@averyyy/pi-server 0.82.1-piclient.2 → 0.82.1-piclient.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/README.md +25 -0
- package/bin/update.js +156 -23
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/pi-server-protocol.d.ts +16 -0
- package/dist/pi-server-protocol.d.ts.map +1 -1
- package/dist/pi-server-protocol.js +30 -1
- package/dist/pi-server-protocol.js.map +1 -1
- package/dist/receive-upload.d.ts +5 -0
- package/dist/receive-upload.d.ts.map +1 -1
- package/dist/receive-upload.js +88 -6
- package/dist/receive-upload.js.map +1 -1
- package/dist/request-chunks.d.ts +33 -0
- package/dist/request-chunks.d.ts.map +1 -1
- package/dist/request-chunks.js +540 -48
- package/dist/request-chunks.js.map +1 -1
- package/dist/server.d.ts +41 -3
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +3088 -366
- package/dist/server.js.map +1 -1
- package/dist/session-persistence.d.ts +39 -1
- package/dist/session-persistence.d.ts.map +1 -1
- package/dist/session-persistence.js +1132 -116
- package/dist/session-persistence.js.map +1 -1
- package/dist/session-store.d.ts +112 -6
- package/dist/session-store.d.ts.map +1 -1
- package/dist/session-store.js +695 -120
- package/dist/session-store.js.map +1 -1
- package/dist/stream-run-persistence.d.ts +163 -0
- package/dist/stream-run-persistence.d.ts.map +1 -0
- package/dist/stream-run-persistence.js +1824 -0
- package/dist/stream-run-persistence.js.map +1 -0
- package/package.json +3 -3
package/dist/server.js
CHANGED
|
@@ -1,19 +1,27 @@
|
|
|
1
|
+
import { Buffer } from "node:buffer";
|
|
2
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
1
3
|
import { createServer } from "node:http";
|
|
2
4
|
import { createRequire } from "node:module";
|
|
5
|
+
import { basename, join } from "node:path";
|
|
3
6
|
import { compact, DEFAULT_COMPACTION_SETTINGS, prepareCompaction, } from "@earendil-works/pi-agent-core";
|
|
4
7
|
import { createModels, createProvider, } from "@earendil-works/pi-ai";
|
|
5
|
-
import { streamSimple } from "@earendil-works/pi-ai/compat";
|
|
8
|
+
import { getCompatProviderExecutionRoute, streamSimple } from "@earendil-works/pi-ai/compat";
|
|
9
|
+
import { hashRemoteProviderExecution } from "@earendil-works/pi-ai/provider-execution-node";
|
|
6
10
|
import { loadConfig } from "./config.js";
|
|
7
11
|
import { encodeErrorEvent, encodeProxyEvent } from "./event-encoding.js";
|
|
12
|
+
import { appendPiServerTreeHash, canonicalJsonStringify, hashPiServerTreeEntry, matchesPiServerStreamBase, PI_SERVER_EMPTY_TREE_HASH, } from "./pi-server-protocol.js";
|
|
8
13
|
import { ReceiveUploadError, receiveUpload } from "./receive-upload.js";
|
|
9
14
|
import { CHUNK_ENDPOINT, receiveRequestChunk } from "./request-chunks.js";
|
|
10
|
-
import { deletePersistedSession, loadPersistedSessions, savePersistedSession } from "./session-persistence.js";
|
|
11
|
-
import {
|
|
15
|
+
import { deletePersistedSession, loadPersistedSessions, preflightPersistedSession, SessionPersistenceCapacityError, savePersistedSession, } from "./session-persistence.js";
|
|
16
|
+
import { applyPiServerCompactionEntry, applySessionMutation, clearAllSessions, configureSessionCapacityLimits, DEFAULT_SESSION_CAPACITY_LIMITS, deleteSession as deleteSessionFromStore, dropLastAssistantError, exportSessionState, getOrCreateSession, getSession, getSessionBranch, listSessions, preflightSessionCapacityMutation, resetSessionCapacityLimits, restoreSessionState, SessionCapacityError, switchSessionLeaf, } from "./session-store.js";
|
|
17
|
+
import { StreamRunConflictError, StreamRunCorruptionError, StreamRunNotFoundError, StreamRunPersistence, StreamRunPersistenceTimeoutError, StreamRunQuotaError, StreamRunStateError, StreamRunStoreLockedError, } from "./stream-run-persistence.js";
|
|
12
18
|
export { loadConfig } from "./config.js";
|
|
13
19
|
const packageMetadata = createRequire(import.meta.url)("../package.json");
|
|
14
20
|
const PI_SERVER_VERSION = packageMetadata.version;
|
|
21
|
+
const PI_SERVER_PROTOCOL_VERSION = 2;
|
|
15
22
|
function createRequestModels(model, options) {
|
|
16
23
|
const models = createModels();
|
|
24
|
+
const streamWithRequestOptions = (requestModel, context, streamOptions) => streamSimple(requestModel, context, { ...options, ...streamOptions });
|
|
17
25
|
models.setProvider(createProvider({
|
|
18
26
|
id: model.provider,
|
|
19
27
|
name: model.provider,
|
|
@@ -21,104 +29,1444 @@ function createRequestModels(model, options) {
|
|
|
21
29
|
auth: {
|
|
22
30
|
apiKey: {
|
|
23
31
|
name: "pi-server request auth",
|
|
24
|
-
resolve: async () => ({
|
|
32
|
+
resolve: async () => ({
|
|
33
|
+
auth: { apiKey: options.apiKey, headers: options.headers },
|
|
34
|
+
env: options.env,
|
|
35
|
+
}),
|
|
25
36
|
},
|
|
26
37
|
},
|
|
27
38
|
api: {
|
|
28
|
-
stream:
|
|
29
|
-
streamSimple:
|
|
39
|
+
stream: streamWithRequestOptions,
|
|
40
|
+
streamSimple: streamWithRequestOptions,
|
|
30
41
|
},
|
|
31
42
|
}));
|
|
32
43
|
return models;
|
|
33
44
|
}
|
|
45
|
+
const jsonResponseCaptures = new WeakMap();
|
|
46
|
+
const serverInitializations = new WeakMap();
|
|
47
|
+
let sessionCapacityMutationQueue = Promise.resolve();
|
|
48
|
+
let piServerOwner;
|
|
49
|
+
let activeServerPersistenceRuntime;
|
|
34
50
|
const STREAM_HEARTBEAT = ": keep-alive\n\n";
|
|
35
51
|
const JSON_HEARTBEAT = " \n";
|
|
36
52
|
const STREAM_HEARTBEAT_INTERVAL_MS = 25_000;
|
|
37
|
-
const
|
|
38
|
-
|
|
53
|
+
const STREAM_RUN_CLEANUP_INTERVAL_MS = 60_000;
|
|
54
|
+
const STREAM_RUN_MAX_RECORDS = 256;
|
|
55
|
+
const STREAM_RUN_MAX_SUBSCRIBERS = 8;
|
|
56
|
+
const STREAM_RUN_ERROR_LOG_CHARS = 1000;
|
|
57
|
+
const STREAM_RUN_BATCH_MAX_EVENTS = 64;
|
|
58
|
+
const STREAM_RUN_BATCH_MAX_BYTES = 64 * 1024;
|
|
59
|
+
const STREAM_RUN_BATCH_DELAY_MS = 12;
|
|
60
|
+
const STREAM_RUN_RESTART_ERROR_MESSAGE = "restart-unknown";
|
|
61
|
+
const STREAM_RUN_MAX_FRAME_BYTES = 256 * 1024 * 1024;
|
|
62
|
+
const STREAM_RUN_MAX_BYTES = 8 * 1024 * 1024 * 1024;
|
|
63
|
+
const STREAM_RUN_MAX_TOTAL_BYTES = 8 * 1024 * 1024 * 1024;
|
|
64
|
+
const STREAM_RUN_TERMINAL_RESERVE_BYTES = 64 * 1024 * 1024;
|
|
65
|
+
export const DEFAULT_STREAM_DRAIN_IDLE_TIMEOUT_MS = 90_000;
|
|
66
|
+
const CHUNK_ACK_HEADER = "X-Pi-Chunk-Ack";
|
|
67
|
+
const COMPACT_RUN_MAX_RECORDS = 32;
|
|
68
|
+
const COMPACT_RUN_MAX_FRAME_BYTES = 64 * 1024 * 1024;
|
|
69
|
+
const COMPACT_RUN_MAX_BYTES = 128 * 1024 * 1024;
|
|
70
|
+
const COMPACT_RUN_MAX_TOTAL_BYTES = 1024 * 1024 * 1024;
|
|
71
|
+
const COMPACT_RUN_TERMINAL_RESERVE_BYTES = 64 * 1024 * 1024;
|
|
72
|
+
const COMPACT_RUN_MAX_SUBSCRIBERS = 8;
|
|
73
|
+
const REQUEST_BODY_MAX_BYTES = 2 * 1024 * 1024;
|
|
74
|
+
const REQUEST_BODY_NO_PROGRESS_TIMEOUT_MS = 90_000;
|
|
75
|
+
const HEADERS_TIMEOUT_MS = 15_000;
|
|
76
|
+
const CHUNK_TARGET_RESPONSE_TTL_MS = 60_000;
|
|
77
|
+
const CHUNK_TARGET_RESPONSE_MAX_RECORDS = 128;
|
|
78
|
+
const CHUNK_TARGET_RESPONSE_MAX_BYTES = 16 * 1024 * 1024;
|
|
79
|
+
const FATAL_SHUTDOWN_GRACE_MS = 10_000;
|
|
80
|
+
class RequestBodyTooLargeError extends Error {
|
|
81
|
+
statusCode = 413;
|
|
82
|
+
}
|
|
83
|
+
class RequestBodyIncompleteError extends Error {
|
|
84
|
+
statusCode = 400;
|
|
85
|
+
}
|
|
86
|
+
class RequestBodyNoProgressError extends Error {
|
|
87
|
+
statusCode = 408;
|
|
88
|
+
}
|
|
89
|
+
class ServerPersistenceUnavailableError extends Error {
|
|
90
|
+
statusCode = 503;
|
|
91
|
+
constructor() {
|
|
92
|
+
super("pi-server durable persistence is unavailable");
|
|
93
|
+
this.name = "ServerPersistenceUnavailableError";
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
function positiveServerOption(value, environmentVariable, fallback, label) {
|
|
97
|
+
const environmentValue = process.env[environmentVariable];
|
|
98
|
+
const resolved = value ?? (environmentValue === undefined ? fallback : Number(environmentValue));
|
|
99
|
+
if (!Number.isSafeInteger(resolved) || resolved <= 0) {
|
|
100
|
+
throw new Error(`${label} (${environmentVariable}) must be a positive safe integer`);
|
|
101
|
+
}
|
|
102
|
+
return resolved;
|
|
103
|
+
}
|
|
104
|
+
function acquirePiServerOwnership() {
|
|
105
|
+
if (piServerOwner) {
|
|
106
|
+
throw new Error("pi-server already owns the process-wide session store; close the existing server before creating another");
|
|
107
|
+
}
|
|
108
|
+
const owner = Symbol("pi-server-owner");
|
|
109
|
+
piServerOwner = owner;
|
|
110
|
+
clearAllSessions();
|
|
111
|
+
resetSessionCapacityLimits();
|
|
112
|
+
sessionCapacityMutationQueue = Promise.resolve();
|
|
113
|
+
return owner;
|
|
114
|
+
}
|
|
115
|
+
function releasePiServerOwnership(owner) {
|
|
116
|
+
if (piServerOwner !== owner)
|
|
117
|
+
return;
|
|
118
|
+
clearAllSessions();
|
|
119
|
+
resetSessionCapacityLimits();
|
|
120
|
+
sessionCapacityMutationQueue = Promise.resolve();
|
|
121
|
+
activeServerPersistenceRuntime = undefined;
|
|
122
|
+
piServerOwner = undefined;
|
|
123
|
+
}
|
|
124
|
+
function assertServerPersistenceHealthy() {
|
|
125
|
+
if (activeServerPersistenceRuntime?.fatalError) {
|
|
126
|
+
throw new ServerPersistenceUnavailableError();
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
export function readBody(req, maxBytes = REQUEST_BODY_MAX_BYTES, noProgressTimeoutMs = REQUEST_BODY_NO_PROGRESS_TIMEOUT_MS) {
|
|
130
|
+
if (!Number.isSafeInteger(noProgressTimeoutMs) || noProgressTimeoutMs <= 0) {
|
|
131
|
+
throw new Error("request body no-progress timeout must be a positive safe integer");
|
|
132
|
+
}
|
|
39
133
|
return new Promise((resolve, reject) => {
|
|
134
|
+
let settled = false;
|
|
135
|
+
let noProgressTimer;
|
|
40
136
|
const chunks = [];
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
137
|
+
let receivedBytes = 0;
|
|
138
|
+
const cleanup = () => {
|
|
139
|
+
req.off("data", onData);
|
|
140
|
+
req.off("end", onEnd);
|
|
141
|
+
req.off("error", onError);
|
|
142
|
+
req.off("aborted", onAborted);
|
|
143
|
+
req.off("close", onClose);
|
|
144
|
+
if (noProgressTimer)
|
|
145
|
+
clearTimeout(noProgressTimer);
|
|
146
|
+
};
|
|
147
|
+
const finish = (error) => {
|
|
148
|
+
if (settled)
|
|
149
|
+
return;
|
|
150
|
+
settled = true;
|
|
151
|
+
cleanup();
|
|
152
|
+
if (error) {
|
|
153
|
+
chunks.length = 0;
|
|
154
|
+
reject(error);
|
|
155
|
+
return;
|
|
156
|
+
}
|
|
157
|
+
resolve(Buffer.concat(chunks).toString("utf-8"));
|
|
158
|
+
};
|
|
159
|
+
const armNoProgressTimer = () => {
|
|
160
|
+
if (noProgressTimer)
|
|
161
|
+
clearTimeout(noProgressTimer);
|
|
162
|
+
noProgressTimer = setTimeout(() => {
|
|
163
|
+
finish(new RequestBodyNoProgressError(`Request body made no progress for ${noProgressTimeoutMs}ms`));
|
|
164
|
+
req.resume();
|
|
165
|
+
}, noProgressTimeoutMs);
|
|
166
|
+
noProgressTimer.unref();
|
|
167
|
+
};
|
|
168
|
+
const onData = (chunk) => {
|
|
169
|
+
if (settled)
|
|
170
|
+
return;
|
|
171
|
+
if (chunk.byteLength === 0)
|
|
172
|
+
return;
|
|
173
|
+
armNoProgressTimer();
|
|
174
|
+
receivedBytes += chunk.byteLength;
|
|
175
|
+
if (receivedBytes > maxBytes) {
|
|
176
|
+
finish(new RequestBodyTooLargeError(`Request body exceeds ${maxBytes} bytes`));
|
|
177
|
+
req.resume();
|
|
178
|
+
return;
|
|
179
|
+
}
|
|
180
|
+
chunks.push(chunk);
|
|
181
|
+
};
|
|
182
|
+
const onEnd = () => finish();
|
|
183
|
+
const onError = (error) => finish(error);
|
|
184
|
+
const onAborted = () => finish(new RequestBodyIncompleteError("Request body was aborted before completion"));
|
|
185
|
+
const onClose = () => finish(new RequestBodyIncompleteError("Request closed before body completion"));
|
|
186
|
+
req.on("data", onData);
|
|
187
|
+
req.once("end", onEnd);
|
|
188
|
+
req.once("error", onError);
|
|
189
|
+
req.once("aborted", onAborted);
|
|
190
|
+
req.once("close", onClose);
|
|
191
|
+
armNoProgressTimer();
|
|
192
|
+
if (req.aborted) {
|
|
193
|
+
onAborted();
|
|
194
|
+
return;
|
|
195
|
+
}
|
|
196
|
+
if (req.destroyed && !req.complete) {
|
|
197
|
+
onClose();
|
|
198
|
+
return;
|
|
199
|
+
}
|
|
200
|
+
const contentLength = Number(req.headers["content-length"]);
|
|
201
|
+
if (Number.isFinite(contentLength) && contentLength > maxBytes) {
|
|
202
|
+
finish(new RequestBodyTooLargeError(`Request body exceeds ${maxBytes} bytes`));
|
|
203
|
+
req.resume();
|
|
204
|
+
}
|
|
205
|
+
});
|
|
206
|
+
}
|
|
207
|
+
function requestErrorStatus(error, defaultStatus) {
|
|
208
|
+
if (error instanceof RequestBodyTooLargeError)
|
|
209
|
+
return error.statusCode;
|
|
210
|
+
if (error instanceof RequestBodyIncompleteError)
|
|
211
|
+
return error.statusCode;
|
|
212
|
+
if (error instanceof RequestBodyNoProgressError)
|
|
213
|
+
return error.statusCode;
|
|
214
|
+
if (error instanceof ServerPersistenceUnavailableError)
|
|
215
|
+
return error.statusCode;
|
|
216
|
+
if (error instanceof SessionCapacityError || error instanceof SessionPersistenceCapacityError)
|
|
217
|
+
return 507;
|
|
218
|
+
if (error instanceof StreamRunNotFoundError)
|
|
219
|
+
return 404;
|
|
220
|
+
if (error instanceof StreamRunConflictError || error instanceof StreamRunStateError)
|
|
221
|
+
return 409;
|
|
222
|
+
if (error instanceof StreamRunQuotaError || error instanceof StreamRunStoreLockedError)
|
|
223
|
+
return 503;
|
|
224
|
+
return defaultStatus;
|
|
225
|
+
}
|
|
226
|
+
function requestErrorBody(error) {
|
|
227
|
+
if (error instanceof SessionPersistenceCapacityError) {
|
|
228
|
+
return {
|
|
229
|
+
error: `Persisted session capacity exceeded: resource=${error.resource}, session=${error.sessionId}, requested=${error.requested}, limit=${error.limit}`,
|
|
230
|
+
code: error.code,
|
|
231
|
+
resource: error.resource,
|
|
232
|
+
sessionId: error.sessionId,
|
|
233
|
+
current: error.current,
|
|
234
|
+
requested: error.requested,
|
|
235
|
+
limit: error.limit,
|
|
236
|
+
retryable: error.retryable,
|
|
237
|
+
artifact: basename(error.path),
|
|
238
|
+
};
|
|
239
|
+
}
|
|
240
|
+
if (error instanceof SessionCapacityError) {
|
|
241
|
+
return {
|
|
242
|
+
error: error.message,
|
|
243
|
+
code: error.code,
|
|
244
|
+
resource: error.resource,
|
|
245
|
+
sessionId: error.sessionId,
|
|
246
|
+
current: error.current,
|
|
247
|
+
requested: error.requested,
|
|
248
|
+
limit: error.limit,
|
|
249
|
+
retryable: error.retryable,
|
|
250
|
+
};
|
|
251
|
+
}
|
|
252
|
+
return { error: error instanceof Error ? error.message : String(error) };
|
|
253
|
+
}
|
|
254
|
+
function sendJson(res, status, body) {
|
|
255
|
+
const data = JSON.stringify(body);
|
|
256
|
+
const capture = jsonResponseCaptures.get(res);
|
|
257
|
+
if (capture) {
|
|
258
|
+
jsonResponseCaptures.delete(res);
|
|
259
|
+
capture({ status, bodyJson: data });
|
|
260
|
+
}
|
|
261
|
+
res.writeHead(status, { "Content-Type": "application/json", "Content-Length": Buffer.byteLength(data) });
|
|
262
|
+
res.end(data);
|
|
263
|
+
}
|
|
264
|
+
function sendCapturedJson(res, response) {
|
|
265
|
+
const data = response.bodyJson;
|
|
266
|
+
res.writeHead(response.status, { "Content-Type": "application/json", "Content-Length": Buffer.byteLength(data) });
|
|
267
|
+
res.end(data);
|
|
268
|
+
}
|
|
269
|
+
function logRequestError(req, error) {
|
|
270
|
+
if (error instanceof SessionCapacityError || error instanceof SessionPersistenceCapacityError)
|
|
271
|
+
return;
|
|
272
|
+
if (error instanceof ServerPersistenceUnavailableError)
|
|
273
|
+
return;
|
|
274
|
+
const message = error instanceof Error ? error.stack || error.message : String(error);
|
|
275
|
+
console.error(`${req.method ?? "UNKNOWN"} ${req.url ?? "/"} failed: ${message}`);
|
|
276
|
+
}
|
|
277
|
+
function authenticate(config, req) {
|
|
278
|
+
if (!config.authToken)
|
|
279
|
+
return true;
|
|
280
|
+
const header = req.headers.authorization;
|
|
281
|
+
if (!header)
|
|
282
|
+
return false;
|
|
283
|
+
const token = header.startsWith("Bearer ") ? header.slice(7) : header;
|
|
284
|
+
return token === config.authToken;
|
|
285
|
+
}
|
|
286
|
+
function sessionResponseBody(session) {
|
|
287
|
+
return {
|
|
288
|
+
protocolVersion: PI_SERVER_PROTOCOL_VERSION,
|
|
289
|
+
sessionId: session.sessionId,
|
|
290
|
+
staticContextHash: session.staticContextHash,
|
|
291
|
+
treeHash: session.treeHash,
|
|
292
|
+
messageCount: session.messages.length,
|
|
293
|
+
entryCount: session.entries.length,
|
|
294
|
+
leafId: session.leafId,
|
|
295
|
+
revision: session.revision,
|
|
296
|
+
};
|
|
297
|
+
}
|
|
298
|
+
function sessionHistoryV2Summary(session) {
|
|
299
|
+
return {
|
|
300
|
+
protocolVersion: PI_SERVER_PROTOCOL_VERSION,
|
|
301
|
+
sessionId: session.sessionId,
|
|
302
|
+
staticContextHash: session.staticContextHash,
|
|
303
|
+
treeHash: session.treeHash,
|
|
304
|
+
messageCount: session.messages.length,
|
|
305
|
+
entryCount: session.entries.length,
|
|
306
|
+
leafId: session.leafId,
|
|
307
|
+
revision: session.revision,
|
|
308
|
+
};
|
|
309
|
+
}
|
|
310
|
+
function currentServerPersistenceRuntime() {
|
|
311
|
+
const runtime = activeServerPersistenceRuntime;
|
|
312
|
+
if (!runtime) {
|
|
313
|
+
throw new ServerPersistenceUnavailableError();
|
|
314
|
+
}
|
|
315
|
+
return runtime;
|
|
316
|
+
}
|
|
317
|
+
function persistSession(config, session) {
|
|
318
|
+
const runtime = currentServerPersistenceRuntime();
|
|
319
|
+
try {
|
|
320
|
+
savePersistedSession(config.sessionStoreDir, session, {
|
|
321
|
+
faultInjector: runtime.sessionFaultInjector,
|
|
322
|
+
});
|
|
323
|
+
}
|
|
324
|
+
catch (error) {
|
|
325
|
+
if (!(error instanceof SessionPersistenceCapacityError)) {
|
|
326
|
+
failStopServerPersistence(runtime, error);
|
|
327
|
+
}
|
|
328
|
+
throw error;
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
function mutateAndPersistSession(config, sessionId, mutate) {
|
|
332
|
+
const runtime = currentServerPersistenceRuntime();
|
|
333
|
+
const previous = getSession(sessionId);
|
|
334
|
+
const previousState = previous ? exportSessionState(previous) : undefined;
|
|
335
|
+
const session = mutate();
|
|
336
|
+
try {
|
|
337
|
+
persistSession(config, session);
|
|
338
|
+
return session;
|
|
339
|
+
}
|
|
340
|
+
catch (error) {
|
|
341
|
+
if (!(error instanceof SessionPersistenceCapacityError))
|
|
342
|
+
throw error;
|
|
343
|
+
try {
|
|
344
|
+
if (previousState) {
|
|
345
|
+
restoreSessionState(previousState);
|
|
346
|
+
}
|
|
347
|
+
else {
|
|
348
|
+
deleteSessionFromStore(sessionId);
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
catch (rollbackError) {
|
|
352
|
+
const failure = new AggregateError([error, rollbackError], `Failed to restore session ${sessionId} after persistence capacity rejection`);
|
|
353
|
+
failStopServerPersistence(runtime, failure);
|
|
354
|
+
throw failure;
|
|
355
|
+
}
|
|
356
|
+
throw error;
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
function runKey(sessionId, runId) {
|
|
360
|
+
return `${sessionId}\0${runId}`;
|
|
361
|
+
}
|
|
362
|
+
function hashStreamRequest(body) {
|
|
363
|
+
const serialized = canonicalJsonStringify({
|
|
364
|
+
sessionId: body.sessionId,
|
|
365
|
+
runMode: body.runMode,
|
|
366
|
+
baseStaticContextHash: body.baseStaticContextHash,
|
|
367
|
+
baseRevision: body.baseRevision,
|
|
368
|
+
baseTreeHash: body.baseTreeHash,
|
|
369
|
+
baseEntryCount: body.baseEntryCount,
|
|
370
|
+
baseLeafId: body.baseLeafId,
|
|
371
|
+
model: body.model,
|
|
372
|
+
options: body.options,
|
|
373
|
+
providerExecutionFingerprint: body.providerExecutionFingerprint,
|
|
374
|
+
staticContext: body.staticContext,
|
|
375
|
+
ephemeralMessages: body.ephemeralMessages,
|
|
376
|
+
contextOverlay: body.contextOverlay,
|
|
377
|
+
});
|
|
378
|
+
if (serialized === undefined) {
|
|
379
|
+
throw new Error("Failed to serialize pi-server stream request");
|
|
380
|
+
}
|
|
381
|
+
return createHash("sha256").update(serialized).digest("hex");
|
|
382
|
+
}
|
|
383
|
+
function sessionIdentity(session) {
|
|
384
|
+
return {
|
|
385
|
+
staticContextHash: session.staticContextHash,
|
|
386
|
+
treeHash: session.treeHash,
|
|
387
|
+
entryCount: session.entries.length,
|
|
388
|
+
leafId: session.leafId,
|
|
389
|
+
revision: session.revision,
|
|
390
|
+
};
|
|
391
|
+
}
|
|
392
|
+
function streamBaseIdentity(session) {
|
|
393
|
+
return {
|
|
394
|
+
baseStaticContextHash: session.staticContextHash,
|
|
395
|
+
baseTreeHash: session.treeHash,
|
|
396
|
+
baseEntryCount: session.entries.length,
|
|
397
|
+
baseLeafId: session.leafId,
|
|
398
|
+
baseRevision: session.revision,
|
|
399
|
+
};
|
|
400
|
+
}
|
|
401
|
+
function hashCompactRequest(body) {
|
|
402
|
+
const serialized = canonicalJsonStringify({
|
|
403
|
+
protocolVersion: body.protocolVersion,
|
|
404
|
+
sessionId: body.sessionId,
|
|
405
|
+
providerExecutionFingerprint: body.providerExecutionFingerprint,
|
|
406
|
+
model: body.model,
|
|
407
|
+
options: body.options,
|
|
408
|
+
settings: body.settings,
|
|
409
|
+
preparation: body.preparation,
|
|
410
|
+
extensionCompaction: body.extensionCompaction,
|
|
411
|
+
customInstructions: body.customInstructions,
|
|
412
|
+
baseStaticContextHash: body.baseStaticContextHash,
|
|
413
|
+
baseTreeHash: body.baseTreeHash,
|
|
414
|
+
baseEntryCount: body.baseEntryCount,
|
|
415
|
+
baseLeafId: body.baseLeafId,
|
|
416
|
+
baseRevision: body.baseRevision,
|
|
417
|
+
streamResponse: body.streamResponse,
|
|
418
|
+
retry: body.retry,
|
|
419
|
+
});
|
|
420
|
+
if (serialized === undefined) {
|
|
421
|
+
throw new Error("Failed to serialize pi-server compaction request");
|
|
422
|
+
}
|
|
423
|
+
return createHash("sha256").update(serialized).digest("hex");
|
|
424
|
+
}
|
|
425
|
+
function providerExecutionParityError(model, fingerprint) {
|
|
426
|
+
if (!model)
|
|
427
|
+
return "model is required before provider execution parity can be verified";
|
|
428
|
+
if (typeof fingerprint !== "string" || !/^[a-f0-9]{64}$/u.test(fingerprint)) {
|
|
429
|
+
return (`providerExecutionFingerprint is required for ${model.provider}/${model.id}; ` +
|
|
430
|
+
"upgrade pi-client and pi-server together");
|
|
431
|
+
}
|
|
432
|
+
const route = getCompatProviderExecutionRoute(model);
|
|
433
|
+
if (route.kind !== "builtin_provider" && route.kind !== "builtin_api") {
|
|
434
|
+
return (`pi-server cannot preserve provider execution parity for ${model.provider}/${model.id}: ` +
|
|
435
|
+
`server route is ${route.kind}; upgrade matching runtimes or use plain pi`);
|
|
436
|
+
}
|
|
437
|
+
const expected = hashRemoteProviderExecution(model, route);
|
|
438
|
+
if (fingerprint !== expected) {
|
|
439
|
+
return (`provider execution fingerprint mismatch for ${model.provider}/${model.id}; ` +
|
|
440
|
+
"pi-client/pi-server runtime versions or provider routes differ, so execution was rejected");
|
|
441
|
+
}
|
|
442
|
+
return undefined;
|
|
443
|
+
}
|
|
444
|
+
function createStreamErrorEvent(errorMessage, reason = "error") {
|
|
445
|
+
return {
|
|
446
|
+
type: "error",
|
|
447
|
+
reason,
|
|
448
|
+
errorMessage,
|
|
449
|
+
usage: {
|
|
450
|
+
input: 0,
|
|
451
|
+
output: 0,
|
|
452
|
+
cacheRead: 0,
|
|
453
|
+
cacheWrite: 0,
|
|
454
|
+
totalTokens: 0,
|
|
455
|
+
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
|
456
|
+
},
|
|
457
|
+
};
|
|
458
|
+
}
|
|
459
|
+
function createRestartFailureTerminal(identity) {
|
|
460
|
+
const result = {
|
|
461
|
+
role: "assistant",
|
|
462
|
+
content: [],
|
|
463
|
+
api: "openai-responses",
|
|
464
|
+
provider: "pi-server",
|
|
465
|
+
model: STREAM_RUN_RESTART_ERROR_MESSAGE,
|
|
466
|
+
usage: {
|
|
467
|
+
input: 0,
|
|
468
|
+
output: 0,
|
|
469
|
+
cacheRead: 0,
|
|
470
|
+
cacheWrite: 0,
|
|
471
|
+
totalTokens: 0,
|
|
472
|
+
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
|
473
|
+
},
|
|
474
|
+
stopReason: "error",
|
|
475
|
+
errorMessage: STREAM_RUN_RESTART_ERROR_MESSAGE,
|
|
476
|
+
timestamp: Date.now(),
|
|
477
|
+
diagnostics: [
|
|
478
|
+
{
|
|
479
|
+
type: "pi_server_run",
|
|
480
|
+
timestamp: Date.now(),
|
|
481
|
+
details: {
|
|
482
|
+
sessionId: identity.sessionId,
|
|
483
|
+
runId: identity.runId,
|
|
484
|
+
requestMac: identity.requestMac,
|
|
485
|
+
restartUnknown: true,
|
|
486
|
+
},
|
|
487
|
+
},
|
|
488
|
+
],
|
|
489
|
+
};
|
|
490
|
+
return {
|
|
491
|
+
event: encodeProxyEvent({
|
|
492
|
+
type: "error",
|
|
493
|
+
reason: "error",
|
|
494
|
+
errorMessage: result.errorMessage,
|
|
495
|
+
usage: result.usage,
|
|
496
|
+
diagnostics: result.diagnostics,
|
|
497
|
+
api: result.api,
|
|
498
|
+
provider: result.provider,
|
|
499
|
+
model: result.model,
|
|
500
|
+
timestamp: result.timestamp,
|
|
501
|
+
}),
|
|
502
|
+
result,
|
|
503
|
+
errorMessage: STREAM_RUN_RESTART_ERROR_MESSAGE,
|
|
504
|
+
};
|
|
505
|
+
}
|
|
506
|
+
function clearStreamRunFlushTimer(run) {
|
|
507
|
+
if (!run.flushTimer)
|
|
508
|
+
return;
|
|
509
|
+
clearTimeout(run.flushTimer);
|
|
510
|
+
run.flushTimer = undefined;
|
|
511
|
+
}
|
|
512
|
+
function releaseTerminalStreamRun(runtime, run) {
|
|
513
|
+
if (run.status === "running" || run.subscribers.size > 0)
|
|
514
|
+
return;
|
|
515
|
+
runtime.activeRuns.delete(runKey(run.sessionId, run.storageRunId));
|
|
516
|
+
}
|
|
517
|
+
function detachStreamRunSubscriber(runtime, run, res) {
|
|
518
|
+
const subscriber = run.subscribers.get(res);
|
|
519
|
+
if (!subscriber)
|
|
520
|
+
return;
|
|
521
|
+
if (subscriber.heartbeat) {
|
|
522
|
+
clearInterval(subscriber.heartbeat);
|
|
523
|
+
}
|
|
524
|
+
subscriber.abortController.abort();
|
|
525
|
+
res.off("close", subscriber.onClose);
|
|
526
|
+
run.subscribers.delete(res);
|
|
527
|
+
releaseTerminalStreamRun(runtime, run);
|
|
528
|
+
}
|
|
529
|
+
export function waitForStreamDrain(res, idleTimeoutMs = DEFAULT_STREAM_DRAIN_IDLE_TIMEOUT_MS) {
|
|
530
|
+
if (!Number.isSafeInteger(idleTimeoutMs) || idleTimeoutMs <= 0) {
|
|
531
|
+
throw new Error("stream drain idle timeout must be a positive safe integer");
|
|
532
|
+
}
|
|
533
|
+
return new Promise((resolve) => {
|
|
534
|
+
if (res.destroyed || res.writableEnded) {
|
|
535
|
+
resolve(false);
|
|
536
|
+
return;
|
|
537
|
+
}
|
|
538
|
+
let settled = false;
|
|
539
|
+
let timeout;
|
|
540
|
+
const cleanup = () => {
|
|
541
|
+
res.off("drain", onDrain);
|
|
542
|
+
res.off("close", onClose);
|
|
543
|
+
if (timeout)
|
|
544
|
+
clearTimeout(timeout);
|
|
545
|
+
};
|
|
546
|
+
const finish = (drained) => {
|
|
547
|
+
if (settled)
|
|
548
|
+
return;
|
|
549
|
+
settled = true;
|
|
550
|
+
cleanup();
|
|
551
|
+
resolve(drained);
|
|
552
|
+
};
|
|
553
|
+
const onDrain = () => finish(true);
|
|
554
|
+
const onClose = () => finish(false);
|
|
555
|
+
res.once("drain", onDrain);
|
|
556
|
+
res.once("close", onClose);
|
|
557
|
+
timeout = setTimeout(() => finish(false), idleTimeoutMs);
|
|
558
|
+
timeout.unref();
|
|
559
|
+
if (res.destroyed || res.writableEnded) {
|
|
560
|
+
onClose();
|
|
561
|
+
}
|
|
562
|
+
});
|
|
563
|
+
}
|
|
564
|
+
async function writeWithStreamDrain(res, data, idleTimeoutMs) {
|
|
565
|
+
if (res.destroyed || res.writableEnded)
|
|
566
|
+
return false;
|
|
567
|
+
try {
|
|
568
|
+
if (res.writableNeedDrain && !(await waitForStreamDrain(res, idleTimeoutMs)))
|
|
569
|
+
return false;
|
|
570
|
+
if (res.destroyed || res.writableEnded)
|
|
571
|
+
return false;
|
|
572
|
+
if (res.write(data))
|
|
573
|
+
return true;
|
|
574
|
+
return waitForStreamDrain(res, idleTimeoutMs);
|
|
575
|
+
}
|
|
576
|
+
catch {
|
|
577
|
+
return false;
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
function messageFromDurableState(state) {
|
|
581
|
+
const result = state.terminal?.result;
|
|
582
|
+
if (typeof result !== "object" || result === null || !("role" in result) || result.role !== "assistant") {
|
|
583
|
+
return undefined;
|
|
584
|
+
}
|
|
585
|
+
return result;
|
|
586
|
+
}
|
|
587
|
+
function applyDurableState(run, state) {
|
|
588
|
+
run.status = state.status;
|
|
589
|
+
run.nextSeq = state.nextSeq;
|
|
590
|
+
run.message = messageFromDurableState(state);
|
|
591
|
+
run.errorMessage = state.terminal?.errorMessage;
|
|
592
|
+
}
|
|
593
|
+
async function pumpStreamRunSubscriber(runtime, run, res, subscriber) {
|
|
594
|
+
try {
|
|
595
|
+
do {
|
|
596
|
+
subscriber.flushAgain = false;
|
|
597
|
+
for await (const frame of runtime.store.iterateEvents(run.sessionId, run.storageRunId, subscriber.cursor, subscriber.abortController.signal)) {
|
|
598
|
+
if (run.subscribers.get(res) !== subscriber || res.destroyed || res.writableEnded)
|
|
599
|
+
return;
|
|
600
|
+
subscriber.cursor = frame.seq + 1;
|
|
601
|
+
if (!res.write(frame.event) && !(await waitForStreamDrain(res, runtime.drainIdleTimeoutMs))) {
|
|
602
|
+
detachStreamRunSubscriber(runtime, run, res);
|
|
603
|
+
if (!res.destroyed)
|
|
604
|
+
res.destroy();
|
|
605
|
+
return;
|
|
606
|
+
}
|
|
607
|
+
}
|
|
608
|
+
if (subscriber.abortController.signal.aborted ||
|
|
609
|
+
run.subscribers.get(res) !== subscriber ||
|
|
610
|
+
res.destroyed ||
|
|
611
|
+
res.writableEnded) {
|
|
612
|
+
return;
|
|
613
|
+
}
|
|
614
|
+
const state = await runtime.store.get(run.sessionId, run.storageRunId);
|
|
615
|
+
if (!state) {
|
|
616
|
+
throw new StreamRunNotFoundError(run.sessionId, run.storageRunId);
|
|
617
|
+
}
|
|
618
|
+
applyDurableState(run, state);
|
|
619
|
+
if (run.status !== "running" && subscriber.cursor >= run.nextSeq) {
|
|
620
|
+
detachStreamRunSubscriber(runtime, run, res);
|
|
621
|
+
if (!res.destroyed && !res.writableEnded)
|
|
622
|
+
res.end();
|
|
623
|
+
return;
|
|
624
|
+
}
|
|
625
|
+
if (subscriber.cursor < run.nextSeq)
|
|
626
|
+
subscriber.flushAgain = true;
|
|
627
|
+
} while (subscriber.flushAgain);
|
|
628
|
+
}
|
|
629
|
+
catch {
|
|
630
|
+
detachStreamRunSubscriber(runtime, run, res);
|
|
631
|
+
if (!res.destroyed)
|
|
632
|
+
res.destroy();
|
|
633
|
+
}
|
|
634
|
+
}
|
|
635
|
+
function addSessionSubscriberPump(runtime, sessionId, pump) {
|
|
636
|
+
let pumps = runtime.subscriberPumps.get(sessionId);
|
|
637
|
+
if (!pumps) {
|
|
638
|
+
pumps = new Set();
|
|
639
|
+
runtime.subscriberPumps.set(sessionId, pumps);
|
|
640
|
+
}
|
|
641
|
+
pumps.add(pump);
|
|
642
|
+
}
|
|
643
|
+
function removeSessionSubscriberPump(runtime, sessionId, pump) {
|
|
644
|
+
const pumps = runtime.subscriberPumps.get(sessionId);
|
|
645
|
+
if (!pumps)
|
|
646
|
+
return;
|
|
647
|
+
pumps.delete(pump);
|
|
648
|
+
if (pumps.size === 0)
|
|
649
|
+
runtime.subscriberPumps.delete(sessionId);
|
|
650
|
+
}
|
|
651
|
+
function startStreamRunSubscriberPump(runtime, run, res) {
|
|
652
|
+
const subscriber = run.subscribers.get(res);
|
|
653
|
+
if (!subscriber || res.destroyed || res.writableEnded)
|
|
654
|
+
return;
|
|
655
|
+
if (subscriber.heartbeatWrite) {
|
|
656
|
+
subscriber.flushAgain = true;
|
|
657
|
+
return;
|
|
658
|
+
}
|
|
659
|
+
if (subscriber.flushing) {
|
|
660
|
+
subscriber.flushAgain = true;
|
|
661
|
+
return;
|
|
662
|
+
}
|
|
663
|
+
subscriber.flushing = true;
|
|
664
|
+
const pump = pumpStreamRunSubscriber(runtime, run, res, subscriber).finally(() => {
|
|
665
|
+
subscriber.flushing = false;
|
|
666
|
+
removeSessionSubscriberPump(runtime, run.sessionId, pump);
|
|
667
|
+
if (subscriber.flushAgain && run.subscribers.get(res) === subscriber) {
|
|
668
|
+
startStreamRunSubscriberPump(runtime, run, res);
|
|
669
|
+
}
|
|
670
|
+
});
|
|
671
|
+
subscriber.pump = pump;
|
|
672
|
+
addSessionSubscriberPump(runtime, run.sessionId, pump);
|
|
673
|
+
}
|
|
674
|
+
function notifyStreamRunSubscribers(runtime, run) {
|
|
675
|
+
for (const res of [...run.subscribers.keys()]) {
|
|
676
|
+
startStreamRunSubscriberPump(runtime, run, res);
|
|
677
|
+
}
|
|
678
|
+
releaseTerminalStreamRun(runtime, run);
|
|
679
|
+
}
|
|
680
|
+
function beginStreamResponse(res) {
|
|
681
|
+
try {
|
|
682
|
+
res.writeHead(200, {
|
|
683
|
+
"Content-Type": "text/event-stream",
|
|
684
|
+
"Cache-Control": "no-cache",
|
|
685
|
+
Connection: "keep-alive",
|
|
686
|
+
});
|
|
687
|
+
res.flushHeaders();
|
|
688
|
+
return true;
|
|
689
|
+
}
|
|
690
|
+
catch {
|
|
691
|
+
res.destroy();
|
|
692
|
+
return false;
|
|
693
|
+
}
|
|
694
|
+
}
|
|
695
|
+
function startStreamRunSubscriberHeartbeat(runtime, run, res, subscriber) {
|
|
696
|
+
if (subscriber.heartbeatWrite || subscriber.flushing || run.subscribers.get(res) !== subscriber)
|
|
697
|
+
return;
|
|
698
|
+
const heartbeatWrite = writeWithStreamDrain(res, STREAM_HEARTBEAT, runtime.drainIdleTimeoutMs)
|
|
699
|
+
.then((succeeded) => {
|
|
700
|
+
if (!succeeded && run.subscribers.get(res) === subscriber) {
|
|
701
|
+
detachStreamRunSubscriber(runtime, run, res);
|
|
702
|
+
if (!res.destroyed)
|
|
703
|
+
res.destroy();
|
|
704
|
+
}
|
|
705
|
+
})
|
|
706
|
+
.catch(() => {
|
|
707
|
+
if (run.subscribers.get(res) === subscriber) {
|
|
708
|
+
detachStreamRunSubscriber(runtime, run, res);
|
|
709
|
+
if (!res.destroyed)
|
|
710
|
+
res.destroy();
|
|
711
|
+
}
|
|
712
|
+
})
|
|
713
|
+
.finally(() => {
|
|
714
|
+
if (subscriber.heartbeatWrite === heartbeatWrite)
|
|
715
|
+
subscriber.heartbeatWrite = undefined;
|
|
716
|
+
if (run.subscribers.get(res) === subscriber && !res.destroyed && !res.writableEnded) {
|
|
717
|
+
startStreamRunSubscriberPump(runtime, run, res);
|
|
718
|
+
}
|
|
719
|
+
});
|
|
720
|
+
subscriber.heartbeatWrite = heartbeatWrite;
|
|
721
|
+
}
|
|
722
|
+
function subscribeToStreamRun(runtime, run, res, cursor) {
|
|
723
|
+
const onClose = () => detachStreamRunSubscriber(runtime, run, res);
|
|
724
|
+
const subscriber = {
|
|
725
|
+
onClose,
|
|
726
|
+
abortController: new AbortController(),
|
|
727
|
+
cursor,
|
|
728
|
+
flushing: false,
|
|
729
|
+
flushAgain: false,
|
|
730
|
+
};
|
|
731
|
+
if (run.status === "running") {
|
|
732
|
+
subscriber.heartbeat = setInterval(() => {
|
|
733
|
+
startStreamRunSubscriberHeartbeat(runtime, run, res, subscriber);
|
|
734
|
+
}, STREAM_HEARTBEAT_INTERVAL_MS);
|
|
735
|
+
subscriber.heartbeat.unref();
|
|
736
|
+
}
|
|
737
|
+
run.subscribers.set(res, subscriber);
|
|
738
|
+
res.once("close", onClose);
|
|
739
|
+
startStreamRunSubscriberHeartbeat(runtime, run, res, subscriber);
|
|
740
|
+
}
|
|
741
|
+
function enqueueStreamRunSetup(runtime, operation) {
|
|
742
|
+
const guardedOperation = () => {
|
|
743
|
+
assertServerPersistenceHealthy();
|
|
744
|
+
return operation();
|
|
745
|
+
};
|
|
746
|
+
const result = runtime.setupQueue.then(guardedOperation, guardedOperation);
|
|
747
|
+
runtime.setupQueue = result.then(() => undefined, () => undefined);
|
|
748
|
+
return result;
|
|
749
|
+
}
|
|
750
|
+
function replayDurableStreamRun(runtime, state, res, cursor) {
|
|
751
|
+
if (cursor > state.nextSeq) {
|
|
752
|
+
sendJson(res, 409, { error: `eventCursor ${cursor} exceeds durable nextSeq ${state.nextSeq}` });
|
|
753
|
+
return Promise.resolve();
|
|
754
|
+
}
|
|
755
|
+
if (!beginStreamResponse(res))
|
|
756
|
+
return Promise.resolve();
|
|
757
|
+
const replay = {
|
|
758
|
+
runId: state.runId,
|
|
759
|
+
response: res,
|
|
760
|
+
abortController: new AbortController(),
|
|
761
|
+
};
|
|
762
|
+
let readers = runtime.replayReaders.get(state.sessionId);
|
|
763
|
+
if (!readers) {
|
|
764
|
+
readers = new Set();
|
|
765
|
+
runtime.replayReaders.set(state.sessionId, readers);
|
|
766
|
+
}
|
|
767
|
+
readers.add(replay);
|
|
768
|
+
const onClose = () => replay.abortController.abort();
|
|
769
|
+
res.once("close", onClose);
|
|
770
|
+
const operation = (async () => {
|
|
771
|
+
try {
|
|
772
|
+
if (!(await writeWithStreamDrain(res, STREAM_HEARTBEAT, runtime.drainIdleTimeoutMs))) {
|
|
773
|
+
replay.abortController.abort();
|
|
774
|
+
if (!res.destroyed)
|
|
775
|
+
res.destroy();
|
|
776
|
+
return;
|
|
777
|
+
}
|
|
778
|
+
for await (const frame of runtime.store.iterateEvents(state.sessionId, state.runId, cursor, replay.abortController.signal)) {
|
|
779
|
+
if (res.destroyed || res.writableEnded || replay.abortController.signal.aborted)
|
|
780
|
+
return;
|
|
781
|
+
if (!res.write(frame.event) && !(await waitForStreamDrain(res, runtime.drainIdleTimeoutMs))) {
|
|
782
|
+
replay.abortController.abort();
|
|
783
|
+
if (!res.destroyed)
|
|
784
|
+
res.destroy();
|
|
785
|
+
return;
|
|
786
|
+
}
|
|
787
|
+
}
|
|
788
|
+
if (!res.destroyed && !res.writableEnded)
|
|
789
|
+
res.end();
|
|
790
|
+
}
|
|
791
|
+
catch (error) {
|
|
792
|
+
if (!res.destroyed)
|
|
793
|
+
res.destroy(error instanceof Error ? error : new Error(String(error)));
|
|
794
|
+
}
|
|
795
|
+
finally {
|
|
796
|
+
res.off("close", onClose);
|
|
797
|
+
const currentReaders = runtime.replayReaders.get(state.sessionId);
|
|
798
|
+
currentReaders?.delete(replay);
|
|
799
|
+
if (currentReaders?.size === 0)
|
|
800
|
+
runtime.replayReaders.delete(state.sessionId);
|
|
801
|
+
}
|
|
802
|
+
})();
|
|
803
|
+
replay.promise = operation;
|
|
804
|
+
return operation;
|
|
805
|
+
}
|
|
806
|
+
function createActiveStreamRun(state, runId, model) {
|
|
807
|
+
return {
|
|
808
|
+
sessionId: state.sessionId,
|
|
809
|
+
runId,
|
|
810
|
+
storageRunId: state.runId,
|
|
811
|
+
requestHash: state.requestMac,
|
|
812
|
+
status: state.status,
|
|
813
|
+
nextSeq: state.nextSeq,
|
|
814
|
+
model,
|
|
815
|
+
contentDeltas: new Map(),
|
|
816
|
+
abortController: new AbortController(),
|
|
817
|
+
subscribers: new Map(),
|
|
818
|
+
providerFailureLogged: false,
|
|
819
|
+
persistenceFailureLogged: false,
|
|
820
|
+
pendingEvents: [],
|
|
821
|
+
pendingEventBytes: 0,
|
|
822
|
+
shuttingDown: false,
|
|
823
|
+
};
|
|
824
|
+
}
|
|
825
|
+
async function handleCancelSessionRun(runtime, body, res) {
|
|
826
|
+
if (!body.sessionId || !body.runId) {
|
|
827
|
+
sendJson(res, 400, { error: "sessionId and runId are required" });
|
|
828
|
+
return;
|
|
829
|
+
}
|
|
830
|
+
const sessionId = body.sessionId;
|
|
831
|
+
const runId = body.runId;
|
|
832
|
+
const key = runKey(sessionId, runId);
|
|
833
|
+
const activeRun = runtime.activeRuns.get(key);
|
|
834
|
+
if (activeRun?.status === "running") {
|
|
835
|
+
activeRun.abortController.abort();
|
|
836
|
+
}
|
|
837
|
+
await enqueueStreamRunSetup(runtime, async () => {
|
|
838
|
+
const state = await runtime.store.get(sessionId, runId);
|
|
839
|
+
if (!state) {
|
|
840
|
+
sendJson(res, 200, { canceled: false, status: "missing" });
|
|
841
|
+
return;
|
|
842
|
+
}
|
|
843
|
+
if (state.status !== "running") {
|
|
844
|
+
sendJson(res, 200, { canceled: false, status: state.status });
|
|
845
|
+
return;
|
|
846
|
+
}
|
|
847
|
+
const run = runtime.activeRuns.get(key);
|
|
848
|
+
if (!run) {
|
|
849
|
+
throw new StreamRunStateError("Durable running stream has no active provider execution");
|
|
850
|
+
}
|
|
851
|
+
try {
|
|
852
|
+
await abortStreamRun(runtime, run, "Request aborted by user", "aborted");
|
|
853
|
+
}
|
|
854
|
+
catch (error) {
|
|
855
|
+
await handleStreamRunPersistenceFailure(runtime, run, error);
|
|
856
|
+
if (runtime.fatalPersistenceError)
|
|
857
|
+
throw runtime.fatalPersistenceError;
|
|
858
|
+
if (run.status === "running")
|
|
859
|
+
throw error;
|
|
860
|
+
}
|
|
861
|
+
sendJson(res, 200, { canceled: true, status: run.status });
|
|
862
|
+
});
|
|
863
|
+
}
|
|
864
|
+
function logProviderStreamFailure(run, model, errorMessage) {
|
|
865
|
+
if (run.providerFailureLogged)
|
|
866
|
+
return;
|
|
867
|
+
run.providerFailureLogged = true;
|
|
868
|
+
const safeErrorMessage = errorMessage.length > STREAM_RUN_ERROR_LOG_CHARS
|
|
869
|
+
? `${errorMessage.slice(0, STREAM_RUN_ERROR_LOG_CHARS - 3)}...`
|
|
870
|
+
: errorMessage;
|
|
871
|
+
console.error(JSON.stringify({
|
|
872
|
+
phase: "provider_stream",
|
|
873
|
+
sessionId: run.sessionId,
|
|
874
|
+
runId: run.runId ?? null,
|
|
875
|
+
model: { provider: model.provider, id: model.id, api: model.api },
|
|
876
|
+
error: safeErrorMessage,
|
|
877
|
+
}));
|
|
878
|
+
}
|
|
879
|
+
function logStreamRunPersistenceFailure(run, error) {
|
|
880
|
+
if (run.persistenceFailureLogged)
|
|
881
|
+
return;
|
|
882
|
+
run.persistenceFailureLogged = true;
|
|
883
|
+
const rawMessage = error instanceof Error ? error.message : String(error);
|
|
884
|
+
const safeErrorMessage = rawMessage.length > STREAM_RUN_ERROR_LOG_CHARS
|
|
885
|
+
? `${rawMessage.slice(0, STREAM_RUN_ERROR_LOG_CHARS - 3)}...`
|
|
886
|
+
: rawMessage;
|
|
887
|
+
console.error(JSON.stringify({
|
|
888
|
+
phase: "stream_persistence",
|
|
889
|
+
sessionId: run.sessionId,
|
|
890
|
+
runId: run.runId ?? run.storageRunId,
|
|
891
|
+
error: safeErrorMessage,
|
|
892
|
+
}));
|
|
893
|
+
}
|
|
894
|
+
function withStreamRunDiagnostic(run, message) {
|
|
895
|
+
return {
|
|
896
|
+
...message,
|
|
897
|
+
diagnostics: [
|
|
898
|
+
...(message.diagnostics ?? []).filter((diagnostic) => diagnostic.type !== "pi_server_run"),
|
|
899
|
+
{
|
|
900
|
+
type: "pi_server_run",
|
|
901
|
+
timestamp: Date.now(),
|
|
902
|
+
details: {
|
|
903
|
+
sessionId: run.sessionId,
|
|
904
|
+
runId: run.runId ?? run.storageRunId,
|
|
905
|
+
requestMac: run.requestHash,
|
|
906
|
+
},
|
|
907
|
+
},
|
|
908
|
+
],
|
|
909
|
+
};
|
|
910
|
+
}
|
|
911
|
+
function createSyntheticAssistantMessage(run, errorMessage, reason) {
|
|
912
|
+
return withStreamRunDiagnostic(run, {
|
|
913
|
+
role: "assistant",
|
|
914
|
+
content: [],
|
|
915
|
+
api: run.model.api,
|
|
916
|
+
provider: run.model.provider,
|
|
917
|
+
model: run.model.id,
|
|
918
|
+
usage: {
|
|
919
|
+
input: 0,
|
|
920
|
+
output: 0,
|
|
921
|
+
cacheRead: 0,
|
|
922
|
+
cacheWrite: 0,
|
|
923
|
+
totalTokens: 0,
|
|
924
|
+
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
|
925
|
+
},
|
|
926
|
+
stopReason: reason,
|
|
927
|
+
errorMessage,
|
|
928
|
+
timestamp: Date.now(),
|
|
929
|
+
});
|
|
930
|
+
}
|
|
931
|
+
function terminalProxyEvent(run, message) {
|
|
932
|
+
if (message.stopReason === "error" || message.stopReason === "aborted") {
|
|
933
|
+
const event = toProxyEvent({ type: "error", reason: message.stopReason, error: message }, run);
|
|
934
|
+
if (!event || event.type !== "error") {
|
|
935
|
+
throw new Error("Failed to encode terminal stream error");
|
|
936
|
+
}
|
|
937
|
+
return event;
|
|
938
|
+
}
|
|
939
|
+
const event = toProxyEvent({ type: "done", reason: message.stopReason, message }, run);
|
|
940
|
+
if (!event || event.type !== "done") {
|
|
941
|
+
throw new Error("Failed to encode terminal stream completion");
|
|
942
|
+
}
|
|
943
|
+
return event;
|
|
944
|
+
}
|
|
945
|
+
async function flushPendingStreamRunEvents(runtime, run) {
|
|
946
|
+
clearStreamRunFlushTimer(run);
|
|
947
|
+
if (run.flushPromise) {
|
|
948
|
+
await run.flushPromise;
|
|
949
|
+
if (run.pendingEvents.length > 0)
|
|
950
|
+
await flushPendingStreamRunEvents(runtime, run);
|
|
951
|
+
return;
|
|
952
|
+
}
|
|
953
|
+
if (run.pendingEvents.length === 0)
|
|
954
|
+
return;
|
|
955
|
+
const operation = (async () => {
|
|
956
|
+
while (run.pendingEvents.length > 0) {
|
|
957
|
+
if (run.persistenceError)
|
|
958
|
+
throw run.persistenceError;
|
|
959
|
+
const events = run.pendingEvents.splice(0);
|
|
960
|
+
run.pendingEventBytes = 0;
|
|
961
|
+
const frames = await runtime.store.appendEvents({
|
|
962
|
+
sessionId: run.sessionId,
|
|
963
|
+
runId: run.storageRunId,
|
|
964
|
+
events,
|
|
965
|
+
expectedSeq: run.nextSeq,
|
|
966
|
+
});
|
|
967
|
+
run.nextSeq += frames.length;
|
|
968
|
+
notifyStreamRunSubscribers(runtime, run);
|
|
969
|
+
}
|
|
970
|
+
})();
|
|
971
|
+
const tracked = operation.catch((error) => {
|
|
972
|
+
run.persistenceError = error instanceof Error ? error : new Error(String(error));
|
|
973
|
+
throw run.persistenceError;
|
|
974
|
+
});
|
|
975
|
+
run.flushPromise = tracked;
|
|
976
|
+
try {
|
|
977
|
+
await tracked;
|
|
978
|
+
}
|
|
979
|
+
finally {
|
|
980
|
+
if (run.flushPromise === tracked)
|
|
981
|
+
run.flushPromise = undefined;
|
|
982
|
+
if (!run.persistenceError && run.pendingEvents.length > 0 && !run.flushTimer) {
|
|
983
|
+
run.flushTimer = setTimeout(() => {
|
|
984
|
+
run.flushTimer = undefined;
|
|
985
|
+
void flushPendingStreamRunEvents(runtime, run).catch((error) => {
|
|
986
|
+
void handleStreamRunPersistenceFailure(runtime, run, error);
|
|
987
|
+
});
|
|
988
|
+
}, STREAM_RUN_BATCH_DELAY_MS);
|
|
989
|
+
run.flushTimer.unref();
|
|
990
|
+
}
|
|
991
|
+
}
|
|
992
|
+
}
|
|
993
|
+
async function queueStreamRunEvent(runtime, run, event) {
|
|
994
|
+
if (run.persistenceError)
|
|
995
|
+
throw run.persistenceError;
|
|
996
|
+
if (run.status !== "running" || run.terminalPromise || run.abortController.signal.aborted)
|
|
997
|
+
return;
|
|
998
|
+
const encoded = encodeProxyEvent(event);
|
|
999
|
+
run.pendingEvents.push(encoded);
|
|
1000
|
+
run.pendingEventBytes += Buffer.byteLength(encoded);
|
|
1001
|
+
if (run.pendingEvents.length >= STREAM_RUN_BATCH_MAX_EVENTS || run.pendingEventBytes >= STREAM_RUN_BATCH_MAX_BYTES) {
|
|
1002
|
+
await flushPendingStreamRunEvents(runtime, run);
|
|
1003
|
+
return;
|
|
1004
|
+
}
|
|
1005
|
+
if (run.flushTimer || run.flushPromise)
|
|
1006
|
+
return;
|
|
1007
|
+
run.flushTimer = setTimeout(() => {
|
|
1008
|
+
run.flushTimer = undefined;
|
|
1009
|
+
void flushPendingStreamRunEvents(runtime, run).catch((error) => {
|
|
1010
|
+
void handleStreamRunPersistenceFailure(runtime, run, error);
|
|
1011
|
+
});
|
|
1012
|
+
}, STREAM_RUN_BATCH_DELAY_MS);
|
|
1013
|
+
run.flushTimer.unref();
|
|
1014
|
+
}
|
|
1015
|
+
async function persistTerminalStreamRun(runtime, run, message, skipPendingFlush = false) {
|
|
1016
|
+
if (run.terminalPromise)
|
|
1017
|
+
return run.terminalPromise;
|
|
1018
|
+
const operation = (async () => {
|
|
1019
|
+
clearStreamRunFlushTimer(run);
|
|
1020
|
+
if (!skipPendingFlush) {
|
|
1021
|
+
await flushPendingStreamRunEvents(runtime, run);
|
|
1022
|
+
}
|
|
1023
|
+
else {
|
|
1024
|
+
run.pendingEvents.length = 0;
|
|
1025
|
+
run.pendingEventBytes = 0;
|
|
1026
|
+
await run.flushPromise?.catch(() => undefined);
|
|
1027
|
+
}
|
|
1028
|
+
if (run.status !== "running")
|
|
1029
|
+
return;
|
|
1030
|
+
const proxyEvent = terminalProxyEvent(run, message);
|
|
1031
|
+
let state;
|
|
1032
|
+
try {
|
|
1033
|
+
state = await runtime.store.settle({
|
|
1034
|
+
sessionId: run.sessionId,
|
|
1035
|
+
runId: run.storageRunId,
|
|
1036
|
+
status: message.stopReason === "error" || message.stopReason === "aborted" ? "failed" : "completed",
|
|
1037
|
+
event: encodeProxyEvent(proxyEvent),
|
|
1038
|
+
result: message,
|
|
1039
|
+
errorMessage: message.errorMessage,
|
|
1040
|
+
expectedSeq: run.nextSeq,
|
|
1041
|
+
});
|
|
1042
|
+
if (!run.runId) {
|
|
1043
|
+
state = await runtime.store.acknowledge(run.sessionId, run.storageRunId);
|
|
1044
|
+
}
|
|
1045
|
+
}
|
|
1046
|
+
catch (error) {
|
|
1047
|
+
run.persistenceError = error instanceof Error ? error : new Error(String(error));
|
|
1048
|
+
throw run.persistenceError;
|
|
1049
|
+
}
|
|
1050
|
+
applyDurableState(run, state);
|
|
1051
|
+
notifyStreamRunSubscribers(runtime, run);
|
|
1052
|
+
})();
|
|
1053
|
+
run.terminalPromise = operation;
|
|
1054
|
+
try {
|
|
1055
|
+
await operation;
|
|
1056
|
+
}
|
|
1057
|
+
finally {
|
|
1058
|
+
if (run.terminalPromise === operation && run.status === "running") {
|
|
1059
|
+
run.terminalPromise = undefined;
|
|
1060
|
+
}
|
|
1061
|
+
}
|
|
1062
|
+
}
|
|
1063
|
+
function markStreamRunPersistenceFatal(runtime, persistenceError) {
|
|
1064
|
+
if (runtime.fatalPersistenceError)
|
|
1065
|
+
return false;
|
|
1066
|
+
runtime.fatalPersistenceError = persistenceError;
|
|
1067
|
+
runtime.closing = true;
|
|
1068
|
+
for (const activeRun of runtime.activeRuns.values()) {
|
|
1069
|
+
activeRun.shuttingDown = true;
|
|
1070
|
+
activeRun.persistenceError ??= persistenceError;
|
|
1071
|
+
clearStreamRunFlushTimer(activeRun);
|
|
1072
|
+
activeRun.abortController.abort();
|
|
1073
|
+
for (const res of [...activeRun.subscribers.keys()]) {
|
|
1074
|
+
detachStreamRunSubscriber(runtime, activeRun, res);
|
|
1075
|
+
res.destroy(new Error("pi-server stopped after an indeterminate stream persistence failure"));
|
|
1076
|
+
}
|
|
1077
|
+
}
|
|
1078
|
+
for (const replays of runtime.replayReaders.values()) {
|
|
1079
|
+
for (const replay of replays) {
|
|
1080
|
+
replay.abortController.abort();
|
|
1081
|
+
replay.response.destroy(new Error("pi-server stopped after an indeterminate stream persistence failure"));
|
|
1082
|
+
}
|
|
1083
|
+
}
|
|
1084
|
+
return true;
|
|
1085
|
+
}
|
|
1086
|
+
function failStopStreamRunPersistence(runtime, error) {
|
|
1087
|
+
const persistenceError = error instanceof Error ? error : new Error(String(error));
|
|
1088
|
+
if (activeServerPersistenceRuntime?.streamRuntime === runtime) {
|
|
1089
|
+
failStopServerPersistence(activeServerPersistenceRuntime, persistenceError);
|
|
1090
|
+
return;
|
|
1091
|
+
}
|
|
1092
|
+
if (!markStreamRunPersistenceFatal(runtime, persistenceError))
|
|
1093
|
+
return;
|
|
1094
|
+
queueMicrotask(() => runtime.onFatalPersistenceError?.(persistenceError));
|
|
1095
|
+
}
|
|
1096
|
+
async function handleStreamRunPersistenceFailure(runtime, run, error) {
|
|
1097
|
+
if (run.status !== "running" || run.shuttingDown)
|
|
1098
|
+
return;
|
|
1099
|
+
logStreamRunPersistenceFailure(run, error);
|
|
1100
|
+
const persistenceError = error instanceof Error ? error : new Error(String(error));
|
|
1101
|
+
if (persistenceError instanceof StreamRunCorruptionError ||
|
|
1102
|
+
persistenceError instanceof StreamRunPersistenceTimeoutError ||
|
|
1103
|
+
persistenceError instanceof StreamRunStoreLockedError ||
|
|
1104
|
+
persistenceError instanceof StreamRunStateError ||
|
|
1105
|
+
persistenceError instanceof StreamRunConflictError) {
|
|
1106
|
+
failStopStreamRunPersistence(runtime, persistenceError);
|
|
1107
|
+
return;
|
|
1108
|
+
}
|
|
1109
|
+
run.abortController.abort();
|
|
1110
|
+
const errorMessage = `pi-server could not durably persist the provider stream: ${persistenceError.message}`;
|
|
1111
|
+
const message = createSyntheticAssistantMessage(run, errorMessage, "error");
|
|
1112
|
+
try {
|
|
1113
|
+
await persistTerminalStreamRun(runtime, run, message, true);
|
|
1114
|
+
}
|
|
1115
|
+
catch (terminalError) {
|
|
1116
|
+
logStreamRunPersistenceFailure(run, terminalError);
|
|
1117
|
+
failStopStreamRunPersistence(runtime, terminalError);
|
|
1118
|
+
}
|
|
1119
|
+
}
|
|
1120
|
+
async function abortStreamRun(runtime, run, errorMessage, reason = "error") {
|
|
1121
|
+
if (run.status !== "running")
|
|
1122
|
+
return;
|
|
1123
|
+
run.abortController.abort();
|
|
1124
|
+
await persistTerminalStreamRun(runtime, run, createSyntheticAssistantMessage(run, errorMessage, reason));
|
|
1125
|
+
}
|
|
1126
|
+
function compactRunKey(sessionId, operationId) {
|
|
1127
|
+
return `${sessionId}\0${operationId}`;
|
|
1128
|
+
}
|
|
1129
|
+
function enqueueCompactSetup(runtime, operation) {
|
|
1130
|
+
const guardedOperation = () => {
|
|
1131
|
+
assertServerPersistenceHealthy();
|
|
1132
|
+
return operation();
|
|
1133
|
+
};
|
|
1134
|
+
const result = runtime.setupQueue.then(guardedOperation, guardedOperation);
|
|
1135
|
+
runtime.setupQueue = result.then(() => undefined, () => undefined);
|
|
1136
|
+
return result;
|
|
1137
|
+
}
|
|
1138
|
+
const sessionDeletionFences = new WeakMap();
|
|
1139
|
+
function sessionDeletionFenceState(runtime) {
|
|
1140
|
+
let state = sessionDeletionFences.get(runtime);
|
|
1141
|
+
if (!state) {
|
|
1142
|
+
state = { versions: new Map(), active: new Map() };
|
|
1143
|
+
sessionDeletionFences.set(runtime, state);
|
|
1144
|
+
}
|
|
1145
|
+
return state;
|
|
1146
|
+
}
|
|
1147
|
+
function advanceSessionDeletionVersion(state, sessionId) {
|
|
1148
|
+
state.versions.set(sessionId, (state.versions.get(sessionId) ?? 0) + 1);
|
|
1149
|
+
}
|
|
1150
|
+
function cleanupSessionDeletionFence(runtime, sessionId) {
|
|
1151
|
+
const state = sessionDeletionFences.get(runtime);
|
|
1152
|
+
if (!state || (state.active.get(sessionId) ?? 0) > 0 || runtime.queues.has(sessionId))
|
|
1153
|
+
return;
|
|
1154
|
+
state.versions.delete(sessionId);
|
|
1155
|
+
if (state.versions.size === 0 && state.active.size === 0)
|
|
1156
|
+
sessionDeletionFences.delete(runtime);
|
|
1157
|
+
}
|
|
1158
|
+
function enqueueSessionMutation(runtime, sessionId, operation, bypassDeletionFence = false, expectedDeletionVersion) {
|
|
1159
|
+
const deletionState = sessionDeletionFenceState(runtime);
|
|
1160
|
+
const deletionVersion = expectedDeletionVersion ?? deletionState.versions.get(sessionId) ?? 0;
|
|
1161
|
+
const previous = runtime.queues.get(sessionId) ?? Promise.resolve();
|
|
1162
|
+
const guardedOperation = () => {
|
|
1163
|
+
assertServerPersistenceHealthy();
|
|
1164
|
+
if (!bypassDeletionFence &&
|
|
1165
|
+
((deletionState.active.get(sessionId) ?? 0) > 0 ||
|
|
1166
|
+
(deletionState.versions.get(sessionId) ?? 0) !== deletionVersion)) {
|
|
1167
|
+
throw new StreamRunConflictError(`Session ${sessionId} was deleted while this mutation was queued; reconcile before retrying`);
|
|
1168
|
+
}
|
|
1169
|
+
return operation();
|
|
1170
|
+
};
|
|
1171
|
+
const result = previous.then(guardedOperation, guardedOperation);
|
|
1172
|
+
const tail = result.then(() => undefined, () => undefined);
|
|
1173
|
+
runtime.queues.set(sessionId, tail);
|
|
1174
|
+
void tail.finally(() => {
|
|
1175
|
+
if (runtime.queues.get(sessionId) === tail) {
|
|
1176
|
+
runtime.queues.delete(sessionId);
|
|
1177
|
+
cleanupSessionDeletionFence(runtime, sessionId);
|
|
1178
|
+
}
|
|
44
1179
|
});
|
|
1180
|
+
return result;
|
|
45
1181
|
}
|
|
46
|
-
function
|
|
47
|
-
const
|
|
48
|
-
|
|
49
|
-
|
|
1182
|
+
function enqueueSessionCapacityMutation(operation) {
|
|
1183
|
+
const guardedOperation = () => {
|
|
1184
|
+
assertServerPersistenceHealthy();
|
|
1185
|
+
return operation();
|
|
1186
|
+
};
|
|
1187
|
+
const result = sessionCapacityMutationQueue.then(guardedOperation, guardedOperation);
|
|
1188
|
+
sessionCapacityMutationQueue = result.then(() => undefined, () => undefined);
|
|
1189
|
+
return result;
|
|
50
1190
|
}
|
|
51
|
-
function
|
|
52
|
-
|
|
53
|
-
console.error(`${req.method ?? "UNKNOWN"} ${req.url ?? "/"} failed: ${message}`);
|
|
1191
|
+
function enqueueSessionStoreMutation(runtime, sessionId, operation, bypassDeletionFence = false) {
|
|
1192
|
+
return enqueueSessionMutation(runtime, sessionId, () => enqueueSessionCapacityMutation(operation), bypassDeletionFence);
|
|
54
1193
|
}
|
|
55
|
-
function
|
|
56
|
-
if (
|
|
57
|
-
return
|
|
58
|
-
|
|
59
|
-
|
|
1194
|
+
function releaseCompactRun(runtime, run) {
|
|
1195
|
+
if (run.status === "running" || run.subscribers.size > 0)
|
|
1196
|
+
return;
|
|
1197
|
+
runtime.activeRuns.delete(compactRunKey(run.sessionId, run.operationId));
|
|
1198
|
+
}
|
|
1199
|
+
function markCompactPersistenceFatal(runtime, failure) {
|
|
1200
|
+
if (runtime.fatalPersistenceError)
|
|
60
1201
|
return false;
|
|
61
|
-
|
|
62
|
-
|
|
1202
|
+
runtime.fatalPersistenceError = failure;
|
|
1203
|
+
runtime.closing = true;
|
|
1204
|
+
for (const run of runtime.activeRuns.values()) {
|
|
1205
|
+
run.abortController.abort();
|
|
1206
|
+
for (const response of run.subscribers) {
|
|
1207
|
+
response.destroy(new Error("pi-server stopped after an indeterminate compaction persistence failure"));
|
|
1208
|
+
}
|
|
1209
|
+
}
|
|
1210
|
+
return true;
|
|
63
1211
|
}
|
|
64
|
-
function
|
|
1212
|
+
function failStopCompactPersistence(runtime, error) {
|
|
1213
|
+
const failure = error instanceof Error ? error : new Error(String(error));
|
|
1214
|
+
if (activeServerPersistenceRuntime?.compactRuntime === runtime) {
|
|
1215
|
+
failStopServerPersistence(activeServerPersistenceRuntime, failure);
|
|
1216
|
+
return;
|
|
1217
|
+
}
|
|
1218
|
+
if (!markCompactPersistenceFatal(runtime, failure))
|
|
1219
|
+
return;
|
|
1220
|
+
queueMicrotask(() => runtime.onFatalPersistenceError?.(failure));
|
|
1221
|
+
}
|
|
1222
|
+
function failStopServerPersistence(runtime, error) {
|
|
1223
|
+
if (runtime.fatalError)
|
|
1224
|
+
return;
|
|
1225
|
+
const failure = error instanceof Error ? error : new Error(String(error));
|
|
1226
|
+
runtime.fatalError = failure;
|
|
1227
|
+
if (runtime.streamRuntime)
|
|
1228
|
+
markStreamRunPersistenceFatal(runtime.streamRuntime, failure);
|
|
1229
|
+
if (runtime.compactRuntime)
|
|
1230
|
+
markCompactPersistenceFatal(runtime.compactRuntime, failure);
|
|
1231
|
+
queueMicrotask(() => runtime.onFatalPersistenceError?.(failure));
|
|
1232
|
+
}
|
|
1233
|
+
function createCompactTerminalPlan(run, response, commit) {
|
|
65
1234
|
return {
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
revision: session.revision,
|
|
1235
|
+
version: 1,
|
|
1236
|
+
sessionId: run.sessionId,
|
|
1237
|
+
operationId: run.operationId,
|
|
1238
|
+
requestHash: run.requestHash,
|
|
1239
|
+
response,
|
|
1240
|
+
...(commit ? { commit } : {}),
|
|
73
1241
|
};
|
|
74
1242
|
}
|
|
75
|
-
function
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
1243
|
+
function createCompactFailureResponse(run, status, error, operationDisposition) {
|
|
1244
|
+
return {
|
|
1245
|
+
status,
|
|
1246
|
+
body: {
|
|
1247
|
+
protocolVersion: PI_SERVER_PROTOCOL_VERSION,
|
|
1248
|
+
sessionId: run.sessionId,
|
|
1249
|
+
operationId: run.operationId,
|
|
1250
|
+
requestHash: run.requestHash,
|
|
1251
|
+
status: operationDisposition === "terminal" ? "failed" : "rejected",
|
|
1252
|
+
httpStatus: status,
|
|
1253
|
+
operationDisposition,
|
|
1254
|
+
error,
|
|
1255
|
+
},
|
|
1256
|
+
};
|
|
80
1257
|
}
|
|
81
|
-
function
|
|
82
|
-
|
|
1258
|
+
function createCompactCapacityFailureResponse(run, error, operationDisposition) {
|
|
1259
|
+
const errorMessage = error instanceof SessionPersistenceCapacityError
|
|
1260
|
+
? `Persisted session capacity exceeded: resource=${error.resource}, session=${error.sessionId}, requested=${error.requested}, limit=${error.limit}`
|
|
1261
|
+
: error.message;
|
|
1262
|
+
const response = createCompactFailureResponse(run, 507, errorMessage, operationDisposition);
|
|
1263
|
+
return {
|
|
1264
|
+
status: response.status,
|
|
1265
|
+
body: {
|
|
1266
|
+
...response.body,
|
|
1267
|
+
code: error.code,
|
|
1268
|
+
resource: error.resource,
|
|
1269
|
+
current: error.current,
|
|
1270
|
+
requested: error.requested,
|
|
1271
|
+
limit: error.limit,
|
|
1272
|
+
retryable: false,
|
|
1273
|
+
...(error instanceof SessionPersistenceCapacityError ? { artifact: basename(error.path) } : {}),
|
|
1274
|
+
},
|
|
1275
|
+
};
|
|
83
1276
|
}
|
|
84
|
-
function
|
|
85
|
-
const
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
runId,
|
|
92
|
-
status: "running",
|
|
93
|
-
events: [],
|
|
94
|
-
createdAt: now,
|
|
95
|
-
updatedAt: now,
|
|
1277
|
+
function createCompactRestartFailureTerminal(identity) {
|
|
1278
|
+
const response = createCompactFailureResponse({ sessionId: identity.sessionId, operationId: identity.runId, requestHash: identity.requestMac }, 500, "restart-unknown: pi-server restarted before compaction reached a durable terminal state; provider execution will not be repeated", "terminal");
|
|
1279
|
+
const result = createCompactTerminalPlan({ sessionId: identity.sessionId, operationId: identity.runId, requestHash: identity.requestMac }, response);
|
|
1280
|
+
return {
|
|
1281
|
+
event: JSON.stringify(result),
|
|
1282
|
+
result,
|
|
1283
|
+
errorMessage: response.body.error,
|
|
96
1284
|
};
|
|
97
|
-
run.status = "running";
|
|
98
|
-
run.updatedAt = now;
|
|
99
|
-
streamRuns.set(runKey(sessionId, runId), run);
|
|
100
|
-
return run;
|
|
101
1285
|
}
|
|
102
|
-
function
|
|
103
|
-
|
|
1286
|
+
function compactTerminalPlanFromState(state) {
|
|
1287
|
+
const result = state.terminal?.result;
|
|
1288
|
+
assertCompactTerminalPlan(result, `${state.sessionId}/${state.runId}`);
|
|
1289
|
+
if (result.sessionId !== state.sessionId ||
|
|
1290
|
+
result.operationId !== state.runId ||
|
|
1291
|
+
result.requestHash !== state.requestMac) {
|
|
1292
|
+
throw new StreamRunCorruptionError(`${state.sessionId}/${state.runId}`, "terminal compaction identity does not match durable run metadata");
|
|
1293
|
+
}
|
|
1294
|
+
return result;
|
|
1295
|
+
}
|
|
1296
|
+
function applyCompactTerminalPlan(config, plan) {
|
|
1297
|
+
const commit = plan.commit;
|
|
1298
|
+
if (!commit)
|
|
1299
|
+
return;
|
|
1300
|
+
mutateAndPersistSession(config, plan.sessionId, () => applyPiServerCompactionEntry(plan.sessionId, {
|
|
1301
|
+
entry: commit.entry,
|
|
1302
|
+
operation: commit.entry.piServerCompactOperation,
|
|
1303
|
+
updatedTreeHash: commit.updatedTreeHash,
|
|
1304
|
+
updatedRevision: commit.updatedRevision,
|
|
1305
|
+
}));
|
|
1306
|
+
}
|
|
1307
|
+
function preflightCompactTerminalPlan(config, plan) {
|
|
1308
|
+
const commit = plan.commit;
|
|
1309
|
+
if (!commit)
|
|
104
1310
|
return;
|
|
105
|
-
|
|
106
|
-
|
|
1311
|
+
const current = getSession(plan.sessionId);
|
|
1312
|
+
if (!current) {
|
|
1313
|
+
throw new Error(`Session ${plan.sessionId} disappeared before compaction persistence preflight`);
|
|
1314
|
+
}
|
|
1315
|
+
const previousState = exportSessionState(current);
|
|
1316
|
+
try {
|
|
1317
|
+
const candidate = applyPiServerCompactionEntry(plan.sessionId, {
|
|
1318
|
+
entry: commit.entry,
|
|
1319
|
+
operation: commit.entry.piServerCompactOperation,
|
|
1320
|
+
updatedTreeHash: commit.updatedTreeHash,
|
|
1321
|
+
updatedRevision: commit.updatedRevision,
|
|
1322
|
+
});
|
|
1323
|
+
preflightPersistedSession(config.sessionStoreDir, candidate);
|
|
1324
|
+
}
|
|
1325
|
+
finally {
|
|
1326
|
+
restoreSessionState(previousState);
|
|
1327
|
+
}
|
|
1328
|
+
}
|
|
1329
|
+
function settleCompactRun(config, runtime, mutations, run, planOrFactory, status = "completed") {
|
|
1330
|
+
if (run.settlement)
|
|
1331
|
+
return run.settlement;
|
|
1332
|
+
const settlement = (async () => {
|
|
1333
|
+
const durablePlan = await enqueueSessionMutation(mutations, run.sessionId, () => enqueueSessionCapacityMutation(async () => {
|
|
1334
|
+
let plan = typeof planOrFactory === "function" ? planOrFactory() : planOrFactory;
|
|
1335
|
+
if (plan.commit) {
|
|
1336
|
+
try {
|
|
1337
|
+
preflightSessionCapacityMutation(run.sessionId, {
|
|
1338
|
+
content: {
|
|
1339
|
+
kind: "append_entries",
|
|
1340
|
+
entries: [plan.commit.entry],
|
|
1341
|
+
leafId: plan.commit.entry.id,
|
|
1342
|
+
},
|
|
1343
|
+
});
|
|
1344
|
+
preflightCompactTerminalPlan(config, plan);
|
|
1345
|
+
}
|
|
1346
|
+
catch (error) {
|
|
1347
|
+
if (!(error instanceof SessionCapacityError) && !(error instanceof SessionPersistenceCapacityError)) {
|
|
1348
|
+
throw error;
|
|
1349
|
+
}
|
|
1350
|
+
plan = createCompactTerminalPlan(run, createCompactCapacityFailureResponse(run, error, "terminal"));
|
|
1351
|
+
}
|
|
1352
|
+
}
|
|
1353
|
+
const durableStatus = plan.commit ? status : "failed";
|
|
1354
|
+
const state = await runtime.store.settle({
|
|
1355
|
+
sessionId: run.sessionId,
|
|
1356
|
+
runId: run.operationId,
|
|
1357
|
+
status: durableStatus,
|
|
1358
|
+
event: JSON.stringify(plan),
|
|
1359
|
+
result: plan,
|
|
1360
|
+
...(durableStatus === "failed"
|
|
1361
|
+
? {
|
|
1362
|
+
errorMessage: "error" in plan.response.body ? plan.response.body.error : "Compaction failed before commit",
|
|
1363
|
+
}
|
|
1364
|
+
: {}),
|
|
1365
|
+
});
|
|
1366
|
+
const settledPlan = compactTerminalPlanFromState(state);
|
|
1367
|
+
applyCompactTerminalPlan(config, settledPlan);
|
|
1368
|
+
return settledPlan;
|
|
1369
|
+
}));
|
|
1370
|
+
run.status = "settled";
|
|
1371
|
+
run.result = durablePlan.response;
|
|
1372
|
+
run.resolveTerminal?.(durablePlan.response);
|
|
1373
|
+
run.resolveTerminal = undefined;
|
|
1374
|
+
run.rejectTerminal = undefined;
|
|
1375
|
+
return durablePlan.response;
|
|
1376
|
+
})().catch((error) => {
|
|
1377
|
+
run.rejectTerminal?.(error);
|
|
1378
|
+
run.resolveTerminal = undefined;
|
|
1379
|
+
run.rejectTerminal = undefined;
|
|
1380
|
+
failStopCompactPersistence(runtime, error);
|
|
1381
|
+
throw error;
|
|
1382
|
+
});
|
|
1383
|
+
run.settlement = settlement;
|
|
1384
|
+
void settlement.finally(() => releaseCompactRun(runtime, run)).catch(() => undefined);
|
|
1385
|
+
return settlement;
|
|
107
1386
|
}
|
|
108
|
-
function
|
|
109
|
-
if (!
|
|
1387
|
+
async function handleCancelSessionCompact(config, runtime, mutations, body, res) {
|
|
1388
|
+
if (!body.sessionId || !body.operationId || !body.requestHash) {
|
|
1389
|
+
sendJson(res, 400, { error: "sessionId, operationId, and requestHash are required" });
|
|
1390
|
+
return;
|
|
1391
|
+
}
|
|
1392
|
+
const action = await enqueueCompactSetup(runtime, async () => {
|
|
1393
|
+
const state = await runtime.store.get(body.sessionId, body.operationId);
|
|
1394
|
+
if (!state)
|
|
1395
|
+
return { kind: "missing" };
|
|
1396
|
+
if (state.requestMac !== body.requestHash) {
|
|
1397
|
+
throw new StreamRunConflictError("operationId is bound to a different compaction request");
|
|
1398
|
+
}
|
|
1399
|
+
if (state.status !== "running") {
|
|
1400
|
+
return { kind: "terminal", state };
|
|
1401
|
+
}
|
|
1402
|
+
const run = runtime.activeRuns.get(compactRunKey(body.sessionId, body.operationId));
|
|
1403
|
+
if (!run) {
|
|
1404
|
+
throw new StreamRunStateError("Durable running compaction has no active provider execution");
|
|
1405
|
+
}
|
|
1406
|
+
run.abortController.abort();
|
|
1407
|
+
const response = createCompactFailureResponse(run, 499, "Compaction aborted by user", "terminal");
|
|
1408
|
+
return {
|
|
1409
|
+
kind: "canceled",
|
|
1410
|
+
promise: settleCompactRun(config, runtime, mutations, run, createCompactTerminalPlan(run, response), "failed"),
|
|
1411
|
+
};
|
|
1412
|
+
});
|
|
1413
|
+
if (action.kind === "missing") {
|
|
1414
|
+
sendJson(res, 404, { error: "compaction operation not found" });
|
|
1415
|
+
return;
|
|
1416
|
+
}
|
|
1417
|
+
if (action.kind === "terminal") {
|
|
1418
|
+
const plan = compactTerminalPlanFromState(action.state);
|
|
1419
|
+
sendJson(res, 200, {
|
|
1420
|
+
canceled: false,
|
|
1421
|
+
status: action.state.status,
|
|
1422
|
+
sessionId: action.state.sessionId,
|
|
1423
|
+
operationId: action.state.runId,
|
|
1424
|
+
requestHash: action.state.requestMac,
|
|
1425
|
+
resultStatus: plan.response.status,
|
|
1426
|
+
terminal: plan.response.body,
|
|
1427
|
+
});
|
|
110
1428
|
return;
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
1429
|
+
}
|
|
1430
|
+
const terminalResponse = await action.promise;
|
|
1431
|
+
sendJson(res, 200, {
|
|
1432
|
+
canceled: true,
|
|
1433
|
+
status: "failed",
|
|
1434
|
+
sessionId: body.sessionId,
|
|
1435
|
+
operationId: body.operationId,
|
|
1436
|
+
requestHash: body.requestHash,
|
|
1437
|
+
resultStatus: 499,
|
|
1438
|
+
terminal: terminalResponse.body,
|
|
1439
|
+
});
|
|
1440
|
+
}
|
|
1441
|
+
function pruneChunkTargetResponses(responses, now = Date.now(), extraBytes = 0) {
|
|
1442
|
+
for (const [requestId, response] of responses) {
|
|
1443
|
+
if (now - response.createdAt >= CHUNK_TARGET_RESPONSE_TTL_MS) {
|
|
1444
|
+
responses.delete(requestId);
|
|
1445
|
+
}
|
|
1446
|
+
}
|
|
1447
|
+
let retainedBytes = 0;
|
|
1448
|
+
for (const response of responses.values())
|
|
1449
|
+
retainedBytes += response.bytes;
|
|
1450
|
+
while (responses.size >= CHUNK_TARGET_RESPONSE_MAX_RECORDS ||
|
|
1451
|
+
retainedBytes + extraBytes > CHUNK_TARGET_RESPONSE_MAX_BYTES) {
|
|
1452
|
+
const oldest = responses.keys().next();
|
|
1453
|
+
if (oldest.done)
|
|
1454
|
+
break;
|
|
1455
|
+
const oldestResponse = responses.get(oldest.value);
|
|
1456
|
+
if (!oldestResponse)
|
|
1457
|
+
throw new Error("Chunk target response cache became inconsistent");
|
|
1458
|
+
retainedBytes -= oldestResponse.bytes;
|
|
1459
|
+
responses.delete(oldest.value);
|
|
1460
|
+
}
|
|
115
1461
|
}
|
|
116
|
-
function
|
|
117
|
-
|
|
1462
|
+
function cacheChunkTargetResponse(responses, requestId, response) {
|
|
1463
|
+
const bytes = Buffer.byteLength(response.bodyJson);
|
|
1464
|
+
if (bytes > CHUNK_TARGET_RESPONSE_MAX_BYTES)
|
|
1465
|
+
return;
|
|
1466
|
+
pruneChunkTargetResponses(responses, Date.now(), bytes);
|
|
1467
|
+
if (responses.size >= CHUNK_TARGET_RESPONSE_MAX_RECORDS)
|
|
118
1468
|
return;
|
|
119
|
-
|
|
120
|
-
run.errorMessage = errorMessage;
|
|
121
|
-
run.updatedAt = Date.now();
|
|
1469
|
+
responses.set(requestId, { ...response, createdAt: Date.now(), bytes });
|
|
122
1470
|
}
|
|
123
1471
|
function sessionHistoryFullResponseBody(session, baseMessageCount) {
|
|
124
1472
|
return {
|
|
@@ -135,6 +1483,12 @@ function sessionHistoryFullResponseBody(session, baseMessageCount) {
|
|
|
135
1483
|
messages: session.messages.slice(baseMessageCount),
|
|
136
1484
|
};
|
|
137
1485
|
}
|
|
1486
|
+
function sessionHistoryV2FullResponseBody(session) {
|
|
1487
|
+
return {
|
|
1488
|
+
...sessionHistoryV2Summary(session),
|
|
1489
|
+
entries: session.entries,
|
|
1490
|
+
};
|
|
1491
|
+
}
|
|
138
1492
|
function sessionTreePatchResponseBody(session, baseMessageCount, entriesFrom, baseRevision) {
|
|
139
1493
|
return {
|
|
140
1494
|
sessionId: session.sessionId,
|
|
@@ -156,6 +1510,174 @@ function sessionTreePatchResponseBody(session, baseMessageCount, entriesFrom, ba
|
|
|
156
1510
|
},
|
|
157
1511
|
};
|
|
158
1512
|
}
|
|
1513
|
+
function sessionTreePatchV2ResponseBody(session, entriesFrom, baseRevision) {
|
|
1514
|
+
return {
|
|
1515
|
+
...sessionHistoryV2Summary(session),
|
|
1516
|
+
treePatch: {
|
|
1517
|
+
entriesFrom,
|
|
1518
|
+
baseRevision,
|
|
1519
|
+
entries: session.entries.slice(entriesFrom),
|
|
1520
|
+
leafId: session.leafId,
|
|
1521
|
+
revision: session.revision,
|
|
1522
|
+
},
|
|
1523
|
+
};
|
|
1524
|
+
}
|
|
1525
|
+
function isRecord(value) {
|
|
1526
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1527
|
+
}
|
|
1528
|
+
function isNonNegativeFiniteNumber(value) {
|
|
1529
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0;
|
|
1530
|
+
}
|
|
1531
|
+
function isCompactionUsage(value) {
|
|
1532
|
+
if (!isRecord(value) || !isRecord(value.cost))
|
|
1533
|
+
return false;
|
|
1534
|
+
if (!isNonNegativeFiniteNumber(value.input) ||
|
|
1535
|
+
!isNonNegativeFiniteNumber(value.output) ||
|
|
1536
|
+
!isNonNegativeFiniteNumber(value.cacheRead) ||
|
|
1537
|
+
!isNonNegativeFiniteNumber(value.cacheWrite) ||
|
|
1538
|
+
!isNonNegativeFiniteNumber(value.totalTokens) ||
|
|
1539
|
+
!isNonNegativeFiniteNumber(value.cost.input) ||
|
|
1540
|
+
!isNonNegativeFiniteNumber(value.cost.output) ||
|
|
1541
|
+
!isNonNegativeFiniteNumber(value.cost.cacheRead) ||
|
|
1542
|
+
!isNonNegativeFiniteNumber(value.cost.cacheWrite) ||
|
|
1543
|
+
!isNonNegativeFiniteNumber(value.cost.total)) {
|
|
1544
|
+
return false;
|
|
1545
|
+
}
|
|
1546
|
+
return ((value.cacheWrite1h === undefined || isNonNegativeFiniteNumber(value.cacheWrite1h)) &&
|
|
1547
|
+
(value.reasoning === undefined || isNonNegativeFiniteNumber(value.reasoning)));
|
|
1548
|
+
}
|
|
1549
|
+
function assertCompactTerminalPlan(value, source) {
|
|
1550
|
+
if (!isRecord(value) ||
|
|
1551
|
+
value.version !== 1 ||
|
|
1552
|
+
typeof value.sessionId !== "string" ||
|
|
1553
|
+
value.sessionId.length === 0 ||
|
|
1554
|
+
typeof value.operationId !== "string" ||
|
|
1555
|
+
value.operationId.length === 0 ||
|
|
1556
|
+
typeof value.requestHash !== "string" ||
|
|
1557
|
+
!/^[a-f0-9]{64}$/u.test(value.requestHash) ||
|
|
1558
|
+
!isRecord(value.response) ||
|
|
1559
|
+
!Number.isSafeInteger(value.response.status) ||
|
|
1560
|
+
value.response.status < 100 ||
|
|
1561
|
+
value.response.status > 599 ||
|
|
1562
|
+
!isRecord(value.response.body)) {
|
|
1563
|
+
throw new StreamRunCorruptionError(source, "terminal compaction plan has invalid identity or response");
|
|
1564
|
+
}
|
|
1565
|
+
if (value.commit === undefined) {
|
|
1566
|
+
if (value.response.body.protocolVersion !== PI_SERVER_PROTOCOL_VERSION ||
|
|
1567
|
+
value.response.body.sessionId !== value.sessionId ||
|
|
1568
|
+
value.response.body.operationId !== value.operationId ||
|
|
1569
|
+
value.response.body.requestHash !== value.requestHash ||
|
|
1570
|
+
value.response.body.status !== "failed" ||
|
|
1571
|
+
value.response.body.httpStatus !== value.response.status ||
|
|
1572
|
+
value.response.body.operationDisposition !== "terminal" ||
|
|
1573
|
+
typeof value.response.body.error !== "string" ||
|
|
1574
|
+
value.response.body.error.length === 0) {
|
|
1575
|
+
throw new StreamRunCorruptionError(source, "non-commit terminal compaction plan has no error response");
|
|
1576
|
+
}
|
|
1577
|
+
return;
|
|
1578
|
+
}
|
|
1579
|
+
if (!isRecord(value.commit) || !isRecord(value.commit.base) || !isRecord(value.commit.entry)) {
|
|
1580
|
+
throw new StreamRunCorruptionError(source, "terminal compaction commit is invalid");
|
|
1581
|
+
}
|
|
1582
|
+
const base = value.commit.base;
|
|
1583
|
+
const entry = value.commit.entry;
|
|
1584
|
+
if (typeof base.staticContextHash !== "string" ||
|
|
1585
|
+
(base.staticContextHash !== "" && !/^[a-f0-9]{64}$/u.test(base.staticContextHash)) ||
|
|
1586
|
+
typeof base.treeHash !== "string" ||
|
|
1587
|
+
!/^[a-f0-9]{64}$/u.test(base.treeHash) ||
|
|
1588
|
+
!Number.isSafeInteger(base.entryCount) ||
|
|
1589
|
+
base.entryCount < 0 ||
|
|
1590
|
+
(base.leafId !== null && (typeof base.leafId !== "string" || base.leafId.length === 0)) ||
|
|
1591
|
+
!Number.isSafeInteger(base.revision) ||
|
|
1592
|
+
base.revision < 0 ||
|
|
1593
|
+
entry.type !== "compaction" ||
|
|
1594
|
+
typeof entry.id !== "string" ||
|
|
1595
|
+
entry.id.length === 0 ||
|
|
1596
|
+
entry.parentId !== base.leafId ||
|
|
1597
|
+
typeof entry.timestamp !== "string" ||
|
|
1598
|
+
typeof entry.summary !== "string" ||
|
|
1599
|
+
typeof entry.firstKeptEntryId !== "string" ||
|
|
1600
|
+
entry.firstKeptEntryId.length === 0 ||
|
|
1601
|
+
!isNonNegativeFiniteNumber(entry.tokensBefore) ||
|
|
1602
|
+
!isRecord(entry.piServerCompactOperation) ||
|
|
1603
|
+
typeof value.commit.updatedTreeHash !== "string" ||
|
|
1604
|
+
!/^[a-f0-9]{64}$/u.test(value.commit.updatedTreeHash) ||
|
|
1605
|
+
!Number.isSafeInteger(value.commit.updatedRevision) ||
|
|
1606
|
+
value.commit.updatedRevision !== base.revision + 1) {
|
|
1607
|
+
throw new StreamRunCorruptionError(source, "terminal compaction commit fields are invalid");
|
|
1608
|
+
}
|
|
1609
|
+
const operation = entry.piServerCompactOperation;
|
|
1610
|
+
if (operation.version !== 1 ||
|
|
1611
|
+
operation.operationId !== value.operationId ||
|
|
1612
|
+
operation.requestHash !== value.requestHash ||
|
|
1613
|
+
operation.baseStaticContextHash !== base.staticContextHash ||
|
|
1614
|
+
operation.baseTreeHash !== base.treeHash ||
|
|
1615
|
+
operation.baseEntryCount !== base.entryCount ||
|
|
1616
|
+
operation.baseLeafId !== base.leafId ||
|
|
1617
|
+
operation.baseRevision !== base.revision ||
|
|
1618
|
+
appendPiServerTreeHash(base.treeHash, hashPiServerTreeEntry(entry)) !==
|
|
1619
|
+
value.commit.updatedTreeHash) {
|
|
1620
|
+
throw new StreamRunCorruptionError(source, "terminal compaction operation metadata is inconsistent");
|
|
1621
|
+
}
|
|
1622
|
+
const body = value.response.body;
|
|
1623
|
+
if (value.response.status !== 200 ||
|
|
1624
|
+
body.protocolVersion !== PI_SERVER_PROTOCOL_VERSION ||
|
|
1625
|
+
body.sessionId !== value.sessionId ||
|
|
1626
|
+
body.operationId !== value.operationId ||
|
|
1627
|
+
body.requestHash !== value.requestHash ||
|
|
1628
|
+
!isRecord(body.treePatch)) {
|
|
1629
|
+
throw new StreamRunCorruptionError(source, "terminal compaction response does not match its commit");
|
|
1630
|
+
}
|
|
1631
|
+
const patch = body.treePatch;
|
|
1632
|
+
if (patch.baseStaticContextHash !== base.staticContextHash ||
|
|
1633
|
+
patch.baseTreeHash !== base.treeHash ||
|
|
1634
|
+
patch.baseEntryCount !== base.entryCount ||
|
|
1635
|
+
patch.baseLeafId !== base.leafId ||
|
|
1636
|
+
patch.baseRevision !== base.revision ||
|
|
1637
|
+
patch.entriesFrom !== base.entryCount ||
|
|
1638
|
+
!Array.isArray(patch.entries) ||
|
|
1639
|
+
patch.entries.length !== 1 ||
|
|
1640
|
+
canonicalJsonStringify(patch.entries[0]) !== canonicalJsonStringify(entry) ||
|
|
1641
|
+
patch.leafId !== entry.id ||
|
|
1642
|
+
patch.revision !== value.commit.updatedRevision ||
|
|
1643
|
+
patch.treeHash !== value.commit.updatedTreeHash) {
|
|
1644
|
+
throw new StreamRunCorruptionError(source, "terminal compaction response patch is inconsistent");
|
|
1645
|
+
}
|
|
1646
|
+
}
|
|
1647
|
+
function normalizeExtensionCompaction(value, activeBranch) {
|
|
1648
|
+
if (!isRecord(value)) {
|
|
1649
|
+
throw new Error("extensionCompaction must be an object");
|
|
1650
|
+
}
|
|
1651
|
+
if (typeof value.summary !== "string") {
|
|
1652
|
+
throw new Error("extensionCompaction.summary must be a string");
|
|
1653
|
+
}
|
|
1654
|
+
if (typeof value.firstKeptEntryId !== "string" || value.firstKeptEntryId.length === 0) {
|
|
1655
|
+
throw new Error("extensionCompaction.firstKeptEntryId must be a non-empty string");
|
|
1656
|
+
}
|
|
1657
|
+
if (!activeBranch.some((entry) => entry.id === value.firstKeptEntryId)) {
|
|
1658
|
+
throw new Error("extensionCompaction.firstKeptEntryId must reference the active session branch");
|
|
1659
|
+
}
|
|
1660
|
+
if (!isNonNegativeFiniteNumber(value.tokensBefore)) {
|
|
1661
|
+
throw new Error("extensionCompaction.tokensBefore must be a non-negative finite number");
|
|
1662
|
+
}
|
|
1663
|
+
if (value.retainedTail !== undefined && !Array.isArray(value.retainedTail)) {
|
|
1664
|
+
throw new Error("extensionCompaction.retainedTail must be an array when provided");
|
|
1665
|
+
}
|
|
1666
|
+
if (value.usage !== undefined && !isCompactionUsage(value.usage)) {
|
|
1667
|
+
throw new Error("extensionCompaction.usage must be a valid non-negative usage object");
|
|
1668
|
+
}
|
|
1669
|
+
return {
|
|
1670
|
+
summary: value.summary,
|
|
1671
|
+
firstKeptEntryId: value.firstKeptEntryId,
|
|
1672
|
+
tokensBefore: value.tokensBefore,
|
|
1673
|
+
...(value.retainedTail !== undefined
|
|
1674
|
+
? { retainedTail: value.retainedTail }
|
|
1675
|
+
: {}),
|
|
1676
|
+
...(value.details !== undefined ? { details: value.details } : {}),
|
|
1677
|
+
...(value.usage !== undefined ? { usage: value.usage } : {}),
|
|
1678
|
+
fromHook: true,
|
|
1679
|
+
};
|
|
1680
|
+
}
|
|
159
1681
|
export function resolveStreamOptions(_config, model, body) {
|
|
160
1682
|
return { model, options: { ...(body.options ?? {}) } };
|
|
161
1683
|
}
|
|
@@ -165,13 +1687,23 @@ function handleSessionInit(config, body, res) {
|
|
|
165
1687
|
return;
|
|
166
1688
|
}
|
|
167
1689
|
if (body.staticContext) {
|
|
168
|
-
|
|
1690
|
+
const session = mutateAndPersistSession(config, body.sessionId, () => applySessionMutation(body.sessionId, { staticContext: body.staticContext }));
|
|
1691
|
+
sendJson(res, 200, sessionResponseBody(session));
|
|
1692
|
+
return;
|
|
169
1693
|
}
|
|
170
|
-
else {
|
|
171
|
-
|
|
1694
|
+
else if (body.staticContextHash !== undefined) {
|
|
1695
|
+
if (!/^[a-f0-9]{64}$/i.test(body.staticContextHash)) {
|
|
1696
|
+
sendJson(res, 400, { error: "staticContextHash must be a 64-character hex digest" });
|
|
1697
|
+
return;
|
|
1698
|
+
}
|
|
1699
|
+
const session = mutateAndPersistSession(config, body.sessionId, () => getOrCreateSession(body.sessionId));
|
|
1700
|
+
sendJson(res, 200, {
|
|
1701
|
+
...sessionResponseBody(session),
|
|
1702
|
+
staticContextRequired: session.staticContextHash !== body.staticContextHash,
|
|
1703
|
+
});
|
|
1704
|
+
return;
|
|
172
1705
|
}
|
|
173
|
-
const session =
|
|
174
|
-
persistSession(config, session);
|
|
1706
|
+
const session = mutateAndPersistSession(config, body.sessionId, () => getOrCreateSession(body.sessionId));
|
|
175
1707
|
sendJson(res, 200, sessionResponseBody(session));
|
|
176
1708
|
}
|
|
177
1709
|
function handleSessionUpdate(config, body, res) {
|
|
@@ -183,9 +1715,7 @@ function handleSessionUpdate(config, body, res) {
|
|
|
183
1715
|
sendJson(res, 400, { error: "staticContext is required for update" });
|
|
184
1716
|
return;
|
|
185
1717
|
}
|
|
186
|
-
|
|
187
|
-
const session = getSession(body.sessionId);
|
|
188
|
-
persistSession(config, session);
|
|
1718
|
+
const session = mutateAndPersistSession(config, body.sessionId, () => applySessionMutation(body.sessionId, { staticContext: body.staticContext }));
|
|
189
1719
|
sendJson(res, 200, sessionResponseBody(session));
|
|
190
1720
|
}
|
|
191
1721
|
function handleSessionSync(config, body, res) {
|
|
@@ -197,11 +1727,10 @@ function handleSessionSync(config, body, res) {
|
|
|
197
1727
|
sendJson(res, 400, { error: "messages is required" });
|
|
198
1728
|
return;
|
|
199
1729
|
}
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
persistSession(config, session);
|
|
1730
|
+
const session = mutateAndPersistSession(config, body.sessionId, () => applySessionMutation(body.sessionId, {
|
|
1731
|
+
...(body.staticContext ? { staticContext: body.staticContext } : {}),
|
|
1732
|
+
content: { kind: "replace_messages", messages: body.messages },
|
|
1733
|
+
}));
|
|
205
1734
|
sendJson(res, 200, sessionResponseBody(session));
|
|
206
1735
|
}
|
|
207
1736
|
function handleSessionAppend(config, body, res) {
|
|
@@ -213,11 +1742,10 @@ function handleSessionAppend(config, body, res) {
|
|
|
213
1742
|
sendJson(res, 400, { error: "messages is required" });
|
|
214
1743
|
return;
|
|
215
1744
|
}
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
persistSession(config, session);
|
|
1745
|
+
const session = mutateAndPersistSession(config, body.sessionId, () => applySessionMutation(body.sessionId, {
|
|
1746
|
+
...(body.staticContext ? { staticContext: body.staticContext } : {}),
|
|
1747
|
+
content: { kind: "append_messages", messages: body.messages },
|
|
1748
|
+
}));
|
|
221
1749
|
sendJson(res, 200, sessionResponseBody(session));
|
|
222
1750
|
}
|
|
223
1751
|
function handleSessionTreeSync(config, body, res) {
|
|
@@ -229,11 +1757,10 @@ function handleSessionTreeSync(config, body, res) {
|
|
|
229
1757
|
sendJson(res, 400, { error: "entries is required" });
|
|
230
1758
|
return;
|
|
231
1759
|
}
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
persistSession(config, session);
|
|
1760
|
+
const session = mutateAndPersistSession(config, body.sessionId, () => applySessionMutation(body.sessionId, {
|
|
1761
|
+
...(body.staticContext ? { staticContext: body.staticContext } : {}),
|
|
1762
|
+
content: { kind: "replace_entries", entries: body.entries, leafId: body.leafId ?? null },
|
|
1763
|
+
}));
|
|
237
1764
|
sendJson(res, 200, sessionResponseBody(session));
|
|
238
1765
|
}
|
|
239
1766
|
function handleSessionTreeAppend(config, body, res) {
|
|
@@ -245,11 +1772,10 @@ function handleSessionTreeAppend(config, body, res) {
|
|
|
245
1772
|
sendJson(res, 400, { error: "entries is required" });
|
|
246
1773
|
return;
|
|
247
1774
|
}
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
persistSession(config, session);
|
|
1775
|
+
const session = mutateAndPersistSession(config, body.sessionId, () => applySessionMutation(body.sessionId, {
|
|
1776
|
+
...(body.staticContext ? { staticContext: body.staticContext } : {}),
|
|
1777
|
+
content: { kind: "append_entries", entries: body.entries, leafId: body.leafId ?? null },
|
|
1778
|
+
}));
|
|
253
1779
|
sendJson(res, 200, sessionResponseBody(session));
|
|
254
1780
|
}
|
|
255
1781
|
function handleSessionTreeSwitch(config, body, res) {
|
|
@@ -257,8 +1783,7 @@ function handleSessionTreeSwitch(config, body, res) {
|
|
|
257
1783
|
sendJson(res, 400, { error: "sessionId is required" });
|
|
258
1784
|
return;
|
|
259
1785
|
}
|
|
260
|
-
const session = switchSessionLeaf(body.sessionId, body.leafId ?? null);
|
|
261
|
-
persistSession(config, session);
|
|
1786
|
+
const session = mutateAndPersistSession(config, body.sessionId, () => switchSessionLeaf(body.sessionId, body.leafId ?? null));
|
|
262
1787
|
sendJson(res, 200, sessionResponseBody(session));
|
|
263
1788
|
}
|
|
264
1789
|
function prepareSessionCompact(body) {
|
|
@@ -272,6 +1797,16 @@ function prepareSessionCompact(body) {
|
|
|
272
1797
|
if (!session) {
|
|
273
1798
|
return { status: 404, body: { error: "session not found" } };
|
|
274
1799
|
}
|
|
1800
|
+
if (body.baseStaticContextHash !== session.staticContextHash ||
|
|
1801
|
+
body.baseTreeHash !== session.treeHash ||
|
|
1802
|
+
body.baseEntryCount !== session.entries.length ||
|
|
1803
|
+
body.baseLeafId !== session.leafId ||
|
|
1804
|
+
body.baseRevision !== session.revision) {
|
|
1805
|
+
return {
|
|
1806
|
+
status: 409,
|
|
1807
|
+
body: { error: "Session base identity does not match the compaction request; reconcile and retry" },
|
|
1808
|
+
};
|
|
1809
|
+
}
|
|
275
1810
|
const entries = getSessionBranch(session);
|
|
276
1811
|
const preparationResult = prepareCompaction(entries, body.settings ?? DEFAULT_COMPACTION_SETTINGS, body.preparation);
|
|
277
1812
|
if (!preparationResult.ok) {
|
|
@@ -280,166 +1815,641 @@ function prepareSessionCompact(body) {
|
|
|
280
1815
|
if (!preparationResult.value) {
|
|
281
1816
|
return { status: 400, body: { error: "Nothing to compact" } };
|
|
282
1817
|
}
|
|
1818
|
+
let extensionCompaction;
|
|
1819
|
+
if (body.extensionCompaction !== undefined) {
|
|
1820
|
+
try {
|
|
1821
|
+
extensionCompaction = normalizeExtensionCompaction(body.extensionCompaction, entries);
|
|
1822
|
+
}
|
|
1823
|
+
catch (error) {
|
|
1824
|
+
return { status: 400, body: { error: error instanceof Error ? error.message : String(error) } };
|
|
1825
|
+
}
|
|
1826
|
+
}
|
|
283
1827
|
const options = body.options ?? {};
|
|
284
1828
|
return {
|
|
285
|
-
session,
|
|
286
1829
|
preparation: preparationResult.value,
|
|
1830
|
+
extensionCompaction,
|
|
287
1831
|
options,
|
|
1832
|
+
baseStaticContextHash: session.staticContextHash,
|
|
1833
|
+
baseTreeHash: session.treeHash,
|
|
1834
|
+
baseEntryCount: session.entries.length,
|
|
1835
|
+
baseLeafId: session.leafId,
|
|
1836
|
+
baseRevision: session.revision,
|
|
288
1837
|
};
|
|
289
1838
|
}
|
|
290
|
-
|
|
291
|
-
const result = await compact(prepared.preparation, createRequestModels(body.model, prepared.options), body.model, body.customInstructions, undefined, prepared.options.reasoning);
|
|
292
|
-
if (!result.ok) {
|
|
293
|
-
return { status: 500, body: { error: result.error.message } };
|
|
294
|
-
}
|
|
295
|
-
const baseTreeHash = prepared.session.treeHash;
|
|
296
|
-
const baseEntryCount = prepared.session.entries.length;
|
|
297
|
-
const compaction = result.value;
|
|
1839
|
+
function createCompactionCommitPlan(body, prepared, requestHash, compaction) {
|
|
298
1840
|
const firstKeptEntryId = compaction.firstKeptEntryId;
|
|
299
1841
|
if (!firstKeptEntryId) {
|
|
300
|
-
return {
|
|
1842
|
+
return createCompactTerminalPlan({ sessionId: body.sessionId, operationId: body.operationId, requestHash }, createCompactFailureResponse({ sessionId: body.sessionId, operationId: body.operationId, requestHash }, 500, "Compaction result is missing firstKeptEntryId", "terminal"));
|
|
301
1843
|
}
|
|
302
|
-
const
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
compaction: normalizedCompaction,
|
|
311
|
-
compactionEntry,
|
|
312
|
-
...sessionResponseBody(updatedSession),
|
|
313
|
-
staticContext: updatedSession.staticContext,
|
|
314
|
-
treePatch: {
|
|
315
|
-
baseTreeHash,
|
|
316
|
-
entriesFrom: baseEntryCount,
|
|
317
|
-
entries: [compactionEntry],
|
|
318
|
-
leafId: updatedSession.leafId,
|
|
319
|
-
revision: updatedSession.revision,
|
|
320
|
-
},
|
|
321
|
-
},
|
|
322
|
-
};
|
|
1844
|
+
const currentSession = getSession(body.sessionId);
|
|
1845
|
+
if (!currentSession ||
|
|
1846
|
+
currentSession.staticContextHash !== prepared.baseStaticContextHash ||
|
|
1847
|
+
currentSession.treeHash !== prepared.baseTreeHash ||
|
|
1848
|
+
currentSession.entries.length !== prepared.baseEntryCount ||
|
|
1849
|
+
currentSession.leafId !== prepared.baseLeafId ||
|
|
1850
|
+
currentSession.revision !== prepared.baseRevision) {
|
|
1851
|
+
return createCompactTerminalPlan({ sessionId: body.sessionId, operationId: body.operationId, requestHash }, createCompactFailureResponse({ sessionId: body.sessionId, operationId: body.operationId, requestHash }, 409, "Session changed while compaction was running; the stale compaction was not committed", "terminal"));
|
|
323
1852
|
}
|
|
324
|
-
|
|
1853
|
+
const operation = {
|
|
1854
|
+
version: 1,
|
|
1855
|
+
operationId: body.operationId,
|
|
1856
|
+
requestHash,
|
|
1857
|
+
baseStaticContextHash: prepared.baseStaticContextHash,
|
|
1858
|
+
baseTreeHash: prepared.baseTreeHash,
|
|
1859
|
+
baseEntryCount: prepared.baseEntryCount,
|
|
1860
|
+
baseLeafId: prepared.baseLeafId,
|
|
1861
|
+
baseRevision: prepared.baseRevision,
|
|
1862
|
+
};
|
|
1863
|
+
const compactionEntry = {
|
|
1864
|
+
type: "compaction",
|
|
1865
|
+
id: randomUUID(),
|
|
1866
|
+
parentId: prepared.baseLeafId,
|
|
1867
|
+
timestamp: new Date().toISOString(),
|
|
1868
|
+
summary: compaction.summary,
|
|
1869
|
+
firstKeptEntryId,
|
|
1870
|
+
tokensBefore: compaction.tokensBefore,
|
|
1871
|
+
...(compaction.retainedTail !== undefined ? { retainedTail: compaction.retainedTail } : {}),
|
|
1872
|
+
...(compaction.details !== undefined ? { details: compaction.details } : {}),
|
|
1873
|
+
...(compaction.usage !== undefined ? { usage: compaction.usage } : {}),
|
|
1874
|
+
...("fromHook" in compaction && compaction.fromHook !== undefined ? { fromHook: compaction.fromHook } : {}),
|
|
1875
|
+
piServerCompactOperation: operation,
|
|
1876
|
+
};
|
|
1877
|
+
const updatedTreeHash = appendPiServerTreeHash(prepared.baseTreeHash, hashPiServerTreeEntry(compactionEntry));
|
|
1878
|
+
const updatedRevision = prepared.baseRevision + 1;
|
|
1879
|
+
const response = {
|
|
325
1880
|
status: 200,
|
|
326
1881
|
body: {
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
1882
|
+
protocolVersion: PI_SERVER_PROTOCOL_VERSION,
|
|
1883
|
+
sessionId: body.sessionId,
|
|
1884
|
+
operationId: body.operationId,
|
|
1885
|
+
requestHash,
|
|
1886
|
+
treePatch: {
|
|
1887
|
+
baseStaticContextHash: prepared.baseStaticContextHash,
|
|
1888
|
+
baseTreeHash: prepared.baseTreeHash,
|
|
1889
|
+
baseEntryCount: prepared.baseEntryCount,
|
|
1890
|
+
baseLeafId: prepared.baseLeafId,
|
|
1891
|
+
baseRevision: prepared.baseRevision,
|
|
1892
|
+
entriesFrom: prepared.baseEntryCount,
|
|
1893
|
+
entries: [compactionEntry],
|
|
1894
|
+
leafId: compactionEntry.id,
|
|
1895
|
+
revision: updatedRevision,
|
|
1896
|
+
treeHash: updatedTreeHash,
|
|
1897
|
+
},
|
|
334
1898
|
},
|
|
335
1899
|
};
|
|
1900
|
+
return createCompactTerminalPlan({ sessionId: body.sessionId, operationId: body.operationId, requestHash }, response, {
|
|
1901
|
+
base: {
|
|
1902
|
+
staticContextHash: prepared.baseStaticContextHash,
|
|
1903
|
+
treeHash: prepared.baseTreeHash,
|
|
1904
|
+
entryCount: prepared.baseEntryCount,
|
|
1905
|
+
leafId: prepared.baseLeafId,
|
|
1906
|
+
revision: prepared.baseRevision,
|
|
1907
|
+
},
|
|
1908
|
+
entry: compactionEntry,
|
|
1909
|
+
updatedTreeHash,
|
|
1910
|
+
updatedRevision,
|
|
1911
|
+
});
|
|
1912
|
+
}
|
|
1913
|
+
async function computeSessionCompaction(body, prepared, signal) {
|
|
1914
|
+
if (prepared.extensionCompaction)
|
|
1915
|
+
return prepared.extensionCompaction;
|
|
1916
|
+
const result = await compact(prepared.preparation, createRequestModels(body.model, prepared.options), body.model, body.customInstructions, signal, prepared.options.reasoning, body.retry);
|
|
1917
|
+
if (!result.ok) {
|
|
1918
|
+
return { status: 500, body: { error: result.error.message } };
|
|
1919
|
+
}
|
|
1920
|
+
return result.value;
|
|
336
1921
|
}
|
|
337
|
-
function
|
|
338
|
-
|
|
339
|
-
res.write(`data: ${JSON.stringify(body)}\n\n`);
|
|
1922
|
+
function encodeServerSentEvent(event, body) {
|
|
1923
|
+
return `event: ${event}\ndata: ${JSON.stringify(body)}\n\n`;
|
|
340
1924
|
}
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
1925
|
+
function attachCompactRunSubscriber(runtime, run, res, drainIdleTimeoutMs) {
|
|
1926
|
+
let detached = false;
|
|
1927
|
+
let resolveClosed;
|
|
1928
|
+
const closed = new Promise((resolve) => {
|
|
1929
|
+
resolveClosed = resolve;
|
|
346
1930
|
});
|
|
347
|
-
|
|
348
|
-
|
|
1931
|
+
const onClose = () => detach(false);
|
|
1932
|
+
const detach = (destroyResponse) => {
|
|
1933
|
+
if (detached)
|
|
1934
|
+
return;
|
|
1935
|
+
detached = true;
|
|
1936
|
+
res.off("close", onClose);
|
|
1937
|
+
run.subscribers.delete(res);
|
|
1938
|
+
releaseCompactRun(runtime, run);
|
|
1939
|
+
resolveClosed?.();
|
|
1940
|
+
if (destroyResponse && !res.destroyed)
|
|
1941
|
+
res.destroy();
|
|
1942
|
+
};
|
|
1943
|
+
run.subscribers.add(res);
|
|
1944
|
+
res.once("close", onClose);
|
|
1945
|
+
let writeTail = Promise.resolve(true);
|
|
1946
|
+
const write = (data) => {
|
|
1947
|
+
const writeResult = writeTail.then(async (previousWriteSucceeded) => {
|
|
1948
|
+
if (!previousWriteSucceeded || detached)
|
|
1949
|
+
return false;
|
|
1950
|
+
const succeeded = await writeWithStreamDrain(res, data, drainIdleTimeoutMs);
|
|
1951
|
+
if (!succeeded)
|
|
1952
|
+
detach(true);
|
|
1953
|
+
return succeeded;
|
|
1954
|
+
});
|
|
1955
|
+
writeTail = writeResult.catch(() => {
|
|
1956
|
+
detach(true);
|
|
1957
|
+
return false;
|
|
1958
|
+
});
|
|
1959
|
+
return writeResult;
|
|
1960
|
+
};
|
|
1961
|
+
return { closed, detach, write };
|
|
1962
|
+
}
|
|
1963
|
+
async function handleSessionCompactStream(runtime, run, res, drainIdleTimeoutMs) {
|
|
1964
|
+
try {
|
|
1965
|
+
res.writeHead(200, {
|
|
1966
|
+
"Content-Type": "text/event-stream",
|
|
1967
|
+
"Cache-Control": "no-cache",
|
|
1968
|
+
Connection: "keep-alive",
|
|
1969
|
+
});
|
|
1970
|
+
res.flushHeaders();
|
|
1971
|
+
}
|
|
1972
|
+
catch {
|
|
1973
|
+
res.destroy();
|
|
1974
|
+
return;
|
|
1975
|
+
}
|
|
1976
|
+
const subscriber = attachCompactRunSubscriber(runtime, run, res, drainIdleTimeoutMs);
|
|
1977
|
+
if (!(await subscriber.write(STREAM_HEARTBEAT)))
|
|
1978
|
+
return;
|
|
349
1979
|
const heartbeat = setInterval(() => {
|
|
350
|
-
|
|
351
|
-
res.write(STREAM_HEARTBEAT);
|
|
352
|
-
}
|
|
1980
|
+
void subscriber.write(STREAM_HEARTBEAT);
|
|
353
1981
|
}, STREAM_HEARTBEAT_INTERVAL_MS);
|
|
354
1982
|
heartbeat.unref();
|
|
355
1983
|
try {
|
|
356
|
-
const
|
|
357
|
-
|
|
1984
|
+
const outcome = await Promise.race([
|
|
1985
|
+
run.promise.then((result) => ({ type: "result", result })),
|
|
1986
|
+
subscriber.closed.then(() => ({ type: "closed" })),
|
|
1987
|
+
]);
|
|
1988
|
+
if (outcome.type === "closed")
|
|
1989
|
+
return;
|
|
1990
|
+
clearInterval(heartbeat);
|
|
1991
|
+
await subscriber.write(encodeServerSentEvent(outcome.result.status >= 400 ? "error" : "result", outcome.result.body));
|
|
1992
|
+
}
|
|
1993
|
+
finally {
|
|
1994
|
+
clearInterval(heartbeat);
|
|
1995
|
+
subscriber.detach(false);
|
|
1996
|
+
if (!res.destroyed && !res.writableEnded) {
|
|
1997
|
+
res.end();
|
|
1998
|
+
}
|
|
1999
|
+
}
|
|
2000
|
+
}
|
|
2001
|
+
async function handleSessionCompactJsonStream(runtime, run, res, drainIdleTimeoutMs) {
|
|
2002
|
+
try {
|
|
2003
|
+
res.writeHead(200, {
|
|
2004
|
+
"Content-Type": "application/json",
|
|
2005
|
+
"Cache-Control": "no-cache",
|
|
2006
|
+
Connection: "keep-alive",
|
|
2007
|
+
});
|
|
2008
|
+
res.flushHeaders();
|
|
2009
|
+
}
|
|
2010
|
+
catch {
|
|
2011
|
+
res.destroy();
|
|
2012
|
+
return;
|
|
358
2013
|
}
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
2014
|
+
const subscriber = attachCompactRunSubscriber(runtime, run, res, drainIdleTimeoutMs);
|
|
2015
|
+
if (!(await subscriber.write(JSON_HEARTBEAT)))
|
|
2016
|
+
return;
|
|
2017
|
+
const heartbeat = setInterval(() => {
|
|
2018
|
+
void subscriber.write(JSON_HEARTBEAT);
|
|
2019
|
+
}, STREAM_HEARTBEAT_INTERVAL_MS);
|
|
2020
|
+
heartbeat.unref();
|
|
2021
|
+
try {
|
|
2022
|
+
const outcome = await Promise.race([
|
|
2023
|
+
run.promise.then((result) => ({ type: "result", result })),
|
|
2024
|
+
subscriber.closed.then(() => ({ type: "closed" })),
|
|
2025
|
+
]);
|
|
2026
|
+
if (outcome.type === "closed")
|
|
2027
|
+
return;
|
|
2028
|
+
clearInterval(heartbeat);
|
|
2029
|
+
await subscriber.write(JSON.stringify(outcome.result.body));
|
|
362
2030
|
}
|
|
363
2031
|
finally {
|
|
364
2032
|
clearInterval(heartbeat);
|
|
365
|
-
|
|
2033
|
+
subscriber.detach(false);
|
|
2034
|
+
if (!res.destroyed && !res.writableEnded) {
|
|
2035
|
+
res.end();
|
|
2036
|
+
}
|
|
2037
|
+
}
|
|
2038
|
+
}
|
|
2039
|
+
function compactNotStartedResponse(body, requestHash, response) {
|
|
2040
|
+
const error = typeof response.body.error === "string" ? response.body.error : "Compaction request was rejected";
|
|
2041
|
+
return createCompactFailureResponse({ sessionId: body.sessionId, operationId: body.operationId, requestHash }, response.status, error, "not_started");
|
|
2042
|
+
}
|
|
2043
|
+
function preflightCompactionCapacity(config, body, prepared, requestHash) {
|
|
2044
|
+
try {
|
|
2045
|
+
const probePlan = createCompactionCommitPlan(body, prepared, requestHash, {
|
|
2046
|
+
summary: "",
|
|
2047
|
+
firstKeptEntryId: prepared.preparation.firstKeptEntryId,
|
|
2048
|
+
tokensBefore: 0,
|
|
2049
|
+
});
|
|
2050
|
+
if (!probePlan.commit) {
|
|
2051
|
+
throw new Error("Failed to construct the minimum compaction persistence probe");
|
|
2052
|
+
}
|
|
2053
|
+
preflightSessionCapacityMutation(body.sessionId, {
|
|
2054
|
+
content: {
|
|
2055
|
+
kind: "append_entries",
|
|
2056
|
+
entries: [probePlan.commit.entry],
|
|
2057
|
+
leafId: probePlan.commit.entry.id,
|
|
2058
|
+
},
|
|
2059
|
+
});
|
|
2060
|
+
preflightCompactTerminalPlan(config, probePlan);
|
|
2061
|
+
}
|
|
2062
|
+
catch (error) {
|
|
2063
|
+
if (!(error instanceof SessionCapacityError) && !(error instanceof SessionPersistenceCapacityError)) {
|
|
2064
|
+
throw error;
|
|
2065
|
+
}
|
|
2066
|
+
return createCompactCapacityFailureResponse({ sessionId: body.sessionId, operationId: body.operationId, requestHash }, error, "not_started");
|
|
2067
|
+
}
|
|
2068
|
+
}
|
|
2069
|
+
async function handleSessionCompact(config, runtime, mutations, body, res) {
|
|
2070
|
+
if (body.protocolVersion !== PI_SERVER_PROTOCOL_VERSION) {
|
|
2071
|
+
sendJson(res, 400, {
|
|
2072
|
+
error: `Unsupported pi-server protocol version: ${body.protocolVersion}`,
|
|
2073
|
+
protocolVersion: PI_SERVER_PROTOCOL_VERSION,
|
|
2074
|
+
});
|
|
2075
|
+
return;
|
|
2076
|
+
}
|
|
2077
|
+
if (!body.sessionId || !body.operationId) {
|
|
2078
|
+
sendJson(res, 400, { error: "sessionId and operationId are required" });
|
|
2079
|
+
return;
|
|
2080
|
+
}
|
|
2081
|
+
if (typeof body.baseStaticContextHash !== "string" ||
|
|
2082
|
+
(body.baseStaticContextHash !== "" && !/^[a-f0-9]{64}$/u.test(body.baseStaticContextHash)) ||
|
|
2083
|
+
typeof body.baseTreeHash !== "string" ||
|
|
2084
|
+
!/^[a-f0-9]{64}$/u.test(body.baseTreeHash) ||
|
|
2085
|
+
!Number.isSafeInteger(body.baseEntryCount) ||
|
|
2086
|
+
body.baseEntryCount < 0 ||
|
|
2087
|
+
(body.baseLeafId !== null && (typeof body.baseLeafId !== "string" || body.baseLeafId.length === 0)) ||
|
|
2088
|
+
!Number.isSafeInteger(body.baseRevision) ||
|
|
2089
|
+
body.baseRevision < 0) {
|
|
2090
|
+
sendJson(res, 400, { error: "A valid complete compaction base identity is required" });
|
|
2091
|
+
return;
|
|
2092
|
+
}
|
|
2093
|
+
if (typeof body.streamResponse !== "boolean") {
|
|
2094
|
+
sendJson(res, 400, { error: "streamResponse is required" });
|
|
2095
|
+
return;
|
|
2096
|
+
}
|
|
2097
|
+
const requestHash = hashCompactRequest(body);
|
|
2098
|
+
const parityError = providerExecutionParityError(body.model, body.providerExecutionFingerprint);
|
|
2099
|
+
if (parityError) {
|
|
2100
|
+
const response = createCompactFailureResponse({ sessionId: body.sessionId, operationId: body.operationId, requestHash }, 409, parityError, "not_started");
|
|
2101
|
+
sendJson(res, response.status, response.body);
|
|
2102
|
+
return;
|
|
2103
|
+
}
|
|
2104
|
+
const action = await enqueueCompactSetup(runtime, async () => {
|
|
2105
|
+
if (runtime.closing)
|
|
2106
|
+
return { kind: "closing" };
|
|
2107
|
+
await runtime.store.prune();
|
|
2108
|
+
const existingState = await runtime.store.get(body.sessionId, body.operationId);
|
|
2109
|
+
if (existingState) {
|
|
2110
|
+
if (existingState.requestMac !== requestHash) {
|
|
2111
|
+
return {
|
|
2112
|
+
kind: "response",
|
|
2113
|
+
response: createCompactFailureResponse({ sessionId: body.sessionId, operationId: body.operationId, requestHash }, 409, "operationId is already bound to a different compaction request", "not_started"),
|
|
2114
|
+
};
|
|
2115
|
+
}
|
|
2116
|
+
if (existingState.status !== "running") {
|
|
2117
|
+
const plan = compactTerminalPlanFromState(existingState);
|
|
2118
|
+
await enqueueSessionStoreMutation(mutations, body.sessionId, () => applyCompactTerminalPlan(config, plan));
|
|
2119
|
+
const replay = {
|
|
2120
|
+
sessionId: body.sessionId,
|
|
2121
|
+
operationId: body.operationId,
|
|
2122
|
+
requestHash,
|
|
2123
|
+
status: "settled",
|
|
2124
|
+
result: plan.response,
|
|
2125
|
+
promise: Promise.resolve(plan.response),
|
|
2126
|
+
abortController: new AbortController(),
|
|
2127
|
+
subscribers: new Set(),
|
|
2128
|
+
};
|
|
2129
|
+
return { kind: "run", run: replay };
|
|
2130
|
+
}
|
|
2131
|
+
const activeRun = runtime.activeRuns.get(compactRunKey(body.sessionId, body.operationId));
|
|
2132
|
+
if (!activeRun) {
|
|
2133
|
+
throw new StreamRunStateError("Durable running compaction has no active provider execution");
|
|
2134
|
+
}
|
|
2135
|
+
return { kind: "run", run: activeRun };
|
|
2136
|
+
}
|
|
2137
|
+
return enqueueSessionMutation(mutations, body.sessionId, async () => {
|
|
2138
|
+
const prepared = prepareSessionCompact(body);
|
|
2139
|
+
if ("status" in prepared) {
|
|
2140
|
+
return {
|
|
2141
|
+
kind: "response",
|
|
2142
|
+
response: compactNotStartedResponse(body, requestHash, prepared),
|
|
2143
|
+
};
|
|
2144
|
+
}
|
|
2145
|
+
const capacityFailure = preflightCompactionCapacity(config, body, prepared, requestHash);
|
|
2146
|
+
if (capacityFailure) {
|
|
2147
|
+
return { kind: "response", response: capacityFailure };
|
|
2148
|
+
}
|
|
2149
|
+
const state = await runtime.store.begin({
|
|
2150
|
+
sessionId: body.sessionId,
|
|
2151
|
+
runId: body.operationId,
|
|
2152
|
+
requestMac: requestHash,
|
|
2153
|
+
});
|
|
2154
|
+
if (state.status !== "running") {
|
|
2155
|
+
throw new StreamRunStateError("New durable compaction did not begin in running state");
|
|
2156
|
+
}
|
|
2157
|
+
const abortController = new AbortController();
|
|
2158
|
+
let resolveTerminal;
|
|
2159
|
+
let rejectTerminal;
|
|
2160
|
+
const terminal = new Promise((resolve, reject) => {
|
|
2161
|
+
resolveTerminal = resolve;
|
|
2162
|
+
rejectTerminal = reject;
|
|
2163
|
+
});
|
|
2164
|
+
const run = {
|
|
2165
|
+
sessionId: body.sessionId,
|
|
2166
|
+
operationId: body.operationId,
|
|
2167
|
+
requestHash,
|
|
2168
|
+
status: "running",
|
|
2169
|
+
promise: terminal,
|
|
2170
|
+
abortController,
|
|
2171
|
+
subscribers: new Set(),
|
|
2172
|
+
resolveTerminal,
|
|
2173
|
+
rejectTerminal,
|
|
2174
|
+
};
|
|
2175
|
+
runtime.activeRuns.set(compactRunKey(body.sessionId, body.operationId), run);
|
|
2176
|
+
void (async () => {
|
|
2177
|
+
let computed;
|
|
2178
|
+
try {
|
|
2179
|
+
computed = await computeSessionCompaction(body, prepared, abortController.signal);
|
|
2180
|
+
}
|
|
2181
|
+
catch (error) {
|
|
2182
|
+
computed = {
|
|
2183
|
+
status: 500,
|
|
2184
|
+
body: { error: error instanceof Error ? error.message : String(error) },
|
|
2185
|
+
};
|
|
2186
|
+
}
|
|
2187
|
+
if ("status" in computed) {
|
|
2188
|
+
const error = typeof computed.body.error === "string" ? computed.body.error : "Compaction provider failed";
|
|
2189
|
+
return settleCompactRun(config, runtime, mutations, run, createCompactTerminalPlan(run, createCompactFailureResponse(run, computed.status, error, "terminal")), "failed");
|
|
2190
|
+
}
|
|
2191
|
+
if (abortController.signal.aborted) {
|
|
2192
|
+
return settleCompactRun(config, runtime, mutations, run, createCompactTerminalPlan(run, createCompactFailureResponse(run, 499, "Compaction aborted by user", "terminal")), "failed");
|
|
2193
|
+
}
|
|
2194
|
+
return settleCompactRun(config, runtime, mutations, run, () => createCompactionCommitPlan(body, prepared, requestHash, computed));
|
|
2195
|
+
})().catch(() => undefined);
|
|
2196
|
+
return { kind: "run", run };
|
|
2197
|
+
});
|
|
2198
|
+
});
|
|
2199
|
+
if (action.kind === "closing") {
|
|
2200
|
+
const response = createCompactFailureResponse({ sessionId: body.sessionId, operationId: body.operationId, requestHash }, 503, "pi-server is closing", "not_started");
|
|
2201
|
+
sendJson(res, response.status, response.body);
|
|
2202
|
+
return;
|
|
2203
|
+
}
|
|
2204
|
+
if (action.kind === "response") {
|
|
2205
|
+
sendJson(res, action.response.status, action.response.body);
|
|
2206
|
+
return;
|
|
2207
|
+
}
|
|
2208
|
+
const run = action.run;
|
|
2209
|
+
if (run.status === "running" && run.subscribers.size >= COMPACT_RUN_MAX_SUBSCRIBERS) {
|
|
2210
|
+
sendJson(res, 503, { error: "Too many subscribers for this compaction" });
|
|
2211
|
+
return;
|
|
2212
|
+
}
|
|
2213
|
+
if (body.streamResponse) {
|
|
2214
|
+
await handleSessionCompactStream(runtime, run, res, runtime.drainIdleTimeoutMs);
|
|
2215
|
+
return;
|
|
2216
|
+
}
|
|
2217
|
+
await handleSessionCompactJsonStream(runtime, run, res, runtime.drainIdleTimeoutMs);
|
|
2218
|
+
}
|
|
2219
|
+
async function handleSessionCompactRecovery(config, runtime, mutations, sessionId, operationId, requestHash, res) {
|
|
2220
|
+
if (!requestHash || !/^[a-f0-9]{64}$/u.test(requestHash)) {
|
|
2221
|
+
sendJson(res, 400, { error: "requestHash is required and must be a lowercase SHA-256 digest" });
|
|
2222
|
+
return;
|
|
2223
|
+
}
|
|
2224
|
+
const action = await enqueueCompactSetup(runtime, async () => {
|
|
2225
|
+
const state = await runtime.store.get(sessionId, operationId);
|
|
2226
|
+
if (!state)
|
|
2227
|
+
return { kind: "missing" };
|
|
2228
|
+
if (state.requestMac !== requestHash) {
|
|
2229
|
+
throw new StreamRunConflictError("operationId is bound to a different compaction request");
|
|
2230
|
+
}
|
|
2231
|
+
if (state.status === "running") {
|
|
2232
|
+
const active = runtime.activeRuns.get(compactRunKey(sessionId, operationId));
|
|
2233
|
+
if (!active) {
|
|
2234
|
+
throw new StreamRunStateError("Durable running compaction has no active provider execution");
|
|
2235
|
+
}
|
|
2236
|
+
return { kind: "run", run: active };
|
|
2237
|
+
}
|
|
2238
|
+
const plan = compactTerminalPlanFromState(state);
|
|
2239
|
+
await enqueueSessionStoreMutation(mutations, sessionId, () => applyCompactTerminalPlan(config, plan));
|
|
2240
|
+
const replay = {
|
|
2241
|
+
sessionId,
|
|
2242
|
+
operationId,
|
|
2243
|
+
requestHash,
|
|
2244
|
+
status: "settled",
|
|
2245
|
+
result: plan.response,
|
|
2246
|
+
promise: Promise.resolve(plan.response),
|
|
2247
|
+
abortController: new AbortController(),
|
|
2248
|
+
subscribers: new Set(),
|
|
2249
|
+
};
|
|
2250
|
+
return { kind: "run", run: replay };
|
|
2251
|
+
});
|
|
2252
|
+
if (action.kind === "missing") {
|
|
2253
|
+
sendJson(res, 404, { error: "compaction operation not found" });
|
|
2254
|
+
return;
|
|
2255
|
+
}
|
|
2256
|
+
await handleSessionCompactStream(runtime, action.run, res, runtime.drainIdleTimeoutMs);
|
|
2257
|
+
}
|
|
2258
|
+
async function handleAcknowledgeSessionCompact(config, runtime, mutations, body, res) {
|
|
2259
|
+
if (!body.sessionId || !body.operationId || !body.requestHash) {
|
|
2260
|
+
sendJson(res, 400, { error: "sessionId, operationId, and requestHash are required" });
|
|
2261
|
+
return;
|
|
2262
|
+
}
|
|
2263
|
+
const state = await enqueueCompactSetup(runtime, async () => {
|
|
2264
|
+
const current = await runtime.store.get(body.sessionId, body.operationId);
|
|
2265
|
+
if (!current)
|
|
2266
|
+
return undefined;
|
|
2267
|
+
if (current.requestMac !== body.requestHash) {
|
|
2268
|
+
throw new StreamRunConflictError("operationId is bound to a different compaction request");
|
|
2269
|
+
}
|
|
2270
|
+
if (current.status === "running") {
|
|
2271
|
+
throw new StreamRunStateError("Cannot acknowledge a running compaction");
|
|
2272
|
+
}
|
|
2273
|
+
const plan = compactTerminalPlanFromState(current);
|
|
2274
|
+
await enqueueSessionStoreMutation(mutations, body.sessionId, () => applyCompactTerminalPlan(config, plan));
|
|
2275
|
+
const acknowledged = await runtime.store.acknowledge(body.sessionId, body.operationId);
|
|
2276
|
+
await runtime.store.prune();
|
|
2277
|
+
return acknowledged;
|
|
2278
|
+
});
|
|
2279
|
+
if (!state) {
|
|
2280
|
+
sendJson(res, 404, { error: "compaction operation not found" });
|
|
2281
|
+
return;
|
|
366
2282
|
}
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
2283
|
+
sendJson(res, 200, {
|
|
2284
|
+
acknowledged: true,
|
|
2285
|
+
sessionId: state.sessionId,
|
|
2286
|
+
operationId: state.runId,
|
|
2287
|
+
requestHash: state.requestMac,
|
|
2288
|
+
status: state.status,
|
|
2289
|
+
acknowledgedAt: state.acknowledgedAt,
|
|
373
2290
|
});
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
const
|
|
377
|
-
|
|
378
|
-
|
|
2291
|
+
}
|
|
2292
|
+
async function reserveSessionCompactRunsForDeletion(runtime, sessionId) {
|
|
2293
|
+
const activeRuns = [...runtime.activeRuns.values()].filter((run) => run.sessionId === sessionId);
|
|
2294
|
+
const reserved = [];
|
|
2295
|
+
for (const run of activeRuns) {
|
|
2296
|
+
run.abortController.abort();
|
|
2297
|
+
if (run.settlement) {
|
|
2298
|
+
await run.settlement;
|
|
2299
|
+
continue;
|
|
379
2300
|
}
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
const
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
finally {
|
|
391
|
-
clearInterval(heartbeat);
|
|
392
|
-
res.end();
|
|
2301
|
+
const plan = createCompactTerminalPlan(run, createCompactFailureResponse(run, 499, "Session deleted while compaction was running", "terminal"));
|
|
2302
|
+
let resolve;
|
|
2303
|
+
let reject;
|
|
2304
|
+
const settlement = new Promise((resolvePromise, rejectPromise) => {
|
|
2305
|
+
resolve = resolvePromise;
|
|
2306
|
+
reject = rejectPromise;
|
|
2307
|
+
});
|
|
2308
|
+
run.settlement = settlement;
|
|
2309
|
+
void settlement.catch(() => undefined);
|
|
2310
|
+
reserved.push({ run, plan, resolve, reject });
|
|
393
2311
|
}
|
|
2312
|
+
return { activeRuns, reserved };
|
|
394
2313
|
}
|
|
395
|
-
async function
|
|
396
|
-
const
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
2314
|
+
async function deleteSessionCompactRunsLocked(runtime, sessionId, activeRuns, reserved) {
|
|
2315
|
+
for (const { run, plan, resolve } of reserved) {
|
|
2316
|
+
const state = await runtime.store.settle({
|
|
2317
|
+
sessionId: run.sessionId,
|
|
2318
|
+
runId: run.operationId,
|
|
2319
|
+
status: "failed",
|
|
2320
|
+
event: JSON.stringify(plan),
|
|
2321
|
+
result: plan,
|
|
2322
|
+
errorMessage: plan.response.body.error,
|
|
2323
|
+
});
|
|
2324
|
+
const durablePlan = compactTerminalPlanFromState(state);
|
|
2325
|
+
run.status = "settled";
|
|
2326
|
+
run.result = durablePlan.response;
|
|
2327
|
+
run.resolveTerminal?.(durablePlan.response);
|
|
2328
|
+
run.resolveTerminal = undefined;
|
|
2329
|
+
run.rejectTerminal = undefined;
|
|
2330
|
+
resolve(durablePlan.response);
|
|
400
2331
|
}
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
2332
|
+
for (const run of activeRuns) {
|
|
2333
|
+
for (const response of run.subscribers)
|
|
2334
|
+
response.destroy();
|
|
2335
|
+
runtime.activeRuns.delete(compactRunKey(run.sessionId, run.operationId));
|
|
2336
|
+
}
|
|
2337
|
+
const states = await runtime.store.list();
|
|
2338
|
+
for (const state of states) {
|
|
2339
|
+
if (state.sessionId !== sessionId)
|
|
2340
|
+
continue;
|
|
2341
|
+
await runtime.store.delete(state.sessionId, state.runId);
|
|
404
2342
|
}
|
|
405
|
-
await handleSessionCompactJsonStream(config, body, prepared, res);
|
|
406
2343
|
}
|
|
407
2344
|
function handleDropLastAssistantError(config, body, res) {
|
|
408
2345
|
if (!body.sessionId) {
|
|
409
2346
|
sendJson(res, 400, { error: "sessionId is required" });
|
|
410
2347
|
return;
|
|
411
2348
|
}
|
|
412
|
-
|
|
413
|
-
const
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
2349
|
+
let dropped = false;
|
|
2350
|
+
const existingSession = getSession(body.sessionId);
|
|
2351
|
+
const session = existingSession
|
|
2352
|
+
? mutateAndPersistSession(config, body.sessionId, () => {
|
|
2353
|
+
dropped = dropLastAssistantError(body.sessionId);
|
|
2354
|
+
const updated = getSession(body.sessionId);
|
|
2355
|
+
if (!updated)
|
|
2356
|
+
throw new Error(`Session ${body.sessionId} disappeared while dropping its assistant error`);
|
|
2357
|
+
return updated;
|
|
2358
|
+
})
|
|
2359
|
+
: undefined;
|
|
417
2360
|
const messageCount = session?.messages.length ?? 0;
|
|
418
2361
|
sendJson(res, 200, { success: true, dropped, messageCount });
|
|
419
2362
|
}
|
|
420
|
-
function handleSessionHistory(sessionId, from, entriesFrom, revision, baseTreeHash, res) {
|
|
2363
|
+
function handleSessionHistory(sessionId, from, entriesFrom, revision, baseTreeHash, protocolVersion, res) {
|
|
421
2364
|
const session = getSession(sessionId);
|
|
422
2365
|
if (!session) {
|
|
423
2366
|
sendJson(res, 404, { error: "session not found" });
|
|
424
2367
|
return;
|
|
425
2368
|
}
|
|
2369
|
+
if (protocolVersion !== undefined && protocolVersion !== PI_SERVER_PROTOCOL_VERSION) {
|
|
2370
|
+
sendJson(res, 400, {
|
|
2371
|
+
error: `Unsupported pi-server protocol version: ${protocolVersion}`,
|
|
2372
|
+
protocolVersion: PI_SERVER_PROTOCOL_VERSION,
|
|
2373
|
+
});
|
|
2374
|
+
return;
|
|
2375
|
+
}
|
|
426
2376
|
const baseMessageCount = from ?? 0;
|
|
427
2377
|
if (entriesFrom !== undefined &&
|
|
428
2378
|
entriesFrom <= session.entries.length &&
|
|
429
2379
|
(revision === undefined || revision <= session.revision) &&
|
|
430
2380
|
(baseTreeHash === undefined || baseTreeHash === session.prefixHashes[entriesFrom])) {
|
|
431
|
-
sendJson(res, 200,
|
|
2381
|
+
sendJson(res, 200, protocolVersion === PI_SERVER_PROTOCOL_VERSION
|
|
2382
|
+
? sessionTreePatchV2ResponseBody(session, entriesFrom, revision)
|
|
2383
|
+
: sessionTreePatchResponseBody(session, baseMessageCount, entriesFrom, revision));
|
|
432
2384
|
return;
|
|
433
2385
|
}
|
|
434
|
-
sendJson(res, 200,
|
|
2386
|
+
sendJson(res, 200, protocolVersion === PI_SERVER_PROTOCOL_VERSION
|
|
2387
|
+
? sessionHistoryV2FullResponseBody(session)
|
|
2388
|
+
: sessionHistoryFullResponseBody(session, baseMessageCount));
|
|
435
2389
|
}
|
|
436
|
-
function handleSessionRun(sessionId, runId, res) {
|
|
437
|
-
const
|
|
438
|
-
|
|
2390
|
+
async function handleSessionRun(runtime, sessionId, runId, res) {
|
|
2391
|
+
const state = await enqueueStreamRunSetup(runtime, async () => {
|
|
2392
|
+
await runtime.store.prune();
|
|
2393
|
+
return runtime.store.get(sessionId, runId);
|
|
2394
|
+
});
|
|
2395
|
+
if (!state) {
|
|
439
2396
|
sendJson(res, 404, { error: "run not found" });
|
|
440
2397
|
return;
|
|
441
2398
|
}
|
|
442
|
-
sendJson(res, 200,
|
|
2399
|
+
sendJson(res, 200, {
|
|
2400
|
+
sessionId: state.sessionId,
|
|
2401
|
+
runId: state.runId,
|
|
2402
|
+
status: state.status,
|
|
2403
|
+
requestMac: state.requestMac,
|
|
2404
|
+
nextSeq: state.nextSeq,
|
|
2405
|
+
message: messageFromDurableState(state),
|
|
2406
|
+
errorMessage: state.terminal?.errorMessage,
|
|
2407
|
+
acknowledgedAt: state.acknowledgedAt,
|
|
2408
|
+
});
|
|
2409
|
+
}
|
|
2410
|
+
async function handleSessionRunEvents(runtime, sessionId, runId, from, res) {
|
|
2411
|
+
const action = await enqueueStreamRunSetup(runtime, async () => {
|
|
2412
|
+
await runtime.store.prune();
|
|
2413
|
+
const state = await runtime.store.get(sessionId, runId);
|
|
2414
|
+
if (!state)
|
|
2415
|
+
return { kind: "missing" };
|
|
2416
|
+
if (from > state.nextSeq) {
|
|
2417
|
+
return { kind: "invalid_cursor", nextSeq: state.nextSeq };
|
|
2418
|
+
}
|
|
2419
|
+
if (state.status !== "running") {
|
|
2420
|
+
return { kind: "replay", promise: replayDurableStreamRun(runtime, state, res, from) };
|
|
2421
|
+
}
|
|
2422
|
+
const run = runtime.activeRuns.get(runKey(sessionId, runId));
|
|
2423
|
+
if (!run) {
|
|
2424
|
+
return { kind: "unavailable" };
|
|
2425
|
+
}
|
|
2426
|
+
if (run.subscribers.size >= STREAM_RUN_MAX_SUBSCRIBERS) {
|
|
2427
|
+
return { kind: "subscriber_limit" };
|
|
2428
|
+
}
|
|
2429
|
+
if (!beginStreamResponse(res))
|
|
2430
|
+
return { kind: "attached" };
|
|
2431
|
+
subscribeToStreamRun(runtime, run, res, from);
|
|
2432
|
+
return { kind: "attached" };
|
|
2433
|
+
});
|
|
2434
|
+
switch (action.kind) {
|
|
2435
|
+
case "missing":
|
|
2436
|
+
sendJson(res, 404, { error: "run not found" });
|
|
2437
|
+
return;
|
|
2438
|
+
case "invalid_cursor":
|
|
2439
|
+
sendJson(res, 409, { error: `from ${from} exceeds durable nextSeq ${action.nextSeq}` });
|
|
2440
|
+
return;
|
|
2441
|
+
case "unavailable":
|
|
2442
|
+
sendJson(res, 409, { error: "Durable running stream has no active provider execution" });
|
|
2443
|
+
return;
|
|
2444
|
+
case "subscriber_limit":
|
|
2445
|
+
sendJson(res, 503, { error: "Too many subscribers for this stream run" });
|
|
2446
|
+
return;
|
|
2447
|
+
case "replay":
|
|
2448
|
+
await action.promise;
|
|
2449
|
+
return;
|
|
2450
|
+
case "attached":
|
|
2451
|
+
return;
|
|
2452
|
+
}
|
|
443
2453
|
}
|
|
444
2454
|
export function buildStreamContext(session, body) {
|
|
445
2455
|
const messages = body.contextOverlay ?? [...session.messages, ...(body.ephemeralMessages ?? [])];
|
|
@@ -449,22 +2459,65 @@ export function buildStreamContext(session, body) {
|
|
|
449
2459
|
tools: session.staticContext?.tools,
|
|
450
2460
|
};
|
|
451
2461
|
}
|
|
452
|
-
function
|
|
453
|
-
if (
|
|
454
|
-
sendJson(res, 400, { error: "sessionId is required" });
|
|
2462
|
+
async function terminateStreamRunWithProviderError(runtime, run, model, errorMessage) {
|
|
2463
|
+
if (run.status !== "running")
|
|
455
2464
|
return;
|
|
2465
|
+
logProviderStreamFailure(run, model, errorMessage);
|
|
2466
|
+
await persistTerminalStreamRun(runtime, run, createSyntheticAssistantMessage(run, errorMessage, "error"));
|
|
2467
|
+
}
|
|
2468
|
+
function executeStreamRun(runtime, run, model, context, streamOptions) {
|
|
2469
|
+
let stream;
|
|
2470
|
+
try {
|
|
2471
|
+
stream = streamSimple(model, context, { ...streamOptions, signal: run.abortController.signal });
|
|
456
2472
|
}
|
|
457
|
-
|
|
458
|
-
|
|
2473
|
+
catch (error) {
|
|
2474
|
+
void terminateStreamRunWithProviderError(runtime, run, model, error instanceof Error ? error.message : String(error)).catch((persistenceError) => {
|
|
2475
|
+
void handleStreamRunPersistenceFailure(runtime, run, persistenceError);
|
|
2476
|
+
});
|
|
459
2477
|
return;
|
|
460
2478
|
}
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
2479
|
+
void (async () => {
|
|
2480
|
+
try {
|
|
2481
|
+
for await (const event of stream) {
|
|
2482
|
+
if (run.status !== "running")
|
|
2483
|
+
return;
|
|
2484
|
+
if (event.type === "error") {
|
|
2485
|
+
if (event.reason === "error") {
|
|
2486
|
+
logProviderStreamFailure(run, model, event.error.errorMessage ?? event.reason);
|
|
2487
|
+
}
|
|
2488
|
+
await persistTerminalStreamRun(runtime, run, withStreamRunDiagnostic(run, event.error));
|
|
2489
|
+
return;
|
|
2490
|
+
}
|
|
2491
|
+
if (event.type === "done") {
|
|
2492
|
+
await persistTerminalStreamRun(runtime, run, withStreamRunDiagnostic(run, event.message));
|
|
2493
|
+
return;
|
|
2494
|
+
}
|
|
2495
|
+
const proxyEvent = toProxyEvent(event, run);
|
|
2496
|
+
if (proxyEvent)
|
|
2497
|
+
await queueStreamRunEvent(runtime, run, proxyEvent);
|
|
2498
|
+
}
|
|
2499
|
+
await terminateStreamRunWithProviderError(runtime, run, model, "Provider stream ended without a terminal event");
|
|
2500
|
+
}
|
|
2501
|
+
catch (error) {
|
|
2502
|
+
if (run.shuttingDown || run.status !== "running")
|
|
2503
|
+
return;
|
|
2504
|
+
if (run.persistenceError) {
|
|
2505
|
+
await handleStreamRunPersistenceFailure(runtime, run, run.persistenceError);
|
|
2506
|
+
return;
|
|
2507
|
+
}
|
|
2508
|
+
await terminateStreamRunWithProviderError(runtime, run, model, error instanceof Error ? error.message : String(error));
|
|
2509
|
+
}
|
|
2510
|
+
})().catch((error) => {
|
|
2511
|
+
void handleStreamRunPersistenceFailure(runtime, run, error);
|
|
2512
|
+
});
|
|
2513
|
+
}
|
|
2514
|
+
async function handleStream(config, runtime, mutations, body, res) {
|
|
2515
|
+
if (!body.sessionId) {
|
|
2516
|
+
sendJson(res, 400, { error: "sessionId is required" });
|
|
2517
|
+
return;
|
|
465
2518
|
}
|
|
466
|
-
if (
|
|
467
|
-
sendJson(res, 400, { error: "
|
|
2519
|
+
if (body.runMode !== undefined && body.runMode !== "main-durable" && body.runMode !== "auxiliary-transient") {
|
|
2520
|
+
sendJson(res, 400, { error: "runMode must be main-durable or auxiliary-transient" });
|
|
468
2521
|
return;
|
|
469
2522
|
}
|
|
470
2523
|
if (body.ephemeralMessages !== undefined && !Array.isArray(body.ephemeralMessages)) {
|
|
@@ -475,71 +2528,259 @@ function handleStream(config, body, res) {
|
|
|
475
2528
|
sendJson(res, 400, { error: "contextOverlay must be an array" });
|
|
476
2529
|
return;
|
|
477
2530
|
}
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
const { model: resolvedModel, options: streamOptions } = resolveStreamOptions(config, body.model, body);
|
|
481
|
-
res.writeHead(200, {
|
|
482
|
-
"Content-Type": "text/event-stream",
|
|
483
|
-
"Cache-Control": "no-cache",
|
|
484
|
-
Connection: "keep-alive",
|
|
485
|
-
});
|
|
486
|
-
res.flushHeaders();
|
|
487
|
-
res.write(STREAM_HEARTBEAT);
|
|
488
|
-
if (existingRun?.status === "completed") {
|
|
489
|
-
for (const event of existingRun.events) {
|
|
490
|
-
res.write(encodeProxyEvent(event));
|
|
491
|
-
}
|
|
492
|
-
res.end();
|
|
2531
|
+
if (body.baseStaticContextHash === undefined) {
|
|
2532
|
+
sendJson(res, 400, { error: "baseStaticContextHash is required" });
|
|
493
2533
|
return;
|
|
494
2534
|
}
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
}, STREAM_HEARTBEAT_INTERVAL_MS);
|
|
501
|
-
heartbeat.unref();
|
|
502
|
-
let stream;
|
|
503
|
-
try {
|
|
504
|
-
stream = streamSimple(resolvedModel, context, streamOptions);
|
|
2535
|
+
if (body.baseStaticContextHash !== "" && !/^[a-f0-9]{64}$/.test(body.baseStaticContextHash)) {
|
|
2536
|
+
sendJson(res, 400, {
|
|
2537
|
+
error: "baseStaticContextHash must be empty or a lowercase 64-character SHA-256 digest",
|
|
2538
|
+
});
|
|
2539
|
+
return;
|
|
505
2540
|
}
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
509
|
-
failStreamRun(run, message);
|
|
510
|
-
res.write(encodeErrorEvent(message));
|
|
511
|
-
res.end();
|
|
2541
|
+
if (body.baseRevision === undefined) {
|
|
2542
|
+
sendJson(res, 400, { error: "baseRevision is required" });
|
|
512
2543
|
return;
|
|
513
2544
|
}
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
2545
|
+
if (!Number.isSafeInteger(body.baseRevision) || body.baseRevision < 0) {
|
|
2546
|
+
sendJson(res, 400, { error: "baseRevision must be a non-negative safe integer" });
|
|
2547
|
+
return;
|
|
2548
|
+
}
|
|
2549
|
+
const baseStaticContextHash = body.baseStaticContextHash;
|
|
2550
|
+
const baseRevision = body.baseRevision;
|
|
2551
|
+
const cursor = body.eventCursor ?? 0;
|
|
2552
|
+
if (!Number.isSafeInteger(cursor) || cursor < 0) {
|
|
2553
|
+
sendJson(res, 400, { error: "eventCursor must be a non-negative safe integer" });
|
|
2554
|
+
return;
|
|
2555
|
+
}
|
|
2556
|
+
if (body.baseTreeHash !== undefined && !/^[a-f0-9]{64}$/.test(body.baseTreeHash)) {
|
|
2557
|
+
sendJson(res, 400, { error: "baseTreeHash must be a lowercase 64-character SHA-256 digest" });
|
|
2558
|
+
return;
|
|
2559
|
+
}
|
|
2560
|
+
if (body.baseEntryCount !== undefined && (!Number.isSafeInteger(body.baseEntryCount) || body.baseEntryCount < 0)) {
|
|
2561
|
+
sendJson(res, 400, { error: "baseEntryCount must be a non-negative safe integer" });
|
|
2562
|
+
return;
|
|
2563
|
+
}
|
|
2564
|
+
if (body.baseLeafId !== undefined &&
|
|
2565
|
+
body.baseLeafId !== null &&
|
|
2566
|
+
(typeof body.baseLeafId !== "string" || body.baseLeafId.length === 0)) {
|
|
2567
|
+
sendJson(res, 400, { error: "baseLeafId must be a non-empty string or null" });
|
|
2568
|
+
return;
|
|
2569
|
+
}
|
|
2570
|
+
const parityError = providerExecutionParityError(body.model, body.providerExecutionFingerprint);
|
|
2571
|
+
if (parityError) {
|
|
2572
|
+
sendJson(res, 409, { error: parityError });
|
|
2573
|
+
return;
|
|
2574
|
+
}
|
|
2575
|
+
const requestHash = hashStreamRequest(body);
|
|
2576
|
+
const storageRunId = body.runId ?? `anonymous:${randomUUID()}`;
|
|
2577
|
+
const deletionVersion = sessionDeletionFences.get(mutations)?.versions.get(body.sessionId) ?? 0;
|
|
2578
|
+
const action = await enqueueStreamRunSetup(runtime, () => enqueueSessionMutation(mutations, body.sessionId, async () => {
|
|
2579
|
+
if (runtime.closing)
|
|
2580
|
+
return { kind: "closing" };
|
|
2581
|
+
await runtime.store.prune();
|
|
2582
|
+
const existingState = body.runId ? await runtime.store.get(body.sessionId, body.runId) : undefined;
|
|
2583
|
+
if (existingState) {
|
|
2584
|
+
if (existingState.requestMac !== requestHash) {
|
|
2585
|
+
throw new StreamRunConflictError("runId is already bound to a different stream request");
|
|
2586
|
+
}
|
|
2587
|
+
if (cursor > existingState.nextSeq) {
|
|
2588
|
+
throw new StreamRunConflictError(`eventCursor ${cursor} exceeds durable nextSeq ${existingState.nextSeq}`);
|
|
2589
|
+
}
|
|
2590
|
+
if (existingState.status !== "running") {
|
|
2591
|
+
return {
|
|
2592
|
+
kind: "replay",
|
|
2593
|
+
promise: replayDurableStreamRun(runtime, existingState, res, cursor),
|
|
2594
|
+
};
|
|
528
2595
|
}
|
|
2596
|
+
const existingRun = runtime.activeRuns.get(runKey(body.sessionId, body.runId));
|
|
2597
|
+
if (!existingRun) {
|
|
2598
|
+
throw new StreamRunStateError("Durable running stream has no active provider execution");
|
|
2599
|
+
}
|
|
2600
|
+
if (existingRun.subscribers.size >= STREAM_RUN_MAX_SUBSCRIBERS) {
|
|
2601
|
+
return { kind: "subscriber_limit" };
|
|
2602
|
+
}
|
|
2603
|
+
if (beginStreamResponse(res)) {
|
|
2604
|
+
subscribeToStreamRun(runtime, existingRun, res, cursor);
|
|
2605
|
+
}
|
|
2606
|
+
return { kind: "attached" };
|
|
529
2607
|
}
|
|
530
|
-
|
|
531
|
-
|
|
2608
|
+
const existingSession = getSession(body.sessionId);
|
|
2609
|
+
if (!existingSession?.staticContext && !body.staticContext) {
|
|
2610
|
+
return { kind: "missing_context" };
|
|
532
2611
|
}
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
2612
|
+
if (body.baseTreeHash === undefined || body.baseEntryCount === undefined || body.baseLeafId === undefined) {
|
|
2613
|
+
return { kind: "missing_tree_base" };
|
|
2614
|
+
}
|
|
2615
|
+
if (!matchesPiServerStreamBase({
|
|
2616
|
+
baseStaticContextHash,
|
|
2617
|
+
baseTreeHash: body.baseTreeHash,
|
|
2618
|
+
baseEntryCount: body.baseEntryCount,
|
|
2619
|
+
baseLeafId: body.baseLeafId,
|
|
2620
|
+
baseRevision,
|
|
2621
|
+
}, existingSession
|
|
2622
|
+
? sessionIdentity(existingSession)
|
|
2623
|
+
: {
|
|
2624
|
+
staticContextHash: "",
|
|
2625
|
+
treeHash: PI_SERVER_EMPTY_TREE_HASH,
|
|
2626
|
+
entryCount: 0,
|
|
2627
|
+
leafId: null,
|
|
2628
|
+
revision: 0,
|
|
2629
|
+
})) {
|
|
2630
|
+
return { kind: "session_conflict" };
|
|
2631
|
+
}
|
|
2632
|
+
const staticContext = body.staticContext;
|
|
2633
|
+
const session = existingSession && !staticContext
|
|
2634
|
+
? existingSession
|
|
2635
|
+
: await enqueueSessionCapacityMutation(() => mutateAndPersistSession(config, body.sessionId, () => applySessionMutation(body.sessionId, staticContext ? { staticContext } : {})));
|
|
2636
|
+
const expectedIdentity = streamBaseIdentity(session);
|
|
2637
|
+
const resolved = resolveStreamOptions(config, body.model, body);
|
|
2638
|
+
const state = await runtime.store.begin({
|
|
2639
|
+
sessionId: body.sessionId,
|
|
2640
|
+
runId: storageRunId,
|
|
2641
|
+
requestMac: requestHash,
|
|
2642
|
+
});
|
|
2643
|
+
const currentSession = getSession(body.sessionId);
|
|
2644
|
+
if (!currentSession || !matchesPiServerStreamBase(expectedIdentity, sessionIdentity(currentSession))) {
|
|
2645
|
+
await runtime.store.rollbackUnstartedBegin({
|
|
2646
|
+
sessionId: body.sessionId,
|
|
2647
|
+
runId: storageRunId,
|
|
2648
|
+
requestMac: requestHash,
|
|
2649
|
+
});
|
|
2650
|
+
return { kind: "session_conflict" };
|
|
2651
|
+
}
|
|
2652
|
+
const context = buildStreamContext(currentSession, body);
|
|
2653
|
+
const run = createActiveStreamRun(state, body.runId, resolved.model);
|
|
2654
|
+
runtime.activeRuns.set(runKey(body.sessionId, storageRunId), run);
|
|
2655
|
+
if (beginStreamResponse(res)) {
|
|
2656
|
+
subscribeToStreamRun(runtime, run, res, cursor);
|
|
2657
|
+
}
|
|
2658
|
+
return { kind: "start", run, context, resolved };
|
|
2659
|
+
}, false, deletionVersion));
|
|
2660
|
+
switch (action.kind) {
|
|
2661
|
+
case "closing":
|
|
2662
|
+
sendJson(res, 503, { error: "pi-server is closing" });
|
|
2663
|
+
return;
|
|
2664
|
+
case "subscriber_limit":
|
|
2665
|
+
sendJson(res, 503, { error: "Too many subscribers for this stream run" });
|
|
2666
|
+
return;
|
|
2667
|
+
case "missing_context":
|
|
2668
|
+
sendJson(res, 400, { error: "Session has no static context. Initialize with /api/session/init first." });
|
|
2669
|
+
return;
|
|
2670
|
+
case "missing_tree_base":
|
|
2671
|
+
sendJson(res, 400, { error: "baseTreeHash, baseEntryCount, and baseLeafId are required for a new run" });
|
|
2672
|
+
return;
|
|
2673
|
+
case "session_conflict":
|
|
2674
|
+
sendJson(res, 409, {
|
|
2675
|
+
error: "Stream request base identity does not match the current server session",
|
|
2676
|
+
});
|
|
2677
|
+
return;
|
|
2678
|
+
case "replay":
|
|
2679
|
+
await action.promise;
|
|
2680
|
+
return;
|
|
2681
|
+
case "attached":
|
|
2682
|
+
return;
|
|
2683
|
+
case "start":
|
|
2684
|
+
executeStreamRun(runtime, action.run, action.resolved.model, action.context, action.resolved.options);
|
|
2685
|
+
return;
|
|
2686
|
+
}
|
|
2687
|
+
}
|
|
2688
|
+
async function handleAcknowledgeSessionRun(runtime, body, res) {
|
|
2689
|
+
if (!body.sessionId || !body.runId) {
|
|
2690
|
+
sendJson(res, 400, { error: "sessionId and runId are required" });
|
|
2691
|
+
return;
|
|
2692
|
+
}
|
|
2693
|
+
const state = await enqueueStreamRunSetup(runtime, () => runtime.store.acknowledge(body.sessionId, body.runId));
|
|
2694
|
+
sendJson(res, 200, {
|
|
2695
|
+
acknowledged: true,
|
|
2696
|
+
sessionId: state.sessionId,
|
|
2697
|
+
runId: state.runId,
|
|
2698
|
+
requestMac: state.requestMac,
|
|
2699
|
+
status: state.status,
|
|
2700
|
+
acknowledgedAt: state.acknowledgedAt,
|
|
540
2701
|
});
|
|
541
2702
|
}
|
|
542
|
-
async function
|
|
2703
|
+
async function deleteSessionStreamRunsLocked(runtime, sessionId) {
|
|
2704
|
+
const activeRuns = [...runtime.activeRuns.values()].filter((run) => run.sessionId === sessionId);
|
|
2705
|
+
for (const run of activeRuns) {
|
|
2706
|
+
await abortStreamRun(runtime, run, "Session deleted while provider stream was running");
|
|
2707
|
+
for (const res of [...run.subscribers.keys()]) {
|
|
2708
|
+
detachStreamRunSubscriber(runtime, run, res);
|
|
2709
|
+
res.destroy();
|
|
2710
|
+
}
|
|
2711
|
+
runtime.activeRuns.delete(runKey(run.sessionId, run.storageRunId));
|
|
2712
|
+
}
|
|
2713
|
+
const replays = [...(runtime.replayReaders.get(sessionId) ?? [])];
|
|
2714
|
+
for (const replay of replays) {
|
|
2715
|
+
replay.abortController.abort();
|
|
2716
|
+
replay.response.destroy();
|
|
2717
|
+
}
|
|
2718
|
+
const pending = [
|
|
2719
|
+
...(runtime.subscriberPumps.get(sessionId) ?? []),
|
|
2720
|
+
...replays.flatMap((replay) => (replay.promise ? [replay.promise] : [])),
|
|
2721
|
+
];
|
|
2722
|
+
if (pending.length > 0) {
|
|
2723
|
+
await Promise.allSettled(pending);
|
|
2724
|
+
}
|
|
2725
|
+
const states = await runtime.store.list();
|
|
2726
|
+
for (const state of states) {
|
|
2727
|
+
if (state.sessionId !== sessionId)
|
|
2728
|
+
continue;
|
|
2729
|
+
await runtime.store.delete(state.sessionId, state.runId);
|
|
2730
|
+
}
|
|
2731
|
+
}
|
|
2732
|
+
async function deleteSessionAtomically(config, deletePersistence, streamRuntime, compactRuntime, sessionMutations, sessionId) {
|
|
2733
|
+
const deletionState = sessionDeletionFenceState(sessionMutations);
|
|
2734
|
+
advanceSessionDeletionVersion(deletionState, sessionId);
|
|
2735
|
+
deletionState.active.set(sessionId, (deletionState.active.get(sessionId) ?? 0) + 1);
|
|
2736
|
+
let fenceClosed = false;
|
|
2737
|
+
const closeFence = () => {
|
|
2738
|
+
if (fenceClosed)
|
|
2739
|
+
return;
|
|
2740
|
+
fenceClosed = true;
|
|
2741
|
+
const remaining = (deletionState.active.get(sessionId) ?? 1) - 1;
|
|
2742
|
+
if (remaining === 0)
|
|
2743
|
+
deletionState.active.delete(sessionId);
|
|
2744
|
+
else
|
|
2745
|
+
deletionState.active.set(sessionId, remaining);
|
|
2746
|
+
advanceSessionDeletionVersion(deletionState, sessionId);
|
|
2747
|
+
cleanupSessionDeletionFence(sessionMutations, sessionId);
|
|
2748
|
+
};
|
|
2749
|
+
let reserved = [];
|
|
2750
|
+
try {
|
|
2751
|
+
await enqueueStreamRunSetup(streamRuntime, () => enqueueCompactSetup(compactRuntime, async () => {
|
|
2752
|
+
const compactDeletion = await reserveSessionCompactRunsForDeletion(compactRuntime, sessionId);
|
|
2753
|
+
reserved = compactDeletion.reserved;
|
|
2754
|
+
await enqueueSessionStoreMutation(sessionMutations, sessionId, async () => {
|
|
2755
|
+
try {
|
|
2756
|
+
await deleteSessionStreamRunsLocked(streamRuntime, sessionId);
|
|
2757
|
+
await deleteSessionCompactRunsLocked(compactRuntime, sessionId, compactDeletion.activeRuns, compactDeletion.reserved);
|
|
2758
|
+
deleteSessionFromStore(sessionId);
|
|
2759
|
+
deletePersistence(config.sessionStoreDir, sessionId);
|
|
2760
|
+
}
|
|
2761
|
+
finally {
|
|
2762
|
+
closeFence();
|
|
2763
|
+
}
|
|
2764
|
+
}, true);
|
|
2765
|
+
}));
|
|
2766
|
+
}
|
|
2767
|
+
catch (error) {
|
|
2768
|
+
for (const { run, reject } of reserved) {
|
|
2769
|
+
if (run.status !== "running")
|
|
2770
|
+
continue;
|
|
2771
|
+
run.rejectTerminal?.(error);
|
|
2772
|
+
run.resolveTerminal = undefined;
|
|
2773
|
+
run.rejectTerminal = undefined;
|
|
2774
|
+
reject(error);
|
|
2775
|
+
}
|
|
2776
|
+
failStopServerPersistence(currentServerPersistenceRuntime(), error);
|
|
2777
|
+
throw error;
|
|
2778
|
+
}
|
|
2779
|
+
finally {
|
|
2780
|
+
closeFence();
|
|
2781
|
+
}
|
|
2782
|
+
}
|
|
2783
|
+
async function handlePostRequest(config, streamRuntime, compactRuntime, sessionMutations, pathname, body, res) {
|
|
543
2784
|
if (pathname === "/api/receive") {
|
|
544
2785
|
try {
|
|
545
2786
|
sendJson(res, 200, receiveUpload(config.uploadDir, body));
|
|
@@ -552,174 +2793,599 @@ async function handlePostRequest(config, pathname, body, res) {
|
|
|
552
2793
|
return true;
|
|
553
2794
|
}
|
|
554
2795
|
if (pathname === "/api/session/init") {
|
|
555
|
-
|
|
2796
|
+
const request = body;
|
|
2797
|
+
await enqueueSessionStoreMutation(sessionMutations, request.sessionId, () => handleSessionInit(config, request, res));
|
|
556
2798
|
return true;
|
|
557
2799
|
}
|
|
558
2800
|
if (pathname === "/api/session/update") {
|
|
559
|
-
|
|
2801
|
+
const request = body;
|
|
2802
|
+
await enqueueSessionStoreMutation(sessionMutations, request.sessionId, () => handleSessionUpdate(config, request, res));
|
|
560
2803
|
return true;
|
|
561
2804
|
}
|
|
562
2805
|
if (pathname === "/api/session/sync") {
|
|
563
|
-
|
|
2806
|
+
const request = body;
|
|
2807
|
+
await enqueueSessionStoreMutation(sessionMutations, request.sessionId, () => handleSessionSync(config, request, res));
|
|
564
2808
|
return true;
|
|
565
2809
|
}
|
|
566
2810
|
if (pathname === "/api/session/append") {
|
|
567
|
-
|
|
2811
|
+
const request = body;
|
|
2812
|
+
await enqueueSessionStoreMutation(sessionMutations, request.sessionId, () => handleSessionAppend(config, request, res));
|
|
568
2813
|
return true;
|
|
569
2814
|
}
|
|
570
2815
|
if (pathname === "/api/session/tree/sync") {
|
|
571
|
-
|
|
2816
|
+
const request = body;
|
|
2817
|
+
await enqueueSessionStoreMutation(sessionMutations, request.sessionId, () => handleSessionTreeSync(config, request, res));
|
|
572
2818
|
return true;
|
|
573
2819
|
}
|
|
574
2820
|
if (pathname === "/api/session/tree/append") {
|
|
575
|
-
|
|
2821
|
+
const request = body;
|
|
2822
|
+
await enqueueSessionStoreMutation(sessionMutations, request.sessionId, () => handleSessionTreeAppend(config, request, res));
|
|
576
2823
|
return true;
|
|
577
2824
|
}
|
|
578
2825
|
if (pathname === "/api/session/tree/switch") {
|
|
579
|
-
|
|
2826
|
+
const request = body;
|
|
2827
|
+
await enqueueSessionStoreMutation(sessionMutations, request.sessionId, () => handleSessionTreeSwitch(config, request, res));
|
|
580
2828
|
return true;
|
|
581
2829
|
}
|
|
582
2830
|
if (pathname === "/api/session/drop-last-assistant-error") {
|
|
583
|
-
|
|
2831
|
+
const request = body;
|
|
2832
|
+
await enqueueSessionStoreMutation(sessionMutations, request.sessionId, () => handleDropLastAssistantError(config, request, res));
|
|
2833
|
+
return true;
|
|
2834
|
+
}
|
|
2835
|
+
if (pathname === "/api/session/run/cancel") {
|
|
2836
|
+
await handleCancelSessionRun(streamRuntime, body, res);
|
|
2837
|
+
return true;
|
|
2838
|
+
}
|
|
2839
|
+
if (pathname === "/api/session/run/ack") {
|
|
2840
|
+
await handleAcknowledgeSessionRun(streamRuntime, body, res);
|
|
2841
|
+
return true;
|
|
2842
|
+
}
|
|
2843
|
+
if (pathname === "/api/session/compact/cancel") {
|
|
2844
|
+
await handleCancelSessionCompact(config, compactRuntime, sessionMutations, body, res);
|
|
2845
|
+
return true;
|
|
2846
|
+
}
|
|
2847
|
+
if (pathname === "/api/session/compact/ack") {
|
|
2848
|
+
await handleAcknowledgeSessionCompact(config, compactRuntime, sessionMutations, body, res);
|
|
584
2849
|
return true;
|
|
585
2850
|
}
|
|
586
2851
|
if (pathname === "/api/session/compact") {
|
|
587
|
-
await handleSessionCompact(config, body, res);
|
|
2852
|
+
await handleSessionCompact(config, compactRuntime, sessionMutations, body, res);
|
|
588
2853
|
return true;
|
|
589
2854
|
}
|
|
590
2855
|
if (pathname === "/api/stream") {
|
|
591
|
-
handleStream(config, body, res);
|
|
2856
|
+
await handleStream(config, streamRuntime, sessionMutations, body, res);
|
|
592
2857
|
return true;
|
|
593
2858
|
}
|
|
594
2859
|
return false;
|
|
595
2860
|
}
|
|
596
|
-
|
|
2861
|
+
function createOwnedPiServer(configOverride, ownership) {
|
|
597
2862
|
const config = loadConfig(configOverride);
|
|
598
|
-
|
|
599
|
-
const
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
2863
|
+
const drainIdleTimeoutMs = positiveServerOption(configOverride?.streamDrainIdleTimeoutMs, "PI_SERVER_STREAM_DRAIN_IDLE_TIMEOUT_MS", DEFAULT_STREAM_DRAIN_IDLE_TIMEOUT_MS, "streamDrainIdleTimeoutMs");
|
|
2864
|
+
const requestBodyNoProgressTimeoutMs = positiveServerOption(configOverride?.requestBodyNoProgressTimeoutMs, "PI_SERVER_REQUEST_BODY_NO_PROGRESS_TIMEOUT_MS", REQUEST_BODY_NO_PROGRESS_TIMEOUT_MS, "requestBodyNoProgressTimeoutMs");
|
|
2865
|
+
const streamRunMaxFrameBytes = positiveServerOption(configOverride?.streamRunMaxFrameBytes, "PI_SERVER_STREAM_RUN_MAX_FRAME_BYTES", STREAM_RUN_MAX_FRAME_BYTES, "streamRunMaxFrameBytes");
|
|
2866
|
+
const streamRunMaxBytes = positiveServerOption(configOverride?.streamRunMaxBytes, "PI_SERVER_STREAM_RUN_MAX_BYTES", STREAM_RUN_MAX_BYTES, "streamRunMaxBytes");
|
|
2867
|
+
const streamRunMaxTotalBytes = positiveServerOption(configOverride?.streamRunMaxTotalBytes, "PI_SERVER_STREAM_RUN_MAX_TOTAL_BYTES", STREAM_RUN_MAX_TOTAL_BYTES, "streamRunMaxTotalBytes");
|
|
2868
|
+
const streamRunTerminalReserveBytes = positiveServerOption(configOverride?.streamRunTerminalReserveBytes, "PI_SERVER_STREAM_RUN_TERMINAL_RESERVE_BYTES", STREAM_RUN_TERMINAL_RESERVE_BYTES, "streamRunTerminalReserveBytes");
|
|
2869
|
+
const streamRunIoNoProgressTimeoutMs = positiveServerOption(configOverride?.streamRunIoNoProgressTimeoutMs, "PI_SERVER_STREAM_RUN_IO_NO_PROGRESS_TIMEOUT_MS", 120_000, "streamRunIoNoProgressTimeoutMs");
|
|
2870
|
+
const compactRunMaxBytes = positiveServerOption(configOverride?.compactRunMaxBytes, "PI_SERVER_COMPACT_RUN_MAX_BYTES", COMPACT_RUN_MAX_BYTES, "compactRunMaxBytes");
|
|
2871
|
+
const compactRunMaxTotalBytes = positiveServerOption(configOverride?.compactRunMaxTotalBytes, "PI_SERVER_COMPACT_RUN_MAX_TOTAL_BYTES", COMPACT_RUN_MAX_TOTAL_BYTES, "compactRunMaxTotalBytes");
|
|
2872
|
+
const compactRunTerminalReserveBytes = positiveServerOption(configOverride?.compactRunTerminalReserveBytes, "PI_SERVER_COMPACT_RUN_TERMINAL_RESERVE_BYTES", COMPACT_RUN_TERMINAL_RESERVE_BYTES, "compactRunTerminalReserveBytes");
|
|
2873
|
+
const compactRunIoNoProgressTimeoutMs = positiveServerOption(configOverride?.compactRunIoNoProgressTimeoutMs, "PI_SERVER_COMPACT_RUN_IO_NO_PROGRESS_TIMEOUT_MS", 120_000, "compactRunIoNoProgressTimeoutMs");
|
|
2874
|
+
const sessionMaxEntries = positiveServerOption(configOverride?.sessionMaxEntries, "PI_SERVER_SESSION_MAX_ENTRIES", DEFAULT_SESSION_CAPACITY_LIMITS.maxEntriesPerSession, "sessionMaxEntries");
|
|
2875
|
+
const sessionMaxLogicalBytes = positiveServerOption(configOverride?.sessionMaxLogicalBytes, "PI_SERVER_SESSION_MAX_LOGICAL_BYTES", DEFAULT_SESSION_CAPACITY_LIMITS.maxLogicalBytesPerSession, "sessionMaxLogicalBytes");
|
|
2876
|
+
const sessionsMaxEntries = positiveServerOption(configOverride?.sessionsMaxEntries, "PI_SERVER_SESSIONS_MAX_ENTRIES", DEFAULT_SESSION_CAPACITY_LIMITS.maxAggregateEntries, "sessionsMaxEntries");
|
|
2877
|
+
const sessionsMaxLogicalBytes = positiveServerOption(configOverride?.sessionsMaxLogicalBytes, "PI_SERVER_SESSIONS_MAX_LOGICAL_BYTES", DEFAULT_SESSION_CAPACITY_LIMITS.maxAggregateLogicalBytes, "sessionsMaxLogicalBytes");
|
|
2878
|
+
const maxLoadedSessions = positiveServerOption(configOverride?.maxLoadedSessions, "PI_SERVER_MAX_LOADED_SESSIONS", DEFAULT_SESSION_CAPACITY_LIMITS.maxLoadedSessions, "maxLoadedSessions");
|
|
2879
|
+
configureSessionCapacityLimits({
|
|
2880
|
+
maxEntriesPerSession: sessionMaxEntries,
|
|
2881
|
+
maxLogicalBytesPerSession: sessionMaxLogicalBytes,
|
|
2882
|
+
maxAggregateEntries: sessionsMaxEntries,
|
|
2883
|
+
maxAggregateLogicalBytes: sessionsMaxLogicalBytes,
|
|
2884
|
+
maxLoadedSessions,
|
|
2885
|
+
});
|
|
2886
|
+
const persistenceRuntime = {
|
|
2887
|
+
onFatalPersistenceError: configOverride?.onFatalStreamPersistenceError,
|
|
2888
|
+
sessionFaultInjector: configOverride?.sessionPersistenceFaultInjector,
|
|
2889
|
+
};
|
|
2890
|
+
activeServerPersistenceRuntime = persistenceRuntime;
|
|
2891
|
+
let streamRuntime;
|
|
2892
|
+
const streamStore = new StreamRunPersistence({
|
|
2893
|
+
rootDir: join(config.sessionStoreDir, ".runs"),
|
|
2894
|
+
lockPath: join(config.sessionStoreDir, ".pi-server-owner.sqlite"),
|
|
2895
|
+
maxRuns: STREAM_RUN_MAX_RECORDS,
|
|
2896
|
+
maxFrameBytes: streamRunMaxFrameBytes,
|
|
2897
|
+
maxRunBytes: streamRunMaxBytes,
|
|
2898
|
+
maxTotalBytes: streamRunMaxTotalBytes,
|
|
2899
|
+
terminalReserveBytes: streamRunTerminalReserveBytes,
|
|
2900
|
+
maxBatchBytes: streamRunMaxFrameBytes,
|
|
2901
|
+
ioNoProgressTimeoutMs: streamRunIoNoProgressTimeoutMs,
|
|
2902
|
+
faultInjector: configOverride?.streamRunFaultInjector,
|
|
2903
|
+
restartFailureEvent: encodeProxyEvent(createStreamErrorEvent(STREAM_RUN_RESTART_ERROR_MESSAGE)),
|
|
2904
|
+
restartFailureMessage: STREAM_RUN_RESTART_ERROR_MESSAGE,
|
|
2905
|
+
restartFailureTerminal: createRestartFailureTerminal,
|
|
2906
|
+
onFatalError: (error) => failStopServerPersistence(persistenceRuntime, error),
|
|
2907
|
+
});
|
|
2908
|
+
streamRuntime = {
|
|
2909
|
+
store: streamStore,
|
|
2910
|
+
activeRuns: new Map(),
|
|
2911
|
+
setupQueue: Promise.resolve(),
|
|
2912
|
+
subscriberPumps: new Map(),
|
|
2913
|
+
replayReaders: new Map(),
|
|
2914
|
+
drainIdleTimeoutMs,
|
|
2915
|
+
closing: false,
|
|
2916
|
+
};
|
|
2917
|
+
persistenceRuntime.streamRuntime = streamRuntime;
|
|
2918
|
+
const sessionMutations = { queues: new Map() };
|
|
2919
|
+
let compactRuntime;
|
|
2920
|
+
const compactStore = new StreamRunPersistence({
|
|
2921
|
+
rootDir: join(config.sessionStoreDir, ".compactions"),
|
|
2922
|
+
lockPath: join(config.sessionStoreDir, ".pi-server-compact-owner.sqlite"),
|
|
2923
|
+
maxRuns: COMPACT_RUN_MAX_RECORDS,
|
|
2924
|
+
maxFrameBytes: COMPACT_RUN_MAX_FRAME_BYTES,
|
|
2925
|
+
maxRunBytes: compactRunMaxBytes,
|
|
2926
|
+
maxTotalBytes: compactRunMaxTotalBytes,
|
|
2927
|
+
terminalReserveBytes: compactRunTerminalReserveBytes,
|
|
2928
|
+
maxBatchBytes: COMPACT_RUN_MAX_FRAME_BYTES,
|
|
2929
|
+
ioNoProgressTimeoutMs: compactRunIoNoProgressTimeoutMs,
|
|
2930
|
+
faultInjector: configOverride?.compactRunFaultInjector,
|
|
2931
|
+
restartFailureMessage: "restart-unknown",
|
|
2932
|
+
restartFailureEvent: JSON.stringify({ error: "restart-unknown" }),
|
|
2933
|
+
restartFailureTerminal: createCompactRestartFailureTerminal,
|
|
2934
|
+
onFatalError: (error) => failStopServerPersistence(persistenceRuntime, error),
|
|
2935
|
+
});
|
|
2936
|
+
compactRuntime = {
|
|
2937
|
+
store: compactStore,
|
|
2938
|
+
activeRuns: new Map(),
|
|
2939
|
+
setupQueue: Promise.resolve(),
|
|
2940
|
+
drainIdleTimeoutMs,
|
|
2941
|
+
closing: false,
|
|
2942
|
+
};
|
|
2943
|
+
persistenceRuntime.compactRuntime = compactRuntime;
|
|
2944
|
+
let streamStoreInitialized = false;
|
|
2945
|
+
let compactStoreInitialized = false;
|
|
2946
|
+
const serverInitialization = (async () => {
|
|
2947
|
+
await streamStore.initialize();
|
|
2948
|
+
streamStoreInitialized = true;
|
|
2949
|
+
await compactStore.initialize();
|
|
2950
|
+
compactStoreInitialized = true;
|
|
2951
|
+
loadPersistedSessions(config.sessionStoreDir);
|
|
2952
|
+
const compactStates = await compactStore.list();
|
|
2953
|
+
for (const compactState of compactStates) {
|
|
2954
|
+
if (compactState.status === "running") {
|
|
2955
|
+
throw new StreamRunStateError("Compaction recovery left a durable operation running");
|
|
2956
|
+
}
|
|
2957
|
+
const state = await compactStore.get(compactState.sessionId, compactState.runId);
|
|
2958
|
+
if (!state) {
|
|
2959
|
+
throw new StreamRunCorruptionError(`${compactState.sessionId}/${compactState.runId}`, "compaction disappeared during startup recovery");
|
|
2960
|
+
}
|
|
2961
|
+
applyCompactTerminalPlan(config, compactTerminalPlanFromState(state));
|
|
615
2962
|
}
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
2963
|
+
})();
|
|
2964
|
+
const serverReady = serverInitialization.catch((error) => {
|
|
2965
|
+
failStopServerPersistence(persistenceRuntime, error);
|
|
2966
|
+
if (streamStoreInitialized && !compactStoreInitialized) {
|
|
2967
|
+
void streamStore.close().catch(() => undefined);
|
|
620
2968
|
}
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
2969
|
+
});
|
|
2970
|
+
const chunkTargetResponses = new Map();
|
|
2971
|
+
const server = createServer(async (req, res) => {
|
|
2972
|
+
try {
|
|
2973
|
+
await serverReady;
|
|
2974
|
+
const url = new URL(req.url ?? "/", `http://${config.host}:${config.port}`);
|
|
2975
|
+
if (persistenceRuntime.fatalError) {
|
|
2976
|
+
sendJson(res, 503, { status: "error", error: "durable operation persistence health failure" });
|
|
627
2977
|
return;
|
|
628
2978
|
}
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
if (entriesFrom !== undefined && (!Number.isInteger(entriesFrom) || entriesFrom < 0)) {
|
|
632
|
-
sendJson(res, 400, { error: "entriesFrom must be a non-negative integer" });
|
|
2979
|
+
if (req.method === "GET" && url.pathname === "/health") {
|
|
2980
|
+
sendJson(res, 200, { status: "ok" });
|
|
633
2981
|
return;
|
|
634
2982
|
}
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
2983
|
+
if (!authenticate(config, req)) {
|
|
2984
|
+
const body = req.method === "GET" && url.pathname === "/"
|
|
2985
|
+
? { error: "Unauthorized", version: PI_SERVER_VERSION }
|
|
2986
|
+
: { error: "Unauthorized" };
|
|
2987
|
+
sendJson(res, 401, body);
|
|
639
2988
|
return;
|
|
640
2989
|
}
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
const
|
|
648
|
-
|
|
649
|
-
|
|
2990
|
+
if (req.method === "GET" && url.pathname === "/api/sessions") {
|
|
2991
|
+
sendJson(res, 200, { sessions: listSessions() });
|
|
2992
|
+
return;
|
|
2993
|
+
}
|
|
2994
|
+
const runEventsMatch = /^\/api\/session\/([^/]+)\/runs\/([^/]+)\/events$/.exec(url.pathname);
|
|
2995
|
+
if (req.method === "GET" && runEventsMatch) {
|
|
2996
|
+
const fromParam = url.searchParams.get("from");
|
|
2997
|
+
const from = fromParam === null ? 0 : Number(fromParam);
|
|
2998
|
+
if (!Number.isSafeInteger(from) || from < 0) {
|
|
2999
|
+
sendJson(res, 400, { error: "from must be a non-negative safe integer" });
|
|
650
3000
|
return;
|
|
651
3001
|
}
|
|
652
|
-
await
|
|
3002
|
+
await handleSessionRunEvents(streamRuntime, decodeURIComponent(runEventsMatch[1]), decodeURIComponent(runEventsMatch[2]), from, res);
|
|
3003
|
+
return;
|
|
3004
|
+
}
|
|
3005
|
+
const runMatch = /^\/api\/session\/([^/]+)\/runs\/([^/]+)$/.exec(url.pathname);
|
|
3006
|
+
if (req.method === "GET" && runMatch) {
|
|
3007
|
+
await handleSessionRun(streamRuntime, decodeURIComponent(runMatch[1]), decodeURIComponent(runMatch[2]), res);
|
|
3008
|
+
return;
|
|
3009
|
+
}
|
|
3010
|
+
const compactMatch = /^\/api\/session\/([^/]+)\/compactions\/([^/]+)$/.exec(url.pathname);
|
|
3011
|
+
if (req.method === "GET" && compactMatch) {
|
|
3012
|
+
await handleSessionCompactRecovery(config, compactRuntime, sessionMutations, decodeURIComponent(compactMatch[1]), decodeURIComponent(compactMatch[2]), url.searchParams.get("requestHash") ?? undefined, res);
|
|
3013
|
+
return;
|
|
653
3014
|
}
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
3015
|
+
if (req.method === "GET" && url.pathname.startsWith("/api/session/") && url.pathname.endsWith("/history")) {
|
|
3016
|
+
const encodedSessionId = url.pathname.slice("/api/session/".length, -"/history".length);
|
|
3017
|
+
const fromParam = url.searchParams.get("from");
|
|
3018
|
+
const from = fromParam === null ? undefined : Number(fromParam);
|
|
3019
|
+
if (from !== undefined && (!Number.isInteger(from) || from < 0)) {
|
|
3020
|
+
sendJson(res, 400, { error: "from must be a non-negative integer" });
|
|
3021
|
+
return;
|
|
658
3022
|
}
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
3023
|
+
const entriesFromParam = url.searchParams.get("entriesFrom");
|
|
3024
|
+
const entriesFrom = entriesFromParam === null ? undefined : Number(entriesFromParam);
|
|
3025
|
+
if (entriesFrom !== undefined && (!Number.isInteger(entriesFrom) || entriesFrom < 0)) {
|
|
3026
|
+
sendJson(res, 400, { error: "entriesFrom must be a non-negative integer" });
|
|
3027
|
+
return;
|
|
662
3028
|
}
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
try {
|
|
668
|
-
const body = JSON.parse(await readBody(req));
|
|
669
|
-
if (await handlePostRequest(config, url.pathname, body, res))
|
|
3029
|
+
const revisionParam = url.searchParams.get("revision");
|
|
3030
|
+
const revision = revisionParam === null ? undefined : Number(revisionParam);
|
|
3031
|
+
if (revision !== undefined && (!Number.isInteger(revision) || revision < 0)) {
|
|
3032
|
+
sendJson(res, 400, { error: "revision must be a non-negative integer" });
|
|
670
3033
|
return;
|
|
3034
|
+
}
|
|
3035
|
+
const protocolVersionParam = url.searchParams.get("protocolVersion");
|
|
3036
|
+
const protocolVersion = protocolVersionParam === null ? undefined : Number(protocolVersionParam);
|
|
3037
|
+
if (protocolVersion !== undefined && !Number.isInteger(protocolVersion)) {
|
|
3038
|
+
sendJson(res, 400, { error: "protocolVersion must be an integer" });
|
|
3039
|
+
return;
|
|
3040
|
+
}
|
|
3041
|
+
handleSessionHistory(decodeURIComponent(encodedSessionId), from, entriesFrom, revision, url.searchParams.get("baseTreeHash") ?? undefined, protocolVersion, res);
|
|
3042
|
+
return;
|
|
671
3043
|
}
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
3044
|
+
if (req.method === "POST" && url.pathname === CHUNK_ENDPOINT) {
|
|
3045
|
+
try {
|
|
3046
|
+
const body = JSON.parse(await readBody(req, REQUEST_BODY_MAX_BYTES, requestBodyNoProgressTimeoutMs));
|
|
3047
|
+
const chunkResult = receiveRequestChunk(body);
|
|
3048
|
+
if (!chunkResult.complete) {
|
|
3049
|
+
res.setHeader(CHUNK_ACK_HEADER, "1");
|
|
3050
|
+
sendJson(res, 200, chunkResult.ack);
|
|
3051
|
+
return;
|
|
3052
|
+
}
|
|
3053
|
+
res.setHeader(CHUNK_ACK_HEADER, "0");
|
|
3054
|
+
const targetBody = JSON.parse(chunkResult.bodyJson);
|
|
3055
|
+
const targetUsesOwnJournal = chunkResult.target === "/api/stream" || chunkResult.target === "/api/session/compact";
|
|
3056
|
+
if (targetUsesOwnJournal) {
|
|
3057
|
+
const record = typeof targetBody === "object" && targetBody !== null
|
|
3058
|
+
? targetBody
|
|
3059
|
+
: undefined;
|
|
3060
|
+
const id = chunkResult.target === "/api/stream" ? record?.runId : record?.operationId;
|
|
3061
|
+
if (typeof id !== "string" || id.length === 0) {
|
|
3062
|
+
sendJson(res, 400, {
|
|
3063
|
+
error: `${chunkResult.target} chunk requests require an idempotency id`,
|
|
3064
|
+
});
|
|
3065
|
+
return;
|
|
3066
|
+
}
|
|
3067
|
+
}
|
|
3068
|
+
else if (chunkResult.replayed) {
|
|
3069
|
+
pruneChunkTargetResponses(chunkTargetResponses);
|
|
3070
|
+
const cachedResponse = chunkTargetResponses.get(chunkResult.requestId);
|
|
3071
|
+
if (!cachedResponse) {
|
|
3072
|
+
sendJson(res, 409, {
|
|
3073
|
+
error: "Chunk target response expired; reconcile target state before retrying",
|
|
3074
|
+
});
|
|
3075
|
+
return;
|
|
3076
|
+
}
|
|
3077
|
+
sendCapturedJson(res, cachedResponse);
|
|
3078
|
+
return;
|
|
3079
|
+
}
|
|
3080
|
+
else {
|
|
3081
|
+
jsonResponseCaptures.set(res, (response) => {
|
|
3082
|
+
cacheChunkTargetResponse(chunkTargetResponses, chunkResult.requestId, response);
|
|
3083
|
+
});
|
|
3084
|
+
}
|
|
3085
|
+
await handlePostRequest(config, streamRuntime, compactRuntime, sessionMutations, chunkResult.target, targetBody, res);
|
|
3086
|
+
if (!targetUsesOwnJournal && jsonResponseCaptures.has(res)) {
|
|
3087
|
+
throw new Error(`Chunk target ${chunkResult.target} ended without a JSON response`);
|
|
3088
|
+
}
|
|
676
3089
|
}
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
res.
|
|
3090
|
+
catch (err) {
|
|
3091
|
+
logRequestError(req, err);
|
|
3092
|
+
if (!res.headersSent) {
|
|
3093
|
+
sendJson(res, requestErrorStatus(err, 400), requestErrorBody(err));
|
|
3094
|
+
}
|
|
3095
|
+
else {
|
|
3096
|
+
res.write(encodeErrorEvent(err instanceof Error ? err.message : String(err)));
|
|
3097
|
+
res.end();
|
|
3098
|
+
}
|
|
3099
|
+
}
|
|
3100
|
+
finally {
|
|
3101
|
+
jsonResponseCaptures.delete(res);
|
|
3102
|
+
}
|
|
3103
|
+
return;
|
|
3104
|
+
}
|
|
3105
|
+
if (req.method === "POST") {
|
|
3106
|
+
try {
|
|
3107
|
+
const body = JSON.parse(await readBody(req, REQUEST_BODY_MAX_BYTES, requestBodyNoProgressTimeoutMs));
|
|
3108
|
+
if (await handlePostRequest(config, streamRuntime, compactRuntime, sessionMutations, url.pathname, body, res)) {
|
|
3109
|
+
return;
|
|
3110
|
+
}
|
|
3111
|
+
}
|
|
3112
|
+
catch (err) {
|
|
3113
|
+
logRequestError(req, err);
|
|
3114
|
+
if (!res.headersSent) {
|
|
3115
|
+
sendJson(res, requestErrorStatus(err, 500), requestErrorBody(err));
|
|
3116
|
+
}
|
|
3117
|
+
else {
|
|
3118
|
+
res.write(encodeErrorEvent(err instanceof Error ? err.message : String(err)));
|
|
3119
|
+
res.end();
|
|
3120
|
+
}
|
|
3121
|
+
return;
|
|
680
3122
|
}
|
|
3123
|
+
}
|
|
3124
|
+
if (req.method === "DELETE" && url.pathname.startsWith("/api/session/")) {
|
|
3125
|
+
const sessionId = decodeURIComponent(url.pathname.slice("/api/session/".length));
|
|
3126
|
+
await deleteSessionAtomically(config, configOverride?.sessionPersistenceDelete ?? deletePersistedSession, streamRuntime, compactRuntime, sessionMutations, sessionId);
|
|
3127
|
+
sendJson(res, 200, { deleted: sessionId });
|
|
681
3128
|
return;
|
|
682
3129
|
}
|
|
3130
|
+
sendJson(res, 404, { error: "Not found" });
|
|
683
3131
|
}
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
3132
|
+
catch (error) {
|
|
3133
|
+
logRequestError(req, error);
|
|
3134
|
+
if (res.headersSent) {
|
|
3135
|
+
res.destroy(error instanceof Error ? error : new Error(String(error)));
|
|
3136
|
+
return;
|
|
3137
|
+
}
|
|
3138
|
+
sendJson(res, error instanceof URIError ? 400 : requestErrorStatus(error, 500), requestErrorBody(error));
|
|
3139
|
+
}
|
|
3140
|
+
});
|
|
3141
|
+
serverInitializations.set(server, serverInitialization);
|
|
3142
|
+
server.requestTimeout = 0;
|
|
3143
|
+
server.headersTimeout = HEADERS_TIMEOUT_MS;
|
|
3144
|
+
const streamRunCleanup = setInterval(() => {
|
|
3145
|
+
void streamStore.prune().catch((error) => {
|
|
3146
|
+
console.error(JSON.stringify({
|
|
3147
|
+
phase: "stream_persistence",
|
|
3148
|
+
operation: "prune",
|
|
3149
|
+
error: error instanceof Error ? error.message : String(error),
|
|
3150
|
+
}));
|
|
3151
|
+
});
|
|
3152
|
+
pruneChunkTargetResponses(chunkTargetResponses);
|
|
3153
|
+
}, STREAM_RUN_CLEANUP_INTERVAL_MS);
|
|
3154
|
+
streamRunCleanup.unref();
|
|
3155
|
+
const compactRunCleanup = setInterval(() => {
|
|
3156
|
+
void compactStore.prune().catch((error) => failStopCompactPersistence(compactRuntime, error));
|
|
3157
|
+
}, STREAM_RUN_CLEANUP_INTERVAL_MS);
|
|
3158
|
+
compactRunCleanup.unref();
|
|
3159
|
+
let shutdownPromise;
|
|
3160
|
+
const shutDownRuns = () => {
|
|
3161
|
+
if (shutdownPromise)
|
|
3162
|
+
return shutdownPromise;
|
|
3163
|
+
clearInterval(streamRunCleanup);
|
|
3164
|
+
clearInterval(compactRunCleanup);
|
|
3165
|
+
streamRuntime.closing = true;
|
|
3166
|
+
compactRuntime.closing = true;
|
|
3167
|
+
const pendingCompactSettlements = [];
|
|
3168
|
+
for (const run of streamRuntime.activeRuns.values()) {
|
|
3169
|
+
run.shuttingDown = true;
|
|
3170
|
+
clearStreamRunFlushTimer(run);
|
|
3171
|
+
run.abortController.abort();
|
|
3172
|
+
for (const res of [...run.subscribers.keys()]) {
|
|
3173
|
+
detachStreamRunSubscriber(streamRuntime, run, res);
|
|
3174
|
+
res.destroy();
|
|
3175
|
+
}
|
|
3176
|
+
}
|
|
3177
|
+
streamRuntime.activeRuns.clear();
|
|
3178
|
+
for (const replays of streamRuntime.replayReaders.values()) {
|
|
3179
|
+
for (const replay of replays) {
|
|
3180
|
+
replay.abortController.abort();
|
|
3181
|
+
replay.response.destroy();
|
|
3182
|
+
}
|
|
3183
|
+
}
|
|
3184
|
+
for (const run of compactRuntime.activeRuns.values()) {
|
|
3185
|
+
run.abortController.abort();
|
|
3186
|
+
pendingCompactSettlements.push(settleCompactRun(config, compactRuntime, sessionMutations, run, createCompactTerminalPlan(run, createCompactFailureResponse(run, 499, "Server closed while compaction was running", "terminal")), "failed"));
|
|
3187
|
+
for (const res of run.subscribers) {
|
|
3188
|
+
res.destroy();
|
|
3189
|
+
}
|
|
690
3190
|
}
|
|
691
|
-
|
|
3191
|
+
compactRuntime.activeRuns.clear();
|
|
3192
|
+
chunkTargetResponses.clear();
|
|
3193
|
+
shutdownPromise = (async () => {
|
|
3194
|
+
const cleanupErrors = [];
|
|
3195
|
+
try {
|
|
3196
|
+
await serverReady;
|
|
3197
|
+
}
|
|
3198
|
+
catch (error) {
|
|
3199
|
+
cleanupErrors.push(error);
|
|
3200
|
+
}
|
|
3201
|
+
try {
|
|
3202
|
+
await streamRuntime.setupQueue;
|
|
3203
|
+
}
|
|
3204
|
+
catch (error) {
|
|
3205
|
+
cleanupErrors.push(error);
|
|
3206
|
+
}
|
|
3207
|
+
try {
|
|
3208
|
+
await compactRuntime.setupQueue;
|
|
3209
|
+
}
|
|
3210
|
+
catch (error) {
|
|
3211
|
+
cleanupErrors.push(error);
|
|
3212
|
+
}
|
|
3213
|
+
if (pendingCompactSettlements.length > 0) {
|
|
3214
|
+
const settlements = await Promise.allSettled(pendingCompactSettlements);
|
|
3215
|
+
for (const settlement of settlements) {
|
|
3216
|
+
if (settlement.status === "rejected")
|
|
3217
|
+
cleanupErrors.push(settlement.reason);
|
|
3218
|
+
}
|
|
3219
|
+
}
|
|
3220
|
+
if (streamStoreInitialized) {
|
|
3221
|
+
try {
|
|
3222
|
+
await streamStore.flush();
|
|
3223
|
+
}
|
|
3224
|
+
catch (error) {
|
|
3225
|
+
cleanupErrors.push(error);
|
|
3226
|
+
}
|
|
3227
|
+
}
|
|
3228
|
+
if (compactStoreInitialized) {
|
|
3229
|
+
try {
|
|
3230
|
+
await compactStore.flush();
|
|
3231
|
+
}
|
|
3232
|
+
catch (error) {
|
|
3233
|
+
cleanupErrors.push(error);
|
|
3234
|
+
}
|
|
3235
|
+
}
|
|
3236
|
+
const pendingStreamReaders = [
|
|
3237
|
+
...[...streamRuntime.subscriberPumps.values()].flatMap((pumps) => [...pumps]),
|
|
3238
|
+
...[...streamRuntime.replayReaders.values()].flatMap((replays) => [...replays].flatMap((replay) => (replay.promise ? [replay.promise] : []))),
|
|
3239
|
+
];
|
|
3240
|
+
if (pendingStreamReaders.length > 0) {
|
|
3241
|
+
await Promise.allSettled(pendingStreamReaders);
|
|
3242
|
+
}
|
|
3243
|
+
if (streamStoreInitialized) {
|
|
3244
|
+
try {
|
|
3245
|
+
await streamStore.close();
|
|
3246
|
+
}
|
|
3247
|
+
catch (error) {
|
|
3248
|
+
cleanupErrors.push(error);
|
|
3249
|
+
}
|
|
3250
|
+
}
|
|
3251
|
+
if (compactStoreInitialized) {
|
|
3252
|
+
try {
|
|
3253
|
+
await compactStore.close();
|
|
3254
|
+
}
|
|
3255
|
+
catch (error) {
|
|
3256
|
+
cleanupErrors.push(error);
|
|
3257
|
+
}
|
|
3258
|
+
}
|
|
3259
|
+
if (cleanupErrors.length === 1)
|
|
3260
|
+
throw cleanupErrors[0];
|
|
3261
|
+
if (cleanupErrors.length > 1) {
|
|
3262
|
+
throw new AggregateError(cleanupErrors, "pi-server shutdown encountered persistence errors");
|
|
3263
|
+
}
|
|
3264
|
+
})();
|
|
3265
|
+
return shutdownPromise;
|
|
3266
|
+
};
|
|
3267
|
+
const closeServer = server.close.bind(server);
|
|
3268
|
+
let ownershipReleased = false;
|
|
3269
|
+
const releaseOwnership = () => {
|
|
3270
|
+
if (ownershipReleased)
|
|
3271
|
+
return;
|
|
3272
|
+
ownershipReleased = true;
|
|
3273
|
+
releasePiServerOwnership(ownership);
|
|
3274
|
+
};
|
|
3275
|
+
server.close = ((callback) => {
|
|
3276
|
+
const cleanup = shutDownRuns();
|
|
3277
|
+
let httpClosed = false;
|
|
3278
|
+
let cleanupDone = false;
|
|
3279
|
+
let finished = false;
|
|
3280
|
+
let closeError;
|
|
3281
|
+
const finish = () => {
|
|
3282
|
+
if (finished || !httpClosed || !cleanupDone)
|
|
3283
|
+
return;
|
|
3284
|
+
finished = true;
|
|
3285
|
+
releaseOwnership();
|
|
3286
|
+
callback?.(closeError);
|
|
3287
|
+
};
|
|
3288
|
+
void cleanup.then(() => {
|
|
3289
|
+
cleanupDone = true;
|
|
3290
|
+
finish();
|
|
3291
|
+
}, (error) => {
|
|
3292
|
+
cleanupDone = true;
|
|
3293
|
+
closeError = error instanceof Error ? error : new Error(String(error));
|
|
3294
|
+
finish();
|
|
3295
|
+
});
|
|
3296
|
+
return closeServer((error) => {
|
|
3297
|
+
httpClosed = true;
|
|
3298
|
+
closeError ??= error;
|
|
3299
|
+
finish();
|
|
3300
|
+
});
|
|
3301
|
+
});
|
|
3302
|
+
server.once("close", () => {
|
|
3303
|
+
void shutDownRuns()
|
|
3304
|
+
.finally(releaseOwnership)
|
|
3305
|
+
.catch(() => undefined);
|
|
692
3306
|
});
|
|
693
3307
|
return server;
|
|
694
3308
|
}
|
|
695
|
-
function
|
|
3309
|
+
export function createPiServer(configOverride) {
|
|
3310
|
+
const ownership = acquirePiServerOwnership();
|
|
3311
|
+
try {
|
|
3312
|
+
return createOwnedPiServer(configOverride, ownership);
|
|
3313
|
+
}
|
|
3314
|
+
catch (error) {
|
|
3315
|
+
releasePiServerOwnership(ownership);
|
|
3316
|
+
throw error;
|
|
3317
|
+
}
|
|
3318
|
+
}
|
|
3319
|
+
function toProxyEvent(event, run) {
|
|
696
3320
|
switch (event.type) {
|
|
697
3321
|
case "start":
|
|
3322
|
+
run.contentDeltas.clear();
|
|
698
3323
|
return { type: "start" };
|
|
699
|
-
case "text_start":
|
|
3324
|
+
case "text_start": {
|
|
3325
|
+
run.contentDeltas.set(event.contentIndex, { type: "text", hash: createHash("sha256"), bytes: 0 });
|
|
700
3326
|
return { type: "text_start", contentIndex: event.contentIndex };
|
|
701
|
-
|
|
3327
|
+
}
|
|
3328
|
+
case "text_delta": {
|
|
3329
|
+
const accumulated = run.contentDeltas.get(event.contentIndex);
|
|
3330
|
+
const digest = accumulated?.type === "text"
|
|
3331
|
+
? accumulated
|
|
3332
|
+
: { type: "text", hash: createHash("sha256"), bytes: 0 };
|
|
3333
|
+
digest.hash.update(event.delta, "utf-8");
|
|
3334
|
+
digest.bytes += Buffer.byteLength(event.delta, "utf-8");
|
|
3335
|
+
run.contentDeltas.set(event.contentIndex, digest);
|
|
702
3336
|
return { type: "text_delta", contentIndex: event.contentIndex, delta: event.delta };
|
|
703
|
-
|
|
3337
|
+
}
|
|
3338
|
+
case "text_end": {
|
|
3339
|
+
const accumulated = run.contentDeltas.get(event.contentIndex);
|
|
3340
|
+
run.contentDeltas.delete(event.contentIndex);
|
|
3341
|
+
const contentBytes = Buffer.byteLength(event.content, "utf-8");
|
|
3342
|
+
const contentHash = createHash("sha256").update(event.content, "utf-8").digest("hex");
|
|
3343
|
+
const deltasMatch = accumulated?.type === "text" &&
|
|
3344
|
+
accumulated.bytes === contentBytes &&
|
|
3345
|
+
accumulated.hash.digest("hex") === contentHash;
|
|
704
3346
|
return {
|
|
705
3347
|
type: "text_end",
|
|
706
3348
|
contentIndex: event.contentIndex,
|
|
3349
|
+
content: deltasMatch ? undefined : event.content,
|
|
707
3350
|
contentSignature: event.partial.content[event.contentIndex]?.type === "text"
|
|
708
3351
|
? event.partial.content[event.contentIndex].textSignature
|
|
709
3352
|
: undefined,
|
|
710
3353
|
};
|
|
711
|
-
|
|
3354
|
+
}
|
|
3355
|
+
case "thinking_start": {
|
|
3356
|
+
run.contentDeltas.set(event.contentIndex, { type: "thinking", hash: createHash("sha256"), bytes: 0 });
|
|
712
3357
|
return { type: "thinking_start", contentIndex: event.contentIndex };
|
|
713
|
-
|
|
3358
|
+
}
|
|
3359
|
+
case "thinking_delta": {
|
|
3360
|
+
const accumulated = run.contentDeltas.get(event.contentIndex);
|
|
3361
|
+
const digest = accumulated?.type === "thinking"
|
|
3362
|
+
? accumulated
|
|
3363
|
+
: { type: "thinking", hash: createHash("sha256"), bytes: 0 };
|
|
3364
|
+
digest.hash.update(event.delta, "utf-8");
|
|
3365
|
+
digest.bytes += Buffer.byteLength(event.delta, "utf-8");
|
|
3366
|
+
run.contentDeltas.set(event.contentIndex, digest);
|
|
714
3367
|
return { type: "thinking_delta", contentIndex: event.contentIndex, delta: event.delta };
|
|
715
|
-
|
|
3368
|
+
}
|
|
3369
|
+
case "thinking_end": {
|
|
3370
|
+
const accumulated = run.contentDeltas.get(event.contentIndex);
|
|
3371
|
+
run.contentDeltas.delete(event.contentIndex);
|
|
3372
|
+
const contentBytes = Buffer.byteLength(event.content, "utf-8");
|
|
3373
|
+
const contentHash = createHash("sha256").update(event.content, "utf-8").digest("hex");
|
|
3374
|
+
const deltasMatch = accumulated?.type === "thinking" &&
|
|
3375
|
+
accumulated.bytes === contentBytes &&
|
|
3376
|
+
accumulated.hash.digest("hex") === contentHash;
|
|
716
3377
|
return {
|
|
717
3378
|
type: "thinking_end",
|
|
718
3379
|
contentIndex: event.contentIndex,
|
|
3380
|
+
content: deltasMatch ? undefined : event.content,
|
|
719
3381
|
contentSignature: event.partial.content[event.contentIndex]?.type === "thinking"
|
|
720
3382
|
? event.partial.content[event.contentIndex].thinkingSignature
|
|
721
3383
|
: undefined,
|
|
3384
|
+
redacted: event.partial.content[event.contentIndex]?.type === "thinking"
|
|
3385
|
+
? event.partial.content[event.contentIndex].redacted
|
|
3386
|
+
: undefined,
|
|
722
3387
|
};
|
|
3388
|
+
}
|
|
723
3389
|
case "toolcall_start":
|
|
724
3390
|
return {
|
|
725
3391
|
type: "toolcall_start",
|
|
@@ -734,15 +3400,40 @@ function toProxyEvent(event) {
|
|
|
734
3400
|
case "toolcall_delta":
|
|
735
3401
|
return { type: "toolcall_delta", contentIndex: event.contentIndex, delta: event.delta };
|
|
736
3402
|
case "toolcall_end":
|
|
737
|
-
return {
|
|
3403
|
+
return {
|
|
3404
|
+
type: "toolcall_end",
|
|
3405
|
+
contentIndex: event.contentIndex,
|
|
3406
|
+
thoughtSignature: event.toolCall.thoughtSignature,
|
|
3407
|
+
toolCall: event.toolCall,
|
|
3408
|
+
};
|
|
738
3409
|
case "done":
|
|
739
|
-
|
|
3410
|
+
run.contentDeltas.clear();
|
|
3411
|
+
return {
|
|
3412
|
+
type: "done",
|
|
3413
|
+
reason: event.reason,
|
|
3414
|
+
usage: event.message.usage,
|
|
3415
|
+
responseModel: event.message.responseModel,
|
|
3416
|
+
responseId: event.message.responseId,
|
|
3417
|
+
diagnostics: event.message.diagnostics,
|
|
3418
|
+
api: event.message.api,
|
|
3419
|
+
provider: event.message.provider,
|
|
3420
|
+
model: event.message.model,
|
|
3421
|
+
timestamp: event.message.timestamp,
|
|
3422
|
+
};
|
|
740
3423
|
case "error":
|
|
3424
|
+
run.contentDeltas.clear();
|
|
741
3425
|
return {
|
|
742
3426
|
type: "error",
|
|
743
3427
|
reason: event.reason,
|
|
744
3428
|
errorMessage: event.error.errorMessage,
|
|
745
3429
|
usage: event.error.usage,
|
|
3430
|
+
responseModel: event.error.responseModel,
|
|
3431
|
+
responseId: event.error.responseId,
|
|
3432
|
+
diagnostics: event.error.diagnostics,
|
|
3433
|
+
api: event.error.api,
|
|
3434
|
+
provider: event.error.provider,
|
|
3435
|
+
model: event.error.model,
|
|
3436
|
+
timestamp: event.error.timestamp,
|
|
746
3437
|
};
|
|
747
3438
|
default:
|
|
748
3439
|
return undefined;
|
|
@@ -750,10 +3441,41 @@ function toProxyEvent(event) {
|
|
|
750
3441
|
}
|
|
751
3442
|
export function startServer(configOverride) {
|
|
752
3443
|
const config = loadConfig(configOverride);
|
|
753
|
-
const
|
|
754
|
-
|
|
755
|
-
|
|
3444
|
+
const externalFatalHandler = configOverride?.onFatalStreamPersistenceError;
|
|
3445
|
+
const fatalShutdownGraceMs = configOverride?.fatalShutdownGraceMs ?? FATAL_SHUTDOWN_GRACE_MS;
|
|
3446
|
+
if (!Number.isSafeInteger(fatalShutdownGraceMs) || fatalShutdownGraceMs <= 0) {
|
|
3447
|
+
throw new Error("fatalShutdownGraceMs must be a positive safe integer");
|
|
3448
|
+
}
|
|
3449
|
+
const fatalExit = configOverride?.fatalExit ?? ((code) => process.exit(code));
|
|
3450
|
+
let server;
|
|
3451
|
+
server = createPiServer({
|
|
3452
|
+
...configOverride,
|
|
3453
|
+
onFatalStreamPersistenceError: (error) => {
|
|
3454
|
+
process.exitCode = 1;
|
|
3455
|
+
const hardExitTimer = setTimeout(() => fatalExit(1), fatalShutdownGraceMs);
|
|
3456
|
+
server.close((closeError) => {
|
|
3457
|
+
if (!closeError)
|
|
3458
|
+
clearTimeout(hardExitTimer);
|
|
3459
|
+
if (closeError) {
|
|
3460
|
+
console.error(JSON.stringify({
|
|
3461
|
+
phase: "stream_persistence",
|
|
3462
|
+
operation: "fatal_shutdown",
|
|
3463
|
+
error: closeError.message,
|
|
3464
|
+
}));
|
|
3465
|
+
}
|
|
3466
|
+
externalFatalHandler?.(error);
|
|
3467
|
+
});
|
|
3468
|
+
},
|
|
756
3469
|
});
|
|
3470
|
+
const initialization = serverInitializations.get(server);
|
|
3471
|
+
if (!initialization) {
|
|
3472
|
+
throw new Error("pi-server initialization promise was not registered");
|
|
3473
|
+
}
|
|
3474
|
+
void initialization.then(() => {
|
|
3475
|
+
server.listen(config.port, config.host, () => {
|
|
3476
|
+
console.log(`pi-server listening on ${config.host}:${config.port}`);
|
|
3477
|
+
});
|
|
3478
|
+
}, () => undefined);
|
|
757
3479
|
return server;
|
|
758
3480
|
}
|
|
759
3481
|
//# sourceMappingURL=server.js.map
|