@pasko70/pibo 1.7.12 → 1.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apps/chat/workflow-manual-trigger-runtime.js +13 -4
- package/dist/core/compaction-prompt.js +87 -52
- package/dist/core/context-guard.js +172 -5
- package/dist/core/provider-recovery.js +64 -0
- package/dist/core/provider-telemetry.js +97 -4
- package/dist/core/routed-session.js +181 -16
- package/dist/core/runtime-telemetry.js +251 -91
- package/dist/core/runtime.js +53 -12
- package/dist/core/session-errors.js +14 -1
- package/dist/core/session-router.js +127 -26
- package/dist/data/telemetry.js +115 -26
- package/dist/gateway/request.js +17 -8
- package/dist/ralph/service.js +3 -5
- package/package.json +1 -1
- package/dist/apps/vscode-artifacts/latest.vsix +0 -0
- package/dist/apps/vscode-artifacts/pibo-vscode-ext-1.7.12.vsix +0 -0
package/dist/core/runtime.js
CHANGED
|
@@ -14,7 +14,7 @@ import { getMcpAgentContextFile } from "../mcp/agent-context.js";
|
|
|
14
14
|
import { createPiboSystemPromptTemplateExtension } from "./system-prompt-template.js";
|
|
15
15
|
import { getActivePiboBasePromptPath } from "./base-prompt.js";
|
|
16
16
|
import { createPiboCompactionPromptExtension } from "./compaction-prompt.js";
|
|
17
|
-
import { createPiboAssistantContextGuardExtension } from "./context-guard.js";
|
|
17
|
+
import { cancelPiboAssistantContextGuardRecovery, createPiboAssistantContextGuardExtension, createPiboAssistantContextGuardRecovery, isPiboAssistantContextGuardRecoveryPending, registerPiboAssistantContextGuardRecovery, } from "./context-guard.js";
|
|
18
18
|
import { getPiPackageRuntimeOptions } from "../pi-packages/runtime.js";
|
|
19
19
|
import { getDefaultPiboWorkspace } from "./workspace.js";
|
|
20
20
|
import { DEFAULT_USER_TIMEZONE } from "./user-settings.js";
|
|
@@ -166,10 +166,10 @@ function getBuiltinToolAllowlist(profile, customTools) {
|
|
|
166
166
|
return undefined;
|
|
167
167
|
return [...selectedBuiltinTools, ...customTools.map((tool) => tool.name)];
|
|
168
168
|
}
|
|
169
|
-
function getProfileExtensionFactories(profile, extensionFactories) {
|
|
169
|
+
function getProfileExtensionFactories(profile, extensionFactories, contextGuardRecovery, getSettingsManager) {
|
|
170
170
|
const piboPromptTemplateExtension = createPiboSystemPromptTemplateExtension();
|
|
171
|
-
const piboCompactionPromptExtension = createPiboCompactionPromptExtension();
|
|
172
|
-
const piboContextGuardExtension = createPiboAssistantContextGuardExtension();
|
|
171
|
+
const piboCompactionPromptExtension = createPiboCompactionPromptExtension({ getSettingsManager });
|
|
172
|
+
const piboContextGuardExtension = createPiboAssistantContextGuardExtension({}, contextGuardRecovery);
|
|
173
173
|
const providerToolExtensions = profile.tools
|
|
174
174
|
.filter((tool) => tool.enabled !== false)
|
|
175
175
|
.filter(isWebSearchProviderTool)
|
|
@@ -234,12 +234,14 @@ export async function createPiboRuntime(options = {}) {
|
|
|
234
234
|
const sessionManager = await createSessionManager(cwd, profile, options.persistSession !== false);
|
|
235
235
|
const authStorage = AuthStorage.create();
|
|
236
236
|
const createRuntime = async ({ cwd: runtimeCwd, agentDir: runtimeAgentDir, sessionManager: runtimeSessionManager, sessionStartEvent, }) => {
|
|
237
|
+
const contextGuardRecovery = createPiboAssistantContextGuardRecovery();
|
|
237
238
|
const contextFiles = await loadContextFiles(runtimeCwd, profile.contextFiles);
|
|
238
239
|
const sessionContextFile = createSessionContextFile({ piboSessionId: profile.sessionId, ...options.sessionContext });
|
|
239
240
|
const installedToolContextFile = getInstalledCliToolContextFile();
|
|
240
241
|
const mcpAgentContextFile = await getMcpAgentContextFile(profile.mcpServers);
|
|
241
242
|
const skillPaths = getEnabledSkillPaths(runtimeCwd, profile);
|
|
242
243
|
const piPackageOptions = getPiPackageRuntimeOptions(runtimeCwd, profile);
|
|
244
|
+
let runtimeSettingsManager;
|
|
243
245
|
const services = await createAgentSessionServices({
|
|
244
246
|
cwd: runtimeCwd,
|
|
245
247
|
agentDir: runtimeAgentDir,
|
|
@@ -247,7 +249,7 @@ export async function createPiboRuntime(options = {}) {
|
|
|
247
249
|
resourceLoaderOptions: {
|
|
248
250
|
...piPackageOptions.resourceLoaderOptions,
|
|
249
251
|
additionalSkillPaths: skillPaths,
|
|
250
|
-
extensionFactories: getProfileExtensionFactories(profile, options.extensionFactories),
|
|
252
|
+
extensionFactories: getProfileExtensionFactories(profile, options.extensionFactories, contextGuardRecovery, () => runtimeSettingsManager),
|
|
251
253
|
noExtensions: true,
|
|
252
254
|
noSkills: true,
|
|
253
255
|
noPromptTemplates: true,
|
|
@@ -264,6 +266,7 @@ export async function createPiboRuntime(options = {}) {
|
|
|
264
266
|
}),
|
|
265
267
|
},
|
|
266
268
|
});
|
|
269
|
+
runtimeSettingsManager = services.settingsManager;
|
|
267
270
|
applyPiboRuntimeRetryDefaults(services.settingsManager, options.retryDefaults);
|
|
268
271
|
registerOpenAiGpt56Models(services.modelRegistry);
|
|
269
272
|
registerMiniMaxProvider(services.modelRegistry);
|
|
@@ -293,6 +296,10 @@ export async function createPiboRuntime(options = {}) {
|
|
|
293
296
|
tools: getBuiltinToolAllowlist(profile, customTools),
|
|
294
297
|
});
|
|
295
298
|
installValidationOutputCompaction(created.session.agent);
|
|
299
|
+
registerPiboAssistantContextGuardRecovery(created.session, contextGuardRecovery);
|
|
300
|
+
if (options.contextGuardTuiQueueOrdering === true) {
|
|
301
|
+
installPiboContextGuardTuiQueueOrdering(created.session);
|
|
302
|
+
}
|
|
296
303
|
const resourceLoader = services.resourceLoader;
|
|
297
304
|
const diagnostics = [
|
|
298
305
|
...piPackageOptions.diagnostics,
|
|
@@ -303,13 +310,14 @@ export async function createPiboRuntime(options = {}) {
|
|
|
303
310
|
message: `Failed to load extension "${path}": ${error}`,
|
|
304
311
|
})),
|
|
305
312
|
];
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
created.session
|
|
313
|
+
const originalDispose = created.session.dispose.bind(created.session);
|
|
314
|
+
created.session.dispose = () => {
|
|
315
|
+
cancelPiboAssistantContextGuardRecovery(created.session, new Error("Context guard recovery cancelled because the Pi session was disposed"));
|
|
316
|
+
if (localRuntimeRegistry) {
|
|
309
317
|
void localRuntimeRegistry.closeControllerSessions(profile.sessionId ?? "local", { force: true });
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
}
|
|
318
|
+
}
|
|
319
|
+
originalDispose();
|
|
320
|
+
};
|
|
313
321
|
return {
|
|
314
322
|
...created,
|
|
315
323
|
services,
|
|
@@ -456,6 +464,39 @@ export async function inspectPiboProfile(options = {}) {
|
|
|
456
464
|
await runtime.dispose();
|
|
457
465
|
}
|
|
458
466
|
}
|
|
467
|
+
function installPiboContextGuardTuiQueueOrdering(session) {
|
|
468
|
+
const originalSubscribe = session.subscribe.bind(session);
|
|
469
|
+
const originalPrompt = session.prompt.bind(session);
|
|
470
|
+
const originalSteer = session.steer.bind(session);
|
|
471
|
+
session.subscribe = ((listener) => originalSubscribe((event) => {
|
|
472
|
+
if (event.type === "compaction_end"
|
|
473
|
+
&& event.result
|
|
474
|
+
&& isPiboAssistantContextGuardRecoveryPending(session)) {
|
|
475
|
+
listener({ ...event, willRetry: true });
|
|
476
|
+
return;
|
|
477
|
+
}
|
|
478
|
+
listener(event);
|
|
479
|
+
}));
|
|
480
|
+
session.prompt = async (text, options) => {
|
|
481
|
+
if (isPiboAssistantContextGuardRecoveryPending(session)) {
|
|
482
|
+
if (!session.isStreaming) {
|
|
483
|
+
await session.followUp(text, options?.images);
|
|
484
|
+
options?.preflightResult?.(true);
|
|
485
|
+
return;
|
|
486
|
+
}
|
|
487
|
+
await originalPrompt(text, { ...options, streamingBehavior: "followUp" });
|
|
488
|
+
return;
|
|
489
|
+
}
|
|
490
|
+
await originalPrompt(text, options);
|
|
491
|
+
};
|
|
492
|
+
session.steer = async (text, images) => {
|
|
493
|
+
if (isPiboAssistantContextGuardRecoveryPending(session)) {
|
|
494
|
+
await session.followUp(text, images);
|
|
495
|
+
return;
|
|
496
|
+
}
|
|
497
|
+
await originalSteer(text, images);
|
|
498
|
+
};
|
|
499
|
+
}
|
|
459
500
|
export async function runPiboTui(options = {}) {
|
|
460
501
|
const profile = options.profile ?? createDefaultPiboProfile();
|
|
461
502
|
const hasEnabledSubagents = profile.subagents.some((subagent) => subagent.enabled !== false);
|
|
@@ -465,7 +506,7 @@ export async function runPiboTui(options = {}) {
|
|
|
465
506
|
process.exitCode = 1;
|
|
466
507
|
return;
|
|
467
508
|
}
|
|
468
|
-
const runtime = await createPiboRuntime({ ...options, profile });
|
|
509
|
+
const runtime = await createPiboRuntime({ ...options, profile, contextGuardTuiQueueOrdering: true });
|
|
469
510
|
try {
|
|
470
511
|
const fatal = runtime.diagnostics.find((diagnostic) => diagnostic.type === "error");
|
|
471
512
|
for (const diagnostic of runtime.diagnostics) {
|
|
@@ -10,6 +10,11 @@ const PROVIDER_NETWORK_ERROR_MARKERS = [
|
|
|
10
10
|
"reset before headers",
|
|
11
11
|
"socket hang up",
|
|
12
12
|
"socket connection was closed",
|
|
13
|
+
"eai_again",
|
|
14
|
+
"enotfound",
|
|
15
|
+
"econnreset",
|
|
16
|
+
"econnrefused",
|
|
17
|
+
"etimedout",
|
|
13
18
|
];
|
|
14
19
|
export function classifySessionErrorMessage(message, options = {}) {
|
|
15
20
|
const normalized = message.toLowerCase();
|
|
@@ -20,7 +25,15 @@ export function classifySessionErrorMessage(message, options = {}) {
|
|
|
20
25
|
return { category: "provider_transport", errorClass: "provider_transport", code: "websocket_error", origin: "provider", retryable: true, userMessage: "The provider WebSocket connection failed." };
|
|
21
26
|
}
|
|
22
27
|
if (normalized.includes("request was aborted") || normalized.includes("aborted")) {
|
|
23
|
-
return { category: "runtime_abort", errorClass: "runtime_abort", code: "request_aborted", origin: "runtime", retryable:
|
|
28
|
+
return { category: "runtime_abort", errorClass: "runtime_abort", code: "request_aborted", origin: "runtime", retryable: false, userMessage: "The active model request was aborted." };
|
|
29
|
+
}
|
|
30
|
+
if (normalized.includes("insufficient_quota")
|
|
31
|
+
|| normalized.includes("quota exceeded")
|
|
32
|
+
|| normalized.includes("out of budget")
|
|
33
|
+
|| normalized.includes("billing")
|
|
34
|
+
|| normalized.includes("usage limit")
|
|
35
|
+
|| normalized.includes("available balance")) {
|
|
36
|
+
return { category: "quota_exhausted", errorClass: "provider_rate_limit", code: "quota_exhausted", origin: "provider", retryable: false, userMessage: "The provider quota or billing limit was reached." };
|
|
24
37
|
}
|
|
25
38
|
if (normalized.includes("rate limit") || normalized.includes("429")) {
|
|
26
39
|
return { category: "rate_limit", errorClass: "provider_rate_limit", code: "rate_limited", origin: "provider", retryable: true, userMessage: "The provider rate limit was reached." };
|
|
@@ -20,6 +20,7 @@ import { withWorkflowSessionKind } from "../sessions/workflow-session-kind.js";
|
|
|
20
20
|
import { PiboRuntimeTelemetryRecorder } from "./runtime-telemetry.js";
|
|
21
21
|
import { createPiboProviderTelemetryExtension } from "./provider-telemetry.js";
|
|
22
22
|
const DEFAULT_SUBAGENT_REPLY_TIMEOUT_MS = 10 * 60 * 1000;
|
|
23
|
+
const DEFAULT_ROUTED_SESSION_IDLE_TIMEOUT_MS = 30 * 60 * 1000;
|
|
23
24
|
export const RALPH_RUNTIME_RETRY_DEFAULTS = {
|
|
24
25
|
enabled: true,
|
|
25
26
|
maxRetries: 7,
|
|
@@ -132,6 +133,8 @@ export class PiboSessionRouter {
|
|
|
132
133
|
signalRegistry;
|
|
133
134
|
runtimeRegistry;
|
|
134
135
|
scheduledRunReminders = new Map();
|
|
136
|
+
idleSessionTimers = new Map();
|
|
137
|
+
routedSessionIdleTimeoutMs;
|
|
135
138
|
baseProfile;
|
|
136
139
|
pluginRegistry;
|
|
137
140
|
sessionStore;
|
|
@@ -146,6 +149,12 @@ export class PiboSessionRouter {
|
|
|
146
149
|
this.telemetryRecorder = this.telemetryStore
|
|
147
150
|
? new PiboRuntimeTelemetryRecorder(this.telemetryStore, undefined, { providerEventMode: providerEventTelemetryModeFromEnv() })
|
|
148
151
|
: undefined;
|
|
152
|
+
const idleTimeoutMs = options.routedSessionIdleTimeoutMs;
|
|
153
|
+
this.routedSessionIdleTimeoutMs = idleTimeoutMs === false
|
|
154
|
+
? false
|
|
155
|
+
: typeof idleTimeoutMs === "number" && Number.isFinite(idleTimeoutMs) && idleTimeoutMs > 0
|
|
156
|
+
? idleTimeoutMs
|
|
157
|
+
: DEFAULT_ROUTED_SESSION_IDLE_TIMEOUT_MS;
|
|
149
158
|
const defaultProfileName = selectDefaultPiboProfileName(this.pluginRegistry);
|
|
150
159
|
this.baseProfile = options.profile ?? createPiboProfileFromRegistryOrDefault(this.pluginRegistry, defaultProfileName);
|
|
151
160
|
this.reliabilityStore = options.reliabilityStore ?? (options.persistSession === false ? undefined : createDefaultPiboReliabilityStore());
|
|
@@ -165,20 +174,29 @@ export class PiboSessionRouter {
|
|
|
165
174
|
}
|
|
166
175
|
async emit(event) {
|
|
167
176
|
const session = await this.getOrCreateSession(event.piboSessionId);
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
+
this.clearIdleSessionTimer(event.piboSessionId);
|
|
178
|
+
try {
|
|
179
|
+
if (event.type === "message") {
|
|
180
|
+
return session.enqueueMessage(event);
|
|
181
|
+
}
|
|
182
|
+
const output = await session.executeAction(event);
|
|
183
|
+
if (event.action === "abort") {
|
|
184
|
+
this.signalRegistry.project({ type: "session_interrupted", piboSessionId: event.piboSessionId, reason: "abort action" });
|
|
185
|
+
}
|
|
186
|
+
if (event.action === "dispose") {
|
|
187
|
+
await this.disposeSessionSubtree(event.piboSessionId, "dispose action", { cancelRuns: true });
|
|
188
|
+
}
|
|
189
|
+
else if (event.action === "kill" || event.action === "kill_all") {
|
|
190
|
+
await this.disposeSessionSubtree(event.piboSessionId, `${event.action} action`, { cancelRuns: event.action === "kill_all" });
|
|
191
|
+
}
|
|
192
|
+
else if (shouldResetSessionAfterAction(event.action)) {
|
|
193
|
+
await this.resetCachedSession(event.piboSessionId, "provider auth changed");
|
|
194
|
+
}
|
|
195
|
+
return output;
|
|
177
196
|
}
|
|
178
|
-
|
|
179
|
-
|
|
197
|
+
finally {
|
|
198
|
+
this.scheduleIdleSessionEvictionIfIdle(event.piboSessionId);
|
|
180
199
|
}
|
|
181
|
-
return output;
|
|
182
200
|
}
|
|
183
201
|
async killSession(piboSessionId, options) {
|
|
184
202
|
const killed = [];
|
|
@@ -190,23 +208,32 @@ export class PiboSessionRouter {
|
|
|
190
208
|
const runs = this.runRegistry.cancelControllerRuns(piboSessionId);
|
|
191
209
|
cancelledRuns.push(...runs.map((r) => r.runId));
|
|
192
210
|
}
|
|
193
|
-
await this.runtimeRegistry.closeControllerSessions(piboSessionId, { force: true });
|
|
194
211
|
this.signalRegistry.project({ type: "session_interrupted", piboSessionId, reason: "kill" });
|
|
195
212
|
const children = await this.killChildSessions(piboSessionId, options);
|
|
196
213
|
killed.push(...children.killed);
|
|
197
214
|
cancelledRuns.push(...children.cancelledRuns);
|
|
215
|
+
await this.disposeSessionSubtree(piboSessionId, "kill", { cancelRuns: false });
|
|
198
216
|
}
|
|
199
217
|
return { killed, cancelledRuns };
|
|
200
218
|
}
|
|
201
|
-
|
|
202
|
-
const
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
219
|
+
async disposeSessionSubtree(piboSessionId, reason, options) {
|
|
220
|
+
const ids = [piboSessionId, ...this.descendantSessionIds(piboSessionId)];
|
|
221
|
+
const sessions = [];
|
|
222
|
+
for (const id of ids) {
|
|
223
|
+
if (options.cancelRuns)
|
|
224
|
+
this.runRegistry.cancelControllerRuns(id);
|
|
225
|
+
this.clearIdleSessionTimer(id);
|
|
207
226
|
this.scheduledRunReminders.delete(id);
|
|
227
|
+
const cached = this.sessions.get(id);
|
|
228
|
+
if (cached)
|
|
229
|
+
sessions.push(cached);
|
|
208
230
|
this.sessions.delete(id);
|
|
209
231
|
}
|
|
232
|
+
await Promise.all(ids.map((id) => this.runtimeRegistry.closeControllerSessions(id, { force: true })));
|
|
233
|
+
await Promise.all(sessions.map((session) => session.dispose()));
|
|
234
|
+
for (const id of ids) {
|
|
235
|
+
this.signalRegistry.project({ type: "session_disposed", piboSessionId: id, reason });
|
|
236
|
+
}
|
|
210
237
|
}
|
|
211
238
|
descendantSessionIds(parentId) {
|
|
212
239
|
const output = [];
|
|
@@ -292,6 +319,8 @@ export class PiboSessionRouter {
|
|
|
292
319
|
const eventWithId = { ...event, id: event.id ?? randomUUID() };
|
|
293
320
|
return await new Promise((resolve, reject) => {
|
|
294
321
|
let settled = false;
|
|
322
|
+
let lastAssistantMessage;
|
|
323
|
+
let timeout;
|
|
295
324
|
const unsubscribe = this.subscribe((output) => {
|
|
296
325
|
if (output.piboSessionId !== eventWithId.piboSessionId ||
|
|
297
326
|
!("eventId" in output) ||
|
|
@@ -299,20 +328,21 @@ export class PiboSessionRouter {
|
|
|
299
328
|
return;
|
|
300
329
|
}
|
|
301
330
|
if (output.type === "assistant_message") {
|
|
302
|
-
|
|
331
|
+
lastAssistantMessage = output;
|
|
332
|
+
}
|
|
333
|
+
else if (output.type === "message_finished") {
|
|
334
|
+
finish(lastAssistantMessage ?? new Error(`Pibo session "${eventWithId.piboSessionId}" finished without an assistant reply`));
|
|
303
335
|
}
|
|
304
336
|
else if (output.type === "session_error") {
|
|
305
337
|
finish(new Error(output.error));
|
|
306
338
|
}
|
|
307
339
|
});
|
|
308
|
-
const timeout = setTimeout(() => {
|
|
309
|
-
finish(new Error(`Timed out waiting for assistant reply from Pibo session "${eventWithId.piboSessionId}"`));
|
|
310
|
-
}, timeoutMs);
|
|
311
340
|
const finish = (result) => {
|
|
312
341
|
if (settled)
|
|
313
342
|
return;
|
|
314
343
|
settled = true;
|
|
315
|
-
|
|
344
|
+
if (timeout)
|
|
345
|
+
clearTimeout(timeout);
|
|
316
346
|
unsubscribe();
|
|
317
347
|
if (result instanceof Error) {
|
|
318
348
|
reject(result);
|
|
@@ -321,12 +351,29 @@ export class PiboSessionRouter {
|
|
|
321
351
|
resolve(result);
|
|
322
352
|
}
|
|
323
353
|
};
|
|
354
|
+
timeout = setTimeout(() => {
|
|
355
|
+
if (settled)
|
|
356
|
+
return;
|
|
357
|
+
settled = true;
|
|
358
|
+
unsubscribe();
|
|
359
|
+
const timeoutError = new Error(`Timed out waiting for assistant reply from Pibo session "${eventWithId.piboSessionId}"`);
|
|
360
|
+
reject(timeoutError);
|
|
361
|
+
void this.emit({
|
|
362
|
+
type: "execution",
|
|
363
|
+
piboSessionId: eventWithId.piboSessionId,
|
|
364
|
+
action: "abort",
|
|
365
|
+
id: randomUUID(),
|
|
366
|
+
}).catch(() => { });
|
|
367
|
+
}, timeoutMs);
|
|
324
368
|
this.emit(eventWithId).catch(finish);
|
|
325
369
|
});
|
|
326
370
|
}
|
|
327
371
|
async disposeAll() {
|
|
328
372
|
const sessions = [...this.sessions.values()];
|
|
329
373
|
this.sessions.clear();
|
|
374
|
+
for (const timer of this.idleSessionTimers.values())
|
|
375
|
+
clearTimeout(timer);
|
|
376
|
+
this.idleSessionTimers.clear();
|
|
330
377
|
this.runRegistry.cancelAll("Pibo session router was disposed.");
|
|
331
378
|
for (const session of sessions)
|
|
332
379
|
this.signalRegistry.project({ type: "session_disposed", piboSessionId: session.getStatus().piboSessionId, reason: "router disposed" });
|
|
@@ -334,10 +381,54 @@ export class PiboSessionRouter {
|
|
|
334
381
|
await this.runtimeRegistry.closeAll({ force: true });
|
|
335
382
|
await Promise.all(sessions.map((session) => session.dispose()));
|
|
336
383
|
}
|
|
384
|
+
clearIdleSessionTimer(piboSessionId) {
|
|
385
|
+
const timer = this.idleSessionTimers.get(piboSessionId);
|
|
386
|
+
if (timer)
|
|
387
|
+
clearTimeout(timer);
|
|
388
|
+
this.idleSessionTimers.delete(piboSessionId);
|
|
389
|
+
}
|
|
390
|
+
scheduleIdleSessionEvictionIfIdle(piboSessionId) {
|
|
391
|
+
if (this.routedSessionIdleTimeoutMs === false)
|
|
392
|
+
return;
|
|
393
|
+
const session = this.sessions.get(piboSessionId);
|
|
394
|
+
if (!session)
|
|
395
|
+
return;
|
|
396
|
+
const status = session.getStatus();
|
|
397
|
+
if (status.disposed || status.processing || status.streaming || status.queuedMessages > 0) {
|
|
398
|
+
this.clearIdleSessionTimer(piboSessionId);
|
|
399
|
+
return;
|
|
400
|
+
}
|
|
401
|
+
this.clearIdleSessionTimer(piboSessionId);
|
|
402
|
+
const timer = setTimeout(() => {
|
|
403
|
+
this.idleSessionTimers.delete(piboSessionId);
|
|
404
|
+
void this.evictIdleSession(piboSessionId, session).catch((error) => {
|
|
405
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
406
|
+
this.emitOutput({
|
|
407
|
+
type: "session_error",
|
|
408
|
+
piboSessionId,
|
|
409
|
+
error: `Failed to dispose idle routed runtime: ${message}`,
|
|
410
|
+
errorDetails: runtimeSessionErrorDetails(message),
|
|
411
|
+
});
|
|
412
|
+
});
|
|
413
|
+
}, this.routedSessionIdleTimeoutMs);
|
|
414
|
+
timer.unref();
|
|
415
|
+
this.idleSessionTimers.set(piboSessionId, timer);
|
|
416
|
+
}
|
|
417
|
+
async evictIdleSession(piboSessionId, expected) {
|
|
418
|
+
const current = this.sessions.get(piboSessionId);
|
|
419
|
+
if (current !== expected)
|
|
420
|
+
return;
|
|
421
|
+
const status = current.getStatus();
|
|
422
|
+
if (status.disposed || status.processing || status.streaming || status.queuedMessages > 0)
|
|
423
|
+
return;
|
|
424
|
+
await this.resetCachedSession(piboSessionId, "routed runtime idle timeout");
|
|
425
|
+
}
|
|
337
426
|
async getOrCreateSession(piboSessionId) {
|
|
338
427
|
const existing = this.sessions.get(piboSessionId);
|
|
339
|
-
if (existing)
|
|
428
|
+
if (existing) {
|
|
429
|
+
this.clearIdleSessionTimer(piboSessionId);
|
|
340
430
|
return existing;
|
|
431
|
+
}
|
|
341
432
|
const pending = this.pendingSessions.get(piboSessionId);
|
|
342
433
|
if (pending)
|
|
343
434
|
return pending;
|
|
@@ -388,7 +479,16 @@ export class PiboSessionRouter {
|
|
|
388
479
|
const initialFastMode = resolvePiboSessionInitialFastMode(piboSession) ?? selectRequestedFastMode(profileForSession(profile, piboSession.piSessionId, parentPiSessionId), modelDefaults) ?? false;
|
|
389
480
|
const session = new RoutedSession(piboSession.id, runtime, this.emitOutput, this.pluginRegistry, this.options.forwardPiEvents ?? false, this.telemetryRecorder
|
|
390
481
|
? (id, event, context) => this.telemetryRecorder?.recordPiEvent(id, event, { session: this.sessionStore.get(id), status: context.status, activeEventId: context.activeEventId })
|
|
391
|
-
: undefined, initialFastMode, (result, event) => this.handleSessionOperation(result, event), (id, opts) => this.killChildSessions(id, opts), (state) =>
|
|
482
|
+
: undefined, initialFastMode, (result, event) => this.handleSessionOperation(result, event), (id, opts) => this.killChildSessions(id, opts), (state) => {
|
|
483
|
+
this.signalRegistry.project({ type: "session_processing_changed", piboSessionId: piboSession.id, processing: state.processing, queuedMessages: state.queuedMessages });
|
|
484
|
+
if (state.disposed || state.processing || state.queuedMessages > 0)
|
|
485
|
+
this.clearIdleSessionTimer(piboSession.id);
|
|
486
|
+
else
|
|
487
|
+
this.scheduleIdleSessionEvictionIfIdle(piboSession.id);
|
|
488
|
+
}, (messages, reason) => this.telemetryRecorder?.recordMessagesInterrupted(messages, {
|
|
489
|
+
session: this.sessionStore.get(piboSession.id),
|
|
490
|
+
status: this.sessions.get(piboSession.id)?.getStatus(),
|
|
491
|
+
}, reason));
|
|
392
492
|
this.sessions.set(piboSession.id, session);
|
|
393
493
|
return session;
|
|
394
494
|
}
|
|
@@ -447,6 +547,7 @@ export class PiboSessionRouter {
|
|
|
447
547
|
}
|
|
448
548
|
async resetCachedSession(piboSessionId, reason) {
|
|
449
549
|
const cached = this.sessions.get(piboSessionId);
|
|
550
|
+
this.clearIdleSessionTimer(piboSessionId);
|
|
450
551
|
this.sessions.delete(piboSessionId);
|
|
451
552
|
await this.runtimeRegistry.closeControllerSessions(piboSessionId, { force: true });
|
|
452
553
|
await cached?.dispose();
|
package/dist/data/telemetry.js
CHANGED
|
@@ -17,6 +17,61 @@ export class TelemetryStore {
|
|
|
17
17
|
getTurnTimeline(turnIdOrEventId, input = {}) {
|
|
18
18
|
return getTelemetryTurnTimeline(this.db, turnIdOrEventId, input);
|
|
19
19
|
}
|
|
20
|
+
getOpenPhaseForTurn(turnId, name) {
|
|
21
|
+
const row = this.db.prepare(`
|
|
22
|
+
SELECT * FROM telemetry_phases
|
|
23
|
+
WHERE turn_id = ? AND name = ? AND status = 'open'
|
|
24
|
+
ORDER BY COALESCE(last_progress_at, started_at) DESC, created_at DESC
|
|
25
|
+
LIMIT 1
|
|
26
|
+
`).get(turnId, name);
|
|
27
|
+
return row ? phaseFromRow(row) : undefined;
|
|
28
|
+
}
|
|
29
|
+
countPhasesForTurn(turnId, name) {
|
|
30
|
+
const row = this.db.prepare("SELECT COUNT(*) AS count FROM telemetry_phases WHERE turn_id = ? AND name = ?").get(turnId, name);
|
|
31
|
+
return Number(row.count);
|
|
32
|
+
}
|
|
33
|
+
listOpenPhasesForTurn(turnId) {
|
|
34
|
+
const rows = this.db.prepare(`
|
|
35
|
+
SELECT * FROM telemetry_phases
|
|
36
|
+
WHERE turn_id = ? AND status = 'open'
|
|
37
|
+
ORDER BY started_at ASC, created_at ASC
|
|
38
|
+
`).all(turnId);
|
|
39
|
+
return rows.map(phaseFromRow);
|
|
40
|
+
}
|
|
41
|
+
getLatestProviderRequestForTurn(turnId) {
|
|
42
|
+
const row = this.db.prepare(`
|
|
43
|
+
SELECT * FROM telemetry_provider_requests
|
|
44
|
+
WHERE turn_id = ?
|
|
45
|
+
ORDER BY started_at DESC, created_at DESC
|
|
46
|
+
LIMIT 1
|
|
47
|
+
`).get(turnId);
|
|
48
|
+
return row ? providerRequestFromRow(row) : undefined;
|
|
49
|
+
}
|
|
50
|
+
getActiveProviderRequestForTurn(turnId) {
|
|
51
|
+
const row = this.db.prepare(`
|
|
52
|
+
SELECT * FROM telemetry_provider_requests
|
|
53
|
+
WHERE turn_id = ? AND status NOT IN ('completed', 'error', 'aborted', 'timeout')
|
|
54
|
+
ORDER BY started_at DESC, created_at DESC
|
|
55
|
+
LIMIT 1
|
|
56
|
+
`).get(turnId);
|
|
57
|
+
return row ? providerRequestFromRow(row) : undefined;
|
|
58
|
+
}
|
|
59
|
+
listActiveProviderRequestsForTurn(turnId) {
|
|
60
|
+
const rows = this.db.prepare(`
|
|
61
|
+
SELECT * FROM telemetry_provider_requests
|
|
62
|
+
WHERE turn_id = ? AND status NOT IN ('completed', 'error', 'aborted', 'timeout')
|
|
63
|
+
ORDER BY started_at ASC, created_at ASC
|
|
64
|
+
`).all(turnId);
|
|
65
|
+
return rows.map(providerRequestFromRow);
|
|
66
|
+
}
|
|
67
|
+
listActiveToolCallsForTurn(turnId) {
|
|
68
|
+
const rows = this.db.prepare(`
|
|
69
|
+
SELECT * FROM telemetry_tool_calls
|
|
70
|
+
WHERE turn_id = ? AND status NOT IN ('ok', 'error', 'aborted', 'timeout')
|
|
71
|
+
ORDER BY created_at ASC
|
|
72
|
+
`).all(turnId);
|
|
73
|
+
return rows.map(toolCallFromRow);
|
|
74
|
+
}
|
|
20
75
|
listProviderEventsPage(providerRequestId, input = {}) {
|
|
21
76
|
return listTelemetryProviderEventsPage(this.db, providerRequestId, input);
|
|
22
77
|
}
|
|
@@ -176,6 +231,53 @@ export class TelemetryStore {
|
|
|
176
231
|
const normalizedDelta = input.normalizedEventDelta ?? (input.normalizedType ? 1 : 0);
|
|
177
232
|
this.incrementProviderCounters(input.providerRequestId, input.eventType, receivedAt, byteSize, parseStatus, normalizedDelta);
|
|
178
233
|
}
|
|
234
|
+
recordProviderProgress(input) {
|
|
235
|
+
const existing = this.getProviderRequest(input.providerRequestId);
|
|
236
|
+
if (!existing)
|
|
237
|
+
return undefined;
|
|
238
|
+
const eventTypeCounts = { ...existing.eventTypeCounts };
|
|
239
|
+
for (const [eventType, delta] of Object.entries(input.eventTypeCounts ?? {})) {
|
|
240
|
+
if (!Number.isFinite(delta) || delta <= 0)
|
|
241
|
+
continue;
|
|
242
|
+
const current = typeof eventTypeCounts[eventType] === "number" ? eventTypeCounts[eventType] : 0;
|
|
243
|
+
eventTypeCounts[eventType] = current + delta;
|
|
244
|
+
}
|
|
245
|
+
const rawEventCount = Math.max(0, input.rawEventCount ?? 0);
|
|
246
|
+
const normalizedEventCount = Math.max(0, input.normalizedEventCount ?? 0);
|
|
247
|
+
const parseErrorCount = Math.max(0, input.parseErrorCount ?? 0);
|
|
248
|
+
const unknownEventCount = Math.max(0, input.unknownEventCount ?? 0);
|
|
249
|
+
const bytesReceived = Math.max(0, input.bytesReceived ?? 0);
|
|
250
|
+
const updatedAt = input.updatedAt ?? input.lastNormalizedEventAt ?? input.lastRawEventAt ?? new Date().toISOString();
|
|
251
|
+
this.db.prepare(`
|
|
252
|
+
UPDATE telemetry_provider_requests SET
|
|
253
|
+
status = CASE WHEN status IN ('completed', 'error', 'aborted', 'timeout') THEN status ELSE COALESCE(?, status) END,
|
|
254
|
+
last_raw_event_at = COALESCE(?, last_raw_event_at),
|
|
255
|
+
last_normalized_event_at = COALESCE(?, last_normalized_event_at),
|
|
256
|
+
upstream_response_id = COALESCE(?, upstream_response_id),
|
|
257
|
+
raw_event_count = raw_event_count + ?,
|
|
258
|
+
normalized_event_count = normalized_event_count + ?,
|
|
259
|
+
parse_error_count = parse_error_count + ?,
|
|
260
|
+
unknown_event_count = unknown_event_count + ?,
|
|
261
|
+
bytes_received = CASE WHEN ? = 0 THEN bytes_received ELSE COALESCE(bytes_received, 0) + ? END,
|
|
262
|
+
event_type_counts_json = ?,
|
|
263
|
+
updated_at = ?
|
|
264
|
+
WHERE provider_request_id = ?
|
|
265
|
+
`).run(input.status ?? null, input.lastRawEventAt ?? null, input.lastNormalizedEventAt ?? null, input.upstreamResponseId ?? null, rawEventCount, normalizedEventCount, parseErrorCount, unknownEventCount, bytesReceived, bytesReceived, JSON.stringify(eventTypeCounts), updatedAt, input.providerRequestId);
|
|
266
|
+
return {
|
|
267
|
+
...existing,
|
|
268
|
+
status: isTerminalProviderRequestStatus(existing.status) ? existing.status : input.status ?? existing.status,
|
|
269
|
+
lastRawEventAt: input.lastRawEventAt ?? existing.lastRawEventAt,
|
|
270
|
+
lastNormalizedEventAt: input.lastNormalizedEventAt ?? existing.lastNormalizedEventAt,
|
|
271
|
+
upstreamResponseId: input.upstreamResponseId ?? existing.upstreamResponseId,
|
|
272
|
+
rawEventCount: existing.rawEventCount + rawEventCount,
|
|
273
|
+
normalizedEventCount: existing.normalizedEventCount + normalizedEventCount,
|
|
274
|
+
parseErrorCount: existing.parseErrorCount + parseErrorCount,
|
|
275
|
+
unknownEventCount: existing.unknownEventCount + unknownEventCount,
|
|
276
|
+
bytesReceived: bytesReceived > 0 ? (existing.bytesReceived ?? 0) + bytesReceived : existing.bytesReceived,
|
|
277
|
+
eventTypeCounts,
|
|
278
|
+
updatedAt,
|
|
279
|
+
};
|
|
280
|
+
}
|
|
179
281
|
appendProviderEventSummary(input) {
|
|
180
282
|
const now = input.updatedAt ?? new Date().toISOString();
|
|
181
283
|
const receivedAt = input.receivedAt ?? now;
|
|
@@ -293,34 +395,15 @@ export class TelemetryStore {
|
|
|
293
395
|
return row.next_sequence;
|
|
294
396
|
}
|
|
295
397
|
incrementProviderCounters(providerRequestId, eventType, receivedAt, byteSize, parseStatus, normalizedDelta) {
|
|
296
|
-
|
|
297
|
-
if (!existing)
|
|
298
|
-
return;
|
|
299
|
-
const eventTypeCounts = { ...existing.eventTypeCounts };
|
|
300
|
-
const currentCount = typeof eventTypeCounts[eventType] === "number" ? eventTypeCounts[eventType] : 0;
|
|
301
|
-
eventTypeCounts[eventType] = currentCount + 1;
|
|
302
|
-
this.upsertProviderRequest({
|
|
398
|
+
this.recordProviderProgress({
|
|
303
399
|
providerRequestId,
|
|
304
|
-
piboSessionId: existing.piboSessionId,
|
|
305
|
-
rootSessionId: existing.rootSessionId,
|
|
306
|
-
roomId: existing.roomId,
|
|
307
|
-
turnId: existing.turnId,
|
|
308
|
-
phaseId: existing.phaseId,
|
|
309
|
-
provider: existing.provider,
|
|
310
|
-
api: existing.api,
|
|
311
|
-
model: existing.model,
|
|
312
|
-
transport: existing.transport,
|
|
313
|
-
serviceTier: existing.serviceTier,
|
|
314
|
-
status: existing.status,
|
|
315
400
|
lastRawEventAt: receivedAt,
|
|
316
|
-
rawEventCount:
|
|
317
|
-
normalizedEventCount:
|
|
318
|
-
parseErrorCount:
|
|
319
|
-
unknownEventCount:
|
|
320
|
-
bytesReceived:
|
|
321
|
-
eventTypeCounts,
|
|
322
|
-
captureMode: existing.captureMode,
|
|
323
|
-
retentionClass: existing.retentionClass,
|
|
401
|
+
rawEventCount: 1,
|
|
402
|
+
normalizedEventCount: normalizedDelta,
|
|
403
|
+
parseErrorCount: parseStatus === "invalid_json" ? 1 : 0,
|
|
404
|
+
unknownEventCount: parseStatus === "unknown_type" ? 1 : 0,
|
|
405
|
+
bytesReceived: byteSize,
|
|
406
|
+
eventTypeCounts: { [eventType]: 1 },
|
|
324
407
|
updatedAt: receivedAt,
|
|
325
408
|
});
|
|
326
409
|
}
|
|
@@ -347,6 +430,9 @@ export class BestEffortTelemetryService {
|
|
|
347
430
|
recordProviderEventSummary(input) {
|
|
348
431
|
this.safe(() => this.store?.recordProviderEventSummary(input));
|
|
349
432
|
}
|
|
433
|
+
recordProviderProgress(input) {
|
|
434
|
+
return this.safe(() => this.store?.recordProviderProgress(input));
|
|
435
|
+
}
|
|
350
436
|
appendProviderEventSummary(input) {
|
|
351
437
|
return this.safe(() => this.store?.appendProviderEventSummary(input));
|
|
352
438
|
}
|
|
@@ -363,6 +449,9 @@ export class BestEffortTelemetryService {
|
|
|
363
449
|
}
|
|
364
450
|
}
|
|
365
451
|
}
|
|
452
|
+
function isTerminalProviderRequestStatus(status) {
|
|
453
|
+
return status === "completed" || status === "error" || status === "aborted" || status === "timeout";
|
|
454
|
+
}
|
|
366
455
|
function fail(message) {
|
|
367
456
|
throw new Error(message);
|
|
368
457
|
}
|
package/dist/gateway/request.js
CHANGED
|
@@ -75,6 +75,7 @@ export async function sendGatewayMessageAndWaitForReply(event, options = {}) {
|
|
|
75
75
|
let settled = false;
|
|
76
76
|
let response;
|
|
77
77
|
let reply;
|
|
78
|
+
let messageFinished = false;
|
|
78
79
|
const timeout = setTimeout(() => {
|
|
79
80
|
finish(new Error(`Timed out waiting for assistant reply from session "${event.piboSessionId}"`));
|
|
80
81
|
}, timeoutMs);
|
|
@@ -91,14 +92,21 @@ export async function sendGatewayMessageAndWaitForReply(event, options = {}) {
|
|
|
91
92
|
resolve(result);
|
|
92
93
|
}
|
|
93
94
|
};
|
|
95
|
+
const finishCompletedReply = () => {
|
|
96
|
+
if (!response?.ok || !messageFinished)
|
|
97
|
+
return;
|
|
98
|
+
finish(reply
|
|
99
|
+
? { response, reply }
|
|
100
|
+
: new Error(`Session "${eventWithId.piboSessionId}" finished without an assistant reply`));
|
|
101
|
+
};
|
|
94
102
|
const handleFrame = (frame) => {
|
|
95
103
|
if (frame.type === "res" && frame.id === id) {
|
|
96
104
|
response = frame;
|
|
97
105
|
if (!frame.ok) {
|
|
98
106
|
finish(new Error(frame.error?.message ?? "Gateway rejected the message"));
|
|
99
107
|
}
|
|
100
|
-
else
|
|
101
|
-
|
|
108
|
+
else {
|
|
109
|
+
finishCompletedReply();
|
|
102
110
|
}
|
|
103
111
|
return;
|
|
104
112
|
}
|
|
@@ -111,13 +119,14 @@ export async function sendGatewayMessageAndWaitForReply(event, options = {}) {
|
|
|
111
119
|
finish(new Error(output.error));
|
|
112
120
|
return;
|
|
113
121
|
}
|
|
114
|
-
if (output.
|
|
115
|
-
|
|
116
|
-
|
|
122
|
+
if (output.piboSessionId !== eventWithId.piboSessionId || !("eventId" in output) || output.eventId !== eventWithId.id)
|
|
123
|
+
return;
|
|
124
|
+
if (output.type === "assistant_message") {
|
|
117
125
|
reply = output;
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
126
|
+
}
|
|
127
|
+
else if (output.type === "message_finished") {
|
|
128
|
+
messageFinished = true;
|
|
129
|
+
finishCompletedReply();
|
|
121
130
|
}
|
|
122
131
|
};
|
|
123
132
|
socket.once("connect", () => {
|