@pasko70/pibo 1.7.11 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/context-guard.js +172 -5
- package/dist/core/provider-telemetry.js +97 -4
- package/dist/core/routed-session.js +110 -16
- package/dist/core/runtime-telemetry.js +254 -94
- package/dist/core/runtime.js +72 -11
- package/dist/core/session-errors.js +16 -0
- package/dist/core/session-router.js +114 -21
- package/dist/data/telemetry.js +115 -26
- package/dist/gateway/cli.js +35 -16
- package/dist/gateway/pidfile.js +90 -44
- package/dist/gateway/server.js +22 -16
- package/dist/gateway/web.js +29 -23
- package/dist/ralph/service.js +60 -20
- package/dist/ralph/store.js +1 -1
- package/package.json +1 -1
- package/dist/apps/vscode-artifacts/latest.vsix +0 -0
- package/dist/apps/vscode-artifacts/pibo-vscode-ext-1.7.11.vsix +0 -0
package/dist/core/runtime.js
CHANGED
|
@@ -14,7 +14,7 @@ import { getMcpAgentContextFile } from "../mcp/agent-context.js";
|
|
|
14
14
|
import { createPiboSystemPromptTemplateExtension } from "./system-prompt-template.js";
|
|
15
15
|
import { getActivePiboBasePromptPath } from "./base-prompt.js";
|
|
16
16
|
import { createPiboCompactionPromptExtension } from "./compaction-prompt.js";
|
|
17
|
-
import { createPiboAssistantContextGuardExtension } from "./context-guard.js";
|
|
17
|
+
import { cancelPiboAssistantContextGuardRecovery, createPiboAssistantContextGuardExtension, createPiboAssistantContextGuardRecovery, isPiboAssistantContextGuardRecoveryPending, registerPiboAssistantContextGuardRecovery, } from "./context-guard.js";
|
|
18
18
|
import { getPiPackageRuntimeOptions } from "../pi-packages/runtime.js";
|
|
19
19
|
import { getDefaultPiboWorkspace } from "./workspace.js";
|
|
20
20
|
import { DEFAULT_USER_TIMEZONE } from "./user-settings.js";
|
|
@@ -25,6 +25,27 @@ import { PIBO_APP_CONTEXT } from "../app-context.js";
|
|
|
25
25
|
import { createRuntimeToolDefinition } from "../tools/runtime/tool.js";
|
|
26
26
|
import { RuntimeSessionRegistry } from "../tools/runtime/registry.js";
|
|
27
27
|
import { compactValidationToolResultForContext } from "./test-output-compaction.js";
|
|
28
|
+
function hasOwnRetrySetting(settings, key) {
|
|
29
|
+
return settings !== undefined && settings !== null && Object.prototype.hasOwnProperty.call(settings, key);
|
|
30
|
+
}
|
|
31
|
+
export function applyPiboRuntimeRetryDefaults(settingsManager, defaults) {
|
|
32
|
+
if (!defaults)
|
|
33
|
+
return;
|
|
34
|
+
const globalRetry = settingsManager.getGlobalSettings().retry;
|
|
35
|
+
const projectRetry = settingsManager.getProjectSettings().retry;
|
|
36
|
+
const overrides = {};
|
|
37
|
+
if (!hasOwnRetrySetting(globalRetry, "enabled") && !hasOwnRetrySetting(projectRetry, "enabled") && defaults.enabled !== undefined) {
|
|
38
|
+
overrides.enabled = defaults.enabled;
|
|
39
|
+
}
|
|
40
|
+
if (!hasOwnRetrySetting(globalRetry, "maxRetries") && !hasOwnRetrySetting(projectRetry, "maxRetries") && defaults.maxRetries !== undefined) {
|
|
41
|
+
overrides.maxRetries = defaults.maxRetries;
|
|
42
|
+
}
|
|
43
|
+
if (!hasOwnRetrySetting(globalRetry, "baseDelayMs") && !hasOwnRetrySetting(projectRetry, "baseDelayMs") && defaults.baseDelayMs !== undefined) {
|
|
44
|
+
overrides.baseDelayMs = defaults.baseDelayMs;
|
|
45
|
+
}
|
|
46
|
+
if (Object.keys(overrides).length > 0)
|
|
47
|
+
settingsManager.applyOverrides({ retry: overrides });
|
|
48
|
+
}
|
|
28
49
|
function resolveProfilePath(cwd, path) {
|
|
29
50
|
return isAbsolute(path) ? path : resolve(cwd, path);
|
|
30
51
|
}
|
|
@@ -145,10 +166,10 @@ function getBuiltinToolAllowlist(profile, customTools) {
|
|
|
145
166
|
return undefined;
|
|
146
167
|
return [...selectedBuiltinTools, ...customTools.map((tool) => tool.name)];
|
|
147
168
|
}
|
|
148
|
-
function getProfileExtensionFactories(profile, extensionFactories) {
|
|
169
|
+
function getProfileExtensionFactories(profile, extensionFactories, contextGuardRecovery) {
|
|
149
170
|
const piboPromptTemplateExtension = createPiboSystemPromptTemplateExtension();
|
|
150
171
|
const piboCompactionPromptExtension = createPiboCompactionPromptExtension();
|
|
151
|
-
const piboContextGuardExtension = createPiboAssistantContextGuardExtension();
|
|
172
|
+
const piboContextGuardExtension = createPiboAssistantContextGuardExtension({}, contextGuardRecovery);
|
|
152
173
|
const providerToolExtensions = profile.tools
|
|
153
174
|
.filter((tool) => tool.enabled !== false)
|
|
154
175
|
.filter(isWebSearchProviderTool)
|
|
@@ -213,6 +234,7 @@ export async function createPiboRuntime(options = {}) {
|
|
|
213
234
|
const sessionManager = await createSessionManager(cwd, profile, options.persistSession !== false);
|
|
214
235
|
const authStorage = AuthStorage.create();
|
|
215
236
|
const createRuntime = async ({ cwd: runtimeCwd, agentDir: runtimeAgentDir, sessionManager: runtimeSessionManager, sessionStartEvent, }) => {
|
|
237
|
+
const contextGuardRecovery = createPiboAssistantContextGuardRecovery();
|
|
216
238
|
const contextFiles = await loadContextFiles(runtimeCwd, profile.contextFiles);
|
|
217
239
|
const sessionContextFile = createSessionContextFile({ piboSessionId: profile.sessionId, ...options.sessionContext });
|
|
218
240
|
const installedToolContextFile = getInstalledCliToolContextFile();
|
|
@@ -226,7 +248,7 @@ export async function createPiboRuntime(options = {}) {
|
|
|
226
248
|
resourceLoaderOptions: {
|
|
227
249
|
...piPackageOptions.resourceLoaderOptions,
|
|
228
250
|
additionalSkillPaths: skillPaths,
|
|
229
|
-
extensionFactories: getProfileExtensionFactories(profile, options.extensionFactories),
|
|
251
|
+
extensionFactories: getProfileExtensionFactories(profile, options.extensionFactories, contextGuardRecovery),
|
|
230
252
|
noExtensions: true,
|
|
231
253
|
noSkills: true,
|
|
232
254
|
noPromptTemplates: true,
|
|
@@ -243,6 +265,7 @@ export async function createPiboRuntime(options = {}) {
|
|
|
243
265
|
}),
|
|
244
266
|
},
|
|
245
267
|
});
|
|
268
|
+
applyPiboRuntimeRetryDefaults(services.settingsManager, options.retryDefaults);
|
|
246
269
|
registerOpenAiGpt56Models(services.modelRegistry);
|
|
247
270
|
registerMiniMaxProvider(services.modelRegistry);
|
|
248
271
|
registerGlmProvider(services.modelRegistry);
|
|
@@ -271,6 +294,10 @@ export async function createPiboRuntime(options = {}) {
|
|
|
271
294
|
tools: getBuiltinToolAllowlist(profile, customTools),
|
|
272
295
|
});
|
|
273
296
|
installValidationOutputCompaction(created.session.agent);
|
|
297
|
+
registerPiboAssistantContextGuardRecovery(created.session, contextGuardRecovery);
|
|
298
|
+
if (options.contextGuardTuiQueueOrdering === true) {
|
|
299
|
+
installPiboContextGuardTuiQueueOrdering(created.session);
|
|
300
|
+
}
|
|
274
301
|
const resourceLoader = services.resourceLoader;
|
|
275
302
|
const diagnostics = [
|
|
276
303
|
...piPackageOptions.diagnostics,
|
|
@@ -281,13 +308,14 @@ export async function createPiboRuntime(options = {}) {
|
|
|
281
308
|
message: `Failed to load extension "${path}": ${error}`,
|
|
282
309
|
})),
|
|
283
310
|
];
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
created.session
|
|
311
|
+
const originalDispose = created.session.dispose.bind(created.session);
|
|
312
|
+
created.session.dispose = () => {
|
|
313
|
+
cancelPiboAssistantContextGuardRecovery(created.session, new Error("Context guard recovery cancelled because the Pi session was disposed"));
|
|
314
|
+
if (localRuntimeRegistry) {
|
|
287
315
|
void localRuntimeRegistry.closeControllerSessions(profile.sessionId ?? "local", { force: true });
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
}
|
|
316
|
+
}
|
|
317
|
+
originalDispose();
|
|
318
|
+
};
|
|
291
319
|
return {
|
|
292
320
|
...created,
|
|
293
321
|
services,
|
|
@@ -434,6 +462,39 @@ export async function inspectPiboProfile(options = {}) {
|
|
|
434
462
|
await runtime.dispose();
|
|
435
463
|
}
|
|
436
464
|
}
|
|
465
|
+
function installPiboContextGuardTuiQueueOrdering(session) {
|
|
466
|
+
const originalSubscribe = session.subscribe.bind(session);
|
|
467
|
+
const originalPrompt = session.prompt.bind(session);
|
|
468
|
+
const originalSteer = session.steer.bind(session);
|
|
469
|
+
session.subscribe = ((listener) => originalSubscribe((event) => {
|
|
470
|
+
if (event.type === "compaction_end"
|
|
471
|
+
&& event.result
|
|
472
|
+
&& isPiboAssistantContextGuardRecoveryPending(session)) {
|
|
473
|
+
listener({ ...event, willRetry: true });
|
|
474
|
+
return;
|
|
475
|
+
}
|
|
476
|
+
listener(event);
|
|
477
|
+
}));
|
|
478
|
+
session.prompt = async (text, options) => {
|
|
479
|
+
if (isPiboAssistantContextGuardRecoveryPending(session)) {
|
|
480
|
+
if (!session.isStreaming) {
|
|
481
|
+
await session.followUp(text, options?.images);
|
|
482
|
+
options?.preflightResult?.(true);
|
|
483
|
+
return;
|
|
484
|
+
}
|
|
485
|
+
await originalPrompt(text, { ...options, streamingBehavior: "followUp" });
|
|
486
|
+
return;
|
|
487
|
+
}
|
|
488
|
+
await originalPrompt(text, options);
|
|
489
|
+
};
|
|
490
|
+
session.steer = async (text, images) => {
|
|
491
|
+
if (isPiboAssistantContextGuardRecoveryPending(session)) {
|
|
492
|
+
await session.followUp(text, images);
|
|
493
|
+
return;
|
|
494
|
+
}
|
|
495
|
+
await originalSteer(text, images);
|
|
496
|
+
};
|
|
497
|
+
}
|
|
437
498
|
export async function runPiboTui(options = {}) {
|
|
438
499
|
const profile = options.profile ?? createDefaultPiboProfile();
|
|
439
500
|
const hasEnabledSubagents = profile.subagents.some((subagent) => subagent.enabled !== false);
|
|
@@ -443,7 +504,7 @@ export async function runPiboTui(options = {}) {
|
|
|
443
504
|
process.exitCode = 1;
|
|
444
505
|
return;
|
|
445
506
|
}
|
|
446
|
-
const runtime = await createPiboRuntime({ ...options, profile });
|
|
507
|
+
const runtime = await createPiboRuntime({ ...options, profile, contextGuardTuiQueueOrdering: true });
|
|
447
508
|
try {
|
|
448
509
|
const fatal = runtime.diagnostics.find((diagnostic) => diagnostic.type === "error");
|
|
449
510
|
for (const diagnostic of runtime.diagnostics) {
|
|
@@ -1,3 +1,16 @@
|
|
|
1
|
+
const PROVIDER_NETWORK_ERROR_MARKERS = [
|
|
2
|
+
"fetch failed",
|
|
3
|
+
"network error",
|
|
4
|
+
"connection error",
|
|
5
|
+
"connection refused",
|
|
6
|
+
"connection lost",
|
|
7
|
+
"connection reset",
|
|
8
|
+
"other side closed",
|
|
9
|
+
"upstream connect",
|
|
10
|
+
"reset before headers",
|
|
11
|
+
"socket hang up",
|
|
12
|
+
"socket connection was closed",
|
|
13
|
+
];
|
|
1
14
|
export function classifySessionErrorMessage(message, options = {}) {
|
|
2
15
|
const normalized = message.toLowerCase();
|
|
3
16
|
if (normalized.includes("context_length_exceeded") || normalized.includes("context window")) {
|
|
@@ -18,6 +31,9 @@ export function classifySessionErrorMessage(message, options = {}) {
|
|
|
18
31
|
if (normalized.includes("timeout") || normalized.includes("timed out")) {
|
|
19
32
|
return { category: "provider_transport", errorClass: "provider_transport", code: "timeout", origin: "provider", retryable: true, userMessage: "The provider request timed out." };
|
|
20
33
|
}
|
|
34
|
+
if (PROVIDER_NETWORK_ERROR_MARKERS.some((marker) => normalized.includes(marker))) {
|
|
35
|
+
return { category: "provider_transport", errorClass: "provider_transport", code: "network_error", origin: "provider", retryable: true, userMessage: "The provider network connection failed." };
|
|
36
|
+
}
|
|
21
37
|
if (/\b5\d\d\b/.test(normalized)) {
|
|
22
38
|
return { category: "provider_server", errorClass: "provider_server", code: "provider_server_error", origin: "provider", retryable: true, userMessage: "The provider returned a server error." };
|
|
23
39
|
}
|
|
@@ -20,6 +20,15 @@ import { withWorkflowSessionKind } from "../sessions/workflow-session-kind.js";
|
|
|
20
20
|
import { PiboRuntimeTelemetryRecorder } from "./runtime-telemetry.js";
|
|
21
21
|
import { createPiboProviderTelemetryExtension } from "./provider-telemetry.js";
|
|
22
22
|
const DEFAULT_SUBAGENT_REPLY_TIMEOUT_MS = 10 * 60 * 1000;
|
|
23
|
+
const DEFAULT_ROUTED_SESSION_IDLE_TIMEOUT_MS = 30 * 60 * 1000;
|
|
24
|
+
export const RALPH_RUNTIME_RETRY_DEFAULTS = {
|
|
25
|
+
enabled: true,
|
|
26
|
+
maxRetries: 7,
|
|
27
|
+
baseDelayMs: 2_000,
|
|
28
|
+
};
|
|
29
|
+
export function resolvePiboSessionRetryDefaults(kind, configured) {
|
|
30
|
+
return configured ?? (kind === "ralph" ? RALPH_RUNTIME_RETRY_DEFAULTS : undefined);
|
|
31
|
+
}
|
|
23
32
|
export function resolvePiboSessionInitialThinkingLevel(session) {
|
|
24
33
|
const value = session.metadata?.initialThinkingLevel;
|
|
25
34
|
return typeof value === "string" && isPiboThinkingLevel(value) ? value : undefined;
|
|
@@ -124,6 +133,8 @@ export class PiboSessionRouter {
|
|
|
124
133
|
signalRegistry;
|
|
125
134
|
runtimeRegistry;
|
|
126
135
|
scheduledRunReminders = new Map();
|
|
136
|
+
idleSessionTimers = new Map();
|
|
137
|
+
routedSessionIdleTimeoutMs;
|
|
127
138
|
baseProfile;
|
|
128
139
|
pluginRegistry;
|
|
129
140
|
sessionStore;
|
|
@@ -138,6 +149,12 @@ export class PiboSessionRouter {
|
|
|
138
149
|
this.telemetryRecorder = this.telemetryStore
|
|
139
150
|
? new PiboRuntimeTelemetryRecorder(this.telemetryStore, undefined, { providerEventMode: providerEventTelemetryModeFromEnv() })
|
|
140
151
|
: undefined;
|
|
152
|
+
const idleTimeoutMs = options.routedSessionIdleTimeoutMs;
|
|
153
|
+
this.routedSessionIdleTimeoutMs = idleTimeoutMs === false
|
|
154
|
+
? false
|
|
155
|
+
: typeof idleTimeoutMs === "number" && Number.isFinite(idleTimeoutMs) && idleTimeoutMs > 0
|
|
156
|
+
? idleTimeoutMs
|
|
157
|
+
: DEFAULT_ROUTED_SESSION_IDLE_TIMEOUT_MS;
|
|
141
158
|
const defaultProfileName = selectDefaultPiboProfileName(this.pluginRegistry);
|
|
142
159
|
this.baseProfile = options.profile ?? createPiboProfileFromRegistryOrDefault(this.pluginRegistry, defaultProfileName);
|
|
143
160
|
this.reliabilityStore = options.reliabilityStore ?? (options.persistSession === false ? undefined : createDefaultPiboReliabilityStore());
|
|
@@ -157,20 +174,29 @@ export class PiboSessionRouter {
|
|
|
157
174
|
}
|
|
158
175
|
async emit(event) {
|
|
159
176
|
const session = await this.getOrCreateSession(event.piboSessionId);
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
177
|
+
this.clearIdleSessionTimer(event.piboSessionId);
|
|
178
|
+
try {
|
|
179
|
+
if (event.type === "message") {
|
|
180
|
+
return session.enqueueMessage(event);
|
|
181
|
+
}
|
|
182
|
+
const output = await session.executeAction(event);
|
|
183
|
+
if (event.action === "abort") {
|
|
184
|
+
this.signalRegistry.project({ type: "session_interrupted", piboSessionId: event.piboSessionId, reason: "abort action" });
|
|
185
|
+
}
|
|
186
|
+
if (event.action === "dispose") {
|
|
187
|
+
await this.disposeSessionSubtree(event.piboSessionId, "dispose action", { cancelRuns: true });
|
|
188
|
+
}
|
|
189
|
+
else if (event.action === "kill" || event.action === "kill_all") {
|
|
190
|
+
await this.disposeSessionSubtree(event.piboSessionId, `${event.action} action`, { cancelRuns: event.action === "kill_all" });
|
|
191
|
+
}
|
|
192
|
+
else if (shouldResetSessionAfterAction(event.action)) {
|
|
193
|
+
await this.resetCachedSession(event.piboSessionId, "provider auth changed");
|
|
194
|
+
}
|
|
195
|
+
return output;
|
|
169
196
|
}
|
|
170
|
-
|
|
171
|
-
|
|
197
|
+
finally {
|
|
198
|
+
this.scheduleIdleSessionEvictionIfIdle(event.piboSessionId);
|
|
172
199
|
}
|
|
173
|
-
return output;
|
|
174
200
|
}
|
|
175
201
|
async killSession(piboSessionId, options) {
|
|
176
202
|
const killed = [];
|
|
@@ -182,23 +208,32 @@ export class PiboSessionRouter {
|
|
|
182
208
|
const runs = this.runRegistry.cancelControllerRuns(piboSessionId);
|
|
183
209
|
cancelledRuns.push(...runs.map((r) => r.runId));
|
|
184
210
|
}
|
|
185
|
-
await this.runtimeRegistry.closeControllerSessions(piboSessionId, { force: true });
|
|
186
211
|
this.signalRegistry.project({ type: "session_interrupted", piboSessionId, reason: "kill" });
|
|
187
212
|
const children = await this.killChildSessions(piboSessionId, options);
|
|
188
213
|
killed.push(...children.killed);
|
|
189
214
|
cancelledRuns.push(...children.cancelledRuns);
|
|
215
|
+
await this.disposeSessionSubtree(piboSessionId, "kill", { cancelRuns: false });
|
|
190
216
|
}
|
|
191
217
|
return { killed, cancelledRuns };
|
|
192
218
|
}
|
|
193
|
-
|
|
194
|
-
const
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
219
|
+
async disposeSessionSubtree(piboSessionId, reason, options) {
|
|
220
|
+
const ids = [piboSessionId, ...this.descendantSessionIds(piboSessionId)];
|
|
221
|
+
const sessions = [];
|
|
222
|
+
for (const id of ids) {
|
|
223
|
+
if (options.cancelRuns)
|
|
224
|
+
this.runRegistry.cancelControllerRuns(id);
|
|
225
|
+
this.clearIdleSessionTimer(id);
|
|
199
226
|
this.scheduledRunReminders.delete(id);
|
|
227
|
+
const cached = this.sessions.get(id);
|
|
228
|
+
if (cached)
|
|
229
|
+
sessions.push(cached);
|
|
200
230
|
this.sessions.delete(id);
|
|
201
231
|
}
|
|
232
|
+
await Promise.all(ids.map((id) => this.runtimeRegistry.closeControllerSessions(id, { force: true })));
|
|
233
|
+
await Promise.all(sessions.map((session) => session.dispose()));
|
|
234
|
+
for (const id of ids) {
|
|
235
|
+
this.signalRegistry.project({ type: "session_disposed", piboSessionId: id, reason });
|
|
236
|
+
}
|
|
202
237
|
}
|
|
203
238
|
descendantSessionIds(parentId) {
|
|
204
239
|
const output = [];
|
|
@@ -319,6 +354,9 @@ export class PiboSessionRouter {
|
|
|
319
354
|
async disposeAll() {
|
|
320
355
|
const sessions = [...this.sessions.values()];
|
|
321
356
|
this.sessions.clear();
|
|
357
|
+
for (const timer of this.idleSessionTimers.values())
|
|
358
|
+
clearTimeout(timer);
|
|
359
|
+
this.idleSessionTimers.clear();
|
|
322
360
|
this.runRegistry.cancelAll("Pibo session router was disposed.");
|
|
323
361
|
for (const session of sessions)
|
|
324
362
|
this.signalRegistry.project({ type: "session_disposed", piboSessionId: session.getStatus().piboSessionId, reason: "router disposed" });
|
|
@@ -326,10 +364,54 @@ export class PiboSessionRouter {
|
|
|
326
364
|
await this.runtimeRegistry.closeAll({ force: true });
|
|
327
365
|
await Promise.all(sessions.map((session) => session.dispose()));
|
|
328
366
|
}
|
|
367
|
+
clearIdleSessionTimer(piboSessionId) {
|
|
368
|
+
const timer = this.idleSessionTimers.get(piboSessionId);
|
|
369
|
+
if (timer)
|
|
370
|
+
clearTimeout(timer);
|
|
371
|
+
this.idleSessionTimers.delete(piboSessionId);
|
|
372
|
+
}
|
|
373
|
+
scheduleIdleSessionEvictionIfIdle(piboSessionId) {
|
|
374
|
+
if (this.routedSessionIdleTimeoutMs === false)
|
|
375
|
+
return;
|
|
376
|
+
const session = this.sessions.get(piboSessionId);
|
|
377
|
+
if (!session)
|
|
378
|
+
return;
|
|
379
|
+
const status = session.getStatus();
|
|
380
|
+
if (status.disposed || status.processing || status.streaming || status.queuedMessages > 0) {
|
|
381
|
+
this.clearIdleSessionTimer(piboSessionId);
|
|
382
|
+
return;
|
|
383
|
+
}
|
|
384
|
+
this.clearIdleSessionTimer(piboSessionId);
|
|
385
|
+
const timer = setTimeout(() => {
|
|
386
|
+
this.idleSessionTimers.delete(piboSessionId);
|
|
387
|
+
void this.evictIdleSession(piboSessionId, session).catch((error) => {
|
|
388
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
389
|
+
this.emitOutput({
|
|
390
|
+
type: "session_error",
|
|
391
|
+
piboSessionId,
|
|
392
|
+
error: `Failed to dispose idle routed runtime: ${message}`,
|
|
393
|
+
errorDetails: runtimeSessionErrorDetails(message),
|
|
394
|
+
});
|
|
395
|
+
});
|
|
396
|
+
}, this.routedSessionIdleTimeoutMs);
|
|
397
|
+
timer.unref();
|
|
398
|
+
this.idleSessionTimers.set(piboSessionId, timer);
|
|
399
|
+
}
|
|
400
|
+
async evictIdleSession(piboSessionId, expected) {
|
|
401
|
+
const current = this.sessions.get(piboSessionId);
|
|
402
|
+
if (current !== expected)
|
|
403
|
+
return;
|
|
404
|
+
const status = current.getStatus();
|
|
405
|
+
if (status.disposed || status.processing || status.streaming || status.queuedMessages > 0)
|
|
406
|
+
return;
|
|
407
|
+
await this.resetCachedSession(piboSessionId, "routed runtime idle timeout");
|
|
408
|
+
}
|
|
329
409
|
async getOrCreateSession(piboSessionId) {
|
|
330
410
|
const existing = this.sessions.get(piboSessionId);
|
|
331
|
-
if (existing)
|
|
411
|
+
if (existing) {
|
|
412
|
+
this.clearIdleSessionTimer(piboSessionId);
|
|
332
413
|
return existing;
|
|
414
|
+
}
|
|
333
415
|
const pending = this.pendingSessions.get(piboSessionId);
|
|
334
416
|
if (pending)
|
|
335
417
|
return pending;
|
|
@@ -360,6 +442,7 @@ export class PiboSessionRouter {
|
|
|
360
442
|
cwd: piboSession.workspace ?? this.options.cwd,
|
|
361
443
|
persistSession: this.options.persistSession,
|
|
362
444
|
thinkingLevel: initialThinkingLevel ?? this.options.thinkingLevel,
|
|
445
|
+
retryDefaults: resolvePiboSessionRetryDefaults(piboSession.kind, this.options.retryDefaults),
|
|
363
446
|
profile: profileForSession(profile, piboSession.piSessionId, parentPiSessionId),
|
|
364
447
|
extensionFactories: [
|
|
365
448
|
...(telemetryExtension ? [telemetryExtension] : []),
|
|
@@ -379,7 +462,16 @@ export class PiboSessionRouter {
|
|
|
379
462
|
const initialFastMode = resolvePiboSessionInitialFastMode(piboSession) ?? selectRequestedFastMode(profileForSession(profile, piboSession.piSessionId, parentPiSessionId), modelDefaults) ?? false;
|
|
380
463
|
const session = new RoutedSession(piboSession.id, runtime, this.emitOutput, this.pluginRegistry, this.options.forwardPiEvents ?? false, this.telemetryRecorder
|
|
381
464
|
? (id, event, context) => this.telemetryRecorder?.recordPiEvent(id, event, { session: this.sessionStore.get(id), status: context.status, activeEventId: context.activeEventId })
|
|
382
|
-
: undefined, initialFastMode, (result, event) => this.handleSessionOperation(result, event), (id, opts) => this.killChildSessions(id, opts), (state) =>
|
|
465
|
+
: undefined, initialFastMode, (result, event) => this.handleSessionOperation(result, event), (id, opts) => this.killChildSessions(id, opts), (state) => {
|
|
466
|
+
this.signalRegistry.project({ type: "session_processing_changed", piboSessionId: piboSession.id, processing: state.processing, queuedMessages: state.queuedMessages });
|
|
467
|
+
if (state.disposed || state.processing || state.queuedMessages > 0)
|
|
468
|
+
this.clearIdleSessionTimer(piboSession.id);
|
|
469
|
+
else
|
|
470
|
+
this.scheduleIdleSessionEvictionIfIdle(piboSession.id);
|
|
471
|
+
}, (messages, reason) => this.telemetryRecorder?.recordMessagesInterrupted(messages, {
|
|
472
|
+
session: this.sessionStore.get(piboSession.id),
|
|
473
|
+
status: this.sessions.get(piboSession.id)?.getStatus(),
|
|
474
|
+
}, reason));
|
|
383
475
|
this.sessions.set(piboSession.id, session);
|
|
384
476
|
return session;
|
|
385
477
|
}
|
|
@@ -438,6 +530,7 @@ export class PiboSessionRouter {
|
|
|
438
530
|
}
|
|
439
531
|
async resetCachedSession(piboSessionId, reason) {
|
|
440
532
|
const cached = this.sessions.get(piboSessionId);
|
|
533
|
+
this.clearIdleSessionTimer(piboSessionId);
|
|
441
534
|
this.sessions.delete(piboSessionId);
|
|
442
535
|
await this.runtimeRegistry.closeControllerSessions(piboSessionId, { force: true });
|
|
443
536
|
await cached?.dispose();
|
package/dist/data/telemetry.js
CHANGED
|
@@ -17,6 +17,61 @@ export class TelemetryStore {
|
|
|
17
17
|
getTurnTimeline(turnIdOrEventId, input = {}) {
|
|
18
18
|
return getTelemetryTurnTimeline(this.db, turnIdOrEventId, input);
|
|
19
19
|
}
|
|
20
|
+
getOpenPhaseForTurn(turnId, name) {
|
|
21
|
+
const row = this.db.prepare(`
|
|
22
|
+
SELECT * FROM telemetry_phases
|
|
23
|
+
WHERE turn_id = ? AND name = ? AND status = 'open'
|
|
24
|
+
ORDER BY COALESCE(last_progress_at, started_at) DESC, created_at DESC
|
|
25
|
+
LIMIT 1
|
|
26
|
+
`).get(turnId, name);
|
|
27
|
+
return row ? phaseFromRow(row) : undefined;
|
|
28
|
+
}
|
|
29
|
+
countPhasesForTurn(turnId, name) {
|
|
30
|
+
const row = this.db.prepare("SELECT COUNT(*) AS count FROM telemetry_phases WHERE turn_id = ? AND name = ?").get(turnId, name);
|
|
31
|
+
return Number(row.count);
|
|
32
|
+
}
|
|
33
|
+
listOpenPhasesForTurn(turnId) {
|
|
34
|
+
const rows = this.db.prepare(`
|
|
35
|
+
SELECT * FROM telemetry_phases
|
|
36
|
+
WHERE turn_id = ? AND status = 'open'
|
|
37
|
+
ORDER BY started_at ASC, created_at ASC
|
|
38
|
+
`).all(turnId);
|
|
39
|
+
return rows.map(phaseFromRow);
|
|
40
|
+
}
|
|
41
|
+
getLatestProviderRequestForTurn(turnId) {
|
|
42
|
+
const row = this.db.prepare(`
|
|
43
|
+
SELECT * FROM telemetry_provider_requests
|
|
44
|
+
WHERE turn_id = ?
|
|
45
|
+
ORDER BY started_at DESC, created_at DESC
|
|
46
|
+
LIMIT 1
|
|
47
|
+
`).get(turnId);
|
|
48
|
+
return row ? providerRequestFromRow(row) : undefined;
|
|
49
|
+
}
|
|
50
|
+
getActiveProviderRequestForTurn(turnId) {
|
|
51
|
+
const row = this.db.prepare(`
|
|
52
|
+
SELECT * FROM telemetry_provider_requests
|
|
53
|
+
WHERE turn_id = ? AND status NOT IN ('completed', 'error', 'aborted', 'timeout')
|
|
54
|
+
ORDER BY started_at DESC, created_at DESC
|
|
55
|
+
LIMIT 1
|
|
56
|
+
`).get(turnId);
|
|
57
|
+
return row ? providerRequestFromRow(row) : undefined;
|
|
58
|
+
}
|
|
59
|
+
listActiveProviderRequestsForTurn(turnId) {
|
|
60
|
+
const rows = this.db.prepare(`
|
|
61
|
+
SELECT * FROM telemetry_provider_requests
|
|
62
|
+
WHERE turn_id = ? AND status NOT IN ('completed', 'error', 'aborted', 'timeout')
|
|
63
|
+
ORDER BY started_at ASC, created_at ASC
|
|
64
|
+
`).all(turnId);
|
|
65
|
+
return rows.map(providerRequestFromRow);
|
|
66
|
+
}
|
|
67
|
+
listActiveToolCallsForTurn(turnId) {
|
|
68
|
+
const rows = this.db.prepare(`
|
|
69
|
+
SELECT * FROM telemetry_tool_calls
|
|
70
|
+
WHERE turn_id = ? AND status NOT IN ('ok', 'error', 'aborted', 'timeout')
|
|
71
|
+
ORDER BY created_at ASC
|
|
72
|
+
`).all(turnId);
|
|
73
|
+
return rows.map(toolCallFromRow);
|
|
74
|
+
}
|
|
20
75
|
listProviderEventsPage(providerRequestId, input = {}) {
|
|
21
76
|
return listTelemetryProviderEventsPage(this.db, providerRequestId, input);
|
|
22
77
|
}
|
|
@@ -176,6 +231,53 @@ export class TelemetryStore {
|
|
|
176
231
|
const normalizedDelta = input.normalizedEventDelta ?? (input.normalizedType ? 1 : 0);
|
|
177
232
|
this.incrementProviderCounters(input.providerRequestId, input.eventType, receivedAt, byteSize, parseStatus, normalizedDelta);
|
|
178
233
|
}
|
|
234
|
+
recordProviderProgress(input) {
|
|
235
|
+
const existing = this.getProviderRequest(input.providerRequestId);
|
|
236
|
+
if (!existing)
|
|
237
|
+
return undefined;
|
|
238
|
+
const eventTypeCounts = { ...existing.eventTypeCounts };
|
|
239
|
+
for (const [eventType, delta] of Object.entries(input.eventTypeCounts ?? {})) {
|
|
240
|
+
if (!Number.isFinite(delta) || delta <= 0)
|
|
241
|
+
continue;
|
|
242
|
+
const current = typeof eventTypeCounts[eventType] === "number" ? eventTypeCounts[eventType] : 0;
|
|
243
|
+
eventTypeCounts[eventType] = current + delta;
|
|
244
|
+
}
|
|
245
|
+
const rawEventCount = Math.max(0, input.rawEventCount ?? 0);
|
|
246
|
+
const normalizedEventCount = Math.max(0, input.normalizedEventCount ?? 0);
|
|
247
|
+
const parseErrorCount = Math.max(0, input.parseErrorCount ?? 0);
|
|
248
|
+
const unknownEventCount = Math.max(0, input.unknownEventCount ?? 0);
|
|
249
|
+
const bytesReceived = Math.max(0, input.bytesReceived ?? 0);
|
|
250
|
+
const updatedAt = input.updatedAt ?? input.lastNormalizedEventAt ?? input.lastRawEventAt ?? new Date().toISOString();
|
|
251
|
+
this.db.prepare(`
|
|
252
|
+
UPDATE telemetry_provider_requests SET
|
|
253
|
+
status = CASE WHEN status IN ('completed', 'error', 'aborted', 'timeout') THEN status ELSE COALESCE(?, status) END,
|
|
254
|
+
last_raw_event_at = COALESCE(?, last_raw_event_at),
|
|
255
|
+
last_normalized_event_at = COALESCE(?, last_normalized_event_at),
|
|
256
|
+
upstream_response_id = COALESCE(?, upstream_response_id),
|
|
257
|
+
raw_event_count = raw_event_count + ?,
|
|
258
|
+
normalized_event_count = normalized_event_count + ?,
|
|
259
|
+
parse_error_count = parse_error_count + ?,
|
|
260
|
+
unknown_event_count = unknown_event_count + ?,
|
|
261
|
+
bytes_received = CASE WHEN ? = 0 THEN bytes_received ELSE COALESCE(bytes_received, 0) + ? END,
|
|
262
|
+
event_type_counts_json = ?,
|
|
263
|
+
updated_at = ?
|
|
264
|
+
WHERE provider_request_id = ?
|
|
265
|
+
`).run(input.status ?? null, input.lastRawEventAt ?? null, input.lastNormalizedEventAt ?? null, input.upstreamResponseId ?? null, rawEventCount, normalizedEventCount, parseErrorCount, unknownEventCount, bytesReceived, bytesReceived, JSON.stringify(eventTypeCounts), updatedAt, input.providerRequestId);
|
|
266
|
+
return {
|
|
267
|
+
...existing,
|
|
268
|
+
status: isTerminalProviderRequestStatus(existing.status) ? existing.status : input.status ?? existing.status,
|
|
269
|
+
lastRawEventAt: input.lastRawEventAt ?? existing.lastRawEventAt,
|
|
270
|
+
lastNormalizedEventAt: input.lastNormalizedEventAt ?? existing.lastNormalizedEventAt,
|
|
271
|
+
upstreamResponseId: input.upstreamResponseId ?? existing.upstreamResponseId,
|
|
272
|
+
rawEventCount: existing.rawEventCount + rawEventCount,
|
|
273
|
+
normalizedEventCount: existing.normalizedEventCount + normalizedEventCount,
|
|
274
|
+
parseErrorCount: existing.parseErrorCount + parseErrorCount,
|
|
275
|
+
unknownEventCount: existing.unknownEventCount + unknownEventCount,
|
|
276
|
+
bytesReceived: bytesReceived > 0 ? (existing.bytesReceived ?? 0) + bytesReceived : existing.bytesReceived,
|
|
277
|
+
eventTypeCounts,
|
|
278
|
+
updatedAt,
|
|
279
|
+
};
|
|
280
|
+
}
|
|
179
281
|
appendProviderEventSummary(input) {
|
|
180
282
|
const now = input.updatedAt ?? new Date().toISOString();
|
|
181
283
|
const receivedAt = input.receivedAt ?? now;
|
|
@@ -293,34 +395,15 @@ export class TelemetryStore {
|
|
|
293
395
|
return row.next_sequence;
|
|
294
396
|
}
|
|
295
397
|
incrementProviderCounters(providerRequestId, eventType, receivedAt, byteSize, parseStatus, normalizedDelta) {
|
|
296
|
-
|
|
297
|
-
if (!existing)
|
|
298
|
-
return;
|
|
299
|
-
const eventTypeCounts = { ...existing.eventTypeCounts };
|
|
300
|
-
const currentCount = typeof eventTypeCounts[eventType] === "number" ? eventTypeCounts[eventType] : 0;
|
|
301
|
-
eventTypeCounts[eventType] = currentCount + 1;
|
|
302
|
-
this.upsertProviderRequest({
|
|
398
|
+
this.recordProviderProgress({
|
|
303
399
|
providerRequestId,
|
|
304
|
-
piboSessionId: existing.piboSessionId,
|
|
305
|
-
rootSessionId: existing.rootSessionId,
|
|
306
|
-
roomId: existing.roomId,
|
|
307
|
-
turnId: existing.turnId,
|
|
308
|
-
phaseId: existing.phaseId,
|
|
309
|
-
provider: existing.provider,
|
|
310
|
-
api: existing.api,
|
|
311
|
-
model: existing.model,
|
|
312
|
-
transport: existing.transport,
|
|
313
|
-
serviceTier: existing.serviceTier,
|
|
314
|
-
status: existing.status,
|
|
315
400
|
lastRawEventAt: receivedAt,
|
|
316
|
-
rawEventCount:
|
|
317
|
-
normalizedEventCount:
|
|
318
|
-
parseErrorCount:
|
|
319
|
-
unknownEventCount:
|
|
320
|
-
bytesReceived:
|
|
321
|
-
eventTypeCounts,
|
|
322
|
-
captureMode: existing.captureMode,
|
|
323
|
-
retentionClass: existing.retentionClass,
|
|
401
|
+
rawEventCount: 1,
|
|
402
|
+
normalizedEventCount: normalizedDelta,
|
|
403
|
+
parseErrorCount: parseStatus === "invalid_json" ? 1 : 0,
|
|
404
|
+
unknownEventCount: parseStatus === "unknown_type" ? 1 : 0,
|
|
405
|
+
bytesReceived: byteSize,
|
|
406
|
+
eventTypeCounts: { [eventType]: 1 },
|
|
324
407
|
updatedAt: receivedAt,
|
|
325
408
|
});
|
|
326
409
|
}
|
|
@@ -347,6 +430,9 @@ export class BestEffortTelemetryService {
|
|
|
347
430
|
recordProviderEventSummary(input) {
|
|
348
431
|
this.safe(() => this.store?.recordProviderEventSummary(input));
|
|
349
432
|
}
|
|
433
|
+
recordProviderProgress(input) {
|
|
434
|
+
return this.safe(() => this.store?.recordProviderProgress(input));
|
|
435
|
+
}
|
|
350
436
|
appendProviderEventSummary(input) {
|
|
351
437
|
return this.safe(() => this.store?.appendProviderEventSummary(input));
|
|
352
438
|
}
|
|
@@ -363,6 +449,9 @@ export class BestEffortTelemetryService {
|
|
|
363
449
|
}
|
|
364
450
|
}
|
|
365
451
|
}
|
|
452
|
+
function isTerminalProviderRequestStatus(status) {
|
|
453
|
+
return status === "completed" || status === "error" || status === "aborted" || status === "timeout";
|
|
454
|
+
}
|
|
366
455
|
function fail(message) {
|
|
367
456
|
throw new Error(message);
|
|
368
457
|
}
|
package/dist/gateway/cli.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { spawn, execFile } from "node:child_process";
|
|
2
|
-
import { existsSync, mkdirSync, openSync, readFileSync } from "node:fs";
|
|
2
|
+
import { existsSync, mkdirSync, openSync, readFileSync, readdirSync } from "node:fs";
|
|
3
3
|
import { homedir } from "node:os";
|
|
4
4
|
import { join } from "node:path";
|
|
5
5
|
import { promisify } from "node:util";
|
|
@@ -87,28 +87,40 @@ function resolveGatewayWebCommand(argv, target) {
|
|
|
87
87
|
];
|
|
88
88
|
return { command: process.execPath, args };
|
|
89
89
|
}
|
|
90
|
-
function
|
|
91
|
-
|
|
90
|
+
function managedGatewayPidPaths(target) {
|
|
91
|
+
const home = managedGatewayHome(target);
|
|
92
|
+
try {
|
|
93
|
+
const legacy = readdirSync(home)
|
|
94
|
+
.filter((name) => /^gateway-\d+\.pid$/.test(name))
|
|
95
|
+
.map((name) => join(home, name));
|
|
96
|
+
return [join(home, "gateway.pid"), ...legacy];
|
|
97
|
+
}
|
|
98
|
+
catch {
|
|
99
|
+
return [join(home, "gateway.pid")];
|
|
100
|
+
}
|
|
92
101
|
}
|
|
93
102
|
function readManagedGatewayPid(target) {
|
|
94
|
-
|
|
95
|
-
const path = managedGatewayPidPath(target);
|
|
96
|
-
if (!existsSync(path))
|
|
97
|
-
return undefined;
|
|
98
|
-
const pid = Number(readFileSync(path, "utf-8").trim());
|
|
99
|
-
if (!Number.isInteger(pid) || pid <= 0)
|
|
100
|
-
return undefined;
|
|
103
|
+
for (const path of managedGatewayPidPaths(target)) {
|
|
101
104
|
try {
|
|
102
|
-
|
|
103
|
-
|
|
105
|
+
if (!existsSync(path))
|
|
106
|
+
continue;
|
|
107
|
+
const pid = Number(readFileSync(path, "utf-8").trim());
|
|
108
|
+
if (!Number.isInteger(pid) || pid <= 0)
|
|
109
|
+
continue;
|
|
110
|
+
try {
|
|
111
|
+
process.kill(pid, 0);
|
|
112
|
+
return pid;
|
|
113
|
+
}
|
|
114
|
+
catch (error) {
|
|
115
|
+
if (error.code === "EPERM")
|
|
116
|
+
return pid;
|
|
117
|
+
}
|
|
104
118
|
}
|
|
105
119
|
catch {
|
|
106
|
-
|
|
120
|
+
// Try the next current or legacy PID file.
|
|
107
121
|
}
|
|
108
122
|
}
|
|
109
|
-
|
|
110
|
-
return undefined;
|
|
111
|
-
}
|
|
123
|
+
return undefined;
|
|
112
124
|
}
|
|
113
125
|
async function waitForTargetGatewayDown(target, maxRetries = 40, intervalMs = 250) {
|
|
114
126
|
const port = targetPort(target);
|
|
@@ -322,6 +334,13 @@ async function runManagedGatewayCommand(target, command, args, argv = process.ar
|
|
|
322
334
|
process.exitCode = 1;
|
|
323
335
|
return true;
|
|
324
336
|
}
|
|
337
|
+
const existingPid = readManagedGatewayPid(target);
|
|
338
|
+
if (existingPid !== undefined) {
|
|
339
|
+
console.error(`Start blocked: ${managedGatewayHome(target)} is already owned by gateway PID ${existingPid}.`);
|
|
340
|
+
console.error(`The configured status port ${targetPort(target)} is not reachable; check the gateway port configuration instead of starting a second gateway with the same PIBO_HOME.`);
|
|
341
|
+
process.exitCode = 1;
|
|
342
|
+
return true;
|
|
343
|
+
}
|
|
325
344
|
console.error(`Starting ${target === "web" ? "production" : "dev"} gateway...`);
|
|
326
345
|
try {
|
|
327
346
|
await runGatewayManager("start", target, argv);
|