@pasko70/pibo 1.8.0 → 1.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apps/chat/workflow-manual-trigger-runtime.js +13 -4
- package/dist/core/compaction-prompt.js +87 -52
- package/dist/core/provider-recovery.js +64 -0
- package/dist/core/provider-telemetry.js +40 -2
- package/dist/core/routed-session.js +97 -19
- package/dist/core/runtime-telemetry.js +84 -46
- package/dist/core/runtime.js +5 -3
- package/dist/core/session-errors.js +14 -1
- package/dist/core/session-router.js +74 -22
- package/dist/data/telemetry-writer.js +114 -0
- package/dist/data/telemetry.js +14 -0
- package/dist/gateway/request.js +17 -8
- package/dist/ralph/service.js +3 -5
- package/package.json +1 -1
|
@@ -184,6 +184,7 @@ function emitMessageAndWaitForAssistant(channelContext, piboSessionId, text, tim
|
|
|
184
184
|
return new Promise((resolve, reject) => {
|
|
185
185
|
const eventId = `wfm_${randomUUID()}`;
|
|
186
186
|
let settled = false;
|
|
187
|
+
let lastAssistantMessage;
|
|
187
188
|
let timeout;
|
|
188
189
|
let unsubscribe = () => { };
|
|
189
190
|
const finish = (value) => {
|
|
@@ -197,16 +198,24 @@ function emitMessageAndWaitForAssistant(channelContext, piboSessionId, text, tim
|
|
|
197
198
|
else
|
|
198
199
|
resolve(value);
|
|
199
200
|
};
|
|
200
|
-
timeout = setTimeout(() =>
|
|
201
|
+
timeout = setTimeout(() => {
|
|
202
|
+
finish(new Error(`Timed out waiting for assistant reply from workflow agent session '${piboSessionId}'.`));
|
|
203
|
+
void channelContext.emit({ type: "execution", piboSessionId, action: "abort", id: `wfm_abort_${randomUUID()}` }).catch(() => { });
|
|
204
|
+
}, timeoutMs);
|
|
201
205
|
unsubscribe = channelContext.subscribe((event) => {
|
|
202
206
|
if (event.piboSessionId !== piboSessionId)
|
|
203
207
|
return;
|
|
208
|
+
if (!("eventId" in event) || event.eventId !== eventId)
|
|
209
|
+
return;
|
|
204
210
|
if (event.type === "assistant_message") {
|
|
205
|
-
|
|
206
|
-
|
|
211
|
+
lastAssistantMessage = event.text;
|
|
212
|
+
return;
|
|
213
|
+
}
|
|
214
|
+
if (event.type === "message_finished") {
|
|
215
|
+
finish(lastAssistantMessage ?? new Error(`Workflow agent session '${piboSessionId}' finished without an assistant reply.`));
|
|
207
216
|
return;
|
|
208
217
|
}
|
|
209
|
-
if (event.type === "session_error"
|
|
218
|
+
if (event.type === "session_error")
|
|
210
219
|
finish(new Error(event.error));
|
|
211
220
|
});
|
|
212
221
|
channelContext.emit({ type: "message", piboSessionId, id: eventId, text, source: "actor" }).catch(finish);
|
|
@@ -1,9 +1,11 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
1
2
|
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
2
3
|
import { readFile, writeFile } from "node:fs/promises";
|
|
3
4
|
import { dirname, resolve } from "node:path";
|
|
4
5
|
import { fileURLToPath } from "node:url";
|
|
5
6
|
import { completeSimple } from "@earendil-works/pi-ai/compat";
|
|
6
7
|
import { buildSessionContext, convertToLlm, serializeConversation, } from "@earendil-works/pi-coding-agent";
|
|
8
|
+
import { isRetryablePiboAssistantError, isRetryablePiboProviderError, resolvePiboProviderRecoverySettings, waitForPiboProviderRecovery, } from "./provider-recovery.js";
|
|
7
9
|
const PROJECT_ROOT = dirname(dirname(dirname(fileURLToPath(import.meta.url))));
|
|
8
10
|
export const PIBO_LIBRARY_COMPACTION_PROMPT_PATH = resolve(PROJECT_ROOT, "context/pibo-compaction-prompt.md");
|
|
9
11
|
function getCompactionPromptStatePath(cwd) {
|
|
@@ -134,38 +136,62 @@ function buildTurnPrefixPrompt(spec, messages) {
|
|
|
134
136
|
const conversationText = serializeConversation(convertToLlm(messages));
|
|
135
137
|
return `<conversation>\n${conversationText}\n</conversation>\n\n${spec.turnPrefixSummaryPrompt}`;
|
|
136
138
|
}
|
|
137
|
-
async function
|
|
138
|
-
const
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
139
|
+
export async function completePiboCompactionSummary(input) {
|
|
140
|
+
const recoverySessionId = input.sessionId ? `compaction-${randomUUID()}` : undefined;
|
|
141
|
+
const completionOptions = {
|
|
142
|
+
maxTokens: input.maxTokens,
|
|
143
|
+
signal: input.signal,
|
|
144
|
+
apiKey: input.apiKey,
|
|
145
|
+
headers: input.headers,
|
|
146
|
+
transport: input.transport,
|
|
147
|
+
sessionId: recoverySessionId,
|
|
148
|
+
timeoutMs: input.timeoutMs,
|
|
149
|
+
websocketConnectTimeoutMs: input.websocketConnectTimeoutMs,
|
|
150
|
+
maxRetries: input.maxRetries,
|
|
151
|
+
maxRetryDelayMs: input.maxRetryDelayMs,
|
|
152
|
+
...(input.model.reasoning && input.thinkingLevel && input.thinkingLevel !== "off"
|
|
153
|
+
? { reasoning: input.thinkingLevel }
|
|
154
|
+
: {}),
|
|
155
|
+
};
|
|
156
|
+
const complete = input.complete ?? completeSimple;
|
|
157
|
+
let recoveryAttempt = 0;
|
|
158
|
+
while (true) {
|
|
159
|
+
let response;
|
|
160
|
+
try {
|
|
161
|
+
response = await complete(input.model, {
|
|
162
|
+
systemPrompt: input.systemPrompt,
|
|
163
|
+
messages: [
|
|
164
|
+
{
|
|
165
|
+
role: "user",
|
|
166
|
+
content: [{ type: "text", text: input.promptText }],
|
|
167
|
+
timestamp: Date.now(),
|
|
168
|
+
},
|
|
169
|
+
],
|
|
170
|
+
}, completionOptions);
|
|
145
171
|
}
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
172
|
+
catch (error) {
|
|
173
|
+
if (input.signal?.aborted || !input.recovery.enabled || !isRetryablePiboProviderError(error))
|
|
174
|
+
throw error;
|
|
175
|
+
recoveryAttempt += 1;
|
|
176
|
+
await waitForPiboProviderRecovery(recoveryAttempt, input.recovery, input.signal);
|
|
177
|
+
continue;
|
|
178
|
+
}
|
|
179
|
+
if (response.stopReason === "aborted") {
|
|
180
|
+
throw new Error("Summarization was aborted");
|
|
181
|
+
}
|
|
182
|
+
if (response.stopReason === "error") {
|
|
183
|
+
const error = new Error(`Summarization failed: ${response.errorMessage || "Unknown error"}`);
|
|
184
|
+
if (!input.recovery.enabled || !isRetryablePiboAssistantError(response))
|
|
185
|
+
throw error;
|
|
186
|
+
recoveryAttempt += 1;
|
|
187
|
+
await waitForPiboProviderRecovery(recoveryAttempt, input.recovery, input.signal);
|
|
188
|
+
continue;
|
|
189
|
+
}
|
|
190
|
+
return response.content
|
|
191
|
+
.filter((content) => content.type === "text")
|
|
192
|
+
.map((content) => content.text)
|
|
193
|
+
.join("\n");
|
|
164
194
|
}
|
|
165
|
-
return response.content
|
|
166
|
-
.filter((content) => content.type === "text")
|
|
167
|
-
.map((content) => content.text)
|
|
168
|
-
.join("\n");
|
|
169
195
|
}
|
|
170
196
|
function computeFileLists(fileOps) {
|
|
171
197
|
const modified = new Set([...fileOps.written, ...fileOps.edited]);
|
|
@@ -183,44 +209,44 @@ function formatFileOperations(readFiles, modifiedFiles) {
|
|
|
183
209
|
}
|
|
184
210
|
async function generatePiboCompaction(input) {
|
|
185
211
|
const { preparation, spec } = input;
|
|
212
|
+
const completionDefaults = {
|
|
213
|
+
model: input.model,
|
|
214
|
+
systemPrompt: spec.systemPrompt,
|
|
215
|
+
apiKey: input.apiKey,
|
|
216
|
+
headers: input.headers,
|
|
217
|
+
signal: input.signal,
|
|
218
|
+
thinkingLevel: input.thinkingLevel,
|
|
219
|
+
transport: input.transport,
|
|
220
|
+
sessionId: input.sessionId,
|
|
221
|
+
timeoutMs: input.timeoutMs,
|
|
222
|
+
websocketConnectTimeoutMs: input.websocketConnectTimeoutMs,
|
|
223
|
+
maxRetries: input.maxRetries,
|
|
224
|
+
maxRetryDelayMs: input.maxRetryDelayMs,
|
|
225
|
+
recovery: input.recovery,
|
|
226
|
+
};
|
|
186
227
|
let summary;
|
|
187
228
|
if (preparation.isSplitTurn && preparation.turnPrefixMessages.length > 0) {
|
|
188
229
|
const [historySummary, turnPrefixSummary] = await Promise.all([
|
|
189
230
|
preparation.messagesToSummarize.length > 0
|
|
190
|
-
?
|
|
191
|
-
|
|
192
|
-
systemPrompt: spec.systemPrompt,
|
|
231
|
+
? completePiboCompactionSummary({
|
|
232
|
+
...completionDefaults,
|
|
193
233
|
promptText: buildSummaryPrompt(spec, preparation.messagesToSummarize, input.customInstructions, preparation.previousSummary),
|
|
194
234
|
maxTokens: Math.floor(0.8 * preparation.settings.reserveTokens),
|
|
195
|
-
apiKey: input.apiKey,
|
|
196
|
-
headers: input.headers,
|
|
197
|
-
signal: input.signal,
|
|
198
|
-
thinkingLevel: input.thinkingLevel,
|
|
199
235
|
})
|
|
200
236
|
: Promise.resolve("No prior history."),
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
systemPrompt: spec.systemPrompt,
|
|
237
|
+
completePiboCompactionSummary({
|
|
238
|
+
...completionDefaults,
|
|
204
239
|
promptText: buildTurnPrefixPrompt(spec, preparation.turnPrefixMessages),
|
|
205
240
|
maxTokens: Math.floor(0.5 * preparation.settings.reserveTokens),
|
|
206
|
-
apiKey: input.apiKey,
|
|
207
|
-
headers: input.headers,
|
|
208
|
-
signal: input.signal,
|
|
209
|
-
thinkingLevel: input.thinkingLevel,
|
|
210
241
|
}),
|
|
211
242
|
]);
|
|
212
243
|
summary = `${historySummary}\n\n---\n\n**Turn Context (split turn):**\n\n${turnPrefixSummary}`;
|
|
213
244
|
}
|
|
214
245
|
else {
|
|
215
|
-
summary = await
|
|
216
|
-
|
|
217
|
-
systemPrompt: spec.systemPrompt,
|
|
246
|
+
summary = await completePiboCompactionSummary({
|
|
247
|
+
...completionDefaults,
|
|
218
248
|
promptText: buildSummaryPrompt(spec, preparation.messagesToSummarize, input.customInstructions, preparation.previousSummary),
|
|
219
249
|
maxTokens: Math.floor(0.8 * preparation.settings.reserveTokens),
|
|
220
|
-
apiKey: input.apiKey,
|
|
221
|
-
headers: input.headers,
|
|
222
|
-
signal: input.signal,
|
|
223
|
-
thinkingLevel: input.thinkingLevel,
|
|
224
250
|
});
|
|
225
251
|
}
|
|
226
252
|
const { readFiles, modifiedFiles } = computeFileLists(preparation.fileOps);
|
|
@@ -232,7 +258,7 @@ async function generatePiboCompaction(input) {
|
|
|
232
258
|
details: { readFiles, modifiedFiles },
|
|
233
259
|
};
|
|
234
260
|
}
|
|
235
|
-
export function createPiboCompactionPromptExtension() {
|
|
261
|
+
export function createPiboCompactionPromptExtension(options = {}) {
|
|
236
262
|
return (pi) => {
|
|
237
263
|
pi.on("session_before_compact", async (event, ctx) => {
|
|
238
264
|
if (!ctx.model)
|
|
@@ -241,6 +267,8 @@ export function createPiboCompactionPromptExtension() {
|
|
|
241
267
|
if (!auth.ok || !auth.apiKey)
|
|
242
268
|
return undefined;
|
|
243
269
|
const sessionContext = buildSessionContext(ctx.sessionManager.getEntries(), ctx.sessionManager.getLeafId());
|
|
270
|
+
const settingsManager = options.getSettingsManager?.();
|
|
271
|
+
const providerRetry = settingsManager?.getProviderRetrySettings();
|
|
244
272
|
return {
|
|
245
273
|
compaction: await generatePiboCompaction({
|
|
246
274
|
preparation: event.preparation,
|
|
@@ -251,6 +279,13 @@ export function createPiboCompactionPromptExtension() {
|
|
|
251
279
|
customInstructions: event.customInstructions,
|
|
252
280
|
signal: event.signal,
|
|
253
281
|
thinkingLevel: sessionContext.thinkingLevel,
|
|
282
|
+
transport: settingsManager?.getTransport(),
|
|
283
|
+
sessionId: ctx.sessionManager.getSessionId(),
|
|
284
|
+
timeoutMs: settingsManager?.getHttpIdleTimeoutMs(),
|
|
285
|
+
websocketConnectTimeoutMs: settingsManager?.getWebSocketConnectTimeoutMs(),
|
|
286
|
+
maxRetries: providerRetry?.maxRetries,
|
|
287
|
+
maxRetryDelayMs: providerRetry?.maxRetryDelayMs,
|
|
288
|
+
recovery: resolvePiboProviderRecoverySettings(settingsManager),
|
|
254
289
|
}),
|
|
255
290
|
};
|
|
256
291
|
});
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { isRetryableAssistantError } from "@earendil-works/pi-ai";
|
|
2
|
+
import { classifySessionErrorMessage } from "./session-errors.js";
|
|
3
|
+
export const PIBO_PROVIDER_RECOVERY_MESSAGE_TYPE = "pibo-provider-recovery-resume";
|
|
4
|
+
export const PIBO_PROVIDER_RECOVERY_PROMPT = "Continue the interrupted task autonomously from the existing session state. The previous provider request failed transiently. Do not wait for more user input, ask the user to repeat the request, or mention this recovery message unless the failure affects the result.";
|
|
5
|
+
const DEFAULT_RECOVERY_BASE_DELAY_MS = 2_000;
|
|
6
|
+
const DEFAULT_RECOVERY_MAX_DELAY_MS = 60_000;
|
|
7
|
+
export class PiboProviderRecoveryCancelledError extends Error {
|
|
8
|
+
constructor(message = "Provider recovery cancelled") {
|
|
9
|
+
super(message);
|
|
10
|
+
this.name = "PiboProviderRecoveryCancelledError";
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
export function resolvePiboProviderRecoverySettings(settingsManager) {
|
|
14
|
+
if (!settingsManager) {
|
|
15
|
+
return {
|
|
16
|
+
enabled: false,
|
|
17
|
+
baseDelayMs: DEFAULT_RECOVERY_BASE_DELAY_MS,
|
|
18
|
+
maxDelayMs: DEFAULT_RECOVERY_MAX_DELAY_MS,
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
const retry = settingsManager.getRetrySettings();
|
|
22
|
+
const provider = settingsManager.getProviderRetrySettings();
|
|
23
|
+
return {
|
|
24
|
+
enabled: retry.enabled,
|
|
25
|
+
baseDelayMs: Math.max(0, retry.baseDelayMs),
|
|
26
|
+
maxDelayMs: provider.maxRetryDelayMs > 0 ? provider.maxRetryDelayMs : DEFAULT_RECOVERY_MAX_DELAY_MS,
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
export function piboProviderRecoveryDelayMs(attempt, settings) {
|
|
30
|
+
const exponent = Math.max(0, Math.floor(attempt) - 1);
|
|
31
|
+
const uncapped = settings.baseDelayMs * 2 ** Math.min(exponent, 30);
|
|
32
|
+
return Math.min(settings.maxDelayMs, Number.isFinite(uncapped) ? uncapped : settings.maxDelayMs);
|
|
33
|
+
}
|
|
34
|
+
export function isRetryablePiboAssistantError(message) {
|
|
35
|
+
if (!message || typeof message !== "object")
|
|
36
|
+
return false;
|
|
37
|
+
const assistantMessage = message;
|
|
38
|
+
if (isRetryableAssistantError(assistantMessage))
|
|
39
|
+
return true;
|
|
40
|
+
return typeof assistantMessage.errorMessage === "string"
|
|
41
|
+
&& classifySessionErrorMessage(assistantMessage.errorMessage, { hasProviderContext: true }).retryable === true;
|
|
42
|
+
}
|
|
43
|
+
export function isRetryablePiboProviderError(error) {
|
|
44
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
45
|
+
return isRetryablePiboAssistantError({ stopReason: "error", errorMessage });
|
|
46
|
+
}
|
|
47
|
+
export async function waitForPiboProviderRecovery(attempt, settings, signal) {
|
|
48
|
+
if (signal?.aborted)
|
|
49
|
+
throw new PiboProviderRecoveryCancelledError();
|
|
50
|
+
const delayMs = piboProviderRecoveryDelayMs(attempt, settings);
|
|
51
|
+
if (delayMs <= 0)
|
|
52
|
+
return;
|
|
53
|
+
await new Promise((resolve, reject) => {
|
|
54
|
+
const onAbort = () => {
|
|
55
|
+
clearTimeout(timer);
|
|
56
|
+
reject(new PiboProviderRecoveryCancelledError());
|
|
57
|
+
};
|
|
58
|
+
const timer = setTimeout(() => {
|
|
59
|
+
signal?.removeEventListener("abort", onAbort);
|
|
60
|
+
resolve();
|
|
61
|
+
}, delayMs);
|
|
62
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
63
|
+
});
|
|
64
|
+
}
|
|
@@ -10,6 +10,10 @@ export class PiboProviderTelemetryRecorder {
|
|
|
10
10
|
this.telemetry = new BestEffortTelemetryService(options.store, options.onError);
|
|
11
11
|
}
|
|
12
12
|
recordRequestStart(payload, options = {}) {
|
|
13
|
+
const capturedOptions = { ...options, at: options.at ?? new Date().toISOString() };
|
|
14
|
+
return this.schedule(() => this.recordRequestStartNow(providerPayloadSnapshot(payload), capturedOptions));
|
|
15
|
+
}
|
|
16
|
+
recordRequestStartNow(payload, options) {
|
|
13
17
|
if (!this.options.store)
|
|
14
18
|
return undefined;
|
|
15
19
|
try {
|
|
@@ -74,6 +78,10 @@ export class PiboProviderTelemetryRecorder {
|
|
|
74
78
|
}
|
|
75
79
|
}
|
|
76
80
|
recordResponse(input) {
|
|
81
|
+
const captured = { status: input.status, at: input.at ?? new Date().toISOString() };
|
|
82
|
+
return this.schedule(() => this.recordResponseNow(captured));
|
|
83
|
+
}
|
|
84
|
+
recordResponseNow(input) {
|
|
77
85
|
if (!this.options.store)
|
|
78
86
|
return undefined;
|
|
79
87
|
try {
|
|
@@ -129,7 +137,14 @@ export class PiboProviderTelemetryRecorder {
|
|
|
129
137
|
// The assistant message boundary ends the provider stream even when the wider
|
|
130
138
|
// Pibo turn continues with a long-running tool or another provider request.
|
|
131
139
|
recordMessageEnd(message, options = {}) {
|
|
132
|
-
if (!
|
|
140
|
+
if (!isAssistantMessage(message))
|
|
141
|
+
return undefined;
|
|
142
|
+
const captured = providerAssistantMessageSnapshot(message);
|
|
143
|
+
const capturedOptions = { ...options, at: options.at ?? new Date().toISOString() };
|
|
144
|
+
return this.schedule(() => this.recordMessageEndNow(captured, capturedOptions));
|
|
145
|
+
}
|
|
146
|
+
recordMessageEndNow(message, options) {
|
|
147
|
+
if (!this.options.store)
|
|
133
148
|
return undefined;
|
|
134
149
|
const status = providerStatusForMessage(message);
|
|
135
150
|
const summary = providerSummaryForStatus(status);
|
|
@@ -144,7 +159,14 @@ export class PiboProviderTelemetryRecorder {
|
|
|
144
159
|
return this.finishActiveProviderRequest(status, options.at ?? new Date().toISOString(), summary, errorMessage, errorDetails?.category ?? errorDetails?.errorClass);
|
|
145
160
|
}
|
|
146
161
|
recordShutdown(reason, at = new Date().toISOString()) {
|
|
147
|
-
return this.finishActiveProviderRequest("aborted", at, reason, undefined, "runtime_abort");
|
|
162
|
+
return this.schedule(() => this.finishActiveProviderRequest("aborted", at, reason, undefined, "runtime_abort"));
|
|
163
|
+
}
|
|
164
|
+
schedule(write) {
|
|
165
|
+
if (this.options.writer) {
|
|
166
|
+
this.options.writer.enqueue(write, this.options.onError);
|
|
167
|
+
return undefined;
|
|
168
|
+
}
|
|
169
|
+
return write();
|
|
148
170
|
}
|
|
149
171
|
finishActiveProviderRequest(status, now, summary, errorMessage, errorCategory) {
|
|
150
172
|
if (!this.options.store)
|
|
@@ -301,6 +323,22 @@ function modelFromContext(ctx) {
|
|
|
301
323
|
const api = typeof candidate.api === "string" && candidate.api.length > 0 ? candidate.api : undefined;
|
|
302
324
|
return provider && id ? { provider, id, api } : undefined;
|
|
303
325
|
}
|
|
326
|
+
function providerAssistantMessageSnapshot(message) {
|
|
327
|
+
return {
|
|
328
|
+
role: message.role,
|
|
329
|
+
stopReason: message.stopReason,
|
|
330
|
+
errorMessage: message.errorMessage,
|
|
331
|
+
api: message.api,
|
|
332
|
+
provider: message.provider,
|
|
333
|
+
model: message.model,
|
|
334
|
+
};
|
|
335
|
+
}
|
|
336
|
+
function providerPayloadSnapshot(payload) {
|
|
337
|
+
return {
|
|
338
|
+
model: modelIdFromPayload(payload),
|
|
339
|
+
service_tier: serviceTierFromPayload(payload),
|
|
340
|
+
};
|
|
341
|
+
}
|
|
304
342
|
function modelIdFromPayload(payload) {
|
|
305
343
|
if (!payload || typeof payload !== "object" || Array.isArray(payload))
|
|
306
344
|
return undefined;
|
|
@@ -3,6 +3,7 @@ import { getOpenAiCodexProviderUsageForActiveModel } from "../auth/openai-codex-
|
|
|
3
3
|
import { normalizeSessionErrorDetails, runtimeSessionErrorDetails } from "./session-errors.js";
|
|
4
4
|
import { expandInlineSkills } from "./skill-expansion.js";
|
|
5
5
|
import { PIBO_CONTEXT_GUARD_RESUME_MESSAGE_TYPE, PIBO_CONTEXT_GUARD_RESUME_PROMPT, cancelPiboAssistantContextGuardRecovery, claimPiboAssistantContextGuardRecovery, waitForPiboAssistantContextGuardRecovery, } from "./context-guard.js";
|
|
6
|
+
import { PIBO_PROVIDER_RECOVERY_MESSAGE_TYPE, PIBO_PROVIDER_RECOVERY_PROMPT, PiboProviderRecoveryCancelledError, isRetryablePiboAssistantError, isRetryablePiboProviderError, resolvePiboProviderRecoverySettings, waitForPiboProviderRecovery, } from "./provider-recovery.js";
|
|
6
7
|
const FAST_SERVICE_TIER = "priority";
|
|
7
8
|
function modelSupportsFastServiceTier(model) {
|
|
8
9
|
if (!model)
|
|
@@ -404,7 +405,10 @@ export class RoutedSession {
|
|
|
404
405
|
activeThinkingIndex;
|
|
405
406
|
nextThinkingIndex = 0;
|
|
406
407
|
pendingAssistantError;
|
|
408
|
+
pendingAssistantErrorRetryable = false;
|
|
407
409
|
activeMessageFailed = false;
|
|
410
|
+
providerRecoveryCancelled = false;
|
|
411
|
+
providerRecoveryAbortController;
|
|
408
412
|
unsubscribe;
|
|
409
413
|
recoverySession;
|
|
410
414
|
isContinuePatched = false;
|
|
@@ -496,6 +500,7 @@ export class RoutedSession {
|
|
|
496
500
|
this.unsubscribe?.();
|
|
497
501
|
const session = this.runtime.session;
|
|
498
502
|
if (this.recoverySession && this.recoverySession !== session) {
|
|
503
|
+
this.cancelProviderRecovery();
|
|
499
504
|
cancelPiboAssistantContextGuardRecovery(this.recoverySession, new Error("Context guard recovery cancelled because the Pi session changed"));
|
|
500
505
|
}
|
|
501
506
|
this.recoverySession = session;
|
|
@@ -506,19 +511,20 @@ export class RoutedSession {
|
|
|
506
511
|
const normalized = normalizePiEvent(this.piboSessionId, event, { contextWindow: numberValue(model?.contextWindow) });
|
|
507
512
|
const candidate = event && typeof event === "object" ? event : undefined;
|
|
508
513
|
const assistantMessageEnded = candidate?.type === "message_end" && isAssistantMessage(candidate.message);
|
|
509
|
-
// Pi
|
|
510
|
-
//
|
|
514
|
+
// Pi gets the first chance to recover through its short retry/compaction loop.
|
|
515
|
+
// Keep the final error pending so the routed turn can continue durable recovery.
|
|
511
516
|
if (assistantMessageEnded && normalized?.type === "session_error") {
|
|
512
517
|
this.pendingAssistantError = this.withActiveMessage(normalized);
|
|
518
|
+
this.pendingAssistantErrorRetryable = isRetryablePiboAssistantError(candidate.message);
|
|
513
519
|
}
|
|
514
520
|
else {
|
|
515
|
-
if (assistantMessageEnded)
|
|
521
|
+
if (assistantMessageEnded) {
|
|
516
522
|
this.pendingAssistantError = undefined;
|
|
523
|
+
this.pendingAssistantErrorRetryable = false;
|
|
524
|
+
}
|
|
517
525
|
if (normalized)
|
|
518
526
|
this.emit(this.withActiveMessage(normalized));
|
|
519
527
|
}
|
|
520
|
-
if (candidate?.type === "agent_settled")
|
|
521
|
-
this.flushPendingAssistantError();
|
|
522
528
|
if (this.forwardPiEvents) {
|
|
523
529
|
this.emit({ type: "pi_event", piboSessionId: this.piboSessionId, event });
|
|
524
530
|
}
|
|
@@ -531,6 +537,80 @@ export class RoutedSession {
|
|
|
531
537
|
this.activeMessageFailed = true;
|
|
532
538
|
this.emit(this.pendingAssistantError);
|
|
533
539
|
this.pendingAssistantError = undefined;
|
|
540
|
+
this.pendingAssistantErrorRetryable = false;
|
|
541
|
+
}
|
|
542
|
+
cancelProviderRecovery() {
|
|
543
|
+
this.providerRecoveryCancelled = true;
|
|
544
|
+
this.providerRecoveryAbortController?.abort();
|
|
545
|
+
this.providerRecoveryAbortController = undefined;
|
|
546
|
+
}
|
|
547
|
+
async resumeContextGuardRecovery(session) {
|
|
548
|
+
while (await waitForPiboAssistantContextGuardRecovery(session)) {
|
|
549
|
+
try {
|
|
550
|
+
await session.sendCustomMessage({
|
|
551
|
+
customType: PIBO_CONTEXT_GUARD_RESUME_MESSAGE_TYPE,
|
|
552
|
+
content: [{ type: "text", text: PIBO_CONTEXT_GUARD_RESUME_PROMPT }],
|
|
553
|
+
display: false,
|
|
554
|
+
}, { triggerTurn: true });
|
|
555
|
+
}
|
|
556
|
+
catch (error) {
|
|
557
|
+
const resumeError = error instanceof Error ? error : new Error(String(error));
|
|
558
|
+
cancelPiboAssistantContextGuardRecovery(session, resumeError);
|
|
559
|
+
throw resumeError;
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
async recoverTransientProviderErrors(session) {
|
|
564
|
+
let attempt = 0;
|
|
565
|
+
while (this.pendingAssistantError && this.pendingAssistantErrorRetryable) {
|
|
566
|
+
if (this.providerRecoveryCancelled)
|
|
567
|
+
throw new PiboProviderRecoveryCancelledError();
|
|
568
|
+
const settings = resolvePiboProviderRecoverySettings(session.settingsManager);
|
|
569
|
+
if (!settings.enabled)
|
|
570
|
+
return;
|
|
571
|
+
attempt += 1;
|
|
572
|
+
this.pendingAssistantError = undefined;
|
|
573
|
+
this.pendingAssistantErrorRetryable = false;
|
|
574
|
+
const controller = new AbortController();
|
|
575
|
+
this.providerRecoveryAbortController = controller;
|
|
576
|
+
try {
|
|
577
|
+
await waitForPiboProviderRecovery(attempt, settings, controller.signal);
|
|
578
|
+
}
|
|
579
|
+
finally {
|
|
580
|
+
if (this.providerRecoveryAbortController === controller) {
|
|
581
|
+
this.providerRecoveryAbortController = undefined;
|
|
582
|
+
}
|
|
583
|
+
}
|
|
584
|
+
if (this.providerRecoveryCancelled || this.disposed || this.runtime.session !== session || !this.activeMessage) {
|
|
585
|
+
throw new PiboProviderRecoveryCancelledError();
|
|
586
|
+
}
|
|
587
|
+
try {
|
|
588
|
+
await session.sendCustomMessage({
|
|
589
|
+
customType: PIBO_PROVIDER_RECOVERY_MESSAGE_TYPE,
|
|
590
|
+
content: [{ type: "text", text: PIBO_PROVIDER_RECOVERY_PROMPT }],
|
|
591
|
+
display: false,
|
|
592
|
+
details: { attempt },
|
|
593
|
+
}, { triggerTurn: true });
|
|
594
|
+
await this.resumeContextGuardRecovery(session);
|
|
595
|
+
if (this.providerRecoveryCancelled)
|
|
596
|
+
throw new PiboProviderRecoveryCancelledError();
|
|
597
|
+
}
|
|
598
|
+
catch (error) {
|
|
599
|
+
if (error instanceof PiboProviderRecoveryCancelledError)
|
|
600
|
+
throw error;
|
|
601
|
+
if (!isRetryablePiboProviderError(error))
|
|
602
|
+
throw error;
|
|
603
|
+
const message = errorMessage(error);
|
|
604
|
+
this.pendingAssistantError = {
|
|
605
|
+
type: "session_error",
|
|
606
|
+
piboSessionId: this.piboSessionId,
|
|
607
|
+
eventId: this.activeMessage.id,
|
|
608
|
+
error: message,
|
|
609
|
+
errorDetails: runtimeSessionErrorDetails(message),
|
|
610
|
+
};
|
|
611
|
+
this.pendingAssistantErrorRetryable = true;
|
|
612
|
+
}
|
|
613
|
+
}
|
|
534
614
|
}
|
|
535
615
|
handleCompactionEvent(event) {
|
|
536
616
|
if (!event || typeof event !== "object")
|
|
@@ -770,6 +850,7 @@ export class RoutedSession {
|
|
|
770
850
|
if (this.disposed)
|
|
771
851
|
return;
|
|
772
852
|
this.notifyMessagesInterrupted(this.activeAndQueuedMessages(), "session disposed");
|
|
853
|
+
this.cancelProviderRecovery();
|
|
773
854
|
this.queue.length = 0;
|
|
774
855
|
this.onStateChange?.({ processing: this.processing, queuedMessages: this.queue.length, disposed: true });
|
|
775
856
|
this.unsubscribe?.();
|
|
@@ -783,6 +864,7 @@ export class RoutedSession {
|
|
|
783
864
|
}
|
|
784
865
|
async kill() {
|
|
785
866
|
this.notifyMessagesInterrupted(this.activeAndQueuedMessages(), "session killed");
|
|
867
|
+
this.cancelProviderRecovery();
|
|
786
868
|
this.queue.length = 0;
|
|
787
869
|
this.onStateChange?.({ processing: this.processing, queuedMessages: this.queue.length, disposed: this.disposed });
|
|
788
870
|
this.cancelContextGuardRecovery("Context guard recovery cancelled because the routed session was killed");
|
|
@@ -801,6 +883,7 @@ export class RoutedSession {
|
|
|
801
883
|
}
|
|
802
884
|
if (this.activeMessage?.id === eventId) {
|
|
803
885
|
this.notifyMessagesInterrupted([this.activeMessage], "message cancelled");
|
|
886
|
+
this.cancelProviderRecovery();
|
|
804
887
|
this.cancelContextGuardRecovery("Context guard recovery cancelled with the active message");
|
|
805
888
|
await this.runtime.session.abort();
|
|
806
889
|
return true;
|
|
@@ -839,7 +922,9 @@ export class RoutedSession {
|
|
|
839
922
|
});
|
|
840
923
|
try {
|
|
841
924
|
this.activeMessage = event;
|
|
925
|
+
this.providerRecoveryCancelled = false;
|
|
842
926
|
this.pendingAssistantError = undefined;
|
|
927
|
+
this.pendingAssistantErrorRetryable = false;
|
|
843
928
|
this.activeMessageFailed = false;
|
|
844
929
|
this.activeAssistantIndex = undefined;
|
|
845
930
|
this.nextAssistantIndex = 0;
|
|
@@ -848,20 +933,9 @@ export class RoutedSession {
|
|
|
848
933
|
const session = this.runtime.session;
|
|
849
934
|
const expandedText = expandInlineSkills(event.text, session.resourceLoader.getSkills().skills);
|
|
850
935
|
await session.prompt(expandedText, { source: promptSource(event.source) });
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
customType: PIBO_CONTEXT_GUARD_RESUME_MESSAGE_TYPE,
|
|
855
|
-
content: [{ type: "text", text: PIBO_CONTEXT_GUARD_RESUME_PROMPT }],
|
|
856
|
-
display: false,
|
|
857
|
-
}, { triggerTurn: true });
|
|
858
|
-
}
|
|
859
|
-
catch (error) {
|
|
860
|
-
const resumeError = error instanceof Error ? error : new Error(String(error));
|
|
861
|
-
cancelPiboAssistantContextGuardRecovery(session, resumeError);
|
|
862
|
-
throw resumeError;
|
|
863
|
-
}
|
|
864
|
-
}
|
|
936
|
+
await this.resumeContextGuardRecovery(session);
|
|
937
|
+
await this.recoverTransientProviderErrors(session);
|
|
938
|
+
this.flushPendingAssistantError();
|
|
865
939
|
if (!this.activeMessageFailed) {
|
|
866
940
|
this.emit({
|
|
867
941
|
type: "message_finished",
|
|
@@ -872,6 +946,8 @@ export class RoutedSession {
|
|
|
872
946
|
}
|
|
873
947
|
}
|
|
874
948
|
catch (error) {
|
|
949
|
+
if (error instanceof PiboProviderRecoveryCancelledError)
|
|
950
|
+
return;
|
|
875
951
|
const message = errorMessage(error);
|
|
876
952
|
this.emit({
|
|
877
953
|
type: "session_error",
|
|
@@ -883,7 +959,9 @@ export class RoutedSession {
|
|
|
883
959
|
}
|
|
884
960
|
finally {
|
|
885
961
|
this.activeMessage = undefined;
|
|
962
|
+
this.providerRecoveryCancelled = false;
|
|
886
963
|
this.pendingAssistantError = undefined;
|
|
964
|
+
this.pendingAssistantErrorRetryable = false;
|
|
887
965
|
this.activeMessageFailed = false;
|
|
888
966
|
this.activeAssistantIndex = undefined;
|
|
889
967
|
this.nextAssistantIndex = 0;
|