@pasko70/pibo 1.8.0 → 1.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apps/chat/workflow-manual-trigger-runtime.js +13 -4
- package/dist/core/compaction-prompt.js +87 -52
- package/dist/core/provider-recovery.js +64 -0
- package/dist/core/routed-session.js +97 -19
- package/dist/core/runtime.js +5 -3
- package/dist/core/session-errors.js +14 -1
- package/dist/core/session-router.js +22 -5
- package/dist/gateway/request.js +17 -8
- package/dist/ralph/service.js +3 -5
- package/package.json +1 -1
|
@@ -184,6 +184,7 @@ function emitMessageAndWaitForAssistant(channelContext, piboSessionId, text, tim
|
|
|
184
184
|
return new Promise((resolve, reject) => {
|
|
185
185
|
const eventId = `wfm_${randomUUID()}`;
|
|
186
186
|
let settled = false;
|
|
187
|
+
let lastAssistantMessage;
|
|
187
188
|
let timeout;
|
|
188
189
|
let unsubscribe = () => { };
|
|
189
190
|
const finish = (value) => {
|
|
@@ -197,16 +198,24 @@ function emitMessageAndWaitForAssistant(channelContext, piboSessionId, text, tim
|
|
|
197
198
|
else
|
|
198
199
|
resolve(value);
|
|
199
200
|
};
|
|
200
|
-
timeout = setTimeout(() =>
|
|
201
|
+
timeout = setTimeout(() => {
|
|
202
|
+
finish(new Error(`Timed out waiting for assistant reply from workflow agent session '${piboSessionId}'.`));
|
|
203
|
+
void channelContext.emit({ type: "execution", piboSessionId, action: "abort", id: `wfm_abort_${randomUUID()}` }).catch(() => { });
|
|
204
|
+
}, timeoutMs);
|
|
201
205
|
unsubscribe = channelContext.subscribe((event) => {
|
|
202
206
|
if (event.piboSessionId !== piboSessionId)
|
|
203
207
|
return;
|
|
208
|
+
if (!("eventId" in event) || event.eventId !== eventId)
|
|
209
|
+
return;
|
|
204
210
|
if (event.type === "assistant_message") {
|
|
205
|
-
|
|
206
|
-
|
|
211
|
+
lastAssistantMessage = event.text;
|
|
212
|
+
return;
|
|
213
|
+
}
|
|
214
|
+
if (event.type === "message_finished") {
|
|
215
|
+
finish(lastAssistantMessage ?? new Error(`Workflow agent session '${piboSessionId}' finished without an assistant reply.`));
|
|
207
216
|
return;
|
|
208
217
|
}
|
|
209
|
-
if (event.type === "session_error"
|
|
218
|
+
if (event.type === "session_error")
|
|
210
219
|
finish(new Error(event.error));
|
|
211
220
|
});
|
|
212
221
|
channelContext.emit({ type: "message", piboSessionId, id: eventId, text, source: "actor" }).catch(finish);
|
|
@@ -1,9 +1,11 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
1
2
|
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
2
3
|
import { readFile, writeFile } from "node:fs/promises";
|
|
3
4
|
import { dirname, resolve } from "node:path";
|
|
4
5
|
import { fileURLToPath } from "node:url";
|
|
5
6
|
import { completeSimple } from "@earendil-works/pi-ai/compat";
|
|
6
7
|
import { buildSessionContext, convertToLlm, serializeConversation, } from "@earendil-works/pi-coding-agent";
|
|
8
|
+
import { isRetryablePiboAssistantError, isRetryablePiboProviderError, resolvePiboProviderRecoverySettings, waitForPiboProviderRecovery, } from "./provider-recovery.js";
|
|
7
9
|
const PROJECT_ROOT = dirname(dirname(dirname(fileURLToPath(import.meta.url))));
|
|
8
10
|
export const PIBO_LIBRARY_COMPACTION_PROMPT_PATH = resolve(PROJECT_ROOT, "context/pibo-compaction-prompt.md");
|
|
9
11
|
function getCompactionPromptStatePath(cwd) {
|
|
@@ -134,38 +136,62 @@ function buildTurnPrefixPrompt(spec, messages) {
|
|
|
134
136
|
const conversationText = serializeConversation(convertToLlm(messages));
|
|
135
137
|
return `<conversation>\n${conversationText}\n</conversation>\n\n${spec.turnPrefixSummaryPrompt}`;
|
|
136
138
|
}
|
|
137
|
-
async function
|
|
138
|
-
const
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
139
|
+
export async function completePiboCompactionSummary(input) {
|
|
140
|
+
const recoverySessionId = input.sessionId ? `compaction-${randomUUID()}` : undefined;
|
|
141
|
+
const completionOptions = {
|
|
142
|
+
maxTokens: input.maxTokens,
|
|
143
|
+
signal: input.signal,
|
|
144
|
+
apiKey: input.apiKey,
|
|
145
|
+
headers: input.headers,
|
|
146
|
+
transport: input.transport,
|
|
147
|
+
sessionId: recoverySessionId,
|
|
148
|
+
timeoutMs: input.timeoutMs,
|
|
149
|
+
websocketConnectTimeoutMs: input.websocketConnectTimeoutMs,
|
|
150
|
+
maxRetries: input.maxRetries,
|
|
151
|
+
maxRetryDelayMs: input.maxRetryDelayMs,
|
|
152
|
+
...(input.model.reasoning && input.thinkingLevel && input.thinkingLevel !== "off"
|
|
153
|
+
? { reasoning: input.thinkingLevel }
|
|
154
|
+
: {}),
|
|
155
|
+
};
|
|
156
|
+
const complete = input.complete ?? completeSimple;
|
|
157
|
+
let recoveryAttempt = 0;
|
|
158
|
+
while (true) {
|
|
159
|
+
let response;
|
|
160
|
+
try {
|
|
161
|
+
response = await complete(input.model, {
|
|
162
|
+
systemPrompt: input.systemPrompt,
|
|
163
|
+
messages: [
|
|
164
|
+
{
|
|
165
|
+
role: "user",
|
|
166
|
+
content: [{ type: "text", text: input.promptText }],
|
|
167
|
+
timestamp: Date.now(),
|
|
168
|
+
},
|
|
169
|
+
],
|
|
170
|
+
}, completionOptions);
|
|
145
171
|
}
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
172
|
+
catch (error) {
|
|
173
|
+
if (input.signal?.aborted || !input.recovery.enabled || !isRetryablePiboProviderError(error))
|
|
174
|
+
throw error;
|
|
175
|
+
recoveryAttempt += 1;
|
|
176
|
+
await waitForPiboProviderRecovery(recoveryAttempt, input.recovery, input.signal);
|
|
177
|
+
continue;
|
|
178
|
+
}
|
|
179
|
+
if (response.stopReason === "aborted") {
|
|
180
|
+
throw new Error("Summarization was aborted");
|
|
181
|
+
}
|
|
182
|
+
if (response.stopReason === "error") {
|
|
183
|
+
const error = new Error(`Summarization failed: ${response.errorMessage || "Unknown error"}`);
|
|
184
|
+
if (!input.recovery.enabled || !isRetryablePiboAssistantError(response))
|
|
185
|
+
throw error;
|
|
186
|
+
recoveryAttempt += 1;
|
|
187
|
+
await waitForPiboProviderRecovery(recoveryAttempt, input.recovery, input.signal);
|
|
188
|
+
continue;
|
|
189
|
+
}
|
|
190
|
+
return response.content
|
|
191
|
+
.filter((content) => content.type === "text")
|
|
192
|
+
.map((content) => content.text)
|
|
193
|
+
.join("\n");
|
|
164
194
|
}
|
|
165
|
-
return response.content
|
|
166
|
-
.filter((content) => content.type === "text")
|
|
167
|
-
.map((content) => content.text)
|
|
168
|
-
.join("\n");
|
|
169
195
|
}
|
|
170
196
|
function computeFileLists(fileOps) {
|
|
171
197
|
const modified = new Set([...fileOps.written, ...fileOps.edited]);
|
|
@@ -183,44 +209,44 @@ function formatFileOperations(readFiles, modifiedFiles) {
|
|
|
183
209
|
}
|
|
184
210
|
async function generatePiboCompaction(input) {
|
|
185
211
|
const { preparation, spec } = input;
|
|
212
|
+
const completionDefaults = {
|
|
213
|
+
model: input.model,
|
|
214
|
+
systemPrompt: spec.systemPrompt,
|
|
215
|
+
apiKey: input.apiKey,
|
|
216
|
+
headers: input.headers,
|
|
217
|
+
signal: input.signal,
|
|
218
|
+
thinkingLevel: input.thinkingLevel,
|
|
219
|
+
transport: input.transport,
|
|
220
|
+
sessionId: input.sessionId,
|
|
221
|
+
timeoutMs: input.timeoutMs,
|
|
222
|
+
websocketConnectTimeoutMs: input.websocketConnectTimeoutMs,
|
|
223
|
+
maxRetries: input.maxRetries,
|
|
224
|
+
maxRetryDelayMs: input.maxRetryDelayMs,
|
|
225
|
+
recovery: input.recovery,
|
|
226
|
+
};
|
|
186
227
|
let summary;
|
|
187
228
|
if (preparation.isSplitTurn && preparation.turnPrefixMessages.length > 0) {
|
|
188
229
|
const [historySummary, turnPrefixSummary] = await Promise.all([
|
|
189
230
|
preparation.messagesToSummarize.length > 0
|
|
190
|
-
?
|
|
191
|
-
|
|
192
|
-
systemPrompt: spec.systemPrompt,
|
|
231
|
+
? completePiboCompactionSummary({
|
|
232
|
+
...completionDefaults,
|
|
193
233
|
promptText: buildSummaryPrompt(spec, preparation.messagesToSummarize, input.customInstructions, preparation.previousSummary),
|
|
194
234
|
maxTokens: Math.floor(0.8 * preparation.settings.reserveTokens),
|
|
195
|
-
apiKey: input.apiKey,
|
|
196
|
-
headers: input.headers,
|
|
197
|
-
signal: input.signal,
|
|
198
|
-
thinkingLevel: input.thinkingLevel,
|
|
199
235
|
})
|
|
200
236
|
: Promise.resolve("No prior history."),
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
systemPrompt: spec.systemPrompt,
|
|
237
|
+
completePiboCompactionSummary({
|
|
238
|
+
...completionDefaults,
|
|
204
239
|
promptText: buildTurnPrefixPrompt(spec, preparation.turnPrefixMessages),
|
|
205
240
|
maxTokens: Math.floor(0.5 * preparation.settings.reserveTokens),
|
|
206
|
-
apiKey: input.apiKey,
|
|
207
|
-
headers: input.headers,
|
|
208
|
-
signal: input.signal,
|
|
209
|
-
thinkingLevel: input.thinkingLevel,
|
|
210
241
|
}),
|
|
211
242
|
]);
|
|
212
243
|
summary = `${historySummary}\n\n---\n\n**Turn Context (split turn):**\n\n${turnPrefixSummary}`;
|
|
213
244
|
}
|
|
214
245
|
else {
|
|
215
|
-
summary = await
|
|
216
|
-
|
|
217
|
-
systemPrompt: spec.systemPrompt,
|
|
246
|
+
summary = await completePiboCompactionSummary({
|
|
247
|
+
...completionDefaults,
|
|
218
248
|
promptText: buildSummaryPrompt(spec, preparation.messagesToSummarize, input.customInstructions, preparation.previousSummary),
|
|
219
249
|
maxTokens: Math.floor(0.8 * preparation.settings.reserveTokens),
|
|
220
|
-
apiKey: input.apiKey,
|
|
221
|
-
headers: input.headers,
|
|
222
|
-
signal: input.signal,
|
|
223
|
-
thinkingLevel: input.thinkingLevel,
|
|
224
250
|
});
|
|
225
251
|
}
|
|
226
252
|
const { readFiles, modifiedFiles } = computeFileLists(preparation.fileOps);
|
|
@@ -232,7 +258,7 @@ async function generatePiboCompaction(input) {
|
|
|
232
258
|
details: { readFiles, modifiedFiles },
|
|
233
259
|
};
|
|
234
260
|
}
|
|
235
|
-
export function createPiboCompactionPromptExtension() {
|
|
261
|
+
export function createPiboCompactionPromptExtension(options = {}) {
|
|
236
262
|
return (pi) => {
|
|
237
263
|
pi.on("session_before_compact", async (event, ctx) => {
|
|
238
264
|
if (!ctx.model)
|
|
@@ -241,6 +267,8 @@ export function createPiboCompactionPromptExtension() {
|
|
|
241
267
|
if (!auth.ok || !auth.apiKey)
|
|
242
268
|
return undefined;
|
|
243
269
|
const sessionContext = buildSessionContext(ctx.sessionManager.getEntries(), ctx.sessionManager.getLeafId());
|
|
270
|
+
const settingsManager = options.getSettingsManager?.();
|
|
271
|
+
const providerRetry = settingsManager?.getProviderRetrySettings();
|
|
244
272
|
return {
|
|
245
273
|
compaction: await generatePiboCompaction({
|
|
246
274
|
preparation: event.preparation,
|
|
@@ -251,6 +279,13 @@ export function createPiboCompactionPromptExtension() {
|
|
|
251
279
|
customInstructions: event.customInstructions,
|
|
252
280
|
signal: event.signal,
|
|
253
281
|
thinkingLevel: sessionContext.thinkingLevel,
|
|
282
|
+
transport: settingsManager?.getTransport(),
|
|
283
|
+
sessionId: ctx.sessionManager.getSessionId(),
|
|
284
|
+
timeoutMs: settingsManager?.getHttpIdleTimeoutMs(),
|
|
285
|
+
websocketConnectTimeoutMs: settingsManager?.getWebSocketConnectTimeoutMs(),
|
|
286
|
+
maxRetries: providerRetry?.maxRetries,
|
|
287
|
+
maxRetryDelayMs: providerRetry?.maxRetryDelayMs,
|
|
288
|
+
recovery: resolvePiboProviderRecoverySettings(settingsManager),
|
|
254
289
|
}),
|
|
255
290
|
};
|
|
256
291
|
});
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { isRetryableAssistantError } from "@earendil-works/pi-ai";
|
|
2
|
+
import { classifySessionErrorMessage } from "./session-errors.js";
|
|
3
|
+
export const PIBO_PROVIDER_RECOVERY_MESSAGE_TYPE = "pibo-provider-recovery-resume";
|
|
4
|
+
export const PIBO_PROVIDER_RECOVERY_PROMPT = "Continue the interrupted task autonomously from the existing session state. The previous provider request failed transiently. Do not wait for more user input, ask the user to repeat the request, or mention this recovery message unless the failure affects the result.";
|
|
5
|
+
const DEFAULT_RECOVERY_BASE_DELAY_MS = 2_000;
|
|
6
|
+
const DEFAULT_RECOVERY_MAX_DELAY_MS = 60_000;
|
|
7
|
+
export class PiboProviderRecoveryCancelledError extends Error {
|
|
8
|
+
constructor(message = "Provider recovery cancelled") {
|
|
9
|
+
super(message);
|
|
10
|
+
this.name = "PiboProviderRecoveryCancelledError";
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
export function resolvePiboProviderRecoverySettings(settingsManager) {
|
|
14
|
+
if (!settingsManager) {
|
|
15
|
+
return {
|
|
16
|
+
enabled: false,
|
|
17
|
+
baseDelayMs: DEFAULT_RECOVERY_BASE_DELAY_MS,
|
|
18
|
+
maxDelayMs: DEFAULT_RECOVERY_MAX_DELAY_MS,
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
const retry = settingsManager.getRetrySettings();
|
|
22
|
+
const provider = settingsManager.getProviderRetrySettings();
|
|
23
|
+
return {
|
|
24
|
+
enabled: retry.enabled,
|
|
25
|
+
baseDelayMs: Math.max(0, retry.baseDelayMs),
|
|
26
|
+
maxDelayMs: provider.maxRetryDelayMs > 0 ? provider.maxRetryDelayMs : DEFAULT_RECOVERY_MAX_DELAY_MS,
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
export function piboProviderRecoveryDelayMs(attempt, settings) {
|
|
30
|
+
const exponent = Math.max(0, Math.floor(attempt) - 1);
|
|
31
|
+
const uncapped = settings.baseDelayMs * 2 ** Math.min(exponent, 30);
|
|
32
|
+
return Math.min(settings.maxDelayMs, Number.isFinite(uncapped) ? uncapped : settings.maxDelayMs);
|
|
33
|
+
}
|
|
34
|
+
export function isRetryablePiboAssistantError(message) {
|
|
35
|
+
if (!message || typeof message !== "object")
|
|
36
|
+
return false;
|
|
37
|
+
const assistantMessage = message;
|
|
38
|
+
if (isRetryableAssistantError(assistantMessage))
|
|
39
|
+
return true;
|
|
40
|
+
return typeof assistantMessage.errorMessage === "string"
|
|
41
|
+
&& classifySessionErrorMessage(assistantMessage.errorMessage, { hasProviderContext: true }).retryable === true;
|
|
42
|
+
}
|
|
43
|
+
export function isRetryablePiboProviderError(error) {
|
|
44
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
45
|
+
return isRetryablePiboAssistantError({ stopReason: "error", errorMessage });
|
|
46
|
+
}
|
|
47
|
+
export async function waitForPiboProviderRecovery(attempt, settings, signal) {
|
|
48
|
+
if (signal?.aborted)
|
|
49
|
+
throw new PiboProviderRecoveryCancelledError();
|
|
50
|
+
const delayMs = piboProviderRecoveryDelayMs(attempt, settings);
|
|
51
|
+
if (delayMs <= 0)
|
|
52
|
+
return;
|
|
53
|
+
await new Promise((resolve, reject) => {
|
|
54
|
+
const onAbort = () => {
|
|
55
|
+
clearTimeout(timer);
|
|
56
|
+
reject(new PiboProviderRecoveryCancelledError());
|
|
57
|
+
};
|
|
58
|
+
const timer = setTimeout(() => {
|
|
59
|
+
signal?.removeEventListener("abort", onAbort);
|
|
60
|
+
resolve();
|
|
61
|
+
}, delayMs);
|
|
62
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
63
|
+
});
|
|
64
|
+
}
|
|
@@ -3,6 +3,7 @@ import { getOpenAiCodexProviderUsageForActiveModel } from "../auth/openai-codex-
|
|
|
3
3
|
import { normalizeSessionErrorDetails, runtimeSessionErrorDetails } from "./session-errors.js";
|
|
4
4
|
import { expandInlineSkills } from "./skill-expansion.js";
|
|
5
5
|
import { PIBO_CONTEXT_GUARD_RESUME_MESSAGE_TYPE, PIBO_CONTEXT_GUARD_RESUME_PROMPT, cancelPiboAssistantContextGuardRecovery, claimPiboAssistantContextGuardRecovery, waitForPiboAssistantContextGuardRecovery, } from "./context-guard.js";
|
|
6
|
+
import { PIBO_PROVIDER_RECOVERY_MESSAGE_TYPE, PIBO_PROVIDER_RECOVERY_PROMPT, PiboProviderRecoveryCancelledError, isRetryablePiboAssistantError, isRetryablePiboProviderError, resolvePiboProviderRecoverySettings, waitForPiboProviderRecovery, } from "./provider-recovery.js";
|
|
6
7
|
const FAST_SERVICE_TIER = "priority";
|
|
7
8
|
function modelSupportsFastServiceTier(model) {
|
|
8
9
|
if (!model)
|
|
@@ -404,7 +405,10 @@ export class RoutedSession {
|
|
|
404
405
|
activeThinkingIndex;
|
|
405
406
|
nextThinkingIndex = 0;
|
|
406
407
|
pendingAssistantError;
|
|
408
|
+
pendingAssistantErrorRetryable = false;
|
|
407
409
|
activeMessageFailed = false;
|
|
410
|
+
providerRecoveryCancelled = false;
|
|
411
|
+
providerRecoveryAbortController;
|
|
408
412
|
unsubscribe;
|
|
409
413
|
recoverySession;
|
|
410
414
|
isContinuePatched = false;
|
|
@@ -496,6 +500,7 @@ export class RoutedSession {
|
|
|
496
500
|
this.unsubscribe?.();
|
|
497
501
|
const session = this.runtime.session;
|
|
498
502
|
if (this.recoverySession && this.recoverySession !== session) {
|
|
503
|
+
this.cancelProviderRecovery();
|
|
499
504
|
cancelPiboAssistantContextGuardRecovery(this.recoverySession, new Error("Context guard recovery cancelled because the Pi session changed"));
|
|
500
505
|
}
|
|
501
506
|
this.recoverySession = session;
|
|
@@ -506,19 +511,20 @@ export class RoutedSession {
|
|
|
506
511
|
const normalized = normalizePiEvent(this.piboSessionId, event, { contextWindow: numberValue(model?.contextWindow) });
|
|
507
512
|
const candidate = event && typeof event === "object" ? event : undefined;
|
|
508
513
|
const assistantMessageEnded = candidate?.type === "message_end" && isAssistantMessage(candidate.message);
|
|
509
|
-
// Pi
|
|
510
|
-
//
|
|
514
|
+
// Pi gets the first chance to recover through its short retry/compaction loop.
|
|
515
|
+
// Keep the final error pending so the routed turn can continue durable recovery.
|
|
511
516
|
if (assistantMessageEnded && normalized?.type === "session_error") {
|
|
512
517
|
this.pendingAssistantError = this.withActiveMessage(normalized);
|
|
518
|
+
this.pendingAssistantErrorRetryable = isRetryablePiboAssistantError(candidate.message);
|
|
513
519
|
}
|
|
514
520
|
else {
|
|
515
|
-
if (assistantMessageEnded)
|
|
521
|
+
if (assistantMessageEnded) {
|
|
516
522
|
this.pendingAssistantError = undefined;
|
|
523
|
+
this.pendingAssistantErrorRetryable = false;
|
|
524
|
+
}
|
|
517
525
|
if (normalized)
|
|
518
526
|
this.emit(this.withActiveMessage(normalized));
|
|
519
527
|
}
|
|
520
|
-
if (candidate?.type === "agent_settled")
|
|
521
|
-
this.flushPendingAssistantError();
|
|
522
528
|
if (this.forwardPiEvents) {
|
|
523
529
|
this.emit({ type: "pi_event", piboSessionId: this.piboSessionId, event });
|
|
524
530
|
}
|
|
@@ -531,6 +537,80 @@ export class RoutedSession {
|
|
|
531
537
|
this.activeMessageFailed = true;
|
|
532
538
|
this.emit(this.pendingAssistantError);
|
|
533
539
|
this.pendingAssistantError = undefined;
|
|
540
|
+
this.pendingAssistantErrorRetryable = false;
|
|
541
|
+
}
|
|
542
|
+
cancelProviderRecovery() {
|
|
543
|
+
this.providerRecoveryCancelled = true;
|
|
544
|
+
this.providerRecoveryAbortController?.abort();
|
|
545
|
+
this.providerRecoveryAbortController = undefined;
|
|
546
|
+
}
|
|
547
|
+
async resumeContextGuardRecovery(session) {
|
|
548
|
+
while (await waitForPiboAssistantContextGuardRecovery(session)) {
|
|
549
|
+
try {
|
|
550
|
+
await session.sendCustomMessage({
|
|
551
|
+
customType: PIBO_CONTEXT_GUARD_RESUME_MESSAGE_TYPE,
|
|
552
|
+
content: [{ type: "text", text: PIBO_CONTEXT_GUARD_RESUME_PROMPT }],
|
|
553
|
+
display: false,
|
|
554
|
+
}, { triggerTurn: true });
|
|
555
|
+
}
|
|
556
|
+
catch (error) {
|
|
557
|
+
const resumeError = error instanceof Error ? error : new Error(String(error));
|
|
558
|
+
cancelPiboAssistantContextGuardRecovery(session, resumeError);
|
|
559
|
+
throw resumeError;
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
async recoverTransientProviderErrors(session) {
|
|
564
|
+
let attempt = 0;
|
|
565
|
+
while (this.pendingAssistantError && this.pendingAssistantErrorRetryable) {
|
|
566
|
+
if (this.providerRecoveryCancelled)
|
|
567
|
+
throw new PiboProviderRecoveryCancelledError();
|
|
568
|
+
const settings = resolvePiboProviderRecoverySettings(session.settingsManager);
|
|
569
|
+
if (!settings.enabled)
|
|
570
|
+
return;
|
|
571
|
+
attempt += 1;
|
|
572
|
+
this.pendingAssistantError = undefined;
|
|
573
|
+
this.pendingAssistantErrorRetryable = false;
|
|
574
|
+
const controller = new AbortController();
|
|
575
|
+
this.providerRecoveryAbortController = controller;
|
|
576
|
+
try {
|
|
577
|
+
await waitForPiboProviderRecovery(attempt, settings, controller.signal);
|
|
578
|
+
}
|
|
579
|
+
finally {
|
|
580
|
+
if (this.providerRecoveryAbortController === controller) {
|
|
581
|
+
this.providerRecoveryAbortController = undefined;
|
|
582
|
+
}
|
|
583
|
+
}
|
|
584
|
+
if (this.providerRecoveryCancelled || this.disposed || this.runtime.session !== session || !this.activeMessage) {
|
|
585
|
+
throw new PiboProviderRecoveryCancelledError();
|
|
586
|
+
}
|
|
587
|
+
try {
|
|
588
|
+
await session.sendCustomMessage({
|
|
589
|
+
customType: PIBO_PROVIDER_RECOVERY_MESSAGE_TYPE,
|
|
590
|
+
content: [{ type: "text", text: PIBO_PROVIDER_RECOVERY_PROMPT }],
|
|
591
|
+
display: false,
|
|
592
|
+
details: { attempt },
|
|
593
|
+
}, { triggerTurn: true });
|
|
594
|
+
await this.resumeContextGuardRecovery(session);
|
|
595
|
+
if (this.providerRecoveryCancelled)
|
|
596
|
+
throw new PiboProviderRecoveryCancelledError();
|
|
597
|
+
}
|
|
598
|
+
catch (error) {
|
|
599
|
+
if (error instanceof PiboProviderRecoveryCancelledError)
|
|
600
|
+
throw error;
|
|
601
|
+
if (!isRetryablePiboProviderError(error))
|
|
602
|
+
throw error;
|
|
603
|
+
const message = errorMessage(error);
|
|
604
|
+
this.pendingAssistantError = {
|
|
605
|
+
type: "session_error",
|
|
606
|
+
piboSessionId: this.piboSessionId,
|
|
607
|
+
eventId: this.activeMessage.id,
|
|
608
|
+
error: message,
|
|
609
|
+
errorDetails: runtimeSessionErrorDetails(message),
|
|
610
|
+
};
|
|
611
|
+
this.pendingAssistantErrorRetryable = true;
|
|
612
|
+
}
|
|
613
|
+
}
|
|
534
614
|
}
|
|
535
615
|
handleCompactionEvent(event) {
|
|
536
616
|
if (!event || typeof event !== "object")
|
|
@@ -770,6 +850,7 @@ export class RoutedSession {
|
|
|
770
850
|
if (this.disposed)
|
|
771
851
|
return;
|
|
772
852
|
this.notifyMessagesInterrupted(this.activeAndQueuedMessages(), "session disposed");
|
|
853
|
+
this.cancelProviderRecovery();
|
|
773
854
|
this.queue.length = 0;
|
|
774
855
|
this.onStateChange?.({ processing: this.processing, queuedMessages: this.queue.length, disposed: true });
|
|
775
856
|
this.unsubscribe?.();
|
|
@@ -783,6 +864,7 @@ export class RoutedSession {
|
|
|
783
864
|
}
|
|
784
865
|
async kill() {
|
|
785
866
|
this.notifyMessagesInterrupted(this.activeAndQueuedMessages(), "session killed");
|
|
867
|
+
this.cancelProviderRecovery();
|
|
786
868
|
this.queue.length = 0;
|
|
787
869
|
this.onStateChange?.({ processing: this.processing, queuedMessages: this.queue.length, disposed: this.disposed });
|
|
788
870
|
this.cancelContextGuardRecovery("Context guard recovery cancelled because the routed session was killed");
|
|
@@ -801,6 +883,7 @@ export class RoutedSession {
|
|
|
801
883
|
}
|
|
802
884
|
if (this.activeMessage?.id === eventId) {
|
|
803
885
|
this.notifyMessagesInterrupted([this.activeMessage], "message cancelled");
|
|
886
|
+
this.cancelProviderRecovery();
|
|
804
887
|
this.cancelContextGuardRecovery("Context guard recovery cancelled with the active message");
|
|
805
888
|
await this.runtime.session.abort();
|
|
806
889
|
return true;
|
|
@@ -839,7 +922,9 @@ export class RoutedSession {
|
|
|
839
922
|
});
|
|
840
923
|
try {
|
|
841
924
|
this.activeMessage = event;
|
|
925
|
+
this.providerRecoveryCancelled = false;
|
|
842
926
|
this.pendingAssistantError = undefined;
|
|
927
|
+
this.pendingAssistantErrorRetryable = false;
|
|
843
928
|
this.activeMessageFailed = false;
|
|
844
929
|
this.activeAssistantIndex = undefined;
|
|
845
930
|
this.nextAssistantIndex = 0;
|
|
@@ -848,20 +933,9 @@ export class RoutedSession {
|
|
|
848
933
|
const session = this.runtime.session;
|
|
849
934
|
const expandedText = expandInlineSkills(event.text, session.resourceLoader.getSkills().skills);
|
|
850
935
|
await session.prompt(expandedText, { source: promptSource(event.source) });
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
customType: PIBO_CONTEXT_GUARD_RESUME_MESSAGE_TYPE,
|
|
855
|
-
content: [{ type: "text", text: PIBO_CONTEXT_GUARD_RESUME_PROMPT }],
|
|
856
|
-
display: false,
|
|
857
|
-
}, { triggerTurn: true });
|
|
858
|
-
}
|
|
859
|
-
catch (error) {
|
|
860
|
-
const resumeError = error instanceof Error ? error : new Error(String(error));
|
|
861
|
-
cancelPiboAssistantContextGuardRecovery(session, resumeError);
|
|
862
|
-
throw resumeError;
|
|
863
|
-
}
|
|
864
|
-
}
|
|
936
|
+
await this.resumeContextGuardRecovery(session);
|
|
937
|
+
await this.recoverTransientProviderErrors(session);
|
|
938
|
+
this.flushPendingAssistantError();
|
|
865
939
|
if (!this.activeMessageFailed) {
|
|
866
940
|
this.emit({
|
|
867
941
|
type: "message_finished",
|
|
@@ -872,6 +946,8 @@ export class RoutedSession {
|
|
|
872
946
|
}
|
|
873
947
|
}
|
|
874
948
|
catch (error) {
|
|
949
|
+
if (error instanceof PiboProviderRecoveryCancelledError)
|
|
950
|
+
return;
|
|
875
951
|
const message = errorMessage(error);
|
|
876
952
|
this.emit({
|
|
877
953
|
type: "session_error",
|
|
@@ -883,7 +959,9 @@ export class RoutedSession {
|
|
|
883
959
|
}
|
|
884
960
|
finally {
|
|
885
961
|
this.activeMessage = undefined;
|
|
962
|
+
this.providerRecoveryCancelled = false;
|
|
886
963
|
this.pendingAssistantError = undefined;
|
|
964
|
+
this.pendingAssistantErrorRetryable = false;
|
|
887
965
|
this.activeMessageFailed = false;
|
|
888
966
|
this.activeAssistantIndex = undefined;
|
|
889
967
|
this.nextAssistantIndex = 0;
|
package/dist/core/runtime.js
CHANGED
|
@@ -166,9 +166,9 @@ function getBuiltinToolAllowlist(profile, customTools) {
|
|
|
166
166
|
return undefined;
|
|
167
167
|
return [...selectedBuiltinTools, ...customTools.map((tool) => tool.name)];
|
|
168
168
|
}
|
|
169
|
-
function getProfileExtensionFactories(profile, extensionFactories, contextGuardRecovery) {
|
|
169
|
+
function getProfileExtensionFactories(profile, extensionFactories, contextGuardRecovery, getSettingsManager) {
|
|
170
170
|
const piboPromptTemplateExtension = createPiboSystemPromptTemplateExtension();
|
|
171
|
-
const piboCompactionPromptExtension = createPiboCompactionPromptExtension();
|
|
171
|
+
const piboCompactionPromptExtension = createPiboCompactionPromptExtension({ getSettingsManager });
|
|
172
172
|
const piboContextGuardExtension = createPiboAssistantContextGuardExtension({}, contextGuardRecovery);
|
|
173
173
|
const providerToolExtensions = profile.tools
|
|
174
174
|
.filter((tool) => tool.enabled !== false)
|
|
@@ -241,6 +241,7 @@ export async function createPiboRuntime(options = {}) {
|
|
|
241
241
|
const mcpAgentContextFile = await getMcpAgentContextFile(profile.mcpServers);
|
|
242
242
|
const skillPaths = getEnabledSkillPaths(runtimeCwd, profile);
|
|
243
243
|
const piPackageOptions = getPiPackageRuntimeOptions(runtimeCwd, profile);
|
|
244
|
+
let runtimeSettingsManager;
|
|
244
245
|
const services = await createAgentSessionServices({
|
|
245
246
|
cwd: runtimeCwd,
|
|
246
247
|
agentDir: runtimeAgentDir,
|
|
@@ -248,7 +249,7 @@ export async function createPiboRuntime(options = {}) {
|
|
|
248
249
|
resourceLoaderOptions: {
|
|
249
250
|
...piPackageOptions.resourceLoaderOptions,
|
|
250
251
|
additionalSkillPaths: skillPaths,
|
|
251
|
-
extensionFactories: getProfileExtensionFactories(profile, options.extensionFactories, contextGuardRecovery),
|
|
252
|
+
extensionFactories: getProfileExtensionFactories(profile, options.extensionFactories, contextGuardRecovery, () => runtimeSettingsManager),
|
|
252
253
|
noExtensions: true,
|
|
253
254
|
noSkills: true,
|
|
254
255
|
noPromptTemplates: true,
|
|
@@ -265,6 +266,7 @@ export async function createPiboRuntime(options = {}) {
|
|
|
265
266
|
}),
|
|
266
267
|
},
|
|
267
268
|
});
|
|
269
|
+
runtimeSettingsManager = services.settingsManager;
|
|
268
270
|
applyPiboRuntimeRetryDefaults(services.settingsManager, options.retryDefaults);
|
|
269
271
|
registerOpenAiGpt56Models(services.modelRegistry);
|
|
270
272
|
registerMiniMaxProvider(services.modelRegistry);
|
|
@@ -10,6 +10,11 @@ const PROVIDER_NETWORK_ERROR_MARKERS = [
|
|
|
10
10
|
"reset before headers",
|
|
11
11
|
"socket hang up",
|
|
12
12
|
"socket connection was closed",
|
|
13
|
+
"eai_again",
|
|
14
|
+
"enotfound",
|
|
15
|
+
"econnreset",
|
|
16
|
+
"econnrefused",
|
|
17
|
+
"etimedout",
|
|
13
18
|
];
|
|
14
19
|
export function classifySessionErrorMessage(message, options = {}) {
|
|
15
20
|
const normalized = message.toLowerCase();
|
|
@@ -20,7 +25,15 @@ export function classifySessionErrorMessage(message, options = {}) {
|
|
|
20
25
|
return { category: "provider_transport", errorClass: "provider_transport", code: "websocket_error", origin: "provider", retryable: true, userMessage: "The provider WebSocket connection failed." };
|
|
21
26
|
}
|
|
22
27
|
if (normalized.includes("request was aborted") || normalized.includes("aborted")) {
|
|
23
|
-
return { category: "runtime_abort", errorClass: "runtime_abort", code: "request_aborted", origin: "runtime", retryable:
|
|
28
|
+
return { category: "runtime_abort", errorClass: "runtime_abort", code: "request_aborted", origin: "runtime", retryable: false, userMessage: "The active model request was aborted." };
|
|
29
|
+
}
|
|
30
|
+
if (normalized.includes("insufficient_quota")
|
|
31
|
+
|| normalized.includes("quota exceeded")
|
|
32
|
+
|| normalized.includes("out of budget")
|
|
33
|
+
|| normalized.includes("billing")
|
|
34
|
+
|| normalized.includes("usage limit")
|
|
35
|
+
|| normalized.includes("available balance")) {
|
|
36
|
+
return { category: "quota_exhausted", errorClass: "provider_rate_limit", code: "quota_exhausted", origin: "provider", retryable: false, userMessage: "The provider quota or billing limit was reached." };
|
|
24
37
|
}
|
|
25
38
|
if (normalized.includes("rate limit") || normalized.includes("429")) {
|
|
26
39
|
return { category: "rate_limit", errorClass: "provider_rate_limit", code: "rate_limited", origin: "provider", retryable: true, userMessage: "The provider rate limit was reached." };
|
|
@@ -319,6 +319,8 @@ export class PiboSessionRouter {
|
|
|
319
319
|
const eventWithId = { ...event, id: event.id ?? randomUUID() };
|
|
320
320
|
return await new Promise((resolve, reject) => {
|
|
321
321
|
let settled = false;
|
|
322
|
+
let lastAssistantMessage;
|
|
323
|
+
let timeout;
|
|
322
324
|
const unsubscribe = this.subscribe((output) => {
|
|
323
325
|
if (output.piboSessionId !== eventWithId.piboSessionId ||
|
|
324
326
|
!("eventId" in output) ||
|
|
@@ -326,20 +328,21 @@ export class PiboSessionRouter {
|
|
|
326
328
|
return;
|
|
327
329
|
}
|
|
328
330
|
if (output.type === "assistant_message") {
|
|
329
|
-
|
|
331
|
+
lastAssistantMessage = output;
|
|
332
|
+
}
|
|
333
|
+
else if (output.type === "message_finished") {
|
|
334
|
+
finish(lastAssistantMessage ?? new Error(`Pibo session "${eventWithId.piboSessionId}" finished without an assistant reply`));
|
|
330
335
|
}
|
|
331
336
|
else if (output.type === "session_error") {
|
|
332
337
|
finish(new Error(output.error));
|
|
333
338
|
}
|
|
334
339
|
});
|
|
335
|
-
const timeout = setTimeout(() => {
|
|
336
|
-
finish(new Error(`Timed out waiting for assistant reply from Pibo session "${eventWithId.piboSessionId}"`));
|
|
337
|
-
}, timeoutMs);
|
|
338
340
|
const finish = (result) => {
|
|
339
341
|
if (settled)
|
|
340
342
|
return;
|
|
341
343
|
settled = true;
|
|
342
|
-
|
|
344
|
+
if (timeout)
|
|
345
|
+
clearTimeout(timeout);
|
|
343
346
|
unsubscribe();
|
|
344
347
|
if (result instanceof Error) {
|
|
345
348
|
reject(result);
|
|
@@ -348,6 +351,20 @@ export class PiboSessionRouter {
|
|
|
348
351
|
resolve(result);
|
|
349
352
|
}
|
|
350
353
|
};
|
|
354
|
+
timeout = setTimeout(() => {
|
|
355
|
+
if (settled)
|
|
356
|
+
return;
|
|
357
|
+
settled = true;
|
|
358
|
+
unsubscribe();
|
|
359
|
+
const timeoutError = new Error(`Timed out waiting for assistant reply from Pibo session "${eventWithId.piboSessionId}"`);
|
|
360
|
+
reject(timeoutError);
|
|
361
|
+
void this.emit({
|
|
362
|
+
type: "execution",
|
|
363
|
+
piboSessionId: eventWithId.piboSessionId,
|
|
364
|
+
action: "abort",
|
|
365
|
+
id: randomUUID(),
|
|
366
|
+
}).catch(() => { });
|
|
367
|
+
}, timeoutMs);
|
|
351
368
|
this.emit(eventWithId).catch(finish);
|
|
352
369
|
});
|
|
353
370
|
}
|
package/dist/gateway/request.js
CHANGED
|
@@ -75,6 +75,7 @@ export async function sendGatewayMessageAndWaitForReply(event, options = {}) {
|
|
|
75
75
|
let settled = false;
|
|
76
76
|
let response;
|
|
77
77
|
let reply;
|
|
78
|
+
let messageFinished = false;
|
|
78
79
|
const timeout = setTimeout(() => {
|
|
79
80
|
finish(new Error(`Timed out waiting for assistant reply from session "${event.piboSessionId}"`));
|
|
80
81
|
}, timeoutMs);
|
|
@@ -91,14 +92,21 @@ export async function sendGatewayMessageAndWaitForReply(event, options = {}) {
|
|
|
91
92
|
resolve(result);
|
|
92
93
|
}
|
|
93
94
|
};
|
|
95
|
+
const finishCompletedReply = () => {
|
|
96
|
+
if (!response?.ok || !messageFinished)
|
|
97
|
+
return;
|
|
98
|
+
finish(reply
|
|
99
|
+
? { response, reply }
|
|
100
|
+
: new Error(`Session "${eventWithId.piboSessionId}" finished without an assistant reply`));
|
|
101
|
+
};
|
|
94
102
|
const handleFrame = (frame) => {
|
|
95
103
|
if (frame.type === "res" && frame.id === id) {
|
|
96
104
|
response = frame;
|
|
97
105
|
if (!frame.ok) {
|
|
98
106
|
finish(new Error(frame.error?.message ?? "Gateway rejected the message"));
|
|
99
107
|
}
|
|
100
|
-
else
|
|
101
|
-
|
|
108
|
+
else {
|
|
109
|
+
finishCompletedReply();
|
|
102
110
|
}
|
|
103
111
|
return;
|
|
104
112
|
}
|
|
@@ -111,13 +119,14 @@ export async function sendGatewayMessageAndWaitForReply(event, options = {}) {
|
|
|
111
119
|
finish(new Error(output.error));
|
|
112
120
|
return;
|
|
113
121
|
}
|
|
114
|
-
if (output.
|
|
115
|
-
|
|
116
|
-
|
|
122
|
+
if (output.piboSessionId !== eventWithId.piboSessionId || !("eventId" in output) || output.eventId !== eventWithId.id)
|
|
123
|
+
return;
|
|
124
|
+
if (output.type === "assistant_message") {
|
|
117
125
|
reply = output;
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
126
|
+
}
|
|
127
|
+
else if (output.type === "message_finished") {
|
|
128
|
+
messageFinished = true;
|
|
129
|
+
finishCompletedReply();
|
|
121
130
|
}
|
|
122
131
|
};
|
|
123
132
|
socket.once("connect", () => {
|
package/dist/ralph/service.js
CHANGED
|
@@ -252,7 +252,7 @@ export class PiboRalphService {
|
|
|
252
252
|
},
|
|
253
253
|
});
|
|
254
254
|
this.store.attachRunSession(job.id, run.id, session.id);
|
|
255
|
-
const finalAnswer = await this.emitMessageAndWait(session.id, buildRalphPrompt(job)
|
|
255
|
+
const finalAnswer = await this.emitMessageAndWait(session.id, buildRalphPrompt(job));
|
|
256
256
|
return { piboSessionId: session.id, finalAnswer };
|
|
257
257
|
}
|
|
258
258
|
resolveTarget(job) { if (job.target.kind === 'room') {
|
|
@@ -263,7 +263,7 @@ export class PiboRalphService {
|
|
|
263
263
|
throw new Error('Target room is archived');
|
|
264
264
|
return { roomId: room.id, workspace: room.workspace ?? getDefaultPiboWorkspace() };
|
|
265
265
|
} const room = this.roomService.ensureDefaultRoom({ name: 'Shared Chat' }); return { roomId: room.id, workspace: room.workspace ?? getDefaultPiboWorkspace() }; }
|
|
266
|
-
async emitMessageAndWait(piboSessionId, text
|
|
266
|
+
async emitMessageAndWait(piboSessionId, text) {
|
|
267
267
|
const eventId = `ralph_msg_${randomUUID()}`;
|
|
268
268
|
return await new Promise((resolve, reject) => {
|
|
269
269
|
let settled = false;
|
|
@@ -308,9 +308,7 @@ export class PiboRalphService {
|
|
|
308
308
|
finish(lastSessionError ? new Error(lastSessionError) : undefined);
|
|
309
309
|
if (event.type === 'session_error') {
|
|
310
310
|
lastSessionError = event.error;
|
|
311
|
-
|
|
312
|
-
if (!providerAttempt || options.isCancelled?.())
|
|
313
|
-
finish(new Error(event.error));
|
|
311
|
+
finish(new Error(event.error));
|
|
314
312
|
}
|
|
315
313
|
});
|
|
316
314
|
this.options.context.emit({ type: 'message', piboSessionId, id: eventId, source: 'service', text }).catch((error) => finish(error instanceof Error ? error : new Error(String(error))));
|