@pasko70/pibo 1.7.12 → 1.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apps/chat/workflow-manual-trigger-runtime.js +13 -4
- package/dist/core/compaction-prompt.js +87 -52
- package/dist/core/context-guard.js +172 -5
- package/dist/core/provider-recovery.js +64 -0
- package/dist/core/provider-telemetry.js +97 -4
- package/dist/core/routed-session.js +181 -16
- package/dist/core/runtime-telemetry.js +251 -91
- package/dist/core/runtime.js +53 -12
- package/dist/core/session-errors.js +14 -1
- package/dist/core/session-router.js +127 -26
- package/dist/data/telemetry.js +115 -26
- package/dist/gateway/request.js +17 -8
- package/dist/ralph/service.js +3 -5
- package/package.json +1 -1
- package/dist/apps/vscode-artifacts/latest.vsix +0 -0
- package/dist/apps/vscode-artifacts/pibo-vscode-ext-1.7.12.vsix +0 -0
|
@@ -184,6 +184,7 @@ function emitMessageAndWaitForAssistant(channelContext, piboSessionId, text, tim
|
|
|
184
184
|
return new Promise((resolve, reject) => {
|
|
185
185
|
const eventId = `wfm_${randomUUID()}`;
|
|
186
186
|
let settled = false;
|
|
187
|
+
let lastAssistantMessage;
|
|
187
188
|
let timeout;
|
|
188
189
|
let unsubscribe = () => { };
|
|
189
190
|
const finish = (value) => {
|
|
@@ -197,16 +198,24 @@ function emitMessageAndWaitForAssistant(channelContext, piboSessionId, text, tim
|
|
|
197
198
|
else
|
|
198
199
|
resolve(value);
|
|
199
200
|
};
|
|
200
|
-
timeout = setTimeout(() =>
|
|
201
|
+
timeout = setTimeout(() => {
|
|
202
|
+
finish(new Error(`Timed out waiting for assistant reply from workflow agent session '${piboSessionId}'.`));
|
|
203
|
+
void channelContext.emit({ type: "execution", piboSessionId, action: "abort", id: `wfm_abort_${randomUUID()}` }).catch(() => { });
|
|
204
|
+
}, timeoutMs);
|
|
201
205
|
unsubscribe = channelContext.subscribe((event) => {
|
|
202
206
|
if (event.piboSessionId !== piboSessionId)
|
|
203
207
|
return;
|
|
208
|
+
if (!("eventId" in event) || event.eventId !== eventId)
|
|
209
|
+
return;
|
|
204
210
|
if (event.type === "assistant_message") {
|
|
205
|
-
|
|
206
|
-
|
|
211
|
+
lastAssistantMessage = event.text;
|
|
212
|
+
return;
|
|
213
|
+
}
|
|
214
|
+
if (event.type === "message_finished") {
|
|
215
|
+
finish(lastAssistantMessage ?? new Error(`Workflow agent session '${piboSessionId}' finished without an assistant reply.`));
|
|
207
216
|
return;
|
|
208
217
|
}
|
|
209
|
-
if (event.type === "session_error"
|
|
218
|
+
if (event.type === "session_error")
|
|
210
219
|
finish(new Error(event.error));
|
|
211
220
|
});
|
|
212
221
|
channelContext.emit({ type: "message", piboSessionId, id: eventId, text, source: "actor" }).catch(finish);
|
|
@@ -1,9 +1,11 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
1
2
|
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
2
3
|
import { readFile, writeFile } from "node:fs/promises";
|
|
3
4
|
import { dirname, resolve } from "node:path";
|
|
4
5
|
import { fileURLToPath } from "node:url";
|
|
5
6
|
import { completeSimple } from "@earendil-works/pi-ai/compat";
|
|
6
7
|
import { buildSessionContext, convertToLlm, serializeConversation, } from "@earendil-works/pi-coding-agent";
|
|
8
|
+
import { isRetryablePiboAssistantError, isRetryablePiboProviderError, resolvePiboProviderRecoverySettings, waitForPiboProviderRecovery, } from "./provider-recovery.js";
|
|
7
9
|
const PROJECT_ROOT = dirname(dirname(dirname(fileURLToPath(import.meta.url))));
|
|
8
10
|
export const PIBO_LIBRARY_COMPACTION_PROMPT_PATH = resolve(PROJECT_ROOT, "context/pibo-compaction-prompt.md");
|
|
9
11
|
function getCompactionPromptStatePath(cwd) {
|
|
@@ -134,38 +136,62 @@ function buildTurnPrefixPrompt(spec, messages) {
|
|
|
134
136
|
const conversationText = serializeConversation(convertToLlm(messages));
|
|
135
137
|
return `<conversation>\n${conversationText}\n</conversation>\n\n${spec.turnPrefixSummaryPrompt}`;
|
|
136
138
|
}
|
|
137
|
-
async function
|
|
138
|
-
const
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
139
|
+
export async function completePiboCompactionSummary(input) {
|
|
140
|
+
const recoverySessionId = input.sessionId ? `compaction-${randomUUID()}` : undefined;
|
|
141
|
+
const completionOptions = {
|
|
142
|
+
maxTokens: input.maxTokens,
|
|
143
|
+
signal: input.signal,
|
|
144
|
+
apiKey: input.apiKey,
|
|
145
|
+
headers: input.headers,
|
|
146
|
+
transport: input.transport,
|
|
147
|
+
sessionId: recoverySessionId,
|
|
148
|
+
timeoutMs: input.timeoutMs,
|
|
149
|
+
websocketConnectTimeoutMs: input.websocketConnectTimeoutMs,
|
|
150
|
+
maxRetries: input.maxRetries,
|
|
151
|
+
maxRetryDelayMs: input.maxRetryDelayMs,
|
|
152
|
+
...(input.model.reasoning && input.thinkingLevel && input.thinkingLevel !== "off"
|
|
153
|
+
? { reasoning: input.thinkingLevel }
|
|
154
|
+
: {}),
|
|
155
|
+
};
|
|
156
|
+
const complete = input.complete ?? completeSimple;
|
|
157
|
+
let recoveryAttempt = 0;
|
|
158
|
+
while (true) {
|
|
159
|
+
let response;
|
|
160
|
+
try {
|
|
161
|
+
response = await complete(input.model, {
|
|
162
|
+
systemPrompt: input.systemPrompt,
|
|
163
|
+
messages: [
|
|
164
|
+
{
|
|
165
|
+
role: "user",
|
|
166
|
+
content: [{ type: "text", text: input.promptText }],
|
|
167
|
+
timestamp: Date.now(),
|
|
168
|
+
},
|
|
169
|
+
],
|
|
170
|
+
}, completionOptions);
|
|
145
171
|
}
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
172
|
+
catch (error) {
|
|
173
|
+
if (input.signal?.aborted || !input.recovery.enabled || !isRetryablePiboProviderError(error))
|
|
174
|
+
throw error;
|
|
175
|
+
recoveryAttempt += 1;
|
|
176
|
+
await waitForPiboProviderRecovery(recoveryAttempt, input.recovery, input.signal);
|
|
177
|
+
continue;
|
|
178
|
+
}
|
|
179
|
+
if (response.stopReason === "aborted") {
|
|
180
|
+
throw new Error("Summarization was aborted");
|
|
181
|
+
}
|
|
182
|
+
if (response.stopReason === "error") {
|
|
183
|
+
const error = new Error(`Summarization failed: ${response.errorMessage || "Unknown error"}`);
|
|
184
|
+
if (!input.recovery.enabled || !isRetryablePiboAssistantError(response))
|
|
185
|
+
throw error;
|
|
186
|
+
recoveryAttempt += 1;
|
|
187
|
+
await waitForPiboProviderRecovery(recoveryAttempt, input.recovery, input.signal);
|
|
188
|
+
continue;
|
|
189
|
+
}
|
|
190
|
+
return response.content
|
|
191
|
+
.filter((content) => content.type === "text")
|
|
192
|
+
.map((content) => content.text)
|
|
193
|
+
.join("\n");
|
|
164
194
|
}
|
|
165
|
-
return response.content
|
|
166
|
-
.filter((content) => content.type === "text")
|
|
167
|
-
.map((content) => content.text)
|
|
168
|
-
.join("\n");
|
|
169
195
|
}
|
|
170
196
|
function computeFileLists(fileOps) {
|
|
171
197
|
const modified = new Set([...fileOps.written, ...fileOps.edited]);
|
|
@@ -183,44 +209,44 @@ function formatFileOperations(readFiles, modifiedFiles) {
|
|
|
183
209
|
}
|
|
184
210
|
async function generatePiboCompaction(input) {
|
|
185
211
|
const { preparation, spec } = input;
|
|
212
|
+
const completionDefaults = {
|
|
213
|
+
model: input.model,
|
|
214
|
+
systemPrompt: spec.systemPrompt,
|
|
215
|
+
apiKey: input.apiKey,
|
|
216
|
+
headers: input.headers,
|
|
217
|
+
signal: input.signal,
|
|
218
|
+
thinkingLevel: input.thinkingLevel,
|
|
219
|
+
transport: input.transport,
|
|
220
|
+
sessionId: input.sessionId,
|
|
221
|
+
timeoutMs: input.timeoutMs,
|
|
222
|
+
websocketConnectTimeoutMs: input.websocketConnectTimeoutMs,
|
|
223
|
+
maxRetries: input.maxRetries,
|
|
224
|
+
maxRetryDelayMs: input.maxRetryDelayMs,
|
|
225
|
+
recovery: input.recovery,
|
|
226
|
+
};
|
|
186
227
|
let summary;
|
|
187
228
|
if (preparation.isSplitTurn && preparation.turnPrefixMessages.length > 0) {
|
|
188
229
|
const [historySummary, turnPrefixSummary] = await Promise.all([
|
|
189
230
|
preparation.messagesToSummarize.length > 0
|
|
190
|
-
?
|
|
191
|
-
|
|
192
|
-
systemPrompt: spec.systemPrompt,
|
|
231
|
+
? completePiboCompactionSummary({
|
|
232
|
+
...completionDefaults,
|
|
193
233
|
promptText: buildSummaryPrompt(spec, preparation.messagesToSummarize, input.customInstructions, preparation.previousSummary),
|
|
194
234
|
maxTokens: Math.floor(0.8 * preparation.settings.reserveTokens),
|
|
195
|
-
apiKey: input.apiKey,
|
|
196
|
-
headers: input.headers,
|
|
197
|
-
signal: input.signal,
|
|
198
|
-
thinkingLevel: input.thinkingLevel,
|
|
199
235
|
})
|
|
200
236
|
: Promise.resolve("No prior history."),
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
systemPrompt: spec.systemPrompt,
|
|
237
|
+
completePiboCompactionSummary({
|
|
238
|
+
...completionDefaults,
|
|
204
239
|
promptText: buildTurnPrefixPrompt(spec, preparation.turnPrefixMessages),
|
|
205
240
|
maxTokens: Math.floor(0.5 * preparation.settings.reserveTokens),
|
|
206
|
-
apiKey: input.apiKey,
|
|
207
|
-
headers: input.headers,
|
|
208
|
-
signal: input.signal,
|
|
209
|
-
thinkingLevel: input.thinkingLevel,
|
|
210
241
|
}),
|
|
211
242
|
]);
|
|
212
243
|
summary = `${historySummary}\n\n---\n\n**Turn Context (split turn):**\n\n${turnPrefixSummary}`;
|
|
213
244
|
}
|
|
214
245
|
else {
|
|
215
|
-
summary = await
|
|
216
|
-
|
|
217
|
-
systemPrompt: spec.systemPrompt,
|
|
246
|
+
summary = await completePiboCompactionSummary({
|
|
247
|
+
...completionDefaults,
|
|
218
248
|
promptText: buildSummaryPrompt(spec, preparation.messagesToSummarize, input.customInstructions, preparation.previousSummary),
|
|
219
249
|
maxTokens: Math.floor(0.8 * preparation.settings.reserveTokens),
|
|
220
|
-
apiKey: input.apiKey,
|
|
221
|
-
headers: input.headers,
|
|
222
|
-
signal: input.signal,
|
|
223
|
-
thinkingLevel: input.thinkingLevel,
|
|
224
250
|
});
|
|
225
251
|
}
|
|
226
252
|
const { readFiles, modifiedFiles } = computeFileLists(preparation.fileOps);
|
|
@@ -232,7 +258,7 @@ async function generatePiboCompaction(input) {
|
|
|
232
258
|
details: { readFiles, modifiedFiles },
|
|
233
259
|
};
|
|
234
260
|
}
|
|
235
|
-
export function createPiboCompactionPromptExtension() {
|
|
261
|
+
export function createPiboCompactionPromptExtension(options = {}) {
|
|
236
262
|
return (pi) => {
|
|
237
263
|
pi.on("session_before_compact", async (event, ctx) => {
|
|
238
264
|
if (!ctx.model)
|
|
@@ -241,6 +267,8 @@ export function createPiboCompactionPromptExtension() {
|
|
|
241
267
|
if (!auth.ok || !auth.apiKey)
|
|
242
268
|
return undefined;
|
|
243
269
|
const sessionContext = buildSessionContext(ctx.sessionManager.getEntries(), ctx.sessionManager.getLeafId());
|
|
270
|
+
const settingsManager = options.getSettingsManager?.();
|
|
271
|
+
const providerRetry = settingsManager?.getProviderRetrySettings();
|
|
244
272
|
return {
|
|
245
273
|
compaction: await generatePiboCompaction({
|
|
246
274
|
preparation: event.preparation,
|
|
@@ -251,6 +279,13 @@ export function createPiboCompactionPromptExtension() {
|
|
|
251
279
|
customInstructions: event.customInstructions,
|
|
252
280
|
signal: event.signal,
|
|
253
281
|
thinkingLevel: sessionContext.thinkingLevel,
|
|
282
|
+
transport: settingsManager?.getTransport(),
|
|
283
|
+
sessionId: ctx.sessionManager.getSessionId(),
|
|
284
|
+
timeoutMs: settingsManager?.getHttpIdleTimeoutMs(),
|
|
285
|
+
websocketConnectTimeoutMs: settingsManager?.getWebSocketConnectTimeoutMs(),
|
|
286
|
+
maxRetries: providerRetry?.maxRetries,
|
|
287
|
+
maxRetryDelayMs: providerRetry?.maxRetryDelayMs,
|
|
288
|
+
recovery: resolvePiboProviderRecoverySettings(settingsManager),
|
|
254
289
|
}),
|
|
255
290
|
};
|
|
256
291
|
});
|
|
@@ -1,7 +1,92 @@
|
|
|
1
1
|
import { DEFAULT_COMPACTION_SETTINGS, buildSessionContext, estimateTokens, } from "@earendil-works/pi-coding-agent";
|
|
2
2
|
export const PIBO_CONTEXT_GUARD_NOTICE = "Context safety interrupted this response before adding it to long-term context. Pibo is compacting the session before continuing.";
|
|
3
|
+
export const PIBO_CONTEXT_GUARD_RESUME_MESSAGE_TYPE = "pibo-context-guard-resume";
|
|
4
|
+
export const PIBO_CONTEXT_GUARD_RESUME_PROMPT = "Continue the interrupted task autonomously from the compacted context. Do not wait for additional user input and do not repeat the context-safety notice.";
|
|
3
5
|
const DEFAULT_MIN_COMPACTION_RESERVE_TOKENS = 1024;
|
|
4
6
|
const FALLBACK_CONTEXT_WINDOW = 0;
|
|
7
|
+
const MAX_CONTEXT_GUARD_RECOVERY_ATTEMPTS = 3;
|
|
8
|
+
const contextGuardRecoveries = new WeakMap();
|
|
9
|
+
export function createPiboAssistantContextGuardRecovery() {
|
|
10
|
+
let pending;
|
|
11
|
+
let claimed = false;
|
|
12
|
+
let cancelError;
|
|
13
|
+
let cancelHandler;
|
|
14
|
+
return {
|
|
15
|
+
begin() {
|
|
16
|
+
if (pending && !pending.settled)
|
|
17
|
+
return;
|
|
18
|
+
cancelError = undefined;
|
|
19
|
+
let resolve;
|
|
20
|
+
const promise = new Promise((done) => {
|
|
21
|
+
resolve = done;
|
|
22
|
+
});
|
|
23
|
+
pending = { promise, resolve, settled: false };
|
|
24
|
+
},
|
|
25
|
+
complete() {
|
|
26
|
+
if (!pending || pending.settled)
|
|
27
|
+
return;
|
|
28
|
+
pending.settled = true;
|
|
29
|
+
pending.resolve({ ok: true });
|
|
30
|
+
},
|
|
31
|
+
fail(error) {
|
|
32
|
+
if (!pending || pending.settled)
|
|
33
|
+
return;
|
|
34
|
+
pending.settled = true;
|
|
35
|
+
pending.resolve({ ok: false, error });
|
|
36
|
+
},
|
|
37
|
+
cancel(error) {
|
|
38
|
+
cancelError = error;
|
|
39
|
+
if (pending && !pending.settled) {
|
|
40
|
+
pending.settled = true;
|
|
41
|
+
pending.resolve({ ok: false, error });
|
|
42
|
+
}
|
|
43
|
+
cancelHandler?.();
|
|
44
|
+
},
|
|
45
|
+
claim() {
|
|
46
|
+
claimed = true;
|
|
47
|
+
},
|
|
48
|
+
isClaimed() {
|
|
49
|
+
return claimed;
|
|
50
|
+
},
|
|
51
|
+
isPending() {
|
|
52
|
+
return pending !== undefined && !pending.settled;
|
|
53
|
+
},
|
|
54
|
+
onCancel(handler) {
|
|
55
|
+
cancelHandler = handler;
|
|
56
|
+
},
|
|
57
|
+
async wait() {
|
|
58
|
+
const current = pending;
|
|
59
|
+
if (!current)
|
|
60
|
+
return false;
|
|
61
|
+
const outcome = await current.promise;
|
|
62
|
+
const cancelled = cancelError;
|
|
63
|
+
if (pending === current) {
|
|
64
|
+
pending = undefined;
|
|
65
|
+
cancelError = undefined;
|
|
66
|
+
}
|
|
67
|
+
if (cancelled)
|
|
68
|
+
throw cancelled;
|
|
69
|
+
if (!outcome.ok)
|
|
70
|
+
throw outcome.error;
|
|
71
|
+
return true;
|
|
72
|
+
},
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
export function registerPiboAssistantContextGuardRecovery(session, recovery) {
|
|
76
|
+
contextGuardRecoveries.set(session, recovery);
|
|
77
|
+
}
|
|
78
|
+
export function claimPiboAssistantContextGuardRecovery(session) {
|
|
79
|
+
contextGuardRecoveries.get(session)?.claim();
|
|
80
|
+
}
|
|
81
|
+
export function cancelPiboAssistantContextGuardRecovery(session, error) {
|
|
82
|
+
contextGuardRecoveries.get(session)?.cancel(error);
|
|
83
|
+
}
|
|
84
|
+
export function isPiboAssistantContextGuardRecoveryPending(session) {
|
|
85
|
+
return contextGuardRecoveries.get(session)?.isPending() ?? false;
|
|
86
|
+
}
|
|
87
|
+
export async function waitForPiboAssistantContextGuardRecovery(session) {
|
|
88
|
+
return await contextGuardRecoveries.get(session)?.wait() ?? false;
|
|
89
|
+
}
|
|
5
90
|
function textTokensByLength(length) {
|
|
6
91
|
return Math.ceil(length / 4);
|
|
7
92
|
}
|
|
@@ -79,15 +164,43 @@ function compactionInstructions(projection) {
|
|
|
79
164
|
: "";
|
|
80
165
|
return `Pibo interrupted the previous assistant response before persisting the full output because it would exceed the safe context budget.${usage} Summarize the durable conversation up to the guard notice, preserve the user's current task and important recent facts, and leave enough context budget for the next response.`;
|
|
81
166
|
}
|
|
82
|
-
export function createPiboAssistantContextGuardExtension(options = {}) {
|
|
167
|
+
export function createPiboAssistantContextGuardExtension(options = {}, recovery = createPiboAssistantContextGuardRecovery()) {
|
|
83
168
|
return (pi) => {
|
|
84
|
-
const state = {
|
|
169
|
+
const state = {
|
|
170
|
+
deltaChars: 0,
|
|
171
|
+
tripped: false,
|
|
172
|
+
compactQueued: false,
|
|
173
|
+
resumeRequested: false,
|
|
174
|
+
resumeInProgress: false,
|
|
175
|
+
resumeSettling: false,
|
|
176
|
+
recoveryAttempts: 0,
|
|
177
|
+
};
|
|
85
178
|
function resetAssistantState() {
|
|
86
179
|
state.deltaChars = 0;
|
|
87
180
|
state.tripped = false;
|
|
88
181
|
state.compactQueued = false;
|
|
89
182
|
state.lastProjection = undefined;
|
|
90
183
|
}
|
|
184
|
+
function clearResumeStartTimer() {
|
|
185
|
+
if (state.resumeStartTimer === undefined)
|
|
186
|
+
return;
|
|
187
|
+
clearTimeout(state.resumeStartTimer);
|
|
188
|
+
state.resumeStartTimer = undefined;
|
|
189
|
+
}
|
|
190
|
+
function resetRecoveryState() {
|
|
191
|
+
clearResumeStartTimer();
|
|
192
|
+
state.tripped = false;
|
|
193
|
+
state.compactQueued = false;
|
|
194
|
+
state.resumeRequested = false;
|
|
195
|
+
state.resumeInProgress = false;
|
|
196
|
+
state.resumeSettling = false;
|
|
197
|
+
state.recoveryAttempts = 0;
|
|
198
|
+
}
|
|
199
|
+
function failRecovery(error) {
|
|
200
|
+
resetRecoveryState();
|
|
201
|
+
recovery.fail(error);
|
|
202
|
+
}
|
|
203
|
+
recovery.onCancel(resetRecoveryState);
|
|
91
204
|
function tripIfNeeded(ctx, assistantTokens) {
|
|
92
205
|
const projection = projectAssistantContextGuard(ctx, assistantTokens, options);
|
|
93
206
|
state.lastProjection = projection;
|
|
@@ -96,6 +209,20 @@ export function createPiboAssistantContextGuardExtension(options = {}) {
|
|
|
96
209
|
state.tripped = true;
|
|
97
210
|
return true;
|
|
98
211
|
}
|
|
212
|
+
pi.on("agent_start", () => {
|
|
213
|
+
if (!state.resumeRequested)
|
|
214
|
+
return;
|
|
215
|
+
clearResumeStartTimer();
|
|
216
|
+
state.resumeRequested = false;
|
|
217
|
+
state.resumeInProgress = true;
|
|
218
|
+
});
|
|
219
|
+
pi.on("agent_settled", () => {
|
|
220
|
+
if (!state.resumeSettling)
|
|
221
|
+
return;
|
|
222
|
+
state.resumeSettling = false;
|
|
223
|
+
state.recoveryAttempts = 0;
|
|
224
|
+
recovery.complete();
|
|
225
|
+
});
|
|
99
226
|
pi.on("message_start", (event) => {
|
|
100
227
|
if (event.message.role === "assistant")
|
|
101
228
|
resetAssistantState();
|
|
@@ -117,10 +244,50 @@ export function createPiboAssistantContextGuardExtension(options = {}) {
|
|
|
117
244
|
return { message: replacementAssistantMessage(event.message) };
|
|
118
245
|
});
|
|
119
246
|
pi.on("agent_end", (_event, ctx) => {
|
|
120
|
-
if (
|
|
247
|
+
if (state.tripped && !state.compactQueued) {
|
|
248
|
+
if (state.recoveryAttempts >= MAX_CONTEXT_GUARD_RECOVERY_ATTEMPTS) {
|
|
249
|
+
failRecovery(new Error(`Context guard recovery exceeded ${MAX_CONTEXT_GUARD_RECOVERY_ATTEMPTS} compaction attempts`));
|
|
250
|
+
return;
|
|
251
|
+
}
|
|
252
|
+
state.compactQueued = true;
|
|
253
|
+
state.recoveryAttempts++;
|
|
254
|
+
recovery.begin();
|
|
255
|
+
let callbackHandled = false;
|
|
256
|
+
ctx.compact({
|
|
257
|
+
customInstructions: compactionInstructions(state.lastProjection),
|
|
258
|
+
onComplete: () => {
|
|
259
|
+
if (callbackHandled || !recovery.isPending())
|
|
260
|
+
return;
|
|
261
|
+
callbackHandled = true;
|
|
262
|
+
state.tripped = false;
|
|
263
|
+
state.compactQueued = false;
|
|
264
|
+
state.resumeRequested = true;
|
|
265
|
+
if (recovery.isClaimed()) {
|
|
266
|
+
recovery.complete();
|
|
267
|
+
return;
|
|
268
|
+
}
|
|
269
|
+
state.resumeStartTimer = setTimeout(() => {
|
|
270
|
+
failRecovery(new Error("Context guard continuation did not start"));
|
|
271
|
+
}, 5000);
|
|
272
|
+
pi.sendMessage({
|
|
273
|
+
customType: PIBO_CONTEXT_GUARD_RESUME_MESSAGE_TYPE,
|
|
274
|
+
content: [{ type: "text", text: PIBO_CONTEXT_GUARD_RESUME_PROMPT }],
|
|
275
|
+
display: false,
|
|
276
|
+
}, { triggerTurn: true });
|
|
277
|
+
},
|
|
278
|
+
onError: (error) => {
|
|
279
|
+
if (callbackHandled || !recovery.isPending())
|
|
280
|
+
return;
|
|
281
|
+
callbackHandled = true;
|
|
282
|
+
failRecovery(error);
|
|
283
|
+
},
|
|
284
|
+
});
|
|
285
|
+
return;
|
|
286
|
+
}
|
|
287
|
+
if (!state.resumeInProgress)
|
|
121
288
|
return;
|
|
122
|
-
state.
|
|
123
|
-
|
|
289
|
+
state.resumeInProgress = false;
|
|
290
|
+
state.resumeSettling = true;
|
|
124
291
|
});
|
|
125
292
|
};
|
|
126
293
|
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { isRetryableAssistantError } from "@earendil-works/pi-ai";
|
|
2
|
+
import { classifySessionErrorMessage } from "./session-errors.js";
|
|
3
|
+
export const PIBO_PROVIDER_RECOVERY_MESSAGE_TYPE = "pibo-provider-recovery-resume";
|
|
4
|
+
export const PIBO_PROVIDER_RECOVERY_PROMPT = "Continue the interrupted task autonomously from the existing session state. The previous provider request failed transiently. Do not wait for more user input, ask the user to repeat the request, or mention this recovery message unless the failure affects the result.";
|
|
5
|
+
const DEFAULT_RECOVERY_BASE_DELAY_MS = 2_000;
|
|
6
|
+
const DEFAULT_RECOVERY_MAX_DELAY_MS = 60_000;
|
|
7
|
+
export class PiboProviderRecoveryCancelledError extends Error {
|
|
8
|
+
constructor(message = "Provider recovery cancelled") {
|
|
9
|
+
super(message);
|
|
10
|
+
this.name = "PiboProviderRecoveryCancelledError";
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
export function resolvePiboProviderRecoverySettings(settingsManager) {
|
|
14
|
+
if (!settingsManager) {
|
|
15
|
+
return {
|
|
16
|
+
enabled: false,
|
|
17
|
+
baseDelayMs: DEFAULT_RECOVERY_BASE_DELAY_MS,
|
|
18
|
+
maxDelayMs: DEFAULT_RECOVERY_MAX_DELAY_MS,
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
const retry = settingsManager.getRetrySettings();
|
|
22
|
+
const provider = settingsManager.getProviderRetrySettings();
|
|
23
|
+
return {
|
|
24
|
+
enabled: retry.enabled,
|
|
25
|
+
baseDelayMs: Math.max(0, retry.baseDelayMs),
|
|
26
|
+
maxDelayMs: provider.maxRetryDelayMs > 0 ? provider.maxRetryDelayMs : DEFAULT_RECOVERY_MAX_DELAY_MS,
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
export function piboProviderRecoveryDelayMs(attempt, settings) {
|
|
30
|
+
const exponent = Math.max(0, Math.floor(attempt) - 1);
|
|
31
|
+
const uncapped = settings.baseDelayMs * 2 ** Math.min(exponent, 30);
|
|
32
|
+
return Math.min(settings.maxDelayMs, Number.isFinite(uncapped) ? uncapped : settings.maxDelayMs);
|
|
33
|
+
}
|
|
34
|
+
export function isRetryablePiboAssistantError(message) {
|
|
35
|
+
if (!message || typeof message !== "object")
|
|
36
|
+
return false;
|
|
37
|
+
const assistantMessage = message;
|
|
38
|
+
if (isRetryableAssistantError(assistantMessage))
|
|
39
|
+
return true;
|
|
40
|
+
return typeof assistantMessage.errorMessage === "string"
|
|
41
|
+
&& classifySessionErrorMessage(assistantMessage.errorMessage, { hasProviderContext: true }).retryable === true;
|
|
42
|
+
}
|
|
43
|
+
export function isRetryablePiboProviderError(error) {
|
|
44
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
45
|
+
return isRetryablePiboAssistantError({ stopReason: "error", errorMessage });
|
|
46
|
+
}
|
|
47
|
+
export async function waitForPiboProviderRecovery(attempt, settings, signal) {
|
|
48
|
+
if (signal?.aborted)
|
|
49
|
+
throw new PiboProviderRecoveryCancelledError();
|
|
50
|
+
const delayMs = piboProviderRecoveryDelayMs(attempt, settings);
|
|
51
|
+
if (delayMs <= 0)
|
|
52
|
+
return;
|
|
53
|
+
await new Promise((resolve, reject) => {
|
|
54
|
+
const onAbort = () => {
|
|
55
|
+
clearTimeout(timer);
|
|
56
|
+
reject(new PiboProviderRecoveryCancelledError());
|
|
57
|
+
};
|
|
58
|
+
const timer = setTimeout(() => {
|
|
59
|
+
signal?.removeEventListener("abort", onAbort);
|
|
60
|
+
resolve();
|
|
61
|
+
}, delayMs);
|
|
62
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
63
|
+
});
|
|
64
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
2
|
import { BestEffortTelemetryService, } from "../data/telemetry.js";
|
|
3
|
+
import { normalizeSessionErrorDetails } from "./session-errors.js";
|
|
3
4
|
export class PiboProviderTelemetryRecorder {
|
|
4
5
|
options;
|
|
5
6
|
telemetry;
|
|
@@ -16,6 +17,9 @@ export class PiboProviderTelemetryRecorder {
|
|
|
16
17
|
if (!turn)
|
|
17
18
|
return undefined;
|
|
18
19
|
const now = options.at ?? new Date().toISOString();
|
|
20
|
+
if (this.activeProviderRequest()) {
|
|
21
|
+
this.finishActiveProviderRequest("aborted", now, "provider request superseded", undefined, "provider_superseded");
|
|
22
|
+
}
|
|
19
23
|
const model = modelInfo(options.model, this.options.model, payload);
|
|
20
24
|
const providerRequestId = `pr_${randomUUID()}`;
|
|
21
25
|
const phaseId = this.nextPhaseId(turn.turnId, "provider_request");
|
|
@@ -83,7 +87,7 @@ export class PiboProviderTelemetryRecorder {
|
|
|
83
87
|
lastProgressAt: now,
|
|
84
88
|
summary: "provider response received",
|
|
85
89
|
});
|
|
86
|
-
const streamPhaseId =
|
|
90
|
+
const streamPhaseId = providerStreamPhaseId(request);
|
|
87
91
|
this.telemetry.upsertPhase({
|
|
88
92
|
phaseId: streamPhaseId,
|
|
89
93
|
turnId: request.turnId,
|
|
@@ -122,6 +126,56 @@ export class PiboProviderTelemetryRecorder {
|
|
|
122
126
|
return undefined;
|
|
123
127
|
}
|
|
124
128
|
}
|
|
129
|
+
// The assistant message boundary ends the provider stream even when the wider
|
|
130
|
+
// Pibo turn continues with a long-running tool or another provider request.
|
|
131
|
+
recordMessageEnd(message, options = {}) {
|
|
132
|
+
if (!this.options.store || !isAssistantMessage(message))
|
|
133
|
+
return undefined;
|
|
134
|
+
const status = providerStatusForMessage(message);
|
|
135
|
+
const summary = providerSummaryForStatus(status);
|
|
136
|
+
const errorMessage = safeMessageError(message);
|
|
137
|
+
const errorDetails = status === "error"
|
|
138
|
+
? normalizeSessionErrorDetails(errorMessage ?? "Provider request failed.", {
|
|
139
|
+
api: optionalString(message.api),
|
|
140
|
+
provider: optionalString(message.provider),
|
|
141
|
+
model: optionalString(message.model),
|
|
142
|
+
})
|
|
143
|
+
: undefined;
|
|
144
|
+
return this.finishActiveProviderRequest(status, options.at ?? new Date().toISOString(), summary, errorMessage, errorDetails?.category ?? errorDetails?.errorClass);
|
|
145
|
+
}
|
|
146
|
+
recordShutdown(reason, at = new Date().toISOString()) {
|
|
147
|
+
return this.finishActiveProviderRequest("aborted", at, reason, undefined, "runtime_abort");
|
|
148
|
+
}
|
|
149
|
+
finishActiveProviderRequest(status, now, summary, errorMessage, errorCategory) {
|
|
150
|
+
if (!this.options.store)
|
|
151
|
+
return undefined;
|
|
152
|
+
try {
|
|
153
|
+
const request = this.activeProviderRequest();
|
|
154
|
+
if (!request)
|
|
155
|
+
return undefined;
|
|
156
|
+
const phaseStatus = status === "completed" ? "ok" : status;
|
|
157
|
+
const requestPhaseId = request.phaseId ?? `${request.turnId}:provider_request`;
|
|
158
|
+
if (this.options.store.getPhase(requestPhaseId)?.status === "open") {
|
|
159
|
+
this.telemetry.finishPhase(requestPhaseId, { status: phaseStatus, endedAt: now, lastProgressAt: now, summary });
|
|
160
|
+
}
|
|
161
|
+
const streamPhaseId = providerStreamPhaseId(request);
|
|
162
|
+
if (this.options.store.getPhase(streamPhaseId)?.status === "open") {
|
|
163
|
+
this.telemetry.finishPhase(streamPhaseId, { status: phaseStatus, endedAt: now, lastProgressAt: now, summary });
|
|
164
|
+
}
|
|
165
|
+
const updated = this.updateProviderRequest(request, {
|
|
166
|
+
status,
|
|
167
|
+
completedAt: now,
|
|
168
|
+
errorCategory: status === "error" ? errorCategory ?? "provider_error" : status === "aborted" ? errorCategory ?? "runtime_abort" : undefined,
|
|
169
|
+
errorMessage,
|
|
170
|
+
});
|
|
171
|
+
this.activeProviderRequestId = undefined;
|
|
172
|
+
return updated;
|
|
173
|
+
}
|
|
174
|
+
catch (error) {
|
|
175
|
+
this.options.onError?.(error);
|
|
176
|
+
return undefined;
|
|
177
|
+
}
|
|
178
|
+
}
|
|
125
179
|
activeTurn() {
|
|
126
180
|
const detail = this.options.store?.getSessionTelemetry(this.options.session.id, { limit: 10 });
|
|
127
181
|
const active = detail?.activeTurn;
|
|
@@ -138,11 +192,10 @@ export class PiboProviderTelemetryRecorder {
|
|
|
138
192
|
const turn = this.activeTurn();
|
|
139
193
|
if (!turn)
|
|
140
194
|
return undefined;
|
|
141
|
-
|
|
142
|
-
return [...(timeline?.providerRequests ?? [])].reverse().find((request) => !isTerminalProviderStatus(request.status));
|
|
195
|
+
return this.options.store?.getActiveProviderRequestForTurn(turn.turnId);
|
|
143
196
|
}
|
|
144
197
|
nextPhaseId(turnId, phaseName) {
|
|
145
|
-
const count = this.options.store?.
|
|
198
|
+
const count = this.options.store?.countPhasesForTurn(turnId, phaseName) ?? 0;
|
|
146
199
|
return count === 0 ? `${turnId}:${phaseName}` : `${turnId}:${phaseName}:${count + 1}`;
|
|
147
200
|
}
|
|
148
201
|
updateProviderRequest(existing, input) {
|
|
@@ -162,12 +215,18 @@ export class PiboProviderTelemetryRecorder {
|
|
|
162
215
|
responseHeadersAt: input.responseHeadersAt,
|
|
163
216
|
firstByteAt: input.firstByteAt,
|
|
164
217
|
httpStatus: input.httpStatus,
|
|
218
|
+
completedAt: input.completedAt,
|
|
165
219
|
rawEventCount: existing.rawEventCount,
|
|
166
220
|
normalizedEventCount: existing.normalizedEventCount,
|
|
167
221
|
parseErrorCount: existing.parseErrorCount,
|
|
168
222
|
unknownEventCount: existing.unknownEventCount,
|
|
169
223
|
bytesReceived: existing.bytesReceived,
|
|
170
224
|
eventTypeCounts: existing.eventTypeCounts,
|
|
225
|
+
eventStreamId: existing.eventStreamId,
|
|
226
|
+
eventId: existing.eventId,
|
|
227
|
+
payloadRef: existing.payloadRef,
|
|
228
|
+
errorCategory: input.errorCategory,
|
|
229
|
+
errorMessage: input.errorMessage,
|
|
171
230
|
captureMode: existing.captureMode,
|
|
172
231
|
retentionClass: existing.retentionClass,
|
|
173
232
|
});
|
|
@@ -182,11 +241,45 @@ export function createPiboProviderTelemetryExtension(options) {
|
|
|
182
241
|
pi.on("after_provider_response", (event) => {
|
|
183
242
|
recorder.recordResponse({ status: event.status, headers: event.headers });
|
|
184
243
|
});
|
|
244
|
+
pi.on("message_end", (event) => {
|
|
245
|
+
recorder.recordMessageEnd(event.message);
|
|
246
|
+
});
|
|
247
|
+
pi.on("session_shutdown", (event) => {
|
|
248
|
+
recorder.recordShutdown(`provider session shutdown: ${event.reason}`);
|
|
249
|
+
});
|
|
185
250
|
};
|
|
186
251
|
}
|
|
187
252
|
export function isTerminalProviderStatus(status) {
|
|
188
253
|
return status === "completed" || status === "error" || status === "aborted" || status === "timeout";
|
|
189
254
|
}
|
|
255
|
+
function isAssistantMessage(message) {
|
|
256
|
+
return Boolean(message && typeof message === "object" && message.role === "assistant");
|
|
257
|
+
}
|
|
258
|
+
function providerStatusForMessage(message) {
|
|
259
|
+
if (message.stopReason === "error")
|
|
260
|
+
return "error";
|
|
261
|
+
if (message.stopReason === "aborted")
|
|
262
|
+
return "aborted";
|
|
263
|
+
return "completed";
|
|
264
|
+
}
|
|
265
|
+
function providerSummaryForStatus(status) {
|
|
266
|
+
if (status === "error")
|
|
267
|
+
return "provider stream failed";
|
|
268
|
+
if (status === "aborted")
|
|
269
|
+
return "provider stream aborted";
|
|
270
|
+
return "provider stream completed";
|
|
271
|
+
}
|
|
272
|
+
function providerStreamPhaseId(request) {
|
|
273
|
+
return `${request.turnId}:provider_stream:${request.providerRequestId}`;
|
|
274
|
+
}
|
|
275
|
+
function safeMessageError(message) {
|
|
276
|
+
return typeof message.errorMessage === "string" && message.errorMessage.trim().length > 0
|
|
277
|
+
? message.errorMessage.replace(/\s+/g, " ").trim().slice(0, 512)
|
|
278
|
+
: undefined;
|
|
279
|
+
}
|
|
280
|
+
function optionalString(value) {
|
|
281
|
+
return typeof value === "string" && value.length > 0 ? value : undefined;
|
|
282
|
+
}
|
|
190
283
|
function modelInfo(primary, fallback, payload) {
|
|
191
284
|
const payloadModel = modelIdFromPayload(payload);
|
|
192
285
|
return {
|