@pasko70/pibo 1.8.0 → 1.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apps/chat/workflow-manual-trigger-runtime.js +13 -4
- package/dist/core/compaction-prompt.js +87 -52
- package/dist/core/provider-recovery.js +64 -0
- package/dist/core/provider-telemetry.js +40 -2
- package/dist/core/routed-session.js +97 -19
- package/dist/core/runtime-telemetry.js +84 -46
- package/dist/core/runtime.js +5 -3
- package/dist/core/session-errors.js +14 -1
- package/dist/core/session-router.js +74 -22
- package/dist/data/telemetry-writer.js +114 -0
- package/dist/data/telemetry.js +14 -0
- package/dist/gateway/request.js +17 -8
- package/dist/ralph/service.js +3 -5
- package/package.json +1 -1
|
@@ -8,6 +8,7 @@ export class PiboRuntimeTelemetryRecorder {
|
|
|
8
8
|
telemetry;
|
|
9
9
|
providerEventMode;
|
|
10
10
|
progressFlushIntervalMs;
|
|
11
|
+
writer;
|
|
11
12
|
pendingProviderProgress = new Map();
|
|
12
13
|
providerRequestCache = new Map();
|
|
13
14
|
lastProviderFlushAtMs = new Map();
|
|
@@ -17,6 +18,7 @@ export class PiboRuntimeTelemetryRecorder {
|
|
|
17
18
|
this.onError = onError;
|
|
18
19
|
this.telemetry = new BestEffortTelemetryService(store, onError);
|
|
19
20
|
this.providerEventMode = options.providerEventMode ?? "aggregate";
|
|
21
|
+
this.writer = options.writer;
|
|
20
22
|
const progressFlushIntervalMs = options.progressFlushIntervalMs;
|
|
21
23
|
this.progressFlushIntervalMs = typeof progressFlushIntervalMs === "number" && Number.isFinite(progressFlushIntervalMs) && progressFlushIntervalMs >= 0
|
|
22
24
|
? progressFlushIntervalMs
|
|
@@ -25,31 +27,43 @@ export class PiboRuntimeTelemetryRecorder {
|
|
|
25
27
|
recordOutput(event, context = {}) {
|
|
26
28
|
if (!this.store)
|
|
27
29
|
return;
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
catch (error) {
|
|
32
|
-
this.onError?.(error);
|
|
33
|
-
}
|
|
30
|
+
const captured = captureTelemetryContext(context);
|
|
31
|
+
const capturedEvent = telemetryOutputEventSnapshot(event);
|
|
32
|
+
this.schedule(() => this.recordOutputUnsafe(capturedEvent, captured));
|
|
34
33
|
}
|
|
35
34
|
recordPiEvent(piboSessionId, event, context = {}) {
|
|
36
35
|
if (!this.store)
|
|
37
36
|
return;
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
}
|
|
37
|
+
const summary = providerEventSummaryForPiEvent(event);
|
|
38
|
+
if (!summary)
|
|
39
|
+
return;
|
|
40
|
+
const captured = captureTelemetryContext(context);
|
|
41
|
+
this.schedule(() => this.recordPiEventSummaryUnsafe(piboSessionId, summary, captured));
|
|
44
42
|
}
|
|
45
43
|
recordMessagesInterrupted(messages, context = {}, reason = "message interrupted") {
|
|
46
44
|
if (!this.store)
|
|
47
45
|
return;
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
46
|
+
const captured = captureTelemetryContext(context);
|
|
47
|
+
const interrupted = messages.flatMap((message) => message.id ? [{ piboSessionId: message.piboSessionId, eventId: message.id }] : []);
|
|
48
|
+
this.schedule(() => {
|
|
49
|
+
for (const message of interrupted) {
|
|
50
|
+
this.recordTurnTerminal(message, captured, "aborted", "abort", reason, "runtime_abort");
|
|
51
|
+
}
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
schedule(write) {
|
|
55
|
+
const guarded = () => {
|
|
56
|
+
try {
|
|
57
|
+
write();
|
|
58
|
+
}
|
|
59
|
+
catch (error) {
|
|
60
|
+
this.onError?.(error);
|
|
61
|
+
}
|
|
62
|
+
};
|
|
63
|
+
if (this.writer)
|
|
64
|
+
this.writer.enqueue(guarded, this.onError);
|
|
65
|
+
else
|
|
66
|
+
guarded();
|
|
53
67
|
}
|
|
54
68
|
recordOutputUnsafe(event, context) {
|
|
55
69
|
switch (event.type) {
|
|
@@ -77,7 +91,7 @@ export class PiboRuntimeTelemetryRecorder {
|
|
|
77
91
|
case "thinking_finished": {
|
|
78
92
|
this.recordProviderStreamProgress(event, context, "reasoning", "reasoning finished", true);
|
|
79
93
|
const turn = this.turnContextForEvent(event.piboSessionId, event.eventId, undefined, context) ?? this.activeTurnContext(event.piboSessionId, context);
|
|
80
|
-
this.finishOpenPhasesByName(turn?.turnId, "reasoning", "ok");
|
|
94
|
+
this.finishOpenPhasesByName(turn?.turnId, "reasoning", "ok", telemetryTimestamp(context));
|
|
81
95
|
return;
|
|
82
96
|
}
|
|
83
97
|
case "tool_call":
|
|
@@ -104,23 +118,21 @@ export class PiboRuntimeTelemetryRecorder {
|
|
|
104
118
|
return;
|
|
105
119
|
}
|
|
106
120
|
}
|
|
107
|
-
|
|
108
|
-
const summary = providerEventSummaryForPiEvent(event);
|
|
109
|
-
if (!summary)
|
|
110
|
-
return;
|
|
121
|
+
recordPiEventSummaryUnsafe(piboSessionId, summary, context) {
|
|
111
122
|
const turn = context.activeEventId
|
|
112
123
|
? this.progressTurnContextForEvent(piboSessionId, context.activeEventId, context)
|
|
113
124
|
: this.activeTurnContext(piboSessionId, context);
|
|
114
125
|
if (!turn)
|
|
115
126
|
return;
|
|
116
|
-
const now =
|
|
127
|
+
const now = telemetryTimestamp(context);
|
|
128
|
+
const nowMs = telemetryTimestampMs(context);
|
|
117
129
|
if (summary.assistantEventType === "start") {
|
|
118
|
-
this.flushProviderProgress(turn.turnId, now, { force: true, includeLatest: true });
|
|
130
|
+
this.flushProviderProgress(turn.turnId, now, { force: true, includeLatest: true, nowMs });
|
|
119
131
|
this.clearProviderProgress(turn.turnId);
|
|
120
132
|
}
|
|
121
133
|
const providerRequest = this.providerEventMode === "detailed"
|
|
122
134
|
? this.providerRequestForTurn(turn.turnId, { includeLatest: summary.messageEnded, refresh: summary.messageEnded })
|
|
123
|
-
: this.accumulateProviderEvent(turn, summary, now);
|
|
135
|
+
: this.accumulateProviderEvent(turn, summary, now, nowMs);
|
|
124
136
|
if (!providerRequest) {
|
|
125
137
|
if (summary.messageEnded)
|
|
126
138
|
this.clearProviderProgress(turn.turnId);
|
|
@@ -146,11 +158,11 @@ export class PiboRuntimeTelemetryRecorder {
|
|
|
146
158
|
}
|
|
147
159
|
if (summary.toolCallId && summary.assistantEventType?.startsWith("toolcall_")) {
|
|
148
160
|
const forceToolProgress = summary.assistantEventType !== "toolcall_delta";
|
|
149
|
-
if (this.shouldPersistProgress(`${turn.turnId}:tool_args:${summary.toolCallId}`, forceToolProgress)) {
|
|
161
|
+
if (this.shouldPersistProgress(`${turn.turnId}:tool_args:${summary.toolCallId}`, forceToolProgress, nowMs)) {
|
|
150
162
|
this.recordPiToolCallProgress(turn, providerRequest.providerRequestId, summary, now);
|
|
151
163
|
}
|
|
152
164
|
}
|
|
153
|
-
if (!summary.messageEnded && !isTerminalProviderStatus(providerRequest.status) && !summary.normalizedType && this.shouldPersistProgress(`${turn.turnId}:provider_stream:${providerRequest.providerRequestId}
|
|
165
|
+
if (!summary.messageEnded && !isTerminalProviderStatus(providerRequest.status) && !summary.normalizedType && this.shouldPersistProgress(`${turn.turnId}:provider_stream:${providerRequest.providerRequestId}`, false, nowMs)) {
|
|
154
166
|
this.startOrProgressPhase(turn, "provider_stream", now, "provider event metadata", { providerRequestId: providerRequest.providerRequestId });
|
|
155
167
|
}
|
|
156
168
|
if (summary.messageEnded)
|
|
@@ -160,7 +172,7 @@ export class PiboRuntimeTelemetryRecorder {
|
|
|
160
172
|
const turn = this.turnContextForEvent(event.piboSessionId, event.eventId, event.source, context);
|
|
161
173
|
if (!turn)
|
|
162
174
|
return;
|
|
163
|
-
const now =
|
|
175
|
+
const now = telemetryTimestamp(context);
|
|
164
176
|
const queueDepth = event.queuedMessages;
|
|
165
177
|
this.telemetry.upsertTurn({
|
|
166
178
|
turnId: turn.turnId,
|
|
@@ -197,7 +209,7 @@ export class PiboRuntimeTelemetryRecorder {
|
|
|
197
209
|
const turn = this.turnContextForEvent(event.piboSessionId, event.eventId, event.source, context);
|
|
198
210
|
if (!turn)
|
|
199
211
|
return;
|
|
200
|
-
const now =
|
|
212
|
+
const now = telemetryTimestamp(context);
|
|
201
213
|
this.telemetry.finishPhase(phaseId(turn.turnId, "queued"), { status: "ok", endedAt: now, lastProgressAt: now });
|
|
202
214
|
this.telemetry.upsertPhase({
|
|
203
215
|
phaseId: phaseId(turn.turnId, "message_started"),
|
|
@@ -244,10 +256,10 @@ export class PiboRuntimeTelemetryRecorder {
|
|
|
244
256
|
: this.activeTurnContext(event.piboSessionId, context);
|
|
245
257
|
if (!turn)
|
|
246
258
|
return;
|
|
247
|
-
const now =
|
|
248
|
-
const providerRequest = this.accumulateNormalizedProviderProgress(turn, now, force);
|
|
259
|
+
const now = telemetryTimestamp(context);
|
|
260
|
+
const providerRequest = this.accumulateNormalizedProviderProgress(turn, now, force, telemetryTimestampMs(context));
|
|
249
261
|
const progressKey = `${turn.turnId}:${phaseName}:${providerRequest?.providerRequestId ?? "none"}`;
|
|
250
|
-
if (!this.shouldPersistProgress(progressKey, force))
|
|
262
|
+
if (!this.shouldPersistProgress(progressKey, force, telemetryTimestampMs(context)))
|
|
251
263
|
return;
|
|
252
264
|
const storedTurn = this.store?.getTurn(turn.turnId);
|
|
253
265
|
if (storedTurn && TERMINAL_TURN_STATUSES.has(storedTurn.status)) {
|
|
@@ -256,7 +268,7 @@ export class PiboRuntimeTelemetryRecorder {
|
|
|
256
268
|
}
|
|
257
269
|
this.closeOpenPhasesByName(turn.turnId, "message_started", "ok", now);
|
|
258
270
|
const providerStreamKey = `${turn.turnId}:provider_stream:${providerRequest?.providerRequestId ?? "none"}`;
|
|
259
|
-
if ((!providerRequest || !isTerminalProviderStatus(providerRequest.status)) && this.shouldPersistProgress(providerStreamKey, force)) {
|
|
271
|
+
if ((!providerRequest || !isTerminalProviderStatus(providerRequest.status)) && this.shouldPersistProgress(providerStreamKey, force, telemetryTimestampMs(context))) {
|
|
260
272
|
this.startOrProgressPhase(turn, "provider_stream", now, "normalized provider stream progress", { providerRequestId: providerRequest?.providerRequestId });
|
|
261
273
|
}
|
|
262
274
|
this.startOrProgressPhase(turn, phaseName, now, summary, { updateTurn: true, providerRequestId: providerRequest?.providerRequestId });
|
|
@@ -265,11 +277,11 @@ export class PiboRuntimeTelemetryRecorder {
|
|
|
265
277
|
const turn = this.turnContextForEvent(event.piboSessionId, event.eventId, undefined, context) ?? this.activeTurnContext(event.piboSessionId, context);
|
|
266
278
|
if (!turn)
|
|
267
279
|
return;
|
|
268
|
-
const now =
|
|
280
|
+
const now = telemetryTimestamp(context);
|
|
269
281
|
this.closeOpenPhasesByName(turn.turnId, "message_started", "ok", now);
|
|
270
282
|
this.closeOpenPhasesByName(turn.turnId, "assistant_text", "ok", now);
|
|
271
283
|
this.closeOpenPhasesByName(turn.turnId, "reasoning", "ok", now);
|
|
272
|
-
const providerRequest = this.accumulateNormalizedProviderProgress(turn, now, true)
|
|
284
|
+
const providerRequest = this.accumulateNormalizedProviderProgress(turn, now, true, telemetryTimestampMs(context))
|
|
273
285
|
?? this.providerRequestForTurn(turn.turnId, { includeLatest: true });
|
|
274
286
|
this.upsertToolCallArgs(turn, {
|
|
275
287
|
toolCallId: event.toolCallId,
|
|
@@ -295,14 +307,14 @@ export class PiboRuntimeTelemetryRecorder {
|
|
|
295
307
|
: this.activeTurnContext(event.piboSessionId, context);
|
|
296
308
|
if (!turn)
|
|
297
309
|
return;
|
|
298
|
-
if (!this.shouldPersistProgress(`${turn.turnId}:tool_execution:${event.toolCallId}`, force))
|
|
310
|
+
if (!this.shouldPersistProgress(`${turn.turnId}:tool_execution:${event.toolCallId}`, force, telemetryTimestampMs(context)))
|
|
299
311
|
return;
|
|
300
312
|
const storedTurn = this.store?.getTurn(turn.turnId);
|
|
301
313
|
if (storedTurn && TERMINAL_TURN_STATUSES.has(storedTurn.status)) {
|
|
302
314
|
this.clearTurnProgress(turn.turnId);
|
|
303
315
|
return;
|
|
304
316
|
}
|
|
305
|
-
const now =
|
|
317
|
+
const now = telemetryTimestamp(context);
|
|
306
318
|
const existing = this.store?.getToolCall(event.toolCallId);
|
|
307
319
|
const args = toolArgsMetadata(event.args, true);
|
|
308
320
|
const providerRequestId = existing?.providerRequestId ?? this.latestProviderRequestForTurn(turn.turnId)?.providerRequestId;
|
|
@@ -336,7 +348,7 @@ export class PiboRuntimeTelemetryRecorder {
|
|
|
336
348
|
const turn = this.turnContextForEvent(event.piboSessionId, event.eventId, undefined, context) ?? this.activeTurnContext(event.piboSessionId, context);
|
|
337
349
|
if (!turn)
|
|
338
350
|
return;
|
|
339
|
-
const now =
|
|
351
|
+
const now = telemetryTimestamp(context);
|
|
340
352
|
const existing = this.store?.getToolCall(event.toolCallId);
|
|
341
353
|
const executionStartedAt = existing?.executionStartedAt;
|
|
342
354
|
this.telemetry.upsertToolCall({
|
|
@@ -425,8 +437,8 @@ export class PiboRuntimeTelemetryRecorder {
|
|
|
425
437
|
this.clearTurnProgress(turn.turnId);
|
|
426
438
|
return;
|
|
427
439
|
}
|
|
428
|
-
const now =
|
|
429
|
-
this.flushProviderProgress(turn.turnId, now, { force: true, includeLatest: true });
|
|
440
|
+
const now = telemetryTimestamp(context);
|
|
441
|
+
this.flushProviderProgress(turn.turnId, now, { force: true, includeLatest: true, nowMs: telemetryTimestampMs(context) });
|
|
430
442
|
this.finishOpenPhases(turn.turnId, terminalPhaseStatus(status), now);
|
|
431
443
|
this.finishActiveProviderRequests(turn.turnId, providerStatusForTurnStatus(status), now, summary, errorCategory);
|
|
432
444
|
this.finishActiveToolCalls(turn.turnId, status, now, summary);
|
|
@@ -540,7 +552,7 @@ export class PiboRuntimeTelemetryRecorder {
|
|
|
540
552
|
this.providerRequestCache.delete(turnId);
|
|
541
553
|
return request;
|
|
542
554
|
}
|
|
543
|
-
accumulateProviderEvent(turn, summary, now) {
|
|
555
|
+
accumulateProviderEvent(turn, summary, now, nowMs) {
|
|
544
556
|
const pending = this.pendingProviderProgress.get(turn.turnId) ?? emptyPendingProviderProgress();
|
|
545
557
|
pending.lastRawEventAt = now;
|
|
546
558
|
pending.upstreamResponseId = summary.upstreamResponseId ?? pending.upstreamResponseId;
|
|
@@ -554,9 +566,10 @@ export class PiboRuntimeTelemetryRecorder {
|
|
|
554
566
|
force: summary.messageEnded,
|
|
555
567
|
includeLatest: summary.messageEnded,
|
|
556
568
|
refresh: summary.messageEnded,
|
|
569
|
+
nowMs,
|
|
557
570
|
});
|
|
558
571
|
}
|
|
559
|
-
accumulateNormalizedProviderProgress(turn, now, force = false) {
|
|
572
|
+
accumulateNormalizedProviderProgress(turn, now, force = false, nowMs = Date.now()) {
|
|
560
573
|
const request = this.providerRequestForTurn(turn.turnId, { includeLatest: force });
|
|
561
574
|
if (!request)
|
|
562
575
|
return undefined;
|
|
@@ -565,14 +578,14 @@ export class PiboRuntimeTelemetryRecorder {
|
|
|
565
578
|
pending.normalizedEventCount += 1;
|
|
566
579
|
this.pendingProviderProgress.set(turn.turnId, pending);
|
|
567
580
|
const flushNow = force || request.status === "started" || request.status === "headers";
|
|
568
|
-
return this.flushProviderProgress(turn.turnId, now, { force: flushNow, includeLatest: force }) ?? request;
|
|
581
|
+
return this.flushProviderProgress(turn.turnId, now, { force: flushNow, includeLatest: force, nowMs }) ?? request;
|
|
569
582
|
}
|
|
570
583
|
flushProviderProgress(turnId, now, options = {}) {
|
|
571
584
|
const pending = this.pendingProviderProgress.get(turnId);
|
|
572
585
|
const request = this.providerRequestForTurn(turnId, options);
|
|
573
586
|
if (!pending)
|
|
574
587
|
return request;
|
|
575
|
-
const nowMs = Date.now();
|
|
588
|
+
const nowMs = options.nowMs ?? Date.now();
|
|
576
589
|
const lastFlushAtMs = this.lastProviderFlushAtMs.get(turnId);
|
|
577
590
|
if (!options.force && lastFlushAtMs !== undefined && nowMs - lastFlushAtMs < this.progressFlushIntervalMs)
|
|
578
591
|
return request;
|
|
@@ -602,8 +615,7 @@ export class PiboRuntimeTelemetryRecorder {
|
|
|
602
615
|
this.providerRequestCache.set(turnId, updated);
|
|
603
616
|
return updated ?? request;
|
|
604
617
|
}
|
|
605
|
-
shouldPersistProgress(key, force = false) {
|
|
606
|
-
const nowMs = Date.now();
|
|
618
|
+
shouldPersistProgress(key, force = false, nowMs = Date.now()) {
|
|
607
619
|
const lastWriteAtMs = this.lastProgressWriteAtMs.get(key);
|
|
608
620
|
if (!force && lastWriteAtMs !== undefined && nowMs - lastWriteAtMs < this.progressFlushIntervalMs)
|
|
609
621
|
return false;
|
|
@@ -960,6 +972,32 @@ function safeJsonByteSize(value) {
|
|
|
960
972
|
function utf8Bytes(value) {
|
|
961
973
|
return Buffer.byteLength(value, "utf8");
|
|
962
974
|
}
|
|
975
|
+
function telemetryOutputEventSnapshot(event) {
|
|
976
|
+
if (event.type === "tool_execution_updated")
|
|
977
|
+
return { ...event, partialResult: undefined };
|
|
978
|
+
if (event.type === "tool_execution_finished")
|
|
979
|
+
return { ...event, result: event.isError ? safeErrorMessage(event.result) : undefined };
|
|
980
|
+
if (event.type === "execution_result")
|
|
981
|
+
return { ...event, result: undefined };
|
|
982
|
+
return { ...event };
|
|
983
|
+
}
|
|
984
|
+
function captureTelemetryContext(context) {
|
|
985
|
+
const parsedAtMs = context.at ? Date.parse(context.at) : Number.NaN;
|
|
986
|
+
const atMs = context.atMs ?? (Number.isFinite(parsedAtMs) ? parsedAtMs : Date.now());
|
|
987
|
+
return {
|
|
988
|
+
...context,
|
|
989
|
+
session: context.session ? { ...context.session, metadata: context.session.metadata ? { ...context.session.metadata } : undefined } : undefined,
|
|
990
|
+
status: context.status ? { ...context.status, activeTools: [...context.status.activeTools], enabledTools: [...context.status.enabledTools] } : undefined,
|
|
991
|
+
at: context.at ?? new Date(atMs).toISOString(),
|
|
992
|
+
atMs,
|
|
993
|
+
};
|
|
994
|
+
}
|
|
995
|
+
function telemetryTimestamp(context) {
|
|
996
|
+
return context.at ?? new Date().toISOString();
|
|
997
|
+
}
|
|
998
|
+
function telemetryTimestampMs(context) {
|
|
999
|
+
return context.atMs ?? Date.now();
|
|
1000
|
+
}
|
|
963
1001
|
export function turnIdForEvent(eventId) {
|
|
964
1002
|
return `turn_${eventId}`;
|
|
965
1003
|
}
|
package/dist/core/runtime.js
CHANGED
|
@@ -166,9 +166,9 @@ function getBuiltinToolAllowlist(profile, customTools) {
|
|
|
166
166
|
return undefined;
|
|
167
167
|
return [...selectedBuiltinTools, ...customTools.map((tool) => tool.name)];
|
|
168
168
|
}
|
|
169
|
-
function getProfileExtensionFactories(profile, extensionFactories, contextGuardRecovery) {
|
|
169
|
+
function getProfileExtensionFactories(profile, extensionFactories, contextGuardRecovery, getSettingsManager) {
|
|
170
170
|
const piboPromptTemplateExtension = createPiboSystemPromptTemplateExtension();
|
|
171
|
-
const piboCompactionPromptExtension = createPiboCompactionPromptExtension();
|
|
171
|
+
const piboCompactionPromptExtension = createPiboCompactionPromptExtension({ getSettingsManager });
|
|
172
172
|
const piboContextGuardExtension = createPiboAssistantContextGuardExtension({}, contextGuardRecovery);
|
|
173
173
|
const providerToolExtensions = profile.tools
|
|
174
174
|
.filter((tool) => tool.enabled !== false)
|
|
@@ -241,6 +241,7 @@ export async function createPiboRuntime(options = {}) {
|
|
|
241
241
|
const mcpAgentContextFile = await getMcpAgentContextFile(profile.mcpServers);
|
|
242
242
|
const skillPaths = getEnabledSkillPaths(runtimeCwd, profile);
|
|
243
243
|
const piPackageOptions = getPiPackageRuntimeOptions(runtimeCwd, profile);
|
|
244
|
+
let runtimeSettingsManager;
|
|
244
245
|
const services = await createAgentSessionServices({
|
|
245
246
|
cwd: runtimeCwd,
|
|
246
247
|
agentDir: runtimeAgentDir,
|
|
@@ -248,7 +249,7 @@ export async function createPiboRuntime(options = {}) {
|
|
|
248
249
|
resourceLoaderOptions: {
|
|
249
250
|
...piPackageOptions.resourceLoaderOptions,
|
|
250
251
|
additionalSkillPaths: skillPaths,
|
|
251
|
-
extensionFactories: getProfileExtensionFactories(profile, options.extensionFactories, contextGuardRecovery),
|
|
252
|
+
extensionFactories: getProfileExtensionFactories(profile, options.extensionFactories, contextGuardRecovery, () => runtimeSettingsManager),
|
|
252
253
|
noExtensions: true,
|
|
253
254
|
noSkills: true,
|
|
254
255
|
noPromptTemplates: true,
|
|
@@ -265,6 +266,7 @@ export async function createPiboRuntime(options = {}) {
|
|
|
265
266
|
}),
|
|
266
267
|
},
|
|
267
268
|
});
|
|
269
|
+
runtimeSettingsManager = services.settingsManager;
|
|
268
270
|
applyPiboRuntimeRetryDefaults(services.settingsManager, options.retryDefaults);
|
|
269
271
|
registerOpenAiGpt56Models(services.modelRegistry);
|
|
270
272
|
registerMiniMaxProvider(services.modelRegistry);
|
|
@@ -10,6 +10,11 @@ const PROVIDER_NETWORK_ERROR_MARKERS = [
|
|
|
10
10
|
"reset before headers",
|
|
11
11
|
"socket hang up",
|
|
12
12
|
"socket connection was closed",
|
|
13
|
+
"eai_again",
|
|
14
|
+
"enotfound",
|
|
15
|
+
"econnreset",
|
|
16
|
+
"econnrefused",
|
|
17
|
+
"etimedout",
|
|
13
18
|
];
|
|
14
19
|
export function classifySessionErrorMessage(message, options = {}) {
|
|
15
20
|
const normalized = message.toLowerCase();
|
|
@@ -20,7 +25,15 @@ export function classifySessionErrorMessage(message, options = {}) {
|
|
|
20
25
|
return { category: "provider_transport", errorClass: "provider_transport", code: "websocket_error", origin: "provider", retryable: true, userMessage: "The provider WebSocket connection failed." };
|
|
21
26
|
}
|
|
22
27
|
if (normalized.includes("request was aborted") || normalized.includes("aborted")) {
|
|
23
|
-
return { category: "runtime_abort", errorClass: "runtime_abort", code: "request_aborted", origin: "runtime", retryable:
|
|
28
|
+
return { category: "runtime_abort", errorClass: "runtime_abort", code: "request_aborted", origin: "runtime", retryable: false, userMessage: "The active model request was aborted." };
|
|
29
|
+
}
|
|
30
|
+
if (normalized.includes("insufficient_quota")
|
|
31
|
+
|| normalized.includes("quota exceeded")
|
|
32
|
+
|| normalized.includes("out of budget")
|
|
33
|
+
|| normalized.includes("billing")
|
|
34
|
+
|| normalized.includes("usage limit")
|
|
35
|
+
|| normalized.includes("available balance")) {
|
|
36
|
+
return { category: "quota_exhausted", errorClass: "provider_rate_limit", code: "quota_exhausted", origin: "provider", retryable: false, userMessage: "The provider quota or billing limit was reached." };
|
|
24
37
|
}
|
|
25
38
|
if (normalized.includes("rate limit") || normalized.includes("429")) {
|
|
26
39
|
return { category: "rate_limit", errorClass: "provider_rate_limit", code: "rate_limited", origin: "provider", retryable: true, userMessage: "The provider rate limit was reached." };
|
|
@@ -19,6 +19,7 @@ import { assertGatewayResourceAvailableForWork } from "./gateway-resource-guard.
|
|
|
19
19
|
import { withWorkflowSessionKind } from "../sessions/workflow-session-kind.js";
|
|
20
20
|
import { PiboRuntimeTelemetryRecorder } from "./runtime-telemetry.js";
|
|
21
21
|
import { createPiboProviderTelemetryExtension } from "./provider-telemetry.js";
|
|
22
|
+
import { AsyncTelemetryWriter } from "../data/telemetry-writer.js";
|
|
22
23
|
const DEFAULT_SUBAGENT_REPLY_TIMEOUT_MS = 10 * 60 * 1000;
|
|
23
24
|
const DEFAULT_ROUTED_SESSION_IDLE_TIMEOUT_MS = 30 * 60 * 1000;
|
|
24
25
|
export const RALPH_RUNTIME_RETRY_DEFAULTS = {
|
|
@@ -140,14 +141,21 @@ export class PiboSessionRouter {
|
|
|
140
141
|
sessionStore;
|
|
141
142
|
reliabilityStore;
|
|
142
143
|
telemetryStore;
|
|
144
|
+
telemetryWriter;
|
|
143
145
|
telemetryRecorder;
|
|
146
|
+
disposePromise;
|
|
147
|
+
closing = false;
|
|
144
148
|
constructor(options = {}) {
|
|
145
149
|
this.options = options;
|
|
146
150
|
this.pluginRegistry = options.pluginRegistry ?? createDefaultPiboPluginRegistry();
|
|
147
151
|
this.sessionStore = options.sessionStore ?? new InMemoryPiboSessionStore();
|
|
148
152
|
this.telemetryStore = options.telemetryStore ?? telemetryStoreFromSessionStore(this.sessionStore);
|
|
153
|
+
this.telemetryWriter = this.telemetryStore ? new AsyncTelemetryWriter(this.telemetryStore) : undefined;
|
|
149
154
|
this.telemetryRecorder = this.telemetryStore
|
|
150
|
-
? new PiboRuntimeTelemetryRecorder(this.telemetryStore, undefined, {
|
|
155
|
+
? new PiboRuntimeTelemetryRecorder(this.telemetryStore, undefined, {
|
|
156
|
+
providerEventMode: providerEventTelemetryModeFromEnv(),
|
|
157
|
+
writer: this.telemetryWriter,
|
|
158
|
+
})
|
|
151
159
|
: undefined;
|
|
152
160
|
const idleTimeoutMs = options.routedSessionIdleTimeoutMs;
|
|
153
161
|
this.routedSessionIdleTimeoutMs = idleTimeoutMs === false
|
|
@@ -173,6 +181,8 @@ export class PiboSessionRouter {
|
|
|
173
181
|
};
|
|
174
182
|
}
|
|
175
183
|
async emit(event) {
|
|
184
|
+
if (this.closing)
|
|
185
|
+
throw new Error("Pibo session router is disposed.");
|
|
176
186
|
const session = await this.getOrCreateSession(event.piboSessionId);
|
|
177
187
|
this.clearIdleSessionTimer(event.piboSessionId);
|
|
178
188
|
try {
|
|
@@ -229,8 +239,13 @@ export class PiboSessionRouter {
|
|
|
229
239
|
sessions.push(cached);
|
|
230
240
|
this.sessions.delete(id);
|
|
231
241
|
}
|
|
232
|
-
|
|
233
|
-
|
|
242
|
+
try {
|
|
243
|
+
await Promise.all(ids.map((id) => this.runtimeRegistry.closeControllerSessions(id, { force: true })));
|
|
244
|
+
await Promise.all(sessions.map((session) => session.dispose()));
|
|
245
|
+
}
|
|
246
|
+
finally {
|
|
247
|
+
await this.telemetryWriter?.flush();
|
|
248
|
+
}
|
|
234
249
|
for (const id of ids) {
|
|
235
250
|
this.signalRegistry.project({ type: "session_disposed", piboSessionId: id, reason });
|
|
236
251
|
}
|
|
@@ -319,6 +334,8 @@ export class PiboSessionRouter {
|
|
|
319
334
|
const eventWithId = { ...event, id: event.id ?? randomUUID() };
|
|
320
335
|
return await new Promise((resolve, reject) => {
|
|
321
336
|
let settled = false;
|
|
337
|
+
let lastAssistantMessage;
|
|
338
|
+
let timeout;
|
|
322
339
|
const unsubscribe = this.subscribe((output) => {
|
|
323
340
|
if (output.piboSessionId !== eventWithId.piboSessionId ||
|
|
324
341
|
!("eventId" in output) ||
|
|
@@ -326,20 +343,21 @@ export class PiboSessionRouter {
|
|
|
326
343
|
return;
|
|
327
344
|
}
|
|
328
345
|
if (output.type === "assistant_message") {
|
|
329
|
-
|
|
346
|
+
lastAssistantMessage = output;
|
|
347
|
+
}
|
|
348
|
+
else if (output.type === "message_finished") {
|
|
349
|
+
finish(lastAssistantMessage ?? new Error(`Pibo session "${eventWithId.piboSessionId}" finished without an assistant reply`));
|
|
330
350
|
}
|
|
331
351
|
else if (output.type === "session_error") {
|
|
332
352
|
finish(new Error(output.error));
|
|
333
353
|
}
|
|
334
354
|
});
|
|
335
|
-
const timeout = setTimeout(() => {
|
|
336
|
-
finish(new Error(`Timed out waiting for assistant reply from Pibo session "${eventWithId.piboSessionId}"`));
|
|
337
|
-
}, timeoutMs);
|
|
338
355
|
const finish = (result) => {
|
|
339
356
|
if (settled)
|
|
340
357
|
return;
|
|
341
358
|
settled = true;
|
|
342
|
-
|
|
359
|
+
if (timeout)
|
|
360
|
+
clearTimeout(timeout);
|
|
343
361
|
unsubscribe();
|
|
344
362
|
if (result instanceof Error) {
|
|
345
363
|
reject(result);
|
|
@@ -348,21 +366,48 @@ export class PiboSessionRouter {
|
|
|
348
366
|
resolve(result);
|
|
349
367
|
}
|
|
350
368
|
};
|
|
369
|
+
timeout = setTimeout(() => {
|
|
370
|
+
if (settled)
|
|
371
|
+
return;
|
|
372
|
+
settled = true;
|
|
373
|
+
unsubscribe();
|
|
374
|
+
const timeoutError = new Error(`Timed out waiting for assistant reply from Pibo session "${eventWithId.piboSessionId}"`);
|
|
375
|
+
reject(timeoutError);
|
|
376
|
+
void this.emit({
|
|
377
|
+
type: "execution",
|
|
378
|
+
piboSessionId: eventWithId.piboSessionId,
|
|
379
|
+
action: "abort",
|
|
380
|
+
id: randomUUID(),
|
|
381
|
+
}).catch(() => { });
|
|
382
|
+
}, timeoutMs);
|
|
351
383
|
this.emit(eventWithId).catch(finish);
|
|
352
384
|
});
|
|
353
385
|
}
|
|
354
386
|
async disposeAll() {
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
this.
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
387
|
+
if (this.disposePromise)
|
|
388
|
+
return this.disposePromise;
|
|
389
|
+
this.closing = true;
|
|
390
|
+
this.disposePromise = this.disposeAllUnsafe();
|
|
391
|
+
return this.disposePromise;
|
|
392
|
+
}
|
|
393
|
+
async disposeAllUnsafe() {
|
|
394
|
+
try {
|
|
395
|
+
await Promise.allSettled([...this.pendingSessions.values()]);
|
|
396
|
+
const sessions = [...this.sessions.values()];
|
|
397
|
+
this.sessions.clear();
|
|
398
|
+
for (const timer of this.idleSessionTimers.values())
|
|
399
|
+
clearTimeout(timer);
|
|
400
|
+
this.idleSessionTimers.clear();
|
|
401
|
+
this.runRegistry.cancelAll("Pibo session router was disposed.");
|
|
402
|
+
for (const session of sessions)
|
|
403
|
+
this.signalRegistry.project({ type: "session_disposed", piboSessionId: session.getStatus().piboSessionId, reason: "router disposed" });
|
|
404
|
+
this.scheduledRunReminders.clear();
|
|
405
|
+
await this.runtimeRegistry.closeAll({ force: true });
|
|
406
|
+
await Promise.all(sessions.map((session) => session.dispose()));
|
|
407
|
+
}
|
|
408
|
+
finally {
|
|
409
|
+
await this.telemetryWriter?.dispose();
|
|
410
|
+
}
|
|
366
411
|
}
|
|
367
412
|
clearIdleSessionTimer(piboSessionId) {
|
|
368
413
|
const timer = this.idleSessionTimers.get(piboSessionId);
|
|
@@ -407,6 +452,8 @@ export class PiboSessionRouter {
|
|
|
407
452
|
await this.resetCachedSession(piboSessionId, "routed runtime idle timeout");
|
|
408
453
|
}
|
|
409
454
|
async getOrCreateSession(piboSessionId) {
|
|
455
|
+
if (this.closing)
|
|
456
|
+
throw new Error("Pibo session router is disposed.");
|
|
410
457
|
const existing = this.sessions.get(piboSessionId);
|
|
411
458
|
if (existing) {
|
|
412
459
|
this.clearIdleSessionTimer(piboSessionId);
|
|
@@ -436,7 +483,7 @@ export class PiboSessionRouter {
|
|
|
436
483
|
const initialThinkingLevel = resolvePiboSessionInitialThinkingLevel(piboSession);
|
|
437
484
|
const userSettings = loadPiboUserSettings();
|
|
438
485
|
const telemetryExtension = this.telemetryStore
|
|
439
|
-
? createPiboProviderTelemetryExtension({ store: this.telemetryStore, session: piboSession, model: activeModel })
|
|
486
|
+
? createPiboProviderTelemetryExtension({ store: this.telemetryStore, writer: this.telemetryWriter, session: piboSession, model: activeModel })
|
|
440
487
|
: undefined;
|
|
441
488
|
const runtime = await createPiboRuntime({
|
|
442
489
|
cwd: piboSession.workspace ?? this.options.cwd,
|
|
@@ -532,8 +579,13 @@ export class PiboSessionRouter {
|
|
|
532
579
|
const cached = this.sessions.get(piboSessionId);
|
|
533
580
|
this.clearIdleSessionTimer(piboSessionId);
|
|
534
581
|
this.sessions.delete(piboSessionId);
|
|
535
|
-
|
|
536
|
-
|
|
582
|
+
try {
|
|
583
|
+
await this.runtimeRegistry.closeControllerSessions(piboSessionId, { force: true });
|
|
584
|
+
await cached?.dispose();
|
|
585
|
+
}
|
|
586
|
+
finally {
|
|
587
|
+
await this.telemetryWriter?.flush();
|
|
588
|
+
}
|
|
537
589
|
if (reason)
|
|
538
590
|
this.signalRegistry.project({ type: "session_disposed", piboSessionId, reason });
|
|
539
591
|
}
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
const DEFAULT_FLUSH_INTERVAL_MS = 25;
|
|
2
|
+
const DEFAULT_MAX_PENDING_OPERATIONS = 1_024;
|
|
3
|
+
/**
|
|
4
|
+
* Gateway-scoped, ordered telemetry writer.
|
|
5
|
+
*
|
|
6
|
+
* Normal writes are deferred briefly so telemetry from multiple routed sessions
|
|
7
|
+
* shares one SQLite transaction. The queue never drops lifecycle events: when
|
|
8
|
+
* the hard bound is reached, it drains immediately in the caller instead.
|
|
9
|
+
*/
|
|
10
|
+
export class AsyncTelemetryWriter {
|
|
11
|
+
store;
|
|
12
|
+
options;
|
|
13
|
+
flushIntervalMs;
|
|
14
|
+
maxPendingOperations;
|
|
15
|
+
pending = [];
|
|
16
|
+
flushTimer;
|
|
17
|
+
flushing = false;
|
|
18
|
+
closed = false;
|
|
19
|
+
constructor(store, options = {}) {
|
|
20
|
+
this.store = store;
|
|
21
|
+
this.options = options;
|
|
22
|
+
this.flushIntervalMs = nonNegativeFinite(options.flushIntervalMs, DEFAULT_FLUSH_INTERVAL_MS);
|
|
23
|
+
this.maxPendingOperations = positiveInteger(options.maxPendingOperations, DEFAULT_MAX_PENDING_OPERATIONS);
|
|
24
|
+
}
|
|
25
|
+
enqueue(write, onError) {
|
|
26
|
+
if (this.closed) {
|
|
27
|
+
this.reportError(new Error("Telemetry writer is closed."), onError);
|
|
28
|
+
return false;
|
|
29
|
+
}
|
|
30
|
+
this.pending.push({ write, onError });
|
|
31
|
+
if (this.pending.length >= this.maxPendingOperations) {
|
|
32
|
+
this.flushNow();
|
|
33
|
+
}
|
|
34
|
+
else {
|
|
35
|
+
this.scheduleFlush();
|
|
36
|
+
}
|
|
37
|
+
return true;
|
|
38
|
+
}
|
|
39
|
+
async flush() {
|
|
40
|
+
this.flushNow();
|
|
41
|
+
}
|
|
42
|
+
async dispose() {
|
|
43
|
+
if (this.closed)
|
|
44
|
+
return;
|
|
45
|
+
this.flushNow();
|
|
46
|
+
this.closed = true;
|
|
47
|
+
}
|
|
48
|
+
scheduleFlush() {
|
|
49
|
+
if (this.flushTimer)
|
|
50
|
+
return;
|
|
51
|
+
this.flushTimer = setTimeout(() => {
|
|
52
|
+
this.flushTimer = undefined;
|
|
53
|
+
this.flushNow();
|
|
54
|
+
}, this.flushIntervalMs);
|
|
55
|
+
this.flushTimer.unref();
|
|
56
|
+
}
|
|
57
|
+
flushNow() {
|
|
58
|
+
if (this.flushing)
|
|
59
|
+
return;
|
|
60
|
+
if (this.flushTimer)
|
|
61
|
+
clearTimeout(this.flushTimer);
|
|
62
|
+
this.flushTimer = undefined;
|
|
63
|
+
this.flushing = true;
|
|
64
|
+
try {
|
|
65
|
+
while (this.pending.length > 0) {
|
|
66
|
+
const batch = this.pending;
|
|
67
|
+
this.pending = [];
|
|
68
|
+
try {
|
|
69
|
+
this.store.transaction(() => {
|
|
70
|
+
for (const operation of batch) {
|
|
71
|
+
try {
|
|
72
|
+
operation.write();
|
|
73
|
+
}
|
|
74
|
+
catch (error) {
|
|
75
|
+
this.reportError(error, operation.onError);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
catch (error) {
|
|
81
|
+
for (const operation of batch)
|
|
82
|
+
this.reportError(error, operation.onError);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
finally {
|
|
87
|
+
this.flushing = false;
|
|
88
|
+
if (!this.closed && this.pending.length > 0)
|
|
89
|
+
this.scheduleFlush();
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
reportError(error, operationHandler) {
|
|
93
|
+
try {
|
|
94
|
+
operationHandler?.(error);
|
|
95
|
+
}
|
|
96
|
+
catch {
|
|
97
|
+
// Telemetry error reporting must not affect runtime work.
|
|
98
|
+
}
|
|
99
|
+
if (operationHandler === this.options.onError)
|
|
100
|
+
return;
|
|
101
|
+
try {
|
|
102
|
+
this.options.onError?.(error);
|
|
103
|
+
}
|
|
104
|
+
catch {
|
|
105
|
+
// Telemetry error reporting must not affect runtime work.
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
function nonNegativeFinite(value, fallback) {
|
|
110
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : fallback;
|
|
111
|
+
}
|
|
112
|
+
function positiveInteger(value, fallback) {
|
|
113
|
+
return typeof value === "number" && Number.isInteger(value) && value > 0 ? value : fallback;
|
|
114
|
+
}
|