@pasko70/pibo 3.4.4 → 3.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-runtime/routed-session.js +111 -25
- package/dist/agent-runtimes/codex-native/turn.js +27 -3
- package/dist/agent-runtimes/pi/adapter.js +1 -0
- package/dist/agent-runtimes/pi/routed-session.js +2 -2
- package/dist/apps/chat/bounded-event-stream.js +98 -0
- package/dist/apps/chat/chat-settings-routes.js +3 -3
- package/dist/apps/chat/data/chat-data-mappers.js +40 -14
- package/dist/apps/chat/data/event-command-service.js +30 -21
- package/dist/apps/chat/data/history-query-service.js +34 -25
- package/dist/apps/chat/data/read-state-service.js +37 -3
- package/dist/apps/chat/data/session-query-service.js +19 -11
- package/dist/apps/chat/data/timeline-query-service.js +19 -7
- package/dist/apps/chat/message-command-dispatcher.js +149 -0
- package/dist/apps/chat/output-compactor.js +9 -0
- package/dist/apps/chat/output-event-policy.js +9 -1
- package/dist/apps/chat/stream.js +21 -3
- package/dist/apps/chat/telemetry-retention-service.js +113 -8
- package/dist/apps/chat/trace-response-cache.js +46 -0
- package/dist/apps/chat/trace-v2.js +12 -6
- package/dist/apps/chat/trace.js +1 -1
- package/dist/apps/chat/web-app.js +645 -297
- package/dist/apps/chat-ui/assets/{dist-V06sfuZa.js → dist-B-auLrzD.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-CH3SvpYV.js → dist-BA_dsINH.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-23lt_7qm.js → dist-eJZar_0-.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-Bf2KScPo.js → dist-wNNR2Bci.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-DsgL8w-W.js → dist-zcmEsIEp.js} +1 -1
- package/dist/apps/chat-ui/assets/index-DEkbN5Vo.js +229 -0
- package/dist/apps/chat-ui/assets/index-DZK6Tzil.css +1 -0
- package/dist/apps/chat-ui/index.html +2 -2
- package/dist/apps/chat-vscode-web/assets/index-DZgW1fCB.js +44 -0
- package/dist/apps/chat-vscode-web/index.html +1 -1
- package/dist/cli-session/localSessionSource.js +3 -2
- package/dist/compute/pool/seeds.js +25 -3
- package/dist/core/events.js +4 -0
- package/dist/core/output-render-sequence.js +65 -6
- package/dist/core/provider-capacity.js +33 -0
- package/dist/core/provider-telemetry.js +21 -6
- package/dist/core/runtime-capacity.js +174 -0
- package/dist/core/runtime-telemetry.js +40 -12
- package/dist/core/session-router.js +178 -13
- package/dist/data/async-chat-reads.js +38 -0
- package/dist/data/async-chat-storage.js +96 -0
- package/dist/data/async-telemetry-maintenance.js +9 -0
- package/dist/data/bounded-worker-client.js +256 -0
- package/dist/data/chat-read-projections.js +159 -0
- package/dist/data/chat-read-worker.js +73 -0
- package/dist/data/chat-storage-worker.js +167 -0
- package/dist/data/ingest-service.js +152 -18
- package/dist/data/message-command-store.js +290 -0
- package/dist/data/payload-store.js +92 -11
- package/dist/data/pibo-store.js +10 -8
- package/dist/data/schema.js +39 -4
- package/dist/data/session-store.js +3 -1
- package/dist/data/storage-backup.js +278 -0
- package/dist/data/storage-maintenance.js +344 -0
- package/dist/data/storage-verification-worker.js +25 -0
- package/dist/data/telemetry-capture.js +188 -0
- package/dist/data/telemetry-command.js +3 -0
- package/dist/data/telemetry-maintenance-worker.js +40 -0
- package/dist/data/telemetry-maintenance.js +110 -0
- package/dist/data/telemetry-retention.js +16 -7
- package/dist/data/telemetry-worker.js +111 -0
- package/dist/data/telemetry-writer.js +150 -83
- package/dist/data/telemetry.js +5 -0
- package/dist/debug/index.js +207 -1
- package/dist/debug/message-queue.js +108 -0
- package/dist/debug/output-collision-repair.js +140 -0
- package/dist/debug/output-integrity.js +38 -2
- package/dist/debug/output-repair.js +1 -0
- package/dist/debug/storage-backup.js +41 -0
- package/dist/debug/storage-maintenance.js +78 -0
- package/dist/debug/telemetry-capture.js +66 -0
- package/dist/gateway/cli.js +71 -7
- package/dist/gateway/server.js +3 -0
- package/dist/providers/openai-gpt56.js +11 -6
- package/dist/reliability/store.js +119 -23
- package/dist/session-ui/terminalRows.js +54 -13
- package/dist/sessions/pibo-data-store.js +19 -14
- package/dist/shared/debug-features.js +4 -0
- package/dist/shared/model-inference-metrics.js +23 -0
- package/dist/shared/trace-event-projection.js +69 -2
- package/dist/shared/trace-history.js +9 -0
- package/dist/shared/trace-live-reducer.js +1 -0
- package/dist/shared/trace-patch-nodes.js +19 -0
- package/dist/web/channel.js +117 -12
- package/dist/web/http.js +135 -41
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
- package/dist/apps/chat-ui/assets/index-BOceJ0jM.css +0 -1
- package/dist/apps/chat-ui/assets/index-BOemYq-V.js +0 -228
- package/dist/apps/chat-vscode-web/assets/index-CMwTB8o8.js +0 -43
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import { previouslyClearedMessages } from "./events.js";
|
|
2
|
+
import { createProviderCapacityExtension } from "./provider-capacity.js";
|
|
3
|
+
import { RuntimeCapacity } from "./runtime-capacity.js";
|
|
1
4
|
import { randomUUID } from "node:crypto";
|
|
2
5
|
import { isDeepStrictEqual } from "node:util";
|
|
3
6
|
import { InitialSessionContext, } from "./profiles.js";
|
|
@@ -172,6 +175,39 @@ function formatRunReminderMessage(notification) {
|
|
|
172
175
|
function isRunReminderServiceMessage(event) {
|
|
173
176
|
return event.source === "service" && event.text.startsWith("<pibo_run_notification>");
|
|
174
177
|
}
|
|
178
|
+
const RUNTIME_QUEUE_CAPACITY_DIMENSIONS = new Set([
|
|
179
|
+
"message_bytes",
|
|
180
|
+
"queue_count",
|
|
181
|
+
"queue_bytes",
|
|
182
|
+
"oldest_wait_age",
|
|
183
|
+
]);
|
|
184
|
+
function runReminderAdmissionDiagnostic(error) {
|
|
185
|
+
if (!error || typeof error !== "object" || !("code" in error) || error.code !== "runtime_capacity_unavailable") {
|
|
186
|
+
return {
|
|
187
|
+
code: "run_reminder_delivery_unavailable",
|
|
188
|
+
warning: "Run reminder deferred: internal delivery is temporarily unavailable; pending run state is retained.",
|
|
189
|
+
};
|
|
190
|
+
}
|
|
191
|
+
const candidate = error;
|
|
192
|
+
const dimension = typeof candidate.dimension === "string"
|
|
193
|
+
&& RUNTIME_QUEUE_CAPACITY_DIMENSIONS.has(candidate.dimension)
|
|
194
|
+
? candidate.dimension
|
|
195
|
+
: undefined;
|
|
196
|
+
const current = candidate.current
|
|
197
|
+
&& ["messageBytes", "queueCount", "queueBytes", "oldestWaitMs"].every((key) => (typeof candidate.current?.[key] === "number" && Number.isFinite(candidate.current[key]) && candidate.current[key] >= 0))
|
|
198
|
+
? candidate.current
|
|
199
|
+
: undefined;
|
|
200
|
+
const limit = typeof candidate.limit === "number" && Number.isFinite(candidate.limit) && candidate.limit >= 0
|
|
201
|
+
? candidate.limit
|
|
202
|
+
: undefined;
|
|
203
|
+
return {
|
|
204
|
+
code: "runtime_capacity_unavailable",
|
|
205
|
+
warning: `Run reminder deferred: runtime queue${dimension ? ` ${dimension}` : ""} capacity is unavailable; pending run state is retained.`,
|
|
206
|
+
...(dimension ? { dimension } : {}),
|
|
207
|
+
...(current ? { current } : {}),
|
|
208
|
+
...(limit !== undefined ? { limit } : {}),
|
|
209
|
+
};
|
|
210
|
+
}
|
|
175
211
|
function isRunReminderContextPressureError(event) {
|
|
176
212
|
return event.errorDetails?.category === "context_overflow"
|
|
177
213
|
|| event.errorDetails?.code === "context_length_exceeded"
|
|
@@ -333,6 +369,10 @@ function providerEventTelemetryModeFromEnv(env = process.env) {
|
|
|
333
369
|
export class PiboSessionRouter {
|
|
334
370
|
options;
|
|
335
371
|
sessions = new Map();
|
|
372
|
+
capacity;
|
|
373
|
+
creatingRuntimes = 0;
|
|
374
|
+
initializationTimings = [];
|
|
375
|
+
pendingStartAborts = new Map();
|
|
336
376
|
pendingSessions = new Map();
|
|
337
377
|
listeners = new Set();
|
|
338
378
|
outputRenderSequencer;
|
|
@@ -358,6 +398,7 @@ export class PiboSessionRouter {
|
|
|
358
398
|
deferredRunReminders = new Map();
|
|
359
399
|
runReminderRecoveries = new Map();
|
|
360
400
|
runReminderGenerations = new Map();
|
|
401
|
+
runReminderAdmissionWarnings = new Map();
|
|
361
402
|
runCancellationHandlers = new Map();
|
|
362
403
|
activeRunExecutions = new Set();
|
|
363
404
|
quiescingSessions = new Set();
|
|
@@ -378,6 +419,7 @@ export class PiboSessionRouter {
|
|
|
378
419
|
closing = false;
|
|
379
420
|
constructor(options = {}) {
|
|
380
421
|
this.options = options;
|
|
422
|
+
this.capacity = new RuntimeCapacity(options.runtimeCapacity);
|
|
381
423
|
this.pluginRegistry = options.pluginRegistry ?? createDefaultPiboPluginRegistry();
|
|
382
424
|
// Historical custom registries supplied only actions/profiles while runtime creation was implicit.
|
|
383
425
|
// Preserve that composition contract during the adapter migration without branching on adapter ids.
|
|
@@ -450,6 +492,16 @@ export class PiboSessionRouter {
|
|
|
450
492
|
async emit(event) {
|
|
451
493
|
if (this.closing)
|
|
452
494
|
throw new Error("Pibo session router is disposed.");
|
|
495
|
+
if (event.type === "execution" && !this.sessions.has(event.piboSessionId) && (event.action === "abort" || event.action === "clear_queue")) {
|
|
496
|
+
this.resolvePiboSession(event.piboSessionId);
|
|
497
|
+
if (event.action === "abort") {
|
|
498
|
+
this.invalidateRunReminders([event.piboSessionId]);
|
|
499
|
+
this.pendingStartAborts.get(event.piboSessionId)?.abort(Object.assign(new Error("Runtime start aborted before message dispatch."), { code: "runtime_start_cancelled" }));
|
|
500
|
+
}
|
|
501
|
+
const output = { type: "execution_result", piboSessionId: event.piboSessionId, eventId: event.id, action: event.action, result: event.action === "abort" ? { aborted: true } : { cleared: previouslyClearedMessages(event) } };
|
|
502
|
+
this.emitOutput(output);
|
|
503
|
+
return output;
|
|
504
|
+
}
|
|
453
505
|
let messageSignalAccepted = false;
|
|
454
506
|
const acceptMessageSignal = () => {
|
|
455
507
|
if (event.type !== "message" || !event.id || messageSignalAccepted)
|
|
@@ -1020,6 +1072,13 @@ export class PiboSessionRouter {
|
|
|
1020
1072
|
listRuns(options = {}) {
|
|
1021
1073
|
return this.runRegistry.listAll(options);
|
|
1022
1074
|
}
|
|
1075
|
+
getRunJobReliabilityStatus() {
|
|
1076
|
+
return this.reliabilityStore?.getRunJobReliabilityStatus() ?? {
|
|
1077
|
+
status: "ok",
|
|
1078
|
+
expiredOrphanRunJobs: 0,
|
|
1079
|
+
orphanRunDeadLetters: 0,
|
|
1080
|
+
};
|
|
1081
|
+
}
|
|
1023
1082
|
getSignalRegistry() {
|
|
1024
1083
|
return this.signalRegistry;
|
|
1025
1084
|
}
|
|
@@ -1126,6 +1185,7 @@ export class PiboSessionRouter {
|
|
|
1126
1185
|
if (this.disposePromise)
|
|
1127
1186
|
return this.disposePromise;
|
|
1128
1187
|
this.closing = true;
|
|
1188
|
+
this.capacity.close();
|
|
1129
1189
|
this.disposePromise = this.disposeAllUnsafe();
|
|
1130
1190
|
return this.disposePromise;
|
|
1131
1191
|
}
|
|
@@ -1245,16 +1305,65 @@ export class PiboSessionRouter {
|
|
|
1245
1305
|
const pending = this.pendingSessions.get(piboSessionId);
|
|
1246
1306
|
if (pending)
|
|
1247
1307
|
return pending;
|
|
1248
|
-
const
|
|
1308
|
+
const startAbort = new AbortController();
|
|
1309
|
+
this.pendingStartAborts.set(piboSessionId, startAbort);
|
|
1310
|
+
const created = this.createRoutedSessionWithinCapacity(piboSessionId, startAbort.signal);
|
|
1249
1311
|
this.pendingSessions.set(piboSessionId, created);
|
|
1250
1312
|
try {
|
|
1251
1313
|
return await created;
|
|
1252
1314
|
}
|
|
1253
1315
|
finally {
|
|
1254
1316
|
this.pendingSessions.delete(piboSessionId);
|
|
1317
|
+
if (this.pendingStartAborts.get(piboSessionId) === startAbort)
|
|
1318
|
+
this.pendingStartAborts.delete(piboSessionId);
|
|
1319
|
+
}
|
|
1320
|
+
}
|
|
1321
|
+
getRuntimeCapacityStatus() { return { ...this.capacity.snapshot(), telemetry: this.telemetryWriter?.status(), activeRuntimes: this.sessions.size, initializingRuntimes: this.creatingRuntimes, recentInitializations: this.initializationTimings.map(item => ({ ...item, phases: { ...item.phases } })) }; }
|
|
1322
|
+
async createRoutedSessionWithinCapacity(piboSessionId, signal) {
|
|
1323
|
+
const stored = this.resolvePiboSession(piboSessionId);
|
|
1324
|
+
const room = typeof stored.metadata?.chatRoomId === "string" ? stored.metadata.chatRoomId : piboSessionId;
|
|
1325
|
+
const queuedAt = performance.now();
|
|
1326
|
+
const lease = await this.capacity.starts.acquire(room, signal);
|
|
1327
|
+
const startedAt = performance.now();
|
|
1328
|
+
const phases = {};
|
|
1329
|
+
let outcome = "failed";
|
|
1330
|
+
try {
|
|
1331
|
+
signal.throwIfAborted();
|
|
1332
|
+
if (this.closing || this.quiescingSessions.has(piboSessionId))
|
|
1333
|
+
throw new Error("Runtime start cancelled during session shutdown.");
|
|
1334
|
+
if (this.sessions.size + this.creatingRuntimes >= this.capacity.maxRuntimes) {
|
|
1335
|
+
const idle = [...this.sessions].find(([id, session]) => { if (this.pendingSessions.has(id))
|
|
1336
|
+
return false; const status = session.getStatus(); return !status.disposed && !status.processing && !status.streaming && status.queuedMessages === 0; });
|
|
1337
|
+
if (idle)
|
|
1338
|
+
await this.evictIdleSession(idle[0], idle[1]);
|
|
1339
|
+
}
|
|
1340
|
+
if (this.sessions.size + this.creatingRuntimes >= this.capacity.maxRuntimes)
|
|
1341
|
+
throw Object.assign(new Error("Active runtime capacity reached."), { code: "runtime_capacity_unavailable" });
|
|
1342
|
+
this.creatingRuntimes++;
|
|
1343
|
+
try {
|
|
1344
|
+
const session = await this.createRoutedSession(piboSessionId, phases);
|
|
1345
|
+
if (signal.aborted) {
|
|
1346
|
+
if (this.sessions.get(piboSessionId) === session)
|
|
1347
|
+
this.sessions.delete(piboSessionId);
|
|
1348
|
+
await this.disposeRoutedSession(piboSessionId, session, "cold start aborted");
|
|
1349
|
+
throw signal.reason;
|
|
1350
|
+
}
|
|
1351
|
+
outcome = "ready";
|
|
1352
|
+
return session;
|
|
1353
|
+
}
|
|
1354
|
+
finally {
|
|
1355
|
+
this.creatingRuntimes--;
|
|
1356
|
+
}
|
|
1357
|
+
}
|
|
1358
|
+
finally {
|
|
1359
|
+
lease.release();
|
|
1360
|
+
this.initializationTimings.push({ sessionId: piboSessionId, waitMs: startedAt - queuedAt, totalMs: performance.now() - startedAt, phases, outcome });
|
|
1361
|
+
if (this.initializationTimings.length > 32)
|
|
1362
|
+
this.initializationTimings.shift();
|
|
1255
1363
|
}
|
|
1256
1364
|
}
|
|
1257
|
-
async createRoutedSession(piboSessionId) {
|
|
1365
|
+
async createRoutedSession(piboSessionId, phases = {}) {
|
|
1366
|
+
let phaseStarted = performance.now();
|
|
1258
1367
|
const piboSession = this.resolvePiboSession(piboSessionId);
|
|
1259
1368
|
let session;
|
|
1260
1369
|
this.signalRegistry.project({ type: "session_created", session: piboSession });
|
|
@@ -1313,6 +1422,8 @@ export class PiboSessionRouter {
|
|
|
1313
1422
|
if (invalidProfile) {
|
|
1314
1423
|
throw new Error(`Runtime profile validation failed: ${invalidProfile.message}`);
|
|
1315
1424
|
}
|
|
1425
|
+
phases.bindingAndProfileMs = performance.now() - phaseStarted;
|
|
1426
|
+
phaseStarted = performance.now();
|
|
1316
1427
|
let historyHandoff;
|
|
1317
1428
|
if (persistedHistoryHandoff?.mode === "import") {
|
|
1318
1429
|
if (!runtimeAdapter.descriptor.capabilities.historyImport) {
|
|
@@ -1336,6 +1447,8 @@ export class PiboSessionRouter {
|
|
|
1336
1447
|
else if (persistedHistoryHandoff?.mode === "fresh") {
|
|
1337
1448
|
historyHandoff = { mode: "fresh" };
|
|
1338
1449
|
}
|
|
1450
|
+
phases.portableHistoryMs = performance.now() - phaseStarted;
|
|
1451
|
+
phaseStarted = performance.now();
|
|
1339
1452
|
const initialFastMode = resolvePiboSessionInitialFastMode(piboSession) ?? selectRequestedFastMode(sessionProfile, modelDefaults) ?? false;
|
|
1340
1453
|
const agentsController = this.createAgentsController(piboSession.id);
|
|
1341
1454
|
const runToolController = this.createRunToolController(piboSession.id);
|
|
@@ -1380,6 +1493,8 @@ export class PiboSessionRouter {
|
|
|
1380
1493
|
this.portableToolSessions.delete(piboSession.id);
|
|
1381
1494
|
throw error;
|
|
1382
1495
|
}
|
|
1496
|
+
phases.resourcesAndToolsMs = performance.now() - phaseStarted;
|
|
1497
|
+
phaseStarted = performance.now();
|
|
1383
1498
|
const bindingSync = { expectedRevision: binding.revision };
|
|
1384
1499
|
const runtimeBindingPersistence = createAgentRuntimeBindingPersistence(this.sessionStore, {
|
|
1385
1500
|
piboSessionId: piboSession.id,
|
|
@@ -1423,6 +1538,7 @@ export class PiboSessionRouter {
|
|
|
1423
1538
|
thinkingLevel: initialThinkingLevel ?? this.options.thinkingLevel,
|
|
1424
1539
|
retryDefaults: resolvePiboSessionRetryDefaults(piboSession.kind, this.options.retryDefaults),
|
|
1425
1540
|
extensionFactories: [
|
|
1541
|
+
createProviderCapacityExtension(this.capacity, piboRoomIdFromMetadata(piboSession.metadata) ?? piboSession.id, Boolean(piboSession.parentId)),
|
|
1426
1542
|
...(telemetryExtension ? [telemetryExtension] : []),
|
|
1427
1543
|
...(this.options.extensionFactories ?? []),
|
|
1428
1544
|
],
|
|
@@ -1492,6 +1608,7 @@ export class PiboSessionRouter {
|
|
|
1492
1608
|
this.runtimeResourceSessions.delete(piboSession.id);
|
|
1493
1609
|
throw error;
|
|
1494
1610
|
}
|
|
1611
|
+
phases.adapterOpenAndBindingMs = performance.now() - phaseStarted;
|
|
1495
1612
|
const resourceInspection = resources.getInspection();
|
|
1496
1613
|
const statusResources = {
|
|
1497
1614
|
enabledSkills: [...new Set(resourceInspection.skills.map((skill) => skill.name))],
|
|
@@ -1533,6 +1650,7 @@ export class PiboSessionRouter {
|
|
|
1533
1650
|
this.handleInterruptedRunReminders(messages);
|
|
1534
1651
|
},
|
|
1535
1652
|
messagePreflight: this.options.messagePreflight,
|
|
1653
|
+
acquireProviderCapacity: runtimeAdapter.descriptor.id === "pi" ? undefined : (provider, signal) => this.capacity.acquireProvider(provider, typeof piboSession.metadata?.chatRoomId === "string" ? piboSession.metadata.chatRoomId : piboSession.id, signal, Boolean(piboSession.parentId)),
|
|
1536
1654
|
modelFallbacks,
|
|
1537
1655
|
getRuntimeAuthStatus: () => runtimeRegistry.getAgentRuntimeAuthStatus(binding.runtimeInstanceId),
|
|
1538
1656
|
startRuntimeAuth: async (input) => {
|
|
@@ -1553,6 +1671,7 @@ export class PiboSessionRouter {
|
|
|
1553
1671
|
},
|
|
1554
1672
|
statusResources,
|
|
1555
1673
|
getToolMetricTokenCalculation: () => loadPiboUserSettings().toolMetrics.tokenCalculation,
|
|
1674
|
+
now: this.options.runtimeQueueNow,
|
|
1556
1675
|
});
|
|
1557
1676
|
this.sessions.set(piboSession.id, session);
|
|
1558
1677
|
return session;
|
|
@@ -2584,6 +2703,7 @@ export class PiboSessionRouter {
|
|
|
2584
2703
|
this.runRegistry.suppressNotification(delivery.piboSessionId, run.runId);
|
|
2585
2704
|
this.clearRunReminderRecovery(delivery);
|
|
2586
2705
|
this.deferredRunReminders.delete(delivery.piboSessionId);
|
|
2706
|
+
this.clearRunReminderAdmissionWarning(delivery.piboSessionId, delivery.generation);
|
|
2587
2707
|
this.sessions.get(delivery.piboSessionId)?.removeQueuedMessages(isRunReminderServiceMessage);
|
|
2588
2708
|
this.scheduleRunReminder(delivery.piboSessionId, false, delivery.generation);
|
|
2589
2709
|
}
|
|
@@ -2673,6 +2793,7 @@ export class PiboSessionRouter {
|
|
|
2673
2793
|
this.scheduledRunReminders.delete(piboSessionId);
|
|
2674
2794
|
this.deferredRunReminders.delete(piboSessionId);
|
|
2675
2795
|
this.runReminderRecoveries.delete(piboSessionId);
|
|
2796
|
+
this.clearRunReminderAdmissionWarning(piboSessionId);
|
|
2676
2797
|
for (const [eventId, delivery] of this.runReminderDeliveries) {
|
|
2677
2798
|
if (delivery.piboSessionId === piboSessionId)
|
|
2678
2799
|
this.runReminderDeliveries.delete(eventId);
|
|
@@ -2710,8 +2831,18 @@ export class PiboSessionRouter {
|
|
|
2710
2831
|
return;
|
|
2711
2832
|
if (deferredGeneration !== undefined)
|
|
2712
2833
|
this.deferredRunReminders.delete(piboSessionId);
|
|
2713
|
-
if (!this.runRegistry.hasPendingNotification(piboSessionId, { includeAlreadyNotified }))
|
|
2834
|
+
if (!this.runRegistry.hasPendingNotification(piboSessionId, { includeAlreadyNotified })) {
|
|
2835
|
+
this.clearRunReminderAdmissionWarning(piboSessionId, expectedGeneration);
|
|
2714
2836
|
return;
|
|
2837
|
+
}
|
|
2838
|
+
if (includeAlreadyNotified) {
|
|
2839
|
+
const status = this.sessions.get(piboSessionId)?.getStatus?.();
|
|
2840
|
+
const effectiveEventIds = new Set([status?.activeEventId, ...(status?.queuedEventIds ?? [])].filter((id) => Boolean(id)));
|
|
2841
|
+
if ([...this.runReminderDeliveries.entries()].some(([eventId, delivery]) => (effectiveEventIds.has(eventId)
|
|
2842
|
+
&& delivery.piboSessionId === piboSessionId
|
|
2843
|
+
&& delivery.generation === expectedGeneration)))
|
|
2844
|
+
return;
|
|
2845
|
+
}
|
|
2715
2846
|
const previous = this.scheduledRunReminders.get(piboSessionId);
|
|
2716
2847
|
if (previous?.generation === expectedGeneration) {
|
|
2717
2848
|
this.scheduledRunReminders.set(piboSessionId, {
|
|
@@ -2727,8 +2858,33 @@ export class PiboSessionRouter {
|
|
|
2727
2858
|
}
|
|
2728
2859
|
refreshQueuedRunReminders(piboSessionId) {
|
|
2729
2860
|
const removed = this.sessions.get(piboSessionId)?.removeQueuedMessages(isRunReminderServiceMessage) ?? 0;
|
|
2730
|
-
if (removed > 0)
|
|
2861
|
+
if (removed > 0) {
|
|
2731
2862
|
this.scheduleRunReminder(piboSessionId, true);
|
|
2863
|
+
}
|
|
2864
|
+
else if (!this.runRegistry.hasPendingNotification(piboSessionId, { includeAlreadyNotified: true })) {
|
|
2865
|
+
this.clearRunReminderAdmissionWarning(piboSessionId);
|
|
2866
|
+
}
|
|
2867
|
+
}
|
|
2868
|
+
clearRunReminderAdmissionWarning(piboSessionId, generation) {
|
|
2869
|
+
if (generation !== undefined && this.runReminderAdmissionWarnings.get(piboSessionId) !== generation)
|
|
2870
|
+
return;
|
|
2871
|
+
this.runReminderAdmissionWarnings.delete(piboSessionId);
|
|
2872
|
+
this.sessions.get(piboSessionId)?.setRunReminderDeferredWarning?.(undefined);
|
|
2873
|
+
}
|
|
2874
|
+
deferRunReminderAdmission(piboSessionId, generation, error) {
|
|
2875
|
+
const diagnostic = runReminderAdmissionDiagnostic(error);
|
|
2876
|
+
this.sessions.get(piboSessionId)?.setRunReminderDeferredWarning?.(diagnostic.warning);
|
|
2877
|
+
if (this.runReminderAdmissionWarnings.get(piboSessionId) === generation)
|
|
2878
|
+
return;
|
|
2879
|
+
this.runReminderAdmissionWarnings.set(piboSessionId, generation);
|
|
2880
|
+
console.warn("[pibo] run reminder deferred", {
|
|
2881
|
+
piboSessionId,
|
|
2882
|
+
generation,
|
|
2883
|
+
code: diagnostic.code,
|
|
2884
|
+
...(diagnostic.dimension ? { dimension: diagnostic.dimension } : {}),
|
|
2885
|
+
...(diagnostic.current ? { current: diagnostic.current } : {}),
|
|
2886
|
+
...(diagnostic.limit !== undefined ? { limit: diagnostic.limit } : {}),
|
|
2887
|
+
});
|
|
2732
2888
|
}
|
|
2733
2889
|
async deliverRunReminder(piboSessionId, expectedGeneration) {
|
|
2734
2890
|
const scheduled = this.scheduledRunReminders.get(piboSessionId);
|
|
@@ -2743,9 +2899,21 @@ export class PiboSessionRouter {
|
|
|
2743
2899
|
const session = await this.getOrCreateSession(piboSessionId);
|
|
2744
2900
|
if (this.closing || this.quiescingSessions.has(piboSessionId) || expectedGeneration !== this.runReminderGeneration(piboSessionId))
|
|
2745
2901
|
return;
|
|
2746
|
-
|
|
2747
|
-
|
|
2902
|
+
// Release the queued snapshot before reserving its replacement. The
|
|
2903
|
+
// interruption callback restores every unconsumed run to pending state.
|
|
2904
|
+
const replaced = session.removeQueuedMessages?.((event) => {
|
|
2905
|
+
if (!isRunReminderServiceMessage(event) || !event.id)
|
|
2906
|
+
return false;
|
|
2907
|
+
const delivery = this.runReminderDeliveries.get(event.id);
|
|
2908
|
+
return delivery?.piboSessionId === piboSessionId && delivery.generation === expectedGeneration;
|
|
2909
|
+
}) ?? 0;
|
|
2910
|
+
notification = this.runRegistry.createNotification(piboSessionId, {
|
|
2911
|
+
includeAlreadyNotified: scheduled.includeAlreadyNotified || replaced > 0,
|
|
2912
|
+
});
|
|
2913
|
+
if (!notification) {
|
|
2914
|
+
this.clearRunReminderAdmissionWarning(piboSessionId, expectedGeneration);
|
|
2748
2915
|
return;
|
|
2916
|
+
}
|
|
2749
2917
|
eventId = randomUUID();
|
|
2750
2918
|
this.runReminderDeliveries.set(eventId, { piboSessionId, generation: expectedGeneration, notification });
|
|
2751
2919
|
session.enqueueMessage({
|
|
@@ -2756,6 +2924,7 @@ export class PiboSessionRouter {
|
|
|
2756
2924
|
id: eventId,
|
|
2757
2925
|
provenance: runReminderProvenance(notification),
|
|
2758
2926
|
});
|
|
2927
|
+
this.clearRunReminderAdmissionWarning(piboSessionId, expectedGeneration);
|
|
2759
2928
|
}
|
|
2760
2929
|
catch (error) {
|
|
2761
2930
|
if (eventId)
|
|
@@ -2764,13 +2933,9 @@ export class PiboSessionRouter {
|
|
|
2764
2933
|
this.runRegistry.releaseNotification(piboSessionId, notification);
|
|
2765
2934
|
if (this.closing || this.quiescingSessions.has(piboSessionId) || expectedGeneration !== this.runReminderGeneration(piboSessionId))
|
|
2766
2935
|
return;
|
|
2767
|
-
|
|
2768
|
-
|
|
2769
|
-
|
|
2770
|
-
piboSessionId,
|
|
2771
|
-
error: message,
|
|
2772
|
-
errorDetails: runtimeSessionErrorDetails(message),
|
|
2773
|
-
});
|
|
2936
|
+
// Reminder delivery is internal recovery work. Preserve it for the next
|
|
2937
|
+
// bounded state transition instead of emitting an unscoped fatal error.
|
|
2938
|
+
this.deferRunReminderAdmission(piboSessionId, expectedGeneration, error);
|
|
2774
2939
|
}
|
|
2775
2940
|
}
|
|
2776
2941
|
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { BoundedWorkerClient } from "./bounded-worker-client.js";
|
|
2
|
+
export const CHAT_READ_METHODS = {
|
|
3
|
+
timeline: ["listEvents", "listSessionEvents", "listAllSessionEvents", "listMessageTurnTimings", "scanMessageTurnTimings", "listTraceEvents", "isPayloadAttachedToTraceNode", "countEventsByType", "getLatestEventSequence", "getLatestStreamId"],
|
|
4
|
+
history: ["listProductHistoryEntries", "getProductHistoryCoverage"],
|
|
5
|
+
navigation: ["unreadCountsPage", "sessionIndexPage"],
|
|
6
|
+
};
|
|
7
|
+
/** Read traffic owns a separate connection and queue from admission and output commits. */
|
|
8
|
+
export class AsyncChatReadQueries {
|
|
9
|
+
path;
|
|
10
|
+
payloadRootDir;
|
|
11
|
+
client;
|
|
12
|
+
closed = false;
|
|
13
|
+
restarts = 0;
|
|
14
|
+
restartAfter = 0;
|
|
15
|
+
timeline;
|
|
16
|
+
navigation;
|
|
17
|
+
history;
|
|
18
|
+
constructor(path, payloadRootDir) {
|
|
19
|
+
this.path = path;
|
|
20
|
+
this.payloadRootDir = payloadRootDir;
|
|
21
|
+
this.client = this.startWorker();
|
|
22
|
+
const group = (name) => Object.fromEntries(CHAT_READ_METHODS[name].map(method => [method, (...args) => this.getClient().request({ group: name, method, args }, { priority: "background", fairnessKey: typeof args[0] === "string" ? args[0] : args[0]?.piboSessionId })]));
|
|
23
|
+
this.timeline = group("timeline");
|
|
24
|
+
this.history = group("history");
|
|
25
|
+
this.navigation = group("navigation");
|
|
26
|
+
}
|
|
27
|
+
startWorker() {
|
|
28
|
+
this.restartAfter = Date.now() + 1000;
|
|
29
|
+
return new BoundedWorkerClient(new URL("./chat-read-worker.js", import.meta.url), { maxPending: 64, reservedControlRequests: 2, reservedControlBytes: 16384, maxPendingBytes: 1024 * 1024, maxMessageBytes: 4 * 1024 * 1024, maxAgeMs: 2000, workerOptions: { workerData: { path: this.path, payloadRootDir: this.payloadRootDir } } });
|
|
30
|
+
}
|
|
31
|
+
getClient() { if (!this.closed && this.client.status().closed && this.client.status().exited && Date.now() >= this.restartAfter) {
|
|
32
|
+
this.restarts++;
|
|
33
|
+
this.client = this.startWorker();
|
|
34
|
+
} return this.client; }
|
|
35
|
+
maintenance(action) { return this.getClient().request({ group: "maintenance", method: action, args: [] }, { priority: "control" }); }
|
|
36
|
+
status() { return { ...this.client.status(), restarts: this.restarts }; }
|
|
37
|
+
async close() { this.closed = true; await this.client.close(); }
|
|
38
|
+
}
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import { BoundedWorkerClient, StorageUnavailableError } from "./bounded-worker-client.js";
|
|
2
|
+
/** Admission/output writer; bulk reads must use a separate worker connection. */
|
|
3
|
+
export class AsyncChatStorage {
|
|
4
|
+
writerClient;
|
|
5
|
+
reader;
|
|
6
|
+
workerUrl;
|
|
7
|
+
workerOptions;
|
|
8
|
+
closed = false;
|
|
9
|
+
writerRestarts = 0;
|
|
10
|
+
readerRestarts = 0;
|
|
11
|
+
writerRestartAfter = 0;
|
|
12
|
+
readerRestartAfter = 0;
|
|
13
|
+
constructor(path, payloadRootDir, options = {}) {
|
|
14
|
+
if (path === ":memory:")
|
|
15
|
+
throw new Error("Worker storage requires a file-backed database.");
|
|
16
|
+
// Admission carries the bounded message plus its event projection; account for both in IPC.
|
|
17
|
+
this.workerOptions = { maxMessageBytes: 4 * 1024 * 1024, reservedControlRequests: 8, reservedControlBytes: 256 * 1024, admissionWindowMs: 1, ...options, workerOptions: { ...options.workerOptions, workerData: { path, payloadRootDir } } };
|
|
18
|
+
this.workerUrl = new URL("./chat-storage-worker.js", import.meta.url);
|
|
19
|
+
this.writerClient = this.startWriter();
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* A crashed worker, or one that breached its execution deadline, must not disable durable
|
|
23
|
+
* admission permanently. Replacement waits for the failed worker to exit and is bounded to one
|
|
24
|
+
* per second, so two writers never own the same SQLite file. Recovery is safe without replay
|
|
25
|
+
* protection because callers reconcile the same client transaction ID and admission is
|
|
26
|
+
* idempotent by request key.
|
|
27
|
+
*/
|
|
28
|
+
get writer() {
|
|
29
|
+
const status = this.writerClient.status();
|
|
30
|
+
if (!this.closed && status.closed && status.exited && Date.now() >= this.writerRestartAfter) {
|
|
31
|
+
this.writerRestarts++;
|
|
32
|
+
this.writerClient = this.startWriter();
|
|
33
|
+
}
|
|
34
|
+
return this.writerClient;
|
|
35
|
+
}
|
|
36
|
+
get readerClient() {
|
|
37
|
+
const status = this.reader?.status();
|
|
38
|
+
if (!this.reader || (!this.closed && status.closed && status.exited && Date.now() >= this.readerRestartAfter)) {
|
|
39
|
+
if (this.reader)
|
|
40
|
+
this.readerRestarts++;
|
|
41
|
+
this.reader = this.startReader();
|
|
42
|
+
}
|
|
43
|
+
return this.reader;
|
|
44
|
+
}
|
|
45
|
+
startWriter() {
|
|
46
|
+
this.writerRestartAfter = Date.now() + 1000;
|
|
47
|
+
return new BoundedWorkerClient(this.workerUrl, this.workerOptions);
|
|
48
|
+
}
|
|
49
|
+
startReader() {
|
|
50
|
+
this.readerRestartAfter = Date.now() + 1000;
|
|
51
|
+
return new BoundedWorkerClient(this.workerUrl, this.workerOptions);
|
|
52
|
+
}
|
|
53
|
+
resolveRoom(roomId, required = false) {
|
|
54
|
+
return this.writer.request({ type: "resolveRoom", roomId, required });
|
|
55
|
+
}
|
|
56
|
+
admit(input, session, text, durableCommand) {
|
|
57
|
+
if (durableCommand && Buffer.byteLength(text) > 1024 * 1024)
|
|
58
|
+
return Promise.reject(Object.assign(new Error("Message exceeds the durable command byte limit."), { code: "command_too_large" }));
|
|
59
|
+
return this.writer.request({ type: "admit", input, session, text, durableCommand }, { fairnessKey: input.roomId, priority: durableCommand?.delivery === "steer" ? "control" : "admission" });
|
|
60
|
+
}
|
|
61
|
+
append(input) {
|
|
62
|
+
return this.writer.request({ type: "append", input });
|
|
63
|
+
}
|
|
64
|
+
find(roomId, actorId, clientTxnId) {
|
|
65
|
+
if (this.closed)
|
|
66
|
+
return Promise.reject(new StorageUnavailableError("storage_closed", "Storage worker is closed."));
|
|
67
|
+
return this.readerClient.request({ type: "find", roomId, actorId, clientTxnId }, { priority: "admission" });
|
|
68
|
+
}
|
|
69
|
+
ingestUser(input) {
|
|
70
|
+
return this.writer.request({ type: "ingestUser", input });
|
|
71
|
+
}
|
|
72
|
+
async ingestOutput(input) {
|
|
73
|
+
const result = await this.writer.request({ type: "ingestOutput", input }, { priority: "output", fairnessKey: input.roomId });
|
|
74
|
+
if (input.event.type === "compaction_end" && result.enrichment?.compactionStats) {
|
|
75
|
+
input.event.compactionStats = result.enrichment.compactionStats;
|
|
76
|
+
}
|
|
77
|
+
return result;
|
|
78
|
+
}
|
|
79
|
+
cancelPendingCommands(sessionId) { return this.writer.request({ type: "cancelPendingCommands", sessionId }, { priority: "control" }); }
|
|
80
|
+
commandReceiptPage(sessionId) { return this.writer.request({ type: "commandReceiptPage", sessionId }, { priority: "control" }); }
|
|
81
|
+
commandReceipts(sessionId) { return this.writer.request({ type: "commandReceipts", sessionId }, { priority: "control" }); }
|
|
82
|
+
commandReceipt(id) { return this.writer.request({ type: "commandReceipt", id }, { priority: "control" }); }
|
|
83
|
+
claimCommand(owner, leaseMs) { return this.writer.request({ type: "claimCommand", owner, leaseMs }, { priority: "background" }); }
|
|
84
|
+
transitionCommand(id, owner, token, state, error) { return this.writer.request({ type: "transitionCommand", id, owner, token, state, error }, { priority: "control" }); }
|
|
85
|
+
heartbeatCommand(id, owner, token, leaseMs) { return this.writer.request({ type: "heartbeatCommand", id, owner, token, leaseMs }, { priority: "control" }); }
|
|
86
|
+
durableQueueHealth() { return this.writer.request({ type: "durableQueueHealth" }, { priority: "control", timeoutMs: 500 }); }
|
|
87
|
+
status() {
|
|
88
|
+
const writer = this.writer.status();
|
|
89
|
+
const reader = this.reader?.status();
|
|
90
|
+
return { ready: writer.ready && (!reader || reader.ready), closed: writer.closed && (!reader || reader.closed), restarts: { writer: this.writerRestarts, reader: this.readerRestarts }, writer, reader };
|
|
91
|
+
}
|
|
92
|
+
async close() {
|
|
93
|
+
this.closed = true;
|
|
94
|
+
await Promise.all([this.writerClient.close(), this.reader?.close()]);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { BoundedWorkerClient } from "./bounded-worker-client.js";
|
|
2
|
+
export class AsyncTelemetryMaintenance {
|
|
3
|
+
client;
|
|
4
|
+
constructor(path) { this.client = new BoundedWorkerClient(new URL("./telemetry-maintenance-worker.js", import.meta.url), { maxPending: 8, maxPendingBytes: 64 * 1024, maxMessageBytes: 16 * 1024, maxAgeMs: 2000, workerOptions: { workerData: { path } } }); }
|
|
5
|
+
command(action, cutoff) { return this.client.request({ action, cutoff }, { priority: "control" }); }
|
|
6
|
+
preview(cutoff) { return this.client.request({ action: "preview", cutoff }, { priority: "background" }); }
|
|
7
|
+
status() { return this.client.status(); }
|
|
8
|
+
close() { return this.client.close(); }
|
|
9
|
+
}
|