@pasko70/pibo 3.5.0 → 3.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-runtime/routed-session.js +80 -19
- package/dist/agent-runtimes/codex-native/turn.js +27 -3
- package/dist/apps/chat/data/chat-data-mappers.js +8 -1
- package/dist/apps/chat/data/read-state-service.js +24 -3
- package/dist/apps/chat/message-command-dispatcher.js +16 -1
- package/dist/apps/chat/web-app.js +40 -19
- package/dist/apps/chat-ui/assets/{dist-D79vyxSX.js → dist-B-auLrzD.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-DFZ8cwh0.js → dist-BA_dsINH.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-D6TjFhAm.js → dist-eJZar_0-.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-BG0n7zLd.js → dist-wNNR2Bci.js} +1 -1
- package/dist/apps/chat-ui/assets/{dist-cOjokPrK.js → dist-zcmEsIEp.js} +1 -1
- package/dist/apps/chat-ui/assets/{index-RMHUTJ62.js → index-DEkbN5Vo.js} +43 -43
- package/dist/apps/chat-ui/assets/index-DZK6Tzil.css +1 -0
- package/dist/apps/chat-ui/index.html +2 -2
- package/dist/apps/chat-vscode-web/assets/{index-xacbCyTx.js → index-DZgW1fCB.js} +11 -11
- package/dist/apps/chat-vscode-web/index.html +1 -1
- package/dist/cli-session/localSessionSource.js +3 -2
- package/dist/core/output-render-sequence.js +63 -6
- package/dist/core/session-router.js +99 -11
- package/dist/data/async-chat-storage.js +7 -2
- package/dist/data/bounded-worker-client.js +1 -1
- package/dist/data/chat-read-projections.js +4 -4
- package/dist/data/chat-storage-worker.js +24 -3
- package/dist/data/ingest-service.js +73 -4
- package/dist/data/message-command-store.js +153 -11
- package/dist/data/schema.js +24 -3
- package/dist/data/storage-maintenance.js +344 -0
- package/dist/data/storage-verification-worker.js +25 -0
- package/dist/debug/index.js +155 -1
- package/dist/debug/message-queue.js +108 -0
- package/dist/debug/output-collision-repair.js +140 -0
- package/dist/debug/output-integrity.js +38 -2
- package/dist/debug/output-repair.js +1 -0
- package/dist/debug/storage-backup.js +12 -4
- package/dist/debug/storage-maintenance.js +78 -0
- package/dist/gateway/cli.js +71 -7
- package/dist/gateway/server.js +1 -0
- package/dist/reliability/store.js +119 -23
- package/dist/session-ui/terminalRows.js +7 -8
- package/dist/sessions/pibo-data-store.js +18 -14
- package/dist/shared/trace-event-projection.js +13 -3
- package/dist/web/channel.js +114 -12
- package/dist/web/http.js +105 -44
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
- package/dist/apps/chat-ui/assets/index-hEkrlRk-.css +0 -1
|
@@ -15,10 +15,32 @@ CREATE INDEX IF NOT EXISTS message_commands_recent ON message_commands(session_i
|
|
|
15
15
|
CREATE INDEX IF NOT EXISTS message_commands_pending ON message_commands(state,created_at,id);
|
|
16
16
|
CREATE INDEX IF NOT EXISTS message_commands_session ON message_commands(session_id,state,created_at,id);
|
|
17
17
|
CREATE INDEX IF NOT EXISTS message_commands_lease ON message_commands(lease_until) WHERE owner IS NOT NULL;
|
|
18
|
+
CREATE INDEX IF NOT EXISTS message_commands_health_scope ON message_commands(state,delivery,room_id,session_id,stream_id,created_at);
|
|
19
|
+
CREATE INDEX IF NOT EXISTS event_log_message_command_terminal ON event_log(session_id,event_id,type,stream_id)
|
|
20
|
+
WHERE event_id IS NOT NULL AND type IN ('message_finished','session_error','message_steered');
|
|
21
|
+
CREATE TABLE IF NOT EXISTS message_command_stats (
|
|
22
|
+
state TEXT NOT NULL, delivery TEXT NOT NULL, command_count INTEGER NOT NULL, payload_bytes INTEGER NOT NULL,
|
|
23
|
+
PRIMARY KEY(state,delivery)
|
|
24
|
+
);
|
|
25
|
+
CREATE TRIGGER IF NOT EXISTS message_command_stats_insert AFTER INSERT ON message_commands BEGIN
|
|
26
|
+
INSERT INTO message_command_stats(state,delivery,command_count,payload_bytes) VALUES(NEW.state,NEW.delivery,1,NEW.payload_bytes)
|
|
27
|
+
ON CONFLICT(state,delivery) DO UPDATE SET command_count=command_count+1,payload_bytes=payload_bytes+NEW.payload_bytes;
|
|
28
|
+
END;
|
|
29
|
+
CREATE TRIGGER IF NOT EXISTS message_command_stats_delete AFTER DELETE ON message_commands BEGIN
|
|
30
|
+
UPDATE message_command_stats SET command_count=command_count-1,payload_bytes=payload_bytes-OLD.payload_bytes WHERE state=OLD.state AND delivery=OLD.delivery;
|
|
31
|
+
END;
|
|
32
|
+
CREATE TRIGGER IF NOT EXISTS message_command_stats_update AFTER UPDATE OF state,delivery,payload_bytes ON message_commands
|
|
33
|
+
WHEN OLD.state<>NEW.state OR OLD.delivery<>NEW.delivery OR OLD.payload_bytes<>NEW.payload_bytes BEGIN
|
|
34
|
+
UPDATE message_command_stats SET command_count=command_count-1,payload_bytes=payload_bytes-OLD.payload_bytes WHERE state=OLD.state AND delivery=OLD.delivery;
|
|
35
|
+
INSERT INTO message_command_stats(state,delivery,command_count,payload_bytes) VALUES(NEW.state,NEW.delivery,1,NEW.payload_bytes)
|
|
36
|
+
ON CONFLICT(state,delivery) DO UPDATE SET command_count=command_count+1,payload_bytes=payload_bytes+NEW.payload_bytes;
|
|
37
|
+
END;
|
|
18
38
|
CREATE TABLE IF NOT EXISTS message_dispatch_clock (id INTEGER PRIMARY KEY CHECK(id=1), sequence INTEGER NOT NULL);
|
|
19
39
|
INSERT OR IGNORE INTO message_dispatch_clock VALUES (1,0);
|
|
20
40
|
CREATE TABLE IF NOT EXISTS message_dispatch_rooms (room_id TEXT PRIMARY KEY, sequence INTEGER NOT NULL);
|
|
21
|
-
|
|
41
|
+
INSERT INTO message_command_stats(state,delivery,command_count,payload_bytes)
|
|
42
|
+
SELECT state,delivery,count(*),COALESCE(sum(payload_bytes),0) FROM message_commands
|
|
43
|
+
WHERE NOT EXISTS (SELECT 1 FROM message_command_stats) GROUP BY state,delivery;
|
|
22
44
|
`;
|
|
23
45
|
export const MESSAGE_COMMAND_LIMITS = Object.freeze({
|
|
24
46
|
queue: { count: 1000, roomCount: 256, sessionCount: 64, bytes: 64 * 1024 * 1024, roomBytes: 16 * 1024 * 1024, sessionBytes: 4 * 1024 * 1024, ageMs: 60 * 60 * 1000, roomAgeMs: 15 * 60 * 1000, sessionAgeMs: 10 * 60 * 1000 },
|
|
@@ -26,6 +48,12 @@ export const MESSAGE_COMMAND_LIMITS = Object.freeze({
|
|
|
26
48
|
dispatch: { queue: 10, roomQueue: 5, steer: 2, roomSteer: 1 },
|
|
27
49
|
});
|
|
28
50
|
const active = "'accepted','waiting_slot','initializing','session_queue','running'";
|
|
51
|
+
const unstarted = "'accepted','waiting_slot'";
|
|
52
|
+
const terminalEvidenceSql = `(SELECT CASE
|
|
53
|
+
WHEN count(DISTINCT CASE WHEN e.type='session_error' THEN 'failed' ELSE 'completed' END)=1
|
|
54
|
+
THEN max(CASE WHEN e.type='session_error' THEN 'failed' ELSE 'completed' END) END
|
|
55
|
+
FROM event_log e WHERE e.session_id=message_commands.session_id AND e.event_id=message_commands.event_id
|
|
56
|
+
AND e.type IN ('message_finished','session_error','message_steered'))`;
|
|
29
57
|
/** Durable receipts are independent of optional trace/telemetry retention. Only the storage worker owns this store. */
|
|
30
58
|
export class MessageCommandStore {
|
|
31
59
|
store;
|
|
@@ -56,36 +84,109 @@ export class MessageCommandStore {
|
|
|
56
84
|
}
|
|
57
85
|
queueStatus(sessionId) {
|
|
58
86
|
const rows = this.store.db.prepare(`SELECT state,delivery,payload_bytes,created_at FROM message_commands WHERE session_id=? AND state IN (${active}) LIMIT 69`).all(sessionId);
|
|
87
|
+
const blocker = this.store.db.prepare("SELECT id,created_at FROM message_commands WHERE session_id=? AND state='interrupted' ORDER BY stream_id LIMIT 1").get(sessionId);
|
|
59
88
|
const summarize = (delivery) => {
|
|
60
89
|
const scoped = rows.filter(row => row.delivery === delivery), limits = MESSAGE_COMMAND_LIMITS[delivery];
|
|
61
90
|
return { count: scoped.length, bytes: scoped.reduce((sum, row) => sum + row.payload_bytes, 0), oldestWaitMs: Math.max(0, ...scoped.filter(row => row.state === "accepted" || row.state === "waiting_slot").map(row => Date.now() - row.created_at)), limits: { count: limits.sessionCount, bytes: limits.sessionBytes, waitMs: limits.sessionAgeMs } };
|
|
62
91
|
};
|
|
63
|
-
return { queue: summarize("queue"), steer: summarize("steer") };
|
|
92
|
+
return { queue: summarize("queue"), steer: summarize("steer"), ...(blocker ? { reconciliation: { required: true, retryable: false, scope: "session", blockingCommandId: blocker.id, blockedSince: blocker.created_at, nextAction: `pibo debug message-queue inspect --session ${sessionId}` } } : {}) };
|
|
64
93
|
}
|
|
65
94
|
get(id) {
|
|
66
95
|
const row = this.store.db.prepare("SELECT * FROM message_commands WHERE id = ?").get(id);
|
|
67
96
|
return row && receipt(row);
|
|
68
97
|
}
|
|
98
|
+
terminalOutcome(sessionId, eventId) {
|
|
99
|
+
const row = this.store.db.prepare(`SELECT CASE WHEN count(DISTINCT CASE WHEN type='session_error' THEN 'failed' ELSE 'completed' END)=1 THEN max(CASE WHEN type='session_error' THEN 'failed' ELSE 'completed' END) END state FROM event_log WHERE session_id=? AND event_id=? AND type IN ('message_finished','session_error','message_steered')`).get(sessionId, eventId);
|
|
100
|
+
return row?.state;
|
|
101
|
+
}
|
|
102
|
+
terminalEvidence(sessionId, eventId) {
|
|
103
|
+
return this.store.db.prepare(`SELECT stream_id,type,CASE WHEN type='session_error' THEN 'failed' ELSE 'completed' END state FROM event_log
|
|
104
|
+
WHERE session_id=? AND event_id=? AND type IN ('message_finished','session_error','message_steered') ORDER BY stream_id LIMIT 8`).all(sessionId, eventId)
|
|
105
|
+
.map(row => ({ streamId: row.stream_id, type: row.type, state: row.state }));
|
|
106
|
+
}
|
|
107
|
+
assertAdmissionUnblocked(sessionId, delivery, beforeStreamId = Number.MAX_SAFE_INTEGER) {
|
|
108
|
+
if (delivery !== "queue")
|
|
109
|
+
return;
|
|
110
|
+
const blocker = this.store.db.prepare("SELECT id,created_at FROM message_commands WHERE session_id=? AND state='interrupted' AND stream_id<? ORDER BY stream_id LIMIT 1").get(sessionId, beforeStreamId);
|
|
111
|
+
if (blocker)
|
|
112
|
+
throw domainError("command_reconciliation_required", "A previous interrupted message requires operator review before this session can accept more messages.", { retryable: false, scope: "session", blockingCommandId: blocker.id, blockedSince: blocker.created_at, oldestWaitAgeMs: Math.max(0, Date.now() - blocker.created_at), nextAction: `pibo debug message-queue inspect --session ${sessionId}` });
|
|
113
|
+
}
|
|
69
114
|
insert(input) {
|
|
70
115
|
const prior = this.find(input.key, input.fingerprint);
|
|
71
116
|
if (prior)
|
|
72
117
|
return prior;
|
|
118
|
+
this.assertAdmissionUnblocked(input.sessionId, input.delivery, input.streamId);
|
|
73
119
|
const limit = MESSAGE_COMMAND_LIMITS[input.delivery];
|
|
74
|
-
const rows = this.store.db.prepare(`SELECT session_id,
|
|
120
|
+
const rows = this.store.db.prepare(`SELECT c.session_id,c.room_id,c.payload_bytes,c.created_at,c.state,
|
|
121
|
+
NOT EXISTS(SELECT 1 FROM message_commands p WHERE p.session_id=c.session_id AND p.state='interrupted' AND p.stream_id<c.stream_id) dispatchable
|
|
122
|
+
FROM message_commands c WHERE c.state IN (${active}) AND c.delivery=? LIMIT ?`).all(input.delivery, limit.count + 1);
|
|
75
123
|
const now = Date.now();
|
|
76
124
|
const exceeds = (scope, count, bytes, ageMs) => scope.length >= count
|
|
77
125
|
|| scope.reduce((n, r) => n + r.payload_bytes, input.payload.byteSize) > bytes
|
|
78
|
-
|| scope.some(r => (r.state === "accepted" || r.state === "waiting_slot") && now - r.created_at >= ageMs);
|
|
126
|
+
|| scope.some(r => r.dispatchable === 1 && (r.state === "accepted" || r.state === "waiting_slot") && now - r.created_at >= ageMs);
|
|
79
127
|
if (exceeds(rows, limit.count, limit.bytes, limit.ageMs)
|
|
80
128
|
|| exceeds(rows.filter(r => r.room_id === input.roomId), limit.roomCount, limit.roomBytes, limit.roomAgeMs)
|
|
81
129
|
|| exceeds(rows.filter(r => r.session_id === input.sessionId), limit.sessionCount, limit.sessionBytes, limit.sessionAgeMs)) {
|
|
82
|
-
throw domainError("command_overloaded", "Durable message queue count, byte or wait-age capacity reached; retry the same transaction later.");
|
|
130
|
+
throw domainError("command_overloaded", "Durable message queue count, byte or dispatchable wait-age capacity reached; retry the same transaction later.", { retryable: true });
|
|
83
131
|
}
|
|
84
132
|
const id = `cmd_${randomUUID()}`;
|
|
85
133
|
const payload = this.store.payloads.commitPreparedPayload(input.payload);
|
|
86
134
|
this.store.db.prepare(`INSERT INTO message_commands (id,request_key,fingerprint,session_id,room_id,event_id,stream_id,payload_ref,payload_bytes,delivery,state,created_at,updated_at) VALUES (?,?,?,?,?,?,?,?,?,?,'accepted',?,?)`).run(id, input.key, input.fingerprint, input.sessionId, input.roomId, input.eventId, input.streamId, payload.id, payload.byteSize, input.delivery, now, now);
|
|
87
135
|
return this.get(id);
|
|
88
136
|
}
|
|
137
|
+
terminalizeBlockedSuccessors(now, limit = 100) {
|
|
138
|
+
const bounded = Math.max(1, Math.min(1000, Math.trunc(limit) || 1));
|
|
139
|
+
return Number(this.store.db.prepare(`UPDATE message_commands SET state='failed',error='Not dispatched: an earlier interrupted message requires reconciliation.',owner = NULL,lease_until=0,updated_at=?
|
|
140
|
+
WHERE id IN (SELECT c.id FROM message_commands c WHERE c.state IN (${unstarted}) AND c.delivery='queue' AND c.owner IS NULL AND EXISTS(
|
|
141
|
+
SELECT 1 FROM message_commands p WHERE p.session_id=c.session_id AND p.state='interrupted' AND p.stream_id<c.stream_id
|
|
142
|
+
) ORDER BY c.created_at,c.id LIMIT ?)`).run(now, bounded).changes);
|
|
143
|
+
}
|
|
144
|
+
/** Bounded, idempotent startup repair. Ambiguous rows cannot starve later evidence-backed rows. */
|
|
145
|
+
reconcileInterrupted(limit = 1000) {
|
|
146
|
+
const bounded = Math.max(1, Math.min(1000, Math.trunc(limit) || 1));
|
|
147
|
+
let rows = [];
|
|
148
|
+
try {
|
|
149
|
+
rows = this.store.db.prepare(`SELECT id FROM message_commands WHERE state='interrupted' AND ${terminalEvidenceSql} IN ('completed','failed') ORDER BY updated_at,id LIMIT ?`).all(bounded);
|
|
150
|
+
}
|
|
151
|
+
catch {
|
|
152
|
+
return { examined: 0, reconciled: 0, blockedSuccessorsFailed: 0, ambiguous: 0, errors: 1 };
|
|
153
|
+
}
|
|
154
|
+
let reconciled = 0, errors = 0;
|
|
155
|
+
for (const row of rows) {
|
|
156
|
+
try {
|
|
157
|
+
reconciled += this.store.transaction(() => Number(this.store.db.prepare(`UPDATE message_commands SET state=${terminalEvidenceSql},error=NULL,owner = NULL,lease_until=0,updated_at=? WHERE id=? AND state='interrupted' AND ${terminalEvidenceSql} IN ('completed','failed')`).run(Date.now(), row.id).changes));
|
|
158
|
+
}
|
|
159
|
+
catch {
|
|
160
|
+
errors++;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
let blockedSuccessorsFailed = 0;
|
|
164
|
+
try {
|
|
165
|
+
blockedSuccessorsFailed = this.store.transaction(() => this.terminalizeBlockedSuccessors(Date.now(), bounded));
|
|
166
|
+
}
|
|
167
|
+
catch {
|
|
168
|
+
errors++;
|
|
169
|
+
}
|
|
170
|
+
const ambiguous = Number(this.store.db.prepare("SELECT count(*) n FROM message_commands WHERE state='interrupted'").get().n);
|
|
171
|
+
return { examined: rows.length, reconciled, blockedSuccessorsFailed, ambiguous, errors };
|
|
172
|
+
}
|
|
173
|
+
recoverExpiredLeasesInTransaction(now, afterSelection) {
|
|
174
|
+
const rows = this.store.db.prepare("SELECT id FROM message_commands WHERE owner IS NOT NULL AND lease_until<=? ORDER BY lease_until,id LIMIT 100").all(now);
|
|
175
|
+
afterSelection?.();
|
|
176
|
+
let changed = 0;
|
|
177
|
+
for (const row of rows) {
|
|
178
|
+
changed += Number(this.store.db.prepare(`UPDATE message_commands SET
|
|
179
|
+
state=CASE WHEN ${terminalEvidenceSql} IN ('completed','failed') THEN ${terminalEvidenceSql} WHEN state='waiting_slot' THEN 'accepted' ELSE 'interrupted' END,
|
|
180
|
+
error=CASE WHEN ${terminalEvidenceSql} IN ('completed','failed') OR state='waiting_slot' THEN NULL ELSE 'Runtime ownership expired; execution requires reconciliation.' END,
|
|
181
|
+
owner = NULL,lease_until=0,updated_at=? WHERE id=? AND owner IS NOT NULL AND lease_until<=? AND state IN (${active})`).run(now, row.id, now).changes);
|
|
182
|
+
}
|
|
183
|
+
this.terminalizeBlockedSuccessors(now);
|
|
184
|
+
return changed;
|
|
185
|
+
}
|
|
186
|
+
/** Public for deterministic recovery tests; production callers use claim(). */
|
|
187
|
+
recoverExpiredLeases(now = Date.now(), barrier) {
|
|
188
|
+
return this.store.transaction(() => this.recoverExpiredLeasesInTransaction(now, barrier));
|
|
189
|
+
}
|
|
89
190
|
claim(owner, leaseMs) {
|
|
90
191
|
const limits = MESSAGE_COMMAND_LIMITS.dispatch;
|
|
91
192
|
const candidateSql = `WITH occupied AS (
|
|
@@ -96,13 +197,11 @@ export class MessageCommandStore {
|
|
|
96
197
|
AND (SELECT count(*) FROM occupied o WHERE o.delivery=c.delivery AND o.room_id=c.room_id) < CASE c.delivery WHEN 'steer' THEN ${limits.roomSteer} ELSE ${limits.roomQueue} END
|
|
97
198
|
AND NOT EXISTS (SELECT 1 FROM message_commands p WHERE p.session_id=c.session_id AND p.state IN (${active},'interrupted') AND p.stream_id<c.stream_id AND (c.delivery='queue' OR p.delivery='steer'))
|
|
98
199
|
ORDER BY CASE c.delivery WHEN 'steer' THEN 0 ELSE 1 END,COALESCE(r.sequence,0),c.stream_id LIMIT 1`;
|
|
99
|
-
// Idle polling must not take the SQLite writer lock. The claim transaction rechecks.
|
|
100
200
|
if (!this.store.db.prepare(candidateSql).get() && !this.store.db.prepare("SELECT 1 FROM message_commands WHERE owner IS NOT NULL AND lease_until <= ? LIMIT 1").get(Date.now()))
|
|
101
201
|
return undefined;
|
|
102
202
|
const row = this.store.transaction(() => {
|
|
103
203
|
const now = Date.now();
|
|
104
|
-
|
|
105
|
-
this.store.db.prepare(`UPDATE message_commands SET state=CASE WHEN state='waiting_slot' THEN 'accepted' ELSE 'interrupted' END, error=CASE WHEN state='waiting_slot' THEN NULL ELSE 'Runtime ownership expired; execution requires reconciliation.' END, owner = NULL,lease_until=0,updated_at=? WHERE id IN (SELECT id FROM message_commands WHERE owner IS NOT NULL AND lease_until <= ? ORDER BY lease_until LIMIT 100)`).run(now, now);
|
|
204
|
+
this.recoverExpiredLeasesInTransaction(now);
|
|
106
205
|
const candidate = this.store.db.prepare(candidateSql).get();
|
|
107
206
|
if (!candidate)
|
|
108
207
|
return undefined;
|
|
@@ -126,7 +225,7 @@ export class MessageCommandStore {
|
|
|
126
225
|
return Number(this.store.db.prepare("UPDATE message_commands SET state='failed',error='Cancelled before runtime dispatch.',owner = NULL,lease_until=0,updated_at=? WHERE session_id=? AND state IN ('accepted','waiting_slot')").run(Date.now(), sessionId).changes);
|
|
127
226
|
}
|
|
128
227
|
transition(id, owner, token, state, error) {
|
|
129
|
-
return Number(this.store.db.prepare(`UPDATE message_commands SET state=?,error=?,updated_at=?,owner = CASE WHEN ? IN ('completed','failed','interrupted') THEN NULL ELSE owner END WHERE id=? AND owner = ? AND token=? AND lease_until>? AND state IN (${active})`).run(state, error?.slice(0, 500) ?? null, Date.now(), state, id, owner, token, Date.now()).changes) === 1;
|
|
228
|
+
return Number(this.store.db.prepare(`UPDATE message_commands SET state=?,error=?,updated_at=?,owner = CASE WHEN ? IN ('completed','failed','interrupted') THEN NULL ELSE owner END,lease_until=CASE WHEN ? IN ('completed','failed','interrupted') THEN 0 ELSE lease_until END WHERE id=? AND owner = ? AND token=? AND lease_until>? AND state IN (${active})`).run(state, error?.slice(0, 500) ?? null, Date.now(), state, state, id, owner, token, Date.now()).changes) === 1;
|
|
130
229
|
}
|
|
131
230
|
heartbeat(id, owner, token, leaseMs) {
|
|
132
231
|
return Number(this.store.db.prepare(`UPDATE message_commands SET lease_until=? WHERE id=? AND owner = ? AND token=? AND lease_until>? AND state IN (${active})`).run(Date.now() + leaseMs, id, owner, token, Date.now()).changes) === 1;
|
|
@@ -139,10 +238,53 @@ export class MessageCommandStore {
|
|
|
139
238
|
if (!state)
|
|
140
239
|
return;
|
|
141
240
|
const eligible = state === "session_queue" ? "'initializing','waiting_slot'" : state === "running" ? "'initializing','waiting_slot','session_queue'" : `${active},'interrupted'`;
|
|
142
|
-
this.store.db.prepare(`UPDATE message_commands SET state=?,error=NULL,updated_at=?,owner = CASE WHEN ? IN ('completed','failed') THEN NULL ELSE owner END WHERE session_id=? AND event_id=? AND state IN (${eligible})`).run(state, Date.now(), state, sessionId, eventId);
|
|
241
|
+
this.store.db.prepare(`UPDATE message_commands SET state=?,error=NULL,updated_at=?,owner = CASE WHEN ? IN ('completed','failed') THEN NULL ELSE owner END,lease_until=CASE WHEN ? IN ('completed','failed') THEN 0 ELSE lease_until END WHERE session_id=? AND event_id=? AND state IN (${eligible})`).run(state, Date.now(), state, state, sessionId, eventId);
|
|
242
|
+
}
|
|
243
|
+
health(now = Date.now(), scopeLimit = 20) {
|
|
244
|
+
const bounded = Math.max(1, Math.min(100, Math.trunc(scopeLimit) || 20));
|
|
245
|
+
const activeRowsLimit = MESSAGE_COMMAND_LIMITS.queue.count + MESSAGE_COMMAND_LIMITS.steer.count + 1;
|
|
246
|
+
const counts = this.store.db.prepare("SELECT state,delivery,command_count count,payload_bytes bytes FROM message_command_stats WHERE command_count>0 ORDER BY state,delivery").all();
|
|
247
|
+
const interrupted = Number(this.store.db.prepare("SELECT count(*) n FROM message_commands WHERE state='interrupted'").get().n);
|
|
248
|
+
const expired = Number(this.store.db.prepare("SELECT count(*) n FROM message_commands WHERE owner IS NOT NULL AND lease_until<=?").get(now).n);
|
|
249
|
+
const blockedRows = this.store.db.prepare(`SELECT c.session_id,c.room_id,c.created_at,p.id blocking_id,p.created_at blocked_since
|
|
250
|
+
FROM message_commands c JOIN message_commands p ON p.id=(SELECT p2.id FROM message_commands p2 WHERE p2.session_id=c.session_id AND p2.state='interrupted' AND p2.stream_id<c.stream_id ORDER BY p2.stream_id LIMIT 1)
|
|
251
|
+
WHERE c.state IN (${unstarted}) AND c.delivery='queue' ORDER BY c.created_at,c.id LIMIT ?`).all(bounded);
|
|
252
|
+
const blockedCount = Number(this.store.db.prepare(`SELECT count(*) n FROM message_commands c WHERE c.state IN (${unstarted}) AND c.delivery='queue' AND EXISTS(SELECT 1 FROM message_commands p WHERE p.session_id=c.session_id AND p.state='interrupted' AND p.stream_id<c.stream_id)`).get().n);
|
|
253
|
+
const oldestDispatchable = this.store.db.prepare(`SELECT created_at FROM message_commands c WHERE c.state IN (${unstarted}) AND NOT EXISTS(SELECT 1 FROM message_commands p WHERE p.session_id=c.session_id AND p.state='interrupted' AND p.stream_id<c.stream_id) ORDER BY created_at LIMIT 1`).get();
|
|
254
|
+
const scopes = new Map();
|
|
255
|
+
for (const row of blockedRows) {
|
|
256
|
+
const prior = scopes.get(row.session_id);
|
|
257
|
+
if (prior)
|
|
258
|
+
prior.blockedSuccessors++;
|
|
259
|
+
else
|
|
260
|
+
scopes.set(row.session_id, { sessionId: row.session_id, roomId: row.room_id, blockingCommandId: row.blocking_id, blockedSince: row.blocked_since, blockedSuccessors: 1 });
|
|
261
|
+
}
|
|
262
|
+
if (interrupted > 0) {
|
|
263
|
+
for (const row of this.store.db.prepare("SELECT session_id,room_id,id,created_at FROM message_commands WHERE state='interrupted' ORDER BY created_at LIMIT ?").all(bounded)) {
|
|
264
|
+
if (!scopes.has(row.session_id))
|
|
265
|
+
scopes.set(row.session_id, { sessionId: row.session_id, roomId: row.room_id, blockingCommandId: row.id, blockedSince: row.created_at, blockedSuccessors: 0 });
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
const activeRows = this.store.db.prepare(`SELECT session_id,room_id,delivery,payload_bytes,created_at,state,NOT EXISTS(SELECT 1 FROM message_commands p WHERE p.session_id=c.session_id AND p.state='interrupted' AND p.stream_id<c.stream_id) dispatchable FROM message_commands c WHERE state IN (${active}) ORDER BY created_at,id LIMIT ?`).all(activeRowsLimit);
|
|
269
|
+
const capacityReasons = [];
|
|
270
|
+
const activeStates = new Set(["accepted", "waiting_slot", "initializing", "session_queue", "running"]);
|
|
271
|
+
const globalTotals = (delivery) => counts.filter(row => row.delivery === delivery && activeStates.has(row.state)).reduce((total, row) => ({ count: total.count + row.count, bytes: total.bytes + row.bytes }), { count: 0, bytes: 0 });
|
|
272
|
+
const summarizeCapacity = (rows, delivery, scope) => { const selected = rows.filter(row => row.delivery === delivery), limits = MESSAGE_COMMAND_LIMITS[delivery], countLimit = scope === "global" ? limits.count : scope === "room" ? limits.roomCount : limits.sessionCount, byteLimit = scope === "global" ? limits.bytes : scope === "room" ? limits.roomBytes : limits.sessionBytes, ageLimit = scope === "global" ? limits.ageMs : scope === "room" ? limits.roomAgeMs : limits.sessionAgeMs, totals = scope === "global" ? globalTotals(delivery) : { count: selected.length, bytes: selected.reduce((sum, row) => sum + row.payload_bytes, 0) }, count = totals.count, bytes = totals.bytes, oldestDispatchableWaitMs = Math.max(0, ...selected.filter(row => row.dispatchable === 1 && (row.state === "accepted" || row.state === "waiting_slot")).map(row => now - row.created_at)), reasons = [...(count >= countLimit ? [`count ${count}/${countLimit}`] : []), ...(bytes >= byteLimit ? [`bytes ${bytes}/${byteLimit}`] : []), ...(oldestDispatchableWaitMs >= ageLimit ? [`dispatchable wait ${oldestDispatchableWaitMs}ms/${ageLimit}ms`] : [])]; return { delivery, count, bytes, oldestDispatchableWaitMs, available: reasons.length === 0, degradedReasons: reasons }; };
|
|
273
|
+
const globalCapacity = ["queue", "steer"].map(delivery => summarizeCapacity(activeRows, delivery, "global"));
|
|
274
|
+
const roomKeys = [...new Set(activeRows.map(row => row.room_id))], sessionKeys = [...new Set(activeRows.map(row => row.session_id))];
|
|
275
|
+
const roomCapacity = roomKeys.flatMap(roomId => ["queue", "steer"].map(delivery => ({ roomId, ...summarizeCapacity(activeRows.filter(row => row.room_id === roomId), delivery, "room") }))).filter(row => !row.available).slice(0, bounded);
|
|
276
|
+
const sessionCapacity = sessionKeys.flatMap(sessionId => ["queue", "steer"].map(delivery => ({ sessionId, roomId: activeRows.find(row => row.session_id === sessionId)?.room_id ?? "unknown", ...summarizeCapacity(activeRows.filter(row => row.session_id === sessionId), delivery, "session") }))).filter(row => !row.available).slice(0, bounded);
|
|
277
|
+
for (const row of globalCapacity.filter(row => !row.available))
|
|
278
|
+
capacityReasons.push(`global ${row.delivery} admission degraded: ${row.degradedReasons.join(", ")}`);
|
|
279
|
+
for (const row of roomCapacity)
|
|
280
|
+
capacityReasons.push(`room ${row.roomId} ${row.delivery} admission degraded: ${row.degradedReasons.join(", ")}`);
|
|
281
|
+
for (const row of sessionCapacity)
|
|
282
|
+
capacityReasons.push(`session ${row.sessionId} ${row.delivery} admission degraded: ${row.degradedReasons.join(", ")}`);
|
|
283
|
+
const degradedReasons = [...(interrupted ? [`${interrupted} interrupted predecessor(s) require reconciliation`] : []), ...(blockedCount ? [`${blockedCount} successor(s) are blocked by FIFO`] : []), ...(expired ? [`${expired} owned lease(s) are expired`] : []), ...capacityReasons];
|
|
284
|
+
return { status: degradedReasons.length ? "degraded" : "healthy", storage: { available: true }, counts, interruptedPredecessors: interrupted, blockedSuccessors: blockedCount, expiredOwnedLeases: expired, oldestDispatchableWaitMs: oldestDispatchable ? Math.max(0, now - oldestDispatchable.created_at) : 0, oldestBlockedWaitMs: blockedRows.length ? Math.max(...blockedRows.map(row => Math.max(0, now - row.created_at))) : 0, affectedScopes: [...scopes.values()].slice(0, bounded), limits: MESSAGE_COMMAND_LIMITS, degradedReasons, admissionCapacity: { global: globalCapacity, rooms: roomCapacity, sessions: sessionCapacity }, bounded: { scopeLimit: bounded, activeRowsLimit, activeRowsTruncated: activeRows.length === activeRowsLimit, affectedScopesTruncated: interrupted > bounded || scopes.size > bounded || blockedCount > blockedRows.length } };
|
|
143
285
|
}
|
|
144
286
|
}
|
|
145
287
|
function receipt(row) {
|
|
146
288
|
return { id: row.id, sessionId: row.session_id, roomId: row.room_id, eventId: row.event_id, streamId: row.stream_id, state: row.state, createdAt: row.created_at, updatedAt: row.updated_at, ...(row.error ? { error: row.error } : {}) };
|
|
147
289
|
}
|
|
148
|
-
function domainError(code, message) { return Object.assign(new Error(message), { code }); }
|
|
290
|
+
function domainError(code, message, details = {}) { return Object.assign(new Error(message), { code, ...details }); }
|
package/dist/data/schema.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { CHAT_READ_PROJECTION_SCHEMA, CHAT_NAVIGATION_REVISION_SCHEMA } from "./chat-read-projections.js";
|
|
2
2
|
import { TELEMETRY_MAINTENANCE_SCHEMA } from "./telemetry-maintenance.js";
|
|
3
3
|
import { MESSAGE_COMMAND_SCHEMA } from "./message-command-store.js";
|
|
4
|
-
export const PIBO_DATA_SCHEMA_VERSION =
|
|
4
|
+
export const PIBO_DATA_SCHEMA_VERSION = 14;
|
|
5
5
|
const NATIVE_HISTORY_FALLBACK_SCHEMA_VERSION = 5;
|
|
6
|
+
const TERMINAL_MESSAGE_UNREAD_SCHEMA_VERSION = 14;
|
|
6
7
|
const retiredScopeColumn = ["owner", "scope"].join("_");
|
|
7
8
|
const payloadTableDefinition = `
|
|
8
9
|
id TEXT PRIMARY KEY,
|
|
@@ -201,6 +202,14 @@ function applyPiboDataSchemaInTransaction(db, hooks, previousVersion, tablesToRe
|
|
|
201
202
|
if (rebuildPayloadTable)
|
|
202
203
|
rebuildPayloadTableForMetadataIdentity(db);
|
|
203
204
|
hooks.afterStep?.("payload-metadata-identity");
|
|
205
|
+
if (previousVersion < TERMINAL_MESSAGE_UNREAD_SCHEMA_VERSION) {
|
|
206
|
+
db.exec(`
|
|
207
|
+
DROP TRIGGER IF EXISTS chat_unread_event_insert;
|
|
208
|
+
DROP TRIGGER IF EXISTS chat_unread_event_update;
|
|
209
|
+
DROP INDEX IF EXISTS idx_event_log_unread_stream;
|
|
210
|
+
DROP INDEX IF EXISTS idx_event_log_unread_session_stream;
|
|
211
|
+
`);
|
|
212
|
+
}
|
|
204
213
|
db.exec(`
|
|
205
214
|
CREATE TABLE IF NOT EXISTS sessions (
|
|
206
215
|
id TEXT PRIMARY KEY,
|
|
@@ -663,8 +672,10 @@ function applyPiboDataSchemaInTransaction(db, hooks, previousVersion, tablesToRe
|
|
|
663
672
|
ON event_log(session_id, stream_id);
|
|
664
673
|
CREATE INDEX IF NOT EXISTS idx_event_log_unread_session_stream
|
|
665
674
|
ON event_log(session_id, stream_id)
|
|
666
|
-
WHERE
|
|
667
|
-
|
|
675
|
+
WHERE type = 'message_finished';
|
|
676
|
+
CREATE INDEX IF NOT EXISTS idx_event_log_unread_error_session_stream
|
|
677
|
+
ON event_log(session_id, stream_id)
|
|
678
|
+
WHERE type = 'session_error';
|
|
668
679
|
CREATE INDEX IF NOT EXISTS idx_event_log_session_sequence_stream
|
|
669
680
|
ON event_log(session_id, session_sequence DESC, stream_id DESC);
|
|
670
681
|
CREATE INDEX IF NOT EXISTS idx_event_log_session_type_sequence_stream
|
|
@@ -769,6 +780,16 @@ function applyPiboDataSchemaInTransaction(db, hooks, previousVersion, tablesToRe
|
|
|
769
780
|
ON telemetry_tool_calls(retention_class, updated_at);
|
|
770
781
|
`);
|
|
771
782
|
db.exec(CHAT_READ_PROJECTION_SCHEMA);
|
|
783
|
+
if (previousVersion < TERMINAL_MESSAGE_UNREAD_SCHEMA_VERSION) {
|
|
784
|
+
db.exec(`
|
|
785
|
+
DELETE FROM chat_unread_counts;
|
|
786
|
+
DELETE FROM chat_unread_index;
|
|
787
|
+
UPDATE chat_read_backfill
|
|
788
|
+
SET event_cursor = 0,
|
|
789
|
+
event_target = (SELECT COALESCE(MAX(stream_id), 0) FROM event_log)
|
|
790
|
+
WHERE id = 1;
|
|
791
|
+
`);
|
|
792
|
+
}
|
|
772
793
|
db.exec(CHAT_NAVIGATION_REVISION_SCHEMA);
|
|
773
794
|
db.exec(TELEMETRY_MAINTENANCE_SCHEMA);
|
|
774
795
|
hooks.afterStep?.("schema");
|