@yeaft/webchat-agent 1.0.295 → 1.0.298
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.js +1 -2
- package/local-runtime/server/db/connection.js +13 -0
- package/local-runtime/server/db/session-db.js +3 -0
- package/local-runtime/server/db/yeaft-session-db.js +9 -0
- package/local-runtime/server/handlers/agent-output.js +27 -0
- package/local-runtime/server/handlers/client-conversation.js +2 -2
- package/local-runtime/server/handlers/client-work-center.js +5 -4
- package/local-runtime/server/session-catalog.js +4 -7
- package/local-runtime/version.json +1 -1
- package/local-runtime/web/app.bundle.js +156 -169
- package/local-runtime/web/app.bundle.js.gz +0 -0
- package/local-runtime/web/index.html +2 -2
- package/local-runtime/web/style.bundle.css +1 -1
- package/local-runtime/web/style.bundle.css.gz +0 -0
- package/package.json +1 -1
- package/yeaft/engine.js +30 -3
- package/yeaft/sessions/session-crud.js +9 -6
- package/yeaft/sessions/session-store.js +1 -0
- package/yeaft/tools/file-edit.js +8 -6
- package/yeaft/tools/file-write.js +2 -2
- package/yeaft/tools/registry.js +28 -10
- package/yeaft/tools/types.js +4 -0
- package/yeaft/web-bridge.js +10 -2
- package/yeaft/work-center/bridge.js +5 -1
- package/yeaft/work-center/controller.js +6 -2
- package/yeaft/work-center/coordinator.js +84 -17
- package/yeaft/work-center/durable-model.js +674 -0
- package/yeaft/work-center/projection.js +10 -1
- package/yeaft/work-center/runner.js +111 -16
- package/yeaft/work-center/service.js +102 -4
- package/yeaft/work-center/store.js +963 -23
|
@@ -6,8 +6,13 @@ import { normalizeEvidence } from './evidence.js';
|
|
|
6
6
|
import { normalizeActionCheckpoint } from './action-checkpoint.js';
|
|
7
7
|
import { currentActionInputEventIds, runMatchesActionIdentity } from './action-identity.js';
|
|
8
8
|
import { canonicalActionInstruction, withoutActionInputContext } from './workflow.js';
|
|
9
|
+
import {
|
|
10
|
+
WORK_CENTER_SCHEMA_VERSION,
|
|
11
|
+
durableId,
|
|
12
|
+
migrateDurableWorkCenterModel,
|
|
13
|
+
} from './durable-model.js';
|
|
9
14
|
|
|
10
|
-
const SCHEMA_VERSION =
|
|
15
|
+
const SCHEMA_VERSION = WORK_CENTER_SCHEMA_VERSION;
|
|
11
16
|
const UNFINISHED_ACTION_STATUSES = "'ready','running','waiting','failed'";
|
|
12
17
|
const MAX_REUSABLE_CONTEXT_ITEMS = 12;
|
|
13
18
|
const MAX_RUN_RESPONSE_CHARS = 65_536;
|
|
@@ -227,6 +232,9 @@ function mapRun(row) {
|
|
|
227
232
|
workItemId: row.work_item_id,
|
|
228
233
|
ownerBootId: row.owner_boot_id,
|
|
229
234
|
leaseEpoch: row.lease_epoch,
|
|
235
|
+
ordinal: Math.max(1, Number(row.ordinal) || 1),
|
|
236
|
+
terminalStatus: row.terminal_status || null,
|
|
237
|
+
terminalAt: row.terminal_at || null,
|
|
230
238
|
status: row.status,
|
|
231
239
|
startedAt: row.started_at,
|
|
232
240
|
expiresAt: row.expires_at,
|
|
@@ -774,6 +782,10 @@ export class WorkItemStore {
|
|
|
774
782
|
this.db.exec('PRAGMA synchronous = NORMAL;');
|
|
775
783
|
this.db.exec('PRAGMA foreign_keys = ON;');
|
|
776
784
|
this.#initSchema();
|
|
785
|
+
this.recoverCoordinatorMailbox();
|
|
786
|
+
this.recoverCoordinatorProviderTurns();
|
|
787
|
+
this.recoverOperations();
|
|
788
|
+
this.recoverEngineTurns();
|
|
777
789
|
this.recoverInterruptedCoordinatorTurns();
|
|
778
790
|
}
|
|
779
791
|
|
|
@@ -939,6 +951,9 @@ export class WorkItemStore {
|
|
|
939
951
|
const storedSchemaVersion = Number(
|
|
940
952
|
this.db.prepare("SELECT value FROM schema_meta WHERE key = 'schema_version'").get()?.value,
|
|
941
953
|
) || 0;
|
|
954
|
+
if (storedSchemaVersion > SCHEMA_VERSION) {
|
|
955
|
+
throw new Error(`Work Center database schema ${storedSchemaVersion} is newer than supported schema ${SCHEMA_VERSION}`);
|
|
956
|
+
}
|
|
942
957
|
|
|
943
958
|
// The feature shipped first as an unmerged PR, but keep the store tolerant
|
|
944
959
|
// of databases created by review builds.
|
|
@@ -1129,16 +1144,520 @@ export class WorkItemStore {
|
|
|
1129
1144
|
}
|
|
1130
1145
|
if (storedSchemaVersion < 20) reconcilePendingActionInputIdentity(this.db, this.now());
|
|
1131
1146
|
if (storedSchemaVersion < 22) repairReviewBuildActionInputIdentity(this.db, this.now());
|
|
1132
|
-
this.db.prepare(`INSERT INTO schema_meta(key, value) VALUES('schema_version', ?)
|
|
1133
|
-
ON CONFLICT(key) DO UPDATE SET value = excluded.value`).run(String(SCHEMA_VERSION));
|
|
1134
1147
|
});
|
|
1135
1148
|
}
|
|
1149
|
+
withTransaction(this.db, () => {
|
|
1150
|
+
migrateDurableWorkCenterModel(this.db, this.now(), storedSchemaVersion || SCHEMA_VERSION);
|
|
1151
|
+
this.db.prepare(`INSERT INTO schema_meta(key, value) VALUES('schema_version', ?)
|
|
1152
|
+
ON CONFLICT(key) DO UPDATE SET value = excluded.value`).run(String(SCHEMA_VERSION));
|
|
1153
|
+
});
|
|
1136
1154
|
}
|
|
1137
1155
|
|
|
1138
1156
|
close() {
|
|
1139
1157
|
this.db.close();
|
|
1140
1158
|
}
|
|
1141
1159
|
|
|
1160
|
+
#nextConversationSequence(conversationId) {
|
|
1161
|
+
return Number(this.db.prepare(`SELECT COALESCE(MAX(sequence), 0) + 1 AS value
|
|
1162
|
+
FROM conversation_entries WHERE conversation_id = ?`).get(conversationId)?.value) || 1;
|
|
1163
|
+
}
|
|
1164
|
+
|
|
1165
|
+
#nextActionEntrySequence(actionId) {
|
|
1166
|
+
return Number(this.db.prepare(`SELECT COALESCE(MAX(sequence), 0) + 1 AS value
|
|
1167
|
+
FROM action_entries WHERE action_id = ?`).get(actionId)?.value) || 1;
|
|
1168
|
+
}
|
|
1169
|
+
|
|
1170
|
+
#appendConversationEntry(workItemId, entry, sourceKey) {
|
|
1171
|
+
const now = this.now();
|
|
1172
|
+
const conversationId = `work-item:${workItemId}`;
|
|
1173
|
+
this.db.prepare(`INSERT INTO conversations
|
|
1174
|
+
(id, work_item_id, status, created_at, updated_at) VALUES (?, ?, 'active', ?, ?)
|
|
1175
|
+
ON CONFLICT(work_item_id) DO UPDATE SET updated_at = excluded.updated_at`).run(
|
|
1176
|
+
conversationId, workItemId, now, now,
|
|
1177
|
+
);
|
|
1178
|
+
const existing = this.db.prepare('SELECT id FROM conversation_entries WHERE source_key = ?').get(sourceKey);
|
|
1179
|
+
if (existing) {
|
|
1180
|
+
this.db.prepare(`UPDATE conversation_entries SET status = ?, text = ?, attachments = ?,
|
|
1181
|
+
payload = ?, updated_at = ? WHERE id = ?`).run(
|
|
1182
|
+
entry.status || 'completed',
|
|
1183
|
+
entry.text || '',
|
|
1184
|
+
stringify(Array.isArray(entry.attachments) ? entry.attachments : []),
|
|
1185
|
+
stringify(entry),
|
|
1186
|
+
Number(entry.updatedAt) || Number(entry.createdAt) || now,
|
|
1187
|
+
existing.id,
|
|
1188
|
+
);
|
|
1189
|
+
return existing.id;
|
|
1190
|
+
}
|
|
1191
|
+
const id = entry.id || durableId('conversation-entry');
|
|
1192
|
+
this.db.prepare(`INSERT INTO conversation_entries
|
|
1193
|
+
(id, conversation_id, work_item_id, sequence, kind, role, status, text, attachments,
|
|
1194
|
+
turn_id, source_key, payload, created_at, updated_at)
|
|
1195
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(
|
|
1196
|
+
id,
|
|
1197
|
+
conversationId,
|
|
1198
|
+
workItemId,
|
|
1199
|
+
this.#nextConversationSequence(conversationId),
|
|
1200
|
+
entry.kind || 'message',
|
|
1201
|
+
entry.role || null,
|
|
1202
|
+
entry.status || 'completed',
|
|
1203
|
+
entry.text || '',
|
|
1204
|
+
stringify(Array.isArray(entry.attachments) ? entry.attachments : []),
|
|
1205
|
+
entry.turnId || null,
|
|
1206
|
+
sourceKey,
|
|
1207
|
+
stringify(entry),
|
|
1208
|
+
Number(entry.createdAt) || now,
|
|
1209
|
+
Number(entry.updatedAt) || Number(entry.createdAt) || now,
|
|
1210
|
+
);
|
|
1211
|
+
return id;
|
|
1212
|
+
}
|
|
1213
|
+
|
|
1214
|
+
#appendActionEntry(input, sourceKey) {
|
|
1215
|
+
const append = () => {
|
|
1216
|
+
const existing = this.db.prepare('SELECT * FROM action_entries WHERE source_key = ?').get(sourceKey);
|
|
1217
|
+
if (existing) return existing;
|
|
1218
|
+
const now = this.now();
|
|
1219
|
+
const status = input.status || 'pending';
|
|
1220
|
+
const id = input.id || durableId('action-entry');
|
|
1221
|
+
this.db.prepare(`INSERT INTO action_entries
|
|
1222
|
+
(id, work_item_id, action_id, run_id, sequence, kind, role, status, text, attachments,
|
|
1223
|
+
source_key, payload, created_at, updated_at, consumed_at)
|
|
1224
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(
|
|
1225
|
+
id,
|
|
1226
|
+
input.workItemId,
|
|
1227
|
+
input.actionId,
|
|
1228
|
+
input.runId || null,
|
|
1229
|
+
this.#nextActionEntrySequence(input.actionId),
|
|
1230
|
+
input.kind || 'message',
|
|
1231
|
+
input.role || null,
|
|
1232
|
+
status,
|
|
1233
|
+
input.text || '',
|
|
1234
|
+
stringify(Array.isArray(input.attachments) ? input.attachments : []),
|
|
1235
|
+
sourceKey,
|
|
1236
|
+
stringify(input.payload || {}),
|
|
1237
|
+
Number(input.createdAt) || now,
|
|
1238
|
+
now,
|
|
1239
|
+
status === 'consumed' ? now : null,
|
|
1240
|
+
);
|
|
1241
|
+
return this.db.prepare('SELECT * FROM action_entries WHERE id = ?').get(id);
|
|
1242
|
+
};
|
|
1243
|
+
return this.db.isTransaction ? append() : withTransaction(this.db, append);
|
|
1244
|
+
}
|
|
1245
|
+
|
|
1246
|
+
appendActionControl(workItemId, actionId, command, options = {}) {
|
|
1247
|
+
if (!['start', 'pause', 'stop'].includes(command)) {
|
|
1248
|
+
throw new Error(`Unsupported Action control command: ${command}`);
|
|
1249
|
+
}
|
|
1250
|
+
const action = this.getAction(actionId);
|
|
1251
|
+
if (!action || action.workItemId !== workItemId) return null;
|
|
1252
|
+
const sourceKey = options.sourceKey || durableId(`action-control-${command}`);
|
|
1253
|
+
return this.#appendActionEntry({
|
|
1254
|
+
workItemId,
|
|
1255
|
+
actionId,
|
|
1256
|
+
runId: options.runId || action.currentRunId || null,
|
|
1257
|
+
kind: 'control',
|
|
1258
|
+
role: options.source || 'user',
|
|
1259
|
+
status: options.status || 'pending',
|
|
1260
|
+
text: options.reason || '',
|
|
1261
|
+
payload: { command, reason: options.reason || '', source: options.source || 'user' },
|
|
1262
|
+
createdAt: options.createdAt,
|
|
1263
|
+
}, sourceKey);
|
|
1264
|
+
}
|
|
1265
|
+
|
|
1266
|
+
enqueueCoordinatorMailbox(workItemId, kind, payload = {}, sourceKey = durableId('mailbox-source')) {
|
|
1267
|
+
const enqueue = () => {
|
|
1268
|
+
const existing = this.db.prepare(`SELECT * FROM coordinator_mailbox_entries
|
|
1269
|
+
WHERE source_key = ?`).get(sourceKey);
|
|
1270
|
+
if (existing) return existing;
|
|
1271
|
+
const now = this.now();
|
|
1272
|
+
const sequence = Number(this.db.prepare(`SELECT COALESCE(MAX(sequence), 0) + 1 AS value
|
|
1273
|
+
FROM coordinator_mailbox_entries WHERE work_item_id = ?`).get(workItemId)?.value) || 1;
|
|
1274
|
+
const id = durableId('coordinator-mailbox');
|
|
1275
|
+
this.db.prepare(`INSERT INTO coordinator_mailbox_entries
|
|
1276
|
+
(id, work_item_id, sequence, kind, status, source_key, payload, created_at, updated_at)
|
|
1277
|
+
VALUES (?, ?, ?, ?, 'pending', ?, ?, ?, ?)`).run(
|
|
1278
|
+
id, workItemId, sequence, kind, sourceKey, stringify(payload), now, now,
|
|
1279
|
+
);
|
|
1280
|
+
return this.db.prepare('SELECT * FROM coordinator_mailbox_entries WHERE id = ?').get(id);
|
|
1281
|
+
};
|
|
1282
|
+
return this.db.isTransaction ? enqueue() : withTransaction(this.db, enqueue);
|
|
1283
|
+
}
|
|
1284
|
+
|
|
1285
|
+
claimCoordinatorTurn(workItemId, turnId, owner, leaseMs = 60_000) {
|
|
1286
|
+
return withTransaction(this.db, () => {
|
|
1287
|
+
const now = this.now();
|
|
1288
|
+
const row = this.db.prepare(`SELECT * FROM coordinator_mailbox_entries
|
|
1289
|
+
WHERE work_item_id = ? AND json_extract(payload, '$.turnId') = ?
|
|
1290
|
+
AND (status = 'pending' OR (status = 'claimed' AND lease_expires_at <= ?))`).get(
|
|
1291
|
+
workItemId, turnId, now,
|
|
1292
|
+
);
|
|
1293
|
+
if (!row) return null;
|
|
1294
|
+
const changed = this.db.prepare(`UPDATE coordinator_mailbox_entries SET status = 'claimed',
|
|
1295
|
+
claim_owner = ?, claim_epoch = claim_epoch + 1, claimed_at = ?, lease_expires_at = ?,
|
|
1296
|
+
updated_at = ? WHERE id = ? AND claim_epoch = ?
|
|
1297
|
+
AND (status = 'pending' OR (status = 'claimed' AND lease_expires_at <= ?))`).run(
|
|
1298
|
+
owner, now, now + leaseMs, now, row.id, row.claim_epoch, now,
|
|
1299
|
+
);
|
|
1300
|
+
if (Number(changed.changes) !== 1) return null;
|
|
1301
|
+
const claimed = this.db.prepare('SELECT * FROM coordinator_mailbox_entries WHERE id = ?').get(row.id);
|
|
1302
|
+
const providerChanged = this.db.prepare(`UPDATE coordinator_provider_turns SET claim_owner = ?,
|
|
1303
|
+
claim_epoch = ?, updated_at = ? WHERE coordinator_turn_id = ?
|
|
1304
|
+
AND status IN ('prepared', 'responded')`).run(
|
|
1305
|
+
owner, claimed.claim_epoch, now, turnId,
|
|
1306
|
+
);
|
|
1307
|
+
const providerCount = Number(this.db.prepare(`SELECT COUNT(*) AS count FROM coordinator_provider_turns
|
|
1308
|
+
WHERE coordinator_turn_id = ? AND status IN ('prepared', 'responded')`).get(turnId)?.count) || 0;
|
|
1309
|
+
if (providerCount !== Number(providerChanged.changes)) {
|
|
1310
|
+
throw new Error('Coordinator provider turn cannot transfer an in-flight dispatch');
|
|
1311
|
+
}
|
|
1312
|
+
return {
|
|
1313
|
+
mailboxId: claimed.id,
|
|
1314
|
+
ownerBootId: owner,
|
|
1315
|
+
claimEpoch: Number(claimed.claim_epoch),
|
|
1316
|
+
leaseExpiresAt: Number(claimed.lease_expires_at),
|
|
1317
|
+
payload: parseJson(claimed.payload, {}),
|
|
1318
|
+
};
|
|
1319
|
+
});
|
|
1320
|
+
}
|
|
1321
|
+
|
|
1322
|
+
claimCoordinatorMailbox(workItemId, owner, leaseMs = 60_000) {
|
|
1323
|
+
return withTransaction(this.db, () => {
|
|
1324
|
+
const now = this.now();
|
|
1325
|
+
const row = this.db.prepare(`SELECT * FROM coordinator_mailbox_entries
|
|
1326
|
+
WHERE work_item_id = ? AND (status = 'pending'
|
|
1327
|
+
OR (status = 'claimed' AND lease_expires_at <= ?))
|
|
1328
|
+
ORDER BY sequence LIMIT 1`).get(workItemId, now);
|
|
1329
|
+
if (!row) return null;
|
|
1330
|
+
const changed = this.db.prepare(`UPDATE coordinator_mailbox_entries SET status = 'claimed',
|
|
1331
|
+
claim_owner = ?, claim_epoch = claim_epoch + 1, claimed_at = ?, lease_expires_at = ?,
|
|
1332
|
+
updated_at = ? WHERE id = ? AND claim_epoch = ?`).run(
|
|
1333
|
+
owner, now, now + leaseMs, now, row.id, row.claim_epoch,
|
|
1334
|
+
);
|
|
1335
|
+
if (Number(changed.changes) !== 1) return null;
|
|
1336
|
+
const claimed = this.db.prepare('SELECT * FROM coordinator_mailbox_entries WHERE id = ?').get(row.id);
|
|
1337
|
+
return { ...claimed, payload: parseJson(claimed.payload, {}) };
|
|
1338
|
+
});
|
|
1339
|
+
}
|
|
1340
|
+
|
|
1341
|
+
renewCoordinatorMailbox(id, owner, claimEpoch, leaseMs = 60_000) {
|
|
1342
|
+
const now = this.now();
|
|
1343
|
+
const changed = this.db.prepare(`UPDATE coordinator_mailbox_entries SET lease_expires_at = ?,
|
|
1344
|
+
updated_at = ? WHERE id = ? AND status = 'claimed' AND claim_owner = ? AND claim_epoch = ?
|
|
1345
|
+
AND lease_expires_at > ?`).run(now + leaseMs, now, id, owner, claimEpoch, now);
|
|
1346
|
+
return Number(changed.changes) === 1;
|
|
1347
|
+
}
|
|
1348
|
+
|
|
1349
|
+
ackCoordinatorMailbox(id, owner, claimEpoch) {
|
|
1350
|
+
const now = this.now();
|
|
1351
|
+
const changed = this.db.prepare(`UPDATE coordinator_mailbox_entries SET status = 'acked',
|
|
1352
|
+
acked_at = ?, updated_at = ? WHERE id = ? AND status = 'claimed' AND claim_owner = ?
|
|
1353
|
+
AND claim_epoch = ?`).run(now, now, id, owner, claimEpoch);
|
|
1354
|
+
return Number(changed.changes) === 1;
|
|
1355
|
+
}
|
|
1356
|
+
|
|
1357
|
+
recoverCoordinatorMailbox() {
|
|
1358
|
+
const now = this.now();
|
|
1359
|
+
return Number(this.db.prepare(`UPDATE coordinator_mailbox_entries SET status = 'pending',
|
|
1360
|
+
claim_owner = NULL, claimed_at = NULL, lease_expires_at = NULL, updated_at = ?
|
|
1361
|
+
WHERE status = 'claimed' AND lease_expires_at <= ?`).run(now, now).changes);
|
|
1362
|
+
}
|
|
1363
|
+
|
|
1364
|
+
prepareCoordinatorProviderTurn(workItemId, coordinatorTurnId, attemptNumber, requestBody, claim = {}) {
|
|
1365
|
+
return withTransaction(this.db, () => {
|
|
1366
|
+
const mailbox = this.#activeCoordinatorMailboxClaim(coordinatorTurnId, claim);
|
|
1367
|
+
if (!mailbox || mailbox.work_item_id !== workItemId) return null;
|
|
1368
|
+
const existing = this.db.prepare(`SELECT * FROM coordinator_provider_turns
|
|
1369
|
+
WHERE coordinator_turn_id = ? AND attempt_number = ?`).get(coordinatorTurnId, attemptNumber);
|
|
1370
|
+
const requestHash = createHash('sha256').update(stableJson(requestBody), 'utf8').digest('hex');
|
|
1371
|
+
if (existing) {
|
|
1372
|
+
if (existing.request_hash !== requestHash) {
|
|
1373
|
+
throw new Error('Prepared Coordinator provider request changed before dispatch');
|
|
1374
|
+
}
|
|
1375
|
+
if (existing.claim_owner !== claim.ownerBootId
|
|
1376
|
+
|| Number(existing.claim_epoch) !== Number(claim.claimEpoch)) return null;
|
|
1377
|
+
return this.#mapCoordinatorProviderTurn(existing);
|
|
1378
|
+
}
|
|
1379
|
+
const now = this.now();
|
|
1380
|
+
const id = durableId('coordinator-provider-turn');
|
|
1381
|
+
this.db.prepare(`INSERT INTO coordinator_provider_turns
|
|
1382
|
+
(id, work_item_id, coordinator_turn_id, attempt_number, status, request_body, request_hash,
|
|
1383
|
+
claim_owner, claim_epoch, prepared_at, updated_at)
|
|
1384
|
+
VALUES (?, ?, ?, ?, 'prepared', ?, ?, ?, ?, ?, ?)`).run(
|
|
1385
|
+
id, workItemId, coordinatorTurnId, attemptNumber, stringify(requestBody), requestHash,
|
|
1386
|
+
claim.ownerBootId, claim.claimEpoch, now, now,
|
|
1387
|
+
);
|
|
1388
|
+
return this.getCoordinatorProviderTurn(id);
|
|
1389
|
+
});
|
|
1390
|
+
}
|
|
1391
|
+
|
|
1392
|
+
#activeCoordinatorMailboxClaim(coordinatorTurnId, claim = {}) {
|
|
1393
|
+
if (!claim.mailboxId || !claim.ownerBootId || !Number.isInteger(Number(claim.claimEpoch))) return null;
|
|
1394
|
+
return this.db.prepare(`SELECT * FROM coordinator_mailbox_entries WHERE id = ? AND status = 'claimed'
|
|
1395
|
+
AND claim_owner = ? AND claim_epoch = ? AND lease_expires_at > ?
|
|
1396
|
+
AND json_extract(payload, '$.turnId') = ?`).get(
|
|
1397
|
+
claim.mailboxId, claim.ownerBootId, Number(claim.claimEpoch), this.now(), coordinatorTurnId,
|
|
1398
|
+
);
|
|
1399
|
+
}
|
|
1400
|
+
|
|
1401
|
+
#mapCoordinatorProviderTurn(row) {
|
|
1402
|
+
return {
|
|
1403
|
+
id: row.id,
|
|
1404
|
+
workItemId: row.work_item_id,
|
|
1405
|
+
coordinatorTurnId: row.coordinator_turn_id,
|
|
1406
|
+
attemptNumber: Number(row.attempt_number),
|
|
1407
|
+
status: row.status,
|
|
1408
|
+
requestBody: parseJson(row.request_body, {}),
|
|
1409
|
+
requestHash: row.request_hash,
|
|
1410
|
+
response: parseJson(row.response, null),
|
|
1411
|
+
responseHash: row.response_hash || null,
|
|
1412
|
+
error: row.error || null,
|
|
1413
|
+
claimOwner: row.claim_owner || null,
|
|
1414
|
+
claimEpoch: Number(row.claim_epoch) || 0,
|
|
1415
|
+
};
|
|
1416
|
+
}
|
|
1417
|
+
|
|
1418
|
+
getCoordinatorProviderTurn(id) {
|
|
1419
|
+
const row = this.db.prepare('SELECT * FROM coordinator_provider_turns WHERE id = ?').get(id);
|
|
1420
|
+
return row ? this.#mapCoordinatorProviderTurn(row) : null;
|
|
1421
|
+
}
|
|
1422
|
+
|
|
1423
|
+
dispatchCoordinatorProviderTurn(id, claim = {}) {
|
|
1424
|
+
return withTransaction(this.db, () => {
|
|
1425
|
+
const existing = this.getCoordinatorProviderTurn(id);
|
|
1426
|
+
if (!existing || !this.#activeCoordinatorMailboxClaim(existing.coordinatorTurnId, claim)) return null;
|
|
1427
|
+
const now = this.now();
|
|
1428
|
+
const changed = this.db.prepare(`UPDATE coordinator_provider_turns SET status = 'dispatching',
|
|
1429
|
+
dispatched_at = ?, updated_at = ? WHERE id = ? AND status = 'prepared'
|
|
1430
|
+
AND claim_owner = ? AND claim_epoch = ?`).run(
|
|
1431
|
+
now, now, id, claim.ownerBootId, Number(claim.claimEpoch),
|
|
1432
|
+
);
|
|
1433
|
+
return Number(changed.changes) === 1 ? this.getCoordinatorProviderTurn(id) : null;
|
|
1434
|
+
});
|
|
1435
|
+
}
|
|
1436
|
+
|
|
1437
|
+
respondCoordinatorProviderTurn(id, requestHash, response, claim = {}) {
|
|
1438
|
+
return withTransaction(this.db, () => {
|
|
1439
|
+
const existing = this.getCoordinatorProviderTurn(id);
|
|
1440
|
+
if (!existing || !this.#activeCoordinatorMailboxClaim(existing.coordinatorTurnId, claim)) return null;
|
|
1441
|
+
const now = this.now();
|
|
1442
|
+
const responseHash = createHash('sha256').update(stableJson(response), 'utf8').digest('hex');
|
|
1443
|
+
const changed = this.db.prepare(`UPDATE coordinator_provider_turns SET status = 'responded',
|
|
1444
|
+
response = ?, response_hash = ?, responded_at = ?, updated_at = ?
|
|
1445
|
+
WHERE id = ? AND status = 'dispatching' AND request_hash = ?
|
|
1446
|
+
AND claim_owner = ? AND claim_epoch = ?`).run(
|
|
1447
|
+
stringify(response), responseHash, now, now, id, requestHash,
|
|
1448
|
+
claim.ownerBootId, Number(claim.claimEpoch),
|
|
1449
|
+
);
|
|
1450
|
+
return Number(changed.changes) === 1 ? this.getCoordinatorProviderTurn(id) : null;
|
|
1451
|
+
});
|
|
1452
|
+
}
|
|
1453
|
+
|
|
1454
|
+
rejectCoordinatorProviderTurn(id, error, claim = {}) {
|
|
1455
|
+
return withTransaction(this.db, () => {
|
|
1456
|
+
const existing = this.getCoordinatorProviderTurn(id);
|
|
1457
|
+
if (!existing || !this.#activeCoordinatorMailboxClaim(existing.coordinatorTurnId, claim)) return null;
|
|
1458
|
+
const now = this.now();
|
|
1459
|
+
const changed = this.db.prepare(`UPDATE coordinator_provider_turns SET status = 'cancelled',
|
|
1460
|
+
error = ?, updated_at = ? WHERE id = ? AND status = 'responded'
|
|
1461
|
+
AND claim_owner = ? AND claim_epoch = ?`).run(
|
|
1462
|
+
String(error?.message || error || 'Coordinator provider response was rejected').slice(0, 8_000),
|
|
1463
|
+
now, id, claim.ownerBootId, Number(claim.claimEpoch),
|
|
1464
|
+
);
|
|
1465
|
+
return Number(changed.changes) === 1 ? this.getCoordinatorProviderTurn(id) : null;
|
|
1466
|
+
});
|
|
1467
|
+
}
|
|
1468
|
+
|
|
1469
|
+
recoverCoordinatorProviderTurns() {
|
|
1470
|
+
const now = this.now();
|
|
1471
|
+
return Number(this.db.prepare(`UPDATE coordinator_provider_turns SET status = 'unknown',
|
|
1472
|
+
error = 'Coordinator provider dispatch outcome is unknown after Agent restart', updated_at = ?
|
|
1473
|
+
WHERE status = 'dispatching' AND NOT EXISTS (
|
|
1474
|
+
SELECT 1 FROM coordinator_mailbox_entries mailbox
|
|
1475
|
+
WHERE json_extract(mailbox.payload, '$.turnId') = coordinator_provider_turns.coordinator_turn_id
|
|
1476
|
+
AND mailbox.status = 'claimed' AND mailbox.claim_owner = coordinator_provider_turns.claim_owner
|
|
1477
|
+
AND mailbox.claim_epoch = coordinator_provider_turns.claim_epoch
|
|
1478
|
+
AND mailbox.lease_expires_at > ?
|
|
1479
|
+
)`).run(now, now).changes);
|
|
1480
|
+
}
|
|
1481
|
+
|
|
1482
|
+
createOperation(input = {}) {
|
|
1483
|
+
const replayPolicy = input.replayPolicy || 'never_automatic';
|
|
1484
|
+
if (!input.idempotencyKey) throw new Error('Operation idempotencyKey is required');
|
|
1485
|
+
if (!['safe', 'probe_first', 'never_automatic'].includes(replayPolicy)) {
|
|
1486
|
+
throw new Error(`Unsupported Operation replay policy: ${replayPolicy}`);
|
|
1487
|
+
}
|
|
1488
|
+
const now = this.now();
|
|
1489
|
+
const id = input.id || durableId('operation');
|
|
1490
|
+
this.db.prepare(`INSERT INTO operations
|
|
1491
|
+
(id, work_item_id, action_id, run_id, engine_turn_id, operation_type, idempotency_key,
|
|
1492
|
+
replay_policy, effect_status, execution_status, payload, created_at, updated_at)
|
|
1493
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1494
|
+
ON CONFLICT(idempotency_key) DO NOTHING`).run(
|
|
1495
|
+
id, input.workItemId, input.actionId || null, input.runId || null, input.engineTurnId || null,
|
|
1496
|
+
input.operationType || 'unknown', input.idempotencyKey, replayPolicy,
|
|
1497
|
+
input.effectStatus || 'pending', input.executionStatus || 'not_started',
|
|
1498
|
+
stringify(input.payload || {}), now, now,
|
|
1499
|
+
);
|
|
1500
|
+
return this.getOperationByKey(input.idempotencyKey);
|
|
1501
|
+
}
|
|
1502
|
+
|
|
1503
|
+
getOperation(id) {
|
|
1504
|
+
const row = this.db.prepare('SELECT * FROM operations WHERE id = ?').get(id);
|
|
1505
|
+
return row ? this.#mapOperation(row) : null;
|
|
1506
|
+
}
|
|
1507
|
+
|
|
1508
|
+
getOperationByKey(idempotencyKey) {
|
|
1509
|
+
const row = this.db.prepare('SELECT * FROM operations WHERE idempotency_key = ?').get(idempotencyKey);
|
|
1510
|
+
return row ? this.#mapOperation(row) : null;
|
|
1511
|
+
}
|
|
1512
|
+
|
|
1513
|
+
#mapOperation(row) {
|
|
1514
|
+
return {
|
|
1515
|
+
id: row.id,
|
|
1516
|
+
workItemId: row.work_item_id,
|
|
1517
|
+
actionId: row.action_id || null,
|
|
1518
|
+
runId: row.run_id || null,
|
|
1519
|
+
engineTurnId: row.engine_turn_id || null,
|
|
1520
|
+
operationType: row.operation_type,
|
|
1521
|
+
idempotencyKey: row.idempotency_key,
|
|
1522
|
+
replayPolicy: row.replay_policy,
|
|
1523
|
+
concurrencyPolicy: row.concurrency_policy,
|
|
1524
|
+
effectStatus: row.effect_status,
|
|
1525
|
+
executionStatus: row.execution_status,
|
|
1526
|
+
effectCutoff: parseJson(row.effect_cutoff, null),
|
|
1527
|
+
grantManifest: parseJson(row.grant_manifest, {}),
|
|
1528
|
+
resourceRelease: parseJson(row.resource_release, {}),
|
|
1529
|
+
supplementalInventory: parseJson(row.supplemental_inventory, {}),
|
|
1530
|
+
payload: parseJson(row.payload, {}),
|
|
1531
|
+
result: parseJson(row.result, null),
|
|
1532
|
+
};
|
|
1533
|
+
}
|
|
1534
|
+
|
|
1535
|
+
operationSafeToProceed(operationOrKey) {
|
|
1536
|
+
const operation = typeof operationOrKey === 'string'
|
|
1537
|
+
? this.getOperationByKey(operationOrKey) : operationOrKey;
|
|
1538
|
+
if (!operation) return false;
|
|
1539
|
+
const effectSafe = ['applied', 'not_applied', 'failed_no_effect'].includes(operation.effectStatus);
|
|
1540
|
+
const executionSafe = ['not_started', 'quiescent'].includes(operation.executionStatus)
|
|
1541
|
+
|| (operation.executionStatus === 'fenced' && operation.effectCutoff?.status === 'current');
|
|
1542
|
+
const manifest = operation.grantManifest || {};
|
|
1543
|
+
const grantSafe = manifest.status === 'closed' && manifest.safetyStatus === 'current'
|
|
1544
|
+
&& manifest.inventoryComplete === true
|
|
1545
|
+
&& (manifest.pendingGrantAttemptIds || []).length === 0
|
|
1546
|
+
&& (manifest.authorityClosures || []).every(closure => closure.status === 'closed');
|
|
1547
|
+
const release = operation.resourceRelease || {};
|
|
1548
|
+
const resourcesSafe = release.status === 'released'
|
|
1549
|
+
&& (release.leases || []).every(lease => ['released', 'expired'].includes(lease.status));
|
|
1550
|
+
const supplemental = operation.supplementalInventory || {};
|
|
1551
|
+
const supplementalSafe = ['clear', 'resolved'].includes(supplemental.status || 'clear');
|
|
1552
|
+
return effectSafe && executionSafe && grantSafe && resourcesSafe && supplementalSafe;
|
|
1553
|
+
}
|
|
1554
|
+
|
|
1555
|
+
#hasBlockingOperation(workItemId, actionId = null) {
|
|
1556
|
+
const rows = this.db.prepare(`SELECT idempotency_key FROM operations
|
|
1557
|
+
WHERE work_item_id = ? AND concurrency_policy = 'blocking'
|
|
1558
|
+
AND (? IS NULL OR action_id IS NULL OR action_id = ?)`)
|
|
1559
|
+
.all(workItemId, actionId, actionId);
|
|
1560
|
+
return rows.some(row => !this.operationSafeToProceed(row.idempotency_key));
|
|
1561
|
+
}
|
|
1562
|
+
|
|
1563
|
+
createAndClaimOperation(input = {}, ownerBootId, leaseEpoch, automatic = true) {
|
|
1564
|
+
return withTransaction(this.db, () => {
|
|
1565
|
+
const active = this.#activeRunRow(input.runId, ownerBootId, leaseEpoch, true);
|
|
1566
|
+
if (!active || active.work_item_id !== input.workItemId || active.action_id !== input.actionId) return null;
|
|
1567
|
+
const operation = this.createOperation(input);
|
|
1568
|
+
if (!operation
|
|
1569
|
+
|| operation.workItemId !== input.workItemId
|
|
1570
|
+
|| operation.actionId !== input.actionId
|
|
1571
|
+
|| operation.runId !== input.runId
|
|
1572
|
+
|| operation.operationType !== (input.operationType || 'unknown')
|
|
1573
|
+
|| operation.executionStatus !== 'not_started') return null;
|
|
1574
|
+
if (automatic && operation.replayPolicy !== 'safe') return null;
|
|
1575
|
+
const now = this.now();
|
|
1576
|
+
const changed = this.db.prepare(`UPDATE operations SET execution_status = 'running',
|
|
1577
|
+
execution_epoch = execution_epoch + 1, owner_boot_id = ?, owner_lease_epoch = ?,
|
|
1578
|
+
claimed_at = ?, updated_at = ? WHERE idempotency_key = ? AND execution_status = 'not_started'
|
|
1579
|
+
AND work_item_id = ? AND action_id = ? AND run_id = ?`).run(
|
|
1580
|
+
ownerBootId, leaseEpoch, now, now, input.idempotencyKey,
|
|
1581
|
+
input.workItemId, input.actionId, input.runId,
|
|
1582
|
+
);
|
|
1583
|
+
return Number(changed.changes) === 1 ? this.getOperationByKey(input.idempotencyKey) : null;
|
|
1584
|
+
});
|
|
1585
|
+
}
|
|
1586
|
+
|
|
1587
|
+
claimOperation(idempotencyKey, ownerBootId, leaseEpoch, automatic = true) {
|
|
1588
|
+
return withTransaction(this.db, () => {
|
|
1589
|
+
const operation = this.getOperationByKey(idempotencyKey);
|
|
1590
|
+
if (!operation || operation.executionStatus !== 'not_started') return null;
|
|
1591
|
+
if (!this.#activeRunRow(operation.runId, ownerBootId, leaseEpoch, true)) return null;
|
|
1592
|
+
if (automatic && operation.replayPolicy !== 'safe') return null;
|
|
1593
|
+
const now = this.now();
|
|
1594
|
+
const changed = this.db.prepare(`UPDATE operations SET execution_status = 'running',
|
|
1595
|
+
execution_epoch = execution_epoch + 1, owner_boot_id = ?, owner_lease_epoch = ?,
|
|
1596
|
+
claimed_at = ?, updated_at = ? WHERE idempotency_key = ? AND execution_status = 'not_started'
|
|
1597
|
+
AND run_id = ?`).run(
|
|
1598
|
+
ownerBootId, leaseEpoch, now, now, idempotencyKey, operation.runId,
|
|
1599
|
+
);
|
|
1600
|
+
return Number(changed.changes) === 1 ? this.getOperationByKey(idempotencyKey) : null;
|
|
1601
|
+
});
|
|
1602
|
+
}
|
|
1603
|
+
|
|
1604
|
+
completeOperation(idempotencyKey, ownerBootId, leaseEpoch, effectStatus, result = null) {
|
|
1605
|
+
if (!['applied', 'not_applied', 'failed_no_effect', 'unknown'].includes(effectStatus)) return false;
|
|
1606
|
+
return withTransaction(this.db, () => {
|
|
1607
|
+
const operation = this.getOperationByKey(idempotencyKey);
|
|
1608
|
+
if (!operation || operation.executionStatus !== 'running'
|
|
1609
|
+
|| operation.payload == null) return false;
|
|
1610
|
+
const ownerMatches = this.db.prepare(`SELECT 1 AS present FROM operations
|
|
1611
|
+
WHERE idempotency_key = ? AND execution_status = 'running'
|
|
1612
|
+
AND owner_boot_id = ? AND owner_lease_epoch = ?`).get(
|
|
1613
|
+
idempotencyKey, ownerBootId, leaseEpoch,
|
|
1614
|
+
);
|
|
1615
|
+
if (!ownerMatches) return false;
|
|
1616
|
+
const now = this.now();
|
|
1617
|
+
const active = this.#activeRunRow(operation.runId, ownerBootId, leaseEpoch, true);
|
|
1618
|
+
if (!active || active.work_item_id !== operation.workItemId
|
|
1619
|
+
|| active.action_id !== operation.actionId) {
|
|
1620
|
+
this.db.prepare(`UPDATE operations SET effect_status = 'unknown',
|
|
1621
|
+
execution_status = 'hazardous_orphan', effect_cutoff = ?, result = ?, completed_at = ?,
|
|
1622
|
+
updated_at = ? WHERE idempotency_key = ? AND execution_status = 'running'
|
|
1623
|
+
AND owner_boot_id = ? AND owner_lease_epoch = ?`).run(
|
|
1624
|
+
stringify({ status: 'stale', closureType: 'late_completion', closedAt: now }),
|
|
1625
|
+
stringify({ attemptedEffectStatus: effectStatus, reportedResult: result }),
|
|
1626
|
+
now, now, idempotencyKey, ownerBootId, leaseEpoch,
|
|
1627
|
+
);
|
|
1628
|
+
return false;
|
|
1629
|
+
}
|
|
1630
|
+
const changed = this.db.prepare(`UPDATE operations SET effect_status = ?, execution_status = ?,
|
|
1631
|
+
effect_cutoff = ?, result = ?, completed_at = ?, updated_at = ? WHERE idempotency_key = ?
|
|
1632
|
+
AND execution_status = 'running' AND owner_boot_id = ? AND owner_lease_epoch = ?`).run(
|
|
1633
|
+
effectStatus,
|
|
1634
|
+
'quiescent',
|
|
1635
|
+
stringify({ status: 'current', closureType: 'quiescent', closedAt: now }),
|
|
1636
|
+
stringify(result), now, now, idempotencyKey, ownerBootId, leaseEpoch,
|
|
1637
|
+
);
|
|
1638
|
+
return Number(changed.changes) === 1;
|
|
1639
|
+
});
|
|
1640
|
+
}
|
|
1641
|
+
|
|
1642
|
+
recoverOperations() {
|
|
1643
|
+
return withTransaction(this.db, () => {
|
|
1644
|
+
const now = this.now();
|
|
1645
|
+
const unstarted = this.db.prepare(`UPDATE operations SET effect_status = 'failed_no_effect',
|
|
1646
|
+
execution_status = 'quiescent', effect_cutoff = ?, result = ?, completed_at = ?, updated_at = ?
|
|
1647
|
+
WHERE effect_status = 'pending' AND execution_status = 'not_started'`).run(
|
|
1648
|
+
stringify({ status: 'current', closureType: 'recovered_before_dispatch', closedAt: now }),
|
|
1649
|
+
stringify({ recovered: true, reason: 'Operation was never claimed before restart' }),
|
|
1650
|
+
now, now,
|
|
1651
|
+
);
|
|
1652
|
+
const hazardous = this.db.prepare(`UPDATE operations SET execution_status = 'hazardous_orphan',
|
|
1653
|
+
effect_status = CASE WHEN effect_status = 'pending' THEN 'unknown' ELSE effect_status END,
|
|
1654
|
+
effect_cutoff = ?, updated_at = ? WHERE execution_status IN ('running', 'cancel_requested')`).run(
|
|
1655
|
+
stringify({ status: 'stale', closureType: 'restart_unknown', closedAt: now }), now,
|
|
1656
|
+
);
|
|
1657
|
+
return Number(unstarted.changes) + Number(hazardous.changes);
|
|
1658
|
+
});
|
|
1659
|
+
}
|
|
1660
|
+
|
|
1142
1661
|
appendEvent(workItemId, type, data = {}, refs = {}) {
|
|
1143
1662
|
const actionGeneration = refs.actionGeneration
|
|
1144
1663
|
?? (refs.actionId ? this.getAction(refs.actionId)?.generation : null)
|
|
@@ -1157,8 +1676,41 @@ export class WorkItemStore {
|
|
|
1157
1676
|
return Number(result.lastInsertRowid);
|
|
1158
1677
|
}
|
|
1159
1678
|
|
|
1160
|
-
|
|
1679
|
+
getCoordinatorClientMessageReceipt(workItemId, clientMessageId) {
|
|
1680
|
+
if (typeof clientMessageId !== 'string' || !clientMessageId) return null;
|
|
1681
|
+
const sourceKey = `client:message:${workItemId}:${clientMessageId}`;
|
|
1682
|
+
const actionReceipt = this.db.prepare('SELECT action_id FROM action_entries WHERE source_key = ?')
|
|
1683
|
+
.get(sourceKey);
|
|
1684
|
+
if (actionReceipt) throw new Error('clientMessageId already belongs to an Action message');
|
|
1685
|
+
const row = this.db.prepare(`SELECT payload FROM coordinator_mailbox_entries
|
|
1686
|
+
WHERE work_item_id = ? AND source_key = ?`).get(workItemId, sourceKey);
|
|
1687
|
+
return row ? parseJson(row.payload, {}) : null;
|
|
1688
|
+
}
|
|
1689
|
+
|
|
1690
|
+
hasActionInputClientMessage(workItemId, actionId, clientMessageId) {
|
|
1691
|
+
if (typeof clientMessageId !== 'string' || !clientMessageId) return false;
|
|
1692
|
+
const sourceKey = `client:message:${workItemId}:${clientMessageId}`;
|
|
1693
|
+
const coordinatorReceipt = this.db.prepare(`SELECT 1 FROM coordinator_mailbox_entries
|
|
1694
|
+
WHERE source_key = ?`).get(sourceKey);
|
|
1695
|
+
if (coordinatorReceipt) throw new Error('clientMessageId already belongs to a Coordinator message');
|
|
1696
|
+
const entry = this.db.prepare('SELECT action_id FROM action_entries WHERE source_key = ?').get(sourceKey)
|
|
1697
|
+
|| this.db.prepare(`SELECT action_id FROM events WHERE work_item_id = ?
|
|
1698
|
+
AND type = 'action.input_added' AND json_extract(data, '$.clientMessageId') = ?`).get(
|
|
1699
|
+
workItemId, clientMessageId,
|
|
1700
|
+
);
|
|
1701
|
+
if (!entry) return false;
|
|
1702
|
+
if (entry.action_id !== actionId) throw new Error('clientMessageId already belongs to another Action');
|
|
1703
|
+
return true;
|
|
1704
|
+
}
|
|
1705
|
+
|
|
1706
|
+
addActionInput(id, input, expected, attachments = null, addedAttachments = [], clientMessageId = null) {
|
|
1161
1707
|
return withTransaction(this.db, () => {
|
|
1708
|
+
const sourceKey = typeof clientMessageId === 'string' && clientMessageId
|
|
1709
|
+
? `client:message:${id}:${clientMessageId}` : null;
|
|
1710
|
+
if (sourceKey) {
|
|
1711
|
+
const existing = this.db.prepare('SELECT id FROM action_entries WHERE source_key = ?').get(sourceKey);
|
|
1712
|
+
if (existing) return this.getWorkItemDetail(id);
|
|
1713
|
+
}
|
|
1162
1714
|
const workItem = this.getWorkItem(id);
|
|
1163
1715
|
if (!workItem) return null;
|
|
1164
1716
|
const expectedGeneration = Number(expected.generation);
|
|
@@ -1265,6 +1817,7 @@ export class WorkItemStore {
|
|
|
1265
1817
|
}
|
|
1266
1818
|
const eventId = this.appendEvent(id, 'action.input_added', {
|
|
1267
1819
|
inputId,
|
|
1820
|
+
clientMessageId,
|
|
1268
1821
|
text: input,
|
|
1269
1822
|
attachments: projectedAttachments,
|
|
1270
1823
|
}, { actionId: action.id, runId: eventRunId, actionGeneration: eventGeneration });
|
|
@@ -1282,6 +1835,31 @@ export class WorkItemStore {
|
|
|
1282
1835
|
input,
|
|
1283
1836
|
stringify(projectedAttachments),
|
|
1284
1837
|
);
|
|
1838
|
+
this.#appendActionEntry({
|
|
1839
|
+
workItemId: id,
|
|
1840
|
+
actionId: action.id,
|
|
1841
|
+
runId: action.currentRunId,
|
|
1842
|
+
kind: 'message',
|
|
1843
|
+
role: 'user',
|
|
1844
|
+
status: 'pending',
|
|
1845
|
+
text: input,
|
|
1846
|
+
attachments: projectedAttachments,
|
|
1847
|
+
payload: { eventId, inputId, actionGeneration: eventGeneration, actionSpecHash: eventSpecHash },
|
|
1848
|
+
createdAt: now,
|
|
1849
|
+
}, sourceKey || `pending_action_inputs:event:${eventId}`);
|
|
1850
|
+
} else if (sourceKey) {
|
|
1851
|
+
this.#appendActionEntry({
|
|
1852
|
+
workItemId: id,
|
|
1853
|
+
actionId: action.id,
|
|
1854
|
+
runId: null,
|
|
1855
|
+
kind: 'message',
|
|
1856
|
+
role: 'user',
|
|
1857
|
+
status: 'consumed',
|
|
1858
|
+
text: input,
|
|
1859
|
+
attachments: projectedAttachments,
|
|
1860
|
+
payload: { eventId, inputId, actionGeneration: eventGeneration, actionSpecHash: eventSpecHash },
|
|
1861
|
+
createdAt: now,
|
|
1862
|
+
}, sourceKey);
|
|
1285
1863
|
}
|
|
1286
1864
|
return this.getWorkItemDetail(id);
|
|
1287
1865
|
});
|
|
@@ -1290,27 +1868,242 @@ export class WorkItemStore {
|
|
|
1290
1868
|
listPendingActionInputs(actionId, runId, ownerBootId, leaseEpoch) {
|
|
1291
1869
|
const active = this.#activeRunRow(runId, ownerBootId, leaseEpoch, true);
|
|
1292
1870
|
if (!active || active.action_id !== actionId) return [];
|
|
1293
|
-
return this.db.prepare(`SELECT
|
|
1294
|
-
|
|
1295
|
-
|
|
1871
|
+
return this.db.prepare(`SELECT p.*, ae.id AS action_entry_id, ae.sequence AS action_entry_sequence
|
|
1872
|
+
FROM pending_action_inputs p
|
|
1873
|
+
LEFT JOIN action_entries ae ON CAST(json_extract(ae.payload, '$.eventId') AS INTEGER) = p.event_id
|
|
1874
|
+
AND ae.action_id = p.action_id
|
|
1875
|
+
WHERE p.action_id = ? AND p.run_id = ? AND p.action_generation = ? AND p.action_spec_hash = ?
|
|
1876
|
+
AND p.consumed_at IS NULL AND p.superseded_at IS NULL ORDER BY p.event_id`).all(
|
|
1296
1877
|
actionId, runId, active.action_generation, active.action_spec_hash,
|
|
1297
1878
|
).map(row => ({
|
|
1298
1879
|
id: String(row.event_id),
|
|
1880
|
+
actionEntryId: row.action_entry_id || null,
|
|
1881
|
+
sequence: row.action_entry_sequence == null ? null : Number(row.action_entry_sequence),
|
|
1299
1882
|
text: row.text || '',
|
|
1300
1883
|
attachments: parseJson(row.attachments, []),
|
|
1301
1884
|
}));
|
|
1302
1885
|
}
|
|
1303
1886
|
|
|
1304
|
-
|
|
1887
|
+
prepareEngineTurn(actionId, runId, ownerBootId, leaseEpoch, inputs = [], request = {}) {
|
|
1305
1888
|
return withTransaction(this.db, () => {
|
|
1306
1889
|
const active = this.#activeRunRow(runId, ownerBootId, leaseEpoch, true);
|
|
1307
|
-
if (!active || active.action_id !== actionId) return
|
|
1308
|
-
const
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1890
|
+
if (!active || active.action_id !== actionId) return null;
|
|
1891
|
+
const eventIds = inputs.map(input => Number(input?.id ?? input)).filter(Number.isInteger);
|
|
1892
|
+
const normalizedEventIds = [...new Set(eventIds)].sort((left, right) => left - right);
|
|
1893
|
+
const reusable = this.db.prepare(`SELECT id FROM engine_turns
|
|
1894
|
+
WHERE run_id = ? AND status = 'prepared' ORDER BY ordinal DESC LIMIT 1`).get(runId);
|
|
1895
|
+
if (reusable) {
|
|
1896
|
+
const turn = this.getEngineTurn(reusable.id);
|
|
1897
|
+
const requestBody = request.requestBody && typeof request.requestBody === 'object'
|
|
1898
|
+
? request.requestBody : {};
|
|
1899
|
+
const requestHash = createHash('sha256').update(stableJson(requestBody), 'utf8').digest('hex');
|
|
1900
|
+
if (turn.requestHash !== requestHash) {
|
|
1901
|
+
throw new Error('Prepared EngineTurn request changed before provider dispatch');
|
|
1902
|
+
}
|
|
1903
|
+
return turn;
|
|
1904
|
+
}
|
|
1905
|
+
const entries = [];
|
|
1906
|
+
for (const eventId of normalizedEventIds) {
|
|
1907
|
+
const row = this.db.prepare(`SELECT p.*, ae.id AS action_entry_id,
|
|
1908
|
+
ae.sequence AS action_entry_sequence FROM pending_action_inputs p
|
|
1909
|
+
JOIN action_entries ae ON CAST(json_extract(ae.payload, '$.eventId') AS INTEGER) = p.event_id
|
|
1910
|
+
AND ae.action_id = p.action_id
|
|
1911
|
+
WHERE p.event_id = ? AND p.action_id = ? AND p.run_id = ? AND p.action_generation = ?
|
|
1912
|
+
AND p.action_spec_hash = ? AND p.consumed_at IS NULL AND p.superseded_at IS NULL
|
|
1913
|
+
AND ae.status = 'pending' AND ae.engine_turn_id IS NULL`).get(
|
|
1914
|
+
eventId, actionId, runId, active.action_generation, active.action_spec_hash,
|
|
1915
|
+
);
|
|
1916
|
+
if (row) entries.push(row);
|
|
1917
|
+
}
|
|
1918
|
+
if (entries.length !== normalizedEventIds.length) return null;
|
|
1919
|
+
const requestBody = request.requestBody && typeof request.requestBody === 'object'
|
|
1920
|
+
? request.requestBody
|
|
1921
|
+
: { actionEntryIds: entries.map(row => row.action_entry_id) };
|
|
1922
|
+
const requestHash = createHash('sha256').update(stableJson(requestBody), 'utf8').digest('hex');
|
|
1923
|
+
const ordinal = Number(this.db.prepare(`SELECT COALESCE(MAX(ordinal), 0) + 1 AS value
|
|
1924
|
+
FROM engine_turns WHERE run_id = ?`).get(runId)?.value) || 1;
|
|
1925
|
+
const turnId = durableId('engine-turn');
|
|
1926
|
+
const requestKey = `run:${runId}:turn:${ordinal}`;
|
|
1927
|
+
const claimedThroughSequence = Math.max(
|
|
1928
|
+
0,
|
|
1929
|
+
...entries.map(row => Number(row.action_entry_sequence) || 0),
|
|
1312
1930
|
);
|
|
1313
|
-
|
|
1931
|
+
const entryIds = entries.map(row => row.action_entry_id);
|
|
1932
|
+
const now = this.now();
|
|
1933
|
+
this.db.prepare(`INSERT INTO engine_turns
|
|
1934
|
+
(id, work_item_id, action_id, run_id, ordinal, status, owner_boot_id, lease_epoch,
|
|
1935
|
+
input_entry_ids, message_entry_ids, claimed_through_sequence, request_body, request_hash,
|
|
1936
|
+
request_key, dispatch_capability, created_at, updated_at)
|
|
1937
|
+
VALUES (?, ?, ?, ?, ?, 'prepared', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(
|
|
1938
|
+
turnId, active.work_item_id, actionId, runId, ordinal, ownerBootId, leaseEpoch,
|
|
1939
|
+
stringify(entryIds), stringify(entryIds), claimedThroughSequence, stringify(requestBody),
|
|
1940
|
+
requestHash, requestKey, request.dispatchCapability || 'unknown', now, now,
|
|
1941
|
+
);
|
|
1942
|
+
const bind = this.db.prepare(`UPDATE action_entries SET status = 'bound', engine_turn_id = ?,
|
|
1943
|
+
run_id = ?, updated_at = ? WHERE id = ? AND status = 'pending' AND engine_turn_id IS NULL`);
|
|
1944
|
+
for (const row of entries) {
|
|
1945
|
+
const changed = bind.run(turnId, runId, now, row.action_entry_id);
|
|
1946
|
+
if (Number(changed.changes) !== 1) throw new Error('ActionEntry changed before EngineTurn prepare');
|
|
1947
|
+
}
|
|
1948
|
+
this.appendEvent(active.work_item_id, 'engine_turn.prepared', {
|
|
1949
|
+
turnId, ordinal, requestHash, actionEntryIds: entryIds,
|
|
1950
|
+
}, { actionId, runId });
|
|
1951
|
+
return this.getEngineTurn(turnId);
|
|
1952
|
+
});
|
|
1953
|
+
}
|
|
1954
|
+
|
|
1955
|
+
getEngineTurn(turnId) {
|
|
1956
|
+
const row = this.db.prepare('SELECT * FROM engine_turns WHERE id = ?').get(turnId);
|
|
1957
|
+
if (!row) return null;
|
|
1958
|
+
return {
|
|
1959
|
+
id: row.id,
|
|
1960
|
+
workItemId: row.work_item_id,
|
|
1961
|
+
actionId: row.action_id,
|
|
1962
|
+
runId: row.run_id,
|
|
1963
|
+
ordinal: Number(row.ordinal),
|
|
1964
|
+
status: row.status,
|
|
1965
|
+
ownerBootId: row.owner_boot_id,
|
|
1966
|
+
leaseEpoch: Number(row.lease_epoch),
|
|
1967
|
+
inputEntryIds: parseJson(row.input_entry_ids, []),
|
|
1968
|
+
requestBody: parseJson(row.request_body, {}),
|
|
1969
|
+
requestHash: row.request_hash,
|
|
1970
|
+
requestKey: row.request_key,
|
|
1971
|
+
dispatchAttempt: Number(row.dispatch_attempt) || 0,
|
|
1972
|
+
dispatchCapability: row.dispatch_capability || 'unknown',
|
|
1973
|
+
claimedAt: row.claimed_at || null,
|
|
1974
|
+
dispatchedAt: row.dispatched_at || null,
|
|
1975
|
+
response: parseJson(row.response, null),
|
|
1976
|
+
responseHash: row.response_hash || null,
|
|
1977
|
+
consumedAt: row.consumed_at || null,
|
|
1978
|
+
error: row.error || null,
|
|
1979
|
+
};
|
|
1980
|
+
}
|
|
1981
|
+
|
|
1982
|
+
claimEngineTurn(turnId, ownerBootId, leaseEpoch) {
|
|
1983
|
+
return withTransaction(this.db, () => {
|
|
1984
|
+
const turn = this.getEngineTurn(turnId);
|
|
1985
|
+
if (!turn || turn.ownerBootId !== ownerBootId || turn.leaseEpoch !== leaseEpoch) return null;
|
|
1986
|
+
if (!this.#activeRunRow(turn.runId, ownerBootId, leaseEpoch, true)) return null;
|
|
1987
|
+
if (turn.status === 'dispatching') return turn;
|
|
1988
|
+
if (turn.status !== 'prepared') return null;
|
|
1989
|
+
const now = this.now();
|
|
1990
|
+
const changed = this.db.prepare(`UPDATE engine_turns SET status = 'dispatching',
|
|
1991
|
+
dispatch_attempt = dispatch_attempt + 1, claimed_at = COALESCE(claimed_at, ?),
|
|
1992
|
+
dispatched_at = ?, updated_at = ? WHERE id = ? AND status = 'prepared'`).run(
|
|
1993
|
+
now, now, now, turnId,
|
|
1994
|
+
);
|
|
1995
|
+
if (Number(changed.changes) !== 1) return null;
|
|
1996
|
+
this.appendEvent(turn.workItemId, 'engine_turn.dispatching', {
|
|
1997
|
+
turnId, requestHash: turn.requestHash, dispatchAttempt: turn.dispatchAttempt + 1,
|
|
1998
|
+
}, { actionId: turn.actionId, runId: turn.runId });
|
|
1999
|
+
return this.getEngineTurn(turnId);
|
|
2000
|
+
});
|
|
2001
|
+
}
|
|
2002
|
+
|
|
2003
|
+
consumeEngineTurn(turnId, ownerBootId, leaseEpoch, result = {}) {
|
|
2004
|
+
return withTransaction(this.db, () => {
|
|
2005
|
+
const turn = this.getEngineTurn(turnId);
|
|
2006
|
+
if (!turn || turn.status !== 'dispatching' || turn.ownerBootId !== ownerBootId
|
|
2007
|
+
|| turn.leaseEpoch !== leaseEpoch) return false;
|
|
2008
|
+
const now = this.now();
|
|
2009
|
+
const response = {
|
|
2010
|
+
text: String(result.responseText || ''),
|
|
2011
|
+
stopReason: result.stopReason || null,
|
|
2012
|
+
toolCalls: Array.isArray(result.toolCalls) ? result.toolCalls : [],
|
|
2013
|
+
thinkingBlocks: Array.isArray(result.thinkingBlocks) ? result.thinkingBlocks : [],
|
|
2014
|
+
};
|
|
2015
|
+
const responseHash = createHash('sha256').update(stableJson(response), 'utf8').digest('hex');
|
|
2016
|
+
const consumed = this.db.prepare(`UPDATE engine_turns SET status = 'responded', response = ?,
|
|
2017
|
+
response_hash = ?, responded_at = ?, consumed_at = ?,
|
|
2018
|
+
consumed_through_sequence = claimed_through_sequence, updated_at = ?
|
|
2019
|
+
WHERE id = ? AND status = 'dispatching' AND dispatch_attempt = ? AND request_hash = ?`).run(
|
|
2020
|
+
stringify(response), responseHash, now, now, now, turnId, turn.dispatchAttempt, turn.requestHash,
|
|
2021
|
+
);
|
|
2022
|
+
if (Number(consumed.changes) !== 1) return false;
|
|
2023
|
+
this.db.prepare(`UPDATE action_entries SET status = 'consumed', consumed_at = ?, updated_at = ?
|
|
2024
|
+
WHERE engine_turn_id = ? AND status = 'bound'`).run(now, now, turnId);
|
|
2025
|
+
const eventIds = this.db.prepare(`SELECT CAST(json_extract(payload, '$.eventId') AS INTEGER) AS event_id
|
|
2026
|
+
FROM action_entries WHERE engine_turn_id = ? AND json_extract(payload, '$.eventId') IS NOT NULL`)
|
|
2027
|
+
.all(turnId).map(row => row.event_id);
|
|
2028
|
+
const acknowledge = this.db.prepare(`UPDATE pending_action_inputs SET consumed_at = ?
|
|
2029
|
+
WHERE event_id = ? AND action_id = ? AND run_id = ? AND consumed_at IS NULL AND superseded_at IS NULL`);
|
|
2030
|
+
for (const eventId of eventIds) acknowledge.run(now, eventId, turn.actionId, turn.runId);
|
|
2031
|
+
this.appendEvent(turn.workItemId, 'engine_turn.responded', {
|
|
2032
|
+
turnId, requestHash: turn.requestHash, responseHash, dispatchAttempt: turn.dispatchAttempt,
|
|
2033
|
+
}, { actionId: turn.actionId, runId: turn.runId });
|
|
2034
|
+
return true;
|
|
2035
|
+
});
|
|
2036
|
+
}
|
|
2037
|
+
|
|
2038
|
+
failEngineTurn(turnId, ownerBootId, leaseEpoch, error) {
|
|
2039
|
+
return withTransaction(this.db, () => {
|
|
2040
|
+
const turn = this.getEngineTurn(turnId);
|
|
2041
|
+
if (!turn || turn.ownerBootId !== ownerBootId || turn.leaseEpoch !== leaseEpoch) {
|
|
2042
|
+
return { allowRetry: false, status: 'stale' };
|
|
2043
|
+
}
|
|
2044
|
+
if (turn.status === 'prepared') return { allowRetry: true, status: 'prepared' };
|
|
2045
|
+
if (turn.status !== 'dispatching') return { allowRetry: false, status: turn.status };
|
|
2046
|
+
const now = this.now();
|
|
2047
|
+
const message = String(error?.message || error || 'Provider dispatch failed').slice(0, 8_000);
|
|
2048
|
+
const changed = this.db.prepare(`UPDATE engine_turns SET status = 'unknown', error = ?,
|
|
2049
|
+
updated_at = ? WHERE id = ? AND status = 'dispatching' AND dispatch_attempt = ?`).run(
|
|
2050
|
+
message, now, turnId, turn.dispatchAttempt,
|
|
2051
|
+
);
|
|
2052
|
+
if (Number(changed.changes) !== 1) return { allowRetry: false, status: 'stale' };
|
|
2053
|
+
this.db.prepare(`UPDATE runs SET status = 'dispatch_unknown', accepting_input = 0,
|
|
2054
|
+
ended_at = ?, error = ? WHERE id = ? AND status = 'running'
|
|
2055
|
+
AND owner_boot_id = ? AND lease_epoch = ?`).run(
|
|
2056
|
+
now, message, turn.runId, ownerBootId, leaseEpoch,
|
|
2057
|
+
);
|
|
2058
|
+
this.db.prepare(`UPDATE actions SET status = 'failed', current_run_id = NULL,
|
|
2059
|
+
updated_at = ? WHERE id = ? AND status = 'running' AND current_run_id = ?
|
|
2060
|
+
AND lease_epoch = ?`).run(now, turn.actionId, turn.runId, leaseEpoch);
|
|
2061
|
+
this.db.prepare(`UPDATE work_items SET status = 'needs_attention', current_action_id = ?,
|
|
2062
|
+
current_run_id = NULL, updated_at = ? WHERE id = ? AND status = 'running'`).run(
|
|
2063
|
+
turn.actionId, now, turn.workItemId,
|
|
2064
|
+
);
|
|
2065
|
+
this.appendEvent(turn.workItemId, 'engine_turn.dispatch_unknown', {
|
|
2066
|
+
turnId, requestHash: turn.requestHash, dispatchAttempt: turn.dispatchAttempt, error: message,
|
|
2067
|
+
}, { actionId: turn.actionId, runId: turn.runId });
|
|
2068
|
+
return { allowRetry: false, status: 'unknown' };
|
|
2069
|
+
});
|
|
2070
|
+
}
|
|
2071
|
+
|
|
2072
|
+
recoverEngineTurns() {
|
|
2073
|
+
return withTransaction(this.db, () => {
|
|
2074
|
+
const now = this.now();
|
|
2075
|
+
const recoverable = this.db.prepare(`SELECT id FROM engine_turns WHERE status = 'prepared'`).all();
|
|
2076
|
+
const dispatching = this.db.prepare(`SELECT * FROM engine_turns WHERE status = 'dispatching'`).all();
|
|
2077
|
+
for (const row of dispatching) {
|
|
2078
|
+
const turn = this.getEngineTurn(row.id);
|
|
2079
|
+
const message = 'Provider dispatch outcome is unknown after Agent restart';
|
|
2080
|
+
this.db.prepare(`UPDATE engine_turns SET status = 'unknown', error = ?, updated_at = ?
|
|
2081
|
+
WHERE id = ? AND status = 'dispatching'`).run(message, now, turn.id);
|
|
2082
|
+
this.db.prepare(`UPDATE runs SET status = 'dispatch_unknown', accepting_input = 0,
|
|
2083
|
+
ended_at = ?, error = ? WHERE id = ? AND status = 'running'`).run(now, message, turn.runId);
|
|
2084
|
+
this.db.prepare(`UPDATE actions SET status = 'failed', current_run_id = NULL,
|
|
2085
|
+
updated_at = ? WHERE id = ? AND status = 'running' AND current_run_id = ?`).run(
|
|
2086
|
+
now, turn.actionId, turn.runId,
|
|
2087
|
+
);
|
|
2088
|
+
this.db.prepare(`UPDATE work_items SET status = 'needs_attention', current_action_id = ?,
|
|
2089
|
+
current_run_id = NULL, updated_at = ? WHERE id = ? AND status = 'running'`).run(
|
|
2090
|
+
turn.actionId, now, turn.workItemId,
|
|
2091
|
+
);
|
|
2092
|
+
this.appendEvent(turn.workItemId, 'engine_turn.dispatch_unknown', {
|
|
2093
|
+
turnId: turn.id, requestHash: turn.requestHash,
|
|
2094
|
+
dispatchAttempt: turn.dispatchAttempt, error: message,
|
|
2095
|
+
}, { actionId: turn.actionId, runId: turn.runId });
|
|
2096
|
+
}
|
|
2097
|
+
return recoverable.map(row => this.getEngineTurn(row.id));
|
|
2098
|
+
});
|
|
2099
|
+
}
|
|
2100
|
+
|
|
2101
|
+
acknowledgeActionInput(eventId, actionId, runId, ownerBootId, leaseEpoch) {
|
|
2102
|
+
const prepared = this.prepareEngineTurn(actionId, runId, ownerBootId, leaseEpoch, [eventId]);
|
|
2103
|
+
if (!prepared) return false;
|
|
2104
|
+
const claimed = this.claimEngineTurn(prepared.id, ownerBootId, leaseEpoch);
|
|
2105
|
+
return !!claimed && this.consumeEngineTurn(prepared.id, ownerBootId, leaseEpoch, {
|
|
2106
|
+
responseText: '', stopReason: 'legacy_acknowledge',
|
|
1314
2107
|
});
|
|
1315
2108
|
}
|
|
1316
2109
|
|
|
@@ -2175,8 +2968,85 @@ export class WorkItemStore {
|
|
|
2175
2968
|
return this.db.prepare(`SELECT * FROM events WHERE action_id = ? ORDER BY id`).all(actionId).map(mapEvent);
|
|
2176
2969
|
}
|
|
2177
2970
|
|
|
2971
|
+
getRecoverableCoordinatorTurns() {
|
|
2972
|
+
return this.db.prepare(`SELECT w.id AS work_item_id, c.payload FROM coordinator_mailbox_entries c
|
|
2973
|
+
JOIN work_items w ON w.id = c.work_item_id
|
|
2974
|
+
WHERE c.status IN ('pending', 'claimed')
|
|
2975
|
+
AND EXISTS (SELECT 1 FROM json_each(w.messages) message
|
|
2976
|
+
WHERE json_extract(message.value, '$.turnId') = json_extract(c.payload, '$.turnId')
|
|
2977
|
+
AND json_extract(message.value, '$.role') = 'assistant'
|
|
2978
|
+
AND json_extract(message.value, '$.status') = 'thinking')
|
|
2979
|
+
AND EXISTS (SELECT 1 FROM coordinator_provider_turns p
|
|
2980
|
+
WHERE p.coordinator_turn_id = json_extract(c.payload, '$.turnId')
|
|
2981
|
+
AND p.status IN ('prepared', 'dispatching', 'responded'))
|
|
2982
|
+
ORDER BY c.created_at`).all().map(row => ({
|
|
2983
|
+
workItemId: row.work_item_id,
|
|
2984
|
+
...parseJson(row.payload, {}),
|
|
2985
|
+
}));
|
|
2986
|
+
}
|
|
2987
|
+
|
|
2988
|
+
resumeCoordinatorTurn(workItemId, turnId, claim = {}) {
|
|
2989
|
+
const detail = this.getWorkItemDetail(workItemId);
|
|
2990
|
+
if (!detail || !this.#activeCoordinatorMailboxClaim(turnId, claim)) return null;
|
|
2991
|
+
const assistant = [...(detail.messages || [])].reverse().find(message => (
|
|
2992
|
+
message.turnId === turnId && message.role === 'assistant' && message.status === 'thinking'
|
|
2993
|
+
));
|
|
2994
|
+
if (!assistant) return null;
|
|
2995
|
+
return {
|
|
2996
|
+
turnId,
|
|
2997
|
+
detail,
|
|
2998
|
+
fence: {
|
|
2999
|
+
workItemId,
|
|
3000
|
+
revision: detail.revision,
|
|
3001
|
+
planRevision: detail.planRevision,
|
|
3002
|
+
ledgerRevision: detail.ledgerRevision,
|
|
3003
|
+
coordinatorRevision: detail.coordinatorRevision,
|
|
3004
|
+
status: detail.status,
|
|
3005
|
+
actionFence: coordinatorActionFence(
|
|
3006
|
+
(detail.actions || []).filter(action => !['completed', 'superseded', 'cancelled'].includes(action.status)),
|
|
3007
|
+
),
|
|
3008
|
+
recovery: assistant.recovery ? { ...assistant.recovery } : null,
|
|
3009
|
+
claim: {
|
|
3010
|
+
mailboxId: claim.mailboxId,
|
|
3011
|
+
ownerBootId: claim.ownerBootId,
|
|
3012
|
+
claimEpoch: Number(claim.claimEpoch),
|
|
3013
|
+
},
|
|
3014
|
+
},
|
|
3015
|
+
};
|
|
3016
|
+
}
|
|
3017
|
+
|
|
3018
|
+
claimStartedCoordinatorTurn(started, ownerBootId, leaseMs = 60_000) {
|
|
3019
|
+
if (!started?.turnId || !started?.detail?.id) return null;
|
|
3020
|
+
const claim = this.claimCoordinatorTurn(started.detail.id, started.turnId, ownerBootId, leaseMs);
|
|
3021
|
+
if (!claim) return null;
|
|
3022
|
+
return {
|
|
3023
|
+
...started,
|
|
3024
|
+
fence: {
|
|
3025
|
+
...started.fence,
|
|
3026
|
+
claim: {
|
|
3027
|
+
mailboxId: claim.mailboxId,
|
|
3028
|
+
ownerBootId: claim.ownerBootId,
|
|
3029
|
+
claimEpoch: claim.claimEpoch,
|
|
3030
|
+
},
|
|
3031
|
+
},
|
|
3032
|
+
};
|
|
3033
|
+
}
|
|
3034
|
+
|
|
2178
3035
|
beginCoordinatorTurn(id, text, expected = {}, options = {}) {
|
|
2179
3036
|
return withTransaction(this.db, () => {
|
|
3037
|
+
const clientMessageId = typeof options.clientMessageId === 'string' && options.clientMessageId
|
|
3038
|
+
? options.clientMessageId : null;
|
|
3039
|
+
if (clientMessageId) {
|
|
3040
|
+
const sourceKey = `client:message:${id}:${clientMessageId}`;
|
|
3041
|
+
const existingAction = this.db.prepare('SELECT 1 FROM action_entries WHERE source_key = ?').get(sourceKey);
|
|
3042
|
+
if (existingAction) throw new Error('clientMessageId already belongs to an Action message');
|
|
3043
|
+
const existing = this.db.prepare(`SELECT payload FROM coordinator_mailbox_entries
|
|
3044
|
+
WHERE source_key = ?`).get(sourceKey);
|
|
3045
|
+
if (existing) {
|
|
3046
|
+
const payload = parseJson(existing.payload, {});
|
|
3047
|
+
return { turnId: payload.turnId, detail: this.getWorkItemDetail(id), duplicate: true };
|
|
3048
|
+
}
|
|
3049
|
+
}
|
|
2180
3050
|
const workItem = this.getWorkItem(id);
|
|
2181
3051
|
if (!workItem) return null;
|
|
2182
3052
|
if (['done', 'cancelled'].includes(workItem.status)) {
|
|
@@ -2242,6 +3112,21 @@ export class WorkItemStore {
|
|
|
2242
3112
|
createdAt: now, updatedAt: now, decision: null,
|
|
2243
3113
|
...(recovery ? { recovery: { ...recovery } } : {}),
|
|
2244
3114
|
};
|
|
3115
|
+
if (userMessage) {
|
|
3116
|
+
this.#appendConversationEntry(
|
|
3117
|
+
id,
|
|
3118
|
+
userMessage,
|
|
3119
|
+
clientMessageId ? `client:conversation:${id}:${clientMessageId}` : `coordinator:turn:${turnId}:user`,
|
|
3120
|
+
);
|
|
3121
|
+
}
|
|
3122
|
+
this.#appendConversationEntry(id, assistantMessage, `coordinator:turn:${turnId}:assistant`);
|
|
3123
|
+
this.enqueueCoordinatorMailbox(id, automaticRecovery ? 'recovery' : 'message', {
|
|
3124
|
+
turnId,
|
|
3125
|
+
text,
|
|
3126
|
+
recovery,
|
|
3127
|
+
clientMessageId,
|
|
3128
|
+
addedAttachments: projectedAttachments,
|
|
3129
|
+
}, clientMessageId ? `client:message:${id}:${clientMessageId}` : `coordinator:turn:${turnId}`);
|
|
2245
3130
|
const messages = [...(workItem.messages || []), ...(userMessage ? [userMessage] : []), assistantMessage]
|
|
2246
3131
|
.slice(-100);
|
|
2247
3132
|
const coordinatorRevision = workItem.coordinatorRevision + 1;
|
|
@@ -2258,6 +3143,7 @@ export class WorkItemStore {
|
|
|
2258
3143
|
if (Number(changed.changes) !== 1) throw new Error('Coordinator turn lost its revision fence');
|
|
2259
3144
|
this.appendEvent(id, automaticRecovery ? 'coordinator.recovery_started' : 'coordinator.turn_started', {
|
|
2260
3145
|
turnId,
|
|
3146
|
+
clientMessageId,
|
|
2261
3147
|
status: 'thinking',
|
|
2262
3148
|
coordinatorRevision,
|
|
2263
3149
|
addedAttachmentCount: projectedAttachments.length,
|
|
@@ -2285,6 +3171,8 @@ export class WorkItemStore {
|
|
|
2285
3171
|
|
|
2286
3172
|
completeCoordinatorTurn(turnId, result, expected = {}) {
|
|
2287
3173
|
return withTransaction(this.db, () => {
|
|
3174
|
+
const claim = expected.claim || {};
|
|
3175
|
+
if (!this.#activeCoordinatorMailboxClaim(turnId, claim)) return null;
|
|
2288
3176
|
const workItem = this.getWorkItem(expected.workItemId);
|
|
2289
3177
|
if (!workItem) return null;
|
|
2290
3178
|
if (workItem.revision !== expected.revision
|
|
@@ -2355,11 +3243,6 @@ export class WorkItemStore {
|
|
|
2355
3243
|
|| this.db.prepare(`SELECT id FROM runs WHERE action_id = ? AND status = 'failed'
|
|
2356
3244
|
ORDER BY ended_at DESC, started_at DESC LIMIT 1`).get(action.id)?.id;
|
|
2357
3245
|
if (!resultRunId) throw new Error('Coordinator human request requires a failed Run');
|
|
2358
|
-
const changedRun = this.db.prepare(`UPDATE runs SET waiting_reason = ?
|
|
2359
|
-
WHERE id = ? AND action_id = ? AND status = 'failed'`).run(question, resultRunId, action.id);
|
|
2360
|
-
if (Number(changedRun.changes) !== 1) {
|
|
2361
|
-
throw new Error('Coordinator human request lost the failed Run fence');
|
|
2362
|
-
}
|
|
2363
3246
|
affectedActionIds = [action.id];
|
|
2364
3247
|
this.appendEvent(workItem.id, 'action.waiting', {
|
|
2365
3248
|
reason: question,
|
|
@@ -2488,6 +3371,14 @@ export class WorkItemStore {
|
|
|
2488
3371
|
affectedActionIds,
|
|
2489
3372
|
},
|
|
2490
3373
|
};
|
|
3374
|
+
this.#appendConversationEntry(
|
|
3375
|
+
workItem.id,
|
|
3376
|
+
messages[assistantIndex],
|
|
3377
|
+
`coordinator:turn:${turnId}:assistant`,
|
|
3378
|
+
);
|
|
3379
|
+
if (!this.ackCoordinatorMailbox(claim.mailboxId, claim.ownerBootId, claim.claimEpoch)) {
|
|
3380
|
+
throw new Error('Coordinator completion lost its mailbox claim');
|
|
3381
|
+
}
|
|
2491
3382
|
const current = this.getWorkItem(workItem.id);
|
|
2492
3383
|
const coordinatorRevision = current.coordinatorRevision + 1;
|
|
2493
3384
|
const changed = decision.kind === 'answer'
|
|
@@ -2518,6 +3409,11 @@ export class WorkItemStore {
|
|
|
2518
3409
|
|
|
2519
3410
|
failCoordinatorTurn(turnId, error, expected = {}) {
|
|
2520
3411
|
return withTransaction(this.db, () => {
|
|
3412
|
+
const claim = expected.claim || {};
|
|
3413
|
+
if (!this.#activeCoordinatorMailboxClaim(turnId, claim)) return null;
|
|
3414
|
+
const responded = this.db.prepare(`SELECT 1 AS present FROM coordinator_provider_turns
|
|
3415
|
+
WHERE coordinator_turn_id = ? AND status = 'responded'`).get(turnId);
|
|
3416
|
+
if (responded) return null;
|
|
2521
3417
|
const workItem = this.getWorkItem(expected.workItemId);
|
|
2522
3418
|
if (!workItem || workItem.coordinatorRevision !== expected.coordinatorRevision) return null;
|
|
2523
3419
|
const messages = [...(workItem.messages || [])];
|
|
@@ -2530,6 +3426,12 @@ export class WorkItemStore {
|
|
|
2530
3426
|
...messages[index], status: 'failed', updatedAt: now,
|
|
2531
3427
|
error: String(error?.message || error || 'Coordinator failed').slice(0, 8_000),
|
|
2532
3428
|
};
|
|
3429
|
+
this.#appendConversationEntry(
|
|
3430
|
+
workItem.id,
|
|
3431
|
+
messages[index],
|
|
3432
|
+
`coordinator:turn:${turnId}:assistant`,
|
|
3433
|
+
);
|
|
3434
|
+
if (!this.ackCoordinatorMailbox(claim.mailboxId, claim.ownerBootId, claim.claimEpoch)) return null;
|
|
2533
3435
|
const changed = this.db.prepare(`UPDATE work_items SET messages = ?, coordinator_revision = coordinator_revision + 1,
|
|
2534
3436
|
updated_at = ? WHERE id = ? AND coordinator_revision = ?`).run(
|
|
2535
3437
|
stringify(messages), now, workItem.id, workItem.coordinatorRevision,
|
|
@@ -2861,6 +3763,20 @@ export class WorkItemStore {
|
|
|
2861
3763
|
|
|
2862
3764
|
retryWorkItemAtomic(id, makeAction, options = {}) {
|
|
2863
3765
|
return withTransaction(this.db, () => {
|
|
3766
|
+
const clientMessageId = typeof options.inputEvent?.clientMessageId === 'string'
|
|
3767
|
+
? options.inputEvent.clientMessageId : null;
|
|
3768
|
+
if (clientMessageId) {
|
|
3769
|
+
const existing = this.db.prepare(`SELECT action_id FROM events WHERE work_item_id = ?
|
|
3770
|
+
AND type = 'action.input_added' AND json_extract(data, '$.clientMessageId') = ?`).get(
|
|
3771
|
+
id, clientMessageId,
|
|
3772
|
+
);
|
|
3773
|
+
if (existing) {
|
|
3774
|
+
if (existing.action_id !== options.inputEvent.targetActionId) {
|
|
3775
|
+
throw new Error('clientMessageId already belongs to another Action');
|
|
3776
|
+
}
|
|
3777
|
+
return this.getWorkItemDetail(id);
|
|
3778
|
+
}
|
|
3779
|
+
}
|
|
2864
3780
|
const workItem = this.getWorkItem(id);
|
|
2865
3781
|
if (!workItem) return null;
|
|
2866
3782
|
const graphMode = isGraphWorkItem(workItem);
|
|
@@ -2958,7 +3874,7 @@ export class WorkItemStore {
|
|
|
2958
3874
|
|
|
2959
3875
|
claimReadyAction(ownerBootId, leaseMs = 60_000) {
|
|
2960
3876
|
return withTransaction(this.db, () => {
|
|
2961
|
-
const
|
|
3877
|
+
const rows = this.db.prepare(`SELECT a.* FROM actions a
|
|
2962
3878
|
JOIN work_items w ON w.id = a.work_item_id
|
|
2963
3879
|
WHERE a.status = 'ready' AND a.current_run_id IS NULL
|
|
2964
3880
|
AND NOT EXISTS (
|
|
@@ -3004,6 +3920,12 @@ export class WorkItemStore {
|
|
|
3004
3920
|
OR (running.work_item_id != a.work_item_id
|
|
3005
3921
|
AND a.workspace_mode != 'read' AND running.workspace_mode != 'read'))
|
|
3006
3922
|
)
|
|
3923
|
+
AND NOT EXISTS (
|
|
3924
|
+
SELECT 1 FROM operations unsafe_operation
|
|
3925
|
+
WHERE unsafe_operation.work_item_id = w.id
|
|
3926
|
+
AND unsafe_operation.concurrency_policy = 'blocking'
|
|
3927
|
+
AND unsafe_operation.effect_status NOT IN ('applied', 'not_applied', 'failed_no_effect')
|
|
3928
|
+
)
|
|
3007
3929
|
AND NOT EXISTS (
|
|
3008
3930
|
SELECT 1 FROM runs deferred
|
|
3009
3931
|
WHERE deferred.action_id = a.id
|
|
@@ -3017,7 +3939,8 @@ export class WorkItemStore {
|
|
|
3017
3939
|
AND blocker_item.workspace_key = w.workspace_key
|
|
3018
3940
|
)
|
|
3019
3941
|
)
|
|
3020
|
-
ORDER BY a.updated_at ASC, a.sequence ASC
|
|
3942
|
+
ORDER BY a.updated_at ASC, a.sequence ASC`).all();
|
|
3943
|
+
const row = rows.find(candidate => !this.#hasBlockingOperation(candidate.work_item_id, candidate.id));
|
|
3021
3944
|
if (!row) return null;
|
|
3022
3945
|
const now = this.now();
|
|
3023
3946
|
let action = mapAction(row);
|
|
@@ -3034,6 +3957,9 @@ export class WorkItemStore {
|
|
|
3034
3957
|
action = promoteReadyActionInputs(this.db, action, readyInputs, now, 'run_claim');
|
|
3035
3958
|
const runId = randomUUID();
|
|
3036
3959
|
const leaseEpoch = Number(action.leaseEpoch) + 1;
|
|
3960
|
+
const priorOrdinal = this.db.prepare(`SELECT MAX(ordinal) AS value FROM runs
|
|
3961
|
+
WHERE action_id = ?`).get(action.id);
|
|
3962
|
+
const runOrdinal = Math.max(0, Number(priorOrdinal?.value) || 0) + 1;
|
|
3037
3963
|
const priorProgress = this.db.prepare(`SELECT MAX(progress_revision) AS value FROM runs
|
|
3038
3964
|
WHERE action_id = ?`).get(action.id);
|
|
3039
3965
|
const progressRevision = Math.max(0, Number(priorProgress?.value) || 0) + 1;
|
|
@@ -3061,14 +3987,15 @@ export class WorkItemStore {
|
|
|
3061
3987
|
);
|
|
3062
3988
|
if (Number(changedWorkItem.changes) !== 1) throw new Error('WorkItem claim lost its Action fence');
|
|
3063
3989
|
this.db.prepare(`INSERT INTO runs
|
|
3064
|
-
(id, action_id, work_item_id, owner_boot_id, lease_epoch, status, started_at,
|
|
3990
|
+
(id, action_id, work_item_id, owner_boot_id, lease_epoch, ordinal, status, started_at,
|
|
3065
3991
|
expires_at, evidence, progress_revision, action_generation, action_spec_hash, action_attempt)
|
|
3066
|
-
VALUES (?, ?, ?, ?, ?, 'running', ?, ?, '[]', ?, ?, ?, ?)`).run(
|
|
3992
|
+
VALUES (?, ?, ?, ?, ?, ?, 'running', ?, ?, '[]', ?, ?, ?, ?)`).run(
|
|
3067
3993
|
runId,
|
|
3068
3994
|
action.id,
|
|
3069
3995
|
action.workItemId,
|
|
3070
3996
|
ownerBootId,
|
|
3071
3997
|
leaseEpoch,
|
|
3998
|
+
runOrdinal,
|
|
3072
3999
|
now,
|
|
3073
4000
|
now + leaseMs,
|
|
3074
4001
|
progressRevision,
|
|
@@ -3583,6 +4510,9 @@ export class WorkItemStore {
|
|
|
3583
4510
|
}, this.#nextSequence(workItem.id), now);
|
|
3584
4511
|
}
|
|
3585
4512
|
let workItemStatus = transition.workItemStatus;
|
|
4513
|
+
if (workItemStatus === 'done' && this.#hasBlockingOperation(workItem.id)) {
|
|
4514
|
+
throw new Error('WorkItem has an unsafe blocking Operation and cannot complete');
|
|
4515
|
+
}
|
|
3586
4516
|
let currentActionId = nextAction?.id ?? (transition.keepCurrentAction ? action.id : null);
|
|
3587
4517
|
let changedWorkItem;
|
|
3588
4518
|
if (transition.planConflict) {
|
|
@@ -3662,6 +4592,9 @@ export class WorkItemStore {
|
|
|
3662
4592
|
|
|
3663
4593
|
recoverInterruptedCoordinatorTurns() {
|
|
3664
4594
|
return withTransaction(this.db, () => {
|
|
4595
|
+
const recoverableTurnIds = new Set(
|
|
4596
|
+
this.getRecoverableCoordinatorTurns().map(turn => turn.turnId),
|
|
4597
|
+
);
|
|
3665
4598
|
const now = this.now();
|
|
3666
4599
|
let recovered = 0;
|
|
3667
4600
|
for (const row of this.db.prepare(`SELECT id, messages, coordinator_revision FROM work_items
|
|
@@ -3670,6 +4603,7 @@ export class WorkItemStore {
|
|
|
3670
4603
|
const messages = parseJson(row.messages, []);
|
|
3671
4604
|
const index = messages.length - 1;
|
|
3672
4605
|
if (index < 0 || messages[index]?.role !== 'assistant' || messages[index]?.status !== 'thinking') continue;
|
|
4606
|
+
if (recoverableTurnIds.has(messages[index].turnId)) continue;
|
|
3673
4607
|
messages[index] = {
|
|
3674
4608
|
...messages[index],
|
|
3675
4609
|
status: 'failed',
|
|
@@ -3682,6 +4616,12 @@ export class WorkItemStore {
|
|
|
3682
4616
|
stringify(messages), now, row.id, row.coordinator_revision,
|
|
3683
4617
|
);
|
|
3684
4618
|
if (Number(changed.changes) !== 1) continue;
|
|
4619
|
+
const turnId = messages[index].turnId || null;
|
|
4620
|
+
if (turnId) {
|
|
4621
|
+
this.db.prepare(`UPDATE coordinator_mailbox_entries SET status = 'acked', acked_at = ?,
|
|
4622
|
+
claim_owner = NULL, claimed_at = NULL, lease_expires_at = NULL, updated_at = ?
|
|
4623
|
+
WHERE json_extract(payload, '$.turnId') = ? AND status != 'acked'`).run(now, now, turnId);
|
|
4624
|
+
}
|
|
3685
4625
|
this.appendEvent(row.id, 'coordinator.turn_interrupted', {
|
|
3686
4626
|
turnId: messages[index].turnId || null,
|
|
3687
4627
|
error: messages[index].error,
|