@yeaft/webchat-agent 1.0.294 → 1.0.296
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.js +1 -2
- package/local-runtime/server/handlers/client-work-center.js +1 -1
- package/local-runtime/version.json +1 -1
- package/local-runtime/web/app.bundle.js +181 -173
- package/local-runtime/web/app.bundle.js.gz +0 -0
- package/local-runtime/web/index.html +2 -2
- package/local-runtime/web/style.bundle.css +1 -1
- package/local-runtime/web/style.bundle.css.gz +0 -0
- package/package.json +1 -1
- package/yeaft/engine.js +27 -3
- package/yeaft/work-center/bridge.js +5 -1
- package/yeaft/work-center/controller.js +6 -2
- package/yeaft/work-center/coordinator.js +51 -11
- package/yeaft/work-center/durable-model.js +618 -0
- package/yeaft/work-center/projection.js +8 -1
- package/yeaft/work-center/runner.js +101 -15
- package/yeaft/work-center/service.js +55 -0
- package/yeaft/work-center/store.js +774 -23
|
@@ -6,8 +6,13 @@ import { normalizeEvidence } from './evidence.js';
|
|
|
6
6
|
import { normalizeActionCheckpoint } from './action-checkpoint.js';
|
|
7
7
|
import { currentActionInputEventIds, runMatchesActionIdentity } from './action-identity.js';
|
|
8
8
|
import { canonicalActionInstruction, withoutActionInputContext } from './workflow.js';
|
|
9
|
+
import {
|
|
10
|
+
WORK_CENTER_SCHEMA_VERSION,
|
|
11
|
+
durableId,
|
|
12
|
+
migrateDurableWorkCenterModel,
|
|
13
|
+
} from './durable-model.js';
|
|
9
14
|
|
|
10
|
-
const SCHEMA_VERSION =
|
|
15
|
+
const SCHEMA_VERSION = WORK_CENTER_SCHEMA_VERSION;
|
|
11
16
|
const UNFINISHED_ACTION_STATUSES = "'ready','running','waiting','failed'";
|
|
12
17
|
const MAX_REUSABLE_CONTEXT_ITEMS = 12;
|
|
13
18
|
const MAX_RUN_RESPONSE_CHARS = 65_536;
|
|
@@ -227,6 +232,9 @@ function mapRun(row) {
|
|
|
227
232
|
workItemId: row.work_item_id,
|
|
228
233
|
ownerBootId: row.owner_boot_id,
|
|
229
234
|
leaseEpoch: row.lease_epoch,
|
|
235
|
+
ordinal: Math.max(1, Number(row.ordinal) || 1),
|
|
236
|
+
terminalStatus: row.terminal_status || null,
|
|
237
|
+
terminalAt: row.terminal_at || null,
|
|
230
238
|
status: row.status,
|
|
231
239
|
startedAt: row.started_at,
|
|
232
240
|
expiresAt: row.expires_at,
|
|
@@ -774,6 +782,10 @@ export class WorkItemStore {
|
|
|
774
782
|
this.db.exec('PRAGMA synchronous = NORMAL;');
|
|
775
783
|
this.db.exec('PRAGMA foreign_keys = ON;');
|
|
776
784
|
this.#initSchema();
|
|
785
|
+
this.recoverCoordinatorMailbox();
|
|
786
|
+
this.recoverCoordinatorProviderTurns();
|
|
787
|
+
this.recoverOperations();
|
|
788
|
+
this.recoverEngineTurns();
|
|
777
789
|
this.recoverInterruptedCoordinatorTurns();
|
|
778
790
|
}
|
|
779
791
|
|
|
@@ -939,6 +951,9 @@ export class WorkItemStore {
|
|
|
939
951
|
const storedSchemaVersion = Number(
|
|
940
952
|
this.db.prepare("SELECT value FROM schema_meta WHERE key = 'schema_version'").get()?.value,
|
|
941
953
|
) || 0;
|
|
954
|
+
if (storedSchemaVersion > SCHEMA_VERSION) {
|
|
955
|
+
throw new Error(`Work Center database schema ${storedSchemaVersion} is newer than supported schema ${SCHEMA_VERSION}`);
|
|
956
|
+
}
|
|
942
957
|
|
|
943
958
|
// The feature shipped first as an unmerged PR, but keep the store tolerant
|
|
944
959
|
// of databases created by review builds.
|
|
@@ -1129,16 +1144,365 @@ export class WorkItemStore {
|
|
|
1129
1144
|
}
|
|
1130
1145
|
if (storedSchemaVersion < 20) reconcilePendingActionInputIdentity(this.db, this.now());
|
|
1131
1146
|
if (storedSchemaVersion < 22) repairReviewBuildActionInputIdentity(this.db, this.now());
|
|
1132
|
-
this.db.prepare(`INSERT INTO schema_meta(key, value) VALUES('schema_version', ?)
|
|
1133
|
-
ON CONFLICT(key) DO UPDATE SET value = excluded.value`).run(String(SCHEMA_VERSION));
|
|
1134
1147
|
});
|
|
1135
1148
|
}
|
|
1149
|
+
withTransaction(this.db, () => {
|
|
1150
|
+
migrateDurableWorkCenterModel(this.db, this.now(), storedSchemaVersion || SCHEMA_VERSION);
|
|
1151
|
+
this.db.prepare(`INSERT INTO schema_meta(key, value) VALUES('schema_version', ?)
|
|
1152
|
+
ON CONFLICT(key) DO UPDATE SET value = excluded.value`).run(String(SCHEMA_VERSION));
|
|
1153
|
+
});
|
|
1136
1154
|
}
|
|
1137
1155
|
|
|
1138
1156
|
close() {
|
|
1139
1157
|
this.db.close();
|
|
1140
1158
|
}
|
|
1141
1159
|
|
|
1160
|
+
#nextConversationSequence(conversationId) {
|
|
1161
|
+
return Number(this.db.prepare(`SELECT COALESCE(MAX(sequence), 0) + 1 AS value
|
|
1162
|
+
FROM conversation_entries WHERE conversation_id = ?`).get(conversationId)?.value) || 1;
|
|
1163
|
+
}
|
|
1164
|
+
|
|
1165
|
+
#nextActionEntrySequence(actionId) {
|
|
1166
|
+
return Number(this.db.prepare(`SELECT COALESCE(MAX(sequence), 0) + 1 AS value
|
|
1167
|
+
FROM action_entries WHERE action_id = ?`).get(actionId)?.value) || 1;
|
|
1168
|
+
}
|
|
1169
|
+
|
|
1170
|
+
#appendConversationEntry(workItemId, entry, sourceKey) {
|
|
1171
|
+
const now = this.now();
|
|
1172
|
+
const conversationId = `work-item:${workItemId}`;
|
|
1173
|
+
this.db.prepare(`INSERT INTO conversations
|
|
1174
|
+
(id, work_item_id, status, created_at, updated_at) VALUES (?, ?, 'active', ?, ?)
|
|
1175
|
+
ON CONFLICT(work_item_id) DO UPDATE SET updated_at = excluded.updated_at`).run(
|
|
1176
|
+
conversationId, workItemId, now, now,
|
|
1177
|
+
);
|
|
1178
|
+
const existing = this.db.prepare('SELECT id FROM conversation_entries WHERE source_key = ?').get(sourceKey);
|
|
1179
|
+
if (existing) {
|
|
1180
|
+
this.db.prepare(`UPDATE conversation_entries SET status = ?, text = ?, attachments = ?,
|
|
1181
|
+
payload = ?, updated_at = ? WHERE id = ?`).run(
|
|
1182
|
+
entry.status || 'completed',
|
|
1183
|
+
entry.text || '',
|
|
1184
|
+
stringify(Array.isArray(entry.attachments) ? entry.attachments : []),
|
|
1185
|
+
stringify(entry),
|
|
1186
|
+
Number(entry.updatedAt) || Number(entry.createdAt) || now,
|
|
1187
|
+
existing.id,
|
|
1188
|
+
);
|
|
1189
|
+
return existing.id;
|
|
1190
|
+
}
|
|
1191
|
+
const id = entry.id || durableId('conversation-entry');
|
|
1192
|
+
this.db.prepare(`INSERT INTO conversation_entries
|
|
1193
|
+
(id, conversation_id, work_item_id, sequence, kind, role, status, text, attachments,
|
|
1194
|
+
turn_id, source_key, payload, created_at, updated_at)
|
|
1195
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(
|
|
1196
|
+
id,
|
|
1197
|
+
conversationId,
|
|
1198
|
+
workItemId,
|
|
1199
|
+
this.#nextConversationSequence(conversationId),
|
|
1200
|
+
entry.kind || 'message',
|
|
1201
|
+
entry.role || null,
|
|
1202
|
+
entry.status || 'completed',
|
|
1203
|
+
entry.text || '',
|
|
1204
|
+
stringify(Array.isArray(entry.attachments) ? entry.attachments : []),
|
|
1205
|
+
entry.turnId || null,
|
|
1206
|
+
sourceKey,
|
|
1207
|
+
stringify(entry),
|
|
1208
|
+
Number(entry.createdAt) || now,
|
|
1209
|
+
Number(entry.updatedAt) || Number(entry.createdAt) || now,
|
|
1210
|
+
);
|
|
1211
|
+
return id;
|
|
1212
|
+
}
|
|
1213
|
+
|
|
1214
|
+
#appendActionEntry(input, sourceKey) {
|
|
1215
|
+
const append = () => {
|
|
1216
|
+
const existing = this.db.prepare('SELECT * FROM action_entries WHERE source_key = ?').get(sourceKey);
|
|
1217
|
+
if (existing) return existing;
|
|
1218
|
+
const now = this.now();
|
|
1219
|
+
const status = input.status || 'pending';
|
|
1220
|
+
const id = input.id || durableId('action-entry');
|
|
1221
|
+
this.db.prepare(`INSERT INTO action_entries
|
|
1222
|
+
(id, work_item_id, action_id, run_id, sequence, kind, role, status, text, attachments,
|
|
1223
|
+
source_key, payload, created_at, updated_at, consumed_at)
|
|
1224
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(
|
|
1225
|
+
id,
|
|
1226
|
+
input.workItemId,
|
|
1227
|
+
input.actionId,
|
|
1228
|
+
input.runId || null,
|
|
1229
|
+
this.#nextActionEntrySequence(input.actionId),
|
|
1230
|
+
input.kind || 'message',
|
|
1231
|
+
input.role || null,
|
|
1232
|
+
status,
|
|
1233
|
+
input.text || '',
|
|
1234
|
+
stringify(Array.isArray(input.attachments) ? input.attachments : []),
|
|
1235
|
+
sourceKey,
|
|
1236
|
+
stringify(input.payload || {}),
|
|
1237
|
+
Number(input.createdAt) || now,
|
|
1238
|
+
now,
|
|
1239
|
+
status === 'consumed' ? now : null,
|
|
1240
|
+
);
|
|
1241
|
+
return this.db.prepare('SELECT * FROM action_entries WHERE id = ?').get(id);
|
|
1242
|
+
};
|
|
1243
|
+
return this.db.isTransaction ? append() : withTransaction(this.db, append);
|
|
1244
|
+
}
|
|
1245
|
+
|
|
1246
|
+
appendActionControl(workItemId, actionId, command, options = {}) {
|
|
1247
|
+
if (!['start', 'pause', 'stop'].includes(command)) {
|
|
1248
|
+
throw new Error(`Unsupported Action control command: ${command}`);
|
|
1249
|
+
}
|
|
1250
|
+
const action = this.getAction(actionId);
|
|
1251
|
+
if (!action || action.workItemId !== workItemId) return null;
|
|
1252
|
+
const sourceKey = options.sourceKey || durableId(`action-control-${command}`);
|
|
1253
|
+
return this.#appendActionEntry({
|
|
1254
|
+
workItemId,
|
|
1255
|
+
actionId,
|
|
1256
|
+
runId: options.runId || action.currentRunId || null,
|
|
1257
|
+
kind: 'control',
|
|
1258
|
+
role: options.source || 'user',
|
|
1259
|
+
status: options.status || 'pending',
|
|
1260
|
+
text: options.reason || '',
|
|
1261
|
+
payload: { command, reason: options.reason || '', source: options.source || 'user' },
|
|
1262
|
+
createdAt: options.createdAt,
|
|
1263
|
+
}, sourceKey);
|
|
1264
|
+
}
|
|
1265
|
+
|
|
1266
|
+
enqueueCoordinatorMailbox(workItemId, kind, payload = {}, sourceKey = durableId('mailbox-source')) {
|
|
1267
|
+
const enqueue = () => {
|
|
1268
|
+
const existing = this.db.prepare(`SELECT * FROM coordinator_mailbox_entries
|
|
1269
|
+
WHERE source_key = ?`).get(sourceKey);
|
|
1270
|
+
if (existing) return existing;
|
|
1271
|
+
const now = this.now();
|
|
1272
|
+
const sequence = Number(this.db.prepare(`SELECT COALESCE(MAX(sequence), 0) + 1 AS value
|
|
1273
|
+
FROM coordinator_mailbox_entries WHERE work_item_id = ?`).get(workItemId)?.value) || 1;
|
|
1274
|
+
const id = durableId('coordinator-mailbox');
|
|
1275
|
+
this.db.prepare(`INSERT INTO coordinator_mailbox_entries
|
|
1276
|
+
(id, work_item_id, sequence, kind, status, source_key, payload, created_at, updated_at)
|
|
1277
|
+
VALUES (?, ?, ?, ?, 'pending', ?, ?, ?, ?)`).run(
|
|
1278
|
+
id, workItemId, sequence, kind, sourceKey, stringify(payload), now, now,
|
|
1279
|
+
);
|
|
1280
|
+
return this.db.prepare('SELECT * FROM coordinator_mailbox_entries WHERE id = ?').get(id);
|
|
1281
|
+
};
|
|
1282
|
+
return this.db.isTransaction ? enqueue() : withTransaction(this.db, enqueue);
|
|
1283
|
+
}
|
|
1284
|
+
|
|
1285
|
+
claimCoordinatorMailbox(workItemId, owner, leaseMs = 60_000) {
|
|
1286
|
+
return withTransaction(this.db, () => {
|
|
1287
|
+
const now = this.now();
|
|
1288
|
+
const row = this.db.prepare(`SELECT * FROM coordinator_mailbox_entries
|
|
1289
|
+
WHERE work_item_id = ? AND (status = 'pending'
|
|
1290
|
+
OR (status = 'claimed' AND lease_expires_at <= ?))
|
|
1291
|
+
ORDER BY sequence LIMIT 1`).get(workItemId, now);
|
|
1292
|
+
if (!row) return null;
|
|
1293
|
+
const changed = this.db.prepare(`UPDATE coordinator_mailbox_entries SET status = 'claimed',
|
|
1294
|
+
claim_owner = ?, claim_epoch = claim_epoch + 1, claimed_at = ?, lease_expires_at = ?,
|
|
1295
|
+
updated_at = ? WHERE id = ? AND claim_epoch = ?`).run(
|
|
1296
|
+
owner, now, now + leaseMs, now, row.id, row.claim_epoch,
|
|
1297
|
+
);
|
|
1298
|
+
if (Number(changed.changes) !== 1) return null;
|
|
1299
|
+
const claimed = this.db.prepare('SELECT * FROM coordinator_mailbox_entries WHERE id = ?').get(row.id);
|
|
1300
|
+
return { ...claimed, payload: parseJson(claimed.payload, {}) };
|
|
1301
|
+
});
|
|
1302
|
+
}
|
|
1303
|
+
|
|
1304
|
+
ackCoordinatorMailbox(id, owner, claimEpoch) {
|
|
1305
|
+
const now = this.now();
|
|
1306
|
+
const changed = this.db.prepare(`UPDATE coordinator_mailbox_entries SET status = 'acked',
|
|
1307
|
+
acked_at = ?, updated_at = ? WHERE id = ? AND status = 'claimed' AND claim_owner = ?
|
|
1308
|
+
AND claim_epoch = ?`).run(now, now, id, owner, claimEpoch);
|
|
1309
|
+
return Number(changed.changes) === 1;
|
|
1310
|
+
}
|
|
1311
|
+
|
|
1312
|
+
recoverCoordinatorMailbox() {
|
|
1313
|
+
const now = this.now();
|
|
1314
|
+
return Number(this.db.prepare(`UPDATE coordinator_mailbox_entries SET status = 'pending',
|
|
1315
|
+
claim_owner = NULL, claimed_at = NULL, lease_expires_at = NULL, updated_at = ?
|
|
1316
|
+
WHERE status = 'claimed' AND lease_expires_at <= ?`).run(now, now).changes);
|
|
1317
|
+
}
|
|
1318
|
+
|
|
1319
|
+
prepareCoordinatorProviderTurn(workItemId, coordinatorTurnId, attemptNumber, requestBody) {
|
|
1320
|
+
return withTransaction(this.db, () => {
|
|
1321
|
+
const existing = this.db.prepare(`SELECT * FROM coordinator_provider_turns
|
|
1322
|
+
WHERE coordinator_turn_id = ? AND attempt_number = ?`).get(coordinatorTurnId, attemptNumber);
|
|
1323
|
+
const requestHash = createHash('sha256').update(stableJson(requestBody), 'utf8').digest('hex');
|
|
1324
|
+
if (existing) {
|
|
1325
|
+
if (existing.request_hash !== requestHash) {
|
|
1326
|
+
throw new Error('Prepared Coordinator provider request changed before dispatch');
|
|
1327
|
+
}
|
|
1328
|
+
return this.#mapCoordinatorProviderTurn(existing);
|
|
1329
|
+
}
|
|
1330
|
+
const now = this.now();
|
|
1331
|
+
const id = durableId('coordinator-provider-turn');
|
|
1332
|
+
this.db.prepare(`INSERT INTO coordinator_provider_turns
|
|
1333
|
+
(id, work_item_id, coordinator_turn_id, attempt_number, status, request_body, request_hash,
|
|
1334
|
+
prepared_at, updated_at) VALUES (?, ?, ?, ?, 'prepared', ?, ?, ?, ?)`).run(
|
|
1335
|
+
id, workItemId, coordinatorTurnId, attemptNumber,
|
|
1336
|
+
stringify(requestBody), requestHash, now, now,
|
|
1337
|
+
);
|
|
1338
|
+
return this.getCoordinatorProviderTurn(id);
|
|
1339
|
+
});
|
|
1340
|
+
}
|
|
1341
|
+
|
|
1342
|
+
#mapCoordinatorProviderTurn(row) {
|
|
1343
|
+
return {
|
|
1344
|
+
id: row.id,
|
|
1345
|
+
workItemId: row.work_item_id,
|
|
1346
|
+
coordinatorTurnId: row.coordinator_turn_id,
|
|
1347
|
+
attemptNumber: Number(row.attempt_number),
|
|
1348
|
+
status: row.status,
|
|
1349
|
+
requestBody: parseJson(row.request_body, {}),
|
|
1350
|
+
requestHash: row.request_hash,
|
|
1351
|
+
response: parseJson(row.response, null),
|
|
1352
|
+
responseHash: row.response_hash || null,
|
|
1353
|
+
error: row.error || null,
|
|
1354
|
+
};
|
|
1355
|
+
}
|
|
1356
|
+
|
|
1357
|
+
getCoordinatorProviderTurn(id) {
|
|
1358
|
+
const row = this.db.prepare('SELECT * FROM coordinator_provider_turns WHERE id = ?').get(id);
|
|
1359
|
+
return row ? this.#mapCoordinatorProviderTurn(row) : null;
|
|
1360
|
+
}
|
|
1361
|
+
|
|
1362
|
+
dispatchCoordinatorProviderTurn(id) {
|
|
1363
|
+
const existing = this.getCoordinatorProviderTurn(id);
|
|
1364
|
+
if (existing?.status === 'dispatching') return existing;
|
|
1365
|
+
const now = this.now();
|
|
1366
|
+
const changed = this.db.prepare(`UPDATE coordinator_provider_turns SET status = 'dispatching',
|
|
1367
|
+
dispatched_at = ?, updated_at = ? WHERE id = ? AND status = 'prepared'`).run(now, now, id);
|
|
1368
|
+
return Number(changed.changes) === 1 ? this.getCoordinatorProviderTurn(id) : null;
|
|
1369
|
+
}
|
|
1370
|
+
|
|
1371
|
+
respondCoordinatorProviderTurn(id, requestHash, response) {
|
|
1372
|
+
const now = this.now();
|
|
1373
|
+
const responseHash = createHash('sha256').update(stableJson(response), 'utf8').digest('hex');
|
|
1374
|
+
const changed = this.db.prepare(`UPDATE coordinator_provider_turns SET status = 'responded',
|
|
1375
|
+
response = ?, response_hash = ?, responded_at = ?, updated_at = ?
|
|
1376
|
+
WHERE id = ? AND status = 'dispatching' AND request_hash = ?`).run(
|
|
1377
|
+
stringify(response), responseHash, now, now, id, requestHash,
|
|
1378
|
+
);
|
|
1379
|
+
return Number(changed.changes) === 1 ? this.getCoordinatorProviderTurn(id) : null;
|
|
1380
|
+
}
|
|
1381
|
+
|
|
1382
|
+
recoverCoordinatorProviderTurns() {
|
|
1383
|
+
const now = this.now();
|
|
1384
|
+
return Number(this.db.prepare(`UPDATE coordinator_provider_turns SET status = 'unknown',
|
|
1385
|
+
error = 'Coordinator provider dispatch outcome is unknown after Agent restart', updated_at = ?
|
|
1386
|
+
WHERE status = 'dispatching'`).run(now).changes);
|
|
1387
|
+
}
|
|
1388
|
+
|
|
1389
|
+
createOperation(input = {}) {
|
|
1390
|
+
const replayPolicy = input.replayPolicy || 'never_automatic';
|
|
1391
|
+
if (!input.idempotencyKey) throw new Error('Operation idempotencyKey is required');
|
|
1392
|
+
if (!['safe', 'probe_first', 'never_automatic'].includes(replayPolicy)) {
|
|
1393
|
+
throw new Error(`Unsupported Operation replay policy: ${replayPolicy}`);
|
|
1394
|
+
}
|
|
1395
|
+
const now = this.now();
|
|
1396
|
+
const id = input.id || durableId('operation');
|
|
1397
|
+
this.db.prepare(`INSERT INTO operations
|
|
1398
|
+
(id, work_item_id, action_id, run_id, engine_turn_id, operation_type, idempotency_key,
|
|
1399
|
+
replay_policy, effect_status, execution_status, payload, created_at, updated_at)
|
|
1400
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1401
|
+
ON CONFLICT(idempotency_key) DO NOTHING`).run(
|
|
1402
|
+
id, input.workItemId, input.actionId || null, input.runId || null, input.engineTurnId || null,
|
|
1403
|
+
input.operationType || 'unknown', input.idempotencyKey, replayPolicy,
|
|
1404
|
+
input.effectStatus || 'pending', input.executionStatus || 'not_started',
|
|
1405
|
+
stringify(input.payload || {}), now, now,
|
|
1406
|
+
);
|
|
1407
|
+
return this.getOperationByKey(input.idempotencyKey);
|
|
1408
|
+
}
|
|
1409
|
+
|
|
1410
|
+
getOperation(id) {
|
|
1411
|
+
const row = this.db.prepare('SELECT * FROM operations WHERE id = ?').get(id);
|
|
1412
|
+
return row ? this.#mapOperation(row) : null;
|
|
1413
|
+
}
|
|
1414
|
+
|
|
1415
|
+
getOperationByKey(idempotencyKey) {
|
|
1416
|
+
const row = this.db.prepare('SELECT * FROM operations WHERE idempotency_key = ?').get(idempotencyKey);
|
|
1417
|
+
return row ? this.#mapOperation(row) : null;
|
|
1418
|
+
}
|
|
1419
|
+
|
|
1420
|
+
#mapOperation(row) {
|
|
1421
|
+
return {
|
|
1422
|
+
id: row.id,
|
|
1423
|
+
workItemId: row.work_item_id,
|
|
1424
|
+
actionId: row.action_id || null,
|
|
1425
|
+
runId: row.run_id || null,
|
|
1426
|
+
engineTurnId: row.engine_turn_id || null,
|
|
1427
|
+
operationType: row.operation_type,
|
|
1428
|
+
idempotencyKey: row.idempotency_key,
|
|
1429
|
+
replayPolicy: row.replay_policy,
|
|
1430
|
+
concurrencyPolicy: row.concurrency_policy,
|
|
1431
|
+
effectStatus: row.effect_status,
|
|
1432
|
+
executionStatus: row.execution_status,
|
|
1433
|
+
effectCutoff: parseJson(row.effect_cutoff, null),
|
|
1434
|
+
grantManifest: parseJson(row.grant_manifest, {}),
|
|
1435
|
+
resourceRelease: parseJson(row.resource_release, {}),
|
|
1436
|
+
supplementalInventory: parseJson(row.supplemental_inventory, {}),
|
|
1437
|
+
payload: parseJson(row.payload, {}),
|
|
1438
|
+
result: parseJson(row.result, null),
|
|
1439
|
+
};
|
|
1440
|
+
}
|
|
1441
|
+
|
|
1442
|
+
operationSafeToProceed(operationOrKey) {
|
|
1443
|
+
const operation = typeof operationOrKey === 'string'
|
|
1444
|
+
? this.getOperationByKey(operationOrKey) : operationOrKey;
|
|
1445
|
+
if (!operation) return false;
|
|
1446
|
+
const effectSafe = ['applied', 'not_applied', 'failed_no_effect'].includes(operation.effectStatus);
|
|
1447
|
+
const executionSafe = ['not_started', 'quiescent'].includes(operation.executionStatus)
|
|
1448
|
+
|| (operation.executionStatus === 'fenced' && operation.effectCutoff?.status === 'current');
|
|
1449
|
+
const manifest = operation.grantManifest || {};
|
|
1450
|
+
const grantSafe = manifest.status === 'closed' && manifest.safetyStatus === 'current'
|
|
1451
|
+
&& manifest.inventoryComplete === true
|
|
1452
|
+
&& (manifest.pendingGrantAttemptIds || []).length === 0
|
|
1453
|
+
&& (manifest.authorityClosures || []).every(closure => closure.status === 'closed');
|
|
1454
|
+
const release = operation.resourceRelease || {};
|
|
1455
|
+
const resourcesSafe = release.status === 'released'
|
|
1456
|
+
&& (release.leases || []).every(lease => ['released', 'expired'].includes(lease.status));
|
|
1457
|
+
const supplemental = operation.supplementalInventory || {};
|
|
1458
|
+
const supplementalSafe = ['clear', 'resolved'].includes(supplemental.status || 'clear');
|
|
1459
|
+
return effectSafe && executionSafe && grantSafe && resourcesSafe && supplementalSafe;
|
|
1460
|
+
}
|
|
1461
|
+
|
|
1462
|
+
#hasBlockingOperation(workItemId, actionId = null) {
|
|
1463
|
+
const rows = this.db.prepare(`SELECT idempotency_key FROM operations
|
|
1464
|
+
WHERE work_item_id = ? AND concurrency_policy = 'blocking'
|
|
1465
|
+
AND (? IS NULL OR action_id IS NULL OR action_id = ?)`)
|
|
1466
|
+
.all(workItemId, actionId, actionId);
|
|
1467
|
+
return rows.some(row => !this.operationSafeToProceed(row.idempotency_key));
|
|
1468
|
+
}
|
|
1469
|
+
|
|
1470
|
+
claimOperation(idempotencyKey, ownerBootId, leaseEpoch, automatic = true) {
|
|
1471
|
+
return withTransaction(this.db, () => {
|
|
1472
|
+
const operation = this.getOperationByKey(idempotencyKey);
|
|
1473
|
+
if (!operation || operation.executionStatus !== 'not_started') return null;
|
|
1474
|
+
if (automatic && operation.replayPolicy !== 'safe') return null;
|
|
1475
|
+
const now = this.now();
|
|
1476
|
+
const changed = this.db.prepare(`UPDATE operations SET execution_status = 'running',
|
|
1477
|
+
owner_boot_id = ?, owner_lease_epoch = ?, claimed_at = ?, updated_at = ?
|
|
1478
|
+
WHERE idempotency_key = ? AND execution_status = 'not_started'`).run(
|
|
1479
|
+
ownerBootId, leaseEpoch, now, now, idempotencyKey,
|
|
1480
|
+
);
|
|
1481
|
+
return Number(changed.changes) === 1 ? this.getOperationByKey(idempotencyKey) : null;
|
|
1482
|
+
});
|
|
1483
|
+
}
|
|
1484
|
+
|
|
1485
|
+
completeOperation(idempotencyKey, ownerBootId, leaseEpoch, effectStatus, result = null) {
|
|
1486
|
+
if (!['applied', 'not_applied', 'failed_no_effect', 'unknown'].includes(effectStatus)) return false;
|
|
1487
|
+
const now = this.now();
|
|
1488
|
+
const changed = this.db.prepare(`UPDATE operations SET effect_status = ?, execution_status = ?,
|
|
1489
|
+
effect_cutoff = ?, result = ?, completed_at = ?, updated_at = ? WHERE idempotency_key = ?
|
|
1490
|
+
AND execution_status = 'running' AND owner_boot_id = ? AND owner_lease_epoch = ?`).run(
|
|
1491
|
+
effectStatus,
|
|
1492
|
+
'quiescent',
|
|
1493
|
+
stringify({ status: 'current', closureType: 'quiescent', closedAt: now }),
|
|
1494
|
+
stringify(result), now, now, idempotencyKey, ownerBootId, leaseEpoch,
|
|
1495
|
+
);
|
|
1496
|
+
return Number(changed.changes) === 1;
|
|
1497
|
+
}
|
|
1498
|
+
|
|
1499
|
+
recoverOperations() {
|
|
1500
|
+
const now = this.now();
|
|
1501
|
+
return Number(this.db.prepare(`UPDATE operations SET execution_status = 'hazardous_orphan',
|
|
1502
|
+
effect_status = CASE WHEN effect_status = 'pending' THEN 'unknown' ELSE effect_status END,
|
|
1503
|
+
updated_at = ? WHERE execution_status IN ('running', 'cancel_requested')`).run(now).changes);
|
|
1504
|
+
}
|
|
1505
|
+
|
|
1142
1506
|
appendEvent(workItemId, type, data = {}, refs = {}) {
|
|
1143
1507
|
const actionGeneration = refs.actionGeneration
|
|
1144
1508
|
?? (refs.actionId ? this.getAction(refs.actionId)?.generation : null)
|
|
@@ -1157,8 +1521,30 @@ export class WorkItemStore {
|
|
|
1157
1521
|
return Number(result.lastInsertRowid);
|
|
1158
1522
|
}
|
|
1159
1523
|
|
|
1160
|
-
|
|
1524
|
+
hasActionInputClientMessage(workItemId, actionId, clientMessageId) {
|
|
1525
|
+
if (typeof clientMessageId !== 'string' || !clientMessageId) return false;
|
|
1526
|
+
const sourceKey = `client:message:${workItemId}:${clientMessageId}`;
|
|
1527
|
+
const coordinatorReceipt = this.db.prepare(`SELECT 1 FROM coordinator_mailbox_entries
|
|
1528
|
+
WHERE source_key = ?`).get(sourceKey);
|
|
1529
|
+
if (coordinatorReceipt) throw new Error('clientMessageId already belongs to a Coordinator message');
|
|
1530
|
+
const entry = this.db.prepare('SELECT action_id FROM action_entries WHERE source_key = ?').get(sourceKey)
|
|
1531
|
+
|| this.db.prepare(`SELECT action_id FROM events WHERE work_item_id = ?
|
|
1532
|
+
AND type = 'action.input_added' AND json_extract(data, '$.clientMessageId') = ?`).get(
|
|
1533
|
+
workItemId, clientMessageId,
|
|
1534
|
+
);
|
|
1535
|
+
if (!entry) return false;
|
|
1536
|
+
if (entry.action_id !== actionId) throw new Error('clientMessageId already belongs to another Action');
|
|
1537
|
+
return true;
|
|
1538
|
+
}
|
|
1539
|
+
|
|
1540
|
+
addActionInput(id, input, expected, attachments = null, addedAttachments = [], clientMessageId = null) {
|
|
1161
1541
|
return withTransaction(this.db, () => {
|
|
1542
|
+
const sourceKey = typeof clientMessageId === 'string' && clientMessageId
|
|
1543
|
+
? `client:message:${id}:${clientMessageId}` : null;
|
|
1544
|
+
if (sourceKey) {
|
|
1545
|
+
const existing = this.db.prepare('SELECT id FROM action_entries WHERE source_key = ?').get(sourceKey);
|
|
1546
|
+
if (existing) return this.getWorkItemDetail(id);
|
|
1547
|
+
}
|
|
1162
1548
|
const workItem = this.getWorkItem(id);
|
|
1163
1549
|
if (!workItem) return null;
|
|
1164
1550
|
const expectedGeneration = Number(expected.generation);
|
|
@@ -1282,6 +1668,31 @@ export class WorkItemStore {
|
|
|
1282
1668
|
input,
|
|
1283
1669
|
stringify(projectedAttachments),
|
|
1284
1670
|
);
|
|
1671
|
+
this.#appendActionEntry({
|
|
1672
|
+
workItemId: id,
|
|
1673
|
+
actionId: action.id,
|
|
1674
|
+
runId: action.currentRunId,
|
|
1675
|
+
kind: 'message',
|
|
1676
|
+
role: 'user',
|
|
1677
|
+
status: 'pending',
|
|
1678
|
+
text: input,
|
|
1679
|
+
attachments: projectedAttachments,
|
|
1680
|
+
payload: { eventId, inputId, actionGeneration: eventGeneration, actionSpecHash: eventSpecHash },
|
|
1681
|
+
createdAt: now,
|
|
1682
|
+
}, sourceKey || `pending_action_inputs:event:${eventId}`);
|
|
1683
|
+
} else if (sourceKey) {
|
|
1684
|
+
this.#appendActionEntry({
|
|
1685
|
+
workItemId: id,
|
|
1686
|
+
actionId: action.id,
|
|
1687
|
+
runId: null,
|
|
1688
|
+
kind: 'message',
|
|
1689
|
+
role: 'user',
|
|
1690
|
+
status: 'consumed',
|
|
1691
|
+
text: input,
|
|
1692
|
+
attachments: projectedAttachments,
|
|
1693
|
+
payload: { eventId, inputId, actionGeneration: eventGeneration, actionSpecHash: eventSpecHash },
|
|
1694
|
+
createdAt: now,
|
|
1695
|
+
}, sourceKey);
|
|
1285
1696
|
}
|
|
1286
1697
|
return this.getWorkItemDetail(id);
|
|
1287
1698
|
});
|
|
@@ -1290,27 +1701,242 @@ export class WorkItemStore {
|
|
|
1290
1701
|
listPendingActionInputs(actionId, runId, ownerBootId, leaseEpoch) {
|
|
1291
1702
|
const active = this.#activeRunRow(runId, ownerBootId, leaseEpoch, true);
|
|
1292
1703
|
if (!active || active.action_id !== actionId) return [];
|
|
1293
|
-
return this.db.prepare(`SELECT
|
|
1294
|
-
|
|
1295
|
-
|
|
1704
|
+
return this.db.prepare(`SELECT p.*, ae.id AS action_entry_id, ae.sequence AS action_entry_sequence
|
|
1705
|
+
FROM pending_action_inputs p
|
|
1706
|
+
LEFT JOIN action_entries ae ON CAST(json_extract(ae.payload, '$.eventId') AS INTEGER) = p.event_id
|
|
1707
|
+
AND ae.action_id = p.action_id
|
|
1708
|
+
WHERE p.action_id = ? AND p.run_id = ? AND p.action_generation = ? AND p.action_spec_hash = ?
|
|
1709
|
+
AND p.consumed_at IS NULL AND p.superseded_at IS NULL ORDER BY p.event_id`).all(
|
|
1296
1710
|
actionId, runId, active.action_generation, active.action_spec_hash,
|
|
1297
1711
|
).map(row => ({
|
|
1298
1712
|
id: String(row.event_id),
|
|
1713
|
+
actionEntryId: row.action_entry_id || null,
|
|
1714
|
+
sequence: row.action_entry_sequence == null ? null : Number(row.action_entry_sequence),
|
|
1299
1715
|
text: row.text || '',
|
|
1300
1716
|
attachments: parseJson(row.attachments, []),
|
|
1301
1717
|
}));
|
|
1302
1718
|
}
|
|
1303
1719
|
|
|
1304
|
-
|
|
1720
|
+
prepareEngineTurn(actionId, runId, ownerBootId, leaseEpoch, inputs = [], request = {}) {
|
|
1305
1721
|
return withTransaction(this.db, () => {
|
|
1306
1722
|
const active = this.#activeRunRow(runId, ownerBootId, leaseEpoch, true);
|
|
1307
|
-
if (!active || active.action_id !== actionId) return
|
|
1308
|
-
const
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1723
|
+
if (!active || active.action_id !== actionId) return null;
|
|
1724
|
+
const eventIds = inputs.map(input => Number(input?.id ?? input)).filter(Number.isInteger);
|
|
1725
|
+
const normalizedEventIds = [...new Set(eventIds)].sort((left, right) => left - right);
|
|
1726
|
+
const reusable = this.db.prepare(`SELECT id FROM engine_turns
|
|
1727
|
+
WHERE run_id = ? AND status = 'prepared' ORDER BY ordinal DESC LIMIT 1`).get(runId);
|
|
1728
|
+
if (reusable) {
|
|
1729
|
+
const turn = this.getEngineTurn(reusable.id);
|
|
1730
|
+
const requestBody = request.requestBody && typeof request.requestBody === 'object'
|
|
1731
|
+
? request.requestBody : {};
|
|
1732
|
+
const requestHash = createHash('sha256').update(stableJson(requestBody), 'utf8').digest('hex');
|
|
1733
|
+
if (turn.requestHash !== requestHash) {
|
|
1734
|
+
throw new Error('Prepared EngineTurn request changed before provider dispatch');
|
|
1735
|
+
}
|
|
1736
|
+
return turn;
|
|
1737
|
+
}
|
|
1738
|
+
const entries = [];
|
|
1739
|
+
for (const eventId of normalizedEventIds) {
|
|
1740
|
+
const row = this.db.prepare(`SELECT p.*, ae.id AS action_entry_id,
|
|
1741
|
+
ae.sequence AS action_entry_sequence FROM pending_action_inputs p
|
|
1742
|
+
JOIN action_entries ae ON CAST(json_extract(ae.payload, '$.eventId') AS INTEGER) = p.event_id
|
|
1743
|
+
AND ae.action_id = p.action_id
|
|
1744
|
+
WHERE p.event_id = ? AND p.action_id = ? AND p.run_id = ? AND p.action_generation = ?
|
|
1745
|
+
AND p.action_spec_hash = ? AND p.consumed_at IS NULL AND p.superseded_at IS NULL
|
|
1746
|
+
AND ae.status = 'pending' AND ae.engine_turn_id IS NULL`).get(
|
|
1747
|
+
eventId, actionId, runId, active.action_generation, active.action_spec_hash,
|
|
1748
|
+
);
|
|
1749
|
+
if (row) entries.push(row);
|
|
1750
|
+
}
|
|
1751
|
+
if (entries.length !== normalizedEventIds.length) return null;
|
|
1752
|
+
const requestBody = request.requestBody && typeof request.requestBody === 'object'
|
|
1753
|
+
? request.requestBody
|
|
1754
|
+
: { actionEntryIds: entries.map(row => row.action_entry_id) };
|
|
1755
|
+
const requestHash = createHash('sha256').update(stableJson(requestBody), 'utf8').digest('hex');
|
|
1756
|
+
const ordinal = Number(this.db.prepare(`SELECT COALESCE(MAX(ordinal), 0) + 1 AS value
|
|
1757
|
+
FROM engine_turns WHERE run_id = ?`).get(runId)?.value) || 1;
|
|
1758
|
+
const turnId = durableId('engine-turn');
|
|
1759
|
+
const requestKey = `run:${runId}:turn:${ordinal}`;
|
|
1760
|
+
const claimedThroughSequence = Math.max(
|
|
1761
|
+
0,
|
|
1762
|
+
...entries.map(row => Number(row.action_entry_sequence) || 0),
|
|
1312
1763
|
);
|
|
1313
|
-
|
|
1764
|
+
const entryIds = entries.map(row => row.action_entry_id);
|
|
1765
|
+
const now = this.now();
|
|
1766
|
+
this.db.prepare(`INSERT INTO engine_turns
|
|
1767
|
+
(id, work_item_id, action_id, run_id, ordinal, status, owner_boot_id, lease_epoch,
|
|
1768
|
+
input_entry_ids, message_entry_ids, claimed_through_sequence, request_body, request_hash,
|
|
1769
|
+
request_key, dispatch_capability, created_at, updated_at)
|
|
1770
|
+
VALUES (?, ?, ?, ?, ?, 'prepared', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(
|
|
1771
|
+
turnId, active.work_item_id, actionId, runId, ordinal, ownerBootId, leaseEpoch,
|
|
1772
|
+
stringify(entryIds), stringify(entryIds), claimedThroughSequence, stringify(requestBody),
|
|
1773
|
+
requestHash, requestKey, request.dispatchCapability || 'unknown', now, now,
|
|
1774
|
+
);
|
|
1775
|
+
const bind = this.db.prepare(`UPDATE action_entries SET status = 'bound', engine_turn_id = ?,
|
|
1776
|
+
run_id = ?, updated_at = ? WHERE id = ? AND status = 'pending' AND engine_turn_id IS NULL`);
|
|
1777
|
+
for (const row of entries) {
|
|
1778
|
+
const changed = bind.run(turnId, runId, now, row.action_entry_id);
|
|
1779
|
+
if (Number(changed.changes) !== 1) throw new Error('ActionEntry changed before EngineTurn prepare');
|
|
1780
|
+
}
|
|
1781
|
+
this.appendEvent(active.work_item_id, 'engine_turn.prepared', {
|
|
1782
|
+
turnId, ordinal, requestHash, actionEntryIds: entryIds,
|
|
1783
|
+
}, { actionId, runId });
|
|
1784
|
+
return this.getEngineTurn(turnId);
|
|
1785
|
+
});
|
|
1786
|
+
}
|
|
1787
|
+
|
|
1788
|
+
getEngineTurn(turnId) {
|
|
1789
|
+
const row = this.db.prepare('SELECT * FROM engine_turns WHERE id = ?').get(turnId);
|
|
1790
|
+
if (!row) return null;
|
|
1791
|
+
return {
|
|
1792
|
+
id: row.id,
|
|
1793
|
+
workItemId: row.work_item_id,
|
|
1794
|
+
actionId: row.action_id,
|
|
1795
|
+
runId: row.run_id,
|
|
1796
|
+
ordinal: Number(row.ordinal),
|
|
1797
|
+
status: row.status,
|
|
1798
|
+
ownerBootId: row.owner_boot_id,
|
|
1799
|
+
leaseEpoch: Number(row.lease_epoch),
|
|
1800
|
+
inputEntryIds: parseJson(row.input_entry_ids, []),
|
|
1801
|
+
requestBody: parseJson(row.request_body, {}),
|
|
1802
|
+
requestHash: row.request_hash,
|
|
1803
|
+
requestKey: row.request_key,
|
|
1804
|
+
dispatchAttempt: Number(row.dispatch_attempt) || 0,
|
|
1805
|
+
dispatchCapability: row.dispatch_capability || 'unknown',
|
|
1806
|
+
claimedAt: row.claimed_at || null,
|
|
1807
|
+
dispatchedAt: row.dispatched_at || null,
|
|
1808
|
+
response: parseJson(row.response, null),
|
|
1809
|
+
responseHash: row.response_hash || null,
|
|
1810
|
+
consumedAt: row.consumed_at || null,
|
|
1811
|
+
error: row.error || null,
|
|
1812
|
+
};
|
|
1813
|
+
}
|
|
1814
|
+
|
|
1815
|
+
claimEngineTurn(turnId, ownerBootId, leaseEpoch) {
|
|
1816
|
+
return withTransaction(this.db, () => {
|
|
1817
|
+
const turn = this.getEngineTurn(turnId);
|
|
1818
|
+
if (!turn || turn.ownerBootId !== ownerBootId || turn.leaseEpoch !== leaseEpoch) return null;
|
|
1819
|
+
if (!this.#activeRunRow(turn.runId, ownerBootId, leaseEpoch, true)) return null;
|
|
1820
|
+
if (turn.status === 'dispatching') return turn;
|
|
1821
|
+
if (turn.status !== 'prepared') return null;
|
|
1822
|
+
const now = this.now();
|
|
1823
|
+
const changed = this.db.prepare(`UPDATE engine_turns SET status = 'dispatching',
|
|
1824
|
+
dispatch_attempt = dispatch_attempt + 1, claimed_at = COALESCE(claimed_at, ?),
|
|
1825
|
+
dispatched_at = ?, updated_at = ? WHERE id = ? AND status = 'prepared'`).run(
|
|
1826
|
+
now, now, now, turnId,
|
|
1827
|
+
);
|
|
1828
|
+
if (Number(changed.changes) !== 1) return null;
|
|
1829
|
+
this.appendEvent(turn.workItemId, 'engine_turn.dispatching', {
|
|
1830
|
+
turnId, requestHash: turn.requestHash, dispatchAttempt: turn.dispatchAttempt + 1,
|
|
1831
|
+
}, { actionId: turn.actionId, runId: turn.runId });
|
|
1832
|
+
return this.getEngineTurn(turnId);
|
|
1833
|
+
});
|
|
1834
|
+
}
|
|
1835
|
+
|
|
1836
|
+
consumeEngineTurn(turnId, ownerBootId, leaseEpoch, result = {}) {
|
|
1837
|
+
return withTransaction(this.db, () => {
|
|
1838
|
+
const turn = this.getEngineTurn(turnId);
|
|
1839
|
+
if (!turn || turn.status !== 'dispatching' || turn.ownerBootId !== ownerBootId
|
|
1840
|
+
|| turn.leaseEpoch !== leaseEpoch) return false;
|
|
1841
|
+
const now = this.now();
|
|
1842
|
+
const response = {
|
|
1843
|
+
text: String(result.responseText || ''),
|
|
1844
|
+
stopReason: result.stopReason || null,
|
|
1845
|
+
toolCalls: Array.isArray(result.toolCalls) ? result.toolCalls : [],
|
|
1846
|
+
thinkingBlocks: Array.isArray(result.thinkingBlocks) ? result.thinkingBlocks : [],
|
|
1847
|
+
};
|
|
1848
|
+
const responseHash = createHash('sha256').update(stableJson(response), 'utf8').digest('hex');
|
|
1849
|
+
const consumed = this.db.prepare(`UPDATE engine_turns SET status = 'responded', response = ?,
|
|
1850
|
+
response_hash = ?, responded_at = ?, consumed_at = ?,
|
|
1851
|
+
consumed_through_sequence = claimed_through_sequence, updated_at = ?
|
|
1852
|
+
WHERE id = ? AND status = 'dispatching' AND dispatch_attempt = ? AND request_hash = ?`).run(
|
|
1853
|
+
stringify(response), responseHash, now, now, now, turnId, turn.dispatchAttempt, turn.requestHash,
|
|
1854
|
+
);
|
|
1855
|
+
if (Number(consumed.changes) !== 1) return false;
|
|
1856
|
+
this.db.prepare(`UPDATE action_entries SET status = 'consumed', consumed_at = ?, updated_at = ?
|
|
1857
|
+
WHERE engine_turn_id = ? AND status = 'bound'`).run(now, now, turnId);
|
|
1858
|
+
const eventIds = this.db.prepare(`SELECT CAST(json_extract(payload, '$.eventId') AS INTEGER) AS event_id
|
|
1859
|
+
FROM action_entries WHERE engine_turn_id = ? AND json_extract(payload, '$.eventId') IS NOT NULL`)
|
|
1860
|
+
.all(turnId).map(row => row.event_id);
|
|
1861
|
+
const acknowledge = this.db.prepare(`UPDATE pending_action_inputs SET consumed_at = ?
|
|
1862
|
+
WHERE event_id = ? AND action_id = ? AND run_id = ? AND consumed_at IS NULL AND superseded_at IS NULL`);
|
|
1863
|
+
for (const eventId of eventIds) acknowledge.run(now, eventId, turn.actionId, turn.runId);
|
|
1864
|
+
this.appendEvent(turn.workItemId, 'engine_turn.responded', {
|
|
1865
|
+
turnId, requestHash: turn.requestHash, responseHash, dispatchAttempt: turn.dispatchAttempt,
|
|
1866
|
+
}, { actionId: turn.actionId, runId: turn.runId });
|
|
1867
|
+
return true;
|
|
1868
|
+
});
|
|
1869
|
+
}
|
|
1870
|
+
|
|
1871
|
+
failEngineTurn(turnId, ownerBootId, leaseEpoch, error) {
|
|
1872
|
+
return withTransaction(this.db, () => {
|
|
1873
|
+
const turn = this.getEngineTurn(turnId);
|
|
1874
|
+
if (!turn || turn.ownerBootId !== ownerBootId || turn.leaseEpoch !== leaseEpoch) {
|
|
1875
|
+
return { allowRetry: false, status: 'stale' };
|
|
1876
|
+
}
|
|
1877
|
+
if (turn.status === 'prepared') return { allowRetry: true, status: 'prepared' };
|
|
1878
|
+
if (turn.status !== 'dispatching') return { allowRetry: false, status: turn.status };
|
|
1879
|
+
const now = this.now();
|
|
1880
|
+
const message = String(error?.message || error || 'Provider dispatch failed').slice(0, 8_000);
|
|
1881
|
+
const changed = this.db.prepare(`UPDATE engine_turns SET status = 'unknown', error = ?,
|
|
1882
|
+
updated_at = ? WHERE id = ? AND status = 'dispatching' AND dispatch_attempt = ?`).run(
|
|
1883
|
+
message, now, turnId, turn.dispatchAttempt,
|
|
1884
|
+
);
|
|
1885
|
+
if (Number(changed.changes) !== 1) return { allowRetry: false, status: 'stale' };
|
|
1886
|
+
this.db.prepare(`UPDATE runs SET status = 'dispatch_unknown', accepting_input = 0,
|
|
1887
|
+
ended_at = ?, error = ? WHERE id = ? AND status = 'running'
|
|
1888
|
+
AND owner_boot_id = ? AND lease_epoch = ?`).run(
|
|
1889
|
+
now, message, turn.runId, ownerBootId, leaseEpoch,
|
|
1890
|
+
);
|
|
1891
|
+
this.db.prepare(`UPDATE actions SET status = 'failed', current_run_id = NULL,
|
|
1892
|
+
updated_at = ? WHERE id = ? AND status = 'running' AND current_run_id = ?
|
|
1893
|
+
AND lease_epoch = ?`).run(now, turn.actionId, turn.runId, leaseEpoch);
|
|
1894
|
+
this.db.prepare(`UPDATE work_items SET status = 'needs_attention', current_action_id = ?,
|
|
1895
|
+
current_run_id = NULL, updated_at = ? WHERE id = ? AND status = 'running'`).run(
|
|
1896
|
+
turn.actionId, now, turn.workItemId,
|
|
1897
|
+
);
|
|
1898
|
+
this.appendEvent(turn.workItemId, 'engine_turn.dispatch_unknown', {
|
|
1899
|
+
turnId, requestHash: turn.requestHash, dispatchAttempt: turn.dispatchAttempt, error: message,
|
|
1900
|
+
}, { actionId: turn.actionId, runId: turn.runId });
|
|
1901
|
+
return { allowRetry: false, status: 'unknown' };
|
|
1902
|
+
});
|
|
1903
|
+
}
|
|
1904
|
+
|
|
1905
|
+
recoverEngineTurns() {
|
|
1906
|
+
return withTransaction(this.db, () => {
|
|
1907
|
+
const now = this.now();
|
|
1908
|
+
const recoverable = this.db.prepare(`SELECT id FROM engine_turns WHERE status = 'prepared'`).all();
|
|
1909
|
+
const dispatching = this.db.prepare(`SELECT * FROM engine_turns WHERE status = 'dispatching'`).all();
|
|
1910
|
+
for (const row of dispatching) {
|
|
1911
|
+
const turn = this.getEngineTurn(row.id);
|
|
1912
|
+
const message = 'Provider dispatch outcome is unknown after Agent restart';
|
|
1913
|
+
this.db.prepare(`UPDATE engine_turns SET status = 'unknown', error = ?, updated_at = ?
|
|
1914
|
+
WHERE id = ? AND status = 'dispatching'`).run(message, now, turn.id);
|
|
1915
|
+
this.db.prepare(`UPDATE runs SET status = 'dispatch_unknown', accepting_input = 0,
|
|
1916
|
+
ended_at = ?, error = ? WHERE id = ? AND status = 'running'`).run(now, message, turn.runId);
|
|
1917
|
+
this.db.prepare(`UPDATE actions SET status = 'failed', current_run_id = NULL,
|
|
1918
|
+
updated_at = ? WHERE id = ? AND status = 'running' AND current_run_id = ?`).run(
|
|
1919
|
+
now, turn.actionId, turn.runId,
|
|
1920
|
+
);
|
|
1921
|
+
this.db.prepare(`UPDATE work_items SET status = 'needs_attention', current_action_id = ?,
|
|
1922
|
+
current_run_id = NULL, updated_at = ? WHERE id = ? AND status = 'running'`).run(
|
|
1923
|
+
turn.actionId, now, turn.workItemId,
|
|
1924
|
+
);
|
|
1925
|
+
this.appendEvent(turn.workItemId, 'engine_turn.dispatch_unknown', {
|
|
1926
|
+
turnId: turn.id, requestHash: turn.requestHash,
|
|
1927
|
+
dispatchAttempt: turn.dispatchAttempt, error: message,
|
|
1928
|
+
}, { actionId: turn.actionId, runId: turn.runId });
|
|
1929
|
+
}
|
|
1930
|
+
return recoverable.map(row => this.getEngineTurn(row.id));
|
|
1931
|
+
});
|
|
1932
|
+
}
|
|
1933
|
+
|
|
1934
|
+
acknowledgeActionInput(eventId, actionId, runId, ownerBootId, leaseEpoch) {
|
|
1935
|
+
const prepared = this.prepareEngineTurn(actionId, runId, ownerBootId, leaseEpoch, [eventId]);
|
|
1936
|
+
if (!prepared) return false;
|
|
1937
|
+
const claimed = this.claimEngineTurn(prepared.id, ownerBootId, leaseEpoch);
|
|
1938
|
+
return !!claimed && this.consumeEngineTurn(prepared.id, ownerBootId, leaseEpoch, {
|
|
1939
|
+
responseText: '', stopReason: 'legacy_acknowledge',
|
|
1314
1940
|
});
|
|
1315
1941
|
}
|
|
1316
1942
|
|
|
@@ -2175,8 +2801,63 @@ export class WorkItemStore {
|
|
|
2175
2801
|
return this.db.prepare(`SELECT * FROM events WHERE action_id = ? ORDER BY id`).all(actionId).map(mapEvent);
|
|
2176
2802
|
}
|
|
2177
2803
|
|
|
2804
|
+
getRecoverableCoordinatorTurns() {
|
|
2805
|
+
return this.db.prepare(`SELECT w.id AS work_item_id, c.payload FROM coordinator_mailbox_entries c
|
|
2806
|
+
JOIN work_items w ON w.id = c.work_item_id
|
|
2807
|
+
WHERE c.status = 'pending'
|
|
2808
|
+
AND EXISTS (SELECT 1 FROM json_each(w.messages) message
|
|
2809
|
+
WHERE json_extract(message.value, '$.turnId') = json_extract(c.payload, '$.turnId')
|
|
2810
|
+
AND json_extract(message.value, '$.role') = 'assistant'
|
|
2811
|
+
AND json_extract(message.value, '$.status') = 'thinking')
|
|
2812
|
+
AND EXISTS (SELECT 1 FROM coordinator_provider_turns p
|
|
2813
|
+
WHERE p.coordinator_turn_id = json_extract(c.payload, '$.turnId')
|
|
2814
|
+
AND p.status IN ('prepared', 'responded'))
|
|
2815
|
+
ORDER BY c.created_at`).all().map(row => ({
|
|
2816
|
+
workItemId: row.work_item_id,
|
|
2817
|
+
...parseJson(row.payload, {}),
|
|
2818
|
+
}));
|
|
2819
|
+
}
|
|
2820
|
+
|
|
2821
|
+
resumeCoordinatorTurn(workItemId, turnId) {
|
|
2822
|
+
const detail = this.getWorkItemDetail(workItemId);
|
|
2823
|
+
if (!detail) return null;
|
|
2824
|
+
const assistant = [...(detail.messages || [])].reverse().find(message => (
|
|
2825
|
+
message.turnId === turnId && message.role === 'assistant' && message.status === 'thinking'
|
|
2826
|
+
));
|
|
2827
|
+
if (!assistant) return null;
|
|
2828
|
+
return {
|
|
2829
|
+
turnId,
|
|
2830
|
+
detail,
|
|
2831
|
+
fence: {
|
|
2832
|
+
workItemId,
|
|
2833
|
+
revision: detail.revision,
|
|
2834
|
+
planRevision: detail.planRevision,
|
|
2835
|
+
ledgerRevision: detail.ledgerRevision,
|
|
2836
|
+
coordinatorRevision: detail.coordinatorRevision,
|
|
2837
|
+
status: detail.status,
|
|
2838
|
+
actionFence: coordinatorActionFence(
|
|
2839
|
+
(detail.actions || []).filter(action => !['completed', 'superseded', 'cancelled'].includes(action.status)),
|
|
2840
|
+
),
|
|
2841
|
+
recovery: assistant.recovery ? { ...assistant.recovery } : null,
|
|
2842
|
+
},
|
|
2843
|
+
};
|
|
2844
|
+
}
|
|
2845
|
+
|
|
2178
2846
|
beginCoordinatorTurn(id, text, expected = {}, options = {}) {
|
|
2179
2847
|
return withTransaction(this.db, () => {
|
|
2848
|
+
const clientMessageId = typeof options.clientMessageId === 'string' && options.clientMessageId
|
|
2849
|
+
? options.clientMessageId : null;
|
|
2850
|
+
if (clientMessageId) {
|
|
2851
|
+
const sourceKey = `client:message:${id}:${clientMessageId}`;
|
|
2852
|
+
const existingAction = this.db.prepare('SELECT 1 FROM action_entries WHERE source_key = ?').get(sourceKey);
|
|
2853
|
+
if (existingAction) throw new Error('clientMessageId already belongs to an Action message');
|
|
2854
|
+
const existing = this.db.prepare(`SELECT payload FROM coordinator_mailbox_entries
|
|
2855
|
+
WHERE source_key = ?`).get(sourceKey);
|
|
2856
|
+
if (existing) {
|
|
2857
|
+
const payload = parseJson(existing.payload, {});
|
|
2858
|
+
return { turnId: payload.turnId, detail: this.getWorkItemDetail(id), duplicate: true };
|
|
2859
|
+
}
|
|
2860
|
+
}
|
|
2180
2861
|
const workItem = this.getWorkItem(id);
|
|
2181
2862
|
if (!workItem) return null;
|
|
2182
2863
|
if (['done', 'cancelled'].includes(workItem.status)) {
|
|
@@ -2242,6 +2923,21 @@ export class WorkItemStore {
|
|
|
2242
2923
|
createdAt: now, updatedAt: now, decision: null,
|
|
2243
2924
|
...(recovery ? { recovery: { ...recovery } } : {}),
|
|
2244
2925
|
};
|
|
2926
|
+
if (userMessage) {
|
|
2927
|
+
this.#appendConversationEntry(
|
|
2928
|
+
id,
|
|
2929
|
+
userMessage,
|
|
2930
|
+
clientMessageId ? `client:conversation:${id}:${clientMessageId}` : `coordinator:turn:${turnId}:user`,
|
|
2931
|
+
);
|
|
2932
|
+
}
|
|
2933
|
+
this.#appendConversationEntry(id, assistantMessage, `coordinator:turn:${turnId}:assistant`);
|
|
2934
|
+
this.enqueueCoordinatorMailbox(id, automaticRecovery ? 'recovery' : 'message', {
|
|
2935
|
+
turnId,
|
|
2936
|
+
text,
|
|
2937
|
+
recovery,
|
|
2938
|
+
clientMessageId,
|
|
2939
|
+
addedAttachments: projectedAttachments,
|
|
2940
|
+
}, clientMessageId ? `client:message:${id}:${clientMessageId}` : `coordinator:turn:${turnId}`);
|
|
2245
2941
|
const messages = [...(workItem.messages || []), ...(userMessage ? [userMessage] : []), assistantMessage]
|
|
2246
2942
|
.slice(-100);
|
|
2247
2943
|
const coordinatorRevision = workItem.coordinatorRevision + 1;
|
|
@@ -2355,11 +3051,6 @@ export class WorkItemStore {
|
|
|
2355
3051
|
|| this.db.prepare(`SELECT id FROM runs WHERE action_id = ? AND status = 'failed'
|
|
2356
3052
|
ORDER BY ended_at DESC, started_at DESC LIMIT 1`).get(action.id)?.id;
|
|
2357
3053
|
if (!resultRunId) throw new Error('Coordinator human request requires a failed Run');
|
|
2358
|
-
const changedRun = this.db.prepare(`UPDATE runs SET waiting_reason = ?
|
|
2359
|
-
WHERE id = ? AND action_id = ? AND status = 'failed'`).run(question, resultRunId, action.id);
|
|
2360
|
-
if (Number(changedRun.changes) !== 1) {
|
|
2361
|
-
throw new Error('Coordinator human request lost the failed Run fence');
|
|
2362
|
-
}
|
|
2363
3054
|
affectedActionIds = [action.id];
|
|
2364
3055
|
this.appendEvent(workItem.id, 'action.waiting', {
|
|
2365
3056
|
reason: question,
|
|
@@ -2488,6 +3179,17 @@ export class WorkItemStore {
|
|
|
2488
3179
|
affectedActionIds,
|
|
2489
3180
|
},
|
|
2490
3181
|
};
|
|
3182
|
+
this.#appendConversationEntry(
|
|
3183
|
+
workItem.id,
|
|
3184
|
+
messages[assistantIndex],
|
|
3185
|
+
`coordinator:turn:${turnId}:assistant`,
|
|
3186
|
+
);
|
|
3187
|
+
const mailbox = this.db.prepare(`SELECT * FROM coordinator_mailbox_entries
|
|
3188
|
+
WHERE json_extract(payload, '$.turnId') = ?`).get(turnId);
|
|
3189
|
+
if (mailbox && mailbox.status !== 'acked') {
|
|
3190
|
+
this.db.prepare(`UPDATE coordinator_mailbox_entries SET status = 'acked', acked_at = ?,
|
|
3191
|
+
updated_at = ? WHERE id = ?`).run(now, now, mailbox.id);
|
|
3192
|
+
}
|
|
2491
3193
|
const current = this.getWorkItem(workItem.id);
|
|
2492
3194
|
const coordinatorRevision = current.coordinatorRevision + 1;
|
|
2493
3195
|
const changed = decision.kind === 'answer'
|
|
@@ -2530,6 +3232,17 @@ export class WorkItemStore {
|
|
|
2530
3232
|
...messages[index], status: 'failed', updatedAt: now,
|
|
2531
3233
|
error: String(error?.message || error || 'Coordinator failed').slice(0, 8_000),
|
|
2532
3234
|
};
|
|
3235
|
+
this.#appendConversationEntry(
|
|
3236
|
+
workItem.id,
|
|
3237
|
+
messages[index],
|
|
3238
|
+
`coordinator:turn:${turnId}:assistant`,
|
|
3239
|
+
);
|
|
3240
|
+
const mailbox = this.db.prepare(`SELECT * FROM coordinator_mailbox_entries
|
|
3241
|
+
WHERE json_extract(payload, '$.turnId') = ?`).get(turnId);
|
|
3242
|
+
if (mailbox && mailbox.status !== 'acked') {
|
|
3243
|
+
this.db.prepare(`UPDATE coordinator_mailbox_entries SET status = 'acked', acked_at = ?,
|
|
3244
|
+
updated_at = ? WHERE id = ?`).run(now, now, mailbox.id);
|
|
3245
|
+
}
|
|
2533
3246
|
const changed = this.db.prepare(`UPDATE work_items SET messages = ?, coordinator_revision = coordinator_revision + 1,
|
|
2534
3247
|
updated_at = ? WHERE id = ? AND coordinator_revision = ?`).run(
|
|
2535
3248
|
stringify(messages), now, workItem.id, workItem.coordinatorRevision,
|
|
@@ -2861,6 +3574,20 @@ export class WorkItemStore {
|
|
|
2861
3574
|
|
|
2862
3575
|
retryWorkItemAtomic(id, makeAction, options = {}) {
|
|
2863
3576
|
return withTransaction(this.db, () => {
|
|
3577
|
+
const clientMessageId = typeof options.inputEvent?.clientMessageId === 'string'
|
|
3578
|
+
? options.inputEvent.clientMessageId : null;
|
|
3579
|
+
if (clientMessageId) {
|
|
3580
|
+
const existing = this.db.prepare(`SELECT action_id FROM events WHERE work_item_id = ?
|
|
3581
|
+
AND type = 'action.input_added' AND json_extract(data, '$.clientMessageId') = ?`).get(
|
|
3582
|
+
id, clientMessageId,
|
|
3583
|
+
);
|
|
3584
|
+
if (existing) {
|
|
3585
|
+
if (existing.action_id !== options.inputEvent.targetActionId) {
|
|
3586
|
+
throw new Error('clientMessageId already belongs to another Action');
|
|
3587
|
+
}
|
|
3588
|
+
return this.getWorkItemDetail(id);
|
|
3589
|
+
}
|
|
3590
|
+
}
|
|
2864
3591
|
const workItem = this.getWorkItem(id);
|
|
2865
3592
|
if (!workItem) return null;
|
|
2866
3593
|
const graphMode = isGraphWorkItem(workItem);
|
|
@@ -2958,7 +3685,7 @@ export class WorkItemStore {
|
|
|
2958
3685
|
|
|
2959
3686
|
claimReadyAction(ownerBootId, leaseMs = 60_000) {
|
|
2960
3687
|
return withTransaction(this.db, () => {
|
|
2961
|
-
const
|
|
3688
|
+
const rows = this.db.prepare(`SELECT a.* FROM actions a
|
|
2962
3689
|
JOIN work_items w ON w.id = a.work_item_id
|
|
2963
3690
|
WHERE a.status = 'ready' AND a.current_run_id IS NULL
|
|
2964
3691
|
AND NOT EXISTS (
|
|
@@ -3004,6 +3731,12 @@ export class WorkItemStore {
|
|
|
3004
3731
|
OR (running.work_item_id != a.work_item_id
|
|
3005
3732
|
AND a.workspace_mode != 'read' AND running.workspace_mode != 'read'))
|
|
3006
3733
|
)
|
|
3734
|
+
AND NOT EXISTS (
|
|
3735
|
+
SELECT 1 FROM operations unsafe_operation
|
|
3736
|
+
WHERE unsafe_operation.work_item_id = w.id
|
|
3737
|
+
AND unsafe_operation.concurrency_policy = 'blocking'
|
|
3738
|
+
AND unsafe_operation.effect_status NOT IN ('applied', 'not_applied', 'failed_no_effect')
|
|
3739
|
+
)
|
|
3007
3740
|
AND NOT EXISTS (
|
|
3008
3741
|
SELECT 1 FROM runs deferred
|
|
3009
3742
|
WHERE deferred.action_id = a.id
|
|
@@ -3017,7 +3750,8 @@ export class WorkItemStore {
|
|
|
3017
3750
|
AND blocker_item.workspace_key = w.workspace_key
|
|
3018
3751
|
)
|
|
3019
3752
|
)
|
|
3020
|
-
ORDER BY a.updated_at ASC, a.sequence ASC
|
|
3753
|
+
ORDER BY a.updated_at ASC, a.sequence ASC`).all();
|
|
3754
|
+
const row = rows.find(candidate => !this.#hasBlockingOperation(candidate.work_item_id, candidate.id));
|
|
3021
3755
|
if (!row) return null;
|
|
3022
3756
|
const now = this.now();
|
|
3023
3757
|
let action = mapAction(row);
|
|
@@ -3034,6 +3768,9 @@ export class WorkItemStore {
|
|
|
3034
3768
|
action = promoteReadyActionInputs(this.db, action, readyInputs, now, 'run_claim');
|
|
3035
3769
|
const runId = randomUUID();
|
|
3036
3770
|
const leaseEpoch = Number(action.leaseEpoch) + 1;
|
|
3771
|
+
const priorOrdinal = this.db.prepare(`SELECT MAX(ordinal) AS value FROM runs
|
|
3772
|
+
WHERE action_id = ?`).get(action.id);
|
|
3773
|
+
const runOrdinal = Math.max(0, Number(priorOrdinal?.value) || 0) + 1;
|
|
3037
3774
|
const priorProgress = this.db.prepare(`SELECT MAX(progress_revision) AS value FROM runs
|
|
3038
3775
|
WHERE action_id = ?`).get(action.id);
|
|
3039
3776
|
const progressRevision = Math.max(0, Number(priorProgress?.value) || 0) + 1;
|
|
@@ -3061,14 +3798,15 @@ export class WorkItemStore {
|
|
|
3061
3798
|
);
|
|
3062
3799
|
if (Number(changedWorkItem.changes) !== 1) throw new Error('WorkItem claim lost its Action fence');
|
|
3063
3800
|
this.db.prepare(`INSERT INTO runs
|
|
3064
|
-
(id, action_id, work_item_id, owner_boot_id, lease_epoch, status, started_at,
|
|
3801
|
+
(id, action_id, work_item_id, owner_boot_id, lease_epoch, ordinal, status, started_at,
|
|
3065
3802
|
expires_at, evidence, progress_revision, action_generation, action_spec_hash, action_attempt)
|
|
3066
|
-
VALUES (?, ?, ?, ?, ?, 'running', ?, ?, '[]', ?, ?, ?, ?)`).run(
|
|
3803
|
+
VALUES (?, ?, ?, ?, ?, ?, 'running', ?, ?, '[]', ?, ?, ?, ?)`).run(
|
|
3067
3804
|
runId,
|
|
3068
3805
|
action.id,
|
|
3069
3806
|
action.workItemId,
|
|
3070
3807
|
ownerBootId,
|
|
3071
3808
|
leaseEpoch,
|
|
3809
|
+
runOrdinal,
|
|
3072
3810
|
now,
|
|
3073
3811
|
now + leaseMs,
|
|
3074
3812
|
progressRevision,
|
|
@@ -3583,6 +4321,9 @@ export class WorkItemStore {
|
|
|
3583
4321
|
}, this.#nextSequence(workItem.id), now);
|
|
3584
4322
|
}
|
|
3585
4323
|
let workItemStatus = transition.workItemStatus;
|
|
4324
|
+
if (workItemStatus === 'done' && this.#hasBlockingOperation(workItem.id)) {
|
|
4325
|
+
throw new Error('WorkItem has an unsafe blocking Operation and cannot complete');
|
|
4326
|
+
}
|
|
3586
4327
|
let currentActionId = nextAction?.id ?? (transition.keepCurrentAction ? action.id : null);
|
|
3587
4328
|
let changedWorkItem;
|
|
3588
4329
|
if (transition.planConflict) {
|
|
@@ -3662,6 +4403,9 @@ export class WorkItemStore {
|
|
|
3662
4403
|
|
|
3663
4404
|
recoverInterruptedCoordinatorTurns() {
|
|
3664
4405
|
return withTransaction(this.db, () => {
|
|
4406
|
+
const recoverableTurnIds = new Set(
|
|
4407
|
+
this.getRecoverableCoordinatorTurns().map(turn => turn.turnId),
|
|
4408
|
+
);
|
|
3665
4409
|
const now = this.now();
|
|
3666
4410
|
let recovered = 0;
|
|
3667
4411
|
for (const row of this.db.prepare(`SELECT id, messages, coordinator_revision FROM work_items
|
|
@@ -3670,6 +4414,7 @@ export class WorkItemStore {
|
|
|
3670
4414
|
const messages = parseJson(row.messages, []);
|
|
3671
4415
|
const index = messages.length - 1;
|
|
3672
4416
|
if (index < 0 || messages[index]?.role !== 'assistant' || messages[index]?.status !== 'thinking') continue;
|
|
4417
|
+
if (recoverableTurnIds.has(messages[index].turnId)) continue;
|
|
3673
4418
|
messages[index] = {
|
|
3674
4419
|
...messages[index],
|
|
3675
4420
|
status: 'failed',
|
|
@@ -3682,6 +4427,12 @@ export class WorkItemStore {
|
|
|
3682
4427
|
stringify(messages), now, row.id, row.coordinator_revision,
|
|
3683
4428
|
);
|
|
3684
4429
|
if (Number(changed.changes) !== 1) continue;
|
|
4430
|
+
const turnId = messages[index].turnId || null;
|
|
4431
|
+
if (turnId) {
|
|
4432
|
+
this.db.prepare(`UPDATE coordinator_mailbox_entries SET status = 'acked', acked_at = ?,
|
|
4433
|
+
claim_owner = NULL, claimed_at = NULL, lease_expires_at = NULL, updated_at = ?
|
|
4434
|
+
WHERE json_extract(payload, '$.turnId') = ? AND status != 'acked'`).run(now, now, turnId);
|
|
4435
|
+
}
|
|
3685
4436
|
this.appendEvent(row.id, 'coordinator.turn_interrupted', {
|
|
3686
4437
|
turnId: messages[index].turnId || null,
|
|
3687
4438
|
error: messages[index].error,
|