conductor-remote 1.106.3 → 1.108.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +65 -44
- package/dist/assets/{PierrePatch-CEuLSHL_.js → PierrePatch-DF-STc4J.js} +1 -1
- package/dist/assets/index-DM7eCDXg.css +1 -0
- package/dist/assets/index-DWbWwEOp.js +59 -0
- package/dist/index.html +2 -2
- package/dist/licenses/vscode-seti.txt +48 -0
- package/dist/sw.js +1 -1
- package/dist-node/src/logbuf.js +4 -3
- package/dist-node/src/mcp-tools.js +48 -138
- package/dist-node/src/model-cache.js +22 -7
- package/dist-node/src/orchestration-db.js +2150 -0
- package/dist-node/src/orchestration-schema.generated.js +306 -0
- package/dist-node/src/orchestration-schema.js +417 -0
- package/dist-node/src/preview-urls.js +1 -1
- package/dist-node/src/reads.js +135 -6
- package/dist-node/src/relay-processes.js +230 -0
- package/dist-node/src/roles.js +64 -12
- package/dist-node/src/routes.js +14 -1
- package/dist-node/src/search-worker.js +3 -1
- package/dist-node/src/server.js +1050 -211
- package/dist-node/src/shared.js +79 -2
- package/dist-node/src/transcript.js +20 -10
- package/dist-node/src/ui-lease-watchdog.js +94 -0
- package/dist-node/src/workflow-coordinator.js +2119 -0
- package/dist-node/src/workflow-effect-runner.js +336 -0
- package/dist-node/src/workflow-http.js +141 -0
- package/dist-node/src/workflow-machine.js +112 -0
- package/dist-node/src/workflow.js +111 -22
- package/dist-node/src/writes.js +157 -12
- package/package.json +9 -2
- package/dist/assets/index-BfTQtpd5.css +0 -1
- package/dist/assets/index-BiWQQKon.js +0 -58
|
@@ -0,0 +1,2150 @@
|
|
|
1
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
2
|
+
import { mkdirSync } from 'node:fs';
|
|
3
|
+
import { dirname } from 'node:path';
|
|
4
|
+
import { DatabaseSync } from 'node:sqlite';
|
|
5
|
+
import { and, asc, desc, eq, notInArray } from 'drizzle-orm';
|
|
6
|
+
import { drizzle } from 'drizzle-orm/node-sqlite';
|
|
7
|
+
import { ORCHESTRATION_BOOTSTRAP_SQL, ORCHESTRATION_SCHEMA_VERSION, ORCHESTRATION_TABLE_NAMES, relayInstanceSelectSchema, relayInstances, uiMutex, uiMutexSelectSchema, uiQuarantine, uiQuarantineSelectSchema, workflowCapabilities, workflowCapabilitySelectSchema, workflowEffectAttemptSelectSchema, workflowEffectAttempts, workflowEffectSelectSchema, workflowEffects, workflowEventSelectSchema, workflowEvents, workflowIdempotency, workflowJobAttemptSelectSchema, workflowJobAttempts, workflowJobSelectSchema, workflowJobs, workflowRunSelectSchema, workflowRuns } from "./orchestration-schema.js";
|
|
8
|
+
import { scrubWorkflowSecrets, withoutWindowEvidence } from "./shared.js";
|
|
9
|
+
import { isTerminalWorkflowJobState, isTerminalWorkflowPhase } from "./workflow-machine.js";
|
|
10
|
+
export { ORCHESTRATION_SCHEMA_VERSION } from "./orchestration-schema.js";
|
|
11
|
+
export const ORCHESTRATION_PROTOCOL_VERSION = 1;
|
|
12
|
+
export class OrchestrationError extends Error {
|
|
13
|
+
}
|
|
14
|
+
export class UnsupportedOrchestrationSchemaError extends OrchestrationError {
|
|
15
|
+
foundVersion;
|
|
16
|
+
supportedVersion = ORCHESTRATION_SCHEMA_VERSION;
|
|
17
|
+
constructor(foundVersion) {
|
|
18
|
+
super(`orchestration schema ${foundVersion} is newer than supported schema ${ORCHESTRATION_SCHEMA_VERSION}`);
|
|
19
|
+
this.name = 'UnsupportedOrchestrationSchemaError';
|
|
20
|
+
this.foundVersion = foundVersion;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
export class IdempotencyConflictError extends OrchestrationError {
|
|
24
|
+
operation;
|
|
25
|
+
clientId;
|
|
26
|
+
constructor(operation, clientId) {
|
|
27
|
+
super(`clientId ${clientId} was already used with a different ${operation} request`);
|
|
28
|
+
this.name = 'IdempotencyConflictError';
|
|
29
|
+
this.operation = operation;
|
|
30
|
+
this.clientId = clientId;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
export class WorkflowTransitionError extends OrchestrationError {
|
|
34
|
+
constructor(message) {
|
|
35
|
+
super(message);
|
|
36
|
+
this.name = 'WorkflowTransitionError';
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
export class UiLeaseUnavailableError extends OrchestrationError {
|
|
40
|
+
result;
|
|
41
|
+
constructor(result) {
|
|
42
|
+
const message = result.status === 'busy'
|
|
43
|
+
? `Conductor's UI is busy — held by relay PID ${result.owner.pid} (${result.reason}). Try again shortly.`
|
|
44
|
+
: `Conductor's UI is quarantined — ${result.quarantine.reason ?? 'confirm Conductor is stable on the phone'}.`;
|
|
45
|
+
super(message);
|
|
46
|
+
this.name = 'UiLeaseUnavailableError';
|
|
47
|
+
this.result = result;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
const ACTIVE_PHASES = [
|
|
51
|
+
'creating_workspace',
|
|
52
|
+
'binding_root',
|
|
53
|
+
'pending_root',
|
|
54
|
+
'exploring',
|
|
55
|
+
'planning',
|
|
56
|
+
'implementing',
|
|
57
|
+
'reviewing',
|
|
58
|
+
'blocked'
|
|
59
|
+
];
|
|
60
|
+
const RUNNABLE_PHASES = ACTIVE_PHASES.filter(phase => phase !== 'blocked');
|
|
61
|
+
const ALL_JOB_STATES = [
|
|
62
|
+
'dormant',
|
|
63
|
+
'queued',
|
|
64
|
+
'owned',
|
|
65
|
+
'opening',
|
|
66
|
+
'configuring',
|
|
67
|
+
'sending',
|
|
68
|
+
'running',
|
|
69
|
+
'returning',
|
|
70
|
+
'returned',
|
|
71
|
+
'failed',
|
|
72
|
+
'cancelled'
|
|
73
|
+
];
|
|
74
|
+
const TERMINAL_EFFECT_STATES = ['committed', 'failed', 'ambiguous', 'cancelled'];
|
|
75
|
+
const migrations = [ORCHESTRATION_BOOTSTRAP_SQL];
|
|
76
|
+
const sqliteRetryWait = new Int32Array(new SharedArrayBuffer(4));
|
|
77
|
+
const nonEmpty = (value, name) => {
|
|
78
|
+
if (!value.trim())
|
|
79
|
+
throw new OrchestrationError(`${name} must not be empty`);
|
|
80
|
+
return value;
|
|
81
|
+
};
|
|
82
|
+
const validateProcessIdentity = (identity, name) => {
|
|
83
|
+
if (!Number.isSafeInteger(identity.pid) || identity.pid <= 0)
|
|
84
|
+
throw new OrchestrationError(`${name} PID is invalid`);
|
|
85
|
+
nonEmpty(identity.processStartedAt, `${name} process-start identity`);
|
|
86
|
+
};
|
|
87
|
+
const validateRelayIdentity = (identity) => {
|
|
88
|
+
nonEmpty(identity.instanceId, 'relay instance ID');
|
|
89
|
+
validateProcessIdentity(identity, 'relay');
|
|
90
|
+
if (!Number.isSafeInteger(identity.protocolVersion) || identity.protocolVersion < 0) {
|
|
91
|
+
throw new OrchestrationError('relay orchestration protocol is invalid');
|
|
92
|
+
}
|
|
93
|
+
};
|
|
94
|
+
const json = (value) => {
|
|
95
|
+
const encoded = JSON.stringify(value);
|
|
96
|
+
if (encoded === undefined)
|
|
97
|
+
throw new OrchestrationError('value is not JSON serializable');
|
|
98
|
+
return encoded;
|
|
99
|
+
};
|
|
100
|
+
const optionalJson = (value) => (value === undefined ? null : json(value));
|
|
101
|
+
const jsonProperty = (key, stored) => stored === null ? {} : { [key]: stored.value };
|
|
102
|
+
const isPromiseLike = (value) => value !== null &&
|
|
103
|
+
(typeof value === 'object' || typeof value === 'function') &&
|
|
104
|
+
typeof value.then === 'function';
|
|
105
|
+
const canonicalize = (value, inArray = false) => {
|
|
106
|
+
if (value === null || typeof value === 'string' || typeof value === 'boolean')
|
|
107
|
+
return value;
|
|
108
|
+
if (typeof value === 'number') {
|
|
109
|
+
if (!Number.isFinite(value))
|
|
110
|
+
throw new OrchestrationError('canonical request contains a non-finite number');
|
|
111
|
+
return Object.is(value, -0) ? 0 : value;
|
|
112
|
+
}
|
|
113
|
+
if (typeof value === 'undefined')
|
|
114
|
+
return inArray ? null : undefined;
|
|
115
|
+
if (Array.isArray(value))
|
|
116
|
+
return value.map(entry => canonicalize(entry, true));
|
|
117
|
+
if (typeof value !== 'object' || Object.getPrototypeOf(value) !== Object.prototype) {
|
|
118
|
+
throw new OrchestrationError('canonical requests must contain only JSON objects and values');
|
|
119
|
+
}
|
|
120
|
+
const result = {};
|
|
121
|
+
for (const key of Object.keys(value).sort()) {
|
|
122
|
+
const normalized = canonicalize(value[key]);
|
|
123
|
+
if (normalized !== undefined)
|
|
124
|
+
result[key] = normalized;
|
|
125
|
+
}
|
|
126
|
+
return result;
|
|
127
|
+
};
|
|
128
|
+
export const canonicalRequestJson = (request) => json(canonicalize(request));
|
|
129
|
+
export const canonicalRequestHash = (request) => createHash('sha256').update(canonicalRequestJson(request)).digest('hex');
|
|
130
|
+
export const hashCapabilityToken = (token) => createHash('sha256').update(nonEmpty(token, 'capability token')).digest('hex');
|
|
131
|
+
const isConstraintError = (error) => error instanceof Error && /constraint failed|unique constraint/i.test(error.message);
|
|
132
|
+
const ownerFromColumns = (row) => {
|
|
133
|
+
if (row.ownerInstanceId === null ||
|
|
134
|
+
row.ownerPid === null ||
|
|
135
|
+
row.ownerProcessStartedAt === null ||
|
|
136
|
+
row.ownerProtocolVersion === null) {
|
|
137
|
+
return undefined;
|
|
138
|
+
}
|
|
139
|
+
return {
|
|
140
|
+
instanceId: row.ownerInstanceId,
|
|
141
|
+
pid: row.ownerPid,
|
|
142
|
+
processStartedAt: row.ownerProcessStartedAt,
|
|
143
|
+
protocolVersion: row.ownerProtocolVersion
|
|
144
|
+
};
|
|
145
|
+
};
|
|
146
|
+
const externalFromColumns = (row) => {
|
|
147
|
+
if (row.externalPid === null || row.externalProcessStartedAt === null)
|
|
148
|
+
return undefined;
|
|
149
|
+
return {
|
|
150
|
+
pid: row.externalPid,
|
|
151
|
+
processStartedAt: row.externalProcessStartedAt,
|
|
152
|
+
...(row.externalProcessGroup === null ? {} : { processGroup: row.externalProcessGroup })
|
|
153
|
+
};
|
|
154
|
+
};
|
|
155
|
+
const sameProcess = (left, right) => left.pid === right.pid && left.processStartedAt === right.processStartedAt;
|
|
156
|
+
const sameOwner = (left, right) => left.instanceId === right.instanceId && left.protocolVersion === right.protocolVersion && sameProcess(left, right);
|
|
157
|
+
const processAuditKey = (identity) => `${identity.pid}:${createHash('sha256')
|
|
158
|
+
.update(`${identity.processStartedAt}\0${identity.processGroup ?? ''}`)
|
|
159
|
+
.digest('hex')
|
|
160
|
+
.slice(0, 16)}`;
|
|
161
|
+
const ownerAuditKey = (owner) => createHash('sha256')
|
|
162
|
+
.update(`${owner.instanceId}\0${owner.pid}\0${owner.processStartedAt}\0${owner.protocolVersion}`)
|
|
163
|
+
.digest('hex')
|
|
164
|
+
.slice(0, 16);
|
|
165
|
+
const defaultScrub = (text) => withoutWindowEvidence(scrubWorkflowSecrets(text))
|
|
166
|
+
.replace(/[\r\n\t]+/g, ' ')
|
|
167
|
+
.replace(/\s{2,}/g, ' ')
|
|
168
|
+
.trim();
|
|
169
|
+
/**
|
|
170
|
+
* Durable, relay-owned Workflow coordinator state. Every mutating method is a
|
|
171
|
+
* short synchronous `BEGIN IMMEDIATE` transaction. `idempotentMutation` callbacks
|
|
172
|
+
* must therefore be synchronous; domain methods called inside one join its transaction.
|
|
173
|
+
*/
|
|
174
|
+
export class OrchestrationDb {
|
|
175
|
+
db;
|
|
176
|
+
orm;
|
|
177
|
+
now;
|
|
178
|
+
processProbe;
|
|
179
|
+
scrubPublicText;
|
|
180
|
+
busyTimeoutMs;
|
|
181
|
+
transactionDepth = 0;
|
|
182
|
+
schemaVersion;
|
|
183
|
+
writable;
|
|
184
|
+
schemaWarning;
|
|
185
|
+
constructor(file, options = {}) {
|
|
186
|
+
if (file !== ':memory:')
|
|
187
|
+
mkdirSync(dirname(file), { recursive: true });
|
|
188
|
+
this.now = options.now ?? Date.now;
|
|
189
|
+
// Without an exact process-identity probe, failure to prove death must fail closed.
|
|
190
|
+
this.processProbe = options.processProbe ?? (() => true);
|
|
191
|
+
this.scrubPublicText = options.scrubPublicText ?? defaultScrub;
|
|
192
|
+
this.busyTimeoutMs = Math.max(0, Math.floor(options.busyTimeoutMs ?? 5000));
|
|
193
|
+
this.db = new DatabaseSync(file);
|
|
194
|
+
this.orm = drizzle({ client: this.db });
|
|
195
|
+
this.db.exec(`PRAGMA busy_timeout = ${this.busyTimeoutMs}`);
|
|
196
|
+
this.db.exec('PRAGMA foreign_keys = ON');
|
|
197
|
+
let found;
|
|
198
|
+
try {
|
|
199
|
+
found = this.detectSchemaVersion();
|
|
200
|
+
}
|
|
201
|
+
catch (error) {
|
|
202
|
+
this.reopenReadOnly(file);
|
|
203
|
+
this.schemaVersion = -1;
|
|
204
|
+
this.writable = false;
|
|
205
|
+
this.schemaWarning = error instanceof Error ? error.message : 'orchestration schema metadata is corrupt';
|
|
206
|
+
return;
|
|
207
|
+
}
|
|
208
|
+
if (found > ORCHESTRATION_SCHEMA_VERSION) {
|
|
209
|
+
this.reopenReadOnly(file);
|
|
210
|
+
this.schemaVersion = found;
|
|
211
|
+
this.writable = false;
|
|
212
|
+
this.schemaWarning = `orchestration schema ${found} requires a newer relay`;
|
|
213
|
+
return;
|
|
214
|
+
}
|
|
215
|
+
// `busy_timeout` does not reliably wait for a concurrent journal-mode change.
|
|
216
|
+
// Relay processes can cold-start together, so retry only these initialization
|
|
217
|
+
// pragmas within the same bounded budget used by SQLite statements.
|
|
218
|
+
this.execInitializationPragma('PRAGMA journal_mode = WAL');
|
|
219
|
+
this.execInitializationPragma('PRAGMA synchronous = NORMAL');
|
|
220
|
+
for (let next = found + 1; next <= ORCHESTRATION_SCHEMA_VERSION; next++)
|
|
221
|
+
this.applyMigration(next);
|
|
222
|
+
let schemaProblem;
|
|
223
|
+
try {
|
|
224
|
+
schemaProblem = this.currentSchemaProblem();
|
|
225
|
+
}
|
|
226
|
+
catch (error) {
|
|
227
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
228
|
+
schemaProblem = `orchestration schema could not be validated: ${detail}`;
|
|
229
|
+
}
|
|
230
|
+
if (schemaProblem) {
|
|
231
|
+
this.reopenReadOnly(file);
|
|
232
|
+
this.schemaVersion = ORCHESTRATION_SCHEMA_VERSION;
|
|
233
|
+
this.writable = false;
|
|
234
|
+
this.schemaWarning = schemaProblem;
|
|
235
|
+
return;
|
|
236
|
+
}
|
|
237
|
+
this.schemaVersion = ORCHESTRATION_SCHEMA_VERSION;
|
|
238
|
+
this.writable = true;
|
|
239
|
+
this.schemaWarning = undefined;
|
|
240
|
+
}
|
|
241
|
+
close() {
|
|
242
|
+
this.db.close();
|
|
243
|
+
}
|
|
244
|
+
reopenReadOnly(file) {
|
|
245
|
+
if (file === ':memory:')
|
|
246
|
+
return;
|
|
247
|
+
this.db.close();
|
|
248
|
+
this.db = new DatabaseSync(file, { readOnly: true });
|
|
249
|
+
this.orm = drizzle({ client: this.db });
|
|
250
|
+
this.db.exec(`PRAGMA busy_timeout = ${this.busyTimeoutMs}`);
|
|
251
|
+
this.db.exec('PRAGMA foreign_keys = ON');
|
|
252
|
+
}
|
|
253
|
+
currentSchemaProblem() {
|
|
254
|
+
const rows = this.db
|
|
255
|
+
.prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name IN (${ORCHESTRATION_TABLE_NAMES.map(() => '?').join(',')})`)
|
|
256
|
+
.all(...ORCHESTRATION_TABLE_NAMES);
|
|
257
|
+
const present = new Set(rows.map(row => row.name));
|
|
258
|
+
const missing = ORCHESTRATION_TABLE_NAMES.filter(table => !present.has(table));
|
|
259
|
+
if (missing.length > 0)
|
|
260
|
+
return `orchestration schema is missing: ${missing.join(', ')}`;
|
|
261
|
+
const mutex = this.db.prepare('SELECT COUNT(*) count FROM ui_mutex WHERE id = 1').get();
|
|
262
|
+
const quarantine = this.db.prepare('SELECT COUNT(*) count FROM ui_quarantine WHERE id = 1').get();
|
|
263
|
+
if (Number(mutex.count) !== 1 || Number(quarantine.count) !== 1) {
|
|
264
|
+
return 'orchestration schema is missing its singleton UI coordination rows';
|
|
265
|
+
}
|
|
266
|
+
return undefined;
|
|
267
|
+
}
|
|
268
|
+
execInitializationPragma(sql) {
|
|
269
|
+
const deadline = Date.now() + this.busyTimeoutMs;
|
|
270
|
+
for (;;) {
|
|
271
|
+
try {
|
|
272
|
+
this.db.exec(sql);
|
|
273
|
+
return;
|
|
274
|
+
}
|
|
275
|
+
catch (error) {
|
|
276
|
+
const busy = error instanceof Error &&
|
|
277
|
+
(error.errcode === 5 || /database is (?:locked|busy)/i.test(error.message));
|
|
278
|
+
const remaining = deadline - Date.now();
|
|
279
|
+
if (!busy || remaining <= 0)
|
|
280
|
+
throw error;
|
|
281
|
+
Atomics.wait(sqliteRetryWait, 0, 0, Math.min(25, remaining));
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
detectSchemaVersion() {
|
|
286
|
+
const table = this.db
|
|
287
|
+
.prepare("SELECT 1 present FROM sqlite_master WHERE type = 'table' AND name = 'orchestration_meta'")
|
|
288
|
+
.get();
|
|
289
|
+
if (!table) {
|
|
290
|
+
const stateTables = ORCHESTRATION_TABLE_NAMES.filter(name => name !== 'orchestration_meta');
|
|
291
|
+
const existing = this.db
|
|
292
|
+
.prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name IN (${stateTables.map(() => '?').join(',')}) ORDER BY name`)
|
|
293
|
+
.all(...stateTables);
|
|
294
|
+
if (existing.length > 0) {
|
|
295
|
+
throw new OrchestrationError(`orchestration schema version metadata is missing (found ${existing.map(row => row.name).join(', ')})`);
|
|
296
|
+
}
|
|
297
|
+
return 0;
|
|
298
|
+
}
|
|
299
|
+
const row = this.db.prepare("SELECT value FROM orchestration_meta WHERE key = 'schema_version'").get();
|
|
300
|
+
if (!row)
|
|
301
|
+
throw new OrchestrationError('orchestration schema version metadata is missing');
|
|
302
|
+
const version = Number(row.value);
|
|
303
|
+
if (!Number.isSafeInteger(version) || version < 0)
|
|
304
|
+
throw new OrchestrationError('invalid orchestration schema version');
|
|
305
|
+
return version;
|
|
306
|
+
}
|
|
307
|
+
applyMigration(version) {
|
|
308
|
+
const migration = migrations[version - 1];
|
|
309
|
+
if (!migration)
|
|
310
|
+
throw new OrchestrationError(`missing orchestration migration ${version}`);
|
|
311
|
+
this.db.exec('BEGIN IMMEDIATE');
|
|
312
|
+
try {
|
|
313
|
+
// Another relay may have migrated after this connection's initial read but
|
|
314
|
+
// before it acquired the write lock. Recheck under the lock and join it.
|
|
315
|
+
const current = this.detectSchemaVersion();
|
|
316
|
+
if (current >= version) {
|
|
317
|
+
this.db.exec('COMMIT');
|
|
318
|
+
return;
|
|
319
|
+
}
|
|
320
|
+
if (current !== version - 1) {
|
|
321
|
+
throw new OrchestrationError(`cannot migrate orchestration schema ${current} to ${version}`);
|
|
322
|
+
}
|
|
323
|
+
this.db.exec(migration);
|
|
324
|
+
this.db
|
|
325
|
+
.prepare('INSERT OR REPLACE INTO orchestration_meta(key, value) VALUES (?, ?)')
|
|
326
|
+
.run('schema_version', String(version));
|
|
327
|
+
this.db.exec('COMMIT');
|
|
328
|
+
}
|
|
329
|
+
catch (error) {
|
|
330
|
+
this.db.exec('ROLLBACK');
|
|
331
|
+
throw error;
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
assertWritable() {
|
|
335
|
+
if (!this.writable && this.schemaVersion > ORCHESTRATION_SCHEMA_VERSION) {
|
|
336
|
+
throw new UnsupportedOrchestrationSchemaError(this.schemaVersion);
|
|
337
|
+
}
|
|
338
|
+
if (this.schemaWarning)
|
|
339
|
+
throw new OrchestrationError(this.schemaWarning);
|
|
340
|
+
if (!this.writable)
|
|
341
|
+
throw new OrchestrationError('orchestration database is read-only');
|
|
342
|
+
}
|
|
343
|
+
immediate(operation) {
|
|
344
|
+
this.assertWritable();
|
|
345
|
+
if (this.transactionDepth > 0)
|
|
346
|
+
return operation();
|
|
347
|
+
this.db.exec('BEGIN IMMEDIATE');
|
|
348
|
+
this.transactionDepth++;
|
|
349
|
+
try {
|
|
350
|
+
const result = operation();
|
|
351
|
+
if (isPromiseLike(result))
|
|
352
|
+
throw new OrchestrationError('orchestration transactions must be synchronous');
|
|
353
|
+
this.db.exec('COMMIT');
|
|
354
|
+
return result;
|
|
355
|
+
}
|
|
356
|
+
catch (error) {
|
|
357
|
+
this.db.exec('ROLLBACK');
|
|
358
|
+
throw error;
|
|
359
|
+
}
|
|
360
|
+
finally {
|
|
361
|
+
this.transactionDepth--;
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
idempotentMutation(operation, clientId, request, mutate, link = {}) {
|
|
365
|
+
nonEmpty(operation, 'operation');
|
|
366
|
+
nonEmpty(clientId, 'clientId');
|
|
367
|
+
const requestHash = canonicalRequestHash(request);
|
|
368
|
+
return this.immediate(() => {
|
|
369
|
+
const existing = this.orm
|
|
370
|
+
.select({ requestHash: workflowIdempotency.requestHash, result: workflowIdempotency.result })
|
|
371
|
+
.from(workflowIdempotency)
|
|
372
|
+
.where(and(eq(workflowIdempotency.operation, operation), eq(workflowIdempotency.clientId, clientId)))
|
|
373
|
+
.get();
|
|
374
|
+
if (existing) {
|
|
375
|
+
if (existing.requestHash !== requestHash)
|
|
376
|
+
throw new IdempotencyConflictError(operation, clientId);
|
|
377
|
+
return { replayed: true, result: existing.result };
|
|
378
|
+
}
|
|
379
|
+
const result = mutate();
|
|
380
|
+
if (isPromiseLike(result))
|
|
381
|
+
throw new OrchestrationError('idempotent mutations must be synchronous');
|
|
382
|
+
const resultRecord = asObject(result);
|
|
383
|
+
const resultRunId = typeof resultRecord.runId === 'string' ? resultRecord.runId : undefined;
|
|
384
|
+
const resultActionId = typeof resultRecord.actionId === 'string' ? resultRecord.actionId : undefined;
|
|
385
|
+
this.db
|
|
386
|
+
.prepare(`INSERT INTO workflow_idempotency
|
|
387
|
+
(operation, client_id, request_hash, result_json, run_id, action_id, created_at)
|
|
388
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`)
|
|
389
|
+
.run(operation, clientId, requestHash, json(result), link.runId ?? resultRunId ?? null, link.actionId ?? resultActionId ?? null, this.now());
|
|
390
|
+
return { replayed: false, result };
|
|
391
|
+
});
|
|
392
|
+
}
|
|
393
|
+
/**
|
|
394
|
+
* Read the stable result before re-running live preconditions. This matters for
|
|
395
|
+
* Workflow Start: after the first accepted request, its root is intentionally no
|
|
396
|
+
* longer pristine and roles.json may have changed, but an identical client retry
|
|
397
|
+
* must still receive the original accepted run.
|
|
398
|
+
*/
|
|
399
|
+
getIdempotentMutation(operation, clientId, request) {
|
|
400
|
+
nonEmpty(operation, 'operation');
|
|
401
|
+
nonEmpty(clientId, 'clientId');
|
|
402
|
+
const existing = this.orm
|
|
403
|
+
.select({ requestHash: workflowIdempotency.requestHash, result: workflowIdempotency.result })
|
|
404
|
+
.from(workflowIdempotency)
|
|
405
|
+
.where(and(eq(workflowIdempotency.operation, operation), eq(workflowIdempotency.clientId, clientId)))
|
|
406
|
+
.get();
|
|
407
|
+
if (!existing)
|
|
408
|
+
return undefined;
|
|
409
|
+
if (existing.requestHash !== canonicalRequestHash(request)) {
|
|
410
|
+
throw new IdempotencyConflictError(operation, clientId);
|
|
411
|
+
}
|
|
412
|
+
return { result: existing.result };
|
|
413
|
+
}
|
|
414
|
+
createWorkflowRun(input) {
|
|
415
|
+
const initialEffects = input.initialEffects ?? (input.initialEffect ? [input.initialEffect] : []);
|
|
416
|
+
if (new Set(initialEffects.map(effect => effect.actionId)).size !== initialEffects.length) {
|
|
417
|
+
throw new OrchestrationError('initial Workflow effects must have unique action IDs');
|
|
418
|
+
}
|
|
419
|
+
// Idempotency belongs to the normalized UI request, not to derived preflight
|
|
420
|
+
// evidence. Re-reading roles or the now-non-pristine root on a transport retry
|
|
421
|
+
// must not turn the same tap into a conflict.
|
|
422
|
+
const request = { objective: input.objective, target: input.target };
|
|
423
|
+
let createdRunId;
|
|
424
|
+
const answer = this.idempotentMutation('start_workflow', input.clientId, request, () => {
|
|
425
|
+
const runId = input.id ?? randomUUID();
|
|
426
|
+
createdRunId = runId;
|
|
427
|
+
const at = this.now();
|
|
428
|
+
const phase = input.target.kind === 'new_workspace' ? 'creating_workspace' : 'pending_root';
|
|
429
|
+
if (input.target.kind === 'existing_session') {
|
|
430
|
+
if (input.workspaceId && input.workspaceId !== input.target.workspaceId) {
|
|
431
|
+
throw new OrchestrationError('workspace binding conflicts with the existing-session target');
|
|
432
|
+
}
|
|
433
|
+
if (input.rootSessionId && input.rootSessionId !== input.target.sessionId) {
|
|
434
|
+
throw new OrchestrationError('root binding conflicts with the existing-session target');
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
try {
|
|
438
|
+
this.db
|
|
439
|
+
.prepare(`INSERT INTO workflow_runs (
|
|
440
|
+
id, objective, target_json, roles_json, phase, workspace_id, root_session_id,
|
|
441
|
+
pristine_evidence_json, delivery_baseline_json, created_at, updated_at
|
|
442
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
443
|
+
.run(runId, nonEmpty(input.objective, 'objective'), json(input.target), json(input.roles), phase, input.target.kind === 'existing_session' ? input.target.workspaceId : (input.workspaceId ?? null), input.target.kind === 'existing_session' ? input.target.sessionId : (input.rootSessionId ?? null), optionalJson(input.pristineEvidence), optionalJson(input.deliveryBaseline), at, at);
|
|
444
|
+
}
|
|
445
|
+
catch (error) {
|
|
446
|
+
if (isConstraintError(error)) {
|
|
447
|
+
throw new WorkflowTransitionError('the root already belongs to an active Workflow');
|
|
448
|
+
}
|
|
449
|
+
throw error;
|
|
450
|
+
}
|
|
451
|
+
this.insertJob({
|
|
452
|
+
id: `${runId}:explore:0`,
|
|
453
|
+
runId,
|
|
454
|
+
logicalKey: 'explore:0',
|
|
455
|
+
role: 'exploration',
|
|
456
|
+
cycle: 0,
|
|
457
|
+
revision: 0,
|
|
458
|
+
resolvedRole: input.roles.exploration,
|
|
459
|
+
prompt: input.bootstrapPrompt ?? input.objective,
|
|
460
|
+
state: 'dormant',
|
|
461
|
+
cancellationGeneration: 0,
|
|
462
|
+
at
|
|
463
|
+
});
|
|
464
|
+
for (const effect of initialEffects) {
|
|
465
|
+
this.insertEffect({
|
|
466
|
+
...effect,
|
|
467
|
+
id: effect.id ?? `${runId}:${effect.actionId}`,
|
|
468
|
+
runId,
|
|
469
|
+
at
|
|
470
|
+
});
|
|
471
|
+
}
|
|
472
|
+
this.appendEvent(runId, 'workflow_started', 'workflow_started', {
|
|
473
|
+
target: input.target,
|
|
474
|
+
bootstrapJob: 'explore:0',
|
|
475
|
+
initialActionIds: initialEffects.map(effect => effect.actionId)
|
|
476
|
+
});
|
|
477
|
+
return { runId };
|
|
478
|
+
}, { runId: input.id, actionId: initialEffects[0]?.actionId });
|
|
479
|
+
const runId = answer.result.runId ?? createdRunId;
|
|
480
|
+
if (!runId)
|
|
481
|
+
throw new OrchestrationError('idempotent Workflow result did not contain a run ID');
|
|
482
|
+
const run = this.getWorkflowRun(runId);
|
|
483
|
+
if (!run)
|
|
484
|
+
throw new OrchestrationError(`Workflow ${runId} disappeared after creation`);
|
|
485
|
+
return { replayed: answer.replayed, run };
|
|
486
|
+
}
|
|
487
|
+
getWorkflowRun(id) {
|
|
488
|
+
const row = this.orm.select().from(workflowRuns).where(eq(workflowRuns.id, id)).get();
|
|
489
|
+
return row ? this.decodeRun(row) : undefined;
|
|
490
|
+
}
|
|
491
|
+
findActiveWorkflowByRoot(rootSessionId) {
|
|
492
|
+
const row = this.orm
|
|
493
|
+
.select()
|
|
494
|
+
.from(workflowRuns)
|
|
495
|
+
.where(and(eq(workflowRuns.rootSessionId, rootSessionId), notInArray(workflowRuns.phase, ['completed', 'cancelled'])))
|
|
496
|
+
.get();
|
|
497
|
+
return row ? this.decodeRun(row) : undefined;
|
|
498
|
+
}
|
|
499
|
+
transitionWorkflowRun(input) {
|
|
500
|
+
return this.immediate(() => {
|
|
501
|
+
const current = this.requireRun(input.runId);
|
|
502
|
+
if (isTerminalWorkflowPhase(current.phase)) {
|
|
503
|
+
throw new WorkflowTransitionError(`terminal Workflow ${current.id} cannot transition from ${current.phase}`);
|
|
504
|
+
}
|
|
505
|
+
this.checkRunGuard(current, input.expectedPhase, input.expectedCancellationGeneration);
|
|
506
|
+
const phase = input.phase ?? current.phase;
|
|
507
|
+
const terminalAt = isTerminalWorkflowPhase(phase) ? (current.terminalAt ?? this.now()) : undefined;
|
|
508
|
+
const blocked = input.blocked === undefined ? current.blocked : (input.blocked ?? undefined);
|
|
509
|
+
const at = this.now();
|
|
510
|
+
try {
|
|
511
|
+
this.db
|
|
512
|
+
.prepare(`UPDATE workflow_runs SET
|
|
513
|
+
phase = ?, cycle = ?, revision = ?, workspace_id = ?, root_session_id = ?,
|
|
514
|
+
pristine_evidence_json = ?, delivery_baseline_json = ?, planning_interpretation = ?,
|
|
515
|
+
blocked_action_id = ?, blocked_error_code = ?, blocked_message = ?, resume_phase = ?,
|
|
516
|
+
retry_class = ?, blocked_candidates_json = ?, blocked_at = ?,
|
|
517
|
+
implementation_batons_delivered = ?, updated_at = ?, terminal_at = ?
|
|
518
|
+
WHERE id = ? AND cancellation_generation = ?`)
|
|
519
|
+
.run(phase, input.cycle ?? current.cycle, input.revision ?? current.revision, input.workspaceId === undefined ? (current.workspaceId ?? null) : input.workspaceId, input.rootSessionId === undefined ? (current.rootSessionId ?? null) : input.rootSessionId, input.pristineEvidence === undefined
|
|
520
|
+
? optionalJson(current.pristineEvidence)
|
|
521
|
+
: optionalJson(input.pristineEvidence), input.deliveryBaseline === undefined
|
|
522
|
+
? optionalJson(current.deliveryBaseline)
|
|
523
|
+
: optionalJson(input.deliveryBaseline), input.planningInterpretation === undefined
|
|
524
|
+
? (current.planningInterpretation ?? null)
|
|
525
|
+
: input.planningInterpretation, blocked?.actionId ?? null, blocked?.errorCode ?? null, blocked?.message ?? null, blocked?.resumePhase ?? null, blocked?.retryClass ?? null, optionalJson(blocked?.candidates), blocked ? (blocked.blockedAt ?? at) : null, input.implementationBatonsDelivered ?? current.implementationBatonsDelivered, at, terminalAt ?? null, input.runId, input.expectedCancellationGeneration);
|
|
526
|
+
}
|
|
527
|
+
catch (error) {
|
|
528
|
+
if (isConstraintError(error)) {
|
|
529
|
+
throw new WorkflowTransitionError('the requested root is already bound to another active Workflow');
|
|
530
|
+
}
|
|
531
|
+
throw error;
|
|
532
|
+
}
|
|
533
|
+
this.appendEvent(input.runId, input.eventKey, input.eventType, input.eventData);
|
|
534
|
+
return this.requireRun(input.runId);
|
|
535
|
+
});
|
|
536
|
+
}
|
|
537
|
+
cancelWorkflowRun(runId, eventKey, eventData) {
|
|
538
|
+
return this.immediate(() => {
|
|
539
|
+
const run = this.requireRun(runId);
|
|
540
|
+
if (run.phase === 'cancelled')
|
|
541
|
+
return run;
|
|
542
|
+
if (run.phase === 'completed')
|
|
543
|
+
throw new WorkflowTransitionError('a completed Workflow cannot be cancelled');
|
|
544
|
+
const at = this.now();
|
|
545
|
+
const generation = run.cancellationGeneration + 1;
|
|
546
|
+
this.db
|
|
547
|
+
.prepare(`UPDATE workflow_runs SET phase = 'cancelled', cancellation_generation = ?, updated_at = ?, terminal_at = ?,
|
|
548
|
+
blocked_action_id = NULL, blocked_error_code = NULL, blocked_message = NULL, resume_phase = NULL,
|
|
549
|
+
retry_class = NULL, blocked_candidates_json = NULL, blocked_at = NULL
|
|
550
|
+
WHERE id = ? AND cancellation_generation = ?`)
|
|
551
|
+
.run(generation, at, at, runId, run.cancellationGeneration);
|
|
552
|
+
this.db
|
|
553
|
+
.prepare(`UPDATE workflow_jobs SET state = 'cancelled', cancellation_generation = ?, updated_at = ?, terminal_at = ?
|
|
554
|
+
WHERE run_id = ? AND state NOT IN ('returned', 'failed', 'cancelled')`)
|
|
555
|
+
.run(generation, at, at, runId);
|
|
556
|
+
this.db
|
|
557
|
+
.prepare(`UPDATE workflow_job_attempts SET state = 'cancelled', updated_at = ?, terminal_at = ?
|
|
558
|
+
WHERE job_id IN (SELECT id FROM workflow_jobs WHERE run_id = ?)
|
|
559
|
+
AND state NOT IN ('returned', 'failed', 'cancelled')`)
|
|
560
|
+
.run(at, at, runId);
|
|
561
|
+
this.db
|
|
562
|
+
.prepare(`UPDATE workflow_effects SET state = 'cancelled', updated_at = ?, terminal_at = ?
|
|
563
|
+
WHERE run_id = ? AND state IN ('prepared', 'failed')`)
|
|
564
|
+
.run(at, at, runId);
|
|
565
|
+
this.db
|
|
566
|
+
.prepare(`UPDATE workflow_effect_attempts SET state = 'cancelled', updated_at = ?, terminal_at = ?
|
|
567
|
+
WHERE effect_id IN (SELECT id FROM workflow_effects WHERE run_id = ?) AND state = 'prepared'`)
|
|
568
|
+
.run(at, at, runId);
|
|
569
|
+
this.appendEvent(runId, eventKey, 'workflow_cancelled', eventData);
|
|
570
|
+
return this.requireRun(runId);
|
|
571
|
+
});
|
|
572
|
+
}
|
|
573
|
+
requireRun(id) {
|
|
574
|
+
const run = this.getWorkflowRun(id);
|
|
575
|
+
if (!run)
|
|
576
|
+
throw new WorkflowTransitionError(`Workflow ${id} does not exist`);
|
|
577
|
+
return run;
|
|
578
|
+
}
|
|
579
|
+
checkRunGuard(run, expectedPhase, expectedCancellationGeneration) {
|
|
580
|
+
const phases = Array.isArray(expectedPhase) ? expectedPhase : expectedPhase ? [expectedPhase] : undefined;
|
|
581
|
+
if (phases && !phases.includes(run.phase)) {
|
|
582
|
+
throw new WorkflowTransitionError(`Workflow ${run.id} is ${run.phase}, expected ${phases.join(' or ')}`);
|
|
583
|
+
}
|
|
584
|
+
if (run.cancellationGeneration !== expectedCancellationGeneration) {
|
|
585
|
+
throw new WorkflowTransitionError(`Workflow ${run.id} cancellation generation changed`);
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
decodeRun(candidate) {
|
|
589
|
+
const row = workflowRunSelectSchema.parse(candidate);
|
|
590
|
+
const blocked = row.blockedActionId
|
|
591
|
+
? {
|
|
592
|
+
actionId: row.blockedActionId,
|
|
593
|
+
errorCode: row.blockedErrorCode ?? 'workflow_blocked',
|
|
594
|
+
message: row.blockedMessage ?? 'Workflow is blocked',
|
|
595
|
+
resumePhase: row.resumePhase ?? 'pending_root',
|
|
596
|
+
retryClass: row.retryClass ?? 'terminal',
|
|
597
|
+
...(row.blockedCandidates ? { candidates: row.blockedCandidates.value } : {}),
|
|
598
|
+
...(row.blockedAt === null ? {} : { blockedAt: row.blockedAt })
|
|
599
|
+
}
|
|
600
|
+
: undefined;
|
|
601
|
+
return {
|
|
602
|
+
id: row.id,
|
|
603
|
+
objective: row.objective,
|
|
604
|
+
target: row.target,
|
|
605
|
+
roles: row.roles,
|
|
606
|
+
phase: row.phase,
|
|
607
|
+
cycle: row.cycle,
|
|
608
|
+
revision: row.revision,
|
|
609
|
+
...(row.workspaceId === null ? {} : { workspaceId: row.workspaceId }),
|
|
610
|
+
...(row.rootSessionId === null ? {} : { rootSessionId: row.rootSessionId }),
|
|
611
|
+
...jsonProperty('pristineEvidence', row.pristineEvidence),
|
|
612
|
+
...jsonProperty('deliveryBaseline', row.deliveryBaseline),
|
|
613
|
+
...(row.planningInterpretation === null ? {} : { planningInterpretation: row.planningInterpretation }),
|
|
614
|
+
cancellationGeneration: row.cancellationGeneration,
|
|
615
|
+
...(blocked ? { blocked } : {}),
|
|
616
|
+
implementationBatonsDelivered: row.implementationBatonsDelivered,
|
|
617
|
+
createdAt: row.createdAt,
|
|
618
|
+
updatedAt: row.updatedAt,
|
|
619
|
+
...(row.terminalAt === null ? {} : { terminalAt: row.terminalAt })
|
|
620
|
+
};
|
|
621
|
+
}
|
|
622
|
+
insertJob(input) {
|
|
623
|
+
this.db
|
|
624
|
+
.prepare(`INSERT INTO workflow_jobs (
|
|
625
|
+
id, run_id, logical_key, role, cycle, revision, resolved_role_json, prompt, state, cancellation_generation,
|
|
626
|
+
transcript_cursor_json, created_at, updated_at
|
|
627
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
628
|
+
.run(input.id, input.runId, input.logicalKey, input.role, input.cycle, input.revision, json(input.resolvedRole), input.prompt, input.state, input.cancellationGeneration, optionalJson(input.transcriptCursor), input.at, input.at);
|
|
629
|
+
}
|
|
630
|
+
createWorkflowJob(input) {
|
|
631
|
+
return this.immediate(() => {
|
|
632
|
+
const run = this.requireRun(input.runId);
|
|
633
|
+
this.checkRunGuard(run, RUNNABLE_PHASES, input.expectedCancellationGeneration);
|
|
634
|
+
const existing = this.orm
|
|
635
|
+
.select()
|
|
636
|
+
.from(workflowJobs)
|
|
637
|
+
.where(and(eq(workflowJobs.runId, input.runId), eq(workflowJobs.logicalKey, input.logicalKey)))
|
|
638
|
+
.get();
|
|
639
|
+
if (existing) {
|
|
640
|
+
const decoded = this.decodeJob(existing);
|
|
641
|
+
if (decoded.role !== input.role ||
|
|
642
|
+
decoded.cycle !== (input.cycle ?? run.cycle) ||
|
|
643
|
+
decoded.revision !== (input.revision ?? run.revision) ||
|
|
644
|
+
canonicalRequestJson(decoded.resolvedRole) !== canonicalRequestJson(input.resolvedRole) ||
|
|
645
|
+
decoded.prompt !== input.prompt) {
|
|
646
|
+
throw new WorkflowTransitionError(`logical job ${input.logicalKey} already has different frozen inputs`);
|
|
647
|
+
}
|
|
648
|
+
return { created: false, job: decoded };
|
|
649
|
+
}
|
|
650
|
+
const id = input.id ?? randomUUID();
|
|
651
|
+
const at = this.now();
|
|
652
|
+
this.insertJob({
|
|
653
|
+
...input,
|
|
654
|
+
id,
|
|
655
|
+
cycle: input.cycle ?? run.cycle,
|
|
656
|
+
revision: input.revision ?? run.revision,
|
|
657
|
+
state: input.state ?? 'queued',
|
|
658
|
+
cancellationGeneration: run.cancellationGeneration,
|
|
659
|
+
at
|
|
660
|
+
});
|
|
661
|
+
this.touchRun(input.runId, at);
|
|
662
|
+
this.appendEvent(input.runId, input.eventKey, 'workflow_job_created', {
|
|
663
|
+
jobId: id,
|
|
664
|
+
logicalKey: input.logicalKey,
|
|
665
|
+
role: input.role,
|
|
666
|
+
cycle: input.cycle ?? run.cycle,
|
|
667
|
+
revision: input.revision ?? run.revision,
|
|
668
|
+
state: input.state ?? 'queued'
|
|
669
|
+
});
|
|
670
|
+
return { created: true, job: this.requireJob(id) };
|
|
671
|
+
});
|
|
672
|
+
}
|
|
673
|
+
activateWorkflowJob(jobId, expectedCancellationGeneration, eventKey, transcriptCursor) {
|
|
674
|
+
return this.updateWorkflowJob({
|
|
675
|
+
jobId,
|
|
676
|
+
expectedStates: ['dormant'],
|
|
677
|
+
expectedCancellationGeneration,
|
|
678
|
+
state: 'queued',
|
|
679
|
+
...(transcriptCursor === undefined ? {} : { transcriptCursor }),
|
|
680
|
+
eventKey,
|
|
681
|
+
eventType: 'workflow_job_activated'
|
|
682
|
+
});
|
|
683
|
+
}
|
|
684
|
+
getWorkflowJob(id) {
|
|
685
|
+
const row = this.orm.select().from(workflowJobs).where(eq(workflowJobs.id, id)).get();
|
|
686
|
+
return row ? this.decodeJob(row) : undefined;
|
|
687
|
+
}
|
|
688
|
+
listWorkflowJobs(runId) {
|
|
689
|
+
return this.orm
|
|
690
|
+
.select()
|
|
691
|
+
.from(workflowJobs)
|
|
692
|
+
.where(eq(workflowJobs.runId, runId))
|
|
693
|
+
.orderBy(asc(workflowJobs.createdAt), asc(workflowJobs.id))
|
|
694
|
+
.all()
|
|
695
|
+
.map(row => this.decodeJob(row));
|
|
696
|
+
}
|
|
697
|
+
claimNextWorkflowJob(owner, runId) {
|
|
698
|
+
validateRelayIdentity(owner);
|
|
699
|
+
return this.immediate(() => {
|
|
700
|
+
const row = this.orm
|
|
701
|
+
.select({ job: workflowJobs })
|
|
702
|
+
.from(workflowJobs)
|
|
703
|
+
.innerJoin(workflowRuns, eq(workflowRuns.id, workflowJobs.runId))
|
|
704
|
+
.where(and(eq(workflowJobs.state, 'queued'), eq(workflowJobs.cancellationGeneration, workflowRuns.cancellationGeneration), notInArray(workflowRuns.phase, ['blocked', 'completed', 'cancelled']), runId === undefined ? undefined : eq(workflowJobs.runId, runId)))
|
|
705
|
+
.orderBy(asc(workflowJobs.createdAt), asc(workflowJobs.id))
|
|
706
|
+
.limit(1)
|
|
707
|
+
.get()?.job;
|
|
708
|
+
if (!row)
|
|
709
|
+
return undefined;
|
|
710
|
+
const at = this.now();
|
|
711
|
+
const result = this.db
|
|
712
|
+
.prepare(`UPDATE workflow_jobs SET state = 'owned', owner_instance_id = ?, owner_pid = ?,
|
|
713
|
+
owner_process_started_at = ?, owner_protocol_version = ?, updated_at = ?
|
|
714
|
+
WHERE id = ? AND state = 'queued'`)
|
|
715
|
+
.run(owner.instanceId, owner.pid, owner.processStartedAt, owner.protocolVersion, at, row.id);
|
|
716
|
+
if (Number(result.changes) !== 1)
|
|
717
|
+
return undefined;
|
|
718
|
+
this.touchRun(row.runId, at);
|
|
719
|
+
this.appendEvent(row.runId, `job_claimed:${row.id}:${row.attemptCount + 1}:${ownerAuditKey(owner)}`, 'workflow_job_claimed', {
|
|
720
|
+
jobId: row.id,
|
|
721
|
+
ownerInstanceId: owner.instanceId
|
|
722
|
+
});
|
|
723
|
+
return this.requireJob(row.id);
|
|
724
|
+
});
|
|
725
|
+
}
|
|
726
|
+
/** Requeue only the pre-effect `owned` state after exact process death is proven. */
|
|
727
|
+
reconcileAbandonedWorkflowJobClaim(input) {
|
|
728
|
+
const observed = this.requireJob(input.jobId);
|
|
729
|
+
if (!observed.owner || observed.state !== 'owned')
|
|
730
|
+
return { status: 'unsafe', job: observed };
|
|
731
|
+
const observedOwner = observed.owner;
|
|
732
|
+
const probe = input.processProbe ?? this.processProbe;
|
|
733
|
+
if (this.probeAlive(observedOwner, probe))
|
|
734
|
+
return { status: 'owner_alive', job: observed };
|
|
735
|
+
return this.immediate(() => {
|
|
736
|
+
const current = this.requireJob(input.jobId);
|
|
737
|
+
if (current.state !== 'owned' ||
|
|
738
|
+
!current.owner ||
|
|
739
|
+
!sameOwner(current.owner, observedOwner) ||
|
|
740
|
+
current.attemptCount !== observed.attemptCount) {
|
|
741
|
+
return { status: 'changed', job: current };
|
|
742
|
+
}
|
|
743
|
+
const at = this.now();
|
|
744
|
+
this.db
|
|
745
|
+
.prepare(`UPDATE workflow_jobs SET state = 'queued', owner_instance_id = NULL, owner_pid = NULL,
|
|
746
|
+
owner_process_started_at = NULL, owner_protocol_version = NULL, updated_at = ? WHERE id = ?`)
|
|
747
|
+
.run(at, current.id);
|
|
748
|
+
this.touchRun(current.runId, at);
|
|
749
|
+
this.appendEvent(current.runId, `${input.eventKey}:${ownerAuditKey(observedOwner)}`, 'workflow_job_claim_recovered', {
|
|
750
|
+
jobId: current.id,
|
|
751
|
+
abandonedInstanceId: observedOwner.instanceId
|
|
752
|
+
});
|
|
753
|
+
return { status: 'requeued', job: this.requireJob(current.id) };
|
|
754
|
+
});
|
|
755
|
+
}
|
|
756
|
+
updateWorkflowJob(input) {
|
|
757
|
+
return this.immediate(() => {
|
|
758
|
+
const job = this.requireJob(input.jobId);
|
|
759
|
+
const run = this.requireRun(job.runId);
|
|
760
|
+
if (isTerminalWorkflowPhase(run.phase) || job.state === 'cancelled') {
|
|
761
|
+
throw new WorkflowTransitionError(`cannot advance terminal job ${job.id}`);
|
|
762
|
+
}
|
|
763
|
+
this.checkRunGuard(run, undefined, input.expectedCancellationGeneration);
|
|
764
|
+
if (!input.expectedStates.includes(job.state)) {
|
|
765
|
+
throw new WorkflowTransitionError(`job ${job.id} is ${job.state}, expected ${input.expectedStates.join(' or ')}`);
|
|
766
|
+
}
|
|
767
|
+
const at = this.now();
|
|
768
|
+
const terminalAt = isTerminalWorkflowJobState(input.state) ? (job.terminalAt ?? at) : null;
|
|
769
|
+
const result = this.db
|
|
770
|
+
.prepare(`UPDATE workflow_jobs SET state = ?, transcript_cursor_json = ?, child_session_id = ?, outcome_json = ?, task_receipt_json = ?,
|
|
771
|
+
baton_receipt_json = ?, owner_instance_id = ?, owner_pid = ?, owner_process_started_at = ?,
|
|
772
|
+
owner_protocol_version = ?, updated_at = ?, terminal_at = ?
|
|
773
|
+
WHERE id = ? AND state = ? AND cancellation_generation = ?`)
|
|
774
|
+
.run(input.state, input.transcriptCursor === undefined
|
|
775
|
+
? optionalJson(job.transcriptCursor)
|
|
776
|
+
: optionalJson(input.transcriptCursor), input.childSessionId === undefined ? (job.childSessionId ?? null) : input.childSessionId, input.outcome === undefined ? optionalJson(job.outcome) : optionalJson(input.outcome), input.taskReceipt === undefined ? optionalJson(job.taskReceipt) : optionalJson(input.taskReceipt), input.batonReceipt === undefined ? optionalJson(job.batonReceipt) : optionalJson(input.batonReceipt), input.clearOwner ? null : (job.owner?.instanceId ?? null), input.clearOwner ? null : (job.owner?.pid ?? null), input.clearOwner ? null : (job.owner?.processStartedAt ?? null), input.clearOwner ? null : (job.owner?.protocolVersion ?? null), at, terminalAt, job.id, job.state, input.expectedCancellationGeneration);
|
|
777
|
+
if (Number(result.changes) !== 1)
|
|
778
|
+
throw new WorkflowTransitionError(`job ${job.id} changed concurrently`);
|
|
779
|
+
this.touchRun(job.runId, at);
|
|
780
|
+
this.appendEvent(job.runId, input.eventKey, input.eventType, { jobId: job.id, ...asObject(input.eventData) });
|
|
781
|
+
return this.requireJob(job.id);
|
|
782
|
+
});
|
|
783
|
+
}
|
|
784
|
+
createWorkflowJobAttempt(input) {
|
|
785
|
+
validateRelayIdentity(input.owner);
|
|
786
|
+
return this.immediate(() => {
|
|
787
|
+
const job = this.requireJob(input.jobId);
|
|
788
|
+
const run = this.requireRun(job.runId);
|
|
789
|
+
if (run.phase === 'completed' ||
|
|
790
|
+
run.phase === 'cancelled' ||
|
|
791
|
+
job.cancellationGeneration !== run.cancellationGeneration) {
|
|
792
|
+
throw new WorkflowTransitionError(`cannot create an attempt for inactive job ${job.id}`);
|
|
793
|
+
}
|
|
794
|
+
if (job.state === 'cancelled' || job.state === 'returned') {
|
|
795
|
+
throw new WorkflowTransitionError(`cannot attempt ${job.state} job ${job.id}`);
|
|
796
|
+
}
|
|
797
|
+
if (job.state !== 'owned' || !job.owner || !sameOwner(job.owner, input.owner)) {
|
|
798
|
+
throw new WorkflowTransitionError(`job ${job.id} is not claimed by this relay`);
|
|
799
|
+
}
|
|
800
|
+
const attemptNumber = job.attemptCount + 1;
|
|
801
|
+
const id = `${job.id}:attempt:${attemptNumber}`;
|
|
802
|
+
const at = this.now();
|
|
803
|
+
this.db
|
|
804
|
+
.prepare(`INSERT INTO workflow_job_attempts (
|
|
805
|
+
id, job_id, attempt_number, state, child_session_id, open_effect_id, configure_effect_id,
|
|
806
|
+
task_effect_id, baton_effect_id, owner_instance_id, owner_pid, owner_process_started_at,
|
|
807
|
+
owner_protocol_version, created_at, updated_at
|
|
808
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
809
|
+
.run(id, job.id, attemptNumber, input.state ?? job.state, input.childSessionId ?? null, input.effectIds?.open ?? null, input.effectIds?.configure ?? null, input.effectIds?.task ?? null, input.effectIds?.baton ?? null, input.owner.instanceId, input.owner.pid, input.owner.processStartedAt, input.owner.protocolVersion, at, at);
|
|
810
|
+
this.db
|
|
811
|
+
.prepare('UPDATE workflow_jobs SET attempt_count = ?, updated_at = ? WHERE id = ?')
|
|
812
|
+
.run(attemptNumber, at, job.id);
|
|
813
|
+
this.touchRun(job.runId, at);
|
|
814
|
+
this.appendEvent(job.runId, `job_attempt:${job.id}:${attemptNumber}`, 'workflow_job_attempt_created', {
|
|
815
|
+
jobId: job.id,
|
|
816
|
+
attemptNumber
|
|
817
|
+
});
|
|
818
|
+
return this.requireJobAttempt(id);
|
|
819
|
+
});
|
|
820
|
+
}
|
|
821
|
+
listWorkflowJobAttempts(jobId) {
|
|
822
|
+
return this.orm
|
|
823
|
+
.select()
|
|
824
|
+
.from(workflowJobAttempts)
|
|
825
|
+
.where(eq(workflowJobAttempts.jobId, jobId))
|
|
826
|
+
.orderBy(asc(workflowJobAttempts.attemptNumber))
|
|
827
|
+
.all()
|
|
828
|
+
.map(row => this.decodeJobAttempt(row));
|
|
829
|
+
}
|
|
830
|
+
updateWorkflowJobAttempt(input) {
|
|
831
|
+
return this.immediate(() => {
|
|
832
|
+
const job = this.requireJob(input.jobId);
|
|
833
|
+
const run = this.requireRun(job.runId);
|
|
834
|
+
if (run.phase === 'completed' ||
|
|
835
|
+
run.phase === 'cancelled' ||
|
|
836
|
+
job.state === 'cancelled' ||
|
|
837
|
+
job.cancellationGeneration !== run.cancellationGeneration) {
|
|
838
|
+
throw new WorkflowTransitionError(`cannot advance an attempt for inactive job ${job.id}`);
|
|
839
|
+
}
|
|
840
|
+
const id = `${job.id}:attempt:${input.attemptNumber}`;
|
|
841
|
+
const attempt = this.requireJobAttempt(id);
|
|
842
|
+
if (attempt.state !== input.expectedState) {
|
|
843
|
+
throw new WorkflowTransitionError(`job attempt ${input.attemptNumber} is ${attempt.state}, expected ${input.expectedState}`);
|
|
844
|
+
}
|
|
845
|
+
const at = this.now();
|
|
846
|
+
const terminalAt = isTerminalWorkflowJobState(input.state) ? at : null;
|
|
847
|
+
const result = this.db
|
|
848
|
+
.prepare(`UPDATE workflow_job_attempts SET state = ?, child_session_id = ?, outcome_json = ?,
|
|
849
|
+
failure_evidence_json = ?, updated_at = ?, terminal_at = ?
|
|
850
|
+
WHERE id = ? AND state = ?`)
|
|
851
|
+
.run(input.state, input.childSessionId === undefined ? (attempt.childSessionId ?? null) : input.childSessionId, input.outcome === undefined ? optionalJson(attempt.outcome) : optionalJson(input.outcome), input.failureEvidence === undefined
|
|
852
|
+
? optionalJson(attempt.failureEvidence)
|
|
853
|
+
: optionalJson(input.failureEvidence), at, terminalAt, id, input.expectedState);
|
|
854
|
+
if (Number(result.changes) !== 1)
|
|
855
|
+
throw new WorkflowTransitionError(`job attempt ${id} changed concurrently`);
|
|
856
|
+
this.touchRun(job.runId, at);
|
|
857
|
+
this.appendEvent(job.runId, input.eventKey, input.eventType, {
|
|
858
|
+
jobId: job.id,
|
|
859
|
+
attemptNumber: input.attemptNumber,
|
|
860
|
+
state: input.state
|
|
861
|
+
});
|
|
862
|
+
return this.requireJobAttempt(id);
|
|
863
|
+
});
|
|
864
|
+
}
|
|
865
|
+
/** Preserve an outcome observed after cancellation without reopening the job. */
|
|
866
|
+
recordLateWorkflowChildResult(input) {
|
|
867
|
+
return this.immediate(() => {
|
|
868
|
+
const job = this.requireJob(input.jobId);
|
|
869
|
+
const run = this.requireRun(input.runId);
|
|
870
|
+
if (job.runId !== run.id)
|
|
871
|
+
throw new WorkflowTransitionError(`job ${job.id} does not belong to Workflow ${run.id}`);
|
|
872
|
+
if (run.phase !== 'cancelled' || job.state !== 'cancelled') {
|
|
873
|
+
throw new WorkflowTransitionError(`job ${job.id} is not a cancelled late-result target`);
|
|
874
|
+
}
|
|
875
|
+
const attemptNumber = input.attemptNumber ?? job.attemptCount;
|
|
876
|
+
const attempt = attemptNumber > 0 ? this.requireJobAttempt(`${job.id}:attempt:${attemptNumber}`) : undefined;
|
|
877
|
+
const prior = this.findEvent(run.id, input.eventKey);
|
|
878
|
+
if (prior) {
|
|
879
|
+
if (prior.type !== 'late_child_result' || canonicalOptional(job.outcome) !== canonicalOptional(input.outcome)) {
|
|
880
|
+
throw new WorkflowTransitionError(`late child event ${input.eventKey} conflicts with existing evidence`);
|
|
881
|
+
}
|
|
882
|
+
return job;
|
|
883
|
+
}
|
|
884
|
+
const at = this.now();
|
|
885
|
+
this.db
|
|
886
|
+
.prepare('UPDATE workflow_jobs SET outcome_json = ?, updated_at = ? WHERE id = ?')
|
|
887
|
+
.run(json(input.outcome), at, job.id);
|
|
888
|
+
if (attempt) {
|
|
889
|
+
this.db
|
|
890
|
+
.prepare('UPDATE workflow_job_attempts SET outcome_json = ?, updated_at = ? WHERE id = ?')
|
|
891
|
+
.run(json(input.outcome), at, attempt.id);
|
|
892
|
+
}
|
|
893
|
+
this.appendEvent(run.id, input.eventKey, 'late_child_result', {
|
|
894
|
+
jobId: job.id,
|
|
895
|
+
attemptNumber,
|
|
896
|
+
outcome: input.outcome
|
|
897
|
+
});
|
|
898
|
+
return this.requireJob(job.id);
|
|
899
|
+
});
|
|
900
|
+
}
|
|
901
|
+
requireJob(id) {
|
|
902
|
+
const job = this.getWorkflowJob(id);
|
|
903
|
+
if (!job)
|
|
904
|
+
throw new WorkflowTransitionError(`Workflow job ${id} does not exist`);
|
|
905
|
+
return job;
|
|
906
|
+
}
|
|
907
|
+
decodeJob(candidate) {
|
|
908
|
+
const row = workflowJobSelectSchema.parse(candidate);
|
|
909
|
+
const owner = ownerFromColumns(row);
|
|
910
|
+
return {
|
|
911
|
+
id: row.id,
|
|
912
|
+
runId: row.runId,
|
|
913
|
+
logicalKey: row.logicalKey,
|
|
914
|
+
role: row.role,
|
|
915
|
+
cycle: row.cycle,
|
|
916
|
+
revision: row.revision,
|
|
917
|
+
resolvedRole: row.resolvedRole,
|
|
918
|
+
prompt: row.prompt,
|
|
919
|
+
state: row.state,
|
|
920
|
+
cancellationGeneration: row.cancellationGeneration,
|
|
921
|
+
...(owner ? { owner } : {}),
|
|
922
|
+
...jsonProperty('transcriptCursor', row.transcriptCursor),
|
|
923
|
+
...(row.childSessionId === null ? {} : { childSessionId: row.childSessionId }),
|
|
924
|
+
...jsonProperty('outcome', row.outcome),
|
|
925
|
+
...jsonProperty('taskReceipt', row.taskReceipt),
|
|
926
|
+
...jsonProperty('batonReceipt', row.batonReceipt),
|
|
927
|
+
attemptCount: row.attemptCount,
|
|
928
|
+
createdAt: row.createdAt,
|
|
929
|
+
updatedAt: row.updatedAt,
|
|
930
|
+
...(row.terminalAt === null ? {} : { terminalAt: row.terminalAt })
|
|
931
|
+
};
|
|
932
|
+
}
|
|
933
|
+
requireJobAttempt(id) {
|
|
934
|
+
const row = this.orm.select().from(workflowJobAttempts).where(eq(workflowJobAttempts.id, id)).get();
|
|
935
|
+
if (!row)
|
|
936
|
+
throw new WorkflowTransitionError(`Workflow job attempt ${id} does not exist`);
|
|
937
|
+
return this.decodeJobAttempt(row);
|
|
938
|
+
}
|
|
939
|
+
decodeJobAttempt(candidate) {
|
|
940
|
+
const row = workflowJobAttemptSelectSchema.parse(candidate);
|
|
941
|
+
const owner = ownerFromColumns(row);
|
|
942
|
+
return {
|
|
943
|
+
id: row.id,
|
|
944
|
+
jobId: row.jobId,
|
|
945
|
+
attemptNumber: row.attemptNumber,
|
|
946
|
+
state: row.state,
|
|
947
|
+
...(row.childSessionId === null ? {} : { childSessionId: row.childSessionId }),
|
|
948
|
+
...(row.openEffectId === null ? {} : { openEffectId: row.openEffectId }),
|
|
949
|
+
...(row.configureEffectId === null ? {} : { configureEffectId: row.configureEffectId }),
|
|
950
|
+
...(row.taskEffectId === null ? {} : { taskEffectId: row.taskEffectId }),
|
|
951
|
+
...(row.batonEffectId === null ? {} : { batonEffectId: row.batonEffectId }),
|
|
952
|
+
...jsonProperty('outcome', row.outcome),
|
|
953
|
+
...jsonProperty('failureEvidence', row.failureEvidence),
|
|
954
|
+
...(owner ? { owner } : {}),
|
|
955
|
+
createdAt: row.createdAt,
|
|
956
|
+
updatedAt: row.updatedAt,
|
|
957
|
+
...(row.terminalAt === null ? {} : { terminalAt: row.terminalAt })
|
|
958
|
+
};
|
|
959
|
+
}
|
|
960
|
+
insertEffect(input) {
|
|
961
|
+
this.db
|
|
962
|
+
.prepare(`INSERT INTO workflow_effects (
|
|
963
|
+
id, run_id, action_id, job_id, kind, state, target_json, inputs_json, baseline_json,
|
|
964
|
+
cursor_json, created_at, updated_at
|
|
965
|
+
) VALUES (?, ?, ?, ?, ?, 'prepared', ?, ?, ?, ?, ?, ?)`)
|
|
966
|
+
.run(input.id, input.runId, input.actionId, input.jobId ?? null, nonEmpty(input.kind, 'effect kind'), optionalJson(input.target), optionalJson(input.inputs), optionalJson(input.baseline), optionalJson(input.cursor), input.at, input.at);
|
|
967
|
+
}
|
|
968
|
+
prepareWorkflowEffect(input) {
|
|
969
|
+
return this.immediate(() => {
|
|
970
|
+
const run = this.requireRun(input.runId);
|
|
971
|
+
this.checkRunGuard(run, ACTIVE_PHASES, input.expectedCancellationGeneration);
|
|
972
|
+
const existing = this.orm
|
|
973
|
+
.select()
|
|
974
|
+
.from(workflowEffects)
|
|
975
|
+
.where(and(eq(workflowEffects.runId, input.runId), eq(workflowEffects.actionId, input.actionId)))
|
|
976
|
+
.get();
|
|
977
|
+
if (existing) {
|
|
978
|
+
const effect = this.decodeEffect(existing);
|
|
979
|
+
const same = effect.kind === input.kind &&
|
|
980
|
+
effect.jobId === input.jobId &&
|
|
981
|
+
canonicalOptional(effect.target) === canonicalOptional(input.target) &&
|
|
982
|
+
canonicalOptional(effect.inputs) === canonicalOptional(input.inputs) &&
|
|
983
|
+
canonicalOptional(effect.baseline) === canonicalOptional(input.baseline) &&
|
|
984
|
+
canonicalOptional(effect.cursor) === canonicalOptional(input.cursor);
|
|
985
|
+
if (!same)
|
|
986
|
+
throw new WorkflowTransitionError(`action ${input.actionId} already has different frozen inputs`);
|
|
987
|
+
return { created: false, effect };
|
|
988
|
+
}
|
|
989
|
+
const id = input.id ?? randomUUID();
|
|
990
|
+
const at = this.now();
|
|
991
|
+
this.insertEffect({ ...input, id, at });
|
|
992
|
+
this.touchRun(input.runId, at);
|
|
993
|
+
this.appendEvent(input.runId, input.eventKey, 'workflow_effect_prepared', {
|
|
994
|
+
effectId: id,
|
|
995
|
+
actionId: input.actionId,
|
|
996
|
+
kind: input.kind,
|
|
997
|
+
jobId: input.jobId
|
|
998
|
+
});
|
|
999
|
+
return { created: true, effect: this.requireEffect(id) };
|
|
1000
|
+
});
|
|
1001
|
+
}
|
|
1002
|
+
getWorkflowEffect(runId, actionId) {
|
|
1003
|
+
const row = this.orm
|
|
1004
|
+
.select()
|
|
1005
|
+
.from(workflowEffects)
|
|
1006
|
+
.where(and(eq(workflowEffects.runId, runId), eq(workflowEffects.actionId, actionId)))
|
|
1007
|
+
.get();
|
|
1008
|
+
return row ? this.decodeEffect(row) : undefined;
|
|
1009
|
+
}
|
|
1010
|
+
listWorkflowEffects(runId) {
|
|
1011
|
+
return this.orm
|
|
1012
|
+
.select()
|
|
1013
|
+
.from(workflowEffects)
|
|
1014
|
+
.where(eq(workflowEffects.runId, runId))
|
|
1015
|
+
.orderBy(asc(workflowEffects.createdAt), asc(workflowEffects.id))
|
|
1016
|
+
.all()
|
|
1017
|
+
.map(row => this.decodeEffect(row));
|
|
1018
|
+
}
|
|
1019
|
+
/** Commit a positively reconciled settings match without entering the UI dispatch boundary. */
|
|
1020
|
+
markWorkflowEffectSatisfiedWithoutDispatch(input) {
|
|
1021
|
+
return this.immediate(() => {
|
|
1022
|
+
const run = this.requireRun(input.runId);
|
|
1023
|
+
this.checkRunGuard(run, RUNNABLE_PHASES, input.expectedCancellationGeneration);
|
|
1024
|
+
const effect = this.requireEffectByAction(input.runId, input.actionId);
|
|
1025
|
+
if (effect.kind !== 'configure_root' && effect.kind !== 'configure_child') {
|
|
1026
|
+
throw new WorkflowTransitionError(`effect ${input.actionId} is not a configuration effect`);
|
|
1027
|
+
}
|
|
1028
|
+
if (effect.state !== 'prepared' || effect.owner || effect.mayExecute) {
|
|
1029
|
+
throw new WorkflowTransitionError(`effect ${input.actionId} is not an unowned prepared configuration`);
|
|
1030
|
+
}
|
|
1031
|
+
const at = this.now();
|
|
1032
|
+
const result = this.db
|
|
1033
|
+
.prepare(`UPDATE workflow_effects SET state = 'committed', receipt_json = ?, updated_at = ?, terminal_at = ?
|
|
1034
|
+
WHERE id = ? AND state = 'prepared' AND owner_instance_id IS NULL AND may_execute = 0`)
|
|
1035
|
+
.run(json(input.receipt), at, at, effect.id);
|
|
1036
|
+
if (Number(result.changes) !== 1) {
|
|
1037
|
+
throw new WorkflowTransitionError(`configuration effect ${input.actionId} changed concurrently`);
|
|
1038
|
+
}
|
|
1039
|
+
this.touchRun(input.runId, at);
|
|
1040
|
+
this.appendEvent(input.runId, input.eventKey ?? `effect_satisfied_without_dispatch:${input.actionId}`, 'workflow_effect_satisfied_without_dispatch', { effectId: effect.id, actionId: effect.actionId, receipt: input.receipt });
|
|
1041
|
+
return this.requireEffect(effect.id);
|
|
1042
|
+
});
|
|
1043
|
+
}
|
|
1044
|
+
claimPreparedWorkflowEffect(input) {
|
|
1045
|
+
validateRelayIdentity(input.owner);
|
|
1046
|
+
return this.immediate(() => {
|
|
1047
|
+
const run = this.requireRun(input.runId);
|
|
1048
|
+
this.checkRunGuard(run, RUNNABLE_PHASES, input.expectedCancellationGeneration);
|
|
1049
|
+
const effect = this.getWorkflowEffect(input.runId, input.actionId);
|
|
1050
|
+
if (effect?.state !== 'prepared' || effect.owner)
|
|
1051
|
+
return undefined;
|
|
1052
|
+
const attemptNumber = effect.attemptCount + 1;
|
|
1053
|
+
const attemptId = `${effect.id}:attempt:${attemptNumber}`;
|
|
1054
|
+
const at = this.now();
|
|
1055
|
+
const result = this.db
|
|
1056
|
+
.prepare(`UPDATE workflow_effects SET owner_instance_id = ?, owner_pid = ?, owner_process_started_at = ?,
|
|
1057
|
+
owner_protocol_version = ?, attempt_count = ?, updated_at = ?
|
|
1058
|
+
WHERE id = ? AND state = 'prepared' AND owner_instance_id IS NULL`)
|
|
1059
|
+
.run(input.owner.instanceId, input.owner.pid, input.owner.processStartedAt, input.owner.protocolVersion, attemptNumber, at, effect.id);
|
|
1060
|
+
if (Number(result.changes) !== 1)
|
|
1061
|
+
return undefined;
|
|
1062
|
+
this.db
|
|
1063
|
+
.prepare(`INSERT INTO workflow_effect_attempts (
|
|
1064
|
+
id, effect_id, attempt_number, state, owner_instance_id, owner_pid,
|
|
1065
|
+
owner_process_started_at, owner_protocol_version, created_at, updated_at
|
|
1066
|
+
) VALUES (?, ?, ?, 'prepared', ?, ?, ?, ?, ?, ?)`)
|
|
1067
|
+
.run(attemptId, effect.id, attemptNumber, input.owner.instanceId, input.owner.pid, input.owner.processStartedAt, input.owner.protocolVersion, at, at);
|
|
1068
|
+
this.touchRun(input.runId, at);
|
|
1069
|
+
this.appendEvent(input.runId, `effect_claimed:${input.actionId}:${attemptNumber}`, 'workflow_effect_claimed', {
|
|
1070
|
+
effectId: effect.id,
|
|
1071
|
+
actionId: input.actionId,
|
|
1072
|
+
attemptNumber
|
|
1073
|
+
});
|
|
1074
|
+
return { effect: this.requireEffect(effect.id), attempt: this.requireEffectAttempt(attemptId) };
|
|
1075
|
+
});
|
|
1076
|
+
}
|
|
1077
|
+
markWorkflowEffectDispatched(input) {
|
|
1078
|
+
validateRelayIdentity(input.owner);
|
|
1079
|
+
if (input.externalProcess)
|
|
1080
|
+
validateProcessIdentity(input.externalProcess, 'external effect process');
|
|
1081
|
+
return this.immediate(() => {
|
|
1082
|
+
const run = this.requireRun(input.runId);
|
|
1083
|
+
if (!RUNNABLE_PHASES.includes(run.phase)) {
|
|
1084
|
+
throw new WorkflowTransitionError(`Workflow ${run.id} cannot dispatch a UI effect while ${run.phase}`);
|
|
1085
|
+
}
|
|
1086
|
+
return this.transitionEffect({
|
|
1087
|
+
...input,
|
|
1088
|
+
from: ['prepared'],
|
|
1089
|
+
to: 'dispatched',
|
|
1090
|
+
mayExecute: input.mayExecute ?? false,
|
|
1091
|
+
eventKey: input.eventKey ?? `effect_dispatched:${input.actionId}:${input.attemptNumber}`,
|
|
1092
|
+
eventType: 'workflow_effect_dispatched'
|
|
1093
|
+
});
|
|
1094
|
+
});
|
|
1095
|
+
}
|
|
1096
|
+
markWorkflowEffectMayExecute(input) {
|
|
1097
|
+
validateRelayIdentity(input.owner);
|
|
1098
|
+
if (input.externalProcess)
|
|
1099
|
+
validateProcessIdentity(input.externalProcess, 'external effect process');
|
|
1100
|
+
return this.immediate(() => {
|
|
1101
|
+
const run = this.requireRun(input.runId);
|
|
1102
|
+
if (!RUNNABLE_PHASES.includes(run.phase)) {
|
|
1103
|
+
throw new WorkflowTransitionError(`Workflow ${run.id} cannot release a UI gate while ${run.phase}`);
|
|
1104
|
+
}
|
|
1105
|
+
const effect = this.requireEffectByAction(input.runId, input.actionId);
|
|
1106
|
+
this.requireEffectOwner(effect, input.owner, input.attemptNumber);
|
|
1107
|
+
if (effect.state !== 'dispatched' || effect.launchNonce !== input.launchNonce) {
|
|
1108
|
+
throw new WorkflowTransitionError(`effect ${input.actionId} is not the matching dispatched attempt`);
|
|
1109
|
+
}
|
|
1110
|
+
const identityKey = input.externalProcess ? processAuditKey(input.externalProcess) : 'in-process';
|
|
1111
|
+
const eventKey = `effect_may_execute:${input.actionId}:${input.attemptNumber}:${identityKey}`;
|
|
1112
|
+
const prior = this.findEvent(input.runId, eventKey);
|
|
1113
|
+
if (prior) {
|
|
1114
|
+
if (prior.type !== 'workflow_effect_may_execute') {
|
|
1115
|
+
throw new WorkflowTransitionError(`effect execution event ${eventKey} conflicts with existing audit`);
|
|
1116
|
+
}
|
|
1117
|
+
return effect;
|
|
1118
|
+
}
|
|
1119
|
+
const at = this.now();
|
|
1120
|
+
const externalProcess = input.externalProcess ?? effect.externalProcess;
|
|
1121
|
+
this.db
|
|
1122
|
+
.prepare(`UPDATE workflow_effects SET may_execute = 1, external_pid = ?, external_process_started_at = ?,
|
|
1123
|
+
external_process_group = ?, updated_at = ? WHERE id = ?`)
|
|
1124
|
+
.run(externalProcess?.pid ?? null, externalProcess?.processStartedAt ?? null, externalProcess?.processGroup ?? null, at, effect.id);
|
|
1125
|
+
this.db
|
|
1126
|
+
.prepare(`UPDATE workflow_effect_attempts SET may_execute = 1, external_pid = ?,
|
|
1127
|
+
external_process_started_at = ?, external_process_group = ?, updated_at = ?
|
|
1128
|
+
WHERE effect_id = ? AND attempt_number = ?`)
|
|
1129
|
+
.run(externalProcess?.pid ?? null, externalProcess?.processStartedAt ?? null, externalProcess?.processGroup ?? null, at, effect.id, input.attemptNumber);
|
|
1130
|
+
this.touchRun(input.runId, at);
|
|
1131
|
+
this.appendEvent(input.runId, eventKey, 'workflow_effect_may_execute', {
|
|
1132
|
+
effectId: effect.id,
|
|
1133
|
+
actionId: input.actionId,
|
|
1134
|
+
attemptNumber: input.attemptNumber,
|
|
1135
|
+
...(input.externalProcess ? { externalProcess: input.externalProcess } : {})
|
|
1136
|
+
});
|
|
1137
|
+
return this.requireEffect(effect.id);
|
|
1138
|
+
});
|
|
1139
|
+
}
|
|
1140
|
+
markWorkflowEffectCommitted(input) {
|
|
1141
|
+
return this.transitionEffect({
|
|
1142
|
+
...input,
|
|
1143
|
+
from: ['dispatched', 'ambiguous'],
|
|
1144
|
+
to: 'committed',
|
|
1145
|
+
eventKey: input.eventKey ?? `effect_committed:${input.actionId}`,
|
|
1146
|
+
eventType: 'workflow_effect_committed'
|
|
1147
|
+
});
|
|
1148
|
+
}
|
|
1149
|
+
markWorkflowEffectFailed(input) {
|
|
1150
|
+
return this.transitionEffect({
|
|
1151
|
+
...input,
|
|
1152
|
+
from: ['prepared'],
|
|
1153
|
+
to: 'failed',
|
|
1154
|
+
eventKey: input.eventKey ?? `effect_failed:${input.actionId}:${input.attemptNumber}`,
|
|
1155
|
+
eventType: 'workflow_effect_failed'
|
|
1156
|
+
});
|
|
1157
|
+
}
|
|
1158
|
+
markWorkflowEffectFailedBeforeMayExecute(input) {
|
|
1159
|
+
return this.transitionEffect({
|
|
1160
|
+
...input,
|
|
1161
|
+
from: ['dispatched'],
|
|
1162
|
+
to: 'failed',
|
|
1163
|
+
requireMayExecute: false,
|
|
1164
|
+
mayExecute: false,
|
|
1165
|
+
eventKey: input.eventKey ?? `effect_failed_before_execute:${input.actionId}:${input.attemptNumber}`,
|
|
1166
|
+
eventType: 'workflow_effect_failed_before_may_execute'
|
|
1167
|
+
});
|
|
1168
|
+
}
|
|
1169
|
+
markWorkflowEffectAmbiguous(input) {
|
|
1170
|
+
return this.transitionEffect({
|
|
1171
|
+
...input,
|
|
1172
|
+
from: ['dispatched'],
|
|
1173
|
+
to: 'ambiguous',
|
|
1174
|
+
eventKey: input.eventKey ?? `effect_ambiguous:${input.actionId}`,
|
|
1175
|
+
eventType: 'workflow_effect_ambiguous'
|
|
1176
|
+
});
|
|
1177
|
+
}
|
|
1178
|
+
markWorkflowEffectReceiptLost(input) {
|
|
1179
|
+
return this.immediate(() => {
|
|
1180
|
+
const effect = this.requireEffectByAction(input.runId, input.actionId);
|
|
1181
|
+
const receipt = asObject(input.expectedReceipt);
|
|
1182
|
+
if (receipt.kind !== 'outbox' || typeof receipt.id !== 'string') {
|
|
1183
|
+
throw new WorkflowTransitionError('only a tagged outbox receipt can become lost');
|
|
1184
|
+
}
|
|
1185
|
+
if (canonicalOptional(effect.receipt) !== canonicalOptional(input.expectedReceipt)) {
|
|
1186
|
+
throw new WorkflowTransitionError(`effect ${input.actionId} receipt changed before loss reconciliation`);
|
|
1187
|
+
}
|
|
1188
|
+
if (effect.state === 'ambiguous' && effect.errorCode === input.errorCode)
|
|
1189
|
+
return effect;
|
|
1190
|
+
if (effect.state !== 'committed') {
|
|
1191
|
+
throw new WorkflowTransitionError(`effect ${input.actionId} is ${effect.state}, expected committed`);
|
|
1192
|
+
}
|
|
1193
|
+
const at = this.now();
|
|
1194
|
+
this.db
|
|
1195
|
+
.prepare(`UPDATE workflow_effects SET state = 'ambiguous', error_code = ?, error_message = ?,
|
|
1196
|
+
updated_at = ?, terminal_at = ? WHERE id = ? AND state = 'committed'`)
|
|
1197
|
+
.run(input.errorCode, input.errorMessage, at, at, effect.id);
|
|
1198
|
+
if (effect.attemptCount > 0) {
|
|
1199
|
+
this.db
|
|
1200
|
+
.prepare(`UPDATE workflow_effect_attempts SET state = 'ambiguous', evidence_json = ?, error_code = ?,
|
|
1201
|
+
error_message = ?, updated_at = ?, terminal_at = ? WHERE effect_id = ? AND attempt_number = ?`)
|
|
1202
|
+
.run(optionalJson(input.evidence), input.errorCode, input.errorMessage, at, at, effect.id, effect.attemptCount);
|
|
1203
|
+
}
|
|
1204
|
+
this.touchRun(input.runId, at);
|
|
1205
|
+
this.appendEvent(input.runId, input.eventKey ?? `effect_receipt_lost:${input.actionId}:${String(receipt.id)}`, 'workflow_effect_receipt_lost', { effectId: effect.id, actionId: input.actionId, errorCode: input.errorCode });
|
|
1206
|
+
return this.requireEffect(effect.id);
|
|
1207
|
+
});
|
|
1208
|
+
}
|
|
1209
|
+
/** Record a post-cancellation receipt without reopening the run or effect. */
|
|
1210
|
+
recordLateWorkflowEffect(input) {
|
|
1211
|
+
return this.immediate(() => {
|
|
1212
|
+
const run = this.requireRun(input.runId);
|
|
1213
|
+
if (run.phase !== 'cancelled') {
|
|
1214
|
+
throw new WorkflowTransitionError(`Workflow ${run.id} is not cancelled`);
|
|
1215
|
+
}
|
|
1216
|
+
const effect = this.requireEffectByAction(run.id, input.actionId);
|
|
1217
|
+
const prior = this.findEvent(run.id, input.eventKey);
|
|
1218
|
+
if (prior) {
|
|
1219
|
+
if (prior.type !== 'late_effect' || canonicalOptional(effect.receipt) !== canonicalOptional(input.receipt)) {
|
|
1220
|
+
throw new WorkflowTransitionError(`late effect event ${input.eventKey} conflicts with existing evidence`);
|
|
1221
|
+
}
|
|
1222
|
+
return effect;
|
|
1223
|
+
}
|
|
1224
|
+
const at = this.now();
|
|
1225
|
+
const settled = effect.state === 'dispatched' || effect.state === 'ambiguous';
|
|
1226
|
+
this.db
|
|
1227
|
+
.prepare(`UPDATE workflow_effects SET state = ?, receipt_json = ?, error_code = ?, error_message = ?,
|
|
1228
|
+
updated_at = ?, terminal_at = COALESCE(terminal_at, ?) WHERE id = ?`)
|
|
1229
|
+
.run(settled ? 'committed' : effect.state, json(input.receipt), settled ? null : (effect.errorCode ?? null), settled ? null : (effect.errorMessage ?? null), at, settled ? at : (effect.terminalAt ?? null), effect.id);
|
|
1230
|
+
if (effect.attemptCount > 0) {
|
|
1231
|
+
this.db
|
|
1232
|
+
.prepare(`UPDATE workflow_effect_attempts SET state = CASE WHEN state IN ('dispatched', 'ambiguous') THEN 'committed' ELSE state END,
|
|
1233
|
+
receipt_json = ?, error_code = CASE WHEN state IN ('dispatched', 'ambiguous') THEN NULL ELSE error_code END,
|
|
1234
|
+
error_message = CASE WHEN state IN ('dispatched', 'ambiguous') THEN NULL ELSE error_message END,
|
|
1235
|
+
updated_at = ?, terminal_at = CASE WHEN state IN ('dispatched', 'ambiguous') THEN COALESCE(terminal_at, ?) ELSE terminal_at END
|
|
1236
|
+
WHERE effect_id = ? AND attempt_number = ?`)
|
|
1237
|
+
.run(json(input.receipt), at, at, effect.id, effect.attemptCount);
|
|
1238
|
+
}
|
|
1239
|
+
const receipt = asObject(input.receipt);
|
|
1240
|
+
const deliveredMessage = receipt.kind === 'message' &&
|
|
1241
|
+
typeof receipt.id === 'string' &&
|
|
1242
|
+
Number.isSafeInteger(receipt.rowid) &&
|
|
1243
|
+
(receipt.turnId === null || typeof receipt.turnId === 'string');
|
|
1244
|
+
if (effect.jobId && deliveredMessage && (effect.kind === 'send_task' || effect.kind === 'return_baton')) {
|
|
1245
|
+
const receiptColumn = effect.kind === 'send_task' ? 'task_receipt_json' : 'baton_receipt_json';
|
|
1246
|
+
this.db
|
|
1247
|
+
.prepare(`UPDATE workflow_jobs SET ${receiptColumn} = ?, updated_at = ? WHERE id = ? AND run_id = ?`)
|
|
1248
|
+
.run(json(input.receipt), at, effect.jobId, run.id);
|
|
1249
|
+
}
|
|
1250
|
+
// A positive receipt is stronger than an earlier ambiguity. Clear only the
|
|
1251
|
+
// hold for this exact effect inside the same transaction; a newer unrelated
|
|
1252
|
+
// quarantine can never be erased by a late callback.
|
|
1253
|
+
this.db
|
|
1254
|
+
.prepare(`UPDATE ui_quarantine SET active = 0, cleared_at = ?, cleared_by = ?
|
|
1255
|
+
WHERE id = 1 AND active = 1 AND (
|
|
1256
|
+
effect_id = ? OR (effect_id IS NULL AND action_id IN (?, ?))
|
|
1257
|
+
)`)
|
|
1258
|
+
.run(at, `late-effect:${input.eventKey}`, effect.id, effect.id, effect.actionId);
|
|
1259
|
+
this.touchRun(run.id, at);
|
|
1260
|
+
this.appendEvent(run.id, input.eventKey, 'late_effect', {
|
|
1261
|
+
effectId: effect.id,
|
|
1262
|
+
actionId: effect.actionId,
|
|
1263
|
+
receipt: input.receipt
|
|
1264
|
+
});
|
|
1265
|
+
return this.requireEffect(effect.id);
|
|
1266
|
+
});
|
|
1267
|
+
}
|
|
1268
|
+
markWorkflowEffectCancelled(input) {
|
|
1269
|
+
return this.transitionEffect({
|
|
1270
|
+
...input,
|
|
1271
|
+
from: ['prepared', 'failed'],
|
|
1272
|
+
to: 'cancelled',
|
|
1273
|
+
eventKey: input.eventKey ?? `effect_cancelled:${input.actionId}`,
|
|
1274
|
+
eventType: 'workflow_effect_cancelled'
|
|
1275
|
+
});
|
|
1276
|
+
}
|
|
1277
|
+
/**
|
|
1278
|
+
* Reconcile an orphan only after the caller checked for a positive receipt.
|
|
1279
|
+
* A dead owner before `mayExecute` is safely reset; every later boundary is ambiguous.
|
|
1280
|
+
*/
|
|
1281
|
+
reconcileAbandonedWorkflowEffect(input) {
|
|
1282
|
+
const observed = this.requireEffectByAction(input.runId, input.actionId);
|
|
1283
|
+
if (!observed.owner)
|
|
1284
|
+
return { status: 'unowned', effect: observed };
|
|
1285
|
+
if (TERMINAL_EFFECT_STATES.includes(observed.state))
|
|
1286
|
+
return { status: 'terminal', effect: observed };
|
|
1287
|
+
const probe = input.processProbe ?? this.processProbe;
|
|
1288
|
+
if (this.probeAlive(observed.owner, probe))
|
|
1289
|
+
return { status: 'owner_alive', effect: observed };
|
|
1290
|
+
if (observed.externalProcess && this.probeAlive(observed.externalProcess, probe)) {
|
|
1291
|
+
return { status: 'external_process_alive', effect: observed };
|
|
1292
|
+
}
|
|
1293
|
+
return this.immediate(() => {
|
|
1294
|
+
const current = this.requireEffectByAction(input.runId, input.actionId);
|
|
1295
|
+
const run = this.requireRun(input.runId);
|
|
1296
|
+
if (!current.owner ||
|
|
1297
|
+
!sameOwner(current.owner, observed.owner) ||
|
|
1298
|
+
current.state !== observed.state ||
|
|
1299
|
+
current.attemptCount !== observed.attemptCount ||
|
|
1300
|
+
current.mayExecute !== observed.mayExecute ||
|
|
1301
|
+
!sameOptionalProcess(current.externalProcess, observed.externalProcess)) {
|
|
1302
|
+
return { status: 'changed', effect: current };
|
|
1303
|
+
}
|
|
1304
|
+
const safelyPrepared = current.state === 'prepared' || (current.state === 'dispatched' && !current.mayExecute);
|
|
1305
|
+
const terminalRun = isTerminalWorkflowPhase(run.phase);
|
|
1306
|
+
const at = this.now();
|
|
1307
|
+
if (safelyPrepared) {
|
|
1308
|
+
this.db
|
|
1309
|
+
.prepare(`UPDATE workflow_effects SET state = ?, owner_instance_id = NULL, owner_pid = NULL,
|
|
1310
|
+
owner_process_started_at = NULL, owner_protocol_version = NULL, launch_nonce = NULL,
|
|
1311
|
+
external_pid = NULL, external_process_started_at = NULL, external_process_group = NULL,
|
|
1312
|
+
may_execute = 0, error_code = NULL, error_message = NULL, updated_at = ?, terminal_at = ?
|
|
1313
|
+
WHERE id = ?`)
|
|
1314
|
+
.run(terminalRun ? 'cancelled' : 'prepared', at, terminalRun ? at : null, current.id);
|
|
1315
|
+
if (current.attemptCount > 0) {
|
|
1316
|
+
this.db
|
|
1317
|
+
.prepare(`UPDATE workflow_effect_attempts SET state = 'cancelled', evidence_json = ?, updated_at = ?, terminal_at = ?
|
|
1318
|
+
WHERE effect_id = ? AND attempt_number = ?`)
|
|
1319
|
+
.run(json({ reason: 'owner_died_before_may_execute' }), at, at, current.id, current.attemptCount);
|
|
1320
|
+
}
|
|
1321
|
+
this.touchRun(current.runId, at);
|
|
1322
|
+
this.appendEvent(current.runId, input.eventKey, terminalRun ? 'workflow_effect_cancelled_after_owner_exit' : 'workflow_effect_safely_recovered', {
|
|
1323
|
+
effectId: current.id,
|
|
1324
|
+
actionId: current.actionId,
|
|
1325
|
+
attemptNumber: current.attemptCount
|
|
1326
|
+
});
|
|
1327
|
+
return {
|
|
1328
|
+
status: terminalRun ? 'terminal' : 'safely_prepared',
|
|
1329
|
+
effect: this.requireEffect(current.id)
|
|
1330
|
+
};
|
|
1331
|
+
}
|
|
1332
|
+
this.db
|
|
1333
|
+
.prepare(`UPDATE workflow_effects SET state = 'ambiguous', error_code = 'ambiguous_effect',
|
|
1334
|
+
error_message = 'The UI action may have executed before its relay owner exited.',
|
|
1335
|
+
updated_at = ?, terminal_at = ? WHERE id = ?`)
|
|
1336
|
+
.run(at, at, current.id);
|
|
1337
|
+
if (current.attemptCount > 0) {
|
|
1338
|
+
this.db
|
|
1339
|
+
.prepare(`UPDATE workflow_effect_attempts SET state = 'ambiguous', error_code = 'ambiguous_effect',
|
|
1340
|
+
error_message = 'The UI action may have executed before its relay owner exited.',
|
|
1341
|
+
updated_at = ?, terminal_at = ? WHERE effect_id = ? AND attempt_number = ?`)
|
|
1342
|
+
.run(at, at, current.id, current.attemptCount);
|
|
1343
|
+
}
|
|
1344
|
+
this.touchRun(current.runId, at);
|
|
1345
|
+
this.appendEvent(current.runId, input.eventKey, 'workflow_effect_abandoned_ambiguous', {
|
|
1346
|
+
effectId: current.id,
|
|
1347
|
+
actionId: current.actionId,
|
|
1348
|
+
attemptNumber: current.attemptCount
|
|
1349
|
+
});
|
|
1350
|
+
return { status: 'ambiguous', effect: this.requireEffect(current.id) };
|
|
1351
|
+
});
|
|
1352
|
+
}
|
|
1353
|
+
retryWorkflowEffect(runId, actionId, eventKey) {
|
|
1354
|
+
return this.immediate(() => {
|
|
1355
|
+
const run = this.requireRun(runId);
|
|
1356
|
+
if (isTerminalWorkflowPhase(run.phase)) {
|
|
1357
|
+
throw new WorkflowTransitionError(`terminal Workflow ${run.id} cannot retry effects`);
|
|
1358
|
+
}
|
|
1359
|
+
const effect = this.requireEffectByAction(runId, actionId);
|
|
1360
|
+
if (effect.state !== 'failed')
|
|
1361
|
+
throw new WorkflowTransitionError(`effect ${actionId} is not deterministically failed`);
|
|
1362
|
+
const at = this.now();
|
|
1363
|
+
this.db
|
|
1364
|
+
.prepare(`UPDATE workflow_effects SET state = 'prepared', owner_instance_id = NULL, owner_pid = NULL,
|
|
1365
|
+
owner_process_started_at = NULL, owner_protocol_version = NULL, launch_nonce = NULL,
|
|
1366
|
+
external_pid = NULL, external_process_started_at = NULL, external_process_group = NULL,
|
|
1367
|
+
may_execute = 0, error_code = NULL, error_message = NULL, updated_at = ?, terminal_at = NULL
|
|
1368
|
+
WHERE id = ?`)
|
|
1369
|
+
.run(at, effect.id);
|
|
1370
|
+
this.touchRun(runId, at);
|
|
1371
|
+
this.appendEvent(runId, eventKey, 'workflow_effect_retry_prepared', {
|
|
1372
|
+
effectId: effect.id,
|
|
1373
|
+
actionId,
|
|
1374
|
+
nextAttempt: effect.attemptCount + 1
|
|
1375
|
+
});
|
|
1376
|
+
return this.requireEffect(effect.id);
|
|
1377
|
+
});
|
|
1378
|
+
}
|
|
1379
|
+
/** Used only behind the explicit, idempotent phone confirmation for duplicate risk. */
|
|
1380
|
+
replayAmbiguousWorkflowEffect(runId, actionId, eventKey) {
|
|
1381
|
+
return this.immediate(() => {
|
|
1382
|
+
const run = this.requireRun(runId);
|
|
1383
|
+
if (isTerminalWorkflowPhase(run.phase)) {
|
|
1384
|
+
throw new WorkflowTransitionError(`terminal Workflow ${run.id} cannot replay effects`);
|
|
1385
|
+
}
|
|
1386
|
+
const effect = this.requireEffectByAction(runId, actionId);
|
|
1387
|
+
if (effect.state !== 'ambiguous') {
|
|
1388
|
+
throw new WorkflowTransitionError(`effect ${actionId} is not ambiguous`);
|
|
1389
|
+
}
|
|
1390
|
+
const at = this.now();
|
|
1391
|
+
this.db
|
|
1392
|
+
.prepare(`UPDATE workflow_effects SET state = 'prepared', owner_instance_id = NULL, owner_pid = NULL,
|
|
1393
|
+
owner_process_started_at = NULL, owner_protocol_version = NULL, launch_nonce = NULL,
|
|
1394
|
+
external_pid = NULL, external_process_started_at = NULL, external_process_group = NULL,
|
|
1395
|
+
may_execute = 0, receipt_json = NULL, error_code = NULL, error_message = NULL,
|
|
1396
|
+
updated_at = ?, terminal_at = NULL WHERE id = ?`)
|
|
1397
|
+
.run(at, effect.id);
|
|
1398
|
+
this.touchRun(runId, at);
|
|
1399
|
+
this.appendEvent(runId, eventKey, 'workflow_effect_risky_replay_prepared', {
|
|
1400
|
+
effectId: effect.id,
|
|
1401
|
+
actionId,
|
|
1402
|
+
previousAttempt: effect.attemptCount,
|
|
1403
|
+
nextAttempt: effect.attemptCount + 1
|
|
1404
|
+
});
|
|
1405
|
+
return this.requireEffect(effect.id);
|
|
1406
|
+
});
|
|
1407
|
+
}
|
|
1408
|
+
transitionEffect(input) {
|
|
1409
|
+
return this.immediate(() => {
|
|
1410
|
+
const effect = this.requireEffectByAction(input.runId, input.actionId);
|
|
1411
|
+
const run = this.requireRun(input.runId);
|
|
1412
|
+
if (!input.from.includes(effect.state)) {
|
|
1413
|
+
throw new WorkflowTransitionError(`effect ${input.actionId} is ${effect.state}, expected ${input.from.join(' or ')}`);
|
|
1414
|
+
}
|
|
1415
|
+
if (input.requireMayExecute !== undefined && effect.mayExecute !== input.requireMayExecute) {
|
|
1416
|
+
throw new WorkflowTransitionError(`effect ${input.actionId} mayExecute evidence changed`);
|
|
1417
|
+
}
|
|
1418
|
+
const attemptNumber = input.attemptNumber ?? (effect.attemptCount > 0 ? effect.attemptCount : undefined);
|
|
1419
|
+
if (input.owner && attemptNumber !== undefined) {
|
|
1420
|
+
this.requireEffectOwner(effect, input.owner, attemptNumber);
|
|
1421
|
+
}
|
|
1422
|
+
const at = this.now();
|
|
1423
|
+
const terminalAt = TERMINAL_EFFECT_STATES.includes(input.to) ? at : null;
|
|
1424
|
+
const externalProcess = input.externalProcess ?? effect.externalProcess;
|
|
1425
|
+
this.db
|
|
1426
|
+
.prepare(`UPDATE workflow_effects SET state = ?, launch_nonce = ?, external_pid = ?,
|
|
1427
|
+
external_process_started_at = ?, external_process_group = ?, may_execute = ?, receipt_json = ?,
|
|
1428
|
+
error_code = ?, error_message = ?, updated_at = ?, terminal_at = ? WHERE id = ?`)
|
|
1429
|
+
.run(input.to, input.launchNonce ?? effect.launchNonce ?? null, externalProcess?.pid ?? null, externalProcess?.processStartedAt ?? null, externalProcess?.processGroup ?? null, input.mayExecute === undefined ? (effect.mayExecute ? 1 : 0) : input.mayExecute ? 1 : 0, input.receipt === undefined ? optionalJson(effect.receipt) : optionalJson(input.receipt), input.errorCode ?? effect.errorCode ?? null, input.errorMessage ?? effect.errorMessage ?? null, at, terminalAt, effect.id);
|
|
1430
|
+
if (attemptNumber !== undefined) {
|
|
1431
|
+
this.db
|
|
1432
|
+
.prepare(`UPDATE workflow_effect_attempts SET state = ?, launch_nonce = ?, external_pid = ?,
|
|
1433
|
+
external_process_started_at = ?, external_process_group = ?, may_execute = ?, receipt_json = ?,
|
|
1434
|
+
evidence_json = ?, error_code = ?, error_message = ?, updated_at = ?, terminal_at = ?
|
|
1435
|
+
WHERE effect_id = ? AND attempt_number = ?`)
|
|
1436
|
+
.run(input.to, input.launchNonce ?? effect.launchNonce ?? null, externalProcess?.pid ?? null, externalProcess?.processStartedAt ?? null, externalProcess?.processGroup ?? null, input.mayExecute === undefined ? (effect.mayExecute ? 1 : 0) : input.mayExecute ? 1 : 0, input.receipt === undefined ? optionalJson(effect.receipt) : optionalJson(input.receipt), optionalJson(input.evidence), input.errorCode ?? null, input.errorMessage ?? null, at, terminalAt, effect.id, attemptNumber);
|
|
1437
|
+
}
|
|
1438
|
+
this.touchRun(input.runId, at);
|
|
1439
|
+
const late = input.to === 'committed' && isTerminalWorkflowPhase(run.phase);
|
|
1440
|
+
this.appendEvent(input.runId, input.eventKey, late ? 'late_effect' : input.eventType, {
|
|
1441
|
+
effectId: effect.id,
|
|
1442
|
+
actionId: input.actionId,
|
|
1443
|
+
attemptNumber,
|
|
1444
|
+
errorCode: input.errorCode,
|
|
1445
|
+
...(late ? { intendedEventType: input.eventType } : {})
|
|
1446
|
+
});
|
|
1447
|
+
return this.requireEffect(effect.id);
|
|
1448
|
+
});
|
|
1449
|
+
}
|
|
1450
|
+
requireEffectByAction(runId, actionId) {
|
|
1451
|
+
const effect = this.getWorkflowEffect(runId, actionId);
|
|
1452
|
+
if (!effect)
|
|
1453
|
+
throw new WorkflowTransitionError(`Workflow effect ${actionId} does not exist`);
|
|
1454
|
+
return effect;
|
|
1455
|
+
}
|
|
1456
|
+
requireEffect(id) {
|
|
1457
|
+
const row = this.orm.select().from(workflowEffects).where(eq(workflowEffects.id, id)).get();
|
|
1458
|
+
if (!row)
|
|
1459
|
+
throw new WorkflowTransitionError(`Workflow effect ${id} does not exist`);
|
|
1460
|
+
return this.decodeEffect(row);
|
|
1461
|
+
}
|
|
1462
|
+
decodeEffect(candidate) {
|
|
1463
|
+
const row = workflowEffectSelectSchema.parse(candidate);
|
|
1464
|
+
const owner = ownerFromColumns(row);
|
|
1465
|
+
const externalProcess = externalFromColumns(row);
|
|
1466
|
+
return {
|
|
1467
|
+
id: row.id,
|
|
1468
|
+
runId: row.runId,
|
|
1469
|
+
actionId: row.actionId,
|
|
1470
|
+
...(row.jobId === null ? {} : { jobId: row.jobId }),
|
|
1471
|
+
kind: row.kind,
|
|
1472
|
+
state: row.state,
|
|
1473
|
+
...jsonProperty('target', row.target),
|
|
1474
|
+
...jsonProperty('inputs', row.inputs),
|
|
1475
|
+
...jsonProperty('baseline', row.baseline),
|
|
1476
|
+
...jsonProperty('cursor', row.cursor),
|
|
1477
|
+
...jsonProperty('receipt', row.receipt),
|
|
1478
|
+
...(row.errorCode === null ? {} : { errorCode: row.errorCode }),
|
|
1479
|
+
...(row.errorMessage === null ? {} : { errorMessage: row.errorMessage }),
|
|
1480
|
+
...(owner ? { owner } : {}),
|
|
1481
|
+
...(row.launchNonce === null ? {} : { launchNonce: row.launchNonce }),
|
|
1482
|
+
...(externalProcess ? { externalProcess } : {}),
|
|
1483
|
+
mayExecute: row.mayExecute,
|
|
1484
|
+
attemptCount: row.attemptCount,
|
|
1485
|
+
createdAt: row.createdAt,
|
|
1486
|
+
updatedAt: row.updatedAt,
|
|
1487
|
+
...(row.terminalAt === null ? {} : { terminalAt: row.terminalAt })
|
|
1488
|
+
};
|
|
1489
|
+
}
|
|
1490
|
+
requireEffectAttempt(id) {
|
|
1491
|
+
const row = this.orm.select().from(workflowEffectAttempts).where(eq(workflowEffectAttempts.id, id)).get();
|
|
1492
|
+
if (!row)
|
|
1493
|
+
throw new WorkflowTransitionError(`Workflow effect attempt ${id} does not exist`);
|
|
1494
|
+
return this.decodeEffectAttempt(row);
|
|
1495
|
+
}
|
|
1496
|
+
listWorkflowEffectAttempts(effectId) {
|
|
1497
|
+
return this.orm
|
|
1498
|
+
.select()
|
|
1499
|
+
.from(workflowEffectAttempts)
|
|
1500
|
+
.where(eq(workflowEffectAttempts.effectId, effectId))
|
|
1501
|
+
.orderBy(asc(workflowEffectAttempts.attemptNumber))
|
|
1502
|
+
.all()
|
|
1503
|
+
.map(row => this.decodeEffectAttempt(row));
|
|
1504
|
+
}
|
|
1505
|
+
decodeEffectAttempt(candidate) {
|
|
1506
|
+
const row = workflowEffectAttemptSelectSchema.parse(candidate);
|
|
1507
|
+
const id = row.id;
|
|
1508
|
+
const owner = ownerFromColumns(row);
|
|
1509
|
+
if (!owner)
|
|
1510
|
+
throw new OrchestrationError(`Workflow effect attempt ${id} has no owner`);
|
|
1511
|
+
const externalProcess = externalFromColumns(row);
|
|
1512
|
+
return {
|
|
1513
|
+
id,
|
|
1514
|
+
effectId: row.effectId,
|
|
1515
|
+
attemptNumber: row.attemptNumber,
|
|
1516
|
+
state: row.state,
|
|
1517
|
+
owner,
|
|
1518
|
+
...(row.launchNonce === null ? {} : { launchNonce: row.launchNonce }),
|
|
1519
|
+
...(externalProcess ? { externalProcess } : {}),
|
|
1520
|
+
mayExecute: row.mayExecute,
|
|
1521
|
+
...jsonProperty('receipt', row.receipt),
|
|
1522
|
+
...jsonProperty('evidence', row.evidence),
|
|
1523
|
+
...(row.errorCode === null ? {} : { errorCode: row.errorCode }),
|
|
1524
|
+
...(row.errorMessage === null ? {} : { errorMessage: row.errorMessage }),
|
|
1525
|
+
createdAt: row.createdAt,
|
|
1526
|
+
updatedAt: row.updatedAt,
|
|
1527
|
+
...(row.terminalAt === null ? {} : { terminalAt: row.terminalAt })
|
|
1528
|
+
};
|
|
1529
|
+
}
|
|
1530
|
+
requireEffectOwner(effect, owner, attemptNumber) {
|
|
1531
|
+
if (!effect.owner || !sameOwner(effect.owner, owner) || effect.attemptCount !== attemptNumber) {
|
|
1532
|
+
throw new WorkflowTransitionError(`effect ${effect.actionId} is not owned by this relay attempt`);
|
|
1533
|
+
}
|
|
1534
|
+
}
|
|
1535
|
+
getWorkflowCapability(tokenHash) {
|
|
1536
|
+
const candidate = this.orm
|
|
1537
|
+
.select()
|
|
1538
|
+
.from(workflowCapabilities)
|
|
1539
|
+
.where(eq(workflowCapabilities.tokenHash, tokenHash.toLowerCase()))
|
|
1540
|
+
.get();
|
|
1541
|
+
if (!candidate)
|
|
1542
|
+
return undefined;
|
|
1543
|
+
const row = workflowCapabilitySelectSchema.parse(candidate);
|
|
1544
|
+
return {
|
|
1545
|
+
id: row.id,
|
|
1546
|
+
tokenHash: row.tokenHash,
|
|
1547
|
+
runId: row.runId,
|
|
1548
|
+
rootSessionId: row.rootSessionId,
|
|
1549
|
+
cycle: row.cycle,
|
|
1550
|
+
phase: row.phase,
|
|
1551
|
+
revision: row.revision,
|
|
1552
|
+
allowedRoles: row.allowedRoles,
|
|
1553
|
+
...(row.issuedWithRowid === null
|
|
1554
|
+
? {}
|
|
1555
|
+
: {
|
|
1556
|
+
issuedWith: {
|
|
1557
|
+
rowid: row.issuedWithRowid,
|
|
1558
|
+
...(row.issuedWithTurnId === null ? {} : { turnId: row.issuedWithTurnId })
|
|
1559
|
+
}
|
|
1560
|
+
}),
|
|
1561
|
+
...(row.consumedAt === null ? {} : { consumedAt: row.consumedAt }),
|
|
1562
|
+
...(row.revokedAt === null ? {} : { revokedAt: row.revokedAt }),
|
|
1563
|
+
createdAt: row.createdAt
|
|
1564
|
+
};
|
|
1565
|
+
}
|
|
1566
|
+
issueWorkflowCapability(input) {
|
|
1567
|
+
if (!/^[a-f\d]{64}$/i.test(input.tokenHash))
|
|
1568
|
+
throw new OrchestrationError('capability token hash must be SHA-256 hex');
|
|
1569
|
+
return this.immediate(() => {
|
|
1570
|
+
const run = this.requireRun(input.runId);
|
|
1571
|
+
if (run.rootSessionId !== input.rootSessionId)
|
|
1572
|
+
throw new WorkflowTransitionError('capability root does not match Workflow');
|
|
1573
|
+
if (run.phase !== input.phase || run.cycle !== input.cycle || run.revision !== input.revision) {
|
|
1574
|
+
throw new WorkflowTransitionError('capability coordinates do not match Workflow');
|
|
1575
|
+
}
|
|
1576
|
+
const id = input.id ?? randomUUID();
|
|
1577
|
+
const at = this.now();
|
|
1578
|
+
this.db
|
|
1579
|
+
.prepare(`INSERT INTO workflow_capabilities (
|
|
1580
|
+
id, token_hash, run_id, root_session_id, cycle, phase, revision, allowed_roles_json,
|
|
1581
|
+
issued_with_rowid, issued_with_turn_id, created_at
|
|
1582
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
1583
|
+
.run(id, input.tokenHash.toLowerCase(), input.runId, input.rootSessionId, input.cycle, input.phase, input.revision, json([...new Set(input.allowedRoles)].sort()), input.issuedWith?.rowid ?? null, input.issuedWith?.turnId ?? null, at);
|
|
1584
|
+
this.appendEvent(input.runId, input.eventKey, 'workflow_capability_issued', {
|
|
1585
|
+
capabilityId: id,
|
|
1586
|
+
phase: input.phase,
|
|
1587
|
+
cycle: input.cycle,
|
|
1588
|
+
revision: input.revision,
|
|
1589
|
+
allowedRoles: input.allowedRoles,
|
|
1590
|
+
issuedWith: input.issuedWith
|
|
1591
|
+
});
|
|
1592
|
+
return id;
|
|
1593
|
+
});
|
|
1594
|
+
}
|
|
1595
|
+
consumeWorkflowCapability(input) {
|
|
1596
|
+
return this.immediate(() => {
|
|
1597
|
+
const capability = this.getWorkflowCapability(input.tokenHash);
|
|
1598
|
+
if (!capability ||
|
|
1599
|
+
capability.runId !== input.runId ||
|
|
1600
|
+
capability.rootSessionId !== input.rootSessionId ||
|
|
1601
|
+
capability.phase !== input.expectedPhase ||
|
|
1602
|
+
capability.cycle !== input.expectedCycle ||
|
|
1603
|
+
capability.revision !== input.expectedRevision ||
|
|
1604
|
+
capability.consumedAt !== undefined ||
|
|
1605
|
+
capability.revokedAt !== undefined ||
|
|
1606
|
+
!capability.allowedRoles.includes(input.role)) {
|
|
1607
|
+
throw new WorkflowTransitionError('Workflow capability is invalid, stale, consumed, or wrong-role');
|
|
1608
|
+
}
|
|
1609
|
+
const run = this.requireRun(input.runId);
|
|
1610
|
+
if (run.rootSessionId !== input.rootSessionId ||
|
|
1611
|
+
run.phase !== input.expectedPhase ||
|
|
1612
|
+
run.cycle !== input.expectedCycle ||
|
|
1613
|
+
run.revision !== input.expectedRevision) {
|
|
1614
|
+
throw new WorkflowTransitionError('Workflow advanced past this capability');
|
|
1615
|
+
}
|
|
1616
|
+
const at = this.now();
|
|
1617
|
+
this.db.prepare('UPDATE workflow_capabilities SET consumed_at = ? WHERE id = ?').run(at, capability.id);
|
|
1618
|
+
this.appendEvent(input.runId, input.eventKey, 'workflow_capability_consumed', {
|
|
1619
|
+
capabilityId: capability.id,
|
|
1620
|
+
role: input.role
|
|
1621
|
+
});
|
|
1622
|
+
return capability.id;
|
|
1623
|
+
});
|
|
1624
|
+
}
|
|
1625
|
+
revokeWorkflowCapabilities(runId, eventKey, phase) {
|
|
1626
|
+
return this.immediate(() => {
|
|
1627
|
+
const at = this.now();
|
|
1628
|
+
const result = this.db
|
|
1629
|
+
.prepare(`UPDATE workflow_capabilities SET revoked_at = ?
|
|
1630
|
+
WHERE run_id = ? AND consumed_at IS NULL AND revoked_at IS NULL AND (? IS NULL OR phase = ?)`)
|
|
1631
|
+
.run(at, runId, phase ?? null, phase ?? null);
|
|
1632
|
+
const count = Number(result.changes);
|
|
1633
|
+
this.appendEvent(runId, eventKey, 'workflow_capabilities_revoked', { phase, count });
|
|
1634
|
+
return count;
|
|
1635
|
+
});
|
|
1636
|
+
}
|
|
1637
|
+
listWorkflowEvents(runId) {
|
|
1638
|
+
return this.orm
|
|
1639
|
+
.select()
|
|
1640
|
+
.from(workflowEvents)
|
|
1641
|
+
.where(eq(workflowEvents.runId, runId))
|
|
1642
|
+
.orderBy(asc(workflowEvents.id))
|
|
1643
|
+
.all()
|
|
1644
|
+
.map(row => this.decodeEvent(row));
|
|
1645
|
+
}
|
|
1646
|
+
findEvent(runId, eventKey) {
|
|
1647
|
+
const row = this.orm
|
|
1648
|
+
.select()
|
|
1649
|
+
.from(workflowEvents)
|
|
1650
|
+
.where(and(eq(workflowEvents.runId, runId), eq(workflowEvents.eventKey, eventKey)))
|
|
1651
|
+
.get();
|
|
1652
|
+
return row ? this.decodeEvent(row) : undefined;
|
|
1653
|
+
}
|
|
1654
|
+
decodeEvent(candidate) {
|
|
1655
|
+
const row = workflowEventSelectSchema.parse(candidate);
|
|
1656
|
+
return {
|
|
1657
|
+
id: row.id,
|
|
1658
|
+
runId: row.runId,
|
|
1659
|
+
eventKey: row.eventKey,
|
|
1660
|
+
type: row.type,
|
|
1661
|
+
...jsonProperty('data', row.data),
|
|
1662
|
+
createdAt: row.createdAt
|
|
1663
|
+
};
|
|
1664
|
+
}
|
|
1665
|
+
appendEvent(runId, eventKey, type, data) {
|
|
1666
|
+
this.db
|
|
1667
|
+
.prepare('INSERT INTO workflow_events(run_id, event_key, type, data_json, created_at) VALUES (?, ?, ?, ?, ?)')
|
|
1668
|
+
.run(runId, nonEmpty(eventKey, 'event key'), nonEmpty(type, 'event type'), optionalJson(data), this.now());
|
|
1669
|
+
}
|
|
1670
|
+
touchRun(runId, at) {
|
|
1671
|
+
this.db.prepare('UPDATE workflow_runs SET updated_at = ? WHERE id = ?').run(at, runId);
|
|
1672
|
+
}
|
|
1673
|
+
registerRelayInstance(input) {
|
|
1674
|
+
validateRelayIdentity(input);
|
|
1675
|
+
return this.immediate(() => {
|
|
1676
|
+
const at = this.now();
|
|
1677
|
+
this.db
|
|
1678
|
+
.prepare(`INSERT INTO relay_instances (
|
|
1679
|
+
instance_id, pid, process_started_at, protocol_version, can_drive_ui,
|
|
1680
|
+
heartbeat_at, registered_at, metadata_json
|
|
1681
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
1682
|
+
ON CONFLICT(instance_id) DO UPDATE SET
|
|
1683
|
+
pid = excluded.pid,
|
|
1684
|
+
process_started_at = excluded.process_started_at,
|
|
1685
|
+
protocol_version = excluded.protocol_version,
|
|
1686
|
+
can_drive_ui = excluded.can_drive_ui,
|
|
1687
|
+
heartbeat_at = excluded.heartbeat_at,
|
|
1688
|
+
registered_at = CASE
|
|
1689
|
+
WHEN relay_instances.pid = excluded.pid
|
|
1690
|
+
AND relay_instances.process_started_at = excluded.process_started_at
|
|
1691
|
+
THEN relay_instances.registered_at ELSE excluded.registered_at END,
|
|
1692
|
+
metadata_json = excluded.metadata_json`)
|
|
1693
|
+
.run(input.instanceId, input.pid, input.processStartedAt, input.protocolVersion, input.canDriveUi === false ? 0 : 1, at, at, optionalJson(input.metadata));
|
|
1694
|
+
return this.requireRelayInstance(input.instanceId);
|
|
1695
|
+
});
|
|
1696
|
+
}
|
|
1697
|
+
heartbeatRelayInstance(identity) {
|
|
1698
|
+
return this.immediate(() => {
|
|
1699
|
+
const result = this.db
|
|
1700
|
+
.prepare(`UPDATE relay_instances SET heartbeat_at = ?
|
|
1701
|
+
WHERE instance_id = ? AND pid = ? AND process_started_at = ? AND protocol_version = ?`)
|
|
1702
|
+
.run(this.now(), identity.instanceId, identity.pid, identity.processStartedAt, identity.protocolVersion);
|
|
1703
|
+
return Number(result.changes) === 1;
|
|
1704
|
+
});
|
|
1705
|
+
}
|
|
1706
|
+
listRelayInstances() {
|
|
1707
|
+
return this.orm
|
|
1708
|
+
.select()
|
|
1709
|
+
.from(relayInstances)
|
|
1710
|
+
.orderBy(asc(relayInstances.registeredAt), asc(relayInstances.instanceId))
|
|
1711
|
+
.all()
|
|
1712
|
+
.map(candidate => {
|
|
1713
|
+
const row = relayInstanceSelectSchema.parse(candidate);
|
|
1714
|
+
return {
|
|
1715
|
+
instanceId: row.instanceId,
|
|
1716
|
+
pid: row.pid,
|
|
1717
|
+
processStartedAt: row.processStartedAt,
|
|
1718
|
+
protocolVersion: row.protocolVersion,
|
|
1719
|
+
canDriveUi: row.canDriveUi,
|
|
1720
|
+
heartbeatAt: row.heartbeatAt,
|
|
1721
|
+
registeredAt: row.registeredAt,
|
|
1722
|
+
...jsonProperty('metadata', row.metadata)
|
|
1723
|
+
};
|
|
1724
|
+
});
|
|
1725
|
+
}
|
|
1726
|
+
findIncompatibleRelayInstances(current, probe = this.processProbe) {
|
|
1727
|
+
return this.listRelayInstances().filter(instance => {
|
|
1728
|
+
if (!instance.canDriveUi || sameOwner(instance, current) || instance.protocolVersion === current.protocolVersion)
|
|
1729
|
+
return false;
|
|
1730
|
+
return this.probeAlive(instance, probe);
|
|
1731
|
+
});
|
|
1732
|
+
}
|
|
1733
|
+
requireRelayInstance(instanceId) {
|
|
1734
|
+
const instance = this.listRelayInstances().find(entry => entry.instanceId === instanceId);
|
|
1735
|
+
if (!instance)
|
|
1736
|
+
throw new OrchestrationError(`relay instance ${instanceId} does not exist`);
|
|
1737
|
+
return instance;
|
|
1738
|
+
}
|
|
1739
|
+
acquireUiLease(input) {
|
|
1740
|
+
validateRelayIdentity(input.owner);
|
|
1741
|
+
const probe = input.processProbe ?? this.processProbe;
|
|
1742
|
+
const observed = this.readUiLeaseOwner();
|
|
1743
|
+
const ownerAlive = observed ? this.probeAlive(observed, probe) : false;
|
|
1744
|
+
const externalAlive = observed?.externalProcess ? this.probeAlive(observed.externalProcess, probe) : false;
|
|
1745
|
+
return this.immediate(() => {
|
|
1746
|
+
this.assertRelayRegistered(input.owner);
|
|
1747
|
+
const quarantine = this.getUiQuarantine();
|
|
1748
|
+
if (quarantine.active && input.priority === 'background')
|
|
1749
|
+
return { status: 'quarantined', quarantine };
|
|
1750
|
+
const current = this.readUiLeaseOwner();
|
|
1751
|
+
if (observed && (!current || !sameLeaseSnapshot(observed, current))) {
|
|
1752
|
+
return current ? { status: 'busy', owner: current, reason: 'changed' } : this.assignUiLease(input);
|
|
1753
|
+
}
|
|
1754
|
+
if (!observed && current)
|
|
1755
|
+
return { status: 'busy', owner: current, reason: 'changed' };
|
|
1756
|
+
if (current) {
|
|
1757
|
+
if (ownerAlive)
|
|
1758
|
+
return { status: 'busy', owner: current, reason: 'owner_alive' };
|
|
1759
|
+
if (current.externalProcess && externalAlive) {
|
|
1760
|
+
return { status: 'busy', owner: current, reason: 'external_process_alive' };
|
|
1761
|
+
}
|
|
1762
|
+
if (current.mayExecute) {
|
|
1763
|
+
this.clearUiMutex(current);
|
|
1764
|
+
const hold = this.activateUiQuarantine({
|
|
1765
|
+
actionId: current.actionId,
|
|
1766
|
+
effectId: current.effectId,
|
|
1767
|
+
reason: 'a dead UI lease owner may have emitted an external action',
|
|
1768
|
+
owner: current,
|
|
1769
|
+
externalProcess: current.externalProcess
|
|
1770
|
+
});
|
|
1771
|
+
if (input.priority === 'background')
|
|
1772
|
+
return { status: 'quarantined', quarantine: hold };
|
|
1773
|
+
return this.assignUiLease(input, current);
|
|
1774
|
+
}
|
|
1775
|
+
this.clearUiMutex(current);
|
|
1776
|
+
return this.assignUiLease(input, current);
|
|
1777
|
+
}
|
|
1778
|
+
return this.assignUiLease(input);
|
|
1779
|
+
});
|
|
1780
|
+
}
|
|
1781
|
+
assertRelayRegistered(owner) {
|
|
1782
|
+
const row = this.db
|
|
1783
|
+
.prepare(`SELECT 1 present FROM relay_instances
|
|
1784
|
+
WHERE instance_id = ? AND pid = ? AND process_started_at = ? AND protocol_version = ? AND can_drive_ui = 1`)
|
|
1785
|
+
.get(owner.instanceId, owner.pid, owner.processStartedAt, owner.protocolVersion);
|
|
1786
|
+
if (!row)
|
|
1787
|
+
throw new OrchestrationError(`relay ${owner.instanceId} is not registered as this live UI-driving process`);
|
|
1788
|
+
}
|
|
1789
|
+
assignUiLease(input, reclaimed) {
|
|
1790
|
+
const nonce = input.nonce ?? randomUUID();
|
|
1791
|
+
const at = this.now();
|
|
1792
|
+
const result = this.db
|
|
1793
|
+
.prepare(`UPDATE ui_mutex SET owner_instance_id = ?, owner_pid = ?, owner_process_started_at = ?,
|
|
1794
|
+
owner_protocol_version = ?, nonce = ?, action_id = ?, effect_id = ?, external_pid = NULL,
|
|
1795
|
+
external_process_started_at = NULL, external_process_group = NULL, may_execute = 0,
|
|
1796
|
+
deadline_at = ?, acquired_at = ?, updated_at = ?
|
|
1797
|
+
WHERE id = 1 AND owner_instance_id IS NULL`)
|
|
1798
|
+
.run(input.owner.instanceId, input.owner.pid, input.owner.processStartedAt, input.owner.protocolVersion, nonce, nonEmpty(input.actionId, 'UI action ID'), input.effectId ?? null, input.deadlineAt, at, at);
|
|
1799
|
+
if (Number(result.changes) !== 1) {
|
|
1800
|
+
const owner = this.readUiLeaseOwner();
|
|
1801
|
+
if (!owner)
|
|
1802
|
+
throw new OrchestrationError('UI mutex changed while being acquired');
|
|
1803
|
+
return { status: 'busy', owner, reason: 'changed' };
|
|
1804
|
+
}
|
|
1805
|
+
return {
|
|
1806
|
+
status: 'acquired',
|
|
1807
|
+
lease: {
|
|
1808
|
+
instanceId: input.owner.instanceId,
|
|
1809
|
+
pid: input.owner.pid,
|
|
1810
|
+
processStartedAt: input.owner.processStartedAt,
|
|
1811
|
+
nonce,
|
|
1812
|
+
actionId: input.actionId,
|
|
1813
|
+
...(input.effectId ? { effectId: input.effectId } : {})
|
|
1814
|
+
},
|
|
1815
|
+
...(reclaimed ? { reclaimed } : {})
|
|
1816
|
+
};
|
|
1817
|
+
}
|
|
1818
|
+
markUiLeaseMayExecute(lease, externalProcess, deadlineAt) {
|
|
1819
|
+
if (externalProcess)
|
|
1820
|
+
validateProcessIdentity(externalProcess, 'external UI process');
|
|
1821
|
+
return this.immediate(() => {
|
|
1822
|
+
const result = this.db
|
|
1823
|
+
.prepare(`UPDATE ui_mutex SET may_execute = 1, external_pid = COALESCE(?, external_pid),
|
|
1824
|
+
external_process_started_at = COALESCE(?, external_process_started_at),
|
|
1825
|
+
external_process_group = CASE WHEN ? IS NULL THEN external_process_group ELSE ? END,
|
|
1826
|
+
deadline_at = COALESCE(?, deadline_at), updated_at = ?
|
|
1827
|
+
WHERE id = 1 AND owner_instance_id = ? AND owner_pid = ? AND owner_process_started_at = ?
|
|
1828
|
+
AND nonce = ? AND action_id = ?`)
|
|
1829
|
+
.run(externalProcess?.pid ?? null, externalProcess?.processStartedAt ?? null, externalProcess?.pid ?? null, externalProcess?.processGroup ?? null, deadlineAt ?? null, this.now(), lease.instanceId, lease.pid, lease.processStartedAt, lease.nonce, lease.actionId);
|
|
1830
|
+
return Number(result.changes) === 1;
|
|
1831
|
+
});
|
|
1832
|
+
}
|
|
1833
|
+
renewUiLease(lease, deadlineAt) {
|
|
1834
|
+
return this.immediate(() => {
|
|
1835
|
+
const result = this.db
|
|
1836
|
+
.prepare(`UPDATE ui_mutex SET deadline_at = ?, updated_at = ?
|
|
1837
|
+
WHERE id = 1 AND owner_instance_id = ? AND owner_pid = ? AND owner_process_started_at = ?
|
|
1838
|
+
AND nonce = ? AND action_id = ?`)
|
|
1839
|
+
.run(deadlineAt, this.now(), lease.instanceId, lease.pid, lease.processStartedAt, lease.nonce, lease.actionId);
|
|
1840
|
+
return Number(result.changes) === 1;
|
|
1841
|
+
});
|
|
1842
|
+
}
|
|
1843
|
+
releaseUiLease(lease) {
|
|
1844
|
+
const observed = this.readUiLeaseOwner();
|
|
1845
|
+
if (observed &&
|
|
1846
|
+
observed.instanceId === lease.instanceId &&
|
|
1847
|
+
observed.pid === lease.pid &&
|
|
1848
|
+
observed.processStartedAt === lease.processStartedAt &&
|
|
1849
|
+
observed.nonce === lease.nonce &&
|
|
1850
|
+
observed.actionId === lease.actionId &&
|
|
1851
|
+
observed.externalProcess &&
|
|
1852
|
+
this.probeAlive(observed.externalProcess, this.processProbe)) {
|
|
1853
|
+
// A returned/failed caller is not proof that a detached helper (or one of
|
|
1854
|
+
// its process-group children) is gone. Retain the durable owner so the
|
|
1855
|
+
// watchdog can terminate and prove the exact group dead before reclaim.
|
|
1856
|
+
return false;
|
|
1857
|
+
}
|
|
1858
|
+
return this.immediate(() => {
|
|
1859
|
+
const result = this.db
|
|
1860
|
+
.prepare(`UPDATE ui_mutex SET owner_instance_id = NULL, owner_pid = NULL, owner_process_started_at = NULL,
|
|
1861
|
+
owner_protocol_version = NULL, nonce = NULL, action_id = NULL, effect_id = NULL,
|
|
1862
|
+
external_pid = NULL, external_process_started_at = NULL, external_process_group = NULL,
|
|
1863
|
+
may_execute = 0, deadline_at = NULL, acquired_at = NULL, updated_at = ?
|
|
1864
|
+
WHERE id = 1 AND owner_instance_id = ? AND owner_pid = ? AND owner_process_started_at = ?
|
|
1865
|
+
AND nonce = ? AND action_id = ?`)
|
|
1866
|
+
.run(this.now(), lease.instanceId, lease.pid, lease.processStartedAt, lease.nonce, lease.actionId);
|
|
1867
|
+
return Number(result.changes) === 1;
|
|
1868
|
+
});
|
|
1869
|
+
}
|
|
1870
|
+
getUiLeaseOwner() {
|
|
1871
|
+
return this.readUiLeaseOwner();
|
|
1872
|
+
}
|
|
1873
|
+
/** Adapter installed with `configureSharedUiLeaseProvider` in `writes.ts`. */
|
|
1874
|
+
createSharedUiLeaseProvider(owner, options = {}) {
|
|
1875
|
+
const leaseMs = Math.max(1_000, options.leaseMs ?? 60_000);
|
|
1876
|
+
return {
|
|
1877
|
+
acquire: async (request) => {
|
|
1878
|
+
const result = this.acquireUiLease({
|
|
1879
|
+
owner,
|
|
1880
|
+
actionId: request.actionId ?? `ui:${owner.instanceId}:${randomUUID()}`,
|
|
1881
|
+
deadlineAt: this.now() + leaseMs,
|
|
1882
|
+
priority: request.priority
|
|
1883
|
+
});
|
|
1884
|
+
if (result.status !== 'acquired')
|
|
1885
|
+
throw new UiLeaseUnavailableError(result);
|
|
1886
|
+
return {
|
|
1887
|
+
markMayExecute: externalProcess => {
|
|
1888
|
+
if (!this.markUiLeaseMayExecute(result.lease, externalProcess)) {
|
|
1889
|
+
throw new OrchestrationError('lost the shared UI lease before dispatch');
|
|
1890
|
+
}
|
|
1891
|
+
},
|
|
1892
|
+
release: () => {
|
|
1893
|
+
if (!this.releaseUiLease(result.lease)) {
|
|
1894
|
+
throw new OrchestrationError('shared UI lease release did not match its durable owner or its external process is still live');
|
|
1895
|
+
}
|
|
1896
|
+
}
|
|
1897
|
+
};
|
|
1898
|
+
}
|
|
1899
|
+
};
|
|
1900
|
+
}
|
|
1901
|
+
readUiLeaseOwner() {
|
|
1902
|
+
const candidate = this.orm.select().from(uiMutex).where(eq(uiMutex.id, 1)).get();
|
|
1903
|
+
if (!candidate)
|
|
1904
|
+
throw new OrchestrationError('orchestration UI mutex singleton is missing');
|
|
1905
|
+
const row = uiMutexSelectSchema.parse(candidate);
|
|
1906
|
+
const owner = ownerFromColumns(row);
|
|
1907
|
+
if (!owner ||
|
|
1908
|
+
row.nonce === null ||
|
|
1909
|
+
row.actionId === null ||
|
|
1910
|
+
row.deadlineAt === null ||
|
|
1911
|
+
row.acquiredAt === null ||
|
|
1912
|
+
row.updatedAt === null) {
|
|
1913
|
+
return undefined;
|
|
1914
|
+
}
|
|
1915
|
+
const externalProcess = externalFromColumns(row);
|
|
1916
|
+
return {
|
|
1917
|
+
...owner,
|
|
1918
|
+
nonce: row.nonce,
|
|
1919
|
+
actionId: row.actionId,
|
|
1920
|
+
...(row.effectId === null ? {} : { effectId: row.effectId }),
|
|
1921
|
+
...(externalProcess ? { externalProcess } : {}),
|
|
1922
|
+
mayExecute: row.mayExecute,
|
|
1923
|
+
deadlineAt: row.deadlineAt,
|
|
1924
|
+
acquiredAt: row.acquiredAt,
|
|
1925
|
+
updatedAt: row.updatedAt
|
|
1926
|
+
};
|
|
1927
|
+
}
|
|
1928
|
+
clearUiMutex(expected) {
|
|
1929
|
+
this.db
|
|
1930
|
+
.prepare(`UPDATE ui_mutex SET owner_instance_id = NULL, owner_pid = NULL, owner_process_started_at = NULL,
|
|
1931
|
+
owner_protocol_version = NULL, nonce = NULL, action_id = NULL, effect_id = NULL,
|
|
1932
|
+
external_pid = NULL, external_process_started_at = NULL, external_process_group = NULL,
|
|
1933
|
+
may_execute = 0, deadline_at = NULL, acquired_at = NULL, updated_at = ?
|
|
1934
|
+
WHERE id = 1 AND owner_instance_id = ? AND owner_pid = ? AND owner_process_started_at = ? AND nonce = ?`)
|
|
1935
|
+
.run(this.now(), expected.instanceId, expected.pid, expected.processStartedAt, expected.nonce);
|
|
1936
|
+
}
|
|
1937
|
+
activateUiQuarantine(input) {
|
|
1938
|
+
return this.immediate(() => {
|
|
1939
|
+
const current = this.getUiQuarantine();
|
|
1940
|
+
if (current.active)
|
|
1941
|
+
return current;
|
|
1942
|
+
const at = this.now();
|
|
1943
|
+
this.db
|
|
1944
|
+
.prepare(`UPDATE ui_quarantine SET active = 1, action_id = ?, effect_id = ?, reason = ?,
|
|
1945
|
+
owner_instance_id = ?, owner_pid = ?, owner_process_started_at = ?, owner_protocol_version = ?,
|
|
1946
|
+
external_pid = ?, external_process_started_at = ?, external_process_group = ?,
|
|
1947
|
+
created_at = ?, cleared_at = NULL, cleared_by = NULL WHERE id = 1`)
|
|
1948
|
+
.run(input.actionId, input.effectId ?? null, nonEmpty(input.reason, 'quarantine reason'), input.owner?.instanceId ?? null, input.owner?.pid ?? null, input.owner?.processStartedAt ?? null, input.owner?.protocolVersion ?? null, input.externalProcess?.pid ?? null, input.externalProcess?.processStartedAt ?? null, input.externalProcess?.processGroup ?? null, at);
|
|
1949
|
+
return this.getUiQuarantine();
|
|
1950
|
+
});
|
|
1951
|
+
}
|
|
1952
|
+
clearUiQuarantine(clearedBy) {
|
|
1953
|
+
return this.immediate(() => {
|
|
1954
|
+
const result = this.db
|
|
1955
|
+
.prepare('UPDATE ui_quarantine SET active = 0, cleared_at = ?, cleared_by = ? WHERE id = 1 AND active = 1')
|
|
1956
|
+
.run(this.now(), nonEmpty(clearedBy, 'quarantine clearer'));
|
|
1957
|
+
return Number(result.changes) === 1;
|
|
1958
|
+
});
|
|
1959
|
+
}
|
|
1960
|
+
getUiQuarantine() {
|
|
1961
|
+
const candidate = this.orm.select().from(uiQuarantine).where(eq(uiQuarantine.id, 1)).get();
|
|
1962
|
+
if (!candidate)
|
|
1963
|
+
throw new OrchestrationError('orchestration UI quarantine singleton is missing');
|
|
1964
|
+
const row = uiQuarantineSelectSchema.parse(candidate);
|
|
1965
|
+
const owner = ownerFromColumns(row);
|
|
1966
|
+
const externalProcess = externalFromColumns(row);
|
|
1967
|
+
return {
|
|
1968
|
+
active: row.active,
|
|
1969
|
+
...(row.actionId === null ? {} : { actionId: row.actionId }),
|
|
1970
|
+
...(row.effectId === null ? {} : { effectId: row.effectId }),
|
|
1971
|
+
...(row.reason === null ? {} : { reason: row.reason }),
|
|
1972
|
+
...(owner ? { owner } : {}),
|
|
1973
|
+
...(externalProcess ? { externalProcess } : {}),
|
|
1974
|
+
...(row.createdAt === null ? {} : { createdAt: row.createdAt }),
|
|
1975
|
+
...(row.clearedAt === null ? {} : { clearedAt: row.clearedAt }),
|
|
1976
|
+
...(row.clearedBy === null ? {} : { clearedBy: row.clearedBy })
|
|
1977
|
+
};
|
|
1978
|
+
}
|
|
1979
|
+
probeAlive(identity, probe) {
|
|
1980
|
+
try {
|
|
1981
|
+
return probe(identity);
|
|
1982
|
+
}
|
|
1983
|
+
catch {
|
|
1984
|
+
// Failure to prove death must never authorize overlap.
|
|
1985
|
+
return true;
|
|
1986
|
+
}
|
|
1987
|
+
}
|
|
1988
|
+
getWorkflowProjection(runId) {
|
|
1989
|
+
const run = this.getWorkflowRun(runId);
|
|
1990
|
+
return run ? this.projectRun(run) : undefined;
|
|
1991
|
+
}
|
|
1992
|
+
listWorkflowProjections(options = {}) {
|
|
1993
|
+
return this.orm
|
|
1994
|
+
.select()
|
|
1995
|
+
.from(workflowRuns)
|
|
1996
|
+
.where(options.includeTerminal ? undefined : notInArray(workflowRuns.phase, ['completed', 'cancelled']))
|
|
1997
|
+
.orderBy(desc(workflowRuns.updatedAt))
|
|
1998
|
+
.all()
|
|
1999
|
+
.map(row => this.projectRun(this.decodeRun(row)));
|
|
2000
|
+
}
|
|
2001
|
+
/**
|
|
2002
|
+
* Active runs always need driving. A cancelled run remains wakeable only while
|
|
2003
|
+
* it has durable evidence that can still settle: an external effect whose
|
|
2004
|
+
* outcome is unknown, an accepted outbox receipt awaiting promotion, or a
|
|
2005
|
+
* delivered child turn whose final outcome has not appeared yet.
|
|
2006
|
+
*/
|
|
2007
|
+
listWorkflowRunIdsNeedingWake() {
|
|
2008
|
+
return this.db
|
|
2009
|
+
.prepare(`SELECT r.id FROM workflow_runs r
|
|
2010
|
+
WHERE r.phase NOT IN ('completed', 'cancelled')
|
|
2011
|
+
OR (
|
|
2012
|
+
r.phase = 'cancelled' AND (
|
|
2013
|
+
EXISTS (
|
|
2014
|
+
SELECT 1 FROM workflow_effects e
|
|
2015
|
+
WHERE e.run_id = r.id AND (
|
|
2016
|
+
e.state IN ('dispatched', 'ambiguous') OR
|
|
2017
|
+
(e.state = 'committed' AND json_extract(e.receipt_json, '$.kind') = 'outbox')
|
|
2018
|
+
)
|
|
2019
|
+
) OR EXISTS (
|
|
2020
|
+
SELECT 1 FROM workflow_jobs j
|
|
2021
|
+
WHERE j.run_id = r.id AND j.state = 'cancelled'
|
|
2022
|
+
AND j.child_session_id IS NOT NULL AND j.outcome_json IS NULL
|
|
2023
|
+
AND (
|
|
2024
|
+
json_extract(j.task_receipt_json, '$.kind') = 'message' OR EXISTS (
|
|
2025
|
+
SELECT 1 FROM workflow_effects task
|
|
2026
|
+
WHERE task.job_id = j.id AND task.kind = 'send_task'
|
|
2027
|
+
AND task.state = 'committed'
|
|
2028
|
+
AND json_extract(task.receipt_json, '$.kind') = 'message'
|
|
2029
|
+
)
|
|
2030
|
+
)
|
|
2031
|
+
)
|
|
2032
|
+
)
|
|
2033
|
+
)
|
|
2034
|
+
ORDER BY r.updated_at DESC, r.id`)
|
|
2035
|
+
.all().map(row => row.id);
|
|
2036
|
+
}
|
|
2037
|
+
projectRun(run) {
|
|
2038
|
+
const jobs = this.listWorkflowJobs(run.id);
|
|
2039
|
+
const summary = (role) => {
|
|
2040
|
+
const selected = jobs.filter(job => job.role === role && job.state !== 'cancelled');
|
|
2041
|
+
return {
|
|
2042
|
+
requested: selected.length,
|
|
2043
|
+
running: selected.filter(job => !isTerminalWorkflowJobState(job.state) && job.state !== 'dormant').length,
|
|
2044
|
+
returned: selected.filter(job => job.state === 'returned').length,
|
|
2045
|
+
failed: selected.filter(job => job.state === 'failed').length
|
|
2046
|
+
};
|
|
2047
|
+
};
|
|
2048
|
+
const publicRole = (role) => ({
|
|
2049
|
+
agentType: role.agentType,
|
|
2050
|
+
model: role.model,
|
|
2051
|
+
...(role.effort === undefined ? {} : { effort: role.effort }),
|
|
2052
|
+
...(role.fast === undefined ? {} : { fast: role.fast })
|
|
2053
|
+
});
|
|
2054
|
+
const blocked = run.blocked;
|
|
2055
|
+
const candidates = (blocked?.candidates ?? []).slice(0, 20).map(candidate => ({
|
|
2056
|
+
id: candidate.id,
|
|
2057
|
+
title: this.scrubPublicText(candidate.title).slice(0, 200),
|
|
2058
|
+
repo: this.scrubPublicText(candidate.repo).slice(0, 200),
|
|
2059
|
+
createdAt: candidate.createdAt
|
|
2060
|
+
}));
|
|
2061
|
+
const blockedEffect = blocked ? this.getWorkflowEffect(run.id, blocked.actionId) : undefined;
|
|
2062
|
+
const adoptionKind = blocked?.candidates?.[0]?.kind ?? (blockedEffect?.kind === 'create_workspace' ? 'workspace' : 'session');
|
|
2063
|
+
// Risky replay still needs the stable action id when reconciliation found no
|
|
2064
|
+
// candidate. The fixed wire deliberately has no second public effect-id field,
|
|
2065
|
+
// so retain the recovery envelope with an empty candidates array.
|
|
2066
|
+
const exposeRecoveryAction = !!blocked &&
|
|
2067
|
+
(candidates.length > 0 || blocked.retryClass === 'ambiguous') &&
|
|
2068
|
+
(adoptionKind === 'workspace' || adoptionKind === 'session');
|
|
2069
|
+
const outstanding = jobs.some(job => !isTerminalWorkflowJobState(job.state));
|
|
2070
|
+
const implementationReturned = jobs.some(job => job.role === 'implementation' && job.state === 'returned');
|
|
2071
|
+
return {
|
|
2072
|
+
id: run.id,
|
|
2073
|
+
...(run.workspaceId ? { workspaceId: run.workspaceId } : {}),
|
|
2074
|
+
...(run.rootSessionId ? { rootSessionId: run.rootSessionId } : {}),
|
|
2075
|
+
phase: run.phase,
|
|
2076
|
+
objectiveExcerpt: this.scrubPublicText(run.objective).slice(0, 240),
|
|
2077
|
+
roles: {
|
|
2078
|
+
planning: publicRole(run.roles.planning),
|
|
2079
|
+
exploration: publicRole(run.roles.exploration),
|
|
2080
|
+
implementation: publicRole(run.roles.implementation)
|
|
2081
|
+
},
|
|
2082
|
+
jobs: { exploration: summary('exploration'), implementation: summary('implementation') },
|
|
2083
|
+
...(blocked
|
|
2084
|
+
? {
|
|
2085
|
+
error: {
|
|
2086
|
+
code: blocked.errorCode,
|
|
2087
|
+
message: this.scrubPublicText(blocked.message).slice(0, 500),
|
|
2088
|
+
retryable: blocked.retryClass === 'deterministic'
|
|
2089
|
+
}
|
|
2090
|
+
}
|
|
2091
|
+
: {}),
|
|
2092
|
+
...(exposeRecoveryAction ? { adoption: { actionId: blocked.actionId, kind: adoptionKind, candidates } } : {}),
|
|
2093
|
+
actions: {
|
|
2094
|
+
canRetry: run.phase === 'blocked' && run.blocked?.retryClass === 'deterministic',
|
|
2095
|
+
canAdopt: run.phase === 'blocked' && candidates.length > 0,
|
|
2096
|
+
canReplayAmbiguous: run.phase === 'blocked' && run.blocked?.retryClass === 'ambiguous',
|
|
2097
|
+
canCancel: !isTerminalWorkflowPhase(run.phase),
|
|
2098
|
+
canComplete: run.phase === 'reviewing' && implementationReturned && !outstanding
|
|
2099
|
+
},
|
|
2100
|
+
createdAt: run.createdAt,
|
|
2101
|
+
updatedAt: run.updatedAt
|
|
2102
|
+
};
|
|
2103
|
+
}
|
|
2104
|
+
compactTerminalRuns(options) {
|
|
2105
|
+
return this.immediate(() => {
|
|
2106
|
+
const rows = this.db
|
|
2107
|
+
.prepare(`SELECT id FROM workflow_runs
|
|
2108
|
+
WHERE phase IN ('completed', 'cancelled') AND terminal_at IS NOT NULL AND terminal_at < ?
|
|
2109
|
+
AND NOT EXISTS (
|
|
2110
|
+
SELECT 1 FROM workflow_events e
|
|
2111
|
+
WHERE e.run_id = workflow_runs.id AND e.type = 'terminal_compacted'
|
|
2112
|
+
)
|
|
2113
|
+
ORDER BY terminal_at LIMIT ?`)
|
|
2114
|
+
.all(options.olderThan, Math.max(1, Math.min(options.limit ?? 20, 100)));
|
|
2115
|
+
for (const row of rows) {
|
|
2116
|
+
const jobs = this.listWorkflowJobs(row.id);
|
|
2117
|
+
const effects = this.db
|
|
2118
|
+
.prepare('SELECT state, COUNT(*) count FROM workflow_effects WHERE run_id = ? GROUP BY state')
|
|
2119
|
+
.all(row.id);
|
|
2120
|
+
this.db.prepare('DELETE FROM workflow_capabilities WHERE run_id = ?').run(row.id);
|
|
2121
|
+
this.db.prepare('DELETE FROM workflow_jobs WHERE run_id = ?').run(row.id);
|
|
2122
|
+
this.db.prepare('DELETE FROM workflow_effects WHERE run_id = ?').run(row.id);
|
|
2123
|
+
this.db.prepare('DELETE FROM workflow_events WHERE run_id = ?').run(row.id);
|
|
2124
|
+
this.appendEvent(row.id, 'terminal_compacted', 'terminal_compacted', {
|
|
2125
|
+
jobs: Object.fromEntries(['exploration', 'implementation'].map(role => [
|
|
2126
|
+
role,
|
|
2127
|
+
Object.fromEntries(ALL_JOB_STATES.map(state => [
|
|
2128
|
+
state,
|
|
2129
|
+
jobs.filter(job => job.role === role && job.state === state).length
|
|
2130
|
+
]))
|
|
2131
|
+
])),
|
|
2132
|
+
effects: Object.fromEntries(effects.map(effect => [effect.state, effect.count]))
|
|
2133
|
+
});
|
|
2134
|
+
}
|
|
2135
|
+
return rows.length;
|
|
2136
|
+
});
|
|
2137
|
+
}
|
|
2138
|
+
}
|
|
2139
|
+
const asObject = (value) => value !== null && typeof value === 'object' && !Array.isArray(value) ? value : {};
|
|
2140
|
+
const canonicalOptional = (value) => value === undefined ? '__orchestration_undefined__' : canonicalRequestJson(value);
|
|
2141
|
+
const sameOptionalProcess = (left, right) => left === undefined
|
|
2142
|
+
? right === undefined
|
|
2143
|
+
: right !== undefined && sameProcess(left, right) && left.processGroup === right.processGroup;
|
|
2144
|
+
const sameLeaseSnapshot = (left, right) => sameOwner(left, right) &&
|
|
2145
|
+
left.nonce === right.nonce &&
|
|
2146
|
+
left.actionId === right.actionId &&
|
|
2147
|
+
left.effectId === right.effectId &&
|
|
2148
|
+
left.mayExecute === right.mayExecute &&
|
|
2149
|
+
left.deadlineAt === right.deadlineAt &&
|
|
2150
|
+
sameOptionalProcess(left.externalProcess, right.externalProcess);
|