@atolis-hq/wake 0.2.28 → 0.2.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/adapters/claude/claude-runner.js +1 -0
- package/dist/src/adapters/codex/codex-runner.js +1 -0
- package/dist/src/adapters/cursor/cursor-runner.js +1 -0
- package/dist/src/adapters/fs/state-store.js +7 -0
- package/dist/src/adapters/runner/cli-command.js +17 -1
- package/dist/src/core/active-run-recovery.js +13 -8
- package/dist/src/core/run-lease.js +38 -0
- package/dist/src/core/stale-run-reconciler.js +100 -15
- package/dist/src/core/tick-runner.js +81 -14
- package/dist/src/domain/schema.js +46 -3
- package/dist/src/lib/lock.js +5 -5
- package/dist/src/lib/process-identity.js +54 -0
- package/dist/src/version.js +1 -1
- package/package.json +1 -1
|
@@ -218,6 +218,7 @@ export function createClaudeRunner(options) {
|
|
|
218
218
|
args,
|
|
219
219
|
cwd: input.workspacePath ?? options.cwd,
|
|
220
220
|
timeoutMs: options.settings.timeoutMs,
|
|
221
|
+
...(input.onProcessStart === undefined ? {} : { onProcessStart: input.onProcessStart }),
|
|
221
222
|
});
|
|
222
223
|
const responseTranscriptPath = await writeRunnerTranscript({
|
|
223
224
|
config: input.config,
|
|
@@ -261,6 +261,7 @@ export function createCodexRunner(options) {
|
|
|
261
261
|
}),
|
|
262
262
|
cwd,
|
|
263
263
|
timeoutMs: options.settings.timeoutMs,
|
|
264
|
+
...(input.onProcessStart === undefined ? {} : { onProcessStart: input.onProcessStart }),
|
|
264
265
|
});
|
|
265
266
|
const responseTranscriptPath = await writeRunnerTranscript({
|
|
266
267
|
config: input.config,
|
|
@@ -212,6 +212,7 @@ export function createCursorRunner(options) {
|
|
|
212
212
|
}),
|
|
213
213
|
cwd,
|
|
214
214
|
timeoutMs: options.settings.timeoutMs,
|
|
215
|
+
...(input.onProcessStart === undefined ? {} : { onProcessStart: input.onProcessStart }),
|
|
215
216
|
});
|
|
216
217
|
const responseTranscriptPath = await writeRunnerTranscript({
|
|
217
218
|
config: input.config,
|
|
@@ -172,6 +172,13 @@ export function createStateStore({ wakeRoot }) {
|
|
|
172
172
|
await writeJsonFile(paths.runDateFile(parsed.startedAt.slice(0, 10), parsed.runId), parsed);
|
|
173
173
|
return parsed;
|
|
174
174
|
},
|
|
175
|
+
async updateRunRecordIf(runId, input) {
|
|
176
|
+
const current = await this.readRunRecord(runId);
|
|
177
|
+
if (current === null || !input.expect(current)) {
|
|
178
|
+
return null;
|
|
179
|
+
}
|
|
180
|
+
return this.writeRunRecord(input.update(current));
|
|
181
|
+
},
|
|
175
182
|
async writeSourceState(record) {
|
|
176
183
|
const parsed = parseSourceStateRecord(record);
|
|
177
184
|
await writeJsonFile(paths.sourceStateFile(parsed.source, parsed.key), parsed);
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { spawn } from 'node:child_process';
|
|
2
|
+
import { readProcessIdentity } from '../../lib/process-identity.js';
|
|
2
3
|
const TIMEOUT_KILL_GRACE_MS = 5_000;
|
|
3
4
|
export function runAgentCliCommand(input) {
|
|
4
5
|
return new Promise((resolve, reject) => {
|
|
@@ -11,6 +12,16 @@ export function runAgentCliCommand(input) {
|
|
|
11
12
|
let stderr = '';
|
|
12
13
|
let timedOut = false;
|
|
13
14
|
let killTimer;
|
|
15
|
+
let startNotification = Promise.resolve();
|
|
16
|
+
let startNotificationError;
|
|
17
|
+
if (input.onProcessStart !== undefined && child.pid !== undefined) {
|
|
18
|
+
const identity = readProcessIdentity(child.pid);
|
|
19
|
+
if (identity !== null) {
|
|
20
|
+
startNotification = input.onProcessStart(identity).catch((error) => {
|
|
21
|
+
startNotificationError = error;
|
|
22
|
+
});
|
|
23
|
+
}
|
|
24
|
+
}
|
|
14
25
|
const timeoutTimer = input.timeoutMs === undefined
|
|
15
26
|
? undefined
|
|
16
27
|
: setTimeout(() => {
|
|
@@ -29,9 +40,14 @@ export function runAgentCliCommand(input) {
|
|
|
29
40
|
clearTimeout(killTimer);
|
|
30
41
|
reject(error);
|
|
31
42
|
});
|
|
32
|
-
child.on('close', (exitCode) => {
|
|
43
|
+
child.on('close', async (exitCode) => {
|
|
33
44
|
clearTimeout(timeoutTimer);
|
|
34
45
|
clearTimeout(killTimer);
|
|
46
|
+
await startNotification;
|
|
47
|
+
if (startNotificationError !== undefined) {
|
|
48
|
+
reject(startNotificationError);
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
35
51
|
resolve({
|
|
36
52
|
stdout,
|
|
37
53
|
stderr,
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import { readFileLockStatus } from '../lib/lock.js';
|
|
2
1
|
import { maxConfiguredRunnerTimeoutMs } from '../domain/runner-routing.js';
|
|
2
|
+
import { isRunLeaseExpired } from './run-lease.js';
|
|
3
|
+
import { processIdentityMatches } from '../lib/process-identity.js';
|
|
3
4
|
import { createOutbox } from './outbox.js';
|
|
4
5
|
import { createProjectionUpdater } from './projection-updater.js';
|
|
5
6
|
import { createStaleRunReconciler } from './stale-run-reconciler.js';
|
|
@@ -14,14 +15,18 @@ export function createActiveRunRecovery(deps) {
|
|
|
14
15
|
stateStore: deps.stateStore,
|
|
15
16
|
projectionUpdater,
|
|
16
17
|
});
|
|
17
|
-
async function isRunningRecordActive(record) {
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
});
|
|
21
|
-
if (!lock.active || lock.metadata === undefined) {
|
|
22
|
-
return false;
|
|
18
|
+
async function isRunningRecordActive(record, now) {
|
|
19
|
+
if (record.lease !== undefined && !isRunLeaseExpired(record, now)) {
|
|
20
|
+
return true;
|
|
23
21
|
}
|
|
24
|
-
return
|
|
22
|
+
return (processIdentityMatches({
|
|
23
|
+
pid: record.agentPid,
|
|
24
|
+
processStartedAt: record.agentProcessStartedAt,
|
|
25
|
+
}) ||
|
|
26
|
+
processIdentityMatches({
|
|
27
|
+
pid: record.workerPid,
|
|
28
|
+
processStartedAt: record.workerProcessStartedAt,
|
|
29
|
+
}));
|
|
25
30
|
}
|
|
26
31
|
const { reconcileStaleRunningRecords } = createStaleRunReconciler({
|
|
27
32
|
config: deps.config,
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
export const runLeaseDurationMs = 60_000;
|
|
3
|
+
export const runLeaseRenewalIntervalMs = 20_000;
|
|
4
|
+
export function createRunLease(input) {
|
|
5
|
+
const acquiredAt = input.clock.now();
|
|
6
|
+
const expiresAt = new Date(acquiredAt.getTime() + runLeaseDurationMs);
|
|
7
|
+
return {
|
|
8
|
+
leaseId: `lease-${randomUUID()}`,
|
|
9
|
+
ownerInstanceId: input.ownerInstanceId,
|
|
10
|
+
acquiredAt: acquiredAt.toISOString(),
|
|
11
|
+
lastRenewedAt: acquiredAt.toISOString(),
|
|
12
|
+
expiresAt: expiresAt.toISOString(),
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
export function isRunLeaseExpired(record, now) {
|
|
16
|
+
if (record.lease === undefined) {
|
|
17
|
+
return true;
|
|
18
|
+
}
|
|
19
|
+
return Date.parse(record.lease.expiresAt) <= now.getTime();
|
|
20
|
+
}
|
|
21
|
+
export async function renewRunLease(input) {
|
|
22
|
+
const now = input.clock.now();
|
|
23
|
+
const renewed = await input.stateStore.updateRunRecordIf(input.runId, {
|
|
24
|
+
expect: (record) => record.status === 'running' &&
|
|
25
|
+
record.lifecycle !== 'TERMINAL' &&
|
|
26
|
+
record.lease?.leaseId === input.leaseId &&
|
|
27
|
+
record.lease.ownerInstanceId === input.ownerInstanceId,
|
|
28
|
+
update: (record) => ({
|
|
29
|
+
...record,
|
|
30
|
+
lease: {
|
|
31
|
+
...record.lease,
|
|
32
|
+
lastRenewedAt: now.toISOString(),
|
|
33
|
+
expiresAt: new Date(now.getTime() + runLeaseDurationMs).toISOString(),
|
|
34
|
+
},
|
|
35
|
+
}),
|
|
36
|
+
});
|
|
37
|
+
return renewed !== null;
|
|
38
|
+
}
|
|
@@ -8,22 +8,107 @@ import { createEventEnvelope } from '../lib/event-log.js';
|
|
|
8
8
|
// `deliverOutboundEvent` is injected rather than imported so this module does
|
|
9
9
|
// not depend on the outbox module directly.
|
|
10
10
|
export function createStaleRunReconciler(deps) {
|
|
11
|
+
function isTerminalLifecycle(lifecycle) {
|
|
12
|
+
return lifecycle === 'TERMINAL';
|
|
13
|
+
}
|
|
14
|
+
function recoveryOutcomeForLifecycle(lifecycle) {
|
|
15
|
+
switch (lifecycle) {
|
|
16
|
+
case 'CREATED':
|
|
17
|
+
case 'CLAIMED':
|
|
18
|
+
case 'PREPARING':
|
|
19
|
+
return 'CANCELED_BY_RECONCILIATION';
|
|
20
|
+
case 'PROCESS_STARTING':
|
|
21
|
+
case 'RUNNING':
|
|
22
|
+
case 'FINALISING':
|
|
23
|
+
return 'STALLED';
|
|
24
|
+
case 'TERMINAL':
|
|
25
|
+
return 'SUPERSEDED';
|
|
26
|
+
}
|
|
27
|
+
}
|
|
11
28
|
async function staleReason(record, now) {
|
|
12
29
|
if (record.status !== 'running') {
|
|
13
30
|
return null;
|
|
14
31
|
}
|
|
15
|
-
if (
|
|
32
|
+
if (isTerminalLifecycle(record.lifecycle)) {
|
|
33
|
+
return null;
|
|
34
|
+
}
|
|
35
|
+
if (deps.isRunningRecordActive === undefined) {
|
|
36
|
+
if (record.lifecycle !== 'CREATED' &&
|
|
37
|
+
record.lifecycle !== 'CLAIMED' &&
|
|
38
|
+
record.lifecycle !== 'PREPARING') {
|
|
39
|
+
const startedAtMs = Date.parse(record.startedAt);
|
|
40
|
+
if (!Number.isFinite(startedAtMs)) {
|
|
41
|
+
return 'timeout';
|
|
42
|
+
}
|
|
43
|
+
return now.getTime() - startedAtMs >= deps.runnerTimeoutMs() ? 'timeout' : null;
|
|
44
|
+
}
|
|
45
|
+
return 'startup-recovery';
|
|
46
|
+
}
|
|
47
|
+
const active = await deps.isRunningRecordActive(record, now);
|
|
48
|
+
if (active) {
|
|
49
|
+
return null;
|
|
50
|
+
}
|
|
51
|
+
if (record.lifecycle === 'RUNNING' || record.lifecycle === 'PROCESS_STARTING') {
|
|
16
52
|
return 'runner-lock-not-active';
|
|
17
53
|
}
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
54
|
+
return 'startup-recovery';
|
|
55
|
+
}
|
|
56
|
+
async function deliverRecoveredLabels(projection, runId, finishedAt) {
|
|
57
|
+
await deps.deliverOutboundEvent(createLabelsEvent({
|
|
58
|
+
projection,
|
|
59
|
+
runId,
|
|
60
|
+
statusLabel: 'wake:status.failed',
|
|
61
|
+
stageLabel: stageLabelForStage(projection.wake.stage),
|
|
62
|
+
workflowLabel: workflowLabelForWorkflowName(workflowNameForProjection(projection, deps.config)),
|
|
63
|
+
occurredAt: finishedAt,
|
|
64
|
+
}));
|
|
65
|
+
}
|
|
66
|
+
async function recoverMissingRunRecordClaims(runRecords, finishedAt) {
|
|
67
|
+
const knownRunIds = new Set(runRecords.map((record) => record.runId));
|
|
68
|
+
const projections = await deps.stateStore.listIssueStates();
|
|
69
|
+
for (const projection of projections) {
|
|
70
|
+
const runId = projection.wake.lastRunId;
|
|
71
|
+
if (runId === undefined ||
|
|
72
|
+
knownRunIds.has(runId) ||
|
|
73
|
+
runRecords.some((record) => record.workItemKey === projection.workItemKey) ||
|
|
74
|
+
projection.context.lastRunSentinel !== undefined) {
|
|
75
|
+
continue;
|
|
76
|
+
}
|
|
77
|
+
const event = createEventEnvelope({
|
|
78
|
+
eventId: `${runId}-missing-run-record-recovered`,
|
|
79
|
+
workItemKey: projection.workItemKey,
|
|
80
|
+
streamScope: 'work-item',
|
|
81
|
+
direction: 'internal',
|
|
82
|
+
sourceSystem: 'wake',
|
|
83
|
+
sourceEventType: 'wake.run.completed',
|
|
84
|
+
sourceRefs: {
|
|
85
|
+
repo: projection.issue.repo,
|
|
86
|
+
issueNumber: projection.issue.number,
|
|
87
|
+
runId,
|
|
88
|
+
},
|
|
89
|
+
occurredAt: finishedAt,
|
|
90
|
+
ingestedAt: finishedAt,
|
|
91
|
+
trigger: 'immediate',
|
|
92
|
+
payload: {
|
|
93
|
+
action: projection.context.lastRunAction,
|
|
94
|
+
sentinel: 'FAILED',
|
|
95
|
+
executionOutcome: 'CANCELED_BY_RECONCILIATION',
|
|
96
|
+
runId,
|
|
97
|
+
reason: 'runner:missing-run-record',
|
|
98
|
+
},
|
|
99
|
+
});
|
|
100
|
+
await deps.stateStore.appendEventEnvelope(event);
|
|
101
|
+
await deps.projectionUpdater.rebuildFromEvents([event]);
|
|
102
|
+
const updatedProjection = await deps.stateStore.readIssueState(projection.workItemKey);
|
|
103
|
+
if (updatedProjection !== null) {
|
|
104
|
+
await deliverRecoveredLabels(updatedProjection, runId, finishedAt);
|
|
105
|
+
}
|
|
21
106
|
}
|
|
22
|
-
return now.getTime() - startedAtMs >= deps.runnerTimeoutMs() ? 'timeout' : null;
|
|
23
107
|
}
|
|
24
108
|
async function reconcileStaleRunningRecords(now) {
|
|
25
109
|
const finishedAt = now.toISOString();
|
|
26
110
|
const runRecords = await deps.stateStore.listRunRecords();
|
|
111
|
+
await recoverMissingRunRecordClaims(runRecords, finishedAt);
|
|
27
112
|
const staleRecords = [];
|
|
28
113
|
for (const record of runRecords) {
|
|
29
114
|
const reason = await staleReason(record, now);
|
|
@@ -46,6 +131,7 @@ export function createStaleRunReconciler(deps) {
|
|
|
46
131
|
if (projection === null || projection.wake.lastRunId !== record.runId || newerCompletedRun) {
|
|
47
132
|
await deps.stateStore.writeRunRecord({
|
|
48
133
|
...record,
|
|
134
|
+
lifecycle: 'TERMINAL',
|
|
49
135
|
status: 'superseded',
|
|
50
136
|
finishedAt,
|
|
51
137
|
executionOutcome: 'SUPERSEDED',
|
|
@@ -58,9 +144,10 @@ export function createStaleRunReconciler(deps) {
|
|
|
58
144
|
});
|
|
59
145
|
continue;
|
|
60
146
|
}
|
|
61
|
-
const staleExecutionOutcome = reason === 'timeout' ? 'TIMED_OUT' :
|
|
147
|
+
const staleExecutionOutcome = reason === 'timeout' ? 'TIMED_OUT' : recoveryOutcomeForLifecycle(record.lifecycle);
|
|
62
148
|
await deps.stateStore.writeRunRecord({
|
|
63
149
|
...record,
|
|
150
|
+
lifecycle: 'TERMINAL',
|
|
64
151
|
status: 'failed',
|
|
65
152
|
finishedAt,
|
|
66
153
|
sentinel: 'FAILED',
|
|
@@ -69,6 +156,7 @@ export function createStaleRunReconciler(deps) {
|
|
|
69
156
|
metadata: {
|
|
70
157
|
...record.metadata,
|
|
71
158
|
reconciledBy: 'stale-running-record',
|
|
159
|
+
recoveryLifecycle: record.lifecycle,
|
|
72
160
|
staleReason: reason,
|
|
73
161
|
timeoutMs: deps.runnerTimeoutMs(),
|
|
74
162
|
},
|
|
@@ -93,7 +181,11 @@ export function createStaleRunReconciler(deps) {
|
|
|
93
181
|
sentinel: 'FAILED',
|
|
94
182
|
executionOutcome: staleExecutionOutcome,
|
|
95
183
|
runId: record.runId,
|
|
96
|
-
reason: reason === 'timeout'
|
|
184
|
+
reason: reason === 'timeout'
|
|
185
|
+
? 'runner:stale-timeout'
|
|
186
|
+
: reason === 'startup-recovery'
|
|
187
|
+
? `runner:recover-${record.lifecycle.toLowerCase()}`
|
|
188
|
+
: 'runner:orphaned-process',
|
|
97
189
|
...(record.routing === undefined ? {} : { routing: record.routing }),
|
|
98
190
|
},
|
|
99
191
|
});
|
|
@@ -101,14 +193,7 @@ export function createStaleRunReconciler(deps) {
|
|
|
101
193
|
await deps.projectionUpdater.rebuildFromEvents([runCompletedEvent]);
|
|
102
194
|
const updatedProjection = await deps.stateStore.readIssueState(projection.workItemKey);
|
|
103
195
|
if (updatedProjection !== null) {
|
|
104
|
-
await
|
|
105
|
-
projection: updatedProjection,
|
|
106
|
-
runId: record.runId,
|
|
107
|
-
statusLabel: 'wake:status.failed',
|
|
108
|
-
stageLabel: stageLabelForStage(updatedProjection.wake.stage),
|
|
109
|
-
workflowLabel: workflowLabelForWorkflowName(workflowNameForProjection(updatedProjection, deps.config)),
|
|
110
|
-
occurredAt: finishedAt,
|
|
111
|
-
}));
|
|
196
|
+
await deliverRecoveredLabels(updatedProjection, record.runId, finishedAt);
|
|
112
197
|
}
|
|
113
198
|
}
|
|
114
199
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { createLifecycleService } from './lifecycle-service.js';
|
|
2
2
|
import { createPolicyEngine } from './policy-engine.js';
|
|
3
3
|
import { createProjectionUpdater } from './projection-updater.js';
|
|
4
|
-
import { acquireFileLock
|
|
4
|
+
import { acquireFileLock } from '../lib/lock.js';
|
|
5
5
|
import { CORRELATION_REGISTERED_EVENT, parseRunnerArtifacts, parseRunnerResult, } from '../domain/schema.js';
|
|
6
6
|
import { maxConfiguredRunnerTimeoutMs, resolveRunnerRouting } from '../domain/runner-routing.js';
|
|
7
7
|
import { awaitingApprovalRunnerSentinel, stageLabelForStage } from '../domain/stages.js';
|
|
@@ -15,6 +15,8 @@ import { createOutbox } from './outbox.js';
|
|
|
15
15
|
import { createEventResolver } from './event-resolver.js';
|
|
16
16
|
import { createStaleRunReconciler } from './stale-run-reconciler.js';
|
|
17
17
|
import { createWorkspaceCleanup } from './workspace-cleanup.js';
|
|
18
|
+
import { createRunLease, isRunLeaseExpired, renewRunLease, runLeaseRenewalIntervalMs, } from './run-lease.js';
|
|
19
|
+
import { currentProcessIdentity, processIdentityMatches } from '../lib/process-identity.js';
|
|
18
20
|
function latestHumanCommentId(candidate) {
|
|
19
21
|
const human = candidate.comments.filter((c) => !c.isBotAuthored);
|
|
20
22
|
return human.at(-1)?.id;
|
|
@@ -68,6 +70,7 @@ export function createTickRunner(deps) {
|
|
|
68
70
|
workspaceManager: deps.workspaceManager,
|
|
69
71
|
projectionUpdater,
|
|
70
72
|
});
|
|
73
|
+
const ownerInstanceId = `instance-${process.pid}-${Date.now()}`;
|
|
71
74
|
function isAwaitingApproval(projection) {
|
|
72
75
|
return projection.context.lastRunSentinel === awaitingApprovalRunnerSentinel;
|
|
73
76
|
}
|
|
@@ -204,14 +207,18 @@ export function createTickRunner(deps) {
|
|
|
204
207
|
function runnerTimeoutMs() {
|
|
205
208
|
return maxConfiguredRunnerTimeoutMs(deps.config);
|
|
206
209
|
}
|
|
207
|
-
async function isRunningRecordActive(record) {
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
});
|
|
211
|
-
if (!lock.active || lock.metadata === undefined) {
|
|
212
|
-
return false;
|
|
210
|
+
async function isRunningRecordActive(record, now) {
|
|
211
|
+
if (record.lease !== undefined && !isRunLeaseExpired(record, now)) {
|
|
212
|
+
return true;
|
|
213
213
|
}
|
|
214
|
-
return
|
|
214
|
+
return (processIdentityMatches({
|
|
215
|
+
pid: record.agentPid,
|
|
216
|
+
processStartedAt: record.agentProcessStartedAt,
|
|
217
|
+
}) ||
|
|
218
|
+
processIdentityMatches({
|
|
219
|
+
pid: record.workerPid,
|
|
220
|
+
processStartedAt: record.workerProcessStartedAt,
|
|
221
|
+
}));
|
|
215
222
|
}
|
|
216
223
|
async function parkConfigDriftedProjections(projections) {
|
|
217
224
|
let parked = false;
|
|
@@ -289,9 +296,7 @@ export function createTickRunner(deps) {
|
|
|
289
296
|
}
|
|
290
297
|
}
|
|
291
298
|
async function runRunnerTick() {
|
|
292
|
-
const lock = await acquireFileLock(deps.stateStore.paths.runnerLockFile
|
|
293
|
-
staleAfterMs: runnerTimeoutMs(),
|
|
294
|
-
});
|
|
299
|
+
const lock = await acquireFileLock(deps.stateStore.paths.runnerLockFile);
|
|
295
300
|
if (!lock.acquired) {
|
|
296
301
|
return { status: 'locked' };
|
|
297
302
|
}
|
|
@@ -439,6 +444,8 @@ export function createTickRunner(deps) {
|
|
|
439
444
|
return { status: 'idle' };
|
|
440
445
|
}
|
|
441
446
|
const runId = `run-${candidate.issue.number}-${deps.clock.now().getTime()}`;
|
|
447
|
+
const lease = createRunLease({ clock: deps.clock, ownerInstanceId });
|
|
448
|
+
const workerIdentity = currentProcessIdentity();
|
|
442
449
|
const runningRecord = {
|
|
443
450
|
schemaVersion: 1,
|
|
444
451
|
runId,
|
|
@@ -446,10 +453,21 @@ export function createTickRunner(deps) {
|
|
|
446
453
|
repo: candidate.issue.repo,
|
|
447
454
|
issueNumber: candidate.issue.number,
|
|
448
455
|
action,
|
|
456
|
+
lifecycle: 'CREATED',
|
|
449
457
|
status: 'running',
|
|
450
458
|
startedAt: nowIso,
|
|
459
|
+
routing,
|
|
460
|
+
lease,
|
|
461
|
+
workerPid: workerIdentity.pid,
|
|
462
|
+
workerProcessStartedAt: workerIdentity.processStartedAt,
|
|
451
463
|
};
|
|
452
464
|
await deps.stateStore.writeRunRecord(runningRecord);
|
|
465
|
+
async function transitionRunLifecycle(lifecycle) {
|
|
466
|
+
await deps.stateStore.writeRunRecord({
|
|
467
|
+
...(await deps.stateStore.readRunRecord(runId)),
|
|
468
|
+
lifecycle,
|
|
469
|
+
});
|
|
470
|
+
}
|
|
453
471
|
const claimedAt = eventStampNow();
|
|
454
472
|
const claimedEvent = createEventEnvelope({
|
|
455
473
|
eventId: `${runId}-claimed`,
|
|
@@ -473,6 +491,7 @@ export function createTickRunner(deps) {
|
|
|
473
491
|
},
|
|
474
492
|
});
|
|
475
493
|
await deps.stateStore.appendEventEnvelope(claimedEvent);
|
|
494
|
+
await transitionRunLifecycle('CLAIMED');
|
|
476
495
|
await projectionUpdater.rebuildFromEvents([claimedEvent]);
|
|
477
496
|
await deliverOutboundEvent(createLabelsEvent({
|
|
478
497
|
projection: candidate,
|
|
@@ -482,7 +501,20 @@ export function createTickRunner(deps) {
|
|
|
482
501
|
workflowLabel: workflowLabelForWorkflowName(workflowName),
|
|
483
502
|
occurredAt: eventStampNow(),
|
|
484
503
|
}));
|
|
504
|
+
let leaseRenewalTimer;
|
|
505
|
+
function startLeaseRenewal() {
|
|
506
|
+
leaseRenewalTimer = setInterval(() => {
|
|
507
|
+
void renewRunLease({
|
|
508
|
+
stateStore: deps.stateStore,
|
|
509
|
+
runId,
|
|
510
|
+
leaseId: lease.leaseId,
|
|
511
|
+
ownerInstanceId,
|
|
512
|
+
clock: deps.clock,
|
|
513
|
+
});
|
|
514
|
+
}, runLeaseRenewalIntervalMs);
|
|
515
|
+
}
|
|
485
516
|
try {
|
|
517
|
+
await transitionRunLifecycle('PREPARING');
|
|
486
518
|
const prepareResult = workspaceMode === 'branch'
|
|
487
519
|
? await deps.workspaceManager.prepareWorkspace({
|
|
488
520
|
workId: candidate.workItemKey,
|
|
@@ -499,7 +531,19 @@ export function createTickRunner(deps) {
|
|
|
499
531
|
const upstreamChanges = 'upstreamChanges' in prepareResult && typeof prepareResult.upstreamChanges === 'string'
|
|
500
532
|
? prepareResult.upstreamChanges
|
|
501
533
|
: undefined;
|
|
534
|
+
const preparedRecord = (await deps.stateStore.readRunRecord(runId));
|
|
535
|
+
await deps.stateStore.writeRunRecord({
|
|
536
|
+
...preparedRecord,
|
|
537
|
+
lifecycle: 'PROCESS_STARTING',
|
|
538
|
+
metadata: {
|
|
539
|
+
...preparedRecord.metadata,
|
|
540
|
+
...(workspacePath === undefined ? {} : { workspacePath }),
|
|
541
|
+
workspaceMode,
|
|
542
|
+
},
|
|
543
|
+
});
|
|
502
544
|
const recentEvents = await deps.stateStore.listEventEnvelopesForWorkItem(candidate.workItemKey, 6);
|
|
545
|
+
await transitionRunLifecycle('RUNNING');
|
|
546
|
+
startLeaseRenewal();
|
|
503
547
|
const runnerResult = await deps.runner.run({
|
|
504
548
|
action,
|
|
505
549
|
projection: candidate,
|
|
@@ -511,7 +555,21 @@ export function createTickRunner(deps) {
|
|
|
511
555
|
...(workspacePath === undefined ? {} : { workspacePath }),
|
|
512
556
|
...(mergeConflictDetected ? { mergeConflictDetected: true } : {}),
|
|
513
557
|
...(upstreamChanges === undefined ? {} : { upstreamChanges }),
|
|
558
|
+
onProcessStart: async (identity) => {
|
|
559
|
+
await deps.stateStore.updateRunRecordIf(runId, {
|
|
560
|
+
expect: (record) => record.status === 'running' &&
|
|
561
|
+
record.lease?.leaseId === lease.leaseId &&
|
|
562
|
+
record.lease.ownerInstanceId === ownerInstanceId,
|
|
563
|
+
update: (record) => ({
|
|
564
|
+
...record,
|
|
565
|
+
agentPid: identity.pid,
|
|
566
|
+
agentProcessStartedAt: identity.processStartedAt,
|
|
567
|
+
}),
|
|
568
|
+
});
|
|
569
|
+
},
|
|
514
570
|
});
|
|
571
|
+
clearInterval(leaseRenewalTimer);
|
|
572
|
+
leaseRenewalTimer = undefined;
|
|
515
573
|
const parsedRunnerResult = parseRunnerResult(runnerResult.result);
|
|
516
574
|
const rawSentinel = parsedRunnerResult.status;
|
|
517
575
|
// Coerce DONE → AWAITING_APPROVAL when the stage requires human sign-off.
|
|
@@ -583,8 +641,11 @@ export function createTickRunner(deps) {
|
|
|
583
641
|
: sentinel === 'AWAITING_APPROVAL'
|
|
584
642
|
? 'AWAITING_APPROVAL'
|
|
585
643
|
: undefined;
|
|
644
|
+
await transitionRunLifecycle('FINALISING');
|
|
645
|
+
const finalisingRecord = (await deps.stateStore.readRunRecord(runId));
|
|
586
646
|
await deps.stateStore.writeRunRecord({
|
|
587
|
-
...
|
|
647
|
+
...finalisingRecord,
|
|
648
|
+
lifecycle: 'TERMINAL',
|
|
588
649
|
status: sentinel === 'DONE'
|
|
589
650
|
? 'completed'
|
|
590
651
|
: sentinel === 'BLOCKED'
|
|
@@ -600,7 +661,10 @@ export function createTickRunner(deps) {
|
|
|
600
661
|
summary: parsedRunnerResult.body,
|
|
601
662
|
...(runnerResult.routing === undefined ? {} : { routing: runnerResult.routing }),
|
|
602
663
|
...(runnerResult.tokenUsage === undefined ? {} : { tokenUsage: runnerResult.tokenUsage }),
|
|
603
|
-
metadata:
|
|
664
|
+
metadata: {
|
|
665
|
+
...finalisingRecord.metadata,
|
|
666
|
+
...resultMetadata,
|
|
667
|
+
},
|
|
604
668
|
});
|
|
605
669
|
const runCompletedEvent = createEventEnvelope({
|
|
606
670
|
eventId: `${runId}-completed`,
|
|
@@ -682,10 +746,13 @@ export function createTickRunner(deps) {
|
|
|
682
746
|
};
|
|
683
747
|
}
|
|
684
748
|
catch (err) {
|
|
749
|
+
clearInterval(leaseRenewalTimer);
|
|
685
750
|
const finishedAt = deps.clock.now().toISOString();
|
|
686
751
|
const sentinel = 'FAILED';
|
|
752
|
+
const failedRecord = (await deps.stateStore.readRunRecord(runId)) ?? runningRecord;
|
|
687
753
|
await deps.stateStore.writeRunRecord({
|
|
688
|
-
...
|
|
754
|
+
...failedRecord,
|
|
755
|
+
lifecycle: 'TERMINAL',
|
|
689
756
|
status: 'failed',
|
|
690
757
|
finishedAt,
|
|
691
758
|
sentinel,
|
|
@@ -34,8 +34,18 @@ export const workflowOutcomeValues = [
|
|
|
34
34
|
'AWAITING_INPUT',
|
|
35
35
|
'CHANGES_REQUESTED',
|
|
36
36
|
];
|
|
37
|
+
export const executionAttemptLifecycleValues = [
|
|
38
|
+
'CREATED',
|
|
39
|
+
'CLAIMED',
|
|
40
|
+
'PREPARING',
|
|
41
|
+
'PROCESS_STARTING',
|
|
42
|
+
'RUNNING',
|
|
43
|
+
'FINALISING',
|
|
44
|
+
'TERMINAL',
|
|
45
|
+
];
|
|
37
46
|
export const executionOutcomeSchema = z.enum(executionOutcomeValues);
|
|
38
47
|
export const workflowOutcomeSchema = z.enum(workflowOutcomeValues);
|
|
48
|
+
export const executionAttemptLifecycleSchema = z.enum(executionAttemptLifecycleValues);
|
|
39
49
|
export const defaultAgentIdentity = 'Wake';
|
|
40
50
|
export const defaultSmokePrompt = `This is ${defaultAgentIdentity}, reply with "hi ${defaultAgentIdentity} only"`;
|
|
41
51
|
const modelOverridesSchema = z
|
|
@@ -291,7 +301,27 @@ const runTokenUsageSchema = z.object({
|
|
|
291
301
|
costUsd: z.number().nonnegative().optional(),
|
|
292
302
|
turns: z.number().nonnegative().optional(),
|
|
293
303
|
});
|
|
294
|
-
|
|
304
|
+
const runLeaseSchema = z.object({
|
|
305
|
+
leaseId: z.string(),
|
|
306
|
+
ownerInstanceId: z.string(),
|
|
307
|
+
acquiredAt: isoTimestampSchema,
|
|
308
|
+
lastRenewedAt: isoTimestampSchema,
|
|
309
|
+
expiresAt: isoTimestampSchema,
|
|
310
|
+
});
|
|
311
|
+
function legacyRunLifecycle(input) {
|
|
312
|
+
if (input.lifecycle !== undefined) {
|
|
313
|
+
return input;
|
|
314
|
+
}
|
|
315
|
+
return {
|
|
316
|
+
...input,
|
|
317
|
+
lifecycle: input.status === 'running' ? 'RUNNING' : 'TERMINAL',
|
|
318
|
+
};
|
|
319
|
+
}
|
|
320
|
+
export const runRecordSchema = z.preprocess((input) => {
|
|
321
|
+
return input !== null && typeof input === 'object'
|
|
322
|
+
? legacyRunLifecycle(input)
|
|
323
|
+
: input;
|
|
324
|
+
}, z.object({
|
|
295
325
|
schemaVersion: z.literal(1),
|
|
296
326
|
runId: z.string(),
|
|
297
327
|
// The work item this run belongs to. Required: run records are Wake-owned
|
|
@@ -306,7 +336,15 @@ export const runRecordSchema = z.object({
|
|
|
306
336
|
repo: z.string(),
|
|
307
337
|
issueNumber: z.number().int().positive(),
|
|
308
338
|
action: identifierSchema,
|
|
309
|
-
|
|
339
|
+
lifecycle: executionAttemptLifecycleSchema,
|
|
340
|
+
status: z.enum([
|
|
341
|
+
'running',
|
|
342
|
+
'completed',
|
|
343
|
+
'awaiting-approval',
|
|
344
|
+
'blocked',
|
|
345
|
+
'failed',
|
|
346
|
+
'superseded',
|
|
347
|
+
]),
|
|
310
348
|
startedAt: isoTimestampSchema,
|
|
311
349
|
finishedAt: isoTimestampSchema.optional(),
|
|
312
350
|
sessionId: z.string().optional(),
|
|
@@ -315,9 +353,14 @@ export const runRecordSchema = z.object({
|
|
|
315
353
|
workflowOutcome: workflowOutcomeSchema.optional(),
|
|
316
354
|
summary: z.string().optional(),
|
|
317
355
|
routing: runnerRoutingSchema.optional(),
|
|
356
|
+
lease: runLeaseSchema.optional(),
|
|
357
|
+
workerPid: z.number().int().positive().optional(),
|
|
358
|
+
workerProcessStartedAt: z.string().optional(),
|
|
359
|
+
agentPid: z.number().int().positive().optional(),
|
|
360
|
+
agentProcessStartedAt: z.string().optional(),
|
|
318
361
|
tokenUsage: runTokenUsageSchema.optional(),
|
|
319
362
|
metadata: z.record(z.string(), z.unknown()).optional(),
|
|
320
|
-
});
|
|
363
|
+
}));
|
|
321
364
|
const runnerHealthEntrySchema = z.object({
|
|
322
365
|
pausedUntil: isoTimestampSchema.optional(),
|
|
323
366
|
// 'reported' = CLI told us the real reset time; 'estimated' = exponential
|
package/dist/src/lib/lock.js
CHANGED
|
@@ -156,11 +156,11 @@ export async function acquireFileLock(path, options) {
|
|
|
156
156
|
}
|
|
157
157
|
catch (error) {
|
|
158
158
|
if (error.code === 'EEXIST') {
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
159
|
+
const status = await readFileLockStatus(path, {
|
|
160
|
+
...(options?.staleAfterMs === undefined ? {} : { staleAfterMs: options.staleAfterMs }),
|
|
161
|
+
now: options?.now ?? new Date(),
|
|
162
|
+
});
|
|
163
|
+
if (!status.active) {
|
|
164
164
|
await rm(path, { force: true });
|
|
165
165
|
try {
|
|
166
166
|
return await tryAcquire();
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { readFileSync } from 'node:fs';
|
|
2
|
+
const currentProcessStartedAt = new Date(Date.now() - process.uptime() * 1000).toISOString();
|
|
3
|
+
function readLinuxProcessStartTicks(pid) {
|
|
4
|
+
try {
|
|
5
|
+
const stat = readFileSync(`/proc/${pid}/stat`, 'utf8');
|
|
6
|
+
const endCommand = stat.lastIndexOf(')');
|
|
7
|
+
if (endCommand === -1) {
|
|
8
|
+
return undefined;
|
|
9
|
+
}
|
|
10
|
+
const fields = stat
|
|
11
|
+
.slice(endCommand + 2)
|
|
12
|
+
.trim()
|
|
13
|
+
.split(/\s+/);
|
|
14
|
+
const startTicks = fields[19];
|
|
15
|
+
return startTicks === undefined ? undefined : `linux-start-ticks:${startTicks}`;
|
|
16
|
+
}
|
|
17
|
+
catch {
|
|
18
|
+
return undefined;
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
export function readProcessIdentity(pid) {
|
|
22
|
+
if (!Number.isInteger(pid) || pid <= 0) {
|
|
23
|
+
return null;
|
|
24
|
+
}
|
|
25
|
+
try {
|
|
26
|
+
process.kill(pid, 0);
|
|
27
|
+
}
|
|
28
|
+
catch (error) {
|
|
29
|
+
if (error.code === 'ESRCH') {
|
|
30
|
+
return null;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
const linuxStartedAt = readLinuxProcessStartTicks(pid);
|
|
34
|
+
if (linuxStartedAt !== undefined) {
|
|
35
|
+
return { pid, processStartedAt: linuxStartedAt };
|
|
36
|
+
}
|
|
37
|
+
if (pid === process.pid) {
|
|
38
|
+
return { pid, processStartedAt: currentProcessStartedAt };
|
|
39
|
+
}
|
|
40
|
+
return null;
|
|
41
|
+
}
|
|
42
|
+
export function currentProcessIdentity() {
|
|
43
|
+
return (readProcessIdentity(process.pid) ?? {
|
|
44
|
+
pid: process.pid,
|
|
45
|
+
processStartedAt: currentProcessStartedAt,
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
export function processIdentityMatches(input) {
|
|
49
|
+
if (input.pid === undefined || input.processStartedAt === undefined) {
|
|
50
|
+
return false;
|
|
51
|
+
}
|
|
52
|
+
const current = readProcessIdentity(input.pid);
|
|
53
|
+
return current?.processStartedAt === input.processStartedAt;
|
|
54
|
+
}
|
package/dist/src/version.js
CHANGED