@atolis-hq/wake 0.2.29 → 0.2.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/adapters/claude/claude-runner.js +1 -0
- package/dist/src/adapters/codex/codex-runner.js +1 -0
- package/dist/src/adapters/cursor/cursor-runner.js +1 -0
- package/dist/src/adapters/fake/fake-ticketing-system.js +5 -0
- package/dist/src/adapters/fs/state-store.js +7 -0
- package/dist/src/adapters/github/github-issues-work-source.js +105 -22
- package/dist/src/adapters/github/github-pull-request-activity-source.js +104 -13
- package/dist/src/adapters/runner/cli-command.js +17 -1
- package/dist/src/core/active-run-recovery.js +13 -8
- package/dist/src/core/event-builders.js +11 -0
- package/dist/src/core/outbox.js +95 -18
- package/dist/src/core/run-lease.js +38 -0
- package/dist/src/core/sink-router.js +37 -26
- package/dist/src/core/stale-run-reconciler.js +2 -6
- package/dist/src/core/tick-runner.js +79 -23
- package/dist/src/domain/schema.js +12 -0
- package/dist/src/lib/lock.js +5 -5
- package/dist/src/lib/process-identity.js +54 -0
- package/dist/src/version.js +1 -1
- package/package.json +1 -1
|
@@ -218,6 +218,7 @@ export function createClaudeRunner(options) {
|
|
|
218
218
|
args,
|
|
219
219
|
cwd: input.workspacePath ?? options.cwd,
|
|
220
220
|
timeoutMs: options.settings.timeoutMs,
|
|
221
|
+
...(input.onProcessStart === undefined ? {} : { onProcessStart: input.onProcessStart }),
|
|
221
222
|
});
|
|
222
223
|
const responseTranscriptPath = await writeRunnerTranscript({
|
|
223
224
|
config: input.config,
|
|
@@ -261,6 +261,7 @@ export function createCodexRunner(options) {
|
|
|
261
261
|
}),
|
|
262
262
|
cwd,
|
|
263
263
|
timeoutMs: options.settings.timeoutMs,
|
|
264
|
+
...(input.onProcessStart === undefined ? {} : { onProcessStart: input.onProcessStart }),
|
|
264
265
|
});
|
|
265
266
|
const responseTranscriptPath = await writeRunnerTranscript({
|
|
266
267
|
config: input.config,
|
|
@@ -212,6 +212,7 @@ export function createCursorRunner(options) {
|
|
|
212
212
|
}),
|
|
213
213
|
cwd,
|
|
214
214
|
timeoutMs: options.settings.timeoutMs,
|
|
215
|
+
...(input.onProcessStart === undefined ? {} : { onProcessStart: input.onProcessStart }),
|
|
215
216
|
});
|
|
216
217
|
const responseTranscriptPath = await writeRunnerTranscript({
|
|
217
218
|
config: input.config,
|
|
@@ -125,6 +125,8 @@ export function createFakeTicketingSystem(options) {
|
|
|
125
125
|
trigger: 'context-only',
|
|
126
126
|
payload: {
|
|
127
127
|
intentEventId: input.event.eventId,
|
|
128
|
+
idempotencyKey: input.event.payload.idempotencyKey,
|
|
129
|
+
deliveryState: 'CONFIRMED',
|
|
128
130
|
labels,
|
|
129
131
|
},
|
|
130
132
|
}),
|
|
@@ -149,6 +151,9 @@ export function createFakeTicketingSystem(options) {
|
|
|
149
151
|
trigger: 'context-only',
|
|
150
152
|
payload: {
|
|
151
153
|
intentEventId: input.event.eventId,
|
|
154
|
+
idempotencyKey: input.event.payload.idempotencyKey,
|
|
155
|
+
deliveryState: 'CONFIRMED',
|
|
156
|
+
providerId: commentId,
|
|
152
157
|
kind: input.event.payload.kind,
|
|
153
158
|
body: input.event.payload.body,
|
|
154
159
|
},
|
|
@@ -172,6 +172,13 @@ export function createStateStore({ wakeRoot }) {
|
|
|
172
172
|
await writeJsonFile(paths.runDateFile(parsed.startedAt.slice(0, 10), parsed.runId), parsed);
|
|
173
173
|
return parsed;
|
|
174
174
|
},
|
|
175
|
+
async updateRunRecordIf(runId, input) {
|
|
176
|
+
const current = await this.readRunRecord(runId);
|
|
177
|
+
if (current === null || !input.expect(current)) {
|
|
178
|
+
return null;
|
|
179
|
+
}
|
|
180
|
+
return this.writeRunRecord(input.update(current));
|
|
181
|
+
},
|
|
175
182
|
async writeSourceState(record) {
|
|
176
183
|
const parsed = parseSourceStateRecord(record);
|
|
177
184
|
await writeJsonFile(paths.sourceStateFile(parsed.source, parsed.key), parsed);
|
|
@@ -18,6 +18,11 @@ const pollOverlapMs = 60 * 60 * 1000;
|
|
|
18
18
|
// expectedEcho bookkeeping surviving a crash (#145).
|
|
19
19
|
const wakeCommentMarker = '<!-- wake:agent -->';
|
|
20
20
|
const githubSource = 'github';
|
|
21
|
+
export function wakeIdempotencyMarker(idempotencyKey) {
|
|
22
|
+
return typeof idempotencyKey === 'string'
|
|
23
|
+
? `<!-- wake:idempotency ${idempotencyKey} -->`
|
|
24
|
+
: undefined;
|
|
25
|
+
}
|
|
21
26
|
function normalizeTicketUpsert(input) {
|
|
22
27
|
// Names the resource, never the work item — the resolver stamps the
|
|
23
28
|
// canonical workItemKey after the poll (spec D1).
|
|
@@ -210,6 +215,7 @@ export function formatWakeComment(payload, controlPlaneUrl) {
|
|
|
210
215
|
const tokens = typeof payload.tokens === 'string' ? payload.tokens : undefined;
|
|
211
216
|
const cost = typeof payload.cost === 'string' ? payload.cost : undefined;
|
|
212
217
|
const workspacePath = typeof payload.workspacePath === 'string' ? payload.workspacePath : undefined;
|
|
218
|
+
const idempotencyMarker = wakeIdempotencyMarker(payload.idempotencyKey);
|
|
213
219
|
const details = [
|
|
214
220
|
action === undefined ? undefined : `stage \`${action}\``,
|
|
215
221
|
runnerName === undefined ? undefined : `runner \`${runnerName}\``,
|
|
@@ -225,7 +231,7 @@ export function formatWakeComment(payload, controlPlaneUrl) {
|
|
|
225
231
|
? defaultAgentIdentity
|
|
226
232
|
: `[${defaultAgentIdentity}](${controlPlaneUrl})`;
|
|
227
233
|
const header = `**${name}** _(Wake ${wakeVersion}${details.length > 0 ? ` · ${details.join(' · ')}` : ''})_`;
|
|
228
|
-
const sections = [wakeCommentMarker, header, body];
|
|
234
|
+
const sections = [wakeCommentMarker, idempotencyMarker, header, body].filter((section) => section !== undefined);
|
|
229
235
|
if (kind === 'approval-request') {
|
|
230
236
|
sections.push('_To approve this work, reply with `/approved`. To request changes, reply with `/changes` followed by your feedback. To ask a question without requesting changes, reply with `/ask` followed by your question._');
|
|
231
237
|
}
|
|
@@ -257,6 +263,33 @@ export function formatWakeComment(payload, controlPlaneUrl) {
|
|
|
257
263
|
}
|
|
258
264
|
return sections.join('\n\n');
|
|
259
265
|
}
|
|
266
|
+
function createIssueCommentPublishedEvent(input) {
|
|
267
|
+
return createEventEnvelope({
|
|
268
|
+
eventId: `${input.event.eventId}-published`,
|
|
269
|
+
workItemKey: input.event.workItemKey,
|
|
270
|
+
streamScope: 'work-item',
|
|
271
|
+
direction: 'outbound',
|
|
272
|
+
sourceSystem: 'github',
|
|
273
|
+
sourceEventType: 'ticket.reply.published',
|
|
274
|
+
sourceRefs: {
|
|
275
|
+
repo: input.repo,
|
|
276
|
+
issueNumber: input.issueNumber,
|
|
277
|
+
...(input.commentId === undefined ? {} : { commentId: input.commentId }),
|
|
278
|
+
},
|
|
279
|
+
occurredAt: input.publishedAt,
|
|
280
|
+
ingestedAt: input.publishedAt,
|
|
281
|
+
trigger: 'context-only',
|
|
282
|
+
payload: {
|
|
283
|
+
intentEventId: input.event.eventId,
|
|
284
|
+
idempotencyKey: input.event.payload.idempotencyKey,
|
|
285
|
+
deliveryState: 'CONFIRMED',
|
|
286
|
+
kind: input.event.payload.kind,
|
|
287
|
+
body: input.event.payload.body,
|
|
288
|
+
providerEventType: 'github.issue.comment.published',
|
|
289
|
+
...(input.commentId === undefined ? {} : { providerId: input.commentId }),
|
|
290
|
+
},
|
|
291
|
+
});
|
|
292
|
+
}
|
|
260
293
|
export function createGitHubIssuesWorkSource(deps) {
|
|
261
294
|
/** O(1): one shard read, then a direct projection read by work id. */
|
|
262
295
|
async function readProjectionForIssue(repo, issueNumber) {
|
|
@@ -398,6 +431,8 @@ export function createGitHubIssuesWorkSource(deps) {
|
|
|
398
431
|
trigger: 'context-only',
|
|
399
432
|
payload: {
|
|
400
433
|
intentEventId: input.event.eventId,
|
|
434
|
+
idempotencyKey: input.event.payload.idempotencyKey,
|
|
435
|
+
deliveryState: 'CONFIRMED',
|
|
401
436
|
...(nextStatusLabel !== undefined ? { statusLabel: nextStatusLabel } : {}),
|
|
402
437
|
...(nextStageLabel !== undefined ? { stageLabel: nextStageLabel } : {}),
|
|
403
438
|
...(nextWorkflowLabel !== undefined ? { workflowLabel: nextWorkflowLabel } : {}),
|
|
@@ -412,27 +447,75 @@ export function createGitHubIssuesWorkSource(deps) {
|
|
|
412
447
|
const response = await deps.client.createComment(owner, repoName, issueNumber, formatWakeComment(input.event.payload, await readControlPlaneUiUrl(deps.config.paths.wakeRoot)));
|
|
413
448
|
const commentId = extractCreatedCommentId(response);
|
|
414
449
|
return [
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
450
|
+
createIssueCommentPublishedEvent({
|
|
451
|
+
event: input.event,
|
|
452
|
+
repo,
|
|
453
|
+
issueNumber,
|
|
454
|
+
publishedAt,
|
|
455
|
+
...(commentId === undefined ? {} : { commentId }),
|
|
456
|
+
}),
|
|
457
|
+
];
|
|
458
|
+
},
|
|
459
|
+
async reconcileIntent(input) {
|
|
460
|
+
const repo = input.event.sourceRefs.repo;
|
|
461
|
+
const issueNumber = input.event.sourceRefs.issueNumber;
|
|
462
|
+
if (repo === undefined || issueNumber === undefined) {
|
|
463
|
+
return [];
|
|
464
|
+
}
|
|
465
|
+
const [owner, repoName] = repo.split('/');
|
|
466
|
+
if (owner === undefined || repoName === undefined) {
|
|
467
|
+
return [];
|
|
468
|
+
}
|
|
469
|
+
const publishedAt = deps.now().toISOString();
|
|
470
|
+
if (input.event.sourceEventType === 'wake.labels.requested') {
|
|
471
|
+
const projection = await deps.stateStore.readIssueState(input.event.workItemKey);
|
|
472
|
+
const currentLabels = projection?.issue.labels ?? [];
|
|
473
|
+
const expected = [
|
|
474
|
+
input.event.payload.statusLabel,
|
|
475
|
+
input.event.payload.stageLabel,
|
|
476
|
+
input.event.payload.workflowLabel,
|
|
477
|
+
].filter((label) => typeof label === 'string');
|
|
478
|
+
if (expected.every((label) => currentLabels.includes(label))) {
|
|
479
|
+
return [
|
|
480
|
+
createEventEnvelope({
|
|
481
|
+
eventId: `${input.event.eventId}-labels-updated`,
|
|
482
|
+
workItemKey: input.event.workItemKey,
|
|
483
|
+
streamScope: 'work-item',
|
|
484
|
+
direction: 'outbound',
|
|
485
|
+
sourceSystem: 'github',
|
|
486
|
+
sourceEventType: 'ticket.labels.updated',
|
|
487
|
+
sourceRefs: { repo, issueNumber },
|
|
488
|
+
occurredAt: publishedAt,
|
|
489
|
+
ingestedAt: publishedAt,
|
|
490
|
+
trigger: 'context-only',
|
|
491
|
+
payload: {
|
|
492
|
+
intentEventId: input.event.eventId,
|
|
493
|
+
idempotencyKey: input.event.payload.idempotencyKey,
|
|
494
|
+
deliveryState: 'CONFIRMED',
|
|
495
|
+
labels: currentLabels,
|
|
496
|
+
providerEventType: 'github.issue.labels.updated',
|
|
497
|
+
},
|
|
498
|
+
}),
|
|
499
|
+
];
|
|
500
|
+
}
|
|
501
|
+
return [];
|
|
502
|
+
}
|
|
503
|
+
const marker = wakeIdempotencyMarker(input.event.payload.idempotencyKey);
|
|
504
|
+
if (marker === undefined) {
|
|
505
|
+
return [];
|
|
506
|
+
}
|
|
507
|
+
const comments = await deps.client.listComments(owner, repoName, issueNumber, deps.config.sources.github.polling.commentPageSize);
|
|
508
|
+
const existing = comments.find((comment) => (comment.body ?? '').includes(marker));
|
|
509
|
+
if (existing === undefined) {
|
|
510
|
+
return [];
|
|
511
|
+
}
|
|
512
|
+
return [
|
|
513
|
+
createIssueCommentPublishedEvent({
|
|
514
|
+
event: input.event,
|
|
515
|
+
repo,
|
|
516
|
+
issueNumber,
|
|
517
|
+
publishedAt,
|
|
518
|
+
commentId: String(existing.id),
|
|
436
519
|
}),
|
|
437
520
|
];
|
|
438
521
|
},
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { buildResourceUri } from '../../domain/resource-uri.js';
|
|
2
2
|
import { createUnkeyedEventEnvelope, createEventEnvelope } from '../../lib/event-log.js';
|
|
3
|
-
import { formatGitHubError, formatWakeComment, readControlPlaneUiUrl, } from './github-issues-work-source.js';
|
|
3
|
+
import { formatGitHubError, formatWakeComment, readControlPlaneUiUrl, wakeIdempotencyMarker, } from './github-issues-work-source.js';
|
|
4
4
|
const githubPrSource = 'github-pr';
|
|
5
5
|
const wakeCommentMarker = '<!-- wake:agent -->';
|
|
6
6
|
function prResourceUri(repo, number) {
|
|
@@ -61,6 +61,27 @@ export function createGitHubPullRequestActivitySource(deps) {
|
|
|
61
61
|
}
|
|
62
62
|
return { owner, repo, repoRef: `${owner}/${repo}`, number: Number(numberStr) };
|
|
63
63
|
}
|
|
64
|
+
function reviewThreadRefFromUri(resourceUri) {
|
|
65
|
+
const locator = resourceUri.split(':').slice(2).join(':');
|
|
66
|
+
const match = /^([^/]+)\/([^#]+)#(\d+)\/rt_(\d+)$/.exec(locator);
|
|
67
|
+
if (match === null) {
|
|
68
|
+
return null;
|
|
69
|
+
}
|
|
70
|
+
const [, owner, repo, numberStr, rootIdStr] = match;
|
|
71
|
+
if (owner === undefined ||
|
|
72
|
+
repo === undefined ||
|
|
73
|
+
numberStr === undefined ||
|
|
74
|
+
rootIdStr === undefined) {
|
|
75
|
+
return null;
|
|
76
|
+
}
|
|
77
|
+
return {
|
|
78
|
+
owner,
|
|
79
|
+
repo,
|
|
80
|
+
repoRef: `${owner}/${repo}`,
|
|
81
|
+
number: Number(numberStr),
|
|
82
|
+
rootId: Number(rootIdStr),
|
|
83
|
+
};
|
|
84
|
+
}
|
|
64
85
|
async function discoverPullRequests(ingestedAt) {
|
|
65
86
|
const seenPrData = new Map();
|
|
66
87
|
const confirmedOpenRepos = new Set();
|
|
@@ -404,19 +425,11 @@ export function createGitHubPullRequestActivitySource(deps) {
|
|
|
404
425
|
}
|
|
405
426
|
const publishedAt = deps.now().toISOString();
|
|
406
427
|
if (resourceUri.startsWith('github:pr-review-thread:')) {
|
|
407
|
-
const
|
|
408
|
-
|
|
409
|
-
if (match === null) {
|
|
410
|
-
throw new Error(`cannot deliver intent ${input.event.eventId}: malformed review-thread uri ${resourceUri}`);
|
|
411
|
-
}
|
|
412
|
-
const [, owner, repo, numberStr, rootIdStr] = match;
|
|
413
|
-
if (owner === undefined ||
|
|
414
|
-
repo === undefined ||
|
|
415
|
-
numberStr === undefined ||
|
|
416
|
-
rootIdStr === undefined) {
|
|
428
|
+
const ref = reviewThreadRefFromUri(resourceUri);
|
|
429
|
+
if (ref === null) {
|
|
417
430
|
throw new Error(`cannot deliver intent ${input.event.eventId}: malformed review-thread uri ${resourceUri}`);
|
|
418
431
|
}
|
|
419
|
-
const response = await deps.client.replyToReviewComment(owner, repo,
|
|
432
|
+
const response = await deps.client.replyToReviewComment(ref.owner, ref.repo, ref.number, ref.rootId, formatWakeComment(input.event.payload, await readControlPlaneUiUrl(deps.config.paths.wakeRoot)));
|
|
420
433
|
return [
|
|
421
434
|
createEventEnvelope({
|
|
422
435
|
eventId: `${input.event.eventId}-published`,
|
|
@@ -434,8 +447,11 @@ export function createGitHubPullRequestActivitySource(deps) {
|
|
|
434
447
|
trigger: 'context-only',
|
|
435
448
|
payload: {
|
|
436
449
|
intentEventId: input.event.eventId,
|
|
450
|
+
idempotencyKey: input.event.payload.idempotencyKey,
|
|
451
|
+
deliveryState: 'CONFIRMED',
|
|
437
452
|
kind: input.event.payload.kind,
|
|
438
453
|
body: input.event.payload.body,
|
|
454
|
+
providerId: response?.id,
|
|
439
455
|
},
|
|
440
456
|
}),
|
|
441
457
|
];
|
|
@@ -444,7 +460,79 @@ export function createGitHubPullRequestActivitySource(deps) {
|
|
|
444
460
|
if (ref === null) {
|
|
445
461
|
throw new Error(`cannot deliver intent ${input.event.eventId}: malformed pr uri ${resourceUri}`);
|
|
446
462
|
}
|
|
447
|
-
await deps.client.createComment(ref.owner, ref.repo, ref.number, formatWakeComment(input.event.payload, await readControlPlaneUiUrl(deps.config.paths.wakeRoot)));
|
|
463
|
+
const response = await deps.client.createComment(ref.owner, ref.repo, ref.number, formatWakeComment(input.event.payload, await readControlPlaneUiUrl(deps.config.paths.wakeRoot)));
|
|
464
|
+
return [
|
|
465
|
+
createEventEnvelope({
|
|
466
|
+
eventId: `${input.event.eventId}-published`,
|
|
467
|
+
workItemKey: input.event.workItemKey,
|
|
468
|
+
streamScope: 'work-item',
|
|
469
|
+
direction: 'outbound',
|
|
470
|
+
sourceSystem: githubPrSource,
|
|
471
|
+
sourceEventType: 'pr.comment.reply.published',
|
|
472
|
+
sourceRefs: { repo: ref.repoRef, resourceUri },
|
|
473
|
+
occurredAt: publishedAt,
|
|
474
|
+
ingestedAt: publishedAt,
|
|
475
|
+
trigger: 'context-only',
|
|
476
|
+
payload: {
|
|
477
|
+
intentEventId: input.event.eventId,
|
|
478
|
+
idempotencyKey: input.event.payload.idempotencyKey,
|
|
479
|
+
deliveryState: 'CONFIRMED',
|
|
480
|
+
kind: input.event.payload.kind,
|
|
481
|
+
body: input.event.payload.body,
|
|
482
|
+
providerId: response?.data?.id,
|
|
483
|
+
},
|
|
484
|
+
}),
|
|
485
|
+
];
|
|
486
|
+
},
|
|
487
|
+
async reconcileIntent(input) {
|
|
488
|
+
const resourceUri = input.event.sourceRefs.resourceUri;
|
|
489
|
+
const marker = wakeIdempotencyMarker(input.event.payload.idempotencyKey);
|
|
490
|
+
if (resourceUri === undefined || marker === undefined) {
|
|
491
|
+
return [];
|
|
492
|
+
}
|
|
493
|
+
const publishedAt = deps.now().toISOString();
|
|
494
|
+
if (resourceUri.startsWith('github:pr-review-thread:')) {
|
|
495
|
+
const ref = reviewThreadRefFromUri(resourceUri);
|
|
496
|
+
if (ref === null) {
|
|
497
|
+
return [];
|
|
498
|
+
}
|
|
499
|
+
const comments = await deps.client.listReviewComments(ref.owner, ref.repo, ref.number, deps.config.sources.github.pullRequests.commentPageSize);
|
|
500
|
+
const existing = comments.find((comment) => (comment.body ?? '').includes(marker));
|
|
501
|
+
if (existing === undefined) {
|
|
502
|
+
return [];
|
|
503
|
+
}
|
|
504
|
+
return [
|
|
505
|
+
createEventEnvelope({
|
|
506
|
+
eventId: `${input.event.eventId}-published`,
|
|
507
|
+
workItemKey: input.event.workItemKey,
|
|
508
|
+
streamScope: 'work-item',
|
|
509
|
+
direction: 'outbound',
|
|
510
|
+
sourceSystem: githubPrSource,
|
|
511
|
+
sourceEventType: 'pr.review-comment.reply.published',
|
|
512
|
+
sourceRefs: { resourceUri, sourceUrl: existing.html_url },
|
|
513
|
+
occurredAt: publishedAt,
|
|
514
|
+
ingestedAt: publishedAt,
|
|
515
|
+
trigger: 'context-only',
|
|
516
|
+
payload: {
|
|
517
|
+
intentEventId: input.event.eventId,
|
|
518
|
+
idempotencyKey: input.event.payload.idempotencyKey,
|
|
519
|
+
deliveryState: 'CONFIRMED',
|
|
520
|
+
kind: input.event.payload.kind,
|
|
521
|
+
body: input.event.payload.body,
|
|
522
|
+
providerId: existing.id,
|
|
523
|
+
},
|
|
524
|
+
}),
|
|
525
|
+
];
|
|
526
|
+
}
|
|
527
|
+
const ref = repoAndNumberFromPrUri(resourceUri);
|
|
528
|
+
if (ref === null) {
|
|
529
|
+
return [];
|
|
530
|
+
}
|
|
531
|
+
const comments = await deps.client.listComments(ref.owner, ref.repo, ref.number, deps.config.sources.github.pullRequests.commentPageSize);
|
|
532
|
+
const existing = comments.find((comment) => (comment.body ?? '').includes(marker));
|
|
533
|
+
if (existing === undefined) {
|
|
534
|
+
return [];
|
|
535
|
+
}
|
|
448
536
|
return [
|
|
449
537
|
createEventEnvelope({
|
|
450
538
|
eventId: `${input.event.eventId}-published`,
|
|
@@ -459,8 +547,11 @@ export function createGitHubPullRequestActivitySource(deps) {
|
|
|
459
547
|
trigger: 'context-only',
|
|
460
548
|
payload: {
|
|
461
549
|
intentEventId: input.event.eventId,
|
|
550
|
+
idempotencyKey: input.event.payload.idempotencyKey,
|
|
551
|
+
deliveryState: 'CONFIRMED',
|
|
462
552
|
kind: input.event.payload.kind,
|
|
463
553
|
body: input.event.payload.body,
|
|
554
|
+
providerId: existing.id,
|
|
464
555
|
},
|
|
465
556
|
}),
|
|
466
557
|
];
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { spawn } from 'node:child_process';
|
|
2
|
+
import { readProcessIdentity } from '../../lib/process-identity.js';
|
|
2
3
|
const TIMEOUT_KILL_GRACE_MS = 5_000;
|
|
3
4
|
export function runAgentCliCommand(input) {
|
|
4
5
|
return new Promise((resolve, reject) => {
|
|
@@ -11,6 +12,16 @@ export function runAgentCliCommand(input) {
|
|
|
11
12
|
let stderr = '';
|
|
12
13
|
let timedOut = false;
|
|
13
14
|
let killTimer;
|
|
15
|
+
let startNotification = Promise.resolve();
|
|
16
|
+
let startNotificationError;
|
|
17
|
+
if (input.onProcessStart !== undefined && child.pid !== undefined) {
|
|
18
|
+
const identity = readProcessIdentity(child.pid);
|
|
19
|
+
if (identity !== null) {
|
|
20
|
+
startNotification = input.onProcessStart(identity).catch((error) => {
|
|
21
|
+
startNotificationError = error;
|
|
22
|
+
});
|
|
23
|
+
}
|
|
24
|
+
}
|
|
14
25
|
const timeoutTimer = input.timeoutMs === undefined
|
|
15
26
|
? undefined
|
|
16
27
|
: setTimeout(() => {
|
|
@@ -29,9 +40,14 @@ export function runAgentCliCommand(input) {
|
|
|
29
40
|
clearTimeout(killTimer);
|
|
30
41
|
reject(error);
|
|
31
42
|
});
|
|
32
|
-
child.on('close', (exitCode) => {
|
|
43
|
+
child.on('close', async (exitCode) => {
|
|
33
44
|
clearTimeout(timeoutTimer);
|
|
34
45
|
clearTimeout(killTimer);
|
|
46
|
+
await startNotification;
|
|
47
|
+
if (startNotificationError !== undefined) {
|
|
48
|
+
reject(startNotificationError);
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
35
51
|
resolve({
|
|
36
52
|
stdout,
|
|
37
53
|
stderr,
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import { readFileLockStatus } from '../lib/lock.js';
|
|
2
1
|
import { maxConfiguredRunnerTimeoutMs } from '../domain/runner-routing.js';
|
|
2
|
+
import { isRunLeaseExpired } from './run-lease.js';
|
|
3
|
+
import { processIdentityMatches } from '../lib/process-identity.js';
|
|
3
4
|
import { createOutbox } from './outbox.js';
|
|
4
5
|
import { createProjectionUpdater } from './projection-updater.js';
|
|
5
6
|
import { createStaleRunReconciler } from './stale-run-reconciler.js';
|
|
@@ -14,14 +15,18 @@ export function createActiveRunRecovery(deps) {
|
|
|
14
15
|
stateStore: deps.stateStore,
|
|
15
16
|
projectionUpdater,
|
|
16
17
|
});
|
|
17
|
-
async function isRunningRecordActive(record) {
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
});
|
|
21
|
-
if (!lock.active || lock.metadata === undefined) {
|
|
22
|
-
return false;
|
|
18
|
+
async function isRunningRecordActive(record, now) {
|
|
19
|
+
if (record.lease !== undefined && !isRunLeaseExpired(record, now)) {
|
|
20
|
+
return true;
|
|
23
21
|
}
|
|
24
|
-
return
|
|
22
|
+
return (processIdentityMatches({
|
|
23
|
+
pid: record.agentPid,
|
|
24
|
+
processStartedAt: record.agentProcessStartedAt,
|
|
25
|
+
}) ||
|
|
26
|
+
processIdentityMatches({
|
|
27
|
+
pid: record.workerPid,
|
|
28
|
+
processStartedAt: record.workerProcessStartedAt,
|
|
29
|
+
}));
|
|
25
30
|
}
|
|
26
31
|
const { reconcileStaleRunningRecords } = createStaleRunReconciler({
|
|
27
32
|
config: deps.config,
|
|
@@ -29,6 +29,8 @@ function isFreshTriggeringComment(candidate) {
|
|
|
29
29
|
export function createPublishIntentEvent(input) {
|
|
30
30
|
const tokenCount = extractTokenCount(input.runnerResult.tokenUsage);
|
|
31
31
|
const duration = formatDuration(input.startedAt, input.occurredAt);
|
|
32
|
+
const failureRepeated = input.runnerResult.failureClass !== undefined &&
|
|
33
|
+
input.previousFailureClass === input.runnerResult.failureClass;
|
|
32
34
|
return createEventEnvelope({
|
|
33
35
|
eventId: `${input.runId}-publish-intent`,
|
|
34
36
|
workItemKey: input.projection.workItemKey,
|
|
@@ -107,6 +109,12 @@ export function createPublishIntentEvent(input) {
|
|
|
107
109
|
? {}
|
|
108
110
|
: { cost: formatCostUsd(input.runnerResult.tokenUsage.costUsd) }),
|
|
109
111
|
...(input.workspacePath === undefined ? {} : { workspacePath: input.workspacePath }),
|
|
112
|
+
idempotencyKey: `${input.runId}:result-comment`,
|
|
113
|
+
deliveryState: 'PENDING',
|
|
114
|
+
...(failureRepeated ? { failureRepeated } : {}),
|
|
115
|
+
...(input.previousFailureClass === undefined
|
|
116
|
+
? {}
|
|
117
|
+
: { previousFailureClass: input.previousFailureClass }),
|
|
110
118
|
},
|
|
111
119
|
derivedHints: {
|
|
112
120
|
stage: input.sentinel === 'DONE' ? 'done' : input.projection.wake.stage,
|
|
@@ -114,6 +122,7 @@ export function createPublishIntentEvent(input) {
|
|
|
114
122
|
});
|
|
115
123
|
}
|
|
116
124
|
export function createLabelsEvent(input) {
|
|
125
|
+
const labelKind = input.statusLabel === 'wake:status.working' ? 'working-label' : 'completion-label';
|
|
117
126
|
return createEventEnvelope({
|
|
118
127
|
eventId: `${input.runId}-labels-${input.statusLabel.replace(/[^a-z0-9]+/gi, '-')}-${input.stageLabel.replace(/[^a-z0-9]+/gi, '-')}-${input.workflowLabel.replace(/[^a-z0-9]+/gi, '-')}`,
|
|
119
128
|
workItemKey: input.projection.workItemKey,
|
|
@@ -134,6 +143,8 @@ export function createLabelsEvent(input) {
|
|
|
134
143
|
stageLabel: input.stageLabel,
|
|
135
144
|
workflowLabel: input.workflowLabel,
|
|
136
145
|
origin: input.projection.origin ?? 'github',
|
|
146
|
+
idempotencyKey: `${input.runId}:${labelKind}`,
|
|
147
|
+
deliveryState: 'PENDING',
|
|
137
148
|
},
|
|
138
149
|
});
|
|
139
150
|
}
|
package/dist/src/core/outbox.js
CHANGED
|
@@ -12,6 +12,10 @@ const outboundConfirmationEventTypes = new Set([
|
|
|
12
12
|
'pr.comment.reply.published',
|
|
13
13
|
'pr.review-comment.reply.published',
|
|
14
14
|
]);
|
|
15
|
+
function intentId(event) {
|
|
16
|
+
const intentEventId = event.payload.intentEventId;
|
|
17
|
+
return typeof intentEventId === 'string' ? intentEventId : undefined;
|
|
18
|
+
}
|
|
15
19
|
// The outbox: outbound delivery (comments, labels) attempted independently of
|
|
16
20
|
// run-outcome recording, with a durable, bounded retry trace. Extracted from
|
|
17
21
|
// tick-runner.ts so it can be exercised in isolation; it has the cleanest
|
|
@@ -33,12 +37,60 @@ export function createOutbox(deps) {
|
|
|
33
37
|
payload: {
|
|
34
38
|
intentEventId: intentEvent.eventId,
|
|
35
39
|
intentEventType: intentEvent.sourceEventType,
|
|
40
|
+
idempotencyKey: intentEvent.payload.idempotencyKey,
|
|
41
|
+
deliveryState: 'PENDING',
|
|
36
42
|
error: err instanceof Error ? err.message : String(err),
|
|
37
43
|
},
|
|
38
44
|
});
|
|
39
45
|
await deps.stateStore.appendEventEnvelope(failureEvent);
|
|
40
46
|
await deps.projectionUpdater.rebuildFromEvents([failureEvent]);
|
|
41
47
|
}
|
|
48
|
+
async function recordSentUnconfirmed(intentEvent) {
|
|
49
|
+
const occurredAt = deps.clock.now().toISOString();
|
|
50
|
+
const sentEvent = createEventEnvelope({
|
|
51
|
+
eventId: `${intentEvent.eventId}-sent-unconfirmed-${randomUUID()}`,
|
|
52
|
+
workItemKey: intentEvent.workItemKey,
|
|
53
|
+
streamScope: 'work-item',
|
|
54
|
+
direction: 'internal',
|
|
55
|
+
sourceSystem: 'wake',
|
|
56
|
+
sourceEventType: 'wake.publish.sent-unconfirmed',
|
|
57
|
+
sourceRefs: intentEvent.sourceRefs,
|
|
58
|
+
occurredAt,
|
|
59
|
+
ingestedAt: occurredAt,
|
|
60
|
+
trigger: 'context-only',
|
|
61
|
+
payload: {
|
|
62
|
+
intentEventId: intentEvent.eventId,
|
|
63
|
+
intentEventType: intentEvent.sourceEventType,
|
|
64
|
+
idempotencyKey: intentEvent.payload.idempotencyKey,
|
|
65
|
+
deliveryState: 'SENT_UNCONFIRMED',
|
|
66
|
+
},
|
|
67
|
+
});
|
|
68
|
+
await deps.stateStore.appendEventEnvelope(sentEvent);
|
|
69
|
+
await deps.projectionUpdater.rebuildFromEvents([sentEvent]);
|
|
70
|
+
}
|
|
71
|
+
async function recordConfirmed(intentEvent, payload = {}) {
|
|
72
|
+
const confirmedAt = deps.clock.now().toISOString();
|
|
73
|
+
const confirmedEvent = createEventEnvelope({
|
|
74
|
+
eventId: `${intentEvent.eventId}-confirmed`,
|
|
75
|
+
workItemKey: intentEvent.workItemKey,
|
|
76
|
+
streamScope: 'work-item',
|
|
77
|
+
direction: 'internal',
|
|
78
|
+
sourceSystem: 'wake',
|
|
79
|
+
sourceEventType: 'wake.publish.confirmed',
|
|
80
|
+
sourceRefs: intentEvent.sourceRefs,
|
|
81
|
+
occurredAt: confirmedAt,
|
|
82
|
+
ingestedAt: confirmedAt,
|
|
83
|
+
trigger: 'context-only',
|
|
84
|
+
payload: {
|
|
85
|
+
intentEventId: intentEvent.eventId,
|
|
86
|
+
idempotencyKey: intentEvent.payload.idempotencyKey,
|
|
87
|
+
deliveryState: 'CONFIRMED',
|
|
88
|
+
...payload,
|
|
89
|
+
},
|
|
90
|
+
});
|
|
91
|
+
const appended = await deps.stateStore.appendEventEnvelope(confirmedEvent);
|
|
92
|
+
await deps.projectionUpdater.rebuildFromEvents([appended]);
|
|
93
|
+
}
|
|
42
94
|
// Outbound delivery (comments, labels) is attempted independently of run-outcome
|
|
43
95
|
// recording: a delivery failure must never rewrite an already-recorded run result
|
|
44
96
|
// (S1), and must always leave a durable, retryable trace instead of being lost
|
|
@@ -48,6 +100,7 @@ export function createOutbox(deps) {
|
|
|
48
100
|
return;
|
|
49
101
|
}
|
|
50
102
|
try {
|
|
103
|
+
await recordSentUnconfirmed(event);
|
|
51
104
|
const deliveryEvents = await deps.outboundSink.deliverIntent({ event });
|
|
52
105
|
for (const deliveryEvent of deliveryEvents) {
|
|
53
106
|
await deps.stateStore.appendEventEnvelope(deliveryEvent);
|
|
@@ -57,21 +110,7 @@ export function createOutbox(deps) {
|
|
|
57
110
|
// No confirmation event was produced (e.g. a no-op label update) but the
|
|
58
111
|
// sink did not throw. Record that delivery was attempted successfully so
|
|
59
112
|
// the outbox scan below does not retry it indefinitely.
|
|
60
|
-
|
|
61
|
-
const confirmedEvent = createEventEnvelope({
|
|
62
|
-
eventId: `${event.eventId}-confirmed`,
|
|
63
|
-
workItemKey: event.workItemKey,
|
|
64
|
-
streamScope: 'work-item',
|
|
65
|
-
direction: 'internal',
|
|
66
|
-
sourceSystem: 'wake',
|
|
67
|
-
sourceEventType: 'wake.publish.confirmed',
|
|
68
|
-
sourceRefs: event.sourceRefs,
|
|
69
|
-
occurredAt: confirmedAt,
|
|
70
|
-
ingestedAt: confirmedAt,
|
|
71
|
-
trigger: 'context-only',
|
|
72
|
-
payload: { intentEventId: event.eventId },
|
|
73
|
-
});
|
|
74
|
-
await deps.stateStore.appendEventEnvelope(confirmedEvent);
|
|
113
|
+
await recordConfirmed(event);
|
|
75
114
|
}
|
|
76
115
|
}
|
|
77
116
|
catch (err) {
|
|
@@ -83,6 +122,18 @@ export function createOutbox(deps) {
|
|
|
83
122
|
await deps.projectionUpdater.rebuildFromEvents([event]);
|
|
84
123
|
await attemptDelivery(event);
|
|
85
124
|
}
|
|
125
|
+
async function suppressOutboundEvent(event, input) {
|
|
126
|
+
const suppressedEvent = await deps.stateStore.appendEventEnvelope({
|
|
127
|
+
...event,
|
|
128
|
+
payload: {
|
|
129
|
+
...event.payload,
|
|
130
|
+
deliveryState: 'CONFIRMED',
|
|
131
|
+
suppressedPublishReason: input.suppressedPublishReason,
|
|
132
|
+
},
|
|
133
|
+
});
|
|
134
|
+
await deps.projectionUpdater.rebuildFromEvents([suppressedEvent]);
|
|
135
|
+
await recordConfirmed(event, { suppressedPublishReason: input.suppressedPublishReason });
|
|
136
|
+
}
|
|
86
137
|
// Adopts the outbox pattern: an intent is only considered delivered once a
|
|
87
138
|
// matching confirmation event exists. Anything left unconfirmed by a prior tick
|
|
88
139
|
// (e.g. the process crashed mid-delivery) is retried here, bounded so a
|
|
@@ -94,9 +145,10 @@ export function createOutbox(deps) {
|
|
|
94
145
|
const events = await deps.stateStore.listEventEnvelopes();
|
|
95
146
|
const confirmedIntentIds = new Set();
|
|
96
147
|
const failureAttempts = new Map();
|
|
148
|
+
const uncertainIntentIds = new Set();
|
|
97
149
|
for (const event of events) {
|
|
98
|
-
const intentEventId = event
|
|
99
|
-
if (
|
|
150
|
+
const intentEventId = intentId(event);
|
|
151
|
+
if (intentEventId === undefined) {
|
|
100
152
|
continue;
|
|
101
153
|
}
|
|
102
154
|
if (outboundConfirmationEventTypes.has(event.sourceEventType)) {
|
|
@@ -105,6 +157,9 @@ export function createOutbox(deps) {
|
|
|
105
157
|
if (event.sourceEventType === 'wake.publish.failed') {
|
|
106
158
|
failureAttempts.set(intentEventId, (failureAttempts.get(intentEventId) ?? 0) + 1);
|
|
107
159
|
}
|
|
160
|
+
if (event.sourceEventType === 'wake.publish.sent-unconfirmed') {
|
|
161
|
+
uncertainIntentIds.add(intentEventId);
|
|
162
|
+
}
|
|
108
163
|
}
|
|
109
164
|
for (const intent of events) {
|
|
110
165
|
if (!outboundIntentEventTypes.has(intent.sourceEventType)) {
|
|
@@ -116,8 +171,30 @@ export function createOutbox(deps) {
|
|
|
116
171
|
if ((failureAttempts.get(intent.eventId) ?? 0) >= outboxMaxAttempts) {
|
|
117
172
|
continue;
|
|
118
173
|
}
|
|
174
|
+
if (uncertainIntentIds.has(intent.eventId) &&
|
|
175
|
+
deps.outboundSink.reconcileIntent !== undefined) {
|
|
176
|
+
try {
|
|
177
|
+
const reconciledEvents = await deps.outboundSink.reconcileIntent({ event: intent });
|
|
178
|
+
for (const reconciledEvent of reconciledEvents) {
|
|
179
|
+
await deps.stateStore.appendEventEnvelope(reconciledEvent);
|
|
180
|
+
}
|
|
181
|
+
await deps.projectionUpdater.rebuildFromEvents(reconciledEvents);
|
|
182
|
+
if (reconciledEvents.length > 0) {
|
|
183
|
+
continue;
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
catch (err) {
|
|
187
|
+
await recordDeliveryFailure(intent, err);
|
|
188
|
+
continue;
|
|
189
|
+
}
|
|
190
|
+
}
|
|
119
191
|
await attemptDelivery(intent);
|
|
120
192
|
}
|
|
121
193
|
}
|
|
122
|
-
return {
|
|
194
|
+
return {
|
|
195
|
+
attemptDelivery,
|
|
196
|
+
deliverOutboundEvent,
|
|
197
|
+
retryUnconfirmedDeliveries,
|
|
198
|
+
suppressOutboundEvent,
|
|
199
|
+
};
|
|
123
200
|
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
export const runLeaseDurationMs = 60_000;
|
|
3
|
+
export const runLeaseRenewalIntervalMs = 20_000;
|
|
4
|
+
export function createRunLease(input) {
|
|
5
|
+
const acquiredAt = input.clock.now();
|
|
6
|
+
const expiresAt = new Date(acquiredAt.getTime() + runLeaseDurationMs);
|
|
7
|
+
return {
|
|
8
|
+
leaseId: `lease-${randomUUID()}`,
|
|
9
|
+
ownerInstanceId: input.ownerInstanceId,
|
|
10
|
+
acquiredAt: acquiredAt.toISOString(),
|
|
11
|
+
lastRenewedAt: acquiredAt.toISOString(),
|
|
12
|
+
expiresAt: expiresAt.toISOString(),
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
export function isRunLeaseExpired(record, now) {
|
|
16
|
+
if (record.lease === undefined) {
|
|
17
|
+
return true;
|
|
18
|
+
}
|
|
19
|
+
return Date.parse(record.lease.expiresAt) <= now.getTime();
|
|
20
|
+
}
|
|
21
|
+
export async function renewRunLease(input) {
|
|
22
|
+
const now = input.clock.now();
|
|
23
|
+
const renewed = await input.stateStore.updateRunRecordIf(input.runId, {
|
|
24
|
+
expect: (record) => record.status === 'running' &&
|
|
25
|
+
record.lifecycle !== 'TERMINAL' &&
|
|
26
|
+
record.lease?.leaseId === input.leaseId &&
|
|
27
|
+
record.lease.ownerInstanceId === input.ownerInstanceId,
|
|
28
|
+
update: (record) => ({
|
|
29
|
+
...record,
|
|
30
|
+
lease: {
|
|
31
|
+
...record.lease,
|
|
32
|
+
lastRenewedAt: now.toISOString(),
|
|
33
|
+
expiresAt: new Date(now.getTime() + runLeaseDurationMs).toISOString(),
|
|
34
|
+
},
|
|
35
|
+
}),
|
|
36
|
+
});
|
|
37
|
+
return renewed !== null;
|
|
38
|
+
}
|
|
@@ -38,36 +38,34 @@ function withSinkRef(event, sink) {
|
|
|
38
38
|
},
|
|
39
39
|
};
|
|
40
40
|
}
|
|
41
|
+
function targetSinksForEvent(input) {
|
|
42
|
+
const targetSinks = new Set();
|
|
43
|
+
const origin = typeof input.event.payload.origin === 'string' ? input.event.payload.origin : undefined;
|
|
44
|
+
const sourceOrigin = input.event.sourceRefs.sink ?? origin;
|
|
45
|
+
if (input.event.sourceEventType === 'wake.labels.requested') {
|
|
46
|
+
const projectionOrigin = typeof input.event.payload.origin === 'string' ? input.event.payload.origin : undefined;
|
|
47
|
+
if (projectionOrigin !== undefined) {
|
|
48
|
+
targetSinks.add(projectionOrigin);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
const resourceUri = input.event.sourceRefs.resourceUri;
|
|
52
|
+
if (input.event.sourceEventType === 'wake.publish.intent.requested' &&
|
|
53
|
+
sourceOrigin !== undefined) {
|
|
54
|
+
const resourceSink = resourceUri === undefined ? sourceOrigin : sinkNameForResourceUri(resourceUri, sourceOrigin);
|
|
55
|
+
targetSinks.add(input.sinksByName.has(resourceSink) ? resourceSink : sourceOrigin);
|
|
56
|
+
}
|
|
57
|
+
for (const [sinkName, sinkConfig] of Object.entries(input.config.sinks ?? {})) {
|
|
58
|
+
if (sinkConfig.subscribe.some((subscription) => subscriptionMatches(input.event, subscription))) {
|
|
59
|
+
targetSinks.add(sinkName);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
return targetSinks;
|
|
63
|
+
}
|
|
41
64
|
export function createOutboundSinkRouter(input) {
|
|
42
65
|
const sinksByName = new Map(input.sinks.map((sink) => [sink.sink, sink]));
|
|
43
66
|
return {
|
|
44
67
|
async deliverIntent({ event }) {
|
|
45
|
-
const targetSinks =
|
|
46
|
-
const origin = typeof event.payload.origin === 'string' ? event.payload.origin : undefined;
|
|
47
|
-
const sourceOrigin = event.sourceRefs.sink ?? origin;
|
|
48
|
-
if (event.sourceEventType === 'wake.labels.requested') {
|
|
49
|
-
const projectionOrigin = typeof event.payload.origin === 'string' ? event.payload.origin : undefined;
|
|
50
|
-
if (projectionOrigin !== undefined) {
|
|
51
|
-
targetSinks.add(projectionOrigin);
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
const resourceUri = event.sourceRefs.resourceUri;
|
|
55
|
-
if (event.sourceEventType === 'wake.publish.intent.requested' && sourceOrigin !== undefined) {
|
|
56
|
-
const resourceSink = resourceUri === undefined
|
|
57
|
-
? sourceOrigin
|
|
58
|
-
: sinkNameForResourceUri(resourceUri, sourceOrigin);
|
|
59
|
-
// A resource-derived sink name (e.g. a PR surface) may not be
|
|
60
|
-
// registered — the source that owns it can be disabled independently
|
|
61
|
-
// of the origin sink. Falling back to sourceOrigin here, rather than
|
|
62
|
-
// silently skipping an unregistered sink below, is what keeps a reply
|
|
63
|
-
// from being dropped-but-marked-delivered when that happens.
|
|
64
|
-
targetSinks.add(sinksByName.has(resourceSink) ? resourceSink : sourceOrigin);
|
|
65
|
-
}
|
|
66
|
-
for (const [sinkName, sinkConfig] of Object.entries(input.config.sinks ?? {})) {
|
|
67
|
-
if (sinkConfig.subscribe.some((subscription) => subscriptionMatches(event, subscription))) {
|
|
68
|
-
targetSinks.add(sinkName);
|
|
69
|
-
}
|
|
70
|
-
}
|
|
68
|
+
const targetSinks = targetSinksForEvent({ event, sinksByName, config: input.config });
|
|
71
69
|
const deliveryEvents = [];
|
|
72
70
|
for (const sinkName of targetSinks) {
|
|
73
71
|
const sink = sinksByName.get(sinkName);
|
|
@@ -79,5 +77,18 @@ export function createOutboundSinkRouter(input) {
|
|
|
79
77
|
}
|
|
80
78
|
return deliveryEvents;
|
|
81
79
|
},
|
|
80
|
+
async reconcileIntent({ event }) {
|
|
81
|
+
const targetSinks = targetSinksForEvent({ event, sinksByName, config: input.config });
|
|
82
|
+
const deliveryEvents = [];
|
|
83
|
+
for (const sinkName of targetSinks) {
|
|
84
|
+
const sink = sinksByName.get(sinkName);
|
|
85
|
+
if (sink?.reconcileIntent === undefined) {
|
|
86
|
+
continue;
|
|
87
|
+
}
|
|
88
|
+
const sinkDeliveryEvents = await sink.reconcileIntent({ event });
|
|
89
|
+
deliveryEvents.push(...sinkDeliveryEvents.map((deliveryEvent) => withSinkRef(deliveryEvent, sinkName)));
|
|
90
|
+
}
|
|
91
|
+
return deliveryEvents;
|
|
92
|
+
},
|
|
82
93
|
};
|
|
83
94
|
}
|
|
@@ -44,13 +44,9 @@ export function createStaleRunReconciler(deps) {
|
|
|
44
44
|
}
|
|
45
45
|
return 'startup-recovery';
|
|
46
46
|
}
|
|
47
|
-
const active = await deps.isRunningRecordActive(record);
|
|
47
|
+
const active = await deps.isRunningRecordActive(record, now);
|
|
48
48
|
if (active) {
|
|
49
|
-
|
|
50
|
-
if (!Number.isFinite(startedAtMs)) {
|
|
51
|
-
return 'timeout';
|
|
52
|
-
}
|
|
53
|
-
return now.getTime() - startedAtMs >= deps.runnerTimeoutMs() ? 'timeout' : null;
|
|
49
|
+
return null;
|
|
54
50
|
}
|
|
55
51
|
if (record.lifecycle === 'RUNNING' || record.lifecycle === 'PROCESS_STARTING') {
|
|
56
52
|
return 'runner-lock-not-active';
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { createLifecycleService } from './lifecycle-service.js';
|
|
2
2
|
import { createPolicyEngine } from './policy-engine.js';
|
|
3
3
|
import { createProjectionUpdater } from './projection-updater.js';
|
|
4
|
-
import { acquireFileLock
|
|
4
|
+
import { acquireFileLock } from '../lib/lock.js';
|
|
5
5
|
import { CORRELATION_REGISTERED_EVENT, parseRunnerArtifacts, parseRunnerResult, } from '../domain/schema.js';
|
|
6
6
|
import { maxConfiguredRunnerTimeoutMs, resolveRunnerRouting } from '../domain/runner-routing.js';
|
|
7
7
|
import { awaitingApprovalRunnerSentinel, stageLabelForStage } from '../domain/stages.js';
|
|
@@ -15,6 +15,8 @@ import { createOutbox } from './outbox.js';
|
|
|
15
15
|
import { createEventResolver } from './event-resolver.js';
|
|
16
16
|
import { createStaleRunReconciler } from './stale-run-reconciler.js';
|
|
17
17
|
import { createWorkspaceCleanup } from './workspace-cleanup.js';
|
|
18
|
+
import { createRunLease, isRunLeaseExpired, renewRunLease, runLeaseRenewalIntervalMs, } from './run-lease.js';
|
|
19
|
+
import { currentProcessIdentity, processIdentityMatches } from '../lib/process-identity.js';
|
|
18
20
|
function latestHumanCommentId(candidate) {
|
|
19
21
|
const human = candidate.comments.filter((c) => !c.isBotAuthored);
|
|
20
22
|
return human.at(-1)?.id;
|
|
@@ -39,7 +41,7 @@ export function createTickRunner(deps) {
|
|
|
39
41
|
resourceIndex: deps.resourceIndex,
|
|
40
42
|
config: deps.config,
|
|
41
43
|
});
|
|
42
|
-
const { deliverOutboundEvent, retryUnconfirmedDeliveries } = createOutbox({
|
|
44
|
+
const { deliverOutboundEvent, retryUnconfirmedDeliveries, suppressOutboundEvent } = createOutbox({
|
|
43
45
|
clock: deps.clock,
|
|
44
46
|
stateStore: deps.stateStore,
|
|
45
47
|
projectionUpdater,
|
|
@@ -68,6 +70,7 @@ export function createTickRunner(deps) {
|
|
|
68
70
|
workspaceManager: deps.workspaceManager,
|
|
69
71
|
projectionUpdater,
|
|
70
72
|
});
|
|
73
|
+
const ownerInstanceId = `instance-${process.pid}-${Date.now()}`;
|
|
71
74
|
function isAwaitingApproval(projection) {
|
|
72
75
|
return projection.context.lastRunSentinel === awaitingApprovalRunnerSentinel;
|
|
73
76
|
}
|
|
@@ -134,6 +137,7 @@ export function createTickRunner(deps) {
|
|
|
134
137
|
relation: 'primary',
|
|
135
138
|
provenance: 'agent-reported',
|
|
136
139
|
registeredBy: input.runId,
|
|
140
|
+
idempotencyKey: `${input.runId}:artifact-registration:${verified.resourceUri}`,
|
|
137
141
|
},
|
|
138
142
|
});
|
|
139
143
|
const appended = await deps.stateStore.appendEventEnvelope(event);
|
|
@@ -204,14 +208,18 @@ export function createTickRunner(deps) {
|
|
|
204
208
|
function runnerTimeoutMs() {
|
|
205
209
|
return maxConfiguredRunnerTimeoutMs(deps.config);
|
|
206
210
|
}
|
|
207
|
-
async function isRunningRecordActive(record) {
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
});
|
|
211
|
-
if (!lock.active || lock.metadata === undefined) {
|
|
212
|
-
return false;
|
|
211
|
+
async function isRunningRecordActive(record, now) {
|
|
212
|
+
if (record.lease !== undefined && !isRunLeaseExpired(record, now)) {
|
|
213
|
+
return true;
|
|
213
214
|
}
|
|
214
|
-
return
|
|
215
|
+
return (processIdentityMatches({
|
|
216
|
+
pid: record.agentPid,
|
|
217
|
+
processStartedAt: record.agentProcessStartedAt,
|
|
218
|
+
}) ||
|
|
219
|
+
processIdentityMatches({
|
|
220
|
+
pid: record.workerPid,
|
|
221
|
+
processStartedAt: record.workerProcessStartedAt,
|
|
222
|
+
}));
|
|
215
223
|
}
|
|
216
224
|
async function parkConfigDriftedProjections(projections) {
|
|
217
225
|
let parked = false;
|
|
@@ -289,9 +297,7 @@ export function createTickRunner(deps) {
|
|
|
289
297
|
}
|
|
290
298
|
}
|
|
291
299
|
async function runRunnerTick() {
|
|
292
|
-
const lock = await acquireFileLock(deps.stateStore.paths.runnerLockFile
|
|
293
|
-
staleAfterMs: runnerTimeoutMs(),
|
|
294
|
-
});
|
|
300
|
+
const lock = await acquireFileLock(deps.stateStore.paths.runnerLockFile);
|
|
295
301
|
if (!lock.acquired) {
|
|
296
302
|
return { status: 'locked' };
|
|
297
303
|
}
|
|
@@ -439,6 +445,8 @@ export function createTickRunner(deps) {
|
|
|
439
445
|
return { status: 'idle' };
|
|
440
446
|
}
|
|
441
447
|
const runId = `run-${candidate.issue.number}-${deps.clock.now().getTime()}`;
|
|
448
|
+
const lease = createRunLease({ clock: deps.clock, ownerInstanceId });
|
|
449
|
+
const workerIdentity = currentProcessIdentity();
|
|
442
450
|
const runningRecord = {
|
|
443
451
|
schemaVersion: 1,
|
|
444
452
|
runId,
|
|
@@ -450,6 +458,9 @@ export function createTickRunner(deps) {
|
|
|
450
458
|
status: 'running',
|
|
451
459
|
startedAt: nowIso,
|
|
452
460
|
routing,
|
|
461
|
+
lease,
|
|
462
|
+
workerPid: workerIdentity.pid,
|
|
463
|
+
workerProcessStartedAt: workerIdentity.processStartedAt,
|
|
453
464
|
};
|
|
454
465
|
await deps.stateStore.writeRunRecord(runningRecord);
|
|
455
466
|
async function transitionRunLifecycle(lifecycle) {
|
|
@@ -491,6 +502,18 @@ export function createTickRunner(deps) {
|
|
|
491
502
|
workflowLabel: workflowLabelForWorkflowName(workflowName),
|
|
492
503
|
occurredAt: eventStampNow(),
|
|
493
504
|
}));
|
|
505
|
+
let leaseRenewalTimer;
|
|
506
|
+
function startLeaseRenewal() {
|
|
507
|
+
leaseRenewalTimer = setInterval(() => {
|
|
508
|
+
void renewRunLease({
|
|
509
|
+
stateStore: deps.stateStore,
|
|
510
|
+
runId,
|
|
511
|
+
leaseId: lease.leaseId,
|
|
512
|
+
ownerInstanceId,
|
|
513
|
+
clock: deps.clock,
|
|
514
|
+
});
|
|
515
|
+
}, runLeaseRenewalIntervalMs);
|
|
516
|
+
}
|
|
494
517
|
try {
|
|
495
518
|
await transitionRunLifecycle('PREPARING');
|
|
496
519
|
const prepareResult = workspaceMode === 'branch'
|
|
@@ -521,6 +544,7 @@ export function createTickRunner(deps) {
|
|
|
521
544
|
});
|
|
522
545
|
const recentEvents = await deps.stateStore.listEventEnvelopesForWorkItem(candidate.workItemKey, 6);
|
|
523
546
|
await transitionRunLifecycle('RUNNING');
|
|
547
|
+
startLeaseRenewal();
|
|
524
548
|
const runnerResult = await deps.runner.run({
|
|
525
549
|
action,
|
|
526
550
|
projection: candidate,
|
|
@@ -532,7 +556,21 @@ export function createTickRunner(deps) {
|
|
|
532
556
|
...(workspacePath === undefined ? {} : { workspacePath }),
|
|
533
557
|
...(mergeConflictDetected ? { mergeConflictDetected: true } : {}),
|
|
534
558
|
...(upstreamChanges === undefined ? {} : { upstreamChanges }),
|
|
559
|
+
onProcessStart: async (identity) => {
|
|
560
|
+
await deps.stateStore.updateRunRecordIf(runId, {
|
|
561
|
+
expect: (record) => record.status === 'running' &&
|
|
562
|
+
record.lease?.leaseId === lease.leaseId &&
|
|
563
|
+
record.lease.ownerInstanceId === ownerInstanceId,
|
|
564
|
+
update: (record) => ({
|
|
565
|
+
...record,
|
|
566
|
+
agentPid: identity.pid,
|
|
567
|
+
agentProcessStartedAt: identity.processStartedAt,
|
|
568
|
+
}),
|
|
569
|
+
});
|
|
570
|
+
},
|
|
535
571
|
});
|
|
572
|
+
clearInterval(leaseRenewalTimer);
|
|
573
|
+
leaseRenewalTimer = undefined;
|
|
536
574
|
const parsedRunnerResult = parseRunnerResult(runnerResult.result);
|
|
537
575
|
const rawSentinel = parsedRunnerResult.status;
|
|
538
576
|
// Coerce DONE → AWAITING_APPROVAL when the stage requires human sign-off.
|
|
@@ -684,23 +722,31 @@ export function createTickRunner(deps) {
|
|
|
684
722
|
workflowLabel: workflowLabelForWorkflowName(workflowName),
|
|
685
723
|
occurredAt: finishedAt,
|
|
686
724
|
}));
|
|
725
|
+
const publishIntent = createPublishIntentEvent({
|
|
726
|
+
projection: candidate,
|
|
727
|
+
runId,
|
|
728
|
+
action,
|
|
729
|
+
runnerResult,
|
|
730
|
+
parsedRunnerResult,
|
|
731
|
+
sentinel,
|
|
732
|
+
occurredAt: finishedAt,
|
|
733
|
+
startedAt: nowIso,
|
|
734
|
+
...(workspacePath === undefined ? {} : { workspacePath }),
|
|
735
|
+
...(typeof candidate.context.lastFailureClass === 'string'
|
|
736
|
+
? { previousFailureClass: candidate.context.lastFailureClass }
|
|
737
|
+
: {}),
|
|
738
|
+
});
|
|
687
739
|
if (shouldPublishRunResult({
|
|
688
740
|
failureClass: runnerResult.failureClass,
|
|
689
741
|
previousFailureClass: candidate.context.lastFailureClass,
|
|
690
742
|
})) {
|
|
691
|
-
const publishIntent = createPublishIntentEvent({
|
|
692
|
-
projection: candidate,
|
|
693
|
-
runId,
|
|
694
|
-
action,
|
|
695
|
-
runnerResult,
|
|
696
|
-
parsedRunnerResult,
|
|
697
|
-
sentinel,
|
|
698
|
-
occurredAt: finishedAt,
|
|
699
|
-
startedAt: nowIso,
|
|
700
|
-
...(workspacePath === undefined ? {} : { workspacePath }),
|
|
701
|
-
});
|
|
702
743
|
await deliverOutboundEvent(publishIntent);
|
|
703
744
|
}
|
|
745
|
+
else {
|
|
746
|
+
await suppressOutboundEvent(publishIntent, {
|
|
747
|
+
suppressedPublishReason: runnerResult.failureClass === 'quota' ? 'quota-failure' : 'repeated-infra-failure',
|
|
748
|
+
});
|
|
749
|
+
}
|
|
704
750
|
return {
|
|
705
751
|
status: 'processed',
|
|
706
752
|
runId,
|
|
@@ -709,6 +755,7 @@ export function createTickRunner(deps) {
|
|
|
709
755
|
};
|
|
710
756
|
}
|
|
711
757
|
catch (err) {
|
|
758
|
+
clearInterval(leaseRenewalTimer);
|
|
712
759
|
const finishedAt = deps.clock.now().toISOString();
|
|
713
760
|
const sentinel = 'FAILED';
|
|
714
761
|
const failedRecord = (await deps.stateStore.readRunRecord(runId)) ?? runningRecord;
|
|
@@ -767,6 +814,7 @@ export function createTickRunner(deps) {
|
|
|
767
814
|
result: errorMessage,
|
|
768
815
|
model: 'unknown',
|
|
769
816
|
cli: 'unknown',
|
|
817
|
+
failureClass: 'infra',
|
|
770
818
|
};
|
|
771
819
|
const parsedInfraFailureResult = parseRunnerResult(infraFailureResult.result);
|
|
772
820
|
const failurePublishIntent = createPublishIntentEvent({
|
|
@@ -778,6 +826,9 @@ export function createTickRunner(deps) {
|
|
|
778
826
|
sentinel,
|
|
779
827
|
occurredAt: finishedAt,
|
|
780
828
|
startedAt: nowIso,
|
|
829
|
+
...(typeof candidate.context.lastFailureClass === 'string'
|
|
830
|
+
? { previousFailureClass: candidate.context.lastFailureClass }
|
|
831
|
+
: {}),
|
|
781
832
|
});
|
|
782
833
|
if (shouldPublishRunResult({
|
|
783
834
|
failureClass: 'infra',
|
|
@@ -785,6 +836,11 @@ export function createTickRunner(deps) {
|
|
|
785
836
|
})) {
|
|
786
837
|
await deliverOutboundEvent(failurePublishIntent);
|
|
787
838
|
}
|
|
839
|
+
else {
|
|
840
|
+
await suppressOutboundEvent(failurePublishIntent, {
|
|
841
|
+
suppressedPublishReason: 'repeated-infra-failure',
|
|
842
|
+
});
|
|
843
|
+
}
|
|
788
844
|
return {
|
|
789
845
|
status: 'processed',
|
|
790
846
|
runId,
|
|
@@ -301,6 +301,13 @@ const runTokenUsageSchema = z.object({
|
|
|
301
301
|
costUsd: z.number().nonnegative().optional(),
|
|
302
302
|
turns: z.number().nonnegative().optional(),
|
|
303
303
|
});
|
|
304
|
+
const runLeaseSchema = z.object({
|
|
305
|
+
leaseId: z.string(),
|
|
306
|
+
ownerInstanceId: z.string(),
|
|
307
|
+
acquiredAt: isoTimestampSchema,
|
|
308
|
+
lastRenewedAt: isoTimestampSchema,
|
|
309
|
+
expiresAt: isoTimestampSchema,
|
|
310
|
+
});
|
|
304
311
|
function legacyRunLifecycle(input) {
|
|
305
312
|
if (input.lifecycle !== undefined) {
|
|
306
313
|
return input;
|
|
@@ -346,6 +353,11 @@ export const runRecordSchema = z.preprocess((input) => {
|
|
|
346
353
|
workflowOutcome: workflowOutcomeSchema.optional(),
|
|
347
354
|
summary: z.string().optional(),
|
|
348
355
|
routing: runnerRoutingSchema.optional(),
|
|
356
|
+
lease: runLeaseSchema.optional(),
|
|
357
|
+
workerPid: z.number().int().positive().optional(),
|
|
358
|
+
workerProcessStartedAt: z.string().optional(),
|
|
359
|
+
agentPid: z.number().int().positive().optional(),
|
|
360
|
+
agentProcessStartedAt: z.string().optional(),
|
|
349
361
|
tokenUsage: runTokenUsageSchema.optional(),
|
|
350
362
|
metadata: z.record(z.string(), z.unknown()).optional(),
|
|
351
363
|
}));
|
package/dist/src/lib/lock.js
CHANGED
|
@@ -156,11 +156,11 @@ export async function acquireFileLock(path, options) {
|
|
|
156
156
|
}
|
|
157
157
|
catch (error) {
|
|
158
158
|
if (error.code === 'EEXIST') {
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
159
|
+
const status = await readFileLockStatus(path, {
|
|
160
|
+
...(options?.staleAfterMs === undefined ? {} : { staleAfterMs: options.staleAfterMs }),
|
|
161
|
+
now: options?.now ?? new Date(),
|
|
162
|
+
});
|
|
163
|
+
if (!status.active) {
|
|
164
164
|
await rm(path, { force: true });
|
|
165
165
|
try {
|
|
166
166
|
return await tryAcquire();
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { readFileSync } from 'node:fs';
|
|
2
|
+
const currentProcessStartedAt = new Date(Date.now() - process.uptime() * 1000).toISOString();
|
|
3
|
+
function readLinuxProcessStartTicks(pid) {
|
|
4
|
+
try {
|
|
5
|
+
const stat = readFileSync(`/proc/${pid}/stat`, 'utf8');
|
|
6
|
+
const endCommand = stat.lastIndexOf(')');
|
|
7
|
+
if (endCommand === -1) {
|
|
8
|
+
return undefined;
|
|
9
|
+
}
|
|
10
|
+
const fields = stat
|
|
11
|
+
.slice(endCommand + 2)
|
|
12
|
+
.trim()
|
|
13
|
+
.split(/\s+/);
|
|
14
|
+
const startTicks = fields[19];
|
|
15
|
+
return startTicks === undefined ? undefined : `linux-start-ticks:${startTicks}`;
|
|
16
|
+
}
|
|
17
|
+
catch {
|
|
18
|
+
return undefined;
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
export function readProcessIdentity(pid) {
|
|
22
|
+
if (!Number.isInteger(pid) || pid <= 0) {
|
|
23
|
+
return null;
|
|
24
|
+
}
|
|
25
|
+
try {
|
|
26
|
+
process.kill(pid, 0);
|
|
27
|
+
}
|
|
28
|
+
catch (error) {
|
|
29
|
+
if (error.code === 'ESRCH') {
|
|
30
|
+
return null;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
const linuxStartedAt = readLinuxProcessStartTicks(pid);
|
|
34
|
+
if (linuxStartedAt !== undefined) {
|
|
35
|
+
return { pid, processStartedAt: linuxStartedAt };
|
|
36
|
+
}
|
|
37
|
+
if (pid === process.pid) {
|
|
38
|
+
return { pid, processStartedAt: currentProcessStartedAt };
|
|
39
|
+
}
|
|
40
|
+
return null;
|
|
41
|
+
}
|
|
42
|
+
export function currentProcessIdentity() {
|
|
43
|
+
return (readProcessIdentity(process.pid) ?? {
|
|
44
|
+
pid: process.pid,
|
|
45
|
+
processStartedAt: currentProcessStartedAt,
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
export function processIdentityMatches(input) {
|
|
49
|
+
if (input.pid === undefined || input.processStartedAt === undefined) {
|
|
50
|
+
return false;
|
|
51
|
+
}
|
|
52
|
+
const current = readProcessIdentity(input.pid);
|
|
53
|
+
return current?.processStartedAt === input.processStartedAt;
|
|
54
|
+
}
|
package/dist/src/version.js
CHANGED