@atolis-hq/wake 0.3.109 → 0.3.111
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/bootstrap/surface-api-work-applications.js +30 -1
- package/dist/src/bootstrap/version.js +1 -1
- package/dist/src/integrations/github/application/inbound-comment-syntax.js +2 -0
- package/dist/src/integrations/github/application/inbound-review-signals.js +52 -7
- package/dist/src/integrations/github/application/inbound-translator.js +51 -2
- package/dist/src/integrations/github/contracts/events.js +13 -0
- package/dist/src/integrations/github/contracts/vocabulary.js +3 -1
- package/dist/src/orchestration/application/coordination-claims.js +33 -1
- package/dist/src/orchestration/application/group-budget-recorder.js +2 -2
- package/dist/src/orchestration/application/orchestration-service.js +50 -4
- package/dist/src/orchestration/application/request-child.js +1 -1
- package/dist/src/orchestration/contracts/event-decoder.js +19 -1
- package/dist/src/orchestration/contracts/events.js +1 -0
- package/dist/src/orchestration/domain/interpreter.js +1 -1
- package/dist/src/orchestration/domain/operator-retry-policy.js +5 -0
- package/dist/src/persistence/filesystem/file-event-journal.js +1 -1
- package/dist/src/persistence/filesystem/file-lock.js +18 -2
- package/dist/src/surfaces/api/presenters/orchestration.js +2 -1
- package/dist/src/surfaces/api/routes/commands.js +1 -1
- package/dist/src/surfaces/api/routes/work.js +1 -0
- package/dist/src/surfaces/web-assets/assets/{index-CZ1hZSda.js → index-DZHWNeNA.js} +2 -2
- package/dist/src/surfaces/web-assets/index.html +1 -1
- package/package.json +1 -1
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { pullRequestProjection } from '../activities/index.js';
|
|
2
2
|
import { executionProjection, runsByWorkflowInstanceProjection, } from '../execution/index.js';
|
|
3
3
|
import { correlationId, EventActorKind } from '../kernel/index.js';
|
|
4
|
-
import { OperatorRetryIneligibleError, orchestrationProjection, selectOperatorRetryTarget, workflowsByWorkItemProjection, } from '../orchestration/index.js';
|
|
4
|
+
import { ApprovalAuthorityKind, GroupBudgetExtensionIneligibleError, isGroupBudgetExtensionEligible, OperatorRetryIneligibleError, orchestrationProjection, selectOperatorRetryTarget, workflowsByWorkItemProjection, } from '../orchestration/index.js';
|
|
5
5
|
import { workCorrelationsProjection, } from '../resources/index.js';
|
|
6
6
|
import { ApiCommandStatus, fromWorkItemKey, presentResource, presentRun, presentWorkflowInstance, presentWorkItem, } from '../surfaces/index.js';
|
|
7
7
|
import { workItemId, WorkStatus } from '../work/index.js';
|
|
@@ -86,6 +86,32 @@ export function createSurfaceWorkApplications(root, now) {
|
|
|
86
86
|
}
|
|
87
87
|
return accepted(command.idempotencyKey, now());
|
|
88
88
|
},
|
|
89
|
+
async extend(key, command) {
|
|
90
|
+
const id = decodeWorkItemId(key);
|
|
91
|
+
if (id === undefined)
|
|
92
|
+
return extendIneligible('Work item was not found');
|
|
93
|
+
const work = await root.work.get(id);
|
|
94
|
+
if (work === null)
|
|
95
|
+
return extendIneligible('Work item was not found');
|
|
96
|
+
if (work.deleted === true)
|
|
97
|
+
return extendIneligible('Work item is deleted');
|
|
98
|
+
if (work.state !== WorkStatus.Open)
|
|
99
|
+
return extendIneligible('Work item is not open');
|
|
100
|
+
const primary = (await root.orchestration.listForWorkItem(id)).find((workflow) => workflow.parentWorkflowInstanceId === undefined);
|
|
101
|
+
if (primary === undefined)
|
|
102
|
+
return extendIneligible('Work item has no primary workflow');
|
|
103
|
+
if (!isGroupBudgetExtensionEligible(primary))
|
|
104
|
+
return extendIneligible('Workflow is not blocked on a gate budget exhaustion');
|
|
105
|
+
try {
|
|
106
|
+
await root.orchestration.extendBlockedGroupBudget(primary.workflowInstanceId, { kind: ApprovalAuthorityKind.Human }, commandContext(command.idempotencyKey, now));
|
|
107
|
+
}
|
|
108
|
+
catch (error) {
|
|
109
|
+
if (error instanceof GroupBudgetExtensionIneligibleError)
|
|
110
|
+
return extendIneligible(error.message);
|
|
111
|
+
throw error;
|
|
112
|
+
}
|
|
113
|
+
return accepted(command.idempotencyKey, now());
|
|
114
|
+
},
|
|
89
115
|
};
|
|
90
116
|
}
|
|
91
117
|
async function workDetail(root, key, now) {
|
|
@@ -203,3 +229,6 @@ function accepted(idempotencyKey, acceptedAt) {
|
|
|
203
229
|
function retryIneligible(detail) {
|
|
204
230
|
return { conflict: true, code: 'retry-ineligible', detail };
|
|
205
231
|
}
|
|
232
|
+
function extendIneligible(detail) {
|
|
233
|
+
return { conflict: true, code: 'extend-ineligible', detail };
|
|
234
|
+
}
|
|
@@ -20,6 +20,8 @@ export function recognizedCommand(body) {
|
|
|
20
20
|
return GitHubBuiltInCommand.Changes;
|
|
21
21
|
if (matchesCommand(line, GitHubBuiltInCommand.Retry))
|
|
22
22
|
return GitHubBuiltInCommand.Retry;
|
|
23
|
+
if (line === GitHubBuiltInCommand.Extend)
|
|
24
|
+
return GitHubBuiltInCommand.Extend;
|
|
23
25
|
return null;
|
|
24
26
|
}
|
|
25
27
|
export function isPlainReply(body) {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/* eslint-disable max-lines */
|
|
2
2
|
import { ActivityOutcomeKind, ReviewActorKind, ReviewDecisionKind, ReviewerAuthorizationSource, createPullRequestService, isReviewAuthorized, } from '../../../activities/index.js';
|
|
3
|
-
import { ApprovalAuthorityKind, selectOperatorRetryTarget, } from '../../../orchestration/index.js';
|
|
3
|
+
import { ApprovalAuthorityKind, isGroupBudgetExtensionEligible, selectOperatorRetryTarget, } from '../../../orchestration/index.js';
|
|
4
4
|
import { BuiltInResourceKind, ResourceCorrelationRole, ResourceStreamKind, resourceId, } from '../../../resources/index.js';
|
|
5
5
|
import { WorkStatus } from '../../../work/index.js';
|
|
6
6
|
import { GitHubBuiltInCommand, UnknownGitHubIdentity } from '../contracts/vocabulary.js';
|
|
@@ -77,15 +77,18 @@ async function applyIssueReviewSignal(input) {
|
|
|
77
77
|
const resource = await resources.get(resourceIdValue);
|
|
78
78
|
const command = recognizedCommand(event.payload.body);
|
|
79
79
|
const plainReply = isPlainReply(event.payload.body);
|
|
80
|
-
if (command
|
|
81
|
-
return
|
|
80
|
+
if (command !== null)
|
|
81
|
+
return applyIssueCommand({
|
|
82
82
|
event,
|
|
83
|
+
command,
|
|
84
|
+
resource,
|
|
83
85
|
resources,
|
|
84
86
|
work,
|
|
87
|
+
lookup,
|
|
85
88
|
orchestration,
|
|
89
|
+
adapter,
|
|
86
90
|
resourceId: resourceIdValue,
|
|
87
91
|
});
|
|
88
|
-
}
|
|
89
92
|
if (resource?.kind === BuiltInResourceKind.PullRequest) {
|
|
90
93
|
await applyWorkflowSignal({
|
|
91
94
|
event,
|
|
@@ -101,9 +104,7 @@ async function applyIssueReviewSignal(input) {
|
|
|
101
104
|
});
|
|
102
105
|
return;
|
|
103
106
|
}
|
|
104
|
-
if (
|
|
105
|
-
await applyIssueApprovalSignal({ ...input, command });
|
|
106
|
-
else if (plainReply) {
|
|
107
|
+
if (plainReply) {
|
|
107
108
|
await applyWorkflowSignal({
|
|
108
109
|
event,
|
|
109
110
|
resources,
|
|
@@ -116,6 +117,50 @@ async function applyIssueReviewSignal(input) {
|
|
|
116
117
|
});
|
|
117
118
|
}
|
|
118
119
|
}
|
|
120
|
+
async function applyIssueCommand(input) {
|
|
121
|
+
const { command, resource, resourceId: resourceIdValue } = input;
|
|
122
|
+
if (command === GitHubBuiltInCommand.Retry)
|
|
123
|
+
return applyIssueRetrySignal({ ...input, resourceId: resourceIdValue });
|
|
124
|
+
if (command === GitHubBuiltInCommand.Extend)
|
|
125
|
+
return applyIssueBudgetExtension({ ...input, resourceId: resourceIdValue });
|
|
126
|
+
if (resource?.kind === BuiltInResourceKind.PullRequest)
|
|
127
|
+
return applyWorkflowSignal({
|
|
128
|
+
...input,
|
|
129
|
+
resourceId: resourceIdValue,
|
|
130
|
+
outcome: command === GitHubBuiltInCommand.Approved
|
|
131
|
+
? ActivityOutcomeKind.Done
|
|
132
|
+
: ActivityOutcomeKind.Rejected,
|
|
133
|
+
acceptWaitingSignal: true,
|
|
134
|
+
resumeBlockedOnChanges: command === GitHubBuiltInCommand.Changes,
|
|
135
|
+
});
|
|
136
|
+
return applyIssueApprovalSignal({
|
|
137
|
+
...input,
|
|
138
|
+
command: command,
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
async function applyIssueBudgetExtension(input) {
|
|
142
|
+
const { event, resources, work, orchestration, resourceId: resourceIdValue } = input;
|
|
143
|
+
if (orchestration === undefined)
|
|
144
|
+
return;
|
|
145
|
+
if (!isReviewAuthorized({
|
|
146
|
+
actorId: event.payload.actor.id,
|
|
147
|
+
actorKind: event.payload.actor.kind,
|
|
148
|
+
resourceAuthorId: UnknownGitHubIdentity,
|
|
149
|
+
authorization: event.payload.authorization ?? { source: ReviewerAuthorizationSource.None },
|
|
150
|
+
}))
|
|
151
|
+
return;
|
|
152
|
+
const workItemIds = (await resources.correlations(resourceIdValue))
|
|
153
|
+
.filter((correlation) => correlation.role === ResourceCorrelationRole.Primary)
|
|
154
|
+
.map((correlation) => correlation.workItemId);
|
|
155
|
+
for (const workItemId of workItemIds) {
|
|
156
|
+
if (!(await isEligibleWorkItem(work, workItemId)))
|
|
157
|
+
continue;
|
|
158
|
+
const primary = (await orchestration.listAll()).find((workflow) => workflow.workItemId === workItemId && workflow.parentWorkflowInstanceId === undefined);
|
|
159
|
+
if (primary === undefined || !isGroupBudgetExtensionEligible(primary))
|
|
160
|
+
continue;
|
|
161
|
+
await orchestration.extendBlockedGroupBudget(primary.workflowInstanceId, { kind: ApprovalAuthorityKind.Human }, commandContext(event));
|
|
162
|
+
}
|
|
163
|
+
}
|
|
119
164
|
async function applyIssueRetrySignal(input) {
|
|
120
165
|
const { event, resources, work, orchestration, resourceId: resourceIdValue } = input;
|
|
121
166
|
if (orchestration === undefined)
|
|
@@ -270,7 +270,7 @@ export class InboundTranslator {
|
|
|
270
270
|
const intake = evaluateIntakeRules(this.intake, gitHubIntakeFacts(payload));
|
|
271
271
|
if (intake.ignored)
|
|
272
272
|
return;
|
|
273
|
-
const identity = await this.resolveIdentity({ adapter: this.adapter, key: payload.externalKey }, intake.admitted);
|
|
273
|
+
const identity = await this.resolveIdentity(event.eventId, { adapter: this.adapter, key: payload.externalKey }, intake.admitted);
|
|
274
274
|
if (identity === null)
|
|
275
275
|
return;
|
|
276
276
|
if (!identity.created) {
|
|
@@ -281,6 +281,7 @@ export class InboundTranslator {
|
|
|
281
281
|
const workItemIdValue = identity.workItemId;
|
|
282
282
|
if (workItemIdValue === null)
|
|
283
283
|
throw new Error('Created identity is missing a WorkItem');
|
|
284
|
+
await this.recordAdmissionStarted(event, identity);
|
|
284
285
|
const isPullRequest = payload.kind === 'pull-request';
|
|
285
286
|
await admitObservedWork(this.admissionServices(), {
|
|
286
287
|
adapter: this.adapter,
|
|
@@ -453,8 +454,13 @@ export class InboundTranslator {
|
|
|
453
454
|
routing: this.routing,
|
|
454
455
|
};
|
|
455
456
|
}
|
|
456
|
-
async resolveIdentity(externalKey, admitted) {
|
|
457
|
+
async resolveIdentity(sourceEventId, externalKey, admitted) {
|
|
457
458
|
const key = `${externalKey.adapter}:${externalKey.key}`;
|
|
459
|
+
const recorded = await this.recordedAdmissionIdentity(sourceEventId);
|
|
460
|
+
if (recorded !== undefined) {
|
|
461
|
+
this.minted.set(key, recorded);
|
|
462
|
+
return this.resumeAdmissionIdentity(recorded);
|
|
463
|
+
}
|
|
458
464
|
const inBatch = this.minted.get(key);
|
|
459
465
|
if (inBatch !== undefined)
|
|
460
466
|
return this.inBatchIdentity(inBatch);
|
|
@@ -468,6 +474,49 @@ export class InboundTranslator {
|
|
|
468
474
|
return null;
|
|
469
475
|
return { ...this.mintIdentity(externalKey), created: true, deleted: false };
|
|
470
476
|
}
|
|
477
|
+
async recordedAdmissionIdentity(sourceEventId) {
|
|
478
|
+
const event = (await this.journal.readStream(integrationStream(this.adapter)))
|
|
479
|
+
.map(selectGitHubAdapterEvent)
|
|
480
|
+
.find((candidate) => candidate?.eventType === GitHubEventType.AdmissionStarted &&
|
|
481
|
+
candidate.payload.sourceEventId === sourceEventId);
|
|
482
|
+
if (event?.eventType !== GitHubEventType.AdmissionStarted)
|
|
483
|
+
return undefined;
|
|
484
|
+
return { resourceId: event.payload.resourceId, workItemId: event.payload.workItemId };
|
|
485
|
+
}
|
|
486
|
+
async resumeAdmissionIdentity(identity) {
|
|
487
|
+
const work = await this.work.get(identity.workItemId);
|
|
488
|
+
if (work?.deleted)
|
|
489
|
+
return { ...identity, created: false, deleted: true };
|
|
490
|
+
const correlation = (await this.resources.correlations(identity.resourceId)).find((candidate) => candidate.role === ResourceCorrelationRole.Primary &&
|
|
491
|
+
candidate.workItemId === identity.workItemId);
|
|
492
|
+
return { ...identity, created: correlation === undefined || work === null, deleted: false };
|
|
493
|
+
}
|
|
494
|
+
async recordAdmissionStarted(source, identity) {
|
|
495
|
+
if (identity.workItemId === null)
|
|
496
|
+
throw new Error('Created identity is missing a WorkItem');
|
|
497
|
+
const stream = integrationStream(this.adapter);
|
|
498
|
+
const eventId = `github:admission-started:${this.adapter}:${source.eventId}`;
|
|
499
|
+
const existing = await this.journal.readStream(stream);
|
|
500
|
+
if (existing.some((event) => event.eventId === eventId))
|
|
501
|
+
return;
|
|
502
|
+
await this.journal.append(stream, existing.length, [
|
|
503
|
+
createEventDraft({
|
|
504
|
+
eventId,
|
|
505
|
+
eventType: GitHubEventType.AdmissionStarted,
|
|
506
|
+
occurredAt: source.occurredAt,
|
|
507
|
+
correlationId: source.correlationId,
|
|
508
|
+
causationId: source.eventId,
|
|
509
|
+
actor: { kind: EventActorKind.Integration, id: this.adapter },
|
|
510
|
+
source: { kind: EventSourceKind.Internal, id: 'github-inbound-translator' },
|
|
511
|
+
stream,
|
|
512
|
+
payload: {
|
|
513
|
+
sourceEventId: source.eventId,
|
|
514
|
+
resourceId: identity.resourceId,
|
|
515
|
+
workItemId: identity.workItemId,
|
|
516
|
+
},
|
|
517
|
+
}),
|
|
518
|
+
]);
|
|
519
|
+
}
|
|
471
520
|
async inBatchIdentity(identity) {
|
|
472
521
|
const existing = await this.work?.get(identity.workItemId);
|
|
473
522
|
return { ...identity, created: false, deleted: existing?.deleted === true };
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
import { ProviderPermission, PullRequestCheckState, PullRequestState, ReviewActorKind, ReviewerAuthorizationSource, } from '../../../activities/index.js';
|
|
3
3
|
import { brandedStringSchema, eventEnvelopeSchema, } from '../../../kernel/index.js';
|
|
4
|
+
import { resourceId } from '../../../resources/index.js';
|
|
4
5
|
import { workItemId } from '../../../work/index.js';
|
|
5
6
|
import { adapterId } from '../../contracts/identifiers.js';
|
|
6
7
|
import { ExternalWorkOutcome } from '../../contracts/outcome-vocabulary.js';
|
|
@@ -10,6 +11,7 @@ export const GitHubEventType = {
|
|
|
10
11
|
CommentObserved: 'integration.github.comment-observed',
|
|
11
12
|
DeliveryObserved: 'integration.github.delivery-observed',
|
|
12
13
|
DeletedWorkObservationSkipped: 'integration.github.deleted-work-observation-skipped',
|
|
14
|
+
AdmissionStarted: 'integration.github.admission-started',
|
|
13
15
|
InboundTranslationRetried: 'integration.github.inbound-translation-retried',
|
|
14
16
|
InboundTranslationRecovered: 'integration.github.inbound-translation-recovered',
|
|
15
17
|
InboundTranslationFailed: 'integration.github.inbound-translation-failed',
|
|
@@ -77,6 +79,17 @@ const eventSchema = z.discriminatedUnion('eventType', [
|
|
|
77
79
|
})
|
|
78
80
|
.strict(),
|
|
79
81
|
}),
|
|
82
|
+
eventEnvelopeSchema.extend({
|
|
83
|
+
eventType: z.literal(GitHubEventType.AdmissionStarted),
|
|
84
|
+
stream: streamSchema,
|
|
85
|
+
payload: z
|
|
86
|
+
.object({
|
|
87
|
+
sourceEventId: z.string(),
|
|
88
|
+
resourceId: brandedStringSchema(resourceId),
|
|
89
|
+
workItemId: brandedStringSchema(workItemId),
|
|
90
|
+
})
|
|
91
|
+
.strict(),
|
|
92
|
+
}),
|
|
80
93
|
eventEnvelopeSchema.extend({
|
|
81
94
|
eventType: z.literal(GitHubEventType.CommentObserved),
|
|
82
95
|
stream: streamSchema,
|
|
@@ -57,7 +57,8 @@ export const GitHubOutboundAction = {
|
|
|
57
57
|
// Comment-channel commands the GitHub adapter recognizes from human replies.
|
|
58
58
|
// /approved and /changes drive issue-review and PR-issue-comment signals;
|
|
59
59
|
// /accepted and /changes drive formal (native) PR review comments; /retry
|
|
60
|
-
// resumes a blocked/failed stage
|
|
60
|
+
// resumes a blocked/failed stage; /extend grants one exhausted gate attempt.
|
|
61
|
+
// Kept as the single source of truth for both
|
|
61
62
|
// recognition (inbound-comment-syntax.ts, review-command-translator.ts) and
|
|
62
63
|
// the commands/instructions surface, so the two cannot drift.
|
|
63
64
|
export const GitHubBuiltInCommand = defineClosedVocabulary({
|
|
@@ -65,4 +66,5 @@ export const GitHubBuiltInCommand = defineClosedVocabulary({
|
|
|
65
66
|
Accepted: '/accepted',
|
|
66
67
|
Changes: '/changes',
|
|
67
68
|
Retry: '/retry',
|
|
69
|
+
Extend: '/extend',
|
|
68
70
|
});
|
|
@@ -2,6 +2,7 @@ import { createEventDraft, EventSourceKind, } from '../../kernel/index.js';
|
|
|
2
2
|
import { selectOrchestrationEvent } from '../contracts/event-decoder.js';
|
|
3
3
|
import { OrchestrationEventType } from '../contracts/events.js';
|
|
4
4
|
import { isOrchestrationGroupStream, primaryOrchestrationGroupStream, } from '../contracts/streams.js';
|
|
5
|
+
import { ApprovalAuthorityKind } from '../contracts/vocabulary.js';
|
|
5
6
|
import { claimWithCasRetry } from './durable-append.js';
|
|
6
7
|
export class CoordinationClaims {
|
|
7
8
|
journal;
|
|
@@ -49,7 +50,7 @@ export class CoordinationClaims {
|
|
|
49
50
|
read: () => this.journal.readStream(stream),
|
|
50
51
|
decode: groupEvents,
|
|
51
52
|
alreadyClaimed: (events) => claimedRequestIds(events).has(request.requestId),
|
|
52
|
-
canAppend: (events) => events.
|
|
53
|
+
canAppend: (_rawEvents, events) => claimedRequestIds(events).size < request.maxPerGroup + budgetGrants(events),
|
|
53
54
|
append: async (sequence) => {
|
|
54
55
|
await this.journal.append(stream, sequence, [
|
|
55
56
|
createEventDraft({
|
|
@@ -70,6 +71,31 @@ export class CoordinationClaims {
|
|
|
70
71
|
return true;
|
|
71
72
|
return claimedRequestIds(groupEvents(await this.journal.readStream(stream))).has(request.requestId);
|
|
72
73
|
}
|
|
74
|
+
async grantBudget(stream, requestId, context, authority) {
|
|
75
|
+
if (authority.kind !== ApprovalAuthorityKind.Human)
|
|
76
|
+
throw new Error('A group budget extension requires human authority');
|
|
77
|
+
await claimWithCasRetry({
|
|
78
|
+
read: () => this.journal.readStream(stream),
|
|
79
|
+
decode: groupEvents,
|
|
80
|
+
alreadyClaimed: (events) => events.some((event) => event.eventType === OrchestrationEventType.GroupBudgetGranted &&
|
|
81
|
+
event.payload.commandId === context.commandId),
|
|
82
|
+
append: async (sequence) => {
|
|
83
|
+
await this.journal.append(stream, sequence, [
|
|
84
|
+
createEventDraft({
|
|
85
|
+
eventId: `${context.commandId}:${OrchestrationEventType.GroupBudgetGranted}:${requestId}`,
|
|
86
|
+
eventType: OrchestrationEventType.GroupBudgetGranted,
|
|
87
|
+
occurredAt: context.occurredAt,
|
|
88
|
+
correlationId: context.correlationId,
|
|
89
|
+
causationId: context.commandId,
|
|
90
|
+
actor: context.actor,
|
|
91
|
+
source: { kind: EventSourceKind.Internal, id: 'orchestration-service' },
|
|
92
|
+
stream,
|
|
93
|
+
payload: { key: stream.id, requestId, commandId: context.commandId },
|
|
94
|
+
}),
|
|
95
|
+
]);
|
|
96
|
+
},
|
|
97
|
+
});
|
|
98
|
+
}
|
|
73
99
|
}
|
|
74
100
|
function primaryOwner(events) {
|
|
75
101
|
for (const event of events) {
|
|
@@ -77,6 +103,7 @@ function primaryOwner(events) {
|
|
|
77
103
|
case OrchestrationEventType.PrimaryClaimed:
|
|
78
104
|
return event.payload.workflowInstanceId;
|
|
79
105
|
case OrchestrationEventType.GroupClaimed:
|
|
106
|
+
case OrchestrationEventType.GroupBudgetGranted:
|
|
80
107
|
break;
|
|
81
108
|
default:
|
|
82
109
|
assertNever(event);
|
|
@@ -90,12 +117,17 @@ function claimedRequestIds(events) {
|
|
|
90
117
|
case OrchestrationEventType.GroupClaimed:
|
|
91
118
|
return [event.payload.requestId];
|
|
92
119
|
case OrchestrationEventType.PrimaryClaimed:
|
|
120
|
+
case OrchestrationEventType.GroupBudgetGranted:
|
|
93
121
|
return [];
|
|
94
122
|
default:
|
|
95
123
|
return assertNever(event);
|
|
96
124
|
}
|
|
97
125
|
}));
|
|
98
126
|
}
|
|
127
|
+
function budgetGrants(events) {
|
|
128
|
+
return events.filter((event) => event.eventType === OrchestrationEventType.GroupBudgetGranted)
|
|
129
|
+
.length;
|
|
130
|
+
}
|
|
99
131
|
function groupEvents(events) {
|
|
100
132
|
return events
|
|
101
133
|
.map(selectOrchestrationEvent)
|
|
@@ -10,9 +10,9 @@ export class GroupBudgetRecorder {
|
|
|
10
10
|
constructor(journal) {
|
|
11
11
|
this.journal = journal;
|
|
12
12
|
}
|
|
13
|
-
async record(parent, metadata, maxPerGroup, context) {
|
|
13
|
+
async record(parent, metadata, workflowName, maxPerGroup, context) {
|
|
14
14
|
const stream = workflowInstanceStream(workflowInstanceId(parent.workflowInstanceId));
|
|
15
|
-
const payload = { ...metadata, maxPerGroup };
|
|
15
|
+
const payload = { ...metadata, workflowName, maxPerGroup };
|
|
16
16
|
for (;;) {
|
|
17
17
|
const events = await this.journal.readStream(stream);
|
|
18
18
|
if (events.some((event) => {
|
|
@@ -1,3 +1,9 @@
|
|
|
1
|
+
import { selectOrchestrationEvent } from '../contracts/event-decoder.js';
|
|
2
|
+
import { OrchestrationEventType } from '../contracts/events.js';
|
|
3
|
+
import { workflowInstanceId as toWorkflowInstanceId, } from '../contracts/identifiers.js';
|
|
4
|
+
import { childOrchestrationGroupStream, workflowInstanceStream } from '../contracts/streams.js';
|
|
5
|
+
import { ApprovalAuthorityKind } from '../contracts/vocabulary.js';
|
|
6
|
+
import { isGroupBudgetExtensionEligible } from '../domain/operator-retry-policy.js';
|
|
1
7
|
import { AcceptActivityOutcome } from './accept-activity-outcome.js';
|
|
2
8
|
import { AcceptSignal } from './accept-signal.js';
|
|
3
9
|
import { AdvanceWorkflow } from './advance-workflow.js';
|
|
@@ -16,18 +22,21 @@ export class OrchestrationService {
|
|
|
16
22
|
advanceWorkflow;
|
|
17
23
|
childWorkflows;
|
|
18
24
|
projections;
|
|
19
|
-
definitions;
|
|
20
25
|
coordinateAcceptSignal = (operation) => operation();
|
|
21
26
|
watchChildCancellation;
|
|
27
|
+
claims;
|
|
28
|
+
journal;
|
|
29
|
+
definitions;
|
|
22
30
|
constructor(journal, work, definitions, projections) {
|
|
23
31
|
this.projections = projections;
|
|
32
|
+
this.journal = journal;
|
|
24
33
|
const repository = new OrchestrationRepository(journal);
|
|
25
|
-
|
|
34
|
+
this.claims = new CoordinationClaims(journal);
|
|
26
35
|
this.definitions = new WorkflowDefinitionRegistry(journal, projections, definitions);
|
|
27
|
-
this.startWorkflow = new StartWorkflow(repository, claims, work, this.definitions);
|
|
36
|
+
this.startWorkflow = new StartWorkflow(repository, this.claims, work, this.definitions);
|
|
28
37
|
this.advanceWorkflow = new AdvanceWorkflow(repository, this.startWorkflow);
|
|
29
38
|
this.acceptWorkflowSignal = new AcceptSignal(repository, this.startWorkflow, work);
|
|
30
|
-
this.childWorkflows = new RequestChild(repository, claims, new GroupBudgetRecorder(journal), this.startWorkflow, this.advanceWorkflow);
|
|
39
|
+
this.childWorkflows = new RequestChild(repository, this.claims, new GroupBudgetRecorder(journal), this.startWorkflow, this.advanceWorkflow);
|
|
31
40
|
this.acceptActivityOutcome = new AcceptActivityOutcome(repository, this.startWorkflow, (context) => this.childWorkflows.reconcileChildCompletions(context));
|
|
32
41
|
}
|
|
33
42
|
start(command, context) {
|
|
@@ -78,6 +87,31 @@ export class OrchestrationService {
|
|
|
78
87
|
resumeBlockedStageForChanges(workflowInstanceId, context) {
|
|
79
88
|
return this.transitionWatchChildren(context, () => this.advanceWorkflow.resumeBlockedStageForChanges(workflowInstanceId, context));
|
|
80
89
|
}
|
|
90
|
+
async extendBlockedGroupBudget(workflowInstanceId, authority, context) {
|
|
91
|
+
if (authority.kind !== ApprovalAuthorityKind.Human)
|
|
92
|
+
throw new GroupBudgetExtensionIneligibleError('A group budget extension requires human authority');
|
|
93
|
+
const parent = await this.advanceWorkflow.get(workflowInstanceId);
|
|
94
|
+
if (parent === null || !isGroupBudgetExtensionEligible(parent))
|
|
95
|
+
throw new GroupBudgetExtensionIneligibleError('Workflow is not blocked on a gate budget exhaustion');
|
|
96
|
+
const exhausted = (await journalGroupBudgetExhaustion(this.journal, parent.workflowInstanceId)).at(-1);
|
|
97
|
+
if (exhausted === undefined)
|
|
98
|
+
throw new GroupBudgetExtensionIneligibleError('Workflow has no durable gate budget exhaustion to extend');
|
|
99
|
+
const workflowName = exhausted.workflowName ??
|
|
100
|
+
this.definitions
|
|
101
|
+
.currentDefinition(parent.workflowName)
|
|
102
|
+
.definition.watches.find((watch) => watch.id === exhausted.watchId)?.workflow;
|
|
103
|
+
if (workflowName === undefined)
|
|
104
|
+
throw new GroupBudgetExtensionIneligibleError('Workflow has no durable gate definition to extend');
|
|
105
|
+
const group = childOrchestrationGroupStream(parent.orchestrationGroupId, exhausted.watchId);
|
|
106
|
+
await this.claims.grantBudget(group, exhausted.requestId, context, authority);
|
|
107
|
+
await this.waitForSignal(parent.workflowInstanceId, parent.waitingFor, context);
|
|
108
|
+
return this.requestChild({
|
|
109
|
+
...exhausted,
|
|
110
|
+
workflowName,
|
|
111
|
+
maxPerGroup: exhausted.maxPerGroup,
|
|
112
|
+
requestId: toWorkflowInstanceId(exhausted.requestId),
|
|
113
|
+
}, { ...context, commandId: `${context.commandId}:dispatch` });
|
|
114
|
+
}
|
|
81
115
|
get(id) {
|
|
82
116
|
return this.advanceWorkflow.get(id);
|
|
83
117
|
}
|
|
@@ -138,4 +172,16 @@ export class OrchestrationService {
|
|
|
138
172
|
return this.transitionWatchChildren(context, () => this.advanceWorkflow.applyResourceTransition(workflowInstanceId, target, evidenceId, context));
|
|
139
173
|
}
|
|
140
174
|
}
|
|
175
|
+
export class GroupBudgetExtensionIneligibleError extends Error {
|
|
176
|
+
constructor(detail) {
|
|
177
|
+
super(detail);
|
|
178
|
+
this.name = 'GroupBudgetExtensionIneligibleError';
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
async function journalGroupBudgetExhaustion(journal, workflowInstanceId) {
|
|
182
|
+
return (await journal.readStream(workflowInstanceStream(workflowInstanceId))).flatMap((event) => {
|
|
183
|
+
const owned = selectOrchestrationEvent(event);
|
|
184
|
+
return owned?.eventType === OrchestrationEventType.GroupBudgetExhausted ? [owned.payload] : [];
|
|
185
|
+
});
|
|
186
|
+
}
|
|
141
187
|
export const createOrchestrationService = (journal, work, definitions, projections) => new OrchestrationService(journal, work, definitions, projections);
|
|
@@ -30,7 +30,7 @@ export class RequestChild {
|
|
|
30
30
|
const group = childOrchestrationGroupStream(parent.view.orchestrationGroupId, request.watchId);
|
|
31
31
|
const metadata = coordinationMetadata(parent.view, request);
|
|
32
32
|
if (!(await this.claims.claimWithinBudget(group, request, context))) {
|
|
33
|
-
await this.groupBudgets.record(parent.view, metadata, request.maxPerGroup, context);
|
|
33
|
+
await this.groupBudgets.record(parent.view, metadata, request.workflowName, request.maxPerGroup, context);
|
|
34
34
|
return { kind: 'group-budget-exhausted', requestId: request.requestId };
|
|
35
35
|
}
|
|
36
36
|
return this.workflows.execute({
|
|
@@ -184,6 +184,16 @@ const groupClaimedEnvelope = childGroupEnvelope(OrchestrationEventType.GroupClai
|
|
|
184
184
|
message: 'Group claim key must identify its stream',
|
|
185
185
|
});
|
|
186
186
|
});
|
|
187
|
+
const groupBudgetGrantedEnvelope = childGroupEnvelope(OrchestrationEventType.GroupBudgetGranted, z
|
|
188
|
+
.object({ key: childGroupIdSchema, requestId: z.string().min(1), commandId: z.string().min(1) })
|
|
189
|
+
.strict()).superRefine((event, context) => {
|
|
190
|
+
if (event.payload.key !== event.stream.id)
|
|
191
|
+
context.addIssue({
|
|
192
|
+
code: 'custom',
|
|
193
|
+
path: ['payload', 'key'],
|
|
194
|
+
message: 'Group budget grant key must identify its stream',
|
|
195
|
+
});
|
|
196
|
+
});
|
|
187
197
|
const eventSchema = z.discriminatedUnion('eventType', [
|
|
188
198
|
workflowEnvelope(OrchestrationEventType.InstanceStarted, z.union([
|
|
189
199
|
z
|
|
@@ -277,9 +287,16 @@ const eventSchema = z.discriminatedUnion('eventType', [
|
|
|
277
287
|
workflowEnvelope(OrchestrationEventType.ChildCompleted, childMetadataSchema),
|
|
278
288
|
workflowEnvelope(OrchestrationEventType.ChildCompletionConsumed, childMetadataSchema),
|
|
279
289
|
workflowEnvelope(OrchestrationEventType.CausalActivationRejected, childMetadataSchema),
|
|
280
|
-
workflowEnvelope(OrchestrationEventType.GroupBudgetExhausted, z
|
|
290
|
+
workflowEnvelope(OrchestrationEventType.GroupBudgetExhausted, z
|
|
291
|
+
.object({
|
|
292
|
+
...childMetadataShape,
|
|
293
|
+
maxPerGroup: z.number().int().positive(),
|
|
294
|
+
workflowName: brandedStringSchema(workflowName).optional(),
|
|
295
|
+
})
|
|
296
|
+
.strict()),
|
|
281
297
|
primaryClaimedEnvelope,
|
|
282
298
|
groupClaimedEnvelope,
|
|
299
|
+
groupBudgetGrantedEnvelope,
|
|
283
300
|
]);
|
|
284
301
|
export function decodeOrchestrationEvent(event) {
|
|
285
302
|
const result = eventSchema.safeParse(event);
|
|
@@ -299,6 +316,7 @@ export function selectWorkflowOrchestrationEvent(event) {
|
|
|
299
316
|
switch (owned.eventType) {
|
|
300
317
|
case OrchestrationEventType.PrimaryClaimed:
|
|
301
318
|
case OrchestrationEventType.GroupClaimed:
|
|
319
|
+
case OrchestrationEventType.GroupBudgetGranted:
|
|
302
320
|
case OrchestrationEventType.WorkflowDefinitionRegistered:
|
|
303
321
|
return null;
|
|
304
322
|
default:
|
|
@@ -27,6 +27,7 @@ export const OrchestrationEventType = {
|
|
|
27
27
|
GroupBudgetExhausted: 'orchestration.group-budget-exhausted',
|
|
28
28
|
PrimaryClaimed: 'orchestration.primary-claimed',
|
|
29
29
|
GroupClaimed: 'orchestration.group-claimed',
|
|
30
|
+
GroupBudgetGranted: 'orchestration.group-budget-granted',
|
|
30
31
|
};
|
|
31
32
|
export const WatchGateVerdictSignal = signalName('orchestration.watch-gate-verdict');
|
|
32
33
|
export const ApprovedSignal = signalName('approved');
|
|
@@ -10,7 +10,7 @@ import { finishRoute } from './transition.js';
|
|
|
10
10
|
export { startInstance } from './activation-policy.js';
|
|
11
11
|
export { acceptSignal, waitForSignal } from './signal-policy.js';
|
|
12
12
|
export { requestSupplementalActivity } from './supplemental-policy.js';
|
|
13
|
-
export { isChangesResumeEligible, isOperatorRetryEligible, requestChangesResume, requestOperatorRetry, selectOperatorRetryTarget, } from './operator-retry-policy.js';
|
|
13
|
+
export { isChangesResumeEligible, isGroupBudgetExtensionEligible, isOperatorRetryEligible, requestChangesResume, requestOperatorRetry, selectOperatorRetryTarget, } from './operator-retry-policy.js';
|
|
14
14
|
export { isRunnerQuotaRetryEligible, requestRunnerQuotaRetry, } from './runner-quota-retry-policy.js';
|
|
15
15
|
export function acceptActivityOutcome(definition, state, input) {
|
|
16
16
|
if (!isPendingOutcome(state, input))
|
|
@@ -60,6 +60,11 @@ export function isChangesResumeEligible(view) {
|
|
|
60
60
|
view.lastOutcome?.kind === ActivityOutcomeKind.Blocked &&
|
|
61
61
|
view.acceptedOutcomes.includes(pending.activationId));
|
|
62
62
|
}
|
|
63
|
+
export function isGroupBudgetExtensionEligible(view) {
|
|
64
|
+
return (view.status === WorkflowStatus.Blocked &&
|
|
65
|
+
view.blockReason?.startsWith('watch group budget exhausted for ') === true &&
|
|
66
|
+
view.waitingFor !== undefined);
|
|
67
|
+
}
|
|
63
68
|
export function requestOperatorRetry(definition, state, input) {
|
|
64
69
|
if (input.commandId.length === 0)
|
|
65
70
|
return { kind: 'ignored', reason: 'operator retry command id is required' };
|
|
@@ -76,7 +76,7 @@ export class FileEventJournal {
|
|
|
76
76
|
this.changeSignalSource.notify();
|
|
77
77
|
}
|
|
78
78
|
return finalizedEnvelopes;
|
|
79
|
-
});
|
|
79
|
+
}, { waitMs: 5_000, retryIntervalMs: 25 });
|
|
80
80
|
}
|
|
81
81
|
// Reads via the persisted per-segment manifest when the in-memory cache is
|
|
82
82
|
// cold (fresh process, or an on-disk change this instance didn't make),
|
|
@@ -317,10 +317,11 @@ async function linuxProcessStartTicks(pid) {
|
|
|
317
317
|
throw new Error('Invalid process start ticks');
|
|
318
318
|
return startTicks;
|
|
319
319
|
}
|
|
320
|
-
export async function withFileLock(path, operation) {
|
|
321
|
-
const lock = await
|
|
320
|
+
export async function withFileLock(path, operation, options = {}) {
|
|
321
|
+
const lock = await acquireFileLockWithWait(path, {
|
|
322
322
|
staleAfterMs: 60_000,
|
|
323
323
|
staleRequiresDeadProcess: true,
|
|
324
|
+
...options,
|
|
324
325
|
});
|
|
325
326
|
if (!lock.acquired)
|
|
326
327
|
throw new Error(`File lock is already held: ${path}`);
|
|
@@ -331,3 +332,18 @@ export async function withFileLock(path, operation) {
|
|
|
331
332
|
await lock.release();
|
|
332
333
|
}
|
|
333
334
|
}
|
|
335
|
+
async function acquireFileLockWithWait(path, options) {
|
|
336
|
+
const waitMs = options.waitMs ?? 0;
|
|
337
|
+
const retryIntervalMs = options.retryIntervalMs ?? 25;
|
|
338
|
+
const deadline = Date.now() + waitMs;
|
|
339
|
+
do {
|
|
340
|
+
const lock = await acquireFileLock(path, options);
|
|
341
|
+
if (lock.acquired || Date.now() >= deadline)
|
|
342
|
+
return lock;
|
|
343
|
+
await delay(Math.min(retryIntervalMs, Math.max(0, deadline - Date.now())));
|
|
344
|
+
} while (Date.now() < deadline);
|
|
345
|
+
return acquireFileLock(path, options);
|
|
346
|
+
}
|
|
347
|
+
function delay(milliseconds) {
|
|
348
|
+
return new Promise((resolve) => setTimeout(resolve, milliseconds));
|
|
349
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { isOperatorRetryEligible, WorkflowStatus, } from '../../../orchestration/index.js';
|
|
1
|
+
import { isGroupBudgetExtensionEligible, isOperatorRetryEligible, WorkflowStatus, } from '../../../orchestration/index.js';
|
|
2
2
|
import { toWorkItemKey } from '../contracts/work.js';
|
|
3
3
|
export function presentWorkflowInstance(value) {
|
|
4
4
|
return {
|
|
@@ -15,6 +15,7 @@ export function presentWorkflowInstance(value) {
|
|
|
15
15
|
? { blockReason: value.blockReason }
|
|
16
16
|
: {}),
|
|
17
17
|
...(isOperatorRetryEligible(value) ? { retryEligible: true } : {}),
|
|
18
|
+
...(isGroupBudgetExtensionEligible(value) ? { extendEligible: true } : {}),
|
|
18
19
|
...(value.waitingFor === undefined
|
|
19
20
|
? {}
|
|
20
21
|
: { waitingFor: { signalKind: value.waitingFor.signalKind } }),
|
|
@@ -35,7 +35,7 @@ async function dispatchAmbiguousRunResolution(applications, pathname, body) {
|
|
|
35
35
|
return accepted(await operation(runId, request), applications.now());
|
|
36
36
|
}
|
|
37
37
|
async function dispatchWorkCommand(applications, pathname, request) {
|
|
38
|
-
const match = /^\/api\/v1\/work-items\/([^/]+)\/commands\/(freeze|unfreeze|delete|retry)$/.exec(pathname);
|
|
38
|
+
const match = /^\/api\/v1\/work-items\/([^/]+)\/commands\/(freeze|unfreeze|delete|retry|extend)$/.exec(pathname);
|
|
39
39
|
if (match === null)
|
|
40
40
|
return undefined;
|
|
41
41
|
const decoded = decodePathSegment(match[1]);
|