@atolis-hq/wake 0.3.83 → 0.3.84
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/activities/agent/agent-activity.js +7 -1
- package/dist/src/activities/agent/agent-result.js +5 -1
- package/dist/src/activities/contracts/vocabulary.js +1 -0
- package/dist/src/bootstrap/version.js +1 -1
- package/dist/src/control-plane/application/advance-once-dispatch.js +55 -9
- package/dist/src/control-plane/application/advance-once.js +24 -6
- package/dist/src/execution/application/execution-activity.js +3 -1
- package/dist/src/execution/contracts/runner.js +1 -0
- package/dist/src/execution/infrastructure/runners/claude.js +33 -8
- package/dist/src/execution/infrastructure/runners/codex.js +37 -0
- package/dist/src/execution/infrastructure/runners/cursor.js +15 -0
- package/dist/src/execution/infrastructure/runners/registry.js +9 -1
- package/dist/src/orchestration/application/advance-workflow.js +24 -1
- package/dist/src/orchestration/application/orchestration-service.js +3 -0
- package/dist/src/orchestration/contracts/event-decoder.js +8 -0
- package/dist/src/orchestration/contracts/events.js +1 -0
- package/dist/src/orchestration/domain/interpreter.js +1 -0
- package/dist/src/orchestration/domain/runner-quota-retry-policy.js +41 -0
- package/dist/src/orchestration/domain/workflow-instance-events.js +5 -0
- package/package.json +1 -1
|
@@ -175,7 +175,13 @@ function agentOutcome(result) {
|
|
|
175
175
|
return {
|
|
176
176
|
kind: ActivityOutcomeKind.Failed,
|
|
177
177
|
data: {
|
|
178
|
-
|
|
178
|
+
// 'provider-quota-exceeded' is a literal, not execution's ProviderQuotaExceededFailureKind
|
|
179
|
+
// import: activities sits below execution in the module dependency order and must not
|
|
180
|
+
// import it. AgentRunnerPort's failure.kind is an open string per the existing runner
|
|
181
|
+
// contract convention; this is the one place that interprets it.
|
|
182
|
+
reason: result.failure?.kind === 'provider-quota-exceeded'
|
|
183
|
+
? ActivityFailureCode.RunnerQuotaExceeded
|
|
184
|
+
: ActivityFailureCode.RunnerFailed,
|
|
179
185
|
...(result.failure === undefined ? {} : { message: result.failure.message }),
|
|
180
186
|
},
|
|
181
187
|
};
|
|
@@ -14,7 +14,11 @@ const blockedReason = z
|
|
|
14
14
|
.strict();
|
|
15
15
|
const failedReason = z
|
|
16
16
|
.object({
|
|
17
|
-
reason: z.enum([
|
|
17
|
+
reason: z.enum([
|
|
18
|
+
ActivityFailureCode.InvalidAgentResult,
|
|
19
|
+
ActivityFailureCode.RunnerFailed,
|
|
20
|
+
ActivityFailureCode.RunnerQuotaExceeded,
|
|
21
|
+
]),
|
|
18
22
|
message: z.string().optional(),
|
|
19
23
|
})
|
|
20
24
|
.strict();
|
|
@@ -31,6 +31,7 @@ export const ActivityFailureCode = defineClosedVocabulary({
|
|
|
31
31
|
InvalidAgentResult: 'invalid-agent-result',
|
|
32
32
|
AmbiguousRunnerResult: 'ambiguous-runner-result',
|
|
33
33
|
RunnerFailed: 'runner-failed',
|
|
34
|
+
RunnerQuotaExceeded: 'runner-quota-exceeded',
|
|
34
35
|
});
|
|
35
36
|
export const ActivityOutcomeKind = defineClosedVocabulary({
|
|
36
37
|
Waiting: 'waiting',
|
|
@@ -1,6 +1,30 @@
|
|
|
1
|
-
|
|
1
|
+
/* eslint-disable complexity, max-lines-per-function */
|
|
2
|
+
import { ActivityFailureCode, ActivityOutcomeKind, } from '../../activities/index.js';
|
|
3
|
+
import { ActivationClaimConflictError, NoEligibleRunnerError, RunStatus, WorkspaceMode, } from '../../execution/index.js';
|
|
2
4
|
import { WorkflowStatus } from '../../orchestration/index.js';
|
|
3
5
|
import { isExecutionFailureTerminal } from './execution-reconciliation.js';
|
|
6
|
+
export function isRunnerQuotaOutcome(outcome) {
|
|
7
|
+
return (outcome.kind === ActivityOutcomeKind.Failed &&
|
|
8
|
+
typeof outcome.data === 'object' &&
|
|
9
|
+
outcome.data !== null &&
|
|
10
|
+
'reason' in outcome.data &&
|
|
11
|
+
outcome.data.reason === ActivityFailureCode.RunnerQuotaExceeded);
|
|
12
|
+
}
|
|
13
|
+
export function runnerQuotaMessage(outcome) {
|
|
14
|
+
const data = outcome.data;
|
|
15
|
+
return typeof data.message === 'string' ? data.message : 'runner reported quota exhaustion';
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* A quota retry deliberately leaves the outcome unaccepted, so a port that
|
|
19
|
+
* never applied it (absent method, or a workflow instance that vanished)
|
|
20
|
+
* leaves the Run permanently unresolved. Surface that rather than going
|
|
21
|
+
* silent; it is an operational defect, not a workflow state.
|
|
22
|
+
*/
|
|
23
|
+
export function reportUnappliedRunnerQuotaRetry(result, input) {
|
|
24
|
+
if (result !== null && result !== undefined)
|
|
25
|
+
return;
|
|
26
|
+
console.error(`Runner-quota retry was not applied for activation ${input.activationId} (run ${input.runId}); its outcome remains unaccepted.`);
|
|
27
|
+
}
|
|
4
28
|
/**
|
|
5
29
|
* Fills open capacity within one Advancement call: dispatches ready
|
|
6
30
|
* activations one at a time, rechecking `maxConcurrentRuns` and reselecting
|
|
@@ -73,18 +97,40 @@ export async function runDispatchLoop(pending, ctx) {
|
|
|
73
97
|
stopReason = { kind: 'no-work' };
|
|
74
98
|
break;
|
|
75
99
|
}
|
|
100
|
+
if (error instanceof NoEligibleRunnerError) {
|
|
101
|
+
stopReason = { kind: 'no-work' };
|
|
102
|
+
break;
|
|
103
|
+
}
|
|
76
104
|
throw error;
|
|
77
105
|
}
|
|
78
106
|
if (run.status === RunStatus.Succeeded && run.outcome !== undefined) {
|
|
79
|
-
if (
|
|
80
|
-
|
|
81
|
-
|
|
107
|
+
if (isRunnerQuotaOutcome(run.outcome)) {
|
|
108
|
+
// No isDispatchPaused check here: a quota retry re-requests the same
|
|
109
|
+
// stage's activity without publishing an outcome or consuming retry
|
|
110
|
+
// budget, so it must not be held behind maintenance pause the way an
|
|
111
|
+
// outward-publishing acceptOutcome call is below.
|
|
112
|
+
const retried = await ctx.orchestration.retryRunnerQuotaFailure?.(selected.workflow.workflowInstanceId, {
|
|
113
|
+
activationId: selected.activation.activationId,
|
|
114
|
+
runId: run.runId,
|
|
115
|
+
runnerName: run.runner?.name ?? 'unknown-runner',
|
|
116
|
+
message: runnerQuotaMessage(run.outcome),
|
|
117
|
+
}, ctx.commandContext(run.runId));
|
|
118
|
+
reportUnappliedRunnerQuotaRetry(retried, {
|
|
119
|
+
activationId: selected.activation.activationId,
|
|
120
|
+
runId: run.runId,
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
else {
|
|
124
|
+
if (await ctx.isDispatchPaused()) {
|
|
125
|
+
stopReason = { kind: 'paused' };
|
|
126
|
+
break;
|
|
127
|
+
}
|
|
128
|
+
await ctx.orchestration.acceptOutcome({
|
|
129
|
+
workflowInstanceId: selected.workflow.workflowInstanceId,
|
|
130
|
+
activationId: selected.activation.activationId,
|
|
131
|
+
outcome: run.outcome,
|
|
132
|
+
}, ctx.commandContext(run.runId));
|
|
82
133
|
}
|
|
83
|
-
await ctx.orchestration.acceptOutcome({
|
|
84
|
-
workflowInstanceId: selected.workflow.workflowInstanceId,
|
|
85
|
-
activationId: selected.activation.activationId,
|
|
86
|
-
outcome: run.outcome,
|
|
87
|
-
}, ctx.commandContext(run.runId));
|
|
88
134
|
}
|
|
89
135
|
if (isExecutionFailureTerminal(run.status))
|
|
90
136
|
await ctx.orchestration.resolveExecutionFailure?.(selected.workflow.workflowInstanceId, {
|
|
@@ -5,7 +5,7 @@ import { isAmbiguityResolutionBlock, WorkflowStatus } from '../../orchestration/
|
|
|
5
5
|
import { WorkStatus } from '../../work/index.js';
|
|
6
6
|
import { ControlStreamKind } from '../contracts/streams.js';
|
|
7
7
|
import { DispatchPolicy } from '../domain/dispatch-policy.js';
|
|
8
|
-
import { runDispatchLoop } from './advance-once-dispatch.js';
|
|
8
|
+
import { isRunnerQuotaOutcome, reportUnappliedRunnerQuotaRetry, runDispatchLoop, runnerQuotaMessage, } from './advance-once-dispatch.js';
|
|
9
9
|
import { findUnresolvedSucceededTerminal, findUnresolvedTerminal, } from './execution-reconciliation.js';
|
|
10
10
|
export function createAdvanceOnce(orchestration, execution, resources, clock, dependencies) {
|
|
11
11
|
const runnerIneligibility = dependencies.runnerIneligibility ?? (async () => new Set());
|
|
@@ -96,11 +96,29 @@ export function createAdvanceOnce(orchestration, execution, resources, clock, de
|
|
|
96
96
|
if (await isDispatchPaused())
|
|
97
97
|
return { kind: 'paused' };
|
|
98
98
|
if (recovery.run.status === RunStatus.Succeeded) {
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
99
|
+
// An agent Run completes after execution.attempt() has already returned,
|
|
100
|
+
// so this reconciliation — not the dispatch loop — is where a real
|
|
101
|
+
// quota-classified outcome is resolved. It must divert here too, or the
|
|
102
|
+
// quota failure is published and charged to the retry budget after all.
|
|
103
|
+
if (isRunnerQuotaOutcome(recovery.run.outcome)) {
|
|
104
|
+
const retried = await orchestration.retryRunnerQuotaFailure?.(recovery.item.workflow.workflowInstanceId, {
|
|
105
|
+
activationId: recovery.item.activation.activationId,
|
|
106
|
+
runId: recovery.run.runId,
|
|
107
|
+
runnerName: recovery.run.runner?.name ?? 'unknown-runner',
|
|
108
|
+
message: runnerQuotaMessage(recovery.run.outcome),
|
|
109
|
+
}, context(recovery.run.runId));
|
|
110
|
+
reportUnappliedRunnerQuotaRetry(retried, {
|
|
111
|
+
activationId: recovery.item.activation.activationId,
|
|
112
|
+
runId: recovery.run.runId,
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
else {
|
|
116
|
+
await orchestration.acceptOutcome({
|
|
117
|
+
workflowInstanceId: recovery.item.workflow.workflowInstanceId,
|
|
118
|
+
activationId: recovery.item.activation.activationId,
|
|
119
|
+
outcome: recovery.run.outcome,
|
|
120
|
+
}, context(recovery.run.runId));
|
|
121
|
+
}
|
|
104
122
|
return {
|
|
105
123
|
kind: 'progressed',
|
|
106
124
|
dispatched: [
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { ExecutionEventType } from '../contracts/events.js';
|
|
2
|
+
import { ProviderQuotaExceededFailureKind } from '../contracts/runner.js';
|
|
2
3
|
import { parseAgentRunnerResponse } from '../infrastructure/agent-runner-adapter.js';
|
|
3
4
|
import { createRunEvent } from './run-lifecycle.js';
|
|
4
5
|
export async function executeActivity(runtime, currentRunId, request) {
|
|
@@ -77,7 +78,8 @@ function runnerResultReporter(runtime, currentRunId, context, activation, reques
|
|
|
77
78
|
payload: { transport: result.transport, agent: parseAgentRunnerResponse(result) },
|
|
78
79
|
});
|
|
79
80
|
await appendIdempotently(runtime.repository, currentRunId, loaded.sequence, event);
|
|
80
|
-
if (result.failure?.kind ===
|
|
81
|
+
if (result.failure?.kind === ProviderQuotaExceededFailureKind &&
|
|
82
|
+
request.runnerName !== undefined)
|
|
81
83
|
await runtime.dependencies.reportRunnerQuota?.({
|
|
82
84
|
runnerName: request.runnerName,
|
|
83
85
|
message: result.failure.message,
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
export const ProviderQuotaExceededFailureKind = 'provider-quota-exceeded';
|
|
1
2
|
// inputTokens/outputTokens/cacheReadTokens/cacheWriteTokens/costUsd are the known
|
|
2
3
|
// numeric keys agent-runner-adapter.ts writes into AgentRunResponse.metadata.
|
|
3
4
|
export function agentTokenUsage(metadata) {
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { ExternalExecutionKind } from '../../../activities/index.js';
|
|
2
|
+
import { ProviderQuotaExceededFailureKind } from '../../contracts/runner.js';
|
|
2
3
|
import { ExecutionCancellationReason, RunStatus } from '../../contracts/vocabulary.js';
|
|
3
4
|
import { runProcess } from '../process-execution.js';
|
|
4
5
|
export function createClaudeRunner(options = {}) {
|
|
@@ -6,6 +7,7 @@ export function createClaudeRunner(options = {}) {
|
|
|
6
7
|
...(options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs }),
|
|
7
8
|
...(options.model === undefined ? {} : { defaultModel: options.model }),
|
|
8
9
|
parseSuccessfulOutput: parseClaudeOutput,
|
|
10
|
+
classifyFailure: classifyClaudeFailure,
|
|
9
11
|
supportsSessionResume: true,
|
|
10
12
|
});
|
|
11
13
|
}
|
|
@@ -19,6 +21,20 @@ export function parseClaudeOutput(stdout, _request) {
|
|
|
19
21
|
...claudeUsage(value),
|
|
20
22
|
};
|
|
21
23
|
}
|
|
24
|
+
// Deliberately narrow: only genuine provider usage/rate-limit phrasing.
|
|
25
|
+
// Auth/login failures (unauthorized, authentication, permission denied, api
|
|
26
|
+
// key) are NOT included here — they are a different failure class and must
|
|
27
|
+
// not be paused-and-retried as if they were transient.
|
|
28
|
+
const claudeQuotaPattern = /rate limit|quota|credit balance|spend limit|usage limit|session limit|too many requests|\b429\b/i;
|
|
29
|
+
export function classifyClaudeFailure(input) {
|
|
30
|
+
// Prefer stderr — a CLI's own diagnostic stream — over stdout, which on a
|
|
31
|
+
// failed run can carry the agent's own generated text; only fall back to
|
|
32
|
+
// stdout when stderr is empty.
|
|
33
|
+
const text = input.stderr.trim().length > 0 ? input.stderr : input.stdout;
|
|
34
|
+
if (!claudeQuotaPattern.test(text))
|
|
35
|
+
return undefined;
|
|
36
|
+
return { kind: ProviderQuotaExceededFailureKind, message: text.trim() };
|
|
37
|
+
}
|
|
22
38
|
function claudeUsage(value) {
|
|
23
39
|
const usage = record(value.usage);
|
|
24
40
|
const input = usage === undefined ? undefined : numeric(usage.input_tokens);
|
|
@@ -98,20 +114,29 @@ export function cliRunner(name, command, args, options = {}) {
|
|
|
98
114
|
transport: RunStatus.Failed,
|
|
99
115
|
output: value.stdout,
|
|
100
116
|
runner: name,
|
|
101
|
-
failure:
|
|
102
|
-
kind: value.timedOut
|
|
103
|
-
? ExecutionCancellationReason.Timeout
|
|
104
|
-
: (value.failureKind ?? 'process-exit'),
|
|
105
|
-
message: value.timedOut
|
|
106
|
-
? `Runner timed out after ${options.timeoutMs}ms`
|
|
107
|
-
: (value.failureMessage ?? (value.stderr || `exit ${value.exitCode}`)),
|
|
108
|
-
},
|
|
117
|
+
failure: failureFor(value, options),
|
|
109
118
|
}),
|
|
110
119
|
cancel: process.cancel,
|
|
111
120
|
};
|
|
112
121
|
},
|
|
113
122
|
};
|
|
114
123
|
}
|
|
124
|
+
function failureFor(value, options) {
|
|
125
|
+
if (value.timedOut)
|
|
126
|
+
return {
|
|
127
|
+
kind: ExecutionCancellationReason.Timeout,
|
|
128
|
+
message: `Runner timed out after ${options.timeoutMs}ms`,
|
|
129
|
+
};
|
|
130
|
+
if (value.failureKind !== undefined)
|
|
131
|
+
return {
|
|
132
|
+
kind: value.failureKind,
|
|
133
|
+
message: value.failureMessage ?? (value.stderr || `exit ${value.exitCode}`),
|
|
134
|
+
};
|
|
135
|
+
const classified = options.classifyFailure?.({ stdout: value.stdout, stderr: value.stderr });
|
|
136
|
+
if (classified !== undefined)
|
|
137
|
+
return classified;
|
|
138
|
+
return { kind: 'process-exit', message: value.stderr || `exit ${value.exitCode}` };
|
|
139
|
+
}
|
|
115
140
|
function parseSuccessfulOutput(stdout, request, parser) {
|
|
116
141
|
if (parser === undefined)
|
|
117
142
|
return {};
|
|
@@ -1,10 +1,47 @@
|
|
|
1
|
+
import { ProviderQuotaExceededFailureKind } from '../../contracts/runner.js';
|
|
1
2
|
import { WorkspaceMode } from '../../contracts/vocabulary.js';
|
|
2
3
|
import { cliRunner } from './claude.js';
|
|
4
|
+
// Deliberately narrow: only genuine provider usage/rate-limit phrasing.
|
|
5
|
+
// Auth/login failures (unauthorized, authentication, api key, not logged
|
|
6
|
+
// in, login required) are NOT included here — they are a different failure
|
|
7
|
+
// class and must not be paused-and-retried as if they were transient.
|
|
8
|
+
const codexQuotaPattern = /usage limit|rate limit|quota|too many requests|credit balance|spend limit|session limit|\b429\b/i;
|
|
9
|
+
export function classifyCodexFailure(input) {
|
|
10
|
+
// Only ever classify from Codex's own structured error/turn.failed
|
|
11
|
+
// message, never from scanning raw stdout — stdout on a failed run can
|
|
12
|
+
// carry the agent's own generated text, which must never be
|
|
13
|
+
// pattern-matched into a false quota classification.
|
|
14
|
+
const structured = extractCodexErrorMessage(input.stdout);
|
|
15
|
+
if (structured === undefined)
|
|
16
|
+
return undefined;
|
|
17
|
+
if (!codexQuotaPattern.test(structured))
|
|
18
|
+
return undefined;
|
|
19
|
+
return { kind: ProviderQuotaExceededFailureKind, message: structured };
|
|
20
|
+
}
|
|
21
|
+
function extractCodexErrorMessage(stdout) {
|
|
22
|
+
for (const line of stdout.split(/\r?\n/)) {
|
|
23
|
+
const trimmed = line.trim();
|
|
24
|
+
if (trimmed.length === 0)
|
|
25
|
+
continue;
|
|
26
|
+
const event = parseLine(trimmed);
|
|
27
|
+
if (event === undefined)
|
|
28
|
+
continue;
|
|
29
|
+
if (event.type === 'error' && typeof event.message === 'string')
|
|
30
|
+
return event.message;
|
|
31
|
+
if (event.type === 'turn.failed') {
|
|
32
|
+
const error = asRecord(event.error);
|
|
33
|
+
if (typeof error?.message === 'string')
|
|
34
|
+
return error.message;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
return undefined;
|
|
38
|
+
}
|
|
3
39
|
export function createCodexRunner(options = {}) {
|
|
4
40
|
return cliRunner('codex', options.command ?? 'codex', (request) => codexCommandArgs(request, options.args, options), {
|
|
5
41
|
...(options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs }),
|
|
6
42
|
...(options.model === undefined ? {} : { defaultModel: options.model }),
|
|
7
43
|
parseSuccessfulOutput: parseCodexOutput,
|
|
44
|
+
classifyFailure: classifyCodexFailure,
|
|
8
45
|
supportsSessionResume: true,
|
|
9
46
|
});
|
|
10
47
|
}
|
|
@@ -1,12 +1,27 @@
|
|
|
1
|
+
import { ProviderQuotaExceededFailureKind } from '../../contracts/runner.js';
|
|
1
2
|
import { WorkspaceMode } from '../../contracts/vocabulary.js';
|
|
2
3
|
import { cliRunner } from './claude.js';
|
|
3
4
|
// Cursor's CLI subcommand, not the closed domain vocabulary word "agent".
|
|
4
5
|
const cursorCliSubcommand = String.fromCharCode(97, 103, 101, 110, 116);
|
|
6
|
+
// Deliberately narrow: only genuine provider usage/rate-limit phrasing.
|
|
7
|
+
// Auth/login failures (unauthorized, authentication, api key) are NOT
|
|
8
|
+
// included here — they are a different failure class and must not be
|
|
9
|
+
// paused-and-retried as if they were transient.
|
|
10
|
+
const cursorQuotaPattern = /rate limit|usage limit|quota|credit balance|spend limit|session limit|too many requests|\b429\b/i;
|
|
11
|
+
export function classifyCursorFailure(input) {
|
|
12
|
+
// Prefer stderr over stdout, same reasoning as Claude's classifier: stdout
|
|
13
|
+
// on a failed run can carry the agent's own generated text.
|
|
14
|
+
const text = input.stderr.trim().length > 0 ? input.stderr : input.stdout;
|
|
15
|
+
if (!cursorQuotaPattern.test(text))
|
|
16
|
+
return undefined;
|
|
17
|
+
return { kind: ProviderQuotaExceededFailureKind, message: text.trim() };
|
|
18
|
+
}
|
|
5
19
|
export function createCursorRunner(options = {}) {
|
|
6
20
|
return cliRunner('cursor', options.command ?? 'cursor', (request) => cursorCommandArgs(request, options.args, options), {
|
|
7
21
|
...(options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs }),
|
|
8
22
|
...(options.model === undefined ? {} : { defaultModel: options.model }),
|
|
9
23
|
parseSuccessfulOutput: parseCursorOutput,
|
|
24
|
+
classifyFailure: classifyCursorFailure,
|
|
10
25
|
supportsSessionResume: true,
|
|
11
26
|
});
|
|
12
27
|
}
|
|
@@ -1,3 +1,11 @@
|
|
|
1
|
+
export class NoEligibleRunnerError extends Error {
|
|
2
|
+
runnerPool;
|
|
3
|
+
constructor(runnerPool) {
|
|
4
|
+
super(`Execution runner pool ${runnerPool} has no eligible runner: all candidates are ineligible`);
|
|
5
|
+
this.runnerPool = runnerPool;
|
|
6
|
+
this.name = 'NoEligibleRunnerError';
|
|
7
|
+
}
|
|
8
|
+
}
|
|
1
9
|
export class RunnerRegistry {
|
|
2
10
|
runnerPools;
|
|
3
11
|
runners;
|
|
@@ -22,5 +30,5 @@ function selectEligibleCandidate(runnerPool, candidates, runners, ineligible) {
|
|
|
22
30
|
throw new Error(`Runner ${name} is not registered`);
|
|
23
31
|
return { name, runner };
|
|
24
32
|
}
|
|
25
|
-
throw new
|
|
33
|
+
throw new NoEligibleRunnerError(runnerPool);
|
|
26
34
|
}
|
|
@@ -3,7 +3,7 @@ import { OrchestrationEventType } from '../contracts/events.js';
|
|
|
3
3
|
import { commandName, workflowInstanceId, } from '../contracts/identifiers.js';
|
|
4
4
|
import { workflowInstanceStream } from '../contracts/streams.js';
|
|
5
5
|
import { WorkflowStatus } from '../contracts/vocabulary.js';
|
|
6
|
-
import { requestChangesResume as decideChangesResume, requestOperatorRetry as decideOperatorRetry, requestSupplementalActivity as decideSupplementalActivity, } from '../domain/interpreter.js';
|
|
6
|
+
import { requestChangesResume as decideChangesResume, requestOperatorRetry as decideOperatorRetry, requestSupplementalActivity as decideSupplementalActivity, requestRunnerQuotaRetry, } from '../domain/interpreter.js';
|
|
7
7
|
import { isAuthorisedActor } from '../domain/supplemental-policy.js';
|
|
8
8
|
import { appendWithIntentRecovery } from './durable-append.js';
|
|
9
9
|
import { acceptResourceTransition, matchResourceTransitions, } from './resource-transition-matching.js';
|
|
@@ -112,6 +112,29 @@ export class AdvanceWorkflow {
|
|
|
112
112
|
]);
|
|
113
113
|
return (await this.repository.load(id)).view;
|
|
114
114
|
}
|
|
115
|
+
async retryRunnerQuotaFailure(id, input, context) {
|
|
116
|
+
const loaded = await this.repository.load(id);
|
|
117
|
+
if (loaded.view === null)
|
|
118
|
+
return null;
|
|
119
|
+
// The retry re-requests the interrupted Activation, not a stage lookup, but
|
|
120
|
+
// the definition must still be resolvable for the retried Activation's own
|
|
121
|
+
// outcome to be routable later; an unavailable definition blocks here.
|
|
122
|
+
const definition = await this.workflows.definitionForOperation(loaded.view, loaded.sequence, context);
|
|
123
|
+
if (definition === null)
|
|
124
|
+
return (await this.repository.loadRequired(id)).view;
|
|
125
|
+
const decision = requestRunnerQuotaRetry(loaded.view, {
|
|
126
|
+
activationId: input.activationId,
|
|
127
|
+
runId: input.runId,
|
|
128
|
+
runnerName: input.runnerName,
|
|
129
|
+
message: input.message,
|
|
130
|
+
occurredAt: context.occurredAt,
|
|
131
|
+
causationId: context.commandId,
|
|
132
|
+
});
|
|
133
|
+
if (decision.kind === 'ignored')
|
|
134
|
+
return loaded.view;
|
|
135
|
+
await this.repository.append(id, loaded.sequence, decision.events);
|
|
136
|
+
return (await this.repository.load(id)).view;
|
|
137
|
+
}
|
|
115
138
|
async retryBlockedFailedStage(id, context) {
|
|
116
139
|
const loaded = await this.repository.load(id);
|
|
117
140
|
if (loaded.view === null)
|
|
@@ -67,6 +67,9 @@ export class OrchestrationService {
|
|
|
67
67
|
resolveExecutionFailure(workflowInstanceId, input, context) {
|
|
68
68
|
return this.transitionWatchChildren(context, () => this.advanceWorkflow.resolveExecutionFailure(workflowInstanceId, input, context));
|
|
69
69
|
}
|
|
70
|
+
retryRunnerQuotaFailure(workflowInstanceId, input, context) {
|
|
71
|
+
return this.transitionWatchChildren(context, () => this.advanceWorkflow.retryRunnerQuotaFailure(workflowInstanceId, input, context));
|
|
72
|
+
}
|
|
70
73
|
retryBlockedFailedStage(workflowInstanceId, context) {
|
|
71
74
|
return this.transitionWatchChildren(context, () => this.advanceWorkflow.retryBlockedFailedStage(workflowInstanceId, context));
|
|
72
75
|
}
|
|
@@ -232,6 +232,14 @@ const eventSchema = z.discriminatedUnion('eventType', [
|
|
|
232
232
|
reason: z.string().min(1),
|
|
233
233
|
})
|
|
234
234
|
.strict()),
|
|
235
|
+
workflowEnvelope(OrchestrationEventType.ActivityRetriedForRunnerQuota, z
|
|
236
|
+
.object({
|
|
237
|
+
activationId: brandedStringSchema(activationId),
|
|
238
|
+
runId: z.string().min(1),
|
|
239
|
+
runnerName: z.string().min(1),
|
|
240
|
+
message: z.string().min(1),
|
|
241
|
+
})
|
|
242
|
+
.strict()),
|
|
235
243
|
workflowEnvelope(OrchestrationEventType.ActivityWaiting, z
|
|
236
244
|
.object({
|
|
237
245
|
activationId: brandedStringSchema(activationId),
|
|
@@ -7,6 +7,7 @@ export const OrchestrationEventType = {
|
|
|
7
7
|
ActivityStarted: 'orchestration.activity-started',
|
|
8
8
|
ActivityOutcomeAccepted: 'orchestration.activity-outcome-accepted',
|
|
9
9
|
ActivityExecutionFailed: 'orchestration.activity-execution-failed',
|
|
10
|
+
ActivityRetriedForRunnerQuota: 'orchestration.activity-retried-for-runner-quota',
|
|
10
11
|
ActivityWaiting: 'orchestration.activity-waiting',
|
|
11
12
|
SignalWaitStarted: 'orchestration.signal-wait-started',
|
|
12
13
|
SignalAccepted: 'orchestration.signal-accepted',
|
|
@@ -11,6 +11,7 @@ export { startInstance } from './activation-policy.js';
|
|
|
11
11
|
export { acceptSignal, waitForSignal } from './signal-policy.js';
|
|
12
12
|
export { requestSupplementalActivity } from './supplemental-policy.js';
|
|
13
13
|
export { isChangesResumeEligible, isOperatorRetryEligible, requestChangesResume, requestOperatorRetry, selectOperatorRetryTarget, } from './operator-retry-policy.js';
|
|
14
|
+
export { isRunnerQuotaRetryEligible, requestRunnerQuotaRetry, } from './runner-quota-retry-policy.js';
|
|
14
15
|
export function acceptActivityOutcome(definition, state, input) {
|
|
15
16
|
if (!isPendingOutcome(state, input))
|
|
16
17
|
return { kind: 'ignored', reason: 'outcome is not for the pending activation' };
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { activationId as toActivationId } from '../../activities/index.js';
|
|
2
|
+
import { OrchestrationEventType, } from '../contracts/events.js';
|
|
3
|
+
import { activation, nextOrdinal, stateDraft } from './decision-events.js';
|
|
4
|
+
export function isRunnerQuotaRetryEligible(view, activationId) {
|
|
5
|
+
return (view.pendingActivation?.activationId === activationId &&
|
|
6
|
+
!view.acceptedOutcomes.includes(toActivationId(activationId)));
|
|
7
|
+
}
|
|
8
|
+
export function requestRunnerQuotaRetry(state, input) {
|
|
9
|
+
if (!isRunnerQuotaRetryEligible(state, input.activationId))
|
|
10
|
+
return {
|
|
11
|
+
kind: 'ignored',
|
|
12
|
+
reason: 'workflow has no matching pending activation for runner-quota retry',
|
|
13
|
+
};
|
|
14
|
+
// Re-request the interrupted Activation itself, not the current stage's
|
|
15
|
+
// configured activity: a supplemental or follow-on Activation runs a
|
|
16
|
+
// different activity and input, and substituting the stage's main one would
|
|
17
|
+
// drop that work and re-run something the runner never attempted.
|
|
18
|
+
const interrupted = state.pendingActivation;
|
|
19
|
+
// No RetryCounted here, deliberately: a quota condition is a runner-capacity
|
|
20
|
+
// fact, not a failed attempt, so it must never consume the route's
|
|
21
|
+
// configured `retry.max` budget.
|
|
22
|
+
const events = [
|
|
23
|
+
stateDraft(state, input, OrchestrationEventType.ActivityRetriedForRunnerQuota, {
|
|
24
|
+
activationId: toActivationId(input.activationId),
|
|
25
|
+
runId: input.runId,
|
|
26
|
+
runnerName: input.runnerName,
|
|
27
|
+
message: input.message,
|
|
28
|
+
}, 1),
|
|
29
|
+
stateDraft(state, input, OrchestrationEventType.ActivityRequested, activation(state.workflowInstanceId, nextOrdinal(state), interrupted.activity, interrupted.input, {
|
|
30
|
+
execution: interrupted.execution,
|
|
31
|
+
...(interrupted.stage === undefined ? {} : { stage: interrupted.stage }),
|
|
32
|
+
...(interrupted.followOnIndex === undefined
|
|
33
|
+
? {}
|
|
34
|
+
: { followOnIndex: interrupted.followOnIndex }),
|
|
35
|
+
...(interrupted.supplemental === undefined
|
|
36
|
+
? {}
|
|
37
|
+
: { supplemental: interrupted.supplemental }),
|
|
38
|
+
}), 2),
|
|
39
|
+
];
|
|
40
|
+
return { kind: 'append', events };
|
|
41
|
+
}
|
|
@@ -34,6 +34,7 @@ function isActivityFact(event) {
|
|
|
34
34
|
case OrchestrationEventType.ActivityStarted:
|
|
35
35
|
case OrchestrationEventType.ActivityOutcomeAccepted:
|
|
36
36
|
case OrchestrationEventType.ActivityExecutionFailed:
|
|
37
|
+
case OrchestrationEventType.ActivityRetriedForRunnerQuota:
|
|
37
38
|
case OrchestrationEventType.ActivityWaiting:
|
|
38
39
|
return true;
|
|
39
40
|
default:
|
|
@@ -61,6 +62,10 @@ function applyActivityFact(state, event) {
|
|
|
61
62
|
state.executionFailure = event.payload;
|
|
62
63
|
updateActivationStatus(state, event.payload.activationId, ActivityActivationStatus.Completed);
|
|
63
64
|
return;
|
|
65
|
+
case OrchestrationEventType.ActivityRetriedForRunnerQuota:
|
|
66
|
+
state.acceptedOutcomes.push(event.payload.activationId);
|
|
67
|
+
updateActivationStatus(state, event.payload.activationId, ActivityActivationStatus.Completed);
|
|
68
|
+
return;
|
|
64
69
|
case OrchestrationEventType.ActivityWaiting:
|
|
65
70
|
applyActivityWaiting(state, event);
|
|
66
71
|
return;
|