@atolis-hq/wake 0.3.82 → 0.3.84
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/activities/agent/agent-activity.js +7 -1
- package/dist/src/activities/agent/agent-result.js +5 -1
- package/dist/src/activities/contracts/vocabulary.js +1 -0
- package/dist/src/bootstrap/index.js +3 -3
- package/dist/src/bootstrap/integration-runtime.js +1 -1
- package/dist/src/bootstrap/persistence-composition.js +1 -0
- package/dist/src/bootstrap/surface-cli-applications.js +12 -8
- package/dist/src/bootstrap/version.js +1 -1
- package/dist/src/control-plane/application/advance-once-dispatch.js +55 -9
- package/dist/src/control-plane/application/advance-once.js +24 -6
- package/dist/src/control-plane/application/control-plane-service.js +23 -9
- package/dist/src/control-plane/contracts/config.js +6 -3
- package/dist/src/execution/application/execution-activity.js +3 -1
- package/dist/src/execution/contracts/runner.js +1 -0
- package/dist/src/execution/infrastructure/runners/claude.js +33 -8
- package/dist/src/execution/infrastructure/runners/codex.js +37 -0
- package/dist/src/execution/infrastructure/runners/cursor.js +15 -0
- package/dist/src/execution/infrastructure/runners/registry.js +9 -1
- package/dist/src/integrations/delivery/application/delivery-outcome-reactor.js +52 -10
- package/dist/src/kernel/contracts/journal-change-signal.js +1 -0
- package/dist/src/kernel/index.js +2 -0
- package/dist/src/kernel/infrastructure/journal-change-signal.js +30 -0
- package/dist/src/orchestration/application/advance-workflow.js +24 -1
- package/dist/src/orchestration/application/orchestration-service.js +3 -0
- package/dist/src/orchestration/contracts/event-decoder.js +8 -0
- package/dist/src/orchestration/contracts/events.js +1 -0
- package/dist/src/orchestration/domain/interpreter.js +1 -0
- package/dist/src/orchestration/domain/runner-quota-retry-policy.js +41 -0
- package/dist/src/orchestration/domain/workflow-instance-events.js +5 -0
- package/dist/src/persistence/filesystem/file-event-journal.js +6 -1
- package/dist/src/persistence/memory/in-memory-event-journal.js +9 -1
- package/package.json +1 -1
|
@@ -175,7 +175,13 @@ function agentOutcome(result) {
|
|
|
175
175
|
return {
|
|
176
176
|
kind: ActivityOutcomeKind.Failed,
|
|
177
177
|
data: {
|
|
178
|
-
|
|
178
|
+
// 'provider-quota-exceeded' is a literal, not execution's ProviderQuotaExceededFailureKind
|
|
179
|
+
// import: activities sits below execution in the module dependency order and must not
|
|
180
|
+
// import it. AgentRunnerPort's failure.kind is an open string per the existing runner
|
|
181
|
+
// contract convention; this is the one place that interprets it.
|
|
182
|
+
reason: result.failure?.kind === 'provider-quota-exceeded'
|
|
183
|
+
? ActivityFailureCode.RunnerQuotaExceeded
|
|
184
|
+
: ActivityFailureCode.RunnerFailed,
|
|
179
185
|
...(result.failure === undefined ? {} : { message: result.failure.message }),
|
|
180
186
|
},
|
|
181
187
|
};
|
|
@@ -14,7 +14,11 @@ const blockedReason = z
|
|
|
14
14
|
.strict();
|
|
15
15
|
const failedReason = z
|
|
16
16
|
.object({
|
|
17
|
-
reason: z.enum([
|
|
17
|
+
reason: z.enum([
|
|
18
|
+
ActivityFailureCode.InvalidAgentResult,
|
|
19
|
+
ActivityFailureCode.RunnerFailed,
|
|
20
|
+
ActivityFailureCode.RunnerQuotaExceeded,
|
|
21
|
+
]),
|
|
18
22
|
message: z.string().optional(),
|
|
19
23
|
})
|
|
20
24
|
.strict();
|
|
@@ -31,6 +31,7 @@ export const ActivityFailureCode = defineClosedVocabulary({
|
|
|
31
31
|
InvalidAgentResult: 'invalid-agent-result',
|
|
32
32
|
AmbiguousRunnerResult: 'ambiguous-runner-result',
|
|
33
33
|
RunnerFailed: 'runner-failed',
|
|
34
|
+
RunnerQuotaExceeded: 'runner-quota-exceeded',
|
|
34
35
|
});
|
|
35
36
|
export const ActivityOutcomeKind = defineClosedVocabulary({
|
|
36
37
|
Waiting: 'waiting',
|
|
@@ -4,8 +4,8 @@ import { createRuntimeProjectionRunner } from './projection-runtime.js';
|
|
|
4
4
|
export function composeDeliveryService(dependencies) {
|
|
5
5
|
return new DeliveryService(dependencies);
|
|
6
6
|
}
|
|
7
|
-
export function composeDeliveryOutcomeReactor(journal, checkpoints, orchestration) {
|
|
8
|
-
return new DeliveryOutcomeReactor(journal, checkpoints, orchestration);
|
|
7
|
+
export function composeDeliveryOutcomeReactor(journal, checkpoints, orchestration, projections) {
|
|
8
|
+
return new DeliveryOutcomeReactor(journal, checkpoints, orchestration, projections);
|
|
9
9
|
}
|
|
10
10
|
export function composeDeliveryRuntime(dependencies) {
|
|
11
11
|
const projectionRunner = createRuntimeProjectionRunner(dependencies.journal, dependencies.projections, dependencies.checkpoints);
|
|
@@ -16,7 +16,7 @@ export function composeDeliveryRuntime(dependencies) {
|
|
|
16
16
|
adapter: dependencies.adapter,
|
|
17
17
|
now: dependencies.now,
|
|
18
18
|
});
|
|
19
|
-
const reactor = composeDeliveryOutcomeReactor(dependencies.journal, dependencies.checkpoints, dependencies.orchestration);
|
|
19
|
+
const reactor = composeDeliveryOutcomeReactor(dependencies.journal, dependencies.checkpoints, dependencies.orchestration, dependencies.projections);
|
|
20
20
|
return {
|
|
21
21
|
async runOnce(signal) {
|
|
22
22
|
await projectionRunner.runRegisteredOnce();
|
|
@@ -120,7 +120,7 @@ export async function composeIntegrationRuntime(input) {
|
|
|
120
120
|
await resourceTransitions.drain();
|
|
121
121
|
return operation();
|
|
122
122
|
});
|
|
123
|
-
const outcomes = new DeliveryOutcomeReactor(input.journal, input.checkpoints, input.orchestration);
|
|
123
|
+
const outcomes = new DeliveryOutcomeReactor(input.journal, input.checkpoints, input.orchestration, input.projections);
|
|
124
124
|
const catchUpProjections = async () => {
|
|
125
125
|
await projectionRunner.runRegisteredOnce();
|
|
126
126
|
};
|
|
@@ -13,6 +13,7 @@ function serializeJournalAppends(journal) {
|
|
|
13
13
|
readStream: (stream) => journal.readStream(stream),
|
|
14
14
|
readAll: (afterGlobalPosition, limit) => journal.readAll(afterGlobalPosition, limit),
|
|
15
15
|
latestGlobalPosition: () => journal.latestGlobalPosition(),
|
|
16
|
+
changeSignal: journal.changeSignal,
|
|
16
17
|
...(journal.readLatest === undefined
|
|
17
18
|
? {}
|
|
18
19
|
: {
|
|
@@ -39,12 +39,13 @@ export function createSurfaceCliApplications(root, api, now) {
|
|
|
39
39
|
// hammered every cycle instead of given a chance to recover.
|
|
40
40
|
const runnerResident = new ResidentHost(runnerTick, (signal, { consecutiveIdleTicks, consecutiveErrorTicks }) => {
|
|
41
41
|
if (consecutiveErrorTicks > 0)
|
|
42
|
-
return sleepUntilAbort(signal,
|
|
42
|
+
return sleepUntilAbort(signal, nextPollBackoffMs(root.config.controlPlane.resident, consecutiveErrorTicks));
|
|
43
|
+
// Genuine idle (no errors): wait for the journal to actually change.
|
|
43
44
|
return consecutiveIdleTicks === 0
|
|
44
45
|
? Promise.resolve()
|
|
45
|
-
:
|
|
46
|
+
: root.journal.changeSignal.waitForChange(signal, JOURNAL_WAIT_FALLBACK_MS);
|
|
46
47
|
}, reportResidentError('runner'));
|
|
47
|
-
const intakeResident = new ResidentHost(intakeHost, (signal, { consecutiveIdleTicks }) => sleepUntilAbort(signal,
|
|
48
|
+
const intakeResident = new ResidentHost(intakeHost, (signal, { consecutiveIdleTicks }) => sleepUntilAbort(signal, nextPollBackoffMs(root.config.controlPlane.resident, consecutiveIdleTicks)), reportResidentError('intake'));
|
|
48
49
|
const servers = new Set();
|
|
49
50
|
const startHttp = createHttpStarter(root, api, servers);
|
|
50
51
|
return {
|
|
@@ -134,8 +135,11 @@ export function createSurfaceCliApplications(root, api, now) {
|
|
|
134
135
|
operational: createOperationalApplications(root),
|
|
135
136
|
};
|
|
136
137
|
}
|
|
138
|
+
// Safety net for a missed in-process notify() or a cross-process writer the
|
|
139
|
+
// EventEmitter can't see; a real append wakes waiters within milliseconds
|
|
140
|
+
// regardless, so there's no operational reason to make this configurable.
|
|
141
|
+
const JOURNAL_WAIT_FALLBACK_MS = 30_000;
|
|
137
142
|
export async function runProjectionPump(root, signal) {
|
|
138
|
-
const intervalMs = 1000;
|
|
139
143
|
while (!signal.aborted) {
|
|
140
144
|
try {
|
|
141
145
|
await root.projectionRunner.runRegisteredOnce();
|
|
@@ -143,12 +147,12 @@ export async function runProjectionPump(root, signal) {
|
|
|
143
147
|
catch (error) {
|
|
144
148
|
process.stderr.write(`Wake projection pump failed: ${error instanceof Error ? error.message : String(error)}\n`);
|
|
145
149
|
}
|
|
146
|
-
await
|
|
150
|
+
await root.journal.changeSignal.waitForChange(signal, JOURNAL_WAIT_FALLBACK_MS);
|
|
147
151
|
}
|
|
148
152
|
}
|
|
149
|
-
function
|
|
150
|
-
const baseMs = resident?.
|
|
151
|
-
const maxMs = resident?.
|
|
153
|
+
function nextPollBackoffMs(resident, consecutiveIdleTicks) {
|
|
154
|
+
const baseMs = resident?.pollBackoffMs ?? 1000;
|
|
155
|
+
const maxMs = resident?.maxPollBackoffMs ?? baseMs * 16;
|
|
152
156
|
return Math.min(baseMs * 2 ** Math.min(consecutiveIdleTicks, 20), maxMs);
|
|
153
157
|
}
|
|
154
158
|
function sleepUntilAbort(signal, milliseconds) {
|
|
@@ -1,6 +1,30 @@
|
|
|
1
|
-
|
|
1
|
+
/* eslint-disable complexity, max-lines-per-function */
|
|
2
|
+
import { ActivityFailureCode, ActivityOutcomeKind, } from '../../activities/index.js';
|
|
3
|
+
import { ActivationClaimConflictError, NoEligibleRunnerError, RunStatus, WorkspaceMode, } from '../../execution/index.js';
|
|
2
4
|
import { WorkflowStatus } from '../../orchestration/index.js';
|
|
3
5
|
import { isExecutionFailureTerminal } from './execution-reconciliation.js';
|
|
6
|
+
export function isRunnerQuotaOutcome(outcome) {
|
|
7
|
+
return (outcome.kind === ActivityOutcomeKind.Failed &&
|
|
8
|
+
typeof outcome.data === 'object' &&
|
|
9
|
+
outcome.data !== null &&
|
|
10
|
+
'reason' in outcome.data &&
|
|
11
|
+
outcome.data.reason === ActivityFailureCode.RunnerQuotaExceeded);
|
|
12
|
+
}
|
|
13
|
+
export function runnerQuotaMessage(outcome) {
|
|
14
|
+
const data = outcome.data;
|
|
15
|
+
return typeof data.message === 'string' ? data.message : 'runner reported quota exhaustion';
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* A quota retry deliberately leaves the outcome unaccepted, so a port that
|
|
19
|
+
* never applied it (absent method, or a workflow instance that vanished)
|
|
20
|
+
* leaves the Run permanently unresolved. Surface that rather than going
|
|
21
|
+
* silent; it is an operational defect, not a workflow state.
|
|
22
|
+
*/
|
|
23
|
+
export function reportUnappliedRunnerQuotaRetry(result, input) {
|
|
24
|
+
if (result !== null && result !== undefined)
|
|
25
|
+
return;
|
|
26
|
+
console.error(`Runner-quota retry was not applied for activation ${input.activationId} (run ${input.runId}); its outcome remains unaccepted.`);
|
|
27
|
+
}
|
|
4
28
|
/**
|
|
5
29
|
* Fills open capacity within one Advancement call: dispatches ready
|
|
6
30
|
* activations one at a time, rechecking `maxConcurrentRuns` and reselecting
|
|
@@ -73,18 +97,40 @@ export async function runDispatchLoop(pending, ctx) {
|
|
|
73
97
|
stopReason = { kind: 'no-work' };
|
|
74
98
|
break;
|
|
75
99
|
}
|
|
100
|
+
if (error instanceof NoEligibleRunnerError) {
|
|
101
|
+
stopReason = { kind: 'no-work' };
|
|
102
|
+
break;
|
|
103
|
+
}
|
|
76
104
|
throw error;
|
|
77
105
|
}
|
|
78
106
|
if (run.status === RunStatus.Succeeded && run.outcome !== undefined) {
|
|
79
|
-
if (
|
|
80
|
-
|
|
81
|
-
|
|
107
|
+
if (isRunnerQuotaOutcome(run.outcome)) {
|
|
108
|
+
// No isDispatchPaused check here: a quota retry re-requests the same
|
|
109
|
+
// stage's activity without publishing an outcome or consuming retry
|
|
110
|
+
// budget, so it must not be held behind maintenance pause the way an
|
|
111
|
+
// outward-publishing acceptOutcome call is below.
|
|
112
|
+
const retried = await ctx.orchestration.retryRunnerQuotaFailure?.(selected.workflow.workflowInstanceId, {
|
|
113
|
+
activationId: selected.activation.activationId,
|
|
114
|
+
runId: run.runId,
|
|
115
|
+
runnerName: run.runner?.name ?? 'unknown-runner',
|
|
116
|
+
message: runnerQuotaMessage(run.outcome),
|
|
117
|
+
}, ctx.commandContext(run.runId));
|
|
118
|
+
reportUnappliedRunnerQuotaRetry(retried, {
|
|
119
|
+
activationId: selected.activation.activationId,
|
|
120
|
+
runId: run.runId,
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
else {
|
|
124
|
+
if (await ctx.isDispatchPaused()) {
|
|
125
|
+
stopReason = { kind: 'paused' };
|
|
126
|
+
break;
|
|
127
|
+
}
|
|
128
|
+
await ctx.orchestration.acceptOutcome({
|
|
129
|
+
workflowInstanceId: selected.workflow.workflowInstanceId,
|
|
130
|
+
activationId: selected.activation.activationId,
|
|
131
|
+
outcome: run.outcome,
|
|
132
|
+
}, ctx.commandContext(run.runId));
|
|
82
133
|
}
|
|
83
|
-
await ctx.orchestration.acceptOutcome({
|
|
84
|
-
workflowInstanceId: selected.workflow.workflowInstanceId,
|
|
85
|
-
activationId: selected.activation.activationId,
|
|
86
|
-
outcome: run.outcome,
|
|
87
|
-
}, ctx.commandContext(run.runId));
|
|
88
134
|
}
|
|
89
135
|
if (isExecutionFailureTerminal(run.status))
|
|
90
136
|
await ctx.orchestration.resolveExecutionFailure?.(selected.workflow.workflowInstanceId, {
|
|
@@ -5,7 +5,7 @@ import { isAmbiguityResolutionBlock, WorkflowStatus } from '../../orchestration/
|
|
|
5
5
|
import { WorkStatus } from '../../work/index.js';
|
|
6
6
|
import { ControlStreamKind } from '../contracts/streams.js';
|
|
7
7
|
import { DispatchPolicy } from '../domain/dispatch-policy.js';
|
|
8
|
-
import { runDispatchLoop } from './advance-once-dispatch.js';
|
|
8
|
+
import { isRunnerQuotaOutcome, reportUnappliedRunnerQuotaRetry, runDispatchLoop, runnerQuotaMessage, } from './advance-once-dispatch.js';
|
|
9
9
|
import { findUnresolvedSucceededTerminal, findUnresolvedTerminal, } from './execution-reconciliation.js';
|
|
10
10
|
export function createAdvanceOnce(orchestration, execution, resources, clock, dependencies) {
|
|
11
11
|
const runnerIneligibility = dependencies.runnerIneligibility ?? (async () => new Set());
|
|
@@ -96,11 +96,29 @@ export function createAdvanceOnce(orchestration, execution, resources, clock, de
|
|
|
96
96
|
if (await isDispatchPaused())
|
|
97
97
|
return { kind: 'paused' };
|
|
98
98
|
if (recovery.run.status === RunStatus.Succeeded) {
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
99
|
+
// An agent Run completes after execution.attempt() has already returned,
|
|
100
|
+
// so this reconciliation — not the dispatch loop — is where a real
|
|
101
|
+
// quota-classified outcome is resolved. It must divert here too, or the
|
|
102
|
+
// quota failure is published and charged to the retry budget after all.
|
|
103
|
+
if (isRunnerQuotaOutcome(recovery.run.outcome)) {
|
|
104
|
+
const retried = await orchestration.retryRunnerQuotaFailure?.(recovery.item.workflow.workflowInstanceId, {
|
|
105
|
+
activationId: recovery.item.activation.activationId,
|
|
106
|
+
runId: recovery.run.runId,
|
|
107
|
+
runnerName: recovery.run.runner?.name ?? 'unknown-runner',
|
|
108
|
+
message: runnerQuotaMessage(recovery.run.outcome),
|
|
109
|
+
}, context(recovery.run.runId));
|
|
110
|
+
reportUnappliedRunnerQuotaRetry(retried, {
|
|
111
|
+
activationId: recovery.item.activation.activationId,
|
|
112
|
+
runId: recovery.run.runId,
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
else {
|
|
116
|
+
await orchestration.acceptOutcome({
|
|
117
|
+
workflowInstanceId: recovery.item.workflow.workflowInstanceId,
|
|
118
|
+
activationId: recovery.item.activation.activationId,
|
|
119
|
+
outcome: recovery.run.outcome,
|
|
120
|
+
}, context(recovery.run.runId));
|
|
121
|
+
}
|
|
104
122
|
return {
|
|
105
123
|
kind: 'progressed',
|
|
106
124
|
dispatched: [
|
|
@@ -2,22 +2,36 @@ import { EventActorKind, correlationId, } from '../../kernel/index.js';
|
|
|
2
2
|
import { ControlEventType, createControlEventDraft, selectControlEvent, } from '../contracts/events.js';
|
|
3
3
|
import { controlPlaneStream } from '../contracts/streams.js';
|
|
4
4
|
export function createControlPlaneService(input) {
|
|
5
|
+
// isPaused() is checked many times per pipeline run, so memoize by
|
|
6
|
+
// journal position to skip the read entirely when nothing has moved.
|
|
7
|
+
let cached;
|
|
5
8
|
return {
|
|
6
9
|
pause: (key) => change(input, key, 'pause'),
|
|
7
10
|
resume: (key) => change(input, key, 'resume'),
|
|
8
11
|
async isPaused() {
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
if (event?.eventType === ControlEventType.DispatchResumed)
|
|
15
|
-
paused = false;
|
|
16
|
-
}
|
|
12
|
+
const position = await input.journal.latestGlobalPosition();
|
|
13
|
+
if (cached !== undefined && cached.position === position)
|
|
14
|
+
return cached.paused;
|
|
15
|
+
const paused = await currentIsPaused(input.journal);
|
|
16
|
+
cached = { position, paused };
|
|
17
17
|
return paused;
|
|
18
18
|
},
|
|
19
19
|
};
|
|
20
20
|
}
|
|
21
|
+
async function currentIsPaused(journal) {
|
|
22
|
+
return isPausedIn(await journal.readStream(controlPlaneStream()));
|
|
23
|
+
}
|
|
24
|
+
function isPausedIn(events) {
|
|
25
|
+
let paused = false;
|
|
26
|
+
for (const envelope of events) {
|
|
27
|
+
const event = selectControlEvent(envelope);
|
|
28
|
+
if (event?.eventType === ControlEventType.DispatchPaused)
|
|
29
|
+
paused = true;
|
|
30
|
+
if (event?.eventType === ControlEventType.DispatchResumed)
|
|
31
|
+
paused = false;
|
|
32
|
+
}
|
|
33
|
+
return paused;
|
|
34
|
+
}
|
|
21
35
|
async function change(input, idempotencyKey, operation) {
|
|
22
36
|
const stream = controlPlaneStream();
|
|
23
37
|
const events = await input.journal.readStream(stream);
|
|
@@ -25,7 +39,7 @@ async function change(input, idempotencyKey, operation) {
|
|
|
25
39
|
const correlation = correlationId(`control:${operation}:${idempotencyKey}`);
|
|
26
40
|
if (events.some((event) => event.eventType === eventType && event.correlationId === correlation))
|
|
27
41
|
return;
|
|
28
|
-
const currentlyPaused =
|
|
42
|
+
const currentlyPaused = isPausedIn(events);
|
|
29
43
|
if ((operation === 'pause' && currentlyPaused) || (operation === 'resume' && !currentlyPaused))
|
|
30
44
|
return;
|
|
31
45
|
const occurredAt = input.clock.now().toISOString();
|
|
@@ -14,10 +14,13 @@ export const controlPlaneConfigSchema = z
|
|
|
14
14
|
schedules: z.array(scheduleSchema).default([]),
|
|
15
15
|
resident: z
|
|
16
16
|
.object({
|
|
17
|
-
|
|
18
|
-
|
|
17
|
+
// Backoff for the resident loop's own retry cadence when idle or
|
|
18
|
+
// erroring, not a per-adapter rate limit (e.g.
|
|
19
|
+
// integrations.github.polling.intervalMs gates the actual call).
|
|
20
|
+
pollBackoffMs: z.number().int().positive().default(1000),
|
|
21
|
+
maxPollBackoffMs: z.number().int().positive().optional(),
|
|
19
22
|
})
|
|
20
23
|
.strict()
|
|
21
|
-
.default({
|
|
24
|
+
.default({ pollBackoffMs: 1000 }),
|
|
22
25
|
})
|
|
23
26
|
.strict();
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { ExecutionEventType } from '../contracts/events.js';
|
|
2
|
+
import { ProviderQuotaExceededFailureKind } from '../contracts/runner.js';
|
|
2
3
|
import { parseAgentRunnerResponse } from '../infrastructure/agent-runner-adapter.js';
|
|
3
4
|
import { createRunEvent } from './run-lifecycle.js';
|
|
4
5
|
export async function executeActivity(runtime, currentRunId, request) {
|
|
@@ -77,7 +78,8 @@ function runnerResultReporter(runtime, currentRunId, context, activation, reques
|
|
|
77
78
|
payload: { transport: result.transport, agent: parseAgentRunnerResponse(result) },
|
|
78
79
|
});
|
|
79
80
|
await appendIdempotently(runtime.repository, currentRunId, loaded.sequence, event);
|
|
80
|
-
if (result.failure?.kind ===
|
|
81
|
+
if (result.failure?.kind === ProviderQuotaExceededFailureKind &&
|
|
82
|
+
request.runnerName !== undefined)
|
|
81
83
|
await runtime.dependencies.reportRunnerQuota?.({
|
|
82
84
|
runnerName: request.runnerName,
|
|
83
85
|
message: result.failure.message,
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
export const ProviderQuotaExceededFailureKind = 'provider-quota-exceeded';
|
|
1
2
|
// inputTokens/outputTokens/cacheReadTokens/cacheWriteTokens/costUsd are the known
|
|
2
3
|
// numeric keys agent-runner-adapter.ts writes into AgentRunResponse.metadata.
|
|
3
4
|
export function agentTokenUsage(metadata) {
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { ExternalExecutionKind } from '../../../activities/index.js';
|
|
2
|
+
import { ProviderQuotaExceededFailureKind } from '../../contracts/runner.js';
|
|
2
3
|
import { ExecutionCancellationReason, RunStatus } from '../../contracts/vocabulary.js';
|
|
3
4
|
import { runProcess } from '../process-execution.js';
|
|
4
5
|
export function createClaudeRunner(options = {}) {
|
|
@@ -6,6 +7,7 @@ export function createClaudeRunner(options = {}) {
|
|
|
6
7
|
...(options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs }),
|
|
7
8
|
...(options.model === undefined ? {} : { defaultModel: options.model }),
|
|
8
9
|
parseSuccessfulOutput: parseClaudeOutput,
|
|
10
|
+
classifyFailure: classifyClaudeFailure,
|
|
9
11
|
supportsSessionResume: true,
|
|
10
12
|
});
|
|
11
13
|
}
|
|
@@ -19,6 +21,20 @@ export function parseClaudeOutput(stdout, _request) {
|
|
|
19
21
|
...claudeUsage(value),
|
|
20
22
|
};
|
|
21
23
|
}
|
|
24
|
+
// Deliberately narrow: only genuine provider usage/rate-limit phrasing.
|
|
25
|
+
// Auth/login failures (unauthorized, authentication, permission denied, api
|
|
26
|
+
// key) are NOT included here — they are a different failure class and must
|
|
27
|
+
// not be paused-and-retried as if they were transient.
|
|
28
|
+
const claudeQuotaPattern = /rate limit|quota|credit balance|spend limit|usage limit|session limit|too many requests|\b429\b/i;
|
|
29
|
+
export function classifyClaudeFailure(input) {
|
|
30
|
+
// Prefer stderr — a CLI's own diagnostic stream — over stdout, which on a
|
|
31
|
+
// failed run can carry the agent's own generated text; only fall back to
|
|
32
|
+
// stdout when stderr is empty.
|
|
33
|
+
const text = input.stderr.trim().length > 0 ? input.stderr : input.stdout;
|
|
34
|
+
if (!claudeQuotaPattern.test(text))
|
|
35
|
+
return undefined;
|
|
36
|
+
return { kind: ProviderQuotaExceededFailureKind, message: text.trim() };
|
|
37
|
+
}
|
|
22
38
|
function claudeUsage(value) {
|
|
23
39
|
const usage = record(value.usage);
|
|
24
40
|
const input = usage === undefined ? undefined : numeric(usage.input_tokens);
|
|
@@ -98,20 +114,29 @@ export function cliRunner(name, command, args, options = {}) {
|
|
|
98
114
|
transport: RunStatus.Failed,
|
|
99
115
|
output: value.stdout,
|
|
100
116
|
runner: name,
|
|
101
|
-
failure:
|
|
102
|
-
kind: value.timedOut
|
|
103
|
-
? ExecutionCancellationReason.Timeout
|
|
104
|
-
: (value.failureKind ?? 'process-exit'),
|
|
105
|
-
message: value.timedOut
|
|
106
|
-
? `Runner timed out after ${options.timeoutMs}ms`
|
|
107
|
-
: (value.failureMessage ?? (value.stderr || `exit ${value.exitCode}`)),
|
|
108
|
-
},
|
|
117
|
+
failure: failureFor(value, options),
|
|
109
118
|
}),
|
|
110
119
|
cancel: process.cancel,
|
|
111
120
|
};
|
|
112
121
|
},
|
|
113
122
|
};
|
|
114
123
|
}
|
|
124
|
+
function failureFor(value, options) {
|
|
125
|
+
if (value.timedOut)
|
|
126
|
+
return {
|
|
127
|
+
kind: ExecutionCancellationReason.Timeout,
|
|
128
|
+
message: `Runner timed out after ${options.timeoutMs}ms`,
|
|
129
|
+
};
|
|
130
|
+
if (value.failureKind !== undefined)
|
|
131
|
+
return {
|
|
132
|
+
kind: value.failureKind,
|
|
133
|
+
message: value.failureMessage ?? (value.stderr || `exit ${value.exitCode}`),
|
|
134
|
+
};
|
|
135
|
+
const classified = options.classifyFailure?.({ stdout: value.stdout, stderr: value.stderr });
|
|
136
|
+
if (classified !== undefined)
|
|
137
|
+
return classified;
|
|
138
|
+
return { kind: 'process-exit', message: value.stderr || `exit ${value.exitCode}` };
|
|
139
|
+
}
|
|
115
140
|
function parseSuccessfulOutput(stdout, request, parser) {
|
|
116
141
|
if (parser === undefined)
|
|
117
142
|
return {};
|
|
@@ -1,10 +1,47 @@
|
|
|
1
|
+
import { ProviderQuotaExceededFailureKind } from '../../contracts/runner.js';
|
|
1
2
|
import { WorkspaceMode } from '../../contracts/vocabulary.js';
|
|
2
3
|
import { cliRunner } from './claude.js';
|
|
4
|
+
// Deliberately narrow: only genuine provider usage/rate-limit phrasing.
|
|
5
|
+
// Auth/login failures (unauthorized, authentication, api key, not logged
|
|
6
|
+
// in, login required) are NOT included here — they are a different failure
|
|
7
|
+
// class and must not be paused-and-retried as if they were transient.
|
|
8
|
+
const codexQuotaPattern = /usage limit|rate limit|quota|too many requests|credit balance|spend limit|session limit|\b429\b/i;
|
|
9
|
+
export function classifyCodexFailure(input) {
|
|
10
|
+
// Only ever classify from Codex's own structured error/turn.failed
|
|
11
|
+
// message, never from scanning raw stdout — stdout on a failed run can
|
|
12
|
+
// carry the agent's own generated text, which must never be
|
|
13
|
+
// pattern-matched into a false quota classification.
|
|
14
|
+
const structured = extractCodexErrorMessage(input.stdout);
|
|
15
|
+
if (structured === undefined)
|
|
16
|
+
return undefined;
|
|
17
|
+
if (!codexQuotaPattern.test(structured))
|
|
18
|
+
return undefined;
|
|
19
|
+
return { kind: ProviderQuotaExceededFailureKind, message: structured };
|
|
20
|
+
}
|
|
21
|
+
function extractCodexErrorMessage(stdout) {
|
|
22
|
+
for (const line of stdout.split(/\r?\n/)) {
|
|
23
|
+
const trimmed = line.trim();
|
|
24
|
+
if (trimmed.length === 0)
|
|
25
|
+
continue;
|
|
26
|
+
const event = parseLine(trimmed);
|
|
27
|
+
if (event === undefined)
|
|
28
|
+
continue;
|
|
29
|
+
if (event.type === 'error' && typeof event.message === 'string')
|
|
30
|
+
return event.message;
|
|
31
|
+
if (event.type === 'turn.failed') {
|
|
32
|
+
const error = asRecord(event.error);
|
|
33
|
+
if (typeof error?.message === 'string')
|
|
34
|
+
return error.message;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
return undefined;
|
|
38
|
+
}
|
|
3
39
|
export function createCodexRunner(options = {}) {
|
|
4
40
|
return cliRunner('codex', options.command ?? 'codex', (request) => codexCommandArgs(request, options.args, options), {
|
|
5
41
|
...(options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs }),
|
|
6
42
|
...(options.model === undefined ? {} : { defaultModel: options.model }),
|
|
7
43
|
parseSuccessfulOutput: parseCodexOutput,
|
|
44
|
+
classifyFailure: classifyCodexFailure,
|
|
8
45
|
supportsSessionResume: true,
|
|
9
46
|
});
|
|
10
47
|
}
|
|
@@ -1,12 +1,27 @@
|
|
|
1
|
+
import { ProviderQuotaExceededFailureKind } from '../../contracts/runner.js';
|
|
1
2
|
import { WorkspaceMode } from '../../contracts/vocabulary.js';
|
|
2
3
|
import { cliRunner } from './claude.js';
|
|
3
4
|
// Cursor's CLI subcommand, not the closed domain vocabulary word "agent".
|
|
4
5
|
const cursorCliSubcommand = String.fromCharCode(97, 103, 101, 110, 116);
|
|
6
|
+
// Deliberately narrow: only genuine provider usage/rate-limit phrasing.
|
|
7
|
+
// Auth/login failures (unauthorized, authentication, api key) are NOT
|
|
8
|
+
// included here — they are a different failure class and must not be
|
|
9
|
+
// paused-and-retried as if they were transient.
|
|
10
|
+
const cursorQuotaPattern = /rate limit|usage limit|quota|credit balance|spend limit|session limit|too many requests|\b429\b/i;
|
|
11
|
+
export function classifyCursorFailure(input) {
|
|
12
|
+
// Prefer stderr over stdout, same reasoning as Claude's classifier: stdout
|
|
13
|
+
// on a failed run can carry the agent's own generated text.
|
|
14
|
+
const text = input.stderr.trim().length > 0 ? input.stderr : input.stdout;
|
|
15
|
+
if (!cursorQuotaPattern.test(text))
|
|
16
|
+
return undefined;
|
|
17
|
+
return { kind: ProviderQuotaExceededFailureKind, message: text.trim() };
|
|
18
|
+
}
|
|
5
19
|
export function createCursorRunner(options = {}) {
|
|
6
20
|
return cliRunner('cursor', options.command ?? 'cursor', (request) => cursorCommandArgs(request, options.args, options), {
|
|
7
21
|
...(options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs }),
|
|
8
22
|
...(options.model === undefined ? {} : { defaultModel: options.model }),
|
|
9
23
|
parseSuccessfulOutput: parseCursorOutput,
|
|
24
|
+
classifyFailure: classifyCursorFailure,
|
|
10
25
|
supportsSessionResume: true,
|
|
11
26
|
});
|
|
12
27
|
}
|
|
@@ -1,3 +1,11 @@
|
|
|
1
|
+
export class NoEligibleRunnerError extends Error {
|
|
2
|
+
runnerPool;
|
|
3
|
+
constructor(runnerPool) {
|
|
4
|
+
super(`Execution runner pool ${runnerPool} has no eligible runner: all candidates are ineligible`);
|
|
5
|
+
this.runnerPool = runnerPool;
|
|
6
|
+
this.name = 'NoEligibleRunnerError';
|
|
7
|
+
}
|
|
8
|
+
}
|
|
1
9
|
export class RunnerRegistry {
|
|
2
10
|
runnerPools;
|
|
3
11
|
runners;
|
|
@@ -22,5 +30,5 @@ function selectEligibleCandidate(runnerPool, candidates, runners, ineligible) {
|
|
|
22
30
|
throw new Error(`Runner ${name} is not registered`);
|
|
23
31
|
return { name, runner };
|
|
24
32
|
}
|
|
25
|
-
throw new
|
|
33
|
+
throw new NoEligibleRunnerError(runnerPool);
|
|
26
34
|
}
|
|
@@ -1,34 +1,63 @@
|
|
|
1
1
|
import { ActivityOutcomeKind, activationId } from '../../../activities/index.js';
|
|
2
|
-
import { EventActorKind } from '../../../kernel/index.js';
|
|
2
|
+
import { EventActorKind, } from '../../../kernel/index.js';
|
|
3
3
|
import { ActivityActivationStatus, workflowInstanceId, } from '../../../orchestration/index.js';
|
|
4
4
|
import { DeliveryEventType, selectDeliveryEvent } from '../contracts/events.js';
|
|
5
5
|
import { DeliveryResultKind } from '../contracts/vocabulary.js';
|
|
6
6
|
const deliveryResultSignalKind = 'delivery-result';
|
|
7
|
+
const pendingNamespace = 'reactor:delivery-outcomes:pending';
|
|
8
|
+
const pendingKey = 'pending-confirmations';
|
|
7
9
|
export class DeliveryOutcomeReactor {
|
|
8
10
|
journal;
|
|
9
11
|
checkpoints;
|
|
10
12
|
orchestration;
|
|
11
|
-
|
|
13
|
+
projections;
|
|
14
|
+
constructor(journal, checkpoints, orchestration, projections) {
|
|
12
15
|
this.journal = journal;
|
|
13
16
|
this.checkpoints = checkpoints;
|
|
14
17
|
this.orchestration = orchestration;
|
|
18
|
+
this.projections = projections;
|
|
15
19
|
}
|
|
16
20
|
async runOnce() {
|
|
17
21
|
const consumer = 'reactor:delivery-outcomes';
|
|
18
22
|
const events = await this.journal.readAll(await this.checkpoints.load(consumer));
|
|
19
|
-
const
|
|
23
|
+
const resolved = new Set();
|
|
24
|
+
const pending = new Map((await this.loadPending()).map((event) => [event.eventId, event]));
|
|
20
25
|
for (const event of events) {
|
|
21
|
-
await this.reconcile(event,
|
|
26
|
+
const matched = await this.reconcile(event, resolved);
|
|
27
|
+
if (matched === false) {
|
|
28
|
+
// reconcile() only ever returns false for a delivery-terminal event
|
|
29
|
+
// (selectDeliveryEvent(event) !== null), so this lookup can't miss.
|
|
30
|
+
const delivery = selectDeliveryEvent(event);
|
|
31
|
+
pending.set(delivery.eventId, event);
|
|
32
|
+
}
|
|
22
33
|
await this.checkpoints.save(consumer, event.globalPosition);
|
|
23
34
|
}
|
|
24
|
-
|
|
25
|
-
|
|
35
|
+
// Catches a confirmation checkpointed before its workflow reached
|
|
36
|
+
// "waiting for delivery" — re-checked here on every call rather than
|
|
37
|
+
// relying on the incremental pass above, which never revisits a
|
|
38
|
+
// position once checkpointed.
|
|
39
|
+
for (const [id, pendingEvent] of pending) {
|
|
40
|
+
if (resolved.has(id)) {
|
|
41
|
+
pending.delete(id);
|
|
42
|
+
continue;
|
|
43
|
+
}
|
|
44
|
+
const matched = await this.reconcile(pendingEvent, resolved);
|
|
45
|
+
if (matched === true)
|
|
46
|
+
pending.delete(id);
|
|
47
|
+
}
|
|
48
|
+
await this.savePending([...pending.values()]);
|
|
26
49
|
return events.length;
|
|
27
50
|
}
|
|
51
|
+
// true: resolved (accepted this call, or already accepted earlier this
|
|
52
|
+
// same call). false: a delivery-terminal event whose workflow isn't
|
|
53
|
+
// waiting on it yet — belongs in the pending set for a later retry. null:
|
|
54
|
+
// not a delivery-terminal event at all.
|
|
28
55
|
async reconcile(event, seen) {
|
|
29
56
|
const delivery = selectDeliveryEvent(event);
|
|
30
|
-
if (delivery === null
|
|
31
|
-
return;
|
|
57
|
+
if (delivery === null)
|
|
58
|
+
return null;
|
|
59
|
+
if (seen.has(delivery.eventId))
|
|
60
|
+
return true;
|
|
32
61
|
const outcome = delivery.eventType === DeliveryEventType.Confirmed ||
|
|
33
62
|
(delivery.eventType === DeliveryEventType.Reconciled &&
|
|
34
63
|
delivery.payload.result === DeliveryResultKind.Confirmed)
|
|
@@ -37,13 +66,13 @@ export class DeliveryOutcomeReactor {
|
|
|
37
66
|
? { kind: ActivityOutcomeKind.Failed, data: { reason: delivery.payload.code } }
|
|
38
67
|
: null;
|
|
39
68
|
if (outcome === null)
|
|
40
|
-
return;
|
|
69
|
+
return null;
|
|
41
70
|
const command = {
|
|
42
71
|
workflowInstanceId: workflowInstanceId(delivery.payload.workflowInstanceId),
|
|
43
72
|
activationId: activationId(delivery.payload.activationId),
|
|
44
73
|
};
|
|
45
74
|
if (!(await this.isAwaitingThisDelivery(command, delivery.payload.intentEventId)))
|
|
46
|
-
return;
|
|
75
|
+
return false;
|
|
47
76
|
seen.add(delivery.eventId);
|
|
48
77
|
await this.orchestration.acceptOutcome({ ...command, outcome }, {
|
|
49
78
|
commandId: delivery.eventId,
|
|
@@ -51,6 +80,7 @@ export class DeliveryOutcomeReactor {
|
|
|
51
80
|
actor: { kind: EventActorKind.System, id: 'delivery-outcome-reactor' },
|
|
52
81
|
occurredAt: event.recordedAt,
|
|
53
82
|
});
|
|
83
|
+
return true;
|
|
54
84
|
}
|
|
55
85
|
/**
|
|
56
86
|
* A delivery's own completion may only resolve the activation that
|
|
@@ -66,4 +96,16 @@ export class DeliveryOutcomeReactor {
|
|
|
66
96
|
view.waitingFor?.signalKind === deliveryResultSignalKind &&
|
|
67
97
|
view.waitingFor.intentEventId === intentEventId);
|
|
68
98
|
}
|
|
99
|
+
async loadPending() {
|
|
100
|
+
const stored = await this.projections.read(pendingNamespace, pendingKey);
|
|
101
|
+
return stored?.value.events ?? [];
|
|
102
|
+
}
|
|
103
|
+
async savePending(events) {
|
|
104
|
+
await this.projections.write({
|
|
105
|
+
namespace: pendingNamespace,
|
|
106
|
+
key: pendingKey,
|
|
107
|
+
lastGlobalPosition: 0,
|
|
108
|
+
value: { events },
|
|
109
|
+
});
|
|
110
|
+
}
|
|
69
111
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
package/dist/src/kernel/index.js
CHANGED
|
@@ -5,6 +5,7 @@ export * from './contracts/event-journal.js';
|
|
|
5
5
|
export * from './contracts/event-schema.js';
|
|
6
6
|
export * from './contracts/events.js';
|
|
7
7
|
export * from './contracts/id-generator.js';
|
|
8
|
+
export * from './contracts/journal-change-signal.js';
|
|
8
9
|
export { causationId, correlationId, eventId } from './contracts/identifiers.js';
|
|
9
10
|
export * from './contracts/projection-store.js';
|
|
10
11
|
export * from './contracts/relations.js';
|
|
@@ -14,5 +15,6 @@ export * from './domain/event-envelope.js';
|
|
|
14
15
|
export * from './domain/match-mode.js';
|
|
15
16
|
export * from './domain/relation.js';
|
|
16
17
|
export * from './infrastructure/cached-journal-view.js';
|
|
18
|
+
export * from './infrastructure/journal-change-signal.js';
|
|
17
19
|
export * from './infrastructure/system-clock.js';
|
|
18
20
|
export * from './infrastructure/ulid-id-generator.js';
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
export class InProcessJournalChangeSignal {
|
|
2
|
+
waiters = [];
|
|
3
|
+
notify() {
|
|
4
|
+
const waiters = this.waiters;
|
|
5
|
+
this.waiters = [];
|
|
6
|
+
for (const resolve of waiters)
|
|
7
|
+
resolve();
|
|
8
|
+
}
|
|
9
|
+
waitForChange(signal, fallbackMs) {
|
|
10
|
+
if (signal.aborted)
|
|
11
|
+
return Promise.resolve();
|
|
12
|
+
return new Promise((resolve) => {
|
|
13
|
+
let settled = false;
|
|
14
|
+
const done = () => {
|
|
15
|
+
if (settled)
|
|
16
|
+
return;
|
|
17
|
+
settled = true;
|
|
18
|
+
clearTimeout(timer);
|
|
19
|
+
signal.removeEventListener('abort', done);
|
|
20
|
+
// Must drop this waiter here too, not just in notify() — otherwise
|
|
21
|
+
// every timeout/abort leaks an entry into `waiters` forever.
|
|
22
|
+
this.waiters = this.waiters.filter((waiter) => waiter !== done);
|
|
23
|
+
resolve();
|
|
24
|
+
};
|
|
25
|
+
this.waiters.push(done);
|
|
26
|
+
const timer = setTimeout(done, fallbackMs);
|
|
27
|
+
signal.addEventListener('abort', done, { once: true });
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
}
|
|
@@ -3,7 +3,7 @@ import { OrchestrationEventType } from '../contracts/events.js';
|
|
|
3
3
|
import { commandName, workflowInstanceId, } from '../contracts/identifiers.js';
|
|
4
4
|
import { workflowInstanceStream } from '../contracts/streams.js';
|
|
5
5
|
import { WorkflowStatus } from '../contracts/vocabulary.js';
|
|
6
|
-
import { requestChangesResume as decideChangesResume, requestOperatorRetry as decideOperatorRetry, requestSupplementalActivity as decideSupplementalActivity, } from '../domain/interpreter.js';
|
|
6
|
+
import { requestChangesResume as decideChangesResume, requestOperatorRetry as decideOperatorRetry, requestSupplementalActivity as decideSupplementalActivity, requestRunnerQuotaRetry, } from '../domain/interpreter.js';
|
|
7
7
|
import { isAuthorisedActor } from '../domain/supplemental-policy.js';
|
|
8
8
|
import { appendWithIntentRecovery } from './durable-append.js';
|
|
9
9
|
import { acceptResourceTransition, matchResourceTransitions, } from './resource-transition-matching.js';
|
|
@@ -112,6 +112,29 @@ export class AdvanceWorkflow {
|
|
|
112
112
|
]);
|
|
113
113
|
return (await this.repository.load(id)).view;
|
|
114
114
|
}
|
|
115
|
+
async retryRunnerQuotaFailure(id, input, context) {
|
|
116
|
+
const loaded = await this.repository.load(id);
|
|
117
|
+
if (loaded.view === null)
|
|
118
|
+
return null;
|
|
119
|
+
// The retry re-requests the interrupted Activation, not a stage lookup, but
|
|
120
|
+
// the definition must still be resolvable for the retried Activation's own
|
|
121
|
+
// outcome to be routable later; an unavailable definition blocks here.
|
|
122
|
+
const definition = await this.workflows.definitionForOperation(loaded.view, loaded.sequence, context);
|
|
123
|
+
if (definition === null)
|
|
124
|
+
return (await this.repository.loadRequired(id)).view;
|
|
125
|
+
const decision = requestRunnerQuotaRetry(loaded.view, {
|
|
126
|
+
activationId: input.activationId,
|
|
127
|
+
runId: input.runId,
|
|
128
|
+
runnerName: input.runnerName,
|
|
129
|
+
message: input.message,
|
|
130
|
+
occurredAt: context.occurredAt,
|
|
131
|
+
causationId: context.commandId,
|
|
132
|
+
});
|
|
133
|
+
if (decision.kind === 'ignored')
|
|
134
|
+
return loaded.view;
|
|
135
|
+
await this.repository.append(id, loaded.sequence, decision.events);
|
|
136
|
+
return (await this.repository.load(id)).view;
|
|
137
|
+
}
|
|
115
138
|
async retryBlockedFailedStage(id, context) {
|
|
116
139
|
const loaded = await this.repository.load(id);
|
|
117
140
|
if (loaded.view === null)
|
|
@@ -67,6 +67,9 @@ export class OrchestrationService {
|
|
|
67
67
|
resolveExecutionFailure(workflowInstanceId, input, context) {
|
|
68
68
|
return this.transitionWatchChildren(context, () => this.advanceWorkflow.resolveExecutionFailure(workflowInstanceId, input, context));
|
|
69
69
|
}
|
|
70
|
+
retryRunnerQuotaFailure(workflowInstanceId, input, context) {
|
|
71
|
+
return this.transitionWatchChildren(context, () => this.advanceWorkflow.retryRunnerQuotaFailure(workflowInstanceId, input, context));
|
|
72
|
+
}
|
|
70
73
|
retryBlockedFailedStage(workflowInstanceId, context) {
|
|
71
74
|
return this.transitionWatchChildren(context, () => this.advanceWorkflow.retryBlockedFailedStage(workflowInstanceId, context));
|
|
72
75
|
}
|
|
@@ -232,6 +232,14 @@ const eventSchema = z.discriminatedUnion('eventType', [
|
|
|
232
232
|
reason: z.string().min(1),
|
|
233
233
|
})
|
|
234
234
|
.strict()),
|
|
235
|
+
workflowEnvelope(OrchestrationEventType.ActivityRetriedForRunnerQuota, z
|
|
236
|
+
.object({
|
|
237
|
+
activationId: brandedStringSchema(activationId),
|
|
238
|
+
runId: z.string().min(1),
|
|
239
|
+
runnerName: z.string().min(1),
|
|
240
|
+
message: z.string().min(1),
|
|
241
|
+
})
|
|
242
|
+
.strict()),
|
|
235
243
|
workflowEnvelope(OrchestrationEventType.ActivityWaiting, z
|
|
236
244
|
.object({
|
|
237
245
|
activationId: brandedStringSchema(activationId),
|
|
@@ -7,6 +7,7 @@ export const OrchestrationEventType = {
|
|
|
7
7
|
ActivityStarted: 'orchestration.activity-started',
|
|
8
8
|
ActivityOutcomeAccepted: 'orchestration.activity-outcome-accepted',
|
|
9
9
|
ActivityExecutionFailed: 'orchestration.activity-execution-failed',
|
|
10
|
+
ActivityRetriedForRunnerQuota: 'orchestration.activity-retried-for-runner-quota',
|
|
10
11
|
ActivityWaiting: 'orchestration.activity-waiting',
|
|
11
12
|
SignalWaitStarted: 'orchestration.signal-wait-started',
|
|
12
13
|
SignalAccepted: 'orchestration.signal-accepted',
|
|
@@ -11,6 +11,7 @@ export { startInstance } from './activation-policy.js';
|
|
|
11
11
|
export { acceptSignal, waitForSignal } from './signal-policy.js';
|
|
12
12
|
export { requestSupplementalActivity } from './supplemental-policy.js';
|
|
13
13
|
export { isChangesResumeEligible, isOperatorRetryEligible, requestChangesResume, requestOperatorRetry, selectOperatorRetryTarget, } from './operator-retry-policy.js';
|
|
14
|
+
export { isRunnerQuotaRetryEligible, requestRunnerQuotaRetry, } from './runner-quota-retry-policy.js';
|
|
14
15
|
export function acceptActivityOutcome(definition, state, input) {
|
|
15
16
|
if (!isPendingOutcome(state, input))
|
|
16
17
|
return { kind: 'ignored', reason: 'outcome is not for the pending activation' };
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { activationId as toActivationId } from '../../activities/index.js';
|
|
2
|
+
import { OrchestrationEventType, } from '../contracts/events.js';
|
|
3
|
+
import { activation, nextOrdinal, stateDraft } from './decision-events.js';
|
|
4
|
+
export function isRunnerQuotaRetryEligible(view, activationId) {
|
|
5
|
+
return (view.pendingActivation?.activationId === activationId &&
|
|
6
|
+
!view.acceptedOutcomes.includes(toActivationId(activationId)));
|
|
7
|
+
}
|
|
8
|
+
export function requestRunnerQuotaRetry(state, input) {
|
|
9
|
+
if (!isRunnerQuotaRetryEligible(state, input.activationId))
|
|
10
|
+
return {
|
|
11
|
+
kind: 'ignored',
|
|
12
|
+
reason: 'workflow has no matching pending activation for runner-quota retry',
|
|
13
|
+
};
|
|
14
|
+
// Re-request the interrupted Activation itself, not the current stage's
|
|
15
|
+
// configured activity: a supplemental or follow-on Activation runs a
|
|
16
|
+
// different activity and input, and substituting the stage's main one would
|
|
17
|
+
// drop that work and re-run something the runner never attempted.
|
|
18
|
+
const interrupted = state.pendingActivation;
|
|
19
|
+
// No RetryCounted here, deliberately: a quota condition is a runner-capacity
|
|
20
|
+
// fact, not a failed attempt, so it must never consume the route's
|
|
21
|
+
// configured `retry.max` budget.
|
|
22
|
+
const events = [
|
|
23
|
+
stateDraft(state, input, OrchestrationEventType.ActivityRetriedForRunnerQuota, {
|
|
24
|
+
activationId: toActivationId(input.activationId),
|
|
25
|
+
runId: input.runId,
|
|
26
|
+
runnerName: input.runnerName,
|
|
27
|
+
message: input.message,
|
|
28
|
+
}, 1),
|
|
29
|
+
stateDraft(state, input, OrchestrationEventType.ActivityRequested, activation(state.workflowInstanceId, nextOrdinal(state), interrupted.activity, interrupted.input, {
|
|
30
|
+
execution: interrupted.execution,
|
|
31
|
+
...(interrupted.stage === undefined ? {} : { stage: interrupted.stage }),
|
|
32
|
+
...(interrupted.followOnIndex === undefined
|
|
33
|
+
? {}
|
|
34
|
+
: { followOnIndex: interrupted.followOnIndex }),
|
|
35
|
+
...(interrupted.supplemental === undefined
|
|
36
|
+
? {}
|
|
37
|
+
: { supplemental: interrupted.supplemental }),
|
|
38
|
+
}), 2),
|
|
39
|
+
];
|
|
40
|
+
return { kind: 'append', events };
|
|
41
|
+
}
|
|
@@ -34,6 +34,7 @@ function isActivityFact(event) {
|
|
|
34
34
|
case OrchestrationEventType.ActivityStarted:
|
|
35
35
|
case OrchestrationEventType.ActivityOutcomeAccepted:
|
|
36
36
|
case OrchestrationEventType.ActivityExecutionFailed:
|
|
37
|
+
case OrchestrationEventType.ActivityRetriedForRunnerQuota:
|
|
37
38
|
case OrchestrationEventType.ActivityWaiting:
|
|
38
39
|
return true;
|
|
39
40
|
default:
|
|
@@ -61,6 +62,10 @@ function applyActivityFact(state, event) {
|
|
|
61
62
|
state.executionFailure = event.payload;
|
|
62
63
|
updateActivationStatus(state, event.payload.activationId, ActivityActivationStatus.Completed);
|
|
63
64
|
return;
|
|
65
|
+
case OrchestrationEventType.ActivityRetriedForRunnerQuota:
|
|
66
|
+
state.acceptedOutcomes.push(event.payload.activationId);
|
|
67
|
+
updateActivationStatus(state, event.payload.activationId, ActivityActivationStatus.Completed);
|
|
68
|
+
return;
|
|
64
69
|
case OrchestrationEventType.ActivityWaiting:
|
|
65
70
|
applyActivityWaiting(state, event);
|
|
66
71
|
return;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { appendFile, mkdir, readdir, readFile, stat } from 'node:fs/promises';
|
|
2
2
|
import { join } from 'node:path';
|
|
3
3
|
import { isDeepStrictEqual } from 'node:util';
|
|
4
|
-
import { decodeEventEnvelope, WrongExpectedSequenceError } from '../../kernel/index.js';
|
|
4
|
+
import { decodeEventEnvelope, InProcessJournalChangeSignal, WrongExpectedSequenceError, } from '../../kernel/index.js';
|
|
5
5
|
import { withFileLock } from './file-lock.js';
|
|
6
6
|
export class FileEventJournal {
|
|
7
7
|
root;
|
|
@@ -10,6 +10,10 @@ export class FileEventJournal {
|
|
|
10
10
|
this.root = root;
|
|
11
11
|
this.clock = clock;
|
|
12
12
|
}
|
|
13
|
+
changeSignalSource = new InProcessJournalChangeSignal();
|
|
14
|
+
get changeSignal() {
|
|
15
|
+
return this.changeSignalSource;
|
|
16
|
+
}
|
|
13
17
|
cached;
|
|
14
18
|
async append(stream, expectedSequence, drafts) {
|
|
15
19
|
return withFileLock(join(this.root, 'locks', 'event-journal.lock'), async () => {
|
|
@@ -56,6 +60,7 @@ export class FileEventJournal {
|
|
|
56
60
|
const file = `${day}.jsonl`;
|
|
57
61
|
await appendFile(join(directory, file), newEnvelopes.map((event) => JSON.stringify(event)).join('\n') + '\n', 'utf8');
|
|
58
62
|
await this.extendCache(file, current, newEnvelopes);
|
|
63
|
+
this.changeSignalSource.notify();
|
|
59
64
|
}
|
|
60
65
|
return finalizedEnvelopes;
|
|
61
66
|
});
|
|
@@ -1,13 +1,17 @@
|
|
|
1
1
|
import { isDeepStrictEqual } from 'node:util';
|
|
2
|
-
import { WrongExpectedSequenceError } from '../../kernel/index.js';
|
|
2
|
+
import { InProcessJournalChangeSignal, WrongExpectedSequenceError } from '../../kernel/index.js';
|
|
3
3
|
export class InMemoryEventJournal {
|
|
4
4
|
clock;
|
|
5
5
|
streams = new Map();
|
|
6
6
|
events = [];
|
|
7
7
|
eventIds = new Map();
|
|
8
|
+
changeSignalSource = new InProcessJournalChangeSignal();
|
|
8
9
|
constructor(clock) {
|
|
9
10
|
this.clock = clock;
|
|
10
11
|
}
|
|
12
|
+
get changeSignal() {
|
|
13
|
+
return this.changeSignalSource;
|
|
14
|
+
}
|
|
11
15
|
async append(stream, expectedSequence, events) {
|
|
12
16
|
validateBatch(stream, events);
|
|
13
17
|
const existingEvents = events.map((draft) => this.eventIds.get(draft.eventId));
|
|
@@ -21,6 +25,7 @@ export class InMemoryEventJournal {
|
|
|
21
25
|
}
|
|
22
26
|
const recordedAt = this.clock.now().toISOString();
|
|
23
27
|
const appended = [];
|
|
28
|
+
let newCount = 0;
|
|
24
29
|
for (const [index, draft] of events.entries()) {
|
|
25
30
|
const prior = existingEvents[index];
|
|
26
31
|
if (prior !== undefined) {
|
|
@@ -37,8 +42,11 @@ export class InMemoryEventJournal {
|
|
|
37
42
|
this.events.push(envelope);
|
|
38
43
|
this.eventIds.set(draft.eventId, { draft, envelope });
|
|
39
44
|
appended.push(envelope);
|
|
45
|
+
newCount += 1;
|
|
40
46
|
}
|
|
41
47
|
this.streams.set(streamKey(stream), streamEvents);
|
|
48
|
+
if (newCount > 0)
|
|
49
|
+
this.changeSignalSource.notify();
|
|
42
50
|
return appended;
|
|
43
51
|
}
|
|
44
52
|
async readStream(stream) {
|