@atolis-hq/wake 0.4.73 → 0.4.75
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/bootstrap/composition-root.js +2 -10
- package/dist/src/bootstrap/runner-tool-notices.d.ts +3 -0
- package/dist/src/bootstrap/runner-tool-notices.js +31 -0
- package/dist/src/bootstrap/surface-api-trigger-applications.js +38 -4
- package/dist/src/bootstrap/surface-api-work-applications.js +5 -3
- package/dist/src/bootstrap/surface-cli-applications.js +4 -1
- package/dist/src/bootstrap/version.js +1 -1
- package/dist/src/control-plane/application/activation-scheduler-ports.d.ts +2 -1
- package/dist/src/control-plane/application/activation-scheduler.js +14 -2
- package/dist/src/control-plane/application/advance-once-dispatch.js +1 -0
- package/dist/src/control-plane/application/trigger-inspection.d.ts +32 -0
- package/dist/src/control-plane/application/trigger-inspection.js +14 -0
- package/dist/src/control-plane/application/trigger-rules.js +3 -0
- package/dist/src/control-plane/application/trigger-service.d.ts +1 -0
- package/dist/src/control-plane/application/trigger-service.js +5 -5
- package/dist/src/execution/application/recovery-service.d.ts +2 -1
- package/dist/src/execution/application/recovery-service.js +20 -11
- package/dist/src/execution/contracts/config.d.ts +2 -1
- package/dist/src/execution/contracts/config.js +6 -5
- package/dist/src/execution/contracts/events.js +1 -1
- package/dist/src/execution/contracts/vocabulary.d.ts +12 -0
- package/dist/src/execution/contracts/vocabulary.js +18 -0
- package/dist/src/execution/index.d.ts +1 -0
- package/dist/src/execution/index.js +1 -0
- package/dist/src/execution/infrastructure/local-process-inspector.d.ts +4 -0
- package/dist/src/execution/infrastructure/local-process-inspector.js +85 -0
- package/dist/src/execution/infrastructure/process-execution.d.ts +1 -0
- package/dist/src/execution/infrastructure/process-execution.js +6 -1
- package/dist/src/execution/infrastructure/runners/claude.js +1 -1
- package/dist/src/node_modules/@atolis-hq/eventing-filesystem/dist/file-lock.d.ts +1 -1
- package/dist/src/node_modules/@atolis-hq/eventing-filesystem/dist/file-lock.d.ts.map +1 -1
- package/dist/src/node_modules/@atolis-hq/eventing-filesystem/dist/file-lock.js +14 -7
- package/dist/src/orchestration/application/advance-workflow.d.ts +7 -8
- package/dist/src/orchestration/application/advance-workflow.js +21 -36
- package/dist/src/orchestration/application/operator-retry-attribution.d.ts +3 -0
- package/dist/src/orchestration/application/operator-retry-attribution.js +7 -0
- package/dist/src/orchestration/application/orchestration-service.d.ts +4 -1
- package/dist/src/orchestration/application/orchestration-service.js +4 -1
- package/dist/src/orchestration/application/resolve-execution-failure.d.ts +14 -0
- package/dist/src/orchestration/application/resolve-execution-failure.js +47 -0
- package/dist/src/surfaces/api/contracts/transport-values.d.ts +5 -1
- package/dist/src/surfaces/api/contracts/transport-values.js +13 -0
- package/dist/src/surfaces/api/contracts/triggers.d.ts +13 -1
- package/dist/src/surfaces/web-assets/assets/index-QnREfUBG.js +1192 -0
- package/dist/src/surfaces/web-assets/index.html +1 -1
- package/package.json +1 -1
- package/dist/src/surfaces/web-assets/assets/index-CxURzTmR.js +0 -1192
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import { readFileSync, readlinkSync } from 'node:fs';
|
|
2
|
+
import { z } from 'zod';
|
|
3
|
+
import { ExternalExecutionKind } from '../../activities/index.js';
|
|
4
|
+
import { ExternalExecutionState } from '../contracts/vocabulary.js';
|
|
5
|
+
function environment() {
|
|
6
|
+
return {
|
|
7
|
+
boot: readFileSync('/proc/sys/kernel/random/boot_id', 'utf8').trim(),
|
|
8
|
+
namespace: readlinkSync('/proc/self/ns/pid'),
|
|
9
|
+
};
|
|
10
|
+
}
|
|
11
|
+
function processStart(pid) {
|
|
12
|
+
const stat = readFileSync(`/proc/${pid}/stat`, 'utf8');
|
|
13
|
+
const fields = stat.slice(stat.lastIndexOf(')') + 2).split(' ');
|
|
14
|
+
const start = fields[19];
|
|
15
|
+
if (fields[0] === 'Z' || fields[0] === 'X')
|
|
16
|
+
return 'terminated';
|
|
17
|
+
if (start === undefined || !/^\d+$/.test(start))
|
|
18
|
+
throw new Error('Invalid process stat');
|
|
19
|
+
return start;
|
|
20
|
+
}
|
|
21
|
+
/** Opaque identity includes PID reuse protection and the inspection environment. */
|
|
22
|
+
export function localProcessIdentity(pid) {
|
|
23
|
+
if (pid === undefined || process.platform !== 'linux')
|
|
24
|
+
return undefined;
|
|
25
|
+
try {
|
|
26
|
+
const start = processStart(pid);
|
|
27
|
+
if (start === 'terminated')
|
|
28
|
+
return undefined;
|
|
29
|
+
return JSON.stringify({ ...environment(), pid, start });
|
|
30
|
+
}
|
|
31
|
+
catch {
|
|
32
|
+
return undefined;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
const processIdentitySchema = z
|
|
36
|
+
.object({
|
|
37
|
+
boot: z.string().min(1),
|
|
38
|
+
namespace: z.string().min(1),
|
|
39
|
+
pid: z.number().int().positive().safe(),
|
|
40
|
+
start: z.string().regex(/^\d+$/),
|
|
41
|
+
})
|
|
42
|
+
.strict();
|
|
43
|
+
function decodeIdentity(id) {
|
|
44
|
+
try {
|
|
45
|
+
const parsed = processIdentitySchema.safeParse(JSON.parse(id));
|
|
46
|
+
return parsed.success ? parsed.data : undefined;
|
|
47
|
+
}
|
|
48
|
+
catch {
|
|
49
|
+
return undefined;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
export function createLocalProcessInspector() {
|
|
53
|
+
return {
|
|
54
|
+
async inspect(reference) {
|
|
55
|
+
const unknown = (reason) => ({ kind: ExternalExecutionState.Unknown, reason });
|
|
56
|
+
const identity = decodeIdentity(reference.id);
|
|
57
|
+
if (reference.kind !== ExternalExecutionKind.Process ||
|
|
58
|
+
identity === undefined ||
|
|
59
|
+
process.platform !== 'linux')
|
|
60
|
+
return unknown('No inspectable local process identity; verify the runner has stopped, then resolve the ambiguous run');
|
|
61
|
+
try {
|
|
62
|
+
const current = environment();
|
|
63
|
+
if (current.boot !== identity.boot)
|
|
64
|
+
return { kind: ExternalExecutionState.Absent };
|
|
65
|
+
if (current.namespace !== identity.namespace)
|
|
66
|
+
return unknown('Runner PID namespace changed; verify the previous container has stopped, then resolve the ambiguous run');
|
|
67
|
+
try {
|
|
68
|
+
return {
|
|
69
|
+
kind: processStart(identity.pid) === identity.start
|
|
70
|
+
? ExternalExecutionState.Running
|
|
71
|
+
: ExternalExecutionState.Absent,
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
catch (error) {
|
|
75
|
+
if (error instanceof Error && 'code' in error && error.code === 'ENOENT')
|
|
76
|
+
return { kind: ExternalExecutionState.Absent };
|
|
77
|
+
throw error;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
catch {
|
|
81
|
+
return unknown('Local process inspection failed; restore /proc access or verify the runner has stopped and resolve the ambiguous run');
|
|
82
|
+
}
|
|
83
|
+
},
|
|
84
|
+
};
|
|
85
|
+
}
|
|
@@ -27,6 +27,7 @@ export interface ProcessExecutionResult {
|
|
|
27
27
|
readonly failureMessage?: string;
|
|
28
28
|
}
|
|
29
29
|
export declare function runProcess(command: string, args: readonly string[], cwd: string | undefined, signal: AbortSignal, options?: ProcessExecutionOptions, shell?: boolean): {
|
|
30
|
+
readonly identity?: string;
|
|
30
31
|
readonly result: Promise<ProcessExecutionResult>;
|
|
31
32
|
cancel(): Promise<void>;
|
|
32
33
|
};
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { spawn } from 'node:child_process';
|
|
2
|
+
import { localProcessIdentity } from './local-process-inspector.js';
|
|
2
3
|
// Agent CLIs can emit arbitrarily large machine-readable transcripts. Capture
|
|
3
4
|
// raw bytes ourselves so overflow never enters a third-party string buffer.
|
|
4
5
|
export const maximumCapturedProcessOutputBytes = 1024 * 1024;
|
|
@@ -9,7 +10,11 @@ export function runProcess(command, args, cwd, signal, options, shell = false) {
|
|
|
9
10
|
...(shell && process.platform !== 'win32' ? { detached: true } : {}),
|
|
10
11
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
11
12
|
});
|
|
12
|
-
|
|
13
|
+
const identity = localProcessIdentity(child.pid);
|
|
14
|
+
return {
|
|
15
|
+
...captureProcessOutput(child, signal, options, shell),
|
|
16
|
+
...(identity === undefined ? {} : { identity }),
|
|
17
|
+
};
|
|
13
18
|
}
|
|
14
19
|
// Lifecycle coordination must share these closures so first-event-wins is atomic.
|
|
15
20
|
// eslint-disable-next-line max-lines-per-function
|
|
@@ -138,7 +138,7 @@ export function cliRunner(name, command, args, options = {}) {
|
|
|
138
138
|
return {
|
|
139
139
|
identity: {
|
|
140
140
|
kind: ExternalExecutionKind.Process,
|
|
141
|
-
id: request.runId,
|
|
141
|
+
id: process.identity ?? request.runId,
|
|
142
142
|
startedAt: new Date().toISOString(),
|
|
143
143
|
},
|
|
144
144
|
result: process.result.then((value) => value.exitCode === 0 && !value.timedOut
|
|
@@ -8,7 +8,7 @@ export interface FileLockMetadata {
|
|
|
8
8
|
export interface FileLockOptions {
|
|
9
9
|
readonly staleAfterMs?: number;
|
|
10
10
|
readonly now?: Date;
|
|
11
|
-
/**
|
|
11
|
+
/** Recover immediately only when process evidence proves the owner generation dead. */
|
|
12
12
|
readonly staleRequiresDeadProcess?: boolean;
|
|
13
13
|
readonly isProcessAlive?: (pid: number) => boolean;
|
|
14
14
|
readonly processIdentity?: (pid: number) => Promise<string | null>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"file-lock.d.ts","sourceRoot":"","sources":["../src/file-lock.ts"],"names":[],"mappings":"AAMA,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,iGAAiG;IACjG,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;CACnC;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC;IACpB,
|
|
1
|
+
{"version":3,"file":"file-lock.d.ts","sourceRoot":"","sources":["../src/file-lock.ts"],"names":[],"mappings":"AAMA,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,iGAAiG;IACjG,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;CACnC;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC;IACpB,uFAAuF;IACvF,QAAQ,CAAC,wBAAwB,CAAC,EAAE,OAAO,CAAC;IAC5C,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC;IACnD,QAAQ,CAAC,eAAe,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IACnE,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;CACnE;AAED,MAAM,WAAW,mBAAoB,SAAQ,eAAe;IAC1D,mFAAmF;IACnF,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,6EAA6E;IAC7E,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;CACnC;AAED,UAAU,mBAAmB;IAC3B,QAAQ,CAAC,QAAQ,EAAE,KAAK,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CACvC;AAUD,wBAAsB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,eAAe;;;;GAY5E;AA0UD,wBAAsB,YAAY,CAAC,KAAK,EACtC,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,OAAO,CAAC,KAAK,CAAC,EAC/B,OAAO,GAAE,mBAAwB,GAChC,OAAO,CAAC,KAAK,CAAC,CAYhB"}
|
|
@@ -226,14 +226,15 @@ async function ownerBlocksAcquisition(path, options) {
|
|
|
226
226
|
lockId: encoded[3],
|
|
227
227
|
};
|
|
228
228
|
}
|
|
229
|
-
if (options?.staleAfterMs === undefined || !isStale(owner, options))
|
|
230
|
-
return true;
|
|
231
229
|
return ownerMetadataBlocks(owner, options);
|
|
232
230
|
}
|
|
233
231
|
async function ownerMetadataBlocks(owner, options) {
|
|
234
|
-
if (options?.
|
|
232
|
+
if (!options?.staleRequiresDeadProcess)
|
|
233
|
+
return options?.staleAfterMs === undefined || !isStale(owner, options);
|
|
234
|
+
const alive = ownerMayBeAlive(options, owner.pid);
|
|
235
|
+
if (alive === null)
|
|
235
236
|
return true;
|
|
236
|
-
if (!
|
|
237
|
+
if (!alive)
|
|
237
238
|
return false;
|
|
238
239
|
const identity = await processIdentity(owner.pid, options);
|
|
239
240
|
if (owner.processIdentity !== undefined)
|
|
@@ -256,8 +257,14 @@ function isProcessAlive(pid) {
|
|
|
256
257
|
process.kill(pid, 0);
|
|
257
258
|
return true;
|
|
258
259
|
}
|
|
259
|
-
catch {
|
|
260
|
-
|
|
260
|
+
catch (error) {
|
|
261
|
+
const code = error.code;
|
|
262
|
+
if (code === 'ESRCH')
|
|
263
|
+
return false;
|
|
264
|
+
// EPERM proves the pid exists; start identity then decides whether it is the owner.
|
|
265
|
+
if (code === 'EPERM')
|
|
266
|
+
return true;
|
|
267
|
+
throw error;
|
|
261
268
|
}
|
|
262
269
|
}
|
|
263
270
|
function ownerMayBeAlive(options, pid) {
|
|
@@ -265,7 +272,7 @@ function ownerMayBeAlive(options, pid) {
|
|
|
265
272
|
return (options.isProcessAlive ?? isProcessAlive)(pid);
|
|
266
273
|
}
|
|
267
274
|
catch {
|
|
268
|
-
return
|
|
275
|
+
return null;
|
|
269
276
|
}
|
|
270
277
|
}
|
|
271
278
|
const processStartToleranceMilliseconds = 1_000;
|
|
@@ -3,7 +3,9 @@ import type { ActivationId, selectActivityEvent } from '../../activities/index.j
|
|
|
3
3
|
import type { TransitionTarget } from '../contracts/config.js';
|
|
4
4
|
import type { SupplementalActivityRequest } from '../contracts/events.js';
|
|
5
5
|
import { type SignalName, type WorkflowInstanceId } from '../contracts/identifiers.js';
|
|
6
|
+
import type { AcceptActivityOutcome } from './accept-activity-outcome.js';
|
|
6
7
|
import type { OrchestrationRepository } from './orchestration-repository.js';
|
|
8
|
+
import { type ExecutionFailureInput } from './resolve-execution-failure.js';
|
|
7
9
|
import type { StartWorkflow } from './start-workflow.js';
|
|
8
10
|
export declare class OperatorRetryIneligibleError extends Error {
|
|
9
11
|
constructor(detail: string);
|
|
@@ -12,16 +14,14 @@ type PersistedEvent = Parameters<typeof selectActivityEvent>[0];
|
|
|
12
14
|
export declare class AdvanceWorkflow {
|
|
13
15
|
private readonly repository;
|
|
14
16
|
private readonly workflows;
|
|
15
|
-
|
|
17
|
+
private readonly acceptOutcome;
|
|
18
|
+
constructor(repository: OrchestrationRepository, workflows: StartWorkflow, acceptOutcome: AcceptActivityOutcome['execute']);
|
|
16
19
|
requestSupplementalActivity(id: WorkflowInstanceId, request: SupplementalActivityRequest, context: CommandContext): Promise<import("../index.js").WorkflowInstanceView>;
|
|
17
20
|
markActivationStarted(id: WorkflowInstanceId, activationId: ActivationId, context: CommandContext): Promise<import("../index.js").WorkflowInstanceView | null>;
|
|
21
|
+
blockAwaitingActivation(id: WorkflowInstanceId, activationId: ActivationId, reason: string, context: CommandContext): Promise<import("../index.js").WorkflowInstanceView | null>;
|
|
18
22
|
block(id: WorkflowInstanceId, reason: string, context: CommandContext): Promise<import("../index.js").WorkflowInstanceView | null>;
|
|
19
23
|
expireTimedOutWaits(context: CommandContext): Promise<import("../index.js").WorkflowInstanceView[]>;
|
|
20
|
-
resolveExecutionFailure(id: WorkflowInstanceId, input:
|
|
21
|
-
readonly activationId: ActivationId;
|
|
22
|
-
readonly runId: string;
|
|
23
|
-
readonly reason: string;
|
|
24
|
-
}, context: CommandContext): Promise<import("../index.js").WorkflowInstanceView | null>;
|
|
24
|
+
resolveExecutionFailure(id: WorkflowInstanceId, input: ExecutionFailureInput, context: CommandContext): Promise<import("../index.js").WorkflowInstanceView | null>;
|
|
25
25
|
retryRunnerQuotaFailure(id: WorkflowInstanceId, input: {
|
|
26
26
|
readonly activationId: ActivationId;
|
|
27
27
|
readonly runId: string;
|
|
@@ -48,5 +48,4 @@ export declare class AdvanceWorkflow {
|
|
|
48
48
|
watch: Readonly<import("../contracts/config.js").CompiledWatch>;
|
|
49
49
|
}[]>;
|
|
50
50
|
}
|
|
51
|
-
export
|
|
52
|
-
export {};
|
|
51
|
+
export { isAmbiguityResolutionBlock } from './resolve-execution-failure.js';
|
|
@@ -7,6 +7,8 @@ import { requestChangesResume as decideChangesResume, requestFreshOperatorRetry
|
|
|
7
7
|
import { isAuthorisedActor } from '../domain/supplemental-policy.js';
|
|
8
8
|
import { appendWithIntentRecovery } from './durable-append.js';
|
|
9
9
|
import { expireTimedOutWaits } from './expire-timed-out-waits.js';
|
|
10
|
+
import { attributeOperatorRetry } from './operator-retry-attribution.js';
|
|
11
|
+
import { resolveExecutionFailure, } from './resolve-execution-failure.js';
|
|
10
12
|
import { acceptResourceTransition, matchResourceTransitions, } from './resource-transition-matching.js';
|
|
11
13
|
import { matchWatches } from './watch-matching.js';
|
|
12
14
|
export class OperatorRetryIneligibleError extends Error {
|
|
@@ -18,9 +20,11 @@ export class OperatorRetryIneligibleError extends Error {
|
|
|
18
20
|
export class AdvanceWorkflow {
|
|
19
21
|
repository;
|
|
20
22
|
workflows;
|
|
21
|
-
|
|
23
|
+
acceptOutcome;
|
|
24
|
+
constructor(repository, workflows, acceptOutcome) {
|
|
22
25
|
this.repository = repository;
|
|
23
26
|
this.workflows = workflows;
|
|
27
|
+
this.acceptOutcome = acceptOutcome;
|
|
24
28
|
}
|
|
25
29
|
async requestSupplementalActivity(id, request, context) {
|
|
26
30
|
const loaded = await this.repository.loadRequired(id);
|
|
@@ -59,6 +63,17 @@ export class AdvanceWorkflow {
|
|
|
59
63
|
await this.repository.append(id, loaded.sequence, [event]);
|
|
60
64
|
return (await this.repository.load(id)).view;
|
|
61
65
|
}
|
|
66
|
+
async blockAwaitingActivation(id, activationId, reason, context) {
|
|
67
|
+
const loaded = await this.repository.load(id);
|
|
68
|
+
// Only a workflow still waiting on this activation may be blocked on its behalf.
|
|
69
|
+
if (loaded.view === null ||
|
|
70
|
+
loaded.view.status === WorkflowStatus.Completed ||
|
|
71
|
+
loaded.view.status === WorkflowStatus.Superseded ||
|
|
72
|
+
loaded.view.pendingActivation?.activationId !== activationId ||
|
|
73
|
+
loaded.view.acceptedOutcomes.includes(activationId))
|
|
74
|
+
return loaded.view;
|
|
75
|
+
return this.block(id, reason, context);
|
|
76
|
+
}
|
|
62
77
|
async block(id, reason, context) {
|
|
63
78
|
const loaded = await this.repository.load(id);
|
|
64
79
|
if (loaded.view === null || loaded.view.status === WorkflowStatus.Blocked)
|
|
@@ -79,37 +94,8 @@ export class AdvanceWorkflow {
|
|
|
79
94
|
async expireTimedOutWaits(context) {
|
|
80
95
|
return expireTimedOutWaits(this.repository, context);
|
|
81
96
|
}
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
if (loaded.view === null ||
|
|
85
|
-
(loaded.view.status === WorkflowStatus.Blocked &&
|
|
86
|
-
!isAmbiguityResolutionBlock(loaded.view.blockReason)) ||
|
|
87
|
-
loaded.view.pendingActivation?.activationId !== input.activationId ||
|
|
88
|
-
loaded.view.acceptedOutcomes.includes(input.activationId))
|
|
89
|
-
return loaded.view;
|
|
90
|
-
await this.repository.append(id, loaded.sequence, [
|
|
91
|
-
workflowEventData({
|
|
92
|
-
eventId: `${context.commandId}:${OrchestrationEventType.ActivityExecutionFailed}`,
|
|
93
|
-
eventType: OrchestrationEventType.ActivityExecutionFailed,
|
|
94
|
-
occurredAt: context.occurredAt,
|
|
95
|
-
correlationId: context.correlationId,
|
|
96
|
-
causationId: context.commandId,
|
|
97
|
-
actor: context.actor,
|
|
98
|
-
source: { kind: EventSourceKind.Internal, id: 'orchestration-service' },
|
|
99
|
-
payload: input,
|
|
100
|
-
}),
|
|
101
|
-
workflowEventData({
|
|
102
|
-
eventId: `${context.commandId}:${OrchestrationEventType.InstanceBlocked}`,
|
|
103
|
-
eventType: OrchestrationEventType.InstanceBlocked,
|
|
104
|
-
occurredAt: context.occurredAt,
|
|
105
|
-
correlationId: context.correlationId,
|
|
106
|
-
causationId: context.commandId,
|
|
107
|
-
actor: context.actor,
|
|
108
|
-
source: { kind: EventSourceKind.Internal, id: 'orchestration-service' },
|
|
109
|
-
payload: { reason: input.reason },
|
|
110
|
-
}),
|
|
111
|
-
]);
|
|
112
|
-
return (await this.repository.load(id)).view;
|
|
97
|
+
resolveExecutionFailure(id, input, context) {
|
|
98
|
+
return resolveExecutionFailure(this.repository, this.acceptOutcome, id, input, context);
|
|
113
99
|
}
|
|
114
100
|
async retryRunnerQuotaFailure(id, input, context) {
|
|
115
101
|
const loaded = await this.repository.load(id);
|
|
@@ -156,9 +142,10 @@ export class AdvanceWorkflow {
|
|
|
156
142
|
});
|
|
157
143
|
if (decision.kind === 'ignored')
|
|
158
144
|
throw new OperatorRetryIneligibleError(decision.reason);
|
|
145
|
+
const attributed = attributeOperatorRetry(decision.events, context.actor);
|
|
159
146
|
const recovered = await appendWithIntentRecovery({
|
|
160
147
|
append: async () => {
|
|
161
|
-
await this.repository.append(id, loaded.sequence,
|
|
148
|
+
await this.repository.append(id, loaded.sequence, attributed);
|
|
162
149
|
},
|
|
163
150
|
load: () => this.repository.loadRequired(id),
|
|
164
151
|
alreadyApplied: (reloaded) => reloaded.view.operatorRetryCommandIds.includes(context.commandId),
|
|
@@ -238,6 +225,4 @@ export class AdvanceWorkflow {
|
|
|
238
225
|
return matchWatches(await this.listAllLoaded(), event, this.workflows, context);
|
|
239
226
|
}
|
|
240
227
|
}
|
|
241
|
-
export
|
|
242
|
-
return reason !== undefined && /^run-ambiguous-after-\d+-attempts$/.test(reason);
|
|
243
|
-
}
|
|
228
|
+
export { isAmbiguityResolutionBlock } from './resolve-execution-failure.js';
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
import type { EventActor } from '@atolis-hq/eventing';
|
|
2
|
+
import { type WorkflowOrchestrationEventData } from '../contracts/events.js';
|
|
3
|
+
export declare function attributeOperatorRetry(events: readonly WorkflowOrchestrationEventData[], actor: EventActor): readonly WorkflowOrchestrationEventData[];
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { OrchestrationEventType, } from '../contracts/events.js';
|
|
2
|
+
import { workflowEventData } from '../domain/decision-events.js';
|
|
3
|
+
export function attributeOperatorRetry(events, actor) {
|
|
4
|
+
return events.map((event) => event.eventType === OrchestrationEventType.OperatorRetryRequested
|
|
5
|
+
? workflowEventData({ ...event, actor })
|
|
6
|
+
: event);
|
|
7
|
+
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { ResourceTransitionEvidence } from './resource-transition-evidence.js';
|
|
2
2
|
import type { CommandContext, EventJournal, ProjectionStore } from '@atolis-hq/eventing';
|
|
3
|
-
import type
|
|
3
|
+
import { type ActivationId, type ActivityOutcome } from '../../activities/index.js';
|
|
4
|
+
import type { ExecutionFailureCode } from '../../execution/index.js';
|
|
4
5
|
import { type WorkItemId, type WorkService } from '../../work/index.js';
|
|
5
6
|
import type { StartWorkflowInstance } from '../contracts/commands.js';
|
|
6
7
|
import type { CompiledWorkflow, TransitionTarget } from '../contracts/config.js';
|
|
@@ -44,10 +45,12 @@ export declare class OrchestrationService {
|
|
|
44
45
|
markActivationStarted(workflowInstanceId: WorkflowInstanceId, activationId: ActivationId, context: CommandContext): Promise<WorkflowInstanceView | null>;
|
|
45
46
|
validateActivationDispatch(workflowInstanceId: WorkflowInstanceId, context: CommandContext): Promise<boolean>;
|
|
46
47
|
block(workflowInstanceId: WorkflowInstanceId, reason: string, context: CommandContext): Promise<WorkflowInstanceView | null>;
|
|
48
|
+
blockAwaitingActivation(workflowInstanceId: WorkflowInstanceId, activationId: ActivationId, reason: string, context: CommandContext): Promise<WorkflowInstanceView | null>;
|
|
47
49
|
resolveExecutionFailure(workflowInstanceId: WorkflowInstanceId, input: {
|
|
48
50
|
readonly activationId: ActivationId;
|
|
49
51
|
readonly runId: string;
|
|
50
52
|
readonly reason: string;
|
|
53
|
+
readonly failureKind?: ExecutionFailureCode;
|
|
51
54
|
}, context: CommandContext): Promise<WorkflowInstanceView | null>;
|
|
52
55
|
retryRunnerQuotaFailure(workflowInstanceId: WorkflowInstanceId, input: {
|
|
53
56
|
readonly activationId: ActivationId;
|
|
@@ -39,7 +39,7 @@ export class OrchestrationService {
|
|
|
39
39
|
this.claims = new CoordinationClaims(journal);
|
|
40
40
|
this.definitions = new WorkflowDefinitionRegistry(journal, projections, definitions);
|
|
41
41
|
this.startWorkflow = new StartWorkflow(repository, this.claims, work, this.definitions);
|
|
42
|
-
this.advanceWorkflow = new AdvanceWorkflow(repository, this.startWorkflow);
|
|
42
|
+
this.advanceWorkflow = new AdvanceWorkflow(repository, this.startWorkflow, (command, context) => this.acceptActivityOutcome.execute(command, context));
|
|
43
43
|
this.acceptWorkflowSignal = new AcceptSignal(repository, this.startWorkflow, work);
|
|
44
44
|
this.childWorkflows = new RequestChild(repository, this.claims, new GroupBudgetRecorder(journal), this.startWorkflow, this.advanceWorkflow);
|
|
45
45
|
this.acceptActivityOutcome = new AcceptActivityOutcome(repository, this.startWorkflow, (context) => this.childWorkflows.reconcileChildCompletions(context));
|
|
@@ -152,6 +152,9 @@ export class OrchestrationService {
|
|
|
152
152
|
block(workflowInstanceId, reason, context) {
|
|
153
153
|
return this.transitionWatchChildren(context, () => this.advanceWorkflow.block(workflowInstanceId, reason, context));
|
|
154
154
|
}
|
|
155
|
+
blockAwaitingActivation(workflowInstanceId, activationId, reason, context) {
|
|
156
|
+
return this.transitionWatchChildren(context, () => this.advanceWorkflow.blockAwaitingActivation(workflowInstanceId, activationId, reason, context));
|
|
157
|
+
}
|
|
155
158
|
resolveExecutionFailure(workflowInstanceId, input, context) {
|
|
156
159
|
return this.transitionWatchChildren(context, () => this.advanceWorkflow.resolveExecutionFailure(workflowInstanceId, input, context));
|
|
157
160
|
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { type CommandContext } from '@atolis-hq/eventing';
|
|
2
|
+
import { type ActivationId } from '../../activities/index.js';
|
|
3
|
+
import { ExecutionFailureCode } from '../../execution/index.js';
|
|
4
|
+
import type { WorkflowInstanceId } from '../contracts/identifiers.js';
|
|
5
|
+
import type { AcceptActivityOutcome } from './accept-activity-outcome.js';
|
|
6
|
+
import type { OrchestrationRepository } from './orchestration-repository.js';
|
|
7
|
+
export interface ExecutionFailureInput {
|
|
8
|
+
readonly activationId: ActivationId;
|
|
9
|
+
readonly runId: string;
|
|
10
|
+
readonly reason: string;
|
|
11
|
+
readonly failureKind?: ExecutionFailureCode;
|
|
12
|
+
}
|
|
13
|
+
export declare function resolveExecutionFailure(repository: OrchestrationRepository, acceptOutcome: AcceptActivityOutcome['execute'], id: WorkflowInstanceId, input: ExecutionFailureInput, context: CommandContext): Promise<import("../index.js").WorkflowInstanceView | null>;
|
|
14
|
+
export declare function isAmbiguityResolutionBlock(reason: string | undefined): boolean;
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { EventSourceKind } from '@atolis-hq/eventing';
|
|
2
|
+
import { ActivityOutcomeKind } from '../../activities/index.js';
|
|
3
|
+
import { ExecutionFailureCode, isAmbiguityResolutionBlockReason } from '../../execution/index.js';
|
|
4
|
+
import { OrchestrationEventType } from '../contracts/events.js';
|
|
5
|
+
import { WorkflowStatus } from '../contracts/vocabulary.js';
|
|
6
|
+
import { workflowEventData } from '../domain/decision-events.js';
|
|
7
|
+
export async function resolveExecutionFailure(repository, acceptOutcome, id, input, context) {
|
|
8
|
+
if (input.failureKind === ExecutionFailureCode.Interrupted)
|
|
9
|
+
return acceptOutcome({
|
|
10
|
+
workflowInstanceId: id,
|
|
11
|
+
activationId: input.activationId,
|
|
12
|
+
outcome: { kind: ActivityOutcomeKind.Failed },
|
|
13
|
+
}, context);
|
|
14
|
+
const loaded = await repository.load(id);
|
|
15
|
+
if (loaded.view === null ||
|
|
16
|
+
(loaded.view.status === WorkflowStatus.Blocked &&
|
|
17
|
+
!isAmbiguityResolutionBlock(loaded.view.blockReason)) ||
|
|
18
|
+
loaded.view.pendingActivation?.activationId !== input.activationId ||
|
|
19
|
+
loaded.view.acceptedOutcomes.includes(input.activationId))
|
|
20
|
+
return loaded.view;
|
|
21
|
+
await repository.append(id, loaded.sequence, [
|
|
22
|
+
workflowEventData({
|
|
23
|
+
eventId: `${context.commandId}:${OrchestrationEventType.ActivityExecutionFailed}`,
|
|
24
|
+
eventType: OrchestrationEventType.ActivityExecutionFailed,
|
|
25
|
+
occurredAt: context.occurredAt,
|
|
26
|
+
correlationId: context.correlationId,
|
|
27
|
+
causationId: context.commandId,
|
|
28
|
+
actor: context.actor,
|
|
29
|
+
source: { kind: EventSourceKind.Internal, id: 'orchestration-service' },
|
|
30
|
+
payload: { activationId: input.activationId, runId: input.runId, reason: input.reason },
|
|
31
|
+
}),
|
|
32
|
+
workflowEventData({
|
|
33
|
+
eventId: `${context.commandId}:${OrchestrationEventType.InstanceBlocked}`,
|
|
34
|
+
eventType: OrchestrationEventType.InstanceBlocked,
|
|
35
|
+
occurredAt: context.occurredAt,
|
|
36
|
+
correlationId: context.correlationId,
|
|
37
|
+
causationId: context.commandId,
|
|
38
|
+
actor: context.actor,
|
|
39
|
+
source: { kind: EventSourceKind.Internal, id: 'orchestration-service' },
|
|
40
|
+
payload: { reason: input.reason },
|
|
41
|
+
}),
|
|
42
|
+
]);
|
|
43
|
+
return (await repository.load(id)).view;
|
|
44
|
+
}
|
|
45
|
+
export function isAmbiguityResolutionBlock(reason) {
|
|
46
|
+
return isAmbiguityResolutionBlockReason(reason);
|
|
47
|
+
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { ActivityOutcomeKind } from '../../../activities/index.js';
|
|
2
2
|
import type { ManualTriggerDisposition, TriggerRequestDisposition, TriggerSkipReason, TriggerSourceKind } from '../../../control-plane/index.js';
|
|
3
|
-
import type { RunStatus } from '../../../execution/index.js';
|
|
3
|
+
import type { RecoveryBlockReasonPrefix, RunStatus } from '../../../execution/index.js';
|
|
4
|
+
import type { WorkflowStatus } from '../../../orchestration/index.js';
|
|
4
5
|
import type { ApiCommandStatus } from './control-plane.js';
|
|
5
6
|
import type { TriggerCommandName } from './triggers.js';
|
|
6
7
|
export declare const AcceptedCommandStatusValue: {
|
|
@@ -53,4 +54,7 @@ export declare const TriggerFinishedStatusValue: {
|
|
|
53
54
|
};
|
|
54
55
|
export declare const ManualTriggerDispositionValue: typeof ManualTriggerDisposition;
|
|
55
56
|
export declare const TriggerCommandNameValue: typeof TriggerCommandName;
|
|
57
|
+
export declare const WorkflowStatusValue: typeof WorkflowStatus;
|
|
58
|
+
export declare const RecoveryBlockReasonPrefixValue: typeof RecoveryBlockReasonPrefix;
|
|
59
|
+
export declare function isAmbiguityResolutionBlockReasonValue(reason: string | undefined): boolean;
|
|
56
60
|
export {};
|
|
@@ -96,3 +96,16 @@ export const TriggerCommandNameValue = {
|
|
|
96
96
|
Resume: triggerCommands[1],
|
|
97
97
|
Run: triggerCommands[2],
|
|
98
98
|
};
|
|
99
|
+
export const WorkflowStatusValue = defineClosedVocabulary({
|
|
100
|
+
Active: 'active',
|
|
101
|
+
Waiting: 'waiting',
|
|
102
|
+
Completed: 'completed',
|
|
103
|
+
Blocked: 'blocked',
|
|
104
|
+
Superseded: 'superseded',
|
|
105
|
+
});
|
|
106
|
+
const recoveryBlockReasonShape = { 'execution-recovery-ambiguous:': true };
|
|
107
|
+
export const RecoveryBlockReasonPrefixValue = Object.keys(recoveryBlockReasonShape)[0];
|
|
108
|
+
export function isAmbiguityResolutionBlockReasonValue(reason) {
|
|
109
|
+
return (reason?.startsWith(RecoveryBlockReasonPrefixValue) === true ||
|
|
110
|
+
(reason !== undefined && /^run-ambiguous-after-\d+-attempts$/.test(reason)));
|
|
111
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { TriggerConfig, TriggerInvocationView, TriggerSkipDiagnostic, TriggerView } from '../../../control-plane/index.js';
|
|
2
2
|
import type { ValueOf } from '../../../kernel/index.js';
|
|
3
|
+
import type { WorkflowStatus } from '../../../orchestration/index.js';
|
|
3
4
|
export declare const TriggerStatus: {
|
|
4
5
|
readonly Enabled: "running" | "paused" | "removed" | "enabled";
|
|
5
6
|
readonly Paused: "running" | "paused" | "removed" | "enabled";
|
|
@@ -15,13 +16,24 @@ export interface TriggerResponse {
|
|
|
15
16
|
readonly author: string;
|
|
16
17
|
readonly occurredAt: string;
|
|
17
18
|
} | null;
|
|
19
|
+
readonly activeInvocation: {
|
|
20
|
+
readonly activeWorkItemId: string;
|
|
21
|
+
readonly workItemKey: string;
|
|
22
|
+
readonly triggerId: string;
|
|
23
|
+
readonly workflowInstanceId: string;
|
|
24
|
+
readonly workflowStatus: WorkflowStatus | null;
|
|
25
|
+
readonly blockReason: string | null;
|
|
26
|
+
readonly retryEligible: boolean;
|
|
27
|
+
} | null;
|
|
18
28
|
readonly triggerId: string;
|
|
19
29
|
readonly status: TriggerStatus;
|
|
20
30
|
readonly definition: TriggerConfig | null;
|
|
21
31
|
readonly paused: boolean;
|
|
22
32
|
readonly pauseReason: string | null;
|
|
23
33
|
readonly nextOccurrence: string | null;
|
|
24
|
-
readonly pending: readonly TriggerView['coalesced'][string]
|
|
34
|
+
readonly pending: readonly (TriggerView['coalesced'][string] & {
|
|
35
|
+
readonly activeWorkItemId: string | null;
|
|
36
|
+
})[];
|
|
25
37
|
readonly latestSkip: TriggerSkipDiagnostic | null;
|
|
26
38
|
readonly invocations: readonly (TriggerInvocationView & {
|
|
27
39
|
readonly workItemKey: string;
|