@shipfox/api-workflows 12.7.0 → 13.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +8 -8
- package/CHANGELOG.md +9 -0
- package/dist/db/index.d.ts +1 -1
- package/dist/db/index.d.ts.map +1 -1
- package/dist/db/index.js +1 -1
- package/dist/db/index.js.map +1 -1
- package/dist/db/job-listeners.d.ts.map +1 -1
- package/dist/db/job-listeners.js +17 -1
- package/dist/db/job-listeners.js.map +1 -1
- package/dist/db/workflow-runs/job-executions.d.ts +2 -3
- package/dist/db/workflow-runs/job-executions.d.ts.map +1 -1
- package/dist/db/workflow-runs/job-executions.js +48 -20
- package/dist/db/workflow-runs/job-executions.js.map +1 -1
- package/dist/db/workflow-runs/outbox.d.ts +15 -0
- package/dist/db/workflow-runs/outbox.d.ts.map +1 -1
- package/dist/db/workflow-runs/outbox.js +35 -1
- package/dist/db/workflow-runs/outbox.js.map +1 -1
- package/dist/db/workflow-runs/run-graph.d.ts.map +1 -1
- package/dist/db/workflow-runs/run-graph.js +12 -0
- package/dist/db/workflow-runs/run-graph.js.map +1 -1
- package/dist/db/workflow-runs/run-status.d.ts.map +1 -1
- package/dist/db/workflow-runs/run-status.js +13 -1
- package/dist/db/workflow-runs/run-status.js.map +1 -1
- package/dist/db/workflow-runs.d.ts +2 -2
- package/dist/db/workflow-runs.d.ts.map +1 -1
- package/dist/db/workflow-runs.js +2 -2
- package/dist/db/workflow-runs.js.map +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -4
- package/dist/index.js.map +1 -1
- package/dist/presentation/index.d.ts +1 -1
- package/dist/presentation/index.d.ts.map +1 -1
- package/dist/presentation/index.js +1 -1
- package/dist/presentation/index.js.map +1 -1
- package/dist/presentation/subscribers/index.d.ts +0 -1
- package/dist/presentation/subscribers/index.d.ts.map +1 -1
- package/dist/presentation/subscribers/index.js +0 -1
- package/dist/presentation/subscribers/index.js.map +1 -1
- package/dist/temporal/activities/index.d.ts +2 -18
- package/dist/temporal/activities/index.d.ts.map +1 -1
- package/dist/temporal/activities/index.js +3 -5
- package/dist/temporal/activities/index.js.map +1 -1
- package/dist/temporal/activities/orchestration-activities.d.ts +5 -14
- package/dist/temporal/activities/orchestration-activities.d.ts.map +1 -1
- package/dist/temporal/activities/orchestration-activities.js +13 -34
- package/dist/temporal/activities/orchestration-activities.js.map +1 -1
- package/dist/temporal/workflows/index.bundle.js +20 -68
- package/dist/temporal/workflows/job-execution-orchestration.d.ts +0 -3
- package/dist/temporal/workflows/job-execution-orchestration.d.ts.map +1 -1
- package/dist/temporal/workflows/job-execution-orchestration.js +17 -41
- package/dist/temporal/workflows/job-execution-orchestration.js.map +1 -1
- package/dist/temporal/workflows/job-listener-orchestration.d.ts +0 -3
- package/dist/temporal/workflows/job-listener-orchestration.d.ts.map +1 -1
- package/dist/temporal/workflows/job-listener-orchestration.js +1 -9
- package/dist/temporal/workflows/job-listener-orchestration.js.map +1 -1
- package/dist/temporal/workflows/run-orchestration.d.ts.map +1 -1
- package/dist/temporal/workflows/run-orchestration.js +2 -18
- package/dist/temporal/workflows/run-orchestration.js.map +1 -1
- package/dist/temporal/workflows/test-env.d.ts +4 -6
- package/dist/temporal/workflows/test-env.d.ts.map +1 -1
- package/dist/temporal/workflows/test-env.js +13 -24
- package/dist/temporal/workflows/test-env.js.map +1 -1
- package/dist/tsconfig.test.tsbuildinfo +1 -1
- package/package.json +3 -3
- package/src/db/index.ts +1 -1
- package/src/db/job-listeners.test.ts +21 -0
- package/src/db/job-listeners.ts +18 -0
- package/src/db/workflow-runs/job-executions.test.ts +110 -12
- package/src/db/workflow-runs/job-executions.ts +54 -29
- package/src/db/workflow-runs/outbox.ts +63 -0
- package/src/db/workflow-runs/run-graph.ts +17 -0
- package/src/db/workflow-runs/run-status.test.ts +13 -0
- package/src/db/workflow-runs/run-status.ts +15 -1
- package/src/db/workflow-runs.ts +3 -1
- package/src/index.ts +1 -5
- package/src/presentation/index.ts +0 -1
- package/src/presentation/subscribers/index.ts +0 -1
- package/src/temporal/activities/index.ts +2 -8
- package/src/temporal/activities/orchestration-activities.test.ts +17 -1
- package/src/temporal/activities/orchestration-activities.ts +13 -46
- package/src/temporal/workflows/job-execution-orchestration.test.ts +6 -25
- package/src/temporal/workflows/job-execution-orchestration.ts +17 -43
- package/src/temporal/workflows/job-listener-orchestration-continuation.test.ts +0 -3
- package/src/temporal/workflows/job-listener-orchestration.test.ts +9 -12
- package/src/temporal/workflows/job-listener-orchestration.ts +0 -8
- package/src/temporal/workflows/run-orchestration.test.ts +22 -33
- package/src/temporal/workflows/run-orchestration.ts +4 -28
- package/src/temporal/workflows/test-env.ts +14 -34
- package/test/fixtures/runners-inter-module.ts +0 -3
- package/test/helpers/workflow-runs.ts +25 -0
- package/tsconfig.build.tsbuildinfo +1 -1
- package/dist/presentation/subscribers/on-runner-job-queued.d.ts +0 -3
- package/dist/presentation/subscribers/on-runner-job-queued.d.ts.map +0 -1
- package/dist/presentation/subscribers/on-runner-job-queued.js +0 -20
- package/dist/presentation/subscribers/on-runner-job-queued.js.map +0 -1
- package/src/presentation/subscribers/on-runner-job-queued.test.ts +0 -57
- package/src/presentation/subscribers/on-runner-job-queued.ts +0 -23
|
@@ -1,20 +1,17 @@
|
|
|
1
1
|
import type {AgentInterModuleClient} from '@shipfox/api-agent-dto/inter-module';
|
|
2
2
|
import type {IntegrationsModuleClient} from '@shipfox/api-integration-core-dto/inter-module';
|
|
3
3
|
import type {ProjectsModuleClient} from '@shipfox/api-projects-dto/inter-module';
|
|
4
|
-
import type {RunnersInterModuleClient} from '@shipfox/api-runners-dto/inter-module';
|
|
5
4
|
import type {SecretsInterModuleClient} from '@shipfox/api-secrets-dto/inter-module';
|
|
6
5
|
import {
|
|
7
6
|
activateJobListenerActivity,
|
|
8
7
|
bulkSetStepStatuses,
|
|
9
|
-
createCancelRunnerJobsActivity,
|
|
10
8
|
createDrainListenerEventsActivity,
|
|
11
|
-
createEnqueueJobExecutionForRunner,
|
|
12
|
-
createReleaseLeaseActivity,
|
|
13
9
|
evaluateJobActivationsActivity,
|
|
14
10
|
failJobExecutionAsTimedOutActivity,
|
|
15
11
|
failRunAsTimedOutActivity,
|
|
16
12
|
loadRunAttemptDag,
|
|
17
13
|
peekListenerBufferActivity,
|
|
14
|
+
queueJobExecutionActivity,
|
|
18
15
|
recordListenerFiringOutcomeActivity,
|
|
19
16
|
resolveJobListenerActivity,
|
|
20
17
|
resolveJobStatusFromJobExecutionsActivity,
|
|
@@ -29,7 +26,6 @@ export function createOrchestrationActivities(params: {
|
|
|
29
26
|
agent: AgentInterModuleClient;
|
|
30
27
|
integrations: IntegrationsModuleClient;
|
|
31
28
|
projects: ProjectsModuleClient;
|
|
32
|
-
runners: RunnersInterModuleClient;
|
|
33
29
|
secrets: Pick<SecretsInterModuleClient, 'getVariablesByNamespace'>;
|
|
34
30
|
}) {
|
|
35
31
|
return {
|
|
@@ -39,8 +35,7 @@ export function createOrchestrationActivities(params: {
|
|
|
39
35
|
setJobExecutionStatus: async (activityParams: Parameters<typeof setJobExecutionStatus>[0]) =>
|
|
40
36
|
await setJobExecutionStatus(activityParams, params.secrets),
|
|
41
37
|
bulkSetStepStatuses,
|
|
42
|
-
|
|
43
|
-
enqueueJobExecutionForRunner: createEnqueueJobExecutionForRunner(params.runners),
|
|
38
|
+
queueJobExecutionActivity,
|
|
44
39
|
evaluateJobActivationsActivity,
|
|
45
40
|
failJobExecutionAsTimedOutActivity: async (
|
|
46
41
|
activityParams: Parameters<typeof failJobExecutionAsTimedOutActivity>[0],
|
|
@@ -61,6 +56,5 @@ export function createOrchestrationActivities(params: {
|
|
|
61
56
|
activityParams: Parameters<typeof resolveLeaseExpiredJobExecutionActivity>[0],
|
|
62
57
|
) => await resolveLeaseExpiredJobExecutionActivity(activityParams, params.secrets),
|
|
63
58
|
resolveJobStatusFromJobExecutionsActivity,
|
|
64
|
-
releaseLeaseActivity: createReleaseLeaseActivity(params.runners),
|
|
65
59
|
};
|
|
66
60
|
}
|
|
@@ -8,7 +8,11 @@ import {
|
|
|
8
8
|
import {createTestSecretsClient} from '#test/fixtures/secrets-inter-module.js';
|
|
9
9
|
import {stripSetupStep} from '#test/fixtures/strip-setup-step.js';
|
|
10
10
|
import {workflowModel} from '#test/index.js';
|
|
11
|
-
import {
|
|
11
|
+
import {
|
|
12
|
+
queueJobExecutionActivity,
|
|
13
|
+
resolveLeaseExpiredJobExecutionActivity,
|
|
14
|
+
setJobStatus,
|
|
15
|
+
} from './orchestration-activities.js';
|
|
12
16
|
|
|
13
17
|
let workspaceId: string;
|
|
14
18
|
let projectId: string;
|
|
@@ -74,6 +78,18 @@ describe('resolveLeaseExpiredJobExecutionActivity', () => {
|
|
|
74
78
|
});
|
|
75
79
|
});
|
|
76
80
|
|
|
81
|
+
describe('queueJobExecutionActivity', () => {
|
|
82
|
+
test('a missing execution fails non-retryably', async () => {
|
|
83
|
+
const error = await queueJobExecutionActivity({
|
|
84
|
+
jobId: crypto.randomUUID(),
|
|
85
|
+
jobExecutionId: crypto.randomUUID(),
|
|
86
|
+
}).catch((err: unknown) => err);
|
|
87
|
+
|
|
88
|
+
expect(error).toBeInstanceOf(ApplicationFailure);
|
|
89
|
+
expect((error as ApplicationFailure).nonRetryable).toBe(true);
|
|
90
|
+
});
|
|
91
|
+
});
|
|
92
|
+
|
|
77
93
|
describe('setJobStatus', () => {
|
|
78
94
|
test('default-gate skipped jobs record an evaluation trace', async () => {
|
|
79
95
|
const run = await createWorkflowRun({
|
|
@@ -1,12 +1,7 @@
|
|
|
1
1
|
import type {AgentInterModuleClient} from '@shipfox/api-agent-dto/inter-module';
|
|
2
2
|
import type {IntegrationsModuleClient} from '@shipfox/api-integration-core-dto/inter-module';
|
|
3
3
|
import type {ProjectsModuleClient} from '@shipfox/api-projects-dto/inter-module';
|
|
4
|
-
import {
|
|
5
|
-
type RunnersInterModuleClient,
|
|
6
|
-
runnersInterModuleContract,
|
|
7
|
-
} from '@shipfox/api-runners-dto/inter-module';
|
|
8
4
|
import type {SecretsInterModuleClient} from '@shipfox/api-secrets-dto/inter-module';
|
|
9
|
-
import {isInterModuleKnownError} from '@shipfox/inter-module';
|
|
10
5
|
import {ApplicationFailure} from '@temporalio/common';
|
|
11
6
|
import {defaultJobConditionTrace} from '#core/condition-trace.js';
|
|
12
7
|
import type {JobStatus, JobStatusReason, ResolutionReason} from '#core/entities/job.js';
|
|
@@ -31,6 +26,7 @@ import {
|
|
|
31
26
|
getWorkflowRunByAttemptId,
|
|
32
27
|
type JobActivationDecision,
|
|
33
28
|
peekListenerBuffer,
|
|
29
|
+
queueJobExecution,
|
|
34
30
|
resolveJobExecutionAfterLeaseExpiry,
|
|
35
31
|
resolveJobListener,
|
|
36
32
|
resolveJobStatusFromJobExecutions,
|
|
@@ -209,48 +205,19 @@ export async function resolveLeaseExpiredJobExecutionActivity(
|
|
|
209
205
|
}
|
|
210
206
|
}
|
|
211
207
|
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
await
|
|
218
|
-
|
|
219
|
-
}
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
return async function enqueueJobExecutionForRunner(params: {
|
|
223
|
-
workspaceId: string;
|
|
224
|
-
workflowRunId: string;
|
|
225
|
-
jobId: string;
|
|
226
|
-
jobExecutionId: string;
|
|
227
|
-
runAttemptId: string;
|
|
228
|
-
projectId: string;
|
|
229
|
-
requiredLabels: string[];
|
|
230
|
-
}): Promise<void> {
|
|
231
|
-
try {
|
|
232
|
-
await runners.enqueueJobExecution({
|
|
233
|
-
workspaceId: params.workspaceId,
|
|
234
|
-
workflowRunId: params.workflowRunId,
|
|
235
|
-
workflowRunAttemptId: params.runAttemptId,
|
|
236
|
-
jobId: params.jobId,
|
|
237
|
-
jobExecutionId: params.jobExecutionId,
|
|
238
|
-
projectId: params.projectId,
|
|
239
|
-
requiredLabels: params.requiredLabels,
|
|
240
|
-
});
|
|
241
|
-
} catch (err) {
|
|
242
|
-
if (isInterModuleKnownError(runnersInterModuleContract.methods.enqueueJobExecution, err)) {
|
|
243
|
-
throw ApplicationFailure.nonRetryable(err.message, err.name);
|
|
244
|
-
}
|
|
245
|
-
throw err;
|
|
208
|
+
export async function queueJobExecutionActivity(params: {
|
|
209
|
+
jobId: string;
|
|
210
|
+
jobExecutionId: string;
|
|
211
|
+
}): Promise<{newVersion: number; status: Exclude<JobStatus, 'skipped'>}> {
|
|
212
|
+
try {
|
|
213
|
+
const execution = await queueJobExecution({jobExecutionId: params.jobExecutionId});
|
|
214
|
+
return {newVersion: execution.version, status: execution.status};
|
|
215
|
+
} catch (err) {
|
|
216
|
+
if (err instanceof JobNotFoundError) {
|
|
217
|
+
throw ApplicationFailure.nonRetryable(err.message, err.name);
|
|
246
218
|
}
|
|
247
|
-
|
|
248
|
-
}
|
|
249
|
-
|
|
250
|
-
export function createCancelRunnerJobsActivity(runners: RunnersInterModuleClient) {
|
|
251
|
-
return async function cancelRunnerJobsActivity(params: {jobIds: string[]}): Promise<void> {
|
|
252
|
-
await runners.cancelJobs(params);
|
|
253
|
-
};
|
|
219
|
+
throw err;
|
|
220
|
+
}
|
|
254
221
|
}
|
|
255
222
|
|
|
256
223
|
export async function failJobExecutionAsTimedOutActivity(
|
|
@@ -107,7 +107,7 @@ async function expectEmptyRequiredLabelsFailure(input: typeof defaultJobInput):
|
|
|
107
107
|
}
|
|
108
108
|
|
|
109
109
|
describe('jobExecutionOrchestration', () => {
|
|
110
|
-
test('runner claim flips status to running, then
|
|
110
|
+
test('runner claim flips status to running, then the terminal fact finishes the execution', async () => {
|
|
111
111
|
setCfg({
|
|
112
112
|
dag: makeDag([dagJob('job-1', 'build')]),
|
|
113
113
|
jobResults: new Map([['job-1', 'succeeded']]),
|
|
@@ -117,7 +117,6 @@ describe('jobExecutionOrchestration', () => {
|
|
|
117
117
|
|
|
118
118
|
expect(result.status).toBe('succeeded');
|
|
119
119
|
expect(finalStatusesFor('job-1')).toEqual(['running', 'succeeded']);
|
|
120
|
-
expect(callsNamed('releaseLeaseActivity')).toHaveLength(1);
|
|
121
120
|
expect(callsNamed('resolveLeaseExpiredJobExecutionActivity')).toHaveLength(0);
|
|
122
121
|
expect(callsNamed('bulkSetStepStatuses')).toHaveLength(0);
|
|
123
122
|
});
|
|
@@ -238,7 +237,7 @@ describe('jobExecutionOrchestration', () => {
|
|
|
238
237
|
});
|
|
239
238
|
|
|
240
239
|
expect(setExecutionStatusCalls()).toHaveLength(0);
|
|
241
|
-
expect(callsNamed('
|
|
240
|
+
expect(callsNamed('queueJobExecutionActivity')).toHaveLength(0);
|
|
242
241
|
});
|
|
243
242
|
|
|
244
243
|
test('whitespace-only required labels fail before the job is marked running', async () => {
|
|
@@ -254,7 +253,7 @@ describe('jobExecutionOrchestration', () => {
|
|
|
254
253
|
});
|
|
255
254
|
|
|
256
255
|
expect(setExecutionStatusCalls()).toHaveLength(0);
|
|
257
|
-
expect(callsNamed('
|
|
256
|
+
expect(callsNamed('queueJobExecutionActivity')).toHaveLength(0);
|
|
258
257
|
});
|
|
259
258
|
|
|
260
259
|
test('finished signal (failed) flips status without sweeping steps', async () => {
|
|
@@ -265,7 +264,6 @@ describe('jobExecutionOrchestration', () => {
|
|
|
265
264
|
expect(result.status).toBe('failed');
|
|
266
265
|
expect(finalStatusesFor('job-2')).toEqual(['running', 'failed']);
|
|
267
266
|
expect(terminalSetJobCall('job-2')?.params.statusReason).toBe('step_failed');
|
|
268
|
-
expect(callsNamed('releaseLeaseActivity')).toHaveLength(1);
|
|
269
267
|
expect(callsNamed('bulkSetStepStatuses')).toHaveLength(0);
|
|
270
268
|
});
|
|
271
269
|
|
|
@@ -289,10 +287,9 @@ describe('jobExecutionOrchestration', () => {
|
|
|
289
287
|
statusReason: 'step_failed',
|
|
290
288
|
},
|
|
291
289
|
});
|
|
292
|
-
expect(callsNamed('releaseLeaseActivity')).toHaveLength(1);
|
|
293
290
|
});
|
|
294
291
|
|
|
295
|
-
test('lease-expired signal resolves via the activity
|
|
292
|
+
test('lease-expired signal resolves via the activity', async () => {
|
|
296
293
|
setCfg({
|
|
297
294
|
dag: makeDag([]),
|
|
298
295
|
jobResults: new Map(),
|
|
@@ -304,7 +301,6 @@ describe('jobExecutionOrchestration', () => {
|
|
|
304
301
|
|
|
305
302
|
expect(result.status).toBe('failed');
|
|
306
303
|
expect(callsNamed('resolveLeaseExpiredJobExecutionActivity')).toHaveLength(1);
|
|
307
|
-
expect(callsNamed('releaseLeaseActivity')).toHaveLength(1);
|
|
308
304
|
expect(callsNamed('failJobExecutionAsTimedOutActivity')).toHaveLength(0);
|
|
309
305
|
});
|
|
310
306
|
|
|
@@ -400,24 +396,10 @@ describe('jobExecutionOrchestration', () => {
|
|
|
400
396
|
const result = await executeJob({...defaultJobInput, jobId: 'job-cancelled'});
|
|
401
397
|
|
|
402
398
|
expect(result).toMatchObject({status: 'cancelled'});
|
|
403
|
-
expect(callsNamed('
|
|
404
|
-
expect(callsNamed('releaseLeaseActivity')).toHaveLength(0);
|
|
399
|
+
expect(callsNamed('queueJobExecutionActivity')).toHaveLength(1);
|
|
405
400
|
});
|
|
406
401
|
|
|
407
|
-
test('
|
|
408
|
-
setCfg({
|
|
409
|
-
dag: makeDag([]),
|
|
410
|
-
jobResults: new Map([['job-rl', 'succeeded']]),
|
|
411
|
-
releaseLeaseError: 'runners db down',
|
|
412
|
-
});
|
|
413
|
-
|
|
414
|
-
const result = await executeJob({...defaultJobInput, jobId: 'job-rl'});
|
|
415
|
-
|
|
416
|
-
expect(result.status).toBe('succeeded');
|
|
417
|
-
expect(callsNamed('releaseLeaseActivity').length).toBeGreaterThanOrEqual(1);
|
|
418
|
-
});
|
|
419
|
-
|
|
420
|
-
test('times out after the execution deadline and fails its steps without releasing the lease', async () => {
|
|
402
|
+
test('times out after the execution deadline and fails its steps', async () => {
|
|
421
403
|
setCfg({
|
|
422
404
|
dag: makeDag([dagJob('job-timeout', 'build')]),
|
|
423
405
|
jobResults: new Map(),
|
|
@@ -437,7 +419,6 @@ describe('jobExecutionOrchestration', () => {
|
|
|
437
419
|
name: 'bulkSetStepStatuses',
|
|
438
420
|
params: {jobExecutionId: 'job-timeout', status: 'failed'},
|
|
439
421
|
});
|
|
440
|
-
expect(callsNamed('releaseLeaseActivity')).toHaveLength(0);
|
|
441
422
|
expect(callsNamed('resolveLeaseExpiredJobExecutionActivity')).toHaveLength(0);
|
|
442
423
|
});
|
|
443
424
|
|
|
@@ -26,14 +26,14 @@ import {remainingMs} from './deadline.js';
|
|
|
26
26
|
* execution pending while it is queued, then performs the versioned pending → running
|
|
27
27
|
* transition after the current claim. One deadline spans both waits; claim never resets
|
|
28
28
|
* it. Signals can arrive in any order, so the precedence is finished > lease expired >
|
|
29
|
-
* claimed > timeout.
|
|
30
|
-
*
|
|
29
|
+
* claimed > timeout. Workflows commits queue and terminal facts with its state transitions;
|
|
30
|
+
* runners reconciles its local queue, lease, and reservation projections asynchronously.
|
|
31
31
|
*/
|
|
32
32
|
|
|
33
33
|
const {
|
|
34
34
|
setJobStatus,
|
|
35
35
|
setJobExecutionStatus,
|
|
36
|
-
|
|
36
|
+
queueJobExecutionActivity,
|
|
37
37
|
bulkSetStepStatuses,
|
|
38
38
|
failJobExecutionAsTimedOutActivity,
|
|
39
39
|
resolveLeaseExpiredJobExecutionActivity,
|
|
@@ -48,14 +48,6 @@ const {resolveJobStatusFromJobExecutionsActivity} = proxyActivities<
|
|
|
48
48
|
retry: {maximumAttempts: 5},
|
|
49
49
|
});
|
|
50
50
|
|
|
51
|
-
// Lease cleanup gets a bounded retry policy of its own: after a few attempts the
|
|
52
|
-
// workflow stops waiting on it (see releaseLeaseBestEffort) rather than blocking
|
|
53
|
-
// the DAG on runners-side availability.
|
|
54
|
-
const {releaseLeaseActivity} = proxyActivities<ReturnType<typeof createOrchestrationActivities>>({
|
|
55
|
-
startToCloseTimeout: '30s',
|
|
56
|
-
retry: {maximumAttempts: 5},
|
|
57
|
-
});
|
|
58
|
-
|
|
59
51
|
const DEFAULT_EXECUTION_MAX_DURATION_MS = 6 * 60 * 60 * 1000;
|
|
60
52
|
|
|
61
53
|
export const jobFinishedSignal =
|
|
@@ -71,12 +63,9 @@ export interface JobClaimedSignalPayload {
|
|
|
71
63
|
export const jobClaimedSignal = defineSignal<[JobClaimedSignalPayload]>(JOB_CLAIMED_SIGNAL);
|
|
72
64
|
|
|
73
65
|
export interface JobExecutionOrchestrationInput {
|
|
74
|
-
workspaceId: string;
|
|
75
|
-
workflowRunId: string;
|
|
76
66
|
runAttemptId: string;
|
|
77
67
|
jobId: string;
|
|
78
68
|
jobExecutionId: string;
|
|
79
|
-
projectId: string;
|
|
80
69
|
jobVersion: number;
|
|
81
70
|
executionVersion: number;
|
|
82
71
|
executionTimeoutMs?: number | null | undefined;
|
|
@@ -89,17 +78,6 @@ export interface JobExecutionOrchestrationResult {
|
|
|
89
78
|
jobVersion: number;
|
|
90
79
|
}
|
|
91
80
|
|
|
92
|
-
async function releaseLeaseBestEffort(jobExecutionId: string): Promise<void> {
|
|
93
|
-
try {
|
|
94
|
-
await releaseLeaseActivity({jobExecutionId});
|
|
95
|
-
} catch (err) {
|
|
96
|
-
log.warn('lease release failed; stuck detector will reap the row', {
|
|
97
|
-
jobExecutionId,
|
|
98
|
-
error: String(err),
|
|
99
|
-
});
|
|
100
|
-
}
|
|
101
|
-
}
|
|
102
|
-
|
|
103
81
|
async function resolveJobStatusOrFailClosed(
|
|
104
82
|
input: JobExecutionOrchestrationInput,
|
|
105
83
|
): Promise<{status: RuntimeCompletionStatus; jobVersion: number}> {
|
|
@@ -121,7 +99,7 @@ async function resolveJobStatusOrFailClosed(
|
|
|
121
99
|
}
|
|
122
100
|
}
|
|
123
101
|
|
|
124
|
-
async function
|
|
102
|
+
async function queueJobExecution(input: JobExecutionOrchestrationInput) {
|
|
125
103
|
if (hasNoRequiredRunnerLabels(input.requiredLabels)) {
|
|
126
104
|
throw ApplicationFailure.nonRetryable(
|
|
127
105
|
`Job ${input.jobId} has no required runner labels`,
|
|
@@ -129,15 +107,12 @@ async function enqueueJobExecution(input: JobExecutionOrchestrationInput): Promi
|
|
|
129
107
|
);
|
|
130
108
|
}
|
|
131
109
|
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
projectId: input.projectId,
|
|
139
|
-
requiredLabels: input.requiredLabels,
|
|
140
|
-
});
|
|
110
|
+
return jobExecutionStartOutcome(
|
|
111
|
+
await queueJobExecutionActivity({
|
|
112
|
+
jobId: input.jobId,
|
|
113
|
+
jobExecutionId: input.jobExecutionId,
|
|
114
|
+
}),
|
|
115
|
+
);
|
|
141
116
|
}
|
|
142
117
|
|
|
143
118
|
async function markJobExecutionRunning(
|
|
@@ -218,7 +193,6 @@ async function resolveFinishedJobExecution({
|
|
|
218
193
|
terminationReason: 'finished',
|
|
219
194
|
status,
|
|
220
195
|
});
|
|
221
|
-
await releaseLeaseBestEffort(input.jobExecutionId);
|
|
222
196
|
return {status, jobVersion: input.jobVersion};
|
|
223
197
|
}
|
|
224
198
|
const resolved = await resolveJobStatusOrFailClosed(input);
|
|
@@ -228,7 +202,6 @@ async function resolveFinishedJobExecution({
|
|
|
228
202
|
terminationReason: 'finished',
|
|
229
203
|
status: resolved.status,
|
|
230
204
|
});
|
|
231
|
-
await releaseLeaseBestEffort(input.jobExecutionId);
|
|
232
205
|
return resolved;
|
|
233
206
|
}
|
|
234
207
|
|
|
@@ -257,7 +230,6 @@ async function resolveLeaseExpiredJobExecution({
|
|
|
257
230
|
terminationReason: 'lease_expired',
|
|
258
231
|
status: leaseExpired.status,
|
|
259
232
|
});
|
|
260
|
-
await releaseLeaseBestEffort(input.jobExecutionId);
|
|
261
233
|
return {status: leaseExpired.status, jobVersion: input.jobVersion};
|
|
262
234
|
}
|
|
263
235
|
const {status, jobVersion} = await resolveJobStatusOrFailClosed(input);
|
|
@@ -267,13 +239,12 @@ async function resolveLeaseExpiredJobExecution({
|
|
|
267
239
|
terminationReason: 'lease_expired',
|
|
268
240
|
status,
|
|
269
241
|
});
|
|
270
|
-
await releaseLeaseBestEffort(input.jobExecutionId);
|
|
271
242
|
return {status, jobVersion};
|
|
272
243
|
}
|
|
273
244
|
|
|
274
|
-
// Timeout backstop. The activity atomically fails the execution, marks
|
|
275
|
-
//
|
|
276
|
-
//
|
|
245
|
+
// Timeout backstop. The activity atomically fails the execution, marks `timed_out_at`, and
|
|
246
|
+
// enqueues the generic terminal execution fact. The runners subscriber then asks the runner to
|
|
247
|
+
// cancel. The lease is intentionally NOT released here.
|
|
277
248
|
async function resolveTimedOutJobExecution({
|
|
278
249
|
input,
|
|
279
250
|
runningVersion,
|
|
@@ -314,7 +285,10 @@ export async function jobExecutionOrchestration(
|
|
|
314
285
|
// Register every signal before enqueue can block or publish a claim/outcome event. The
|
|
315
286
|
// handlers retain signals that arrive while the enqueue activity is in flight.
|
|
316
287
|
registerJobExecutionSignalHandlers(input.jobExecutionId, signals);
|
|
317
|
-
await
|
|
288
|
+
const queued = await queueJobExecution(input);
|
|
289
|
+
if (queued.kind === 'terminal') {
|
|
290
|
+
return {status: queued.result.status, jobVersion: input.jobVersion};
|
|
291
|
+
}
|
|
318
292
|
|
|
319
293
|
const timeoutMs = input.executionTimeoutMs ?? DEFAULT_EXECUTION_MAX_DURATION_MS;
|
|
320
294
|
const deadline = Date.now() + timeoutMs;
|
|
@@ -63,9 +63,6 @@ describe('listener continue-as-new guard', () => {
|
|
|
63
63
|
|
|
64
64
|
it('carries listener loop state into the continued run', () => {
|
|
65
65
|
const input: JobListenerOrchestrationInput = {
|
|
66
|
-
workspaceId: 'workspace-1',
|
|
67
|
-
workflowRunId: 'run-1',
|
|
68
|
-
projectId: 'project-1',
|
|
69
66
|
runAttemptId: 'attempt-1',
|
|
70
67
|
jobId: 'job-1',
|
|
71
68
|
jobVersion: 3,
|
|
@@ -197,7 +197,7 @@ describe('jobListenerOrchestration', () => {
|
|
|
197
197
|
|
|
198
198
|
expect(listenerFiringOutcomeCalls()).toHaveLength(0);
|
|
199
199
|
expect(resolveJobListenerCalls().map((c) => c.params.reason)).toEqual(['until']);
|
|
200
|
-
expect(callsNamed('
|
|
200
|
+
expect(callsNamed('queueJobExecutionActivity')).toHaveLength(0);
|
|
201
201
|
});
|
|
202
202
|
|
|
203
203
|
test('resolves with timeout when the listening deadline elapses', async () => {
|
|
@@ -233,9 +233,10 @@ describe('jobListenerOrchestration', () => {
|
|
|
233
233
|
|
|
234
234
|
await runListener({maxExecutions: 1});
|
|
235
235
|
|
|
236
|
-
expect(callsNamed('
|
|
237
|
-
expect(callsNamed('
|
|
238
|
-
|
|
236
|
+
expect(callsNamed('queueJobExecutionActivity')).toHaveLength(1);
|
|
237
|
+
expect(callsNamed('queueJobExecutionActivity')[0]?.params).toMatchObject({
|
|
238
|
+
jobId,
|
|
239
|
+
jobExecutionId: 'exec-1',
|
|
239
240
|
});
|
|
240
241
|
expect(listenerFiringOutcomeCalls().map((c) => c.params.outcome)).toEqual(['succeeded']);
|
|
241
242
|
});
|
|
@@ -250,7 +251,7 @@ describe('jobListenerOrchestration', () => {
|
|
|
250
251
|
await runListener({maxExecutions: 1});
|
|
251
252
|
|
|
252
253
|
// The child actually ran (enqueued), then returned failed.
|
|
253
|
-
expect(callsNamed('
|
|
254
|
+
expect(callsNamed('queueJobExecutionActivity')).toHaveLength(1);
|
|
254
255
|
expect(listenerFiringOutcomeCalls().map((c) => c.params.outcome)).toEqual(['failed']);
|
|
255
256
|
expect(resolveJobListenerCalls().map((c) => c.params.reason)).toEqual(['max_executions']);
|
|
256
257
|
});
|
|
@@ -265,7 +266,7 @@ describe('jobListenerOrchestration', () => {
|
|
|
265
266
|
await runListener({maxExecutions: 2});
|
|
266
267
|
|
|
267
268
|
// Sequence 1 failed at materialization (no child enqueued); sequence 2 ran and succeeded.
|
|
268
|
-
expect(callsNamed('
|
|
269
|
+
expect(callsNamed('queueJobExecutionActivity')).toHaveLength(1);
|
|
269
270
|
expect(listenerFiringOutcomeCalls().map((c) => c.params.outcome)).toEqual([
|
|
270
271
|
'failed',
|
|
271
272
|
'succeeded',
|
|
@@ -369,15 +370,12 @@ describe('jobListenerOrchestration', () => {
|
|
|
369
370
|
});
|
|
370
371
|
|
|
371
372
|
const handle = await startListener({onResolve: 'cancel'});
|
|
372
|
-
await waitForActivity('
|
|
373
|
+
await waitForActivity('queueJobExecutionActivity');
|
|
373
374
|
await handle.signal(LISTENER_RESOLVE_SIGNAL);
|
|
374
375
|
await handle.result();
|
|
375
376
|
|
|
376
377
|
expect(settleListenerCalls().map((c) => c.params.status)).toEqual(['cancelled']);
|
|
377
378
|
expect(listenerFiringOutcomeCalls().map((c) => c.params.outcome)).toEqual(['cancelled']);
|
|
378
|
-
expect(callsNamed('cancelRunnerJobsActivity')).toEqual([
|
|
379
|
-
{name: 'cancelRunnerJobsActivity', params: {jobIds: [jobId]}},
|
|
380
|
-
]);
|
|
381
379
|
expect(resolveJobListenerCalls().map((c) => c.params.reason)).toEqual(['until']);
|
|
382
380
|
});
|
|
383
381
|
|
|
@@ -390,7 +388,7 @@ describe('jobListenerOrchestration', () => {
|
|
|
390
388
|
});
|
|
391
389
|
|
|
392
390
|
const handle = await startListener({onResolve: 'finish'});
|
|
393
|
-
await waitForActivity('
|
|
391
|
+
await waitForActivity('queueJobExecutionActivity');
|
|
394
392
|
// Resolution arrives mid-firing; finish mode must wait for the child, not cancel it.
|
|
395
393
|
await handle.signal(LISTENER_RESOLVE_SIGNAL);
|
|
396
394
|
const child = testEnv.client.workflow.getHandle(`job:${jobId}`);
|
|
@@ -398,7 +396,6 @@ describe('jobListenerOrchestration', () => {
|
|
|
398
396
|
await handle.result();
|
|
399
397
|
|
|
400
398
|
expect(settleListenerCalls()).toHaveLength(0);
|
|
401
|
-
expect(callsNamed('cancelRunnerJobsActivity')).toHaveLength(0);
|
|
402
399
|
expect(listenerFiringOutcomeCalls().map((c) => c.params.outcome)).toEqual(['succeeded']);
|
|
403
400
|
expect(resolveJobListenerCalls().map((c) => c.params.reason)).toEqual(['until']);
|
|
404
401
|
});
|
|
@@ -25,7 +25,6 @@ const {
|
|
|
25
25
|
resolveJobListenerActivity,
|
|
26
26
|
settleListenerJobExecutionActivity,
|
|
27
27
|
recordListenerFiringOutcomeActivity,
|
|
28
|
-
cancelRunnerJobsActivity,
|
|
29
28
|
} = proxyActivities<ReturnType<typeof createOrchestrationActivities>>({
|
|
30
29
|
startToCloseTimeout: '30s',
|
|
31
30
|
});
|
|
@@ -34,9 +33,6 @@ export const listenerEventsAvailableSignal = defineSignal<[]>(LISTENER_EVENTS_AV
|
|
|
34
33
|
export const listenerResolveSignal = defineSignal<[]>(LISTENER_RESOLVE_SIGNAL);
|
|
35
34
|
|
|
36
35
|
export interface JobListenerOrchestrationInput {
|
|
37
|
-
workspaceId: string;
|
|
38
|
-
workflowRunId: string;
|
|
39
|
-
projectId: string;
|
|
40
36
|
runAttemptId: string;
|
|
41
37
|
jobId: string;
|
|
42
38
|
jobVersion: number;
|
|
@@ -413,9 +409,6 @@ async function runListenerExecution(params: {
|
|
|
413
409
|
workflowIdReusePolicy: 'ALLOW_DUPLICATE',
|
|
414
410
|
args: [
|
|
415
411
|
{
|
|
416
|
-
workspaceId: params.input.workspaceId,
|
|
417
|
-
workflowRunId: params.input.workflowRunId,
|
|
418
|
-
projectId: params.input.projectId,
|
|
419
412
|
jobId: params.input.jobId,
|
|
420
413
|
jobExecutionId: params.jobExecutionId,
|
|
421
414
|
runAttemptId: params.input.runAttemptId,
|
|
@@ -447,7 +440,6 @@ async function runListenerExecution(params: {
|
|
|
447
440
|
status: 'cancelled',
|
|
448
441
|
});
|
|
449
442
|
await recordListenerFiringOutcomeActivity({outcome: 'cancelled'});
|
|
450
|
-
await cancelRunnerJobsActivity({jobIds: [params.input.jobId]});
|
|
451
443
|
return;
|
|
452
444
|
}
|
|
453
445
|
}
|