@shipfox/api-workflows 12.6.0 → 13.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +8 -8
- package/CHANGELOG.md +23 -0
- package/dist/core/entities/workflow-run.d.ts +3 -0
- package/dist/core/entities/workflow-run.d.ts.map +1 -1
- package/dist/core/entities/workflow-run.js.map +1 -1
- package/dist/db/index.d.ts +1 -1
- package/dist/db/index.d.ts.map +1 -1
- package/dist/db/index.js +1 -1
- package/dist/db/index.js.map +1 -1
- package/dist/db/job-listeners.d.ts.map +1 -1
- package/dist/db/job-listeners.js +17 -1
- package/dist/db/job-listeners.js.map +1 -1
- package/dist/db/workflow-runs/job-executions.d.ts +2 -3
- package/dist/db/workflow-runs/job-executions.d.ts.map +1 -1
- package/dist/db/workflow-runs/job-executions.js +48 -20
- package/dist/db/workflow-runs/job-executions.js.map +1 -1
- package/dist/db/workflow-runs/outbox.d.ts +15 -0
- package/dist/db/workflow-runs/outbox.d.ts.map +1 -1
- package/dist/db/workflow-runs/outbox.js +35 -1
- package/dist/db/workflow-runs/outbox.js.map +1 -1
- package/dist/db/workflow-runs/queries.d.ts +5 -3
- package/dist/db/workflow-runs/queries.d.ts.map +1 -1
- package/dist/db/workflow-runs/queries.js +20 -6
- package/dist/db/workflow-runs/queries.js.map +1 -1
- package/dist/db/workflow-runs/run-graph.d.ts.map +1 -1
- package/dist/db/workflow-runs/run-graph.js +12 -0
- package/dist/db/workflow-runs/run-graph.js.map +1 -1
- package/dist/db/workflow-runs/run-status.d.ts.map +1 -1
- package/dist/db/workflow-runs/run-status.js +13 -1
- package/dist/db/workflow-runs/run-status.js.map +1 -1
- package/dist/db/workflow-runs.d.ts +2 -2
- package/dist/db/workflow-runs.d.ts.map +1 -1
- package/dist/db/workflow-runs.js +2 -2
- package/dist/db/workflow-runs.js.map +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -4
- package/dist/index.js.map +1 -1
- package/dist/presentation/dto/workflow-run.d.ts.map +1 -1
- package/dist/presentation/dto/workflow-run.js +4 -2
- package/dist/presentation/dto/workflow-run.js.map +1 -1
- package/dist/presentation/index.d.ts +1 -1
- package/dist/presentation/index.d.ts.map +1 -1
- package/dist/presentation/index.js +1 -1
- package/dist/presentation/index.js.map +1 -1
- package/dist/presentation/routes/get-run.d.ts.map +1 -1
- package/dist/presentation/routes/get-run.js +2 -1
- package/dist/presentation/routes/get-run.js.map +1 -1
- package/dist/presentation/subscribers/index.d.ts +0 -1
- package/dist/presentation/subscribers/index.d.ts.map +1 -1
- package/dist/presentation/subscribers/index.js +0 -1
- package/dist/presentation/subscribers/index.js.map +1 -1
- package/dist/temporal/activities/index.d.ts +2 -18
- package/dist/temporal/activities/index.d.ts.map +1 -1
- package/dist/temporal/activities/index.js +3 -5
- package/dist/temporal/activities/index.js.map +1 -1
- package/dist/temporal/activities/orchestration-activities.d.ts +5 -14
- package/dist/temporal/activities/orchestration-activities.d.ts.map +1 -1
- package/dist/temporal/activities/orchestration-activities.js +13 -34
- package/dist/temporal/activities/orchestration-activities.js.map +1 -1
- package/dist/temporal/workflows/index.bundle.js +20 -68
- package/dist/temporal/workflows/job-execution-orchestration.d.ts +0 -3
- package/dist/temporal/workflows/job-execution-orchestration.d.ts.map +1 -1
- package/dist/temporal/workflows/job-execution-orchestration.js +17 -41
- package/dist/temporal/workflows/job-execution-orchestration.js.map +1 -1
- package/dist/temporal/workflows/job-listener-orchestration.d.ts +0 -3
- package/dist/temporal/workflows/job-listener-orchestration.d.ts.map +1 -1
- package/dist/temporal/workflows/job-listener-orchestration.js +1 -9
- package/dist/temporal/workflows/job-listener-orchestration.js.map +1 -1
- package/dist/temporal/workflows/run-orchestration.d.ts.map +1 -1
- package/dist/temporal/workflows/run-orchestration.js +2 -18
- package/dist/temporal/workflows/run-orchestration.js.map +1 -1
- package/dist/temporal/workflows/test-env.d.ts +4 -6
- package/dist/temporal/workflows/test-env.d.ts.map +1 -1
- package/dist/temporal/workflows/test-env.js +13 -24
- package/dist/temporal/workflows/test-env.js.map +1 -1
- package/dist/tsconfig.test.tsbuildinfo +1 -1
- package/package.json +3 -3
- package/src/core/entities/workflow-run.ts +3 -0
- package/src/db/index.ts +1 -1
- package/src/db/job-listeners.test.ts +21 -0
- package/src/db/job-listeners.ts +18 -0
- package/src/db/workflow-runs/job-executions.test.ts +110 -12
- package/src/db/workflow-runs/job-executions.ts +54 -29
- package/src/db/workflow-runs/outbox.ts +63 -0
- package/src/db/workflow-runs/queries.test.ts +27 -0
- package/src/db/workflow-runs/queries.ts +31 -5
- package/src/db/workflow-runs/run-graph.ts +17 -0
- package/src/db/workflow-runs/run-status.test.ts +13 -0
- package/src/db/workflow-runs/run-status.ts +15 -1
- package/src/db/workflow-runs.ts +3 -1
- package/src/index.ts +1 -5
- package/src/presentation/dto/workflow-run.ts +7 -1
- package/src/presentation/index.ts +0 -1
- package/src/presentation/routes/get-run.test.ts +31 -0
- package/src/presentation/routes/get-run.ts +1 -0
- package/src/presentation/routes/list-runs.test.ts +5 -1
- package/src/presentation/subscribers/index.ts +0 -1
- package/src/temporal/activities/index.ts +2 -8
- package/src/temporal/activities/orchestration-activities.test.ts +17 -1
- package/src/temporal/activities/orchestration-activities.ts +13 -46
- package/src/temporal/workflows/job-execution-orchestration.test.ts +6 -25
- package/src/temporal/workflows/job-execution-orchestration.ts +17 -43
- package/src/temporal/workflows/job-listener-orchestration-continuation.test.ts +0 -3
- package/src/temporal/workflows/job-listener-orchestration.test.ts +9 -12
- package/src/temporal/workflows/job-listener-orchestration.ts +0 -8
- package/src/temporal/workflows/run-orchestration.test.ts +22 -33
- package/src/temporal/workflows/run-orchestration.ts +4 -28
- package/src/temporal/workflows/test-env.ts +14 -34
- package/test/fixtures/runners-inter-module.ts +0 -3
- package/test/helpers/workflow-runs.ts +25 -0
- package/tsconfig.build.tsbuildinfo +1 -1
- package/dist/presentation/subscribers/on-runner-job-queued.d.ts +0 -3
- package/dist/presentation/subscribers/on-runner-job-queued.d.ts.map +0 -1
- package/dist/presentation/subscribers/on-runner-job-queued.js +0 -20
- package/dist/presentation/subscribers/on-runner-job-queued.js.map +0 -1
- package/src/presentation/subscribers/on-runner-job-queued.test.ts +0 -57
- package/src/presentation/subscribers/on-runner-job-queued.ts +0 -23
|
@@ -4,6 +4,7 @@ import {nextStepForJob} from '#core/job-execution.js';
|
|
|
4
4
|
import {
|
|
5
5
|
buildModel,
|
|
6
6
|
createTestRun,
|
|
7
|
+
jobExecutionTerminatedEvents,
|
|
7
8
|
jobTerminatedEvents,
|
|
8
9
|
runCancelledEvents,
|
|
9
10
|
runTerminatedEvents,
|
|
@@ -16,6 +17,7 @@ import {
|
|
|
16
17
|
cancelWorkflowRun,
|
|
17
18
|
createRerunWorkflowRun,
|
|
18
19
|
createWorkflowRun,
|
|
20
|
+
getFirstJobExecutionByJobId,
|
|
19
21
|
getJobsByWorkflowRunId,
|
|
20
22
|
getStepsByJobId,
|
|
21
23
|
getWorkflowRunById,
|
|
@@ -257,6 +259,8 @@ describe('workflow run queries', () => {
|
|
|
257
259
|
const [runningJobExecution, succeededJob, skippedJob] = await getJobsByWorkflowRunId(run.id);
|
|
258
260
|
if (!runningJobExecution || !succeededJob || !skippedJob) throw new Error('Expected jobs');
|
|
259
261
|
await updateJobStatus({jobId: runningJobExecution.id, status: 'running', expectedVersion: 1});
|
|
262
|
+
const execution = await getFirstJobExecutionByJobId(runningJobExecution.id);
|
|
263
|
+
if (!execution) throw new Error('Expected running job execution');
|
|
260
264
|
await nextStepForJob(runningJobExecution.id);
|
|
261
265
|
await updateJobStatus({jobId: succeededJob.id, status: 'succeeded', expectedVersion: 1});
|
|
262
266
|
await updateJobStatus({
|
|
@@ -299,6 +303,15 @@ describe('workflow run queries', () => {
|
|
|
299
303
|
statusReason: 'run_cancelled',
|
|
300
304
|
}),
|
|
301
305
|
]);
|
|
306
|
+
expect(await jobExecutionTerminatedEvents(execution.id)).toEqual([
|
|
307
|
+
expect.objectContaining({
|
|
308
|
+
jobId: runningJobExecution.id,
|
|
309
|
+
jobExecutionId: execution.id,
|
|
310
|
+
workflowRunId: run.id,
|
|
311
|
+
status: 'cancelled',
|
|
312
|
+
statusReason: 'run_cancelled',
|
|
313
|
+
}),
|
|
314
|
+
]);
|
|
302
315
|
expect(await stepAttemptTerminatedEvents(runningJobExecution.id)).toHaveLength(1);
|
|
303
316
|
expect(await jobTerminatedEvents(succeededJob.id)).toHaveLength(1);
|
|
304
317
|
expect(await jobTerminatedEvents(skippedJob.id)).toHaveLength(1);
|
|
@@ -19,6 +19,7 @@ import {steps} from '../schema/steps.js';
|
|
|
19
19
|
import {workflowRunAttempts} from '../schema/workflow-run-attempts.js';
|
|
20
20
|
import {toWorkflowRun, workflowRuns} from '../schema/workflow-runs.js';
|
|
21
21
|
import {updateJobStatusAtVersion} from './jobs.js';
|
|
22
|
+
import {writeJobExecutionTerminatedOutbox} from './outbox.js';
|
|
22
23
|
import {
|
|
23
24
|
lockWorkflowRun,
|
|
24
25
|
TERMINAL_EXECUTION_STATUSES,
|
|
@@ -114,8 +115,21 @@ async function terminateRunAttempt(
|
|
|
114
115
|
notInArray(jobExecutions.status, TERMINAL_EXECUTION_STATUSES),
|
|
115
116
|
),
|
|
116
117
|
)
|
|
117
|
-
.returning({
|
|
118
|
+
.returning({
|
|
119
|
+
id: jobExecutions.id,
|
|
120
|
+
jobId: jobExecutions.jobId,
|
|
121
|
+
status: jobExecutions.status,
|
|
122
|
+
statusReason: jobExecutions.statusReason,
|
|
123
|
+
statusReasonMessage: jobExecutions.statusReasonMessage,
|
|
124
|
+
});
|
|
118
125
|
for (const jobExecution of terminatedExecutions) {
|
|
126
|
+
await writeJobExecutionTerminatedOutbox(tx, {
|
|
127
|
+
jobId: jobExecution.jobId,
|
|
128
|
+
jobExecutionId: jobExecution.id,
|
|
129
|
+
status: jobExecution.status,
|
|
130
|
+
statusReason: jobExecution.statusReason,
|
|
131
|
+
statusReasonMessage: jobExecution.statusReasonMessage,
|
|
132
|
+
});
|
|
119
133
|
await bulkUpdateStepStatuses(
|
|
120
134
|
{jobExecutionId: jobExecution.id, status: spec.terminalStatus},
|
|
121
135
|
tx,
|
package/src/db/workflow-runs.ts
CHANGED
|
@@ -9,7 +9,7 @@ export {
|
|
|
9
9
|
getJobExecutionsByJobId,
|
|
10
10
|
getJobExecutionsByWorkflowRunAttemptId,
|
|
11
11
|
getLatestJobExecutionByJobId,
|
|
12
|
-
|
|
12
|
+
queueJobExecution,
|
|
13
13
|
recordJobExecutionStartedAt,
|
|
14
14
|
resolveJobExecutionAfterLeaseExpiry,
|
|
15
15
|
updateJobExecutionStatus,
|
|
@@ -35,6 +35,8 @@ export {
|
|
|
35
35
|
updateJobStatusAtVersion,
|
|
36
36
|
} from './workflow-runs/jobs.js';
|
|
37
37
|
export {
|
|
38
|
+
writeJobExecutionQueuedOutbox,
|
|
39
|
+
writeJobExecutionTerminatedOutbox,
|
|
38
40
|
writeJobStepsSettledOutbox,
|
|
39
41
|
writeStepAttemptTerminatedOutbox,
|
|
40
42
|
writeStepRestartEnqueuedOutbox,
|
package/src/index.ts
CHANGED
|
@@ -9,7 +9,6 @@ import type {ProjectsModuleClient} from '@shipfox/api-projects-dto/inter-module'
|
|
|
9
9
|
import {
|
|
10
10
|
RUNNER_JOB_CLAIMED,
|
|
11
11
|
RUNNER_JOB_LEASE_EXPIRED,
|
|
12
|
-
RUNNER_JOB_QUEUED,
|
|
13
12
|
type RunnersEventMap,
|
|
14
13
|
} from '@shipfox/api-runners-dto';
|
|
15
14
|
import type {RunnersInterModuleClient} from '@shipfox/api-runners-dto/inter-module';
|
|
@@ -35,7 +34,6 @@ import {
|
|
|
35
34
|
onJobTerminatedFailureAnnotation,
|
|
36
35
|
onRunnerJobClaimed,
|
|
37
36
|
onRunnerJobLeaseExpired,
|
|
38
|
-
onRunnerJobQueued,
|
|
39
37
|
onStepAttemptTerminatedFailureAnnotation,
|
|
40
38
|
onWorkflowRunAttemptCreated,
|
|
41
39
|
onWorkflowRunCancelled,
|
|
@@ -128,15 +126,13 @@ export function createWorkflowsModule({
|
|
|
128
126
|
),
|
|
129
127
|
subscriber(WORKFLOWS_JOB_TERMINATED, onJobTerminatedFailureAnnotation(annotations)),
|
|
130
128
|
subscriber(RUNNER_JOB_LEASE_EXPIRED, onRunnerJobLeaseExpired),
|
|
131
|
-
subscriber(RUNNER_JOB_QUEUED, onRunnerJobQueued),
|
|
132
129
|
subscriber(RUNNER_JOB_CLAIMED, onRunnerJobClaimed),
|
|
133
130
|
],
|
|
134
131
|
workers: [
|
|
135
132
|
{
|
|
136
133
|
taskQueue: WORKFLOWS_TASK_QUEUE,
|
|
137
134
|
workflowsPath,
|
|
138
|
-
activities: () =>
|
|
139
|
-
createOrchestrationActivities({agent, integrations, projects, runners, secrets}),
|
|
135
|
+
activities: () => createOrchestrationActivities({agent, integrations, projects, secrets}),
|
|
140
136
|
workflows: [],
|
|
141
137
|
},
|
|
142
138
|
],
|
|
@@ -33,7 +33,12 @@ export function toRunDto(run: WorkflowRun, latestAttempt = run.currentAttempt):
|
|
|
33
33
|
};
|
|
34
34
|
}
|
|
35
35
|
|
|
36
|
-
const EMPTY_JOBS: WorkflowRunJobsSummary = {
|
|
36
|
+
const EMPTY_JOBS: WorkflowRunJobsSummary = {
|
|
37
|
+
preview: [],
|
|
38
|
+
statusCounts: [],
|
|
39
|
+
rawStatusCounts: [],
|
|
40
|
+
hasStartedJobExecution: false,
|
|
41
|
+
};
|
|
37
42
|
|
|
38
43
|
export function toRunListItemDto(
|
|
39
44
|
run: WorkflowRun,
|
|
@@ -53,6 +58,7 @@ export function toRunListItemDto(
|
|
|
53
58
|
})),
|
|
54
59
|
job_status_counts: jobs.rawStatusCounts.map(({status, count}) => ({status, count})),
|
|
55
60
|
job_display_status_counts: jobs.statusCounts.map(({status, count}) => ({status, count})),
|
|
61
|
+
has_started_job_execution: jobs.hasStartedJobExecution,
|
|
56
62
|
};
|
|
57
63
|
}
|
|
58
64
|
|
|
@@ -21,6 +21,7 @@ import {
|
|
|
21
21
|
getJobExecutionsByJobId,
|
|
22
22
|
getJobsByWorkflowRunId,
|
|
23
23
|
getStepsByJobId,
|
|
24
|
+
recordJobExecutionStartedAt,
|
|
24
25
|
updateWorkflowRunStatus,
|
|
25
26
|
} from '#db/workflow-runs.js';
|
|
26
27
|
import {workflowModel} from '#test/index.js';
|
|
@@ -137,6 +138,36 @@ describe('GET /api/workflows/runs/:id', () => {
|
|
|
137
138
|
expect(body.jobs[0]).not.toHaveProperty('finished_at');
|
|
138
139
|
});
|
|
139
140
|
|
|
141
|
+
// The run list decides this from the same executions, so a run detail that answered it from
|
|
142
|
+
// its own payload could drift from the row the user clicked through.
|
|
143
|
+
test('reports whether any job execution of the attempt reached its runner', async () => {
|
|
144
|
+
const run = await createWorkflowRun({
|
|
145
|
+
workspaceId,
|
|
146
|
+
projectId: crypto.randomUUID(),
|
|
147
|
+
definitionId: crypto.randomUUID(),
|
|
148
|
+
model: workflowModel({name: 'Test', jobs: {build: {steps: [{run: 'npm build'}]}}}),
|
|
149
|
+
triggerPayload: {
|
|
150
|
+
source: 'manual',
|
|
151
|
+
event: 'fire',
|
|
152
|
+
subscriptionId: crypto.randomUUID(),
|
|
153
|
+
userId: crypto.randomUUID(),
|
|
154
|
+
},
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
const before = await app.inject({method: 'GET', url: `/api/workflows/runs/${run.id}`});
|
|
158
|
+
expect(before.json().has_started_job_execution).toBe(false);
|
|
159
|
+
|
|
160
|
+
const [job] = await getJobsByWorkflowRunId(run.id);
|
|
161
|
+
const [execution] = await getJobExecutionsByJobId(job?.id ?? '');
|
|
162
|
+
await recordJobExecutionStartedAt({
|
|
163
|
+
jobExecutionId: execution?.id ?? '',
|
|
164
|
+
startedAt: new Date('2026-05-07T01:00:05.000Z'),
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
const after = await app.inject({method: 'GET', url: `/api/workflows/runs/${run.id}`});
|
|
168
|
+
expect(after.json().has_started_job_execution).toBe(true);
|
|
169
|
+
});
|
|
170
|
+
|
|
140
171
|
test('does not aggregate latest_attempt for a first attempt without lineage', async () => {
|
|
141
172
|
const latestAttemptSpy = vi.spyOn(dbIndex, 'getLatestAttempt');
|
|
142
173
|
const run = await createWorkflowRun({
|
|
@@ -99,7 +99,11 @@ describe('GET /api/workflows/runs', () => {
|
|
|
99
99
|
expect(body.runs[0].workflow_name).toBeDefined();
|
|
100
100
|
expect(body.runs[0].trigger_source).toBe('manual');
|
|
101
101
|
// The runs list carries run-level timing (null until the run starts).
|
|
102
|
-
expect(body.runs[0]).toMatchObject({
|
|
102
|
+
expect(body.runs[0]).toMatchObject({
|
|
103
|
+
started_at: null,
|
|
104
|
+
finished_at: null,
|
|
105
|
+
has_started_job_execution: false,
|
|
106
|
+
});
|
|
103
107
|
expect(body.next_cursor).toBeNull();
|
|
104
108
|
expect(body.filtered_total_count).toBe(2);
|
|
105
109
|
});
|
|
@@ -6,6 +6,5 @@ export {onJobEventDelivered} from './on-job-event-delivered.js';
|
|
|
6
6
|
export {onJobStepsSettled} from './on-job-steps-settled.js';
|
|
7
7
|
export {onRunnerJobClaimed} from './on-runner-job-claimed.js';
|
|
8
8
|
export {onRunnerJobLeaseExpired} from './on-runner-job-lease-expired.js';
|
|
9
|
-
export {onRunnerJobQueued} from './on-runner-job-queued.js';
|
|
10
9
|
export {onWorkflowRunAttemptCreated} from './on-workflow-run-attempt-created.js';
|
|
11
10
|
export {onWorkflowRunCancelled} from './on-workflow-run-cancelled.js';
|
|
@@ -1,20 +1,17 @@
|
|
|
1
1
|
import type {AgentInterModuleClient} from '@shipfox/api-agent-dto/inter-module';
|
|
2
2
|
import type {IntegrationsModuleClient} from '@shipfox/api-integration-core-dto/inter-module';
|
|
3
3
|
import type {ProjectsModuleClient} from '@shipfox/api-projects-dto/inter-module';
|
|
4
|
-
import type {RunnersInterModuleClient} from '@shipfox/api-runners-dto/inter-module';
|
|
5
4
|
import type {SecretsInterModuleClient} from '@shipfox/api-secrets-dto/inter-module';
|
|
6
5
|
import {
|
|
7
6
|
activateJobListenerActivity,
|
|
8
7
|
bulkSetStepStatuses,
|
|
9
|
-
createCancelRunnerJobsActivity,
|
|
10
8
|
createDrainListenerEventsActivity,
|
|
11
|
-
createEnqueueJobExecutionForRunner,
|
|
12
|
-
createReleaseLeaseActivity,
|
|
13
9
|
evaluateJobActivationsActivity,
|
|
14
10
|
failJobExecutionAsTimedOutActivity,
|
|
15
11
|
failRunAsTimedOutActivity,
|
|
16
12
|
loadRunAttemptDag,
|
|
17
13
|
peekListenerBufferActivity,
|
|
14
|
+
queueJobExecutionActivity,
|
|
18
15
|
recordListenerFiringOutcomeActivity,
|
|
19
16
|
resolveJobListenerActivity,
|
|
20
17
|
resolveJobStatusFromJobExecutionsActivity,
|
|
@@ -29,7 +26,6 @@ export function createOrchestrationActivities(params: {
|
|
|
29
26
|
agent: AgentInterModuleClient;
|
|
30
27
|
integrations: IntegrationsModuleClient;
|
|
31
28
|
projects: ProjectsModuleClient;
|
|
32
|
-
runners: RunnersInterModuleClient;
|
|
33
29
|
secrets: Pick<SecretsInterModuleClient, 'getVariablesByNamespace'>;
|
|
34
30
|
}) {
|
|
35
31
|
return {
|
|
@@ -39,8 +35,7 @@ export function createOrchestrationActivities(params: {
|
|
|
39
35
|
setJobExecutionStatus: async (activityParams: Parameters<typeof setJobExecutionStatus>[0]) =>
|
|
40
36
|
await setJobExecutionStatus(activityParams, params.secrets),
|
|
41
37
|
bulkSetStepStatuses,
|
|
42
|
-
|
|
43
|
-
enqueueJobExecutionForRunner: createEnqueueJobExecutionForRunner(params.runners),
|
|
38
|
+
queueJobExecutionActivity,
|
|
44
39
|
evaluateJobActivationsActivity,
|
|
45
40
|
failJobExecutionAsTimedOutActivity: async (
|
|
46
41
|
activityParams: Parameters<typeof failJobExecutionAsTimedOutActivity>[0],
|
|
@@ -61,6 +56,5 @@ export function createOrchestrationActivities(params: {
|
|
|
61
56
|
activityParams: Parameters<typeof resolveLeaseExpiredJobExecutionActivity>[0],
|
|
62
57
|
) => await resolveLeaseExpiredJobExecutionActivity(activityParams, params.secrets),
|
|
63
58
|
resolveJobStatusFromJobExecutionsActivity,
|
|
64
|
-
releaseLeaseActivity: createReleaseLeaseActivity(params.runners),
|
|
65
59
|
};
|
|
66
60
|
}
|
|
@@ -8,7 +8,11 @@ import {
|
|
|
8
8
|
import {createTestSecretsClient} from '#test/fixtures/secrets-inter-module.js';
|
|
9
9
|
import {stripSetupStep} from '#test/fixtures/strip-setup-step.js';
|
|
10
10
|
import {workflowModel} from '#test/index.js';
|
|
11
|
-
import {
|
|
11
|
+
import {
|
|
12
|
+
queueJobExecutionActivity,
|
|
13
|
+
resolveLeaseExpiredJobExecutionActivity,
|
|
14
|
+
setJobStatus,
|
|
15
|
+
} from './orchestration-activities.js';
|
|
12
16
|
|
|
13
17
|
let workspaceId: string;
|
|
14
18
|
let projectId: string;
|
|
@@ -74,6 +78,18 @@ describe('resolveLeaseExpiredJobExecutionActivity', () => {
|
|
|
74
78
|
});
|
|
75
79
|
});
|
|
76
80
|
|
|
81
|
+
describe('queueJobExecutionActivity', () => {
|
|
82
|
+
test('a missing execution fails non-retryably', async () => {
|
|
83
|
+
const error = await queueJobExecutionActivity({
|
|
84
|
+
jobId: crypto.randomUUID(),
|
|
85
|
+
jobExecutionId: crypto.randomUUID(),
|
|
86
|
+
}).catch((err: unknown) => err);
|
|
87
|
+
|
|
88
|
+
expect(error).toBeInstanceOf(ApplicationFailure);
|
|
89
|
+
expect((error as ApplicationFailure).nonRetryable).toBe(true);
|
|
90
|
+
});
|
|
91
|
+
});
|
|
92
|
+
|
|
77
93
|
describe('setJobStatus', () => {
|
|
78
94
|
test('default-gate skipped jobs record an evaluation trace', async () => {
|
|
79
95
|
const run = await createWorkflowRun({
|
|
@@ -1,12 +1,7 @@
|
|
|
1
1
|
import type {AgentInterModuleClient} from '@shipfox/api-agent-dto/inter-module';
|
|
2
2
|
import type {IntegrationsModuleClient} from '@shipfox/api-integration-core-dto/inter-module';
|
|
3
3
|
import type {ProjectsModuleClient} from '@shipfox/api-projects-dto/inter-module';
|
|
4
|
-
import {
|
|
5
|
-
type RunnersInterModuleClient,
|
|
6
|
-
runnersInterModuleContract,
|
|
7
|
-
} from '@shipfox/api-runners-dto/inter-module';
|
|
8
4
|
import type {SecretsInterModuleClient} from '@shipfox/api-secrets-dto/inter-module';
|
|
9
|
-
import {isInterModuleKnownError} from '@shipfox/inter-module';
|
|
10
5
|
import {ApplicationFailure} from '@temporalio/common';
|
|
11
6
|
import {defaultJobConditionTrace} from '#core/condition-trace.js';
|
|
12
7
|
import type {JobStatus, JobStatusReason, ResolutionReason} from '#core/entities/job.js';
|
|
@@ -31,6 +26,7 @@ import {
|
|
|
31
26
|
getWorkflowRunByAttemptId,
|
|
32
27
|
type JobActivationDecision,
|
|
33
28
|
peekListenerBuffer,
|
|
29
|
+
queueJobExecution,
|
|
34
30
|
resolveJobExecutionAfterLeaseExpiry,
|
|
35
31
|
resolveJobListener,
|
|
36
32
|
resolveJobStatusFromJobExecutions,
|
|
@@ -209,48 +205,19 @@ export async function resolveLeaseExpiredJobExecutionActivity(
|
|
|
209
205
|
}
|
|
210
206
|
}
|
|
211
207
|
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
await
|
|
218
|
-
|
|
219
|
-
}
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
return async function enqueueJobExecutionForRunner(params: {
|
|
223
|
-
workspaceId: string;
|
|
224
|
-
workflowRunId: string;
|
|
225
|
-
jobId: string;
|
|
226
|
-
jobExecutionId: string;
|
|
227
|
-
runAttemptId: string;
|
|
228
|
-
projectId: string;
|
|
229
|
-
requiredLabels: string[];
|
|
230
|
-
}): Promise<void> {
|
|
231
|
-
try {
|
|
232
|
-
await runners.enqueueJobExecution({
|
|
233
|
-
workspaceId: params.workspaceId,
|
|
234
|
-
workflowRunId: params.workflowRunId,
|
|
235
|
-
workflowRunAttemptId: params.runAttemptId,
|
|
236
|
-
jobId: params.jobId,
|
|
237
|
-
jobExecutionId: params.jobExecutionId,
|
|
238
|
-
projectId: params.projectId,
|
|
239
|
-
requiredLabels: params.requiredLabels,
|
|
240
|
-
});
|
|
241
|
-
} catch (err) {
|
|
242
|
-
if (isInterModuleKnownError(runnersInterModuleContract.methods.enqueueJobExecution, err)) {
|
|
243
|
-
throw ApplicationFailure.nonRetryable(err.message, err.name);
|
|
244
|
-
}
|
|
245
|
-
throw err;
|
|
208
|
+
export async function queueJobExecutionActivity(params: {
|
|
209
|
+
jobId: string;
|
|
210
|
+
jobExecutionId: string;
|
|
211
|
+
}): Promise<{newVersion: number; status: Exclude<JobStatus, 'skipped'>}> {
|
|
212
|
+
try {
|
|
213
|
+
const execution = await queueJobExecution({jobExecutionId: params.jobExecutionId});
|
|
214
|
+
return {newVersion: execution.version, status: execution.status};
|
|
215
|
+
} catch (err) {
|
|
216
|
+
if (err instanceof JobNotFoundError) {
|
|
217
|
+
throw ApplicationFailure.nonRetryable(err.message, err.name);
|
|
246
218
|
}
|
|
247
|
-
|
|
248
|
-
}
|
|
249
|
-
|
|
250
|
-
export function createCancelRunnerJobsActivity(runners: RunnersInterModuleClient) {
|
|
251
|
-
return async function cancelRunnerJobsActivity(params: {jobIds: string[]}): Promise<void> {
|
|
252
|
-
await runners.cancelJobs(params);
|
|
253
|
-
};
|
|
219
|
+
throw err;
|
|
220
|
+
}
|
|
254
221
|
}
|
|
255
222
|
|
|
256
223
|
export async function failJobExecutionAsTimedOutActivity(
|
|
@@ -107,7 +107,7 @@ async function expectEmptyRequiredLabelsFailure(input: typeof defaultJobInput):
|
|
|
107
107
|
}
|
|
108
108
|
|
|
109
109
|
describe('jobExecutionOrchestration', () => {
|
|
110
|
-
test('runner claim flips status to running, then
|
|
110
|
+
test('runner claim flips status to running, then the terminal fact finishes the execution', async () => {
|
|
111
111
|
setCfg({
|
|
112
112
|
dag: makeDag([dagJob('job-1', 'build')]),
|
|
113
113
|
jobResults: new Map([['job-1', 'succeeded']]),
|
|
@@ -117,7 +117,6 @@ describe('jobExecutionOrchestration', () => {
|
|
|
117
117
|
|
|
118
118
|
expect(result.status).toBe('succeeded');
|
|
119
119
|
expect(finalStatusesFor('job-1')).toEqual(['running', 'succeeded']);
|
|
120
|
-
expect(callsNamed('releaseLeaseActivity')).toHaveLength(1);
|
|
121
120
|
expect(callsNamed('resolveLeaseExpiredJobExecutionActivity')).toHaveLength(0);
|
|
122
121
|
expect(callsNamed('bulkSetStepStatuses')).toHaveLength(0);
|
|
123
122
|
});
|
|
@@ -238,7 +237,7 @@ describe('jobExecutionOrchestration', () => {
|
|
|
238
237
|
});
|
|
239
238
|
|
|
240
239
|
expect(setExecutionStatusCalls()).toHaveLength(0);
|
|
241
|
-
expect(callsNamed('
|
|
240
|
+
expect(callsNamed('queueJobExecutionActivity')).toHaveLength(0);
|
|
242
241
|
});
|
|
243
242
|
|
|
244
243
|
test('whitespace-only required labels fail before the job is marked running', async () => {
|
|
@@ -254,7 +253,7 @@ describe('jobExecutionOrchestration', () => {
|
|
|
254
253
|
});
|
|
255
254
|
|
|
256
255
|
expect(setExecutionStatusCalls()).toHaveLength(0);
|
|
257
|
-
expect(callsNamed('
|
|
256
|
+
expect(callsNamed('queueJobExecutionActivity')).toHaveLength(0);
|
|
258
257
|
});
|
|
259
258
|
|
|
260
259
|
test('finished signal (failed) flips status without sweeping steps', async () => {
|
|
@@ -265,7 +264,6 @@ describe('jobExecutionOrchestration', () => {
|
|
|
265
264
|
expect(result.status).toBe('failed');
|
|
266
265
|
expect(finalStatusesFor('job-2')).toEqual(['running', 'failed']);
|
|
267
266
|
expect(terminalSetJobCall('job-2')?.params.statusReason).toBe('step_failed');
|
|
268
|
-
expect(callsNamed('releaseLeaseActivity')).toHaveLength(1);
|
|
269
267
|
expect(callsNamed('bulkSetStepStatuses')).toHaveLength(0);
|
|
270
268
|
});
|
|
271
269
|
|
|
@@ -289,10 +287,9 @@ describe('jobExecutionOrchestration', () => {
|
|
|
289
287
|
statusReason: 'step_failed',
|
|
290
288
|
},
|
|
291
289
|
});
|
|
292
|
-
expect(callsNamed('releaseLeaseActivity')).toHaveLength(1);
|
|
293
290
|
});
|
|
294
291
|
|
|
295
|
-
test('lease-expired signal resolves via the activity
|
|
292
|
+
test('lease-expired signal resolves via the activity', async () => {
|
|
296
293
|
setCfg({
|
|
297
294
|
dag: makeDag([]),
|
|
298
295
|
jobResults: new Map(),
|
|
@@ -304,7 +301,6 @@ describe('jobExecutionOrchestration', () => {
|
|
|
304
301
|
|
|
305
302
|
expect(result.status).toBe('failed');
|
|
306
303
|
expect(callsNamed('resolveLeaseExpiredJobExecutionActivity')).toHaveLength(1);
|
|
307
|
-
expect(callsNamed('releaseLeaseActivity')).toHaveLength(1);
|
|
308
304
|
expect(callsNamed('failJobExecutionAsTimedOutActivity')).toHaveLength(0);
|
|
309
305
|
});
|
|
310
306
|
|
|
@@ -400,24 +396,10 @@ describe('jobExecutionOrchestration', () => {
|
|
|
400
396
|
const result = await executeJob({...defaultJobInput, jobId: 'job-cancelled'});
|
|
401
397
|
|
|
402
398
|
expect(result).toMatchObject({status: 'cancelled'});
|
|
403
|
-
expect(callsNamed('
|
|
404
|
-
expect(callsNamed('releaseLeaseActivity')).toHaveLength(0);
|
|
399
|
+
expect(callsNamed('queueJobExecutionActivity')).toHaveLength(1);
|
|
405
400
|
});
|
|
406
401
|
|
|
407
|
-
test('
|
|
408
|
-
setCfg({
|
|
409
|
-
dag: makeDag([]),
|
|
410
|
-
jobResults: new Map([['job-rl', 'succeeded']]),
|
|
411
|
-
releaseLeaseError: 'runners db down',
|
|
412
|
-
});
|
|
413
|
-
|
|
414
|
-
const result = await executeJob({...defaultJobInput, jobId: 'job-rl'});
|
|
415
|
-
|
|
416
|
-
expect(result.status).toBe('succeeded');
|
|
417
|
-
expect(callsNamed('releaseLeaseActivity').length).toBeGreaterThanOrEqual(1);
|
|
418
|
-
});
|
|
419
|
-
|
|
420
|
-
test('times out after the execution deadline and fails its steps without releasing the lease', async () => {
|
|
402
|
+
test('times out after the execution deadline and fails its steps', async () => {
|
|
421
403
|
setCfg({
|
|
422
404
|
dag: makeDag([dagJob('job-timeout', 'build')]),
|
|
423
405
|
jobResults: new Map(),
|
|
@@ -437,7 +419,6 @@ describe('jobExecutionOrchestration', () => {
|
|
|
437
419
|
name: 'bulkSetStepStatuses',
|
|
438
420
|
params: {jobExecutionId: 'job-timeout', status: 'failed'},
|
|
439
421
|
});
|
|
440
|
-
expect(callsNamed('releaseLeaseActivity')).toHaveLength(0);
|
|
441
422
|
expect(callsNamed('resolveLeaseExpiredJobExecutionActivity')).toHaveLength(0);
|
|
442
423
|
});
|
|
443
424
|
|
|
@@ -26,14 +26,14 @@ import {remainingMs} from './deadline.js';
|
|
|
26
26
|
* execution pending while it is queued, then performs the versioned pending → running
|
|
27
27
|
* transition after the current claim. One deadline spans both waits; claim never resets
|
|
28
28
|
* it. Signals can arrive in any order, so the precedence is finished > lease expired >
|
|
29
|
-
* claimed > timeout.
|
|
30
|
-
*
|
|
29
|
+
* claimed > timeout. Workflows commits queue and terminal facts with its state transitions;
|
|
30
|
+
* runners reconciles its local queue, lease, and reservation projections asynchronously.
|
|
31
31
|
*/
|
|
32
32
|
|
|
33
33
|
const {
|
|
34
34
|
setJobStatus,
|
|
35
35
|
setJobExecutionStatus,
|
|
36
|
-
|
|
36
|
+
queueJobExecutionActivity,
|
|
37
37
|
bulkSetStepStatuses,
|
|
38
38
|
failJobExecutionAsTimedOutActivity,
|
|
39
39
|
resolveLeaseExpiredJobExecutionActivity,
|
|
@@ -48,14 +48,6 @@ const {resolveJobStatusFromJobExecutionsActivity} = proxyActivities<
|
|
|
48
48
|
retry: {maximumAttempts: 5},
|
|
49
49
|
});
|
|
50
50
|
|
|
51
|
-
// Lease cleanup gets a bounded retry policy of its own: after a few attempts the
|
|
52
|
-
// workflow stops waiting on it (see releaseLeaseBestEffort) rather than blocking
|
|
53
|
-
// the DAG on runners-side availability.
|
|
54
|
-
const {releaseLeaseActivity} = proxyActivities<ReturnType<typeof createOrchestrationActivities>>({
|
|
55
|
-
startToCloseTimeout: '30s',
|
|
56
|
-
retry: {maximumAttempts: 5},
|
|
57
|
-
});
|
|
58
|
-
|
|
59
51
|
const DEFAULT_EXECUTION_MAX_DURATION_MS = 6 * 60 * 60 * 1000;
|
|
60
52
|
|
|
61
53
|
export const jobFinishedSignal =
|
|
@@ -71,12 +63,9 @@ export interface JobClaimedSignalPayload {
|
|
|
71
63
|
export const jobClaimedSignal = defineSignal<[JobClaimedSignalPayload]>(JOB_CLAIMED_SIGNAL);
|
|
72
64
|
|
|
73
65
|
export interface JobExecutionOrchestrationInput {
|
|
74
|
-
workspaceId: string;
|
|
75
|
-
workflowRunId: string;
|
|
76
66
|
runAttemptId: string;
|
|
77
67
|
jobId: string;
|
|
78
68
|
jobExecutionId: string;
|
|
79
|
-
projectId: string;
|
|
80
69
|
jobVersion: number;
|
|
81
70
|
executionVersion: number;
|
|
82
71
|
executionTimeoutMs?: number | null | undefined;
|
|
@@ -89,17 +78,6 @@ export interface JobExecutionOrchestrationResult {
|
|
|
89
78
|
jobVersion: number;
|
|
90
79
|
}
|
|
91
80
|
|
|
92
|
-
async function releaseLeaseBestEffort(jobExecutionId: string): Promise<void> {
|
|
93
|
-
try {
|
|
94
|
-
await releaseLeaseActivity({jobExecutionId});
|
|
95
|
-
} catch (err) {
|
|
96
|
-
log.warn('lease release failed; stuck detector will reap the row', {
|
|
97
|
-
jobExecutionId,
|
|
98
|
-
error: String(err),
|
|
99
|
-
});
|
|
100
|
-
}
|
|
101
|
-
}
|
|
102
|
-
|
|
103
81
|
async function resolveJobStatusOrFailClosed(
|
|
104
82
|
input: JobExecutionOrchestrationInput,
|
|
105
83
|
): Promise<{status: RuntimeCompletionStatus; jobVersion: number}> {
|
|
@@ -121,7 +99,7 @@ async function resolveJobStatusOrFailClosed(
|
|
|
121
99
|
}
|
|
122
100
|
}
|
|
123
101
|
|
|
124
|
-
async function
|
|
102
|
+
async function queueJobExecution(input: JobExecutionOrchestrationInput) {
|
|
125
103
|
if (hasNoRequiredRunnerLabels(input.requiredLabels)) {
|
|
126
104
|
throw ApplicationFailure.nonRetryable(
|
|
127
105
|
`Job ${input.jobId} has no required runner labels`,
|
|
@@ -129,15 +107,12 @@ async function enqueueJobExecution(input: JobExecutionOrchestrationInput): Promi
|
|
|
129
107
|
);
|
|
130
108
|
}
|
|
131
109
|
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
projectId: input.projectId,
|
|
139
|
-
requiredLabels: input.requiredLabels,
|
|
140
|
-
});
|
|
110
|
+
return jobExecutionStartOutcome(
|
|
111
|
+
await queueJobExecutionActivity({
|
|
112
|
+
jobId: input.jobId,
|
|
113
|
+
jobExecutionId: input.jobExecutionId,
|
|
114
|
+
}),
|
|
115
|
+
);
|
|
141
116
|
}
|
|
142
117
|
|
|
143
118
|
async function markJobExecutionRunning(
|
|
@@ -218,7 +193,6 @@ async function resolveFinishedJobExecution({
|
|
|
218
193
|
terminationReason: 'finished',
|
|
219
194
|
status,
|
|
220
195
|
});
|
|
221
|
-
await releaseLeaseBestEffort(input.jobExecutionId);
|
|
222
196
|
return {status, jobVersion: input.jobVersion};
|
|
223
197
|
}
|
|
224
198
|
const resolved = await resolveJobStatusOrFailClosed(input);
|
|
@@ -228,7 +202,6 @@ async function resolveFinishedJobExecution({
|
|
|
228
202
|
terminationReason: 'finished',
|
|
229
203
|
status: resolved.status,
|
|
230
204
|
});
|
|
231
|
-
await releaseLeaseBestEffort(input.jobExecutionId);
|
|
232
205
|
return resolved;
|
|
233
206
|
}
|
|
234
207
|
|
|
@@ -257,7 +230,6 @@ async function resolveLeaseExpiredJobExecution({
|
|
|
257
230
|
terminationReason: 'lease_expired',
|
|
258
231
|
status: leaseExpired.status,
|
|
259
232
|
});
|
|
260
|
-
await releaseLeaseBestEffort(input.jobExecutionId);
|
|
261
233
|
return {status: leaseExpired.status, jobVersion: input.jobVersion};
|
|
262
234
|
}
|
|
263
235
|
const {status, jobVersion} = await resolveJobStatusOrFailClosed(input);
|
|
@@ -267,13 +239,12 @@ async function resolveLeaseExpiredJobExecution({
|
|
|
267
239
|
terminationReason: 'lease_expired',
|
|
268
240
|
status,
|
|
269
241
|
});
|
|
270
|
-
await releaseLeaseBestEffort(input.jobExecutionId);
|
|
271
242
|
return {status, jobVersion};
|
|
272
243
|
}
|
|
273
244
|
|
|
274
|
-
// Timeout backstop. The activity atomically fails the execution, marks
|
|
275
|
-
//
|
|
276
|
-
//
|
|
245
|
+
// Timeout backstop. The activity atomically fails the execution, marks `timed_out_at`, and
|
|
246
|
+
// enqueues the generic terminal execution fact. The runners subscriber then asks the runner to
|
|
247
|
+
// cancel. The lease is intentionally NOT released here.
|
|
277
248
|
async function resolveTimedOutJobExecution({
|
|
278
249
|
input,
|
|
279
250
|
runningVersion,
|
|
@@ -314,7 +285,10 @@ export async function jobExecutionOrchestration(
|
|
|
314
285
|
// Register every signal before enqueue can block or publish a claim/outcome event. The
|
|
315
286
|
// handlers retain signals that arrive while the enqueue activity is in flight.
|
|
316
287
|
registerJobExecutionSignalHandlers(input.jobExecutionId, signals);
|
|
317
|
-
await
|
|
288
|
+
const queued = await queueJobExecution(input);
|
|
289
|
+
if (queued.kind === 'terminal') {
|
|
290
|
+
return {status: queued.result.status, jobVersion: input.jobVersion};
|
|
291
|
+
}
|
|
318
292
|
|
|
319
293
|
const timeoutMs = input.executionTimeoutMs ?? DEFAULT_EXECUTION_MAX_DURATION_MS;
|
|
320
294
|
const deadline = Date.now() + timeoutMs;
|
|
@@ -63,9 +63,6 @@ describe('listener continue-as-new guard', () => {
|
|
|
63
63
|
|
|
64
64
|
it('carries listener loop state into the continued run', () => {
|
|
65
65
|
const input: JobListenerOrchestrationInput = {
|
|
66
|
-
workspaceId: 'workspace-1',
|
|
67
|
-
workflowRunId: 'run-1',
|
|
68
|
-
projectId: 'project-1',
|
|
69
66
|
runAttemptId: 'attempt-1',
|
|
70
67
|
jobId: 'job-1',
|
|
71
68
|
jobVersion: 3,
|