@shipfox/api-workflows 12.7.0 → 13.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +8 -8
- package/CHANGELOG.md +9 -0
- package/dist/db/index.d.ts +1 -1
- package/dist/db/index.d.ts.map +1 -1
- package/dist/db/index.js +1 -1
- package/dist/db/index.js.map +1 -1
- package/dist/db/job-listeners.d.ts.map +1 -1
- package/dist/db/job-listeners.js +17 -1
- package/dist/db/job-listeners.js.map +1 -1
- package/dist/db/workflow-runs/job-executions.d.ts +2 -3
- package/dist/db/workflow-runs/job-executions.d.ts.map +1 -1
- package/dist/db/workflow-runs/job-executions.js +48 -20
- package/dist/db/workflow-runs/job-executions.js.map +1 -1
- package/dist/db/workflow-runs/outbox.d.ts +15 -0
- package/dist/db/workflow-runs/outbox.d.ts.map +1 -1
- package/dist/db/workflow-runs/outbox.js +35 -1
- package/dist/db/workflow-runs/outbox.js.map +1 -1
- package/dist/db/workflow-runs/run-graph.d.ts.map +1 -1
- package/dist/db/workflow-runs/run-graph.js +12 -0
- package/dist/db/workflow-runs/run-graph.js.map +1 -1
- package/dist/db/workflow-runs/run-status.d.ts.map +1 -1
- package/dist/db/workflow-runs/run-status.js +13 -1
- package/dist/db/workflow-runs/run-status.js.map +1 -1
- package/dist/db/workflow-runs.d.ts +2 -2
- package/dist/db/workflow-runs.d.ts.map +1 -1
- package/dist/db/workflow-runs.js +2 -2
- package/dist/db/workflow-runs.js.map +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -4
- package/dist/index.js.map +1 -1
- package/dist/presentation/index.d.ts +1 -1
- package/dist/presentation/index.d.ts.map +1 -1
- package/dist/presentation/index.js +1 -1
- package/dist/presentation/index.js.map +1 -1
- package/dist/presentation/subscribers/index.d.ts +0 -1
- package/dist/presentation/subscribers/index.d.ts.map +1 -1
- package/dist/presentation/subscribers/index.js +0 -1
- package/dist/presentation/subscribers/index.js.map +1 -1
- package/dist/temporal/activities/index.d.ts +2 -18
- package/dist/temporal/activities/index.d.ts.map +1 -1
- package/dist/temporal/activities/index.js +3 -5
- package/dist/temporal/activities/index.js.map +1 -1
- package/dist/temporal/activities/orchestration-activities.d.ts +5 -14
- package/dist/temporal/activities/orchestration-activities.d.ts.map +1 -1
- package/dist/temporal/activities/orchestration-activities.js +13 -34
- package/dist/temporal/activities/orchestration-activities.js.map +1 -1
- package/dist/temporal/workflows/index.bundle.js +20 -68
- package/dist/temporal/workflows/job-execution-orchestration.d.ts +0 -3
- package/dist/temporal/workflows/job-execution-orchestration.d.ts.map +1 -1
- package/dist/temporal/workflows/job-execution-orchestration.js +17 -41
- package/dist/temporal/workflows/job-execution-orchestration.js.map +1 -1
- package/dist/temporal/workflows/job-listener-orchestration.d.ts +0 -3
- package/dist/temporal/workflows/job-listener-orchestration.d.ts.map +1 -1
- package/dist/temporal/workflows/job-listener-orchestration.js +1 -9
- package/dist/temporal/workflows/job-listener-orchestration.js.map +1 -1
- package/dist/temporal/workflows/run-orchestration.d.ts.map +1 -1
- package/dist/temporal/workflows/run-orchestration.js +2 -18
- package/dist/temporal/workflows/run-orchestration.js.map +1 -1
- package/dist/temporal/workflows/test-env.d.ts +4 -6
- package/dist/temporal/workflows/test-env.d.ts.map +1 -1
- package/dist/temporal/workflows/test-env.js +13 -24
- package/dist/temporal/workflows/test-env.js.map +1 -1
- package/dist/tsconfig.test.tsbuildinfo +1 -1
- package/package.json +3 -3
- package/src/db/index.ts +1 -1
- package/src/db/job-listeners.test.ts +21 -0
- package/src/db/job-listeners.ts +18 -0
- package/src/db/workflow-runs/job-executions.test.ts +110 -12
- package/src/db/workflow-runs/job-executions.ts +54 -29
- package/src/db/workflow-runs/outbox.ts +63 -0
- package/src/db/workflow-runs/run-graph.ts +17 -0
- package/src/db/workflow-runs/run-status.test.ts +13 -0
- package/src/db/workflow-runs/run-status.ts +15 -1
- package/src/db/workflow-runs.ts +3 -1
- package/src/index.ts +1 -5
- package/src/presentation/index.ts +0 -1
- package/src/presentation/subscribers/index.ts +0 -1
- package/src/temporal/activities/index.ts +2 -8
- package/src/temporal/activities/orchestration-activities.test.ts +17 -1
- package/src/temporal/activities/orchestration-activities.ts +13 -46
- package/src/temporal/workflows/job-execution-orchestration.test.ts +6 -25
- package/src/temporal/workflows/job-execution-orchestration.ts +17 -43
- package/src/temporal/workflows/job-listener-orchestration-continuation.test.ts +0 -3
- package/src/temporal/workflows/job-listener-orchestration.test.ts +9 -12
- package/src/temporal/workflows/job-listener-orchestration.ts +0 -8
- package/src/temporal/workflows/run-orchestration.test.ts +22 -33
- package/src/temporal/workflows/run-orchestration.ts +4 -28
- package/src/temporal/workflows/test-env.ts +14 -34
- package/test/fixtures/runners-inter-module.ts +0 -3
- package/test/helpers/workflow-runs.ts +25 -0
- package/tsconfig.build.tsbuildinfo +1 -1
- package/dist/presentation/subscribers/on-runner-job-queued.d.ts +0 -3
- package/dist/presentation/subscribers/on-runner-job-queued.d.ts.map +0 -1
- package/dist/presentation/subscribers/on-runner-job-queued.js +0 -20
- package/dist/presentation/subscribers/on-runner-job-queued.js.map +0 -1
- package/src/presentation/subscribers/on-runner-job-queued.test.ts +0 -57
- package/src/presentation/subscribers/on-runner-job-queued.ts +0 -23
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@shipfox/api-workflows",
|
|
3
3
|
"license": "MIT",
|
|
4
|
-
"version": "
|
|
4
|
+
"version": "13.0.0",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
7
7
|
"url": "git+https://github.com/ShipfoxHQ/shipfox.git",
|
|
@@ -30,9 +30,9 @@
|
|
|
30
30
|
"@shipfox/api-definitions-dto": "12.3.0",
|
|
31
31
|
"@shipfox/api-projects-dto": "12.2.0",
|
|
32
32
|
"@shipfox/api-integration-core-dto": "12.2.0",
|
|
33
|
-
"@shipfox/api-runners-dto": "
|
|
33
|
+
"@shipfox/api-runners-dto": "13.0.0",
|
|
34
34
|
"@shipfox/api-secrets-dto": "12.0.0",
|
|
35
|
-
"@shipfox/api-workflows-dto": "
|
|
35
|
+
"@shipfox/api-workflows-dto": "13.0.0",
|
|
36
36
|
"@shipfox/api-workspaces-dto": "12.0.0",
|
|
37
37
|
"@shipfox/config": "1.2.4",
|
|
38
38
|
"@shipfox/inter-module": "0.2.3",
|
package/src/db/index.ts
CHANGED
|
@@ -15,6 +15,7 @@ import {
|
|
|
15
15
|
drainListenerEventsIntoExecution,
|
|
16
16
|
peekListenerBuffer,
|
|
17
17
|
resolveJobListener,
|
|
18
|
+
settleListenerJobExecution,
|
|
18
19
|
} from '#db/job-listeners.js';
|
|
19
20
|
import {jobExecutions} from '#db/schema/job-executions.js';
|
|
20
21
|
import {jobListenerEvents} from '#db/schema/job-listener-events.js';
|
|
@@ -22,6 +23,7 @@ import {jobs} from '#db/schema/jobs.js';
|
|
|
22
23
|
import {workflowsOutbox} from '#db/schema/outbox.js';
|
|
23
24
|
import {steps} from '#db/schema/steps.js';
|
|
24
25
|
import {workflowRunAttempts} from '#db/schema/workflow-run-attempts.js';
|
|
26
|
+
import {jobExecutionTerminatedEvents} from '#test/helpers/workflow-runs.js';
|
|
25
27
|
import {jobFactory, workflowModel, workflowRunFactory} from '#test/index.js';
|
|
26
28
|
import {getJobsByWorkflowRunId, updateJobExecutionStatus} from './workflow-runs.js';
|
|
27
29
|
|
|
@@ -127,6 +129,25 @@ function expectListeningPayload(
|
|
|
127
129
|
expect(payload.mode).toBe('listening');
|
|
128
130
|
}
|
|
129
131
|
|
|
132
|
+
describe('settleListenerJobExecution', () => {
|
|
133
|
+
it('writes a terminal job-execution fact for a cancelled listener firing', async () => {
|
|
134
|
+
const job = await createListeningJob();
|
|
135
|
+
const execution = await insertExecution(job.id, 1, 'running');
|
|
136
|
+
|
|
137
|
+
await settleListenerJobExecution({jobExecutionId: execution.id, status: 'cancelled'});
|
|
138
|
+
await settleListenerJobExecution({jobExecutionId: execution.id, status: 'cancelled'});
|
|
139
|
+
|
|
140
|
+
expect(await jobExecutionTerminatedEvents(execution.id)).toEqual([
|
|
141
|
+
expect.objectContaining({
|
|
142
|
+
jobId: job.id,
|
|
143
|
+
jobExecutionId: execution.id,
|
|
144
|
+
status: 'cancelled',
|
|
145
|
+
statusReason: 'run_cancelled',
|
|
146
|
+
}),
|
|
147
|
+
]);
|
|
148
|
+
});
|
|
149
|
+
});
|
|
150
|
+
|
|
130
151
|
async function createInactiveListeningJobWithMatchers(params: {
|
|
131
152
|
readonly on: readonly JobListeningTrigger[];
|
|
132
153
|
readonly until?: readonly JobListeningTrigger[] | null;
|
package/src/db/job-listeners.ts
CHANGED
|
@@ -48,6 +48,7 @@ import {
|
|
|
48
48
|
getDirectDependencyJobContexts,
|
|
49
49
|
loadReferencedVariables,
|
|
50
50
|
updateJobStatusAtVersion,
|
|
51
|
+
writeJobExecutionTerminatedOutbox,
|
|
51
52
|
} from './workflow-runs.js';
|
|
52
53
|
|
|
53
54
|
const TERMINAL_EXECUTION_STATUSES: JobExecutionStatus[] = ['succeeded', 'failed', 'cancelled'];
|
|
@@ -481,6 +482,13 @@ export async function settleListenerJobExecution(params: {
|
|
|
481
482
|
)
|
|
482
483
|
.returning();
|
|
483
484
|
if (!execution) return false;
|
|
485
|
+
await writeJobExecutionTerminatedOutbox(tx, {
|
|
486
|
+
jobId: execution.jobId,
|
|
487
|
+
jobExecutionId: execution.id,
|
|
488
|
+
status: execution.status,
|
|
489
|
+
statusReason: execution.statusReason,
|
|
490
|
+
statusReasonMessage: execution.statusReasonMessage,
|
|
491
|
+
});
|
|
484
492
|
await bulkUpdateStepStatuses(
|
|
485
493
|
{jobExecutionId: params.jobExecutionId, status: params.status},
|
|
486
494
|
tx,
|
|
@@ -615,6 +623,16 @@ async function persistMaterializedListenerExecution(
|
|
|
615
623
|
.returning();
|
|
616
624
|
if (!execution) throw new Error('Insert returned no rows');
|
|
617
625
|
|
|
626
|
+
if (execution.status === 'failed') {
|
|
627
|
+
await writeJobExecutionTerminatedOutbox(tx, {
|
|
628
|
+
jobId: execution.jobId,
|
|
629
|
+
jobExecutionId: execution.id,
|
|
630
|
+
status: execution.status,
|
|
631
|
+
statusReason: execution.statusReason,
|
|
632
|
+
statusReasonMessage: execution.statusReasonMessage,
|
|
633
|
+
});
|
|
634
|
+
}
|
|
635
|
+
|
|
618
636
|
await tx
|
|
619
637
|
.update(jobListenerEvents)
|
|
620
638
|
.set({consumedByExecutionId: execution.id})
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {WORKFLOWS_JOB_EXECUTION_QUEUED} from '@shipfox/api-workflows-dto';
|
|
2
2
|
import {and, eq, sql} from 'drizzle-orm';
|
|
3
3
|
import {
|
|
4
4
|
InterpolationUnresolvableError,
|
|
@@ -10,18 +10,23 @@ import {
|
|
|
10
10
|
MAX_JOB_OUTPUT_ENTRIES,
|
|
11
11
|
MAX_JOB_OUTPUT_VALUE_BYTES,
|
|
12
12
|
} from '#core/step-config/job-output-limits.js';
|
|
13
|
-
import {
|
|
13
|
+
import {
|
|
14
|
+
buildModel,
|
|
15
|
+
jobExecutionTerminatedEvents,
|
|
16
|
+
template,
|
|
17
|
+
workflowRunAttemptId,
|
|
18
|
+
} from '#test/helpers/workflow-runs.js';
|
|
14
19
|
import {db} from '../db.js';
|
|
15
20
|
import {workflowsOutbox} from '../schema/outbox.js';
|
|
16
21
|
import {
|
|
17
22
|
applyStepResult,
|
|
18
23
|
createWorkflowRun,
|
|
19
|
-
failJobExecutionAsTimedOut,
|
|
20
24
|
finishStepAttempt,
|
|
21
25
|
getFirstJobExecutionByJobId,
|
|
22
26
|
getJobsByWorkflowRunId,
|
|
23
27
|
getStepsByJobId,
|
|
24
28
|
markStepRunning,
|
|
29
|
+
queueJobExecution,
|
|
25
30
|
resolveJobExecutionAfterLeaseExpiry,
|
|
26
31
|
updateJobExecutionStatus,
|
|
27
32
|
} from '../workflow-runs.js';
|
|
@@ -38,7 +43,7 @@ describe('workflow run job executions', () => {
|
|
|
38
43
|
definitionId = crypto.randomUUID();
|
|
39
44
|
});
|
|
40
45
|
|
|
41
|
-
test('
|
|
46
|
+
test('records and publishes the queue fact once', async () => {
|
|
42
47
|
const run = await createWorkflowRun({
|
|
43
48
|
workspaceId,
|
|
44
49
|
projectId,
|
|
@@ -57,26 +62,119 @@ describe('workflow run job executions', () => {
|
|
|
57
62
|
if (!execution) throw new Error('Expected job execution');
|
|
58
63
|
const actualAttemptId = await workflowRunAttemptId(run.id);
|
|
59
64
|
|
|
60
|
-
await
|
|
61
|
-
|
|
62
|
-
workflowRunAttemptId: crypto.randomUUID(),
|
|
63
|
-
expectedVersion: execution.version,
|
|
64
|
-
});
|
|
65
|
+
const queued = await queueJobExecution({jobExecutionId: execution.id});
|
|
66
|
+
await queueJobExecution({jobExecutionId: execution.id});
|
|
65
67
|
|
|
66
|
-
const
|
|
68
|
+
const events = await db()
|
|
67
69
|
.select({payload: workflowsOutbox.payload})
|
|
68
70
|
.from(workflowsOutbox)
|
|
69
71
|
.where(
|
|
70
72
|
and(
|
|
71
|
-
eq(workflowsOutbox.eventType,
|
|
73
|
+
eq(workflowsOutbox.eventType, WORKFLOWS_JOB_EXECUTION_QUEUED),
|
|
72
74
|
sql`${workflowsOutbox.payload}->>'jobExecutionId' = ${execution.id}`,
|
|
73
75
|
),
|
|
74
76
|
);
|
|
75
|
-
expect(
|
|
77
|
+
expect(events).toHaveLength(1);
|
|
78
|
+
expect(events[0]?.payload).toMatchObject({
|
|
76
79
|
jobId: job.id,
|
|
77
80
|
jobExecutionId: execution.id,
|
|
81
|
+
workflowRunId: run.id,
|
|
78
82
|
workflowRunAttemptId: actualAttemptId,
|
|
83
|
+
workspaceId,
|
|
84
|
+
projectId,
|
|
85
|
+
requiredLabels: ['ubuntu-latest'],
|
|
86
|
+
queuedAt: queued.queuedAt?.toISOString(),
|
|
87
|
+
});
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
test('writes one terminal fact when a job execution becomes terminal', async () => {
|
|
91
|
+
const run = await createWorkflowRun({
|
|
92
|
+
workspaceId,
|
|
93
|
+
projectId,
|
|
94
|
+
definitionId,
|
|
95
|
+
model: buildModel({jobs: {build: {steps: [{run: 'echo build'}]}}}),
|
|
96
|
+
triggerPayload: {
|
|
97
|
+
source: 'manual',
|
|
98
|
+
event: 'fire',
|
|
99
|
+
subscriptionId: crypto.randomUUID(),
|
|
100
|
+
userId: crypto.randomUUID(),
|
|
101
|
+
},
|
|
102
|
+
});
|
|
103
|
+
const [job] = await getJobsByWorkflowRunId(run.id);
|
|
104
|
+
if (!job) throw new Error('Expected workflow job');
|
|
105
|
+
const execution = await getFirstJobExecutionByJobId(job.id);
|
|
106
|
+
if (!execution) throw new Error('Expected job execution');
|
|
107
|
+
|
|
108
|
+
const running = await updateJobExecutionStatus({
|
|
109
|
+
jobExecutionId: execution.id,
|
|
110
|
+
status: 'running',
|
|
111
|
+
expectedVersion: execution.version,
|
|
79
112
|
});
|
|
113
|
+
expect(await jobExecutionTerminatedEvents(execution.id)).toHaveLength(0);
|
|
114
|
+
|
|
115
|
+
await updateJobExecutionStatus({
|
|
116
|
+
jobExecutionId: execution.id,
|
|
117
|
+
status: 'cancelled',
|
|
118
|
+
expectedVersion: running.version,
|
|
119
|
+
statusReason: 'run_cancelled',
|
|
120
|
+
});
|
|
121
|
+
await updateJobExecutionStatus({
|
|
122
|
+
jobExecutionId: execution.id,
|
|
123
|
+
status: 'failed',
|
|
124
|
+
expectedVersion: running.version + 1,
|
|
125
|
+
statusReason: 'unknown',
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
expect(await jobExecutionTerminatedEvents(execution.id)).toEqual([
|
|
129
|
+
expect.objectContaining({
|
|
130
|
+
jobId: job.id,
|
|
131
|
+
jobExecutionId: execution.id,
|
|
132
|
+
workflowRunId: run.id,
|
|
133
|
+
status: 'cancelled',
|
|
134
|
+
statusReason: 'run_cancelled',
|
|
135
|
+
}),
|
|
136
|
+
]);
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
test('does not publish a queue fact after the execution is terminal', async () => {
|
|
140
|
+
const run = await createWorkflowRun({
|
|
141
|
+
workspaceId,
|
|
142
|
+
projectId,
|
|
143
|
+
definitionId,
|
|
144
|
+
model: buildModel({jobs: {build: {steps: [{run: 'echo build'}]}}}),
|
|
145
|
+
triggerPayload: {
|
|
146
|
+
source: 'manual',
|
|
147
|
+
event: 'fire',
|
|
148
|
+
subscriptionId: crypto.randomUUID(),
|
|
149
|
+
userId: crypto.randomUUID(),
|
|
150
|
+
},
|
|
151
|
+
});
|
|
152
|
+
const [job] = await getJobsByWorkflowRunId(run.id);
|
|
153
|
+
if (!job) throw new Error('Expected workflow job');
|
|
154
|
+
const execution = await getFirstJobExecutionByJobId(job.id);
|
|
155
|
+
if (!execution) throw new Error('Expected job execution');
|
|
156
|
+
|
|
157
|
+
await updateJobExecutionStatus({
|
|
158
|
+
jobExecutionId: execution.id,
|
|
159
|
+
status: 'cancelled',
|
|
160
|
+
expectedVersion: execution.version,
|
|
161
|
+
statusReason: 'run_cancelled',
|
|
162
|
+
});
|
|
163
|
+
const after = await queueJobExecution({jobExecutionId: execution.id});
|
|
164
|
+
|
|
165
|
+
expect(after.status).toBe('cancelled');
|
|
166
|
+
expect(after.queuedAt).toBeNull();
|
|
167
|
+
expect(
|
|
168
|
+
await db()
|
|
169
|
+
.select()
|
|
170
|
+
.from(workflowsOutbox)
|
|
171
|
+
.where(
|
|
172
|
+
and(
|
|
173
|
+
eq(workflowsOutbox.eventType, WORKFLOWS_JOB_EXECUTION_QUEUED),
|
|
174
|
+
sql`${workflowsOutbox.payload}->>'jobExecutionId' = ${execution.id}`,
|
|
175
|
+
),
|
|
176
|
+
),
|
|
177
|
+
).toHaveLength(0);
|
|
80
178
|
});
|
|
81
179
|
|
|
82
180
|
test('does not cancel steps when lease expiry loses the execution version race', async () => {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {readPersistedWorkflowModel} from '@shipfox/api-definitions-dto';
|
|
2
2
|
import type {SecretsInterModuleClient} from '@shipfox/api-secrets-dto/inter-module';
|
|
3
|
-
import {
|
|
3
|
+
import {canonicalizeLabels} from '@shipfox/runner-labels';
|
|
4
4
|
import {and, asc, desc, eq, isNull, notInArray, sql} from 'drizzle-orm';
|
|
5
5
|
import type {JobStatusReason} from '#core/entities/job.js';
|
|
6
6
|
import type {JobExecution, JobExecutionStatus} from '#core/entities/job-execution.js';
|
|
@@ -22,7 +22,6 @@ import {
|
|
|
22
22
|
recordWorkflowJobExecutionTimedOut,
|
|
23
23
|
} from '#metrics/instance.js';
|
|
24
24
|
import {db, type Tx} from '../db.js';
|
|
25
|
-
import {writeWorkflowsOutboxEvent} from '../outbox-writes.js';
|
|
26
25
|
import {jobExecutions, toJobExecution} from '../schema/job-executions.js';
|
|
27
26
|
import {jobs, toJob} from '../schema/jobs.js';
|
|
28
27
|
import {stepAttempts, toStepAttempt} from '../schema/step-attempts.js';
|
|
@@ -30,12 +29,9 @@ import {steps, toStep} from '../schema/steps.js';
|
|
|
30
29
|
import {workflowRunAttempts} from '../schema/workflow-run-attempts.js';
|
|
31
30
|
import {toWorkflowRun, workflowRuns} from '../schema/workflow-runs.js';
|
|
32
31
|
import {getDirectDependencyJobContexts} from './jobs.js';
|
|
32
|
+
import {writeJobExecutionQueuedOutbox, writeJobExecutionTerminatedOutbox} from './outbox.js';
|
|
33
33
|
import {loadReferencedVariables} from './runs.js';
|
|
34
|
-
import {
|
|
35
|
-
getWorkflowContextForJob,
|
|
36
|
-
optimisticLockRetry,
|
|
37
|
-
TERMINAL_EXECUTION_STATUSES,
|
|
38
|
-
} from './shared.js';
|
|
34
|
+
import {optimisticLockRetry, TERMINAL_EXECUTION_STATUSES} from './shared.js';
|
|
39
35
|
import {bulkUpdateStepStatuses, getStepsByJobExecutionIdForUpdate} from './steps.js';
|
|
40
36
|
|
|
41
37
|
async function getJobExecutionFallbackName(tx: Tx, jobExecutionId: string): Promise<string> {
|
|
@@ -282,6 +278,15 @@ async function updateJobExecutionStatusAtVersion(
|
|
|
282
278
|
|
|
283
279
|
const row = rows[0];
|
|
284
280
|
if (!row) return null;
|
|
281
|
+
if (TERMINAL_EXECUTION_STATUSES.includes(row.status)) {
|
|
282
|
+
await writeJobExecutionTerminatedOutbox(tx, {
|
|
283
|
+
jobId: row.jobId,
|
|
284
|
+
jobExecutionId: row.id,
|
|
285
|
+
status: row.status,
|
|
286
|
+
statusReason: row.statusReason,
|
|
287
|
+
statusReasonMessage: row.statusReasonMessage,
|
|
288
|
+
});
|
|
289
|
+
}
|
|
285
290
|
return {
|
|
286
291
|
execution: toJobExecution(row, await getJobExecutionFallbackName(tx, row.id)),
|
|
287
292
|
changed: true,
|
|
@@ -336,17 +341,49 @@ export async function updateJobExecutionStatus(
|
|
|
336
341
|
return result.execution;
|
|
337
342
|
}
|
|
338
343
|
|
|
339
|
-
export async function
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
344
|
+
export async function queueJobExecution(params: {jobExecutionId: string}): Promise<JobExecution> {
|
|
345
|
+
const result = await db().transaction(async (tx) => {
|
|
346
|
+
const [row] = await tx
|
|
347
|
+
.select({jobExecution: jobExecutions, job: jobs})
|
|
348
|
+
.from(jobExecutions)
|
|
349
|
+
.innerJoin(jobs, eq(jobExecutions.jobId, jobs.id))
|
|
350
|
+
.where(eq(jobExecutions.id, params.jobExecutionId))
|
|
351
|
+
.limit(1)
|
|
352
|
+
.for('update');
|
|
353
|
+
if (!row) throw new JobNotFoundError(params.jobExecutionId);
|
|
354
|
+
|
|
355
|
+
const existing = toJobExecution(row.jobExecution, row.job.name ?? row.job.key);
|
|
356
|
+
if (existing.queuedAt !== null || TERMINAL_EXECUTION_STATUSES.includes(existing.status)) {
|
|
357
|
+
return {execution: existing, changed: false};
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
const requiredLabels = [...canonicalizeLabels(existing.runner ?? row.job.runner ?? [])];
|
|
361
|
+
if (requiredLabels.length === 0) {
|
|
362
|
+
throw new Error(`Job execution ${params.jobExecutionId} has no required runner labels`);
|
|
363
|
+
}
|
|
348
364
|
|
|
349
|
-
|
|
365
|
+
const [queued] = await tx
|
|
366
|
+
.update(jobExecutions)
|
|
367
|
+
.set({queuedAt: sql`now()`})
|
|
368
|
+
.where(eq(jobExecutions.id, params.jobExecutionId))
|
|
369
|
+
.returning();
|
|
370
|
+
if (!queued?.queuedAt) throw new Error(`Cannot queue job execution ${params.jobExecutionId}`);
|
|
371
|
+
|
|
372
|
+
await writeJobExecutionQueuedOutbox(tx, {
|
|
373
|
+
jobId: queued.jobId,
|
|
374
|
+
jobExecutionId: queued.id,
|
|
375
|
+
requiredLabels,
|
|
376
|
+
queuedAt: queued.queuedAt,
|
|
377
|
+
});
|
|
378
|
+
|
|
379
|
+
return {
|
|
380
|
+
execution: toJobExecution(queued, row.job.name ?? row.job.key),
|
|
381
|
+
changed: true,
|
|
382
|
+
};
|
|
383
|
+
});
|
|
384
|
+
|
|
385
|
+
if (result.changed) recordWorkflowJobExecutionQueued();
|
|
386
|
+
return result.execution;
|
|
350
387
|
}
|
|
351
388
|
|
|
352
389
|
export async function recordJobExecutionStartedAt(params: {
|
|
@@ -393,18 +430,6 @@ export async function failJobExecutionAsTimedOut(params: {
|
|
|
393
430
|
failureMessage: `Optimistic lock failure: job execution ${params.jobExecutionId} version ${params.expectedVersion}`,
|
|
394
431
|
});
|
|
395
432
|
|
|
396
|
-
if (updated.changed) {
|
|
397
|
-
const identity = await getWorkflowContextForJob(updated.execution.jobId, tx);
|
|
398
|
-
await writeWorkflowsOutboxEvent(tx, {
|
|
399
|
-
type: WORKFLOWS_JOB_EXECUTION_TIMED_OUT,
|
|
400
|
-
payload: {
|
|
401
|
-
jobId: identity.jobId,
|
|
402
|
-
jobExecutionId: params.jobExecutionId,
|
|
403
|
-
workflowRunAttemptId: identity.workflowRunAttemptId,
|
|
404
|
-
},
|
|
405
|
-
});
|
|
406
|
-
}
|
|
407
|
-
|
|
408
433
|
return updated;
|
|
409
434
|
});
|
|
410
435
|
|
|
@@ -1,16 +1,79 @@
|
|
|
1
1
|
import {
|
|
2
2
|
type LogOutcomeDto,
|
|
3
|
+
WORKFLOWS_JOB_EXECUTION_QUEUED,
|
|
4
|
+
WORKFLOWS_JOB_EXECUTION_TERMINATED,
|
|
3
5
|
WORKFLOWS_JOB_STEPS_SETTLED,
|
|
4
6
|
WORKFLOWS_STEP_ATTEMPT_TERMINATED,
|
|
5
7
|
WORKFLOWS_STEP_RESTART_ENQUEUED,
|
|
6
8
|
} from '@shipfox/api-workflows-dto';
|
|
7
9
|
import {eq} from 'drizzle-orm';
|
|
10
|
+
import type {JobStatusReason} from '#core/entities/job.js';
|
|
11
|
+
import type {JobExecutionStatus} from '#core/entities/job-execution.js';
|
|
8
12
|
import type {Tx} from '../db.js';
|
|
9
13
|
import {writeWorkflowsOutboxEvent} from '../outbox-writes.js';
|
|
10
14
|
import {jobExecutions} from '../schema/job-executions.js';
|
|
11
15
|
import {steps} from '../schema/steps.js';
|
|
12
16
|
import {getWorkflowContextForJob} from './shared.js';
|
|
13
17
|
|
|
18
|
+
export async function writeJobExecutionQueuedOutbox(
|
|
19
|
+
tx: Tx,
|
|
20
|
+
params: {
|
|
21
|
+
jobId: string;
|
|
22
|
+
jobExecutionId: string;
|
|
23
|
+
requiredLabels: string[];
|
|
24
|
+
queuedAt: Date;
|
|
25
|
+
},
|
|
26
|
+
): Promise<void> {
|
|
27
|
+
const identity = await getWorkflowContextForJob(params.jobId, tx);
|
|
28
|
+
|
|
29
|
+
await writeWorkflowsOutboxEvent(tx, {
|
|
30
|
+
type: WORKFLOWS_JOB_EXECUTION_QUEUED,
|
|
31
|
+
payload: {
|
|
32
|
+
jobId: params.jobId,
|
|
33
|
+
jobExecutionId: params.jobExecutionId,
|
|
34
|
+
workflowRunId: identity.workflowRunId,
|
|
35
|
+
workflowRunAttemptId: identity.workflowRunAttemptId,
|
|
36
|
+
workspaceId: identity.workspaceId,
|
|
37
|
+
projectId: identity.projectId,
|
|
38
|
+
requiredLabels: params.requiredLabels,
|
|
39
|
+
queuedAt: params.queuedAt.toISOString(),
|
|
40
|
+
},
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export async function writeJobExecutionTerminatedOutbox(
|
|
45
|
+
tx: Tx,
|
|
46
|
+
params: {
|
|
47
|
+
jobId: string;
|
|
48
|
+
jobExecutionId: string;
|
|
49
|
+
status: JobExecutionStatus;
|
|
50
|
+
statusReason: JobStatusReason | null;
|
|
51
|
+
statusReasonMessage?: string | null | undefined;
|
|
52
|
+
},
|
|
53
|
+
): Promise<void> {
|
|
54
|
+
if (
|
|
55
|
+
params.status !== 'succeeded' &&
|
|
56
|
+
params.status !== 'failed' &&
|
|
57
|
+
params.status !== 'cancelled'
|
|
58
|
+
) {
|
|
59
|
+
throw new Error(`Cannot enqueue terminal job execution event for status ${params.status}`);
|
|
60
|
+
}
|
|
61
|
+
const identity = await getWorkflowContextForJob(params.jobId, tx);
|
|
62
|
+
|
|
63
|
+
await writeWorkflowsOutboxEvent(tx, {
|
|
64
|
+
type: WORKFLOWS_JOB_EXECUTION_TERMINATED,
|
|
65
|
+
payload: {
|
|
66
|
+
jobId: params.jobId,
|
|
67
|
+
jobExecutionId: params.jobExecutionId,
|
|
68
|
+
workflowRunId: identity.workflowRunId,
|
|
69
|
+
workflowRunAttemptId: identity.workflowRunAttemptId,
|
|
70
|
+
status: params.status,
|
|
71
|
+
statusReason: params.statusReason,
|
|
72
|
+
statusReasonMessage: params.statusReasonMessage ?? null,
|
|
73
|
+
},
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
|
|
14
77
|
// Enqueue the steps-settled signal in the same transaction as the final per-step
|
|
15
78
|
// result, so per-step execution observes it exactly once (the outbox is at-least-once;
|
|
16
79
|
// the job workflow dedupes the signal). Drives the Temporal JOB_FINISHED_SIGNAL; the
|
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
} from '../schema/job-executions.js';
|
|
11
11
|
import {type JobCreateDb, type JobDb, jobs} from '../schema/jobs.js';
|
|
12
12
|
import {type StepCreateDb, steps} from '../schema/steps.js';
|
|
13
|
+
import {writeJobExecutionTerminatedOutbox} from './outbox.js';
|
|
13
14
|
|
|
14
15
|
type MaterializedRunGraphJobExecution = Omit<JobExecutionCreateDb, 'finishedAt' | 'jobId'> & {
|
|
15
16
|
readonly finishedAt?: JobExecutionCreateDb['finishedAt'] | SQL | undefined;
|
|
@@ -59,6 +60,22 @@ export async function persistMaterializedRunGraph(
|
|
|
59
60
|
? []
|
|
60
61
|
: await tx.insert(jobExecutions).values(jobExecutionValues).returning();
|
|
61
62
|
|
|
63
|
+
for (const jobExecution of jobExecutionRows) {
|
|
64
|
+
if (
|
|
65
|
+
jobExecution.status === 'succeeded' ||
|
|
66
|
+
jobExecution.status === 'failed' ||
|
|
67
|
+
jobExecution.status === 'cancelled'
|
|
68
|
+
) {
|
|
69
|
+
await writeJobExecutionTerminatedOutbox(tx, {
|
|
70
|
+
jobId: jobExecution.jobId,
|
|
71
|
+
jobExecutionId: jobExecution.id,
|
|
72
|
+
status: jobExecution.status,
|
|
73
|
+
statusReason: jobExecution.statusReason,
|
|
74
|
+
statusReasonMessage: jobExecution.statusReasonMessage,
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
62
79
|
const jobById = new Map(jobRows.map((job) => [job.id, job]));
|
|
63
80
|
const materializedJobByJobId = new Map(
|
|
64
81
|
jobRows.flatMap((jobRow, index) => {
|
|
@@ -4,6 +4,7 @@ import {nextStepForJob} from '#core/job-execution.js';
|
|
|
4
4
|
import {
|
|
5
5
|
buildModel,
|
|
6
6
|
createTestRun,
|
|
7
|
+
jobExecutionTerminatedEvents,
|
|
7
8
|
jobTerminatedEvents,
|
|
8
9
|
runCancelledEvents,
|
|
9
10
|
runTerminatedEvents,
|
|
@@ -16,6 +17,7 @@ import {
|
|
|
16
17
|
cancelWorkflowRun,
|
|
17
18
|
createRerunWorkflowRun,
|
|
18
19
|
createWorkflowRun,
|
|
20
|
+
getFirstJobExecutionByJobId,
|
|
19
21
|
getJobsByWorkflowRunId,
|
|
20
22
|
getStepsByJobId,
|
|
21
23
|
getWorkflowRunById,
|
|
@@ -257,6 +259,8 @@ describe('workflow run queries', () => {
|
|
|
257
259
|
const [runningJobExecution, succeededJob, skippedJob] = await getJobsByWorkflowRunId(run.id);
|
|
258
260
|
if (!runningJobExecution || !succeededJob || !skippedJob) throw new Error('Expected jobs');
|
|
259
261
|
await updateJobStatus({jobId: runningJobExecution.id, status: 'running', expectedVersion: 1});
|
|
262
|
+
const execution = await getFirstJobExecutionByJobId(runningJobExecution.id);
|
|
263
|
+
if (!execution) throw new Error('Expected running job execution');
|
|
260
264
|
await nextStepForJob(runningJobExecution.id);
|
|
261
265
|
await updateJobStatus({jobId: succeededJob.id, status: 'succeeded', expectedVersion: 1});
|
|
262
266
|
await updateJobStatus({
|
|
@@ -299,6 +303,15 @@ describe('workflow run queries', () => {
|
|
|
299
303
|
statusReason: 'run_cancelled',
|
|
300
304
|
}),
|
|
301
305
|
]);
|
|
306
|
+
expect(await jobExecutionTerminatedEvents(execution.id)).toEqual([
|
|
307
|
+
expect.objectContaining({
|
|
308
|
+
jobId: runningJobExecution.id,
|
|
309
|
+
jobExecutionId: execution.id,
|
|
310
|
+
workflowRunId: run.id,
|
|
311
|
+
status: 'cancelled',
|
|
312
|
+
statusReason: 'run_cancelled',
|
|
313
|
+
}),
|
|
314
|
+
]);
|
|
302
315
|
expect(await stepAttemptTerminatedEvents(runningJobExecution.id)).toHaveLength(1);
|
|
303
316
|
expect(await jobTerminatedEvents(succeededJob.id)).toHaveLength(1);
|
|
304
317
|
expect(await jobTerminatedEvents(skippedJob.id)).toHaveLength(1);
|
|
@@ -19,6 +19,7 @@ import {steps} from '../schema/steps.js';
|
|
|
19
19
|
import {workflowRunAttempts} from '../schema/workflow-run-attempts.js';
|
|
20
20
|
import {toWorkflowRun, workflowRuns} from '../schema/workflow-runs.js';
|
|
21
21
|
import {updateJobStatusAtVersion} from './jobs.js';
|
|
22
|
+
import {writeJobExecutionTerminatedOutbox} from './outbox.js';
|
|
22
23
|
import {
|
|
23
24
|
lockWorkflowRun,
|
|
24
25
|
TERMINAL_EXECUTION_STATUSES,
|
|
@@ -114,8 +115,21 @@ async function terminateRunAttempt(
|
|
|
114
115
|
notInArray(jobExecutions.status, TERMINAL_EXECUTION_STATUSES),
|
|
115
116
|
),
|
|
116
117
|
)
|
|
117
|
-
.returning({
|
|
118
|
+
.returning({
|
|
119
|
+
id: jobExecutions.id,
|
|
120
|
+
jobId: jobExecutions.jobId,
|
|
121
|
+
status: jobExecutions.status,
|
|
122
|
+
statusReason: jobExecutions.statusReason,
|
|
123
|
+
statusReasonMessage: jobExecutions.statusReasonMessage,
|
|
124
|
+
});
|
|
118
125
|
for (const jobExecution of terminatedExecutions) {
|
|
126
|
+
await writeJobExecutionTerminatedOutbox(tx, {
|
|
127
|
+
jobId: jobExecution.jobId,
|
|
128
|
+
jobExecutionId: jobExecution.id,
|
|
129
|
+
status: jobExecution.status,
|
|
130
|
+
statusReason: jobExecution.statusReason,
|
|
131
|
+
statusReasonMessage: jobExecution.statusReasonMessage,
|
|
132
|
+
});
|
|
119
133
|
await bulkUpdateStepStatuses(
|
|
120
134
|
{jobExecutionId: jobExecution.id, status: spec.terminalStatus},
|
|
121
135
|
tx,
|
package/src/db/workflow-runs.ts
CHANGED
|
@@ -9,7 +9,7 @@ export {
|
|
|
9
9
|
getJobExecutionsByJobId,
|
|
10
10
|
getJobExecutionsByWorkflowRunAttemptId,
|
|
11
11
|
getLatestJobExecutionByJobId,
|
|
12
|
-
|
|
12
|
+
queueJobExecution,
|
|
13
13
|
recordJobExecutionStartedAt,
|
|
14
14
|
resolveJobExecutionAfterLeaseExpiry,
|
|
15
15
|
updateJobExecutionStatus,
|
|
@@ -35,6 +35,8 @@ export {
|
|
|
35
35
|
updateJobStatusAtVersion,
|
|
36
36
|
} from './workflow-runs/jobs.js';
|
|
37
37
|
export {
|
|
38
|
+
writeJobExecutionQueuedOutbox,
|
|
39
|
+
writeJobExecutionTerminatedOutbox,
|
|
38
40
|
writeJobStepsSettledOutbox,
|
|
39
41
|
writeStepAttemptTerminatedOutbox,
|
|
40
42
|
writeStepRestartEnqueuedOutbox,
|
package/src/index.ts
CHANGED
|
@@ -9,7 +9,6 @@ import type {ProjectsModuleClient} from '@shipfox/api-projects-dto/inter-module'
|
|
|
9
9
|
import {
|
|
10
10
|
RUNNER_JOB_CLAIMED,
|
|
11
11
|
RUNNER_JOB_LEASE_EXPIRED,
|
|
12
|
-
RUNNER_JOB_QUEUED,
|
|
13
12
|
type RunnersEventMap,
|
|
14
13
|
} from '@shipfox/api-runners-dto';
|
|
15
14
|
import type {RunnersInterModuleClient} from '@shipfox/api-runners-dto/inter-module';
|
|
@@ -35,7 +34,6 @@ import {
|
|
|
35
34
|
onJobTerminatedFailureAnnotation,
|
|
36
35
|
onRunnerJobClaimed,
|
|
37
36
|
onRunnerJobLeaseExpired,
|
|
38
|
-
onRunnerJobQueued,
|
|
39
37
|
onStepAttemptTerminatedFailureAnnotation,
|
|
40
38
|
onWorkflowRunAttemptCreated,
|
|
41
39
|
onWorkflowRunCancelled,
|
|
@@ -128,15 +126,13 @@ export function createWorkflowsModule({
|
|
|
128
126
|
),
|
|
129
127
|
subscriber(WORKFLOWS_JOB_TERMINATED, onJobTerminatedFailureAnnotation(annotations)),
|
|
130
128
|
subscriber(RUNNER_JOB_LEASE_EXPIRED, onRunnerJobLeaseExpired),
|
|
131
|
-
subscriber(RUNNER_JOB_QUEUED, onRunnerJobQueued),
|
|
132
129
|
subscriber(RUNNER_JOB_CLAIMED, onRunnerJobClaimed),
|
|
133
130
|
],
|
|
134
131
|
workers: [
|
|
135
132
|
{
|
|
136
133
|
taskQueue: WORKFLOWS_TASK_QUEUE,
|
|
137
134
|
workflowsPath,
|
|
138
|
-
activities: () =>
|
|
139
|
-
createOrchestrationActivities({agent, integrations, projects, runners, secrets}),
|
|
135
|
+
activities: () => createOrchestrationActivities({agent, integrations, projects, secrets}),
|
|
140
136
|
workflows: [],
|
|
141
137
|
},
|
|
142
138
|
],
|
|
@@ -6,6 +6,5 @@ export {onJobEventDelivered} from './on-job-event-delivered.js';
|
|
|
6
6
|
export {onJobStepsSettled} from './on-job-steps-settled.js';
|
|
7
7
|
export {onRunnerJobClaimed} from './on-runner-job-claimed.js';
|
|
8
8
|
export {onRunnerJobLeaseExpired} from './on-runner-job-lease-expired.js';
|
|
9
|
-
export {onRunnerJobQueued} from './on-runner-job-queued.js';
|
|
10
9
|
export {onWorkflowRunAttemptCreated} from './on-workflow-run-attempt-created.js';
|
|
11
10
|
export {onWorkflowRunCancelled} from './on-workflow-run-cancelled.js';
|