@shipfox/api-workflows 12.6.0 → 13.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +8 -8
- package/CHANGELOG.md +23 -0
- package/dist/core/entities/workflow-run.d.ts +3 -0
- package/dist/core/entities/workflow-run.d.ts.map +1 -1
- package/dist/core/entities/workflow-run.js.map +1 -1
- package/dist/db/index.d.ts +1 -1
- package/dist/db/index.d.ts.map +1 -1
- package/dist/db/index.js +1 -1
- package/dist/db/index.js.map +1 -1
- package/dist/db/job-listeners.d.ts.map +1 -1
- package/dist/db/job-listeners.js +17 -1
- package/dist/db/job-listeners.js.map +1 -1
- package/dist/db/workflow-runs/job-executions.d.ts +2 -3
- package/dist/db/workflow-runs/job-executions.d.ts.map +1 -1
- package/dist/db/workflow-runs/job-executions.js +48 -20
- package/dist/db/workflow-runs/job-executions.js.map +1 -1
- package/dist/db/workflow-runs/outbox.d.ts +15 -0
- package/dist/db/workflow-runs/outbox.d.ts.map +1 -1
- package/dist/db/workflow-runs/outbox.js +35 -1
- package/dist/db/workflow-runs/outbox.js.map +1 -1
- package/dist/db/workflow-runs/queries.d.ts +5 -3
- package/dist/db/workflow-runs/queries.d.ts.map +1 -1
- package/dist/db/workflow-runs/queries.js +20 -6
- package/dist/db/workflow-runs/queries.js.map +1 -1
- package/dist/db/workflow-runs/run-graph.d.ts.map +1 -1
- package/dist/db/workflow-runs/run-graph.js +12 -0
- package/dist/db/workflow-runs/run-graph.js.map +1 -1
- package/dist/db/workflow-runs/run-status.d.ts.map +1 -1
- package/dist/db/workflow-runs/run-status.js +13 -1
- package/dist/db/workflow-runs/run-status.js.map +1 -1
- package/dist/db/workflow-runs.d.ts +2 -2
- package/dist/db/workflow-runs.d.ts.map +1 -1
- package/dist/db/workflow-runs.js +2 -2
- package/dist/db/workflow-runs.js.map +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -4
- package/dist/index.js.map +1 -1
- package/dist/presentation/dto/workflow-run.d.ts.map +1 -1
- package/dist/presentation/dto/workflow-run.js +4 -2
- package/dist/presentation/dto/workflow-run.js.map +1 -1
- package/dist/presentation/index.d.ts +1 -1
- package/dist/presentation/index.d.ts.map +1 -1
- package/dist/presentation/index.js +1 -1
- package/dist/presentation/index.js.map +1 -1
- package/dist/presentation/routes/get-run.d.ts.map +1 -1
- package/dist/presentation/routes/get-run.js +2 -1
- package/dist/presentation/routes/get-run.js.map +1 -1
- package/dist/presentation/subscribers/index.d.ts +0 -1
- package/dist/presentation/subscribers/index.d.ts.map +1 -1
- package/dist/presentation/subscribers/index.js +0 -1
- package/dist/presentation/subscribers/index.js.map +1 -1
- package/dist/temporal/activities/index.d.ts +2 -18
- package/dist/temporal/activities/index.d.ts.map +1 -1
- package/dist/temporal/activities/index.js +3 -5
- package/dist/temporal/activities/index.js.map +1 -1
- package/dist/temporal/activities/orchestration-activities.d.ts +5 -14
- package/dist/temporal/activities/orchestration-activities.d.ts.map +1 -1
- package/dist/temporal/activities/orchestration-activities.js +13 -34
- package/dist/temporal/activities/orchestration-activities.js.map +1 -1
- package/dist/temporal/workflows/index.bundle.js +20 -68
- package/dist/temporal/workflows/job-execution-orchestration.d.ts +0 -3
- package/dist/temporal/workflows/job-execution-orchestration.d.ts.map +1 -1
- package/dist/temporal/workflows/job-execution-orchestration.js +17 -41
- package/dist/temporal/workflows/job-execution-orchestration.js.map +1 -1
- package/dist/temporal/workflows/job-listener-orchestration.d.ts +0 -3
- package/dist/temporal/workflows/job-listener-orchestration.d.ts.map +1 -1
- package/dist/temporal/workflows/job-listener-orchestration.js +1 -9
- package/dist/temporal/workflows/job-listener-orchestration.js.map +1 -1
- package/dist/temporal/workflows/run-orchestration.d.ts.map +1 -1
- package/dist/temporal/workflows/run-orchestration.js +2 -18
- package/dist/temporal/workflows/run-orchestration.js.map +1 -1
- package/dist/temporal/workflows/test-env.d.ts +4 -6
- package/dist/temporal/workflows/test-env.d.ts.map +1 -1
- package/dist/temporal/workflows/test-env.js +13 -24
- package/dist/temporal/workflows/test-env.js.map +1 -1
- package/dist/tsconfig.test.tsbuildinfo +1 -1
- package/package.json +3 -3
- package/src/core/entities/workflow-run.ts +3 -0
- package/src/db/index.ts +1 -1
- package/src/db/job-listeners.test.ts +21 -0
- package/src/db/job-listeners.ts +18 -0
- package/src/db/workflow-runs/job-executions.test.ts +110 -12
- package/src/db/workflow-runs/job-executions.ts +54 -29
- package/src/db/workflow-runs/outbox.ts +63 -0
- package/src/db/workflow-runs/queries.test.ts +27 -0
- package/src/db/workflow-runs/queries.ts +31 -5
- package/src/db/workflow-runs/run-graph.ts +17 -0
- package/src/db/workflow-runs/run-status.test.ts +13 -0
- package/src/db/workflow-runs/run-status.ts +15 -1
- package/src/db/workflow-runs.ts +3 -1
- package/src/index.ts +1 -5
- package/src/presentation/dto/workflow-run.ts +7 -1
- package/src/presentation/index.ts +0 -1
- package/src/presentation/routes/get-run.test.ts +31 -0
- package/src/presentation/routes/get-run.ts +1 -0
- package/src/presentation/routes/list-runs.test.ts +5 -1
- package/src/presentation/subscribers/index.ts +0 -1
- package/src/temporal/activities/index.ts +2 -8
- package/src/temporal/activities/orchestration-activities.test.ts +17 -1
- package/src/temporal/activities/orchestration-activities.ts +13 -46
- package/src/temporal/workflows/job-execution-orchestration.test.ts +6 -25
- package/src/temporal/workflows/job-execution-orchestration.ts +17 -43
- package/src/temporal/workflows/job-listener-orchestration-continuation.test.ts +0 -3
- package/src/temporal/workflows/job-listener-orchestration.test.ts +9 -12
- package/src/temporal/workflows/job-listener-orchestration.ts +0 -8
- package/src/temporal/workflows/run-orchestration.test.ts +22 -33
- package/src/temporal/workflows/run-orchestration.ts +4 -28
- package/src/temporal/workflows/test-env.ts +14 -34
- package/test/fixtures/runners-inter-module.ts +0 -3
- package/test/helpers/workflow-runs.ts +25 -0
- package/tsconfig.build.tsbuildinfo +1 -1
- package/dist/presentation/subscribers/on-runner-job-queued.d.ts +0 -3
- package/dist/presentation/subscribers/on-runner-job-queued.d.ts.map +0 -1
- package/dist/presentation/subscribers/on-runner-job-queued.js +0 -20
- package/dist/presentation/subscribers/on-runner-job-queued.js.map +0 -1
- package/src/presentation/subscribers/on-runner-job-queued.test.ts +0 -57
- package/src/presentation/subscribers/on-runner-job-queued.ts +0 -23
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@shipfox/api-workflows",
|
|
3
3
|
"license": "MIT",
|
|
4
|
-
"version": "
|
|
4
|
+
"version": "13.0.0",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
7
7
|
"url": "git+https://github.com/ShipfoxHQ/shipfox.git",
|
|
@@ -30,9 +30,9 @@
|
|
|
30
30
|
"@shipfox/api-definitions-dto": "12.3.0",
|
|
31
31
|
"@shipfox/api-projects-dto": "12.2.0",
|
|
32
32
|
"@shipfox/api-integration-core-dto": "12.2.0",
|
|
33
|
-
"@shipfox/api-runners-dto": "
|
|
33
|
+
"@shipfox/api-runners-dto": "13.0.0",
|
|
34
34
|
"@shipfox/api-secrets-dto": "12.0.0",
|
|
35
|
-
"@shipfox/api-workflows-dto": "
|
|
35
|
+
"@shipfox/api-workflows-dto": "13.0.0",
|
|
36
36
|
"@shipfox/api-workspaces-dto": "12.0.0",
|
|
37
37
|
"@shipfox/config": "1.2.4",
|
|
38
38
|
"@shipfox/inter-module": "0.2.3",
|
|
@@ -107,4 +107,7 @@ export interface WorkflowRunDetail extends WorkflowRun {
|
|
|
107
107
|
runAttempt: WorkflowRunAttempt;
|
|
108
108
|
latestAttempt: number;
|
|
109
109
|
jobs: WorkflowJobDetail[];
|
|
110
|
+
/** Whether any job execution of this attempt reached its runner, decided here so the detail
|
|
111
|
+
* and the list cannot answer it differently. */
|
|
112
|
+
hasStartedJobExecution: boolean;
|
|
110
113
|
}
|
package/src/db/index.ts
CHANGED
|
@@ -15,6 +15,7 @@ import {
|
|
|
15
15
|
drainListenerEventsIntoExecution,
|
|
16
16
|
peekListenerBuffer,
|
|
17
17
|
resolveJobListener,
|
|
18
|
+
settleListenerJobExecution,
|
|
18
19
|
} from '#db/job-listeners.js';
|
|
19
20
|
import {jobExecutions} from '#db/schema/job-executions.js';
|
|
20
21
|
import {jobListenerEvents} from '#db/schema/job-listener-events.js';
|
|
@@ -22,6 +23,7 @@ import {jobs} from '#db/schema/jobs.js';
|
|
|
22
23
|
import {workflowsOutbox} from '#db/schema/outbox.js';
|
|
23
24
|
import {steps} from '#db/schema/steps.js';
|
|
24
25
|
import {workflowRunAttempts} from '#db/schema/workflow-run-attempts.js';
|
|
26
|
+
import {jobExecutionTerminatedEvents} from '#test/helpers/workflow-runs.js';
|
|
25
27
|
import {jobFactory, workflowModel, workflowRunFactory} from '#test/index.js';
|
|
26
28
|
import {getJobsByWorkflowRunId, updateJobExecutionStatus} from './workflow-runs.js';
|
|
27
29
|
|
|
@@ -127,6 +129,25 @@ function expectListeningPayload(
|
|
|
127
129
|
expect(payload.mode).toBe('listening');
|
|
128
130
|
}
|
|
129
131
|
|
|
132
|
+
describe('settleListenerJobExecution', () => {
|
|
133
|
+
it('writes a terminal job-execution fact for a cancelled listener firing', async () => {
|
|
134
|
+
const job = await createListeningJob();
|
|
135
|
+
const execution = await insertExecution(job.id, 1, 'running');
|
|
136
|
+
|
|
137
|
+
await settleListenerJobExecution({jobExecutionId: execution.id, status: 'cancelled'});
|
|
138
|
+
await settleListenerJobExecution({jobExecutionId: execution.id, status: 'cancelled'});
|
|
139
|
+
|
|
140
|
+
expect(await jobExecutionTerminatedEvents(execution.id)).toEqual([
|
|
141
|
+
expect.objectContaining({
|
|
142
|
+
jobId: job.id,
|
|
143
|
+
jobExecutionId: execution.id,
|
|
144
|
+
status: 'cancelled',
|
|
145
|
+
statusReason: 'run_cancelled',
|
|
146
|
+
}),
|
|
147
|
+
]);
|
|
148
|
+
});
|
|
149
|
+
});
|
|
150
|
+
|
|
130
151
|
async function createInactiveListeningJobWithMatchers(params: {
|
|
131
152
|
readonly on: readonly JobListeningTrigger[];
|
|
132
153
|
readonly until?: readonly JobListeningTrigger[] | null;
|
package/src/db/job-listeners.ts
CHANGED
|
@@ -48,6 +48,7 @@ import {
|
|
|
48
48
|
getDirectDependencyJobContexts,
|
|
49
49
|
loadReferencedVariables,
|
|
50
50
|
updateJobStatusAtVersion,
|
|
51
|
+
writeJobExecutionTerminatedOutbox,
|
|
51
52
|
} from './workflow-runs.js';
|
|
52
53
|
|
|
53
54
|
const TERMINAL_EXECUTION_STATUSES: JobExecutionStatus[] = ['succeeded', 'failed', 'cancelled'];
|
|
@@ -481,6 +482,13 @@ export async function settleListenerJobExecution(params: {
|
|
|
481
482
|
)
|
|
482
483
|
.returning();
|
|
483
484
|
if (!execution) return false;
|
|
485
|
+
await writeJobExecutionTerminatedOutbox(tx, {
|
|
486
|
+
jobId: execution.jobId,
|
|
487
|
+
jobExecutionId: execution.id,
|
|
488
|
+
status: execution.status,
|
|
489
|
+
statusReason: execution.statusReason,
|
|
490
|
+
statusReasonMessage: execution.statusReasonMessage,
|
|
491
|
+
});
|
|
484
492
|
await bulkUpdateStepStatuses(
|
|
485
493
|
{jobExecutionId: params.jobExecutionId, status: params.status},
|
|
486
494
|
tx,
|
|
@@ -615,6 +623,16 @@ async function persistMaterializedListenerExecution(
|
|
|
615
623
|
.returning();
|
|
616
624
|
if (!execution) throw new Error('Insert returned no rows');
|
|
617
625
|
|
|
626
|
+
if (execution.status === 'failed') {
|
|
627
|
+
await writeJobExecutionTerminatedOutbox(tx, {
|
|
628
|
+
jobId: execution.jobId,
|
|
629
|
+
jobExecutionId: execution.id,
|
|
630
|
+
status: execution.status,
|
|
631
|
+
statusReason: execution.statusReason,
|
|
632
|
+
statusReasonMessage: execution.statusReasonMessage,
|
|
633
|
+
});
|
|
634
|
+
}
|
|
635
|
+
|
|
618
636
|
await tx
|
|
619
637
|
.update(jobListenerEvents)
|
|
620
638
|
.set({consumedByExecutionId: execution.id})
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {WORKFLOWS_JOB_EXECUTION_QUEUED} from '@shipfox/api-workflows-dto';
|
|
2
2
|
import {and, eq, sql} from 'drizzle-orm';
|
|
3
3
|
import {
|
|
4
4
|
InterpolationUnresolvableError,
|
|
@@ -10,18 +10,23 @@ import {
|
|
|
10
10
|
MAX_JOB_OUTPUT_ENTRIES,
|
|
11
11
|
MAX_JOB_OUTPUT_VALUE_BYTES,
|
|
12
12
|
} from '#core/step-config/job-output-limits.js';
|
|
13
|
-
import {
|
|
13
|
+
import {
|
|
14
|
+
buildModel,
|
|
15
|
+
jobExecutionTerminatedEvents,
|
|
16
|
+
template,
|
|
17
|
+
workflowRunAttemptId,
|
|
18
|
+
} from '#test/helpers/workflow-runs.js';
|
|
14
19
|
import {db} from '../db.js';
|
|
15
20
|
import {workflowsOutbox} from '../schema/outbox.js';
|
|
16
21
|
import {
|
|
17
22
|
applyStepResult,
|
|
18
23
|
createWorkflowRun,
|
|
19
|
-
failJobExecutionAsTimedOut,
|
|
20
24
|
finishStepAttempt,
|
|
21
25
|
getFirstJobExecutionByJobId,
|
|
22
26
|
getJobsByWorkflowRunId,
|
|
23
27
|
getStepsByJobId,
|
|
24
28
|
markStepRunning,
|
|
29
|
+
queueJobExecution,
|
|
25
30
|
resolveJobExecutionAfterLeaseExpiry,
|
|
26
31
|
updateJobExecutionStatus,
|
|
27
32
|
} from '../workflow-runs.js';
|
|
@@ -38,7 +43,7 @@ describe('workflow run job executions', () => {
|
|
|
38
43
|
definitionId = crypto.randomUUID();
|
|
39
44
|
});
|
|
40
45
|
|
|
41
|
-
test('
|
|
46
|
+
test('records and publishes the queue fact once', async () => {
|
|
42
47
|
const run = await createWorkflowRun({
|
|
43
48
|
workspaceId,
|
|
44
49
|
projectId,
|
|
@@ -57,26 +62,119 @@ describe('workflow run job executions', () => {
|
|
|
57
62
|
if (!execution) throw new Error('Expected job execution');
|
|
58
63
|
const actualAttemptId = await workflowRunAttemptId(run.id);
|
|
59
64
|
|
|
60
|
-
await
|
|
61
|
-
|
|
62
|
-
workflowRunAttemptId: crypto.randomUUID(),
|
|
63
|
-
expectedVersion: execution.version,
|
|
64
|
-
});
|
|
65
|
+
const queued = await queueJobExecution({jobExecutionId: execution.id});
|
|
66
|
+
await queueJobExecution({jobExecutionId: execution.id});
|
|
65
67
|
|
|
66
|
-
const
|
|
68
|
+
const events = await db()
|
|
67
69
|
.select({payload: workflowsOutbox.payload})
|
|
68
70
|
.from(workflowsOutbox)
|
|
69
71
|
.where(
|
|
70
72
|
and(
|
|
71
|
-
eq(workflowsOutbox.eventType,
|
|
73
|
+
eq(workflowsOutbox.eventType, WORKFLOWS_JOB_EXECUTION_QUEUED),
|
|
72
74
|
sql`${workflowsOutbox.payload}->>'jobExecutionId' = ${execution.id}`,
|
|
73
75
|
),
|
|
74
76
|
);
|
|
75
|
-
expect(
|
|
77
|
+
expect(events).toHaveLength(1);
|
|
78
|
+
expect(events[0]?.payload).toMatchObject({
|
|
76
79
|
jobId: job.id,
|
|
77
80
|
jobExecutionId: execution.id,
|
|
81
|
+
workflowRunId: run.id,
|
|
78
82
|
workflowRunAttemptId: actualAttemptId,
|
|
83
|
+
workspaceId,
|
|
84
|
+
projectId,
|
|
85
|
+
requiredLabels: ['ubuntu-latest'],
|
|
86
|
+
queuedAt: queued.queuedAt?.toISOString(),
|
|
87
|
+
});
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
test('writes one terminal fact when a job execution becomes terminal', async () => {
|
|
91
|
+
const run = await createWorkflowRun({
|
|
92
|
+
workspaceId,
|
|
93
|
+
projectId,
|
|
94
|
+
definitionId,
|
|
95
|
+
model: buildModel({jobs: {build: {steps: [{run: 'echo build'}]}}}),
|
|
96
|
+
triggerPayload: {
|
|
97
|
+
source: 'manual',
|
|
98
|
+
event: 'fire',
|
|
99
|
+
subscriptionId: crypto.randomUUID(),
|
|
100
|
+
userId: crypto.randomUUID(),
|
|
101
|
+
},
|
|
102
|
+
});
|
|
103
|
+
const [job] = await getJobsByWorkflowRunId(run.id);
|
|
104
|
+
if (!job) throw new Error('Expected workflow job');
|
|
105
|
+
const execution = await getFirstJobExecutionByJobId(job.id);
|
|
106
|
+
if (!execution) throw new Error('Expected job execution');
|
|
107
|
+
|
|
108
|
+
const running = await updateJobExecutionStatus({
|
|
109
|
+
jobExecutionId: execution.id,
|
|
110
|
+
status: 'running',
|
|
111
|
+
expectedVersion: execution.version,
|
|
79
112
|
});
|
|
113
|
+
expect(await jobExecutionTerminatedEvents(execution.id)).toHaveLength(0);
|
|
114
|
+
|
|
115
|
+
await updateJobExecutionStatus({
|
|
116
|
+
jobExecutionId: execution.id,
|
|
117
|
+
status: 'cancelled',
|
|
118
|
+
expectedVersion: running.version,
|
|
119
|
+
statusReason: 'run_cancelled',
|
|
120
|
+
});
|
|
121
|
+
await updateJobExecutionStatus({
|
|
122
|
+
jobExecutionId: execution.id,
|
|
123
|
+
status: 'failed',
|
|
124
|
+
expectedVersion: running.version + 1,
|
|
125
|
+
statusReason: 'unknown',
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
expect(await jobExecutionTerminatedEvents(execution.id)).toEqual([
|
|
129
|
+
expect.objectContaining({
|
|
130
|
+
jobId: job.id,
|
|
131
|
+
jobExecutionId: execution.id,
|
|
132
|
+
workflowRunId: run.id,
|
|
133
|
+
status: 'cancelled',
|
|
134
|
+
statusReason: 'run_cancelled',
|
|
135
|
+
}),
|
|
136
|
+
]);
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
test('does not publish a queue fact after the execution is terminal', async () => {
|
|
140
|
+
const run = await createWorkflowRun({
|
|
141
|
+
workspaceId,
|
|
142
|
+
projectId,
|
|
143
|
+
definitionId,
|
|
144
|
+
model: buildModel({jobs: {build: {steps: [{run: 'echo build'}]}}}),
|
|
145
|
+
triggerPayload: {
|
|
146
|
+
source: 'manual',
|
|
147
|
+
event: 'fire',
|
|
148
|
+
subscriptionId: crypto.randomUUID(),
|
|
149
|
+
userId: crypto.randomUUID(),
|
|
150
|
+
},
|
|
151
|
+
});
|
|
152
|
+
const [job] = await getJobsByWorkflowRunId(run.id);
|
|
153
|
+
if (!job) throw new Error('Expected workflow job');
|
|
154
|
+
const execution = await getFirstJobExecutionByJobId(job.id);
|
|
155
|
+
if (!execution) throw new Error('Expected job execution');
|
|
156
|
+
|
|
157
|
+
await updateJobExecutionStatus({
|
|
158
|
+
jobExecutionId: execution.id,
|
|
159
|
+
status: 'cancelled',
|
|
160
|
+
expectedVersion: execution.version,
|
|
161
|
+
statusReason: 'run_cancelled',
|
|
162
|
+
});
|
|
163
|
+
const after = await queueJobExecution({jobExecutionId: execution.id});
|
|
164
|
+
|
|
165
|
+
expect(after.status).toBe('cancelled');
|
|
166
|
+
expect(after.queuedAt).toBeNull();
|
|
167
|
+
expect(
|
|
168
|
+
await db()
|
|
169
|
+
.select()
|
|
170
|
+
.from(workflowsOutbox)
|
|
171
|
+
.where(
|
|
172
|
+
and(
|
|
173
|
+
eq(workflowsOutbox.eventType, WORKFLOWS_JOB_EXECUTION_QUEUED),
|
|
174
|
+
sql`${workflowsOutbox.payload}->>'jobExecutionId' = ${execution.id}`,
|
|
175
|
+
),
|
|
176
|
+
),
|
|
177
|
+
).toHaveLength(0);
|
|
80
178
|
});
|
|
81
179
|
|
|
82
180
|
test('does not cancel steps when lease expiry loses the execution version race', async () => {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {readPersistedWorkflowModel} from '@shipfox/api-definitions-dto';
|
|
2
2
|
import type {SecretsInterModuleClient} from '@shipfox/api-secrets-dto/inter-module';
|
|
3
|
-
import {
|
|
3
|
+
import {canonicalizeLabels} from '@shipfox/runner-labels';
|
|
4
4
|
import {and, asc, desc, eq, isNull, notInArray, sql} from 'drizzle-orm';
|
|
5
5
|
import type {JobStatusReason} from '#core/entities/job.js';
|
|
6
6
|
import type {JobExecution, JobExecutionStatus} from '#core/entities/job-execution.js';
|
|
@@ -22,7 +22,6 @@ import {
|
|
|
22
22
|
recordWorkflowJobExecutionTimedOut,
|
|
23
23
|
} from '#metrics/instance.js';
|
|
24
24
|
import {db, type Tx} from '../db.js';
|
|
25
|
-
import {writeWorkflowsOutboxEvent} from '../outbox-writes.js';
|
|
26
25
|
import {jobExecutions, toJobExecution} from '../schema/job-executions.js';
|
|
27
26
|
import {jobs, toJob} from '../schema/jobs.js';
|
|
28
27
|
import {stepAttempts, toStepAttempt} from '../schema/step-attempts.js';
|
|
@@ -30,12 +29,9 @@ import {steps, toStep} from '../schema/steps.js';
|
|
|
30
29
|
import {workflowRunAttempts} from '../schema/workflow-run-attempts.js';
|
|
31
30
|
import {toWorkflowRun, workflowRuns} from '../schema/workflow-runs.js';
|
|
32
31
|
import {getDirectDependencyJobContexts} from './jobs.js';
|
|
32
|
+
import {writeJobExecutionQueuedOutbox, writeJobExecutionTerminatedOutbox} from './outbox.js';
|
|
33
33
|
import {loadReferencedVariables} from './runs.js';
|
|
34
|
-
import {
|
|
35
|
-
getWorkflowContextForJob,
|
|
36
|
-
optimisticLockRetry,
|
|
37
|
-
TERMINAL_EXECUTION_STATUSES,
|
|
38
|
-
} from './shared.js';
|
|
34
|
+
import {optimisticLockRetry, TERMINAL_EXECUTION_STATUSES} from './shared.js';
|
|
39
35
|
import {bulkUpdateStepStatuses, getStepsByJobExecutionIdForUpdate} from './steps.js';
|
|
40
36
|
|
|
41
37
|
async function getJobExecutionFallbackName(tx: Tx, jobExecutionId: string): Promise<string> {
|
|
@@ -282,6 +278,15 @@ async function updateJobExecutionStatusAtVersion(
|
|
|
282
278
|
|
|
283
279
|
const row = rows[0];
|
|
284
280
|
if (!row) return null;
|
|
281
|
+
if (TERMINAL_EXECUTION_STATUSES.includes(row.status)) {
|
|
282
|
+
await writeJobExecutionTerminatedOutbox(tx, {
|
|
283
|
+
jobId: row.jobId,
|
|
284
|
+
jobExecutionId: row.id,
|
|
285
|
+
status: row.status,
|
|
286
|
+
statusReason: row.statusReason,
|
|
287
|
+
statusReasonMessage: row.statusReasonMessage,
|
|
288
|
+
});
|
|
289
|
+
}
|
|
285
290
|
return {
|
|
286
291
|
execution: toJobExecution(row, await getJobExecutionFallbackName(tx, row.id)),
|
|
287
292
|
changed: true,
|
|
@@ -336,17 +341,49 @@ export async function updateJobExecutionStatus(
|
|
|
336
341
|
return result.execution;
|
|
337
342
|
}
|
|
338
343
|
|
|
339
|
-
export async function
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
344
|
+
export async function queueJobExecution(params: {jobExecutionId: string}): Promise<JobExecution> {
|
|
345
|
+
const result = await db().transaction(async (tx) => {
|
|
346
|
+
const [row] = await tx
|
|
347
|
+
.select({jobExecution: jobExecutions, job: jobs})
|
|
348
|
+
.from(jobExecutions)
|
|
349
|
+
.innerJoin(jobs, eq(jobExecutions.jobId, jobs.id))
|
|
350
|
+
.where(eq(jobExecutions.id, params.jobExecutionId))
|
|
351
|
+
.limit(1)
|
|
352
|
+
.for('update');
|
|
353
|
+
if (!row) throw new JobNotFoundError(params.jobExecutionId);
|
|
354
|
+
|
|
355
|
+
const existing = toJobExecution(row.jobExecution, row.job.name ?? row.job.key);
|
|
356
|
+
if (existing.queuedAt !== null || TERMINAL_EXECUTION_STATUSES.includes(existing.status)) {
|
|
357
|
+
return {execution: existing, changed: false};
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
const requiredLabels = [...canonicalizeLabels(existing.runner ?? row.job.runner ?? [])];
|
|
361
|
+
if (requiredLabels.length === 0) {
|
|
362
|
+
throw new Error(`Job execution ${params.jobExecutionId} has no required runner labels`);
|
|
363
|
+
}
|
|
348
364
|
|
|
349
|
-
|
|
365
|
+
const [queued] = await tx
|
|
366
|
+
.update(jobExecutions)
|
|
367
|
+
.set({queuedAt: sql`now()`})
|
|
368
|
+
.where(eq(jobExecutions.id, params.jobExecutionId))
|
|
369
|
+
.returning();
|
|
370
|
+
if (!queued?.queuedAt) throw new Error(`Cannot queue job execution ${params.jobExecutionId}`);
|
|
371
|
+
|
|
372
|
+
await writeJobExecutionQueuedOutbox(tx, {
|
|
373
|
+
jobId: queued.jobId,
|
|
374
|
+
jobExecutionId: queued.id,
|
|
375
|
+
requiredLabels,
|
|
376
|
+
queuedAt: queued.queuedAt,
|
|
377
|
+
});
|
|
378
|
+
|
|
379
|
+
return {
|
|
380
|
+
execution: toJobExecution(queued, row.job.name ?? row.job.key),
|
|
381
|
+
changed: true,
|
|
382
|
+
};
|
|
383
|
+
});
|
|
384
|
+
|
|
385
|
+
if (result.changed) recordWorkflowJobExecutionQueued();
|
|
386
|
+
return result.execution;
|
|
350
387
|
}
|
|
351
388
|
|
|
352
389
|
export async function recordJobExecutionStartedAt(params: {
|
|
@@ -393,18 +430,6 @@ export async function failJobExecutionAsTimedOut(params: {
|
|
|
393
430
|
failureMessage: `Optimistic lock failure: job execution ${params.jobExecutionId} version ${params.expectedVersion}`,
|
|
394
431
|
});
|
|
395
432
|
|
|
396
|
-
if (updated.changed) {
|
|
397
|
-
const identity = await getWorkflowContextForJob(updated.execution.jobId, tx);
|
|
398
|
-
await writeWorkflowsOutboxEvent(tx, {
|
|
399
|
-
type: WORKFLOWS_JOB_EXECUTION_TIMED_OUT,
|
|
400
|
-
payload: {
|
|
401
|
-
jobId: identity.jobId,
|
|
402
|
-
jobExecutionId: params.jobExecutionId,
|
|
403
|
-
workflowRunAttemptId: identity.workflowRunAttemptId,
|
|
404
|
-
},
|
|
405
|
-
});
|
|
406
|
-
}
|
|
407
|
-
|
|
408
433
|
return updated;
|
|
409
434
|
});
|
|
410
435
|
|
|
@@ -1,16 +1,79 @@
|
|
|
1
1
|
import {
|
|
2
2
|
type LogOutcomeDto,
|
|
3
|
+
WORKFLOWS_JOB_EXECUTION_QUEUED,
|
|
4
|
+
WORKFLOWS_JOB_EXECUTION_TERMINATED,
|
|
3
5
|
WORKFLOWS_JOB_STEPS_SETTLED,
|
|
4
6
|
WORKFLOWS_STEP_ATTEMPT_TERMINATED,
|
|
5
7
|
WORKFLOWS_STEP_RESTART_ENQUEUED,
|
|
6
8
|
} from '@shipfox/api-workflows-dto';
|
|
7
9
|
import {eq} from 'drizzle-orm';
|
|
10
|
+
import type {JobStatusReason} from '#core/entities/job.js';
|
|
11
|
+
import type {JobExecutionStatus} from '#core/entities/job-execution.js';
|
|
8
12
|
import type {Tx} from '../db.js';
|
|
9
13
|
import {writeWorkflowsOutboxEvent} from '../outbox-writes.js';
|
|
10
14
|
import {jobExecutions} from '../schema/job-executions.js';
|
|
11
15
|
import {steps} from '../schema/steps.js';
|
|
12
16
|
import {getWorkflowContextForJob} from './shared.js';
|
|
13
17
|
|
|
18
|
+
export async function writeJobExecutionQueuedOutbox(
|
|
19
|
+
tx: Tx,
|
|
20
|
+
params: {
|
|
21
|
+
jobId: string;
|
|
22
|
+
jobExecutionId: string;
|
|
23
|
+
requiredLabels: string[];
|
|
24
|
+
queuedAt: Date;
|
|
25
|
+
},
|
|
26
|
+
): Promise<void> {
|
|
27
|
+
const identity = await getWorkflowContextForJob(params.jobId, tx);
|
|
28
|
+
|
|
29
|
+
await writeWorkflowsOutboxEvent(tx, {
|
|
30
|
+
type: WORKFLOWS_JOB_EXECUTION_QUEUED,
|
|
31
|
+
payload: {
|
|
32
|
+
jobId: params.jobId,
|
|
33
|
+
jobExecutionId: params.jobExecutionId,
|
|
34
|
+
workflowRunId: identity.workflowRunId,
|
|
35
|
+
workflowRunAttemptId: identity.workflowRunAttemptId,
|
|
36
|
+
workspaceId: identity.workspaceId,
|
|
37
|
+
projectId: identity.projectId,
|
|
38
|
+
requiredLabels: params.requiredLabels,
|
|
39
|
+
queuedAt: params.queuedAt.toISOString(),
|
|
40
|
+
},
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export async function writeJobExecutionTerminatedOutbox(
|
|
45
|
+
tx: Tx,
|
|
46
|
+
params: {
|
|
47
|
+
jobId: string;
|
|
48
|
+
jobExecutionId: string;
|
|
49
|
+
status: JobExecutionStatus;
|
|
50
|
+
statusReason: JobStatusReason | null;
|
|
51
|
+
statusReasonMessage?: string | null | undefined;
|
|
52
|
+
},
|
|
53
|
+
): Promise<void> {
|
|
54
|
+
if (
|
|
55
|
+
params.status !== 'succeeded' &&
|
|
56
|
+
params.status !== 'failed' &&
|
|
57
|
+
params.status !== 'cancelled'
|
|
58
|
+
) {
|
|
59
|
+
throw new Error(`Cannot enqueue terminal job execution event for status ${params.status}`);
|
|
60
|
+
}
|
|
61
|
+
const identity = await getWorkflowContextForJob(params.jobId, tx);
|
|
62
|
+
|
|
63
|
+
await writeWorkflowsOutboxEvent(tx, {
|
|
64
|
+
type: WORKFLOWS_JOB_EXECUTION_TERMINATED,
|
|
65
|
+
payload: {
|
|
66
|
+
jobId: params.jobId,
|
|
67
|
+
jobExecutionId: params.jobExecutionId,
|
|
68
|
+
workflowRunId: identity.workflowRunId,
|
|
69
|
+
workflowRunAttemptId: identity.workflowRunAttemptId,
|
|
70
|
+
status: params.status,
|
|
71
|
+
statusReason: params.statusReason,
|
|
72
|
+
statusReasonMessage: params.statusReasonMessage ?? null,
|
|
73
|
+
},
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
|
|
14
77
|
// Enqueue the steps-settled signal in the same transaction as the final per-step
|
|
15
78
|
// result, so per-step execution observes it exactly once (the outbox is at-least-once;
|
|
16
79
|
// the job workflow dedupes the signal). Drives the Temporal JOB_FINISHED_SIGNAL; the
|
|
@@ -14,6 +14,7 @@ import {
|
|
|
14
14
|
listRunAttempts,
|
|
15
15
|
listWorkflowRunJobSummaries,
|
|
16
16
|
listWorkflowRunsByProject,
|
|
17
|
+
recordJobExecutionStartedAt,
|
|
17
18
|
updateJobExecutionStatus,
|
|
18
19
|
updateJobStatus,
|
|
19
20
|
updateWorkflowRunStatus,
|
|
@@ -273,6 +274,32 @@ describe('workflow run queries', () => {
|
|
|
273
274
|
]);
|
|
274
275
|
|
|
275
276
|
expect(summary.get(run.id)?.statusCounts).toEqual([{status: 'pending', count: 2}]);
|
|
277
|
+
expect(summary.get(run.id)?.hasStartedJobExecution).toBe(false);
|
|
278
|
+
});
|
|
279
|
+
|
|
280
|
+
test('reports when any current-attempt job execution has started', async () => {
|
|
281
|
+
const run = await createWorkflowRun({
|
|
282
|
+
workspaceId,
|
|
283
|
+
projectId,
|
|
284
|
+
definitionId,
|
|
285
|
+
model: buildModel({jobs: {build: {steps: [{run: 'echo build'}]}}}),
|
|
286
|
+
triggerPayload: manualTrigger(),
|
|
287
|
+
});
|
|
288
|
+
const [job] = await getJobsByWorkflowRunId(run.id);
|
|
289
|
+
if (!job) throw new Error('Expected workflow job');
|
|
290
|
+
const execution = await getFirstJobExecutionByJobId(job.id);
|
|
291
|
+
if (!execution) throw new Error('Expected workflow job execution');
|
|
292
|
+
|
|
293
|
+
await recordJobExecutionStartedAt({
|
|
294
|
+
jobExecutionId: execution.id,
|
|
295
|
+
startedAt: new Date('2026-05-07T01:00:05.000Z'),
|
|
296
|
+
});
|
|
297
|
+
|
|
298
|
+
const summary = await listWorkflowRunJobSummaries([
|
|
299
|
+
{id: run.id, currentAttempt: run.currentAttempt},
|
|
300
|
+
]);
|
|
301
|
+
|
|
302
|
+
expect(summary.get(run.id)?.hasStartedJobExecution).toBe(true);
|
|
276
303
|
});
|
|
277
304
|
|
|
278
305
|
test('returns execution evidence and counts its display status', async () => {
|
|
@@ -213,7 +213,8 @@ export interface WorkflowRunJobSummaryTarget {
|
|
|
213
213
|
}
|
|
214
214
|
|
|
215
215
|
/**
|
|
216
|
-
* A page row's jobs: a bounded slice to draw,
|
|
216
|
+
* A page row's jobs: a bounded slice to draw, totals describing all of them, and whether any
|
|
217
|
+
* execution has reached a runner.
|
|
217
218
|
*
|
|
218
219
|
* The preview is what a row can show; `statusCounts` is what it can say. Keeping the counts
|
|
219
220
|
* server-side is what lets a row report a failure that sits past the preview. Counts use the
|
|
@@ -225,6 +226,7 @@ export interface WorkflowRunJobsSummary {
|
|
|
225
226
|
preview: WorkflowRunJobSummary[];
|
|
226
227
|
statusCounts: WorkflowRunJobStatusCount[];
|
|
227
228
|
rawStatusCounts: WorkflowRunJobRawStatusCount[];
|
|
229
|
+
hasStartedJobExecution: boolean;
|
|
228
230
|
}
|
|
229
231
|
|
|
230
232
|
export interface WorkflowRunJobStatusCount {
|
|
@@ -253,8 +255,8 @@ export interface WorkflowRunJobRawStatusCount {
|
|
|
253
255
|
* attempt 1's run metadata with attempt 2's jobs, and the row would report a status its strip
|
|
254
256
|
* contradicts.
|
|
255
257
|
*
|
|
256
|
-
* The
|
|
257
|
-
* instant, and the strip combines them into a single glyph row, so under the default
|
|
258
|
+
* The reads share one repeatable-read snapshot. They describe the same jobs and executions at
|
|
259
|
+
* the same instant, and the strip combines them into a single glyph row, so under the default
|
|
258
260
|
* read-committed isolation a job settling between the statements would draw a pending glyph
|
|
259
261
|
* beside a summary counting it as failed. Sequential inside one transaction is the cost of
|
|
260
262
|
* that; at a four-second poll the extra round trip does not register.
|
|
@@ -321,6 +323,14 @@ export async function listWorkflowRunJobSummaries(
|
|
|
321
323
|
.leftJoin(selectedExecution, eq(selectedExecution.jobId, jobs.id))
|
|
322
324
|
.where(attemptFilter)
|
|
323
325
|
.as('ranked');
|
|
326
|
+
// Correlated per job rather than a grouped scan of `job_executions`: an aggregate over the
|
|
327
|
+
// whole table cannot have the page's filter pushed into it, so it would read every execution
|
|
328
|
+
// ever recorded on each poll and grow with history rather than with the page.
|
|
329
|
+
const hasStartedJobExecution = sql<boolean>`bool_or(exists (
|
|
330
|
+
select 1
|
|
331
|
+
from ${jobExecutions}
|
|
332
|
+
where ${jobExecutions.jobId} = ${jobs.id} and ${jobExecutions.startedAt} is not null
|
|
333
|
+
))`;
|
|
324
334
|
|
|
325
335
|
const executionDisplayStatus = sql<JobExecutionStatus | null>`
|
|
326
336
|
${selectedExecution.executionStatus}
|
|
@@ -356,6 +366,7 @@ export async function listWorkflowRunJobSummaries(
|
|
|
356
366
|
rawStatus: jobs.status,
|
|
357
367
|
status: displayStatus,
|
|
358
368
|
count: count(),
|
|
369
|
+
hasStartedJobExecution,
|
|
359
370
|
})
|
|
360
371
|
.from(jobs)
|
|
361
372
|
.innerJoin(workflowRunAttempts, eq(jobs.workflowRunAttemptId, workflowRunAttempts.id))
|
|
@@ -370,10 +381,18 @@ export async function listWorkflowRunJobSummaries(
|
|
|
370
381
|
for (const {workflowRunId, ...summary} of previewRows) {
|
|
371
382
|
summaryFor(summaries, workflowRunId).preview.push(summary);
|
|
372
383
|
}
|
|
373
|
-
for (const {
|
|
384
|
+
for (const {
|
|
385
|
+
workflowRunId,
|
|
386
|
+
rawStatus,
|
|
387
|
+
status,
|
|
388
|
+
count: statusCount,
|
|
389
|
+
hasStartedJobExecution,
|
|
390
|
+
} of countRows) {
|
|
374
391
|
const summary = summaryFor(summaries, workflowRunId);
|
|
375
392
|
appendStatusCount(summary.rawStatusCounts, rawStatus, statusCount);
|
|
376
393
|
appendStatusCount(summary.statusCounts, status, statusCount);
|
|
394
|
+
// One row per (run, verdict, display status), so the run's answer is the OR of its groups.
|
|
395
|
+
summary.hasStartedJobExecution ||= hasStartedJobExecution;
|
|
377
396
|
}
|
|
378
397
|
|
|
379
398
|
return summaries;
|
|
@@ -385,7 +404,12 @@ function summaryFor(
|
|
|
385
404
|
): WorkflowRunJobsSummary {
|
|
386
405
|
const existing = summaries.get(workflowRunId);
|
|
387
406
|
if (existing) return existing;
|
|
388
|
-
const created: WorkflowRunJobsSummary = {
|
|
407
|
+
const created: WorkflowRunJobsSummary = {
|
|
408
|
+
preview: [],
|
|
409
|
+
statusCounts: [],
|
|
410
|
+
rawStatusCounts: [],
|
|
411
|
+
hasStartedJobExecution: false,
|
|
412
|
+
};
|
|
389
413
|
summaries.set(workflowRunId, created);
|
|
390
414
|
return created;
|
|
391
415
|
}
|
|
@@ -606,6 +630,8 @@ function hydrateWorkflowRunDetail(
|
|
|
606
630
|
runAttempt: toWorkflowRunAttempt(attempt),
|
|
607
631
|
latestAttempt,
|
|
608
632
|
jobs: [],
|
|
633
|
+
// Read off the same rows the executions come from, so the flag cannot contradict them.
|
|
634
|
+
hasStartedJobExecution: rows.some((row) => row.jobExecution?.startedAt != null),
|
|
609
635
|
};
|
|
610
636
|
const jobById = new Map<string, WorkflowJobDetail>();
|
|
611
637
|
const jobExecutionById = new Map<string, JobExecutionDetail>();
|
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
} from '../schema/job-executions.js';
|
|
11
11
|
import {type JobCreateDb, type JobDb, jobs} from '../schema/jobs.js';
|
|
12
12
|
import {type StepCreateDb, steps} from '../schema/steps.js';
|
|
13
|
+
import {writeJobExecutionTerminatedOutbox} from './outbox.js';
|
|
13
14
|
|
|
14
15
|
type MaterializedRunGraphJobExecution = Omit<JobExecutionCreateDb, 'finishedAt' | 'jobId'> & {
|
|
15
16
|
readonly finishedAt?: JobExecutionCreateDb['finishedAt'] | SQL | undefined;
|
|
@@ -59,6 +60,22 @@ export async function persistMaterializedRunGraph(
|
|
|
59
60
|
? []
|
|
60
61
|
: await tx.insert(jobExecutions).values(jobExecutionValues).returning();
|
|
61
62
|
|
|
63
|
+
for (const jobExecution of jobExecutionRows) {
|
|
64
|
+
if (
|
|
65
|
+
jobExecution.status === 'succeeded' ||
|
|
66
|
+
jobExecution.status === 'failed' ||
|
|
67
|
+
jobExecution.status === 'cancelled'
|
|
68
|
+
) {
|
|
69
|
+
await writeJobExecutionTerminatedOutbox(tx, {
|
|
70
|
+
jobId: jobExecution.jobId,
|
|
71
|
+
jobExecutionId: jobExecution.id,
|
|
72
|
+
status: jobExecution.status,
|
|
73
|
+
statusReason: jobExecution.statusReason,
|
|
74
|
+
statusReasonMessage: jobExecution.statusReasonMessage,
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
62
79
|
const jobById = new Map(jobRows.map((job) => [job.id, job]));
|
|
63
80
|
const materializedJobByJobId = new Map(
|
|
64
81
|
jobRows.flatMap((jobRow, index) => {
|