@shipfox/api-runners 12.7.0 → 13.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +4 -4
- package/CHANGELOG.md +12 -0
- package/dist/db/index.d.ts +1 -1
- package/dist/db/index.d.ts.map +1 -1
- package/dist/db/index.js +1 -1
- package/dist/db/index.js.map +1 -1
- package/dist/db/job-executions.d.ts +8 -24
- package/dist/db/job-executions.d.ts.map +1 -1
- package/dist/db/job-executions.js +62 -81
- package/dist/db/job-executions.js.map +1 -1
- package/dist/db/reservations.d.ts.map +1 -1
- package/dist/db/reservations.js +6 -4
- package/dist/db/reservations.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -4
- package/dist/index.js.map +1 -1
- package/dist/presentation/index.d.ts +2 -1
- package/dist/presentation/index.d.ts.map +1 -1
- package/dist/presentation/index.js +2 -1
- package/dist/presentation/index.js.map +1 -1
- package/dist/presentation/inter-module.d.ts +0 -1
- package/dist/presentation/inter-module.d.ts.map +1 -1
- package/dist/presentation/inter-module.js +2 -25
- package/dist/presentation/inter-module.js.map +1 -1
- package/dist/presentation/subscribers/on-workflows-job-execution-queued.d.ts +3 -0
- package/dist/presentation/subscribers/on-workflows-job-execution-queued.d.ts.map +1 -0
- package/dist/presentation/subscribers/on-workflows-job-execution-queued.js +21 -0
- package/dist/presentation/subscribers/on-workflows-job-execution-queued.js.map +1 -0
- package/dist/presentation/subscribers/on-workflows-job-execution-terminated.d.ts +3 -0
- package/dist/presentation/subscribers/on-workflows-job-execution-terminated.d.ts.map +1 -0
- package/dist/presentation/subscribers/{on-workflows-job-execution-timed-out.js → on-workflows-job-execution-terminated.js} +5 -4
- package/dist/presentation/subscribers/on-workflows-job-execution-terminated.js.map +1 -0
- package/dist/tsconfig.test.tsbuildinfo +1 -1
- package/package.json +3 -3
- package/src/db/index.ts +0 -2
- package/src/db/job-executions.test.ts +105 -335
- package/src/db/job-executions.ts +70 -91
- package/src/db/reservations.ts +6 -3
- package/src/db/runner-instances.test.ts +100 -0
- package/src/index.test.ts +1 -0
- package/src/index.ts +8 -7
- package/src/presentation/index.ts +2 -1
- package/src/presentation/inter-module.ts +2 -40
- package/src/presentation/subscribers/on-workflows-job-execution-queued.test.ts +57 -0
- package/src/presentation/subscribers/on-workflows-job-execution-queued.ts +26 -0
- package/src/presentation/subscribers/on-workflows-job-execution-terminated.test.ts +142 -0
- package/src/presentation/subscribers/{on-workflows-job-execution-timed-out.ts → on-workflows-job-execution-terminated.ts} +5 -4
- package/test/factories/pending-job.ts +3 -0
- package/tsconfig.build.tsbuildinfo +1 -1
- package/dist/presentation/subscribers/on-workflows-job-execution-timed-out.d.ts +0 -3
- package/dist/presentation/subscribers/on-workflows-job-execution-timed-out.d.ts.map +0 -1
- package/dist/presentation/subscribers/on-workflows-job-execution-timed-out.js.map +0 -1
- package/src/presentation/inter-module.test.ts +0 -15
- package/src/presentation/subscribers/on-workflows-job-execution-timed-out.test.ts +0 -163
package/src/db/job-executions.ts
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
RUNNER_JOB_CLAIMED,
|
|
3
3
|
RUNNER_JOB_LEASE_EXPIRED,
|
|
4
|
-
RUNNER_JOB_QUEUED,
|
|
5
4
|
type RunnersEventMap,
|
|
6
5
|
type RunnerToolCapabilitiesDto,
|
|
7
6
|
} from '@shipfox/api-runners-dto';
|
|
@@ -54,6 +53,39 @@ async function lockJobExecution(tx: Tx, jobExecutionId: string): Promise<void> {
|
|
|
54
53
|
);
|
|
55
54
|
}
|
|
56
55
|
|
|
56
|
+
async function releaseReservationsForTerminalRunningRows(
|
|
57
|
+
tx: Tx,
|
|
58
|
+
rows: ReadonlyArray<{
|
|
59
|
+
provisionerId: string | null;
|
|
60
|
+
providerRunnerId: string | null;
|
|
61
|
+
}>,
|
|
62
|
+
): Promise<void> {
|
|
63
|
+
// Converge after the lease fact changes. The reservation helper locks each provider runner
|
|
64
|
+
// and releases only when the runner is terminal and no uncancelled lease remains.
|
|
65
|
+
const providerRunnerIdsByProvisionerId = new Map<string, Set<string>>();
|
|
66
|
+
for (const row of rows) {
|
|
67
|
+
if (row.provisionerId === null || row.providerRunnerId === null) continue;
|
|
68
|
+
const providerRunnerIds =
|
|
69
|
+
providerRunnerIdsByProvisionerId.get(row.provisionerId) ?? new Set<string>();
|
|
70
|
+
providerRunnerIds.add(row.providerRunnerId);
|
|
71
|
+
providerRunnerIdsByProvisionerId.set(row.provisionerId, providerRunnerIds);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
for (const provisionerId of [...providerRunnerIdsByProvisionerId.keys()].sort()) {
|
|
75
|
+
const providerRunnerIds = providerRunnerIdsByProvisionerId.get(provisionerId);
|
|
76
|
+
if (!providerRunnerIds) continue;
|
|
77
|
+
await releaseTerminalRunnerInstanceReservationsByIds(tx, {
|
|
78
|
+
workspaceId: null,
|
|
79
|
+
provisionerId,
|
|
80
|
+
providerRunnerIds: [...providerRunnerIds].sort(),
|
|
81
|
+
requireUnlinkedSession: false,
|
|
82
|
+
// Lock and re-check the runner row locally so lease finalization remains retryable
|
|
83
|
+
// without a workflow/runner scope lock.
|
|
84
|
+
requireTerminalState: false,
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
57
89
|
export interface EnqueueJobExecutionParams {
|
|
58
90
|
workspaceId: string;
|
|
59
91
|
workflowRunId: string;
|
|
@@ -62,6 +94,7 @@ export interface EnqueueJobExecutionParams {
|
|
|
62
94
|
jobExecutionId: string;
|
|
63
95
|
projectId: string;
|
|
64
96
|
requiredLabels: string[];
|
|
97
|
+
queuedAt: Date;
|
|
65
98
|
}
|
|
66
99
|
|
|
67
100
|
export async function getWorkspaceJobCounts(params: {
|
|
@@ -92,11 +125,9 @@ export async function getWorkspaceJobCounts(params: {
|
|
|
92
125
|
}));
|
|
93
126
|
}
|
|
94
127
|
|
|
95
|
-
//
|
|
96
|
-
//
|
|
97
|
-
//
|
|
98
|
-
// The per-execution advisory lock serializes retries with lease expiry/reconciliation, and the
|
|
99
|
-
// durable lease-expired event prevents a retry from re-queueing an execution already reaped.
|
|
128
|
+
// The workflows outbox delivers queue facts at least once. The per-execution advisory lock and
|
|
129
|
+
// pending/running checks make replay a no-op across the claim transition. A durable lease-expired
|
|
130
|
+
// event also prevents a delayed replay from resurrecting an execution already reaped locally.
|
|
100
131
|
export async function enqueueJobExecution(params: EnqueueJobExecutionParams): Promise<void> {
|
|
101
132
|
const requiredLabels = [...canonicalizeLabels(params.requiredLabels)];
|
|
102
133
|
if (requiredLabels.length === 0) throw new EmptyRequiredLabelsError();
|
|
@@ -104,6 +135,13 @@ export async function enqueueJobExecution(params: EnqueueJobExecutionParams): Pr
|
|
|
104
135
|
const enqueued = await db().transaction(async (tx) => {
|
|
105
136
|
await lockJobExecution(tx, params.jobExecutionId);
|
|
106
137
|
|
|
138
|
+
const [running] = await tx
|
|
139
|
+
.select({jobExecutionId: runningJobExecutions.jobExecutionId})
|
|
140
|
+
.from(runningJobExecutions)
|
|
141
|
+
.where(eq(runningJobExecutions.jobExecutionId, params.jobExecutionId))
|
|
142
|
+
.limit(1);
|
|
143
|
+
if (running) return false;
|
|
144
|
+
|
|
107
145
|
const [leaseExpired] = await tx
|
|
108
146
|
.select({id: runnersOutbox.id})
|
|
109
147
|
.from(runnersOutbox)
|
|
@@ -126,26 +164,12 @@ export async function enqueueJobExecution(params: EnqueueJobExecutionParams): Pr
|
|
|
126
164
|
jobExecutionId: params.jobExecutionId,
|
|
127
165
|
projectId: params.projectId,
|
|
128
166
|
requiredLabels,
|
|
167
|
+
createdAt: params.queuedAt,
|
|
129
168
|
})
|
|
130
169
|
.onConflictDoNothing({target: pendingJobExecutions.jobExecutionId})
|
|
131
170
|
.returning({createdAt: pendingJobExecutions.createdAt});
|
|
132
171
|
|
|
133
|
-
|
|
134
|
-
// emitted the queued event (durably, in the outbox), so re-emitting would
|
|
135
|
-
// only add a redundant row the subscriber coalesces away. Skip it.
|
|
136
|
-
if (!inserted) return false;
|
|
137
|
-
|
|
138
|
-
await writeOutboxEvent<RunnersEventMap>(tx, runnersOutbox, {
|
|
139
|
-
type: RUNNER_JOB_QUEUED,
|
|
140
|
-
payload: {
|
|
141
|
-
workflowRunId: params.workflowRunId,
|
|
142
|
-
workflowRunAttemptId: params.workflowRunAttemptId,
|
|
143
|
-
jobId: params.jobId,
|
|
144
|
-
jobExecutionId: params.jobExecutionId,
|
|
145
|
-
queuedAt: inserted.createdAt.toISOString(),
|
|
146
|
-
},
|
|
147
|
-
});
|
|
148
|
-
return true;
|
|
172
|
+
return inserted !== undefined;
|
|
149
173
|
});
|
|
150
174
|
|
|
151
175
|
if (enqueued) jobExecutionEnqueuedCount.add(1);
|
|
@@ -403,64 +427,18 @@ async function touchRunnerSessionLiveness(params: {
|
|
|
403
427
|
);
|
|
404
428
|
}
|
|
405
429
|
|
|
406
|
-
/**
|
|
407
|
-
* Releases a job execution's lease when the orchestration workflow finalizes it: deletes the
|
|
408
|
-
* running-job-execution row AND any lingering pending row for the same execution, in one tx.
|
|
409
|
-
* When the deleted lease was the last one for a terminal provider runner, it also releases the
|
|
410
|
-
* runner's reservation in the same runner-owned transaction. Terminal reports and workflow lease
|
|
411
|
-
* finalization remain independently retryable; they do not share a cross-module scope lock.
|
|
412
|
-
* Idempotent (0-row no-op), no token scope (the workflow is authoritative), and
|
|
413
|
-
* emits no event: the workflow already owns the outcome. Sweeping the pending row
|
|
414
|
-
* too closes the at-least-once window where an enqueue retry left an orphan that a
|
|
415
|
-
* later claim would otherwise pick up for an already-finished job execution.
|
|
416
|
-
*/
|
|
417
|
-
export async function releaseJobExecution(params: {jobExecutionId: string}): Promise<void> {
|
|
418
|
-
await db().transaction(async (tx) => {
|
|
419
|
-
await lockJobExecution(tx, params.jobExecutionId);
|
|
420
|
-
|
|
421
|
-
// Delete pending before running to match `claimPendingJobExecution`'s lock-acquisition
|
|
422
|
-
// order (it locks the pending row first, then the running row). A concurrent
|
|
423
|
-
// claim picking up an orphan pending row for this same job execution would otherwise
|
|
424
|
-
// deadlock against the reverse order here.
|
|
425
|
-
await tx
|
|
426
|
-
.delete(pendingJobExecutions)
|
|
427
|
-
.where(eq(pendingJobExecutions.jobExecutionId, params.jobExecutionId));
|
|
428
|
-
const deletedRunningRows = await tx
|
|
429
|
-
.delete(runningJobExecutions)
|
|
430
|
-
.where(eq(runningJobExecutions.jobExecutionId, params.jobExecutionId))
|
|
431
|
-
.returning({
|
|
432
|
-
workspaceId: runningJobExecutions.workspaceId,
|
|
433
|
-
provisionerId: runningJobExecutions.provisionerId,
|
|
434
|
-
providerRunnerId: runningJobExecutions.providerRunnerId,
|
|
435
|
-
});
|
|
436
|
-
|
|
437
|
-
const deletedRunningRow = deletedRunningRows[0];
|
|
438
|
-
if (deletedRunningRow?.provisionerId && deletedRunningRow.providerRunnerId) {
|
|
439
|
-
await releaseTerminalRunnerInstanceReservationsByIds(tx, {
|
|
440
|
-
workspaceId: null,
|
|
441
|
-
provisionerId: deletedRunningRow.provisionerId,
|
|
442
|
-
providerRunnerIds: [deletedRunningRow.providerRunnerId],
|
|
443
|
-
requireUnlinkedSession: false,
|
|
444
|
-
// Lock and re-check the runner row locally so terminal reporting remains retryable
|
|
445
|
-
// without a workflow/runner scope lock.
|
|
446
|
-
requireTerminalState: false,
|
|
447
|
-
});
|
|
448
|
-
}
|
|
449
|
-
});
|
|
450
|
-
}
|
|
451
|
-
|
|
452
430
|
/**
|
|
453
431
|
* Reaps stale leases (bounded by `limit`), emitting one
|
|
454
432
|
* `runners.job.lease_expired` event per reaped job execution.
|
|
455
433
|
*
|
|
456
434
|
* The cutoff is re-checked in the DELETE, not just the locking subquery, so a
|
|
457
435
|
* heartbeat landing mid-call spares the live row. Each reaped execution also sweeps its
|
|
458
|
-
* pending row
|
|
459
|
-
* that a later claim re-runs as an already-finished job execution.
|
|
436
|
+
* pending row so an at-least-once queue replay cannot leave an orphan.
|
|
460
437
|
*
|
|
461
|
-
* Locks pending-then-running to match `claimPendingJobExecution
|
|
462
|
-
* `reconcileTerminalJobExecution`. The stale candidate scan intentionally does not lock
|
|
463
|
-
* rows first; the running-row DELETE re-checks the stale predicate after the
|
|
438
|
+
* Locks pending-then-running to match `claimPendingJobExecution` and
|
|
439
|
+
* `reconcileTerminalJobExecution`. The stale candidate scan intentionally does not lock
|
|
440
|
+
* running rows first; the running-row DELETE re-checks the stale predicate after the
|
|
441
|
+
* pending-row sweep.
|
|
464
442
|
*/
|
|
465
443
|
export async function expireStuckJobExecutions(params: {
|
|
466
444
|
thresholdSeconds: number;
|
|
@@ -537,10 +515,14 @@ export async function expireStuckJobExecutions(params: {
|
|
|
537
515
|
workflowRunAttemptId: runningJobExecutions.workflowRunAttemptId,
|
|
538
516
|
jobId: runningJobExecutions.jobId,
|
|
539
517
|
jobExecutionId: runningJobExecutions.jobExecutionId,
|
|
518
|
+
provisionerId: runningJobExecutions.provisionerId,
|
|
519
|
+
providerRunnerId: runningJobExecutions.providerRunnerId,
|
|
540
520
|
});
|
|
541
521
|
|
|
542
522
|
if (deleted.length === 0) return [];
|
|
543
523
|
|
|
524
|
+
await releaseReservationsForTerminalRunningRows(tx, deleted);
|
|
525
|
+
|
|
544
526
|
await writeOutboxEvents<RunnersEventMap>(
|
|
545
527
|
tx,
|
|
546
528
|
runnersOutbox,
|
|
@@ -555,7 +537,12 @@ export async function expireStuckJobExecutions(params: {
|
|
|
555
537
|
})),
|
|
556
538
|
);
|
|
557
539
|
|
|
558
|
-
return deleted
|
|
540
|
+
return deleted.map(({workflowRunId, workflowRunAttemptId, jobId, jobExecutionId}) => ({
|
|
541
|
+
workflowRunId,
|
|
542
|
+
workflowRunAttemptId,
|
|
543
|
+
jobId,
|
|
544
|
+
jobExecutionId,
|
|
545
|
+
}));
|
|
559
546
|
});
|
|
560
547
|
|
|
561
548
|
if (reaped.length > 0) jobExecutionLeaseExpiredCount.add(reaped.length);
|
|
@@ -803,8 +790,8 @@ export async function recordHeartbeat(params: {
|
|
|
803
790
|
|
|
804
791
|
/**
|
|
805
792
|
* Reconciles a terminal job execution with runner state in one transaction: removes its pending
|
|
806
|
-
* queue row
|
|
807
|
-
* idempotent and preserves the first cancellation-request timestamp.
|
|
793
|
+
* queue row, requests cancellation on its running lease, and converges any linked reservation.
|
|
794
|
+
* The operation is idempotent and preserves the first cancellation-request timestamp.
|
|
808
795
|
*/
|
|
809
796
|
export async function reconcileTerminalJobExecution(params: {
|
|
810
797
|
jobExecutionId: string;
|
|
@@ -818,25 +805,17 @@ export async function reconcileTerminalJobExecution(params: {
|
|
|
818
805
|
await tx
|
|
819
806
|
.delete(pendingJobExecutions)
|
|
820
807
|
.where(eq(pendingJobExecutions.jobExecutionId, params.jobExecutionId));
|
|
821
|
-
await tx
|
|
808
|
+
const cancelledRunningRows = await tx
|
|
822
809
|
.update(runningJobExecutions)
|
|
823
810
|
.set({
|
|
824
811
|
cancellationRequestedAt: sql`COALESCE(${runningJobExecutions.cancellationRequestedAt}, now())`,
|
|
825
812
|
})
|
|
826
|
-
.where(eq(runningJobExecutions.jobExecutionId, params.jobExecutionId))
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
if (params.jobIds.length === 0) return;
|
|
813
|
+
.where(eq(runningJobExecutions.jobExecutionId, params.jobExecutionId))
|
|
814
|
+
.returning({
|
|
815
|
+
provisionerId: runningJobExecutions.provisionerId,
|
|
816
|
+
providerRunnerId: runningJobExecutions.providerRunnerId,
|
|
817
|
+
});
|
|
832
818
|
|
|
833
|
-
|
|
834
|
-
await tx.delete(pendingJobExecutions).where(inArray(pendingJobExecutions.jobId, params.jobIds));
|
|
835
|
-
await tx
|
|
836
|
-
.update(runningJobExecutions)
|
|
837
|
-
.set({
|
|
838
|
-
cancellationRequestedAt: sql`COALESCE(${runningJobExecutions.cancellationRequestedAt}, now())`,
|
|
839
|
-
})
|
|
840
|
-
.where(inArray(runningJobExecutions.jobId, params.jobIds));
|
|
819
|
+
await releaseReservationsForTerminalRunningRows(tx, cancelledRunningRows);
|
|
841
820
|
});
|
|
842
821
|
}
|
package/src/db/reservations.ts
CHANGED
|
@@ -939,7 +939,9 @@ export async function releaseTerminalRunnerInstanceReservationsByIds(
|
|
|
939
939
|
params.workspaceId === null
|
|
940
940
|
? sql``
|
|
941
941
|
: sql`and ${eq(reservations.workspaceId, params.workspaceId)}`;
|
|
942
|
-
|
|
942
|
+
// Cancellation keeps the lease row so the provisioner can observe its terminate intent.
|
|
943
|
+
// Once the runner is terminal, only an uncancelled lease should block reservation release.
|
|
944
|
+
const noUncancelledRunningJobPredicate = notExists(
|
|
943
945
|
tx
|
|
944
946
|
.select({id: runningJobExecutions.id})
|
|
945
947
|
.from(runningJobExecutions)
|
|
@@ -947,6 +949,7 @@ export async function releaseTerminalRunnerInstanceReservationsByIds(
|
|
|
947
949
|
and(
|
|
948
950
|
eq(runningJobExecutions.provisionerId, params.provisionerId),
|
|
949
951
|
eq(runningJobExecutions.providerRunnerId, providerRunners.providerRunnerId),
|
|
952
|
+
isNull(runningJobExecutions.cancellationRequestedAt),
|
|
950
953
|
params.workspaceId === null
|
|
951
954
|
? undefined
|
|
952
955
|
: eq(runningJobExecutions.workspaceId, params.workspaceId),
|
|
@@ -1103,7 +1106,7 @@ export async function releaseTerminalRunnerInstanceReservationsByIds(
|
|
|
1103
1106
|
params.requireUnlinkedSession === false
|
|
1104
1107
|
? undefined
|
|
1105
1108
|
: isNull(providerRunners.runnerSessionId),
|
|
1106
|
-
|
|
1109
|
+
noUncancelledRunningJobPredicate,
|
|
1107
1110
|
reportFreshnessPredicate,
|
|
1108
1111
|
isNull(providerRunners.reservationReleasedAt),
|
|
1109
1112
|
),
|
|
@@ -1137,7 +1140,7 @@ export async function releaseTerminalRunnerInstanceReservationsByIds(
|
|
|
1137
1140
|
params.requireUnlinkedSession === false
|
|
1138
1141
|
? undefined
|
|
1139
1142
|
: isNull(providerRunners.runnerSessionId),
|
|
1140
|
-
|
|
1143
|
+
noUncancelledRunningJobPredicate,
|
|
1141
1144
|
reportFreshnessPredicate,
|
|
1142
1145
|
isNull(providerRunners.reservationReleasedAt),
|
|
1143
1146
|
),
|
|
@@ -1145,6 +1145,106 @@ describe('reportRunnerInstances', () => {
|
|
|
1145
1145
|
expect(reservation?.count).toBe(1);
|
|
1146
1146
|
});
|
|
1147
1147
|
|
|
1148
|
+
it('keeps a reservation when a terminal runner has cancelled and uncancelled jobs', async () => {
|
|
1149
|
+
const reservationId = await createReservation(1);
|
|
1150
|
+
const runnerSession = await runnerSessionFactory.create({workspaceId});
|
|
1151
|
+
await providerRunnerFactory.create({
|
|
1152
|
+
workspaceId,
|
|
1153
|
+
provisionerId,
|
|
1154
|
+
providerRunnerId: 'mixed-terminal-runner',
|
|
1155
|
+
reservationId,
|
|
1156
|
+
runnerSessionId: runnerSession.id,
|
|
1157
|
+
state: 'running',
|
|
1158
|
+
});
|
|
1159
|
+
await insertRunningJobRow({
|
|
1160
|
+
workspaceId,
|
|
1161
|
+
provisionerId,
|
|
1162
|
+
providerRunnerId: 'mixed-terminal-runner',
|
|
1163
|
+
startedAt: new Date('2025-01-01T00:00:00.000Z'),
|
|
1164
|
+
cancellationRequestedAt: new Date('2025-01-01T00:01:00.000Z'),
|
|
1165
|
+
});
|
|
1166
|
+
await insertRunningJobRow({
|
|
1167
|
+
workspaceId,
|
|
1168
|
+
provisionerId,
|
|
1169
|
+
providerRunnerId: 'mixed-terminal-runner',
|
|
1170
|
+
startedAt: new Date('2025-01-01T00:02:00.000Z'),
|
|
1171
|
+
});
|
|
1172
|
+
|
|
1173
|
+
const result = await reportRunnerInstances({
|
|
1174
|
+
scope: 'workspace',
|
|
1175
|
+
workspaceId,
|
|
1176
|
+
provisionerId,
|
|
1177
|
+
events: [
|
|
1178
|
+
event({
|
|
1179
|
+
providerRunnerId: 'mixed-terminal-runner',
|
|
1180
|
+
reservationId,
|
|
1181
|
+
state: 'terminated',
|
|
1182
|
+
runnerSessionId: runnerSession.id,
|
|
1183
|
+
}),
|
|
1184
|
+
],
|
|
1185
|
+
});
|
|
1186
|
+
|
|
1187
|
+
const [providerRunner] = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
1188
|
+
const [reservation] = await reservationRowsFor({workspaceId, provisionerId});
|
|
1189
|
+
expect(result).toEqual({accepted: 1, reservationsReleased: 0, terminateIntentsHonored: []});
|
|
1190
|
+
expect(providerRunner).toMatchObject({state: 'terminated', reservationReleasedAt: null});
|
|
1191
|
+
expect(reservation?.count).toBe(1);
|
|
1192
|
+
});
|
|
1193
|
+
|
|
1194
|
+
it('releases a reservation when a terminal runner only has a cancelled job', async () => {
|
|
1195
|
+
const reservationId = await createReservation(1);
|
|
1196
|
+
const runnerSession = await runnerSessionFactory.create({workspaceId});
|
|
1197
|
+
await providerRunnerFactory.create({
|
|
1198
|
+
workspaceId,
|
|
1199
|
+
provisionerId,
|
|
1200
|
+
providerRunnerId: 'cancelled-terminal-runner',
|
|
1201
|
+
reservationId,
|
|
1202
|
+
runnerSessionId: runnerSession.id,
|
|
1203
|
+
state: 'running',
|
|
1204
|
+
});
|
|
1205
|
+
await insertRunningJobRow({
|
|
1206
|
+
workspaceId,
|
|
1207
|
+
provisionerId,
|
|
1208
|
+
providerRunnerId: 'cancelled-terminal-runner',
|
|
1209
|
+
cancellationRequestedAt: new Date('2025-01-01T00:01:00.000Z'),
|
|
1210
|
+
});
|
|
1211
|
+
|
|
1212
|
+
const result = await reportRunnerInstances({
|
|
1213
|
+
scope: 'workspace',
|
|
1214
|
+
workspaceId,
|
|
1215
|
+
provisionerId,
|
|
1216
|
+
events: [
|
|
1217
|
+
event({
|
|
1218
|
+
providerRunnerId: 'cancelled-terminal-runner',
|
|
1219
|
+
reservationId,
|
|
1220
|
+
state: 'terminated',
|
|
1221
|
+
runnerSessionId: runnerSession.id,
|
|
1222
|
+
}),
|
|
1223
|
+
],
|
|
1224
|
+
});
|
|
1225
|
+
|
|
1226
|
+
const [providerRunner] = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
1227
|
+
const reservationRows = await reservationRowsFor({workspaceId, provisionerId});
|
|
1228
|
+
const runningJobRows = await db()
|
|
1229
|
+
.select()
|
|
1230
|
+
.from(runningJobExecutions)
|
|
1231
|
+
.where(eq(runningJobExecutions.providerRunnerId, 'cancelled-terminal-runner'));
|
|
1232
|
+
expect(result).toEqual({
|
|
1233
|
+
accepted: 1,
|
|
1234
|
+
reservationsReleased: 1,
|
|
1235
|
+
terminateIntentsHonored: [
|
|
1236
|
+
{providerRunnerId: 'cancelled-terminal-runner', reason: 'job-cancelled'},
|
|
1237
|
+
],
|
|
1238
|
+
});
|
|
1239
|
+
expect(providerRunner).toMatchObject({
|
|
1240
|
+
state: 'terminated',
|
|
1241
|
+
reservationReleasedAt: expect.any(Date),
|
|
1242
|
+
});
|
|
1243
|
+
expect(reservationRows).toHaveLength(0);
|
|
1244
|
+
expect(runningJobRows).toHaveLength(1);
|
|
1245
|
+
expect(runningJobRows[0]?.cancellationRequestedAt).toBeInstanceOf(Date);
|
|
1246
|
+
});
|
|
1247
|
+
|
|
1148
1248
|
it('uses the consumed ephemeral token session before releasing a terminal report', async () => {
|
|
1149
1249
|
const reservationId = await createReservation(1);
|
|
1150
1250
|
const token = await ephemeralRegistrationTokenFactory.create({
|
package/src/index.test.ts
CHANGED
|
@@ -10,6 +10,7 @@ describe('createRunnersModule', () => {
|
|
|
10
10
|
expect(module.name).toBe('runners');
|
|
11
11
|
expect(module.auth).toHaveLength(3);
|
|
12
12
|
expect(module.routes).toHaveLength(13);
|
|
13
|
+
expect(module.subscribers).toHaveLength(2);
|
|
13
14
|
expect(module.workers).toHaveLength(1);
|
|
14
15
|
});
|
|
15
16
|
|
package/src/index.ts
CHANGED
|
@@ -4,7 +4,8 @@ import type {AuthInterModuleClient} from '@shipfox/api-auth-dto/inter-module';
|
|
|
4
4
|
import {administrationActionEventSchemas} from '@shipfox/api-common-dto';
|
|
5
5
|
import {runnersEventSchemas} from '@shipfox/api-runners-dto';
|
|
6
6
|
import {
|
|
7
|
-
|
|
7
|
+
WORKFLOWS_JOB_EXECUTION_QUEUED,
|
|
8
|
+
WORKFLOWS_JOB_EXECUTION_TERMINATED,
|
|
8
9
|
type WorkflowsEventMapDto,
|
|
9
10
|
} from '@shipfox/api-workflows-dto';
|
|
10
11
|
import {type ShipfoxModule, subscriberFactory} from '@shipfox/node-module';
|
|
@@ -16,7 +17,8 @@ import {
|
|
|
16
17
|
createRunnerControlSessionAuthMethod,
|
|
17
18
|
createRunnerRegistrationTokenAuthMethod,
|
|
18
19
|
createRunnerRoutes,
|
|
19
|
-
|
|
20
|
+
onWorkflowsJobExecutionQueued,
|
|
21
|
+
onWorkflowsJobExecutionTerminated,
|
|
20
22
|
} from '#presentation/index.js';
|
|
21
23
|
import {createRunnersInterModulePresentation} from '#presentation/inter-module.js';
|
|
22
24
|
import {createRunnersMaintenanceActivities} from '#temporal/activities/index.js';
|
|
@@ -30,12 +32,8 @@ export {
|
|
|
30
32
|
unadvertisedRunnerTools,
|
|
31
33
|
} from '#core/runner-tool-capabilities.js';
|
|
32
34
|
export {
|
|
33
|
-
cancelRunnerJobs,
|
|
34
|
-
type EnqueueJobExecutionParams,
|
|
35
|
-
enqueueJobExecution,
|
|
36
35
|
getWorkspaceJobCounts,
|
|
37
36
|
isJobLeaseActive,
|
|
38
|
-
releaseJobExecution,
|
|
39
37
|
} from '#db/index.js';
|
|
40
38
|
export type {
|
|
41
39
|
CreateRunnersModuleOptions,
|
|
@@ -64,7 +62,10 @@ export function createRunnersModule({
|
|
|
64
62
|
publishers: [
|
|
65
63
|
{name: 'runners', table: runnersOutbox, db, eventSchemas: runnersPublisherEventSchemas},
|
|
66
64
|
],
|
|
67
|
-
subscribers: [
|
|
65
|
+
subscribers: [
|
|
66
|
+
subscriber(WORKFLOWS_JOB_EXECUTION_QUEUED, onWorkflowsJobExecutionQueued),
|
|
67
|
+
subscriber(WORKFLOWS_JOB_EXECUTION_TERMINATED, onWorkflowsJobExecutionTerminated),
|
|
68
|
+
],
|
|
68
69
|
workers: [
|
|
69
70
|
{
|
|
70
71
|
taskQueue: RUNNERS_MAINTENANCE_TASK_QUEUE,
|
|
@@ -4,4 +4,5 @@ export {
|
|
|
4
4
|
createRunnerRegistrationTokenAuthMethod,
|
|
5
5
|
} from './auth/index.js';
|
|
6
6
|
export {createRunnerRoutes, createRunnerRoutes as routes} from './routes/index.js';
|
|
7
|
-
export {
|
|
7
|
+
export {onWorkflowsJobExecutionQueued} from './subscribers/on-workflows-job-execution-queued.js';
|
|
8
|
+
export {onWorkflowsJobExecutionTerminated} from './subscribers/on-workflows-job-execution-terminated.js';
|
|
@@ -1,39 +1,12 @@
|
|
|
1
1
|
import {runnersInterModuleContract} from '@shipfox/api-runners-dto/inter-module';
|
|
2
|
-
import {
|
|
3
|
-
createInterModuleKnownError,
|
|
4
|
-
defineInterModulePresentation,
|
|
5
|
-
type InterModulePresentation,
|
|
6
|
-
} from '@shipfox/inter-module';
|
|
7
|
-
import {EmptyRequiredLabelsError} from '#core/errors.js';
|
|
2
|
+
import {defineInterModulePresentation, type InterModulePresentation} from '@shipfox/inter-module';
|
|
8
3
|
import {getEffectiveRunnerToolCapabilities} from '#core/runner-tool-capabilities.js';
|
|
9
|
-
import {
|
|
10
|
-
cancelRunnerJobs,
|
|
11
|
-
enqueueJobExecution,
|
|
12
|
-
getWorkspaceJobCounts,
|
|
13
|
-
isJobLeaseActive,
|
|
14
|
-
releaseJobExecution,
|
|
15
|
-
} from '#db/job-executions.js';
|
|
4
|
+
import {getWorkspaceJobCounts, isJobLeaseActive} from '#db/job-executions.js';
|
|
16
5
|
|
|
17
6
|
export function createRunnersInterModulePresentation(): InterModulePresentation<
|
|
18
7
|
typeof runnersInterModuleContract
|
|
19
8
|
> {
|
|
20
9
|
return defineInterModulePresentation(runnersInterModuleContract, {
|
|
21
|
-
enqueueJobExecution: async (input) => {
|
|
22
|
-
try {
|
|
23
|
-
await enqueueJobExecution(input);
|
|
24
|
-
return {};
|
|
25
|
-
} catch (error) {
|
|
26
|
-
throw toEnqueueJobExecutionKnownError(error);
|
|
27
|
-
}
|
|
28
|
-
},
|
|
29
|
-
releaseJobExecution: async (input) => {
|
|
30
|
-
await releaseJobExecution(input);
|
|
31
|
-
return {};
|
|
32
|
-
},
|
|
33
|
-
cancelJobs: async (input) => {
|
|
34
|
-
await cancelRunnerJobs(input);
|
|
35
|
-
return {};
|
|
36
|
-
},
|
|
37
10
|
getLeaseState: async (input) => ({active: await isJobLeaseActive(input)}),
|
|
38
11
|
getEffectiveRunnerToolCapabilities: async (input) => {
|
|
39
12
|
const result = await getEffectiveRunnerToolCapabilities(input);
|
|
@@ -44,14 +17,3 @@ export function createRunnersInterModulePresentation(): InterModulePresentation<
|
|
|
44
17
|
}),
|
|
45
18
|
});
|
|
46
19
|
}
|
|
47
|
-
|
|
48
|
-
export function toEnqueueJobExecutionKnownError(error: unknown): unknown {
|
|
49
|
-
if (error instanceof EmptyRequiredLabelsError) {
|
|
50
|
-
return createInterModuleKnownError(
|
|
51
|
-
runnersInterModuleContract.methods.enqueueJobExecution,
|
|
52
|
-
'empty-required-labels',
|
|
53
|
-
{},
|
|
54
|
-
);
|
|
55
|
-
}
|
|
56
|
-
return error;
|
|
57
|
-
}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import type {WorkflowsJobExecutionQueuedEventDto} from '@shipfox/api-workflows-dto';
|
|
2
|
+
import {eq} from 'drizzle-orm';
|
|
3
|
+
import {db} from '#db/db.js';
|
|
4
|
+
import {claimPendingJobExecution} from '#db/job-executions.js';
|
|
5
|
+
import {pendingJobExecutions} from '#db/schema/pending-job-executions.js';
|
|
6
|
+
import {runningJobExecutions} from '#db/schema/running-job-executions.js';
|
|
7
|
+
import {runnerSessionFactory} from '#test/index.js';
|
|
8
|
+
import {onWorkflowsJobExecutionQueued} from './on-workflows-job-execution-queued.js';
|
|
9
|
+
|
|
10
|
+
describe('onWorkflowsJobExecutionQueued', () => {
|
|
11
|
+
it('queues from the workflow timestamp and is idempotent across claim', async () => {
|
|
12
|
+
const workspaceId = crypto.randomUUID();
|
|
13
|
+
const payload: WorkflowsJobExecutionQueuedEventDto = {
|
|
14
|
+
jobId: crypto.randomUUID(),
|
|
15
|
+
jobExecutionId: crypto.randomUUID(),
|
|
16
|
+
workflowRunId: crypto.randomUUID(),
|
|
17
|
+
workflowRunAttemptId: crypto.randomUUID(),
|
|
18
|
+
workspaceId,
|
|
19
|
+
projectId: crypto.randomUUID(),
|
|
20
|
+
requiredLabels: ['linux'],
|
|
21
|
+
queuedAt: '2026-08-11T08:00:00.000Z',
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
await onWorkflowsJobExecutionQueued(payload);
|
|
25
|
+
await onWorkflowsJobExecutionQueued(payload);
|
|
26
|
+
|
|
27
|
+
const pending = await db()
|
|
28
|
+
.select()
|
|
29
|
+
.from(pendingJobExecutions)
|
|
30
|
+
.where(eq(pendingJobExecutions.jobExecutionId, payload.jobExecutionId));
|
|
31
|
+
expect(pending).toHaveLength(1);
|
|
32
|
+
expect(pending[0]?.createdAt.toISOString()).toBe(payload.queuedAt);
|
|
33
|
+
|
|
34
|
+
const session = await runnerSessionFactory.create({workspaceId});
|
|
35
|
+
await claimPendingJobExecution({
|
|
36
|
+
workspaceId,
|
|
37
|
+
runnerSessionId: session.id,
|
|
38
|
+
sessionLabels: ['linux'],
|
|
39
|
+
maxClaims: null,
|
|
40
|
+
runnerSessionLivenessThrottleSeconds: 10,
|
|
41
|
+
});
|
|
42
|
+
await onWorkflowsJobExecutionQueued(payload);
|
|
43
|
+
|
|
44
|
+
expect(
|
|
45
|
+
await db()
|
|
46
|
+
.select()
|
|
47
|
+
.from(pendingJobExecutions)
|
|
48
|
+
.where(eq(pendingJobExecutions.jobExecutionId, payload.jobExecutionId)),
|
|
49
|
+
).toHaveLength(0);
|
|
50
|
+
expect(
|
|
51
|
+
await db()
|
|
52
|
+
.select()
|
|
53
|
+
.from(runningJobExecutions)
|
|
54
|
+
.where(eq(runningJobExecutions.jobExecutionId, payload.jobExecutionId)),
|
|
55
|
+
).toHaveLength(1);
|
|
56
|
+
});
|
|
57
|
+
});
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type {WorkflowsJobExecutionQueuedEventDto} from '@shipfox/api-workflows-dto';
|
|
2
|
+
import {logger} from '@shipfox/node-opentelemetry';
|
|
3
|
+
import {enqueueJobExecution} from '#db/job-executions.js';
|
|
4
|
+
|
|
5
|
+
export async function onWorkflowsJobExecutionQueued(
|
|
6
|
+
payload: WorkflowsJobExecutionQueuedEventDto,
|
|
7
|
+
): Promise<void> {
|
|
8
|
+
logger().info(
|
|
9
|
+
{
|
|
10
|
+
jobId: payload.jobId,
|
|
11
|
+
jobExecutionId: payload.jobExecutionId,
|
|
12
|
+
workflowRunAttemptId: payload.workflowRunAttemptId,
|
|
13
|
+
},
|
|
14
|
+
'Queueing runner job execution from workflow fact',
|
|
15
|
+
);
|
|
16
|
+
await enqueueJobExecution({
|
|
17
|
+
workspaceId: payload.workspaceId,
|
|
18
|
+
workflowRunId: payload.workflowRunId,
|
|
19
|
+
workflowRunAttemptId: payload.workflowRunAttemptId,
|
|
20
|
+
jobId: payload.jobId,
|
|
21
|
+
jobExecutionId: payload.jobExecutionId,
|
|
22
|
+
projectId: payload.projectId,
|
|
23
|
+
requiredLabels: payload.requiredLabels,
|
|
24
|
+
queuedAt: new Date(payload.queuedAt),
|
|
25
|
+
});
|
|
26
|
+
}
|