@shipfox/api-runners 12.6.0 → 13.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +4 -4
- package/CHANGELOG.md +20 -0
- package/dist/db/index.d.ts +1 -1
- package/dist/db/index.d.ts.map +1 -1
- package/dist/db/index.js +1 -1
- package/dist/db/index.js.map +1 -1
- package/dist/db/job-executions.d.ts +8 -21
- package/dist/db/job-executions.d.ts.map +1 -1
- package/dist/db/job-executions.js +63 -60
- package/dist/db/job-executions.js.map +1 -1
- package/dist/db/provisioner-tokens.d.ts.map +1 -1
- package/dist/db/provisioner-tokens.js +14 -2
- package/dist/db/provisioner-tokens.js.map +1 -1
- package/dist/db/reservations.d.ts +10 -0
- package/dist/db/reservations.d.ts.map +1 -1
- package/dist/db/reservations.js +139 -17
- package/dist/db/reservations.js.map +1 -1
- package/dist/db/runner-assignments.d.ts.map +1 -1
- package/dist/db/runner-assignments.js +8 -5
- package/dist/db/runner-assignments.js.map +1 -1
- package/dist/db/runner-instances.d.ts +0 -1
- package/dist/db/runner-instances.d.ts.map +1 -1
- package/dist/db/runner-instances.js +85 -100
- package/dist/db/runner-instances.js.map +1 -1
- package/dist/db/runner-states.d.ts +3 -1
- package/dist/db/runner-states.d.ts.map +1 -1
- package/dist/db/runner-states.js +6 -4
- package/dist/db/runner-states.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -4
- package/dist/index.js.map +1 -1
- package/dist/presentation/index.d.ts +2 -1
- package/dist/presentation/index.d.ts.map +1 -1
- package/dist/presentation/index.js +2 -1
- package/dist/presentation/index.js.map +1 -1
- package/dist/presentation/inter-module.d.ts +0 -1
- package/dist/presentation/inter-module.d.ts.map +1 -1
- package/dist/presentation/inter-module.js +2 -25
- package/dist/presentation/inter-module.js.map +1 -1
- package/dist/presentation/subscribers/on-workflows-job-execution-queued.d.ts +3 -0
- package/dist/presentation/subscribers/on-workflows-job-execution-queued.d.ts.map +1 -0
- package/dist/presentation/subscribers/on-workflows-job-execution-queued.js +21 -0
- package/dist/presentation/subscribers/on-workflows-job-execution-queued.js.map +1 -0
- package/dist/presentation/subscribers/on-workflows-job-execution-terminated.d.ts +3 -0
- package/dist/presentation/subscribers/on-workflows-job-execution-terminated.d.ts.map +1 -0
- package/dist/presentation/subscribers/{on-workflows-job-execution-timed-out.js → on-workflows-job-execution-terminated.js} +5 -4
- package/dist/presentation/subscribers/on-workflows-job-execution-terminated.js.map +1 -0
- package/dist/tsconfig.test.tsbuildinfo +1 -1
- package/package.json +3 -3
- package/src/db/index.ts +0 -2
- package/src/db/job-executions.test.ts +106 -217
- package/src/db/job-executions.ts +71 -70
- package/src/db/provisioner-tokens.test.ts +19 -0
- package/src/db/provisioner-tokens.ts +13 -2
- package/src/db/reservations.test.ts +138 -0
- package/src/db/reservations.ts +314 -35
- package/src/db/runner-assignments.test.ts +20 -0
- package/src/db/runner-assignments.ts +6 -7
- package/src/db/runner-instances.test.ts +350 -15
- package/src/db/runner-instances.ts +136 -198
- package/src/db/runner-states.test.ts +16 -0
- package/src/db/runner-states.ts +9 -4
- package/src/index.test.ts +1 -0
- package/src/index.ts +8 -7
- package/src/presentation/index.ts +2 -1
- package/src/presentation/inter-module.ts +2 -40
- package/src/presentation/subscribers/on-workflows-job-execution-queued.test.ts +57 -0
- package/src/presentation/subscribers/on-workflows-job-execution-queued.ts +26 -0
- package/src/presentation/subscribers/on-workflows-job-execution-terminated.test.ts +142 -0
- package/src/presentation/subscribers/{on-workflows-job-execution-timed-out.ts → on-workflows-job-execution-terminated.ts} +5 -4
- package/test/factories/pending-job.ts +3 -0
- package/tsconfig.build.tsbuildinfo +1 -1
- package/dist/presentation/subscribers/on-workflows-job-execution-timed-out.d.ts +0 -3
- package/dist/presentation/subscribers/on-workflows-job-execution-timed-out.d.ts.map +0 -1
- package/dist/presentation/subscribers/on-workflows-job-execution-timed-out.js.map +0 -1
- package/src/presentation/inter-module.test.ts +0 -15
- package/src/presentation/subscribers/on-workflows-job-execution-timed-out.test.ts +0 -163
|
@@ -28,9 +28,9 @@ import {
|
|
|
28
28
|
type RunnerInstanceBoundJobExecution,
|
|
29
29
|
} from './job-executions.js';
|
|
30
30
|
import {lockRunnerReservationAdvisoryKeysTx} from './reservation-locks.js';
|
|
31
|
-
import {
|
|
31
|
+
import {releaseTerminalRunnerInstanceReservationsByIds} from './reservations.js';
|
|
32
32
|
import {validateRunnerReservationCapacityTx} from './runner-assignments.js';
|
|
33
|
-
import {terminalStates} from './runner-states.js';
|
|
33
|
+
import {activeStates, terminalStates} from './runner-states.js';
|
|
34
34
|
import {ephemeralRegistrationTokens} from './schema/ephemeral-registration-tokens.js';
|
|
35
35
|
import {provisionerTokens} from './schema/provisioner-tokens.js';
|
|
36
36
|
import {reservations} from './schema/reservations.js';
|
|
@@ -39,11 +39,6 @@ import {providerRunners, toRunnerInstance} from './schema/runner-instances.js';
|
|
|
39
39
|
import {runnerSessions} from './schema/runner-sessions.js';
|
|
40
40
|
import {runningJobExecutions} from './schema/running-job-executions.js';
|
|
41
41
|
|
|
42
|
-
export const activeStates = [
|
|
43
|
-
'starting',
|
|
44
|
-
'running',
|
|
45
|
-
'stopping',
|
|
46
|
-
] as const satisfies readonly RunnerInstanceState[];
|
|
47
42
|
export const divergenceCountStates = ['starting', 'running'] as const satisfies readonly Extract<
|
|
48
43
|
RunnerInstanceState,
|
|
49
44
|
'starting' | 'running'
|
|
@@ -157,6 +152,14 @@ export async function reportRunnerInstances(params: ReportRunnerInstancesParams)
|
|
|
157
152
|
|
|
158
153
|
const events = await hydrateRunnerSessionIdsFromConsumedTokens(tx, params, aggregatedEvents);
|
|
159
154
|
const reservationSafeEvents = await guardReportedReservationIdsTx(tx, params, events);
|
|
155
|
+
const existingReportedAtByProviderRunnerId = hasTerminalEvent
|
|
156
|
+
? await listExistingProviderRunnerReportedAtTx(tx, params, reservationSafeEvents)
|
|
157
|
+
: new Map<string, Date>();
|
|
158
|
+
const freshTerminalEvents = reservationSafeEvents.filter((event) => {
|
|
159
|
+
if (!isTerminalState(event.state)) return false;
|
|
160
|
+
const existingReportedAt = existingReportedAtByProviderRunnerId.get(event.providerRunnerId);
|
|
161
|
+
return existingReportedAt === undefined || event.reportedAt >= existingReportedAt;
|
|
162
|
+
});
|
|
160
163
|
const terminateIntentsHonored = await listTerminateIntentsHonoredByTerminatedReportsTx(
|
|
161
164
|
tx,
|
|
162
165
|
params,
|
|
@@ -214,9 +217,10 @@ export async function reportRunnerInstances(params: ReportRunnerInstancesParams)
|
|
|
214
217
|
`,
|
|
215
218
|
});
|
|
216
219
|
|
|
217
|
-
const reservationsReleased =
|
|
218
|
-
|
|
219
|
-
|
|
220
|
+
const reservationsReleased =
|
|
221
|
+
freshTerminalEvents.length > 0
|
|
222
|
+
? await releaseTerminalRunnerInstanceReservations(tx, params, freshTerminalEvents)
|
|
223
|
+
: 0;
|
|
220
224
|
|
|
221
225
|
return {
|
|
222
226
|
accepted: reservationSafeEvents.length,
|
|
@@ -226,21 +230,73 @@ export async function reportRunnerInstances(params: ReportRunnerInstancesParams)
|
|
|
226
230
|
});
|
|
227
231
|
}
|
|
228
232
|
|
|
233
|
+
async function listExistingProviderRunnerReportedAtTx(
|
|
234
|
+
tx: Tx,
|
|
235
|
+
params: ReportRunnerInstancesParams,
|
|
236
|
+
events: RunnerInstanceReportRow[],
|
|
237
|
+
): Promise<Map<string, Date>> {
|
|
238
|
+
const providerRunnerIds = [...new Set(events.map((event) => event.providerRunnerId))];
|
|
239
|
+
if (providerRunnerIds.length === 0) return new Map();
|
|
240
|
+
|
|
241
|
+
const rows = await tx
|
|
242
|
+
.select({
|
|
243
|
+
providerRunnerId: providerRunners.providerRunnerId,
|
|
244
|
+
reportedAt: providerRunners.reportedAt,
|
|
245
|
+
})
|
|
246
|
+
.from(providerRunners)
|
|
247
|
+
.where(
|
|
248
|
+
and(
|
|
249
|
+
eq(providerRunners.provisionerId, params.provisionerId),
|
|
250
|
+
inArray(providerRunners.providerRunnerId, providerRunnerIds),
|
|
251
|
+
),
|
|
252
|
+
);
|
|
253
|
+
return new Map(
|
|
254
|
+
rows.flatMap((row) =>
|
|
255
|
+
row.providerRunnerId ? [[row.providerRunnerId, row.reportedAt] as const] : [],
|
|
256
|
+
),
|
|
257
|
+
);
|
|
258
|
+
}
|
|
259
|
+
|
|
229
260
|
async function guardReportedReservationIdsTx(
|
|
230
261
|
tx: Tx,
|
|
231
262
|
params: ReportRunnerInstancesParams,
|
|
232
263
|
events: RunnerInstanceReportRow[],
|
|
233
264
|
): Promise<RunnerInstanceReportRow[]> {
|
|
234
|
-
const
|
|
265
|
+
const reportedReservationIds = [
|
|
235
266
|
...new Set(events.flatMap((event) => (event.reservationId ? [event.reservationId] : []))),
|
|
236
267
|
].sort();
|
|
237
|
-
|
|
268
|
+
const providerRunnerIds = [...new Set(events.map((event) => event.providerRunnerId))];
|
|
269
|
+
if (reportedReservationIds.length === 0 && !events.some((event) => isTerminalState(event.state)))
|
|
270
|
+
return events;
|
|
271
|
+
|
|
272
|
+
// Lock every reservation currently associated with the reported runners before taking a
|
|
273
|
+
// runner row lock. Terminal reports may omit reservationId, or carry a stale one; discovering
|
|
274
|
+
// the stored IDs here keeps report projection in the same reservation-then-runner order as
|
|
275
|
+
// assignment and terminal cleanup.
|
|
276
|
+
const existingReservationRows = await tx
|
|
277
|
+
.select({
|
|
278
|
+
reservationId: providerRunners.reservationId,
|
|
279
|
+
intendedReservationId: providerRunners.intendedReservationId,
|
|
280
|
+
})
|
|
281
|
+
.from(providerRunners)
|
|
282
|
+
.where(
|
|
283
|
+
and(
|
|
284
|
+
eq(providerRunners.provisionerId, params.provisionerId),
|
|
285
|
+
inArray(providerRunners.providerRunnerId, providerRunnerIds),
|
|
286
|
+
),
|
|
287
|
+
);
|
|
288
|
+
const storedReservationIds = existingReservationRows.flatMap((row) =>
|
|
289
|
+
[row.reservationId, row.intendedReservationId].filter(
|
|
290
|
+
(reservationId): reservationId is string => reservationId !== null,
|
|
291
|
+
),
|
|
292
|
+
);
|
|
293
|
+
const reservationIds = [...new Set([...reportedReservationIds, ...storedReservationIds])].sort();
|
|
294
|
+
if (reservationIds.length > 0)
|
|
295
|
+
await lockRunnerReservationAdvisoryKeysTx(tx, {
|
|
296
|
+
provisionerId: params.provisionerId,
|
|
297
|
+
reservationIds,
|
|
298
|
+
});
|
|
238
299
|
|
|
239
|
-
await lockRunnerReservationAdvisoryKeysTx(tx, {
|
|
240
|
-
provisionerId: params.provisionerId,
|
|
241
|
-
reservationIds,
|
|
242
|
-
});
|
|
243
|
-
const providerRunnerIds = events.map((event) => event.providerRunnerId);
|
|
244
300
|
const existingRows = await tx
|
|
245
301
|
.select({
|
|
246
302
|
providerRunnerId: providerRunners.providerRunnerId,
|
|
@@ -713,16 +769,15 @@ export async function reconcileRunnerInstances(
|
|
|
713
769
|
),
|
|
714
770
|
),
|
|
715
771
|
)
|
|
716
|
-
.returning({providerRunnerId: providerRunners.providerRunnerId});
|
|
772
|
+
.returning({id: providerRunners.id, providerRunnerId: providerRunners.providerRunnerId});
|
|
717
773
|
|
|
718
774
|
absentIds = updated.flatMap((row) => (row.providerRunnerId ? [row.providerRunnerId] : []));
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
}
|
|
775
|
+
reservationsReleased = await releaseTerminalRunnerInstanceReservationsByIds(tx, {
|
|
776
|
+
workspaceId: params.workspaceId,
|
|
777
|
+
provisionerId: params.provisionerId,
|
|
778
|
+
runnerInstanceIds: updated.map((row) => row.id),
|
|
779
|
+
requireUnlinkedSession: false,
|
|
780
|
+
});
|
|
726
781
|
}
|
|
727
782
|
}
|
|
728
783
|
|
|
@@ -782,10 +837,41 @@ export async function reapStaleRunnerInstances(params: {
|
|
|
782
837
|
.orderBy(asc(providerRunners.updatedAt), asc(providerRunners.id))
|
|
783
838
|
.limit(params.limit);
|
|
784
839
|
|
|
785
|
-
|
|
840
|
+
const terminalReservationRows =
|
|
841
|
+
candidateRows.length === 0
|
|
842
|
+
? await tx
|
|
843
|
+
.select({
|
|
844
|
+
id: providerRunners.id,
|
|
845
|
+
workspaceId: providerRunners.workspaceId,
|
|
846
|
+
provisionerId: providerRunners.provisionerId,
|
|
847
|
+
providerRunnerId: providerRunners.providerRunnerId,
|
|
848
|
+
})
|
|
849
|
+
.from(providerRunners)
|
|
850
|
+
.where(
|
|
851
|
+
and(
|
|
852
|
+
inArray(providerRunners.state, terminalStates),
|
|
853
|
+
isNotNull(providerRunners.runnerSessionId),
|
|
854
|
+
lt(providerRunners.updatedAt, cutoff),
|
|
855
|
+
isNull(providerRunners.reservationReleasedAt),
|
|
856
|
+
or(
|
|
857
|
+
isNotNull(providerRunners.reservationId),
|
|
858
|
+
isNotNull(providerRunners.intendedReservationId),
|
|
859
|
+
),
|
|
860
|
+
),
|
|
861
|
+
)
|
|
862
|
+
.orderBy(asc(providerRunners.updatedAt), asc(providerRunners.id))
|
|
863
|
+
.limit(params.limit)
|
|
864
|
+
: [];
|
|
865
|
+
|
|
866
|
+
if (candidateRows.length === 0 && terminalReservationRows.length === 0)
|
|
867
|
+
return {reaped: 0, reservationsReleased: 0};
|
|
786
868
|
|
|
787
869
|
const workspaceIds = [
|
|
788
|
-
...new Set(
|
|
870
|
+
...new Set(
|
|
871
|
+
[...candidateRows, ...terminalReservationRows].flatMap((row) =>
|
|
872
|
+
row.workspaceId ? [row.workspaceId] : [],
|
|
873
|
+
),
|
|
874
|
+
),
|
|
789
875
|
].sort();
|
|
790
876
|
for (const workspaceId of workspaceIds) {
|
|
791
877
|
await tx.execute(sql`select pg_advisory_xact_lock(hashtext(${workspaceId}))`);
|
|
@@ -809,6 +895,7 @@ export async function reapStaleRunnerInstances(params: {
|
|
|
809
895
|
),
|
|
810
896
|
)
|
|
811
897
|
.returning({
|
|
898
|
+
id: providerRunners.id,
|
|
812
899
|
workspaceId: providerRunners.workspaceId,
|
|
813
900
|
provisionerId: providerRunners.provisionerId,
|
|
814
901
|
providerRunnerId: providerRunners.providerRunnerId,
|
|
@@ -819,7 +906,15 @@ export async function reapStaleRunnerInstances(params: {
|
|
|
819
906
|
reservationsReleased += await releaseTerminalRunnerInstanceReservationsByIds(tx, {
|
|
820
907
|
workspaceId: group.workspaceId,
|
|
821
908
|
provisionerId: group.provisionerId,
|
|
822
|
-
|
|
909
|
+
runnerInstanceIds: group.runnerInstanceIds,
|
|
910
|
+
requireUnlinkedSession: false,
|
|
911
|
+
});
|
|
912
|
+
}
|
|
913
|
+
for (const group of groupRunnerInstanceIds(terminalReservationRows)) {
|
|
914
|
+
reservationsReleased += await releaseTerminalRunnerInstanceReservationsByIds(tx, {
|
|
915
|
+
workspaceId: group.workspaceId,
|
|
916
|
+
provisionerId: group.provisionerId,
|
|
917
|
+
runnerInstanceIds: group.runnerInstanceIds,
|
|
823
918
|
requireUnlinkedSession: false,
|
|
824
919
|
});
|
|
825
920
|
}
|
|
@@ -840,174 +935,13 @@ async function releaseTerminalRunnerInstanceReservations(
|
|
|
840
935
|
workspaceId: params.workspaceId,
|
|
841
936
|
provisionerId: params.provisionerId,
|
|
842
937
|
providerRunnerIds: terminalEvents.map((event) => event.providerRunnerId),
|
|
938
|
+
reportedAtByProviderRunnerId: new Map(
|
|
939
|
+
terminalEvents.map((event) => [event.providerRunnerId, event.reportedAt]),
|
|
940
|
+
),
|
|
941
|
+
requireUnlinkedSession: false,
|
|
843
942
|
});
|
|
844
943
|
}
|
|
845
944
|
|
|
846
|
-
async function releaseTerminalRunnerInstanceReservationsByIds(
|
|
847
|
-
tx: Tx,
|
|
848
|
-
params: {
|
|
849
|
-
workspaceId: string | null;
|
|
850
|
-
provisionerId: string;
|
|
851
|
-
providerRunnerIds: string[];
|
|
852
|
-
requireUnlinkedSession?: boolean;
|
|
853
|
-
},
|
|
854
|
-
): Promise<number> {
|
|
855
|
-
if (params.providerRunnerIds.length === 0) return 0;
|
|
856
|
-
|
|
857
|
-
const reservationWorkspacePredicate =
|
|
858
|
-
params.workspaceId === null
|
|
859
|
-
? sql``
|
|
860
|
-
: sql`and ${eq(reservations.workspaceId, params.workspaceId)}`;
|
|
861
|
-
|
|
862
|
-
const rows = await tx
|
|
863
|
-
.select({
|
|
864
|
-
id: providerRunners.id,
|
|
865
|
-
releaseReservationId: sql<string | null>`coalesce(
|
|
866
|
-
(select ${reservations.id}
|
|
867
|
-
from ${reservations}
|
|
868
|
-
where ${reservations.id} = ${providerRunners.intendedReservationId}
|
|
869
|
-
and ${reservations.provisionerId} = ${params.provisionerId}
|
|
870
|
-
${reservationWorkspacePredicate}),
|
|
871
|
-
(select ${reservations.id}
|
|
872
|
-
from ${reservations}
|
|
873
|
-
where ${reservations.id} = ${providerRunners.reservationId}
|
|
874
|
-
and ${reservations.provisionerId} = ${params.provisionerId}
|
|
875
|
-
${reservationWorkspacePredicate})
|
|
876
|
-
)`,
|
|
877
|
-
releaseReservationWorkspaceId: sql<string | null>`coalesce(
|
|
878
|
-
(select ${reservations.workspaceId}
|
|
879
|
-
from ${reservations}
|
|
880
|
-
where ${reservations.id} = ${providerRunners.intendedReservationId}
|
|
881
|
-
and ${reservations.provisionerId} = ${params.provisionerId}
|
|
882
|
-
${reservationWorkspacePredicate}),
|
|
883
|
-
(select ${reservations.workspaceId}
|
|
884
|
-
from ${reservations}
|
|
885
|
-
where ${reservations.id} = ${providerRunners.reservationId}
|
|
886
|
-
and ${reservations.provisionerId} = ${params.provisionerId}
|
|
887
|
-
${reservationWorkspacePredicate})
|
|
888
|
-
)`,
|
|
889
|
-
})
|
|
890
|
-
.from(providerRunners)
|
|
891
|
-
.where(
|
|
892
|
-
and(
|
|
893
|
-
eq(providerRunners.provisionerId, params.provisionerId),
|
|
894
|
-
inArray(providerRunners.providerRunnerId, params.providerRunnerIds),
|
|
895
|
-
inArray(providerRunners.state, terminalStates),
|
|
896
|
-
or(
|
|
897
|
-
isNotNull(providerRunners.reservationId),
|
|
898
|
-
isNotNull(providerRunners.intendedReservationId),
|
|
899
|
-
),
|
|
900
|
-
params.workspaceId === null
|
|
901
|
-
? undefined
|
|
902
|
-
: or(
|
|
903
|
-
and(
|
|
904
|
-
eq(providerRunners.workspaceId, params.workspaceId),
|
|
905
|
-
or(
|
|
906
|
-
exists(
|
|
907
|
-
tx
|
|
908
|
-
.select({id: reservations.id})
|
|
909
|
-
.from(reservations)
|
|
910
|
-
.where(
|
|
911
|
-
and(
|
|
912
|
-
eq(reservations.workspaceId, params.workspaceId),
|
|
913
|
-
eq(reservations.provisionerId, params.provisionerId),
|
|
914
|
-
eq(reservations.id, providerRunners.reservationId),
|
|
915
|
-
),
|
|
916
|
-
),
|
|
917
|
-
),
|
|
918
|
-
exists(
|
|
919
|
-
tx
|
|
920
|
-
.select({id: reservations.id})
|
|
921
|
-
.from(reservations)
|
|
922
|
-
.where(
|
|
923
|
-
and(
|
|
924
|
-
eq(reservations.workspaceId, params.workspaceId),
|
|
925
|
-
eq(reservations.provisionerId, params.provisionerId),
|
|
926
|
-
eq(reservations.id, providerRunners.intendedReservationId),
|
|
927
|
-
),
|
|
928
|
-
),
|
|
929
|
-
),
|
|
930
|
-
),
|
|
931
|
-
),
|
|
932
|
-
and(
|
|
933
|
-
isNull(providerRunners.workspaceId),
|
|
934
|
-
isNotNull(providerRunners.intendedReservationId),
|
|
935
|
-
exists(
|
|
936
|
-
tx
|
|
937
|
-
.select({id: reservations.id})
|
|
938
|
-
.from(reservations)
|
|
939
|
-
.where(
|
|
940
|
-
and(
|
|
941
|
-
eq(reservations.workspaceId, params.workspaceId),
|
|
942
|
-
eq(reservations.provisionerId, params.provisionerId),
|
|
943
|
-
eq(reservations.id, providerRunners.intendedReservationId),
|
|
944
|
-
),
|
|
945
|
-
),
|
|
946
|
-
),
|
|
947
|
-
),
|
|
948
|
-
),
|
|
949
|
-
params.requireUnlinkedSession === false
|
|
950
|
-
? undefined
|
|
951
|
-
: isNull(providerRunners.runnerSessionId),
|
|
952
|
-
isNull(providerRunners.reservationReleasedAt),
|
|
953
|
-
),
|
|
954
|
-
)
|
|
955
|
-
.for('update');
|
|
956
|
-
|
|
957
|
-
if (rows.length === 0) return 0;
|
|
958
|
-
|
|
959
|
-
const updated = await tx
|
|
960
|
-
.update(providerRunners)
|
|
961
|
-
.set({
|
|
962
|
-
intendedReservationId: null,
|
|
963
|
-
reservationReleasedAt: sql`now()`,
|
|
964
|
-
updatedAt: sql`now()`,
|
|
965
|
-
})
|
|
966
|
-
.where(
|
|
967
|
-
and(
|
|
968
|
-
inArray(
|
|
969
|
-
providerRunners.id,
|
|
970
|
-
rows.map((row) => row.id),
|
|
971
|
-
),
|
|
972
|
-
params.requireUnlinkedSession === false
|
|
973
|
-
? undefined
|
|
974
|
-
: isNull(providerRunners.runnerSessionId),
|
|
975
|
-
isNull(providerRunners.reservationReleasedAt),
|
|
976
|
-
),
|
|
977
|
-
)
|
|
978
|
-
.returning({id: providerRunners.id});
|
|
979
|
-
|
|
980
|
-
const releasesByReservationId = new Map<
|
|
981
|
-
string,
|
|
982
|
-
{workspaceId: string; reservationId: string; count: number}
|
|
983
|
-
>();
|
|
984
|
-
const updatedIds = new Set(updated.map((row) => row.id));
|
|
985
|
-
for (const row of rows) {
|
|
986
|
-
if (!updatedIds.has(row.id)) continue;
|
|
987
|
-
if (!row.releaseReservationId || !row.releaseReservationWorkspaceId) continue;
|
|
988
|
-
const key = `${row.releaseReservationWorkspaceId}:${row.releaseReservationId}`;
|
|
989
|
-
const release = releasesByReservationId.get(key) ?? {
|
|
990
|
-
workspaceId: row.releaseReservationWorkspaceId,
|
|
991
|
-
reservationId: row.releaseReservationId,
|
|
992
|
-
count: 0,
|
|
993
|
-
};
|
|
994
|
-
release.count += 1;
|
|
995
|
-
releasesByReservationId.set(key, release);
|
|
996
|
-
}
|
|
997
|
-
|
|
998
|
-
if (releasesByReservationId.size === 0) return 0;
|
|
999
|
-
|
|
1000
|
-
let released = 0;
|
|
1001
|
-
for (const release of releasesByReservationId.values()) {
|
|
1002
|
-
released += await releaseReservationUnits(tx, {
|
|
1003
|
-
workspaceId: release.workspaceId,
|
|
1004
|
-
provisionerId: params.provisionerId,
|
|
1005
|
-
releases: [{reservationId: release.reservationId, count: release.count}],
|
|
1006
|
-
});
|
|
1007
|
-
}
|
|
1008
|
-
return released;
|
|
1009
|
-
}
|
|
1010
|
-
|
|
1011
945
|
function staleRunnerInstanceCutoff(thresholdSeconds: number): SQL {
|
|
1012
946
|
return sql`now() - (${thresholdSeconds} || ' seconds')::interval`;
|
|
1013
947
|
}
|
|
@@ -1058,24 +992,28 @@ function staleRunnerInstanceWhere(tx: Tx, cutoff: SQL): SQL<boolean> {
|
|
|
1058
992
|
|
|
1059
993
|
function groupRunnerInstanceIds(
|
|
1060
994
|
rows: Array<{
|
|
995
|
+
id: string;
|
|
1061
996
|
workspaceId: string | null;
|
|
1062
997
|
provisionerId: string;
|
|
1063
998
|
providerRunnerId: string | null;
|
|
1064
999
|
}>,
|
|
1065
|
-
): Array<{
|
|
1000
|
+
): Array<{
|
|
1001
|
+
workspaceId: string | null;
|
|
1002
|
+
provisionerId: string;
|
|
1003
|
+
runnerInstanceIds: string[];
|
|
1004
|
+
}> {
|
|
1066
1005
|
const groups = new Map<
|
|
1067
1006
|
string,
|
|
1068
|
-
{workspaceId: string | null; provisionerId: string;
|
|
1007
|
+
{workspaceId: string | null; provisionerId: string; runnerInstanceIds: string[]}
|
|
1069
1008
|
>();
|
|
1070
1009
|
for (const row of rows) {
|
|
1071
|
-
if (!row.providerRunnerId) continue;
|
|
1072
1010
|
const key = `${row.workspaceId ?? ''}:${row.provisionerId}`;
|
|
1073
1011
|
const group = groups.get(key) ?? {
|
|
1074
1012
|
workspaceId: row.workspaceId,
|
|
1075
1013
|
provisionerId: row.provisionerId,
|
|
1076
|
-
|
|
1014
|
+
runnerInstanceIds: [],
|
|
1077
1015
|
};
|
|
1078
|
-
group.
|
|
1016
|
+
group.runnerInstanceIds.push(row.id);
|
|
1079
1017
|
groups.set(key, group);
|
|
1080
1018
|
}
|
|
1081
1019
|
return [...groups.values()];
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import {providerRunnerStateSchema} from '@shipfox/api-runners-dto';
|
|
2
|
+
import {activeStates, terminalStates} from '#db/runner-states.js';
|
|
3
|
+
|
|
4
|
+
describe('runner state partitions', () => {
|
|
5
|
+
it('covers every provider state exactly once', () => {
|
|
6
|
+
const states = [...activeStates, ...terminalStates];
|
|
7
|
+
|
|
8
|
+
expect(new Set(states).size).toBe(states.length);
|
|
9
|
+
expect([...states].sort()).toEqual([...providerRunnerStateSchema.options].sort());
|
|
10
|
+
expect([...terminalStates]).toEqual(
|
|
11
|
+
providerRunnerStateSchema.options.filter(
|
|
12
|
+
(state) => !activeStates.includes(state as (typeof activeStates)[number]),
|
|
13
|
+
),
|
|
14
|
+
);
|
|
15
|
+
});
|
|
16
|
+
});
|
package/src/db/runner-states.ts
CHANGED
|
@@ -1,7 +1,12 @@
|
|
|
1
|
+
import {providerRunnerStateSchema} from '@shipfox/api-runners-dto';
|
|
1
2
|
import type {RunnerInstanceState} from '#core/entities/runner-instance.js';
|
|
2
3
|
|
|
3
|
-
export const
|
|
4
|
-
'
|
|
5
|
-
'
|
|
6
|
-
'
|
|
4
|
+
export const activeStates = [
|
|
5
|
+
'starting',
|
|
6
|
+
'running',
|
|
7
|
+
'stopping',
|
|
7
8
|
] as const satisfies readonly RunnerInstanceState[];
|
|
9
|
+
|
|
10
|
+
export const terminalStates = providerRunnerStateSchema.options.filter(
|
|
11
|
+
(state) => !activeStates.includes(state as (typeof activeStates)[number]),
|
|
12
|
+
) as readonly Extract<RunnerInstanceState, 'stopped' | 'failed' | 'terminated'>[];
|
package/src/index.test.ts
CHANGED
|
@@ -10,6 +10,7 @@ describe('createRunnersModule', () => {
|
|
|
10
10
|
expect(module.name).toBe('runners');
|
|
11
11
|
expect(module.auth).toHaveLength(3);
|
|
12
12
|
expect(module.routes).toHaveLength(13);
|
|
13
|
+
expect(module.subscribers).toHaveLength(2);
|
|
13
14
|
expect(module.workers).toHaveLength(1);
|
|
14
15
|
});
|
|
15
16
|
|
package/src/index.ts
CHANGED
|
@@ -4,7 +4,8 @@ import type {AuthInterModuleClient} from '@shipfox/api-auth-dto/inter-module';
|
|
|
4
4
|
import {administrationActionEventSchemas} from '@shipfox/api-common-dto';
|
|
5
5
|
import {runnersEventSchemas} from '@shipfox/api-runners-dto';
|
|
6
6
|
import {
|
|
7
|
-
|
|
7
|
+
WORKFLOWS_JOB_EXECUTION_QUEUED,
|
|
8
|
+
WORKFLOWS_JOB_EXECUTION_TERMINATED,
|
|
8
9
|
type WorkflowsEventMapDto,
|
|
9
10
|
} from '@shipfox/api-workflows-dto';
|
|
10
11
|
import {type ShipfoxModule, subscriberFactory} from '@shipfox/node-module';
|
|
@@ -16,7 +17,8 @@ import {
|
|
|
16
17
|
createRunnerControlSessionAuthMethod,
|
|
17
18
|
createRunnerRegistrationTokenAuthMethod,
|
|
18
19
|
createRunnerRoutes,
|
|
19
|
-
|
|
20
|
+
onWorkflowsJobExecutionQueued,
|
|
21
|
+
onWorkflowsJobExecutionTerminated,
|
|
20
22
|
} from '#presentation/index.js';
|
|
21
23
|
import {createRunnersInterModulePresentation} from '#presentation/inter-module.js';
|
|
22
24
|
import {createRunnersMaintenanceActivities} from '#temporal/activities/index.js';
|
|
@@ -30,12 +32,8 @@ export {
|
|
|
30
32
|
unadvertisedRunnerTools,
|
|
31
33
|
} from '#core/runner-tool-capabilities.js';
|
|
32
34
|
export {
|
|
33
|
-
cancelRunnerJobs,
|
|
34
|
-
type EnqueueJobExecutionParams,
|
|
35
|
-
enqueueJobExecution,
|
|
36
35
|
getWorkspaceJobCounts,
|
|
37
36
|
isJobLeaseActive,
|
|
38
|
-
releaseJobExecution,
|
|
39
37
|
} from '#db/index.js';
|
|
40
38
|
export type {
|
|
41
39
|
CreateRunnersModuleOptions,
|
|
@@ -64,7 +62,10 @@ export function createRunnersModule({
|
|
|
64
62
|
publishers: [
|
|
65
63
|
{name: 'runners', table: runnersOutbox, db, eventSchemas: runnersPublisherEventSchemas},
|
|
66
64
|
],
|
|
67
|
-
subscribers: [
|
|
65
|
+
subscribers: [
|
|
66
|
+
subscriber(WORKFLOWS_JOB_EXECUTION_QUEUED, onWorkflowsJobExecutionQueued),
|
|
67
|
+
subscriber(WORKFLOWS_JOB_EXECUTION_TERMINATED, onWorkflowsJobExecutionTerminated),
|
|
68
|
+
],
|
|
68
69
|
workers: [
|
|
69
70
|
{
|
|
70
71
|
taskQueue: RUNNERS_MAINTENANCE_TASK_QUEUE,
|
|
@@ -4,4 +4,5 @@ export {
|
|
|
4
4
|
createRunnerRegistrationTokenAuthMethod,
|
|
5
5
|
} from './auth/index.js';
|
|
6
6
|
export {createRunnerRoutes, createRunnerRoutes as routes} from './routes/index.js';
|
|
7
|
-
export {
|
|
7
|
+
export {onWorkflowsJobExecutionQueued} from './subscribers/on-workflows-job-execution-queued.js';
|
|
8
|
+
export {onWorkflowsJobExecutionTerminated} from './subscribers/on-workflows-job-execution-terminated.js';
|
|
@@ -1,39 +1,12 @@
|
|
|
1
1
|
import {runnersInterModuleContract} from '@shipfox/api-runners-dto/inter-module';
|
|
2
|
-
import {
|
|
3
|
-
createInterModuleKnownError,
|
|
4
|
-
defineInterModulePresentation,
|
|
5
|
-
type InterModulePresentation,
|
|
6
|
-
} from '@shipfox/inter-module';
|
|
7
|
-
import {EmptyRequiredLabelsError} from '#core/errors.js';
|
|
2
|
+
import {defineInterModulePresentation, type InterModulePresentation} from '@shipfox/inter-module';
|
|
8
3
|
import {getEffectiveRunnerToolCapabilities} from '#core/runner-tool-capabilities.js';
|
|
9
|
-
import {
|
|
10
|
-
cancelRunnerJobs,
|
|
11
|
-
enqueueJobExecution,
|
|
12
|
-
getWorkspaceJobCounts,
|
|
13
|
-
isJobLeaseActive,
|
|
14
|
-
releaseJobExecution,
|
|
15
|
-
} from '#db/job-executions.js';
|
|
4
|
+
import {getWorkspaceJobCounts, isJobLeaseActive} from '#db/job-executions.js';
|
|
16
5
|
|
|
17
6
|
export function createRunnersInterModulePresentation(): InterModulePresentation<
|
|
18
7
|
typeof runnersInterModuleContract
|
|
19
8
|
> {
|
|
20
9
|
return defineInterModulePresentation(runnersInterModuleContract, {
|
|
21
|
-
enqueueJobExecution: async (input) => {
|
|
22
|
-
try {
|
|
23
|
-
await enqueueJobExecution(input);
|
|
24
|
-
return {};
|
|
25
|
-
} catch (error) {
|
|
26
|
-
throw toEnqueueJobExecutionKnownError(error);
|
|
27
|
-
}
|
|
28
|
-
},
|
|
29
|
-
releaseJobExecution: async (input) => {
|
|
30
|
-
await releaseJobExecution(input);
|
|
31
|
-
return {};
|
|
32
|
-
},
|
|
33
|
-
cancelJobs: async (input) => {
|
|
34
|
-
await cancelRunnerJobs(input);
|
|
35
|
-
return {};
|
|
36
|
-
},
|
|
37
10
|
getLeaseState: async (input) => ({active: await isJobLeaseActive(input)}),
|
|
38
11
|
getEffectiveRunnerToolCapabilities: async (input) => {
|
|
39
12
|
const result = await getEffectiveRunnerToolCapabilities(input);
|
|
@@ -44,14 +17,3 @@ export function createRunnersInterModulePresentation(): InterModulePresentation<
|
|
|
44
17
|
}),
|
|
45
18
|
});
|
|
46
19
|
}
|
|
47
|
-
|
|
48
|
-
export function toEnqueueJobExecutionKnownError(error: unknown): unknown {
|
|
49
|
-
if (error instanceof EmptyRequiredLabelsError) {
|
|
50
|
-
return createInterModuleKnownError(
|
|
51
|
-
runnersInterModuleContract.methods.enqueueJobExecution,
|
|
52
|
-
'empty-required-labels',
|
|
53
|
-
{},
|
|
54
|
-
);
|
|
55
|
-
}
|
|
56
|
-
return error;
|
|
57
|
-
}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import type {WorkflowsJobExecutionQueuedEventDto} from '@shipfox/api-workflows-dto';
|
|
2
|
+
import {eq} from 'drizzle-orm';
|
|
3
|
+
import {db} from '#db/db.js';
|
|
4
|
+
import {claimPendingJobExecution} from '#db/job-executions.js';
|
|
5
|
+
import {pendingJobExecutions} from '#db/schema/pending-job-executions.js';
|
|
6
|
+
import {runningJobExecutions} from '#db/schema/running-job-executions.js';
|
|
7
|
+
import {runnerSessionFactory} from '#test/index.js';
|
|
8
|
+
import {onWorkflowsJobExecutionQueued} from './on-workflows-job-execution-queued.js';
|
|
9
|
+
|
|
10
|
+
describe('onWorkflowsJobExecutionQueued', () => {
|
|
11
|
+
it('queues from the workflow timestamp and is idempotent across claim', async () => {
|
|
12
|
+
const workspaceId = crypto.randomUUID();
|
|
13
|
+
const payload: WorkflowsJobExecutionQueuedEventDto = {
|
|
14
|
+
jobId: crypto.randomUUID(),
|
|
15
|
+
jobExecutionId: crypto.randomUUID(),
|
|
16
|
+
workflowRunId: crypto.randomUUID(),
|
|
17
|
+
workflowRunAttemptId: crypto.randomUUID(),
|
|
18
|
+
workspaceId,
|
|
19
|
+
projectId: crypto.randomUUID(),
|
|
20
|
+
requiredLabels: ['linux'],
|
|
21
|
+
queuedAt: '2026-08-11T08:00:00.000Z',
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
await onWorkflowsJobExecutionQueued(payload);
|
|
25
|
+
await onWorkflowsJobExecutionQueued(payload);
|
|
26
|
+
|
|
27
|
+
const pending = await db()
|
|
28
|
+
.select()
|
|
29
|
+
.from(pendingJobExecutions)
|
|
30
|
+
.where(eq(pendingJobExecutions.jobExecutionId, payload.jobExecutionId));
|
|
31
|
+
expect(pending).toHaveLength(1);
|
|
32
|
+
expect(pending[0]?.createdAt.toISOString()).toBe(payload.queuedAt);
|
|
33
|
+
|
|
34
|
+
const session = await runnerSessionFactory.create({workspaceId});
|
|
35
|
+
await claimPendingJobExecution({
|
|
36
|
+
workspaceId,
|
|
37
|
+
runnerSessionId: session.id,
|
|
38
|
+
sessionLabels: ['linux'],
|
|
39
|
+
maxClaims: null,
|
|
40
|
+
runnerSessionLivenessThrottleSeconds: 10,
|
|
41
|
+
});
|
|
42
|
+
await onWorkflowsJobExecutionQueued(payload);
|
|
43
|
+
|
|
44
|
+
expect(
|
|
45
|
+
await db()
|
|
46
|
+
.select()
|
|
47
|
+
.from(pendingJobExecutions)
|
|
48
|
+
.where(eq(pendingJobExecutions.jobExecutionId, payload.jobExecutionId)),
|
|
49
|
+
).toHaveLength(0);
|
|
50
|
+
expect(
|
|
51
|
+
await db()
|
|
52
|
+
.select()
|
|
53
|
+
.from(runningJobExecutions)
|
|
54
|
+
.where(eq(runningJobExecutions.jobExecutionId, payload.jobExecutionId)),
|
|
55
|
+
).toHaveLength(1);
|
|
56
|
+
});
|
|
57
|
+
});
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type {WorkflowsJobExecutionQueuedEventDto} from '@shipfox/api-workflows-dto';
|
|
2
|
+
import {logger} from '@shipfox/node-opentelemetry';
|
|
3
|
+
import {enqueueJobExecution} from '#db/job-executions.js';
|
|
4
|
+
|
|
5
|
+
export async function onWorkflowsJobExecutionQueued(
|
|
6
|
+
payload: WorkflowsJobExecutionQueuedEventDto,
|
|
7
|
+
): Promise<void> {
|
|
8
|
+
logger().info(
|
|
9
|
+
{
|
|
10
|
+
jobId: payload.jobId,
|
|
11
|
+
jobExecutionId: payload.jobExecutionId,
|
|
12
|
+
workflowRunAttemptId: payload.workflowRunAttemptId,
|
|
13
|
+
},
|
|
14
|
+
'Queueing runner job execution from workflow fact',
|
|
15
|
+
);
|
|
16
|
+
await enqueueJobExecution({
|
|
17
|
+
workspaceId: payload.workspaceId,
|
|
18
|
+
workflowRunId: payload.workflowRunId,
|
|
19
|
+
workflowRunAttemptId: payload.workflowRunAttemptId,
|
|
20
|
+
jobId: payload.jobId,
|
|
21
|
+
jobExecutionId: payload.jobExecutionId,
|
|
22
|
+
projectId: payload.projectId,
|
|
23
|
+
requiredLabels: payload.requiredLabels,
|
|
24
|
+
queuedAt: new Date(payload.queuedAt),
|
|
25
|
+
});
|
|
26
|
+
}
|