@shipfox/api-runners 12.3.0 → 12.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +4 -4
- package/CHANGELOG.md +36 -0
- package/dist/config.d.ts +1 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +9 -2
- package/dist/config.js.map +1 -1
- package/dist/core/demand.d.ts +4 -2
- package/dist/core/demand.d.ts.map +1 -1
- package/dist/core/demand.js +22 -6
- package/dist/core/demand.js.map +1 -1
- package/dist/core/entities/reservation.d.ts +1 -0
- package/dist/core/entities/reservation.d.ts.map +1 -1
- package/dist/core/entities/reservation.js.map +1 -1
- package/dist/core/entities/runner-instance.d.ts +3 -0
- package/dist/core/entities/runner-instance.d.ts.map +1 -1
- package/dist/core/entities/runner-instance.js.map +1 -1
- package/dist/core/errors.d.ts +5 -1
- package/dist/core/errors.d.ts.map +1 -1
- package/dist/core/errors.js +9 -2
- package/dist/core/errors.js.map +1 -1
- package/dist/core/runner-activation.d.ts +4 -1
- package/dist/core/runner-activation.d.ts.map +1 -1
- package/dist/core/runner-activation.js +32 -3
- package/dist/core/runner-activation.js.map +1 -1
- package/dist/core/runner-control-sessions.d.ts +8 -4
- package/dist/core/runner-control-sessions.d.ts.map +1 -1
- package/dist/core/runner-control-sessions.js +116 -20
- package/dist/core/runner-control-sessions.js.map +1 -1
- package/dist/core/runner-instances.d.ts +1 -0
- package/dist/core/runner-instances.d.ts.map +1 -1
- package/dist/core/runner-instances.js +1 -0
- package/dist/core/runner-instances.js.map +1 -1
- package/dist/db/db.d.ts +68 -0
- package/dist/db/db.d.ts.map +1 -1
- package/dist/db/ephemeral-registration-tokens.d.ts.map +1 -1
- package/dist/db/ephemeral-registration-tokens.js +1 -1
- package/dist/db/ephemeral-registration-tokens.js.map +1 -1
- package/dist/db/job-executions.d.ts.map +1 -1
- package/dist/db/job-executions.js +35 -4
- package/dist/db/job-executions.js.map +1 -1
- package/dist/db/reservation-locks.d.ts +6 -0
- package/dist/db/reservation-locks.d.ts.map +1 -0
- package/dist/db/reservation-locks.js +15 -0
- package/dist/db/reservation-locks.js.map +1 -0
- package/dist/db/reservations.d.ts +17 -3
- package/dist/db/reservations.d.ts.map +1 -1
- package/dist/db/reservations.js +246 -51
- package/dist/db/reservations.js.map +1 -1
- package/dist/db/runner-assignments.d.ts +25 -1
- package/dist/db/runner-assignments.d.ts.map +1 -1
- package/dist/db/runner-assignments.js +160 -18
- package/dist/db/runner-assignments.js.map +1 -1
- package/dist/db/runner-instances.d.ts +11 -2
- package/dist/db/runner-instances.d.ts.map +1 -1
- package/dist/db/runner-instances.js +167 -24
- package/dist/db/runner-instances.js.map +1 -1
- package/dist/db/runner-sessions.d.ts.map +1 -1
- package/dist/db/runner-sessions.js +14 -1
- package/dist/db/runner-sessions.js.map +1 -1
- package/dist/db/runner-states.d.ts +2 -0
- package/dist/db/runner-states.d.ts.map +1 -0
- package/dist/db/runner-states.js +7 -0
- package/dist/db/runner-states.js.map +1 -0
- package/dist/db/schema/reservations.d.ts +18 -0
- package/dist/db/schema/reservations.d.ts.map +1 -1
- package/dist/db/schema/reservations.js +7 -1
- package/dist/db/schema/reservations.js.map +1 -1
- package/dist/db/schema/runner-instances.d.ts +18 -0
- package/dist/db/schema/runner-instances.d.ts.map +1 -1
- package/dist/db/schema/runner-instances.js +8 -0
- package/dist/db/schema/runner-instances.js.map +1 -1
- package/dist/metrics/index.d.ts +2 -2
- package/dist/metrics/index.d.ts.map +1 -1
- package/dist/metrics/index.js +1 -1
- package/dist/metrics/index.js.map +1 -1
- package/dist/metrics/instance.d.ts +72 -2
- package/dist/metrics/instance.d.ts.map +1 -1
- package/dist/metrics/instance.js +161 -1
- package/dist/metrics/instance.js.map +1 -1
- package/dist/metrics/service.d.ts.map +1 -1
- package/dist/metrics/service.js +25 -4
- package/dist/metrics/service.js.map +1 -1
- package/dist/presentation/dto/poll-demand.d.ts.map +1 -1
- package/dist/presentation/dto/poll-demand.js +3 -0
- package/dist/presentation/dto/poll-demand.js.map +1 -1
- package/dist/presentation/dto/runner-instances.d.ts +4 -1
- package/dist/presentation/dto/runner-instances.d.ts.map +1 -1
- package/dist/presentation/dto/runner-instances.js +4 -1
- package/dist/presentation/dto/runner-instances.js.map +1 -1
- package/dist/presentation/routes/assign-runner-instances.d.ts.map +1 -1
- package/dist/presentation/routes/assign-runner-instances.js +10 -4
- package/dist/presentation/routes/assign-runner-instances.js.map +1 -1
- package/dist/presentation/routes/poll-demand.d.ts.map +1 -1
- package/dist/presentation/routes/poll-demand.js +1 -0
- package/dist/presentation/routes/poll-demand.js.map +1 -1
- package/dist/presentation/routes/reconcile-runner-instances.d.ts.map +1 -1
- package/dist/presentation/routes/reconcile-runner-instances.js +4 -2
- package/dist/presentation/routes/reconcile-runner-instances.js.map +1 -1
- package/dist/presentation/routes/runner-enrollment.d.ts +4 -1
- package/dist/presentation/routes/runner-enrollment.d.ts.map +1 -1
- package/dist/presentation/routes/runner-enrollment.js +11 -6
- package/dist/presentation/routes/runner-enrollment.js.map +1 -1
- package/dist/tsconfig.test.tsbuildinfo +1 -1
- package/drizzle/0005_previous_lenny_balinger.sql +5 -0
- package/drizzle/meta/0005_snapshot.json +2220 -0
- package/drizzle/meta/_journal.json +7 -0
- package/package.json +6 -6
- package/src/config.test.ts +22 -0
- package/src/config.ts +15 -2
- package/src/core/demand.test.ts +192 -3
- package/src/core/demand.ts +44 -4
- package/src/core/entities/reservation.ts +1 -0
- package/src/core/entities/runner-instance.ts +4 -0
- package/src/core/errors.ts +30 -1
- package/src/core/runner-activation.ts +48 -2
- package/src/core/runner-control-sessions.test.ts +135 -3
- package/src/core/runner-control-sessions.ts +141 -19
- package/src/core/runner-instances.test.ts +19 -0
- package/src/core/runner-instances.ts +2 -0
- package/src/core/runner-sessions.test.ts +121 -1
- package/src/db/ephemeral-registration-tokens.test.ts +20 -1
- package/src/db/ephemeral-registration-tokens.ts +1 -0
- package/src/db/job-executions.test.ts +98 -12
- package/src/db/job-executions.ts +52 -4
- package/src/db/reservation-locks.ts +22 -0
- package/src/db/reservations.test.ts +733 -22
- package/src/db/reservations.ts +408 -144
- package/src/db/runner-assignments.test.ts +105 -1
- package/src/db/runner-assignments.ts +245 -25
- package/src/db/runner-instances.test.ts +357 -2
- package/src/db/runner-instances.ts +289 -55
- package/src/db/runner-sessions.ts +19 -1
- package/src/db/runner-states.ts +7 -0
- package/src/db/schema/reservations.ts +5 -1
- package/src/db/schema/runner-instances.ts +9 -0
- package/src/metrics/index.ts +24 -1
- package/src/metrics/instance.test.ts +118 -0
- package/src/metrics/instance.ts +247 -3
- package/src/metrics/service.test.ts +46 -1
- package/src/metrics/service.ts +55 -8
- package/src/presentation/dto/poll-demand.ts +3 -0
- package/src/presentation/dto/runner-instances.ts +8 -1
- package/src/presentation/routes/assign-runner-instances.ts +14 -2
- package/src/presentation/routes/late-runner-enrollment.test.ts +219 -2
- package/src/presentation/routes/poll-demand.test.ts +56 -0
- package/src/presentation/routes/poll-demand.ts +1 -0
- package/src/presentation/routes/reconcile-runner-instances.test.ts +34 -2
- package/src/presentation/routes/reconcile-runner-instances.ts +7 -1
- package/src/presentation/routes/runner-enrollment.test.ts +316 -3
- package/src/presentation/routes/runner-enrollment.ts +8 -5
- package/test/factories/reservation.ts +2 -0
- package/test/factories/runner-instance.ts +6 -0
- package/tsconfig.build.tsbuildinfo +1 -1
|
@@ -9,6 +9,7 @@ import {runnerControlSessions} from '#db/schema/runner-control-sessions.js';
|
|
|
9
9
|
import {providerRunners} from '#db/schema/runner-instances.js';
|
|
10
10
|
import {
|
|
11
11
|
type RunnerReservationPromotionFailureReason,
|
|
12
|
+
runnerActivationTokenNotIssuedCount,
|
|
12
13
|
runnerReservationPromotionFailureCount,
|
|
13
14
|
} from '#metrics/instance.js';
|
|
14
15
|
import {enrollRunnerControlSession} from './runner-control-sessions.js';
|
|
@@ -167,7 +168,7 @@ describe('enrollRunnerControlSession', () => {
|
|
|
167
168
|
});
|
|
168
169
|
});
|
|
169
170
|
|
|
170
|
-
it('
|
|
171
|
+
it('repairs assignment metadata when a provider report precedes assignment commit', async () => {
|
|
171
172
|
const provisionerId = crypto.randomUUID();
|
|
172
173
|
const reservation = await createReservation({provisionerId});
|
|
173
174
|
const capabilities: RunnerToolCapabilitiesDto = {
|
|
@@ -190,8 +191,10 @@ describe('enrollRunnerControlSession', () => {
|
|
|
190
191
|
});
|
|
191
192
|
const [runner] = await db()
|
|
192
193
|
.select({
|
|
194
|
+
workspaceId: providerRunners.workspaceId,
|
|
193
195
|
reservationId: providerRunners.reservationId,
|
|
194
196
|
assignedAt: providerRunners.assignedAt,
|
|
197
|
+
intendedReservationId: providerRunners.intendedReservationId,
|
|
195
198
|
labels: providerRunners.labels,
|
|
196
199
|
providerKind: providerRunners.providerKind,
|
|
197
200
|
protocolVersion: providerRunners.protocolVersion,
|
|
@@ -201,10 +204,12 @@ describe('enrollRunnerControlSession', () => {
|
|
|
201
204
|
.from(providerRunners)
|
|
202
205
|
.where(eq(providerRunners.id, runnerInstanceId));
|
|
203
206
|
|
|
204
|
-
expect(activationToken).
|
|
207
|
+
expect(activationToken).toEqual(expect.any(String));
|
|
205
208
|
expect(runner).toEqual({
|
|
209
|
+
workspaceId: reservation.workspaceId,
|
|
206
210
|
reservationId: reservation.id,
|
|
207
|
-
assignedAt:
|
|
211
|
+
assignedAt: expect.any(Date),
|
|
212
|
+
intendedReservationId: null,
|
|
208
213
|
labels: ['linux'],
|
|
209
214
|
providerKind: 'ec2',
|
|
210
215
|
protocolVersion: '1',
|
|
@@ -213,6 +218,62 @@ describe('enrollRunnerControlSession', () => {
|
|
|
213
218
|
});
|
|
214
219
|
});
|
|
215
220
|
|
|
221
|
+
it('records an activation-token reason when enrollment sees an existing runner session', async () => {
|
|
222
|
+
const provisionerId = crypto.randomUUID();
|
|
223
|
+
const reservation = await createReservation({provisionerId});
|
|
224
|
+
const runnerInstanceId = await createRunner({
|
|
225
|
+
provisionerId,
|
|
226
|
+
intendedReservationId: reservation.id,
|
|
227
|
+
});
|
|
228
|
+
await db()
|
|
229
|
+
.update(providerRunners)
|
|
230
|
+
.set({runnerSessionId: crypto.randomUUID()})
|
|
231
|
+
.where(eq(providerRunners.id, runnerInstanceId));
|
|
232
|
+
const addSpy = vi.spyOn(runnerActivationTokenNotIssuedCount, 'add');
|
|
233
|
+
|
|
234
|
+
try {
|
|
235
|
+
const activationToken = await enrollRunnerControlSession({
|
|
236
|
+
runnerInstanceId,
|
|
237
|
+
provisionerId,
|
|
238
|
+
labels: ['linux'],
|
|
239
|
+
providerKind: 'docker',
|
|
240
|
+
protocolVersion: '1',
|
|
241
|
+
});
|
|
242
|
+
const [runner] = await db()
|
|
243
|
+
.select({
|
|
244
|
+
workspaceId: providerRunners.workspaceId,
|
|
245
|
+
reservationId: providerRunners.reservationId,
|
|
246
|
+
assignedAt: providerRunners.assignedAt,
|
|
247
|
+
intendedReservationId: providerRunners.intendedReservationId,
|
|
248
|
+
runnerSessionId: providerRunners.runnerSessionId,
|
|
249
|
+
state: providerRunners.state,
|
|
250
|
+
})
|
|
251
|
+
.from(providerRunners)
|
|
252
|
+
.where(eq(providerRunners.id, runnerInstanceId));
|
|
253
|
+
|
|
254
|
+
expect(activationToken).toBeNull();
|
|
255
|
+
expect(runner).toEqual({
|
|
256
|
+
workspaceId: reservation.workspaceId,
|
|
257
|
+
reservationId: reservation.id,
|
|
258
|
+
assignedAt: expect.any(Date),
|
|
259
|
+
intendedReservationId: null,
|
|
260
|
+
runnerSessionId: expect.any(String),
|
|
261
|
+
state: 'running',
|
|
262
|
+
});
|
|
263
|
+
// The test setup uses a shared NoopMeter counter; count only this metric's label shape.
|
|
264
|
+
const activationTokenCalls = addSpy.mock.calls.filter(
|
|
265
|
+
([value, attributes]) =>
|
|
266
|
+
value === 1 &&
|
|
267
|
+
(attributes as {reason?: string; surface?: string} | undefined)?.reason ===
|
|
268
|
+
'existing-session' &&
|
|
269
|
+
(attributes as {reason?: string; surface?: string} | undefined)?.surface === 'enrollment',
|
|
270
|
+
);
|
|
271
|
+
expect(activationTokenCalls).toHaveLength(1);
|
|
272
|
+
} finally {
|
|
273
|
+
addSpy.mockRestore();
|
|
274
|
+
}
|
|
275
|
+
});
|
|
276
|
+
|
|
216
277
|
it('updates metadata during enrollment for an unassigned runner', async () => {
|
|
217
278
|
const provisionerId = crypto.randomUUID();
|
|
218
279
|
const capabilities: RunnerToolCapabilitiesDto = {
|
|
@@ -298,6 +359,77 @@ describe('enrollRunnerControlSession', () => {
|
|
|
298
359
|
vi.resetModules();
|
|
299
360
|
}
|
|
300
361
|
});
|
|
362
|
+
|
|
363
|
+
it('records a rolled-back assignment rejection without recording its assignment timing', async () => {
|
|
364
|
+
const provisionerId = crypto.randomUUID();
|
|
365
|
+
const reservation = await createReservation({provisionerId});
|
|
366
|
+
const runnerInstanceId = await createRunner({
|
|
367
|
+
provisionerId,
|
|
368
|
+
intendedReservationId: reservation.id,
|
|
369
|
+
});
|
|
370
|
+
const assignmentTimingRecord = vi.fn();
|
|
371
|
+
const assignmentRejectionRecord = vi.fn();
|
|
372
|
+
const promotionFailureRecord = vi.fn();
|
|
373
|
+
|
|
374
|
+
vi.resetModules();
|
|
375
|
+
const {RunnerInstanceNotAssignableError} = await import('#core/errors.js');
|
|
376
|
+
const activationIssueError = new RunnerInstanceNotAssignableError(
|
|
377
|
+
runnerInstanceId,
|
|
378
|
+
'capacity-exhausted',
|
|
379
|
+
);
|
|
380
|
+
vi.doMock('#metrics/index.js', async (importOriginal) => ({
|
|
381
|
+
...(await importOriginal<typeof import('#metrics/index.js')>()),
|
|
382
|
+
recordProviderRunnerControlSessionToAssignment: assignmentTimingRecord,
|
|
383
|
+
recordProviderRunnerAssignmentRejected: assignmentRejectionRecord,
|
|
384
|
+
recordRunnerReservationPromotionFailure: promotionFailureRecord,
|
|
385
|
+
}));
|
|
386
|
+
vi.doMock('./runner-activation.js', async (importOriginal) => ({
|
|
387
|
+
...(await importOriginal<typeof import('./runner-activation.js')>()),
|
|
388
|
+
issueRunnerActivationTokenTx: vi.fn().mockRejectedValue(activationIssueError),
|
|
389
|
+
}));
|
|
390
|
+
|
|
391
|
+
const {closePostgresClient, createPostgresClient} = await import('@shipfox/node-postgres');
|
|
392
|
+
createPostgresClient();
|
|
393
|
+
|
|
394
|
+
try {
|
|
395
|
+
const {enrollRunnerControlSession: enroll} = await import('./runner-control-sessions.js');
|
|
396
|
+
const result = await enroll({
|
|
397
|
+
runnerInstanceId,
|
|
398
|
+
provisionerId,
|
|
399
|
+
labels: ['linux'],
|
|
400
|
+
providerKind: 'docker',
|
|
401
|
+
protocolVersion: '1',
|
|
402
|
+
});
|
|
403
|
+
const [runner] = await db()
|
|
404
|
+
.select({
|
|
405
|
+
intendedReservationId: providerRunners.intendedReservationId,
|
|
406
|
+
reservationId: providerRunners.reservationId,
|
|
407
|
+
workspaceId: providerRunners.workspaceId,
|
|
408
|
+
assignedAt: providerRunners.assignedAt,
|
|
409
|
+
})
|
|
410
|
+
.from(providerRunners)
|
|
411
|
+
.where(eq(providerRunners.id, runnerInstanceId));
|
|
412
|
+
|
|
413
|
+
expect(result).toBeNull();
|
|
414
|
+
expect(runner).toEqual({
|
|
415
|
+
intendedReservationId: reservation.id,
|
|
416
|
+
reservationId: null,
|
|
417
|
+
workspaceId: null,
|
|
418
|
+
assignedAt: null,
|
|
419
|
+
});
|
|
420
|
+
expect(assignmentTimingRecord).not.toHaveBeenCalled();
|
|
421
|
+
expect(assignmentRejectionRecord).toHaveBeenCalledWith({
|
|
422
|
+
reason: 'capacity-exhausted',
|
|
423
|
+
surface: 'enrollment',
|
|
424
|
+
});
|
|
425
|
+
expect(promotionFailureRecord).toHaveBeenCalledWith('not-assignable');
|
|
426
|
+
} finally {
|
|
427
|
+
await closePostgresClient();
|
|
428
|
+
vi.doUnmock('#metrics/index.js');
|
|
429
|
+
vi.doUnmock('./runner-activation.js');
|
|
430
|
+
vi.resetModules();
|
|
431
|
+
}
|
|
432
|
+
});
|
|
301
433
|
});
|
|
302
434
|
|
|
303
435
|
async function expectPromotionFailure(params: {
|
|
@@ -5,6 +5,7 @@ import {extractDisplayPrefix, generateOpaqueToken, hashOpaqueToken} from '@shipf
|
|
|
5
5
|
import {and, eq, gt, isNull, notInArray, or, sql} from 'drizzle-orm';
|
|
6
6
|
import {config} from '#config.js';
|
|
7
7
|
import {
|
|
8
|
+
getRunnerAssignmentRejectionReason,
|
|
8
9
|
ReservationExpiredError,
|
|
9
10
|
ReservationNotFoundError,
|
|
10
11
|
RunnerInstanceAlreadyAssignedError,
|
|
@@ -12,13 +13,20 @@ import {
|
|
|
12
13
|
} from '#core/errors.js';
|
|
13
14
|
import {sanitizeRunnerLabels} from '#core/runner-labels.js';
|
|
14
15
|
import {db, type schema, type Tx} from '#db/db.js';
|
|
15
|
-
import {
|
|
16
|
-
|
|
16
|
+
import {
|
|
17
|
+
assignRunnerInstancesTx,
|
|
18
|
+
validateRunnerReservationCapacityTx,
|
|
19
|
+
} from '#db/runner-assignments.js';
|
|
20
|
+
import {terminalStates} from '#db/runner-states.js';
|
|
17
21
|
import {provisionerTokens} from '#db/schema/provisioner-tokens.js';
|
|
18
22
|
import {runnerBootstrapTokens, runnerControlSessions} from '#db/schema/runner-control-sessions.js';
|
|
19
23
|
import {providerRunners} from '#db/schema/runner-instances.js';
|
|
20
24
|
import {
|
|
21
25
|
type RunnerReservationPromotionFailureReason,
|
|
26
|
+
recordProviderRunnerAssignmentRejected,
|
|
27
|
+
recordProviderRunnerControlSessionToAssignment,
|
|
28
|
+
recordProviderRunnerCreatedToControlSession,
|
|
29
|
+
recordRunnerReservationCapacityFailure,
|
|
22
30
|
recordRunnerReservationPromotionFailure,
|
|
23
31
|
} from '#metrics/index.js';
|
|
24
32
|
import {issueRunnerActivationTokenTx} from './runner-activation.js';
|
|
@@ -45,27 +53,72 @@ export async function createRunnerInstancesWithBootstrapTokens(params: {
|
|
|
45
53
|
reservationId?: string | null;
|
|
46
54
|
}>;
|
|
47
55
|
ttlSeconds: number;
|
|
48
|
-
}): Promise<
|
|
56
|
+
}): Promise<{
|
|
57
|
+
runnerInstances: Array<{
|
|
58
|
+
runnerInstanceId: string;
|
|
59
|
+
bootstrapToken: string;
|
|
60
|
+
requestIndex: number;
|
|
61
|
+
}>;
|
|
62
|
+
reservationUnavailable: boolean;
|
|
63
|
+
}> {
|
|
49
64
|
const expiresAt = new Date(Date.now() + params.ttlSeconds * 1000);
|
|
50
65
|
return await db().transaction(async (tx) => {
|
|
66
|
+
const reservationValidation = await validateRunnerReservationCapacityTx(tx, {
|
|
67
|
+
provisionerId: params.provisionerId,
|
|
68
|
+
requests: params.runnerInstances.reduce(
|
|
69
|
+
(requests, runner) => {
|
|
70
|
+
if (!runner.reservationId) return requests;
|
|
71
|
+
const request = requests.find(
|
|
72
|
+
(candidate) => candidate.reservationId === runner.reservationId,
|
|
73
|
+
);
|
|
74
|
+
if (request) request.count += 1;
|
|
75
|
+
else requests.push({reservationId: runner.reservationId, count: 1});
|
|
76
|
+
return requests;
|
|
77
|
+
},
|
|
78
|
+
[] as Array<{reservationId: string; count: number}>,
|
|
79
|
+
),
|
|
80
|
+
});
|
|
81
|
+
for (const {reason, count} of reservationValidation.unavailableByReservation.values())
|
|
82
|
+
recordRunnerReservationCapacityFailure(reason, count);
|
|
83
|
+
const remainingAcceptedByReservation = new Map(reservationValidation.acceptedByReservation);
|
|
84
|
+
const runnerInstances = params.runnerInstances.flatMap((runner, requestIndex) => {
|
|
85
|
+
if (!runner.reservationId) return [{...runner, requestIndex}];
|
|
86
|
+
const remaining = remainingAcceptedByReservation.get(runner.reservationId) ?? 0;
|
|
87
|
+
if (remaining === 0) return [];
|
|
88
|
+
remainingAcceptedByReservation.set(runner.reservationId, remaining - 1);
|
|
89
|
+
return [{...runner, requestIndex}];
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
if (runnerInstances.length === 0)
|
|
93
|
+
return {
|
|
94
|
+
runnerInstances: [],
|
|
95
|
+
reservationUnavailable: reservationValidation.unavailableByReservation.size > 0,
|
|
96
|
+
};
|
|
97
|
+
|
|
51
98
|
const instances = await tx
|
|
52
99
|
.insert(providerRunners)
|
|
53
100
|
.values(
|
|
54
|
-
|
|
101
|
+
runnerInstances.map((runner) => ({
|
|
55
102
|
provisionerId: params.provisionerId,
|
|
56
103
|
providerKind: params.providerKind ?? null,
|
|
57
104
|
templateKey: runner.templateKey ?? null,
|
|
58
105
|
intendedReservationId: runner.reservationId ?? null,
|
|
106
|
+
launchKind: runner.reservationId ? ('demand' as const) : ('warm' as const),
|
|
59
107
|
state: 'starting' as const,
|
|
60
108
|
labels: [],
|
|
61
109
|
reportedAt: new Date(),
|
|
62
110
|
})),
|
|
63
111
|
)
|
|
64
112
|
.returning({id: providerRunners.id});
|
|
65
|
-
const results = instances.map((instance) =>
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
113
|
+
const results = instances.map((instance, index) => {
|
|
114
|
+
const runner = runnerInstances[index];
|
|
115
|
+
if (!runner) throw new Error('Runner instance insert returned an unexpected row count');
|
|
116
|
+
return {
|
|
117
|
+
runnerInstanceId: instance.id,
|
|
118
|
+
bootstrapToken: generateOpaqueToken('runnerBootstrapToken'),
|
|
119
|
+
requestIndex: runner.requestIndex,
|
|
120
|
+
};
|
|
121
|
+
});
|
|
69
122
|
await tx.insert(runnerBootstrapTokens).values(
|
|
70
123
|
results.map((result) => ({
|
|
71
124
|
runnerInstanceId: result.runnerInstanceId,
|
|
@@ -75,7 +128,10 @@ export async function createRunnerInstancesWithBootstrapTokens(params: {
|
|
|
75
128
|
expiresAt,
|
|
76
129
|
})),
|
|
77
130
|
);
|
|
78
|
-
return
|
|
131
|
+
return {
|
|
132
|
+
runnerInstances: results,
|
|
133
|
+
reservationUnavailable: reservationValidation.unavailableByReservation.size > 0,
|
|
134
|
+
};
|
|
79
135
|
});
|
|
80
136
|
}
|
|
81
137
|
|
|
@@ -85,7 +141,7 @@ export async function exchangeRunnerBootstrapToken(params: {
|
|
|
85
141
|
}): Promise<{runnerInstanceId: string; controlSessionToken: string; expiresAt: Date}> {
|
|
86
142
|
const controlSessionToken = generateOpaqueToken('runnerControlSession');
|
|
87
143
|
const expiresAt = new Date(Date.now() + params.ttlSeconds * 1000);
|
|
88
|
-
|
|
144
|
+
const result = await db().transaction(async (tx) => {
|
|
89
145
|
const [bootstrap] = await tx
|
|
90
146
|
.update(runnerBootstrapTokens)
|
|
91
147
|
.set({consumedAt: sql`now()`})
|
|
@@ -99,6 +155,21 @@ export async function exchangeRunnerBootstrapToken(params: {
|
|
|
99
155
|
)
|
|
100
156
|
.returning();
|
|
101
157
|
if (!bootstrap) throw new RunnerBootstrapTokenInvalidError();
|
|
158
|
+
const [runner] = await tx
|
|
159
|
+
.select({
|
|
160
|
+
runnerInstanceId: providerRunners.id,
|
|
161
|
+
createdAt: providerRunners.createdAt,
|
|
162
|
+
provider: providerRunners.providerKind,
|
|
163
|
+
launchKind: providerRunners.launchKind,
|
|
164
|
+
})
|
|
165
|
+
.from(providerRunners)
|
|
166
|
+
.where(
|
|
167
|
+
and(
|
|
168
|
+
eq(providerRunners.id, bootstrap.runnerInstanceId),
|
|
169
|
+
eq(providerRunners.provisionerId, bootstrap.provisionerId),
|
|
170
|
+
),
|
|
171
|
+
)
|
|
172
|
+
.limit(1);
|
|
102
173
|
const [session] = await tx
|
|
103
174
|
.insert(runnerControlSessions)
|
|
104
175
|
.values({
|
|
@@ -108,10 +179,28 @@ export async function exchangeRunnerBootstrapToken(params: {
|
|
|
108
179
|
prefix: extractDisplayPrefix(controlSessionToken),
|
|
109
180
|
expiresAt,
|
|
110
181
|
})
|
|
111
|
-
.returning({
|
|
182
|
+
.returning({createdAt: runnerControlSessions.createdAt});
|
|
112
183
|
if (!session) throw new Error('Runner control session insert returned no row');
|
|
113
|
-
return {
|
|
184
|
+
return {
|
|
185
|
+
runner,
|
|
186
|
+
session,
|
|
187
|
+
runnerInstanceId: bootstrap.runnerInstanceId,
|
|
188
|
+
controlSessionToken,
|
|
189
|
+
expiresAt,
|
|
190
|
+
};
|
|
114
191
|
});
|
|
192
|
+
if (result.runner)
|
|
193
|
+
recordProviderRunnerCreatedToControlSession({
|
|
194
|
+
durationMilliseconds: result.session.createdAt.getTime() - result.runner.createdAt.getTime(),
|
|
195
|
+
provider: result.runner.provider,
|
|
196
|
+
launchKind: result.runner.launchKind,
|
|
197
|
+
runnerInstanceId: result.runner.runnerInstanceId,
|
|
198
|
+
});
|
|
199
|
+
return {
|
|
200
|
+
runnerInstanceId: result.runnerInstanceId,
|
|
201
|
+
controlSessionToken: result.controlSessionToken,
|
|
202
|
+
expiresAt: result.expiresAt,
|
|
203
|
+
};
|
|
115
204
|
}
|
|
116
205
|
|
|
117
206
|
export async function resolveRunnerControlSession(rawToken: string) {
|
|
@@ -143,7 +232,7 @@ export async function enrollRunnerControlSession(params: {
|
|
|
143
232
|
providerKind: string;
|
|
144
233
|
protocolVersion: string;
|
|
145
234
|
}): Promise<string | null> {
|
|
146
|
-
|
|
235
|
+
const result = await db().transaction(async (tx) => {
|
|
147
236
|
const [current] = await tx
|
|
148
237
|
.select({
|
|
149
238
|
intendedReservationId: providerRunners.intendedReservationId,
|
|
@@ -214,22 +303,38 @@ export async function enrollRunnerControlSession(params: {
|
|
|
214
303
|
await updateRunnerControlSessionLastSeen(tx, params.runnerInstanceId, params.provisionerId);
|
|
215
304
|
|
|
216
305
|
const intendedReservationId = updated.intendedReservationId;
|
|
217
|
-
if (!intendedReservationId)
|
|
306
|
+
if (!intendedReservationId)
|
|
307
|
+
return {
|
|
308
|
+
activationToken: null,
|
|
309
|
+
controlSessionToAssignment: [],
|
|
310
|
+
assignmentRejectedReason: null,
|
|
311
|
+
promotionFailureReason: null,
|
|
312
|
+
};
|
|
218
313
|
|
|
219
314
|
try {
|
|
220
|
-
|
|
221
|
-
await assignRunnerInstancesTx(promotionTx, {
|
|
315
|
+
const promotion = await tx.transaction(async (promotionTx) => {
|
|
316
|
+
const assignment = await assignRunnerInstancesTx(promotionTx, {
|
|
222
317
|
provisionerId: params.provisionerId,
|
|
223
318
|
reservationId: intendedReservationId,
|
|
224
319
|
runnerInstanceIds: [params.runnerInstanceId],
|
|
320
|
+
surface: 'enrollment',
|
|
225
321
|
});
|
|
226
|
-
|
|
322
|
+
const activationToken = await issueRunnerActivationTokenTx(promotionTx, {
|
|
227
323
|
runnerInstanceId: params.runnerInstanceId,
|
|
228
324
|
provisionerId: params.provisionerId,
|
|
229
325
|
ttlSeconds: config.RUNNER_ACTIVATION_TOKEN_TTL_SECONDS,
|
|
326
|
+
surface: 'enrollment',
|
|
230
327
|
});
|
|
328
|
+
return {activationToken, controlSessionToAssignment: assignment.controlSessionToAssignment};
|
|
231
329
|
});
|
|
330
|
+
return {
|
|
331
|
+
activationToken: promotion.activationToken,
|
|
332
|
+
controlSessionToAssignment: promotion.controlSessionToAssignment,
|
|
333
|
+
assignmentRejectedReason: null,
|
|
334
|
+
promotionFailureReason: null,
|
|
335
|
+
};
|
|
232
336
|
} catch (error) {
|
|
337
|
+
const assignmentRejectionReason = getRunnerAssignmentRejectionReason(error);
|
|
233
338
|
const expectedFailureReason = getRunnerReservationPromotionFailureReason(error);
|
|
234
339
|
const details = {
|
|
235
340
|
err: error,
|
|
@@ -238,7 +343,6 @@ export async function enrollRunnerControlSession(params: {
|
|
|
238
343
|
reservationId: intendedReservationId,
|
|
239
344
|
};
|
|
240
345
|
if (expectedFailureReason) {
|
|
241
|
-
recordRunnerReservationPromotionFailure(expectedFailureReason);
|
|
242
346
|
logger().debug(details, 'Runner reservation could not be promoted during enrollment');
|
|
243
347
|
} else {
|
|
244
348
|
logger().error(
|
|
@@ -246,9 +350,27 @@ export async function enrollRunnerControlSession(params: {
|
|
|
246
350
|
'Unexpected failure promoting runner reservation during enrollment',
|
|
247
351
|
);
|
|
248
352
|
}
|
|
249
|
-
return
|
|
353
|
+
return {
|
|
354
|
+
activationToken: null,
|
|
355
|
+
controlSessionToAssignment: [],
|
|
356
|
+
assignmentRejectedReason: assignmentRejectionReason,
|
|
357
|
+
promotionFailureReason: expectedFailureReason,
|
|
358
|
+
};
|
|
250
359
|
}
|
|
251
360
|
});
|
|
361
|
+
for (const observation of result.controlSessionToAssignment)
|
|
362
|
+
recordProviderRunnerControlSessionToAssignment(observation);
|
|
363
|
+
// These counters intentionally overlap: assignment_rejected records the assignment-facing
|
|
364
|
+
// reason, while reservation_promotion_failure records the enrollment recovery outcome. They
|
|
365
|
+
// answer different questions and must not be summed.
|
|
366
|
+
if (result.assignmentRejectedReason)
|
|
367
|
+
recordProviderRunnerAssignmentRejected({
|
|
368
|
+
reason: result.assignmentRejectedReason,
|
|
369
|
+
surface: 'enrollment',
|
|
370
|
+
});
|
|
371
|
+
if (result.promotionFailureReason)
|
|
372
|
+
recordRunnerReservationPromotionFailure(result.promotionFailureReason);
|
|
373
|
+
return result.activationToken;
|
|
252
374
|
}
|
|
253
375
|
|
|
254
376
|
function getRunnerReservationPromotionFailureReason(
|
|
@@ -54,18 +54,37 @@ describe('reconcileRunnerInstancesFromDbResult', () => {
|
|
|
54
54
|
|
|
55
55
|
expect(result[0]?.desiredIntent).toBe('terminate');
|
|
56
56
|
});
|
|
57
|
+
|
|
58
|
+
it('returns the intended reservation when a runner is not assigned yet', () => {
|
|
59
|
+
const intendedReservationId = crypto.randomUUID();
|
|
60
|
+
const result = reconcileRunnerInstancesFromDbResult({
|
|
61
|
+
observedRunnerInstanceIds: ['provisioned-runner-1'],
|
|
62
|
+
observedRows: [
|
|
63
|
+
providerRunner({providerRunnerId: 'provisioned-runner-1', intendedReservationId}),
|
|
64
|
+
],
|
|
65
|
+
boundJobExecutionsByRunnerInstanceId: new Map(),
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
expect(result[0]).toMatchObject({
|
|
69
|
+
reservationId: null,
|
|
70
|
+
intendedReservationId,
|
|
71
|
+
});
|
|
72
|
+
});
|
|
57
73
|
});
|
|
58
74
|
|
|
59
75
|
function providerRunner(params: {
|
|
60
76
|
providerRunnerId: string;
|
|
61
77
|
state?: 'starting' | 'running' | 'stopping' | 'stopped' | 'failed' | 'terminated';
|
|
78
|
+
intendedReservationId?: string | null;
|
|
62
79
|
}) {
|
|
63
80
|
return {
|
|
64
81
|
id: crypto.randomUUID(),
|
|
65
82
|
workspaceId: crypto.randomUUID(),
|
|
66
83
|
provisionerId: crypto.randomUUID(),
|
|
67
84
|
providerRunnerId: params.providerRunnerId,
|
|
85
|
+
intendedReservationId: params.intendedReservationId ?? null,
|
|
68
86
|
reservationId: null,
|
|
87
|
+
launchKind: 'manual' as const,
|
|
69
88
|
templateKey: 'linux',
|
|
70
89
|
labels: ['linux'],
|
|
71
90
|
state: params.state ?? 'running',
|
|
@@ -62,6 +62,7 @@ export interface ReconciledBoundJobExecution {
|
|
|
62
62
|
export interface ReconciledRunnerInstance {
|
|
63
63
|
providerRunnerId: string;
|
|
64
64
|
state: RunnerInstanceState | null;
|
|
65
|
+
intendedReservationId: string | null;
|
|
65
66
|
reservationId: string | null;
|
|
66
67
|
runnerSessionId: string | null;
|
|
67
68
|
boundJobExecution: ReconciledBoundJobExecution | null;
|
|
@@ -157,6 +158,7 @@ export function reconcileRunnerInstancesFromDbResult(params: {
|
|
|
157
158
|
return {
|
|
158
159
|
providerRunnerId,
|
|
159
160
|
state: row?.state ?? null,
|
|
161
|
+
intendedReservationId: row?.intendedReservationId ?? null,
|
|
160
162
|
reservationId: row?.reservationId ?? null,
|
|
161
163
|
runnerSessionId: row?.runnerSessionId ?? null,
|
|
162
164
|
boundJobExecution: boundJobExecution
|
|
@@ -1,9 +1,15 @@
|
|
|
1
|
+
import {vi} from '@shipfox/vitest/vi';
|
|
1
2
|
import {and, eq} from 'drizzle-orm';
|
|
2
3
|
import {db} from '#db/db.js';
|
|
3
4
|
import {ephemeralRegistrationTokens} from '#db/schema/ephemeral-registration-tokens.js';
|
|
4
5
|
import {runnerActivationTokens} from '#db/schema/runner-activation-tokens.js';
|
|
6
|
+
import type {RunnerInstanceInsertDb} from '#db/schema/runner-instances.js';
|
|
5
7
|
import {providerRunners} from '#db/schema/runner-instances.js';
|
|
6
8
|
import {runnerSessions} from '#db/schema/runner-sessions.js';
|
|
9
|
+
import {
|
|
10
|
+
type RunnerActivationTokenNotIssuedReason,
|
|
11
|
+
runnerActivationTokenNotIssuedCount,
|
|
12
|
+
} from '#metrics/instance.js';
|
|
7
13
|
import {
|
|
8
14
|
ephemeralRegistrationTokenFactory,
|
|
9
15
|
getRunnerSessionTokenClaims,
|
|
@@ -18,9 +24,34 @@ import {
|
|
|
18
24
|
RegistrationTokenWorkspaceMismatchError,
|
|
19
25
|
RunnerLabelsReservedError,
|
|
20
26
|
} from './errors.js';
|
|
21
|
-
import {issueRunnerActivationToken} from './runner-activation.js';
|
|
27
|
+
import {getRunnerAssignment, issueRunnerActivationToken} from './runner-activation.js';
|
|
22
28
|
import {registerRunnerSession} from './runner-sessions.js';
|
|
23
29
|
|
|
30
|
+
const activationTokenNotIssuedCases: Array<{
|
|
31
|
+
reason: RunnerActivationTokenNotIssuedReason;
|
|
32
|
+
update: Partial<RunnerInstanceInsertDb>;
|
|
33
|
+
provisionerId?: string;
|
|
34
|
+
}> = [
|
|
35
|
+
{reason: 'runner-not-found', update: {}, provisionerId: crypto.randomUUID()},
|
|
36
|
+
{reason: 'missing-workspace', update: {workspaceId: null}},
|
|
37
|
+
{reason: 'existing-session', update: {runnerSessionId: crypto.randomUUID()}},
|
|
38
|
+
{reason: 'not-running', update: {state: 'starting'}},
|
|
39
|
+
];
|
|
40
|
+
|
|
41
|
+
function activationTokenMetricCalls(spy: {mock: {calls: unknown[][]}}): unknown[][] {
|
|
42
|
+
// The test setup uses a shared NoopMeter counter, so filter calls to this metric before counting.
|
|
43
|
+
return spy.mock.calls.filter(([, attributes]) => {
|
|
44
|
+
if (typeof attributes !== 'object' || attributes === null) return false;
|
|
45
|
+
const reason = (attributes as {reason?: unknown}).reason;
|
|
46
|
+
const surface = (attributes as {surface?: unknown}).surface;
|
|
47
|
+
return (
|
|
48
|
+
['runner-not-found', 'missing-workspace', 'existing-session', 'not-running'].includes(
|
|
49
|
+
String(reason),
|
|
50
|
+
) && ['enrollment', 'poll'].includes(String(surface))
|
|
51
|
+
);
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
|
|
24
55
|
describe('registerRunnerSession', () => {
|
|
25
56
|
let workspaceId: string;
|
|
26
57
|
let registrationTokenId: string;
|
|
@@ -245,11 +276,13 @@ describe('activation runner sessions', () => {
|
|
|
245
276
|
runnerInstanceId,
|
|
246
277
|
provisionerId,
|
|
247
278
|
ttlSeconds: 60,
|
|
279
|
+
surface: 'poll',
|
|
248
280
|
});
|
|
249
281
|
const replacementToken = await issueRunnerActivationToken({
|
|
250
282
|
runnerInstanceId,
|
|
251
283
|
provisionerId,
|
|
252
284
|
ttlSeconds: 60,
|
|
285
|
+
surface: 'poll',
|
|
253
286
|
});
|
|
254
287
|
|
|
255
288
|
const tokens = await db()
|
|
@@ -265,11 +298,97 @@ describe('activation runner sessions', () => {
|
|
|
265
298
|
expect(tokens.filter((token) => token.revokedAt !== null)).toHaveLength(1);
|
|
266
299
|
});
|
|
267
300
|
|
|
301
|
+
it.each(
|
|
302
|
+
activationTokenNotIssuedCases,
|
|
303
|
+
)('records $reason when direct activation-token issuance is skipped', async ({
|
|
304
|
+
reason,
|
|
305
|
+
update,
|
|
306
|
+
provisionerId: caseProvisionerId,
|
|
307
|
+
}) => {
|
|
308
|
+
const addSpy = vi.spyOn(runnerActivationTokenNotIssuedCount, 'add');
|
|
309
|
+
|
|
310
|
+
try {
|
|
311
|
+
if (Object.keys(update).length > 0) {
|
|
312
|
+
await db()
|
|
313
|
+
.update(providerRunners)
|
|
314
|
+
.set(update)
|
|
315
|
+
.where(eq(providerRunners.id, runnerInstanceId));
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
const activationToken = await issueRunnerActivationToken({
|
|
319
|
+
runnerInstanceId,
|
|
320
|
+
provisionerId: caseProvisionerId ?? provisionerId,
|
|
321
|
+
ttlSeconds: 60,
|
|
322
|
+
surface: 'poll',
|
|
323
|
+
});
|
|
324
|
+
|
|
325
|
+
expect(activationToken).toBeNull();
|
|
326
|
+
expect(activationTokenMetricCalls(addSpy)).toEqual([[1, {reason, surface: 'poll'}]]);
|
|
327
|
+
} finally {
|
|
328
|
+
addSpy.mockRestore();
|
|
329
|
+
}
|
|
330
|
+
});
|
|
331
|
+
|
|
332
|
+
it('does not record a metric when assignment polling finds no assignment', async () => {
|
|
333
|
+
const addSpy = vi.spyOn(runnerActivationTokenNotIssuedCount, 'add');
|
|
334
|
+
|
|
335
|
+
try {
|
|
336
|
+
await db()
|
|
337
|
+
.update(providerRunners)
|
|
338
|
+
.set({workspaceId: null})
|
|
339
|
+
.where(eq(providerRunners.id, runnerInstanceId));
|
|
340
|
+
|
|
341
|
+
const assignment = await getRunnerAssignment({runnerInstanceId, provisionerId});
|
|
342
|
+
|
|
343
|
+
expect(assignment).toBeNull();
|
|
344
|
+
expect(activationTokenMetricCalls(addSpy)).toHaveLength(0);
|
|
345
|
+
} finally {
|
|
346
|
+
addSpy.mockRestore();
|
|
347
|
+
}
|
|
348
|
+
});
|
|
349
|
+
|
|
350
|
+
it('does not poll for an assignment while the runner is not running', async () => {
|
|
351
|
+
const addSpy = vi.spyOn(runnerActivationTokenNotIssuedCount, 'add');
|
|
352
|
+
|
|
353
|
+
try {
|
|
354
|
+
await db()
|
|
355
|
+
.update(providerRunners)
|
|
356
|
+
.set({state: 'starting'})
|
|
357
|
+
.where(eq(providerRunners.id, runnerInstanceId));
|
|
358
|
+
|
|
359
|
+
const assignment = await getRunnerAssignment({runnerInstanceId, provisionerId});
|
|
360
|
+
|
|
361
|
+
expect(assignment).toBeNull();
|
|
362
|
+
expect(activationTokenMetricCalls(addSpy)).toHaveLength(0);
|
|
363
|
+
} finally {
|
|
364
|
+
addSpy.mockRestore();
|
|
365
|
+
}
|
|
366
|
+
});
|
|
367
|
+
|
|
368
|
+
it('does not record a metric when activation-token issuance succeeds', async () => {
|
|
369
|
+
const addSpy = vi.spyOn(runnerActivationTokenNotIssuedCount, 'add');
|
|
370
|
+
|
|
371
|
+
try {
|
|
372
|
+
const activationToken = await issueRunnerActivationToken({
|
|
373
|
+
runnerInstanceId,
|
|
374
|
+
provisionerId,
|
|
375
|
+
ttlSeconds: 60,
|
|
376
|
+
surface: 'poll',
|
|
377
|
+
});
|
|
378
|
+
|
|
379
|
+
expect(activationToken).toEqual(expect.any(String));
|
|
380
|
+
expect(activationTokenMetricCalls(addSpy)).toHaveLength(0);
|
|
381
|
+
} finally {
|
|
382
|
+
addSpy.mockRestore();
|
|
383
|
+
}
|
|
384
|
+
});
|
|
385
|
+
|
|
268
386
|
it('allows only one concurrent registration to consume an activation token', async () => {
|
|
269
387
|
const rawToken = await issueRunnerActivationToken({
|
|
270
388
|
runnerInstanceId,
|
|
271
389
|
provisionerId,
|
|
272
390
|
ttlSeconds: 60,
|
|
391
|
+
surface: 'poll',
|
|
273
392
|
});
|
|
274
393
|
const [activationToken] = await db()
|
|
275
394
|
.select()
|
|
@@ -314,6 +433,7 @@ describe('activation runner sessions', () => {
|
|
|
314
433
|
runnerInstanceId,
|
|
315
434
|
provisionerId,
|
|
316
435
|
ttlSeconds: 60,
|
|
436
|
+
surface: 'poll',
|
|
317
437
|
});
|
|
318
438
|
const [activationToken] = await db()
|
|
319
439
|
.select()
|