@shipfox/api-runners 12.3.0 → 12.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (153) hide show
  1. package/.turbo/turbo-build.log +4 -4
  2. package/CHANGELOG.md +36 -0
  3. package/dist/config.d.ts +1 -0
  4. package/dist/config.d.ts.map +1 -1
  5. package/dist/config.js +9 -2
  6. package/dist/config.js.map +1 -1
  7. package/dist/core/demand.d.ts +4 -2
  8. package/dist/core/demand.d.ts.map +1 -1
  9. package/dist/core/demand.js +22 -6
  10. package/dist/core/demand.js.map +1 -1
  11. package/dist/core/entities/reservation.d.ts +1 -0
  12. package/dist/core/entities/reservation.d.ts.map +1 -1
  13. package/dist/core/entities/reservation.js.map +1 -1
  14. package/dist/core/entities/runner-instance.d.ts +3 -0
  15. package/dist/core/entities/runner-instance.d.ts.map +1 -1
  16. package/dist/core/entities/runner-instance.js.map +1 -1
  17. package/dist/core/errors.d.ts +5 -1
  18. package/dist/core/errors.d.ts.map +1 -1
  19. package/dist/core/errors.js +9 -2
  20. package/dist/core/errors.js.map +1 -1
  21. package/dist/core/runner-activation.d.ts +4 -1
  22. package/dist/core/runner-activation.d.ts.map +1 -1
  23. package/dist/core/runner-activation.js +32 -3
  24. package/dist/core/runner-activation.js.map +1 -1
  25. package/dist/core/runner-control-sessions.d.ts +8 -4
  26. package/dist/core/runner-control-sessions.d.ts.map +1 -1
  27. package/dist/core/runner-control-sessions.js +116 -20
  28. package/dist/core/runner-control-sessions.js.map +1 -1
  29. package/dist/core/runner-instances.d.ts +1 -0
  30. package/dist/core/runner-instances.d.ts.map +1 -1
  31. package/dist/core/runner-instances.js +1 -0
  32. package/dist/core/runner-instances.js.map +1 -1
  33. package/dist/db/db.d.ts +68 -0
  34. package/dist/db/db.d.ts.map +1 -1
  35. package/dist/db/ephemeral-registration-tokens.d.ts.map +1 -1
  36. package/dist/db/ephemeral-registration-tokens.js +1 -1
  37. package/dist/db/ephemeral-registration-tokens.js.map +1 -1
  38. package/dist/db/job-executions.d.ts.map +1 -1
  39. package/dist/db/job-executions.js +35 -4
  40. package/dist/db/job-executions.js.map +1 -1
  41. package/dist/db/reservation-locks.d.ts +6 -0
  42. package/dist/db/reservation-locks.d.ts.map +1 -0
  43. package/dist/db/reservation-locks.js +15 -0
  44. package/dist/db/reservation-locks.js.map +1 -0
  45. package/dist/db/reservations.d.ts +17 -3
  46. package/dist/db/reservations.d.ts.map +1 -1
  47. package/dist/db/reservations.js +246 -51
  48. package/dist/db/reservations.js.map +1 -1
  49. package/dist/db/runner-assignments.d.ts +25 -1
  50. package/dist/db/runner-assignments.d.ts.map +1 -1
  51. package/dist/db/runner-assignments.js +160 -18
  52. package/dist/db/runner-assignments.js.map +1 -1
  53. package/dist/db/runner-instances.d.ts +11 -2
  54. package/dist/db/runner-instances.d.ts.map +1 -1
  55. package/dist/db/runner-instances.js +167 -24
  56. package/dist/db/runner-instances.js.map +1 -1
  57. package/dist/db/runner-sessions.d.ts.map +1 -1
  58. package/dist/db/runner-sessions.js +14 -1
  59. package/dist/db/runner-sessions.js.map +1 -1
  60. package/dist/db/runner-states.d.ts +2 -0
  61. package/dist/db/runner-states.d.ts.map +1 -0
  62. package/dist/db/runner-states.js +7 -0
  63. package/dist/db/runner-states.js.map +1 -0
  64. package/dist/db/schema/reservations.d.ts +18 -0
  65. package/dist/db/schema/reservations.d.ts.map +1 -1
  66. package/dist/db/schema/reservations.js +7 -1
  67. package/dist/db/schema/reservations.js.map +1 -1
  68. package/dist/db/schema/runner-instances.d.ts +18 -0
  69. package/dist/db/schema/runner-instances.d.ts.map +1 -1
  70. package/dist/db/schema/runner-instances.js +8 -0
  71. package/dist/db/schema/runner-instances.js.map +1 -1
  72. package/dist/metrics/index.d.ts +2 -2
  73. package/dist/metrics/index.d.ts.map +1 -1
  74. package/dist/metrics/index.js +1 -1
  75. package/dist/metrics/index.js.map +1 -1
  76. package/dist/metrics/instance.d.ts +72 -2
  77. package/dist/metrics/instance.d.ts.map +1 -1
  78. package/dist/metrics/instance.js +161 -1
  79. package/dist/metrics/instance.js.map +1 -1
  80. package/dist/metrics/service.d.ts.map +1 -1
  81. package/dist/metrics/service.js +25 -4
  82. package/dist/metrics/service.js.map +1 -1
  83. package/dist/presentation/dto/poll-demand.d.ts.map +1 -1
  84. package/dist/presentation/dto/poll-demand.js +3 -0
  85. package/dist/presentation/dto/poll-demand.js.map +1 -1
  86. package/dist/presentation/dto/runner-instances.d.ts +4 -1
  87. package/dist/presentation/dto/runner-instances.d.ts.map +1 -1
  88. package/dist/presentation/dto/runner-instances.js +4 -1
  89. package/dist/presentation/dto/runner-instances.js.map +1 -1
  90. package/dist/presentation/routes/assign-runner-instances.d.ts.map +1 -1
  91. package/dist/presentation/routes/assign-runner-instances.js +10 -4
  92. package/dist/presentation/routes/assign-runner-instances.js.map +1 -1
  93. package/dist/presentation/routes/poll-demand.d.ts.map +1 -1
  94. package/dist/presentation/routes/poll-demand.js +1 -0
  95. package/dist/presentation/routes/poll-demand.js.map +1 -1
  96. package/dist/presentation/routes/reconcile-runner-instances.d.ts.map +1 -1
  97. package/dist/presentation/routes/reconcile-runner-instances.js +4 -2
  98. package/dist/presentation/routes/reconcile-runner-instances.js.map +1 -1
  99. package/dist/presentation/routes/runner-enrollment.d.ts +4 -1
  100. package/dist/presentation/routes/runner-enrollment.d.ts.map +1 -1
  101. package/dist/presentation/routes/runner-enrollment.js +11 -6
  102. package/dist/presentation/routes/runner-enrollment.js.map +1 -1
  103. package/dist/tsconfig.test.tsbuildinfo +1 -1
  104. package/drizzle/0005_previous_lenny_balinger.sql +5 -0
  105. package/drizzle/meta/0005_snapshot.json +2220 -0
  106. package/drizzle/meta/_journal.json +7 -0
  107. package/package.json +6 -6
  108. package/src/config.test.ts +22 -0
  109. package/src/config.ts +15 -2
  110. package/src/core/demand.test.ts +192 -3
  111. package/src/core/demand.ts +44 -4
  112. package/src/core/entities/reservation.ts +1 -0
  113. package/src/core/entities/runner-instance.ts +4 -0
  114. package/src/core/errors.ts +30 -1
  115. package/src/core/runner-activation.ts +48 -2
  116. package/src/core/runner-control-sessions.test.ts +135 -3
  117. package/src/core/runner-control-sessions.ts +141 -19
  118. package/src/core/runner-instances.test.ts +19 -0
  119. package/src/core/runner-instances.ts +2 -0
  120. package/src/core/runner-sessions.test.ts +121 -1
  121. package/src/db/ephemeral-registration-tokens.test.ts +20 -1
  122. package/src/db/ephemeral-registration-tokens.ts +1 -0
  123. package/src/db/job-executions.test.ts +98 -12
  124. package/src/db/job-executions.ts +52 -4
  125. package/src/db/reservation-locks.ts +22 -0
  126. package/src/db/reservations.test.ts +733 -22
  127. package/src/db/reservations.ts +408 -144
  128. package/src/db/runner-assignments.test.ts +105 -1
  129. package/src/db/runner-assignments.ts +245 -25
  130. package/src/db/runner-instances.test.ts +357 -2
  131. package/src/db/runner-instances.ts +289 -55
  132. package/src/db/runner-sessions.ts +19 -1
  133. package/src/db/runner-states.ts +7 -0
  134. package/src/db/schema/reservations.ts +5 -1
  135. package/src/db/schema/runner-instances.ts +9 -0
  136. package/src/metrics/index.ts +24 -1
  137. package/src/metrics/instance.test.ts +118 -0
  138. package/src/metrics/instance.ts +247 -3
  139. package/src/metrics/service.test.ts +46 -1
  140. package/src/metrics/service.ts +55 -8
  141. package/src/presentation/dto/poll-demand.ts +3 -0
  142. package/src/presentation/dto/runner-instances.ts +8 -1
  143. package/src/presentation/routes/assign-runner-instances.ts +14 -2
  144. package/src/presentation/routes/late-runner-enrollment.test.ts +219 -2
  145. package/src/presentation/routes/poll-demand.test.ts +56 -0
  146. package/src/presentation/routes/poll-demand.ts +1 -0
  147. package/src/presentation/routes/reconcile-runner-instances.test.ts +34 -2
  148. package/src/presentation/routes/reconcile-runner-instances.ts +7 -1
  149. package/src/presentation/routes/runner-enrollment.test.ts +316 -3
  150. package/src/presentation/routes/runner-enrollment.ts +8 -5
  151. package/test/factories/reservation.ts +2 -0
  152. package/test/factories/runner-instance.ts +6 -0
  153. package/tsconfig.build.tsbuildinfo +1 -1
@@ -9,6 +9,7 @@ import {runnerControlSessions} from '#db/schema/runner-control-sessions.js';
9
9
  import {providerRunners} from '#db/schema/runner-instances.js';
10
10
  import {
11
11
  type RunnerReservationPromotionFailureReason,
12
+ runnerActivationTokenNotIssuedCount,
12
13
  runnerReservationPromotionFailureCount,
13
14
  } from '#metrics/instance.js';
14
15
  import {enrollRunnerControlSession} from './runner-control-sessions.js';
@@ -167,7 +168,7 @@ describe('enrollRunnerControlSession', () => {
167
168
  });
168
169
  });
169
170
 
170
- it('updates metadata when a provider report precedes assignment commit', async () => {
171
+ it('repairs assignment metadata when a provider report precedes assignment commit', async () => {
171
172
  const provisionerId = crypto.randomUUID();
172
173
  const reservation = await createReservation({provisionerId});
173
174
  const capabilities: RunnerToolCapabilitiesDto = {
@@ -190,8 +191,10 @@ describe('enrollRunnerControlSession', () => {
190
191
  });
191
192
  const [runner] = await db()
192
193
  .select({
194
+ workspaceId: providerRunners.workspaceId,
193
195
  reservationId: providerRunners.reservationId,
194
196
  assignedAt: providerRunners.assignedAt,
197
+ intendedReservationId: providerRunners.intendedReservationId,
195
198
  labels: providerRunners.labels,
196
199
  providerKind: providerRunners.providerKind,
197
200
  protocolVersion: providerRunners.protocolVersion,
@@ -201,10 +204,12 @@ describe('enrollRunnerControlSession', () => {
201
204
  .from(providerRunners)
202
205
  .where(eq(providerRunners.id, runnerInstanceId));
203
206
 
204
- expect(activationToken).toBeNull();
207
+ expect(activationToken).toEqual(expect.any(String));
205
208
  expect(runner).toEqual({
209
+ workspaceId: reservation.workspaceId,
206
210
  reservationId: reservation.id,
207
- assignedAt: null,
211
+ assignedAt: expect.any(Date),
212
+ intendedReservationId: null,
208
213
  labels: ['linux'],
209
214
  providerKind: 'ec2',
210
215
  protocolVersion: '1',
@@ -213,6 +218,62 @@ describe('enrollRunnerControlSession', () => {
213
218
  });
214
219
  });
215
220
 
221
+ it('records an activation-token reason when enrollment sees an existing runner session', async () => {
222
+ const provisionerId = crypto.randomUUID();
223
+ const reservation = await createReservation({provisionerId});
224
+ const runnerInstanceId = await createRunner({
225
+ provisionerId,
226
+ intendedReservationId: reservation.id,
227
+ });
228
+ await db()
229
+ .update(providerRunners)
230
+ .set({runnerSessionId: crypto.randomUUID()})
231
+ .where(eq(providerRunners.id, runnerInstanceId));
232
+ const addSpy = vi.spyOn(runnerActivationTokenNotIssuedCount, 'add');
233
+
234
+ try {
235
+ const activationToken = await enrollRunnerControlSession({
236
+ runnerInstanceId,
237
+ provisionerId,
238
+ labels: ['linux'],
239
+ providerKind: 'docker',
240
+ protocolVersion: '1',
241
+ });
242
+ const [runner] = await db()
243
+ .select({
244
+ workspaceId: providerRunners.workspaceId,
245
+ reservationId: providerRunners.reservationId,
246
+ assignedAt: providerRunners.assignedAt,
247
+ intendedReservationId: providerRunners.intendedReservationId,
248
+ runnerSessionId: providerRunners.runnerSessionId,
249
+ state: providerRunners.state,
250
+ })
251
+ .from(providerRunners)
252
+ .where(eq(providerRunners.id, runnerInstanceId));
253
+
254
+ expect(activationToken).toBeNull();
255
+ expect(runner).toEqual({
256
+ workspaceId: reservation.workspaceId,
257
+ reservationId: reservation.id,
258
+ assignedAt: expect.any(Date),
259
+ intendedReservationId: null,
260
+ runnerSessionId: expect.any(String),
261
+ state: 'running',
262
+ });
263
+ // The test setup uses a shared NoopMeter counter; count only this metric's label shape.
264
+ const activationTokenCalls = addSpy.mock.calls.filter(
265
+ ([value, attributes]) =>
266
+ value === 1 &&
267
+ (attributes as {reason?: string; surface?: string} | undefined)?.reason ===
268
+ 'existing-session' &&
269
+ (attributes as {reason?: string; surface?: string} | undefined)?.surface === 'enrollment',
270
+ );
271
+ expect(activationTokenCalls).toHaveLength(1);
272
+ } finally {
273
+ addSpy.mockRestore();
274
+ }
275
+ });
276
+
216
277
  it('updates metadata during enrollment for an unassigned runner', async () => {
217
278
  const provisionerId = crypto.randomUUID();
218
279
  const capabilities: RunnerToolCapabilitiesDto = {
@@ -298,6 +359,77 @@ describe('enrollRunnerControlSession', () => {
298
359
  vi.resetModules();
299
360
  }
300
361
  });
362
+
363
+ it('records a rolled-back assignment rejection without recording its assignment timing', async () => {
364
+ const provisionerId = crypto.randomUUID();
365
+ const reservation = await createReservation({provisionerId});
366
+ const runnerInstanceId = await createRunner({
367
+ provisionerId,
368
+ intendedReservationId: reservation.id,
369
+ });
370
+ const assignmentTimingRecord = vi.fn();
371
+ const assignmentRejectionRecord = vi.fn();
372
+ const promotionFailureRecord = vi.fn();
373
+
374
+ vi.resetModules();
375
+ const {RunnerInstanceNotAssignableError} = await import('#core/errors.js');
376
+ const activationIssueError = new RunnerInstanceNotAssignableError(
377
+ runnerInstanceId,
378
+ 'capacity-exhausted',
379
+ );
380
+ vi.doMock('#metrics/index.js', async (importOriginal) => ({
381
+ ...(await importOriginal<typeof import('#metrics/index.js')>()),
382
+ recordProviderRunnerControlSessionToAssignment: assignmentTimingRecord,
383
+ recordProviderRunnerAssignmentRejected: assignmentRejectionRecord,
384
+ recordRunnerReservationPromotionFailure: promotionFailureRecord,
385
+ }));
386
+ vi.doMock('./runner-activation.js', async (importOriginal) => ({
387
+ ...(await importOriginal<typeof import('./runner-activation.js')>()),
388
+ issueRunnerActivationTokenTx: vi.fn().mockRejectedValue(activationIssueError),
389
+ }));
390
+
391
+ const {closePostgresClient, createPostgresClient} = await import('@shipfox/node-postgres');
392
+ createPostgresClient();
393
+
394
+ try {
395
+ const {enrollRunnerControlSession: enroll} = await import('./runner-control-sessions.js');
396
+ const result = await enroll({
397
+ runnerInstanceId,
398
+ provisionerId,
399
+ labels: ['linux'],
400
+ providerKind: 'docker',
401
+ protocolVersion: '1',
402
+ });
403
+ const [runner] = await db()
404
+ .select({
405
+ intendedReservationId: providerRunners.intendedReservationId,
406
+ reservationId: providerRunners.reservationId,
407
+ workspaceId: providerRunners.workspaceId,
408
+ assignedAt: providerRunners.assignedAt,
409
+ })
410
+ .from(providerRunners)
411
+ .where(eq(providerRunners.id, runnerInstanceId));
412
+
413
+ expect(result).toBeNull();
414
+ expect(runner).toEqual({
415
+ intendedReservationId: reservation.id,
416
+ reservationId: null,
417
+ workspaceId: null,
418
+ assignedAt: null,
419
+ });
420
+ expect(assignmentTimingRecord).not.toHaveBeenCalled();
421
+ expect(assignmentRejectionRecord).toHaveBeenCalledWith({
422
+ reason: 'capacity-exhausted',
423
+ surface: 'enrollment',
424
+ });
425
+ expect(promotionFailureRecord).toHaveBeenCalledWith('not-assignable');
426
+ } finally {
427
+ await closePostgresClient();
428
+ vi.doUnmock('#metrics/index.js');
429
+ vi.doUnmock('./runner-activation.js');
430
+ vi.resetModules();
431
+ }
432
+ });
301
433
  });
302
434
 
303
435
  async function expectPromotionFailure(params: {
@@ -5,6 +5,7 @@ import {extractDisplayPrefix, generateOpaqueToken, hashOpaqueToken} from '@shipf
5
5
  import {and, eq, gt, isNull, notInArray, or, sql} from 'drizzle-orm';
6
6
  import {config} from '#config.js';
7
7
  import {
8
+ getRunnerAssignmentRejectionReason,
8
9
  ReservationExpiredError,
9
10
  ReservationNotFoundError,
10
11
  RunnerInstanceAlreadyAssignedError,
@@ -12,13 +13,20 @@ import {
12
13
  } from '#core/errors.js';
13
14
  import {sanitizeRunnerLabels} from '#core/runner-labels.js';
14
15
  import {db, type schema, type Tx} from '#db/db.js';
15
- import {assignRunnerInstancesTx} from '#db/runner-assignments.js';
16
- import {terminalStates} from '#db/runner-instances.js';
16
+ import {
17
+ assignRunnerInstancesTx,
18
+ validateRunnerReservationCapacityTx,
19
+ } from '#db/runner-assignments.js';
20
+ import {terminalStates} from '#db/runner-states.js';
17
21
  import {provisionerTokens} from '#db/schema/provisioner-tokens.js';
18
22
  import {runnerBootstrapTokens, runnerControlSessions} from '#db/schema/runner-control-sessions.js';
19
23
  import {providerRunners} from '#db/schema/runner-instances.js';
20
24
  import {
21
25
  type RunnerReservationPromotionFailureReason,
26
+ recordProviderRunnerAssignmentRejected,
27
+ recordProviderRunnerControlSessionToAssignment,
28
+ recordProviderRunnerCreatedToControlSession,
29
+ recordRunnerReservationCapacityFailure,
22
30
  recordRunnerReservationPromotionFailure,
23
31
  } from '#metrics/index.js';
24
32
  import {issueRunnerActivationTokenTx} from './runner-activation.js';
@@ -45,27 +53,72 @@ export async function createRunnerInstancesWithBootstrapTokens(params: {
45
53
  reservationId?: string | null;
46
54
  }>;
47
55
  ttlSeconds: number;
48
- }): Promise<Array<{runnerInstanceId: string; bootstrapToken: string}>> {
56
+ }): Promise<{
57
+ runnerInstances: Array<{
58
+ runnerInstanceId: string;
59
+ bootstrapToken: string;
60
+ requestIndex: number;
61
+ }>;
62
+ reservationUnavailable: boolean;
63
+ }> {
49
64
  const expiresAt = new Date(Date.now() + params.ttlSeconds * 1000);
50
65
  return await db().transaction(async (tx) => {
66
+ const reservationValidation = await validateRunnerReservationCapacityTx(tx, {
67
+ provisionerId: params.provisionerId,
68
+ requests: params.runnerInstances.reduce(
69
+ (requests, runner) => {
70
+ if (!runner.reservationId) return requests;
71
+ const request = requests.find(
72
+ (candidate) => candidate.reservationId === runner.reservationId,
73
+ );
74
+ if (request) request.count += 1;
75
+ else requests.push({reservationId: runner.reservationId, count: 1});
76
+ return requests;
77
+ },
78
+ [] as Array<{reservationId: string; count: number}>,
79
+ ),
80
+ });
81
+ for (const {reason, count} of reservationValidation.unavailableByReservation.values())
82
+ recordRunnerReservationCapacityFailure(reason, count);
83
+ const remainingAcceptedByReservation = new Map(reservationValidation.acceptedByReservation);
84
+ const runnerInstances = params.runnerInstances.flatMap((runner, requestIndex) => {
85
+ if (!runner.reservationId) return [{...runner, requestIndex}];
86
+ const remaining = remainingAcceptedByReservation.get(runner.reservationId) ?? 0;
87
+ if (remaining === 0) return [];
88
+ remainingAcceptedByReservation.set(runner.reservationId, remaining - 1);
89
+ return [{...runner, requestIndex}];
90
+ });
91
+
92
+ if (runnerInstances.length === 0)
93
+ return {
94
+ runnerInstances: [],
95
+ reservationUnavailable: reservationValidation.unavailableByReservation.size > 0,
96
+ };
97
+
51
98
  const instances = await tx
52
99
  .insert(providerRunners)
53
100
  .values(
54
- params.runnerInstances.map((runner) => ({
101
+ runnerInstances.map((runner) => ({
55
102
  provisionerId: params.provisionerId,
56
103
  providerKind: params.providerKind ?? null,
57
104
  templateKey: runner.templateKey ?? null,
58
105
  intendedReservationId: runner.reservationId ?? null,
106
+ launchKind: runner.reservationId ? ('demand' as const) : ('warm' as const),
59
107
  state: 'starting' as const,
60
108
  labels: [],
61
109
  reportedAt: new Date(),
62
110
  })),
63
111
  )
64
112
  .returning({id: providerRunners.id});
65
- const results = instances.map((instance) => ({
66
- runnerInstanceId: instance.id,
67
- bootstrapToken: generateOpaqueToken('runnerBootstrapToken'),
68
- }));
113
+ const results = instances.map((instance, index) => {
114
+ const runner = runnerInstances[index];
115
+ if (!runner) throw new Error('Runner instance insert returned an unexpected row count');
116
+ return {
117
+ runnerInstanceId: instance.id,
118
+ bootstrapToken: generateOpaqueToken('runnerBootstrapToken'),
119
+ requestIndex: runner.requestIndex,
120
+ };
121
+ });
69
122
  await tx.insert(runnerBootstrapTokens).values(
70
123
  results.map((result) => ({
71
124
  runnerInstanceId: result.runnerInstanceId,
@@ -75,7 +128,10 @@ export async function createRunnerInstancesWithBootstrapTokens(params: {
75
128
  expiresAt,
76
129
  })),
77
130
  );
78
- return results;
131
+ return {
132
+ runnerInstances: results,
133
+ reservationUnavailable: reservationValidation.unavailableByReservation.size > 0,
134
+ };
79
135
  });
80
136
  }
81
137
 
@@ -85,7 +141,7 @@ export async function exchangeRunnerBootstrapToken(params: {
85
141
  }): Promise<{runnerInstanceId: string; controlSessionToken: string; expiresAt: Date}> {
86
142
  const controlSessionToken = generateOpaqueToken('runnerControlSession');
87
143
  const expiresAt = new Date(Date.now() + params.ttlSeconds * 1000);
88
- return await db().transaction(async (tx) => {
144
+ const result = await db().transaction(async (tx) => {
89
145
  const [bootstrap] = await tx
90
146
  .update(runnerBootstrapTokens)
91
147
  .set({consumedAt: sql`now()`})
@@ -99,6 +155,21 @@ export async function exchangeRunnerBootstrapToken(params: {
99
155
  )
100
156
  .returning();
101
157
  if (!bootstrap) throw new RunnerBootstrapTokenInvalidError();
158
+ const [runner] = await tx
159
+ .select({
160
+ runnerInstanceId: providerRunners.id,
161
+ createdAt: providerRunners.createdAt,
162
+ provider: providerRunners.providerKind,
163
+ launchKind: providerRunners.launchKind,
164
+ })
165
+ .from(providerRunners)
166
+ .where(
167
+ and(
168
+ eq(providerRunners.id, bootstrap.runnerInstanceId),
169
+ eq(providerRunners.provisionerId, bootstrap.provisionerId),
170
+ ),
171
+ )
172
+ .limit(1);
102
173
  const [session] = await tx
103
174
  .insert(runnerControlSessions)
104
175
  .values({
@@ -108,10 +179,28 @@ export async function exchangeRunnerBootstrapToken(params: {
108
179
  prefix: extractDisplayPrefix(controlSessionToken),
109
180
  expiresAt,
110
181
  })
111
- .returning({id: runnerControlSessions.id});
182
+ .returning({createdAt: runnerControlSessions.createdAt});
112
183
  if (!session) throw new Error('Runner control session insert returned no row');
113
- return {runnerInstanceId: bootstrap.runnerInstanceId, controlSessionToken, expiresAt};
184
+ return {
185
+ runner,
186
+ session,
187
+ runnerInstanceId: bootstrap.runnerInstanceId,
188
+ controlSessionToken,
189
+ expiresAt,
190
+ };
114
191
  });
192
+ if (result.runner)
193
+ recordProviderRunnerCreatedToControlSession({
194
+ durationMilliseconds: result.session.createdAt.getTime() - result.runner.createdAt.getTime(),
195
+ provider: result.runner.provider,
196
+ launchKind: result.runner.launchKind,
197
+ runnerInstanceId: result.runner.runnerInstanceId,
198
+ });
199
+ return {
200
+ runnerInstanceId: result.runnerInstanceId,
201
+ controlSessionToken: result.controlSessionToken,
202
+ expiresAt: result.expiresAt,
203
+ };
115
204
  }
116
205
 
117
206
  export async function resolveRunnerControlSession(rawToken: string) {
@@ -143,7 +232,7 @@ export async function enrollRunnerControlSession(params: {
143
232
  providerKind: string;
144
233
  protocolVersion: string;
145
234
  }): Promise<string | null> {
146
- return await db().transaction(async (tx) => {
235
+ const result = await db().transaction(async (tx) => {
147
236
  const [current] = await tx
148
237
  .select({
149
238
  intendedReservationId: providerRunners.intendedReservationId,
@@ -214,22 +303,38 @@ export async function enrollRunnerControlSession(params: {
214
303
  await updateRunnerControlSessionLastSeen(tx, params.runnerInstanceId, params.provisionerId);
215
304
 
216
305
  const intendedReservationId = updated.intendedReservationId;
217
- if (!intendedReservationId) return null;
306
+ if (!intendedReservationId)
307
+ return {
308
+ activationToken: null,
309
+ controlSessionToAssignment: [],
310
+ assignmentRejectedReason: null,
311
+ promotionFailureReason: null,
312
+ };
218
313
 
219
314
  try {
220
- return await tx.transaction(async (promotionTx) => {
221
- await assignRunnerInstancesTx(promotionTx, {
315
+ const promotion = await tx.transaction(async (promotionTx) => {
316
+ const assignment = await assignRunnerInstancesTx(promotionTx, {
222
317
  provisionerId: params.provisionerId,
223
318
  reservationId: intendedReservationId,
224
319
  runnerInstanceIds: [params.runnerInstanceId],
320
+ surface: 'enrollment',
225
321
  });
226
- return await issueRunnerActivationTokenTx(promotionTx, {
322
+ const activationToken = await issueRunnerActivationTokenTx(promotionTx, {
227
323
  runnerInstanceId: params.runnerInstanceId,
228
324
  provisionerId: params.provisionerId,
229
325
  ttlSeconds: config.RUNNER_ACTIVATION_TOKEN_TTL_SECONDS,
326
+ surface: 'enrollment',
230
327
  });
328
+ return {activationToken, controlSessionToAssignment: assignment.controlSessionToAssignment};
231
329
  });
330
+ return {
331
+ activationToken: promotion.activationToken,
332
+ controlSessionToAssignment: promotion.controlSessionToAssignment,
333
+ assignmentRejectedReason: null,
334
+ promotionFailureReason: null,
335
+ };
232
336
  } catch (error) {
337
+ const assignmentRejectionReason = getRunnerAssignmentRejectionReason(error);
233
338
  const expectedFailureReason = getRunnerReservationPromotionFailureReason(error);
234
339
  const details = {
235
340
  err: error,
@@ -238,7 +343,6 @@ export async function enrollRunnerControlSession(params: {
238
343
  reservationId: intendedReservationId,
239
344
  };
240
345
  if (expectedFailureReason) {
241
- recordRunnerReservationPromotionFailure(expectedFailureReason);
242
346
  logger().debug(details, 'Runner reservation could not be promoted during enrollment');
243
347
  } else {
244
348
  logger().error(
@@ -246,9 +350,27 @@ export async function enrollRunnerControlSession(params: {
246
350
  'Unexpected failure promoting runner reservation during enrollment',
247
351
  );
248
352
  }
249
- return null;
353
+ return {
354
+ activationToken: null,
355
+ controlSessionToAssignment: [],
356
+ assignmentRejectedReason: assignmentRejectionReason,
357
+ promotionFailureReason: expectedFailureReason,
358
+ };
250
359
  }
251
360
  });
361
+ for (const observation of result.controlSessionToAssignment)
362
+ recordProviderRunnerControlSessionToAssignment(observation);
363
+ // These counters intentionally overlap: assignment_rejected records the assignment-facing
364
+ // reason, while reservation_promotion_failure records the enrollment recovery outcome. They
365
+ // answer different questions and must not be summed.
366
+ if (result.assignmentRejectedReason)
367
+ recordProviderRunnerAssignmentRejected({
368
+ reason: result.assignmentRejectedReason,
369
+ surface: 'enrollment',
370
+ });
371
+ if (result.promotionFailureReason)
372
+ recordRunnerReservationPromotionFailure(result.promotionFailureReason);
373
+ return result.activationToken;
252
374
  }
253
375
 
254
376
  function getRunnerReservationPromotionFailureReason(
@@ -54,18 +54,37 @@ describe('reconcileRunnerInstancesFromDbResult', () => {
54
54
 
55
55
  expect(result[0]?.desiredIntent).toBe('terminate');
56
56
  });
57
+
58
+ it('returns the intended reservation when a runner is not assigned yet', () => {
59
+ const intendedReservationId = crypto.randomUUID();
60
+ const result = reconcileRunnerInstancesFromDbResult({
61
+ observedRunnerInstanceIds: ['provisioned-runner-1'],
62
+ observedRows: [
63
+ providerRunner({providerRunnerId: 'provisioned-runner-1', intendedReservationId}),
64
+ ],
65
+ boundJobExecutionsByRunnerInstanceId: new Map(),
66
+ });
67
+
68
+ expect(result[0]).toMatchObject({
69
+ reservationId: null,
70
+ intendedReservationId,
71
+ });
72
+ });
57
73
  });
58
74
 
59
75
  function providerRunner(params: {
60
76
  providerRunnerId: string;
61
77
  state?: 'starting' | 'running' | 'stopping' | 'stopped' | 'failed' | 'terminated';
78
+ intendedReservationId?: string | null;
62
79
  }) {
63
80
  return {
64
81
  id: crypto.randomUUID(),
65
82
  workspaceId: crypto.randomUUID(),
66
83
  provisionerId: crypto.randomUUID(),
67
84
  providerRunnerId: params.providerRunnerId,
85
+ intendedReservationId: params.intendedReservationId ?? null,
68
86
  reservationId: null,
87
+ launchKind: 'manual' as const,
69
88
  templateKey: 'linux',
70
89
  labels: ['linux'],
71
90
  state: params.state ?? 'running',
@@ -62,6 +62,7 @@ export interface ReconciledBoundJobExecution {
62
62
  export interface ReconciledRunnerInstance {
63
63
  providerRunnerId: string;
64
64
  state: RunnerInstanceState | null;
65
+ intendedReservationId: string | null;
65
66
  reservationId: string | null;
66
67
  runnerSessionId: string | null;
67
68
  boundJobExecution: ReconciledBoundJobExecution | null;
@@ -157,6 +158,7 @@ export function reconcileRunnerInstancesFromDbResult(params: {
157
158
  return {
158
159
  providerRunnerId,
159
160
  state: row?.state ?? null,
161
+ intendedReservationId: row?.intendedReservationId ?? null,
160
162
  reservationId: row?.reservationId ?? null,
161
163
  runnerSessionId: row?.runnerSessionId ?? null,
162
164
  boundJobExecution: boundJobExecution
@@ -1,9 +1,15 @@
1
+ import {vi} from '@shipfox/vitest/vi';
1
2
  import {and, eq} from 'drizzle-orm';
2
3
  import {db} from '#db/db.js';
3
4
  import {ephemeralRegistrationTokens} from '#db/schema/ephemeral-registration-tokens.js';
4
5
  import {runnerActivationTokens} from '#db/schema/runner-activation-tokens.js';
6
+ import type {RunnerInstanceInsertDb} from '#db/schema/runner-instances.js';
5
7
  import {providerRunners} from '#db/schema/runner-instances.js';
6
8
  import {runnerSessions} from '#db/schema/runner-sessions.js';
9
+ import {
10
+ type RunnerActivationTokenNotIssuedReason,
11
+ runnerActivationTokenNotIssuedCount,
12
+ } from '#metrics/instance.js';
7
13
  import {
8
14
  ephemeralRegistrationTokenFactory,
9
15
  getRunnerSessionTokenClaims,
@@ -18,9 +24,34 @@ import {
18
24
  RegistrationTokenWorkspaceMismatchError,
19
25
  RunnerLabelsReservedError,
20
26
  } from './errors.js';
21
- import {issueRunnerActivationToken} from './runner-activation.js';
27
+ import {getRunnerAssignment, issueRunnerActivationToken} from './runner-activation.js';
22
28
  import {registerRunnerSession} from './runner-sessions.js';
23
29
 
30
+ const activationTokenNotIssuedCases: Array<{
31
+ reason: RunnerActivationTokenNotIssuedReason;
32
+ update: Partial<RunnerInstanceInsertDb>;
33
+ provisionerId?: string;
34
+ }> = [
35
+ {reason: 'runner-not-found', update: {}, provisionerId: crypto.randomUUID()},
36
+ {reason: 'missing-workspace', update: {workspaceId: null}},
37
+ {reason: 'existing-session', update: {runnerSessionId: crypto.randomUUID()}},
38
+ {reason: 'not-running', update: {state: 'starting'}},
39
+ ];
40
+
41
+ function activationTokenMetricCalls(spy: {mock: {calls: unknown[][]}}): unknown[][] {
42
+ // The test setup uses a shared NoopMeter counter, so filter calls to this metric before counting.
43
+ return spy.mock.calls.filter(([, attributes]) => {
44
+ if (typeof attributes !== 'object' || attributes === null) return false;
45
+ const reason = (attributes as {reason?: unknown}).reason;
46
+ const surface = (attributes as {surface?: unknown}).surface;
47
+ return (
48
+ ['runner-not-found', 'missing-workspace', 'existing-session', 'not-running'].includes(
49
+ String(reason),
50
+ ) && ['enrollment', 'poll'].includes(String(surface))
51
+ );
52
+ });
53
+ }
54
+
24
55
  describe('registerRunnerSession', () => {
25
56
  let workspaceId: string;
26
57
  let registrationTokenId: string;
@@ -245,11 +276,13 @@ describe('activation runner sessions', () => {
245
276
  runnerInstanceId,
246
277
  provisionerId,
247
278
  ttlSeconds: 60,
279
+ surface: 'poll',
248
280
  });
249
281
  const replacementToken = await issueRunnerActivationToken({
250
282
  runnerInstanceId,
251
283
  provisionerId,
252
284
  ttlSeconds: 60,
285
+ surface: 'poll',
253
286
  });
254
287
 
255
288
  const tokens = await db()
@@ -265,11 +298,97 @@ describe('activation runner sessions', () => {
265
298
  expect(tokens.filter((token) => token.revokedAt !== null)).toHaveLength(1);
266
299
  });
267
300
 
301
+ it.each(
302
+ activationTokenNotIssuedCases,
303
+ )('records $reason when direct activation-token issuance is skipped', async ({
304
+ reason,
305
+ update,
306
+ provisionerId: caseProvisionerId,
307
+ }) => {
308
+ const addSpy = vi.spyOn(runnerActivationTokenNotIssuedCount, 'add');
309
+
310
+ try {
311
+ if (Object.keys(update).length > 0) {
312
+ await db()
313
+ .update(providerRunners)
314
+ .set(update)
315
+ .where(eq(providerRunners.id, runnerInstanceId));
316
+ }
317
+
318
+ const activationToken = await issueRunnerActivationToken({
319
+ runnerInstanceId,
320
+ provisionerId: caseProvisionerId ?? provisionerId,
321
+ ttlSeconds: 60,
322
+ surface: 'poll',
323
+ });
324
+
325
+ expect(activationToken).toBeNull();
326
+ expect(activationTokenMetricCalls(addSpy)).toEqual([[1, {reason, surface: 'poll'}]]);
327
+ } finally {
328
+ addSpy.mockRestore();
329
+ }
330
+ });
331
+
332
+ it('does not record a metric when assignment polling finds no assignment', async () => {
333
+ const addSpy = vi.spyOn(runnerActivationTokenNotIssuedCount, 'add');
334
+
335
+ try {
336
+ await db()
337
+ .update(providerRunners)
338
+ .set({workspaceId: null})
339
+ .where(eq(providerRunners.id, runnerInstanceId));
340
+
341
+ const assignment = await getRunnerAssignment({runnerInstanceId, provisionerId});
342
+
343
+ expect(assignment).toBeNull();
344
+ expect(activationTokenMetricCalls(addSpy)).toHaveLength(0);
345
+ } finally {
346
+ addSpy.mockRestore();
347
+ }
348
+ });
349
+
350
+ it('does not poll for an assignment while the runner is not running', async () => {
351
+ const addSpy = vi.spyOn(runnerActivationTokenNotIssuedCount, 'add');
352
+
353
+ try {
354
+ await db()
355
+ .update(providerRunners)
356
+ .set({state: 'starting'})
357
+ .where(eq(providerRunners.id, runnerInstanceId));
358
+
359
+ const assignment = await getRunnerAssignment({runnerInstanceId, provisionerId});
360
+
361
+ expect(assignment).toBeNull();
362
+ expect(activationTokenMetricCalls(addSpy)).toHaveLength(0);
363
+ } finally {
364
+ addSpy.mockRestore();
365
+ }
366
+ });
367
+
368
+ it('does not record a metric when activation-token issuance succeeds', async () => {
369
+ const addSpy = vi.spyOn(runnerActivationTokenNotIssuedCount, 'add');
370
+
371
+ try {
372
+ const activationToken = await issueRunnerActivationToken({
373
+ runnerInstanceId,
374
+ provisionerId,
375
+ ttlSeconds: 60,
376
+ surface: 'poll',
377
+ });
378
+
379
+ expect(activationToken).toEqual(expect.any(String));
380
+ expect(activationTokenMetricCalls(addSpy)).toHaveLength(0);
381
+ } finally {
382
+ addSpy.mockRestore();
383
+ }
384
+ });
385
+
268
386
  it('allows only one concurrent registration to consume an activation token', async () => {
269
387
  const rawToken = await issueRunnerActivationToken({
270
388
  runnerInstanceId,
271
389
  provisionerId,
272
390
  ttlSeconds: 60,
391
+ surface: 'poll',
273
392
  });
274
393
  const [activationToken] = await db()
275
394
  .select()
@@ -314,6 +433,7 @@ describe('activation runner sessions', () => {
314
433
  runnerInstanceId,
315
434
  provisionerId,
316
435
  ttlSeconds: 60,
436
+ surface: 'poll',
317
437
  });
318
438
  const [activationToken] = await db()
319
439
  .select()