@shipfox/api-runners 12.2.0 → 12.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (149) hide show
  1. package/.turbo/turbo-build.log +4 -4
  2. package/CHANGELOG.md +34 -0
  3. package/dist/config.d.ts +1 -0
  4. package/dist/config.d.ts.map +1 -1
  5. package/dist/config.js +10 -3
  6. package/dist/config.js.map +1 -1
  7. package/dist/core/demand.d.ts +4 -2
  8. package/dist/core/demand.d.ts.map +1 -1
  9. package/dist/core/demand.js +22 -6
  10. package/dist/core/demand.js.map +1 -1
  11. package/dist/core/entities/reservation.d.ts +1 -0
  12. package/dist/core/entities/reservation.d.ts.map +1 -1
  13. package/dist/core/entities/reservation.js.map +1 -1
  14. package/dist/core/entities/runner-instance.d.ts +3 -0
  15. package/dist/core/entities/runner-instance.d.ts.map +1 -1
  16. package/dist/core/entities/runner-instance.js.map +1 -1
  17. package/dist/core/runner-activation.d.ts +4 -1
  18. package/dist/core/runner-activation.d.ts.map +1 -1
  19. package/dist/core/runner-activation.js +32 -3
  20. package/dist/core/runner-activation.js.map +1 -1
  21. package/dist/core/runner-control-sessions.d.ts +8 -4
  22. package/dist/core/runner-control-sessions.d.ts.map +1 -1
  23. package/dist/core/runner-control-sessions.js +69 -11
  24. package/dist/core/runner-control-sessions.js.map +1 -1
  25. package/dist/core/runner-instances.d.ts +1 -0
  26. package/dist/core/runner-instances.d.ts.map +1 -1
  27. package/dist/core/runner-instances.js +1 -0
  28. package/dist/core/runner-instances.js.map +1 -1
  29. package/dist/db/db.d.ts +68 -0
  30. package/dist/db/db.d.ts.map +1 -1
  31. package/dist/db/ephemeral-registration-tokens.d.ts.map +1 -1
  32. package/dist/db/ephemeral-registration-tokens.js +1 -1
  33. package/dist/db/ephemeral-registration-tokens.js.map +1 -1
  34. package/dist/db/index.d.ts +1 -1
  35. package/dist/db/index.d.ts.map +1 -1
  36. package/dist/db/index.js +1 -1
  37. package/dist/db/index.js.map +1 -1
  38. package/dist/db/reservation-locks.d.ts +6 -0
  39. package/dist/db/reservation-locks.d.ts.map +1 -0
  40. package/dist/db/reservation-locks.js +15 -0
  41. package/dist/db/reservation-locks.js.map +1 -0
  42. package/dist/db/reservations.d.ts +17 -3
  43. package/dist/db/reservations.d.ts.map +1 -1
  44. package/dist/db/reservations.js +256 -45
  45. package/dist/db/reservations.js.map +1 -1
  46. package/dist/db/runner-assignments.d.ts +17 -0
  47. package/dist/db/runner-assignments.d.ts.map +1 -1
  48. package/dist/db/runner-assignments.js +103 -8
  49. package/dist/db/runner-assignments.js.map +1 -1
  50. package/dist/db/runner-instances.d.ts +5 -2
  51. package/dist/db/runner-instances.d.ts.map +1 -1
  52. package/dist/db/runner-instances.js +134 -25
  53. package/dist/db/runner-instances.js.map +1 -1
  54. package/dist/db/runner-sessions.d.ts.map +1 -1
  55. package/dist/db/runner-sessions.js +22 -15
  56. package/dist/db/runner-sessions.js.map +1 -1
  57. package/dist/db/runner-states.d.ts +2 -0
  58. package/dist/db/runner-states.d.ts.map +1 -0
  59. package/dist/db/runner-states.js +7 -0
  60. package/dist/db/runner-states.js.map +1 -0
  61. package/dist/db/schema/reservations.d.ts +18 -0
  62. package/dist/db/schema/reservations.d.ts.map +1 -1
  63. package/dist/db/schema/reservations.js +7 -1
  64. package/dist/db/schema/reservations.js.map +1 -1
  65. package/dist/db/schema/runner-instances.d.ts +18 -0
  66. package/dist/db/schema/runner-instances.d.ts.map +1 -1
  67. package/dist/db/schema/runner-instances.js +8 -0
  68. package/dist/db/schema/runner-instances.js.map +1 -1
  69. package/dist/metrics/index.d.ts +2 -2
  70. package/dist/metrics/index.d.ts.map +1 -1
  71. package/dist/metrics/index.js +1 -1
  72. package/dist/metrics/index.js.map +1 -1
  73. package/dist/metrics/instance.d.ts +24 -2
  74. package/dist/metrics/instance.d.ts.map +1 -1
  75. package/dist/metrics/instance.js +23 -0
  76. package/dist/metrics/instance.js.map +1 -1
  77. package/dist/metrics/service.d.ts.map +1 -1
  78. package/dist/metrics/service.js +20 -4
  79. package/dist/metrics/service.js.map +1 -1
  80. package/dist/presentation/dto/poll-demand.d.ts.map +1 -1
  81. package/dist/presentation/dto/poll-demand.js +3 -0
  82. package/dist/presentation/dto/poll-demand.js.map +1 -1
  83. package/dist/presentation/dto/runner-instances.d.ts +4 -1
  84. package/dist/presentation/dto/runner-instances.d.ts.map +1 -1
  85. package/dist/presentation/dto/runner-instances.js +4 -1
  86. package/dist/presentation/dto/runner-instances.js.map +1 -1
  87. package/dist/presentation/routes/assign-runner-instances.d.ts.map +1 -1
  88. package/dist/presentation/routes/assign-runner-instances.js +3 -3
  89. package/dist/presentation/routes/assign-runner-instances.js.map +1 -1
  90. package/dist/presentation/routes/poll-demand.d.ts.map +1 -1
  91. package/dist/presentation/routes/poll-demand.js +1 -0
  92. package/dist/presentation/routes/poll-demand.js.map +1 -1
  93. package/dist/presentation/routes/reconcile-runner-instances.d.ts.map +1 -1
  94. package/dist/presentation/routes/reconcile-runner-instances.js +4 -2
  95. package/dist/presentation/routes/reconcile-runner-instances.js.map +1 -1
  96. package/dist/presentation/routes/runner-enrollment.d.ts +4 -1
  97. package/dist/presentation/routes/runner-enrollment.d.ts.map +1 -1
  98. package/dist/presentation/routes/runner-enrollment.js +11 -6
  99. package/dist/presentation/routes/runner-enrollment.js.map +1 -1
  100. package/dist/tsconfig.test.tsbuildinfo +1 -1
  101. package/drizzle/0005_previous_lenny_balinger.sql +5 -0
  102. package/drizzle/meta/0005_snapshot.json +2220 -0
  103. package/drizzle/meta/_journal.json +7 -0
  104. package/package.json +4 -4
  105. package/src/config.test.ts +24 -2
  106. package/src/config.ts +16 -3
  107. package/src/core/demand.test.ts +192 -3
  108. package/src/core/demand.ts +44 -4
  109. package/src/core/entities/reservation.ts +1 -0
  110. package/src/core/entities/runner-instance.ts +4 -0
  111. package/src/core/runner-activation.ts +48 -2
  112. package/src/core/runner-control-sessions.test.ts +230 -4
  113. package/src/core/runner-control-sessions.ts +84 -10
  114. package/src/core/runner-instances.test.ts +19 -0
  115. package/src/core/runner-instances.ts +2 -0
  116. package/src/core/runner-sessions.test.ts +121 -1
  117. package/src/db/ephemeral-registration-tokens.test.ts +20 -1
  118. package/src/db/ephemeral-registration-tokens.ts +1 -0
  119. package/src/db/index.ts +1 -0
  120. package/src/db/reservation-locks.ts +22 -0
  121. package/src/db/reservations.test.ts +1110 -31
  122. package/src/db/reservations.ts +434 -105
  123. package/src/db/runner-assignments.test.ts +105 -1
  124. package/src/db/runner-assignments.ts +178 -9
  125. package/src/db/runner-instances.test.ts +496 -2
  126. package/src/db/runner-instances.ts +247 -56
  127. package/src/db/runner-sessions.ts +26 -15
  128. package/src/db/runner-states.ts +7 -0
  129. package/src/db/schema/reservations.ts +5 -1
  130. package/src/db/schema/runner-instances.ts +9 -0
  131. package/src/metrics/index.ts +10 -1
  132. package/src/metrics/instance.ts +59 -2
  133. package/src/metrics/service.test.ts +106 -0
  134. package/src/metrics/service.ts +24 -4
  135. package/src/presentation/dto/poll-demand.ts +3 -0
  136. package/src/presentation/dto/runner-instances.ts +8 -1
  137. package/src/presentation/routes/assign-runner-instances.ts +6 -2
  138. package/src/presentation/routes/late-runner-enrollment.test.ts +447 -0
  139. package/src/presentation/routes/poll-demand.test.ts +79 -6
  140. package/src/presentation/routes/poll-demand.ts +1 -0
  141. package/src/presentation/routes/reconcile-runner-instances.test.ts +34 -2
  142. package/src/presentation/routes/reconcile-runner-instances.ts +7 -1
  143. package/src/presentation/routes/runner-enrollment.test.ts +316 -3
  144. package/src/presentation/routes/runner-enrollment.ts +8 -5
  145. package/test/factories/reservation.ts +2 -0
  146. package/test/factories/runner-instance.ts +6 -0
  147. package/test/fixtures/late-runner-enrollment.ts +103 -0
  148. package/test/index.ts +5 -0
  149. package/tsconfig.build.tsbuildinfo +1 -1
@@ -1,17 +1,21 @@
1
+ import type {RunnerToolCapabilitiesDto} from '@shipfox/api-runners-dto';
1
2
  import {afterEach, vi} from '@shipfox/vitest/vi';
2
- import {inArray} from 'drizzle-orm';
3
+ import {eq, inArray} from 'drizzle-orm';
3
4
  import {db} from '#db/db.js';
5
+ import {provisionerTokens} from '#db/schema/provisioner-tokens.js';
4
6
  import {reservations} from '#db/schema/reservations.js';
5
7
  import {runnerActivationTokens} from '#db/schema/runner-activation-tokens.js';
6
8
  import {runnerControlSessions} from '#db/schema/runner-control-sessions.js';
7
9
  import {providerRunners} from '#db/schema/runner-instances.js';
8
10
  import {
9
11
  type RunnerReservationPromotionFailureReason,
12
+ runnerActivationTokenNotIssuedCount,
10
13
  runnerReservationPromotionFailureCount,
11
14
  } from '#metrics/instance.js';
12
15
  import {enrollRunnerControlSession} from './runner-control-sessions.js';
13
16
 
14
17
  const createdReservationIds = new Set<string>();
18
+ const createdProvisionerTokenIds = new Set<string>();
15
19
  const createdRunnerInstanceIds = new Set<string>();
16
20
 
17
21
  afterEach(async () => {
@@ -29,9 +33,14 @@ afterEach(async () => {
29
33
  }
30
34
  if (reservationIds.length > 0)
31
35
  await db().delete(reservations).where(inArray(reservations.id, reservationIds));
36
+ if (createdProvisionerTokenIds.size > 0)
37
+ await db()
38
+ .delete(provisionerTokens)
39
+ .where(inArray(provisionerTokens.id, [...createdProvisionerTokenIds]));
32
40
 
33
41
  createdRunnerInstanceIds.clear();
34
42
  createdReservationIds.clear();
43
+ createdProvisionerTokenIds.clear();
35
44
  });
36
45
 
37
46
  describe('enrollRunnerControlSession', () => {
@@ -110,6 +119,199 @@ describe('enrollRunnerControlSession', () => {
110
119
  }
111
120
  });
112
121
 
122
+ it('preserves committed assignment metadata during a later enrollment', async () => {
123
+ const provisionerId = crypto.randomUUID();
124
+ const reservation = await createReservation({provisionerId});
125
+ const committedCapabilities: RunnerToolCapabilitiesDto = {
126
+ harnesses: {pi: {tools: ['read']}},
127
+ };
128
+ const runnerInstanceId = await createRunner({
129
+ provisionerId,
130
+ intendedReservationId: reservation.id,
131
+ reservationId: reservation.id,
132
+ workspaceId: reservation.workspaceId,
133
+ assignedAt: new Date(),
134
+ labels: ['linux'],
135
+ providerKind: 'ec2',
136
+ protocolVersion: '1',
137
+ capabilities: committedCapabilities,
138
+ });
139
+
140
+ const activationToken = await enrollRunnerControlSession({
141
+ runnerInstanceId,
142
+ provisionerId,
143
+ labels: ['gpu'],
144
+ capabilities: {harnesses: {claude: {tools: ['bash']}}},
145
+ providerKind: 'docker',
146
+ protocolVersion: '2',
147
+ });
148
+ const [runner] = await db()
149
+ .select({
150
+ reservationId: providerRunners.reservationId,
151
+ labels: providerRunners.labels,
152
+ providerKind: providerRunners.providerKind,
153
+ protocolVersion: providerRunners.protocolVersion,
154
+ capabilities: providerRunners.capabilities,
155
+ state: providerRunners.state,
156
+ })
157
+ .from(providerRunners)
158
+ .where(eq(providerRunners.id, runnerInstanceId));
159
+
160
+ expect(activationToken).toEqual(expect.any(String));
161
+ expect(runner).toEqual({
162
+ reservationId: reservation.id,
163
+ labels: ['linux'],
164
+ providerKind: 'ec2',
165
+ protocolVersion: '1',
166
+ capabilities: committedCapabilities,
167
+ state: 'running',
168
+ });
169
+ });
170
+
171
+ it('repairs assignment metadata when a provider report precedes assignment commit', async () => {
172
+ const provisionerId = crypto.randomUUID();
173
+ const reservation = await createReservation({provisionerId});
174
+ const capabilities: RunnerToolCapabilitiesDto = {
175
+ harnesses: {pi: {tools: ['read']}},
176
+ };
177
+ const runnerInstanceId = await createRunner({
178
+ provisionerId,
179
+ intendedReservationId: reservation.id,
180
+ reservationId: reservation.id,
181
+ assignedAt: null,
182
+ });
183
+
184
+ const activationToken = await enrollRunnerControlSession({
185
+ runnerInstanceId,
186
+ provisionerId,
187
+ labels: ['linux'],
188
+ capabilities,
189
+ providerKind: 'ec2',
190
+ protocolVersion: '1',
191
+ });
192
+ const [runner] = await db()
193
+ .select({
194
+ workspaceId: providerRunners.workspaceId,
195
+ reservationId: providerRunners.reservationId,
196
+ assignedAt: providerRunners.assignedAt,
197
+ intendedReservationId: providerRunners.intendedReservationId,
198
+ labels: providerRunners.labels,
199
+ providerKind: providerRunners.providerKind,
200
+ protocolVersion: providerRunners.protocolVersion,
201
+ capabilities: providerRunners.capabilities,
202
+ state: providerRunners.state,
203
+ })
204
+ .from(providerRunners)
205
+ .where(eq(providerRunners.id, runnerInstanceId));
206
+
207
+ expect(activationToken).toEqual(expect.any(String));
208
+ expect(runner).toEqual({
209
+ workspaceId: reservation.workspaceId,
210
+ reservationId: reservation.id,
211
+ assignedAt: expect.any(Date),
212
+ intendedReservationId: null,
213
+ labels: ['linux'],
214
+ providerKind: 'ec2',
215
+ protocolVersion: '1',
216
+ capabilities,
217
+ state: 'running',
218
+ });
219
+ });
220
+
221
+ it('records an activation-token reason when enrollment sees an existing runner session', async () => {
222
+ const provisionerId = crypto.randomUUID();
223
+ const reservation = await createReservation({provisionerId});
224
+ const runnerInstanceId = await createRunner({
225
+ provisionerId,
226
+ intendedReservationId: reservation.id,
227
+ });
228
+ await db()
229
+ .update(providerRunners)
230
+ .set({runnerSessionId: crypto.randomUUID()})
231
+ .where(eq(providerRunners.id, runnerInstanceId));
232
+ const addSpy = vi.spyOn(runnerActivationTokenNotIssuedCount, 'add');
233
+
234
+ try {
235
+ const activationToken = await enrollRunnerControlSession({
236
+ runnerInstanceId,
237
+ provisionerId,
238
+ labels: ['linux'],
239
+ providerKind: 'docker',
240
+ protocolVersion: '1',
241
+ });
242
+ const [runner] = await db()
243
+ .select({
244
+ workspaceId: providerRunners.workspaceId,
245
+ reservationId: providerRunners.reservationId,
246
+ assignedAt: providerRunners.assignedAt,
247
+ intendedReservationId: providerRunners.intendedReservationId,
248
+ runnerSessionId: providerRunners.runnerSessionId,
249
+ state: providerRunners.state,
250
+ })
251
+ .from(providerRunners)
252
+ .where(eq(providerRunners.id, runnerInstanceId));
253
+
254
+ expect(activationToken).toBeNull();
255
+ expect(runner).toEqual({
256
+ workspaceId: reservation.workspaceId,
257
+ reservationId: reservation.id,
258
+ assignedAt: expect.any(Date),
259
+ intendedReservationId: null,
260
+ runnerSessionId: expect.any(String),
261
+ state: 'running',
262
+ });
263
+ // The test setup uses a shared NoopMeter counter; count only this metric's label shape.
264
+ const activationTokenCalls = addSpy.mock.calls.filter(
265
+ ([value, attributes]) =>
266
+ value === 1 &&
267
+ (attributes as {reason?: string; surface?: string} | undefined)?.reason ===
268
+ 'existing-session' &&
269
+ (attributes as {reason?: string; surface?: string} | undefined)?.surface === 'enrollment',
270
+ );
271
+ expect(activationTokenCalls).toHaveLength(1);
272
+ } finally {
273
+ addSpy.mockRestore();
274
+ }
275
+ });
276
+
277
+ it('updates metadata during enrollment for an unassigned runner', async () => {
278
+ const provisionerId = crypto.randomUUID();
279
+ const capabilities: RunnerToolCapabilitiesDto = {
280
+ harnesses: {pi: {tools: ['read']}},
281
+ };
282
+ const runnerInstanceId = await createRunner({provisionerId});
283
+
284
+ const activationToken = await enrollRunnerControlSession({
285
+ runnerInstanceId,
286
+ provisionerId,
287
+ labels: ['linux'],
288
+ capabilities,
289
+ providerKind: 'docker',
290
+ protocolVersion: '1',
291
+ });
292
+ const [runner] = await db()
293
+ .select({
294
+ reservationId: providerRunners.reservationId,
295
+ labels: providerRunners.labels,
296
+ providerKind: providerRunners.providerKind,
297
+ protocolVersion: providerRunners.protocolVersion,
298
+ capabilities: providerRunners.capabilities,
299
+ state: providerRunners.state,
300
+ })
301
+ .from(providerRunners)
302
+ .where(eq(providerRunners.id, runnerInstanceId));
303
+
304
+ expect(activationToken).toBeNull();
305
+ expect(runner).toEqual({
306
+ reservationId: null,
307
+ labels: ['linux'],
308
+ providerKind: 'docker',
309
+ protocolVersion: '1',
310
+ capabilities,
311
+ state: 'running',
312
+ });
313
+ });
314
+
113
315
  it('keeps unexpected promotion failures on the error-log path', async () => {
114
316
  const provisionerId = crypto.randomUUID();
115
317
  const reservation = await createReservation({provisionerId});
@@ -207,18 +409,42 @@ async function createReservation(params: {
207
409
 
208
410
  async function createRunner(params: {
209
411
  provisionerId: string;
210
- intendedReservationId: string;
211
- reservationId?: string;
412
+ intendedReservationId?: string | null;
413
+ reservationId?: string | null;
414
+ workspaceId?: string | null;
212
415
  labels?: string[];
416
+ providerKind?: string | null;
417
+ protocolVersion?: string | null;
418
+ capabilities?: RunnerToolCapabilitiesDto | null;
419
+ assignedAt?: Date | null;
213
420
  }): Promise<string> {
421
+ const defaultAssignedAt = params.reservationId ? new Date() : null;
422
+
423
+ if (!createdProvisionerTokenIds.has(params.provisionerId)) {
424
+ await db().insert(provisionerTokens).values({
425
+ id: params.provisionerId,
426
+ scope: 'installation',
427
+ workspaceId: null,
428
+ hashedToken: crypto.randomUUID(),
429
+ prefix: 'test',
430
+ createdByUserId: crypto.randomUUID(),
431
+ });
432
+ createdProvisionerTokenIds.add(params.provisionerId);
433
+ }
434
+
214
435
  const [runner] = await db()
215
436
  .insert(providerRunners)
216
437
  .values({
217
438
  provisionerId: params.provisionerId,
218
- intendedReservationId: params.intendedReservationId,
439
+ intendedReservationId: params.intendedReservationId ?? null,
219
440
  reservationId: params.reservationId ?? null,
441
+ workspaceId: params.workspaceId ?? null,
220
442
  providerRunnerId: crypto.randomUUID(),
221
443
  labels: params.labels ?? [],
444
+ providerKind: params.providerKind ?? null,
445
+ protocolVersion: params.protocolVersion ?? null,
446
+ capabilities: params.capabilities ?? null,
447
+ assignedAt: params.assignedAt === undefined ? defaultAssignedAt : params.assignedAt,
222
448
  state: 'starting',
223
449
  reportedAt: new Date(),
224
450
  })
@@ -12,13 +12,17 @@ import {
12
12
  } from '#core/errors.js';
13
13
  import {sanitizeRunnerLabels} from '#core/runner-labels.js';
14
14
  import {db, type schema, type Tx} from '#db/db.js';
15
- import {assignRunnerInstancesTx} from '#db/runner-assignments.js';
16
- import {terminalStates} from '#db/runner-instances.js';
15
+ import {
16
+ assignRunnerInstancesTx,
17
+ validateRunnerReservationCapacityTx,
18
+ } from '#db/runner-assignments.js';
19
+ import {terminalStates} from '#db/runner-states.js';
17
20
  import {provisionerTokens} from '#db/schema/provisioner-tokens.js';
18
21
  import {runnerBootstrapTokens, runnerControlSessions} from '#db/schema/runner-control-sessions.js';
19
22
  import {providerRunners} from '#db/schema/runner-instances.js';
20
23
  import {
21
24
  type RunnerReservationPromotionFailureReason,
25
+ recordRunnerReservationCapacityFailure,
22
26
  recordRunnerReservationPromotionFailure,
23
27
  } from '#metrics/index.js';
24
28
  import {issueRunnerActivationTokenTx} from './runner-activation.js';
@@ -45,27 +49,72 @@ export async function createRunnerInstancesWithBootstrapTokens(params: {
45
49
  reservationId?: string | null;
46
50
  }>;
47
51
  ttlSeconds: number;
48
- }): Promise<Array<{runnerInstanceId: string; bootstrapToken: string}>> {
52
+ }): Promise<{
53
+ runnerInstances: Array<{
54
+ runnerInstanceId: string;
55
+ bootstrapToken: string;
56
+ requestIndex: number;
57
+ }>;
58
+ reservationUnavailable: boolean;
59
+ }> {
49
60
  const expiresAt = new Date(Date.now() + params.ttlSeconds * 1000);
50
61
  return await db().transaction(async (tx) => {
62
+ const reservationValidation = await validateRunnerReservationCapacityTx(tx, {
63
+ provisionerId: params.provisionerId,
64
+ requests: params.runnerInstances.reduce(
65
+ (requests, runner) => {
66
+ if (!runner.reservationId) return requests;
67
+ const request = requests.find(
68
+ (candidate) => candidate.reservationId === runner.reservationId,
69
+ );
70
+ if (request) request.count += 1;
71
+ else requests.push({reservationId: runner.reservationId, count: 1});
72
+ return requests;
73
+ },
74
+ [] as Array<{reservationId: string; count: number}>,
75
+ ),
76
+ });
77
+ for (const {reason, count} of reservationValidation.unavailableByReservation.values())
78
+ recordRunnerReservationCapacityFailure(reason, count);
79
+ const remainingAcceptedByReservation = new Map(reservationValidation.acceptedByReservation);
80
+ const runnerInstances = params.runnerInstances.flatMap((runner, requestIndex) => {
81
+ if (!runner.reservationId) return [{...runner, requestIndex}];
82
+ const remaining = remainingAcceptedByReservation.get(runner.reservationId) ?? 0;
83
+ if (remaining === 0) return [];
84
+ remainingAcceptedByReservation.set(runner.reservationId, remaining - 1);
85
+ return [{...runner, requestIndex}];
86
+ });
87
+
88
+ if (runnerInstances.length === 0)
89
+ return {
90
+ runnerInstances: [],
91
+ reservationUnavailable: reservationValidation.unavailableByReservation.size > 0,
92
+ };
93
+
51
94
  const instances = await tx
52
95
  .insert(providerRunners)
53
96
  .values(
54
- params.runnerInstances.map((runner) => ({
97
+ runnerInstances.map((runner) => ({
55
98
  provisionerId: params.provisionerId,
56
99
  providerKind: params.providerKind ?? null,
57
100
  templateKey: runner.templateKey ?? null,
58
101
  intendedReservationId: runner.reservationId ?? null,
102
+ launchKind: runner.reservationId ? ('demand' as const) : ('warm' as const),
59
103
  state: 'starting' as const,
60
104
  labels: [],
61
105
  reportedAt: new Date(),
62
106
  })),
63
107
  )
64
108
  .returning({id: providerRunners.id});
65
- const results = instances.map((instance) => ({
66
- runnerInstanceId: instance.id,
67
- bootstrapToken: generateOpaqueToken('runnerBootstrapToken'),
68
- }));
109
+ const results = instances.map((instance, index) => {
110
+ const runner = runnerInstances[index];
111
+ if (!runner) throw new Error('Runner instance insert returned an unexpected row count');
112
+ return {
113
+ runnerInstanceId: instance.id,
114
+ bootstrapToken: generateOpaqueToken('runnerBootstrapToken'),
115
+ requestIndex: runner.requestIndex,
116
+ };
117
+ });
69
118
  await tx.insert(runnerBootstrapTokens).values(
70
119
  results.map((result) => ({
71
120
  runnerInstanceId: result.runnerInstanceId,
@@ -75,7 +124,10 @@ export async function createRunnerInstancesWithBootstrapTokens(params: {
75
124
  expiresAt,
76
125
  })),
77
126
  );
78
- return results;
127
+ return {
128
+ runnerInstances: results,
129
+ reservationUnavailable: reservationValidation.unavailableByReservation.size > 0,
130
+ };
79
131
  });
80
132
  }
81
133
 
@@ -163,7 +215,9 @@ export async function enrollRunnerControlSession(params: {
163
215
  sql`select pg_advisory_xact_lock(hashtext(${`runners_assignment:${params.provisionerId}:${current.intendedReservationId}`}))`,
164
216
  );
165
217
 
166
- const [updated] = await tx
218
+ // Provider reports may populate reservationId before the assignment commits. assignedAt is
219
+ // written by the assignment transaction, so keep the guard on that write boundary.
220
+ const [metadataUpdated] = await tx
167
221
  .update(providerRunners)
168
222
  .set({
169
223
  labels: sanitizeRunnerLabels(params.labels, {
@@ -173,6 +227,25 @@ export async function enrollRunnerControlSession(params: {
173
227
  providerKind: params.providerKind,
174
228
  protocolVersion: params.protocolVersion,
175
229
  capabilities: params.capabilities ?? null,
230
+ })
231
+ .where(
232
+ and(
233
+ eq(providerRunners.id, params.runnerInstanceId),
234
+ eq(providerRunners.provisionerId, params.provisionerId),
235
+ isNull(providerRunners.assignedAt),
236
+ notInArray(providerRunners.state, [...terminalStates]),
237
+ ),
238
+ )
239
+ .returning({id: providerRunners.id});
240
+ if (!metadataUpdated)
241
+ logger().debug(
242
+ {runnerInstanceId: params.runnerInstanceId, provisionerId: params.provisionerId},
243
+ 'Runner enrollment metadata update skipped for assigned or terminal runner',
244
+ );
245
+
246
+ const [updated] = await tx
247
+ .update(providerRunners)
248
+ .set({
176
249
  state: 'running',
177
250
  reportedAt: sql`now()`,
178
251
  updatedAt: sql`now()`,
@@ -206,6 +279,7 @@ export async function enrollRunnerControlSession(params: {
206
279
  runnerInstanceId: params.runnerInstanceId,
207
280
  provisionerId: params.provisionerId,
208
281
  ttlSeconds: config.RUNNER_ACTIVATION_TOKEN_TTL_SECONDS,
282
+ surface: 'enrollment',
209
283
  });
210
284
  });
211
285
  } catch (error) {
@@ -54,18 +54,37 @@ describe('reconcileRunnerInstancesFromDbResult', () => {
54
54
 
55
55
  expect(result[0]?.desiredIntent).toBe('terminate');
56
56
  });
57
+
58
+ it('returns the intended reservation when a runner is not assigned yet', () => {
59
+ const intendedReservationId = crypto.randomUUID();
60
+ const result = reconcileRunnerInstancesFromDbResult({
61
+ observedRunnerInstanceIds: ['provisioned-runner-1'],
62
+ observedRows: [
63
+ providerRunner({providerRunnerId: 'provisioned-runner-1', intendedReservationId}),
64
+ ],
65
+ boundJobExecutionsByRunnerInstanceId: new Map(),
66
+ });
67
+
68
+ expect(result[0]).toMatchObject({
69
+ reservationId: null,
70
+ intendedReservationId,
71
+ });
72
+ });
57
73
  });
58
74
 
59
75
  function providerRunner(params: {
60
76
  providerRunnerId: string;
61
77
  state?: 'starting' | 'running' | 'stopping' | 'stopped' | 'failed' | 'terminated';
78
+ intendedReservationId?: string | null;
62
79
  }) {
63
80
  return {
64
81
  id: crypto.randomUUID(),
65
82
  workspaceId: crypto.randomUUID(),
66
83
  provisionerId: crypto.randomUUID(),
67
84
  providerRunnerId: params.providerRunnerId,
85
+ intendedReservationId: params.intendedReservationId ?? null,
68
86
  reservationId: null,
87
+ launchKind: 'manual' as const,
69
88
  templateKey: 'linux',
70
89
  labels: ['linux'],
71
90
  state: params.state ?? 'running',
@@ -62,6 +62,7 @@ export interface ReconciledBoundJobExecution {
62
62
  export interface ReconciledRunnerInstance {
63
63
  providerRunnerId: string;
64
64
  state: RunnerInstanceState | null;
65
+ intendedReservationId: string | null;
65
66
  reservationId: string | null;
66
67
  runnerSessionId: string | null;
67
68
  boundJobExecution: ReconciledBoundJobExecution | null;
@@ -157,6 +158,7 @@ export function reconcileRunnerInstancesFromDbResult(params: {
157
158
  return {
158
159
  providerRunnerId,
159
160
  state: row?.state ?? null,
161
+ intendedReservationId: row?.intendedReservationId ?? null,
160
162
  reservationId: row?.reservationId ?? null,
161
163
  runnerSessionId: row?.runnerSessionId ?? null,
162
164
  boundJobExecution: boundJobExecution
@@ -1,9 +1,15 @@
1
+ import {vi} from '@shipfox/vitest/vi';
1
2
  import {and, eq} from 'drizzle-orm';
2
3
  import {db} from '#db/db.js';
3
4
  import {ephemeralRegistrationTokens} from '#db/schema/ephemeral-registration-tokens.js';
4
5
  import {runnerActivationTokens} from '#db/schema/runner-activation-tokens.js';
6
+ import type {RunnerInstanceInsertDb} from '#db/schema/runner-instances.js';
5
7
  import {providerRunners} from '#db/schema/runner-instances.js';
6
8
  import {runnerSessions} from '#db/schema/runner-sessions.js';
9
+ import {
10
+ type RunnerActivationTokenNotIssuedReason,
11
+ runnerActivationTokenNotIssuedCount,
12
+ } from '#metrics/instance.js';
7
13
  import {
8
14
  ephemeralRegistrationTokenFactory,
9
15
  getRunnerSessionTokenClaims,
@@ -18,9 +24,34 @@ import {
18
24
  RegistrationTokenWorkspaceMismatchError,
19
25
  RunnerLabelsReservedError,
20
26
  } from './errors.js';
21
- import {issueRunnerActivationToken} from './runner-activation.js';
27
+ import {getRunnerAssignment, issueRunnerActivationToken} from './runner-activation.js';
22
28
  import {registerRunnerSession} from './runner-sessions.js';
23
29
 
30
+ const activationTokenNotIssuedCases: Array<{
31
+ reason: RunnerActivationTokenNotIssuedReason;
32
+ update: Partial<RunnerInstanceInsertDb>;
33
+ provisionerId?: string;
34
+ }> = [
35
+ {reason: 'runner-not-found', update: {}, provisionerId: crypto.randomUUID()},
36
+ {reason: 'missing-workspace', update: {workspaceId: null}},
37
+ {reason: 'existing-session', update: {runnerSessionId: crypto.randomUUID()}},
38
+ {reason: 'not-running', update: {state: 'starting'}},
39
+ ];
40
+
41
+ function activationTokenMetricCalls(spy: {mock: {calls: unknown[][]}}): unknown[][] {
42
+ // The test setup uses a shared NoopMeter counter, so filter calls to this metric before counting.
43
+ return spy.mock.calls.filter(([, attributes]) => {
44
+ if (typeof attributes !== 'object' || attributes === null) return false;
45
+ const reason = (attributes as {reason?: unknown}).reason;
46
+ const surface = (attributes as {surface?: unknown}).surface;
47
+ return (
48
+ ['runner-not-found', 'missing-workspace', 'existing-session', 'not-running'].includes(
49
+ String(reason),
50
+ ) && ['enrollment', 'poll'].includes(String(surface))
51
+ );
52
+ });
53
+ }
54
+
24
55
  describe('registerRunnerSession', () => {
25
56
  let workspaceId: string;
26
57
  let registrationTokenId: string;
@@ -245,11 +276,13 @@ describe('activation runner sessions', () => {
245
276
  runnerInstanceId,
246
277
  provisionerId,
247
278
  ttlSeconds: 60,
279
+ surface: 'poll',
248
280
  });
249
281
  const replacementToken = await issueRunnerActivationToken({
250
282
  runnerInstanceId,
251
283
  provisionerId,
252
284
  ttlSeconds: 60,
285
+ surface: 'poll',
253
286
  });
254
287
 
255
288
  const tokens = await db()
@@ -265,11 +298,97 @@ describe('activation runner sessions', () => {
265
298
  expect(tokens.filter((token) => token.revokedAt !== null)).toHaveLength(1);
266
299
  });
267
300
 
301
+ it.each(
302
+ activationTokenNotIssuedCases,
303
+ )('records $reason when direct activation-token issuance is skipped', async ({
304
+ reason,
305
+ update,
306
+ provisionerId: caseProvisionerId,
307
+ }) => {
308
+ const addSpy = vi.spyOn(runnerActivationTokenNotIssuedCount, 'add');
309
+
310
+ try {
311
+ if (Object.keys(update).length > 0) {
312
+ await db()
313
+ .update(providerRunners)
314
+ .set(update)
315
+ .where(eq(providerRunners.id, runnerInstanceId));
316
+ }
317
+
318
+ const activationToken = await issueRunnerActivationToken({
319
+ runnerInstanceId,
320
+ provisionerId: caseProvisionerId ?? provisionerId,
321
+ ttlSeconds: 60,
322
+ surface: 'poll',
323
+ });
324
+
325
+ expect(activationToken).toBeNull();
326
+ expect(activationTokenMetricCalls(addSpy)).toEqual([[1, {reason, surface: 'poll'}]]);
327
+ } finally {
328
+ addSpy.mockRestore();
329
+ }
330
+ });
331
+
332
+ it('does not record a metric when assignment polling finds no assignment', async () => {
333
+ const addSpy = vi.spyOn(runnerActivationTokenNotIssuedCount, 'add');
334
+
335
+ try {
336
+ await db()
337
+ .update(providerRunners)
338
+ .set({workspaceId: null})
339
+ .where(eq(providerRunners.id, runnerInstanceId));
340
+
341
+ const assignment = await getRunnerAssignment({runnerInstanceId, provisionerId});
342
+
343
+ expect(assignment).toBeNull();
344
+ expect(activationTokenMetricCalls(addSpy)).toHaveLength(0);
345
+ } finally {
346
+ addSpy.mockRestore();
347
+ }
348
+ });
349
+
350
+ it('does not poll for an assignment while the runner is not running', async () => {
351
+ const addSpy = vi.spyOn(runnerActivationTokenNotIssuedCount, 'add');
352
+
353
+ try {
354
+ await db()
355
+ .update(providerRunners)
356
+ .set({state: 'starting'})
357
+ .where(eq(providerRunners.id, runnerInstanceId));
358
+
359
+ const assignment = await getRunnerAssignment({runnerInstanceId, provisionerId});
360
+
361
+ expect(assignment).toBeNull();
362
+ expect(activationTokenMetricCalls(addSpy)).toHaveLength(0);
363
+ } finally {
364
+ addSpy.mockRestore();
365
+ }
366
+ });
367
+
368
+ it('does not record a metric when activation-token issuance succeeds', async () => {
369
+ const addSpy = vi.spyOn(runnerActivationTokenNotIssuedCount, 'add');
370
+
371
+ try {
372
+ const activationToken = await issueRunnerActivationToken({
373
+ runnerInstanceId,
374
+ provisionerId,
375
+ ttlSeconds: 60,
376
+ surface: 'poll',
377
+ });
378
+
379
+ expect(activationToken).toEqual(expect.any(String));
380
+ expect(activationTokenMetricCalls(addSpy)).toHaveLength(0);
381
+ } finally {
382
+ addSpy.mockRestore();
383
+ }
384
+ });
385
+
268
386
  it('allows only one concurrent registration to consume an activation token', async () => {
269
387
  const rawToken = await issueRunnerActivationToken({
270
388
  runnerInstanceId,
271
389
  provisionerId,
272
390
  ttlSeconds: 60,
391
+ surface: 'poll',
273
392
  });
274
393
  const [activationToken] = await db()
275
394
  .select()
@@ -314,6 +433,7 @@ describe('activation runner sessions', () => {
314
433
  runnerInstanceId,
315
434
  provisionerId,
316
435
  ttlSeconds: 60,
436
+ surface: 'poll',
317
437
  });
318
438
  const [activationToken] = await db()
319
439
  .select()