@shipfox/api-runners 12.2.0 → 12.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +4 -4
- package/CHANGELOG.md +34 -0
- package/dist/config.d.ts +1 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +10 -3
- package/dist/config.js.map +1 -1
- package/dist/core/demand.d.ts +4 -2
- package/dist/core/demand.d.ts.map +1 -1
- package/dist/core/demand.js +22 -6
- package/dist/core/demand.js.map +1 -1
- package/dist/core/entities/reservation.d.ts +1 -0
- package/dist/core/entities/reservation.d.ts.map +1 -1
- package/dist/core/entities/reservation.js.map +1 -1
- package/dist/core/entities/runner-instance.d.ts +3 -0
- package/dist/core/entities/runner-instance.d.ts.map +1 -1
- package/dist/core/entities/runner-instance.js.map +1 -1
- package/dist/core/runner-activation.d.ts +4 -1
- package/dist/core/runner-activation.d.ts.map +1 -1
- package/dist/core/runner-activation.js +32 -3
- package/dist/core/runner-activation.js.map +1 -1
- package/dist/core/runner-control-sessions.d.ts +8 -4
- package/dist/core/runner-control-sessions.d.ts.map +1 -1
- package/dist/core/runner-control-sessions.js +69 -11
- package/dist/core/runner-control-sessions.js.map +1 -1
- package/dist/core/runner-instances.d.ts +1 -0
- package/dist/core/runner-instances.d.ts.map +1 -1
- package/dist/core/runner-instances.js +1 -0
- package/dist/core/runner-instances.js.map +1 -1
- package/dist/db/db.d.ts +68 -0
- package/dist/db/db.d.ts.map +1 -1
- package/dist/db/ephemeral-registration-tokens.d.ts.map +1 -1
- package/dist/db/ephemeral-registration-tokens.js +1 -1
- package/dist/db/ephemeral-registration-tokens.js.map +1 -1
- package/dist/db/index.d.ts +1 -1
- package/dist/db/index.d.ts.map +1 -1
- package/dist/db/index.js +1 -1
- package/dist/db/index.js.map +1 -1
- package/dist/db/reservation-locks.d.ts +6 -0
- package/dist/db/reservation-locks.d.ts.map +1 -0
- package/dist/db/reservation-locks.js +15 -0
- package/dist/db/reservation-locks.js.map +1 -0
- package/dist/db/reservations.d.ts +17 -3
- package/dist/db/reservations.d.ts.map +1 -1
- package/dist/db/reservations.js +256 -45
- package/dist/db/reservations.js.map +1 -1
- package/dist/db/runner-assignments.d.ts +17 -0
- package/dist/db/runner-assignments.d.ts.map +1 -1
- package/dist/db/runner-assignments.js +103 -8
- package/dist/db/runner-assignments.js.map +1 -1
- package/dist/db/runner-instances.d.ts +5 -2
- package/dist/db/runner-instances.d.ts.map +1 -1
- package/dist/db/runner-instances.js +134 -25
- package/dist/db/runner-instances.js.map +1 -1
- package/dist/db/runner-sessions.d.ts.map +1 -1
- package/dist/db/runner-sessions.js +22 -15
- package/dist/db/runner-sessions.js.map +1 -1
- package/dist/db/runner-states.d.ts +2 -0
- package/dist/db/runner-states.d.ts.map +1 -0
- package/dist/db/runner-states.js +7 -0
- package/dist/db/runner-states.js.map +1 -0
- package/dist/db/schema/reservations.d.ts +18 -0
- package/dist/db/schema/reservations.d.ts.map +1 -1
- package/dist/db/schema/reservations.js +7 -1
- package/dist/db/schema/reservations.js.map +1 -1
- package/dist/db/schema/runner-instances.d.ts +18 -0
- package/dist/db/schema/runner-instances.d.ts.map +1 -1
- package/dist/db/schema/runner-instances.js +8 -0
- package/dist/db/schema/runner-instances.js.map +1 -1
- package/dist/metrics/index.d.ts +2 -2
- package/dist/metrics/index.d.ts.map +1 -1
- package/dist/metrics/index.js +1 -1
- package/dist/metrics/index.js.map +1 -1
- package/dist/metrics/instance.d.ts +24 -2
- package/dist/metrics/instance.d.ts.map +1 -1
- package/dist/metrics/instance.js +23 -0
- package/dist/metrics/instance.js.map +1 -1
- package/dist/metrics/service.d.ts.map +1 -1
- package/dist/metrics/service.js +20 -4
- package/dist/metrics/service.js.map +1 -1
- package/dist/presentation/dto/poll-demand.d.ts.map +1 -1
- package/dist/presentation/dto/poll-demand.js +3 -0
- package/dist/presentation/dto/poll-demand.js.map +1 -1
- package/dist/presentation/dto/runner-instances.d.ts +4 -1
- package/dist/presentation/dto/runner-instances.d.ts.map +1 -1
- package/dist/presentation/dto/runner-instances.js +4 -1
- package/dist/presentation/dto/runner-instances.js.map +1 -1
- package/dist/presentation/routes/assign-runner-instances.d.ts.map +1 -1
- package/dist/presentation/routes/assign-runner-instances.js +3 -3
- package/dist/presentation/routes/assign-runner-instances.js.map +1 -1
- package/dist/presentation/routes/poll-demand.d.ts.map +1 -1
- package/dist/presentation/routes/poll-demand.js +1 -0
- package/dist/presentation/routes/poll-demand.js.map +1 -1
- package/dist/presentation/routes/reconcile-runner-instances.d.ts.map +1 -1
- package/dist/presentation/routes/reconcile-runner-instances.js +4 -2
- package/dist/presentation/routes/reconcile-runner-instances.js.map +1 -1
- package/dist/presentation/routes/runner-enrollment.d.ts +4 -1
- package/dist/presentation/routes/runner-enrollment.d.ts.map +1 -1
- package/dist/presentation/routes/runner-enrollment.js +11 -6
- package/dist/presentation/routes/runner-enrollment.js.map +1 -1
- package/dist/tsconfig.test.tsbuildinfo +1 -1
- package/drizzle/0005_previous_lenny_balinger.sql +5 -0
- package/drizzle/meta/0005_snapshot.json +2220 -0
- package/drizzle/meta/_journal.json +7 -0
- package/package.json +4 -4
- package/src/config.test.ts +24 -2
- package/src/config.ts +16 -3
- package/src/core/demand.test.ts +192 -3
- package/src/core/demand.ts +44 -4
- package/src/core/entities/reservation.ts +1 -0
- package/src/core/entities/runner-instance.ts +4 -0
- package/src/core/runner-activation.ts +48 -2
- package/src/core/runner-control-sessions.test.ts +230 -4
- package/src/core/runner-control-sessions.ts +84 -10
- package/src/core/runner-instances.test.ts +19 -0
- package/src/core/runner-instances.ts +2 -0
- package/src/core/runner-sessions.test.ts +121 -1
- package/src/db/ephemeral-registration-tokens.test.ts +20 -1
- package/src/db/ephemeral-registration-tokens.ts +1 -0
- package/src/db/index.ts +1 -0
- package/src/db/reservation-locks.ts +22 -0
- package/src/db/reservations.test.ts +1110 -31
- package/src/db/reservations.ts +434 -105
- package/src/db/runner-assignments.test.ts +105 -1
- package/src/db/runner-assignments.ts +178 -9
- package/src/db/runner-instances.test.ts +496 -2
- package/src/db/runner-instances.ts +247 -56
- package/src/db/runner-sessions.ts +26 -15
- package/src/db/runner-states.ts +7 -0
- package/src/db/schema/reservations.ts +5 -1
- package/src/db/schema/runner-instances.ts +9 -0
- package/src/metrics/index.ts +10 -1
- package/src/metrics/instance.ts +59 -2
- package/src/metrics/service.test.ts +106 -0
- package/src/metrics/service.ts +24 -4
- package/src/presentation/dto/poll-demand.ts +3 -0
- package/src/presentation/dto/runner-instances.ts +8 -1
- package/src/presentation/routes/assign-runner-instances.ts +6 -2
- package/src/presentation/routes/late-runner-enrollment.test.ts +447 -0
- package/src/presentation/routes/poll-demand.test.ts +79 -6
- package/src/presentation/routes/poll-demand.ts +1 -0
- package/src/presentation/routes/reconcile-runner-instances.test.ts +34 -2
- package/src/presentation/routes/reconcile-runner-instances.ts +7 -1
- package/src/presentation/routes/runner-enrollment.test.ts +316 -3
- package/src/presentation/routes/runner-enrollment.ts +8 -5
- package/test/factories/reservation.ts +2 -0
- package/test/factories/runner-instance.ts +6 -0
- package/test/fixtures/late-runner-enrollment.ts +103 -0
- package/test/index.ts +5 -0
- package/tsconfig.build.tsbuildinfo +1 -1
|
@@ -1,17 +1,21 @@
|
|
|
1
|
+
import type {RunnerToolCapabilitiesDto} from '@shipfox/api-runners-dto';
|
|
1
2
|
import {afterEach, vi} from '@shipfox/vitest/vi';
|
|
2
|
-
import {inArray} from 'drizzle-orm';
|
|
3
|
+
import {eq, inArray} from 'drizzle-orm';
|
|
3
4
|
import {db} from '#db/db.js';
|
|
5
|
+
import {provisionerTokens} from '#db/schema/provisioner-tokens.js';
|
|
4
6
|
import {reservations} from '#db/schema/reservations.js';
|
|
5
7
|
import {runnerActivationTokens} from '#db/schema/runner-activation-tokens.js';
|
|
6
8
|
import {runnerControlSessions} from '#db/schema/runner-control-sessions.js';
|
|
7
9
|
import {providerRunners} from '#db/schema/runner-instances.js';
|
|
8
10
|
import {
|
|
9
11
|
type RunnerReservationPromotionFailureReason,
|
|
12
|
+
runnerActivationTokenNotIssuedCount,
|
|
10
13
|
runnerReservationPromotionFailureCount,
|
|
11
14
|
} from '#metrics/instance.js';
|
|
12
15
|
import {enrollRunnerControlSession} from './runner-control-sessions.js';
|
|
13
16
|
|
|
14
17
|
const createdReservationIds = new Set<string>();
|
|
18
|
+
const createdProvisionerTokenIds = new Set<string>();
|
|
15
19
|
const createdRunnerInstanceIds = new Set<string>();
|
|
16
20
|
|
|
17
21
|
afterEach(async () => {
|
|
@@ -29,9 +33,14 @@ afterEach(async () => {
|
|
|
29
33
|
}
|
|
30
34
|
if (reservationIds.length > 0)
|
|
31
35
|
await db().delete(reservations).where(inArray(reservations.id, reservationIds));
|
|
36
|
+
if (createdProvisionerTokenIds.size > 0)
|
|
37
|
+
await db()
|
|
38
|
+
.delete(provisionerTokens)
|
|
39
|
+
.where(inArray(provisionerTokens.id, [...createdProvisionerTokenIds]));
|
|
32
40
|
|
|
33
41
|
createdRunnerInstanceIds.clear();
|
|
34
42
|
createdReservationIds.clear();
|
|
43
|
+
createdProvisionerTokenIds.clear();
|
|
35
44
|
});
|
|
36
45
|
|
|
37
46
|
describe('enrollRunnerControlSession', () => {
|
|
@@ -110,6 +119,199 @@ describe('enrollRunnerControlSession', () => {
|
|
|
110
119
|
}
|
|
111
120
|
});
|
|
112
121
|
|
|
122
|
+
it('preserves committed assignment metadata during a later enrollment', async () => {
|
|
123
|
+
const provisionerId = crypto.randomUUID();
|
|
124
|
+
const reservation = await createReservation({provisionerId});
|
|
125
|
+
const committedCapabilities: RunnerToolCapabilitiesDto = {
|
|
126
|
+
harnesses: {pi: {tools: ['read']}},
|
|
127
|
+
};
|
|
128
|
+
const runnerInstanceId = await createRunner({
|
|
129
|
+
provisionerId,
|
|
130
|
+
intendedReservationId: reservation.id,
|
|
131
|
+
reservationId: reservation.id,
|
|
132
|
+
workspaceId: reservation.workspaceId,
|
|
133
|
+
assignedAt: new Date(),
|
|
134
|
+
labels: ['linux'],
|
|
135
|
+
providerKind: 'ec2',
|
|
136
|
+
protocolVersion: '1',
|
|
137
|
+
capabilities: committedCapabilities,
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
const activationToken = await enrollRunnerControlSession({
|
|
141
|
+
runnerInstanceId,
|
|
142
|
+
provisionerId,
|
|
143
|
+
labels: ['gpu'],
|
|
144
|
+
capabilities: {harnesses: {claude: {tools: ['bash']}}},
|
|
145
|
+
providerKind: 'docker',
|
|
146
|
+
protocolVersion: '2',
|
|
147
|
+
});
|
|
148
|
+
const [runner] = await db()
|
|
149
|
+
.select({
|
|
150
|
+
reservationId: providerRunners.reservationId,
|
|
151
|
+
labels: providerRunners.labels,
|
|
152
|
+
providerKind: providerRunners.providerKind,
|
|
153
|
+
protocolVersion: providerRunners.protocolVersion,
|
|
154
|
+
capabilities: providerRunners.capabilities,
|
|
155
|
+
state: providerRunners.state,
|
|
156
|
+
})
|
|
157
|
+
.from(providerRunners)
|
|
158
|
+
.where(eq(providerRunners.id, runnerInstanceId));
|
|
159
|
+
|
|
160
|
+
expect(activationToken).toEqual(expect.any(String));
|
|
161
|
+
expect(runner).toEqual({
|
|
162
|
+
reservationId: reservation.id,
|
|
163
|
+
labels: ['linux'],
|
|
164
|
+
providerKind: 'ec2',
|
|
165
|
+
protocolVersion: '1',
|
|
166
|
+
capabilities: committedCapabilities,
|
|
167
|
+
state: 'running',
|
|
168
|
+
});
|
|
169
|
+
});
|
|
170
|
+
|
|
171
|
+
it('repairs assignment metadata when a provider report precedes assignment commit', async () => {
|
|
172
|
+
const provisionerId = crypto.randomUUID();
|
|
173
|
+
const reservation = await createReservation({provisionerId});
|
|
174
|
+
const capabilities: RunnerToolCapabilitiesDto = {
|
|
175
|
+
harnesses: {pi: {tools: ['read']}},
|
|
176
|
+
};
|
|
177
|
+
const runnerInstanceId = await createRunner({
|
|
178
|
+
provisionerId,
|
|
179
|
+
intendedReservationId: reservation.id,
|
|
180
|
+
reservationId: reservation.id,
|
|
181
|
+
assignedAt: null,
|
|
182
|
+
});
|
|
183
|
+
|
|
184
|
+
const activationToken = await enrollRunnerControlSession({
|
|
185
|
+
runnerInstanceId,
|
|
186
|
+
provisionerId,
|
|
187
|
+
labels: ['linux'],
|
|
188
|
+
capabilities,
|
|
189
|
+
providerKind: 'ec2',
|
|
190
|
+
protocolVersion: '1',
|
|
191
|
+
});
|
|
192
|
+
const [runner] = await db()
|
|
193
|
+
.select({
|
|
194
|
+
workspaceId: providerRunners.workspaceId,
|
|
195
|
+
reservationId: providerRunners.reservationId,
|
|
196
|
+
assignedAt: providerRunners.assignedAt,
|
|
197
|
+
intendedReservationId: providerRunners.intendedReservationId,
|
|
198
|
+
labels: providerRunners.labels,
|
|
199
|
+
providerKind: providerRunners.providerKind,
|
|
200
|
+
protocolVersion: providerRunners.protocolVersion,
|
|
201
|
+
capabilities: providerRunners.capabilities,
|
|
202
|
+
state: providerRunners.state,
|
|
203
|
+
})
|
|
204
|
+
.from(providerRunners)
|
|
205
|
+
.where(eq(providerRunners.id, runnerInstanceId));
|
|
206
|
+
|
|
207
|
+
expect(activationToken).toEqual(expect.any(String));
|
|
208
|
+
expect(runner).toEqual({
|
|
209
|
+
workspaceId: reservation.workspaceId,
|
|
210
|
+
reservationId: reservation.id,
|
|
211
|
+
assignedAt: expect.any(Date),
|
|
212
|
+
intendedReservationId: null,
|
|
213
|
+
labels: ['linux'],
|
|
214
|
+
providerKind: 'ec2',
|
|
215
|
+
protocolVersion: '1',
|
|
216
|
+
capabilities,
|
|
217
|
+
state: 'running',
|
|
218
|
+
});
|
|
219
|
+
});
|
|
220
|
+
|
|
221
|
+
it('records an activation-token reason when enrollment sees an existing runner session', async () => {
|
|
222
|
+
const provisionerId = crypto.randomUUID();
|
|
223
|
+
const reservation = await createReservation({provisionerId});
|
|
224
|
+
const runnerInstanceId = await createRunner({
|
|
225
|
+
provisionerId,
|
|
226
|
+
intendedReservationId: reservation.id,
|
|
227
|
+
});
|
|
228
|
+
await db()
|
|
229
|
+
.update(providerRunners)
|
|
230
|
+
.set({runnerSessionId: crypto.randomUUID()})
|
|
231
|
+
.where(eq(providerRunners.id, runnerInstanceId));
|
|
232
|
+
const addSpy = vi.spyOn(runnerActivationTokenNotIssuedCount, 'add');
|
|
233
|
+
|
|
234
|
+
try {
|
|
235
|
+
const activationToken = await enrollRunnerControlSession({
|
|
236
|
+
runnerInstanceId,
|
|
237
|
+
provisionerId,
|
|
238
|
+
labels: ['linux'],
|
|
239
|
+
providerKind: 'docker',
|
|
240
|
+
protocolVersion: '1',
|
|
241
|
+
});
|
|
242
|
+
const [runner] = await db()
|
|
243
|
+
.select({
|
|
244
|
+
workspaceId: providerRunners.workspaceId,
|
|
245
|
+
reservationId: providerRunners.reservationId,
|
|
246
|
+
assignedAt: providerRunners.assignedAt,
|
|
247
|
+
intendedReservationId: providerRunners.intendedReservationId,
|
|
248
|
+
runnerSessionId: providerRunners.runnerSessionId,
|
|
249
|
+
state: providerRunners.state,
|
|
250
|
+
})
|
|
251
|
+
.from(providerRunners)
|
|
252
|
+
.where(eq(providerRunners.id, runnerInstanceId));
|
|
253
|
+
|
|
254
|
+
expect(activationToken).toBeNull();
|
|
255
|
+
expect(runner).toEqual({
|
|
256
|
+
workspaceId: reservation.workspaceId,
|
|
257
|
+
reservationId: reservation.id,
|
|
258
|
+
assignedAt: expect.any(Date),
|
|
259
|
+
intendedReservationId: null,
|
|
260
|
+
runnerSessionId: expect.any(String),
|
|
261
|
+
state: 'running',
|
|
262
|
+
});
|
|
263
|
+
// The test setup uses a shared NoopMeter counter; count only this metric's label shape.
|
|
264
|
+
const activationTokenCalls = addSpy.mock.calls.filter(
|
|
265
|
+
([value, attributes]) =>
|
|
266
|
+
value === 1 &&
|
|
267
|
+
(attributes as {reason?: string; surface?: string} | undefined)?.reason ===
|
|
268
|
+
'existing-session' &&
|
|
269
|
+
(attributes as {reason?: string; surface?: string} | undefined)?.surface === 'enrollment',
|
|
270
|
+
);
|
|
271
|
+
expect(activationTokenCalls).toHaveLength(1);
|
|
272
|
+
} finally {
|
|
273
|
+
addSpy.mockRestore();
|
|
274
|
+
}
|
|
275
|
+
});
|
|
276
|
+
|
|
277
|
+
it('updates metadata during enrollment for an unassigned runner', async () => {
|
|
278
|
+
const provisionerId = crypto.randomUUID();
|
|
279
|
+
const capabilities: RunnerToolCapabilitiesDto = {
|
|
280
|
+
harnesses: {pi: {tools: ['read']}},
|
|
281
|
+
};
|
|
282
|
+
const runnerInstanceId = await createRunner({provisionerId});
|
|
283
|
+
|
|
284
|
+
const activationToken = await enrollRunnerControlSession({
|
|
285
|
+
runnerInstanceId,
|
|
286
|
+
provisionerId,
|
|
287
|
+
labels: ['linux'],
|
|
288
|
+
capabilities,
|
|
289
|
+
providerKind: 'docker',
|
|
290
|
+
protocolVersion: '1',
|
|
291
|
+
});
|
|
292
|
+
const [runner] = await db()
|
|
293
|
+
.select({
|
|
294
|
+
reservationId: providerRunners.reservationId,
|
|
295
|
+
labels: providerRunners.labels,
|
|
296
|
+
providerKind: providerRunners.providerKind,
|
|
297
|
+
protocolVersion: providerRunners.protocolVersion,
|
|
298
|
+
capabilities: providerRunners.capabilities,
|
|
299
|
+
state: providerRunners.state,
|
|
300
|
+
})
|
|
301
|
+
.from(providerRunners)
|
|
302
|
+
.where(eq(providerRunners.id, runnerInstanceId));
|
|
303
|
+
|
|
304
|
+
expect(activationToken).toBeNull();
|
|
305
|
+
expect(runner).toEqual({
|
|
306
|
+
reservationId: null,
|
|
307
|
+
labels: ['linux'],
|
|
308
|
+
providerKind: 'docker',
|
|
309
|
+
protocolVersion: '1',
|
|
310
|
+
capabilities,
|
|
311
|
+
state: 'running',
|
|
312
|
+
});
|
|
313
|
+
});
|
|
314
|
+
|
|
113
315
|
it('keeps unexpected promotion failures on the error-log path', async () => {
|
|
114
316
|
const provisionerId = crypto.randomUUID();
|
|
115
317
|
const reservation = await createReservation({provisionerId});
|
|
@@ -207,18 +409,42 @@ async function createReservation(params: {
|
|
|
207
409
|
|
|
208
410
|
async function createRunner(params: {
|
|
209
411
|
provisionerId: string;
|
|
210
|
-
intendedReservationId
|
|
211
|
-
reservationId?: string;
|
|
412
|
+
intendedReservationId?: string | null;
|
|
413
|
+
reservationId?: string | null;
|
|
414
|
+
workspaceId?: string | null;
|
|
212
415
|
labels?: string[];
|
|
416
|
+
providerKind?: string | null;
|
|
417
|
+
protocolVersion?: string | null;
|
|
418
|
+
capabilities?: RunnerToolCapabilitiesDto | null;
|
|
419
|
+
assignedAt?: Date | null;
|
|
213
420
|
}): Promise<string> {
|
|
421
|
+
const defaultAssignedAt = params.reservationId ? new Date() : null;
|
|
422
|
+
|
|
423
|
+
if (!createdProvisionerTokenIds.has(params.provisionerId)) {
|
|
424
|
+
await db().insert(provisionerTokens).values({
|
|
425
|
+
id: params.provisionerId,
|
|
426
|
+
scope: 'installation',
|
|
427
|
+
workspaceId: null,
|
|
428
|
+
hashedToken: crypto.randomUUID(),
|
|
429
|
+
prefix: 'test',
|
|
430
|
+
createdByUserId: crypto.randomUUID(),
|
|
431
|
+
});
|
|
432
|
+
createdProvisionerTokenIds.add(params.provisionerId);
|
|
433
|
+
}
|
|
434
|
+
|
|
214
435
|
const [runner] = await db()
|
|
215
436
|
.insert(providerRunners)
|
|
216
437
|
.values({
|
|
217
438
|
provisionerId: params.provisionerId,
|
|
218
|
-
intendedReservationId: params.intendedReservationId,
|
|
439
|
+
intendedReservationId: params.intendedReservationId ?? null,
|
|
219
440
|
reservationId: params.reservationId ?? null,
|
|
441
|
+
workspaceId: params.workspaceId ?? null,
|
|
220
442
|
providerRunnerId: crypto.randomUUID(),
|
|
221
443
|
labels: params.labels ?? [],
|
|
444
|
+
providerKind: params.providerKind ?? null,
|
|
445
|
+
protocolVersion: params.protocolVersion ?? null,
|
|
446
|
+
capabilities: params.capabilities ?? null,
|
|
447
|
+
assignedAt: params.assignedAt === undefined ? defaultAssignedAt : params.assignedAt,
|
|
222
448
|
state: 'starting',
|
|
223
449
|
reportedAt: new Date(),
|
|
224
450
|
})
|
|
@@ -12,13 +12,17 @@ import {
|
|
|
12
12
|
} from '#core/errors.js';
|
|
13
13
|
import {sanitizeRunnerLabels} from '#core/runner-labels.js';
|
|
14
14
|
import {db, type schema, type Tx} from '#db/db.js';
|
|
15
|
-
import {
|
|
16
|
-
|
|
15
|
+
import {
|
|
16
|
+
assignRunnerInstancesTx,
|
|
17
|
+
validateRunnerReservationCapacityTx,
|
|
18
|
+
} from '#db/runner-assignments.js';
|
|
19
|
+
import {terminalStates} from '#db/runner-states.js';
|
|
17
20
|
import {provisionerTokens} from '#db/schema/provisioner-tokens.js';
|
|
18
21
|
import {runnerBootstrapTokens, runnerControlSessions} from '#db/schema/runner-control-sessions.js';
|
|
19
22
|
import {providerRunners} from '#db/schema/runner-instances.js';
|
|
20
23
|
import {
|
|
21
24
|
type RunnerReservationPromotionFailureReason,
|
|
25
|
+
recordRunnerReservationCapacityFailure,
|
|
22
26
|
recordRunnerReservationPromotionFailure,
|
|
23
27
|
} from '#metrics/index.js';
|
|
24
28
|
import {issueRunnerActivationTokenTx} from './runner-activation.js';
|
|
@@ -45,27 +49,72 @@ export async function createRunnerInstancesWithBootstrapTokens(params: {
|
|
|
45
49
|
reservationId?: string | null;
|
|
46
50
|
}>;
|
|
47
51
|
ttlSeconds: number;
|
|
48
|
-
}): Promise<
|
|
52
|
+
}): Promise<{
|
|
53
|
+
runnerInstances: Array<{
|
|
54
|
+
runnerInstanceId: string;
|
|
55
|
+
bootstrapToken: string;
|
|
56
|
+
requestIndex: number;
|
|
57
|
+
}>;
|
|
58
|
+
reservationUnavailable: boolean;
|
|
59
|
+
}> {
|
|
49
60
|
const expiresAt = new Date(Date.now() + params.ttlSeconds * 1000);
|
|
50
61
|
return await db().transaction(async (tx) => {
|
|
62
|
+
const reservationValidation = await validateRunnerReservationCapacityTx(tx, {
|
|
63
|
+
provisionerId: params.provisionerId,
|
|
64
|
+
requests: params.runnerInstances.reduce(
|
|
65
|
+
(requests, runner) => {
|
|
66
|
+
if (!runner.reservationId) return requests;
|
|
67
|
+
const request = requests.find(
|
|
68
|
+
(candidate) => candidate.reservationId === runner.reservationId,
|
|
69
|
+
);
|
|
70
|
+
if (request) request.count += 1;
|
|
71
|
+
else requests.push({reservationId: runner.reservationId, count: 1});
|
|
72
|
+
return requests;
|
|
73
|
+
},
|
|
74
|
+
[] as Array<{reservationId: string; count: number}>,
|
|
75
|
+
),
|
|
76
|
+
});
|
|
77
|
+
for (const {reason, count} of reservationValidation.unavailableByReservation.values())
|
|
78
|
+
recordRunnerReservationCapacityFailure(reason, count);
|
|
79
|
+
const remainingAcceptedByReservation = new Map(reservationValidation.acceptedByReservation);
|
|
80
|
+
const runnerInstances = params.runnerInstances.flatMap((runner, requestIndex) => {
|
|
81
|
+
if (!runner.reservationId) return [{...runner, requestIndex}];
|
|
82
|
+
const remaining = remainingAcceptedByReservation.get(runner.reservationId) ?? 0;
|
|
83
|
+
if (remaining === 0) return [];
|
|
84
|
+
remainingAcceptedByReservation.set(runner.reservationId, remaining - 1);
|
|
85
|
+
return [{...runner, requestIndex}];
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
if (runnerInstances.length === 0)
|
|
89
|
+
return {
|
|
90
|
+
runnerInstances: [],
|
|
91
|
+
reservationUnavailable: reservationValidation.unavailableByReservation.size > 0,
|
|
92
|
+
};
|
|
93
|
+
|
|
51
94
|
const instances = await tx
|
|
52
95
|
.insert(providerRunners)
|
|
53
96
|
.values(
|
|
54
|
-
|
|
97
|
+
runnerInstances.map((runner) => ({
|
|
55
98
|
provisionerId: params.provisionerId,
|
|
56
99
|
providerKind: params.providerKind ?? null,
|
|
57
100
|
templateKey: runner.templateKey ?? null,
|
|
58
101
|
intendedReservationId: runner.reservationId ?? null,
|
|
102
|
+
launchKind: runner.reservationId ? ('demand' as const) : ('warm' as const),
|
|
59
103
|
state: 'starting' as const,
|
|
60
104
|
labels: [],
|
|
61
105
|
reportedAt: new Date(),
|
|
62
106
|
})),
|
|
63
107
|
)
|
|
64
108
|
.returning({id: providerRunners.id});
|
|
65
|
-
const results = instances.map((instance) =>
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
109
|
+
const results = instances.map((instance, index) => {
|
|
110
|
+
const runner = runnerInstances[index];
|
|
111
|
+
if (!runner) throw new Error('Runner instance insert returned an unexpected row count');
|
|
112
|
+
return {
|
|
113
|
+
runnerInstanceId: instance.id,
|
|
114
|
+
bootstrapToken: generateOpaqueToken('runnerBootstrapToken'),
|
|
115
|
+
requestIndex: runner.requestIndex,
|
|
116
|
+
};
|
|
117
|
+
});
|
|
69
118
|
await tx.insert(runnerBootstrapTokens).values(
|
|
70
119
|
results.map((result) => ({
|
|
71
120
|
runnerInstanceId: result.runnerInstanceId,
|
|
@@ -75,7 +124,10 @@ export async function createRunnerInstancesWithBootstrapTokens(params: {
|
|
|
75
124
|
expiresAt,
|
|
76
125
|
})),
|
|
77
126
|
);
|
|
78
|
-
return
|
|
127
|
+
return {
|
|
128
|
+
runnerInstances: results,
|
|
129
|
+
reservationUnavailable: reservationValidation.unavailableByReservation.size > 0,
|
|
130
|
+
};
|
|
79
131
|
});
|
|
80
132
|
}
|
|
81
133
|
|
|
@@ -163,7 +215,9 @@ export async function enrollRunnerControlSession(params: {
|
|
|
163
215
|
sql`select pg_advisory_xact_lock(hashtext(${`runners_assignment:${params.provisionerId}:${current.intendedReservationId}`}))`,
|
|
164
216
|
);
|
|
165
217
|
|
|
166
|
-
|
|
218
|
+
// Provider reports may populate reservationId before the assignment commits. assignedAt is
|
|
219
|
+
// written by the assignment transaction, so keep the guard on that write boundary.
|
|
220
|
+
const [metadataUpdated] = await tx
|
|
167
221
|
.update(providerRunners)
|
|
168
222
|
.set({
|
|
169
223
|
labels: sanitizeRunnerLabels(params.labels, {
|
|
@@ -173,6 +227,25 @@ export async function enrollRunnerControlSession(params: {
|
|
|
173
227
|
providerKind: params.providerKind,
|
|
174
228
|
protocolVersion: params.protocolVersion,
|
|
175
229
|
capabilities: params.capabilities ?? null,
|
|
230
|
+
})
|
|
231
|
+
.where(
|
|
232
|
+
and(
|
|
233
|
+
eq(providerRunners.id, params.runnerInstanceId),
|
|
234
|
+
eq(providerRunners.provisionerId, params.provisionerId),
|
|
235
|
+
isNull(providerRunners.assignedAt),
|
|
236
|
+
notInArray(providerRunners.state, [...terminalStates]),
|
|
237
|
+
),
|
|
238
|
+
)
|
|
239
|
+
.returning({id: providerRunners.id});
|
|
240
|
+
if (!metadataUpdated)
|
|
241
|
+
logger().debug(
|
|
242
|
+
{runnerInstanceId: params.runnerInstanceId, provisionerId: params.provisionerId},
|
|
243
|
+
'Runner enrollment metadata update skipped for assigned or terminal runner',
|
|
244
|
+
);
|
|
245
|
+
|
|
246
|
+
const [updated] = await tx
|
|
247
|
+
.update(providerRunners)
|
|
248
|
+
.set({
|
|
176
249
|
state: 'running',
|
|
177
250
|
reportedAt: sql`now()`,
|
|
178
251
|
updatedAt: sql`now()`,
|
|
@@ -206,6 +279,7 @@ export async function enrollRunnerControlSession(params: {
|
|
|
206
279
|
runnerInstanceId: params.runnerInstanceId,
|
|
207
280
|
provisionerId: params.provisionerId,
|
|
208
281
|
ttlSeconds: config.RUNNER_ACTIVATION_TOKEN_TTL_SECONDS,
|
|
282
|
+
surface: 'enrollment',
|
|
209
283
|
});
|
|
210
284
|
});
|
|
211
285
|
} catch (error) {
|
|
@@ -54,18 +54,37 @@ describe('reconcileRunnerInstancesFromDbResult', () => {
|
|
|
54
54
|
|
|
55
55
|
expect(result[0]?.desiredIntent).toBe('terminate');
|
|
56
56
|
});
|
|
57
|
+
|
|
58
|
+
it('returns the intended reservation when a runner is not assigned yet', () => {
|
|
59
|
+
const intendedReservationId = crypto.randomUUID();
|
|
60
|
+
const result = reconcileRunnerInstancesFromDbResult({
|
|
61
|
+
observedRunnerInstanceIds: ['provisioned-runner-1'],
|
|
62
|
+
observedRows: [
|
|
63
|
+
providerRunner({providerRunnerId: 'provisioned-runner-1', intendedReservationId}),
|
|
64
|
+
],
|
|
65
|
+
boundJobExecutionsByRunnerInstanceId: new Map(),
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
expect(result[0]).toMatchObject({
|
|
69
|
+
reservationId: null,
|
|
70
|
+
intendedReservationId,
|
|
71
|
+
});
|
|
72
|
+
});
|
|
57
73
|
});
|
|
58
74
|
|
|
59
75
|
function providerRunner(params: {
|
|
60
76
|
providerRunnerId: string;
|
|
61
77
|
state?: 'starting' | 'running' | 'stopping' | 'stopped' | 'failed' | 'terminated';
|
|
78
|
+
intendedReservationId?: string | null;
|
|
62
79
|
}) {
|
|
63
80
|
return {
|
|
64
81
|
id: crypto.randomUUID(),
|
|
65
82
|
workspaceId: crypto.randomUUID(),
|
|
66
83
|
provisionerId: crypto.randomUUID(),
|
|
67
84
|
providerRunnerId: params.providerRunnerId,
|
|
85
|
+
intendedReservationId: params.intendedReservationId ?? null,
|
|
68
86
|
reservationId: null,
|
|
87
|
+
launchKind: 'manual' as const,
|
|
69
88
|
templateKey: 'linux',
|
|
70
89
|
labels: ['linux'],
|
|
71
90
|
state: params.state ?? 'running',
|
|
@@ -62,6 +62,7 @@ export interface ReconciledBoundJobExecution {
|
|
|
62
62
|
export interface ReconciledRunnerInstance {
|
|
63
63
|
providerRunnerId: string;
|
|
64
64
|
state: RunnerInstanceState | null;
|
|
65
|
+
intendedReservationId: string | null;
|
|
65
66
|
reservationId: string | null;
|
|
66
67
|
runnerSessionId: string | null;
|
|
67
68
|
boundJobExecution: ReconciledBoundJobExecution | null;
|
|
@@ -157,6 +158,7 @@ export function reconcileRunnerInstancesFromDbResult(params: {
|
|
|
157
158
|
return {
|
|
158
159
|
providerRunnerId,
|
|
159
160
|
state: row?.state ?? null,
|
|
161
|
+
intendedReservationId: row?.intendedReservationId ?? null,
|
|
160
162
|
reservationId: row?.reservationId ?? null,
|
|
161
163
|
runnerSessionId: row?.runnerSessionId ?? null,
|
|
162
164
|
boundJobExecution: boundJobExecution
|
|
@@ -1,9 +1,15 @@
|
|
|
1
|
+
import {vi} from '@shipfox/vitest/vi';
|
|
1
2
|
import {and, eq} from 'drizzle-orm';
|
|
2
3
|
import {db} from '#db/db.js';
|
|
3
4
|
import {ephemeralRegistrationTokens} from '#db/schema/ephemeral-registration-tokens.js';
|
|
4
5
|
import {runnerActivationTokens} from '#db/schema/runner-activation-tokens.js';
|
|
6
|
+
import type {RunnerInstanceInsertDb} from '#db/schema/runner-instances.js';
|
|
5
7
|
import {providerRunners} from '#db/schema/runner-instances.js';
|
|
6
8
|
import {runnerSessions} from '#db/schema/runner-sessions.js';
|
|
9
|
+
import {
|
|
10
|
+
type RunnerActivationTokenNotIssuedReason,
|
|
11
|
+
runnerActivationTokenNotIssuedCount,
|
|
12
|
+
} from '#metrics/instance.js';
|
|
7
13
|
import {
|
|
8
14
|
ephemeralRegistrationTokenFactory,
|
|
9
15
|
getRunnerSessionTokenClaims,
|
|
@@ -18,9 +24,34 @@ import {
|
|
|
18
24
|
RegistrationTokenWorkspaceMismatchError,
|
|
19
25
|
RunnerLabelsReservedError,
|
|
20
26
|
} from './errors.js';
|
|
21
|
-
import {issueRunnerActivationToken} from './runner-activation.js';
|
|
27
|
+
import {getRunnerAssignment, issueRunnerActivationToken} from './runner-activation.js';
|
|
22
28
|
import {registerRunnerSession} from './runner-sessions.js';
|
|
23
29
|
|
|
30
|
+
const activationTokenNotIssuedCases: Array<{
|
|
31
|
+
reason: RunnerActivationTokenNotIssuedReason;
|
|
32
|
+
update: Partial<RunnerInstanceInsertDb>;
|
|
33
|
+
provisionerId?: string;
|
|
34
|
+
}> = [
|
|
35
|
+
{reason: 'runner-not-found', update: {}, provisionerId: crypto.randomUUID()},
|
|
36
|
+
{reason: 'missing-workspace', update: {workspaceId: null}},
|
|
37
|
+
{reason: 'existing-session', update: {runnerSessionId: crypto.randomUUID()}},
|
|
38
|
+
{reason: 'not-running', update: {state: 'starting'}},
|
|
39
|
+
];
|
|
40
|
+
|
|
41
|
+
function activationTokenMetricCalls(spy: {mock: {calls: unknown[][]}}): unknown[][] {
|
|
42
|
+
// The test setup uses a shared NoopMeter counter, so filter calls to this metric before counting.
|
|
43
|
+
return spy.mock.calls.filter(([, attributes]) => {
|
|
44
|
+
if (typeof attributes !== 'object' || attributes === null) return false;
|
|
45
|
+
const reason = (attributes as {reason?: unknown}).reason;
|
|
46
|
+
const surface = (attributes as {surface?: unknown}).surface;
|
|
47
|
+
return (
|
|
48
|
+
['runner-not-found', 'missing-workspace', 'existing-session', 'not-running'].includes(
|
|
49
|
+
String(reason),
|
|
50
|
+
) && ['enrollment', 'poll'].includes(String(surface))
|
|
51
|
+
);
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
|
|
24
55
|
describe('registerRunnerSession', () => {
|
|
25
56
|
let workspaceId: string;
|
|
26
57
|
let registrationTokenId: string;
|
|
@@ -245,11 +276,13 @@ describe('activation runner sessions', () => {
|
|
|
245
276
|
runnerInstanceId,
|
|
246
277
|
provisionerId,
|
|
247
278
|
ttlSeconds: 60,
|
|
279
|
+
surface: 'poll',
|
|
248
280
|
});
|
|
249
281
|
const replacementToken = await issueRunnerActivationToken({
|
|
250
282
|
runnerInstanceId,
|
|
251
283
|
provisionerId,
|
|
252
284
|
ttlSeconds: 60,
|
|
285
|
+
surface: 'poll',
|
|
253
286
|
});
|
|
254
287
|
|
|
255
288
|
const tokens = await db()
|
|
@@ -265,11 +298,97 @@ describe('activation runner sessions', () => {
|
|
|
265
298
|
expect(tokens.filter((token) => token.revokedAt !== null)).toHaveLength(1);
|
|
266
299
|
});
|
|
267
300
|
|
|
301
|
+
it.each(
|
|
302
|
+
activationTokenNotIssuedCases,
|
|
303
|
+
)('records $reason when direct activation-token issuance is skipped', async ({
|
|
304
|
+
reason,
|
|
305
|
+
update,
|
|
306
|
+
provisionerId: caseProvisionerId,
|
|
307
|
+
}) => {
|
|
308
|
+
const addSpy = vi.spyOn(runnerActivationTokenNotIssuedCount, 'add');
|
|
309
|
+
|
|
310
|
+
try {
|
|
311
|
+
if (Object.keys(update).length > 0) {
|
|
312
|
+
await db()
|
|
313
|
+
.update(providerRunners)
|
|
314
|
+
.set(update)
|
|
315
|
+
.where(eq(providerRunners.id, runnerInstanceId));
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
const activationToken = await issueRunnerActivationToken({
|
|
319
|
+
runnerInstanceId,
|
|
320
|
+
provisionerId: caseProvisionerId ?? provisionerId,
|
|
321
|
+
ttlSeconds: 60,
|
|
322
|
+
surface: 'poll',
|
|
323
|
+
});
|
|
324
|
+
|
|
325
|
+
expect(activationToken).toBeNull();
|
|
326
|
+
expect(activationTokenMetricCalls(addSpy)).toEqual([[1, {reason, surface: 'poll'}]]);
|
|
327
|
+
} finally {
|
|
328
|
+
addSpy.mockRestore();
|
|
329
|
+
}
|
|
330
|
+
});
|
|
331
|
+
|
|
332
|
+
it('does not record a metric when assignment polling finds no assignment', async () => {
|
|
333
|
+
const addSpy = vi.spyOn(runnerActivationTokenNotIssuedCount, 'add');
|
|
334
|
+
|
|
335
|
+
try {
|
|
336
|
+
await db()
|
|
337
|
+
.update(providerRunners)
|
|
338
|
+
.set({workspaceId: null})
|
|
339
|
+
.where(eq(providerRunners.id, runnerInstanceId));
|
|
340
|
+
|
|
341
|
+
const assignment = await getRunnerAssignment({runnerInstanceId, provisionerId});
|
|
342
|
+
|
|
343
|
+
expect(assignment).toBeNull();
|
|
344
|
+
expect(activationTokenMetricCalls(addSpy)).toHaveLength(0);
|
|
345
|
+
} finally {
|
|
346
|
+
addSpy.mockRestore();
|
|
347
|
+
}
|
|
348
|
+
});
|
|
349
|
+
|
|
350
|
+
it('does not poll for an assignment while the runner is not running', async () => {
|
|
351
|
+
const addSpy = vi.spyOn(runnerActivationTokenNotIssuedCount, 'add');
|
|
352
|
+
|
|
353
|
+
try {
|
|
354
|
+
await db()
|
|
355
|
+
.update(providerRunners)
|
|
356
|
+
.set({state: 'starting'})
|
|
357
|
+
.where(eq(providerRunners.id, runnerInstanceId));
|
|
358
|
+
|
|
359
|
+
const assignment = await getRunnerAssignment({runnerInstanceId, provisionerId});
|
|
360
|
+
|
|
361
|
+
expect(assignment).toBeNull();
|
|
362
|
+
expect(activationTokenMetricCalls(addSpy)).toHaveLength(0);
|
|
363
|
+
} finally {
|
|
364
|
+
addSpy.mockRestore();
|
|
365
|
+
}
|
|
366
|
+
});
|
|
367
|
+
|
|
368
|
+
it('does not record a metric when activation-token issuance succeeds', async () => {
|
|
369
|
+
const addSpy = vi.spyOn(runnerActivationTokenNotIssuedCount, 'add');
|
|
370
|
+
|
|
371
|
+
try {
|
|
372
|
+
const activationToken = await issueRunnerActivationToken({
|
|
373
|
+
runnerInstanceId,
|
|
374
|
+
provisionerId,
|
|
375
|
+
ttlSeconds: 60,
|
|
376
|
+
surface: 'poll',
|
|
377
|
+
});
|
|
378
|
+
|
|
379
|
+
expect(activationToken).toEqual(expect.any(String));
|
|
380
|
+
expect(activationTokenMetricCalls(addSpy)).toHaveLength(0);
|
|
381
|
+
} finally {
|
|
382
|
+
addSpy.mockRestore();
|
|
383
|
+
}
|
|
384
|
+
});
|
|
385
|
+
|
|
268
386
|
it('allows only one concurrent registration to consume an activation token', async () => {
|
|
269
387
|
const rawToken = await issueRunnerActivationToken({
|
|
270
388
|
runnerInstanceId,
|
|
271
389
|
provisionerId,
|
|
272
390
|
ttlSeconds: 60,
|
|
391
|
+
surface: 'poll',
|
|
273
392
|
});
|
|
274
393
|
const [activationToken] = await db()
|
|
275
394
|
.select()
|
|
@@ -314,6 +433,7 @@ describe('activation runner sessions', () => {
|
|
|
314
433
|
runnerInstanceId,
|
|
315
434
|
provisionerId,
|
|
316
435
|
ttlSeconds: 60,
|
|
436
|
+
surface: 'poll',
|
|
317
437
|
});
|
|
318
438
|
const [activationToken] = await db()
|
|
319
439
|
.select()
|