@shipfox/api-runners 12.0.0 → 12.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +4 -4
- package/CHANGELOG.md +30 -0
- package/dist/config.d.ts +3 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +24 -2
- package/dist/config.js.map +1 -1
- package/dist/core/errors.d.ts +4 -0
- package/dist/core/errors.d.ts.map +1 -1
- package/dist/core/errors.js +6 -0
- package/dist/core/errors.js.map +1 -1
- package/dist/core/index.d.ts +1 -1
- package/dist/core/index.d.ts.map +1 -1
- package/dist/core/index.js +1 -1
- package/dist/core/index.js.map +1 -1
- package/dist/core/rate-limit.d.ts.map +1 -1
- package/dist/core/runner-control-sessions.d.ts.map +1 -1
- package/dist/core/runner-control-sessions.js +23 -9
- package/dist/core/runner-control-sessions.js.map +1 -1
- package/dist/core/runner-instances.d.ts +1 -0
- package/dist/core/runner-instances.d.ts.map +1 -1
- package/dist/core/runner-instances.js.map +1 -1
- package/dist/core/runner-labels.d.ts +11 -0
- package/dist/core/runner-labels.d.ts.map +1 -0
- package/dist/core/runner-labels.js +41 -0
- package/dist/core/runner-labels.js.map +1 -0
- package/dist/core/runner-sessions.d.ts.map +1 -1
- package/dist/core/runner-sessions.js +5 -1
- package/dist/core/runner-sessions.js.map +1 -1
- package/dist/db/db.d.ts +64 -64
- package/dist/db/ephemeral-registration-tokens.d.ts.map +1 -1
- package/dist/db/ephemeral-registration-tokens.js +12 -2
- package/dist/db/ephemeral-registration-tokens.js.map +1 -1
- package/dist/db/reservations.d.ts.map +1 -1
- package/dist/db/reservations.js +15 -5
- package/dist/db/reservations.js.map +1 -1
- package/dist/db/runner-instances.d.ts +1 -0
- package/dist/db/runner-instances.d.ts.map +1 -1
- package/dist/db/runner-instances.js +6 -4
- package/dist/db/runner-instances.js.map +1 -1
- package/dist/db/runner-sessions.d.ts.map +1 -1
- package/dist/db/runner-sessions.js +12 -1
- package/dist/db/runner-sessions.js.map +1 -1
- package/dist/db/schema/admin-command-results.d.ts +1 -1
- package/dist/db/schema/ephemeral-registration-tokens.d.ts +1 -1
- package/dist/db/schema/manual-registration-tokens.d.ts +1 -1
- package/dist/db/schema/outbox.d.ts +1 -1
- package/dist/db/schema/pending-job-executions.d.ts +3 -3
- package/dist/db/schema/provisioner-capability-snapshots.d.ts +3 -3
- package/dist/db/schema/provisioner-tokens.d.ts +2 -2
- package/dist/db/schema/rate-limits.d.ts +1 -1
- package/dist/db/schema/reservations.d.ts +3 -3
- package/dist/db/schema/runner-activation-tokens.d.ts +1 -1
- package/dist/db/schema/runner-control-sessions.d.ts +2 -2
- package/dist/db/schema/runner-instances.d.ts +4 -4
- package/dist/db/schema/runner-instances.d.ts.map +1 -1
- package/dist/db/schema/runner-instances.js +1 -0
- package/dist/db/schema/runner-instances.js.map +1 -1
- package/dist/db/schema/runner-sessions.d.ts +4 -4
- package/dist/db/schema/running-job-executions.d.ts +5 -5
- package/dist/metrics/index.d.ts +2 -1
- package/dist/metrics/index.d.ts.map +1 -1
- package/dist/metrics/index.js +1 -1
- package/dist/metrics/index.js.map +1 -1
- package/dist/metrics/instance.d.ts +13 -8
- package/dist/metrics/instance.d.ts.map +1 -1
- package/dist/metrics/instance.js +8 -0
- package/dist/metrics/instance.js.map +1 -1
- package/dist/presentation/dto/admin-provisioner-token.d.ts +1 -1
- package/dist/presentation/dto/admin-provisioner-token.d.ts.map +1 -1
- package/dist/presentation/dto/admin-runner-instances.d.ts +2 -2
- package/dist/presentation/dto/admin-runner-instances.d.ts.map +1 -1
- package/dist/presentation/routes/poll-demand.d.ts.map +1 -1
- package/dist/presentation/routes/poll-demand.js +12 -6
- package/dist/presentation/routes/poll-demand.js.map +1 -1
- package/dist/presentation/routes/rate-limit.d.ts.map +1 -1
- package/dist/presentation/routes/register.d.ts.map +1 -1
- package/dist/presentation/routes/register.js +9 -1
- package/dist/presentation/routes/register.js.map +1 -1
- package/dist/presentation/routes/report-runner-instances.d.ts.map +1 -1
- package/dist/presentation/routes/report-runner-instances.js +1 -0
- package/dist/presentation/routes/report-runner-instances.js.map +1 -1
- package/dist/tsconfig.test.tsbuildinfo +1 -1
- package/drizzle/0004_huge_skreet.sql +1 -0
- package/drizzle/meta/0004_snapshot.json +2194 -0
- package/drizzle/meta/_journal.json +7 -0
- package/package.json +9 -9
- package/src/config.test.ts +81 -0
- package/src/config.ts +39 -2
- package/src/core/errors.ts +9 -0
- package/src/core/index.ts +1 -0
- package/src/core/runner-control-sessions.test.ts +239 -0
- package/src/core/runner-control-sessions.ts +29 -10
- package/src/core/runner-instances.ts +1 -0
- package/src/core/runner-labels.test.ts +53 -0
- package/src/core/runner-labels.ts +51 -0
- package/src/core/runner-sessions.test.ts +34 -1
- package/src/core/runner-sessions.ts +8 -1
- package/src/db/ephemeral-registration-tokens.ts +15 -1
- package/src/db/reservations.test.ts +169 -0
- package/src/db/reservations.ts +42 -8
- package/src/db/runner-instances.test.ts +45 -1
- package/src/db/runner-instances.ts +7 -2
- package/src/db/runner-sessions.ts +14 -1
- package/src/db/schema/runner-instances.ts +3 -0
- package/src/metrics/index.ts +2 -0
- package/src/metrics/instance.ts +18 -0
- package/src/presentation/routes/poll-demand.test.ts +99 -3
- package/src/presentation/routes/poll-demand.ts +18 -6
- package/src/presentation/routes/register.test.ts +116 -2
- package/src/presentation/routes/register.ts +7 -0
- package/src/presentation/routes/report-runner-instances.test.ts +33 -0
- package/src/presentation/routes/report-runner-instances.ts +1 -0
- package/src/presentation/routes/runner-enrollment.test.ts +65 -5
- package/src/temporal/activities/maintenance-activities.test.ts +24 -31
- package/test/env.ts +1 -0
- package/tsconfig.build.tsbuildinfo +1 -1
|
@@ -10,7 +10,7 @@ import {
|
|
|
10
10
|
listProvisionerTerminateIntents,
|
|
11
11
|
reapStaleRunnerInstances,
|
|
12
12
|
reconcileRunnerInstances,
|
|
13
|
-
reportRunnerInstances,
|
|
13
|
+
reportRunnerInstances as reportRunnerInstancesDb,
|
|
14
14
|
} from '#db/runner-instances.js';
|
|
15
15
|
import {provisionerTokens} from '#db/schema/provisioner-tokens.js';
|
|
16
16
|
import {reservations} from '#db/schema/reservations.js';
|
|
@@ -25,6 +25,12 @@ import {
|
|
|
25
25
|
runnerSessionFactory,
|
|
26
26
|
} from '#test/index.js';
|
|
27
27
|
|
|
28
|
+
type ReportRunnerInstancesTestParams = Parameters<typeof reportRunnerInstancesDb>[0];
|
|
29
|
+
|
|
30
|
+
function reportRunnerInstances(params: ReportRunnerInstancesTestParams) {
|
|
31
|
+
return reportRunnerInstancesDb(params);
|
|
32
|
+
}
|
|
33
|
+
|
|
28
34
|
function providerRunnerRowsFor(params: {workspaceId: string; provisionerId: string}) {
|
|
29
35
|
return db()
|
|
30
36
|
.select()
|
|
@@ -93,6 +99,7 @@ describe('reportRunnerInstances', () => {
|
|
|
93
99
|
const reportedAt = new Date();
|
|
94
100
|
|
|
95
101
|
const report = await reportRunnerInstances({
|
|
102
|
+
scope: 'installation',
|
|
96
103
|
workspaceId: null,
|
|
97
104
|
provisionerId,
|
|
98
105
|
events: [event({providerRunnerId: 'installation-runner', reportedAt})],
|
|
@@ -118,6 +125,7 @@ describe('reportRunnerInstances', () => {
|
|
|
118
125
|
const reportedAt = new Date();
|
|
119
126
|
|
|
120
127
|
const result = await reportRunnerInstances({
|
|
128
|
+
scope: 'workspace',
|
|
121
129
|
workspaceId,
|
|
122
130
|
provisionerId,
|
|
123
131
|
events: [
|
|
@@ -140,6 +148,7 @@ describe('reportRunnerInstances', () => {
|
|
|
140
148
|
const reportedAt = new Date();
|
|
141
149
|
|
|
142
150
|
const result = await reportRunnerInstances({
|
|
151
|
+
scope: 'workspace',
|
|
143
152
|
workspaceId,
|
|
144
153
|
provisionerId,
|
|
145
154
|
events: [
|
|
@@ -160,6 +169,7 @@ describe('reportRunnerInstances', () => {
|
|
|
160
169
|
it('accepts delayed events that move the lifecycle forward', async () => {
|
|
161
170
|
const newest = new Date();
|
|
162
171
|
await reportRunnerInstances({
|
|
172
|
+
scope: 'workspace',
|
|
163
173
|
workspaceId,
|
|
164
174
|
provisionerId,
|
|
165
175
|
events: [
|
|
@@ -168,6 +178,7 @@ describe('reportRunnerInstances', () => {
|
|
|
168
178
|
});
|
|
169
179
|
|
|
170
180
|
await reportRunnerInstances({
|
|
181
|
+
scope: 'workspace',
|
|
171
182
|
workspaceId,
|
|
172
183
|
provisionerId,
|
|
173
184
|
events: [
|
|
@@ -188,6 +199,7 @@ describe('reportRunnerInstances', () => {
|
|
|
188
199
|
it('rejects older out-of-order events in the same lifecycle state', async () => {
|
|
189
200
|
const newest = new Date();
|
|
190
201
|
await reportRunnerInstances({
|
|
202
|
+
scope: 'workspace',
|
|
191
203
|
workspaceId,
|
|
192
204
|
provisionerId,
|
|
193
205
|
events: [
|
|
@@ -201,6 +213,7 @@ describe('reportRunnerInstances', () => {
|
|
|
201
213
|
});
|
|
202
214
|
|
|
203
215
|
await reportRunnerInstances({
|
|
216
|
+
scope: 'workspace',
|
|
204
217
|
workspaceId,
|
|
205
218
|
provisionerId,
|
|
206
219
|
events: [
|
|
@@ -222,6 +235,7 @@ describe('reportRunnerInstances', () => {
|
|
|
222
235
|
const reservationId = await createReservation(1);
|
|
223
236
|
const reportedAt = new Date('2025-01-01T00:00:00.000Z');
|
|
224
237
|
await reportRunnerInstances({
|
|
238
|
+
scope: 'workspace',
|
|
225
239
|
workspaceId,
|
|
226
240
|
provisionerId,
|
|
227
241
|
events: [
|
|
@@ -235,6 +249,7 @@ describe('reportRunnerInstances', () => {
|
|
|
235
249
|
});
|
|
236
250
|
|
|
237
251
|
const terminal = await reportRunnerInstances({
|
|
252
|
+
scope: 'workspace',
|
|
238
253
|
workspaceId,
|
|
239
254
|
provisionerId,
|
|
240
255
|
events: [
|
|
@@ -247,6 +262,7 @@ describe('reportRunnerInstances', () => {
|
|
|
247
262
|
],
|
|
248
263
|
});
|
|
249
264
|
const revived = await reportRunnerInstances({
|
|
265
|
+
scope: 'workspace',
|
|
250
266
|
workspaceId,
|
|
251
267
|
provisionerId,
|
|
252
268
|
events: [
|
|
@@ -270,6 +286,7 @@ describe('reportRunnerInstances', () => {
|
|
|
270
286
|
|
|
271
287
|
it('clamps future reported times so they do not pin provisioned runner state', async () => {
|
|
272
288
|
await reportRunnerInstances({
|
|
289
|
+
scope: 'workspace',
|
|
273
290
|
workspaceId,
|
|
274
291
|
provisionerId,
|
|
275
292
|
events: [
|
|
@@ -282,6 +299,7 @@ describe('reportRunnerInstances', () => {
|
|
|
282
299
|
});
|
|
283
300
|
|
|
284
301
|
await reportRunnerInstances({
|
|
302
|
+
scope: 'workspace',
|
|
285
303
|
workspaceId,
|
|
286
304
|
provisionerId,
|
|
287
305
|
events: [
|
|
@@ -305,6 +323,7 @@ describe('reportRunnerInstances', () => {
|
|
|
305
323
|
const terminatedAt = new Date('2025-01-01T00:03:00.000Z');
|
|
306
324
|
|
|
307
325
|
const result = await reportRunnerInstances({
|
|
326
|
+
scope: 'workspace',
|
|
308
327
|
workspaceId,
|
|
309
328
|
provisionerId,
|
|
310
329
|
events: [
|
|
@@ -346,6 +365,7 @@ describe('reportRunnerInstances', () => {
|
|
|
346
365
|
const terminatedAt = new Date('2025-01-01T00:03:00.000Z');
|
|
347
366
|
const startedAt = new Date('2025-01-01T00:00:00.000Z');
|
|
348
367
|
await reportRunnerInstances({
|
|
368
|
+
scope: 'workspace',
|
|
349
369
|
workspaceId,
|
|
350
370
|
provisionerId,
|
|
351
371
|
events: [
|
|
@@ -358,6 +378,7 @@ describe('reportRunnerInstances', () => {
|
|
|
358
378
|
});
|
|
359
379
|
|
|
360
380
|
await reportRunnerInstances({
|
|
381
|
+
scope: 'workspace',
|
|
361
382
|
workspaceId,
|
|
362
383
|
provisionerId,
|
|
363
384
|
events: [
|
|
@@ -378,6 +399,7 @@ describe('reportRunnerInstances', () => {
|
|
|
378
399
|
|
|
379
400
|
it('uses server update time for active provisioned runner windows', async () => {
|
|
380
401
|
await reportRunnerInstances({
|
|
402
|
+
scope: 'workspace',
|
|
381
403
|
workspaceId,
|
|
382
404
|
provisionerId,
|
|
383
405
|
events: [
|
|
@@ -401,6 +423,7 @@ describe('reportRunnerInstances', () => {
|
|
|
401
423
|
const reservationId = await createReservation(2);
|
|
402
424
|
|
|
403
425
|
const result = await reportRunnerInstances({
|
|
426
|
+
scope: 'workspace',
|
|
404
427
|
workspaceId,
|
|
405
428
|
provisionerId,
|
|
406
429
|
events: [event({providerRunnerId: 'provisioned-runner-1', reservationId, state: 'failed'})],
|
|
@@ -429,6 +452,7 @@ describe('reportRunnerInstances', () => {
|
|
|
429
452
|
if (!runner.providerRunnerId) throw new Error('Runner instance provider id missing');
|
|
430
453
|
|
|
431
454
|
const result = await reportRunnerInstances({
|
|
455
|
+
scope: 'workspace',
|
|
432
456
|
workspaceId,
|
|
433
457
|
provisionerId,
|
|
434
458
|
events: [event({providerRunnerId: runner.providerRunnerId, state: 'failed'})],
|
|
@@ -479,6 +503,7 @@ describe('reportRunnerInstances', () => {
|
|
|
479
503
|
});
|
|
480
504
|
|
|
481
505
|
const result = await reportRunnerInstances({
|
|
506
|
+
scope: 'installation',
|
|
482
507
|
workspaceId: null,
|
|
483
508
|
provisionerId: installationProvisioner.id,
|
|
484
509
|
events: [event({providerRunnerId: 'installation-pre-enrollment-runner', state: 'failed'})],
|
|
@@ -521,6 +546,7 @@ describe('reportRunnerInstances', () => {
|
|
|
521
546
|
});
|
|
522
547
|
|
|
523
548
|
const result = await reportRunnerInstances({
|
|
549
|
+
scope: 'installation',
|
|
524
550
|
workspaceId: null,
|
|
525
551
|
provisionerId: installationProvisioner.id,
|
|
526
552
|
events: [
|
|
@@ -552,6 +578,7 @@ describe('reportRunnerInstances', () => {
|
|
|
552
578
|
});
|
|
553
579
|
|
|
554
580
|
const result = await reportRunnerInstances({
|
|
581
|
+
scope: 'workspace',
|
|
555
582
|
workspaceId,
|
|
556
583
|
provisionerId,
|
|
557
584
|
events: [
|
|
@@ -581,11 +608,13 @@ describe('reportRunnerInstances', () => {
|
|
|
581
608
|
const reservationId = await createReservation(2);
|
|
582
609
|
|
|
583
610
|
await reportRunnerInstances({
|
|
611
|
+
scope: 'workspace',
|
|
584
612
|
workspaceId,
|
|
585
613
|
provisionerId,
|
|
586
614
|
events: [event({providerRunnerId: 'provisioned-runner-1', reservationId, state: 'failed'})],
|
|
587
615
|
});
|
|
588
616
|
const result = await reportRunnerInstances({
|
|
617
|
+
scope: 'workspace',
|
|
589
618
|
workspaceId,
|
|
590
619
|
provisionerId,
|
|
591
620
|
events: [event({providerRunnerId: 'provisioned-runner-1', reservationId, state: 'failed'})],
|
|
@@ -600,6 +629,7 @@ describe('reportRunnerInstances', () => {
|
|
|
600
629
|
const reservationId = await createReservation(2);
|
|
601
630
|
const failedAt = new Date();
|
|
602
631
|
await reportRunnerInstances({
|
|
632
|
+
scope: 'workspace',
|
|
603
633
|
workspaceId,
|
|
604
634
|
provisionerId,
|
|
605
635
|
events: [
|
|
@@ -613,6 +643,7 @@ describe('reportRunnerInstances', () => {
|
|
|
613
643
|
});
|
|
614
644
|
|
|
615
645
|
await reportRunnerInstances({
|
|
646
|
+
scope: 'workspace',
|
|
616
647
|
workspaceId,
|
|
617
648
|
provisionerId,
|
|
618
649
|
events: [
|
|
@@ -636,6 +667,7 @@ describe('reportRunnerInstances', () => {
|
|
|
636
667
|
const reservationId = await createReservation(2);
|
|
637
668
|
|
|
638
669
|
const result = await reportRunnerInstances({
|
|
670
|
+
scope: 'workspace',
|
|
639
671
|
workspaceId,
|
|
640
672
|
provisionerId,
|
|
641
673
|
events: [
|
|
@@ -656,6 +688,7 @@ describe('reportRunnerInstances', () => {
|
|
|
656
688
|
const reservationId = await createReservation(3);
|
|
657
689
|
|
|
658
690
|
const result = await reportRunnerInstances({
|
|
691
|
+
scope: 'workspace',
|
|
659
692
|
workspaceId,
|
|
660
693
|
provisionerId,
|
|
661
694
|
events: [
|
|
@@ -673,6 +706,7 @@ describe('reportRunnerInstances', () => {
|
|
|
673
706
|
const reservationId = await createReservation(1);
|
|
674
707
|
|
|
675
708
|
const result = await reportRunnerInstances({
|
|
709
|
+
scope: 'workspace',
|
|
676
710
|
workspaceId,
|
|
677
711
|
provisionerId,
|
|
678
712
|
events: [event({providerRunnerId: 'provisioned-runner-1', reservationId, state: 'failed'})],
|
|
@@ -695,6 +729,7 @@ describe('reportRunnerInstances', () => {
|
|
|
695
729
|
if (!reservation) throw new Error('Expected reservation');
|
|
696
730
|
|
|
697
731
|
const result = await reportRunnerInstances({
|
|
732
|
+
scope: 'workspace',
|
|
698
733
|
workspaceId,
|
|
699
734
|
provisionerId,
|
|
700
735
|
events: [
|
|
@@ -715,6 +750,7 @@ describe('reportRunnerInstances', () => {
|
|
|
715
750
|
const reservationId = await createReservation(1);
|
|
716
751
|
|
|
717
752
|
const result = await reportRunnerInstances({
|
|
753
|
+
scope: 'workspace',
|
|
718
754
|
workspaceId,
|
|
719
755
|
provisionerId,
|
|
720
756
|
events: [
|
|
@@ -750,6 +786,7 @@ describe('reportRunnerInstances', () => {
|
|
|
750
786
|
});
|
|
751
787
|
|
|
752
788
|
const result = await reportRunnerInstances({
|
|
789
|
+
scope: 'workspace',
|
|
753
790
|
workspaceId,
|
|
754
791
|
provisionerId,
|
|
755
792
|
events: [
|
|
@@ -776,6 +813,7 @@ describe('reportRunnerInstances', () => {
|
|
|
776
813
|
const reportedAt = new Date('2025-01-01T00:00:00.000Z');
|
|
777
814
|
|
|
778
815
|
const result = await reportRunnerInstances({
|
|
816
|
+
scope: 'workspace',
|
|
779
817
|
workspaceId,
|
|
780
818
|
provisionerId,
|
|
781
819
|
events: [
|
|
@@ -819,11 +857,13 @@ describe('reportRunnerInstances', () => {
|
|
|
819
857
|
});
|
|
820
858
|
|
|
821
859
|
const first = await reportRunnerInstances({
|
|
860
|
+
scope: 'workspace',
|
|
822
861
|
workspaceId,
|
|
823
862
|
provisionerId,
|
|
824
863
|
events: [event({providerRunnerId: 'provisioned-runner-1', state: 'terminated'})],
|
|
825
864
|
});
|
|
826
865
|
const second = await reportRunnerInstances({
|
|
866
|
+
scope: 'workspace',
|
|
827
867
|
workspaceId,
|
|
828
868
|
provisionerId,
|
|
829
869
|
events: [event({providerRunnerId: 'provisioned-runner-1', state: 'terminated'})],
|
|
@@ -1365,6 +1405,7 @@ describe('reapStaleRunnerInstances', () => {
|
|
|
1365
1405
|
|
|
1366
1406
|
await lockHolderReady.promise;
|
|
1367
1407
|
const report = reportRunnerInstances({
|
|
1408
|
+
scope: 'workspace',
|
|
1368
1409
|
workspaceId,
|
|
1369
1410
|
provisionerId,
|
|
1370
1411
|
events: [
|
|
@@ -1845,6 +1886,7 @@ describe('reconcileRunnerInstances', () => {
|
|
|
1845
1886
|
});
|
|
1846
1887
|
|
|
1847
1888
|
await reportRunnerInstances({
|
|
1889
|
+
scope: 'workspace',
|
|
1848
1890
|
workspaceId,
|
|
1849
1891
|
provisionerId,
|
|
1850
1892
|
events: [
|
|
@@ -1877,6 +1919,7 @@ describe('reconcileRunnerInstances', () => {
|
|
|
1877
1919
|
|
|
1878
1920
|
await lockHolderReady.promise;
|
|
1879
1921
|
const report = reportRunnerInstances({
|
|
1922
|
+
scope: 'workspace',
|
|
1880
1923
|
workspaceId,
|
|
1881
1924
|
provisionerId,
|
|
1882
1925
|
events: [
|
|
@@ -2026,6 +2069,7 @@ describe('runner instance provider attachment', () => {
|
|
|
2026
2069
|
providerRunnerId,
|
|
2027
2070
|
});
|
|
2028
2071
|
await reportRunnerInstances({
|
|
2072
|
+
scope: 'installation',
|
|
2029
2073
|
workspaceId: null,
|
|
2030
2074
|
provisionerId,
|
|
2031
2075
|
events: [
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import {logger} from '@shipfox/node-opentelemetry';
|
|
2
|
-
import {canonicalizeLabels} from '@shipfox/runner-labels';
|
|
3
2
|
import {
|
|
4
3
|
and,
|
|
5
4
|
asc,
|
|
@@ -19,6 +18,7 @@ import {
|
|
|
19
18
|
} from 'drizzle-orm';
|
|
20
19
|
import {alias} from 'drizzle-orm/pg-core';
|
|
21
20
|
import type {RunnerInstance, RunnerInstanceState} from '#core/entities/runner-instance.js';
|
|
21
|
+
import {sanitizeRunnerLabels} from '#core/runner-labels.js';
|
|
22
22
|
import type {Tx} from './db.js';
|
|
23
23
|
import {db} from './db.js';
|
|
24
24
|
import {
|
|
@@ -74,6 +74,7 @@ export interface RunnerInstanceReportEvent {
|
|
|
74
74
|
}
|
|
75
75
|
|
|
76
76
|
export interface ReportRunnerInstancesParams {
|
|
77
|
+
scope: 'installation' | 'workspace';
|
|
77
78
|
workspaceId: string | null;
|
|
78
79
|
provisionerId: string;
|
|
79
80
|
events: RunnerInstanceReportEvent[];
|
|
@@ -165,7 +166,11 @@ export async function reportRunnerInstances(params: ReportRunnerInstancesParams)
|
|
|
165
166
|
providerRunnerId: event.providerRunnerId,
|
|
166
167
|
reservationId: event.reservationId,
|
|
167
168
|
templateKey: event.templateKey,
|
|
168
|
-
labels:
|
|
169
|
+
labels: sanitizeRunnerLabels(event.labels, {
|
|
170
|
+
scope: params.scope,
|
|
171
|
+
logLevel: 'debug',
|
|
172
|
+
source: 'runner instance report',
|
|
173
|
+
}),
|
|
169
174
|
state: event.state,
|
|
170
175
|
reason: event.reason,
|
|
171
176
|
runnerSessionId: event.runnerSessionId,
|
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
import type {RunnerToolCapabilitiesDto} from '@shipfox/api-runners-dto';
|
|
2
2
|
import {and, asc, eq, gt, inArray, isNull, lt, notExists, or, sql} from 'drizzle-orm';
|
|
3
3
|
import type {RunnerSession} from '#core/entities/runner-session.js';
|
|
4
|
+
import {EmptyRunnerLabelsError} from '#core/errors.js';
|
|
5
|
+
import {sanitizeRunnerLabelsOrThrow} from '#core/runner-labels.js';
|
|
4
6
|
import {db} from './db.js';
|
|
7
|
+
import {provisionerTokens} from './schema/provisioner-tokens.js';
|
|
5
8
|
import {runnerActivationTokens} from './schema/runner-activation-tokens.js';
|
|
6
9
|
import {runnerControlSessions} from './schema/runner-control-sessions.js';
|
|
7
10
|
import {providerRunners} from './schema/runner-instances.js';
|
|
@@ -79,6 +82,16 @@ export async function createRunnerSessionConsumingActivationToken(params: {
|
|
|
79
82
|
.for('update');
|
|
80
83
|
if (!token?.workspaceId || !token.providerRunnerId)
|
|
81
84
|
throw new Error('Runner activation token is invalid, expired, or has already been used');
|
|
85
|
+
const [provisioner] = await tx
|
|
86
|
+
.select({scope: provisionerTokens.scope})
|
|
87
|
+
.from(provisionerTokens)
|
|
88
|
+
.where(eq(provisionerTokens.id, token.provisionerId))
|
|
89
|
+
.limit(1);
|
|
90
|
+
const labels = sanitizeRunnerLabelsOrThrow(params.labels, {
|
|
91
|
+
scope: provisioner?.scope ?? 'workspace',
|
|
92
|
+
source: 'activation runner registration',
|
|
93
|
+
});
|
|
94
|
+
if (labels.length === 0) throw new EmptyRunnerLabelsError();
|
|
82
95
|
const [session] = await tx
|
|
83
96
|
.insert(runnerSessions)
|
|
84
97
|
.values({
|
|
@@ -89,7 +102,7 @@ export async function createRunnerSessionConsumingActivationToken(params: {
|
|
|
89
102
|
runnerInstanceId: token.runnerInstanceId,
|
|
90
103
|
provisionerId: token.provisionerId,
|
|
91
104
|
providerRunnerId: token.providerRunnerId,
|
|
92
|
-
labels
|
|
105
|
+
labels,
|
|
93
106
|
toolCapabilities: params.toolCapabilities ?? null,
|
|
94
107
|
toolCapabilitiesReportedAt: params.toolCapabilities ? sql`now()` : null,
|
|
95
108
|
maxClaims: 1,
|
|
@@ -60,6 +60,9 @@ export const providerRunners = pgTable(
|
|
|
60
60
|
.where(
|
|
61
61
|
sql`${table.intendedReservationId} is not null and ${table.reservationReleasedAt} is null`,
|
|
62
62
|
),
|
|
63
|
+
index('runners_runner_instances_intended_reservation_idx')
|
|
64
|
+
.on(table.intendedReservationId)
|
|
65
|
+
.where(sql`${table.intendedReservationId} is not null`),
|
|
63
66
|
index('runners_runner_instances_provisioner_reservation_idx')
|
|
64
67
|
.on(table.provisionerId, table.reservationId)
|
|
65
68
|
.where(sql`${table.reservationId} is not null`),
|
package/src/metrics/index.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
export type {RunnerReservationPromotionFailureReason} from './instance.js';
|
|
1
2
|
export {
|
|
2
3
|
jobExecutionClaimedCount,
|
|
3
4
|
jobExecutionEnqueuedCount,
|
|
@@ -7,6 +8,7 @@ export {
|
|
|
7
8
|
providerRunnerReconcileCallCount,
|
|
8
9
|
providerRunnerTerminateIntentHonoredCount,
|
|
9
10
|
providerRunnerTerminateIntentIssuedCount,
|
|
11
|
+
recordRunnerReservationPromotionFailure,
|
|
10
12
|
recordRunnersRateLimitCheck,
|
|
11
13
|
recordRunnersRateLimitPruneFailure,
|
|
12
14
|
} from './instance.js';
|
package/src/metrics/instance.ts
CHANGED
|
@@ -83,6 +83,18 @@ export const reservationReleasedCount = meter.createCounter<Record<string, never
|
|
|
83
83
|
{description: 'Reservation units released from terminal provisioned runner reports'},
|
|
84
84
|
);
|
|
85
85
|
|
|
86
|
+
export type RunnerReservationPromotionFailureReason =
|
|
87
|
+
| 'reservation-expired'
|
|
88
|
+
| 'reservation-not-found'
|
|
89
|
+
| 'already-assigned'
|
|
90
|
+
| 'not-assignable';
|
|
91
|
+
|
|
92
|
+
export const runnerReservationPromotionFailureCount = meter.createCounter<{
|
|
93
|
+
reason: RunnerReservationPromotionFailureReason;
|
|
94
|
+
}>('runners_reservation_promotion_failures', {
|
|
95
|
+
description: 'Runner reservation promotion failures during enrollment by reason',
|
|
96
|
+
});
|
|
97
|
+
|
|
86
98
|
export type RunnersRateLimitAction = 'provisioner-mint' | 'ephemeral-register';
|
|
87
99
|
export type RunnersRateLimitScope = 'provisioner' | 'ephemeral-token';
|
|
88
100
|
export type RunnersRateLimitOutcome = 'allowed' | 'blocked' | 'unavailable';
|
|
@@ -107,6 +119,12 @@ function recordMetric(record: () => void): void {
|
|
|
107
119
|
}
|
|
108
120
|
}
|
|
109
121
|
|
|
122
|
+
export function recordRunnerReservationPromotionFailure(
|
|
123
|
+
reason: RunnerReservationPromotionFailureReason,
|
|
124
|
+
): void {
|
|
125
|
+
recordMetric(() => runnerReservationPromotionFailureCount.add(1, {reason}));
|
|
126
|
+
}
|
|
127
|
+
|
|
110
128
|
export function recordRunnersRateLimitCheck(params: {
|
|
111
129
|
action: RunnersRateLimitAction;
|
|
112
130
|
scope: RunnersRateLimitScope;
|
|
@@ -87,6 +87,7 @@ describe('POST /provisioners/demand/poll', () => {
|
|
|
87
87
|
it('returns demand stats and reservations when matching demand exists', async () => {
|
|
88
88
|
await pendingJobFactory.create({workspaceId, requiredLabels: ['linux']});
|
|
89
89
|
|
|
90
|
+
const requestStartedAt = Date.now();
|
|
90
91
|
const res = await app.inject({
|
|
91
92
|
method: 'POST',
|
|
92
93
|
url: '/provisioners/demand/poll',
|
|
@@ -102,6 +103,43 @@ describe('POST /provisioners/demand/poll', () => {
|
|
|
102
103
|
});
|
|
103
104
|
expect(res.json().reservations[0].reservation_id).toEqual(expect.any(String));
|
|
104
105
|
expect(res.json().reservations[0].expires_at).toEqual(expect.any(String));
|
|
106
|
+
const expiresAt = Date.parse(res.json().reservations[0].expires_at);
|
|
107
|
+
expect(expiresAt).toBeGreaterThanOrEqual(requestStartedAt + 55_000);
|
|
108
|
+
expect(expiresAt).toBeLessThan(requestStartedAt + 65_000);
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
it('clamps a requested reservation TTL for workspace provisioners', async () => {
|
|
112
|
+
await pendingJobFactory.create({workspaceId, requiredLabels: ['linux']});
|
|
113
|
+
|
|
114
|
+
const requestStartedAt = Date.now();
|
|
115
|
+
const res = await app.inject({
|
|
116
|
+
method: 'POST',
|
|
117
|
+
url: '/provisioners/demand/poll',
|
|
118
|
+
headers: {authorization: `Bearer ${VALID_PROVISIONER_TOKEN}`},
|
|
119
|
+
payload: body({max_reservations: 1, reservation_ttl_seconds: 600}),
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
expect(res.statusCode).toBe(200);
|
|
123
|
+
const expiresAt = Date.parse(res.json().reservations[0].expires_at);
|
|
124
|
+
expect(expiresAt).toBeGreaterThanOrEqual(requestStartedAt + 295_000);
|
|
125
|
+
expect(expiresAt).toBeLessThan(requestStartedAt + 305_000);
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
it('applies a requested reservation TTL below the ceiling', async () => {
|
|
129
|
+
await pendingJobFactory.create({workspaceId, requiredLabels: ['linux']});
|
|
130
|
+
|
|
131
|
+
const requestStartedAt = Date.now();
|
|
132
|
+
const res = await app.inject({
|
|
133
|
+
method: 'POST',
|
|
134
|
+
url: '/provisioners/demand/poll',
|
|
135
|
+
headers: {authorization: `Bearer ${VALID_PROVISIONER_TOKEN}`},
|
|
136
|
+
payload: body({max_reservations: 1, reservation_ttl_seconds: 120}),
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
expect(res.statusCode).toBe(200);
|
|
140
|
+
const expiresAt = Date.parse(res.json().reservations[0].expires_at);
|
|
141
|
+
expect(expiresAt).toBeGreaterThanOrEqual(requestStartedAt + 115_000);
|
|
142
|
+
expect(expiresAt).toBeLessThan(requestStartedAt + 125_000);
|
|
105
143
|
});
|
|
106
144
|
|
|
107
145
|
it('rejects installation provisioner credentials from workspace demand polling', async () => {
|
|
@@ -173,6 +211,39 @@ describe('POST /provisioners/demand/poll', () => {
|
|
|
173
211
|
expect(clearedSnapshots).toEqual([]);
|
|
174
212
|
});
|
|
175
213
|
|
|
214
|
+
it('strips reserved labels from workspace template advertisements', async () => {
|
|
215
|
+
const res = await app.inject({
|
|
216
|
+
method: 'POST',
|
|
217
|
+
url: '/provisioners/demand/poll',
|
|
218
|
+
headers: {authorization: `Bearer ${VALID_PROVISIONER_TOKEN}`},
|
|
219
|
+
payload: {
|
|
220
|
+
wait_seconds: 0,
|
|
221
|
+
max_reservations: 0,
|
|
222
|
+
templates: [
|
|
223
|
+
{
|
|
224
|
+
template_key: 'linux',
|
|
225
|
+
labels: ['linux', 'shipfox-managed'],
|
|
226
|
+
available_slots: 1,
|
|
227
|
+
starting: 0,
|
|
228
|
+
running: 0,
|
|
229
|
+
},
|
|
230
|
+
],
|
|
231
|
+
},
|
|
232
|
+
});
|
|
233
|
+
|
|
234
|
+
const snapshots = await db()
|
|
235
|
+
.select()
|
|
236
|
+
.from(provisionerCapabilitySnapshots)
|
|
237
|
+
.where(eq(provisionerCapabilitySnapshots.provisionerId, provisionerTokenId));
|
|
238
|
+
|
|
239
|
+
expect(res.statusCode).toBe(200);
|
|
240
|
+
expect(snapshots).toEqual([
|
|
241
|
+
expect.objectContaining({
|
|
242
|
+
labels: ['linux'],
|
|
243
|
+
}),
|
|
244
|
+
]);
|
|
245
|
+
});
|
|
246
|
+
|
|
176
247
|
it('returns terminate intent ids for active provisioned runners with cancelled latest jobs', async () => {
|
|
177
248
|
await providerRunnerFactory.create({
|
|
178
249
|
workspaceId,
|
|
@@ -330,10 +401,13 @@ describe('POST /provisioners/demand/poll', () => {
|
|
|
330
401
|
expect(res.statusCode).toBe(401);
|
|
331
402
|
});
|
|
332
403
|
|
|
333
|
-
function body(params: {max_reservations: number}) {
|
|
404
|
+
function body(params: {max_reservations: number; reservation_ttl_seconds?: number}) {
|
|
334
405
|
return {
|
|
335
406
|
wait_seconds: 0,
|
|
336
407
|
max_reservations: params.max_reservations,
|
|
408
|
+
...(params.reservation_ttl_seconds === undefined
|
|
409
|
+
? {}
|
|
410
|
+
: {reservation_ttl_seconds: params.reservation_ttl_seconds}),
|
|
337
411
|
templates: [
|
|
338
412
|
{
|
|
339
413
|
template_key: 'linux',
|
|
@@ -377,6 +451,7 @@ describe('POST /provisioners/demand/poll with installation provisioning configur
|
|
|
377
451
|
let app: FastifyInstance;
|
|
378
452
|
let workspaceId: string;
|
|
379
453
|
let provisionerTokenId: string;
|
|
454
|
+
const filterEligibleWorkspaceIds = vi.fn().mockResolvedValue(new Set<string>());
|
|
380
455
|
|
|
381
456
|
const fakeProvisionerAuth: AuthMethod = {
|
|
382
457
|
name: AUTH_PROVISIONER_TOKEN,
|
|
@@ -402,7 +477,7 @@ describe('POST /provisioners/demand/poll with installation provisioning configur
|
|
|
402
477
|
],
|
|
403
478
|
routes: createRunnerRoutes(runnersTestAuthClient, {
|
|
404
479
|
installationProvisioning: {
|
|
405
|
-
policy: {filterEligibleWorkspaceIds
|
|
480
|
+
policy: {filterEligibleWorkspaceIds},
|
|
406
481
|
},
|
|
407
482
|
}),
|
|
408
483
|
swagger: false,
|
|
@@ -444,10 +519,31 @@ describe('POST /provisioners/demand/poll with installation provisioning configur
|
|
|
444
519
|
expect(res.statusCode).toBe(200);
|
|
445
520
|
});
|
|
446
521
|
|
|
447
|
-
|
|
522
|
+
it('clamps a requested reservation TTL for installation provisioners', async () => {
|
|
523
|
+
await pendingJobFactory.create({workspaceId, requiredLabels: ['linux']});
|
|
524
|
+
filterEligibleWorkspaceIds.mockResolvedValueOnce(new Set([workspaceId]));
|
|
525
|
+
|
|
526
|
+
const requestStartedAt = Date.now();
|
|
527
|
+
const res = await app.inject({
|
|
528
|
+
method: 'POST',
|
|
529
|
+
url: '/provisioners/demand/poll',
|
|
530
|
+
headers: {authorization: `Bearer ${INSTALLATION_PROVISIONER_TOKEN}`},
|
|
531
|
+
payload: body({max_reservations: 1, reservation_ttl_seconds: 600}),
|
|
532
|
+
});
|
|
533
|
+
|
|
534
|
+
expect(res.statusCode).toBe(200);
|
|
535
|
+
const expiresAt = Date.parse(res.json().reservations[0].expires_at);
|
|
536
|
+
expect(expiresAt).toBeGreaterThanOrEqual(requestStartedAt + 295_000);
|
|
537
|
+
expect(expiresAt).toBeLessThan(requestStartedAt + 305_000);
|
|
538
|
+
});
|
|
539
|
+
|
|
540
|
+
function body(params: {max_reservations: number; reservation_ttl_seconds?: number}) {
|
|
448
541
|
return {
|
|
449
542
|
wait_seconds: 0,
|
|
450
543
|
max_reservations: params.max_reservations,
|
|
544
|
+
...(params.reservation_ttl_seconds === undefined
|
|
545
|
+
? {}
|
|
546
|
+
: {reservation_ttl_seconds: params.reservation_ttl_seconds}),
|
|
451
547
|
templates: [
|
|
452
548
|
{
|
|
453
549
|
template_key: 'linux',
|
|
@@ -9,6 +9,7 @@ import {ClientError, defineRoute} from '@shipfox/node-fastify';
|
|
|
9
9
|
import {logger} from '@shipfox/node-opentelemetry';
|
|
10
10
|
import {config} from '#config.js';
|
|
11
11
|
import {pollDemand, releaseReservationGrants} from '#core/demand.js';
|
|
12
|
+
import {sanitizeRunnerLabels} from '#core/runner-labels.js';
|
|
12
13
|
import {publishWorkspaceProvisionerCapabilitySnapshot} from '#db/provisioner-capability-snapshots.js';
|
|
13
14
|
import {
|
|
14
15
|
listQueuedDemandWorkspaceIds,
|
|
@@ -21,10 +22,17 @@ import {toPollDemandResponseDto} from '#presentation/dto/index.js';
|
|
|
21
22
|
|
|
22
23
|
const TERMINATE_INTENT_LIMIT = 1000;
|
|
23
24
|
|
|
24
|
-
function toReservationTemplates(
|
|
25
|
+
function toReservationTemplates(
|
|
26
|
+
templates: PollDemandTemplateDto[],
|
|
27
|
+
scope: 'installation' | 'workspace',
|
|
28
|
+
): ReservationTemplate[] {
|
|
25
29
|
return templates.map((template) => ({
|
|
26
30
|
templateKey: template.template_key,
|
|
27
|
-
labels: template.labels,
|
|
31
|
+
labels: sanitizeRunnerLabels(template.labels, {
|
|
32
|
+
scope,
|
|
33
|
+
logLevel: 'debug',
|
|
34
|
+
source: 'provisioner template advertisement',
|
|
35
|
+
}),
|
|
28
36
|
availableSlots: template.available_slots,
|
|
29
37
|
starting: template.starting,
|
|
30
38
|
running: template.running,
|
|
@@ -66,6 +74,10 @@ export function createPollDemandRoute(options: CreateRunnersModuleOptions = {})
|
|
|
66
74
|
}
|
|
67
75
|
});
|
|
68
76
|
const provisionerContext = requireProvisionerContext(request);
|
|
77
|
+
const ttlSeconds = Math.min(
|
|
78
|
+
request.body.reservation_ttl_seconds ?? config.RESERVATION_TTL_SECONDS,
|
|
79
|
+
config.RESERVATION_TTL_MAX_SECONDS,
|
|
80
|
+
);
|
|
69
81
|
if (provisionerContext.scope === 'installation') {
|
|
70
82
|
if (!options.installationProvisioning) {
|
|
71
83
|
throw new ClientError(
|
|
@@ -81,11 +93,11 @@ export function createPollDemandRoute(options: CreateRunnersModuleOptions = {})
|
|
|
81
93
|
await options.installationProvisioning.policy.filterEligibleWorkspaceIds(
|
|
82
94
|
candidateWorkspaceIds,
|
|
83
95
|
);
|
|
84
|
-
const templates = toReservationTemplates(request.body.templates);
|
|
96
|
+
const templates = toReservationTemplates(request.body.templates, 'installation');
|
|
85
97
|
const result = await pollInstallationDemandAndReserve({
|
|
86
98
|
provisionerId: provisionerContext.provisionerTokenId,
|
|
87
99
|
maxReservations: request.body.max_reservations,
|
|
88
|
-
ttlSeconds
|
|
100
|
+
ttlSeconds,
|
|
89
101
|
templates,
|
|
90
102
|
capabilityWindowSeconds: config.PROVISIONER_ACTIVE_WINDOW_SECONDS,
|
|
91
103
|
eligibleWorkspaceIds,
|
|
@@ -101,7 +113,7 @@ export function createPollDemandRoute(options: CreateRunnersModuleOptions = {})
|
|
|
101
113
|
}
|
|
102
114
|
const {provisionerTokenId, workspaceId} = requireWorkspaceProvisionerContext(request);
|
|
103
115
|
|
|
104
|
-
const templates = toReservationTemplates(request.body.templates);
|
|
116
|
+
const templates = toReservationTemplates(request.body.templates, 'workspace');
|
|
105
117
|
await publishWorkspaceProvisionerCapabilitySnapshot({
|
|
106
118
|
workspaceId,
|
|
107
119
|
provisionerId: provisionerTokenId,
|
|
@@ -113,7 +125,7 @@ export function createPollDemandRoute(options: CreateRunnersModuleOptions = {})
|
|
|
113
125
|
provisionerId: provisionerTokenId,
|
|
114
126
|
maxReservations: request.body.max_reservations,
|
|
115
127
|
waitSeconds: request.body.wait_seconds,
|
|
116
|
-
ttlSeconds
|
|
128
|
+
ttlSeconds,
|
|
117
129
|
terminateIntentLimit: TERMINATE_INTENT_LIMIT,
|
|
118
130
|
templates,
|
|
119
131
|
signal: abortController.signal,
|