@shipfox/api-runners 12.0.0 → 12.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. package/.turbo/turbo-build.log +4 -4
  2. package/CHANGELOG.md +30 -0
  3. package/dist/config.d.ts +3 -0
  4. package/dist/config.d.ts.map +1 -1
  5. package/dist/config.js +24 -2
  6. package/dist/config.js.map +1 -1
  7. package/dist/core/errors.d.ts +4 -0
  8. package/dist/core/errors.d.ts.map +1 -1
  9. package/dist/core/errors.js +6 -0
  10. package/dist/core/errors.js.map +1 -1
  11. package/dist/core/index.d.ts +1 -1
  12. package/dist/core/index.d.ts.map +1 -1
  13. package/dist/core/index.js +1 -1
  14. package/dist/core/index.js.map +1 -1
  15. package/dist/core/rate-limit.d.ts.map +1 -1
  16. package/dist/core/runner-control-sessions.d.ts.map +1 -1
  17. package/dist/core/runner-control-sessions.js +23 -9
  18. package/dist/core/runner-control-sessions.js.map +1 -1
  19. package/dist/core/runner-instances.d.ts +1 -0
  20. package/dist/core/runner-instances.d.ts.map +1 -1
  21. package/dist/core/runner-instances.js.map +1 -1
  22. package/dist/core/runner-labels.d.ts +11 -0
  23. package/dist/core/runner-labels.d.ts.map +1 -0
  24. package/dist/core/runner-labels.js +41 -0
  25. package/dist/core/runner-labels.js.map +1 -0
  26. package/dist/core/runner-sessions.d.ts.map +1 -1
  27. package/dist/core/runner-sessions.js +5 -1
  28. package/dist/core/runner-sessions.js.map +1 -1
  29. package/dist/db/db.d.ts +64 -64
  30. package/dist/db/ephemeral-registration-tokens.d.ts.map +1 -1
  31. package/dist/db/ephemeral-registration-tokens.js +12 -2
  32. package/dist/db/ephemeral-registration-tokens.js.map +1 -1
  33. package/dist/db/reservations.d.ts.map +1 -1
  34. package/dist/db/reservations.js +15 -5
  35. package/dist/db/reservations.js.map +1 -1
  36. package/dist/db/runner-instances.d.ts +1 -0
  37. package/dist/db/runner-instances.d.ts.map +1 -1
  38. package/dist/db/runner-instances.js +6 -4
  39. package/dist/db/runner-instances.js.map +1 -1
  40. package/dist/db/runner-sessions.d.ts.map +1 -1
  41. package/dist/db/runner-sessions.js +12 -1
  42. package/dist/db/runner-sessions.js.map +1 -1
  43. package/dist/db/schema/admin-command-results.d.ts +1 -1
  44. package/dist/db/schema/ephemeral-registration-tokens.d.ts +1 -1
  45. package/dist/db/schema/manual-registration-tokens.d.ts +1 -1
  46. package/dist/db/schema/outbox.d.ts +1 -1
  47. package/dist/db/schema/pending-job-executions.d.ts +3 -3
  48. package/dist/db/schema/provisioner-capability-snapshots.d.ts +3 -3
  49. package/dist/db/schema/provisioner-tokens.d.ts +2 -2
  50. package/dist/db/schema/rate-limits.d.ts +1 -1
  51. package/dist/db/schema/reservations.d.ts +3 -3
  52. package/dist/db/schema/runner-activation-tokens.d.ts +1 -1
  53. package/dist/db/schema/runner-control-sessions.d.ts +2 -2
  54. package/dist/db/schema/runner-instances.d.ts +4 -4
  55. package/dist/db/schema/runner-instances.d.ts.map +1 -1
  56. package/dist/db/schema/runner-instances.js +1 -0
  57. package/dist/db/schema/runner-instances.js.map +1 -1
  58. package/dist/db/schema/runner-sessions.d.ts +4 -4
  59. package/dist/db/schema/running-job-executions.d.ts +5 -5
  60. package/dist/metrics/index.d.ts +2 -1
  61. package/dist/metrics/index.d.ts.map +1 -1
  62. package/dist/metrics/index.js +1 -1
  63. package/dist/metrics/index.js.map +1 -1
  64. package/dist/metrics/instance.d.ts +13 -8
  65. package/dist/metrics/instance.d.ts.map +1 -1
  66. package/dist/metrics/instance.js +8 -0
  67. package/dist/metrics/instance.js.map +1 -1
  68. package/dist/presentation/dto/admin-provisioner-token.d.ts +1 -1
  69. package/dist/presentation/dto/admin-provisioner-token.d.ts.map +1 -1
  70. package/dist/presentation/dto/admin-runner-instances.d.ts +2 -2
  71. package/dist/presentation/dto/admin-runner-instances.d.ts.map +1 -1
  72. package/dist/presentation/routes/poll-demand.d.ts.map +1 -1
  73. package/dist/presentation/routes/poll-demand.js +12 -6
  74. package/dist/presentation/routes/poll-demand.js.map +1 -1
  75. package/dist/presentation/routes/rate-limit.d.ts.map +1 -1
  76. package/dist/presentation/routes/register.d.ts.map +1 -1
  77. package/dist/presentation/routes/register.js +9 -1
  78. package/dist/presentation/routes/register.js.map +1 -1
  79. package/dist/presentation/routes/report-runner-instances.d.ts.map +1 -1
  80. package/dist/presentation/routes/report-runner-instances.js +1 -0
  81. package/dist/presentation/routes/report-runner-instances.js.map +1 -1
  82. package/dist/tsconfig.test.tsbuildinfo +1 -1
  83. package/drizzle/0004_huge_skreet.sql +1 -0
  84. package/drizzle/meta/0004_snapshot.json +2194 -0
  85. package/drizzle/meta/_journal.json +7 -0
  86. package/package.json +9 -9
  87. package/src/config.test.ts +81 -0
  88. package/src/config.ts +39 -2
  89. package/src/core/errors.ts +9 -0
  90. package/src/core/index.ts +1 -0
  91. package/src/core/runner-control-sessions.test.ts +239 -0
  92. package/src/core/runner-control-sessions.ts +29 -10
  93. package/src/core/runner-instances.ts +1 -0
  94. package/src/core/runner-labels.test.ts +53 -0
  95. package/src/core/runner-labels.ts +51 -0
  96. package/src/core/runner-sessions.test.ts +34 -1
  97. package/src/core/runner-sessions.ts +8 -1
  98. package/src/db/ephemeral-registration-tokens.ts +15 -1
  99. package/src/db/reservations.test.ts +169 -0
  100. package/src/db/reservations.ts +42 -8
  101. package/src/db/runner-instances.test.ts +45 -1
  102. package/src/db/runner-instances.ts +7 -2
  103. package/src/db/runner-sessions.ts +14 -1
  104. package/src/db/schema/runner-instances.ts +3 -0
  105. package/src/metrics/index.ts +2 -0
  106. package/src/metrics/instance.ts +18 -0
  107. package/src/presentation/routes/poll-demand.test.ts +99 -3
  108. package/src/presentation/routes/poll-demand.ts +18 -6
  109. package/src/presentation/routes/register.test.ts +116 -2
  110. package/src/presentation/routes/register.ts +7 -0
  111. package/src/presentation/routes/report-runner-instances.test.ts +33 -0
  112. package/src/presentation/routes/report-runner-instances.ts +1 -0
  113. package/src/presentation/routes/runner-enrollment.test.ts +65 -5
  114. package/src/temporal/activities/maintenance-activities.test.ts +24 -31
  115. package/test/env.ts +1 -0
  116. package/tsconfig.build.tsbuildinfo +1 -1
@@ -10,7 +10,7 @@ import {
10
10
  listProvisionerTerminateIntents,
11
11
  reapStaleRunnerInstances,
12
12
  reconcileRunnerInstances,
13
- reportRunnerInstances,
13
+ reportRunnerInstances as reportRunnerInstancesDb,
14
14
  } from '#db/runner-instances.js';
15
15
  import {provisionerTokens} from '#db/schema/provisioner-tokens.js';
16
16
  import {reservations} from '#db/schema/reservations.js';
@@ -25,6 +25,12 @@ import {
25
25
  runnerSessionFactory,
26
26
  } from '#test/index.js';
27
27
 
28
+ type ReportRunnerInstancesTestParams = Parameters<typeof reportRunnerInstancesDb>[0];
29
+
30
+ function reportRunnerInstances(params: ReportRunnerInstancesTestParams) {
31
+ return reportRunnerInstancesDb(params);
32
+ }
33
+
28
34
  function providerRunnerRowsFor(params: {workspaceId: string; provisionerId: string}) {
29
35
  return db()
30
36
  .select()
@@ -93,6 +99,7 @@ describe('reportRunnerInstances', () => {
93
99
  const reportedAt = new Date();
94
100
 
95
101
  const report = await reportRunnerInstances({
102
+ scope: 'installation',
96
103
  workspaceId: null,
97
104
  provisionerId,
98
105
  events: [event({providerRunnerId: 'installation-runner', reportedAt})],
@@ -118,6 +125,7 @@ describe('reportRunnerInstances', () => {
118
125
  const reportedAt = new Date();
119
126
 
120
127
  const result = await reportRunnerInstances({
128
+ scope: 'workspace',
121
129
  workspaceId,
122
130
  provisionerId,
123
131
  events: [
@@ -140,6 +148,7 @@ describe('reportRunnerInstances', () => {
140
148
  const reportedAt = new Date();
141
149
 
142
150
  const result = await reportRunnerInstances({
151
+ scope: 'workspace',
143
152
  workspaceId,
144
153
  provisionerId,
145
154
  events: [
@@ -160,6 +169,7 @@ describe('reportRunnerInstances', () => {
160
169
  it('accepts delayed events that move the lifecycle forward', async () => {
161
170
  const newest = new Date();
162
171
  await reportRunnerInstances({
172
+ scope: 'workspace',
163
173
  workspaceId,
164
174
  provisionerId,
165
175
  events: [
@@ -168,6 +178,7 @@ describe('reportRunnerInstances', () => {
168
178
  });
169
179
 
170
180
  await reportRunnerInstances({
181
+ scope: 'workspace',
171
182
  workspaceId,
172
183
  provisionerId,
173
184
  events: [
@@ -188,6 +199,7 @@ describe('reportRunnerInstances', () => {
188
199
  it('rejects older out-of-order events in the same lifecycle state', async () => {
189
200
  const newest = new Date();
190
201
  await reportRunnerInstances({
202
+ scope: 'workspace',
191
203
  workspaceId,
192
204
  provisionerId,
193
205
  events: [
@@ -201,6 +213,7 @@ describe('reportRunnerInstances', () => {
201
213
  });
202
214
 
203
215
  await reportRunnerInstances({
216
+ scope: 'workspace',
204
217
  workspaceId,
205
218
  provisionerId,
206
219
  events: [
@@ -222,6 +235,7 @@ describe('reportRunnerInstances', () => {
222
235
  const reservationId = await createReservation(1);
223
236
  const reportedAt = new Date('2025-01-01T00:00:00.000Z');
224
237
  await reportRunnerInstances({
238
+ scope: 'workspace',
225
239
  workspaceId,
226
240
  provisionerId,
227
241
  events: [
@@ -235,6 +249,7 @@ describe('reportRunnerInstances', () => {
235
249
  });
236
250
 
237
251
  const terminal = await reportRunnerInstances({
252
+ scope: 'workspace',
238
253
  workspaceId,
239
254
  provisionerId,
240
255
  events: [
@@ -247,6 +262,7 @@ describe('reportRunnerInstances', () => {
247
262
  ],
248
263
  });
249
264
  const revived = await reportRunnerInstances({
265
+ scope: 'workspace',
250
266
  workspaceId,
251
267
  provisionerId,
252
268
  events: [
@@ -270,6 +286,7 @@ describe('reportRunnerInstances', () => {
270
286
 
271
287
  it('clamps future reported times so they do not pin provisioned runner state', async () => {
272
288
  await reportRunnerInstances({
289
+ scope: 'workspace',
273
290
  workspaceId,
274
291
  provisionerId,
275
292
  events: [
@@ -282,6 +299,7 @@ describe('reportRunnerInstances', () => {
282
299
  });
283
300
 
284
301
  await reportRunnerInstances({
302
+ scope: 'workspace',
285
303
  workspaceId,
286
304
  provisionerId,
287
305
  events: [
@@ -305,6 +323,7 @@ describe('reportRunnerInstances', () => {
305
323
  const terminatedAt = new Date('2025-01-01T00:03:00.000Z');
306
324
 
307
325
  const result = await reportRunnerInstances({
326
+ scope: 'workspace',
308
327
  workspaceId,
309
328
  provisionerId,
310
329
  events: [
@@ -346,6 +365,7 @@ describe('reportRunnerInstances', () => {
346
365
  const terminatedAt = new Date('2025-01-01T00:03:00.000Z');
347
366
  const startedAt = new Date('2025-01-01T00:00:00.000Z');
348
367
  await reportRunnerInstances({
368
+ scope: 'workspace',
349
369
  workspaceId,
350
370
  provisionerId,
351
371
  events: [
@@ -358,6 +378,7 @@ describe('reportRunnerInstances', () => {
358
378
  });
359
379
 
360
380
  await reportRunnerInstances({
381
+ scope: 'workspace',
361
382
  workspaceId,
362
383
  provisionerId,
363
384
  events: [
@@ -378,6 +399,7 @@ describe('reportRunnerInstances', () => {
378
399
 
379
400
  it('uses server update time for active provisioned runner windows', async () => {
380
401
  await reportRunnerInstances({
402
+ scope: 'workspace',
381
403
  workspaceId,
382
404
  provisionerId,
383
405
  events: [
@@ -401,6 +423,7 @@ describe('reportRunnerInstances', () => {
401
423
  const reservationId = await createReservation(2);
402
424
 
403
425
  const result = await reportRunnerInstances({
426
+ scope: 'workspace',
404
427
  workspaceId,
405
428
  provisionerId,
406
429
  events: [event({providerRunnerId: 'provisioned-runner-1', reservationId, state: 'failed'})],
@@ -429,6 +452,7 @@ describe('reportRunnerInstances', () => {
429
452
  if (!runner.providerRunnerId) throw new Error('Runner instance provider id missing');
430
453
 
431
454
  const result = await reportRunnerInstances({
455
+ scope: 'workspace',
432
456
  workspaceId,
433
457
  provisionerId,
434
458
  events: [event({providerRunnerId: runner.providerRunnerId, state: 'failed'})],
@@ -479,6 +503,7 @@ describe('reportRunnerInstances', () => {
479
503
  });
480
504
 
481
505
  const result = await reportRunnerInstances({
506
+ scope: 'installation',
482
507
  workspaceId: null,
483
508
  provisionerId: installationProvisioner.id,
484
509
  events: [event({providerRunnerId: 'installation-pre-enrollment-runner', state: 'failed'})],
@@ -521,6 +546,7 @@ describe('reportRunnerInstances', () => {
521
546
  });
522
547
 
523
548
  const result = await reportRunnerInstances({
549
+ scope: 'installation',
524
550
  workspaceId: null,
525
551
  provisionerId: installationProvisioner.id,
526
552
  events: [
@@ -552,6 +578,7 @@ describe('reportRunnerInstances', () => {
552
578
  });
553
579
 
554
580
  const result = await reportRunnerInstances({
581
+ scope: 'workspace',
555
582
  workspaceId,
556
583
  provisionerId,
557
584
  events: [
@@ -581,11 +608,13 @@ describe('reportRunnerInstances', () => {
581
608
  const reservationId = await createReservation(2);
582
609
 
583
610
  await reportRunnerInstances({
611
+ scope: 'workspace',
584
612
  workspaceId,
585
613
  provisionerId,
586
614
  events: [event({providerRunnerId: 'provisioned-runner-1', reservationId, state: 'failed'})],
587
615
  });
588
616
  const result = await reportRunnerInstances({
617
+ scope: 'workspace',
589
618
  workspaceId,
590
619
  provisionerId,
591
620
  events: [event({providerRunnerId: 'provisioned-runner-1', reservationId, state: 'failed'})],
@@ -600,6 +629,7 @@ describe('reportRunnerInstances', () => {
600
629
  const reservationId = await createReservation(2);
601
630
  const failedAt = new Date();
602
631
  await reportRunnerInstances({
632
+ scope: 'workspace',
603
633
  workspaceId,
604
634
  provisionerId,
605
635
  events: [
@@ -613,6 +643,7 @@ describe('reportRunnerInstances', () => {
613
643
  });
614
644
 
615
645
  await reportRunnerInstances({
646
+ scope: 'workspace',
616
647
  workspaceId,
617
648
  provisionerId,
618
649
  events: [
@@ -636,6 +667,7 @@ describe('reportRunnerInstances', () => {
636
667
  const reservationId = await createReservation(2);
637
668
 
638
669
  const result = await reportRunnerInstances({
670
+ scope: 'workspace',
639
671
  workspaceId,
640
672
  provisionerId,
641
673
  events: [
@@ -656,6 +688,7 @@ describe('reportRunnerInstances', () => {
656
688
  const reservationId = await createReservation(3);
657
689
 
658
690
  const result = await reportRunnerInstances({
691
+ scope: 'workspace',
659
692
  workspaceId,
660
693
  provisionerId,
661
694
  events: [
@@ -673,6 +706,7 @@ describe('reportRunnerInstances', () => {
673
706
  const reservationId = await createReservation(1);
674
707
 
675
708
  const result = await reportRunnerInstances({
709
+ scope: 'workspace',
676
710
  workspaceId,
677
711
  provisionerId,
678
712
  events: [event({providerRunnerId: 'provisioned-runner-1', reservationId, state: 'failed'})],
@@ -695,6 +729,7 @@ describe('reportRunnerInstances', () => {
695
729
  if (!reservation) throw new Error('Expected reservation');
696
730
 
697
731
  const result = await reportRunnerInstances({
732
+ scope: 'workspace',
698
733
  workspaceId,
699
734
  provisionerId,
700
735
  events: [
@@ -715,6 +750,7 @@ describe('reportRunnerInstances', () => {
715
750
  const reservationId = await createReservation(1);
716
751
 
717
752
  const result = await reportRunnerInstances({
753
+ scope: 'workspace',
718
754
  workspaceId,
719
755
  provisionerId,
720
756
  events: [
@@ -750,6 +786,7 @@ describe('reportRunnerInstances', () => {
750
786
  });
751
787
 
752
788
  const result = await reportRunnerInstances({
789
+ scope: 'workspace',
753
790
  workspaceId,
754
791
  provisionerId,
755
792
  events: [
@@ -776,6 +813,7 @@ describe('reportRunnerInstances', () => {
776
813
  const reportedAt = new Date('2025-01-01T00:00:00.000Z');
777
814
 
778
815
  const result = await reportRunnerInstances({
816
+ scope: 'workspace',
779
817
  workspaceId,
780
818
  provisionerId,
781
819
  events: [
@@ -819,11 +857,13 @@ describe('reportRunnerInstances', () => {
819
857
  });
820
858
 
821
859
  const first = await reportRunnerInstances({
860
+ scope: 'workspace',
822
861
  workspaceId,
823
862
  provisionerId,
824
863
  events: [event({providerRunnerId: 'provisioned-runner-1', state: 'terminated'})],
825
864
  });
826
865
  const second = await reportRunnerInstances({
866
+ scope: 'workspace',
827
867
  workspaceId,
828
868
  provisionerId,
829
869
  events: [event({providerRunnerId: 'provisioned-runner-1', state: 'terminated'})],
@@ -1365,6 +1405,7 @@ describe('reapStaleRunnerInstances', () => {
1365
1405
 
1366
1406
  await lockHolderReady.promise;
1367
1407
  const report = reportRunnerInstances({
1408
+ scope: 'workspace',
1368
1409
  workspaceId,
1369
1410
  provisionerId,
1370
1411
  events: [
@@ -1845,6 +1886,7 @@ describe('reconcileRunnerInstances', () => {
1845
1886
  });
1846
1887
 
1847
1888
  await reportRunnerInstances({
1889
+ scope: 'workspace',
1848
1890
  workspaceId,
1849
1891
  provisionerId,
1850
1892
  events: [
@@ -1877,6 +1919,7 @@ describe('reconcileRunnerInstances', () => {
1877
1919
 
1878
1920
  await lockHolderReady.promise;
1879
1921
  const report = reportRunnerInstances({
1922
+ scope: 'workspace',
1880
1923
  workspaceId,
1881
1924
  provisionerId,
1882
1925
  events: [
@@ -2026,6 +2069,7 @@ describe('runner instance provider attachment', () => {
2026
2069
  providerRunnerId,
2027
2070
  });
2028
2071
  await reportRunnerInstances({
2072
+ scope: 'installation',
2029
2073
  workspaceId: null,
2030
2074
  provisionerId,
2031
2075
  events: [
@@ -1,5 +1,4 @@
1
1
  import {logger} from '@shipfox/node-opentelemetry';
2
- import {canonicalizeLabels} from '@shipfox/runner-labels';
3
2
  import {
4
3
  and,
5
4
  asc,
@@ -19,6 +18,7 @@ import {
19
18
  } from 'drizzle-orm';
20
19
  import {alias} from 'drizzle-orm/pg-core';
21
20
  import type {RunnerInstance, RunnerInstanceState} from '#core/entities/runner-instance.js';
21
+ import {sanitizeRunnerLabels} from '#core/runner-labels.js';
22
22
  import type {Tx} from './db.js';
23
23
  import {db} from './db.js';
24
24
  import {
@@ -74,6 +74,7 @@ export interface RunnerInstanceReportEvent {
74
74
  }
75
75
 
76
76
  export interface ReportRunnerInstancesParams {
77
+ scope: 'installation' | 'workspace';
77
78
  workspaceId: string | null;
78
79
  provisionerId: string;
79
80
  events: RunnerInstanceReportEvent[];
@@ -165,7 +166,11 @@ export async function reportRunnerInstances(params: ReportRunnerInstancesParams)
165
166
  providerRunnerId: event.providerRunnerId,
166
167
  reservationId: event.reservationId,
167
168
  templateKey: event.templateKey,
168
- labels: [...canonicalizeLabels(event.labels)],
169
+ labels: sanitizeRunnerLabels(event.labels, {
170
+ scope: params.scope,
171
+ logLevel: 'debug',
172
+ source: 'runner instance report',
173
+ }),
169
174
  state: event.state,
170
175
  reason: event.reason,
171
176
  runnerSessionId: event.runnerSessionId,
@@ -1,7 +1,10 @@
1
1
  import type {RunnerToolCapabilitiesDto} from '@shipfox/api-runners-dto';
2
2
  import {and, asc, eq, gt, inArray, isNull, lt, notExists, or, sql} from 'drizzle-orm';
3
3
  import type {RunnerSession} from '#core/entities/runner-session.js';
4
+ import {EmptyRunnerLabelsError} from '#core/errors.js';
5
+ import {sanitizeRunnerLabelsOrThrow} from '#core/runner-labels.js';
4
6
  import {db} from './db.js';
7
+ import {provisionerTokens} from './schema/provisioner-tokens.js';
5
8
  import {runnerActivationTokens} from './schema/runner-activation-tokens.js';
6
9
  import {runnerControlSessions} from './schema/runner-control-sessions.js';
7
10
  import {providerRunners} from './schema/runner-instances.js';
@@ -79,6 +82,16 @@ export async function createRunnerSessionConsumingActivationToken(params: {
79
82
  .for('update');
80
83
  if (!token?.workspaceId || !token.providerRunnerId)
81
84
  throw new Error('Runner activation token is invalid, expired, or has already been used');
85
+ const [provisioner] = await tx
86
+ .select({scope: provisionerTokens.scope})
87
+ .from(provisionerTokens)
88
+ .where(eq(provisionerTokens.id, token.provisionerId))
89
+ .limit(1);
90
+ const labels = sanitizeRunnerLabelsOrThrow(params.labels, {
91
+ scope: provisioner?.scope ?? 'workspace',
92
+ source: 'activation runner registration',
93
+ });
94
+ if (labels.length === 0) throw new EmptyRunnerLabelsError();
82
95
  const [session] = await tx
83
96
  .insert(runnerSessions)
84
97
  .values({
@@ -89,7 +102,7 @@ export async function createRunnerSessionConsumingActivationToken(params: {
89
102
  runnerInstanceId: token.runnerInstanceId,
90
103
  provisionerId: token.provisionerId,
91
104
  providerRunnerId: token.providerRunnerId,
92
- labels: params.labels,
105
+ labels,
93
106
  toolCapabilities: params.toolCapabilities ?? null,
94
107
  toolCapabilitiesReportedAt: params.toolCapabilities ? sql`now()` : null,
95
108
  maxClaims: 1,
@@ -60,6 +60,9 @@ export const providerRunners = pgTable(
60
60
  .where(
61
61
  sql`${table.intendedReservationId} is not null and ${table.reservationReleasedAt} is null`,
62
62
  ),
63
+ index('runners_runner_instances_intended_reservation_idx')
64
+ .on(table.intendedReservationId)
65
+ .where(sql`${table.intendedReservationId} is not null`),
63
66
  index('runners_runner_instances_provisioner_reservation_idx')
64
67
  .on(table.provisionerId, table.reservationId)
65
68
  .where(sql`${table.reservationId} is not null`),
@@ -1,3 +1,4 @@
1
+ export type {RunnerReservationPromotionFailureReason} from './instance.js';
1
2
  export {
2
3
  jobExecutionClaimedCount,
3
4
  jobExecutionEnqueuedCount,
@@ -7,6 +8,7 @@ export {
7
8
  providerRunnerReconcileCallCount,
8
9
  providerRunnerTerminateIntentHonoredCount,
9
10
  providerRunnerTerminateIntentIssuedCount,
11
+ recordRunnerReservationPromotionFailure,
10
12
  recordRunnersRateLimitCheck,
11
13
  recordRunnersRateLimitPruneFailure,
12
14
  } from './instance.js';
@@ -83,6 +83,18 @@ export const reservationReleasedCount = meter.createCounter<Record<string, never
83
83
  {description: 'Reservation units released from terminal provisioned runner reports'},
84
84
  );
85
85
 
86
+ export type RunnerReservationPromotionFailureReason =
87
+ | 'reservation-expired'
88
+ | 'reservation-not-found'
89
+ | 'already-assigned'
90
+ | 'not-assignable';
91
+
92
+ export const runnerReservationPromotionFailureCount = meter.createCounter<{
93
+ reason: RunnerReservationPromotionFailureReason;
94
+ }>('runners_reservation_promotion_failures', {
95
+ description: 'Runner reservation promotion failures during enrollment by reason',
96
+ });
97
+
86
98
  export type RunnersRateLimitAction = 'provisioner-mint' | 'ephemeral-register';
87
99
  export type RunnersRateLimitScope = 'provisioner' | 'ephemeral-token';
88
100
  export type RunnersRateLimitOutcome = 'allowed' | 'blocked' | 'unavailable';
@@ -107,6 +119,12 @@ function recordMetric(record: () => void): void {
107
119
  }
108
120
  }
109
121
 
122
+ export function recordRunnerReservationPromotionFailure(
123
+ reason: RunnerReservationPromotionFailureReason,
124
+ ): void {
125
+ recordMetric(() => runnerReservationPromotionFailureCount.add(1, {reason}));
126
+ }
127
+
110
128
  export function recordRunnersRateLimitCheck(params: {
111
129
  action: RunnersRateLimitAction;
112
130
  scope: RunnersRateLimitScope;
@@ -87,6 +87,7 @@ describe('POST /provisioners/demand/poll', () => {
87
87
  it('returns demand stats and reservations when matching demand exists', async () => {
88
88
  await pendingJobFactory.create({workspaceId, requiredLabels: ['linux']});
89
89
 
90
+ const requestStartedAt = Date.now();
90
91
  const res = await app.inject({
91
92
  method: 'POST',
92
93
  url: '/provisioners/demand/poll',
@@ -102,6 +103,43 @@ describe('POST /provisioners/demand/poll', () => {
102
103
  });
103
104
  expect(res.json().reservations[0].reservation_id).toEqual(expect.any(String));
104
105
  expect(res.json().reservations[0].expires_at).toEqual(expect.any(String));
106
+ const expiresAt = Date.parse(res.json().reservations[0].expires_at);
107
+ expect(expiresAt).toBeGreaterThanOrEqual(requestStartedAt + 55_000);
108
+ expect(expiresAt).toBeLessThan(requestStartedAt + 65_000);
109
+ });
110
+
111
+ it('clamps a requested reservation TTL for workspace provisioners', async () => {
112
+ await pendingJobFactory.create({workspaceId, requiredLabels: ['linux']});
113
+
114
+ const requestStartedAt = Date.now();
115
+ const res = await app.inject({
116
+ method: 'POST',
117
+ url: '/provisioners/demand/poll',
118
+ headers: {authorization: `Bearer ${VALID_PROVISIONER_TOKEN}`},
119
+ payload: body({max_reservations: 1, reservation_ttl_seconds: 600}),
120
+ });
121
+
122
+ expect(res.statusCode).toBe(200);
123
+ const expiresAt = Date.parse(res.json().reservations[0].expires_at);
124
+ expect(expiresAt).toBeGreaterThanOrEqual(requestStartedAt + 295_000);
125
+ expect(expiresAt).toBeLessThan(requestStartedAt + 305_000);
126
+ });
127
+
128
+ it('applies a requested reservation TTL below the ceiling', async () => {
129
+ await pendingJobFactory.create({workspaceId, requiredLabels: ['linux']});
130
+
131
+ const requestStartedAt = Date.now();
132
+ const res = await app.inject({
133
+ method: 'POST',
134
+ url: '/provisioners/demand/poll',
135
+ headers: {authorization: `Bearer ${VALID_PROVISIONER_TOKEN}`},
136
+ payload: body({max_reservations: 1, reservation_ttl_seconds: 120}),
137
+ });
138
+
139
+ expect(res.statusCode).toBe(200);
140
+ const expiresAt = Date.parse(res.json().reservations[0].expires_at);
141
+ expect(expiresAt).toBeGreaterThanOrEqual(requestStartedAt + 115_000);
142
+ expect(expiresAt).toBeLessThan(requestStartedAt + 125_000);
105
143
  });
106
144
 
107
145
  it('rejects installation provisioner credentials from workspace demand polling', async () => {
@@ -173,6 +211,39 @@ describe('POST /provisioners/demand/poll', () => {
173
211
  expect(clearedSnapshots).toEqual([]);
174
212
  });
175
213
 
214
+ it('strips reserved labels from workspace template advertisements', async () => {
215
+ const res = await app.inject({
216
+ method: 'POST',
217
+ url: '/provisioners/demand/poll',
218
+ headers: {authorization: `Bearer ${VALID_PROVISIONER_TOKEN}`},
219
+ payload: {
220
+ wait_seconds: 0,
221
+ max_reservations: 0,
222
+ templates: [
223
+ {
224
+ template_key: 'linux',
225
+ labels: ['linux', 'shipfox-managed'],
226
+ available_slots: 1,
227
+ starting: 0,
228
+ running: 0,
229
+ },
230
+ ],
231
+ },
232
+ });
233
+
234
+ const snapshots = await db()
235
+ .select()
236
+ .from(provisionerCapabilitySnapshots)
237
+ .where(eq(provisionerCapabilitySnapshots.provisionerId, provisionerTokenId));
238
+
239
+ expect(res.statusCode).toBe(200);
240
+ expect(snapshots).toEqual([
241
+ expect.objectContaining({
242
+ labels: ['linux'],
243
+ }),
244
+ ]);
245
+ });
246
+
176
247
  it('returns terminate intent ids for active provisioned runners with cancelled latest jobs', async () => {
177
248
  await providerRunnerFactory.create({
178
249
  workspaceId,
@@ -330,10 +401,13 @@ describe('POST /provisioners/demand/poll', () => {
330
401
  expect(res.statusCode).toBe(401);
331
402
  });
332
403
 
333
- function body(params: {max_reservations: number}) {
404
+ function body(params: {max_reservations: number; reservation_ttl_seconds?: number}) {
334
405
  return {
335
406
  wait_seconds: 0,
336
407
  max_reservations: params.max_reservations,
408
+ ...(params.reservation_ttl_seconds === undefined
409
+ ? {}
410
+ : {reservation_ttl_seconds: params.reservation_ttl_seconds}),
337
411
  templates: [
338
412
  {
339
413
  template_key: 'linux',
@@ -377,6 +451,7 @@ describe('POST /provisioners/demand/poll with installation provisioning configur
377
451
  let app: FastifyInstance;
378
452
  let workspaceId: string;
379
453
  let provisionerTokenId: string;
454
+ const filterEligibleWorkspaceIds = vi.fn().mockResolvedValue(new Set<string>());
380
455
 
381
456
  const fakeProvisionerAuth: AuthMethod = {
382
457
  name: AUTH_PROVISIONER_TOKEN,
@@ -402,7 +477,7 @@ describe('POST /provisioners/demand/poll with installation provisioning configur
402
477
  ],
403
478
  routes: createRunnerRoutes(runnersTestAuthClient, {
404
479
  installationProvisioning: {
405
- policy: {filterEligibleWorkspaceIds: vi.fn().mockResolvedValue(new Set())},
480
+ policy: {filterEligibleWorkspaceIds},
406
481
  },
407
482
  }),
408
483
  swagger: false,
@@ -444,10 +519,31 @@ describe('POST /provisioners/demand/poll with installation provisioning configur
444
519
  expect(res.statusCode).toBe(200);
445
520
  });
446
521
 
447
- function body(params: {max_reservations: number}) {
522
+ it('clamps a requested reservation TTL for installation provisioners', async () => {
523
+ await pendingJobFactory.create({workspaceId, requiredLabels: ['linux']});
524
+ filterEligibleWorkspaceIds.mockResolvedValueOnce(new Set([workspaceId]));
525
+
526
+ const requestStartedAt = Date.now();
527
+ const res = await app.inject({
528
+ method: 'POST',
529
+ url: '/provisioners/demand/poll',
530
+ headers: {authorization: `Bearer ${INSTALLATION_PROVISIONER_TOKEN}`},
531
+ payload: body({max_reservations: 1, reservation_ttl_seconds: 600}),
532
+ });
533
+
534
+ expect(res.statusCode).toBe(200);
535
+ const expiresAt = Date.parse(res.json().reservations[0].expires_at);
536
+ expect(expiresAt).toBeGreaterThanOrEqual(requestStartedAt + 295_000);
537
+ expect(expiresAt).toBeLessThan(requestStartedAt + 305_000);
538
+ });
539
+
540
+ function body(params: {max_reservations: number; reservation_ttl_seconds?: number}) {
448
541
  return {
449
542
  wait_seconds: 0,
450
543
  max_reservations: params.max_reservations,
544
+ ...(params.reservation_ttl_seconds === undefined
545
+ ? {}
546
+ : {reservation_ttl_seconds: params.reservation_ttl_seconds}),
451
547
  templates: [
452
548
  {
453
549
  template_key: 'linux',
@@ -9,6 +9,7 @@ import {ClientError, defineRoute} from '@shipfox/node-fastify';
9
9
  import {logger} from '@shipfox/node-opentelemetry';
10
10
  import {config} from '#config.js';
11
11
  import {pollDemand, releaseReservationGrants} from '#core/demand.js';
12
+ import {sanitizeRunnerLabels} from '#core/runner-labels.js';
12
13
  import {publishWorkspaceProvisionerCapabilitySnapshot} from '#db/provisioner-capability-snapshots.js';
13
14
  import {
14
15
  listQueuedDemandWorkspaceIds,
@@ -21,10 +22,17 @@ import {toPollDemandResponseDto} from '#presentation/dto/index.js';
21
22
 
22
23
  const TERMINATE_INTENT_LIMIT = 1000;
23
24
 
24
- function toReservationTemplates(templates: PollDemandTemplateDto[]): ReservationTemplate[] {
25
+ function toReservationTemplates(
26
+ templates: PollDemandTemplateDto[],
27
+ scope: 'installation' | 'workspace',
28
+ ): ReservationTemplate[] {
25
29
  return templates.map((template) => ({
26
30
  templateKey: template.template_key,
27
- labels: template.labels,
31
+ labels: sanitizeRunnerLabels(template.labels, {
32
+ scope,
33
+ logLevel: 'debug',
34
+ source: 'provisioner template advertisement',
35
+ }),
28
36
  availableSlots: template.available_slots,
29
37
  starting: template.starting,
30
38
  running: template.running,
@@ -66,6 +74,10 @@ export function createPollDemandRoute(options: CreateRunnersModuleOptions = {})
66
74
  }
67
75
  });
68
76
  const provisionerContext = requireProvisionerContext(request);
77
+ const ttlSeconds = Math.min(
78
+ request.body.reservation_ttl_seconds ?? config.RESERVATION_TTL_SECONDS,
79
+ config.RESERVATION_TTL_MAX_SECONDS,
80
+ );
69
81
  if (provisionerContext.scope === 'installation') {
70
82
  if (!options.installationProvisioning) {
71
83
  throw new ClientError(
@@ -81,11 +93,11 @@ export function createPollDemandRoute(options: CreateRunnersModuleOptions = {})
81
93
  await options.installationProvisioning.policy.filterEligibleWorkspaceIds(
82
94
  candidateWorkspaceIds,
83
95
  );
84
- const templates = toReservationTemplates(request.body.templates);
96
+ const templates = toReservationTemplates(request.body.templates, 'installation');
85
97
  const result = await pollInstallationDemandAndReserve({
86
98
  provisionerId: provisionerContext.provisionerTokenId,
87
99
  maxReservations: request.body.max_reservations,
88
- ttlSeconds: config.RESERVATION_TTL_SECONDS,
100
+ ttlSeconds,
89
101
  templates,
90
102
  capabilityWindowSeconds: config.PROVISIONER_ACTIVE_WINDOW_SECONDS,
91
103
  eligibleWorkspaceIds,
@@ -101,7 +113,7 @@ export function createPollDemandRoute(options: CreateRunnersModuleOptions = {})
101
113
  }
102
114
  const {provisionerTokenId, workspaceId} = requireWorkspaceProvisionerContext(request);
103
115
 
104
- const templates = toReservationTemplates(request.body.templates);
116
+ const templates = toReservationTemplates(request.body.templates, 'workspace');
105
117
  await publishWorkspaceProvisionerCapabilitySnapshot({
106
118
  workspaceId,
107
119
  provisionerId: provisionerTokenId,
@@ -113,7 +125,7 @@ export function createPollDemandRoute(options: CreateRunnersModuleOptions = {})
113
125
  provisionerId: provisionerTokenId,
114
126
  maxReservations: request.body.max_reservations,
115
127
  waitSeconds: request.body.wait_seconds,
116
- ttlSeconds: config.RESERVATION_TTL_SECONDS,
128
+ ttlSeconds,
117
129
  terminateIntentLimit: TERMINATE_INTENT_LIMIT,
118
130
  templates,
119
131
  signal: abortController.signal,