@shipfox/api-runners 12.1.0 → 12.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. package/.turbo/turbo-build.log +4 -4
  2. package/CHANGELOG.md +23 -0
  3. package/dist/config.d.ts +3 -0
  4. package/dist/config.d.ts.map +1 -1
  5. package/dist/config.js +24 -2
  6. package/dist/config.js.map +1 -1
  7. package/dist/core/errors.d.ts +4 -0
  8. package/dist/core/errors.d.ts.map +1 -1
  9. package/dist/core/errors.js +6 -0
  10. package/dist/core/errors.js.map +1 -1
  11. package/dist/core/index.d.ts +1 -1
  12. package/dist/core/index.d.ts.map +1 -1
  13. package/dist/core/index.js +1 -1
  14. package/dist/core/index.js.map +1 -1
  15. package/dist/core/rate-limit.d.ts.map +1 -1
  16. package/dist/core/runner-control-sessions.d.ts.map +1 -1
  17. package/dist/core/runner-control-sessions.js +23 -9
  18. package/dist/core/runner-control-sessions.js.map +1 -1
  19. package/dist/core/runner-instances.d.ts +1 -0
  20. package/dist/core/runner-instances.d.ts.map +1 -1
  21. package/dist/core/runner-instances.js.map +1 -1
  22. package/dist/core/runner-labels.d.ts +11 -0
  23. package/dist/core/runner-labels.d.ts.map +1 -0
  24. package/dist/core/runner-labels.js +41 -0
  25. package/dist/core/runner-labels.js.map +1 -0
  26. package/dist/core/runner-sessions.d.ts.map +1 -1
  27. package/dist/core/runner-sessions.js +5 -1
  28. package/dist/core/runner-sessions.js.map +1 -1
  29. package/dist/db/db.d.ts +64 -64
  30. package/dist/db/ephemeral-registration-tokens.d.ts.map +1 -1
  31. package/dist/db/ephemeral-registration-tokens.js +12 -2
  32. package/dist/db/ephemeral-registration-tokens.js.map +1 -1
  33. package/dist/db/reservations.d.ts.map +1 -1
  34. package/dist/db/reservations.js +15 -5
  35. package/dist/db/reservations.js.map +1 -1
  36. package/dist/db/runner-instances.d.ts +1 -0
  37. package/dist/db/runner-instances.d.ts.map +1 -1
  38. package/dist/db/runner-instances.js +6 -4
  39. package/dist/db/runner-instances.js.map +1 -1
  40. package/dist/db/runner-sessions.d.ts.map +1 -1
  41. package/dist/db/runner-sessions.js +12 -1
  42. package/dist/db/runner-sessions.js.map +1 -1
  43. package/dist/db/schema/admin-command-results.d.ts +1 -1
  44. package/dist/db/schema/ephemeral-registration-tokens.d.ts +1 -1
  45. package/dist/db/schema/manual-registration-tokens.d.ts +1 -1
  46. package/dist/db/schema/outbox.d.ts +1 -1
  47. package/dist/db/schema/pending-job-executions.d.ts +3 -3
  48. package/dist/db/schema/provisioner-capability-snapshots.d.ts +3 -3
  49. package/dist/db/schema/provisioner-tokens.d.ts +2 -2
  50. package/dist/db/schema/rate-limits.d.ts +1 -1
  51. package/dist/db/schema/reservations.d.ts +3 -3
  52. package/dist/db/schema/runner-activation-tokens.d.ts +1 -1
  53. package/dist/db/schema/runner-control-sessions.d.ts +2 -2
  54. package/dist/db/schema/runner-instances.d.ts +4 -4
  55. package/dist/db/schema/runner-instances.d.ts.map +1 -1
  56. package/dist/db/schema/runner-instances.js +1 -0
  57. package/dist/db/schema/runner-instances.js.map +1 -1
  58. package/dist/db/schema/runner-sessions.d.ts +4 -4
  59. package/dist/db/schema/running-job-executions.d.ts +5 -5
  60. package/dist/metrics/index.d.ts +2 -1
  61. package/dist/metrics/index.d.ts.map +1 -1
  62. package/dist/metrics/index.js +1 -1
  63. package/dist/metrics/index.js.map +1 -1
  64. package/dist/metrics/instance.d.ts +13 -8
  65. package/dist/metrics/instance.d.ts.map +1 -1
  66. package/dist/metrics/instance.js +8 -0
  67. package/dist/metrics/instance.js.map +1 -1
  68. package/dist/presentation/dto/admin-provisioner-token.d.ts +1 -1
  69. package/dist/presentation/dto/admin-provisioner-token.d.ts.map +1 -1
  70. package/dist/presentation/dto/admin-runner-instances.d.ts +2 -2
  71. package/dist/presentation/dto/admin-runner-instances.d.ts.map +1 -1
  72. package/dist/presentation/routes/poll-demand.d.ts.map +1 -1
  73. package/dist/presentation/routes/poll-demand.js +12 -6
  74. package/dist/presentation/routes/poll-demand.js.map +1 -1
  75. package/dist/presentation/routes/rate-limit.d.ts.map +1 -1
  76. package/dist/presentation/routes/register.d.ts.map +1 -1
  77. package/dist/presentation/routes/register.js +9 -1
  78. package/dist/presentation/routes/register.js.map +1 -1
  79. package/dist/presentation/routes/report-runner-instances.d.ts.map +1 -1
  80. package/dist/presentation/routes/report-runner-instances.js +1 -0
  81. package/dist/presentation/routes/report-runner-instances.js.map +1 -1
  82. package/dist/tsconfig.test.tsbuildinfo +1 -1
  83. package/drizzle/0004_huge_skreet.sql +1 -0
  84. package/drizzle/meta/0004_snapshot.json +2194 -0
  85. package/drizzle/meta/_journal.json +7 -0
  86. package/package.json +9 -9
  87. package/src/config.test.ts +81 -0
  88. package/src/config.ts +39 -2
  89. package/src/core/errors.ts +9 -0
  90. package/src/core/index.ts +1 -0
  91. package/src/core/runner-control-sessions.test.ts +239 -0
  92. package/src/core/runner-control-sessions.ts +29 -10
  93. package/src/core/runner-instances.ts +1 -0
  94. package/src/core/runner-labels.test.ts +53 -0
  95. package/src/core/runner-labels.ts +51 -0
  96. package/src/core/runner-sessions.test.ts +34 -1
  97. package/src/core/runner-sessions.ts +8 -1
  98. package/src/db/ephemeral-registration-tokens.ts +15 -1
  99. package/src/db/reservations.test.ts +169 -0
  100. package/src/db/reservations.ts +42 -8
  101. package/src/db/runner-instances.test.ts +45 -1
  102. package/src/db/runner-instances.ts +7 -2
  103. package/src/db/runner-sessions.ts +14 -1
  104. package/src/db/schema/runner-instances.ts +3 -0
  105. package/src/metrics/index.ts +2 -0
  106. package/src/metrics/instance.ts +18 -0
  107. package/src/presentation/routes/poll-demand.test.ts +99 -3
  108. package/src/presentation/routes/poll-demand.ts +18 -6
  109. package/src/presentation/routes/register.test.ts +116 -2
  110. package/src/presentation/routes/register.ts +7 -0
  111. package/src/presentation/routes/report-runner-instances.test.ts +33 -0
  112. package/src/presentation/routes/report-runner-instances.ts +1 -0
  113. package/src/presentation/routes/runner-enrollment.test.ts +65 -5
  114. package/src/temporal/activities/maintenance-activities.test.ts +24 -31
  115. package/test/env.ts +1 -0
  116. package/tsconfig.build.tsbuildinfo +1 -1
@@ -29,6 +29,13 @@
29
29
  "when": 1785235910167,
30
30
  "tag": "0003_flaky_sally_floyd",
31
31
  "breakpoints": true
32
+ },
33
+ {
34
+ "idx": 4,
35
+ "version": "7",
36
+ "when": 1786017247182,
37
+ "tag": "0004_huge_skreet",
38
+ "breakpoints": true
32
39
  }
33
40
  ]
34
41
  }
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@shipfox/api-runners",
3
3
  "license": "MIT",
4
- "version": "12.1.0",
4
+ "version": "12.2.0",
5
5
  "repository": {
6
6
  "type": "git",
7
7
  "url": "git+https://github.com/ShipfoxHQ/shipfox.git",
@@ -22,24 +22,24 @@
22
22
  "drizzle-orm": "^0.45.2",
23
23
  "zod": "^4.4.3",
24
24
  "@shipfox/api-common-dto": "12.0.0",
25
- "@shipfox/api-auth-context": "12.0.0",
25
+ "@shipfox/api-auth-context": "12.2.0",
26
26
  "@shipfox/api-auth-dto": "12.0.0",
27
- "@shipfox/api-runners-dto": "12.0.0",
28
- "@shipfox/api-workflows-dto": "12.1.0",
27
+ "@shipfox/api-runners-dto": "12.2.0",
28
+ "@shipfox/api-workflows-dto": "12.2.0",
29
29
  "@shipfox/config": "1.2.4",
30
30
  "@shipfox/inter-module": "0.2.3",
31
31
  "@shipfox/node-drizzle": "0.3.5",
32
32
  "@shipfox/node-error-monitoring": "0.3.0",
33
33
  "@shipfox/node-auth-root-key": "0.2.3",
34
- "@shipfox/node-fastify": "0.4.1",
35
- "@shipfox/node-module": "1.0.5",
36
- "@shipfox/node-opentelemetry": "0.6.3",
34
+ "@shipfox/node-fastify": "0.4.2",
35
+ "@shipfox/node-opentelemetry": "0.6.4",
37
36
  "@shipfox/node-outbox": "0.2.6",
38
37
  "@shipfox/node-postgres": "0.5.0",
39
38
  "@shipfox/node-rate-limit": "0.4.0",
40
- "@shipfox/node-temporal": "0.4.4",
39
+ "@shipfox/node-temporal": "0.4.5",
40
+ "@shipfox/node-module": "1.0.6",
41
41
  "@shipfox/node-tokens": "0.3.2",
42
- "@shipfox/runner-labels": "0.1.3"
42
+ "@shipfox/runner-labels": "0.2.0"
43
43
  },
44
44
  "imports": {
45
45
  "#*": "./dist/*"
@@ -1,6 +1,42 @@
1
1
  import {RUNNER_ASSIGNMENT_POLL_DEFAULT_WAIT_SECONDS} from '@shipfox/api-runners-dto';
2
2
  import {vi} from '@shipfox/vitest/vi';
3
3
 
4
+ describe('RUNNER_RESERVED_LABELS', () => {
5
+ afterEach(() => {
6
+ vi.unstubAllEnvs();
7
+ vi.resetModules();
8
+ });
9
+
10
+ it('defaults to no reserved labels', async () => {
11
+ vi.stubEnv('RUNNER_RESERVED_LABELS', undefined);
12
+ vi.resetModules();
13
+
14
+ const {config, runnerReservedLabels} = await import('#config.js');
15
+
16
+ expect(config.RUNNER_RESERVED_LABELS).toBe('');
17
+ expect(runnerReservedLabels).toEqual([]);
18
+ });
19
+
20
+ it('parses comma-separated labels for case-insensitive matching', async () => {
21
+ vi.stubEnv('RUNNER_RESERVED_LABELS', ' ShipFox-Managed, Linux,shipfox-managed ');
22
+ vi.resetModules();
23
+
24
+ const {config, runnerReservedLabels} = await import('#config.js');
25
+
26
+ expect(config.RUNNER_RESERVED_LABELS).toBe(' ShipFox-Managed, Linux,shipfox-managed ');
27
+ expect(runnerReservedLabels).toEqual(['linux', 'shipfox-managed']);
28
+ });
29
+
30
+ it('rejects invalid runner labels', async () => {
31
+ vi.stubEnv('RUNNER_RESERVED_LABELS', 'linux,has space');
32
+ vi.resetModules();
33
+
34
+ await expect(import('#config.js')).rejects.toThrow(
35
+ 'RUNNER_RESERVED_LABELS contains invalid runner label(s): has space',
36
+ );
37
+ });
38
+ });
39
+
4
40
  describe('EPHEMERAL_REGISTRATION_TOKEN_TTL_SECONDS validation', () => {
5
41
  afterEach(() => {
6
42
  vi.unstubAllEnvs();
@@ -49,6 +85,51 @@ describe('runner assignment polling defaults', () => {
49
85
  await expect(import('#config.js')).rejects.toThrow('RUNNER_ASSIGNMENT_POLL_MAX_WAIT_SECONDS');
50
86
  });
51
87
  });
88
+
89
+ describe('reservation TTL ceiling validation', () => {
90
+ afterEach(() => {
91
+ vi.unstubAllEnvs();
92
+ vi.resetModules();
93
+ });
94
+
95
+ it.each([
96
+ '0',
97
+ '-5',
98
+ '1.5',
99
+ '3601',
100
+ ])('fails startup when RESERVATION_TTL_MAX_SECONDS is %s', async (value) => {
101
+ vi.stubEnv('RESERVATION_TTL_MAX_SECONDS', value);
102
+ vi.resetModules();
103
+
104
+ await expect(import('#config.js')).rejects.toThrow('RESERVATION_TTL_MAX_SECONDS');
105
+ });
106
+
107
+ it('defaults high enough for the EC2 registration deadline', async () => {
108
+ vi.resetModules();
109
+
110
+ const {config} = await import('#config.js');
111
+
112
+ expect(config.RESERVATION_TTL_MAX_SECONDS).toBe(300);
113
+ });
114
+
115
+ it('accepts a whole-second ceiling inside the hard maximum', async () => {
116
+ vi.stubEnv('RESERVATION_TTL_MAX_SECONDS', '600');
117
+ vi.resetModules();
118
+
119
+ const {config} = await import('#config.js');
120
+
121
+ expect(config.RESERVATION_TTL_MAX_SECONDS).toBe(600);
122
+ });
123
+
124
+ it('fails startup when the default reservation TTL is above the ceiling', async () => {
125
+ vi.stubEnv('RESERVATION_TTL_SECONDS', '600');
126
+ vi.stubEnv('RESERVATION_TTL_MAX_SECONDS', '300');
127
+ vi.resetModules();
128
+
129
+ await expect(import('#config.js')).rejects.toThrow('RESERVATION_TTL_SECONDS');
130
+ });
131
+ });
132
+
52
133
  describe('REGISTRATION_TOKEN_BATCH_MAX validation', () => {
53
134
  afterEach(() => {
54
135
  vi.unstubAllEnvs();
package/src/config.ts CHANGED
@@ -1,10 +1,12 @@
1
1
  import {RUNNER_ASSIGNMENT_POLL_DEFAULT_WAIT_SECONDS} from '@shipfox/api-runners-dto';
2
- import {bool, createConfig, num} from '@shipfox/config';
2
+ import {bool, createConfig, num, str} from '@shipfox/config';
3
+ import {findInvalidLabels, parseLabelList} from '@shipfox/runner-labels';
3
4
  import {STUCK_JOB_THRESHOLD_SECONDS} from '#core/maintenance-policy.js';
4
5
 
5
6
  const EPHEMERAL_REGISTRATION_TOKEN_TTL_HARD_MAX_SECONDS = 3600;
6
7
  const REGISTRATION_TOKEN_BATCH_HARD_MAX = 1000;
7
8
  const RUNNER_CONTROL_PLANE_TOKEN_TTL_HARD_MAX_SECONDS = 3600;
9
+ const RESERVATION_TTL_HARD_MAX_SECONDS = 3600;
8
10
 
9
11
  export const config = createConfig({
10
12
  RUNNER_BOOTSTRAP_TOKEN_TTL_SECONDS: num({
@@ -27,6 +29,10 @@ export const config = createConfig({
27
29
  desc: 'Delay between durable assignment reads while a runner-control session waits for an assignment, in milliseconds.',
28
30
  default: 250,
29
31
  }),
32
+ RUNNER_RESERVED_LABELS: str({
33
+ desc: 'Comma-separated labels that only installation-scope provisioners may advertise. Any other provisioner or runner has them removed.',
34
+ default: '',
35
+ }),
30
36
  EPHEMERAL_REGISTRATION_TOKEN_TTL_SECONDS: num({
31
37
  desc: `Lifetime of a provisioner-minted runner registration token, in seconds. The token can be exchanged once by a runner before this time passes. Set this between 1 and ${EPHEMERAL_REGISTRATION_TOKEN_TTL_HARD_MAX_SECONDS}.`,
32
38
  default: 300,
@@ -56,9 +62,13 @@ export const config = createConfig({
56
62
  default: 250,
57
63
  }),
58
64
  RESERVATION_TTL_SECONDS: num({
59
- desc: 'Lifetime of a count-based runner reservation, in seconds. Expired reservations stop counting against queued demand.',
65
+ desc: 'Default lifetime of a count-based runner reservation, in seconds. Used when a provisioner does not request a provider-specific value. Expired reservations stop counting against queued demand.',
60
66
  default: 60,
61
67
  }),
68
+ RESERVATION_TTL_MAX_SECONDS: num({
69
+ desc: `Server-side ceiling for a count-based runner reservation lifetime, in seconds. Set it at least as high as every provider registration deadline so provider-specific values can cover boot and enrollment. Set this between 1 and ${RESERVATION_TTL_HARD_MAX_SECONDS}.`,
70
+ default: 300,
71
+ }),
62
72
  RESERVATION_LONG_POLL_MAX_WAIT_SECONDS: num({
63
73
  desc: 'Maximum time the provisioner demand poll endpoint waits for reservable demand before returning, in seconds.',
64
74
  default: 30,
@@ -133,6 +143,17 @@ export const config = createConfig({
133
143
  }),
134
144
  });
135
145
 
146
+ const parsedRunnerReservedLabels = parseLabelList(config.RUNNER_RESERVED_LABELS);
147
+ const invalidRunnerReservedLabels = findInvalidLabels(parsedRunnerReservedLabels);
148
+
149
+ if (invalidRunnerReservedLabels.length > 0) {
150
+ throw new Error(
151
+ `RUNNER_RESERVED_LABELS contains invalid runner label(s): ${invalidRunnerReservedLabels.join(', ')}`,
152
+ );
153
+ }
154
+
155
+ export const runnerReservedLabels = parsedRunnerReservedLabels;
156
+
136
157
  if (
137
158
  !Number.isInteger(config.EPHEMERAL_REGISTRATION_TOKEN_TTL_SECONDS) ||
138
159
  config.EPHEMERAL_REGISTRATION_TOKEN_TTL_SECONDS < 1 ||
@@ -209,6 +230,22 @@ if (!Number.isInteger(config.RESERVATION_TTL_SECONDS) || config.RESERVATION_TTL_
209
230
  );
210
231
  }
211
232
 
233
+ if (
234
+ !Number.isInteger(config.RESERVATION_TTL_MAX_SECONDS) ||
235
+ config.RESERVATION_TTL_MAX_SECONDS < 1 ||
236
+ config.RESERVATION_TTL_MAX_SECONDS > RESERVATION_TTL_HARD_MAX_SECONDS
237
+ ) {
238
+ throw new Error(
239
+ `RESERVATION_TTL_MAX_SECONDS (${config.RESERVATION_TTL_MAX_SECONDS}) must be a whole number of seconds between 1 and ${RESERVATION_TTL_HARD_MAX_SECONDS}.`,
240
+ );
241
+ }
242
+
243
+ if (config.RESERVATION_TTL_SECONDS > config.RESERVATION_TTL_MAX_SECONDS) {
244
+ throw new Error(
245
+ `RESERVATION_TTL_SECONDS (${config.RESERVATION_TTL_SECONDS}) must not be greater than RESERVATION_TTL_MAX_SECONDS (${config.RESERVATION_TTL_MAX_SECONDS}).`,
246
+ );
247
+ }
248
+
212
249
  if (
213
250
  !Number.isInteger(config.RESERVATION_LONG_POLL_MAX_WAIT_SECONDS) ||
214
251
  config.RESERVATION_LONG_POLL_MAX_WAIT_SECONDS < 0
@@ -40,6 +40,15 @@ export class EmptyRunnerLabelsError extends Error {
40
40
  }
41
41
  }
42
42
 
43
+ export class RunnerLabelsReservedError extends Error {
44
+ constructor(public readonly labels: string[]) {
45
+ super(
46
+ `All supplied runner labels are reserved for installation-scope provisioners: ${labels.join(', ')}`,
47
+ );
48
+ this.name = 'RunnerLabelsReservedError';
49
+ }
50
+ }
51
+
43
52
  export class EmptyRequiredLabelsError extends Error {
44
53
  constructor() {
45
54
  super('Required labels cannot be empty');
package/src/core/index.ts CHANGED
@@ -25,6 +25,7 @@ export {
25
25
  ProvisionerTokenNotFoundError,
26
26
  RegistrationTokenConsumedError,
27
27
  RegistrationTokenExpiredError,
28
+ RunnerLabelsReservedError,
28
29
  RunnerSessionExhaustedError,
29
30
  RunningJobExecutionNotFoundError,
30
31
  } from './errors.js';
@@ -0,0 +1,239 @@
1
+ import {afterEach, vi} from '@shipfox/vitest/vi';
2
+ import {inArray} from 'drizzle-orm';
3
+ import {db} from '#db/db.js';
4
+ import {reservations} from '#db/schema/reservations.js';
5
+ import {runnerActivationTokens} from '#db/schema/runner-activation-tokens.js';
6
+ import {runnerControlSessions} from '#db/schema/runner-control-sessions.js';
7
+ import {providerRunners} from '#db/schema/runner-instances.js';
8
+ import {
9
+ type RunnerReservationPromotionFailureReason,
10
+ runnerReservationPromotionFailureCount,
11
+ } from '#metrics/instance.js';
12
+ import {enrollRunnerControlSession} from './runner-control-sessions.js';
13
+
14
+ const createdReservationIds = new Set<string>();
15
+ const createdRunnerInstanceIds = new Set<string>();
16
+
17
+ afterEach(async () => {
18
+ const runnerInstanceIds = [...createdRunnerInstanceIds];
19
+ const reservationIds = [...createdReservationIds];
20
+
21
+ if (runnerInstanceIds.length > 0) {
22
+ await db()
23
+ .delete(runnerActivationTokens)
24
+ .where(inArray(runnerActivationTokens.runnerInstanceId, runnerInstanceIds));
25
+ await db()
26
+ .delete(runnerControlSessions)
27
+ .where(inArray(runnerControlSessions.runnerInstanceId, runnerInstanceIds));
28
+ await db().delete(providerRunners).where(inArray(providerRunners.id, runnerInstanceIds));
29
+ }
30
+ if (reservationIds.length > 0)
31
+ await db().delete(reservations).where(inArray(reservations.id, reservationIds));
32
+
33
+ createdRunnerInstanceIds.clear();
34
+ createdReservationIds.clear();
35
+ });
36
+
37
+ describe('enrollRunnerControlSession', () => {
38
+ it('counts expired reservation promotion failures', async () => {
39
+ const provisionerId = crypto.randomUUID();
40
+ const reservation = await createReservation({
41
+ provisionerId,
42
+ expiresAt: new Date(Date.now() - 1_000),
43
+ });
44
+
45
+ await expectPromotionFailure({
46
+ provisionerId,
47
+ intendedReservationId: reservation.id,
48
+ reason: 'reservation-expired',
49
+ });
50
+ });
51
+
52
+ it('counts missing reservation promotion failures', async () => {
53
+ await expectPromotionFailure({
54
+ provisionerId: crypto.randomUUID(),
55
+ intendedReservationId: crypto.randomUUID(),
56
+ reason: 'reservation-not-found',
57
+ });
58
+ });
59
+
60
+ it('counts already-assigned reservation promotion failures', async () => {
61
+ const provisionerId = crypto.randomUUID();
62
+ const intendedReservation = await createReservation({provisionerId});
63
+ const assignedReservation = await createReservation({provisionerId});
64
+
65
+ await expectPromotionFailure({
66
+ provisionerId,
67
+ intendedReservationId: intendedReservation.id,
68
+ reservationId: assignedReservation.id,
69
+ reason: 'already-assigned',
70
+ });
71
+ });
72
+
73
+ it('counts not-assignable reservation promotion failures', async () => {
74
+ const provisionerId = crypto.randomUUID();
75
+ const reservation = await createReservation({
76
+ provisionerId,
77
+ requiredLabels: ['gpu'],
78
+ });
79
+
80
+ await expectPromotionFailure({
81
+ provisionerId,
82
+ intendedReservationId: reservation.id,
83
+ labels: ['linux'],
84
+ reason: 'not-assignable',
85
+ });
86
+ });
87
+
88
+ it('does not count successful reservation promotion', async () => {
89
+ const provisionerId = crypto.randomUUID();
90
+ const reservation = await createReservation({provisionerId});
91
+ const runnerInstanceId = await createRunner({
92
+ provisionerId,
93
+ intendedReservationId: reservation.id,
94
+ });
95
+ const addSpy = vi.spyOn(runnerReservationPromotionFailureCount, 'add');
96
+
97
+ try {
98
+ const activationToken = await enrollRunnerControlSession({
99
+ runnerInstanceId,
100
+ provisionerId,
101
+ labels: ['linux'],
102
+ providerKind: 'docker',
103
+ protocolVersion: '1',
104
+ });
105
+
106
+ expect(activationToken).toEqual(expect.any(String));
107
+ expect(addSpy).not.toHaveBeenCalled();
108
+ } finally {
109
+ addSpy.mockRestore();
110
+ }
111
+ });
112
+
113
+ it('keeps unexpected promotion failures on the error-log path', async () => {
114
+ const provisionerId = crypto.randomUUID();
115
+ const reservation = await createReservation({provisionerId});
116
+ const runnerInstanceId = await createRunner({
117
+ provisionerId,
118
+ intendedReservationId: reservation.id,
119
+ });
120
+ const unexpectedError = new Error('database unavailable');
121
+ const debugLog = vi.fn();
122
+ const errorLog = vi.fn();
123
+
124
+ vi.resetModules();
125
+ vi.doMock('@shipfox/node-opentelemetry', async (importOriginal) => ({
126
+ ...(await importOriginal<typeof import('@shipfox/node-opentelemetry')>()),
127
+ logger: () => ({debug: debugLog, error: errorLog}),
128
+ }));
129
+ vi.doMock('#db/runner-assignments.js', async (importOriginal) => ({
130
+ ...(await importOriginal<typeof import('#db/runner-assignments.js')>()),
131
+ assignRunnerInstancesTx: vi.fn().mockRejectedValue(unexpectedError),
132
+ }));
133
+
134
+ const {closePostgresClient, createPostgresClient} = await import('@shipfox/node-postgres');
135
+ createPostgresClient();
136
+
137
+ try {
138
+ const {enrollRunnerControlSession: enroll} = await import('./runner-control-sessions.js');
139
+ const result = await enroll({
140
+ runnerInstanceId,
141
+ provisionerId,
142
+ labels: ['linux'],
143
+ providerKind: 'docker',
144
+ protocolVersion: '1',
145
+ });
146
+
147
+ expect(result).toBeNull();
148
+ expect(debugLog).not.toHaveBeenCalled();
149
+ expect(errorLog).toHaveBeenCalledWith(
150
+ expect.objectContaining({err: unexpectedError}),
151
+ 'Unexpected failure promoting runner reservation during enrollment',
152
+ );
153
+ } finally {
154
+ await closePostgresClient();
155
+ vi.doUnmock('@shipfox/node-opentelemetry');
156
+ vi.doUnmock('#db/runner-assignments.js');
157
+ vi.resetModules();
158
+ }
159
+ });
160
+ });
161
+
162
+ async function expectPromotionFailure(params: {
163
+ provisionerId: string;
164
+ intendedReservationId: string;
165
+ reservationId?: string;
166
+ labels?: string[];
167
+ reason: RunnerReservationPromotionFailureReason;
168
+ }): Promise<void> {
169
+ const runnerInstanceId = await createRunner(params);
170
+ const addSpy = vi.spyOn(runnerReservationPromotionFailureCount, 'add');
171
+
172
+ try {
173
+ const result = await enrollRunnerControlSession({
174
+ runnerInstanceId,
175
+ provisionerId: params.provisionerId,
176
+ labels: params.labels ?? ['linux'],
177
+ providerKind: 'docker',
178
+ protocolVersion: '1',
179
+ });
180
+
181
+ expect(result).toBeNull();
182
+ expect(addSpy).toHaveBeenCalledWith(1, {reason: params.reason});
183
+ } finally {
184
+ addSpy.mockRestore();
185
+ }
186
+ }
187
+
188
+ async function createReservation(params: {
189
+ provisionerId: string;
190
+ requiredLabels?: string[];
191
+ expiresAt?: Date;
192
+ }) {
193
+ const [reservation] = await db()
194
+ .insert(reservations)
195
+ .values({
196
+ workspaceId: crypto.randomUUID(),
197
+ provisionerId: params.provisionerId,
198
+ requiredLabels: params.requiredLabels ?? ['linux'],
199
+ count: 1,
200
+ expiresAt: params.expiresAt ?? new Date(Date.now() + 60_000),
201
+ })
202
+ .returning();
203
+ if (!reservation) throw new Error('Expected reservation');
204
+ createdReservationIds.add(reservation.id);
205
+ return reservation;
206
+ }
207
+
208
+ async function createRunner(params: {
209
+ provisionerId: string;
210
+ intendedReservationId: string;
211
+ reservationId?: string;
212
+ labels?: string[];
213
+ }): Promise<string> {
214
+ const [runner] = await db()
215
+ .insert(providerRunners)
216
+ .values({
217
+ provisionerId: params.provisionerId,
218
+ intendedReservationId: params.intendedReservationId,
219
+ reservationId: params.reservationId ?? null,
220
+ providerRunnerId: crypto.randomUUID(),
221
+ labels: params.labels ?? [],
222
+ state: 'starting',
223
+ reportedAt: new Date(),
224
+ })
225
+ .returning({id: providerRunners.id});
226
+ if (!runner) throw new Error('Expected runner instance');
227
+ createdRunnerInstanceIds.add(runner.id);
228
+
229
+ await db()
230
+ .insert(runnerControlSessions)
231
+ .values({
232
+ runnerInstanceId: runner.id,
233
+ provisionerId: params.provisionerId,
234
+ hashedToken: crypto.randomUUID(),
235
+ prefix: 'test',
236
+ expiresAt: new Date(Date.now() + 60_000),
237
+ });
238
+ return runner.id;
239
+ }
@@ -2,7 +2,6 @@ import type {RunnerToolCapabilitiesDto} from '@shipfox/api-runners-dto';
2
2
  import type {NodePgDatabase} from '@shipfox/node-drizzle';
3
3
  import {logger} from '@shipfox/node-opentelemetry';
4
4
  import {extractDisplayPrefix, generateOpaqueToken, hashOpaqueToken} from '@shipfox/node-tokens';
5
- import {canonicalizeLabels} from '@shipfox/runner-labels';
6
5
  import {and, eq, gt, isNull, notInArray, or, sql} from 'drizzle-orm';
7
6
  import {config} from '#config.js';
8
7
  import {
@@ -11,12 +10,17 @@ import {
11
10
  RunnerInstanceAlreadyAssignedError,
12
11
  RunnerInstanceNotAssignableError,
13
12
  } from '#core/errors.js';
13
+ import {sanitizeRunnerLabels} from '#core/runner-labels.js';
14
14
  import {db, type schema, type Tx} from '#db/db.js';
15
15
  import {assignRunnerInstancesTx} from '#db/runner-assignments.js';
16
16
  import {terminalStates} from '#db/runner-instances.js';
17
17
  import {provisionerTokens} from '#db/schema/provisioner-tokens.js';
18
18
  import {runnerBootstrapTokens, runnerControlSessions} from '#db/schema/runner-control-sessions.js';
19
19
  import {providerRunners} from '#db/schema/runner-instances.js';
20
+ import {
21
+ type RunnerReservationPromotionFailureReason,
22
+ recordRunnerReservationPromotionFailure,
23
+ } from '#metrics/index.js';
20
24
  import {issueRunnerActivationTokenTx} from './runner-activation.js';
21
25
 
22
26
  export class RunnerBootstrapTokenInvalidError extends Error {
@@ -141,8 +145,12 @@ export async function enrollRunnerControlSession(params: {
141
145
  }): Promise<string | null> {
142
146
  return await db().transaction(async (tx) => {
143
147
  const [current] = await tx
144
- .select({intendedReservationId: providerRunners.intendedReservationId})
148
+ .select({
149
+ intendedReservationId: providerRunners.intendedReservationId,
150
+ provisionerScope: provisionerTokens.scope,
151
+ })
145
152
  .from(providerRunners)
153
+ .innerJoin(provisionerTokens, eq(provisionerTokens.id, providerRunners.provisionerId))
146
154
  .where(
147
155
  and(
148
156
  eq(providerRunners.id, params.runnerInstanceId),
@@ -158,7 +166,10 @@ export async function enrollRunnerControlSession(params: {
158
166
  const [updated] = await tx
159
167
  .update(providerRunners)
160
168
  .set({
161
- labels: [...canonicalizeLabels(params.labels)],
169
+ labels: sanitizeRunnerLabels(params.labels, {
170
+ scope: current.provisionerScope,
171
+ source: 'runner control enrollment',
172
+ }),
162
173
  providerKind: params.providerKind,
163
174
  protocolVersion: params.protocolVersion,
164
175
  capabilities: params.capabilities ?? null,
@@ -198,29 +209,37 @@ export async function enrollRunnerControlSession(params: {
198
209
  });
199
210
  });
200
211
  } catch (error) {
201
- const expectedFailure =
202
- error instanceof ReservationExpiredError ||
203
- error instanceof ReservationNotFoundError ||
204
- error instanceof RunnerInstanceAlreadyAssignedError ||
205
- error instanceof RunnerInstanceNotAssignableError;
212
+ const expectedFailureReason = getRunnerReservationPromotionFailureReason(error);
206
213
  const details = {
207
214
  err: error,
208
215
  runnerInstanceId: params.runnerInstanceId,
209
216
  provisionerId: params.provisionerId,
210
217
  reservationId: intendedReservationId,
211
218
  };
212
- if (expectedFailure)
219
+ if (expectedFailureReason) {
220
+ recordRunnerReservationPromotionFailure(expectedFailureReason);
213
221
  logger().debug(details, 'Runner reservation could not be promoted during enrollment');
214
- else
222
+ } else {
215
223
  logger().error(
216
224
  details,
217
225
  'Unexpected failure promoting runner reservation during enrollment',
218
226
  );
227
+ }
219
228
  return null;
220
229
  }
221
230
  });
222
231
  }
223
232
 
233
+ function getRunnerReservationPromotionFailureReason(
234
+ error: unknown,
235
+ ): RunnerReservationPromotionFailureReason | null {
236
+ if (error instanceof ReservationExpiredError) return 'reservation-expired';
237
+ if (error instanceof ReservationNotFoundError) return 'reservation-not-found';
238
+ if (error instanceof RunnerInstanceAlreadyAssignedError) return 'already-assigned';
239
+ if (error instanceof RunnerInstanceNotAssignableError) return 'not-assignable';
240
+ return null;
241
+ }
242
+
224
243
  export async function attachRunnerControlProviderId(params: {
225
244
  runnerInstanceId: string;
226
245
  provisionerId: string;
@@ -23,6 +23,7 @@ import {
23
23
  import {config} from '../config.js';
24
24
 
25
25
  export interface ReportRunnerInstancesParams {
26
+ scope: 'installation' | 'workspace';
26
27
  workspaceId: string | null;
27
28
  provisionerId: string;
28
29
  events: RunnerInstanceReportEvent[];
@@ -0,0 +1,53 @@
1
+ import {RunnerLabelsReservedError} from './errors.js';
2
+ import {sanitizeRunnerLabels, sanitizeRunnerLabelsOrThrow} from './runner-labels.js';
3
+
4
+ describe('sanitizeRunnerLabels', () => {
5
+ it('strips reserved labels from non-installation sources', () => {
6
+ const labels = sanitizeRunnerLabels([' Linux ', 'shipfox-managed', 'linux'], {
7
+ scope: 'workspace',
8
+ source: 'test',
9
+ reservedLabels: ['Shipfox-Managed'],
10
+ });
11
+
12
+ expect(labels).toEqual(['linux']);
13
+ });
14
+
15
+ it('preserves reserved labels for installation sources', () => {
16
+ const labels = sanitizeRunnerLabels([' Linux ', 'shipfox-managed'], {
17
+ scope: 'installation',
18
+ source: 'test',
19
+ reservedLabels: ['shipfox-managed'],
20
+ });
21
+
22
+ expect(labels).toEqual(['linux', 'shipfox-managed']);
23
+ });
24
+
25
+ it('does not change labels when no reserved labels are configured', () => {
26
+ const labels = sanitizeRunnerLabels([' Linux ', 'linux'], {
27
+ scope: 'manual',
28
+ source: 'test',
29
+ reservedLabels: [],
30
+ });
31
+
32
+ expect(labels).toEqual(['linux']);
33
+ });
34
+
35
+ it('uses the configured reserved labels when none are provided explicitly', () => {
36
+ const labels = sanitizeRunnerLabels(['linux', 'shipfox-managed'], {
37
+ scope: 'workspace',
38
+ source: 'test',
39
+ });
40
+
41
+ expect(labels).toEqual(['linux']);
42
+ });
43
+
44
+ it('throws when sanitization removes every canonical label', () => {
45
+ expect(() =>
46
+ sanitizeRunnerLabelsOrThrow([' Shipfox-Managed '], {
47
+ scope: 'workspace',
48
+ source: 'test',
49
+ reservedLabels: ['shipfox-managed'],
50
+ }),
51
+ ).toThrow(RunnerLabelsReservedError);
52
+ });
53
+ });