@shipfox/api-runners 12.0.0 → 12.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +4 -4
- package/CHANGELOG.md +30 -0
- package/dist/config.d.ts +3 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +24 -2
- package/dist/config.js.map +1 -1
- package/dist/core/errors.d.ts +4 -0
- package/dist/core/errors.d.ts.map +1 -1
- package/dist/core/errors.js +6 -0
- package/dist/core/errors.js.map +1 -1
- package/dist/core/index.d.ts +1 -1
- package/dist/core/index.d.ts.map +1 -1
- package/dist/core/index.js +1 -1
- package/dist/core/index.js.map +1 -1
- package/dist/core/rate-limit.d.ts.map +1 -1
- package/dist/core/runner-control-sessions.d.ts.map +1 -1
- package/dist/core/runner-control-sessions.js +23 -9
- package/dist/core/runner-control-sessions.js.map +1 -1
- package/dist/core/runner-instances.d.ts +1 -0
- package/dist/core/runner-instances.d.ts.map +1 -1
- package/dist/core/runner-instances.js.map +1 -1
- package/dist/core/runner-labels.d.ts +11 -0
- package/dist/core/runner-labels.d.ts.map +1 -0
- package/dist/core/runner-labels.js +41 -0
- package/dist/core/runner-labels.js.map +1 -0
- package/dist/core/runner-sessions.d.ts.map +1 -1
- package/dist/core/runner-sessions.js +5 -1
- package/dist/core/runner-sessions.js.map +1 -1
- package/dist/db/db.d.ts +64 -64
- package/dist/db/ephemeral-registration-tokens.d.ts.map +1 -1
- package/dist/db/ephemeral-registration-tokens.js +12 -2
- package/dist/db/ephemeral-registration-tokens.js.map +1 -1
- package/dist/db/reservations.d.ts.map +1 -1
- package/dist/db/reservations.js +15 -5
- package/dist/db/reservations.js.map +1 -1
- package/dist/db/runner-instances.d.ts +1 -0
- package/dist/db/runner-instances.d.ts.map +1 -1
- package/dist/db/runner-instances.js +6 -4
- package/dist/db/runner-instances.js.map +1 -1
- package/dist/db/runner-sessions.d.ts.map +1 -1
- package/dist/db/runner-sessions.js +12 -1
- package/dist/db/runner-sessions.js.map +1 -1
- package/dist/db/schema/admin-command-results.d.ts +1 -1
- package/dist/db/schema/ephemeral-registration-tokens.d.ts +1 -1
- package/dist/db/schema/manual-registration-tokens.d.ts +1 -1
- package/dist/db/schema/outbox.d.ts +1 -1
- package/dist/db/schema/pending-job-executions.d.ts +3 -3
- package/dist/db/schema/provisioner-capability-snapshots.d.ts +3 -3
- package/dist/db/schema/provisioner-tokens.d.ts +2 -2
- package/dist/db/schema/rate-limits.d.ts +1 -1
- package/dist/db/schema/reservations.d.ts +3 -3
- package/dist/db/schema/runner-activation-tokens.d.ts +1 -1
- package/dist/db/schema/runner-control-sessions.d.ts +2 -2
- package/dist/db/schema/runner-instances.d.ts +4 -4
- package/dist/db/schema/runner-instances.d.ts.map +1 -1
- package/dist/db/schema/runner-instances.js +1 -0
- package/dist/db/schema/runner-instances.js.map +1 -1
- package/dist/db/schema/runner-sessions.d.ts +4 -4
- package/dist/db/schema/running-job-executions.d.ts +5 -5
- package/dist/metrics/index.d.ts +2 -1
- package/dist/metrics/index.d.ts.map +1 -1
- package/dist/metrics/index.js +1 -1
- package/dist/metrics/index.js.map +1 -1
- package/dist/metrics/instance.d.ts +13 -8
- package/dist/metrics/instance.d.ts.map +1 -1
- package/dist/metrics/instance.js +8 -0
- package/dist/metrics/instance.js.map +1 -1
- package/dist/presentation/dto/admin-provisioner-token.d.ts +1 -1
- package/dist/presentation/dto/admin-provisioner-token.d.ts.map +1 -1
- package/dist/presentation/dto/admin-runner-instances.d.ts +2 -2
- package/dist/presentation/dto/admin-runner-instances.d.ts.map +1 -1
- package/dist/presentation/routes/poll-demand.d.ts.map +1 -1
- package/dist/presentation/routes/poll-demand.js +12 -6
- package/dist/presentation/routes/poll-demand.js.map +1 -1
- package/dist/presentation/routes/rate-limit.d.ts.map +1 -1
- package/dist/presentation/routes/register.d.ts.map +1 -1
- package/dist/presentation/routes/register.js +9 -1
- package/dist/presentation/routes/register.js.map +1 -1
- package/dist/presentation/routes/report-runner-instances.d.ts.map +1 -1
- package/dist/presentation/routes/report-runner-instances.js +1 -0
- package/dist/presentation/routes/report-runner-instances.js.map +1 -1
- package/dist/tsconfig.test.tsbuildinfo +1 -1
- package/drizzle/0004_huge_skreet.sql +1 -0
- package/drizzle/meta/0004_snapshot.json +2194 -0
- package/drizzle/meta/_journal.json +7 -0
- package/package.json +9 -9
- package/src/config.test.ts +81 -0
- package/src/config.ts +39 -2
- package/src/core/errors.ts +9 -0
- package/src/core/index.ts +1 -0
- package/src/core/runner-control-sessions.test.ts +239 -0
- package/src/core/runner-control-sessions.ts +29 -10
- package/src/core/runner-instances.ts +1 -0
- package/src/core/runner-labels.test.ts +53 -0
- package/src/core/runner-labels.ts +51 -0
- package/src/core/runner-sessions.test.ts +34 -1
- package/src/core/runner-sessions.ts +8 -1
- package/src/db/ephemeral-registration-tokens.ts +15 -1
- package/src/db/reservations.test.ts +169 -0
- package/src/db/reservations.ts +42 -8
- package/src/db/runner-instances.test.ts +45 -1
- package/src/db/runner-instances.ts +7 -2
- package/src/db/runner-sessions.ts +14 -1
- package/src/db/schema/runner-instances.ts +3 -0
- package/src/metrics/index.ts +2 -0
- package/src/metrics/instance.ts +18 -0
- package/src/presentation/routes/poll-demand.test.ts +99 -3
- package/src/presentation/routes/poll-demand.ts +18 -6
- package/src/presentation/routes/register.test.ts +116 -2
- package/src/presentation/routes/register.ts +7 -0
- package/src/presentation/routes/report-runner-instances.test.ts +33 -0
- package/src/presentation/routes/report-runner-instances.ts +1 -0
- package/src/presentation/routes/runner-enrollment.test.ts +65 -5
- package/src/temporal/activities/maintenance-activities.test.ts +24 -31
- package/test/env.ts +1 -0
- package/tsconfig.build.tsbuildinfo +1 -1
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@shipfox/api-runners",
|
|
3
3
|
"license": "MIT",
|
|
4
|
-
"version": "12.
|
|
4
|
+
"version": "12.2.0",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
7
7
|
"url": "git+https://github.com/ShipfoxHQ/shipfox.git",
|
|
@@ -22,24 +22,24 @@
|
|
|
22
22
|
"drizzle-orm": "^0.45.2",
|
|
23
23
|
"zod": "^4.4.3",
|
|
24
24
|
"@shipfox/api-common-dto": "12.0.0",
|
|
25
|
-
"@shipfox/api-auth-context": "12.
|
|
25
|
+
"@shipfox/api-auth-context": "12.2.0",
|
|
26
26
|
"@shipfox/api-auth-dto": "12.0.0",
|
|
27
|
-
"@shipfox/api-runners-dto": "12.
|
|
28
|
-
"@shipfox/api-workflows-dto": "12.
|
|
27
|
+
"@shipfox/api-runners-dto": "12.2.0",
|
|
28
|
+
"@shipfox/api-workflows-dto": "12.2.0",
|
|
29
29
|
"@shipfox/config": "1.2.4",
|
|
30
30
|
"@shipfox/inter-module": "0.2.3",
|
|
31
31
|
"@shipfox/node-drizzle": "0.3.5",
|
|
32
32
|
"@shipfox/node-error-monitoring": "0.3.0",
|
|
33
33
|
"@shipfox/node-auth-root-key": "0.2.3",
|
|
34
|
-
"@shipfox/node-fastify": "0.4.
|
|
35
|
-
"@shipfox/node-
|
|
36
|
-
"@shipfox/node-opentelemetry": "0.6.3",
|
|
34
|
+
"@shipfox/node-fastify": "0.4.2",
|
|
35
|
+
"@shipfox/node-opentelemetry": "0.6.4",
|
|
37
36
|
"@shipfox/node-outbox": "0.2.6",
|
|
38
37
|
"@shipfox/node-postgres": "0.5.0",
|
|
39
38
|
"@shipfox/node-rate-limit": "0.4.0",
|
|
40
|
-
"@shipfox/node-temporal": "0.4.
|
|
39
|
+
"@shipfox/node-temporal": "0.4.5",
|
|
40
|
+
"@shipfox/node-module": "1.0.6",
|
|
41
41
|
"@shipfox/node-tokens": "0.3.2",
|
|
42
|
-
"@shipfox/runner-labels": "0.
|
|
42
|
+
"@shipfox/runner-labels": "0.2.0"
|
|
43
43
|
},
|
|
44
44
|
"imports": {
|
|
45
45
|
"#*": "./dist/*"
|
package/src/config.test.ts
CHANGED
|
@@ -1,6 +1,42 @@
|
|
|
1
1
|
import {RUNNER_ASSIGNMENT_POLL_DEFAULT_WAIT_SECONDS} from '@shipfox/api-runners-dto';
|
|
2
2
|
import {vi} from '@shipfox/vitest/vi';
|
|
3
3
|
|
|
4
|
+
describe('RUNNER_RESERVED_LABELS', () => {
|
|
5
|
+
afterEach(() => {
|
|
6
|
+
vi.unstubAllEnvs();
|
|
7
|
+
vi.resetModules();
|
|
8
|
+
});
|
|
9
|
+
|
|
10
|
+
it('defaults to no reserved labels', async () => {
|
|
11
|
+
vi.stubEnv('RUNNER_RESERVED_LABELS', undefined);
|
|
12
|
+
vi.resetModules();
|
|
13
|
+
|
|
14
|
+
const {config, runnerReservedLabels} = await import('#config.js');
|
|
15
|
+
|
|
16
|
+
expect(config.RUNNER_RESERVED_LABELS).toBe('');
|
|
17
|
+
expect(runnerReservedLabels).toEqual([]);
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
it('parses comma-separated labels for case-insensitive matching', async () => {
|
|
21
|
+
vi.stubEnv('RUNNER_RESERVED_LABELS', ' ShipFox-Managed, Linux,shipfox-managed ');
|
|
22
|
+
vi.resetModules();
|
|
23
|
+
|
|
24
|
+
const {config, runnerReservedLabels} = await import('#config.js');
|
|
25
|
+
|
|
26
|
+
expect(config.RUNNER_RESERVED_LABELS).toBe(' ShipFox-Managed, Linux,shipfox-managed ');
|
|
27
|
+
expect(runnerReservedLabels).toEqual(['linux', 'shipfox-managed']);
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
it('rejects invalid runner labels', async () => {
|
|
31
|
+
vi.stubEnv('RUNNER_RESERVED_LABELS', 'linux,has space');
|
|
32
|
+
vi.resetModules();
|
|
33
|
+
|
|
34
|
+
await expect(import('#config.js')).rejects.toThrow(
|
|
35
|
+
'RUNNER_RESERVED_LABELS contains invalid runner label(s): has space',
|
|
36
|
+
);
|
|
37
|
+
});
|
|
38
|
+
});
|
|
39
|
+
|
|
4
40
|
describe('EPHEMERAL_REGISTRATION_TOKEN_TTL_SECONDS validation', () => {
|
|
5
41
|
afterEach(() => {
|
|
6
42
|
vi.unstubAllEnvs();
|
|
@@ -49,6 +85,51 @@ describe('runner assignment polling defaults', () => {
|
|
|
49
85
|
await expect(import('#config.js')).rejects.toThrow('RUNNER_ASSIGNMENT_POLL_MAX_WAIT_SECONDS');
|
|
50
86
|
});
|
|
51
87
|
});
|
|
88
|
+
|
|
89
|
+
describe('reservation TTL ceiling validation', () => {
|
|
90
|
+
afterEach(() => {
|
|
91
|
+
vi.unstubAllEnvs();
|
|
92
|
+
vi.resetModules();
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
it.each([
|
|
96
|
+
'0',
|
|
97
|
+
'-5',
|
|
98
|
+
'1.5',
|
|
99
|
+
'3601',
|
|
100
|
+
])('fails startup when RESERVATION_TTL_MAX_SECONDS is %s', async (value) => {
|
|
101
|
+
vi.stubEnv('RESERVATION_TTL_MAX_SECONDS', value);
|
|
102
|
+
vi.resetModules();
|
|
103
|
+
|
|
104
|
+
await expect(import('#config.js')).rejects.toThrow('RESERVATION_TTL_MAX_SECONDS');
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
it('defaults high enough for the EC2 registration deadline', async () => {
|
|
108
|
+
vi.resetModules();
|
|
109
|
+
|
|
110
|
+
const {config} = await import('#config.js');
|
|
111
|
+
|
|
112
|
+
expect(config.RESERVATION_TTL_MAX_SECONDS).toBe(300);
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
it('accepts a whole-second ceiling inside the hard maximum', async () => {
|
|
116
|
+
vi.stubEnv('RESERVATION_TTL_MAX_SECONDS', '600');
|
|
117
|
+
vi.resetModules();
|
|
118
|
+
|
|
119
|
+
const {config} = await import('#config.js');
|
|
120
|
+
|
|
121
|
+
expect(config.RESERVATION_TTL_MAX_SECONDS).toBe(600);
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
it('fails startup when the default reservation TTL is above the ceiling', async () => {
|
|
125
|
+
vi.stubEnv('RESERVATION_TTL_SECONDS', '600');
|
|
126
|
+
vi.stubEnv('RESERVATION_TTL_MAX_SECONDS', '300');
|
|
127
|
+
vi.resetModules();
|
|
128
|
+
|
|
129
|
+
await expect(import('#config.js')).rejects.toThrow('RESERVATION_TTL_SECONDS');
|
|
130
|
+
});
|
|
131
|
+
});
|
|
132
|
+
|
|
52
133
|
describe('REGISTRATION_TOKEN_BATCH_MAX validation', () => {
|
|
53
134
|
afterEach(() => {
|
|
54
135
|
vi.unstubAllEnvs();
|
package/src/config.ts
CHANGED
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
import {RUNNER_ASSIGNMENT_POLL_DEFAULT_WAIT_SECONDS} from '@shipfox/api-runners-dto';
|
|
2
|
-
import {bool, createConfig, num} from '@shipfox/config';
|
|
2
|
+
import {bool, createConfig, num, str} from '@shipfox/config';
|
|
3
|
+
import {findInvalidLabels, parseLabelList} from '@shipfox/runner-labels';
|
|
3
4
|
import {STUCK_JOB_THRESHOLD_SECONDS} from '#core/maintenance-policy.js';
|
|
4
5
|
|
|
5
6
|
const EPHEMERAL_REGISTRATION_TOKEN_TTL_HARD_MAX_SECONDS = 3600;
|
|
6
7
|
const REGISTRATION_TOKEN_BATCH_HARD_MAX = 1000;
|
|
7
8
|
const RUNNER_CONTROL_PLANE_TOKEN_TTL_HARD_MAX_SECONDS = 3600;
|
|
9
|
+
const RESERVATION_TTL_HARD_MAX_SECONDS = 3600;
|
|
8
10
|
|
|
9
11
|
export const config = createConfig({
|
|
10
12
|
RUNNER_BOOTSTRAP_TOKEN_TTL_SECONDS: num({
|
|
@@ -27,6 +29,10 @@ export const config = createConfig({
|
|
|
27
29
|
desc: 'Delay between durable assignment reads while a runner-control session waits for an assignment, in milliseconds.',
|
|
28
30
|
default: 250,
|
|
29
31
|
}),
|
|
32
|
+
RUNNER_RESERVED_LABELS: str({
|
|
33
|
+
desc: 'Comma-separated labels that only installation-scope provisioners may advertise. Any other provisioner or runner has them removed.',
|
|
34
|
+
default: '',
|
|
35
|
+
}),
|
|
30
36
|
EPHEMERAL_REGISTRATION_TOKEN_TTL_SECONDS: num({
|
|
31
37
|
desc: `Lifetime of a provisioner-minted runner registration token, in seconds. The token can be exchanged once by a runner before this time passes. Set this between 1 and ${EPHEMERAL_REGISTRATION_TOKEN_TTL_HARD_MAX_SECONDS}.`,
|
|
32
38
|
default: 300,
|
|
@@ -56,9 +62,13 @@ export const config = createConfig({
|
|
|
56
62
|
default: 250,
|
|
57
63
|
}),
|
|
58
64
|
RESERVATION_TTL_SECONDS: num({
|
|
59
|
-
desc: '
|
|
65
|
+
desc: 'Default lifetime of a count-based runner reservation, in seconds. Used when a provisioner does not request a provider-specific value. Expired reservations stop counting against queued demand.',
|
|
60
66
|
default: 60,
|
|
61
67
|
}),
|
|
68
|
+
RESERVATION_TTL_MAX_SECONDS: num({
|
|
69
|
+
desc: `Server-side ceiling for a count-based runner reservation lifetime, in seconds. Set it at least as high as every provider registration deadline so provider-specific values can cover boot and enrollment. Set this between 1 and ${RESERVATION_TTL_HARD_MAX_SECONDS}.`,
|
|
70
|
+
default: 300,
|
|
71
|
+
}),
|
|
62
72
|
RESERVATION_LONG_POLL_MAX_WAIT_SECONDS: num({
|
|
63
73
|
desc: 'Maximum time the provisioner demand poll endpoint waits for reservable demand before returning, in seconds.',
|
|
64
74
|
default: 30,
|
|
@@ -133,6 +143,17 @@ export const config = createConfig({
|
|
|
133
143
|
}),
|
|
134
144
|
});
|
|
135
145
|
|
|
146
|
+
const parsedRunnerReservedLabels = parseLabelList(config.RUNNER_RESERVED_LABELS);
|
|
147
|
+
const invalidRunnerReservedLabels = findInvalidLabels(parsedRunnerReservedLabels);
|
|
148
|
+
|
|
149
|
+
if (invalidRunnerReservedLabels.length > 0) {
|
|
150
|
+
throw new Error(
|
|
151
|
+
`RUNNER_RESERVED_LABELS contains invalid runner label(s): ${invalidRunnerReservedLabels.join(', ')}`,
|
|
152
|
+
);
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
export const runnerReservedLabels = parsedRunnerReservedLabels;
|
|
156
|
+
|
|
136
157
|
if (
|
|
137
158
|
!Number.isInteger(config.EPHEMERAL_REGISTRATION_TOKEN_TTL_SECONDS) ||
|
|
138
159
|
config.EPHEMERAL_REGISTRATION_TOKEN_TTL_SECONDS < 1 ||
|
|
@@ -209,6 +230,22 @@ if (!Number.isInteger(config.RESERVATION_TTL_SECONDS) || config.RESERVATION_TTL_
|
|
|
209
230
|
);
|
|
210
231
|
}
|
|
211
232
|
|
|
233
|
+
if (
|
|
234
|
+
!Number.isInteger(config.RESERVATION_TTL_MAX_SECONDS) ||
|
|
235
|
+
config.RESERVATION_TTL_MAX_SECONDS < 1 ||
|
|
236
|
+
config.RESERVATION_TTL_MAX_SECONDS > RESERVATION_TTL_HARD_MAX_SECONDS
|
|
237
|
+
) {
|
|
238
|
+
throw new Error(
|
|
239
|
+
`RESERVATION_TTL_MAX_SECONDS (${config.RESERVATION_TTL_MAX_SECONDS}) must be a whole number of seconds between 1 and ${RESERVATION_TTL_HARD_MAX_SECONDS}.`,
|
|
240
|
+
);
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
if (config.RESERVATION_TTL_SECONDS > config.RESERVATION_TTL_MAX_SECONDS) {
|
|
244
|
+
throw new Error(
|
|
245
|
+
`RESERVATION_TTL_SECONDS (${config.RESERVATION_TTL_SECONDS}) must not be greater than RESERVATION_TTL_MAX_SECONDS (${config.RESERVATION_TTL_MAX_SECONDS}).`,
|
|
246
|
+
);
|
|
247
|
+
}
|
|
248
|
+
|
|
212
249
|
if (
|
|
213
250
|
!Number.isInteger(config.RESERVATION_LONG_POLL_MAX_WAIT_SECONDS) ||
|
|
214
251
|
config.RESERVATION_LONG_POLL_MAX_WAIT_SECONDS < 0
|
package/src/core/errors.ts
CHANGED
|
@@ -40,6 +40,15 @@ export class EmptyRunnerLabelsError extends Error {
|
|
|
40
40
|
}
|
|
41
41
|
}
|
|
42
42
|
|
|
43
|
+
export class RunnerLabelsReservedError extends Error {
|
|
44
|
+
constructor(public readonly labels: string[]) {
|
|
45
|
+
super(
|
|
46
|
+
`All supplied runner labels are reserved for installation-scope provisioners: ${labels.join(', ')}`,
|
|
47
|
+
);
|
|
48
|
+
this.name = 'RunnerLabelsReservedError';
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
43
52
|
export class EmptyRequiredLabelsError extends Error {
|
|
44
53
|
constructor() {
|
|
45
54
|
super('Required labels cannot be empty');
|
package/src/core/index.ts
CHANGED
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
import {afterEach, vi} from '@shipfox/vitest/vi';
|
|
2
|
+
import {inArray} from 'drizzle-orm';
|
|
3
|
+
import {db} from '#db/db.js';
|
|
4
|
+
import {reservations} from '#db/schema/reservations.js';
|
|
5
|
+
import {runnerActivationTokens} from '#db/schema/runner-activation-tokens.js';
|
|
6
|
+
import {runnerControlSessions} from '#db/schema/runner-control-sessions.js';
|
|
7
|
+
import {providerRunners} from '#db/schema/runner-instances.js';
|
|
8
|
+
import {
|
|
9
|
+
type RunnerReservationPromotionFailureReason,
|
|
10
|
+
runnerReservationPromotionFailureCount,
|
|
11
|
+
} from '#metrics/instance.js';
|
|
12
|
+
import {enrollRunnerControlSession} from './runner-control-sessions.js';
|
|
13
|
+
|
|
14
|
+
const createdReservationIds = new Set<string>();
|
|
15
|
+
const createdRunnerInstanceIds = new Set<string>();
|
|
16
|
+
|
|
17
|
+
afterEach(async () => {
|
|
18
|
+
const runnerInstanceIds = [...createdRunnerInstanceIds];
|
|
19
|
+
const reservationIds = [...createdReservationIds];
|
|
20
|
+
|
|
21
|
+
if (runnerInstanceIds.length > 0) {
|
|
22
|
+
await db()
|
|
23
|
+
.delete(runnerActivationTokens)
|
|
24
|
+
.where(inArray(runnerActivationTokens.runnerInstanceId, runnerInstanceIds));
|
|
25
|
+
await db()
|
|
26
|
+
.delete(runnerControlSessions)
|
|
27
|
+
.where(inArray(runnerControlSessions.runnerInstanceId, runnerInstanceIds));
|
|
28
|
+
await db().delete(providerRunners).where(inArray(providerRunners.id, runnerInstanceIds));
|
|
29
|
+
}
|
|
30
|
+
if (reservationIds.length > 0)
|
|
31
|
+
await db().delete(reservations).where(inArray(reservations.id, reservationIds));
|
|
32
|
+
|
|
33
|
+
createdRunnerInstanceIds.clear();
|
|
34
|
+
createdReservationIds.clear();
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
describe('enrollRunnerControlSession', () => {
|
|
38
|
+
it('counts expired reservation promotion failures', async () => {
|
|
39
|
+
const provisionerId = crypto.randomUUID();
|
|
40
|
+
const reservation = await createReservation({
|
|
41
|
+
provisionerId,
|
|
42
|
+
expiresAt: new Date(Date.now() - 1_000),
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
await expectPromotionFailure({
|
|
46
|
+
provisionerId,
|
|
47
|
+
intendedReservationId: reservation.id,
|
|
48
|
+
reason: 'reservation-expired',
|
|
49
|
+
});
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
it('counts missing reservation promotion failures', async () => {
|
|
53
|
+
await expectPromotionFailure({
|
|
54
|
+
provisionerId: crypto.randomUUID(),
|
|
55
|
+
intendedReservationId: crypto.randomUUID(),
|
|
56
|
+
reason: 'reservation-not-found',
|
|
57
|
+
});
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
it('counts already-assigned reservation promotion failures', async () => {
|
|
61
|
+
const provisionerId = crypto.randomUUID();
|
|
62
|
+
const intendedReservation = await createReservation({provisionerId});
|
|
63
|
+
const assignedReservation = await createReservation({provisionerId});
|
|
64
|
+
|
|
65
|
+
await expectPromotionFailure({
|
|
66
|
+
provisionerId,
|
|
67
|
+
intendedReservationId: intendedReservation.id,
|
|
68
|
+
reservationId: assignedReservation.id,
|
|
69
|
+
reason: 'already-assigned',
|
|
70
|
+
});
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
it('counts not-assignable reservation promotion failures', async () => {
|
|
74
|
+
const provisionerId = crypto.randomUUID();
|
|
75
|
+
const reservation = await createReservation({
|
|
76
|
+
provisionerId,
|
|
77
|
+
requiredLabels: ['gpu'],
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
await expectPromotionFailure({
|
|
81
|
+
provisionerId,
|
|
82
|
+
intendedReservationId: reservation.id,
|
|
83
|
+
labels: ['linux'],
|
|
84
|
+
reason: 'not-assignable',
|
|
85
|
+
});
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
it('does not count successful reservation promotion', async () => {
|
|
89
|
+
const provisionerId = crypto.randomUUID();
|
|
90
|
+
const reservation = await createReservation({provisionerId});
|
|
91
|
+
const runnerInstanceId = await createRunner({
|
|
92
|
+
provisionerId,
|
|
93
|
+
intendedReservationId: reservation.id,
|
|
94
|
+
});
|
|
95
|
+
const addSpy = vi.spyOn(runnerReservationPromotionFailureCount, 'add');
|
|
96
|
+
|
|
97
|
+
try {
|
|
98
|
+
const activationToken = await enrollRunnerControlSession({
|
|
99
|
+
runnerInstanceId,
|
|
100
|
+
provisionerId,
|
|
101
|
+
labels: ['linux'],
|
|
102
|
+
providerKind: 'docker',
|
|
103
|
+
protocolVersion: '1',
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
expect(activationToken).toEqual(expect.any(String));
|
|
107
|
+
expect(addSpy).not.toHaveBeenCalled();
|
|
108
|
+
} finally {
|
|
109
|
+
addSpy.mockRestore();
|
|
110
|
+
}
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
it('keeps unexpected promotion failures on the error-log path', async () => {
|
|
114
|
+
const provisionerId = crypto.randomUUID();
|
|
115
|
+
const reservation = await createReservation({provisionerId});
|
|
116
|
+
const runnerInstanceId = await createRunner({
|
|
117
|
+
provisionerId,
|
|
118
|
+
intendedReservationId: reservation.id,
|
|
119
|
+
});
|
|
120
|
+
const unexpectedError = new Error('database unavailable');
|
|
121
|
+
const debugLog = vi.fn();
|
|
122
|
+
const errorLog = vi.fn();
|
|
123
|
+
|
|
124
|
+
vi.resetModules();
|
|
125
|
+
vi.doMock('@shipfox/node-opentelemetry', async (importOriginal) => ({
|
|
126
|
+
...(await importOriginal<typeof import('@shipfox/node-opentelemetry')>()),
|
|
127
|
+
logger: () => ({debug: debugLog, error: errorLog}),
|
|
128
|
+
}));
|
|
129
|
+
vi.doMock('#db/runner-assignments.js', async (importOriginal) => ({
|
|
130
|
+
...(await importOriginal<typeof import('#db/runner-assignments.js')>()),
|
|
131
|
+
assignRunnerInstancesTx: vi.fn().mockRejectedValue(unexpectedError),
|
|
132
|
+
}));
|
|
133
|
+
|
|
134
|
+
const {closePostgresClient, createPostgresClient} = await import('@shipfox/node-postgres');
|
|
135
|
+
createPostgresClient();
|
|
136
|
+
|
|
137
|
+
try {
|
|
138
|
+
const {enrollRunnerControlSession: enroll} = await import('./runner-control-sessions.js');
|
|
139
|
+
const result = await enroll({
|
|
140
|
+
runnerInstanceId,
|
|
141
|
+
provisionerId,
|
|
142
|
+
labels: ['linux'],
|
|
143
|
+
providerKind: 'docker',
|
|
144
|
+
protocolVersion: '1',
|
|
145
|
+
});
|
|
146
|
+
|
|
147
|
+
expect(result).toBeNull();
|
|
148
|
+
expect(debugLog).not.toHaveBeenCalled();
|
|
149
|
+
expect(errorLog).toHaveBeenCalledWith(
|
|
150
|
+
expect.objectContaining({err: unexpectedError}),
|
|
151
|
+
'Unexpected failure promoting runner reservation during enrollment',
|
|
152
|
+
);
|
|
153
|
+
} finally {
|
|
154
|
+
await closePostgresClient();
|
|
155
|
+
vi.doUnmock('@shipfox/node-opentelemetry');
|
|
156
|
+
vi.doUnmock('#db/runner-assignments.js');
|
|
157
|
+
vi.resetModules();
|
|
158
|
+
}
|
|
159
|
+
});
|
|
160
|
+
});
|
|
161
|
+
|
|
162
|
+
async function expectPromotionFailure(params: {
|
|
163
|
+
provisionerId: string;
|
|
164
|
+
intendedReservationId: string;
|
|
165
|
+
reservationId?: string;
|
|
166
|
+
labels?: string[];
|
|
167
|
+
reason: RunnerReservationPromotionFailureReason;
|
|
168
|
+
}): Promise<void> {
|
|
169
|
+
const runnerInstanceId = await createRunner(params);
|
|
170
|
+
const addSpy = vi.spyOn(runnerReservationPromotionFailureCount, 'add');
|
|
171
|
+
|
|
172
|
+
try {
|
|
173
|
+
const result = await enrollRunnerControlSession({
|
|
174
|
+
runnerInstanceId,
|
|
175
|
+
provisionerId: params.provisionerId,
|
|
176
|
+
labels: params.labels ?? ['linux'],
|
|
177
|
+
providerKind: 'docker',
|
|
178
|
+
protocolVersion: '1',
|
|
179
|
+
});
|
|
180
|
+
|
|
181
|
+
expect(result).toBeNull();
|
|
182
|
+
expect(addSpy).toHaveBeenCalledWith(1, {reason: params.reason});
|
|
183
|
+
} finally {
|
|
184
|
+
addSpy.mockRestore();
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
async function createReservation(params: {
|
|
189
|
+
provisionerId: string;
|
|
190
|
+
requiredLabels?: string[];
|
|
191
|
+
expiresAt?: Date;
|
|
192
|
+
}) {
|
|
193
|
+
const [reservation] = await db()
|
|
194
|
+
.insert(reservations)
|
|
195
|
+
.values({
|
|
196
|
+
workspaceId: crypto.randomUUID(),
|
|
197
|
+
provisionerId: params.provisionerId,
|
|
198
|
+
requiredLabels: params.requiredLabels ?? ['linux'],
|
|
199
|
+
count: 1,
|
|
200
|
+
expiresAt: params.expiresAt ?? new Date(Date.now() + 60_000),
|
|
201
|
+
})
|
|
202
|
+
.returning();
|
|
203
|
+
if (!reservation) throw new Error('Expected reservation');
|
|
204
|
+
createdReservationIds.add(reservation.id);
|
|
205
|
+
return reservation;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
async function createRunner(params: {
|
|
209
|
+
provisionerId: string;
|
|
210
|
+
intendedReservationId: string;
|
|
211
|
+
reservationId?: string;
|
|
212
|
+
labels?: string[];
|
|
213
|
+
}): Promise<string> {
|
|
214
|
+
const [runner] = await db()
|
|
215
|
+
.insert(providerRunners)
|
|
216
|
+
.values({
|
|
217
|
+
provisionerId: params.provisionerId,
|
|
218
|
+
intendedReservationId: params.intendedReservationId,
|
|
219
|
+
reservationId: params.reservationId ?? null,
|
|
220
|
+
providerRunnerId: crypto.randomUUID(),
|
|
221
|
+
labels: params.labels ?? [],
|
|
222
|
+
state: 'starting',
|
|
223
|
+
reportedAt: new Date(),
|
|
224
|
+
})
|
|
225
|
+
.returning({id: providerRunners.id});
|
|
226
|
+
if (!runner) throw new Error('Expected runner instance');
|
|
227
|
+
createdRunnerInstanceIds.add(runner.id);
|
|
228
|
+
|
|
229
|
+
await db()
|
|
230
|
+
.insert(runnerControlSessions)
|
|
231
|
+
.values({
|
|
232
|
+
runnerInstanceId: runner.id,
|
|
233
|
+
provisionerId: params.provisionerId,
|
|
234
|
+
hashedToken: crypto.randomUUID(),
|
|
235
|
+
prefix: 'test',
|
|
236
|
+
expiresAt: new Date(Date.now() + 60_000),
|
|
237
|
+
});
|
|
238
|
+
return runner.id;
|
|
239
|
+
}
|
|
@@ -2,7 +2,6 @@ import type {RunnerToolCapabilitiesDto} from '@shipfox/api-runners-dto';
|
|
|
2
2
|
import type {NodePgDatabase} from '@shipfox/node-drizzle';
|
|
3
3
|
import {logger} from '@shipfox/node-opentelemetry';
|
|
4
4
|
import {extractDisplayPrefix, generateOpaqueToken, hashOpaqueToken} from '@shipfox/node-tokens';
|
|
5
|
-
import {canonicalizeLabels} from '@shipfox/runner-labels';
|
|
6
5
|
import {and, eq, gt, isNull, notInArray, or, sql} from 'drizzle-orm';
|
|
7
6
|
import {config} from '#config.js';
|
|
8
7
|
import {
|
|
@@ -11,12 +10,17 @@ import {
|
|
|
11
10
|
RunnerInstanceAlreadyAssignedError,
|
|
12
11
|
RunnerInstanceNotAssignableError,
|
|
13
12
|
} from '#core/errors.js';
|
|
13
|
+
import {sanitizeRunnerLabels} from '#core/runner-labels.js';
|
|
14
14
|
import {db, type schema, type Tx} from '#db/db.js';
|
|
15
15
|
import {assignRunnerInstancesTx} from '#db/runner-assignments.js';
|
|
16
16
|
import {terminalStates} from '#db/runner-instances.js';
|
|
17
17
|
import {provisionerTokens} from '#db/schema/provisioner-tokens.js';
|
|
18
18
|
import {runnerBootstrapTokens, runnerControlSessions} from '#db/schema/runner-control-sessions.js';
|
|
19
19
|
import {providerRunners} from '#db/schema/runner-instances.js';
|
|
20
|
+
import {
|
|
21
|
+
type RunnerReservationPromotionFailureReason,
|
|
22
|
+
recordRunnerReservationPromotionFailure,
|
|
23
|
+
} from '#metrics/index.js';
|
|
20
24
|
import {issueRunnerActivationTokenTx} from './runner-activation.js';
|
|
21
25
|
|
|
22
26
|
export class RunnerBootstrapTokenInvalidError extends Error {
|
|
@@ -141,8 +145,12 @@ export async function enrollRunnerControlSession(params: {
|
|
|
141
145
|
}): Promise<string | null> {
|
|
142
146
|
return await db().transaction(async (tx) => {
|
|
143
147
|
const [current] = await tx
|
|
144
|
-
.select({
|
|
148
|
+
.select({
|
|
149
|
+
intendedReservationId: providerRunners.intendedReservationId,
|
|
150
|
+
provisionerScope: provisionerTokens.scope,
|
|
151
|
+
})
|
|
145
152
|
.from(providerRunners)
|
|
153
|
+
.innerJoin(provisionerTokens, eq(provisionerTokens.id, providerRunners.provisionerId))
|
|
146
154
|
.where(
|
|
147
155
|
and(
|
|
148
156
|
eq(providerRunners.id, params.runnerInstanceId),
|
|
@@ -158,7 +166,10 @@ export async function enrollRunnerControlSession(params: {
|
|
|
158
166
|
const [updated] = await tx
|
|
159
167
|
.update(providerRunners)
|
|
160
168
|
.set({
|
|
161
|
-
labels:
|
|
169
|
+
labels: sanitizeRunnerLabels(params.labels, {
|
|
170
|
+
scope: current.provisionerScope,
|
|
171
|
+
source: 'runner control enrollment',
|
|
172
|
+
}),
|
|
162
173
|
providerKind: params.providerKind,
|
|
163
174
|
protocolVersion: params.protocolVersion,
|
|
164
175
|
capabilities: params.capabilities ?? null,
|
|
@@ -198,29 +209,37 @@ export async function enrollRunnerControlSession(params: {
|
|
|
198
209
|
});
|
|
199
210
|
});
|
|
200
211
|
} catch (error) {
|
|
201
|
-
const
|
|
202
|
-
error instanceof ReservationExpiredError ||
|
|
203
|
-
error instanceof ReservationNotFoundError ||
|
|
204
|
-
error instanceof RunnerInstanceAlreadyAssignedError ||
|
|
205
|
-
error instanceof RunnerInstanceNotAssignableError;
|
|
212
|
+
const expectedFailureReason = getRunnerReservationPromotionFailureReason(error);
|
|
206
213
|
const details = {
|
|
207
214
|
err: error,
|
|
208
215
|
runnerInstanceId: params.runnerInstanceId,
|
|
209
216
|
provisionerId: params.provisionerId,
|
|
210
217
|
reservationId: intendedReservationId,
|
|
211
218
|
};
|
|
212
|
-
if (
|
|
219
|
+
if (expectedFailureReason) {
|
|
220
|
+
recordRunnerReservationPromotionFailure(expectedFailureReason);
|
|
213
221
|
logger().debug(details, 'Runner reservation could not be promoted during enrollment');
|
|
214
|
-
else
|
|
222
|
+
} else {
|
|
215
223
|
logger().error(
|
|
216
224
|
details,
|
|
217
225
|
'Unexpected failure promoting runner reservation during enrollment',
|
|
218
226
|
);
|
|
227
|
+
}
|
|
219
228
|
return null;
|
|
220
229
|
}
|
|
221
230
|
});
|
|
222
231
|
}
|
|
223
232
|
|
|
233
|
+
function getRunnerReservationPromotionFailureReason(
|
|
234
|
+
error: unknown,
|
|
235
|
+
): RunnerReservationPromotionFailureReason | null {
|
|
236
|
+
if (error instanceof ReservationExpiredError) return 'reservation-expired';
|
|
237
|
+
if (error instanceof ReservationNotFoundError) return 'reservation-not-found';
|
|
238
|
+
if (error instanceof RunnerInstanceAlreadyAssignedError) return 'already-assigned';
|
|
239
|
+
if (error instanceof RunnerInstanceNotAssignableError) return 'not-assignable';
|
|
240
|
+
return null;
|
|
241
|
+
}
|
|
242
|
+
|
|
224
243
|
export async function attachRunnerControlProviderId(params: {
|
|
225
244
|
runnerInstanceId: string;
|
|
226
245
|
provisionerId: string;
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import {RunnerLabelsReservedError} from './errors.js';
|
|
2
|
+
import {sanitizeRunnerLabels, sanitizeRunnerLabelsOrThrow} from './runner-labels.js';
|
|
3
|
+
|
|
4
|
+
describe('sanitizeRunnerLabels', () => {
|
|
5
|
+
it('strips reserved labels from non-installation sources', () => {
|
|
6
|
+
const labels = sanitizeRunnerLabels([' Linux ', 'shipfox-managed', 'linux'], {
|
|
7
|
+
scope: 'workspace',
|
|
8
|
+
source: 'test',
|
|
9
|
+
reservedLabels: ['Shipfox-Managed'],
|
|
10
|
+
});
|
|
11
|
+
|
|
12
|
+
expect(labels).toEqual(['linux']);
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
it('preserves reserved labels for installation sources', () => {
|
|
16
|
+
const labels = sanitizeRunnerLabels([' Linux ', 'shipfox-managed'], {
|
|
17
|
+
scope: 'installation',
|
|
18
|
+
source: 'test',
|
|
19
|
+
reservedLabels: ['shipfox-managed'],
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
expect(labels).toEqual(['linux', 'shipfox-managed']);
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
it('does not change labels when no reserved labels are configured', () => {
|
|
26
|
+
const labels = sanitizeRunnerLabels([' Linux ', 'linux'], {
|
|
27
|
+
scope: 'manual',
|
|
28
|
+
source: 'test',
|
|
29
|
+
reservedLabels: [],
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
expect(labels).toEqual(['linux']);
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
it('uses the configured reserved labels when none are provided explicitly', () => {
|
|
36
|
+
const labels = sanitizeRunnerLabels(['linux', 'shipfox-managed'], {
|
|
37
|
+
scope: 'workspace',
|
|
38
|
+
source: 'test',
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
expect(labels).toEqual(['linux']);
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
it('throws when sanitization removes every canonical label', () => {
|
|
45
|
+
expect(() =>
|
|
46
|
+
sanitizeRunnerLabelsOrThrow([' Shipfox-Managed '], {
|
|
47
|
+
scope: 'workspace',
|
|
48
|
+
source: 'test',
|
|
49
|
+
reservedLabels: ['shipfox-managed'],
|
|
50
|
+
}),
|
|
51
|
+
).toThrow(RunnerLabelsReservedError);
|
|
52
|
+
});
|
|
53
|
+
});
|