@shipfox/api-runners 12.2.0 → 12.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +4 -4
- package/CHANGELOG.md +34 -0
- package/dist/config.d.ts +1 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +10 -3
- package/dist/config.js.map +1 -1
- package/dist/core/demand.d.ts +4 -2
- package/dist/core/demand.d.ts.map +1 -1
- package/dist/core/demand.js +22 -6
- package/dist/core/demand.js.map +1 -1
- package/dist/core/entities/reservation.d.ts +1 -0
- package/dist/core/entities/reservation.d.ts.map +1 -1
- package/dist/core/entities/reservation.js.map +1 -1
- package/dist/core/entities/runner-instance.d.ts +3 -0
- package/dist/core/entities/runner-instance.d.ts.map +1 -1
- package/dist/core/entities/runner-instance.js.map +1 -1
- package/dist/core/runner-activation.d.ts +4 -1
- package/dist/core/runner-activation.d.ts.map +1 -1
- package/dist/core/runner-activation.js +32 -3
- package/dist/core/runner-activation.js.map +1 -1
- package/dist/core/runner-control-sessions.d.ts +8 -4
- package/dist/core/runner-control-sessions.d.ts.map +1 -1
- package/dist/core/runner-control-sessions.js +69 -11
- package/dist/core/runner-control-sessions.js.map +1 -1
- package/dist/core/runner-instances.d.ts +1 -0
- package/dist/core/runner-instances.d.ts.map +1 -1
- package/dist/core/runner-instances.js +1 -0
- package/dist/core/runner-instances.js.map +1 -1
- package/dist/db/db.d.ts +68 -0
- package/dist/db/db.d.ts.map +1 -1
- package/dist/db/ephemeral-registration-tokens.d.ts.map +1 -1
- package/dist/db/ephemeral-registration-tokens.js +1 -1
- package/dist/db/ephemeral-registration-tokens.js.map +1 -1
- package/dist/db/index.d.ts +1 -1
- package/dist/db/index.d.ts.map +1 -1
- package/dist/db/index.js +1 -1
- package/dist/db/index.js.map +1 -1
- package/dist/db/reservation-locks.d.ts +6 -0
- package/dist/db/reservation-locks.d.ts.map +1 -0
- package/dist/db/reservation-locks.js +15 -0
- package/dist/db/reservation-locks.js.map +1 -0
- package/dist/db/reservations.d.ts +17 -3
- package/dist/db/reservations.d.ts.map +1 -1
- package/dist/db/reservations.js +256 -45
- package/dist/db/reservations.js.map +1 -1
- package/dist/db/runner-assignments.d.ts +17 -0
- package/dist/db/runner-assignments.d.ts.map +1 -1
- package/dist/db/runner-assignments.js +103 -8
- package/dist/db/runner-assignments.js.map +1 -1
- package/dist/db/runner-instances.d.ts +5 -2
- package/dist/db/runner-instances.d.ts.map +1 -1
- package/dist/db/runner-instances.js +134 -25
- package/dist/db/runner-instances.js.map +1 -1
- package/dist/db/runner-sessions.d.ts.map +1 -1
- package/dist/db/runner-sessions.js +22 -15
- package/dist/db/runner-sessions.js.map +1 -1
- package/dist/db/runner-states.d.ts +2 -0
- package/dist/db/runner-states.d.ts.map +1 -0
- package/dist/db/runner-states.js +7 -0
- package/dist/db/runner-states.js.map +1 -0
- package/dist/db/schema/reservations.d.ts +18 -0
- package/dist/db/schema/reservations.d.ts.map +1 -1
- package/dist/db/schema/reservations.js +7 -1
- package/dist/db/schema/reservations.js.map +1 -1
- package/dist/db/schema/runner-instances.d.ts +18 -0
- package/dist/db/schema/runner-instances.d.ts.map +1 -1
- package/dist/db/schema/runner-instances.js +8 -0
- package/dist/db/schema/runner-instances.js.map +1 -1
- package/dist/metrics/index.d.ts +2 -2
- package/dist/metrics/index.d.ts.map +1 -1
- package/dist/metrics/index.js +1 -1
- package/dist/metrics/index.js.map +1 -1
- package/dist/metrics/instance.d.ts +24 -2
- package/dist/metrics/instance.d.ts.map +1 -1
- package/dist/metrics/instance.js +23 -0
- package/dist/metrics/instance.js.map +1 -1
- package/dist/metrics/service.d.ts.map +1 -1
- package/dist/metrics/service.js +20 -4
- package/dist/metrics/service.js.map +1 -1
- package/dist/presentation/dto/poll-demand.d.ts.map +1 -1
- package/dist/presentation/dto/poll-demand.js +3 -0
- package/dist/presentation/dto/poll-demand.js.map +1 -1
- package/dist/presentation/dto/runner-instances.d.ts +4 -1
- package/dist/presentation/dto/runner-instances.d.ts.map +1 -1
- package/dist/presentation/dto/runner-instances.js +4 -1
- package/dist/presentation/dto/runner-instances.js.map +1 -1
- package/dist/presentation/routes/assign-runner-instances.d.ts.map +1 -1
- package/dist/presentation/routes/assign-runner-instances.js +3 -3
- package/dist/presentation/routes/assign-runner-instances.js.map +1 -1
- package/dist/presentation/routes/poll-demand.d.ts.map +1 -1
- package/dist/presentation/routes/poll-demand.js +1 -0
- package/dist/presentation/routes/poll-demand.js.map +1 -1
- package/dist/presentation/routes/reconcile-runner-instances.d.ts.map +1 -1
- package/dist/presentation/routes/reconcile-runner-instances.js +4 -2
- package/dist/presentation/routes/reconcile-runner-instances.js.map +1 -1
- package/dist/presentation/routes/runner-enrollment.d.ts +4 -1
- package/dist/presentation/routes/runner-enrollment.d.ts.map +1 -1
- package/dist/presentation/routes/runner-enrollment.js +11 -6
- package/dist/presentation/routes/runner-enrollment.js.map +1 -1
- package/dist/tsconfig.test.tsbuildinfo +1 -1
- package/drizzle/0005_previous_lenny_balinger.sql +5 -0
- package/drizzle/meta/0005_snapshot.json +2220 -0
- package/drizzle/meta/_journal.json +7 -0
- package/package.json +4 -4
- package/src/config.test.ts +24 -2
- package/src/config.ts +16 -3
- package/src/core/demand.test.ts +192 -3
- package/src/core/demand.ts +44 -4
- package/src/core/entities/reservation.ts +1 -0
- package/src/core/entities/runner-instance.ts +4 -0
- package/src/core/runner-activation.ts +48 -2
- package/src/core/runner-control-sessions.test.ts +230 -4
- package/src/core/runner-control-sessions.ts +84 -10
- package/src/core/runner-instances.test.ts +19 -0
- package/src/core/runner-instances.ts +2 -0
- package/src/core/runner-sessions.test.ts +121 -1
- package/src/db/ephemeral-registration-tokens.test.ts +20 -1
- package/src/db/ephemeral-registration-tokens.ts +1 -0
- package/src/db/index.ts +1 -0
- package/src/db/reservation-locks.ts +22 -0
- package/src/db/reservations.test.ts +1110 -31
- package/src/db/reservations.ts +434 -105
- package/src/db/runner-assignments.test.ts +105 -1
- package/src/db/runner-assignments.ts +178 -9
- package/src/db/runner-instances.test.ts +496 -2
- package/src/db/runner-instances.ts +247 -56
- package/src/db/runner-sessions.ts +26 -15
- package/src/db/runner-states.ts +7 -0
- package/src/db/schema/reservations.ts +5 -1
- package/src/db/schema/runner-instances.ts +9 -0
- package/src/metrics/index.ts +10 -1
- package/src/metrics/instance.ts +59 -2
- package/src/metrics/service.test.ts +106 -0
- package/src/metrics/service.ts +24 -4
- package/src/presentation/dto/poll-demand.ts +3 -0
- package/src/presentation/dto/runner-instances.ts +8 -1
- package/src/presentation/routes/assign-runner-instances.ts +6 -2
- package/src/presentation/routes/late-runner-enrollment.test.ts +447 -0
- package/src/presentation/routes/poll-demand.test.ts +79 -6
- package/src/presentation/routes/poll-demand.ts +1 -0
- package/src/presentation/routes/reconcile-runner-instances.test.ts +34 -2
- package/src/presentation/routes/reconcile-runner-instances.ts +7 -1
- package/src/presentation/routes/runner-enrollment.test.ts +316 -3
- package/src/presentation/routes/runner-enrollment.ts +8 -5
- package/test/factories/reservation.ts +2 -0
- package/test/factories/runner-instance.ts +6 -0
- package/test/fixtures/late-runner-enrollment.ts +103 -0
- package/test/index.ts +5 -0
- package/tsconfig.build.tsbuildinfo +1 -1
package/.turbo/turbo-build.log
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
$ shipfox-swc && shipfox-temporal-bundle dist/temporal/workflows/index.js
|
|
2
|
-
Successfully compiled:
|
|
3
|
-
2026-08-
|
|
2
|
+
Successfully compiled: 104 files with swc (886.73ms)
|
|
3
|
+
2026-08-08T09:17:02.061Z [INFO] asset workflow-bundle-0c3da2487e29e352d64e.js 3.17 MiB [emitted] [immutable] (name: main)
|
|
4
4
|
orphan modules 33.5 KiB [orphan] 21 modules
|
|
5
5
|
runtime modules 1.13 KiB 5 modules
|
|
6
6
|
modules by path ../../../node_modules/.pnpm/ 970 KiB 196 modules
|
|
@@ -14,6 +14,6 @@ optional modules 30 bytes [optional]
|
|
|
14
14
|
__temporal_custom_payload_converter (ignored) 15 bytes [optional] [built] [code generated]
|
|
15
15
|
__temporal_custom_failure_converter (ignored) 15 bytes [optional] [built] [code generated]
|
|
16
16
|
../../shared/node/temporal/dist/workflow-error-interceptor.js 1.21 KiB [built] [code generated]
|
|
17
|
-
webpack 5.107.2 compiled successfully in
|
|
18
|
-
2026-08-
|
|
17
|
+
webpack 5.107.2 compiled successfully in 12068 ms
|
|
18
|
+
2026-08-08T09:17:02.096Z [INFO] Workflow bundle created { size: '3.17MB' }
|
|
19
19
|
/home/runner/work/shipfox/shipfox/libs/api/runners/dist/temporal/workflows/index.js: 3326598 bytes
|
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,39 @@
|
|
|
1
1
|
# @shipfox/api-runners
|
|
2
2
|
|
|
3
|
+
## 12.4.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- 9e16946: Runners are enrolled only when the reservation still has capacity, and partial-enrollment errors now report reservation shortfalls separately from launch grants.
|
|
8
|
+
- 4fa8526: Adds opt-in reconciliation data so provisioners can retry intended reservation assignments.
|
|
9
|
+
|
|
10
|
+
### Patch Changes
|
|
11
|
+
|
|
12
|
+
- 99d798e: Repairs provider-reported runner assignments before activation and preserves reservation capacity during retries.
|
|
13
|
+
- c98cd02: Deduct only pending launch units from provisioner capacity across reservation polls.
|
|
14
|
+
- 67cc90d: Avoid provisional bound reservation writes when no runner can be rebound.
|
|
15
|
+
- 2dbe43b: Split rebound and launch reservations so rebinding an idle runner does not create duplicate launch capacity.
|
|
16
|
+
- e930624: Adds low-cardinality metrics and debug logs that identify why runner activation token issuance stops.
|
|
17
|
+
- 57a6407: Reclaims demand-backed runners that never activate after their independent recovery window expires.
|
|
18
|
+
- e41fc8d: Require provisioner reservation token and assignment flows to use launch reservations.
|
|
19
|
+
- 0c2cee4: Align demand polling and reservation cleanup runner row lock ordering.
|
|
20
|
+
Recheck runner eligibility before binding after row-lock acquisition.
|
|
21
|
+
- Updated dependencies [9e16946]
|
|
22
|
+
- Updated dependencies [4fa8526]
|
|
23
|
+
- @shipfox/api-runners-dto@12.4.0
|
|
24
|
+
|
|
25
|
+
## 12.3.0
|
|
26
|
+
|
|
27
|
+
### Patch Changes
|
|
28
|
+
|
|
29
|
+
- ed3ea25: Raise the default runner reservation TTL ceiling to 600 seconds so longer EC2 launch budgets are accepted while the cap still limits unreleased-reservation demand.
|
|
30
|
+
- 72fb19d: Allow idle runners with expired or deleted intended or assigned reservations to be rebound on the next demand poll.
|
|
31
|
+
- 32bc717: Preserves committed runner assignment metadata during enrollment.
|
|
32
|
+
- 4bd55e8: Adds a service gauge for enrolled runners without a recent provisioner report after the stale-runner grace window.
|
|
33
|
+
- 4a5e847: Uses provider reservation expiry as the activation grace period before stale runner assignments are rebound.
|
|
34
|
+
- Updated dependencies [4b0731e]
|
|
35
|
+
- @shipfox/api-workflows-dto@12.3.0
|
|
36
|
+
|
|
3
37
|
## 12.2.0
|
|
4
38
|
|
|
5
39
|
### Minor Changes
|
package/dist/config.d.ts
CHANGED
|
@@ -18,6 +18,7 @@ export declare const config: Readonly<{
|
|
|
18
18
|
RESERVATION_POLL_INTERVAL_MS: number;
|
|
19
19
|
RESERVATION_POLL_MAX_INTERVAL_MS: number;
|
|
20
20
|
RUNNER_ACTIVE_WINDOW_SECONDS: number;
|
|
21
|
+
RUNNER_DEMAND_ACTIVATION_TIMEOUT_SECONDS: number;
|
|
21
22
|
RUNNER_TOOL_CAPABILITIES_STALE_AFTER_SECONDS: number;
|
|
22
23
|
RUNNER_NO_FIRST_HEARTBEAT_GRACE_SECONDS: number;
|
|
23
24
|
RUNNER_RECONCILE_TERMINATE_GRACE_SECONDS: number;
|
package/dist/config.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAUA,eAAO,MAAM,MAAM
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAUA,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kDAyIjB,CAAC;AAWH,eAAO,MAAM,oBAAoB,mBAA6B,CAAC"}
|
package/dist/config.js
CHANGED
|
@@ -60,12 +60,12 @@ export const config = createConfig({
|
|
|
60
60
|
default: 250
|
|
61
61
|
}),
|
|
62
62
|
RESERVATION_TTL_SECONDS: num({
|
|
63
|
-
desc: '
|
|
63
|
+
desc: 'Activation grace period for a rebound runner reservation, in seconds. A runner without a session can be rebound again only after this deadline. It is also the default lifetime for launch reservations when a provisioner does not request a provider-specific value.',
|
|
64
64
|
default: 60
|
|
65
65
|
}),
|
|
66
66
|
RESERVATION_TTL_MAX_SECONDS: num({
|
|
67
|
-
desc: `Server-side ceiling for a
|
|
68
|
-
default:
|
|
67
|
+
desc: `Server-side ceiling for a launch reservation lifetime, in seconds. Set it at least as high as every provider registration deadline plus its launch headroom so provider-specific values can cover the launch gap, boot, and enrollment. Set this between 1 and ${RESERVATION_TTL_HARD_MAX_SECONDS}.`,
|
|
68
|
+
default: 600
|
|
69
69
|
}),
|
|
70
70
|
RESERVATION_LONG_POLL_MAX_WAIT_SECONDS: num({
|
|
71
71
|
desc: 'Maximum time the provisioner demand poll endpoint waits for reservable demand before returning, in seconds.',
|
|
@@ -83,6 +83,10 @@ export const config = createConfig({
|
|
|
83
83
|
desc: 'Time window, in seconds, used to list active runners from recent heartbeats and provisioned runner reports.',
|
|
84
84
|
default: 60
|
|
85
85
|
}),
|
|
86
|
+
RUNNER_DEMAND_ACTIVATION_TIMEOUT_SECONDS: num({
|
|
87
|
+
desc: 'Minimum time, in seconds, before a demand-backed runner without a runner session can be reclaimed when neither its current nor intended reservation is live. Set this above the provider registration deadline when startup reservations remain live during boot.',
|
|
88
|
+
default: 300
|
|
89
|
+
}),
|
|
86
90
|
RUNNER_TOOL_CAPABILITIES_STALE_AFTER_SECONDS: num({
|
|
87
91
|
desc: 'Time window, in seconds, after which a runner tool capability report is treated as stale. Set this higher than the runner heartbeat interval so active runners keep their advertised tools fresh.',
|
|
88
92
|
default: 60
|
|
@@ -223,6 +227,9 @@ if (!Number.isInteger(config.RESERVATION_POLL_MAX_INTERVAL_MS) || config.RESERVA
|
|
|
223
227
|
if (!Number.isInteger(config.RUNNER_ACTIVE_WINDOW_SECONDS) || config.RUNNER_ACTIVE_WINDOW_SECONDS < 1) {
|
|
224
228
|
throw new Error(`RUNNER_ACTIVE_WINDOW_SECONDS (${config.RUNNER_ACTIVE_WINDOW_SECONDS}) must be a whole number of seconds >= 1.`);
|
|
225
229
|
}
|
|
230
|
+
if (!Number.isInteger(config.RUNNER_DEMAND_ACTIVATION_TIMEOUT_SECONDS) || config.RUNNER_DEMAND_ACTIVATION_TIMEOUT_SECONDS < 1) {
|
|
231
|
+
throw new Error(`RUNNER_DEMAND_ACTIVATION_TIMEOUT_SECONDS (${config.RUNNER_DEMAND_ACTIVATION_TIMEOUT_SECONDS}) must be a whole number of seconds >= 1.`);
|
|
232
|
+
}
|
|
226
233
|
if (!Number.isInteger(config.RUNNER_TOOL_CAPABILITIES_STALE_AFTER_SECONDS) || config.RUNNER_TOOL_CAPABILITIES_STALE_AFTER_SECONDS < 1) {
|
|
227
234
|
throw new Error(`RUNNER_TOOL_CAPABILITIES_STALE_AFTER_SECONDS (${config.RUNNER_TOOL_CAPABILITIES_STALE_AFTER_SECONDS}) must be a whole number of seconds >= 1.`);
|
|
228
235
|
}
|
package/dist/config.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/config.ts"],"sourcesContent":["import {RUNNER_ASSIGNMENT_POLL_DEFAULT_WAIT_SECONDS} from '@shipfox/api-runners-dto';\nimport {bool, createConfig, num, str} from '@shipfox/config';\nimport {findInvalidLabels, parseLabelList} from '@shipfox/runner-labels';\nimport {STUCK_JOB_THRESHOLD_SECONDS} from '#core/maintenance-policy.js';\n\nconst EPHEMERAL_REGISTRATION_TOKEN_TTL_HARD_MAX_SECONDS = 3600;\nconst REGISTRATION_TOKEN_BATCH_HARD_MAX = 1000;\nconst RUNNER_CONTROL_PLANE_TOKEN_TTL_HARD_MAX_SECONDS = 3600;\nconst RESERVATION_TTL_HARD_MAX_SECONDS = 3600;\n\nexport const config = createConfig({\n RUNNER_BOOTSTRAP_TOKEN_TTL_SECONDS: num({\n desc: 'Lifetime of a one-use runner bootstrap token in seconds. The token can only create one runner-control session before it expires.',\n default: 300,\n }),\n RUNNER_CONTROL_SESSION_TTL_SECONDS: num({\n desc: 'Lifetime of a pre-workspace runner-control session in seconds. This session can only enroll, attach its provider identity, and report its own liveness.',\n default: 3600,\n }),\n RUNNER_ACTIVATION_TOKEN_TTL_SECONDS: num({\n desc: 'Lifetime of a one-use activation token in seconds. The token is issued only to an assigned runner-control session and creates one workspace runner session.',\n default: 300,\n }),\n RUNNER_ASSIGNMENT_POLL_MAX_WAIT_SECONDS: num({\n desc: 'Maximum server-side cap for the per-request runner assignment wait, in seconds. Managed runners request their own bounded wait with wait_seconds.',\n default: RUNNER_ASSIGNMENT_POLL_DEFAULT_WAIT_SECONDS,\n }),\n RUNNER_ASSIGNMENT_POLL_INTERVAL_MS: num({\n desc: 'Delay between durable assignment reads while a runner-control session waits for an assignment, in milliseconds.',\n default: 250,\n }),\n RUNNER_RESERVED_LABELS: str({\n desc: 'Comma-separated labels that only installation-scope provisioners may advertise. Any other provisioner or runner has them removed.',\n default: '',\n }),\n EPHEMERAL_REGISTRATION_TOKEN_TTL_SECONDS: num({\n desc: `Lifetime of a provisioner-minted runner registration token, in seconds. The token can be exchanged once by a runner before this time passes. Set this between 1 and ${EPHEMERAL_REGISTRATION_TOKEN_TTL_HARD_MAX_SECONDS}.`,\n default: 300,\n }),\n REGISTRATION_TOKEN_BATCH_MAX: num({\n desc: `Maximum number of runner registration tokens a provisioner can mint in one batch request. Set this between 1 and ${REGISTRATION_TOKEN_BATCH_HARD_MAX}.`,\n default: 500,\n }),\n PROVISIONER_MINT_RATE_LIMIT_MAX_REQUESTS: num({\n desc: 'Maximum number of batch runner registration token mint requests one provisioner can make per rate-limit window.',\n default: 120,\n }),\n PROVISIONER_MINT_RATE_LIMIT_WINDOW_SECONDS: num({\n desc: 'Rate-limit window for provisioner batch runner registration token mint requests, in seconds.',\n default: 60,\n }),\n EPHEMERAL_REGISTER_RATE_LIMIT_MAX_REQUESTS: num({\n desc: 'Maximum number of runner registration attempts one ephemeral registration token can make per rate-limit window.',\n default: 5,\n }),\n EPHEMERAL_REGISTER_RATE_LIMIT_WINDOW_SECONDS: num({\n desc: 'Rate-limit window for runner registration attempts with one ephemeral registration token, in seconds.',\n default: 60,\n }),\n RUNNERS_RATE_LIMIT_TIMEOUT_MS: num({\n desc: 'Maximum time, in milliseconds, a runners rate-limit storage check may wait before the request fails closed.',\n default: 250,\n }),\n RESERVATION_TTL_SECONDS: num({\n desc: 'Default lifetime of a count-based runner reservation, in seconds. Used when a provisioner does not request a provider-specific value. Expired reservations stop counting against queued demand.',\n default: 60,\n }),\n RESERVATION_TTL_MAX_SECONDS: num({\n desc: `Server-side ceiling for a count-based runner reservation lifetime, in seconds. Set it at least as high as every provider registration deadline so provider-specific values can cover boot and enrollment. Set this between 1 and ${RESERVATION_TTL_HARD_MAX_SECONDS}.`,\n default: 300,\n }),\n RESERVATION_LONG_POLL_MAX_WAIT_SECONDS: num({\n desc: 'Maximum time the provisioner demand poll endpoint waits for reservable demand before returning, in seconds.',\n default: 30,\n }),\n RESERVATION_POLL_INTERVAL_MS: num({\n desc: 'Initial interval between demand re-checks while a provisioner request is waiting, in milliseconds.',\n default: 1000,\n }),\n RESERVATION_POLL_MAX_INTERVAL_MS: num({\n desc: 'Maximum backoff interval between demand re-checks while a provisioner request is waiting, in milliseconds.',\n default: 5000,\n }),\n RUNNER_ACTIVE_WINDOW_SECONDS: num({\n desc: 'Time window, in seconds, used to list active runners from recent heartbeats and provisioned runner reports.',\n default: 60,\n }),\n RUNNER_TOOL_CAPABILITIES_STALE_AFTER_SECONDS: num({\n desc: 'Time window, in seconds, after which a runner tool capability report is treated as stale. Set this higher than the runner heartbeat interval so active runners keep their advertised tools fresh.',\n default: 60,\n }),\n RUNNER_NO_FIRST_HEARTBEAT_GRACE_SECONDS: num({\n desc: 'Grace window, in seconds, before maintenance expires a claimed job that has not sent its first heartbeat. Set this lower than the normal stuck-job threshold so startup crashes release work quickly.',\n default: 60,\n }),\n RUNNER_RECONCILE_TERMINATE_GRACE_SECONDS: num({\n desc: 'Grace window, in seconds, before reconcile marks an absent provisioned runner as terminated. Set this higher than the provisioner report interval so a transient empty or partial observed set does not kill a live runner.',\n default: 120,\n }),\n RUNNER_STALE_PROVISIONED_RUNNER_THRESHOLD_SECONDS: num({\n desc: 'Time, in seconds, after which a provisioned runner with no recent report, no live provisioner, no live runner session, and no running job is marked failed by backend maintenance.',\n default: 300,\n }),\n RUNNER_STALE_PROVISIONED_RUNNER_REAPER_LIMIT: num({\n desc: 'Maximum number of stale provisioned runners the maintenance worker marks failed in one run. Higher values clear backlogs faster but hold database locks longer.',\n default: 100,\n }),\n RUNNER_SESSION_LIVENESS_THROTTLE_SECONDS: num({\n desc: 'Minimum time, in seconds, between runner session liveness writes from job request polls. Set this lower than RUNNER_STALE_PROVISIONED_RUNNER_THRESHOLD_SECONDS so active idle runners stay fresh.',\n default: 10,\n }),\n RUNNER_SESSION_MANUAL_RETENTION_DAYS: num({\n desc: 'How long manual runner sessions are retained before maintenance deletes them, in days. Set this longer than AUTH_RUNNER_SESSION_TOKEN_EXPIRES_IN so a valid session token never outlives its row.',\n default: 30,\n }),\n RUNNER_SESSION_EPHEMERAL_RETENTION_DAYS: num({\n desc: 'How long ephemeral runner sessions are retained before maintenance deletes them, in days. Set this longer than AUTH_RUNNER_SESSION_TOKEN_EXPIRES_IN so a valid session token never outlives its row.',\n default: 7,\n }),\n RUNNER_SESSION_GC_BATCH_SIZE: num({\n desc: 'Maximum number of expired runner sessions maintenance deletes in one pass.',\n default: 1000,\n }),\n RUNNER_EPHEMERAL_TOKEN_RETENTION_DAYS: num({\n desc: 'How long consumed or expired ephemeral registration tokens are retained for audit and debugging before maintenance deletes them, in days. Active tokens that are neither consumed nor expired are never deleted. This window is independent of the runner-session retention windows.',\n default: 7,\n }),\n RUNNER_EPHEMERAL_TOKEN_GC_BATCH_SIZE: num({\n desc: 'Maximum number of consumed or expired ephemeral registration tokens maintenance deletes in one pass.',\n default: 1000,\n }),\n PROVISIONER_ACTIVE_WINDOW_SECONDS: num({\n desc: 'Time window, in seconds, used to list active provisioners from recent authenticated requests.',\n default: 120,\n }),\n PROVISIONER_LAST_SEEN_THROTTLE_SECONDS: num({\n desc: 'Minimum time, in seconds, between last-seen writes for one provisioner token.',\n default: 10,\n }),\n PROVISIONED_RUNNER_COUNT_DIVERGENCE_TEMPLATE_KEY_LABEL_ENABLED: bool({\n desc: 'Whether runners_provider_runner_count_divergence includes template_key as a metric label. Use true or false. Defaults to false because template keys can create high-cardinality metric series. Set true only when template keys are bounded and stable.',\n default: false,\n }),\n});\n\nconst parsedRunnerReservedLabels = parseLabelList(config.RUNNER_RESERVED_LABELS);\nconst invalidRunnerReservedLabels = findInvalidLabels(parsedRunnerReservedLabels);\n\nif (invalidRunnerReservedLabels.length > 0) {\n throw new Error(\n `RUNNER_RESERVED_LABELS contains invalid runner label(s): ${invalidRunnerReservedLabels.join(', ')}`,\n );\n}\n\nexport const runnerReservedLabels = parsedRunnerReservedLabels;\n\nif (\n !Number.isInteger(config.EPHEMERAL_REGISTRATION_TOKEN_TTL_SECONDS) ||\n config.EPHEMERAL_REGISTRATION_TOKEN_TTL_SECONDS < 1 ||\n config.EPHEMERAL_REGISTRATION_TOKEN_TTL_SECONDS >\n EPHEMERAL_REGISTRATION_TOKEN_TTL_HARD_MAX_SECONDS\n) {\n throw new Error(\n `EPHEMERAL_REGISTRATION_TOKEN_TTL_SECONDS (${config.EPHEMERAL_REGISTRATION_TOKEN_TTL_SECONDS}) must be a whole number of seconds between 1 and ${EPHEMERAL_REGISTRATION_TOKEN_TTL_HARD_MAX_SECONDS}.`,\n );\n}\n\nfor (const [name, value] of [\n ['RUNNER_BOOTSTRAP_TOKEN_TTL_SECONDS', config.RUNNER_BOOTSTRAP_TOKEN_TTL_SECONDS],\n ['RUNNER_CONTROL_SESSION_TTL_SECONDS', config.RUNNER_CONTROL_SESSION_TTL_SECONDS],\n ['RUNNER_ACTIVATION_TOKEN_TTL_SECONDS', config.RUNNER_ACTIVATION_TOKEN_TTL_SECONDS],\n] as const) {\n if (\n !Number.isInteger(value) ||\n value < 1 ||\n value > RUNNER_CONTROL_PLANE_TOKEN_TTL_HARD_MAX_SECONDS\n ) {\n throw new Error(\n `${name} (${value}) must be a whole number of seconds between 1 and ${RUNNER_CONTROL_PLANE_TOKEN_TTL_HARD_MAX_SECONDS}.`,\n );\n }\n}\n\nif (\n !Number.isInteger(config.RUNNER_ASSIGNMENT_POLL_MAX_WAIT_SECONDS) ||\n config.RUNNER_ASSIGNMENT_POLL_MAX_WAIT_SECONDS < 1\n) {\n throw new Error(\n `RUNNER_ASSIGNMENT_POLL_MAX_WAIT_SECONDS (${config.RUNNER_ASSIGNMENT_POLL_MAX_WAIT_SECONDS}) must be a whole number >= 1.`,\n );\n}\n\nif (\n !Number.isInteger(config.RUNNER_ASSIGNMENT_POLL_INTERVAL_MS) ||\n config.RUNNER_ASSIGNMENT_POLL_INTERVAL_MS < 0\n) {\n throw new Error(\n `RUNNER_ASSIGNMENT_POLL_INTERVAL_MS (${config.RUNNER_ASSIGNMENT_POLL_INTERVAL_MS}) must be a whole number >= 0.`,\n );\n}\n\nif (\n !Number.isInteger(config.REGISTRATION_TOKEN_BATCH_MAX) ||\n config.REGISTRATION_TOKEN_BATCH_MAX < 1 ||\n config.REGISTRATION_TOKEN_BATCH_MAX > REGISTRATION_TOKEN_BATCH_HARD_MAX\n) {\n throw new Error(\n `REGISTRATION_TOKEN_BATCH_MAX (${config.REGISTRATION_TOKEN_BATCH_MAX}) must be a whole number between 1 and ${REGISTRATION_TOKEN_BATCH_HARD_MAX}.`,\n );\n}\n\nfor (const [name, value] of [\n ['PROVISIONER_MINT_RATE_LIMIT_MAX_REQUESTS', config.PROVISIONER_MINT_RATE_LIMIT_MAX_REQUESTS],\n ['PROVISIONER_MINT_RATE_LIMIT_WINDOW_SECONDS', config.PROVISIONER_MINT_RATE_LIMIT_WINDOW_SECONDS],\n ['EPHEMERAL_REGISTER_RATE_LIMIT_MAX_REQUESTS', config.EPHEMERAL_REGISTER_RATE_LIMIT_MAX_REQUESTS],\n [\n 'EPHEMERAL_REGISTER_RATE_LIMIT_WINDOW_SECONDS',\n config.EPHEMERAL_REGISTER_RATE_LIMIT_WINDOW_SECONDS,\n ],\n ['RUNNERS_RATE_LIMIT_TIMEOUT_MS', config.RUNNERS_RATE_LIMIT_TIMEOUT_MS],\n] as const) {\n if (!Number.isInteger(value) || value < 1) {\n throw new Error(`${name} (${value}) must be a whole number >= 1.`);\n }\n}\n\nif (!Number.isInteger(config.RESERVATION_TTL_SECONDS) || config.RESERVATION_TTL_SECONDS < 1) {\n throw new Error(\n `RESERVATION_TTL_SECONDS (${config.RESERVATION_TTL_SECONDS}) must be a whole number of seconds >= 1.`,\n );\n}\n\nif (\n !Number.isInteger(config.RESERVATION_TTL_MAX_SECONDS) ||\n config.RESERVATION_TTL_MAX_SECONDS < 1 ||\n config.RESERVATION_TTL_MAX_SECONDS > RESERVATION_TTL_HARD_MAX_SECONDS\n) {\n throw new Error(\n `RESERVATION_TTL_MAX_SECONDS (${config.RESERVATION_TTL_MAX_SECONDS}) must be a whole number of seconds between 1 and ${RESERVATION_TTL_HARD_MAX_SECONDS}.`,\n );\n}\n\nif (config.RESERVATION_TTL_SECONDS > config.RESERVATION_TTL_MAX_SECONDS) {\n throw new Error(\n `RESERVATION_TTL_SECONDS (${config.RESERVATION_TTL_SECONDS}) must not be greater than RESERVATION_TTL_MAX_SECONDS (${config.RESERVATION_TTL_MAX_SECONDS}).`,\n );\n}\n\nif (\n !Number.isInteger(config.RESERVATION_LONG_POLL_MAX_WAIT_SECONDS) ||\n config.RESERVATION_LONG_POLL_MAX_WAIT_SECONDS < 0\n) {\n throw new Error(\n `RESERVATION_LONG_POLL_MAX_WAIT_SECONDS (${config.RESERVATION_LONG_POLL_MAX_WAIT_SECONDS}) must be a whole number of seconds >= 0.`,\n );\n}\n\nif (\n !Number.isInteger(config.RESERVATION_POLL_INTERVAL_MS) ||\n config.RESERVATION_POLL_INTERVAL_MS < 1\n) {\n throw new Error(\n `RESERVATION_POLL_INTERVAL_MS (${config.RESERVATION_POLL_INTERVAL_MS}) must be a whole number of milliseconds >= 1.`,\n );\n}\n\nif (\n !Number.isInteger(config.RESERVATION_POLL_MAX_INTERVAL_MS) ||\n config.RESERVATION_POLL_MAX_INTERVAL_MS < config.RESERVATION_POLL_INTERVAL_MS\n) {\n throw new Error(\n `RESERVATION_POLL_MAX_INTERVAL_MS (${config.RESERVATION_POLL_MAX_INTERVAL_MS}) must be a whole number of milliseconds >= RESERVATION_POLL_INTERVAL_MS (${config.RESERVATION_POLL_INTERVAL_MS}).`,\n );\n}\n\nif (\n !Number.isInteger(config.RUNNER_ACTIVE_WINDOW_SECONDS) ||\n config.RUNNER_ACTIVE_WINDOW_SECONDS < 1\n) {\n throw new Error(\n `RUNNER_ACTIVE_WINDOW_SECONDS (${config.RUNNER_ACTIVE_WINDOW_SECONDS}) must be a whole number of seconds >= 1.`,\n );\n}\n\nif (\n !Number.isInteger(config.RUNNER_TOOL_CAPABILITIES_STALE_AFTER_SECONDS) ||\n config.RUNNER_TOOL_CAPABILITIES_STALE_AFTER_SECONDS < 1\n) {\n throw new Error(\n `RUNNER_TOOL_CAPABILITIES_STALE_AFTER_SECONDS (${config.RUNNER_TOOL_CAPABILITIES_STALE_AFTER_SECONDS}) must be a whole number of seconds >= 1.`,\n );\n}\n\nif (\n !Number.isInteger(config.RUNNER_NO_FIRST_HEARTBEAT_GRACE_SECONDS) ||\n config.RUNNER_NO_FIRST_HEARTBEAT_GRACE_SECONDS < 1 ||\n config.RUNNER_NO_FIRST_HEARTBEAT_GRACE_SECONDS >= STUCK_JOB_THRESHOLD_SECONDS\n) {\n throw new Error(\n `RUNNER_NO_FIRST_HEARTBEAT_GRACE_SECONDS (${config.RUNNER_NO_FIRST_HEARTBEAT_GRACE_SECONDS}) must be a whole number of seconds >= 1 and < ${STUCK_JOB_THRESHOLD_SECONDS}.`,\n );\n}\n\nif (\n !Number.isInteger(config.RUNNER_RECONCILE_TERMINATE_GRACE_SECONDS) ||\n config.RUNNER_RECONCILE_TERMINATE_GRACE_SECONDS < 1\n) {\n throw new Error(\n `RUNNER_RECONCILE_TERMINATE_GRACE_SECONDS (${config.RUNNER_RECONCILE_TERMINATE_GRACE_SECONDS}) must be a whole number of seconds >= 1.`,\n );\n}\n\nif (\n !Number.isInteger(config.RUNNER_STALE_PROVISIONED_RUNNER_THRESHOLD_SECONDS) ||\n config.RUNNER_STALE_PROVISIONED_RUNNER_THRESHOLD_SECONDS < 1\n) {\n throw new Error(\n `RUNNER_STALE_PROVISIONED_RUNNER_THRESHOLD_SECONDS (${config.RUNNER_STALE_PROVISIONED_RUNNER_THRESHOLD_SECONDS}) must be a whole number of seconds >= 1.`,\n );\n}\n\nif (\n !Number.isInteger(config.RUNNER_STALE_PROVISIONED_RUNNER_REAPER_LIMIT) ||\n config.RUNNER_STALE_PROVISIONED_RUNNER_REAPER_LIMIT < 1\n) {\n throw new Error(\n `RUNNER_STALE_PROVISIONED_RUNNER_REAPER_LIMIT (${config.RUNNER_STALE_PROVISIONED_RUNNER_REAPER_LIMIT}) must be a whole number >= 1.`,\n );\n}\n\nif (\n !Number.isInteger(config.RUNNER_SESSION_LIVENESS_THROTTLE_SECONDS) ||\n config.RUNNER_SESSION_LIVENESS_THROTTLE_SECONDS < 1\n) {\n throw new Error(\n `RUNNER_SESSION_LIVENESS_THROTTLE_SECONDS (${config.RUNNER_SESSION_LIVENESS_THROTTLE_SECONDS}) must be a whole number of seconds >= 1.`,\n );\n}\n\nif (\n !Number.isInteger(config.RUNNER_SESSION_MANUAL_RETENTION_DAYS) ||\n config.RUNNER_SESSION_MANUAL_RETENTION_DAYS < 1\n) {\n throw new Error(\n `RUNNER_SESSION_MANUAL_RETENTION_DAYS (${config.RUNNER_SESSION_MANUAL_RETENTION_DAYS}) must be a whole number of days >= 1.`,\n );\n}\n\nif (\n !Number.isInteger(config.RUNNER_SESSION_EPHEMERAL_RETENTION_DAYS) ||\n config.RUNNER_SESSION_EPHEMERAL_RETENTION_DAYS < 1\n) {\n throw new Error(\n `RUNNER_SESSION_EPHEMERAL_RETENTION_DAYS (${config.RUNNER_SESSION_EPHEMERAL_RETENTION_DAYS}) must be a whole number of days >= 1.`,\n );\n}\n\nif (\n !Number.isInteger(config.RUNNER_SESSION_GC_BATCH_SIZE) ||\n config.RUNNER_SESSION_GC_BATCH_SIZE < 1\n) {\n throw new Error(\n `RUNNER_SESSION_GC_BATCH_SIZE (${config.RUNNER_SESSION_GC_BATCH_SIZE}) must be a whole number >= 1.`,\n );\n}\n\nif (\n !Number.isInteger(config.RUNNER_EPHEMERAL_TOKEN_RETENTION_DAYS) ||\n config.RUNNER_EPHEMERAL_TOKEN_RETENTION_DAYS < 1\n) {\n throw new Error(\n `RUNNER_EPHEMERAL_TOKEN_RETENTION_DAYS (${config.RUNNER_EPHEMERAL_TOKEN_RETENTION_DAYS}) must be a whole number of days >= 1.`,\n );\n}\n\nif (\n !Number.isInteger(config.RUNNER_EPHEMERAL_TOKEN_GC_BATCH_SIZE) ||\n config.RUNNER_EPHEMERAL_TOKEN_GC_BATCH_SIZE < 1\n) {\n throw new Error(\n `RUNNER_EPHEMERAL_TOKEN_GC_BATCH_SIZE (${config.RUNNER_EPHEMERAL_TOKEN_GC_BATCH_SIZE}) must be a whole number >= 1.`,\n );\n}\n\nif (\n !Number.isInteger(config.PROVISIONER_ACTIVE_WINDOW_SECONDS) ||\n config.PROVISIONER_ACTIVE_WINDOW_SECONDS < 1\n) {\n throw new Error(\n `PROVISIONER_ACTIVE_WINDOW_SECONDS (${config.PROVISIONER_ACTIVE_WINDOW_SECONDS}) must be a whole number of seconds >= 1.`,\n );\n}\n\nif (\n !Number.isInteger(config.PROVISIONER_LAST_SEEN_THROTTLE_SECONDS) ||\n config.PROVISIONER_LAST_SEEN_THROTTLE_SECONDS < 1\n) {\n throw new Error(\n `PROVISIONER_LAST_SEEN_THROTTLE_SECONDS (${config.PROVISIONER_LAST_SEEN_THROTTLE_SECONDS}) must be a whole number of seconds >= 1.`,\n );\n}\n\nif (\n config.RUNNER_STALE_PROVISIONED_RUNNER_THRESHOLD_SECONDS <=\n config.PROVISIONER_LAST_SEEN_THROTTLE_SECONDS\n) {\n throw new Error(\n `RUNNER_STALE_PROVISIONED_RUNNER_THRESHOLD_SECONDS (${config.RUNNER_STALE_PROVISIONED_RUNNER_THRESHOLD_SECONDS}) must be greater than PROVISIONER_LAST_SEEN_THROTTLE_SECONDS (${config.PROVISIONER_LAST_SEEN_THROTTLE_SECONDS}).`,\n );\n}\n\nif (\n config.RUNNER_STALE_PROVISIONED_RUNNER_THRESHOLD_SECONDS <=\n config.RUNNER_SESSION_LIVENESS_THROTTLE_SECONDS\n) {\n throw new Error(\n `RUNNER_STALE_PROVISIONED_RUNNER_THRESHOLD_SECONDS (${config.RUNNER_STALE_PROVISIONED_RUNNER_THRESHOLD_SECONDS}) must be greater than RUNNER_SESSION_LIVENESS_THROTTLE_SECONDS (${config.RUNNER_SESSION_LIVENESS_THROTTLE_SECONDS}).`,\n );\n}\n"],"names":["RUNNER_ASSIGNMENT_POLL_DEFAULT_WAIT_SECONDS","bool","createConfig","num","str","findInvalidLabels","parseLabelList","STUCK_JOB_THRESHOLD_SECONDS","EPHEMERAL_REGISTRATION_TOKEN_TTL_HARD_MAX_SECONDS","REGISTRATION_TOKEN_BATCH_HARD_MAX","RUNNER_CONTROL_PLANE_TOKEN_TTL_HARD_MAX_SECONDS","RESERVATION_TTL_HARD_MAX_SECONDS","config","RUNNER_BOOTSTRAP_TOKEN_TTL_SECONDS","desc","default","RUNNER_CONTROL_SESSION_TTL_SECONDS","RUNNER_ACTIVATION_TOKEN_TTL_SECONDS","RUNNER_ASSIGNMENT_POLL_MAX_WAIT_SECONDS","RUNNER_ASSIGNMENT_POLL_INTERVAL_MS","RUNNER_RESERVED_LABELS","EPHEMERAL_REGISTRATION_TOKEN_TTL_SECONDS","REGISTRATION_TOKEN_BATCH_MAX","PROVISIONER_MINT_RATE_LIMIT_MAX_REQUESTS","PROVISIONER_MINT_RATE_LIMIT_WINDOW_SECONDS","EPHEMERAL_REGISTER_RATE_LIMIT_MAX_REQUESTS","EPHEMERAL_REGISTER_RATE_LIMIT_WINDOW_SECONDS","RUNNERS_RATE_LIMIT_TIMEOUT_MS","RESERVATION_TTL_SECONDS","RESERVATION_TTL_MAX_SECONDS","RESERVATION_LONG_POLL_MAX_WAIT_SECONDS","RESERVATION_POLL_INTERVAL_MS","RESERVATION_POLL_MAX_INTERVAL_MS","RUNNER_ACTIVE_WINDOW_SECONDS","RUNNER_TOOL_CAPABILITIES_STALE_AFTER_SECONDS","RUNNER_NO_FIRST_HEARTBEAT_GRACE_SECONDS","RUNNER_RECONCILE_TERMINATE_GRACE_SECONDS","RUNNER_STALE_PROVISIONED_RUNNER_THRESHOLD_SECONDS","RUNNER_STALE_PROVISIONED_RUNNER_REAPER_LIMIT","RUNNER_SESSION_LIVENESS_THROTTLE_SECONDS","RUNNER_SESSION_MANUAL_RETENTION_DAYS","RUNNER_SESSION_EPHEMERAL_RETENTION_DAYS","RUNNER_SESSION_GC_BATCH_SIZE","RUNNER_EPHEMERAL_TOKEN_RETENTION_DAYS","RUNNER_EPHEMERAL_TOKEN_GC_BATCH_SIZE","PROVISIONER_ACTIVE_WINDOW_SECONDS","PROVISIONER_LAST_SEEN_THROTTLE_SECONDS","PROVISIONED_RUNNER_COUNT_DIVERGENCE_TEMPLATE_KEY_LABEL_ENABLED","parsedRunnerReservedLabels","invalidRunnerReservedLabels","length","Error","join","runnerReservedLabels","Number","isInteger","name","value"],"mappings":"AAAA,SAAQA,2CAA2C,QAAO,2BAA2B;AACrF,SAAQC,IAAI,EAAEC,YAAY,EAAEC,GAAG,EAAEC,GAAG,QAAO,kBAAkB;AAC7D,SAAQC,iBAAiB,EAAEC,cAAc,QAAO,yBAAyB;AACzE,SAAQC,2BAA2B,QAAO,8BAA8B;AAExE,MAAMC,oDAAoD;AAC1D,MAAMC,oCAAoC;AAC1C,MAAMC,kDAAkD;AACxD,MAAMC,mCAAmC;AAEzC,OAAO,MAAMC,SAASV,aAAa;IACjCW,oCAAoCV,IAAI;QACtCW,MAAM;QACNC,SAAS;IACX;IACAC,oCAAoCb,IAAI;QACtCW,MAAM;QACNC,SAAS;IACX;IACAE,qCAAqCd,IAAI;QACvCW,MAAM;QACNC,SAAS;IACX;IACAG,yCAAyCf,IAAI;QAC3CW,MAAM;QACNC,SAASf;IACX;IACAmB,oCAAoChB,IAAI;QACtCW,MAAM;QACNC,SAAS;IACX;IACAK,wBAAwBhB,IAAI;QAC1BU,MAAM;QACNC,SAAS;IACX;IACAM,0CAA0ClB,IAAI;QAC5CW,MAAM,CAAC,oKAAoK,EAAEN,kDAAkD,CAAC,CAAC;QACjOO,SAAS;IACX;IACAO,8BAA8BnB,IAAI;QAChCW,MAAM,CAAC,iHAAiH,EAAEL,kCAAkC,CAAC,CAAC;QAC9JM,SAAS;IACX;IACAQ,0CAA0CpB,IAAI;QAC5CW,MAAM;QACNC,SAAS;IACX;IACAS,4CAA4CrB,IAAI;QAC9CW,MAAM;QACNC,SAAS;IACX;IACAU,4CAA4CtB,IAAI;QAC9CW,MAAM;QACNC,SAAS;IACX;IACAW,8CAA8CvB,IAAI;QAChDW,MAAM;QACNC,SAAS;IACX;IACAY,+BAA+BxB,IAAI;QACjCW,MAAM;QACNC,SAAS;IACX;IACAa,yBAAyBzB,IAAI;QAC3BW,MAAM;QACNC,SAAS;IACX;IACAc,6BAA6B1B,IAAI;QAC/BW,MAAM,CAAC,iOAAiO,EAAEH,iCAAiC,CAAC,CAAC;QAC7QI,SAAS;IACX;IACAe,wCAAwC3B,IAAI;QAC1CW,MAAM;QACNC,SAAS;IACX;IACAgB,8BAA8B5B,IAAI;QAChCW,MAAM;QACNC,SAAS;IACX;IACAiB,kCAAkC7B,IAAI;QACpCW,MAAM;QACNC,SAAS;IACX;IACAkB,8BAA8B9B,IAAI;QAChCW,MAAM;QACNC,SAAS;IACX;IACAmB,8CAA8C/B,IAAI;QAChDW,MAAM;QACNC,SAAS;IACX;IACAoB,yCAAyChC,IAAI;QAC3CW,MAAM;QACNC,SAAS;IACX;IACAqB,0CAA0CjC,IAAI;QAC5CW,MAAM;QACNC,SAAS;IACX;IACAsB,mDAAmDlC,IAAI;QACrDW,MAAM;QACNC,SAAS;IACX;IACAuB,8CAA8CnC,IAAI;QAChDW,MAAM;QACNC,SAAS;IACX;IACAwB,0CAA0CpC,IAAI;QAC5CW,MAAM;QACNC,SAAS;IACX;IACAyB,sCAAsCrC,IAAI;QACxCW,MAAM;QACNC,SAAS;IACX;IACA0B,yCAAyCtC,IAAI;QAC3CW,MAAM;QACNC,SAAS;IACX;IACA2B,8BAA8BvC,IAAI;QAChCW,MAAM;QACNC,SAAS;IACX;IACA4B,uCAAuCxC,IAAI;QACzCW,MAAM;QACNC,SAAS;IACX;IACA6B,sCAAsCzC,IAAI;QACxCW,MAAM;QACNC,SAAS;IACX;IACA8B,mCAAmC1C,IAAI;QACrCW,MAAM;QACNC,SAAS;IACX;IACA+B,wCAAwC3C,IAAI;QAC1CW,MAAM;QACNC,SAAS;IACX;IACAgC,gEAAgE9C,KAAK;QACnEa,MAAM;QACNC,SAAS;IACX;AACF,GAAG;AAEH,MAAMiC,6BAA6B1C,eAAeM,OAAOQ,sBAAsB;AAC/E,MAAM6B,8BAA8B5C,kBAAkB2C;AAEtD,IAAIC,4BAA4BC,MAAM,GAAG,GAAG;IAC1C,MAAM,IAAIC,MACR,CAAC,yDAAyD,EAAEF,4BAA4BG,IAAI,CAAC,OAAO;AAExG;AAEA,OAAO,MAAMC,uBAAuBL,2BAA2B;AAE/D,IACE,CAACM,OAAOC,SAAS,CAAC3C,OAAOS,wCAAwC,KACjET,OAAOS,wCAAwC,GAAG,KAClDT,OAAOS,wCAAwC,GAC7Cb,mDACF;IACA,MAAM,IAAI2C,MACR,CAAC,0CAA0C,EAAEvC,OAAOS,wCAAwC,CAAC,kDAAkD,EAAEb,kDAAkD,CAAC,CAAC;AAEzM;AAEA,KAAK,MAAM,CAACgD,MAAMC,MAAM,IAAI;IAC1B;QAAC;QAAsC7C,OAAOC,kCAAkC;KAAC;IACjF;QAAC;QAAsCD,OAAOI,kCAAkC;KAAC;IACjF;QAAC;QAAuCJ,OAAOK,mCAAmC;KAAC;CACpF,CAAW;IACV,IACE,CAACqC,OAAOC,SAAS,CAACE,UAClBA,QAAQ,KACRA,QAAQ/C,iDACR;QACA,MAAM,IAAIyC,MACR,GAAGK,KAAK,EAAE,EAAEC,MAAM,kDAAkD,EAAE/C,gDAAgD,CAAC,CAAC;IAE5H;AACF;AAEA,IACE,CAAC4C,OAAOC,SAAS,CAAC3C,OAAOM,uCAAuC,KAChEN,OAAOM,uCAAuC,GAAG,GACjD;IACA,MAAM,IAAIiC,MACR,CAAC,yCAAyC,EAAEvC,OAAOM,uCAAuC,CAAC,8BAA8B,CAAC;AAE9H;AAEA,IACE,CAACoC,OAAOC,SAAS,CAAC3C,OAAOO,kCAAkC,KAC3DP,OAAOO,kCAAkC,GAAG,GAC5C;IACA,MAAM,IAAIgC,MACR,CAAC,oCAAoC,EAAEvC,OAAOO,kCAAkC,CAAC,8BAA8B,CAAC;AAEpH;AAEA,IACE,CAACmC,OAAOC,SAAS,CAAC3C,OAAOU,4BAA4B,KACrDV,OAAOU,4BAA4B,GAAG,KACtCV,OAAOU,4BAA4B,GAAGb,mCACtC;IACA,MAAM,IAAI0C,MACR,CAAC,8BAA8B,EAAEvC,OAAOU,4BAA4B,CAAC,uCAAuC,EAAEb,kCAAkC,CAAC,CAAC;AAEtJ;AAEA,KAAK,MAAM,CAAC+C,MAAMC,MAAM,IAAI;IAC1B;QAAC;QAA4C7C,OAAOW,wCAAwC;KAAC;IAC7F;QAAC;QAA8CX,OAAOY,0CAA0C;KAAC;IACjG;QAAC;QAA8CZ,OAAOa,0CAA0C;KAAC;IACjG;QACE;QACAb,OAAOc,4CAA4C;KACpD;IACD;QAAC;QAAiCd,OAAOe,6BAA6B;KAAC;CACxE,CAAW;IACV,IAAI,CAAC2B,OAAOC,SAAS,CAACE,UAAUA,QAAQ,GAAG;QACzC,MAAM,IAAIN,MAAM,GAAGK,KAAK,EAAE,EAAEC,MAAM,8BAA8B,CAAC;IACnE;AACF;AAEA,IAAI,CAACH,OAAOC,SAAS,CAAC3C,OAAOgB,uBAAuB,KAAKhB,OAAOgB,uBAAuB,GAAG,GAAG;IAC3F,MAAM,IAAIuB,MACR,CAAC,yBAAyB,EAAEvC,OAAOgB,uBAAuB,CAAC,yCAAyC,CAAC;AAEzG;AAEA,IACE,CAAC0B,OAAOC,SAAS,CAAC3C,OAAOiB,2BAA2B,KACpDjB,OAAOiB,2BAA2B,GAAG,KACrCjB,OAAOiB,2BAA2B,GAAGlB,kCACrC;IACA,MAAM,IAAIwC,MACR,CAAC,6BAA6B,EAAEvC,OAAOiB,2BAA2B,CAAC,kDAAkD,EAAElB,iCAAiC,CAAC,CAAC;AAE9J;AAEA,IAAIC,OAAOgB,uBAAuB,GAAGhB,OAAOiB,2BAA2B,EAAE;IACvE,MAAM,IAAIsB,MACR,CAAC,yBAAyB,EAAEvC,OAAOgB,uBAAuB,CAAC,wDAAwD,EAAEhB,OAAOiB,2BAA2B,CAAC,EAAE,CAAC;AAE/J;AAEA,IACE,CAACyB,OAAOC,SAAS,CAAC3C,OAAOkB,sCAAsC,KAC/DlB,OAAOkB,sCAAsC,GAAG,GAChD;IACA,MAAM,IAAIqB,MACR,CAAC,wCAAwC,EAAEvC,OAAOkB,sCAAsC,CAAC,yCAAyC,CAAC;AAEvI;AAEA,IACE,CAACwB,OAAOC,SAAS,CAAC3C,OAAOmB,4BAA4B,KACrDnB,OAAOmB,4BAA4B,GAAG,GACtC;IACA,MAAM,IAAIoB,MACR,CAAC,8BAA8B,EAAEvC,OAAOmB,4BAA4B,CAAC,8CAA8C,CAAC;AAExH;AAEA,IACE,CAACuB,OAAOC,SAAS,CAAC3C,OAAOoB,gCAAgC,KACzDpB,OAAOoB,gCAAgC,GAAGpB,OAAOmB,4BAA4B,EAC7E;IACA,MAAM,IAAIoB,MACR,CAAC,kCAAkC,EAAEvC,OAAOoB,gCAAgC,CAAC,0EAA0E,EAAEpB,OAAOmB,4BAA4B,CAAC,EAAE,CAAC;AAEpM;AAEA,IACE,CAACuB,OAAOC,SAAS,CAAC3C,OAAOqB,4BAA4B,KACrDrB,OAAOqB,4BAA4B,GAAG,GACtC;IACA,MAAM,IAAIkB,MACR,CAAC,8BAA8B,EAAEvC,OAAOqB,4BAA4B,CAAC,yCAAyC,CAAC;AAEnH;AAEA,IACE,CAACqB,OAAOC,SAAS,CAAC3C,OAAOsB,4CAA4C,KACrEtB,OAAOsB,4CAA4C,GAAG,GACtD;IACA,MAAM,IAAIiB,MACR,CAAC,8CAA8C,EAAEvC,OAAOsB,4CAA4C,CAAC,yCAAyC,CAAC;AAEnJ;AAEA,IACE,CAACoB,OAAOC,SAAS,CAAC3C,OAAOuB,uCAAuC,KAChEvB,OAAOuB,uCAAuC,GAAG,KACjDvB,OAAOuB,uCAAuC,IAAI5B,6BAClD;IACA,MAAM,IAAI4C,MACR,CAAC,yCAAyC,EAAEvC,OAAOuB,uCAAuC,CAAC,+CAA+C,EAAE5B,4BAA4B,CAAC,CAAC;AAE9K;AAEA,IACE,CAAC+C,OAAOC,SAAS,CAAC3C,OAAOwB,wCAAwC,KACjExB,OAAOwB,wCAAwC,GAAG,GAClD;IACA,MAAM,IAAIe,MACR,CAAC,0CAA0C,EAAEvC,OAAOwB,wCAAwC,CAAC,yCAAyC,CAAC;AAE3I;AAEA,IACE,CAACkB,OAAOC,SAAS,CAAC3C,OAAOyB,iDAAiD,KAC1EzB,OAAOyB,iDAAiD,GAAG,GAC3D;IACA,MAAM,IAAIc,MACR,CAAC,mDAAmD,EAAEvC,OAAOyB,iDAAiD,CAAC,yCAAyC,CAAC;AAE7J;AAEA,IACE,CAACiB,OAAOC,SAAS,CAAC3C,OAAO0B,4CAA4C,KACrE1B,OAAO0B,4CAA4C,GAAG,GACtD;IACA,MAAM,IAAIa,MACR,CAAC,8CAA8C,EAAEvC,OAAO0B,4CAA4C,CAAC,8BAA8B,CAAC;AAExI;AAEA,IACE,CAACgB,OAAOC,SAAS,CAAC3C,OAAO2B,wCAAwC,KACjE3B,OAAO2B,wCAAwC,GAAG,GAClD;IACA,MAAM,IAAIY,MACR,CAAC,0CAA0C,EAAEvC,OAAO2B,wCAAwC,CAAC,yCAAyC,CAAC;AAE3I;AAEA,IACE,CAACe,OAAOC,SAAS,CAAC3C,OAAO4B,oCAAoC,KAC7D5B,OAAO4B,oCAAoC,GAAG,GAC9C;IACA,MAAM,IAAIW,MACR,CAAC,sCAAsC,EAAEvC,OAAO4B,oCAAoC,CAAC,sCAAsC,CAAC;AAEhI;AAEA,IACE,CAACc,OAAOC,SAAS,CAAC3C,OAAO6B,uCAAuC,KAChE7B,OAAO6B,uCAAuC,GAAG,GACjD;IACA,MAAM,IAAIU,MACR,CAAC,yCAAyC,EAAEvC,OAAO6B,uCAAuC,CAAC,sCAAsC,CAAC;AAEtI;AAEA,IACE,CAACa,OAAOC,SAAS,CAAC3C,OAAO8B,4BAA4B,KACrD9B,OAAO8B,4BAA4B,GAAG,GACtC;IACA,MAAM,IAAIS,MACR,CAAC,8BAA8B,EAAEvC,OAAO8B,4BAA4B,CAAC,8BAA8B,CAAC;AAExG;AAEA,IACE,CAACY,OAAOC,SAAS,CAAC3C,OAAO+B,qCAAqC,KAC9D/B,OAAO+B,qCAAqC,GAAG,GAC/C;IACA,MAAM,IAAIQ,MACR,CAAC,uCAAuC,EAAEvC,OAAO+B,qCAAqC,CAAC,sCAAsC,CAAC;AAElI;AAEA,IACE,CAACW,OAAOC,SAAS,CAAC3C,OAAOgC,oCAAoC,KAC7DhC,OAAOgC,oCAAoC,GAAG,GAC9C;IACA,MAAM,IAAIO,MACR,CAAC,sCAAsC,EAAEvC,OAAOgC,oCAAoC,CAAC,8BAA8B,CAAC;AAExH;AAEA,IACE,CAACU,OAAOC,SAAS,CAAC3C,OAAOiC,iCAAiC,KAC1DjC,OAAOiC,iCAAiC,GAAG,GAC3C;IACA,MAAM,IAAIM,MACR,CAAC,mCAAmC,EAAEvC,OAAOiC,iCAAiC,CAAC,yCAAyC,CAAC;AAE7H;AAEA,IACE,CAACS,OAAOC,SAAS,CAAC3C,OAAOkC,sCAAsC,KAC/DlC,OAAOkC,sCAAsC,GAAG,GAChD;IACA,MAAM,IAAIK,MACR,CAAC,wCAAwC,EAAEvC,OAAOkC,sCAAsC,CAAC,yCAAyC,CAAC;AAEvI;AAEA,IACElC,OAAOyB,iDAAiD,IACxDzB,OAAOkC,sCAAsC,EAC7C;IACA,MAAM,IAAIK,MACR,CAAC,mDAAmD,EAAEvC,OAAOyB,iDAAiD,CAAC,+DAA+D,EAAEzB,OAAOkC,sCAAsC,CAAC,EAAE,CAAC;AAErO;AAEA,IACElC,OAAOyB,iDAAiD,IACxDzB,OAAO2B,wCAAwC,EAC/C;IACA,MAAM,IAAIY,MACR,CAAC,mDAAmD,EAAEvC,OAAOyB,iDAAiD,CAAC,iEAAiE,EAAEzB,OAAO2B,wCAAwC,CAAC,EAAE,CAAC;AAEzO"}
|
|
1
|
+
{"version":3,"sources":["../src/config.ts"],"sourcesContent":["import {RUNNER_ASSIGNMENT_POLL_DEFAULT_WAIT_SECONDS} from '@shipfox/api-runners-dto';\nimport {bool, createConfig, num, str} from '@shipfox/config';\nimport {findInvalidLabels, parseLabelList} from '@shipfox/runner-labels';\nimport {STUCK_JOB_THRESHOLD_SECONDS} from '#core/maintenance-policy.js';\n\nconst EPHEMERAL_REGISTRATION_TOKEN_TTL_HARD_MAX_SECONDS = 3600;\nconst REGISTRATION_TOKEN_BATCH_HARD_MAX = 1000;\nconst RUNNER_CONTROL_PLANE_TOKEN_TTL_HARD_MAX_SECONDS = 3600;\nconst RESERVATION_TTL_HARD_MAX_SECONDS = 3600;\n\nexport const config = createConfig({\n RUNNER_BOOTSTRAP_TOKEN_TTL_SECONDS: num({\n desc: 'Lifetime of a one-use runner bootstrap token in seconds. The token can only create one runner-control session before it expires.',\n default: 300,\n }),\n RUNNER_CONTROL_SESSION_TTL_SECONDS: num({\n desc: 'Lifetime of a pre-workspace runner-control session in seconds. This session can only enroll, attach its provider identity, and report its own liveness.',\n default: 3600,\n }),\n RUNNER_ACTIVATION_TOKEN_TTL_SECONDS: num({\n desc: 'Lifetime of a one-use activation token in seconds. The token is issued only to an assigned runner-control session and creates one workspace runner session.',\n default: 300,\n }),\n RUNNER_ASSIGNMENT_POLL_MAX_WAIT_SECONDS: num({\n desc: 'Maximum server-side cap for the per-request runner assignment wait, in seconds. Managed runners request their own bounded wait with wait_seconds.',\n default: RUNNER_ASSIGNMENT_POLL_DEFAULT_WAIT_SECONDS,\n }),\n RUNNER_ASSIGNMENT_POLL_INTERVAL_MS: num({\n desc: 'Delay between durable assignment reads while a runner-control session waits for an assignment, in milliseconds.',\n default: 250,\n }),\n RUNNER_RESERVED_LABELS: str({\n desc: 'Comma-separated labels that only installation-scope provisioners may advertise. Any other provisioner or runner has them removed.',\n default: '',\n }),\n EPHEMERAL_REGISTRATION_TOKEN_TTL_SECONDS: num({\n desc: `Lifetime of a provisioner-minted runner registration token, in seconds. The token can be exchanged once by a runner before this time passes. Set this between 1 and ${EPHEMERAL_REGISTRATION_TOKEN_TTL_HARD_MAX_SECONDS}.`,\n default: 300,\n }),\n REGISTRATION_TOKEN_BATCH_MAX: num({\n desc: `Maximum number of runner registration tokens a provisioner can mint in one batch request. Set this between 1 and ${REGISTRATION_TOKEN_BATCH_HARD_MAX}.`,\n default: 500,\n }),\n PROVISIONER_MINT_RATE_LIMIT_MAX_REQUESTS: num({\n desc: 'Maximum number of batch runner registration token mint requests one provisioner can make per rate-limit window.',\n default: 120,\n }),\n PROVISIONER_MINT_RATE_LIMIT_WINDOW_SECONDS: num({\n desc: 'Rate-limit window for provisioner batch runner registration token mint requests, in seconds.',\n default: 60,\n }),\n EPHEMERAL_REGISTER_RATE_LIMIT_MAX_REQUESTS: num({\n desc: 'Maximum number of runner registration attempts one ephemeral registration token can make per rate-limit window.',\n default: 5,\n }),\n EPHEMERAL_REGISTER_RATE_LIMIT_WINDOW_SECONDS: num({\n desc: 'Rate-limit window for runner registration attempts with one ephemeral registration token, in seconds.',\n default: 60,\n }),\n RUNNERS_RATE_LIMIT_TIMEOUT_MS: num({\n desc: 'Maximum time, in milliseconds, a runners rate-limit storage check may wait before the request fails closed.',\n default: 250,\n }),\n RESERVATION_TTL_SECONDS: num({\n desc: 'Activation grace period for a rebound runner reservation, in seconds. A runner without a session can be rebound again only after this deadline. It is also the default lifetime for launch reservations when a provisioner does not request a provider-specific value.',\n default: 60,\n }),\n RESERVATION_TTL_MAX_SECONDS: num({\n desc: `Server-side ceiling for a launch reservation lifetime, in seconds. Set it at least as high as every provider registration deadline plus its launch headroom so provider-specific values can cover the launch gap, boot, and enrollment. Set this between 1 and ${RESERVATION_TTL_HARD_MAX_SECONDS}.`,\n default: 600,\n }),\n RESERVATION_LONG_POLL_MAX_WAIT_SECONDS: num({\n desc: 'Maximum time the provisioner demand poll endpoint waits for reservable demand before returning, in seconds.',\n default: 30,\n }),\n RESERVATION_POLL_INTERVAL_MS: num({\n desc: 'Initial interval between demand re-checks while a provisioner request is waiting, in milliseconds.',\n default: 1000,\n }),\n RESERVATION_POLL_MAX_INTERVAL_MS: num({\n desc: 'Maximum backoff interval between demand re-checks while a provisioner request is waiting, in milliseconds.',\n default: 5000,\n }),\n RUNNER_ACTIVE_WINDOW_SECONDS: num({\n desc: 'Time window, in seconds, used to list active runners from recent heartbeats and provisioned runner reports.',\n default: 60,\n }),\n RUNNER_DEMAND_ACTIVATION_TIMEOUT_SECONDS: num({\n desc: 'Minimum time, in seconds, before a demand-backed runner without a runner session can be reclaimed when neither its current nor intended reservation is live. Set this above the provider registration deadline when startup reservations remain live during boot.',\n default: 300,\n }),\n RUNNER_TOOL_CAPABILITIES_STALE_AFTER_SECONDS: num({\n desc: 'Time window, in seconds, after which a runner tool capability report is treated as stale. Set this higher than the runner heartbeat interval so active runners keep their advertised tools fresh.',\n default: 60,\n }),\n RUNNER_NO_FIRST_HEARTBEAT_GRACE_SECONDS: num({\n desc: 'Grace window, in seconds, before maintenance expires a claimed job that has not sent its first heartbeat. Set this lower than the normal stuck-job threshold so startup crashes release work quickly.',\n default: 60,\n }),\n RUNNER_RECONCILE_TERMINATE_GRACE_SECONDS: num({\n desc: 'Grace window, in seconds, before reconcile marks an absent provisioned runner as terminated. Set this higher than the provisioner report interval so a transient empty or partial observed set does not kill a live runner.',\n default: 120,\n }),\n RUNNER_STALE_PROVISIONED_RUNNER_THRESHOLD_SECONDS: num({\n desc: 'Time, in seconds, after which a provisioned runner with no recent report, no live provisioner, no live runner session, and no running job is marked failed by backend maintenance.',\n default: 300,\n }),\n RUNNER_STALE_PROVISIONED_RUNNER_REAPER_LIMIT: num({\n desc: 'Maximum number of stale provisioned runners the maintenance worker marks failed in one run. Higher values clear backlogs faster but hold database locks longer.',\n default: 100,\n }),\n RUNNER_SESSION_LIVENESS_THROTTLE_SECONDS: num({\n desc: 'Minimum time, in seconds, between runner session liveness writes from job request polls. Set this lower than RUNNER_STALE_PROVISIONED_RUNNER_THRESHOLD_SECONDS so active idle runners stay fresh.',\n default: 10,\n }),\n RUNNER_SESSION_MANUAL_RETENTION_DAYS: num({\n desc: 'How long manual runner sessions are retained before maintenance deletes them, in days. Set this longer than AUTH_RUNNER_SESSION_TOKEN_EXPIRES_IN so a valid session token never outlives its row.',\n default: 30,\n }),\n RUNNER_SESSION_EPHEMERAL_RETENTION_DAYS: num({\n desc: 'How long ephemeral runner sessions are retained before maintenance deletes them, in days. Set this longer than AUTH_RUNNER_SESSION_TOKEN_EXPIRES_IN so a valid session token never outlives its row.',\n default: 7,\n }),\n RUNNER_SESSION_GC_BATCH_SIZE: num({\n desc: 'Maximum number of expired runner sessions maintenance deletes in one pass.',\n default: 1000,\n }),\n RUNNER_EPHEMERAL_TOKEN_RETENTION_DAYS: num({\n desc: 'How long consumed or expired ephemeral registration tokens are retained for audit and debugging before maintenance deletes them, in days. Active tokens that are neither consumed nor expired are never deleted. This window is independent of the runner-session retention windows.',\n default: 7,\n }),\n RUNNER_EPHEMERAL_TOKEN_GC_BATCH_SIZE: num({\n desc: 'Maximum number of consumed or expired ephemeral registration tokens maintenance deletes in one pass.',\n default: 1000,\n }),\n PROVISIONER_ACTIVE_WINDOW_SECONDS: num({\n desc: 'Time window, in seconds, used to list active provisioners from recent authenticated requests.',\n default: 120,\n }),\n PROVISIONER_LAST_SEEN_THROTTLE_SECONDS: num({\n desc: 'Minimum time, in seconds, between last-seen writes for one provisioner token.',\n default: 10,\n }),\n PROVISIONED_RUNNER_COUNT_DIVERGENCE_TEMPLATE_KEY_LABEL_ENABLED: bool({\n desc: 'Whether runners_provider_runner_count_divergence includes template_key as a metric label. Use true or false. Defaults to false because template keys can create high-cardinality metric series. Set true only when template keys are bounded and stable.',\n default: false,\n }),\n});\n\nconst parsedRunnerReservedLabels = parseLabelList(config.RUNNER_RESERVED_LABELS);\nconst invalidRunnerReservedLabels = findInvalidLabels(parsedRunnerReservedLabels);\n\nif (invalidRunnerReservedLabels.length > 0) {\n throw new Error(\n `RUNNER_RESERVED_LABELS contains invalid runner label(s): ${invalidRunnerReservedLabels.join(', ')}`,\n );\n}\n\nexport const runnerReservedLabels = parsedRunnerReservedLabels;\n\nif (\n !Number.isInteger(config.EPHEMERAL_REGISTRATION_TOKEN_TTL_SECONDS) ||\n config.EPHEMERAL_REGISTRATION_TOKEN_TTL_SECONDS < 1 ||\n config.EPHEMERAL_REGISTRATION_TOKEN_TTL_SECONDS >\n EPHEMERAL_REGISTRATION_TOKEN_TTL_HARD_MAX_SECONDS\n) {\n throw new Error(\n `EPHEMERAL_REGISTRATION_TOKEN_TTL_SECONDS (${config.EPHEMERAL_REGISTRATION_TOKEN_TTL_SECONDS}) must be a whole number of seconds between 1 and ${EPHEMERAL_REGISTRATION_TOKEN_TTL_HARD_MAX_SECONDS}.`,\n );\n}\n\nfor (const [name, value] of [\n ['RUNNER_BOOTSTRAP_TOKEN_TTL_SECONDS', config.RUNNER_BOOTSTRAP_TOKEN_TTL_SECONDS],\n ['RUNNER_CONTROL_SESSION_TTL_SECONDS', config.RUNNER_CONTROL_SESSION_TTL_SECONDS],\n ['RUNNER_ACTIVATION_TOKEN_TTL_SECONDS', config.RUNNER_ACTIVATION_TOKEN_TTL_SECONDS],\n] as const) {\n if (\n !Number.isInteger(value) ||\n value < 1 ||\n value > RUNNER_CONTROL_PLANE_TOKEN_TTL_HARD_MAX_SECONDS\n ) {\n throw new Error(\n `${name} (${value}) must be a whole number of seconds between 1 and ${RUNNER_CONTROL_PLANE_TOKEN_TTL_HARD_MAX_SECONDS}.`,\n );\n }\n}\n\nif (\n !Number.isInteger(config.RUNNER_ASSIGNMENT_POLL_MAX_WAIT_SECONDS) ||\n config.RUNNER_ASSIGNMENT_POLL_MAX_WAIT_SECONDS < 1\n) {\n throw new Error(\n `RUNNER_ASSIGNMENT_POLL_MAX_WAIT_SECONDS (${config.RUNNER_ASSIGNMENT_POLL_MAX_WAIT_SECONDS}) must be a whole number >= 1.`,\n );\n}\n\nif (\n !Number.isInteger(config.RUNNER_ASSIGNMENT_POLL_INTERVAL_MS) ||\n config.RUNNER_ASSIGNMENT_POLL_INTERVAL_MS < 0\n) {\n throw new Error(\n `RUNNER_ASSIGNMENT_POLL_INTERVAL_MS (${config.RUNNER_ASSIGNMENT_POLL_INTERVAL_MS}) must be a whole number >= 0.`,\n );\n}\n\nif (\n !Number.isInteger(config.REGISTRATION_TOKEN_BATCH_MAX) ||\n config.REGISTRATION_TOKEN_BATCH_MAX < 1 ||\n config.REGISTRATION_TOKEN_BATCH_MAX > REGISTRATION_TOKEN_BATCH_HARD_MAX\n) {\n throw new Error(\n `REGISTRATION_TOKEN_BATCH_MAX (${config.REGISTRATION_TOKEN_BATCH_MAX}) must be a whole number between 1 and ${REGISTRATION_TOKEN_BATCH_HARD_MAX}.`,\n );\n}\n\nfor (const [name, value] of [\n ['PROVISIONER_MINT_RATE_LIMIT_MAX_REQUESTS', config.PROVISIONER_MINT_RATE_LIMIT_MAX_REQUESTS],\n ['PROVISIONER_MINT_RATE_LIMIT_WINDOW_SECONDS', config.PROVISIONER_MINT_RATE_LIMIT_WINDOW_SECONDS],\n ['EPHEMERAL_REGISTER_RATE_LIMIT_MAX_REQUESTS', config.EPHEMERAL_REGISTER_RATE_LIMIT_MAX_REQUESTS],\n [\n 'EPHEMERAL_REGISTER_RATE_LIMIT_WINDOW_SECONDS',\n config.EPHEMERAL_REGISTER_RATE_LIMIT_WINDOW_SECONDS,\n ],\n ['RUNNERS_RATE_LIMIT_TIMEOUT_MS', config.RUNNERS_RATE_LIMIT_TIMEOUT_MS],\n] as const) {\n if (!Number.isInteger(value) || value < 1) {\n throw new Error(`${name} (${value}) must be a whole number >= 1.`);\n }\n}\n\nif (!Number.isInteger(config.RESERVATION_TTL_SECONDS) || config.RESERVATION_TTL_SECONDS < 1) {\n throw new Error(\n `RESERVATION_TTL_SECONDS (${config.RESERVATION_TTL_SECONDS}) must be a whole number of seconds >= 1.`,\n );\n}\n\nif (\n !Number.isInteger(config.RESERVATION_TTL_MAX_SECONDS) ||\n config.RESERVATION_TTL_MAX_SECONDS < 1 ||\n config.RESERVATION_TTL_MAX_SECONDS > RESERVATION_TTL_HARD_MAX_SECONDS\n) {\n throw new Error(\n `RESERVATION_TTL_MAX_SECONDS (${config.RESERVATION_TTL_MAX_SECONDS}) must be a whole number of seconds between 1 and ${RESERVATION_TTL_HARD_MAX_SECONDS}.`,\n );\n}\n\nif (config.RESERVATION_TTL_SECONDS > config.RESERVATION_TTL_MAX_SECONDS) {\n throw new Error(\n `RESERVATION_TTL_SECONDS (${config.RESERVATION_TTL_SECONDS}) must not be greater than RESERVATION_TTL_MAX_SECONDS (${config.RESERVATION_TTL_MAX_SECONDS}).`,\n );\n}\n\nif (\n !Number.isInteger(config.RESERVATION_LONG_POLL_MAX_WAIT_SECONDS) ||\n config.RESERVATION_LONG_POLL_MAX_WAIT_SECONDS < 0\n) {\n throw new Error(\n `RESERVATION_LONG_POLL_MAX_WAIT_SECONDS (${config.RESERVATION_LONG_POLL_MAX_WAIT_SECONDS}) must be a whole number of seconds >= 0.`,\n );\n}\n\nif (\n !Number.isInteger(config.RESERVATION_POLL_INTERVAL_MS) ||\n config.RESERVATION_POLL_INTERVAL_MS < 1\n) {\n throw new Error(\n `RESERVATION_POLL_INTERVAL_MS (${config.RESERVATION_POLL_INTERVAL_MS}) must be a whole number of milliseconds >= 1.`,\n );\n}\n\nif (\n !Number.isInteger(config.RESERVATION_POLL_MAX_INTERVAL_MS) ||\n config.RESERVATION_POLL_MAX_INTERVAL_MS < config.RESERVATION_POLL_INTERVAL_MS\n) {\n throw new Error(\n `RESERVATION_POLL_MAX_INTERVAL_MS (${config.RESERVATION_POLL_MAX_INTERVAL_MS}) must be a whole number of milliseconds >= RESERVATION_POLL_INTERVAL_MS (${config.RESERVATION_POLL_INTERVAL_MS}).`,\n );\n}\n\nif (\n !Number.isInteger(config.RUNNER_ACTIVE_WINDOW_SECONDS) ||\n config.RUNNER_ACTIVE_WINDOW_SECONDS < 1\n) {\n throw new Error(\n `RUNNER_ACTIVE_WINDOW_SECONDS (${config.RUNNER_ACTIVE_WINDOW_SECONDS}) must be a whole number of seconds >= 1.`,\n );\n}\n\nif (\n !Number.isInteger(config.RUNNER_DEMAND_ACTIVATION_TIMEOUT_SECONDS) ||\n config.RUNNER_DEMAND_ACTIVATION_TIMEOUT_SECONDS < 1\n) {\n throw new Error(\n `RUNNER_DEMAND_ACTIVATION_TIMEOUT_SECONDS (${config.RUNNER_DEMAND_ACTIVATION_TIMEOUT_SECONDS}) must be a whole number of seconds >= 1.`,\n );\n}\n\nif (\n !Number.isInteger(config.RUNNER_TOOL_CAPABILITIES_STALE_AFTER_SECONDS) ||\n config.RUNNER_TOOL_CAPABILITIES_STALE_AFTER_SECONDS < 1\n) {\n throw new Error(\n `RUNNER_TOOL_CAPABILITIES_STALE_AFTER_SECONDS (${config.RUNNER_TOOL_CAPABILITIES_STALE_AFTER_SECONDS}) must be a whole number of seconds >= 1.`,\n );\n}\n\nif (\n !Number.isInteger(config.RUNNER_NO_FIRST_HEARTBEAT_GRACE_SECONDS) ||\n config.RUNNER_NO_FIRST_HEARTBEAT_GRACE_SECONDS < 1 ||\n config.RUNNER_NO_FIRST_HEARTBEAT_GRACE_SECONDS >= STUCK_JOB_THRESHOLD_SECONDS\n) {\n throw new Error(\n `RUNNER_NO_FIRST_HEARTBEAT_GRACE_SECONDS (${config.RUNNER_NO_FIRST_HEARTBEAT_GRACE_SECONDS}) must be a whole number of seconds >= 1 and < ${STUCK_JOB_THRESHOLD_SECONDS}.`,\n );\n}\n\nif (\n !Number.isInteger(config.RUNNER_RECONCILE_TERMINATE_GRACE_SECONDS) ||\n config.RUNNER_RECONCILE_TERMINATE_GRACE_SECONDS < 1\n) {\n throw new Error(\n `RUNNER_RECONCILE_TERMINATE_GRACE_SECONDS (${config.RUNNER_RECONCILE_TERMINATE_GRACE_SECONDS}) must be a whole number of seconds >= 1.`,\n );\n}\n\nif (\n !Number.isInteger(config.RUNNER_STALE_PROVISIONED_RUNNER_THRESHOLD_SECONDS) ||\n config.RUNNER_STALE_PROVISIONED_RUNNER_THRESHOLD_SECONDS < 1\n) {\n throw new Error(\n `RUNNER_STALE_PROVISIONED_RUNNER_THRESHOLD_SECONDS (${config.RUNNER_STALE_PROVISIONED_RUNNER_THRESHOLD_SECONDS}) must be a whole number of seconds >= 1.`,\n );\n}\n\nif (\n !Number.isInteger(config.RUNNER_STALE_PROVISIONED_RUNNER_REAPER_LIMIT) ||\n config.RUNNER_STALE_PROVISIONED_RUNNER_REAPER_LIMIT < 1\n) {\n throw new Error(\n `RUNNER_STALE_PROVISIONED_RUNNER_REAPER_LIMIT (${config.RUNNER_STALE_PROVISIONED_RUNNER_REAPER_LIMIT}) must be a whole number >= 1.`,\n );\n}\n\nif (\n !Number.isInteger(config.RUNNER_SESSION_LIVENESS_THROTTLE_SECONDS) ||\n config.RUNNER_SESSION_LIVENESS_THROTTLE_SECONDS < 1\n) {\n throw new Error(\n `RUNNER_SESSION_LIVENESS_THROTTLE_SECONDS (${config.RUNNER_SESSION_LIVENESS_THROTTLE_SECONDS}) must be a whole number of seconds >= 1.`,\n );\n}\n\nif (\n !Number.isInteger(config.RUNNER_SESSION_MANUAL_RETENTION_DAYS) ||\n config.RUNNER_SESSION_MANUAL_RETENTION_DAYS < 1\n) {\n throw new Error(\n `RUNNER_SESSION_MANUAL_RETENTION_DAYS (${config.RUNNER_SESSION_MANUAL_RETENTION_DAYS}) must be a whole number of days >= 1.`,\n );\n}\n\nif (\n !Number.isInteger(config.RUNNER_SESSION_EPHEMERAL_RETENTION_DAYS) ||\n config.RUNNER_SESSION_EPHEMERAL_RETENTION_DAYS < 1\n) {\n throw new Error(\n `RUNNER_SESSION_EPHEMERAL_RETENTION_DAYS (${config.RUNNER_SESSION_EPHEMERAL_RETENTION_DAYS}) must be a whole number of days >= 1.`,\n );\n}\n\nif (\n !Number.isInteger(config.RUNNER_SESSION_GC_BATCH_SIZE) ||\n config.RUNNER_SESSION_GC_BATCH_SIZE < 1\n) {\n throw new Error(\n `RUNNER_SESSION_GC_BATCH_SIZE (${config.RUNNER_SESSION_GC_BATCH_SIZE}) must be a whole number >= 1.`,\n );\n}\n\nif (\n !Number.isInteger(config.RUNNER_EPHEMERAL_TOKEN_RETENTION_DAYS) ||\n config.RUNNER_EPHEMERAL_TOKEN_RETENTION_DAYS < 1\n) {\n throw new Error(\n `RUNNER_EPHEMERAL_TOKEN_RETENTION_DAYS (${config.RUNNER_EPHEMERAL_TOKEN_RETENTION_DAYS}) must be a whole number of days >= 1.`,\n );\n}\n\nif (\n !Number.isInteger(config.RUNNER_EPHEMERAL_TOKEN_GC_BATCH_SIZE) ||\n config.RUNNER_EPHEMERAL_TOKEN_GC_BATCH_SIZE < 1\n) {\n throw new Error(\n `RUNNER_EPHEMERAL_TOKEN_GC_BATCH_SIZE (${config.RUNNER_EPHEMERAL_TOKEN_GC_BATCH_SIZE}) must be a whole number >= 1.`,\n );\n}\n\nif (\n !Number.isInteger(config.PROVISIONER_ACTIVE_WINDOW_SECONDS) ||\n config.PROVISIONER_ACTIVE_WINDOW_SECONDS < 1\n) {\n throw new Error(\n `PROVISIONER_ACTIVE_WINDOW_SECONDS (${config.PROVISIONER_ACTIVE_WINDOW_SECONDS}) must be a whole number of seconds >= 1.`,\n );\n}\n\nif (\n !Number.isInteger(config.PROVISIONER_LAST_SEEN_THROTTLE_SECONDS) ||\n config.PROVISIONER_LAST_SEEN_THROTTLE_SECONDS < 1\n) {\n throw new Error(\n `PROVISIONER_LAST_SEEN_THROTTLE_SECONDS (${config.PROVISIONER_LAST_SEEN_THROTTLE_SECONDS}) must be a whole number of seconds >= 1.`,\n );\n}\n\nif (\n config.RUNNER_STALE_PROVISIONED_RUNNER_THRESHOLD_SECONDS <=\n config.PROVISIONER_LAST_SEEN_THROTTLE_SECONDS\n) {\n throw new Error(\n `RUNNER_STALE_PROVISIONED_RUNNER_THRESHOLD_SECONDS (${config.RUNNER_STALE_PROVISIONED_RUNNER_THRESHOLD_SECONDS}) must be greater than PROVISIONER_LAST_SEEN_THROTTLE_SECONDS (${config.PROVISIONER_LAST_SEEN_THROTTLE_SECONDS}).`,\n );\n}\n\nif (\n config.RUNNER_STALE_PROVISIONED_RUNNER_THRESHOLD_SECONDS <=\n config.RUNNER_SESSION_LIVENESS_THROTTLE_SECONDS\n) {\n throw new Error(\n `RUNNER_STALE_PROVISIONED_RUNNER_THRESHOLD_SECONDS (${config.RUNNER_STALE_PROVISIONED_RUNNER_THRESHOLD_SECONDS}) must be greater than RUNNER_SESSION_LIVENESS_THROTTLE_SECONDS (${config.RUNNER_SESSION_LIVENESS_THROTTLE_SECONDS}).`,\n );\n}\n"],"names":["RUNNER_ASSIGNMENT_POLL_DEFAULT_WAIT_SECONDS","bool","createConfig","num","str","findInvalidLabels","parseLabelList","STUCK_JOB_THRESHOLD_SECONDS","EPHEMERAL_REGISTRATION_TOKEN_TTL_HARD_MAX_SECONDS","REGISTRATION_TOKEN_BATCH_HARD_MAX","RUNNER_CONTROL_PLANE_TOKEN_TTL_HARD_MAX_SECONDS","RESERVATION_TTL_HARD_MAX_SECONDS","config","RUNNER_BOOTSTRAP_TOKEN_TTL_SECONDS","desc","default","RUNNER_CONTROL_SESSION_TTL_SECONDS","RUNNER_ACTIVATION_TOKEN_TTL_SECONDS","RUNNER_ASSIGNMENT_POLL_MAX_WAIT_SECONDS","RUNNER_ASSIGNMENT_POLL_INTERVAL_MS","RUNNER_RESERVED_LABELS","EPHEMERAL_REGISTRATION_TOKEN_TTL_SECONDS","REGISTRATION_TOKEN_BATCH_MAX","PROVISIONER_MINT_RATE_LIMIT_MAX_REQUESTS","PROVISIONER_MINT_RATE_LIMIT_WINDOW_SECONDS","EPHEMERAL_REGISTER_RATE_LIMIT_MAX_REQUESTS","EPHEMERAL_REGISTER_RATE_LIMIT_WINDOW_SECONDS","RUNNERS_RATE_LIMIT_TIMEOUT_MS","RESERVATION_TTL_SECONDS","RESERVATION_TTL_MAX_SECONDS","RESERVATION_LONG_POLL_MAX_WAIT_SECONDS","RESERVATION_POLL_INTERVAL_MS","RESERVATION_POLL_MAX_INTERVAL_MS","RUNNER_ACTIVE_WINDOW_SECONDS","RUNNER_DEMAND_ACTIVATION_TIMEOUT_SECONDS","RUNNER_TOOL_CAPABILITIES_STALE_AFTER_SECONDS","RUNNER_NO_FIRST_HEARTBEAT_GRACE_SECONDS","RUNNER_RECONCILE_TERMINATE_GRACE_SECONDS","RUNNER_STALE_PROVISIONED_RUNNER_THRESHOLD_SECONDS","RUNNER_STALE_PROVISIONED_RUNNER_REAPER_LIMIT","RUNNER_SESSION_LIVENESS_THROTTLE_SECONDS","RUNNER_SESSION_MANUAL_RETENTION_DAYS","RUNNER_SESSION_EPHEMERAL_RETENTION_DAYS","RUNNER_SESSION_GC_BATCH_SIZE","RUNNER_EPHEMERAL_TOKEN_RETENTION_DAYS","RUNNER_EPHEMERAL_TOKEN_GC_BATCH_SIZE","PROVISIONER_ACTIVE_WINDOW_SECONDS","PROVISIONER_LAST_SEEN_THROTTLE_SECONDS","PROVISIONED_RUNNER_COUNT_DIVERGENCE_TEMPLATE_KEY_LABEL_ENABLED","parsedRunnerReservedLabels","invalidRunnerReservedLabels","length","Error","join","runnerReservedLabels","Number","isInteger","name","value"],"mappings":"AAAA,SAAQA,2CAA2C,QAAO,2BAA2B;AACrF,SAAQC,IAAI,EAAEC,YAAY,EAAEC,GAAG,EAAEC,GAAG,QAAO,kBAAkB;AAC7D,SAAQC,iBAAiB,EAAEC,cAAc,QAAO,yBAAyB;AACzE,SAAQC,2BAA2B,QAAO,8BAA8B;AAExE,MAAMC,oDAAoD;AAC1D,MAAMC,oCAAoC;AAC1C,MAAMC,kDAAkD;AACxD,MAAMC,mCAAmC;AAEzC,OAAO,MAAMC,SAASV,aAAa;IACjCW,oCAAoCV,IAAI;QACtCW,MAAM;QACNC,SAAS;IACX;IACAC,oCAAoCb,IAAI;QACtCW,MAAM;QACNC,SAAS;IACX;IACAE,qCAAqCd,IAAI;QACvCW,MAAM;QACNC,SAAS;IACX;IACAG,yCAAyCf,IAAI;QAC3CW,MAAM;QACNC,SAASf;IACX;IACAmB,oCAAoChB,IAAI;QACtCW,MAAM;QACNC,SAAS;IACX;IACAK,wBAAwBhB,IAAI;QAC1BU,MAAM;QACNC,SAAS;IACX;IACAM,0CAA0ClB,IAAI;QAC5CW,MAAM,CAAC,oKAAoK,EAAEN,kDAAkD,CAAC,CAAC;QACjOO,SAAS;IACX;IACAO,8BAA8BnB,IAAI;QAChCW,MAAM,CAAC,iHAAiH,EAAEL,kCAAkC,CAAC,CAAC;QAC9JM,SAAS;IACX;IACAQ,0CAA0CpB,IAAI;QAC5CW,MAAM;QACNC,SAAS;IACX;IACAS,4CAA4CrB,IAAI;QAC9CW,MAAM;QACNC,SAAS;IACX;IACAU,4CAA4CtB,IAAI;QAC9CW,MAAM;QACNC,SAAS;IACX;IACAW,8CAA8CvB,IAAI;QAChDW,MAAM;QACNC,SAAS;IACX;IACAY,+BAA+BxB,IAAI;QACjCW,MAAM;QACNC,SAAS;IACX;IACAa,yBAAyBzB,IAAI;QAC3BW,MAAM;QACNC,SAAS;IACX;IACAc,6BAA6B1B,IAAI;QAC/BW,MAAM,CAAC,+PAA+P,EAAEH,iCAAiC,CAAC,CAAC;QAC3SI,SAAS;IACX;IACAe,wCAAwC3B,IAAI;QAC1CW,MAAM;QACNC,SAAS;IACX;IACAgB,8BAA8B5B,IAAI;QAChCW,MAAM;QACNC,SAAS;IACX;IACAiB,kCAAkC7B,IAAI;QACpCW,MAAM;QACNC,SAAS;IACX;IACAkB,8BAA8B9B,IAAI;QAChCW,MAAM;QACNC,SAAS;IACX;IACAmB,0CAA0C/B,IAAI;QAC5CW,MAAM;QACNC,SAAS;IACX;IACAoB,8CAA8ChC,IAAI;QAChDW,MAAM;QACNC,SAAS;IACX;IACAqB,yCAAyCjC,IAAI;QAC3CW,MAAM;QACNC,SAAS;IACX;IACAsB,0CAA0ClC,IAAI;QAC5CW,MAAM;QACNC,SAAS;IACX;IACAuB,mDAAmDnC,IAAI;QACrDW,MAAM;QACNC,SAAS;IACX;IACAwB,8CAA8CpC,IAAI;QAChDW,MAAM;QACNC,SAAS;IACX;IACAyB,0CAA0CrC,IAAI;QAC5CW,MAAM;QACNC,SAAS;IACX;IACA0B,sCAAsCtC,IAAI;QACxCW,MAAM;QACNC,SAAS;IACX;IACA2B,yCAAyCvC,IAAI;QAC3CW,MAAM;QACNC,SAAS;IACX;IACA4B,8BAA8BxC,IAAI;QAChCW,MAAM;QACNC,SAAS;IACX;IACA6B,uCAAuCzC,IAAI;QACzCW,MAAM;QACNC,SAAS;IACX;IACA8B,sCAAsC1C,IAAI;QACxCW,MAAM;QACNC,SAAS;IACX;IACA+B,mCAAmC3C,IAAI;QACrCW,MAAM;QACNC,SAAS;IACX;IACAgC,wCAAwC5C,IAAI;QAC1CW,MAAM;QACNC,SAAS;IACX;IACAiC,gEAAgE/C,KAAK;QACnEa,MAAM;QACNC,SAAS;IACX;AACF,GAAG;AAEH,MAAMkC,6BAA6B3C,eAAeM,OAAOQ,sBAAsB;AAC/E,MAAM8B,8BAA8B7C,kBAAkB4C;AAEtD,IAAIC,4BAA4BC,MAAM,GAAG,GAAG;IAC1C,MAAM,IAAIC,MACR,CAAC,yDAAyD,EAAEF,4BAA4BG,IAAI,CAAC,OAAO;AAExG;AAEA,OAAO,MAAMC,uBAAuBL,2BAA2B;AAE/D,IACE,CAACM,OAAOC,SAAS,CAAC5C,OAAOS,wCAAwC,KACjET,OAAOS,wCAAwC,GAAG,KAClDT,OAAOS,wCAAwC,GAC7Cb,mDACF;IACA,MAAM,IAAI4C,MACR,CAAC,0CAA0C,EAAExC,OAAOS,wCAAwC,CAAC,kDAAkD,EAAEb,kDAAkD,CAAC,CAAC;AAEzM;AAEA,KAAK,MAAM,CAACiD,MAAMC,MAAM,IAAI;IAC1B;QAAC;QAAsC9C,OAAOC,kCAAkC;KAAC;IACjF;QAAC;QAAsCD,OAAOI,kCAAkC;KAAC;IACjF;QAAC;QAAuCJ,OAAOK,mCAAmC;KAAC;CACpF,CAAW;IACV,IACE,CAACsC,OAAOC,SAAS,CAACE,UAClBA,QAAQ,KACRA,QAAQhD,iDACR;QACA,MAAM,IAAI0C,MACR,GAAGK,KAAK,EAAE,EAAEC,MAAM,kDAAkD,EAAEhD,gDAAgD,CAAC,CAAC;IAE5H;AACF;AAEA,IACE,CAAC6C,OAAOC,SAAS,CAAC5C,OAAOM,uCAAuC,KAChEN,OAAOM,uCAAuC,GAAG,GACjD;IACA,MAAM,IAAIkC,MACR,CAAC,yCAAyC,EAAExC,OAAOM,uCAAuC,CAAC,8BAA8B,CAAC;AAE9H;AAEA,IACE,CAACqC,OAAOC,SAAS,CAAC5C,OAAOO,kCAAkC,KAC3DP,OAAOO,kCAAkC,GAAG,GAC5C;IACA,MAAM,IAAIiC,MACR,CAAC,oCAAoC,EAAExC,OAAOO,kCAAkC,CAAC,8BAA8B,CAAC;AAEpH;AAEA,IACE,CAACoC,OAAOC,SAAS,CAAC5C,OAAOU,4BAA4B,KACrDV,OAAOU,4BAA4B,GAAG,KACtCV,OAAOU,4BAA4B,GAAGb,mCACtC;IACA,MAAM,IAAI2C,MACR,CAAC,8BAA8B,EAAExC,OAAOU,4BAA4B,CAAC,uCAAuC,EAAEb,kCAAkC,CAAC,CAAC;AAEtJ;AAEA,KAAK,MAAM,CAACgD,MAAMC,MAAM,IAAI;IAC1B;QAAC;QAA4C9C,OAAOW,wCAAwC;KAAC;IAC7F;QAAC;QAA8CX,OAAOY,0CAA0C;KAAC;IACjG;QAAC;QAA8CZ,OAAOa,0CAA0C;KAAC;IACjG;QACE;QACAb,OAAOc,4CAA4C;KACpD;IACD;QAAC;QAAiCd,OAAOe,6BAA6B;KAAC;CACxE,CAAW;IACV,IAAI,CAAC4B,OAAOC,SAAS,CAACE,UAAUA,QAAQ,GAAG;QACzC,MAAM,IAAIN,MAAM,GAAGK,KAAK,EAAE,EAAEC,MAAM,8BAA8B,CAAC;IACnE;AACF;AAEA,IAAI,CAACH,OAAOC,SAAS,CAAC5C,OAAOgB,uBAAuB,KAAKhB,OAAOgB,uBAAuB,GAAG,GAAG;IAC3F,MAAM,IAAIwB,MACR,CAAC,yBAAyB,EAAExC,OAAOgB,uBAAuB,CAAC,yCAAyC,CAAC;AAEzG;AAEA,IACE,CAAC2B,OAAOC,SAAS,CAAC5C,OAAOiB,2BAA2B,KACpDjB,OAAOiB,2BAA2B,GAAG,KACrCjB,OAAOiB,2BAA2B,GAAGlB,kCACrC;IACA,MAAM,IAAIyC,MACR,CAAC,6BAA6B,EAAExC,OAAOiB,2BAA2B,CAAC,kDAAkD,EAAElB,iCAAiC,CAAC,CAAC;AAE9J;AAEA,IAAIC,OAAOgB,uBAAuB,GAAGhB,OAAOiB,2BAA2B,EAAE;IACvE,MAAM,IAAIuB,MACR,CAAC,yBAAyB,EAAExC,OAAOgB,uBAAuB,CAAC,wDAAwD,EAAEhB,OAAOiB,2BAA2B,CAAC,EAAE,CAAC;AAE/J;AAEA,IACE,CAAC0B,OAAOC,SAAS,CAAC5C,OAAOkB,sCAAsC,KAC/DlB,OAAOkB,sCAAsC,GAAG,GAChD;IACA,MAAM,IAAIsB,MACR,CAAC,wCAAwC,EAAExC,OAAOkB,sCAAsC,CAAC,yCAAyC,CAAC;AAEvI;AAEA,IACE,CAACyB,OAAOC,SAAS,CAAC5C,OAAOmB,4BAA4B,KACrDnB,OAAOmB,4BAA4B,GAAG,GACtC;IACA,MAAM,IAAIqB,MACR,CAAC,8BAA8B,EAAExC,OAAOmB,4BAA4B,CAAC,8CAA8C,CAAC;AAExH;AAEA,IACE,CAACwB,OAAOC,SAAS,CAAC5C,OAAOoB,gCAAgC,KACzDpB,OAAOoB,gCAAgC,GAAGpB,OAAOmB,4BAA4B,EAC7E;IACA,MAAM,IAAIqB,MACR,CAAC,kCAAkC,EAAExC,OAAOoB,gCAAgC,CAAC,0EAA0E,EAAEpB,OAAOmB,4BAA4B,CAAC,EAAE,CAAC;AAEpM;AAEA,IACE,CAACwB,OAAOC,SAAS,CAAC5C,OAAOqB,4BAA4B,KACrDrB,OAAOqB,4BAA4B,GAAG,GACtC;IACA,MAAM,IAAImB,MACR,CAAC,8BAA8B,EAAExC,OAAOqB,4BAA4B,CAAC,yCAAyC,CAAC;AAEnH;AAEA,IACE,CAACsB,OAAOC,SAAS,CAAC5C,OAAOsB,wCAAwC,KACjEtB,OAAOsB,wCAAwC,GAAG,GAClD;IACA,MAAM,IAAIkB,MACR,CAAC,0CAA0C,EAAExC,OAAOsB,wCAAwC,CAAC,yCAAyC,CAAC;AAE3I;AAEA,IACE,CAACqB,OAAOC,SAAS,CAAC5C,OAAOuB,4CAA4C,KACrEvB,OAAOuB,4CAA4C,GAAG,GACtD;IACA,MAAM,IAAIiB,MACR,CAAC,8CAA8C,EAAExC,OAAOuB,4CAA4C,CAAC,yCAAyC,CAAC;AAEnJ;AAEA,IACE,CAACoB,OAAOC,SAAS,CAAC5C,OAAOwB,uCAAuC,KAChExB,OAAOwB,uCAAuC,GAAG,KACjDxB,OAAOwB,uCAAuC,IAAI7B,6BAClD;IACA,MAAM,IAAI6C,MACR,CAAC,yCAAyC,EAAExC,OAAOwB,uCAAuC,CAAC,+CAA+C,EAAE7B,4BAA4B,CAAC,CAAC;AAE9K;AAEA,IACE,CAACgD,OAAOC,SAAS,CAAC5C,OAAOyB,wCAAwC,KACjEzB,OAAOyB,wCAAwC,GAAG,GAClD;IACA,MAAM,IAAIe,MACR,CAAC,0CAA0C,EAAExC,OAAOyB,wCAAwC,CAAC,yCAAyC,CAAC;AAE3I;AAEA,IACE,CAACkB,OAAOC,SAAS,CAAC5C,OAAO0B,iDAAiD,KAC1E1B,OAAO0B,iDAAiD,GAAG,GAC3D;IACA,MAAM,IAAIc,MACR,CAAC,mDAAmD,EAAExC,OAAO0B,iDAAiD,CAAC,yCAAyC,CAAC;AAE7J;AAEA,IACE,CAACiB,OAAOC,SAAS,CAAC5C,OAAO2B,4CAA4C,KACrE3B,OAAO2B,4CAA4C,GAAG,GACtD;IACA,MAAM,IAAIa,MACR,CAAC,8CAA8C,EAAExC,OAAO2B,4CAA4C,CAAC,8BAA8B,CAAC;AAExI;AAEA,IACE,CAACgB,OAAOC,SAAS,CAAC5C,OAAO4B,wCAAwC,KACjE5B,OAAO4B,wCAAwC,GAAG,GAClD;IACA,MAAM,IAAIY,MACR,CAAC,0CAA0C,EAAExC,OAAO4B,wCAAwC,CAAC,yCAAyC,CAAC;AAE3I;AAEA,IACE,CAACe,OAAOC,SAAS,CAAC5C,OAAO6B,oCAAoC,KAC7D7B,OAAO6B,oCAAoC,GAAG,GAC9C;IACA,MAAM,IAAIW,MACR,CAAC,sCAAsC,EAAExC,OAAO6B,oCAAoC,CAAC,sCAAsC,CAAC;AAEhI;AAEA,IACE,CAACc,OAAOC,SAAS,CAAC5C,OAAO8B,uCAAuC,KAChE9B,OAAO8B,uCAAuC,GAAG,GACjD;IACA,MAAM,IAAIU,MACR,CAAC,yCAAyC,EAAExC,OAAO8B,uCAAuC,CAAC,sCAAsC,CAAC;AAEtI;AAEA,IACE,CAACa,OAAOC,SAAS,CAAC5C,OAAO+B,4BAA4B,KACrD/B,OAAO+B,4BAA4B,GAAG,GACtC;IACA,MAAM,IAAIS,MACR,CAAC,8BAA8B,EAAExC,OAAO+B,4BAA4B,CAAC,8BAA8B,CAAC;AAExG;AAEA,IACE,CAACY,OAAOC,SAAS,CAAC5C,OAAOgC,qCAAqC,KAC9DhC,OAAOgC,qCAAqC,GAAG,GAC/C;IACA,MAAM,IAAIQ,MACR,CAAC,uCAAuC,EAAExC,OAAOgC,qCAAqC,CAAC,sCAAsC,CAAC;AAElI;AAEA,IACE,CAACW,OAAOC,SAAS,CAAC5C,OAAOiC,oCAAoC,KAC7DjC,OAAOiC,oCAAoC,GAAG,GAC9C;IACA,MAAM,IAAIO,MACR,CAAC,sCAAsC,EAAExC,OAAOiC,oCAAoC,CAAC,8BAA8B,CAAC;AAExH;AAEA,IACE,CAACU,OAAOC,SAAS,CAAC5C,OAAOkC,iCAAiC,KAC1DlC,OAAOkC,iCAAiC,GAAG,GAC3C;IACA,MAAM,IAAIM,MACR,CAAC,mCAAmC,EAAExC,OAAOkC,iCAAiC,CAAC,yCAAyC,CAAC;AAE7H;AAEA,IACE,CAACS,OAAOC,SAAS,CAAC5C,OAAOmC,sCAAsC,KAC/DnC,OAAOmC,sCAAsC,GAAG,GAChD;IACA,MAAM,IAAIK,MACR,CAAC,wCAAwC,EAAExC,OAAOmC,sCAAsC,CAAC,yCAAyC,CAAC;AAEvI;AAEA,IACEnC,OAAO0B,iDAAiD,IACxD1B,OAAOmC,sCAAsC,EAC7C;IACA,MAAM,IAAIK,MACR,CAAC,mDAAmD,EAAExC,OAAO0B,iDAAiD,CAAC,+DAA+D,EAAE1B,OAAOmC,sCAAsC,CAAC,EAAE,CAAC;AAErO;AAEA,IACEnC,OAAO0B,iDAAiD,IACxD1B,OAAO4B,wCAAwC,EAC/C;IACA,MAAM,IAAIY,MACR,CAAC,mDAAmD,EAAExC,OAAO0B,iDAAiD,CAAC,iEAAiE,EAAE1B,OAAO4B,wCAAwC,CAAC,EAAE,CAAC;AAEzO"}
|
package/dist/core/demand.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { type DemandStat, type ReservationGrant, type ReservationTemplate } from '#db/reservations.js';
|
|
2
|
-
import { type ActiveRunnerInstanceTemplateCount } from '#db/runner-instances.js';
|
|
2
|
+
import { type ActiveRunnerInstanceTemplateCount, type RunnerInstanceTerminateIntent } from '#db/runner-instances.js';
|
|
3
3
|
export interface PollDemandParams {
|
|
4
4
|
workspaceId: string;
|
|
5
5
|
provisionerId: string;
|
|
@@ -14,6 +14,8 @@ export interface PollDemandResult {
|
|
|
14
14
|
stats: DemandStat[];
|
|
15
15
|
reservations: ReservationGrant[];
|
|
16
16
|
terminateRunnerInstanceIds: string[];
|
|
17
|
+
/** Units reserved this poll. A bound-only allocation carries no launch grant, so this is what ends the long poll. */
|
|
18
|
+
newlyReservedCount?: number;
|
|
17
19
|
}
|
|
18
20
|
export interface RunnerInstanceCountDivergence {
|
|
19
21
|
templateKey: string;
|
|
@@ -23,7 +25,7 @@ export interface RunnerInstanceCountDivergence {
|
|
|
23
25
|
}
|
|
24
26
|
export declare function pollDemand(params: PollDemandParams): Promise<PollDemandResult>;
|
|
25
27
|
export declare function releaseReservationGrants(reservations: ReservationGrant[]): Promise<void>;
|
|
26
|
-
export declare function shouldReturn(result: PollDemandResult, maxReservations: number, totalCapacity: number, deadlinePassed: boolean): boolean;
|
|
28
|
+
export declare function shouldReturn(result: PollDemandResult, maxReservations: number, totalCapacity: number, deadlinePassed: boolean, terminateIntents?: readonly RunnerInstanceTerminateIntent[]): boolean;
|
|
27
29
|
export declare function nextBackoffInterval(ms: number): number;
|
|
28
30
|
export declare function withJitter(ms: number): number;
|
|
29
31
|
export declare function calculateRunnerInstanceCountDivergences(params: {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"demand.d.ts","sourceRoot":"","sources":["../../src/core/demand.ts"],"names":[],"mappings":"AAIA,OAAO,EACL,KAAK,UAAU,EAGf,KAAK,gBAAgB,EACrB,KAAK,mBAAmB,EACzB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,KAAK,iCAAiC,
|
|
1
|
+
{"version":3,"file":"demand.d.ts","sourceRoot":"","sources":["../../src/core/demand.ts"],"names":[],"mappings":"AAIA,OAAO,EACL,KAAK,UAAU,EAGf,KAAK,gBAAgB,EACrB,KAAK,mBAAmB,EACzB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,KAAK,iCAAiC,EAGtC,KAAK,6BAA6B,EACnC,MAAM,yBAAyB,CAAC;AAOjC,MAAM,WAAW,gBAAgB;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,SAAS,EAAE,mBAAmB,EAAE,CAAC;IACjC,MAAM,EAAE,WAAW,CAAC;CACrB;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,YAAY,EAAE,gBAAgB,EAAE,CAAC;IACjC,0BAA0B,EAAE,MAAM,EAAE,CAAC;IACrC,qHAAqH;IACrH,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,6BAA6B;IAC5C,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,UAAU,GAAG,SAAS,CAAC;IAC9B,SAAS,EAAE,gBAAgB,GAAG,mBAAmB,CAAC;IAClD,KAAK,EAAE,MAAM,CAAC;CACf;AAQD,wBAAsB,UAAU,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAsGpF;AAED,wBAAsB,wBAAwB,CAAC,YAAY,EAAE,gBAAgB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAK9F;AAED,wBAAgB,YAAY,CAC1B,MAAM,EAAE,gBAAgB,EACxB,eAAe,EAAE,MAAM,EACvB,aAAa,EAAE,MAAM,EACrB,cAAc,EAAE,OAAO,EACvB,gBAAgB,CAAC,EAAE,SAAS,6BAA6B,EAAE,GAC1D,OAAO,CAcT;AAED,wBAAgB,mBAAmB,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAEtD;AAED,wBAAgB,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAE7C;AAED,wBAAgB,uCAAuC,CAAC,MAAM,EAAE;IAC9D,mBAAmB,EAAE,mBAAmB,EAAE,CAAC;IAC3C,aAAa,EAAE,iCAAiC,EAAE,CAAC;CACpD,GAAG,6BAA6B,EAAE,CA4BlC"}
|
package/dist/core/demand.js
CHANGED
|
@@ -4,7 +4,7 @@ import { config } from '#config.js';
|
|
|
4
4
|
import { db } from '#db/db.js';
|
|
5
5
|
import { deleteReservationsByIds, pollDemandAndReserveTx } from '#db/reservations.js';
|
|
6
6
|
import { listActiveRunnerInstanceCountsByTemplateTx, listProvisionerTerminateIntentRowsTx } from '#db/runner-instances.js';
|
|
7
|
-
import { providerRunnerCountDivergenceCount, providerRunnerTerminateIntentIssuedCount } from '#metrics/instance.js';
|
|
7
|
+
import { providerRunnerCountDivergenceCount, providerRunnerTerminateIntentIssuedCount, recordProviderRunnerActivationOutcome } from '#metrics/instance.js';
|
|
8
8
|
export async function pollDemand(params) {
|
|
9
9
|
const waitSeconds = Math.min(params.waitSeconds ?? config.RESERVATION_LONG_POLL_MAX_WAIT_SECONDS, config.RESERVATION_LONG_POLL_MAX_WAIT_SECONDS);
|
|
10
10
|
const deadlineMs = Date.now() + Math.max(0, waitSeconds) * 1000;
|
|
@@ -29,6 +29,7 @@ export async function pollDemand(params) {
|
|
|
29
29
|
provisionerId: params.provisionerId,
|
|
30
30
|
maxReservations: params.maxReservations,
|
|
31
31
|
ttlSeconds: params.ttlSeconds,
|
|
32
|
+
activationGraceSeconds: config.RESERVATION_TTL_SECONDS,
|
|
32
33
|
templates: params.templates
|
|
33
34
|
});
|
|
34
35
|
const terminateIntents = await listProvisionerTerminateIntentRowsTx(tx, {
|
|
@@ -36,11 +37,16 @@ export async function pollDemand(params) {
|
|
|
36
37
|
provisionerId: params.provisionerId,
|
|
37
38
|
limit: params.terminateIntentLimit
|
|
38
39
|
});
|
|
40
|
+
const newlyReservedCount = demand.newlyReservedUnits.reduce((total, reservation)=>total + reservation.count, 0);
|
|
39
41
|
const result = {
|
|
40
|
-
|
|
42
|
+
stats: demand.stats,
|
|
43
|
+
reservations: demand.reservations,
|
|
44
|
+
...newlyReservedCount > 0 ? {
|
|
45
|
+
newlyReservedCount
|
|
46
|
+
} : {},
|
|
41
47
|
terminateRunnerInstanceIds: terminateIntents.map((intent)=>intent.providerRunnerId)
|
|
42
48
|
};
|
|
43
|
-
if (!shouldReturn(result, params.maxReservations, totalCapacity, deadlinePassed)) {
|
|
49
|
+
if (!shouldReturn(result, params.maxReservations, totalCapacity, deadlinePassed, terminateIntents)) {
|
|
44
50
|
return {
|
|
45
51
|
result,
|
|
46
52
|
terminateIntents,
|
|
@@ -64,7 +70,7 @@ export async function pollDemand(params) {
|
|
|
64
70
|
return previousSnapshot.result;
|
|
65
71
|
}
|
|
66
72
|
lastSnapshot = snapshot;
|
|
67
|
-
if (shouldReturn(lastSnapshot.result, params.maxReservations, totalCapacity, deadlinePassed)) {
|
|
73
|
+
if (shouldReturn(lastSnapshot.result, params.maxReservations, totalCapacity, deadlinePassed, lastSnapshot.terminateIntents)) {
|
|
68
74
|
recordPollDemandMetrics(params, lastSnapshot);
|
|
69
75
|
return lastSnapshot.result;
|
|
70
76
|
}
|
|
@@ -81,10 +87,14 @@ export async function pollDemand(params) {
|
|
|
81
87
|
}
|
|
82
88
|
}
|
|
83
89
|
export async function releaseReservationGrants(reservations) {
|
|
90
|
+
// Fully adopted reservations are intentionally absent from this list: deleting
|
|
91
|
+
// one on disconnect would orphan the runner bound to that reservation. They
|
|
92
|
+
// remain protected until terminal cleanup or expiry.
|
|
84
93
|
await deleteReservationsByIds(reservations.map((reservation)=>reservation.reservationId));
|
|
85
94
|
}
|
|
86
|
-
export function shouldReturn(result, maxReservations, totalCapacity, deadlinePassed) {
|
|
87
|
-
|
|
95
|
+
export function shouldReturn(result, maxReservations, totalCapacity, deadlinePassed, terminateIntents) {
|
|
96
|
+
const hasImmediateTerminateIntent = terminateIntents?.some((intent)=>intent.reason !== 'activation-timeout' || !intent.activationTimeoutRetry) ?? result.terminateRunnerInstanceIds.length > 0;
|
|
97
|
+
return maxReservations === 0 || totalCapacity === 0 || result.reservations.length > 0 || (result.newlyReservedCount ?? 0) > 0 || hasImmediateTerminateIntent || deadlinePassed;
|
|
88
98
|
}
|
|
89
99
|
export function nextBackoffInterval(ms) {
|
|
90
100
|
return Math.min(ms * 1.5, config.RESERVATION_POLL_MAX_INTERVAL_MS);
|
|
@@ -147,6 +157,12 @@ function recordPollDemandMetrics(params, snapshot) {
|
|
|
147
157
|
surface: 'poll-demand',
|
|
148
158
|
reason: intent.reason
|
|
149
159
|
});
|
|
160
|
+
// Retries redeliver an intent already counted on its first emission.
|
|
161
|
+
if (intent.reason === 'activation-timeout' && !intent.activationTimeoutRetry) {
|
|
162
|
+
recordProviderRunnerActivationOutcome({
|
|
163
|
+
outcome: 'reaped'
|
|
164
|
+
});
|
|
165
|
+
}
|
|
150
166
|
}
|
|
151
167
|
}
|
|
152
168
|
function addCount(counts, key, count) {
|
package/dist/core/demand.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/core/demand.ts"],"sourcesContent":["import {setTimeout as sleep} from 'node:timers/promises';\nimport {logger} from '@shipfox/node-opentelemetry';\nimport {config} from '#config.js';\nimport {db} from '#db/db.js';\nimport {\n type DemandStat,\n deleteReservationsByIds,\n pollDemandAndReserveTx,\n type ReservationGrant,\n type ReservationTemplate,\n} from '#db/reservations.js';\nimport {\n type ActiveRunnerInstanceTemplateCount,\n listActiveRunnerInstanceCountsByTemplateTx,\n listProvisionerTerminateIntentRowsTx,\n type RunnerInstanceTerminateIntent,\n} from '#db/runner-instances.js';\nimport {\n providerRunnerCountDivergenceCount,\n providerRunnerTerminateIntentIssuedCount,\n} from '#metrics/instance.js';\n\nexport interface PollDemandParams {\n workspaceId: string;\n provisionerId: string;\n maxReservations: number;\n waitSeconds?: number | undefined;\n ttlSeconds: number;\n terminateIntentLimit: number;\n templates: ReservationTemplate[];\n signal: AbortSignal;\n}\n\nexport interface PollDemandResult {\n stats: DemandStat[];\n reservations: ReservationGrant[];\n terminateRunnerInstanceIds: string[];\n}\n\nexport interface RunnerInstanceCountDivergence {\n templateKey: string;\n state: 'starting' | 'running';\n direction: 'backend-higher' | 'advertised-higher';\n delta: number;\n}\n\ninterface PollDemandSnapshot {\n result: PollDemandResult;\n divergences: RunnerInstanceCountDivergence[];\n terminateIntents: RunnerInstanceTerminateIntent[];\n}\n\nexport async function pollDemand(params: PollDemandParams): Promise<PollDemandResult> {\n const waitSeconds = Math.min(\n params.waitSeconds ?? config.RESERVATION_LONG_POLL_MAX_WAIT_SECONDS,\n config.RESERVATION_LONG_POLL_MAX_WAIT_SECONDS,\n );\n const deadlineMs = Date.now() + Math.max(0, waitSeconds) * 1000;\n const totalCapacity = params.templates.reduce(\n (total, template) => total + template.availableSlots,\n 0,\n );\n let interval = config.RESERVATION_POLL_INTERVAL_MS;\n let lastSnapshot: PollDemandSnapshot = {\n result: {stats: [], reservations: [], terminateRunnerInstanceIds: []},\n divergences: [],\n terminateIntents: [],\n };\n\n while (true) {\n if (params.signal.aborted) return lastSnapshot.result;\n\n const previousSnapshot = lastSnapshot;\n const deadlinePassed = Date.now() >= deadlineMs;\n const snapshot = await db().transaction(async (tx) => {\n const demand = await pollDemandAndReserveTx(tx, {\n workspaceId: params.workspaceId,\n provisionerId: params.provisionerId,\n maxReservations: params.maxReservations,\n ttlSeconds: params.ttlSeconds,\n templates: params.templates,\n });\n const terminateIntents = await listProvisionerTerminateIntentRowsTx(tx, {\n workspaceId: params.workspaceId,\n provisionerId: params.provisionerId,\n limit: params.terminateIntentLimit,\n });\n const result: PollDemandResult = {\n ...demand,\n terminateRunnerInstanceIds: terminateIntents.map((intent) => intent.providerRunnerId),\n };\n\n if (!shouldReturn(result, params.maxReservations, totalCapacity, deadlinePassed)) {\n return {result, terminateIntents, divergences: []};\n }\n\n return {\n result,\n terminateIntents,\n divergences: calculateRunnerInstanceCountDivergences({\n advertisedTemplates: params.templates,\n backendCounts: await listActiveRunnerInstanceCountsByTemplateTx(tx, {\n workspaceId: params.workspaceId,\n provisionerId: params.provisionerId,\n }),\n }),\n };\n });\n if (params.signal.aborted) {\n await releaseReservationGrants(snapshot.result.reservations);\n return previousSnapshot.result;\n }\n\n lastSnapshot = snapshot;\n\n if (shouldReturn(lastSnapshot.result, params.maxReservations, totalCapacity, deadlinePassed)) {\n recordPollDemandMetrics(params, lastSnapshot);\n return lastSnapshot.result;\n }\n\n const remainingWaitMs = Math.max(0, deadlineMs - Date.now());\n try {\n await sleep(Math.min(withJitter(interval), remainingWaitMs), undefined, {\n signal: params.signal,\n });\n } catch (error) {\n if (params.signal.aborted) return lastSnapshot.result;\n throw error;\n }\n interval = nextBackoffInterval(interval);\n }\n}\n\nexport async function releaseReservationGrants(reservations: ReservationGrant[]): Promise<void> {\n await deleteReservationsByIds(reservations.map((reservation) => reservation.reservationId));\n}\n\nexport function shouldReturn(\n result: PollDemandResult,\n maxReservations: number,\n totalCapacity: number,\n deadlinePassed: boolean,\n): boolean {\n return (\n maxReservations === 0 ||\n totalCapacity === 0 ||\n result.reservations.length > 0 ||\n result.terminateRunnerInstanceIds.length > 0 ||\n deadlinePassed\n );\n}\n\nexport function nextBackoffInterval(ms: number): number {\n return Math.min(ms * 1.5, config.RESERVATION_POLL_MAX_INTERVAL_MS);\n}\n\nexport function withJitter(ms: number): number {\n return Math.random() * ms;\n}\n\nexport function calculateRunnerInstanceCountDivergences(params: {\n advertisedTemplates: ReservationTemplate[];\n backendCounts: ActiveRunnerInstanceTemplateCount[];\n}): RunnerInstanceCountDivergence[] {\n const advertisedCounts = new Map<string, number>();\n const backendCounts = new Map<string, number>();\n\n for (const template of params.advertisedTemplates) {\n addCount(advertisedCounts, countKey(template.templateKey, 'starting'), template.starting);\n addCount(advertisedCounts, countKey(template.templateKey, 'running'), template.running);\n }\n for (const count of params.backendCounts) {\n addCount(backendCounts, countKey(count.templateKey, count.state), count.count);\n }\n\n const keys = [...new Set([...advertisedCounts.keys(), ...backendCounts.keys()])].sort();\n return keys.flatMap((key) => {\n const advertised = advertisedCounts.get(key) ?? 0;\n const backend = backendCounts.get(key) ?? 0;\n if (advertised === backend) return [];\n\n const [templateKey, state] = splitCountKey(key);\n return [\n {\n templateKey,\n state,\n direction: backend > advertised ? 'backend-higher' : 'advertised-higher',\n delta: Math.abs(backend - advertised),\n },\n ];\n });\n}\n\nfunction recordPollDemandMetrics(params: PollDemandParams, snapshot: PollDemandSnapshot): void {\n for (const divergence of snapshot.divergences) {\n logger().debug(\n {\n workspaceId: params.workspaceId,\n provisionerId: params.provisionerId,\n templateKey: divergence.templateKey,\n state: divergence.state,\n direction: divergence.direction,\n delta: divergence.delta,\n },\n 'provisioned runner count divergence observed',\n );\n\n const attributes = {\n state: divergence.state,\n direction: divergence.direction,\n ...(config.PROVISIONED_RUNNER_COUNT_DIVERGENCE_TEMPLATE_KEY_LABEL_ENABLED\n ? {template_key: divergence.templateKey}\n : {}),\n };\n providerRunnerCountDivergenceCount.add(divergence.delta, attributes);\n }\n for (const intent of snapshot.terminateIntents) {\n providerRunnerTerminateIntentIssuedCount.add(1, {\n surface: 'poll-demand',\n reason: intent.reason,\n });\n }\n}\n\nfunction addCount(counts: Map<string, number>, key: string, count: number): void {\n counts.set(key, (counts.get(key) ?? 0) + count);\n}\n\nfunction countKey(templateKey: string, state: 'starting' | 'running'): string {\n return `${templateKey}\\0${state}`;\n}\n\nfunction splitCountKey(key: string): [string, 'starting' | 'running'] {\n const [templateKey, state] = key.split('\\0');\n if (!templateKey || (state !== 'starting' && state !== 'running')) {\n throw new Error(`Invalid provisioned runner count key: ${key}`);\n }\n return [templateKey, state];\n}\n"],"names":["setTimeout","sleep","logger","config","db","deleteReservationsByIds","pollDemandAndReserveTx","listActiveRunnerInstanceCountsByTemplateTx","listProvisionerTerminateIntentRowsTx","providerRunnerCountDivergenceCount","providerRunnerTerminateIntentIssuedCount","pollDemand","params","waitSeconds","Math","min","RESERVATION_LONG_POLL_MAX_WAIT_SECONDS","deadlineMs","Date","now","max","totalCapacity","templates","reduce","total","template","availableSlots","interval","RESERVATION_POLL_INTERVAL_MS","lastSnapshot","result","stats","reservations","terminateRunnerInstanceIds","divergences","terminateIntents","signal","aborted","previousSnapshot","deadlinePassed","snapshot","transaction","tx","demand","workspaceId","provisionerId","maxReservations","ttlSeconds","limit","terminateIntentLimit","map","intent","providerRunnerId","shouldReturn","calculateRunnerInstanceCountDivergences","advertisedTemplates","backendCounts","releaseReservationGrants","recordPollDemandMetrics","remainingWaitMs","withJitter","undefined","error","nextBackoffInterval","reservation","reservationId","length","ms","RESERVATION_POLL_MAX_INTERVAL_MS","random","advertisedCounts","Map","addCount","countKey","templateKey","starting","running","count","state","keys","Set","sort","flatMap","key","advertised","get","backend","splitCountKey","direction","delta","abs","divergence","debug","attributes","PROVISIONED_RUNNER_COUNT_DIVERGENCE_TEMPLATE_KEY_LABEL_ENABLED","template_key","add","surface","reason","counts","set","split","Error"],"mappings":"AAAA,SAAQA,cAAcC,KAAK,QAAO,uBAAuB;AACzD,SAAQC,MAAM,QAAO,8BAA8B;AACnD,SAAQC,MAAM,QAAO,aAAa;AAClC,SAAQC,EAAE,QAAO,YAAY;AAC7B,SAEEC,uBAAuB,EACvBC,sBAAsB,QAGjB,sBAAsB;AAC7B,SAEEC,0CAA0C,EAC1CC,oCAAoC,QAE/B,0BAA0B;AACjC,SACEC,kCAAkC,EAClCC,wCAAwC,QACnC,uBAAuB;AAgC9B,OAAO,eAAeC,WAAWC,MAAwB;IACvD,MAAMC,cAAcC,KAAKC,GAAG,CAC1BH,OAAOC,WAAW,IAAIV,OAAOa,sCAAsC,EACnEb,OAAOa,sCAAsC;IAE/C,MAAMC,aAAaC,KAAKC,GAAG,KAAKL,KAAKM,GAAG,CAAC,GAAGP,eAAe;IAC3D,MAAMQ,gBAAgBT,OAAOU,SAAS,CAACC,MAAM,CAC3C,CAACC,OAAOC,WAAaD,QAAQC,SAASC,cAAc,EACpD;IAEF,IAAIC,WAAWxB,OAAOyB,4BAA4B;IAClD,IAAIC,eAAmC;QACrCC,QAAQ;YAACC,OAAO,EAAE;YAAEC,cAAc,EAAE;YAAEC,4BAA4B,EAAE;QAAA;QACpEC,aAAa,EAAE;QACfC,kBAAkB,EAAE;IACtB;IAEA,MAAO,KAAM;QACX,IAAIvB,OAAOwB,MAAM,CAACC,OAAO,EAAE,OAAOR,aAAaC,MAAM;QAErD,MAAMQ,mBAAmBT;QACzB,MAAMU,iBAAiBrB,KAAKC,GAAG,MAAMF;QACrC,MAAMuB,WAAW,MAAMpC,KAAKqC,WAAW,CAAC,OAAOC;YAC7C,MAAMC,SAAS,MAAMrC,uBAAuBoC,IAAI;gBAC9CE,aAAahC,OAAOgC,WAAW;gBAC/BC,eAAejC,OAAOiC,aAAa;gBACnCC,iBAAiBlC,OAAOkC,eAAe;gBACvCC,YAAYnC,OAAOmC,UAAU;gBAC7BzB,WAAWV,OAAOU,SAAS;YAC7B;YACA,MAAMa,mBAAmB,MAAM3B,qCAAqCkC,IAAI;gBACtEE,aAAahC,OAAOgC,WAAW;gBAC/BC,eAAejC,OAAOiC,aAAa;gBACnCG,OAAOpC,OAAOqC,oBAAoB;YACpC;YACA,MAAMnB,SAA2B;gBAC/B,GAAGa,MAAM;gBACTV,4BAA4BE,iBAAiBe,GAAG,CAAC,CAACC,SAAWA,OAAOC,gBAAgB;YACtF;YAEA,IAAI,CAACC,aAAavB,QAAQlB,OAAOkC,eAAe,EAAEzB,eAAekB,iBAAiB;gBAChF,OAAO;oBAACT;oBAAQK;oBAAkBD,aAAa,EAAE;gBAAA;YACnD;YAEA,OAAO;gBACLJ;gBACAK;gBACAD,aAAaoB,wCAAwC;oBACnDC,qBAAqB3C,OAAOU,SAAS;oBACrCkC,eAAe,MAAMjD,2CAA2CmC,IAAI;wBAClEE,aAAahC,OAAOgC,WAAW;wBAC/BC,eAAejC,OAAOiC,aAAa;oBACrC;gBACF;YACF;QACF;QACA,IAAIjC,OAAOwB,MAAM,CAACC,OAAO,EAAE;YACzB,MAAMoB,yBAAyBjB,SAASV,MAAM,CAACE,YAAY;YAC3D,OAAOM,iBAAiBR,MAAM;QAChC;QAEAD,eAAeW;QAEf,IAAIa,aAAaxB,aAAaC,MAAM,EAAElB,OAAOkC,eAAe,EAAEzB,eAAekB,iBAAiB;YAC5FmB,wBAAwB9C,QAAQiB;YAChC,OAAOA,aAAaC,MAAM;QAC5B;QAEA,MAAM6B,kBAAkB7C,KAAKM,GAAG,CAAC,GAAGH,aAAaC,KAAKC,GAAG;QACzD,IAAI;YACF,MAAMlB,MAAMa,KAAKC,GAAG,CAAC6C,WAAWjC,WAAWgC,kBAAkBE,WAAW;gBACtEzB,QAAQxB,OAAOwB,MAAM;YACvB;QACF,EAAE,OAAO0B,OAAO;YACd,IAAIlD,OAAOwB,MAAM,CAACC,OAAO,EAAE,OAAOR,aAAaC,MAAM;YACrD,MAAMgC;QACR;QACAnC,WAAWoC,oBAAoBpC;IACjC;AACF;AAEA,OAAO,eAAe8B,yBAAyBzB,YAAgC;IAC7E,MAAM3B,wBAAwB2B,aAAakB,GAAG,CAAC,CAACc,cAAgBA,YAAYC,aAAa;AAC3F;AAEA,OAAO,SAASZ,aACdvB,MAAwB,EACxBgB,eAAuB,EACvBzB,aAAqB,EACrBkB,cAAuB;IAEvB,OACEO,oBAAoB,KACpBzB,kBAAkB,KAClBS,OAAOE,YAAY,CAACkC,MAAM,GAAG,KAC7BpC,OAAOG,0BAA0B,CAACiC,MAAM,GAAG,KAC3C3B;AAEJ;AAEA,OAAO,SAASwB,oBAAoBI,EAAU;IAC5C,OAAOrD,KAAKC,GAAG,CAACoD,KAAK,KAAKhE,OAAOiE,gCAAgC;AACnE;AAEA,OAAO,SAASR,WAAWO,EAAU;IACnC,OAAOrD,KAAKuD,MAAM,KAAKF;AACzB;AAEA,OAAO,SAASb,wCAAwC1C,MAGvD;IACC,MAAM0D,mBAAmB,IAAIC;IAC7B,MAAMf,gBAAgB,IAAIe;IAE1B,KAAK,MAAM9C,YAAYb,OAAO2C,mBAAmB,CAAE;QACjDiB,SAASF,kBAAkBG,SAAShD,SAASiD,WAAW,EAAE,aAAajD,SAASkD,QAAQ;QACxFH,SAASF,kBAAkBG,SAAShD,SAASiD,WAAW,EAAE,YAAYjD,SAASmD,OAAO;IACxF;IACA,KAAK,MAAMC,SAASjE,OAAO4C,aAAa,CAAE;QACxCgB,SAAShB,eAAeiB,SAASI,MAAMH,WAAW,EAAEG,MAAMC,KAAK,GAAGD,MAAMA,KAAK;IAC/E;IAEA,MAAME,OAAO;WAAI,IAAIC,IAAI;eAAIV,iBAAiBS,IAAI;eAAOvB,cAAcuB,IAAI;SAAG;KAAE,CAACE,IAAI;IACrF,OAAOF,KAAKG,OAAO,CAAC,CAACC;QACnB,MAAMC,aAAad,iBAAiBe,GAAG,CAACF,QAAQ;QAChD,MAAMG,UAAU9B,cAAc6B,GAAG,CAACF,QAAQ;QAC1C,IAAIC,eAAeE,SAAS,OAAO,EAAE;QAErC,MAAM,CAACZ,aAAaI,MAAM,GAAGS,cAAcJ;QAC3C,OAAO;YACL;gBACET;gBACAI;gBACAU,WAAWF,UAAUF,aAAa,mBAAmB;gBACrDK,OAAO3E,KAAK4E,GAAG,CAACJ,UAAUF;YAC5B;SACD;IACH;AACF;AAEA,SAAS1B,wBAAwB9C,MAAwB,EAAE4B,QAA4B;IACrF,KAAK,MAAMmD,cAAcnD,SAASN,WAAW,CAAE;QAC7ChC,SAAS0F,KAAK,CACZ;YACEhD,aAAahC,OAAOgC,WAAW;YAC/BC,eAAejC,OAAOiC,aAAa;YACnC6B,aAAaiB,WAAWjB,WAAW;YACnCI,OAAOa,WAAWb,KAAK;YACvBU,WAAWG,WAAWH,SAAS;YAC/BC,OAAOE,WAAWF,KAAK;QACzB,GACA;QAGF,MAAMI,aAAa;YACjBf,OAAOa,WAAWb,KAAK;YACvBU,WAAWG,WAAWH,SAAS;YAC/B,GAAIrF,OAAO2F,8DAA8D,GACrE;gBAACC,cAAcJ,WAAWjB,WAAW;YAAA,IACrC,CAAC,CAAC;QACR;QACAjE,mCAAmCuF,GAAG,CAACL,WAAWF,KAAK,EAAEI;IAC3D;IACA,KAAK,MAAM1C,UAAUX,SAASL,gBAAgB,CAAE;QAC9CzB,yCAAyCsF,GAAG,CAAC,GAAG;YAC9CC,SAAS;YACTC,QAAQ/C,OAAO+C,MAAM;QACvB;IACF;AACF;AAEA,SAAS1B,SAAS2B,MAA2B,EAAEhB,GAAW,EAAEN,KAAa;IACvEsB,OAAOC,GAAG,CAACjB,KAAK,AAACgB,CAAAA,OAAOd,GAAG,CAACF,QAAQ,CAAA,IAAKN;AAC3C;AAEA,SAASJ,SAASC,WAAmB,EAAEI,KAA6B;IAClE,OAAO,GAAGJ,YAAY,EAAE,EAAEI,OAAO;AACnC;AAEA,SAASS,cAAcJ,GAAW;IAChC,MAAM,CAACT,aAAaI,MAAM,GAAGK,IAAIkB,KAAK,CAAC;IACvC,IAAI,CAAC3B,eAAgBI,UAAU,cAAcA,UAAU,WAAY;QACjE,MAAM,IAAIwB,MAAM,CAAC,sCAAsC,EAAEnB,KAAK;IAChE;IACA,OAAO;QAACT;QAAaI;KAAM;AAC7B"}
|
|
1
|
+
{"version":3,"sources":["../../src/core/demand.ts"],"sourcesContent":["import {setTimeout as sleep} from 'node:timers/promises';\nimport {logger} from '@shipfox/node-opentelemetry';\nimport {config} from '#config.js';\nimport {db} from '#db/db.js';\nimport {\n type DemandStat,\n deleteReservationsByIds,\n pollDemandAndReserveTx,\n type ReservationGrant,\n type ReservationTemplate,\n} from '#db/reservations.js';\nimport {\n type ActiveRunnerInstanceTemplateCount,\n listActiveRunnerInstanceCountsByTemplateTx,\n listProvisionerTerminateIntentRowsTx,\n type RunnerInstanceTerminateIntent,\n} from '#db/runner-instances.js';\nimport {\n providerRunnerCountDivergenceCount,\n providerRunnerTerminateIntentIssuedCount,\n recordProviderRunnerActivationOutcome,\n} from '#metrics/instance.js';\n\nexport interface PollDemandParams {\n workspaceId: string;\n provisionerId: string;\n maxReservations: number;\n waitSeconds?: number | undefined;\n ttlSeconds: number;\n terminateIntentLimit: number;\n templates: ReservationTemplate[];\n signal: AbortSignal;\n}\n\nexport interface PollDemandResult {\n stats: DemandStat[];\n reservations: ReservationGrant[];\n terminateRunnerInstanceIds: string[];\n /** Units reserved this poll. A bound-only allocation carries no launch grant, so this is what ends the long poll. */\n newlyReservedCount?: number;\n}\n\nexport interface RunnerInstanceCountDivergence {\n templateKey: string;\n state: 'starting' | 'running';\n direction: 'backend-higher' | 'advertised-higher';\n delta: number;\n}\n\ninterface PollDemandSnapshot {\n result: PollDemandResult;\n divergences: RunnerInstanceCountDivergence[];\n terminateIntents: RunnerInstanceTerminateIntent[];\n}\n\nexport async function pollDemand(params: PollDemandParams): Promise<PollDemandResult> {\n const waitSeconds = Math.min(\n params.waitSeconds ?? config.RESERVATION_LONG_POLL_MAX_WAIT_SECONDS,\n config.RESERVATION_LONG_POLL_MAX_WAIT_SECONDS,\n );\n const deadlineMs = Date.now() + Math.max(0, waitSeconds) * 1000;\n const totalCapacity = params.templates.reduce(\n (total, template) => total + template.availableSlots,\n 0,\n );\n let interval = config.RESERVATION_POLL_INTERVAL_MS;\n let lastSnapshot: PollDemandSnapshot = {\n result: {stats: [], reservations: [], terminateRunnerInstanceIds: []},\n divergences: [],\n terminateIntents: [],\n };\n\n while (true) {\n if (params.signal.aborted) return lastSnapshot.result;\n\n const previousSnapshot = lastSnapshot;\n const deadlinePassed = Date.now() >= deadlineMs;\n const snapshot = await db().transaction(async (tx) => {\n const demand = await pollDemandAndReserveTx(tx, {\n workspaceId: params.workspaceId,\n provisionerId: params.provisionerId,\n maxReservations: params.maxReservations,\n ttlSeconds: params.ttlSeconds,\n activationGraceSeconds: config.RESERVATION_TTL_SECONDS,\n templates: params.templates,\n });\n const terminateIntents = await listProvisionerTerminateIntentRowsTx(tx, {\n workspaceId: params.workspaceId,\n provisionerId: params.provisionerId,\n limit: params.terminateIntentLimit,\n });\n const newlyReservedCount = demand.newlyReservedUnits.reduce(\n (total, reservation) => total + reservation.count,\n 0,\n );\n const result: PollDemandResult = {\n stats: demand.stats,\n reservations: demand.reservations,\n ...(newlyReservedCount > 0 ? {newlyReservedCount} : {}),\n terminateRunnerInstanceIds: terminateIntents.map((intent) => intent.providerRunnerId),\n };\n\n if (\n !shouldReturn(\n result,\n params.maxReservations,\n totalCapacity,\n deadlinePassed,\n terminateIntents,\n )\n ) {\n return {result, terminateIntents, divergences: []};\n }\n\n return {\n result,\n terminateIntents,\n divergences: calculateRunnerInstanceCountDivergences({\n advertisedTemplates: params.templates,\n backendCounts: await listActiveRunnerInstanceCountsByTemplateTx(tx, {\n workspaceId: params.workspaceId,\n provisionerId: params.provisionerId,\n }),\n }),\n };\n });\n if (params.signal.aborted) {\n await releaseReservationGrants(snapshot.result.reservations);\n return previousSnapshot.result;\n }\n\n lastSnapshot = snapshot;\n\n if (\n shouldReturn(\n lastSnapshot.result,\n params.maxReservations,\n totalCapacity,\n deadlinePassed,\n lastSnapshot.terminateIntents,\n )\n ) {\n recordPollDemandMetrics(params, lastSnapshot);\n return lastSnapshot.result;\n }\n\n const remainingWaitMs = Math.max(0, deadlineMs - Date.now());\n try {\n await sleep(Math.min(withJitter(interval), remainingWaitMs), undefined, {\n signal: params.signal,\n });\n } catch (error) {\n if (params.signal.aborted) return lastSnapshot.result;\n throw error;\n }\n interval = nextBackoffInterval(interval);\n }\n}\n\nexport async function releaseReservationGrants(reservations: ReservationGrant[]): Promise<void> {\n // Fully adopted reservations are intentionally absent from this list: deleting\n // one on disconnect would orphan the runner bound to that reservation. They\n // remain protected until terminal cleanup or expiry.\n await deleteReservationsByIds(reservations.map((reservation) => reservation.reservationId));\n}\n\nexport function shouldReturn(\n result: PollDemandResult,\n maxReservations: number,\n totalCapacity: number,\n deadlinePassed: boolean,\n terminateIntents?: readonly RunnerInstanceTerminateIntent[],\n): boolean {\n const hasImmediateTerminateIntent =\n terminateIntents?.some(\n (intent) => intent.reason !== 'activation-timeout' || !intent.activationTimeoutRetry,\n ) ?? result.terminateRunnerInstanceIds.length > 0;\n\n return (\n maxReservations === 0 ||\n totalCapacity === 0 ||\n result.reservations.length > 0 ||\n (result.newlyReservedCount ?? 0) > 0 ||\n hasImmediateTerminateIntent ||\n deadlinePassed\n );\n}\n\nexport function nextBackoffInterval(ms: number): number {\n return Math.min(ms * 1.5, config.RESERVATION_POLL_MAX_INTERVAL_MS);\n}\n\nexport function withJitter(ms: number): number {\n return Math.random() * ms;\n}\n\nexport function calculateRunnerInstanceCountDivergences(params: {\n advertisedTemplates: ReservationTemplate[];\n backendCounts: ActiveRunnerInstanceTemplateCount[];\n}): RunnerInstanceCountDivergence[] {\n const advertisedCounts = new Map<string, number>();\n const backendCounts = new Map<string, number>();\n\n for (const template of params.advertisedTemplates) {\n addCount(advertisedCounts, countKey(template.templateKey, 'starting'), template.starting);\n addCount(advertisedCounts, countKey(template.templateKey, 'running'), template.running);\n }\n for (const count of params.backendCounts) {\n addCount(backendCounts, countKey(count.templateKey, count.state), count.count);\n }\n\n const keys = [...new Set([...advertisedCounts.keys(), ...backendCounts.keys()])].sort();\n return keys.flatMap((key) => {\n const advertised = advertisedCounts.get(key) ?? 0;\n const backend = backendCounts.get(key) ?? 0;\n if (advertised === backend) return [];\n\n const [templateKey, state] = splitCountKey(key);\n return [\n {\n templateKey,\n state,\n direction: backend > advertised ? 'backend-higher' : 'advertised-higher',\n delta: Math.abs(backend - advertised),\n },\n ];\n });\n}\n\nfunction recordPollDemandMetrics(params: PollDemandParams, snapshot: PollDemandSnapshot): void {\n for (const divergence of snapshot.divergences) {\n logger().debug(\n {\n workspaceId: params.workspaceId,\n provisionerId: params.provisionerId,\n templateKey: divergence.templateKey,\n state: divergence.state,\n direction: divergence.direction,\n delta: divergence.delta,\n },\n 'provisioned runner count divergence observed',\n );\n\n const attributes = {\n state: divergence.state,\n direction: divergence.direction,\n ...(config.PROVISIONED_RUNNER_COUNT_DIVERGENCE_TEMPLATE_KEY_LABEL_ENABLED\n ? {template_key: divergence.templateKey}\n : {}),\n };\n providerRunnerCountDivergenceCount.add(divergence.delta, attributes);\n }\n for (const intent of snapshot.terminateIntents) {\n providerRunnerTerminateIntentIssuedCount.add(1, {\n surface: 'poll-demand',\n reason: intent.reason,\n });\n // Retries redeliver an intent already counted on its first emission.\n if (intent.reason === 'activation-timeout' && !intent.activationTimeoutRetry) {\n recordProviderRunnerActivationOutcome({outcome: 'reaped'});\n }\n }\n}\n\nfunction addCount(counts: Map<string, number>, key: string, count: number): void {\n counts.set(key, (counts.get(key) ?? 0) + count);\n}\n\nfunction countKey(templateKey: string, state: 'starting' | 'running'): string {\n return `${templateKey}\\0${state}`;\n}\n\nfunction splitCountKey(key: string): [string, 'starting' | 'running'] {\n const [templateKey, state] = key.split('\\0');\n if (!templateKey || (state !== 'starting' && state !== 'running')) {\n throw new Error(`Invalid provisioned runner count key: ${key}`);\n }\n return [templateKey, state];\n}\n"],"names":["setTimeout","sleep","logger","config","db","deleteReservationsByIds","pollDemandAndReserveTx","listActiveRunnerInstanceCountsByTemplateTx","listProvisionerTerminateIntentRowsTx","providerRunnerCountDivergenceCount","providerRunnerTerminateIntentIssuedCount","recordProviderRunnerActivationOutcome","pollDemand","params","waitSeconds","Math","min","RESERVATION_LONG_POLL_MAX_WAIT_SECONDS","deadlineMs","Date","now","max","totalCapacity","templates","reduce","total","template","availableSlots","interval","RESERVATION_POLL_INTERVAL_MS","lastSnapshot","result","stats","reservations","terminateRunnerInstanceIds","divergences","terminateIntents","signal","aborted","previousSnapshot","deadlinePassed","snapshot","transaction","tx","demand","workspaceId","provisionerId","maxReservations","ttlSeconds","activationGraceSeconds","RESERVATION_TTL_SECONDS","limit","terminateIntentLimit","newlyReservedCount","newlyReservedUnits","reservation","count","map","intent","providerRunnerId","shouldReturn","calculateRunnerInstanceCountDivergences","advertisedTemplates","backendCounts","releaseReservationGrants","recordPollDemandMetrics","remainingWaitMs","withJitter","undefined","error","nextBackoffInterval","reservationId","hasImmediateTerminateIntent","some","reason","activationTimeoutRetry","length","ms","RESERVATION_POLL_MAX_INTERVAL_MS","random","advertisedCounts","Map","addCount","countKey","templateKey","starting","running","state","keys","Set","sort","flatMap","key","advertised","get","backend","splitCountKey","direction","delta","abs","divergence","debug","attributes","PROVISIONED_RUNNER_COUNT_DIVERGENCE_TEMPLATE_KEY_LABEL_ENABLED","template_key","add","surface","outcome","counts","set","split","Error"],"mappings":"AAAA,SAAQA,cAAcC,KAAK,QAAO,uBAAuB;AACzD,SAAQC,MAAM,QAAO,8BAA8B;AACnD,SAAQC,MAAM,QAAO,aAAa;AAClC,SAAQC,EAAE,QAAO,YAAY;AAC7B,SAEEC,uBAAuB,EACvBC,sBAAsB,QAGjB,sBAAsB;AAC7B,SAEEC,0CAA0C,EAC1CC,oCAAoC,QAE/B,0BAA0B;AACjC,SACEC,kCAAkC,EAClCC,wCAAwC,EACxCC,qCAAqC,QAChC,uBAAuB;AAkC9B,OAAO,eAAeC,WAAWC,MAAwB;IACvD,MAAMC,cAAcC,KAAKC,GAAG,CAC1BH,OAAOC,WAAW,IAAIX,OAAOc,sCAAsC,EACnEd,OAAOc,sCAAsC;IAE/C,MAAMC,aAAaC,KAAKC,GAAG,KAAKL,KAAKM,GAAG,CAAC,GAAGP,eAAe;IAC3D,MAAMQ,gBAAgBT,OAAOU,SAAS,CAACC,MAAM,CAC3C,CAACC,OAAOC,WAAaD,QAAQC,SAASC,cAAc,EACpD;IAEF,IAAIC,WAAWzB,OAAO0B,4BAA4B;IAClD,IAAIC,eAAmC;QACrCC,QAAQ;YAACC,OAAO,EAAE;YAAEC,cAAc,EAAE;YAAEC,4BAA4B,EAAE;QAAA;QACpEC,aAAa,EAAE;QACfC,kBAAkB,EAAE;IACtB;IAEA,MAAO,KAAM;QACX,IAAIvB,OAAOwB,MAAM,CAACC,OAAO,EAAE,OAAOR,aAAaC,MAAM;QAErD,MAAMQ,mBAAmBT;QACzB,MAAMU,iBAAiBrB,KAAKC,GAAG,MAAMF;QACrC,MAAMuB,WAAW,MAAMrC,KAAKsC,WAAW,CAAC,OAAOC;YAC7C,MAAMC,SAAS,MAAMtC,uBAAuBqC,IAAI;gBAC9CE,aAAahC,OAAOgC,WAAW;gBAC/BC,eAAejC,OAAOiC,aAAa;gBACnCC,iBAAiBlC,OAAOkC,eAAe;gBACvCC,YAAYnC,OAAOmC,UAAU;gBAC7BC,wBAAwB9C,OAAO+C,uBAAuB;gBACtD3B,WAAWV,OAAOU,SAAS;YAC7B;YACA,MAAMa,mBAAmB,MAAM5B,qCAAqCmC,IAAI;gBACtEE,aAAahC,OAAOgC,WAAW;gBAC/BC,eAAejC,OAAOiC,aAAa;gBACnCK,OAAOtC,OAAOuC,oBAAoB;YACpC;YACA,MAAMC,qBAAqBT,OAAOU,kBAAkB,CAAC9B,MAAM,CACzD,CAACC,OAAO8B,cAAgB9B,QAAQ8B,YAAYC,KAAK,EACjD;YAEF,MAAMzB,SAA2B;gBAC/BC,OAAOY,OAAOZ,KAAK;gBACnBC,cAAcW,OAAOX,YAAY;gBACjC,GAAIoB,qBAAqB,IAAI;oBAACA;gBAAkB,IAAI,CAAC,CAAC;gBACtDnB,4BAA4BE,iBAAiBqB,GAAG,CAAC,CAACC,SAAWA,OAAOC,gBAAgB;YACtF;YAEA,IACE,CAACC,aACC7B,QACAlB,OAAOkC,eAAe,EACtBzB,eACAkB,gBACAJ,mBAEF;gBACA,OAAO;oBAACL;oBAAQK;oBAAkBD,aAAa,EAAE;gBAAA;YACnD;YAEA,OAAO;gBACLJ;gBACAK;gBACAD,aAAa0B,wCAAwC;oBACnDC,qBAAqBjD,OAAOU,SAAS;oBACrCwC,eAAe,MAAMxD,2CAA2CoC,IAAI;wBAClEE,aAAahC,OAAOgC,WAAW;wBAC/BC,eAAejC,OAAOiC,aAAa;oBACrC;gBACF;YACF;QACF;QACA,IAAIjC,OAAOwB,MAAM,CAACC,OAAO,EAAE;YACzB,MAAM0B,yBAAyBvB,SAASV,MAAM,CAACE,YAAY;YAC3D,OAAOM,iBAAiBR,MAAM;QAChC;QAEAD,eAAeW;QAEf,IACEmB,aACE9B,aAAaC,MAAM,EACnBlB,OAAOkC,eAAe,EACtBzB,eACAkB,gBACAV,aAAaM,gBAAgB,GAE/B;YACA6B,wBAAwBpD,QAAQiB;YAChC,OAAOA,aAAaC,MAAM;QAC5B;QAEA,MAAMmC,kBAAkBnD,KAAKM,GAAG,CAAC,GAAGH,aAAaC,KAAKC,GAAG;QACzD,IAAI;YACF,MAAMnB,MAAMc,KAAKC,GAAG,CAACmD,WAAWvC,WAAWsC,kBAAkBE,WAAW;gBACtE/B,QAAQxB,OAAOwB,MAAM;YACvB;QACF,EAAE,OAAOgC,OAAO;YACd,IAAIxD,OAAOwB,MAAM,CAACC,OAAO,EAAE,OAAOR,aAAaC,MAAM;YACrD,MAAMsC;QACR;QACAzC,WAAW0C,oBAAoB1C;IACjC;AACF;AAEA,OAAO,eAAeoC,yBAAyB/B,YAAgC;IAC7E,+EAA+E;IAC/E,4EAA4E;IAC5E,qDAAqD;IACrD,MAAM5B,wBAAwB4B,aAAawB,GAAG,CAAC,CAACF,cAAgBA,YAAYgB,aAAa;AAC3F;AAEA,OAAO,SAASX,aACd7B,MAAwB,EACxBgB,eAAuB,EACvBzB,aAAqB,EACrBkB,cAAuB,EACvBJ,gBAA2D;IAE3D,MAAMoC,8BACJpC,kBAAkBqC,KAChB,CAACf,SAAWA,OAAOgB,MAAM,KAAK,wBAAwB,CAAChB,OAAOiB,sBAAsB,KACjF5C,OAAOG,0BAA0B,CAAC0C,MAAM,GAAG;IAElD,OACE7B,oBAAoB,KACpBzB,kBAAkB,KAClBS,OAAOE,YAAY,CAAC2C,MAAM,GAAG,KAC7B,AAAC7C,CAAAA,OAAOsB,kBAAkB,IAAI,CAAA,IAAK,KACnCmB,+BACAhC;AAEJ;AAEA,OAAO,SAAS8B,oBAAoBO,EAAU;IAC5C,OAAO9D,KAAKC,GAAG,CAAC6D,KAAK,KAAK1E,OAAO2E,gCAAgC;AACnE;AAEA,OAAO,SAASX,WAAWU,EAAU;IACnC,OAAO9D,KAAKgE,MAAM,KAAKF;AACzB;AAEA,OAAO,SAAShB,wCAAwChD,MAGvD;IACC,MAAMmE,mBAAmB,IAAIC;IAC7B,MAAMlB,gBAAgB,IAAIkB;IAE1B,KAAK,MAAMvD,YAAYb,OAAOiD,mBAAmB,CAAE;QACjDoB,SAASF,kBAAkBG,SAASzD,SAAS0D,WAAW,EAAE,aAAa1D,SAAS2D,QAAQ;QACxFH,SAASF,kBAAkBG,SAASzD,SAAS0D,WAAW,EAAE,YAAY1D,SAAS4D,OAAO;IACxF;IACA,KAAK,MAAM9B,SAAS3C,OAAOkD,aAAa,CAAE;QACxCmB,SAASnB,eAAeoB,SAAS3B,MAAM4B,WAAW,EAAE5B,MAAM+B,KAAK,GAAG/B,MAAMA,KAAK;IAC/E;IAEA,MAAMgC,OAAO;WAAI,IAAIC,IAAI;eAAIT,iBAAiBQ,IAAI;eAAOzB,cAAcyB,IAAI;SAAG;KAAE,CAACE,IAAI;IACrF,OAAOF,KAAKG,OAAO,CAAC,CAACC;QACnB,MAAMC,aAAab,iBAAiBc,GAAG,CAACF,QAAQ;QAChD,MAAMG,UAAUhC,cAAc+B,GAAG,CAACF,QAAQ;QAC1C,IAAIC,eAAeE,SAAS,OAAO,EAAE;QAErC,MAAM,CAACX,aAAaG,MAAM,GAAGS,cAAcJ;QAC3C,OAAO;YACL;gBACER;gBACAG;gBACAU,WAAWF,UAAUF,aAAa,mBAAmB;gBACrDK,OAAOnF,KAAKoF,GAAG,CAACJ,UAAUF;YAC5B;SACD;IACH;AACF;AAEA,SAAS5B,wBAAwBpD,MAAwB,EAAE4B,QAA4B;IACrF,KAAK,MAAM2D,cAAc3D,SAASN,WAAW,CAAE;QAC7CjC,SAASmG,KAAK,CACZ;YACExD,aAAahC,OAAOgC,WAAW;YAC/BC,eAAejC,OAAOiC,aAAa;YACnCsC,aAAagB,WAAWhB,WAAW;YACnCG,OAAOa,WAAWb,KAAK;YACvBU,WAAWG,WAAWH,SAAS;YAC/BC,OAAOE,WAAWF,KAAK;QACzB,GACA;QAGF,MAAMI,aAAa;YACjBf,OAAOa,WAAWb,KAAK;YACvBU,WAAWG,WAAWH,SAAS;YAC/B,GAAI9F,OAAOoG,8DAA8D,GACrE;gBAACC,cAAcJ,WAAWhB,WAAW;YAAA,IACrC,CAAC,CAAC;QACR;QACA3E,mCAAmCgG,GAAG,CAACL,WAAWF,KAAK,EAAEI;IAC3D;IACA,KAAK,MAAM5C,UAAUjB,SAASL,gBAAgB,CAAE;QAC9C1B,yCAAyC+F,GAAG,CAAC,GAAG;YAC9CC,SAAS;YACThC,QAAQhB,OAAOgB,MAAM;QACvB;QACA,qEAAqE;QACrE,IAAIhB,OAAOgB,MAAM,KAAK,wBAAwB,CAAChB,OAAOiB,sBAAsB,EAAE;YAC5EhE,sCAAsC;gBAACgG,SAAS;YAAQ;QAC1D;IACF;AACF;AAEA,SAASzB,SAAS0B,MAA2B,EAAEhB,GAAW,EAAEpC,KAAa;IACvEoD,OAAOC,GAAG,CAACjB,KAAK,AAACgB,CAAAA,OAAOd,GAAG,CAACF,QAAQ,CAAA,IAAKpC;AAC3C;AAEA,SAAS2B,SAASC,WAAmB,EAAEG,KAA6B;IAClE,OAAO,GAAGH,YAAY,EAAE,EAAEG,OAAO;AACnC;AAEA,SAASS,cAAcJ,GAAW;IAChC,MAAM,CAACR,aAAaG,MAAM,GAAGK,IAAIkB,KAAK,CAAC;IACvC,IAAI,CAAC1B,eAAgBG,UAAU,cAAcA,UAAU,WAAY;QACjE,MAAM,IAAIwB,MAAM,CAAC,sCAAsC,EAAEnB,KAAK;IAChE;IACA,OAAO;QAACR;QAAaG;KAAM;AAC7B"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"reservation.d.ts","sourceRoot":"","sources":["../../../src/core/entities/reservation.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,IAAI,CAAC;IAChB,SAAS,EAAE,IAAI,CAAC;CACjB"}
|
|
1
|
+
{"version":3,"file":"reservation.d.ts","sourceRoot":"","sources":["../../../src/core/entities/reservation.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,OAAO,GAAG,QAAQ,CAAC;IACzB,SAAS,EAAE,IAAI,CAAC;IAChB,SAAS,EAAE,IAAI,CAAC;CACjB"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../src/core/entities/reservation.ts"],"sourcesContent":["export interface Reservation {\n id: string;\n workspaceId: string;\n provisionerId: string;\n requiredLabels: string[];\n count: number;\n createdAt: Date;\n expiresAt: Date;\n}\n"],"names":[],"mappings":"AAAA,
|
|
1
|
+
{"version":3,"sources":["../../../src/core/entities/reservation.ts"],"sourcesContent":["export interface Reservation {\n id: string;\n workspaceId: string;\n provisionerId: string;\n requiredLabels: string[];\n count: number;\n kind: 'bound' | 'launch';\n createdAt: Date;\n expiresAt: Date;\n}\n"],"names":[],"mappings":"AAAA,WASC"}
|
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
export type RunnerInstanceState = 'starting' | 'running' | 'stopping' | 'stopped' | 'failed' | 'terminated';
|
|
2
|
+
export type RunnerInstanceLaunchKind = 'demand' | 'warm' | 'manual';
|
|
2
3
|
export interface RunnerInstance {
|
|
3
4
|
id: string;
|
|
4
5
|
workspaceId: string | null;
|
|
5
6
|
provisionerId: string;
|
|
6
7
|
providerRunnerId: string;
|
|
8
|
+
intendedReservationId: string | null;
|
|
7
9
|
reservationId: string | null;
|
|
10
|
+
launchKind: RunnerInstanceLaunchKind;
|
|
8
11
|
assignedAt?: Date | null;
|
|
9
12
|
templateKey: string | null;
|
|
10
13
|
labels: string[];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runner-instance.d.ts","sourceRoot":"","sources":["../../../src/core/entities/runner-instance.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,mBAAmB,GAC3B,UAAU,GACV,SAAS,GACT,UAAU,GACV,SAAS,GACT,QAAQ,GACR,YAAY,CAAC;AAEjB,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,UAAU,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC;IACzB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,KAAK,EAAE,mBAAmB,CAAC;IAC3B,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,UAAU,EAAE,IAAI,CAAC;IACjB,SAAS,EAAE,IAAI,GAAG,IAAI,CAAC;IACvB,UAAU,EAAE,IAAI,GAAG,IAAI,CAAC;IACxB,SAAS,EAAE,IAAI,GAAG,IAAI,CAAC;IACvB,QAAQ,EAAE,IAAI,GAAG,IAAI,CAAC;IACtB,YAAY,EAAE,IAAI,GAAG,IAAI,CAAC;IAC1B,qBAAqB,EAAE,IAAI,GAAG,IAAI,CAAC;IACnC,SAAS,EAAE,IAAI,CAAC;IAChB,SAAS,EAAE,IAAI,CAAC;CACjB"}
|
|
1
|
+
{"version":3,"file":"runner-instance.d.ts","sourceRoot":"","sources":["../../../src/core/entities/runner-instance.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,mBAAmB,GAC3B,UAAU,GACV,SAAS,GACT,UAAU,GACV,SAAS,GACT,QAAQ,GACR,YAAY,CAAC;AAEjB,MAAM,MAAM,wBAAwB,GAAG,QAAQ,GAAG,MAAM,GAAG,QAAQ,CAAC;AAEpE,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,UAAU,EAAE,wBAAwB,CAAC;IACrC,UAAU,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC;IACzB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,KAAK,EAAE,mBAAmB,CAAC;IAC3B,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,UAAU,EAAE,IAAI,CAAC;IACjB,SAAS,EAAE,IAAI,GAAG,IAAI,CAAC;IACvB,UAAU,EAAE,IAAI,GAAG,IAAI,CAAC;IACxB,SAAS,EAAE,IAAI,GAAG,IAAI,CAAC;IACvB,QAAQ,EAAE,IAAI,GAAG,IAAI,CAAC;IACtB,YAAY,EAAE,IAAI,GAAG,IAAI,CAAC;IAC1B,qBAAqB,EAAE,IAAI,GAAG,IAAI,CAAC;IACnC,SAAS,EAAE,IAAI,CAAC;IAChB,SAAS,EAAE,IAAI,CAAC;CACjB"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../src/core/entities/runner-instance.ts"],"sourcesContent":["export type RunnerInstanceState =\n | 'starting'\n | 'running'\n | 'stopping'\n | 'stopped'\n | 'failed'\n | 'terminated';\n\nexport interface RunnerInstance {\n id: string;\n workspaceId: string | null;\n provisionerId: string;\n providerRunnerId: string;\n reservationId: string | null;\n assignedAt?: Date | null;\n templateKey: string | null;\n labels: string[];\n state: RunnerInstanceState;\n reason: string | null;\n runnerSessionId: string | null;\n providerKind: string | null;\n reportedAt: Date;\n startedAt: Date | null;\n stoppingAt: Date | null;\n stoppedAt: Date | null;\n failedAt: Date | null;\n terminatedAt: Date | null;\n reservationReleasedAt: Date | null;\n createdAt: Date;\n updatedAt: Date;\n}\n"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"sources":["../../../src/core/entities/runner-instance.ts"],"sourcesContent":["export type RunnerInstanceState =\n | 'starting'\n | 'running'\n | 'stopping'\n | 'stopped'\n | 'failed'\n | 'terminated';\n\nexport type RunnerInstanceLaunchKind = 'demand' | 'warm' | 'manual';\n\nexport interface RunnerInstance {\n id: string;\n workspaceId: string | null;\n provisionerId: string;\n providerRunnerId: string;\n intendedReservationId: string | null;\n reservationId: string | null;\n launchKind: RunnerInstanceLaunchKind;\n assignedAt?: Date | null;\n templateKey: string | null;\n labels: string[];\n state: RunnerInstanceState;\n reason: string | null;\n runnerSessionId: string | null;\n providerKind: string | null;\n reportedAt: Date;\n startedAt: Date | null;\n stoppingAt: Date | null;\n stoppedAt: Date | null;\n failedAt: Date | null;\n terminatedAt: Date | null;\n reservationReleasedAt: Date | null;\n createdAt: Date;\n updatedAt: Date;\n}\n"],"names":[],"mappings":"AAUA,WAwBC"}
|
|
@@ -1,19 +1,22 @@
|
|
|
1
1
|
import type { Tx } from '#db/db.js';
|
|
2
|
+
import { type RunnerActivationTokenNotIssuedSurface } from '#metrics/instance.js';
|
|
2
3
|
export declare function issueRunnerActivationToken(params: {
|
|
3
4
|
runnerInstanceId: string;
|
|
4
5
|
provisionerId: string;
|
|
5
6
|
ttlSeconds: number;
|
|
7
|
+
surface: RunnerActivationTokenNotIssuedSurface;
|
|
6
8
|
}): Promise<string | null>;
|
|
7
9
|
export declare function issueRunnerActivationTokenTx(tx: Tx, params: {
|
|
8
10
|
runnerInstanceId: string;
|
|
9
11
|
provisionerId: string;
|
|
10
12
|
ttlSeconds: number;
|
|
13
|
+
surface: RunnerActivationTokenNotIssuedSurface;
|
|
11
14
|
}): Promise<string | null>;
|
|
12
15
|
export declare function getRunnerAssignment(params: {
|
|
13
16
|
runnerInstanceId: string;
|
|
14
17
|
provisionerId: string;
|
|
15
18
|
}): Promise<{
|
|
16
|
-
workspaceId: string
|
|
19
|
+
workspaceId: string;
|
|
17
20
|
runnerSessionId: string | null;
|
|
18
21
|
} | null>;
|
|
19
22
|
//# sourceMappingURL=runner-activation.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runner-activation.d.ts","sourceRoot":"","sources":["../../src/core/runner-activation.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"runner-activation.d.ts","sourceRoot":"","sources":["../../src/core/runner-activation.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAC,EAAE,EAAC,MAAM,WAAW,CAAC;AAKlC,OAAO,EAEL,KAAK,qCAAqC,EAE3C,MAAM,sBAAsB,CAAC;AAE9B,wBAAsB,0BAA0B,CAAC,MAAM,EAAE;IACvD,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,qCAAqC,CAAC;CAChD,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAEzB;AAED,wBAAsB,4BAA4B,CAChD,EAAE,EAAE,EAAE,EACN,MAAM,EAAE;IACN,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,qCAAqC,CAAC;CAChD,GACA,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CA2CxB;AAED,wBAAsB,mBAAmB,CAAC,MAAM,EAAE;IAChD,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;CACvB;;;UAkBA"}
|
|
@@ -1,8 +1,10 @@
|
|
|
1
|
+
import { logger } from '@shipfox/node-opentelemetry';
|
|
1
2
|
import { extractDisplayPrefix, generateOpaqueToken, hashOpaqueToken } from '@shipfox/node-tokens';
|
|
2
3
|
import { and, eq, isNull, sql } from 'drizzle-orm';
|
|
3
4
|
import { db } from '#db/db.js';
|
|
4
5
|
import { runnerActivationTokens } from '#db/schema/runner-activation-tokens.js';
|
|
5
6
|
import { providerRunners } from '#db/schema/runner-instances.js';
|
|
7
|
+
import { recordRunnerActivationTokenNotIssued } from '#metrics/instance.js';
|
|
6
8
|
export async function issueRunnerActivationToken(params) {
|
|
7
9
|
return await db().transaction(async (tx)=>issueRunnerActivationTokenTx(tx, params));
|
|
8
10
|
}
|
|
@@ -13,7 +15,11 @@ export async function issueRunnerActivationTokenTx(tx, params) {
|
|
|
13
15
|
runnerSessionId: providerRunners.runnerSessionId,
|
|
14
16
|
state: providerRunners.state
|
|
15
17
|
}).from(providerRunners).where(and(eq(providerRunners.id, params.runnerInstanceId), eq(providerRunners.provisionerId, params.provisionerId))).limit(1).for('update');
|
|
16
|
-
|
|
18
|
+
const notIssuedReason = getRunnerActivationTokenNotIssuedReason(runner);
|
|
19
|
+
if (notIssuedReason) {
|
|
20
|
+
recordRunnerActivationTokenNotIssuedWithLog(notIssuedReason, params);
|
|
21
|
+
return null;
|
|
22
|
+
}
|
|
17
23
|
await tx.update(runnerActivationTokens).set({
|
|
18
24
|
revokedAt: sql`now()`
|
|
19
25
|
}).where(and(eq(runnerActivationTokens.runnerInstanceId, params.runnerInstanceId), isNull(runnerActivationTokens.consumedAt), isNull(runnerActivationTokens.revokedAt)));
|
|
@@ -29,9 +35,32 @@ export async function issueRunnerActivationTokenTx(tx, params) {
|
|
|
29
35
|
export async function getRunnerAssignment(params) {
|
|
30
36
|
const [runner] = await db().select({
|
|
31
37
|
workspaceId: providerRunners.workspaceId,
|
|
32
|
-
runnerSessionId: providerRunners.runnerSessionId
|
|
38
|
+
runnerSessionId: providerRunners.runnerSessionId,
|
|
39
|
+
state: providerRunners.state
|
|
33
40
|
}).from(providerRunners).where(and(eq(providerRunners.id, params.runnerInstanceId), eq(providerRunners.provisionerId, params.provisionerId))).limit(1);
|
|
34
|
-
return runner?.workspaceId && !runner.runnerSessionId
|
|
41
|
+
return runner?.workspaceId && !runner.runnerSessionId && runner.state === 'running' ? {
|
|
42
|
+
workspaceId: runner.workspaceId,
|
|
43
|
+
runnerSessionId: runner.runnerSessionId
|
|
44
|
+
} : null;
|
|
45
|
+
}
|
|
46
|
+
function getRunnerActivationTokenNotIssuedReason(runner) {
|
|
47
|
+
if (!runner) return 'runner-not-found';
|
|
48
|
+
if (!runner.workspaceId) return 'missing-workspace';
|
|
49
|
+
if (runner.runnerSessionId) return 'existing-session';
|
|
50
|
+
if (runner.state !== 'running') return 'not-running';
|
|
51
|
+
return null;
|
|
52
|
+
}
|
|
53
|
+
function recordRunnerActivationTokenNotIssuedWithLog(reason, params) {
|
|
54
|
+
recordRunnerActivationTokenNotIssued({
|
|
55
|
+
reason,
|
|
56
|
+
surface: params.surface
|
|
57
|
+
});
|
|
58
|
+
logger().debug({
|
|
59
|
+
runnerInstanceId: params.runnerInstanceId,
|
|
60
|
+
provisionerId: params.provisionerId,
|
|
61
|
+
reason,
|
|
62
|
+
surface: params.surface
|
|
63
|
+
}, 'Runner activation token was not issued');
|
|
35
64
|
}
|
|
36
65
|
|
|
37
66
|
//# sourceMappingURL=runner-activation.js.map
|