@shipfox/api-runners 5.0.0 → 6.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +6 -6
- package/CHANGELOG.md +61 -0
- package/dist/config.d.ts +5 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +53 -6
- package/dist/config.js.map +1 -1
- package/dist/core/capacity-assignments.d.ts +7 -0
- package/dist/core/capacity-assignments.d.ts.map +1 -0
- package/dist/core/capacity-assignments.js +6 -0
- package/dist/core/capacity-assignments.js.map +1 -0
- package/dist/core/demand.d.ts +6 -6
- package/dist/core/demand.d.ts.map +1 -1
- package/dist/core/demand.js +10 -10
- package/dist/core/demand.js.map +1 -1
- package/dist/core/entities/capacity-assignment.d.ts +9 -0
- package/dist/core/entities/capacity-assignment.d.ts.map +1 -0
- package/dist/core/entities/capacity-assignment.js +3 -0
- package/dist/core/entities/capacity-assignment.js.map +1 -0
- package/dist/core/entities/ephemeral-registration-token.d.ts +1 -1
- package/dist/core/entities/ephemeral-registration-token.d.ts.map +1 -1
- package/dist/core/entities/ephemeral-registration-token.js.map +1 -1
- package/dist/core/entities/provisioner-capability-snapshot.d.ts +13 -0
- package/dist/core/entities/provisioner-capability-snapshot.d.ts.map +1 -0
- package/dist/core/entities/provisioner-capability-snapshot.js +3 -0
- package/dist/core/entities/provisioner-capability-snapshot.js.map +1 -0
- package/dist/core/entities/provisioner-token.d.ts +10 -2
- package/dist/core/entities/provisioner-token.d.ts.map +1 -1
- package/dist/core/entities/provisioner-token.js.map +1 -1
- package/dist/core/entities/{provisioned-runner.d.ts → runner-instance.d.ts} +6 -6
- package/dist/core/entities/runner-instance.d.ts.map +1 -0
- package/dist/core/entities/runner-instance.js +3 -0
- package/dist/core/entities/runner-instance.js.map +1 -0
- package/dist/core/entities/runner-session.d.ts +4 -2
- package/dist/core/entities/runner-session.d.ts.map +1 -1
- package/dist/core/entities/runner-session.js.map +1 -1
- package/dist/core/ephemeral-registration-tokens.d.ts +5 -5
- package/dist/core/ephemeral-registration-tokens.d.ts.map +1 -1
- package/dist/core/ephemeral-registration-tokens.js +12 -12
- package/dist/core/ephemeral-registration-tokens.js.map +1 -1
- package/dist/core/errors.d.ts +16 -4
- package/dist/core/errors.d.ts.map +1 -1
- package/dist/core/errors.js +22 -4
- package/dist/core/errors.js.map +1 -1
- package/dist/core/index.d.ts +10 -4
- package/dist/core/index.d.ts.map +1 -1
- package/dist/core/index.js +6 -2
- package/dist/core/index.js.map +1 -1
- package/dist/core/job-executions.d.ts +2 -0
- package/dist/core/job-executions.d.ts.map +1 -1
- package/dist/core/job-executions.js +1 -2
- package/dist/core/job-executions.js.map +1 -1
- package/dist/core/maintenance.d.ts +1 -1
- package/dist/core/maintenance.d.ts.map +1 -1
- package/dist/core/maintenance.js +5 -5
- package/dist/core/maintenance.js.map +1 -1
- package/dist/core/provisioner-capability-snapshots.d.ts +5 -0
- package/dist/core/provisioner-capability-snapshots.d.ts.map +1 -0
- package/dist/core/provisioner-capability-snapshots.js +10 -0
- package/dist/core/provisioner-capability-snapshots.js.map +1 -0
- package/dist/core/provisioner-tokens.d.ts +10 -0
- package/dist/core/provisioner-tokens.d.ts.map +1 -1
- package/dist/core/provisioner-tokens.js +23 -1
- package/dist/core/provisioner-tokens.js.map +1 -1
- package/dist/core/rate-limit.d.ts.map +1 -1
- package/dist/core/rate-limit.js +2 -7
- package/dist/core/rate-limit.js.map +1 -1
- package/dist/core/runner-activation.d.ts +13 -0
- package/dist/core/runner-activation.d.ts.map +1 -0
- package/dist/core/runner-activation.js +36 -0
- package/dist/core/runner-activation.js.map +1 -0
- package/dist/core/runner-control-sessions.d.ts +46 -0
- package/dist/core/runner-control-sessions.d.ts.map +1 -0
- package/dist/core/runner-control-sessions.js +121 -0
- package/dist/core/runner-control-sessions.js.map +1 -0
- package/dist/core/runner-instances.d.ts +79 -0
- package/dist/core/runner-instances.d.ts.map +1 -0
- package/dist/core/runner-instances.js +168 -0
- package/dist/core/runner-instances.js.map +1 -0
- package/dist/core/runner-sessions.d.ts +8 -2
- package/dist/core/runner-sessions.d.ts.map +1 -1
- package/dist/core/runner-sessions.js +8 -5
- package/dist/core/runner-sessions.js.map +1 -1
- package/dist/db/capacity-assignments.d.ts +7 -0
- package/dist/db/capacity-assignments.d.ts.map +1 -0
- package/dist/db/capacity-assignments.js +72 -0
- package/dist/db/capacity-assignments.js.map +1 -0
- package/dist/db/db.d.ts +2837 -1009
- package/dist/db/db.d.ts.map +1 -1
- package/dist/db/db.js +11 -2
- package/dist/db/db.js.map +1 -1
- package/dist/db/ephemeral-registration-tokens.d.ts +2 -2
- package/dist/db/ephemeral-registration-tokens.d.ts.map +1 -1
- package/dist/db/ephemeral-registration-tokens.js +21 -21
- package/dist/db/ephemeral-registration-tokens.js.map +1 -1
- package/dist/db/index.d.ts +7 -4
- package/dist/db/index.d.ts.map +1 -1
- package/dist/db/index.js +5 -2
- package/dist/db/index.js.map +1 -1
- package/dist/db/job-executions.d.ts +6 -6
- package/dist/db/job-executions.d.ts.map +1 -1
- package/dist/db/job-executions.js +20 -19
- package/dist/db/job-executions.js.map +1 -1
- package/dist/db/provisioner-capability-snapshots.d.ts +21 -0
- package/dist/db/provisioner-capability-snapshots.d.ts.map +1 -0
- package/dist/db/provisioner-capability-snapshots.js +57 -0
- package/dist/db/provisioner-capability-snapshots.js.map +1 -0
- package/dist/db/provisioner-tokens.d.ts +7 -2
- package/dist/db/provisioner-tokens.d.ts.map +1 -1
- package/dist/db/provisioner-tokens.js +59 -10
- package/dist/db/provisioner-tokens.js.map +1 -1
- package/dist/db/reservations.d.ts +18 -0
- package/dist/db/reservations.d.ts.map +1 -1
- package/dist/db/reservations.js +88 -1
- package/dist/db/reservations.js.map +1 -1
- package/dist/db/runner-instances.d.ts +91 -0
- package/dist/db/runner-instances.d.ts.map +1 -0
- package/dist/db/runner-instances.js +511 -0
- package/dist/db/runner-instances.js.map +1 -0
- package/dist/db/runner-sessions.d.ts +5 -0
- package/dist/db/runner-sessions.d.ts.map +1 -1
- package/dist/db/runner-sessions.js +48 -2
- package/dist/db/runner-sessions.js.map +1 -1
- package/dist/db/schema/capacity-assignments.d.ts +113 -0
- package/dist/db/schema/capacity-assignments.d.ts.map +1 -0
- package/dist/db/schema/capacity-assignments.js +29 -0
- package/dist/db/schema/capacity-assignments.js.map +1 -0
- package/dist/db/schema/ephemeral-registration-tokens.d.ts +2 -2
- package/dist/db/schema/ephemeral-registration-tokens.js +3 -3
- package/dist/db/schema/ephemeral-registration-tokens.js.map +1 -1
- package/dist/db/schema/provisioner-capability-snapshots.d.ts +207 -0
- package/dist/db/schema/provisioner-capability-snapshots.d.ts.map +1 -0
- package/dist/db/schema/provisioner-capability-snapshots.js +38 -0
- package/dist/db/schema/provisioner-capability-snapshots.js.map +1 -0
- package/dist/db/schema/provisioner-tokens.d.ts +19 -1
- package/dist/db/schema/provisioner-tokens.d.ts.map +1 -1
- package/dist/db/schema/provisioner-tokens.js +29 -3
- package/dist/db/schema/provisioner-tokens.js.map +1 -1
- package/dist/db/schema/runner-activation-tokens.d.ts +161 -0
- package/dist/db/schema/runner-activation-tokens.d.ts.map +1 -0
- package/dist/db/schema/runner-activation-tokens.js +27 -0
- package/dist/db/schema/runner-activation-tokens.js.map +1 -0
- package/dist/db/schema/runner-control-sessions.d.ts +338 -0
- package/dist/db/schema/runner-control-sessions.d.ts.map +1 -0
- package/dist/db/schema/runner-control-sessions.js +52 -0
- package/dist/db/schema/runner-control-sessions.js.map +1 -0
- package/dist/db/schema/{provisioned-runners.d.ts → runner-instances.d.ts} +88 -34
- package/dist/db/schema/runner-instances.d.ts.map +1 -0
- package/dist/db/schema/{provisioned-runners.js → runner-instances.js} +19 -14
- package/dist/db/schema/runner-instances.js.map +1 -0
- package/dist/db/schema/runner-sessions.d.ts +39 -5
- package/dist/db/schema/runner-sessions.d.ts.map +1 -1
- package/dist/db/schema/runner-sessions.js +13 -6
- package/dist/db/schema/runner-sessions.js.map +1 -1
- package/dist/db/schema/running-job-executions.d.ts +2 -2
- package/dist/db/schema/running-job-executions.js +4 -4
- package/dist/db/schema/running-job-executions.js.map +1 -1
- package/dist/index.d.ts +7 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +45 -38
- package/dist/index.js.map +1 -1
- package/dist/installation-provisioning.d.ts +14 -0
- package/dist/installation-provisioning.d.ts.map +1 -0
- package/dist/installation-provisioning.js +7 -0
- package/dist/installation-provisioning.js.map +1 -0
- package/dist/metrics/index.d.ts +1 -1
- package/dist/metrics/index.d.ts.map +1 -1
- package/dist/metrics/index.js +1 -1
- package/dist/metrics/index.js.map +1 -1
- package/dist/metrics/instance.d.ts +11 -7
- package/dist/metrics/instance.d.ts.map +1 -1
- package/dist/metrics/instance.js +13 -7
- package/dist/metrics/instance.js.map +1 -1
- package/dist/presentation/auth/index.d.ts +1 -0
- package/dist/presentation/auth/index.d.ts.map +1 -1
- package/dist/presentation/auth/index.js +1 -0
- package/dist/presentation/auth/index.js.map +1 -1
- package/dist/presentation/auth/provisioner-token-auth.d.ts.map +1 -1
- package/dist/presentation/auth/provisioner-token-auth.js +5 -1
- package/dist/presentation/auth/provisioner-token-auth.js.map +1 -1
- package/dist/presentation/auth/runner-control-session-auth.d.ts +4 -0
- package/dist/presentation/auth/runner-control-session-auth.d.ts.map +1 -0
- package/dist/presentation/auth/runner-control-session-auth.js +29 -0
- package/dist/presentation/auth/runner-control-session-auth.js.map +1 -0
- package/dist/presentation/auth/runner-registration-token-auth.d.ts +5 -1
- package/dist/presentation/auth/runner-registration-token-auth.d.ts.map +1 -1
- package/dist/presentation/auth/runner-registration-token-auth.js +21 -1
- package/dist/presentation/auth/runner-registration-token-auth.js.map +1 -1
- package/dist/presentation/dto/index.d.ts +2 -2
- package/dist/presentation/dto/index.d.ts.map +1 -1
- package/dist/presentation/dto/index.js +2 -2
- package/dist/presentation/dto/index.js.map +1 -1
- package/dist/presentation/dto/mint-registration-tokens.d.ts +5 -3
- package/dist/presentation/dto/mint-registration-tokens.d.ts.map +1 -1
- package/dist/presentation/dto/mint-registration-tokens.js +4 -4
- package/dist/presentation/dto/mint-registration-tokens.js.map +1 -1
- package/dist/presentation/dto/poll-demand.d.ts.map +1 -1
- package/dist/presentation/dto/poll-demand.js +7 -1
- package/dist/presentation/dto/poll-demand.js.map +1 -1
- package/dist/presentation/dto/provisioner-token.d.ts +1 -0
- package/dist/presentation/dto/provisioner-token.d.ts.map +1 -1
- package/dist/presentation/dto/provisioner-token.js +5 -1
- package/dist/presentation/dto/provisioner-token.js.map +1 -1
- package/dist/presentation/dto/{provisioned-runners.d.ts → runner-instances.d.ts} +9 -9
- package/dist/presentation/dto/runner-instances.d.ts.map +1 -0
- package/dist/presentation/dto/{provisioned-runners.js → runner-instances.js} +6 -6
- package/dist/presentation/dto/runner-instances.js.map +1 -0
- package/dist/presentation/index.d.ts +2 -2
- package/dist/presentation/index.d.ts.map +1 -1
- package/dist/presentation/index.js +2 -2
- package/dist/presentation/index.js.map +1 -1
- package/dist/presentation/inter-module.d.ts +5 -0
- package/dist/presentation/inter-module.d.ts.map +1 -0
- package/dist/presentation/inter-module.js +43 -0
- package/dist/presentation/inter-module.js.map +1 -0
- package/dist/presentation/routes/assign-capacity.d.ts +2 -0
- package/dist/presentation/routes/assign-capacity.d.ts.map +1 -0
- package/dist/presentation/routes/assign-capacity.js +44 -0
- package/dist/presentation/routes/assign-capacity.js.map +1 -0
- package/dist/presentation/routes/get-provisioner-me.d.ts.map +1 -1
- package/dist/presentation/routes/get-provisioner-me.js +5 -1
- package/dist/presentation/routes/get-provisioner-me.js.map +1 -1
- package/dist/presentation/routes/heartbeat.d.ts +2 -1
- package/dist/presentation/routes/heartbeat.d.ts.map +1 -1
- package/dist/presentation/routes/heartbeat.js +60 -50
- package/dist/presentation/routes/heartbeat.js.map +1 -1
- package/dist/presentation/routes/index.d.ts +3 -1
- package/dist/presentation/routes/index.d.ts.map +1 -1
- package/dist/presentation/routes/index.js +83 -50
- package/dist/presentation/routes/index.js.map +1 -1
- package/dist/presentation/routes/mint-registration-tokens.d.ts.map +1 -1
- package/dist/presentation/routes/mint-registration-tokens.js +5 -5
- package/dist/presentation/routes/mint-registration-tokens.js.map +1 -1
- package/dist/presentation/routes/planned-capacity.d.ts +3 -0
- package/dist/presentation/routes/planned-capacity.d.ts.map +1 -0
- package/dist/presentation/routes/planned-capacity.js +54 -0
- package/dist/presentation/routes/planned-capacity.js.map +1 -0
- package/dist/presentation/routes/poll-demand.d.ts +2 -0
- package/dist/presentation/routes/poll-demand.d.ts.map +1 -1
- package/dist/presentation/routes/poll-demand.js +86 -45
- package/dist/presentation/routes/poll-demand.js.map +1 -1
- package/dist/presentation/routes/reconcile-runner-instances.d.ts +2 -0
- package/dist/presentation/routes/reconcile-runner-instances.d.ts.map +1 -0
- package/dist/presentation/routes/reconcile-runner-instances.js +27 -0
- package/dist/presentation/routes/reconcile-runner-instances.js.map +1 -0
- package/dist/presentation/routes/register.d.ts +2 -1
- package/dist/presentation/routes/register.d.ts.map +1 -1
- package/dist/presentation/routes/register.js +55 -52
- package/dist/presentation/routes/register.js.map +1 -1
- package/dist/presentation/routes/report-runner-instances.d.ts +2 -0
- package/dist/presentation/routes/report-runner-instances.d.ts.map +1 -0
- package/dist/presentation/routes/report-runner-instances.js +37 -0
- package/dist/presentation/routes/report-runner-instances.js.map +1 -0
- package/dist/presentation/routes/request-job.d.ts +2 -1
- package/dist/presentation/routes/request-job.d.ts.map +1 -1
- package/dist/presentation/routes/request-job.js +39 -36
- package/dist/presentation/routes/request-job.js.map +1 -1
- package/dist/presentation/routes/runner-enrollment.d.ts +7 -0
- package/dist/presentation/routes/runner-enrollment.d.ts.map +1 -0
- package/dist/presentation/routes/runner-enrollment.js +185 -0
- package/dist/presentation/routes/runner-enrollment.js.map +1 -0
- package/dist/temporal/activities/index.d.ts +2 -2
- package/dist/temporal/activities/index.d.ts.map +1 -1
- package/dist/temporal/activities/index.js +2 -2
- package/dist/temporal/activities/index.js.map +1 -1
- package/dist/temporal/activities/maintenance-activities.d.ts +1 -1
- package/dist/temporal/activities/maintenance-activities.d.ts.map +1 -1
- package/dist/temporal/activities/maintenance-activities.js +3 -3
- package/dist/temporal/activities/maintenance-activities.js.map +1 -1
- package/dist/temporal/workflows/index.bundle.js +3 -3
- package/dist/temporal/workflows/stuck-job-detector.js +2 -2
- package/dist/temporal/workflows/stuck-job-detector.js.map +1 -1
- package/dist/tsconfig.test.tsbuildinfo +1 -1
- package/drizzle/0000_initial.sql +102 -24
- package/drizzle/meta/0000_snapshot.json +483 -63
- package/package.json +21 -16
- package/src/config.test.ts +0 -26
- package/src/config.ts +47 -6
- package/src/core/capacity-assignments.ts +10 -0
- package/src/core/demand.test.ts +29 -29
- package/src/core/demand.ts +22 -22
- package/src/core/entities/capacity-assignment.ts +8 -0
- package/src/core/entities/ephemeral-registration-token.ts +1 -1
- package/src/core/entities/provisioner-capability-snapshot.ts +12 -0
- package/src/core/entities/provisioner-token.ts +8 -2
- package/src/core/entities/{provisioned-runner.ts → runner-instance.ts} +5 -5
- package/src/core/entities/runner-session.ts +4 -2
- package/src/core/ephemeral-registration-tokens.test.ts +11 -11
- package/src/core/ephemeral-registration-tokens.ts +18 -20
- package/src/core/errors.ts +23 -4
- package/src/core/index.ts +39 -17
- package/src/core/job-executions.ts +3 -2
- package/src/core/maintenance.test.ts +13 -17
- package/src/core/maintenance.ts +5 -5
- package/src/core/provisioner-capability-snapshots.test.ts +35 -0
- package/src/core/provisioner-capability-snapshots.ts +12 -0
- package/src/core/provisioner-tokens.test.ts +20 -0
- package/src/core/provisioner-tokens.ts +33 -0
- package/src/core/rate-limit.test.ts +11 -27
- package/src/core/rate-limit.ts +3 -11
- package/src/core/runner-activation.ts +72 -0
- package/src/core/runner-control-sessions.ts +190 -0
- package/src/core/{provisioned-runners.test.ts → runner-instances.test.ts} +24 -24
- package/src/core/runner-instances.ts +310 -0
- package/src/core/runner-sessions.test.ts +112 -22
- package/src/core/runner-sessions.ts +25 -14
- package/src/db/capacity-assignments.test.ts +159 -0
- package/src/db/capacity-assignments.ts +116 -0
- package/src/db/db.ts +11 -2
- package/src/db/ephemeral-registration-tokens.test.ts +9 -9
- package/src/db/ephemeral-registration-tokens.ts +26 -26
- package/src/db/index.ts +31 -21
- package/src/db/job-executions.test.ts +13 -11
- package/src/db/job-executions.ts +33 -27
- package/src/db/provisioner-capability-snapshots.test.ts +175 -0
- package/src/db/provisioner-capability-snapshots.ts +130 -0
- package/src/db/provisioner-tokens.test.ts +139 -2
- package/src/db/provisioner-tokens.ts +123 -15
- package/src/db/reservations.test.ts +49 -0
- package/src/db/reservations.ts +146 -1
- package/src/db/{provisioned-runners.test.ts → runner-instances.test.ts} +429 -353
- package/src/db/runner-instances.ts +942 -0
- package/src/db/runner-sessions.test.ts +19 -6
- package/src/db/runner-sessions.ts +78 -2
- package/src/db/schema/capacity-assignments.ts +34 -0
- package/src/db/schema/ephemeral-registration-tokens.ts +4 -4
- package/src/db/schema/provisioner-capability-snapshots.ts +46 -0
- package/src/db/schema/provisioner-tokens.ts +33 -2
- package/src/db/schema/runner-activation-tokens.ts +22 -0
- package/src/db/schema/runner-control-sessions.ts +46 -0
- package/src/db/schema/{provisioned-runners.ts → runner-instances.ts} +26 -27
- package/src/db/schema/runner-sessions.ts +11 -7
- package/src/db/schema/running-job-executions.ts +5 -5
- package/src/index.test.ts +26 -0
- package/src/index.ts +39 -19
- package/src/installation-provisioning.ts +14 -0
- package/src/metrics/index.ts +5 -5
- package/src/metrics/instance.ts +25 -14
- package/src/presentation/auth/index.ts +4 -0
- package/src/presentation/auth/provisioner-token-auth.ts +8 -4
- package/src/presentation/auth/runner-control-session-auth.ts +31 -0
- package/src/presentation/auth/runner-registration-token-auth.ts +39 -3
- package/src/presentation/dto/index.ts +5 -5
- package/src/presentation/dto/mint-registration-tokens.ts +8 -15
- package/src/presentation/dto/poll-demand.ts +3 -1
- package/src/presentation/dto/provisioner-token.ts +5 -0
- package/src/presentation/dto/{provisioned-runners.ts → runner-instances.ts} +13 -15
- package/src/presentation/index.ts +2 -1
- package/src/presentation/inter-module.test.ts +15 -0
- package/src/presentation/inter-module.ts +53 -0
- package/src/presentation/routes/assign-capacity.ts +48 -0
- package/src/presentation/routes/get-provisioner-me.ts +3 -4
- package/src/presentation/routes/heartbeat.test.ts +4 -3
- package/src/presentation/routes/heartbeat.ts +67 -64
- package/src/presentation/routes/index.ts +93 -43
- package/src/presentation/routes/list-active-runners.test.ts +23 -23
- package/src/presentation/routes/manual-registration-tokens.test.ts +4 -3
- package/src/presentation/routes/mint-registration-tokens.test.ts +12 -12
- package/src/presentation/routes/mint-registration-tokens.ts +5 -7
- package/src/presentation/routes/planned-capacity.ts +49 -0
- package/src/presentation/routes/poll-demand-release.test.ts +4 -4
- package/src/presentation/routes/poll-demand.test.ts +161 -22
- package/src/presentation/routes/poll-demand.ts +105 -45
- package/src/presentation/routes/provisioner-me.test.ts +37 -3
- package/src/presentation/routes/{reconcile-provisioned-runners.test.ts → reconcile-runner-instances.test.ts} +66 -61
- package/src/presentation/routes/reconcile-runner-instances.ts +30 -0
- package/src/presentation/routes/register.test.ts +9 -5
- package/src/presentation/routes/register.ts +61 -57
- package/src/presentation/routes/{report-provisioned-runners.test.ts → report-runner-instances.test.ts} +28 -28
- package/src/presentation/routes/{report-provisioned-runners.ts → report-runner-instances.ts} +14 -14
- package/src/presentation/routes/request-job.test.ts +5 -3
- package/src/presentation/routes/request-job.ts +41 -37
- package/src/presentation/routes/runner-enrollment.test.ts +259 -0
- package/src/presentation/routes/runner-enrollment.ts +174 -0
- package/src/temporal/activities/index.ts +2 -2
- package/src/temporal/activities/maintenance-activities.test.ts +7 -7
- package/src/temporal/activities/maintenance-activities.ts +3 -3
- package/src/temporal/workflows/stuck-job-detector.test.ts +3 -3
- package/src/temporal/workflows/stuck-job-detector.ts +2 -2
- package/test/env.ts +1 -3
- package/test/factories/ephemeral-registration-token.ts +2 -2
- package/test/factories/provisioner-token.ts +3 -1
- package/test/factories/runner-instance.ts +57 -0
- package/test/factories/runner-session.ts +3 -1
- package/test/fixtures/auth-inter-module.ts +30 -0
- package/test/globalSetup.ts +5 -1
- package/test/index.ts +2 -1
- package/tsconfig.build.tsbuildinfo +1 -1
- package/dist/core/entities/provisioned-runner.d.ts.map +0 -1
- package/dist/core/entities/provisioned-runner.js +0 -3
- package/dist/core/entities/provisioned-runner.js.map +0 -1
- package/dist/core/provisioned-runners.d.ts +0 -67
- package/dist/core/provisioned-runners.d.ts.map +0 -1
- package/dist/core/provisioned-runners.js +0 -162
- package/dist/core/provisioned-runners.js.map +0 -1
- package/dist/db/provisioned-runners.d.ts +0 -79
- package/dist/db/provisioned-runners.d.ts.map +0 -1
- package/dist/db/provisioned-runners.js +0 -466
- package/dist/db/provisioned-runners.js.map +0 -1
- package/dist/db/schema/provisioned-runners.d.ts.map +0 -1
- package/dist/db/schema/provisioned-runners.js.map +0 -1
- package/dist/presentation/dto/provisioned-runners.d.ts.map +0 -1
- package/dist/presentation/dto/provisioned-runners.js.map +0 -1
- package/dist/presentation/routes/reconcile-provisioned-runners.d.ts +0 -2
- package/dist/presentation/routes/reconcile-provisioned-runners.d.ts.map +0 -1
- package/dist/presentation/routes/reconcile-provisioned-runners.js +0 -27
- package/dist/presentation/routes/reconcile-provisioned-runners.js.map +0 -1
- package/dist/presentation/routes/report-provisioned-runners.d.ts +0 -2
- package/dist/presentation/routes/report-provisioned-runners.d.ts.map +0 -1
- package/dist/presentation/routes/report-provisioned-runners.js +0 -37
- package/dist/presentation/routes/report-provisioned-runners.js.map +0 -1
- package/src/core/provisioned-runners.ts +0 -301
- package/src/db/provisioned-runners.ts +0 -886
- package/src/presentation/routes/reconcile-provisioned-runners.ts +0 -30
- package/test/factories/provisioned-runner.ts +0 -78
|
@@ -3,35 +3,37 @@ import {and, desc, eq, inArray, or, sql} from 'drizzle-orm';
|
|
|
3
3
|
import {db} from '#db/db.js';
|
|
4
4
|
import {createRunnerSessionConsumingEphemeralToken} from '#db/ephemeral-registration-tokens.js';
|
|
5
5
|
import {
|
|
6
|
-
|
|
7
|
-
|
|
6
|
+
attachProviderRunnerId,
|
|
7
|
+
createPlannedProvisionedCapacity,
|
|
8
|
+
listActiveRunnerInstanceCountsByTemplateTx,
|
|
9
|
+
listActiveRunnerInstances,
|
|
8
10
|
listProvisionerTerminateIntentRowsTx,
|
|
9
11
|
listProvisionerTerminateIntents,
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
} from '#db/
|
|
14
|
-
import {provisionedRunners} from '#db/schema/provisioned-runners.js';
|
|
12
|
+
reapStaleRunnerInstances,
|
|
13
|
+
reconcileRunnerInstances,
|
|
14
|
+
reportRunnerInstances,
|
|
15
|
+
} from '#db/runner-instances.js';
|
|
15
16
|
import {provisionerTokens} from '#db/schema/provisioner-tokens.js';
|
|
16
17
|
import {reservations} from '#db/schema/reservations.js';
|
|
18
|
+
import {providerRunners} from '#db/schema/runner-instances.js';
|
|
17
19
|
import {runnerSessions} from '#db/schema/runner-sessions.js';
|
|
18
20
|
import {runningJobExecutions} from '#db/schema/running-job-executions.js';
|
|
19
21
|
import {
|
|
20
22
|
ephemeralRegistrationTokenFactory,
|
|
21
|
-
|
|
23
|
+
providerRunnerFactory,
|
|
22
24
|
provisionerTokenFactory,
|
|
23
25
|
reservationFactory,
|
|
24
26
|
runnerSessionFactory,
|
|
25
27
|
} from '#test/index.js';
|
|
26
28
|
|
|
27
|
-
function
|
|
29
|
+
function providerRunnerRowsFor(params: {workspaceId: string; provisionerId: string}) {
|
|
28
30
|
return db()
|
|
29
31
|
.select()
|
|
30
|
-
.from(
|
|
32
|
+
.from(providerRunners)
|
|
31
33
|
.where(
|
|
32
34
|
and(
|
|
33
|
-
eq(
|
|
34
|
-
eq(
|
|
35
|
+
eq(providerRunners.workspaceId, params.workspaceId),
|
|
36
|
+
eq(providerRunners.provisionerId, params.provisionerId),
|
|
35
37
|
),
|
|
36
38
|
);
|
|
37
39
|
}
|
|
@@ -51,7 +53,7 @@ function reservationRowsFor(params: {workspaceId: string; provisionerId: string}
|
|
|
51
53
|
async function insertRunningJobRow(params: {
|
|
52
54
|
workspaceId: string;
|
|
53
55
|
provisionerId: string;
|
|
54
|
-
|
|
56
|
+
providerRunnerId: string;
|
|
55
57
|
jobExecutionId?: string;
|
|
56
58
|
startedAt?: Date;
|
|
57
59
|
cancellationRequestedAt?: Date | null;
|
|
@@ -70,7 +72,7 @@ async function insertRunningJobRow(params: {
|
|
|
70
72
|
projectId: crypto.randomUUID(),
|
|
71
73
|
runnerSessionId: runnerSession.id,
|
|
72
74
|
provisionerId: params.provisionerId,
|
|
73
|
-
|
|
75
|
+
providerRunnerId: params.providerRunnerId,
|
|
74
76
|
requiredLabels: ['linux'],
|
|
75
77
|
runnerLabels: ['linux'],
|
|
76
78
|
startedAt,
|
|
@@ -79,7 +81,7 @@ async function insertRunningJobRow(params: {
|
|
|
79
81
|
});
|
|
80
82
|
}
|
|
81
83
|
|
|
82
|
-
describe('
|
|
84
|
+
describe('reportRunnerInstances', () => {
|
|
83
85
|
let workspaceId: string;
|
|
84
86
|
let provisionerId: string;
|
|
85
87
|
|
|
@@ -88,23 +90,48 @@ describe('reportProvisionedRunners', () => {
|
|
|
88
90
|
provisionerId = crypto.randomUUID();
|
|
89
91
|
});
|
|
90
92
|
|
|
93
|
+
it('reports and reconciles installation capacity without a workspace assignment', async () => {
|
|
94
|
+
const reportedAt = new Date();
|
|
95
|
+
|
|
96
|
+
const report = await reportRunnerInstances({
|
|
97
|
+
workspaceId: null,
|
|
98
|
+
provisionerId,
|
|
99
|
+
events: [event({providerRunnerId: 'installation-runner', reportedAt})],
|
|
100
|
+
});
|
|
101
|
+
const reconcile = await reconcileRunnerInstances({
|
|
102
|
+
workspaceId: null,
|
|
103
|
+
provisionerId,
|
|
104
|
+
observedRunnerInstanceIds: ['installation-runner'],
|
|
105
|
+
terminateGraceSeconds: 60,
|
|
106
|
+
});
|
|
107
|
+
const [row] = await db()
|
|
108
|
+
.select()
|
|
109
|
+
.from(providerRunners)
|
|
110
|
+
.where(eq(providerRunners.providerRunnerId, 'installation-runner'));
|
|
111
|
+
|
|
112
|
+
expect(report).toEqual({accepted: 1, reservationsReleased: 0, terminateIntentsHonored: []});
|
|
113
|
+
expect(reconcile.absentIds).toEqual([]);
|
|
114
|
+
expect(reconcile.observedRows).toMatchObject([{workspaceId: null}]);
|
|
115
|
+
expect(row).toMatchObject({workspaceId: null, provisionerId, reportedAt});
|
|
116
|
+
});
|
|
117
|
+
|
|
91
118
|
it('dedupes duplicate provisioned runner ids in one batch', async () => {
|
|
92
119
|
const reportedAt = new Date();
|
|
93
120
|
|
|
94
|
-
const result = await
|
|
121
|
+
const result = await reportRunnerInstances({
|
|
95
122
|
workspaceId,
|
|
96
123
|
provisionerId,
|
|
97
124
|
events: [
|
|
98
|
-
event({
|
|
125
|
+
event({providerRunnerId: 'provisioned-runner-1', state: 'starting', reportedAt}),
|
|
99
126
|
event({
|
|
100
|
-
|
|
127
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
101
128
|
state: 'running',
|
|
102
129
|
reportedAt: new Date(reportedAt.getTime() + 1),
|
|
103
130
|
}),
|
|
104
131
|
],
|
|
105
132
|
});
|
|
106
133
|
|
|
107
|
-
const rows = await
|
|
134
|
+
const rows = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
108
135
|
expect(result).toEqual({accepted: 1, reservationsReleased: 0, terminateIntentsHonored: []});
|
|
109
136
|
expect(rows).toHaveLength(1);
|
|
110
137
|
expect(rows[0]?.state).toBe('running');
|
|
@@ -113,19 +140,19 @@ describe('reportProvisionedRunners', () => {
|
|
|
113
140
|
it('uses state progression to dedupe equal-timestamp provisioned runner reports', async () => {
|
|
114
141
|
const reportedAt = new Date();
|
|
115
142
|
|
|
116
|
-
const result = await
|
|
143
|
+
const result = await reportRunnerInstances({
|
|
117
144
|
workspaceId,
|
|
118
145
|
provisionerId,
|
|
119
146
|
events: [
|
|
120
|
-
event({
|
|
121
|
-
event({
|
|
122
|
-
event({
|
|
123
|
-
event({
|
|
147
|
+
event({providerRunnerId: 'provisioned-runner-1', state: 'running', reportedAt}),
|
|
148
|
+
event({providerRunnerId: 'provisioned-runner-1', state: 'failed', reportedAt}),
|
|
149
|
+
event({providerRunnerId: 'provisioned-runner-2', state: 'failed', reportedAt}),
|
|
150
|
+
event({providerRunnerId: 'provisioned-runner-2', state: 'running', reportedAt}),
|
|
124
151
|
],
|
|
125
152
|
});
|
|
126
153
|
|
|
127
|
-
const rows = await
|
|
128
|
-
|
|
154
|
+
const rows = await providerRunnerRowsFor({workspaceId, provisionerId}).orderBy(
|
|
155
|
+
providerRunners.providerRunnerId,
|
|
129
156
|
);
|
|
130
157
|
expect(result).toEqual({accepted: 2, reservationsReleased: 0, terminateIntentsHonored: []});
|
|
131
158
|
expect(rows.map((row) => row.state)).toEqual(['failed', 'failed']);
|
|
@@ -133,20 +160,20 @@ describe('reportProvisionedRunners', () => {
|
|
|
133
160
|
|
|
134
161
|
it('accepts delayed events that move the lifecycle forward', async () => {
|
|
135
162
|
const newest = new Date();
|
|
136
|
-
await
|
|
163
|
+
await reportRunnerInstances({
|
|
137
164
|
workspaceId,
|
|
138
165
|
provisionerId,
|
|
139
166
|
events: [
|
|
140
|
-
event({
|
|
167
|
+
event({providerRunnerId: 'provisioned-runner-1', state: 'running', reportedAt: newest}),
|
|
141
168
|
],
|
|
142
169
|
});
|
|
143
170
|
|
|
144
|
-
await
|
|
171
|
+
await reportRunnerInstances({
|
|
145
172
|
workspaceId,
|
|
146
173
|
provisionerId,
|
|
147
174
|
events: [
|
|
148
175
|
event({
|
|
149
|
-
|
|
176
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
150
177
|
state: 'failed',
|
|
151
178
|
reason: 'late stale failure',
|
|
152
179
|
reportedAt: new Date(newest.getTime() - 1_000),
|
|
@@ -154,19 +181,19 @@ describe('reportProvisionedRunners', () => {
|
|
|
154
181
|
],
|
|
155
182
|
});
|
|
156
183
|
|
|
157
|
-
const rows = await
|
|
184
|
+
const rows = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
158
185
|
expect(rows[0]?.state).toBe('failed');
|
|
159
186
|
expect(rows[0]?.reason).toBe('late stale failure');
|
|
160
187
|
});
|
|
161
188
|
|
|
162
189
|
it('rejects older out-of-order events in the same lifecycle state', async () => {
|
|
163
190
|
const newest = new Date();
|
|
164
|
-
await
|
|
191
|
+
await reportRunnerInstances({
|
|
165
192
|
workspaceId,
|
|
166
193
|
provisionerId,
|
|
167
194
|
events: [
|
|
168
195
|
event({
|
|
169
|
-
|
|
196
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
170
197
|
state: 'running',
|
|
171
198
|
reason: 'fresh',
|
|
172
199
|
reportedAt: newest,
|
|
@@ -174,12 +201,12 @@ describe('reportProvisionedRunners', () => {
|
|
|
174
201
|
],
|
|
175
202
|
});
|
|
176
203
|
|
|
177
|
-
await
|
|
204
|
+
await reportRunnerInstances({
|
|
178
205
|
workspaceId,
|
|
179
206
|
provisionerId,
|
|
180
207
|
events: [
|
|
181
208
|
event({
|
|
182
|
-
|
|
209
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
183
210
|
state: 'running',
|
|
184
211
|
reason: 'stale',
|
|
185
212
|
reportedAt: new Date(newest.getTime() - 1_000),
|
|
@@ -187,7 +214,7 @@ describe('reportProvisionedRunners', () => {
|
|
|
187
214
|
],
|
|
188
215
|
});
|
|
189
216
|
|
|
190
|
-
const rows = await
|
|
217
|
+
const rows = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
191
218
|
expect(rows[0]?.state).toBe('running');
|
|
192
219
|
expect(rows[0]?.reason).toBe('fresh');
|
|
193
220
|
});
|
|
@@ -195,12 +222,12 @@ describe('reportProvisionedRunners', () => {
|
|
|
195
222
|
it('does not let equal-timestamp lower-priority reports flip terminal state', async () => {
|
|
196
223
|
const reservationId = await createReservation(1);
|
|
197
224
|
const reportedAt = new Date('2025-01-01T00:00:00.000Z');
|
|
198
|
-
await
|
|
225
|
+
await reportRunnerInstances({
|
|
199
226
|
workspaceId,
|
|
200
227
|
provisionerId,
|
|
201
228
|
events: [
|
|
202
229
|
event({
|
|
203
|
-
|
|
230
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
204
231
|
reservationId,
|
|
205
232
|
state: 'running',
|
|
206
233
|
reportedAt,
|
|
@@ -208,24 +235,24 @@ describe('reportProvisionedRunners', () => {
|
|
|
208
235
|
],
|
|
209
236
|
});
|
|
210
237
|
|
|
211
|
-
const terminal = await
|
|
238
|
+
const terminal = await reportRunnerInstances({
|
|
212
239
|
workspaceId,
|
|
213
240
|
provisionerId,
|
|
214
241
|
events: [
|
|
215
242
|
event({
|
|
216
|
-
|
|
243
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
217
244
|
reservationId,
|
|
218
245
|
state: 'failed',
|
|
219
246
|
reportedAt,
|
|
220
247
|
}),
|
|
221
248
|
],
|
|
222
249
|
});
|
|
223
|
-
const revived = await
|
|
250
|
+
const revived = await reportRunnerInstances({
|
|
224
251
|
workspaceId,
|
|
225
252
|
provisionerId,
|
|
226
253
|
events: [
|
|
227
254
|
event({
|
|
228
|
-
|
|
255
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
229
256
|
reservationId,
|
|
230
257
|
state: 'running',
|
|
231
258
|
reportedAt,
|
|
@@ -233,41 +260,41 @@ describe('reportProvisionedRunners', () => {
|
|
|
233
260
|
],
|
|
234
261
|
});
|
|
235
262
|
|
|
236
|
-
const
|
|
263
|
+
const providerRunnerRows = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
237
264
|
const reservationRows = await reservationRowsFor({workspaceId, provisionerId});
|
|
238
265
|
expect(terminal).toEqual({accepted: 1, reservationsReleased: 1, terminateIntentsHonored: []});
|
|
239
266
|
expect(revived).toEqual({accepted: 1, reservationsReleased: 0, terminateIntentsHonored: []});
|
|
240
|
-
expect(
|
|
241
|
-
expect(
|
|
267
|
+
expect(providerRunnerRows[0]?.state).toBe('failed');
|
|
268
|
+
expect(providerRunnerRows[0]?.reservationReleasedAt).toBeInstanceOf(Date);
|
|
242
269
|
expect(reservationRows).toHaveLength(0);
|
|
243
270
|
});
|
|
244
271
|
|
|
245
272
|
it('clamps future reported times so they do not pin provisioned runner state', async () => {
|
|
246
|
-
await
|
|
273
|
+
await reportRunnerInstances({
|
|
247
274
|
workspaceId,
|
|
248
275
|
provisionerId,
|
|
249
276
|
events: [
|
|
250
277
|
event({
|
|
251
|
-
|
|
278
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
252
279
|
state: 'starting',
|
|
253
280
|
reportedAt: new Date(Date.now() + 60 * 60 * 1000),
|
|
254
281
|
}),
|
|
255
282
|
],
|
|
256
283
|
});
|
|
257
284
|
|
|
258
|
-
await
|
|
285
|
+
await reportRunnerInstances({
|
|
259
286
|
workspaceId,
|
|
260
287
|
provisionerId,
|
|
261
288
|
events: [
|
|
262
289
|
event({
|
|
263
|
-
|
|
290
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
264
291
|
state: 'running',
|
|
265
292
|
reportedAt: new Date(),
|
|
266
293
|
}),
|
|
267
294
|
],
|
|
268
295
|
});
|
|
269
296
|
|
|
270
|
-
const rows = await
|
|
297
|
+
const rows = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
271
298
|
expect(rows[0]?.state).toBe('running');
|
|
272
299
|
expect(rows[0]?.reportedAt.getTime()).toBeLessThan(Date.now() + 10_000);
|
|
273
300
|
});
|
|
@@ -278,34 +305,34 @@ describe('reportProvisionedRunners', () => {
|
|
|
278
305
|
const stoppedAt = new Date('2025-01-01T00:02:00.000Z');
|
|
279
306
|
const terminatedAt = new Date('2025-01-01T00:03:00.000Z');
|
|
280
307
|
|
|
281
|
-
const result = await
|
|
308
|
+
const result = await reportRunnerInstances({
|
|
282
309
|
workspaceId,
|
|
283
310
|
provisionerId,
|
|
284
311
|
events: [
|
|
285
312
|
event({
|
|
286
|
-
|
|
313
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
287
314
|
state: 'running',
|
|
288
315
|
reportedAt: startedAt,
|
|
289
316
|
}),
|
|
290
317
|
event({
|
|
291
|
-
|
|
318
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
292
319
|
state: 'stopping',
|
|
293
320
|
reportedAt: stoppingAt,
|
|
294
321
|
}),
|
|
295
322
|
event({
|
|
296
|
-
|
|
323
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
297
324
|
state: 'stopped',
|
|
298
325
|
reportedAt: stoppedAt,
|
|
299
326
|
}),
|
|
300
327
|
event({
|
|
301
|
-
|
|
328
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
302
329
|
state: 'terminated',
|
|
303
330
|
reportedAt: terminatedAt,
|
|
304
331
|
}),
|
|
305
332
|
],
|
|
306
333
|
});
|
|
307
334
|
|
|
308
|
-
const rows = await
|
|
335
|
+
const rows = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
309
336
|
expect(result).toEqual({accepted: 1, reservationsReleased: 0, terminateIntentsHonored: []});
|
|
310
337
|
expect(rows[0]?.state).toBe('terminated');
|
|
311
338
|
expect(rows[0]?.reportedAt.toISOString()).toBe(terminatedAt.toISOString());
|
|
@@ -319,31 +346,31 @@ describe('reportProvisionedRunners', () => {
|
|
|
319
346
|
it('records delayed lower-state milestones without reviving current state', async () => {
|
|
320
347
|
const terminatedAt = new Date('2025-01-01T00:03:00.000Z');
|
|
321
348
|
const startedAt = new Date('2025-01-01T00:00:00.000Z');
|
|
322
|
-
await
|
|
349
|
+
await reportRunnerInstances({
|
|
323
350
|
workspaceId,
|
|
324
351
|
provisionerId,
|
|
325
352
|
events: [
|
|
326
353
|
event({
|
|
327
|
-
|
|
354
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
328
355
|
state: 'terminated',
|
|
329
356
|
reportedAt: terminatedAt,
|
|
330
357
|
}),
|
|
331
358
|
],
|
|
332
359
|
});
|
|
333
360
|
|
|
334
|
-
await
|
|
361
|
+
await reportRunnerInstances({
|
|
335
362
|
workspaceId,
|
|
336
363
|
provisionerId,
|
|
337
364
|
events: [
|
|
338
365
|
event({
|
|
339
|
-
|
|
366
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
340
367
|
state: 'running',
|
|
341
368
|
reportedAt: startedAt,
|
|
342
369
|
}),
|
|
343
370
|
],
|
|
344
371
|
});
|
|
345
372
|
|
|
346
|
-
const rows = await
|
|
373
|
+
const rows = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
347
374
|
expect(rows[0]?.state).toBe('terminated');
|
|
348
375
|
expect(rows[0]?.reportedAt.toISOString()).toBe(terminatedAt.toISOString());
|
|
349
376
|
expect(rows[0]?.startedAt?.toISOString()).toBe(startedAt.toISOString());
|
|
@@ -351,22 +378,22 @@ describe('reportProvisionedRunners', () => {
|
|
|
351
378
|
});
|
|
352
379
|
|
|
353
380
|
it('uses server update time for active provisioned runner windows', async () => {
|
|
354
|
-
await
|
|
381
|
+
await reportRunnerInstances({
|
|
355
382
|
workspaceId,
|
|
356
383
|
provisionerId,
|
|
357
384
|
events: [
|
|
358
385
|
event({
|
|
359
|
-
|
|
386
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
360
387
|
state: 'running',
|
|
361
388
|
reportedAt: new Date(Date.now() + 60 * 60 * 1000),
|
|
362
389
|
}),
|
|
363
390
|
],
|
|
364
391
|
});
|
|
365
392
|
await db().execute(
|
|
366
|
-
sql`UPDATE
|
|
393
|
+
sql`UPDATE runners_runner_instances SET updated_at = now() - interval '10 minutes'`,
|
|
367
394
|
);
|
|
368
395
|
|
|
369
|
-
const active = await
|
|
396
|
+
const active = await listActiveRunnerInstances({workspaceId, windowSeconds: 60});
|
|
370
397
|
|
|
371
398
|
expect(active).toEqual([]);
|
|
372
399
|
});
|
|
@@ -374,19 +401,17 @@ describe('reportProvisionedRunners', () => {
|
|
|
374
401
|
it('releases one reservation unit for a terminal unclaimed provisioned runner', async () => {
|
|
375
402
|
const reservationId = await createReservation(2);
|
|
376
403
|
|
|
377
|
-
const result = await
|
|
404
|
+
const result = await reportRunnerInstances({
|
|
378
405
|
workspaceId,
|
|
379
406
|
provisionerId,
|
|
380
|
-
events: [
|
|
381
|
-
event({provisionedRunnerId: 'provisioned-runner-1', reservationId, state: 'failed'}),
|
|
382
|
-
],
|
|
407
|
+
events: [event({providerRunnerId: 'provisioned-runner-1', reservationId, state: 'failed'})],
|
|
383
408
|
});
|
|
384
409
|
|
|
385
410
|
const reservationRows = await reservationRowsFor({workspaceId, provisionerId});
|
|
386
|
-
const
|
|
411
|
+
const providerRunnerRows = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
387
412
|
expect(result).toEqual({accepted: 1, reservationsReleased: 1, terminateIntentsHonored: []});
|
|
388
413
|
expect(reservationRows[0]?.count).toBe(1);
|
|
389
|
-
expect(
|
|
414
|
+
expect(providerRunnerRows[0]?.reservationReleasedAt).toBeInstanceOf(Date);
|
|
390
415
|
});
|
|
391
416
|
|
|
392
417
|
it('does not release a reservation owned by another workspace or provisioner', async () => {
|
|
@@ -399,17 +424,17 @@ describe('reportProvisionedRunners', () => {
|
|
|
399
424
|
provisionerId: crypto.randomUUID(),
|
|
400
425
|
});
|
|
401
426
|
|
|
402
|
-
const result = await
|
|
427
|
+
const result = await reportRunnerInstances({
|
|
403
428
|
workspaceId,
|
|
404
429
|
provisionerId,
|
|
405
430
|
events: [
|
|
406
431
|
event({
|
|
407
|
-
|
|
432
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
408
433
|
reservationId: otherWorkspaceReservationId,
|
|
409
434
|
state: 'failed',
|
|
410
435
|
}),
|
|
411
436
|
event({
|
|
412
|
-
|
|
437
|
+
providerRunnerId: 'provisioned-runner-2',
|
|
413
438
|
reservationId: peerProvisionerReservationId,
|
|
414
439
|
state: 'failed',
|
|
415
440
|
}),
|
|
@@ -428,19 +453,15 @@ describe('reportProvisionedRunners', () => {
|
|
|
428
453
|
it('releases a reservation only once across repeated terminal reports', async () => {
|
|
429
454
|
const reservationId = await createReservation(2);
|
|
430
455
|
|
|
431
|
-
await
|
|
456
|
+
await reportRunnerInstances({
|
|
432
457
|
workspaceId,
|
|
433
458
|
provisionerId,
|
|
434
|
-
events: [
|
|
435
|
-
event({provisionedRunnerId: 'provisioned-runner-1', reservationId, state: 'failed'}),
|
|
436
|
-
],
|
|
459
|
+
events: [event({providerRunnerId: 'provisioned-runner-1', reservationId, state: 'failed'})],
|
|
437
460
|
});
|
|
438
|
-
const result = await
|
|
461
|
+
const result = await reportRunnerInstances({
|
|
439
462
|
workspaceId,
|
|
440
463
|
provisionerId,
|
|
441
|
-
events: [
|
|
442
|
-
event({provisionedRunnerId: 'provisioned-runner-1', reservationId, state: 'failed'}),
|
|
443
|
-
],
|
|
464
|
+
events: [event({providerRunnerId: 'provisioned-runner-1', reservationId, state: 'failed'})],
|
|
444
465
|
});
|
|
445
466
|
|
|
446
467
|
const reservationRows = await reservationRowsFor({workspaceId, provisionerId});
|
|
@@ -451,12 +472,12 @@ describe('reportProvisionedRunners', () => {
|
|
|
451
472
|
it('does not let a newer running report revive a terminal provisioned runner', async () => {
|
|
452
473
|
const reservationId = await createReservation(2);
|
|
453
474
|
const failedAt = new Date();
|
|
454
|
-
await
|
|
475
|
+
await reportRunnerInstances({
|
|
455
476
|
workspaceId,
|
|
456
477
|
provisionerId,
|
|
457
478
|
events: [
|
|
458
479
|
event({
|
|
459
|
-
|
|
480
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
460
481
|
reservationId,
|
|
461
482
|
state: 'failed',
|
|
462
483
|
reportedAt: failedAt,
|
|
@@ -464,12 +485,12 @@ describe('reportProvisionedRunners', () => {
|
|
|
464
485
|
],
|
|
465
486
|
});
|
|
466
487
|
|
|
467
|
-
await
|
|
488
|
+
await reportRunnerInstances({
|
|
468
489
|
workspaceId,
|
|
469
490
|
provisionerId,
|
|
470
491
|
events: [
|
|
471
492
|
event({
|
|
472
|
-
|
|
493
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
473
494
|
reservationId,
|
|
474
495
|
state: 'running',
|
|
475
496
|
reportedAt: new Date(failedAt.getTime() + 1_000),
|
|
@@ -477,42 +498,42 @@ describe('reportProvisionedRunners', () => {
|
|
|
477
498
|
],
|
|
478
499
|
});
|
|
479
500
|
|
|
480
|
-
const
|
|
501
|
+
const providerRunnerRows = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
481
502
|
const reservationRows = await reservationRowsFor({workspaceId, provisionerId});
|
|
482
|
-
expect(
|
|
483
|
-
expect(
|
|
503
|
+
expect(providerRunnerRows[0]?.state).toBe('failed');
|
|
504
|
+
expect(providerRunnerRows[0]?.reservationReleasedAt).toBeInstanceOf(Date);
|
|
484
505
|
expect(reservationRows[0]?.count).toBe(1);
|
|
485
506
|
});
|
|
486
507
|
|
|
487
508
|
it('tracks provider cleanup as terminated', async () => {
|
|
488
509
|
const reservationId = await createReservation(2);
|
|
489
510
|
|
|
490
|
-
const result = await
|
|
511
|
+
const result = await reportRunnerInstances({
|
|
491
512
|
workspaceId,
|
|
492
513
|
provisionerId,
|
|
493
514
|
events: [
|
|
494
|
-
event({
|
|
515
|
+
event({providerRunnerId: 'provisioned-runner-1', reservationId, state: 'terminated'}),
|
|
495
516
|
],
|
|
496
517
|
});
|
|
497
518
|
|
|
498
519
|
const reservationRows = await reservationRowsFor({workspaceId, provisionerId});
|
|
499
|
-
const
|
|
520
|
+
const providerRunnerRows = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
500
521
|
expect(result).toEqual({accepted: 1, reservationsReleased: 1, terminateIntentsHonored: []});
|
|
501
522
|
expect(reservationRows[0]?.count).toBe(1);
|
|
502
|
-
expect(
|
|
503
|
-
expect(
|
|
504
|
-
expect(
|
|
523
|
+
expect(providerRunnerRows[0]?.state).toBe('terminated');
|
|
524
|
+
expect(providerRunnerRows[0]?.terminatedAt).toBeInstanceOf(Date);
|
|
525
|
+
expect(providerRunnerRows[0]?.reservationReleasedAt).toBeInstanceOf(Date);
|
|
505
526
|
});
|
|
506
527
|
|
|
507
528
|
it('releases multiple units from the same reservation in one batch', async () => {
|
|
508
529
|
const reservationId = await createReservation(3);
|
|
509
530
|
|
|
510
|
-
const result = await
|
|
531
|
+
const result = await reportRunnerInstances({
|
|
511
532
|
workspaceId,
|
|
512
533
|
provisionerId,
|
|
513
534
|
events: [
|
|
514
|
-
event({
|
|
515
|
-
event({
|
|
535
|
+
event({providerRunnerId: 'provisioned-runner-1', reservationId, state: 'failed'}),
|
|
536
|
+
event({providerRunnerId: 'provisioned-runner-2', reservationId, state: 'stopped'}),
|
|
516
537
|
],
|
|
517
538
|
});
|
|
518
539
|
|
|
@@ -524,12 +545,10 @@ describe('reportProvisionedRunners', () => {
|
|
|
524
545
|
it('deletes a one-unit reservation instead of violating the positive count check', async () => {
|
|
525
546
|
const reservationId = await createReservation(1);
|
|
526
547
|
|
|
527
|
-
const result = await
|
|
548
|
+
const result = await reportRunnerInstances({
|
|
528
549
|
workspaceId,
|
|
529
550
|
provisionerId,
|
|
530
|
-
events: [
|
|
531
|
-
event({provisionedRunnerId: 'provisioned-runner-1', reservationId, state: 'failed'}),
|
|
532
|
-
],
|
|
551
|
+
events: [event({providerRunnerId: 'provisioned-runner-1', reservationId, state: 'failed'})],
|
|
533
552
|
});
|
|
534
553
|
|
|
535
554
|
const reservationRows = await reservationRowsFor({workspaceId, provisionerId});
|
|
@@ -548,32 +567,32 @@ describe('reportProvisionedRunners', () => {
|
|
|
548
567
|
const [reservation] = await reservationRowsFor({workspaceId, provisionerId});
|
|
549
568
|
if (!reservation) throw new Error('Expected reservation');
|
|
550
569
|
|
|
551
|
-
const result = await
|
|
570
|
+
const result = await reportRunnerInstances({
|
|
552
571
|
workspaceId,
|
|
553
572
|
provisionerId,
|
|
554
573
|
events: [
|
|
555
574
|
event({
|
|
556
|
-
|
|
575
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
557
576
|
reservationId: reservation.id,
|
|
558
577
|
state: 'failed',
|
|
559
578
|
}),
|
|
560
579
|
],
|
|
561
580
|
});
|
|
562
581
|
|
|
563
|
-
const
|
|
582
|
+
const providerRunnerRows = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
564
583
|
expect(result).toEqual({accepted: 1, reservationsReleased: 0, terminateIntentsHonored: []});
|
|
565
|
-
expect(
|
|
584
|
+
expect(providerRunnerRows[0]?.reservationReleasedAt).toBeInstanceOf(Date);
|
|
566
585
|
});
|
|
567
586
|
|
|
568
587
|
it('does not release a reservation for a provisioned runner that already has a runner session', async () => {
|
|
569
588
|
const reservationId = await createReservation(1);
|
|
570
589
|
|
|
571
|
-
const result = await
|
|
590
|
+
const result = await reportRunnerInstances({
|
|
572
591
|
workspaceId,
|
|
573
592
|
provisionerId,
|
|
574
593
|
events: [
|
|
575
594
|
event({
|
|
576
|
-
|
|
595
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
577
596
|
reservationId,
|
|
578
597
|
state: 'failed',
|
|
579
598
|
runnerSessionId: crypto.randomUUID(),
|
|
@@ -582,10 +601,10 @@ describe('reportProvisionedRunners', () => {
|
|
|
582
601
|
});
|
|
583
602
|
|
|
584
603
|
const reservationRows = await reservationRowsFor({workspaceId, provisionerId});
|
|
585
|
-
const
|
|
604
|
+
const providerRunnerRows = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
586
605
|
expect(result).toEqual({accepted: 1, reservationsReleased: 0, terminateIntentsHonored: []});
|
|
587
606
|
expect(reservationRows[0]?.count).toBe(1);
|
|
588
|
-
expect(
|
|
607
|
+
expect(providerRunnerRows[0]?.reservationReleasedAt).toBeNull();
|
|
589
608
|
});
|
|
590
609
|
|
|
591
610
|
it('uses the consumed ephemeral token session before releasing a terminal report', async () => {
|
|
@@ -594,7 +613,7 @@ describe('reportProvisionedRunners', () => {
|
|
|
594
613
|
workspaceId,
|
|
595
614
|
provisionerId,
|
|
596
615
|
reservationId,
|
|
597
|
-
|
|
616
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
598
617
|
});
|
|
599
618
|
const session = await createRunnerSessionConsumingEphemeralToken({
|
|
600
619
|
ephemeralTokenId: token.id,
|
|
@@ -603,12 +622,12 @@ describe('reportProvisionedRunners', () => {
|
|
|
603
622
|
maxClaims: 1,
|
|
604
623
|
});
|
|
605
624
|
|
|
606
|
-
const result = await
|
|
625
|
+
const result = await reportRunnerInstances({
|
|
607
626
|
workspaceId,
|
|
608
627
|
provisionerId,
|
|
609
628
|
events: [
|
|
610
629
|
event({
|
|
611
|
-
|
|
630
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
612
631
|
reservationId,
|
|
613
632
|
state: 'failed',
|
|
614
633
|
runnerSessionId: crypto.randomUUID(),
|
|
@@ -617,11 +636,11 @@ describe('reportProvisionedRunners', () => {
|
|
|
617
636
|
});
|
|
618
637
|
|
|
619
638
|
const reservationRows = await reservationRowsFor({workspaceId, provisionerId});
|
|
620
|
-
const
|
|
639
|
+
const providerRunnerRows = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
621
640
|
expect(result).toEqual({accepted: 1, reservationsReleased: 0, terminateIntentsHonored: []});
|
|
622
641
|
expect(reservationRows[0]?.count).toBe(1);
|
|
623
|
-
expect(
|
|
624
|
-
expect(
|
|
642
|
+
expect(providerRunnerRows[0]?.runnerSessionId).toBe(session.id);
|
|
643
|
+
expect(providerRunnerRows[0]?.reservationReleasedAt).toBeNull();
|
|
625
644
|
});
|
|
626
645
|
|
|
627
646
|
it('preserves claimed runner session metadata when a terminal state wins the batch', async () => {
|
|
@@ -629,19 +648,19 @@ describe('reportProvisionedRunners', () => {
|
|
|
629
648
|
const runnerSessionId = crypto.randomUUID();
|
|
630
649
|
const reportedAt = new Date('2025-01-01T00:00:00.000Z');
|
|
631
650
|
|
|
632
|
-
const result = await
|
|
651
|
+
const result = await reportRunnerInstances({
|
|
633
652
|
workspaceId,
|
|
634
653
|
provisionerId,
|
|
635
654
|
events: [
|
|
636
655
|
event({
|
|
637
|
-
|
|
656
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
638
657
|
reservationId,
|
|
639
658
|
state: 'running',
|
|
640
659
|
runnerSessionId,
|
|
641
660
|
reportedAt,
|
|
642
661
|
}),
|
|
643
662
|
event({
|
|
644
|
-
|
|
663
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
645
664
|
reservationId,
|
|
646
665
|
state: 'failed',
|
|
647
666
|
reportedAt: new Date(reportedAt.getTime() + 1_000),
|
|
@@ -650,41 +669,41 @@ describe('reportProvisionedRunners', () => {
|
|
|
650
669
|
});
|
|
651
670
|
|
|
652
671
|
const reservationRows = await reservationRowsFor({workspaceId, provisionerId});
|
|
653
|
-
const
|
|
672
|
+
const providerRunnerRows = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
654
673
|
expect(result).toEqual({accepted: 1, reservationsReleased: 0, terminateIntentsHonored: []});
|
|
655
674
|
expect(reservationRows[0]?.count).toBe(1);
|
|
656
|
-
expect(
|
|
657
|
-
expect(
|
|
658
|
-
expect(
|
|
675
|
+
expect(providerRunnerRows[0]?.state).toBe('failed');
|
|
676
|
+
expect(providerRunnerRows[0]?.runnerSessionId).toBe(runnerSessionId);
|
|
677
|
+
expect(providerRunnerRows[0]?.reservationReleasedAt).toBeNull();
|
|
659
678
|
});
|
|
660
679
|
|
|
661
680
|
it('returns honored terminate intents only for the first active-to-terminated transition', async () => {
|
|
662
|
-
await
|
|
681
|
+
await providerRunnerFactory.create({
|
|
663
682
|
workspaceId,
|
|
664
683
|
provisionerId,
|
|
665
|
-
|
|
684
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
666
685
|
state: 'running',
|
|
667
686
|
});
|
|
668
687
|
await insertRunningJobRow({
|
|
669
688
|
workspaceId,
|
|
670
689
|
provisionerId,
|
|
671
|
-
|
|
690
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
672
691
|
cancellationRequestedAt: new Date('2025-01-01T00:01:00.000Z'),
|
|
673
692
|
});
|
|
674
693
|
|
|
675
|
-
const first = await
|
|
694
|
+
const first = await reportRunnerInstances({
|
|
676
695
|
workspaceId,
|
|
677
696
|
provisionerId,
|
|
678
|
-
events: [event({
|
|
697
|
+
events: [event({providerRunnerId: 'provisioned-runner-1', state: 'terminated'})],
|
|
679
698
|
});
|
|
680
|
-
const second = await
|
|
699
|
+
const second = await reportRunnerInstances({
|
|
681
700
|
workspaceId,
|
|
682
701
|
provisionerId,
|
|
683
|
-
events: [event({
|
|
702
|
+
events: [event({providerRunnerId: 'provisioned-runner-1', state: 'terminated'})],
|
|
684
703
|
});
|
|
685
704
|
|
|
686
705
|
expect(first.terminateIntentsHonored).toEqual([
|
|
687
|
-
{
|
|
706
|
+
{providerRunnerId: 'provisioned-runner-1', reason: 'job-cancelled'},
|
|
688
707
|
]);
|
|
689
708
|
expect(second.terminateIntentsHonored).toEqual([]);
|
|
690
709
|
});
|
|
@@ -716,7 +735,7 @@ describe('reportProvisionedRunners', () => {
|
|
|
716
735
|
}
|
|
717
736
|
|
|
718
737
|
function event(params: {
|
|
719
|
-
|
|
738
|
+
providerRunnerId?: string;
|
|
720
739
|
reservationId?: string | null;
|
|
721
740
|
state?: 'starting' | 'running' | 'stopping' | 'stopped' | 'failed' | 'terminated';
|
|
722
741
|
reportedAt?: Date;
|
|
@@ -724,7 +743,7 @@ describe('reportProvisionedRunners', () => {
|
|
|
724
743
|
runnerSessionId?: string | null;
|
|
725
744
|
}) {
|
|
726
745
|
return {
|
|
727
|
-
|
|
746
|
+
providerRunnerId: params.providerRunnerId ?? 'provisioned-runner-1',
|
|
728
747
|
reservationId: params.reservationId ?? null,
|
|
729
748
|
templateKey: 'linux',
|
|
730
749
|
labels: ['linux'],
|
|
@@ -737,7 +756,7 @@ describe('reportProvisionedRunners', () => {
|
|
|
737
756
|
}
|
|
738
757
|
});
|
|
739
758
|
|
|
740
|
-
describe('
|
|
759
|
+
describe('listActiveRunnerInstanceCountsByTemplateTx', () => {
|
|
741
760
|
let workspaceId: string;
|
|
742
761
|
let provisionerId: string;
|
|
743
762
|
|
|
@@ -747,44 +766,44 @@ describe('listActiveProvisionedRunnerCountsByTemplateTx', () => {
|
|
|
747
766
|
});
|
|
748
767
|
|
|
749
768
|
it('groups starting and running provisioned runners by template and ignores non-divergence states', async () => {
|
|
750
|
-
await
|
|
769
|
+
await providerRunnerFactory.create({
|
|
751
770
|
workspaceId,
|
|
752
771
|
provisionerId,
|
|
753
|
-
|
|
772
|
+
providerRunnerId: 'starting-1',
|
|
754
773
|
templateKey: 'linux',
|
|
755
774
|
state: 'starting',
|
|
756
775
|
});
|
|
757
|
-
await
|
|
776
|
+
await providerRunnerFactory.create({
|
|
758
777
|
workspaceId,
|
|
759
778
|
provisionerId,
|
|
760
|
-
|
|
779
|
+
providerRunnerId: 'running-1',
|
|
761
780
|
templateKey: 'linux',
|
|
762
781
|
state: 'running',
|
|
763
782
|
});
|
|
764
|
-
await
|
|
783
|
+
await providerRunnerFactory.create({
|
|
765
784
|
workspaceId,
|
|
766
785
|
provisionerId,
|
|
767
|
-
|
|
786
|
+
providerRunnerId: 'running-2',
|
|
768
787
|
templateKey: 'linux',
|
|
769
788
|
state: 'running',
|
|
770
789
|
});
|
|
771
|
-
await
|
|
790
|
+
await providerRunnerFactory.create({
|
|
772
791
|
workspaceId,
|
|
773
792
|
provisionerId,
|
|
774
|
-
|
|
793
|
+
providerRunnerId: 'stopping-1',
|
|
775
794
|
templateKey: 'linux',
|
|
776
795
|
state: 'stopping',
|
|
777
796
|
});
|
|
778
|
-
await
|
|
797
|
+
await providerRunnerFactory.create({
|
|
779
798
|
workspaceId,
|
|
780
799
|
provisionerId,
|
|
781
|
-
|
|
800
|
+
providerRunnerId: 'null-template',
|
|
782
801
|
templateKey: null,
|
|
783
802
|
state: 'running',
|
|
784
803
|
});
|
|
785
804
|
|
|
786
805
|
const result = await db().transaction((tx) =>
|
|
787
|
-
|
|
806
|
+
listActiveRunnerInstanceCountsByTemplateTx(tx, {workspaceId, provisionerId}),
|
|
788
807
|
);
|
|
789
808
|
|
|
790
809
|
expect(result).toEqual(
|
|
@@ -807,11 +826,11 @@ describe('listProvisionerTerminateIntents', () => {
|
|
|
807
826
|
});
|
|
808
827
|
|
|
809
828
|
it('includes active provisioned runners whose latest bound job is cancelled', async () => {
|
|
810
|
-
await
|
|
829
|
+
await createRunnerInstance({providerRunnerId: 'provisioned-runner-1'});
|
|
811
830
|
await insertRunningJobRow({
|
|
812
831
|
workspaceId,
|
|
813
832
|
provisionerId,
|
|
814
|
-
|
|
833
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
815
834
|
cancellationRequestedAt: new Date('2025-01-01T00:01:00.000Z'),
|
|
816
835
|
});
|
|
817
836
|
|
|
@@ -821,11 +840,11 @@ describe('listProvisionerTerminateIntents', () => {
|
|
|
821
840
|
});
|
|
822
841
|
|
|
823
842
|
it('returns structured rows with bounded reasons from the shared query', async () => {
|
|
824
|
-
await
|
|
843
|
+
await createRunnerInstance({providerRunnerId: 'provisioned-runner-1'});
|
|
825
844
|
await insertRunningJobRow({
|
|
826
845
|
workspaceId,
|
|
827
846
|
provisionerId,
|
|
828
|
-
|
|
847
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
829
848
|
cancellationRequestedAt: new Date('2025-01-01T00:01:00.000Z'),
|
|
830
849
|
});
|
|
831
850
|
|
|
@@ -833,17 +852,15 @@ describe('listProvisionerTerminateIntents', () => {
|
|
|
833
852
|
listProvisionerTerminateIntentRowsTx(tx, {workspaceId, provisionerId, limit: 1000}),
|
|
834
853
|
);
|
|
835
854
|
|
|
836
|
-
expect(result).toEqual([
|
|
837
|
-
{provisionedRunnerId: 'provisioned-runner-1', reason: 'job-cancelled'},
|
|
838
|
-
]);
|
|
855
|
+
expect(result).toEqual([{providerRunnerId: 'provisioned-runner-1', reason: 'job-cancelled'}]);
|
|
839
856
|
});
|
|
840
857
|
|
|
841
858
|
it('excludes active provisioned runners whose latest bound job is healthy', async () => {
|
|
842
|
-
await
|
|
859
|
+
await createRunnerInstance({providerRunnerId: 'provisioned-runner-1'});
|
|
843
860
|
await insertRunningJobRow({
|
|
844
861
|
workspaceId,
|
|
845
862
|
provisionerId,
|
|
846
|
-
|
|
863
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
847
864
|
});
|
|
848
865
|
|
|
849
866
|
const result = await listProvisionerTerminateIntents({workspaceId, provisionerId, limit: 1000});
|
|
@@ -852,14 +869,14 @@ describe('listProvisionerTerminateIntents', () => {
|
|
|
852
869
|
});
|
|
853
870
|
|
|
854
871
|
it('excludes terminal provisioned runners with cancelled bound jobs', async () => {
|
|
855
|
-
await
|
|
856
|
-
|
|
872
|
+
await createRunnerInstance({
|
|
873
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
857
874
|
state: 'terminated',
|
|
858
875
|
});
|
|
859
876
|
await insertRunningJobRow({
|
|
860
877
|
workspaceId,
|
|
861
878
|
provisionerId,
|
|
862
|
-
|
|
879
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
863
880
|
cancellationRequestedAt: new Date('2025-01-01T00:01:00.000Z'),
|
|
864
881
|
});
|
|
865
882
|
|
|
@@ -869,18 +886,18 @@ describe('listProvisionerTerminateIntents', () => {
|
|
|
869
886
|
});
|
|
870
887
|
|
|
871
888
|
it('excludes a cancelled job when it is not the latest bound job', async () => {
|
|
872
|
-
await
|
|
889
|
+
await createRunnerInstance({providerRunnerId: 'provisioned-runner-1'});
|
|
873
890
|
await insertRunningJobRow({
|
|
874
891
|
workspaceId,
|
|
875
892
|
provisionerId,
|
|
876
|
-
|
|
893
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
877
894
|
startedAt: new Date('2025-01-01T00:00:00.000Z'),
|
|
878
895
|
cancellationRequestedAt: new Date('2025-01-01T00:01:00.000Z'),
|
|
879
896
|
});
|
|
880
897
|
await insertRunningJobRow({
|
|
881
898
|
workspaceId,
|
|
882
899
|
provisionerId,
|
|
883
|
-
|
|
900
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
884
901
|
startedAt: new Date('2025-01-01T00:02:00.000Z'),
|
|
885
902
|
});
|
|
886
903
|
|
|
@@ -891,13 +908,13 @@ describe('listProvisionerTerminateIntents', () => {
|
|
|
891
908
|
|
|
892
909
|
it('excludes cancelled jobs for another provisioner', async () => {
|
|
893
910
|
const otherProvisionerId = crypto.randomUUID();
|
|
894
|
-
await
|
|
895
|
-
|
|
911
|
+
await createRunnerInstance({
|
|
912
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
896
913
|
provisionerId: otherProvisionerId,
|
|
897
914
|
});
|
|
898
915
|
await insertRunningJobRow({
|
|
899
916
|
workspaceId,
|
|
900
|
-
|
|
917
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
901
918
|
provisionerId: otherProvisionerId,
|
|
902
919
|
cancellationRequestedAt: new Date('2025-01-01T00:01:00.000Z'),
|
|
903
920
|
});
|
|
@@ -908,18 +925,18 @@ describe('listProvisionerTerminateIntents', () => {
|
|
|
908
925
|
});
|
|
909
926
|
|
|
910
927
|
it('returns one id for duplicate cancelled bound jobs on the same provisioned runner', async () => {
|
|
911
|
-
await
|
|
928
|
+
await createRunnerInstance({providerRunnerId: 'provisioned-runner-1'});
|
|
912
929
|
await insertRunningJobRow({
|
|
913
930
|
workspaceId,
|
|
914
931
|
provisionerId,
|
|
915
|
-
|
|
932
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
916
933
|
startedAt: new Date('2025-01-01T00:00:00.000Z'),
|
|
917
934
|
cancellationRequestedAt: new Date('2025-01-01T00:01:00.000Z'),
|
|
918
935
|
});
|
|
919
936
|
await insertRunningJobRow({
|
|
920
937
|
workspaceId,
|
|
921
938
|
provisionerId,
|
|
922
|
-
|
|
939
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
923
940
|
startedAt: new Date('2025-01-01T00:00:00.000Z'),
|
|
924
941
|
cancellationRequestedAt: new Date('2025-01-01T00:01:00.000Z'),
|
|
925
942
|
});
|
|
@@ -930,16 +947,16 @@ describe('listProvisionerTerminateIntents', () => {
|
|
|
930
947
|
});
|
|
931
948
|
|
|
932
949
|
it('returns a deterministic subset when the limit truncates results', async () => {
|
|
933
|
-
for (const
|
|
950
|
+
for (const providerRunnerId of [
|
|
934
951
|
'provisioned-runner-c',
|
|
935
952
|
'provisioned-runner-a',
|
|
936
953
|
'provisioned-runner-b',
|
|
937
954
|
]) {
|
|
938
|
-
await
|
|
955
|
+
await createRunnerInstance({providerRunnerId});
|
|
939
956
|
await insertRunningJobRow({
|
|
940
957
|
workspaceId,
|
|
941
958
|
provisionerId,
|
|
942
|
-
|
|
959
|
+
providerRunnerId,
|
|
943
960
|
cancellationRequestedAt: new Date('2025-01-01T00:01:00.000Z'),
|
|
944
961
|
});
|
|
945
962
|
}
|
|
@@ -949,21 +966,21 @@ describe('listProvisionerTerminateIntents', () => {
|
|
|
949
966
|
expect(result).toEqual(['provisioned-runner-a', 'provisioned-runner-b']);
|
|
950
967
|
});
|
|
951
968
|
|
|
952
|
-
async function
|
|
953
|
-
|
|
969
|
+
async function createRunnerInstance(params: {
|
|
970
|
+
providerRunnerId: string;
|
|
954
971
|
provisionerId?: string;
|
|
955
972
|
state?: 'starting' | 'running' | 'stopping' | 'stopped' | 'failed' | 'terminated';
|
|
956
973
|
}) {
|
|
957
|
-
return await
|
|
974
|
+
return await providerRunnerFactory.create({
|
|
958
975
|
workspaceId,
|
|
959
976
|
provisionerId: params.provisionerId ?? provisionerId,
|
|
960
|
-
|
|
977
|
+
providerRunnerId: params.providerRunnerId,
|
|
961
978
|
state: params.state ?? 'running',
|
|
962
979
|
});
|
|
963
980
|
}
|
|
964
981
|
});
|
|
965
982
|
|
|
966
|
-
describe('
|
|
983
|
+
describe('reapStaleRunnerInstances', () => {
|
|
967
984
|
let workspaceId: string;
|
|
968
985
|
let provisionerId: string;
|
|
969
986
|
|
|
@@ -975,58 +992,83 @@ describe('reapStaleProvisionedRunners', () => {
|
|
|
975
992
|
|
|
976
993
|
it('fails stale unclaimed provisioned runners and releases reservations', async () => {
|
|
977
994
|
const reservationId = await createReservation(2);
|
|
978
|
-
await
|
|
979
|
-
|
|
995
|
+
await createRunnerInstance({
|
|
996
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
980
997
|
reservationId,
|
|
981
998
|
reportedAt: staleAt(),
|
|
982
999
|
updatedAt: staleAt(),
|
|
983
1000
|
});
|
|
984
1001
|
|
|
985
|
-
const result = await
|
|
1002
|
+
const result = await reapStaleRunnerInstances({thresholdSeconds: 60, limit: 100});
|
|
986
1003
|
|
|
987
|
-
const [
|
|
1004
|
+
const [providerRunner] = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
988
1005
|
const [reservation] = await reservationRowsFor({workspaceId, provisionerId});
|
|
989
1006
|
expect(result).toEqual({reaped: 1, reservationsReleased: 1});
|
|
990
|
-
expect(
|
|
1007
|
+
expect(providerRunner).toMatchObject({
|
|
991
1008
|
state: 'failed',
|
|
992
1009
|
reason: 'stale-provisioner',
|
|
993
1010
|
});
|
|
994
|
-
expect(
|
|
995
|
-
expect(
|
|
1011
|
+
expect(providerRunner?.failedAt).toBeInstanceOf(Date);
|
|
1012
|
+
expect(providerRunner?.reservationReleasedAt).toBeInstanceOf(Date);
|
|
996
1013
|
expect(reservation?.id).toBe(reservationId);
|
|
997
1014
|
expect(reservation?.count).toBe(1);
|
|
998
1015
|
});
|
|
999
1016
|
|
|
1017
|
+
it('fails stale planned installation capacity before provider attachment', async () => {
|
|
1018
|
+
const installationProvisioner = await provisionerTokenFactory.create({
|
|
1019
|
+
scope: 'installation',
|
|
1020
|
+
workspaceId: null,
|
|
1021
|
+
});
|
|
1022
|
+
const capacity = await createPlannedProvisionedCapacity({
|
|
1023
|
+
provisionerId: installationProvisioner.id,
|
|
1024
|
+
providerKind: 'docker',
|
|
1025
|
+
templateKey: null,
|
|
1026
|
+
});
|
|
1027
|
+
await db()
|
|
1028
|
+
.update(providerRunners)
|
|
1029
|
+
.set({reportedAt: staleAt(), updatedAt: staleAt()})
|
|
1030
|
+
.where(eq(providerRunners.id, capacity.capacityId));
|
|
1031
|
+
|
|
1032
|
+
const result = await reapStaleRunnerInstances({thresholdSeconds: 60, limit: 100});
|
|
1033
|
+
const [row] = await db()
|
|
1034
|
+
.select()
|
|
1035
|
+
.from(providerRunners)
|
|
1036
|
+
.where(eq(providerRunners.id, capacity.capacityId));
|
|
1037
|
+
|
|
1038
|
+
expect(result.reaped).toBe(1);
|
|
1039
|
+
expect(row).toMatchObject({workspaceId: null, providerRunnerId: null, state: 'failed'});
|
|
1040
|
+
});
|
|
1041
|
+
|
|
1000
1042
|
it('skips fresh rows, live provisioners, terminal rows, running jobs, and fresh sessions', async () => {
|
|
1001
|
-
await
|
|
1002
|
-
|
|
1043
|
+
await createRunnerInstance({
|
|
1044
|
+
providerRunnerId: 'fresh-row',
|
|
1003
1045
|
reportedAt: new Date(),
|
|
1004
1046
|
updatedAt: new Date(),
|
|
1005
1047
|
});
|
|
1006
|
-
await
|
|
1007
|
-
|
|
1048
|
+
await createRunnerInstance({
|
|
1049
|
+
providerRunnerId: 'live-provisioner',
|
|
1008
1050
|
reportedAt: staleAt(),
|
|
1009
1051
|
updatedAt: staleAt(),
|
|
1010
1052
|
});
|
|
1011
|
-
await
|
|
1012
|
-
|
|
1053
|
+
await createRunnerInstance({
|
|
1054
|
+
providerRunnerId: 'terminal-row',
|
|
1013
1055
|
reportedAt: staleAt(),
|
|
1014
1056
|
updatedAt: staleAt(),
|
|
1015
1057
|
state: 'failed',
|
|
1016
1058
|
});
|
|
1017
|
-
await
|
|
1018
|
-
|
|
1059
|
+
await createRunnerInstance({
|
|
1060
|
+
providerRunnerId: 'running-job',
|
|
1019
1061
|
reportedAt: staleAt(),
|
|
1020
1062
|
updatedAt: staleAt(),
|
|
1021
1063
|
});
|
|
1022
|
-
await insertRunningJob({
|
|
1023
|
-
const freshSession = await
|
|
1024
|
-
|
|
1064
|
+
await insertRunningJob({providerRunnerId: 'running-job'});
|
|
1065
|
+
const freshSession = await createRunnerInstance({
|
|
1066
|
+
providerRunnerId: 'fresh-session',
|
|
1025
1067
|
reportedAt: staleAt(),
|
|
1026
1068
|
updatedAt: staleAt(),
|
|
1027
1069
|
});
|
|
1028
1070
|
await createLinkedSession({
|
|
1029
|
-
|
|
1071
|
+
providerRunnerId: freshSession.providerRunnerId,
|
|
1030
1072
|
updatedAt: new Date(),
|
|
1031
1073
|
});
|
|
1032
1074
|
await db()
|
|
@@ -1034,13 +1076,13 @@ describe('reapStaleProvisionedRunners', () => {
|
|
|
1034
1076
|
.set({lastSeenAt: new Date()})
|
|
1035
1077
|
.where(eq(provisionerTokens.id, provisionerId));
|
|
1036
1078
|
|
|
1037
|
-
const result = await
|
|
1079
|
+
const result = await reapStaleRunnerInstances({thresholdSeconds: 60, limit: 100});
|
|
1038
1080
|
|
|
1039
|
-
const rows = await
|
|
1040
|
-
|
|
1081
|
+
const rows = await providerRunnerRowsFor({workspaceId, provisionerId}).orderBy(
|
|
1082
|
+
providerRunners.providerRunnerId,
|
|
1041
1083
|
);
|
|
1042
1084
|
expect(result).toEqual({reaped: 0, reservationsReleased: 0});
|
|
1043
|
-
expect(rows.map((row) => [row.
|
|
1085
|
+
expect(rows.map((row) => [row.providerRunnerId, row.state])).toEqual([
|
|
1044
1086
|
['fresh-row', 'running'],
|
|
1045
1087
|
['fresh-session', 'running'],
|
|
1046
1088
|
['live-provisioner', 'running'],
|
|
@@ -1051,29 +1093,29 @@ describe('reapStaleProvisionedRunners', () => {
|
|
|
1051
1093
|
|
|
1052
1094
|
it('releases reservations for stale rows whose linked session is no longer live', async () => {
|
|
1053
1095
|
const reservationId = await createReservation(1);
|
|
1054
|
-
await
|
|
1055
|
-
|
|
1096
|
+
await createRunnerInstance({
|
|
1097
|
+
providerRunnerId: 'stale-session',
|
|
1056
1098
|
reservationId,
|
|
1057
1099
|
runnerSessionId: '00000000-0000-4000-8000-000000000001',
|
|
1058
1100
|
reportedAt: staleAt(),
|
|
1059
1101
|
updatedAt: staleAt(),
|
|
1060
1102
|
});
|
|
1061
1103
|
const session = await createLinkedSession({
|
|
1062
|
-
|
|
1104
|
+
providerRunnerId: 'stale-session',
|
|
1063
1105
|
updatedAt: staleAt(),
|
|
1064
1106
|
});
|
|
1065
1107
|
await db()
|
|
1066
|
-
.update(
|
|
1108
|
+
.update(providerRunners)
|
|
1067
1109
|
.set({runnerSessionId: session.id})
|
|
1068
|
-
.where(eq(
|
|
1110
|
+
.where(eq(providerRunners.providerRunnerId, 'stale-session'));
|
|
1069
1111
|
|
|
1070
|
-
const result = await
|
|
1112
|
+
const result = await reapStaleRunnerInstances({thresholdSeconds: 60, limit: 100});
|
|
1071
1113
|
|
|
1072
1114
|
const reservationRows = await reservationRowsFor({workspaceId, provisionerId});
|
|
1073
|
-
const [
|
|
1115
|
+
const [providerRunner] = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
1074
1116
|
expect(result).toEqual({reaped: 1, reservationsReleased: 1});
|
|
1075
|
-
expect(
|
|
1076
|
-
expect(
|
|
1117
|
+
expect(providerRunner?.state).toBe('failed');
|
|
1118
|
+
expect(providerRunner?.reservationReleasedAt).toBeInstanceOf(Date);
|
|
1077
1119
|
expect(reservationRows.find((row) => row.id === reservationId)).toBeUndefined();
|
|
1078
1120
|
});
|
|
1079
1121
|
|
|
@@ -1082,36 +1124,36 @@ describe('reapStaleProvisionedRunners', () => {
|
|
|
1082
1124
|
const expiredReservationId = await createReservation(1, {
|
|
1083
1125
|
expiresAt: new Date(Date.now() - 60_000),
|
|
1084
1126
|
});
|
|
1085
|
-
await
|
|
1086
|
-
|
|
1127
|
+
await createRunnerInstance({
|
|
1128
|
+
providerRunnerId: 'stale-1',
|
|
1087
1129
|
reservationId: activeReservationId,
|
|
1088
1130
|
reportedAt: staleAt(240_000),
|
|
1089
1131
|
updatedAt: staleAt(240_000),
|
|
1090
1132
|
});
|
|
1091
|
-
await
|
|
1092
|
-
|
|
1133
|
+
await createRunnerInstance({
|
|
1134
|
+
providerRunnerId: 'stale-2',
|
|
1093
1135
|
reservationId: expiredReservationId,
|
|
1094
1136
|
reportedAt: staleAt(180_000),
|
|
1095
1137
|
updatedAt: staleAt(180_000),
|
|
1096
1138
|
});
|
|
1097
|
-
await
|
|
1098
|
-
|
|
1139
|
+
await createRunnerInstance({
|
|
1140
|
+
providerRunnerId: 'stale-3',
|
|
1099
1141
|
reportedAt: staleAt(120_000),
|
|
1100
1142
|
updatedAt: staleAt(120_000),
|
|
1101
1143
|
});
|
|
1102
1144
|
|
|
1103
|
-
const first = await
|
|
1104
|
-
const second = await
|
|
1145
|
+
const first = await reapStaleRunnerInstances({thresholdSeconds: 60, limit: 2});
|
|
1146
|
+
const second = await reapStaleRunnerInstances({thresholdSeconds: 60, limit: 2});
|
|
1105
1147
|
|
|
1106
1148
|
const reservationRows = await reservationRowsFor({workspaceId, provisionerId});
|
|
1107
|
-
const
|
|
1149
|
+
const providerRunnerRows = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
1108
1150
|
expect(first).toEqual({reaped: 2, reservationsReleased: 1});
|
|
1109
1151
|
expect(second).toEqual({reaped: 1, reservationsReleased: 0});
|
|
1110
1152
|
expect(reservationRows.find((row) => row.id === activeReservationId)?.count).toBe(1);
|
|
1111
1153
|
expect(reservationRows.find((row) => row.id === expiredReservationId)?.count).toBe(1);
|
|
1112
|
-
expect(
|
|
1154
|
+
expect(providerRunnerRows.every((row) => row.state === 'failed')).toBe(true);
|
|
1113
1155
|
expect(
|
|
1114
|
-
|
|
1156
|
+
providerRunnerRows
|
|
1115
1157
|
.filter((row) => row.reservationId)
|
|
1116
1158
|
.every((row) => row.reservationReleasedAt instanceof Date),
|
|
1117
1159
|
).toBe(true);
|
|
@@ -1119,8 +1161,8 @@ describe('reapStaleProvisionedRunners', () => {
|
|
|
1119
1161
|
|
|
1120
1162
|
it('does not double-release reservations when terminal report and reaper queue on the workspace lock', async () => {
|
|
1121
1163
|
const reservationId = await createReservation(2);
|
|
1122
|
-
await
|
|
1123
|
-
|
|
1164
|
+
await createRunnerInstance({
|
|
1165
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
1124
1166
|
reservationId,
|
|
1125
1167
|
reportedAt: staleAt(),
|
|
1126
1168
|
updatedAt: staleAt(),
|
|
@@ -1134,12 +1176,12 @@ describe('reapStaleProvisionedRunners', () => {
|
|
|
1134
1176
|
});
|
|
1135
1177
|
|
|
1136
1178
|
await lockHolderReady.promise;
|
|
1137
|
-
const report =
|
|
1179
|
+
const report = reportRunnerInstances({
|
|
1138
1180
|
workspaceId,
|
|
1139
1181
|
provisionerId,
|
|
1140
1182
|
events: [
|
|
1141
1183
|
event({
|
|
1142
|
-
|
|
1184
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
1143
1185
|
reservationId,
|
|
1144
1186
|
state: 'failed',
|
|
1145
1187
|
reportedAt: new Date(),
|
|
@@ -1147,7 +1189,7 @@ describe('reapStaleProvisionedRunners', () => {
|
|
|
1147
1189
|
],
|
|
1148
1190
|
});
|
|
1149
1191
|
await waitForLockWait({queryLike: '%pg_advisory_xact_lock%'});
|
|
1150
|
-
const reaper =
|
|
1192
|
+
const reaper = reapStaleRunnerInstances({thresholdSeconds: 60, limit: 100});
|
|
1151
1193
|
try {
|
|
1152
1194
|
await waitForLockWait({minWaiters: 2, queryLike: '%pg_advisory_xact_lock%'});
|
|
1153
1195
|
} finally {
|
|
@@ -1156,9 +1198,9 @@ describe('reapStaleProvisionedRunners', () => {
|
|
|
1156
1198
|
const [reportResult, reaperResult] = await Promise.all([report, reaper, lockHolder]);
|
|
1157
1199
|
|
|
1158
1200
|
const [reservation] = await reservationRowsFor({workspaceId, provisionerId});
|
|
1159
|
-
const [
|
|
1201
|
+
const [providerRunner] = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
1160
1202
|
expect(reportResult.reservationsReleased + reaperResult.reservationsReleased).toBe(1);
|
|
1161
|
-
expect(
|
|
1203
|
+
expect(providerRunner?.reservationReleasedAt).toBeInstanceOf(Date);
|
|
1162
1204
|
expect(reservation?.count).toBe(1);
|
|
1163
1205
|
});
|
|
1164
1206
|
|
|
@@ -1185,38 +1227,38 @@ describe('reapStaleProvisionedRunners', () => {
|
|
|
1185
1227
|
return reservation.id;
|
|
1186
1228
|
}
|
|
1187
1229
|
|
|
1188
|
-
async function
|
|
1189
|
-
|
|
1230
|
+
async function createRunnerInstance(params: {
|
|
1231
|
+
providerRunnerId: string;
|
|
1190
1232
|
reservationId?: string | null;
|
|
1191
1233
|
runnerSessionId?: string | null;
|
|
1192
1234
|
reportedAt: Date;
|
|
1193
1235
|
updatedAt: Date;
|
|
1194
1236
|
state?: 'starting' | 'running' | 'stopping' | 'stopped' | 'failed' | 'terminated';
|
|
1195
1237
|
}) {
|
|
1196
|
-
const row = await
|
|
1238
|
+
const row = await providerRunnerFactory.create({
|
|
1197
1239
|
workspaceId,
|
|
1198
1240
|
provisionerId,
|
|
1199
|
-
|
|
1241
|
+
providerRunnerId: params.providerRunnerId,
|
|
1200
1242
|
reservationId: params.reservationId ?? null,
|
|
1201
1243
|
runnerSessionId: params.runnerSessionId ?? null,
|
|
1202
1244
|
reportedAt: params.reportedAt,
|
|
1203
1245
|
state: params.state ?? 'running',
|
|
1204
1246
|
});
|
|
1205
1247
|
await db()
|
|
1206
|
-
.update(
|
|
1248
|
+
.update(providerRunners)
|
|
1207
1249
|
.set({updatedAt: params.updatedAt})
|
|
1208
|
-
.where(eq(
|
|
1250
|
+
.where(eq(providerRunners.id, row.id));
|
|
1209
1251
|
return {...row, updatedAt: params.updatedAt};
|
|
1210
1252
|
}
|
|
1211
1253
|
|
|
1212
|
-
async function createLinkedSession(params: {
|
|
1254
|
+
async function createLinkedSession(params: {providerRunnerId: string; updatedAt: Date}) {
|
|
1213
1255
|
const session = await runnerSessionFactory.create({workspaceId});
|
|
1214
1256
|
await db()
|
|
1215
1257
|
.update(runnerSessions)
|
|
1216
1258
|
.set({
|
|
1217
1259
|
registrationTokenKind: 'ephemeral',
|
|
1218
1260
|
provisionerId,
|
|
1219
|
-
|
|
1261
|
+
providerRunnerId: params.providerRunnerId,
|
|
1220
1262
|
maxClaims: 1,
|
|
1221
1263
|
updatedAt: params.updatedAt,
|
|
1222
1264
|
})
|
|
@@ -1224,7 +1266,7 @@ describe('reapStaleProvisionedRunners', () => {
|
|
|
1224
1266
|
return session;
|
|
1225
1267
|
}
|
|
1226
1268
|
|
|
1227
|
-
async function insertRunningJob(params: {
|
|
1269
|
+
async function insertRunningJob(params: {providerRunnerId: string}) {
|
|
1228
1270
|
const runnerSession = await runnerSessionFactory.create({workspaceId});
|
|
1229
1271
|
|
|
1230
1272
|
await db()
|
|
@@ -1238,7 +1280,7 @@ describe('reapStaleProvisionedRunners', () => {
|
|
|
1238
1280
|
projectId: crypto.randomUUID(),
|
|
1239
1281
|
runnerSessionId: runnerSession.id,
|
|
1240
1282
|
provisionerId,
|
|
1241
|
-
|
|
1283
|
+
providerRunnerId: params.providerRunnerId,
|
|
1242
1284
|
requiredLabels: ['linux'],
|
|
1243
1285
|
runnerLabels: ['linux'],
|
|
1244
1286
|
startedAt: new Date('2025-01-01T00:00:00.000Z'),
|
|
@@ -1251,13 +1293,13 @@ describe('reapStaleProvisionedRunners', () => {
|
|
|
1251
1293
|
}
|
|
1252
1294
|
|
|
1253
1295
|
function event(params: {
|
|
1254
|
-
|
|
1296
|
+
providerRunnerId: string;
|
|
1255
1297
|
reservationId?: string | null;
|
|
1256
1298
|
state?: 'starting' | 'running' | 'stopping' | 'stopped' | 'failed' | 'terminated';
|
|
1257
1299
|
reportedAt?: Date;
|
|
1258
1300
|
}) {
|
|
1259
1301
|
return {
|
|
1260
|
-
|
|
1302
|
+
providerRunnerId: params.providerRunnerId,
|
|
1261
1303
|
reservationId: params.reservationId ?? null,
|
|
1262
1304
|
templateKey: 'linux',
|
|
1263
1305
|
labels: ['linux'],
|
|
@@ -1270,7 +1312,7 @@ describe('reapStaleProvisionedRunners', () => {
|
|
|
1270
1312
|
}
|
|
1271
1313
|
});
|
|
1272
1314
|
|
|
1273
|
-
describe('
|
|
1315
|
+
describe('reconcileRunnerInstances', () => {
|
|
1274
1316
|
let workspaceId: string;
|
|
1275
1317
|
let provisionerId: string;
|
|
1276
1318
|
|
|
@@ -1281,76 +1323,76 @@ describe('reconcileProvisionedRunners', () => {
|
|
|
1281
1323
|
|
|
1282
1324
|
it('terminates stale absent provisioned runners and releases reservations', async () => {
|
|
1283
1325
|
const reservationId = await createReservation(2);
|
|
1284
|
-
await
|
|
1285
|
-
|
|
1326
|
+
await createRunnerInstance({
|
|
1327
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
1286
1328
|
reservationId,
|
|
1287
1329
|
reportedAt: staleReportedAt(),
|
|
1288
1330
|
});
|
|
1289
1331
|
|
|
1290
|
-
const result = await
|
|
1332
|
+
const result = await reconcileRunnerInstances({
|
|
1291
1333
|
workspaceId,
|
|
1292
1334
|
provisionerId,
|
|
1293
|
-
|
|
1335
|
+
observedRunnerInstanceIds: ['observed-runner'],
|
|
1294
1336
|
terminateGraceSeconds: 60,
|
|
1295
1337
|
});
|
|
1296
1338
|
|
|
1297
|
-
const [
|
|
1339
|
+
const [providerRunner] = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
1298
1340
|
const [reservation] = await reservationRowsFor({workspaceId, provisionerId});
|
|
1299
1341
|
expect(result.absentIds).toEqual(['provisioned-runner-1']);
|
|
1300
1342
|
expect(result.reservationsReleased).toBe(1);
|
|
1301
|
-
expect(
|
|
1302
|
-
expect(
|
|
1343
|
+
expect(providerRunner?.state).toBe('terminated');
|
|
1344
|
+
expect(providerRunner?.terminatedAt).toBeInstanceOf(Date);
|
|
1303
1345
|
expect(reservation?.count).toBe(1);
|
|
1304
1346
|
});
|
|
1305
1347
|
|
|
1306
1348
|
it('treats an empty observed set as read-only', async () => {
|
|
1307
1349
|
const reservationId = await createReservation(2);
|
|
1308
|
-
await
|
|
1309
|
-
|
|
1350
|
+
await createRunnerInstance({
|
|
1351
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
1310
1352
|
reservationId,
|
|
1311
1353
|
reportedAt: staleReportedAt(),
|
|
1312
1354
|
});
|
|
1313
1355
|
|
|
1314
|
-
const result = await
|
|
1356
|
+
const result = await reconcileRunnerInstances({
|
|
1315
1357
|
workspaceId,
|
|
1316
1358
|
provisionerId,
|
|
1317
|
-
|
|
1359
|
+
observedRunnerInstanceIds: [],
|
|
1318
1360
|
terminateGraceSeconds: 60,
|
|
1319
1361
|
});
|
|
1320
1362
|
|
|
1321
|
-
const [
|
|
1363
|
+
const [providerRunner] = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
1322
1364
|
const [reservation] = await reservationRowsFor({workspaceId, provisionerId});
|
|
1323
1365
|
expect(result.absentIds).toEqual([]);
|
|
1324
1366
|
expect(result.reservationsReleased).toBe(0);
|
|
1325
|
-
expect(
|
|
1326
|
-
expect(
|
|
1327
|
-
expect(
|
|
1367
|
+
expect(providerRunner?.state).toBe('running');
|
|
1368
|
+
expect(providerRunner?.terminatedAt).toBeNull();
|
|
1369
|
+
expect(providerRunner?.reservationReleasedAt).toBeNull();
|
|
1328
1370
|
expect(reservation?.count).toBe(2);
|
|
1329
1371
|
});
|
|
1330
1372
|
|
|
1331
1373
|
it('keeps fresh absent provisioned runners inside the grace window', async () => {
|
|
1332
|
-
await
|
|
1333
|
-
|
|
1374
|
+
await createRunnerInstance({
|
|
1375
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
1334
1376
|
reportedAt: new Date(),
|
|
1335
1377
|
});
|
|
1336
1378
|
|
|
1337
|
-
const result = await
|
|
1379
|
+
const result = await reconcileRunnerInstances({
|
|
1338
1380
|
workspaceId,
|
|
1339
1381
|
provisionerId,
|
|
1340
|
-
|
|
1382
|
+
observedRunnerInstanceIds: ['observed-runner'],
|
|
1341
1383
|
terminateGraceSeconds: 60,
|
|
1342
1384
|
});
|
|
1343
1385
|
|
|
1344
|
-
const [
|
|
1386
|
+
const [providerRunner] = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
1345
1387
|
expect(result.absentIds).toEqual([]);
|
|
1346
1388
|
expect(result.reservationsReleased).toBe(0);
|
|
1347
|
-
expect(
|
|
1389
|
+
expect(providerRunner?.state).toBe('running');
|
|
1348
1390
|
});
|
|
1349
1391
|
|
|
1350
1392
|
it('respects a fresh report that commits after reconcile selects a stale absent row', async () => {
|
|
1351
1393
|
const reservationId = await createReservation(1);
|
|
1352
|
-
await
|
|
1353
|
-
|
|
1394
|
+
await createRunnerInstance({
|
|
1395
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
1354
1396
|
reservationId,
|
|
1355
1397
|
reportedAt: staleReportedAt(),
|
|
1356
1398
|
});
|
|
@@ -1359,13 +1401,13 @@ describe('reconcileProvisionedRunners', () => {
|
|
|
1359
1401
|
|
|
1360
1402
|
const reportTransaction = db().transaction(async (tx) => {
|
|
1361
1403
|
await tx
|
|
1362
|
-
.update(
|
|
1404
|
+
.update(providerRunners)
|
|
1363
1405
|
.set({reportedAt: sql`now()`, updatedAt: sql`now()`})
|
|
1364
1406
|
.where(
|
|
1365
1407
|
and(
|
|
1366
|
-
eq(
|
|
1367
|
-
eq(
|
|
1368
|
-
eq(
|
|
1408
|
+
eq(providerRunners.workspaceId, workspaceId),
|
|
1409
|
+
eq(providerRunners.provisionerId, provisionerId),
|
|
1410
|
+
eq(providerRunners.providerRunnerId, 'provisioned-runner-1'),
|
|
1369
1411
|
),
|
|
1370
1412
|
);
|
|
1371
1413
|
reportTransactionUpdated.resolve();
|
|
@@ -1373,50 +1415,50 @@ describe('reconcileProvisionedRunners', () => {
|
|
|
1373
1415
|
});
|
|
1374
1416
|
|
|
1375
1417
|
await reportTransactionUpdated.promise;
|
|
1376
|
-
const reconcile =
|
|
1418
|
+
const reconcile = reconcileRunnerInstances({
|
|
1377
1419
|
workspaceId,
|
|
1378
1420
|
provisionerId,
|
|
1379
|
-
|
|
1421
|
+
observedRunnerInstanceIds: ['observed-runner'],
|
|
1380
1422
|
terminateGraceSeconds: 60,
|
|
1381
1423
|
});
|
|
1382
1424
|
try {
|
|
1383
|
-
await waitForLockWait({queryLike: '%
|
|
1425
|
+
await waitForLockWait({queryLike: '%runner_instances%'});
|
|
1384
1426
|
} finally {
|
|
1385
1427
|
releaseReportTransaction.resolve();
|
|
1386
1428
|
}
|
|
1387
1429
|
const [result] = await Promise.all([reconcile, reportTransaction]);
|
|
1388
1430
|
|
|
1389
|
-
const [
|
|
1431
|
+
const [providerRunner] = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
1390
1432
|
const [reservation] = await reservationRowsFor({workspaceId, provisionerId});
|
|
1391
1433
|
expect(result.absentIds).toEqual([]);
|
|
1392
1434
|
expect(result.reservationsReleased).toBe(0);
|
|
1393
|
-
expect(
|
|
1394
|
-
expect(
|
|
1435
|
+
expect(providerRunner?.state).toBe('running');
|
|
1436
|
+
expect(providerRunner?.reservationReleasedAt).toBeNull();
|
|
1395
1437
|
expect(reservation?.count).toBe(1);
|
|
1396
1438
|
});
|
|
1397
1439
|
|
|
1398
1440
|
it('terminates only stale rows when the observed set is non-empty', async () => {
|
|
1399
|
-
await
|
|
1400
|
-
|
|
1441
|
+
await createRunnerInstance({
|
|
1442
|
+
providerRunnerId: 'stale-runner',
|
|
1401
1443
|
reportedAt: staleReportedAt(),
|
|
1402
1444
|
});
|
|
1403
|
-
await
|
|
1404
|
-
|
|
1445
|
+
await createRunnerInstance({
|
|
1446
|
+
providerRunnerId: 'fresh-runner',
|
|
1405
1447
|
reportedAt: new Date(),
|
|
1406
1448
|
});
|
|
1407
1449
|
|
|
1408
|
-
const result = await
|
|
1450
|
+
const result = await reconcileRunnerInstances({
|
|
1409
1451
|
workspaceId,
|
|
1410
1452
|
provisionerId,
|
|
1411
|
-
|
|
1453
|
+
observedRunnerInstanceIds: ['observed-runner'],
|
|
1412
1454
|
terminateGraceSeconds: 60,
|
|
1413
1455
|
});
|
|
1414
1456
|
|
|
1415
|
-
const rows = await
|
|
1416
|
-
|
|
1457
|
+
const rows = await providerRunnerRowsFor({workspaceId, provisionerId}).orderBy(
|
|
1458
|
+
providerRunners.providerRunnerId,
|
|
1417
1459
|
);
|
|
1418
1460
|
expect(result.absentIds).toEqual(['stale-runner']);
|
|
1419
|
-
expect(rows.map((row) => [row.
|
|
1461
|
+
expect(rows.map((row) => [row.providerRunnerId, row.state])).toEqual([
|
|
1420
1462
|
['fresh-runner', 'running'],
|
|
1421
1463
|
['stale-runner', 'terminated'],
|
|
1422
1464
|
]);
|
|
@@ -1428,64 +1470,62 @@ describe('reconcileProvisionedRunners', () => {
|
|
|
1428
1470
|
const expiredReservationId = await createReservation(1, {
|
|
1429
1471
|
expiresAt: new Date(Date.now() - 60_000),
|
|
1430
1472
|
});
|
|
1431
|
-
await
|
|
1432
|
-
|
|
1473
|
+
await createRunnerInstance({
|
|
1474
|
+
providerRunnerId: 'shared-1',
|
|
1433
1475
|
reservationId: sharedReservationId,
|
|
1434
1476
|
reportedAt: staleReportedAt(),
|
|
1435
1477
|
});
|
|
1436
|
-
await
|
|
1437
|
-
|
|
1478
|
+
await createRunnerInstance({
|
|
1479
|
+
providerRunnerId: 'shared-2',
|
|
1438
1480
|
reservationId: sharedReservationId,
|
|
1439
1481
|
reportedAt: staleReportedAt(),
|
|
1440
1482
|
});
|
|
1441
|
-
await
|
|
1442
|
-
|
|
1483
|
+
await createRunnerInstance({
|
|
1484
|
+
providerRunnerId: 'one-unit',
|
|
1443
1485
|
reservationId: oneUnitReservationId,
|
|
1444
1486
|
reportedAt: staleReportedAt(),
|
|
1445
1487
|
});
|
|
1446
|
-
await
|
|
1447
|
-
|
|
1488
|
+
await createRunnerInstance({
|
|
1489
|
+
providerRunnerId: 'expired',
|
|
1448
1490
|
reservationId: expiredReservationId,
|
|
1449
1491
|
reportedAt: staleReportedAt(),
|
|
1450
1492
|
});
|
|
1451
1493
|
|
|
1452
|
-
const result = await
|
|
1494
|
+
const result = await reconcileRunnerInstances({
|
|
1453
1495
|
workspaceId,
|
|
1454
1496
|
provisionerId,
|
|
1455
|
-
|
|
1497
|
+
observedRunnerInstanceIds: ['observed-runner'],
|
|
1456
1498
|
terminateGraceSeconds: 60,
|
|
1457
1499
|
});
|
|
1458
1500
|
|
|
1459
1501
|
const reservationRows = await reservationRowsFor({workspaceId, provisionerId});
|
|
1460
|
-
const
|
|
1502
|
+
const providerRunnerRows = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
1461
1503
|
expect(result.reservationsReleased).toBe(3);
|
|
1462
1504
|
expect(reservationRows).toHaveLength(2);
|
|
1463
1505
|
expect(reservationRows.find((row) => row.id === sharedReservationId)?.count).toBe(1);
|
|
1464
1506
|
expect(reservationRows.find((row) => row.id === expiredReservationId)?.count).toBe(1);
|
|
1465
1507
|
expect(reservationRows.find((row) => row.id === oneUnitReservationId)).toBeUndefined();
|
|
1466
|
-
expect(
|
|
1467
|
-
true,
|
|
1468
|
-
);
|
|
1508
|
+
expect(providerRunnerRows.every((row) => row.reservationReleasedAt instanceof Date)).toBe(true);
|
|
1469
1509
|
});
|
|
1470
1510
|
|
|
1471
1511
|
it('is idempotent across repeated reconciles', async () => {
|
|
1472
1512
|
const reservationId = await createReservation(2);
|
|
1473
|
-
await
|
|
1474
|
-
|
|
1513
|
+
await createRunnerInstance({
|
|
1514
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
1475
1515
|
reservationId,
|
|
1476
1516
|
reportedAt: staleReportedAt(),
|
|
1477
1517
|
});
|
|
1478
1518
|
|
|
1479
|
-
const first = await
|
|
1519
|
+
const first = await reconcileRunnerInstances({
|
|
1480
1520
|
workspaceId,
|
|
1481
1521
|
provisionerId,
|
|
1482
|
-
|
|
1522
|
+
observedRunnerInstanceIds: ['observed-runner'],
|
|
1483
1523
|
terminateGraceSeconds: 60,
|
|
1484
1524
|
});
|
|
1485
|
-
const second = await
|
|
1525
|
+
const second = await reconcileRunnerInstances({
|
|
1486
1526
|
workspaceId,
|
|
1487
1527
|
provisionerId,
|
|
1488
|
-
|
|
1528
|
+
observedRunnerInstanceIds: ['observed-runner'],
|
|
1489
1529
|
terminateGraceSeconds: 60,
|
|
1490
1530
|
});
|
|
1491
1531
|
|
|
@@ -1498,53 +1538,53 @@ describe('reconcileProvisionedRunners', () => {
|
|
|
1498
1538
|
it('does not touch provisioned runners from another workspace or provisioner', async () => {
|
|
1499
1539
|
const otherWorkspaceId = crypto.randomUUID();
|
|
1500
1540
|
const otherProvisionerId = crypto.randomUUID();
|
|
1501
|
-
await
|
|
1502
|
-
|
|
1541
|
+
await createRunnerInstance({
|
|
1542
|
+
providerRunnerId: 'owned-runner',
|
|
1503
1543
|
reportedAt: staleReportedAt(),
|
|
1504
1544
|
});
|
|
1505
|
-
await
|
|
1545
|
+
await providerRunnerFactory.create({
|
|
1506
1546
|
workspaceId: otherWorkspaceId,
|
|
1507
1547
|
provisionerId,
|
|
1508
|
-
|
|
1548
|
+
providerRunnerId: 'other-workspace-runner',
|
|
1509
1549
|
reportedAt: staleReportedAt(),
|
|
1510
1550
|
state: 'running',
|
|
1511
1551
|
});
|
|
1512
|
-
await
|
|
1552
|
+
await providerRunnerFactory.create({
|
|
1513
1553
|
workspaceId,
|
|
1514
1554
|
provisionerId: otherProvisionerId,
|
|
1515
|
-
|
|
1555
|
+
providerRunnerId: 'other-provisioner-runner',
|
|
1516
1556
|
reportedAt: staleReportedAt(),
|
|
1517
1557
|
state: 'running',
|
|
1518
1558
|
});
|
|
1519
1559
|
|
|
1520
|
-
await
|
|
1560
|
+
await reconcileRunnerInstances({
|
|
1521
1561
|
workspaceId,
|
|
1522
1562
|
provisionerId,
|
|
1523
|
-
|
|
1563
|
+
observedRunnerInstanceIds: ['observed-runner'],
|
|
1524
1564
|
terminateGraceSeconds: 60,
|
|
1525
1565
|
});
|
|
1526
1566
|
|
|
1527
1567
|
const rows = await db()
|
|
1528
1568
|
.select()
|
|
1529
|
-
.from(
|
|
1569
|
+
.from(providerRunners)
|
|
1530
1570
|
.where(
|
|
1531
1571
|
or(
|
|
1532
1572
|
and(
|
|
1533
|
-
eq(
|
|
1534
|
-
eq(
|
|
1573
|
+
eq(providerRunners.workspaceId, workspaceId),
|
|
1574
|
+
eq(providerRunners.provisionerId, provisionerId),
|
|
1535
1575
|
),
|
|
1536
1576
|
and(
|
|
1537
|
-
eq(
|
|
1538
|
-
eq(
|
|
1577
|
+
eq(providerRunners.workspaceId, otherWorkspaceId),
|
|
1578
|
+
eq(providerRunners.provisionerId, provisionerId),
|
|
1539
1579
|
),
|
|
1540
1580
|
and(
|
|
1541
|
-
eq(
|
|
1542
|
-
eq(
|
|
1581
|
+
eq(providerRunners.workspaceId, workspaceId),
|
|
1582
|
+
eq(providerRunners.provisionerId, otherProvisionerId),
|
|
1543
1583
|
),
|
|
1544
1584
|
),
|
|
1545
1585
|
)
|
|
1546
|
-
.orderBy(
|
|
1547
|
-
expect(rows.map((row) => [row.
|
|
1586
|
+
.orderBy(providerRunners.providerRunnerId);
|
|
1587
|
+
expect(rows.map((row) => [row.providerRunnerId, row.state])).toEqual([
|
|
1548
1588
|
['other-provisioner-runner', 'running'],
|
|
1549
1589
|
['other-workspace-runner', 'running'],
|
|
1550
1590
|
['owned-runner', 'terminated'],
|
|
@@ -1553,30 +1593,30 @@ describe('reconcileProvisionedRunners', () => {
|
|
|
1553
1593
|
|
|
1554
1594
|
it('terminates session-bound absent runners without releasing their reservation', async () => {
|
|
1555
1595
|
const reservationId = await createReservation(1);
|
|
1556
|
-
await
|
|
1557
|
-
|
|
1596
|
+
await createRunnerInstance({
|
|
1597
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
1558
1598
|
reservationId,
|
|
1559
1599
|
runnerSessionId: crypto.randomUUID(),
|
|
1560
1600
|
reportedAt: staleReportedAt(),
|
|
1561
1601
|
});
|
|
1562
1602
|
|
|
1563
|
-
const result = await
|
|
1603
|
+
const result = await reconcileRunnerInstances({
|
|
1564
1604
|
workspaceId,
|
|
1565
1605
|
provisionerId,
|
|
1566
|
-
|
|
1606
|
+
observedRunnerInstanceIds: ['observed-runner'],
|
|
1567
1607
|
terminateGraceSeconds: 60,
|
|
1568
1608
|
});
|
|
1569
1609
|
|
|
1570
|
-
const [
|
|
1610
|
+
const [providerRunner] = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
1571
1611
|
const [reservation] = await reservationRowsFor({workspaceId, provisionerId});
|
|
1572
1612
|
expect(result.reservationsReleased).toBe(0);
|
|
1573
|
-
expect(
|
|
1574
|
-
expect(
|
|
1613
|
+
expect(providerRunner?.state).toBe('terminated');
|
|
1614
|
+
expect(providerRunner?.reservationReleasedAt).toBeNull();
|
|
1575
1615
|
expect(reservation?.count).toBe(1);
|
|
1576
1616
|
});
|
|
1577
1617
|
|
|
1578
1618
|
it('returns a deterministic newest running job execution bound to an observed provisioned runner', async () => {
|
|
1579
|
-
await
|
|
1619
|
+
await createRunnerInstance({providerRunnerId: 'provisioned-runner-1'});
|
|
1580
1620
|
const lowerJobId = '00000000-0000-4000-8000-000000000001';
|
|
1581
1621
|
const higherJobId = '00000000-0000-4000-8000-000000000002';
|
|
1582
1622
|
const lowerJobExecutionId = '10000000-0000-4000-8000-000000000001';
|
|
@@ -1584,58 +1624,58 @@ describe('reconcileProvisionedRunners', () => {
|
|
|
1584
1624
|
await insertRunningJob({
|
|
1585
1625
|
jobId: lowerJobId,
|
|
1586
1626
|
jobExecutionId: lowerJobExecutionId,
|
|
1587
|
-
|
|
1627
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
1588
1628
|
startedAt: new Date('2025-01-01T00:00:00.000Z'),
|
|
1589
1629
|
});
|
|
1590
1630
|
await insertRunningJob({
|
|
1591
1631
|
jobId: higherJobId,
|
|
1592
1632
|
jobExecutionId: higherJobExecutionId,
|
|
1593
|
-
|
|
1633
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
1594
1634
|
startedAt: new Date('2025-01-01T00:00:00.000Z'),
|
|
1595
1635
|
});
|
|
1596
1636
|
|
|
1597
|
-
const result = await
|
|
1637
|
+
const result = await reconcileRunnerInstances({
|
|
1598
1638
|
workspaceId,
|
|
1599
1639
|
provisionerId,
|
|
1600
|
-
|
|
1640
|
+
observedRunnerInstanceIds: ['provisioned-runner-1'],
|
|
1601
1641
|
terminateGraceSeconds: 60,
|
|
1602
1642
|
});
|
|
1603
1643
|
|
|
1604
1644
|
expect(
|
|
1605
|
-
result.
|
|
1645
|
+
result.boundJobExecutionsByRunnerInstanceId.get('provisioned-runner-1')?.jobExecutionId,
|
|
1606
1646
|
).toBe(higherJobExecutionId);
|
|
1607
1647
|
});
|
|
1608
1648
|
|
|
1609
1649
|
it('does not let a later running report revive a reconcile-terminated runner', async () => {
|
|
1610
1650
|
const reportedAt = staleReportedAt();
|
|
1611
|
-
await
|
|
1612
|
-
await
|
|
1651
|
+
await createRunnerInstance({providerRunnerId: 'provisioned-runner-1', reportedAt});
|
|
1652
|
+
await reconcileRunnerInstances({
|
|
1613
1653
|
workspaceId,
|
|
1614
1654
|
provisionerId,
|
|
1615
|
-
|
|
1655
|
+
observedRunnerInstanceIds: ['observed-runner'],
|
|
1616
1656
|
terminateGraceSeconds: 60,
|
|
1617
1657
|
});
|
|
1618
1658
|
|
|
1619
|
-
await
|
|
1659
|
+
await reportRunnerInstances({
|
|
1620
1660
|
workspaceId,
|
|
1621
1661
|
provisionerId,
|
|
1622
1662
|
events: [
|
|
1623
1663
|
event({
|
|
1624
|
-
|
|
1664
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
1625
1665
|
state: 'running',
|
|
1626
1666
|
reportedAt: new Date(reportedAt.getTime() + 120_000),
|
|
1627
1667
|
}),
|
|
1628
1668
|
],
|
|
1629
1669
|
});
|
|
1630
1670
|
|
|
1631
|
-
const [
|
|
1632
|
-
expect(
|
|
1671
|
+
const [providerRunner] = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
1672
|
+
expect(providerRunner?.state).toBe('terminated');
|
|
1633
1673
|
});
|
|
1634
1674
|
|
|
1635
1675
|
it('does not double-release reservations when terminal report and reconcile queue on the workspace lock', async () => {
|
|
1636
1676
|
const reservationId = await createReservation(2);
|
|
1637
|
-
await
|
|
1638
|
-
|
|
1677
|
+
await createRunnerInstance({
|
|
1678
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
1639
1679
|
reservationId,
|
|
1640
1680
|
reportedAt: staleReportedAt(),
|
|
1641
1681
|
});
|
|
@@ -1648,12 +1688,12 @@ describe('reconcileProvisionedRunners', () => {
|
|
|
1648
1688
|
});
|
|
1649
1689
|
|
|
1650
1690
|
await lockHolderReady.promise;
|
|
1651
|
-
const report =
|
|
1691
|
+
const report = reportRunnerInstances({
|
|
1652
1692
|
workspaceId,
|
|
1653
1693
|
provisionerId,
|
|
1654
1694
|
events: [
|
|
1655
1695
|
event({
|
|
1656
|
-
|
|
1696
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
1657
1697
|
reservationId,
|
|
1658
1698
|
state: 'failed',
|
|
1659
1699
|
reportedAt: new Date(),
|
|
@@ -1661,10 +1701,10 @@ describe('reconcileProvisionedRunners', () => {
|
|
|
1661
1701
|
],
|
|
1662
1702
|
});
|
|
1663
1703
|
await waitForLockWait({queryLike: '%pg_advisory_xact_lock%'});
|
|
1664
|
-
const reconcile =
|
|
1704
|
+
const reconcile = reconcileRunnerInstances({
|
|
1665
1705
|
workspaceId,
|
|
1666
1706
|
provisionerId,
|
|
1667
|
-
|
|
1707
|
+
observedRunnerInstanceIds: ['observed-runner'],
|
|
1668
1708
|
terminateGraceSeconds: 60,
|
|
1669
1709
|
});
|
|
1670
1710
|
try {
|
|
@@ -1674,13 +1714,13 @@ describe('reconcileProvisionedRunners', () => {
|
|
|
1674
1714
|
}
|
|
1675
1715
|
const [reportResult, reconcileResult] = await Promise.all([report, reconcile, lockHolder]);
|
|
1676
1716
|
|
|
1677
|
-
const [
|
|
1717
|
+
const [providerRunner] = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
1678
1718
|
const [reservation] = await reservationRowsFor({workspaceId, provisionerId});
|
|
1679
1719
|
expect(reportResult.reservationsReleased + reconcileResult.reservationsReleased).toBe(1);
|
|
1680
|
-
expect(
|
|
1720
|
+
expect(providerRunner?.state).toSatisfy(
|
|
1681
1721
|
(state: string | undefined) => state === 'failed' || state === 'terminated',
|
|
1682
1722
|
);
|
|
1683
|
-
expect(
|
|
1723
|
+
expect(providerRunner?.reservationReleasedAt).toBeInstanceOf(Date);
|
|
1684
1724
|
expect(reservation?.count).toBe(1);
|
|
1685
1725
|
});
|
|
1686
1726
|
|
|
@@ -1707,16 +1747,16 @@ describe('reconcileProvisionedRunners', () => {
|
|
|
1707
1747
|
return reservation.id;
|
|
1708
1748
|
}
|
|
1709
1749
|
|
|
1710
|
-
async function
|
|
1711
|
-
|
|
1750
|
+
async function createRunnerInstance(params: {
|
|
1751
|
+
providerRunnerId: string;
|
|
1712
1752
|
reservationId?: string | null;
|
|
1713
1753
|
runnerSessionId?: string | null;
|
|
1714
1754
|
reportedAt?: Date;
|
|
1715
1755
|
}) {
|
|
1716
|
-
return await
|
|
1756
|
+
return await providerRunnerFactory.create({
|
|
1717
1757
|
workspaceId,
|
|
1718
1758
|
provisionerId,
|
|
1719
|
-
|
|
1759
|
+
providerRunnerId: params.providerRunnerId,
|
|
1720
1760
|
reservationId: params.reservationId ?? null,
|
|
1721
1761
|
runnerSessionId: params.runnerSessionId ?? null,
|
|
1722
1762
|
reportedAt: params.reportedAt ?? new Date(),
|
|
@@ -1727,7 +1767,7 @@ describe('reconcileProvisionedRunners', () => {
|
|
|
1727
1767
|
async function insertRunningJob(params: {
|
|
1728
1768
|
jobId: string;
|
|
1729
1769
|
jobExecutionId?: string;
|
|
1730
|
-
|
|
1770
|
+
providerRunnerId: string;
|
|
1731
1771
|
startedAt: Date;
|
|
1732
1772
|
}) {
|
|
1733
1773
|
const runnerSession = await runnerSessionFactory.create({workspaceId});
|
|
@@ -1743,7 +1783,7 @@ describe('reconcileProvisionedRunners', () => {
|
|
|
1743
1783
|
projectId: crypto.randomUUID(),
|
|
1744
1784
|
runnerSessionId: runnerSession.id,
|
|
1745
1785
|
provisionerId,
|
|
1746
|
-
|
|
1786
|
+
providerRunnerId: params.providerRunnerId,
|
|
1747
1787
|
requiredLabels: ['linux'],
|
|
1748
1788
|
runnerLabels: ['linux'],
|
|
1749
1789
|
startedAt: params.startedAt,
|
|
@@ -1756,13 +1796,13 @@ describe('reconcileProvisionedRunners', () => {
|
|
|
1756
1796
|
}
|
|
1757
1797
|
|
|
1758
1798
|
function event(params: {
|
|
1759
|
-
|
|
1799
|
+
providerRunnerId: string;
|
|
1760
1800
|
reservationId?: string | null;
|
|
1761
1801
|
state?: 'starting' | 'running' | 'stopping' | 'stopped' | 'failed' | 'terminated';
|
|
1762
1802
|
reportedAt?: Date;
|
|
1763
1803
|
}) {
|
|
1764
1804
|
return {
|
|
1765
|
-
|
|
1805
|
+
providerRunnerId: params.providerRunnerId,
|
|
1766
1806
|
reservationId: params.reservationId ?? null,
|
|
1767
1807
|
templateKey: 'linux',
|
|
1768
1808
|
labels: ['linux'],
|
|
@@ -1775,6 +1815,42 @@ describe('reconcileProvisionedRunners', () => {
|
|
|
1775
1815
|
}
|
|
1776
1816
|
});
|
|
1777
1817
|
|
|
1818
|
+
describe('planned provisioned capacity', () => {
|
|
1819
|
+
it('belongs to its provisioner until it receives one provider runner identity', async () => {
|
|
1820
|
+
const provisionerId = crypto.randomUUID();
|
|
1821
|
+
const capacity = await createPlannedProvisionedCapacity({
|
|
1822
|
+
provisionerId,
|
|
1823
|
+
providerKind: 'docker',
|
|
1824
|
+
templateKey: 'linux',
|
|
1825
|
+
});
|
|
1826
|
+
|
|
1827
|
+
const attached = await attachProviderRunnerId({
|
|
1828
|
+
capacityId: capacity.capacityId,
|
|
1829
|
+
provisionerId,
|
|
1830
|
+
providerRunnerId: 'container-1',
|
|
1831
|
+
});
|
|
1832
|
+
const rebound = await attachProviderRunnerId({
|
|
1833
|
+
capacityId: capacity.capacityId,
|
|
1834
|
+
provisionerId,
|
|
1835
|
+
providerRunnerId: 'container-2',
|
|
1836
|
+
});
|
|
1837
|
+
const [row] = await db()
|
|
1838
|
+
.select()
|
|
1839
|
+
.from(providerRunners)
|
|
1840
|
+
.where(eq(providerRunners.id, capacity.capacityId));
|
|
1841
|
+
|
|
1842
|
+
expect(attached).toBe(true);
|
|
1843
|
+
expect(rebound).toBe(false);
|
|
1844
|
+
expect(row).toMatchObject({
|
|
1845
|
+
workspaceId: null,
|
|
1846
|
+
provisionerId,
|
|
1847
|
+
providerRunnerId: 'container-1',
|
|
1848
|
+
labels: [],
|
|
1849
|
+
state: 'starting',
|
|
1850
|
+
});
|
|
1851
|
+
});
|
|
1852
|
+
});
|
|
1853
|
+
|
|
1778
1854
|
function deferred<T>() {
|
|
1779
1855
|
let resolve!: (value: T | PromiseLike<T>) => void;
|
|
1780
1856
|
let reject!: (reason?: unknown) => void;
|