@shipfox/api-runners 5.0.0 → 7.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +6 -6
- package/CHANGELOG.md +80 -0
- package/dist/config.d.ts +5 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +54 -7
- package/dist/config.js.map +1 -1
- package/dist/core/demand.d.ts +6 -6
- package/dist/core/demand.d.ts.map +1 -1
- package/dist/core/demand.js +10 -10
- package/dist/core/demand.js.map +1 -1
- package/dist/core/entities/ephemeral-registration-token.d.ts +1 -1
- package/dist/core/entities/ephemeral-registration-token.d.ts.map +1 -1
- package/dist/core/entities/ephemeral-registration-token.js.map +1 -1
- package/dist/core/entities/provisioner-capability-snapshot.d.ts +13 -0
- package/dist/core/entities/provisioner-capability-snapshot.d.ts.map +1 -0
- package/dist/core/entities/provisioner-capability-snapshot.js +3 -0
- package/dist/core/entities/provisioner-capability-snapshot.js.map +1 -0
- package/dist/core/entities/provisioner-token.d.ts +10 -2
- package/dist/core/entities/provisioner-token.d.ts.map +1 -1
- package/dist/core/entities/provisioner-token.js.map +1 -1
- package/dist/core/entities/{provisioned-runner.d.ts → runner-instance.d.ts} +7 -6
- package/dist/core/entities/runner-instance.d.ts.map +1 -0
- package/dist/core/entities/runner-instance.js +3 -0
- package/dist/core/entities/runner-instance.js.map +1 -0
- package/dist/core/entities/runner-session.d.ts +4 -2
- package/dist/core/entities/runner-session.d.ts.map +1 -1
- package/dist/core/entities/runner-session.js.map +1 -1
- package/dist/core/ephemeral-registration-tokens.d.ts +5 -5
- package/dist/core/ephemeral-registration-tokens.d.ts.map +1 -1
- package/dist/core/ephemeral-registration-tokens.js +12 -12
- package/dist/core/ephemeral-registration-tokens.js.map +1 -1
- package/dist/core/errors.d.ts +16 -4
- package/dist/core/errors.d.ts.map +1 -1
- package/dist/core/errors.js +22 -4
- package/dist/core/errors.js.map +1 -1
- package/dist/core/index.d.ts +9 -5
- package/dist/core/index.d.ts.map +1 -1
- package/dist/core/index.js +6 -3
- package/dist/core/index.js.map +1 -1
- package/dist/core/job-executions.d.ts +2 -0
- package/dist/core/job-executions.d.ts.map +1 -1
- package/dist/core/job-executions.js +1 -2
- package/dist/core/job-executions.js.map +1 -1
- package/dist/core/maintenance.d.ts +1 -1
- package/dist/core/maintenance.d.ts.map +1 -1
- package/dist/core/maintenance.js +5 -5
- package/dist/core/maintenance.js.map +1 -1
- package/dist/core/provisioner-capability-snapshots.d.ts +5 -0
- package/dist/core/provisioner-capability-snapshots.d.ts.map +1 -0
- package/dist/core/provisioner-capability-snapshots.js +10 -0
- package/dist/core/provisioner-capability-snapshots.js.map +1 -0
- package/dist/core/provisioner-tokens.d.ts +10 -0
- package/dist/core/provisioner-tokens.d.ts.map +1 -1
- package/dist/core/provisioner-tokens.js +23 -1
- package/dist/core/provisioner-tokens.js.map +1 -1
- package/dist/core/rate-limit.d.ts.map +1 -1
- package/dist/core/rate-limit.js +2 -7
- package/dist/core/rate-limit.js.map +1 -1
- package/dist/core/runner-activation.d.ts +13 -0
- package/dist/core/runner-activation.d.ts.map +1 -0
- package/dist/core/runner-activation.js +36 -0
- package/dist/core/runner-activation.js.map +1 -0
- package/dist/core/runner-assignments.d.ts +6 -0
- package/dist/core/runner-assignments.d.ts.map +1 -0
- package/dist/core/runner-assignments.js +6 -0
- package/dist/core/runner-assignments.js.map +1 -0
- package/dist/core/runner-control-sessions.d.ts +46 -0
- package/dist/core/runner-control-sessions.d.ts.map +1 -0
- package/dist/core/runner-control-sessions.js +121 -0
- package/dist/core/runner-control-sessions.js.map +1 -0
- package/dist/core/runner-instances.d.ts +72 -0
- package/dist/core/runner-instances.d.ts.map +1 -0
- package/dist/core/runner-instances.js +165 -0
- package/dist/core/runner-instances.js.map +1 -0
- package/dist/core/runner-sessions.d.ts +8 -2
- package/dist/core/runner-sessions.d.ts.map +1 -1
- package/dist/core/runner-sessions.js +8 -5
- package/dist/core/runner-sessions.js.map +1 -1
- package/dist/db/db.d.ts +2698 -1054
- package/dist/db/db.d.ts.map +1 -1
- package/dist/db/db.js +9 -2
- package/dist/db/db.js.map +1 -1
- package/dist/db/ephemeral-registration-tokens.d.ts +2 -2
- package/dist/db/ephemeral-registration-tokens.d.ts.map +1 -1
- package/dist/db/ephemeral-registration-tokens.js +21 -21
- package/dist/db/ephemeral-registration-tokens.js.map +1 -1
- package/dist/db/index.d.ts +7 -4
- package/dist/db/index.d.ts.map +1 -1
- package/dist/db/index.js +5 -2
- package/dist/db/index.js.map +1 -1
- package/dist/db/job-executions.d.ts +6 -6
- package/dist/db/job-executions.d.ts.map +1 -1
- package/dist/db/job-executions.js +20 -19
- package/dist/db/job-executions.js.map +1 -1
- package/dist/db/provisioner-capability-snapshots.d.ts +21 -0
- package/dist/db/provisioner-capability-snapshots.d.ts.map +1 -0
- package/dist/db/provisioner-capability-snapshots.js +57 -0
- package/dist/db/provisioner-capability-snapshots.js.map +1 -0
- package/dist/db/provisioner-tokens.d.ts +7 -2
- package/dist/db/provisioner-tokens.d.ts.map +1 -1
- package/dist/db/provisioner-tokens.js +59 -10
- package/dist/db/provisioner-tokens.js.map +1 -1
- package/dist/db/reservations.d.ts +18 -0
- package/dist/db/reservations.d.ts.map +1 -1
- package/dist/db/reservations.js +88 -1
- package/dist/db/reservations.js.map +1 -1
- package/dist/db/runner-assignments.d.ts +7 -0
- package/dist/db/runner-assignments.d.ts.map +1 -0
- package/dist/db/runner-assignments.js +55 -0
- package/dist/db/runner-assignments.js.map +1 -0
- package/dist/db/runner-instances.d.ts +84 -0
- package/dist/db/runner-instances.d.ts.map +1 -0
- package/dist/db/runner-instances.js +497 -0
- package/dist/db/runner-instances.js.map +1 -0
- package/dist/db/runner-sessions.d.ts +5 -0
- package/dist/db/runner-sessions.d.ts.map +1 -1
- package/dist/db/runner-sessions.js +48 -2
- package/dist/db/runner-sessions.js.map +1 -1
- package/dist/db/schema/ephemeral-registration-tokens.d.ts +2 -2
- package/dist/db/schema/ephemeral-registration-tokens.js +3 -3
- package/dist/db/schema/ephemeral-registration-tokens.js.map +1 -1
- package/dist/db/schema/provisioner-capability-snapshots.d.ts +207 -0
- package/dist/db/schema/provisioner-capability-snapshots.d.ts.map +1 -0
- package/dist/db/schema/provisioner-capability-snapshots.js +38 -0
- package/dist/db/schema/provisioner-capability-snapshots.js.map +1 -0
- package/dist/db/schema/provisioner-tokens.d.ts +19 -1
- package/dist/db/schema/provisioner-tokens.d.ts.map +1 -1
- package/dist/db/schema/provisioner-tokens.js +29 -3
- package/dist/db/schema/provisioner-tokens.js.map +1 -1
- package/dist/db/schema/runner-activation-tokens.d.ts +161 -0
- package/dist/db/schema/runner-activation-tokens.d.ts.map +1 -0
- package/dist/db/schema/runner-activation-tokens.js +27 -0
- package/dist/db/schema/runner-activation-tokens.js.map +1 -0
- package/dist/db/schema/runner-control-sessions.d.ts +338 -0
- package/dist/db/schema/runner-control-sessions.d.ts.map +1 -0
- package/dist/db/schema/runner-control-sessions.js +52 -0
- package/dist/db/schema/runner-control-sessions.js.map +1 -0
- package/dist/db/schema/{provisioned-runners.d.ts → runner-instances.d.ts} +105 -34
- package/dist/db/schema/runner-instances.d.ts.map +1 -0
- package/dist/db/schema/{provisioned-runners.js → runner-instances.js} +23 -14
- package/dist/db/schema/runner-instances.js.map +1 -0
- package/dist/db/schema/runner-sessions.d.ts +39 -5
- package/dist/db/schema/runner-sessions.d.ts.map +1 -1
- package/dist/db/schema/runner-sessions.js +13 -6
- package/dist/db/schema/runner-sessions.js.map +1 -1
- package/dist/db/schema/running-job-executions.d.ts +2 -2
- package/dist/db/schema/running-job-executions.js +4 -4
- package/dist/db/schema/running-job-executions.js.map +1 -1
- package/dist/index.d.ts +6 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +45 -39
- package/dist/index.js.map +1 -1
- package/dist/installation-provisioning.d.ts +14 -0
- package/dist/installation-provisioning.d.ts.map +1 -0
- package/dist/installation-provisioning.js +7 -0
- package/dist/installation-provisioning.js.map +1 -0
- package/dist/metrics/index.d.ts +1 -1
- package/dist/metrics/index.d.ts.map +1 -1
- package/dist/metrics/index.js +1 -1
- package/dist/metrics/index.js.map +1 -1
- package/dist/metrics/instance.d.ts +11 -7
- package/dist/metrics/instance.d.ts.map +1 -1
- package/dist/metrics/instance.js +13 -7
- package/dist/metrics/instance.js.map +1 -1
- package/dist/presentation/auth/index.d.ts +1 -0
- package/dist/presentation/auth/index.d.ts.map +1 -1
- package/dist/presentation/auth/index.js +1 -0
- package/dist/presentation/auth/index.js.map +1 -1
- package/dist/presentation/auth/provisioner-token-auth.d.ts.map +1 -1
- package/dist/presentation/auth/provisioner-token-auth.js +5 -1
- package/dist/presentation/auth/provisioner-token-auth.js.map +1 -1
- package/dist/presentation/auth/runner-control-session-auth.d.ts +4 -0
- package/dist/presentation/auth/runner-control-session-auth.d.ts.map +1 -0
- package/dist/presentation/auth/runner-control-session-auth.js +29 -0
- package/dist/presentation/auth/runner-control-session-auth.js.map +1 -0
- package/dist/presentation/auth/runner-registration-token-auth.d.ts +5 -1
- package/dist/presentation/auth/runner-registration-token-auth.d.ts.map +1 -1
- package/dist/presentation/auth/runner-registration-token-auth.js +21 -1
- package/dist/presentation/auth/runner-registration-token-auth.js.map +1 -1
- package/dist/presentation/dto/index.d.ts +1 -2
- package/dist/presentation/dto/index.d.ts.map +1 -1
- package/dist/presentation/dto/index.js +1 -2
- package/dist/presentation/dto/index.js.map +1 -1
- package/dist/presentation/dto/poll-demand.d.ts.map +1 -1
- package/dist/presentation/dto/poll-demand.js +7 -1
- package/dist/presentation/dto/poll-demand.js.map +1 -1
- package/dist/presentation/dto/provisioner-token.d.ts +1 -0
- package/dist/presentation/dto/provisioner-token.d.ts.map +1 -1
- package/dist/presentation/dto/provisioner-token.js +5 -1
- package/dist/presentation/dto/provisioner-token.js.map +1 -1
- package/dist/presentation/dto/{provisioned-runners.d.ts → runner-instances.d.ts} +9 -9
- package/dist/presentation/dto/runner-instances.d.ts.map +1 -0
- package/dist/presentation/dto/{provisioned-runners.js → runner-instances.js} +6 -6
- package/dist/presentation/dto/runner-instances.js.map +1 -0
- package/dist/presentation/index.d.ts +2 -2
- package/dist/presentation/index.d.ts.map +1 -1
- package/dist/presentation/index.js +2 -2
- package/dist/presentation/index.js.map +1 -1
- package/dist/presentation/inter-module.d.ts +5 -0
- package/dist/presentation/inter-module.d.ts.map +1 -0
- package/dist/presentation/inter-module.js +43 -0
- package/dist/presentation/inter-module.js.map +1 -0
- package/dist/presentation/routes/assign-runner-instances.d.ts +2 -0
- package/dist/presentation/routes/assign-runner-instances.d.ts.map +1 -0
- package/dist/presentation/routes/assign-runner-instances.js +40 -0
- package/dist/presentation/routes/assign-runner-instances.js.map +1 -0
- package/dist/presentation/routes/get-provisioner-me.d.ts.map +1 -1
- package/dist/presentation/routes/get-provisioner-me.js +5 -1
- package/dist/presentation/routes/get-provisioner-me.js.map +1 -1
- package/dist/presentation/routes/heartbeat.d.ts +2 -1
- package/dist/presentation/routes/heartbeat.d.ts.map +1 -1
- package/dist/presentation/routes/heartbeat.js +60 -50
- package/dist/presentation/routes/heartbeat.js.map +1 -1
- package/dist/presentation/routes/index.d.ts +3 -1
- package/dist/presentation/routes/index.d.ts.map +1 -1
- package/dist/presentation/routes/index.js +80 -51
- package/dist/presentation/routes/index.js.map +1 -1
- package/dist/presentation/routes/poll-demand.d.ts +2 -0
- package/dist/presentation/routes/poll-demand.d.ts.map +1 -1
- package/dist/presentation/routes/poll-demand.js +86 -45
- package/dist/presentation/routes/poll-demand.js.map +1 -1
- package/dist/presentation/routes/reconcile-runner-instances.d.ts +2 -0
- package/dist/presentation/routes/reconcile-runner-instances.d.ts.map +1 -0
- package/dist/presentation/routes/reconcile-runner-instances.js +27 -0
- package/dist/presentation/routes/reconcile-runner-instances.js.map +1 -0
- package/dist/presentation/routes/register.d.ts +2 -1
- package/dist/presentation/routes/register.d.ts.map +1 -1
- package/dist/presentation/routes/register.js +55 -52
- package/dist/presentation/routes/register.js.map +1 -1
- package/dist/presentation/routes/report-runner-instances.d.ts +2 -0
- package/dist/presentation/routes/report-runner-instances.d.ts.map +1 -0
- package/dist/presentation/routes/report-runner-instances.js +37 -0
- package/dist/presentation/routes/report-runner-instances.js.map +1 -0
- package/dist/presentation/routes/request-job.d.ts +2 -1
- package/dist/presentation/routes/request-job.d.ts.map +1 -1
- package/dist/presentation/routes/request-job.js +39 -36
- package/dist/presentation/routes/request-job.js.map +1 -1
- package/dist/presentation/routes/runner-enrollment.d.ts +8 -0
- package/dist/presentation/routes/runner-enrollment.d.ts.map +1 -0
- package/dist/presentation/routes/runner-enrollment.js +212 -0
- package/dist/presentation/routes/runner-enrollment.js.map +1 -0
- package/dist/temporal/activities/index.d.ts +2 -2
- package/dist/temporal/activities/index.d.ts.map +1 -1
- package/dist/temporal/activities/index.js +2 -2
- package/dist/temporal/activities/index.js.map +1 -1
- package/dist/temporal/activities/maintenance-activities.d.ts +1 -1
- package/dist/temporal/activities/maintenance-activities.d.ts.map +1 -1
- package/dist/temporal/activities/maintenance-activities.js +3 -3
- package/dist/temporal/activities/maintenance-activities.js.map +1 -1
- package/dist/temporal/workflows/index.bundle.js +3 -3
- package/dist/temporal/workflows/stuck-job-detector.js +2 -2
- package/dist/temporal/workflows/stuck-job-detector.js.map +1 -1
- package/dist/tsconfig.test.tsbuildinfo +1 -1
- package/drizzle/0000_initial.sql +91 -24
- package/drizzle/meta/0000_snapshot.json +392 -63
- package/package.json +21 -16
- package/src/config.test.ts +0 -26
- package/src/config.ts +48 -7
- package/src/core/demand.test.ts +29 -29
- package/src/core/demand.ts +22 -22
- package/src/core/entities/ephemeral-registration-token.ts +1 -1
- package/src/core/entities/provisioner-capability-snapshot.ts +12 -0
- package/src/core/entities/provisioner-token.ts +8 -2
- package/src/core/entities/{provisioned-runner.ts → runner-instance.ts} +6 -5
- package/src/core/entities/runner-session.ts +4 -2
- package/src/core/ephemeral-registration-tokens.test.ts +11 -11
- package/src/core/ephemeral-registration-tokens.ts +18 -20
- package/src/core/errors.ts +23 -4
- package/src/core/index.ts +37 -22
- package/src/core/job-executions.ts +3 -2
- package/src/core/maintenance.test.ts +13 -17
- package/src/core/maintenance.ts +5 -5
- package/src/core/provisioner-capability-snapshots.test.ts +35 -0
- package/src/core/provisioner-capability-snapshots.ts +12 -0
- package/src/core/provisioner-tokens.test.ts +20 -0
- package/src/core/provisioner-tokens.ts +33 -0
- package/src/core/rate-limit.test.ts +11 -27
- package/src/core/rate-limit.ts +3 -11
- package/src/core/runner-activation.ts +72 -0
- package/src/core/runner-assignments.ts +9 -0
- package/src/core/runner-control-sessions.ts +190 -0
- package/src/core/{provisioned-runners.test.ts → runner-instances.test.ts} +24 -24
- package/src/core/runner-instances.ts +301 -0
- package/src/core/runner-sessions.test.ts +112 -22
- package/src/core/runner-sessions.ts +25 -14
- package/src/db/db.ts +9 -2
- package/src/db/ephemeral-registration-tokens.test.ts +9 -9
- package/src/db/ephemeral-registration-tokens.ts +26 -26
- package/src/db/index.ts +30 -21
- package/src/db/job-executions.test.ts +13 -11
- package/src/db/job-executions.ts +33 -27
- package/src/db/provisioner-capability-snapshots.test.ts +175 -0
- package/src/db/provisioner-capability-snapshots.ts +130 -0
- package/src/db/provisioner-tokens.test.ts +139 -2
- package/src/db/provisioner-tokens.ts +123 -15
- package/src/db/reservations.test.ts +49 -0
- package/src/db/reservations.ts +146 -1
- package/src/db/runner-assignments.test.ts +163 -0
- package/src/db/runner-assignments.ts +115 -0
- package/src/db/{provisioned-runners.test.ts → runner-instances.test.ts} +442 -353
- package/src/db/runner-instances.ts +922 -0
- package/src/db/runner-sessions.test.ts +19 -6
- package/src/db/runner-sessions.ts +78 -2
- package/src/db/schema/ephemeral-registration-tokens.ts +4 -4
- package/src/db/schema/provisioner-capability-snapshots.ts +46 -0
- package/src/db/schema/provisioner-tokens.ts +33 -2
- package/src/db/schema/runner-activation-tokens.ts +22 -0
- package/src/db/schema/runner-control-sessions.ts +46 -0
- package/src/db/schema/{provisioned-runners.ts → runner-instances.ts} +28 -27
- package/src/db/schema/runner-sessions.ts +11 -7
- package/src/db/schema/running-job-executions.ts +5 -5
- package/src/index.test.ts +26 -0
- package/src/index.ts +38 -27
- package/src/installation-provisioning.ts +14 -0
- package/src/metrics/index.ts +5 -5
- package/src/metrics/instance.ts +25 -14
- package/src/presentation/auth/index.ts +4 -0
- package/src/presentation/auth/provisioner-token-auth.ts +8 -4
- package/src/presentation/auth/runner-control-session-auth.ts +31 -0
- package/src/presentation/auth/runner-registration-token-auth.ts +39 -3
- package/src/presentation/dto/index.ts +4 -8
- package/src/presentation/dto/poll-demand.ts +3 -1
- package/src/presentation/dto/provisioner-token.ts +5 -0
- package/src/presentation/dto/{provisioned-runners.ts → runner-instances.ts} +13 -15
- package/src/presentation/index.ts +2 -1
- package/src/presentation/inter-module.test.ts +15 -0
- package/src/presentation/inter-module.ts +53 -0
- package/src/presentation/routes/assign-runner-instances.ts +49 -0
- package/src/presentation/routes/get-provisioner-me.ts +3 -4
- package/src/presentation/routes/heartbeat.test.ts +4 -3
- package/src/presentation/routes/heartbeat.ts +67 -64
- package/src/presentation/routes/index.ts +91 -44
- package/src/presentation/routes/list-active-runners.test.ts +23 -23
- package/src/presentation/routes/manual-registration-tokens.test.ts +4 -3
- package/src/presentation/routes/poll-demand-release.test.ts +4 -4
- package/src/presentation/routes/poll-demand.test.ts +161 -22
- package/src/presentation/routes/poll-demand.ts +105 -45
- package/src/presentation/routes/provisioner-me.test.ts +37 -3
- package/src/presentation/routes/{reconcile-provisioned-runners.test.ts → reconcile-runner-instances.test.ts} +66 -61
- package/src/presentation/routes/reconcile-runner-instances.ts +30 -0
- package/src/presentation/routes/register.test.ts +9 -5
- package/src/presentation/routes/register.ts +61 -57
- package/src/presentation/routes/{report-provisioned-runners.test.ts → report-runner-instances.test.ts} +28 -28
- package/src/presentation/routes/{report-provisioned-runners.ts → report-runner-instances.ts} +14 -14
- package/src/presentation/routes/request-job.test.ts +5 -3
- package/src/presentation/routes/request-job.ts +41 -37
- package/src/presentation/routes/runner-enrollment.test.ts +319 -0
- package/src/presentation/routes/runner-enrollment.ts +196 -0
- package/src/temporal/activities/index.ts +2 -2
- package/src/temporal/activities/maintenance-activities.test.ts +7 -7
- package/src/temporal/activities/maintenance-activities.ts +3 -3
- package/src/temporal/workflows/stuck-job-detector.test.ts +3 -3
- package/src/temporal/workflows/stuck-job-detector.ts +2 -2
- package/test/env.ts +1 -3
- package/test/factories/ephemeral-registration-token.ts +2 -2
- package/test/factories/provisioner-token.ts +3 -1
- package/test/factories/runner-instance.ts +57 -0
- package/test/factories/runner-session.ts +3 -1
- package/test/fixtures/auth-inter-module.ts +30 -0
- package/test/globalSetup.ts +4 -1
- package/test/index.ts +2 -1
- package/tsconfig.build.tsbuildinfo +1 -1
- package/dist/core/entities/provisioned-runner.d.ts.map +0 -1
- package/dist/core/entities/provisioned-runner.js +0 -3
- package/dist/core/entities/provisioned-runner.js.map +0 -1
- package/dist/core/provisioned-runners.d.ts +0 -67
- package/dist/core/provisioned-runners.d.ts.map +0 -1
- package/dist/core/provisioned-runners.js +0 -162
- package/dist/core/provisioned-runners.js.map +0 -1
- package/dist/db/provisioned-runners.d.ts +0 -79
- package/dist/db/provisioned-runners.d.ts.map +0 -1
- package/dist/db/provisioned-runners.js +0 -466
- package/dist/db/provisioned-runners.js.map +0 -1
- package/dist/db/schema/provisioned-runners.d.ts.map +0 -1
- package/dist/db/schema/provisioned-runners.js.map +0 -1
- package/dist/presentation/dto/mint-registration-tokens.d.ts +0 -5
- package/dist/presentation/dto/mint-registration-tokens.d.ts.map +0 -1
- package/dist/presentation/dto/mint-registration-tokens.js +0 -19
- package/dist/presentation/dto/mint-registration-tokens.js.map +0 -1
- package/dist/presentation/dto/provisioned-runners.d.ts.map +0 -1
- package/dist/presentation/dto/provisioned-runners.js.map +0 -1
- package/dist/presentation/routes/mint-registration-tokens.d.ts +0 -2
- package/dist/presentation/routes/mint-registration-tokens.d.ts.map +0 -1
- package/dist/presentation/routes/mint-registration-tokens.js +0 -74
- package/dist/presentation/routes/mint-registration-tokens.js.map +0 -1
- package/dist/presentation/routes/reconcile-provisioned-runners.d.ts +0 -2
- package/dist/presentation/routes/reconcile-provisioned-runners.d.ts.map +0 -1
- package/dist/presentation/routes/reconcile-provisioned-runners.js +0 -27
- package/dist/presentation/routes/reconcile-provisioned-runners.js.map +0 -1
- package/dist/presentation/routes/report-provisioned-runners.d.ts +0 -2
- package/dist/presentation/routes/report-provisioned-runners.d.ts.map +0 -1
- package/dist/presentation/routes/report-provisioned-runners.js +0 -37
- package/dist/presentation/routes/report-provisioned-runners.js.map +0 -1
- package/src/core/provisioned-runners.ts +0 -301
- package/src/db/provisioned-runners.ts +0 -886
- package/src/presentation/dto/mint-registration-tokens.ts +0 -35
- package/src/presentation/routes/mint-registration-tokens.test.ts +0 -453
- package/src/presentation/routes/mint-registration-tokens.ts +0 -82
- package/src/presentation/routes/reconcile-provisioned-runners.ts +0 -30
- package/test/factories/provisioned-runner.ts +0 -78
|
@@ -3,35 +3,36 @@ import {and, desc, eq, inArray, or, sql} from 'drizzle-orm';
|
|
|
3
3
|
import {db} from '#db/db.js';
|
|
4
4
|
import {createRunnerSessionConsumingEphemeralToken} from '#db/ephemeral-registration-tokens.js';
|
|
5
5
|
import {
|
|
6
|
-
|
|
7
|
-
|
|
6
|
+
attachRunnerInstanceProviderId,
|
|
7
|
+
listActiveRunnerInstanceCountsByTemplateTx,
|
|
8
|
+
listActiveRunnerInstances,
|
|
8
9
|
listProvisionerTerminateIntentRowsTx,
|
|
9
10
|
listProvisionerTerminateIntents,
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
} from '#db/
|
|
14
|
-
import {provisionedRunners} from '#db/schema/provisioned-runners.js';
|
|
11
|
+
reapStaleRunnerInstances,
|
|
12
|
+
reconcileRunnerInstances,
|
|
13
|
+
reportRunnerInstances,
|
|
14
|
+
} from '#db/runner-instances.js';
|
|
15
15
|
import {provisionerTokens} from '#db/schema/provisioner-tokens.js';
|
|
16
16
|
import {reservations} from '#db/schema/reservations.js';
|
|
17
|
+
import {providerRunners} from '#db/schema/runner-instances.js';
|
|
17
18
|
import {runnerSessions} from '#db/schema/runner-sessions.js';
|
|
18
19
|
import {runningJobExecutions} from '#db/schema/running-job-executions.js';
|
|
19
20
|
import {
|
|
20
21
|
ephemeralRegistrationTokenFactory,
|
|
21
|
-
|
|
22
|
+
providerRunnerFactory,
|
|
22
23
|
provisionerTokenFactory,
|
|
23
24
|
reservationFactory,
|
|
24
25
|
runnerSessionFactory,
|
|
25
26
|
} from '#test/index.js';
|
|
26
27
|
|
|
27
|
-
function
|
|
28
|
+
function providerRunnerRowsFor(params: {workspaceId: string; provisionerId: string}) {
|
|
28
29
|
return db()
|
|
29
30
|
.select()
|
|
30
|
-
.from(
|
|
31
|
+
.from(providerRunners)
|
|
31
32
|
.where(
|
|
32
33
|
and(
|
|
33
|
-
eq(
|
|
34
|
-
eq(
|
|
34
|
+
eq(providerRunners.workspaceId, params.workspaceId),
|
|
35
|
+
eq(providerRunners.provisionerId, params.provisionerId),
|
|
35
36
|
),
|
|
36
37
|
);
|
|
37
38
|
}
|
|
@@ -51,7 +52,7 @@ function reservationRowsFor(params: {workspaceId: string; provisionerId: string}
|
|
|
51
52
|
async function insertRunningJobRow(params: {
|
|
52
53
|
workspaceId: string;
|
|
53
54
|
provisionerId: string;
|
|
54
|
-
|
|
55
|
+
providerRunnerId: string;
|
|
55
56
|
jobExecutionId?: string;
|
|
56
57
|
startedAt?: Date;
|
|
57
58
|
cancellationRequestedAt?: Date | null;
|
|
@@ -70,7 +71,7 @@ async function insertRunningJobRow(params: {
|
|
|
70
71
|
projectId: crypto.randomUUID(),
|
|
71
72
|
runnerSessionId: runnerSession.id,
|
|
72
73
|
provisionerId: params.provisionerId,
|
|
73
|
-
|
|
74
|
+
providerRunnerId: params.providerRunnerId,
|
|
74
75
|
requiredLabels: ['linux'],
|
|
75
76
|
runnerLabels: ['linux'],
|
|
76
77
|
startedAt,
|
|
@@ -79,7 +80,7 @@ async function insertRunningJobRow(params: {
|
|
|
79
80
|
});
|
|
80
81
|
}
|
|
81
82
|
|
|
82
|
-
describe('
|
|
83
|
+
describe('reportRunnerInstances', () => {
|
|
83
84
|
let workspaceId: string;
|
|
84
85
|
let provisionerId: string;
|
|
85
86
|
|
|
@@ -88,23 +89,48 @@ describe('reportProvisionedRunners', () => {
|
|
|
88
89
|
provisionerId = crypto.randomUUID();
|
|
89
90
|
});
|
|
90
91
|
|
|
92
|
+
it('reports and reconciles installation capacity without a workspace assignment', async () => {
|
|
93
|
+
const reportedAt = new Date();
|
|
94
|
+
|
|
95
|
+
const report = await reportRunnerInstances({
|
|
96
|
+
workspaceId: null,
|
|
97
|
+
provisionerId,
|
|
98
|
+
events: [event({providerRunnerId: 'installation-runner', reportedAt})],
|
|
99
|
+
});
|
|
100
|
+
const reconcile = await reconcileRunnerInstances({
|
|
101
|
+
workspaceId: null,
|
|
102
|
+
provisionerId,
|
|
103
|
+
observedRunnerInstanceIds: ['installation-runner'],
|
|
104
|
+
terminateGraceSeconds: 60,
|
|
105
|
+
});
|
|
106
|
+
const [row] = await db()
|
|
107
|
+
.select()
|
|
108
|
+
.from(providerRunners)
|
|
109
|
+
.where(eq(providerRunners.providerRunnerId, 'installation-runner'));
|
|
110
|
+
|
|
111
|
+
expect(report).toEqual({accepted: 1, reservationsReleased: 0, terminateIntentsHonored: []});
|
|
112
|
+
expect(reconcile.absentIds).toEqual([]);
|
|
113
|
+
expect(reconcile.observedRows).toMatchObject([{workspaceId: null}]);
|
|
114
|
+
expect(row).toMatchObject({workspaceId: null, provisionerId, reportedAt});
|
|
115
|
+
});
|
|
116
|
+
|
|
91
117
|
it('dedupes duplicate provisioned runner ids in one batch', async () => {
|
|
92
118
|
const reportedAt = new Date();
|
|
93
119
|
|
|
94
|
-
const result = await
|
|
120
|
+
const result = await reportRunnerInstances({
|
|
95
121
|
workspaceId,
|
|
96
122
|
provisionerId,
|
|
97
123
|
events: [
|
|
98
|
-
event({
|
|
124
|
+
event({providerRunnerId: 'provisioned-runner-1', state: 'starting', reportedAt}),
|
|
99
125
|
event({
|
|
100
|
-
|
|
126
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
101
127
|
state: 'running',
|
|
102
128
|
reportedAt: new Date(reportedAt.getTime() + 1),
|
|
103
129
|
}),
|
|
104
130
|
],
|
|
105
131
|
});
|
|
106
132
|
|
|
107
|
-
const rows = await
|
|
133
|
+
const rows = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
108
134
|
expect(result).toEqual({accepted: 1, reservationsReleased: 0, terminateIntentsHonored: []});
|
|
109
135
|
expect(rows).toHaveLength(1);
|
|
110
136
|
expect(rows[0]?.state).toBe('running');
|
|
@@ -113,19 +139,19 @@ describe('reportProvisionedRunners', () => {
|
|
|
113
139
|
it('uses state progression to dedupe equal-timestamp provisioned runner reports', async () => {
|
|
114
140
|
const reportedAt = new Date();
|
|
115
141
|
|
|
116
|
-
const result = await
|
|
142
|
+
const result = await reportRunnerInstances({
|
|
117
143
|
workspaceId,
|
|
118
144
|
provisionerId,
|
|
119
145
|
events: [
|
|
120
|
-
event({
|
|
121
|
-
event({
|
|
122
|
-
event({
|
|
123
|
-
event({
|
|
146
|
+
event({providerRunnerId: 'provisioned-runner-1', state: 'running', reportedAt}),
|
|
147
|
+
event({providerRunnerId: 'provisioned-runner-1', state: 'failed', reportedAt}),
|
|
148
|
+
event({providerRunnerId: 'provisioned-runner-2', state: 'failed', reportedAt}),
|
|
149
|
+
event({providerRunnerId: 'provisioned-runner-2', state: 'running', reportedAt}),
|
|
124
150
|
],
|
|
125
151
|
});
|
|
126
152
|
|
|
127
|
-
const rows = await
|
|
128
|
-
|
|
153
|
+
const rows = await providerRunnerRowsFor({workspaceId, provisionerId}).orderBy(
|
|
154
|
+
providerRunners.providerRunnerId,
|
|
129
155
|
);
|
|
130
156
|
expect(result).toEqual({accepted: 2, reservationsReleased: 0, terminateIntentsHonored: []});
|
|
131
157
|
expect(rows.map((row) => row.state)).toEqual(['failed', 'failed']);
|
|
@@ -133,20 +159,20 @@ describe('reportProvisionedRunners', () => {
|
|
|
133
159
|
|
|
134
160
|
it('accepts delayed events that move the lifecycle forward', async () => {
|
|
135
161
|
const newest = new Date();
|
|
136
|
-
await
|
|
162
|
+
await reportRunnerInstances({
|
|
137
163
|
workspaceId,
|
|
138
164
|
provisionerId,
|
|
139
165
|
events: [
|
|
140
|
-
event({
|
|
166
|
+
event({providerRunnerId: 'provisioned-runner-1', state: 'running', reportedAt: newest}),
|
|
141
167
|
],
|
|
142
168
|
});
|
|
143
169
|
|
|
144
|
-
await
|
|
170
|
+
await reportRunnerInstances({
|
|
145
171
|
workspaceId,
|
|
146
172
|
provisionerId,
|
|
147
173
|
events: [
|
|
148
174
|
event({
|
|
149
|
-
|
|
175
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
150
176
|
state: 'failed',
|
|
151
177
|
reason: 'late stale failure',
|
|
152
178
|
reportedAt: new Date(newest.getTime() - 1_000),
|
|
@@ -154,19 +180,19 @@ describe('reportProvisionedRunners', () => {
|
|
|
154
180
|
],
|
|
155
181
|
});
|
|
156
182
|
|
|
157
|
-
const rows = await
|
|
183
|
+
const rows = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
158
184
|
expect(rows[0]?.state).toBe('failed');
|
|
159
185
|
expect(rows[0]?.reason).toBe('late stale failure');
|
|
160
186
|
});
|
|
161
187
|
|
|
162
188
|
it('rejects older out-of-order events in the same lifecycle state', async () => {
|
|
163
189
|
const newest = new Date();
|
|
164
|
-
await
|
|
190
|
+
await reportRunnerInstances({
|
|
165
191
|
workspaceId,
|
|
166
192
|
provisionerId,
|
|
167
193
|
events: [
|
|
168
194
|
event({
|
|
169
|
-
|
|
195
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
170
196
|
state: 'running',
|
|
171
197
|
reason: 'fresh',
|
|
172
198
|
reportedAt: newest,
|
|
@@ -174,12 +200,12 @@ describe('reportProvisionedRunners', () => {
|
|
|
174
200
|
],
|
|
175
201
|
});
|
|
176
202
|
|
|
177
|
-
await
|
|
203
|
+
await reportRunnerInstances({
|
|
178
204
|
workspaceId,
|
|
179
205
|
provisionerId,
|
|
180
206
|
events: [
|
|
181
207
|
event({
|
|
182
|
-
|
|
208
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
183
209
|
state: 'running',
|
|
184
210
|
reason: 'stale',
|
|
185
211
|
reportedAt: new Date(newest.getTime() - 1_000),
|
|
@@ -187,7 +213,7 @@ describe('reportProvisionedRunners', () => {
|
|
|
187
213
|
],
|
|
188
214
|
});
|
|
189
215
|
|
|
190
|
-
const rows = await
|
|
216
|
+
const rows = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
191
217
|
expect(rows[0]?.state).toBe('running');
|
|
192
218
|
expect(rows[0]?.reason).toBe('fresh');
|
|
193
219
|
});
|
|
@@ -195,12 +221,12 @@ describe('reportProvisionedRunners', () => {
|
|
|
195
221
|
it('does not let equal-timestamp lower-priority reports flip terminal state', async () => {
|
|
196
222
|
const reservationId = await createReservation(1);
|
|
197
223
|
const reportedAt = new Date('2025-01-01T00:00:00.000Z');
|
|
198
|
-
await
|
|
224
|
+
await reportRunnerInstances({
|
|
199
225
|
workspaceId,
|
|
200
226
|
provisionerId,
|
|
201
227
|
events: [
|
|
202
228
|
event({
|
|
203
|
-
|
|
229
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
204
230
|
reservationId,
|
|
205
231
|
state: 'running',
|
|
206
232
|
reportedAt,
|
|
@@ -208,24 +234,24 @@ describe('reportProvisionedRunners', () => {
|
|
|
208
234
|
],
|
|
209
235
|
});
|
|
210
236
|
|
|
211
|
-
const terminal = await
|
|
237
|
+
const terminal = await reportRunnerInstances({
|
|
212
238
|
workspaceId,
|
|
213
239
|
provisionerId,
|
|
214
240
|
events: [
|
|
215
241
|
event({
|
|
216
|
-
|
|
242
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
217
243
|
reservationId,
|
|
218
244
|
state: 'failed',
|
|
219
245
|
reportedAt,
|
|
220
246
|
}),
|
|
221
247
|
],
|
|
222
248
|
});
|
|
223
|
-
const revived = await
|
|
249
|
+
const revived = await reportRunnerInstances({
|
|
224
250
|
workspaceId,
|
|
225
251
|
provisionerId,
|
|
226
252
|
events: [
|
|
227
253
|
event({
|
|
228
|
-
|
|
254
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
229
255
|
reservationId,
|
|
230
256
|
state: 'running',
|
|
231
257
|
reportedAt,
|
|
@@ -233,41 +259,41 @@ describe('reportProvisionedRunners', () => {
|
|
|
233
259
|
],
|
|
234
260
|
});
|
|
235
261
|
|
|
236
|
-
const
|
|
262
|
+
const providerRunnerRows = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
237
263
|
const reservationRows = await reservationRowsFor({workspaceId, provisionerId});
|
|
238
264
|
expect(terminal).toEqual({accepted: 1, reservationsReleased: 1, terminateIntentsHonored: []});
|
|
239
265
|
expect(revived).toEqual({accepted: 1, reservationsReleased: 0, terminateIntentsHonored: []});
|
|
240
|
-
expect(
|
|
241
|
-
expect(
|
|
266
|
+
expect(providerRunnerRows[0]?.state).toBe('failed');
|
|
267
|
+
expect(providerRunnerRows[0]?.reservationReleasedAt).toBeInstanceOf(Date);
|
|
242
268
|
expect(reservationRows).toHaveLength(0);
|
|
243
269
|
});
|
|
244
270
|
|
|
245
271
|
it('clamps future reported times so they do not pin provisioned runner state', async () => {
|
|
246
|
-
await
|
|
272
|
+
await reportRunnerInstances({
|
|
247
273
|
workspaceId,
|
|
248
274
|
provisionerId,
|
|
249
275
|
events: [
|
|
250
276
|
event({
|
|
251
|
-
|
|
277
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
252
278
|
state: 'starting',
|
|
253
279
|
reportedAt: new Date(Date.now() + 60 * 60 * 1000),
|
|
254
280
|
}),
|
|
255
281
|
],
|
|
256
282
|
});
|
|
257
283
|
|
|
258
|
-
await
|
|
284
|
+
await reportRunnerInstances({
|
|
259
285
|
workspaceId,
|
|
260
286
|
provisionerId,
|
|
261
287
|
events: [
|
|
262
288
|
event({
|
|
263
|
-
|
|
289
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
264
290
|
state: 'running',
|
|
265
291
|
reportedAt: new Date(),
|
|
266
292
|
}),
|
|
267
293
|
],
|
|
268
294
|
});
|
|
269
295
|
|
|
270
|
-
const rows = await
|
|
296
|
+
const rows = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
271
297
|
expect(rows[0]?.state).toBe('running');
|
|
272
298
|
expect(rows[0]?.reportedAt.getTime()).toBeLessThan(Date.now() + 10_000);
|
|
273
299
|
});
|
|
@@ -278,34 +304,34 @@ describe('reportProvisionedRunners', () => {
|
|
|
278
304
|
const stoppedAt = new Date('2025-01-01T00:02:00.000Z');
|
|
279
305
|
const terminatedAt = new Date('2025-01-01T00:03:00.000Z');
|
|
280
306
|
|
|
281
|
-
const result = await
|
|
307
|
+
const result = await reportRunnerInstances({
|
|
282
308
|
workspaceId,
|
|
283
309
|
provisionerId,
|
|
284
310
|
events: [
|
|
285
311
|
event({
|
|
286
|
-
|
|
312
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
287
313
|
state: 'running',
|
|
288
314
|
reportedAt: startedAt,
|
|
289
315
|
}),
|
|
290
316
|
event({
|
|
291
|
-
|
|
317
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
292
318
|
state: 'stopping',
|
|
293
319
|
reportedAt: stoppingAt,
|
|
294
320
|
}),
|
|
295
321
|
event({
|
|
296
|
-
|
|
322
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
297
323
|
state: 'stopped',
|
|
298
324
|
reportedAt: stoppedAt,
|
|
299
325
|
}),
|
|
300
326
|
event({
|
|
301
|
-
|
|
327
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
302
328
|
state: 'terminated',
|
|
303
329
|
reportedAt: terminatedAt,
|
|
304
330
|
}),
|
|
305
331
|
],
|
|
306
332
|
});
|
|
307
333
|
|
|
308
|
-
const rows = await
|
|
334
|
+
const rows = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
309
335
|
expect(result).toEqual({accepted: 1, reservationsReleased: 0, terminateIntentsHonored: []});
|
|
310
336
|
expect(rows[0]?.state).toBe('terminated');
|
|
311
337
|
expect(rows[0]?.reportedAt.toISOString()).toBe(terminatedAt.toISOString());
|
|
@@ -319,31 +345,31 @@ describe('reportProvisionedRunners', () => {
|
|
|
319
345
|
it('records delayed lower-state milestones without reviving current state', async () => {
|
|
320
346
|
const terminatedAt = new Date('2025-01-01T00:03:00.000Z');
|
|
321
347
|
const startedAt = new Date('2025-01-01T00:00:00.000Z');
|
|
322
|
-
await
|
|
348
|
+
await reportRunnerInstances({
|
|
323
349
|
workspaceId,
|
|
324
350
|
provisionerId,
|
|
325
351
|
events: [
|
|
326
352
|
event({
|
|
327
|
-
|
|
353
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
328
354
|
state: 'terminated',
|
|
329
355
|
reportedAt: terminatedAt,
|
|
330
356
|
}),
|
|
331
357
|
],
|
|
332
358
|
});
|
|
333
359
|
|
|
334
|
-
await
|
|
360
|
+
await reportRunnerInstances({
|
|
335
361
|
workspaceId,
|
|
336
362
|
provisionerId,
|
|
337
363
|
events: [
|
|
338
364
|
event({
|
|
339
|
-
|
|
365
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
340
366
|
state: 'running',
|
|
341
367
|
reportedAt: startedAt,
|
|
342
368
|
}),
|
|
343
369
|
],
|
|
344
370
|
});
|
|
345
371
|
|
|
346
|
-
const rows = await
|
|
372
|
+
const rows = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
347
373
|
expect(rows[0]?.state).toBe('terminated');
|
|
348
374
|
expect(rows[0]?.reportedAt.toISOString()).toBe(terminatedAt.toISOString());
|
|
349
375
|
expect(rows[0]?.startedAt?.toISOString()).toBe(startedAt.toISOString());
|
|
@@ -351,22 +377,22 @@ describe('reportProvisionedRunners', () => {
|
|
|
351
377
|
});
|
|
352
378
|
|
|
353
379
|
it('uses server update time for active provisioned runner windows', async () => {
|
|
354
|
-
await
|
|
380
|
+
await reportRunnerInstances({
|
|
355
381
|
workspaceId,
|
|
356
382
|
provisionerId,
|
|
357
383
|
events: [
|
|
358
384
|
event({
|
|
359
|
-
|
|
385
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
360
386
|
state: 'running',
|
|
361
387
|
reportedAt: new Date(Date.now() + 60 * 60 * 1000),
|
|
362
388
|
}),
|
|
363
389
|
],
|
|
364
390
|
});
|
|
365
391
|
await db().execute(
|
|
366
|
-
sql`UPDATE
|
|
392
|
+
sql`UPDATE runners_runner_instances SET updated_at = now() - interval '10 minutes'`,
|
|
367
393
|
);
|
|
368
394
|
|
|
369
|
-
const active = await
|
|
395
|
+
const active = await listActiveRunnerInstances({workspaceId, windowSeconds: 60});
|
|
370
396
|
|
|
371
397
|
expect(active).toEqual([]);
|
|
372
398
|
});
|
|
@@ -374,19 +400,17 @@ describe('reportProvisionedRunners', () => {
|
|
|
374
400
|
it('releases one reservation unit for a terminal unclaimed provisioned runner', async () => {
|
|
375
401
|
const reservationId = await createReservation(2);
|
|
376
402
|
|
|
377
|
-
const result = await
|
|
403
|
+
const result = await reportRunnerInstances({
|
|
378
404
|
workspaceId,
|
|
379
405
|
provisionerId,
|
|
380
|
-
events: [
|
|
381
|
-
event({provisionedRunnerId: 'provisioned-runner-1', reservationId, state: 'failed'}),
|
|
382
|
-
],
|
|
406
|
+
events: [event({providerRunnerId: 'provisioned-runner-1', reservationId, state: 'failed'})],
|
|
383
407
|
});
|
|
384
408
|
|
|
385
409
|
const reservationRows = await reservationRowsFor({workspaceId, provisionerId});
|
|
386
|
-
const
|
|
410
|
+
const providerRunnerRows = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
387
411
|
expect(result).toEqual({accepted: 1, reservationsReleased: 1, terminateIntentsHonored: []});
|
|
388
412
|
expect(reservationRows[0]?.count).toBe(1);
|
|
389
|
-
expect(
|
|
413
|
+
expect(providerRunnerRows[0]?.reservationReleasedAt).toBeInstanceOf(Date);
|
|
390
414
|
});
|
|
391
415
|
|
|
392
416
|
it('does not release a reservation owned by another workspace or provisioner', async () => {
|
|
@@ -399,17 +423,17 @@ describe('reportProvisionedRunners', () => {
|
|
|
399
423
|
provisionerId: crypto.randomUUID(),
|
|
400
424
|
});
|
|
401
425
|
|
|
402
|
-
const result = await
|
|
426
|
+
const result = await reportRunnerInstances({
|
|
403
427
|
workspaceId,
|
|
404
428
|
provisionerId,
|
|
405
429
|
events: [
|
|
406
430
|
event({
|
|
407
|
-
|
|
431
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
408
432
|
reservationId: otherWorkspaceReservationId,
|
|
409
433
|
state: 'failed',
|
|
410
434
|
}),
|
|
411
435
|
event({
|
|
412
|
-
|
|
436
|
+
providerRunnerId: 'provisioned-runner-2',
|
|
413
437
|
reservationId: peerProvisionerReservationId,
|
|
414
438
|
state: 'failed',
|
|
415
439
|
}),
|
|
@@ -428,19 +452,15 @@ describe('reportProvisionedRunners', () => {
|
|
|
428
452
|
it('releases a reservation only once across repeated terminal reports', async () => {
|
|
429
453
|
const reservationId = await createReservation(2);
|
|
430
454
|
|
|
431
|
-
await
|
|
455
|
+
await reportRunnerInstances({
|
|
432
456
|
workspaceId,
|
|
433
457
|
provisionerId,
|
|
434
|
-
events: [
|
|
435
|
-
event({provisionedRunnerId: 'provisioned-runner-1', reservationId, state: 'failed'}),
|
|
436
|
-
],
|
|
458
|
+
events: [event({providerRunnerId: 'provisioned-runner-1', reservationId, state: 'failed'})],
|
|
437
459
|
});
|
|
438
|
-
const result = await
|
|
460
|
+
const result = await reportRunnerInstances({
|
|
439
461
|
workspaceId,
|
|
440
462
|
provisionerId,
|
|
441
|
-
events: [
|
|
442
|
-
event({provisionedRunnerId: 'provisioned-runner-1', reservationId, state: 'failed'}),
|
|
443
|
-
],
|
|
463
|
+
events: [event({providerRunnerId: 'provisioned-runner-1', reservationId, state: 'failed'})],
|
|
444
464
|
});
|
|
445
465
|
|
|
446
466
|
const reservationRows = await reservationRowsFor({workspaceId, provisionerId});
|
|
@@ -451,12 +471,12 @@ describe('reportProvisionedRunners', () => {
|
|
|
451
471
|
it('does not let a newer running report revive a terminal provisioned runner', async () => {
|
|
452
472
|
const reservationId = await createReservation(2);
|
|
453
473
|
const failedAt = new Date();
|
|
454
|
-
await
|
|
474
|
+
await reportRunnerInstances({
|
|
455
475
|
workspaceId,
|
|
456
476
|
provisionerId,
|
|
457
477
|
events: [
|
|
458
478
|
event({
|
|
459
|
-
|
|
479
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
460
480
|
reservationId,
|
|
461
481
|
state: 'failed',
|
|
462
482
|
reportedAt: failedAt,
|
|
@@ -464,12 +484,12 @@ describe('reportProvisionedRunners', () => {
|
|
|
464
484
|
],
|
|
465
485
|
});
|
|
466
486
|
|
|
467
|
-
await
|
|
487
|
+
await reportRunnerInstances({
|
|
468
488
|
workspaceId,
|
|
469
489
|
provisionerId,
|
|
470
490
|
events: [
|
|
471
491
|
event({
|
|
472
|
-
|
|
492
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
473
493
|
reservationId,
|
|
474
494
|
state: 'running',
|
|
475
495
|
reportedAt: new Date(failedAt.getTime() + 1_000),
|
|
@@ -477,42 +497,42 @@ describe('reportProvisionedRunners', () => {
|
|
|
477
497
|
],
|
|
478
498
|
});
|
|
479
499
|
|
|
480
|
-
const
|
|
500
|
+
const providerRunnerRows = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
481
501
|
const reservationRows = await reservationRowsFor({workspaceId, provisionerId});
|
|
482
|
-
expect(
|
|
483
|
-
expect(
|
|
502
|
+
expect(providerRunnerRows[0]?.state).toBe('failed');
|
|
503
|
+
expect(providerRunnerRows[0]?.reservationReleasedAt).toBeInstanceOf(Date);
|
|
484
504
|
expect(reservationRows[0]?.count).toBe(1);
|
|
485
505
|
});
|
|
486
506
|
|
|
487
507
|
it('tracks provider cleanup as terminated', async () => {
|
|
488
508
|
const reservationId = await createReservation(2);
|
|
489
509
|
|
|
490
|
-
const result = await
|
|
510
|
+
const result = await reportRunnerInstances({
|
|
491
511
|
workspaceId,
|
|
492
512
|
provisionerId,
|
|
493
513
|
events: [
|
|
494
|
-
event({
|
|
514
|
+
event({providerRunnerId: 'provisioned-runner-1', reservationId, state: 'terminated'}),
|
|
495
515
|
],
|
|
496
516
|
});
|
|
497
517
|
|
|
498
518
|
const reservationRows = await reservationRowsFor({workspaceId, provisionerId});
|
|
499
|
-
const
|
|
519
|
+
const providerRunnerRows = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
500
520
|
expect(result).toEqual({accepted: 1, reservationsReleased: 1, terminateIntentsHonored: []});
|
|
501
521
|
expect(reservationRows[0]?.count).toBe(1);
|
|
502
|
-
expect(
|
|
503
|
-
expect(
|
|
504
|
-
expect(
|
|
522
|
+
expect(providerRunnerRows[0]?.state).toBe('terminated');
|
|
523
|
+
expect(providerRunnerRows[0]?.terminatedAt).toBeInstanceOf(Date);
|
|
524
|
+
expect(providerRunnerRows[0]?.reservationReleasedAt).toBeInstanceOf(Date);
|
|
505
525
|
});
|
|
506
526
|
|
|
507
527
|
it('releases multiple units from the same reservation in one batch', async () => {
|
|
508
528
|
const reservationId = await createReservation(3);
|
|
509
529
|
|
|
510
|
-
const result = await
|
|
530
|
+
const result = await reportRunnerInstances({
|
|
511
531
|
workspaceId,
|
|
512
532
|
provisionerId,
|
|
513
533
|
events: [
|
|
514
|
-
event({
|
|
515
|
-
event({
|
|
534
|
+
event({providerRunnerId: 'provisioned-runner-1', reservationId, state: 'failed'}),
|
|
535
|
+
event({providerRunnerId: 'provisioned-runner-2', reservationId, state: 'stopped'}),
|
|
516
536
|
],
|
|
517
537
|
});
|
|
518
538
|
|
|
@@ -524,12 +544,10 @@ describe('reportProvisionedRunners', () => {
|
|
|
524
544
|
it('deletes a one-unit reservation instead of violating the positive count check', async () => {
|
|
525
545
|
const reservationId = await createReservation(1);
|
|
526
546
|
|
|
527
|
-
const result = await
|
|
547
|
+
const result = await reportRunnerInstances({
|
|
528
548
|
workspaceId,
|
|
529
549
|
provisionerId,
|
|
530
|
-
events: [
|
|
531
|
-
event({provisionedRunnerId: 'provisioned-runner-1', reservationId, state: 'failed'}),
|
|
532
|
-
],
|
|
550
|
+
events: [event({providerRunnerId: 'provisioned-runner-1', reservationId, state: 'failed'})],
|
|
533
551
|
});
|
|
534
552
|
|
|
535
553
|
const reservationRows = await reservationRowsFor({workspaceId, provisionerId});
|
|
@@ -548,32 +566,32 @@ describe('reportProvisionedRunners', () => {
|
|
|
548
566
|
const [reservation] = await reservationRowsFor({workspaceId, provisionerId});
|
|
549
567
|
if (!reservation) throw new Error('Expected reservation');
|
|
550
568
|
|
|
551
|
-
const result = await
|
|
569
|
+
const result = await reportRunnerInstances({
|
|
552
570
|
workspaceId,
|
|
553
571
|
provisionerId,
|
|
554
572
|
events: [
|
|
555
573
|
event({
|
|
556
|
-
|
|
574
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
557
575
|
reservationId: reservation.id,
|
|
558
576
|
state: 'failed',
|
|
559
577
|
}),
|
|
560
578
|
],
|
|
561
579
|
});
|
|
562
580
|
|
|
563
|
-
const
|
|
581
|
+
const providerRunnerRows = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
564
582
|
expect(result).toEqual({accepted: 1, reservationsReleased: 0, terminateIntentsHonored: []});
|
|
565
|
-
expect(
|
|
583
|
+
expect(providerRunnerRows[0]?.reservationReleasedAt).toBeInstanceOf(Date);
|
|
566
584
|
});
|
|
567
585
|
|
|
568
586
|
it('does not release a reservation for a provisioned runner that already has a runner session', async () => {
|
|
569
587
|
const reservationId = await createReservation(1);
|
|
570
588
|
|
|
571
|
-
const result = await
|
|
589
|
+
const result = await reportRunnerInstances({
|
|
572
590
|
workspaceId,
|
|
573
591
|
provisionerId,
|
|
574
592
|
events: [
|
|
575
593
|
event({
|
|
576
|
-
|
|
594
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
577
595
|
reservationId,
|
|
578
596
|
state: 'failed',
|
|
579
597
|
runnerSessionId: crypto.randomUUID(),
|
|
@@ -582,10 +600,10 @@ describe('reportProvisionedRunners', () => {
|
|
|
582
600
|
});
|
|
583
601
|
|
|
584
602
|
const reservationRows = await reservationRowsFor({workspaceId, provisionerId});
|
|
585
|
-
const
|
|
603
|
+
const providerRunnerRows = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
586
604
|
expect(result).toEqual({accepted: 1, reservationsReleased: 0, terminateIntentsHonored: []});
|
|
587
605
|
expect(reservationRows[0]?.count).toBe(1);
|
|
588
|
-
expect(
|
|
606
|
+
expect(providerRunnerRows[0]?.reservationReleasedAt).toBeNull();
|
|
589
607
|
});
|
|
590
608
|
|
|
591
609
|
it('uses the consumed ephemeral token session before releasing a terminal report', async () => {
|
|
@@ -594,7 +612,7 @@ describe('reportProvisionedRunners', () => {
|
|
|
594
612
|
workspaceId,
|
|
595
613
|
provisionerId,
|
|
596
614
|
reservationId,
|
|
597
|
-
|
|
615
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
598
616
|
});
|
|
599
617
|
const session = await createRunnerSessionConsumingEphemeralToken({
|
|
600
618
|
ephemeralTokenId: token.id,
|
|
@@ -603,12 +621,12 @@ describe('reportProvisionedRunners', () => {
|
|
|
603
621
|
maxClaims: 1,
|
|
604
622
|
});
|
|
605
623
|
|
|
606
|
-
const result = await
|
|
624
|
+
const result = await reportRunnerInstances({
|
|
607
625
|
workspaceId,
|
|
608
626
|
provisionerId,
|
|
609
627
|
events: [
|
|
610
628
|
event({
|
|
611
|
-
|
|
629
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
612
630
|
reservationId,
|
|
613
631
|
state: 'failed',
|
|
614
632
|
runnerSessionId: crypto.randomUUID(),
|
|
@@ -617,11 +635,11 @@ describe('reportProvisionedRunners', () => {
|
|
|
617
635
|
});
|
|
618
636
|
|
|
619
637
|
const reservationRows = await reservationRowsFor({workspaceId, provisionerId});
|
|
620
|
-
const
|
|
638
|
+
const providerRunnerRows = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
621
639
|
expect(result).toEqual({accepted: 1, reservationsReleased: 0, terminateIntentsHonored: []});
|
|
622
640
|
expect(reservationRows[0]?.count).toBe(1);
|
|
623
|
-
expect(
|
|
624
|
-
expect(
|
|
641
|
+
expect(providerRunnerRows[0]?.runnerSessionId).toBe(session.id);
|
|
642
|
+
expect(providerRunnerRows[0]?.reservationReleasedAt).toBeNull();
|
|
625
643
|
});
|
|
626
644
|
|
|
627
645
|
it('preserves claimed runner session metadata when a terminal state wins the batch', async () => {
|
|
@@ -629,19 +647,19 @@ describe('reportProvisionedRunners', () => {
|
|
|
629
647
|
const runnerSessionId = crypto.randomUUID();
|
|
630
648
|
const reportedAt = new Date('2025-01-01T00:00:00.000Z');
|
|
631
649
|
|
|
632
|
-
const result = await
|
|
650
|
+
const result = await reportRunnerInstances({
|
|
633
651
|
workspaceId,
|
|
634
652
|
provisionerId,
|
|
635
653
|
events: [
|
|
636
654
|
event({
|
|
637
|
-
|
|
655
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
638
656
|
reservationId,
|
|
639
657
|
state: 'running',
|
|
640
658
|
runnerSessionId,
|
|
641
659
|
reportedAt,
|
|
642
660
|
}),
|
|
643
661
|
event({
|
|
644
|
-
|
|
662
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
645
663
|
reservationId,
|
|
646
664
|
state: 'failed',
|
|
647
665
|
reportedAt: new Date(reportedAt.getTime() + 1_000),
|
|
@@ -650,41 +668,41 @@ describe('reportProvisionedRunners', () => {
|
|
|
650
668
|
});
|
|
651
669
|
|
|
652
670
|
const reservationRows = await reservationRowsFor({workspaceId, provisionerId});
|
|
653
|
-
const
|
|
671
|
+
const providerRunnerRows = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
654
672
|
expect(result).toEqual({accepted: 1, reservationsReleased: 0, terminateIntentsHonored: []});
|
|
655
673
|
expect(reservationRows[0]?.count).toBe(1);
|
|
656
|
-
expect(
|
|
657
|
-
expect(
|
|
658
|
-
expect(
|
|
674
|
+
expect(providerRunnerRows[0]?.state).toBe('failed');
|
|
675
|
+
expect(providerRunnerRows[0]?.runnerSessionId).toBe(runnerSessionId);
|
|
676
|
+
expect(providerRunnerRows[0]?.reservationReleasedAt).toBeNull();
|
|
659
677
|
});
|
|
660
678
|
|
|
661
679
|
it('returns honored terminate intents only for the first active-to-terminated transition', async () => {
|
|
662
|
-
await
|
|
680
|
+
await providerRunnerFactory.create({
|
|
663
681
|
workspaceId,
|
|
664
682
|
provisionerId,
|
|
665
|
-
|
|
683
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
666
684
|
state: 'running',
|
|
667
685
|
});
|
|
668
686
|
await insertRunningJobRow({
|
|
669
687
|
workspaceId,
|
|
670
688
|
provisionerId,
|
|
671
|
-
|
|
689
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
672
690
|
cancellationRequestedAt: new Date('2025-01-01T00:01:00.000Z'),
|
|
673
691
|
});
|
|
674
692
|
|
|
675
|
-
const first = await
|
|
693
|
+
const first = await reportRunnerInstances({
|
|
676
694
|
workspaceId,
|
|
677
695
|
provisionerId,
|
|
678
|
-
events: [event({
|
|
696
|
+
events: [event({providerRunnerId: 'provisioned-runner-1', state: 'terminated'})],
|
|
679
697
|
});
|
|
680
|
-
const second = await
|
|
698
|
+
const second = await reportRunnerInstances({
|
|
681
699
|
workspaceId,
|
|
682
700
|
provisionerId,
|
|
683
|
-
events: [event({
|
|
701
|
+
events: [event({providerRunnerId: 'provisioned-runner-1', state: 'terminated'})],
|
|
684
702
|
});
|
|
685
703
|
|
|
686
704
|
expect(first.terminateIntentsHonored).toEqual([
|
|
687
|
-
{
|
|
705
|
+
{providerRunnerId: 'provisioned-runner-1', reason: 'job-cancelled'},
|
|
688
706
|
]);
|
|
689
707
|
expect(second.terminateIntentsHonored).toEqual([]);
|
|
690
708
|
});
|
|
@@ -716,7 +734,7 @@ describe('reportProvisionedRunners', () => {
|
|
|
716
734
|
}
|
|
717
735
|
|
|
718
736
|
function event(params: {
|
|
719
|
-
|
|
737
|
+
providerRunnerId?: string;
|
|
720
738
|
reservationId?: string | null;
|
|
721
739
|
state?: 'starting' | 'running' | 'stopping' | 'stopped' | 'failed' | 'terminated';
|
|
722
740
|
reportedAt?: Date;
|
|
@@ -724,7 +742,7 @@ describe('reportProvisionedRunners', () => {
|
|
|
724
742
|
runnerSessionId?: string | null;
|
|
725
743
|
}) {
|
|
726
744
|
return {
|
|
727
|
-
|
|
745
|
+
providerRunnerId: params.providerRunnerId ?? 'provisioned-runner-1',
|
|
728
746
|
reservationId: params.reservationId ?? null,
|
|
729
747
|
templateKey: 'linux',
|
|
730
748
|
labels: ['linux'],
|
|
@@ -737,7 +755,7 @@ describe('reportProvisionedRunners', () => {
|
|
|
737
755
|
}
|
|
738
756
|
});
|
|
739
757
|
|
|
740
|
-
describe('
|
|
758
|
+
describe('listActiveRunnerInstanceCountsByTemplateTx', () => {
|
|
741
759
|
let workspaceId: string;
|
|
742
760
|
let provisionerId: string;
|
|
743
761
|
|
|
@@ -747,44 +765,44 @@ describe('listActiveProvisionedRunnerCountsByTemplateTx', () => {
|
|
|
747
765
|
});
|
|
748
766
|
|
|
749
767
|
it('groups starting and running provisioned runners by template and ignores non-divergence states', async () => {
|
|
750
|
-
await
|
|
768
|
+
await providerRunnerFactory.create({
|
|
751
769
|
workspaceId,
|
|
752
770
|
provisionerId,
|
|
753
|
-
|
|
771
|
+
providerRunnerId: 'starting-1',
|
|
754
772
|
templateKey: 'linux',
|
|
755
773
|
state: 'starting',
|
|
756
774
|
});
|
|
757
|
-
await
|
|
775
|
+
await providerRunnerFactory.create({
|
|
758
776
|
workspaceId,
|
|
759
777
|
provisionerId,
|
|
760
|
-
|
|
778
|
+
providerRunnerId: 'running-1',
|
|
761
779
|
templateKey: 'linux',
|
|
762
780
|
state: 'running',
|
|
763
781
|
});
|
|
764
|
-
await
|
|
782
|
+
await providerRunnerFactory.create({
|
|
765
783
|
workspaceId,
|
|
766
784
|
provisionerId,
|
|
767
|
-
|
|
785
|
+
providerRunnerId: 'running-2',
|
|
768
786
|
templateKey: 'linux',
|
|
769
787
|
state: 'running',
|
|
770
788
|
});
|
|
771
|
-
await
|
|
789
|
+
await providerRunnerFactory.create({
|
|
772
790
|
workspaceId,
|
|
773
791
|
provisionerId,
|
|
774
|
-
|
|
792
|
+
providerRunnerId: 'stopping-1',
|
|
775
793
|
templateKey: 'linux',
|
|
776
794
|
state: 'stopping',
|
|
777
795
|
});
|
|
778
|
-
await
|
|
796
|
+
await providerRunnerFactory.create({
|
|
779
797
|
workspaceId,
|
|
780
798
|
provisionerId,
|
|
781
|
-
|
|
799
|
+
providerRunnerId: 'null-template',
|
|
782
800
|
templateKey: null,
|
|
783
801
|
state: 'running',
|
|
784
802
|
});
|
|
785
803
|
|
|
786
804
|
const result = await db().transaction((tx) =>
|
|
787
|
-
|
|
805
|
+
listActiveRunnerInstanceCountsByTemplateTx(tx, {workspaceId, provisionerId}),
|
|
788
806
|
);
|
|
789
807
|
|
|
790
808
|
expect(result).toEqual(
|
|
@@ -807,11 +825,11 @@ describe('listProvisionerTerminateIntents', () => {
|
|
|
807
825
|
});
|
|
808
826
|
|
|
809
827
|
it('includes active provisioned runners whose latest bound job is cancelled', async () => {
|
|
810
|
-
await
|
|
828
|
+
await createRunnerInstance({providerRunnerId: 'provisioned-runner-1'});
|
|
811
829
|
await insertRunningJobRow({
|
|
812
830
|
workspaceId,
|
|
813
831
|
provisionerId,
|
|
814
|
-
|
|
832
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
815
833
|
cancellationRequestedAt: new Date('2025-01-01T00:01:00.000Z'),
|
|
816
834
|
});
|
|
817
835
|
|
|
@@ -821,11 +839,11 @@ describe('listProvisionerTerminateIntents', () => {
|
|
|
821
839
|
});
|
|
822
840
|
|
|
823
841
|
it('returns structured rows with bounded reasons from the shared query', async () => {
|
|
824
|
-
await
|
|
842
|
+
await createRunnerInstance({providerRunnerId: 'provisioned-runner-1'});
|
|
825
843
|
await insertRunningJobRow({
|
|
826
844
|
workspaceId,
|
|
827
845
|
provisionerId,
|
|
828
|
-
|
|
846
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
829
847
|
cancellationRequestedAt: new Date('2025-01-01T00:01:00.000Z'),
|
|
830
848
|
});
|
|
831
849
|
|
|
@@ -833,17 +851,15 @@ describe('listProvisionerTerminateIntents', () => {
|
|
|
833
851
|
listProvisionerTerminateIntentRowsTx(tx, {workspaceId, provisionerId, limit: 1000}),
|
|
834
852
|
);
|
|
835
853
|
|
|
836
|
-
expect(result).toEqual([
|
|
837
|
-
{provisionedRunnerId: 'provisioned-runner-1', reason: 'job-cancelled'},
|
|
838
|
-
]);
|
|
854
|
+
expect(result).toEqual([{providerRunnerId: 'provisioned-runner-1', reason: 'job-cancelled'}]);
|
|
839
855
|
});
|
|
840
856
|
|
|
841
857
|
it('excludes active provisioned runners whose latest bound job is healthy', async () => {
|
|
842
|
-
await
|
|
858
|
+
await createRunnerInstance({providerRunnerId: 'provisioned-runner-1'});
|
|
843
859
|
await insertRunningJobRow({
|
|
844
860
|
workspaceId,
|
|
845
861
|
provisionerId,
|
|
846
|
-
|
|
862
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
847
863
|
});
|
|
848
864
|
|
|
849
865
|
const result = await listProvisionerTerminateIntents({workspaceId, provisionerId, limit: 1000});
|
|
@@ -852,14 +868,14 @@ describe('listProvisionerTerminateIntents', () => {
|
|
|
852
868
|
});
|
|
853
869
|
|
|
854
870
|
it('excludes terminal provisioned runners with cancelled bound jobs', async () => {
|
|
855
|
-
await
|
|
856
|
-
|
|
871
|
+
await createRunnerInstance({
|
|
872
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
857
873
|
state: 'terminated',
|
|
858
874
|
});
|
|
859
875
|
await insertRunningJobRow({
|
|
860
876
|
workspaceId,
|
|
861
877
|
provisionerId,
|
|
862
|
-
|
|
878
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
863
879
|
cancellationRequestedAt: new Date('2025-01-01T00:01:00.000Z'),
|
|
864
880
|
});
|
|
865
881
|
|
|
@@ -869,18 +885,18 @@ describe('listProvisionerTerminateIntents', () => {
|
|
|
869
885
|
});
|
|
870
886
|
|
|
871
887
|
it('excludes a cancelled job when it is not the latest bound job', async () => {
|
|
872
|
-
await
|
|
888
|
+
await createRunnerInstance({providerRunnerId: 'provisioned-runner-1'});
|
|
873
889
|
await insertRunningJobRow({
|
|
874
890
|
workspaceId,
|
|
875
891
|
provisionerId,
|
|
876
|
-
|
|
892
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
877
893
|
startedAt: new Date('2025-01-01T00:00:00.000Z'),
|
|
878
894
|
cancellationRequestedAt: new Date('2025-01-01T00:01:00.000Z'),
|
|
879
895
|
});
|
|
880
896
|
await insertRunningJobRow({
|
|
881
897
|
workspaceId,
|
|
882
898
|
provisionerId,
|
|
883
|
-
|
|
899
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
884
900
|
startedAt: new Date('2025-01-01T00:02:00.000Z'),
|
|
885
901
|
});
|
|
886
902
|
|
|
@@ -891,13 +907,13 @@ describe('listProvisionerTerminateIntents', () => {
|
|
|
891
907
|
|
|
892
908
|
it('excludes cancelled jobs for another provisioner', async () => {
|
|
893
909
|
const otherProvisionerId = crypto.randomUUID();
|
|
894
|
-
await
|
|
895
|
-
|
|
910
|
+
await createRunnerInstance({
|
|
911
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
896
912
|
provisionerId: otherProvisionerId,
|
|
897
913
|
});
|
|
898
914
|
await insertRunningJobRow({
|
|
899
915
|
workspaceId,
|
|
900
|
-
|
|
916
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
901
917
|
provisionerId: otherProvisionerId,
|
|
902
918
|
cancellationRequestedAt: new Date('2025-01-01T00:01:00.000Z'),
|
|
903
919
|
});
|
|
@@ -908,18 +924,18 @@ describe('listProvisionerTerminateIntents', () => {
|
|
|
908
924
|
});
|
|
909
925
|
|
|
910
926
|
it('returns one id for duplicate cancelled bound jobs on the same provisioned runner', async () => {
|
|
911
|
-
await
|
|
927
|
+
await createRunnerInstance({providerRunnerId: 'provisioned-runner-1'});
|
|
912
928
|
await insertRunningJobRow({
|
|
913
929
|
workspaceId,
|
|
914
930
|
provisionerId,
|
|
915
|
-
|
|
931
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
916
932
|
startedAt: new Date('2025-01-01T00:00:00.000Z'),
|
|
917
933
|
cancellationRequestedAt: new Date('2025-01-01T00:01:00.000Z'),
|
|
918
934
|
});
|
|
919
935
|
await insertRunningJobRow({
|
|
920
936
|
workspaceId,
|
|
921
937
|
provisionerId,
|
|
922
|
-
|
|
938
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
923
939
|
startedAt: new Date('2025-01-01T00:00:00.000Z'),
|
|
924
940
|
cancellationRequestedAt: new Date('2025-01-01T00:01:00.000Z'),
|
|
925
941
|
});
|
|
@@ -930,16 +946,16 @@ describe('listProvisionerTerminateIntents', () => {
|
|
|
930
946
|
});
|
|
931
947
|
|
|
932
948
|
it('returns a deterministic subset when the limit truncates results', async () => {
|
|
933
|
-
for (const
|
|
949
|
+
for (const providerRunnerId of [
|
|
934
950
|
'provisioned-runner-c',
|
|
935
951
|
'provisioned-runner-a',
|
|
936
952
|
'provisioned-runner-b',
|
|
937
953
|
]) {
|
|
938
|
-
await
|
|
954
|
+
await createRunnerInstance({providerRunnerId});
|
|
939
955
|
await insertRunningJobRow({
|
|
940
956
|
workspaceId,
|
|
941
957
|
provisionerId,
|
|
942
|
-
|
|
958
|
+
providerRunnerId,
|
|
943
959
|
cancellationRequestedAt: new Date('2025-01-01T00:01:00.000Z'),
|
|
944
960
|
});
|
|
945
961
|
}
|
|
@@ -949,21 +965,21 @@ describe('listProvisionerTerminateIntents', () => {
|
|
|
949
965
|
expect(result).toEqual(['provisioned-runner-a', 'provisioned-runner-b']);
|
|
950
966
|
});
|
|
951
967
|
|
|
952
|
-
async function
|
|
953
|
-
|
|
968
|
+
async function createRunnerInstance(params: {
|
|
969
|
+
providerRunnerId: string;
|
|
954
970
|
provisionerId?: string;
|
|
955
971
|
state?: 'starting' | 'running' | 'stopping' | 'stopped' | 'failed' | 'terminated';
|
|
956
972
|
}) {
|
|
957
|
-
return await
|
|
973
|
+
return await providerRunnerFactory.create({
|
|
958
974
|
workspaceId,
|
|
959
975
|
provisionerId: params.provisionerId ?? provisionerId,
|
|
960
|
-
|
|
976
|
+
providerRunnerId: params.providerRunnerId,
|
|
961
977
|
state: params.state ?? 'running',
|
|
962
978
|
});
|
|
963
979
|
}
|
|
964
980
|
});
|
|
965
981
|
|
|
966
|
-
describe('
|
|
982
|
+
describe('reapStaleRunnerInstances', () => {
|
|
967
983
|
let workspaceId: string;
|
|
968
984
|
let provisionerId: string;
|
|
969
985
|
|
|
@@ -975,58 +991,90 @@ describe('reapStaleProvisionedRunners', () => {
|
|
|
975
991
|
|
|
976
992
|
it('fails stale unclaimed provisioned runners and releases reservations', async () => {
|
|
977
993
|
const reservationId = await createReservation(2);
|
|
978
|
-
await
|
|
979
|
-
|
|
994
|
+
await createRunnerInstance({
|
|
995
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
980
996
|
reservationId,
|
|
981
997
|
reportedAt: staleAt(),
|
|
982
998
|
updatedAt: staleAt(),
|
|
983
999
|
});
|
|
984
1000
|
|
|
985
|
-
const result = await
|
|
1001
|
+
const result = await reapStaleRunnerInstances({thresholdSeconds: 60, limit: 100});
|
|
986
1002
|
|
|
987
|
-
const [
|
|
1003
|
+
const [providerRunner] = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
988
1004
|
const [reservation] = await reservationRowsFor({workspaceId, provisionerId});
|
|
989
1005
|
expect(result).toEqual({reaped: 1, reservationsReleased: 1});
|
|
990
|
-
expect(
|
|
1006
|
+
expect(providerRunner).toMatchObject({
|
|
991
1007
|
state: 'failed',
|
|
992
1008
|
reason: 'stale-provisioner',
|
|
993
1009
|
});
|
|
994
|
-
expect(
|
|
995
|
-
expect(
|
|
1010
|
+
expect(providerRunner?.failedAt).toBeInstanceOf(Date);
|
|
1011
|
+
expect(providerRunner?.reservationReleasedAt).toBeInstanceOf(Date);
|
|
996
1012
|
expect(reservation?.id).toBe(reservationId);
|
|
997
1013
|
expect(reservation?.count).toBe(1);
|
|
998
1014
|
});
|
|
999
1015
|
|
|
1016
|
+
it('fails a stale installation runner instance before provider attachment', async () => {
|
|
1017
|
+
const installationProvisioner = await provisionerTokenFactory.create({
|
|
1018
|
+
scope: 'installation',
|
|
1019
|
+
workspaceId: null,
|
|
1020
|
+
});
|
|
1021
|
+
await db()
|
|
1022
|
+
.update(provisionerTokens)
|
|
1023
|
+
.set({lastSeenAt: null})
|
|
1024
|
+
.where(eq(provisionerTokens.id, installationProvisioner.id));
|
|
1025
|
+
const [instance] = await db()
|
|
1026
|
+
.insert(providerRunners)
|
|
1027
|
+
.values({
|
|
1028
|
+
provisionerId: installationProvisioner.id,
|
|
1029
|
+
providerKind: 'docker',
|
|
1030
|
+
labels: [],
|
|
1031
|
+
state: 'starting',
|
|
1032
|
+
reportedAt: staleAt(),
|
|
1033
|
+
updatedAt: staleAt(),
|
|
1034
|
+
})
|
|
1035
|
+
.returning({id: providerRunners.id});
|
|
1036
|
+
if (!instance) throw new Error('Runner instance insert returned no row');
|
|
1037
|
+
|
|
1038
|
+
const result = await reapStaleRunnerInstances({thresholdSeconds: 60, limit: 100});
|
|
1039
|
+
const [row] = await db()
|
|
1040
|
+
.select()
|
|
1041
|
+
.from(providerRunners)
|
|
1042
|
+
.where(eq(providerRunners.id, instance.id));
|
|
1043
|
+
|
|
1044
|
+
expect(result.reaped).toBe(1);
|
|
1045
|
+
expect(row).toMatchObject({workspaceId: null, providerRunnerId: null, state: 'failed'});
|
|
1046
|
+
});
|
|
1047
|
+
|
|
1000
1048
|
it('skips fresh rows, live provisioners, terminal rows, running jobs, and fresh sessions', async () => {
|
|
1001
|
-
await
|
|
1002
|
-
|
|
1049
|
+
await createRunnerInstance({
|
|
1050
|
+
providerRunnerId: 'fresh-row',
|
|
1003
1051
|
reportedAt: new Date(),
|
|
1004
1052
|
updatedAt: new Date(),
|
|
1005
1053
|
});
|
|
1006
|
-
await
|
|
1007
|
-
|
|
1054
|
+
await createRunnerInstance({
|
|
1055
|
+
providerRunnerId: 'live-provisioner',
|
|
1008
1056
|
reportedAt: staleAt(),
|
|
1009
1057
|
updatedAt: staleAt(),
|
|
1010
1058
|
});
|
|
1011
|
-
await
|
|
1012
|
-
|
|
1059
|
+
await createRunnerInstance({
|
|
1060
|
+
providerRunnerId: 'terminal-row',
|
|
1013
1061
|
reportedAt: staleAt(),
|
|
1014
1062
|
updatedAt: staleAt(),
|
|
1015
1063
|
state: 'failed',
|
|
1016
1064
|
});
|
|
1017
|
-
await
|
|
1018
|
-
|
|
1065
|
+
await createRunnerInstance({
|
|
1066
|
+
providerRunnerId: 'running-job',
|
|
1019
1067
|
reportedAt: staleAt(),
|
|
1020
1068
|
updatedAt: staleAt(),
|
|
1021
1069
|
});
|
|
1022
|
-
await insertRunningJob({
|
|
1023
|
-
const freshSession = await
|
|
1024
|
-
|
|
1070
|
+
await insertRunningJob({providerRunnerId: 'running-job'});
|
|
1071
|
+
const freshSession = await createRunnerInstance({
|
|
1072
|
+
providerRunnerId: 'fresh-session',
|
|
1025
1073
|
reportedAt: staleAt(),
|
|
1026
1074
|
updatedAt: staleAt(),
|
|
1027
1075
|
});
|
|
1028
1076
|
await createLinkedSession({
|
|
1029
|
-
|
|
1077
|
+
providerRunnerId: freshSession.providerRunnerId,
|
|
1030
1078
|
updatedAt: new Date(),
|
|
1031
1079
|
});
|
|
1032
1080
|
await db()
|
|
@@ -1034,13 +1082,13 @@ describe('reapStaleProvisionedRunners', () => {
|
|
|
1034
1082
|
.set({lastSeenAt: new Date()})
|
|
1035
1083
|
.where(eq(provisionerTokens.id, provisionerId));
|
|
1036
1084
|
|
|
1037
|
-
const result = await
|
|
1085
|
+
const result = await reapStaleRunnerInstances({thresholdSeconds: 60, limit: 100});
|
|
1038
1086
|
|
|
1039
|
-
const rows = await
|
|
1040
|
-
|
|
1087
|
+
const rows = await providerRunnerRowsFor({workspaceId, provisionerId}).orderBy(
|
|
1088
|
+
providerRunners.providerRunnerId,
|
|
1041
1089
|
);
|
|
1042
1090
|
expect(result).toEqual({reaped: 0, reservationsReleased: 0});
|
|
1043
|
-
expect(rows.map((row) => [row.
|
|
1091
|
+
expect(rows.map((row) => [row.providerRunnerId, row.state])).toEqual([
|
|
1044
1092
|
['fresh-row', 'running'],
|
|
1045
1093
|
['fresh-session', 'running'],
|
|
1046
1094
|
['live-provisioner', 'running'],
|
|
@@ -1051,29 +1099,29 @@ describe('reapStaleProvisionedRunners', () => {
|
|
|
1051
1099
|
|
|
1052
1100
|
it('releases reservations for stale rows whose linked session is no longer live', async () => {
|
|
1053
1101
|
const reservationId = await createReservation(1);
|
|
1054
|
-
await
|
|
1055
|
-
|
|
1102
|
+
await createRunnerInstance({
|
|
1103
|
+
providerRunnerId: 'stale-session',
|
|
1056
1104
|
reservationId,
|
|
1057
1105
|
runnerSessionId: '00000000-0000-4000-8000-000000000001',
|
|
1058
1106
|
reportedAt: staleAt(),
|
|
1059
1107
|
updatedAt: staleAt(),
|
|
1060
1108
|
});
|
|
1061
1109
|
const session = await createLinkedSession({
|
|
1062
|
-
|
|
1110
|
+
providerRunnerId: 'stale-session',
|
|
1063
1111
|
updatedAt: staleAt(),
|
|
1064
1112
|
});
|
|
1065
1113
|
await db()
|
|
1066
|
-
.update(
|
|
1114
|
+
.update(providerRunners)
|
|
1067
1115
|
.set({runnerSessionId: session.id})
|
|
1068
|
-
.where(eq(
|
|
1116
|
+
.where(eq(providerRunners.providerRunnerId, 'stale-session'));
|
|
1069
1117
|
|
|
1070
|
-
const result = await
|
|
1118
|
+
const result = await reapStaleRunnerInstances({thresholdSeconds: 60, limit: 100});
|
|
1071
1119
|
|
|
1072
1120
|
const reservationRows = await reservationRowsFor({workspaceId, provisionerId});
|
|
1073
|
-
const [
|
|
1121
|
+
const [providerRunner] = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
1074
1122
|
expect(result).toEqual({reaped: 1, reservationsReleased: 1});
|
|
1075
|
-
expect(
|
|
1076
|
-
expect(
|
|
1123
|
+
expect(providerRunner?.state).toBe('failed');
|
|
1124
|
+
expect(providerRunner?.reservationReleasedAt).toBeInstanceOf(Date);
|
|
1077
1125
|
expect(reservationRows.find((row) => row.id === reservationId)).toBeUndefined();
|
|
1078
1126
|
});
|
|
1079
1127
|
|
|
@@ -1082,36 +1130,36 @@ describe('reapStaleProvisionedRunners', () => {
|
|
|
1082
1130
|
const expiredReservationId = await createReservation(1, {
|
|
1083
1131
|
expiresAt: new Date(Date.now() - 60_000),
|
|
1084
1132
|
});
|
|
1085
|
-
await
|
|
1086
|
-
|
|
1133
|
+
await createRunnerInstance({
|
|
1134
|
+
providerRunnerId: 'stale-1',
|
|
1087
1135
|
reservationId: activeReservationId,
|
|
1088
1136
|
reportedAt: staleAt(240_000),
|
|
1089
1137
|
updatedAt: staleAt(240_000),
|
|
1090
1138
|
});
|
|
1091
|
-
await
|
|
1092
|
-
|
|
1139
|
+
await createRunnerInstance({
|
|
1140
|
+
providerRunnerId: 'stale-2',
|
|
1093
1141
|
reservationId: expiredReservationId,
|
|
1094
1142
|
reportedAt: staleAt(180_000),
|
|
1095
1143
|
updatedAt: staleAt(180_000),
|
|
1096
1144
|
});
|
|
1097
|
-
await
|
|
1098
|
-
|
|
1145
|
+
await createRunnerInstance({
|
|
1146
|
+
providerRunnerId: 'stale-3',
|
|
1099
1147
|
reportedAt: staleAt(120_000),
|
|
1100
1148
|
updatedAt: staleAt(120_000),
|
|
1101
1149
|
});
|
|
1102
1150
|
|
|
1103
|
-
const first = await
|
|
1104
|
-
const second = await
|
|
1151
|
+
const first = await reapStaleRunnerInstances({thresholdSeconds: 60, limit: 2});
|
|
1152
|
+
const second = await reapStaleRunnerInstances({thresholdSeconds: 60, limit: 2});
|
|
1105
1153
|
|
|
1106
1154
|
const reservationRows = await reservationRowsFor({workspaceId, provisionerId});
|
|
1107
|
-
const
|
|
1155
|
+
const providerRunnerRows = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
1108
1156
|
expect(first).toEqual({reaped: 2, reservationsReleased: 1});
|
|
1109
1157
|
expect(second).toEqual({reaped: 1, reservationsReleased: 0});
|
|
1110
1158
|
expect(reservationRows.find((row) => row.id === activeReservationId)?.count).toBe(1);
|
|
1111
1159
|
expect(reservationRows.find((row) => row.id === expiredReservationId)?.count).toBe(1);
|
|
1112
|
-
expect(
|
|
1160
|
+
expect(providerRunnerRows.every((row) => row.state === 'failed')).toBe(true);
|
|
1113
1161
|
expect(
|
|
1114
|
-
|
|
1162
|
+
providerRunnerRows
|
|
1115
1163
|
.filter((row) => row.reservationId)
|
|
1116
1164
|
.every((row) => row.reservationReleasedAt instanceof Date),
|
|
1117
1165
|
).toBe(true);
|
|
@@ -1119,8 +1167,8 @@ describe('reapStaleProvisionedRunners', () => {
|
|
|
1119
1167
|
|
|
1120
1168
|
it('does not double-release reservations when terminal report and reaper queue on the workspace lock', async () => {
|
|
1121
1169
|
const reservationId = await createReservation(2);
|
|
1122
|
-
await
|
|
1123
|
-
|
|
1170
|
+
await createRunnerInstance({
|
|
1171
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
1124
1172
|
reservationId,
|
|
1125
1173
|
reportedAt: staleAt(),
|
|
1126
1174
|
updatedAt: staleAt(),
|
|
@@ -1134,12 +1182,12 @@ describe('reapStaleProvisionedRunners', () => {
|
|
|
1134
1182
|
});
|
|
1135
1183
|
|
|
1136
1184
|
await lockHolderReady.promise;
|
|
1137
|
-
const report =
|
|
1185
|
+
const report = reportRunnerInstances({
|
|
1138
1186
|
workspaceId,
|
|
1139
1187
|
provisionerId,
|
|
1140
1188
|
events: [
|
|
1141
1189
|
event({
|
|
1142
|
-
|
|
1190
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
1143
1191
|
reservationId,
|
|
1144
1192
|
state: 'failed',
|
|
1145
1193
|
reportedAt: new Date(),
|
|
@@ -1147,7 +1195,7 @@ describe('reapStaleProvisionedRunners', () => {
|
|
|
1147
1195
|
],
|
|
1148
1196
|
});
|
|
1149
1197
|
await waitForLockWait({queryLike: '%pg_advisory_xact_lock%'});
|
|
1150
|
-
const reaper =
|
|
1198
|
+
const reaper = reapStaleRunnerInstances({thresholdSeconds: 60, limit: 100});
|
|
1151
1199
|
try {
|
|
1152
1200
|
await waitForLockWait({minWaiters: 2, queryLike: '%pg_advisory_xact_lock%'});
|
|
1153
1201
|
} finally {
|
|
@@ -1156,9 +1204,9 @@ describe('reapStaleProvisionedRunners', () => {
|
|
|
1156
1204
|
const [reportResult, reaperResult] = await Promise.all([report, reaper, lockHolder]);
|
|
1157
1205
|
|
|
1158
1206
|
const [reservation] = await reservationRowsFor({workspaceId, provisionerId});
|
|
1159
|
-
const [
|
|
1207
|
+
const [providerRunner] = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
1160
1208
|
expect(reportResult.reservationsReleased + reaperResult.reservationsReleased).toBe(1);
|
|
1161
|
-
expect(
|
|
1209
|
+
expect(providerRunner?.reservationReleasedAt).toBeInstanceOf(Date);
|
|
1162
1210
|
expect(reservation?.count).toBe(1);
|
|
1163
1211
|
});
|
|
1164
1212
|
|
|
@@ -1185,38 +1233,38 @@ describe('reapStaleProvisionedRunners', () => {
|
|
|
1185
1233
|
return reservation.id;
|
|
1186
1234
|
}
|
|
1187
1235
|
|
|
1188
|
-
async function
|
|
1189
|
-
|
|
1236
|
+
async function createRunnerInstance(params: {
|
|
1237
|
+
providerRunnerId: string;
|
|
1190
1238
|
reservationId?: string | null;
|
|
1191
1239
|
runnerSessionId?: string | null;
|
|
1192
1240
|
reportedAt: Date;
|
|
1193
1241
|
updatedAt: Date;
|
|
1194
1242
|
state?: 'starting' | 'running' | 'stopping' | 'stopped' | 'failed' | 'terminated';
|
|
1195
1243
|
}) {
|
|
1196
|
-
const row = await
|
|
1244
|
+
const row = await providerRunnerFactory.create({
|
|
1197
1245
|
workspaceId,
|
|
1198
1246
|
provisionerId,
|
|
1199
|
-
|
|
1247
|
+
providerRunnerId: params.providerRunnerId,
|
|
1200
1248
|
reservationId: params.reservationId ?? null,
|
|
1201
1249
|
runnerSessionId: params.runnerSessionId ?? null,
|
|
1202
1250
|
reportedAt: params.reportedAt,
|
|
1203
1251
|
state: params.state ?? 'running',
|
|
1204
1252
|
});
|
|
1205
1253
|
await db()
|
|
1206
|
-
.update(
|
|
1254
|
+
.update(providerRunners)
|
|
1207
1255
|
.set({updatedAt: params.updatedAt})
|
|
1208
|
-
.where(eq(
|
|
1256
|
+
.where(eq(providerRunners.id, row.id));
|
|
1209
1257
|
return {...row, updatedAt: params.updatedAt};
|
|
1210
1258
|
}
|
|
1211
1259
|
|
|
1212
|
-
async function createLinkedSession(params: {
|
|
1260
|
+
async function createLinkedSession(params: {providerRunnerId: string; updatedAt: Date}) {
|
|
1213
1261
|
const session = await runnerSessionFactory.create({workspaceId});
|
|
1214
1262
|
await db()
|
|
1215
1263
|
.update(runnerSessions)
|
|
1216
1264
|
.set({
|
|
1217
1265
|
registrationTokenKind: 'ephemeral',
|
|
1218
1266
|
provisionerId,
|
|
1219
|
-
|
|
1267
|
+
providerRunnerId: params.providerRunnerId,
|
|
1220
1268
|
maxClaims: 1,
|
|
1221
1269
|
updatedAt: params.updatedAt,
|
|
1222
1270
|
})
|
|
@@ -1224,7 +1272,7 @@ describe('reapStaleProvisionedRunners', () => {
|
|
|
1224
1272
|
return session;
|
|
1225
1273
|
}
|
|
1226
1274
|
|
|
1227
|
-
async function insertRunningJob(params: {
|
|
1275
|
+
async function insertRunningJob(params: {providerRunnerId: string}) {
|
|
1228
1276
|
const runnerSession = await runnerSessionFactory.create({workspaceId});
|
|
1229
1277
|
|
|
1230
1278
|
await db()
|
|
@@ -1238,7 +1286,7 @@ describe('reapStaleProvisionedRunners', () => {
|
|
|
1238
1286
|
projectId: crypto.randomUUID(),
|
|
1239
1287
|
runnerSessionId: runnerSession.id,
|
|
1240
1288
|
provisionerId,
|
|
1241
|
-
|
|
1289
|
+
providerRunnerId: params.providerRunnerId,
|
|
1242
1290
|
requiredLabels: ['linux'],
|
|
1243
1291
|
runnerLabels: ['linux'],
|
|
1244
1292
|
startedAt: new Date('2025-01-01T00:00:00.000Z'),
|
|
@@ -1251,13 +1299,13 @@ describe('reapStaleProvisionedRunners', () => {
|
|
|
1251
1299
|
}
|
|
1252
1300
|
|
|
1253
1301
|
function event(params: {
|
|
1254
|
-
|
|
1302
|
+
providerRunnerId: string;
|
|
1255
1303
|
reservationId?: string | null;
|
|
1256
1304
|
state?: 'starting' | 'running' | 'stopping' | 'stopped' | 'failed' | 'terminated';
|
|
1257
1305
|
reportedAt?: Date;
|
|
1258
1306
|
}) {
|
|
1259
1307
|
return {
|
|
1260
|
-
|
|
1308
|
+
providerRunnerId: params.providerRunnerId,
|
|
1261
1309
|
reservationId: params.reservationId ?? null,
|
|
1262
1310
|
templateKey: 'linux',
|
|
1263
1311
|
labels: ['linux'],
|
|
@@ -1270,7 +1318,7 @@ describe('reapStaleProvisionedRunners', () => {
|
|
|
1270
1318
|
}
|
|
1271
1319
|
});
|
|
1272
1320
|
|
|
1273
|
-
describe('
|
|
1321
|
+
describe('reconcileRunnerInstances', () => {
|
|
1274
1322
|
let workspaceId: string;
|
|
1275
1323
|
let provisionerId: string;
|
|
1276
1324
|
|
|
@@ -1281,76 +1329,76 @@ describe('reconcileProvisionedRunners', () => {
|
|
|
1281
1329
|
|
|
1282
1330
|
it('terminates stale absent provisioned runners and releases reservations', async () => {
|
|
1283
1331
|
const reservationId = await createReservation(2);
|
|
1284
|
-
await
|
|
1285
|
-
|
|
1332
|
+
await createRunnerInstance({
|
|
1333
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
1286
1334
|
reservationId,
|
|
1287
1335
|
reportedAt: staleReportedAt(),
|
|
1288
1336
|
});
|
|
1289
1337
|
|
|
1290
|
-
const result = await
|
|
1338
|
+
const result = await reconcileRunnerInstances({
|
|
1291
1339
|
workspaceId,
|
|
1292
1340
|
provisionerId,
|
|
1293
|
-
|
|
1341
|
+
observedRunnerInstanceIds: ['observed-runner'],
|
|
1294
1342
|
terminateGraceSeconds: 60,
|
|
1295
1343
|
});
|
|
1296
1344
|
|
|
1297
|
-
const [
|
|
1345
|
+
const [providerRunner] = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
1298
1346
|
const [reservation] = await reservationRowsFor({workspaceId, provisionerId});
|
|
1299
1347
|
expect(result.absentIds).toEqual(['provisioned-runner-1']);
|
|
1300
1348
|
expect(result.reservationsReleased).toBe(1);
|
|
1301
|
-
expect(
|
|
1302
|
-
expect(
|
|
1349
|
+
expect(providerRunner?.state).toBe('terminated');
|
|
1350
|
+
expect(providerRunner?.terminatedAt).toBeInstanceOf(Date);
|
|
1303
1351
|
expect(reservation?.count).toBe(1);
|
|
1304
1352
|
});
|
|
1305
1353
|
|
|
1306
1354
|
it('treats an empty observed set as read-only', async () => {
|
|
1307
1355
|
const reservationId = await createReservation(2);
|
|
1308
|
-
await
|
|
1309
|
-
|
|
1356
|
+
await createRunnerInstance({
|
|
1357
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
1310
1358
|
reservationId,
|
|
1311
1359
|
reportedAt: staleReportedAt(),
|
|
1312
1360
|
});
|
|
1313
1361
|
|
|
1314
|
-
const result = await
|
|
1362
|
+
const result = await reconcileRunnerInstances({
|
|
1315
1363
|
workspaceId,
|
|
1316
1364
|
provisionerId,
|
|
1317
|
-
|
|
1365
|
+
observedRunnerInstanceIds: [],
|
|
1318
1366
|
terminateGraceSeconds: 60,
|
|
1319
1367
|
});
|
|
1320
1368
|
|
|
1321
|
-
const [
|
|
1369
|
+
const [providerRunner] = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
1322
1370
|
const [reservation] = await reservationRowsFor({workspaceId, provisionerId});
|
|
1323
1371
|
expect(result.absentIds).toEqual([]);
|
|
1324
1372
|
expect(result.reservationsReleased).toBe(0);
|
|
1325
|
-
expect(
|
|
1326
|
-
expect(
|
|
1327
|
-
expect(
|
|
1373
|
+
expect(providerRunner?.state).toBe('running');
|
|
1374
|
+
expect(providerRunner?.terminatedAt).toBeNull();
|
|
1375
|
+
expect(providerRunner?.reservationReleasedAt).toBeNull();
|
|
1328
1376
|
expect(reservation?.count).toBe(2);
|
|
1329
1377
|
});
|
|
1330
1378
|
|
|
1331
1379
|
it('keeps fresh absent provisioned runners inside the grace window', async () => {
|
|
1332
|
-
await
|
|
1333
|
-
|
|
1380
|
+
await createRunnerInstance({
|
|
1381
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
1334
1382
|
reportedAt: new Date(),
|
|
1335
1383
|
});
|
|
1336
1384
|
|
|
1337
|
-
const result = await
|
|
1385
|
+
const result = await reconcileRunnerInstances({
|
|
1338
1386
|
workspaceId,
|
|
1339
1387
|
provisionerId,
|
|
1340
|
-
|
|
1388
|
+
observedRunnerInstanceIds: ['observed-runner'],
|
|
1341
1389
|
terminateGraceSeconds: 60,
|
|
1342
1390
|
});
|
|
1343
1391
|
|
|
1344
|
-
const [
|
|
1392
|
+
const [providerRunner] = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
1345
1393
|
expect(result.absentIds).toEqual([]);
|
|
1346
1394
|
expect(result.reservationsReleased).toBe(0);
|
|
1347
|
-
expect(
|
|
1395
|
+
expect(providerRunner?.state).toBe('running');
|
|
1348
1396
|
});
|
|
1349
1397
|
|
|
1350
1398
|
it('respects a fresh report that commits after reconcile selects a stale absent row', async () => {
|
|
1351
1399
|
const reservationId = await createReservation(1);
|
|
1352
|
-
await
|
|
1353
|
-
|
|
1400
|
+
await createRunnerInstance({
|
|
1401
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
1354
1402
|
reservationId,
|
|
1355
1403
|
reportedAt: staleReportedAt(),
|
|
1356
1404
|
});
|
|
@@ -1359,13 +1407,13 @@ describe('reconcileProvisionedRunners', () => {
|
|
|
1359
1407
|
|
|
1360
1408
|
const reportTransaction = db().transaction(async (tx) => {
|
|
1361
1409
|
await tx
|
|
1362
|
-
.update(
|
|
1410
|
+
.update(providerRunners)
|
|
1363
1411
|
.set({reportedAt: sql`now()`, updatedAt: sql`now()`})
|
|
1364
1412
|
.where(
|
|
1365
1413
|
and(
|
|
1366
|
-
eq(
|
|
1367
|
-
eq(
|
|
1368
|
-
eq(
|
|
1414
|
+
eq(providerRunners.workspaceId, workspaceId),
|
|
1415
|
+
eq(providerRunners.provisionerId, provisionerId),
|
|
1416
|
+
eq(providerRunners.providerRunnerId, 'provisioned-runner-1'),
|
|
1369
1417
|
),
|
|
1370
1418
|
);
|
|
1371
1419
|
reportTransactionUpdated.resolve();
|
|
@@ -1373,50 +1421,50 @@ describe('reconcileProvisionedRunners', () => {
|
|
|
1373
1421
|
});
|
|
1374
1422
|
|
|
1375
1423
|
await reportTransactionUpdated.promise;
|
|
1376
|
-
const reconcile =
|
|
1424
|
+
const reconcile = reconcileRunnerInstances({
|
|
1377
1425
|
workspaceId,
|
|
1378
1426
|
provisionerId,
|
|
1379
|
-
|
|
1427
|
+
observedRunnerInstanceIds: ['observed-runner'],
|
|
1380
1428
|
terminateGraceSeconds: 60,
|
|
1381
1429
|
});
|
|
1382
1430
|
try {
|
|
1383
|
-
await waitForLockWait({queryLike: '%
|
|
1431
|
+
await waitForLockWait({queryLike: '%runner_instances%'});
|
|
1384
1432
|
} finally {
|
|
1385
1433
|
releaseReportTransaction.resolve();
|
|
1386
1434
|
}
|
|
1387
1435
|
const [result] = await Promise.all([reconcile, reportTransaction]);
|
|
1388
1436
|
|
|
1389
|
-
const [
|
|
1437
|
+
const [providerRunner] = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
1390
1438
|
const [reservation] = await reservationRowsFor({workspaceId, provisionerId});
|
|
1391
1439
|
expect(result.absentIds).toEqual([]);
|
|
1392
1440
|
expect(result.reservationsReleased).toBe(0);
|
|
1393
|
-
expect(
|
|
1394
|
-
expect(
|
|
1441
|
+
expect(providerRunner?.state).toBe('running');
|
|
1442
|
+
expect(providerRunner?.reservationReleasedAt).toBeNull();
|
|
1395
1443
|
expect(reservation?.count).toBe(1);
|
|
1396
1444
|
});
|
|
1397
1445
|
|
|
1398
1446
|
it('terminates only stale rows when the observed set is non-empty', async () => {
|
|
1399
|
-
await
|
|
1400
|
-
|
|
1447
|
+
await createRunnerInstance({
|
|
1448
|
+
providerRunnerId: 'stale-runner',
|
|
1401
1449
|
reportedAt: staleReportedAt(),
|
|
1402
1450
|
});
|
|
1403
|
-
await
|
|
1404
|
-
|
|
1451
|
+
await createRunnerInstance({
|
|
1452
|
+
providerRunnerId: 'fresh-runner',
|
|
1405
1453
|
reportedAt: new Date(),
|
|
1406
1454
|
});
|
|
1407
1455
|
|
|
1408
|
-
const result = await
|
|
1456
|
+
const result = await reconcileRunnerInstances({
|
|
1409
1457
|
workspaceId,
|
|
1410
1458
|
provisionerId,
|
|
1411
|
-
|
|
1459
|
+
observedRunnerInstanceIds: ['observed-runner'],
|
|
1412
1460
|
terminateGraceSeconds: 60,
|
|
1413
1461
|
});
|
|
1414
1462
|
|
|
1415
|
-
const rows = await
|
|
1416
|
-
|
|
1463
|
+
const rows = await providerRunnerRowsFor({workspaceId, provisionerId}).orderBy(
|
|
1464
|
+
providerRunners.providerRunnerId,
|
|
1417
1465
|
);
|
|
1418
1466
|
expect(result.absentIds).toEqual(['stale-runner']);
|
|
1419
|
-
expect(rows.map((row) => [row.
|
|
1467
|
+
expect(rows.map((row) => [row.providerRunnerId, row.state])).toEqual([
|
|
1420
1468
|
['fresh-runner', 'running'],
|
|
1421
1469
|
['stale-runner', 'terminated'],
|
|
1422
1470
|
]);
|
|
@@ -1428,64 +1476,62 @@ describe('reconcileProvisionedRunners', () => {
|
|
|
1428
1476
|
const expiredReservationId = await createReservation(1, {
|
|
1429
1477
|
expiresAt: new Date(Date.now() - 60_000),
|
|
1430
1478
|
});
|
|
1431
|
-
await
|
|
1432
|
-
|
|
1479
|
+
await createRunnerInstance({
|
|
1480
|
+
providerRunnerId: 'shared-1',
|
|
1433
1481
|
reservationId: sharedReservationId,
|
|
1434
1482
|
reportedAt: staleReportedAt(),
|
|
1435
1483
|
});
|
|
1436
|
-
await
|
|
1437
|
-
|
|
1484
|
+
await createRunnerInstance({
|
|
1485
|
+
providerRunnerId: 'shared-2',
|
|
1438
1486
|
reservationId: sharedReservationId,
|
|
1439
1487
|
reportedAt: staleReportedAt(),
|
|
1440
1488
|
});
|
|
1441
|
-
await
|
|
1442
|
-
|
|
1489
|
+
await createRunnerInstance({
|
|
1490
|
+
providerRunnerId: 'one-unit',
|
|
1443
1491
|
reservationId: oneUnitReservationId,
|
|
1444
1492
|
reportedAt: staleReportedAt(),
|
|
1445
1493
|
});
|
|
1446
|
-
await
|
|
1447
|
-
|
|
1494
|
+
await createRunnerInstance({
|
|
1495
|
+
providerRunnerId: 'expired',
|
|
1448
1496
|
reservationId: expiredReservationId,
|
|
1449
1497
|
reportedAt: staleReportedAt(),
|
|
1450
1498
|
});
|
|
1451
1499
|
|
|
1452
|
-
const result = await
|
|
1500
|
+
const result = await reconcileRunnerInstances({
|
|
1453
1501
|
workspaceId,
|
|
1454
1502
|
provisionerId,
|
|
1455
|
-
|
|
1503
|
+
observedRunnerInstanceIds: ['observed-runner'],
|
|
1456
1504
|
terminateGraceSeconds: 60,
|
|
1457
1505
|
});
|
|
1458
1506
|
|
|
1459
1507
|
const reservationRows = await reservationRowsFor({workspaceId, provisionerId});
|
|
1460
|
-
const
|
|
1508
|
+
const providerRunnerRows = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
1461
1509
|
expect(result.reservationsReleased).toBe(3);
|
|
1462
1510
|
expect(reservationRows).toHaveLength(2);
|
|
1463
1511
|
expect(reservationRows.find((row) => row.id === sharedReservationId)?.count).toBe(1);
|
|
1464
1512
|
expect(reservationRows.find((row) => row.id === expiredReservationId)?.count).toBe(1);
|
|
1465
1513
|
expect(reservationRows.find((row) => row.id === oneUnitReservationId)).toBeUndefined();
|
|
1466
|
-
expect(
|
|
1467
|
-
true,
|
|
1468
|
-
);
|
|
1514
|
+
expect(providerRunnerRows.every((row) => row.reservationReleasedAt instanceof Date)).toBe(true);
|
|
1469
1515
|
});
|
|
1470
1516
|
|
|
1471
1517
|
it('is idempotent across repeated reconciles', async () => {
|
|
1472
1518
|
const reservationId = await createReservation(2);
|
|
1473
|
-
await
|
|
1474
|
-
|
|
1519
|
+
await createRunnerInstance({
|
|
1520
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
1475
1521
|
reservationId,
|
|
1476
1522
|
reportedAt: staleReportedAt(),
|
|
1477
1523
|
});
|
|
1478
1524
|
|
|
1479
|
-
const first = await
|
|
1525
|
+
const first = await reconcileRunnerInstances({
|
|
1480
1526
|
workspaceId,
|
|
1481
1527
|
provisionerId,
|
|
1482
|
-
|
|
1528
|
+
observedRunnerInstanceIds: ['observed-runner'],
|
|
1483
1529
|
terminateGraceSeconds: 60,
|
|
1484
1530
|
});
|
|
1485
|
-
const second = await
|
|
1531
|
+
const second = await reconcileRunnerInstances({
|
|
1486
1532
|
workspaceId,
|
|
1487
1533
|
provisionerId,
|
|
1488
|
-
|
|
1534
|
+
observedRunnerInstanceIds: ['observed-runner'],
|
|
1489
1535
|
terminateGraceSeconds: 60,
|
|
1490
1536
|
});
|
|
1491
1537
|
|
|
@@ -1498,53 +1544,53 @@ describe('reconcileProvisionedRunners', () => {
|
|
|
1498
1544
|
it('does not touch provisioned runners from another workspace or provisioner', async () => {
|
|
1499
1545
|
const otherWorkspaceId = crypto.randomUUID();
|
|
1500
1546
|
const otherProvisionerId = crypto.randomUUID();
|
|
1501
|
-
await
|
|
1502
|
-
|
|
1547
|
+
await createRunnerInstance({
|
|
1548
|
+
providerRunnerId: 'owned-runner',
|
|
1503
1549
|
reportedAt: staleReportedAt(),
|
|
1504
1550
|
});
|
|
1505
|
-
await
|
|
1551
|
+
await providerRunnerFactory.create({
|
|
1506
1552
|
workspaceId: otherWorkspaceId,
|
|
1507
1553
|
provisionerId,
|
|
1508
|
-
|
|
1554
|
+
providerRunnerId: 'other-workspace-runner',
|
|
1509
1555
|
reportedAt: staleReportedAt(),
|
|
1510
1556
|
state: 'running',
|
|
1511
1557
|
});
|
|
1512
|
-
await
|
|
1558
|
+
await providerRunnerFactory.create({
|
|
1513
1559
|
workspaceId,
|
|
1514
1560
|
provisionerId: otherProvisionerId,
|
|
1515
|
-
|
|
1561
|
+
providerRunnerId: 'other-provisioner-runner',
|
|
1516
1562
|
reportedAt: staleReportedAt(),
|
|
1517
1563
|
state: 'running',
|
|
1518
1564
|
});
|
|
1519
1565
|
|
|
1520
|
-
await
|
|
1566
|
+
await reconcileRunnerInstances({
|
|
1521
1567
|
workspaceId,
|
|
1522
1568
|
provisionerId,
|
|
1523
|
-
|
|
1569
|
+
observedRunnerInstanceIds: ['observed-runner'],
|
|
1524
1570
|
terminateGraceSeconds: 60,
|
|
1525
1571
|
});
|
|
1526
1572
|
|
|
1527
1573
|
const rows = await db()
|
|
1528
1574
|
.select()
|
|
1529
|
-
.from(
|
|
1575
|
+
.from(providerRunners)
|
|
1530
1576
|
.where(
|
|
1531
1577
|
or(
|
|
1532
1578
|
and(
|
|
1533
|
-
eq(
|
|
1534
|
-
eq(
|
|
1579
|
+
eq(providerRunners.workspaceId, workspaceId),
|
|
1580
|
+
eq(providerRunners.provisionerId, provisionerId),
|
|
1535
1581
|
),
|
|
1536
1582
|
and(
|
|
1537
|
-
eq(
|
|
1538
|
-
eq(
|
|
1583
|
+
eq(providerRunners.workspaceId, otherWorkspaceId),
|
|
1584
|
+
eq(providerRunners.provisionerId, provisionerId),
|
|
1539
1585
|
),
|
|
1540
1586
|
and(
|
|
1541
|
-
eq(
|
|
1542
|
-
eq(
|
|
1587
|
+
eq(providerRunners.workspaceId, workspaceId),
|
|
1588
|
+
eq(providerRunners.provisionerId, otherProvisionerId),
|
|
1543
1589
|
),
|
|
1544
1590
|
),
|
|
1545
1591
|
)
|
|
1546
|
-
.orderBy(
|
|
1547
|
-
expect(rows.map((row) => [row.
|
|
1592
|
+
.orderBy(providerRunners.providerRunnerId);
|
|
1593
|
+
expect(rows.map((row) => [row.providerRunnerId, row.state])).toEqual([
|
|
1548
1594
|
['other-provisioner-runner', 'running'],
|
|
1549
1595
|
['other-workspace-runner', 'running'],
|
|
1550
1596
|
['owned-runner', 'terminated'],
|
|
@@ -1553,30 +1599,30 @@ describe('reconcileProvisionedRunners', () => {
|
|
|
1553
1599
|
|
|
1554
1600
|
it('terminates session-bound absent runners without releasing their reservation', async () => {
|
|
1555
1601
|
const reservationId = await createReservation(1);
|
|
1556
|
-
await
|
|
1557
|
-
|
|
1602
|
+
await createRunnerInstance({
|
|
1603
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
1558
1604
|
reservationId,
|
|
1559
1605
|
runnerSessionId: crypto.randomUUID(),
|
|
1560
1606
|
reportedAt: staleReportedAt(),
|
|
1561
1607
|
});
|
|
1562
1608
|
|
|
1563
|
-
const result = await
|
|
1609
|
+
const result = await reconcileRunnerInstances({
|
|
1564
1610
|
workspaceId,
|
|
1565
1611
|
provisionerId,
|
|
1566
|
-
|
|
1612
|
+
observedRunnerInstanceIds: ['observed-runner'],
|
|
1567
1613
|
terminateGraceSeconds: 60,
|
|
1568
1614
|
});
|
|
1569
1615
|
|
|
1570
|
-
const [
|
|
1616
|
+
const [providerRunner] = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
1571
1617
|
const [reservation] = await reservationRowsFor({workspaceId, provisionerId});
|
|
1572
1618
|
expect(result.reservationsReleased).toBe(0);
|
|
1573
|
-
expect(
|
|
1574
|
-
expect(
|
|
1619
|
+
expect(providerRunner?.state).toBe('terminated');
|
|
1620
|
+
expect(providerRunner?.reservationReleasedAt).toBeNull();
|
|
1575
1621
|
expect(reservation?.count).toBe(1);
|
|
1576
1622
|
});
|
|
1577
1623
|
|
|
1578
1624
|
it('returns a deterministic newest running job execution bound to an observed provisioned runner', async () => {
|
|
1579
|
-
await
|
|
1625
|
+
await createRunnerInstance({providerRunnerId: 'provisioned-runner-1'});
|
|
1580
1626
|
const lowerJobId = '00000000-0000-4000-8000-000000000001';
|
|
1581
1627
|
const higherJobId = '00000000-0000-4000-8000-000000000002';
|
|
1582
1628
|
const lowerJobExecutionId = '10000000-0000-4000-8000-000000000001';
|
|
@@ -1584,58 +1630,58 @@ describe('reconcileProvisionedRunners', () => {
|
|
|
1584
1630
|
await insertRunningJob({
|
|
1585
1631
|
jobId: lowerJobId,
|
|
1586
1632
|
jobExecutionId: lowerJobExecutionId,
|
|
1587
|
-
|
|
1633
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
1588
1634
|
startedAt: new Date('2025-01-01T00:00:00.000Z'),
|
|
1589
1635
|
});
|
|
1590
1636
|
await insertRunningJob({
|
|
1591
1637
|
jobId: higherJobId,
|
|
1592
1638
|
jobExecutionId: higherJobExecutionId,
|
|
1593
|
-
|
|
1639
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
1594
1640
|
startedAt: new Date('2025-01-01T00:00:00.000Z'),
|
|
1595
1641
|
});
|
|
1596
1642
|
|
|
1597
|
-
const result = await
|
|
1643
|
+
const result = await reconcileRunnerInstances({
|
|
1598
1644
|
workspaceId,
|
|
1599
1645
|
provisionerId,
|
|
1600
|
-
|
|
1646
|
+
observedRunnerInstanceIds: ['provisioned-runner-1'],
|
|
1601
1647
|
terminateGraceSeconds: 60,
|
|
1602
1648
|
});
|
|
1603
1649
|
|
|
1604
1650
|
expect(
|
|
1605
|
-
result.
|
|
1651
|
+
result.boundJobExecutionsByRunnerInstanceId.get('provisioned-runner-1')?.jobExecutionId,
|
|
1606
1652
|
).toBe(higherJobExecutionId);
|
|
1607
1653
|
});
|
|
1608
1654
|
|
|
1609
1655
|
it('does not let a later running report revive a reconcile-terminated runner', async () => {
|
|
1610
1656
|
const reportedAt = staleReportedAt();
|
|
1611
|
-
await
|
|
1612
|
-
await
|
|
1657
|
+
await createRunnerInstance({providerRunnerId: 'provisioned-runner-1', reportedAt});
|
|
1658
|
+
await reconcileRunnerInstances({
|
|
1613
1659
|
workspaceId,
|
|
1614
1660
|
provisionerId,
|
|
1615
|
-
|
|
1661
|
+
observedRunnerInstanceIds: ['observed-runner'],
|
|
1616
1662
|
terminateGraceSeconds: 60,
|
|
1617
1663
|
});
|
|
1618
1664
|
|
|
1619
|
-
await
|
|
1665
|
+
await reportRunnerInstances({
|
|
1620
1666
|
workspaceId,
|
|
1621
1667
|
provisionerId,
|
|
1622
1668
|
events: [
|
|
1623
1669
|
event({
|
|
1624
|
-
|
|
1670
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
1625
1671
|
state: 'running',
|
|
1626
1672
|
reportedAt: new Date(reportedAt.getTime() + 120_000),
|
|
1627
1673
|
}),
|
|
1628
1674
|
],
|
|
1629
1675
|
});
|
|
1630
1676
|
|
|
1631
|
-
const [
|
|
1632
|
-
expect(
|
|
1677
|
+
const [providerRunner] = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
1678
|
+
expect(providerRunner?.state).toBe('terminated');
|
|
1633
1679
|
});
|
|
1634
1680
|
|
|
1635
1681
|
it('does not double-release reservations when terminal report and reconcile queue on the workspace lock', async () => {
|
|
1636
1682
|
const reservationId = await createReservation(2);
|
|
1637
|
-
await
|
|
1638
|
-
|
|
1683
|
+
await createRunnerInstance({
|
|
1684
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
1639
1685
|
reservationId,
|
|
1640
1686
|
reportedAt: staleReportedAt(),
|
|
1641
1687
|
});
|
|
@@ -1648,12 +1694,12 @@ describe('reconcileProvisionedRunners', () => {
|
|
|
1648
1694
|
});
|
|
1649
1695
|
|
|
1650
1696
|
await lockHolderReady.promise;
|
|
1651
|
-
const report =
|
|
1697
|
+
const report = reportRunnerInstances({
|
|
1652
1698
|
workspaceId,
|
|
1653
1699
|
provisionerId,
|
|
1654
1700
|
events: [
|
|
1655
1701
|
event({
|
|
1656
|
-
|
|
1702
|
+
providerRunnerId: 'provisioned-runner-1',
|
|
1657
1703
|
reservationId,
|
|
1658
1704
|
state: 'failed',
|
|
1659
1705
|
reportedAt: new Date(),
|
|
@@ -1661,10 +1707,10 @@ describe('reconcileProvisionedRunners', () => {
|
|
|
1661
1707
|
],
|
|
1662
1708
|
});
|
|
1663
1709
|
await waitForLockWait({queryLike: '%pg_advisory_xact_lock%'});
|
|
1664
|
-
const reconcile =
|
|
1710
|
+
const reconcile = reconcileRunnerInstances({
|
|
1665
1711
|
workspaceId,
|
|
1666
1712
|
provisionerId,
|
|
1667
|
-
|
|
1713
|
+
observedRunnerInstanceIds: ['observed-runner'],
|
|
1668
1714
|
terminateGraceSeconds: 60,
|
|
1669
1715
|
});
|
|
1670
1716
|
try {
|
|
@@ -1674,13 +1720,13 @@ describe('reconcileProvisionedRunners', () => {
|
|
|
1674
1720
|
}
|
|
1675
1721
|
const [reportResult, reconcileResult] = await Promise.all([report, reconcile, lockHolder]);
|
|
1676
1722
|
|
|
1677
|
-
const [
|
|
1723
|
+
const [providerRunner] = await providerRunnerRowsFor({workspaceId, provisionerId});
|
|
1678
1724
|
const [reservation] = await reservationRowsFor({workspaceId, provisionerId});
|
|
1679
1725
|
expect(reportResult.reservationsReleased + reconcileResult.reservationsReleased).toBe(1);
|
|
1680
|
-
expect(
|
|
1726
|
+
expect(providerRunner?.state).toSatisfy(
|
|
1681
1727
|
(state: string | undefined) => state === 'failed' || state === 'terminated',
|
|
1682
1728
|
);
|
|
1683
|
-
expect(
|
|
1729
|
+
expect(providerRunner?.reservationReleasedAt).toBeInstanceOf(Date);
|
|
1684
1730
|
expect(reservation?.count).toBe(1);
|
|
1685
1731
|
});
|
|
1686
1732
|
|
|
@@ -1707,16 +1753,16 @@ describe('reconcileProvisionedRunners', () => {
|
|
|
1707
1753
|
return reservation.id;
|
|
1708
1754
|
}
|
|
1709
1755
|
|
|
1710
|
-
async function
|
|
1711
|
-
|
|
1756
|
+
async function createRunnerInstance(params: {
|
|
1757
|
+
providerRunnerId: string;
|
|
1712
1758
|
reservationId?: string | null;
|
|
1713
1759
|
runnerSessionId?: string | null;
|
|
1714
1760
|
reportedAt?: Date;
|
|
1715
1761
|
}) {
|
|
1716
|
-
return await
|
|
1762
|
+
return await providerRunnerFactory.create({
|
|
1717
1763
|
workspaceId,
|
|
1718
1764
|
provisionerId,
|
|
1719
|
-
|
|
1765
|
+
providerRunnerId: params.providerRunnerId,
|
|
1720
1766
|
reservationId: params.reservationId ?? null,
|
|
1721
1767
|
runnerSessionId: params.runnerSessionId ?? null,
|
|
1722
1768
|
reportedAt: params.reportedAt ?? new Date(),
|
|
@@ -1727,7 +1773,7 @@ describe('reconcileProvisionedRunners', () => {
|
|
|
1727
1773
|
async function insertRunningJob(params: {
|
|
1728
1774
|
jobId: string;
|
|
1729
1775
|
jobExecutionId?: string;
|
|
1730
|
-
|
|
1776
|
+
providerRunnerId: string;
|
|
1731
1777
|
startedAt: Date;
|
|
1732
1778
|
}) {
|
|
1733
1779
|
const runnerSession = await runnerSessionFactory.create({workspaceId});
|
|
@@ -1743,7 +1789,7 @@ describe('reconcileProvisionedRunners', () => {
|
|
|
1743
1789
|
projectId: crypto.randomUUID(),
|
|
1744
1790
|
runnerSessionId: runnerSession.id,
|
|
1745
1791
|
provisionerId,
|
|
1746
|
-
|
|
1792
|
+
providerRunnerId: params.providerRunnerId,
|
|
1747
1793
|
requiredLabels: ['linux'],
|
|
1748
1794
|
runnerLabels: ['linux'],
|
|
1749
1795
|
startedAt: params.startedAt,
|
|
@@ -1756,13 +1802,13 @@ describe('reconcileProvisionedRunners', () => {
|
|
|
1756
1802
|
}
|
|
1757
1803
|
|
|
1758
1804
|
function event(params: {
|
|
1759
|
-
|
|
1805
|
+
providerRunnerId: string;
|
|
1760
1806
|
reservationId?: string | null;
|
|
1761
1807
|
state?: 'starting' | 'running' | 'stopping' | 'stopped' | 'failed' | 'terminated';
|
|
1762
1808
|
reportedAt?: Date;
|
|
1763
1809
|
}) {
|
|
1764
1810
|
return {
|
|
1765
|
-
|
|
1811
|
+
providerRunnerId: params.providerRunnerId,
|
|
1766
1812
|
reservationId: params.reservationId ?? null,
|
|
1767
1813
|
templateKey: 'linux',
|
|
1768
1814
|
labels: ['linux'],
|
|
@@ -1775,6 +1821,49 @@ describe('reconcileProvisionedRunners', () => {
|
|
|
1775
1821
|
}
|
|
1776
1822
|
});
|
|
1777
1823
|
|
|
1824
|
+
describe('runner instance provider attachment', () => {
|
|
1825
|
+
it('belongs to its provisioner until it receives one provider runner identity', async () => {
|
|
1826
|
+
const provisionerId = crypto.randomUUID();
|
|
1827
|
+
const [instance] = await db()
|
|
1828
|
+
.insert(providerRunners)
|
|
1829
|
+
.values({
|
|
1830
|
+
provisionerId,
|
|
1831
|
+
providerKind: 'docker',
|
|
1832
|
+
templateKey: 'linux',
|
|
1833
|
+
labels: [],
|
|
1834
|
+
state: 'starting',
|
|
1835
|
+
reportedAt: new Date(),
|
|
1836
|
+
})
|
|
1837
|
+
.returning({id: providerRunners.id});
|
|
1838
|
+
if (!instance) throw new Error('Runner instance insert returned no row');
|
|
1839
|
+
|
|
1840
|
+
const attached = await attachRunnerInstanceProviderId({
|
|
1841
|
+
runnerInstanceId: instance.id,
|
|
1842
|
+
provisionerId,
|
|
1843
|
+
providerRunnerId: 'container-1',
|
|
1844
|
+
});
|
|
1845
|
+
const rebound = await attachRunnerInstanceProviderId({
|
|
1846
|
+
runnerInstanceId: instance.id,
|
|
1847
|
+
provisionerId,
|
|
1848
|
+
providerRunnerId: 'container-2',
|
|
1849
|
+
});
|
|
1850
|
+
const [row] = await db()
|
|
1851
|
+
.select()
|
|
1852
|
+
.from(providerRunners)
|
|
1853
|
+
.where(eq(providerRunners.id, instance.id));
|
|
1854
|
+
|
|
1855
|
+
expect(attached).toBe(true);
|
|
1856
|
+
expect(rebound).toBe(false);
|
|
1857
|
+
expect(row).toMatchObject({
|
|
1858
|
+
workspaceId: null,
|
|
1859
|
+
provisionerId,
|
|
1860
|
+
providerRunnerId: 'container-1',
|
|
1861
|
+
labels: [],
|
|
1862
|
+
state: 'starting',
|
|
1863
|
+
});
|
|
1864
|
+
});
|
|
1865
|
+
});
|
|
1866
|
+
|
|
1778
1867
|
function deferred<T>() {
|
|
1779
1868
|
let resolve!: (value: T | PromiseLike<T>) => void;
|
|
1780
1869
|
let reject!: (reason?: unknown) => void;
|