npm - @shipfox/api-runners - Versions diffs - 13.0.0 → 13.1.0 - Mend

@shipfox/api-runners 13.0.0 → 13.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (10) hide show

package/.turbo/turbo-build.log +4 -4
package/CHANGELOG.md +8 -0
package/dist/db/job-executions.d.ts.map +1 -1
package/dist/db/job-executions.js +44 -2
package/dist/db/job-executions.js.map +1 -1
package/dist/tsconfig.test.tsbuildinfo +1 -1
package/package.json +2 -2
package/src/db/job-executions.test.ts +100 -8
package/src/db/job-executions.ts +75 -9
package/tsconfig.build.tsbuildinfo +1 -1

package/.turbo/turbo-build.log CHANGED Viewed

@@ -1,6 +1,6 @@
 $ shipfox-swc && shipfox-temporal-bundle dist/temporal/workflows/index.js
-Successfully compiled: 105 files with swc (500.52ms)
-2026-08-14T18:57:08.640Z [INFO] asset workflow-bundle-0c3da2487e29e352d64e.js 3.17 MiB [emitted] [immutable] (name: main)
+Successfully compiled: 105 files with swc (548ms)
+2026-08-23T10:00:31.690Z [INFO] asset workflow-bundle-0c3da2487e29e352d64e.js 3.17 MiB [emitted] [immutable] (name: main)
 orphan modules 33.5 KiB [orphan] 21 modules
 runtime modules 1.13 KiB 5 modules
 modules by path ../../../node_modules/.pnpm/ 970 KiB 196 modules
@@ -14,6 +14,6 @@ optional modules 30 bytes [optional]
   __temporal_custom_payload_converter (ignored) 15 bytes [optional] [built] [code generated]
   __temporal_custom_failure_converter (ignored) 15 bytes [optional] [built] [code generated]
 ../../shared/node/temporal/dist/workflow-error-interceptor.js 1.21 KiB [built] [code generated]
-webpack 5.107.2 compiled successfully in 6538 ms
-2026-08-14T18:57:08.663Z [INFO] Workflow bundle created { size: '3.17MB' }
+webpack 5.107.2 compiled successfully in 9390 ms
+2026-08-23T10:00:31.741Z [INFO] Workflow bundle created { size: '3.17MB' }
 /home/runner/work/shipfox/shipfox/libs/api/runners/dist/temporal/workflows/index.js: 3326598 bytes

package/CHANGELOG.md CHANGED Viewed

@@ -1,5 +1,13 @@
 # @shipfox/api-runners
+## 13.1.0
+### Patch Changes
+- 7de2e30: Release runner reservation units when a provisioned runner claims its first job.
+- Updated dependencies [0d3c2e3]
+  - @shipfox/api-workflows-dto@13.1.0
 ## 13.0.0
 ### Major Changes

package/dist/db/job-executions.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"job-executions.d.ts","sourceRoot":"","sources":["../../src/db/job-executions.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,KAAK,yBAAyB,EAC/B,MAAM,0BAA0B,CAAC;AAiClC,OAAO,KAAK,EAAC,EAAE,EAAC,MAAM,SAAS,CAAC;~~AAkDhC~~,MAAM,WAAW,yBAAyB;IACxC,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,QAAQ,EAAE,IAAI,CAAC;CAChB;AAED,wBAAsB,qBAAqB,CAAC,MAAM,EAAE;IAClD,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB,GAAG,OAAO,CAAC,KAAK,CAAC;IAAC,WAAW,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAC,CAAC,CAAC,CAwBzE;AAKD,wBAAsB,mBAAmB,CAAC,MAAM,EAAE,yBAAyB,GAAG,OAAO,CAAC,IAAI,CAAC,CA6C1F;AAED,MAAM,WAAW,mBAAmB;IAClC,aAAa,EAAE,MAAM,CAAC;IACtB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,yBAAyB;IACxC,aAAa,EAAE,MAAM,CAAC;IACtB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,SAAS,EAAE,IAAI,CAAC;IAChB,eAAe,EAAE,IAAI,CAAC;CACvB;AAED,MAAM,WAAW,+BAA+B;IAC9C,aAAa,EAAE,MAAM,CAAC;IACtB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,gBAAgB,EAAE,MAAM,CAAC;IACzB,SAAS,EAAE,IAAI,CAAC;IAChB,eAAe,EAAE,IAAI,CAAC;IACtB,uBAAuB,EAAE,IAAI,GAAG,IAAI,CAAC;CACtC;AAED,wBAAsB,wBAAwB,CAAC,MAAM,EAAE;IACrD,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,oCAAoC,EAAE,MAAM,CAAC;CAC9C,GAAG,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC,~~CAiMtC~~;AAmBD;;;;;;;;;;;;GAYG;AACH,wBAAsB,wBAAwB,CAAC,MAAM,EAAE;IACrD,gBAAgB,EAAE,MAAM,CAAC;IACzB,4BAA4B,EAAE,MAAM,CAAC;IACrC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,GAAG,OAAO,CACT,KAAK,CAAC;IACJ,aAAa,EAAE,MAAM,CAAC;IACtB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC,CACH,CAiGA;AAED,wBAAsB,yBAAyB,IAAI,OAAO,CAAC;IACzD,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;CAC9B,CAAC,CAOD;AAED,wBAAsB,8BAA8B,CAAC,MAAM,EAAE;IAC3D,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,GAAG,OAAO,CAAC,yBAAyB,EAAE,CAAC,CAyBvC;AAED,wBAAsB,0CAA0C,CAC9D,EAAE,EAAE,EAAE,EACN,MAAM,EAAE;IACN,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,MAAM,EAAE,CAAC;CAC7B,GACA,OAAO,CAAC,+BAA+B,EAAE,CAAC,CA2E5C;AAMD,wBAAsB,gBAAgB,CAAC,MAAM,EAAE;IAC7C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;CACzB,GAAG,OAAO,CAAC,OAAO,CAAC,CAcnB;AAED,wBAAsB,eAAe,CAAC,MAAM,EAAE;IAC5C,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,CAAC,EAAE,yBAAyB,GAAG,IAAI,CAAC;CACrD,GAAG,OAAO,CAAC;IACV,qBAAqB,EAAE,OAAO,CAAC;IAC/B,wBAAwB,EAAE,yBAAyB,GAAG,IAAI,CAAC;IAC3D,uBAAuB,EAAE,yBAAyB,GAAG,IAAI,CAAC;IAC1D,mBAAmB,EAAE;QACnB,aAAa,EAAE,MAAM,CAAC;QACtB,oBAAoB,EAAE,MAAM,CAAC;QAC7B,KAAK,EAAE,MAAM,CAAC;QACd,cAAc,EAAE,MAAM,CAAC;QACvB,SAAS,EAAE,MAAM,CAAC;QAClB,WAAW,EAAE,MAAM,CAAC;QACpB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;CACH,CAAC,CAoED;AAED;;;;GAIG;AACH,wBAAsB,6BAA6B,CAAC,MAAM,EAAE;IAC1D,cAAc,EAAE,MAAM,CAAC;CACxB,GAAG,OAAO,CAAC,IAAI,CAAC,CAuBhB"}
1	+ {"version":3,"file":"job-executions.d.ts","sourceRoot":"","sources":["../../src/db/job-executions.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,KAAK,yBAAyB,EAC/B,MAAM,0BAA0B,CAAC;AAiClC,OAAO,KAAK,EAAC,EAAE,EAAC,MAAM,SAAS,CAAC;AAuDhC,MAAM,WAAW,yBAAyB;IACxC,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,QAAQ,EAAE,IAAI,CAAC;CAChB;AAED,wBAAsB,qBAAqB,CAAC,MAAM,EAAE;IAClD,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB,GAAG,OAAO,CAAC,KAAK,CAAC;IAAC,WAAW,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAC,CAAC,CAAC,CAwBzE;AAKD,wBAAsB,mBAAmB,CAAC,MAAM,EAAE,yBAAyB,GAAG,OAAO,CAAC,IAAI,CAAC,CA6C1F;AAED,MAAM,WAAW,mBAAmB;IAClC,aAAa,EAAE,MAAM,CAAC;IACtB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,yBAAyB;IACxC,aAAa,EAAE,MAAM,CAAC;IACtB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,SAAS,EAAE,IAAI,CAAC;IAChB,eAAe,EAAE,IAAI,CAAC;CACvB;AAED,MAAM,WAAW,+BAA+B;IAC9C,aAAa,EAAE,MAAM,CAAC;IACtB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,gBAAgB,EAAE,MAAM,CAAC;IACzB,SAAS,EAAE,IAAI,CAAC;IAChB,eAAe,EAAE,IAAI,CAAC;IACtB,uBAAuB,EAAE,IAAI,GAAG,IAAI,CAAC;CACtC;AAED,wBAAsB,wBAAwB,CAAC,MAAM,EAAE;IACrD,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,oCAAoC,EAAE,MAAM,CAAC;CAC9C,GAAG,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC,CA8PtC;AAmBD;;;;;;;;;;;;GAYG;AACH,wBAAsB,wBAAwB,CAAC,MAAM,EAAE;IACrD,gBAAgB,EAAE,MAAM,CAAC;IACzB,4BAA4B,EAAE,MAAM,CAAC;IACrC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,GAAG,OAAO,CACT,KAAK,CAAC;IACJ,aAAa,EAAE,MAAM,CAAC;IACtB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC,CACH,CAiGA;AAED,wBAAsB,yBAAyB,IAAI,OAAO,CAAC;IACzD,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;CAC9B,CAAC,CAOD;AAED,wBAAsB,8BAA8B,CAAC,MAAM,EAAE;IAC3D,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,GAAG,OAAO,CAAC,yBAAyB,EAAE,CAAC,CAyBvC;AAED,wBAAsB,0CAA0C,CAC9D,EAAE,EAAE,EAAE,EACN,MAAM,EAAE;IACN,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,MAAM,EAAE,CAAC;CAC7B,GACA,OAAO,CAAC,+BAA+B,EAAE,CAAC,CA2E5C;AAMD,wBAAsB,gBAAgB,CAAC,MAAM,EAAE;IAC7C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;CACzB,GAAG,OAAO,CAAC,OAAO,CAAC,CAcnB;AAED,wBAAsB,eAAe,CAAC,MAAM,EAAE;IAC5C,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,CAAC,EAAE,yBAAyB,GAAG,IAAI,CAAC;CACrD,GAAG,OAAO,CAAC;IACV,qBAAqB,EAAE,OAAO,CAAC;IAC/B,wBAAwB,EAAE,yBAAyB,GAAG,IAAI,CAAC;IAC3D,uBAAuB,EAAE,yBAAyB,GAAG,IAAI,CAAC;IAC1D,mBAAmB,EAAE;QACnB,aAAa,EAAE,MAAM,CAAC;QACtB,oBAAoB,EAAE,MAAM,CAAC;QAC7B,KAAK,EAAE,MAAM,CAAC;QACd,cAAc,EAAE,MAAM,CAAC;QACvB,SAAS,EAAE,MAAM,CAAC;QAClB,WAAW,EAAE,MAAM,CAAC;QACpB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;CACH,CAAC,CAoED;AAED;;;;GAIG;AACH,wBAAsB,6BAA6B,CAAC,MAAM,EAAE;IAC1D,cAAc,EAAE,MAAM,CAAC;CACxB,GAAG,OAAO,CAAC,IAAI,CAAC,CAuBhB"}

package/dist/db/job-executions.js CHANGED Viewed

@@ -6,9 +6,11 @@ import { and, arrayContained, asc, count, desc, eq, gt, inArray, isNotNull, isNu
 import { EmptyRequiredLabelsError, RunnerSessionExhaustedError, RunningJobExecutionNotFoundError } from '#core/errors.js';
 import { jobExecutionEnqueuedCount, jobExecutionLeaseExpiredCount, recordJobExecutionQueueTime, recordProviderRunnerActivationToFirstClaim } from '#metrics/instance.js';
 import { db } from './db.js';
-import { releaseTerminalRunnerInstanceReservationsByIds } from './reservations.js';
+import { lockRunnerReservationAdvisoryKeysTx } from './reservation-locks.js';
+import { releaseReservationUnits, releaseTerminalRunnerInstanceReservationsByIds } from './reservations.js';
 import { runnersOutbox } from './schema/outbox.js';
 import { pendingJobExecutions } from './schema/pending-job-executions.js';
+import { reservations } from './schema/reservations.js';
 import { providerRunners } from './schema/runner-instances.js';
 import { runnerSessions } from './schema/runner-sessions.js';
 import { runningJobExecutions } from './schema/running-job-executions.js';
@@ -136,6 +138,20 @@ export async function claimPendingJobExecution(params) {
             provisionerId = session.provisionerId;
             providerRunnerId = session.providerRunnerId;
         }
+        const runnerInstanceCondition = runnerInstanceId ? eq(providerRunners.id, runnerInstanceId) : provisionerId && providerRunnerId ? and(eq(providerRunners.provisionerId, provisionerId), eq(providerRunners.providerRunnerId, providerRunnerId)) : null;
+        if (runnerInstanceCondition && provisionerId) {
+            const [runner] = await tx.select({
+                reservationId: providerRunners.reservationId,
+                intendedReservationId: providerRunners.intendedReservationId
+            }).from(providerRunners).where(runnerInstanceCondition).limit(1);
+            await lockRunnerReservationAdvisoryKeysTx(tx, {
+                provisionerId,
+                reservationIds: [
+                    runner?.reservationId,
+                    runner?.intendedReservationId
+                ].filter((reservationId)=>reservationId !== null && reservationId !== undefined)
+            });
+        }
         // `id` is a uuidv7 (time-ordered), so it is a deterministic FIFO tiebreaker
         // for rows sharing a created_at within a batch. Lock only the FIFO candidate before
         // attempting its execution advisory lock; putting pg_try_advisory_xact_lock in this
@@ -180,7 +196,6 @@ export async function claimPendingJobExecution(params) {
             provider: null,
             launchKind: params.maxClaims === null ? 'manual' : 'unknown'
         };
-        const runnerInstanceCondition = runnerInstanceId ? eq(providerRunners.id, runnerInstanceId) : provisionerId && providerRunnerId ? and(eq(providerRunners.provisionerId, provisionerId), eq(providerRunners.providerRunnerId, providerRunnerId)) : null;
         if (runnerInstanceCondition) {
             const [claimedRunner] = await tx.update(providerRunners).set({
                 firstClaimedAt: sql`coalesce(${providerRunners.firstClaimedAt}, ${claimed.claimedAt})`,
@@ -191,6 +206,8 @@ export async function claimPendingJobExecution(params) {
                 provider: providerRunners.providerKind,
                 launchKind: providerRunners.launchKind,
                 runnerInstanceId: providerRunners.id,
+                reservationId: providerRunners.reservationId,
+                intendedReservationId: providerRunners.intendedReservationId,
                 sessionCreatedAtEpochMs: sql`(
             select extract(epoch from ${runnerSessions.createdAt})::double precision * 1000
             from ${runnerSessions}
@@ -201,6 +218,31 @@ export async function claimPendingJobExecution(params) {
                 queueTimeObservation.provider = claimedRunner.provider;
                 queueTimeObservation.launchKind = claimedRunner.launchKind;
             }
+            if (claimedRunner?.isFirstClaim) {
+                const [releasedRunner] = await tx.update(providerRunners).set({
+                    intendedReservationId: null,
+                    reservationReleasedAt: sql`now()`,
+                    updatedAt: sql`now()`
+                }).where(and(eq(providerRunners.id, claimedRunner.runnerInstanceId), or(isNotNull(providerRunners.reservationId), isNotNull(providerRunners.intendedReservationId)), isNull(providerRunners.reservationReleasedAt))).returning({
+                    provisionerId: providerRunners.provisionerId
+                });
+                const reservationId = claimedRunner.intendedReservationId ?? claimedRunner.reservationId;
+                if (releasedRunner?.provisionerId && reservationId) {
+                    const [reservation] = await tx.select({
+                        workspaceId: reservations.workspaceId
+                    }).from(reservations).where(and(eq(reservations.id, reservationId), eq(reservations.provisionerId, releasedRunner.provisionerId))).limit(1);
+                    if (reservation) await releaseReservationUnits(tx, {
+                        workspaceId: reservation.workspaceId,
+                        provisionerId: releasedRunner.provisionerId,
+                        releases: [
+                            {
+                                reservationId,
+                                count: 1
+                            }
+                        ]
+                    });
+                }
+            }
             if (claimedRunner?.isFirstClaim && claimedRunner.firstClaimedAt !== null && claimedRunner.sessionCreatedAtEpochMs !== null && claimedRunner.sessionCreatedAtEpochMs !== undefined) activationToFirstClaimObservation = {
                 durationMilliseconds: claimedRunner.firstClaimedAt.getTime() - claimedRunner.sessionCreatedAtEpochMs,
                 provider: claimedRunner.provider,

package/dist/db/job-executions.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"sources":["../../src/db/job-executions.ts"],"sourcesContent":["import {\n RUNNER_JOB_CLAIMED,\n RUNNER_JOB_LEASE_EXPIRED,\n type RunnersEventMap,\n type RunnerToolCapabilitiesDto,\n} from '@shipfox/api-runners-dto';\nimport {logger} from '@shipfox/node-opentelemetry';\nimport {writeOutboxEvent, writeOutboxEvents} from '@shipfox/node-outbox';\nimport {canonicalizeLabels} from '@shipfox/runner-labels';\nimport {\n and,\n arrayContained,\n asc,\n count,\n desc,\n eq,\n gt,\n inArray,\n isNotNull,\n isNull,\n lt,\n lte,\n or,\n sql,\n} from 'drizzle-orm';\nimport {\n EmptyRequiredLabelsError,\n RunnerSessionExhaustedError,\n RunningJobExecutionNotFoundError,\n} from '#core/errors.js';\nimport {\n type JobExecutionQueueTimeObservation,\n jobExecutionEnqueuedCount,\n jobExecutionLeaseExpiredCount,\n type ProviderRunnerLifecycleObservation,\n recordJobExecutionQueueTime,\n recordProviderRunnerActivationToFirstClaim,\n} from '#metrics/instance.js';\nimport type {Tx} from './db.js';\nimport {db} from './db.js';\nimport {releaseTerminalRunnerInstanceReservationsByIds} from './reservations.js';\nimport {runnersOutbox} from './schema/outbox.js';\nimport {pendingJobExecutions} from './schema/pending-job-executions.js';\nimport {providerRunners} from './schema/runner-instances.js';\nimport {runnerSessions} from './schema/runner-sessions.js';\nimport {runningJobExecutions} from './schema/running-job-executions.js';\n\nconst runnerJobExecutionLockPrefix = 'runners_job_execution:';\n\nasync function lockJobExecution(tx: Tx, jobExecutionId: string): Promise<void> {\n await tx.execute(\n sql`select pg_advisory_xact_lock(hashtext(${`${runnerJobExecutionLockPrefix}${jobExecutionId}`}))`,\n );\n}\n\nasync function releaseReservationsForTerminalRunningRows(\n tx: Tx,\n rows: ReadonlyArray<{\n provisionerId: string \| null;\n providerRunnerId: string \| null;\n }>,\n): Promise<void> {\n // Converge after the lease fact changes. The reservation helper locks each provider runner\n // and releases only when the runner is terminal and no uncancelled lease remains.\n const providerRunnerIdsByProvisionerId = new Map<string, Set<string>>();\n for (const row of rows) {\n if (row.provisionerId === null \|\| row.providerRunnerId === null) continue;\n const providerRunnerIds =\n providerRunnerIdsByProvisionerId.get(row.provisionerId) ?? new Set<string>();\n providerRunnerIds.add(row.providerRunnerId);\n providerRunnerIdsByProvisionerId.set(row.provisionerId, providerRunnerIds);\n }\n\n for (const provisionerId of [...providerRunnerIdsByProvisionerId.keys()].sort()) {\n const providerRunnerIds = providerRunnerIdsByProvisionerId.get(provisionerId);\n if (!providerRunnerIds) continue;\n await releaseTerminalRunnerInstanceReservationsByIds(tx, {\n workspaceId: null,\n provisionerId,\n providerRunnerIds: [...providerRunnerIds].sort(),\n requireUnlinkedSession: false,\n // Lock and re-check the runner row locally so lease finalization remains retryable\n // without a workflow/runner scope lock.\n requireTerminalState: false,\n });\n }\n}\n\nexport interface EnqueueJobExecutionParams {\n workspaceId: string;\n workflowRunId: string;\n workflowRunAttemptId: string;\n jobId: string;\n jobExecutionId: string;\n projectId: string;\n requiredLabels: string[];\n queuedAt: Date;\n}\n\nexport async function getWorkspaceJobCounts(params: {\n workspaceIds: string[];\n}): Promise<Array<{workspaceId: string; queued: number; running: number}>> {\n const [pendingRows, runningRows] = await Promise.all([\n db()\n .select({workspaceId: pendingJobExecutions.workspaceId, count: count()})\n .from(pendingJobExecutions)\n .where(inArray(pendingJobExecutions.workspaceId, params.workspaceIds))\n .groupBy(pendingJobExecutions.workspaceId),\n db()\n .select({workspaceId: runningJobExecutions.workspaceId, count: count()})\n .from(runningJobExecutions)\n .where(inArray(runningJobExecutions.workspaceId, params.workspaceIds))\n .groupBy(runningJobExecutions.workspaceId),\n ]);\n\n const queuedByWorkspace = new Map(pendingRows.map((row) => [row.workspaceId, Number(row.count)]));\n const runningByWorkspace = new Map(\n runningRows.map((row) => [row.workspaceId, Number(row.count)]),\n );\n\n return params.workspaceIds.map((workspaceId) => ({\n workspaceId,\n queued: queuedByWorkspace.get(workspaceId) ?? 0,\n running: runningByWorkspace.get(workspaceId) ?? 0,\n }));\n}\n\n// The workflows outbox delivers queue facts at least once. The per-execution advisory lock and\n// pending/running checks make replay a no-op across the claim transition. A durable lease-expired\n// event also prevents a delayed replay from resurrecting an execution already reaped locally.\nexport async function enqueueJobExecution(params: EnqueueJobExecutionParams): Promise<void> {\n const requiredLabels = [...canonicalizeLabels(params.requiredLabels)];\n if (requiredLabels.length === 0) throw new EmptyRequiredLabelsError();\n\n const enqueued = await db().transaction(async (tx) => {\n await lockJobExecution(tx, params.jobExecutionId);\n\n const [running] = await tx\n .select({jobExecutionId: runningJobExecutions.jobExecutionId})\n .from(runningJobExecutions)\n .where(eq(runningJobExecutions.jobExecutionId, params.jobExecutionId))\n .limit(1);\n if (running) return false;\n\n const [leaseExpired] = await tx\n .select({id: runnersOutbox.id})\n .from(runnersOutbox)\n .where(\n and(\n eq(runnersOutbox.eventType, RUNNER_JOB_LEASE_EXPIRED),\n sql`${runnersOutbox.payload}->>'jobExecutionId' = ${params.jobExecutionId}`,\n ),\n )\n .limit(1);\n if (leaseExpired) return false;\n\n const [inserted] = await tx\n .insert(pendingJobExecutions)\n .values({\n workspaceId: params.workspaceId,\n workflowRunId: params.workflowRunId,\n workflowRunAttemptId: params.workflowRunAttemptId,\n jobId: params.jobId,\n jobExecutionId: params.jobExecutionId,\n projectId: params.projectId,\n requiredLabels,\n createdAt: params.queuedAt,\n })\n .onConflictDoNothing({target: pendingJobExecutions.jobExecutionId})\n .returning({createdAt: pendingJobExecutions.createdAt});\n\n return inserted !== undefined;\n });\n\n if (enqueued) jobExecutionEnqueuedCount.add(1);\n}\n\nexport interface ClaimedJobExecution {\n workflowRunId: string;\n workflowRunAttemptId: string;\n jobId: string;\n jobExecutionId: string;\n projectId: string;\n}\n\nexport interface ActiveRunningJobExecution {\n workflowRunId: string;\n workflowRunAttemptId: string;\n jobId: string;\n jobExecutionId: string;\n projectId: string;\n runnerSessionId: string;\n provisionerId: string \| null;\n providerRunnerId: string \| null;\n requiredLabels: string[];\n runnerLabels: string[];\n startedAt: Date;\n lastHeartbeatAt: Date;\n}\n\nexport interface RunnerInstanceBoundJobExecution {\n workflowRunId: string;\n workflowRunAttemptId: string;\n jobId: string;\n jobExecutionId: string;\n providerRunnerId: string;\n startedAt: Date;\n lastHeartbeatAt: Date;\n cancellationRequestedAt: Date \| null;\n}\n\nexport async function claimPendingJobExecution(params: {\n workspaceId: string;\n runnerSessionId: string;\n sessionLabels: string[];\n maxClaims: number \| null;\n runnerSessionLivenessThrottleSeconds: number;\n}): Promise<ClaimedJobExecution \| null> {\n await touchRunnerSessionLiveness({\n workspaceId: params.workspaceId,\n runnerSessionId: params.runnerSessionId,\n throttleSeconds: params.runnerSessionLivenessThrottleSeconds,\n });\n\n if (params.sessionLabels.length === 0) return null;\n\n let activationToFirstClaimObservation: ProviderRunnerLifecycleObservation \| null = null;\n let queueTimeObservation: JobExecutionQueueTimeObservation \| null = null;\n const result = await db().transaction(async (tx) => {\n let provisionerId: string \| null = null;\n let providerRunnerId: string \| null = null;\n let runnerInstanceId: string \| null = null;\n\n if (params.maxClaims !== null) {\n const [session] = await tx\n .select({\n maxClaims: runnerSessions.maxClaims,\n claimsUsed: runnerSessions.claimsUsed,\n revokedAt: runnerSessions.revokedAt,\n runnerInstanceId: runnerSessions.runnerInstanceId,\n provisionerId: runnerSessions.provisionerId,\n providerRunnerId: runnerSessions.providerRunnerId,\n })\n .from(runnerSessions)\n .where(eq(runnerSessions.id, params.runnerSessionId))\n .limit(1)\n .for('update');\n\n if (\n !session \|\|\n session.revokedAt \|\|\n session.maxClaims === null \|\|\n session.claimsUsed >= session.maxClaims\n ) {\n throw new RunnerSessionExhaustedError(params.runnerSessionId);\n }\n\n // Ephemeral sessions are the only capped sessions, and the DB check keeps\n // their provisioned-runner link present as a pair.\n runnerInstanceId = session.runnerInstanceId;\n provisionerId = session.provisionerId;\n providerRunnerId = session.providerRunnerId;\n }\n\n // `id` is a uuidv7 (time-ordered), so it is a deterministic FIFO tiebreaker\n // for rows sharing a created_at within a batch. Lock only the FIFO candidate before\n // attempting its execution advisory lock; putting pg_try_advisory_xact_lock in this\n // predicate would evaluate it while scanning and temporarily lock many queue entries.\n const candidate = tx\n .select()\n .from(pendingJobExecutions)\n .where(\n and(\n eq(pendingJobExecutions.workspaceId, params.workspaceId),\n arrayContained(pendingJobExecutions.requiredLabels, params.sessionLabels),\n ),\n )\n .orderBy(asc(pendingJobExecutions.createdAt), asc(pendingJobExecutions.id))\n .limit(1)\n .for('update', {skipLocked: true})\n .as('pending_candidate');\n\n const [row] = await tx\n .select()\n .from(candidate)\n .where(\n sql`\n pg_try_advisory_xact_lock(\n hashtext(${runnerJobExecutionLockPrefix} \|\| ${candidate.jobExecutionId}::text)\n )\n `,\n );\n\n if (!row) return null;\n\n await tx.delete(pendingJobExecutions).where(eq(pendingJobExecutions.id, row.id));\n\n // An enqueue retry that lands after a prior claim can leave an orphan pending\n // row whose jobExecutionId is already in `runners_running_jobs`. Insert-or-skip\n // on the jobExecutionId unique constraint: when the job execution is already running\n // the insert touches no row, so we commit the orphan's deletion and return\n // null rather than let the unique violation roll the claim back into a poison\n // loop. The runner just re-polls for a real job execution.\n const inserted = await tx\n .insert(runningJobExecutions)\n .values({\n workspaceId: row.workspaceId,\n workflowRunId: row.workflowRunId,\n workflowRunAttemptId: row.workflowRunAttemptId,\n jobId: row.jobId,\n jobExecutionId: row.jobExecutionId,\n projectId: row.projectId,\n runnerSessionId: params.runnerSessionId,\n provisionerId,\n providerRunnerId,\n requiredLabels: row.requiredLabels,\n runnerLabels: params.sessionLabels,\n })\n .onConflictDoNothing({target: runningJobExecutions.jobExecutionId})\n .returning({claimedAt: runningJobExecutions.startedAt});\n\n const claimed = inserted[0];\n if (!claimed) return null;\n\n queueTimeObservation = {\n durationMilliseconds: claimed.claimedAt.getTime() - row.createdAt.getTime(),\n provider: null,\n launchKind: params.maxClaims === null ? 'manual' : 'unknown',\n };\n\n const runnerInstanceCondition = runnerInstanceId\n ? eq(providerRunners.id, runnerInstanceId)\n : provisionerId && providerRunnerId\n ? and(\n eq(providerRunners.provisionerId, provisionerId),\n eq(providerRunners.providerRunnerId, providerRunnerId),\n )\n : null;\n if (runnerInstanceCondition) {\n const [claimedRunner] = await tx\n .update(providerRunners)\n .set({\n firstClaimedAt: sql`coalesce(${providerRunners.firstClaimedAt}, ${claimed.claimedAt})`,\n updatedAt: sql`now()`,\n })\n .where(runnerInstanceCondition)\n .returning({\n firstClaimedAt: providerRunners.firstClaimedAt,\n isFirstClaim: sql<boolean>`${providerRunners.firstClaimedAt} = ${claimed.claimedAt}`,\n provider: providerRunners.providerKind,\n launchKind: providerRunners.launchKind,\n runnerInstanceId: providerRunners.id,\n sessionCreatedAtEpochMs: sql<number \| null>`(\n select extract(epoch from ${runnerSessions.createdAt})::double precision * 1000\n from ${runnerSessions}\n where ${runnerSessions.id} = ${params.runnerSessionId}\n )`,\n });\n if (claimedRunner && queueTimeObservation) {\n queueTimeObservation.provider = claimedRunner.provider;\n queueTimeObservation.launchKind = claimedRunner.launchKind;\n }\n if (\n claimedRunner?.isFirstClaim &&\n claimedRunner.firstClaimedAt !== null &&\n claimedRunner.sessionCreatedAtEpochMs !== null &&\n claimedRunner.sessionCreatedAtEpochMs !== undefined\n )\n activationToFirstClaimObservation = {\n durationMilliseconds:\n claimedRunner.firstClaimedAt.getTime() - claimedRunner.sessionCreatedAtEpochMs,\n provider: claimedRunner.provider,\n launchKind: claimedRunner.launchKind,\n runnerInstanceId: claimedRunner.runnerInstanceId,\n };\n }\n\n if (params.maxClaims !== null) {\n await tx\n .update(runnerSessions)\n .set({claimsUsed: sql`${runnerSessions.claimsUsed} + 1`, updatedAt: sql`now()`})\n .where(eq(runnerSessions.id, params.runnerSessionId));\n }\n\n // The running-row insert is the runner claiming the job execution. Emit in the same tx; the\n // payload carries the row's own claim instant so a consumer records the true time,\n // not the outbox drain time.\n await writeOutboxEvent<RunnersEventMap>(tx, runnersOutbox, {\n type: RUNNER_JOB_CLAIMED,\n payload: {\n workflowRunId: row.workflowRunId,\n workflowRunAttemptId: row.workflowRunAttemptId,\n jobId: row.jobId,\n jobExecutionId: row.jobExecutionId,\n claimedAt: claimed.claimedAt.toISOString(),\n },\n });\n\n return {\n workflowRunId: row.workflowRunId,\n workflowRunAttemptId: row.workflowRunAttemptId,\n jobId: row.jobId,\n jobExecutionId: row.jobExecutionId,\n projectId: row.projectId,\n };\n });\n if (queueTimeObservation) recordJobExecutionQueueTime(queueTimeObservation);\n if (activationToFirstClaimObservation)\n recordProviderRunnerActivationToFirstClaim(activationToFirstClaimObservation);\n return result;\n}\n\nasync function touchRunnerSessionLiveness(params: {\n workspaceId: string;\n runnerSessionId: string;\n throttleSeconds: number;\n}): Promise<void> {\n await db()\n .update(runnerSessions)\n .set({updatedAt: sql`now()`})\n .where(\n and(\n eq(runnerSessions.id, params.runnerSessionId),\n eq(runnerSessions.workspaceId, params.workspaceId),\n sql`${runnerSessions.updatedAt} < now() - (${params.throttleSeconds} \|\| ' seconds')::interval`,\n ),\n );\n}\n\n/*\n Reaps stale leases (bounded by `limit`), emitting one\n * `runners.job.lease_expired` event per reaped job execution.\n \n The cutoff is re-checked in the DELETE, not just the locking subquery, so a\n * heartbeat landing mid-call spares the live row. Each reaped execution also sweeps its\n * pending row so an at-least-once queue replay cannot leave an orphan.\n \n Locks pending-then-running to match `claimPendingJobExecution` and\n * `reconcileTerminalJobExecution`. The stale candidate scan intentionally does not lock\n * running rows first; the running-row DELETE re-checks the stale predicate after the\n * pending-row sweep.\n /\nexport async function expireStuckJobExecutions(params: {\n thresholdSeconds: number;\n noFirstHeartbeatGraceSeconds: number;\n limit?: number;\n}): Promise<\n Array<{\n workflowRunId: string;\n workflowRunAttemptId: string;\n jobId: string;\n jobExecutionId: string;\n }>\n> {\n const reaped = await db().transaction(async (tx) => {\n const heartbeatCutoff = sql`now() - (${params.thresholdSeconds} \|\| ' seconds')::interval`;\n const firstHeartbeatCutoff = sql`now() - (${params.noFirstHeartbeatGraceSeconds} \|\| ' seconds')::interval`;\n const stalePredicate = or(\n and(\n isNull(runningJobExecutions.firstHeartbeatAt),\n lte(runningJobExecutions.lastHeartbeatAt, runningJobExecutions.startedAt),\n lt(runningJobExecutions.startedAt, firstHeartbeatCutoff),\n ),\n and(\n or(\n isNotNull(runningJobExecutions.firstHeartbeatAt),\n gt(runningJobExecutions.lastHeartbeatAt, runningJobExecutions.startedAt),\n ),\n lt(runningJobExecutions.lastHeartbeatAt, heartbeatCutoff),\n ),\n );\n\n const staleRows = await tx\n .select({\n id: runningJobExecutions.id,\n jobExecutionId: runningJobExecutions.jobExecutionId,\n })\n .from(runningJobExecutions)\n .where(\n and(\n stalePredicate,\n sql`\n pg_try_advisory_xact_lock(\n hashtext(${runnerJobExecutionLockPrefix} \|\| ${runningJobExecutions.jobExecutionId}::text)\n )\n `,\n ),\n )\n .orderBy(\n asc(\n sql`CASE WHEN ${runningJobExecutions.firstHeartbeatAt} IS NULL AND ${runningJobExecutions.lastHeartbeatAt} <= ${runningJobExecutions.startedAt} THEN ${runningJobExecutions.startedAt} ELSE ${runningJobExecutions.lastHeartbeatAt} END`,\n ),\n asc(runningJobExecutions.id),\n )\n .limit(params.limit ?? 100);\n\n if (staleRows.length === 0) return [];\n\n const staleIds = staleRows.map((row) => row.id);\n const staleJobExecutionIds = staleRows.map((row) => row.jobExecutionId);\n\n // Sweep pending rows first so this transaction cannot hold a running-row lock while waiting\n // for a pending-row lock held by reconciliation or a claim of an orphan pending row. The\n // advisory lock acquired in the candidate scan also makes enqueue retries wait until this\n // transaction has either reaped the execution or released its lock without selecting it.\n await tx\n .delete(pendingJobExecutions)\n .where(inArray(pendingJobExecutions.jobExecutionId, staleJobExecutionIds));\n\n const deleted = await tx\n .delete(runningJobExecutions)\n .where(and(inArray(runningJobExecutions.id, staleIds), stalePredicate))\n .returning({\n workflowRunId: runningJobExecutions.workflowRunId,\n workflowRunAttemptId: runningJobExecutions.workflowRunAttemptId,\n jobId: runningJobExecutions.jobId,\n jobExecutionId: runningJobExecutions.jobExecutionId,\n provisionerId: runningJobExecutions.provisionerId,\n providerRunnerId: runningJobExecutions.providerRunnerId,\n });\n\n if (deleted.length === 0) return [];\n\n await releaseReservationsForTerminalRunningRows(tx, deleted);\n\n await writeOutboxEvents<RunnersEventMap>(\n tx,\n runnersOutbox,\n deleted.map((row) => ({\n type: RUNNER_JOB_LEASE_EXPIRED,\n payload: {\n workflowRunId: row.workflowRunId,\n workflowRunAttemptId: row.workflowRunAttemptId,\n jobId: row.jobId,\n jobExecutionId: row.jobExecutionId,\n },\n })),\n );\n\n return deleted.map(({workflowRunId, workflowRunAttemptId, jobId, jobExecutionId}) => ({\n workflowRunId,\n workflowRunAttemptId,\n jobId,\n jobExecutionId,\n }));\n });\n\n if (reaped.length > 0) jobExecutionLeaseExpiredCount.add(reaped.length);\n\n return reaped;\n}\n\nexport async function getJobExecutionQueueDepth(): Promise<{\n pendingJobExecutions: number;\n runningJobExecutions: number;\n}> {\n const [pending] = await db().select({value: count()}).from(pendingJobExecutions);\n const [running] = await db().select({value: count()}).from(runningJobExecutions);\n return {\n pendingJobExecutions: pending?.value ?? 0,\n runningJobExecutions: running?.value ?? 0,\n };\n}\n\nexport async function listActiveRunningJobExecutions(params: {\n workspaceId: string;\n windowSeconds: number;\n limit?: number;\n}): Promise<ActiveRunningJobExecution[]> {\n return await db()\n .select({\n jobId: runningJobExecutions.jobId,\n jobExecutionId: runningJobExecutions.jobExecutionId,\n workflowRunId: runningJobExecutions.workflowRunId,\n workflowRunAttemptId: runningJobExecutions.workflowRunAttemptId,\n projectId: runningJobExecutions.projectId,\n runnerSessionId: runningJobExecutions.runnerSessionId,\n provisionerId: runningJobExecutions.provisionerId,\n providerRunnerId: runningJobExecutions.providerRunnerId,\n requiredLabels: runningJobExecutions.requiredLabels,\n runnerLabels: runningJobExecutions.runnerLabels,\n startedAt: runningJobExecutions.startedAt,\n lastHeartbeatAt: runningJobExecutions.lastHeartbeatAt,\n })\n .from(runningJobExecutions)\n .where(\n and(\n eq(runningJobExecutions.workspaceId, params.workspaceId),\n sql`${runningJobExecutions.lastHeartbeatAt} > now() - (${params.windowSeconds} \|\| ' seconds')::interval`,\n ),\n )\n .orderBy(desc(runningJobExecutions.lastHeartbeatAt), desc(runningJobExecutions.id))\n .limit(params.limit ?? 1000);\n}\n\nexport async function listRunningJobExecutionsByRunnerInstanceTx(\n tx: Tx,\n params: {\n workspaceId: string;\n provisionerId: string;\n providerRunnerIds: string[];\n },\n): Promise<RunnerInstanceBoundJobExecution[]> {\n if (params.providerRunnerIds.length === 0) return [];\n\n const duplicateRows = await tx\n .select({\n providerRunnerId: runningJobExecutions.providerRunnerId,\n count: count(),\n })\n .from(runningJobExecutions)\n .where(\n and(\n eq(runningJobExecutions.workspaceId, params.workspaceId),\n eq(runningJobExecutions.provisionerId, params.provisionerId),\n inArray(runningJobExecutions.providerRunnerId, params.providerRunnerIds),\n ),\n )\n .groupBy(runningJobExecutions.providerRunnerId)\n .having(sql`count() > 1`);\n\n const duplicateRunnerInstanceIds = duplicateRows.flatMap((row) =>\n row.providerRunnerId ? [row.providerRunnerId] : [],\n );\n if (duplicateRunnerInstanceIds.length > 0) {\n logger().warn(\n {\n workspaceId: params.workspaceId,\n provisionerId: params.provisionerId,\n providerRunnerIds: duplicateRunnerInstanceIds,\n },\n 'multiple running job executions are bound to the same provisioned runner',\n );\n }\n\n const result = await tx.execute<{\n workflowRunId: string;\n workflowRunAttemptId: string;\n jobId: string;\n jobExecutionId: string;\n providerRunnerId: string;\n startedAt: Date \| string;\n lastHeartbeatAt: Date \| string;\n cancellationRequestedAt: Date \| string \| null;\n }>(sql`\n SELECT DISTINCT ON (${runningJobExecutions.providerRunnerId})\n ${runningJobExecutions.workflowRunId} AS \"workflowRunId\",\n ${runningJobExecutions.workflowRunAttemptId} AS \"workflowRunAttemptId\",\n ${runningJobExecutions.jobId} AS \"jobId\",\n ${runningJobExecutions.jobExecutionId} AS \"jobExecutionId\",\n ${runningJobExecutions.providerRunnerId} AS \"providerRunnerId\",\n ${runningJobExecutions.startedAt} AS \"startedAt\",\n ${runningJobExecutions.lastHeartbeatAt} AS \"lastHeartbeatAt\",\n ${runningJobExecutions.cancellationRequestedAt} AS \"cancellationRequestedAt\"\n FROM ${runningJobExecutions}\n WHERE\n ${runningJobExecutions.workspaceId} = ${params.workspaceId}\n AND ${runningJobExecutions.provisionerId} = ${params.provisionerId}\n AND ${runningJobExecutions.providerRunnerId} IN (${sql.join(\n params.providerRunnerIds.map((providerRunnerId) => sql`${providerRunnerId}`),\n sql`, `,\n )})\n ORDER BY ${runningJobExecutions.providerRunnerId}, ${runningJobExecutions.startedAt} DESC, ${runningJobExecutions.jobExecutionId} DESC\n `);\n\n return result.rows.map((row) => ({\n workflowRunId: row.workflowRunId,\n workflowRunAttemptId: row.workflowRunAttemptId,\n jobId: row.jobId,\n jobExecutionId: row.jobExecutionId,\n providerRunnerId: row.providerRunnerId,\n startedAt: toDate(row.startedAt),\n lastHeartbeatAt: toDate(row.lastHeartbeatAt),\n cancellationRequestedAt: row.cancellationRequestedAt\n ? toDate(row.cancellationRequestedAt)\n : null,\n }));\n}\n\nfunction toDate(value: Date \| string): Date {\n return value instanceof Date ? value : new Date(value);\n}\n\nexport async function isJobLeaseActive(params: {\n jobId?: string;\n jobExecutionId: string;\n runnerSessionId: string;\n}): Promise<boolean> {\n const [row] = await db()\n .select({id: runningJobExecutions.id})\n .from(runningJobExecutions)\n .where(\n and(\n eq(runningJobExecutions.jobExecutionId, params.jobExecutionId),\n eq(runningJobExecutions.runnerSessionId, params.runnerSessionId),\n params.jobId === undefined ? undefined : eq(runningJobExecutions.jobId, params.jobId),\n ),\n )\n .limit(1);\n\n return row !== undefined;\n}\n\nexport async function recordHeartbeat(params: {\n jobExecutionId: string;\n runnerSessionId: string;\n toolCapabilities?: RunnerToolCapabilitiesDto \| null;\n}): Promise<{\n cancellationRequested: boolean;\n previousToolCapabilities: RunnerToolCapabilitiesDto \| null;\n currentToolCapabilities: RunnerToolCapabilitiesDto \| null;\n runningJobExecution: {\n workflowRunId: string;\n workflowRunAttemptId: string;\n jobId: string;\n jobExecutionId: string;\n projectId: string;\n workspaceId: string;\n runnerSessionId: string;\n };\n}> {\n const result = await db().transaction(async (tx) => {\n const updated = await tx\n .update(runningJobExecutions)\n .set({\n firstHeartbeatAt: sql`COALESCE(${runningJobExecutions.firstHeartbeatAt}, now())`,\n lastHeartbeatAt: sql`now()`,\n })\n .where(\n and(\n eq(runningJobExecutions.jobExecutionId, params.jobExecutionId),\n eq(runningJobExecutions.runnerSessionId, params.runnerSessionId),\n ),\n )\n .returning({\n cancellationRequestedAt: runningJobExecutions.cancellationRequestedAt,\n workflowRunId: runningJobExecutions.workflowRunId,\n workflowRunAttemptId: runningJobExecutions.workflowRunAttemptId,\n jobId: runningJobExecutions.jobId,\n jobExecutionId: runningJobExecutions.jobExecutionId,\n projectId: runningJobExecutions.projectId,\n workspaceId: runningJobExecutions.workspaceId,\n runnerSessionId: runningJobExecutions.runnerSessionId,\n });\n\n const row = updated[0];\n if (!row) throw new RunningJobExecutionNotFoundError(params.jobExecutionId);\n\n const [previous] = await tx\n .select({toolCapabilities: runnerSessions.toolCapabilities})\n .from(runnerSessions)\n .where(eq(runnerSessions.id, params.runnerSessionId))\n .limit(1)\n .for('update');\n\n const [session] = await tx\n .update(runnerSessions)\n .set({\n toolCapabilities: params.toolCapabilities ?? null,\n toolCapabilitiesReportedAt: params.toolCapabilities ? sql`now()` : null,\n updatedAt: sql`now()`,\n })\n .where(eq(runnerSessions.id, params.runnerSessionId))\n .returning({toolCapabilities: runnerSessions.toolCapabilities});\n\n return {\n row,\n previousToolCapabilities: previous?.toolCapabilities ?? null,\n currentToolCapabilities: session?.toolCapabilities ?? null,\n };\n });\n\n const row = result.row;\n if (!row) throw new RunningJobExecutionNotFoundError(params.jobExecutionId);\n return {\n cancellationRequested: row.cancellationRequestedAt !== null,\n previousToolCapabilities: result.previousToolCapabilities,\n currentToolCapabilities: result.currentToolCapabilities,\n runningJobExecution: {\n workflowRunId: row.workflowRunId,\n workflowRunAttemptId: row.workflowRunAttemptId,\n jobId: row.jobId,\n jobExecutionId: row.jobExecutionId,\n projectId: row.projectId,\n workspaceId: row.workspaceId,\n runnerSessionId: row.runnerSessionId,\n },\n };\n}\n\n/*\n Reconciles a terminal job execution with runner state in one transaction: removes its pending\n * queue row, requests cancellation on its running lease, and converges any linked reservation.\n * The operation is idempotent and preserves the first cancellation-request timestamp.\n */\nexport async function reconcileTerminalJobExecution(params: {\n jobExecutionId: string;\n}): Promise<void> {\n await db().transaction(async (tx) => {\n await lockJobExecution(tx, params.jobExecutionId);\n\n // Delete pending before updating running to match claim/release lock order. Claim locks\n // pending rows with SKIP LOCKED before inserting the running lease, so this ordering makes\n // a concurrent terminal reconciliation either win before claim or cancel the new lease.\n await tx\n .delete(pendingJobExecutions)\n .where(eq(pendingJobExecutions.jobExecutionId, params.jobExecutionId));\n const cancelledRunningRows = await tx\n .update(runningJobExecutions)\n .set({\n cancellationRequestedAt: sql`COALESCE(${runningJobExecutions.cancellationRequestedAt}, now())`,\n })\n .where(eq(runningJobExecutions.jobExecutionId, params.jobExecutionId))\n .returning({\n provisionerId: runningJobExecutions.provisionerId,\n providerRunnerId: runningJobExecutions.providerRunnerId,\n });\n\n await releaseReservationsForTerminalRunningRows(tx, cancelledRunningRows);\n });\n}\n"],"names":["RUNNER_JOB_CLAIMED","RUNNER_JOB_LEASE_EXPIRED","logger","writeOutboxEvent","writeOutboxEvents","canonicalizeLabels","and","arrayContained","asc","count","desc","eq","gt","inArray","isNotNull","isNull","lt","lte","or","sql","EmptyRequiredLabelsError","RunnerSessionExhaustedError","RunningJobExecutionNotFoundError","jobExecutionEnqueuedCount","jobExecutionLeaseExpiredCount","recordJobExecutionQueueTime","recordProviderRunnerActivationToFirstClaim","db","releaseTerminalRunnerInstanceReservationsByIds","runnersOutbox","pendingJobExecutions","providerRunners","runnerSessions","runningJobExecutions","runnerJobExecutionLockPrefix","lockJobExecution","tx","jobExecutionId","execute","releaseReservationsForTerminalRunningRows","rows","providerRunnerIdsByProvisionerId","Map","row","provisionerId","providerRunnerId","providerRunnerIds","get","Set","add","set","keys","sort","workspaceId","requireUnlinkedSession","requireTerminalState","getWorkspaceJobCounts","params","pendingRows","runningRows","Promise","all","select","from","where","workspaceIds","groupBy","queuedByWorkspace","map","Number","runningByWorkspace","queued","running","enqueueJobExecution","requiredLabels","length","enqueued","transaction","limit","leaseExpired","id","eventType","payload","inserted","insert","values","workflowRunId","workflowRunAttemptId","jobId","projectId","createdAt","queuedAt","onConflictDoNothing","target","returning","undefined","claimPendingJobExecution","touchRunnerSessionLiveness","runnerSessionId","throttleSeconds","runnerSessionLivenessThrottleSeconds","sessionLabels","activationToFirstClaimObservation","queueTimeObservation","result","runnerInstanceId","maxClaims","session","claimsUsed","revokedAt","for","candidate","orderBy","skipLocked","as","delete","runnerLabels","claimedAt","startedAt","claimed","durationMilliseconds","getTime","provider","launchKind","runnerInstanceCondition","claimedRunner","update","firstClaimedAt","updatedAt","isFirstClaim","providerKind","sessionCreatedAtEpochMs","type","toISOString","expireStuckJobExecutions","reaped","heartbeatCutoff","thresholdSeconds","firstHeartbeatCutoff","noFirstHeartbeatGraceSeconds","stalePredicate","firstHeartbeatAt","lastHeartbeatAt","staleRows","staleIds","staleJobExecutionIds","deleted","getJobExecutionQueueDepth","pending","value","listActiveRunningJobExecutions","windowSeconds","listRunningJobExecutionsByRunnerInstanceTx","duplicateRows","having","duplicateRunnerInstanceIds","flatMap","warn","cancellationRequestedAt","join","toDate","Date","isJobLeaseActive","recordHeartbeat","updated","previous","toolCapabilities","toolCapabilitiesReportedAt","previousToolCapabilities","currentToolCapabilities","cancellationRequested","runningJobExecution","reconcileTerminalJobExecution","cancelledRunningRows"],"mappings":"AAAA,SACEA,kBAAkB,EAClBC,wBAAwB,QAGnB,2BAA2B;AAClC,SAAQC,MAAM,QAAO,8BAA8B;AACnD,SAAQC,gBAAgB,EAAEC,iBAAiB,QAAO,uBAAuB;AACzE,SAAQC,kBAAkB,QAAO,yBAAyB;AAC1D,SACEC,GAAG,EACHC,cAAc,EACdC,GAAG,EACHC,KAAK,EACLC,IAAI,EACJC,EAAE,EACFC,EAAE,EACFC,OAAO,EACPC,SAAS,EACTC,MAAM,EACNC,EAAE,EACFC,GAAG,EACHC,EAAE,EACFC,GAAG,QACE,cAAc;AACrB,SACEC,wBAAwB,EACxBC,2BAA2B,EAC3BC,gCAAgC,QAC3B,kBAAkB;AACzB,SAEEC,yBAAyB,EACzBC,6BAA6B,EAE7BC,2BAA2B,EAC3BC,0CAA0C,QACrC,uBAAuB;AAE9B,SAAQC,EAAE,QAAO,UAAU;AAC3B,SAAQC,8CAA8C,QAAO,oBAAoB;AACjF,SAAQC,aAAa,QAAO,qBAAqB;AACjD,SAAQC,oBAAoB,QAAO,qCAAqC;AACxE,SAAQC,eAAe,QAAO,+BAA+B;AAC7D,SAAQC,cAAc,QAAO,8BAA8B;AAC3D,SAAQC,oBAAoB,QAAO,qCAAqC;AAExE,MAAMC,+BAA+B;AAErC,eAAeC,iBAAiBC,EAAM,EAAEC,cAAsB;IAC5D,MAAMD,GAAGE,OAAO,CACdnB,GAAG,CAAC,sCAAsC,EAAE,GAAGe,+BAA+BG,gBAAgB,CAAC,EAAE,CAAC;AAEtG;AAEA,eAAeE,0CACbH,EAAM,EACNI,IAGE;IAEF,2FAA2F;IAC3F,kFAAkF;IAClF,MAAMC,mCAAmC,IAAIC;IAC7C,KAAK,MAAMC,OAAOH,KAAM;QACtB,IAAIG,IAAIC,aAAa,KAAK,QAAQD,IAAIE,gBAAgB,KAAK,MAAM;QACjE,MAAMC,oBACJL,iCAAiCM,GAAG,CAACJ,IAAIC,aAAa,KAAK,IAAII;QACjEF,kBAAkBG,GAAG,CAACN,IAAIE,gBAAgB;QAC1CJ,iCAAiCS,GAAG,CAACP,IAAIC,aAAa,EAAEE;IAC1D;IAEA,KAAK,MAAMF,iBAAiB;WAAIH,iCAAiCU,IAAI;KAAG,CAACC,IAAI,GAAI;QAC/E,MAAMN,oBAAoBL,iCAAiCM,GAAG,CAACH;QAC/D,IAAI,CAACE,mBAAmB;QACxB,MAAMlB,+CAA+CQ,IAAI;YACvDiB,aAAa;YACbT;YACAE,mBAAmB;mBAAIA;aAAkB,CAACM,IAAI;YAC9CE,wBAAwB;YACxB,mFAAmF;YACnF,wCAAwC;YACxCC,sBAAsB;QACxB;IACF;AACF;AAaA,OAAO,eAAeC,sBAAsBC,MAE3C;IACC,MAAM,CAACC,aAAaC,YAAY,GAAG,MAAMC,QAAQC,GAAG,CAAC;QACnDlC,KACGmC,MAAM,CAAC;YAACT,aAAavB,qBAAqBuB,WAAW;YAAE5C,OAAOA;QAAO,GACrEsD,IAAI,CAACjC,sBACLkC,KAAK,CAACnD,QAAQiB,qBAAqBuB,WAAW,EAAEI,OAAOQ,YAAY,GACnEC,OAAO,CAACpC,qBAAqBuB,WAAW;QAC3C1B,KACGmC,MAAM,CAAC;YAACT,aAAapB,qBAAqBoB,WAAW;YAAE5C,OAAOA;QAAO,GACrEsD,IAAI,CAAC9B,sBACL+B,KAAK,CAACnD,QAAQoB,qBAAqBoB,WAAW,EAAEI,OAAOQ,YAAY,GACnEC,OAAO,CAACjC,qBAAqBoB,WAAW;KAC5C;IAED,MAAMc,oBAAoB,IAAIzB,IAAIgB,YAAYU,GAAG,CAAC,CAACzB,MAAQ;YAACA,IAAIU,WAAW;YAAEgB,OAAO1B,IAAIlC,KAAK;SAAE;IAC/F,MAAM6D,qBAAqB,IAAI5B,IAC7BiB,YAAYS,GAAG,CAAC,CAACzB,MAAQ;YAACA,IAAIU,WAAW;YAAEgB,OAAO1B,IAAIlC,KAAK;SAAE;IAG/D,OAAOgD,OAAOQ,YAAY,CAACG,GAAG,CAAC,CAACf,cAAiB,CAAA;YAC/CA;YACAkB,QAAQJ,kBAAkBpB,GAAG,CAACM,gBAAgB;YAC9CmB,SAASF,mBAAmBvB,GAAG,CAACM,gBAAgB;QAClD,CAAA;AACF;AAEA,+FAA+F;AAC/F,kGAAkG;AAClG,8FAA8F;AAC9F,OAAO,eAAeoB,oBAAoBhB,MAAiC;IACzE,MAAMiB,iBAAiB;WAAIrE,mBAAmBoD,OAAOiB,cAAc;KAAE;IACrE,IAAIA,eAAeC,MAAM,KAAK,GAAG,MAAM,IAAIvD;IAE3C,MAAMwD,WAAW,MAAMjD,KAAKkD,WAAW,CAAC,OAAOzC;QAC7C,MAAMD,iBAAiBC,IAAIqB,OAAOpB,cAAc;QAEhD,MAAM,CAACmC,QAAQ,GAAG,MAAMpC,GACrB0B,MAAM,CAAC;YAACzB,gBAAgBJ,qBAAqBI,cAAc;QAAA,GAC3D0B,IAAI,CAAC9B,sBACL+B,KAAK,CAACrD,GAAGsB,qBAAqBI,cAAc,EAAEoB,OAAOpB,cAAc,GACnEyC,KAAK,CAAC;QACT,IAAIN,SAAS,OAAO;QAEpB,MAAM,CAACO,aAAa,GAAG,MAAM3C,GAC1B0B,MAAM,CAAC;YAACkB,IAAInD,cAAcmD,EAAE;QAAA,GAC5BjB,IAAI,CAAClC,eACLmC,KAAK,CACJ1D,IACEK,GAAGkB,cAAcoD,SAAS,EAAEhF,2BAC5BkB,GAAG,CAAC,EAAEU,cAAcqD,OAAO,CAAC,sBAAsB,EAAEzB,OAAOpB,cAAc,CAAC,CAAC,GAG9EyC,KAAK,CAAC;QACT,IAAIC,cAAc,OAAO;QAEzB,MAAM,CAACI,SAAS,GAAG,MAAM/C,GACtBgD,MAAM,CAACtD,sBACPuD,MAAM,CAAC;YACNhC,aAAaI,OAAOJ,WAAW;YAC/BiC,eAAe7B,OAAO6B,aAAa;YACnCC,sBAAsB9B,OAAO8B,oBAAoB;YACjDC,OAAO/B,OAAO+B,KAAK;YACnBnD,gBAAgBoB,OAAOpB,cAAc;YACrCoD,WAAWhC,OAAOgC,SAAS;YAC3Bf;YACAgB,WAAWjC,OAAOkC,QAAQ;QAC5B,GACCC,mBAAmB,CAAC;YAACC,QAAQ/D,qBAAqBO,cAAc;QAAA,GAChEyD,SAAS,CAAC;YAACJ,WAAW5D,qBAAqB4D,SAAS;QAAA;QAEvD,OAAOP,aAAaY;IACtB;IAEA,IAAInB,UAAUrD,0BAA0B0B,GAAG,CAAC;AAC9C;AAoCA,OAAO,eAAe+C,yBAAyBvC,MAM9C;IACC,MAAMwC,2BAA2B;QAC/B5C,aAAaI,OAAOJ,WAAW;QAC/B6C,iBAAiBzC,OAAOyC,eAAe;QACvCC,iBAAiB1C,OAAO2C,oCAAoC;IAC9D;IAEA,IAAI3C,OAAO4C,aAAa,CAAC1B,MAAM,KAAK,GAAG,OAAO;IAE9C,IAAI2B,oCAA+E;IACnF,IAAIC,uBAAgE;IACpE,MAAMC,SAAS,MAAM7E,KAAKkD,WAAW,CAAC,OAAOzC;QAC3C,IAAIQ,gBAA+B;QACnC,IAAIC,mBAAkC;QACtC,IAAI4D,mBAAkC;QAEtC,IAAIhD,OAAOiD,SAAS,KAAK,MAAM;YAC7B,MAAM,CAACC,QAAQ,GAAG,MAAMvE,GACrB0B,MAAM,CAAC;gBACN4C,WAAW1E,eAAe0E,SAAS;gBACnCE,YAAY5E,eAAe4E,UAAU;gBACrCC,WAAW7E,eAAe6E,SAAS;gBACnCJ,kBAAkBzE,eAAeyE,gBAAgB;gBACjD7D,eAAeZ,eAAeY,aAAa;gBAC3CC,kBAAkBb,eAAea,gBAAgB;YACnD,GACCkB,IAAI,CAAC/B,gBACLgC,KAAK,CAACrD,GAAGqB,eAAegD,EAAE,EAAEvB,OAAOyC,eAAe,GAClDpB,KAAK,CAAC,GACNgC,GAAG,CAAC;YAEP,IACE,CAACH,WACDA,QAAQE,SAAS,IACjBF,QAAQD,SAAS,KAAK,QACtBC,QAAQC,UAAU,IAAID,QAAQD,SAAS,EACvC;gBACA,MAAM,IAAIrF,4BAA4BoC,OAAOyC,eAAe;YAC9D;YAEA,0EAA0E;YAC1E,mDAAmD;YACnDO,mBAAmBE,QAAQF,gBAAgB;YAC3C7D,gBAAgB+D,QAAQ/D,aAAa;YACrCC,mBAAmB8D,QAAQ9D,gBAAgB;QAC7C;QAEA,4EAA4E;QAC5E,oFAAoF;QACpF,oFAAoF;QACpF,sFAAsF;QACtF,MAAMkE,YAAY3E,GACf0B,MAAM,GACNC,IAAI,CAACjC,sBACLkC,KAAK,CACJ1D,IACEK,GAAGmB,qBAAqBuB,WAAW,EAAEI,OAAOJ,WAAW,GACvD9C,eAAeuB,qBAAqB4C,cAAc,EAAEjB,OAAO4C,aAAa,IAG3EW,OAAO,CAACxG,IAAIsB,qBAAqB4D,SAAS,GAAGlF,IAAIsB,qBAAqBkD,EAAE,GACxEF,KAAK,CAAC,GACNgC,GAAG,CAAC,UAAU;YAACG,YAAY;QAAI,GAC/BC,EAAE,CAAC;QAEN,MAAM,CAACvE,IAAI,GAAG,MAAMP,GACjB0B,MAAM,GACNC,IAAI,CAACgD,WACL/C,KAAK,CACJ7C,GAAG,CAAC;;qBAES,EAAEe,6BAA6B,IAAI,EAAE6E,UAAU1E,cAAc,CAAC;;QAE3E,CAAC;QAGL,IAAI,CAACM,KAAK,OAAO;QAEjB,MAAMP,GAAG+E,MAAM,CAACrF,sBAAsBkC,KAAK,CAACrD,GAAGmB,qBAAqBkD,EAAE,EAAErC,IAAIqC,EAAE;QAE9E,8EAA8E;QAC9E,gFAAgF;QAChF,qFAAqF;QACrF,2EAA2E;QAC3E,8EAA8E;QAC9E,2DAA2D;QAC3D,MAAMG,WAAW,MAAM/C,GACpBgD,MAAM,CAACnD,sBACPoD,MAAM,CAAC;YACNhC,aAAaV,IAAIU,WAAW;YAC5BiC,eAAe3C,IAAI2C,aAAa;YAChCC,sBAAsB5C,IAAI4C,oBAAoB;YAC9CC,OAAO7C,IAAI6C,KAAK;YAChBnD,gBAAgBM,IAAIN,cAAc;YAClCoD,WAAW9C,IAAI8C,SAAS;YACxBS,iBAAiBzC,OAAOyC,eAAe;YACvCtD;YACAC;YACA6B,gBAAgB/B,IAAI+B,cAAc;YAClC0C,cAAc3D,OAAO4C,aAAa;QACpC,GACCT,mBAAmB,CAAC;YAACC,QAAQ5D,qBAAqBI,cAAc;QAAA,GAChEyD,SAAS,CAAC;YAACuB,WAAWpF,qBAAqBqF,SAAS;QAAA;QAEvD,MAAMC,UAAUpC,QAAQ,CAAC,EAAE;QAC3B,IAAI,CAACoC,SAAS,OAAO;QAErBhB,uBAAuB;YACrBiB,sBAAsBD,QAAQF,SAAS,CAACI,OAAO,KAAK9E,IAAI+C,SAAS,CAAC+B,OAAO;YACzEC,UAAU;YACVC,YAAYlE,OAAOiD,SAAS,KAAK,OAAO,WAAW;QACrD;QAEA,MAAMkB,0BAA0BnB,mBAC5B9F,GAAGoB,gBAAgBiD,EAAE,EAAEyB,oBACvB7D,iBAAiBC,mBACfvC,IACEK,GAAGoB,gBAAgBa,aAAa,EAAEA,gBAClCjC,GAAGoB,gBAAgBc,gBAAgB,EAAEA,qBAEvC;QACN,IAAI+E,yBAAyB;YAC3B,MAAM,CAACC,cAAc,GAAG,MAAMzF,GAC3B0F,MAAM,CAAC/F,iBACPmB,GAAG,CAAC;gBACH6E,gBAAgB5G,GAAG,CAAC,SAAS,EAAEY,gBAAgBgG,cAAc,CAAC,EAAE,EAAER,QAAQF,SAAS,CAAC,CAAC,CAAC;gBACtFW,WAAW7G,GAAG,CAAC,KAAK,CAAC;YACvB,GACC6C,KAAK,CAAC4D,yBACN9B,SAAS,CAAC;gBACTiC,gBAAgBhG,gBAAgBgG,cAAc;gBAC9CE,cAAc9G,GAAY,CAAC,EAAEY,gBAAgBgG,cAAc,CAAC,GAAG,EAAER,QAAQF,SAAS,CAAC,CAAC;gBACpFK,UAAU3F,gBAAgBmG,YAAY;gBACtCP,YAAY5F,gBAAgB4F,UAAU;gBACtClB,kBAAkB1E,gBAAgBiD,EAAE;gBACpCmD,yBAAyBhH,GAAkB,CAAC;sCAChB,EAAEa,eAAe0D,SAAS,CAAC;iBAChD,EAAE1D,eAAe;kBAChB,EAAEA,eAAegD,EAAE,CAAC,GAAG,EAAEvB,OAAOyC,eAAe,CAAC;WACvD,CAAC;YACJ;YACF,IAAI2B,iBAAiBtB,sBAAsB;gBACzCA,qBAAqBmB,QAAQ,GAAGG,cAAcH,QAAQ;gBACtDnB,qBAAqBoB,UAAU,GAAGE,cAAcF,UAAU;YAC5D;YACA,IACEE,eAAeI,gBACfJ,cAAcE,cAAc,KAAK,QACjCF,cAAcM,uBAAuB,KAAK,QAC1CN,cAAcM,uBAAuB,KAAKpC,WAE1CO,oCAAoC;gBAClCkB,sBACEK,cAAcE,cAAc,CAACN,OAAO,KAAKI,cAAcM,uBAAuB;gBAChFT,UAAUG,cAAcH,QAAQ;gBAChCC,YAAYE,cAAcF,UAAU;gBACpClB,kBAAkBoB,cAAcpB,gBAAgB;YAClD;QACJ;QAEA,IAAIhD,OAAOiD,SAAS,KAAK,MAAM;YAC7B,MAAMtE,GACH0F,MAAM,CAAC9F,gBACPkB,GAAG,CAAC;gBAAC0D,YAAYzF,GAAG,CAAC,EAAEa,eAAe4E,UAAU,CAAC,IAAI,CAAC;gBAAEoB,WAAW7G,GAAG,CAAC,KAAK,CAAC;YAAA,GAC7E6C,KAAK,CAACrD,GAAGqB,eAAegD,EAAE,EAAEvB,OAAOyC,eAAe;QACvD;QAEA,4FAA4F;QAC5F,mFAAmF;QACnF,6BAA6B;QAC7B,MAAM/F,iBAAkCiC,IAAIP,eAAe;YACzDuG,MAAMpI;YACNkF,SAAS;gBACPI,eAAe3C,IAAI2C,aAAa;gBAChCC,sBAAsB5C,IAAI4C,oBAAoB;gBAC9CC,OAAO7C,IAAI6C,KAAK;gBAChBnD,gBAAgBM,IAAIN,cAAc;gBAClCgF,WAAWE,QAAQF,SAAS,CAACgB,WAAW;YAC1C;QACF;QAEA,OAAO;YACL/C,eAAe3C,IAAI2C,aAAa;YAChCC,sBAAsB5C,IAAI4C,oBAAoB;YAC9CC,OAAO7C,IAAI6C,KAAK;YAChBnD,gBAAgBM,IAAIN,cAAc;YAClCoD,WAAW9C,IAAI8C,SAAS;QAC1B;IACF;IACA,IAAIc,sBAAsB9E,4BAA4B8E;IACtD,IAAID,mCACF5E,2CAA2C4E;IAC7C,OAAOE;AACT;AAEA,eAAeP,2BAA2BxC,MAIzC;IACC,MAAM9B,KACHmG,MAAM,CAAC9F,gBACPkB,GAAG,CAAC;QAAC8E,WAAW7G,GAAG,CAAC,KAAK,CAAC;IAAA,GAC1B6C,KAAK,CACJ1D,IACEK,GAAGqB,eAAegD,EAAE,EAAEvB,OAAOyC,eAAe,GAC5CvF,GAAGqB,eAAeqB,WAAW,EAAEI,OAAOJ,WAAW,GACjDlC,GAAG,CAAC,EAAEa,eAAegG,SAAS,CAAC,YAAY,EAAEvE,OAAO0C,eAAe,CAAC,yBAAyB,CAAC;AAGtG;AAEA;;;;;;;;;;;;CAYC,GACD,OAAO,eAAemC,yBAAyB7E,MAI9C;IAQC,MAAM8E,SAAS,MAAM5G,KAAKkD,WAAW,CAAC,OAAOzC;QAC3C,MAAMoG,kBAAkBrH,GAAG,CAAC,SAAS,EAAEsC,OAAOgF,gBAAgB,CAAC,yBAAyB,CAAC;QACzF,MAAMC,uBAAuBvH,GAAG,CAAC,SAAS,EAAEsC,OAAOkF,4BAA4B,CAAC,yBAAyB,CAAC;QAC1G,MAAMC,iBAAiB1H,GACrBZ,IACES,OAAOkB,qBAAqB4G,gBAAgB,GAC5C5H,IAAIgB,qBAAqB6G,eAAe,EAAE7G,qBAAqBqF,SAAS,GACxEtG,GAAGiB,qBAAqBqF,SAAS,EAAEoB,wBAErCpI,IACEY,GACEJ,UAAUmB,qBAAqB4G,gBAAgB,GAC/CjI,GAAGqB,qBAAqB6G,eAAe,EAAE7G,qBAAqBqF,SAAS,IAEzEtG,GAAGiB,qBAAqB6G,eAAe,EAAEN;QAI7C,MAAMO,YAAY,MAAM3G,GACrB0B,MAAM,CAAC;YACNkB,IAAI/C,qBAAqB+C,EAAE;YAC3B3C,gBAAgBJ,qBAAqBI,cAAc;QACrD,GACC0B,IAAI,CAAC9B,sBACL+B,KAAK,CACJ1D,IACEsI,gBACAzH,GAAG,CAAC;;uBAES,EAAEe,6BAA6B,IAAI,EAAED,qBAAqBI,cAAc,CAAC;;UAEtF,CAAC,GAGJ2E,OAAO,CACNxG,IACEW,GAAG,CAAC,UAAU,EAAEc,qBAAqB4G,gBAAgB,CAAC,aAAa,EAAE5G,qBAAqB6G,eAAe,CAAC,IAAI,EAAE7G,qBAAqBqF,SAAS,CAAC,MAAM,EAAErF,qBAAqBqF,SAAS,CAAC,MAAM,EAAErF,qBAAqB6G,eAAe,CAAC,IAAI,CAAC,GAE1OtI,IAAIyB,qBAAqB+C,EAAE,GAE5BF,KAAK,CAACrB,OAAOqB,KAAK,IAAI;QAEzB,IAAIiE,UAAUpE,MAAM,KAAK,GAAG,OAAO,EAAE;QAErC,MAAMqE,WAAWD,UAAU3E,GAAG,CAAC,CAACzB,MAAQA,IAAIqC,EAAE;QAC9C,MAAMiE,uBAAuBF,UAAU3E,GAAG,CAAC,CAACzB,MAAQA,IAAIN,cAAc;QAEtE,4FAA4F;QAC5F,yFAAyF;QACzF,0FAA0F;QAC1F,yFAAyF;QACzF,MAAMD,GACH+E,MAAM,CAACrF,sBACPkC,KAAK,CAACnD,QAAQiB,qBAAqBO,cAAc,EAAE4G;QAEtD,MAAMC,UAAU,MAAM9G,GACnB+E,MAAM,CAAClF,sBACP+B,KAAK,CAAC1D,IAAIO,QAAQoB,qBAAqB+C,EAAE,EAAEgE,WAAWJ,iBACtD9C,SAAS,CAAC;YACTR,eAAerD,qBAAqBqD,aAAa;YACjDC,sBAAsBtD,qBAAqBsD,oBAAoB;YAC/DC,OAAOvD,qBAAqBuD,KAAK;YACjCnD,gBAAgBJ,qBAAqBI,cAAc;YACnDO,eAAeX,qBAAqBW,aAAa;YACjDC,kBAAkBZ,qBAAqBY,gBAAgB;QACzD;QAEF,IAAIqG,QAAQvE,MAAM,KAAK,GAAG,OAAO,EAAE;QAEnC,MAAMpC,0CAA0CH,IAAI8G;QAEpD,MAAM9I,kBACJgC,IACAP,eACAqH,QAAQ9E,GAAG,CAAC,CAACzB,MAAS,CAAA;gBACpByF,MAAMnI;gBACNiF,SAAS;oBACPI,eAAe3C,IAAI2C,aAAa;oBAChCC,sBAAsB5C,IAAI4C,oBAAoB;oBAC9CC,OAAO7C,IAAI6C,KAAK;oBAChBnD,gBAAgBM,IAAIN,cAAc;gBACpC;YACF,CAAA;QAGF,OAAO6G,QAAQ9E,GAAG,CAAC,CAAC,EAACkB,aAAa,EAAEC,oBAAoB,EAAEC,KAAK,EAAEnD,cAAc,EAAC,GAAM,CAAA;gBACpFiD;gBACAC;gBACAC;gBACAnD;YACF,CAAA;IACF;IAEA,IAAIkG,OAAO5D,MAAM,GAAG,GAAGnD,8BAA8ByB,GAAG,CAACsF,OAAO5D,MAAM;IAEtE,OAAO4D;AACT;AAEA,OAAO,eAAeY;IAIpB,MAAM,CAACC,QAAQ,GAAG,MAAMzH,KAAKmC,MAAM,CAAC;QAACuF,OAAO5I;IAAO,GAAGsD,IAAI,CAACjC;IAC3D,MAAM,CAAC0C,QAAQ,GAAG,MAAM7C,KAAKmC,MAAM,CAAC;QAACuF,OAAO5I;IAAO,GAAGsD,IAAI,CAAC9B;IAC3D,OAAO;QACLH,sBAAsBsH,SAASC,SAAS;QACxCpH,sBAAsBuC,SAAS6E,SAAS;IAC1C;AACF;AAEA,OAAO,eAAeC,+BAA+B7F,MAIpD;IACC,OAAO,MAAM9B,KACVmC,MAAM,CAAC;QACN0B,OAAOvD,qBAAqBuD,KAAK;QACjCnD,gBAAgBJ,qBAAqBI,cAAc;QACnDiD,eAAerD,qBAAqBqD,aAAa;QACjDC,sBAAsBtD,qBAAqBsD,oBAAoB;QAC/DE,WAAWxD,qBAAqBwD,SAAS;QACzCS,iBAAiBjE,qBAAqBiE,eAAe;QACrDtD,eAAeX,qBAAqBW,aAAa;QACjDC,kBAAkBZ,qBAAqBY,gBAAgB;QACvD6B,gBAAgBzC,qBAAqByC,cAAc;QACnD0C,cAAcnF,qBAAqBmF,YAAY;QAC/CE,WAAWrF,qBAAqBqF,SAAS;QACzCwB,iBAAiB7G,qBAAqB6G,eAAe;IACvD,GACC/E,IAAI,CAAC9B,sBACL+B,KAAK,CACJ1D,IACEK,GAAGsB,qBAAqBoB,WAAW,EAAEI,OAAOJ,WAAW,GACvDlC,GAAG,CAAC,EAAEc,qBAAqB6G,eAAe,CAAC,YAAY,EAAErF,OAAO8F,aAAa,CAAC,yBAAyB,CAAC,GAG3GvC,OAAO,CAACtG,KAAKuB,qBAAqB6G,eAAe,GAAGpI,KAAKuB,qBAAqB+C,EAAE,GAChFF,KAAK,CAACrB,OAAOqB,KAAK,IAAI;AAC3B;AAEA,OAAO,eAAe0E,2CACpBpH,EAAM,EACNqB,MAIC;IAED,IAAIA,OAAOX,iBAAiB,CAAC6B,MAAM,KAAK,GAAG,OAAO,EAAE;IAEpD,MAAM8E,gBAAgB,MAAMrH,GACzB0B,MAAM,CAAC;QACNjB,kBAAkBZ,qBAAqBY,gBAAgB;QACvDpC,OAAOA;IACT,GACCsD,IAAI,CAAC9B,sBACL+B,KAAK,CACJ1D,IACEK,GAAGsB,qBAAqBoB,WAAW,EAAEI,OAAOJ,WAAW,GACvD1C,GAAGsB,qBAAqBW,aAAa,EAAEa,OAAOb,aAAa,GAC3D/B,QAAQoB,qBAAqBY,gBAAgB,EAAEY,OAAOX,iBAAiB,IAG1EoB,OAAO,CAACjC,qBAAqBY,gBAAgB,EAC7C6G,MAAM,CAACvI,GAAG,CAAC,YAAY,CAAC;IAE3B,MAAMwI,6BAA6BF,cAAcG,OAAO,CAAC,CAACjH,MACxDA,IAAIE,gBAAgB,GAAG;YAACF,IAAIE,gBAAgB;SAAC,GAAG,EAAE;IAEpD,IAAI8G,2BAA2BhF,MAAM,GAAG,GAAG;QACzCzE,SAAS2J,IAAI,CACX;YACExG,aAAaI,OAAOJ,WAAW;YAC/BT,eAAea,OAAOb,aAAa;YACnCE,mBAAmB6G;QACrB,GACA;IAEJ;IAEA,MAAMnD,SAAS,MAAMpE,GAAGE,OAAO,CAS5BnB,GAAG,CAAC;wBACe,EAAEc,qBAAqBY,gBAAgB,CAAC;MAC1D,EAAEZ,qBAAqBqD,aAAa,CAAC;MACrC,EAAErD,qBAAqBsD,oBAAoB,CAAC;MAC5C,EAAEtD,qBAAqBuD,KAAK,CAAC;MAC7B,EAAEvD,qBAAqBI,cAAc,CAAC;MACtC,EAAEJ,qBAAqBY,gBAAgB,CAAC;MACxC,EAAEZ,qBAAqBqF,SAAS,CAAC;MACjC,EAAErF,qBAAqB6G,eAAe,CAAC;MACvC,EAAE7G,qBAAqB6H,uBAAuB,CAAC;SAC5C,EAAE7H,qBAAqB;;MAE1B,EAAEA,qBAAqBoB,WAAW,CAAC,GAAG,EAAEI,OAAOJ,WAAW,CAAC;UACvD,EAAEpB,qBAAqBW,aAAa,CAAC,GAAG,EAAEa,OAAOb,aAAa,CAAC;UAC/D,EAAEX,qBAAqBY,gBAAgB,CAAC,KAAK,EAAE1B,IAAI4I,IAAI,CACzDtG,OAAOX,iBAAiB,CAACsB,GAAG,CAAC,CAACvB,mBAAqB1B,GAAG,CAAC,EAAE0B,iBAAiB,CAAC,GAC3E1B,GAAG,CAAC,EAAE,CAAC,EACP;aACK,EAAEc,qBAAqBY,gBAAgB,CAAC,EAAE,EAAEZ,qBAAqBqF,SAAS,CAAC,OAAO,EAAErF,qBAAqBI,cAAc,CAAC;EACnI,CAAC;IAED,OAAOmE,OAAOhE,IAAI,CAAC4B,GAAG,CAAC,CAACzB,MAAS,CAAA;YAC/B2C,eAAe3C,IAAI2C,aAAa;YAChCC,sBAAsB5C,IAAI4C,oBAAoB;YAC9CC,OAAO7C,IAAI6C,KAAK;YAChBnD,gBAAgBM,IAAIN,cAAc;YAClCQ,kBAAkBF,IAAIE,gBAAgB;YACtCyE,WAAW0C,OAAOrH,IAAI2E,SAAS;YAC/BwB,iBAAiBkB,OAAOrH,IAAImG,eAAe;YAC3CgB,yBAAyBnH,IAAImH,uBAAuB,GAChDE,OAAOrH,IAAImH,uBAAuB,IAClC;QACN,CAAA;AACF;AAEA,SAASE,OAAOX,KAAoB;IAClC,OAAOA,iBAAiBY,OAAOZ,QAAQ,IAAIY,KAAKZ;AAClD;AAEA,OAAO,eAAea,iBAAiBzG,MAItC;IACC,MAAM,CAACd,IAAI,GAAG,MAAMhB,KACjBmC,MAAM,CAAC;QAACkB,IAAI/C,qBAAqB+C,EAAE;IAAA,GACnCjB,IAAI,CAAC9B,sBACL+B,KAAK,CACJ1D,IACEK,GAAGsB,qBAAqBI,cAAc,EAAEoB,OAAOpB,cAAc,GAC7D1B,GAAGsB,qBAAqBiE,eAAe,EAAEzC,OAAOyC,eAAe,GAC/DzC,OAAO+B,KAAK,KAAKO,YAAYA,YAAYpF,GAAGsB,qBAAqBuD,KAAK,EAAE/B,OAAO+B,KAAK,IAGvFV,KAAK,CAAC;IAET,OAAOnC,QAAQoD;AACjB;AAEA,OAAO,eAAeoE,gBAAgB1G,MAIrC;IAcC,MAAM+C,SAAS,MAAM7E,KAAKkD,WAAW,CAAC,OAAOzC;QAC3C,MAAMgI,UAAU,MAAMhI,GACnB0F,MAAM,CAAC7F,sBACPiB,GAAG,CAAC;YACH2F,kBAAkB1H,GAAG,CAAC,SAAS,EAAEc,qBAAqB4G,gBAAgB,CAAC,QAAQ,CAAC;YAChFC,iBAAiB3H,GAAG,CAAC,KAAK,CAAC;QAC7B,GACC6C,KAAK,CACJ1D,IACEK,GAAGsB,qBAAqBI,cAAc,EAAEoB,OAAOpB,cAAc,GAC7D1B,GAAGsB,qBAAqBiE,eAAe,EAAEzC,OAAOyC,eAAe,IAGlEJ,SAAS,CAAC;YACTgE,yBAAyB7H,qBAAqB6H,uBAAuB;YACrExE,eAAerD,qBAAqBqD,aAAa;YACjDC,sBAAsBtD,qBAAqBsD,oBAAoB;YAC/DC,OAAOvD,qBAAqBuD,KAAK;YACjCnD,gBAAgBJ,qBAAqBI,cAAc;YACnDoD,WAAWxD,qBAAqBwD,SAAS;YACzCpC,aAAapB,qBAAqBoB,WAAW;YAC7C6C,iBAAiBjE,qBAAqBiE,eAAe;QACvD;QAEF,MAAMvD,MAAMyH,OAAO,CAAC,EAAE;QACtB,IAAI,CAACzH,KAAK,MAAM,IAAIrB,iCAAiCmC,OAAOpB,cAAc;QAE1E,MAAM,CAACgI,SAAS,GAAG,MAAMjI,GACtB0B,MAAM,CAAC;YAACwG,kBAAkBtI,eAAesI,gBAAgB;QAAA,GACzDvG,IAAI,CAAC/B,gBACLgC,KAAK,CAACrD,GAAGqB,eAAegD,EAAE,EAAEvB,OAAOyC,eAAe,GAClDpB,KAAK,CAAC,GACNgC,GAAG,CAAC;QAEP,MAAM,CAACH,QAAQ,GAAG,MAAMvE,GACrB0F,MAAM,CAAC9F,gBACPkB,GAAG,CAAC;YACHoH,kBAAkB7G,OAAO6G,gBAAgB,IAAI;YAC7CC,4BAA4B9G,OAAO6G,gBAAgB,GAAGnJ,GAAG,CAAC,KAAK,CAAC,GAAG;YACnE6G,WAAW7G,GAAG,CAAC,KAAK,CAAC;QACvB,GACC6C,KAAK,CAACrD,GAAGqB,eAAegD,EAAE,EAAEvB,OAAOyC,eAAe,GAClDJ,SAAS,CAAC;YAACwE,kBAAkBtI,eAAesI,gBAAgB;QAAA;QAE/D,OAAO;YACL3H;YACA6H,0BAA0BH,UAAUC,oBAAoB;YACxDG,yBAAyB9D,SAAS2D,oBAAoB;QACxD;IACF;IAEA,MAAM3H,MAAM6D,OAAO7D,GAAG;IACtB,IAAI,CAACA,KAAK,MAAM,IAAIrB,iCAAiCmC,OAAOpB,cAAc;IAC1E,OAAO;QACLqI,uBAAuB/H,IAAImH,uBAAuB,KAAK;QACvDU,0BAA0BhE,OAAOgE,wBAAwB;QACzDC,yBAAyBjE,OAAOiE,uBAAuB;QACvDE,qBAAqB;YACnBrF,eAAe3C,IAAI2C,aAAa;YAChCC,sBAAsB5C,IAAI4C,oBAAoB;YAC9CC,OAAO7C,IAAI6C,KAAK;YAChBnD,gBAAgBM,IAAIN,cAAc;YAClCoD,WAAW9C,IAAI8C,SAAS;YACxBpC,aAAaV,IAAIU,WAAW;YAC5B6C,iBAAiBvD,IAAIuD,eAAe;QACtC;IACF;AACF;AAEA;;;;CAIC,GACD,OAAO,eAAe0E,8BAA8BnH,MAEnD;IACC,MAAM9B,KAAKkD,WAAW,CAAC,OAAOzC;QAC5B,MAAMD,iBAAiBC,IAAIqB,OAAOpB,cAAc;QAEhD,wFAAwF;QACxF,2FAA2F;QAC3F,wFAAwF;QACxF,MAAMD,GACH+E,MAAM,CAACrF,sBACPkC,KAAK,CAACrD,GAAGmB,qBAAqBO,cAAc,EAAEoB,OAAOpB,cAAc;QACtE,MAAMwI,uBAAuB,MAAMzI,GAChC0F,MAAM,CAAC7F,sBACPiB,GAAG,CAAC;YACH4G,yBAAyB3I,GAAG,CAAC,SAAS,EAAEc,qBAAqB6H,uBAAuB,CAAC,QAAQ,CAAC;QAChG,GACC9F,KAAK,CAACrD,GAAGsB,qBAAqBI,cAAc,EAAEoB,OAAOpB,cAAc,GACnEyD,SAAS,CAAC;YACTlD,eAAeX,qBAAqBW,aAAa;YACjDC,kBAAkBZ,qBAAqBY,gBAAgB;QACzD;QAEF,MAAMN,0CAA0CH,IAAIyI;IACtD;AACF"}
1	+ {"version":3,"sources":["../../src/db/job-executions.ts"],"sourcesContent":["import {\n RUNNER_JOB_CLAIMED,\n RUNNER_JOB_LEASE_EXPIRED,\n type RunnersEventMap,\n type RunnerToolCapabilitiesDto,\n} from '@shipfox/api-runners-dto';\nimport {logger} from '@shipfox/node-opentelemetry';\nimport {writeOutboxEvent, writeOutboxEvents} from '@shipfox/node-outbox';\nimport {canonicalizeLabels} from '@shipfox/runner-labels';\nimport {\n and,\n arrayContained,\n asc,\n count,\n desc,\n eq,\n gt,\n inArray,\n isNotNull,\n isNull,\n lt,\n lte,\n or,\n sql,\n} from 'drizzle-orm';\nimport {\n EmptyRequiredLabelsError,\n RunnerSessionExhaustedError,\n RunningJobExecutionNotFoundError,\n} from '#core/errors.js';\nimport {\n type JobExecutionQueueTimeObservation,\n jobExecutionEnqueuedCount,\n jobExecutionLeaseExpiredCount,\n type ProviderRunnerLifecycleObservation,\n recordJobExecutionQueueTime,\n recordProviderRunnerActivationToFirstClaim,\n} from '#metrics/instance.js';\nimport type {Tx} from './db.js';\nimport {db} from './db.js';\nimport {lockRunnerReservationAdvisoryKeysTx} from './reservation-locks.js';\nimport {\n releaseReservationUnits,\n releaseTerminalRunnerInstanceReservationsByIds,\n} from './reservations.js';\nimport {runnersOutbox} from './schema/outbox.js';\nimport {pendingJobExecutions} from './schema/pending-job-executions.js';\nimport {reservations} from './schema/reservations.js';\nimport {providerRunners} from './schema/runner-instances.js';\nimport {runnerSessions} from './schema/runner-sessions.js';\nimport {runningJobExecutions} from './schema/running-job-executions.js';\n\nconst runnerJobExecutionLockPrefix = 'runners_job_execution:';\n\nasync function lockJobExecution(tx: Tx, jobExecutionId: string): Promise<void> {\n await tx.execute(\n sql`select pg_advisory_xact_lock(hashtext(${`${runnerJobExecutionLockPrefix}${jobExecutionId}`}))`,\n );\n}\n\nasync function releaseReservationsForTerminalRunningRows(\n tx: Tx,\n rows: ReadonlyArray<{\n provisionerId: string \| null;\n providerRunnerId: string \| null;\n }>,\n): Promise<void> {\n // Converge after the lease fact changes. The reservation helper locks each provider runner\n // and releases only when the runner is terminal and no uncancelled lease remains.\n const providerRunnerIdsByProvisionerId = new Map<string, Set<string>>();\n for (const row of rows) {\n if (row.provisionerId === null \|\| row.providerRunnerId === null) continue;\n const providerRunnerIds =\n providerRunnerIdsByProvisionerId.get(row.provisionerId) ?? new Set<string>();\n providerRunnerIds.add(row.providerRunnerId);\n providerRunnerIdsByProvisionerId.set(row.provisionerId, providerRunnerIds);\n }\n\n for (const provisionerId of [...providerRunnerIdsByProvisionerId.keys()].sort()) {\n const providerRunnerIds = providerRunnerIdsByProvisionerId.get(provisionerId);\n if (!providerRunnerIds) continue;\n await releaseTerminalRunnerInstanceReservationsByIds(tx, {\n workspaceId: null,\n provisionerId,\n providerRunnerIds: [...providerRunnerIds].sort(),\n requireUnlinkedSession: false,\n // Lock and re-check the runner row locally so lease finalization remains retryable\n // without a workflow/runner scope lock.\n requireTerminalState: false,\n });\n }\n}\n\nexport interface EnqueueJobExecutionParams {\n workspaceId: string;\n workflowRunId: string;\n workflowRunAttemptId: string;\n jobId: string;\n jobExecutionId: string;\n projectId: string;\n requiredLabels: string[];\n queuedAt: Date;\n}\n\nexport async function getWorkspaceJobCounts(params: {\n workspaceIds: string[];\n}): Promise<Array<{workspaceId: string; queued: number; running: number}>> {\n const [pendingRows, runningRows] = await Promise.all([\n db()\n .select({workspaceId: pendingJobExecutions.workspaceId, count: count()})\n .from(pendingJobExecutions)\n .where(inArray(pendingJobExecutions.workspaceId, params.workspaceIds))\n .groupBy(pendingJobExecutions.workspaceId),\n db()\n .select({workspaceId: runningJobExecutions.workspaceId, count: count()})\n .from(runningJobExecutions)\n .where(inArray(runningJobExecutions.workspaceId, params.workspaceIds))\n .groupBy(runningJobExecutions.workspaceId),\n ]);\n\n const queuedByWorkspace = new Map(pendingRows.map((row) => [row.workspaceId, Number(row.count)]));\n const runningByWorkspace = new Map(\n runningRows.map((row) => [row.workspaceId, Number(row.count)]),\n );\n\n return params.workspaceIds.map((workspaceId) => ({\n workspaceId,\n queued: queuedByWorkspace.get(workspaceId) ?? 0,\n running: runningByWorkspace.get(workspaceId) ?? 0,\n }));\n}\n\n// The workflows outbox delivers queue facts at least once. The per-execution advisory lock and\n// pending/running checks make replay a no-op across the claim transition. A durable lease-expired\n// event also prevents a delayed replay from resurrecting an execution already reaped locally.\nexport async function enqueueJobExecution(params: EnqueueJobExecutionParams): Promise<void> {\n const requiredLabels = [...canonicalizeLabels(params.requiredLabels)];\n if (requiredLabels.length === 0) throw new EmptyRequiredLabelsError();\n\n const enqueued = await db().transaction(async (tx) => {\n await lockJobExecution(tx, params.jobExecutionId);\n\n const [running] = await tx\n .select({jobExecutionId: runningJobExecutions.jobExecutionId})\n .from(runningJobExecutions)\n .where(eq(runningJobExecutions.jobExecutionId, params.jobExecutionId))\n .limit(1);\n if (running) return false;\n\n const [leaseExpired] = await tx\n .select({id: runnersOutbox.id})\n .from(runnersOutbox)\n .where(\n and(\n eq(runnersOutbox.eventType, RUNNER_JOB_LEASE_EXPIRED),\n sql`${runnersOutbox.payload}->>'jobExecutionId' = ${params.jobExecutionId}`,\n ),\n )\n .limit(1);\n if (leaseExpired) return false;\n\n const [inserted] = await tx\n .insert(pendingJobExecutions)\n .values({\n workspaceId: params.workspaceId,\n workflowRunId: params.workflowRunId,\n workflowRunAttemptId: params.workflowRunAttemptId,\n jobId: params.jobId,\n jobExecutionId: params.jobExecutionId,\n projectId: params.projectId,\n requiredLabels,\n createdAt: params.queuedAt,\n })\n .onConflictDoNothing({target: pendingJobExecutions.jobExecutionId})\n .returning({createdAt: pendingJobExecutions.createdAt});\n\n return inserted !== undefined;\n });\n\n if (enqueued) jobExecutionEnqueuedCount.add(1);\n}\n\nexport interface ClaimedJobExecution {\n workflowRunId: string;\n workflowRunAttemptId: string;\n jobId: string;\n jobExecutionId: string;\n projectId: string;\n}\n\nexport interface ActiveRunningJobExecution {\n workflowRunId: string;\n workflowRunAttemptId: string;\n jobId: string;\n jobExecutionId: string;\n projectId: string;\n runnerSessionId: string;\n provisionerId: string \| null;\n providerRunnerId: string \| null;\n requiredLabels: string[];\n runnerLabels: string[];\n startedAt: Date;\n lastHeartbeatAt: Date;\n}\n\nexport interface RunnerInstanceBoundJobExecution {\n workflowRunId: string;\n workflowRunAttemptId: string;\n jobId: string;\n jobExecutionId: string;\n providerRunnerId: string;\n startedAt: Date;\n lastHeartbeatAt: Date;\n cancellationRequestedAt: Date \| null;\n}\n\nexport async function claimPendingJobExecution(params: {\n workspaceId: string;\n runnerSessionId: string;\n sessionLabels: string[];\n maxClaims: number \| null;\n runnerSessionLivenessThrottleSeconds: number;\n}): Promise<ClaimedJobExecution \| null> {\n await touchRunnerSessionLiveness({\n workspaceId: params.workspaceId,\n runnerSessionId: params.runnerSessionId,\n throttleSeconds: params.runnerSessionLivenessThrottleSeconds,\n });\n\n if (params.sessionLabels.length === 0) return null;\n\n let activationToFirstClaimObservation: ProviderRunnerLifecycleObservation \| null = null;\n let queueTimeObservation: JobExecutionQueueTimeObservation \| null = null;\n const result = await db().transaction(async (tx) => {\n let provisionerId: string \| null = null;\n let providerRunnerId: string \| null = null;\n let runnerInstanceId: string \| null = null;\n\n if (params.maxClaims !== null) {\n const [session] = await tx\n .select({\n maxClaims: runnerSessions.maxClaims,\n claimsUsed: runnerSessions.claimsUsed,\n revokedAt: runnerSessions.revokedAt,\n runnerInstanceId: runnerSessions.runnerInstanceId,\n provisionerId: runnerSessions.provisionerId,\n providerRunnerId: runnerSessions.providerRunnerId,\n })\n .from(runnerSessions)\n .where(eq(runnerSessions.id, params.runnerSessionId))\n .limit(1)\n .for('update');\n\n if (\n !session \|\|\n session.revokedAt \|\|\n session.maxClaims === null \|\|\n session.claimsUsed >= session.maxClaims\n ) {\n throw new RunnerSessionExhaustedError(params.runnerSessionId);\n }\n\n // Ephemeral sessions are the only capped sessions, and the DB check keeps\n // their provisioned-runner link present as a pair.\n runnerInstanceId = session.runnerInstanceId;\n provisionerId = session.provisionerId;\n providerRunnerId = session.providerRunnerId;\n }\n\n const runnerInstanceCondition = runnerInstanceId\n ? eq(providerRunners.id, runnerInstanceId)\n : provisionerId && providerRunnerId\n ? and(\n eq(providerRunners.provisionerId, provisionerId),\n eq(providerRunners.providerRunnerId, providerRunnerId),\n )\n : null;\n if (runnerInstanceCondition && provisionerId) {\n const [runner] = await tx\n .select({\n reservationId: providerRunners.reservationId,\n intendedReservationId: providerRunners.intendedReservationId,\n })\n .from(providerRunners)\n .where(runnerInstanceCondition)\n .limit(1);\n await lockRunnerReservationAdvisoryKeysTx(tx, {\n provisionerId,\n reservationIds: [runner?.reservationId, runner?.intendedReservationId].filter(\n (reservationId): reservationId is string =>\n reservationId !== null && reservationId !== undefined,\n ),\n });\n }\n\n // `id` is a uuidv7 (time-ordered), so it is a deterministic FIFO tiebreaker\n // for rows sharing a created_at within a batch. Lock only the FIFO candidate before\n // attempting its execution advisory lock; putting pg_try_advisory_xact_lock in this\n // predicate would evaluate it while scanning and temporarily lock many queue entries.\n const candidate = tx\n .select()\n .from(pendingJobExecutions)\n .where(\n and(\n eq(pendingJobExecutions.workspaceId, params.workspaceId),\n arrayContained(pendingJobExecutions.requiredLabels, params.sessionLabels),\n ),\n )\n .orderBy(asc(pendingJobExecutions.createdAt), asc(pendingJobExecutions.id))\n .limit(1)\n .for('update', {skipLocked: true})\n .as('pending_candidate');\n\n const [row] = await tx\n .select()\n .from(candidate)\n .where(\n sql`\n pg_try_advisory_xact_lock(\n hashtext(${runnerJobExecutionLockPrefix} \|\| ${candidate.jobExecutionId}::text)\n )\n `,\n );\n\n if (!row) return null;\n\n await tx.delete(pendingJobExecutions).where(eq(pendingJobExecutions.id, row.id));\n\n // An enqueue retry that lands after a prior claim can leave an orphan pending\n // row whose jobExecutionId is already in `runners_running_jobs`. Insert-or-skip\n // on the jobExecutionId unique constraint: when the job execution is already running\n // the insert touches no row, so we commit the orphan's deletion and return\n // null rather than let the unique violation roll the claim back into a poison\n // loop. The runner just re-polls for a real job execution.\n const inserted = await tx\n .insert(runningJobExecutions)\n .values({\n workspaceId: row.workspaceId,\n workflowRunId: row.workflowRunId,\n workflowRunAttemptId: row.workflowRunAttemptId,\n jobId: row.jobId,\n jobExecutionId: row.jobExecutionId,\n projectId: row.projectId,\n runnerSessionId: params.runnerSessionId,\n provisionerId,\n providerRunnerId,\n requiredLabels: row.requiredLabels,\n runnerLabels: params.sessionLabels,\n })\n .onConflictDoNothing({target: runningJobExecutions.jobExecutionId})\n .returning({claimedAt: runningJobExecutions.startedAt});\n\n const claimed = inserted[0];\n if (!claimed) return null;\n\n queueTimeObservation = {\n durationMilliseconds: claimed.claimedAt.getTime() - row.createdAt.getTime(),\n provider: null,\n launchKind: params.maxClaims === null ? 'manual' : 'unknown',\n };\n\n if (runnerInstanceCondition) {\n const [claimedRunner] = await tx\n .update(providerRunners)\n .set({\n firstClaimedAt: sql`coalesce(${providerRunners.firstClaimedAt}, ${claimed.claimedAt})`,\n updatedAt: sql`now()`,\n })\n .where(runnerInstanceCondition)\n .returning({\n firstClaimedAt: providerRunners.firstClaimedAt,\n isFirstClaim: sql<boolean>`${providerRunners.firstClaimedAt} = ${claimed.claimedAt}`,\n provider: providerRunners.providerKind,\n launchKind: providerRunners.launchKind,\n runnerInstanceId: providerRunners.id,\n reservationId: providerRunners.reservationId,\n intendedReservationId: providerRunners.intendedReservationId,\n sessionCreatedAtEpochMs: sql<number \| null>`(\n select extract(epoch from ${runnerSessions.createdAt})::double precision * 1000\n from ${runnerSessions}\n where ${runnerSessions.id} = ${params.runnerSessionId}\n )`,\n });\n if (claimedRunner && queueTimeObservation) {\n queueTimeObservation.provider = claimedRunner.provider;\n queueTimeObservation.launchKind = claimedRunner.launchKind;\n }\n if (claimedRunner?.isFirstClaim) {\n const [releasedRunner] = await tx\n .update(providerRunners)\n .set({\n intendedReservationId: null,\n reservationReleasedAt: sql`now()`,\n updatedAt: sql`now()`,\n })\n .where(\n and(\n eq(providerRunners.id, claimedRunner.runnerInstanceId),\n or(\n isNotNull(providerRunners.reservationId),\n isNotNull(providerRunners.intendedReservationId),\n ),\n isNull(providerRunners.reservationReleasedAt),\n ),\n )\n .returning({\n provisionerId: providerRunners.provisionerId,\n });\n const reservationId = claimedRunner.intendedReservationId ?? claimedRunner.reservationId;\n if (releasedRunner?.provisionerId && reservationId) {\n const [reservation] = await tx\n .select({workspaceId: reservations.workspaceId})\n .from(reservations)\n .where(\n and(\n eq(reservations.id, reservationId),\n eq(reservations.provisionerId, releasedRunner.provisionerId),\n ),\n )\n .limit(1);\n if (reservation)\n await releaseReservationUnits(tx, {\n workspaceId: reservation.workspaceId,\n provisionerId: releasedRunner.provisionerId,\n releases: [{reservationId, count: 1}],\n });\n }\n }\n if (\n claimedRunner?.isFirstClaim &&\n claimedRunner.firstClaimedAt !== null &&\n claimedRunner.sessionCreatedAtEpochMs !== null &&\n claimedRunner.sessionCreatedAtEpochMs !== undefined\n )\n activationToFirstClaimObservation = {\n durationMilliseconds:\n claimedRunner.firstClaimedAt.getTime() - claimedRunner.sessionCreatedAtEpochMs,\n provider: claimedRunner.provider,\n launchKind: claimedRunner.launchKind,\n runnerInstanceId: claimedRunner.runnerInstanceId,\n };\n }\n\n if (params.maxClaims !== null) {\n await tx\n .update(runnerSessions)\n .set({claimsUsed: sql`${runnerSessions.claimsUsed} + 1`, updatedAt: sql`now()`})\n .where(eq(runnerSessions.id, params.runnerSessionId));\n }\n\n // The running-row insert is the runner claiming the job execution. Emit in the same tx; the\n // payload carries the row's own claim instant so a consumer records the true time,\n // not the outbox drain time.\n await writeOutboxEvent<RunnersEventMap>(tx, runnersOutbox, {\n type: RUNNER_JOB_CLAIMED,\n payload: {\n workflowRunId: row.workflowRunId,\n workflowRunAttemptId: row.workflowRunAttemptId,\n jobId: row.jobId,\n jobExecutionId: row.jobExecutionId,\n claimedAt: claimed.claimedAt.toISOString(),\n },\n });\n\n return {\n workflowRunId: row.workflowRunId,\n workflowRunAttemptId: row.workflowRunAttemptId,\n jobId: row.jobId,\n jobExecutionId: row.jobExecutionId,\n projectId: row.projectId,\n };\n });\n if (queueTimeObservation) recordJobExecutionQueueTime(queueTimeObservation);\n if (activationToFirstClaimObservation)\n recordProviderRunnerActivationToFirstClaim(activationToFirstClaimObservation);\n return result;\n}\n\nasync function touchRunnerSessionLiveness(params: {\n workspaceId: string;\n runnerSessionId: string;\n throttleSeconds: number;\n}): Promise<void> {\n await db()\n .update(runnerSessions)\n .set({updatedAt: sql`now()`})\n .where(\n and(\n eq(runnerSessions.id, params.runnerSessionId),\n eq(runnerSessions.workspaceId, params.workspaceId),\n sql`${runnerSessions.updatedAt} < now() - (${params.throttleSeconds} \|\| ' seconds')::interval`,\n ),\n );\n}\n\n/*\n Reaps stale leases (bounded by `limit`), emitting one\n * `runners.job.lease_expired` event per reaped job execution.\n \n The cutoff is re-checked in the DELETE, not just the locking subquery, so a\n * heartbeat landing mid-call spares the live row. Each reaped execution also sweeps its\n * pending row so an at-least-once queue replay cannot leave an orphan.\n \n Locks pending-then-running to match `claimPendingJobExecution` and\n * `reconcileTerminalJobExecution`. The stale candidate scan intentionally does not lock\n * running rows first; the running-row DELETE re-checks the stale predicate after the\n * pending-row sweep.\n /\nexport async function expireStuckJobExecutions(params: {\n thresholdSeconds: number;\n noFirstHeartbeatGraceSeconds: number;\n limit?: number;\n}): Promise<\n Array<{\n workflowRunId: string;\n workflowRunAttemptId: string;\n jobId: string;\n jobExecutionId: string;\n }>\n> {\n const reaped = await db().transaction(async (tx) => {\n const heartbeatCutoff = sql`now() - (${params.thresholdSeconds} \|\| ' seconds')::interval`;\n const firstHeartbeatCutoff = sql`now() - (${params.noFirstHeartbeatGraceSeconds} \|\| ' seconds')::interval`;\n const stalePredicate = or(\n and(\n isNull(runningJobExecutions.firstHeartbeatAt),\n lte(runningJobExecutions.lastHeartbeatAt, runningJobExecutions.startedAt),\n lt(runningJobExecutions.startedAt, firstHeartbeatCutoff),\n ),\n and(\n or(\n isNotNull(runningJobExecutions.firstHeartbeatAt),\n gt(runningJobExecutions.lastHeartbeatAt, runningJobExecutions.startedAt),\n ),\n lt(runningJobExecutions.lastHeartbeatAt, heartbeatCutoff),\n ),\n );\n\n const staleRows = await tx\n .select({\n id: runningJobExecutions.id,\n jobExecutionId: runningJobExecutions.jobExecutionId,\n })\n .from(runningJobExecutions)\n .where(\n and(\n stalePredicate,\n sql`\n pg_try_advisory_xact_lock(\n hashtext(${runnerJobExecutionLockPrefix} \|\| ${runningJobExecutions.jobExecutionId}::text)\n )\n `,\n ),\n )\n .orderBy(\n asc(\n sql`CASE WHEN ${runningJobExecutions.firstHeartbeatAt} IS NULL AND ${runningJobExecutions.lastHeartbeatAt} <= ${runningJobExecutions.startedAt} THEN ${runningJobExecutions.startedAt} ELSE ${runningJobExecutions.lastHeartbeatAt} END`,\n ),\n asc(runningJobExecutions.id),\n )\n .limit(params.limit ?? 100);\n\n if (staleRows.length === 0) return [];\n\n const staleIds = staleRows.map((row) => row.id);\n const staleJobExecutionIds = staleRows.map((row) => row.jobExecutionId);\n\n // Sweep pending rows first so this transaction cannot hold a running-row lock while waiting\n // for a pending-row lock held by reconciliation or a claim of an orphan pending row. The\n // advisory lock acquired in the candidate scan also makes enqueue retries wait until this\n // transaction has either reaped the execution or released its lock without selecting it.\n await tx\n .delete(pendingJobExecutions)\n .where(inArray(pendingJobExecutions.jobExecutionId, staleJobExecutionIds));\n\n const deleted = await tx\n .delete(runningJobExecutions)\n .where(and(inArray(runningJobExecutions.id, staleIds), stalePredicate))\n .returning({\n workflowRunId: runningJobExecutions.workflowRunId,\n workflowRunAttemptId: runningJobExecutions.workflowRunAttemptId,\n jobId: runningJobExecutions.jobId,\n jobExecutionId: runningJobExecutions.jobExecutionId,\n provisionerId: runningJobExecutions.provisionerId,\n providerRunnerId: runningJobExecutions.providerRunnerId,\n });\n\n if (deleted.length === 0) return [];\n\n await releaseReservationsForTerminalRunningRows(tx, deleted);\n\n await writeOutboxEvents<RunnersEventMap>(\n tx,\n runnersOutbox,\n deleted.map((row) => ({\n type: RUNNER_JOB_LEASE_EXPIRED,\n payload: {\n workflowRunId: row.workflowRunId,\n workflowRunAttemptId: row.workflowRunAttemptId,\n jobId: row.jobId,\n jobExecutionId: row.jobExecutionId,\n },\n })),\n );\n\n return deleted.map(({workflowRunId, workflowRunAttemptId, jobId, jobExecutionId}) => ({\n workflowRunId,\n workflowRunAttemptId,\n jobId,\n jobExecutionId,\n }));\n });\n\n if (reaped.length > 0) jobExecutionLeaseExpiredCount.add(reaped.length);\n\n return reaped;\n}\n\nexport async function getJobExecutionQueueDepth(): Promise<{\n pendingJobExecutions: number;\n runningJobExecutions: number;\n}> {\n const [pending] = await db().select({value: count()}).from(pendingJobExecutions);\n const [running] = await db().select({value: count()}).from(runningJobExecutions);\n return {\n pendingJobExecutions: pending?.value ?? 0,\n runningJobExecutions: running?.value ?? 0,\n };\n}\n\nexport async function listActiveRunningJobExecutions(params: {\n workspaceId: string;\n windowSeconds: number;\n limit?: number;\n}): Promise<ActiveRunningJobExecution[]> {\n return await db()\n .select({\n jobId: runningJobExecutions.jobId,\n jobExecutionId: runningJobExecutions.jobExecutionId,\n workflowRunId: runningJobExecutions.workflowRunId,\n workflowRunAttemptId: runningJobExecutions.workflowRunAttemptId,\n projectId: runningJobExecutions.projectId,\n runnerSessionId: runningJobExecutions.runnerSessionId,\n provisionerId: runningJobExecutions.provisionerId,\n providerRunnerId: runningJobExecutions.providerRunnerId,\n requiredLabels: runningJobExecutions.requiredLabels,\n runnerLabels: runningJobExecutions.runnerLabels,\n startedAt: runningJobExecutions.startedAt,\n lastHeartbeatAt: runningJobExecutions.lastHeartbeatAt,\n })\n .from(runningJobExecutions)\n .where(\n and(\n eq(runningJobExecutions.workspaceId, params.workspaceId),\n sql`${runningJobExecutions.lastHeartbeatAt} > now() - (${params.windowSeconds} \|\| ' seconds')::interval`,\n ),\n )\n .orderBy(desc(runningJobExecutions.lastHeartbeatAt), desc(runningJobExecutions.id))\n .limit(params.limit ?? 1000);\n}\n\nexport async function listRunningJobExecutionsByRunnerInstanceTx(\n tx: Tx,\n params: {\n workspaceId: string;\n provisionerId: string;\n providerRunnerIds: string[];\n },\n): Promise<RunnerInstanceBoundJobExecution[]> {\n if (params.providerRunnerIds.length === 0) return [];\n\n const duplicateRows = await tx\n .select({\n providerRunnerId: runningJobExecutions.providerRunnerId,\n count: count(),\n })\n .from(runningJobExecutions)\n .where(\n and(\n eq(runningJobExecutions.workspaceId, params.workspaceId),\n eq(runningJobExecutions.provisionerId, params.provisionerId),\n inArray(runningJobExecutions.providerRunnerId, params.providerRunnerIds),\n ),\n )\n .groupBy(runningJobExecutions.providerRunnerId)\n .having(sql`count() > 1`);\n\n const duplicateRunnerInstanceIds = duplicateRows.flatMap((row) =>\n row.providerRunnerId ? [row.providerRunnerId] : [],\n );\n if (duplicateRunnerInstanceIds.length > 0) {\n logger().warn(\n {\n workspaceId: params.workspaceId,\n provisionerId: params.provisionerId,\n providerRunnerIds: duplicateRunnerInstanceIds,\n },\n 'multiple running job executions are bound to the same provisioned runner',\n );\n }\n\n const result = await tx.execute<{\n workflowRunId: string;\n workflowRunAttemptId: string;\n jobId: string;\n jobExecutionId: string;\n providerRunnerId: string;\n startedAt: Date \| string;\n lastHeartbeatAt: Date \| string;\n cancellationRequestedAt: Date \| string \| null;\n }>(sql`\n SELECT DISTINCT ON (${runningJobExecutions.providerRunnerId})\n ${runningJobExecutions.workflowRunId} AS \"workflowRunId\",\n ${runningJobExecutions.workflowRunAttemptId} AS \"workflowRunAttemptId\",\n ${runningJobExecutions.jobId} AS \"jobId\",\n ${runningJobExecutions.jobExecutionId} AS \"jobExecutionId\",\n ${runningJobExecutions.providerRunnerId} AS \"providerRunnerId\",\n ${runningJobExecutions.startedAt} AS \"startedAt\",\n ${runningJobExecutions.lastHeartbeatAt} AS \"lastHeartbeatAt\",\n ${runningJobExecutions.cancellationRequestedAt} AS \"cancellationRequestedAt\"\n FROM ${runningJobExecutions}\n WHERE\n ${runningJobExecutions.workspaceId} = ${params.workspaceId}\n AND ${runningJobExecutions.provisionerId} = ${params.provisionerId}\n AND ${runningJobExecutions.providerRunnerId} IN (${sql.join(\n params.providerRunnerIds.map((providerRunnerId) => sql`${providerRunnerId}`),\n sql`, `,\n )})\n ORDER BY ${runningJobExecutions.providerRunnerId}, ${runningJobExecutions.startedAt} DESC, ${runningJobExecutions.jobExecutionId} DESC\n `);\n\n return result.rows.map((row) => ({\n workflowRunId: row.workflowRunId,\n workflowRunAttemptId: row.workflowRunAttemptId,\n jobId: row.jobId,\n jobExecutionId: row.jobExecutionId,\n providerRunnerId: row.providerRunnerId,\n startedAt: toDate(row.startedAt),\n lastHeartbeatAt: toDate(row.lastHeartbeatAt),\n cancellationRequestedAt: row.cancellationRequestedAt\n ? toDate(row.cancellationRequestedAt)\n : null,\n }));\n}\n\nfunction toDate(value: Date \| string): Date {\n return value instanceof Date ? value : new Date(value);\n}\n\nexport async function isJobLeaseActive(params: {\n jobId?: string;\n jobExecutionId: string;\n runnerSessionId: string;\n}): Promise<boolean> {\n const [row] = await db()\n .select({id: runningJobExecutions.id})\n .from(runningJobExecutions)\n .where(\n and(\n eq(runningJobExecutions.jobExecutionId, params.jobExecutionId),\n eq(runningJobExecutions.runnerSessionId, params.runnerSessionId),\n params.jobId === undefined ? undefined : eq(runningJobExecutions.jobId, params.jobId),\n ),\n )\n .limit(1);\n\n return row !== undefined;\n}\n\nexport async function recordHeartbeat(params: {\n jobExecutionId: string;\n runnerSessionId: string;\n toolCapabilities?: RunnerToolCapabilitiesDto \| null;\n}): Promise<{\n cancellationRequested: boolean;\n previousToolCapabilities: RunnerToolCapabilitiesDto \| null;\n currentToolCapabilities: RunnerToolCapabilitiesDto \| null;\n runningJobExecution: {\n workflowRunId: string;\n workflowRunAttemptId: string;\n jobId: string;\n jobExecutionId: string;\n projectId: string;\n workspaceId: string;\n runnerSessionId: string;\n };\n}> {\n const result = await db().transaction(async (tx) => {\n const updated = await tx\n .update(runningJobExecutions)\n .set({\n firstHeartbeatAt: sql`COALESCE(${runningJobExecutions.firstHeartbeatAt}, now())`,\n lastHeartbeatAt: sql`now()`,\n })\n .where(\n and(\n eq(runningJobExecutions.jobExecutionId, params.jobExecutionId),\n eq(runningJobExecutions.runnerSessionId, params.runnerSessionId),\n ),\n )\n .returning({\n cancellationRequestedAt: runningJobExecutions.cancellationRequestedAt,\n workflowRunId: runningJobExecutions.workflowRunId,\n workflowRunAttemptId: runningJobExecutions.workflowRunAttemptId,\n jobId: runningJobExecutions.jobId,\n jobExecutionId: runningJobExecutions.jobExecutionId,\n projectId: runningJobExecutions.projectId,\n workspaceId: runningJobExecutions.workspaceId,\n runnerSessionId: runningJobExecutions.runnerSessionId,\n });\n\n const row = updated[0];\n if (!row) throw new RunningJobExecutionNotFoundError(params.jobExecutionId);\n\n const [previous] = await tx\n .select({toolCapabilities: runnerSessions.toolCapabilities})\n .from(runnerSessions)\n .where(eq(runnerSessions.id, params.runnerSessionId))\n .limit(1)\n .for('update');\n\n const [session] = await tx\n .update(runnerSessions)\n .set({\n toolCapabilities: params.toolCapabilities ?? null,\n toolCapabilitiesReportedAt: params.toolCapabilities ? sql`now()` : null,\n updatedAt: sql`now()`,\n })\n .where(eq(runnerSessions.id, params.runnerSessionId))\n .returning({toolCapabilities: runnerSessions.toolCapabilities});\n\n return {\n row,\n previousToolCapabilities: previous?.toolCapabilities ?? null,\n currentToolCapabilities: session?.toolCapabilities ?? null,\n };\n });\n\n const row = result.row;\n if (!row) throw new RunningJobExecutionNotFoundError(params.jobExecutionId);\n return {\n cancellationRequested: row.cancellationRequestedAt !== null,\n previousToolCapabilities: result.previousToolCapabilities,\n currentToolCapabilities: result.currentToolCapabilities,\n runningJobExecution: {\n workflowRunId: row.workflowRunId,\n workflowRunAttemptId: row.workflowRunAttemptId,\n jobId: row.jobId,\n jobExecutionId: row.jobExecutionId,\n projectId: row.projectId,\n workspaceId: row.workspaceId,\n runnerSessionId: row.runnerSessionId,\n },\n };\n}\n\n/*\n Reconciles a terminal job execution with runner state in one transaction: removes its pending\n * queue row, requests cancellation on its running lease, and converges any linked reservation.\n * The operation is idempotent and preserves the first cancellation-request timestamp.\n */\nexport async function reconcileTerminalJobExecution(params: {\n jobExecutionId: string;\n}): Promise<void> {\n await db().transaction(async (tx) => {\n await lockJobExecution(tx, params.jobExecutionId);\n\n // Delete pending before updating running to match claim/release lock order. Claim locks\n // pending rows with SKIP LOCKED before inserting the running lease, so this ordering makes\n // a concurrent terminal reconciliation either win before claim or cancel the new lease.\n await tx\n .delete(pendingJobExecutions)\n .where(eq(pendingJobExecutions.jobExecutionId, params.jobExecutionId));\n const cancelledRunningRows = await tx\n .update(runningJobExecutions)\n .set({\n cancellationRequestedAt: sql`COALESCE(${runningJobExecutions.cancellationRequestedAt}, now())`,\n })\n .where(eq(runningJobExecutions.jobExecutionId, params.jobExecutionId))\n .returning({\n provisionerId: runningJobExecutions.provisionerId,\n providerRunnerId: runningJobExecutions.providerRunnerId,\n });\n\n await releaseReservationsForTerminalRunningRows(tx, cancelledRunningRows);\n });\n}\n"],"names":["RUNNER_JOB_CLAIMED","RUNNER_JOB_LEASE_EXPIRED","logger","writeOutboxEvent","writeOutboxEvents","canonicalizeLabels","and","arrayContained","asc","count","desc","eq","gt","inArray","isNotNull","isNull","lt","lte","or","sql","EmptyRequiredLabelsError","RunnerSessionExhaustedError","RunningJobExecutionNotFoundError","jobExecutionEnqueuedCount","jobExecutionLeaseExpiredCount","recordJobExecutionQueueTime","recordProviderRunnerActivationToFirstClaim","db","lockRunnerReservationAdvisoryKeysTx","releaseReservationUnits","releaseTerminalRunnerInstanceReservationsByIds","runnersOutbox","pendingJobExecutions","reservations","providerRunners","runnerSessions","runningJobExecutions","runnerJobExecutionLockPrefix","lockJobExecution","tx","jobExecutionId","execute","releaseReservationsForTerminalRunningRows","rows","providerRunnerIdsByProvisionerId","Map","row","provisionerId","providerRunnerId","providerRunnerIds","get","Set","add","set","keys","sort","workspaceId","requireUnlinkedSession","requireTerminalState","getWorkspaceJobCounts","params","pendingRows","runningRows","Promise","all","select","from","where","workspaceIds","groupBy","queuedByWorkspace","map","Number","runningByWorkspace","queued","running","enqueueJobExecution","requiredLabels","length","enqueued","transaction","limit","leaseExpired","id","eventType","payload","inserted","insert","values","workflowRunId","workflowRunAttemptId","jobId","projectId","createdAt","queuedAt","onConflictDoNothing","target","returning","undefined","claimPendingJobExecution","touchRunnerSessionLiveness","runnerSessionId","throttleSeconds","runnerSessionLivenessThrottleSeconds","sessionLabels","activationToFirstClaimObservation","queueTimeObservation","result","runnerInstanceId","maxClaims","session","claimsUsed","revokedAt","for","runnerInstanceCondition","runner","reservationId","intendedReservationId","reservationIds","filter","candidate","orderBy","skipLocked","as","delete","runnerLabels","claimedAt","startedAt","claimed","durationMilliseconds","getTime","provider","launchKind","claimedRunner","update","firstClaimedAt","updatedAt","isFirstClaim","providerKind","sessionCreatedAtEpochMs","releasedRunner","reservationReleasedAt","reservation","releases","type","toISOString","expireStuckJobExecutions","reaped","heartbeatCutoff","thresholdSeconds","firstHeartbeatCutoff","noFirstHeartbeatGraceSeconds","stalePredicate","firstHeartbeatAt","lastHeartbeatAt","staleRows","staleIds","staleJobExecutionIds","deleted","getJobExecutionQueueDepth","pending","value","listActiveRunningJobExecutions","windowSeconds","listRunningJobExecutionsByRunnerInstanceTx","duplicateRows","having","duplicateRunnerInstanceIds","flatMap","warn","cancellationRequestedAt","join","toDate","Date","isJobLeaseActive","recordHeartbeat","updated","previous","toolCapabilities","toolCapabilitiesReportedAt","previousToolCapabilities","currentToolCapabilities","cancellationRequested","runningJobExecution","reconcileTerminalJobExecution","cancelledRunningRows"],"mappings":"AAAA,SACEA,kBAAkB,EAClBC,wBAAwB,QAGnB,2BAA2B;AAClC,SAAQC,MAAM,QAAO,8BAA8B;AACnD,SAAQC,gBAAgB,EAAEC,iBAAiB,QAAO,uBAAuB;AACzE,SAAQC,kBAAkB,QAAO,yBAAyB;AAC1D,SACEC,GAAG,EACHC,cAAc,EACdC,GAAG,EACHC,KAAK,EACLC,IAAI,EACJC,EAAE,EACFC,EAAE,EACFC,OAAO,EACPC,SAAS,EACTC,MAAM,EACNC,EAAE,EACFC,GAAG,EACHC,EAAE,EACFC,GAAG,QACE,cAAc;AACrB,SACEC,wBAAwB,EACxBC,2BAA2B,EAC3BC,gCAAgC,QAC3B,kBAAkB;AACzB,SAEEC,yBAAyB,EACzBC,6BAA6B,EAE7BC,2BAA2B,EAC3BC,0CAA0C,QACrC,uBAAuB;AAE9B,SAAQC,EAAE,QAAO,UAAU;AAC3B,SAAQC,mCAAmC,QAAO,yBAAyB;AAC3E,SACEC,uBAAuB,EACvBC,8CAA8C,QACzC,oBAAoB;AAC3B,SAAQC,aAAa,QAAO,qBAAqB;AACjD,SAAQC,oBAAoB,QAAO,qCAAqC;AACxE,SAAQC,YAAY,QAAO,2BAA2B;AACtD,SAAQC,eAAe,QAAO,+BAA+B;AAC7D,SAAQC,cAAc,QAAO,8BAA8B;AAC3D,SAAQC,oBAAoB,QAAO,qCAAqC;AAExE,MAAMC,+BAA+B;AAErC,eAAeC,iBAAiBC,EAAM,EAAEC,cAAsB;IAC5D,MAAMD,GAAGE,OAAO,CACdtB,GAAG,CAAC,sCAAsC,EAAE,GAAGkB,+BAA+BG,gBAAgB,CAAC,EAAE,CAAC;AAEtG;AAEA,eAAeE,0CACbH,EAAM,EACNI,IAGE;IAEF,2FAA2F;IAC3F,kFAAkF;IAClF,MAAMC,mCAAmC,IAAIC;IAC7C,KAAK,MAAMC,OAAOH,KAAM;QACtB,IAAIG,IAAIC,aAAa,KAAK,QAAQD,IAAIE,gBAAgB,KAAK,MAAM;QACjE,MAAMC,oBACJL,iCAAiCM,GAAG,CAACJ,IAAIC,aAAa,KAAK,IAAII;QACjEF,kBAAkBG,GAAG,CAACN,IAAIE,gBAAgB;QAC1CJ,iCAAiCS,GAAG,CAACP,IAAIC,aAAa,EAAEE;IAC1D;IAEA,KAAK,MAAMF,iBAAiB;WAAIH,iCAAiCU,IAAI;KAAG,CAACC,IAAI,GAAI;QAC/E,MAAMN,oBAAoBL,iCAAiCM,GAAG,CAACH;QAC/D,IAAI,CAACE,mBAAmB;QACxB,MAAMnB,+CAA+CS,IAAI;YACvDiB,aAAa;YACbT;YACAE,mBAAmB;mBAAIA;aAAkB,CAACM,IAAI;YAC9CE,wBAAwB;YACxB,mFAAmF;YACnF,wCAAwC;YACxCC,sBAAsB;QACxB;IACF;AACF;AAaA,OAAO,eAAeC,sBAAsBC,MAE3C;IACC,MAAM,CAACC,aAAaC,YAAY,GAAG,MAAMC,QAAQC,GAAG,CAAC;QACnDrC,KACGsC,MAAM,CAAC;YAACT,aAAaxB,qBAAqBwB,WAAW;YAAE/C,OAAOA;QAAO,GACrEyD,IAAI,CAAClC,sBACLmC,KAAK,CAACtD,QAAQmB,qBAAqBwB,WAAW,EAAEI,OAAOQ,YAAY,GACnEC,OAAO,CAACrC,qBAAqBwB,WAAW;QAC3C7B,KACGsC,MAAM,CAAC;YAACT,aAAapB,qBAAqBoB,WAAW;YAAE/C,OAAOA;QAAO,GACrEyD,IAAI,CAAC9B,sBACL+B,KAAK,CAACtD,QAAQuB,qBAAqBoB,WAAW,EAAEI,OAAOQ,YAAY,GACnEC,OAAO,CAACjC,qBAAqBoB,WAAW;KAC5C;IAED,MAAMc,oBAAoB,IAAIzB,IAAIgB,YAAYU,GAAG,CAAC,CAACzB,MAAQ;YAACA,IAAIU,WAAW;YAAEgB,OAAO1B,IAAIrC,KAAK;SAAE;IAC/F,MAAMgE,qBAAqB,IAAI5B,IAC7BiB,YAAYS,GAAG,CAAC,CAACzB,MAAQ;YAACA,IAAIU,WAAW;YAAEgB,OAAO1B,IAAIrC,KAAK;SAAE;IAG/D,OAAOmD,OAAOQ,YAAY,CAACG,GAAG,CAAC,CAACf,cAAiB,CAAA;YAC/CA;YACAkB,QAAQJ,kBAAkBpB,GAAG,CAACM,gBAAgB;YAC9CmB,SAASF,mBAAmBvB,GAAG,CAACM,gBAAgB;QAClD,CAAA;AACF;AAEA,+FAA+F;AAC/F,kGAAkG;AAClG,8FAA8F;AAC9F,OAAO,eAAeoB,oBAAoBhB,MAAiC;IACzE,MAAMiB,iBAAiB;WAAIxE,mBAAmBuD,OAAOiB,cAAc;KAAE;IACrE,IAAIA,eAAeC,MAAM,KAAK,GAAG,MAAM,IAAI1D;IAE3C,MAAM2D,WAAW,MAAMpD,KAAKqD,WAAW,CAAC,OAAOzC;QAC7C,MAAMD,iBAAiBC,IAAIqB,OAAOpB,cAAc;QAEhD,MAAM,CAACmC,QAAQ,GAAG,MAAMpC,GACrB0B,MAAM,CAAC;YAACzB,gBAAgBJ,qBAAqBI,cAAc;QAAA,GAC3D0B,IAAI,CAAC9B,sBACL+B,KAAK,CAACxD,GAAGyB,qBAAqBI,cAAc,EAAEoB,OAAOpB,cAAc,GACnEyC,KAAK,CAAC;QACT,IAAIN,SAAS,OAAO;QAEpB,MAAM,CAACO,aAAa,GAAG,MAAM3C,GAC1B0B,MAAM,CAAC;YAACkB,IAAIpD,cAAcoD,EAAE;QAAA,GAC5BjB,IAAI,CAACnC,eACLoC,KAAK,CACJ7D,IACEK,GAAGoB,cAAcqD,SAAS,EAAEnF,2BAC5BkB,GAAG,CAAC,EAAEY,cAAcsD,OAAO,CAAC,sBAAsB,EAAEzB,OAAOpB,cAAc,CAAC,CAAC,GAG9EyC,KAAK,CAAC;QACT,IAAIC,cAAc,OAAO;QAEzB,MAAM,CAACI,SAAS,GAAG,MAAM/C,GACtBgD,MAAM,CAACvD,sBACPwD,MAAM,CAAC;YACNhC,aAAaI,OAAOJ,WAAW;YAC/BiC,eAAe7B,OAAO6B,aAAa;YACnCC,sBAAsB9B,OAAO8B,oBAAoB;YACjDC,OAAO/B,OAAO+B,KAAK;YACnBnD,gBAAgBoB,OAAOpB,cAAc;YACrCoD,WAAWhC,OAAOgC,SAAS;YAC3Bf;YACAgB,WAAWjC,OAAOkC,QAAQ;QAC5B,GACCC,mBAAmB,CAAC;YAACC,QAAQhE,qBAAqBQ,cAAc;QAAA,GAChEyD,SAAS,CAAC;YAACJ,WAAW7D,qBAAqB6D,SAAS;QAAA;QAEvD,OAAOP,aAAaY;IACtB;IAEA,IAAInB,UAAUxD,0BAA0B6B,GAAG,CAAC;AAC9C;AAoCA,OAAO,eAAe+C,yBAAyBvC,MAM9C;IACC,MAAMwC,2BAA2B;QAC/B5C,aAAaI,OAAOJ,WAAW;QAC/B6C,iBAAiBzC,OAAOyC,eAAe;QACvCC,iBAAiB1C,OAAO2C,oCAAoC;IAC9D;IAEA,IAAI3C,OAAO4C,aAAa,CAAC1B,MAAM,KAAK,GAAG,OAAO;IAE9C,IAAI2B,oCAA+E;IACnF,IAAIC,uBAAgE;IACpE,MAAMC,SAAS,MAAMhF,KAAKqD,WAAW,CAAC,OAAOzC;QAC3C,IAAIQ,gBAA+B;QACnC,IAAIC,mBAAkC;QACtC,IAAI4D,mBAAkC;QAEtC,IAAIhD,OAAOiD,SAAS,KAAK,MAAM;YAC7B,MAAM,CAACC,QAAQ,GAAG,MAAMvE,GACrB0B,MAAM,CAAC;gBACN4C,WAAW1E,eAAe0E,SAAS;gBACnCE,YAAY5E,eAAe4E,UAAU;gBACrCC,WAAW7E,eAAe6E,SAAS;gBACnCJ,kBAAkBzE,eAAeyE,gBAAgB;gBACjD7D,eAAeZ,eAAeY,aAAa;gBAC3CC,kBAAkBb,eAAea,gBAAgB;YACnD,GACCkB,IAAI,CAAC/B,gBACLgC,KAAK,CAACxD,GAAGwB,eAAegD,EAAE,EAAEvB,OAAOyC,eAAe,GAClDpB,KAAK,CAAC,GACNgC,GAAG,CAAC;YAEP,IACE,CAACH,WACDA,QAAQE,SAAS,IACjBF,QAAQD,SAAS,KAAK,QACtBC,QAAQC,UAAU,IAAID,QAAQD,SAAS,EACvC;gBACA,MAAM,IAAIxF,4BAA4BuC,OAAOyC,eAAe;YAC9D;YAEA,0EAA0E;YAC1E,mDAAmD;YACnDO,mBAAmBE,QAAQF,gBAAgB;YAC3C7D,gBAAgB+D,QAAQ/D,aAAa;YACrCC,mBAAmB8D,QAAQ9D,gBAAgB;QAC7C;QAEA,MAAMkE,0BAA0BN,mBAC5BjG,GAAGuB,gBAAgBiD,EAAE,EAAEyB,oBACvB7D,iBAAiBC,mBACf1C,IACEK,GAAGuB,gBAAgBa,aAAa,EAAEA,gBAClCpC,GAAGuB,gBAAgBc,gBAAgB,EAAEA,qBAEvC;QACN,IAAIkE,2BAA2BnE,eAAe;YAC5C,MAAM,CAACoE,OAAO,GAAG,MAAM5E,GACpB0B,MAAM,CAAC;gBACNmD,eAAelF,gBAAgBkF,aAAa;gBAC5CC,uBAAuBnF,gBAAgBmF,qBAAqB;YAC9D,GACCnD,IAAI,CAAChC,iBACLiC,KAAK,CAAC+C,yBACNjC,KAAK,CAAC;YACT,MAAMrD,oCAAoCW,IAAI;gBAC5CQ;gBACAuE,gBAAgB;oBAACH,QAAQC;oBAAeD,QAAQE;iBAAsB,CAACE,MAAM,CAC3E,CAACH,gBACCA,kBAAkB,QAAQA,kBAAkBlB;YAElD;QACF;QAEA,4EAA4E;QAC5E,oFAAoF;QACpF,oFAAoF;QACpF,sFAAsF;QACtF,MAAMsB,YAAYjF,GACf0B,MAAM,GACNC,IAAI,CAAClC,sBACLmC,KAAK,CACJ7D,IACEK,GAAGqB,qBAAqBwB,WAAW,EAAEI,OAAOJ,WAAW,GACvDjD,eAAeyB,qBAAqB6C,cAAc,EAAEjB,OAAO4C,aAAa,IAG3EiB,OAAO,CAACjH,IAAIwB,qBAAqB6D,SAAS,GAAGrF,IAAIwB,qBAAqBmD,EAAE,GACxEF,KAAK,CAAC,GACNgC,GAAG,CAAC,UAAU;YAACS,YAAY;QAAI,GAC/BC,EAAE,CAAC;QAEN,MAAM,CAAC7E,IAAI,GAAG,MAAMP,GACjB0B,MAAM,GACNC,IAAI,CAACsD,WACLrD,KAAK,CACJhD,GAAG,CAAC;;qBAES,EAAEkB,6BAA6B,IAAI,EAAEmF,UAAUhF,cAAc,CAAC;;QAE3E,CAAC;QAGL,IAAI,CAACM,KAAK,OAAO;QAEjB,MAAMP,GAAGqF,MAAM,CAAC5F,sBAAsBmC,KAAK,CAACxD,GAAGqB,qBAAqBmD,EAAE,EAAErC,IAAIqC,EAAE;QAE9E,8EAA8E;QAC9E,gFAAgF;QAChF,qFAAqF;QACrF,2EAA2E;QAC3E,8EAA8E;QAC9E,2DAA2D;QAC3D,MAAMG,WAAW,MAAM/C,GACpBgD,MAAM,CAACnD,sBACPoD,MAAM,CAAC;YACNhC,aAAaV,IAAIU,WAAW;YAC5BiC,eAAe3C,IAAI2C,aAAa;YAChCC,sBAAsB5C,IAAI4C,oBAAoB;YAC9CC,OAAO7C,IAAI6C,KAAK;YAChBnD,gBAAgBM,IAAIN,cAAc;YAClCoD,WAAW9C,IAAI8C,SAAS;YACxBS,iBAAiBzC,OAAOyC,eAAe;YACvCtD;YACAC;YACA6B,gBAAgB/B,IAAI+B,cAAc;YAClCgD,cAAcjE,OAAO4C,aAAa;QACpC,GACCT,mBAAmB,CAAC;YAACC,QAAQ5D,qBAAqBI,cAAc;QAAA,GAChEyD,SAAS,CAAC;YAAC6B,WAAW1F,qBAAqB2F,SAAS;QAAA;QAEvD,MAAMC,UAAU1C,QAAQ,CAAC,EAAE;QAC3B,IAAI,CAAC0C,SAAS,OAAO;QAErBtB,uBAAuB;YACrBuB,sBAAsBD,QAAQF,SAAS,CAACI,OAAO,KAAKpF,IAAI+C,SAAS,CAACqC,OAAO;YACzEC,UAAU;YACVC,YAAYxE,OAAOiD,SAAS,KAAK,OAAO,WAAW;QACrD;QAEA,IAAIK,yBAAyB;YAC3B,MAAM,CAACmB,cAAc,GAAG,MAAM9F,GAC3B+F,MAAM,CAACpG,iBACPmB,GAAG,CAAC;gBACHkF,gBAAgBpH,GAAG,CAAC,SAAS,EAAEe,gBAAgBqG,cAAc,CAAC,EAAE,EAAEP,QAAQF,SAAS,CAAC,CAAC,CAAC;gBACtFU,WAAWrH,GAAG,CAAC,KAAK,CAAC;YACvB,GACCgD,KAAK,CAAC+C,yBACNjB,SAAS,CAAC;gBACTsC,gBAAgBrG,gBAAgBqG,cAAc;gBAC9CE,cAActH,GAAY,CAAC,EAAEe,gBAAgBqG,cAAc,CAAC,GAAG,EAAEP,QAAQF,SAAS,CAAC,CAAC;gBACpFK,UAAUjG,gBAAgBwG,YAAY;gBACtCN,YAAYlG,gBAAgBkG,UAAU;gBACtCxB,kBAAkB1E,gBAAgBiD,EAAE;gBACpCiC,eAAelF,gBAAgBkF,aAAa;gBAC5CC,uBAAuBnF,gBAAgBmF,qBAAqB;gBAC5DsB,yBAAyBxH,GAAkB,CAAC;sCAChB,EAAEgB,eAAe0D,SAAS,CAAC;iBAChD,EAAE1D,eAAe;kBAChB,EAAEA,eAAegD,EAAE,CAAC,GAAG,EAAEvB,OAAOyC,eAAe,CAAC;WACvD,CAAC;YACJ;YACF,IAAIgC,iBAAiB3B,sBAAsB;gBACzCA,qBAAqByB,QAAQ,GAAGE,cAAcF,QAAQ;gBACtDzB,qBAAqB0B,UAAU,GAAGC,cAAcD,UAAU;YAC5D;YACA,IAAIC,eAAeI,cAAc;gBAC/B,MAAM,CAACG,eAAe,GAAG,MAAMrG,GAC5B+F,MAAM,CAACpG,iBACPmB,GAAG,CAAC;oBACHgE,uBAAuB;oBACvBwB,uBAAuB1H,GAAG,CAAC,KAAK,CAAC;oBACjCqH,WAAWrH,GAAG,CAAC,KAAK,CAAC;gBACvB,GACCgD,KAAK,CACJ7D,IACEK,GAAGuB,gBAAgBiD,EAAE,EAAEkD,cAAczB,gBAAgB,GACrD1F,GACEJ,UAAUoB,gBAAgBkF,aAAa,GACvCtG,UAAUoB,gBAAgBmF,qBAAqB,IAEjDtG,OAAOmB,gBAAgB2G,qBAAqB,IAG/C5C,SAAS,CAAC;oBACTlD,eAAeb,gBAAgBa,aAAa;gBAC9C;gBACF,MAAMqE,gBAAgBiB,cAAchB,qBAAqB,IAAIgB,cAAcjB,aAAa;gBACxF,IAAIwB,gBAAgB7F,iBAAiBqE,eAAe;oBAClD,MAAM,CAAC0B,YAAY,GAAG,MAAMvG,GACzB0B,MAAM,CAAC;wBAACT,aAAavB,aAAauB,WAAW;oBAAA,GAC7CU,IAAI,CAACjC,cACLkC,KAAK,CACJ7D,IACEK,GAAGsB,aAAakD,EAAE,EAAEiC,gBACpBzG,GAAGsB,aAAac,aAAa,EAAE6F,eAAe7F,aAAa,IAG9DkC,KAAK,CAAC;oBACT,IAAI6D,aACF,MAAMjH,wBAAwBU,IAAI;wBAChCiB,aAAasF,YAAYtF,WAAW;wBACpCT,eAAe6F,eAAe7F,aAAa;wBAC3CgG,UAAU;4BAAC;gCAAC3B;gCAAe3G,OAAO;4BAAC;yBAAE;oBACvC;gBACJ;YACF;YACA,IACE4H,eAAeI,gBACfJ,cAAcE,cAAc,KAAK,QACjCF,cAAcM,uBAAuB,KAAK,QAC1CN,cAAcM,uBAAuB,KAAKzC,WAE1CO,oCAAoC;gBAClCwB,sBACEI,cAAcE,cAAc,CAACL,OAAO,KAAKG,cAAcM,uBAAuB;gBAChFR,UAAUE,cAAcF,QAAQ;gBAChCC,YAAYC,cAAcD,UAAU;gBACpCxB,kBAAkByB,cAAczB,gBAAgB;YAClD;QACJ;QAEA,IAAIhD,OAAOiD,SAAS,KAAK,MAAM;YAC7B,MAAMtE,GACH+F,MAAM,CAACnG,gBACPkB,GAAG,CAAC;gBAAC0D,YAAY5F,GAAG,CAAC,EAAEgB,eAAe4E,UAAU,CAAC,IAAI,CAAC;gBAAEyB,WAAWrH,GAAG,CAAC,KAAK,CAAC;YAAA,GAC7EgD,KAAK,CAACxD,GAAGwB,eAAegD,EAAE,EAAEvB,OAAOyC,eAAe;QACvD;QAEA,4FAA4F;QAC5F,mFAAmF;QACnF,6BAA6B;QAC7B,MAAMlG,iBAAkCoC,IAAIR,eAAe;YACzDiH,MAAMhJ;YACNqF,SAAS;gBACPI,eAAe3C,IAAI2C,aAAa;gBAChCC,sBAAsB5C,IAAI4C,oBAAoB;gBAC9CC,OAAO7C,IAAI6C,KAAK;gBAChBnD,gBAAgBM,IAAIN,cAAc;gBAClCsF,WAAWE,QAAQF,SAAS,CAACmB,WAAW;YAC1C;QACF;QAEA,OAAO;YACLxD,eAAe3C,IAAI2C,aAAa;YAChCC,sBAAsB5C,IAAI4C,oBAAoB;YAC9CC,OAAO7C,IAAI6C,KAAK;YAChBnD,gBAAgBM,IAAIN,cAAc;YAClCoD,WAAW9C,IAAI8C,SAAS;QAC1B;IACF;IACA,IAAIc,sBAAsBjF,4BAA4BiF;IACtD,IAAID,mCACF/E,2CAA2C+E;IAC7C,OAAOE;AACT;AAEA,eAAeP,2BAA2BxC,MAIzC;IACC,MAAMjC,KACH2G,MAAM,CAACnG,gBACPkB,GAAG,CAAC;QAACmF,WAAWrH,GAAG,CAAC,KAAK,CAAC;IAAA,GAC1BgD,KAAK,CACJ7D,IACEK,GAAGwB,eAAegD,EAAE,EAAEvB,OAAOyC,eAAe,GAC5C1F,GAAGwB,eAAeqB,WAAW,EAAEI,OAAOJ,WAAW,GACjDrC,GAAG,CAAC,EAAEgB,eAAeqG,SAAS,CAAC,YAAY,EAAE5E,OAAO0C,eAAe,CAAC,yBAAyB,CAAC;AAGtG;AAEA;;;;;;;;;;;;CAYC,GACD,OAAO,eAAe4C,yBAAyBtF,MAI9C;IAQC,MAAMuF,SAAS,MAAMxH,KAAKqD,WAAW,CAAC,OAAOzC;QAC3C,MAAM6G,kBAAkBjI,GAAG,CAAC,SAAS,EAAEyC,OAAOyF,gBAAgB,CAAC,yBAAyB,CAAC;QACzF,MAAMC,uBAAuBnI,GAAG,CAAC,SAAS,EAAEyC,OAAO2F,4BAA4B,CAAC,yBAAyB,CAAC;QAC1G,MAAMC,iBAAiBtI,GACrBZ,IACES,OAAOqB,qBAAqBqH,gBAAgB,GAC5CxI,IAAImB,qBAAqBsH,eAAe,EAAEtH,qBAAqB2F,SAAS,GACxE/G,GAAGoB,qBAAqB2F,SAAS,EAAEuB,wBAErChJ,IACEY,GACEJ,UAAUsB,qBAAqBqH,gBAAgB,GAC/C7I,GAAGwB,qBAAqBsH,eAAe,EAAEtH,qBAAqB2F,SAAS,IAEzE/G,GAAGoB,qBAAqBsH,eAAe,EAAEN;QAI7C,MAAMO,YAAY,MAAMpH,GACrB0B,MAAM,CAAC;YACNkB,IAAI/C,qBAAqB+C,EAAE;YAC3B3C,gBAAgBJ,qBAAqBI,cAAc;QACrD,GACC0B,IAAI,CAAC9B,sBACL+B,KAAK,CACJ7D,IACEkJ,gBACArI,GAAG,CAAC;;uBAES,EAAEkB,6BAA6B,IAAI,EAAED,qBAAqBI,cAAc,CAAC;;UAEtF,CAAC,GAGJiF,OAAO,CACNjH,IACEW,GAAG,CAAC,UAAU,EAAEiB,qBAAqBqH,gBAAgB,CAAC,aAAa,EAAErH,qBAAqBsH,eAAe,CAAC,IAAI,EAAEtH,qBAAqB2F,SAAS,CAAC,MAAM,EAAE3F,qBAAqB2F,SAAS,CAAC,MAAM,EAAE3F,qBAAqBsH,eAAe,CAAC,IAAI,CAAC,GAE1OlJ,IAAI4B,qBAAqB+C,EAAE,GAE5BF,KAAK,CAACrB,OAAOqB,KAAK,IAAI;QAEzB,IAAI0E,UAAU7E,MAAM,KAAK,GAAG,OAAO,EAAE;QAErC,MAAM8E,WAAWD,UAAUpF,GAAG,CAAC,CAACzB,MAAQA,IAAIqC,EAAE;QAC9C,MAAM0E,uBAAuBF,UAAUpF,GAAG,CAAC,CAACzB,MAAQA,IAAIN,cAAc;QAEtE,4FAA4F;QAC5F,yFAAyF;QACzF,0FAA0F;QAC1F,yFAAyF;QACzF,MAAMD,GACHqF,MAAM,CAAC5F,sBACPmC,KAAK,CAACtD,QAAQmB,qBAAqBQ,cAAc,EAAEqH;QAEtD,MAAMC,UAAU,MAAMvH,GACnBqF,MAAM,CAACxF,sBACP+B,KAAK,CAAC7D,IAAIO,QAAQuB,qBAAqB+C,EAAE,EAAEyE,WAAWJ,iBACtDvD,SAAS,CAAC;YACTR,eAAerD,qBAAqBqD,aAAa;YACjDC,sBAAsBtD,qBAAqBsD,oBAAoB;YAC/DC,OAAOvD,qBAAqBuD,KAAK;YACjCnD,gBAAgBJ,qBAAqBI,cAAc;YACnDO,eAAeX,qBAAqBW,aAAa;YACjDC,kBAAkBZ,qBAAqBY,gBAAgB;QACzD;QAEF,IAAI8G,QAAQhF,MAAM,KAAK,GAAG,OAAO,EAAE;QAEnC,MAAMpC,0CAA0CH,IAAIuH;QAEpD,MAAM1J,kBACJmC,IACAR,eACA+H,QAAQvF,GAAG,CAAC,CAACzB,MAAS,CAAA;gBACpBkG,MAAM/I;gBACNoF,SAAS;oBACPI,eAAe3C,IAAI2C,aAAa;oBAChCC,sBAAsB5C,IAAI4C,oBAAoB;oBAC9CC,OAAO7C,IAAI6C,KAAK;oBAChBnD,gBAAgBM,IAAIN,cAAc;gBACpC;YACF,CAAA;QAGF,OAAOsH,QAAQvF,GAAG,CAAC,CAAC,EAACkB,aAAa,EAAEC,oBAAoB,EAAEC,KAAK,EAAEnD,cAAc,EAAC,GAAM,CAAA;gBACpFiD;gBACAC;gBACAC;gBACAnD;YACF,CAAA;IACF;IAEA,IAAI2G,OAAOrE,MAAM,GAAG,GAAGtD,8BAA8B4B,GAAG,CAAC+F,OAAOrE,MAAM;IAEtE,OAAOqE;AACT;AAEA,OAAO,eAAeY;IAIpB,MAAM,CAACC,QAAQ,GAAG,MAAMrI,KAAKsC,MAAM,CAAC;QAACgG,OAAOxJ;IAAO,GAAGyD,IAAI,CAAClC;IAC3D,MAAM,CAAC2C,QAAQ,GAAG,MAAMhD,KAAKsC,MAAM,CAAC;QAACgG,OAAOxJ;IAAO,GAAGyD,IAAI,CAAC9B;IAC3D,OAAO;QACLJ,sBAAsBgI,SAASC,SAAS;QACxC7H,sBAAsBuC,SAASsF,SAAS;IAC1C;AACF;AAEA,OAAO,eAAeC,+BAA+BtG,MAIpD;IACC,OAAO,MAAMjC,KACVsC,MAAM,CAAC;QACN0B,OAAOvD,qBAAqBuD,KAAK;QACjCnD,gBAAgBJ,qBAAqBI,cAAc;QACnDiD,eAAerD,qBAAqBqD,aAAa;QACjDC,sBAAsBtD,qBAAqBsD,oBAAoB;QAC/DE,WAAWxD,qBAAqBwD,SAAS;QACzCS,iBAAiBjE,qBAAqBiE,eAAe;QACrDtD,eAAeX,qBAAqBW,aAAa;QACjDC,kBAAkBZ,qBAAqBY,gBAAgB;QACvD6B,gBAAgBzC,qBAAqByC,cAAc;QACnDgD,cAAczF,qBAAqByF,YAAY;QAC/CE,WAAW3F,qBAAqB2F,SAAS;QACzC2B,iBAAiBtH,qBAAqBsH,eAAe;IACvD,GACCxF,IAAI,CAAC9B,sBACL+B,KAAK,CACJ7D,IACEK,GAAGyB,qBAAqBoB,WAAW,EAAEI,OAAOJ,WAAW,GACvDrC,GAAG,CAAC,EAAEiB,qBAAqBsH,eAAe,CAAC,YAAY,EAAE9F,OAAOuG,aAAa,CAAC,yBAAyB,CAAC,GAG3G1C,OAAO,CAAC/G,KAAK0B,qBAAqBsH,eAAe,GAAGhJ,KAAK0B,qBAAqB+C,EAAE,GAChFF,KAAK,CAACrB,OAAOqB,KAAK,IAAI;AAC3B;AAEA,OAAO,eAAemF,2CACpB7H,EAAM,EACNqB,MAIC;IAED,IAAIA,OAAOX,iBAAiB,CAAC6B,MAAM,KAAK,GAAG,OAAO,EAAE;IAEpD,MAAMuF,gBAAgB,MAAM9H,GACzB0B,MAAM,CAAC;QACNjB,kBAAkBZ,qBAAqBY,gBAAgB;QACvDvC,OAAOA;IACT,GACCyD,IAAI,CAAC9B,sBACL+B,KAAK,CACJ7D,IACEK,GAAGyB,qBAAqBoB,WAAW,EAAEI,OAAOJ,WAAW,GACvD7C,GAAGyB,qBAAqBW,aAAa,EAAEa,OAAOb,aAAa,GAC3DlC,QAAQuB,qBAAqBY,gBAAgB,EAAEY,OAAOX,iBAAiB,IAG1EoB,OAAO,CAACjC,qBAAqBY,gBAAgB,EAC7CsH,MAAM,CAACnJ,GAAG,CAAC,YAAY,CAAC;IAE3B,MAAMoJ,6BAA6BF,cAAcG,OAAO,CAAC,CAAC1H,MACxDA,IAAIE,gBAAgB,GAAG;YAACF,IAAIE,gBAAgB;SAAC,GAAG,EAAE;IAEpD,IAAIuH,2BAA2BzF,MAAM,GAAG,GAAG;QACzC5E,SAASuK,IAAI,CACX;YACEjH,aAAaI,OAAOJ,WAAW;YAC/BT,eAAea,OAAOb,aAAa;YACnCE,mBAAmBsH;QACrB,GACA;IAEJ;IAEA,MAAM5D,SAAS,MAAMpE,GAAGE,OAAO,CAS5BtB,GAAG,CAAC;wBACe,EAAEiB,qBAAqBY,gBAAgB,CAAC;MAC1D,EAAEZ,qBAAqBqD,aAAa,CAAC;MACrC,EAAErD,qBAAqBsD,oBAAoB,CAAC;MAC5C,EAAEtD,qBAAqBuD,KAAK,CAAC;MAC7B,EAAEvD,qBAAqBI,cAAc,CAAC;MACtC,EAAEJ,qBAAqBY,gBAAgB,CAAC;MACxC,EAAEZ,qBAAqB2F,SAAS,CAAC;MACjC,EAAE3F,qBAAqBsH,eAAe,CAAC;MACvC,EAAEtH,qBAAqBsI,uBAAuB,CAAC;SAC5C,EAAEtI,qBAAqB;;MAE1B,EAAEA,qBAAqBoB,WAAW,CAAC,GAAG,EAAEI,OAAOJ,WAAW,CAAC;UACvD,EAAEpB,qBAAqBW,aAAa,CAAC,GAAG,EAAEa,OAAOb,aAAa,CAAC;UAC/D,EAAEX,qBAAqBY,gBAAgB,CAAC,KAAK,EAAE7B,IAAIwJ,IAAI,CACzD/G,OAAOX,iBAAiB,CAACsB,GAAG,CAAC,CAACvB,mBAAqB7B,GAAG,CAAC,EAAE6B,iBAAiB,CAAC,GAC3E7B,GAAG,CAAC,EAAE,CAAC,EACP;aACK,EAAEiB,qBAAqBY,gBAAgB,CAAC,EAAE,EAAEZ,qBAAqB2F,SAAS,CAAC,OAAO,EAAE3F,qBAAqBI,cAAc,CAAC;EACnI,CAAC;IAED,OAAOmE,OAAOhE,IAAI,CAAC4B,GAAG,CAAC,CAACzB,MAAS,CAAA;YAC/B2C,eAAe3C,IAAI2C,aAAa;YAChCC,sBAAsB5C,IAAI4C,oBAAoB;YAC9CC,OAAO7C,IAAI6C,KAAK;YAChBnD,gBAAgBM,IAAIN,cAAc;YAClCQ,kBAAkBF,IAAIE,gBAAgB;YACtC+E,WAAW6C,OAAO9H,IAAIiF,SAAS;YAC/B2B,iBAAiBkB,OAAO9H,IAAI4G,eAAe;YAC3CgB,yBAAyB5H,IAAI4H,uBAAuB,GAChDE,OAAO9H,IAAI4H,uBAAuB,IAClC;QACN,CAAA;AACF;AAEA,SAASE,OAAOX,KAAoB;IAClC,OAAOA,iBAAiBY,OAAOZ,QAAQ,IAAIY,KAAKZ;AAClD;AAEA,OAAO,eAAea,iBAAiBlH,MAItC;IACC,MAAM,CAACd,IAAI,GAAG,MAAMnB,KACjBsC,MAAM,CAAC;QAACkB,IAAI/C,qBAAqB+C,EAAE;IAAA,GACnCjB,IAAI,CAAC9B,sBACL+B,KAAK,CACJ7D,IACEK,GAAGyB,qBAAqBI,cAAc,EAAEoB,OAAOpB,cAAc,GAC7D7B,GAAGyB,qBAAqBiE,eAAe,EAAEzC,OAAOyC,eAAe,GAC/DzC,OAAO+B,KAAK,KAAKO,YAAYA,YAAYvF,GAAGyB,qBAAqBuD,KAAK,EAAE/B,OAAO+B,KAAK,IAGvFV,KAAK,CAAC;IAET,OAAOnC,QAAQoD;AACjB;AAEA,OAAO,eAAe6E,gBAAgBnH,MAIrC;IAcC,MAAM+C,SAAS,MAAMhF,KAAKqD,WAAW,CAAC,OAAOzC;QAC3C,MAAMyI,UAAU,MAAMzI,GACnB+F,MAAM,CAAClG,sBACPiB,GAAG,CAAC;YACHoG,kBAAkBtI,GAAG,CAAC,SAAS,EAAEiB,qBAAqBqH,gBAAgB,CAAC,QAAQ,CAAC;YAChFC,iBAAiBvI,GAAG,CAAC,KAAK,CAAC;QAC7B,GACCgD,KAAK,CACJ7D,IACEK,GAAGyB,qBAAqBI,cAAc,EAAEoB,OAAOpB,cAAc,GAC7D7B,GAAGyB,qBAAqBiE,eAAe,EAAEzC,OAAOyC,eAAe,IAGlEJ,SAAS,CAAC;YACTyE,yBAAyBtI,qBAAqBsI,uBAAuB;YACrEjF,eAAerD,qBAAqBqD,aAAa;YACjDC,sBAAsBtD,qBAAqBsD,oBAAoB;YAC/DC,OAAOvD,qBAAqBuD,KAAK;YACjCnD,gBAAgBJ,qBAAqBI,cAAc;YACnDoD,WAAWxD,qBAAqBwD,SAAS;YACzCpC,aAAapB,qBAAqBoB,WAAW;YAC7C6C,iBAAiBjE,qBAAqBiE,eAAe;QACvD;QAEF,MAAMvD,MAAMkI,OAAO,CAAC,EAAE;QACtB,IAAI,CAAClI,KAAK,MAAM,IAAIxB,iCAAiCsC,OAAOpB,cAAc;QAE1E,MAAM,CAACyI,SAAS,GAAG,MAAM1I,GACtB0B,MAAM,CAAC;YAACiH,kBAAkB/I,eAAe+I,gBAAgB;QAAA,GACzDhH,IAAI,CAAC/B,gBACLgC,KAAK,CAACxD,GAAGwB,eAAegD,EAAE,EAAEvB,OAAOyC,eAAe,GAClDpB,KAAK,CAAC,GACNgC,GAAG,CAAC;QAEP,MAAM,CAACH,QAAQ,GAAG,MAAMvE,GACrB+F,MAAM,CAACnG,gBACPkB,GAAG,CAAC;YACH6H,kBAAkBtH,OAAOsH,gBAAgB,IAAI;YAC7CC,4BAA4BvH,OAAOsH,gBAAgB,GAAG/J,GAAG,CAAC,KAAK,CAAC,GAAG;YACnEqH,WAAWrH,GAAG,CAAC,KAAK,CAAC;QACvB,GACCgD,KAAK,CAACxD,GAAGwB,eAAegD,EAAE,EAAEvB,OAAOyC,eAAe,GAClDJ,SAAS,CAAC;YAACiF,kBAAkB/I,eAAe+I,gBAAgB;QAAA;QAE/D,OAAO;YACLpI;YACAsI,0BAA0BH,UAAUC,oBAAoB;YACxDG,yBAAyBvE,SAASoE,oBAAoB;QACxD;IACF;IAEA,MAAMpI,MAAM6D,OAAO7D,GAAG;IACtB,IAAI,CAACA,KAAK,MAAM,IAAIxB,iCAAiCsC,OAAOpB,cAAc;IAC1E,OAAO;QACL8I,uBAAuBxI,IAAI4H,uBAAuB,KAAK;QACvDU,0BAA0BzE,OAAOyE,wBAAwB;QACzDC,yBAAyB1E,OAAO0E,uBAAuB;QACvDE,qBAAqB;YACnB9F,eAAe3C,IAAI2C,aAAa;YAChCC,sBAAsB5C,IAAI4C,oBAAoB;YAC9CC,OAAO7C,IAAI6C,KAAK;YAChBnD,gBAAgBM,IAAIN,cAAc;YAClCoD,WAAW9C,IAAI8C,SAAS;YACxBpC,aAAaV,IAAIU,WAAW;YAC5B6C,iBAAiBvD,IAAIuD,eAAe;QACtC;IACF;AACF;AAEA;;;;CAIC,GACD,OAAO,eAAemF,8BAA8B5H,MAEnD;IACC,MAAMjC,KAAKqD,WAAW,CAAC,OAAOzC;QAC5B,MAAMD,iBAAiBC,IAAIqB,OAAOpB,cAAc;QAEhD,wFAAwF;QACxF,2FAA2F;QAC3F,wFAAwF;QACxF,MAAMD,GACHqF,MAAM,CAAC5F,sBACPmC,KAAK,CAACxD,GAAGqB,qBAAqBQ,cAAc,EAAEoB,OAAOpB,cAAc;QACtE,MAAMiJ,uBAAuB,MAAMlJ,GAChC+F,MAAM,CAAClG,sBACPiB,GAAG,CAAC;YACHqH,yBAAyBvJ,GAAG,CAAC,SAAS,EAAEiB,qBAAqBsI,uBAAuB,CAAC,QAAQ,CAAC;QAChG,GACCvG,KAAK,CAACxD,GAAGyB,qBAAqBI,cAAc,EAAEoB,OAAOpB,cAAc,GACnEyD,SAAS,CAAC;YACTlD,eAAeX,qBAAqBW,aAAa;YACjDC,kBAAkBZ,qBAAqBY,gBAAgB;QACzD;QAEF,MAAMN,0CAA0CH,IAAIkJ;IACtD;AACF"}