@shipfox/api-runners 12.7.0 → 13.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +4 -4
- package/CHANGELOG.md +20 -0
- package/dist/db/index.d.ts +1 -1
- package/dist/db/index.d.ts.map +1 -1
- package/dist/db/index.js +1 -1
- package/dist/db/index.js.map +1 -1
- package/dist/db/job-executions.d.ts +8 -24
- package/dist/db/job-executions.d.ts.map +1 -1
- package/dist/db/job-executions.js +106 -83
- package/dist/db/job-executions.js.map +1 -1
- package/dist/db/reservations.d.ts.map +1 -1
- package/dist/db/reservations.js +6 -4
- package/dist/db/reservations.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -4
- package/dist/index.js.map +1 -1
- package/dist/presentation/index.d.ts +2 -1
- package/dist/presentation/index.d.ts.map +1 -1
- package/dist/presentation/index.js +2 -1
- package/dist/presentation/index.js.map +1 -1
- package/dist/presentation/inter-module.d.ts +0 -1
- package/dist/presentation/inter-module.d.ts.map +1 -1
- package/dist/presentation/inter-module.js +2 -25
- package/dist/presentation/inter-module.js.map +1 -1
- package/dist/presentation/subscribers/on-workflows-job-execution-queued.d.ts +3 -0
- package/dist/presentation/subscribers/on-workflows-job-execution-queued.d.ts.map +1 -0
- package/dist/presentation/subscribers/on-workflows-job-execution-queued.js +21 -0
- package/dist/presentation/subscribers/on-workflows-job-execution-queued.js.map +1 -0
- package/dist/presentation/subscribers/on-workflows-job-execution-terminated.d.ts +3 -0
- package/dist/presentation/subscribers/on-workflows-job-execution-terminated.d.ts.map +1 -0
- package/dist/presentation/subscribers/{on-workflows-job-execution-timed-out.js → on-workflows-job-execution-terminated.js} +5 -4
- package/dist/presentation/subscribers/on-workflows-job-execution-terminated.js.map +1 -0
- package/dist/tsconfig.test.tsbuildinfo +1 -1
- package/package.json +3 -3
- package/src/db/index.ts +0 -2
- package/src/db/job-executions.test.ts +197 -335
- package/src/db/job-executions.ts +145 -100
- package/src/db/reservations.ts +6 -3
- package/src/db/runner-instances.test.ts +100 -0
- package/src/index.test.ts +1 -0
- package/src/index.ts +8 -7
- package/src/presentation/index.ts +2 -1
- package/src/presentation/inter-module.ts +2 -40
- package/src/presentation/subscribers/on-workflows-job-execution-queued.test.ts +57 -0
- package/src/presentation/subscribers/on-workflows-job-execution-queued.ts +26 -0
- package/src/presentation/subscribers/on-workflows-job-execution-terminated.test.ts +142 -0
- package/src/presentation/subscribers/{on-workflows-job-execution-timed-out.ts → on-workflows-job-execution-terminated.ts} +5 -4
- package/test/factories/pending-job.ts +3 -0
- package/tsconfig.build.tsbuildinfo +1 -1
- package/dist/presentation/subscribers/on-workflows-job-execution-timed-out.d.ts +0 -3
- package/dist/presentation/subscribers/on-workflows-job-execution-timed-out.d.ts.map +0 -1
- package/dist/presentation/subscribers/on-workflows-job-execution-timed-out.js.map +0 -1
- package/src/presentation/inter-module.test.ts +0 -15
- package/src/presentation/subscribers/on-workflows-job-execution-timed-out.test.ts +0 -163
package/.turbo/turbo-build.log
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
$ shipfox-swc && shipfox-temporal-bundle dist/temporal/workflows/index.js
|
|
2
|
-
Successfully compiled:
|
|
3
|
-
2026-08-
|
|
2
|
+
Successfully compiled: 105 files with swc (548ms)
|
|
3
|
+
2026-08-23T10:00:31.690Z [INFO] asset workflow-bundle-0c3da2487e29e352d64e.js 3.17 MiB [emitted] [immutable] (name: main)
|
|
4
4
|
orphan modules 33.5 KiB [orphan] 21 modules
|
|
5
5
|
runtime modules 1.13 KiB 5 modules
|
|
6
6
|
modules by path ../../../node_modules/.pnpm/ 970 KiB 196 modules
|
|
@@ -14,6 +14,6 @@ optional modules 30 bytes [optional]
|
|
|
14
14
|
__temporal_custom_payload_converter (ignored) 15 bytes [optional] [built] [code generated]
|
|
15
15
|
__temporal_custom_failure_converter (ignored) 15 bytes [optional] [built] [code generated]
|
|
16
16
|
../../shared/node/temporal/dist/workflow-error-interceptor.js 1.21 KiB [built] [code generated]
|
|
17
|
-
webpack 5.107.2 compiled successfully in
|
|
18
|
-
2026-08-
|
|
17
|
+
webpack 5.107.2 compiled successfully in 9390 ms
|
|
18
|
+
2026-08-23T10:00:31.741Z [INFO] Workflow bundle created { size: '3.17MB' }
|
|
19
19
|
/home/runner/work/shipfox/shipfox/libs/api/runners/dist/temporal/workflows/index.js: 3326598 bytes
|
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,25 @@
|
|
|
1
1
|
# @shipfox/api-runners
|
|
2
2
|
|
|
3
|
+
## 13.1.0
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- 7de2e30: Release runner reservation units when a provisioned runner claims its first job.
|
|
8
|
+
- Updated dependencies [0d3c2e3]
|
|
9
|
+
- @shipfox/api-workflows-dto@13.1.0
|
|
10
|
+
|
|
11
|
+
## 13.0.0
|
|
12
|
+
|
|
13
|
+
### Major Changes
|
|
14
|
+
|
|
15
|
+
- af6b31e: Expose ordered queued and terminal workflow job-execution facts, and release terminal runner reservations once no uncancelled lease remains.
|
|
16
|
+
|
|
17
|
+
### Patch Changes
|
|
18
|
+
|
|
19
|
+
- Updated dependencies [af6b31e]
|
|
20
|
+
- @shipfox/api-runners-dto@13.0.0
|
|
21
|
+
- @shipfox/api-workflows-dto@13.0.0
|
|
22
|
+
|
|
3
23
|
## 12.7.0
|
|
4
24
|
|
|
5
25
|
### Patch Changes
|
package/dist/db/index.d.ts
CHANGED
|
@@ -4,7 +4,7 @@ export { closeDb, db, schema } from './db.js';
|
|
|
4
4
|
export type { CreateEphemeralRegistrationTokenParams, DeleteExpiredEphemeralRegistrationTokensParams, } from './ephemeral-registration-tokens.js';
|
|
5
5
|
export { createEphemeralRegistrationToken, createRunnerSessionConsumingEphemeralToken, deleteExpiredEphemeralRegistrationTokens, resolveEphemeralRegistrationTokenByHash, } from './ephemeral-registration-tokens.js';
|
|
6
6
|
export type { ClaimedJobExecution, EnqueueJobExecutionParams, RunnerInstanceBoundJobExecution, } from './job-executions.js';
|
|
7
|
-
export {
|
|
7
|
+
export { claimPendingJobExecution, enqueueJobExecution, expireStuckJobExecutions, getWorkspaceJobCounts, isJobLeaseActive, listActiveRunningJobExecutions, listRunningJobExecutionsByRunnerInstanceTx, recordHeartbeat, } from './job-executions.js';
|
|
8
8
|
export type { CreateManualRegistrationTokenParams } from './manual-registration-tokens.js';
|
|
9
9
|
export { createManualRegistrationToken, listUsableManualRegistrationTokensByWorkspaceId, resolveManualRegistrationTokenByHash, revokeManualRegistrationToken, } from './manual-registration-tokens.js';
|
|
10
10
|
export { hasActiveWorkspaceProvisionerCapability, listActiveWorkspaceProvisionerCapabilitySnapshots, listStaleWorkspaceProvisionerCapabilitySnapshots, publishWorkspaceProvisionerCapabilitySnapshot, } from './provisioner-capability-snapshots.js';
|
package/dist/db/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/db/index.ts"],"names":[],"mappings":"AAGA,YAAY,EACV,mCAAmC,EACnC,8BAA8B,EAC9B,sCAAsC,EACtC,sCAAsC,GACvC,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAC,gCAAgC,EAAC,MAAM,6BAA6B,CAAC;AAC7E,OAAO,EAAC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAC,MAAM,SAAS,CAAC;AAC5C,YAAY,EACV,sCAAsC,EACtC,8CAA8C,GAC/C,MAAM,oCAAoC,CAAC;AAC5C,OAAO,EACL,gCAAgC,EAChC,0CAA0C,EAC1C,wCAAwC,EACxC,uCAAuC,GACxC,MAAM,oCAAoC,CAAC;AAC5C,YAAY,EACV,mBAAmB,EACnB,yBAAyB,EACzB,+BAA+B,GAChC,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/db/index.ts"],"names":[],"mappings":"AAGA,YAAY,EACV,mCAAmC,EACnC,8BAA8B,EAC9B,sCAAsC,EACtC,sCAAsC,GACvC,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAC,gCAAgC,EAAC,MAAM,6BAA6B,CAAC;AAC7E,OAAO,EAAC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAC,MAAM,SAAS,CAAC;AAC5C,YAAY,EACV,sCAAsC,EACtC,8CAA8C,GAC/C,MAAM,oCAAoC,CAAC;AAC5C,OAAO,EACL,gCAAgC,EAChC,0CAA0C,EAC1C,wCAAwC,EACxC,uCAAuC,GACxC,MAAM,oCAAoC,CAAC;AAC5C,YAAY,EACV,mBAAmB,EACnB,yBAAyB,EACzB,+BAA+B,GAChC,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,wBAAwB,EACxB,mBAAmB,EACnB,wBAAwB,EACxB,qBAAqB,EACrB,gBAAgB,EAChB,8BAA8B,EAC9B,0CAA0C,EAC1C,eAAe,GAChB,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAC,mCAAmC,EAAC,MAAM,iCAAiC,CAAC;AACzF,OAAO,EACL,6BAA6B,EAC7B,+CAA+C,EAC/C,oCAAoC,EACpC,6BAA6B,GAC9B,MAAM,iCAAiC,CAAC;AACzC,OAAO,EACL,uCAAuC,EACvC,iDAAiD,EACjD,gDAAgD,EAChD,6CAA6C,GAC9C,MAAM,uCAAuC,CAAC;AAC/C,YAAY,EAAC,4BAA4B,EAAC,MAAM,yBAAyB,CAAC;AAC1E,OAAO,EACL,2CAA2C,EAC3C,sBAAsB,EACtB,2BAA2B,EAC3B,iCAAiC,EACjC,wCAAwC,EACxC,6BAA6B,EAC7B,2CAA2C,EAC3C,sBAAsB,EACtB,wBAAwB,GACzB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,uBAAuB,EACvB,6BAA6B,GAC9B,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,UAAU,EACV,0BAA0B,EAC1B,gBAAgB,EAChB,mBAAmB,GACpB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,yBAAyB,EACzB,uBAAuB,EACvB,oBAAoB,EACpB,uBAAuB,GACxB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAC,qBAAqB,EAAC,MAAM,yBAAyB,CAAC;AAC9D,YAAY,EACV,iCAAiC,EACjC,8BAA8B,EAC9B,gCAAgC,EAChC,8BAA8B,EAC9B,2BAA2B,EAC3B,yBAAyB,EACzB,6BAA6B,EAC7B,mCAAmC,GACpC,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,8BAA8B,EAC9B,iCAAiC,EACjC,eAAe,EACf,0CAA0C,EAC1C,yBAAyB,EACzB,oCAAoC,EACpC,+BAA+B,EAC/B,wBAAwB,EACxB,wBAAwB,EACxB,qBAAqB,GACtB,MAAM,uBAAuB,CAAC;AAC/B,YAAY,EACV,yBAAyB,EACzB,iCAAiC,GAClC,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAC,mBAAmB,EAAE,2BAA2B,EAAC,MAAM,sBAAsB,CAAC;AACtF,OAAO,EAAC,aAAa,EAAC,MAAM,oBAAoB,CAAC;AACjD,OAAO,EAAC,qBAAqB,EAAE,qBAAqB,EAAC,MAAM,qCAAqC,CAAC;AAEjG,eAAO,MAAM,cAAc,QAAoE,CAAC"}
|
package/dist/db/index.js
CHANGED
|
@@ -3,7 +3,7 @@ import { fileURLToPath } from 'node:url';
|
|
|
3
3
|
export { listAdministratorRunnerInstances } from './admin-runner-instances.js';
|
|
4
4
|
export { closeDb, db, schema } from './db.js';
|
|
5
5
|
export { createEphemeralRegistrationToken, createRunnerSessionConsumingEphemeralToken, deleteExpiredEphemeralRegistrationTokens, resolveEphemeralRegistrationTokenByHash } from './ephemeral-registration-tokens.js';
|
|
6
|
-
export {
|
|
6
|
+
export { claimPendingJobExecution, enqueueJobExecution, expireStuckJobExecutions, getWorkspaceJobCounts, isJobLeaseActive, listActiveRunningJobExecutions, listRunningJobExecutionsByRunnerInstanceTx, recordHeartbeat } from './job-executions.js';
|
|
7
7
|
export { createManualRegistrationToken, listUsableManualRegistrationTokensByWorkspaceId, resolveManualRegistrationTokenByHash, revokeManualRegistrationToken } from './manual-registration-tokens.js';
|
|
8
8
|
export { hasActiveWorkspaceProvisionerCapability, listActiveWorkspaceProvisionerCapabilitySnapshots, listStaleWorkspaceProvisionerCapabilitySnapshots, publishWorkspaceProvisionerCapabilitySnapshot } from './provisioner-capability-snapshots.js';
|
|
9
9
|
export { createInstallationProvisionerTokenWithAudit, createProvisionerToken, listActiveProvisionerTokens, listInstallationProvisionerTokens, listUsableProvisionerTokensByWorkspaceId, resolveProvisionerTokenByHash, revokeInstallationProvisionerTokenWithAudit, revokeProvisionerToken, touchProvisionerLastSeen } from './provisioner-tokens.js';
|
package/dist/db/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/db/index.ts"],"sourcesContent":["import {dirname, resolve} from 'node:path';\nimport {fileURLToPath} from 'node:url';\n\nexport type {\n AdministratorRunnerAssignmentFilter,\n AdministratorRunnerInstanceRow,\n ListAdministratorRunnerInstancesParams,\n ListAdministratorRunnerInstancesResult,\n} from './admin-runner-instances.js';\nexport {listAdministratorRunnerInstances} from './admin-runner-instances.js';\nexport {closeDb, db, schema} from './db.js';\nexport type {\n CreateEphemeralRegistrationTokenParams,\n DeleteExpiredEphemeralRegistrationTokensParams,\n} from './ephemeral-registration-tokens.js';\nexport {\n createEphemeralRegistrationToken,\n createRunnerSessionConsumingEphemeralToken,\n deleteExpiredEphemeralRegistrationTokens,\n resolveEphemeralRegistrationTokenByHash,\n} from './ephemeral-registration-tokens.js';\nexport type {\n ClaimedJobExecution,\n EnqueueJobExecutionParams,\n RunnerInstanceBoundJobExecution,\n} from './job-executions.js';\nexport {\n
|
|
1
|
+
{"version":3,"sources":["../../src/db/index.ts"],"sourcesContent":["import {dirname, resolve} from 'node:path';\nimport {fileURLToPath} from 'node:url';\n\nexport type {\n AdministratorRunnerAssignmentFilter,\n AdministratorRunnerInstanceRow,\n ListAdministratorRunnerInstancesParams,\n ListAdministratorRunnerInstancesResult,\n} from './admin-runner-instances.js';\nexport {listAdministratorRunnerInstances} from './admin-runner-instances.js';\nexport {closeDb, db, schema} from './db.js';\nexport type {\n CreateEphemeralRegistrationTokenParams,\n DeleteExpiredEphemeralRegistrationTokensParams,\n} from './ephemeral-registration-tokens.js';\nexport {\n createEphemeralRegistrationToken,\n createRunnerSessionConsumingEphemeralToken,\n deleteExpiredEphemeralRegistrationTokens,\n resolveEphemeralRegistrationTokenByHash,\n} from './ephemeral-registration-tokens.js';\nexport type {\n ClaimedJobExecution,\n EnqueueJobExecutionParams,\n RunnerInstanceBoundJobExecution,\n} from './job-executions.js';\nexport {\n claimPendingJobExecution,\n enqueueJobExecution,\n expireStuckJobExecutions,\n getWorkspaceJobCounts,\n isJobLeaseActive,\n listActiveRunningJobExecutions,\n listRunningJobExecutionsByRunnerInstanceTx,\n recordHeartbeat,\n} from './job-executions.js';\nexport type {CreateManualRegistrationTokenParams} from './manual-registration-tokens.js';\nexport {\n createManualRegistrationToken,\n listUsableManualRegistrationTokensByWorkspaceId,\n resolveManualRegistrationTokenByHash,\n revokeManualRegistrationToken,\n} from './manual-registration-tokens.js';\nexport {\n hasActiveWorkspaceProvisionerCapability,\n listActiveWorkspaceProvisionerCapabilitySnapshots,\n listStaleWorkspaceProvisionerCapabilitySnapshots,\n publishWorkspaceProvisionerCapabilitySnapshot,\n} from './provisioner-capability-snapshots.js';\nexport type {CreateProvisionerTokenParams} from './provisioner-tokens.js';\nexport {\n createInstallationProvisionerTokenWithAudit,\n createProvisionerToken,\n listActiveProvisionerTokens,\n listInstallationProvisionerTokens,\n listUsableProvisionerTokensByWorkspaceId,\n resolveProvisionerTokenByHash,\n revokeInstallationProvisionerTokenWithAudit,\n revokeProvisionerToken,\n touchProvisionerLastSeen,\n} from './provisioner-tokens.js';\nexport {\n consumeRunnersRateLimit,\n pruneExpiredRunnersRateLimits,\n} from './rate-limits.js';\nexport type {\n DemandStat,\n PollDemandAndReserveParams,\n ReservationGrant,\n ReservationTemplate,\n} from './reservations.js';\nexport {\n deleteExpiredReservations,\n deleteReservationsByIds,\n pollDemandAndReserve,\n releaseReservationUnits,\n} from './reservations.js';\nexport {assignRunnerInstances} from './runner-assignments.js';\nexport type {\n ActiveRunnerInstanceTemplateCount,\n ReapStaleRunnerInstancesResult,\n ReconcileRunnerInstancesDbResult,\n ReconcileRunnerInstancesParams,\n ReportRunnerInstancesParams,\n RunnerInstanceReportEvent,\n RunnerInstanceTerminateIntent,\n RunnerInstanceTerminateIntentReason,\n} from './runner-instances.js';\nexport {\n attachRunnerInstanceProviderId,\n countStaleEnrolledRunnerInstances,\n isTerminalState,\n listActiveRunnerInstanceCountsByTemplateTx,\n listActiveRunnerInstances,\n listProvisionerTerminateIntentRowsTx,\n listProvisionerTerminateIntents,\n reapStaleRunnerInstances,\n reconcileRunnerInstances,\n reportRunnerInstances,\n} from './runner-instances.js';\nexport type {\n CreateRunnerSessionParams,\n DeleteExpiredRunnerSessionsParams,\n} from './runner-sessions.js';\nexport {createRunnerSession, deleteExpiredRunnerSessions} from './runner-sessions.js';\nexport {runnersOutbox} from './schema/outbox.js';\nexport {runnerBootstrapTokens, runnerControlSessions} from './schema/runner-control-sessions.js';\n\nexport const migrationsPath = resolve(dirname(fileURLToPath(import.meta.url)), '../../drizzle');\n"],"names":["dirname","resolve","fileURLToPath","listAdministratorRunnerInstances","closeDb","db","schema","createEphemeralRegistrationToken","createRunnerSessionConsumingEphemeralToken","deleteExpiredEphemeralRegistrationTokens","resolveEphemeralRegistrationTokenByHash","claimPendingJobExecution","enqueueJobExecution","expireStuckJobExecutions","getWorkspaceJobCounts","isJobLeaseActive","listActiveRunningJobExecutions","listRunningJobExecutionsByRunnerInstanceTx","recordHeartbeat","createManualRegistrationToken","listUsableManualRegistrationTokensByWorkspaceId","resolveManualRegistrationTokenByHash","revokeManualRegistrationToken","hasActiveWorkspaceProvisionerCapability","listActiveWorkspaceProvisionerCapabilitySnapshots","listStaleWorkspaceProvisionerCapabilitySnapshots","publishWorkspaceProvisionerCapabilitySnapshot","createInstallationProvisionerTokenWithAudit","createProvisionerToken","listActiveProvisionerTokens","listInstallationProvisionerTokens","listUsableProvisionerTokensByWorkspaceId","resolveProvisionerTokenByHash","revokeInstallationProvisionerTokenWithAudit","revokeProvisionerToken","touchProvisionerLastSeen","consumeRunnersRateLimit","pruneExpiredRunnersRateLimits","deleteExpiredReservations","deleteReservationsByIds","pollDemandAndReserve","releaseReservationUnits","assignRunnerInstances","attachRunnerInstanceProviderId","countStaleEnrolledRunnerInstances","isTerminalState","listActiveRunnerInstanceCountsByTemplateTx","listActiveRunnerInstances","listProvisionerTerminateIntentRowsTx","listProvisionerTerminateIntents","reapStaleRunnerInstances","reconcileRunnerInstances","reportRunnerInstances","createRunnerSession","deleteExpiredRunnerSessions","runnersOutbox","runnerBootstrapTokens","runnerControlSessions","migrationsPath","url"],"mappings":"AAAA,SAAQA,OAAO,EAAEC,OAAO,QAAO,YAAY;AAC3C,SAAQC,aAAa,QAAO,WAAW;AAQvC,SAAQC,gCAAgC,QAAO,8BAA8B;AAC7E,SAAQC,OAAO,EAAEC,EAAE,EAAEC,MAAM,QAAO,UAAU;AAK5C,SACEC,gCAAgC,EAChCC,0CAA0C,EAC1CC,wCAAwC,EACxCC,uCAAuC,QAClC,qCAAqC;AAM5C,SACEC,wBAAwB,EACxBC,mBAAmB,EACnBC,wBAAwB,EACxBC,qBAAqB,EACrBC,gBAAgB,EAChBC,8BAA8B,EAC9BC,0CAA0C,EAC1CC,eAAe,QACV,sBAAsB;AAE7B,SACEC,6BAA6B,EAC7BC,+CAA+C,EAC/CC,oCAAoC,EACpCC,6BAA6B,QACxB,kCAAkC;AACzC,SACEC,uCAAuC,EACvCC,iDAAiD,EACjDC,gDAAgD,EAChDC,6CAA6C,QACxC,wCAAwC;AAE/C,SACEC,2CAA2C,EAC3CC,sBAAsB,EACtBC,2BAA2B,EAC3BC,iCAAiC,EACjCC,wCAAwC,EACxCC,6BAA6B,EAC7BC,2CAA2C,EAC3CC,sBAAsB,EACtBC,wBAAwB,QACnB,0BAA0B;AACjC,SACEC,uBAAuB,EACvBC,6BAA6B,QACxB,mBAAmB;AAO1B,SACEC,yBAAyB,EACzBC,uBAAuB,EACvBC,oBAAoB,EACpBC,uBAAuB,QAClB,oBAAoB;AAC3B,SAAQC,qBAAqB,QAAO,0BAA0B;AAW9D,SACEC,8BAA8B,EAC9BC,iCAAiC,EACjCC,eAAe,EACfC,0CAA0C,EAC1CC,yBAAyB,EACzBC,oCAAoC,EACpCC,+BAA+B,EAC/BC,wBAAwB,EACxBC,wBAAwB,EACxBC,qBAAqB,QAChB,wBAAwB;AAK/B,SAAQC,mBAAmB,EAAEC,2BAA2B,QAAO,uBAAuB;AACtF,SAAQC,aAAa,QAAO,qBAAqB;AACjD,SAAQC,qBAAqB,EAAEC,qBAAqB,QAAO,sCAAsC;AAEjG,OAAO,MAAMC,iBAAiBzD,QAAQD,QAAQE,cAAc,YAAYyD,GAAG,IAAI,iBAAiB"}
|
|
@@ -8,6 +8,7 @@ export interface EnqueueJobExecutionParams {
|
|
|
8
8
|
jobExecutionId: string;
|
|
9
9
|
projectId: string;
|
|
10
10
|
requiredLabels: string[];
|
|
11
|
+
queuedAt: Date;
|
|
11
12
|
}
|
|
12
13
|
export declare function getWorkspaceJobCounts(params: {
|
|
13
14
|
workspaceIds: string[];
|
|
@@ -55,32 +56,18 @@ export declare function claimPendingJobExecution(params: {
|
|
|
55
56
|
maxClaims: number | null;
|
|
56
57
|
runnerSessionLivenessThrottleSeconds: number;
|
|
57
58
|
}): Promise<ClaimedJobExecution | null>;
|
|
58
|
-
/**
|
|
59
|
-
* Releases a job execution's lease when the orchestration workflow finalizes it: deletes the
|
|
60
|
-
* running-job-execution row AND any lingering pending row for the same execution, in one tx.
|
|
61
|
-
* When the deleted lease was the last one for a terminal provider runner, it also releases the
|
|
62
|
-
* runner's reservation in the same runner-owned transaction. Terminal reports and workflow lease
|
|
63
|
-
* finalization remain independently retryable; they do not share a cross-module scope lock.
|
|
64
|
-
* Idempotent (0-row no-op), no token scope (the workflow is authoritative), and
|
|
65
|
-
* emits no event: the workflow already owns the outcome. Sweeping the pending row
|
|
66
|
-
* too closes the at-least-once window where an enqueue retry left an orphan that a
|
|
67
|
-
* later claim would otherwise pick up for an already-finished job execution.
|
|
68
|
-
*/
|
|
69
|
-
export declare function releaseJobExecution(params: {
|
|
70
|
-
jobExecutionId: string;
|
|
71
|
-
}): Promise<void>;
|
|
72
59
|
/**
|
|
73
60
|
* Reaps stale leases (bounded by `limit`), emitting one
|
|
74
61
|
* `runners.job.lease_expired` event per reaped job execution.
|
|
75
62
|
*
|
|
76
63
|
* The cutoff is re-checked in the DELETE, not just the locking subquery, so a
|
|
77
64
|
* heartbeat landing mid-call spares the live row. Each reaped execution also sweeps its
|
|
78
|
-
* pending row
|
|
79
|
-
* that a later claim re-runs as an already-finished job execution.
|
|
65
|
+
* pending row so an at-least-once queue replay cannot leave an orphan.
|
|
80
66
|
*
|
|
81
|
-
* Locks pending-then-running to match `claimPendingJobExecution
|
|
82
|
-
* `reconcileTerminalJobExecution`. The stale candidate scan intentionally does not lock
|
|
83
|
-
* rows first; the running-row DELETE re-checks the stale predicate after the
|
|
67
|
+
* Locks pending-then-running to match `claimPendingJobExecution` and
|
|
68
|
+
* `reconcileTerminalJobExecution`. The stale candidate scan intentionally does not lock
|
|
69
|
+
* running rows first; the running-row DELETE re-checks the stale predicate after the
|
|
70
|
+
* pending-row sweep.
|
|
84
71
|
*/
|
|
85
72
|
export declare function expireStuckJobExecutions(params: {
|
|
86
73
|
thresholdSeconds: number;
|
|
@@ -131,13 +118,10 @@ export declare function recordHeartbeat(params: {
|
|
|
131
118
|
}>;
|
|
132
119
|
/**
|
|
133
120
|
* Reconciles a terminal job execution with runner state in one transaction: removes its pending
|
|
134
|
-
* queue row
|
|
135
|
-
* idempotent and preserves the first cancellation-request timestamp.
|
|
121
|
+
* queue row, requests cancellation on its running lease, and converges any linked reservation.
|
|
122
|
+
* The operation is idempotent and preserves the first cancellation-request timestamp.
|
|
136
123
|
*/
|
|
137
124
|
export declare function reconcileTerminalJobExecution(params: {
|
|
138
125
|
jobExecutionId: string;
|
|
139
126
|
}): Promise<void>;
|
|
140
|
-
export declare function cancelRunnerJobs(params: {
|
|
141
|
-
jobIds: string[];
|
|
142
|
-
}): Promise<void>;
|
|
143
127
|
//# sourceMappingURL=job-executions.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"job-executions.d.ts","sourceRoot":"","sources":["../../src/db/job-executions.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"job-executions.d.ts","sourceRoot":"","sources":["../../src/db/job-executions.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,KAAK,yBAAyB,EAC/B,MAAM,0BAA0B,CAAC;AAiClC,OAAO,KAAK,EAAC,EAAE,EAAC,MAAM,SAAS,CAAC;AAuDhC,MAAM,WAAW,yBAAyB;IACxC,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,QAAQ,EAAE,IAAI,CAAC;CAChB;AAED,wBAAsB,qBAAqB,CAAC,MAAM,EAAE;IAClD,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB,GAAG,OAAO,CAAC,KAAK,CAAC;IAAC,WAAW,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAC,CAAC,CAAC,CAwBzE;AAKD,wBAAsB,mBAAmB,CAAC,MAAM,EAAE,yBAAyB,GAAG,OAAO,CAAC,IAAI,CAAC,CA6C1F;AAED,MAAM,WAAW,mBAAmB;IAClC,aAAa,EAAE,MAAM,CAAC;IACtB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,yBAAyB;IACxC,aAAa,EAAE,MAAM,CAAC;IACtB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,SAAS,EAAE,IAAI,CAAC;IAChB,eAAe,EAAE,IAAI,CAAC;CACvB;AAED,MAAM,WAAW,+BAA+B;IAC9C,aAAa,EAAE,MAAM,CAAC;IACtB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,gBAAgB,EAAE,MAAM,CAAC;IACzB,SAAS,EAAE,IAAI,CAAC;IAChB,eAAe,EAAE,IAAI,CAAC;IACtB,uBAAuB,EAAE,IAAI,GAAG,IAAI,CAAC;CACtC;AAED,wBAAsB,wBAAwB,CAAC,MAAM,EAAE;IACrD,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,oCAAoC,EAAE,MAAM,CAAC;CAC9C,GAAG,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC,CA8PtC;AAmBD;;;;;;;;;;;;GAYG;AACH,wBAAsB,wBAAwB,CAAC,MAAM,EAAE;IACrD,gBAAgB,EAAE,MAAM,CAAC;IACzB,4BAA4B,EAAE,MAAM,CAAC;IACrC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,GAAG,OAAO,CACT,KAAK,CAAC;IACJ,aAAa,EAAE,MAAM,CAAC;IACtB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC,CACH,CAiGA;AAED,wBAAsB,yBAAyB,IAAI,OAAO,CAAC;IACzD,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;CAC9B,CAAC,CAOD;AAED,wBAAsB,8BAA8B,CAAC,MAAM,EAAE;IAC3D,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,GAAG,OAAO,CAAC,yBAAyB,EAAE,CAAC,CAyBvC;AAED,wBAAsB,0CAA0C,CAC9D,EAAE,EAAE,EAAE,EACN,MAAM,EAAE;IACN,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,MAAM,EAAE,CAAC;CAC7B,GACA,OAAO,CAAC,+BAA+B,EAAE,CAAC,CA2E5C;AAMD,wBAAsB,gBAAgB,CAAC,MAAM,EAAE;IAC7C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;CACzB,GAAG,OAAO,CAAC,OAAO,CAAC,CAcnB;AAED,wBAAsB,eAAe,CAAC,MAAM,EAAE;IAC5C,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,CAAC,EAAE,yBAAyB,GAAG,IAAI,CAAC;CACrD,GAAG,OAAO,CAAC;IACV,qBAAqB,EAAE,OAAO,CAAC;IAC/B,wBAAwB,EAAE,yBAAyB,GAAG,IAAI,CAAC;IAC3D,uBAAuB,EAAE,yBAAyB,GAAG,IAAI,CAAC;IAC1D,mBAAmB,EAAE;QACnB,aAAa,EAAE,MAAM,CAAC;QACtB,oBAAoB,EAAE,MAAM,CAAC;QAC7B,KAAK,EAAE,MAAM,CAAC;QACd,cAAc,EAAE,MAAM,CAAC;QACvB,SAAS,EAAE,MAAM,CAAC;QAClB,WAAW,EAAE,MAAM,CAAC;QACpB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;CACH,CAAC,CAoED;AAED;;;;GAIG;AACH,wBAAsB,6BAA6B,CAAC,MAAM,EAAE;IAC1D,cAAc,EAAE,MAAM,CAAC;CACxB,GAAG,OAAO,CAAC,IAAI,CAAC,CAuBhB"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { RUNNER_JOB_CLAIMED, RUNNER_JOB_LEASE_EXPIRED
|
|
1
|
+
import { RUNNER_JOB_CLAIMED, RUNNER_JOB_LEASE_EXPIRED } from '@shipfox/api-runners-dto';
|
|
2
2
|
import { logger } from '@shipfox/node-opentelemetry';
|
|
3
3
|
import { writeOutboxEvent, writeOutboxEvents } from '@shipfox/node-outbox';
|
|
4
4
|
import { canonicalizeLabels } from '@shipfox/runner-labels';
|
|
@@ -6,9 +6,11 @@ import { and, arrayContained, asc, count, desc, eq, gt, inArray, isNotNull, isNu
|
|
|
6
6
|
import { EmptyRequiredLabelsError, RunnerSessionExhaustedError, RunningJobExecutionNotFoundError } from '#core/errors.js';
|
|
7
7
|
import { jobExecutionEnqueuedCount, jobExecutionLeaseExpiredCount, recordJobExecutionQueueTime, recordProviderRunnerActivationToFirstClaim } from '#metrics/instance.js';
|
|
8
8
|
import { db } from './db.js';
|
|
9
|
-
import {
|
|
9
|
+
import { lockRunnerReservationAdvisoryKeysTx } from './reservation-locks.js';
|
|
10
|
+
import { releaseReservationUnits, releaseTerminalRunnerInstanceReservationsByIds } from './reservations.js';
|
|
10
11
|
import { runnersOutbox } from './schema/outbox.js';
|
|
11
12
|
import { pendingJobExecutions } from './schema/pending-job-executions.js';
|
|
13
|
+
import { reservations } from './schema/reservations.js';
|
|
12
14
|
import { providerRunners } from './schema/runner-instances.js';
|
|
13
15
|
import { runnerSessions } from './schema/runner-sessions.js';
|
|
14
16
|
import { runningJobExecutions } from './schema/running-job-executions.js';
|
|
@@ -16,6 +18,34 @@ const runnerJobExecutionLockPrefix = 'runners_job_execution:';
|
|
|
16
18
|
async function lockJobExecution(tx, jobExecutionId) {
|
|
17
19
|
await tx.execute(sql`select pg_advisory_xact_lock(hashtext(${`${runnerJobExecutionLockPrefix}${jobExecutionId}`}))`);
|
|
18
20
|
}
|
|
21
|
+
async function releaseReservationsForTerminalRunningRows(tx, rows) {
|
|
22
|
+
// Converge after the lease fact changes. The reservation helper locks each provider runner
|
|
23
|
+
// and releases only when the runner is terminal and no uncancelled lease remains.
|
|
24
|
+
const providerRunnerIdsByProvisionerId = new Map();
|
|
25
|
+
for (const row of rows){
|
|
26
|
+
if (row.provisionerId === null || row.providerRunnerId === null) continue;
|
|
27
|
+
const providerRunnerIds = providerRunnerIdsByProvisionerId.get(row.provisionerId) ?? new Set();
|
|
28
|
+
providerRunnerIds.add(row.providerRunnerId);
|
|
29
|
+
providerRunnerIdsByProvisionerId.set(row.provisionerId, providerRunnerIds);
|
|
30
|
+
}
|
|
31
|
+
for (const provisionerId of [
|
|
32
|
+
...providerRunnerIdsByProvisionerId.keys()
|
|
33
|
+
].sort()){
|
|
34
|
+
const providerRunnerIds = providerRunnerIdsByProvisionerId.get(provisionerId);
|
|
35
|
+
if (!providerRunnerIds) continue;
|
|
36
|
+
await releaseTerminalRunnerInstanceReservationsByIds(tx, {
|
|
37
|
+
workspaceId: null,
|
|
38
|
+
provisionerId,
|
|
39
|
+
providerRunnerIds: [
|
|
40
|
+
...providerRunnerIds
|
|
41
|
+
].sort(),
|
|
42
|
+
requireUnlinkedSession: false,
|
|
43
|
+
// Lock and re-check the runner row locally so lease finalization remains retryable
|
|
44
|
+
// without a workflow/runner scope lock.
|
|
45
|
+
requireTerminalState: false
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
}
|
|
19
49
|
export async function getWorkspaceJobCounts(params) {
|
|
20
50
|
const [pendingRows, runningRows] = await Promise.all([
|
|
21
51
|
db().select({
|
|
@@ -41,11 +71,9 @@ export async function getWorkspaceJobCounts(params) {
|
|
|
41
71
|
running: runningByWorkspace.get(workspaceId) ?? 0
|
|
42
72
|
}));
|
|
43
73
|
}
|
|
44
|
-
//
|
|
45
|
-
//
|
|
46
|
-
//
|
|
47
|
-
// The per-execution advisory lock serializes retries with lease expiry/reconciliation, and the
|
|
48
|
-
// durable lease-expired event prevents a retry from re-queueing an execution already reaped.
|
|
74
|
+
// The workflows outbox delivers queue facts at least once. The per-execution advisory lock and
|
|
75
|
+
// pending/running checks make replay a no-op across the claim transition. A durable lease-expired
|
|
76
|
+
// event also prevents a delayed replay from resurrecting an execution already reaped locally.
|
|
49
77
|
export async function enqueueJobExecution(params) {
|
|
50
78
|
const requiredLabels = [
|
|
51
79
|
...canonicalizeLabels(params.requiredLabels)
|
|
@@ -53,6 +81,10 @@ export async function enqueueJobExecution(params) {
|
|
|
53
81
|
if (requiredLabels.length === 0) throw new EmptyRequiredLabelsError();
|
|
54
82
|
const enqueued = await db().transaction(async (tx)=>{
|
|
55
83
|
await lockJobExecution(tx, params.jobExecutionId);
|
|
84
|
+
const [running] = await tx.select({
|
|
85
|
+
jobExecutionId: runningJobExecutions.jobExecutionId
|
|
86
|
+
}).from(runningJobExecutions).where(eq(runningJobExecutions.jobExecutionId, params.jobExecutionId)).limit(1);
|
|
87
|
+
if (running) return false;
|
|
56
88
|
const [leaseExpired] = await tx.select({
|
|
57
89
|
id: runnersOutbox.id
|
|
58
90
|
}).from(runnersOutbox).where(and(eq(runnersOutbox.eventType, RUNNER_JOB_LEASE_EXPIRED), sql`${runnersOutbox.payload}->>'jobExecutionId' = ${params.jobExecutionId}`)).limit(1);
|
|
@@ -64,27 +96,14 @@ export async function enqueueJobExecution(params) {
|
|
|
64
96
|
jobId: params.jobId,
|
|
65
97
|
jobExecutionId: params.jobExecutionId,
|
|
66
98
|
projectId: params.projectId,
|
|
67
|
-
requiredLabels
|
|
99
|
+
requiredLabels,
|
|
100
|
+
createdAt: params.queuedAt
|
|
68
101
|
}).onConflictDoNothing({
|
|
69
102
|
target: pendingJobExecutions.jobExecutionId
|
|
70
103
|
}).returning({
|
|
71
104
|
createdAt: pendingJobExecutions.createdAt
|
|
72
105
|
});
|
|
73
|
-
|
|
74
|
-
// emitted the queued event (durably, in the outbox), so re-emitting would
|
|
75
|
-
// only add a redundant row the subscriber coalesces away. Skip it.
|
|
76
|
-
if (!inserted) return false;
|
|
77
|
-
await writeOutboxEvent(tx, runnersOutbox, {
|
|
78
|
-
type: RUNNER_JOB_QUEUED,
|
|
79
|
-
payload: {
|
|
80
|
-
workflowRunId: params.workflowRunId,
|
|
81
|
-
workflowRunAttemptId: params.workflowRunAttemptId,
|
|
82
|
-
jobId: params.jobId,
|
|
83
|
-
jobExecutionId: params.jobExecutionId,
|
|
84
|
-
queuedAt: inserted.createdAt.toISOString()
|
|
85
|
-
}
|
|
86
|
-
});
|
|
87
|
-
return true;
|
|
106
|
+
return inserted !== undefined;
|
|
88
107
|
});
|
|
89
108
|
if (enqueued) jobExecutionEnqueuedCount.add(1);
|
|
90
109
|
}
|
|
@@ -119,6 +138,20 @@ export async function claimPendingJobExecution(params) {
|
|
|
119
138
|
provisionerId = session.provisionerId;
|
|
120
139
|
providerRunnerId = session.providerRunnerId;
|
|
121
140
|
}
|
|
141
|
+
const runnerInstanceCondition = runnerInstanceId ? eq(providerRunners.id, runnerInstanceId) : provisionerId && providerRunnerId ? and(eq(providerRunners.provisionerId, provisionerId), eq(providerRunners.providerRunnerId, providerRunnerId)) : null;
|
|
142
|
+
if (runnerInstanceCondition && provisionerId) {
|
|
143
|
+
const [runner] = await tx.select({
|
|
144
|
+
reservationId: providerRunners.reservationId,
|
|
145
|
+
intendedReservationId: providerRunners.intendedReservationId
|
|
146
|
+
}).from(providerRunners).where(runnerInstanceCondition).limit(1);
|
|
147
|
+
await lockRunnerReservationAdvisoryKeysTx(tx, {
|
|
148
|
+
provisionerId,
|
|
149
|
+
reservationIds: [
|
|
150
|
+
runner?.reservationId,
|
|
151
|
+
runner?.intendedReservationId
|
|
152
|
+
].filter((reservationId)=>reservationId !== null && reservationId !== undefined)
|
|
153
|
+
});
|
|
154
|
+
}
|
|
122
155
|
// `id` is a uuidv7 (time-ordered), so it is a deterministic FIFO tiebreaker
|
|
123
156
|
// for rows sharing a created_at within a batch. Lock only the FIFO candidate before
|
|
124
157
|
// attempting its execution advisory lock; putting pg_try_advisory_xact_lock in this
|
|
@@ -163,7 +196,6 @@ export async function claimPendingJobExecution(params) {
|
|
|
163
196
|
provider: null,
|
|
164
197
|
launchKind: params.maxClaims === null ? 'manual' : 'unknown'
|
|
165
198
|
};
|
|
166
|
-
const runnerInstanceCondition = runnerInstanceId ? eq(providerRunners.id, runnerInstanceId) : provisionerId && providerRunnerId ? and(eq(providerRunners.provisionerId, provisionerId), eq(providerRunners.providerRunnerId, providerRunnerId)) : null;
|
|
167
199
|
if (runnerInstanceCondition) {
|
|
168
200
|
const [claimedRunner] = await tx.update(providerRunners).set({
|
|
169
201
|
firstClaimedAt: sql`coalesce(${providerRunners.firstClaimedAt}, ${claimed.claimedAt})`,
|
|
@@ -174,6 +206,8 @@ export async function claimPendingJobExecution(params) {
|
|
|
174
206
|
provider: providerRunners.providerKind,
|
|
175
207
|
launchKind: providerRunners.launchKind,
|
|
176
208
|
runnerInstanceId: providerRunners.id,
|
|
209
|
+
reservationId: providerRunners.reservationId,
|
|
210
|
+
intendedReservationId: providerRunners.intendedReservationId,
|
|
177
211
|
sessionCreatedAtEpochMs: sql`(
|
|
178
212
|
select extract(epoch from ${runnerSessions.createdAt})::double precision * 1000
|
|
179
213
|
from ${runnerSessions}
|
|
@@ -184,6 +218,31 @@ export async function claimPendingJobExecution(params) {
|
|
|
184
218
|
queueTimeObservation.provider = claimedRunner.provider;
|
|
185
219
|
queueTimeObservation.launchKind = claimedRunner.launchKind;
|
|
186
220
|
}
|
|
221
|
+
if (claimedRunner?.isFirstClaim) {
|
|
222
|
+
const [releasedRunner] = await tx.update(providerRunners).set({
|
|
223
|
+
intendedReservationId: null,
|
|
224
|
+
reservationReleasedAt: sql`now()`,
|
|
225
|
+
updatedAt: sql`now()`
|
|
226
|
+
}).where(and(eq(providerRunners.id, claimedRunner.runnerInstanceId), or(isNotNull(providerRunners.reservationId), isNotNull(providerRunners.intendedReservationId)), isNull(providerRunners.reservationReleasedAt))).returning({
|
|
227
|
+
provisionerId: providerRunners.provisionerId
|
|
228
|
+
});
|
|
229
|
+
const reservationId = claimedRunner.intendedReservationId ?? claimedRunner.reservationId;
|
|
230
|
+
if (releasedRunner?.provisionerId && reservationId) {
|
|
231
|
+
const [reservation] = await tx.select({
|
|
232
|
+
workspaceId: reservations.workspaceId
|
|
233
|
+
}).from(reservations).where(and(eq(reservations.id, reservationId), eq(reservations.provisionerId, releasedRunner.provisionerId))).limit(1);
|
|
234
|
+
if (reservation) await releaseReservationUnits(tx, {
|
|
235
|
+
workspaceId: reservation.workspaceId,
|
|
236
|
+
provisionerId: releasedRunner.provisionerId,
|
|
237
|
+
releases: [
|
|
238
|
+
{
|
|
239
|
+
reservationId,
|
|
240
|
+
count: 1
|
|
241
|
+
}
|
|
242
|
+
]
|
|
243
|
+
});
|
|
244
|
+
}
|
|
245
|
+
}
|
|
187
246
|
if (claimedRunner?.isFirstClaim && claimedRunner.firstClaimedAt !== null && claimedRunner.sessionCreatedAtEpochMs !== null && claimedRunner.sessionCreatedAtEpochMs !== undefined) activationToFirstClaimObservation = {
|
|
188
247
|
durationMilliseconds: claimedRunner.firstClaimedAt.getTime() - claimedRunner.sessionCreatedAtEpochMs,
|
|
189
248
|
provider: claimedRunner.provider,
|
|
@@ -227,57 +286,18 @@ async function touchRunnerSessionLiveness(params) {
|
|
|
227
286
|
updatedAt: sql`now()`
|
|
228
287
|
}).where(and(eq(runnerSessions.id, params.runnerSessionId), eq(runnerSessions.workspaceId, params.workspaceId), sql`${runnerSessions.updatedAt} < now() - (${params.throttleSeconds} || ' seconds')::interval`));
|
|
229
288
|
}
|
|
230
|
-
/**
|
|
231
|
-
* Releases a job execution's lease when the orchestration workflow finalizes it: deletes the
|
|
232
|
-
* running-job-execution row AND any lingering pending row for the same execution, in one tx.
|
|
233
|
-
* When the deleted lease was the last one for a terminal provider runner, it also releases the
|
|
234
|
-
* runner's reservation in the same runner-owned transaction. Terminal reports and workflow lease
|
|
235
|
-
* finalization remain independently retryable; they do not share a cross-module scope lock.
|
|
236
|
-
* Idempotent (0-row no-op), no token scope (the workflow is authoritative), and
|
|
237
|
-
* emits no event: the workflow already owns the outcome. Sweeping the pending row
|
|
238
|
-
* too closes the at-least-once window where an enqueue retry left an orphan that a
|
|
239
|
-
* later claim would otherwise pick up for an already-finished job execution.
|
|
240
|
-
*/ export async function releaseJobExecution(params) {
|
|
241
|
-
await db().transaction(async (tx)=>{
|
|
242
|
-
await lockJobExecution(tx, params.jobExecutionId);
|
|
243
|
-
// Delete pending before running to match `claimPendingJobExecution`'s lock-acquisition
|
|
244
|
-
// order (it locks the pending row first, then the running row). A concurrent
|
|
245
|
-
// claim picking up an orphan pending row for this same job execution would otherwise
|
|
246
|
-
// deadlock against the reverse order here.
|
|
247
|
-
await tx.delete(pendingJobExecutions).where(eq(pendingJobExecutions.jobExecutionId, params.jobExecutionId));
|
|
248
|
-
const deletedRunningRows = await tx.delete(runningJobExecutions).where(eq(runningJobExecutions.jobExecutionId, params.jobExecutionId)).returning({
|
|
249
|
-
workspaceId: runningJobExecutions.workspaceId,
|
|
250
|
-
provisionerId: runningJobExecutions.provisionerId,
|
|
251
|
-
providerRunnerId: runningJobExecutions.providerRunnerId
|
|
252
|
-
});
|
|
253
|
-
const deletedRunningRow = deletedRunningRows[0];
|
|
254
|
-
if (deletedRunningRow?.provisionerId && deletedRunningRow.providerRunnerId) {
|
|
255
|
-
await releaseTerminalRunnerInstanceReservationsByIds(tx, {
|
|
256
|
-
workspaceId: null,
|
|
257
|
-
provisionerId: deletedRunningRow.provisionerId,
|
|
258
|
-
providerRunnerIds: [
|
|
259
|
-
deletedRunningRow.providerRunnerId
|
|
260
|
-
],
|
|
261
|
-
requireUnlinkedSession: false,
|
|
262
|
-
// Lock and re-check the runner row locally so terminal reporting remains retryable
|
|
263
|
-
// without a workflow/runner scope lock.
|
|
264
|
-
requireTerminalState: false
|
|
265
|
-
});
|
|
266
|
-
}
|
|
267
|
-
});
|
|
268
|
-
}
|
|
269
289
|
/**
|
|
270
290
|
* Reaps stale leases (bounded by `limit`), emitting one
|
|
271
291
|
* `runners.job.lease_expired` event per reaped job execution.
|
|
272
292
|
*
|
|
273
293
|
* The cutoff is re-checked in the DELETE, not just the locking subquery, so a
|
|
274
294
|
* heartbeat landing mid-call spares the live row. Each reaped execution also sweeps its
|
|
275
|
-
* pending row
|
|
276
|
-
* that a later claim re-runs as an already-finished job execution.
|
|
295
|
+
* pending row so an at-least-once queue replay cannot leave an orphan.
|
|
277
296
|
*
|
|
278
|
-
* Locks pending-then-running to match `claimPendingJobExecution
|
|
279
|
-
* `reconcileTerminalJobExecution`. The stale candidate scan intentionally does not lock
|
|
280
|
-
* rows first; the running-row DELETE re-checks the stale predicate after the
|
|
297
|
+
* Locks pending-then-running to match `claimPendingJobExecution` and
|
|
298
|
+
* `reconcileTerminalJobExecution`. The stale candidate scan intentionally does not lock
|
|
299
|
+
* running rows first; the running-row DELETE re-checks the stale predicate after the
|
|
300
|
+
* pending-row sweep.
|
|
281
301
|
*/ export async function expireStuckJobExecutions(params) {
|
|
282
302
|
const reaped = await db().transaction(async (tx)=>{
|
|
283
303
|
const heartbeatCutoff = sql`now() - (${params.thresholdSeconds} || ' seconds')::interval`;
|
|
@@ -303,9 +323,12 @@ async function touchRunnerSessionLiveness(params) {
|
|
|
303
323
|
workflowRunId: runningJobExecutions.workflowRunId,
|
|
304
324
|
workflowRunAttemptId: runningJobExecutions.workflowRunAttemptId,
|
|
305
325
|
jobId: runningJobExecutions.jobId,
|
|
306
|
-
jobExecutionId: runningJobExecutions.jobExecutionId
|
|
326
|
+
jobExecutionId: runningJobExecutions.jobExecutionId,
|
|
327
|
+
provisionerId: runningJobExecutions.provisionerId,
|
|
328
|
+
providerRunnerId: runningJobExecutions.providerRunnerId
|
|
307
329
|
});
|
|
308
330
|
if (deleted.length === 0) return [];
|
|
331
|
+
await releaseReservationsForTerminalRunningRows(tx, deleted);
|
|
309
332
|
await writeOutboxEvents(tx, runnersOutbox, deleted.map((row)=>({
|
|
310
333
|
type: RUNNER_JOB_LEASE_EXPIRED,
|
|
311
334
|
payload: {
|
|
@@ -315,7 +338,12 @@ async function touchRunnerSessionLiveness(params) {
|
|
|
315
338
|
jobExecutionId: row.jobExecutionId
|
|
316
339
|
}
|
|
317
340
|
})));
|
|
318
|
-
return deleted
|
|
341
|
+
return deleted.map(({ workflowRunId, workflowRunAttemptId, jobId, jobExecutionId })=>({
|
|
342
|
+
workflowRunId,
|
|
343
|
+
workflowRunAttemptId,
|
|
344
|
+
jobId,
|
|
345
|
+
jobExecutionId
|
|
346
|
+
}));
|
|
319
347
|
});
|
|
320
348
|
if (reaped.length > 0) jobExecutionLeaseExpiredCount.add(reaped.length);
|
|
321
349
|
return reaped;
|
|
@@ -453,8 +481,8 @@ export async function recordHeartbeat(params) {
|
|
|
453
481
|
}
|
|
454
482
|
/**
|
|
455
483
|
* Reconciles a terminal job execution with runner state in one transaction: removes its pending
|
|
456
|
-
* queue row
|
|
457
|
-
* idempotent and preserves the first cancellation-request timestamp.
|
|
484
|
+
* queue row, requests cancellation on its running lease, and converges any linked reservation.
|
|
485
|
+
* The operation is idempotent and preserves the first cancellation-request timestamp.
|
|
458
486
|
*/ export async function reconcileTerminalJobExecution(params) {
|
|
459
487
|
await db().transaction(async (tx)=>{
|
|
460
488
|
await lockJobExecution(tx, params.jobExecutionId);
|
|
@@ -462,18 +490,13 @@ export async function recordHeartbeat(params) {
|
|
|
462
490
|
// pending rows with SKIP LOCKED before inserting the running lease, so this ordering makes
|
|
463
491
|
// a concurrent terminal reconciliation either win before claim or cancel the new lease.
|
|
464
492
|
await tx.delete(pendingJobExecutions).where(eq(pendingJobExecutions.jobExecutionId, params.jobExecutionId));
|
|
465
|
-
await tx.update(runningJobExecutions).set({
|
|
493
|
+
const cancelledRunningRows = await tx.update(runningJobExecutions).set({
|
|
466
494
|
cancellationRequestedAt: sql`COALESCE(${runningJobExecutions.cancellationRequestedAt}, now())`
|
|
467
|
-
}).where(eq(runningJobExecutions.jobExecutionId, params.jobExecutionId))
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
await db().transaction(async (tx)=>{
|
|
473
|
-
await tx.delete(pendingJobExecutions).where(inArray(pendingJobExecutions.jobId, params.jobIds));
|
|
474
|
-
await tx.update(runningJobExecutions).set({
|
|
475
|
-
cancellationRequestedAt: sql`COALESCE(${runningJobExecutions.cancellationRequestedAt}, now())`
|
|
476
|
-
}).where(inArray(runningJobExecutions.jobId, params.jobIds));
|
|
495
|
+
}).where(eq(runningJobExecutions.jobExecutionId, params.jobExecutionId)).returning({
|
|
496
|
+
provisionerId: runningJobExecutions.provisionerId,
|
|
497
|
+
providerRunnerId: runningJobExecutions.providerRunnerId
|
|
498
|
+
});
|
|
499
|
+
await releaseReservationsForTerminalRunningRows(tx, cancelledRunningRows);
|
|
477
500
|
});
|
|
478
501
|
}
|
|
479
502
|
|