@shipfox/api-runners 13.0.0 → 14.0.0

package/dist/db/job-executions.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/db/job-executions.ts"],"sourcesContent":["import {\n  RUNNER_JOB_CLAIMED,\n  RUNNER_JOB_LEASE_EXPIRED,\n  type RunnersEventMap,\n  type RunnerToolCapabilitiesDto,\n} from '@shipfox/api-runners-dto';\nimport {logger} from '@shipfox/node-opentelemetry';\nimport {writeOutboxEvent, writeOutboxEvents} from '@shipfox/node-outbox';\nimport {canonicalizeLabels} from '@shipfox/runner-labels';\nimport {\n  and,\n  arrayContained,\n  asc,\n  count,\n  desc,\n  eq,\n  gt,\n  inArray,\n  isNotNull,\n  isNull,\n  lt,\n  lte,\n  or,\n  sql,\n} from 'drizzle-orm';\nimport {\n  EmptyRequiredLabelsError,\n  RunnerSessionExhaustedError,\n  RunningJobExecutionNotFoundError,\n} from '#core/errors.js';\nimport {\n  type JobExecutionQueueTimeObservation,\n  jobExecutionEnqueuedCount,\n  jobExecutionLeaseExpiredCount,\n  type ProviderRunnerLifecycleObservation,\n  recordJobExecutionQueueTime,\n  recordProviderRunnerActivationToFirstClaim,\n} from '#metrics/instance.js';\nimport type {Tx} from './db.js';\nimport {db} from './db.js';\nimport {releaseTerminalRunnerInstanceReservationsByIds} from './reservations.js';\nimport {runnersOutbox} from './schema/outbox.js';\nimport {pendingJobExecutions} from './schema/pending-job-executions.js';\nimport {providerRunners} from './schema/runner-instances.js';\nimport {runnerSessions} from './schema/runner-sessions.js';\nimport {runningJobExecutions} from './schema/running-job-executions.js';\n\nconst runnerJobExecutionLockPrefix = 'runners_job_execution:';\n\nasync function lockJobExecution(tx: Tx, jobExecutionId: string): Promise<void> {\n  await tx.execute(\n    sql`select pg_advisory_xact_lock(hashtext(${`${runnerJobExecutionLockPrefix}${jobExecutionId}`}))`,\n  );\n}\n\nasync function releaseReservationsForTerminalRunningRows(\n  tx: Tx,\n  rows: ReadonlyArray<{\n    provisionerId: string | null;\n    providerRunnerId: string | null;\n  }>,\n): Promise<void> {\n  // Converge after the lease fact changes. The reservation helper locks each provider runner\n  // and releases only when the runner is terminal and no uncancelled lease remains.\n  const providerRunnerIdsByProvisionerId = new Map<string, Set<string>>();\n  for (const row of rows) {\n    if (row.provisionerId === null || row.providerRunnerId === null) continue;\n    const providerRunnerIds =\n      providerRunnerIdsByProvisionerId.get(row.provisionerId) ?? new Set<string>();\n    providerRunnerIds.add(row.providerRunnerId);\n    providerRunnerIdsByProvisionerId.set(row.provisionerId, providerRunnerIds);\n  }\n\n  for (const provisionerId of [...providerRunnerIdsByProvisionerId.keys()].sort()) {\n    const providerRunnerIds = providerRunnerIdsByProvisionerId.get(provisionerId);\n    if (!providerRunnerIds) continue;\n    await releaseTerminalRunnerInstanceReservationsByIds(tx, {\n      workspaceId: null,\n      provisionerId,\n      providerRunnerIds: [...providerRunnerIds].sort(),\n      requireUnlinkedSession: false,\n      // Lock and re-check the runner row locally so lease finalization remains retryable\n      // without a workflow/runner scope lock.\n      requireTerminalState: false,\n    });\n  }\n}\n\nexport interface EnqueueJobExecutionParams {\n  workspaceId: string;\n  workflowRunId: string;\n  workflowRunAttemptId: string;\n  jobId: string;\n  jobExecutionId: string;\n  projectId: string;\n  requiredLabels: string[];\n  queuedAt: Date;\n}\n\nexport async function getWorkspaceJobCounts(params: {\n  workspaceIds: string[];\n}): Promise<Array<{workspaceId: string; queued: number; running: number}>> {\n  const [pendingRows, runningRows] = await Promise.all([\n    db()\n      .select({workspaceId: pendingJobExecutions.workspaceId, count: count()})\n      .from(pendingJobExecutions)\n      .where(inArray(pendingJobExecutions.workspaceId, params.workspaceIds))\n      .groupBy(pendingJobExecutions.workspaceId),\n    db()\n      .select({workspaceId: runningJobExecutions.workspaceId, count: count()})\n      .from(runningJobExecutions)\n      .where(inArray(runningJobExecutions.workspaceId, params.workspaceIds))\n      .groupBy(runningJobExecutions.workspaceId),\n  ]);\n\n  const queuedByWorkspace = new Map(pendingRows.map((row) => [row.workspaceId, Number(row.count)]));\n  const runningByWorkspace = new Map(\n    runningRows.map((row) => [row.workspaceId, Number(row.count)]),\n  );\n\n  return params.workspaceIds.map((workspaceId) => ({\n    workspaceId,\n    queued: queuedByWorkspace.get(workspaceId) ?? 0,\n    running: runningByWorkspace.get(workspaceId) ?? 0,\n  }));\n}\n\n// The workflows outbox delivers queue facts at least once. The per-execution advisory lock and\n// pending/running checks make replay a no-op across the claim transition. A durable lease-expired\n// event also prevents a delayed replay from resurrecting an execution already reaped locally.\nexport async function enqueueJobExecution(params: EnqueueJobExecutionParams): Promise<void> {\n  const requiredLabels = [...canonicalizeLabels(params.requiredLabels)];\n  if (requiredLabels.length === 0) throw new EmptyRequiredLabelsError();\n\n  const enqueued = await db().transaction(async (tx) => {\n    await lockJobExecution(tx, params.jobExecutionId);\n\n    const [running] = await tx\n      .select({jobExecutionId: runningJobExecutions.jobExecutionId})\n      .from(runningJobExecutions)\n      .where(eq(runningJobExecutions.jobExecutionId, params.jobExecutionId))\n      .limit(1);\n    if (running) return false;\n\n    const [leaseExpired] = await tx\n      .select({id: runnersOutbox.id})\n      .from(runnersOutbox)\n      .where(\n        and(\n          eq(runnersOutbox.eventType, RUNNER_JOB_LEASE_EXPIRED),\n          sql`${runnersOutbox.payload}->>'jobExecutionId' = ${params.jobExecutionId}`,\n        ),\n      )\n      .limit(1);\n    if (leaseExpired) return false;\n\n    const [inserted] = await tx\n      .insert(pendingJobExecutions)\n      .values({\n        workspaceId: params.workspaceId,\n        workflowRunId: params.workflowRunId,\n        workflowRunAttemptId: params.workflowRunAttemptId,\n        jobId: params.jobId,\n        jobExecutionId: params.jobExecutionId,\n        projectId: params.projectId,\n        requiredLabels,\n        createdAt: params.queuedAt,\n      })\n      .onConflictDoNothing({target: pendingJobExecutions.jobExecutionId})\n      .returning({createdAt: pendingJobExecutions.createdAt});\n\n    return inserted !== undefined;\n  });\n\n  if (enqueued) jobExecutionEnqueuedCount.add(1);\n}\n\nexport interface ClaimedJobExecution {\n  workflowRunId: string;\n  workflowRunAttemptId: string;\n  jobId: string;\n  jobExecutionId: string;\n  projectId: string;\n}\n\nexport interface ActiveRunningJobExecution {\n  workflowRunId: string;\n  workflowRunAttemptId: string;\n  jobId: string;\n  jobExecutionId: string;\n  projectId: string;\n  runnerSessionId: string;\n  provisionerId: string | null;\n  providerRunnerId: string | null;\n  requiredLabels: string[];\n  runnerLabels: string[];\n  startedAt: Date;\n  lastHeartbeatAt: Date;\n}\n\nexport interface RunnerInstanceBoundJobExecution {\n  workflowRunId: string;\n  workflowRunAttemptId: string;\n  jobId: string;\n  jobExecutionId: string;\n  providerRunnerId: string;\n  startedAt: Date;\n  lastHeartbeatAt: Date;\n  cancellationRequestedAt: Date | null;\n}\n\nexport async function claimPendingJobExecution(params: {\n  workspaceId: string;\n  runnerSessionId: string;\n  sessionLabels: string[];\n  maxClaims: number | null;\n  runnerSessionLivenessThrottleSeconds: number;\n}): Promise<ClaimedJobExecution | null> {\n  await touchRunnerSessionLiveness({\n    workspaceId: params.workspaceId,\n    runnerSessionId: params.runnerSessionId,\n    throttleSeconds: params.runnerSessionLivenessThrottleSeconds,\n  });\n\n  if (params.sessionLabels.length === 0) return null;\n\n  let activationToFirstClaimObservation: ProviderRunnerLifecycleObservation | null = null;\n  let queueTimeObservation: JobExecutionQueueTimeObservation | null = null;\n  const result = await db().transaction(async (tx) => {\n    let provisionerId: string | null = null;\n    let providerRunnerId: string | null = null;\n    let runnerInstanceId: string | null = null;\n\n    if (params.maxClaims !== null) {\n      const [session] = await tx\n        .select({\n          maxClaims: runnerSessions.maxClaims,\n          claimsUsed: runnerSessions.claimsUsed,\n          revokedAt: runnerSessions.revokedAt,\n          runnerInstanceId: runnerSessions.runnerInstanceId,\n          provisionerId: runnerSessions.provisionerId,\n          providerRunnerId: runnerSessions.providerRunnerId,\n        })\n        .from(runnerSessions)\n        .where(eq(runnerSessions.id, params.runnerSessionId))\n        .limit(1)\n        .for('update');\n\n      if (\n        !session ||\n        session.revokedAt ||\n        session.maxClaims === null ||\n        session.claimsUsed >= session.maxClaims\n      ) {\n        throw new RunnerSessionExhaustedError(params.runnerSessionId);\n      }\n\n      // Ephemeral sessions are the only capped sessions, and the DB check keeps\n      // their provisioned-runner link present as a pair.\n      runnerInstanceId = session.runnerInstanceId;\n      provisionerId = session.provisionerId;\n      providerRunnerId = session.providerRunnerId;\n    }\n\n    // `id` is a uuidv7 (time-ordered), so it is a deterministic FIFO tiebreaker\n    // for rows sharing a created_at within a batch. Lock only the FIFO candidate before\n    // attempting its execution advisory lock; putting pg_try_advisory_xact_lock in this\n    // predicate would evaluate it while scanning and temporarily lock many queue entries.\n    const candidate = tx\n      .select()\n      .from(pendingJobExecutions)\n      .where(\n        and(\n          eq(pendingJobExecutions.workspaceId, params.workspaceId),\n          arrayContained(pendingJobExecutions.requiredLabels, params.sessionLabels),\n        ),\n      )\n      .orderBy(asc(pendingJobExecutions.createdAt), asc(pendingJobExecutions.id))\n      .limit(1)\n      .for('update', {skipLocked: true})\n      .as('pending_candidate');\n\n    const [row] = await tx\n      .select()\n      .from(candidate)\n      .where(\n        sql`\n          pg_try_advisory_xact_lock(\n            hashtext(${runnerJobExecutionLockPrefix} || ${candidate.jobExecutionId}::text)\n          )\n        `,\n      );\n\n    if (!row) return null;\n\n    await tx.delete(pendingJobExecutions).where(eq(pendingJobExecutions.id, row.id));\n\n    // An enqueue retry that lands after a prior claim can leave an orphan pending\n    // row whose jobExecutionId is already in `runners_running_jobs`. Insert-or-skip\n    // on the jobExecutionId unique constraint: when the job execution is already running\n    // the insert touches no row, so we commit the orphan's deletion and return\n    // null rather than let the unique violation roll the claim back into a poison\n    // loop. The runner just re-polls for a real job execution.\n    const inserted = await tx\n      .insert(runningJobExecutions)\n      .values({\n        workspaceId: row.workspaceId,\n        workflowRunId: row.workflowRunId,\n        workflowRunAttemptId: row.workflowRunAttemptId,\n        jobId: row.jobId,\n        jobExecutionId: row.jobExecutionId,\n        projectId: row.projectId,\n        runnerSessionId: params.runnerSessionId,\n        provisionerId,\n        providerRunnerId,\n        requiredLabels: row.requiredLabels,\n        runnerLabels: params.sessionLabels,\n      })\n      .onConflictDoNothing({target: runningJobExecutions.jobExecutionId})\n      .returning({claimedAt: runningJobExecutions.startedAt});\n\n    const claimed = inserted[0];\n    if (!claimed) return null;\n\n    queueTimeObservation = {\n      durationMilliseconds: claimed.claimedAt.getTime() - row.createdAt.getTime(),\n      provider: null,\n      launchKind: params.maxClaims === null ? 'manual' : 'unknown',\n    };\n\n    const runnerInstanceCondition = runnerInstanceId\n      ? eq(providerRunners.id, runnerInstanceId)\n      : provisionerId && providerRunnerId\n        ? and(\n            eq(providerRunners.provisionerId, provisionerId),\n            eq(providerRunners.providerRunnerId, providerRunnerId),\n          )\n        : null;\n    if (runnerInstanceCondition) {\n      const [claimedRunner] = await tx\n        .update(providerRunners)\n        .set({\n          firstClaimedAt: sql`coalesce(${providerRunners.firstClaimedAt}, ${claimed.claimedAt})`,\n          updatedAt: sql`now()`,\n        })\n        .where(runnerInstanceCondition)\n        .returning({\n          firstClaimedAt: providerRunners.firstClaimedAt,\n          isFirstClaim: sql<boolean>`${providerRunners.firstClaimedAt} = ${claimed.claimedAt}`,\n          provider: providerRunners.providerKind,\n          launchKind: providerRunners.launchKind,\n          runnerInstanceId: providerRunners.id,\n          sessionCreatedAtEpochMs: sql<number | null>`(\n            select extract(epoch from ${runnerSessions.createdAt})::double precision * 1000\n            from ${runnerSessions}\n            where ${runnerSessions.id} = ${params.runnerSessionId}\n          )`,\n        });\n      if (claimedRunner && queueTimeObservation) {\n        queueTimeObservation.provider = claimedRunner.provider;\n        queueTimeObservation.launchKind = claimedRunner.launchKind;\n      }\n      if (\n        claimedRunner?.isFirstClaim &&\n        claimedRunner.firstClaimedAt !== null &&\n        claimedRunner.sessionCreatedAtEpochMs !== null &&\n        claimedRunner.sessionCreatedAtEpochMs !== undefined\n      )\n        activationToFirstClaimObservation = {\n          durationMilliseconds:\n            claimedRunner.firstClaimedAt.getTime() - claimedRunner.sessionCreatedAtEpochMs,\n          provider: claimedRunner.provider,\n          launchKind: claimedRunner.launchKind,\n          runnerInstanceId: claimedRunner.runnerInstanceId,\n        };\n    }\n\n    if (params.maxClaims !== null) {\n      await tx\n        .update(runnerSessions)\n        .set({claimsUsed: sql`${runnerSessions.claimsUsed} + 1`, updatedAt: sql`now()`})\n        .where(eq(runnerSessions.id, params.runnerSessionId));\n    }\n\n    // The running-row insert is the runner claiming the job execution. Emit in the same tx; the\n    // payload carries the row's own claim instant so a consumer records the true time,\n    // not the outbox drain time.\n    await writeOutboxEvent<RunnersEventMap>(tx, runnersOutbox, {\n      type: RUNNER_JOB_CLAIMED,\n      payload: {\n        workflowRunId: row.workflowRunId,\n        workflowRunAttemptId: row.workflowRunAttemptId,\n        jobId: row.jobId,\n        jobExecutionId: row.jobExecutionId,\n        claimedAt: claimed.claimedAt.toISOString(),\n      },\n    });\n\n    return {\n      workflowRunId: row.workflowRunId,\n      workflowRunAttemptId: row.workflowRunAttemptId,\n      jobId: row.jobId,\n      jobExecutionId: row.jobExecutionId,\n      projectId: row.projectId,\n    };\n  });\n  if (queueTimeObservation) recordJobExecutionQueueTime(queueTimeObservation);\n  if (activationToFirstClaimObservation)\n    recordProviderRunnerActivationToFirstClaim(activationToFirstClaimObservation);\n  return result;\n}\n\nasync function touchRunnerSessionLiveness(params: {\n  workspaceId: string;\n  runnerSessionId: string;\n  throttleSeconds: number;\n}): Promise<void> {\n  await db()\n    .update(runnerSessions)\n    .set({updatedAt: sql`now()`})\n    .where(\n      and(\n        eq(runnerSessions.id, params.runnerSessionId),\n        eq(runnerSessions.workspaceId, params.workspaceId),\n        sql`${runnerSessions.updatedAt} < now() - (${params.throttleSeconds} || ' seconds')::interval`,\n      ),\n    );\n}\n\n/**\n * Reaps stale leases (bounded by `limit`), emitting one\n * `runners.job.lease_expired` event per reaped job execution.\n *\n * The cutoff is re-checked in the DELETE, not just the locking subquery, so a\n * heartbeat landing mid-call spares the live row. Each reaped execution also sweeps its\n * pending row so an at-least-once queue replay cannot leave an orphan.\n *\n * Locks pending-then-running to match `claimPendingJobExecution` and\n * `reconcileTerminalJobExecution`. The stale candidate scan intentionally does not lock\n * running rows first; the running-row DELETE re-checks the stale predicate after the\n * pending-row sweep.\n */\nexport async function expireStuckJobExecutions(params: {\n  thresholdSeconds: number;\n  noFirstHeartbeatGraceSeconds: number;\n  limit?: number;\n}): Promise<\n  Array<{\n    workflowRunId: string;\n    workflowRunAttemptId: string;\n    jobId: string;\n    jobExecutionId: string;\n  }>\n> {\n  const reaped = await db().transaction(async (tx) => {\n    const heartbeatCutoff = sql`now() - (${params.thresholdSeconds} || ' seconds')::interval`;\n    const firstHeartbeatCutoff = sql`now() - (${params.noFirstHeartbeatGraceSeconds} || ' seconds')::interval`;\n    const stalePredicate = or(\n      and(\n        isNull(runningJobExecutions.firstHeartbeatAt),\n        lte(runningJobExecutions.lastHeartbeatAt, runningJobExecutions.startedAt),\n        lt(runningJobExecutions.startedAt, firstHeartbeatCutoff),\n      ),\n      and(\n        or(\n          isNotNull(runningJobExecutions.firstHeartbeatAt),\n          gt(runningJobExecutions.lastHeartbeatAt, runningJobExecutions.startedAt),\n        ),\n        lt(runningJobExecutions.lastHeartbeatAt, heartbeatCutoff),\n      ),\n    );\n\n    const staleRows = await tx\n      .select({\n        id: runningJobExecutions.id,\n        jobExecutionId: runningJobExecutions.jobExecutionId,\n      })\n      .from(runningJobExecutions)\n      .where(\n        and(\n          stalePredicate,\n          sql`\n            pg_try_advisory_xact_lock(\n              hashtext(${runnerJobExecutionLockPrefix} || ${runningJobExecutions.jobExecutionId}::text)\n            )\n          `,\n        ),\n      )\n      .orderBy(\n        asc(\n          sql`CASE WHEN ${runningJobExecutions.firstHeartbeatAt} IS NULL AND ${runningJobExecutions.lastHeartbeatAt} <= ${runningJobExecutions.startedAt} THEN ${runningJobExecutions.startedAt} ELSE ${runningJobExecutions.lastHeartbeatAt} END`,\n        ),\n        asc(runningJobExecutions.id),\n      )\n      .limit(params.limit ?? 100);\n\n    if (staleRows.length === 0) return [];\n\n    const staleIds = staleRows.map((row) => row.id);\n    const staleJobExecutionIds = staleRows.map((row) => row.jobExecutionId);\n\n    // Sweep pending rows first so this transaction cannot hold a running-row lock while waiting\n    // for a pending-row lock held by reconciliation or a claim of an orphan pending row. The\n    // advisory lock acquired in the candidate scan also makes enqueue retries wait until this\n    // transaction has either reaped the execution or released its lock without selecting it.\n    await tx\n      .delete(pendingJobExecutions)\n      .where(inArray(pendingJobExecutions.jobExecutionId, staleJobExecutionIds));\n\n    const deleted = await tx\n      .delete(runningJobExecutions)\n      .where(and(inArray(runningJobExecutions.id, staleIds), stalePredicate))\n      .returning({\n        workflowRunId: runningJobExecutions.workflowRunId,\n        workflowRunAttemptId: runningJobExecutions.workflowRunAttemptId,\n        jobId: runningJobExecutions.jobId,\n        jobExecutionId: runningJobExecutions.jobExecutionId,\n        provisionerId: runningJobExecutions.provisionerId,\n        providerRunnerId: runningJobExecutions.providerRunnerId,\n      });\n\n    if (deleted.length === 0) return [];\n\n    await releaseReservationsForTerminalRunningRows(tx, deleted);\n\n    await writeOutboxEvents<RunnersEventMap>(\n      tx,\n      runnersOutbox,\n      deleted.map((row) => ({\n        type: RUNNER_JOB_LEASE_EXPIRED,\n        payload: {\n          workflowRunId: row.workflowRunId,\n          workflowRunAttemptId: row.workflowRunAttemptId,\n          jobId: row.jobId,\n          jobExecutionId: row.jobExecutionId,\n        },\n      })),\n    );\n\n    return deleted.map(({workflowRunId, workflowRunAttemptId, jobId, jobExecutionId}) => ({\n      workflowRunId,\n      workflowRunAttemptId,\n      jobId,\n      jobExecutionId,\n    }));\n  });\n\n  if (reaped.length > 0) jobExecutionLeaseExpiredCount.add(reaped.length);\n\n  return reaped;\n}\n\nexport async function getJobExecutionQueueDepth(): Promise<{\n  pendingJobExecutions: number;\n  runningJobExecutions: number;\n}> {\n  const [pending] = await db().select({value: count()}).from(pendingJobExecutions);\n  const [running] = await db().select({value: count()}).from(runningJobExecutions);\n  return {\n    pendingJobExecutions: pending?.value ?? 0,\n    runningJobExecutions: running?.value ?? 0,\n  };\n}\n\nexport async function listActiveRunningJobExecutions(params: {\n  workspaceId: string;\n  windowSeconds: number;\n  limit?: number;\n}): Promise<ActiveRunningJobExecution[]> {\n  return await db()\n    .select({\n      jobId: runningJobExecutions.jobId,\n      jobExecutionId: runningJobExecutions.jobExecutionId,\n      workflowRunId: runningJobExecutions.workflowRunId,\n      workflowRunAttemptId: runningJobExecutions.workflowRunAttemptId,\n      projectId: runningJobExecutions.projectId,\n      runnerSessionId: runningJobExecutions.runnerSessionId,\n      provisionerId: runningJobExecutions.provisionerId,\n      providerRunnerId: runningJobExecutions.providerRunnerId,\n      requiredLabels: runningJobExecutions.requiredLabels,\n      runnerLabels: runningJobExecutions.runnerLabels,\n      startedAt: runningJobExecutions.startedAt,\n      lastHeartbeatAt: runningJobExecutions.lastHeartbeatAt,\n    })\n    .from(runningJobExecutions)\n    .where(\n      and(\n        eq(runningJobExecutions.workspaceId, params.workspaceId),\n        sql`${runningJobExecutions.lastHeartbeatAt} > now() - (${params.windowSeconds} || ' seconds')::interval`,\n      ),\n    )\n    .orderBy(desc(runningJobExecutions.lastHeartbeatAt), desc(runningJobExecutions.id))\n    .limit(params.limit ?? 1000);\n}\n\nexport async function listRunningJobExecutionsByRunnerInstanceTx(\n  tx: Tx,\n  params: {\n    workspaceId: string;\n    provisionerId: string;\n    providerRunnerIds: string[];\n  },\n): Promise<RunnerInstanceBoundJobExecution[]> {\n  if (params.providerRunnerIds.length === 0) return [];\n\n  const duplicateRows = await tx\n    .select({\n      providerRunnerId: runningJobExecutions.providerRunnerId,\n      count: count(),\n    })\n    .from(runningJobExecutions)\n    .where(\n      and(\n        eq(runningJobExecutions.workspaceId, params.workspaceId),\n        eq(runningJobExecutions.provisionerId, params.provisionerId),\n        inArray(runningJobExecutions.providerRunnerId, params.providerRunnerIds),\n      ),\n    )\n    .groupBy(runningJobExecutions.providerRunnerId)\n    .having(sql`count(*) > 1`);\n\n  const duplicateRunnerInstanceIds = duplicateRows.flatMap((row) =>\n    row.providerRunnerId ? [row.providerRunnerId] : [],\n  );\n  if (duplicateRunnerInstanceIds.length > 0) {\n    logger().warn(\n      {\n        workspaceId: params.workspaceId,\n        provisionerId: params.provisionerId,\n        providerRunnerIds: duplicateRunnerInstanceIds,\n      },\n      'multiple running job executions are bound to the same provisioned runner',\n    );\n  }\n\n  const result = await tx.execute<{\n    workflowRunId: string;\n    workflowRunAttemptId: string;\n    jobId: string;\n    jobExecutionId: string;\n    providerRunnerId: string;\n    startedAt: Date | string;\n    lastHeartbeatAt: Date | string;\n    cancellationRequestedAt: Date | string | null;\n  }>(sql`\n    SELECT DISTINCT ON (${runningJobExecutions.providerRunnerId})\n      ${runningJobExecutions.workflowRunId} AS \"workflowRunId\",\n      ${runningJobExecutions.workflowRunAttemptId} AS \"workflowRunAttemptId\",\n      ${runningJobExecutions.jobId} AS \"jobId\",\n      ${runningJobExecutions.jobExecutionId} AS \"jobExecutionId\",\n      ${runningJobExecutions.providerRunnerId} AS \"providerRunnerId\",\n      ${runningJobExecutions.startedAt} AS \"startedAt\",\n      ${runningJobExecutions.lastHeartbeatAt} AS \"lastHeartbeatAt\",\n      ${runningJobExecutions.cancellationRequestedAt} AS \"cancellationRequestedAt\"\n    FROM ${runningJobExecutions}\n    WHERE\n      ${runningJobExecutions.workspaceId} = ${params.workspaceId}\n      AND ${runningJobExecutions.provisionerId} = ${params.provisionerId}\n      AND ${runningJobExecutions.providerRunnerId} IN (${sql.join(\n        params.providerRunnerIds.map((providerRunnerId) => sql`${providerRunnerId}`),\n        sql`, `,\n      )})\n    ORDER BY ${runningJobExecutions.providerRunnerId}, ${runningJobExecutions.startedAt} DESC, ${runningJobExecutions.jobExecutionId} DESC\n  `);\n\n  return result.rows.map((row) => ({\n    workflowRunId: row.workflowRunId,\n    workflowRunAttemptId: row.workflowRunAttemptId,\n    jobId: row.jobId,\n    jobExecutionId: row.jobExecutionId,\n    providerRunnerId: row.providerRunnerId,\n    startedAt: toDate(row.startedAt),\n    lastHeartbeatAt: toDate(row.lastHeartbeatAt),\n    cancellationRequestedAt: row.cancellationRequestedAt\n      ? toDate(row.cancellationRequestedAt)\n      : null,\n  }));\n}\n\nfunction toDate(value: Date | string): Date {\n  return value instanceof Date ? value : new Date(value);\n}\n\nexport async function isJobLeaseActive(params: {\n  jobId?: string;\n  jobExecutionId: string;\n  runnerSessionId: string;\n}): Promise<boolean> {\n  const [row] = await db()\n    .select({id: runningJobExecutions.id})\n    .from(runningJobExecutions)\n    .where(\n      and(\n        eq(runningJobExecutions.jobExecutionId, params.jobExecutionId),\n        eq(runningJobExecutions.runnerSessionId, params.runnerSessionId),\n        params.jobId === undefined ? undefined : eq(runningJobExecutions.jobId, params.jobId),\n      ),\n    )\n    .limit(1);\n\n  return row !== undefined;\n}\n\nexport async function recordHeartbeat(params: {\n  jobExecutionId: string;\n  runnerSessionId: string;\n  toolCapabilities?: RunnerToolCapabilitiesDto | null;\n}): Promise<{\n  cancellationRequested: boolean;\n  previousToolCapabilities: RunnerToolCapabilitiesDto | null;\n  currentToolCapabilities: RunnerToolCapabilitiesDto | null;\n  runningJobExecution: {\n    workflowRunId: string;\n    workflowRunAttemptId: string;\n    jobId: string;\n    jobExecutionId: string;\n    projectId: string;\n    workspaceId: string;\n    runnerSessionId: string;\n  };\n}> {\n  const result = await db().transaction(async (tx) => {\n    const updated = await tx\n      .update(runningJobExecutions)\n      .set({\n        firstHeartbeatAt: sql`COALESCE(${runningJobExecutions.firstHeartbeatAt}, now())`,\n        lastHeartbeatAt: sql`now()`,\n      })\n      .where(\n        and(\n          eq(runningJobExecutions.jobExecutionId, params.jobExecutionId),\n          eq(runningJobExecutions.runnerSessionId, params.runnerSessionId),\n        ),\n      )\n      .returning({\n        cancellationRequestedAt: runningJobExecutions.cancellationRequestedAt,\n        workflowRunId: runningJobExecutions.workflowRunId,\n        workflowRunAttemptId: runningJobExecutions.workflowRunAttemptId,\n        jobId: runningJobExecutions.jobId,\n        jobExecutionId: runningJobExecutions.jobExecutionId,\n        projectId: runningJobExecutions.projectId,\n        workspaceId: runningJobExecutions.workspaceId,\n        runnerSessionId: runningJobExecutions.runnerSessionId,\n      });\n\n    const row = updated[0];\n    if (!row) throw new RunningJobExecutionNotFoundError(params.jobExecutionId);\n\n    const [previous] = await tx\n      .select({toolCapabilities: runnerSessions.toolCapabilities})\n      .from(runnerSessions)\n      .where(eq(runnerSessions.id, params.runnerSessionId))\n      .limit(1)\n      .for('update');\n\n    const [session] = await tx\n      .update(runnerSessions)\n      .set({\n        toolCapabilities: params.toolCapabilities ?? null,\n        toolCapabilitiesReportedAt: params.toolCapabilities ? sql`now()` : null,\n        updatedAt: sql`now()`,\n      })\n      .where(eq(runnerSessions.id, params.runnerSessionId))\n      .returning({toolCapabilities: runnerSessions.toolCapabilities});\n\n    return {\n      row,\n      previousToolCapabilities: previous?.toolCapabilities ?? null,\n      currentToolCapabilities: session?.toolCapabilities ?? null,\n    };\n  });\n\n  const row = result.row;\n  if (!row) throw new RunningJobExecutionNotFoundError(params.jobExecutionId);\n  return {\n    cancellationRequested: row.cancellationRequestedAt !== null,\n    previousToolCapabilities: result.previousToolCapabilities,\n    currentToolCapabilities: result.currentToolCapabilities,\n    runningJobExecution: {\n      workflowRunId: row.workflowRunId,\n      workflowRunAttemptId: row.workflowRunAttemptId,\n      jobId: row.jobId,\n      jobExecutionId: row.jobExecutionId,\n      projectId: row.projectId,\n      workspaceId: row.workspaceId,\n      runnerSessionId: row.runnerSessionId,\n    },\n  };\n}\n\n/**\n * Reconciles a terminal job execution with runner state in one transaction: removes its pending\n * queue row, requests cancellation on its running lease, and converges any linked reservation.\n * The operation is idempotent and preserves the first cancellation-request timestamp.\n */\nexport async function reconcileTerminalJobExecution(params: {\n  jobExecutionId: string;\n}): Promise<void> {\n  await db().transaction(async (tx) => {\n    await lockJobExecution(tx, params.jobExecutionId);\n\n    // Delete pending before updating running to match claim/release lock order. Claim locks\n    // pending rows with SKIP LOCKED before inserting the running lease, so this ordering makes\n    // a concurrent terminal reconciliation either win before claim or cancel the new lease.\n    await tx\n      .delete(pendingJobExecutions)\n      .where(eq(pendingJobExecutions.jobExecutionId, params.jobExecutionId));\n    const cancelledRunningRows = await tx\n      .update(runningJobExecutions)\n      .set({\n        cancellationRequestedAt: sql`COALESCE(${runningJobExecutions.cancellationRequestedAt}, now())`,\n      })\n      .where(eq(runningJobExecutions.jobExecutionId, params.jobExecutionId))\n      .returning({\n        provisionerId: runningJobExecutions.provisionerId,\n        providerRunnerId: runningJobExecutions.providerRunnerId,\n      });\n\n    await releaseReservationsForTerminalRunningRows(tx, cancelledRunningRows);\n  });\n}\n"],"names":["RUNNER_JOB_CLAIMED","RUNNER_JOB_LEASE_EXPIRED","logger","writeOutboxEvent","writeOutboxEvents","canonicalizeLabels","and","arrayContained","asc","count","desc","eq","gt","inArray","isNotNull","isNull","lt","lte","or","sql","EmptyRequiredLabelsError","RunnerSessionExhaustedError","RunningJobExecutionNotFoundError","jobExecutionEnqueuedCount","jobExecutionLeaseExpiredCount","recordJobExecutionQueueTime","recordProviderRunnerActivationToFirstClaim","db","releaseTerminalRunnerInstanceReservationsByIds","runnersOutbox","pendingJobExecutions","providerRunners","runnerSessions","runningJobExecutions","runnerJobExecutionLockPrefix","lockJobExecution","tx","jobExecutionId","execute","releaseReservationsForTerminalRunningRows","rows","providerRunnerIdsByProvisionerId","Map","row","provisionerId","providerRunnerId","providerRunnerIds","get","Set","add","set","keys","sort","workspaceId","requireUnlinkedSession","requireTerminalState","getWorkspaceJobCounts","params","pendingRows","runningRows","Promise","all","select","from","where","workspaceIds","groupBy","queuedByWorkspace","map","Number","runningByWorkspace","queued","running","enqueueJobExecution","requiredLabels","length","enqueued","transaction","limit","leaseExpired","id","eventType","payload","inserted","insert","values","workflowRunId","workflowRunAttemptId","jobId","projectId","createdAt","queuedAt","onConflictDoNothing","target","returning","undefined","claimPendingJobExecution","touchRunnerSessionLiveness","runnerSessionId","throttleSeconds","runnerSessionLivenessThrottleSeconds","sessionLabels","activationToFirstClaimObservation","queueTimeObservation","result","runnerInstanceId","maxClaims","session","claimsUsed","revokedAt","for","candidate","orderBy","skipLocked","as","delete","runnerLabels","claimedAt","startedAt","claimed","durationMilliseconds","getTime","provider","launchKind","runnerInstanceCondition","claimedRunner","update","firstClaimedAt","updatedAt","isFirstClaim","providerKind","sessionCreatedAtEpochMs","type","toISOString","expireStuckJobExecutions","reaped","heartbeatCutoff","thresholdSeconds","firstHeartbeatCutoff","noFirstHeartbeatGraceSeconds","stalePredicate","firstHeartbeatAt","lastHeartbeatAt","staleRows","staleIds","staleJobExecutionIds","deleted","getJobExecutionQueueDepth","pending","value","listActiveRunningJobExecutions","windowSeconds","listRunningJobExecutionsByRunnerInstanceTx","duplicateRows","having","duplicateRunnerInstanceIds","flatMap","warn","cancellationRequestedAt","join","toDate","Date","isJobLeaseActive","recordHeartbeat","updated","previous","toolCapabilities","toolCapabilitiesReportedAt","previousToolCapabilities","currentToolCapabilities","cancellationRequested","runningJobExecution","reconcileTerminalJobExecution","cancelledRunningRows"],"mappings":"AAAA,SACEA,kBAAkB,EAClBC,wBAAwB,QAGnB,2BAA2B;AAClC,SAAQC,MAAM,QAAO,8BAA8B;AACnD,SAAQC,gBAAgB,EAAEC,iBAAiB,QAAO,uBAAuB;AACzE,SAAQC,kBAAkB,QAAO,yBAAyB;AAC1D,SACEC,GAAG,EACHC,cAAc,EACdC,GAAG,EACHC,KAAK,EACLC,IAAI,EACJC,EAAE,EACFC,EAAE,EACFC,OAAO,EACPC,SAAS,EACTC,MAAM,EACNC,EAAE,EACFC,GAAG,EACHC,EAAE,EACFC,GAAG,QACE,cAAc;AACrB,SACEC,wBAAwB,EACxBC,2BAA2B,EAC3BC,gCAAgC,QAC3B,kBAAkB;AACzB,SAEEC,yBAAyB,EACzBC,6BAA6B,EAE7BC,2BAA2B,EAC3BC,0CAA0C,QACrC,uBAAuB;AAE9B,SAAQC,EAAE,QAAO,UAAU;AAC3B,SAAQC,8CAA8C,QAAO,oBAAoB;AACjF,SAAQC,aAAa,QAAO,qBAAqB;AACjD,SAAQC,oBAAoB,QAAO,qCAAqC;AACxE,SAAQC,eAAe,QAAO,+BAA+B;AAC7D,SAAQC,cAAc,QAAO,8BAA8B;AAC3D,SAAQC,oBAAoB,QAAO,qCAAqC;AAExE,MAAMC,+BAA+B;AAErC,eAAeC,iBAAiBC,EAAM,EAAEC,cAAsB;IAC5D,MAAMD,GAAGE,OAAO,CACdnB,GAAG,CAAC,sCAAsC,EAAE,GAAGe,+BAA+BG,gBAAgB,CAAC,EAAE,CAAC;AAEtG;AAEA,eAAeE,0CACbH,EAAM,EACNI,IAGE;IAEF,2FAA2F;IAC3F,kFAAkF;IAClF,MAAMC,mCAAmC,IAAIC;IAC7C,KAAK,MAAMC,OAAOH,KAAM;QACtB,IAAIG,IAAIC,aAAa,KAAK,QAAQD,IAAIE,gBAAgB,KAAK,MAAM;QACjE,MAAMC,oBACJL,iCAAiCM,GAAG,CAACJ,IAAIC,aAAa,KAAK,IAAII;QACjEF,kBAAkBG,GAAG,CAACN,IAAIE,gBAAgB;QAC1CJ,iCAAiCS,GAAG,CAACP,IAAIC,aAAa,EAAEE;IAC1D;IAEA,KAAK,MAAMF,iBAAiB;WAAIH,iCAAiCU,IAAI;KAAG,CAACC,IAAI,GAAI;QAC/E,MAAMN,oBAAoBL,iCAAiCM,GAAG,CAACH;QAC/D,IAAI,CAACE,mBAAmB;QACxB,MAAMlB,+CAA+CQ,IAAI;YACvDiB,aAAa;YACbT;YACAE,mBAAmB;mBAAIA;aAAkB,CAACM,IAAI;YAC9CE,wBAAwB;YACxB,mFAAmF;YACnF,wCAAwC;YACxCC,sBAAsB;QACxB;IACF;AACF;AAaA,OAAO,eAAeC,sBAAsBC,MAE3C;IACC,MAAM,CAACC,aAAaC,YAAY,GAAG,MAAMC,QAAQC,GAAG,CAAC;QACnDlC,KACGmC,MAAM,CAAC;YAACT,aAAavB,qBAAqBuB,WAAW;YAAE5C,OAAOA;QAAO,GACrEsD,IAAI,CAACjC,sBACLkC,KAAK,CAACnD,QAAQiB,qBAAqBuB,WAAW,EAAEI,OAAOQ,YAAY,GACnEC,OAAO,CAACpC,qBAAqBuB,WAAW;QAC3C1B,KACGmC,MAAM,CAAC;YAACT,aAAapB,qBAAqBoB,WAAW;YAAE5C,OAAOA;QAAO,GACrEsD,IAAI,CAAC9B,sBACL+B,KAAK,CAACnD,QAAQoB,qBAAqBoB,WAAW,EAAEI,OAAOQ,YAAY,GACnEC,OAAO,CAACjC,qBAAqBoB,WAAW;KAC5C;IAED,MAAMc,oBAAoB,IAAIzB,IAAIgB,YAAYU,GAAG,CAAC,CAACzB,MAAQ;YAACA,IAAIU,WAAW;YAAEgB,OAAO1B,IAAIlC,KAAK;SAAE;IAC/F,MAAM6D,qBAAqB,IAAI5B,IAC7BiB,YAAYS,GAAG,CAAC,CAACzB,MAAQ;YAACA,IAAIU,WAAW;YAAEgB,OAAO1B,IAAIlC,KAAK;SAAE;IAG/D,OAAOgD,OAAOQ,YAAY,CAACG,GAAG,CAAC,CAACf,cAAiB,CAAA;YAC/CA;YACAkB,QAAQJ,kBAAkBpB,GAAG,CAACM,gBAAgB;YAC9CmB,SAASF,mBAAmBvB,GAAG,CAACM,gBAAgB;QAClD,CAAA;AACF;AAEA,+FAA+F;AAC/F,kGAAkG;AAClG,8FAA8F;AAC9F,OAAO,eAAeoB,oBAAoBhB,MAAiC;IACzE,MAAMiB,iBAAiB;WAAIrE,mBAAmBoD,OAAOiB,cAAc;KAAE;IACrE,IAAIA,eAAeC,MAAM,KAAK,GAAG,MAAM,IAAIvD;IAE3C,MAAMwD,WAAW,MAAMjD,KAAKkD,WAAW,CAAC,OAAOzC;QAC7C,MAAMD,iBAAiBC,IAAIqB,OAAOpB,cAAc;QAEhD,MAAM,CAACmC,QAAQ,GAAG,MAAMpC,GACrB0B,MAAM,CAAC;YAACzB,gBAAgBJ,qBAAqBI,cAAc;QAAA,GAC3D0B,IAAI,CAAC9B,sBACL+B,KAAK,CAACrD,GAAGsB,qBAAqBI,cAAc,EAAEoB,OAAOpB,cAAc,GACnEyC,KAAK,CAAC;QACT,IAAIN,SAAS,OAAO;QAEpB,MAAM,CAACO,aAAa,GAAG,MAAM3C,GAC1B0B,MAAM,CAAC;YAACkB,IAAInD,cAAcmD,EAAE;QAAA,GAC5BjB,IAAI,CAAClC,eACLmC,KAAK,CACJ1D,IACEK,GAAGkB,cAAcoD,SAAS,EAAEhF,2BAC5BkB,GAAG,CAAC,EAAEU,cAAcqD,OAAO,CAAC,sBAAsB,EAAEzB,OAAOpB,cAAc,CAAC,CAAC,GAG9EyC,KAAK,CAAC;QACT,IAAIC,cAAc,OAAO;QAEzB,MAAM,CAACI,SAAS,GAAG,MAAM/C,GACtBgD,MAAM,CAACtD,sBACPuD,MAAM,CAAC;YACNhC,aAAaI,OAAOJ,WAAW;YAC/BiC,eAAe7B,OAAO6B,aAAa;YACnCC,sBAAsB9B,OAAO8B,oBAAoB;YACjDC,OAAO/B,OAAO+B,KAAK;YACnBnD,gBAAgBoB,OAAOpB,cAAc;YACrCoD,WAAWhC,OAAOgC,SAAS;YAC3Bf;YACAgB,WAAWjC,OAAOkC,QAAQ;QAC5B,GACCC,mBAAmB,CAAC;YAACC,QAAQ/D,qBAAqBO,cAAc;QAAA,GAChEyD,SAAS,CAAC;YAACJ,WAAW5D,qBAAqB4D,SAAS;QAAA;QAEvD,OAAOP,aAAaY;IACtB;IAEA,IAAInB,UAAUrD,0BAA0B0B,GAAG,CAAC;AAC9C;AAoCA,OAAO,eAAe+C,yBAAyBvC,MAM9C;IACC,MAAMwC,2BAA2B;QAC/B5C,aAAaI,OAAOJ,WAAW;QAC/B6C,iBAAiBzC,OAAOyC,eAAe;QACvCC,iBAAiB1C,OAAO2C,oCAAoC;IAC9D;IAEA,IAAI3C,OAAO4C,aAAa,CAAC1B,MAAM,KAAK,GAAG,OAAO;IAE9C,IAAI2B,oCAA+E;IACnF,IAAIC,uBAAgE;IACpE,MAAMC,SAAS,MAAM7E,KAAKkD,WAAW,CAAC,OAAOzC;QAC3C,IAAIQ,gBAA+B;QACnC,IAAIC,mBAAkC;QACtC,IAAI4D,mBAAkC;QAEtC,IAAIhD,OAAOiD,SAAS,KAAK,MAAM;YAC7B,MAAM,CAACC,QAAQ,GAAG,MAAMvE,GACrB0B,MAAM,CAAC;gBACN4C,WAAW1E,eAAe0E,SAAS;gBACnCE,YAAY5E,eAAe4E,UAAU;gBACrCC,WAAW7E,eAAe6E,SAAS;gBACnCJ,kBAAkBzE,eAAeyE,gBAAgB;gBACjD7D,eAAeZ,eAAeY,aAAa;gBAC3CC,kBAAkBb,eAAea,gBAAgB;YACnD,GACCkB,IAAI,CAAC/B,gBACLgC,KAAK,CAACrD,GAAGqB,eAAegD,EAAE,EAAEvB,OAAOyC,eAAe,GAClDpB,KAAK,CAAC,GACNgC,GAAG,CAAC;YAEP,IACE,CAACH,WACDA,QAAQE,SAAS,IACjBF,QAAQD,SAAS,KAAK,QACtBC,QAAQC,UAAU,IAAID,QAAQD,SAAS,EACvC;gBACA,MAAM,IAAIrF,4BAA4BoC,OAAOyC,eAAe;YAC9D;YAEA,0EAA0E;YAC1E,mDAAmD;YACnDO,mBAAmBE,QAAQF,gBAAgB;YAC3C7D,gBAAgB+D,QAAQ/D,aAAa;YACrCC,mBAAmB8D,QAAQ9D,gBAAgB;QAC7C;QAEA,4EAA4E;QAC5E,oFAAoF;QACpF,oFAAoF;QACpF,sFAAsF;QACtF,MAAMkE,YAAY3E,GACf0B,MAAM,GACNC,IAAI,CAACjC,sBACLkC,KAAK,CACJ1D,IACEK,GAAGmB,qBAAqBuB,WAAW,EAAEI,OAAOJ,WAAW,GACvD9C,eAAeuB,qBAAqB4C,cAAc,EAAEjB,OAAO4C,aAAa,IAG3EW,OAAO,CAACxG,IAAIsB,qBAAqB4D,SAAS,GAAGlF,IAAIsB,qBAAqBkD,EAAE,GACxEF,KAAK,CAAC,GACNgC,GAAG,CAAC,UAAU;YAACG,YAAY;QAAI,GAC/BC,EAAE,CAAC;QAEN,MAAM,CAACvE,IAAI,GAAG,MAAMP,GACjB0B,MAAM,GACNC,IAAI,CAACgD,WACL/C,KAAK,CACJ7C,GAAG,CAAC;;qBAES,EAAEe,6BAA6B,IAAI,EAAE6E,UAAU1E,cAAc,CAAC;;QAE3E,CAAC;QAGL,IAAI,CAACM,KAAK,OAAO;QAEjB,MAAMP,GAAG+E,MAAM,CAACrF,sBAAsBkC,KAAK,CAACrD,GAAGmB,qBAAqBkD,EAAE,EAAErC,IAAIqC,EAAE;QAE9E,8EAA8E;QAC9E,gFAAgF;QAChF,qFAAqF;QACrF,2EAA2E;QAC3E,8EAA8E;QAC9E,2DAA2D;QAC3D,MAAMG,WAAW,MAAM/C,GACpBgD,MAAM,CAACnD,sBACPoD,MAAM,CAAC;YACNhC,aAAaV,IAAIU,WAAW;YAC5BiC,eAAe3C,IAAI2C,aAAa;YAChCC,sBAAsB5C,IAAI4C,oBAAoB;YAC9CC,OAAO7C,IAAI6C,KAAK;YAChBnD,gBAAgBM,IAAIN,cAAc;YAClCoD,WAAW9C,IAAI8C,SAAS;YACxBS,iBAAiBzC,OAAOyC,eAAe;YACvCtD;YACAC;YACA6B,gBAAgB/B,IAAI+B,cAAc;YAClC0C,cAAc3D,OAAO4C,aAAa;QACpC,GACCT,mBAAmB,CAAC;YAACC,QAAQ5D,qBAAqBI,cAAc;QAAA,GAChEyD,SAAS,CAAC;YAACuB,WAAWpF,qBAAqBqF,SAAS;QAAA;QAEvD,MAAMC,UAAUpC,QAAQ,CAAC,EAAE;QAC3B,IAAI,CAACoC,SAAS,OAAO;QAErBhB,uBAAuB;YACrBiB,sBAAsBD,QAAQF,SAAS,CAACI,OAAO,KAAK9E,IAAI+C,SAAS,CAAC+B,OAAO;YACzEC,UAAU;YACVC,YAAYlE,OAAOiD,SAAS,KAAK,OAAO,WAAW;QACrD;QAEA,MAAMkB,0BAA0BnB,mBAC5B9F,GAAGoB,gBAAgBiD,EAAE,EAAEyB,oBACvB7D,iBAAiBC,mBACfvC,IACEK,GAAGoB,gBAAgBa,aAAa,EAAEA,gBAClCjC,GAAGoB,gBAAgBc,gBAAgB,EAAEA,qBAEvC;QACN,IAAI+E,yBAAyB;YAC3B,MAAM,CAACC,cAAc,GAAG,MAAMzF,GAC3B0F,MAAM,CAAC/F,iBACPmB,GAAG,CAAC;gBACH6E,gBAAgB5G,GAAG,CAAC,SAAS,EAAEY,gBAAgBgG,cAAc,CAAC,EAAE,EAAER,QAAQF,SAAS,CAAC,CAAC,CAAC;gBACtFW,WAAW7G,GAAG,CAAC,KAAK,CAAC;YACvB,GACC6C,KAAK,CAAC4D,yBACN9B,SAAS,CAAC;gBACTiC,gBAAgBhG,gBAAgBgG,cAAc;gBAC9CE,cAAc9G,GAAY,CAAC,EAAEY,gBAAgBgG,cAAc,CAAC,GAAG,EAAER,QAAQF,SAAS,CAAC,CAAC;gBACpFK,UAAU3F,gBAAgBmG,YAAY;gBACtCP,YAAY5F,gBAAgB4F,UAAU;gBACtClB,kBAAkB1E,gBAAgBiD,EAAE;gBACpCmD,yBAAyBhH,GAAkB,CAAC;sCAChB,EAAEa,eAAe0D,SAAS,CAAC;iBAChD,EAAE1D,eAAe;kBAChB,EAAEA,eAAegD,EAAE,CAAC,GAAG,EAAEvB,OAAOyC,eAAe,CAAC;WACvD,CAAC;YACJ;YACF,IAAI2B,iBAAiBtB,sBAAsB;gBACzCA,qBAAqBmB,QAAQ,GAAGG,cAAcH,QAAQ;gBACtDnB,qBAAqBoB,UAAU,GAAGE,cAAcF,UAAU;YAC5D;YACA,IACEE,eAAeI,gBACfJ,cAAcE,cAAc,KAAK,QACjCF,cAAcM,uBAAuB,KAAK,QAC1CN,cAAcM,uBAAuB,KAAKpC,WAE1CO,oCAAoC;gBAClCkB,sBACEK,cAAcE,cAAc,CAACN,OAAO,KAAKI,cAAcM,uBAAuB;gBAChFT,UAAUG,cAAcH,QAAQ;gBAChCC,YAAYE,cAAcF,UAAU;gBACpClB,kBAAkBoB,cAAcpB,gBAAgB;YAClD;QACJ;QAEA,IAAIhD,OAAOiD,SAAS,KAAK,MAAM;YAC7B,MAAMtE,GACH0F,MAAM,CAAC9F,gBACPkB,GAAG,CAAC;gBAAC0D,YAAYzF,GAAG,CAAC,EAAEa,eAAe4E,UAAU,CAAC,IAAI,CAAC;gBAAEoB,WAAW7G,GAAG,CAAC,KAAK,CAAC;YAAA,GAC7E6C,KAAK,CAACrD,GAAGqB,eAAegD,EAAE,EAAEvB,OAAOyC,eAAe;QACvD;QAEA,4FAA4F;QAC5F,mFAAmF;QACnF,6BAA6B;QAC7B,MAAM/F,iBAAkCiC,IAAIP,eAAe;YACzDuG,MAAMpI;YACNkF,SAAS;gBACPI,eAAe3C,IAAI2C,aAAa;gBAChCC,sBAAsB5C,IAAI4C,oBAAoB;gBAC9CC,OAAO7C,IAAI6C,KAAK;gBAChBnD,gBAAgBM,IAAIN,cAAc;gBAClCgF,WAAWE,QAAQF,SAAS,CAACgB,WAAW;YAC1C;QACF;QAEA,OAAO;YACL/C,eAAe3C,IAAI2C,aAAa;YAChCC,sBAAsB5C,IAAI4C,oBAAoB;YAC9CC,OAAO7C,IAAI6C,KAAK;YAChBnD,gBAAgBM,IAAIN,cAAc;YAClCoD,WAAW9C,IAAI8C,SAAS;QAC1B;IACF;IACA,IAAIc,sBAAsB9E,4BAA4B8E;IACtD,IAAID,mCACF5E,2CAA2C4E;IAC7C,OAAOE;AACT;AAEA,eAAeP,2BAA2BxC,MAIzC;IACC,MAAM9B,KACHmG,MAAM,CAAC9F,gBACPkB,GAAG,CAAC;QAAC8E,WAAW7G,GAAG,CAAC,KAAK,CAAC;IAAA,GAC1B6C,KAAK,CACJ1D,IACEK,GAAGqB,eAAegD,EAAE,EAAEvB,OAAOyC,eAAe,GAC5CvF,GAAGqB,eAAeqB,WAAW,EAAEI,OAAOJ,WAAW,GACjDlC,GAAG,CAAC,EAAEa,eAAegG,SAAS,CAAC,YAAY,EAAEvE,OAAO0C,eAAe,CAAC,yBAAyB,CAAC;AAGtG;AAEA;;;;;;;;;;;;CAYC,GACD,OAAO,eAAemC,yBAAyB7E,MAI9C;IAQC,MAAM8E,SAAS,MAAM5G,KAAKkD,WAAW,CAAC,OAAOzC;QAC3C,MAAMoG,kBAAkBrH,GAAG,CAAC,SAAS,EAAEsC,OAAOgF,gBAAgB,CAAC,yBAAyB,CAAC;QACzF,MAAMC,uBAAuBvH,GAAG,CAAC,SAAS,EAAEsC,OAAOkF,4BAA4B,CAAC,yBAAyB,CAAC;QAC1G,MAAMC,iBAAiB1H,GACrBZ,IACES,OAAOkB,qBAAqB4G,gBAAgB,GAC5C5H,IAAIgB,qBAAqB6G,eAAe,EAAE7G,qBAAqBqF,SAAS,GACxEtG,GAAGiB,qBAAqBqF,SAAS,EAAEoB,wBAErCpI,IACEY,GACEJ,UAAUmB,qBAAqB4G,gBAAgB,GAC/CjI,GAAGqB,qBAAqB6G,eAAe,EAAE7G,qBAAqBqF,SAAS,IAEzEtG,GAAGiB,qBAAqB6G,eAAe,EAAEN;QAI7C,MAAMO,YAAY,MAAM3G,GACrB0B,MAAM,CAAC;YACNkB,IAAI/C,qBAAqB+C,EAAE;YAC3B3C,gBAAgBJ,qBAAqBI,cAAc;QACrD,GACC0B,IAAI,CAAC9B,sBACL+B,KAAK,CACJ1D,IACEsI,gBACAzH,GAAG,CAAC;;uBAES,EAAEe,6BAA6B,IAAI,EAAED,qBAAqBI,cAAc,CAAC;;UAEtF,CAAC,GAGJ2E,OAAO,CACNxG,IACEW,GAAG,CAAC,UAAU,EAAEc,qBAAqB4G,gBAAgB,CAAC,aAAa,EAAE5G,qBAAqB6G,eAAe,CAAC,IAAI,EAAE7G,qBAAqBqF,SAAS,CAAC,MAAM,EAAErF,qBAAqBqF,SAAS,CAAC,MAAM,EAAErF,qBAAqB6G,eAAe,CAAC,IAAI,CAAC,GAE1OtI,IAAIyB,qBAAqB+C,EAAE,GAE5BF,KAAK,CAACrB,OAAOqB,KAAK,IAAI;QAEzB,IAAIiE,UAAUpE,MAAM,KAAK,GAAG,OAAO,EAAE;QAErC,MAAMqE,WAAWD,UAAU3E,GAAG,CAAC,CAACzB,MAAQA,IAAIqC,EAAE;QAC9C,MAAMiE,uBAAuBF,UAAU3E,GAAG,CAAC,CAACzB,MAAQA,IAAIN,cAAc;QAEtE,4FAA4F;QAC5F,yFAAyF;QACzF,0FAA0F;QAC1F,yFAAyF;QACzF,MAAMD,GACH+E,MAAM,CAACrF,sBACPkC,KAAK,CAACnD,QAAQiB,qBAAqBO,cAAc,EAAE4G;QAEtD,MAAMC,UAAU,MAAM9G,GACnB+E,MAAM,CAAClF,sBACP+B,KAAK,CAAC1D,IAAIO,QAAQoB,qBAAqB+C,EAAE,EAAEgE,WAAWJ,iBACtD9C,SAAS,CAAC;YACTR,eAAerD,qBAAqBqD,aAAa;YACjDC,sBAAsBtD,qBAAqBsD,oBAAoB;YAC/DC,OAAOvD,qBAAqBuD,KAAK;YACjCnD,gBAAgBJ,qBAAqBI,cAAc;YACnDO,eAAeX,qBAAqBW,aAAa;YACjDC,kBAAkBZ,qBAAqBY,gBAAgB;QACzD;QAEF,IAAIqG,QAAQvE,MAAM,KAAK,GAAG,OAAO,EAAE;QAEnC,MAAMpC,0CAA0CH,IAAI8G;QAEpD,MAAM9I,kBACJgC,IACAP,eACAqH,QAAQ9E,GAAG,CAAC,CAACzB,MAAS,CAAA;gBACpByF,MAAMnI;gBACNiF,SAAS;oBACPI,eAAe3C,IAAI2C,aAAa;oBAChCC,sBAAsB5C,IAAI4C,oBAAoB;oBAC9CC,OAAO7C,IAAI6C,KAAK;oBAChBnD,gBAAgBM,IAAIN,cAAc;gBACpC;YACF,CAAA;QAGF,OAAO6G,QAAQ9E,GAAG,CAAC,CAAC,EAACkB,aAAa,EAAEC,oBAAoB,EAAEC,KAAK,EAAEnD,cAAc,EAAC,GAAM,CAAA;gBACpFiD;gBACAC;gBACAC;gBACAnD;YACF,CAAA;IACF;IAEA,IAAIkG,OAAO5D,MAAM,GAAG,GAAGnD,8BAA8ByB,GAAG,CAACsF,OAAO5D,MAAM;IAEtE,OAAO4D;AACT;AAEA,OAAO,eAAeY;IAIpB,MAAM,CAACC,QAAQ,GAAG,MAAMzH,KAAKmC,MAAM,CAAC;QAACuF,OAAO5I;IAAO,GAAGsD,IAAI,CAACjC;IAC3D,MAAM,CAAC0C,QAAQ,GAAG,MAAM7C,KAAKmC,MAAM,CAAC;QAACuF,OAAO5I;IAAO,GAAGsD,IAAI,CAAC9B;IAC3D,OAAO;QACLH,sBAAsBsH,SAASC,SAAS;QACxCpH,sBAAsBuC,SAAS6E,SAAS;IAC1C;AACF;AAEA,OAAO,eAAeC,+BAA+B7F,MAIpD;IACC,OAAO,MAAM9B,KACVmC,MAAM,CAAC;QACN0B,OAAOvD,qBAAqBuD,KAAK;QACjCnD,gBAAgBJ,qBAAqBI,cAAc;QACnDiD,eAAerD,qBAAqBqD,aAAa;QACjDC,sBAAsBtD,qBAAqBsD,oBAAoB;QAC/DE,WAAWxD,qBAAqBwD,SAAS;QACzCS,iBAAiBjE,qBAAqBiE,eAAe;QACrDtD,eAAeX,qBAAqBW,aAAa;QACjDC,kBAAkBZ,qBAAqBY,gBAAgB;QACvD6B,gBAAgBzC,qBAAqByC,cAAc;QACnD0C,cAAcnF,qBAAqBmF,YAAY;QAC/CE,WAAWrF,qBAAqBqF,SAAS;QACzCwB,iBAAiB7G,qBAAqB6G,eAAe;IACvD,GACC/E,IAAI,CAAC9B,sBACL+B,KAAK,CACJ1D,IACEK,GAAGsB,qBAAqBoB,WAAW,EAAEI,OAAOJ,WAAW,GACvDlC,GAAG,CAAC,EAAEc,qBAAqB6G,eAAe,CAAC,YAAY,EAAErF,OAAO8F,aAAa,CAAC,yBAAyB,CAAC,GAG3GvC,OAAO,CAACtG,KAAKuB,qBAAqB6G,eAAe,GAAGpI,KAAKuB,qBAAqB+C,EAAE,GAChFF,KAAK,CAACrB,OAAOqB,KAAK,IAAI;AAC3B;AAEA,OAAO,eAAe0E,2CACpBpH,EAAM,EACNqB,MAIC;IAED,IAAIA,OAAOX,iBAAiB,CAAC6B,MAAM,KAAK,GAAG,OAAO,EAAE;IAEpD,MAAM8E,gBAAgB,MAAMrH,GACzB0B,MAAM,CAAC;QACNjB,kBAAkBZ,qBAAqBY,gBAAgB;QACvDpC,OAAOA;IACT,GACCsD,IAAI,CAAC9B,sBACL+B,KAAK,CACJ1D,IACEK,GAAGsB,qBAAqBoB,WAAW,EAAEI,OAAOJ,WAAW,GACvD1C,GAAGsB,qBAAqBW,aAAa,EAAEa,OAAOb,aAAa,GAC3D/B,QAAQoB,qBAAqBY,gBAAgB,EAAEY,OAAOX,iBAAiB,IAG1EoB,OAAO,CAACjC,qBAAqBY,gBAAgB,EAC7C6G,MAAM,CAACvI,GAAG,CAAC,YAAY,CAAC;IAE3B,MAAMwI,6BAA6BF,cAAcG,OAAO,CAAC,CAACjH,MACxDA,IAAIE,gBAAgB,GAAG;YAACF,IAAIE,gBAAgB;SAAC,GAAG,EAAE;IAEpD,IAAI8G,2BAA2BhF,MAAM,GAAG,GAAG;QACzCzE,SAAS2J,IAAI,CACX;YACExG,aAAaI,OAAOJ,WAAW;YAC/BT,eAAea,OAAOb,aAAa;YACnCE,mBAAmB6G;QACrB,GACA;IAEJ;IAEA,MAAMnD,SAAS,MAAMpE,GAAGE,OAAO,CAS5BnB,GAAG,CAAC;wBACe,EAAEc,qBAAqBY,gBAAgB,CAAC;MAC1D,EAAEZ,qBAAqBqD,aAAa,CAAC;MACrC,EAAErD,qBAAqBsD,oBAAoB,CAAC;MAC5C,EAAEtD,qBAAqBuD,KAAK,CAAC;MAC7B,EAAEvD,qBAAqBI,cAAc,CAAC;MACtC,EAAEJ,qBAAqBY,gBAAgB,CAAC;MACxC,EAAEZ,qBAAqBqF,SAAS,CAAC;MACjC,EAAErF,qBAAqB6G,eAAe,CAAC;MACvC,EAAE7G,qBAAqB6H,uBAAuB,CAAC;SAC5C,EAAE7H,qBAAqB;;MAE1B,EAAEA,qBAAqBoB,WAAW,CAAC,GAAG,EAAEI,OAAOJ,WAAW,CAAC;UACvD,EAAEpB,qBAAqBW,aAAa,CAAC,GAAG,EAAEa,OAAOb,aAAa,CAAC;UAC/D,EAAEX,qBAAqBY,gBAAgB,CAAC,KAAK,EAAE1B,IAAI4I,IAAI,CACzDtG,OAAOX,iBAAiB,CAACsB,GAAG,CAAC,CAACvB,mBAAqB1B,GAAG,CAAC,EAAE0B,iBAAiB,CAAC,GAC3E1B,GAAG,CAAC,EAAE,CAAC,EACP;aACK,EAAEc,qBAAqBY,gBAAgB,CAAC,EAAE,EAAEZ,qBAAqBqF,SAAS,CAAC,OAAO,EAAErF,qBAAqBI,cAAc,CAAC;EACnI,CAAC;IAED,OAAOmE,OAAOhE,IAAI,CAAC4B,GAAG,CAAC,CAACzB,MAAS,CAAA;YAC/B2C,eAAe3C,IAAI2C,aAAa;YAChCC,sBAAsB5C,IAAI4C,oBAAoB;YAC9CC,OAAO7C,IAAI6C,KAAK;YAChBnD,gBAAgBM,IAAIN,cAAc;YAClCQ,kBAAkBF,IAAIE,gBAAgB;YACtCyE,WAAW0C,OAAOrH,IAAI2E,SAAS;YAC/BwB,iBAAiBkB,OAAOrH,IAAImG,eAAe;YAC3CgB,yBAAyBnH,IAAImH,uBAAuB,GAChDE,OAAOrH,IAAImH,uBAAuB,IAClC;QACN,CAAA;AACF;AAEA,SAASE,OAAOX,KAAoB;IAClC,OAAOA,iBAAiBY,OAAOZ,QAAQ,IAAIY,KAAKZ;AAClD;AAEA,OAAO,eAAea,iBAAiBzG,MAItC;IACC,MAAM,CAACd,IAAI,GAAG,MAAMhB,KACjBmC,MAAM,CAAC;QAACkB,IAAI/C,qBAAqB+C,EAAE;IAAA,GACnCjB,IAAI,CAAC9B,sBACL+B,KAAK,CACJ1D,IACEK,GAAGsB,qBAAqBI,cAAc,EAAEoB,OAAOpB,cAAc,GAC7D1B,GAAGsB,qBAAqBiE,eAAe,EAAEzC,OAAOyC,eAAe,GAC/DzC,OAAO+B,KAAK,KAAKO,YAAYA,YAAYpF,GAAGsB,qBAAqBuD,KAAK,EAAE/B,OAAO+B,KAAK,IAGvFV,KAAK,CAAC;IAET,OAAOnC,QAAQoD;AACjB;AAEA,OAAO,eAAeoE,gBAAgB1G,MAIrC;IAcC,MAAM+C,SAAS,MAAM7E,KAAKkD,WAAW,CAAC,OAAOzC;QAC3C,MAAMgI,UAAU,MAAMhI,GACnB0F,MAAM,CAAC7F,sBACPiB,GAAG,CAAC;YACH2F,kBAAkB1H,GAAG,CAAC,SAAS,EAAEc,qBAAqB4G,gBAAgB,CAAC,QAAQ,CAAC;YAChFC,iBAAiB3H,GAAG,CAAC,KAAK,CAAC;QAC7B,GACC6C,KAAK,CACJ1D,IACEK,GAAGsB,qBAAqBI,cAAc,EAAEoB,OAAOpB,cAAc,GAC7D1B,GAAGsB,qBAAqBiE,eAAe,EAAEzC,OAAOyC,eAAe,IAGlEJ,SAAS,CAAC;YACTgE,yBAAyB7H,qBAAqB6H,uBAAuB;YACrExE,eAAerD,qBAAqBqD,aAAa;YACjDC,sBAAsBtD,qBAAqBsD,oBAAoB;YAC/DC,OAAOvD,qBAAqBuD,KAAK;YACjCnD,gBAAgBJ,qBAAqBI,cAAc;YACnDoD,WAAWxD,qBAAqBwD,SAAS;YACzCpC,aAAapB,qBAAqBoB,WAAW;YAC7C6C,iBAAiBjE,qBAAqBiE,eAAe;QACvD;QAEF,MAAMvD,MAAMyH,OAAO,CAAC,EAAE;QACtB,IAAI,CAACzH,KAAK,MAAM,IAAIrB,iCAAiCmC,OAAOpB,cAAc;QAE1E,MAAM,CAACgI,SAAS,GAAG,MAAMjI,GACtB0B,MAAM,CAAC;YAACwG,kBAAkBtI,eAAesI,gBAAgB;QAAA,GACzDvG,IAAI,CAAC/B,gBACLgC,KAAK,CAACrD,GAAGqB,eAAegD,EAAE,EAAEvB,OAAOyC,eAAe,GAClDpB,KAAK,CAAC,GACNgC,GAAG,CAAC;QAEP,MAAM,CAACH,QAAQ,GAAG,MAAMvE,GACrB0F,MAAM,CAAC9F,gBACPkB,GAAG,CAAC;YACHoH,kBAAkB7G,OAAO6G,gBAAgB,IAAI;YAC7CC,4BAA4B9G,OAAO6G,gBAAgB,GAAGnJ,GAAG,CAAC,KAAK,CAAC,GAAG;YACnE6G,WAAW7G,GAAG,CAAC,KAAK,CAAC;QACvB,GACC6C,KAAK,CAACrD,GAAGqB,eAAegD,EAAE,EAAEvB,OAAOyC,eAAe,GAClDJ,SAAS,CAAC;YAACwE,kBAAkBtI,eAAesI,gBAAgB;QAAA;QAE/D,OAAO;YACL3H;YACA6H,0BAA0BH,UAAUC,oBAAoB;YACxDG,yBAAyB9D,SAAS2D,oBAAoB;QACxD;IACF;IAEA,MAAM3H,MAAM6D,OAAO7D,GAAG;IACtB,IAAI,CAACA,KAAK,MAAM,IAAIrB,iCAAiCmC,OAAOpB,cAAc;IAC1E,OAAO;QACLqI,uBAAuB/H,IAAImH,uBAAuB,KAAK;QACvDU,0BAA0BhE,OAAOgE,wBAAwB;QACzDC,yBAAyBjE,OAAOiE,uBAAuB;QACvDE,qBAAqB;YACnBrF,eAAe3C,IAAI2C,aAAa;YAChCC,sBAAsB5C,IAAI4C,oBAAoB;YAC9CC,OAAO7C,IAAI6C,KAAK;YAChBnD,gBAAgBM,IAAIN,cAAc;YAClCoD,WAAW9C,IAAI8C,SAAS;YACxBpC,aAAaV,IAAIU,WAAW;YAC5B6C,iBAAiBvD,IAAIuD,eAAe;QACtC;IACF;AACF;AAEA;;;;CAIC,GACD,OAAO,eAAe0E,8BAA8BnH,MAEnD;IACC,MAAM9B,KAAKkD,WAAW,CAAC,OAAOzC;QAC5B,MAAMD,iBAAiBC,IAAIqB,OAAOpB,cAAc;QAEhD,wFAAwF;QACxF,2FAA2F;QAC3F,wFAAwF;QACxF,MAAMD,GACH+E,MAAM,CAACrF,sBACPkC,KAAK,CAACrD,GAAGmB,qBAAqBO,cAAc,EAAEoB,OAAOpB,cAAc;QACtE,MAAMwI,uBAAuB,MAAMzI,GAChC0F,MAAM,CAAC7F,sBACPiB,GAAG,CAAC;YACH4G,yBAAyB3I,GAAG,CAAC,SAAS,EAAEc,qBAAqB6H,uBAAuB,CAAC,QAAQ,CAAC;QAChG,GACC9F,KAAK,CAACrD,GAAGsB,qBAAqBI,cAAc,EAAEoB,OAAOpB,cAAc,GACnEyD,SAAS,CAAC;YACTlD,eAAeX,qBAAqBW,aAAa;YACjDC,kBAAkBZ,qBAAqBY,gBAAgB;QACzD;QAEF,MAAMN,0CAA0CH,IAAIyI;IACtD;AACF"}
+{"version":3,"sources":["../../src/db/job-executions.ts"],"sourcesContent":["import {\n  RUNNER_JOB_CLAIMED,\n  RUNNER_JOB_LEASE_EXPIRED,\n  type RunnersEventMap,\n  type RunnerToolCapabilitiesDto,\n} from '@shipfox/api-runners-dto';\nimport {logger} from '@shipfox/node-opentelemetry';\nimport {writeOutboxEvent, writeOutboxEvents} from '@shipfox/node-outbox';\nimport {canonicalizeLabels} from '@shipfox/runner-labels';\nimport {\n  and,\n  arrayContained,\n  asc,\n  count,\n  desc,\n  eq,\n  gt,\n  inArray,\n  isNotNull,\n  isNull,\n  lt,\n  lte,\n  or,\n  sql,\n} from 'drizzle-orm';\nimport {\n  EmptyRequiredLabelsError,\n  RunnerSessionExhaustedError,\n  RunningJobExecutionNotFoundError,\n} from '#core/errors.js';\nimport {\n  type JobExecutionQueueTimeObservation,\n  jobExecutionEnqueuedCount,\n  jobExecutionLeaseExpiredCount,\n  type ProviderRunnerLifecycleObservation,\n  recordJobExecutionQueueTime,\n  recordProviderRunnerActivationToFirstClaim,\n} from '#metrics/instance.js';\nimport type {Tx} from './db.js';\nimport {db} from './db.js';\nimport {lockRunnerReservationAdvisoryKeysTx} from './reservation-locks.js';\nimport {\n  releaseReservationUnits,\n  releaseTerminalRunnerInstanceReservationsByIds,\n} from './reservations.js';\nimport {runnersOutbox} from './schema/outbox.js';\nimport {pendingJobExecutions} from './schema/pending-job-executions.js';\nimport {reservations} from './schema/reservations.js';\nimport {providerRunners} from './schema/runner-instances.js';\nimport {runnerSessions} from './schema/runner-sessions.js';\nimport {runningJobExecutions} from './schema/running-job-executions.js';\n\nconst runnerJobExecutionLockPrefix = 'runners_job_execution:';\n\nasync function lockJobExecution(tx: Tx, jobExecutionId: string): Promise<void> {\n  await tx.execute(\n    sql`select pg_advisory_xact_lock(hashtext(${`${runnerJobExecutionLockPrefix}${jobExecutionId}`}))`,\n  );\n}\n\nasync function releaseReservationsForTerminalRunningRows(\n  tx: Tx,\n  rows: ReadonlyArray<{\n    provisionerId: string | null;\n    providerRunnerId: string | null;\n  }>,\n): Promise<void> {\n  // Converge after the lease fact changes. The reservation helper locks each provider runner\n  // and releases only when the runner is terminal and no uncancelled lease remains.\n  const providerRunnerIdsByProvisionerId = new Map<string, Set<string>>();\n  for (const row of rows) {\n    if (row.provisionerId === null || row.providerRunnerId === null) continue;\n    const providerRunnerIds =\n      providerRunnerIdsByProvisionerId.get(row.provisionerId) ?? new Set<string>();\n    providerRunnerIds.add(row.providerRunnerId);\n    providerRunnerIdsByProvisionerId.set(row.provisionerId, providerRunnerIds);\n  }\n\n  for (const provisionerId of [...providerRunnerIdsByProvisionerId.keys()].sort()) {\n    const providerRunnerIds = providerRunnerIdsByProvisionerId.get(provisionerId);\n    if (!providerRunnerIds) continue;\n    await releaseTerminalRunnerInstanceReservationsByIds(tx, {\n      workspaceId: null,\n      provisionerId,\n      providerRunnerIds: [...providerRunnerIds].sort(),\n      requireUnlinkedSession: false,\n      // Lock and re-check the runner row locally so lease finalization remains retryable\n      // without a workflow/runner scope lock.\n      requireTerminalState: false,\n    });\n  }\n}\n\nexport interface EnqueueJobExecutionParams {\n  workspaceId: string;\n  workflowRunId: string;\n  workflowRunAttemptId: string;\n  jobId: string;\n  jobExecutionId: string;\n  projectId: string;\n  requiredLabels: string[];\n  queuedAt: Date;\n}\n\nexport async function getWorkspaceJobCounts(params: {\n  workspaceIds: string[];\n}): Promise<Array<{workspaceId: string; queued: number; running: number}>> {\n  const [pendingRows, runningRows] = await Promise.all([\n    db()\n      .select({workspaceId: pendingJobExecutions.workspaceId, count: count()})\n      .from(pendingJobExecutions)\n      .where(inArray(pendingJobExecutions.workspaceId, params.workspaceIds))\n      .groupBy(pendingJobExecutions.workspaceId),\n    db()\n      .select({workspaceId: runningJobExecutions.workspaceId, count: count()})\n      .from(runningJobExecutions)\n      .where(inArray(runningJobExecutions.workspaceId, params.workspaceIds))\n      .groupBy(runningJobExecutions.workspaceId),\n  ]);\n\n  const queuedByWorkspace = new Map(pendingRows.map((row) => [row.workspaceId, Number(row.count)]));\n  const runningByWorkspace = new Map(\n    runningRows.map((row) => [row.workspaceId, Number(row.count)]),\n  );\n\n  return params.workspaceIds.map((workspaceId) => ({\n    workspaceId,\n    queued: queuedByWorkspace.get(workspaceId) ?? 0,\n    running: runningByWorkspace.get(workspaceId) ?? 0,\n  }));\n}\n\n// The workflows outbox delivers queue facts at least once. The per-execution advisory lock and\n// pending/running checks make replay a no-op across the claim transition. A durable lease-expired\n// event also prevents a delayed replay from resurrecting an execution already reaped locally.\nexport async function enqueueJobExecution(params: EnqueueJobExecutionParams): Promise<void> {\n  const requiredLabels = [...canonicalizeLabels(params.requiredLabels)];\n  if (requiredLabels.length === 0) throw new EmptyRequiredLabelsError();\n\n  const enqueued = await db().transaction(async (tx) => {\n    await lockJobExecution(tx, params.jobExecutionId);\n\n    const [running] = await tx\n      .select({jobExecutionId: runningJobExecutions.jobExecutionId})\n      .from(runningJobExecutions)\n      .where(eq(runningJobExecutions.jobExecutionId, params.jobExecutionId))\n      .limit(1);\n    if (running) return false;\n\n    const [leaseExpired] = await tx\n      .select({id: runnersOutbox.id})\n      .from(runnersOutbox)\n      .where(\n        and(\n          eq(runnersOutbox.eventType, RUNNER_JOB_LEASE_EXPIRED),\n          sql`${runnersOutbox.payload}->>'jobExecutionId' = ${params.jobExecutionId}`,\n        ),\n      )\n      .limit(1);\n    if (leaseExpired) return false;\n\n    const [inserted] = await tx\n      .insert(pendingJobExecutions)\n      .values({\n        workspaceId: params.workspaceId,\n        workflowRunId: params.workflowRunId,\n        workflowRunAttemptId: params.workflowRunAttemptId,\n        jobId: params.jobId,\n        jobExecutionId: params.jobExecutionId,\n        projectId: params.projectId,\n        requiredLabels,\n        createdAt: params.queuedAt,\n      })\n      .onConflictDoNothing({target: pendingJobExecutions.jobExecutionId})\n      .returning({createdAt: pendingJobExecutions.createdAt});\n\n    return inserted !== undefined;\n  });\n\n  if (enqueued) jobExecutionEnqueuedCount.add(1);\n}\n\nexport interface ClaimedJobExecution {\n  workflowRunId: string;\n  workflowRunAttemptId: string;\n  jobId: string;\n  jobExecutionId: string;\n  projectId: string;\n}\n\nexport interface ActiveRunningJobExecution {\n  workflowRunId: string;\n  workflowRunAttemptId: string;\n  jobId: string;\n  jobExecutionId: string;\n  projectId: string;\n  runnerSessionId: string;\n  provisionerId: string | null;\n  providerRunnerId: string | null;\n  requiredLabels: string[];\n  runnerLabels: string[];\n  startedAt: Date;\n  lastHeartbeatAt: Date;\n}\n\nexport interface RunnerInstanceBoundJobExecution {\n  workflowRunId: string;\n  workflowRunAttemptId: string;\n  jobId: string;\n  jobExecutionId: string;\n  providerRunnerId: string;\n  startedAt: Date;\n  lastHeartbeatAt: Date;\n  cancellationRequestedAt: Date | null;\n}\n\nexport async function claimPendingJobExecution(params: {\n  workspaceId: string;\n  runnerSessionId: string;\n  sessionLabels: string[];\n  maxClaims: number | null;\n  runnerSessionLivenessThrottleSeconds: number;\n}): Promise<ClaimedJobExecution | null> {\n  await touchRunnerSessionLiveness({\n    workspaceId: params.workspaceId,\n    runnerSessionId: params.runnerSessionId,\n    throttleSeconds: params.runnerSessionLivenessThrottleSeconds,\n  });\n\n  if (params.sessionLabels.length === 0) return null;\n\n  let activationToFirstClaimObservation: ProviderRunnerLifecycleObservation | null = null;\n  let queueTimeObservation: JobExecutionQueueTimeObservation | null = null;\n  const result = await db().transaction(async (tx) => {\n    let provisionerId: string | null = null;\n    let providerRunnerId: string | null = null;\n    let runnerInstanceId: string | null = null;\n\n    if (params.maxClaims !== null) {\n      const [session] = await tx\n        .select({\n          maxClaims: runnerSessions.maxClaims,\n          claimsUsed: runnerSessions.claimsUsed,\n          revokedAt: runnerSessions.revokedAt,\n          runnerInstanceId: runnerSessions.runnerInstanceId,\n          provisionerId: runnerSessions.provisionerId,\n          providerRunnerId: runnerSessions.providerRunnerId,\n        })\n        .from(runnerSessions)\n        .where(eq(runnerSessions.id, params.runnerSessionId))\n        .limit(1)\n        .for('update');\n\n      if (\n        !session ||\n        session.revokedAt ||\n        session.maxClaims === null ||\n        session.claimsUsed >= session.maxClaims\n      ) {\n        throw new RunnerSessionExhaustedError(params.runnerSessionId);\n      }\n\n      // Ephemeral sessions are the only capped sessions, and the DB check keeps\n      // their provisioned-runner link present as a pair.\n      runnerInstanceId = session.runnerInstanceId;\n      provisionerId = session.provisionerId;\n      providerRunnerId = session.providerRunnerId;\n    }\n\n    const runnerInstanceCondition = runnerInstanceId\n      ? eq(providerRunners.id, runnerInstanceId)\n      : provisionerId && providerRunnerId\n        ? and(\n            eq(providerRunners.provisionerId, provisionerId),\n            eq(providerRunners.providerRunnerId, providerRunnerId),\n          )\n        : null;\n    if (runnerInstanceCondition && provisionerId) {\n      const [runner] = await tx\n        .select({\n          reservationId: providerRunners.reservationId,\n          intendedReservationId: providerRunners.intendedReservationId,\n        })\n        .from(providerRunners)\n        .where(runnerInstanceCondition)\n        .limit(1);\n      await lockRunnerReservationAdvisoryKeysTx(tx, {\n        provisionerId,\n        reservationIds: [runner?.reservationId, runner?.intendedReservationId].filter(\n          (reservationId): reservationId is string =>\n            reservationId !== null && reservationId !== undefined,\n        ),\n      });\n    }\n\n    // `id` is a uuidv7 (time-ordered), so it is a deterministic FIFO tiebreaker\n    // for rows sharing a created_at within a batch. Lock only the FIFO candidate before\n    // attempting its execution advisory lock; putting pg_try_advisory_xact_lock in this\n    // predicate would evaluate it while scanning and temporarily lock many queue entries.\n    const candidate = tx\n      .select()\n      .from(pendingJobExecutions)\n      .where(\n        and(\n          eq(pendingJobExecutions.workspaceId, params.workspaceId),\n          arrayContained(pendingJobExecutions.requiredLabels, params.sessionLabels),\n        ),\n      )\n      .orderBy(asc(pendingJobExecutions.createdAt), asc(pendingJobExecutions.id))\n      .limit(1)\n      .for('update', {skipLocked: true})\n      .as('pending_candidate');\n\n    const [row] = await tx\n      .select()\n      .from(candidate)\n      .where(\n        sql`\n          pg_try_advisory_xact_lock(\n            hashtext(${runnerJobExecutionLockPrefix} || ${candidate.jobExecutionId}::text)\n          )\n        `,\n      );\n\n    if (!row) return null;\n\n    await tx.delete(pendingJobExecutions).where(eq(pendingJobExecutions.id, row.id));\n\n    // An enqueue retry that lands after a prior claim can leave an orphan pending\n    // row whose jobExecutionId is already in `runners_running_jobs`. Insert-or-skip\n    // on the jobExecutionId unique constraint: when the job execution is already running\n    // the insert touches no row, so we commit the orphan's deletion and return\n    // null rather than let the unique violation roll the claim back into a poison\n    // loop. The runner just re-polls for a real job execution.\n    const inserted = await tx\n      .insert(runningJobExecutions)\n      .values({\n        workspaceId: row.workspaceId,\n        workflowRunId: row.workflowRunId,\n        workflowRunAttemptId: row.workflowRunAttemptId,\n        jobId: row.jobId,\n        jobExecutionId: row.jobExecutionId,\n        projectId: row.projectId,\n        runnerSessionId: params.runnerSessionId,\n        provisionerId,\n        providerRunnerId,\n        requiredLabels: row.requiredLabels,\n        runnerLabels: params.sessionLabels,\n      })\n      .onConflictDoNothing({target: runningJobExecutions.jobExecutionId})\n      .returning({claimedAt: runningJobExecutions.startedAt});\n\n    const claimed = inserted[0];\n    if (!claimed) return null;\n\n    queueTimeObservation = {\n      durationMilliseconds: claimed.claimedAt.getTime() - row.createdAt.getTime(),\n      provider: null,\n      launchKind: params.maxClaims === null ? 'manual' : 'unknown',\n    };\n\n    if (runnerInstanceCondition) {\n      const [claimedRunner] = await tx\n        .update(providerRunners)\n        .set({\n          firstClaimedAt: sql`coalesce(${providerRunners.firstClaimedAt}, ${claimed.claimedAt})`,\n          updatedAt: sql`now()`,\n        })\n        .where(runnerInstanceCondition)\n        .returning({\n          firstClaimedAt: providerRunners.firstClaimedAt,\n          isFirstClaim: sql<boolean>`${providerRunners.firstClaimedAt} = ${claimed.claimedAt}`,\n          provider: providerRunners.providerKind,\n          launchKind: providerRunners.launchKind,\n          runnerInstanceId: providerRunners.id,\n          reservationId: providerRunners.reservationId,\n          intendedReservationId: providerRunners.intendedReservationId,\n          sessionCreatedAtEpochMs: sql<number | null>`(\n            select extract(epoch from ${runnerSessions.createdAt})::double precision * 1000\n            from ${runnerSessions}\n            where ${runnerSessions.id} = ${params.runnerSessionId}\n          )`,\n        });\n      if (claimedRunner && queueTimeObservation) {\n        queueTimeObservation.provider = claimedRunner.provider;\n        queueTimeObservation.launchKind = claimedRunner.launchKind;\n      }\n      if (claimedRunner?.isFirstClaim) {\n        const [releasedRunner] = await tx\n          .update(providerRunners)\n          .set({\n            intendedReservationId: null,\n            reservationReleasedAt: sql`now()`,\n            updatedAt: sql`now()`,\n          })\n          .where(\n            and(\n              eq(providerRunners.id, claimedRunner.runnerInstanceId),\n              or(\n                isNotNull(providerRunners.reservationId),\n                isNotNull(providerRunners.intendedReservationId),\n              ),\n              isNull(providerRunners.reservationReleasedAt),\n            ),\n          )\n          .returning({\n            provisionerId: providerRunners.provisionerId,\n          });\n        const reservationId = claimedRunner.intendedReservationId ?? claimedRunner.reservationId;\n        if (releasedRunner?.provisionerId && reservationId) {\n          const [reservation] = await tx\n            .select({workspaceId: reservations.workspaceId})\n            .from(reservations)\n            .where(\n              and(\n                eq(reservations.id, reservationId),\n                eq(reservations.provisionerId, releasedRunner.provisionerId),\n              ),\n            )\n            .limit(1);\n          if (reservation)\n            await releaseReservationUnits(tx, {\n              workspaceId: reservation.workspaceId,\n              provisionerId: releasedRunner.provisionerId,\n              releases: [{reservationId, count: 1}],\n            });\n        }\n      }\n      if (\n        claimedRunner?.isFirstClaim &&\n        claimedRunner.firstClaimedAt !== null &&\n        claimedRunner.sessionCreatedAtEpochMs !== null &&\n        claimedRunner.sessionCreatedAtEpochMs !== undefined\n      )\n        activationToFirstClaimObservation = {\n          durationMilliseconds:\n            claimedRunner.firstClaimedAt.getTime() - claimedRunner.sessionCreatedAtEpochMs,\n          provider: claimedRunner.provider,\n          launchKind: claimedRunner.launchKind,\n          runnerInstanceId: claimedRunner.runnerInstanceId,\n        };\n    }\n\n    if (params.maxClaims !== null) {\n      await tx\n        .update(runnerSessions)\n        .set({claimsUsed: sql`${runnerSessions.claimsUsed} + 1`, updatedAt: sql`now()`})\n        .where(eq(runnerSessions.id, params.runnerSessionId));\n    }\n\n    // The running-row insert is the runner claiming the job execution. Emit in the same tx; the\n    // payload carries the row's own claim instant so a consumer records the true time,\n    // not the outbox drain time.\n    await writeOutboxEvent<RunnersEventMap>(tx, runnersOutbox, {\n      type: RUNNER_JOB_CLAIMED,\n      payload: {\n        workflowRunId: row.workflowRunId,\n        workflowRunAttemptId: row.workflowRunAttemptId,\n        jobId: row.jobId,\n        jobExecutionId: row.jobExecutionId,\n        claimedAt: claimed.claimedAt.toISOString(),\n      },\n    });\n\n    return {\n      workflowRunId: row.workflowRunId,\n      workflowRunAttemptId: row.workflowRunAttemptId,\n      jobId: row.jobId,\n      jobExecutionId: row.jobExecutionId,\n      projectId: row.projectId,\n    };\n  });\n  if (queueTimeObservation) recordJobExecutionQueueTime(queueTimeObservation);\n  if (activationToFirstClaimObservation)\n    recordProviderRunnerActivationToFirstClaim(activationToFirstClaimObservation);\n  return result;\n}\n\nasync function touchRunnerSessionLiveness(params: {\n  workspaceId: string;\n  runnerSessionId: string;\n  throttleSeconds: number;\n}): Promise<void> {\n  await db()\n    .update(runnerSessions)\n    .set({updatedAt: sql`now()`})\n    .where(\n      and(\n        eq(runnerSessions.id, params.runnerSessionId),\n        eq(runnerSessions.workspaceId, params.workspaceId),\n        sql`${runnerSessions.updatedAt} < now() - (${params.throttleSeconds} || ' seconds')::interval`,\n      ),\n    );\n}\n\n/**\n * Reaps stale leases (bounded by `limit`), emitting one\n * `runners.job.lease_expired` event per reaped job execution.\n *\n * The cutoff is re-checked in the DELETE, not just the locking subquery, so a\n * heartbeat landing mid-call spares the live row. Each reaped execution also sweeps its\n * pending row so an at-least-once queue replay cannot leave an orphan.\n *\n * Locks pending-then-running to match `claimPendingJobExecution` and\n * `reconcileTerminalJobExecution`. The stale candidate scan intentionally does not lock\n * running rows first; the running-row DELETE re-checks the stale predicate after the\n * pending-row sweep.\n */\nexport async function expireStuckJobExecutions(params: {\n  thresholdSeconds: number;\n  noFirstHeartbeatGraceSeconds: number;\n  limit?: number;\n}): Promise<\n  Array<{\n    workflowRunId: string;\n    workflowRunAttemptId: string;\n    jobId: string;\n    jobExecutionId: string;\n  }>\n> {\n  const reaped = await db().transaction(async (tx) => {\n    const heartbeatCutoff = sql`now() - (${params.thresholdSeconds} || ' seconds')::interval`;\n    const firstHeartbeatCutoff = sql`now() - (${params.noFirstHeartbeatGraceSeconds} || ' seconds')::interval`;\n    const stalePredicate = or(\n      and(\n        isNull(runningJobExecutions.firstHeartbeatAt),\n        lte(runningJobExecutions.lastHeartbeatAt, runningJobExecutions.startedAt),\n        lt(runningJobExecutions.startedAt, firstHeartbeatCutoff),\n      ),\n      and(\n        or(\n          isNotNull(runningJobExecutions.firstHeartbeatAt),\n          gt(runningJobExecutions.lastHeartbeatAt, runningJobExecutions.startedAt),\n        ),\n        lt(runningJobExecutions.lastHeartbeatAt, heartbeatCutoff),\n      ),\n    );\n\n    const staleRows = await tx\n      .select({\n        id: runningJobExecutions.id,\n        jobExecutionId: runningJobExecutions.jobExecutionId,\n      })\n      .from(runningJobExecutions)\n      .where(\n        and(\n          stalePredicate,\n          sql`\n            pg_try_advisory_xact_lock(\n              hashtext(${runnerJobExecutionLockPrefix} || ${runningJobExecutions.jobExecutionId}::text)\n            )\n          `,\n        ),\n      )\n      .orderBy(\n        asc(\n          sql`CASE WHEN ${runningJobExecutions.firstHeartbeatAt} IS NULL AND ${runningJobExecutions.lastHeartbeatAt} <= ${runningJobExecutions.startedAt} THEN ${runningJobExecutions.startedAt} ELSE ${runningJobExecutions.lastHeartbeatAt} END`,\n        ),\n        asc(runningJobExecutions.id),\n      )\n      .limit(params.limit ?? 100);\n\n    if (staleRows.length === 0) return [];\n\n    const staleIds = staleRows.map((row) => row.id);\n    const staleJobExecutionIds = staleRows.map((row) => row.jobExecutionId);\n\n    // Sweep pending rows first so this transaction cannot hold a running-row lock while waiting\n    // for a pending-row lock held by reconciliation or a claim of an orphan pending row. The\n    // advisory lock acquired in the candidate scan also makes enqueue retries wait until this\n    // transaction has either reaped the execution or released its lock without selecting it.\n    await tx\n      .delete(pendingJobExecutions)\n      .where(inArray(pendingJobExecutions.jobExecutionId, staleJobExecutionIds));\n\n    const deleted = await tx\n      .delete(runningJobExecutions)\n      .where(and(inArray(runningJobExecutions.id, staleIds), stalePredicate))\n      .returning({\n        workflowRunId: runningJobExecutions.workflowRunId,\n        workflowRunAttemptId: runningJobExecutions.workflowRunAttemptId,\n        jobId: runningJobExecutions.jobId,\n        jobExecutionId: runningJobExecutions.jobExecutionId,\n        provisionerId: runningJobExecutions.provisionerId,\n        providerRunnerId: runningJobExecutions.providerRunnerId,\n      });\n\n    if (deleted.length === 0) return [];\n\n    await releaseReservationsForTerminalRunningRows(tx, deleted);\n\n    await writeOutboxEvents<RunnersEventMap>(\n      tx,\n      runnersOutbox,\n      deleted.map((row) => ({\n        type: RUNNER_JOB_LEASE_EXPIRED,\n        payload: {\n          workflowRunId: row.workflowRunId,\n          workflowRunAttemptId: row.workflowRunAttemptId,\n          jobId: row.jobId,\n          jobExecutionId: row.jobExecutionId,\n        },\n      })),\n    );\n\n    return deleted.map(({workflowRunId, workflowRunAttemptId, jobId, jobExecutionId}) => ({\n      workflowRunId,\n      workflowRunAttemptId,\n      jobId,\n      jobExecutionId,\n    }));\n  });\n\n  if (reaped.length > 0) jobExecutionLeaseExpiredCount.add(reaped.length);\n\n  return reaped;\n}\n\nexport async function getJobExecutionQueueDepth(): Promise<{\n  pendingJobExecutions: number;\n  runningJobExecutions: number;\n}> {\n  const [pending] = await db().select({value: count()}).from(pendingJobExecutions);\n  const [running] = await db().select({value: count()}).from(runningJobExecutions);\n  return {\n    pendingJobExecutions: pending?.value ?? 0,\n    runningJobExecutions: running?.value ?? 0,\n  };\n}\n\nexport async function listActiveRunningJobExecutions(params: {\n  workspaceId: string;\n  windowSeconds: number;\n  limit?: number;\n}): Promise<ActiveRunningJobExecution[]> {\n  return await db()\n    .select({\n      jobId: runningJobExecutions.jobId,\n      jobExecutionId: runningJobExecutions.jobExecutionId,\n      workflowRunId: runningJobExecutions.workflowRunId,\n      workflowRunAttemptId: runningJobExecutions.workflowRunAttemptId,\n      projectId: runningJobExecutions.projectId,\n      runnerSessionId: runningJobExecutions.runnerSessionId,\n      provisionerId: runningJobExecutions.provisionerId,\n      providerRunnerId: runningJobExecutions.providerRunnerId,\n      requiredLabels: runningJobExecutions.requiredLabels,\n      runnerLabels: runningJobExecutions.runnerLabels,\n      startedAt: runningJobExecutions.startedAt,\n      lastHeartbeatAt: runningJobExecutions.lastHeartbeatAt,\n    })\n    .from(runningJobExecutions)\n    .where(\n      and(\n        eq(runningJobExecutions.workspaceId, params.workspaceId),\n        sql`${runningJobExecutions.lastHeartbeatAt} > now() - (${params.windowSeconds} || ' seconds')::interval`,\n      ),\n    )\n    .orderBy(desc(runningJobExecutions.lastHeartbeatAt), desc(runningJobExecutions.id))\n    .limit(params.limit ?? 1000);\n}\n\nexport async function listRunningJobExecutionsByRunnerInstanceTx(\n  tx: Tx,\n  params: {\n    workspaceId: string;\n    provisionerId: string;\n    providerRunnerIds: string[];\n  },\n): Promise<RunnerInstanceBoundJobExecution[]> {\n  if (params.providerRunnerIds.length === 0) return [];\n\n  const duplicateRows = await tx\n    .select({\n      providerRunnerId: runningJobExecutions.providerRunnerId,\n      count: count(),\n    })\n    .from(runningJobExecutions)\n    .where(\n      and(\n        eq(runningJobExecutions.workspaceId, params.workspaceId),\n        eq(runningJobExecutions.provisionerId, params.provisionerId),\n        inArray(runningJobExecutions.providerRunnerId, params.providerRunnerIds),\n      ),\n    )\n    .groupBy(runningJobExecutions.providerRunnerId)\n    .having(sql`count(*) > 1`);\n\n  const duplicateRunnerInstanceIds = duplicateRows.flatMap((row) =>\n    row.providerRunnerId ? [row.providerRunnerId] : [],\n  );\n  if (duplicateRunnerInstanceIds.length > 0) {\n    logger().warn(\n      {\n        workspaceId: params.workspaceId,\n        provisionerId: params.provisionerId,\n        providerRunnerIds: duplicateRunnerInstanceIds,\n      },\n      'multiple running job executions are bound to the same provisioned runner',\n    );\n  }\n\n  const result = await tx.execute<{\n    workflowRunId: string;\n    workflowRunAttemptId: string;\n    jobId: string;\n    jobExecutionId: string;\n    providerRunnerId: string;\n    startedAt: Date | string;\n    lastHeartbeatAt: Date | string;\n    cancellationRequestedAt: Date | string | null;\n  }>(sql`\n    SELECT DISTINCT ON (${runningJobExecutions.providerRunnerId})\n      ${runningJobExecutions.workflowRunId} AS \"workflowRunId\",\n      ${runningJobExecutions.workflowRunAttemptId} AS \"workflowRunAttemptId\",\n      ${runningJobExecutions.jobId} AS \"jobId\",\n      ${runningJobExecutions.jobExecutionId} AS \"jobExecutionId\",\n      ${runningJobExecutions.providerRunnerId} AS \"providerRunnerId\",\n      ${runningJobExecutions.startedAt} AS \"startedAt\",\n      ${runningJobExecutions.lastHeartbeatAt} AS \"lastHeartbeatAt\",\n      ${runningJobExecutions.cancellationRequestedAt} AS \"cancellationRequestedAt\"\n    FROM ${runningJobExecutions}\n    WHERE\n      ${runningJobExecutions.workspaceId} = ${params.workspaceId}\n      AND ${runningJobExecutions.provisionerId} = ${params.provisionerId}\n      AND ${runningJobExecutions.providerRunnerId} IN (${sql.join(\n        params.providerRunnerIds.map((providerRunnerId) => sql`${providerRunnerId}`),\n        sql`, `,\n      )})\n    ORDER BY ${runningJobExecutions.providerRunnerId}, ${runningJobExecutions.startedAt} DESC, ${runningJobExecutions.jobExecutionId} DESC\n  `);\n\n  return result.rows.map((row) => ({\n    workflowRunId: row.workflowRunId,\n    workflowRunAttemptId: row.workflowRunAttemptId,\n    jobId: row.jobId,\n    jobExecutionId: row.jobExecutionId,\n    providerRunnerId: row.providerRunnerId,\n    startedAt: toDate(row.startedAt),\n    lastHeartbeatAt: toDate(row.lastHeartbeatAt),\n    cancellationRequestedAt: row.cancellationRequestedAt\n      ? toDate(row.cancellationRequestedAt)\n      : null,\n  }));\n}\n\nfunction toDate(value: Date | string): Date {\n  return value instanceof Date ? value : new Date(value);\n}\n\nexport async function isJobLeaseActive(params: {\n  jobId?: string;\n  jobExecutionId: string;\n  runnerSessionId: string;\n}): Promise<boolean> {\n  const [row] = await db()\n    .select({id: runningJobExecutions.id})\n    .from(runningJobExecutions)\n    .where(\n      and(\n        eq(runningJobExecutions.jobExecutionId, params.jobExecutionId),\n        eq(runningJobExecutions.runnerSessionId, params.runnerSessionId),\n        params.jobId === undefined ? undefined : eq(runningJobExecutions.jobId, params.jobId),\n      ),\n    )\n    .limit(1);\n\n  return row !== undefined;\n}\n\nexport async function recordHeartbeat(params: {\n  jobExecutionId: string;\n  runnerSessionId: string;\n  toolCapabilities?: RunnerToolCapabilitiesDto | null;\n}): Promise<{\n  cancellationRequested: boolean;\n  previousToolCapabilities: RunnerToolCapabilitiesDto | null;\n  currentToolCapabilities: RunnerToolCapabilitiesDto | null;\n  runningJobExecution: {\n    workflowRunId: string;\n    workflowRunAttemptId: string;\n    jobId: string;\n    jobExecutionId: string;\n    projectId: string;\n    workspaceId: string;\n    runnerSessionId: string;\n  };\n}> {\n  const result = await db().transaction(async (tx) => {\n    const updated = await tx\n      .update(runningJobExecutions)\n      .set({\n        firstHeartbeatAt: sql`COALESCE(${runningJobExecutions.firstHeartbeatAt}, now())`,\n        lastHeartbeatAt: sql`now()`,\n      })\n      .where(\n        and(\n          eq(runningJobExecutions.jobExecutionId, params.jobExecutionId),\n          eq(runningJobExecutions.runnerSessionId, params.runnerSessionId),\n        ),\n      )\n      .returning({\n        cancellationRequestedAt: runningJobExecutions.cancellationRequestedAt,\n        workflowRunId: runningJobExecutions.workflowRunId,\n        workflowRunAttemptId: runningJobExecutions.workflowRunAttemptId,\n        jobId: runningJobExecutions.jobId,\n        jobExecutionId: runningJobExecutions.jobExecutionId,\n        projectId: runningJobExecutions.projectId,\n        workspaceId: runningJobExecutions.workspaceId,\n        runnerSessionId: runningJobExecutions.runnerSessionId,\n      });\n\n    const row = updated[0];\n    if (!row) throw new RunningJobExecutionNotFoundError(params.jobExecutionId);\n\n    const [previous] = await tx\n      .select({toolCapabilities: runnerSessions.toolCapabilities})\n      .from(runnerSessions)\n      .where(eq(runnerSessions.id, params.runnerSessionId))\n      .limit(1)\n      .for('update');\n\n    const [session] = await tx\n      .update(runnerSessions)\n      .set({\n        toolCapabilities: params.toolCapabilities ?? null,\n        toolCapabilitiesReportedAt: params.toolCapabilities ? sql`now()` : null,\n        updatedAt: sql`now()`,\n      })\n      .where(eq(runnerSessions.id, params.runnerSessionId))\n      .returning({toolCapabilities: runnerSessions.toolCapabilities});\n\n    return {\n      row,\n      previousToolCapabilities: previous?.toolCapabilities ?? null,\n      currentToolCapabilities: session?.toolCapabilities ?? null,\n    };\n  });\n\n  const row = result.row;\n  if (!row) throw new RunningJobExecutionNotFoundError(params.jobExecutionId);\n  return {\n    cancellationRequested: row.cancellationRequestedAt !== null,\n    previousToolCapabilities: result.previousToolCapabilities,\n    currentToolCapabilities: result.currentToolCapabilities,\n    runningJobExecution: {\n      workflowRunId: row.workflowRunId,\n      workflowRunAttemptId: row.workflowRunAttemptId,\n      jobId: row.jobId,\n      jobExecutionId: row.jobExecutionId,\n      projectId: row.projectId,\n      workspaceId: row.workspaceId,\n      runnerSessionId: row.runnerSessionId,\n    },\n  };\n}\n\n/**\n * Reconciles a terminal job execution with runner state in one transaction: removes its pending\n * queue row, requests cancellation on its running lease, and converges any linked reservation.\n * The operation is idempotent and preserves the first cancellation-request timestamp.\n */\nexport async function reconcileTerminalJobExecution(params: {\n  jobExecutionId: string;\n}): Promise<void> {\n  await db().transaction(async (tx) => {\n    await lockJobExecution(tx, params.jobExecutionId);\n\n    // Delete pending before updating running to match claim/release lock order. Claim locks\n    // pending rows with SKIP LOCKED before inserting the running lease, so this ordering makes\n    // a concurrent terminal reconciliation either win before claim or cancel the new lease.\n    await tx\n      .delete(pendingJobExecutions)\n      .where(eq(pendingJobExecutions.jobExecutionId, params.jobExecutionId));\n    const cancelledRunningRows = await tx\n      .update(runningJobExecutions)\n      .set({\n        cancellationRequestedAt: sql`COALESCE(${runningJobExecutions.cancellationRequestedAt}, now())`,\n      })\n      .where(eq(runningJobExecutions.jobExecutionId, params.jobExecutionId))\n      .returning({\n        provisionerId: runningJobExecutions.provisionerId,\n        providerRunnerId: runningJobExecutions.providerRunnerId,\n      });\n\n    await releaseReservationsForTerminalRunningRows(tx, cancelledRunningRows);\n  });\n}\n"],"names":["RUNNER_JOB_CLAIMED","RUNNER_JOB_LEASE_EXPIRED","logger","writeOutboxEvent","writeOutboxEvents","canonicalizeLabels","and","arrayContained","asc","count","desc","eq","gt","inArray","isNotNull","isNull","lt","lte","or","sql","EmptyRequiredLabelsError","RunnerSessionExhaustedError","RunningJobExecutionNotFoundError","jobExecutionEnqueuedCount","jobExecutionLeaseExpiredCount","recordJobExecutionQueueTime","recordProviderRunnerActivationToFirstClaim","db","lockRunnerReservationAdvisoryKeysTx","releaseReservationUnits","releaseTerminalRunnerInstanceReservationsByIds","runnersOutbox","pendingJobExecutions","reservations","providerRunners","runnerSessions","runningJobExecutions","runnerJobExecutionLockPrefix","lockJobExecution","tx","jobExecutionId","execute","releaseReservationsForTerminalRunningRows","rows","providerRunnerIdsByProvisionerId","Map","row","provisionerId","providerRunnerId","providerRunnerIds","get","Set","add","set","keys","sort","workspaceId","requireUnlinkedSession","requireTerminalState","getWorkspaceJobCounts","params","pendingRows","runningRows","Promise","all","select","from","where","workspaceIds","groupBy","queuedByWorkspace","map","Number","runningByWorkspace","queued","running","enqueueJobExecution","requiredLabels","length","enqueued","transaction","limit","leaseExpired","id","eventType","payload","inserted","insert","values","workflowRunId","workflowRunAttemptId","jobId","projectId","createdAt","queuedAt","onConflictDoNothing","target","returning","undefined","claimPendingJobExecution","touchRunnerSessionLiveness","runnerSessionId","throttleSeconds","runnerSessionLivenessThrottleSeconds","sessionLabels","activationToFirstClaimObservation","queueTimeObservation","result","runnerInstanceId","maxClaims","session","claimsUsed","revokedAt","for","runnerInstanceCondition","runner","reservationId","intendedReservationId","reservationIds","filter","candidate","orderBy","skipLocked","as","delete","runnerLabels","claimedAt","startedAt","claimed","durationMilliseconds","getTime","provider","launchKind","claimedRunner","update","firstClaimedAt","updatedAt","isFirstClaim","providerKind","sessionCreatedAtEpochMs","releasedRunner","reservationReleasedAt","reservation","releases","type","toISOString","expireStuckJobExecutions","reaped","heartbeatCutoff","thresholdSeconds","firstHeartbeatCutoff","noFirstHeartbeatGraceSeconds","stalePredicate","firstHeartbeatAt","lastHeartbeatAt","staleRows","staleIds","staleJobExecutionIds","deleted","getJobExecutionQueueDepth","pending","value","listActiveRunningJobExecutions","windowSeconds","listRunningJobExecutionsByRunnerInstanceTx","duplicateRows","having","duplicateRunnerInstanceIds","flatMap","warn","cancellationRequestedAt","join","toDate","Date","isJobLeaseActive","recordHeartbeat","updated","previous","toolCapabilities","toolCapabilitiesReportedAt","previousToolCapabilities","currentToolCapabilities","cancellationRequested","runningJobExecution","reconcileTerminalJobExecution","cancelledRunningRows"],"mappings":"AAAA,SACEA,kBAAkB,EAClBC,wBAAwB,QAGnB,2BAA2B;AAClC,SAAQC,MAAM,QAAO,8BAA8B;AACnD,SAAQC,gBAAgB,EAAEC,iBAAiB,QAAO,uBAAuB;AACzE,SAAQC,kBAAkB,QAAO,yBAAyB;AAC1D,SACEC,GAAG,EACHC,cAAc,EACdC,GAAG,EACHC,KAAK,EACLC,IAAI,EACJC,EAAE,EACFC,EAAE,EACFC,OAAO,EACPC,SAAS,EACTC,MAAM,EACNC,EAAE,EACFC,GAAG,EACHC,EAAE,EACFC,GAAG,QACE,cAAc;AACrB,SACEC,wBAAwB,EACxBC,2BAA2B,EAC3BC,gCAAgC,QAC3B,kBAAkB;AACzB,SAEEC,yBAAyB,EACzBC,6BAA6B,EAE7BC,2BAA2B,EAC3BC,0CAA0C,QACrC,uBAAuB;AAE9B,SAAQC,EAAE,QAAO,UAAU;AAC3B,SAAQC,mCAAmC,QAAO,yBAAyB;AAC3E,SACEC,uBAAuB,EACvBC,8CAA8C,QACzC,oBAAoB;AAC3B,SAAQC,aAAa,QAAO,qBAAqB;AACjD,SAAQC,oBAAoB,QAAO,qCAAqC;AACxE,SAAQC,YAAY,QAAO,2BAA2B;AACtD,SAAQC,eAAe,QAAO,+BAA+B;AAC7D,SAAQC,cAAc,QAAO,8BAA8B;AAC3D,SAAQC,oBAAoB,QAAO,qCAAqC;AAExE,MAAMC,+BAA+B;AAErC,eAAeC,iBAAiBC,EAAM,EAAEC,cAAsB;IAC5D,MAAMD,GAAGE,OAAO,CACdtB,GAAG,CAAC,sCAAsC,EAAE,GAAGkB,+BAA+BG,gBAAgB,CAAC,EAAE,CAAC;AAEtG;AAEA,eAAeE,0CACbH,EAAM,EACNI,IAGE;IAEF,2FAA2F;IAC3F,kFAAkF;IAClF,MAAMC,mCAAmC,IAAIC;IAC7C,KAAK,MAAMC,OAAOH,KAAM;QACtB,IAAIG,IAAIC,aAAa,KAAK,QAAQD,IAAIE,gBAAgB,KAAK,MAAM;QACjE,MAAMC,oBACJL,iCAAiCM,GAAG,CAACJ,IAAIC,aAAa,KAAK,IAAII;QACjEF,kBAAkBG,GAAG,CAACN,IAAIE,gBAAgB;QAC1CJ,iCAAiCS,GAAG,CAACP,IAAIC,aAAa,EAAEE;IAC1D;IAEA,KAAK,MAAMF,iBAAiB;WAAIH,iCAAiCU,IAAI;KAAG,CAACC,IAAI,GAAI;QAC/E,MAAMN,oBAAoBL,iCAAiCM,GAAG,CAACH;QAC/D,IAAI,CAACE,mBAAmB;QACxB,MAAMnB,+CAA+CS,IAAI;YACvDiB,aAAa;YACbT;YACAE,mBAAmB;mBAAIA;aAAkB,CAACM,IAAI;YAC9CE,wBAAwB;YACxB,mFAAmF;YACnF,wCAAwC;YACxCC,sBAAsB;QACxB;IACF;AACF;AAaA,OAAO,eAAeC,sBAAsBC,MAE3C;IACC,MAAM,CAACC,aAAaC,YAAY,GAAG,MAAMC,QAAQC,GAAG,CAAC;QACnDrC,KACGsC,MAAM,CAAC;YAACT,aAAaxB,qBAAqBwB,WAAW;YAAE/C,OAAOA;QAAO,GACrEyD,IAAI,CAAClC,sBACLmC,KAAK,CAACtD,QAAQmB,qBAAqBwB,WAAW,EAAEI,OAAOQ,YAAY,GACnEC,OAAO,CAACrC,qBAAqBwB,WAAW;QAC3C7B,KACGsC,MAAM,CAAC;YAACT,aAAapB,qBAAqBoB,WAAW;YAAE/C,OAAOA;QAAO,GACrEyD,IAAI,CAAC9B,sBACL+B,KAAK,CAACtD,QAAQuB,qBAAqBoB,WAAW,EAAEI,OAAOQ,YAAY,GACnEC,OAAO,CAACjC,qBAAqBoB,WAAW;KAC5C;IAED,MAAMc,oBAAoB,IAAIzB,IAAIgB,YAAYU,GAAG,CAAC,CAACzB,MAAQ;YAACA,IAAIU,WAAW;YAAEgB,OAAO1B,IAAIrC,KAAK;SAAE;IAC/F,MAAMgE,qBAAqB,IAAI5B,IAC7BiB,YAAYS,GAAG,CAAC,CAACzB,MAAQ;YAACA,IAAIU,WAAW;YAAEgB,OAAO1B,IAAIrC,KAAK;SAAE;IAG/D,OAAOmD,OAAOQ,YAAY,CAACG,GAAG,CAAC,CAACf,cAAiB,CAAA;YAC/CA;YACAkB,QAAQJ,kBAAkBpB,GAAG,CAACM,gBAAgB;YAC9CmB,SAASF,mBAAmBvB,GAAG,CAACM,gBAAgB;QAClD,CAAA;AACF;AAEA,+FAA+F;AAC/F,kGAAkG;AAClG,8FAA8F;AAC9F,OAAO,eAAeoB,oBAAoBhB,MAAiC;IACzE,MAAMiB,iBAAiB;WAAIxE,mBAAmBuD,OAAOiB,cAAc;KAAE;IACrE,IAAIA,eAAeC,MAAM,KAAK,GAAG,MAAM,IAAI1D;IAE3C,MAAM2D,WAAW,MAAMpD,KAAKqD,WAAW,CAAC,OAAOzC;QAC7C,MAAMD,iBAAiBC,IAAIqB,OAAOpB,cAAc;QAEhD,MAAM,CAACmC,QAAQ,GAAG,MAAMpC,GACrB0B,MAAM,CAAC;YAACzB,gBAAgBJ,qBAAqBI,cAAc;QAAA,GAC3D0B,IAAI,CAAC9B,sBACL+B,KAAK,CAACxD,GAAGyB,qBAAqBI,cAAc,EAAEoB,OAAOpB,cAAc,GACnEyC,KAAK,CAAC;QACT,IAAIN,SAAS,OAAO;QAEpB,MAAM,CAACO,aAAa,GAAG,MAAM3C,GAC1B0B,MAAM,CAAC;YAACkB,IAAIpD,cAAcoD,EAAE;QAAA,GAC5BjB,IAAI,CAACnC,eACLoC,KAAK,CACJ7D,IACEK,GAAGoB,cAAcqD,SAAS,EAAEnF,2BAC5BkB,GAAG,CAAC,EAAEY,cAAcsD,OAAO,CAAC,sBAAsB,EAAEzB,OAAOpB,cAAc,CAAC,CAAC,GAG9EyC,KAAK,CAAC;QACT,IAAIC,cAAc,OAAO;QAEzB,MAAM,CAACI,SAAS,GAAG,MAAM/C,GACtBgD,MAAM,CAACvD,sBACPwD,MAAM,CAAC;YACNhC,aAAaI,OAAOJ,WAAW;YAC/BiC,eAAe7B,OAAO6B,aAAa;YACnCC,sBAAsB9B,OAAO8B,oBAAoB;YACjDC,OAAO/B,OAAO+B,KAAK;YACnBnD,gBAAgBoB,OAAOpB,cAAc;YACrCoD,WAAWhC,OAAOgC,SAAS;YAC3Bf;YACAgB,WAAWjC,OAAOkC,QAAQ;QAC5B,GACCC,mBAAmB,CAAC;YAACC,QAAQhE,qBAAqBQ,cAAc;QAAA,GAChEyD,SAAS,CAAC;YAACJ,WAAW7D,qBAAqB6D,SAAS;QAAA;QAEvD,OAAOP,aAAaY;IACtB;IAEA,IAAInB,UAAUxD,0BAA0B6B,GAAG,CAAC;AAC9C;AAoCA,OAAO,eAAe+C,yBAAyBvC,MAM9C;IACC,MAAMwC,2BAA2B;QAC/B5C,aAAaI,OAAOJ,WAAW;QAC/B6C,iBAAiBzC,OAAOyC,eAAe;QACvCC,iBAAiB1C,OAAO2C,oCAAoC;IAC9D;IAEA,IAAI3C,OAAO4C,aAAa,CAAC1B,MAAM,KAAK,GAAG,OAAO;IAE9C,IAAI2B,oCAA+E;IACnF,IAAIC,uBAAgE;IACpE,MAAMC,SAAS,MAAMhF,KAAKqD,WAAW,CAAC,OAAOzC;QAC3C,IAAIQ,gBAA+B;QACnC,IAAIC,mBAAkC;QACtC,IAAI4D,mBAAkC;QAEtC,IAAIhD,OAAOiD,SAAS,KAAK,MAAM;YAC7B,MAAM,CAACC,QAAQ,GAAG,MAAMvE,GACrB0B,MAAM,CAAC;gBACN4C,WAAW1E,eAAe0E,SAAS;gBACnCE,YAAY5E,eAAe4E,UAAU;gBACrCC,WAAW7E,eAAe6E,SAAS;gBACnCJ,kBAAkBzE,eAAeyE,gBAAgB;gBACjD7D,eAAeZ,eAAeY,aAAa;gBAC3CC,kBAAkBb,eAAea,gBAAgB;YACnD,GACCkB,IAAI,CAAC/B,gBACLgC,KAAK,CAACxD,GAAGwB,eAAegD,EAAE,EAAEvB,OAAOyC,eAAe,GAClDpB,KAAK,CAAC,GACNgC,GAAG,CAAC;YAEP,IACE,CAACH,WACDA,QAAQE,SAAS,IACjBF,QAAQD,SAAS,KAAK,QACtBC,QAAQC,UAAU,IAAID,QAAQD,SAAS,EACvC;gBACA,MAAM,IAAIxF,4BAA4BuC,OAAOyC,eAAe;YAC9D;YAEA,0EAA0E;YAC1E,mDAAmD;YACnDO,mBAAmBE,QAAQF,gBAAgB;YAC3C7D,gBAAgB+D,QAAQ/D,aAAa;YACrCC,mBAAmB8D,QAAQ9D,gBAAgB;QAC7C;QAEA,MAAMkE,0BAA0BN,mBAC5BjG,GAAGuB,gBAAgBiD,EAAE,EAAEyB,oBACvB7D,iBAAiBC,mBACf1C,IACEK,GAAGuB,gBAAgBa,aAAa,EAAEA,gBAClCpC,GAAGuB,gBAAgBc,gBAAgB,EAAEA,qBAEvC;QACN,IAAIkE,2BAA2BnE,eAAe;YAC5C,MAAM,CAACoE,OAAO,GAAG,MAAM5E,GACpB0B,MAAM,CAAC;gBACNmD,eAAelF,gBAAgBkF,aAAa;gBAC5CC,uBAAuBnF,gBAAgBmF,qBAAqB;YAC9D,GACCnD,IAAI,CAAChC,iBACLiC,KAAK,CAAC+C,yBACNjC,KAAK,CAAC;YACT,MAAMrD,oCAAoCW,IAAI;gBAC5CQ;gBACAuE,gBAAgB;oBAACH,QAAQC;oBAAeD,QAAQE;iBAAsB,CAACE,MAAM,CAC3E,CAACH,gBACCA,kBAAkB,QAAQA,kBAAkBlB;YAElD;QACF;QAEA,4EAA4E;QAC5E,oFAAoF;QACpF,oFAAoF;QACpF,sFAAsF;QACtF,MAAMsB,YAAYjF,GACf0B,MAAM,GACNC,IAAI,CAAClC,sBACLmC,KAAK,CACJ7D,IACEK,GAAGqB,qBAAqBwB,WAAW,EAAEI,OAAOJ,WAAW,GACvDjD,eAAeyB,qBAAqB6C,cAAc,EAAEjB,OAAO4C,aAAa,IAG3EiB,OAAO,CAACjH,IAAIwB,qBAAqB6D,SAAS,GAAGrF,IAAIwB,qBAAqBmD,EAAE,GACxEF,KAAK,CAAC,GACNgC,GAAG,CAAC,UAAU;YAACS,YAAY;QAAI,GAC/BC,EAAE,CAAC;QAEN,MAAM,CAAC7E,IAAI,GAAG,MAAMP,GACjB0B,MAAM,GACNC,IAAI,CAACsD,WACLrD,KAAK,CACJhD,GAAG,CAAC;;qBAES,EAAEkB,6BAA6B,IAAI,EAAEmF,UAAUhF,cAAc,CAAC;;QAE3E,CAAC;QAGL,IAAI,CAACM,KAAK,OAAO;QAEjB,MAAMP,GAAGqF,MAAM,CAAC5F,sBAAsBmC,KAAK,CAACxD,GAAGqB,qBAAqBmD,EAAE,EAAErC,IAAIqC,EAAE;QAE9E,8EAA8E;QAC9E,gFAAgF;QAChF,qFAAqF;QACrF,2EAA2E;QAC3E,8EAA8E;QAC9E,2DAA2D;QAC3D,MAAMG,WAAW,MAAM/C,GACpBgD,MAAM,CAACnD,sBACPoD,MAAM,CAAC;YACNhC,aAAaV,IAAIU,WAAW;YAC5BiC,eAAe3C,IAAI2C,aAAa;YAChCC,sBAAsB5C,IAAI4C,oBAAoB;YAC9CC,OAAO7C,IAAI6C,KAAK;YAChBnD,gBAAgBM,IAAIN,cAAc;YAClCoD,WAAW9C,IAAI8C,SAAS;YACxBS,iBAAiBzC,OAAOyC,eAAe;YACvCtD;YACAC;YACA6B,gBAAgB/B,IAAI+B,cAAc;YAClCgD,cAAcjE,OAAO4C,aAAa;QACpC,GACCT,mBAAmB,CAAC;YAACC,QAAQ5D,qBAAqBI,cAAc;QAAA,GAChEyD,SAAS,CAAC;YAAC6B,WAAW1F,qBAAqB2F,SAAS;QAAA;QAEvD,MAAMC,UAAU1C,QAAQ,CAAC,EAAE;QAC3B,IAAI,CAAC0C,SAAS,OAAO;QAErBtB,uBAAuB;YACrBuB,sBAAsBD,QAAQF,SAAS,CAACI,OAAO,KAAKpF,IAAI+C,SAAS,CAACqC,OAAO;YACzEC,UAAU;YACVC,YAAYxE,OAAOiD,SAAS,KAAK,OAAO,WAAW;QACrD;QAEA,IAAIK,yBAAyB;YAC3B,MAAM,CAACmB,cAAc,GAAG,MAAM9F,GAC3B+F,MAAM,CAACpG,iBACPmB,GAAG,CAAC;gBACHkF,gBAAgBpH,GAAG,CAAC,SAAS,EAAEe,gBAAgBqG,cAAc,CAAC,EAAE,EAAEP,QAAQF,SAAS,CAAC,CAAC,CAAC;gBACtFU,WAAWrH,GAAG,CAAC,KAAK,CAAC;YACvB,GACCgD,KAAK,CAAC+C,yBACNjB,SAAS,CAAC;gBACTsC,gBAAgBrG,gBAAgBqG,cAAc;gBAC9CE,cAActH,GAAY,CAAC,EAAEe,gBAAgBqG,cAAc,CAAC,GAAG,EAAEP,QAAQF,SAAS,CAAC,CAAC;gBACpFK,UAAUjG,gBAAgBwG,YAAY;gBACtCN,YAAYlG,gBAAgBkG,UAAU;gBACtCxB,kBAAkB1E,gBAAgBiD,EAAE;gBACpCiC,eAAelF,gBAAgBkF,aAAa;gBAC5CC,uBAAuBnF,gBAAgBmF,qBAAqB;gBAC5DsB,yBAAyBxH,GAAkB,CAAC;sCAChB,EAAEgB,eAAe0D,SAAS,CAAC;iBAChD,EAAE1D,eAAe;kBAChB,EAAEA,eAAegD,EAAE,CAAC,GAAG,EAAEvB,OAAOyC,eAAe,CAAC;WACvD,CAAC;YACJ;YACF,IAAIgC,iBAAiB3B,sBAAsB;gBACzCA,qBAAqByB,QAAQ,GAAGE,cAAcF,QAAQ;gBACtDzB,qBAAqB0B,UAAU,GAAGC,cAAcD,UAAU;YAC5D;YACA,IAAIC,eAAeI,cAAc;gBAC/B,MAAM,CAACG,eAAe,GAAG,MAAMrG,GAC5B+F,MAAM,CAACpG,iBACPmB,GAAG,CAAC;oBACHgE,uBAAuB;oBACvBwB,uBAAuB1H,GAAG,CAAC,KAAK,CAAC;oBACjCqH,WAAWrH,GAAG,CAAC,KAAK,CAAC;gBACvB,GACCgD,KAAK,CACJ7D,IACEK,GAAGuB,gBAAgBiD,EAAE,EAAEkD,cAAczB,gBAAgB,GACrD1F,GACEJ,UAAUoB,gBAAgBkF,aAAa,GACvCtG,UAAUoB,gBAAgBmF,qBAAqB,IAEjDtG,OAAOmB,gBAAgB2G,qBAAqB,IAG/C5C,SAAS,CAAC;oBACTlD,eAAeb,gBAAgBa,aAAa;gBAC9C;gBACF,MAAMqE,gBAAgBiB,cAAchB,qBAAqB,IAAIgB,cAAcjB,aAAa;gBACxF,IAAIwB,gBAAgB7F,iBAAiBqE,eAAe;oBAClD,MAAM,CAAC0B,YAAY,GAAG,MAAMvG,GACzB0B,MAAM,CAAC;wBAACT,aAAavB,aAAauB,WAAW;oBAAA,GAC7CU,IAAI,CAACjC,cACLkC,KAAK,CACJ7D,IACEK,GAAGsB,aAAakD,EAAE,EAAEiC,gBACpBzG,GAAGsB,aAAac,aAAa,EAAE6F,eAAe7F,aAAa,IAG9DkC,KAAK,CAAC;oBACT,IAAI6D,aACF,MAAMjH,wBAAwBU,IAAI;wBAChCiB,aAAasF,YAAYtF,WAAW;wBACpCT,eAAe6F,eAAe7F,aAAa;wBAC3CgG,UAAU;4BAAC;gCAAC3B;gCAAe3G,OAAO;4BAAC;yBAAE;oBACvC;gBACJ;YACF;YACA,IACE4H,eAAeI,gBACfJ,cAAcE,cAAc,KAAK,QACjCF,cAAcM,uBAAuB,KAAK,QAC1CN,cAAcM,uBAAuB,KAAKzC,WAE1CO,oCAAoC;gBAClCwB,sBACEI,cAAcE,cAAc,CAACL,OAAO,KAAKG,cAAcM,uBAAuB;gBAChFR,UAAUE,cAAcF,QAAQ;gBAChCC,YAAYC,cAAcD,UAAU;gBACpCxB,kBAAkByB,cAAczB,gBAAgB;YAClD;QACJ;QAEA,IAAIhD,OAAOiD,SAAS,KAAK,MAAM;YAC7B,MAAMtE,GACH+F,MAAM,CAACnG,gBACPkB,GAAG,CAAC;gBAAC0D,YAAY5F,GAAG,CAAC,EAAEgB,eAAe4E,UAAU,CAAC,IAAI,CAAC;gBAAEyB,WAAWrH,GAAG,CAAC,KAAK,CAAC;YAAA,GAC7EgD,KAAK,CAACxD,GAAGwB,eAAegD,EAAE,EAAEvB,OAAOyC,eAAe;QACvD;QAEA,4FAA4F;QAC5F,mFAAmF;QACnF,6BAA6B;QAC7B,MAAMlG,iBAAkCoC,IAAIR,eAAe;YACzDiH,MAAMhJ;YACNqF,SAAS;gBACPI,eAAe3C,IAAI2C,aAAa;gBAChCC,sBAAsB5C,IAAI4C,oBAAoB;gBAC9CC,OAAO7C,IAAI6C,KAAK;gBAChBnD,gBAAgBM,IAAIN,cAAc;gBAClCsF,WAAWE,QAAQF,SAAS,CAACmB,WAAW;YAC1C;QACF;QAEA,OAAO;YACLxD,eAAe3C,IAAI2C,aAAa;YAChCC,sBAAsB5C,IAAI4C,oBAAoB;YAC9CC,OAAO7C,IAAI6C,KAAK;YAChBnD,gBAAgBM,IAAIN,cAAc;YAClCoD,WAAW9C,IAAI8C,SAAS;QAC1B;IACF;IACA,IAAIc,sBAAsBjF,4BAA4BiF;IACtD,IAAID,mCACF/E,2CAA2C+E;IAC7C,OAAOE;AACT;AAEA,eAAeP,2BAA2BxC,MAIzC;IACC,MAAMjC,KACH2G,MAAM,CAACnG,gBACPkB,GAAG,CAAC;QAACmF,WAAWrH,GAAG,CAAC,KAAK,CAAC;IAAA,GAC1BgD,KAAK,CACJ7D,IACEK,GAAGwB,eAAegD,EAAE,EAAEvB,OAAOyC,eAAe,GAC5C1F,GAAGwB,eAAeqB,WAAW,EAAEI,OAAOJ,WAAW,GACjDrC,GAAG,CAAC,EAAEgB,eAAeqG,SAAS,CAAC,YAAY,EAAE5E,OAAO0C,eAAe,CAAC,yBAAyB,CAAC;AAGtG;AAEA;;;;;;;;;;;;CAYC,GACD,OAAO,eAAe4C,yBAAyBtF,MAI9C;IAQC,MAAMuF,SAAS,MAAMxH,KAAKqD,WAAW,CAAC,OAAOzC;QAC3C,MAAM6G,kBAAkBjI,GAAG,CAAC,SAAS,EAAEyC,OAAOyF,gBAAgB,CAAC,yBAAyB,CAAC;QACzF,MAAMC,uBAAuBnI,GAAG,CAAC,SAAS,EAAEyC,OAAO2F,4BAA4B,CAAC,yBAAyB,CAAC;QAC1G,MAAMC,iBAAiBtI,GACrBZ,IACES,OAAOqB,qBAAqBqH,gBAAgB,GAC5CxI,IAAImB,qBAAqBsH,eAAe,EAAEtH,qBAAqB2F,SAAS,GACxE/G,GAAGoB,qBAAqB2F,SAAS,EAAEuB,wBAErChJ,IACEY,GACEJ,UAAUsB,qBAAqBqH,gBAAgB,GAC/C7I,GAAGwB,qBAAqBsH,eAAe,EAAEtH,qBAAqB2F,SAAS,IAEzE/G,GAAGoB,qBAAqBsH,eAAe,EAAEN;QAI7C,MAAMO,YAAY,MAAMpH,GACrB0B,MAAM,CAAC;YACNkB,IAAI/C,qBAAqB+C,EAAE;YAC3B3C,gBAAgBJ,qBAAqBI,cAAc;QACrD,GACC0B,IAAI,CAAC9B,sBACL+B,KAAK,CACJ7D,IACEkJ,gBACArI,GAAG,CAAC;;uBAES,EAAEkB,6BAA6B,IAAI,EAAED,qBAAqBI,cAAc,CAAC;;UAEtF,CAAC,GAGJiF,OAAO,CACNjH,IACEW,GAAG,CAAC,UAAU,EAAEiB,qBAAqBqH,gBAAgB,CAAC,aAAa,EAAErH,qBAAqBsH,eAAe,CAAC,IAAI,EAAEtH,qBAAqB2F,SAAS,CAAC,MAAM,EAAE3F,qBAAqB2F,SAAS,CAAC,MAAM,EAAE3F,qBAAqBsH,eAAe,CAAC,IAAI,CAAC,GAE1OlJ,IAAI4B,qBAAqB+C,EAAE,GAE5BF,KAAK,CAACrB,OAAOqB,KAAK,IAAI;QAEzB,IAAI0E,UAAU7E,MAAM,KAAK,GAAG,OAAO,EAAE;QAErC,MAAM8E,WAAWD,UAAUpF,GAAG,CAAC,CAACzB,MAAQA,IAAIqC,EAAE;QAC9C,MAAM0E,uBAAuBF,UAAUpF,GAAG,CAAC,CAACzB,MAAQA,IAAIN,cAAc;QAEtE,4FAA4F;QAC5F,yFAAyF;QACzF,0FAA0F;QAC1F,yFAAyF;QACzF,MAAMD,GACHqF,MAAM,CAAC5F,sBACPmC,KAAK,CAACtD,QAAQmB,qBAAqBQ,cAAc,EAAEqH;QAEtD,MAAMC,UAAU,MAAMvH,GACnBqF,MAAM,CAACxF,sBACP+B,KAAK,CAAC7D,IAAIO,QAAQuB,qBAAqB+C,EAAE,EAAEyE,WAAWJ,iBACtDvD,SAAS,CAAC;YACTR,eAAerD,qBAAqBqD,aAAa;YACjDC,sBAAsBtD,qBAAqBsD,oBAAoB;YAC/DC,OAAOvD,qBAAqBuD,KAAK;YACjCnD,gBAAgBJ,qBAAqBI,cAAc;YACnDO,eAAeX,qBAAqBW,aAAa;YACjDC,kBAAkBZ,qBAAqBY,gBAAgB;QACzD;QAEF,IAAI8G,QAAQhF,MAAM,KAAK,GAAG,OAAO,EAAE;QAEnC,MAAMpC,0CAA0CH,IAAIuH;QAEpD,MAAM1J,kBACJmC,IACAR,eACA+H,QAAQvF,GAAG,CAAC,CAACzB,MAAS,CAAA;gBACpBkG,MAAM/I;gBACNoF,SAAS;oBACPI,eAAe3C,IAAI2C,aAAa;oBAChCC,sBAAsB5C,IAAI4C,oBAAoB;oBAC9CC,OAAO7C,IAAI6C,KAAK;oBAChBnD,gBAAgBM,IAAIN,cAAc;gBACpC;YACF,CAAA;QAGF,OAAOsH,QAAQvF,GAAG,CAAC,CAAC,EAACkB,aAAa,EAAEC,oBAAoB,EAAEC,KAAK,EAAEnD,cAAc,EAAC,GAAM,CAAA;gBACpFiD;gBACAC;gBACAC;gBACAnD;YACF,CAAA;IACF;IAEA,IAAI2G,OAAOrE,MAAM,GAAG,GAAGtD,8BAA8B4B,GAAG,CAAC+F,OAAOrE,MAAM;IAEtE,OAAOqE;AACT;AAEA,OAAO,eAAeY;IAIpB,MAAM,CAACC,QAAQ,GAAG,MAAMrI,KAAKsC,MAAM,CAAC;QAACgG,OAAOxJ;IAAO,GAAGyD,IAAI,CAAClC;IAC3D,MAAM,CAAC2C,QAAQ,GAAG,MAAMhD,KAAKsC,MAAM,CAAC;QAACgG,OAAOxJ;IAAO,GAAGyD,IAAI,CAAC9B;IAC3D,OAAO;QACLJ,sBAAsBgI,SAASC,SAAS;QACxC7H,sBAAsBuC,SAASsF,SAAS;IAC1C;AACF;AAEA,OAAO,eAAeC,+BAA+BtG,MAIpD;IACC,OAAO,MAAMjC,KACVsC,MAAM,CAAC;QACN0B,OAAOvD,qBAAqBuD,KAAK;QACjCnD,gBAAgBJ,qBAAqBI,cAAc;QACnDiD,eAAerD,qBAAqBqD,aAAa;QACjDC,sBAAsBtD,qBAAqBsD,oBAAoB;QAC/DE,WAAWxD,qBAAqBwD,SAAS;QACzCS,iBAAiBjE,qBAAqBiE,eAAe;QACrDtD,eAAeX,qBAAqBW,aAAa;QACjDC,kBAAkBZ,qBAAqBY,gBAAgB;QACvD6B,gBAAgBzC,qBAAqByC,cAAc;QACnDgD,cAAczF,qBAAqByF,YAAY;QAC/CE,WAAW3F,qBAAqB2F,SAAS;QACzC2B,iBAAiBtH,qBAAqBsH,eAAe;IACvD,GACCxF,IAAI,CAAC9B,sBACL+B,KAAK,CACJ7D,IACEK,GAAGyB,qBAAqBoB,WAAW,EAAEI,OAAOJ,WAAW,GACvDrC,GAAG,CAAC,EAAEiB,qBAAqBsH,eAAe,CAAC,YAAY,EAAE9F,OAAOuG,aAAa,CAAC,yBAAyB,CAAC,GAG3G1C,OAAO,CAAC/G,KAAK0B,qBAAqBsH,eAAe,GAAGhJ,KAAK0B,qBAAqB+C,EAAE,GAChFF,KAAK,CAACrB,OAAOqB,KAAK,IAAI;AAC3B;AAEA,OAAO,eAAemF,2CACpB7H,EAAM,EACNqB,MAIC;IAED,IAAIA,OAAOX,iBAAiB,CAAC6B,MAAM,KAAK,GAAG,OAAO,EAAE;IAEpD,MAAMuF,gBAAgB,MAAM9H,GACzB0B,MAAM,CAAC;QACNjB,kBAAkBZ,qBAAqBY,gBAAgB;QACvDvC,OAAOA;IACT,GACCyD,IAAI,CAAC9B,sBACL+B,KAAK,CACJ7D,IACEK,GAAGyB,qBAAqBoB,WAAW,EAAEI,OAAOJ,WAAW,GACvD7C,GAAGyB,qBAAqBW,aAAa,EAAEa,OAAOb,aAAa,GAC3DlC,QAAQuB,qBAAqBY,gBAAgB,EAAEY,OAAOX,iBAAiB,IAG1EoB,OAAO,CAACjC,qBAAqBY,gBAAgB,EAC7CsH,MAAM,CAACnJ,GAAG,CAAC,YAAY,CAAC;IAE3B,MAAMoJ,6BAA6BF,cAAcG,OAAO,CAAC,CAAC1H,MACxDA,IAAIE,gBAAgB,GAAG;YAACF,IAAIE,gBAAgB;SAAC,GAAG,EAAE;IAEpD,IAAIuH,2BAA2BzF,MAAM,GAAG,GAAG;QACzC5E,SAASuK,IAAI,CACX;YACEjH,aAAaI,OAAOJ,WAAW;YAC/BT,eAAea,OAAOb,aAAa;YACnCE,mBAAmBsH;QACrB,GACA;IAEJ;IAEA,MAAM5D,SAAS,MAAMpE,GAAGE,OAAO,CAS5BtB,GAAG,CAAC;wBACe,EAAEiB,qBAAqBY,gBAAgB,CAAC;MAC1D,EAAEZ,qBAAqBqD,aAAa,CAAC;MACrC,EAAErD,qBAAqBsD,oBAAoB,CAAC;MAC5C,EAAEtD,qBAAqBuD,KAAK,CAAC;MAC7B,EAAEvD,qBAAqBI,cAAc,CAAC;MACtC,EAAEJ,qBAAqBY,gBAAgB,CAAC;MACxC,EAAEZ,qBAAqB2F,SAAS,CAAC;MACjC,EAAE3F,qBAAqBsH,eAAe,CAAC;MACvC,EAAEtH,qBAAqBsI,uBAAuB,CAAC;SAC5C,EAAEtI,qBAAqB;;MAE1B,EAAEA,qBAAqBoB,WAAW,CAAC,GAAG,EAAEI,OAAOJ,WAAW,CAAC;UACvD,EAAEpB,qBAAqBW,aAAa,CAAC,GAAG,EAAEa,OAAOb,aAAa,CAAC;UAC/D,EAAEX,qBAAqBY,gBAAgB,CAAC,KAAK,EAAE7B,IAAIwJ,IAAI,CACzD/G,OAAOX,iBAAiB,CAACsB,GAAG,CAAC,CAACvB,mBAAqB7B,GAAG,CAAC,EAAE6B,iBAAiB,CAAC,GAC3E7B,GAAG,CAAC,EAAE,CAAC,EACP;aACK,EAAEiB,qBAAqBY,gBAAgB,CAAC,EAAE,EAAEZ,qBAAqB2F,SAAS,CAAC,OAAO,EAAE3F,qBAAqBI,cAAc,CAAC;EACnI,CAAC;IAED,OAAOmE,OAAOhE,IAAI,CAAC4B,GAAG,CAAC,CAACzB,MAAS,CAAA;YAC/B2C,eAAe3C,IAAI2C,aAAa;YAChCC,sBAAsB5C,IAAI4C,oBAAoB;YAC9CC,OAAO7C,IAAI6C,KAAK;YAChBnD,gBAAgBM,IAAIN,cAAc;YAClCQ,kBAAkBF,IAAIE,gBAAgB;YACtC+E,WAAW6C,OAAO9H,IAAIiF,SAAS;YAC/B2B,iBAAiBkB,OAAO9H,IAAI4G,eAAe;YAC3CgB,yBAAyB5H,IAAI4H,uBAAuB,GAChDE,OAAO9H,IAAI4H,uBAAuB,IAClC;QACN,CAAA;AACF;AAEA,SAASE,OAAOX,KAAoB;IAClC,OAAOA,iBAAiBY,OAAOZ,QAAQ,IAAIY,KAAKZ;AAClD;AAEA,OAAO,eAAea,iBAAiBlH,MAItC;IACC,MAAM,CAACd,IAAI,GAAG,MAAMnB,KACjBsC,MAAM,CAAC;QAACkB,IAAI/C,qBAAqB+C,EAAE;IAAA,GACnCjB,IAAI,CAAC9B,sBACL+B,KAAK,CACJ7D,IACEK,GAAGyB,qBAAqBI,cAAc,EAAEoB,OAAOpB,cAAc,GAC7D7B,GAAGyB,qBAAqBiE,eAAe,EAAEzC,OAAOyC,eAAe,GAC/DzC,OAAO+B,KAAK,KAAKO,YAAYA,YAAYvF,GAAGyB,qBAAqBuD,KAAK,EAAE/B,OAAO+B,KAAK,IAGvFV,KAAK,CAAC;IAET,OAAOnC,QAAQoD;AACjB;AAEA,OAAO,eAAe6E,gBAAgBnH,MAIrC;IAcC,MAAM+C,SAAS,MAAMhF,KAAKqD,WAAW,CAAC,OAAOzC;QAC3C,MAAMyI,UAAU,MAAMzI,GACnB+F,MAAM,CAAClG,sBACPiB,GAAG,CAAC;YACHoG,kBAAkBtI,GAAG,CAAC,SAAS,EAAEiB,qBAAqBqH,gBAAgB,CAAC,QAAQ,CAAC;YAChFC,iBAAiBvI,GAAG,CAAC,KAAK,CAAC;QAC7B,GACCgD,KAAK,CACJ7D,IACEK,GAAGyB,qBAAqBI,cAAc,EAAEoB,OAAOpB,cAAc,GAC7D7B,GAAGyB,qBAAqBiE,eAAe,EAAEzC,OAAOyC,eAAe,IAGlEJ,SAAS,CAAC;YACTyE,yBAAyBtI,qBAAqBsI,uBAAuB;YACrEjF,eAAerD,qBAAqBqD,aAAa;YACjDC,sBAAsBtD,qBAAqBsD,oBAAoB;YAC/DC,OAAOvD,qBAAqBuD,KAAK;YACjCnD,gBAAgBJ,qBAAqBI,cAAc;YACnDoD,WAAWxD,qBAAqBwD,SAAS;YACzCpC,aAAapB,qBAAqBoB,WAAW;YAC7C6C,iBAAiBjE,qBAAqBiE,eAAe;QACvD;QAEF,MAAMvD,MAAMkI,OAAO,CAAC,EAAE;QACtB,IAAI,CAAClI,KAAK,MAAM,IAAIxB,iCAAiCsC,OAAOpB,cAAc;QAE1E,MAAM,CAACyI,SAAS,GAAG,MAAM1I,GACtB0B,MAAM,CAAC;YAACiH,kBAAkB/I,eAAe+I,gBAAgB;QAAA,GACzDhH,IAAI,CAAC/B,gBACLgC,KAAK,CAACxD,GAAGwB,eAAegD,EAAE,EAAEvB,OAAOyC,eAAe,GAClDpB,KAAK,CAAC,GACNgC,GAAG,CAAC;QAEP,MAAM,CAACH,QAAQ,GAAG,MAAMvE,GACrB+F,MAAM,CAACnG,gBACPkB,GAAG,CAAC;YACH6H,kBAAkBtH,OAAOsH,gBAAgB,IAAI;YAC7CC,4BAA4BvH,OAAOsH,gBAAgB,GAAG/J,GAAG,CAAC,KAAK,CAAC,GAAG;YACnEqH,WAAWrH,GAAG,CAAC,KAAK,CAAC;QACvB,GACCgD,KAAK,CAACxD,GAAGwB,eAAegD,EAAE,EAAEvB,OAAOyC,eAAe,GAClDJ,SAAS,CAAC;YAACiF,kBAAkB/I,eAAe+I,gBAAgB;QAAA;QAE/D,OAAO;YACLpI;YACAsI,0BAA0BH,UAAUC,oBAAoB;YACxDG,yBAAyBvE,SAASoE,oBAAoB;QACxD;IACF;IAEA,MAAMpI,MAAM6D,OAAO7D,GAAG;IACtB,IAAI,CAACA,KAAK,MAAM,IAAIxB,iCAAiCsC,OAAOpB,cAAc;IAC1E,OAAO;QACL8I,uBAAuBxI,IAAI4H,uBAAuB,KAAK;QACvDU,0BAA0BzE,OAAOyE,wBAAwB;QACzDC,yBAAyB1E,OAAO0E,uBAAuB;QACvDE,qBAAqB;YACnB9F,eAAe3C,IAAI2C,aAAa;YAChCC,sBAAsB5C,IAAI4C,oBAAoB;YAC9CC,OAAO7C,IAAI6C,KAAK;YAChBnD,gBAAgBM,IAAIN,cAAc;YAClCoD,WAAW9C,IAAI8C,SAAS;YACxBpC,aAAaV,IAAIU,WAAW;YAC5B6C,iBAAiBvD,IAAIuD,eAAe;QACtC;IACF;AACF;AAEA;;;;CAIC,GACD,OAAO,eAAemF,8BAA8B5H,MAEnD;IACC,MAAMjC,KAAKqD,WAAW,CAAC,OAAOzC;QAC5B,MAAMD,iBAAiBC,IAAIqB,OAAOpB,cAAc;QAEhD,wFAAwF;QACxF,2FAA2F;QAC3F,wFAAwF;QACxF,MAAMD,GACHqF,MAAM,CAAC5F,sBACPmC,KAAK,CAACxD,GAAGqB,qBAAqBQ,cAAc,EAAEoB,OAAOpB,cAAc;QACtE,MAAMiJ,uBAAuB,MAAMlJ,GAChC+F,MAAM,CAAClG,sBACPiB,GAAG,CAAC;YACHqH,yBAAyBvJ,GAAG,CAAC,SAAS,EAAEiB,qBAAqBsI,uBAAuB,CAAC,QAAQ,CAAC;QAChG,GACCvG,KAAK,CAACxD,GAAGyB,qBAAqBI,cAAc,EAAEoB,OAAOpB,cAAc,GACnEyD,SAAS,CAAC;YACTlD,eAAeX,qBAAqBW,aAAa;YACjDC,kBAAkBZ,qBAAqBY,gBAAgB;QACzD;QAEF,MAAMN,0CAA0CH,IAAIkJ;IACtD;AACF"}

package/dist/presentation/routes/list-active-provisioners.d.ts

@@ -1,2 +1,6 @@
+import { installationRunnersStatus } from '#core/index.js';
+type InstallationRunnersStatusReader = () => ReturnType<typeof installationRunnersStatus>;
+export declare function createListActiveProvisionersRoute(readInstallationRunnersStatus?: InstallationRunnersStatusReader): import("@shipfox/node-fastify").RouteDefinition;
 export declare const listActiveProvisionersRoute: import("@shipfox/node-fastify").RouteDefinition;
+export {};
 //# sourceMappingURL=list-active-provisioners.d.ts.map

package/dist/presentation/routes/list-active-provisioners.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"list-active-provisioners.d.ts","sourceRoot":"","sources":["../../../src/presentation/routes/list-active-provisioners.ts"],"names":[],"mappings":"AAOA,eAAO,MAAM,2BAA2B,iDAiBtC,CAAC"}
+{"version":3,"file":"list-active-provisioners.d.ts","sourceRoot":"","sources":["../../../src/presentation/routes/list-active-provisioners.ts"],"names":[],"mappings":"AAIA,OAAO,EAAC,yBAAyB,EAAyB,MAAM,gBAAgB,CAAC;AAGjF,KAAK,+BAA+B,GAAG,MAAM,UAAU,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAE1F,wBAAgB,iCAAiC,CAC/C,6BAA6B,GAAE,+BAA2D,mDA6B3F;AAED,eAAO,MAAM,2BAA2B,iDAAsC,CAAC"}

package/dist/presentation/routes/list-active-provisioners.js

@@ -2,31 +2,43 @@ import { requireWorkspaceAccess } from '@shipfox/api-auth-context';
 import { listActiveProvisionersResponseSchema } from '@shipfox/api-runners-dto';
 import { defineRoute } from '@shipfox/node-fastify';
 import { z } from 'zod';
-import { listActiveProvisioners } from '#core/index.js';
+import { installationRunnersStatus, listActiveProvisioners } from '#core/index.js';
 import { toActiveProvisionerDto } from '#presentation/dto/index.js';
-export const listActiveProvisionersRoute = defineRoute({
-    method: 'GET',
-    path: '/',
-    description: 'List active provisioners for a workspace',
-    schema: {
-        params: z.object({
-            workspaceId: z.string().uuid()
-        }),
-        response: {
-            200: listActiveProvisionersResponseSchema
+export function createListActiveProvisionersRoute(readInstallationRunnersStatus = installationRunnersStatus) {
+    return defineRoute({
+        method: 'GET',
+        path: '/',
+        description: 'List active provisioners for a workspace',
+        schema: {
+            params: z.object({
+                workspaceId: z.string().uuid()
+            }),
+            response: {
+                200: listActiveProvisionersResponseSchema
+            }
+        },
+        handler: async (request)=>{
+            const { workspaceId } = request.params;
+            requireWorkspaceAccess({
+                request,
+                workspaceId
+            });
+            const [provisioners, installationRunners] = await Promise.all([
+                listActiveProvisioners(workspaceId),
+                readInstallationRunnersStatus().catch((error)=>{
+                    request.log.warn({
+                        err: error
+                    }, 'Could not resolve installation runner status');
+                    return 'none';
+                })
+            ]);
+            return {
+                provisioners: provisioners.map(toActiveProvisionerDto),
+                installation_runners: installationRunners
+            };
         }
-    },
-    handler: async (request)=>{
-        const { workspaceId } = request.params;
-        requireWorkspaceAccess({
-            request,
-            workspaceId
-        });
-        const provisioners = await listActiveProvisioners(workspaceId);
-        return {
-            provisioners: provisioners.map(toActiveProvisionerDto)
-        };
-    }
-});
+    });
+}
+export const listActiveProvisionersRoute = createListActiveProvisionersRoute();
 
 //# sourceMappingURL=list-active-provisioners.js.map

package/dist/presentation/routes/list-active-provisioners.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/presentation/routes/list-active-provisioners.ts"],"sourcesContent":["import {requireWorkspaceAccess} from '@shipfox/api-auth-context';\nimport {listActiveProvisionersResponseSchema} from '@shipfox/api-runners-dto';\nimport {defineRoute} from '@shipfox/node-fastify';\nimport {z} from 'zod';\nimport {listActiveProvisioners} from '#core/index.js';\nimport {toActiveProvisionerDto} from '#presentation/dto/index.js';\n\nexport const listActiveProvisionersRoute = defineRoute({\n  method: 'GET',\n  path: '/',\n  description: 'List active provisioners for a workspace',\n  schema: {\n    params: z.object({workspaceId: z.string().uuid()}),\n    response: {\n      200: listActiveProvisionersResponseSchema,\n    },\n  },\n  handler: async (request) => {\n    const {workspaceId} = request.params;\n    requireWorkspaceAccess({request, workspaceId});\n\n    const provisioners = await listActiveProvisioners(workspaceId);\n    return {provisioners: provisioners.map(toActiveProvisionerDto)};\n  },\n});\n"],"names":["requireWorkspaceAccess","listActiveProvisionersResponseSchema","defineRoute","z","listActiveProvisioners","toActiveProvisionerDto","listActiveProvisionersRoute","method","path","description","schema","params","object","workspaceId","string","uuid","response","handler","request","provisioners","map"],"mappings":"AAAA,SAAQA,sBAAsB,QAAO,4BAA4B;AACjE,SAAQC,oCAAoC,QAAO,2BAA2B;AAC9E,SAAQC,WAAW,QAAO,wBAAwB;AAClD,SAAQC,CAAC,QAAO,MAAM;AACtB,SAAQC,sBAAsB,QAAO,iBAAiB;AACtD,SAAQC,sBAAsB,QAAO,6BAA6B;AAElE,OAAO,MAAMC,8BAA8BJ,YAAY;IACrDK,QAAQ;IACRC,MAAM;IACNC,aAAa;IACbC,QAAQ;QACNC,QAAQR,EAAES,MAAM,CAAC;YAACC,aAAaV,EAAEW,MAAM,GAAGC,IAAI;QAAE;QAChDC,UAAU;YACR,KAAKf;QACP;IACF;IACAgB,SAAS,OAAOC;QACd,MAAM,EAACL,WAAW,EAAC,GAAGK,QAAQP,MAAM;QACpCX,uBAAuB;YAACkB;YAASL;QAAW;QAE5C,MAAMM,eAAe,MAAMf,uBAAuBS;QAClD,OAAO;YAACM,cAAcA,aAAaC,GAAG,CAACf;QAAuB;IAChE;AACF,GAAG"}
+{"version":3,"sources":["../../../src/presentation/routes/list-active-provisioners.ts"],"sourcesContent":["import {requireWorkspaceAccess} from '@shipfox/api-auth-context';\nimport {listActiveProvisionersResponseSchema} from '@shipfox/api-runners-dto';\nimport {defineRoute} from '@shipfox/node-fastify';\nimport {z} from 'zod';\nimport {installationRunnersStatus, listActiveProvisioners} from '#core/index.js';\nimport {toActiveProvisionerDto} from '#presentation/dto/index.js';\n\ntype InstallationRunnersStatusReader = () => ReturnType<typeof installationRunnersStatus>;\n\nexport function createListActiveProvisionersRoute(\n  readInstallationRunnersStatus: InstallationRunnersStatusReader = installationRunnersStatus,\n) {\n  return defineRoute({\n    method: 'GET',\n    path: '/',\n    description: 'List active provisioners for a workspace',\n    schema: {\n      params: z.object({workspaceId: z.string().uuid()}),\n      response: {\n        200: listActiveProvisionersResponseSchema,\n      },\n    },\n    handler: async (request) => {\n      const {workspaceId} = request.params;\n      requireWorkspaceAccess({request, workspaceId});\n\n      const [provisioners, installationRunners] = await Promise.all([\n        listActiveProvisioners(workspaceId),\n        readInstallationRunnersStatus().catch((error: unknown) => {\n          request.log.warn({err: error}, 'Could not resolve installation runner status');\n          return 'none' as const;\n        }),\n      ]);\n      return {\n        provisioners: provisioners.map(toActiveProvisionerDto),\n        installation_runners: installationRunners,\n      };\n    },\n  });\n}\n\nexport const listActiveProvisionersRoute = createListActiveProvisionersRoute();\n"],"names":["requireWorkspaceAccess","listActiveProvisionersResponseSchema","defineRoute","z","installationRunnersStatus","listActiveProvisioners","toActiveProvisionerDto","createListActiveProvisionersRoute","readInstallationRunnersStatus","method","path","description","schema","params","object","workspaceId","string","uuid","response","handler","request","provisioners","installationRunners","Promise","all","catch","error","log","warn","err","map","installation_runners","listActiveProvisionersRoute"],"mappings":"AAAA,SAAQA,sBAAsB,QAAO,4BAA4B;AACjE,SAAQC,oCAAoC,QAAO,2BAA2B;AAC9E,SAAQC,WAAW,QAAO,wBAAwB;AAClD,SAAQC,CAAC,QAAO,MAAM;AACtB,SAAQC,yBAAyB,EAAEC,sBAAsB,QAAO,iBAAiB;AACjF,SAAQC,sBAAsB,QAAO,6BAA6B;AAIlE,OAAO,SAASC,kCACdC,gCAAiEJ,yBAAyB;IAE1F,OAAOF,YAAY;QACjBO,QAAQ;QACRC,MAAM;QACNC,aAAa;QACbC,QAAQ;YACNC,QAAQV,EAAEW,MAAM,CAAC;gBAACC,aAAaZ,EAAEa,MAAM,GAAGC,IAAI;YAAE;YAChDC,UAAU;gBACR,KAAKjB;YACP;QACF;QACAkB,SAAS,OAAOC;YACd,MAAM,EAACL,WAAW,EAAC,GAAGK,QAAQP,MAAM;YACpCb,uBAAuB;gBAACoB;gBAASL;YAAW;YAE5C,MAAM,CAACM,cAAcC,oBAAoB,GAAG,MAAMC,QAAQC,GAAG,CAAC;gBAC5DnB,uBAAuBU;gBACvBP,gCAAgCiB,KAAK,CAAC,CAACC;oBACrCN,QAAQO,GAAG,CAACC,IAAI,CAAC;wBAACC,KAAKH;oBAAK,GAAG;oBAC/B,OAAO;gBACT;aACD;YACD,OAAO;gBACLL,cAAcA,aAAaS,GAAG,CAACxB;gBAC/ByB,sBAAsBT;YACxB;QACF;IACF;AACF;AAEA,OAAO,MAAMU,8BAA8BzB,oCAAoC"}