@shipfox/api-runners 12.4.0 → 12.5.0

package/dist/db/job-executions.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/db/job-executions.ts"],"sourcesContent":["import {\n  RUNNER_JOB_CLAIMED,\n  RUNNER_JOB_LEASE_EXPIRED,\n  RUNNER_JOB_QUEUED,\n  type RunnersEventMap,\n  type RunnerToolCapabilitiesDto,\n} from '@shipfox/api-runners-dto';\nimport {logger} from '@shipfox/node-opentelemetry';\nimport {writeOutboxEvent, writeOutboxEvents} from '@shipfox/node-outbox';\nimport {canonicalizeLabels} from '@shipfox/runner-labels';\nimport {\n  and,\n  arrayContained,\n  asc,\n  count,\n  desc,\n  eq,\n  gt,\n  inArray,\n  isNotNull,\n  isNull,\n  lt,\n  lte,\n  or,\n  sql,\n} from 'drizzle-orm';\nimport {\n  EmptyRequiredLabelsError,\n  RunnerSessionExhaustedError,\n  RunningJobExecutionNotFoundError,\n} from '#core/errors.js';\nimport {jobExecutionEnqueuedCount, jobExecutionLeaseExpiredCount} from '#metrics/instance.js';\nimport type {Tx} from './db.js';\nimport {db} from './db.js';\nimport {runnersOutbox} from './schema/outbox.js';\nimport {pendingJobExecutions} from './schema/pending-job-executions.js';\nimport {providerRunners} from './schema/runner-instances.js';\nimport {runnerSessions} from './schema/runner-sessions.js';\nimport {runningJobExecutions} from './schema/running-job-executions.js';\n\nconst runnerJobExecutionLockPrefix = 'runners_job_execution:';\n\nasync function lockJobExecution(tx: Tx, jobExecutionId: string): Promise<void> {\n  await tx.execute(\n    sql`select pg_advisory_xact_lock(hashtext(${`${runnerJobExecutionLockPrefix}${jobExecutionId}`}))`,\n  );\n}\n\nexport interface EnqueueJobExecutionParams {\n  workspaceId: string;\n  workflowRunId: string;\n  workflowRunAttemptId: string;\n  jobId: string;\n  jobExecutionId: string;\n  projectId: string;\n  requiredLabels: string[];\n}\n\nexport async function getWorkspaceJobCounts(params: {\n  workspaceIds: string[];\n}): Promise<Array<{workspaceId: string; queued: number; running: number}>> {\n  const [pendingRows, runningRows] = await Promise.all([\n    db()\n      .select({workspaceId: pendingJobExecutions.workspaceId, count: count()})\n      .from(pendingJobExecutions)\n      .where(inArray(pendingJobExecutions.workspaceId, params.workspaceIds))\n      .groupBy(pendingJobExecutions.workspaceId),\n    db()\n      .select({workspaceId: runningJobExecutions.workspaceId, count: count()})\n      .from(runningJobExecutions)\n      .where(inArray(runningJobExecutions.workspaceId, params.workspaceIds))\n      .groupBy(runningJobExecutions.workspaceId),\n  ]);\n\n  const queuedByWorkspace = new Map(pendingRows.map((row) => [row.workspaceId, Number(row.count)]));\n  const runningByWorkspace = new Map(\n    runningRows.map((row) => [row.workspaceId, Number(row.count)]),\n  );\n\n  return params.workspaceIds.map((workspaceId) => ({\n    workspaceId,\n    queued: queuedByWorkspace.get(workspaceId) ?? 0,\n    running: runningByWorkspace.get(workspaceId) ?? 0,\n  }));\n}\n\n// Idempotent while the job execution is still pending: a duplicate jobExecutionId already in\n// `runners_pending_jobs` is a no-op. Temporal retries the enqueue activity at-least-once, so a\n// unique-violation throw on a retry-after-lost-result would permanently fail a healthy execution.\n// The per-execution advisory lock serializes retries with lease expiry/reconciliation, and the\n// durable lease-expired event prevents a retry from re-queueing an execution already reaped.\nexport async function enqueueJobExecution(params: EnqueueJobExecutionParams): Promise<void> {\n  const requiredLabels = [...canonicalizeLabels(params.requiredLabels)];\n  if (requiredLabels.length === 0) throw new EmptyRequiredLabelsError();\n\n  const enqueued = await db().transaction(async (tx) => {\n    await lockJobExecution(tx, params.jobExecutionId);\n\n    const [leaseExpired] = await tx\n      .select({id: runnersOutbox.id})\n      .from(runnersOutbox)\n      .where(\n        and(\n          eq(runnersOutbox.eventType, RUNNER_JOB_LEASE_EXPIRED),\n          sql`${runnersOutbox.payload}->>'jobExecutionId' = ${params.jobExecutionId}`,\n        ),\n      )\n      .limit(1);\n    if (leaseExpired) return false;\n\n    const [inserted] = await tx\n      .insert(pendingJobExecutions)\n      .values({\n        workspaceId: params.workspaceId,\n        workflowRunId: params.workflowRunId,\n        workflowRunAttemptId: params.workflowRunAttemptId,\n        jobId: params.jobId,\n        jobExecutionId: params.jobExecutionId,\n        projectId: params.projectId,\n        requiredLabels,\n      })\n      .onConflictDoNothing({target: pendingJobExecutions.jobExecutionId})\n      .returning({createdAt: pendingJobExecutions.createdAt});\n\n    // A retry that hits the conflict inserts nothing: the first enqueue already\n    // emitted the queued event (durably, in the outbox), so re-emitting would\n    // only add a redundant row the subscriber coalesces away. Skip it.\n    if (!inserted) return false;\n\n    await writeOutboxEvent<RunnersEventMap>(tx, runnersOutbox, {\n      type: RUNNER_JOB_QUEUED,\n      payload: {\n        workflowRunId: params.workflowRunId,\n        workflowRunAttemptId: params.workflowRunAttemptId,\n        jobId: params.jobId,\n        jobExecutionId: params.jobExecutionId,\n        queuedAt: inserted.createdAt.toISOString(),\n      },\n    });\n    return true;\n  });\n\n  if (enqueued) jobExecutionEnqueuedCount.add(1);\n}\n\nexport interface ClaimedJobExecution {\n  workflowRunId: string;\n  workflowRunAttemptId: string;\n  jobId: string;\n  jobExecutionId: string;\n  projectId: string;\n}\n\nexport interface ActiveRunningJobExecution {\n  workflowRunId: string;\n  workflowRunAttemptId: string;\n  jobId: string;\n  jobExecutionId: string;\n  projectId: string;\n  runnerSessionId: string;\n  provisionerId: string | null;\n  providerRunnerId: string | null;\n  requiredLabels: string[];\n  runnerLabels: string[];\n  startedAt: Date;\n  lastHeartbeatAt: Date;\n}\n\nexport interface RunnerInstanceBoundJobExecution {\n  workflowRunId: string;\n  workflowRunAttemptId: string;\n  jobId: string;\n  jobExecutionId: string;\n  providerRunnerId: string;\n  startedAt: Date;\n  lastHeartbeatAt: Date;\n  cancellationRequestedAt: Date | null;\n}\n\nexport async function claimPendingJobExecution(params: {\n  workspaceId: string;\n  runnerSessionId: string;\n  sessionLabels: string[];\n  maxClaims: number | null;\n  runnerSessionLivenessThrottleSeconds: number;\n}): Promise<ClaimedJobExecution | null> {\n  await touchRunnerSessionLiveness({\n    workspaceId: params.workspaceId,\n    runnerSessionId: params.runnerSessionId,\n    throttleSeconds: params.runnerSessionLivenessThrottleSeconds,\n  });\n\n  if (params.sessionLabels.length === 0) return null;\n\n  return await db().transaction(async (tx) => {\n    let provisionerId: string | null = null;\n    let providerRunnerId: string | null = null;\n    let runnerInstanceId: string | null = null;\n\n    if (params.maxClaims !== null) {\n      const [session] = await tx\n        .select({\n          maxClaims: runnerSessions.maxClaims,\n          claimsUsed: runnerSessions.claimsUsed,\n          revokedAt: runnerSessions.revokedAt,\n          runnerInstanceId: runnerSessions.runnerInstanceId,\n          provisionerId: runnerSessions.provisionerId,\n          providerRunnerId: runnerSessions.providerRunnerId,\n        })\n        .from(runnerSessions)\n        .where(eq(runnerSessions.id, params.runnerSessionId))\n        .limit(1)\n        .for('update');\n\n      if (\n        !session ||\n        session.revokedAt ||\n        session.maxClaims === null ||\n        session.claimsUsed >= session.maxClaims\n      ) {\n        throw new RunnerSessionExhaustedError(params.runnerSessionId);\n      }\n\n      // Ephemeral sessions are the only capped sessions, and the DB check keeps\n      // their provisioned-runner link present as a pair.\n      runnerInstanceId = session.runnerInstanceId;\n      provisionerId = session.provisionerId;\n      providerRunnerId = session.providerRunnerId;\n    }\n\n    // `id` is a uuidv7 (time-ordered), so it is a deterministic FIFO tiebreaker\n    // for rows sharing a created_at within a batch. Lock only the FIFO candidate before\n    // attempting its execution advisory lock; putting pg_try_advisory_xact_lock in this\n    // predicate would evaluate it while scanning and temporarily lock many queue entries.\n    const candidate = tx\n      .select()\n      .from(pendingJobExecutions)\n      .where(\n        and(\n          eq(pendingJobExecutions.workspaceId, params.workspaceId),\n          arrayContained(pendingJobExecutions.requiredLabels, params.sessionLabels),\n        ),\n      )\n      .orderBy(asc(pendingJobExecutions.createdAt), asc(pendingJobExecutions.id))\n      .limit(1)\n      .for('update', {skipLocked: true})\n      .as('pending_candidate');\n\n    const [row] = await tx\n      .select()\n      .from(candidate)\n      .where(\n        sql`\n          pg_try_advisory_xact_lock(\n            hashtext(${runnerJobExecutionLockPrefix} || ${candidate.jobExecutionId}::text)\n          )\n        `,\n      );\n\n    if (!row) return null;\n\n    await tx.delete(pendingJobExecutions).where(eq(pendingJobExecutions.id, row.id));\n\n    // An enqueue retry that lands after a prior claim can leave an orphan pending\n    // row whose jobExecutionId is already in `runners_running_jobs`. Insert-or-skip\n    // on the jobExecutionId unique constraint: when the job execution is already running\n    // the insert touches no row, so we commit the orphan's deletion and return\n    // null rather than let the unique violation roll the claim back into a poison\n    // loop. The runner just re-polls for a real job execution.\n    const inserted = await tx\n      .insert(runningJobExecutions)\n      .values({\n        workspaceId: row.workspaceId,\n        workflowRunId: row.workflowRunId,\n        workflowRunAttemptId: row.workflowRunAttemptId,\n        jobId: row.jobId,\n        jobExecutionId: row.jobExecutionId,\n        projectId: row.projectId,\n        runnerSessionId: params.runnerSessionId,\n        provisionerId,\n        providerRunnerId,\n        requiredLabels: row.requiredLabels,\n        runnerLabels: params.sessionLabels,\n      })\n      .onConflictDoNothing({target: runningJobExecutions.jobExecutionId})\n      .returning({claimedAt: runningJobExecutions.startedAt});\n\n    const claimed = inserted[0];\n    if (!claimed) return null;\n\n    const runnerInstanceCondition = runnerInstanceId\n      ? eq(providerRunners.id, runnerInstanceId)\n      : provisionerId && providerRunnerId\n        ? and(\n            eq(providerRunners.provisionerId, provisionerId),\n            eq(providerRunners.providerRunnerId, providerRunnerId),\n          )\n        : null;\n    if (runnerInstanceCondition) {\n      await tx\n        .update(providerRunners)\n        .set({\n          firstClaimedAt: sql`coalesce(${providerRunners.firstClaimedAt}, ${claimed.claimedAt})`,\n          updatedAt: sql`now()`,\n        })\n        .where(runnerInstanceCondition);\n    }\n\n    if (params.maxClaims !== null) {\n      await tx\n        .update(runnerSessions)\n        .set({claimsUsed: sql`${runnerSessions.claimsUsed} + 1`, updatedAt: sql`now()`})\n        .where(eq(runnerSessions.id, params.runnerSessionId));\n    }\n\n    // The running-row insert is the runner claiming the job execution. Emit in the same tx; the\n    // payload carries the row's own claim instant so a consumer records the true time,\n    // not the outbox drain time.\n    await writeOutboxEvent<RunnersEventMap>(tx, runnersOutbox, {\n      type: RUNNER_JOB_CLAIMED,\n      payload: {\n        workflowRunId: row.workflowRunId,\n        workflowRunAttemptId: row.workflowRunAttemptId,\n        jobId: row.jobId,\n        jobExecutionId: row.jobExecutionId,\n        claimedAt: claimed.claimedAt.toISOString(),\n      },\n    });\n\n    return {\n      workflowRunId: row.workflowRunId,\n      workflowRunAttemptId: row.workflowRunAttemptId,\n      jobId: row.jobId,\n      jobExecutionId: row.jobExecutionId,\n      projectId: row.projectId,\n    };\n  });\n}\n\nasync function touchRunnerSessionLiveness(params: {\n  workspaceId: string;\n  runnerSessionId: string;\n  throttleSeconds: number;\n}): Promise<void> {\n  await db()\n    .update(runnerSessions)\n    .set({updatedAt: sql`now()`})\n    .where(\n      and(\n        eq(runnerSessions.id, params.runnerSessionId),\n        eq(runnerSessions.workspaceId, params.workspaceId),\n        sql`${runnerSessions.updatedAt} < now() - (${params.throttleSeconds} || ' seconds')::interval`,\n      ),\n    );\n}\n\n/**\n * Releases a job execution's lease when the orchestration workflow finalizes it: deletes the\n * running-job-execution row AND any lingering pending row for the same execution, in one tx.\n * Idempotent (0-row no-op), no token scope (the workflow is authoritative), and\n * emits no event: the workflow already owns the outcome. Sweeping the pending row\n * too closes the at-least-once window where an enqueue retry left an orphan that a\n * later claim would otherwise pick up for an already-finished job execution.\n */\nexport async function releaseJobExecution(params: {jobExecutionId: string}): Promise<void> {\n  await db().transaction(async (tx) => {\n    await lockJobExecution(tx, params.jobExecutionId);\n\n    // Delete pending before running to match `claimPendingJobExecution`'s lock-acquisition\n    // order (it locks the pending row first, then the running row). A concurrent\n    // claim picking up an orphan pending row for this same job execution would otherwise\n    // deadlock against the reverse order here.\n    await tx\n      .delete(pendingJobExecutions)\n      .where(eq(pendingJobExecutions.jobExecutionId, params.jobExecutionId));\n    await tx\n      .delete(runningJobExecutions)\n      .where(eq(runningJobExecutions.jobExecutionId, params.jobExecutionId));\n  });\n}\n\n/**\n * Reaps stale leases (bounded by `limit`), emitting one\n * `runners.job.lease_expired` event per reaped job execution.\n *\n * The cutoff is re-checked in the DELETE, not just the locking subquery, so a\n * heartbeat landing mid-call spares the live row. Each reaped execution also sweeps its\n * pending row: a failed best-effort `releaseJobExecution` would otherwise leave an orphan\n * that a later claim re-runs as an already-finished job execution.\n *\n * Locks pending-then-running to match `claimPendingJobExecution`, `releaseJobExecution`, and\n * `reconcileTerminalJobExecution`. The stale candidate scan intentionally does not lock running\n * rows first; the running-row DELETE re-checks the stale predicate after the pending-row sweep.\n */\nexport async function expireStuckJobExecutions(params: {\n  thresholdSeconds: number;\n  noFirstHeartbeatGraceSeconds: number;\n  limit?: number;\n}): Promise<\n  Array<{\n    workflowRunId: string;\n    workflowRunAttemptId: string;\n    jobId: string;\n    jobExecutionId: string;\n  }>\n> {\n  const reaped = await db().transaction(async (tx) => {\n    const heartbeatCutoff = sql`now() - (${params.thresholdSeconds} || ' seconds')::interval`;\n    const firstHeartbeatCutoff = sql`now() - (${params.noFirstHeartbeatGraceSeconds} || ' seconds')::interval`;\n    const stalePredicate = or(\n      and(\n        isNull(runningJobExecutions.firstHeartbeatAt),\n        lte(runningJobExecutions.lastHeartbeatAt, runningJobExecutions.startedAt),\n        lt(runningJobExecutions.startedAt, firstHeartbeatCutoff),\n      ),\n      and(\n        or(\n          isNotNull(runningJobExecutions.firstHeartbeatAt),\n          gt(runningJobExecutions.lastHeartbeatAt, runningJobExecutions.startedAt),\n        ),\n        lt(runningJobExecutions.lastHeartbeatAt, heartbeatCutoff),\n      ),\n    );\n\n    const staleRows = await tx\n      .select({\n        id: runningJobExecutions.id,\n        jobExecutionId: runningJobExecutions.jobExecutionId,\n      })\n      .from(runningJobExecutions)\n      .where(\n        and(\n          stalePredicate,\n          sql`\n            pg_try_advisory_xact_lock(\n              hashtext(${runnerJobExecutionLockPrefix} || ${runningJobExecutions.jobExecutionId}::text)\n            )\n          `,\n        ),\n      )\n      .orderBy(\n        asc(\n          sql`CASE WHEN ${runningJobExecutions.firstHeartbeatAt} IS NULL AND ${runningJobExecutions.lastHeartbeatAt} <= ${runningJobExecutions.startedAt} THEN ${runningJobExecutions.startedAt} ELSE ${runningJobExecutions.lastHeartbeatAt} END`,\n        ),\n        asc(runningJobExecutions.id),\n      )\n      .limit(params.limit ?? 100);\n\n    if (staleRows.length === 0) return [];\n\n    const staleIds = staleRows.map((row) => row.id);\n    const staleJobExecutionIds = staleRows.map((row) => row.jobExecutionId);\n\n    // Sweep pending rows first so this transaction cannot hold a running-row lock while waiting\n    // for a pending-row lock held by reconciliation or a claim of an orphan pending row. The\n    // advisory lock acquired in the candidate scan also makes enqueue retries wait until this\n    // transaction has either reaped the execution or released its lock without selecting it.\n    await tx\n      .delete(pendingJobExecutions)\n      .where(inArray(pendingJobExecutions.jobExecutionId, staleJobExecutionIds));\n\n    const deleted = await tx\n      .delete(runningJobExecutions)\n      .where(and(inArray(runningJobExecutions.id, staleIds), stalePredicate))\n      .returning({\n        workflowRunId: runningJobExecutions.workflowRunId,\n        workflowRunAttemptId: runningJobExecutions.workflowRunAttemptId,\n        jobId: runningJobExecutions.jobId,\n        jobExecutionId: runningJobExecutions.jobExecutionId,\n      });\n\n    if (deleted.length === 0) return [];\n\n    await writeOutboxEvents<RunnersEventMap>(\n      tx,\n      runnersOutbox,\n      deleted.map((row) => ({\n        type: RUNNER_JOB_LEASE_EXPIRED,\n        payload: {\n          workflowRunId: row.workflowRunId,\n          workflowRunAttemptId: row.workflowRunAttemptId,\n          jobId: row.jobId,\n          jobExecutionId: row.jobExecutionId,\n        },\n      })),\n    );\n\n    return deleted;\n  });\n\n  if (reaped.length > 0) jobExecutionLeaseExpiredCount.add(reaped.length);\n\n  return reaped;\n}\n\nexport async function getJobExecutionQueueDepth(): Promise<{\n  pendingJobExecutions: number;\n  runningJobExecutions: number;\n}> {\n  const [pending] = await db().select({value: count()}).from(pendingJobExecutions);\n  const [running] = await db().select({value: count()}).from(runningJobExecutions);\n  return {\n    pendingJobExecutions: pending?.value ?? 0,\n    runningJobExecutions: running?.value ?? 0,\n  };\n}\n\nexport async function listActiveRunningJobExecutions(params: {\n  workspaceId: string;\n  windowSeconds: number;\n  limit?: number;\n}): Promise<ActiveRunningJobExecution[]> {\n  return await db()\n    .select({\n      jobId: runningJobExecutions.jobId,\n      jobExecutionId: runningJobExecutions.jobExecutionId,\n      workflowRunId: runningJobExecutions.workflowRunId,\n      workflowRunAttemptId: runningJobExecutions.workflowRunAttemptId,\n      projectId: runningJobExecutions.projectId,\n      runnerSessionId: runningJobExecutions.runnerSessionId,\n      provisionerId: runningJobExecutions.provisionerId,\n      providerRunnerId: runningJobExecutions.providerRunnerId,\n      requiredLabels: runningJobExecutions.requiredLabels,\n      runnerLabels: runningJobExecutions.runnerLabels,\n      startedAt: runningJobExecutions.startedAt,\n      lastHeartbeatAt: runningJobExecutions.lastHeartbeatAt,\n    })\n    .from(runningJobExecutions)\n    .where(\n      and(\n        eq(runningJobExecutions.workspaceId, params.workspaceId),\n        sql`${runningJobExecutions.lastHeartbeatAt} > now() - (${params.windowSeconds} || ' seconds')::interval`,\n      ),\n    )\n    .orderBy(desc(runningJobExecutions.lastHeartbeatAt), desc(runningJobExecutions.id))\n    .limit(params.limit ?? 1000);\n}\n\nexport async function listRunningJobExecutionsByRunnerInstanceTx(\n  tx: Tx,\n  params: {\n    workspaceId: string;\n    provisionerId: string;\n    providerRunnerIds: string[];\n  },\n): Promise<RunnerInstanceBoundJobExecution[]> {\n  if (params.providerRunnerIds.length === 0) return [];\n\n  const duplicateRows = await tx\n    .select({\n      providerRunnerId: runningJobExecutions.providerRunnerId,\n      count: count(),\n    })\n    .from(runningJobExecutions)\n    .where(\n      and(\n        eq(runningJobExecutions.workspaceId, params.workspaceId),\n        eq(runningJobExecutions.provisionerId, params.provisionerId),\n        inArray(runningJobExecutions.providerRunnerId, params.providerRunnerIds),\n      ),\n    )\n    .groupBy(runningJobExecutions.providerRunnerId)\n    .having(sql`count(*) > 1`);\n\n  const duplicateRunnerInstanceIds = duplicateRows.flatMap((row) =>\n    row.providerRunnerId ? [row.providerRunnerId] : [],\n  );\n  if (duplicateRunnerInstanceIds.length > 0) {\n    logger().warn(\n      {\n        workspaceId: params.workspaceId,\n        provisionerId: params.provisionerId,\n        providerRunnerIds: duplicateRunnerInstanceIds,\n      },\n      'multiple running job executions are bound to the same provisioned runner',\n    );\n  }\n\n  const result = await tx.execute<{\n    workflowRunId: string;\n    workflowRunAttemptId: string;\n    jobId: string;\n    jobExecutionId: string;\n    providerRunnerId: string;\n    startedAt: Date | string;\n    lastHeartbeatAt: Date | string;\n    cancellationRequestedAt: Date | string | null;\n  }>(sql`\n    SELECT DISTINCT ON (${runningJobExecutions.providerRunnerId})\n      ${runningJobExecutions.workflowRunId} AS \"workflowRunId\",\n      ${runningJobExecutions.workflowRunAttemptId} AS \"workflowRunAttemptId\",\n      ${runningJobExecutions.jobId} AS \"jobId\",\n      ${runningJobExecutions.jobExecutionId} AS \"jobExecutionId\",\n      ${runningJobExecutions.providerRunnerId} AS \"providerRunnerId\",\n      ${runningJobExecutions.startedAt} AS \"startedAt\",\n      ${runningJobExecutions.lastHeartbeatAt} AS \"lastHeartbeatAt\",\n      ${runningJobExecutions.cancellationRequestedAt} AS \"cancellationRequestedAt\"\n    FROM ${runningJobExecutions}\n    WHERE\n      ${runningJobExecutions.workspaceId} = ${params.workspaceId}\n      AND ${runningJobExecutions.provisionerId} = ${params.provisionerId}\n      AND ${runningJobExecutions.providerRunnerId} IN (${sql.join(\n        params.providerRunnerIds.map((providerRunnerId) => sql`${providerRunnerId}`),\n        sql`, `,\n      )})\n    ORDER BY ${runningJobExecutions.providerRunnerId}, ${runningJobExecutions.startedAt} DESC, ${runningJobExecutions.jobExecutionId} DESC\n  `);\n\n  return result.rows.map((row) => ({\n    workflowRunId: row.workflowRunId,\n    workflowRunAttemptId: row.workflowRunAttemptId,\n    jobId: row.jobId,\n    jobExecutionId: row.jobExecutionId,\n    providerRunnerId: row.providerRunnerId,\n    startedAt: toDate(row.startedAt),\n    lastHeartbeatAt: toDate(row.lastHeartbeatAt),\n    cancellationRequestedAt: row.cancellationRequestedAt\n      ? toDate(row.cancellationRequestedAt)\n      : null,\n  }));\n}\n\nfunction toDate(value: Date | string): Date {\n  return value instanceof Date ? value : new Date(value);\n}\n\nexport async function isJobLeaseActive(params: {\n  jobId?: string;\n  jobExecutionId: string;\n  runnerSessionId: string;\n}): Promise<boolean> {\n  const [row] = await db()\n    .select({id: runningJobExecutions.id})\n    .from(runningJobExecutions)\n    .where(\n      and(\n        eq(runningJobExecutions.jobExecutionId, params.jobExecutionId),\n        eq(runningJobExecutions.runnerSessionId, params.runnerSessionId),\n        params.jobId === undefined ? undefined : eq(runningJobExecutions.jobId, params.jobId),\n      ),\n    )\n    .limit(1);\n\n  return row !== undefined;\n}\n\nexport async function recordHeartbeat(params: {\n  jobExecutionId: string;\n  runnerSessionId: string;\n  toolCapabilities?: RunnerToolCapabilitiesDto | null;\n}): Promise<{\n  cancellationRequested: boolean;\n  previousToolCapabilities: RunnerToolCapabilitiesDto | null;\n  currentToolCapabilities: RunnerToolCapabilitiesDto | null;\n  runningJobExecution: {\n    workflowRunId: string;\n    workflowRunAttemptId: string;\n    jobId: string;\n    jobExecutionId: string;\n    projectId: string;\n    workspaceId: string;\n    runnerSessionId: string;\n  };\n}> {\n  const result = await db().transaction(async (tx) => {\n    const updated = await tx\n      .update(runningJobExecutions)\n      .set({\n        firstHeartbeatAt: sql`COALESCE(${runningJobExecutions.firstHeartbeatAt}, now())`,\n        lastHeartbeatAt: sql`now()`,\n      })\n      .where(\n        and(\n          eq(runningJobExecutions.jobExecutionId, params.jobExecutionId),\n          eq(runningJobExecutions.runnerSessionId, params.runnerSessionId),\n        ),\n      )\n      .returning({\n        cancellationRequestedAt: runningJobExecutions.cancellationRequestedAt,\n        workflowRunId: runningJobExecutions.workflowRunId,\n        workflowRunAttemptId: runningJobExecutions.workflowRunAttemptId,\n        jobId: runningJobExecutions.jobId,\n        jobExecutionId: runningJobExecutions.jobExecutionId,\n        projectId: runningJobExecutions.projectId,\n        workspaceId: runningJobExecutions.workspaceId,\n        runnerSessionId: runningJobExecutions.runnerSessionId,\n      });\n\n    const row = updated[0];\n    if (!row) throw new RunningJobExecutionNotFoundError(params.jobExecutionId);\n\n    const [previous] = await tx\n      .select({toolCapabilities: runnerSessions.toolCapabilities})\n      .from(runnerSessions)\n      .where(eq(runnerSessions.id, params.runnerSessionId))\n      .limit(1)\n      .for('update');\n\n    const [session] = await tx\n      .update(runnerSessions)\n      .set({\n        toolCapabilities: params.toolCapabilities ?? null,\n        toolCapabilitiesReportedAt: params.toolCapabilities ? sql`now()` : null,\n        updatedAt: sql`now()`,\n      })\n      .where(eq(runnerSessions.id, params.runnerSessionId))\n      .returning({toolCapabilities: runnerSessions.toolCapabilities});\n\n    return {\n      row,\n      previousToolCapabilities: previous?.toolCapabilities ?? null,\n      currentToolCapabilities: session?.toolCapabilities ?? null,\n    };\n  });\n\n  const row = result.row;\n  if (!row) throw new RunningJobExecutionNotFoundError(params.jobExecutionId);\n  return {\n    cancellationRequested: row.cancellationRequestedAt !== null,\n    previousToolCapabilities: result.previousToolCapabilities,\n    currentToolCapabilities: result.currentToolCapabilities,\n    runningJobExecution: {\n      workflowRunId: row.workflowRunId,\n      workflowRunAttemptId: row.workflowRunAttemptId,\n      jobId: row.jobId,\n      jobExecutionId: row.jobExecutionId,\n      projectId: row.projectId,\n      workspaceId: row.workspaceId,\n      runnerSessionId: row.runnerSessionId,\n    },\n  };\n}\n\n/**\n * Reconciles a terminal job execution with runner state in one transaction: removes its pending\n * queue row and requests cancellation on its running lease, if either exists. The operation is\n * idempotent and preserves the first cancellation-request timestamp.\n */\nexport async function reconcileTerminalJobExecution(params: {\n  jobExecutionId: string;\n}): Promise<void> {\n  await db().transaction(async (tx) => {\n    await lockJobExecution(tx, params.jobExecutionId);\n\n    // Delete pending before updating running to match claim/release lock order. Claim locks\n    // pending rows with SKIP LOCKED before inserting the running lease, so this ordering makes\n    // a concurrent terminal reconciliation either win before claim or cancel the new lease.\n    await tx\n      .delete(pendingJobExecutions)\n      .where(eq(pendingJobExecutions.jobExecutionId, params.jobExecutionId));\n    await tx\n      .update(runningJobExecutions)\n      .set({\n        cancellationRequestedAt: sql`COALESCE(${runningJobExecutions.cancellationRequestedAt}, now())`,\n      })\n      .where(eq(runningJobExecutions.jobExecutionId, params.jobExecutionId));\n  });\n}\n\nexport async function cancelRunnerJobs(params: {jobIds: string[]}): Promise<void> {\n  if (params.jobIds.length === 0) return;\n\n  await db().transaction(async (tx) => {\n    await tx.delete(pendingJobExecutions).where(inArray(pendingJobExecutions.jobId, params.jobIds));\n    await tx\n      .update(runningJobExecutions)\n      .set({\n        cancellationRequestedAt: sql`COALESCE(${runningJobExecutions.cancellationRequestedAt}, now())`,\n      })\n      .where(inArray(runningJobExecutions.jobId, params.jobIds));\n  });\n}\n"],"names":["RUNNER_JOB_CLAIMED","RUNNER_JOB_LEASE_EXPIRED","RUNNER_JOB_QUEUED","logger","writeOutboxEvent","writeOutboxEvents","canonicalizeLabels","and","arrayContained","asc","count","desc","eq","gt","inArray","isNotNull","isNull","lt","lte","or","sql","EmptyRequiredLabelsError","RunnerSessionExhaustedError","RunningJobExecutionNotFoundError","jobExecutionEnqueuedCount","jobExecutionLeaseExpiredCount","db","runnersOutbox","pendingJobExecutions","providerRunners","runnerSessions","runningJobExecutions","runnerJobExecutionLockPrefix","lockJobExecution","tx","jobExecutionId","execute","getWorkspaceJobCounts","params","pendingRows","runningRows","Promise","all","select","workspaceId","from","where","workspaceIds","groupBy","queuedByWorkspace","Map","map","row","Number","runningByWorkspace","queued","get","running","enqueueJobExecution","requiredLabels","length","enqueued","transaction","leaseExpired","id","eventType","payload","limit","inserted","insert","values","workflowRunId","workflowRunAttemptId","jobId","projectId","onConflictDoNothing","target","returning","createdAt","type","queuedAt","toISOString","add","claimPendingJobExecution","touchRunnerSessionLiveness","runnerSessionId","throttleSeconds","runnerSessionLivenessThrottleSeconds","sessionLabels","provisionerId","providerRunnerId","runnerInstanceId","maxClaims","session","claimsUsed","revokedAt","for","candidate","orderBy","skipLocked","as","delete","runnerLabels","claimedAt","startedAt","claimed","runnerInstanceCondition","update","set","firstClaimedAt","updatedAt","releaseJobExecution","expireStuckJobExecutions","reaped","heartbeatCutoff","thresholdSeconds","firstHeartbeatCutoff","noFirstHeartbeatGraceSeconds","stalePredicate","firstHeartbeatAt","lastHeartbeatAt","staleRows","staleIds","staleJobExecutionIds","deleted","getJobExecutionQueueDepth","pending","value","listActiveRunningJobExecutions","windowSeconds","listRunningJobExecutionsByRunnerInstanceTx","providerRunnerIds","duplicateRows","having","duplicateRunnerInstanceIds","flatMap","warn","result","cancellationRequestedAt","join","rows","toDate","Date","isJobLeaseActive","undefined","recordHeartbeat","updated","previous","toolCapabilities","toolCapabilitiesReportedAt","previousToolCapabilities","currentToolCapabilities","cancellationRequested","runningJobExecution","reconcileTerminalJobExecution","cancelRunnerJobs","jobIds"],"mappings":"AAAA,SACEA,kBAAkB,EAClBC,wBAAwB,EACxBC,iBAAiB,QAGZ,2BAA2B;AAClC,SAAQC,MAAM,QAAO,8BAA8B;AACnD,SAAQC,gBAAgB,EAAEC,iBAAiB,QAAO,uBAAuB;AACzE,SAAQC,kBAAkB,QAAO,yBAAyB;AAC1D,SACEC,GAAG,EACHC,cAAc,EACdC,GAAG,EACHC,KAAK,EACLC,IAAI,EACJC,EAAE,EACFC,EAAE,EACFC,OAAO,EACPC,SAAS,EACTC,MAAM,EACNC,EAAE,EACFC,GAAG,EACHC,EAAE,EACFC,GAAG,QACE,cAAc;AACrB,SACEC,wBAAwB,EACxBC,2BAA2B,EAC3BC,gCAAgC,QAC3B,kBAAkB;AACzB,SAAQC,yBAAyB,EAAEC,6BAA6B,QAAO,uBAAuB;AAE9F,SAAQC,EAAE,QAAO,UAAU;AAC3B,SAAQC,aAAa,QAAO,qBAAqB;AACjD,SAAQC,oBAAoB,QAAO,qCAAqC;AACxE,SAAQC,eAAe,QAAO,+BAA+B;AAC7D,SAAQC,cAAc,QAAO,8BAA8B;AAC3D,SAAQC,oBAAoB,QAAO,qCAAqC;AAExE,MAAMC,+BAA+B;AAErC,eAAeC,iBAAiBC,EAAM,EAAEC,cAAsB;IAC5D,MAAMD,GAAGE,OAAO,CACdhB,GAAG,CAAC,sCAAsC,EAAE,GAAGY,+BAA+BG,gBAAgB,CAAC,EAAE,CAAC;AAEtG;AAYA,OAAO,eAAeE,sBAAsBC,MAE3C;IACC,MAAM,CAACC,aAAaC,YAAY,GAAG,MAAMC,QAAQC,GAAG,CAAC;QACnDhB,KACGiB,MAAM,CAAC;YAACC,aAAahB,qBAAqBgB,WAAW;YAAElC,OAAOA;QAAO,GACrEmC,IAAI,CAACjB,sBACLkB,KAAK,CAAChC,QAAQc,qBAAqBgB,WAAW,EAAEN,OAAOS,YAAY,GACnEC,OAAO,CAACpB,qBAAqBgB,WAAW;QAC3ClB,KACGiB,MAAM,CAAC;YAACC,aAAab,qBAAqBa,WAAW;YAAElC,OAAOA;QAAO,GACrEmC,IAAI,CAACd,sBACLe,KAAK,CAAChC,QAAQiB,qBAAqBa,WAAW,EAAEN,OAAOS,YAAY,GACnEC,OAAO,CAACjB,qBAAqBa,WAAW;KAC5C;IAED,MAAMK,oBAAoB,IAAIC,IAAIX,YAAYY,GAAG,CAAC,CAACC,MAAQ;YAACA,IAAIR,WAAW;YAAES,OAAOD,IAAI1C,KAAK;SAAE;IAC/F,MAAM4C,qBAAqB,IAAIJ,IAC7BV,YAAYW,GAAG,CAAC,CAACC,MAAQ;YAACA,IAAIR,WAAW;YAAES,OAAOD,IAAI1C,KAAK;SAAE;IAG/D,OAAO4B,OAAOS,YAAY,CAACI,GAAG,CAAC,CAACP,cAAiB,CAAA;YAC/CA;YACAW,QAAQN,kBAAkBO,GAAG,CAACZ,gBAAgB;YAC9Ca,SAASH,mBAAmBE,GAAG,CAACZ,gBAAgB;QAClD,CAAA;AACF;AAEA,6FAA6F;AAC7F,+FAA+F;AAC/F,kGAAkG;AAClG,+FAA+F;AAC/F,6FAA6F;AAC7F,OAAO,eAAec,oBAAoBpB,MAAiC;IACzE,MAAMqB,iBAAiB;WAAIrD,mBAAmBgC,OAAOqB,cAAc;KAAE;IACrE,IAAIA,eAAeC,MAAM,KAAK,GAAG,MAAM,IAAIvC;IAE3C,MAAMwC,WAAW,MAAMnC,KAAKoC,WAAW,CAAC,OAAO5B;QAC7C,MAAMD,iBAAiBC,IAAII,OAAOH,cAAc;QAEhD,MAAM,CAAC4B,aAAa,GAAG,MAAM7B,GAC1BS,MAAM,CAAC;YAACqB,IAAIrC,cAAcqC,EAAE;QAAA,GAC5BnB,IAAI,CAAClB,eACLmB,KAAK,CACJvC,IACEK,GAAGe,cAAcsC,SAAS,EAAEhE,2BAC5BmB,GAAG,CAAC,EAAEO,cAAcuC,OAAO,CAAC,sBAAsB,EAAE5B,OAAOH,cAAc,CAAC,CAAC,GAG9EgC,KAAK,CAAC;QACT,IAAIJ,cAAc,OAAO;QAEzB,MAAM,CAACK,SAAS,GAAG,MAAMlC,GACtBmC,MAAM,CAACzC,sBACP0C,MAAM,CAAC;YACN1B,aAAaN,OAAOM,WAAW;YAC/B2B,eAAejC,OAAOiC,aAAa;YACnCC,sBAAsBlC,OAAOkC,oBAAoB;YACjDC,OAAOnC,OAAOmC,KAAK;YACnBtC,gBAAgBG,OAAOH,cAAc;YACrCuC,WAAWpC,OAAOoC,SAAS;YAC3Bf;QACF,GACCgB,mBAAmB,CAAC;YAACC,QAAQhD,qBAAqBO,cAAc;QAAA,GAChE0C,SAAS,CAAC;YAACC,WAAWlD,qBAAqBkD,SAAS;QAAA;QAEvD,4EAA4E;QAC5E,0EAA0E;QAC1E,mEAAmE;QACnE,IAAI,CAACV,UAAU,OAAO;QAEtB,MAAMhE,iBAAkC8B,IAAIP,eAAe;YACzDoD,MAAM7E;YACNgE,SAAS;gBACPK,eAAejC,OAAOiC,aAAa;gBACnCC,sBAAsBlC,OAAOkC,oBAAoB;gBACjDC,OAAOnC,OAAOmC,KAAK;gBACnBtC,gBAAgBG,OAAOH,cAAc;gBACrC6C,UAAUZ,SAASU,SAAS,CAACG,WAAW;YAC1C;QACF;QACA,OAAO;IACT;IAEA,IAAIpB,UAAUrC,0BAA0B0D,GAAG,CAAC;AAC9C;AAoCA,OAAO,eAAeC,yBAAyB7C,MAM9C;IACC,MAAM8C,2BAA2B;QAC/BxC,aAAaN,OAAOM,WAAW;QAC/ByC,iBAAiB/C,OAAO+C,eAAe;QACvCC,iBAAiBhD,OAAOiD,oCAAoC;IAC9D;IAEA,IAAIjD,OAAOkD,aAAa,CAAC5B,MAAM,KAAK,GAAG,OAAO;IAE9C,OAAO,MAAMlC,KAAKoC,WAAW,CAAC,OAAO5B;QACnC,IAAIuD,gBAA+B;QACnC,IAAIC,mBAAkC;QACtC,IAAIC,mBAAkC;QAEtC,IAAIrD,OAAOsD,SAAS,KAAK,MAAM;YAC7B,MAAM,CAACC,QAAQ,GAAG,MAAM3D,GACrBS,MAAM,CAAC;gBACNiD,WAAW9D,eAAe8D,SAAS;gBACnCE,YAAYhE,eAAegE,UAAU;gBACrCC,WAAWjE,eAAeiE,SAAS;gBACnCJ,kBAAkB7D,eAAe6D,gBAAgB;gBACjDF,eAAe3D,eAAe2D,aAAa;gBAC3CC,kBAAkB5D,eAAe4D,gBAAgB;YACnD,GACC7C,IAAI,CAACf,gBACLgB,KAAK,CAAClC,GAAGkB,eAAekC,EAAE,EAAE1B,OAAO+C,eAAe,GAClDlB,KAAK,CAAC,GACN6B,GAAG,CAAC;YAEP,IACE,CAACH,WACDA,QAAQE,SAAS,IACjBF,QAAQD,SAAS,KAAK,QACtBC,QAAQC,UAAU,IAAID,QAAQD,SAAS,EACvC;gBACA,MAAM,IAAItE,4BAA4BgB,OAAO+C,eAAe;YAC9D;YAEA,0EAA0E;YAC1E,mDAAmD;YACnDM,mBAAmBE,QAAQF,gBAAgB;YAC3CF,gBAAgBI,QAAQJ,aAAa;YACrCC,mBAAmBG,QAAQH,gBAAgB;QAC7C;QAEA,4EAA4E;QAC5E,oFAAoF;QACpF,oFAAoF;QACpF,sFAAsF;QACtF,MAAMO,YAAY/D,GACfS,MAAM,GACNE,IAAI,CAACjB,sBACLkB,KAAK,CACJvC,IACEK,GAAGgB,qBAAqBgB,WAAW,EAAEN,OAAOM,WAAW,GACvDpC,eAAeoB,qBAAqB+B,cAAc,EAAErB,OAAOkD,aAAa,IAG3EU,OAAO,CAACzF,IAAImB,qBAAqBkD,SAAS,GAAGrE,IAAImB,qBAAqBoC,EAAE,GACxEG,KAAK,CAAC,GACN6B,GAAG,CAAC,UAAU;YAACG,YAAY;QAAI,GAC/BC,EAAE,CAAC;QAEN,MAAM,CAAChD,IAAI,GAAG,MAAMlB,GACjBS,MAAM,GACNE,IAAI,CAACoD,WACLnD,KAAK,CACJ1B,GAAG,CAAC;;qBAES,EAAEY,6BAA6B,IAAI,EAAEiE,UAAU9D,cAAc,CAAC;;QAE3E,CAAC;QAGL,IAAI,CAACiB,KAAK,OAAO;QAEjB,MAAMlB,GAAGmE,MAAM,CAACzE,sBAAsBkB,KAAK,CAAClC,GAAGgB,qBAAqBoC,EAAE,EAAEZ,IAAIY,EAAE;QAE9E,8EAA8E;QAC9E,gFAAgF;QAChF,qFAAqF;QACrF,2EAA2E;QAC3E,8EAA8E;QAC9E,2DAA2D;QAC3D,MAAMI,WAAW,MAAMlC,GACpBmC,MAAM,CAACtC,sBACPuC,MAAM,CAAC;YACN1B,aAAaQ,IAAIR,WAAW;YAC5B2B,eAAenB,IAAImB,aAAa;YAChCC,sBAAsBpB,IAAIoB,oBAAoB;YAC9CC,OAAOrB,IAAIqB,KAAK;YAChBtC,gBAAgBiB,IAAIjB,cAAc;YAClCuC,WAAWtB,IAAIsB,SAAS;YACxBW,iBAAiB/C,OAAO+C,eAAe;YACvCI;YACAC;YACA/B,gBAAgBP,IAAIO,cAAc;YAClC2C,cAAchE,OAAOkD,aAAa;QACpC,GACCb,mBAAmB,CAAC;YAACC,QAAQ7C,qBAAqBI,cAAc;QAAA,GAChE0C,SAAS,CAAC;YAAC0B,WAAWxE,qBAAqByE,SAAS;QAAA;QAEvD,MAAMC,UAAUrC,QAAQ,CAAC,EAAE;QAC3B,IAAI,CAACqC,SAAS,OAAO;QAErB,MAAMC,0BAA0Bf,mBAC5B/E,GAAGiB,gBAAgBmC,EAAE,EAAE2B,oBACvBF,iBAAiBC,mBACfnF,IACEK,GAAGiB,gBAAgB4D,aAAa,EAAEA,gBAClC7E,GAAGiB,gBAAgB6D,gBAAgB,EAAEA,qBAEvC;QACN,IAAIgB,yBAAyB;YAC3B,MAAMxE,GACHyE,MAAM,CAAC9E,iBACP+E,GAAG,CAAC;gBACHC,gBAAgBzF,GAAG,CAAC,SAAS,EAAES,gBAAgBgF,cAAc,CAAC,EAAE,EAAEJ,QAAQF,SAAS,CAAC,CAAC,CAAC;gBACtFO,WAAW1F,GAAG,CAAC,KAAK,CAAC;YACvB,GACC0B,KAAK,CAAC4D;QACX;QAEA,IAAIpE,OAAOsD,SAAS,KAAK,MAAM;YAC7B,MAAM1D,GACHyE,MAAM,CAAC7E,gBACP8E,GAAG,CAAC;gBAACd,YAAY1E,GAAG,CAAC,EAAEU,eAAegE,UAAU,CAAC,IAAI,CAAC;gBAAEgB,WAAW1F,GAAG,CAAC,KAAK,CAAC;YAAA,GAC7E0B,KAAK,CAAClC,GAAGkB,eAAekC,EAAE,EAAE1B,OAAO+C,eAAe;QACvD;QAEA,4FAA4F;QAC5F,mFAAmF;QACnF,6BAA6B;QAC7B,MAAMjF,iBAAkC8B,IAAIP,eAAe;YACzDoD,MAAM/E;YACNkE,SAAS;gBACPK,eAAenB,IAAImB,aAAa;gBAChCC,sBAAsBpB,IAAIoB,oBAAoB;gBAC9CC,OAAOrB,IAAIqB,KAAK;gBAChBtC,gBAAgBiB,IAAIjB,cAAc;gBAClCoE,WAAWE,QAAQF,SAAS,CAACtB,WAAW;YAC1C;QACF;QAEA,OAAO;YACLV,eAAenB,IAAImB,aAAa;YAChCC,sBAAsBpB,IAAIoB,oBAAoB;YAC9CC,OAAOrB,IAAIqB,KAAK;YAChBtC,gBAAgBiB,IAAIjB,cAAc;YAClCuC,WAAWtB,IAAIsB,SAAS;QAC1B;IACF;AACF;AAEA,eAAeU,2BAA2B9C,MAIzC;IACC,MAAMZ,KACHiF,MAAM,CAAC7E,gBACP8E,GAAG,CAAC;QAACE,WAAW1F,GAAG,CAAC,KAAK,CAAC;IAAA,GAC1B0B,KAAK,CACJvC,IACEK,GAAGkB,eAAekC,EAAE,EAAE1B,OAAO+C,eAAe,GAC5CzE,GAAGkB,eAAec,WAAW,EAAEN,OAAOM,WAAW,GACjDxB,GAAG,CAAC,EAAEU,eAAegF,SAAS,CAAC,YAAY,EAAExE,OAAOgD,eAAe,CAAC,yBAAyB,CAAC;AAGtG;AAEA;;;;;;;CAOC,GACD,OAAO,eAAeyB,oBAAoBzE,MAAgC;IACxE,MAAMZ,KAAKoC,WAAW,CAAC,OAAO5B;QAC5B,MAAMD,iBAAiBC,IAAII,OAAOH,cAAc;QAEhD,uFAAuF;QACvF,6EAA6E;QAC7E,qFAAqF;QACrF,2CAA2C;QAC3C,MAAMD,GACHmE,MAAM,CAACzE,sBACPkB,KAAK,CAAClC,GAAGgB,qBAAqBO,cAAc,EAAEG,OAAOH,cAAc;QACtE,MAAMD,GACHmE,MAAM,CAACtE,sBACPe,KAAK,CAAClC,GAAGmB,qBAAqBI,cAAc,EAAEG,OAAOH,cAAc;IACxE;AACF;AAEA;;;;;;;;;;;;CAYC,GACD,OAAO,eAAe6E,yBAAyB1E,MAI9C;IAQC,MAAM2E,SAAS,MAAMvF,KAAKoC,WAAW,CAAC,OAAO5B;QAC3C,MAAMgF,kBAAkB9F,GAAG,CAAC,SAAS,EAAEkB,OAAO6E,gBAAgB,CAAC,yBAAyB,CAAC;QACzF,MAAMC,uBAAuBhG,GAAG,CAAC,SAAS,EAAEkB,OAAO+E,4BAA4B,CAAC,yBAAyB,CAAC;QAC1G,MAAMC,iBAAiBnG,GACrBZ,IACES,OAAOe,qBAAqBwF,gBAAgB,GAC5CrG,IAAIa,qBAAqByF,eAAe,EAAEzF,qBAAqByE,SAAS,GACxEvF,GAAGc,qBAAqByE,SAAS,EAAEY,wBAErC7G,IACEY,GACEJ,UAAUgB,qBAAqBwF,gBAAgB,GAC/C1G,GAAGkB,qBAAqByF,eAAe,EAAEzF,qBAAqByE,SAAS,IAEzEvF,GAAGc,qBAAqByF,eAAe,EAAEN;QAI7C,MAAMO,YAAY,MAAMvF,GACrBS,MAAM,CAAC;YACNqB,IAAIjC,qBAAqBiC,EAAE;YAC3B7B,gBAAgBJ,qBAAqBI,cAAc;QACrD,GACCU,IAAI,CAACd,sBACLe,KAAK,CACJvC,IACE+G,gBACAlG,GAAG,CAAC;;uBAES,EAAEY,6BAA6B,IAAI,EAAED,qBAAqBI,cAAc,CAAC;;UAEtF,CAAC,GAGJ+D,OAAO,CACNzF,IACEW,GAAG,CAAC,UAAU,EAAEW,qBAAqBwF,gBAAgB,CAAC,aAAa,EAAExF,qBAAqByF,eAAe,CAAC,IAAI,EAAEzF,qBAAqByE,SAAS,CAAC,MAAM,EAAEzE,qBAAqByE,SAAS,CAAC,MAAM,EAAEzE,qBAAqByF,eAAe,CAAC,IAAI,CAAC,GAE1O/G,IAAIsB,qBAAqBiC,EAAE,GAE5BG,KAAK,CAAC7B,OAAO6B,KAAK,IAAI;QAEzB,IAAIsD,UAAU7D,MAAM,KAAK,GAAG,OAAO,EAAE;QAErC,MAAM8D,WAAWD,UAAUtE,GAAG,CAAC,CAACC,MAAQA,IAAIY,EAAE;QAC9C,MAAM2D,uBAAuBF,UAAUtE,GAAG,CAAC,CAACC,MAAQA,IAAIjB,cAAc;QAEtE,4FAA4F;QAC5F,yFAAyF;QACzF,0FAA0F;QAC1F,yFAAyF;QACzF,MAAMD,GACHmE,MAAM,CAACzE,sBACPkB,KAAK,CAAChC,QAAQc,qBAAqBO,cAAc,EAAEwF;QAEtD,MAAMC,UAAU,MAAM1F,GACnBmE,MAAM,CAACtE,sBACPe,KAAK,CAACvC,IAAIO,QAAQiB,qBAAqBiC,EAAE,EAAE0D,WAAWJ,iBACtDzC,SAAS,CAAC;YACTN,eAAexC,qBAAqBwC,aAAa;YACjDC,sBAAsBzC,qBAAqByC,oBAAoB;YAC/DC,OAAO1C,qBAAqB0C,KAAK;YACjCtC,gBAAgBJ,qBAAqBI,cAAc;QACrD;QAEF,IAAIyF,QAAQhE,MAAM,KAAK,GAAG,OAAO,EAAE;QAEnC,MAAMvD,kBACJ6B,IACAP,eACAiG,QAAQzE,GAAG,CAAC,CAACC,MAAS,CAAA;gBACpB2B,MAAM9E;gBACNiE,SAAS;oBACPK,eAAenB,IAAImB,aAAa;oBAChCC,sBAAsBpB,IAAIoB,oBAAoB;oBAC9CC,OAAOrB,IAAIqB,KAAK;oBAChBtC,gBAAgBiB,IAAIjB,cAAc;gBACpC;YACF,CAAA;QAGF,OAAOyF;IACT;IAEA,IAAIX,OAAOrD,MAAM,GAAG,GAAGnC,8BAA8ByD,GAAG,CAAC+B,OAAOrD,MAAM;IAEtE,OAAOqD;AACT;AAEA,OAAO,eAAeY;IAIpB,MAAM,CAACC,QAAQ,GAAG,MAAMpG,KAAKiB,MAAM,CAAC;QAACoF,OAAOrH;IAAO,GAAGmC,IAAI,CAACjB;IAC3D,MAAM,CAAC6B,QAAQ,GAAG,MAAM/B,KAAKiB,MAAM,CAAC;QAACoF,OAAOrH;IAAO,GAAGmC,IAAI,CAACd;IAC3D,OAAO;QACLH,sBAAsBkG,SAASC,SAAS;QACxChG,sBAAsB0B,SAASsE,SAAS;IAC1C;AACF;AAEA,OAAO,eAAeC,+BAA+B1F,MAIpD;IACC,OAAO,MAAMZ,KACViB,MAAM,CAAC;QACN8B,OAAO1C,qBAAqB0C,KAAK;QACjCtC,gBAAgBJ,qBAAqBI,cAAc;QACnDoC,eAAexC,qBAAqBwC,aAAa;QACjDC,sBAAsBzC,qBAAqByC,oBAAoB;QAC/DE,WAAW3C,qBAAqB2C,SAAS;QACzCW,iBAAiBtD,qBAAqBsD,eAAe;QACrDI,eAAe1D,qBAAqB0D,aAAa;QACjDC,kBAAkB3D,qBAAqB2D,gBAAgB;QACvD/B,gBAAgB5B,qBAAqB4B,cAAc;QACnD2C,cAAcvE,qBAAqBuE,YAAY;QAC/CE,WAAWzE,qBAAqByE,SAAS;QACzCgB,iBAAiBzF,qBAAqByF,eAAe;IACvD,GACC3E,IAAI,CAACd,sBACLe,KAAK,CACJvC,IACEK,GAAGmB,qBAAqBa,WAAW,EAAEN,OAAOM,WAAW,GACvDxB,GAAG,CAAC,EAAEW,qBAAqByF,eAAe,CAAC,YAAY,EAAElF,OAAO2F,aAAa,CAAC,yBAAyB,CAAC,GAG3G/B,OAAO,CAACvF,KAAKoB,qBAAqByF,eAAe,GAAG7G,KAAKoB,qBAAqBiC,EAAE,GAChFG,KAAK,CAAC7B,OAAO6B,KAAK,IAAI;AAC3B;AAEA,OAAO,eAAe+D,2CACpBhG,EAAM,EACNI,MAIC;IAED,IAAIA,OAAO6F,iBAAiB,CAACvE,MAAM,KAAK,GAAG,OAAO,EAAE;IAEpD,MAAMwE,gBAAgB,MAAMlG,GACzBS,MAAM,CAAC;QACN+C,kBAAkB3D,qBAAqB2D,gBAAgB;QACvDhF,OAAOA;IACT,GACCmC,IAAI,CAACd,sBACLe,KAAK,CACJvC,IACEK,GAAGmB,qBAAqBa,WAAW,EAAEN,OAAOM,WAAW,GACvDhC,GAAGmB,qBAAqB0D,aAAa,EAAEnD,OAAOmD,aAAa,GAC3D3E,QAAQiB,qBAAqB2D,gBAAgB,EAAEpD,OAAO6F,iBAAiB,IAG1EnF,OAAO,CAACjB,qBAAqB2D,gBAAgB,EAC7C2C,MAAM,CAACjH,GAAG,CAAC,YAAY,CAAC;IAE3B,MAAMkH,6BAA6BF,cAAcG,OAAO,CAAC,CAACnF,MACxDA,IAAIsC,gBAAgB,GAAG;YAACtC,IAAIsC,gBAAgB;SAAC,GAAG,EAAE;IAEpD,IAAI4C,2BAA2B1E,MAAM,GAAG,GAAG;QACzCzD,SAASqI,IAAI,CACX;YACE5F,aAAaN,OAAOM,WAAW;YAC/B6C,eAAenD,OAAOmD,aAAa;YACnC0C,mBAAmBG;QACrB,GACA;IAEJ;IAEA,MAAMG,SAAS,MAAMvG,GAAGE,OAAO,CAS5BhB,GAAG,CAAC;wBACe,EAAEW,qBAAqB2D,gBAAgB,CAAC;MAC1D,EAAE3D,qBAAqBwC,aAAa,CAAC;MACrC,EAAExC,qBAAqByC,oBAAoB,CAAC;MAC5C,EAAEzC,qBAAqB0C,KAAK,CAAC;MAC7B,EAAE1C,qBAAqBI,cAAc,CAAC;MACtC,EAAEJ,qBAAqB2D,gBAAgB,CAAC;MACxC,EAAE3D,qBAAqByE,SAAS,CAAC;MACjC,EAAEzE,qBAAqByF,eAAe,CAAC;MACvC,EAAEzF,qBAAqB2G,uBAAuB,CAAC;SAC5C,EAAE3G,qBAAqB;;MAE1B,EAAEA,qBAAqBa,WAAW,CAAC,GAAG,EAAEN,OAAOM,WAAW,CAAC;UACvD,EAAEb,qBAAqB0D,aAAa,CAAC,GAAG,EAAEnD,OAAOmD,aAAa,CAAC;UAC/D,EAAE1D,qBAAqB2D,gBAAgB,CAAC,KAAK,EAAEtE,IAAIuH,IAAI,CACzDrG,OAAO6F,iBAAiB,CAAChF,GAAG,CAAC,CAACuC,mBAAqBtE,GAAG,CAAC,EAAEsE,iBAAiB,CAAC,GAC3EtE,GAAG,CAAC,EAAE,CAAC,EACP;aACK,EAAEW,qBAAqB2D,gBAAgB,CAAC,EAAE,EAAE3D,qBAAqByE,SAAS,CAAC,OAAO,EAAEzE,qBAAqBI,cAAc,CAAC;EACnI,CAAC;IAED,OAAOsG,OAAOG,IAAI,CAACzF,GAAG,CAAC,CAACC,MAAS,CAAA;YAC/BmB,eAAenB,IAAImB,aAAa;YAChCC,sBAAsBpB,IAAIoB,oBAAoB;YAC9CC,OAAOrB,IAAIqB,KAAK;YAChBtC,gBAAgBiB,IAAIjB,cAAc;YAClCuD,kBAAkBtC,IAAIsC,gBAAgB;YACtCc,WAAWqC,OAAOzF,IAAIoD,SAAS;YAC/BgB,iBAAiBqB,OAAOzF,IAAIoE,eAAe;YAC3CkB,yBAAyBtF,IAAIsF,uBAAuB,GAChDG,OAAOzF,IAAIsF,uBAAuB,IAClC;QACN,CAAA;AACF;AAEA,SAASG,OAAOd,KAAoB;IAClC,OAAOA,iBAAiBe,OAAOf,QAAQ,IAAIe,KAAKf;AAClD;AAEA,OAAO,eAAegB,iBAAiBzG,MAItC;IACC,MAAM,CAACc,IAAI,GAAG,MAAM1B,KACjBiB,MAAM,CAAC;QAACqB,IAAIjC,qBAAqBiC,EAAE;IAAA,GACnCnB,IAAI,CAACd,sBACLe,KAAK,CACJvC,IACEK,GAAGmB,qBAAqBI,cAAc,EAAEG,OAAOH,cAAc,GAC7DvB,GAAGmB,qBAAqBsD,eAAe,EAAE/C,OAAO+C,eAAe,GAC/D/C,OAAOmC,KAAK,KAAKuE,YAAYA,YAAYpI,GAAGmB,qBAAqB0C,KAAK,EAAEnC,OAAOmC,KAAK,IAGvFN,KAAK,CAAC;IAET,OAAOf,QAAQ4F;AACjB;AAEA,OAAO,eAAeC,gBAAgB3G,MAIrC;IAcC,MAAMmG,SAAS,MAAM/G,KAAKoC,WAAW,CAAC,OAAO5B;QAC3C,MAAMgH,UAAU,MAAMhH,GACnByE,MAAM,CAAC5E,sBACP6E,GAAG,CAAC;YACHW,kBAAkBnG,GAAG,CAAC,SAAS,EAAEW,qBAAqBwF,gBAAgB,CAAC,QAAQ,CAAC;YAChFC,iBAAiBpG,GAAG,CAAC,KAAK,CAAC;QAC7B,GACC0B,KAAK,CACJvC,IACEK,GAAGmB,qBAAqBI,cAAc,EAAEG,OAAOH,cAAc,GAC7DvB,GAAGmB,qBAAqBsD,eAAe,EAAE/C,OAAO+C,eAAe,IAGlER,SAAS,CAAC;YACT6D,yBAAyB3G,qBAAqB2G,uBAAuB;YACrEnE,eAAexC,qBAAqBwC,aAAa;YACjDC,sBAAsBzC,qBAAqByC,oBAAoB;YAC/DC,OAAO1C,qBAAqB0C,KAAK;YACjCtC,gBAAgBJ,qBAAqBI,cAAc;YACnDuC,WAAW3C,qBAAqB2C,SAAS;YACzC9B,aAAab,qBAAqBa,WAAW;YAC7CyC,iBAAiBtD,qBAAqBsD,eAAe;QACvD;QAEF,MAAMjC,MAAM8F,OAAO,CAAC,EAAE;QACtB,IAAI,CAAC9F,KAAK,MAAM,IAAI7B,iCAAiCe,OAAOH,cAAc;QAE1E,MAAM,CAACgH,SAAS,GAAG,MAAMjH,GACtBS,MAAM,CAAC;YAACyG,kBAAkBtH,eAAesH,gBAAgB;QAAA,GACzDvG,IAAI,CAACf,gBACLgB,KAAK,CAAClC,GAAGkB,eAAekC,EAAE,EAAE1B,OAAO+C,eAAe,GAClDlB,KAAK,CAAC,GACN6B,GAAG,CAAC;QAEP,MAAM,CAACH,QAAQ,GAAG,MAAM3D,GACrByE,MAAM,CAAC7E,gBACP8E,GAAG,CAAC;YACHwC,kBAAkB9G,OAAO8G,gBAAgB,IAAI;YAC7CC,4BAA4B/G,OAAO8G,gBAAgB,GAAGhI,GAAG,CAAC,KAAK,CAAC,GAAG;YACnE0F,WAAW1F,GAAG,CAAC,KAAK,CAAC;QACvB,GACC0B,KAAK,CAAClC,GAAGkB,eAAekC,EAAE,EAAE1B,OAAO+C,eAAe,GAClDR,SAAS,CAAC;YAACuE,kBAAkBtH,eAAesH,gBAAgB;QAAA;QAE/D,OAAO;YACLhG;YACAkG,0BAA0BH,UAAUC,oBAAoB;YACxDG,yBAAyB1D,SAASuD,oBAAoB;QACxD;IACF;IAEA,MAAMhG,MAAMqF,OAAOrF,GAAG;IACtB,IAAI,CAACA,KAAK,MAAM,IAAI7B,iCAAiCe,OAAOH,cAAc;IAC1E,OAAO;QACLqH,uBAAuBpG,IAAIsF,uBAAuB,KAAK;QACvDY,0BAA0Bb,OAAOa,wBAAwB;QACzDC,yBAAyBd,OAAOc,uBAAuB;QACvDE,qBAAqB;YACnBlF,eAAenB,IAAImB,aAAa;YAChCC,sBAAsBpB,IAAIoB,oBAAoB;YAC9CC,OAAOrB,IAAIqB,KAAK;YAChBtC,gBAAgBiB,IAAIjB,cAAc;YAClCuC,WAAWtB,IAAIsB,SAAS;YACxB9B,aAAaQ,IAAIR,WAAW;YAC5ByC,iBAAiBjC,IAAIiC,eAAe;QACtC;IACF;AACF;AAEA;;;;CAIC,GACD,OAAO,eAAeqE,8BAA8BpH,MAEnD;IACC,MAAMZ,KAAKoC,WAAW,CAAC,OAAO5B;QAC5B,MAAMD,iBAAiBC,IAAII,OAAOH,cAAc;QAEhD,wFAAwF;QACxF,2FAA2F;QAC3F,wFAAwF;QACxF,MAAMD,GACHmE,MAAM,CAACzE,sBACPkB,KAAK,CAAClC,GAAGgB,qBAAqBO,cAAc,EAAEG,OAAOH,cAAc;QACtE,MAAMD,GACHyE,MAAM,CAAC5E,sBACP6E,GAAG,CAAC;YACH8B,yBAAyBtH,GAAG,CAAC,SAAS,EAAEW,qBAAqB2G,uBAAuB,CAAC,QAAQ,CAAC;QAChG,GACC5F,KAAK,CAAClC,GAAGmB,qBAAqBI,cAAc,EAAEG,OAAOH,cAAc;IACxE;AACF;AAEA,OAAO,eAAewH,iBAAiBrH,MAA0B;IAC/D,IAAIA,OAAOsH,MAAM,CAAChG,MAAM,KAAK,GAAG;IAEhC,MAAMlC,KAAKoC,WAAW,CAAC,OAAO5B;QAC5B,MAAMA,GAAGmE,MAAM,CAACzE,sBAAsBkB,KAAK,CAAChC,QAAQc,qBAAqB6C,KAAK,EAAEnC,OAAOsH,MAAM;QAC7F,MAAM1H,GACHyE,MAAM,CAAC5E,sBACP6E,GAAG,CAAC;YACH8B,yBAAyBtH,GAAG,CAAC,SAAS,EAAEW,qBAAqB2G,uBAAuB,CAAC,QAAQ,CAAC;QAChG,GACC5F,KAAK,CAAChC,QAAQiB,qBAAqB0C,KAAK,EAAEnC,OAAOsH,MAAM;IAC5D;AACF"}
+{"version":3,"sources":["../../src/db/job-executions.ts"],"sourcesContent":["import {\n  RUNNER_JOB_CLAIMED,\n  RUNNER_JOB_LEASE_EXPIRED,\n  RUNNER_JOB_QUEUED,\n  type RunnersEventMap,\n  type RunnerToolCapabilitiesDto,\n} from '@shipfox/api-runners-dto';\nimport {logger} from '@shipfox/node-opentelemetry';\nimport {writeOutboxEvent, writeOutboxEvents} from '@shipfox/node-outbox';\nimport {canonicalizeLabels} from '@shipfox/runner-labels';\nimport {\n  and,\n  arrayContained,\n  asc,\n  count,\n  desc,\n  eq,\n  gt,\n  inArray,\n  isNotNull,\n  isNull,\n  lt,\n  lte,\n  or,\n  sql,\n} from 'drizzle-orm';\nimport {\n  EmptyRequiredLabelsError,\n  RunnerSessionExhaustedError,\n  RunningJobExecutionNotFoundError,\n} from '#core/errors.js';\nimport {\n  type JobExecutionQueueTimeObservation,\n  jobExecutionEnqueuedCount,\n  jobExecutionLeaseExpiredCount,\n  type ProviderRunnerLifecycleObservation,\n  recordJobExecutionQueueTime,\n  recordProviderRunnerActivationToFirstClaim,\n} from '#metrics/instance.js';\nimport type {Tx} from './db.js';\nimport {db} from './db.js';\nimport {runnersOutbox} from './schema/outbox.js';\nimport {pendingJobExecutions} from './schema/pending-job-executions.js';\nimport {providerRunners} from './schema/runner-instances.js';\nimport {runnerSessions} from './schema/runner-sessions.js';\nimport {runningJobExecutions} from './schema/running-job-executions.js';\n\nconst runnerJobExecutionLockPrefix = 'runners_job_execution:';\n\nasync function lockJobExecution(tx: Tx, jobExecutionId: string): Promise<void> {\n  await tx.execute(\n    sql`select pg_advisory_xact_lock(hashtext(${`${runnerJobExecutionLockPrefix}${jobExecutionId}`}))`,\n  );\n}\n\nexport interface EnqueueJobExecutionParams {\n  workspaceId: string;\n  workflowRunId: string;\n  workflowRunAttemptId: string;\n  jobId: string;\n  jobExecutionId: string;\n  projectId: string;\n  requiredLabels: string[];\n}\n\nexport async function getWorkspaceJobCounts(params: {\n  workspaceIds: string[];\n}): Promise<Array<{workspaceId: string; queued: number; running: number}>> {\n  const [pendingRows, runningRows] = await Promise.all([\n    db()\n      .select({workspaceId: pendingJobExecutions.workspaceId, count: count()})\n      .from(pendingJobExecutions)\n      .where(inArray(pendingJobExecutions.workspaceId, params.workspaceIds))\n      .groupBy(pendingJobExecutions.workspaceId),\n    db()\n      .select({workspaceId: runningJobExecutions.workspaceId, count: count()})\n      .from(runningJobExecutions)\n      .where(inArray(runningJobExecutions.workspaceId, params.workspaceIds))\n      .groupBy(runningJobExecutions.workspaceId),\n  ]);\n\n  const queuedByWorkspace = new Map(pendingRows.map((row) => [row.workspaceId, Number(row.count)]));\n  const runningByWorkspace = new Map(\n    runningRows.map((row) => [row.workspaceId, Number(row.count)]),\n  );\n\n  return params.workspaceIds.map((workspaceId) => ({\n    workspaceId,\n    queued: queuedByWorkspace.get(workspaceId) ?? 0,\n    running: runningByWorkspace.get(workspaceId) ?? 0,\n  }));\n}\n\n// Idempotent while the job execution is still pending: a duplicate jobExecutionId already in\n// `runners_pending_jobs` is a no-op. Temporal retries the enqueue activity at-least-once, so a\n// unique-violation throw on a retry-after-lost-result would permanently fail a healthy execution.\n// The per-execution advisory lock serializes retries with lease expiry/reconciliation, and the\n// durable lease-expired event prevents a retry from re-queueing an execution already reaped.\nexport async function enqueueJobExecution(params: EnqueueJobExecutionParams): Promise<void> {\n  const requiredLabels = [...canonicalizeLabels(params.requiredLabels)];\n  if (requiredLabels.length === 0) throw new EmptyRequiredLabelsError();\n\n  const enqueued = await db().transaction(async (tx) => {\n    await lockJobExecution(tx, params.jobExecutionId);\n\n    const [leaseExpired] = await tx\n      .select({id: runnersOutbox.id})\n      .from(runnersOutbox)\n      .where(\n        and(\n          eq(runnersOutbox.eventType, RUNNER_JOB_LEASE_EXPIRED),\n          sql`${runnersOutbox.payload}->>'jobExecutionId' = ${params.jobExecutionId}`,\n        ),\n      )\n      .limit(1);\n    if (leaseExpired) return false;\n\n    const [inserted] = await tx\n      .insert(pendingJobExecutions)\n      .values({\n        workspaceId: params.workspaceId,\n        workflowRunId: params.workflowRunId,\n        workflowRunAttemptId: params.workflowRunAttemptId,\n        jobId: params.jobId,\n        jobExecutionId: params.jobExecutionId,\n        projectId: params.projectId,\n        requiredLabels,\n      })\n      .onConflictDoNothing({target: pendingJobExecutions.jobExecutionId})\n      .returning({createdAt: pendingJobExecutions.createdAt});\n\n    // A retry that hits the conflict inserts nothing: the first enqueue already\n    // emitted the queued event (durably, in the outbox), so re-emitting would\n    // only add a redundant row the subscriber coalesces away. Skip it.\n    if (!inserted) return false;\n\n    await writeOutboxEvent<RunnersEventMap>(tx, runnersOutbox, {\n      type: RUNNER_JOB_QUEUED,\n      payload: {\n        workflowRunId: params.workflowRunId,\n        workflowRunAttemptId: params.workflowRunAttemptId,\n        jobId: params.jobId,\n        jobExecutionId: params.jobExecutionId,\n        queuedAt: inserted.createdAt.toISOString(),\n      },\n    });\n    return true;\n  });\n\n  if (enqueued) jobExecutionEnqueuedCount.add(1);\n}\n\nexport interface ClaimedJobExecution {\n  workflowRunId: string;\n  workflowRunAttemptId: string;\n  jobId: string;\n  jobExecutionId: string;\n  projectId: string;\n}\n\nexport interface ActiveRunningJobExecution {\n  workflowRunId: string;\n  workflowRunAttemptId: string;\n  jobId: string;\n  jobExecutionId: string;\n  projectId: string;\n  runnerSessionId: string;\n  provisionerId: string | null;\n  providerRunnerId: string | null;\n  requiredLabels: string[];\n  runnerLabels: string[];\n  startedAt: Date;\n  lastHeartbeatAt: Date;\n}\n\nexport interface RunnerInstanceBoundJobExecution {\n  workflowRunId: string;\n  workflowRunAttemptId: string;\n  jobId: string;\n  jobExecutionId: string;\n  providerRunnerId: string;\n  startedAt: Date;\n  lastHeartbeatAt: Date;\n  cancellationRequestedAt: Date | null;\n}\n\nexport async function claimPendingJobExecution(params: {\n  workspaceId: string;\n  runnerSessionId: string;\n  sessionLabels: string[];\n  maxClaims: number | null;\n  runnerSessionLivenessThrottleSeconds: number;\n}): Promise<ClaimedJobExecution | null> {\n  await touchRunnerSessionLiveness({\n    workspaceId: params.workspaceId,\n    runnerSessionId: params.runnerSessionId,\n    throttleSeconds: params.runnerSessionLivenessThrottleSeconds,\n  });\n\n  if (params.sessionLabels.length === 0) return null;\n\n  let activationToFirstClaimObservation: ProviderRunnerLifecycleObservation | null = null;\n  let queueTimeObservation: JobExecutionQueueTimeObservation | null = null;\n  const result = await db().transaction(async (tx) => {\n    let provisionerId: string | null = null;\n    let providerRunnerId: string | null = null;\n    let runnerInstanceId: string | null = null;\n\n    if (params.maxClaims !== null) {\n      const [session] = await tx\n        .select({\n          maxClaims: runnerSessions.maxClaims,\n          claimsUsed: runnerSessions.claimsUsed,\n          revokedAt: runnerSessions.revokedAt,\n          runnerInstanceId: runnerSessions.runnerInstanceId,\n          provisionerId: runnerSessions.provisionerId,\n          providerRunnerId: runnerSessions.providerRunnerId,\n        })\n        .from(runnerSessions)\n        .where(eq(runnerSessions.id, params.runnerSessionId))\n        .limit(1)\n        .for('update');\n\n      if (\n        !session ||\n        session.revokedAt ||\n        session.maxClaims === null ||\n        session.claimsUsed >= session.maxClaims\n      ) {\n        throw new RunnerSessionExhaustedError(params.runnerSessionId);\n      }\n\n      // Ephemeral sessions are the only capped sessions, and the DB check keeps\n      // their provisioned-runner link present as a pair.\n      runnerInstanceId = session.runnerInstanceId;\n      provisionerId = session.provisionerId;\n      providerRunnerId = session.providerRunnerId;\n    }\n\n    // `id` is a uuidv7 (time-ordered), so it is a deterministic FIFO tiebreaker\n    // for rows sharing a created_at within a batch. Lock only the FIFO candidate before\n    // attempting its execution advisory lock; putting pg_try_advisory_xact_lock in this\n    // predicate would evaluate it while scanning and temporarily lock many queue entries.\n    const candidate = tx\n      .select()\n      .from(pendingJobExecutions)\n      .where(\n        and(\n          eq(pendingJobExecutions.workspaceId, params.workspaceId),\n          arrayContained(pendingJobExecutions.requiredLabels, params.sessionLabels),\n        ),\n      )\n      .orderBy(asc(pendingJobExecutions.createdAt), asc(pendingJobExecutions.id))\n      .limit(1)\n      .for('update', {skipLocked: true})\n      .as('pending_candidate');\n\n    const [row] = await tx\n      .select()\n      .from(candidate)\n      .where(\n        sql`\n          pg_try_advisory_xact_lock(\n            hashtext(${runnerJobExecutionLockPrefix} || ${candidate.jobExecutionId}::text)\n          )\n        `,\n      );\n\n    if (!row) return null;\n\n    await tx.delete(pendingJobExecutions).where(eq(pendingJobExecutions.id, row.id));\n\n    // An enqueue retry that lands after a prior claim can leave an orphan pending\n    // row whose jobExecutionId is already in `runners_running_jobs`. Insert-or-skip\n    // on the jobExecutionId unique constraint: when the job execution is already running\n    // the insert touches no row, so we commit the orphan's deletion and return\n    // null rather than let the unique violation roll the claim back into a poison\n    // loop. The runner just re-polls for a real job execution.\n    const inserted = await tx\n      .insert(runningJobExecutions)\n      .values({\n        workspaceId: row.workspaceId,\n        workflowRunId: row.workflowRunId,\n        workflowRunAttemptId: row.workflowRunAttemptId,\n        jobId: row.jobId,\n        jobExecutionId: row.jobExecutionId,\n        projectId: row.projectId,\n        runnerSessionId: params.runnerSessionId,\n        provisionerId,\n        providerRunnerId,\n        requiredLabels: row.requiredLabels,\n        runnerLabels: params.sessionLabels,\n      })\n      .onConflictDoNothing({target: runningJobExecutions.jobExecutionId})\n      .returning({claimedAt: runningJobExecutions.startedAt});\n\n    const claimed = inserted[0];\n    if (!claimed) return null;\n\n    queueTimeObservation = {\n      durationMilliseconds: claimed.claimedAt.getTime() - row.createdAt.getTime(),\n      provider: null,\n      launchKind: params.maxClaims === null ? 'manual' : 'unknown',\n    };\n\n    const runnerInstanceCondition = runnerInstanceId\n      ? eq(providerRunners.id, runnerInstanceId)\n      : provisionerId && providerRunnerId\n        ? and(\n            eq(providerRunners.provisionerId, provisionerId),\n            eq(providerRunners.providerRunnerId, providerRunnerId),\n          )\n        : null;\n    if (runnerInstanceCondition) {\n      const [claimedRunner] = await tx\n        .update(providerRunners)\n        .set({\n          firstClaimedAt: sql`coalesce(${providerRunners.firstClaimedAt}, ${claimed.claimedAt})`,\n          updatedAt: sql`now()`,\n        })\n        .where(runnerInstanceCondition)\n        .returning({\n          firstClaimedAt: providerRunners.firstClaimedAt,\n          isFirstClaim: sql<boolean>`${providerRunners.firstClaimedAt} = ${claimed.claimedAt}`,\n          provider: providerRunners.providerKind,\n          launchKind: providerRunners.launchKind,\n          runnerInstanceId: providerRunners.id,\n          sessionCreatedAtEpochMs: sql<number | null>`(\n            select extract(epoch from ${runnerSessions.createdAt})::double precision * 1000\n            from ${runnerSessions}\n            where ${runnerSessions.id} = ${params.runnerSessionId}\n          )`,\n        });\n      if (claimedRunner && queueTimeObservation) {\n        queueTimeObservation.provider = claimedRunner.provider;\n        queueTimeObservation.launchKind = claimedRunner.launchKind;\n      }\n      if (\n        claimedRunner?.isFirstClaim &&\n        claimedRunner.firstClaimedAt !== null &&\n        claimedRunner.sessionCreatedAtEpochMs !== null &&\n        claimedRunner.sessionCreatedAtEpochMs !== undefined\n      )\n        activationToFirstClaimObservation = {\n          durationMilliseconds:\n            claimedRunner.firstClaimedAt.getTime() - claimedRunner.sessionCreatedAtEpochMs,\n          provider: claimedRunner.provider,\n          launchKind: claimedRunner.launchKind,\n          runnerInstanceId: claimedRunner.runnerInstanceId,\n        };\n    }\n\n    if (params.maxClaims !== null) {\n      await tx\n        .update(runnerSessions)\n        .set({claimsUsed: sql`${runnerSessions.claimsUsed} + 1`, updatedAt: sql`now()`})\n        .where(eq(runnerSessions.id, params.runnerSessionId));\n    }\n\n    // The running-row insert is the runner claiming the job execution. Emit in the same tx; the\n    // payload carries the row's own claim instant so a consumer records the true time,\n    // not the outbox drain time.\n    await writeOutboxEvent<RunnersEventMap>(tx, runnersOutbox, {\n      type: RUNNER_JOB_CLAIMED,\n      payload: {\n        workflowRunId: row.workflowRunId,\n        workflowRunAttemptId: row.workflowRunAttemptId,\n        jobId: row.jobId,\n        jobExecutionId: row.jobExecutionId,\n        claimedAt: claimed.claimedAt.toISOString(),\n      },\n    });\n\n    return {\n      workflowRunId: row.workflowRunId,\n      workflowRunAttemptId: row.workflowRunAttemptId,\n      jobId: row.jobId,\n      jobExecutionId: row.jobExecutionId,\n      projectId: row.projectId,\n    };\n  });\n  if (queueTimeObservation) recordJobExecutionQueueTime(queueTimeObservation);\n  if (activationToFirstClaimObservation)\n    recordProviderRunnerActivationToFirstClaim(activationToFirstClaimObservation);\n  return result;\n}\n\nasync function touchRunnerSessionLiveness(params: {\n  workspaceId: string;\n  runnerSessionId: string;\n  throttleSeconds: number;\n}): Promise<void> {\n  await db()\n    .update(runnerSessions)\n    .set({updatedAt: sql`now()`})\n    .where(\n      and(\n        eq(runnerSessions.id, params.runnerSessionId),\n        eq(runnerSessions.workspaceId, params.workspaceId),\n        sql`${runnerSessions.updatedAt} < now() - (${params.throttleSeconds} || ' seconds')::interval`,\n      ),\n    );\n}\n\n/**\n * Releases a job execution's lease when the orchestration workflow finalizes it: deletes the\n * running-job-execution row AND any lingering pending row for the same execution, in one tx.\n * Idempotent (0-row no-op), no token scope (the workflow is authoritative), and\n * emits no event: the workflow already owns the outcome. Sweeping the pending row\n * too closes the at-least-once window where an enqueue retry left an orphan that a\n * later claim would otherwise pick up for an already-finished job execution.\n */\nexport async function releaseJobExecution(params: {jobExecutionId: string}): Promise<void> {\n  await db().transaction(async (tx) => {\n    await lockJobExecution(tx, params.jobExecutionId);\n\n    // Delete pending before running to match `claimPendingJobExecution`'s lock-acquisition\n    // order (it locks the pending row first, then the running row). A concurrent\n    // claim picking up an orphan pending row for this same job execution would otherwise\n    // deadlock against the reverse order here.\n    await tx\n      .delete(pendingJobExecutions)\n      .where(eq(pendingJobExecutions.jobExecutionId, params.jobExecutionId));\n    await tx\n      .delete(runningJobExecutions)\n      .where(eq(runningJobExecutions.jobExecutionId, params.jobExecutionId));\n  });\n}\n\n/**\n * Reaps stale leases (bounded by `limit`), emitting one\n * `runners.job.lease_expired` event per reaped job execution.\n *\n * The cutoff is re-checked in the DELETE, not just the locking subquery, so a\n * heartbeat landing mid-call spares the live row. Each reaped execution also sweeps its\n * pending row: a failed best-effort `releaseJobExecution` would otherwise leave an orphan\n * that a later claim re-runs as an already-finished job execution.\n *\n * Locks pending-then-running to match `claimPendingJobExecution`, `releaseJobExecution`, and\n * `reconcileTerminalJobExecution`. The stale candidate scan intentionally does not lock running\n * rows first; the running-row DELETE re-checks the stale predicate after the pending-row sweep.\n */\nexport async function expireStuckJobExecutions(params: {\n  thresholdSeconds: number;\n  noFirstHeartbeatGraceSeconds: number;\n  limit?: number;\n}): Promise<\n  Array<{\n    workflowRunId: string;\n    workflowRunAttemptId: string;\n    jobId: string;\n    jobExecutionId: string;\n  }>\n> {\n  const reaped = await db().transaction(async (tx) => {\n    const heartbeatCutoff = sql`now() - (${params.thresholdSeconds} || ' seconds')::interval`;\n    const firstHeartbeatCutoff = sql`now() - (${params.noFirstHeartbeatGraceSeconds} || ' seconds')::interval`;\n    const stalePredicate = or(\n      and(\n        isNull(runningJobExecutions.firstHeartbeatAt),\n        lte(runningJobExecutions.lastHeartbeatAt, runningJobExecutions.startedAt),\n        lt(runningJobExecutions.startedAt, firstHeartbeatCutoff),\n      ),\n      and(\n        or(\n          isNotNull(runningJobExecutions.firstHeartbeatAt),\n          gt(runningJobExecutions.lastHeartbeatAt, runningJobExecutions.startedAt),\n        ),\n        lt(runningJobExecutions.lastHeartbeatAt, heartbeatCutoff),\n      ),\n    );\n\n    const staleRows = await tx\n      .select({\n        id: runningJobExecutions.id,\n        jobExecutionId: runningJobExecutions.jobExecutionId,\n      })\n      .from(runningJobExecutions)\n      .where(\n        and(\n          stalePredicate,\n          sql`\n            pg_try_advisory_xact_lock(\n              hashtext(${runnerJobExecutionLockPrefix} || ${runningJobExecutions.jobExecutionId}::text)\n            )\n          `,\n        ),\n      )\n      .orderBy(\n        asc(\n          sql`CASE WHEN ${runningJobExecutions.firstHeartbeatAt} IS NULL AND ${runningJobExecutions.lastHeartbeatAt} <= ${runningJobExecutions.startedAt} THEN ${runningJobExecutions.startedAt} ELSE ${runningJobExecutions.lastHeartbeatAt} END`,\n        ),\n        asc(runningJobExecutions.id),\n      )\n      .limit(params.limit ?? 100);\n\n    if (staleRows.length === 0) return [];\n\n    const staleIds = staleRows.map((row) => row.id);\n    const staleJobExecutionIds = staleRows.map((row) => row.jobExecutionId);\n\n    // Sweep pending rows first so this transaction cannot hold a running-row lock while waiting\n    // for a pending-row lock held by reconciliation or a claim of an orphan pending row. The\n    // advisory lock acquired in the candidate scan also makes enqueue retries wait until this\n    // transaction has either reaped the execution or released its lock without selecting it.\n    await tx\n      .delete(pendingJobExecutions)\n      .where(inArray(pendingJobExecutions.jobExecutionId, staleJobExecutionIds));\n\n    const deleted = await tx\n      .delete(runningJobExecutions)\n      .where(and(inArray(runningJobExecutions.id, staleIds), stalePredicate))\n      .returning({\n        workflowRunId: runningJobExecutions.workflowRunId,\n        workflowRunAttemptId: runningJobExecutions.workflowRunAttemptId,\n        jobId: runningJobExecutions.jobId,\n        jobExecutionId: runningJobExecutions.jobExecutionId,\n      });\n\n    if (deleted.length === 0) return [];\n\n    await writeOutboxEvents<RunnersEventMap>(\n      tx,\n      runnersOutbox,\n      deleted.map((row) => ({\n        type: RUNNER_JOB_LEASE_EXPIRED,\n        payload: {\n          workflowRunId: row.workflowRunId,\n          workflowRunAttemptId: row.workflowRunAttemptId,\n          jobId: row.jobId,\n          jobExecutionId: row.jobExecutionId,\n        },\n      })),\n    );\n\n    return deleted;\n  });\n\n  if (reaped.length > 0) jobExecutionLeaseExpiredCount.add(reaped.length);\n\n  return reaped;\n}\n\nexport async function getJobExecutionQueueDepth(): Promise<{\n  pendingJobExecutions: number;\n  runningJobExecutions: number;\n}> {\n  const [pending] = await db().select({value: count()}).from(pendingJobExecutions);\n  const [running] = await db().select({value: count()}).from(runningJobExecutions);\n  return {\n    pendingJobExecutions: pending?.value ?? 0,\n    runningJobExecutions: running?.value ?? 0,\n  };\n}\n\nexport async function listActiveRunningJobExecutions(params: {\n  workspaceId: string;\n  windowSeconds: number;\n  limit?: number;\n}): Promise<ActiveRunningJobExecution[]> {\n  return await db()\n    .select({\n      jobId: runningJobExecutions.jobId,\n      jobExecutionId: runningJobExecutions.jobExecutionId,\n      workflowRunId: runningJobExecutions.workflowRunId,\n      workflowRunAttemptId: runningJobExecutions.workflowRunAttemptId,\n      projectId: runningJobExecutions.projectId,\n      runnerSessionId: runningJobExecutions.runnerSessionId,\n      provisionerId: runningJobExecutions.provisionerId,\n      providerRunnerId: runningJobExecutions.providerRunnerId,\n      requiredLabels: runningJobExecutions.requiredLabels,\n      runnerLabels: runningJobExecutions.runnerLabels,\n      startedAt: runningJobExecutions.startedAt,\n      lastHeartbeatAt: runningJobExecutions.lastHeartbeatAt,\n    })\n    .from(runningJobExecutions)\n    .where(\n      and(\n        eq(runningJobExecutions.workspaceId, params.workspaceId),\n        sql`${runningJobExecutions.lastHeartbeatAt} > now() - (${params.windowSeconds} || ' seconds')::interval`,\n      ),\n    )\n    .orderBy(desc(runningJobExecutions.lastHeartbeatAt), desc(runningJobExecutions.id))\n    .limit(params.limit ?? 1000);\n}\n\nexport async function listRunningJobExecutionsByRunnerInstanceTx(\n  tx: Tx,\n  params: {\n    workspaceId: string;\n    provisionerId: string;\n    providerRunnerIds: string[];\n  },\n): Promise<RunnerInstanceBoundJobExecution[]> {\n  if (params.providerRunnerIds.length === 0) return [];\n\n  const duplicateRows = await tx\n    .select({\n      providerRunnerId: runningJobExecutions.providerRunnerId,\n      count: count(),\n    })\n    .from(runningJobExecutions)\n    .where(\n      and(\n        eq(runningJobExecutions.workspaceId, params.workspaceId),\n        eq(runningJobExecutions.provisionerId, params.provisionerId),\n        inArray(runningJobExecutions.providerRunnerId, params.providerRunnerIds),\n      ),\n    )\n    .groupBy(runningJobExecutions.providerRunnerId)\n    .having(sql`count(*) > 1`);\n\n  const duplicateRunnerInstanceIds = duplicateRows.flatMap((row) =>\n    row.providerRunnerId ? [row.providerRunnerId] : [],\n  );\n  if (duplicateRunnerInstanceIds.length > 0) {\n    logger().warn(\n      {\n        workspaceId: params.workspaceId,\n        provisionerId: params.provisionerId,\n        providerRunnerIds: duplicateRunnerInstanceIds,\n      },\n      'multiple running job executions are bound to the same provisioned runner',\n    );\n  }\n\n  const result = await tx.execute<{\n    workflowRunId: string;\n    workflowRunAttemptId: string;\n    jobId: string;\n    jobExecutionId: string;\n    providerRunnerId: string;\n    startedAt: Date | string;\n    lastHeartbeatAt: Date | string;\n    cancellationRequestedAt: Date | string | null;\n  }>(sql`\n    SELECT DISTINCT ON (${runningJobExecutions.providerRunnerId})\n      ${runningJobExecutions.workflowRunId} AS \"workflowRunId\",\n      ${runningJobExecutions.workflowRunAttemptId} AS \"workflowRunAttemptId\",\n      ${runningJobExecutions.jobId} AS \"jobId\",\n      ${runningJobExecutions.jobExecutionId} AS \"jobExecutionId\",\n      ${runningJobExecutions.providerRunnerId} AS \"providerRunnerId\",\n      ${runningJobExecutions.startedAt} AS \"startedAt\",\n      ${runningJobExecutions.lastHeartbeatAt} AS \"lastHeartbeatAt\",\n      ${runningJobExecutions.cancellationRequestedAt} AS \"cancellationRequestedAt\"\n    FROM ${runningJobExecutions}\n    WHERE\n      ${runningJobExecutions.workspaceId} = ${params.workspaceId}\n      AND ${runningJobExecutions.provisionerId} = ${params.provisionerId}\n      AND ${runningJobExecutions.providerRunnerId} IN (${sql.join(\n        params.providerRunnerIds.map((providerRunnerId) => sql`${providerRunnerId}`),\n        sql`, `,\n      )})\n    ORDER BY ${runningJobExecutions.providerRunnerId}, ${runningJobExecutions.startedAt} DESC, ${runningJobExecutions.jobExecutionId} DESC\n  `);\n\n  return result.rows.map((row) => ({\n    workflowRunId: row.workflowRunId,\n    workflowRunAttemptId: row.workflowRunAttemptId,\n    jobId: row.jobId,\n    jobExecutionId: row.jobExecutionId,\n    providerRunnerId: row.providerRunnerId,\n    startedAt: toDate(row.startedAt),\n    lastHeartbeatAt: toDate(row.lastHeartbeatAt),\n    cancellationRequestedAt: row.cancellationRequestedAt\n      ? toDate(row.cancellationRequestedAt)\n      : null,\n  }));\n}\n\nfunction toDate(value: Date | string): Date {\n  return value instanceof Date ? value : new Date(value);\n}\n\nexport async function isJobLeaseActive(params: {\n  jobId?: string;\n  jobExecutionId: string;\n  runnerSessionId: string;\n}): Promise<boolean> {\n  const [row] = await db()\n    .select({id: runningJobExecutions.id})\n    .from(runningJobExecutions)\n    .where(\n      and(\n        eq(runningJobExecutions.jobExecutionId, params.jobExecutionId),\n        eq(runningJobExecutions.runnerSessionId, params.runnerSessionId),\n        params.jobId === undefined ? undefined : eq(runningJobExecutions.jobId, params.jobId),\n      ),\n    )\n    .limit(1);\n\n  return row !== undefined;\n}\n\nexport async function recordHeartbeat(params: {\n  jobExecutionId: string;\n  runnerSessionId: string;\n  toolCapabilities?: RunnerToolCapabilitiesDto | null;\n}): Promise<{\n  cancellationRequested: boolean;\n  previousToolCapabilities: RunnerToolCapabilitiesDto | null;\n  currentToolCapabilities: RunnerToolCapabilitiesDto | null;\n  runningJobExecution: {\n    workflowRunId: string;\n    workflowRunAttemptId: string;\n    jobId: string;\n    jobExecutionId: string;\n    projectId: string;\n    workspaceId: string;\n    runnerSessionId: string;\n  };\n}> {\n  const result = await db().transaction(async (tx) => {\n    const updated = await tx\n      .update(runningJobExecutions)\n      .set({\n        firstHeartbeatAt: sql`COALESCE(${runningJobExecutions.firstHeartbeatAt}, now())`,\n        lastHeartbeatAt: sql`now()`,\n      })\n      .where(\n        and(\n          eq(runningJobExecutions.jobExecutionId, params.jobExecutionId),\n          eq(runningJobExecutions.runnerSessionId, params.runnerSessionId),\n        ),\n      )\n      .returning({\n        cancellationRequestedAt: runningJobExecutions.cancellationRequestedAt,\n        workflowRunId: runningJobExecutions.workflowRunId,\n        workflowRunAttemptId: runningJobExecutions.workflowRunAttemptId,\n        jobId: runningJobExecutions.jobId,\n        jobExecutionId: runningJobExecutions.jobExecutionId,\n        projectId: runningJobExecutions.projectId,\n        workspaceId: runningJobExecutions.workspaceId,\n        runnerSessionId: runningJobExecutions.runnerSessionId,\n      });\n\n    const row = updated[0];\n    if (!row) throw new RunningJobExecutionNotFoundError(params.jobExecutionId);\n\n    const [previous] = await tx\n      .select({toolCapabilities: runnerSessions.toolCapabilities})\n      .from(runnerSessions)\n      .where(eq(runnerSessions.id, params.runnerSessionId))\n      .limit(1)\n      .for('update');\n\n    const [session] = await tx\n      .update(runnerSessions)\n      .set({\n        toolCapabilities: params.toolCapabilities ?? null,\n        toolCapabilitiesReportedAt: params.toolCapabilities ? sql`now()` : null,\n        updatedAt: sql`now()`,\n      })\n      .where(eq(runnerSessions.id, params.runnerSessionId))\n      .returning({toolCapabilities: runnerSessions.toolCapabilities});\n\n    return {\n      row,\n      previousToolCapabilities: previous?.toolCapabilities ?? null,\n      currentToolCapabilities: session?.toolCapabilities ?? null,\n    };\n  });\n\n  const row = result.row;\n  if (!row) throw new RunningJobExecutionNotFoundError(params.jobExecutionId);\n  return {\n    cancellationRequested: row.cancellationRequestedAt !== null,\n    previousToolCapabilities: result.previousToolCapabilities,\n    currentToolCapabilities: result.currentToolCapabilities,\n    runningJobExecution: {\n      workflowRunId: row.workflowRunId,\n      workflowRunAttemptId: row.workflowRunAttemptId,\n      jobId: row.jobId,\n      jobExecutionId: row.jobExecutionId,\n      projectId: row.projectId,\n      workspaceId: row.workspaceId,\n      runnerSessionId: row.runnerSessionId,\n    },\n  };\n}\n\n/**\n * Reconciles a terminal job execution with runner state in one transaction: removes its pending\n * queue row and requests cancellation on its running lease, if either exists. The operation is\n * idempotent and preserves the first cancellation-request timestamp.\n */\nexport async function reconcileTerminalJobExecution(params: {\n  jobExecutionId: string;\n}): Promise<void> {\n  await db().transaction(async (tx) => {\n    await lockJobExecution(tx, params.jobExecutionId);\n\n    // Delete pending before updating running to match claim/release lock order. Claim locks\n    // pending rows with SKIP LOCKED before inserting the running lease, so this ordering makes\n    // a concurrent terminal reconciliation either win before claim or cancel the new lease.\n    await tx\n      .delete(pendingJobExecutions)\n      .where(eq(pendingJobExecutions.jobExecutionId, params.jobExecutionId));\n    await tx\n      .update(runningJobExecutions)\n      .set({\n        cancellationRequestedAt: sql`COALESCE(${runningJobExecutions.cancellationRequestedAt}, now())`,\n      })\n      .where(eq(runningJobExecutions.jobExecutionId, params.jobExecutionId));\n  });\n}\n\nexport async function cancelRunnerJobs(params: {jobIds: string[]}): Promise<void> {\n  if (params.jobIds.length === 0) return;\n\n  await db().transaction(async (tx) => {\n    await tx.delete(pendingJobExecutions).where(inArray(pendingJobExecutions.jobId, params.jobIds));\n    await tx\n      .update(runningJobExecutions)\n      .set({\n        cancellationRequestedAt: sql`COALESCE(${runningJobExecutions.cancellationRequestedAt}, now())`,\n      })\n      .where(inArray(runningJobExecutions.jobId, params.jobIds));\n  });\n}\n"],"names":["RUNNER_JOB_CLAIMED","RUNNER_JOB_LEASE_EXPIRED","RUNNER_JOB_QUEUED","logger","writeOutboxEvent","writeOutboxEvents","canonicalizeLabels","and","arrayContained","asc","count","desc","eq","gt","inArray","isNotNull","isNull","lt","lte","or","sql","EmptyRequiredLabelsError","RunnerSessionExhaustedError","RunningJobExecutionNotFoundError","jobExecutionEnqueuedCount","jobExecutionLeaseExpiredCount","recordJobExecutionQueueTime","recordProviderRunnerActivationToFirstClaim","db","runnersOutbox","pendingJobExecutions","providerRunners","runnerSessions","runningJobExecutions","runnerJobExecutionLockPrefix","lockJobExecution","tx","jobExecutionId","execute","getWorkspaceJobCounts","params","pendingRows","runningRows","Promise","all","select","workspaceId","from","where","workspaceIds","groupBy","queuedByWorkspace","Map","map","row","Number","runningByWorkspace","queued","get","running","enqueueJobExecution","requiredLabels","length","enqueued","transaction","leaseExpired","id","eventType","payload","limit","inserted","insert","values","workflowRunId","workflowRunAttemptId","jobId","projectId","onConflictDoNothing","target","returning","createdAt","type","queuedAt","toISOString","add","claimPendingJobExecution","touchRunnerSessionLiveness","runnerSessionId","throttleSeconds","runnerSessionLivenessThrottleSeconds","sessionLabels","activationToFirstClaimObservation","queueTimeObservation","result","provisionerId","providerRunnerId","runnerInstanceId","maxClaims","session","claimsUsed","revokedAt","for","candidate","orderBy","skipLocked","as","delete","runnerLabels","claimedAt","startedAt","claimed","durationMilliseconds","getTime","provider","launchKind","runnerInstanceCondition","claimedRunner","update","set","firstClaimedAt","updatedAt","isFirstClaim","providerKind","sessionCreatedAtEpochMs","undefined","releaseJobExecution","expireStuckJobExecutions","reaped","heartbeatCutoff","thresholdSeconds","firstHeartbeatCutoff","noFirstHeartbeatGraceSeconds","stalePredicate","firstHeartbeatAt","lastHeartbeatAt","staleRows","staleIds","staleJobExecutionIds","deleted","getJobExecutionQueueDepth","pending","value","listActiveRunningJobExecutions","windowSeconds","listRunningJobExecutionsByRunnerInstanceTx","providerRunnerIds","duplicateRows","having","duplicateRunnerInstanceIds","flatMap","warn","cancellationRequestedAt","join","rows","toDate","Date","isJobLeaseActive","recordHeartbeat","updated","previous","toolCapabilities","toolCapabilitiesReportedAt","previousToolCapabilities","currentToolCapabilities","cancellationRequested","runningJobExecution","reconcileTerminalJobExecution","cancelRunnerJobs","jobIds"],"mappings":"AAAA,SACEA,kBAAkB,EAClBC,wBAAwB,EACxBC,iBAAiB,QAGZ,2BAA2B;AAClC,SAAQC,MAAM,QAAO,8BAA8B;AACnD,SAAQC,gBAAgB,EAAEC,iBAAiB,QAAO,uBAAuB;AACzE,SAAQC,kBAAkB,QAAO,yBAAyB;AAC1D,SACEC,GAAG,EACHC,cAAc,EACdC,GAAG,EACHC,KAAK,EACLC,IAAI,EACJC,EAAE,EACFC,EAAE,EACFC,OAAO,EACPC,SAAS,EACTC,MAAM,EACNC,EAAE,EACFC,GAAG,EACHC,EAAE,EACFC,GAAG,QACE,cAAc;AACrB,SACEC,wBAAwB,EACxBC,2BAA2B,EAC3BC,gCAAgC,QAC3B,kBAAkB;AACzB,SAEEC,yBAAyB,EACzBC,6BAA6B,EAE7BC,2BAA2B,EAC3BC,0CAA0C,QACrC,uBAAuB;AAE9B,SAAQC,EAAE,QAAO,UAAU;AAC3B,SAAQC,aAAa,QAAO,qBAAqB;AACjD,SAAQC,oBAAoB,QAAO,qCAAqC;AACxE,SAAQC,eAAe,QAAO,+BAA+B;AAC7D,SAAQC,cAAc,QAAO,8BAA8B;AAC3D,SAAQC,oBAAoB,QAAO,qCAAqC;AAExE,MAAMC,+BAA+B;AAErC,eAAeC,iBAAiBC,EAAM,EAAEC,cAAsB;IAC5D,MAAMD,GAAGE,OAAO,CACdlB,GAAG,CAAC,sCAAsC,EAAE,GAAGc,+BAA+BG,gBAAgB,CAAC,EAAE,CAAC;AAEtG;AAYA,OAAO,eAAeE,sBAAsBC,MAE3C;IACC,MAAM,CAACC,aAAaC,YAAY,GAAG,MAAMC,QAAQC,GAAG,CAAC;QACnDhB,KACGiB,MAAM,CAAC;YAACC,aAAahB,qBAAqBgB,WAAW;YAAEpC,OAAOA;QAAO,GACrEqC,IAAI,CAACjB,sBACLkB,KAAK,CAAClC,QAAQgB,qBAAqBgB,WAAW,EAAEN,OAAOS,YAAY,GACnEC,OAAO,CAACpB,qBAAqBgB,WAAW;QAC3ClB,KACGiB,MAAM,CAAC;YAACC,aAAab,qBAAqBa,WAAW;YAAEpC,OAAOA;QAAO,GACrEqC,IAAI,CAACd,sBACLe,KAAK,CAAClC,QAAQmB,qBAAqBa,WAAW,EAAEN,OAAOS,YAAY,GACnEC,OAAO,CAACjB,qBAAqBa,WAAW;KAC5C;IAED,MAAMK,oBAAoB,IAAIC,IAAIX,YAAYY,GAAG,CAAC,CAACC,MAAQ;YAACA,IAAIR,WAAW;YAAES,OAAOD,IAAI5C,KAAK;SAAE;IAC/F,MAAM8C,qBAAqB,IAAIJ,IAC7BV,YAAYW,GAAG,CAAC,CAACC,MAAQ;YAACA,IAAIR,WAAW;YAAES,OAAOD,IAAI5C,KAAK;SAAE;IAG/D,OAAO8B,OAAOS,YAAY,CAACI,GAAG,CAAC,CAACP,cAAiB,CAAA;YAC/CA;YACAW,QAAQN,kBAAkBO,GAAG,CAACZ,gBAAgB;YAC9Ca,SAASH,mBAAmBE,GAAG,CAACZ,gBAAgB;QAClD,CAAA;AACF;AAEA,6FAA6F;AAC7F,+FAA+F;AAC/F,kGAAkG;AAClG,+FAA+F;AAC/F,6FAA6F;AAC7F,OAAO,eAAec,oBAAoBpB,MAAiC;IACzE,MAAMqB,iBAAiB;WAAIvD,mBAAmBkC,OAAOqB,cAAc;KAAE;IACrE,IAAIA,eAAeC,MAAM,KAAK,GAAG,MAAM,IAAIzC;IAE3C,MAAM0C,WAAW,MAAMnC,KAAKoC,WAAW,CAAC,OAAO5B;QAC7C,MAAMD,iBAAiBC,IAAII,OAAOH,cAAc;QAEhD,MAAM,CAAC4B,aAAa,GAAG,MAAM7B,GAC1BS,MAAM,CAAC;YAACqB,IAAIrC,cAAcqC,EAAE;QAAA,GAC5BnB,IAAI,CAAClB,eACLmB,KAAK,CACJzC,IACEK,GAAGiB,cAAcsC,SAAS,EAAElE,2BAC5BmB,GAAG,CAAC,EAAES,cAAcuC,OAAO,CAAC,sBAAsB,EAAE5B,OAAOH,cAAc,CAAC,CAAC,GAG9EgC,KAAK,CAAC;QACT,IAAIJ,cAAc,OAAO;QAEzB,MAAM,CAACK,SAAS,GAAG,MAAMlC,GACtBmC,MAAM,CAACzC,sBACP0C,MAAM,CAAC;YACN1B,aAAaN,OAAOM,WAAW;YAC/B2B,eAAejC,OAAOiC,aAAa;YACnCC,sBAAsBlC,OAAOkC,oBAAoB;YACjDC,OAAOnC,OAAOmC,KAAK;YACnBtC,gBAAgBG,OAAOH,cAAc;YACrCuC,WAAWpC,OAAOoC,SAAS;YAC3Bf;QACF,GACCgB,mBAAmB,CAAC;YAACC,QAAQhD,qBAAqBO,cAAc;QAAA,GAChE0C,SAAS,CAAC;YAACC,WAAWlD,qBAAqBkD,SAAS;QAAA;QAEvD,4EAA4E;QAC5E,0EAA0E;QAC1E,mEAAmE;QACnE,IAAI,CAACV,UAAU,OAAO;QAEtB,MAAMlE,iBAAkCgC,IAAIP,eAAe;YACzDoD,MAAM/E;YACNkE,SAAS;gBACPK,eAAejC,OAAOiC,aAAa;gBACnCC,sBAAsBlC,OAAOkC,oBAAoB;gBACjDC,OAAOnC,OAAOmC,KAAK;gBACnBtC,gBAAgBG,OAAOH,cAAc;gBACrC6C,UAAUZ,SAASU,SAAS,CAACG,WAAW;YAC1C;QACF;QACA,OAAO;IACT;IAEA,IAAIpB,UAAUvC,0BAA0B4D,GAAG,CAAC;AAC9C;AAoCA,OAAO,eAAeC,yBAAyB7C,MAM9C;IACC,MAAM8C,2BAA2B;QAC/BxC,aAAaN,OAAOM,WAAW;QAC/ByC,iBAAiB/C,OAAO+C,eAAe;QACvCC,iBAAiBhD,OAAOiD,oCAAoC;IAC9D;IAEA,IAAIjD,OAAOkD,aAAa,CAAC5B,MAAM,KAAK,GAAG,OAAO;IAE9C,IAAI6B,oCAA+E;IACnF,IAAIC,uBAAgE;IACpE,MAAMC,SAAS,MAAMjE,KAAKoC,WAAW,CAAC,OAAO5B;QAC3C,IAAI0D,gBAA+B;QACnC,IAAIC,mBAAkC;QACtC,IAAIC,mBAAkC;QAEtC,IAAIxD,OAAOyD,SAAS,KAAK,MAAM;YAC7B,MAAM,CAACC,QAAQ,GAAG,MAAM9D,GACrBS,MAAM,CAAC;gBACNoD,WAAWjE,eAAeiE,SAAS;gBACnCE,YAAYnE,eAAemE,UAAU;gBACrCC,WAAWpE,eAAeoE,SAAS;gBACnCJ,kBAAkBhE,eAAegE,gBAAgB;gBACjDF,eAAe9D,eAAe8D,aAAa;gBAC3CC,kBAAkB/D,eAAe+D,gBAAgB;YACnD,GACChD,IAAI,CAACf,gBACLgB,KAAK,CAACpC,GAAGoB,eAAekC,EAAE,EAAE1B,OAAO+C,eAAe,GAClDlB,KAAK,CAAC,GACNgC,GAAG,CAAC;YAEP,IACE,CAACH,WACDA,QAAQE,SAAS,IACjBF,QAAQD,SAAS,KAAK,QACtBC,QAAQC,UAAU,IAAID,QAAQD,SAAS,EACvC;gBACA,MAAM,IAAI3E,4BAA4BkB,OAAO+C,eAAe;YAC9D;YAEA,0EAA0E;YAC1E,mDAAmD;YACnDS,mBAAmBE,QAAQF,gBAAgB;YAC3CF,gBAAgBI,QAAQJ,aAAa;YACrCC,mBAAmBG,QAAQH,gBAAgB;QAC7C;QAEA,4EAA4E;QAC5E,oFAAoF;QACpF,oFAAoF;QACpF,sFAAsF;QACtF,MAAMO,YAAYlE,GACfS,MAAM,GACNE,IAAI,CAACjB,sBACLkB,KAAK,CACJzC,IACEK,GAAGkB,qBAAqBgB,WAAW,EAAEN,OAAOM,WAAW,GACvDtC,eAAesB,qBAAqB+B,cAAc,EAAErB,OAAOkD,aAAa,IAG3Ea,OAAO,CAAC9F,IAAIqB,qBAAqBkD,SAAS,GAAGvE,IAAIqB,qBAAqBoC,EAAE,GACxEG,KAAK,CAAC,GACNgC,GAAG,CAAC,UAAU;YAACG,YAAY;QAAI,GAC/BC,EAAE,CAAC;QAEN,MAAM,CAACnD,IAAI,GAAG,MAAMlB,GACjBS,MAAM,GACNE,IAAI,CAACuD,WACLtD,KAAK,CACJ5B,GAAG,CAAC;;qBAES,EAAEc,6BAA6B,IAAI,EAAEoE,UAAUjE,cAAc,CAAC;;QAE3E,CAAC;QAGL,IAAI,CAACiB,KAAK,OAAO;QAEjB,MAAMlB,GAAGsE,MAAM,CAAC5E,sBAAsBkB,KAAK,CAACpC,GAAGkB,qBAAqBoC,EAAE,EAAEZ,IAAIY,EAAE;QAE9E,8EAA8E;QAC9E,gFAAgF;QAChF,qFAAqF;QACrF,2EAA2E;QAC3E,8EAA8E;QAC9E,2DAA2D;QAC3D,MAAMI,WAAW,MAAMlC,GACpBmC,MAAM,CAACtC,sBACPuC,MAAM,CAAC;YACN1B,aAAaQ,IAAIR,WAAW;YAC5B2B,eAAenB,IAAImB,aAAa;YAChCC,sBAAsBpB,IAAIoB,oBAAoB;YAC9CC,OAAOrB,IAAIqB,KAAK;YAChBtC,gBAAgBiB,IAAIjB,cAAc;YAClCuC,WAAWtB,IAAIsB,SAAS;YACxBW,iBAAiB/C,OAAO+C,eAAe;YACvCO;YACAC;YACAlC,gBAAgBP,IAAIO,cAAc;YAClC8C,cAAcnE,OAAOkD,aAAa;QACpC,GACCb,mBAAmB,CAAC;YAACC,QAAQ7C,qBAAqBI,cAAc;QAAA,GAChE0C,SAAS,CAAC;YAAC6B,WAAW3E,qBAAqB4E,SAAS;QAAA;QAEvD,MAAMC,UAAUxC,QAAQ,CAAC,EAAE;QAC3B,IAAI,CAACwC,SAAS,OAAO;QAErBlB,uBAAuB;YACrBmB,sBAAsBD,QAAQF,SAAS,CAACI,OAAO,KAAK1D,IAAI0B,SAAS,CAACgC,OAAO;YACzEC,UAAU;YACVC,YAAY1E,OAAOyD,SAAS,KAAK,OAAO,WAAW;QACrD;QAEA,MAAMkB,0BAA0BnB,mBAC5BpF,GAAGmB,gBAAgBmC,EAAE,EAAE8B,oBACvBF,iBAAiBC,mBACfxF,IACEK,GAAGmB,gBAAgB+D,aAAa,EAAEA,gBAClClF,GAAGmB,gBAAgBgE,gBAAgB,EAAEA,qBAEvC;QACN,IAAIoB,yBAAyB;YAC3B,MAAM,CAACC,cAAc,GAAG,MAAMhF,GAC3BiF,MAAM,CAACtF,iBACPuF,GAAG,CAAC;gBACHC,gBAAgBnG,GAAG,CAAC,SAAS,EAAEW,gBAAgBwF,cAAc,CAAC,EAAE,EAAET,QAAQF,SAAS,CAAC,CAAC,CAAC;gBACtFY,WAAWpG,GAAG,CAAC,KAAK,CAAC;YACvB,GACC4B,KAAK,CAACmE,yBACNpC,SAAS,CAAC;gBACTwC,gBAAgBxF,gBAAgBwF,cAAc;gBAC9CE,cAAcrG,GAAY,CAAC,EAAEW,gBAAgBwF,cAAc,CAAC,GAAG,EAAET,QAAQF,SAAS,CAAC,CAAC;gBACpFK,UAAUlF,gBAAgB2F,YAAY;gBACtCR,YAAYnF,gBAAgBmF,UAAU;gBACtClB,kBAAkBjE,gBAAgBmC,EAAE;gBACpCyD,yBAAyBvG,GAAkB,CAAC;sCAChB,EAAEY,eAAegD,SAAS,CAAC;iBAChD,EAAEhD,eAAe;kBAChB,EAAEA,eAAekC,EAAE,CAAC,GAAG,EAAE1B,OAAO+C,eAAe,CAAC;WACvD,CAAC;YACJ;YACF,IAAI6B,iBAAiBxB,sBAAsB;gBACzCA,qBAAqBqB,QAAQ,GAAGG,cAAcH,QAAQ;gBACtDrB,qBAAqBsB,UAAU,GAAGE,cAAcF,UAAU;YAC5D;YACA,IACEE,eAAeK,gBACfL,cAAcG,cAAc,KAAK,QACjCH,cAAcO,uBAAuB,KAAK,QAC1CP,cAAcO,uBAAuB,KAAKC,WAE1CjC,oCAAoC;gBAClCoB,sBACEK,cAAcG,cAAc,CAACP,OAAO,KAAKI,cAAcO,uBAAuB;gBAChFV,UAAUG,cAAcH,QAAQ;gBAChCC,YAAYE,cAAcF,UAAU;gBACpClB,kBAAkBoB,cAAcpB,gBAAgB;YAClD;QACJ;QAEA,IAAIxD,OAAOyD,SAAS,KAAK,MAAM;YAC7B,MAAM7D,GACHiF,MAAM,CAACrF,gBACPsF,GAAG,CAAC;gBAACnB,YAAY/E,GAAG,CAAC,EAAEY,eAAemE,UAAU,CAAC,IAAI,CAAC;gBAAEqB,WAAWpG,GAAG,CAAC,KAAK,CAAC;YAAA,GAC7E4B,KAAK,CAACpC,GAAGoB,eAAekC,EAAE,EAAE1B,OAAO+C,eAAe;QACvD;QAEA,4FAA4F;QAC5F,mFAAmF;QACnF,6BAA6B;QAC7B,MAAMnF,iBAAkCgC,IAAIP,eAAe;YACzDoD,MAAMjF;YACNoE,SAAS;gBACPK,eAAenB,IAAImB,aAAa;gBAChCC,sBAAsBpB,IAAIoB,oBAAoB;gBAC9CC,OAAOrB,IAAIqB,KAAK;gBAChBtC,gBAAgBiB,IAAIjB,cAAc;gBAClCuE,WAAWE,QAAQF,SAAS,CAACzB,WAAW;YAC1C;QACF;QAEA,OAAO;YACLV,eAAenB,IAAImB,aAAa;YAChCC,sBAAsBpB,IAAIoB,oBAAoB;YAC9CC,OAAOrB,IAAIqB,KAAK;YAChBtC,gBAAgBiB,IAAIjB,cAAc;YAClCuC,WAAWtB,IAAIsB,SAAS;QAC1B;IACF;IACA,IAAIgB,sBAAsBlE,4BAA4BkE;IACtD,IAAID,mCACFhE,2CAA2CgE;IAC7C,OAAOE;AACT;AAEA,eAAeP,2BAA2B9C,MAIzC;IACC,MAAMZ,KACHyF,MAAM,CAACrF,gBACPsF,GAAG,CAAC;QAACE,WAAWpG,GAAG,CAAC,KAAK,CAAC;IAAA,GAC1B4B,KAAK,CACJzC,IACEK,GAAGoB,eAAekC,EAAE,EAAE1B,OAAO+C,eAAe,GAC5C3E,GAAGoB,eAAec,WAAW,EAAEN,OAAOM,WAAW,GACjD1B,GAAG,CAAC,EAAEY,eAAewF,SAAS,CAAC,YAAY,EAAEhF,OAAOgD,eAAe,CAAC,yBAAyB,CAAC;AAGtG;AAEA;;;;;;;CAOC,GACD,OAAO,eAAeqC,oBAAoBrF,MAAgC;IACxE,MAAMZ,KAAKoC,WAAW,CAAC,OAAO5B;QAC5B,MAAMD,iBAAiBC,IAAII,OAAOH,cAAc;QAEhD,uFAAuF;QACvF,6EAA6E;QAC7E,qFAAqF;QACrF,2CAA2C;QAC3C,MAAMD,GACHsE,MAAM,CAAC5E,sBACPkB,KAAK,CAACpC,GAAGkB,qBAAqBO,cAAc,EAAEG,OAAOH,cAAc;QACtE,MAAMD,GACHsE,MAAM,CAACzE,sBACPe,KAAK,CAACpC,GAAGqB,qBAAqBI,cAAc,EAAEG,OAAOH,cAAc;IACxE;AACF;AAEA;;;;;;;;;;;;CAYC,GACD,OAAO,eAAeyF,yBAAyBtF,MAI9C;IAQC,MAAMuF,SAAS,MAAMnG,KAAKoC,WAAW,CAAC,OAAO5B;QAC3C,MAAM4F,kBAAkB5G,GAAG,CAAC,SAAS,EAAEoB,OAAOyF,gBAAgB,CAAC,yBAAyB,CAAC;QACzF,MAAMC,uBAAuB9G,GAAG,CAAC,SAAS,EAAEoB,OAAO2F,4BAA4B,CAAC,yBAAyB,CAAC;QAC1G,MAAMC,iBAAiBjH,GACrBZ,IACES,OAAOiB,qBAAqBoG,gBAAgB,GAC5CnH,IAAIe,qBAAqBqG,eAAe,EAAErG,qBAAqB4E,SAAS,GACxE5F,GAAGgB,qBAAqB4E,SAAS,EAAEqB,wBAErC3H,IACEY,GACEJ,UAAUkB,qBAAqBoG,gBAAgB,GAC/CxH,GAAGoB,qBAAqBqG,eAAe,EAAErG,qBAAqB4E,SAAS,IAEzE5F,GAAGgB,qBAAqBqG,eAAe,EAAEN;QAI7C,MAAMO,YAAY,MAAMnG,GACrBS,MAAM,CAAC;YACNqB,IAAIjC,qBAAqBiC,EAAE;YAC3B7B,gBAAgBJ,qBAAqBI,cAAc;QACrD,GACCU,IAAI,CAACd,sBACLe,KAAK,CACJzC,IACE6H,gBACAhH,GAAG,CAAC;;uBAES,EAAEc,6BAA6B,IAAI,EAAED,qBAAqBI,cAAc,CAAC;;UAEtF,CAAC,GAGJkE,OAAO,CACN9F,IACEW,GAAG,CAAC,UAAU,EAAEa,qBAAqBoG,gBAAgB,CAAC,aAAa,EAAEpG,qBAAqBqG,eAAe,CAAC,IAAI,EAAErG,qBAAqB4E,SAAS,CAAC,MAAM,EAAE5E,qBAAqB4E,SAAS,CAAC,MAAM,EAAE5E,qBAAqBqG,eAAe,CAAC,IAAI,CAAC,GAE1O7H,IAAIwB,qBAAqBiC,EAAE,GAE5BG,KAAK,CAAC7B,OAAO6B,KAAK,IAAI;QAEzB,IAAIkE,UAAUzE,MAAM,KAAK,GAAG,OAAO,EAAE;QAErC,MAAM0E,WAAWD,UAAUlF,GAAG,CAAC,CAACC,MAAQA,IAAIY,EAAE;QAC9C,MAAMuE,uBAAuBF,UAAUlF,GAAG,CAAC,CAACC,MAAQA,IAAIjB,cAAc;QAEtE,4FAA4F;QAC5F,yFAAyF;QACzF,0FAA0F;QAC1F,yFAAyF;QACzF,MAAMD,GACHsE,MAAM,CAAC5E,sBACPkB,KAAK,CAAClC,QAAQgB,qBAAqBO,cAAc,EAAEoG;QAEtD,MAAMC,UAAU,MAAMtG,GACnBsE,MAAM,CAACzE,sBACPe,KAAK,CAACzC,IAAIO,QAAQmB,qBAAqBiC,EAAE,EAAEsE,WAAWJ,iBACtDrD,SAAS,CAAC;YACTN,eAAexC,qBAAqBwC,aAAa;YACjDC,sBAAsBzC,qBAAqByC,oBAAoB;YAC/DC,OAAO1C,qBAAqB0C,KAAK;YACjCtC,gBAAgBJ,qBAAqBI,cAAc;QACrD;QAEF,IAAIqG,QAAQ5E,MAAM,KAAK,GAAG,OAAO,EAAE;QAEnC,MAAMzD,kBACJ+B,IACAP,eACA6G,QAAQrF,GAAG,CAAC,CAACC,MAAS,CAAA;gBACpB2B,MAAMhF;gBACNmE,SAAS;oBACPK,eAAenB,IAAImB,aAAa;oBAChCC,sBAAsBpB,IAAIoB,oBAAoB;oBAC9CC,OAAOrB,IAAIqB,KAAK;oBAChBtC,gBAAgBiB,IAAIjB,cAAc;gBACpC;YACF,CAAA;QAGF,OAAOqG;IACT;IAEA,IAAIX,OAAOjE,MAAM,GAAG,GAAGrC,8BAA8B2D,GAAG,CAAC2C,OAAOjE,MAAM;IAEtE,OAAOiE;AACT;AAEA,OAAO,eAAeY;IAIpB,MAAM,CAACC,QAAQ,GAAG,MAAMhH,KAAKiB,MAAM,CAAC;QAACgG,OAAOnI;IAAO,GAAGqC,IAAI,CAACjB;IAC3D,MAAM,CAAC6B,QAAQ,GAAG,MAAM/B,KAAKiB,MAAM,CAAC;QAACgG,OAAOnI;IAAO,GAAGqC,IAAI,CAACd;IAC3D,OAAO;QACLH,sBAAsB8G,SAASC,SAAS;QACxC5G,sBAAsB0B,SAASkF,SAAS;IAC1C;AACF;AAEA,OAAO,eAAeC,+BAA+BtG,MAIpD;IACC,OAAO,MAAMZ,KACViB,MAAM,CAAC;QACN8B,OAAO1C,qBAAqB0C,KAAK;QACjCtC,gBAAgBJ,qBAAqBI,cAAc;QACnDoC,eAAexC,qBAAqBwC,aAAa;QACjDC,sBAAsBzC,qBAAqByC,oBAAoB;QAC/DE,WAAW3C,qBAAqB2C,SAAS;QACzCW,iBAAiBtD,qBAAqBsD,eAAe;QACrDO,eAAe7D,qBAAqB6D,aAAa;QACjDC,kBAAkB9D,qBAAqB8D,gBAAgB;QACvDlC,gBAAgB5B,qBAAqB4B,cAAc;QACnD8C,cAAc1E,qBAAqB0E,YAAY;QAC/CE,WAAW5E,qBAAqB4E,SAAS;QACzCyB,iBAAiBrG,qBAAqBqG,eAAe;IACvD,GACCvF,IAAI,CAACd,sBACLe,KAAK,CACJzC,IACEK,GAAGqB,qBAAqBa,WAAW,EAAEN,OAAOM,WAAW,GACvD1B,GAAG,CAAC,EAAEa,qBAAqBqG,eAAe,CAAC,YAAY,EAAE9F,OAAOuG,aAAa,CAAC,yBAAyB,CAAC,GAG3GxC,OAAO,CAAC5F,KAAKsB,qBAAqBqG,eAAe,GAAG3H,KAAKsB,qBAAqBiC,EAAE,GAChFG,KAAK,CAAC7B,OAAO6B,KAAK,IAAI;AAC3B;AAEA,OAAO,eAAe2E,2CACpB5G,EAAM,EACNI,MAIC;IAED,IAAIA,OAAOyG,iBAAiB,CAACnF,MAAM,KAAK,GAAG,OAAO,EAAE;IAEpD,MAAMoF,gBAAgB,MAAM9G,GACzBS,MAAM,CAAC;QACNkD,kBAAkB9D,qBAAqB8D,gBAAgB;QACvDrF,OAAOA;IACT,GACCqC,IAAI,CAACd,sBACLe,KAAK,CACJzC,IACEK,GAAGqB,qBAAqBa,WAAW,EAAEN,OAAOM,WAAW,GACvDlC,GAAGqB,qBAAqB6D,aAAa,EAAEtD,OAAOsD,aAAa,GAC3DhF,QAAQmB,qBAAqB8D,gBAAgB,EAAEvD,OAAOyG,iBAAiB,IAG1E/F,OAAO,CAACjB,qBAAqB8D,gBAAgB,EAC7CoD,MAAM,CAAC/H,GAAG,CAAC,YAAY,CAAC;IAE3B,MAAMgI,6BAA6BF,cAAcG,OAAO,CAAC,CAAC/F,MACxDA,IAAIyC,gBAAgB,GAAG;YAACzC,IAAIyC,gBAAgB;SAAC,GAAG,EAAE;IAEpD,IAAIqD,2BAA2BtF,MAAM,GAAG,GAAG;QACzC3D,SAASmJ,IAAI,CACX;YACExG,aAAaN,OAAOM,WAAW;YAC/BgD,eAAetD,OAAOsD,aAAa;YACnCmD,mBAAmBG;QACrB,GACA;IAEJ;IAEA,MAAMvD,SAAS,MAAMzD,GAAGE,OAAO,CAS5BlB,GAAG,CAAC;wBACe,EAAEa,qBAAqB8D,gBAAgB,CAAC;MAC1D,EAAE9D,qBAAqBwC,aAAa,CAAC;MACrC,EAAExC,qBAAqByC,oBAAoB,CAAC;MAC5C,EAAEzC,qBAAqB0C,KAAK,CAAC;MAC7B,EAAE1C,qBAAqBI,cAAc,CAAC;MACtC,EAAEJ,qBAAqB8D,gBAAgB,CAAC;MACxC,EAAE9D,qBAAqB4E,SAAS,CAAC;MACjC,EAAE5E,qBAAqBqG,eAAe,CAAC;MACvC,EAAErG,qBAAqBsH,uBAAuB,CAAC;SAC5C,EAAEtH,qBAAqB;;MAE1B,EAAEA,qBAAqBa,WAAW,CAAC,GAAG,EAAEN,OAAOM,WAAW,CAAC;UACvD,EAAEb,qBAAqB6D,aAAa,CAAC,GAAG,EAAEtD,OAAOsD,aAAa,CAAC;UAC/D,EAAE7D,qBAAqB8D,gBAAgB,CAAC,KAAK,EAAE3E,IAAIoI,IAAI,CACzDhH,OAAOyG,iBAAiB,CAAC5F,GAAG,CAAC,CAAC0C,mBAAqB3E,GAAG,CAAC,EAAE2E,iBAAiB,CAAC,GAC3E3E,GAAG,CAAC,EAAE,CAAC,EACP;aACK,EAAEa,qBAAqB8D,gBAAgB,CAAC,EAAE,EAAE9D,qBAAqB4E,SAAS,CAAC,OAAO,EAAE5E,qBAAqBI,cAAc,CAAC;EACnI,CAAC;IAED,OAAOwD,OAAO4D,IAAI,CAACpG,GAAG,CAAC,CAACC,MAAS,CAAA;YAC/BmB,eAAenB,IAAImB,aAAa;YAChCC,sBAAsBpB,IAAIoB,oBAAoB;YAC9CC,OAAOrB,IAAIqB,KAAK;YAChBtC,gBAAgBiB,IAAIjB,cAAc;YAClC0D,kBAAkBzC,IAAIyC,gBAAgB;YACtCc,WAAW6C,OAAOpG,IAAIuD,SAAS;YAC/ByB,iBAAiBoB,OAAOpG,IAAIgF,eAAe;YAC3CiB,yBAAyBjG,IAAIiG,uBAAuB,GAChDG,OAAOpG,IAAIiG,uBAAuB,IAClC;QACN,CAAA;AACF;AAEA,SAASG,OAAOb,KAAoB;IAClC,OAAOA,iBAAiBc,OAAOd,QAAQ,IAAIc,KAAKd;AAClD;AAEA,OAAO,eAAee,iBAAiBpH,MAItC;IACC,MAAM,CAACc,IAAI,GAAG,MAAM1B,KACjBiB,MAAM,CAAC;QAACqB,IAAIjC,qBAAqBiC,EAAE;IAAA,GACnCnB,IAAI,CAACd,sBACLe,KAAK,CACJzC,IACEK,GAAGqB,qBAAqBI,cAAc,EAAEG,OAAOH,cAAc,GAC7DzB,GAAGqB,qBAAqBsD,eAAe,EAAE/C,OAAO+C,eAAe,GAC/D/C,OAAOmC,KAAK,KAAKiD,YAAYA,YAAYhH,GAAGqB,qBAAqB0C,KAAK,EAAEnC,OAAOmC,KAAK,IAGvFN,KAAK,CAAC;IAET,OAAOf,QAAQsE;AACjB;AAEA,OAAO,eAAeiC,gBAAgBrH,MAIrC;IAcC,MAAMqD,SAAS,MAAMjE,KAAKoC,WAAW,CAAC,OAAO5B;QAC3C,MAAM0H,UAAU,MAAM1H,GACnBiF,MAAM,CAACpF,sBACPqF,GAAG,CAAC;YACHe,kBAAkBjH,GAAG,CAAC,SAAS,EAAEa,qBAAqBoG,gBAAgB,CAAC,QAAQ,CAAC;YAChFC,iBAAiBlH,GAAG,CAAC,KAAK,CAAC;QAC7B,GACC4B,KAAK,CACJzC,IACEK,GAAGqB,qBAAqBI,cAAc,EAAEG,OAAOH,cAAc,GAC7DzB,GAAGqB,qBAAqBsD,eAAe,EAAE/C,OAAO+C,eAAe,IAGlER,SAAS,CAAC;YACTwE,yBAAyBtH,qBAAqBsH,uBAAuB;YACrE9E,eAAexC,qBAAqBwC,aAAa;YACjDC,sBAAsBzC,qBAAqByC,oBAAoB;YAC/DC,OAAO1C,qBAAqB0C,KAAK;YACjCtC,gBAAgBJ,qBAAqBI,cAAc;YACnDuC,WAAW3C,qBAAqB2C,SAAS;YACzC9B,aAAab,qBAAqBa,WAAW;YAC7CyC,iBAAiBtD,qBAAqBsD,eAAe;QACvD;QAEF,MAAMjC,MAAMwG,OAAO,CAAC,EAAE;QACtB,IAAI,CAACxG,KAAK,MAAM,IAAI/B,iCAAiCiB,OAAOH,cAAc;QAE1E,MAAM,CAAC0H,SAAS,GAAG,MAAM3H,GACtBS,MAAM,CAAC;YAACmH,kBAAkBhI,eAAegI,gBAAgB;QAAA,GACzDjH,IAAI,CAACf,gBACLgB,KAAK,CAACpC,GAAGoB,eAAekC,EAAE,EAAE1B,OAAO+C,eAAe,GAClDlB,KAAK,CAAC,GACNgC,GAAG,CAAC;QAEP,MAAM,CAACH,QAAQ,GAAG,MAAM9D,GACrBiF,MAAM,CAACrF,gBACPsF,GAAG,CAAC;YACH0C,kBAAkBxH,OAAOwH,gBAAgB,IAAI;YAC7CC,4BAA4BzH,OAAOwH,gBAAgB,GAAG5I,GAAG,CAAC,KAAK,CAAC,GAAG;YACnEoG,WAAWpG,GAAG,CAAC,KAAK,CAAC;QACvB,GACC4B,KAAK,CAACpC,GAAGoB,eAAekC,EAAE,EAAE1B,OAAO+C,eAAe,GAClDR,SAAS,CAAC;YAACiF,kBAAkBhI,eAAegI,gBAAgB;QAAA;QAE/D,OAAO;YACL1G;YACA4G,0BAA0BH,UAAUC,oBAAoB;YACxDG,yBAAyBjE,SAAS8D,oBAAoB;QACxD;IACF;IAEA,MAAM1G,MAAMuC,OAAOvC,GAAG;IACtB,IAAI,CAACA,KAAK,MAAM,IAAI/B,iCAAiCiB,OAAOH,cAAc;IAC1E,OAAO;QACL+H,uBAAuB9G,IAAIiG,uBAAuB,KAAK;QACvDW,0BAA0BrE,OAAOqE,wBAAwB;QACzDC,yBAAyBtE,OAAOsE,uBAAuB;QACvDE,qBAAqB;YACnB5F,eAAenB,IAAImB,aAAa;YAChCC,sBAAsBpB,IAAIoB,oBAAoB;YAC9CC,OAAOrB,IAAIqB,KAAK;YAChBtC,gBAAgBiB,IAAIjB,cAAc;YAClCuC,WAAWtB,IAAIsB,SAAS;YACxB9B,aAAaQ,IAAIR,WAAW;YAC5ByC,iBAAiBjC,IAAIiC,eAAe;QACtC;IACF;AACF;AAEA;;;;CAIC,GACD,OAAO,eAAe+E,8BAA8B9H,MAEnD;IACC,MAAMZ,KAAKoC,WAAW,CAAC,OAAO5B;QAC5B,MAAMD,iBAAiBC,IAAII,OAAOH,cAAc;QAEhD,wFAAwF;QACxF,2FAA2F;QAC3F,wFAAwF;QACxF,MAAMD,GACHsE,MAAM,CAAC5E,sBACPkB,KAAK,CAACpC,GAAGkB,qBAAqBO,cAAc,EAAEG,OAAOH,cAAc;QACtE,MAAMD,GACHiF,MAAM,CAACpF,sBACPqF,GAAG,CAAC;YACHiC,yBAAyBnI,GAAG,CAAC,SAAS,EAAEa,qBAAqBsH,uBAAuB,CAAC,QAAQ,CAAC;QAChG,GACCvG,KAAK,CAACpC,GAAGqB,qBAAqBI,cAAc,EAAEG,OAAOH,cAAc;IACxE;AACF;AAEA,OAAO,eAAekI,iBAAiB/H,MAA0B;IAC/D,IAAIA,OAAOgI,MAAM,CAAC1G,MAAM,KAAK,GAAG;IAEhC,MAAMlC,KAAKoC,WAAW,CAAC,OAAO5B;QAC5B,MAAMA,GAAGsE,MAAM,CAAC5E,sBAAsBkB,KAAK,CAAClC,QAAQgB,qBAAqB6C,KAAK,EAAEnC,OAAOgI,MAAM;QAC7F,MAAMpI,GACHiF,MAAM,CAACpF,sBACPqF,GAAG,CAAC;YACHiC,yBAAyBnI,GAAG,CAAC,SAAS,EAAEa,qBAAqBsH,uBAAuB,CAAC,QAAQ,CAAC;QAChG,GACCvG,KAAK,CAAClC,QAAQmB,qBAAqB0C,KAAK,EAAEnC,OAAOgI,MAAM;IAC5D;AACF"}

package/dist/db/runner-assignments.d.ts

@@ -1,3 +1,4 @@
+import { type ProviderRunnerAssignmentObservation, type RunnerAssignmentSurface } from '#metrics/instance.js';
 import type { Tx } from './db.js';
 /** Atomically consumes reservation units by writing the immutable assignment on each runner. */
 export declare function assignRunnerInstances(params: {
@@ -5,11 +6,16 @@ export declare function assignRunnerInstances(params: {
     reservationId: string;
     runnerInstanceIds: string[];
 }): Promise<string[]>;
+interface AssignRunnerInstancesTxResult {
+    runnerInstanceIds: string[];
+    controlSessionToAssignment: ProviderRunnerAssignmentObservation[];
+}
 export declare function assignRunnerInstancesTx(tx: Tx, params: {
     provisionerId: string;
     reservationId: string;
     runnerInstanceIds: string[];
-}): Promise<string[]>;
+    surface: RunnerAssignmentSurface;
+}): Promise<AssignRunnerInstancesTxResult>;
 export type RunnerReservationCapacityFailureReason = 'reservation-not-found' | 'reservation-kind-mismatch' | 'reservation-expired' | 'capacity-exhausted';
 export interface RunnerReservationCapacityValidation {
     acceptedByReservation: Map<string, number>;
@@ -27,4 +33,5 @@ export declare function validateRunnerReservationCapacityTx(tx: Tx, params: {
 }, options?: {
     advisoryLocksHeld?: boolean;
 }): Promise<RunnerReservationCapacityValidation>;
+export {};
 //# sourceMappingURL=runner-assignments.d.ts.map

package/dist/db/runner-assignments.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"runner-assignments.d.ts","sourceRoot":"","sources":["../../src/db/runner-assignments.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAC,EAAE,EAAC,MAAM,SAAS,CAAC;AAQhC,gGAAgG;AAChG,wBAAsB,qBAAqB,CAAC,MAAM,EAAE;IAClD,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,MAAM,EAAE,CAAC;CAC7B,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAEpB;AAED,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,EAAE,EACN,MAAM,EAAE;IACN,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,MAAM,EAAE,CAAC;CAC7B,GACA,OAAO,CAAC,MAAM,EAAE,CAAC,CAkJnB;AAED,MAAM,MAAM,sCAAsC,GAC9C,uBAAuB,GACvB,2BAA2B,GAC3B,qBAAqB,GACrB,oBAAoB,CAAC;AAEzB,MAAM,WAAW,mCAAmC;IAClD,qBAAqB,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC3C,wBAAwB,EAAE,GAAG,CAC3B,MAAM,EACN;QAAC,MAAM,EAAE,sCAAsC,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAC,CAChE,CAAC;CACH;AAED,wBAAsB,mCAAmC,CACvD,EAAE,EAAE,EAAE,EACN,MAAM,EAAE;IACN,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,SAAS;QAAC,aAAa,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAC,EAAE,CAAC;CAC7D,EACD,OAAO,GAAE;IAAC,iBAAiB,CAAC,EAAE,OAAO,CAAA;CAAM,GAC1C,OAAO,CAAC,mCAAmC,CAAC,CAmI9C"}
+{"version":3,"file":"runner-assignments.d.ts","sourceRoot":"","sources":["../../src/db/runner-assignments.ts"],"names":[],"mappings":"AAOA,OAAO,EACL,KAAK,mCAAmC,EACxC,KAAK,uBAAuB,EAE7B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,KAAK,EAAC,EAAE,EAAC,MAAM,SAAS,CAAC;AAQhC,gGAAgG;AAChG,wBAAsB,qBAAqB,CAAC,MAAM,EAAE;IAClD,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,MAAM,EAAE,CAAC;CAC7B,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAOpB;AAED,UAAU,6BAA6B;IACrC,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,0BAA0B,EAAE,mCAAmC,EAAE,CAAC;CACnE;AAED,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,EAAE,EACN,MAAM,EAAE;IACN,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,OAAO,EAAE,uBAAuB,CAAC;CAClC,GACA,OAAO,CAAC,6BAA6B,CAAC,CAqLxC;AAED,MAAM,MAAM,sCAAsC,GAC9C,uBAAuB,GACvB,2BAA2B,GAC3B,qBAAqB,GACrB,oBAAoB,CAAC;AAEzB,MAAM,WAAW,mCAAmC;IAClD,qBAAqB,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC3C,wBAAwB,EAAE,GAAG,CAC3B,MAAM,EACN;QAAC,MAAM,EAAE,sCAAsC,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAC,CAChE,CAAC;CACH;AAED,wBAAsB,mCAAmC,CACvD,EAAE,EAAE,EAAE,EACN,MAAM,EAAE;IACN,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,SAAS;QAAC,aAAa,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAC,EAAE,CAAC;CAC7D,EACD,OAAO,GAAE;IAAC,iBAAiB,CAAC,EAAE,OAAO,CAAA;CAAM,GAC1C,OAAO,CAAC,mCAAmC,CAAC,CAmI9C"}

package/dist/db/runner-assignments.js

@@ -1,5 +1,6 @@
 import { and, eq, inArray, isNull, ne, notInArray, or, sql } from 'drizzle-orm';
 import { ReservationExpiredError, ReservationNotFoundError, RunnerInstanceAlreadyAssignedError, RunnerInstanceNotAssignableError } from '#core/errors.js';
+import { recordProviderRunnerControlSessionToAssignment } from '#metrics/instance.js';
 import { db } from './db.js';
 import { lockRunnerReservationAdvisoryKeysTx } from './reservation-locks.js';
 import { terminalStates } from './runner-states.js';
@@ -7,7 +8,12 @@ import { reservations } from './schema/reservations.js';
 import { runnerControlSessions } from './schema/runner-control-sessions.js';
 import { providerRunners } from './schema/runner-instances.js';
 /** Atomically consumes reservation units by writing the immutable assignment on each runner. */ export async function assignRunnerInstances(params) {
-    return await db().transaction(async (tx)=>assignRunnerInstancesTx(tx, params));
+    const result = await db().transaction(async (tx)=>assignRunnerInstancesTx(tx, {
+            ...params,
+            surface: 'provisioner'
+        }));
+    for (const observation of result.controlSessionToAssignment)recordProviderRunnerControlSessionToAssignment(observation);
+    return result.runnerInstanceIds;
 }
 export async function assignRunnerInstancesTx(tx, params) {
     const runnerInstanceIds = [
@@ -26,6 +32,8 @@ export async function assignRunnerInstancesTx(tx, params) {
         workspaceId: providerRunners.workspaceId,
         assignedAt: providerRunners.assignedAt,
         providerRunnerId: providerRunners.providerRunnerId,
+        provider: providerRunners.providerKind,
+        launchKind: providerRunners.launchKind,
         labels: providerRunners.labels,
         state: providerRunners.state
     }).from(providerRunners).where(and(eq(providerRunners.provisionerId, params.provisionerId), inArray(providerRunners.id, runnerInstanceIds))).for('update');
@@ -36,19 +44,28 @@ export async function assignRunnerInstancesTx(tx, params) {
             intendedReservationId: null,
             updatedAt: sql`now()`
         }).where(and(eq(providerRunners.provisionerId, params.provisionerId), inArray(providerRunners.id, runnerInstanceIds)));
-        return params.runnerInstanceIds;
+        return {
+            runnerInstanceIds: params.runnerInstanceIds,
+            controlSessionToAssignment: []
+        };
     }
     const [reservation] = await tx.select().from(reservations).where(and(eq(reservations.id, params.reservationId), eq(reservations.provisionerId, params.provisionerId), eq(reservations.kind, 'launch'))).limit(1).for('update');
     if (!reservation) throw new ReservationNotFoundError(params.reservationId);
     if (reservation.expiresAt <= new Date()) throw new ReservationExpiredError(params.reservationId);
-    if (runnerRows.length !== runnerInstanceIds.length) throw new RunnerInstanceNotAssignableError(runnerInstanceIds[0] ?? '');
+    if (runnerRows.length !== runnerInstanceIds.length) throw new RunnerInstanceNotAssignableError(runnerInstanceIds[0] ?? '', 'runner-not-found');
     const activeControlSessions = await tx.select({
-        runnerInstanceId: runnerControlSessions.runnerInstanceId
+        runnerInstanceId: runnerControlSessions.runnerInstanceId,
+        createdAt: runnerControlSessions.createdAt
     }).from(runnerControlSessions).where(and(inArray(runnerControlSessions.runnerInstanceId, runnerRows.map((runner)=>runner.id)), isNull(runnerControlSessions.closedAt)));
     const runnerInstanceIdsWithControlSession = new Set(activeControlSessions.map((session)=>session.runnerInstanceId));
+    const controlSessionCreatedAtByRunner = new Map(activeControlSessions.map((session)=>[
+            session.runnerInstanceId,
+            session.createdAt
+        ]));
     const runners = runnerRows.map((runner)=>({
             ...runner,
-            controlSessionId: runnerInstanceIdsWithControlSession.has(runner.id) ? runner.id : null
+            controlSessionId: runnerInstanceIdsWithControlSession.has(runner.id) ? runner.id : null,
+            controlSessionCreatedAt: controlSessionCreatedAtByRunner.get(runner.id) ?? null
         }));
     const alreadyAssigned = runners.filter((runner)=>runner.reservationId !== null);
     if (alreadyAssigned.some((runner)=>runner.reservationId !== reservation.id)) throw new RunnerInstanceAlreadyAssignedError(alreadyAssigned[0]?.id ?? '');
@@ -59,20 +76,50 @@ export async function assignRunnerInstancesTx(tx, params) {
     }).from(providerRunners).where(and(eq(providerRunners.provisionerId, params.provisionerId), notInArray(providerRunners.id, newRunnerIds), or(and(eq(providerRunners.reservationId, reservation.id), isNull(providerRunners.reservationReleasedAt)), and(eq(providerRunners.intendedReservationId, reservation.id), isNull(providerRunners.reservationReleasedAt), notInArray(providerRunners.state, [
         ...terminalStates
     ])))));
-    if ((assignedCount[0]?.count ?? 0) + newRunners.length > reservation.count) throw new RunnerInstanceNotAssignableError(newRunners[0]?.id ?? '');
+    if ((assignedCount[0]?.count ?? 0) + newRunners.length > reservation.count) throw new RunnerInstanceNotAssignableError(newRunners[0]?.id ?? '', 'capacity-exhausted');
     for (const runner of newRunners){
-        if (runner.state !== 'running' || !runner.providerRunnerId || !runner.controlSessionId || !reservation.requiredLabels.every((label)=>runner.labels.includes(label))) throw new RunnerInstanceNotAssignableError(runner.id);
+        if (runner.state !== 'running') throw new RunnerInstanceNotAssignableError(runner.id, 'runner-not-running');
+        if (!runner.providerRunnerId) throw new RunnerInstanceNotAssignableError(runner.id, 'provider-identity-missing');
+        if (!runner.controlSessionId) throw new RunnerInstanceNotAssignableError(runner.id, 'control-session-not-active');
+        if (!reservation.requiredLabels.every((label)=>runner.labels.includes(label))) throw new RunnerInstanceNotAssignableError(runner.id, 'labels-mismatch');
     }
     if (newRunners.length > 0) {
-        await tx.update(providerRunners).set({
+        const assignedRows = await tx.update(providerRunners).set({
             workspaceId: reservation.workspaceId,
             reservationId: reservation.id,
             intendedReservationId: null,
             assignedAt: sql`now()`,
             updatedAt: sql`now()`
-        }).where(inArray(providerRunners.id, newRunners.map((runner)=>runner.id)));
+        }).where(inArray(providerRunners.id, newRunners.map((runner)=>runner.id))).returning({
+            id: providerRunners.id,
+            assignedAt: providerRunners.assignedAt
+        });
+        const runnersById = new Map(newRunners.map((runner)=>[
+                runner.id,
+                runner
+            ]));
+        const controlSessionToAssignment = [];
+        for (const assigned of assignedRows){
+            const runner = runnersById.get(assigned.id);
+            const controlSessionCreatedAt = runner?.controlSessionCreatedAt;
+            if (!runner || !assigned.assignedAt || !controlSessionCreatedAt) continue;
+            controlSessionToAssignment.push({
+                durationMilliseconds: assigned.assignedAt.getTime() - controlSessionCreatedAt.getTime(),
+                provider: runner.provider,
+                launchKind: runner.launchKind,
+                surface: params.surface,
+                runnerInstanceId: runner.id
+            });
+        }
+        return {
+            runnerInstanceIds: params.runnerInstanceIds,
+            controlSessionToAssignment
+        };
     }
-    return params.runnerInstanceIds;
+    return {
+        runnerInstanceIds: params.runnerInstanceIds,
+        controlSessionToAssignment: []
+    };
 }
 export async function validateRunnerReservationCapacityTx(tx, params, options = {}) {
     const requestedByReservation = new Map();

package/dist/db/runner-assignments.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/db/runner-assignments.ts"],"sourcesContent":["import {and, eq, inArray, isNull, ne, notInArray, or, sql} from 'drizzle-orm';\nimport {\n  ReservationExpiredError,\n  ReservationNotFoundError,\n  RunnerInstanceAlreadyAssignedError,\n  RunnerInstanceNotAssignableError,\n} from '#core/errors.js';\nimport type {Tx} from './db.js';\nimport {db} from './db.js';\nimport {lockRunnerReservationAdvisoryKeysTx} from './reservation-locks.js';\nimport {terminalStates} from './runner-states.js';\nimport {reservations} from './schema/reservations.js';\nimport {runnerControlSessions} from './schema/runner-control-sessions.js';\nimport {providerRunners} from './schema/runner-instances.js';\n\n/** Atomically consumes reservation units by writing the immutable assignment on each runner. */\nexport async function assignRunnerInstances(params: {\n  provisionerId: string;\n  reservationId: string;\n  runnerInstanceIds: string[];\n}): Promise<string[]> {\n  return await db().transaction(async (tx) => assignRunnerInstancesTx(tx, params));\n}\n\nexport async function assignRunnerInstancesTx(\n  tx: Tx,\n  params: {\n    provisionerId: string;\n    reservationId: string;\n    runnerInstanceIds: string[];\n  },\n): Promise<string[]> {\n  const runnerInstanceIds = [...params.runnerInstanceIds].sort();\n  await lockRunnerReservationAdvisoryKeysTx(tx, {\n    provisionerId: params.provisionerId,\n    reservationIds: [params.reservationId],\n  });\n  const runnerRows = await tx\n    .select({\n      id: providerRunners.id,\n      reservationId: providerRunners.reservationId,\n      intendedReservationId: providerRunners.intendedReservationId,\n      workspaceId: providerRunners.workspaceId,\n      assignedAt: providerRunners.assignedAt,\n      providerRunnerId: providerRunners.providerRunnerId,\n      labels: providerRunners.labels,\n      state: providerRunners.state,\n    })\n    .from(providerRunners)\n    .where(\n      and(\n        eq(providerRunners.provisionerId, params.provisionerId),\n        inArray(providerRunners.id, runnerInstanceIds),\n      ),\n    )\n    .for('update');\n\n  // The assignment outlives its short reservation row. A retry after maintenance\n  // cleanup is complete when every owned runner already carries the committed assignment.\n  if (\n    runnerInstanceIds.length > 0 &&\n    runnerRows.length === runnerInstanceIds.length &&\n    runnerRows.every(\n      (runner) =>\n        runner.reservationId === params.reservationId &&\n        runner.assignedAt !== null &&\n        runner.workspaceId !== null,\n    )\n  ) {\n    await tx\n      .update(providerRunners)\n      .set({intendedReservationId: null, updatedAt: sql`now()`})\n      .where(\n        and(\n          eq(providerRunners.provisionerId, params.provisionerId),\n          inArray(providerRunners.id, runnerInstanceIds),\n        ),\n      );\n    return params.runnerInstanceIds;\n  }\n\n  const [reservation] = await tx\n    .select()\n    .from(reservations)\n    .where(\n      and(\n        eq(reservations.id, params.reservationId),\n        eq(reservations.provisionerId, params.provisionerId),\n        eq(reservations.kind, 'launch'),\n      ),\n    )\n    .limit(1)\n    .for('update');\n  if (!reservation) throw new ReservationNotFoundError(params.reservationId);\n  if (reservation.expiresAt <= new Date()) throw new ReservationExpiredError(params.reservationId);\n  if (runnerRows.length !== runnerInstanceIds.length)\n    throw new RunnerInstanceNotAssignableError(runnerInstanceIds[0] ?? '');\n\n  const activeControlSessions = await tx\n    .select({runnerInstanceId: runnerControlSessions.runnerInstanceId})\n    .from(runnerControlSessions)\n    .where(\n      and(\n        inArray(\n          runnerControlSessions.runnerInstanceId,\n          runnerRows.map((runner) => runner.id),\n        ),\n        isNull(runnerControlSessions.closedAt),\n      ),\n    );\n  const runnerInstanceIdsWithControlSession = new Set(\n    activeControlSessions.map((session) => session.runnerInstanceId),\n  );\n  const runners = runnerRows.map((runner) => ({\n    ...runner,\n    controlSessionId: runnerInstanceIdsWithControlSession.has(runner.id) ? runner.id : null,\n  }));\n\n  const alreadyAssigned = runners.filter((runner) => runner.reservationId !== null);\n  if (alreadyAssigned.some((runner) => runner.reservationId !== reservation.id))\n    throw new RunnerInstanceAlreadyAssignedError(alreadyAssigned[0]?.id ?? '');\n  const newRunners = runners.filter(\n    (runner) =>\n      runner.reservationId === null ||\n      (runner.reservationId === reservation.id &&\n        (runner.assignedAt === null || runner.workspaceId === null)),\n  );\n  const newRunnerIds = newRunners.map((runner) => runner.id);\n  const assignedCount = await tx\n    .select({count: sql<number>`count(*)::int`})\n    .from(providerRunners)\n    .where(\n      and(\n        eq(providerRunners.provisionerId, params.provisionerId),\n        notInArray(providerRunners.id, newRunnerIds),\n        or(\n          and(\n            eq(providerRunners.reservationId, reservation.id),\n            isNull(providerRunners.reservationReleasedAt),\n          ),\n          and(\n            eq(providerRunners.intendedReservationId, reservation.id),\n            isNull(providerRunners.reservationReleasedAt),\n            notInArray(providerRunners.state, [...terminalStates]),\n          ),\n        ),\n      ),\n    );\n  if ((assignedCount[0]?.count ?? 0) + newRunners.length > reservation.count)\n    throw new RunnerInstanceNotAssignableError(newRunners[0]?.id ?? '');\n  for (const runner of newRunners) {\n    if (\n      runner.state !== 'running' ||\n      !runner.providerRunnerId ||\n      !runner.controlSessionId ||\n      !reservation.requiredLabels.every((label) => runner.labels.includes(label))\n    )\n      throw new RunnerInstanceNotAssignableError(runner.id);\n  }\n  if (newRunners.length > 0) {\n    await tx\n      .update(providerRunners)\n      .set({\n        workspaceId: reservation.workspaceId,\n        reservationId: reservation.id,\n        intendedReservationId: null,\n        assignedAt: sql`now()`,\n        updatedAt: sql`now()`,\n      })\n      .where(\n        inArray(\n          providerRunners.id,\n          newRunners.map((runner) => runner.id),\n        ),\n      );\n  }\n  return params.runnerInstanceIds;\n}\n\nexport type RunnerReservationCapacityFailureReason =\n  | 'reservation-not-found'\n  | 'reservation-kind-mismatch'\n  | 'reservation-expired'\n  | 'capacity-exhausted';\n\nexport interface RunnerReservationCapacityValidation {\n  acceptedByReservation: Map<string, number>;\n  unavailableByReservation: Map<\n    string,\n    {reason: RunnerReservationCapacityFailureReason; count: number}\n  >;\n}\n\nexport async function validateRunnerReservationCapacityTx(\n  tx: Tx,\n  params: {\n    provisionerId: string;\n    requests: readonly {reservationId: string; count: number}[];\n  },\n  options: {advisoryLocksHeld?: boolean} = {},\n): Promise<RunnerReservationCapacityValidation> {\n  const requestedByReservation = new Map<string, number>();\n  for (const request of params.requests) {\n    if (request.count <= 0) continue;\n    requestedByReservation.set(\n      request.reservationId,\n      (requestedByReservation.get(request.reservationId) ?? 0) + request.count,\n    );\n  }\n  const reservationIds = [...requestedByReservation.keys()].sort();\n  if (reservationIds.length === 0)\n    return {acceptedByReservation: new Map(), unavailableByReservation: new Map()};\n\n  if (!options.advisoryLocksHeld)\n    await lockRunnerReservationAdvisoryKeysTx(tx, {\n      provisionerId: params.provisionerId,\n      reservationIds,\n    });\n\n  const reservationRows = await tx\n    .select({\n      id: reservations.id,\n      count: reservations.count,\n      expiresAt: reservations.expiresAt,\n      isExpired: sql<boolean>`${reservations.expiresAt} <= now()`,\n    })\n    .from(reservations)\n    .where(\n      and(\n        eq(reservations.provisionerId, params.provisionerId),\n        inArray(reservations.id, reservationIds),\n        eq(reservations.kind, 'launch'),\n      ),\n    )\n    .for('update');\n  const nonLaunchReservationRows = await tx\n    .select({id: reservations.id})\n    .from(reservations)\n    .where(\n      and(\n        eq(reservations.provisionerId, params.provisionerId),\n        inArray(reservations.id, reservationIds),\n        ne(reservations.kind, 'launch'),\n      ),\n    )\n    .for('update');\n  const reservationsById = new Map(\n    reservationRows.map((reservation) => [reservation.id, reservation]),\n  );\n  const nonLaunchReservationIds = new Set(\n    nonLaunchReservationRows.map((reservation) => reservation.id),\n  );\n  const requestedReservationIds = new Set(reservationIds);\n  const usedRunnerIdsByReservation = new Map<string, Set<string>>();\n  const usedRunnerRows = await tx\n    .select({\n      id: providerRunners.id,\n      reservationId: providerRunners.reservationId,\n      intendedReservationId: providerRunners.intendedReservationId,\n      state: providerRunners.state,\n    })\n    .from(providerRunners)\n    .where(\n      and(\n        eq(providerRunners.provisionerId, params.provisionerId),\n        isNull(providerRunners.reservationReleasedAt),\n        or(\n          inArray(providerRunners.reservationId, reservationIds),\n          and(\n            inArray(providerRunners.intendedReservationId, reservationIds),\n            notInArray(providerRunners.state, [...terminalStates]),\n          ),\n        ),\n      ),\n    );\n  for (const runner of usedRunnerRows) {\n    if (runner.reservationId && requestedReservationIds.has(runner.reservationId)) {\n      const usedRunnerIds =\n        usedRunnerIdsByReservation.get(runner.reservationId) ?? new Set<string>();\n      usedRunnerIds.add(runner.id);\n      usedRunnerIdsByReservation.set(runner.reservationId, usedRunnerIds);\n    }\n    if (\n      runner.intendedReservationId &&\n      requestedReservationIds.has(runner.intendedReservationId) &&\n      !terminalStates.includes(runner.state as (typeof terminalStates)[number])\n    ) {\n      const usedRunnerIds =\n        usedRunnerIdsByReservation.get(runner.intendedReservationId) ?? new Set<string>();\n      usedRunnerIds.add(runner.id);\n      usedRunnerIdsByReservation.set(runner.intendedReservationId, usedRunnerIds);\n    }\n  }\n\n  const acceptedByReservation = new Map<string, number>();\n  const unavailableByReservation = new Map<\n    string,\n    {reason: RunnerReservationCapacityFailureReason; count: number}\n  >();\n  for (const reservationId of reservationIds) {\n    const requested = requestedByReservation.get(reservationId) ?? 0;\n    const reservation = reservationsById.get(reservationId);\n    if (!reservation) {\n      unavailableByReservation.set(reservationId, {\n        reason: nonLaunchReservationIds.has(reservationId)\n          ? 'reservation-kind-mismatch'\n          : 'reservation-not-found',\n        count: requested,\n      });\n      continue;\n    }\n    if (reservation.isExpired) {\n      unavailableByReservation.set(reservationId, {\n        reason: 'reservation-expired',\n        count: requested,\n      });\n      continue;\n    }\n\n    const used = usedRunnerIdsByReservation.get(reservationId)?.size ?? 0;\n    const accepted = Math.min(requested, Math.max(0, reservation.count - used));\n    acceptedByReservation.set(reservationId, accepted);\n    if (accepted < requested) {\n      unavailableByReservation.set(reservationId, {\n        reason: 'capacity-exhausted',\n        count: requested - accepted,\n      });\n    }\n  }\n\n  return {acceptedByReservation, unavailableByReservation};\n}\n"],"names":["and","eq","inArray","isNull","ne","notInArray","or","sql","ReservationExpiredError","ReservationNotFoundError","RunnerInstanceAlreadyAssignedError","RunnerInstanceNotAssignableError","db","lockRunnerReservationAdvisoryKeysTx","terminalStates","reservations","runnerControlSessions","providerRunners","assignRunnerInstances","params","transaction","tx","assignRunnerInstancesTx","runnerInstanceIds","sort","provisionerId","reservationIds","reservationId","runnerRows","select","id","intendedReservationId","workspaceId","assignedAt","providerRunnerId","labels","state","from","where","for","length","every","runner","update","set","updatedAt","reservation","kind","limit","expiresAt","Date","activeControlSessions","runnerInstanceId","map","closedAt","runnerInstanceIdsWithControlSession","Set","session","runners","controlSessionId","has","alreadyAssigned","filter","some","newRunners","newRunnerIds","assignedCount","count","reservationReleasedAt","requiredLabels","label","includes","validateRunnerReservationCapacityTx","options","requestedByReservation","Map","request","requests","get","keys","acceptedByReservation","unavailableByReservation","advisoryLocksHeld","reservationRows","isExpired","nonLaunchReservationRows","reservationsById","nonLaunchReservationIds","requestedReservationIds","usedRunnerIdsByReservation","usedRunnerRows","usedRunnerIds","add","requested","reason","used","size","accepted","Math","min","max"],"mappings":"AAAA,SAAQA,GAAG,EAAEC,EAAE,EAAEC,OAAO,EAAEC,MAAM,EAAEC,EAAE,EAAEC,UAAU,EAAEC,EAAE,EAAEC,GAAG,QAAO,cAAc;AAC9E,SACEC,uBAAuB,EACvBC,wBAAwB,EACxBC,kCAAkC,EAClCC,gCAAgC,QAC3B,kBAAkB;AAEzB,SAAQC,EAAE,QAAO,UAAU;AAC3B,SAAQC,mCAAmC,QAAO,yBAAyB;AAC3E,SAAQC,cAAc,QAAO,qBAAqB;AAClD,SAAQC,YAAY,QAAO,2BAA2B;AACtD,SAAQC,qBAAqB,QAAO,sCAAsC;AAC1E,SAAQC,eAAe,QAAO,+BAA+B;AAE7D,8FAA8F,GAC9F,OAAO,eAAeC,sBAAsBC,MAI3C;IACC,OAAO,MAAMP,KAAKQ,WAAW,CAAC,OAAOC,KAAOC,wBAAwBD,IAAIF;AAC1E;AAEA,OAAO,eAAeG,wBACpBD,EAAM,EACNF,MAIC;IAED,MAAMI,oBAAoB;WAAIJ,OAAOI,iBAAiB;KAAC,CAACC,IAAI;IAC5D,MAAMX,oCAAoCQ,IAAI;QAC5CI,eAAeN,OAAOM,aAAa;QACnCC,gBAAgB;YAACP,OAAOQ,aAAa;SAAC;IACxC;IACA,MAAMC,aAAa,MAAMP,GACtBQ,MAAM,CAAC;QACNC,IAAIb,gBAAgBa,EAAE;QACtBH,eAAeV,gBAAgBU,aAAa;QAC5CI,uBAAuBd,gBAAgBc,qBAAqB;QAC5DC,aAAaf,gBAAgBe,WAAW;QACxCC,YAAYhB,gBAAgBgB,UAAU;QACtCC,kBAAkBjB,gBAAgBiB,gBAAgB;QAClDC,QAAQlB,gBAAgBkB,MAAM;QAC9BC,OAAOnB,gBAAgBmB,KAAK;IAC9B,GACCC,IAAI,CAACpB,iBACLqB,KAAK,CACJtC,IACEC,GAAGgB,gBAAgBQ,aAAa,EAAEN,OAAOM,aAAa,GACtDvB,QAAQe,gBAAgBa,EAAE,EAAEP,qBAG/BgB,GAAG,CAAC;IAEP,+EAA+E;IAC/E,wFAAwF;IACxF,IACEhB,kBAAkBiB,MAAM,GAAG,KAC3BZ,WAAWY,MAAM,KAAKjB,kBAAkBiB,MAAM,IAC9CZ,WAAWa,KAAK,CACd,CAACC,SACCA,OAAOf,aAAa,KAAKR,OAAOQ,aAAa,IAC7Ce,OAAOT,UAAU,KAAK,QACtBS,OAAOV,WAAW,KAAK,OAE3B;QACA,MAAMX,GACHsB,MAAM,CAAC1B,iBACP2B,GAAG,CAAC;YAACb,uBAAuB;YAAMc,WAAWtC,GAAG,CAAC,KAAK,CAAC;QAAA,GACvD+B,KAAK,CACJtC,IACEC,GAAGgB,gBAAgBQ,aAAa,EAAEN,OAAOM,aAAa,GACtDvB,QAAQe,gBAAgBa,EAAE,EAAEP;QAGlC,OAAOJ,OAAOI,iBAAiB;IACjC;IAEA,MAAM,CAACuB,YAAY,GAAG,MAAMzB,GACzBQ,MAAM,GACNQ,IAAI,CAACtB,cACLuB,KAAK,CACJtC,IACEC,GAAGc,aAAae,EAAE,EAAEX,OAAOQ,aAAa,GACxC1B,GAAGc,aAAaU,aAAa,EAAEN,OAAOM,aAAa,GACnDxB,GAAGc,aAAagC,IAAI,EAAE,YAGzBC,KAAK,CAAC,GACNT,GAAG,CAAC;IACP,IAAI,CAACO,aAAa,MAAM,IAAIrC,yBAAyBU,OAAOQ,aAAa;IACzE,IAAImB,YAAYG,SAAS,IAAI,IAAIC,QAAQ,MAAM,IAAI1C,wBAAwBW,OAAOQ,aAAa;IAC/F,IAAIC,WAAWY,MAAM,KAAKjB,kBAAkBiB,MAAM,EAChD,MAAM,IAAI7B,iCAAiCY,iBAAiB,CAAC,EAAE,IAAI;IAErE,MAAM4B,wBAAwB,MAAM9B,GACjCQ,MAAM,CAAC;QAACuB,kBAAkBpC,sBAAsBoC,gBAAgB;IAAA,GAChEf,IAAI,CAACrB,uBACLsB,KAAK,CACJtC,IACEE,QACEc,sBAAsBoC,gBAAgB,EACtCxB,WAAWyB,GAAG,CAAC,CAACX,SAAWA,OAAOZ,EAAE,IAEtC3B,OAAOa,sBAAsBsC,QAAQ;IAG3C,MAAMC,sCAAsC,IAAIC,IAC9CL,sBAAsBE,GAAG,CAAC,CAACI,UAAYA,QAAQL,gBAAgB;IAEjE,MAAMM,UAAU9B,WAAWyB,GAAG,CAAC,CAACX,SAAY,CAAA;YAC1C,GAAGA,MAAM;YACTiB,kBAAkBJ,oCAAoCK,GAAG,CAAClB,OAAOZ,EAAE,IAAIY,OAAOZ,EAAE,GAAG;QACrF,CAAA;IAEA,MAAM+B,kBAAkBH,QAAQI,MAAM,CAAC,CAACpB,SAAWA,OAAOf,aAAa,KAAK;IAC5E,IAAIkC,gBAAgBE,IAAI,CAAC,CAACrB,SAAWA,OAAOf,aAAa,KAAKmB,YAAYhB,EAAE,GAC1E,MAAM,IAAIpB,mCAAmCmD,eAAe,CAAC,EAAE,EAAE/B,MAAM;IACzE,MAAMkC,aAAaN,QAAQI,MAAM,CAC/B,CAACpB,SACCA,OAAOf,aAAa,KAAK,QACxBe,OAAOf,aAAa,KAAKmB,YAAYhB,EAAE,IACrCY,CAAAA,OAAOT,UAAU,KAAK,QAAQS,OAAOV,WAAW,KAAK,IAAG;IAE/D,MAAMiC,eAAeD,WAAWX,GAAG,CAAC,CAACX,SAAWA,OAAOZ,EAAE;IACzD,MAAMoC,gBAAgB,MAAM7C,GACzBQ,MAAM,CAAC;QAACsC,OAAO5D,GAAW,CAAC,aAAa,CAAC;IAAA,GACzC8B,IAAI,CAACpB,iBACLqB,KAAK,CACJtC,IACEC,GAAGgB,gBAAgBQ,aAAa,EAAEN,OAAOM,aAAa,GACtDpB,WAAWY,gBAAgBa,EAAE,EAAEmC,eAC/B3D,GACEN,IACEC,GAAGgB,gBAAgBU,aAAa,EAAEmB,YAAYhB,EAAE,GAChD3B,OAAOc,gBAAgBmD,qBAAqB,IAE9CpE,IACEC,GAAGgB,gBAAgBc,qBAAqB,EAAEe,YAAYhB,EAAE,GACxD3B,OAAOc,gBAAgBmD,qBAAqB,GAC5C/D,WAAWY,gBAAgBmB,KAAK,EAAE;WAAItB;KAAe;IAK/D,IAAI,AAACoD,CAAAA,aAAa,CAAC,EAAE,EAAEC,SAAS,CAAA,IAAKH,WAAWxB,MAAM,GAAGM,YAAYqB,KAAK,EACxE,MAAM,IAAIxD,iCAAiCqD,UAAU,CAAC,EAAE,EAAElC,MAAM;IAClE,KAAK,MAAMY,UAAUsB,WAAY;QAC/B,IACEtB,OAAON,KAAK,KAAK,aACjB,CAACM,OAAOR,gBAAgB,IACxB,CAACQ,OAAOiB,gBAAgB,IACxB,CAACb,YAAYuB,cAAc,CAAC5B,KAAK,CAAC,CAAC6B,QAAU5B,OAAOP,MAAM,CAACoC,QAAQ,CAACD,SAEpE,MAAM,IAAI3D,iCAAiC+B,OAAOZ,EAAE;IACxD;IACA,IAAIkC,WAAWxB,MAAM,GAAG,GAAG;QACzB,MAAMnB,GACHsB,MAAM,CAAC1B,iBACP2B,GAAG,CAAC;YACHZ,aAAac,YAAYd,WAAW;YACpCL,eAAemB,YAAYhB,EAAE;YAC7BC,uBAAuB;YACvBE,YAAY1B,GAAG,CAAC,KAAK,CAAC;YACtBsC,WAAWtC,GAAG,CAAC,KAAK,CAAC;QACvB,GACC+B,KAAK,CACJpC,QACEe,gBAAgBa,EAAE,EAClBkC,WAAWX,GAAG,CAAC,CAACX,SAAWA,OAAOZ,EAAE;IAG5C;IACA,OAAOX,OAAOI,iBAAiB;AACjC;AAgBA,OAAO,eAAeiD,oCACpBnD,EAAM,EACNF,MAGC,EACDsD,UAAyC,CAAC,CAAC;IAE3C,MAAMC,yBAAyB,IAAIC;IACnC,KAAK,MAAMC,WAAWzD,OAAO0D,QAAQ,CAAE;QACrC,IAAID,QAAQT,KAAK,IAAI,GAAG;QACxBO,uBAAuB9B,GAAG,CACxBgC,QAAQjD,aAAa,EACrB,AAAC+C,CAAAA,uBAAuBI,GAAG,CAACF,QAAQjD,aAAa,KAAK,CAAA,IAAKiD,QAAQT,KAAK;IAE5E;IACA,MAAMzC,iBAAiB;WAAIgD,uBAAuBK,IAAI;KAAG,CAACvD,IAAI;IAC9D,IAAIE,eAAec,MAAM,KAAK,GAC5B,OAAO;QAACwC,uBAAuB,IAAIL;QAAOM,0BAA0B,IAAIN;IAAK;IAE/E,IAAI,CAACF,QAAQS,iBAAiB,EAC5B,MAAMrE,oCAAoCQ,IAAI;QAC5CI,eAAeN,OAAOM,aAAa;QACnCC;IACF;IAEF,MAAMyD,kBAAkB,MAAM9D,GAC3BQ,MAAM,CAAC;QACNC,IAAIf,aAAae,EAAE;QACnBqC,OAAOpD,aAAaoD,KAAK;QACzBlB,WAAWlC,aAAakC,SAAS;QACjCmC,WAAW7E,GAAY,CAAC,EAAEQ,aAAakC,SAAS,CAAC,SAAS,CAAC;IAC7D,GACCZ,IAAI,CAACtB,cACLuB,KAAK,CACJtC,IACEC,GAAGc,aAAaU,aAAa,EAAEN,OAAOM,aAAa,GACnDvB,QAAQa,aAAae,EAAE,EAAEJ,iBACzBzB,GAAGc,aAAagC,IAAI,EAAE,YAGzBR,GAAG,CAAC;IACP,MAAM8C,2BAA2B,MAAMhE,GACpCQ,MAAM,CAAC;QAACC,IAAIf,aAAae,EAAE;IAAA,GAC3BO,IAAI,CAACtB,cACLuB,KAAK,CACJtC,IACEC,GAAGc,aAAaU,aAAa,EAAEN,OAAOM,aAAa,GACnDvB,QAAQa,aAAae,EAAE,EAAEJ,iBACzBtB,GAAGW,aAAagC,IAAI,EAAE,YAGzBR,GAAG,CAAC;IACP,MAAM+C,mBAAmB,IAAIX,IAC3BQ,gBAAgB9B,GAAG,CAAC,CAACP,cAAgB;YAACA,YAAYhB,EAAE;YAAEgB;SAAY;IAEpE,MAAMyC,0BAA0B,IAAI/B,IAClC6B,yBAAyBhC,GAAG,CAAC,CAACP,cAAgBA,YAAYhB,EAAE;IAE9D,MAAM0D,0BAA0B,IAAIhC,IAAI9B;IACxC,MAAM+D,6BAA6B,IAAId;IACvC,MAAMe,iBAAiB,MAAMrE,GAC1BQ,MAAM,CAAC;QACNC,IAAIb,gBAAgBa,EAAE;QACtBH,eAAeV,gBAAgBU,aAAa;QAC5CI,uBAAuBd,gBAAgBc,qBAAqB;QAC5DK,OAAOnB,gBAAgBmB,KAAK;IAC9B,GACCC,IAAI,CAACpB,iBACLqB,KAAK,CACJtC,IACEC,GAAGgB,gBAAgBQ,aAAa,EAAEN,OAAOM,aAAa,GACtDtB,OAAOc,gBAAgBmD,qBAAqB,GAC5C9D,GACEJ,QAAQe,gBAAgBU,aAAa,EAAED,iBACvC1B,IACEE,QAAQe,gBAAgBc,qBAAqB,EAAEL,iBAC/CrB,WAAWY,gBAAgBmB,KAAK,EAAE;WAAItB;KAAe;IAK/D,KAAK,MAAM4B,UAAUgD,eAAgB;QACnC,IAAIhD,OAAOf,aAAa,IAAI6D,wBAAwB5B,GAAG,CAAClB,OAAOf,aAAa,GAAG;YAC7E,MAAMgE,gBACJF,2BAA2BX,GAAG,CAACpC,OAAOf,aAAa,KAAK,IAAI6B;YAC9DmC,cAAcC,GAAG,CAAClD,OAAOZ,EAAE;YAC3B2D,2BAA2B7C,GAAG,CAACF,OAAOf,aAAa,EAAEgE;QACvD;QACA,IACEjD,OAAOX,qBAAqB,IAC5ByD,wBAAwB5B,GAAG,CAAClB,OAAOX,qBAAqB,KACxD,CAACjB,eAAeyD,QAAQ,CAAC7B,OAAON,KAAK,GACrC;YACA,MAAMuD,gBACJF,2BAA2BX,GAAG,CAACpC,OAAOX,qBAAqB,KAAK,IAAIyB;YACtEmC,cAAcC,GAAG,CAAClD,OAAOZ,EAAE;YAC3B2D,2BAA2B7C,GAAG,CAACF,OAAOX,qBAAqB,EAAE4D;QAC/D;IACF;IAEA,MAAMX,wBAAwB,IAAIL;IAClC,MAAMM,2BAA2B,IAAIN;IAIrC,KAAK,MAAMhD,iBAAiBD,eAAgB;QAC1C,MAAMmE,YAAYnB,uBAAuBI,GAAG,CAACnD,kBAAkB;QAC/D,MAAMmB,cAAcwC,iBAAiBR,GAAG,CAACnD;QACzC,IAAI,CAACmB,aAAa;YAChBmC,yBAAyBrC,GAAG,CAACjB,eAAe;gBAC1CmE,QAAQP,wBAAwB3B,GAAG,CAACjC,iBAChC,8BACA;gBACJwC,OAAO0B;YACT;YACA;QACF;QACA,IAAI/C,YAAYsC,SAAS,EAAE;YACzBH,yBAAyBrC,GAAG,CAACjB,eAAe;gBAC1CmE,QAAQ;gBACR3B,OAAO0B;YACT;YACA;QACF;QAEA,MAAME,OAAON,2BAA2BX,GAAG,CAACnD,gBAAgBqE,QAAQ;QACpE,MAAMC,WAAWC,KAAKC,GAAG,CAACN,WAAWK,KAAKE,GAAG,CAAC,GAAGtD,YAAYqB,KAAK,GAAG4B;QACrEf,sBAAsBpC,GAAG,CAACjB,eAAesE;QACzC,IAAIA,WAAWJ,WAAW;YACxBZ,yBAAyBrC,GAAG,CAACjB,eAAe;gBAC1CmE,QAAQ;gBACR3B,OAAO0B,YAAYI;YACrB;QACF;IACF;IAEA,OAAO;QAACjB;QAAuBC;IAAwB;AACzD"}
+{"version":3,"sources":["../../src/db/runner-assignments.ts"],"sourcesContent":["import {and, eq, inArray, isNull, ne, notInArray, or, sql} from 'drizzle-orm';\nimport {\n  ReservationExpiredError,\n  ReservationNotFoundError,\n  RunnerInstanceAlreadyAssignedError,\n  RunnerInstanceNotAssignableError,\n} from '#core/errors.js';\nimport {\n  type ProviderRunnerAssignmentObservation,\n  type RunnerAssignmentSurface,\n  recordProviderRunnerControlSessionToAssignment,\n} from '#metrics/instance.js';\nimport type {Tx} from './db.js';\nimport {db} from './db.js';\nimport {lockRunnerReservationAdvisoryKeysTx} from './reservation-locks.js';\nimport {terminalStates} from './runner-states.js';\nimport {reservations} from './schema/reservations.js';\nimport {runnerControlSessions} from './schema/runner-control-sessions.js';\nimport {providerRunners} from './schema/runner-instances.js';\n\n/** Atomically consumes reservation units by writing the immutable assignment on each runner. */\nexport async function assignRunnerInstances(params: {\n  provisionerId: string;\n  reservationId: string;\n  runnerInstanceIds: string[];\n}): Promise<string[]> {\n  const result = await db().transaction(async (tx) =>\n    assignRunnerInstancesTx(tx, {...params, surface: 'provisioner'}),\n  );\n  for (const observation of result.controlSessionToAssignment)\n    recordProviderRunnerControlSessionToAssignment(observation);\n  return result.runnerInstanceIds;\n}\n\ninterface AssignRunnerInstancesTxResult {\n  runnerInstanceIds: string[];\n  controlSessionToAssignment: ProviderRunnerAssignmentObservation[];\n}\n\nexport async function assignRunnerInstancesTx(\n  tx: Tx,\n  params: {\n    provisionerId: string;\n    reservationId: string;\n    runnerInstanceIds: string[];\n    surface: RunnerAssignmentSurface;\n  },\n): Promise<AssignRunnerInstancesTxResult> {\n  const runnerInstanceIds = [...params.runnerInstanceIds].sort();\n  await lockRunnerReservationAdvisoryKeysTx(tx, {\n    provisionerId: params.provisionerId,\n    reservationIds: [params.reservationId],\n  });\n  const runnerRows = await tx\n    .select({\n      id: providerRunners.id,\n      reservationId: providerRunners.reservationId,\n      intendedReservationId: providerRunners.intendedReservationId,\n      workspaceId: providerRunners.workspaceId,\n      assignedAt: providerRunners.assignedAt,\n      providerRunnerId: providerRunners.providerRunnerId,\n      provider: providerRunners.providerKind,\n      launchKind: providerRunners.launchKind,\n      labels: providerRunners.labels,\n      state: providerRunners.state,\n    })\n    .from(providerRunners)\n    .where(\n      and(\n        eq(providerRunners.provisionerId, params.provisionerId),\n        inArray(providerRunners.id, runnerInstanceIds),\n      ),\n    )\n    .for('update');\n\n  // The assignment outlives its short reservation row. A retry after maintenance\n  // cleanup is complete when every owned runner already carries the committed assignment.\n  if (\n    runnerInstanceIds.length > 0 &&\n    runnerRows.length === runnerInstanceIds.length &&\n    runnerRows.every(\n      (runner) =>\n        runner.reservationId === params.reservationId &&\n        runner.assignedAt !== null &&\n        runner.workspaceId !== null,\n    )\n  ) {\n    await tx\n      .update(providerRunners)\n      .set({intendedReservationId: null, updatedAt: sql`now()`})\n      .where(\n        and(\n          eq(providerRunners.provisionerId, params.provisionerId),\n          inArray(providerRunners.id, runnerInstanceIds),\n        ),\n      );\n    return {\n      runnerInstanceIds: params.runnerInstanceIds,\n      controlSessionToAssignment: [],\n    };\n  }\n\n  const [reservation] = await tx\n    .select()\n    .from(reservations)\n    .where(\n      and(\n        eq(reservations.id, params.reservationId),\n        eq(reservations.provisionerId, params.provisionerId),\n        eq(reservations.kind, 'launch'),\n      ),\n    )\n    .limit(1)\n    .for('update');\n  if (!reservation) throw new ReservationNotFoundError(params.reservationId);\n  if (reservation.expiresAt <= new Date()) throw new ReservationExpiredError(params.reservationId);\n  if (runnerRows.length !== runnerInstanceIds.length)\n    throw new RunnerInstanceNotAssignableError(runnerInstanceIds[0] ?? '', 'runner-not-found');\n\n  const activeControlSessions = await tx\n    .select({\n      runnerInstanceId: runnerControlSessions.runnerInstanceId,\n      createdAt: runnerControlSessions.createdAt,\n    })\n    .from(runnerControlSessions)\n    .where(\n      and(\n        inArray(\n          runnerControlSessions.runnerInstanceId,\n          runnerRows.map((runner) => runner.id),\n        ),\n        isNull(runnerControlSessions.closedAt),\n      ),\n    );\n  const runnerInstanceIdsWithControlSession = new Set(\n    activeControlSessions.map((session) => session.runnerInstanceId),\n  );\n  const controlSessionCreatedAtByRunner = new Map(\n    activeControlSessions.map((session) => [session.runnerInstanceId, session.createdAt]),\n  );\n  const runners = runnerRows.map((runner) => ({\n    ...runner,\n    controlSessionId: runnerInstanceIdsWithControlSession.has(runner.id) ? runner.id : null,\n    controlSessionCreatedAt: controlSessionCreatedAtByRunner.get(runner.id) ?? null,\n  }));\n\n  const alreadyAssigned = runners.filter((runner) => runner.reservationId !== null);\n  if (alreadyAssigned.some((runner) => runner.reservationId !== reservation.id))\n    throw new RunnerInstanceAlreadyAssignedError(alreadyAssigned[0]?.id ?? '');\n  const newRunners = runners.filter(\n    (runner) =>\n      runner.reservationId === null ||\n      (runner.reservationId === reservation.id &&\n        (runner.assignedAt === null || runner.workspaceId === null)),\n  );\n  const newRunnerIds = newRunners.map((runner) => runner.id);\n  const assignedCount = await tx\n    .select({count: sql<number>`count(*)::int`})\n    .from(providerRunners)\n    .where(\n      and(\n        eq(providerRunners.provisionerId, params.provisionerId),\n        notInArray(providerRunners.id, newRunnerIds),\n        or(\n          and(\n            eq(providerRunners.reservationId, reservation.id),\n            isNull(providerRunners.reservationReleasedAt),\n          ),\n          and(\n            eq(providerRunners.intendedReservationId, reservation.id),\n            isNull(providerRunners.reservationReleasedAt),\n            notInArray(providerRunners.state, [...terminalStates]),\n          ),\n        ),\n      ),\n    );\n  if ((assignedCount[0]?.count ?? 0) + newRunners.length > reservation.count)\n    throw new RunnerInstanceNotAssignableError(newRunners[0]?.id ?? '', 'capacity-exhausted');\n  for (const runner of newRunners) {\n    if (runner.state !== 'running')\n      throw new RunnerInstanceNotAssignableError(runner.id, 'runner-not-running');\n    if (!runner.providerRunnerId)\n      throw new RunnerInstanceNotAssignableError(runner.id, 'provider-identity-missing');\n    if (!runner.controlSessionId)\n      throw new RunnerInstanceNotAssignableError(runner.id, 'control-session-not-active');\n    if (!reservation.requiredLabels.every((label) => runner.labels.includes(label)))\n      throw new RunnerInstanceNotAssignableError(runner.id, 'labels-mismatch');\n  }\n  if (newRunners.length > 0) {\n    const assignedRows = await tx\n      .update(providerRunners)\n      .set({\n        workspaceId: reservation.workspaceId,\n        reservationId: reservation.id,\n        intendedReservationId: null,\n        assignedAt: sql`now()`,\n        updatedAt: sql`now()`,\n      })\n      .where(\n        inArray(\n          providerRunners.id,\n          newRunners.map((runner) => runner.id),\n        ),\n      )\n      .returning({id: providerRunners.id, assignedAt: providerRunners.assignedAt});\n    const runnersById = new Map(newRunners.map((runner) => [runner.id, runner]));\n    const controlSessionToAssignment: ProviderRunnerAssignmentObservation[] = [];\n    for (const assigned of assignedRows) {\n      const runner = runnersById.get(assigned.id);\n      const controlSessionCreatedAt = runner?.controlSessionCreatedAt;\n      if (!runner || !assigned.assignedAt || !controlSessionCreatedAt) continue;\n      controlSessionToAssignment.push({\n        durationMilliseconds: assigned.assignedAt.getTime() - controlSessionCreatedAt.getTime(),\n        provider: runner.provider,\n        launchKind: runner.launchKind,\n        surface: params.surface,\n        runnerInstanceId: runner.id,\n      });\n    }\n    return {\n      runnerInstanceIds: params.runnerInstanceIds,\n      controlSessionToAssignment,\n    };\n  }\n  return {\n    runnerInstanceIds: params.runnerInstanceIds,\n    controlSessionToAssignment: [],\n  };\n}\n\nexport type RunnerReservationCapacityFailureReason =\n  | 'reservation-not-found'\n  | 'reservation-kind-mismatch'\n  | 'reservation-expired'\n  | 'capacity-exhausted';\n\nexport interface RunnerReservationCapacityValidation {\n  acceptedByReservation: Map<string, number>;\n  unavailableByReservation: Map<\n    string,\n    {reason: RunnerReservationCapacityFailureReason; count: number}\n  >;\n}\n\nexport async function validateRunnerReservationCapacityTx(\n  tx: Tx,\n  params: {\n    provisionerId: string;\n    requests: readonly {reservationId: string; count: number}[];\n  },\n  options: {advisoryLocksHeld?: boolean} = {},\n): Promise<RunnerReservationCapacityValidation> {\n  const requestedByReservation = new Map<string, number>();\n  for (const request of params.requests) {\n    if (request.count <= 0) continue;\n    requestedByReservation.set(\n      request.reservationId,\n      (requestedByReservation.get(request.reservationId) ?? 0) + request.count,\n    );\n  }\n  const reservationIds = [...requestedByReservation.keys()].sort();\n  if (reservationIds.length === 0)\n    return {acceptedByReservation: new Map(), unavailableByReservation: new Map()};\n\n  if (!options.advisoryLocksHeld)\n    await lockRunnerReservationAdvisoryKeysTx(tx, {\n      provisionerId: params.provisionerId,\n      reservationIds,\n    });\n\n  const reservationRows = await tx\n    .select({\n      id: reservations.id,\n      count: reservations.count,\n      expiresAt: reservations.expiresAt,\n      isExpired: sql<boolean>`${reservations.expiresAt} <= now()`,\n    })\n    .from(reservations)\n    .where(\n      and(\n        eq(reservations.provisionerId, params.provisionerId),\n        inArray(reservations.id, reservationIds),\n        eq(reservations.kind, 'launch'),\n      ),\n    )\n    .for('update');\n  const nonLaunchReservationRows = await tx\n    .select({id: reservations.id})\n    .from(reservations)\n    .where(\n      and(\n        eq(reservations.provisionerId, params.provisionerId),\n        inArray(reservations.id, reservationIds),\n        ne(reservations.kind, 'launch'),\n      ),\n    )\n    .for('update');\n  const reservationsById = new Map(\n    reservationRows.map((reservation) => [reservation.id, reservation]),\n  );\n  const nonLaunchReservationIds = new Set(\n    nonLaunchReservationRows.map((reservation) => reservation.id),\n  );\n  const requestedReservationIds = new Set(reservationIds);\n  const usedRunnerIdsByReservation = new Map<string, Set<string>>();\n  const usedRunnerRows = await tx\n    .select({\n      id: providerRunners.id,\n      reservationId: providerRunners.reservationId,\n      intendedReservationId: providerRunners.intendedReservationId,\n      state: providerRunners.state,\n    })\n    .from(providerRunners)\n    .where(\n      and(\n        eq(providerRunners.provisionerId, params.provisionerId),\n        isNull(providerRunners.reservationReleasedAt),\n        or(\n          inArray(providerRunners.reservationId, reservationIds),\n          and(\n            inArray(providerRunners.intendedReservationId, reservationIds),\n            notInArray(providerRunners.state, [...terminalStates]),\n          ),\n        ),\n      ),\n    );\n  for (const runner of usedRunnerRows) {\n    if (runner.reservationId && requestedReservationIds.has(runner.reservationId)) {\n      const usedRunnerIds =\n        usedRunnerIdsByReservation.get(runner.reservationId) ?? new Set<string>();\n      usedRunnerIds.add(runner.id);\n      usedRunnerIdsByReservation.set(runner.reservationId, usedRunnerIds);\n    }\n    if (\n      runner.intendedReservationId &&\n      requestedReservationIds.has(runner.intendedReservationId) &&\n      !terminalStates.includes(runner.state as (typeof terminalStates)[number])\n    ) {\n      const usedRunnerIds =\n        usedRunnerIdsByReservation.get(runner.intendedReservationId) ?? new Set<string>();\n      usedRunnerIds.add(runner.id);\n      usedRunnerIdsByReservation.set(runner.intendedReservationId, usedRunnerIds);\n    }\n  }\n\n  const acceptedByReservation = new Map<string, number>();\n  const unavailableByReservation = new Map<\n    string,\n    {reason: RunnerReservationCapacityFailureReason; count: number}\n  >();\n  for (const reservationId of reservationIds) {\n    const requested = requestedByReservation.get(reservationId) ?? 0;\n    const reservation = reservationsById.get(reservationId);\n    if (!reservation) {\n      unavailableByReservation.set(reservationId, {\n        reason: nonLaunchReservationIds.has(reservationId)\n          ? 'reservation-kind-mismatch'\n          : 'reservation-not-found',\n        count: requested,\n      });\n      continue;\n    }\n    if (reservation.isExpired) {\n      unavailableByReservation.set(reservationId, {\n        reason: 'reservation-expired',\n        count: requested,\n      });\n      continue;\n    }\n\n    const used = usedRunnerIdsByReservation.get(reservationId)?.size ?? 0;\n    const accepted = Math.min(requested, Math.max(0, reservation.count - used));\n    acceptedByReservation.set(reservationId, accepted);\n    if (accepted < requested) {\n      unavailableByReservation.set(reservationId, {\n        reason: 'capacity-exhausted',\n        count: requested - accepted,\n      });\n    }\n  }\n\n  return {acceptedByReservation, unavailableByReservation};\n}\n"],"names":["and","eq","inArray","isNull","ne","notInArray","or","sql","ReservationExpiredError","ReservationNotFoundError","RunnerInstanceAlreadyAssignedError","RunnerInstanceNotAssignableError","recordProviderRunnerControlSessionToAssignment","db","lockRunnerReservationAdvisoryKeysTx","terminalStates","reservations","runnerControlSessions","providerRunners","assignRunnerInstances","params","result","transaction","tx","assignRunnerInstancesTx","surface","observation","controlSessionToAssignment","runnerInstanceIds","sort","provisionerId","reservationIds","reservationId","runnerRows","select","id","intendedReservationId","workspaceId","assignedAt","providerRunnerId","provider","providerKind","launchKind","labels","state","from","where","for","length","every","runner","update","set","updatedAt","reservation","kind","limit","expiresAt","Date","activeControlSessions","runnerInstanceId","createdAt","map","closedAt","runnerInstanceIdsWithControlSession","Set","session","controlSessionCreatedAtByRunner","Map","runners","controlSessionId","has","controlSessionCreatedAt","get","alreadyAssigned","filter","some","newRunners","newRunnerIds","assignedCount","count","reservationReleasedAt","requiredLabels","label","includes","assignedRows","returning","runnersById","assigned","push","durationMilliseconds","getTime","validateRunnerReservationCapacityTx","options","requestedByReservation","request","requests","keys","acceptedByReservation","unavailableByReservation","advisoryLocksHeld","reservationRows","isExpired","nonLaunchReservationRows","reservationsById","nonLaunchReservationIds","requestedReservationIds","usedRunnerIdsByReservation","usedRunnerRows","usedRunnerIds","add","requested","reason","used","size","accepted","Math","min","max"],"mappings":"AAAA,SAAQA,GAAG,EAAEC,EAAE,EAAEC,OAAO,EAAEC,MAAM,EAAEC,EAAE,EAAEC,UAAU,EAAEC,EAAE,EAAEC,GAAG,QAAO,cAAc;AAC9E,SACEC,uBAAuB,EACvBC,wBAAwB,EACxBC,kCAAkC,EAClCC,gCAAgC,QAC3B,kBAAkB;AACzB,SAGEC,8CAA8C,QACzC,uBAAuB;AAE9B,SAAQC,EAAE,QAAO,UAAU;AAC3B,SAAQC,mCAAmC,QAAO,yBAAyB;AAC3E,SAAQC,cAAc,QAAO,qBAAqB;AAClD,SAAQC,YAAY,QAAO,2BAA2B;AACtD,SAAQC,qBAAqB,QAAO,sCAAsC;AAC1E,SAAQC,eAAe,QAAO,+BAA+B;AAE7D,8FAA8F,GAC9F,OAAO,eAAeC,sBAAsBC,MAI3C;IACC,MAAMC,SAAS,MAAMR,KAAKS,WAAW,CAAC,OAAOC,KAC3CC,wBAAwBD,IAAI;YAAC,GAAGH,MAAM;YAAEK,SAAS;QAAa;IAEhE,KAAK,MAAMC,eAAeL,OAAOM,0BAA0B,CACzDf,+CAA+Cc;IACjD,OAAOL,OAAOO,iBAAiB;AACjC;AAOA,OAAO,eAAeJ,wBACpBD,EAAM,EACNH,MAKC;IAED,MAAMQ,oBAAoB;WAAIR,OAAOQ,iBAAiB;KAAC,CAACC,IAAI;IAC5D,MAAMf,oCAAoCS,IAAI;QAC5CO,eAAeV,OAAOU,aAAa;QACnCC,gBAAgB;YAACX,OAAOY,aAAa;SAAC;IACxC;IACA,MAAMC,aAAa,MAAMV,GACtBW,MAAM,CAAC;QACNC,IAAIjB,gBAAgBiB,EAAE;QACtBH,eAAed,gBAAgBc,aAAa;QAC5CI,uBAAuBlB,gBAAgBkB,qBAAqB;QAC5DC,aAAanB,gBAAgBmB,WAAW;QACxCC,YAAYpB,gBAAgBoB,UAAU;QACtCC,kBAAkBrB,gBAAgBqB,gBAAgB;QAClDC,UAAUtB,gBAAgBuB,YAAY;QACtCC,YAAYxB,gBAAgBwB,UAAU;QACtCC,QAAQzB,gBAAgByB,MAAM;QAC9BC,OAAO1B,gBAAgB0B,KAAK;IAC9B,GACCC,IAAI,CAAC3B,iBACL4B,KAAK,CACJ9C,IACEC,GAAGiB,gBAAgBY,aAAa,EAAEV,OAAOU,aAAa,GACtD5B,QAAQgB,gBAAgBiB,EAAE,EAAEP,qBAG/BmB,GAAG,CAAC;IAEP,+EAA+E;IAC/E,wFAAwF;IACxF,IACEnB,kBAAkBoB,MAAM,GAAG,KAC3Bf,WAAWe,MAAM,KAAKpB,kBAAkBoB,MAAM,IAC9Cf,WAAWgB,KAAK,CACd,CAACC,SACCA,OAAOlB,aAAa,KAAKZ,OAAOY,aAAa,IAC7CkB,OAAOZ,UAAU,KAAK,QACtBY,OAAOb,WAAW,KAAK,OAE3B;QACA,MAAMd,GACH4B,MAAM,CAACjC,iBACPkC,GAAG,CAAC;YAAChB,uBAAuB;YAAMiB,WAAW9C,GAAG,CAAC,KAAK,CAAC;QAAA,GACvDuC,KAAK,CACJ9C,IACEC,GAAGiB,gBAAgBY,aAAa,EAAEV,OAAOU,aAAa,GACtD5B,QAAQgB,gBAAgBiB,EAAE,EAAEP;QAGlC,OAAO;YACLA,mBAAmBR,OAAOQ,iBAAiB;YAC3CD,4BAA4B,EAAE;QAChC;IACF;IAEA,MAAM,CAAC2B,YAAY,GAAG,MAAM/B,GACzBW,MAAM,GACNW,IAAI,CAAC7B,cACL8B,KAAK,CACJ9C,IACEC,GAAGe,aAAamB,EAAE,EAAEf,OAAOY,aAAa,GACxC/B,GAAGe,aAAac,aAAa,EAAEV,OAAOU,aAAa,GACnD7B,GAAGe,aAAauC,IAAI,EAAE,YAGzBC,KAAK,CAAC,GACNT,GAAG,CAAC;IACP,IAAI,CAACO,aAAa,MAAM,IAAI7C,yBAAyBW,OAAOY,aAAa;IACzE,IAAIsB,YAAYG,SAAS,IAAI,IAAIC,QAAQ,MAAM,IAAIlD,wBAAwBY,OAAOY,aAAa;IAC/F,IAAIC,WAAWe,MAAM,KAAKpB,kBAAkBoB,MAAM,EAChD,MAAM,IAAIrC,iCAAiCiB,iBAAiB,CAAC,EAAE,IAAI,IAAI;IAEzE,MAAM+B,wBAAwB,MAAMpC,GACjCW,MAAM,CAAC;QACN0B,kBAAkB3C,sBAAsB2C,gBAAgB;QACxDC,WAAW5C,sBAAsB4C,SAAS;IAC5C,GACChB,IAAI,CAAC5B,uBACL6B,KAAK,CACJ9C,IACEE,QACEe,sBAAsB2C,gBAAgB,EACtC3B,WAAW6B,GAAG,CAAC,CAACZ,SAAWA,OAAOf,EAAE,IAEtChC,OAAOc,sBAAsB8C,QAAQ;IAG3C,MAAMC,sCAAsC,IAAIC,IAC9CN,sBAAsBG,GAAG,CAAC,CAACI,UAAYA,QAAQN,gBAAgB;IAEjE,MAAMO,kCAAkC,IAAIC,IAC1CT,sBAAsBG,GAAG,CAAC,CAACI,UAAY;YAACA,QAAQN,gBAAgB;YAAEM,QAAQL,SAAS;SAAC;IAEtF,MAAMQ,UAAUpC,WAAW6B,GAAG,CAAC,CAACZ,SAAY,CAAA;YAC1C,GAAGA,MAAM;YACToB,kBAAkBN,oCAAoCO,GAAG,CAACrB,OAAOf,EAAE,IAAIe,OAAOf,EAAE,GAAG;YACnFqC,yBAAyBL,gCAAgCM,GAAG,CAACvB,OAAOf,EAAE,KAAK;QAC7E,CAAA;IAEA,MAAMuC,kBAAkBL,QAAQM,MAAM,CAAC,CAACzB,SAAWA,OAAOlB,aAAa,KAAK;IAC5E,IAAI0C,gBAAgBE,IAAI,CAAC,CAAC1B,SAAWA,OAAOlB,aAAa,KAAKsB,YAAYnB,EAAE,GAC1E,MAAM,IAAIzB,mCAAmCgE,eAAe,CAAC,EAAE,EAAEvC,MAAM;IACzE,MAAM0C,aAAaR,QAAQM,MAAM,CAC/B,CAACzB,SACCA,OAAOlB,aAAa,KAAK,QACxBkB,OAAOlB,aAAa,KAAKsB,YAAYnB,EAAE,IACrCe,CAAAA,OAAOZ,UAAU,KAAK,QAAQY,OAAOb,WAAW,KAAK,IAAG;IAE/D,MAAMyC,eAAeD,WAAWf,GAAG,CAAC,CAACZ,SAAWA,OAAOf,EAAE;IACzD,MAAM4C,gBAAgB,MAAMxD,GACzBW,MAAM,CAAC;QAAC8C,OAAOzE,GAAW,CAAC,aAAa,CAAC;IAAA,GACzCsC,IAAI,CAAC3B,iBACL4B,KAAK,CACJ9C,IACEC,GAAGiB,gBAAgBY,aAAa,EAAEV,OAAOU,aAAa,GACtDzB,WAAWa,gBAAgBiB,EAAE,EAAE2C,eAC/BxE,GACEN,IACEC,GAAGiB,gBAAgBc,aAAa,EAAEsB,YAAYnB,EAAE,GAChDhC,OAAOe,gBAAgB+D,qBAAqB,IAE9CjF,IACEC,GAAGiB,gBAAgBkB,qBAAqB,EAAEkB,YAAYnB,EAAE,GACxDhC,OAAOe,gBAAgB+D,qBAAqB,GAC5C5E,WAAWa,gBAAgB0B,KAAK,EAAE;WAAI7B;KAAe;IAK/D,IAAI,AAACgE,CAAAA,aAAa,CAAC,EAAE,EAAEC,SAAS,CAAA,IAAKH,WAAW7B,MAAM,GAAGM,YAAY0B,KAAK,EACxE,MAAM,IAAIrE,iCAAiCkE,UAAU,CAAC,EAAE,EAAE1C,MAAM,IAAI;IACtE,KAAK,MAAMe,UAAU2B,WAAY;QAC/B,IAAI3B,OAAON,KAAK,KAAK,WACnB,MAAM,IAAIjC,iCAAiCuC,OAAOf,EAAE,EAAE;QACxD,IAAI,CAACe,OAAOX,gBAAgB,EAC1B,MAAM,IAAI5B,iCAAiCuC,OAAOf,EAAE,EAAE;QACxD,IAAI,CAACe,OAAOoB,gBAAgB,EAC1B,MAAM,IAAI3D,iCAAiCuC,OAAOf,EAAE,EAAE;QACxD,IAAI,CAACmB,YAAY4B,cAAc,CAACjC,KAAK,CAAC,CAACkC,QAAUjC,OAAOP,MAAM,CAACyC,QAAQ,CAACD,SACtE,MAAM,IAAIxE,iCAAiCuC,OAAOf,EAAE,EAAE;IAC1D;IACA,IAAI0C,WAAW7B,MAAM,GAAG,GAAG;QACzB,MAAMqC,eAAe,MAAM9D,GACxB4B,MAAM,CAACjC,iBACPkC,GAAG,CAAC;YACHf,aAAaiB,YAAYjB,WAAW;YACpCL,eAAesB,YAAYnB,EAAE;YAC7BC,uBAAuB;YACvBE,YAAY/B,GAAG,CAAC,KAAK,CAAC;YACtB8C,WAAW9C,GAAG,CAAC,KAAK,CAAC;QACvB,GACCuC,KAAK,CACJ5C,QACEgB,gBAAgBiB,EAAE,EAClB0C,WAAWf,GAAG,CAAC,CAACZ,SAAWA,OAAOf,EAAE,IAGvCmD,SAAS,CAAC;YAACnD,IAAIjB,gBAAgBiB,EAAE;YAAEG,YAAYpB,gBAAgBoB,UAAU;QAAA;QAC5E,MAAMiD,cAAc,IAAInB,IAAIS,WAAWf,GAAG,CAAC,CAACZ,SAAW;gBAACA,OAAOf,EAAE;gBAAEe;aAAO;QAC1E,MAAMvB,6BAAoE,EAAE;QAC5E,KAAK,MAAM6D,YAAYH,aAAc;YACnC,MAAMnC,SAASqC,YAAYd,GAAG,CAACe,SAASrD,EAAE;YAC1C,MAAMqC,0BAA0BtB,QAAQsB;YACxC,IAAI,CAACtB,UAAU,CAACsC,SAASlD,UAAU,IAAI,CAACkC,yBAAyB;YACjE7C,2BAA2B8D,IAAI,CAAC;gBAC9BC,sBAAsBF,SAASlD,UAAU,CAACqD,OAAO,KAAKnB,wBAAwBmB,OAAO;gBACrFnD,UAAUU,OAAOV,QAAQ;gBACzBE,YAAYQ,OAAOR,UAAU;gBAC7BjB,SAASL,OAAOK,OAAO;gBACvBmC,kBAAkBV,OAAOf,EAAE;YAC7B;QACF;QACA,OAAO;YACLP,mBAAmBR,OAAOQ,iBAAiB;YAC3CD;QACF;IACF;IACA,OAAO;QACLC,mBAAmBR,OAAOQ,iBAAiB;QAC3CD,4BAA4B,EAAE;IAChC;AACF;AAgBA,OAAO,eAAeiE,oCACpBrE,EAAM,EACNH,MAGC,EACDyE,UAAyC,CAAC,CAAC;IAE3C,MAAMC,yBAAyB,IAAI1B;IACnC,KAAK,MAAM2B,WAAW3E,OAAO4E,QAAQ,CAAE;QACrC,IAAID,QAAQf,KAAK,IAAI,GAAG;QACxBc,uBAAuB1C,GAAG,CACxB2C,QAAQ/D,aAAa,EACrB,AAAC8D,CAAAA,uBAAuBrB,GAAG,CAACsB,QAAQ/D,aAAa,KAAK,CAAA,IAAK+D,QAAQf,KAAK;IAE5E;IACA,MAAMjD,iBAAiB;WAAI+D,uBAAuBG,IAAI;KAAG,CAACpE,IAAI;IAC9D,IAAIE,eAAeiB,MAAM,KAAK,GAC5B,OAAO;QAACkD,uBAAuB,IAAI9B;QAAO+B,0BAA0B,IAAI/B;IAAK;IAE/E,IAAI,CAACyB,QAAQO,iBAAiB,EAC5B,MAAMtF,oCAAoCS,IAAI;QAC5CO,eAAeV,OAAOU,aAAa;QACnCC;IACF;IAEF,MAAMsE,kBAAkB,MAAM9E,GAC3BW,MAAM,CAAC;QACNC,IAAInB,aAAamB,EAAE;QACnB6C,OAAOhE,aAAagE,KAAK;QACzBvB,WAAWzC,aAAayC,SAAS;QACjC6C,WAAW/F,GAAY,CAAC,EAAES,aAAayC,SAAS,CAAC,SAAS,CAAC;IAC7D,GACCZ,IAAI,CAAC7B,cACL8B,KAAK,CACJ9C,IACEC,GAAGe,aAAac,aAAa,EAAEV,OAAOU,aAAa,GACnD5B,QAAQc,aAAamB,EAAE,EAAEJ,iBACzB9B,GAAGe,aAAauC,IAAI,EAAE,YAGzBR,GAAG,CAAC;IACP,MAAMwD,2BAA2B,MAAMhF,GACpCW,MAAM,CAAC;QAACC,IAAInB,aAAamB,EAAE;IAAA,GAC3BU,IAAI,CAAC7B,cACL8B,KAAK,CACJ9C,IACEC,GAAGe,aAAac,aAAa,EAAEV,OAAOU,aAAa,GACnD5B,QAAQc,aAAamB,EAAE,EAAEJ,iBACzB3B,GAAGY,aAAauC,IAAI,EAAE,YAGzBR,GAAG,CAAC;IACP,MAAMyD,mBAAmB,IAAIpC,IAC3BiC,gBAAgBvC,GAAG,CAAC,CAACR,cAAgB;YAACA,YAAYnB,EAAE;YAAEmB;SAAY;IAEpE,MAAMmD,0BAA0B,IAAIxC,IAClCsC,yBAAyBzC,GAAG,CAAC,CAACR,cAAgBA,YAAYnB,EAAE;IAE9D,MAAMuE,0BAA0B,IAAIzC,IAAIlC;IACxC,MAAM4E,6BAA6B,IAAIvC;IACvC,MAAMwC,iBAAiB,MAAMrF,GAC1BW,MAAM,CAAC;QACNC,IAAIjB,gBAAgBiB,EAAE;QACtBH,eAAed,gBAAgBc,aAAa;QAC5CI,uBAAuBlB,gBAAgBkB,qBAAqB;QAC5DQ,OAAO1B,gBAAgB0B,KAAK;IAC9B,GACCC,IAAI,CAAC3B,iBACL4B,KAAK,CACJ9C,IACEC,GAAGiB,gBAAgBY,aAAa,EAAEV,OAAOU,aAAa,GACtD3B,OAAOe,gBAAgB+D,qBAAqB,GAC5C3E,GACEJ,QAAQgB,gBAAgBc,aAAa,EAAED,iBACvC/B,IACEE,QAAQgB,gBAAgBkB,qBAAqB,EAAEL,iBAC/C1B,WAAWa,gBAAgB0B,KAAK,EAAE;WAAI7B;KAAe;IAK/D,KAAK,MAAMmC,UAAU0D,eAAgB;QACnC,IAAI1D,OAAOlB,aAAa,IAAI0E,wBAAwBnC,GAAG,CAACrB,OAAOlB,aAAa,GAAG;YAC7E,MAAM6E,gBACJF,2BAA2BlC,GAAG,CAACvB,OAAOlB,aAAa,KAAK,IAAIiC;YAC9D4C,cAAcC,GAAG,CAAC5D,OAAOf,EAAE;YAC3BwE,2BAA2BvD,GAAG,CAACF,OAAOlB,aAAa,EAAE6E;QACvD;QACA,IACE3D,OAAOd,qBAAqB,IAC5BsE,wBAAwBnC,GAAG,CAACrB,OAAOd,qBAAqB,KACxD,CAACrB,eAAeqE,QAAQ,CAAClC,OAAON,KAAK,GACrC;YACA,MAAMiE,gBACJF,2BAA2BlC,GAAG,CAACvB,OAAOd,qBAAqB,KAAK,IAAI6B;YACtE4C,cAAcC,GAAG,CAAC5D,OAAOf,EAAE;YAC3BwE,2BAA2BvD,GAAG,CAACF,OAAOd,qBAAqB,EAAEyE;QAC/D;IACF;IAEA,MAAMX,wBAAwB,IAAI9B;IAClC,MAAM+B,2BAA2B,IAAI/B;IAIrC,KAAK,MAAMpC,iBAAiBD,eAAgB;QAC1C,MAAMgF,YAAYjB,uBAAuBrB,GAAG,CAACzC,kBAAkB;QAC/D,MAAMsB,cAAckD,iBAAiB/B,GAAG,CAACzC;QACzC,IAAI,CAACsB,aAAa;YAChB6C,yBAAyB/C,GAAG,CAACpB,eAAe;gBAC1CgF,QAAQP,wBAAwBlC,GAAG,CAACvC,iBAChC,8BACA;gBACJgD,OAAO+B;YACT;YACA;QACF;QACA,IAAIzD,YAAYgD,SAAS,EAAE;YACzBH,yBAAyB/C,GAAG,CAACpB,eAAe;gBAC1CgF,QAAQ;gBACRhC,OAAO+B;YACT;YACA;QACF;QAEA,MAAME,OAAON,2BAA2BlC,GAAG,CAACzC,gBAAgBkF,QAAQ;QACpE,MAAMC,WAAWC,KAAKC,GAAG,CAACN,WAAWK,KAAKE,GAAG,CAAC,GAAGhE,YAAY0B,KAAK,GAAGiC;QACrEf,sBAAsB9C,GAAG,CAACpB,eAAemF;QACzC,IAAIA,WAAWJ,WAAW;YACxBZ,yBAAyB/C,GAAG,CAACpB,eAAe;gBAC1CgF,QAAQ;gBACRhC,OAAO+B,YAAYI;YACrB;QACF;IACF;IAEA,OAAO;QAACjB;QAAuBC;IAAwB;AACzD"}

package/dist/db/runner-instances.d.ts

@@ -64,6 +64,15 @@ export declare function listActiveRunnerInstanceCountsByTemplateTx(tx: Tx, param
 export declare function countStaleEnrolledRunnerInstances(params: {
     graceSeconds: number;
 }): Promise<number>;
+export type ProviderRunnerPhase = 'control_session' | 'enrollment' | 'assignment' | 'activation' | 'idle';
+export interface ProviderRunnerPhaseMetric {
+    phase: ProviderRunnerPhase;
+    provider: string;
+    launchKind: 'demand' | 'warm' | 'manual';
+    count: number;
+    oldestAgeMilliseconds: number;
+}
+export declare function listProviderRunnerByPhaseMetrics(): Promise<ProviderRunnerPhaseMetric[]>;
 export declare function listActiveRunnerInstances(params: {
     workspaceId: string;
     windowSeconds: number;

package/dist/db/runner-instances.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"runner-instances.d.ts","sourceRoot":"","sources":["../../src/db/runner-instances.ts"],"names":[],"mappings":"AAoBA,OAAO,KAAK,EAAC,cAAc,EAAE,mBAAmB,EAAC,MAAM,mCAAmC,CAAC;AAG3F,OAAO,KAAK,EAAC,EAAE,EAAC,MAAM,SAAS,CAAC;AAEhC,OAAO,EAEL,KAAK,+BAA+B,EACrC,MAAM,qBAAqB,CAAC;AAa7B,eAAO,MAAM,YAAY,8CAI0B,CAAC;AACpD,eAAO,MAAM,qBAAqB,kCAG/B,CAAC;AAEJ,MAAM,MAAM,mCAAmC,GAAG,oBAAoB,GAAG,eAAe,CAAC;AAEzF,MAAM,WAAW,6BAA6B;IAC5C,gBAAgB,EAAE,MAAM,CAAC;IACzB,MAAM,EAAE,mCAAmC,CAAC;IAC5C,sBAAsB,CAAC,EAAE,OAAO,CAAC;CAClC;AAED,MAAM,WAAW,iCAAiC;IAChD,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,CAAC,OAAO,qBAAqB,CAAC,CAAC,MAAM,CAAC,CAAC;IAC9C,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,yBAAyB;IACxC,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,KAAK,EAAE,mBAAmB,CAAC;IAC3B,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,UAAU,EAAE,IAAI,CAAC;CAClB;AAED,MAAM,WAAW,2BAA2B;IAC1C,KAAK,EAAE,cAAc,GAAG,WAAW,CAAC;IACpC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,yBAAyB,EAAE,CAAC;CACrC;AAED,MAAM,WAAW,8BAA8B;IAC7C,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,yBAAyB,EAAE,MAAM,EAAE,CAAC;IACpC,qBAAqB,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,WAAW,gCAAgC;IAC/C,YAAY,EAAE,cAAc,EAAE,CAAC;IAC/B,oCAAoC,EAAE,GAAG,CAAC,MAAM,EAAE,+BAA+B,CAAC,CAAC;IACnF,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,oBAAoB,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,8BAA8B;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,oBAAoB,EAAE,MAAM,CAAC;CAC9B;AAED,wBAAsB,8BAA8B,CAAC,MAAM,EAAE;IAC3D,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;CAC1B,GAAG,OAAO,CAAC,OAAO,CAAC,CAcnB;AAiBD,wBAAsB,qBAAqB,CAAC,MAAM,EAAE,2BAA2B,GAAG,OAAO,CAAC;IACxF,QAAQ,EAAE,MAAM,CAAC;IACjB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,uBAAuB,EAAE,6BAA6B,EAAE,CAAC;CAC1D,CAAC,CAoFD;AA6FD,wBAAsB,0CAA0C,CAC9D,EAAE,EAAE,EAAE,EACN,MAAM,EAAE;IAAC,WAAW,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,MAAM,CAAA;CAAC,GACnD,OAAO,CAAC,iCAAiC,EAAE,CAAC,CAuB9C;AAED,wBAAsB,iCAAiC,CAAC,MAAM,EAAE;IAC9D,YAAY,EAAE,MAAM,CAAC;CACtB,GAAG,OAAO,CAAC,MAAM,CAAC,CA0BlB;AAED,wBAAsB,yBAAyB,CAAC,MAAM,EAAE;IACtD,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAe5B;AAED,wBAAsB,+BAA+B,CAAC,MAAM,EAAE;IAC5D,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;CACf,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAKpB;AAED,wBAAsB,oCAAoC,CACxD,EAAE,EAAE,EAAE,EACN,MAAM,EAAE;IACN,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;CACf,GACA,OAAO,CAAC,6BAA6B,EAAE,CAAC,CAqC1C;AAqID,wBAAsB,wBAAwB,CAC5C,MAAM,EAAE,8BAA8B,GACrC,OAAO,CAAC,gCAAgC,CAAC,CAqG3C;AAED,wBAAsB,wBAAwB,CAAC,MAAM,EAAE;IACrD,gBAAgB,EAAE,MAAM,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC;CACf,GAAG,OAAO,CAAC,8BAA8B,CAAC,CA4D1C;AAkcD,wBAAgB,eAAe,CAAC,KAAK,EAAE,mBAAmB,GAAG,OAAO,CAEnE"}
+{"version":3,"file":"runner-instances.d.ts","sourceRoot":"","sources":["../../src/db/runner-instances.ts"],"names":[],"mappings":"AAoBA,OAAO,KAAK,EAAC,cAAc,EAAE,mBAAmB,EAAC,MAAM,mCAAmC,CAAC;AAG3F,OAAO,KAAK,EAAC,EAAE,EAAC,MAAM,SAAS,CAAC;AAEhC,OAAO,EAEL,KAAK,+BAA+B,EACrC,MAAM,qBAAqB,CAAC;AAa7B,eAAO,MAAM,YAAY,8CAI0B,CAAC;AACpD,eAAO,MAAM,qBAAqB,kCAG/B,CAAC;AAEJ,MAAM,MAAM,mCAAmC,GAAG,oBAAoB,GAAG,eAAe,CAAC;AAEzF,MAAM,WAAW,6BAA6B;IAC5C,gBAAgB,EAAE,MAAM,CAAC;IACzB,MAAM,EAAE,mCAAmC,CAAC;IAC5C,sBAAsB,CAAC,EAAE,OAAO,CAAC;CAClC;AAED,MAAM,WAAW,iCAAiC;IAChD,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,CAAC,OAAO,qBAAqB,CAAC,CAAC,MAAM,CAAC,CAAC;IAC9C,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,yBAAyB;IACxC,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,KAAK,EAAE,mBAAmB,CAAC;IAC3B,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,UAAU,EAAE,IAAI,CAAC;CAClB;AAED,MAAM,WAAW,2BAA2B;IAC1C,KAAK,EAAE,cAAc,GAAG,WAAW,CAAC;IACpC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,yBAAyB,EAAE,CAAC;CACrC;AAED,MAAM,WAAW,8BAA8B;IAC7C,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,yBAAyB,EAAE,MAAM,EAAE,CAAC;IACpC,qBAAqB,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,WAAW,gCAAgC;IAC/C,YAAY,EAAE,cAAc,EAAE,CAAC;IAC/B,oCAAoC,EAAE,GAAG,CAAC,MAAM,EAAE,+BAA+B,CAAC,CAAC;IACnF,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,oBAAoB,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,8BAA8B;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,oBAAoB,EAAE,MAAM,CAAC;CAC9B;AAED,wBAAsB,8BAA8B,CAAC,MAAM,EAAE;IAC3D,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;CAC1B,GAAG,OAAO,CAAC,OAAO,CAAC,CAcnB;AAiBD,wBAAsB,qBAAqB,CAAC,MAAM,EAAE,2BAA2B,GAAG,OAAO,CAAC;IACxF,QAAQ,EAAE,MAAM,CAAC;IACjB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,uBAAuB,EAAE,6BAA6B,EAAE,CAAC;CAC1D,CAAC,CAoFD;AA6FD,wBAAsB,0CAA0C,CAC9D,EAAE,EAAE,EAAE,EACN,MAAM,EAAE;IAAC,WAAW,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,MAAM,CAAA;CAAC,GACnD,OAAO,CAAC,iCAAiC,EAAE,CAAC,CAuB9C;AAED,wBAAsB,iCAAiC,CAAC,MAAM,EAAE;IAC9D,YAAY,EAAE,MAAM,CAAC;CACtB,GAAG,OAAO,CAAC,MAAM,CAAC,CA0BlB;AAED,MAAM,MAAM,mBAAmB,GAC3B,iBAAiB,GACjB,YAAY,GACZ,YAAY,GACZ,YAAY,GACZ,MAAM,CAAC;AAEX,MAAM,WAAW,yBAAyB;IACxC,KAAK,EAAE,mBAAmB,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,QAAQ,GAAG,MAAM,GAAG,QAAQ,CAAC;IACzC,KAAK,EAAE,MAAM,CAAC;IACd,qBAAqB,EAAE,MAAM,CAAC;CAC/B;AAED,wBAAsB,gCAAgC,IAAI,OAAO,CAAC,yBAAyB,EAAE,CAAC,CAyD7F;AAED,wBAAsB,yBAAyB,CAAC,MAAM,EAAE;IACtD,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAe5B;AAED,wBAAsB,+BAA+B,CAAC,MAAM,EAAE;IAC5D,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;CACf,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAKpB;AAED,wBAAsB,oCAAoC,CACxD,EAAE,EAAE,EAAE,EACN,MAAM,EAAE;IACN,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;CACf,GACA,OAAO,CAAC,6BAA6B,EAAE,CAAC,CAqC1C;AAqID,wBAAsB,wBAAwB,CAC5C,MAAM,EAAE,8BAA8B,GACrC,OAAO,CAAC,gCAAgC,CAAC,CAqG3C;AAED,wBAAsB,wBAAwB,CAAC,MAAM,EAAE;IACrD,gBAAgB,EAAE,MAAM,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC;CACf,GAAG,OAAO,CAAC,8BAA8B,CAAC,CA4D1C;AAkcD,wBAAgB,eAAe,CAAC,KAAK,EAAE,mBAAmB,GAAG,OAAO,CAEnE"}

package/dist/db/runner-instances.js

@@ -210,6 +210,49 @@ export async function countStaleEnrolledRunnerInstances(params) {
     }).from(runnerControlSessions).where(and(eq(runnerControlSessions.runnerInstanceId, providerRunners.id), isNull(runnerControlSessions.closedAt), gt(runnerControlSessions.expiresAt, sql`now()`))))));
     return row?.count ?? 0;
 }
+export async function listProviderRunnerByPhaseMetrics() {
+    const phase = sql`case
+    when ${runnerControlSessions.id} is null then 'control_session'
+    when ${providerRunners.state} <> 'running' then 'enrollment'
+    when ${providerRunners.intendedReservationId} is not null
+      and ${providerRunners.workspaceId} is null then 'assignment'
+    when ${providerRunners.workspaceId} is not null then 'activation'
+    else 'idle'
+  end`;
+    const startedAt = sql`case
+    when ${runnerControlSessions.id} is null then ${providerRunners.createdAt}
+    when ${providerRunners.state} <> 'running' then ${runnerControlSessions.createdAt}
+    when ${providerRunners.intendedReservationId} is not null
+      and ${providerRunners.workspaceId} is null then ${runnerControlSessions.createdAt}
+    when ${providerRunners.workspaceId} is not null
+      then coalesce(${providerRunners.assignedAt}, ${runnerControlSessions.createdAt})
+    else coalesce(
+      ${runnerControlSessions.createdAt},
+      ${providerRunners.assignedAt},
+      ${providerRunners.createdAt}
+    )
+  end`;
+    const rows = await db().select({
+        phase,
+        provider: sql`coalesce(${providerRunners.providerKind}, 'unknown')`,
+        launchKind: providerRunners.launchKind,
+        count: sql`count(*)::int`,
+        oldestAgeMilliseconds: sql`coalesce(
+        max(extract(epoch from (now() - (${startedAt}))) * 1000),
+        0
+      )::double precision`
+    }).from(providerRunners).leftJoin(runnerControlSessions, and(eq(runnerControlSessions.runnerInstanceId, providerRunners.id), isNull(runnerControlSessions.closedAt), gt(runnerControlSessions.expiresAt, sql`now()`))).where(and(inArray(providerRunners.state, [
+        'starting',
+        'running'
+    ]), isNull(providerRunners.runnerSessionId))).groupBy(phase, providerRunners.providerKind, providerRunners.launchKind);
+    return rows.map((row)=>({
+            phase: row.phase,
+            provider: row.provider,
+            launchKind: row.launchKind,
+            count: row.count,
+            oldestAgeMilliseconds: Math.max(0, row.oldestAgeMilliseconds)
+        }));
+}
 export async function listActiveRunnerInstances(params) {
     const rows = await db().select().from(providerRunners).where(and(eq(providerRunners.workspaceId, params.workspaceId), inArray(providerRunners.state, activeStates), sql`${providerRunners.updatedAt} > now() - (${params.windowSeconds} || ' seconds')::interval`)).orderBy(desc(providerRunners.updatedAt), desc(providerRunners.id)).limit(params.limit ?? 1000);
     return rows.map(toRunnerInstance);