@shipfox/api-runners 12.7.0 → 13.1.0

package/dist/db/job-executions.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/db/job-executions.ts"],"sourcesContent":["import {\n  RUNNER_JOB_CLAIMED,\n  RUNNER_JOB_LEASE_EXPIRED,\n  RUNNER_JOB_QUEUED,\n  type RunnersEventMap,\n  type RunnerToolCapabilitiesDto,\n} from '@shipfox/api-runners-dto';\nimport {logger} from '@shipfox/node-opentelemetry';\nimport {writeOutboxEvent, writeOutboxEvents} from '@shipfox/node-outbox';\nimport {canonicalizeLabels} from '@shipfox/runner-labels';\nimport {\n  and,\n  arrayContained,\n  asc,\n  count,\n  desc,\n  eq,\n  gt,\n  inArray,\n  isNotNull,\n  isNull,\n  lt,\n  lte,\n  or,\n  sql,\n} from 'drizzle-orm';\nimport {\n  EmptyRequiredLabelsError,\n  RunnerSessionExhaustedError,\n  RunningJobExecutionNotFoundError,\n} from '#core/errors.js';\nimport {\n  type JobExecutionQueueTimeObservation,\n  jobExecutionEnqueuedCount,\n  jobExecutionLeaseExpiredCount,\n  type ProviderRunnerLifecycleObservation,\n  recordJobExecutionQueueTime,\n  recordProviderRunnerActivationToFirstClaim,\n} from '#metrics/instance.js';\nimport type {Tx} from './db.js';\nimport {db} from './db.js';\nimport {releaseTerminalRunnerInstanceReservationsByIds} from './reservations.js';\nimport {runnersOutbox} from './schema/outbox.js';\nimport {pendingJobExecutions} from './schema/pending-job-executions.js';\nimport {providerRunners} from './schema/runner-instances.js';\nimport {runnerSessions} from './schema/runner-sessions.js';\nimport {runningJobExecutions} from './schema/running-job-executions.js';\n\nconst runnerJobExecutionLockPrefix = 'runners_job_execution:';\n\nasync function lockJobExecution(tx: Tx, jobExecutionId: string): Promise<void> {\n  await tx.execute(\n    sql`select pg_advisory_xact_lock(hashtext(${`${runnerJobExecutionLockPrefix}${jobExecutionId}`}))`,\n  );\n}\n\nexport interface EnqueueJobExecutionParams {\n  workspaceId: string;\n  workflowRunId: string;\n  workflowRunAttemptId: string;\n  jobId: string;\n  jobExecutionId: string;\n  projectId: string;\n  requiredLabels: string[];\n}\n\nexport async function getWorkspaceJobCounts(params: {\n  workspaceIds: string[];\n}): Promise<Array<{workspaceId: string; queued: number; running: number}>> {\n  const [pendingRows, runningRows] = await Promise.all([\n    db()\n      .select({workspaceId: pendingJobExecutions.workspaceId, count: count()})\n      .from(pendingJobExecutions)\n      .where(inArray(pendingJobExecutions.workspaceId, params.workspaceIds))\n      .groupBy(pendingJobExecutions.workspaceId),\n    db()\n      .select({workspaceId: runningJobExecutions.workspaceId, count: count()})\n      .from(runningJobExecutions)\n      .where(inArray(runningJobExecutions.workspaceId, params.workspaceIds))\n      .groupBy(runningJobExecutions.workspaceId),\n  ]);\n\n  const queuedByWorkspace = new Map(pendingRows.map((row) => [row.workspaceId, Number(row.count)]));\n  const runningByWorkspace = new Map(\n    runningRows.map((row) => [row.workspaceId, Number(row.count)]),\n  );\n\n  return params.workspaceIds.map((workspaceId) => ({\n    workspaceId,\n    queued: queuedByWorkspace.get(workspaceId) ?? 0,\n    running: runningByWorkspace.get(workspaceId) ?? 0,\n  }));\n}\n\n// Idempotent while the job execution is still pending: a duplicate jobExecutionId already in\n// `runners_pending_jobs` is a no-op. Temporal retries the enqueue activity at-least-once, so a\n// unique-violation throw on a retry-after-lost-result would permanently fail a healthy execution.\n// The per-execution advisory lock serializes retries with lease expiry/reconciliation, and the\n// durable lease-expired event prevents a retry from re-queueing an execution already reaped.\nexport async function enqueueJobExecution(params: EnqueueJobExecutionParams): Promise<void> {\n  const requiredLabels = [...canonicalizeLabels(params.requiredLabels)];\n  if (requiredLabels.length === 0) throw new EmptyRequiredLabelsError();\n\n  const enqueued = await db().transaction(async (tx) => {\n    await lockJobExecution(tx, params.jobExecutionId);\n\n    const [leaseExpired] = await tx\n      .select({id: runnersOutbox.id})\n      .from(runnersOutbox)\n      .where(\n        and(\n          eq(runnersOutbox.eventType, RUNNER_JOB_LEASE_EXPIRED),\n          sql`${runnersOutbox.payload}->>'jobExecutionId' = ${params.jobExecutionId}`,\n        ),\n      )\n      .limit(1);\n    if (leaseExpired) return false;\n\n    const [inserted] = await tx\n      .insert(pendingJobExecutions)\n      .values({\n        workspaceId: params.workspaceId,\n        workflowRunId: params.workflowRunId,\n        workflowRunAttemptId: params.workflowRunAttemptId,\n        jobId: params.jobId,\n        jobExecutionId: params.jobExecutionId,\n        projectId: params.projectId,\n        requiredLabels,\n      })\n      .onConflictDoNothing({target: pendingJobExecutions.jobExecutionId})\n      .returning({createdAt: pendingJobExecutions.createdAt});\n\n    // A retry that hits the conflict inserts nothing: the first enqueue already\n    // emitted the queued event (durably, in the outbox), so re-emitting would\n    // only add a redundant row the subscriber coalesces away. Skip it.\n    if (!inserted) return false;\n\n    await writeOutboxEvent<RunnersEventMap>(tx, runnersOutbox, {\n      type: RUNNER_JOB_QUEUED,\n      payload: {\n        workflowRunId: params.workflowRunId,\n        workflowRunAttemptId: params.workflowRunAttemptId,\n        jobId: params.jobId,\n        jobExecutionId: params.jobExecutionId,\n        queuedAt: inserted.createdAt.toISOString(),\n      },\n    });\n    return true;\n  });\n\n  if (enqueued) jobExecutionEnqueuedCount.add(1);\n}\n\nexport interface ClaimedJobExecution {\n  workflowRunId: string;\n  workflowRunAttemptId: string;\n  jobId: string;\n  jobExecutionId: string;\n  projectId: string;\n}\n\nexport interface ActiveRunningJobExecution {\n  workflowRunId: string;\n  workflowRunAttemptId: string;\n  jobId: string;\n  jobExecutionId: string;\n  projectId: string;\n  runnerSessionId: string;\n  provisionerId: string | null;\n  providerRunnerId: string | null;\n  requiredLabels: string[];\n  runnerLabels: string[];\n  startedAt: Date;\n  lastHeartbeatAt: Date;\n}\n\nexport interface RunnerInstanceBoundJobExecution {\n  workflowRunId: string;\n  workflowRunAttemptId: string;\n  jobId: string;\n  jobExecutionId: string;\n  providerRunnerId: string;\n  startedAt: Date;\n  lastHeartbeatAt: Date;\n  cancellationRequestedAt: Date | null;\n}\n\nexport async function claimPendingJobExecution(params: {\n  workspaceId: string;\n  runnerSessionId: string;\n  sessionLabels: string[];\n  maxClaims: number | null;\n  runnerSessionLivenessThrottleSeconds: number;\n}): Promise<ClaimedJobExecution | null> {\n  await touchRunnerSessionLiveness({\n    workspaceId: params.workspaceId,\n    runnerSessionId: params.runnerSessionId,\n    throttleSeconds: params.runnerSessionLivenessThrottleSeconds,\n  });\n\n  if (params.sessionLabels.length === 0) return null;\n\n  let activationToFirstClaimObservation: ProviderRunnerLifecycleObservation | null = null;\n  let queueTimeObservation: JobExecutionQueueTimeObservation | null = null;\n  const result = await db().transaction(async (tx) => {\n    let provisionerId: string | null = null;\n    let providerRunnerId: string | null = null;\n    let runnerInstanceId: string | null = null;\n\n    if (params.maxClaims !== null) {\n      const [session] = await tx\n        .select({\n          maxClaims: runnerSessions.maxClaims,\n          claimsUsed: runnerSessions.claimsUsed,\n          revokedAt: runnerSessions.revokedAt,\n          runnerInstanceId: runnerSessions.runnerInstanceId,\n          provisionerId: runnerSessions.provisionerId,\n          providerRunnerId: runnerSessions.providerRunnerId,\n        })\n        .from(runnerSessions)\n        .where(eq(runnerSessions.id, params.runnerSessionId))\n        .limit(1)\n        .for('update');\n\n      if (\n        !session ||\n        session.revokedAt ||\n        session.maxClaims === null ||\n        session.claimsUsed >= session.maxClaims\n      ) {\n        throw new RunnerSessionExhaustedError(params.runnerSessionId);\n      }\n\n      // Ephemeral sessions are the only capped sessions, and the DB check keeps\n      // their provisioned-runner link present as a pair.\n      runnerInstanceId = session.runnerInstanceId;\n      provisionerId = session.provisionerId;\n      providerRunnerId = session.providerRunnerId;\n    }\n\n    // `id` is a uuidv7 (time-ordered), so it is a deterministic FIFO tiebreaker\n    // for rows sharing a created_at within a batch. Lock only the FIFO candidate before\n    // attempting its execution advisory lock; putting pg_try_advisory_xact_lock in this\n    // predicate would evaluate it while scanning and temporarily lock many queue entries.\n    const candidate = tx\n      .select()\n      .from(pendingJobExecutions)\n      .where(\n        and(\n          eq(pendingJobExecutions.workspaceId, params.workspaceId),\n          arrayContained(pendingJobExecutions.requiredLabels, params.sessionLabels),\n        ),\n      )\n      .orderBy(asc(pendingJobExecutions.createdAt), asc(pendingJobExecutions.id))\n      .limit(1)\n      .for('update', {skipLocked: true})\n      .as('pending_candidate');\n\n    const [row] = await tx\n      .select()\n      .from(candidate)\n      .where(\n        sql`\n          pg_try_advisory_xact_lock(\n            hashtext(${runnerJobExecutionLockPrefix} || ${candidate.jobExecutionId}::text)\n          )\n        `,\n      );\n\n    if (!row) return null;\n\n    await tx.delete(pendingJobExecutions).where(eq(pendingJobExecutions.id, row.id));\n\n    // An enqueue retry that lands after a prior claim can leave an orphan pending\n    // row whose jobExecutionId is already in `runners_running_jobs`. Insert-or-skip\n    // on the jobExecutionId unique constraint: when the job execution is already running\n    // the insert touches no row, so we commit the orphan's deletion and return\n    // null rather than let the unique violation roll the claim back into a poison\n    // loop. The runner just re-polls for a real job execution.\n    const inserted = await tx\n      .insert(runningJobExecutions)\n      .values({\n        workspaceId: row.workspaceId,\n        workflowRunId: row.workflowRunId,\n        workflowRunAttemptId: row.workflowRunAttemptId,\n        jobId: row.jobId,\n        jobExecutionId: row.jobExecutionId,\n        projectId: row.projectId,\n        runnerSessionId: params.runnerSessionId,\n        provisionerId,\n        providerRunnerId,\n        requiredLabels: row.requiredLabels,\n        runnerLabels: params.sessionLabels,\n      })\n      .onConflictDoNothing({target: runningJobExecutions.jobExecutionId})\n      .returning({claimedAt: runningJobExecutions.startedAt});\n\n    const claimed = inserted[0];\n    if (!claimed) return null;\n\n    queueTimeObservation = {\n      durationMilliseconds: claimed.claimedAt.getTime() - row.createdAt.getTime(),\n      provider: null,\n      launchKind: params.maxClaims === null ? 'manual' : 'unknown',\n    };\n\n    const runnerInstanceCondition = runnerInstanceId\n      ? eq(providerRunners.id, runnerInstanceId)\n      : provisionerId && providerRunnerId\n        ? and(\n            eq(providerRunners.provisionerId, provisionerId),\n            eq(providerRunners.providerRunnerId, providerRunnerId),\n          )\n        : null;\n    if (runnerInstanceCondition) {\n      const [claimedRunner] = await tx\n        .update(providerRunners)\n        .set({\n          firstClaimedAt: sql`coalesce(${providerRunners.firstClaimedAt}, ${claimed.claimedAt})`,\n          updatedAt: sql`now()`,\n        })\n        .where(runnerInstanceCondition)\n        .returning({\n          firstClaimedAt: providerRunners.firstClaimedAt,\n          isFirstClaim: sql<boolean>`${providerRunners.firstClaimedAt} = ${claimed.claimedAt}`,\n          provider: providerRunners.providerKind,\n          launchKind: providerRunners.launchKind,\n          runnerInstanceId: providerRunners.id,\n          sessionCreatedAtEpochMs: sql<number | null>`(\n            select extract(epoch from ${runnerSessions.createdAt})::double precision * 1000\n            from ${runnerSessions}\n            where ${runnerSessions.id} = ${params.runnerSessionId}\n          )`,\n        });\n      if (claimedRunner && queueTimeObservation) {\n        queueTimeObservation.provider = claimedRunner.provider;\n        queueTimeObservation.launchKind = claimedRunner.launchKind;\n      }\n      if (\n        claimedRunner?.isFirstClaim &&\n        claimedRunner.firstClaimedAt !== null &&\n        claimedRunner.sessionCreatedAtEpochMs !== null &&\n        claimedRunner.sessionCreatedAtEpochMs !== undefined\n      )\n        activationToFirstClaimObservation = {\n          durationMilliseconds:\n            claimedRunner.firstClaimedAt.getTime() - claimedRunner.sessionCreatedAtEpochMs,\n          provider: claimedRunner.provider,\n          launchKind: claimedRunner.launchKind,\n          runnerInstanceId: claimedRunner.runnerInstanceId,\n        };\n    }\n\n    if (params.maxClaims !== null) {\n      await tx\n        .update(runnerSessions)\n        .set({claimsUsed: sql`${runnerSessions.claimsUsed} + 1`, updatedAt: sql`now()`})\n        .where(eq(runnerSessions.id, params.runnerSessionId));\n    }\n\n    // The running-row insert is the runner claiming the job execution. Emit in the same tx; the\n    // payload carries the row's own claim instant so a consumer records the true time,\n    // not the outbox drain time.\n    await writeOutboxEvent<RunnersEventMap>(tx, runnersOutbox, {\n      type: RUNNER_JOB_CLAIMED,\n      payload: {\n        workflowRunId: row.workflowRunId,\n        workflowRunAttemptId: row.workflowRunAttemptId,\n        jobId: row.jobId,\n        jobExecutionId: row.jobExecutionId,\n        claimedAt: claimed.claimedAt.toISOString(),\n      },\n    });\n\n    return {\n      workflowRunId: row.workflowRunId,\n      workflowRunAttemptId: row.workflowRunAttemptId,\n      jobId: row.jobId,\n      jobExecutionId: row.jobExecutionId,\n      projectId: row.projectId,\n    };\n  });\n  if (queueTimeObservation) recordJobExecutionQueueTime(queueTimeObservation);\n  if (activationToFirstClaimObservation)\n    recordProviderRunnerActivationToFirstClaim(activationToFirstClaimObservation);\n  return result;\n}\n\nasync function touchRunnerSessionLiveness(params: {\n  workspaceId: string;\n  runnerSessionId: string;\n  throttleSeconds: number;\n}): Promise<void> {\n  await db()\n    .update(runnerSessions)\n    .set({updatedAt: sql`now()`})\n    .where(\n      and(\n        eq(runnerSessions.id, params.runnerSessionId),\n        eq(runnerSessions.workspaceId, params.workspaceId),\n        sql`${runnerSessions.updatedAt} < now() - (${params.throttleSeconds} || ' seconds')::interval`,\n      ),\n    );\n}\n\n/**\n * Releases a job execution's lease when the orchestration workflow finalizes it: deletes the\n * running-job-execution row AND any lingering pending row for the same execution, in one tx.\n * When the deleted lease was the last one for a terminal provider runner, it also releases the\n * runner's reservation in the same runner-owned transaction. Terminal reports and workflow lease\n * finalization remain independently retryable; they do not share a cross-module scope lock.\n * Idempotent (0-row no-op), no token scope (the workflow is authoritative), and\n * emits no event: the workflow already owns the outcome. Sweeping the pending row\n * too closes the at-least-once window where an enqueue retry left an orphan that a\n * later claim would otherwise pick up for an already-finished job execution.\n */\nexport async function releaseJobExecution(params: {jobExecutionId: string}): Promise<void> {\n  await db().transaction(async (tx) => {\n    await lockJobExecution(tx, params.jobExecutionId);\n\n    // Delete pending before running to match `claimPendingJobExecution`'s lock-acquisition\n    // order (it locks the pending row first, then the running row). A concurrent\n    // claim picking up an orphan pending row for this same job execution would otherwise\n    // deadlock against the reverse order here.\n    await tx\n      .delete(pendingJobExecutions)\n      .where(eq(pendingJobExecutions.jobExecutionId, params.jobExecutionId));\n    const deletedRunningRows = await tx\n      .delete(runningJobExecutions)\n      .where(eq(runningJobExecutions.jobExecutionId, params.jobExecutionId))\n      .returning({\n        workspaceId: runningJobExecutions.workspaceId,\n        provisionerId: runningJobExecutions.provisionerId,\n        providerRunnerId: runningJobExecutions.providerRunnerId,\n      });\n\n    const deletedRunningRow = deletedRunningRows[0];\n    if (deletedRunningRow?.provisionerId && deletedRunningRow.providerRunnerId) {\n      await releaseTerminalRunnerInstanceReservationsByIds(tx, {\n        workspaceId: null,\n        provisionerId: deletedRunningRow.provisionerId,\n        providerRunnerIds: [deletedRunningRow.providerRunnerId],\n        requireUnlinkedSession: false,\n        // Lock and re-check the runner row locally so terminal reporting remains retryable\n        // without a workflow/runner scope lock.\n        requireTerminalState: false,\n      });\n    }\n  });\n}\n\n/**\n * Reaps stale leases (bounded by `limit`), emitting one\n * `runners.job.lease_expired` event per reaped job execution.\n *\n * The cutoff is re-checked in the DELETE, not just the locking subquery, so a\n * heartbeat landing mid-call spares the live row. Each reaped execution also sweeps its\n * pending row: a failed best-effort `releaseJobExecution` would otherwise leave an orphan\n * that a later claim re-runs as an already-finished job execution.\n *\n * Locks pending-then-running to match `claimPendingJobExecution`, `releaseJobExecution`, and\n * `reconcileTerminalJobExecution`. The stale candidate scan intentionally does not lock running\n * rows first; the running-row DELETE re-checks the stale predicate after the pending-row sweep.\n */\nexport async function expireStuckJobExecutions(params: {\n  thresholdSeconds: number;\n  noFirstHeartbeatGraceSeconds: number;\n  limit?: number;\n}): Promise<\n  Array<{\n    workflowRunId: string;\n    workflowRunAttemptId: string;\n    jobId: string;\n    jobExecutionId: string;\n  }>\n> {\n  const reaped = await db().transaction(async (tx) => {\n    const heartbeatCutoff = sql`now() - (${params.thresholdSeconds} || ' seconds')::interval`;\n    const firstHeartbeatCutoff = sql`now() - (${params.noFirstHeartbeatGraceSeconds} || ' seconds')::interval`;\n    const stalePredicate = or(\n      and(\n        isNull(runningJobExecutions.firstHeartbeatAt),\n        lte(runningJobExecutions.lastHeartbeatAt, runningJobExecutions.startedAt),\n        lt(runningJobExecutions.startedAt, firstHeartbeatCutoff),\n      ),\n      and(\n        or(\n          isNotNull(runningJobExecutions.firstHeartbeatAt),\n          gt(runningJobExecutions.lastHeartbeatAt, runningJobExecutions.startedAt),\n        ),\n        lt(runningJobExecutions.lastHeartbeatAt, heartbeatCutoff),\n      ),\n    );\n\n    const staleRows = await tx\n      .select({\n        id: runningJobExecutions.id,\n        jobExecutionId: runningJobExecutions.jobExecutionId,\n      })\n      .from(runningJobExecutions)\n      .where(\n        and(\n          stalePredicate,\n          sql`\n            pg_try_advisory_xact_lock(\n              hashtext(${runnerJobExecutionLockPrefix} || ${runningJobExecutions.jobExecutionId}::text)\n            )\n          `,\n        ),\n      )\n      .orderBy(\n        asc(\n          sql`CASE WHEN ${runningJobExecutions.firstHeartbeatAt} IS NULL AND ${runningJobExecutions.lastHeartbeatAt} <= ${runningJobExecutions.startedAt} THEN ${runningJobExecutions.startedAt} ELSE ${runningJobExecutions.lastHeartbeatAt} END`,\n        ),\n        asc(runningJobExecutions.id),\n      )\n      .limit(params.limit ?? 100);\n\n    if (staleRows.length === 0) return [];\n\n    const staleIds = staleRows.map((row) => row.id);\n    const staleJobExecutionIds = staleRows.map((row) => row.jobExecutionId);\n\n    // Sweep pending rows first so this transaction cannot hold a running-row lock while waiting\n    // for a pending-row lock held by reconciliation or a claim of an orphan pending row. The\n    // advisory lock acquired in the candidate scan also makes enqueue retries wait until this\n    // transaction has either reaped the execution or released its lock without selecting it.\n    await tx\n      .delete(pendingJobExecutions)\n      .where(inArray(pendingJobExecutions.jobExecutionId, staleJobExecutionIds));\n\n    const deleted = await tx\n      .delete(runningJobExecutions)\n      .where(and(inArray(runningJobExecutions.id, staleIds), stalePredicate))\n      .returning({\n        workflowRunId: runningJobExecutions.workflowRunId,\n        workflowRunAttemptId: runningJobExecutions.workflowRunAttemptId,\n        jobId: runningJobExecutions.jobId,\n        jobExecutionId: runningJobExecutions.jobExecutionId,\n      });\n\n    if (deleted.length === 0) return [];\n\n    await writeOutboxEvents<RunnersEventMap>(\n      tx,\n      runnersOutbox,\n      deleted.map((row) => ({\n        type: RUNNER_JOB_LEASE_EXPIRED,\n        payload: {\n          workflowRunId: row.workflowRunId,\n          workflowRunAttemptId: row.workflowRunAttemptId,\n          jobId: row.jobId,\n          jobExecutionId: row.jobExecutionId,\n        },\n      })),\n    );\n\n    return deleted;\n  });\n\n  if (reaped.length > 0) jobExecutionLeaseExpiredCount.add(reaped.length);\n\n  return reaped;\n}\n\nexport async function getJobExecutionQueueDepth(): Promise<{\n  pendingJobExecutions: number;\n  runningJobExecutions: number;\n}> {\n  const [pending] = await db().select({value: count()}).from(pendingJobExecutions);\n  const [running] = await db().select({value: count()}).from(runningJobExecutions);\n  return {\n    pendingJobExecutions: pending?.value ?? 0,\n    runningJobExecutions: running?.value ?? 0,\n  };\n}\n\nexport async function listActiveRunningJobExecutions(params: {\n  workspaceId: string;\n  windowSeconds: number;\n  limit?: number;\n}): Promise<ActiveRunningJobExecution[]> {\n  return await db()\n    .select({\n      jobId: runningJobExecutions.jobId,\n      jobExecutionId: runningJobExecutions.jobExecutionId,\n      workflowRunId: runningJobExecutions.workflowRunId,\n      workflowRunAttemptId: runningJobExecutions.workflowRunAttemptId,\n      projectId: runningJobExecutions.projectId,\n      runnerSessionId: runningJobExecutions.runnerSessionId,\n      provisionerId: runningJobExecutions.provisionerId,\n      providerRunnerId: runningJobExecutions.providerRunnerId,\n      requiredLabels: runningJobExecutions.requiredLabels,\n      runnerLabels: runningJobExecutions.runnerLabels,\n      startedAt: runningJobExecutions.startedAt,\n      lastHeartbeatAt: runningJobExecutions.lastHeartbeatAt,\n    })\n    .from(runningJobExecutions)\n    .where(\n      and(\n        eq(runningJobExecutions.workspaceId, params.workspaceId),\n        sql`${runningJobExecutions.lastHeartbeatAt} > now() - (${params.windowSeconds} || ' seconds')::interval`,\n      ),\n    )\n    .orderBy(desc(runningJobExecutions.lastHeartbeatAt), desc(runningJobExecutions.id))\n    .limit(params.limit ?? 1000);\n}\n\nexport async function listRunningJobExecutionsByRunnerInstanceTx(\n  tx: Tx,\n  params: {\n    workspaceId: string;\n    provisionerId: string;\n    providerRunnerIds: string[];\n  },\n): Promise<RunnerInstanceBoundJobExecution[]> {\n  if (params.providerRunnerIds.length === 0) return [];\n\n  const duplicateRows = await tx\n    .select({\n      providerRunnerId: runningJobExecutions.providerRunnerId,\n      count: count(),\n    })\n    .from(runningJobExecutions)\n    .where(\n      and(\n        eq(runningJobExecutions.workspaceId, params.workspaceId),\n        eq(runningJobExecutions.provisionerId, params.provisionerId),\n        inArray(runningJobExecutions.providerRunnerId, params.providerRunnerIds),\n      ),\n    )\n    .groupBy(runningJobExecutions.providerRunnerId)\n    .having(sql`count(*) > 1`);\n\n  const duplicateRunnerInstanceIds = duplicateRows.flatMap((row) =>\n    row.providerRunnerId ? [row.providerRunnerId] : [],\n  );\n  if (duplicateRunnerInstanceIds.length > 0) {\n    logger().warn(\n      {\n        workspaceId: params.workspaceId,\n        provisionerId: params.provisionerId,\n        providerRunnerIds: duplicateRunnerInstanceIds,\n      },\n      'multiple running job executions are bound to the same provisioned runner',\n    );\n  }\n\n  const result = await tx.execute<{\n    workflowRunId: string;\n    workflowRunAttemptId: string;\n    jobId: string;\n    jobExecutionId: string;\n    providerRunnerId: string;\n    startedAt: Date | string;\n    lastHeartbeatAt: Date | string;\n    cancellationRequestedAt: Date | string | null;\n  }>(sql`\n    SELECT DISTINCT ON (${runningJobExecutions.providerRunnerId})\n      ${runningJobExecutions.workflowRunId} AS \"workflowRunId\",\n      ${runningJobExecutions.workflowRunAttemptId} AS \"workflowRunAttemptId\",\n      ${runningJobExecutions.jobId} AS \"jobId\",\n      ${runningJobExecutions.jobExecutionId} AS \"jobExecutionId\",\n      ${runningJobExecutions.providerRunnerId} AS \"providerRunnerId\",\n      ${runningJobExecutions.startedAt} AS \"startedAt\",\n      ${runningJobExecutions.lastHeartbeatAt} AS \"lastHeartbeatAt\",\n      ${runningJobExecutions.cancellationRequestedAt} AS \"cancellationRequestedAt\"\n    FROM ${runningJobExecutions}\n    WHERE\n      ${runningJobExecutions.workspaceId} = ${params.workspaceId}\n      AND ${runningJobExecutions.provisionerId} = ${params.provisionerId}\n      AND ${runningJobExecutions.providerRunnerId} IN (${sql.join(\n        params.providerRunnerIds.map((providerRunnerId) => sql`${providerRunnerId}`),\n        sql`, `,\n      )})\n    ORDER BY ${runningJobExecutions.providerRunnerId}, ${runningJobExecutions.startedAt} DESC, ${runningJobExecutions.jobExecutionId} DESC\n  `);\n\n  return result.rows.map((row) => ({\n    workflowRunId: row.workflowRunId,\n    workflowRunAttemptId: row.workflowRunAttemptId,\n    jobId: row.jobId,\n    jobExecutionId: row.jobExecutionId,\n    providerRunnerId: row.providerRunnerId,\n    startedAt: toDate(row.startedAt),\n    lastHeartbeatAt: toDate(row.lastHeartbeatAt),\n    cancellationRequestedAt: row.cancellationRequestedAt\n      ? toDate(row.cancellationRequestedAt)\n      : null,\n  }));\n}\n\nfunction toDate(value: Date | string): Date {\n  return value instanceof Date ? value : new Date(value);\n}\n\nexport async function isJobLeaseActive(params: {\n  jobId?: string;\n  jobExecutionId: string;\n  runnerSessionId: string;\n}): Promise<boolean> {\n  const [row] = await db()\n    .select({id: runningJobExecutions.id})\n    .from(runningJobExecutions)\n    .where(\n      and(\n        eq(runningJobExecutions.jobExecutionId, params.jobExecutionId),\n        eq(runningJobExecutions.runnerSessionId, params.runnerSessionId),\n        params.jobId === undefined ? undefined : eq(runningJobExecutions.jobId, params.jobId),\n      ),\n    )\n    .limit(1);\n\n  return row !== undefined;\n}\n\nexport async function recordHeartbeat(params: {\n  jobExecutionId: string;\n  runnerSessionId: string;\n  toolCapabilities?: RunnerToolCapabilitiesDto | null;\n}): Promise<{\n  cancellationRequested: boolean;\n  previousToolCapabilities: RunnerToolCapabilitiesDto | null;\n  currentToolCapabilities: RunnerToolCapabilitiesDto | null;\n  runningJobExecution: {\n    workflowRunId: string;\n    workflowRunAttemptId: string;\n    jobId: string;\n    jobExecutionId: string;\n    projectId: string;\n    workspaceId: string;\n    runnerSessionId: string;\n  };\n}> {\n  const result = await db().transaction(async (tx) => {\n    const updated = await tx\n      .update(runningJobExecutions)\n      .set({\n        firstHeartbeatAt: sql`COALESCE(${runningJobExecutions.firstHeartbeatAt}, now())`,\n        lastHeartbeatAt: sql`now()`,\n      })\n      .where(\n        and(\n          eq(runningJobExecutions.jobExecutionId, params.jobExecutionId),\n          eq(runningJobExecutions.runnerSessionId, params.runnerSessionId),\n        ),\n      )\n      .returning({\n        cancellationRequestedAt: runningJobExecutions.cancellationRequestedAt,\n        workflowRunId: runningJobExecutions.workflowRunId,\n        workflowRunAttemptId: runningJobExecutions.workflowRunAttemptId,\n        jobId: runningJobExecutions.jobId,\n        jobExecutionId: runningJobExecutions.jobExecutionId,\n        projectId: runningJobExecutions.projectId,\n        workspaceId: runningJobExecutions.workspaceId,\n        runnerSessionId: runningJobExecutions.runnerSessionId,\n      });\n\n    const row = updated[0];\n    if (!row) throw new RunningJobExecutionNotFoundError(params.jobExecutionId);\n\n    const [previous] = await tx\n      .select({toolCapabilities: runnerSessions.toolCapabilities})\n      .from(runnerSessions)\n      .where(eq(runnerSessions.id, params.runnerSessionId))\n      .limit(1)\n      .for('update');\n\n    const [session] = await tx\n      .update(runnerSessions)\n      .set({\n        toolCapabilities: params.toolCapabilities ?? null,\n        toolCapabilitiesReportedAt: params.toolCapabilities ? sql`now()` : null,\n        updatedAt: sql`now()`,\n      })\n      .where(eq(runnerSessions.id, params.runnerSessionId))\n      .returning({toolCapabilities: runnerSessions.toolCapabilities});\n\n    return {\n      row,\n      previousToolCapabilities: previous?.toolCapabilities ?? null,\n      currentToolCapabilities: session?.toolCapabilities ?? null,\n    };\n  });\n\n  const row = result.row;\n  if (!row) throw new RunningJobExecutionNotFoundError(params.jobExecutionId);\n  return {\n    cancellationRequested: row.cancellationRequestedAt !== null,\n    previousToolCapabilities: result.previousToolCapabilities,\n    currentToolCapabilities: result.currentToolCapabilities,\n    runningJobExecution: {\n      workflowRunId: row.workflowRunId,\n      workflowRunAttemptId: row.workflowRunAttemptId,\n      jobId: row.jobId,\n      jobExecutionId: row.jobExecutionId,\n      projectId: row.projectId,\n      workspaceId: row.workspaceId,\n      runnerSessionId: row.runnerSessionId,\n    },\n  };\n}\n\n/**\n * Reconciles a terminal job execution with runner state in one transaction: removes its pending\n * queue row and requests cancellation on its running lease, if either exists. The operation is\n * idempotent and preserves the first cancellation-request timestamp.\n */\nexport async function reconcileTerminalJobExecution(params: {\n  jobExecutionId: string;\n}): Promise<void> {\n  await db().transaction(async (tx) => {\n    await lockJobExecution(tx, params.jobExecutionId);\n\n    // Delete pending before updating running to match claim/release lock order. Claim locks\n    // pending rows with SKIP LOCKED before inserting the running lease, so this ordering makes\n    // a concurrent terminal reconciliation either win before claim or cancel the new lease.\n    await tx\n      .delete(pendingJobExecutions)\n      .where(eq(pendingJobExecutions.jobExecutionId, params.jobExecutionId));\n    await tx\n      .update(runningJobExecutions)\n      .set({\n        cancellationRequestedAt: sql`COALESCE(${runningJobExecutions.cancellationRequestedAt}, now())`,\n      })\n      .where(eq(runningJobExecutions.jobExecutionId, params.jobExecutionId));\n  });\n}\n\nexport async function cancelRunnerJobs(params: {jobIds: string[]}): Promise<void> {\n  if (params.jobIds.length === 0) return;\n\n  await db().transaction(async (tx) => {\n    await tx.delete(pendingJobExecutions).where(inArray(pendingJobExecutions.jobId, params.jobIds));\n    await tx\n      .update(runningJobExecutions)\n      .set({\n        cancellationRequestedAt: sql`COALESCE(${runningJobExecutions.cancellationRequestedAt}, now())`,\n      })\n      .where(inArray(runningJobExecutions.jobId, params.jobIds));\n  });\n}\n"],"names":["RUNNER_JOB_CLAIMED","RUNNER_JOB_LEASE_EXPIRED","RUNNER_JOB_QUEUED","logger","writeOutboxEvent","writeOutboxEvents","canonicalizeLabels","and","arrayContained","asc","count","desc","eq","gt","inArray","isNotNull","isNull","lt","lte","or","sql","EmptyRequiredLabelsError","RunnerSessionExhaustedError","RunningJobExecutionNotFoundError","jobExecutionEnqueuedCount","jobExecutionLeaseExpiredCount","recordJobExecutionQueueTime","recordProviderRunnerActivationToFirstClaim","db","releaseTerminalRunnerInstanceReservationsByIds","runnersOutbox","pendingJobExecutions","providerRunners","runnerSessions","runningJobExecutions","runnerJobExecutionLockPrefix","lockJobExecution","tx","jobExecutionId","execute","getWorkspaceJobCounts","params","pendingRows","runningRows","Promise","all","select","workspaceId","from","where","workspaceIds","groupBy","queuedByWorkspace","Map","map","row","Number","runningByWorkspace","queued","get","running","enqueueJobExecution","requiredLabels","length","enqueued","transaction","leaseExpired","id","eventType","payload","limit","inserted","insert","values","workflowRunId","workflowRunAttemptId","jobId","projectId","onConflictDoNothing","target","returning","createdAt","type","queuedAt","toISOString","add","claimPendingJobExecution","touchRunnerSessionLiveness","runnerSessionId","throttleSeconds","runnerSessionLivenessThrottleSeconds","sessionLabels","activationToFirstClaimObservation","queueTimeObservation","result","provisionerId","providerRunnerId","runnerInstanceId","maxClaims","session","claimsUsed","revokedAt","for","candidate","orderBy","skipLocked","as","delete","runnerLabels","claimedAt","startedAt","claimed","durationMilliseconds","getTime","provider","launchKind","runnerInstanceCondition","claimedRunner","update","set","firstClaimedAt","updatedAt","isFirstClaim","providerKind","sessionCreatedAtEpochMs","undefined","releaseJobExecution","deletedRunningRows","deletedRunningRow","providerRunnerIds","requireUnlinkedSession","requireTerminalState","expireStuckJobExecutions","reaped","heartbeatCutoff","thresholdSeconds","firstHeartbeatCutoff","noFirstHeartbeatGraceSeconds","stalePredicate","firstHeartbeatAt","lastHeartbeatAt","staleRows","staleIds","staleJobExecutionIds","deleted","getJobExecutionQueueDepth","pending","value","listActiveRunningJobExecutions","windowSeconds","listRunningJobExecutionsByRunnerInstanceTx","duplicateRows","having","duplicateRunnerInstanceIds","flatMap","warn","cancellationRequestedAt","join","rows","toDate","Date","isJobLeaseActive","recordHeartbeat","updated","previous","toolCapabilities","toolCapabilitiesReportedAt","previousToolCapabilities","currentToolCapabilities","cancellationRequested","runningJobExecution","reconcileTerminalJobExecution","cancelRunnerJobs","jobIds"],"mappings":"AAAA,SACEA,kBAAkB,EAClBC,wBAAwB,EACxBC,iBAAiB,QAGZ,2BAA2B;AAClC,SAAQC,MAAM,QAAO,8BAA8B;AACnD,SAAQC,gBAAgB,EAAEC,iBAAiB,QAAO,uBAAuB;AACzE,SAAQC,kBAAkB,QAAO,yBAAyB;AAC1D,SACEC,GAAG,EACHC,cAAc,EACdC,GAAG,EACHC,KAAK,EACLC,IAAI,EACJC,EAAE,EACFC,EAAE,EACFC,OAAO,EACPC,SAAS,EACTC,MAAM,EACNC,EAAE,EACFC,GAAG,EACHC,EAAE,EACFC,GAAG,QACE,cAAc;AACrB,SACEC,wBAAwB,EACxBC,2BAA2B,EAC3BC,gCAAgC,QAC3B,kBAAkB;AACzB,SAEEC,yBAAyB,EACzBC,6BAA6B,EAE7BC,2BAA2B,EAC3BC,0CAA0C,QACrC,uBAAuB;AAE9B,SAAQC,EAAE,QAAO,UAAU;AAC3B,SAAQC,8CAA8C,QAAO,oBAAoB;AACjF,SAAQC,aAAa,QAAO,qBAAqB;AACjD,SAAQC,oBAAoB,QAAO,qCAAqC;AACxE,SAAQC,eAAe,QAAO,+BAA+B;AAC7D,SAAQC,cAAc,QAAO,8BAA8B;AAC3D,SAAQC,oBAAoB,QAAO,qCAAqC;AAExE,MAAMC,+BAA+B;AAErC,eAAeC,iBAAiBC,EAAM,EAAEC,cAAsB;IAC5D,MAAMD,GAAGE,OAAO,CACdnB,GAAG,CAAC,sCAAsC,EAAE,GAAGe,+BAA+BG,gBAAgB,CAAC,EAAE,CAAC;AAEtG;AAYA,OAAO,eAAeE,sBAAsBC,MAE3C;IACC,MAAM,CAACC,aAAaC,YAAY,GAAG,MAAMC,QAAQC,GAAG,CAAC;QACnDjB,KACGkB,MAAM,CAAC;YAACC,aAAahB,qBAAqBgB,WAAW;YAAErC,OAAOA;QAAO,GACrEsC,IAAI,CAACjB,sBACLkB,KAAK,CAACnC,QAAQiB,qBAAqBgB,WAAW,EAAEN,OAAOS,YAAY,GACnEC,OAAO,CAACpB,qBAAqBgB,WAAW;QAC3CnB,KACGkB,MAAM,CAAC;YAACC,aAAab,qBAAqBa,WAAW;YAAErC,OAAOA;QAAO,GACrEsC,IAAI,CAACd,sBACLe,KAAK,CAACnC,QAAQoB,qBAAqBa,WAAW,EAAEN,OAAOS,YAAY,GACnEC,OAAO,CAACjB,qBAAqBa,WAAW;KAC5C;IAED,MAAMK,oBAAoB,IAAIC,IAAIX,YAAYY,GAAG,CAAC,CAACC,MAAQ;YAACA,IAAIR,WAAW;YAAES,OAAOD,IAAI7C,KAAK;SAAE;IAC/F,MAAM+C,qBAAqB,IAAIJ,IAC7BV,YAAYW,GAAG,CAAC,CAACC,MAAQ;YAACA,IAAIR,WAAW;YAAES,OAAOD,IAAI7C,KAAK;SAAE;IAG/D,OAAO+B,OAAOS,YAAY,CAACI,GAAG,CAAC,CAACP,cAAiB,CAAA;YAC/CA;YACAW,QAAQN,kBAAkBO,GAAG,CAACZ,gBAAgB;YAC9Ca,SAASH,mBAAmBE,GAAG,CAACZ,gBAAgB;QAClD,CAAA;AACF;AAEA,6FAA6F;AAC7F,+FAA+F;AAC/F,kGAAkG;AAClG,+FAA+F;AAC/F,6FAA6F;AAC7F,OAAO,eAAec,oBAAoBpB,MAAiC;IACzE,MAAMqB,iBAAiB;WAAIxD,mBAAmBmC,OAAOqB,cAAc;KAAE;IACrE,IAAIA,eAAeC,MAAM,KAAK,GAAG,MAAM,IAAI1C;IAE3C,MAAM2C,WAAW,MAAMpC,KAAKqC,WAAW,CAAC,OAAO5B;QAC7C,MAAMD,iBAAiBC,IAAII,OAAOH,cAAc;QAEhD,MAAM,CAAC4B,aAAa,GAAG,MAAM7B,GAC1BS,MAAM,CAAC;YAACqB,IAAIrC,cAAcqC,EAAE;QAAA,GAC5BnB,IAAI,CAAClB,eACLmB,KAAK,CACJ1C,IACEK,GAAGkB,cAAcsC,SAAS,EAAEnE,2BAC5BmB,GAAG,CAAC,EAAEU,cAAcuC,OAAO,CAAC,sBAAsB,EAAE5B,OAAOH,cAAc,CAAC,CAAC,GAG9EgC,KAAK,CAAC;QACT,IAAIJ,cAAc,OAAO;QAEzB,MAAM,CAACK,SAAS,GAAG,MAAMlC,GACtBmC,MAAM,CAACzC,sBACP0C,MAAM,CAAC;YACN1B,aAAaN,OAAOM,WAAW;YAC/B2B,eAAejC,OAAOiC,aAAa;YACnCC,sBAAsBlC,OAAOkC,oBAAoB;YACjDC,OAAOnC,OAAOmC,KAAK;YACnBtC,gBAAgBG,OAAOH,cAAc;YACrCuC,WAAWpC,OAAOoC,SAAS;YAC3Bf;QACF,GACCgB,mBAAmB,CAAC;YAACC,QAAQhD,qBAAqBO,cAAc;QAAA,GAChE0C,SAAS,CAAC;YAACC,WAAWlD,qBAAqBkD,SAAS;QAAA;QAEvD,4EAA4E;QAC5E,0EAA0E;QAC1E,mEAAmE;QACnE,IAAI,CAACV,UAAU,OAAO;QAEtB,MAAMnE,iBAAkCiC,IAAIP,eAAe;YACzDoD,MAAMhF;YACNmE,SAAS;gBACPK,eAAejC,OAAOiC,aAAa;gBACnCC,sBAAsBlC,OAAOkC,oBAAoB;gBACjDC,OAAOnC,OAAOmC,KAAK;gBACnBtC,gBAAgBG,OAAOH,cAAc;gBACrC6C,UAAUZ,SAASU,SAAS,CAACG,WAAW;YAC1C;QACF;QACA,OAAO;IACT;IAEA,IAAIpB,UAAUxC,0BAA0B6D,GAAG,CAAC;AAC9C;AAoCA,OAAO,eAAeC,yBAAyB7C,MAM9C;IACC,MAAM8C,2BAA2B;QAC/BxC,aAAaN,OAAOM,WAAW;QAC/ByC,iBAAiB/C,OAAO+C,eAAe;QACvCC,iBAAiBhD,OAAOiD,oCAAoC;IAC9D;IAEA,IAAIjD,OAAOkD,aAAa,CAAC5B,MAAM,KAAK,GAAG,OAAO;IAE9C,IAAI6B,oCAA+E;IACnF,IAAIC,uBAAgE;IACpE,MAAMC,SAAS,MAAMlE,KAAKqC,WAAW,CAAC,OAAO5B;QAC3C,IAAI0D,gBAA+B;QACnC,IAAIC,mBAAkC;QACtC,IAAIC,mBAAkC;QAEtC,IAAIxD,OAAOyD,SAAS,KAAK,MAAM;YAC7B,MAAM,CAACC,QAAQ,GAAG,MAAM9D,GACrBS,MAAM,CAAC;gBACNoD,WAAWjE,eAAeiE,SAAS;gBACnCE,YAAYnE,eAAemE,UAAU;gBACrCC,WAAWpE,eAAeoE,SAAS;gBACnCJ,kBAAkBhE,eAAegE,gBAAgB;gBACjDF,eAAe9D,eAAe8D,aAAa;gBAC3CC,kBAAkB/D,eAAe+D,gBAAgB;YACnD,GACChD,IAAI,CAACf,gBACLgB,KAAK,CAACrC,GAAGqB,eAAekC,EAAE,EAAE1B,OAAO+C,eAAe,GAClDlB,KAAK,CAAC,GACNgC,GAAG,CAAC;YAEP,IACE,CAACH,WACDA,QAAQE,SAAS,IACjBF,QAAQD,SAAS,KAAK,QACtBC,QAAQC,UAAU,IAAID,QAAQD,SAAS,EACvC;gBACA,MAAM,IAAI5E,4BAA4BmB,OAAO+C,eAAe;YAC9D;YAEA,0EAA0E;YAC1E,mDAAmD;YACnDS,mBAAmBE,QAAQF,gBAAgB;YAC3CF,gBAAgBI,QAAQJ,aAAa;YACrCC,mBAAmBG,QAAQH,gBAAgB;QAC7C;QAEA,4EAA4E;QAC5E,oFAAoF;QACpF,oFAAoF;QACpF,sFAAsF;QACtF,MAAMO,YAAYlE,GACfS,MAAM,GACNE,IAAI,CAACjB,sBACLkB,KAAK,CACJ1C,IACEK,GAAGmB,qBAAqBgB,WAAW,EAAEN,OAAOM,WAAW,GACvDvC,eAAeuB,qBAAqB+B,cAAc,EAAErB,OAAOkD,aAAa,IAG3Ea,OAAO,CAAC/F,IAAIsB,qBAAqBkD,SAAS,GAAGxE,IAAIsB,qBAAqBoC,EAAE,GACxEG,KAAK,CAAC,GACNgC,GAAG,CAAC,UAAU;YAACG,YAAY;QAAI,GAC/BC,EAAE,CAAC;QAEN,MAAM,CAACnD,IAAI,GAAG,MAAMlB,GACjBS,MAAM,GACNE,IAAI,CAACuD,WACLtD,KAAK,CACJ7B,GAAG,CAAC;;qBAES,EAAEe,6BAA6B,IAAI,EAAEoE,UAAUjE,cAAc,CAAC;;QAE3E,CAAC;QAGL,IAAI,CAACiB,KAAK,OAAO;QAEjB,MAAMlB,GAAGsE,MAAM,CAAC5E,sBAAsBkB,KAAK,CAACrC,GAAGmB,qBAAqBoC,EAAE,EAAEZ,IAAIY,EAAE;QAE9E,8EAA8E;QAC9E,gFAAgF;QAChF,qFAAqF;QACrF,2EAA2E;QAC3E,8EAA8E;QAC9E,2DAA2D;QAC3D,MAAMI,WAAW,MAAMlC,GACpBmC,MAAM,CAACtC,sBACPuC,MAAM,CAAC;YACN1B,aAAaQ,IAAIR,WAAW;YAC5B2B,eAAenB,IAAImB,aAAa;YAChCC,sBAAsBpB,IAAIoB,oBAAoB;YAC9CC,OAAOrB,IAAIqB,KAAK;YAChBtC,gBAAgBiB,IAAIjB,cAAc;YAClCuC,WAAWtB,IAAIsB,SAAS;YACxBW,iBAAiB/C,OAAO+C,eAAe;YACvCO;YACAC;YACAlC,gBAAgBP,IAAIO,cAAc;YAClC8C,cAAcnE,OAAOkD,aAAa;QACpC,GACCb,mBAAmB,CAAC;YAACC,QAAQ7C,qBAAqBI,cAAc;QAAA,GAChE0C,SAAS,CAAC;YAAC6B,WAAW3E,qBAAqB4E,SAAS;QAAA;QAEvD,MAAMC,UAAUxC,QAAQ,CAAC,EAAE;QAC3B,IAAI,CAACwC,SAAS,OAAO;QAErBlB,uBAAuB;YACrBmB,sBAAsBD,QAAQF,SAAS,CAACI,OAAO,KAAK1D,IAAI0B,SAAS,CAACgC,OAAO;YACzEC,UAAU;YACVC,YAAY1E,OAAOyD,SAAS,KAAK,OAAO,WAAW;QACrD;QAEA,MAAMkB,0BAA0BnB,mBAC5BrF,GAAGoB,gBAAgBmC,EAAE,EAAE8B,oBACvBF,iBAAiBC,mBACfzF,IACEK,GAAGoB,gBAAgB+D,aAAa,EAAEA,gBAClCnF,GAAGoB,gBAAgBgE,gBAAgB,EAAEA,qBAEvC;QACN,IAAIoB,yBAAyB;YAC3B,MAAM,CAACC,cAAc,GAAG,MAAMhF,GAC3BiF,MAAM,CAACtF,iBACPuF,GAAG,CAAC;gBACHC,gBAAgBpG,GAAG,CAAC,SAAS,EAAEY,gBAAgBwF,cAAc,CAAC,EAAE,EAAET,QAAQF,SAAS,CAAC,CAAC,CAAC;gBACtFY,WAAWrG,GAAG,CAAC,KAAK,CAAC;YACvB,GACC6B,KAAK,CAACmE,yBACNpC,SAAS,CAAC;gBACTwC,gBAAgBxF,gBAAgBwF,cAAc;gBAC9CE,cAActG,GAAY,CAAC,EAAEY,gBAAgBwF,cAAc,CAAC,GAAG,EAAET,QAAQF,SAAS,CAAC,CAAC;gBACpFK,UAAUlF,gBAAgB2F,YAAY;gBACtCR,YAAYnF,gBAAgBmF,UAAU;gBACtClB,kBAAkBjE,gBAAgBmC,EAAE;gBACpCyD,yBAAyBxG,GAAkB,CAAC;sCAChB,EAAEa,eAAegD,SAAS,CAAC;iBAChD,EAAEhD,eAAe;kBAChB,EAAEA,eAAekC,EAAE,CAAC,GAAG,EAAE1B,OAAO+C,eAAe,CAAC;WACvD,CAAC;YACJ;YACF,IAAI6B,iBAAiBxB,sBAAsB;gBACzCA,qBAAqBqB,QAAQ,GAAGG,cAAcH,QAAQ;gBACtDrB,qBAAqBsB,UAAU,GAAGE,cAAcF,UAAU;YAC5D;YACA,IACEE,eAAeK,gBACfL,cAAcG,cAAc,KAAK,QACjCH,cAAcO,uBAAuB,KAAK,QAC1CP,cAAcO,uBAAuB,KAAKC,WAE1CjC,oCAAoC;gBAClCoB,sBACEK,cAAcG,cAAc,CAACP,OAAO,KAAKI,cAAcO,uBAAuB;gBAChFV,UAAUG,cAAcH,QAAQ;gBAChCC,YAAYE,cAAcF,UAAU;gBACpClB,kBAAkBoB,cAAcpB,gBAAgB;YAClD;QACJ;QAEA,IAAIxD,OAAOyD,SAAS,KAAK,MAAM;YAC7B,MAAM7D,GACHiF,MAAM,CAACrF,gBACPsF,GAAG,CAAC;gBAACnB,YAAYhF,GAAG,CAAC,EAAEa,eAAemE,UAAU,CAAC,IAAI,CAAC;gBAAEqB,WAAWrG,GAAG,CAAC,KAAK,CAAC;YAAA,GAC7E6B,KAAK,CAACrC,GAAGqB,eAAekC,EAAE,EAAE1B,OAAO+C,eAAe;QACvD;QAEA,4FAA4F;QAC5F,mFAAmF;QACnF,6BAA6B;QAC7B,MAAMpF,iBAAkCiC,IAAIP,eAAe;YACzDoD,MAAMlF;YACNqE,SAAS;gBACPK,eAAenB,IAAImB,aAAa;gBAChCC,sBAAsBpB,IAAIoB,oBAAoB;gBAC9CC,OAAOrB,IAAIqB,KAAK;gBAChBtC,gBAAgBiB,IAAIjB,cAAc;gBAClCuE,WAAWE,QAAQF,SAAS,CAACzB,WAAW;YAC1C;QACF;QAEA,OAAO;YACLV,eAAenB,IAAImB,aAAa;YAChCC,sBAAsBpB,IAAIoB,oBAAoB;YAC9CC,OAAOrB,IAAIqB,KAAK;YAChBtC,gBAAgBiB,IAAIjB,cAAc;YAClCuC,WAAWtB,IAAIsB,SAAS;QAC1B;IACF;IACA,IAAIgB,sBAAsBnE,4BAA4BmE;IACtD,IAAID,mCACFjE,2CAA2CiE;IAC7C,OAAOE;AACT;AAEA,eAAeP,2BAA2B9C,MAIzC;IACC,MAAMb,KACH0F,MAAM,CAACrF,gBACPsF,GAAG,CAAC;QAACE,WAAWrG,GAAG,CAAC,KAAK,CAAC;IAAA,GAC1B6B,KAAK,CACJ1C,IACEK,GAAGqB,eAAekC,EAAE,EAAE1B,OAAO+C,eAAe,GAC5C5E,GAAGqB,eAAec,WAAW,EAAEN,OAAOM,WAAW,GACjD3B,GAAG,CAAC,EAAEa,eAAewF,SAAS,CAAC,YAAY,EAAEhF,OAAOgD,eAAe,CAAC,yBAAyB,CAAC;AAGtG;AAEA;;;;;;;;;;CAUC,GACD,OAAO,eAAeqC,oBAAoBrF,MAAgC;IACxE,MAAMb,KAAKqC,WAAW,CAAC,OAAO5B;QAC5B,MAAMD,iBAAiBC,IAAII,OAAOH,cAAc;QAEhD,uFAAuF;QACvF,6EAA6E;QAC7E,qFAAqF;QACrF,2CAA2C;QAC3C,MAAMD,GACHsE,MAAM,CAAC5E,sBACPkB,KAAK,CAACrC,GAAGmB,qBAAqBO,cAAc,EAAEG,OAAOH,cAAc;QACtE,MAAMyF,qBAAqB,MAAM1F,GAC9BsE,MAAM,CAACzE,sBACPe,KAAK,CAACrC,GAAGsB,qBAAqBI,cAAc,EAAEG,OAAOH,cAAc,GACnE0C,SAAS,CAAC;YACTjC,aAAab,qBAAqBa,WAAW;YAC7CgD,eAAe7D,qBAAqB6D,aAAa;YACjDC,kBAAkB9D,qBAAqB8D,gBAAgB;QACzD;QAEF,MAAMgC,oBAAoBD,kBAAkB,CAAC,EAAE;QAC/C,IAAIC,mBAAmBjC,iBAAiBiC,kBAAkBhC,gBAAgB,EAAE;YAC1E,MAAMnE,+CAA+CQ,IAAI;gBACvDU,aAAa;gBACbgD,eAAeiC,kBAAkBjC,aAAa;gBAC9CkC,mBAAmB;oBAACD,kBAAkBhC,gBAAgB;iBAAC;gBACvDkC,wBAAwB;gBACxB,mFAAmF;gBACnF,wCAAwC;gBACxCC,sBAAsB;YACxB;QACF;IACF;AACF;AAEA;;;;;;;;;;;;CAYC,GACD,OAAO,eAAeC,yBAAyB3F,MAI9C;IAQC,MAAM4F,SAAS,MAAMzG,KAAKqC,WAAW,CAAC,OAAO5B;QAC3C,MAAMiG,kBAAkBlH,GAAG,CAAC,SAAS,EAAEqB,OAAO8F,gBAAgB,CAAC,yBAAyB,CAAC;QACzF,MAAMC,uBAAuBpH,GAAG,CAAC,SAAS,EAAEqB,OAAOgG,4BAA4B,CAAC,yBAAyB,CAAC;QAC1G,MAAMC,iBAAiBvH,GACrBZ,IACES,OAAOkB,qBAAqByG,gBAAgB,GAC5CzH,IAAIgB,qBAAqB0G,eAAe,EAAE1G,qBAAqB4E,SAAS,GACxE7F,GAAGiB,qBAAqB4E,SAAS,EAAE0B,wBAErCjI,IACEY,GACEJ,UAAUmB,qBAAqByG,gBAAgB,GAC/C9H,GAAGqB,qBAAqB0G,eAAe,EAAE1G,qBAAqB4E,SAAS,IAEzE7F,GAAGiB,qBAAqB0G,eAAe,EAAEN;QAI7C,MAAMO,YAAY,MAAMxG,GACrBS,MAAM,CAAC;YACNqB,IAAIjC,qBAAqBiC,EAAE;YAC3B7B,gBAAgBJ,qBAAqBI,cAAc;QACrD,GACCU,IAAI,CAACd,sBACLe,KAAK,CACJ1C,IACEmI,gBACAtH,GAAG,CAAC;;uBAES,EAAEe,6BAA6B,IAAI,EAAED,qBAAqBI,cAAc,CAAC;;UAEtF,CAAC,GAGJkE,OAAO,CACN/F,IACEW,GAAG,CAAC,UAAU,EAAEc,qBAAqByG,gBAAgB,CAAC,aAAa,EAAEzG,qBAAqB0G,eAAe,CAAC,IAAI,EAAE1G,qBAAqB4E,SAAS,CAAC,MAAM,EAAE5E,qBAAqB4E,SAAS,CAAC,MAAM,EAAE5E,qBAAqB0G,eAAe,CAAC,IAAI,CAAC,GAE1OnI,IAAIyB,qBAAqBiC,EAAE,GAE5BG,KAAK,CAAC7B,OAAO6B,KAAK,IAAI;QAEzB,IAAIuE,UAAU9E,MAAM,KAAK,GAAG,OAAO,EAAE;QAErC,MAAM+E,WAAWD,UAAUvF,GAAG,CAAC,CAACC,MAAQA,IAAIY,EAAE;QAC9C,MAAM4E,uBAAuBF,UAAUvF,GAAG,CAAC,CAACC,MAAQA,IAAIjB,cAAc;QAEtE,4FAA4F;QAC5F,yFAAyF;QACzF,0FAA0F;QAC1F,yFAAyF;QACzF,MAAMD,GACHsE,MAAM,CAAC5E,sBACPkB,KAAK,CAACnC,QAAQiB,qBAAqBO,cAAc,EAAEyG;QAEtD,MAAMC,UAAU,MAAM3G,GACnBsE,MAAM,CAACzE,sBACPe,KAAK,CAAC1C,IAAIO,QAAQoB,qBAAqBiC,EAAE,EAAE2E,WAAWJ,iBACtD1D,SAAS,CAAC;YACTN,eAAexC,qBAAqBwC,aAAa;YACjDC,sBAAsBzC,qBAAqByC,oBAAoB;YAC/DC,OAAO1C,qBAAqB0C,KAAK;YACjCtC,gBAAgBJ,qBAAqBI,cAAc;QACrD;QAEF,IAAI0G,QAAQjF,MAAM,KAAK,GAAG,OAAO,EAAE;QAEnC,MAAM1D,kBACJgC,IACAP,eACAkH,QAAQ1F,GAAG,CAAC,CAACC,MAAS,CAAA;gBACpB2B,MAAMjF;gBACNoE,SAAS;oBACPK,eAAenB,IAAImB,aAAa;oBAChCC,sBAAsBpB,IAAIoB,oBAAoB;oBAC9CC,OAAOrB,IAAIqB,KAAK;oBAChBtC,gBAAgBiB,IAAIjB,cAAc;gBACpC;YACF,CAAA;QAGF,OAAO0G;IACT;IAEA,IAAIX,OAAOtE,MAAM,GAAG,GAAGtC,8BAA8B4D,GAAG,CAACgD,OAAOtE,MAAM;IAEtE,OAAOsE;AACT;AAEA,OAAO,eAAeY;IAIpB,MAAM,CAACC,QAAQ,GAAG,MAAMtH,KAAKkB,MAAM,CAAC;QAACqG,OAAOzI;IAAO,GAAGsC,IAAI,CAACjB;IAC3D,MAAM,CAAC6B,QAAQ,GAAG,MAAMhC,KAAKkB,MAAM,CAAC;QAACqG,OAAOzI;IAAO,GAAGsC,IAAI,CAACd;IAC3D,OAAO;QACLH,sBAAsBmH,SAASC,SAAS;QACxCjH,sBAAsB0B,SAASuF,SAAS;IAC1C;AACF;AAEA,OAAO,eAAeC,+BAA+B3G,MAIpD;IACC,OAAO,MAAMb,KACVkB,MAAM,CAAC;QACN8B,OAAO1C,qBAAqB0C,KAAK;QACjCtC,gBAAgBJ,qBAAqBI,cAAc;QACnDoC,eAAexC,qBAAqBwC,aAAa;QACjDC,sBAAsBzC,qBAAqByC,oBAAoB;QAC/DE,WAAW3C,qBAAqB2C,SAAS;QACzCW,iBAAiBtD,qBAAqBsD,eAAe;QACrDO,eAAe7D,qBAAqB6D,aAAa;QACjDC,kBAAkB9D,qBAAqB8D,gBAAgB;QACvDlC,gBAAgB5B,qBAAqB4B,cAAc;QACnD8C,cAAc1E,qBAAqB0E,YAAY;QAC/CE,WAAW5E,qBAAqB4E,SAAS;QACzC8B,iBAAiB1G,qBAAqB0G,eAAe;IACvD,GACC5F,IAAI,CAACd,sBACLe,KAAK,CACJ1C,IACEK,GAAGsB,qBAAqBa,WAAW,EAAEN,OAAOM,WAAW,GACvD3B,GAAG,CAAC,EAAEc,qBAAqB0G,eAAe,CAAC,YAAY,EAAEnG,OAAO4G,aAAa,CAAC,yBAAyB,CAAC,GAG3G7C,OAAO,CAAC7F,KAAKuB,qBAAqB0G,eAAe,GAAGjI,KAAKuB,qBAAqBiC,EAAE,GAChFG,KAAK,CAAC7B,OAAO6B,KAAK,IAAI;AAC3B;AAEA,OAAO,eAAegF,2CACpBjH,EAAM,EACNI,MAIC;IAED,IAAIA,OAAOwF,iBAAiB,CAAClE,MAAM,KAAK,GAAG,OAAO,EAAE;IAEpD,MAAMwF,gBAAgB,MAAMlH,GACzBS,MAAM,CAAC;QACNkD,kBAAkB9D,qBAAqB8D,gBAAgB;QACvDtF,OAAOA;IACT,GACCsC,IAAI,CAACd,sBACLe,KAAK,CACJ1C,IACEK,GAAGsB,qBAAqBa,WAAW,EAAEN,OAAOM,WAAW,GACvDnC,GAAGsB,qBAAqB6D,aAAa,EAAEtD,OAAOsD,aAAa,GAC3DjF,QAAQoB,qBAAqB8D,gBAAgB,EAAEvD,OAAOwF,iBAAiB,IAG1E9E,OAAO,CAACjB,qBAAqB8D,gBAAgB,EAC7CwD,MAAM,CAACpI,GAAG,CAAC,YAAY,CAAC;IAE3B,MAAMqI,6BAA6BF,cAAcG,OAAO,CAAC,CAACnG,MACxDA,IAAIyC,gBAAgB,GAAG;YAACzC,IAAIyC,gBAAgB;SAAC,GAAG,EAAE;IAEpD,IAAIyD,2BAA2B1F,MAAM,GAAG,GAAG;QACzC5D,SAASwJ,IAAI,CACX;YACE5G,aAAaN,OAAOM,WAAW;YAC/BgD,eAAetD,OAAOsD,aAAa;YACnCkC,mBAAmBwB;QACrB,GACA;IAEJ;IAEA,MAAM3D,SAAS,MAAMzD,GAAGE,OAAO,CAS5BnB,GAAG,CAAC;wBACe,EAAEc,qBAAqB8D,gBAAgB,CAAC;MAC1D,EAAE9D,qBAAqBwC,aAAa,CAAC;MACrC,EAAExC,qBAAqByC,oBAAoB,CAAC;MAC5C,EAAEzC,qBAAqB0C,KAAK,CAAC;MAC7B,EAAE1C,qBAAqBI,cAAc,CAAC;MACtC,EAAEJ,qBAAqB8D,gBAAgB,CAAC;MACxC,EAAE9D,qBAAqB4E,SAAS,CAAC;MACjC,EAAE5E,qBAAqB0G,eAAe,CAAC;MACvC,EAAE1G,qBAAqB0H,uBAAuB,CAAC;SAC5C,EAAE1H,qBAAqB;;MAE1B,EAAEA,qBAAqBa,WAAW,CAAC,GAAG,EAAEN,OAAOM,WAAW,CAAC;UACvD,EAAEb,qBAAqB6D,aAAa,CAAC,GAAG,EAAEtD,OAAOsD,aAAa,CAAC;UAC/D,EAAE7D,qBAAqB8D,gBAAgB,CAAC,KAAK,EAAE5E,IAAIyI,IAAI,CACzDpH,OAAOwF,iBAAiB,CAAC3E,GAAG,CAAC,CAAC0C,mBAAqB5E,GAAG,CAAC,EAAE4E,iBAAiB,CAAC,GAC3E5E,GAAG,CAAC,EAAE,CAAC,EACP;aACK,EAAEc,qBAAqB8D,gBAAgB,CAAC,EAAE,EAAE9D,qBAAqB4E,SAAS,CAAC,OAAO,EAAE5E,qBAAqBI,cAAc,CAAC;EACnI,CAAC;IAED,OAAOwD,OAAOgE,IAAI,CAACxG,GAAG,CAAC,CAACC,MAAS,CAAA;YAC/BmB,eAAenB,IAAImB,aAAa;YAChCC,sBAAsBpB,IAAIoB,oBAAoB;YAC9CC,OAAOrB,IAAIqB,KAAK;YAChBtC,gBAAgBiB,IAAIjB,cAAc;YAClC0D,kBAAkBzC,IAAIyC,gBAAgB;YACtCc,WAAWiD,OAAOxG,IAAIuD,SAAS;YAC/B8B,iBAAiBmB,OAAOxG,IAAIqF,eAAe;YAC3CgB,yBAAyBrG,IAAIqG,uBAAuB,GAChDG,OAAOxG,IAAIqG,uBAAuB,IAClC;QACN,CAAA;AACF;AAEA,SAASG,OAAOZ,KAAoB;IAClC,OAAOA,iBAAiBa,OAAOb,QAAQ,IAAIa,KAAKb;AAClD;AAEA,OAAO,eAAec,iBAAiBxH,MAItC;IACC,MAAM,CAACc,IAAI,GAAG,MAAM3B,KACjBkB,MAAM,CAAC;QAACqB,IAAIjC,qBAAqBiC,EAAE;IAAA,GACnCnB,IAAI,CAACd,sBACLe,KAAK,CACJ1C,IACEK,GAAGsB,qBAAqBI,cAAc,EAAEG,OAAOH,cAAc,GAC7D1B,GAAGsB,qBAAqBsD,eAAe,EAAE/C,OAAO+C,eAAe,GAC/D/C,OAAOmC,KAAK,KAAKiD,YAAYA,YAAYjH,GAAGsB,qBAAqB0C,KAAK,EAAEnC,OAAOmC,KAAK,IAGvFN,KAAK,CAAC;IAET,OAAOf,QAAQsE;AACjB;AAEA,OAAO,eAAeqC,gBAAgBzH,MAIrC;IAcC,MAAMqD,SAAS,MAAMlE,KAAKqC,WAAW,CAAC,OAAO5B;QAC3C,MAAM8H,UAAU,MAAM9H,GACnBiF,MAAM,CAACpF,sBACPqF,GAAG,CAAC;YACHoB,kBAAkBvH,GAAG,CAAC,SAAS,EAAEc,qBAAqByG,gBAAgB,CAAC,QAAQ,CAAC;YAChFC,iBAAiBxH,GAAG,CAAC,KAAK,CAAC;QAC7B,GACC6B,KAAK,CACJ1C,IACEK,GAAGsB,qBAAqBI,cAAc,EAAEG,OAAOH,cAAc,GAC7D1B,GAAGsB,qBAAqBsD,eAAe,EAAE/C,OAAO+C,eAAe,IAGlER,SAAS,CAAC;YACT4E,yBAAyB1H,qBAAqB0H,uBAAuB;YACrElF,eAAexC,qBAAqBwC,aAAa;YACjDC,sBAAsBzC,qBAAqByC,oBAAoB;YAC/DC,OAAO1C,qBAAqB0C,KAAK;YACjCtC,gBAAgBJ,qBAAqBI,cAAc;YACnDuC,WAAW3C,qBAAqB2C,SAAS;YACzC9B,aAAab,qBAAqBa,WAAW;YAC7CyC,iBAAiBtD,qBAAqBsD,eAAe;QACvD;QAEF,MAAMjC,MAAM4G,OAAO,CAAC,EAAE;QACtB,IAAI,CAAC5G,KAAK,MAAM,IAAIhC,iCAAiCkB,OAAOH,cAAc;QAE1E,MAAM,CAAC8H,SAAS,GAAG,MAAM/H,GACtBS,MAAM,CAAC;YAACuH,kBAAkBpI,eAAeoI,gBAAgB;QAAA,GACzDrH,IAAI,CAACf,gBACLgB,KAAK,CAACrC,GAAGqB,eAAekC,EAAE,EAAE1B,OAAO+C,eAAe,GAClDlB,KAAK,CAAC,GACNgC,GAAG,CAAC;QAEP,MAAM,CAACH,QAAQ,GAAG,MAAM9D,GACrBiF,MAAM,CAACrF,gBACPsF,GAAG,CAAC;YACH8C,kBAAkB5H,OAAO4H,gBAAgB,IAAI;YAC7CC,4BAA4B7H,OAAO4H,gBAAgB,GAAGjJ,GAAG,CAAC,KAAK,CAAC,GAAG;YACnEqG,WAAWrG,GAAG,CAAC,KAAK,CAAC;QACvB,GACC6B,KAAK,CAACrC,GAAGqB,eAAekC,EAAE,EAAE1B,OAAO+C,eAAe,GAClDR,SAAS,CAAC;YAACqF,kBAAkBpI,eAAeoI,gBAAgB;QAAA;QAE/D,OAAO;YACL9G;YACAgH,0BAA0BH,UAAUC,oBAAoB;YACxDG,yBAAyBrE,SAASkE,oBAAoB;QACxD;IACF;IAEA,MAAM9G,MAAMuC,OAAOvC,GAAG;IACtB,IAAI,CAACA,KAAK,MAAM,IAAIhC,iCAAiCkB,OAAOH,cAAc;IAC1E,OAAO;QACLmI,uBAAuBlH,IAAIqG,uBAAuB,KAAK;QACvDW,0BAA0BzE,OAAOyE,wBAAwB;QACzDC,yBAAyB1E,OAAO0E,uBAAuB;QACvDE,qBAAqB;YACnBhG,eAAenB,IAAImB,aAAa;YAChCC,sBAAsBpB,IAAIoB,oBAAoB;YAC9CC,OAAOrB,IAAIqB,KAAK;YAChBtC,gBAAgBiB,IAAIjB,cAAc;YAClCuC,WAAWtB,IAAIsB,SAAS;YACxB9B,aAAaQ,IAAIR,WAAW;YAC5ByC,iBAAiBjC,IAAIiC,eAAe;QACtC;IACF;AACF;AAEA;;;;CAIC,GACD,OAAO,eAAemF,8BAA8BlI,MAEnD;IACC,MAAMb,KAAKqC,WAAW,CAAC,OAAO5B;QAC5B,MAAMD,iBAAiBC,IAAII,OAAOH,cAAc;QAEhD,wFAAwF;QACxF,2FAA2F;QAC3F,wFAAwF;QACxF,MAAMD,GACHsE,MAAM,CAAC5E,sBACPkB,KAAK,CAACrC,GAAGmB,qBAAqBO,cAAc,EAAEG,OAAOH,cAAc;QACtE,MAAMD,GACHiF,MAAM,CAACpF,sBACPqF,GAAG,CAAC;YACHqC,yBAAyBxI,GAAG,CAAC,SAAS,EAAEc,qBAAqB0H,uBAAuB,CAAC,QAAQ,CAAC;QAChG,GACC3G,KAAK,CAACrC,GAAGsB,qBAAqBI,cAAc,EAAEG,OAAOH,cAAc;IACxE;AACF;AAEA,OAAO,eAAesI,iBAAiBnI,MAA0B;IAC/D,IAAIA,OAAOoI,MAAM,CAAC9G,MAAM,KAAK,GAAG;IAEhC,MAAMnC,KAAKqC,WAAW,CAAC,OAAO5B;QAC5B,MAAMA,GAAGsE,MAAM,CAAC5E,sBAAsBkB,KAAK,CAACnC,QAAQiB,qBAAqB6C,KAAK,EAAEnC,OAAOoI,MAAM;QAC7F,MAAMxI,GACHiF,MAAM,CAACpF,sBACPqF,GAAG,CAAC;YACHqC,yBAAyBxI,GAAG,CAAC,SAAS,EAAEc,qBAAqB0H,uBAAuB,CAAC,QAAQ,CAAC;QAChG,GACC3G,KAAK,CAACnC,QAAQoB,qBAAqB0C,KAAK,EAAEnC,OAAOoI,MAAM;IAC5D;AACF"}
+{"version":3,"sources":["../../src/db/job-executions.ts"],"sourcesContent":["import {\n  RUNNER_JOB_CLAIMED,\n  RUNNER_JOB_LEASE_EXPIRED,\n  type RunnersEventMap,\n  type RunnerToolCapabilitiesDto,\n} from '@shipfox/api-runners-dto';\nimport {logger} from '@shipfox/node-opentelemetry';\nimport {writeOutboxEvent, writeOutboxEvents} from '@shipfox/node-outbox';\nimport {canonicalizeLabels} from '@shipfox/runner-labels';\nimport {\n  and,\n  arrayContained,\n  asc,\n  count,\n  desc,\n  eq,\n  gt,\n  inArray,\n  isNotNull,\n  isNull,\n  lt,\n  lte,\n  or,\n  sql,\n} from 'drizzle-orm';\nimport {\n  EmptyRequiredLabelsError,\n  RunnerSessionExhaustedError,\n  RunningJobExecutionNotFoundError,\n} from '#core/errors.js';\nimport {\n  type JobExecutionQueueTimeObservation,\n  jobExecutionEnqueuedCount,\n  jobExecutionLeaseExpiredCount,\n  type ProviderRunnerLifecycleObservation,\n  recordJobExecutionQueueTime,\n  recordProviderRunnerActivationToFirstClaim,\n} from '#metrics/instance.js';\nimport type {Tx} from './db.js';\nimport {db} from './db.js';\nimport {lockRunnerReservationAdvisoryKeysTx} from './reservation-locks.js';\nimport {\n  releaseReservationUnits,\n  releaseTerminalRunnerInstanceReservationsByIds,\n} from './reservations.js';\nimport {runnersOutbox} from './schema/outbox.js';\nimport {pendingJobExecutions} from './schema/pending-job-executions.js';\nimport {reservations} from './schema/reservations.js';\nimport {providerRunners} from './schema/runner-instances.js';\nimport {runnerSessions} from './schema/runner-sessions.js';\nimport {runningJobExecutions} from './schema/running-job-executions.js';\n\nconst runnerJobExecutionLockPrefix = 'runners_job_execution:';\n\nasync function lockJobExecution(tx: Tx, jobExecutionId: string): Promise<void> {\n  await tx.execute(\n    sql`select pg_advisory_xact_lock(hashtext(${`${runnerJobExecutionLockPrefix}${jobExecutionId}`}))`,\n  );\n}\n\nasync function releaseReservationsForTerminalRunningRows(\n  tx: Tx,\n  rows: ReadonlyArray<{\n    provisionerId: string | null;\n    providerRunnerId: string | null;\n  }>,\n): Promise<void> {\n  // Converge after the lease fact changes. The reservation helper locks each provider runner\n  // and releases only when the runner is terminal and no uncancelled lease remains.\n  const providerRunnerIdsByProvisionerId = new Map<string, Set<string>>();\n  for (const row of rows) {\n    if (row.provisionerId === null || row.providerRunnerId === null) continue;\n    const providerRunnerIds =\n      providerRunnerIdsByProvisionerId.get(row.provisionerId) ?? new Set<string>();\n    providerRunnerIds.add(row.providerRunnerId);\n    providerRunnerIdsByProvisionerId.set(row.provisionerId, providerRunnerIds);\n  }\n\n  for (const provisionerId of [...providerRunnerIdsByProvisionerId.keys()].sort()) {\n    const providerRunnerIds = providerRunnerIdsByProvisionerId.get(provisionerId);\n    if (!providerRunnerIds) continue;\n    await releaseTerminalRunnerInstanceReservationsByIds(tx, {\n      workspaceId: null,\n      provisionerId,\n      providerRunnerIds: [...providerRunnerIds].sort(),\n      requireUnlinkedSession: false,\n      // Lock and re-check the runner row locally so lease finalization remains retryable\n      // without a workflow/runner scope lock.\n      requireTerminalState: false,\n    });\n  }\n}\n\nexport interface EnqueueJobExecutionParams {\n  workspaceId: string;\n  workflowRunId: string;\n  workflowRunAttemptId: string;\n  jobId: string;\n  jobExecutionId: string;\n  projectId: string;\n  requiredLabels: string[];\n  queuedAt: Date;\n}\n\nexport async function getWorkspaceJobCounts(params: {\n  workspaceIds: string[];\n}): Promise<Array<{workspaceId: string; queued: number; running: number}>> {\n  const [pendingRows, runningRows] = await Promise.all([\n    db()\n      .select({workspaceId: pendingJobExecutions.workspaceId, count: count()})\n      .from(pendingJobExecutions)\n      .where(inArray(pendingJobExecutions.workspaceId, params.workspaceIds))\n      .groupBy(pendingJobExecutions.workspaceId),\n    db()\n      .select({workspaceId: runningJobExecutions.workspaceId, count: count()})\n      .from(runningJobExecutions)\n      .where(inArray(runningJobExecutions.workspaceId, params.workspaceIds))\n      .groupBy(runningJobExecutions.workspaceId),\n  ]);\n\n  const queuedByWorkspace = new Map(pendingRows.map((row) => [row.workspaceId, Number(row.count)]));\n  const runningByWorkspace = new Map(\n    runningRows.map((row) => [row.workspaceId, Number(row.count)]),\n  );\n\n  return params.workspaceIds.map((workspaceId) => ({\n    workspaceId,\n    queued: queuedByWorkspace.get(workspaceId) ?? 0,\n    running: runningByWorkspace.get(workspaceId) ?? 0,\n  }));\n}\n\n// The workflows outbox delivers queue facts at least once. The per-execution advisory lock and\n// pending/running checks make replay a no-op across the claim transition. A durable lease-expired\n// event also prevents a delayed replay from resurrecting an execution already reaped locally.\nexport async function enqueueJobExecution(params: EnqueueJobExecutionParams): Promise<void> {\n  const requiredLabels = [...canonicalizeLabels(params.requiredLabels)];\n  if (requiredLabels.length === 0) throw new EmptyRequiredLabelsError();\n\n  const enqueued = await db().transaction(async (tx) => {\n    await lockJobExecution(tx, params.jobExecutionId);\n\n    const [running] = await tx\n      .select({jobExecutionId: runningJobExecutions.jobExecutionId})\n      .from(runningJobExecutions)\n      .where(eq(runningJobExecutions.jobExecutionId, params.jobExecutionId))\n      .limit(1);\n    if (running) return false;\n\n    const [leaseExpired] = await tx\n      .select({id: runnersOutbox.id})\n      .from(runnersOutbox)\n      .where(\n        and(\n          eq(runnersOutbox.eventType, RUNNER_JOB_LEASE_EXPIRED),\n          sql`${runnersOutbox.payload}->>'jobExecutionId' = ${params.jobExecutionId}`,\n        ),\n      )\n      .limit(1);\n    if (leaseExpired) return false;\n\n    const [inserted] = await tx\n      .insert(pendingJobExecutions)\n      .values({\n        workspaceId: params.workspaceId,\n        workflowRunId: params.workflowRunId,\n        workflowRunAttemptId: params.workflowRunAttemptId,\n        jobId: params.jobId,\n        jobExecutionId: params.jobExecutionId,\n        projectId: params.projectId,\n        requiredLabels,\n        createdAt: params.queuedAt,\n      })\n      .onConflictDoNothing({target: pendingJobExecutions.jobExecutionId})\n      .returning({createdAt: pendingJobExecutions.createdAt});\n\n    return inserted !== undefined;\n  });\n\n  if (enqueued) jobExecutionEnqueuedCount.add(1);\n}\n\nexport interface ClaimedJobExecution {\n  workflowRunId: string;\n  workflowRunAttemptId: string;\n  jobId: string;\n  jobExecutionId: string;\n  projectId: string;\n}\n\nexport interface ActiveRunningJobExecution {\n  workflowRunId: string;\n  workflowRunAttemptId: string;\n  jobId: string;\n  jobExecutionId: string;\n  projectId: string;\n  runnerSessionId: string;\n  provisionerId: string | null;\n  providerRunnerId: string | null;\n  requiredLabels: string[];\n  runnerLabels: string[];\n  startedAt: Date;\n  lastHeartbeatAt: Date;\n}\n\nexport interface RunnerInstanceBoundJobExecution {\n  workflowRunId: string;\n  workflowRunAttemptId: string;\n  jobId: string;\n  jobExecutionId: string;\n  providerRunnerId: string;\n  startedAt: Date;\n  lastHeartbeatAt: Date;\n  cancellationRequestedAt: Date | null;\n}\n\nexport async function claimPendingJobExecution(params: {\n  workspaceId: string;\n  runnerSessionId: string;\n  sessionLabels: string[];\n  maxClaims: number | null;\n  runnerSessionLivenessThrottleSeconds: number;\n}): Promise<ClaimedJobExecution | null> {\n  await touchRunnerSessionLiveness({\n    workspaceId: params.workspaceId,\n    runnerSessionId: params.runnerSessionId,\n    throttleSeconds: params.runnerSessionLivenessThrottleSeconds,\n  });\n\n  if (params.sessionLabels.length === 0) return null;\n\n  let activationToFirstClaimObservation: ProviderRunnerLifecycleObservation | null = null;\n  let queueTimeObservation: JobExecutionQueueTimeObservation | null = null;\n  const result = await db().transaction(async (tx) => {\n    let provisionerId: string | null = null;\n    let providerRunnerId: string | null = null;\n    let runnerInstanceId: string | null = null;\n\n    if (params.maxClaims !== null) {\n      const [session] = await tx\n        .select({\n          maxClaims: runnerSessions.maxClaims,\n          claimsUsed: runnerSessions.claimsUsed,\n          revokedAt: runnerSessions.revokedAt,\n          runnerInstanceId: runnerSessions.runnerInstanceId,\n          provisionerId: runnerSessions.provisionerId,\n          providerRunnerId: runnerSessions.providerRunnerId,\n        })\n        .from(runnerSessions)\n        .where(eq(runnerSessions.id, params.runnerSessionId))\n        .limit(1)\n        .for('update');\n\n      if (\n        !session ||\n        session.revokedAt ||\n        session.maxClaims === null ||\n        session.claimsUsed >= session.maxClaims\n      ) {\n        throw new RunnerSessionExhaustedError(params.runnerSessionId);\n      }\n\n      // Ephemeral sessions are the only capped sessions, and the DB check keeps\n      // their provisioned-runner link present as a pair.\n      runnerInstanceId = session.runnerInstanceId;\n      provisionerId = session.provisionerId;\n      providerRunnerId = session.providerRunnerId;\n    }\n\n    const runnerInstanceCondition = runnerInstanceId\n      ? eq(providerRunners.id, runnerInstanceId)\n      : provisionerId && providerRunnerId\n        ? and(\n            eq(providerRunners.provisionerId, provisionerId),\n            eq(providerRunners.providerRunnerId, providerRunnerId),\n          )\n        : null;\n    if (runnerInstanceCondition && provisionerId) {\n      const [runner] = await tx\n        .select({\n          reservationId: providerRunners.reservationId,\n          intendedReservationId: providerRunners.intendedReservationId,\n        })\n        .from(providerRunners)\n        .where(runnerInstanceCondition)\n        .limit(1);\n      await lockRunnerReservationAdvisoryKeysTx(tx, {\n        provisionerId,\n        reservationIds: [runner?.reservationId, runner?.intendedReservationId].filter(\n          (reservationId): reservationId is string =>\n            reservationId !== null && reservationId !== undefined,\n        ),\n      });\n    }\n\n    // `id` is a uuidv7 (time-ordered), so it is a deterministic FIFO tiebreaker\n    // for rows sharing a created_at within a batch. Lock only the FIFO candidate before\n    // attempting its execution advisory lock; putting pg_try_advisory_xact_lock in this\n    // predicate would evaluate it while scanning and temporarily lock many queue entries.\n    const candidate = tx\n      .select()\n      .from(pendingJobExecutions)\n      .where(\n        and(\n          eq(pendingJobExecutions.workspaceId, params.workspaceId),\n          arrayContained(pendingJobExecutions.requiredLabels, params.sessionLabels),\n        ),\n      )\n      .orderBy(asc(pendingJobExecutions.createdAt), asc(pendingJobExecutions.id))\n      .limit(1)\n      .for('update', {skipLocked: true})\n      .as('pending_candidate');\n\n    const [row] = await tx\n      .select()\n      .from(candidate)\n      .where(\n        sql`\n          pg_try_advisory_xact_lock(\n            hashtext(${runnerJobExecutionLockPrefix} || ${candidate.jobExecutionId}::text)\n          )\n        `,\n      );\n\n    if (!row) return null;\n\n    await tx.delete(pendingJobExecutions).where(eq(pendingJobExecutions.id, row.id));\n\n    // An enqueue retry that lands after a prior claim can leave an orphan pending\n    // row whose jobExecutionId is already in `runners_running_jobs`. Insert-or-skip\n    // on the jobExecutionId unique constraint: when the job execution is already running\n    // the insert touches no row, so we commit the orphan's deletion and return\n    // null rather than let the unique violation roll the claim back into a poison\n    // loop. The runner just re-polls for a real job execution.\n    const inserted = await tx\n      .insert(runningJobExecutions)\n      .values({\n        workspaceId: row.workspaceId,\n        workflowRunId: row.workflowRunId,\n        workflowRunAttemptId: row.workflowRunAttemptId,\n        jobId: row.jobId,\n        jobExecutionId: row.jobExecutionId,\n        projectId: row.projectId,\n        runnerSessionId: params.runnerSessionId,\n        provisionerId,\n        providerRunnerId,\n        requiredLabels: row.requiredLabels,\n        runnerLabels: params.sessionLabels,\n      })\n      .onConflictDoNothing({target: runningJobExecutions.jobExecutionId})\n      .returning({claimedAt: runningJobExecutions.startedAt});\n\n    const claimed = inserted[0];\n    if (!claimed) return null;\n\n    queueTimeObservation = {\n      durationMilliseconds: claimed.claimedAt.getTime() - row.createdAt.getTime(),\n      provider: null,\n      launchKind: params.maxClaims === null ? 'manual' : 'unknown',\n    };\n\n    if (runnerInstanceCondition) {\n      const [claimedRunner] = await tx\n        .update(providerRunners)\n        .set({\n          firstClaimedAt: sql`coalesce(${providerRunners.firstClaimedAt}, ${claimed.claimedAt})`,\n          updatedAt: sql`now()`,\n        })\n        .where(runnerInstanceCondition)\n        .returning({\n          firstClaimedAt: providerRunners.firstClaimedAt,\n          isFirstClaim: sql<boolean>`${providerRunners.firstClaimedAt} = ${claimed.claimedAt}`,\n          provider: providerRunners.providerKind,\n          launchKind: providerRunners.launchKind,\n          runnerInstanceId: providerRunners.id,\n          reservationId: providerRunners.reservationId,\n          intendedReservationId: providerRunners.intendedReservationId,\n          sessionCreatedAtEpochMs: sql<number | null>`(\n            select extract(epoch from ${runnerSessions.createdAt})::double precision * 1000\n            from ${runnerSessions}\n            where ${runnerSessions.id} = ${params.runnerSessionId}\n          )`,\n        });\n      if (claimedRunner && queueTimeObservation) {\n        queueTimeObservation.provider = claimedRunner.provider;\n        queueTimeObservation.launchKind = claimedRunner.launchKind;\n      }\n      if (claimedRunner?.isFirstClaim) {\n        const [releasedRunner] = await tx\n          .update(providerRunners)\n          .set({\n            intendedReservationId: null,\n            reservationReleasedAt: sql`now()`,\n            updatedAt: sql`now()`,\n          })\n          .where(\n            and(\n              eq(providerRunners.id, claimedRunner.runnerInstanceId),\n              or(\n                isNotNull(providerRunners.reservationId),\n                isNotNull(providerRunners.intendedReservationId),\n              ),\n              isNull(providerRunners.reservationReleasedAt),\n            ),\n          )\n          .returning({\n            provisionerId: providerRunners.provisionerId,\n          });\n        const reservationId = claimedRunner.intendedReservationId ?? claimedRunner.reservationId;\n        if (releasedRunner?.provisionerId && reservationId) {\n          const [reservation] = await tx\n            .select({workspaceId: reservations.workspaceId})\n            .from(reservations)\n            .where(\n              and(\n                eq(reservations.id, reservationId),\n                eq(reservations.provisionerId, releasedRunner.provisionerId),\n              ),\n            )\n            .limit(1);\n          if (reservation)\n            await releaseReservationUnits(tx, {\n              workspaceId: reservation.workspaceId,\n              provisionerId: releasedRunner.provisionerId,\n              releases: [{reservationId, count: 1}],\n            });\n        }\n      }\n      if (\n        claimedRunner?.isFirstClaim &&\n        claimedRunner.firstClaimedAt !== null &&\n        claimedRunner.sessionCreatedAtEpochMs !== null &&\n        claimedRunner.sessionCreatedAtEpochMs !== undefined\n      )\n        activationToFirstClaimObservation = {\n          durationMilliseconds:\n            claimedRunner.firstClaimedAt.getTime() - claimedRunner.sessionCreatedAtEpochMs,\n          provider: claimedRunner.provider,\n          launchKind: claimedRunner.launchKind,\n          runnerInstanceId: claimedRunner.runnerInstanceId,\n        };\n    }\n\n    if (params.maxClaims !== null) {\n      await tx\n        .update(runnerSessions)\n        .set({claimsUsed: sql`${runnerSessions.claimsUsed} + 1`, updatedAt: sql`now()`})\n        .where(eq(runnerSessions.id, params.runnerSessionId));\n    }\n\n    // The running-row insert is the runner claiming the job execution. Emit in the same tx; the\n    // payload carries the row's own claim instant so a consumer records the true time,\n    // not the outbox drain time.\n    await writeOutboxEvent<RunnersEventMap>(tx, runnersOutbox, {\n      type: RUNNER_JOB_CLAIMED,\n      payload: {\n        workflowRunId: row.workflowRunId,\n        workflowRunAttemptId: row.workflowRunAttemptId,\n        jobId: row.jobId,\n        jobExecutionId: row.jobExecutionId,\n        claimedAt: claimed.claimedAt.toISOString(),\n      },\n    });\n\n    return {\n      workflowRunId: row.workflowRunId,\n      workflowRunAttemptId: row.workflowRunAttemptId,\n      jobId: row.jobId,\n      jobExecutionId: row.jobExecutionId,\n      projectId: row.projectId,\n    };\n  });\n  if (queueTimeObservation) recordJobExecutionQueueTime(queueTimeObservation);\n  if (activationToFirstClaimObservation)\n    recordProviderRunnerActivationToFirstClaim(activationToFirstClaimObservation);\n  return result;\n}\n\nasync function touchRunnerSessionLiveness(params: {\n  workspaceId: string;\n  runnerSessionId: string;\n  throttleSeconds: number;\n}): Promise<void> {\n  await db()\n    .update(runnerSessions)\n    .set({updatedAt: sql`now()`})\n    .where(\n      and(\n        eq(runnerSessions.id, params.runnerSessionId),\n        eq(runnerSessions.workspaceId, params.workspaceId),\n        sql`${runnerSessions.updatedAt} < now() - (${params.throttleSeconds} || ' seconds')::interval`,\n      ),\n    );\n}\n\n/**\n * Reaps stale leases (bounded by `limit`), emitting one\n * `runners.job.lease_expired` event per reaped job execution.\n *\n * The cutoff is re-checked in the DELETE, not just the locking subquery, so a\n * heartbeat landing mid-call spares the live row. Each reaped execution also sweeps its\n * pending row so an at-least-once queue replay cannot leave an orphan.\n *\n * Locks pending-then-running to match `claimPendingJobExecution` and\n * `reconcileTerminalJobExecution`. The stale candidate scan intentionally does not lock\n * running rows first; the running-row DELETE re-checks the stale predicate after the\n * pending-row sweep.\n */\nexport async function expireStuckJobExecutions(params: {\n  thresholdSeconds: number;\n  noFirstHeartbeatGraceSeconds: number;\n  limit?: number;\n}): Promise<\n  Array<{\n    workflowRunId: string;\n    workflowRunAttemptId: string;\n    jobId: string;\n    jobExecutionId: string;\n  }>\n> {\n  const reaped = await db().transaction(async (tx) => {\n    const heartbeatCutoff = sql`now() - (${params.thresholdSeconds} || ' seconds')::interval`;\n    const firstHeartbeatCutoff = sql`now() - (${params.noFirstHeartbeatGraceSeconds} || ' seconds')::interval`;\n    const stalePredicate = or(\n      and(\n        isNull(runningJobExecutions.firstHeartbeatAt),\n        lte(runningJobExecutions.lastHeartbeatAt, runningJobExecutions.startedAt),\n        lt(runningJobExecutions.startedAt, firstHeartbeatCutoff),\n      ),\n      and(\n        or(\n          isNotNull(runningJobExecutions.firstHeartbeatAt),\n          gt(runningJobExecutions.lastHeartbeatAt, runningJobExecutions.startedAt),\n        ),\n        lt(runningJobExecutions.lastHeartbeatAt, heartbeatCutoff),\n      ),\n    );\n\n    const staleRows = await tx\n      .select({\n        id: runningJobExecutions.id,\n        jobExecutionId: runningJobExecutions.jobExecutionId,\n      })\n      .from(runningJobExecutions)\n      .where(\n        and(\n          stalePredicate,\n          sql`\n            pg_try_advisory_xact_lock(\n              hashtext(${runnerJobExecutionLockPrefix} || ${runningJobExecutions.jobExecutionId}::text)\n            )\n          `,\n        ),\n      )\n      .orderBy(\n        asc(\n          sql`CASE WHEN ${runningJobExecutions.firstHeartbeatAt} IS NULL AND ${runningJobExecutions.lastHeartbeatAt} <= ${runningJobExecutions.startedAt} THEN ${runningJobExecutions.startedAt} ELSE ${runningJobExecutions.lastHeartbeatAt} END`,\n        ),\n        asc(runningJobExecutions.id),\n      )\n      .limit(params.limit ?? 100);\n\n    if (staleRows.length === 0) return [];\n\n    const staleIds = staleRows.map((row) => row.id);\n    const staleJobExecutionIds = staleRows.map((row) => row.jobExecutionId);\n\n    // Sweep pending rows first so this transaction cannot hold a running-row lock while waiting\n    // for a pending-row lock held by reconciliation or a claim of an orphan pending row. The\n    // advisory lock acquired in the candidate scan also makes enqueue retries wait until this\n    // transaction has either reaped the execution or released its lock without selecting it.\n    await tx\n      .delete(pendingJobExecutions)\n      .where(inArray(pendingJobExecutions.jobExecutionId, staleJobExecutionIds));\n\n    const deleted = await tx\n      .delete(runningJobExecutions)\n      .where(and(inArray(runningJobExecutions.id, staleIds), stalePredicate))\n      .returning({\n        workflowRunId: runningJobExecutions.workflowRunId,\n        workflowRunAttemptId: runningJobExecutions.workflowRunAttemptId,\n        jobId: runningJobExecutions.jobId,\n        jobExecutionId: runningJobExecutions.jobExecutionId,\n        provisionerId: runningJobExecutions.provisionerId,\n        providerRunnerId: runningJobExecutions.providerRunnerId,\n      });\n\n    if (deleted.length === 0) return [];\n\n    await releaseReservationsForTerminalRunningRows(tx, deleted);\n\n    await writeOutboxEvents<RunnersEventMap>(\n      tx,\n      runnersOutbox,\n      deleted.map((row) => ({\n        type: RUNNER_JOB_LEASE_EXPIRED,\n        payload: {\n          workflowRunId: row.workflowRunId,\n          workflowRunAttemptId: row.workflowRunAttemptId,\n          jobId: row.jobId,\n          jobExecutionId: row.jobExecutionId,\n        },\n      })),\n    );\n\n    return deleted.map(({workflowRunId, workflowRunAttemptId, jobId, jobExecutionId}) => ({\n      workflowRunId,\n      workflowRunAttemptId,\n      jobId,\n      jobExecutionId,\n    }));\n  });\n\n  if (reaped.length > 0) jobExecutionLeaseExpiredCount.add(reaped.length);\n\n  return reaped;\n}\n\nexport async function getJobExecutionQueueDepth(): Promise<{\n  pendingJobExecutions: number;\n  runningJobExecutions: number;\n}> {\n  const [pending] = await db().select({value: count()}).from(pendingJobExecutions);\n  const [running] = await db().select({value: count()}).from(runningJobExecutions);\n  return {\n    pendingJobExecutions: pending?.value ?? 0,\n    runningJobExecutions: running?.value ?? 0,\n  };\n}\n\nexport async function listActiveRunningJobExecutions(params: {\n  workspaceId: string;\n  windowSeconds: number;\n  limit?: number;\n}): Promise<ActiveRunningJobExecution[]> {\n  return await db()\n    .select({\n      jobId: runningJobExecutions.jobId,\n      jobExecutionId: runningJobExecutions.jobExecutionId,\n      workflowRunId: runningJobExecutions.workflowRunId,\n      workflowRunAttemptId: runningJobExecutions.workflowRunAttemptId,\n      projectId: runningJobExecutions.projectId,\n      runnerSessionId: runningJobExecutions.runnerSessionId,\n      provisionerId: runningJobExecutions.provisionerId,\n      providerRunnerId: runningJobExecutions.providerRunnerId,\n      requiredLabels: runningJobExecutions.requiredLabels,\n      runnerLabels: runningJobExecutions.runnerLabels,\n      startedAt: runningJobExecutions.startedAt,\n      lastHeartbeatAt: runningJobExecutions.lastHeartbeatAt,\n    })\n    .from(runningJobExecutions)\n    .where(\n      and(\n        eq(runningJobExecutions.workspaceId, params.workspaceId),\n        sql`${runningJobExecutions.lastHeartbeatAt} > now() - (${params.windowSeconds} || ' seconds')::interval`,\n      ),\n    )\n    .orderBy(desc(runningJobExecutions.lastHeartbeatAt), desc(runningJobExecutions.id))\n    .limit(params.limit ?? 1000);\n}\n\nexport async function listRunningJobExecutionsByRunnerInstanceTx(\n  tx: Tx,\n  params: {\n    workspaceId: string;\n    provisionerId: string;\n    providerRunnerIds: string[];\n  },\n): Promise<RunnerInstanceBoundJobExecution[]> {\n  if (params.providerRunnerIds.length === 0) return [];\n\n  const duplicateRows = await tx\n    .select({\n      providerRunnerId: runningJobExecutions.providerRunnerId,\n      count: count(),\n    })\n    .from(runningJobExecutions)\n    .where(\n      and(\n        eq(runningJobExecutions.workspaceId, params.workspaceId),\n        eq(runningJobExecutions.provisionerId, params.provisionerId),\n        inArray(runningJobExecutions.providerRunnerId, params.providerRunnerIds),\n      ),\n    )\n    .groupBy(runningJobExecutions.providerRunnerId)\n    .having(sql`count(*) > 1`);\n\n  const duplicateRunnerInstanceIds = duplicateRows.flatMap((row) =>\n    row.providerRunnerId ? [row.providerRunnerId] : [],\n  );\n  if (duplicateRunnerInstanceIds.length > 0) {\n    logger().warn(\n      {\n        workspaceId: params.workspaceId,\n        provisionerId: params.provisionerId,\n        providerRunnerIds: duplicateRunnerInstanceIds,\n      },\n      'multiple running job executions are bound to the same provisioned runner',\n    );\n  }\n\n  const result = await tx.execute<{\n    workflowRunId: string;\n    workflowRunAttemptId: string;\n    jobId: string;\n    jobExecutionId: string;\n    providerRunnerId: string;\n    startedAt: Date | string;\n    lastHeartbeatAt: Date | string;\n    cancellationRequestedAt: Date | string | null;\n  }>(sql`\n    SELECT DISTINCT ON (${runningJobExecutions.providerRunnerId})\n      ${runningJobExecutions.workflowRunId} AS \"workflowRunId\",\n      ${runningJobExecutions.workflowRunAttemptId} AS \"workflowRunAttemptId\",\n      ${runningJobExecutions.jobId} AS \"jobId\",\n      ${runningJobExecutions.jobExecutionId} AS \"jobExecutionId\",\n      ${runningJobExecutions.providerRunnerId} AS \"providerRunnerId\",\n      ${runningJobExecutions.startedAt} AS \"startedAt\",\n      ${runningJobExecutions.lastHeartbeatAt} AS \"lastHeartbeatAt\",\n      ${runningJobExecutions.cancellationRequestedAt} AS \"cancellationRequestedAt\"\n    FROM ${runningJobExecutions}\n    WHERE\n      ${runningJobExecutions.workspaceId} = ${params.workspaceId}\n      AND ${runningJobExecutions.provisionerId} = ${params.provisionerId}\n      AND ${runningJobExecutions.providerRunnerId} IN (${sql.join(\n        params.providerRunnerIds.map((providerRunnerId) => sql`${providerRunnerId}`),\n        sql`, `,\n      )})\n    ORDER BY ${runningJobExecutions.providerRunnerId}, ${runningJobExecutions.startedAt} DESC, ${runningJobExecutions.jobExecutionId} DESC\n  `);\n\n  return result.rows.map((row) => ({\n    workflowRunId: row.workflowRunId,\n    workflowRunAttemptId: row.workflowRunAttemptId,\n    jobId: row.jobId,\n    jobExecutionId: row.jobExecutionId,\n    providerRunnerId: row.providerRunnerId,\n    startedAt: toDate(row.startedAt),\n    lastHeartbeatAt: toDate(row.lastHeartbeatAt),\n    cancellationRequestedAt: row.cancellationRequestedAt\n      ? toDate(row.cancellationRequestedAt)\n      : null,\n  }));\n}\n\nfunction toDate(value: Date | string): Date {\n  return value instanceof Date ? value : new Date(value);\n}\n\nexport async function isJobLeaseActive(params: {\n  jobId?: string;\n  jobExecutionId: string;\n  runnerSessionId: string;\n}): Promise<boolean> {\n  const [row] = await db()\n    .select({id: runningJobExecutions.id})\n    .from(runningJobExecutions)\n    .where(\n      and(\n        eq(runningJobExecutions.jobExecutionId, params.jobExecutionId),\n        eq(runningJobExecutions.runnerSessionId, params.runnerSessionId),\n        params.jobId === undefined ? undefined : eq(runningJobExecutions.jobId, params.jobId),\n      ),\n    )\n    .limit(1);\n\n  return row !== undefined;\n}\n\nexport async function recordHeartbeat(params: {\n  jobExecutionId: string;\n  runnerSessionId: string;\n  toolCapabilities?: RunnerToolCapabilitiesDto | null;\n}): Promise<{\n  cancellationRequested: boolean;\n  previousToolCapabilities: RunnerToolCapabilitiesDto | null;\n  currentToolCapabilities: RunnerToolCapabilitiesDto | null;\n  runningJobExecution: {\n    workflowRunId: string;\n    workflowRunAttemptId: string;\n    jobId: string;\n    jobExecutionId: string;\n    projectId: string;\n    workspaceId: string;\n    runnerSessionId: string;\n  };\n}> {\n  const result = await db().transaction(async (tx) => {\n    const updated = await tx\n      .update(runningJobExecutions)\n      .set({\n        firstHeartbeatAt: sql`COALESCE(${runningJobExecutions.firstHeartbeatAt}, now())`,\n        lastHeartbeatAt: sql`now()`,\n      })\n      .where(\n        and(\n          eq(runningJobExecutions.jobExecutionId, params.jobExecutionId),\n          eq(runningJobExecutions.runnerSessionId, params.runnerSessionId),\n        ),\n      )\n      .returning({\n        cancellationRequestedAt: runningJobExecutions.cancellationRequestedAt,\n        workflowRunId: runningJobExecutions.workflowRunId,\n        workflowRunAttemptId: runningJobExecutions.workflowRunAttemptId,\n        jobId: runningJobExecutions.jobId,\n        jobExecutionId: runningJobExecutions.jobExecutionId,\n        projectId: runningJobExecutions.projectId,\n        workspaceId: runningJobExecutions.workspaceId,\n        runnerSessionId: runningJobExecutions.runnerSessionId,\n      });\n\n    const row = updated[0];\n    if (!row) throw new RunningJobExecutionNotFoundError(params.jobExecutionId);\n\n    const [previous] = await tx\n      .select({toolCapabilities: runnerSessions.toolCapabilities})\n      .from(runnerSessions)\n      .where(eq(runnerSessions.id, params.runnerSessionId))\n      .limit(1)\n      .for('update');\n\n    const [session] = await tx\n      .update(runnerSessions)\n      .set({\n        toolCapabilities: params.toolCapabilities ?? null,\n        toolCapabilitiesReportedAt: params.toolCapabilities ? sql`now()` : null,\n        updatedAt: sql`now()`,\n      })\n      .where(eq(runnerSessions.id, params.runnerSessionId))\n      .returning({toolCapabilities: runnerSessions.toolCapabilities});\n\n    return {\n      row,\n      previousToolCapabilities: previous?.toolCapabilities ?? null,\n      currentToolCapabilities: session?.toolCapabilities ?? null,\n    };\n  });\n\n  const row = result.row;\n  if (!row) throw new RunningJobExecutionNotFoundError(params.jobExecutionId);\n  return {\n    cancellationRequested: row.cancellationRequestedAt !== null,\n    previousToolCapabilities: result.previousToolCapabilities,\n    currentToolCapabilities: result.currentToolCapabilities,\n    runningJobExecution: {\n      workflowRunId: row.workflowRunId,\n      workflowRunAttemptId: row.workflowRunAttemptId,\n      jobId: row.jobId,\n      jobExecutionId: row.jobExecutionId,\n      projectId: row.projectId,\n      workspaceId: row.workspaceId,\n      runnerSessionId: row.runnerSessionId,\n    },\n  };\n}\n\n/**\n * Reconciles a terminal job execution with runner state in one transaction: removes its pending\n * queue row, requests cancellation on its running lease, and converges any linked reservation.\n * The operation is idempotent and preserves the first cancellation-request timestamp.\n */\nexport async function reconcileTerminalJobExecution(params: {\n  jobExecutionId: string;\n}): Promise<void> {\n  await db().transaction(async (tx) => {\n    await lockJobExecution(tx, params.jobExecutionId);\n\n    // Delete pending before updating running to match claim/release lock order. Claim locks\n    // pending rows with SKIP LOCKED before inserting the running lease, so this ordering makes\n    // a concurrent terminal reconciliation either win before claim or cancel the new lease.\n    await tx\n      .delete(pendingJobExecutions)\n      .where(eq(pendingJobExecutions.jobExecutionId, params.jobExecutionId));\n    const cancelledRunningRows = await tx\n      .update(runningJobExecutions)\n      .set({\n        cancellationRequestedAt: sql`COALESCE(${runningJobExecutions.cancellationRequestedAt}, now())`,\n      })\n      .where(eq(runningJobExecutions.jobExecutionId, params.jobExecutionId))\n      .returning({\n        provisionerId: runningJobExecutions.provisionerId,\n        providerRunnerId: runningJobExecutions.providerRunnerId,\n      });\n\n    await releaseReservationsForTerminalRunningRows(tx, cancelledRunningRows);\n  });\n}\n"],"names":["RUNNER_JOB_CLAIMED","RUNNER_JOB_LEASE_EXPIRED","logger","writeOutboxEvent","writeOutboxEvents","canonicalizeLabels","and","arrayContained","asc","count","desc","eq","gt","inArray","isNotNull","isNull","lt","lte","or","sql","EmptyRequiredLabelsError","RunnerSessionExhaustedError","RunningJobExecutionNotFoundError","jobExecutionEnqueuedCount","jobExecutionLeaseExpiredCount","recordJobExecutionQueueTime","recordProviderRunnerActivationToFirstClaim","db","lockRunnerReservationAdvisoryKeysTx","releaseReservationUnits","releaseTerminalRunnerInstanceReservationsByIds","runnersOutbox","pendingJobExecutions","reservations","providerRunners","runnerSessions","runningJobExecutions","runnerJobExecutionLockPrefix","lockJobExecution","tx","jobExecutionId","execute","releaseReservationsForTerminalRunningRows","rows","providerRunnerIdsByProvisionerId","Map","row","provisionerId","providerRunnerId","providerRunnerIds","get","Set","add","set","keys","sort","workspaceId","requireUnlinkedSession","requireTerminalState","getWorkspaceJobCounts","params","pendingRows","runningRows","Promise","all","select","from","where","workspaceIds","groupBy","queuedByWorkspace","map","Number","runningByWorkspace","queued","running","enqueueJobExecution","requiredLabels","length","enqueued","transaction","limit","leaseExpired","id","eventType","payload","inserted","insert","values","workflowRunId","workflowRunAttemptId","jobId","projectId","createdAt","queuedAt","onConflictDoNothing","target","returning","undefined","claimPendingJobExecution","touchRunnerSessionLiveness","runnerSessionId","throttleSeconds","runnerSessionLivenessThrottleSeconds","sessionLabels","activationToFirstClaimObservation","queueTimeObservation","result","runnerInstanceId","maxClaims","session","claimsUsed","revokedAt","for","runnerInstanceCondition","runner","reservationId","intendedReservationId","reservationIds","filter","candidate","orderBy","skipLocked","as","delete","runnerLabels","claimedAt","startedAt","claimed","durationMilliseconds","getTime","provider","launchKind","claimedRunner","update","firstClaimedAt","updatedAt","isFirstClaim","providerKind","sessionCreatedAtEpochMs","releasedRunner","reservationReleasedAt","reservation","releases","type","toISOString","expireStuckJobExecutions","reaped","heartbeatCutoff","thresholdSeconds","firstHeartbeatCutoff","noFirstHeartbeatGraceSeconds","stalePredicate","firstHeartbeatAt","lastHeartbeatAt","staleRows","staleIds","staleJobExecutionIds","deleted","getJobExecutionQueueDepth","pending","value","listActiveRunningJobExecutions","windowSeconds","listRunningJobExecutionsByRunnerInstanceTx","duplicateRows","having","duplicateRunnerInstanceIds","flatMap","warn","cancellationRequestedAt","join","toDate","Date","isJobLeaseActive","recordHeartbeat","updated","previous","toolCapabilities","toolCapabilitiesReportedAt","previousToolCapabilities","currentToolCapabilities","cancellationRequested","runningJobExecution","reconcileTerminalJobExecution","cancelledRunningRows"],"mappings":"AAAA,SACEA,kBAAkB,EAClBC,wBAAwB,QAGnB,2BAA2B;AAClC,SAAQC,MAAM,QAAO,8BAA8B;AACnD,SAAQC,gBAAgB,EAAEC,iBAAiB,QAAO,uBAAuB;AACzE,SAAQC,kBAAkB,QAAO,yBAAyB;AAC1D,SACEC,GAAG,EACHC,cAAc,EACdC,GAAG,EACHC,KAAK,EACLC,IAAI,EACJC,EAAE,EACFC,EAAE,EACFC,OAAO,EACPC,SAAS,EACTC,MAAM,EACNC,EAAE,EACFC,GAAG,EACHC,EAAE,EACFC,GAAG,QACE,cAAc;AACrB,SACEC,wBAAwB,EACxBC,2BAA2B,EAC3BC,gCAAgC,QAC3B,kBAAkB;AACzB,SAEEC,yBAAyB,EACzBC,6BAA6B,EAE7BC,2BAA2B,EAC3BC,0CAA0C,QACrC,uBAAuB;AAE9B,SAAQC,EAAE,QAAO,UAAU;AAC3B,SAAQC,mCAAmC,QAAO,yBAAyB;AAC3E,SACEC,uBAAuB,EACvBC,8CAA8C,QACzC,oBAAoB;AAC3B,SAAQC,aAAa,QAAO,qBAAqB;AACjD,SAAQC,oBAAoB,QAAO,qCAAqC;AACxE,SAAQC,YAAY,QAAO,2BAA2B;AACtD,SAAQC,eAAe,QAAO,+BAA+B;AAC7D,SAAQC,cAAc,QAAO,8BAA8B;AAC3D,SAAQC,oBAAoB,QAAO,qCAAqC;AAExE,MAAMC,+BAA+B;AAErC,eAAeC,iBAAiBC,EAAM,EAAEC,cAAsB;IAC5D,MAAMD,GAAGE,OAAO,CACdtB,GAAG,CAAC,sCAAsC,EAAE,GAAGkB,+BAA+BG,gBAAgB,CAAC,EAAE,CAAC;AAEtG;AAEA,eAAeE,0CACbH,EAAM,EACNI,IAGE;IAEF,2FAA2F;IAC3F,kFAAkF;IAClF,MAAMC,mCAAmC,IAAIC;IAC7C,KAAK,MAAMC,OAAOH,KAAM;QACtB,IAAIG,IAAIC,aAAa,KAAK,QAAQD,IAAIE,gBAAgB,KAAK,MAAM;QACjE,MAAMC,oBACJL,iCAAiCM,GAAG,CAACJ,IAAIC,aAAa,KAAK,IAAII;QACjEF,kBAAkBG,GAAG,CAACN,IAAIE,gBAAgB;QAC1CJ,iCAAiCS,GAAG,CAACP,IAAIC,aAAa,EAAEE;IAC1D;IAEA,KAAK,MAAMF,iBAAiB;WAAIH,iCAAiCU,IAAI;KAAG,CAACC,IAAI,GAAI;QAC/E,MAAMN,oBAAoBL,iCAAiCM,GAAG,CAACH;QAC/D,IAAI,CAACE,mBAAmB;QACxB,MAAMnB,+CAA+CS,IAAI;YACvDiB,aAAa;YACbT;YACAE,mBAAmB;mBAAIA;aAAkB,CAACM,IAAI;YAC9CE,wBAAwB;YACxB,mFAAmF;YACnF,wCAAwC;YACxCC,sBAAsB;QACxB;IACF;AACF;AAaA,OAAO,eAAeC,sBAAsBC,MAE3C;IACC,MAAM,CAACC,aAAaC,YAAY,GAAG,MAAMC,QAAQC,GAAG,CAAC;QACnDrC,KACGsC,MAAM,CAAC;YAACT,aAAaxB,qBAAqBwB,WAAW;YAAE/C,OAAOA;QAAO,GACrEyD,IAAI,CAAClC,sBACLmC,KAAK,CAACtD,QAAQmB,qBAAqBwB,WAAW,EAAEI,OAAOQ,YAAY,GACnEC,OAAO,CAACrC,qBAAqBwB,WAAW;QAC3C7B,KACGsC,MAAM,CAAC;YAACT,aAAapB,qBAAqBoB,WAAW;YAAE/C,OAAOA;QAAO,GACrEyD,IAAI,CAAC9B,sBACL+B,KAAK,CAACtD,QAAQuB,qBAAqBoB,WAAW,EAAEI,OAAOQ,YAAY,GACnEC,OAAO,CAACjC,qBAAqBoB,WAAW;KAC5C;IAED,MAAMc,oBAAoB,IAAIzB,IAAIgB,YAAYU,GAAG,CAAC,CAACzB,MAAQ;YAACA,IAAIU,WAAW;YAAEgB,OAAO1B,IAAIrC,KAAK;SAAE;IAC/F,MAAMgE,qBAAqB,IAAI5B,IAC7BiB,YAAYS,GAAG,CAAC,CAACzB,MAAQ;YAACA,IAAIU,WAAW;YAAEgB,OAAO1B,IAAIrC,KAAK;SAAE;IAG/D,OAAOmD,OAAOQ,YAAY,CAACG,GAAG,CAAC,CAACf,cAAiB,CAAA;YAC/CA;YACAkB,QAAQJ,kBAAkBpB,GAAG,CAACM,gBAAgB;YAC9CmB,SAASF,mBAAmBvB,GAAG,CAACM,gBAAgB;QAClD,CAAA;AACF;AAEA,+FAA+F;AAC/F,kGAAkG;AAClG,8FAA8F;AAC9F,OAAO,eAAeoB,oBAAoBhB,MAAiC;IACzE,MAAMiB,iBAAiB;WAAIxE,mBAAmBuD,OAAOiB,cAAc;KAAE;IACrE,IAAIA,eAAeC,MAAM,KAAK,GAAG,MAAM,IAAI1D;IAE3C,MAAM2D,WAAW,MAAMpD,KAAKqD,WAAW,CAAC,OAAOzC;QAC7C,MAAMD,iBAAiBC,IAAIqB,OAAOpB,cAAc;QAEhD,MAAM,CAACmC,QAAQ,GAAG,MAAMpC,GACrB0B,MAAM,CAAC;YAACzB,gBAAgBJ,qBAAqBI,cAAc;QAAA,GAC3D0B,IAAI,CAAC9B,sBACL+B,KAAK,CAACxD,GAAGyB,qBAAqBI,cAAc,EAAEoB,OAAOpB,cAAc,GACnEyC,KAAK,CAAC;QACT,IAAIN,SAAS,OAAO;QAEpB,MAAM,CAACO,aAAa,GAAG,MAAM3C,GAC1B0B,MAAM,CAAC;YAACkB,IAAIpD,cAAcoD,EAAE;QAAA,GAC5BjB,IAAI,CAACnC,eACLoC,KAAK,CACJ7D,IACEK,GAAGoB,cAAcqD,SAAS,EAAEnF,2BAC5BkB,GAAG,CAAC,EAAEY,cAAcsD,OAAO,CAAC,sBAAsB,EAAEzB,OAAOpB,cAAc,CAAC,CAAC,GAG9EyC,KAAK,CAAC;QACT,IAAIC,cAAc,OAAO;QAEzB,MAAM,CAACI,SAAS,GAAG,MAAM/C,GACtBgD,MAAM,CAACvD,sBACPwD,MAAM,CAAC;YACNhC,aAAaI,OAAOJ,WAAW;YAC/BiC,eAAe7B,OAAO6B,aAAa;YACnCC,sBAAsB9B,OAAO8B,oBAAoB;YACjDC,OAAO/B,OAAO+B,KAAK;YACnBnD,gBAAgBoB,OAAOpB,cAAc;YACrCoD,WAAWhC,OAAOgC,SAAS;YAC3Bf;YACAgB,WAAWjC,OAAOkC,QAAQ;QAC5B,GACCC,mBAAmB,CAAC;YAACC,QAAQhE,qBAAqBQ,cAAc;QAAA,GAChEyD,SAAS,CAAC;YAACJ,WAAW7D,qBAAqB6D,SAAS;QAAA;QAEvD,OAAOP,aAAaY;IACtB;IAEA,IAAInB,UAAUxD,0BAA0B6B,GAAG,CAAC;AAC9C;AAoCA,OAAO,eAAe+C,yBAAyBvC,MAM9C;IACC,MAAMwC,2BAA2B;QAC/B5C,aAAaI,OAAOJ,WAAW;QAC/B6C,iBAAiBzC,OAAOyC,eAAe;QACvCC,iBAAiB1C,OAAO2C,oCAAoC;IAC9D;IAEA,IAAI3C,OAAO4C,aAAa,CAAC1B,MAAM,KAAK,GAAG,OAAO;IAE9C,IAAI2B,oCAA+E;IACnF,IAAIC,uBAAgE;IACpE,MAAMC,SAAS,MAAMhF,KAAKqD,WAAW,CAAC,OAAOzC;QAC3C,IAAIQ,gBAA+B;QACnC,IAAIC,mBAAkC;QACtC,IAAI4D,mBAAkC;QAEtC,IAAIhD,OAAOiD,SAAS,KAAK,MAAM;YAC7B,MAAM,CAACC,QAAQ,GAAG,MAAMvE,GACrB0B,MAAM,CAAC;gBACN4C,WAAW1E,eAAe0E,SAAS;gBACnCE,YAAY5E,eAAe4E,UAAU;gBACrCC,WAAW7E,eAAe6E,SAAS;gBACnCJ,kBAAkBzE,eAAeyE,gBAAgB;gBACjD7D,eAAeZ,eAAeY,aAAa;gBAC3CC,kBAAkBb,eAAea,gBAAgB;YACnD,GACCkB,IAAI,CAAC/B,gBACLgC,KAAK,CAACxD,GAAGwB,eAAegD,EAAE,EAAEvB,OAAOyC,eAAe,GAClDpB,KAAK,CAAC,GACNgC,GAAG,CAAC;YAEP,IACE,CAACH,WACDA,QAAQE,SAAS,IACjBF,QAAQD,SAAS,KAAK,QACtBC,QAAQC,UAAU,IAAID,QAAQD,SAAS,EACvC;gBACA,MAAM,IAAIxF,4BAA4BuC,OAAOyC,eAAe;YAC9D;YAEA,0EAA0E;YAC1E,mDAAmD;YACnDO,mBAAmBE,QAAQF,gBAAgB;YAC3C7D,gBAAgB+D,QAAQ/D,aAAa;YACrCC,mBAAmB8D,QAAQ9D,gBAAgB;QAC7C;QAEA,MAAMkE,0BAA0BN,mBAC5BjG,GAAGuB,gBAAgBiD,EAAE,EAAEyB,oBACvB7D,iBAAiBC,mBACf1C,IACEK,GAAGuB,gBAAgBa,aAAa,EAAEA,gBAClCpC,GAAGuB,gBAAgBc,gBAAgB,EAAEA,qBAEvC;QACN,IAAIkE,2BAA2BnE,eAAe;YAC5C,MAAM,CAACoE,OAAO,GAAG,MAAM5E,GACpB0B,MAAM,CAAC;gBACNmD,eAAelF,gBAAgBkF,aAAa;gBAC5CC,uBAAuBnF,gBAAgBmF,qBAAqB;YAC9D,GACCnD,IAAI,CAAChC,iBACLiC,KAAK,CAAC+C,yBACNjC,KAAK,CAAC;YACT,MAAMrD,oCAAoCW,IAAI;gBAC5CQ;gBACAuE,gBAAgB;oBAACH,QAAQC;oBAAeD,QAAQE;iBAAsB,CAACE,MAAM,CAC3E,CAACH,gBACCA,kBAAkB,QAAQA,kBAAkBlB;YAElD;QACF;QAEA,4EAA4E;QAC5E,oFAAoF;QACpF,oFAAoF;QACpF,sFAAsF;QACtF,MAAMsB,YAAYjF,GACf0B,MAAM,GACNC,IAAI,CAAClC,sBACLmC,KAAK,CACJ7D,IACEK,GAAGqB,qBAAqBwB,WAAW,EAAEI,OAAOJ,WAAW,GACvDjD,eAAeyB,qBAAqB6C,cAAc,EAAEjB,OAAO4C,aAAa,IAG3EiB,OAAO,CAACjH,IAAIwB,qBAAqB6D,SAAS,GAAGrF,IAAIwB,qBAAqBmD,EAAE,GACxEF,KAAK,CAAC,GACNgC,GAAG,CAAC,UAAU;YAACS,YAAY;QAAI,GAC/BC,EAAE,CAAC;QAEN,MAAM,CAAC7E,IAAI,GAAG,MAAMP,GACjB0B,MAAM,GACNC,IAAI,CAACsD,WACLrD,KAAK,CACJhD,GAAG,CAAC;;qBAES,EAAEkB,6BAA6B,IAAI,EAAEmF,UAAUhF,cAAc,CAAC;;QAE3E,CAAC;QAGL,IAAI,CAACM,KAAK,OAAO;QAEjB,MAAMP,GAAGqF,MAAM,CAAC5F,sBAAsBmC,KAAK,CAACxD,GAAGqB,qBAAqBmD,EAAE,EAAErC,IAAIqC,EAAE;QAE9E,8EAA8E;QAC9E,gFAAgF;QAChF,qFAAqF;QACrF,2EAA2E;QAC3E,8EAA8E;QAC9E,2DAA2D;QAC3D,MAAMG,WAAW,MAAM/C,GACpBgD,MAAM,CAACnD,sBACPoD,MAAM,CAAC;YACNhC,aAAaV,IAAIU,WAAW;YAC5BiC,eAAe3C,IAAI2C,aAAa;YAChCC,sBAAsB5C,IAAI4C,oBAAoB;YAC9CC,OAAO7C,IAAI6C,KAAK;YAChBnD,gBAAgBM,IAAIN,cAAc;YAClCoD,WAAW9C,IAAI8C,SAAS;YACxBS,iBAAiBzC,OAAOyC,eAAe;YACvCtD;YACAC;YACA6B,gBAAgB/B,IAAI+B,cAAc;YAClCgD,cAAcjE,OAAO4C,aAAa;QACpC,GACCT,mBAAmB,CAAC;YAACC,QAAQ5D,qBAAqBI,cAAc;QAAA,GAChEyD,SAAS,CAAC;YAAC6B,WAAW1F,qBAAqB2F,SAAS;QAAA;QAEvD,MAAMC,UAAU1C,QAAQ,CAAC,EAAE;QAC3B,IAAI,CAAC0C,SAAS,OAAO;QAErBtB,uBAAuB;YACrBuB,sBAAsBD,QAAQF,SAAS,CAACI,OAAO,KAAKpF,IAAI+C,SAAS,CAACqC,OAAO;YACzEC,UAAU;YACVC,YAAYxE,OAAOiD,SAAS,KAAK,OAAO,WAAW;QACrD;QAEA,IAAIK,yBAAyB;YAC3B,MAAM,CAACmB,cAAc,GAAG,MAAM9F,GAC3B+F,MAAM,CAACpG,iBACPmB,GAAG,CAAC;gBACHkF,gBAAgBpH,GAAG,CAAC,SAAS,EAAEe,gBAAgBqG,cAAc,CAAC,EAAE,EAAEP,QAAQF,SAAS,CAAC,CAAC,CAAC;gBACtFU,WAAWrH,GAAG,CAAC,KAAK,CAAC;YACvB,GACCgD,KAAK,CAAC+C,yBACNjB,SAAS,CAAC;gBACTsC,gBAAgBrG,gBAAgBqG,cAAc;gBAC9CE,cAActH,GAAY,CAAC,EAAEe,gBAAgBqG,cAAc,CAAC,GAAG,EAAEP,QAAQF,SAAS,CAAC,CAAC;gBACpFK,UAAUjG,gBAAgBwG,YAAY;gBACtCN,YAAYlG,gBAAgBkG,UAAU;gBACtCxB,kBAAkB1E,gBAAgBiD,EAAE;gBACpCiC,eAAelF,gBAAgBkF,aAAa;gBAC5CC,uBAAuBnF,gBAAgBmF,qBAAqB;gBAC5DsB,yBAAyBxH,GAAkB,CAAC;sCAChB,EAAEgB,eAAe0D,SAAS,CAAC;iBAChD,EAAE1D,eAAe;kBAChB,EAAEA,eAAegD,EAAE,CAAC,GAAG,EAAEvB,OAAOyC,eAAe,CAAC;WACvD,CAAC;YACJ;YACF,IAAIgC,iBAAiB3B,sBAAsB;gBACzCA,qBAAqByB,QAAQ,GAAGE,cAAcF,QAAQ;gBACtDzB,qBAAqB0B,UAAU,GAAGC,cAAcD,UAAU;YAC5D;YACA,IAAIC,eAAeI,cAAc;gBAC/B,MAAM,CAACG,eAAe,GAAG,MAAMrG,GAC5B+F,MAAM,CAACpG,iBACPmB,GAAG,CAAC;oBACHgE,uBAAuB;oBACvBwB,uBAAuB1H,GAAG,CAAC,KAAK,CAAC;oBACjCqH,WAAWrH,GAAG,CAAC,KAAK,CAAC;gBACvB,GACCgD,KAAK,CACJ7D,IACEK,GAAGuB,gBAAgBiD,EAAE,EAAEkD,cAAczB,gBAAgB,GACrD1F,GACEJ,UAAUoB,gBAAgBkF,aAAa,GACvCtG,UAAUoB,gBAAgBmF,qBAAqB,IAEjDtG,OAAOmB,gBAAgB2G,qBAAqB,IAG/C5C,SAAS,CAAC;oBACTlD,eAAeb,gBAAgBa,aAAa;gBAC9C;gBACF,MAAMqE,gBAAgBiB,cAAchB,qBAAqB,IAAIgB,cAAcjB,aAAa;gBACxF,IAAIwB,gBAAgB7F,iBAAiBqE,eAAe;oBAClD,MAAM,CAAC0B,YAAY,GAAG,MAAMvG,GACzB0B,MAAM,CAAC;wBAACT,aAAavB,aAAauB,WAAW;oBAAA,GAC7CU,IAAI,CAACjC,cACLkC,KAAK,CACJ7D,IACEK,GAAGsB,aAAakD,EAAE,EAAEiC,gBACpBzG,GAAGsB,aAAac,aAAa,EAAE6F,eAAe7F,aAAa,IAG9DkC,KAAK,CAAC;oBACT,IAAI6D,aACF,MAAMjH,wBAAwBU,IAAI;wBAChCiB,aAAasF,YAAYtF,WAAW;wBACpCT,eAAe6F,eAAe7F,aAAa;wBAC3CgG,UAAU;4BAAC;gCAAC3B;gCAAe3G,OAAO;4BAAC;yBAAE;oBACvC;gBACJ;YACF;YACA,IACE4H,eAAeI,gBACfJ,cAAcE,cAAc,KAAK,QACjCF,cAAcM,uBAAuB,KAAK,QAC1CN,cAAcM,uBAAuB,KAAKzC,WAE1CO,oCAAoC;gBAClCwB,sBACEI,cAAcE,cAAc,CAACL,OAAO,KAAKG,cAAcM,uBAAuB;gBAChFR,UAAUE,cAAcF,QAAQ;gBAChCC,YAAYC,cAAcD,UAAU;gBACpCxB,kBAAkByB,cAAczB,gBAAgB;YAClD;QACJ;QAEA,IAAIhD,OAAOiD,SAAS,KAAK,MAAM;YAC7B,MAAMtE,GACH+F,MAAM,CAACnG,gBACPkB,GAAG,CAAC;gBAAC0D,YAAY5F,GAAG,CAAC,EAAEgB,eAAe4E,UAAU,CAAC,IAAI,CAAC;gBAAEyB,WAAWrH,GAAG,CAAC,KAAK,CAAC;YAAA,GAC7EgD,KAAK,CAACxD,GAAGwB,eAAegD,EAAE,EAAEvB,OAAOyC,eAAe;QACvD;QAEA,4FAA4F;QAC5F,mFAAmF;QACnF,6BAA6B;QAC7B,MAAMlG,iBAAkCoC,IAAIR,eAAe;YACzDiH,MAAMhJ;YACNqF,SAAS;gBACPI,eAAe3C,IAAI2C,aAAa;gBAChCC,sBAAsB5C,IAAI4C,oBAAoB;gBAC9CC,OAAO7C,IAAI6C,KAAK;gBAChBnD,gBAAgBM,IAAIN,cAAc;gBAClCsF,WAAWE,QAAQF,SAAS,CAACmB,WAAW;YAC1C;QACF;QAEA,OAAO;YACLxD,eAAe3C,IAAI2C,aAAa;YAChCC,sBAAsB5C,IAAI4C,oBAAoB;YAC9CC,OAAO7C,IAAI6C,KAAK;YAChBnD,gBAAgBM,IAAIN,cAAc;YAClCoD,WAAW9C,IAAI8C,SAAS;QAC1B;IACF;IACA,IAAIc,sBAAsBjF,4BAA4BiF;IACtD,IAAID,mCACF/E,2CAA2C+E;IAC7C,OAAOE;AACT;AAEA,eAAeP,2BAA2BxC,MAIzC;IACC,MAAMjC,KACH2G,MAAM,CAACnG,gBACPkB,GAAG,CAAC;QAACmF,WAAWrH,GAAG,CAAC,KAAK,CAAC;IAAA,GAC1BgD,KAAK,CACJ7D,IACEK,GAAGwB,eAAegD,EAAE,EAAEvB,OAAOyC,eAAe,GAC5C1F,GAAGwB,eAAeqB,WAAW,EAAEI,OAAOJ,WAAW,GACjDrC,GAAG,CAAC,EAAEgB,eAAeqG,SAAS,CAAC,YAAY,EAAE5E,OAAO0C,eAAe,CAAC,yBAAyB,CAAC;AAGtG;AAEA;;;;;;;;;;;;CAYC,GACD,OAAO,eAAe4C,yBAAyBtF,MAI9C;IAQC,MAAMuF,SAAS,MAAMxH,KAAKqD,WAAW,CAAC,OAAOzC;QAC3C,MAAM6G,kBAAkBjI,GAAG,CAAC,SAAS,EAAEyC,OAAOyF,gBAAgB,CAAC,yBAAyB,CAAC;QACzF,MAAMC,uBAAuBnI,GAAG,CAAC,SAAS,EAAEyC,OAAO2F,4BAA4B,CAAC,yBAAyB,CAAC;QAC1G,MAAMC,iBAAiBtI,GACrBZ,IACES,OAAOqB,qBAAqBqH,gBAAgB,GAC5CxI,IAAImB,qBAAqBsH,eAAe,EAAEtH,qBAAqB2F,SAAS,GACxE/G,GAAGoB,qBAAqB2F,SAAS,EAAEuB,wBAErChJ,IACEY,GACEJ,UAAUsB,qBAAqBqH,gBAAgB,GAC/C7I,GAAGwB,qBAAqBsH,eAAe,EAAEtH,qBAAqB2F,SAAS,IAEzE/G,GAAGoB,qBAAqBsH,eAAe,EAAEN;QAI7C,MAAMO,YAAY,MAAMpH,GACrB0B,MAAM,CAAC;YACNkB,IAAI/C,qBAAqB+C,EAAE;YAC3B3C,gBAAgBJ,qBAAqBI,cAAc;QACrD,GACC0B,IAAI,CAAC9B,sBACL+B,KAAK,CACJ7D,IACEkJ,gBACArI,GAAG,CAAC;;uBAES,EAAEkB,6BAA6B,IAAI,EAAED,qBAAqBI,cAAc,CAAC;;UAEtF,CAAC,GAGJiF,OAAO,CACNjH,IACEW,GAAG,CAAC,UAAU,EAAEiB,qBAAqBqH,gBAAgB,CAAC,aAAa,EAAErH,qBAAqBsH,eAAe,CAAC,IAAI,EAAEtH,qBAAqB2F,SAAS,CAAC,MAAM,EAAE3F,qBAAqB2F,SAAS,CAAC,MAAM,EAAE3F,qBAAqBsH,eAAe,CAAC,IAAI,CAAC,GAE1OlJ,IAAI4B,qBAAqB+C,EAAE,GAE5BF,KAAK,CAACrB,OAAOqB,KAAK,IAAI;QAEzB,IAAI0E,UAAU7E,MAAM,KAAK,GAAG,OAAO,EAAE;QAErC,MAAM8E,WAAWD,UAAUpF,GAAG,CAAC,CAACzB,MAAQA,IAAIqC,EAAE;QAC9C,MAAM0E,uBAAuBF,UAAUpF,GAAG,CAAC,CAACzB,MAAQA,IAAIN,cAAc;QAEtE,4FAA4F;QAC5F,yFAAyF;QACzF,0FAA0F;QAC1F,yFAAyF;QACzF,MAAMD,GACHqF,MAAM,CAAC5F,sBACPmC,KAAK,CAACtD,QAAQmB,qBAAqBQ,cAAc,EAAEqH;QAEtD,MAAMC,UAAU,MAAMvH,GACnBqF,MAAM,CAACxF,sBACP+B,KAAK,CAAC7D,IAAIO,QAAQuB,qBAAqB+C,EAAE,EAAEyE,WAAWJ,iBACtDvD,SAAS,CAAC;YACTR,eAAerD,qBAAqBqD,aAAa;YACjDC,sBAAsBtD,qBAAqBsD,oBAAoB;YAC/DC,OAAOvD,qBAAqBuD,KAAK;YACjCnD,gBAAgBJ,qBAAqBI,cAAc;YACnDO,eAAeX,qBAAqBW,aAAa;YACjDC,kBAAkBZ,qBAAqBY,gBAAgB;QACzD;QAEF,IAAI8G,QAAQhF,MAAM,KAAK,GAAG,OAAO,EAAE;QAEnC,MAAMpC,0CAA0CH,IAAIuH;QAEpD,MAAM1J,kBACJmC,IACAR,eACA+H,QAAQvF,GAAG,CAAC,CAACzB,MAAS,CAAA;gBACpBkG,MAAM/I;gBACNoF,SAAS;oBACPI,eAAe3C,IAAI2C,aAAa;oBAChCC,sBAAsB5C,IAAI4C,oBAAoB;oBAC9CC,OAAO7C,IAAI6C,KAAK;oBAChBnD,gBAAgBM,IAAIN,cAAc;gBACpC;YACF,CAAA;QAGF,OAAOsH,QAAQvF,GAAG,CAAC,CAAC,EAACkB,aAAa,EAAEC,oBAAoB,EAAEC,KAAK,EAAEnD,cAAc,EAAC,GAAM,CAAA;gBACpFiD;gBACAC;gBACAC;gBACAnD;YACF,CAAA;IACF;IAEA,IAAI2G,OAAOrE,MAAM,GAAG,GAAGtD,8BAA8B4B,GAAG,CAAC+F,OAAOrE,MAAM;IAEtE,OAAOqE;AACT;AAEA,OAAO,eAAeY;IAIpB,MAAM,CAACC,QAAQ,GAAG,MAAMrI,KAAKsC,MAAM,CAAC;QAACgG,OAAOxJ;IAAO,GAAGyD,IAAI,CAAClC;IAC3D,MAAM,CAAC2C,QAAQ,GAAG,MAAMhD,KAAKsC,MAAM,CAAC;QAACgG,OAAOxJ;IAAO,GAAGyD,IAAI,CAAC9B;IAC3D,OAAO;QACLJ,sBAAsBgI,SAASC,SAAS;QACxC7H,sBAAsBuC,SAASsF,SAAS;IAC1C;AACF;AAEA,OAAO,eAAeC,+BAA+BtG,MAIpD;IACC,OAAO,MAAMjC,KACVsC,MAAM,CAAC;QACN0B,OAAOvD,qBAAqBuD,KAAK;QACjCnD,gBAAgBJ,qBAAqBI,cAAc;QACnDiD,eAAerD,qBAAqBqD,aAAa;QACjDC,sBAAsBtD,qBAAqBsD,oBAAoB;QAC/DE,WAAWxD,qBAAqBwD,SAAS;QACzCS,iBAAiBjE,qBAAqBiE,eAAe;QACrDtD,eAAeX,qBAAqBW,aAAa;QACjDC,kBAAkBZ,qBAAqBY,gBAAgB;QACvD6B,gBAAgBzC,qBAAqByC,cAAc;QACnDgD,cAAczF,qBAAqByF,YAAY;QAC/CE,WAAW3F,qBAAqB2F,SAAS;QACzC2B,iBAAiBtH,qBAAqBsH,eAAe;IACvD,GACCxF,IAAI,CAAC9B,sBACL+B,KAAK,CACJ7D,IACEK,GAAGyB,qBAAqBoB,WAAW,EAAEI,OAAOJ,WAAW,GACvDrC,GAAG,CAAC,EAAEiB,qBAAqBsH,eAAe,CAAC,YAAY,EAAE9F,OAAOuG,aAAa,CAAC,yBAAyB,CAAC,GAG3G1C,OAAO,CAAC/G,KAAK0B,qBAAqBsH,eAAe,GAAGhJ,KAAK0B,qBAAqB+C,EAAE,GAChFF,KAAK,CAACrB,OAAOqB,KAAK,IAAI;AAC3B;AAEA,OAAO,eAAemF,2CACpB7H,EAAM,EACNqB,MAIC;IAED,IAAIA,OAAOX,iBAAiB,CAAC6B,MAAM,KAAK,GAAG,OAAO,EAAE;IAEpD,MAAMuF,gBAAgB,MAAM9H,GACzB0B,MAAM,CAAC;QACNjB,kBAAkBZ,qBAAqBY,gBAAgB;QACvDvC,OAAOA;IACT,GACCyD,IAAI,CAAC9B,sBACL+B,KAAK,CACJ7D,IACEK,GAAGyB,qBAAqBoB,WAAW,EAAEI,OAAOJ,WAAW,GACvD7C,GAAGyB,qBAAqBW,aAAa,EAAEa,OAAOb,aAAa,GAC3DlC,QAAQuB,qBAAqBY,gBAAgB,EAAEY,OAAOX,iBAAiB,IAG1EoB,OAAO,CAACjC,qBAAqBY,gBAAgB,EAC7CsH,MAAM,CAACnJ,GAAG,CAAC,YAAY,CAAC;IAE3B,MAAMoJ,6BAA6BF,cAAcG,OAAO,CAAC,CAAC1H,MACxDA,IAAIE,gBAAgB,GAAG;YAACF,IAAIE,gBAAgB;SAAC,GAAG,EAAE;IAEpD,IAAIuH,2BAA2BzF,MAAM,GAAG,GAAG;QACzC5E,SAASuK,IAAI,CACX;YACEjH,aAAaI,OAAOJ,WAAW;YAC/BT,eAAea,OAAOb,aAAa;YACnCE,mBAAmBsH;QACrB,GACA;IAEJ;IAEA,MAAM5D,SAAS,MAAMpE,GAAGE,OAAO,CAS5BtB,GAAG,CAAC;wBACe,EAAEiB,qBAAqBY,gBAAgB,CAAC;MAC1D,EAAEZ,qBAAqBqD,aAAa,CAAC;MACrC,EAAErD,qBAAqBsD,oBAAoB,CAAC;MAC5C,EAAEtD,qBAAqBuD,KAAK,CAAC;MAC7B,EAAEvD,qBAAqBI,cAAc,CAAC;MACtC,EAAEJ,qBAAqBY,gBAAgB,CAAC;MACxC,EAAEZ,qBAAqB2F,SAAS,CAAC;MACjC,EAAE3F,qBAAqBsH,eAAe,CAAC;MACvC,EAAEtH,qBAAqBsI,uBAAuB,CAAC;SAC5C,EAAEtI,qBAAqB;;MAE1B,EAAEA,qBAAqBoB,WAAW,CAAC,GAAG,EAAEI,OAAOJ,WAAW,CAAC;UACvD,EAAEpB,qBAAqBW,aAAa,CAAC,GAAG,EAAEa,OAAOb,aAAa,CAAC;UAC/D,EAAEX,qBAAqBY,gBAAgB,CAAC,KAAK,EAAE7B,IAAIwJ,IAAI,CACzD/G,OAAOX,iBAAiB,CAACsB,GAAG,CAAC,CAACvB,mBAAqB7B,GAAG,CAAC,EAAE6B,iBAAiB,CAAC,GAC3E7B,GAAG,CAAC,EAAE,CAAC,EACP;aACK,EAAEiB,qBAAqBY,gBAAgB,CAAC,EAAE,EAAEZ,qBAAqB2F,SAAS,CAAC,OAAO,EAAE3F,qBAAqBI,cAAc,CAAC;EACnI,CAAC;IAED,OAAOmE,OAAOhE,IAAI,CAAC4B,GAAG,CAAC,CAACzB,MAAS,CAAA;YAC/B2C,eAAe3C,IAAI2C,aAAa;YAChCC,sBAAsB5C,IAAI4C,oBAAoB;YAC9CC,OAAO7C,IAAI6C,KAAK;YAChBnD,gBAAgBM,IAAIN,cAAc;YAClCQ,kBAAkBF,IAAIE,gBAAgB;YACtC+E,WAAW6C,OAAO9H,IAAIiF,SAAS;YAC/B2B,iBAAiBkB,OAAO9H,IAAI4G,eAAe;YAC3CgB,yBAAyB5H,IAAI4H,uBAAuB,GAChDE,OAAO9H,IAAI4H,uBAAuB,IAClC;QACN,CAAA;AACF;AAEA,SAASE,OAAOX,KAAoB;IAClC,OAAOA,iBAAiBY,OAAOZ,QAAQ,IAAIY,KAAKZ;AAClD;AAEA,OAAO,eAAea,iBAAiBlH,MAItC;IACC,MAAM,CAACd,IAAI,GAAG,MAAMnB,KACjBsC,MAAM,CAAC;QAACkB,IAAI/C,qBAAqB+C,EAAE;IAAA,GACnCjB,IAAI,CAAC9B,sBACL+B,KAAK,CACJ7D,IACEK,GAAGyB,qBAAqBI,cAAc,EAAEoB,OAAOpB,cAAc,GAC7D7B,GAAGyB,qBAAqBiE,eAAe,EAAEzC,OAAOyC,eAAe,GAC/DzC,OAAO+B,KAAK,KAAKO,YAAYA,YAAYvF,GAAGyB,qBAAqBuD,KAAK,EAAE/B,OAAO+B,KAAK,IAGvFV,KAAK,CAAC;IAET,OAAOnC,QAAQoD;AACjB;AAEA,OAAO,eAAe6E,gBAAgBnH,MAIrC;IAcC,MAAM+C,SAAS,MAAMhF,KAAKqD,WAAW,CAAC,OAAOzC;QAC3C,MAAMyI,UAAU,MAAMzI,GACnB+F,MAAM,CAAClG,sBACPiB,GAAG,CAAC;YACHoG,kBAAkBtI,GAAG,CAAC,SAAS,EAAEiB,qBAAqBqH,gBAAgB,CAAC,QAAQ,CAAC;YAChFC,iBAAiBvI,GAAG,CAAC,KAAK,CAAC;QAC7B,GACCgD,KAAK,CACJ7D,IACEK,GAAGyB,qBAAqBI,cAAc,EAAEoB,OAAOpB,cAAc,GAC7D7B,GAAGyB,qBAAqBiE,eAAe,EAAEzC,OAAOyC,eAAe,IAGlEJ,SAAS,CAAC;YACTyE,yBAAyBtI,qBAAqBsI,uBAAuB;YACrEjF,eAAerD,qBAAqBqD,aAAa;YACjDC,sBAAsBtD,qBAAqBsD,oBAAoB;YAC/DC,OAAOvD,qBAAqBuD,KAAK;YACjCnD,gBAAgBJ,qBAAqBI,cAAc;YACnDoD,WAAWxD,qBAAqBwD,SAAS;YACzCpC,aAAapB,qBAAqBoB,WAAW;YAC7C6C,iBAAiBjE,qBAAqBiE,eAAe;QACvD;QAEF,MAAMvD,MAAMkI,OAAO,CAAC,EAAE;QACtB,IAAI,CAAClI,KAAK,MAAM,IAAIxB,iCAAiCsC,OAAOpB,cAAc;QAE1E,MAAM,CAACyI,SAAS,GAAG,MAAM1I,GACtB0B,MAAM,CAAC;YAACiH,kBAAkB/I,eAAe+I,gBAAgB;QAAA,GACzDhH,IAAI,CAAC/B,gBACLgC,KAAK,CAACxD,GAAGwB,eAAegD,EAAE,EAAEvB,OAAOyC,eAAe,GAClDpB,KAAK,CAAC,GACNgC,GAAG,CAAC;QAEP,MAAM,CAACH,QAAQ,GAAG,MAAMvE,GACrB+F,MAAM,CAACnG,gBACPkB,GAAG,CAAC;YACH6H,kBAAkBtH,OAAOsH,gBAAgB,IAAI;YAC7CC,4BAA4BvH,OAAOsH,gBAAgB,GAAG/J,GAAG,CAAC,KAAK,CAAC,GAAG;YACnEqH,WAAWrH,GAAG,CAAC,KAAK,CAAC;QACvB,GACCgD,KAAK,CAACxD,GAAGwB,eAAegD,EAAE,EAAEvB,OAAOyC,eAAe,GAClDJ,SAAS,CAAC;YAACiF,kBAAkB/I,eAAe+I,gBAAgB;QAAA;QAE/D,OAAO;YACLpI;YACAsI,0BAA0BH,UAAUC,oBAAoB;YACxDG,yBAAyBvE,SAASoE,oBAAoB;QACxD;IACF;IAEA,MAAMpI,MAAM6D,OAAO7D,GAAG;IACtB,IAAI,CAACA,KAAK,MAAM,IAAIxB,iCAAiCsC,OAAOpB,cAAc;IAC1E,OAAO;QACL8I,uBAAuBxI,IAAI4H,uBAAuB,KAAK;QACvDU,0BAA0BzE,OAAOyE,wBAAwB;QACzDC,yBAAyB1E,OAAO0E,uBAAuB;QACvDE,qBAAqB;YACnB9F,eAAe3C,IAAI2C,aAAa;YAChCC,sBAAsB5C,IAAI4C,oBAAoB;YAC9CC,OAAO7C,IAAI6C,KAAK;YAChBnD,gBAAgBM,IAAIN,cAAc;YAClCoD,WAAW9C,IAAI8C,SAAS;YACxBpC,aAAaV,IAAIU,WAAW;YAC5B6C,iBAAiBvD,IAAIuD,eAAe;QACtC;IACF;AACF;AAEA;;;;CAIC,GACD,OAAO,eAAemF,8BAA8B5H,MAEnD;IACC,MAAMjC,KAAKqD,WAAW,CAAC,OAAOzC;QAC5B,MAAMD,iBAAiBC,IAAIqB,OAAOpB,cAAc;QAEhD,wFAAwF;QACxF,2FAA2F;QAC3F,wFAAwF;QACxF,MAAMD,GACHqF,MAAM,CAAC5F,sBACPmC,KAAK,CAACxD,GAAGqB,qBAAqBQ,cAAc,EAAEoB,OAAOpB,cAAc;QACtE,MAAMiJ,uBAAuB,MAAMlJ,GAChC+F,MAAM,CAAClG,sBACPiB,GAAG,CAAC;YACHqH,yBAAyBvJ,GAAG,CAAC,SAAS,EAAEiB,qBAAqBsI,uBAAuB,CAAC,QAAQ,CAAC;QAChG,GACCvG,KAAK,CAACxD,GAAGyB,qBAAqBI,cAAc,EAAEoB,OAAOpB,cAAc,GACnEyD,SAAS,CAAC;YACTlD,eAAeX,qBAAqBW,aAAa;YACjDC,kBAAkBZ,qBAAqBY,gBAAgB;QACzD;QAEF,MAAMN,0CAA0CH,IAAIkJ;IACtD;AACF"}

package/dist/db/reservations.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"reservations.d.ts","sourceRoot":"","sources":["../../src/db/reservations.ts"],"names":[],"mappings":"AAmBA,OAAO,KAAK,EAAC,EAAE,EAAC,MAAM,SAAS,CAAC;AAahC,MAAM,WAAW,mBAAmB;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,UAAU;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,IAAI,CAAC;CACtB;AAED,MAAM,WAAW,gBAAgB;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,IAAI,CAAC;CACjB;AAED,MAAM,WAAW,0BAA0B;IACzC,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,qFAAqF;IACrF,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,SAAS,EAAE,mBAAmB,EAAE,CAAC;IACjC,uBAAuB,CAAC,EAAE,MAAM,CAAC;CAClC;AAED,MAAM,WAAW,sCAAsC;IACrD,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,qFAAqF;IACrF,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,SAAS,EAAE,mBAAmB,EAAE,CAAC;IACjC,uBAAuB,EAAE,MAAM,CAAC;IAChC,oBAAoB,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IAC1C,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,cAAc,CAAC,EAAE,CAAC,YAAY,EAAE,gBAAgB,EAAE,KAAK,IAAI,CAAC;CAC7D;AAoCD,UAAU,mBAAmB;IAC3B,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,UAAU,0BAA0B;IAClC,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,+DAA+D;IAC/D,YAAY,EAAE,gBAAgB,EAAE,CAAC;IACjC,yEAAyE;IACzE,kBAAkB,EAAE,mBAAmB,EAAE,CAAC;CAC3C;AAED,wBAAsB,oBAAoB,CACxC,MAAM,EAAE,0BAA0B,GACjC,OAAO,CAAC;IAAC,KAAK,EAAE,UAAU,EAAE,CAAC;IAAC,YAAY,EAAE,gBAAgB,EAAE,CAAA;CAAC,CAAC,CAKlE;AAED,wBAAsB,sBAAsB,CAC1C,EAAE,EAAE,EAAE,EACN,MAAM,EAAE,0BAA0B,GACjC,OAAO,CAAC,0BAA0B,CAAC,CAGrC;AAED,wBAAsB,gCAAgC,CACpD,MAAM,EAAE,sCAAsC,GAC7C,OAAO,CAAC;IACT,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,YAAY,EAAE,gBAAgB,EAAE,CAAC;IACjC,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B,CAAC,CAsDD;AA+dD,wBAAsB,4BAA4B,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,CAMtE;AA0BD,wBAAsB,yBAAyB,CAAC,MAAM,CAAC,EAAE;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAc1F;AAED,wBAAsB,uBAAuB,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAM5E;AAwHD,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,EAAE,EACN,MAAM,EAAE;IACN,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,KAAK,CAAC;QAAC,aAAa,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAC,CAAC,CAAC;CACzD,GACA,OAAO,CAAC,MAAM,CAAC,CAoDjB;AAED,wBAAsB,8CAA8C,CAClE,EAAE,EAAE,EAAE,EACN,MAAM,EAAE;IACN,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B,4BAA4B,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IACzD,sBAAsB,CAAC,EAAE,OAAO,CAAC;IACjC,mFAAmF;IACnF,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAChC,GACA,OAAO,CAAC,MAAM,CAAC,CA4PjB"}
+{"version":3,"file":"reservations.d.ts","sourceRoot":"","sources":["../../src/db/reservations.ts"],"names":[],"mappings":"AAmBA,OAAO,KAAK,EAAC,EAAE,EAAC,MAAM,SAAS,CAAC;AAahC,MAAM,WAAW,mBAAmB;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,UAAU;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,IAAI,CAAC;CACtB;AAED,MAAM,WAAW,gBAAgB;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,IAAI,CAAC;CACjB;AAED,MAAM,WAAW,0BAA0B;IACzC,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,qFAAqF;IACrF,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,SAAS,EAAE,mBAAmB,EAAE,CAAC;IACjC,uBAAuB,CAAC,EAAE,MAAM,CAAC;CAClC;AAED,MAAM,WAAW,sCAAsC;IACrD,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,qFAAqF;IACrF,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,SAAS,EAAE,mBAAmB,EAAE,CAAC;IACjC,uBAAuB,EAAE,MAAM,CAAC;IAChC,oBAAoB,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IAC1C,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,cAAc,CAAC,EAAE,CAAC,YAAY,EAAE,gBAAgB,EAAE,KAAK,IAAI,CAAC;CAC7D;AAoCD,UAAU,mBAAmB;IAC3B,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,UAAU,0BAA0B;IAClC,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,+DAA+D;IAC/D,YAAY,EAAE,gBAAgB,EAAE,CAAC;IACjC,yEAAyE;IACzE,kBAAkB,EAAE,mBAAmB,EAAE,CAAC;CAC3C;AAED,wBAAsB,oBAAoB,CACxC,MAAM,EAAE,0BAA0B,GACjC,OAAO,CAAC;IAAC,KAAK,EAAE,UAAU,EAAE,CAAC;IAAC,YAAY,EAAE,gBAAgB,EAAE,CAAA;CAAC,CAAC,CAKlE;AAED,wBAAsB,sBAAsB,CAC1C,EAAE,EAAE,EAAE,EACN,MAAM,EAAE,0BAA0B,GACjC,OAAO,CAAC,0BAA0B,CAAC,CAGrC;AAED,wBAAsB,gCAAgC,CACpD,MAAM,EAAE,sCAAsC,GAC7C,OAAO,CAAC;IACT,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,YAAY,EAAE,gBAAgB,EAAE,CAAC;IACjC,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B,CAAC,CAsDD;AA+dD,wBAAsB,4BAA4B,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,CAMtE;AA0BD,wBAAsB,yBAAyB,CAAC,MAAM,CAAC,EAAE;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAc1F;AAED,wBAAsB,uBAAuB,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAM5E;AAwHD,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,EAAE,EACN,MAAM,EAAE;IACN,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,KAAK,CAAC;QAAC,aAAa,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAC,CAAC,CAAC;CACzD,GACA,OAAO,CAAC,MAAM,CAAC,CAoDjB;AAED,wBAAsB,8CAA8C,CAClE,EAAE,EAAE,EAAE,EACN,MAAM,EAAE;IACN,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B,4BAA4B,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IACzD,sBAAsB,CAAC,EAAE,OAAO,CAAC;IACjC,mFAAmF;IACnF,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAChC,GACA,OAAO,CAAC,MAAM,CAAC,CA+PjB"}

package/dist/db/reservations.js

@@ -491,9 +491,11 @@ export async function releaseReservationUnits(tx, params) {
 export async function releaseTerminalRunnerInstanceReservationsByIds(tx, params) {
     if ((params.providerRunnerIds?.length ?? 0) === 0 && (params.runnerInstanceIds?.length ?? 0) === 0) return 0;
     const reservationWorkspacePredicate = params.workspaceId === null ? sql`` : sql`and ${eq(reservations.workspaceId, params.workspaceId)}`;
-    const noRunningJobPredicate = notExists(tx.select({
+    // Cancellation keeps the lease row so the provisioner can observe its terminate intent.
+    // Once the runner is terminal, only an uncancelled lease should block reservation release.
+    const noUncancelledRunningJobPredicate = notExists(tx.select({
         id: runningJobExecutions.id
-    }).from(runningJobExecutions).where(and(eq(runningJobExecutions.provisionerId, params.provisionerId), eq(runningJobExecutions.providerRunnerId, providerRunners.providerRunnerId), params.workspaceId === null ? undefined : eq(runningJobExecutions.workspaceId, params.workspaceId))));
+    }).from(runningJobExecutions).where(and(eq(runningJobExecutions.provisionerId, params.provisionerId), eq(runningJobExecutions.providerRunnerId, providerRunners.providerRunnerId), isNull(runningJobExecutions.cancellationRequestedAt), params.workspaceId === null ? undefined : eq(runningJobExecutions.workspaceId, params.workspaceId))));
     const reportFreshnessPredicate = params.reportedAtByProviderRunnerId && params.reportedAtByProviderRunnerId.size > 0 ? or(...[
         ...params.reportedAtByProviderRunnerId
     ].map(([providerRunnerId, reportedAt])=>and(eq(providerRunners.providerRunnerId, providerRunnerId), lte(providerRunners.reportedAt, reportedAt)))) : undefined;
@@ -548,7 +550,7 @@ export async function releaseTerminalRunnerInstanceReservationsByIds(tx, params)
         id: reservations.id
     }).from(reservations).where(and(eq(reservations.workspaceId, params.workspaceId), eq(reservations.provisionerId, params.provisionerId), eq(reservations.id, providerRunners.intendedReservationId)))))), and(isNull(providerRunners.workspaceId), isNotNull(providerRunners.intendedReservationId), exists(tx.select({
         id: reservations.id
-    }).from(reservations).where(and(eq(reservations.workspaceId, params.workspaceId), eq(reservations.provisionerId, params.provisionerId), eq(reservations.id, providerRunners.intendedReservationId)))))), params.requireUnlinkedSession === false ? undefined : isNull(providerRunners.runnerSessionId), noRunningJobPredicate, reportFreshnessPredicate, isNull(providerRunners.reservationReleasedAt))).for('update');
+    }).from(reservations).where(and(eq(reservations.workspaceId, params.workspaceId), eq(reservations.provisionerId, params.provisionerId), eq(reservations.id, providerRunners.intendedReservationId)))))), params.requireUnlinkedSession === false ? undefined : isNull(providerRunners.runnerSessionId), noUncancelledRunningJobPredicate, reportFreshnessPredicate, isNull(providerRunners.reservationReleasedAt))).for('update');
     if (rows.length === 0) return 0;
     // The lease-finalization path deliberately locks active rows too. If a terminal report is
     // concurrently projecting the runner state, PostgreSQL rechecks this row after the lock and
@@ -559,7 +561,7 @@ export async function releaseTerminalRunnerInstanceReservationsByIds(tx, params)
         intendedReservationId: null,
         reservationReleasedAt: sql`now()`,
         updatedAt: sql`now()`
-    }).where(and(inArray(providerRunners.id, terminalRows.map((row)=>row.id)), params.requireUnlinkedSession === false ? undefined : isNull(providerRunners.runnerSessionId), noRunningJobPredicate, reportFreshnessPredicate, isNull(providerRunners.reservationReleasedAt))).returning({
+    }).where(and(inArray(providerRunners.id, terminalRows.map((row)=>row.id)), params.requireUnlinkedSession === false ? undefined : isNull(providerRunners.runnerSessionId), noUncancelledRunningJobPredicate, reportFreshnessPredicate, isNull(providerRunners.reservationReleasedAt))).returning({
         id: providerRunners.id
     });
     const releasesByReservationId = new Map();