@shipfox/api-runners 5.0.0 → 6.0.0

package/dist/db/runner-instances.js

@@ -0,0 +1,511 @@
+import { logger } from '@shipfox/node-opentelemetry';
+import { canonicalizeLabels } from '@shipfox/runner-labels';
+import { and, asc, desc, eq, exists, gt, inArray, isNotNull, isNull, lt, notExists, notInArray, or, sql } from 'drizzle-orm';
+import { alias } from 'drizzle-orm/pg-core';
+import { db } from './db.js';
+import { listRunningJobExecutionsByRunnerInstanceTx } from './job-executions.js';
+import { releaseReservationUnits } from './reservations.js';
+import { ephemeralRegistrationTokens } from './schema/ephemeral-registration-tokens.js';
+import { provisionerTokens } from './schema/provisioner-tokens.js';
+import { providerRunners, toRunnerInstance } from './schema/runner-instances.js';
+import { runnerSessions } from './schema/runner-sessions.js';
+import { runningJobExecutions } from './schema/running-job-executions.js';
+export const terminalStates = [
+    'stopped',
+    'failed',
+    'terminated'
+];
+export const activeStates = [
+    'starting',
+    'running',
+    'stopping'
+];
+export const divergenceCountStates = [
+    'starting',
+    'running'
+];
+export async function createPlannedProvisionedCapacity(params) {
+    const [row] = await db().insert(providerRunners).values({
+        provisionerId: params.provisionerId,
+        providerKind: params.providerKind,
+        templateKey: params.templateKey,
+        state: 'starting',
+        labels: [],
+        reportedAt: new Date()
+    }).returning({
+        capacityId: providerRunners.id
+    });
+    if (!row) throw new Error('Planned capacity insert returned no row');
+    return row;
+}
+export async function attachProviderRunnerId(params) {
+    const updated = await db().update(providerRunners).set({
+        providerRunnerId: params.providerRunnerId,
+        updatedAt: sql`now()`
+    }).where(and(eq(providerRunners.id, params.capacityId), eq(providerRunners.provisionerId, params.provisionerId), isNull(providerRunners.providerRunnerId), notInArray(providerRunners.state, [
+        ...terminalStates
+    ]))).returning({
+        id: providerRunners.id
+    });
+    return updated.length === 1;
+}
+export async function reportRunnerInstances(params) {
+    if (params.events.length === 0) return {
+        accepted: 0,
+        reservationsReleased: 0,
+        terminateIntentsHonored: []
+    };
+    return await db().transaction(async (tx)=>{
+        const receivedAt = new Date();
+        const aggregatedEvents = aggregateEvents(params.events, receivedAt);
+        const hasTerminalEvent = aggregatedEvents.some((event)=>isTerminalState(event.state));
+        if (hasTerminalEvent) {
+            await tx.execute(sql`select pg_advisory_xact_lock(hashtext(${params.workspaceId ?? params.provisionerId}))`);
+        }
+        const events = await hydrateRunnerSessionIdsFromConsumedTokens(tx, params, aggregatedEvents);
+        const terminateIntentsHonored = await listTerminateIntentsHonoredByTerminatedReportsTx(tx, params, events);
+        const values = events.map((event)=>({
+                workspaceId: params.workspaceId,
+                provisionerId: params.provisionerId,
+                providerRunnerId: event.providerRunnerId,
+                reservationId: event.reservationId,
+                templateKey: event.templateKey,
+                labels: [
+                    ...canonicalizeLabels(event.labels)
+                ],
+                state: event.state,
+                reason: event.reason,
+                runnerSessionId: event.runnerSessionId,
+                providerKind: event.providerKind,
+                reportedAt: event.reportedAt > receivedAt ? receivedAt : event.reportedAt,
+                startedAt: event.startedAt,
+                stoppingAt: event.stoppingAt,
+                stoppedAt: event.stoppedAt,
+                failedAt: event.failedAt,
+                terminatedAt: event.terminatedAt
+            }));
+        await tx.insert(providerRunners).values(values).onConflictDoUpdate({
+            target: [
+                providerRunners.provisionerId,
+                providerRunners.providerRunnerId
+            ],
+            targetWhere: isNotNull(providerRunners.providerRunnerId),
+            set: {
+                reservationId: sql`CASE WHEN ${providerRunnerProjectionUpdateCondition()} THEN coalesce(excluded.reservation_id, ${providerRunners.reservationId}) ELSE ${providerRunners.reservationId} END`,
+                templateKey: sql`CASE WHEN ${providerRunnerProjectionUpdateCondition()} THEN coalesce(excluded.template_key, ${providerRunners.templateKey}) ELSE ${providerRunners.templateKey} END`,
+                labels: sql`CASE WHEN ${providerRunnerProjectionUpdateCondition()} THEN excluded.labels ELSE ${providerRunners.labels} END`,
+                state: sql`CASE WHEN ${providerRunnerProjectionUpdateCondition()} THEN excluded.state ELSE ${providerRunners.state} END`,
+                reason: sql`CASE WHEN ${providerRunnerProjectionUpdateCondition()} THEN excluded.reason ELSE ${providerRunners.reason} END`,
+                runnerSessionId: sql`CASE WHEN ${providerRunnerProjectionUpdateCondition()} THEN coalesce(${providerRunners.runnerSessionId}, excluded.runner_session_id) ELSE ${providerRunners.runnerSessionId} END`,
+                providerKind: sql`CASE WHEN ${providerRunnerProjectionUpdateCondition()} THEN coalesce(excluded.provider_kind, ${providerRunners.providerKind}) ELSE ${providerRunners.providerKind} END`,
+                reportedAt: sql`CASE WHEN ${providerRunnerProjectionUpdateCondition()} THEN excluded.reported_at ELSE ${providerRunners.reportedAt} END`,
+                startedAt: firstObservedAt(providerRunners.startedAt, sql`excluded.started_at`),
+                stoppingAt: firstObservedAt(providerRunners.stoppingAt, sql`excluded.stopping_at`),
+                stoppedAt: firstObservedAt(providerRunners.stoppedAt, sql`excluded.stopped_at`),
+                failedAt: firstObservedAt(providerRunners.failedAt, sql`excluded.failed_at`),
+                terminatedAt: firstObservedAt(providerRunners.terminatedAt, sql`excluded.terminated_at`),
+                updatedAt: sql`now()`
+            },
+            setWhere: sql`
+          ${providerRunnerProjectionUpdateCondition()}
+          OR ${providerRunnerMilestoneUpdateCondition()}
+        `
+        });
+        const reservationsReleased = hasTerminalEvent ? await releaseTerminalRunnerInstanceReservations(tx, params, events) : 0;
+        return {
+            accepted: events.length,
+            reservationsReleased,
+            terminateIntentsHonored
+        };
+    });
+}
+export async function listActiveRunnerInstanceCountsByTemplateTx(tx, params) {
+    const rows = await tx.select({
+        templateKey: providerRunners.templateKey,
+        state: providerRunners.state,
+        count: sql`count(*)::int`
+    }).from(providerRunners).where(and(eq(providerRunners.workspaceId, params.workspaceId), eq(providerRunners.provisionerId, params.provisionerId), inArray(providerRunners.state, divergenceCountStates), isNotNull(providerRunners.templateKey))).groupBy(providerRunners.templateKey, providerRunners.state);
+    return rows.flatMap((row)=>row.templateKey && isDivergenceCountState(row.state) ? [
+            {
+                templateKey: row.templateKey,
+                state: row.state,
+                count: row.count
+            }
+        ] : []);
+}
+export async function listActiveRunnerInstances(params) {
+    const rows = await db().select().from(providerRunners).where(and(eq(providerRunners.workspaceId, params.workspaceId), inArray(providerRunners.state, activeStates), sql`${providerRunners.updatedAt} > now() - (${params.windowSeconds} || ' seconds')::interval`)).orderBy(desc(providerRunners.updatedAt), desc(providerRunners.id)).limit(params.limit ?? 1000);
+    return rows.map(toRunnerInstance);
+}
+export async function listProvisionerTerminateIntents(params) {
+    return await db().transaction(async (tx)=>{
+        const rows = await listProvisionerTerminateIntentRowsTx(tx, params);
+        return rows.map((row)=>row.providerRunnerId);
+    });
+}
+export async function listProvisionerTerminateIntentRowsTx(tx, params) {
+    const rows = await provisionerTerminateIntentsQuery(tx, params).orderBy(asc(providerRunners.providerRunnerId)).limit(params.limit + 1);
+    const truncated = rows.length > params.limit;
+    const returnedRows = truncated ? rows.slice(0, params.limit) : rows;
+    if (truncated) {
+        logger().warn({
+            workspaceId: params.workspaceId,
+            provisionerId: params.provisionerId,
+            limit: params.limit,
+            returnedCount: returnedRows.length
+        }, 'provisioner terminate intents truncated by poll-demand limit');
+    }
+    return returnedRows.flatMap((row)=>row.providerRunnerId ? [
+            {
+                ...row,
+                providerRunnerId: row.providerRunnerId
+            }
+        ] : []);
+}
+function provisionerTerminateIntentsQuery(tx, params) {
+    const newerRunningJobExecutions = alias(runningJobExecutions, 'newer_running_jobs');
+    return tx.select({
+        providerRunnerId: providerRunners.providerRunnerId,
+        reason: sql`'job-cancelled'`
+    }).from(runningJobExecutions).innerJoin(providerRunners, and(eq(providerRunners.workspaceId, runningJobExecutions.workspaceId), eq(providerRunners.provisionerId, runningJobExecutions.provisionerId), eq(providerRunners.providerRunnerId, runningJobExecutions.providerRunnerId))).where(and(eq(runningJobExecutions.workspaceId, params.workspaceId), eq(runningJobExecutions.provisionerId, params.provisionerId), isNotNull(providerRunners.providerRunnerId), isNotNull(runningJobExecutions.cancellationRequestedAt), inArray(providerRunners.state, activeStates), params.providerRunnerIds && params.providerRunnerIds.length > 0 ? inArray(providerRunners.providerRunnerId, params.providerRunnerIds) : undefined, notExists(tx.select({
+        id: newerRunningJobExecutions.id
+    }).from(newerRunningJobExecutions).where(and(eq(newerRunningJobExecutions.workspaceId, runningJobExecutions.workspaceId), eq(newerRunningJobExecutions.provisionerId, runningJobExecutions.provisionerId), eq(newerRunningJobExecutions.providerRunnerId, runningJobExecutions.providerRunnerId), or(gt(newerRunningJobExecutions.startedAt, runningJobExecutions.startedAt), and(eq(newerRunningJobExecutions.startedAt, runningJobExecutions.startedAt), gt(newerRunningJobExecutions.jobExecutionId, runningJobExecutions.jobExecutionId)))))))).groupBy(providerRunners.providerRunnerId);
+}
+async function listTerminateIntentsHonoredByTerminatedReportsTx(tx, params, events) {
+    if (!params.workspaceId) return [];
+    const terminatedRunnerInstanceIds = [
+        ...new Set(events.filter((event)=>event.state === 'terminated').map((event)=>event.providerRunnerId))
+    ];
+    if (terminatedRunnerInstanceIds.length === 0) return [];
+    const rows = await provisionerTerminateIntentsQuery(tx, {
+        workspaceId: params.workspaceId,
+        provisionerId: params.provisionerId,
+        providerRunnerIds: terminatedRunnerInstanceIds
+    }).orderBy(asc(providerRunners.providerRunnerId));
+    return rows.flatMap((row)=>row.providerRunnerId ? [
+            {
+                ...row,
+                providerRunnerId: row.providerRunnerId
+            }
+        ] : []);
+}
+export async function reconcileRunnerInstances(params) {
+    const observedRunnerInstanceIds = [
+        ...new Set(params.observedRunnerInstanceIds)
+    ];
+    return await db().transaction(async (tx)=>{
+        await tx.execute(sql`select pg_advisory_xact_lock(hashtext(${params.workspaceId ?? params.provisionerId}))`);
+        let absentIds = [];
+        let reservationsReleased = 0;
+        if (observedRunnerInstanceIds.length > 0) {
+            const staleAbsentRows = await tx.select({
+                id: providerRunners.id,
+                providerRunnerId: providerRunners.providerRunnerId
+            }).from(providerRunners).where(and(params.workspaceId ? eq(providerRunners.workspaceId, params.workspaceId) : isNull(providerRunners.workspaceId), eq(providerRunners.provisionerId, params.provisionerId), inArray(providerRunners.state, activeStates), lt(providerRunners.reportedAt, sql`now() - (${params.terminateGraceSeconds} || ' seconds')::interval`), notInArray(providerRunners.providerRunnerId, observedRunnerInstanceIds)));
+            if (staleAbsentRows.length > 0) {
+                const updated = await tx.update(providerRunners).set({
+                    state: 'terminated',
+                    terminatedAt: sql`coalesce(${providerRunners.terminatedAt}, now())`,
+                    updatedAt: sql`now()`
+                }).where(and(inArray(providerRunners.id, staleAbsentRows.map((row)=>row.id)), inArray(providerRunners.state, activeStates), lt(providerRunners.reportedAt, sql`now() - (${params.terminateGraceSeconds} || ' seconds')::interval`))).returning({
+                    providerRunnerId: providerRunners.providerRunnerId
+                });
+                absentIds = updated.flatMap((row)=>row.providerRunnerId ? [
+                        row.providerRunnerId
+                    ] : []);
+                if (params.workspaceId) {
+                    reservationsReleased = await releaseTerminalRunnerInstanceReservationsByIds(tx, {
+                        workspaceId: params.workspaceId,
+                        provisionerId: params.provisionerId,
+                        providerRunnerIds: absentIds
+                    });
+                }
+            }
+        }
+        const observedRows = observedRunnerInstanceIds.length === 0 ? [] : (await tx.select().from(providerRunners).where(and(params.workspaceId ? eq(providerRunners.workspaceId, params.workspaceId) : isNull(providerRunners.workspaceId), eq(providerRunners.provisionerId, params.provisionerId), inArray(providerRunners.providerRunnerId, observedRunnerInstanceIds)))).map(toRunnerInstance);
+        const boundJobExecutions = params.workspaceId ? await listRunningJobExecutionsByRunnerInstanceTx(tx, {
+            workspaceId: params.workspaceId,
+            provisionerId: params.provisionerId,
+            providerRunnerIds: observedRunnerInstanceIds
+        }) : [];
+        return {
+            observedRows,
+            boundJobExecutionsByRunnerInstanceId: new Map(boundJobExecutions.map((jobExecution)=>[
+                    jobExecution.providerRunnerId,
+                    jobExecution
+                ])),
+            absentIds,
+            reservationsReleased
+        };
+    });
+}
+export async function reapStaleRunnerInstances(params) {
+    const cutoff = staleRunnerInstanceCutoff(params.thresholdSeconds);
+    return await db().transaction(async (tx)=>{
+        const candidateRows = await tx.select({
+            id: providerRunners.id,
+            workspaceId: providerRunners.workspaceId,
+            provisionerId: providerRunners.provisionerId,
+            providerRunnerId: providerRunners.providerRunnerId
+        }).from(providerRunners).where(staleRunnerInstanceWhere(tx, cutoff)).orderBy(asc(providerRunners.updatedAt), asc(providerRunners.id)).limit(params.limit);
+        if (candidateRows.length === 0) return {
+            reaped: 0,
+            reservationsReleased: 0
+        };
+        const workspaceIds = [
+            ...new Set(candidateRows.flatMap((row)=>row.workspaceId ? [
+                    row.workspaceId
+                ] : []))
+        ].sort();
+        for (const workspaceId of workspaceIds){
+            await tx.execute(sql`select pg_advisory_xact_lock(hashtext(${workspaceId}))`);
+        }
+        const updatedRows = await tx.update(providerRunners).set({
+            state: 'failed',
+            reason: 'stale-provisioner',
+            failedAt: sql`coalesce(${providerRunners.failedAt}, now())`,
+            updatedAt: sql`now()`
+        }).where(and(inArray(providerRunners.id, candidateRows.map((row)=>row.id)), staleRunnerInstanceWhere(tx, cutoff))).returning({
+            workspaceId: providerRunners.workspaceId,
+            provisionerId: providerRunners.provisionerId,
+            providerRunnerId: providerRunners.providerRunnerId
+        });
+        let reservationsReleased = 0;
+        for (const group of groupRunnerInstanceIds(updatedRows)){
+            reservationsReleased += await releaseTerminalRunnerInstanceReservationsByIds(tx, {
+                workspaceId: group.workspaceId,
+                provisionerId: group.provisionerId,
+                providerRunnerIds: group.providerRunnerIds,
+                requireUnlinkedSession: false
+            });
+        }
+        return {
+            reaped: updatedRows.length,
+            reservationsReleased
+        };
+    });
+}
+async function releaseTerminalRunnerInstanceReservations(tx, params, events) {
+    const terminalEvents = events.filter((event)=>isTerminalState(event.state));
+    if (terminalEvents.length === 0 || !params.workspaceId) return 0;
+    return await releaseTerminalRunnerInstanceReservationsByIds(tx, {
+        workspaceId: params.workspaceId,
+        provisionerId: params.provisionerId,
+        providerRunnerIds: terminalEvents.map((event)=>event.providerRunnerId)
+    });
+}
+async function releaseTerminalRunnerInstanceReservationsByIds(tx, params) {
+    if (params.providerRunnerIds.length === 0) return 0;
+    const rows = await tx.select({
+        id: providerRunners.id,
+        reservationId: providerRunners.reservationId
+    }).from(providerRunners).where(and(eq(providerRunners.workspaceId, params.workspaceId), eq(providerRunners.provisionerId, params.provisionerId), inArray(providerRunners.providerRunnerId, params.providerRunnerIds), inArray(providerRunners.state, terminalStates), isNotNull(providerRunners.reservationId), params.requireUnlinkedSession === false ? undefined : isNull(providerRunners.runnerSessionId), isNull(providerRunners.reservationReleasedAt)));
+    if (rows.length === 0) return 0;
+    const updated = await tx.update(providerRunners).set({
+        reservationReleasedAt: sql`now()`,
+        updatedAt: sql`now()`
+    }).where(and(inArray(providerRunners.id, rows.map((row)=>row.id)), params.requireUnlinkedSession === false ? undefined : isNull(providerRunners.runnerSessionId), isNull(providerRunners.reservationReleasedAt))).returning({
+        reservationId: providerRunners.reservationId
+    });
+    const releasesByReservationId = new Map();
+    for (const row of updated){
+        if (!row.reservationId) continue;
+        releasesByReservationId.set(row.reservationId, (releasesByReservationId.get(row.reservationId) ?? 0) + 1);
+    }
+    if (releasesByReservationId.size === 0) return 0;
+    return await releaseReservationUnits(tx, {
+        workspaceId: params.workspaceId,
+        provisionerId: params.provisionerId,
+        releases: [
+            ...releasesByReservationId
+        ].map(([reservationId, count])=>({
+                reservationId,
+                count
+            }))
+    });
+}
+function staleRunnerInstanceCutoff(thresholdSeconds) {
+    return sql`now() - (${thresholdSeconds} || ' seconds')::interval`;
+}
+function staleRunnerInstanceWhere(tx, cutoff) {
+    return and(inArray(providerRunners.state, activeStates), lt(providerRunners.reportedAt, cutoff), lt(providerRunners.updatedAt, cutoff), exists(tx.select({
+        id: provisionerTokens.id
+    }).from(provisionerTokens).where(and(eq(provisionerTokens.id, providerRunners.provisionerId), or(isNull(provisionerTokens.lastSeenAt), lt(provisionerTokens.lastSeenAt, cutoff))))), notExists(tx.select({
+        id: runningJobExecutions.id
+    }).from(runningJobExecutions).where(and(eq(runningJobExecutions.workspaceId, providerRunners.workspaceId), eq(runningJobExecutions.provisionerId, providerRunners.provisionerId), eq(runningJobExecutions.providerRunnerId, providerRunners.providerRunnerId)))), notExists(tx.select({
+        id: runnerSessions.id
+    }).from(runnerSessions).where(and(eq(runnerSessions.workspaceId, providerRunners.workspaceId), eq(runnerSessions.provisionerId, providerRunners.provisionerId), eq(runnerSessions.providerRunnerId, providerRunners.providerRunnerId), sql`${runnerSessions.updatedAt} >= ${cutoff}`))));
+}
+function groupRunnerInstanceIds(rows) {
+    const groups = new Map();
+    for (const row of rows){
+        if (!row.workspaceId || !row.providerRunnerId) continue;
+        const key = `${row.workspaceId}:${row.provisionerId}`;
+        const group = groups.get(key) ?? {
+            workspaceId: row.workspaceId,
+            provisionerId: row.provisionerId,
+            providerRunnerIds: []
+        };
+        group.providerRunnerIds.push(row.providerRunnerId);
+        groups.set(key, group);
+    }
+    return [
+        ...groups.values()
+    ];
+}
+async function hydrateRunnerSessionIdsFromConsumedTokens(tx, params, events) {
+    if (!params.workspaceId) return events;
+    const providerRunnerIds = [
+        ...new Set(events.map((event)=>event.providerRunnerId))
+    ];
+    if (providerRunnerIds.length === 0) return events;
+    const tokenRows = await tx.select({
+        providerRunnerId: ephemeralRegistrationTokens.providerRunnerId,
+        consumedSessionId: ephemeralRegistrationTokens.consumedSessionId
+    }).from(ephemeralRegistrationTokens).where(and(eq(ephemeralRegistrationTokens.workspaceId, params.workspaceId), eq(ephemeralRegistrationTokens.provisionerId, params.provisionerId), inArray(ephemeralRegistrationTokens.providerRunnerId, providerRunnerIds), isNotNull(ephemeralRegistrationTokens.consumedSessionId))).orderBy(desc(ephemeralRegistrationTokens.consumedAt), desc(ephemeralRegistrationTokens.createdAt));
+    const consumedSessionIdsByRunnerInstanceId = new Map();
+    for (const row of tokenRows){
+        if (!row.consumedSessionId) continue;
+        if (consumedSessionIdsByRunnerInstanceId.has(row.providerRunnerId)) continue;
+        consumedSessionIdsByRunnerInstanceId.set(row.providerRunnerId, row.consumedSessionId);
+    }
+    if (consumedSessionIdsByRunnerInstanceId.size === 0) return events;
+    return events.map((event)=>{
+        const consumedSessionId = consumedSessionIdsByRunnerInstanceId.get(event.providerRunnerId);
+        if (!consumedSessionId || event.runnerSessionId === consumedSessionId) return event;
+        return {
+            ...event,
+            runnerSessionId: consumedSessionId
+        };
+    });
+}
+function aggregateEvents(events, receivedAt) {
+    const byRunnerInstanceId = new Map();
+    for (const rawEvent of events){
+        const event = toRunnerInstanceReportRow(rawEvent, receivedAt);
+        const existing = byRunnerInstanceId.get(event.providerRunnerId);
+        if (existing) mergeMilestones(existing, event);
+        if (!existing || compareRunnerInstanceReportEvents(event, existing) > 0) {
+            byRunnerInstanceId.set(event.providerRunnerId, existing ? mergeProjectionMetadata(event, existing) : event);
+        }
+    }
+    return [
+        ...byRunnerInstanceId.values()
+    ];
+}
+function toRunnerInstanceReportRow(event, receivedAt) {
+    const reportedAt = event.reportedAt > receivedAt ? receivedAt : event.reportedAt;
+    return {
+        ...event,
+        reportedAt,
+        startedAt: event.state === 'running' ? reportedAt : null,
+        stoppingAt: event.state === 'stopping' ? reportedAt : null,
+        stoppedAt: event.state === 'stopped' ? reportedAt : null,
+        failedAt: event.state === 'failed' ? reportedAt : null,
+        terminatedAt: event.state === 'terminated' ? reportedAt : null
+    };
+}
+function mergeMilestones(target, source) {
+    target.startedAt = earliestDate(target.startedAt, source.startedAt);
+    target.stoppingAt = earliestDate(target.stoppingAt, source.stoppingAt);
+    target.stoppedAt = earliestDate(target.stoppedAt, source.stoppedAt);
+    target.failedAt = earliestDate(target.failedAt, source.failedAt);
+    target.terminatedAt = earliestDate(target.terminatedAt, source.terminatedAt);
+}
+function mergeProjectionMetadata(event, existing) {
+    return {
+        ...event,
+        reservationId: event.reservationId ?? existing.reservationId,
+        templateKey: event.templateKey ?? existing.templateKey,
+        runnerSessionId: event.runnerSessionId ?? existing.runnerSessionId,
+        providerKind: event.providerKind ?? existing.providerKind,
+        ...pickMilestones(existing)
+    };
+}
+function pickMilestones(event) {
+    return {
+        startedAt: event.startedAt,
+        stoppingAt: event.stoppingAt,
+        stoppedAt: event.stoppedAt,
+        failedAt: event.failedAt,
+        terminatedAt: event.terminatedAt
+    };
+}
+function earliestDate(a, b) {
+    if (!a) return b;
+    if (!b) return a;
+    return a < b ? a : b;
+}
+function compareRunnerInstanceReportEvents(a, b) {
+    const timeDelta = a.reportedAt.getTime() - b.reportedAt.getTime();
+    const rankDelta = getRunnerInstanceStateRank(a.state) - getRunnerInstanceStateRank(b.state);
+    if (rankDelta !== 0) return rankDelta;
+    return timeDelta;
+}
+function getRunnerInstanceStateRank(state) {
+    switch(state){
+        case 'starting':
+            return 1;
+        case 'running':
+            return 2;
+        case 'stopping':
+            return 3;
+        case 'stopped':
+            return 4;
+        case 'failed':
+            return 5;
+        case 'terminated':
+            return 6;
+    }
+}
+function providerRunnerStateRank(state) {
+    return sql`
+    CASE ${state}
+      WHEN 'starting' THEN 1
+      WHEN 'running' THEN 2
+      WHEN 'stopping' THEN 3
+      WHEN 'stopped' THEN 4
+      WHEN 'failed' THEN 5
+      WHEN 'terminated' THEN 6
+      ELSE 0
+    END
+  `;
+}
+function providerRunnerProjectionUpdateCondition() {
+    return sql`
+    ${providerRunnerStateRank(sql`excluded.state`)} > ${providerRunnerStateRank(providerRunners.state)}
+    OR (
+      ${providerRunnerStateRank(sql`excluded.state`)} = ${providerRunnerStateRank(providerRunners.state)}
+      AND excluded.reported_at >= ${providerRunners.reportedAt}
+    )
+  `;
+}
+function providerRunnerMilestoneUpdateCondition() {
+    return sql`
+    ${milestoneNeedsUpdate(providerRunners.startedAt, sql`excluded.started_at`)}
+    OR ${milestoneNeedsUpdate(providerRunners.stoppingAt, sql`excluded.stopping_at`)}
+    OR ${milestoneNeedsUpdate(providerRunners.stoppedAt, sql`excluded.stopped_at`)}
+    OR ${milestoneNeedsUpdate(providerRunners.failedAt, sql`excluded.failed_at`)}
+    OR ${milestoneNeedsUpdate(providerRunners.terminatedAt, sql`excluded.terminated_at`)}
+  `;
+}
+function milestoneNeedsUpdate(current, incoming) {
+    return sql`${incoming} IS NOT NULL AND (${current} IS NULL OR ${incoming} < ${current})`;
+}
+function firstObservedAt(current, incoming) {
+    return sql`
+    CASE
+      WHEN ${incoming} IS NULL THEN ${current}
+      WHEN ${current} IS NULL THEN ${incoming}
+      ELSE least(${current}, ${incoming})
+    END
+  `;
+}
+export function isTerminalState(state) {
+    return terminalStates.includes(state);
+}
+function isDivergenceCountState(state) {
+    return divergenceCountStates.includes(state);
+}
+
+//# sourceMappingURL=runner-instances.js.map

package/dist/db/runner-instances.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/db/runner-instances.ts"],"sourcesContent":["import {logger} from '@shipfox/node-opentelemetry';\nimport {canonicalizeLabels} from '@shipfox/runner-labels';\nimport {\n  and,\n  asc,\n  desc,\n  eq,\n  exists,\n  gt,\n  inArray,\n  isNotNull,\n  isNull,\n  lt,\n  notExists,\n  notInArray,\n  or,\n  type SQL,\n  sql,\n} from 'drizzle-orm';\nimport {alias} from 'drizzle-orm/pg-core';\nimport type {RunnerInstance, RunnerInstanceState} from '#core/entities/runner-instance.js';\nimport type {Tx} from './db.js';\nimport {db} from './db.js';\nimport {\n  listRunningJobExecutionsByRunnerInstanceTx,\n  type RunnerInstanceBoundJobExecution,\n} from './job-executions.js';\nimport {releaseReservationUnits} from './reservations.js';\nimport {ephemeralRegistrationTokens} from './schema/ephemeral-registration-tokens.js';\nimport {provisionerTokens} from './schema/provisioner-tokens.js';\nimport {providerRunners, toRunnerInstance} from './schema/runner-instances.js';\nimport {runnerSessions} from './schema/runner-sessions.js';\nimport {runningJobExecutions} from './schema/running-job-executions.js';\n\nexport const terminalStates = [\n  'stopped',\n  'failed',\n  'terminated',\n] as const satisfies readonly RunnerInstanceState[];\nexport const activeStates = [\n  'starting',\n  'running',\n  'stopping',\n] as const satisfies readonly RunnerInstanceState[];\nexport const divergenceCountStates = ['starting', 'running'] as const satisfies readonly Extract<\n  RunnerInstanceState,\n  'starting' | 'running'\n>[];\n\nexport type RunnerInstanceTerminateIntentReason = 'job-cancelled';\n\nexport interface RunnerInstanceTerminateIntent {\n  providerRunnerId: string;\n  reason: RunnerInstanceTerminateIntentReason;\n}\n\nexport interface ActiveRunnerInstanceTemplateCount {\n  templateKey: string;\n  state: (typeof divergenceCountStates)[number];\n  count: number;\n}\n\nexport interface RunnerInstanceReportEvent {\n  providerRunnerId: string;\n  reservationId: string | null;\n  templateKey: string | null;\n  labels: string[];\n  state: RunnerInstanceState;\n  reason: string | null;\n  runnerSessionId: string | null;\n  providerKind: string | null;\n  reportedAt: Date;\n}\n\nexport interface ReportRunnerInstancesParams {\n  workspaceId: string | null;\n  provisionerId: string;\n  events: RunnerInstanceReportEvent[];\n}\n\nexport interface ReconcileRunnerInstancesParams {\n  workspaceId: string | null;\n  provisionerId: string;\n  observedRunnerInstanceIds: string[];\n  terminateGraceSeconds: number;\n}\n\nexport interface ReconcileRunnerInstancesDbResult {\n  observedRows: RunnerInstance[];\n  boundJobExecutionsByRunnerInstanceId: Map<string, RunnerInstanceBoundJobExecution>;\n  absentIds: string[];\n  reservationsReleased: number;\n}\n\nexport interface ReapStaleRunnerInstancesResult {\n  reaped: number;\n  reservationsReleased: number;\n}\n\nexport async function createPlannedProvisionedCapacity(params: {\n  provisionerId: string;\n  providerKind: string | null;\n  templateKey: string | null;\n}): Promise<{capacityId: string}> {\n  const [row] = await db()\n    .insert(providerRunners)\n    .values({\n      provisionerId: params.provisionerId,\n      providerKind: params.providerKind,\n      templateKey: params.templateKey,\n      state: 'starting',\n      labels: [],\n      reportedAt: new Date(),\n    })\n    .returning({capacityId: providerRunners.id});\n  if (!row) throw new Error('Planned capacity insert returned no row');\n  return row;\n}\n\nexport async function attachProviderRunnerId(params: {\n  capacityId: string;\n  provisionerId: string;\n  providerRunnerId: string;\n}): Promise<boolean> {\n  const updated = await db()\n    .update(providerRunners)\n    .set({providerRunnerId: params.providerRunnerId, updatedAt: sql`now()`})\n    .where(\n      and(\n        eq(providerRunners.id, params.capacityId),\n        eq(providerRunners.provisionerId, params.provisionerId),\n        isNull(providerRunners.providerRunnerId),\n        notInArray(providerRunners.state, [...terminalStates]),\n      ),\n    )\n    .returning({id: providerRunners.id});\n  return updated.length === 1;\n}\n\ninterface RunnerInstanceReportRow extends RunnerInstanceReportEvent {\n  startedAt: Date | null;\n  stoppingAt: Date | null;\n  stoppedAt: Date | null;\n  failedAt: Date | null;\n  terminatedAt: Date | null;\n}\n\ntype RunnerInstanceMilestoneColumn =\n  | typeof providerRunners.startedAt\n  | typeof providerRunners.stoppingAt\n  | typeof providerRunners.stoppedAt\n  | typeof providerRunners.failedAt\n  | typeof providerRunners.terminatedAt;\n\nexport async function reportRunnerInstances(params: ReportRunnerInstancesParams): Promise<{\n  accepted: number;\n  reservationsReleased: number;\n  terminateIntentsHonored: RunnerInstanceTerminateIntent[];\n}> {\n  if (params.events.length === 0)\n    return {accepted: 0, reservationsReleased: 0, terminateIntentsHonored: []};\n\n  return await db().transaction(async (tx) => {\n    const receivedAt = new Date();\n    const aggregatedEvents = aggregateEvents(params.events, receivedAt);\n    const hasTerminalEvent = aggregatedEvents.some((event) => isTerminalState(event.state));\n\n    if (hasTerminalEvent) {\n      await tx.execute(\n        sql`select pg_advisory_xact_lock(hashtext(${params.workspaceId ?? params.provisionerId}))`,\n      );\n    }\n\n    const events = await hydrateRunnerSessionIdsFromConsumedTokens(tx, params, aggregatedEvents);\n    const terminateIntentsHonored = await listTerminateIntentsHonoredByTerminatedReportsTx(\n      tx,\n      params,\n      events,\n    );\n\n    const values = events.map((event) => ({\n      workspaceId: params.workspaceId,\n      provisionerId: params.provisionerId,\n      providerRunnerId: event.providerRunnerId,\n      reservationId: event.reservationId,\n      templateKey: event.templateKey,\n      labels: [...canonicalizeLabels(event.labels)],\n      state: event.state,\n      reason: event.reason,\n      runnerSessionId: event.runnerSessionId,\n      providerKind: event.providerKind,\n      reportedAt: event.reportedAt > receivedAt ? receivedAt : event.reportedAt,\n      startedAt: event.startedAt,\n      stoppingAt: event.stoppingAt,\n      stoppedAt: event.stoppedAt,\n      failedAt: event.failedAt,\n      terminatedAt: event.terminatedAt,\n    }));\n\n    await tx\n      .insert(providerRunners)\n      .values(values)\n      .onConflictDoUpdate({\n        target: [providerRunners.provisionerId, providerRunners.providerRunnerId],\n        targetWhere: isNotNull(providerRunners.providerRunnerId),\n        set: {\n          reservationId: sql`CASE WHEN ${providerRunnerProjectionUpdateCondition()} THEN coalesce(excluded.reservation_id, ${providerRunners.reservationId}) ELSE ${providerRunners.reservationId} END`,\n          templateKey: sql`CASE WHEN ${providerRunnerProjectionUpdateCondition()} THEN coalesce(excluded.template_key, ${providerRunners.templateKey}) ELSE ${providerRunners.templateKey} END`,\n          labels: sql`CASE WHEN ${providerRunnerProjectionUpdateCondition()} THEN excluded.labels ELSE ${providerRunners.labels} END`,\n          state: sql`CASE WHEN ${providerRunnerProjectionUpdateCondition()} THEN excluded.state ELSE ${providerRunners.state} END`,\n          reason: sql`CASE WHEN ${providerRunnerProjectionUpdateCondition()} THEN excluded.reason ELSE ${providerRunners.reason} END`,\n          runnerSessionId: sql`CASE WHEN ${providerRunnerProjectionUpdateCondition()} THEN coalesce(${providerRunners.runnerSessionId}, excluded.runner_session_id) ELSE ${providerRunners.runnerSessionId} END`,\n          providerKind: sql`CASE WHEN ${providerRunnerProjectionUpdateCondition()} THEN coalesce(excluded.provider_kind, ${providerRunners.providerKind}) ELSE ${providerRunners.providerKind} END`,\n          reportedAt: sql`CASE WHEN ${providerRunnerProjectionUpdateCondition()} THEN excluded.reported_at ELSE ${providerRunners.reportedAt} END`,\n          startedAt: firstObservedAt(providerRunners.startedAt, sql`excluded.started_at`),\n          stoppingAt: firstObservedAt(providerRunners.stoppingAt, sql`excluded.stopping_at`),\n          stoppedAt: firstObservedAt(providerRunners.stoppedAt, sql`excluded.stopped_at`),\n          failedAt: firstObservedAt(providerRunners.failedAt, sql`excluded.failed_at`),\n          terminatedAt: firstObservedAt(providerRunners.terminatedAt, sql`excluded.terminated_at`),\n          updatedAt: sql`now()`,\n        },\n        setWhere: sql`\n          ${providerRunnerProjectionUpdateCondition()}\n          OR ${providerRunnerMilestoneUpdateCondition()}\n        `,\n      });\n\n    const reservationsReleased = hasTerminalEvent\n      ? await releaseTerminalRunnerInstanceReservations(tx, params, events)\n      : 0;\n\n    return {accepted: events.length, reservationsReleased, terminateIntentsHonored};\n  });\n}\n\nexport async function listActiveRunnerInstanceCountsByTemplateTx(\n  tx: Tx,\n  params: {workspaceId: string; provisionerId: string},\n): Promise<ActiveRunnerInstanceTemplateCount[]> {\n  const rows = await tx\n    .select({\n      templateKey: providerRunners.templateKey,\n      state: providerRunners.state,\n      count: sql<number>`count(*)::int`,\n    })\n    .from(providerRunners)\n    .where(\n      and(\n        eq(providerRunners.workspaceId, params.workspaceId),\n        eq(providerRunners.provisionerId, params.provisionerId),\n        inArray(providerRunners.state, divergenceCountStates),\n        isNotNull(providerRunners.templateKey),\n      ),\n    )\n    .groupBy(providerRunners.templateKey, providerRunners.state);\n\n  return rows.flatMap((row) =>\n    row.templateKey && isDivergenceCountState(row.state)\n      ? [{templateKey: row.templateKey, state: row.state, count: row.count}]\n      : [],\n  );\n}\n\nexport async function listActiveRunnerInstances(params: {\n  workspaceId: string;\n  windowSeconds: number;\n  limit?: number;\n}): Promise<RunnerInstance[]> {\n  const rows = await db()\n    .select()\n    .from(providerRunners)\n    .where(\n      and(\n        eq(providerRunners.workspaceId, params.workspaceId),\n        inArray(providerRunners.state, activeStates),\n        sql`${providerRunners.updatedAt} > now() - (${params.windowSeconds} || ' seconds')::interval`,\n      ),\n    )\n    .orderBy(desc(providerRunners.updatedAt), desc(providerRunners.id))\n    .limit(params.limit ?? 1000);\n\n  return rows.map(toRunnerInstance);\n}\n\nexport async function listProvisionerTerminateIntents(params: {\n  workspaceId: string;\n  provisionerId: string;\n  limit: number;\n}): Promise<string[]> {\n  return await db().transaction(async (tx) => {\n    const rows = await listProvisionerTerminateIntentRowsTx(tx, params);\n    return rows.map((row) => row.providerRunnerId);\n  });\n}\n\nexport async function listProvisionerTerminateIntentRowsTx(\n  tx: Tx,\n  params: {\n    workspaceId: string;\n    provisionerId: string;\n    limit: number;\n  },\n): Promise<RunnerInstanceTerminateIntent[]> {\n  const rows = await provisionerTerminateIntentsQuery(tx, params)\n    .orderBy(asc(providerRunners.providerRunnerId))\n    .limit(params.limit + 1);\n\n  const truncated = rows.length > params.limit;\n  const returnedRows = truncated ? rows.slice(0, params.limit) : rows;\n  if (truncated) {\n    logger().warn(\n      {\n        workspaceId: params.workspaceId,\n        provisionerId: params.provisionerId,\n        limit: params.limit,\n        returnedCount: returnedRows.length,\n      },\n      'provisioner terminate intents truncated by poll-demand limit',\n    );\n  }\n\n  return returnedRows.flatMap((row) =>\n    row.providerRunnerId ? [{...row, providerRunnerId: row.providerRunnerId}] : [],\n  );\n}\n\nfunction provisionerTerminateIntentsQuery(\n  tx: Tx,\n  params: {workspaceId: string; provisionerId: string; providerRunnerIds?: string[]},\n) {\n  const newerRunningJobExecutions = alias(runningJobExecutions, 'newer_running_jobs');\n\n  return tx\n    .select({\n      providerRunnerId: providerRunners.providerRunnerId,\n      reason: sql<RunnerInstanceTerminateIntentReason>`'job-cancelled'`,\n    })\n    .from(runningJobExecutions)\n    .innerJoin(\n      providerRunners,\n      and(\n        eq(providerRunners.workspaceId, runningJobExecutions.workspaceId),\n        eq(providerRunners.provisionerId, runningJobExecutions.provisionerId),\n        eq(providerRunners.providerRunnerId, runningJobExecutions.providerRunnerId),\n      ),\n    )\n    .where(\n      and(\n        eq(runningJobExecutions.workspaceId, params.workspaceId),\n        eq(runningJobExecutions.provisionerId, params.provisionerId),\n        isNotNull(providerRunners.providerRunnerId),\n        isNotNull(runningJobExecutions.cancellationRequestedAt),\n        inArray(providerRunners.state, activeStates),\n        params.providerRunnerIds && params.providerRunnerIds.length > 0\n          ? inArray(providerRunners.providerRunnerId, params.providerRunnerIds)\n          : undefined,\n        notExists(\n          tx\n            .select({id: newerRunningJobExecutions.id})\n            .from(newerRunningJobExecutions)\n            .where(\n              and(\n                eq(newerRunningJobExecutions.workspaceId, runningJobExecutions.workspaceId),\n                eq(newerRunningJobExecutions.provisionerId, runningJobExecutions.provisionerId),\n                eq(\n                  newerRunningJobExecutions.providerRunnerId,\n                  runningJobExecutions.providerRunnerId,\n                ),\n                or(\n                  gt(newerRunningJobExecutions.startedAt, runningJobExecutions.startedAt),\n                  and(\n                    eq(newerRunningJobExecutions.startedAt, runningJobExecutions.startedAt),\n                    gt(\n                      newerRunningJobExecutions.jobExecutionId,\n                      runningJobExecutions.jobExecutionId,\n                    ),\n                  ),\n                ),\n              ),\n            ),\n        ),\n      ),\n    )\n    .groupBy(providerRunners.providerRunnerId);\n}\n\nasync function listTerminateIntentsHonoredByTerminatedReportsTx(\n  tx: Tx,\n  params: ReportRunnerInstancesParams,\n  events: RunnerInstanceReportEvent[],\n): Promise<RunnerInstanceTerminateIntent[]> {\n  if (!params.workspaceId) return [];\n  const terminatedRunnerInstanceIds = [\n    ...new Set(\n      events.filter((event) => event.state === 'terminated').map((event) => event.providerRunnerId),\n    ),\n  ];\n  if (terminatedRunnerInstanceIds.length === 0) return [];\n\n  const rows = await provisionerTerminateIntentsQuery(tx, {\n    workspaceId: params.workspaceId,\n    provisionerId: params.provisionerId,\n    providerRunnerIds: terminatedRunnerInstanceIds,\n  }).orderBy(asc(providerRunners.providerRunnerId));\n  return rows.flatMap((row) =>\n    row.providerRunnerId ? [{...row, providerRunnerId: row.providerRunnerId}] : [],\n  );\n}\n\nexport async function reconcileRunnerInstances(\n  params: ReconcileRunnerInstancesParams,\n): Promise<ReconcileRunnerInstancesDbResult> {\n  const observedRunnerInstanceIds = [...new Set(params.observedRunnerInstanceIds)];\n\n  return await db().transaction(async (tx) => {\n    await tx.execute(\n      sql`select pg_advisory_xact_lock(hashtext(${params.workspaceId ?? params.provisionerId}))`,\n    );\n\n    let absentIds: string[] = [];\n    let reservationsReleased = 0;\n    if (observedRunnerInstanceIds.length > 0) {\n      const staleAbsentRows = await tx\n        .select({\n          id: providerRunners.id,\n          providerRunnerId: providerRunners.providerRunnerId,\n        })\n        .from(providerRunners)\n        .where(\n          and(\n            params.workspaceId\n              ? eq(providerRunners.workspaceId, params.workspaceId)\n              : isNull(providerRunners.workspaceId),\n            eq(providerRunners.provisionerId, params.provisionerId),\n            inArray(providerRunners.state, activeStates),\n            lt(\n              providerRunners.reportedAt,\n              sql`now() - (${params.terminateGraceSeconds} || ' seconds')::interval`,\n            ),\n            notInArray(providerRunners.providerRunnerId, observedRunnerInstanceIds),\n          ),\n        );\n\n      if (staleAbsentRows.length > 0) {\n        const updated = await tx\n          .update(providerRunners)\n          .set({\n            state: 'terminated',\n            terminatedAt: sql`coalesce(${providerRunners.terminatedAt}, now())`,\n            updatedAt: sql`now()`,\n          })\n          .where(\n            and(\n              inArray(\n                providerRunners.id,\n                staleAbsentRows.map((row) => row.id),\n              ),\n              inArray(providerRunners.state, activeStates),\n              lt(\n                providerRunners.reportedAt,\n                sql`now() - (${params.terminateGraceSeconds} || ' seconds')::interval`,\n              ),\n            ),\n          )\n          .returning({providerRunnerId: providerRunners.providerRunnerId});\n\n        absentIds = updated.flatMap((row) => (row.providerRunnerId ? [row.providerRunnerId] : []));\n        if (params.workspaceId) {\n          reservationsReleased = await releaseTerminalRunnerInstanceReservationsByIds(tx, {\n            workspaceId: params.workspaceId,\n            provisionerId: params.provisionerId,\n            providerRunnerIds: absentIds,\n          });\n        }\n      }\n    }\n\n    const observedRows =\n      observedRunnerInstanceIds.length === 0\n        ? []\n        : (\n            await tx\n              .select()\n              .from(providerRunners)\n              .where(\n                and(\n                  params.workspaceId\n                    ? eq(providerRunners.workspaceId, params.workspaceId)\n                    : isNull(providerRunners.workspaceId),\n                  eq(providerRunners.provisionerId, params.provisionerId),\n                  inArray(providerRunners.providerRunnerId, observedRunnerInstanceIds),\n                ),\n              )\n          ).map(toRunnerInstance);\n\n    const boundJobExecutions = params.workspaceId\n      ? await listRunningJobExecutionsByRunnerInstanceTx(tx, {\n          workspaceId: params.workspaceId,\n          provisionerId: params.provisionerId,\n          providerRunnerIds: observedRunnerInstanceIds,\n        })\n      : [];\n\n    return {\n      observedRows,\n      boundJobExecutionsByRunnerInstanceId: new Map(\n        boundJobExecutions.map((jobExecution) => [jobExecution.providerRunnerId, jobExecution]),\n      ),\n      absentIds,\n      reservationsReleased,\n    };\n  });\n}\n\nexport async function reapStaleRunnerInstances(params: {\n  thresholdSeconds: number;\n  limit: number;\n}): Promise<ReapStaleRunnerInstancesResult> {\n  const cutoff = staleRunnerInstanceCutoff(params.thresholdSeconds);\n\n  return await db().transaction(async (tx) => {\n    const candidateRows = await tx\n      .select({\n        id: providerRunners.id,\n        workspaceId: providerRunners.workspaceId,\n        provisionerId: providerRunners.provisionerId,\n        providerRunnerId: providerRunners.providerRunnerId,\n      })\n      .from(providerRunners)\n      .where(staleRunnerInstanceWhere(tx, cutoff))\n      .orderBy(asc(providerRunners.updatedAt), asc(providerRunners.id))\n      .limit(params.limit);\n\n    if (candidateRows.length === 0) return {reaped: 0, reservationsReleased: 0};\n\n    const workspaceIds = [\n      ...new Set(candidateRows.flatMap((row) => (row.workspaceId ? [row.workspaceId] : []))),\n    ].sort();\n    for (const workspaceId of workspaceIds) {\n      await tx.execute(sql`select pg_advisory_xact_lock(hashtext(${workspaceId}))`);\n    }\n\n    const updatedRows = await tx\n      .update(providerRunners)\n      .set({\n        state: 'failed',\n        reason: 'stale-provisioner',\n        failedAt: sql`coalesce(${providerRunners.failedAt}, now())`,\n        updatedAt: sql`now()`,\n      })\n      .where(\n        and(\n          inArray(\n            providerRunners.id,\n            candidateRows.map((row) => row.id),\n          ),\n          staleRunnerInstanceWhere(tx, cutoff),\n        ),\n      )\n      .returning({\n        workspaceId: providerRunners.workspaceId,\n        provisionerId: providerRunners.provisionerId,\n        providerRunnerId: providerRunners.providerRunnerId,\n      });\n\n    let reservationsReleased = 0;\n    for (const group of groupRunnerInstanceIds(updatedRows)) {\n      reservationsReleased += await releaseTerminalRunnerInstanceReservationsByIds(tx, {\n        workspaceId: group.workspaceId,\n        provisionerId: group.provisionerId,\n        providerRunnerIds: group.providerRunnerIds,\n        requireUnlinkedSession: false,\n      });\n    }\n\n    return {reaped: updatedRows.length, reservationsReleased};\n  });\n}\n\nasync function releaseTerminalRunnerInstanceReservations(\n  tx: Tx,\n  params: ReportRunnerInstancesParams,\n  events: RunnerInstanceReportEvent[],\n): Promise<number> {\n  const terminalEvents = events.filter((event) => isTerminalState(event.state));\n  if (terminalEvents.length === 0 || !params.workspaceId) return 0;\n\n  return await releaseTerminalRunnerInstanceReservationsByIds(tx, {\n    workspaceId: params.workspaceId,\n    provisionerId: params.provisionerId,\n    providerRunnerIds: terminalEvents.map((event) => event.providerRunnerId),\n  });\n}\n\nasync function releaseTerminalRunnerInstanceReservationsByIds(\n  tx: Tx,\n  params: {\n    workspaceId: string;\n    provisionerId: string;\n    providerRunnerIds: string[];\n    requireUnlinkedSession?: boolean;\n  },\n): Promise<number> {\n  if (params.providerRunnerIds.length === 0) return 0;\n\n  const rows = await tx\n    .select({\n      id: providerRunners.id,\n      reservationId: providerRunners.reservationId,\n    })\n    .from(providerRunners)\n    .where(\n      and(\n        eq(providerRunners.workspaceId, params.workspaceId),\n        eq(providerRunners.provisionerId, params.provisionerId),\n        inArray(providerRunners.providerRunnerId, params.providerRunnerIds),\n        inArray(providerRunners.state, terminalStates),\n        isNotNull(providerRunners.reservationId),\n        params.requireUnlinkedSession === false\n          ? undefined\n          : isNull(providerRunners.runnerSessionId),\n        isNull(providerRunners.reservationReleasedAt),\n      ),\n    );\n\n  if (rows.length === 0) return 0;\n\n  const updated = await tx\n    .update(providerRunners)\n    .set({reservationReleasedAt: sql`now()`, updatedAt: sql`now()`})\n    .where(\n      and(\n        inArray(\n          providerRunners.id,\n          rows.map((row) => row.id),\n        ),\n        params.requireUnlinkedSession === false\n          ? undefined\n          : isNull(providerRunners.runnerSessionId),\n        isNull(providerRunners.reservationReleasedAt),\n      ),\n    )\n    .returning({reservationId: providerRunners.reservationId});\n\n  const releasesByReservationId = new Map<string, number>();\n  for (const row of updated) {\n    if (!row.reservationId) continue;\n    releasesByReservationId.set(\n      row.reservationId,\n      (releasesByReservationId.get(row.reservationId) ?? 0) + 1,\n    );\n  }\n\n  if (releasesByReservationId.size === 0) return 0;\n\n  return await releaseReservationUnits(tx, {\n    workspaceId: params.workspaceId,\n    provisionerId: params.provisionerId,\n    releases: [...releasesByReservationId].map(([reservationId, count]) => ({\n      reservationId,\n      count,\n    })),\n  });\n}\n\nfunction staleRunnerInstanceCutoff(thresholdSeconds: number): SQL {\n  return sql`now() - (${thresholdSeconds} || ' seconds')::interval`;\n}\n\nfunction staleRunnerInstanceWhere(tx: Tx, cutoff: SQL): SQL<boolean> {\n  return and(\n    inArray(providerRunners.state, activeStates),\n    lt(providerRunners.reportedAt, cutoff),\n    lt(providerRunners.updatedAt, cutoff),\n    exists(\n      tx\n        .select({id: provisionerTokens.id})\n        .from(provisionerTokens)\n        .where(\n          and(\n            eq(provisionerTokens.id, providerRunners.provisionerId),\n            or(isNull(provisionerTokens.lastSeenAt), lt(provisionerTokens.lastSeenAt, cutoff)),\n          ),\n        ),\n    ),\n    notExists(\n      tx\n        .select({id: runningJobExecutions.id})\n        .from(runningJobExecutions)\n        .where(\n          and(\n            eq(runningJobExecutions.workspaceId, providerRunners.workspaceId),\n            eq(runningJobExecutions.provisionerId, providerRunners.provisionerId),\n            eq(runningJobExecutions.providerRunnerId, providerRunners.providerRunnerId),\n          ),\n        ),\n    ),\n    notExists(\n      tx\n        .select({id: runnerSessions.id})\n        .from(runnerSessions)\n        .where(\n          and(\n            eq(runnerSessions.workspaceId, providerRunners.workspaceId),\n            eq(runnerSessions.provisionerId, providerRunners.provisionerId),\n            eq(runnerSessions.providerRunnerId, providerRunners.providerRunnerId),\n            sql`${runnerSessions.updatedAt} >= ${cutoff}`,\n          ),\n        ),\n    ),\n  ) as SQL<boolean>;\n}\n\nfunction groupRunnerInstanceIds(\n  rows: Array<{\n    workspaceId: string | null;\n    provisionerId: string;\n    providerRunnerId: string | null;\n  }>,\n): Array<{workspaceId: string; provisionerId: string; providerRunnerIds: string[]}> {\n  const groups = new Map<\n    string,\n    {workspaceId: string; provisionerId: string; providerRunnerIds: string[]}\n  >();\n  for (const row of rows) {\n    if (!row.workspaceId || !row.providerRunnerId) continue;\n    const key = `${row.workspaceId}:${row.provisionerId}`;\n    const group = groups.get(key) ?? {\n      workspaceId: row.workspaceId,\n      provisionerId: row.provisionerId,\n      providerRunnerIds: [],\n    };\n    group.providerRunnerIds.push(row.providerRunnerId);\n    groups.set(key, group);\n  }\n  return [...groups.values()];\n}\n\nasync function hydrateRunnerSessionIdsFromConsumedTokens(\n  tx: Tx,\n  params: ReportRunnerInstancesParams,\n  events: RunnerInstanceReportRow[],\n): Promise<RunnerInstanceReportRow[]> {\n  if (!params.workspaceId) return events;\n  const providerRunnerIds = [...new Set(events.map((event) => event.providerRunnerId))];\n  if (providerRunnerIds.length === 0) return events;\n\n  const tokenRows = await tx\n    .select({\n      providerRunnerId: ephemeralRegistrationTokens.providerRunnerId,\n      consumedSessionId: ephemeralRegistrationTokens.consumedSessionId,\n    })\n    .from(ephemeralRegistrationTokens)\n    .where(\n      and(\n        eq(ephemeralRegistrationTokens.workspaceId, params.workspaceId),\n        eq(ephemeralRegistrationTokens.provisionerId, params.provisionerId),\n        inArray(ephemeralRegistrationTokens.providerRunnerId, providerRunnerIds),\n        isNotNull(ephemeralRegistrationTokens.consumedSessionId),\n      ),\n    )\n    .orderBy(\n      desc(ephemeralRegistrationTokens.consumedAt),\n      desc(ephemeralRegistrationTokens.createdAt),\n    );\n\n  const consumedSessionIdsByRunnerInstanceId = new Map<string, string>();\n  for (const row of tokenRows) {\n    if (!row.consumedSessionId) continue;\n    if (consumedSessionIdsByRunnerInstanceId.has(row.providerRunnerId)) continue;\n    consumedSessionIdsByRunnerInstanceId.set(row.providerRunnerId, row.consumedSessionId);\n  }\n\n  if (consumedSessionIdsByRunnerInstanceId.size === 0) return events;\n\n  return events.map((event) => {\n    const consumedSessionId = consumedSessionIdsByRunnerInstanceId.get(event.providerRunnerId);\n    if (!consumedSessionId || event.runnerSessionId === consumedSessionId) return event;\n    return {...event, runnerSessionId: consumedSessionId};\n  });\n}\n\nfunction aggregateEvents(\n  events: RunnerInstanceReportEvent[],\n  receivedAt: Date,\n): RunnerInstanceReportRow[] {\n  const byRunnerInstanceId = new Map<string, RunnerInstanceReportRow>();\n  for (const rawEvent of events) {\n    const event = toRunnerInstanceReportRow(rawEvent, receivedAt);\n    const existing = byRunnerInstanceId.get(event.providerRunnerId);\n    if (existing) mergeMilestones(existing, event);\n    if (!existing || compareRunnerInstanceReportEvents(event, existing) > 0) {\n      byRunnerInstanceId.set(\n        event.providerRunnerId,\n        existing ? mergeProjectionMetadata(event, existing) : event,\n      );\n    }\n  }\n  return [...byRunnerInstanceId.values()];\n}\n\nfunction toRunnerInstanceReportRow(\n  event: RunnerInstanceReportEvent,\n  receivedAt: Date,\n): RunnerInstanceReportRow {\n  const reportedAt = event.reportedAt > receivedAt ? receivedAt : event.reportedAt;\n  return {\n    ...event,\n    reportedAt,\n    startedAt: event.state === 'running' ? reportedAt : null,\n    stoppingAt: event.state === 'stopping' ? reportedAt : null,\n    stoppedAt: event.state === 'stopped' ? reportedAt : null,\n    failedAt: event.state === 'failed' ? reportedAt : null,\n    terminatedAt: event.state === 'terminated' ? reportedAt : null,\n  };\n}\n\nfunction mergeMilestones(target: RunnerInstanceReportRow, source: RunnerInstanceReportRow) {\n  target.startedAt = earliestDate(target.startedAt, source.startedAt);\n  target.stoppingAt = earliestDate(target.stoppingAt, source.stoppingAt);\n  target.stoppedAt = earliestDate(target.stoppedAt, source.stoppedAt);\n  target.failedAt = earliestDate(target.failedAt, source.failedAt);\n  target.terminatedAt = earliestDate(target.terminatedAt, source.terminatedAt);\n}\n\nfunction mergeProjectionMetadata(\n  event: RunnerInstanceReportRow,\n  existing: RunnerInstanceReportRow,\n): RunnerInstanceReportRow {\n  return {\n    ...event,\n    reservationId: event.reservationId ?? existing.reservationId,\n    templateKey: event.templateKey ?? existing.templateKey,\n    runnerSessionId: event.runnerSessionId ?? existing.runnerSessionId,\n    providerKind: event.providerKind ?? existing.providerKind,\n    ...pickMilestones(existing),\n  };\n}\n\nfunction pickMilestones(event: RunnerInstanceReportRow) {\n  return {\n    startedAt: event.startedAt,\n    stoppingAt: event.stoppingAt,\n    stoppedAt: event.stoppedAt,\n    failedAt: event.failedAt,\n    terminatedAt: event.terminatedAt,\n  };\n}\n\nfunction earliestDate(a: Date | null, b: Date | null): Date | null {\n  if (!a) return b;\n  if (!b) return a;\n  return a < b ? a : b;\n}\n\nfunction compareRunnerInstanceReportEvents(\n  a: RunnerInstanceReportEvent,\n  b: RunnerInstanceReportEvent,\n): number {\n  const timeDelta = a.reportedAt.getTime() - b.reportedAt.getTime();\n  const rankDelta = getRunnerInstanceStateRank(a.state) - getRunnerInstanceStateRank(b.state);\n  if (rankDelta !== 0) return rankDelta;\n  return timeDelta;\n}\n\nfunction getRunnerInstanceStateRank(state: RunnerInstanceState): number {\n  switch (state) {\n    case 'starting':\n      return 1;\n    case 'running':\n      return 2;\n    case 'stopping':\n      return 3;\n    case 'stopped':\n      return 4;\n    case 'failed':\n      return 5;\n    case 'terminated':\n      return 6;\n  }\n}\n\nfunction providerRunnerStateRank(state: SQL | typeof providerRunners.state): SQL<number> {\n  return sql<number>`\n    CASE ${state}\n      WHEN 'starting' THEN 1\n      WHEN 'running' THEN 2\n      WHEN 'stopping' THEN 3\n      WHEN 'stopped' THEN 4\n      WHEN 'failed' THEN 5\n      WHEN 'terminated' THEN 6\n      ELSE 0\n    END\n  `;\n}\n\nfunction providerRunnerProjectionUpdateCondition(): SQL<boolean> {\n  return sql<boolean>`\n    ${providerRunnerStateRank(sql`excluded.state`)} > ${providerRunnerStateRank(providerRunners.state)}\n    OR (\n      ${providerRunnerStateRank(sql`excluded.state`)} = ${providerRunnerStateRank(providerRunners.state)}\n      AND excluded.reported_at >= ${providerRunners.reportedAt}\n    )\n  `;\n}\n\nfunction providerRunnerMilestoneUpdateCondition(): SQL<boolean> {\n  return sql<boolean>`\n    ${milestoneNeedsUpdate(providerRunners.startedAt, sql`excluded.started_at`)}\n    OR ${milestoneNeedsUpdate(providerRunners.stoppingAt, sql`excluded.stopping_at`)}\n    OR ${milestoneNeedsUpdate(providerRunners.stoppedAt, sql`excluded.stopped_at`)}\n    OR ${milestoneNeedsUpdate(providerRunners.failedAt, sql`excluded.failed_at`)}\n    OR ${milestoneNeedsUpdate(providerRunners.terminatedAt, sql`excluded.terminated_at`)}\n  `;\n}\n\nfunction milestoneNeedsUpdate(\n  current: SQL | RunnerInstanceMilestoneColumn,\n  incoming: SQL,\n): SQL<boolean> {\n  return sql<boolean>`${incoming} IS NOT NULL AND (${current} IS NULL OR ${incoming} < ${current})`;\n}\n\nfunction firstObservedAt(current: SQL | RunnerInstanceMilestoneColumn, incoming: SQL) {\n  return sql`\n    CASE\n      WHEN ${incoming} IS NULL THEN ${current}\n      WHEN ${current} IS NULL THEN ${incoming}\n      ELSE least(${current}, ${incoming})\n    END\n  `;\n}\n\nexport function isTerminalState(state: RunnerInstanceState): boolean {\n  return terminalStates.includes(state as (typeof terminalStates)[number]);\n}\n\nfunction isDivergenceCountState(\n  state: RunnerInstanceState,\n): state is (typeof divergenceCountStates)[number] {\n  return divergenceCountStates.includes(state as (typeof divergenceCountStates)[number]);\n}\n"],"names":["logger","canonicalizeLabels","and","asc","desc","eq","exists","gt","inArray","isNotNull","isNull","lt","notExists","notInArray","or","sql","alias","db","listRunningJobExecutionsByRunnerInstanceTx","releaseReservationUnits","ephemeralRegistrationTokens","provisionerTokens","providerRunners","toRunnerInstance","runnerSessions","runningJobExecutions","terminalStates","activeStates","divergenceCountStates","createPlannedProvisionedCapacity","params","row","insert","values","provisionerId","providerKind","templateKey","state","labels","reportedAt","Date","returning","capacityId","id","Error","attachProviderRunnerId","updated","update","set","providerRunnerId","updatedAt","where","length","reportRunnerInstances","events","accepted","reservationsReleased","terminateIntentsHonored","transaction","tx","receivedAt","aggregatedEvents","aggregateEvents","hasTerminalEvent","some","event","isTerminalState","execute","workspaceId","hydrateRunnerSessionIdsFromConsumedTokens","listTerminateIntentsHonoredByTerminatedReportsTx","map","reservationId","reason","runnerSessionId","startedAt","stoppingAt","stoppedAt","failedAt","terminatedAt","onConflictDoUpdate","target","targetWhere","providerRunnerProjectionUpdateCondition","firstObservedAt","setWhere","providerRunnerMilestoneUpdateCondition","releaseTerminalRunnerInstanceReservations","listActiveRunnerInstanceCountsByTemplateTx","rows","select","count","from","groupBy","flatMap","isDivergenceCountState","listActiveRunnerInstances","windowSeconds","orderBy","limit","listProvisionerTerminateIntents","listProvisionerTerminateIntentRowsTx","provisionerTerminateIntentsQuery","truncated","returnedRows","slice","warn","returnedCount","newerRunningJobExecutions","innerJoin","cancellationRequestedAt","providerRunnerIds","undefined","jobExecutionId","terminatedRunnerInstanceIds","Set","filter","reconcileRunnerInstances","observedRunnerInstanceIds","absentIds","staleAbsentRows","terminateGraceSeconds","releaseTerminalRunnerInstanceReservationsByIds","observedRows","boundJobExecutions","boundJobExecutionsByRunnerInstanceId","Map","jobExecution","reapStaleRunnerInstances","cutoff","staleRunnerInstanceCutoff","thresholdSeconds","candidateRows","staleRunnerInstanceWhere","reaped","workspaceIds","sort","updatedRows","group","groupRunnerInstanceIds","requireUnlinkedSession","terminalEvents","reservationReleasedAt","releasesByReservationId","get","size","releases","lastSeenAt","groups","key","push","tokenRows","consumedSessionId","consumedAt","createdAt","consumedSessionIdsByRunnerInstanceId","has","byRunnerInstanceId","rawEvent","toRunnerInstanceReportRow","existing","mergeMilestones","compareRunnerInstanceReportEvents","mergeProjectionMetadata","source","earliestDate","pickMilestones","a","b","timeDelta","getTime","rankDelta","getRunnerInstanceStateRank","providerRunnerStateRank","milestoneNeedsUpdate","current","incoming","includes"],"mappings":"AAAA,SAAQA,MAAM,QAAO,8BAA8B;AACnD,SAAQC,kBAAkB,QAAO,yBAAyB;AAC1D,SACEC,GAAG,EACHC,GAAG,EACHC,IAAI,EACJC,EAAE,EACFC,MAAM,EACNC,EAAE,EACFC,OAAO,EACPC,SAAS,EACTC,MAAM,EACNC,EAAE,EACFC,SAAS,EACTC,UAAU,EACVC,EAAE,EAEFC,GAAG,QACE,cAAc;AACrB,SAAQC,KAAK,QAAO,sBAAsB;AAG1C,SAAQC,EAAE,QAAO,UAAU;AAC3B,SACEC,0CAA0C,QAErC,sBAAsB;AAC7B,SAAQC,uBAAuB,QAAO,oBAAoB;AAC1D,SAAQC,2BAA2B,QAAO,4CAA4C;AACtF,SAAQC,iBAAiB,QAAO,iCAAiC;AACjE,SAAQC,eAAe,EAAEC,gBAAgB,QAAO,+BAA+B;AAC/E,SAAQC,cAAc,QAAO,8BAA8B;AAC3D,SAAQC,oBAAoB,QAAO,qCAAqC;AAExE,OAAO,MAAMC,iBAAiB;IAC5B;IACA;IACA;CACD,CAAmD;AACpD,OAAO,MAAMC,eAAe;IAC1B;IACA;IACA;CACD,CAAmD;AACpD,OAAO,MAAMC,wBAAwB;IAAC;IAAY;CAAU,CAGxD;AAoDJ,OAAO,eAAeC,iCAAiCC,MAItD;IACC,MAAM,CAACC,IAAI,GAAG,MAAMd,KACjBe,MAAM,CAACV,iBACPW,MAAM,CAAC;QACNC,eAAeJ,OAAOI,aAAa;QACnCC,cAAcL,OAAOK,YAAY;QACjCC,aAAaN,OAAOM,WAAW;QAC/BC,OAAO;QACPC,QAAQ,EAAE;QACVC,YAAY,IAAIC;IAClB,GACCC,SAAS,CAAC;QAACC,YAAYpB,gBAAgBqB,EAAE;IAAA;IAC5C,IAAI,CAACZ,KAAK,MAAM,IAAIa,MAAM;IAC1B,OAAOb;AACT;AAEA,OAAO,eAAec,uBAAuBf,MAI5C;IACC,MAAMgB,UAAU,MAAM7B,KACnB8B,MAAM,CAACzB,iBACP0B,GAAG,CAAC;QAACC,kBAAkBnB,OAAOmB,gBAAgB;QAAEC,WAAWnC,GAAG,CAAC,KAAK,CAAC;IAAA,GACrEoC,KAAK,CACJjD,IACEG,GAAGiB,gBAAgBqB,EAAE,EAAEb,OAAOY,UAAU,GACxCrC,GAAGiB,gBAAgBY,aAAa,EAAEJ,OAAOI,aAAa,GACtDxB,OAAOY,gBAAgB2B,gBAAgB,GACvCpC,WAAWS,gBAAgBe,KAAK,EAAE;WAAIX;KAAe,IAGxDe,SAAS,CAAC;QAACE,IAAIrB,gBAAgBqB,EAAE;IAAA;IACpC,OAAOG,QAAQM,MAAM,KAAK;AAC5B;AAiBA,OAAO,eAAeC,sBAAsBvB,MAAmC;IAK7E,IAAIA,OAAOwB,MAAM,CAACF,MAAM,KAAK,GAC3B,OAAO;QAACG,UAAU;QAAGC,sBAAsB;QAAGC,yBAAyB,EAAE;IAAA;IAE3E,OAAO,MAAMxC,KAAKyC,WAAW,CAAC,OAAOC;QACnC,MAAMC,aAAa,IAAIpB;QACvB,MAAMqB,mBAAmBC,gBAAgBhC,OAAOwB,MAAM,EAAEM;QACxD,MAAMG,mBAAmBF,iBAAiBG,IAAI,CAAC,CAACC,QAAUC,gBAAgBD,MAAM5B,KAAK;QAErF,IAAI0B,kBAAkB;YACpB,MAAMJ,GAAGQ,OAAO,CACdpD,GAAG,CAAC,sCAAsC,EAAEe,OAAOsC,WAAW,IAAItC,OAAOI,aAAa,CAAC,EAAE,CAAC;QAE9F;QAEA,MAAMoB,SAAS,MAAMe,0CAA0CV,IAAI7B,QAAQ+B;QAC3E,MAAMJ,0BAA0B,MAAMa,iDACpCX,IACA7B,QACAwB;QAGF,MAAMrB,SAASqB,OAAOiB,GAAG,CAAC,CAACN,QAAW,CAAA;gBACpCG,aAAatC,OAAOsC,WAAW;gBAC/BlC,eAAeJ,OAAOI,aAAa;gBACnCe,kBAAkBgB,MAAMhB,gBAAgB;gBACxCuB,eAAeP,MAAMO,aAAa;gBAClCpC,aAAa6B,MAAM7B,WAAW;gBAC9BE,QAAQ;uBAAIrC,mBAAmBgE,MAAM3B,MAAM;iBAAE;gBAC7CD,OAAO4B,MAAM5B,KAAK;gBAClBoC,QAAQR,MAAMQ,MAAM;gBACpBC,iBAAiBT,MAAMS,eAAe;gBACtCvC,cAAc8B,MAAM9B,YAAY;gBAChCI,YAAY0B,MAAM1B,UAAU,GAAGqB,aAAaA,aAAaK,MAAM1B,UAAU;gBACzEoC,WAAWV,MAAMU,SAAS;gBAC1BC,YAAYX,MAAMW,UAAU;gBAC5BC,WAAWZ,MAAMY,SAAS;gBAC1BC,UAAUb,MAAMa,QAAQ;gBACxBC,cAAcd,MAAMc,YAAY;YAClC,CAAA;QAEA,MAAMpB,GACH3B,MAAM,CAACV,iBACPW,MAAM,CAACA,QACP+C,kBAAkB,CAAC;YAClBC,QAAQ;gBAAC3D,gBAAgBY,aAAa;gBAAEZ,gBAAgB2B,gBAAgB;aAAC;YACzEiC,aAAazE,UAAUa,gBAAgB2B,gBAAgB;YACvDD,KAAK;gBACHwB,eAAezD,GAAG,CAAC,UAAU,EAAEoE,0CAA0C,wCAAwC,EAAE7D,gBAAgBkD,aAAa,CAAC,OAAO,EAAElD,gBAAgBkD,aAAa,CAAC,IAAI,CAAC;gBAC7LpC,aAAarB,GAAG,CAAC,UAAU,EAAEoE,0CAA0C,sCAAsC,EAAE7D,gBAAgBc,WAAW,CAAC,OAAO,EAAEd,gBAAgBc,WAAW,CAAC,IAAI,CAAC;gBACrLE,QAAQvB,GAAG,CAAC,UAAU,EAAEoE,0CAA0C,2BAA2B,EAAE7D,gBAAgBgB,MAAM,CAAC,IAAI,CAAC;gBAC3HD,OAAOtB,GAAG,CAAC,UAAU,EAAEoE,0CAA0C,0BAA0B,EAAE7D,gBAAgBe,KAAK,CAAC,IAAI,CAAC;gBACxHoC,QAAQ1D,GAAG,CAAC,UAAU,EAAEoE,0CAA0C,2BAA2B,EAAE7D,gBAAgBmD,MAAM,CAAC,IAAI,CAAC;gBAC3HC,iBAAiB3D,GAAG,CAAC,UAAU,EAAEoE,0CAA0C,eAAe,EAAE7D,gBAAgBoD,eAAe,CAAC,mCAAmC,EAAEpD,gBAAgBoD,eAAe,CAAC,IAAI,CAAC;gBACtMvC,cAAcpB,GAAG,CAAC,UAAU,EAAEoE,0CAA0C,uCAAuC,EAAE7D,gBAAgBa,YAAY,CAAC,OAAO,EAAEb,gBAAgBa,YAAY,CAAC,IAAI,CAAC;gBACzLI,YAAYxB,GAAG,CAAC,UAAU,EAAEoE,0CAA0C,gCAAgC,EAAE7D,gBAAgBiB,UAAU,CAAC,IAAI,CAAC;gBACxIoC,WAAWS,gBAAgB9D,gBAAgBqD,SAAS,EAAE5D,GAAG,CAAC,mBAAmB,CAAC;gBAC9E6D,YAAYQ,gBAAgB9D,gBAAgBsD,UAAU,EAAE7D,GAAG,CAAC,oBAAoB,CAAC;gBACjF8D,WAAWO,gBAAgB9D,gBAAgBuD,SAAS,EAAE9D,GAAG,CAAC,mBAAmB,CAAC;gBAC9E+D,UAAUM,gBAAgB9D,gBAAgBwD,QAAQ,EAAE/D,GAAG,CAAC,kBAAkB,CAAC;gBAC3EgE,cAAcK,gBAAgB9D,gBAAgByD,YAAY,EAAEhE,GAAG,CAAC,sBAAsB,CAAC;gBACvFmC,WAAWnC,GAAG,CAAC,KAAK,CAAC;YACvB;YACAsE,UAAUtE,GAAG,CAAC;UACZ,EAAEoE,0CAA0C;aACzC,EAAEG,yCAAyC;QAChD,CAAC;QACH;QAEF,MAAM9B,uBAAuBO,mBACzB,MAAMwB,0CAA0C5B,IAAI7B,QAAQwB,UAC5D;QAEJ,OAAO;YAACC,UAAUD,OAAOF,MAAM;YAAEI;YAAsBC;QAAuB;IAChF;AACF;AAEA,OAAO,eAAe+B,2CACpB7B,EAAM,EACN7B,MAAoD;IAEpD,MAAM2D,OAAO,MAAM9B,GAChB+B,MAAM,CAAC;QACNtD,aAAad,gBAAgBc,WAAW;QACxCC,OAAOf,gBAAgBe,KAAK;QAC5BsD,OAAO5E,GAAW,CAAC,aAAa,CAAC;IACnC,GACC6E,IAAI,CAACtE,iBACL6B,KAAK,CACJjD,IACEG,GAAGiB,gBAAgB8C,WAAW,EAAEtC,OAAOsC,WAAW,GAClD/D,GAAGiB,gBAAgBY,aAAa,EAAEJ,OAAOI,aAAa,GACtD1B,QAAQc,gBAAgBe,KAAK,EAAET,wBAC/BnB,UAAUa,gBAAgBc,WAAW,IAGxCyD,OAAO,CAACvE,gBAAgBc,WAAW,EAAEd,gBAAgBe,KAAK;IAE7D,OAAOoD,KAAKK,OAAO,CAAC,CAAC/D,MACnBA,IAAIK,WAAW,IAAI2D,uBAAuBhE,IAAIM,KAAK,IAC/C;YAAC;gBAACD,aAAaL,IAAIK,WAAW;gBAAEC,OAAON,IAAIM,KAAK;gBAAEsD,OAAO5D,IAAI4D,KAAK;YAAA;SAAE,GACpE,EAAE;AAEV;AAEA,OAAO,eAAeK,0BAA0BlE,MAI/C;IACC,MAAM2D,OAAO,MAAMxE,KAChByE,MAAM,GACNE,IAAI,CAACtE,iBACL6B,KAAK,CACJjD,IACEG,GAAGiB,gBAAgB8C,WAAW,EAAEtC,OAAOsC,WAAW,GAClD5D,QAAQc,gBAAgBe,KAAK,EAAEV,eAC/BZ,GAAG,CAAC,EAAEO,gBAAgB4B,SAAS,CAAC,YAAY,EAAEpB,OAAOmE,aAAa,CAAC,yBAAyB,CAAC,GAGhGC,OAAO,CAAC9F,KAAKkB,gBAAgB4B,SAAS,GAAG9C,KAAKkB,gBAAgBqB,EAAE,GAChEwD,KAAK,CAACrE,OAAOqE,KAAK,IAAI;IAEzB,OAAOV,KAAKlB,GAAG,CAAChD;AAClB;AAEA,OAAO,eAAe6E,gCAAgCtE,MAIrD;IACC,OAAO,MAAMb,KAAKyC,WAAW,CAAC,OAAOC;QACnC,MAAM8B,OAAO,MAAMY,qCAAqC1C,IAAI7B;QAC5D,OAAO2D,KAAKlB,GAAG,CAAC,CAACxC,MAAQA,IAAIkB,gBAAgB;IAC/C;AACF;AAEA,OAAO,eAAeoD,qCACpB1C,EAAM,EACN7B,MAIC;IAED,MAAM2D,OAAO,MAAMa,iCAAiC3C,IAAI7B,QACrDoE,OAAO,CAAC/F,IAAImB,gBAAgB2B,gBAAgB,GAC5CkD,KAAK,CAACrE,OAAOqE,KAAK,GAAG;IAExB,MAAMI,YAAYd,KAAKrC,MAAM,GAAGtB,OAAOqE,KAAK;IAC5C,MAAMK,eAAeD,YAAYd,KAAKgB,KAAK,CAAC,GAAG3E,OAAOqE,KAAK,IAAIV;IAC/D,IAAIc,WAAW;QACbvG,SAAS0G,IAAI,CACX;YACEtC,aAAatC,OAAOsC,WAAW;YAC/BlC,eAAeJ,OAAOI,aAAa;YACnCiE,OAAOrE,OAAOqE,KAAK;YACnBQ,eAAeH,aAAapD,MAAM;QACpC,GACA;IAEJ;IAEA,OAAOoD,aAAaV,OAAO,CAAC,CAAC/D,MAC3BA,IAAIkB,gBAAgB,GAAG;YAAC;gBAAC,GAAGlB,GAAG;gBAAEkB,kBAAkBlB,IAAIkB,gBAAgB;YAAA;SAAE,GAAG,EAAE;AAElF;AAEA,SAASqD,iCACP3C,EAAM,EACN7B,MAAkF;IAElF,MAAM8E,4BAA4B5F,MAAMS,sBAAsB;IAE9D,OAAOkC,GACJ+B,MAAM,CAAC;QACNzC,kBAAkB3B,gBAAgB2B,gBAAgB;QAClDwB,QAAQ1D,GAAwC,CAAC,eAAe,CAAC;IACnE,GACC6E,IAAI,CAACnE,sBACLoF,SAAS,CACRvF,iBACApB,IACEG,GAAGiB,gBAAgB8C,WAAW,EAAE3C,qBAAqB2C,WAAW,GAChE/D,GAAGiB,gBAAgBY,aAAa,EAAET,qBAAqBS,aAAa,GACpE7B,GAAGiB,gBAAgB2B,gBAAgB,EAAExB,qBAAqBwB,gBAAgB,IAG7EE,KAAK,CACJjD,IACEG,GAAGoB,qBAAqB2C,WAAW,EAAEtC,OAAOsC,WAAW,GACvD/D,GAAGoB,qBAAqBS,aAAa,EAAEJ,OAAOI,aAAa,GAC3DzB,UAAUa,gBAAgB2B,gBAAgB,GAC1CxC,UAAUgB,qBAAqBqF,uBAAuB,GACtDtG,QAAQc,gBAAgBe,KAAK,EAAEV,eAC/BG,OAAOiF,iBAAiB,IAAIjF,OAAOiF,iBAAiB,CAAC3D,MAAM,GAAG,IAC1D5C,QAAQc,gBAAgB2B,gBAAgB,EAAEnB,OAAOiF,iBAAiB,IAClEC,WACJpG,UACE+C,GACG+B,MAAM,CAAC;QAAC/C,IAAIiE,0BAA0BjE,EAAE;IAAA,GACxCiD,IAAI,CAACgB,2BACLzD,KAAK,CACJjD,IACEG,GAAGuG,0BAA0BxC,WAAW,EAAE3C,qBAAqB2C,WAAW,GAC1E/D,GAAGuG,0BAA0B1E,aAAa,EAAET,qBAAqBS,aAAa,GAC9E7B,GACEuG,0BAA0B3D,gBAAgB,EAC1CxB,qBAAqBwB,gBAAgB,GAEvCnC,GACEP,GAAGqG,0BAA0BjC,SAAS,EAAElD,qBAAqBkD,SAAS,GACtEzE,IACEG,GAAGuG,0BAA0BjC,SAAS,EAAElD,qBAAqBkD,SAAS,GACtEpE,GACEqG,0BAA0BK,cAAc,EACxCxF,qBAAqBwF,cAAc,SASpDpB,OAAO,CAACvE,gBAAgB2B,gBAAgB;AAC7C;AAEA,eAAeqB,iDACbX,EAAM,EACN7B,MAAmC,EACnCwB,MAAmC;IAEnC,IAAI,CAACxB,OAAOsC,WAAW,EAAE,OAAO,EAAE;IAClC,MAAM8C,8BAA8B;WAC/B,IAAIC,IACL7D,OAAO8D,MAAM,CAAC,CAACnD,QAAUA,MAAM5B,KAAK,KAAK,cAAckC,GAAG,CAAC,CAACN,QAAUA,MAAMhB,gBAAgB;KAE/F;IACD,IAAIiE,4BAA4B9D,MAAM,KAAK,GAAG,OAAO,EAAE;IAEvD,MAAMqC,OAAO,MAAMa,iCAAiC3C,IAAI;QACtDS,aAAatC,OAAOsC,WAAW;QAC/BlC,eAAeJ,OAAOI,aAAa;QACnC6E,mBAAmBG;IACrB,GAAGhB,OAAO,CAAC/F,IAAImB,gBAAgB2B,gBAAgB;IAC/C,OAAOwC,KAAKK,OAAO,CAAC,CAAC/D,MACnBA,IAAIkB,gBAAgB,GAAG;YAAC;gBAAC,GAAGlB,GAAG;gBAAEkB,kBAAkBlB,IAAIkB,gBAAgB;YAAA;SAAE,GAAG,EAAE;AAElF;AAEA,OAAO,eAAeoE,yBACpBvF,MAAsC;IAEtC,MAAMwF,4BAA4B;WAAI,IAAIH,IAAIrF,OAAOwF,yBAAyB;KAAE;IAEhF,OAAO,MAAMrG,KAAKyC,WAAW,CAAC,OAAOC;QACnC,MAAMA,GAAGQ,OAAO,CACdpD,GAAG,CAAC,sCAAsC,EAAEe,OAAOsC,WAAW,IAAItC,OAAOI,aAAa,CAAC,EAAE,CAAC;QAG5F,IAAIqF,YAAsB,EAAE;QAC5B,IAAI/D,uBAAuB;QAC3B,IAAI8D,0BAA0BlE,MAAM,GAAG,GAAG;YACxC,MAAMoE,kBAAkB,MAAM7D,GAC3B+B,MAAM,CAAC;gBACN/C,IAAIrB,gBAAgBqB,EAAE;gBACtBM,kBAAkB3B,gBAAgB2B,gBAAgB;YACpD,GACC2C,IAAI,CAACtE,iBACL6B,KAAK,CACJjD,IACE4B,OAAOsC,WAAW,GACd/D,GAAGiB,gBAAgB8C,WAAW,EAAEtC,OAAOsC,WAAW,IAClD1D,OAAOY,gBAAgB8C,WAAW,GACtC/D,GAAGiB,gBAAgBY,aAAa,EAAEJ,OAAOI,aAAa,GACtD1B,QAAQc,gBAAgBe,KAAK,EAAEV,eAC/BhB,GACEW,gBAAgBiB,UAAU,EAC1BxB,GAAG,CAAC,SAAS,EAAEe,OAAO2F,qBAAqB,CAAC,yBAAyB,CAAC,GAExE5G,WAAWS,gBAAgB2B,gBAAgB,EAAEqE;YAInD,IAAIE,gBAAgBpE,MAAM,GAAG,GAAG;gBAC9B,MAAMN,UAAU,MAAMa,GACnBZ,MAAM,CAACzB,iBACP0B,GAAG,CAAC;oBACHX,OAAO;oBACP0C,cAAchE,GAAG,CAAC,SAAS,EAAEO,gBAAgByD,YAAY,CAAC,QAAQ,CAAC;oBACnE7B,WAAWnC,GAAG,CAAC,KAAK,CAAC;gBACvB,GACCoC,KAAK,CACJjD,IACEM,QACEc,gBAAgBqB,EAAE,EAClB6E,gBAAgBjD,GAAG,CAAC,CAACxC,MAAQA,IAAIY,EAAE,IAErCnC,QAAQc,gBAAgBe,KAAK,EAAEV,eAC/BhB,GACEW,gBAAgBiB,UAAU,EAC1BxB,GAAG,CAAC,SAAS,EAAEe,OAAO2F,qBAAqB,CAAC,yBAAyB,CAAC,IAI3EhF,SAAS,CAAC;oBAACQ,kBAAkB3B,gBAAgB2B,gBAAgB;gBAAA;gBAEhEsE,YAAYzE,QAAQgD,OAAO,CAAC,CAAC/D,MAASA,IAAIkB,gBAAgB,GAAG;wBAAClB,IAAIkB,gBAAgB;qBAAC,GAAG,EAAE;gBACxF,IAAInB,OAAOsC,WAAW,EAAE;oBACtBZ,uBAAuB,MAAMkE,+CAA+C/D,IAAI;wBAC9ES,aAAatC,OAAOsC,WAAW;wBAC/BlC,eAAeJ,OAAOI,aAAa;wBACnC6E,mBAAmBQ;oBACrB;gBACF;YACF;QACF;QAEA,MAAMI,eACJL,0BAA0BlE,MAAM,KAAK,IACjC,EAAE,GACF,AACE,CAAA,MAAMO,GACH+B,MAAM,GACNE,IAAI,CAACtE,iBACL6B,KAAK,CACJjD,IACE4B,OAAOsC,WAAW,GACd/D,GAAGiB,gBAAgB8C,WAAW,EAAEtC,OAAOsC,WAAW,IAClD1D,OAAOY,gBAAgB8C,WAAW,GACtC/D,GAAGiB,gBAAgBY,aAAa,EAAEJ,OAAOI,aAAa,GACtD1B,QAAQc,gBAAgB2B,gBAAgB,EAAEqE,4BAE9C,EACF/C,GAAG,CAAChD;QAEZ,MAAMqG,qBAAqB9F,OAAOsC,WAAW,GACzC,MAAMlD,2CAA2CyC,IAAI;YACnDS,aAAatC,OAAOsC,WAAW;YAC/BlC,eAAeJ,OAAOI,aAAa;YACnC6E,mBAAmBO;QACrB,KACA,EAAE;QAEN,OAAO;YACLK;YACAE,sCAAsC,IAAIC,IACxCF,mBAAmBrD,GAAG,CAAC,CAACwD,eAAiB;oBAACA,aAAa9E,gBAAgB;oBAAE8E;iBAAa;YAExFR;YACA/D;QACF;IACF;AACF;AAEA,OAAO,eAAewE,yBAAyBlG,MAG9C;IACC,MAAMmG,SAASC,0BAA0BpG,OAAOqG,gBAAgB;IAEhE,OAAO,MAAMlH,KAAKyC,WAAW,CAAC,OAAOC;QACnC,MAAMyE,gBAAgB,MAAMzE,GACzB+B,MAAM,CAAC;YACN/C,IAAIrB,gBAAgBqB,EAAE;YACtByB,aAAa9C,gBAAgB8C,WAAW;YACxClC,eAAeZ,gBAAgBY,aAAa;YAC5Ce,kBAAkB3B,gBAAgB2B,gBAAgB;QACpD,GACC2C,IAAI,CAACtE,iBACL6B,KAAK,CAACkF,yBAAyB1E,IAAIsE,SACnC/B,OAAO,CAAC/F,IAAImB,gBAAgB4B,SAAS,GAAG/C,IAAImB,gBAAgBqB,EAAE,GAC9DwD,KAAK,CAACrE,OAAOqE,KAAK;QAErB,IAAIiC,cAAchF,MAAM,KAAK,GAAG,OAAO;YAACkF,QAAQ;YAAG9E,sBAAsB;QAAC;QAE1E,MAAM+E,eAAe;eAChB,IAAIpB,IAAIiB,cAActC,OAAO,CAAC,CAAC/D,MAASA,IAAIqC,WAAW,GAAG;oBAACrC,IAAIqC,WAAW;iBAAC,GAAG,EAAE;SACpF,CAACoE,IAAI;QACN,KAAK,MAAMpE,eAAemE,aAAc;YACtC,MAAM5E,GAAGQ,OAAO,CAACpD,GAAG,CAAC,sCAAsC,EAAEqD,YAAY,EAAE,CAAC;QAC9E;QAEA,MAAMqE,cAAc,MAAM9E,GACvBZ,MAAM,CAACzB,iBACP0B,GAAG,CAAC;YACHX,OAAO;YACPoC,QAAQ;YACRK,UAAU/D,GAAG,CAAC,SAAS,EAAEO,gBAAgBwD,QAAQ,CAAC,QAAQ,CAAC;YAC3D5B,WAAWnC,GAAG,CAAC,KAAK,CAAC;QACvB,GACCoC,KAAK,CACJjD,IACEM,QACEc,gBAAgBqB,EAAE,EAClByF,cAAc7D,GAAG,CAAC,CAACxC,MAAQA,IAAIY,EAAE,IAEnC0F,yBAAyB1E,IAAIsE,UAGhCxF,SAAS,CAAC;YACT2B,aAAa9C,gBAAgB8C,WAAW;YACxClC,eAAeZ,gBAAgBY,aAAa;YAC5Ce,kBAAkB3B,gBAAgB2B,gBAAgB;QACpD;QAEF,IAAIO,uBAAuB;QAC3B,KAAK,MAAMkF,SAASC,uBAAuBF,aAAc;YACvDjF,wBAAwB,MAAMkE,+CAA+C/D,IAAI;gBAC/ES,aAAasE,MAAMtE,WAAW;gBAC9BlC,eAAewG,MAAMxG,aAAa;gBAClC6E,mBAAmB2B,MAAM3B,iBAAiB;gBAC1C6B,wBAAwB;YAC1B;QACF;QAEA,OAAO;YAACN,QAAQG,YAAYrF,MAAM;YAAEI;QAAoB;IAC1D;AACF;AAEA,eAAe+B,0CACb5B,EAAM,EACN7B,MAAmC,EACnCwB,MAAmC;IAEnC,MAAMuF,iBAAiBvF,OAAO8D,MAAM,CAAC,CAACnD,QAAUC,gBAAgBD,MAAM5B,KAAK;IAC3E,IAAIwG,eAAezF,MAAM,KAAK,KAAK,CAACtB,OAAOsC,WAAW,EAAE,OAAO;IAE/D,OAAO,MAAMsD,+CAA+C/D,IAAI;QAC9DS,aAAatC,OAAOsC,WAAW;QAC/BlC,eAAeJ,OAAOI,aAAa;QACnC6E,mBAAmB8B,eAAetE,GAAG,CAAC,CAACN,QAAUA,MAAMhB,gBAAgB;IACzE;AACF;AAEA,eAAeyE,+CACb/D,EAAM,EACN7B,MAKC;IAED,IAAIA,OAAOiF,iBAAiB,CAAC3D,MAAM,KAAK,GAAG,OAAO;IAElD,MAAMqC,OAAO,MAAM9B,GAChB+B,MAAM,CAAC;QACN/C,IAAIrB,gBAAgBqB,EAAE;QACtB6B,eAAelD,gBAAgBkD,aAAa;IAC9C,GACCoB,IAAI,CAACtE,iBACL6B,KAAK,CACJjD,IACEG,GAAGiB,gBAAgB8C,WAAW,EAAEtC,OAAOsC,WAAW,GAClD/D,GAAGiB,gBAAgBY,aAAa,EAAEJ,OAAOI,aAAa,GACtD1B,QAAQc,gBAAgB2B,gBAAgB,EAAEnB,OAAOiF,iBAAiB,GAClEvG,QAAQc,gBAAgBe,KAAK,EAAEX,iBAC/BjB,UAAUa,gBAAgBkD,aAAa,GACvC1C,OAAO8G,sBAAsB,KAAK,QAC9B5B,YACAtG,OAAOY,gBAAgBoD,eAAe,GAC1ChE,OAAOY,gBAAgBwH,qBAAqB;IAIlD,IAAIrD,KAAKrC,MAAM,KAAK,GAAG,OAAO;IAE9B,MAAMN,UAAU,MAAMa,GACnBZ,MAAM,CAACzB,iBACP0B,GAAG,CAAC;QAAC8F,uBAAuB/H,GAAG,CAAC,KAAK,CAAC;QAAEmC,WAAWnC,GAAG,CAAC,KAAK,CAAC;IAAA,GAC7DoC,KAAK,CACJjD,IACEM,QACEc,gBAAgBqB,EAAE,EAClB8C,KAAKlB,GAAG,CAAC,CAACxC,MAAQA,IAAIY,EAAE,IAE1Bb,OAAO8G,sBAAsB,KAAK,QAC9B5B,YACAtG,OAAOY,gBAAgBoD,eAAe,GAC1ChE,OAAOY,gBAAgBwH,qBAAqB,IAG/CrG,SAAS,CAAC;QAAC+B,eAAelD,gBAAgBkD,aAAa;IAAA;IAE1D,MAAMuE,0BAA0B,IAAIjB;IACpC,KAAK,MAAM/F,OAAOe,QAAS;QACzB,IAAI,CAACf,IAAIyC,aAAa,EAAE;QACxBuE,wBAAwB/F,GAAG,CACzBjB,IAAIyC,aAAa,EACjB,AAACuE,CAAAA,wBAAwBC,GAAG,CAACjH,IAAIyC,aAAa,KAAK,CAAA,IAAK;IAE5D;IAEA,IAAIuE,wBAAwBE,IAAI,KAAK,GAAG,OAAO;IAE/C,OAAO,MAAM9H,wBAAwBwC,IAAI;QACvCS,aAAatC,OAAOsC,WAAW;QAC/BlC,eAAeJ,OAAOI,aAAa;QACnCgH,UAAU;eAAIH;SAAwB,CAACxE,GAAG,CAAC,CAAC,CAACC,eAAemB,MAAM,GAAM,CAAA;gBACtEnB;gBACAmB;YACF,CAAA;IACF;AACF;AAEA,SAASuC,0BAA0BC,gBAAwB;IACzD,OAAOpH,GAAG,CAAC,SAAS,EAAEoH,iBAAiB,yBAAyB,CAAC;AACnE;AAEA,SAASE,yBAAyB1E,EAAM,EAAEsE,MAAW;IACnD,OAAO/H,IACLM,QAAQc,gBAAgBe,KAAK,EAAEV,eAC/BhB,GAAGW,gBAAgBiB,UAAU,EAAE0F,SAC/BtH,GAAGW,gBAAgB4B,SAAS,EAAE+E,SAC9B3H,OACEqD,GACG+B,MAAM,CAAC;QAAC/C,IAAItB,kBAAkBsB,EAAE;IAAA,GAChCiD,IAAI,CAACvE,mBACL8B,KAAK,CACJjD,IACEG,GAAGgB,kBAAkBsB,EAAE,EAAErB,gBAAgBY,aAAa,GACtDpB,GAAGJ,OAAOW,kBAAkB8H,UAAU,GAAGxI,GAAGU,kBAAkB8H,UAAU,EAAElB,aAIlFrH,UACE+C,GACG+B,MAAM,CAAC;QAAC/C,IAAIlB,qBAAqBkB,EAAE;IAAA,GACnCiD,IAAI,CAACnE,sBACL0B,KAAK,CACJjD,IACEG,GAAGoB,qBAAqB2C,WAAW,EAAE9C,gBAAgB8C,WAAW,GAChE/D,GAAGoB,qBAAqBS,aAAa,EAAEZ,gBAAgBY,aAAa,GACpE7B,GAAGoB,qBAAqBwB,gBAAgB,EAAE3B,gBAAgB2B,gBAAgB,MAIlFrC,UACE+C,GACG+B,MAAM,CAAC;QAAC/C,IAAInB,eAAemB,EAAE;IAAA,GAC7BiD,IAAI,CAACpE,gBACL2B,KAAK,CACJjD,IACEG,GAAGmB,eAAe4C,WAAW,EAAE9C,gBAAgB8C,WAAW,GAC1D/D,GAAGmB,eAAeU,aAAa,EAAEZ,gBAAgBY,aAAa,GAC9D7B,GAAGmB,eAAeyB,gBAAgB,EAAE3B,gBAAgB2B,gBAAgB,GACpElC,GAAG,CAAC,EAAES,eAAe0B,SAAS,CAAC,IAAI,EAAE+E,OAAO,CAAC;AAKzD;AAEA,SAASU,uBACPlD,IAIE;IAEF,MAAM2D,SAAS,IAAItB;IAInB,KAAK,MAAM/F,OAAO0D,KAAM;QACtB,IAAI,CAAC1D,IAAIqC,WAAW,IAAI,CAACrC,IAAIkB,gBAAgB,EAAE;QAC/C,MAAMoG,MAAM,GAAGtH,IAAIqC,WAAW,CAAC,CAAC,EAAErC,IAAIG,aAAa,EAAE;QACrD,MAAMwG,QAAQU,OAAOJ,GAAG,CAACK,QAAQ;YAC/BjF,aAAarC,IAAIqC,WAAW;YAC5BlC,eAAeH,IAAIG,aAAa;YAChC6E,mBAAmB,EAAE;QACvB;QACA2B,MAAM3B,iBAAiB,CAACuC,IAAI,CAACvH,IAAIkB,gBAAgB;QACjDmG,OAAOpG,GAAG,CAACqG,KAAKX;IAClB;IACA,OAAO;WAAIU,OAAOnH,MAAM;KAAG;AAC7B;AAEA,eAAeoC,0CACbV,EAAM,EACN7B,MAAmC,EACnCwB,MAAiC;IAEjC,IAAI,CAACxB,OAAOsC,WAAW,EAAE,OAAOd;IAChC,MAAMyD,oBAAoB;WAAI,IAAII,IAAI7D,OAAOiB,GAAG,CAAC,CAACN,QAAUA,MAAMhB,gBAAgB;KAAG;IACrF,IAAI8D,kBAAkB3D,MAAM,KAAK,GAAG,OAAOE;IAE3C,MAAMiG,YAAY,MAAM5F,GACrB+B,MAAM,CAAC;QACNzC,kBAAkB7B,4BAA4B6B,gBAAgB;QAC9DuG,mBAAmBpI,4BAA4BoI,iBAAiB;IAClE,GACC5D,IAAI,CAACxE,6BACL+B,KAAK,CACJjD,IACEG,GAAGe,4BAA4BgD,WAAW,EAAEtC,OAAOsC,WAAW,GAC9D/D,GAAGe,4BAA4Bc,aAAa,EAAEJ,OAAOI,aAAa,GAClE1B,QAAQY,4BAA4B6B,gBAAgB,EAAE8D,oBACtDtG,UAAUW,4BAA4BoI,iBAAiB,IAG1DtD,OAAO,CACN9F,KAAKgB,4BAA4BqI,UAAU,GAC3CrJ,KAAKgB,4BAA4BsI,SAAS;IAG9C,MAAMC,uCAAuC,IAAI7B;IACjD,KAAK,MAAM/F,OAAOwH,UAAW;QAC3B,IAAI,CAACxH,IAAIyH,iBAAiB,EAAE;QAC5B,IAAIG,qCAAqCC,GAAG,CAAC7H,IAAIkB,gBAAgB,GAAG;QACpE0G,qCAAqC3G,GAAG,CAACjB,IAAIkB,gBAAgB,EAAElB,IAAIyH,iBAAiB;IACtF;IAEA,IAAIG,qCAAqCV,IAAI,KAAK,GAAG,OAAO3F;IAE5D,OAAOA,OAAOiB,GAAG,CAAC,CAACN;QACjB,MAAMuF,oBAAoBG,qCAAqCX,GAAG,CAAC/E,MAAMhB,gBAAgB;QACzF,IAAI,CAACuG,qBAAqBvF,MAAMS,eAAe,KAAK8E,mBAAmB,OAAOvF;QAC9E,OAAO;YAAC,GAAGA,KAAK;YAAES,iBAAiB8E;QAAiB;IACtD;AACF;AAEA,SAAS1F,gBACPR,MAAmC,EACnCM,UAAgB;IAEhB,MAAMiG,qBAAqB,IAAI/B;IAC/B,KAAK,MAAMgC,YAAYxG,OAAQ;QAC7B,MAAMW,QAAQ8F,0BAA0BD,UAAUlG;QAClD,MAAMoG,WAAWH,mBAAmBb,GAAG,CAAC/E,MAAMhB,gBAAgB;QAC9D,IAAI+G,UAAUC,gBAAgBD,UAAU/F;QACxC,IAAI,CAAC+F,YAAYE,kCAAkCjG,OAAO+F,YAAY,GAAG;YACvEH,mBAAmB7G,GAAG,CACpBiB,MAAMhB,gBAAgB,EACtB+G,WAAWG,wBAAwBlG,OAAO+F,YAAY/F;QAE1D;IACF;IACA,OAAO;WAAI4F,mBAAmB5H,MAAM;KAAG;AACzC;AAEA,SAAS8H,0BACP9F,KAAgC,EAChCL,UAAgB;IAEhB,MAAMrB,aAAa0B,MAAM1B,UAAU,GAAGqB,aAAaA,aAAaK,MAAM1B,UAAU;IAChF,OAAO;QACL,GAAG0B,KAAK;QACR1B;QACAoC,WAAWV,MAAM5B,KAAK,KAAK,YAAYE,aAAa;QACpDqC,YAAYX,MAAM5B,KAAK,KAAK,aAAaE,aAAa;QACtDsC,WAAWZ,MAAM5B,KAAK,KAAK,YAAYE,aAAa;QACpDuC,UAAUb,MAAM5B,KAAK,KAAK,WAAWE,aAAa;QAClDwC,cAAcd,MAAM5B,KAAK,KAAK,eAAeE,aAAa;IAC5D;AACF;AAEA,SAAS0H,gBAAgBhF,MAA+B,EAAEmF,MAA+B;IACvFnF,OAAON,SAAS,GAAG0F,aAAapF,OAAON,SAAS,EAAEyF,OAAOzF,SAAS;IAClEM,OAAOL,UAAU,GAAGyF,aAAapF,OAAOL,UAAU,EAAEwF,OAAOxF,UAAU;IACrEK,OAAOJ,SAAS,GAAGwF,aAAapF,OAAOJ,SAAS,EAAEuF,OAAOvF,SAAS;IAClEI,OAAOH,QAAQ,GAAGuF,aAAapF,OAAOH,QAAQ,EAAEsF,OAAOtF,QAAQ;IAC/DG,OAAOF,YAAY,GAAGsF,aAAapF,OAAOF,YAAY,EAAEqF,OAAOrF,YAAY;AAC7E;AAEA,SAASoF,wBACPlG,KAA8B,EAC9B+F,QAAiC;IAEjC,OAAO;QACL,GAAG/F,KAAK;QACRO,eAAeP,MAAMO,aAAa,IAAIwF,SAASxF,aAAa;QAC5DpC,aAAa6B,MAAM7B,WAAW,IAAI4H,SAAS5H,WAAW;QACtDsC,iBAAiBT,MAAMS,eAAe,IAAIsF,SAAStF,eAAe;QAClEvC,cAAc8B,MAAM9B,YAAY,IAAI6H,SAAS7H,YAAY;QACzD,GAAGmI,eAAeN,SAAS;IAC7B;AACF;AAEA,SAASM,eAAerG,KAA8B;IACpD,OAAO;QACLU,WAAWV,MAAMU,SAAS;QAC1BC,YAAYX,MAAMW,UAAU;QAC5BC,WAAWZ,MAAMY,SAAS;QAC1BC,UAAUb,MAAMa,QAAQ;QACxBC,cAAcd,MAAMc,YAAY;IAClC;AACF;AAEA,SAASsF,aAAaE,CAAc,EAAEC,CAAc;IAClD,IAAI,CAACD,GAAG,OAAOC;IACf,IAAI,CAACA,GAAG,OAAOD;IACf,OAAOA,IAAIC,IAAID,IAAIC;AACrB;AAEA,SAASN,kCACPK,CAA4B,EAC5BC,CAA4B;IAE5B,MAAMC,YAAYF,EAAEhI,UAAU,CAACmI,OAAO,KAAKF,EAAEjI,UAAU,CAACmI,OAAO;IAC/D,MAAMC,YAAYC,2BAA2BL,EAAElI,KAAK,IAAIuI,2BAA2BJ,EAAEnI,KAAK;IAC1F,IAAIsI,cAAc,GAAG,OAAOA;IAC5B,OAAOF;AACT;AAEA,SAASG,2BAA2BvI,KAA0B;IAC5D,OAAQA;QACN,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;QACT,KAAK;YACH,OAAO;IACX;AACF;AAEA,SAASwI,wBAAwBxI,KAAyC;IACxE,OAAOtB,GAAW,CAAC;SACZ,EAAEsB,MAAM;;;;;;;;;EASf,CAAC;AACH;AAEA,SAAS8C;IACP,OAAOpE,GAAY,CAAC;IAClB,EAAE8J,wBAAwB9J,GAAG,CAAC,cAAc,CAAC,EAAE,GAAG,EAAE8J,wBAAwBvJ,gBAAgBe,KAAK,EAAE;;MAEjG,EAAEwI,wBAAwB9J,GAAG,CAAC,cAAc,CAAC,EAAE,GAAG,EAAE8J,wBAAwBvJ,gBAAgBe,KAAK,EAAE;kCACvE,EAAEf,gBAAgBiB,UAAU,CAAC;;EAE7D,CAAC;AACH;AAEA,SAAS+C;IACP,OAAOvE,GAAY,CAAC;IAClB,EAAE+J,qBAAqBxJ,gBAAgBqD,SAAS,EAAE5D,GAAG,CAAC,mBAAmB,CAAC,EAAE;OACzE,EAAE+J,qBAAqBxJ,gBAAgBsD,UAAU,EAAE7D,GAAG,CAAC,oBAAoB,CAAC,EAAE;OAC9E,EAAE+J,qBAAqBxJ,gBAAgBuD,SAAS,EAAE9D,GAAG,CAAC,mBAAmB,CAAC,EAAE;OAC5E,EAAE+J,qBAAqBxJ,gBAAgBwD,QAAQ,EAAE/D,GAAG,CAAC,kBAAkB,CAAC,EAAE;OAC1E,EAAE+J,qBAAqBxJ,gBAAgByD,YAAY,EAAEhE,GAAG,CAAC,sBAAsB,CAAC,EAAE;EACvF,CAAC;AACH;AAEA,SAAS+J,qBACPC,OAA4C,EAC5CC,QAAa;IAEb,OAAOjK,GAAY,CAAC,EAAEiK,SAAS,kBAAkB,EAAED,QAAQ,YAAY,EAAEC,SAAS,GAAG,EAAED,QAAQ,CAAC,CAAC;AACnG;AAEA,SAAS3F,gBAAgB2F,OAA4C,EAAEC,QAAa;IAClF,OAAOjK,GAAG,CAAC;;WAEF,EAAEiK,SAAS,cAAc,EAAED,QAAQ;WACnC,EAAEA,QAAQ,cAAc,EAAEC,SAAS;iBAC7B,EAAED,QAAQ,EAAE,EAAEC,SAAS;;EAEtC,CAAC;AACH;AAEA,OAAO,SAAS9G,gBAAgB7B,KAA0B;IACxD,OAAOX,eAAeuJ,QAAQ,CAAC5I;AACjC;AAEA,SAAS0D,uBACP1D,KAA0B;IAE1B,OAAOT,sBAAsBqJ,QAAQ,CAAC5I;AACxC"}

package/dist/db/runner-sessions.d.ts

@@ -9,6 +9,11 @@ export interface CreateRunnerSessionParams {
 }
 export declare function createRunnerSession(params: CreateRunnerSessionParams): Promise<RunnerSession>;
 export declare function getRunnerSessionById(runnerSessionId: string): Promise<RunnerSession | null>;
+export declare function createRunnerSessionConsumingActivationToken(params: {
+    activationTokenId: string;
+    labels: string[];
+    toolCapabilities?: RunnerToolCapabilitiesDto | null;
+}): Promise<RunnerSession>;
 export interface DeleteExpiredRunnerSessionsParams {
     manualRetentionDays: number;
     ephemeralRetentionDays: number;