@shipfox/api-runners 12.3.0 → 12.4.1

package/src/metrics/instance.test.ts

@@ -0,0 +1,118 @@
+const metricMocks = vi.hoisted(() => {
+  const histograms = new Map<string, {record: ReturnType<typeof vi.fn>}>();
+  const counters = new Map<string, {add: ReturnType<typeof vi.fn>}>();
+  const createHistogram = vi.fn((name: string) => {
+    const histogram = {record: vi.fn()};
+    histograms.set(name, histogram);
+    return histogram;
+  });
+  const createCounter = vi.fn((name: string) => {
+    const counter = {add: vi.fn()};
+    counters.set(name, counter);
+    return counter;
+  });
+
+  return {counters, createCounter, createHistogram, histograms};
+});
+
+vi.mock('@shipfox/node-opentelemetry', () => ({
+  instanceMetrics: {
+    getMeter: () => ({
+      createCounter: metricMocks.createCounter,
+      createHistogram: metricMocks.createHistogram,
+    }),
+  },
+  logger: () => ({debug: vi.fn()}),
+}));
+
+let metrics: typeof import('./instance.js');
+
+beforeAll(async () => {
+  vi.resetModules();
+  metricMocks.counters.clear();
+  metricMocks.createCounter.mockClear();
+  metricMocks.createHistogram.mockClear();
+  metricMocks.histograms.clear();
+  metrics = await import('./instance.js');
+});
+
+describe('runner lifecycle metrics', () => {
+  it('defines provider-runner histograms with millisecond units and buckets', () => {
+    const calls = metricMocks.createHistogram.mock.calls as unknown as Array<
+      [string, {unit?: string; advice?: {explicitBucketBoundaries?: number[]}}]
+    >;
+
+    expect(
+      calls
+        .filter(([name]) => name.startsWith('runners_provider_runner_'))
+        .map(([name, options]) => ({
+          name,
+          unit: options.unit,
+          buckets: options.advice?.explicitBucketBoundaries,
+        })),
+    ).toEqual([
+      {
+        name: 'runners_provider_runner_created_to_control_session',
+        unit: 'ms',
+        buckets: [
+          100, 500, 1_000, 5_000, 10_000, 15_000, 20_000, 30_000, 45_000, 60_000, 90_000, 120_000,
+          300_000, 600_000,
+        ],
+      },
+      {
+        name: 'runners_provider_runner_control_session_to_assignment',
+        unit: 'ms',
+        buckets: [
+          100, 500, 1_000, 5_000, 10_000, 15_000, 20_000, 30_000, 45_000, 60_000, 90_000, 120_000,
+          300_000, 600_000,
+        ],
+      },
+      {
+        name: 'runners_provider_runner_assignment_to_activation',
+        unit: 'ms',
+        buckets: [10, 25, 50, 100, 250, 500, 1_000, 5_000, 10_000],
+      },
+      {
+        name: 'runners_provider_runner_activation_to_first_claim',
+        unit: 'ms',
+        buckets: [10, 25, 50, 100, 250, 500, 1_000, 5_000, 10_000],
+      },
+    ]);
+  });
+
+  it('records lifecycle durations without converting milliseconds', () => {
+    metrics.recordProviderRunnerCreatedToControlSession({
+      durationMilliseconds: 1_234,
+      provider: 'ec2',
+      launchKind: 'demand',
+    });
+
+    expect(
+      metricMocks.histograms.get('runners_provider_runner_created_to_control_session')?.record,
+    ).toHaveBeenCalledWith(1_234, {provider: 'ec2', launch_kind: 'demand'});
+  });
+});
+
+describe('job execution queue time metrics', () => {
+  it('records milliseconds with bounded labels and ignores negative durations', () => {
+    const record = metricMocks.histograms.get('runners_job_execution_queue_time')?.record;
+    expect(record).toBeDefined();
+
+    metrics.recordJobExecutionQueueTime({
+      durationMilliseconds: 1_234,
+      provider: null,
+      launchKind: 'manual',
+    });
+    metrics.recordJobExecutionQueueTime({
+      durationMilliseconds: -1,
+      provider: 'ec2',
+      launchKind: 'demand',
+    });
+
+    expect(record).toHaveBeenCalledTimes(1);
+    expect(record).toHaveBeenCalledWith(1_234, {
+      provider: 'unknown',
+      launch_kind: 'manual',
+    });
+  });
+});

package/src/metrics/instance.ts

@@ -1,7 +1,116 @@
-import {instanceMetrics} from '@shipfox/node-opentelemetry';
+import {instanceMetrics, logger} from '@shipfox/node-opentelemetry';
+import type {RunnerAssignmentRejectionReason} from '#core/errors.js';
 
 const meter = instanceMetrics.getMeter('runners');
 
+export type RunnerLaunchKind = 'demand' | 'warm' | 'manual';
+export type RunnerAssignmentSurface = 'provisioner' | 'enrollment';
+
+type ProviderRunnerLifecycleLabels = {
+  provider: string;
+  launch_kind: RunnerLaunchKind;
+};
+
+type ProviderRunnerAssignmentLifecycleLabels = ProviderRunnerLifecycleLabels & {
+  surface: RunnerAssignmentSurface;
+};
+
+type JobExecutionQueueTimeLabels = {
+  provider: string;
+  launch_kind: RunnerLaunchKind | 'unknown';
+};
+
+export interface JobExecutionQueueTimeObservation {
+  durationMilliseconds: number;
+  provider: string | null;
+  launchKind: RunnerLaunchKind | 'unknown';
+}
+
+export interface ProviderRunnerLifecycleObservation {
+  durationMilliseconds: number;
+  provider: string | null;
+  launchKind: RunnerLaunchKind;
+  runnerInstanceId?: string;
+}
+
+export interface ProviderRunnerAssignmentObservation extends ProviderRunnerLifecycleObservation {
+  surface: RunnerAssignmentSurface;
+}
+
+export const UNKNOWN_PROVIDER_KIND = 'unknown';
+
+const lifecycleDurationBuckets = {
+  long: [
+    100, 500, 1_000, 5_000, 10_000, 15_000, 20_000, 30_000, 45_000, 60_000, 90_000, 120_000,
+    300_000, 600_000,
+  ],
+  short: [10, 25, 50, 100, 250, 500, 1_000, 5_000, 10_000],
+};
+
+const queueTimeBucketsMilliseconds = [
+  100, 500, 1_000, 5_000, 10_000, 15_000, 20_000, 30_000, 45_000, 60_000, 90_000, 120_000, 300_000,
+  600_000, 900_000, 1_800_000, 3_600_000, 7_200_000, 14_400_000,
+];
+
+// Keep the ordered lifecycle phases as separate histograms so long boot times do not flatten
+// short handoffs. Assignment is the only phase with a surface dimension.
+export const providerRunnerCreatedToControlSessionDuration =
+  meter.createHistogram<ProviderRunnerLifecycleLabels>(
+    'runners_provider_runner_created_to_control_session',
+    {
+      description: 'Provider runner row creation to control-session creation duration',
+      unit: 'ms',
+      advice: {explicitBucketBoundaries: lifecycleDurationBuckets.long},
+    },
+  );
+
+export const providerRunnerControlSessionToAssignmentDuration =
+  meter.createHistogram<ProviderRunnerAssignmentLifecycleLabels>(
+    'runners_provider_runner_control_session_to_assignment',
+    {
+      description: 'Provider runner control-session creation to reservation assignment duration',
+      unit: 'ms',
+      advice: {explicitBucketBoundaries: lifecycleDurationBuckets.long},
+    },
+  );
+
+export const providerRunnerAssignmentToActivationDuration =
+  meter.createHistogram<ProviderRunnerLifecycleLabels>(
+    'runners_provider_runner_assignment_to_activation',
+    {
+      description:
+        'Provider runner reservation assignment to workspace runner-session creation duration',
+      unit: 'ms',
+      advice: {explicitBucketBoundaries: lifecycleDurationBuckets.short},
+    },
+  );
+
+export const providerRunnerActivationToFirstClaimDuration =
+  meter.createHistogram<ProviderRunnerLifecycleLabels>(
+    'runners_provider_runner_activation_to_first_claim',
+    {
+      description: 'Provider runner workspace runner-session creation to first job claim duration',
+      unit: 'ms',
+      advice: {explicitBucketBoundaries: lifecycleDurationBuckets.short},
+    },
+  );
+
+export const jobExecutionQueueTimeDuration = meter.createHistogram<JobExecutionQueueTimeLabels>(
+  'runners_job_execution_queue_time',
+  {
+    description: 'Job execution pending queue duration from enqueue to runner claim',
+    unit: 'ms',
+    advice: {explicitBucketBoundaries: queueTimeBucketsMilliseconds},
+  },
+);
+
+export const providerRunnerAssignmentRejectedCount = meter.createCounter<{
+  reason: RunnerAssignmentRejectionReason;
+  surface: RunnerAssignmentSurface;
+}>('runners_provider_runner_assignment_rejected', {
+  description: 'Provider runner assignment operations rejected by bounded reason and surface',
+});
+
 export const jobExecutionEnqueuedCount = meter.createCounter<Record<string, never>>(
   'runners_job_execution_enqueued',
   {
@@ -31,6 +140,21 @@ export const runnerBootstrapExchangeCount = meter.createCounter<{
   description: 'Runner bootstrap-token exchanges by outcome',
 });
 
+export type RunnerActivationTokenNotIssuedReason =
+  | 'runner-not-found'
+  | 'missing-workspace'
+  | 'existing-session'
+  | 'not-running';
+
+export type RunnerActivationTokenNotIssuedSurface = 'enrollment' | 'poll';
+
+export const runnerActivationTokenNotIssuedCount = meter.createCounter<{
+  reason: RunnerActivationTokenNotIssuedReason;
+  surface: RunnerActivationTokenNotIssuedSurface;
+}>('runners_runner_activation_token_not_issued', {
+  description: 'Runner activation token issuance skips by reason and surface',
+});
+
 export const runnerControlHeartbeatCount = meter.createCounter<Record<string, never>>(
   'runners_runner_control_heartbeat',
   {description: 'Pre-workspace runner-control heartbeats accepted'},
@@ -67,17 +191,23 @@ export const providerRunnerAbsentTerminatedCount = meter.createCounter<Record<st
 
 export const providerRunnerTerminateIntentIssuedCount = meter.createCounter<{
   surface: 'poll-demand' | 'reconcile';
-  reason: 'job-cancelled' | 'terminal-state';
+  reason: 'activation-timeout' | 'job-cancelled' | 'terminal-state';
 }>('runners_provider_runner_terminate_intent_issued', {
   description: 'Provisioned runner terminate intents returned to provisioners',
 });
 
 export const providerRunnerTerminateIntentHonoredCount = meter.createCounter<{
-  reason: 'job-cancelled';
+  reason: 'activation-timeout' | 'job-cancelled';
 }>('runners_provider_runner_terminate_intent_honored', {
   description: 'Provisioned runner terminate intents honored by first transition to terminated',
 });
 
+export const providerRunnerActivationOutcomeCount = meter.createCounter<{
+  outcome: 'reaped' | 'rebound';
+}>('runners_provider_runner_activation_outcome', {
+  description: 'Demand-backed runner activation outcomes by recovery action',
+});
+
 export const reservationReleasedCount = meter.createCounter<Record<string, never>>(
   'runners_reservation_released',
   {description: 'Reservation units released from terminal provisioned runner reports'},
@@ -95,6 +225,18 @@ export const runnerReservationPromotionFailureCount = meter.createCounter<{
   description: 'Runner reservation promotion failures during enrollment by reason',
 });
 
+export type RunnerReservationCapacityFailureReason =
+  | 'reservation-not-found'
+  | 'reservation-kind-mismatch'
+  | 'reservation-expired'
+  | 'capacity-exhausted';
+
+export const runnerReservationCapacityFailureCount = meter.createCounter<{
+  reason: RunnerReservationCapacityFailureReason;
+}>('runners_reservation_capacity_failures', {
+  description: 'Runner reservation admission shortfalls by reason',
+});
+
 export type RunnersRateLimitAction = 'provisioner-mint' | 'ephemeral-register';
 export type RunnersRateLimitScope = 'provisioner' | 'ephemeral-token';
 export type RunnersRateLimitOutcome = 'allowed' | 'blocked' | 'unavailable';
@@ -119,12 +261,105 @@ function recordMetric(record: () => void): void {
   }
 }
 
+function resolveProviderRunnerMetricProvider(params: {
+  provider: string | null;
+  runnerInstanceId?: string;
+}): string {
+  if (params.provider) return params.provider;
+  logger().debug(
+    {runnerInstanceId: params.runnerInstanceId},
+    'Provider runner metric missing provider kind',
+  );
+  return UNKNOWN_PROVIDER_KIND;
+}
+
+export function recordProviderRunnerCreatedToControlSession(
+  params: ProviderRunnerLifecycleObservation,
+): void {
+  if (params.durationMilliseconds < 0) return;
+  recordMetric(() =>
+    providerRunnerCreatedToControlSessionDuration.record(params.durationMilliseconds, {
+      provider: resolveProviderRunnerMetricProvider(params),
+      launch_kind: params.launchKind,
+    }),
+  );
+}
+
+export function recordProviderRunnerControlSessionToAssignment(
+  params: ProviderRunnerAssignmentObservation,
+): void {
+  if (params.durationMilliseconds < 0) return;
+  recordMetric(() =>
+    providerRunnerControlSessionToAssignmentDuration.record(params.durationMilliseconds, {
+      provider: resolveProviderRunnerMetricProvider(params),
+      launch_kind: params.launchKind,
+      surface: params.surface,
+    }),
+  );
+}
+
+export function recordProviderRunnerAssignmentToActivation(
+  params: ProviderRunnerLifecycleObservation,
+): void {
+  if (params.durationMilliseconds < 0) return;
+  recordMetric(() =>
+    providerRunnerAssignmentToActivationDuration.record(params.durationMilliseconds, {
+      provider: resolveProviderRunnerMetricProvider(params),
+      launch_kind: params.launchKind,
+    }),
+  );
+}
+
+export function recordProviderRunnerActivationToFirstClaim(
+  params: ProviderRunnerLifecycleObservation,
+): void {
+  if (params.durationMilliseconds < 0) return;
+  recordMetric(() =>
+    providerRunnerActivationToFirstClaimDuration.record(params.durationMilliseconds, {
+      provider: resolveProviderRunnerMetricProvider(params),
+      launch_kind: params.launchKind,
+    }),
+  );
+}
+
+export function recordJobExecutionQueueTime(params: JobExecutionQueueTimeObservation): void {
+  if (params.durationMilliseconds < 0) return;
+  recordMetric(() =>
+    jobExecutionQueueTimeDuration.record(params.durationMilliseconds, {
+      provider: params.provider ?? UNKNOWN_PROVIDER_KIND,
+      launch_kind: params.launchKind,
+    }),
+  );
+}
+
+export function recordProviderRunnerAssignmentRejected(params: {
+  reason: RunnerAssignmentRejectionReason;
+  surface: RunnerAssignmentSurface;
+}): void {
+  recordMetric(() => providerRunnerAssignmentRejectedCount.add(1, params));
+}
+
 export function recordRunnerReservationPromotionFailure(
   reason: RunnerReservationPromotionFailureReason,
 ): void {
   recordMetric(() => runnerReservationPromotionFailureCount.add(1, {reason}));
 }
 
+export function recordRunnerActivationTokenNotIssued(params: {
+  reason: RunnerActivationTokenNotIssuedReason;
+  surface: RunnerActivationTokenNotIssuedSurface;
+}): void {
+  recordMetric(() => runnerActivationTokenNotIssuedCount.add(1, params));
+}
+
+export function recordRunnerReservationCapacityFailure(
+  reason: RunnerReservationCapacityFailureReason,
+  count: number,
+): void {
+  if (count <= 0) return;
+  recordMetric(() => runnerReservationCapacityFailureCount.add(count, {reason}));
+}
+
 export function recordRunnersRateLimitCheck(params: {
   action: RunnersRateLimitAction;
   scope: RunnersRateLimitScope;
@@ -142,3 +377,12 @@ export function recordRunnersRateLimitCheck(params: {
 export function recordRunnersRateLimitPruneFailure(): void {
   recordMetric(() => rateLimitPruneFailureCount.add(1));
 }
+
+export function recordProviderRunnerActivationOutcome(params: {
+  outcome: 'reaped' | 'rebound';
+  count?: number;
+}): void {
+  recordMetric(() =>
+    providerRunnerActivationOutcomeCount.add(params.count ?? 1, {outcome: params.outcome}),
+  );
+}

package/src/metrics/service.test.ts

@@ -2,11 +2,15 @@ const mocks = vi.hoisted(() => {
   const gauges = {
     enrolledRunnersWithoutRecentReport: {},
     pendingJobExecutions: {},
+    providerRunnersByPhase: {},
+    providerRunnersByPhaseOldestAge: {},
     runningJobExecutions: {},
   };
   const gaugeByName = {
     runners_enrolled_without_recent_report: gauges.enrolledRunnersWithoutRecentReport,
     runners_pending_job_executions: gauges.pendingJobExecutions,
+    runners_provider_runner_by_phase: gauges.providerRunnersByPhase,
+    runners_provider_runner_by_phase_oldest_age: gauges.providerRunnersByPhaseOldestAge,
     runners_running_job_executions: gauges.runningJobExecutions,
   };
   return {
@@ -16,6 +20,7 @@ const mocks = vi.hoisted(() => {
     gauges,
     getMeter: vi.fn(),
     getJobExecutionQueueDepth: vi.fn(),
+    listProviderRunnerByPhaseMetrics: vi.fn(),
     getServiceMetricsProvider: vi.fn(),
   };
 });
@@ -31,9 +36,17 @@ vi.mock('#db/job-executions.js', () => ({
 }));
 vi.mock('#db/runner-instances.js', () => ({
   countStaleEnrolledRunnerInstances: mocks.countStaleEnrolledRunnerInstances,
+  listProviderRunnerByPhaseMetrics: mocks.listProviderRunnerByPhaseMetrics,
 }));
 
-import {registerRunnersServiceMetrics} from './service.js';
+let registerRunnersServiceMetrics: typeof import('./service.js').registerRunnersServiceMetrics;
+
+beforeAll(async () => {
+  // This package intentionally runs with isolate: false. Reset the shared module cache before
+  // importing the service so a previous file's real OpenTelemetry module cannot bypass this mock.
+  vi.resetModules();
+  ({registerRunnersServiceMetrics} = await import('./service.js'));
+});
 
 describe('registerRunnersServiceMetrics', () => {
   beforeEach(() => {
@@ -41,12 +54,14 @@ describe('registerRunnersServiceMetrics', () => {
     mocks.countStaleEnrolledRunnerInstances.mockReset();
     mocks.createObservableGauge.mockClear();
     mocks.getJobExecutionQueueDepth.mockReset();
+    mocks.listProviderRunnerByPhaseMetrics.mockReset();
     mocks.getMeter.mockReset();
     mocks.getServiceMetricsProvider.mockReset();
     mocks.getJobExecutionQueueDepth.mockResolvedValue({
       pendingJobExecutions: 0,
       runningJobExecutions: 0,
     });
+    mocks.listProviderRunnerByPhaseMetrics.mockResolvedValue([]);
     mocks.getMeter.mockReturnValue({
       createObservableGauge: mocks.createObservableGauge,
       addBatchObservableCallback: mocks.addBatchObservableCallback,
@@ -96,4 +111,34 @@ describe('registerRunnersServiceMetrics', () => {
     expect(observer.observe).toHaveBeenCalledWith(mocks.gauges.runningJobExecutions, 4);
     expect(observer.observe).toHaveBeenCalledTimes(2);
   });
+
+  it('observes provider runners by lifecycle phase', async () => {
+    mocks.listProviderRunnerByPhaseMetrics.mockResolvedValue([
+      {
+        phase: 'assignment',
+        provider: 'ec2',
+        launchKind: 'demand',
+        count: 3,
+        oldestAgeMilliseconds: 42,
+      },
+    ]);
+
+    registerRunnersServiceMetrics();
+    const callback = mocks.addBatchObservableCallback.mock.calls[0]?.[0];
+    if (typeof callback !== 'function') throw new Error('Expected metrics callback');
+    const observer = {observe: vi.fn()};
+
+    await callback(observer);
+
+    expect(observer.observe).toHaveBeenCalledWith(mocks.gauges.providerRunnersByPhase, 3, {
+      phase: 'assignment',
+      provider: 'ec2',
+      launch_kind: 'demand',
+    });
+    expect(observer.observe).toHaveBeenCalledWith(
+      mocks.gauges.providerRunnersByPhaseOldestAge,
+      42,
+      {phase: 'assignment', provider: 'ec2', launch_kind: 'demand'},
+    );
+  });
 });

package/src/metrics/service.ts

@@ -1,7 +1,17 @@
 import {getServiceMetricsProvider} from '@shipfox/node-opentelemetry';
 import {config} from '#config.js';
 import {getJobExecutionQueueDepth} from '#db/job-executions.js';
-import {countStaleEnrolledRunnerInstances} from '#db/runner-instances.js';
+import {
+  countStaleEnrolledRunnerInstances,
+  listProviderRunnerByPhaseMetrics,
+  type ProviderRunnerPhaseMetric,
+} from '#db/runner-instances.js';
+
+type ProviderRunnerPhaseLabels = {
+  phase: ProviderRunnerPhaseMetric['phase'];
+  provider: string;
+  launch_kind: ProviderRunnerPhaseMetric['launchKind'];
+};
 
 export function registerRunnersServiceMetrics(): void {
   const meter = getServiceMetricsProvider().getMeter('runners');
@@ -19,15 +29,31 @@ export function registerRunnersServiceMetrics(): void {
         'Running enrolled runners with a live control session, no workspace or runner session, and no recent provisioner report after the stale-runner grace window',
     },
   );
+  const providerRunnersByPhase = meter.createObservableGauge<ProviderRunnerPhaseLabels>(
+    'runners_provider_runner_by_phase',
+    {
+      description:
+        'Provider runners by lifecycle phase, including idle runners without an active assignment',
+    },
+  );
+  const providerRunnersByPhaseOldestAge = meter.createObservableGauge<ProviderRunnerPhaseLabels>(
+    'runners_provider_runner_by_phase_oldest_age',
+    {
+      description: 'Oldest provider runner age by lifecycle phase',
+      unit: 'ms',
+    },
+  );
 
   meter.addBatchObservableCallback(
     async (observer) => {
-      const [depthResult, staleEnrolledRunnerCountResult] = await Promise.allSettled([
-        getJobExecutionQueueDepth(),
-        countStaleEnrolledRunnerInstances({
-          graceSeconds: config.RUNNER_STALE_PROVISIONED_RUNNER_THRESHOLD_SECONDS,
-        }),
-      ]);
+      const [depthResult, staleEnrolledRunnerCountResult, providerRunnersByPhaseResult] =
+        await Promise.allSettled([
+          getJobExecutionQueueDepth(),
+          countStaleEnrolledRunnerInstances({
+            graceSeconds: config.RUNNER_STALE_PROVISIONED_RUNNER_THRESHOLD_SECONDS,
+          }),
+          listProviderRunnerByPhaseMetrics(),
+        ]);
 
       if (depthResult.status === 'fulfilled') {
         observer.observe(pendingJobExecutions, depthResult.value.pendingJobExecutions);
@@ -36,7 +62,28 @@ export function registerRunnersServiceMetrics(): void {
       if (staleEnrolledRunnerCountResult.status === 'fulfilled') {
         observer.observe(enrolledRunnersWithoutRecentReport, staleEnrolledRunnerCountResult.value);
       }
+      if (providerRunnersByPhaseResult.status === 'fulfilled') {
+        for (const metric of providerRunnersByPhaseResult.value) {
+          const attributes: ProviderRunnerPhaseLabels = {
+            phase: metric.phase,
+            provider: metric.provider,
+            launch_kind: metric.launchKind,
+          };
+          observer.observe(providerRunnersByPhase, metric.count, attributes);
+          observer.observe(
+            providerRunnersByPhaseOldestAge,
+            metric.oldestAgeMilliseconds,
+            attributes,
+          );
+        }
+      }
     },
-    [pendingJobExecutions, runningJobExecutions, enrolledRunnersWithoutRecentReport],
+    [
+      pendingJobExecutions,
+      runningJobExecutions,
+      enrolledRunnersWithoutRecentReport,
+      providerRunnersByPhase,
+      providerRunnersByPhaseOldestAge,
+    ],
   );
 }

package/src/presentation/dto/poll-demand.ts

@@ -17,6 +17,9 @@ export function toPollDemandResponseDto(result: PollDemandResult): PollDemandRes
       count: reservation.count,
       expires_at: reservation.expiresAt.toISOString(),
     })),
+    ...(result.newlyReservedCount !== undefined
+      ? {newly_reserved_count: result.newlyReservedCount}
+      : {}),
     terminate_provider_runner_ids: result.terminateRunnerInstanceIds,
   };
 }

package/src/presentation/dto/runner-instances.ts

@@ -14,10 +14,14 @@ export function toReportRunnerInstancesResponseDto(result: ReportRunnerInstances
   };
 }
 
-export function toReconcileRunnerInstancesResponseDto(result: ReconcileRunnerInstancesResult): {
+export function toReconcileRunnerInstancesResponseDto(
+  result: ReconcileRunnerInstancesResult,
+  options: {includeIntendedReservationId: boolean},
+): {
   runners: Array<{
     provider_runner_id: string;
     state: ReconcileRunnerInstancesResult['runners'][number]['state'];
+    intended_reservation_id?: string | null;
     reservation_id: string | null;
     runner_session_id: string | null;
     bound_job: {
@@ -34,6 +38,9 @@ export function toReconcileRunnerInstancesResponseDto(result: ReconcileRunnerIns
     runners: result.runners.map((runner) => ({
       provider_runner_id: runner.providerRunnerId,
       state: runner.state,
+      ...(options.includeIntendedReservationId
+        ? {intended_reservation_id: runner.intendedReservationId}
+        : {}),
       reservation_id: runner.reservationId,
       runner_session_id: runner.runnerSessionId,
       bound_job: runner.boundJobExecution

package/src/presentation/routes/assign-runner-instances.ts

@@ -2,15 +2,19 @@ import {requireProvisionerContext} from '@shipfox/api-auth-context';
 import {
   assignRunnerInstancesBodySchema,
   assignRunnerInstancesResponseSchema,
+  RESERVATION_EXPIRED_ERROR_CODE,
+  RUNNER_INSTANCE_NOT_ASSIGNABLE_ERROR_CODE,
 } from '@shipfox/api-runners-dto';
 import {ClientError, defineRoute} from '@shipfox/node-fastify';
 import {
+  getRunnerAssignmentRejectionReason,
   ReservationExpiredError,
   ReservationNotFoundError,
   RunnerInstanceAlreadyAssignedError,
   RunnerInstanceNotAssignableError,
 } from '#core/errors.js';
 import {assignRunnerInstances} from '#core/runner-assignments.js';
+import {recordProviderRunnerAssignmentRejected} from '#metrics/instance.js';
 
 export const assignRunnerInstancesRoute = defineRoute({
   method: 'POST',
@@ -21,17 +25,25 @@ export const assignRunnerInstancesRoute = defineRoute({
     response: {200: assignRunnerInstancesResponseSchema},
   },
   errorHandler: (error) => {
+    const rejectionReason = getRunnerAssignmentRejectionReason(error);
+    if (rejectionReason)
+      recordProviderRunnerAssignmentRejected({
+        reason: rejectionReason,
+        surface: 'provisioner',
+      });
     if (error instanceof ReservationNotFoundError)
       throw new ClientError('Reservation not found', 'reservation-not-found', {status: 404});
     if (error instanceof ReservationExpiredError)
-      throw new ClientError('Reservation has expired', 'reservation-expired', {status: 409});
+      throw new ClientError('Reservation has expired', RESERVATION_EXPIRED_ERROR_CODE, {
+        status: 409,
+      });
     if (
       error instanceof RunnerInstanceAlreadyAssignedError ||
       error instanceof RunnerInstanceNotAssignableError
     )
       throw new ClientError(
         'Runner instance cannot be assigned',
-        'runner-instance-not-assignable',
+        RUNNER_INSTANCE_NOT_ASSIGNABLE_ERROR_CODE,
         {status: 409},
       );
     throw error;