@shipfox/api-runners 12.4.0 → 12.5.0

package/src/metrics/instance.ts

@@ -1,7 +1,116 @@
-import {instanceMetrics} from '@shipfox/node-opentelemetry';
+import {instanceMetrics, logger} from '@shipfox/node-opentelemetry';
+import type {RunnerAssignmentRejectionReason} from '#core/errors.js';
 
 const meter = instanceMetrics.getMeter('runners');
 
+export type RunnerLaunchKind = 'demand' | 'warm' | 'manual';
+export type RunnerAssignmentSurface = 'provisioner' | 'enrollment';
+
+type ProviderRunnerLifecycleLabels = {
+  provider: string;
+  launch_kind: RunnerLaunchKind;
+};
+
+type ProviderRunnerAssignmentLifecycleLabels = ProviderRunnerLifecycleLabels & {
+  surface: RunnerAssignmentSurface;
+};
+
+type JobExecutionQueueTimeLabels = {
+  provider: string;
+  launch_kind: RunnerLaunchKind | 'unknown';
+};
+
+export interface JobExecutionQueueTimeObservation {
+  durationMilliseconds: number;
+  provider: string | null;
+  launchKind: RunnerLaunchKind | 'unknown';
+}
+
+export interface ProviderRunnerLifecycleObservation {
+  durationMilliseconds: number;
+  provider: string | null;
+  launchKind: RunnerLaunchKind;
+  runnerInstanceId?: string;
+}
+
+export interface ProviderRunnerAssignmentObservation extends ProviderRunnerLifecycleObservation {
+  surface: RunnerAssignmentSurface;
+}
+
+export const UNKNOWN_PROVIDER_KIND = 'unknown';
+
+const lifecycleDurationBuckets = {
+  long: [
+    100, 500, 1_000, 5_000, 10_000, 15_000, 20_000, 30_000, 45_000, 60_000, 90_000, 120_000,
+    300_000, 600_000,
+  ],
+  short: [10, 25, 50, 100, 250, 500, 1_000, 5_000, 10_000],
+};
+
+const queueTimeBucketsMilliseconds = [
+  100, 500, 1_000, 5_000, 10_000, 15_000, 20_000, 30_000, 45_000, 60_000, 90_000, 120_000, 300_000,
+  600_000, 900_000, 1_800_000, 3_600_000, 7_200_000, 14_400_000,
+];
+
+// Keep the ordered lifecycle phases as separate histograms so long boot times do not flatten
+// short handoffs. Assignment is the only phase with a surface dimension.
+export const providerRunnerCreatedToControlSessionDuration =
+  meter.createHistogram<ProviderRunnerLifecycleLabels>(
+    'runners_provider_runner_created_to_control_session',
+    {
+      description: 'Provider runner row creation to control-session creation duration',
+      unit: 'ms',
+      advice: {explicitBucketBoundaries: lifecycleDurationBuckets.long},
+    },
+  );
+
+export const providerRunnerControlSessionToAssignmentDuration =
+  meter.createHistogram<ProviderRunnerAssignmentLifecycleLabels>(
+    'runners_provider_runner_control_session_to_assignment',
+    {
+      description: 'Provider runner control-session creation to reservation assignment duration',
+      unit: 'ms',
+      advice: {explicitBucketBoundaries: lifecycleDurationBuckets.long},
+    },
+  );
+
+export const providerRunnerAssignmentToActivationDuration =
+  meter.createHistogram<ProviderRunnerLifecycleLabels>(
+    'runners_provider_runner_assignment_to_activation',
+    {
+      description:
+        'Provider runner reservation assignment to workspace runner-session creation duration',
+      unit: 'ms',
+      advice: {explicitBucketBoundaries: lifecycleDurationBuckets.short},
+    },
+  );
+
+export const providerRunnerActivationToFirstClaimDuration =
+  meter.createHistogram<ProviderRunnerLifecycleLabels>(
+    'runners_provider_runner_activation_to_first_claim',
+    {
+      description: 'Provider runner workspace runner-session creation to first job claim duration',
+      unit: 'ms',
+      advice: {explicitBucketBoundaries: lifecycleDurationBuckets.short},
+    },
+  );
+
+export const jobExecutionQueueTimeDuration = meter.createHistogram<JobExecutionQueueTimeLabels>(
+  'runners_job_execution_queue_time',
+  {
+    description: 'Job execution pending queue duration from enqueue to runner claim',
+    unit: 'ms',
+    advice: {explicitBucketBoundaries: queueTimeBucketsMilliseconds},
+  },
+);
+
+export const providerRunnerAssignmentRejectedCount = meter.createCounter<{
+  reason: RunnerAssignmentRejectionReason;
+  surface: RunnerAssignmentSurface;
+}>('runners_provider_runner_assignment_rejected', {
+  description: 'Provider runner assignment operations rejected by bounded reason and surface',
+});
+
 export const jobExecutionEnqueuedCount = meter.createCounter<Record<string, never>>(
   'runners_job_execution_enqueued',
   {
@@ -152,6 +261,84 @@ function recordMetric(record: () => void): void {
   }
 }
 
+function resolveProviderRunnerMetricProvider(params: {
+  provider: string | null;
+  runnerInstanceId?: string;
+}): string {
+  if (params.provider) return params.provider;
+  logger().debug(
+    {runnerInstanceId: params.runnerInstanceId},
+    'Provider runner metric missing provider kind',
+  );
+  return UNKNOWN_PROVIDER_KIND;
+}
+
+export function recordProviderRunnerCreatedToControlSession(
+  params: ProviderRunnerLifecycleObservation,
+): void {
+  if (params.durationMilliseconds < 0) return;
+  recordMetric(() =>
+    providerRunnerCreatedToControlSessionDuration.record(params.durationMilliseconds, {
+      provider: resolveProviderRunnerMetricProvider(params),
+      launch_kind: params.launchKind,
+    }),
+  );
+}
+
+export function recordProviderRunnerControlSessionToAssignment(
+  params: ProviderRunnerAssignmentObservation,
+): void {
+  if (params.durationMilliseconds < 0) return;
+  recordMetric(() =>
+    providerRunnerControlSessionToAssignmentDuration.record(params.durationMilliseconds, {
+      provider: resolveProviderRunnerMetricProvider(params),
+      launch_kind: params.launchKind,
+      surface: params.surface,
+    }),
+  );
+}
+
+export function recordProviderRunnerAssignmentToActivation(
+  params: ProviderRunnerLifecycleObservation,
+): void {
+  if (params.durationMilliseconds < 0) return;
+  recordMetric(() =>
+    providerRunnerAssignmentToActivationDuration.record(params.durationMilliseconds, {
+      provider: resolveProviderRunnerMetricProvider(params),
+      launch_kind: params.launchKind,
+    }),
+  );
+}
+
+export function recordProviderRunnerActivationToFirstClaim(
+  params: ProviderRunnerLifecycleObservation,
+): void {
+  if (params.durationMilliseconds < 0) return;
+  recordMetric(() =>
+    providerRunnerActivationToFirstClaimDuration.record(params.durationMilliseconds, {
+      provider: resolveProviderRunnerMetricProvider(params),
+      launch_kind: params.launchKind,
+    }),
+  );
+}
+
+export function recordJobExecutionQueueTime(params: JobExecutionQueueTimeObservation): void {
+  if (params.durationMilliseconds < 0) return;
+  recordMetric(() =>
+    jobExecutionQueueTimeDuration.record(params.durationMilliseconds, {
+      provider: params.provider ?? UNKNOWN_PROVIDER_KIND,
+      launch_kind: params.launchKind,
+    }),
+  );
+}
+
+export function recordProviderRunnerAssignmentRejected(params: {
+  reason: RunnerAssignmentRejectionReason;
+  surface: RunnerAssignmentSurface;
+}): void {
+  recordMetric(() => providerRunnerAssignmentRejectedCount.add(1, params));
+}
+
 export function recordRunnerReservationPromotionFailure(
   reason: RunnerReservationPromotionFailureReason,
 ): void {

package/src/metrics/service.test.ts

@@ -2,11 +2,15 @@ const mocks = vi.hoisted(() => {
   const gauges = {
     enrolledRunnersWithoutRecentReport: {},
     pendingJobExecutions: {},
+    providerRunnersByPhase: {},
+    providerRunnersByPhaseOldestAge: {},
     runningJobExecutions: {},
   };
   const gaugeByName = {
     runners_enrolled_without_recent_report: gauges.enrolledRunnersWithoutRecentReport,
     runners_pending_job_executions: gauges.pendingJobExecutions,
+    runners_provider_runner_by_phase: gauges.providerRunnersByPhase,
+    runners_provider_runner_by_phase_oldest_age: gauges.providerRunnersByPhaseOldestAge,
     runners_running_job_executions: gauges.runningJobExecutions,
   };
   return {
@@ -16,6 +20,7 @@ const mocks = vi.hoisted(() => {
     gauges,
     getMeter: vi.fn(),
     getJobExecutionQueueDepth: vi.fn(),
+    listProviderRunnerByPhaseMetrics: vi.fn(),
     getServiceMetricsProvider: vi.fn(),
   };
 });
@@ -31,6 +36,7 @@ vi.mock('#db/job-executions.js', () => ({
 }));
 vi.mock('#db/runner-instances.js', () => ({
   countStaleEnrolledRunnerInstances: mocks.countStaleEnrolledRunnerInstances,
+  listProviderRunnerByPhaseMetrics: mocks.listProviderRunnerByPhaseMetrics,
 }));
 
 let registerRunnersServiceMetrics: typeof import('./service.js').registerRunnersServiceMetrics;
@@ -48,12 +54,14 @@ describe('registerRunnersServiceMetrics', () => {
     mocks.countStaleEnrolledRunnerInstances.mockReset();
     mocks.createObservableGauge.mockClear();
     mocks.getJobExecutionQueueDepth.mockReset();
+    mocks.listProviderRunnerByPhaseMetrics.mockReset();
     mocks.getMeter.mockReset();
     mocks.getServiceMetricsProvider.mockReset();
     mocks.getJobExecutionQueueDepth.mockResolvedValue({
       pendingJobExecutions: 0,
       runningJobExecutions: 0,
     });
+    mocks.listProviderRunnerByPhaseMetrics.mockResolvedValue([]);
     mocks.getMeter.mockReturnValue({
       createObservableGauge: mocks.createObservableGauge,
       addBatchObservableCallback: mocks.addBatchObservableCallback,
@@ -103,4 +111,34 @@ describe('registerRunnersServiceMetrics', () => {
     expect(observer.observe).toHaveBeenCalledWith(mocks.gauges.runningJobExecutions, 4);
     expect(observer.observe).toHaveBeenCalledTimes(2);
   });
+
+  it('observes provider runners by lifecycle phase', async () => {
+    mocks.listProviderRunnerByPhaseMetrics.mockResolvedValue([
+      {
+        phase: 'assignment',
+        provider: 'ec2',
+        launchKind: 'demand',
+        count: 3,
+        oldestAgeMilliseconds: 42,
+      },
+    ]);
+
+    registerRunnersServiceMetrics();
+    const callback = mocks.addBatchObservableCallback.mock.calls[0]?.[0];
+    if (typeof callback !== 'function') throw new Error('Expected metrics callback');
+    const observer = {observe: vi.fn()};
+
+    await callback(observer);
+
+    expect(observer.observe).toHaveBeenCalledWith(mocks.gauges.providerRunnersByPhase, 3, {
+      phase: 'assignment',
+      provider: 'ec2',
+      launch_kind: 'demand',
+    });
+    expect(observer.observe).toHaveBeenCalledWith(
+      mocks.gauges.providerRunnersByPhaseOldestAge,
+      42,
+      {phase: 'assignment', provider: 'ec2', launch_kind: 'demand'},
+    );
+  });
 });

package/src/metrics/service.ts

@@ -1,7 +1,17 @@
 import {getServiceMetricsProvider} from '@shipfox/node-opentelemetry';
 import {config} from '#config.js';
 import {getJobExecutionQueueDepth} from '#db/job-executions.js';
-import {countStaleEnrolledRunnerInstances} from '#db/runner-instances.js';
+import {
+  countStaleEnrolledRunnerInstances,
+  listProviderRunnerByPhaseMetrics,
+  type ProviderRunnerPhaseMetric,
+} from '#db/runner-instances.js';
+
+type ProviderRunnerPhaseLabels = {
+  phase: ProviderRunnerPhaseMetric['phase'];
+  provider: string;
+  launch_kind: ProviderRunnerPhaseMetric['launchKind'];
+};
 
 export function registerRunnersServiceMetrics(): void {
   const meter = getServiceMetricsProvider().getMeter('runners');
@@ -19,15 +29,31 @@ export function registerRunnersServiceMetrics(): void {
         'Running enrolled runners with a live control session, no workspace or runner session, and no recent provisioner report after the stale-runner grace window',
     },
   );
+  const providerRunnersByPhase = meter.createObservableGauge<ProviderRunnerPhaseLabels>(
+    'runners_provider_runner_by_phase',
+    {
+      description:
+        'Provider runners by lifecycle phase, including idle runners without an active assignment',
+    },
+  );
+  const providerRunnersByPhaseOldestAge = meter.createObservableGauge<ProviderRunnerPhaseLabels>(
+    'runners_provider_runner_by_phase_oldest_age',
+    {
+      description: 'Oldest provider runner age by lifecycle phase',
+      unit: 'ms',
+    },
+  );
 
   meter.addBatchObservableCallback(
     async (observer) => {
-      const [depthResult, staleEnrolledRunnerCountResult] = await Promise.allSettled([
-        getJobExecutionQueueDepth(),
-        countStaleEnrolledRunnerInstances({
-          graceSeconds: config.RUNNER_STALE_PROVISIONED_RUNNER_THRESHOLD_SECONDS,
-        }),
-      ]);
+      const [depthResult, staleEnrolledRunnerCountResult, providerRunnersByPhaseResult] =
+        await Promise.allSettled([
+          getJobExecutionQueueDepth(),
+          countStaleEnrolledRunnerInstances({
+            graceSeconds: config.RUNNER_STALE_PROVISIONED_RUNNER_THRESHOLD_SECONDS,
+          }),
+          listProviderRunnerByPhaseMetrics(),
+        ]);
 
       if (depthResult.status === 'fulfilled') {
         observer.observe(pendingJobExecutions, depthResult.value.pendingJobExecutions);
@@ -36,7 +62,28 @@ export function registerRunnersServiceMetrics(): void {
       if (staleEnrolledRunnerCountResult.status === 'fulfilled') {
         observer.observe(enrolledRunnersWithoutRecentReport, staleEnrolledRunnerCountResult.value);
       }
+      if (providerRunnersByPhaseResult.status === 'fulfilled') {
+        for (const metric of providerRunnersByPhaseResult.value) {
+          const attributes: ProviderRunnerPhaseLabels = {
+            phase: metric.phase,
+            provider: metric.provider,
+            launch_kind: metric.launchKind,
+          };
+          observer.observe(providerRunnersByPhase, metric.count, attributes);
+          observer.observe(
+            providerRunnersByPhaseOldestAge,
+            metric.oldestAgeMilliseconds,
+            attributes,
+          );
+        }
+      }
     },
-    [pendingJobExecutions, runningJobExecutions, enrolledRunnersWithoutRecentReport],
+    [
+      pendingJobExecutions,
+      runningJobExecutions,
+      enrolledRunnersWithoutRecentReport,
+      providerRunnersByPhase,
+      providerRunnersByPhaseOldestAge,
+    ],
   );
 }

package/src/presentation/routes/assign-runner-instances.ts

@@ -7,12 +7,14 @@ import {
 } from '@shipfox/api-runners-dto';
 import {ClientError, defineRoute} from '@shipfox/node-fastify';
 import {
+  getRunnerAssignmentRejectionReason,
   ReservationExpiredError,
   ReservationNotFoundError,
   RunnerInstanceAlreadyAssignedError,
   RunnerInstanceNotAssignableError,
 } from '#core/errors.js';
 import {assignRunnerInstances} from '#core/runner-assignments.js';
+import {recordProviderRunnerAssignmentRejected} from '#metrics/instance.js';
 
 export const assignRunnerInstancesRoute = defineRoute({
   method: 'POST',
@@ -23,6 +25,12 @@ export const assignRunnerInstancesRoute = defineRoute({
     response: {200: assignRunnerInstancesResponseSchema},
   },
   errorHandler: (error) => {
+    const rejectionReason = getRunnerAssignmentRejectionReason(error);
+    if (rejectionReason)
+      recordProviderRunnerAssignmentRejected({
+        reason: rejectionReason,
+        surface: 'provisioner',
+      });
     if (error instanceof ReservationNotFoundError)
       throw new ClientError('Reservation not found', 'reservation-not-found', {status: 404});
     if (error instanceof ReservationExpiredError)