@shipfox/api-runners 12.2.0 → 12.4.0

package/src/metrics/instance.ts

@@ -31,6 +31,21 @@ export const runnerBootstrapExchangeCount = meter.createCounter<{
   description: 'Runner bootstrap-token exchanges by outcome',
 });
 
+export type RunnerActivationTokenNotIssuedReason =
+  | 'runner-not-found'
+  | 'missing-workspace'
+  | 'existing-session'
+  | 'not-running';
+
+export type RunnerActivationTokenNotIssuedSurface = 'enrollment' | 'poll';
+
+export const runnerActivationTokenNotIssuedCount = meter.createCounter<{
+  reason: RunnerActivationTokenNotIssuedReason;
+  surface: RunnerActivationTokenNotIssuedSurface;
+}>('runners_runner_activation_token_not_issued', {
+  description: 'Runner activation token issuance skips by reason and surface',
+});
+
 export const runnerControlHeartbeatCount = meter.createCounter<Record<string, never>>(
   'runners_runner_control_heartbeat',
   {description: 'Pre-workspace runner-control heartbeats accepted'},
@@ -67,17 +82,23 @@ export const providerRunnerAbsentTerminatedCount = meter.createCounter<Record<st
 
 export const providerRunnerTerminateIntentIssuedCount = meter.createCounter<{
   surface: 'poll-demand' | 'reconcile';
-  reason: 'job-cancelled' | 'terminal-state';
+  reason: 'activation-timeout' | 'job-cancelled' | 'terminal-state';
 }>('runners_provider_runner_terminate_intent_issued', {
   description: 'Provisioned runner terminate intents returned to provisioners',
 });
 
 export const providerRunnerTerminateIntentHonoredCount = meter.createCounter<{
-  reason: 'job-cancelled';
+  reason: 'activation-timeout' | 'job-cancelled';
 }>('runners_provider_runner_terminate_intent_honored', {
   description: 'Provisioned runner terminate intents honored by first transition to terminated',
 });
 
+export const providerRunnerActivationOutcomeCount = meter.createCounter<{
+  outcome: 'reaped' | 'rebound';
+}>('runners_provider_runner_activation_outcome', {
+  description: 'Demand-backed runner activation outcomes by recovery action',
+});
+
 export const reservationReleasedCount = meter.createCounter<Record<string, never>>(
   'runners_reservation_released',
   {description: 'Reservation units released from terminal provisioned runner reports'},
@@ -95,6 +116,18 @@ export const runnerReservationPromotionFailureCount = meter.createCounter<{
   description: 'Runner reservation promotion failures during enrollment by reason',
 });
 
+export type RunnerReservationCapacityFailureReason =
+  | 'reservation-not-found'
+  | 'reservation-kind-mismatch'
+  | 'reservation-expired'
+  | 'capacity-exhausted';
+
+export const runnerReservationCapacityFailureCount = meter.createCounter<{
+  reason: RunnerReservationCapacityFailureReason;
+}>('runners_reservation_capacity_failures', {
+  description: 'Runner reservation admission shortfalls by reason',
+});
+
 export type RunnersRateLimitAction = 'provisioner-mint' | 'ephemeral-register';
 export type RunnersRateLimitScope = 'provisioner' | 'ephemeral-token';
 export type RunnersRateLimitOutcome = 'allowed' | 'blocked' | 'unavailable';
@@ -125,6 +158,21 @@ export function recordRunnerReservationPromotionFailure(
   recordMetric(() => runnerReservationPromotionFailureCount.add(1, {reason}));
 }
 
+export function recordRunnerActivationTokenNotIssued(params: {
+  reason: RunnerActivationTokenNotIssuedReason;
+  surface: RunnerActivationTokenNotIssuedSurface;
+}): void {
+  recordMetric(() => runnerActivationTokenNotIssuedCount.add(1, params));
+}
+
+export function recordRunnerReservationCapacityFailure(
+  reason: RunnerReservationCapacityFailureReason,
+  count: number,
+): void {
+  if (count <= 0) return;
+  recordMetric(() => runnerReservationCapacityFailureCount.add(count, {reason}));
+}
+
 export function recordRunnersRateLimitCheck(params: {
   action: RunnersRateLimitAction;
   scope: RunnersRateLimitScope;
@@ -142,3 +190,12 @@ export function recordRunnersRateLimitCheck(params: {
 export function recordRunnersRateLimitPruneFailure(): void {
   recordMetric(() => rateLimitPruneFailureCount.add(1));
 }
+
+export function recordProviderRunnerActivationOutcome(params: {
+  outcome: 'reaped' | 'rebound';
+  count?: number;
+}): void {
+  recordMetric(() =>
+    providerRunnerActivationOutcomeCount.add(params.count ?? 1, {outcome: params.outcome}),
+  );
+}

package/src/metrics/service.test.ts

@@ -0,0 +1,106 @@
+const mocks = vi.hoisted(() => {
+  const gauges = {
+    enrolledRunnersWithoutRecentReport: {},
+    pendingJobExecutions: {},
+    runningJobExecutions: {},
+  };
+  const gaugeByName = {
+    runners_enrolled_without_recent_report: gauges.enrolledRunnersWithoutRecentReport,
+    runners_pending_job_executions: gauges.pendingJobExecutions,
+    runners_running_job_executions: gauges.runningJobExecutions,
+  };
+  return {
+    addBatchObservableCallback: vi.fn(),
+    countStaleEnrolledRunnerInstances: vi.fn(),
+    createObservableGauge: vi.fn((name: string) => gaugeByName[name as keyof typeof gaugeByName]),
+    gauges,
+    getMeter: vi.fn(),
+    getJobExecutionQueueDepth: vi.fn(),
+    getServiceMetricsProvider: vi.fn(),
+  };
+});
+
+vi.mock('@shipfox/node-opentelemetry', () => ({
+  getServiceMetricsProvider: mocks.getServiceMetricsProvider,
+}));
+vi.mock('#config.js', () => ({
+  config: {RUNNER_STALE_PROVISIONED_RUNNER_THRESHOLD_SECONDS: 300},
+}));
+vi.mock('#db/job-executions.js', () => ({
+  getJobExecutionQueueDepth: mocks.getJobExecutionQueueDepth,
+}));
+vi.mock('#db/runner-instances.js', () => ({
+  countStaleEnrolledRunnerInstances: mocks.countStaleEnrolledRunnerInstances,
+}));
+
+let registerRunnersServiceMetrics: typeof import('./service.js').registerRunnersServiceMetrics;
+
+beforeAll(async () => {
+  // This package intentionally runs with isolate: false. Reset the shared module cache before
+  // importing the service so a previous file's real OpenTelemetry module cannot bypass this mock.
+  vi.resetModules();
+  ({registerRunnersServiceMetrics} = await import('./service.js'));
+});
+
+describe('registerRunnersServiceMetrics', () => {
+  beforeEach(() => {
+    mocks.addBatchObservableCallback.mockReset();
+    mocks.countStaleEnrolledRunnerInstances.mockReset();
+    mocks.createObservableGauge.mockClear();
+    mocks.getJobExecutionQueueDepth.mockReset();
+    mocks.getMeter.mockReset();
+    mocks.getServiceMetricsProvider.mockReset();
+    mocks.getJobExecutionQueueDepth.mockResolvedValue({
+      pendingJobExecutions: 0,
+      runningJobExecutions: 0,
+    });
+    mocks.getMeter.mockReturnValue({
+      createObservableGauge: mocks.createObservableGauge,
+      addBatchObservableCallback: mocks.addBatchObservableCallback,
+    });
+    mocks.getServiceMetricsProvider.mockReturnValue({getMeter: mocks.getMeter});
+  });
+
+  it('observes enrolled runners without recent reports after the grace window', async () => {
+    mocks.countStaleEnrolledRunnerInstances.mockResolvedValue(2);
+
+    registerRunnersServiceMetrics();
+    const callback = mocks.addBatchObservableCallback.mock.calls[0]?.[0];
+    if (typeof callback !== 'function') throw new Error('Expected metrics callback');
+    const observer = {observe: vi.fn()};
+
+    await callback(observer);
+
+    expect(mocks.createObservableGauge).toHaveBeenCalledWith(
+      'runners_enrolled_without_recent_report',
+      {
+        description:
+          'Running enrolled runners with a live control session, no workspace or runner session, and no recent provisioner report after the stale-runner grace window',
+      },
+    );
+    expect(mocks.countStaleEnrolledRunnerInstances).toHaveBeenCalledWith({graceSeconds: 300});
+    expect(observer.observe).toHaveBeenCalledWith(
+      mocks.gauges.enrolledRunnersWithoutRecentReport,
+      2,
+    );
+  });
+
+  it('keeps queue gauges observable when the enrolled-runner query fails', async () => {
+    mocks.getJobExecutionQueueDepth.mockResolvedValue({
+      pendingJobExecutions: 3,
+      runningJobExecutions: 4,
+    });
+    mocks.countStaleEnrolledRunnerInstances.mockRejectedValue(new Error('database unavailable'));
+
+    registerRunnersServiceMetrics();
+    const callback = mocks.addBatchObservableCallback.mock.calls[0]?.[0];
+    if (typeof callback !== 'function') throw new Error('Expected metrics callback');
+    const observer = {observe: vi.fn()};
+
+    await callback(observer);
+
+    expect(observer.observe).toHaveBeenCalledWith(mocks.gauges.pendingJobExecutions, 3);
+    expect(observer.observe).toHaveBeenCalledWith(mocks.gauges.runningJobExecutions, 4);
+    expect(observer.observe).toHaveBeenCalledTimes(2);
+  });
+});

package/src/metrics/service.ts

@@ -1,5 +1,7 @@
 import {getServiceMetricsProvider} from '@shipfox/node-opentelemetry';
+import {config} from '#config.js';
 import {getJobExecutionQueueDepth} from '#db/job-executions.js';
+import {countStaleEnrolledRunnerInstances} from '#db/runner-instances.js';
 
 export function registerRunnersServiceMetrics(): void {
   const meter = getServiceMetricsProvider().getMeter('runners');
@@ -10,13 +12,31 @@ export function registerRunnersServiceMetrics(): void {
   const runningJobExecutions = meter.createObservableGauge('runners_running_job_executions', {
     description: 'Job executions currently claimed by a runner and in progress',
   });
+  const enrolledRunnersWithoutRecentReport = meter.createObservableGauge(
+    'runners_enrolled_without_recent_report',
+    {
+      description:
+        'Running enrolled runners with a live control session, no workspace or runner session, and no recent provisioner report after the stale-runner grace window',
+    },
+  );
 
   meter.addBatchObservableCallback(
     async (observer) => {
-      const depth = await getJobExecutionQueueDepth();
-      observer.observe(pendingJobExecutions, depth.pendingJobExecutions);
-      observer.observe(runningJobExecutions, depth.runningJobExecutions);
+      const [depthResult, staleEnrolledRunnerCountResult] = await Promise.allSettled([
+        getJobExecutionQueueDepth(),
+        countStaleEnrolledRunnerInstances({
+          graceSeconds: config.RUNNER_STALE_PROVISIONED_RUNNER_THRESHOLD_SECONDS,
+        }),
+      ]);
+
+      if (depthResult.status === 'fulfilled') {
+        observer.observe(pendingJobExecutions, depthResult.value.pendingJobExecutions);
+        observer.observe(runningJobExecutions, depthResult.value.runningJobExecutions);
+      }
+      if (staleEnrolledRunnerCountResult.status === 'fulfilled') {
+        observer.observe(enrolledRunnersWithoutRecentReport, staleEnrolledRunnerCountResult.value);
+      }
     },
-    [pendingJobExecutions, runningJobExecutions],
+    [pendingJobExecutions, runningJobExecutions, enrolledRunnersWithoutRecentReport],
   );
 }

package/src/presentation/dto/poll-demand.ts

@@ -17,6 +17,9 @@ export function toPollDemandResponseDto(result: PollDemandResult): PollDemandRes
       count: reservation.count,
       expires_at: reservation.expiresAt.toISOString(),
     })),
+    ...(result.newlyReservedCount !== undefined
+      ? {newly_reserved_count: result.newlyReservedCount}
+      : {}),
     terminate_provider_runner_ids: result.terminateRunnerInstanceIds,
   };
 }

package/src/presentation/dto/runner-instances.ts

@@ -14,10 +14,14 @@ export function toReportRunnerInstancesResponseDto(result: ReportRunnerInstances
   };
 }
 
-export function toReconcileRunnerInstancesResponseDto(result: ReconcileRunnerInstancesResult): {
+export function toReconcileRunnerInstancesResponseDto(
+  result: ReconcileRunnerInstancesResult,
+  options: {includeIntendedReservationId: boolean},
+): {
   runners: Array<{
     provider_runner_id: string;
     state: ReconcileRunnerInstancesResult['runners'][number]['state'];
+    intended_reservation_id?: string | null;
     reservation_id: string | null;
     runner_session_id: string | null;
     bound_job: {
@@ -34,6 +38,9 @@ export function toReconcileRunnerInstancesResponseDto(result: ReconcileRunnerIns
     runners: result.runners.map((runner) => ({
       provider_runner_id: runner.providerRunnerId,
       state: runner.state,
+      ...(options.includeIntendedReservationId
+        ? {intended_reservation_id: runner.intendedReservationId}
+        : {}),
       reservation_id: runner.reservationId,
       runner_session_id: runner.runnerSessionId,
       bound_job: runner.boundJobExecution

package/src/presentation/routes/assign-runner-instances.ts

@@ -2,6 +2,8 @@ import {requireProvisionerContext} from '@shipfox/api-auth-context';
 import {
   assignRunnerInstancesBodySchema,
   assignRunnerInstancesResponseSchema,
+  RESERVATION_EXPIRED_ERROR_CODE,
+  RUNNER_INSTANCE_NOT_ASSIGNABLE_ERROR_CODE,
 } from '@shipfox/api-runners-dto';
 import {ClientError, defineRoute} from '@shipfox/node-fastify';
 import {
@@ -24,14 +26,16 @@ export const assignRunnerInstancesRoute = defineRoute({
     if (error instanceof ReservationNotFoundError)
       throw new ClientError('Reservation not found', 'reservation-not-found', {status: 404});
     if (error instanceof ReservationExpiredError)
-      throw new ClientError('Reservation has expired', 'reservation-expired', {status: 409});
+      throw new ClientError('Reservation has expired', RESERVATION_EXPIRED_ERROR_CODE, {
+        status: 409,
+      });
     if (
       error instanceof RunnerInstanceAlreadyAssignedError ||
       error instanceof RunnerInstanceNotAssignableError
     )
       throw new ClientError(
         'Runner instance cannot be assigned',
-        'runner-instance-not-assignable',
+        RUNNER_INSTANCE_NOT_ASSIGNABLE_ERROR_CODE,
         {status: 409},
       );
     throw error;