@shipfox/api-runners 12.2.0 → 12.4.0

package/drizzle/meta/_journal.json

@@ -36,6 +36,13 @@
       "when": 1786017247182,
       "tag": "0004_huge_skreet",
       "breakpoints": true
+    },
+    {
+      "idx": 5,
+      "version": "7",
+      "when": 1786106547187,
+      "tag": "0005_previous_lenny_balinger",
+      "breakpoints": true
     }
   ]
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@shipfox/api-runners",
   "license": "MIT",
-  "version": "12.2.0",
+  "version": "12.4.0",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/ShipfoxHQ/shipfox.git",
@@ -24,20 +24,20 @@
     "@shipfox/api-common-dto": "12.0.0",
     "@shipfox/api-auth-context": "12.2.0",
     "@shipfox/api-auth-dto": "12.0.0",
-    "@shipfox/api-runners-dto": "12.2.0",
-    "@shipfox/api-workflows-dto": "12.2.0",
+    "@shipfox/api-runners-dto": "12.4.0",
+    "@shipfox/api-workflows-dto": "12.3.0",
     "@shipfox/config": "1.2.4",
     "@shipfox/inter-module": "0.2.3",
     "@shipfox/node-drizzle": "0.3.5",
     "@shipfox/node-error-monitoring": "0.3.0",
     "@shipfox/node-auth-root-key": "0.2.3",
     "@shipfox/node-fastify": "0.4.2",
+    "@shipfox/node-module": "1.0.6",
     "@shipfox/node-opentelemetry": "0.6.4",
     "@shipfox/node-outbox": "0.2.6",
     "@shipfox/node-postgres": "0.5.0",
     "@shipfox/node-rate-limit": "0.4.0",
     "@shipfox/node-temporal": "0.4.5",
-    "@shipfox/node-module": "1.0.6",
     "@shipfox/node-tokens": "0.3.2",
     "@shipfox/runner-labels": "0.2.0"
   },

package/src/config.test.ts

@@ -86,6 +86,28 @@ describe('runner assignment polling defaults', () => {
   });
 });
 
+describe('RUNNER_DEMAND_ACTIVATION_TIMEOUT_SECONDS validation', () => {
+  afterEach(() => {
+    vi.unstubAllEnvs();
+    vi.resetModules();
+  });
+
+  it('defaults to a five-minute recovery window', async () => {
+    vi.resetModules();
+
+    const {config} = await import('#config.js');
+
+    expect(config.RUNNER_DEMAND_ACTIVATION_TIMEOUT_SECONDS).toBe(300);
+  });
+
+  it.each(['0', '-5', '1.5'])('fails startup when the value is %s', async (value) => {
+    vi.stubEnv('RUNNER_DEMAND_ACTIVATION_TIMEOUT_SECONDS', value);
+    vi.resetModules();
+
+    await expect(import('#config.js')).rejects.toThrow('RUNNER_DEMAND_ACTIVATION_TIMEOUT_SECONDS');
+  });
+});
+
 describe('reservation TTL ceiling validation', () => {
   afterEach(() => {
     vi.unstubAllEnvs();
@@ -104,12 +126,12 @@ describe('reservation TTL ceiling validation', () => {
     await expect(import('#config.js')).rejects.toThrow('RESERVATION_TTL_MAX_SECONDS');
   });
 
-  it('defaults high enough for the EC2 registration deadline', async () => {
+  it('defaults high enough for provider registration deadlines and launch headroom', async () => {
     vi.resetModules();
 
     const {config} = await import('#config.js');
 
-    expect(config.RESERVATION_TTL_MAX_SECONDS).toBe(300);
+    expect(config.RESERVATION_TTL_MAX_SECONDS).toBe(600);
   });
 
   it('accepts a whole-second ceiling inside the hard maximum', async () => {

package/src/config.ts

@@ -62,12 +62,12 @@ export const config = createConfig({
     default: 250,
   }),
   RESERVATION_TTL_SECONDS: num({
-    desc: 'Default lifetime of a count-based runner reservation, in seconds. Used when a provisioner does not request a provider-specific value. Expired reservations stop counting against queued demand.',
+    desc: 'Activation grace period for a rebound runner reservation, in seconds. A runner without a session can be rebound again only after this deadline. It is also the default lifetime for launch reservations when a provisioner does not request a provider-specific value.',
     default: 60,
   }),
   RESERVATION_TTL_MAX_SECONDS: num({
-    desc: `Server-side ceiling for a count-based runner reservation lifetime, in seconds. Set it at least as high as every provider registration deadline so provider-specific values can cover boot and enrollment. Set this between 1 and ${RESERVATION_TTL_HARD_MAX_SECONDS}.`,
-    default: 300,
+    desc: `Server-side ceiling for a launch reservation lifetime, in seconds. Set it at least as high as every provider registration deadline plus its launch headroom so provider-specific values can cover the launch gap, boot, and enrollment. Set this between 1 and ${RESERVATION_TTL_HARD_MAX_SECONDS}.`,
+    default: 600,
   }),
   RESERVATION_LONG_POLL_MAX_WAIT_SECONDS: num({
     desc: 'Maximum time the provisioner demand poll endpoint waits for reservable demand before returning, in seconds.',
@@ -85,6 +85,10 @@ export const config = createConfig({
     desc: 'Time window, in seconds, used to list active runners from recent heartbeats and provisioned runner reports.',
     default: 60,
   }),
+  RUNNER_DEMAND_ACTIVATION_TIMEOUT_SECONDS: num({
+    desc: 'Minimum time, in seconds, before a demand-backed runner without a runner session can be reclaimed when neither its current nor intended reservation is live. Set this above the provider registration deadline when startup reservations remain live during boot.',
+    default: 300,
+  }),
   RUNNER_TOOL_CAPABILITIES_STALE_AFTER_SECONDS: num({
     desc: 'Time window, in seconds, after which a runner tool capability report is treated as stale. Set this higher than the runner heartbeat interval so active runners keep their advertised tools fresh.',
     default: 60,
@@ -282,6 +286,15 @@ if (
   );
 }
 
+if (
+  !Number.isInteger(config.RUNNER_DEMAND_ACTIVATION_TIMEOUT_SECONDS) ||
+  config.RUNNER_DEMAND_ACTIVATION_TIMEOUT_SECONDS < 1
+) {
+  throw new Error(
+    `RUNNER_DEMAND_ACTIVATION_TIMEOUT_SECONDS (${config.RUNNER_DEMAND_ACTIVATION_TIMEOUT_SECONDS}) must be a whole number of seconds >= 1.`,
+  );
+}
+
 if (
   !Number.isInteger(config.RUNNER_TOOL_CAPABILITIES_STALE_AFTER_SECONDS) ||
   config.RUNNER_TOOL_CAPABILITIES_STALE_AFTER_SECONDS < 1

package/src/core/demand.test.ts

@@ -37,6 +37,18 @@ describe('shouldReturn', () => {
     expect(result).toBe(true);
   });
 
+  it('returns true when only a rebound reservation was created', () => {
+    const result = shouldReturn({...emptyResult, newlyReservedCount: 1}, 1, 1, false);
+
+    expect(result).toBe(true);
+  });
+
+  it('returns true when demand was fully adopted by idle runners', () => {
+    const result = shouldReturn({...emptyResult, newlyReservedCount: 3}, 1, 3, false);
+
+    expect(result).toBe(true);
+  });
+
   it('returns true when terminate intents exist', () => {
     const result = shouldReturn(
       {...emptyResult, terminateRunnerInstanceIds: ['provisioned-runner-1']},
@@ -48,6 +60,55 @@ describe('shouldReturn', () => {
     expect(result).toBe(true);
   });
 
+  it('continues waiting for a retry-only activation-timeout intent', () => {
+    const result = shouldReturn(
+      {...emptyResult, terminateRunnerInstanceIds: ['stale-demand-runner']},
+      1,
+      1,
+      false,
+      [
+        {
+          providerRunnerId: 'stale-demand-runner',
+          reason: 'activation-timeout',
+          activationTimeoutRetry: true,
+        },
+      ],
+    );
+
+    expect(result).toBe(false);
+  });
+
+  it('returns true for a first-delivery activation-timeout intent', () => {
+    const result = shouldReturn(
+      {...emptyResult, terminateRunnerInstanceIds: ['stale-demand-runner']},
+      1,
+      1,
+      false,
+      [{providerRunnerId: 'stale-demand-runner', reason: 'activation-timeout'}],
+    );
+
+    expect(result).toBe(true);
+  });
+
+  it('returns true for a job-cancelled intent alongside a retried activation timeout', () => {
+    const result = shouldReturn(
+      {...emptyResult, terminateRunnerInstanceIds: ['stale-demand-runner', 'cancelled-runner']},
+      1,
+      1,
+      false,
+      [
+        {
+          providerRunnerId: 'stale-demand-runner',
+          reason: 'activation-timeout',
+          activationTimeoutRetry: true,
+        },
+        {providerRunnerId: 'cancelled-runner', reason: 'job-cancelled'},
+      ],
+    );
+
+    expect(result).toBe(true);
+  });
+
   it('returns true when the deadline passed', () => {
     const result = shouldReturn(emptyResult, 1, 1, true);
 
@@ -81,7 +142,9 @@ describe('pollDemand', () => {
 
   it('returns immediately when terminate intents exist without reservation demand', async () => {
     vi.resetModules();
-    const pollDemandAndReserveTx = vi.fn().mockResolvedValue({stats: [], reservations: []});
+    const pollDemandAndReserveTx = vi
+      .fn()
+      .mockResolvedValue({stats: [], reservations: [], newlyReservedUnits: []});
     const listProvisionerTerminateIntentRowsTx = vi
       .fn()
       .mockResolvedValue([{providerRunnerId: 'provisioned-runner-1', reason: 'job-cancelled'}]);
@@ -129,10 +192,134 @@ describe('pollDemand', () => {
     }
   });
 
+  it('does not return immediately for retry-only activation-timeout intents', async () => {
+    vi.resetModules();
+    const abortController = new AbortController();
+    let listCalls = 0;
+    const pollDemandAndReserveTx = vi
+      .fn()
+      .mockResolvedValue({stats: [], reservations: [], newlyReservedUnits: []});
+    const listProvisionerTerminateIntentRowsTx = vi.fn().mockImplementation(() => {
+      listCalls += 1;
+      if (listCalls === 2) abortController.abort();
+      return [
+        {
+          providerRunnerId: 'stale-demand-runner',
+          reason: 'activation-timeout',
+          activationTimeoutRetry: true,
+        },
+      ];
+    });
+    const listActiveRunnerInstanceCountsByTemplateTx = vi.fn().mockResolvedValue([]);
+    vi.doMock('#db/db.js', () => ({
+      db: () => ({transaction: (callback: (tx: unknown) => Promise<unknown>) => callback({})}),
+    }));
+    vi.doMock('#db/reservations.js', () => ({
+      deleteReservationsByIds: vi.fn(),
+      pollDemandAndReserveTx,
+    }));
+    vi.doMock('#db/runner-instances.js', () => ({
+      listActiveRunnerInstanceCountsByTemplateTx,
+      listProvisionerTerminateIntentRowsTx,
+    }));
+    vi.doMock('#metrics/instance.js', () => ({
+      providerRunnerActivationOutcomeCount: {add: vi.fn()},
+      providerRunnerCountDivergenceCount: {add: vi.fn()},
+      providerRunnerTerminateIntentIssuedCount: {add: vi.fn()},
+    }));
+    const random = vi.spyOn(Math, 'random').mockReturnValue(0);
+
+    try {
+      const {pollDemand: mockedPollDemand} = await import('#core/demand.js');
+
+      const result = await mockedPollDemand({
+        workspaceId: crypto.randomUUID(),
+        provisionerId: crypto.randomUUID(),
+        maxReservations: 1,
+        waitSeconds: 60,
+        ttlSeconds: 60,
+        terminateIntentLimit: 1000,
+        templates: [
+          {templateKey: 'linux', labels: ['linux'], availableSlots: 1, starting: 0, running: 0},
+        ],
+        signal: abortController.signal,
+      });
+
+      expect(result).toEqual({
+        stats: [],
+        reservations: [],
+        terminateRunnerInstanceIds: ['stale-demand-runner'],
+      });
+      expect(listProvisionerTerminateIntentRowsTx).toHaveBeenCalledTimes(2);
+    } finally {
+      random.mockRestore();
+      vi.doUnmock('#db/db.js');
+      vi.doUnmock('#db/reservations.js');
+      vi.doUnmock('#db/runner-instances.js');
+      vi.doUnmock('#metrics/instance.js');
+      vi.resetModules();
+    }
+  });
+
+  it('returns immediately when demand is fully adopted by idle runners', async () => {
+    vi.resetModules();
+    const pollDemandAndReserveTx = vi.fn().mockResolvedValue({
+      stats: [],
+      reservations: [],
+      newlyReservedUnits: [{labels: ['linux'], count: 3}],
+    });
+    const listProvisionerTerminateIntentRowsTx = vi.fn().mockResolvedValue([]);
+    const listActiveRunnerInstanceCountsByTemplateTx = vi.fn().mockResolvedValue([]);
+    vi.doMock('#db/db.js', () => ({
+      db: () => ({transaction: (callback: (tx: unknown) => Promise<unknown>) => callback({})}),
+    }));
+    vi.doMock('#db/reservations.js', () => ({
+      deleteReservationsByIds: vi.fn(),
+      pollDemandAndReserveTx,
+    }));
+    vi.doMock('#db/runner-instances.js', () => ({
+      listActiveRunnerInstanceCountsByTemplateTx,
+      listProvisionerTerminateIntentRowsTx,
+    }));
+
+    try {
+      const {pollDemand: mockedPollDemand} = await import('#core/demand.js');
+
+      const result = await mockedPollDemand({
+        workspaceId: crypto.randomUUID(),
+        provisionerId: crypto.randomUUID(),
+        maxReservations: 3,
+        waitSeconds: 60,
+        ttlSeconds: 60,
+        terminateIntentLimit: 1000,
+        templates: [
+          {templateKey: 'linux', labels: ['linux'], availableSlots: 3, starting: 0, running: 3},
+        ],
+        signal: new AbortController().signal,
+      });
+
+      expect(result).toEqual({
+        stats: [],
+        reservations: [],
+        newlyReservedCount: 3,
+        terminateRunnerInstanceIds: [],
+      });
+      expect(pollDemandAndReserveTx).toHaveBeenCalledOnce();
+      expect(listActiveRunnerInstanceCountsByTemplateTx).toHaveBeenCalledOnce();
+    } finally {
+      vi.doUnmock('#db/db.js');
+      vi.doUnmock('#db/reservations.js');
+      vi.doUnmock('#db/runner-instances.js');
+      vi.resetModules();
+    }
+  });
+
   it('does not calculate divergence for a non-returned long-poll retry', async () => {
     vi.resetModules();
     const abortController = new AbortController();
-    const pollDemandAndReserveTx = vi.fn().mockResolvedValue({stats: [], reservations: []});
+    const pollDemandAndReserveTx = vi
+      .fn()
+      .mockResolvedValue({stats: [], reservations: [], newlyReservedUnits: []});
     const listProvisionerTerminateIntentRowsTx = vi.fn().mockImplementation(() => {
       abortController.abort();
       return [];
@@ -180,7 +367,9 @@ describe('pollDemand', () => {
     vi.resetModules();
     vi.stubEnv('PROVISIONED_RUNNER_COUNT_DIVERGENCE_TEMPLATE_KEY_LABEL_ENABLED', 'true');
     const divergenceAdd = vi.fn();
-    const pollDemandAndReserveTx = vi.fn().mockResolvedValue({stats: [], reservations: []});
+    const pollDemandAndReserveTx = vi
+      .fn()
+      .mockResolvedValue({stats: [], reservations: [], newlyReservedUnits: []});
     const listProvisionerTerminateIntentRowsTx = vi.fn().mockResolvedValue([]);
     const listActiveRunnerInstanceCountsByTemplateTx = vi
       .fn()

package/src/core/demand.ts

@@ -18,6 +18,7 @@ import {
 import {
   providerRunnerCountDivergenceCount,
   providerRunnerTerminateIntentIssuedCount,
+  recordProviderRunnerActivationOutcome,
 } from '#metrics/instance.js';
 
 export interface PollDemandParams {
@@ -35,6 +36,8 @@ export interface PollDemandResult {
   stats: DemandStat[];
   reservations: ReservationGrant[];
   terminateRunnerInstanceIds: string[];
+  /** Units reserved this poll. A bound-only allocation carries no launch grant, so this is what ends the long poll. */
+  newlyReservedCount?: number;
 }
 
 export interface RunnerInstanceCountDivergence {
@@ -78,6 +81,7 @@ export async function pollDemand(params: PollDemandParams): Promise<PollDemandRe
         provisionerId: params.provisionerId,
         maxReservations: params.maxReservations,
         ttlSeconds: params.ttlSeconds,
+        activationGraceSeconds: config.RESERVATION_TTL_SECONDS,
         templates: params.templates,
       });
       const terminateIntents = await listProvisionerTerminateIntentRowsTx(tx, {
@@ -85,12 +89,26 @@ export async function pollDemand(params: PollDemandParams): Promise<PollDemandRe
         provisionerId: params.provisionerId,
         limit: params.terminateIntentLimit,
       });
+      const newlyReservedCount = demand.newlyReservedUnits.reduce(
+        (total, reservation) => total + reservation.count,
+        0,
+      );
       const result: PollDemandResult = {
-        ...demand,
+        stats: demand.stats,
+        reservations: demand.reservations,
+        ...(newlyReservedCount > 0 ? {newlyReservedCount} : {}),
         terminateRunnerInstanceIds: terminateIntents.map((intent) => intent.providerRunnerId),
       };
 
-      if (!shouldReturn(result, params.maxReservations, totalCapacity, deadlinePassed)) {
+      if (
+        !shouldReturn(
+          result,
+          params.maxReservations,
+          totalCapacity,
+          deadlinePassed,
+          terminateIntents,
+        )
+      ) {
         return {result, terminateIntents, divergences: []};
       }
 
@@ -113,7 +131,15 @@ export async function pollDemand(params: PollDemandParams): Promise<PollDemandRe
 
     lastSnapshot = snapshot;
 
-    if (shouldReturn(lastSnapshot.result, params.maxReservations, totalCapacity, deadlinePassed)) {
+    if (
+      shouldReturn(
+        lastSnapshot.result,
+        params.maxReservations,
+        totalCapacity,
+        deadlinePassed,
+        lastSnapshot.terminateIntents,
+      )
+    ) {
       recordPollDemandMetrics(params, lastSnapshot);
       return lastSnapshot.result;
     }
@@ -132,6 +158,9 @@ export async function pollDemand(params: PollDemandParams): Promise<PollDemandRe
 }
 
 export async function releaseReservationGrants(reservations: ReservationGrant[]): Promise<void> {
+  // Fully adopted reservations are intentionally absent from this list: deleting
+  // one on disconnect would orphan the runner bound to that reservation. They
+  // remain protected until terminal cleanup or expiry.
   await deleteReservationsByIds(reservations.map((reservation) => reservation.reservationId));
 }
 
@@ -140,12 +169,19 @@ export function shouldReturn(
   maxReservations: number,
   totalCapacity: number,
   deadlinePassed: boolean,
+  terminateIntents?: readonly RunnerInstanceTerminateIntent[],
 ): boolean {
+  const hasImmediateTerminateIntent =
+    terminateIntents?.some(
+      (intent) => intent.reason !== 'activation-timeout' || !intent.activationTimeoutRetry,
+    ) ?? result.terminateRunnerInstanceIds.length > 0;
+
   return (
     maxReservations === 0 ||
     totalCapacity === 0 ||
     result.reservations.length > 0 ||
-    result.terminateRunnerInstanceIds.length > 0 ||
+    (result.newlyReservedCount ?? 0) > 0 ||
+    hasImmediateTerminateIntent ||
     deadlinePassed
   );
 }
@@ -219,6 +255,10 @@ function recordPollDemandMetrics(params: PollDemandParams, snapshot: PollDemandS
       surface: 'poll-demand',
       reason: intent.reason,
     });
+    // Retries redeliver an intent already counted on its first emission.
+    if (intent.reason === 'activation-timeout' && !intent.activationTimeoutRetry) {
+      recordProviderRunnerActivationOutcome({outcome: 'reaped'});
+    }
   }
 }
 

package/src/core/entities/reservation.ts

@@ -4,6 +4,7 @@ export interface Reservation {
   provisionerId: string;
   requiredLabels: string[];
   count: number;
+  kind: 'bound' | 'launch';
   createdAt: Date;
   expiresAt: Date;
 }

package/src/core/entities/runner-instance.ts

@@ -6,12 +6,16 @@ export type RunnerInstanceState =
   | 'failed'
   | 'terminated';
 
+export type RunnerInstanceLaunchKind = 'demand' | 'warm' | 'manual';
+
 export interface RunnerInstance {
   id: string;
   workspaceId: string | null;
   provisionerId: string;
   providerRunnerId: string;
+  intendedReservationId: string | null;
   reservationId: string | null;
+  launchKind: RunnerInstanceLaunchKind;
   assignedAt?: Date | null;
   templateKey: string | null;
   labels: string[];

package/src/core/runner-activation.ts

@@ -1,14 +1,22 @@
+import {logger} from '@shipfox/node-opentelemetry';
 import {extractDisplayPrefix, generateOpaqueToken, hashOpaqueToken} from '@shipfox/node-tokens';
 import {and, eq, isNull, sql} from 'drizzle-orm';
 import type {Tx} from '#db/db.js';
 import {db} from '#db/db.js';
 import {runnerActivationTokens} from '#db/schema/runner-activation-tokens.js';
+import type {RunnerInstanceDb} from '#db/schema/runner-instances.js';
 import {providerRunners} from '#db/schema/runner-instances.js';
+import {
+  type RunnerActivationTokenNotIssuedReason,
+  type RunnerActivationTokenNotIssuedSurface,
+  recordRunnerActivationTokenNotIssued,
+} from '#metrics/instance.js';
 
 export async function issueRunnerActivationToken(params: {
   runnerInstanceId: string;
   provisionerId: string;
   ttlSeconds: number;
+  surface: RunnerActivationTokenNotIssuedSurface;
 }): Promise<string | null> {
   return await db().transaction(async (tx) => issueRunnerActivationTokenTx(tx, params));
 }
@@ -19,6 +27,7 @@ export async function issueRunnerActivationTokenTx(
     runnerInstanceId: string;
     provisionerId: string;
     ttlSeconds: number;
+    surface: RunnerActivationTokenNotIssuedSurface;
   },
 ): Promise<string | null> {
   await tx.execute(
@@ -39,7 +48,11 @@ export async function issueRunnerActivationTokenTx(
     )
     .limit(1)
     .for('update');
-  if (!runner?.workspaceId || runner.runnerSessionId || runner.state !== 'running') return null;
+  const notIssuedReason = getRunnerActivationTokenNotIssuedReason(runner);
+  if (notIssuedReason) {
+    recordRunnerActivationTokenNotIssuedWithLog(notIssuedReason, params);
+    return null;
+  }
 
   await tx
     .update(runnerActivationTokens)
@@ -69,6 +82,7 @@ export async function getRunnerAssignment(params: {
     .select({
       workspaceId: providerRunners.workspaceId,
       runnerSessionId: providerRunners.runnerSessionId,
+      state: providerRunners.state,
     })
     .from(providerRunners)
     .where(
@@ -78,5 +92,37 @@ export async function getRunnerAssignment(params: {
       ),
     )
     .limit(1);
-  return runner?.workspaceId && !runner.runnerSessionId ? runner : null;
+  return runner?.workspaceId && !runner.runnerSessionId && runner.state === 'running'
+    ? {workspaceId: runner.workspaceId, runnerSessionId: runner.runnerSessionId}
+    : null;
+}
+
+function getRunnerActivationTokenNotIssuedReason(
+  runner: Pick<RunnerInstanceDb, 'workspaceId' | 'runnerSessionId' | 'state'> | undefined,
+): RunnerActivationTokenNotIssuedReason | null {
+  if (!runner) return 'runner-not-found';
+  if (!runner.workspaceId) return 'missing-workspace';
+  if (runner.runnerSessionId) return 'existing-session';
+  if (runner.state !== 'running') return 'not-running';
+  return null;
+}
+
+function recordRunnerActivationTokenNotIssuedWithLog(
+  reason: RunnerActivationTokenNotIssuedReason,
+  params: {
+    runnerInstanceId: string;
+    provisionerId: string;
+    surface: RunnerActivationTokenNotIssuedSurface;
+  },
+): void {
+  recordRunnerActivationTokenNotIssued({reason, surface: params.surface});
+  logger().debug(
+    {
+      runnerInstanceId: params.runnerInstanceId,
+      provisionerId: params.provisionerId,
+      reason,
+      surface: params.surface,
+    },
+    'Runner activation token was not issued',
+  );
 }