@shipfox/api-runners 10.0.0 → 10.2.0

package/src/db/reservations.ts

@@ -1,11 +1,26 @@
 import {canonicalizeLabels} from '@shipfox/runner-labels';
-import {and, asc, eq, gt, inArray, lt, sql} from 'drizzle-orm';
+import {
+  and,
+  arrayContains,
+  asc,
+  eq,
+  exists,
+  gt,
+  inArray,
+  isNotNull,
+  isNull,
+  lt,
+  sql,
+} from 'drizzle-orm';
 import type {Tx} from './db.js';
 import {db} from './db.js';
 import {pendingJobExecutions} from './schema/pending-job-executions.js';
 import {provisionerCapabilitySnapshots} from './schema/provisioner-capability-snapshots.js';
 import {provisionerTokens} from './schema/provisioner-tokens.js';
 import {reservations} from './schema/reservations.js';
+import {runnerActivationTokens} from './schema/runner-activation-tokens.js';
+import {runnerControlSessions} from './schema/runner-control-sessions.js';
+import {providerRunners} from './schema/runner-instances.js';
 
 export interface ReservationTemplate {
   templateKey: string;
@@ -243,6 +258,13 @@ async function pollDemandAndReserveLockedTx(
 
       remainingMaxReservations -= grant;
       drawSlots(satisfyingTemplates, grant);
+      await bindIdleRunnerInstancesTx(tx, {
+        provisionerId: params.provisionerId,
+        reservationId: inserted.id,
+        requiredLabels: demand.requiredLabels,
+        count: grant,
+        workspaceId: params.workspaceId,
+      });
       reservedAfterGrant += grant;
       grants.push({
         reservationId: inserted.id,
@@ -265,6 +287,71 @@ async function pollDemandAndReserveLockedTx(
   return {stats, reservations: grants};
 }
 
+async function bindIdleRunnerInstancesTx(
+  tx: Tx,
+  params: {
+    provisionerId: string;
+    reservationId: string;
+    workspaceId: string;
+    requiredLabels: string[];
+    count: number;
+  },
+): Promise<void> {
+  const idleRunners = await tx
+    .select({id: providerRunners.id})
+    .from(providerRunners)
+    .where(
+      and(
+        eq(providerRunners.provisionerId, params.provisionerId),
+        isNull(providerRunners.workspaceId),
+        isNull(providerRunners.reservationId),
+        isNull(providerRunners.intendedReservationId),
+        isNull(providerRunners.runnerSessionId),
+        isNotNull(providerRunners.providerRunnerId),
+        eq(providerRunners.state, 'running'),
+        arrayContains(providerRunners.labels, params.requiredLabels),
+        exists(
+          tx
+            .select({id: runnerControlSessions.id})
+            .from(runnerControlSessions)
+            .where(
+              and(
+                eq(runnerControlSessions.runnerInstanceId, providerRunners.id),
+                eq(runnerControlSessions.provisionerId, params.provisionerId),
+                isNull(runnerControlSessions.closedAt),
+                gt(runnerControlSessions.expiresAt, sql`now()`),
+              ),
+            ),
+        ),
+      ),
+    )
+    .orderBy(asc(providerRunners.createdAt), asc(providerRunners.id))
+    .limit(params.count)
+    .for('update');
+
+  if (idleRunners.length === 0) return;
+
+  await tx
+    .update(providerRunners)
+    .set({
+      workspaceId: params.workspaceId,
+      reservationId: params.reservationId,
+      assignedAt: sql`now()`,
+      updatedAt: sql`now()`,
+    })
+    .where(
+      and(
+        inArray(
+          providerRunners.id,
+          idleRunners.map((runner) => runner.id),
+        ),
+        isNull(providerRunners.workspaceId),
+        isNull(providerRunners.reservationId),
+        isNull(providerRunners.runnerSessionId),
+      ),
+    );
+}
+
 async function listInstallationDemandWorkspaceIds(eligibleWorkspaceIds: ReadonlySet<string>) {
   if (eligibleWorkspaceIds.size === 0) return [];
   const rows = await db()
@@ -315,27 +402,95 @@ async function listActiveWorkspaceCapabilityLabelsTx(
 }
 
 export async function deleteExpiredReservations(params?: {limit?: number}): Promise<number> {
-  const expiredIds = db()
-    .select({id: reservations.id})
-    .from(reservations)
-    .where(lt(reservations.expiresAt, sql`now()`))
-    .orderBy(asc(reservations.expiresAt))
-    .limit(params?.limit ?? 1000);
+  return await db().transaction(async (tx) => {
+    const expiredRows = await tx
+      .select({id: reservations.id})
+      .from(reservations)
+      .where(lt(reservations.expiresAt, sql`now()`))
+      .orderBy(asc(reservations.expiresAt))
+      .limit(params?.limit ?? 1000);
+    return await deleteReservationsWithCleanupTx(
+      tx,
+      expiredRows.map((reservation) => reservation.id),
+      {expiredOnly: true},
+    );
+  });
+}
 
-  const deleted = await db()
-    .delete(reservations)
-    .where(inArray(reservations.id, expiredIds))
-    .returning({id: reservations.id});
+export async function deleteReservationsByIds(ids: string[]): Promise<number> {
+  if (ids.length === 0) return 0;
 
-  return deleted.length;
+  return await db().transaction(async (tx) => {
+    return await deleteReservationsWithCleanupTx(tx, ids);
+  });
 }
 
-export async function deleteReservationsByIds(ids: string[]): Promise<number> {
+async function deleteReservationsWithCleanupTx(
+  tx: Tx,
+  ids: string[],
+  params: {expiredOnly?: boolean} = {},
+): Promise<number> {
   if (ids.length === 0) return 0;
 
-  const deleted = await db()
+  const assignedRunners = await tx
+    .select({id: providerRunners.id, reservationId: providerRunners.reservationId})
+    .from(providerRunners)
+    .where(
+      and(
+        inArray(providerRunners.reservationId, ids),
+        isNull(providerRunners.runnerSessionId),
+        isNull(providerRunners.reservationReleasedAt),
+      ),
+    )
+    .for('update');
+  const reservationRows = await tx
+    .select({id: reservations.id})
+    .from(reservations)
+    .where(
+      and(
+        inArray(reservations.id, ids),
+        params.expiredOnly ? lt(reservations.expiresAt, sql`now()`) : undefined,
+      ),
+    )
+    .for('update');
+  const reservationIds = reservationRows.map((reservation) => reservation.id);
+
+  if (reservationIds.length === 0) return 0;
+
+  const assignedRunnerIds = assignedRunners
+    .filter((runner) => runner.reservationId && reservationIds.includes(runner.reservationId))
+    .map((runner) => runner.id);
+  if (assignedRunnerIds.length > 0) {
+    await tx
+      .update(runnerActivationTokens)
+      .set({revokedAt: sql`now()`})
+      .where(
+        and(
+          inArray(runnerActivationTokens.runnerInstanceId, assignedRunnerIds),
+          isNull(runnerActivationTokens.consumedAt),
+          isNull(runnerActivationTokens.revokedAt),
+        ),
+      );
+    await tx
+      .update(providerRunners)
+      .set({
+        workspaceId: null,
+        reservationId: null,
+        assignedAt: null,
+        updatedAt: sql`now()`,
+      })
+      .where(
+        and(
+          inArray(providerRunners.id, assignedRunnerIds),
+          isNull(providerRunners.runnerSessionId),
+          isNull(providerRunners.reservationReleasedAt),
+        ),
+      );
+  }
+
+  const deleted = await tx
     .delete(reservations)
-    .where(inArray(reservations.id, ids))
+    .where(inArray(reservations.id, reservationIds))
     .returning({id: reservations.id});
 
   return deleted.length;

package/src/db/schema/admin-command-results.ts

@@ -0,0 +1,29 @@
+import {uuidv7PrimaryKey} from '@shipfox/node-drizzle';
+import {jsonb, text, timestamp, uniqueIndex, uuid} from 'drizzle-orm/pg-core';
+import {pgTable} from './common.js';
+
+export interface StoredProvisionerTokenAdministrationResult {
+  provisionerTokenId: string;
+  correlationId: string;
+}
+
+export const runnersAdminCommandResults = pgTable(
+  'admin_command_results',
+  {
+    id: uuidv7PrimaryKey(),
+    actorId: uuid('actor_id').notNull(),
+    idempotencyKeyFingerprint: text('idempotency_key_fingerprint').notNull(),
+    command: text('command').notNull(),
+    requestFingerprint: text('request_fingerprint').notNull(),
+    result: jsonb('result').$type<StoredProvisionerTokenAdministrationResult>().notNull(),
+    createdAt: timestamp('created_at', {withTimezone: true}).notNull().defaultNow(),
+  },
+  (table) => [
+    uniqueIndex('runners_admin_command_results_actor_key_unique').on(
+      table.actorId,
+      table.idempotencyKeyFingerprint,
+    ),
+  ],
+);
+
+export type RunnersAdminCommandResultDb = typeof runnersAdminCommandResults.$inferSelect;

package/src/db/schema/runner-instances.ts

@@ -31,6 +31,7 @@ export const providerRunners = pgTable(
     state: providerRunnerStateEnum('state').notNull(),
     reason: text('reason'),
     runnerSessionId: uuid('runner_session_id'),
+    firstClaimedAt: timestamp('first_claimed_at', {withTimezone: true}),
     providerKind: text('provider_kind'),
     protocolVersion: text('protocol_version'),
     capabilities: jsonb('capabilities').$type<RunnerToolCapabilitiesDto | null>(),

package/src/index.test.ts

@@ -9,7 +9,7 @@ describe('createRunnersModule', () => {
 
     expect(module.name).toBe('runners');
     expect(module.auth).toHaveLength(3);
-    expect(module.routes).toHaveLength(11);
+    expect(module.routes).toHaveLength(13);
     expect(module.workers).toHaveLength(1);
   });
 

package/src/index.ts

@@ -1,6 +1,7 @@
 import {dirname, resolve} from 'node:path';
 import {fileURLToPath} from 'node:url';
 import type {AuthInterModuleClient} from '@shipfox/api-auth-dto/inter-module';
+import {administrationActionEventSchemas} from '@shipfox/api-common-dto';
 import {runnersEventSchemas} from '@shipfox/api-runners-dto';
 import {
   WORKFLOWS_JOB_EXECUTION_TIMED_OUT,
@@ -21,6 +22,8 @@ import {createRunnersInterModulePresentation} from '#presentation/inter-module.j
 import {createRunnersMaintenanceActivities} from '#temporal/activities/index.js';
 import {RUNNERS_MAINTENANCE_TASK_QUEUE} from '#temporal/constants.js';
 
+const runnersPublisherEventSchemas = {...runnersEventSchemas, ...administrationActionEventSchemas};
+
 export {
   type EffectiveRunnerToolCapabilitiesResult,
   getEffectiveRunnerToolCapabilities,
@@ -58,7 +61,9 @@ export function createRunnersModule({
     ],
     routes: createRunnerRoutes(auth, options),
     metrics: registerRunnersServiceMetrics,
-    publishers: [{name: 'runners', table: runnersOutbox, db, eventSchemas: runnersEventSchemas}],
+    publishers: [
+      {name: 'runners', table: runnersOutbox, db, eventSchemas: runnersPublisherEventSchemas},
+    ],
     subscribers: [subscriber(WORKFLOWS_JOB_EXECUTION_TIMED_OUT, onWorkflowsJobExecutionTimedOut)],
     workers: [
       {

package/src/presentation/dto/admin-provisioner-token.ts

@@ -0,0 +1,23 @@
+import type {ProvisionerToken} from '#core/entities/provisioner-token.js';
+import {installationProvisionerTokenStatus} from '#core/provisioner-tokens.js';
+
+export function toAdministratorProvisionerTokenDto(token: ProvisionerToken) {
+  if (token.scope !== 'installation') {
+    throw new Error('Workspace provisioner tokens cannot be returned from administrator routes');
+  }
+
+  return {
+    id: token.id,
+    scope: 'installation' as const,
+    prefix: token.prefix,
+    name: token.name,
+    status: installationProvisionerTokenStatus(token),
+    created_by_user_id: token.createdByUserId,
+    revoked_by_user_id: token.revokedByUserId,
+    expires_at: token.expiresAt?.toISOString() ?? null,
+    revoked_at: token.revokedAt?.toISOString() ?? null,
+    last_seen_at: token.lastSeenAt?.toISOString() ?? null,
+    created_at: token.createdAt.toISOString(),
+    updated_at: token.updatedAt.toISOString(),
+  };
+}

package/src/presentation/dto/admin-runner-instances.ts

@@ -0,0 +1,19 @@
+import type {RunnerAdministratorInstance} from '#core/admin-runner-instances.js';
+
+export function toRunnerAdministratorInstanceDto(runner: RunnerAdministratorInstance) {
+  return {
+    id: runner.id,
+    lifecycle_state: runner.lifecycleState,
+    compute_state: runner.computeState,
+    enrollment_state: runner.enrollmentState,
+    assignment_presence: runner.assignmentPresence,
+    assigned_workspace: runner.assignedWorkspace,
+    labels: runner.labels,
+    created_at: runner.createdAt.toISOString(),
+    last_heartbeat_at: runner.lastHeartbeatAt.toISOString(),
+    closure_reason: runner.closureReason,
+    closed_at: runner.closedAt?.toISOString() ?? null,
+    provisioner: runner.provisioner,
+    reconciliation_status: runner.reconciliationStatus,
+  };
+}

package/src/presentation/dto/index.ts

@@ -1,4 +1,6 @@
 export {toActiveProvisionerDto} from './active-provisioner.js';
+export {toAdministratorProvisionerTokenDto} from './admin-provisioner-token.js';
+export {toRunnerAdministratorInstanceDto} from './admin-runner-instances.js';
 export {toManualRegistrationTokenDto} from './manual-registration-token.js';
 export {toPollDemandResponseDto} from './poll-demand.js';
 export {toProvisionerTokenDto} from './provisioner-token.js';

package/src/presentation/routes/admin-provisioner-tokens.test.ts

@@ -0,0 +1,227 @@
+import {
+  AUTH_LEASED_JOB,
+  AUTH_PROVISIONER_TOKEN,
+  AUTH_RUNNER_REGISTRATION_TOKEN,
+  AUTH_RUNNER_SESSION,
+  AUTH_USER,
+  buildUserContext,
+  setUserContext,
+} from '@shipfox/api-auth-context';
+import {
+  type AuthInterModuleClient,
+  authInterModuleContract,
+} from '@shipfox/api-auth-dto/inter-module';
+import {createInterModuleKnownError} from '@shipfox/inter-module';
+import {type AuthMethod, ClientError, closeApp, createApp} from '@shipfox/node-fastify';
+import {hashOpaqueToken} from '@shipfox/node-tokens';
+import {vi} from '@shipfox/vitest/vi';
+import {eq} from 'drizzle-orm';
+import type {FastifyInstance, FastifyRequest} from 'fastify';
+import {db} from '#db/db.js';
+import {runnersAdminCommandResults} from '#db/schema/admin-command-results.js';
+import {runnersOutbox} from '#db/schema/outbox.js';
+import {provisionerTokens} from '#db/schema/provisioner-tokens.js';
+import {provisionerTokenFactory, runnersTestAuthClient} from '#test/index.js';
+import {createRunnerRoutes} from './index.js';
+
+const USER_ID = '00000000-0000-4000-8000-000000000001';
+
+const fakeUserAuth: AuthMethod = {
+  name: AUTH_USER,
+  authenticate: (request: FastifyRequest) => {
+    if (request.headers.authorization !== 'Bearer user') {
+      throw new ClientError('Invalid user token', 'unauthorized', {status: 401});
+    }
+    setUserContext(
+      request,
+      buildUserContext({
+        userId: USER_ID,
+        email: 'admin@example.com',
+        name: 'Administrator',
+        memberships: [],
+      }),
+    );
+    return Promise.resolve();
+  },
+};
+
+const passthroughAuth = (name: string): AuthMethod => ({
+  name,
+  authenticate: () => Promise.resolve(),
+});
+
+describe('administrator installation provisioner tokens', () => {
+  let app: FastifyInstance;
+  let auth: AuthInterModuleClient;
+
+  beforeEach(async () => {
+    await closeApp();
+    auth = {
+      ...runnersTestAuthClient,
+      requireAdminRole: vi.fn().mockResolvedValue({role: 'admin-owner'}),
+    };
+    app = await createApp({
+      auth: [
+        fakeUserAuth,
+        passthroughAuth(AUTH_RUNNER_REGISTRATION_TOKEN),
+        passthroughAuth(AUTH_RUNNER_SESSION),
+        passthroughAuth(AUTH_LEASED_JOB),
+        passthroughAuth(AUTH_PROVISIONER_TOKEN),
+      ],
+      routes: createRunnerRoutes(auth),
+      swagger: false,
+    });
+    await app.ready();
+  });
+
+  afterEach(async () => {
+    await db()
+      .delete(runnersAdminCommandResults)
+      .where(eq(runnersAdminCommandResults.actorId, USER_ID));
+    await db().delete(provisionerTokens).where(eq(provisionerTokens.createdByUserId, USER_ID));
+    await db()
+      .delete(runnersOutbox)
+      .where(eq(runnersOutbox.eventType, 'administration.action.performed'));
+    await closeApp();
+  });
+
+  test('creates, lists, and revokes only installation metadata with idempotent retries', async () => {
+    const createKey = `create-${crypto.randomUUID()}`;
+    const createRequest = {
+      method: 'POST' as const,
+      url: '/admin/runners/provisioner-tokens',
+      headers: {authorization: 'Bearer user', 'idempotency-key': createKey},
+      payload: {name: 'Cloud managed runners', reason: 'Rotate the installation credential'},
+    };
+    const created = await app.inject(createRequest);
+    expect(created.statusCode).toBe(201);
+    const createdBody = created.json();
+    expect(createdBody).toMatchObject({
+      scope: 'installation',
+      name: 'Cloud managed runners',
+      status: 'active',
+      raw_token: expect.any(String),
+      correlation_id: expect.any(String),
+    });
+
+    const retry = await app.inject(createRequest);
+    expect(retry.statusCode).toBe(201);
+    expect(retry.json()).toMatchObject({
+      id: createdBody.id,
+      raw_token: createdBody.raw_token,
+      correlation_id: createdBody.correlation_id,
+    });
+
+    const rows = await db().select().from(runnersAdminCommandResults);
+    expect(rows).toHaveLength(1);
+    expect(JSON.stringify(rows[0]?.result)).not.toContain(createdBody.raw_token);
+    expect(JSON.stringify(rows[0]?.result)).not.toContain(hashOpaqueToken(createdBody.raw_token));
+
+    const workspaceToken = await provisionerTokenFactory.create({
+      workspaceId: crypto.randomUUID(),
+      createdByUserId: USER_ID,
+    });
+    const list = await app.inject({
+      method: 'GET',
+      url: '/admin/runners/provisioner-tokens?status=active',
+      headers: {authorization: 'Bearer user'},
+    });
+    expect(list.statusCode).toBe(200);
+    const listedToken = list
+      .json()
+      .tokens.find((token: {id: string}) => token.id === createdBody.id);
+    expect(listedToken).toMatchObject({id: createdBody.id, scope: 'installation'});
+    expect(listedToken).not.toHaveProperty('raw_token');
+    expect(listedToken).not.toHaveProperty('hashed_token');
+    expect(list.json().tokens.map((token: {id: string}) => token.id)).not.toContain(
+      workspaceToken.id,
+    );
+
+    vi.mocked(auth.requireAdminRole).mockResolvedValue({role: 'admin-operator'});
+    const revokeKey = `revoke-${crypto.randomUUID()}`;
+    const revokeRequest = {
+      method: 'POST' as const,
+      url: `/admin/runners/provisioner-tokens/${createdBody.id}/revoke`,
+      headers: {authorization: 'Bearer user', 'idempotency-key': revokeKey},
+      payload: {reason: 'Replace the installation credential'},
+    };
+    const revoked = await app.inject(revokeRequest);
+    expect(revoked.statusCode).toBe(200);
+    expect(revoked.json()).toMatchObject({id: createdBody.id, status: 'revoked'});
+    expect(revoked.json()).not.toHaveProperty('raw_token');
+
+    const revokeRetry = await app.inject(revokeRequest);
+    expect(revokeRetry.statusCode).toBe(200);
+    expect(revokeRetry.json()).toMatchObject({
+      id: createdBody.id,
+      correlation_id: revoked.json().correlation_id,
+    });
+
+    const noOpRevoke = await app.inject({
+      ...revokeRequest,
+      headers: {...revokeRequest.headers, 'idempotency-key': `revoke-no-op-${crypto.randomUUID()}`},
+    });
+    expect(noOpRevoke.statusCode).toBe(200);
+    expect(noOpRevoke.json()).toMatchObject({
+      id: createdBody.id,
+      status: 'revoked',
+      revoked_by_user_id: USER_ID,
+    });
+
+    const events = (await db().select().from(runnersOutbox)).filter(
+      (event) => event.eventType === 'administration.action.performed',
+    );
+    expect(events).toHaveLength(2);
+    expect(events.map((event) => event.eventType)).toEqual([
+      'administration.action.performed',
+      'administration.action.performed',
+    ]);
+    expect(JSON.stringify(events)).not.toContain(createdBody.raw_token);
+    expect(JSON.stringify(events)).not.toContain(hashOpaqueToken(createdBody.raw_token));
+  });
+
+  test('fails loudly when an idempotent create replay cannot reproduce its raw token', async () => {
+    const createKey = `create-${crypto.randomUUID()}`;
+    const createRequest = {
+      method: 'POST' as const,
+      url: '/admin/runners/provisioner-tokens',
+      headers: {authorization: 'Bearer user', 'idempotency-key': createKey},
+      payload: {reason: 'Create the installation credential'},
+    };
+    const created = await app.inject(createRequest);
+    expect(created.statusCode).toBe(201);
+    const createdBody = created.json();
+
+    await db()
+      .update(provisionerTokens)
+      .set({hashedToken: hashOpaqueToken('a different token')})
+      .where(eq(provisionerTokens.id, createdBody.id));
+
+    const replay = await app.inject(createRequest);
+    expect(replay.statusCode).toBe(409);
+    expect(replay.json()).toMatchObject({code: 'idempotency-replay-unavailable'});
+  });
+
+  test('rejects administrators without the required role', async () => {
+    vi.mocked(auth.requireAdminRole).mockRejectedValueOnce(
+      createInterModuleKnownError(
+        authInterModuleContract.methods.requireAdminRole,
+        'admin-role-required',
+        {requiredRole: 'admin-owner'},
+      ),
+    );
+
+    const response = await app.inject({
+      method: 'POST',
+      url: '/admin/runners/provisioner-tokens',
+      headers: {authorization: 'Bearer user', 'idempotency-key': 'owner-required'},
+      payload: {reason: 'Create the installation credential'},
+    });
+
+    expect(response.statusCode).toBe(403);
+    expect(response.json()).toMatchObject({
+      code: 'forbidden',
+      details: {required_role: 'admin-owner'},
+    });
+  });
+});