@shipfox/api-runners 4.0.0 → 6.0.0

package/src/presentation/routes/runner-enrollment.ts

@@ -0,0 +1,174 @@
+import {
+  requireProvisionerContext,
+  requireRunnerControlSessionContext,
+} from '@shipfox/api-auth-context';
+import {
+  attachRunnerControlProviderIdBodySchema,
+  createRunnerInstancesBodySchema,
+  createRunnerInstancesResponseSchema,
+  runnerAssignmentPollResponseSchema,
+  runnerBootstrapExchangeBodySchema,
+  runnerBootstrapExchangeResponseSchema,
+  runnerControlHeartbeatResponseSchema,
+  runnerEnrollmentBodySchema,
+} from '@shipfox/api-runners-dto';
+import {ClientError, defineRoute} from '@shipfox/node-fastify';
+import {z} from 'zod';
+import {config} from '#config.js';
+import {getRunnerAssignment, issueRunnerActivationToken} from '#core/runner-activation.js';
+import {
+  attachRunnerControlProviderId,
+  createRunnerInstancesWithBootstrapTokens,
+  enrollRunnerControlSession,
+  exchangeRunnerBootstrapToken,
+  RunnerBootstrapTokenInvalidError,
+  RunnerControlSessionInvalidError,
+  touchRunnerControlSession,
+} from '#core/runner-control-sessions.js';
+import {runnerBootstrapExchangeCount, runnerControlHeartbeatCount} from '#metrics/instance.js';
+import {authenticateRunnerControlSession} from '#presentation/auth/index.js';
+
+export const createRunnerInstancesRoute = defineRoute({
+  method: 'POST',
+  path: '/runner-instances/batch',
+  description:
+    'Create provisioner-owned runner instances and one-use bootstrap tokens before launch',
+  schema: {
+    body: createRunnerInstancesBodySchema,
+    response: {200: createRunnerInstancesResponseSchema},
+  },
+  handler: async (request) => {
+    const {provisionerTokenId} = requireProvisionerContext(request);
+    const results = await createRunnerInstancesWithBootstrapTokens({
+      provisionerId: provisionerTokenId,
+      ...(request.body.provider_kind ? {providerKind: request.body.provider_kind} : {}),
+      runnerInstances: request.body.runner_instances.map((runner) =>
+        runner.template_key ? {templateKey: runner.template_key} : {},
+      ),
+      ttlSeconds: config.RUNNER_BOOTSTRAP_TOKEN_TTL_SECONDS,
+    });
+    return {
+      runner_instances: results.map((result) => ({
+        runner_instance_id: result.runnerInstanceId,
+        bootstrap_token: result.bootstrapToken,
+      })),
+    };
+  },
+});
+
+export const exchangeRunnerBootstrapRoute = defineRoute({
+  method: 'POST',
+  path: '/exchange',
+  description: 'Exchange a one-use runner bootstrap token for a workspace-neutral control session',
+  schema: {
+    body: runnerBootstrapExchangeBodySchema,
+    response: {200: runnerBootstrapExchangeResponseSchema},
+  },
+  errorHandler: (error) => {
+    if (error instanceof RunnerBootstrapTokenInvalidError)
+      runnerBootstrapExchangeCount.add(1, {outcome: 'rejected'});
+    if (error instanceof RunnerBootstrapTokenInvalidError)
+      throw new ClientError(error.message, 'runner-bootstrap-token-invalid', {status: 401});
+    throw error;
+  },
+  handler: async (request) => {
+    const result = await exchangeRunnerBootstrapToken({
+      rawToken: request.body.bootstrap_token,
+      ttlSeconds: config.RUNNER_CONTROL_SESSION_TTL_SECONDS,
+    });
+    runnerBootstrapExchangeCount.add(1, {outcome: 'accepted'});
+    return {
+      runner_instance_id: result.runnerInstanceId,
+      control_session_token: result.controlSessionToken,
+      expires_at: result.expiresAt.toISOString(),
+    };
+  },
+});
+
+export const enrollRunnerRoute = defineRoute({
+  method: 'POST',
+  path: '/enrollment',
+  description: 'Declare the authenticated runner instance labels and protocol capabilities',
+  schema: {body: runnerEnrollmentBodySchema, response: {204: z.null()}},
+  preHandler: authenticateRunnerControlSession,
+  errorHandler: (error) => {
+    if (error instanceof RunnerControlSessionInvalidError)
+      throw new ClientError(error.message, 'runner-control-session-invalid', {status: 409});
+    throw error;
+  },
+  handler: async (request, reply) => {
+    const session = requireRunnerControlSessionContext(request);
+    await enrollRunnerControlSession({
+      runnerInstanceId: session.runnerInstanceId,
+      provisionerId: session.provisionerId,
+      labels: request.body.labels,
+      ...(request.body.capabilities ? {capabilities: request.body.capabilities} : {}),
+      providerKind: request.body.provider_kind,
+      protocolVersion: request.body.protocol_version,
+    });
+    return await reply.code(204).send();
+  },
+});
+
+export const attachRunnerControlProviderIdRoute = defineRoute({
+  method: 'POST',
+  path: '/provider-runner',
+  description: 'Attach the authenticated runner instance provider identity once',
+  schema: {
+    body: attachRunnerControlProviderIdBodySchema,
+    response: {200: z.object({attached: z.boolean()})},
+  },
+  preHandler: authenticateRunnerControlSession,
+  handler: async (request) => {
+    const session = requireRunnerControlSessionContext(request);
+    return {
+      attached: await attachRunnerControlProviderId({
+        ...session,
+        providerRunnerId: request.body.provider_runner_id,
+      }),
+    };
+  },
+});
+
+export const runnerControlHeartbeatRoute = defineRoute({
+  method: 'POST',
+  path: '/heartbeat',
+  description: 'Record liveness for only the authenticated runner instance',
+  schema: {response: {200: runnerControlHeartbeatResponseSchema}},
+  preHandler: authenticateRunnerControlSession,
+  handler: async (request) => {
+    const session = requireRunnerControlSessionContext(request);
+    await touchRunnerControlSession(session.runnerInstanceId, session.provisionerId);
+    runnerControlHeartbeatCount.add(1);
+    return {ok: true};
+  },
+});
+
+export const runnerAssignmentPollRoute = defineRoute({
+  method: 'GET',
+  path: '/assignment',
+  description: 'Wait for only the authenticated runner instance assignment',
+  schema: {response: {200: runnerAssignmentPollResponseSchema}},
+  preHandler: authenticateRunnerControlSession,
+  handler: async (request, reply) => {
+    const session = requireRunnerControlSessionContext(request);
+    const deadline = Date.now() + config.RUNNER_ASSIGNMENT_POLL_MAX_WAIT_SECONDS * 1000;
+    const abortController = new AbortController();
+    reply.raw.on('close', () => abortController.abort());
+    while (Date.now() <= deadline) {
+      if (abortController.signal.aborted) return {activation_token: null};
+      const assignment = await getRunnerAssignment(session);
+      if (assignment) {
+        const activationToken = await issueRunnerActivationToken({
+          ...session,
+          ttlSeconds: config.RUNNER_ACTIVATION_TOKEN_TTL_SECONDS,
+        });
+        return {activation_token: activationToken};
+      }
+      await new Promise((resolve) =>
+        setTimeout(resolve, config.RUNNER_ASSIGNMENT_POLL_INTERVAL_MS),
+      );
+    }
+    return {activation_token: null};
+  },
+});

package/src/temporal/activities/index.ts

@@ -3,7 +3,7 @@ import {
   deleteExpiredReservationsActivity,
   deleteExpiredRunnerSessionsActivity,
   detectAndExpireStuckJobsActivity,
-  reapStaleProvisionedRunnersActivity,
+  reapStaleRunnerInstancesActivity,
 } from './maintenance-activities.js';
 
 export function createRunnersMaintenanceActivities() {
@@ -12,6 +12,6 @@ export function createRunnersMaintenanceActivities() {
     deleteExpiredReservationsActivity,
     deleteExpiredRunnerSessionsActivity,
     detectAndExpireStuckJobsActivity,
-    reapStaleProvisionedRunnersActivity,
+    reapStaleRunnerInstancesActivity,
   };
 }

package/src/temporal/activities/maintenance-activities.test.ts

@@ -3,14 +3,14 @@ import {
   deleteExpiredRunnerReservations,
   deleteExpiredRunnerSessions,
   detectAndExpireStuckJobs,
-  reapStaleProvisionedRunners,
+  reapStaleRunnerInstances,
 } from '#core/maintenance.js';
 import {
   deleteExpiredEphemeralRegistrationTokensActivity,
   deleteExpiredReservationsActivity,
   deleteExpiredRunnerSessionsActivity,
   detectAndExpireStuckJobsActivity,
-  reapStaleProvisionedRunnersActivity,
+  reapStaleRunnerInstancesActivity,
 } from './maintenance-activities.js';
 
 vi.mock('#core/maintenance.js', () => ({
@@ -18,7 +18,7 @@ vi.mock('#core/maintenance.js', () => ({
   deleteExpiredRunnerReservations: vi.fn(),
   deleteExpiredRunnerSessions: vi.fn(),
   detectAndExpireStuckJobs: vi.fn(),
-  reapStaleProvisionedRunners: vi.fn(),
+  reapStaleRunnerInstances: vi.fn(),
 }));
 
 beforeEach(() => {
@@ -47,20 +47,20 @@ describe('deleteExpiredReservationsActivity', () => {
   });
 });
 
-describe('reapStaleProvisionedRunnersActivity', () => {
+describe('reapStaleRunnerInstancesActivity', () => {
   it('delegates to core maintenance', async () => {
-    vi.mocked(reapStaleProvisionedRunners).mockResolvedValueOnce({
+    vi.mocked(reapStaleRunnerInstances).mockResolvedValueOnce({
       reaped: 4,
       reservationsReleased: 2,
     });
 
-    const result = await reapStaleProvisionedRunnersActivity({
+    const result = await reapStaleRunnerInstancesActivity({
       thresholdSeconds: 300,
       limit: 100,
     });
 
     expect(result).toEqual({reaped: 4, reservationsReleased: 2});
-    expect(reapStaleProvisionedRunners).toHaveBeenCalledWith({
+    expect(reapStaleRunnerInstances).toHaveBeenCalledWith({
       thresholdSeconds: 300,
       limit: 100,
     });

package/src/temporal/activities/maintenance-activities.ts

@@ -3,7 +3,7 @@ import {
   deleteExpiredRunnerReservations,
   deleteExpiredRunnerSessions,
   detectAndExpireStuckJobs,
-  reapStaleProvisionedRunners,
+  reapStaleRunnerInstances,
 } from '#core/maintenance.js';
 
 export function detectAndExpireStuckJobsActivity(params: {
@@ -18,11 +18,11 @@ export function deleteExpiredReservationsActivity(params?: {
   return deleteExpiredRunnerReservations(params);
 }
 
-export function reapStaleProvisionedRunnersActivity(params?: {
+export function reapStaleRunnerInstancesActivity(params?: {
   thresholdSeconds: number;
   limit: number;
 }): Promise<{reaped: number; reservationsReleased: number}> {
-  return reapStaleProvisionedRunners(params);
+  return reapStaleRunnerInstances(params);
 }
 
 export function deleteExpiredRunnerSessionsActivity(params?: {

package/src/temporal/workflows/stuck-job-detector.test.ts

@@ -3,7 +3,7 @@ const mocks = vi.hoisted(() => ({
   deleteExpiredReservationsActivity: vi.fn(),
   deleteExpiredRunnerSessionsActivity: vi.fn(),
   detectAndExpireStuckJobsActivity: vi.fn(),
-  reapStaleProvisionedRunnersActivity: vi.fn(),
+  reapStaleRunnerInstancesActivity: vi.fn(),
   info: vi.fn(),
   warn: vi.fn(),
 }));
@@ -19,7 +19,7 @@ vi.mock('@temporalio/workflow', () => ({
     deleteExpiredReservationsActivity: mocks.deleteExpiredReservationsActivity,
     deleteExpiredRunnerSessionsActivity: mocks.deleteExpiredRunnerSessionsActivity,
     detectAndExpireStuckJobsActivity: mocks.detectAndExpireStuckJobsActivity,
-    reapStaleProvisionedRunnersActivity: mocks.reapStaleProvisionedRunnersActivity,
+    reapStaleRunnerInstancesActivity: mocks.reapStaleRunnerInstancesActivity,
   })),
 }));
 
@@ -30,7 +30,7 @@ describe('stuckJobDetector', () => {
     mocks.deleteExpiredReservationsActivity.mockResolvedValue({deleted: 0});
     mocks.deleteExpiredRunnerSessionsActivity.mockResolvedValue({deleted: 0});
     mocks.detectAndExpireStuckJobsActivity.mockResolvedValue({expired: 0});
-    mocks.reapStaleProvisionedRunnersActivity.mockResolvedValue({
+    mocks.reapStaleRunnerInstancesActivity.mockResolvedValue({
       reaped: 0,
       reservationsReleased: 0,
     });

package/src/temporal/workflows/stuck-job-detector.ts

@@ -8,7 +8,7 @@ const {
   deleteExpiredReservationsActivity,
   deleteExpiredRunnerSessionsActivity,
   detectAndExpireStuckJobsActivity,
-  reapStaleProvisionedRunnersActivity,
+  reapStaleRunnerInstancesActivity,
 } = proxyActivities<ReturnType<typeof createRunnersMaintenanceActivities>>({
   startToCloseTimeout: '60s',
 });
@@ -57,7 +57,7 @@ export async function stuckJobDetector(): Promise<void> {
     });
   }
 
-  const {reaped, reservationsReleased} = await reapStaleProvisionedRunnersActivity();
+  const {reaped, reservationsReleased} = await reapStaleRunnerInstancesActivity();
   if (reaped > 0) {
     log.info('Stuck-job detector reaped stale provisioned runners', {
       reaped,

package/test/env.ts

@@ -5,7 +5,5 @@ process.env.POSTGRES_PASSWORD ??= 'password';
 process.env.POSTGRES_DATABASE = 'api_test';
 process.env.POSTGRES_MAX_CONNECTIONS ??= '5';
 process.env.WORKSPACE_JWT_SECRET = 'test-secret';
-process.env.AUTH_JWT_SECRET = 'test-secret';
-process.env.AUTH_JOB_LEASE_TOKEN_SECRET = 'test-lease-secret';
-process.env.AUTH_RUNNER_SESSION_TOKEN_SECRET = 'test-runner-session-secret';
+process.env.AUTH_ROOT_KEY = 'MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=';
 process.env.TZ = 'UTC';

package/test/factories/ephemeral-registration-token.ts

@@ -18,7 +18,7 @@ export const ephemeralRegistrationTokenFactory = Factory.define<
       workspaceId: token.workspaceId,
       provisionerId: token.provisionerId,
       reservationId: token.reservationId,
-      provisionedRunnerId: token.provisionedRunnerId,
+      providerRunnerId: token.providerRunnerId,
       hashedToken: token.hashedToken,
       prefix: token.prefix,
       expiresAt: token.expiresAt,
@@ -30,7 +30,7 @@ export const ephemeralRegistrationTokenFactory = Factory.define<
     workspaceId: crypto.randomUUID(),
     provisionerId: crypto.randomUUID(),
     reservationId: null,
-    provisionedRunnerId: `provisioned-runner-${crypto.randomUUID()}`,
+    providerRunnerId: `provisioned-runner-${crypto.randomUUID()}`,
     hashedToken: hashOpaqueToken(rawToken),
     prefix: extractDisplayPrefix(rawToken),
     expiresAt: new Date(Date.now() + 60_000),

package/test/factories/provisioner-token.ts

@@ -15,7 +15,8 @@ export const provisionerTokenFactory = Factory.define<
 
   onCreate((token) => {
     return createProvisionerToken({
-      workspaceId: token.workspaceId,
+      scope: token.scope,
+      workspaceId: token.scope === 'workspace' ? token.workspaceId : undefined,
       hashedToken: token.hashedToken,
       prefix: token.prefix,
       name: token.name ?? undefined,
@@ -26,6 +27,7 @@ export const provisionerTokenFactory = Factory.define<
 
   return {
     id: crypto.randomUUID(),
+    scope: 'workspace',
     workspaceId: crypto.randomUUID(),
     hashedToken: hashOpaqueToken(rawToken),
     prefix: extractDisplayPrefix(rawToken),

package/test/factories/runner-instance.ts

@@ -0,0 +1,57 @@
+import {Factory} from 'fishery';
+import type {RunnerInstance} from '#core/entities/runner-instance.js';
+import {db} from '#db/db.js';
+import {providerRunners, toRunnerInstance} from '#db/schema/runner-instances.js';
+
+export const providerRunnerFactory = Factory.define<RunnerInstance>(({onCreate}) => {
+  onCreate(async (providerRunner) => {
+    const [row] = await db()
+      .insert(providerRunners)
+      .values({
+        workspaceId: providerRunner.workspaceId,
+        provisionerId: providerRunner.provisionerId,
+        providerRunnerId: providerRunner.providerRunnerId,
+        reservationId: providerRunner.reservationId,
+        templateKey: providerRunner.templateKey,
+        labels: providerRunner.labels,
+        state: providerRunner.state,
+        reason: providerRunner.reason,
+        runnerSessionId: providerRunner.runnerSessionId,
+        providerKind: providerRunner.providerKind,
+        reportedAt: providerRunner.reportedAt,
+        startedAt: providerRunner.startedAt,
+        stoppingAt: providerRunner.stoppingAt,
+        stoppedAt: providerRunner.stoppedAt,
+        failedAt: providerRunner.failedAt,
+        terminatedAt: providerRunner.terminatedAt,
+        reservationReleasedAt: providerRunner.reservationReleasedAt,
+      })
+      .returning();
+
+    if (!row) throw new Error('Insert returned no rows');
+    return toRunnerInstance(row);
+  });
+
+  return {
+    id: crypto.randomUUID(),
+    workspaceId: crypto.randomUUID(),
+    provisionerId: crypto.randomUUID(),
+    providerRunnerId: crypto.randomUUID(),
+    reservationId: null,
+    templateKey: 'linux',
+    labels: ['linux'],
+    state: 'running',
+    reason: null,
+    runnerSessionId: null,
+    providerKind: 'docker',
+    reportedAt: new Date(),
+    startedAt: null,
+    stoppingAt: null,
+    stoppedAt: null,
+    failedAt: null,
+    terminatedAt: null,
+    reservationReleasedAt: null,
+    createdAt: new Date(),
+    updatedAt: new Date(),
+  };
+});

package/test/factories/runner-session.ts

@@ -19,13 +19,15 @@ export const runnerSessionFactory = Factory.define<RunnerSession>(({onCreate}) =
     scope: 'workspace',
     registrationTokenId: crypto.randomUUID(),
     registrationTokenKind: 'manual',
+    runnerInstanceId: null,
     provisionerId: null,
-    provisionedRunnerId: null,
+    providerRunnerId: null,
     labels: ['linux', 'x64'],
     toolCapabilities: null,
     toolCapabilitiesReportedAt: null,
     maxClaims: null,
     claimsUsed: 0,
+    revokedAt: null,
     createdAt: new Date(),
     updatedAt: new Date(),
   };

package/test/fixtures/auth-inter-module.ts

@@ -0,0 +1,30 @@
+import {JOB_LEASE_TOKEN_AUDIENCE, RUNNER_SESSION_TOKEN_AUDIENCE} from '@shipfox/api-auth-dto';
+import type {AuthInterModuleClient} from '@shipfox/api-auth-dto/inter-module';
+import {jobLeaseTokenKey, runnerSessionTokenKey} from '@shipfox/node-auth-root-key';
+import {signHs256} from '@shipfox/node-jwt';
+
+const leaseSecret = jobLeaseTokenKey();
+const runnerSessionSecret = runnerSessionTokenKey();
+
+export const runnersTestAuthClient: AuthInterModuleClient = {
+  async mintRunnerSessionToken(claims) {
+    return {
+      token: await signHs256({
+        payload: claims,
+        secret: runnerSessionSecret,
+        expiresIn: '1h',
+        audience: RUNNER_SESSION_TOKEN_AUDIENCE,
+      }),
+    };
+  },
+  async mintJobLeaseToken(claims) {
+    return {
+      token: await signHs256({
+        payload: claims,
+        secret: leaseSecret,
+        expiresIn: '90m',
+        audience: JOB_LEASE_TOKEN_AUDIENCE,
+      }),
+    };
+  },
+};

package/test/globalSetup.ts

@@ -8,9 +8,13 @@ export async function setup() {
   createPostgresClient();
 
   await runMigrations(db(), migrationsPath, '__drizzle_migrations_runners');
-  await db().execute(sql`TRUNCATE runners_provisioned_runners, runners_reservations CASCADE`);
+  await db().execute(sql`TRUNCATE runners_provisioner_capability_snapshots CASCADE`);
+  await db().execute(sql`TRUNCATE runners_capacity_assignments CASCADE`);
+  await db().execute(sql`TRUNCATE runners_runner_instances, runners_reservations CASCADE`);
   await db().execute(sql`TRUNCATE runners_provisioner_tokens CASCADE`);
   await db().execute(sql`TRUNCATE runners_ephemeral_registration_tokens CASCADE`);
+  await db().execute(sql`TRUNCATE runners_runner_bootstrap_tokens CASCADE`);
+  await db().execute(sql`TRUNCATE runners_runner_control_sessions CASCADE`);
   await db().execute(sql`TRUNCATE runners_runner_sessions CASCADE`);
   await db().execute(sql`TRUNCATE runners_rate_limits CASCADE`);
   await db().execute(sql`TRUNCATE runners_manual_registration_tokens CASCADE`);

package/test/index.ts

@@ -7,10 +7,11 @@ export {
   manualRegistrationTokenFactory,
 } from './factories/manual-registration-token.js';
 export {pendingJobFactory} from './factories/pending-job.js';
-export {provisionedRunnerFactory} from './factories/provisioned-runner.js';
 export {
   type ProvisionerTokenFactoryTransientParams,
   provisionerTokenFactory,
 } from './factories/provisioner-token.js';
 export {reservationFactory} from './factories/reservation.js';
+export {providerRunnerFactory} from './factories/runner-instance.js';
 export {runnerSessionFactory} from './factories/runner-session.js';
+export {runnersTestAuthClient} from './fixtures/auth-inter-module.js';