@shipfox/api-runners 2.0.0

package/src/presentation/routes/report-provisioned-runners.test.ts

@@ -0,0 +1,249 @@
+import {
+  AUTH_LEASED_JOB,
+  AUTH_PROVISIONER_TOKEN,
+  AUTH_RUNNER_REGISTRATION_TOKEN,
+  AUTH_RUNNER_SESSION,
+  AUTH_USER,
+  setProvisionerContext,
+} from '@shipfox/api-auth-context';
+import {
+  type AuthMethod,
+  ClientError,
+  closeApp,
+  createApp,
+  extractBearerToken,
+} from '@shipfox/node-fastify';
+import {vi} from '@shipfox/vitest/vi';
+import {and, eq} from 'drizzle-orm';
+import type {FastifyInstance, FastifyRequest} from 'fastify';
+import {db} from '#db/db.js';
+import {provisionedRunners} from '#db/schema/provisioned-runners.js';
+import {runningJobExecutions} from '#db/schema/running-job-executions.js';
+import {provisionedRunnerTerminateIntentHonoredCount} from '#metrics/instance.js';
+import {provisionedRunnerFactory, runnerSessionFactory} from '#test/index.js';
+import {runnerRoutes} from './index.js';
+
+const VALID_PROVISIONER_TOKEN = 'valid-provisioner-token';
+
+const passthroughAuth = (name: string): AuthMethod => ({
+  name,
+  authenticate: () => Promise.resolve(),
+});
+
+describe('POST /provisioners/provisioned-runners/report', () => {
+  let app: FastifyInstance;
+  let workspaceId: string;
+  let provisionerTokenId: string;
+
+  const fakeProvisionerAuth: AuthMethod = {
+    name: AUTH_PROVISIONER_TOKEN,
+    authenticate: (request: FastifyRequest) => {
+      const rawToken = extractBearerToken(request.headers.authorization);
+      if (rawToken !== VALID_PROVISIONER_TOKEN) {
+        throw new ClientError('Invalid provisioner token', 'unauthorized', {status: 401});
+      }
+      setProvisionerContext(request, {workspaceId, provisionerTokenId});
+      return Promise.resolve();
+    },
+  };
+
+  beforeAll(async () => {
+    app = await createApp({
+      auth: [
+        passthroughAuth(AUTH_USER),
+        passthroughAuth(AUTH_RUNNER_REGISTRATION_TOKEN),
+        passthroughAuth(AUTH_RUNNER_SESSION),
+        passthroughAuth(AUTH_LEASED_JOB),
+        fakeProvisionerAuth,
+      ],
+      routes: runnerRoutes,
+      swagger: false,
+    });
+    await app.ready();
+  });
+
+  afterAll(async () => {
+    await closeApp();
+  });
+
+  beforeEach(() => {
+    workspaceId = crypto.randomUUID();
+    provisionerTokenId = crypto.randomUUID();
+  });
+
+  it('accepts provisioned runner lifecycle reports from provisioner auth', async () => {
+    const reportedAt = '2025-01-01T00:00:00.000Z';
+
+    const res = await app.inject({
+      method: 'POST',
+      url: '/provisioners/provisioned-runners/report',
+      headers: {authorization: `Bearer ${VALID_PROVISIONER_TOKEN}`},
+      payload: {
+        events: [
+          {
+            provisioned_runner_id: 'provisioned-runner-1',
+            labels: ['linux'],
+            state: 'starting',
+            reported_at: reportedAt,
+            provider_kind: 'docker',
+          },
+        ],
+      },
+    });
+
+    const rows = await db()
+      .select()
+      .from(provisionedRunners)
+      .where(
+        and(
+          eq(provisionedRunners.workspaceId, workspaceId),
+          eq(provisionedRunners.provisionerId, provisionerTokenId),
+        ),
+      );
+    expect(res.statusCode).toBe(200);
+    expect(res.json()).toEqual({accepted: 1, reservations_released: 0});
+    expect(rows[0]).toMatchObject({
+      workspaceId,
+      provisionerId: provisionerTokenId,
+      provisionedRunnerId: 'provisioned-runner-1',
+      state: 'starting',
+      labels: ['linux'],
+      providerKind: 'docker',
+    });
+    expect(rows[0]?.reportedAt.toISOString()).toBe(reportedAt);
+  });
+
+  it('returns 400 when the batch exceeds the DTO limit', async () => {
+    const event = {
+      provisioned_runner_id: 'provisioned-runner-1',
+      labels: ['linux'],
+      state: 'running',
+      reported_at: new Date().toISOString(),
+    };
+
+    const res = await app.inject({
+      method: 'POST',
+      url: '/provisioners/provisioned-runners/report',
+      headers: {authorization: `Bearer ${VALID_PROVISIONER_TOKEN}`},
+      payload: {
+        events: Array.from({length: 1001}, (_, index) => ({
+          ...event,
+          provisioned_runner_id: `provisioned-runner-${index}`,
+        })),
+      },
+    });
+
+    expect(res.statusCode).toBe(400);
+  });
+
+  it('increments the honored terminate-intent metric once for a terminated report', async () => {
+    const honoredSpy = vi.spyOn(provisionedRunnerTerminateIntentHonoredCount, 'add');
+    const runnerSession = await runnerSessionFactory.create({workspaceId});
+    await provisionedRunnerFactory.create({
+      workspaceId,
+      provisionerId: provisionerTokenId,
+      provisionedRunnerId: 'provisioned-runner-1',
+      state: 'running',
+    });
+    await db()
+      .insert(runningJobExecutions)
+      .values({
+        workspaceId,
+        workflowRunId: crypto.randomUUID(),
+        workflowRunAttemptId: crypto.randomUUID(),
+        jobId: crypto.randomUUID(),
+        jobExecutionId: crypto.randomUUID(),
+        projectId: crypto.randomUUID(),
+        runnerSessionId: runnerSession.id,
+        provisionerId: provisionerTokenId,
+        provisionedRunnerId: 'provisioned-runner-1',
+        requiredLabels: ['linux'],
+        runnerLabels: ['linux'],
+        startedAt: new Date('2025-01-01T00:00:00.000Z'),
+        lastHeartbeatAt: new Date('2025-01-01T00:00:00.000Z'),
+        cancellationRequestedAt: new Date('2025-01-01T00:01:00.000Z'),
+      });
+    const honoredCallsBefore = honoredSpy.mock.calls.length;
+
+    const first = await app.inject({
+      method: 'POST',
+      url: '/provisioners/provisioned-runners/report',
+      headers: {authorization: `Bearer ${VALID_PROVISIONER_TOKEN}`},
+      payload: {
+        events: [
+          {
+            provisioned_runner_id: 'provisioned-runner-1',
+            labels: ['linux'],
+            state: 'terminated',
+            reported_at: new Date().toISOString(),
+          },
+        ],
+      },
+    });
+    const second = await app.inject({
+      method: 'POST',
+      url: '/provisioners/provisioned-runners/report',
+      headers: {authorization: `Bearer ${VALID_PROVISIONER_TOKEN}`},
+      payload: {
+        events: [
+          {
+            provisioned_runner_id: 'provisioned-runner-1',
+            labels: ['linux'],
+            state: 'terminated',
+            reported_at: new Date().toISOString(),
+          },
+        ],
+      },
+    });
+
+    expect(first.statusCode).toBe(200);
+    expect(second.statusCode).toBe(200);
+    const honoredCalls = honoredSpy.mock.calls
+      .slice(honoredCallsBefore)
+      .filter(
+        ([value, attributes]) =>
+          value === 1 && JSON.stringify(attributes) === JSON.stringify({reason: 'job-cancelled'}),
+      );
+    expect(honoredCalls).toHaveLength(1);
+  });
+
+  it('returns 400 for provider-sensitive extra fields', async () => {
+    const res = await app.inject({
+      method: 'POST',
+      url: '/provisioners/provisioned-runners/report',
+      headers: {authorization: `Bearer ${VALID_PROVISIONER_TOKEN}`},
+      payload: {
+        events: [
+          {
+            provisioned_runner_id: 'provisioned-runner-1',
+            labels: ['linux'],
+            state: 'running',
+            reported_at: new Date().toISOString(),
+            hostname: 'worker-1',
+          },
+        ],
+      },
+    });
+
+    expect(res.statusCode).toBe(400);
+  });
+
+  it('returns 401 without provisioner auth', async () => {
+    const res = await app.inject({
+      method: 'POST',
+      url: '/provisioners/provisioned-runners/report',
+      payload: {
+        events: [
+          {
+            provisioned_runner_id: 'provisioned-runner-1',
+            labels: ['linux'],
+            state: 'running',
+            reported_at: new Date().toISOString(),
+          },
+        ],
+      },
+    });
+
+    expect(res.statusCode).toBe(401);
+  });
+});

package/src/presentation/routes/report-provisioned-runners.ts

@@ -0,0 +1,40 @@
+import {requireProvisionerContext} from '@shipfox/api-auth-context';
+import {
+  reportProvisionedRunnersBodySchema,
+  reportProvisionedRunnersResponseSchema,
+} from '@shipfox/api-runners-dto';
+import {defineRoute} from '@shipfox/node-fastify';
+import {reportProvisionedRunners} from '#core/index.js';
+import {toReportProvisionedRunnersResponseDto} from '#presentation/dto/index.js';
+
+export const reportProvisionedRunnersRoute = defineRoute({
+  method: 'POST',
+  path: '/provisioned-runners/report',
+  description: 'Report provisioned runner lifecycle state',
+  schema: {
+    body: reportProvisionedRunnersBodySchema,
+    response: {
+      200: reportProvisionedRunnersResponseSchema,
+    },
+  },
+  handler: async (request) => {
+    const {provisionerTokenId, workspaceId} = requireProvisionerContext(request);
+    const result = await reportProvisionedRunners({
+      workspaceId,
+      provisionerId: provisionerTokenId,
+      events: request.body.events.map((event) => ({
+        provisionedRunnerId: event.provisioned_runner_id,
+        reservationId: event.reservation_id ?? null,
+        templateKey: event.template_key ?? null,
+        labels: event.labels,
+        state: event.state,
+        reason: event.reason ?? null,
+        runnerSessionId: event.runner_session_id ?? null,
+        providerKind: event.provider_kind ?? null,
+        reportedAt: new Date(event.reported_at),
+      })),
+    });
+
+    return toReportProvisionedRunnersResponseDto(result);
+  },
+});

package/src/presentation/routes/request-job.test.ts

@@ -0,0 +1,222 @@
+import {
+  createLeaseTokenAuthMethod,
+  createRunnerSessionAuthMethod,
+  verifyJobLeaseToken,
+} from '@shipfox/api-auth';
+import {AUTH_PROVISIONER_TOKEN, AUTH_USER} from '@shipfox/api-auth-context';
+import {RUNNER_SESSION_TOKEN_AUDIENCE} from '@shipfox/api-auth-dto';
+import type {AuthMethod} from '@shipfox/node-fastify';
+import {closeApp, createApp} from '@shipfox/node-fastify';
+import {signHs256} from '@shipfox/node-jwt';
+import {generateOpaqueToken} from '@shipfox/node-tokens';
+import type {FastifyInstance} from 'fastify';
+import {createRunnerRegistrationTokenAuthMethod} from '#presentation/auth/index.js';
+import {
+  ephemeralRegistrationTokenFactory,
+  manualRegistrationTokenFactory,
+  pendingJobFactory,
+} from '#test/index.js';
+import {runnerRoutes} from './index.js';
+
+const fakeUserAuth: AuthMethod = {
+  name: AUTH_USER,
+  authenticate: () => Promise.resolve(),
+};
+
+const fakeProvisionerAuth: AuthMethod = {
+  name: AUTH_PROVISIONER_TOKEN,
+  authenticate: () => Promise.resolve(),
+};
+
+describe('POST /runners/jobs/request', () => {
+  let app: FastifyInstance;
+  let rawToken: string;
+  let workspaceId: string;
+  let sessionToken: string;
+  let runnerSessionId: string;
+
+  beforeAll(async () => {
+    app = await createApp({
+      auth: [
+        fakeUserAuth,
+        createRunnerRegistrationTokenAuthMethod(),
+        createRunnerSessionAuthMethod(),
+        createLeaseTokenAuthMethod(),
+        fakeProvisionerAuth,
+      ],
+      routes: runnerRoutes,
+      swagger: false,
+    });
+    await app.ready();
+  });
+
+  afterAll(async () => {
+    await closeApp();
+  });
+
+  beforeEach(async () => {
+    rawToken = `sf_mrt_${crypto.randomUUID()}`;
+    workspaceId = crypto.randomUUID();
+    await manualRegistrationTokenFactory.create({workspaceId}, {transient: {rawToken}});
+    const registered = await registerSession(rawToken);
+    sessionToken = registered.sessionToken;
+    runnerSessionId = registered.runnerSessionId;
+  });
+
+  async function registerSession(
+    token: string,
+  ): Promise<{sessionToken: string; runnerSessionId: string}> {
+    const res = await app.inject({
+      method: 'POST',
+      url: '/runners/register',
+      headers: {authorization: `Bearer ${token}`},
+      payload: {labels: ['Linux', 'x64']},
+    });
+    expect(res.statusCode).toBe(200);
+    const body = res.json();
+    return {sessionToken: body.session_token, runnerSessionId: body.session_id};
+  }
+
+  it('returns 401 without authorization', async () => {
+    const res = await app.inject({
+      method: 'POST',
+      url: '/runners/jobs/request',
+    });
+
+    expect(res.statusCode).toBe(401);
+  });
+
+  it('returns 401 with an invalid runner session token', async () => {
+    const res = await app.inject({
+      method: 'POST',
+      url: '/runners/jobs/request',
+      headers: {authorization: 'Bearer invalid'},
+    });
+
+    expect(res.statusCode).toBe(401);
+  });
+
+  it('returns 200 with the job ids and a verifiable lease token when a job is available', async () => {
+    const created = await pendingJobFactory.create({workspaceId});
+
+    const res = await app.inject({
+      method: 'POST',
+      url: '/runners/jobs/request',
+      headers: {authorization: `Bearer ${sessionToken}`},
+    });
+
+    expect(res.statusCode).toBe(200);
+    const body = res.json();
+    expect(body.job_id).toBe(created.jobId);
+    expect(body.workflow_run_id).toBe(created.workflowRunId);
+    expect(body.workflow_run_attempt_id).toBe(created.workflowRunAttemptId);
+    expect(typeof body.lease_token).toBe('string');
+    expect(body.job_name).toBeUndefined();
+    expect(body.steps).toBeUndefined();
+
+    const claims = await verifyJobLeaseToken(body.lease_token);
+    expect(claims).toMatchObject({
+      jobId: created.jobId,
+      workflowRunId: created.workflowRunId,
+      workflowRunAttemptId: created.workflowRunAttemptId,
+      projectId: created.projectId,
+      workspaceId,
+      runnerSessionId,
+    });
+  });
+
+  it('returns 204 when no jobs are available for the session workspace', async () => {
+    await pendingJobFactory.create({workspaceId: crypto.randomUUID()});
+
+    const res = await app.inject({
+      method: 'POST',
+      url: '/runners/jobs/request',
+      headers: {authorization: `Bearer ${sessionToken}`},
+    });
+
+    expect(res.statusCode).toBe(204);
+  });
+
+  it('returns 204 when no pending job matches the session labels', async () => {
+    await pendingJobFactory.create({workspaceId, requiredLabels: ['macos']});
+
+    const res = await app.inject({
+      method: 'POST',
+      url: '/runners/jobs/request',
+      headers: {authorization: `Bearer ${sessionToken}`},
+    });
+
+    expect(res.statusCode).toBe(204);
+  });
+
+  it('claims multiple jobs from one manual session', async () => {
+    const first = await pendingJobFactory.create({workspaceId});
+    const second = await pendingJobFactory.create({workspaceId});
+
+    const firstRes = await app.inject({
+      method: 'POST',
+      url: '/runners/jobs/request',
+      headers: {authorization: `Bearer ${sessionToken}`},
+    });
+    const secondRes = await app.inject({
+      method: 'POST',
+      url: '/runners/jobs/request',
+      headers: {authorization: `Bearer ${sessionToken}`},
+    });
+
+    expect(firstRes.statusCode).toBe(200);
+    expect(firstRes.json().job_id).toBe(first.jobId);
+    expect(secondRes.statusCode).toBe(200);
+    expect(secondRes.json().job_id).toBe(second.jobId);
+  });
+
+  it('returns 409 after an ephemeral session claims one job', async () => {
+    const ephemeralRawToken = generateOpaqueToken('ephemeralRegistrationToken');
+    await ephemeralRegistrationTokenFactory.create(
+      {workspaceId},
+      {transient: {rawToken: ephemeralRawToken}},
+    );
+    const registered = await registerSession(ephemeralRawToken);
+    const created = await pendingJobFactory.create({workspaceId});
+    await pendingJobFactory.create({workspaceId});
+
+    const firstRes = await app.inject({
+      method: 'POST',
+      url: '/runners/jobs/request',
+      headers: {authorization: `Bearer ${registered.sessionToken}`},
+    });
+    const secondRes = await app.inject({
+      method: 'POST',
+      url: '/runners/jobs/request',
+      headers: {authorization: `Bearer ${registered.sessionToken}`},
+    });
+
+    expect(firstRes.statusCode).toBe(200);
+    expect(firstRes.json().job_id).toBe(created.jobId);
+    expect(secondRes.statusCode).toBe(409);
+    expect(secondRes.json().code).toBe('runner-session-exhausted');
+  });
+
+  it('returns 401 when the runner session token is expired', async () => {
+    const expiredSessionToken = await signHs256({
+      payload: {
+        runnerSessionId,
+        workspaceId,
+        scope: 'workspace',
+        labels: ['linux', 'x64'],
+        maxClaims: null,
+      },
+      secret: process.env.AUTH_RUNNER_SESSION_TOKEN_SECRET ?? 'test-runner-session-secret',
+      expiresIn: '-1s',
+      audience: RUNNER_SESSION_TOKEN_AUDIENCE,
+    });
+
+    const res = await app.inject({
+      method: 'POST',
+      url: '/runners/jobs/request',
+      headers: {authorization: `Bearer ${expiredSessionToken}`},
+    });
+
+    expect(res.statusCode).toBe(401);
+  });
+});

package/src/presentation/routes/request-job.ts

@@ -0,0 +1,47 @@
+import {requireRunnerSessionContext} from '@shipfox/api-auth-context';
+import {claimedJobResponseSchema} from '@shipfox/api-runners-dto';
+import {ClientError, defineRoute} from '@shipfox/node-fastify';
+import {RunnerSessionExhaustedError} from '#core/errors.js';
+import {claimJobExecution} from '#core/job-executions.js';
+
+export const requestJobRoute = defineRoute({
+  method: 'POST',
+  path: '/request',
+  description: 'Claim the next available job execution and receive its lease token',
+  schema: {
+    response: {
+      200: claimedJobResponseSchema,
+    },
+  },
+  errorHandler: (error) => {
+    if (error instanceof RunnerSessionExhaustedError) {
+      throw new ClientError('Runner session claim limit exhausted', 'runner-session-exhausted', {
+        status: 409,
+      });
+    }
+    throw error;
+  },
+  handler: async (_request, reply) => {
+    const runner = requireRunnerSessionContext(_request);
+
+    const jobExecution = await claimJobExecution({
+      workspaceId: runner.workspaceId,
+      runnerSessionId: runner.runnerSessionId,
+      sessionLabels: runner.labels,
+      maxClaims: runner.maxClaims,
+    });
+
+    if (!jobExecution) {
+      reply.status(204);
+      return;
+    }
+
+    return {
+      workflow_run_id: jobExecution.workflowRunId,
+      workflow_run_attempt_id: jobExecution.workflowRunAttemptId,
+      job_id: jobExecution.jobId,
+      job_execution_id: jobExecution.jobExecutionId,
+      lease_token: jobExecution.leaseToken,
+    };
+  },
+});

package/src/presentation/routes/revoke-manual-registration-token.ts

@@ -0,0 +1,35 @@
+import {revokeManualRegistrationTokenResponseSchema} from '@shipfox/api-runners-dto';
+import {ClientError, defineRoute} from '@shipfox/node-fastify';
+import {z} from 'zod';
+import {
+  ManualRegistrationTokenNotFoundError,
+  revokeWorkspaceManualRegistrationToken,
+} from '#core/index.js';
+import {toManualRegistrationTokenDto} from '#presentation/dto/index.js';
+import {requireManualRegistrationTokenWorkspaceMembership} from './workspace-membership.js';
+
+export const revokeManualRegistrationTokenRoute = defineRoute({
+  method: 'POST',
+  path: '/:tokenId/revoke',
+  description: 'Stop a manual registration token from being used to connect to your account',
+  schema: {
+    params: z.object({workspaceId: z.string().uuid(), tokenId: z.string().uuid()}),
+    response: {
+      200: revokeManualRegistrationTokenResponseSchema,
+    },
+  },
+  errorHandler: (error) => {
+    if (error instanceof ManualRegistrationTokenNotFoundError) {
+      throw new ClientError('Manual registration token not found', 'not-found', {status: 404});
+    }
+    throw error;
+  },
+  handler: async (request) => {
+    const {workspaceId, tokenId} = request.params;
+    requireManualRegistrationTokenWorkspaceMembership({request, workspaceId});
+
+    const revoked = await revokeWorkspaceManualRegistrationToken({tokenId, workspaceId});
+
+    return toManualRegistrationTokenDto(revoked);
+  },
+});

package/src/presentation/routes/revoke-provisioner-token.ts

@@ -0,0 +1,37 @@
+import {requireUserContext, requireWorkspaceAccess} from '@shipfox/api-auth-context';
+import {revokeProvisionerTokenResponseSchema} from '@shipfox/api-runners-dto';
+import {ClientError, defineRoute} from '@shipfox/node-fastify';
+import {z} from 'zod';
+import {ProvisionerTokenNotFoundError, revokeWorkspaceProvisionerToken} from '#core/index.js';
+import {toProvisionerTokenDto} from '#presentation/dto/index.js';
+
+export const revokeProvisionerTokenRoute = defineRoute({
+  method: 'POST',
+  path: '/:tokenId/revoke',
+  description: 'Stop a provisioner token from being used to connect to your account',
+  schema: {
+    params: z.object({workspaceId: z.string().uuid(), tokenId: z.string().uuid()}),
+    response: {
+      200: revokeProvisionerTokenResponseSchema,
+    },
+  },
+  errorHandler: (error) => {
+    if (error instanceof ProvisionerTokenNotFoundError) {
+      throw new ClientError('Provisioner token not found', 'not-found', {status: 404});
+    }
+    throw error;
+  },
+  handler: async (request) => {
+    const {workspaceId, tokenId} = request.params;
+    requireWorkspaceAccess({request, workspaceId});
+    const user = requireUserContext(request);
+
+    const revoked = await revokeWorkspaceProvisionerToken({
+      tokenId,
+      workspaceId,
+      revokedByUserId: user.userId,
+    });
+
+    return toProvisionerTokenDto(revoked);
+  },
+});

package/src/presentation/routes/workspace-membership.ts

@@ -0,0 +1,9 @@
+import {requireWorkspaceAccess} from '@shipfox/api-auth-context';
+import type {FastifyRequest} from 'fastify';
+
+export function requireManualRegistrationTokenWorkspaceMembership(params: {
+  request: FastifyRequest;
+  workspaceId: string;
+}): void {
+  requireWorkspaceAccess(params);
+}