@takaro/http 0.0.0-next.0da151e

package/src/controllers/meta.ts

@@ -0,0 +1,268 @@
+import { Controller, Get, getMetadataArgsStorage, Res } from 'routing-controllers';
+import { Response } from 'express';
+import { validationMetadatasToSchemas } from 'class-validator-jsonschema';
+import { routingControllersToSpec, ResponseSchema } from 'routing-controllers-openapi';
+import { IsBoolean } from 'class-validator';
+import { getMetrics, health } from '@takaro/util';
+import { OpenAPIObject } from 'openapi3-ts';
+import { PERMISSIONS } from '@takaro/auth';
+import { EventMapping } from '@takaro/modules';
+import { randomUUID } from 'crypto';
+import dedent from 'dedent';
+
+let spec: OpenAPIObject | undefined;
+
+export class HealthOutputDTO {
+  @IsBoolean()
+  healthy!: boolean;
+}
+
+function addSearchExamples(original: OpenAPIObject): OpenAPIObject {
+  // Copy the spec so we don't mutate the original
+  const spec = JSON.parse(JSON.stringify(original));
+
+  // Add some examples and info to all the POST /search endpoints
+  Object.keys(spec.paths).forEach((pathKey) => {
+    const pathItem = spec?.paths[pathKey];
+    Object.keys(pathItem).forEach((method) => {
+      const operation = pathItem[method];
+      if (method === 'post' && pathKey.endsWith('/search')) {
+        const standardExamples = {
+          list: {
+            summary: 'List all',
+            value: {},
+          },
+          advanced: {
+            summary: 'Advanced search',
+            description: dedent`All /search endpoints allow you to combine different filters, search terms, ranges, and extend options.
+            Filters are exact matches, search terms are partial matches, ranges are greater than or less than comparisons, 
+            and extend allows you to include related entities in the response.
+            
+            Ranges allow you to make queries like "all records created in the last 7 days" or "all records with an age greater than 18".
+
+            In search and filter sections, you pass an array of values for each property 
+            These values are OR'ed together. So we'll get 2 records back in this case, if the IDs exist.
+
+            Eg: \`{"filters": {"id": ["${randomUUID()}", "${randomUUID()}"]}}\`
+
+            Different filters will be AND'ed together.
+            This will return all records where the name is John and the age is 19.
+
+            Eg: \`{"filters": {"name": "John", "age": 19}}\`
+            
+            The extend parameter allows including related data:
+            Eg: \`{"extend": ["roles", "gameServers"]}\`
+            `,
+            value: {
+              filters: {
+                id: ['ea85ddf4-2885-482f-adc6-548fbe3fd8af'],
+              },
+              greaterThan: { createdAt: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000).toISOString() },
+            },
+          },
+        };
+
+        if (!operation.requestBody) {
+          operation.requestBody = {
+            content: {
+              'application/json': {
+                examples: standardExamples,
+              },
+            },
+          };
+        }
+
+        operation.requestBody.content['application/json'].examples = {
+          ...standardExamples,
+          ...operation.requestBody.content['application/json'].examples,
+        };
+      }
+    });
+  });
+
+  return spec;
+}
+
+@Controller()
+export class Meta {
+  @Get('/healthz')
+  @ResponseSchema(HealthOutputDTO)
+  async getHealth() {
+    return { healthy: true };
+  }
+
+  @Get('/readyz')
+  @ResponseSchema(HealthOutputDTO)
+  async getReadiness() {
+    const healthy = await health.check();
+    return { healthy };
+  }
+
+  @Get('/openapi.json')
+  async getOpenApi() {
+    if (spec) return spec;
+
+    const { getMetadataStorage } = await import('class-validator');
+    const classTransformerStorage = await import(
+      // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+      // @ts-ignore were doing an import of internal code and ts doesnt like that
+      // But this does work, trust me bro...
+      'class-transformer/cjs/storage.js'
+    );
+
+    const metadataArgsStorage = getMetadataArgsStorage();
+    const metadataStorage = getMetadataStorage();
+    const schemas = validationMetadatasToSchemas({
+      refPointerPrefix: '#/components/schemas/',
+      classTransformerMetadataStorage: classTransformerStorage.defaultMetadataStorage,
+      classValidatorMetadataStorage: metadataStorage,
+      forbidNonWhitelisted: true,
+    });
+
+    spec = routingControllersToSpec(
+      metadataArgsStorage,
+      {},
+      {
+        info: {
+          title: `Takaro ${process.env.PACKAGE || 'API'}`,
+          version: `${process.env.TAKARO_VERSION} - ${process.env.TAKARO_COMMIT} `,
+          contact: {
+            name: 'Takaro Team',
+            email: 'support@takaro.io',
+            url: 'https://takaro.io',
+          },
+        },
+        components: {
+          schemas,
+          securitySchemes: {
+            adminAuth: {
+              description: 'Used for system administration, like creating or deleting domains',
+              type: 'apiKey',
+              in: 'header',
+              name: 'x-takaro-admin-token',
+            },
+            domainAuth: {
+              description: 'Used for anything inside a domain. Players, GameServers, etc.',
+              type: 'apiKey',
+              in: 'cookie',
+              name: 'takaro-token',
+            },
+          },
+        },
+      },
+    );
+
+    // Add required permissions to operation descriptions
+
+    const requiredPermsRegex = /authMiddleware\((.+)\)/;
+
+    metadataArgsStorage.uses.forEach((use) => {
+      const requiredPerms =
+        use.middleware.name
+          .match(requiredPermsRegex)?.[1]
+          .split(',')
+          .map((p) => `\`${p}\``)
+          .join(', ') || [];
+
+      const operationId = `${use.target.name}.${use.method}`;
+
+      if (!requiredPerms.length) return;
+
+      // Find the corresponding path and method in spec
+      Object.keys(spec?.paths ?? []).forEach((pathKey) => {
+        const pathItem = spec?.paths[pathKey];
+        Object.keys(pathItem).forEach((method) => {
+          const operation = pathItem[method];
+          if (operation.operationId === operationId) {
+            // Update the description with required permissions
+            operation.description = (operation.description || '') + `\n\n Required permissions: ${requiredPerms}`;
+          }
+        });
+      });
+    });
+
+    // Add the operationId to the description, this helps users find the corresponding function call in the API client.
+    Object.keys(spec.paths).forEach((pathKey) => {
+      const pathItem = spec?.paths[pathKey];
+      Object.keys(pathItem).forEach((method) => {
+        const operation = pathItem[method];
+        // Api client exposes it as roleControllerSearch
+        // Current value is RoleController.search so lets adjust
+        // Capitalize the part after . and remove the .
+        const split = operation.operationId.split('.');
+        const cleanOperationId = split[0] + split[1].charAt(0).toUpperCase() + split[1].slice(1);
+        operation.description = (operation.description || '') + `<br> OperationId: \`${cleanOperationId}\``;
+      });
+    });
+
+    if (spec.components?.schemas) {
+      spec.components.schemas.PERMISSIONS = {
+        enum: Object.values(PERMISSIONS),
+      };
+    }
+
+    // Force event meta to be the correct types
+    // TODO: figure out how to do this 'properly' with class-validator
+    const allEvents = Object.values(EventMapping).map((e) => e.name);
+
+    const eventOutputMetaSchema = spec.components?.schemas?.EventOutputDTO;
+    if (eventOutputMetaSchema && 'properties' in eventOutputMetaSchema && eventOutputMetaSchema.properties) {
+      eventOutputMetaSchema.properties.meta = {
+        oneOf: [...allEvents.map((e) => ({ $ref: `#/components/schemas/${e}` }))],
+      };
+    }
+
+    spec = addSearchExamples(spec);
+    return spec;
+  }
+
+  @Get('/')
+  getRoot(@Res() res: Response) {
+    return res.redirect('/api.html');
+  }
+
+  @Get('/api.html')
+  getOpenApiHtml() {
+    return `<!DOCTYPE html>
+    <html>
+      <head>
+        <meta charset="utf-8" />
+        <script
+          type="module"
+          src="https://cdn.jsdelivr.net/npm/rapidoc@9.3.8"
+        ></script>
+      </head>
+      <body>
+        <rapi-doc
+          spec-url="/openapi.json"
+          render-style="read"
+          fill-request-fields-with-example="false"
+          persist-auth="true"
+
+          sort-tags="true"
+          sort-endpoints-by="method"
+
+          show-method-in-nav-bar="as-colored-block"
+          show-header="false"
+          allow-authentication="true"
+          allow-server-selection="false"
+
+          schema-style="table"
+          schema-expand-level="1"
+          default-schema-tab="schema"
+
+          primary-color="#664de5"
+          bg-color="#151515"
+          text-color="#c2c2c2"
+          header-color="#353535"
+        />
+      </body>
+    </html>
+    `;
+  }
+
+  @Get('/metrics')
+  getMetrics() {
+    return getMetrics();
+  }
+}

package/src/main.ts

@@ -0,0 +1,8 @@
+export { HTTP } from './app.js';
+
+export * from './middleware/metrics.js';
+export { createRateLimitMiddleware } from './middleware/rateLimit.js';
+export { getAdminBasicAuth } from './middleware/basicAuth.js';
+export { paginationMiddleware } from './middleware/paginationMiddleware.js';
+export { apiResponse, APIOutput } from './util/apiResponse.js';
+export { ErrorHandler } from './middleware/errorHandler.js';

package/src/middleware/__tests__/paginationMiddleware.unit.test.ts

@@ -0,0 +1,49 @@
+import { sandbox, expect } from '@takaro/test';
+import { NextFunction, Request, Response } from 'express';
+import { errors } from '@takaro/util';
+import { paginationMiddleware } from '../paginationMiddleware.js';
+import { describe, it } from 'node:test';
+
+async function runPagination(page?: number, limit?: number) {
+  const req = { query: { page, limit } } as unknown as Request;
+  const res = { locals: {} } as Response;
+  const next = sandbox.stub<errors.ValidationError[]>();
+  await paginationMiddleware(req, res, next as unknown as NextFunction);
+  return { req, res, next };
+}
+
+describe('pagination middleware', () => {
+  it('Handles setting defaults', async () => {
+    const { res, next } = await runPagination();
+    expect(res.locals.page).to.equal(0);
+    expect(res.locals.limit).to.equal(100);
+    expect(next).to.have.been.calledOnce;
+  });
+  it('Works when pagination is passed', async () => {
+    const { res, next } = await runPagination(2, 20);
+    expect(res.locals.page).to.equal(2);
+    expect(res.locals.limit).to.equal(20);
+    expect(next).to.have.been.calledOnce;
+  });
+  it('Handles negative limit', async () => {
+    const { next } = await runPagination(1, -1);
+    expect(next).to.have.been.calledOnce;
+    const callArg = next.getCalls()[0].args[0];
+    expect(callArg).to.be.an.instanceOf(Error);
+    expect(callArg.message).to.equal('Invalid pagination: limit must be greater than or equal to 1');
+  });
+  it('Handles negative page', async () => {
+    const { next } = await runPagination(-1, 5);
+    expect(next).to.have.been.calledOnce;
+    const callArg = next.getCalls()[0].args[0];
+    expect(callArg).to.be.an.instanceOf(Error);
+    expect(callArg.message).to.equal('Invalid pagination: page must be greater than or equal to 0');
+  });
+  it('Handles limit too high', async () => {
+    const { next } = await runPagination(1, 1001);
+    expect(next).to.have.been.calledOnce;
+    const callArg = next.getCalls()[0].args[0];
+    expect(callArg).to.be.an.instanceOf(Error);
+    expect(callArg.message).to.equal('Invalid pagination: limit must be less than or equal to 1000');
+  });
+});

package/src/middleware/__tests__/rateLimit.integration.test.ts

@@ -0,0 +1,130 @@
+import { Response, NextFunction, Request } from 'express';
+import { Redis } from '@takaro/db';
+import { Controller, UseBefore, Get } from 'routing-controllers';
+import { HTTP } from '../../main.js';
+import { createRateLimitMiddleware } from '../rateLimit.js';
+import { ctx } from '@takaro/util';
+import supertest from 'supertest';
+import { describe, beforeEach, afterEach, after, it } from 'node:test';
+
+describe('rateLimit middleware', () => {
+  let http: HTTP;
+  beforeEach(async () => {
+    @Controller()
+    class TestController {
+      @Get('/low-limit')
+      @UseBefore(
+        await createRateLimitMiddleware({
+          max: 5,
+          windowSeconds: 5,
+          useInMemory: false,
+        }),
+      )
+      getLow() {
+        return 'Hello World';
+      }
+
+      @Get('/high-limit')
+      @UseBefore(
+        await createRateLimitMiddleware({
+          max: 15,
+          windowSeconds: 5,
+          useInMemory: false,
+        }),
+      )
+      getHigh() {
+        return 'Hello World';
+      }
+
+      @Get('/authenticated')
+      @UseBefore(
+        (req: Request, _res: Response, next: NextFunction) => {
+          ctx.addData({ user: req.query.user as string });
+          next();
+        },
+        await createRateLimitMiddleware({ max: 5, windowSeconds: 5, useInMemory: false }),
+      )
+      getAuthenticated() {
+        return 'Hello World';
+      }
+    }
+
+    http = new HTTP(
+      {
+        controllers: [TestController],
+      },
+      { port: undefined },
+    );
+
+    await http.start();
+  });
+
+  afterEach(async () => {
+    await http.stop();
+
+    const redis = await Redis.getClient('http:rateLimit');
+    await redis.flushAll();
+  });
+
+  after(async () => {
+    await Redis.destroy();
+  });
+
+  it('should limit requests', async () => {
+    const agent = supertest(http.expressInstance);
+    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+    // @ts-ignore
+    agent.get('/low-limit').set('X-Forwarded-For', '127.0.0.2');
+
+    for (let i = 0; i < 4; i++) {
+      // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+      // @ts-ignore
+      await agent.get('/low-limit').expect(200);
+    }
+    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+    // @ts-ignore
+    await agent.get('/low-limit').expect(429);
+  });
+
+  it('should apply distinct limits per user', async () => {
+    const agent = supertest(http.expressInstance);
+
+    for (let i = 0; i < 4; i++) {
+      // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+      // @ts-ignore
+      await agent.get('/authenticated?user=1').expect(200);
+    }
+    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+    // @ts-ignore
+    await agent.get('/authenticated?user=1').expect(429);
+
+    for (let i = 0; i < 4; i++) {
+      // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+      // @ts-ignore
+      await agent.get('/authenticated?user=2').expect(200);
+    }
+    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+    // @ts-ignore
+    await agent.get('/authenticated?user=2').expect(429);
+  });
+
+  it('Should accurately report metadata info via HTTP headers', async () => {
+    const agent = supertest(http.expressInstance);
+
+    for (let i = 1; i < 5; i++) {
+      await agent
+        .get('/low-limit')
+        .expect(200)
+        .expect('x-ratelimit-remaining', (5 - i).toString())
+        .expect('x-ratelimit-limit', '5')
+        .expect('x-ratelimit-reset', /\d+/);
+    }
+
+    agent
+      .get('/low-limit')
+      .expect(429)
+      .expect('x-ratelimit-remaining', '0')
+      .expect('x-ratelimit-limit', '5')
+      .expect('x-ratelimit-reset', /\d+/);
+  });
+});

package/src/middleware/basicAuth.ts

@@ -0,0 +1,34 @@
+import { Request, Response, NextFunction } from 'express';
+
+export function getAdminBasicAuth(secret: string) {
+  return function adminBasicAuth(req: Request, res: Response, next: NextFunction) {
+    const auth = req.headers.authorization;
+
+    // If no auth header is present, prompt for credentials
+    if (!auth) {
+      res.set('WWW-Authenticate', 'Basic realm="Admin Access"');
+      res.status(401).send('Authentication required');
+      return;
+    }
+
+    const [type, credentials] = auth.split(' ');
+
+    // Verify auth type is Basic
+    if (type !== 'Basic') {
+      res.set('WWW-Authenticate', 'Basic realm="Admin Access"');
+      res.status(401).send('Invalid authentication type');
+      return;
+    }
+
+    // Decode and verify credentials
+    const [username, password] = Buffer.from(credentials, 'base64').toString().split(':');
+
+    if (username !== 'admin' || password !== secret) {
+      res.set('WWW-Authenticate', 'Basic realm="Admin Access"');
+      res.status(401).send('Invalid credentials');
+      return;
+    }
+
+    next();
+  };
+}

package/src/middleware/errorHandler.ts

@@ -0,0 +1,95 @@
+import { Request, Response, NextFunction } from 'express';
+import { HttpError } from 'routing-controllers';
+import { logger, errors } from '@takaro/util';
+import { apiResponse } from '../util/apiResponse.js';
+import { ValidationError } from 'class-validator';
+import { MulterError } from 'multer';
+
+const log = logger('errorHandler');
+
+export async function ErrorHandler(
+  originalError: Error,
+  req: Request,
+  res: Response,
+
+  _next: NextFunction,
+) {
+  let status = 500;
+  let parsedError = new errors.InternalServerError();
+
+  if (originalError.name === 'BadRequestError') {
+    if (Object.prototype.hasOwnProperty.call(originalError, 'errors')) {
+      // @ts-expect-error Error typing is weird in ts... but we validate during runtime so should be OK
+      const validationErrors = originalError['errors'] as ValidationError[];
+      parsedError = new errors.ValidationError('Validation error', validationErrors);
+      log.warn('⚠️ Validation errror', {
+        details: validationErrors.map((e) => JSON.stringify(e.constraints, null, 2)),
+      });
+    }
+  }
+
+  if (originalError instanceof MulterError) {
+    status = 400;
+    parsedError = new errors.BadRequestError('Invalid file');
+    if (originalError.code === 'LIMIT_FIELD_VALUE') {
+      status = 400;
+      parsedError = new errors.BadRequestError('File too large');
+    }
+  }
+
+  if (originalError instanceof HttpError) {
+    status = originalError.httpCode;
+  }
+
+  if (originalError.name === 'UniqueViolationError') {
+    status = 409;
+    parsedError = new errors.ConflictError('Unique constraint violation');
+  }
+
+  if (originalError.name === 'NotNullViolationError') {
+    status = 400;
+    parsedError = new errors.BadRequestError('Missing required field');
+  }
+
+  if (originalError.name === 'CheckViolationError') {
+    status = 400;
+    parsedError = new errors.BadRequestError('Invalid data provided');
+  }
+
+  if (originalError.name === 'DataError' && originalError.message.includes('invalid input syntax for type uuid')) {
+    status = 400;
+    parsedError = new errors.BadRequestError('Invalid UUID. Passed a string instead of a UUID');
+  }
+
+  if ('constraint' in originalError && originalError['constraint'] === 'currency_positive') {
+    status = 400;
+    parsedError = new errors.BadRequestError('Not enough currency');
+  }
+
+  if (originalError instanceof errors.TakaroError) {
+    status = originalError.http;
+    parsedError = originalError;
+  }
+
+  // If error is a JSON.parse error
+  if (originalError instanceof SyntaxError) {
+    if (
+      originalError.message.includes('Unexpected token') ||
+      originalError.message.includes('Unexpected end of JSON input') ||
+      originalError.message.includes('Expected property name or')
+    ) {
+      status = 400;
+      parsedError = new errors.BadRequestError('Invalid JSON');
+    }
+  }
+
+  log.error(originalError);
+  if (status >= 500) {
+    log.error(`🔴 FAIL ${req.method} ${req.originalUrl}`, parsedError);
+  } else {
+    log.warn(`⚠️ FAIL ${req.method} ${req.originalUrl}`, parsedError);
+  }
+
+  res.status(status).json(apiResponse({}, { error: parsedError, req, res }));
+  res.end();
+}

package/src/middleware/logger.ts

@@ -0,0 +1,62 @@
+import { NextFunction, Request, Response } from 'express';
+import { logger, ctx } from '@takaro/util';
+import { context, trace } from '@opentelemetry/api';
+
+const SUPPRESS_BODY_KEYWORDS = ['password', 'newPassword'];
+const HIDDEN_ROUTES = ['/metrics', '/health', '/healthz', '/ready', '/readyz', '/queues/api/queues'];
+const log = logger('http');
+
+/**
+ * This middleware is called very early in the request lifecycle, so it's
+ * we leverage this fact to inject the context tracking at this stage
+ */
+export const LoggingMiddleware = ctx.wrap('HTTP', loggingMiddleware) as typeof loggingMiddleware;
+
+async function loggingMiddleware(req: Request, res: Response, next: NextFunction) {
+  if (HIDDEN_ROUTES.some((route) => req.originalUrl.startsWith(route))) {
+    return next();
+  }
+
+  const requestStartMs = Date.now();
+
+  const hideData = SUPPRESS_BODY_KEYWORDS.some((keyword) => (JSON.stringify(req.body) || '').includes(keyword));
+
+  log.debug(`⬇️ ${req.method} ${req.originalUrl}`, {
+    ip: req.ip,
+    method: req.method,
+    path: req.originalUrl,
+    body: hideData ? { suppressed_output: true } : req.body,
+  });
+
+  const span = trace.getSpan(context.active());
+
+  if (span) {
+    // get the trace ID from the span and set it in the headers
+    const traceId = span.spanContext().traceId;
+    res.header('X-Trace-Id', traceId);
+  }
+
+  // Log on API Call Finish to add responseTime
+  res.once('finish', () => {
+    const responseTime = Date.now() - requestStartMs;
+
+    log.info(`⬆️ ${req.method} ${req.originalUrl}`, {
+      responseTime,
+      requestMethod: req.method,
+      requestUrl: req.originalUrl,
+      requestSize: req.headers['content-length'],
+      status: res.statusCode,
+      responseSize: res.getHeader('Content-Length'),
+      userAgent: req.get('User-Agent'),
+      remoteIp: req.ip,
+      serverIp: '127.0.0.1',
+      referer: req.get('Referer'),
+      cacheLookup: false,
+      cacheHit: false,
+      cacheValidatedWithOriginServer: false,
+      protocol: req.protocol,
+    });
+  });
+
+  next();
+}

package/src/middleware/metrics.ts

@@ -0,0 +1,42 @@
+import { Request, Response, NextFunction } from 'express';
+import { Counter, Histogram } from 'prom-client';
+
+const counter = new Counter({
+  name: 'http_requests_total',
+  help: 'Total number of HTTP requests made',
+  labelNames: ['path', 'method', 'status'],
+});
+
+const histogram = new Histogram({
+  name: 'http_request_duration_seconds',
+  help: 'Duration of HTTP requests in seconds',
+  labelNames: ['path', 'method', 'status'],
+  buckets: [0.1, 0.5, 1, 2, 5, 10],
+});
+
+export async function metricsMiddleware(req: Request, res: Response, next: NextFunction) {
+  const rawPath = req.path;
+  const method = req.method;
+  // Filter out anything that looks like a UUID from path
+  const path = rawPath.replace(/\/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}/g, '/:id');
+
+  const start = Date.now();
+
+  try {
+    next();
+  } finally {
+    counter.inc({
+      path,
+      method,
+      status: res.statusCode.toString(),
+    });
+    histogram.observe(
+      {
+        path,
+        method,
+        status: res.statusCode.toString(),
+      },
+      (Date.now() - start) / 1000,
+    );
+  }
+}