@takaro/http 0.0.1

package/src/middleware/__tests__/paginationMiddleware.unit.test.ts

@@ -0,0 +1,48 @@
+import { sandbox, expect } from '@takaro/test';
+import { NextFunction, Request, Response } from 'express';
+import { errors } from '@takaro/util';
+import { paginationMiddleware } from '../paginationMiddleware.js';
+
+async function runPagination(page?: number, limit?: number) {
+  const req = { query: { page, limit } } as unknown as Request;
+  const res = { locals: {} } as Response;
+  const next = sandbox.stub<errors.ValidationError[]>();
+  await paginationMiddleware(req, res, next as unknown as NextFunction);
+  return { req, res, next };
+}
+
+describe('pagination middleware', () => {
+  it('Handles setting defaults', async () => {
+    const { res, next } = await runPagination();
+    expect(res.locals.page).to.equal(0);
+    expect(res.locals.limit).to.equal(100);
+    expect(next).to.have.been.calledOnce;
+  });
+  it('Works when pagination is passed', async () => {
+    const { res, next } = await runPagination(2, 20);
+    expect(res.locals.page).to.equal(2);
+    expect(res.locals.limit).to.equal(20);
+    expect(next).to.have.been.calledOnce;
+  });
+  it('Handles negative limit', async () => {
+    const { next } = await runPagination(1, -1);
+    expect(next).to.have.been.calledOnce;
+    const callArg = next.getCalls()[0].args[0];
+    expect(callArg).to.be.an.instanceOf(Error);
+    expect(callArg.message).to.equal('Invalid pagination: limit must be greater than or equal to 1');
+  });
+  it('Handles negative page', async () => {
+    const { next } = await runPagination(-1, 5);
+    expect(next).to.have.been.calledOnce;
+    const callArg = next.getCalls()[0].args[0];
+    expect(callArg).to.be.an.instanceOf(Error);
+    expect(callArg.message).to.equal('Invalid pagination: page must be greater than or equal to 0');
+  });
+  it('Handles limit too high', async () => {
+    const { next } = await runPagination(1, 1001);
+    expect(next).to.have.been.calledOnce;
+    const callArg = next.getCalls()[0].args[0];
+    expect(callArg).to.be.an.instanceOf(Error);
+    expect(callArg.message).to.equal('Invalid pagination: limit must be less than or equal to 1000');
+  });
+});

package/src/middleware/__tests__/rateLimit.integration.test.ts

@@ -0,0 +1,125 @@
+import { Response, NextFunction, Request } from 'express';
+import { Redis } from '@takaro/db';
+import { expect } from '@takaro/test';
+import { Controller, UseBefore, Get } from 'routing-controllers';
+import { HTTP } from '../../main.js';
+import { createRateLimitMiddleware } from '../rateLimit.js';
+import { ctx } from '@takaro/util';
+import supertest from 'supertest';
+
+describe('rateLimit middleware', () => {
+  let http: HTTP;
+  beforeEach(async () => {
+    @Controller()
+    class TestController {
+      @Get('/low-limit')
+      @UseBefore(
+        await createRateLimitMiddleware({
+          max: 5,
+          windowSeconds: 5,
+          useInMemory: false,
+        })
+      )
+      getLow() {
+        return 'Hello World';
+      }
+
+      @Get('/high-limit')
+      @UseBefore(
+        await createRateLimitMiddleware({
+          max: 15,
+          windowSeconds: 5,
+          useInMemory: false,
+        })
+      )
+      getHigh() {
+        return 'Hello World';
+      }
+
+      @Get('/authenticated')
+      @UseBefore((req: Request, _res: Response, next: NextFunction) => {
+        ctx.addData({ user: req.query.user as string });
+        next();
+      }, await createRateLimitMiddleware({ max: 5, windowSeconds: 5, useInMemory: false }))
+      getAuthenticated() {
+        return 'Hello World';
+      }
+    }
+
+    http = new HTTP(
+      {
+        controllers: [TestController],
+      },
+      { port: undefined }
+    );
+
+    await http.start();
+  });
+
+  afterEach(async () => {
+    await http.stop();
+
+    const redis = await Redis.getClient('http:rateLimit');
+    await redis.flushAll();
+  });
+
+  after(async () => {
+    await Redis.destroy();
+  });
+
+  it('should limit requests', async () => {
+    const agent = supertest(http.expressInstance);
+    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+    // @ts-ignore
+    agent.get('/low-limit').set('X-Forwarded-For', '127.0.0.2');
+
+    for (let i = 0; i < 4; i++) {
+      // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+      // @ts-ignore
+      await agent.get('/low-limit').expect(200);
+    }
+    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+    // @ts-ignore
+    await agent.get('/low-limit').expect(429);
+  });
+
+  it('should apply distinct limits per user', async () => {
+    const agent = supertest(http.expressInstance);
+
+    for (let i = 0; i < 4; i++) {
+      // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+      // @ts-ignore
+      await agent.get('/authenticated?user=1').expect(200);
+    }
+    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+    // @ts-ignore
+    await agent.get('/authenticated?user=1').expect(429);
+
+    for (let i = 0; i < 4; i++) {
+      // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+      // @ts-ignore
+      await agent.get('/authenticated?user=2').expect(200);
+    }
+    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+    // @ts-ignore
+    await agent.get('/authenticated?user=2').expect(429);
+  });
+
+  it('Should accurately report metadata info via HTTP headers', async () => {
+    const agent = supertest(http.expressInstance);
+
+    for (let i = 1; i < 5; i++) {
+      // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+      // @ts-ignore
+      const res = await agent.get('/low-limit').expect(200);
+      expect(res.header['x-ratelimit-remaining']).to.equal((5 - i).toString());
+      expect(res.header['x-ratelimit-limit']).to.equal('5');
+      expect(res.header['x-ratelimit-reset']).to.be.a('string');
+    }
+    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+    // @ts-ignore
+    const res = await agent.get('/low-limit').expect(429);
+    expect(res.header['x-ratelimit-remaining']).to.equal('0');
+    expect(res.header['x-ratelimit-limit']).to.equal('5');
+  });
+});

package/src/middleware/adminAuth.ts

@@ -0,0 +1,33 @@
+import { Request, Response, NextFunction } from 'express';
+import { errors, logger } from '@takaro/util';
+import { ory, AUDIENCES } from '@takaro/auth';
+
+const log = logger('http:middleware:adminAuth');
+
+export async function adminAuthMiddleware(request: Request, response: Response, next: NextFunction) {
+  try {
+    const rawToken = request.headers['authorization']?.replace('Bearer ', '');
+
+    if (!rawToken) {
+      log.warn('No token provided');
+      return next(new errors.UnauthorizedError());
+    }
+
+    const token = await ory.introspectToken(rawToken);
+
+    if (!token.active) {
+      log.warn('Token is not active');
+      return next(new errors.ForbiddenError());
+    }
+
+    if (!token.aud.includes(AUDIENCES.TAKARO_API_ADMIN)) {
+      log.warn('Token is not for admin API', { token });
+      return next(new errors.ForbiddenError());
+    }
+
+    return next();
+  } catch (error) {
+    log.error('Unexpected error', { error });
+    next(new errors.ForbiddenError());
+  }
+}

package/src/middleware/errorHandler.ts

@@ -0,0 +1,67 @@
+import { Request, Response, NextFunction } from 'express';
+import { HttpError } from 'routing-controllers';
+import { logger, errors } from '@takaro/util';
+import { apiResponse } from '../util/apiResponse.js';
+import { ValidationError } from 'class-validator';
+
+const log = logger('errorHandler');
+
+export async function ErrorHandler(
+  originalError: Error,
+  req: Request,
+  res: Response,
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  _next: NextFunction
+) {
+  let status = 500;
+  let parsedError = new errors.InternalServerError();
+
+  if (originalError.name === 'BadRequestError') {
+    if (originalError.hasOwnProperty('errors')) {
+      // @ts-expect-error Error typing is weird in ts... but we validate during runtime so should be OK
+      const validationErrors = originalError['errors'] as ValidationError[];
+      parsedError = new errors.ValidationError('Validation error', validationErrors);
+      log.warn('⚠️ Validation errror', { details: validationErrors.map((e) => JSON.stringify(e.target, null, 2)) });
+    }
+  }
+
+  if (originalError instanceof HttpError) {
+    status = originalError.httpCode;
+  }
+
+  if (originalError.name === 'UniqueViolationError') {
+    status = 409;
+    parsedError = new errors.ConflictError('Unique constraint violation');
+  }
+
+  if (originalError.name === 'NotNullViolationError') {
+    status = 400;
+    parsedError = new errors.BadRequestError('Missing required field');
+  }
+
+  if (originalError instanceof errors.TakaroError) {
+    status = originalError.http;
+    parsedError = originalError;
+  }
+
+  // If error is a JSON.parse error
+  if (originalError instanceof SyntaxError) {
+    if (
+      originalError.message.includes('Unexpected token') ||
+      originalError.message.includes('Unexpected end of JSON input')
+    ) {
+      status = 400;
+      parsedError = new errors.BadRequestError('Invalid JSON');
+    }
+  }
+
+  log.error(originalError);
+  if (status >= 500) {
+    log.error(`🔴 FAIL ${req.method} ${req.originalUrl}`, parsedError);
+  } else {
+    log.warn(`⚠️ FAIL ${req.method} ${req.originalUrl}`, parsedError);
+  }
+
+  res.status(status).json(apiResponse({}, { error: parsedError, req, res }));
+  return res.end();
+}

package/src/middleware/logger.ts

@@ -0,0 +1,62 @@
+import { NextFunction, Request, Response } from 'express';
+import { logger, ctx } from '@takaro/util';
+
+const SUPPRESS_BODY_KEYWORDS = ['password', 'newPassword'];
+const HIDDEN_ROUTES = ['/metrics', '/health', '/healthz', '/ready', '/readyz', '/queues/api/queues'];
+import { context, trace } from '@opentelemetry/api';
+const log = logger('http');
+
+/**
+ * This middleware is called very early in the request lifecycle, so it's
+ * we leverage this fact to inject the context tracking at this stage
+ */
+export const LoggingMiddleware = ctx.wrap('HTTP', loggingMiddleware);
+
+async function loggingMiddleware(req: Request, res: Response, next: NextFunction) {
+  if (HIDDEN_ROUTES.some((route) => req.originalUrl.startsWith(route))) {
+    return next();
+  }
+
+  const requestStartMs = Date.now();
+
+  const hideData = SUPPRESS_BODY_KEYWORDS.some((keyword) => (JSON.stringify(req.body) || '').includes(keyword));
+
+  log.debug(`⬇️ ${req.method} ${req.originalUrl}`, {
+    ip: req.ip,
+    method: req.method,
+    path: req.originalUrl,
+    body: hideData ? { suppressed_output: true } : req.body,
+  });
+
+  const span = trace.getSpan(context.active());
+
+  if (span) {
+    // get the trace ID from the span and set it in the headers
+    const traceId = span.spanContext().traceId;
+    res.header('X-Trace-Id', traceId);
+  }
+
+  // Log on API Call Finish to add responseTime
+  res.once('finish', () => {
+    const responseTime = Date.now() - requestStartMs;
+
+    log.info(`⬆️ ${req.method} ${req.originalUrl}`, {
+      responseTime,
+      requestMethod: req.method,
+      requestUrl: req.originalUrl,
+      requestSize: req.headers['content-length'],
+      status: res.statusCode,
+      responseSize: res.getHeader('Content-Length'),
+      userAgent: req.get('User-Agent'),
+      remoteIp: req.ip,
+      serverIp: '127.0.0.1',
+      referer: req.get('Referer'),
+      cacheLookup: false,
+      cacheHit: false,
+      cacheValidatedWithOriginServer: false,
+      protocol: req.protocol,
+    });
+  });
+
+  next();
+}

package/src/middleware/metrics.ts

@@ -0,0 +1,44 @@
+import { Request, Response, NextFunction } from 'express';
+import { Counter, Histogram } from 'prom-client';
+
+const counter = new Counter({
+  name: 'http_requests_total',
+  help: 'Total number of HTTP requests made',
+  labelNames: ['path', 'method', 'status'],
+});
+
+const histogram = new Histogram({
+  name: 'http_request_duration_seconds',
+  help: 'Duration of HTTP requests in seconds',
+  labelNames: ['path', 'method', 'status'],
+  buckets: [0.1, 0.5, 1, 2, 5, 10],
+});
+
+export async function metricsMiddleware(req: Request, res: Response, next: NextFunction) {
+  const rawPath = req.path;
+  const method = req.method;
+  // Filter out anything that looks like a UUID from path
+  const path = rawPath.replace(/\/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}/g, '/:id');
+
+  const start = Date.now();
+
+  try {
+    await next();
+  } catch (error) {
+    throw error;
+  } finally {
+    counter.inc({
+      path,
+      method,
+      status: res.statusCode.toString(),
+    });
+    histogram.observe(
+      {
+        path,
+        method,
+        status: res.statusCode.toString(),
+      },
+      (Date.now() - start) / 1000
+    );
+  }
+}

package/src/middleware/paginationMiddleware.ts

@@ -0,0 +1,29 @@
+import { NextFunction, Request, Response } from 'express';
+import * as yup from 'yup';
+import { errors, logger } from '@takaro/util';
+
+const log = logger('http:pagination');
+
+const paginationSchema = yup.object({
+  page: yup.number().default(0).min(0),
+  limit: yup.number().default(100).min(1).max(1000),
+});
+
+export async function paginationMiddleware(req: Request, res: Response, next: NextFunction): Promise<void> {
+  const merged = { ...req.query, ...req.body };
+  try {
+    const result = await paginationSchema.validate(merged);
+
+    res.locals.page = result.page;
+    res.locals.limit = result.limit;
+
+    next();
+  } catch (error) {
+    if (error instanceof yup.ValidationError) {
+      next(new errors.ValidationError('Invalid pagination', [error]));
+    } else {
+      log.error('Unexpected error', error);
+      next(new errors.InternalServerError());
+    }
+  }
+}

package/src/middleware/rateLimit.ts

@@ -0,0 +1,78 @@
+import { Request, Response, NextFunction } from 'express';
+import { Redis } from '@takaro/db';
+import { logger, errors, ctx } from '@takaro/util';
+import { RateLimiterRes, RateLimiterRedis, RateLimiterMemory } from 'rate-limiter-flexible';
+
+export interface IRateLimitMiddlewareOptions {
+  max: number;
+  windowSeconds: number;
+  keyPrefix?: string;
+  useInMemory?: boolean;
+}
+
+export async function createRateLimitMiddleware(opts: IRateLimitMiddlewareOptions) {
+  const log = logger('http:rateLimit');
+  const redis = await Redis.getClient('http:rateLimit', {
+    // Legacy mode is required for rate-limiter-flexible which isn't updated
+    // to use the new v4 redis API.
+    // See: https://github.com/animir/node-rate-limiter-flexible/wiki/Redis#usage
+    legacyMode: true,
+  });
+
+  // We create a randomHash to use in Redis keys
+  // This makes sure that each endpoint can get different rate limits without too much hassle
+  const randomHash = Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15);
+
+  let rateLimiter: RateLimiterMemory | RateLimiterRedis;
+
+  if (opts.useInMemory) {
+    rateLimiter = new RateLimiterMemory({
+      points: opts.max,
+      duration: opts.windowSeconds,
+      keyPrefix: `http:rateLimit:${opts.keyPrefix ? opts.keyPrefix : randomHash}`,
+    });
+  } else {
+    rateLimiter = new RateLimiterRedis({
+      points: opts.max,
+      duration: opts.windowSeconds,
+      storeClient: redis,
+      keyPrefix: `http:rateLimit:${opts.keyPrefix ? opts.keyPrefix : randomHash}`,
+    });
+  }
+
+  return async (req: Request, res: Response, next: NextFunction) => {
+    const ctxData = ctx.data;
+    let limitedKey = null;
+    if (ctxData.user) {
+      limitedKey = ctxData.user;
+    } else {
+      // TODO: should handle case ip is undefined
+      limitedKey = req.ip!;
+    }
+
+    let rateLimiterRes: RateLimiterRes | null = null;
+
+    try {
+      rateLimiterRes = await rateLimiter.consume(limitedKey);
+    } catch (err) {
+      if (err instanceof RateLimiterRes) {
+        rateLimiterRes = err;
+        log.warn(`rate limited, try again in ${err.msBeforeNext}ms`);
+      } else {
+        throw err;
+      }
+    }
+
+    if (rateLimiterRes) {
+      res.set('X-RateLimit-Limit', opts.max.toString());
+      res.set('X-RateLimit-Remaining', rateLimiterRes.remainingPoints.toString());
+      res.set('X-RateLimit-Reset', rateLimiterRes.msBeforeNext.toString());
+
+      if (rateLimiterRes.remainingPoints === 0) {
+        next(new errors.TooManyRequestsError());
+      }
+    }
+
+    return next();
+  };
+}

package/src/util/apiResponse.ts

@@ -0,0 +1,106 @@
+import { errors, isTakaroDTO, TakaroDTO } from '@takaro/util';
+import { Type } from 'class-transformer';
+import { IsISO8601, IsNumber, IsOptional, ValidateNested } from 'class-validator';
+import { IsString, ValidationError as ClassValidatorError } from 'class-validator';
+import { Request, Response } from 'express';
+
+class ErrorOutput {
+  @IsString()
+  code?: string;
+
+  @IsString()
+  message?: string;
+
+  @IsString()
+  details?: string | ClassValidatorError[];
+}
+
+class MetadataOutput {
+  @IsString()
+  @IsISO8601()
+  serverTime!: string;
+
+  @Type(() => ErrorOutput)
+  @ValidateNested()
+  error?: ErrorOutput;
+
+  /**
+   * The page number of the response
+   */
+  @IsNumber()
+  @IsOptional()
+  page?: number;
+
+  /**
+   * The number of items returned in the response (aka page size)
+   */
+  @IsNumber()
+  @IsOptional()
+  limit?: number;
+
+  /**
+   * The total number of items in the collection
+   */
+  @IsNumber()
+  @IsOptional()
+  total?: number;
+}
+export class APIOutput<T> extends TakaroDTO<APIOutput<T>> {
+  @Type(() => MetadataOutput)
+  @ValidateNested()
+  meta!: MetadataOutput;
+
+  data!: T;
+}
+
+interface IApiResponseOptions {
+  error?: Error;
+  meta?: Record<string, string | number>;
+  req: Request;
+  res: Response;
+}
+
+export function apiResponse(data: unknown = {}, opts?: IApiResponseOptions): APIOutput<unknown> {
+  const returnVal = new APIOutput<unknown>();
+
+  returnVal.meta = new MetadataOutput();
+  returnVal.data = {};
+
+  if (opts?.error) {
+    returnVal.meta.error = new ErrorOutput();
+
+    returnVal.meta.error.code = String(opts.error.name);
+
+    if ('details' in opts.error) {
+      if (opts.error instanceof errors.ValidationError) {
+        returnVal.meta.error.details = opts.error.details as ClassValidatorError[];
+      } else {
+        returnVal.meta.error.details = opts.error.details as string;
+      }
+    }
+
+    returnVal.meta.error.message = String(opts.error.message);
+  }
+
+  if (opts?.meta) {
+    returnVal.meta.page = opts?.res.locals.page;
+    returnVal.meta.limit = opts?.res.locals.limit;
+    returnVal.meta.total = opts?.meta.total as number;
+  }
+
+  if (isTakaroDTO(data)) {
+    returnVal.data = data.toJSON();
+  } else if (Array.isArray(data)) {
+    returnVal.data = data.map((item) => {
+      if (isTakaroDTO(item)) {
+        return item.toJSON();
+      }
+
+      return item;
+    });
+  } else {
+    returnVal.data = data;
+  }
+
+  return returnVal;
+}

package/tsconfig.build.json

@@ -0,0 +1,9 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "rootDir": "./src",
+    "outDir": "dist"
+  },
+  "include": ["src/**/*"],
+  "exclude": ["src/**/*.test.ts"]
+}

package/tsconfig.json

@@ -0,0 +1,8 @@
+{
+  "extends": "../../tsconfig.esm.json",
+  "compilerOptions": {
+    "rootDir": "src",
+    "outDir": "dist"
+  },
+  "include": ["src/**/*"]
+}

package/typedoc.json

@@ -0,0 +1,3 @@
+{
+  "entryPoints": ["src/main.ts"]
+}