@terreno/api 0.0.1

package/src/example.ts

@@ -0,0 +1,99 @@
+import express from "express";
+import mongoose, {model, Schema} from "mongoose";
+import passportLocalMongoose from "passport-local-mongoose";
+
+import {modelRouter, type modelRouterOptions} from "./api";
+import {addAuthRoutes, setupAuth} from "./auth";
+import {setupServer} from "./expressServer";
+import {logger} from "./logger";
+import {Permissions} from "./permissions";
+import {baseUserPlugin, createdUpdatedPlugin} from "./plugins";
+
+mongoose
+  .connect("mongodb://localhost:27017/example")
+  .then(() => {
+    logger.debug("Connected to mongo");
+  })
+  .catch((err) => {
+    logger.error(`Error connecting to mongo ${err}`);
+  });
+
+interface User {
+  admin: boolean;
+  username: string;
+}
+
+interface Food {
+  name: string;
+  calories: number;
+  created: Date;
+  ownerId: mongoose.Types.ObjectId | User;
+  hidden?: boolean;
+}
+
+const userSchema = new Schema<User>({
+  admin: {default: false, type: Boolean},
+  username: String,
+});
+
+userSchema.plugin(passportLocalMongoose as any, {usernameField: "email"});
+userSchema.plugin(createdUpdatedPlugin);
+userSchema.plugin(baseUserPlugin);
+const UserModel = model<User>("User", userSchema);
+
+const schema = new Schema<Food>({
+  calories: Number,
+  created: Date,
+  hidden: {default: false, type: Boolean},
+  name: String,
+  ownerId: {ref: "User", type: "ObjectId"},
+});
+
+const FoodModel = model<Food>("Food", schema);
+
+function getBaseServer() {
+  const app = express();
+
+  app.all("/*", (req, res, next) => {
+    res.header("Access-Control-Allow-Origin", "*");
+    res.header("Access-Control-Allow-Headers", "*");
+    // intercepts OPTIONS method
+    if (req.method === "OPTIONS") {
+      res.send(200);
+    } else {
+      next();
+    }
+  });
+  app.use(express.json());
+  setupAuth(app, UserModel as any);
+  addAuthRoutes(app, UserModel as any);
+
+  function addRoutes(router: express.Router, options?: Partial<modelRouterOptions<any>>): void {
+    router.use(
+      "/food",
+      modelRouter(FoodModel, {
+        ...options,
+        openApiOverwrite: {
+          get: {responses: {200: {description: "Get all the food"}}},
+        },
+        permissions: {
+          create: [Permissions.IsAuthenticated],
+          delete: [Permissions.IsAdmin],
+          list: [Permissions.IsAny],
+          read: [Permissions.IsAny],
+          update: [Permissions.IsOwner],
+        },
+        queryFields: ["name", "calories", "created", "ownerId", "hidden"],
+      })
+    );
+  }
+
+  return setupServer({
+    addRoutes,
+    loggingOptions: {
+      level: "debug",
+    },
+    userModel: UserModel as any,
+  });
+}
+getBaseServer();

package/src/express.d.ts

@@ -0,0 +1,5 @@
+declare namespace Express {
+  export interface Request {
+    user?: {_id: string | ObjectId; id: string; admin: boolean};
+  }
+}

package/src/expressServer.ts

@@ -0,0 +1,387 @@
+import * as Sentry from "@sentry/node";
+import openapi from "@wesleytodd/openapi";
+import cors from "cors";
+import cron from "cron";
+import express, {type Router} from "express";
+import type jwt from "jsonwebtoken";
+import cloneDeep from "lodash/cloneDeep";
+import onFinished from "on-finished";
+import passport from "passport";
+import qs from "qs";
+
+import type {modelRouterOptions} from "./api";
+import {addAuthRoutes, addMeRoutes, setupAuth, type UserModel as UserMongooseModel} from "./auth";
+import {apiErrorMiddleware, apiUnauthorizedMiddleware} from "./errors";
+import {type LoggingOptions, logger, setupLogging} from "./logger";
+import {sendToSlack} from "./notifiers";
+import {openApiEtagMiddleware} from "./openApiEtag";
+
+const SLOW_READ_MAX = 200;
+const SLOW_WRITE_MAX = 500;
+const IS_JEST = process.env.JEST_WORKER_ID !== undefined;
+
+export function setupEnvironment(): void {
+  if (!process.env.TOKEN_ISSUER) {
+    throw new Error("TOKEN_ISSUER must be set in env.");
+  }
+  if (!process.env.TOKEN_SECRET) {
+    throw new Error("TOKEN_SECRET must be set.");
+  }
+  if (!process.env.REFRESH_TOKEN_SECRET) {
+    throw new Error("REFRESH_TOKEN_SECRET must be set.");
+  }
+  if (!process.env.SESSION_SECRET) {
+    throw new Error("SESSION_SECRET must be set.");
+  }
+  if (!process.env.TOKEN_EXPIRES_IN && !IS_JEST) {
+    logger.warn("TOKEN_EXPIRES_IN is not set so using default.");
+  }
+  if (!process.env.REFRESH_TOKEN_EXPIRES_IN && !IS_JEST) {
+    logger.warn("REFRESH_TOKEN_EXPIRES_IN not set so using default.");
+  }
+}
+
+export type AddRoutes = (router: Router, options?: Partial<modelRouterOptions<any>>) => void;
+
+const logRequestsFinished = (req: any, res: any, startTime: bigint) => {
+  const options = (res.locals.loggingOptions ?? {}) as LoggingOptions;
+
+  const slowReadMs = options.logSlowRequestsReadMs ?? SLOW_READ_MAX;
+  const slowWriteMs = options.logSlowRequestsWriteMs ?? SLOW_WRITE_MAX;
+
+  const diff = process.hrtime.bigint() - startTime;
+  const diffInMs = Number(diff) / 1000000;
+  let pathName = "unknown";
+  if (req.route && req.routeMount) {
+    pathName = `${req.routeMount}${req.route.path}`;
+  } else if (req.route) {
+    pathName = req.route.path;
+  } else if (res.statusCode < 400) {
+    logger.warn(`Request without route: ${req.originalUrl}`);
+  }
+  if (process.env.DISABLE_LOG_ALL_REQUESTS !== "true") {
+    logger.debug(`${req.method} -> ${req.originalUrl} ${res.statusCode} ${`${diffInMs}ms`}`);
+  }
+  if (options.logSlowRequests) {
+    if (diffInMs > slowReadMs && req.method === "GET") {
+      logger.warn(
+        `Slow GET request, ${JSON.stringify({
+          pathName,
+          requestTime: diffInMs,
+          url: req.originalUrl,
+        })}`
+      );
+    } else if (diffInMs > slowWriteMs) {
+      logger.warn(
+        `Slow write request ${JSON.stringify({
+          pathName,
+          requestTime: diffInMs,
+          url: req.originalUrl,
+        })}`
+      );
+    }
+  }
+};
+
+export function logRequests(req: any, res: any, next: any) {
+  const startTime = process.hrtime.bigint();
+
+  let userString = "";
+  if (req.user) {
+    let type = "User";
+    if (req.user?.admin) {
+      type = "Admin";
+    } else if (req.user?.testUser) {
+      type = "Test User";
+    } else if (req.user?.type) {
+      type = req.user?.type;
+    }
+    userString = ` <${type}:${req.user.id}>`;
+  }
+
+  let body = "";
+  if (req.body && Object.keys(req.body).length > 0) {
+    const bodyCopy = cloneDeep(req.body);
+    if (bodyCopy.password) {
+      bodyCopy.password = "<PASSWORD>";
+    }
+    body = ` Body: ${JSON.stringify(bodyCopy)}`;
+  }
+
+  if (process.env.DISABLE_LOG_ALL_REQUESTS !== "true") {
+    logger.debug(`${req.method} <- ${req.url}${userString}${body}`);
+  }
+  onFinished(res, () => logRequestsFinished(req, res, startTime));
+  next();
+}
+
+export function createRouter(rootPath: string, addRoutes: AddRoutes, middleware: any[] = []) {
+  function routePathMiddleware(req: any, _res: any, next: any) {
+    if (!req.routeMount) {
+      req.routeMount = [];
+    }
+    req.routeMount.push(rootPath);
+    next();
+  }
+
+  const router = express.Router();
+  router.use(routePathMiddleware);
+  addRoutes(router);
+  return [rootPath, ...middleware, router];
+}
+
+export function createRouterWithAuth(
+  rootPath: string,
+  addRoutes: (router: Router) => void,
+  middleware: any[] = []
+) {
+  return createRouter(rootPath, addRoutes, [
+    passport.authenticate("firebase-jwt", {session: false}),
+    ...middleware,
+  ]);
+}
+
+export interface AuthOptions {
+  generateJWTPayload?: (user: any) => Record<string, any>;
+  generateTokenExpiration?: (user: any) => number | jwt.SignOptions["expiresIn"];
+  generateRefreshTokenExpiration?: (user: any) => number | jwt.SignOptions["expiresIn"];
+}
+
+interface InitializeRoutesOptions {
+  corsOrigin?:
+    | string
+    | boolean
+    | RegExp
+    | Array<boolean | string | RegExp>
+    | ((
+        requestOrigin: string | undefined,
+        callback: (
+          err: Error | null,
+          origin?: boolean | string | RegExp | Array<boolean | string | RegExp>
+        ) => void
+      ) => void);
+  addMiddleware?: AddRoutes;
+  // The maximum number of array elements to parse in a query string. Defaults to 200.
+  arrayLimit?: number;
+  // Whether requests should be logged. In production, you may want to disable this if using another
+  // logger (e.g. Google Cloud).
+  logRequests?: boolean;
+  loggingOptions?: LoggingOptions;
+  authOptions?: AuthOptions;
+}
+
+function initializeRoutes(
+  UserModel: UserMongooseModel,
+  addRoutes: AddRoutes,
+  options: InitializeRoutesOptions = {}
+) {
+  const app = express();
+
+  // TODO: Log a warning when we hit the array limit.
+  app.set("query parser", (str: string) => qs.parse(str, {arrayLimit: options.arrayLimit ?? 200}));
+
+  app.use(
+    cors({
+      origin: options.corsOrigin ?? "*",
+    })
+  );
+
+  if (options.addMiddleware) {
+    options.addMiddleware(app);
+  }
+
+  app.use(express.json());
+
+  // Add login/signup/refresh_token before the JWT/auth middlewares
+  addAuthRoutes(app, UserModel as any, options?.authOptions);
+
+  setupAuth(app as any, UserModel as any);
+
+  if (options.logRequests !== false) {
+    app.use(logRequests);
+  }
+
+  // Store the logging options on the request so we can access them later.
+  app.use((_req, res, next) => {
+    res.locals.loggingOptions = options.loggingOptions;
+    next();
+  });
+
+  // Add Sentry scopes for session, transaction, and userId if any are set
+  app.all("*", (req: any, _res: any, next: any) => {
+    const transactionId = req.header("X-Transaction-ID");
+    const sessionId = req.header("X-Session-ID");
+    if (transactionId) {
+      Sentry.getCurrentScope().setTag("transaction_id", transactionId);
+    }
+    if (sessionId) {
+      Sentry.getCurrentScope().setTag("session_id", sessionId);
+    }
+    if (req.user?._id) {
+      Sentry.getCurrentScope().setTag("user", req.user._id);
+    }
+    next();
+  });
+
+  // Add ETag middleware for OpenAPI JSON endpoint before the openapi middleware
+  app.use(openApiEtagMiddleware);
+
+  const oapi = openapi({
+    info: {
+      description: "Generated docs from an Express api",
+      title: "Express Application",
+      version: "1.0.0",
+    },
+    openapi: "3.0.0",
+  });
+  app.use(oapi);
+
+  if (process.env.ENABLE_SWAGGER === "true") {
+    app.use("/swagger", oapi.swaggerui());
+  }
+
+  addMeRoutes(app, UserModel as any, options?.authOptions);
+  addRoutes(app, {openApi: oapi});
+
+  Sentry.setupExpressErrorHandler(app);
+
+  // Catch any thrown APIErrors and return them in an OpenAPI compatible format
+  app.use(apiUnauthorizedMiddleware);
+  app.use(apiErrorMiddleware);
+
+  app.use(function onError(err: any, _req: any, res: any, _next: any) {
+    logger.error(`Fallthrough error: ${err}${err?.stack ? `\n${err.stack}` : ""}}`);
+    Sentry.captureException(err);
+    res.statusCode = 500;
+    res.end(`${res.sentry}\n`);
+  });
+
+  return app;
+}
+
+export interface SetupServerOptions {
+  userModel: UserMongooseModel;
+  addRoutes: AddRoutes;
+  loggingOptions?: LoggingOptions;
+  authOptions?: AuthOptions;
+  skipListen?: boolean;
+  corsOrigin?:
+    | string
+    | boolean
+    | RegExp
+    | Array<boolean | string | RegExp>
+    | ((
+        requestOrigin: string | undefined,
+        callback: (
+          err: Error | null,
+          origin?: boolean | string | RegExp | Array<boolean | string | RegExp>
+        ) => void
+      ) => void);
+  addMiddleware?: AddRoutes;
+  ignoreTraces?: string[];
+  sentryOptions?: Sentry.NodeOptions;
+}
+
+// Sets up the routes and returns a function to launch the API.
+export function setupServer(options: SetupServerOptions) {
+  const UserModel = options.userModel;
+  const addRoutes = options.addRoutes;
+
+  setupLogging(options.loggingOptions);
+
+  let app: express.Application;
+  try {
+    app = initializeRoutes(UserModel, addRoutes, {
+      addMiddleware: options.addMiddleware,
+      authOptions: options.authOptions,
+      corsOrigin: options.corsOrigin,
+    });
+  } catch (error: any) {
+    logger.error(`Error initializing routes: ${error.stack}`);
+    throw error;
+  }
+
+  if (!options.skipListen) {
+    const port = process.env.PORT || "9000";
+    try {
+      app.listen(port, () => {
+        logger.info(`Listening on port ${port}`);
+      });
+    } catch (error) {
+      logger.error(`Error trying to start HTTP server: ${error}\n${(error as any).stack}`);
+      process.exit(1);
+    }
+  }
+  return app;
+}
+
+// Convenience method to execute cronjobs with an always-running server.
+export function cronjob(
+  name: string,
+  schedule: "hourly" | "minutely" | string,
+  callback: () => void
+) {
+  let _cronSchedule = schedule;
+  if (schedule === "hourly") {
+    _cronSchedule = "0 * * * *";
+  } else if (schedule === "minutely") {
+    _cronSchedule = "* * * * *";
+  }
+  logger.info(`Adding cronjob ${name}, running at: ${schedule}`);
+  try {
+    new cron.CronJob(schedule, callback, null, true, "America/Chicago");
+  } catch (error) {
+    throw new Error(`Failed to create cronjob: ${error}`);
+  }
+}
+
+export interface WrapScriptOptions {
+  onFinish?: (result?: any) => void | Promise<void>;
+  terminateTimeout?: number; // in seconds, defaults to 300. Set to 0 to have no termination timeout.
+  slackChannel?: string;
+}
+// Wrap up a script with some helpers, such as catching errors, reporting them to sentry, notifying
+// Slack of runs, etc. Also supports timeouts.
+export async function wrapScript(func: () => Promise<any>, options: WrapScriptOptions = {}) {
+  const name = require.main?.filename.split("/").slice(-1)[0].replace(".ts", "");
+  logger.info(`Running script ${name}`);
+  await sendToSlack(`Running script ${name}`, {
+    slackChannel: options.slackChannel,
+  });
+
+  if (options.terminateTimeout !== 0) {
+    const warnTime = ((options.terminateTimeout ?? 300) / 2) * 1000;
+    const closeTime = (options.terminateTimeout ?? 300) * 1000;
+    setTimeout(async () => {
+      const msg = `Script ${name} is taking a while, currently ${warnTime / 1000} seconds`;
+      await sendToSlack(msg);
+      logger.warn(msg);
+    }, warnTime);
+
+    setTimeout(async () => {
+      const msg = `Script ${name} took too long, exiting`;
+      await sendToSlack(msg);
+      logger.error(msg);
+      Sentry.captureException(new Error(`Script ${name} took too long, exiting`));
+      await Sentry.flush();
+      process.exit(2);
+    }, closeTime);
+  }
+
+  let result: any;
+  try {
+    result = await func();
+    if (options.onFinish) {
+      await options.onFinish(result);
+    }
+  } catch (error) {
+    Sentry.captureException(error);
+    logger.error(`Error running script ${name}: ${error}\n${(error as Error).stack}`);
+    await sendToSlack(`Error running script ${name}: ${error}\n${(error as Error).stack}`);
+    await Sentry.flush();
+    process.exit(1);
+  }
+  await sendToSlack(`Success running script ${name}: ${result}`);
+  // Unclear why we have to exit here to prevent the script for continuing to run.
+  process.exit(0);
+}

package/src/index.ts

@@ -0,0 +1,14 @@
+export * from "./api";
+export * from "./auth";
+export * from "./errors";
+export * from "./expressServer";
+export * from "./logger";
+export * from "./middleware";
+export * from "./notifiers";
+export * from "./openApiBuilder";
+export * from "./openApiEtag";
+export * from "./permissions";
+export * from "./plugins";
+export * from "./populate";
+export * from "./transformers";
+export * from "./utils";

package/src/logger.ts

@@ -0,0 +1,190 @@
+import fs from "node:fs";
+import {inspect} from "node:util";
+import * as Sentry from "@sentry/node";
+import winston from "winston";
+
+function isPrimitive(val: any) {
+  return val === null || (typeof val !== "object" && typeof val !== "function");
+}
+
+function formatWithInspect(val: any) {
+  const prefix = isPrimitive(val) ? "" : "\n";
+  const shouldFormat = typeof val !== "string";
+
+  return prefix + (shouldFormat ? inspect(val, {colors: true, depth: null}) : val);
+}
+
+// Winston doesn't operate like console.log by default, e.g. `logger.error('error',
+// error)` only prints the message and no args. Add handling for all the args,
+// while also supporting splat logging.
+function printf(timestamp = false) {
+  return (info: winston.Logform.TransformableInfo) => {
+    const msg = formatWithInspect(info.message);
+    const splatArgs = (info[Symbol.for("splat") as any] || []) as any[];
+    const rest = splatArgs.map((data: any) => formatWithInspect(data)).join(" ");
+    if (timestamp) {
+      return `${info.timestamp} - ${info.level}: ${msg} ${rest}`;
+    }
+    return `${info.level}: ${msg} ${rest}`;
+  };
+}
+
+// Setup a global, default rejection handler.
+winston.add(
+  new winston.transports.Console({
+    debugStdout: true,
+    format: winston.format.combine(
+      winston.format.colorize(),
+      winston.format.simple(),
+      winston.format.printf(printf(false))
+    ),
+    handleExceptions: true,
+    handleRejections: true,
+    level: "error",
+  })
+);
+
+// Setup a default console logger.
+export const winstonLogger = winston.createLogger({
+  level: "debug",
+  transports: [
+    new winston.transports.Console({
+      debugStdout: true,
+      format: winston.format.combine(
+        winston.format.colorize(),
+        winston.format.simple(),
+        winston.format.printf(printf(false))
+      ),
+      handleExceptions: true,
+      handleRejections: true,
+      level: "debug",
+    }),
+  ],
+});
+
+// Helper function to send logs to Sentry if enabled
+function sendToSentry(message: string, level: "debug" | "info" | "warn" | "error") {
+  if (process.env.USE_SENTRY_LOGGING === "true" && Sentry.logger) {
+    Sentry.logger[level](message);
+  }
+}
+
+export const logger = {
+  // simple way to log a caught exception. e.g. promise().catch(logger.catch)
+  catch: (e: unknown) => {
+    const errorMsg = `Caught: ${(e as Error)?.message} ${(e as Error)?.stack}`;
+    winstonLogger.error(errorMsg);
+    if (process.env.USE_SENTRY_LOGGING === "true") {
+      if (e instanceof Error) {
+        Sentry.captureException(e);
+      } else if (Sentry.logger) {
+        Sentry.logger.error(errorMsg);
+      }
+    }
+  },
+  debug: (msg: string, ...args: unknown[]) => {
+    winstonLogger.debug(msg, ...args);
+    sendToSentry(msg, "debug");
+  },
+  error: (msg: string, ...args: unknown[]) => {
+    winstonLogger.error(msg, ...args);
+    sendToSentry(msg, "error");
+  },
+  info: (msg: string, ...args: unknown[]) => {
+    winstonLogger.info(msg, ...args);
+    sendToSentry(msg, "info");
+  },
+  warn: (msg: string, ...args: unknown[]) => {
+    winstonLogger.warn(msg, ...args);
+    sendToSentry(msg, "warn");
+  },
+};
+
+export interface LoggingOptions {
+  level?: "debug" | "info" | "warn" | "error";
+  transports?: winston.transport[];
+  disableFileLogging?: boolean;
+  disableConsoleLogging?: boolean;
+  disableConsoleColors?: boolean;
+  showConsoleTimestamps?: boolean;
+  logDirectory?: string;
+  logRequests?: boolean;
+  // Whether to log when requests are slow.
+  logSlowRequests?: boolean;
+  // The threshold in ms for logging slow requests. Defaults to 200ms for read requests.
+  logSlowRequestsReadMs?: number;
+  // The threshold in ms for logging slow requests. Defaults to 500ms for write requests.
+  logSlowRequestsWriteMs?: number;
+}
+
+export function setupLogging(options?: LoggingOptions) {
+  winstonLogger.clear();
+  if (!options?.disableConsoleLogging) {
+    const formats: any[] = [winston.format.simple()];
+    if (!options?.disableConsoleColors) {
+      formats.push(winston.format.colorize());
+    }
+    formats.push(winston.format.printf(printf(options?.showConsoleTimestamps)));
+    winstonLogger.add(
+      new winston.transports.Console({
+        debugStdout: !options?.level || options?.level === "debug",
+        format: winston.format.combine(...formats),
+        level: options?.level ?? "debug",
+      })
+    );
+  }
+  if (!options?.disableFileLogging) {
+    const logDirectory = options?.logDirectory ?? "./log";
+    if (!fs.existsSync(logDirectory)) {
+      fs.mkdirSync(logDirectory, {recursive: true});
+    }
+
+    const FILE_LOG_DEFAULTS = {
+      colorize: false,
+      compress: true,
+      dirname: logDirectory,
+      format: winston.format.simple(),
+      // 30 days of retention
+      maxFiles: 30,
+      // 50MB max file size
+      maxSize: 1024 * 1024 * 50,
+      // Only readable by server user
+      options: {mode: 0o600},
+    };
+
+    winstonLogger.add(
+      new winston.transports.Stream({
+        ...FILE_LOG_DEFAULTS,
+        handleExceptions: true,
+        level: "error",
+        // Use stream so we can open log in append mode rather than overwriting.
+        stream: fs.createWriteStream("error.log", {flags: "a"}),
+      })
+    );
+
+    winstonLogger.add(
+      new winston.transports.Stream({
+        ...FILE_LOG_DEFAULTS,
+        level: "info",
+        // Use stream so we can open log in append mode rather than overwriting.
+        stream: fs.createWriteStream("out.log", {flags: "a"}),
+      })
+    );
+    if (!options?.level || options?.level === "debug") {
+      winstonLogger.add(
+        new winston.transports.Stream({
+          ...FILE_LOG_DEFAULTS,
+          level: "debug",
+          // Use stream so we can open log in append mode rather than overwriting.
+          stream: fs.createWriteStream("debug.log", {flags: "a"}),
+        })
+      );
+    }
+  }
+
+  if (options?.transports) {
+    for (const transport of options.transports) {
+      winstonLogger.add(transport);
+    }
+  }
+}