kayvee 3.17.0 → 4.0.0

package/lib/middleware.ts

@@ -1,317 +0,0 @@
-/**
- * Module dependencies.
- * @private
- */
-
-var fs = require("fs");
-var path = require("path");
-
-var kayvee = require("../lib/kayvee");
-var KayveeLogger = require("../lib/logger/logger");
-var morgan = require("morgan");
-var _ = require("underscore");
-
-/**
- * all relative files path in a directory
- */
-
-function walkDirSync(dir, files = []) {
-  const list = fs.readdirSync(dir);
-  list.forEach((file) => {
-    const f = path.join(dir, file);
-    if (fs.statSync(path.join(dir, file)).isDirectory()) {
-      walkDirSync(f, files);
-    } else {
-      files.push(f);
-    }
-  });
-  return files.map((f) => path.relative(dir, f));
-}
-
-/**
- * returns a middleware function that checks if path exists in dir.
- *
- * Files in the directory are prefixed by base_path and compared to
- * req.path
- */
-
-function skip_path(dir, base_path = "/") {
-  let files = walkDirSync(dir);
-  files = files.map((file) => path.join(base_path, file));
-  console.error(
-    `KayveeMiddleware: Skipping successful requests for files in ${dir} at ${base_path}`,
-  );
-  return (req, res) => _(files).contains(req.path) && res.statusCode < 400;
-}
-
-/**
- * request path
- */
-
-function getBaseUrl(req) {
-  var url = req.originalUrl || req.url;
-  var parsed = require("url").parse(url, true);
-  return parsed.pathname;
-}
-
-/**
- * request query params
- */
-
-function getQueryParams(req) {
-  var url = req.originalUrl || req.url;
-  var parsed = require("url").parse(url, true);
-  var parsedQueryString = require("qs").parse(parsed.search, {
-    allowPrototypes: false,
-    ignoreQueryPrefix: true,
-  });
-  return `?${require("qs").stringify(parsedQueryString)}`;
-}
-
-/**
- * response size
- */
-
-function getResponseSize(res) {
-  var result = undefined;
-  var headers = res.headers || res._headers;
-  if (headers && headers["content-length"]) {
-    result = Number(headers["content-length"]);
-  } else if (res.data) {
-    result = res.data.length;
-  }
-  return result;
-}
-
-/**
- * response time in nanoseconds
- */
-
-function getResponseTimeNs(req, res) {
-  if (!req._startAt || !res._startAt) {
-    // missing request and/or response start time
-    return undefined;
-  }
-
-  // calculate diff
-  var ns = (res._startAt[0] - req._startAt[0]) * 1e9 + (res._startAt[1] - req._startAt[1]);
-  return ns;
-}
-
-/**
- * IP address that sent the request.
- *
- * `req.ip` is defined in Express: http://expressjs.com/en/api.html#req.ip
- */
-function getIp(req) {
-  var remoteAddress = req.connection ? req.connection.remoteAddress : undefined;
-  return req.ip || remoteAddress;
-}
-
-/**
- * Log level
- */
-function getLogLevel(req, res) {
-  const statusCode = res.statusCode;
-  let result;
-  if (statusCode >= 499) {
-    result = KayveeLogger.Error;
-  } else {
-    result = KayveeLogger.Info;
-  }
-  return result;
-}
-
-/*
- * Default handlers
- */
-var defaultHandlers = [
-  // Request method
-  (req) => ({ method: req.method }),
-  // Path (URL without query params)
-  (req) => ({ path: getBaseUrl(req) }),
-  // Query params
-  (req) => ({ params: getQueryParams(req) }),
-  // Response size
-  (req, res) => ({ "response-size": getResponseSize(res) }),
-  // Response time (ns)
-  (req, res) => ({ "response-time": getResponseTimeNs(req, res) }),
-  // Status code
-  (req, res) => ({ "status-code": res.statusCode }),
-  // Ip address
-  (req) => ({ ip: getIp(req) }),
-  // Via -- what library/code produced this log?
-  () => ({ via: "kayvee-middleware" }),
-
-  // Kayvee's reserved fields
-  // Log level
-  (req, res) => ({ level: getLogLevel(req, res) }),
-  // Source -- which app emitted this log?
-  // -> Gets passed in among `options` during library initialization
-  // Title
-  () => ({ title: "request-finished" }),
-];
-
-const defaultContextHandlers = [];
-
-function handlerData(handlers, ...args) {
-  const data = {};
-  handlers.forEach((h) => {
-    try {
-      const handler_data = h(...args);
-      _.extend(data, handler_data);
-    } catch (e) {
-      // ignore invalid handler
-    }
-  });
-  return data;
-}
-
-class ContextLogger {
-  logger = null;
-  handlers = [];
-  args = [];
-
-  constructor(logger, handlers, ...args) {
-    this.logger = logger;
-    this.handlers = handlers;
-    this.args = args;
-  }
-
-  _contextualData(data) {
-    return _.extend(handlerData(this.handlers, ...this.args), data);
-  }
-}
-
-for (const func of KayveeLogger.LEVELS) {
-  ContextLogger.prototype[func] = function (title) {
-    this[`${func}D`](title, {});
-  };
-  ContextLogger.prototype[`${func}D`] = function (title, data) {
-    this.logger[`${func}D`](title, this._contextualData(data));
-  };
-}
-
-for (const func of KayveeLogger.METRICS) {
-  ContextLogger.prototype[func] = function (title, value) {
-    this[`${func}D`](title, value, {});
-  };
-  ContextLogger.prototype[`${func}D`] = function (title, value, data) {
-    this.logger[`${func}D`](title, value, this._contextualData(data));
-  };
-}
-
-/*
- * User configuration is passed via an `options` object.
- * Results from configuration are prioritized such that (`base_handlers` > `handlers` > `headers`).
- *
- * // `headers` - logs these request headers, if they exist
- *
- * headers: e.g. ['X-Request-Id', 'Host']
- *
- * // `handlers` - an array of functions of that return dicts to be logged.
- *
- * handlers: e.g. [function(request, response) { return {"key":"val"}]
- *
- * // `base_handlers` - an array of functions of that return dicts to be logged.
- * // Barring exceptional circumstances, `base_handlers` should not be overriden by the user.
- * // `base_handlers` defaults to a core set of handlers to run... see `defaultHandlers`.
- * //
- * // Separating `base_handlers` from `handlers` is done to ensure that reserved keys
- * // don't accidentally get overriden by custom handlers. This can now only happen if
- * // the user explicitly overrides `base_handlers`.
- *
- * base_handlers: e.g. [function(request, response) { return {"key":"val"}]
- *
- */
-
-var formatLine = (options_arg) => {
-  var options = options_arg || {};
-
-  // `source` is the one required field
-  if (!options.source) {
-    throw Error("Missing required config for 'source' in Kayvee middleware 'options'");
-  }
-
-  const router = KayveeLogger.getGlobalRouter();
-
-  return (tokens, req, res) => {
-    // Build a dict of data to log
-    var data = { _kvmeta: undefined }; // Adding _kvmeta here to make typescript compile happy
-
-    // Add user-configured request headers
-    var custom_headers = options.headers || [];
-    var header_data = {};
-    custom_headers.forEach((h) => {
-      // Header field names are case insensitive, so let's be consistent
-      var lc = h.toLowerCase();
-      header_data[lc] = req.headers[lc];
-    });
-    _.extend(data, header_data);
-
-    // Run user-configured handlers; add custom data
-    var custom_handlers = options.handlers || [];
-
-    // Allow user to override `base_handlers`; provide sane default set of handlers
-    var base_handlers = options.base_handlers || defaultHandlers;
-    base_handlers = base_handlers.concat([() => ({ source: options.source })]);
-
-    // Execute custom-handlers THEN base-handlers
-    const all_handlers = custom_handlers.concat(base_handlers);
-    _.extend(data, handlerData(all_handlers, req, res));
-
-    if (router) {
-      data._kvmeta = router.route(data);
-    }
-
-    return kayvee.format(data);
-  };
-};
-
-const defaultContextLoggerOpts = {
-  enabled: true,
-  handlers: defaultContextHandlers,
-};
-
-/**
- * Module exports.
- * @public
- */
-
-if (process.env.NODE_ENV === "test") {
-  module.exports = (clever_options, morgan_options = { skip: null }) => {
-    if (clever_options.ignore_dir) {
-      morgan_options.skip = skip_path(
-        clever_options.ignore_dir.directory,
-        clever_options.ignore_dir.path,
-      );
-    }
-    return morgan(formatLine(clever_options), morgan_options);
-  };
-  module.exports.ContextLogger = ContextLogger;
-} else {
-  module.exports = (clever_options, context_logger_options = defaultContextLoggerOpts) => {
-    // `source` is the one required field
-    if (!clever_options.source) {
-      throw new Error("Missing required config for 'source' in Kayvee middleware 'options'");
-    }
-    const logger = new KayveeLogger(clever_options.source);
-    const morgan_options = {
-      stream: process.stderr,
-      skip: null,
-    };
-    if (clever_options.ignore_dir) {
-      morgan_options.skip = skip_path(
-        clever_options.ignore_dir.directory,
-        clever_options.ignore_dir.path,
-      );
-    }
-    const morgan_logger = morgan(formatLine(clever_options), morgan_options);
-    return (req, res, next) => {
-      if (context_logger_options.enabled) {
-        req.log = new ContextLogger(logger, context_logger_options.handlers, req);
-      }
-      morgan_logger(req, res, next);
-    };
-  };
-}

package/lib/router/index.ts

@@ -1,234 +0,0 @@
-var fs = require("fs");
-var jsonschema = require("jsonschema");
-var schema = require("./schema_definitions");
-var yaml = require("js-yaml");
-var _ = require("underscore");
-
-var packageJson = require("../../package.json");
-const kvVersion = packageJson.version;
-const teamName = process.env._TEAM_OWNER || "UNSET";
-
-const reEnvvarTokens = new RegExp("\\$\\{(.+?)\\}", "g");
-const reFieldTokens = new RegExp("%\\{(.+?)\\}", "g");
-
-// For performance reason this code is intentionally redundant and not-inlined.
-// Removing redundancy and inlining this function some how makes performance worst.
-function substituteEnvVars(obj, subber) {
-  const rtn = {};
-  const replacer = (s) => s.replace(reEnvvarTokens, (__, p1) => subber(p1));
-
-  for (const key in obj) {
-    const val = obj[key];
-
-    if (Array.isArray(val)) {
-      const updatedVals = Array(val.length);
-      for (let i = 0; i < val.length; i++) {
-        updatedVals[i] = replacer(val[i]);
-      }
-      rtn[key] = updatedVals;
-    } else {
-      rtn[key] = replacer(val);
-    }
-  }
-
-  return rtn;
-}
-
-function deepKey(obj, key) {
-  const path = key.split(".");
-
-  let idx = 0;
-  let val = obj;
-  do {
-    val = val[path[idx++]];
-  } while (val && idx < path.length);
-
-  return val;
-}
-
-function fieldMatches(obj, field, values) {
-  const val = obj[field] || deepKey(obj, field);
-
-  if (val == null || val === "") {
-    return false;
-  }
-
-  if (values[0] === "*") {
-    return true;
-  }
-
-  for (let i = 0; i < values.length; i++) {
-    if (values[i] === val) {
-      return true;
-    }
-  }
-
-  return false;
-}
-
-class Rule {
-  name = null;
-  matchers = null;
-  output = null;
-
-  constructor(name, matchers, output) {
-    this.name = name;
-    this.matchers = matchers;
-
-    const envMissing = [];
-    this.output = substituteEnvVars(output, (k) => {
-      const val = process.env[k];
-      if (val == null) {
-        envMissing.push(k);
-      }
-      return val;
-    });
-
-    if (envMissing.length > 0) {
-      throw new Error(`Missing env var(s): ${envMissing.join(", ")}`);
-    }
-
-    Object.keys(matchers).forEach((field) => {
-      const fieldVals = matchers[field];
-      if (fieldVals.indexOf("*") !== -1 && fieldVals.length > 1) {
-        throw new Error(
-          `Invalid matcher values in ${name}.${field}.\n` +
-            "Wildcard matcher can't co-exist with other matchers.",
-        );
-      }
-    });
-
-    if (this.output.type === "alerts" || this.output.type === "metrics") {
-      this.output.value_field = this.output.value_field || "value";
-    }
-
-    this.output.rule = this.name;
-  }
-
-  // matches returns true if `msg` matches against this rule
-  matches(msg) {
-    for (const field in this.matchers) {
-      if (!fieldMatches(msg, field, this.matchers[field])) {
-        return false;
-      }
-    }
-
-    return true;
-  }
-
-  // returns the output with kv substitutions performed
-  outputFor(msg) {
-    const rtn = {};
-    const subst = (__, k) => msg[k] || deepKey(msg, k) || "KEY_NOT_FOUND";
-    const replacer = (s) => s.replace(reFieldTokens, subst);
-
-    for (const key in this.output) {
-      const val = this.output[key];
-
-      if (Array.isArray(val)) {
-        const updatedVals = Array(val.length);
-        for (let i = 0; i < val.length; i++) {
-          updatedVals[i] = replacer(val[i]);
-        }
-        rtn[key] = updatedVals;
-      } else {
-        rtn[key] = replacer(val);
-      }
-    }
-
-    return rtn;
-  }
-}
-
-// validateKVConfig ensures that `routes` matches the config schema. We have this
-// function instead of just doing a plain jsonschema.validate in order to get
-// better error messages for the "output" object (by default jsonschema would
-// just tell you that the output block doesn't match any of the known output
-// formats, but won't tell you what's wrong because it doesn't let you
-// condition on the output.type property).
-function validateKVConfig(config) {
-  const validator = new jsonschema.Validator();
-  const results = validator.validate(config, schema);
-
-  return {
-    valid: results.valid,
-    errors: results.errors.map((err) => err.stack),
-  };
-}
-
-// parseConfig parses and validates the configuration passed as a string. It
-// returns an object of the form {valid, rules, errors}, where valid is true if
-// it was successfully parsed, rules is an array of rules, and errors is an
-// array of errors.
-function parseConfig(fileString) {
-  let config;
-  try {
-    config = yaml.safeLoad(fileString);
-  } catch (e) {
-    return { valid: false, rules: [], errors: [e] };
-  }
-  const validateRes = validateKVConfig(config);
-  if (!validateRes.valid) {
-    return _.assign(validateRes, { rules: [] });
-  }
-  try {
-    const rulesObj = _.mapObject(
-      config.routes,
-      (elem, name) => new Rule(name, elem.matchers, elem.output),
-    );
-    const rules = _.values(rulesObj);
-    return { valid: true, rules, errors: [] };
-  } catch (e) {
-    return { valid: false, rules: [], errors: [e] };
-  }
-}
-
-class Router {
-  rules = null;
-
-  constructor(rules) {
-    this.rules = rules || [];
-  }
-
-  // loadConfig reads in the config located at `filename` and sets the routing
-  // rules to what it finds there. It should be a YAML-formatted file with
-  // routing rules placed under the `routes` key on the root object.
-  loadConfig(filename) {
-    const data = fs.readFileSync(filename, "utf8");
-    this._loadConfigString(data);
-  }
-
-  _loadConfigString(configStr) {
-    const parsedRules = parseConfig(configStr);
-    if (!parsedRules.valid) {
-      throw new Error(parsedRules.errors);
-    }
-    this.rules = parsedRules.rules;
-  }
-
-  // route matches the log line `msg` against all loaded rules and returns a
-  // metadata object describing the outputs it should be sent to based on that
-  // matching. logger.ts will attach this to log lines under the `_kvmeta`
-  // property.
-  route(msg) {
-    const outputs = [];
-    for (let i = 0; i < this.rules.length; i++) {
-      const rule = this.rules[i];
-      if (rule.matches(msg)) {
-        outputs.push(rule.outputFor(msg));
-      }
-    }
-
-    return {
-      team: teamName,
-      kv_version: kvVersion,
-      kv_language: "js",
-      routes: outputs,
-    };
-  }
-}
-
-module.exports = {
-  Router,
-  Rule,
-};

package/lib/router/schema_definitions.json

@@ -1,158 +0,0 @@
-{
-  "description": "Last modified: 02/08/2017",
-  "required": ["routes"],
-  "properties": {
-    "routes": { "$ref": "#/definitions/routes" }
-  },
-  "definitions": {
-    "routes": {
-      "type": "object",
-      "additionalProperties": false,
-      "patternProperties": {
-        "^[a-zA-Z0-9-_]+$": { "$ref": "#/definitions/rule" }
-      }
-    },
-    "rule": {
-      "title": "Rule",
-      "type": "object",
-      "required": ["matchers", "output"],
-      "additionalProperties": false,
-      "properties": {
-        "matchers": { "$ref": "#/definitions/matchers" },
-        "output": { "$ref": "#/definitions/output" }
-      }
-    },
-    "matchers": {
-      "type": "object",
-      "minProperties": 1,
-      "additionalProperties": false,
-      "patternProperties": {
-        "^[^%\\${}]+$": { "$ref": "#/definitions/matcherArr" }
-      }
-    },
-    "output": {
-      "type": "object",
-      "oneOf": [
-        { "$ref": "#/definitions/metricsOutput" },
-        { "$ref": "#/definitions/alertsOutput" },
-        { "$ref": "#/definitions/analyticsOutput" },
-        { "$ref": "#/definitions/notificationsOutput" }
-      ]
-    },
-    "metricsOutput": {
-      "title": "Metrics Output",
-      "type": "object",
-      "oneOf": [
-        {
-          "additionalProperties": false,
-          "required": ["type", "series", "dimensions"],
-          "properties": {
-            "type": { "type": "string", "pattern": "^metrics$" },
-            "series": { "$ref": "#/definitions/envVarSubstValue" },
-            "dimensions": { "$ref": "#/definitions/flatValueArr" }
-          }
-        },
-        {
-          "additionalProperties": false,
-          "required": ["type", "series", "dimensions", "value_field"],
-          "properties": {
-            "type": { "type": "string", "pattern": "^metrics$" },
-            "series": { "$ref": "#/definitions/envVarSubstValue" },
-            "dimensions": { "$ref": "#/definitions/flatValueArr" },
-            "value_field": { "$ref": "#/definitions/flatValue" }
-          }
-        }
-      ]
-    },
-    "alertsOutput": {
-      "type": "object",
-      "oneOf": [
-        {
-          "additionalProperties": false,
-          "required": ["type", "series", "dimensions", "stat_type"],
-          "properties": {
-            "type": { "type": "string", "pattern": "^alerts$" },
-            "series": { "$ref": "#/definitions/envVarSubstValue" },
-            "dimensions": {
-              "type": "array",
-              "uniqueItems": true,
-              "items": { "$ref": "#/definitions/flatValue" }
-            },
-            "stat_type": { "type": "string", "enum": ["counter", "gauge"] }
-          }
-        },
-        {
-          "additionalProperties": false,
-          "required": ["type", "series", "dimensions", "stat_type", "value_field"],
-          "properties": {
-            "type": { "type": "string", "pattern": "^alerts$" },
-            "series": { "$ref": "#/definitions/envVarSubstValue" },
-            "dimensions": {
-              "type": "array",
-              "uniqueItems": true,
-              "items": { "$ref": "#/definitions/flatValue" }
-            },
-            "stat_type": { "type": "string", "enum": ["counter", "gauge"] },
-            "value_field": { "$ref": "#/definitions/flatValue" }
-          }
-        }
-      ]
-    },
-    "analyticsOutput": {
-      "type": "object",
-      "additionalProperties": false,
-      "required": ["type", "series"],
-      "properties": {
-        "type": {
-          "type": "string",
-          "pattern": "^analytics$"
-        },
-        "series": { "$ref": "#/definitions/envVarSubstValue" }
-      }
-    },
-    "notificationsOutput": {
-      "type": "object",
-      "additionalProperties": false,
-      "required": ["type", "channel", "icon", "message", "user"],
-      "properties": {
-        "type": {
-          "type": "string",
-          "pattern": "^notifications$"
-        },
-        "channel": { "$ref": "#/definitions/kvSubstValue" },
-        "icon": { "$ref": "#/definitions/envVarSubstValue" },
-        "message": { "$ref": "#/definitions/kvSubstValue" },
-        "user": { "$ref": "#/definitions/envVarSubstValue" }
-      }
-    },
-    "flatValue": {
-      "type": "string",
-      "pattern": "^[^%\\${}]+$"
-    },
-    "flatValueArr": {
-      "type": "array",
-      "minItems": 1,
-      "uniqueItems": true,
-      "items": { "$ref": "#/definitions/flatValue" }
-    },
-    "matcherArr": {
-      "type": "array",
-      "minItems": 1,
-      "uniqueItems": true,
-      "items": {
-        "oneOf": [
-          { "$ref": "#/definitions/flatValue" },
-          { "type": "boolean" }
-        ]
-      }
-    },
-    "envVarSubstValue": {
-      "type": "string",
-      "pattern": "^([^\\$%{}]|\\$\\{[^%\\${}]+\\})+$"
-    },
-    "kvSubstValue": {
-      "type": "string",
-      "pattern": "^([^\\$%{}]|\\$\\{[^%\\${}]+\\}|%\\{[^%\\${}]+\\})+$"
-    }
-  }
-}