apitally 0.10.0 → 0.11.1

package/dist/express/middleware.cjs

@@ -39,7 +39,7 @@ module.exports = __toCommonJS(middleware_exports);
 var import_perf_hooks = require("perf_hooks");
 
 // src/common/client.ts
-var import_crypto3 = require("crypto");
+var import_crypto5 = require("crypto");
 var import_fetch_retry = __toESM(require("fetch-retry"), 1);
 
 // src/common/consumerRegistry.ts
@@ -208,8 +208,405 @@ var _RequestCounter = class _RequestCounter {
 __name(_RequestCounter, "RequestCounter");
 var RequestCounter = _RequestCounter;
 
-// src/common/serverErrorCounter.ts
+// src/common/requestLogger.ts
+var import_buffer2 = require("buffer");
+var import_crypto2 = require("crypto");
+var import_fs2 = require("fs");
+var import_os2 = require("os");
+var import_path2 = require("path");
+
+// src/common/tempGzipFile.ts
+var import_buffer = require("buffer");
 var import_crypto = require("crypto");
+var import_fs = require("fs");
+var import_os = require("os");
+var import_path = require("path");
+var import_zlib = require("zlib");
+var _TempGzipFile = class _TempGzipFile {
+  uuid;
+  filePath;
+  gzip;
+  writeStream;
+  readyPromise;
+  constructor() {
+    this.uuid = (0, import_crypto.randomUUID)();
+    this.filePath = (0, import_path.join)((0, import_os.tmpdir)(), `apitally-${this.uuid}.gz`);
+    this.writeStream = (0, import_fs.createWriteStream)(this.filePath);
+    this.readyPromise = new Promise((resolve, reject) => {
+      this.writeStream.once("ready", resolve);
+      this.writeStream.once("error", reject);
+    });
+    this.gzip = (0, import_zlib.createGzip)();
+    this.gzip.pipe(this.writeStream);
+  }
+  get size() {
+    return this.writeStream.bytesWritten;
+  }
+  async writeLine(data) {
+    await this.readyPromise;
+    return new Promise((resolve, reject) => {
+      this.gzip.write(import_buffer.Buffer.concat([
+        data,
+        import_buffer.Buffer.from("\n")
+      ]), (error) => {
+        if (error) {
+          reject(error);
+        } else {
+          resolve();
+        }
+      });
+    });
+  }
+  async getContent() {
+    return new Promise((resolve, reject) => {
+      (0, import_fs.readFile)(this.filePath, (error, data) => {
+        if (error) {
+          reject(error);
+        } else {
+          resolve(data);
+        }
+      });
+    });
+  }
+  async close() {
+    return new Promise((resolve) => {
+      this.gzip.end(() => {
+        resolve();
+      });
+    });
+  }
+  async delete() {
+    await this.close();
+    (0, import_fs.unlinkSync)(this.filePath);
+  }
+};
+__name(_TempGzipFile, "TempGzipFile");
+var TempGzipFile = _TempGzipFile;
+
+// src/common/requestLogger.ts
+var MAX_BODY_SIZE = 5e4;
+var MAX_FILE_SIZE = 1e6;
+var MAX_FILES = 50;
+var MAX_PENDING_WRITES = 100;
+var BODY_TOO_LARGE = import_buffer2.Buffer.from("<body too large>");
+var BODY_MASKED = import_buffer2.Buffer.from("<masked>");
+var MASKED = "******";
+var ALLOWED_CONTENT_TYPES = [
+  "application/json",
+  "text/plain"
+];
+var EXCLUDE_PATH_PATTERNS = [
+  /\/_?healthz?$/i,
+  /\/_?health[_-]?checks?$/i,
+  /\/_?heart[_-]?beats?$/i,
+  /\/ping$/i,
+  /\/ready$/i,
+  /\/live$/i
+];
+var EXCLUDE_USER_AGENT_PATTERNS = [
+  /health[-_ ]?check/i,
+  /microsoft-azure-application-lb/i,
+  /googlehc/i,
+  /kube-probe/i
+];
+var MASK_QUERY_PARAM_PATTERNS = [
+  /auth/i,
+  /api-?key/i,
+  /secret/i,
+  /token/i,
+  /password/i,
+  /pwd/i
+];
+var MASK_HEADER_PATTERNS = [
+  /auth/i,
+  /api-?key/i,
+  /secret/i,
+  /token/i,
+  /cookie/i
+];
+var DEFAULT_CONFIG = {
+  enabled: false,
+  logQueryParams: true,
+  logRequestHeaders: false,
+  logRequestBody: false,
+  logResponseHeaders: true,
+  logResponseBody: false,
+  maskQueryParams: [],
+  maskHeaders: [],
+  excludePaths: []
+};
+var _RequestLogger = class _RequestLogger {
+  config;
+  enabled;
+  suspendUntil = null;
+  pendingWrites = [];
+  currentFile = null;
+  files = [];
+  maintainIntervalId;
+  constructor(config) {
+    this.config = {
+      ...DEFAULT_CONFIG,
+      ...config
+    };
+    this.enabled = this.config.enabled && checkWritableFs();
+    if (this.enabled) {
+      this.maintainIntervalId = setInterval(() => {
+        this.maintain();
+      }, 1e3);
+    }
+  }
+  shouldExcludePath(urlPath) {
+    const patterns = [
+      ...this.config.excludePaths,
+      ...EXCLUDE_PATH_PATTERNS
+    ];
+    return matchPatterns(urlPath, patterns);
+  }
+  shouldExcludeUserAgent(userAgent) {
+    return userAgent ? matchPatterns(userAgent, EXCLUDE_USER_AGENT_PATTERNS) : false;
+  }
+  shouldMaskQueryParam(name) {
+    const patterns = [
+      ...this.config.maskQueryParams,
+      ...MASK_QUERY_PARAM_PATTERNS
+    ];
+    return matchPatterns(name, patterns);
+  }
+  shouldMaskHeader(name) {
+    const patterns = [
+      ...this.config.maskHeaders,
+      ...MASK_HEADER_PATTERNS
+    ];
+    return matchPatterns(name, patterns);
+  }
+  hasSupportedContentType(headers) {
+    var _a2;
+    const contentType = (_a2 = headers.find(([k]) => k.toLowerCase() === "content-type")) == null ? void 0 : _a2[1];
+    return contentType !== void 0 && ALLOWED_CONTENT_TYPES.some((t) => contentType.startsWith(t));
+  }
+  maskQueryParams(search) {
+    const params = new URLSearchParams(search);
+    for (const [key] of params) {
+      if (this.shouldMaskQueryParam(key)) {
+        params.set(key, MASKED);
+      }
+    }
+    return params.toString();
+  }
+  maskHeaders(headers) {
+    return headers.map(([k, v]) => [
+      k,
+      this.shouldMaskHeader(k) ? MASKED : v
+    ]);
+  }
+  logRequest(request, response) {
+    var _a2, _b, _c;
+    if (!this.enabled || this.suspendUntil !== null) return;
+    const url = new URL(request.url);
+    const path = request.path ?? url.pathname;
+    const userAgent = (_a2 = request.headers.find(([k]) => k.toLowerCase() === "user-agent")) == null ? void 0 : _a2[1];
+    if (this.shouldExcludePath(path) || this.shouldExcludeUserAgent(userAgent) || (((_c = (_b = this.config).excludeCallback) == null ? void 0 : _c.call(_b, request, response)) ?? false)) {
+      return;
+    }
+    url.search = this.config.logQueryParams ? this.maskQueryParams(url.search) : "";
+    request.url = url.toString();
+    request.headers = this.config.logRequestHeaders ? this.maskHeaders(request.headers) : [];
+    response.headers = this.config.logResponseHeaders ? this.maskHeaders(response.headers) : [];
+    if (!this.config.logRequestBody || !this.hasSupportedContentType(request.headers)) {
+      request.body = void 0;
+    } else if (request.body) {
+      if (request.body.length > MAX_BODY_SIZE) {
+        request.body = BODY_TOO_LARGE;
+      } else if (this.config.maskRequestBodyCallback) {
+        try {
+          request.body = this.config.maskRequestBodyCallback(request) ?? BODY_MASKED;
+          if (request.body.length > MAX_BODY_SIZE) {
+            request.body = BODY_TOO_LARGE;
+          }
+        } catch {
+          request.body = void 0;
+        }
+      }
+    }
+    if (!this.config.logResponseBody || !this.hasSupportedContentType(response.headers)) {
+      response.body = void 0;
+    } else if (response.body) {
+      if (response.body.length > MAX_BODY_SIZE) {
+        response.body = BODY_TOO_LARGE;
+      } else if (this.config.maskResponseBodyCallback) {
+        try {
+          response.body = this.config.maskResponseBodyCallback(request, response) ?? BODY_MASKED;
+          if (response.body.length > MAX_BODY_SIZE) {
+            response.body = BODY_TOO_LARGE;
+          }
+        } catch {
+          response.body = void 0;
+        }
+      }
+    }
+    const item = {
+      uuid: (0, import_crypto2.randomUUID)(),
+      request: skipEmptyValues(request),
+      response: skipEmptyValues(response)
+    };
+    [
+      item.request.body,
+      item.response.body
+    ].forEach((body) => {
+      if (body) {
+        body.toJSON = function() {
+          return this.toString("base64");
+        };
+      }
+    });
+    this.pendingWrites.push(JSON.stringify(item));
+    if (this.pendingWrites.length > MAX_PENDING_WRITES) {
+      this.pendingWrites.shift();
+    }
+  }
+  async writeToFile() {
+    if (!this.enabled || this.pendingWrites.length === 0) {
+      return;
+    }
+    if (!this.currentFile) {
+      this.currentFile = new TempGzipFile();
+    }
+    while (this.pendingWrites.length > 0) {
+      const item = this.pendingWrites.shift();
+      if (item) {
+        await this.currentFile.writeLine(import_buffer2.Buffer.from(item));
+      }
+    }
+  }
+  getFile() {
+    return this.files.shift();
+  }
+  retryFileLater(file) {
+    this.files.unshift(file);
+  }
+  async rotateFile() {
+    if (this.currentFile) {
+      await this.currentFile.close();
+      this.files.push(this.currentFile);
+      this.currentFile = null;
+    }
+  }
+  async maintain() {
+    await this.writeToFile();
+    if (this.currentFile && this.currentFile.size > MAX_FILE_SIZE) {
+      await this.rotateFile();
+    }
+    while (this.files.length > MAX_FILES) {
+      const file = this.files.shift();
+      file == null ? void 0 : file.delete();
+    }
+    if (this.suspendUntil !== null && this.suspendUntil < Date.now()) {
+      this.suspendUntil = null;
+    }
+  }
+  async clear() {
+    this.pendingWrites = [];
+    await this.rotateFile();
+    this.files.forEach((file) => {
+      file.delete();
+    });
+    this.files = [];
+  }
+  async close() {
+    this.enabled = false;
+    await this.clear();
+    if (this.maintainIntervalId) {
+      clearInterval(this.maintainIntervalId);
+    }
+  }
+};
+__name(_RequestLogger, "RequestLogger");
+var RequestLogger = _RequestLogger;
+function convertHeaders(headers) {
+  if (headers instanceof Headers) {
+    return Array.from(headers.entries());
+  }
+  return Object.entries(headers).flatMap(([key, value]) => {
+    if (value === void 0) {
+      return [];
+    }
+    if (Array.isArray(value)) {
+      return value.map((v) => [
+        key,
+        v
+      ]);
+    }
+    return [
+      [
+        key,
+        value.toString()
+      ]
+    ];
+  });
+}
+__name(convertHeaders, "convertHeaders");
+function convertBody(body, contentType) {
+  if (!body || !contentType) {
+    return;
+  }
+  try {
+    if (contentType.startsWith("application/json")) {
+      if (isValidJsonString(body)) {
+        return import_buffer2.Buffer.from(body);
+      } else {
+        return import_buffer2.Buffer.from(JSON.stringify(body));
+      }
+    }
+    if (contentType.startsWith("text/") && typeof body === "string") {
+      return import_buffer2.Buffer.from(body);
+    }
+  } catch (error) {
+    return;
+  }
+}
+__name(convertBody, "convertBody");
+function isValidJsonString(body) {
+  if (typeof body !== "string") {
+    return false;
+  }
+  try {
+    JSON.parse(body);
+    return true;
+  } catch {
+    return false;
+  }
+}
+__name(isValidJsonString, "isValidJsonString");
+function matchPatterns(value, patterns) {
+  return patterns.some((pattern) => {
+    return pattern.test(value);
+  });
+}
+__name(matchPatterns, "matchPatterns");
+function skipEmptyValues(data) {
+  return Object.fromEntries(Object.entries(data).filter(([_, v]) => {
+    if (v == null || Number.isNaN(v)) return false;
+    if (Array.isArray(v) || import_buffer2.Buffer.isBuffer(v) || typeof v === "string") {
+      return v.length > 0;
+    }
+    return true;
+  }));
+}
+__name(skipEmptyValues, "skipEmptyValues");
+function checkWritableFs() {
+  try {
+    const testPath = (0, import_path2.join)((0, import_os2.tmpdir)(), `apitally-${(0, import_crypto2.randomUUID)()}`);
+    (0, import_fs2.writeFileSync)(testPath, "test");
+    (0, import_fs2.unlinkSync)(testPath);
+    return true;
+  } catch (error) {
+    return false;
+  }
+}
+__name(checkWritableFs, "checkWritableFs");
+
+// src/common/serverErrorCounter.ts
+var import_crypto3 = require("crypto");
 var MAX_MSG_LENGTH = 2048;
 var MAX_STACKTRACE_LENGTH = 65536;
 var _ServerErrorCounter = class _ServerErrorCounter {
@@ -261,7 +658,7 @@ var _ServerErrorCounter = class _ServerErrorCounter {
       serverError.msg.trim(),
       serverError.traceback.trim()
     ].join("|");
-    return (0, import_crypto.createHash)("md5").update(hashInput).digest("hex");
+    return (0, import_crypto3.createHash)("md5").update(hashInput).digest("hex");
   }
   getTruncatedMessage(msg) {
     msg = msg.trim();
@@ -307,7 +704,7 @@ __name(_ServerErrorCounter, "ServerErrorCounter");
 var ServerErrorCounter = _ServerErrorCounter;
 
 // src/common/validationErrorCounter.ts
-var import_crypto2 = require("crypto");
+var import_crypto4 = require("crypto");
 var _ValidationErrorCounter = class _ValidationErrorCounter {
   errorCounts;
   errorDetails;
@@ -351,7 +748,7 @@ var _ValidationErrorCounter = class _ValidationErrorCounter {
       validationError.msg.trim(),
       validationError.type
     ].join("|");
-    return (0, import_crypto2.createHash)("md5").update(hashInput).digest("hex");
+    return (0, import_crypto4.createHash)("md5").update(hashInput).digest("hex");
   }
 };
 __name(_ValidationErrorCounter, "ValidationErrorCounter");
@@ -380,11 +777,12 @@ var _ApitallyClient = class _ApitallyClient {
   startupData;
   startupDataSent = false;
   requestCounter;
+  requestLogger;
   validationErrorCounter;
   serverErrorCounter;
   consumerRegistry;
   logger;
-  constructor({ clientId, env = "dev", logger }) {
+  constructor({ clientId, env = "dev", requestLoggingConfig, logger }) {
     if (_ApitallyClient.instance) {
       throw new Error("Apitally client is already initialized");
     }
@@ -397,9 +795,10 @@ var _ApitallyClient = class _ApitallyClient {
     _ApitallyClient.instance = this;
     this.clientId = clientId;
     this.env = env;
-    this.instanceUuid = (0, import_crypto3.randomUUID)();
+    this.instanceUuid = (0, import_crypto5.randomUUID)();
     this.syncDataQueue = [];
     this.requestCounter = new RequestCounter();
+    this.requestLogger = new RequestLogger(requestLoggingConfig);
     this.validationErrorCounter = new ValidationErrorCounter();
     this.serverErrorCounter = new ServerErrorCounter();
     this.consumerRegistry = new ConsumerRegistry();
@@ -421,6 +820,8 @@ var _ApitallyClient = class _ApitallyClient {
   async handleShutdown() {
     this.stopSync();
     await this.sendSyncData();
+    await this.sendLogData();
+    await this.requestLogger.close();
     _ApitallyClient.instance = void 0;
   }
   getHubUrlPrefix() {
@@ -467,7 +868,8 @@ var _ApitallyClient = class _ApitallyClient {
   async sync() {
     try {
       const promises = [
-        this.sendSyncData()
+        this.sendSyncData(),
+        this.sendLogData()
       ];
       if (!this.startupDataSent) {
         promises.push(this.sendStartupData());
@@ -495,7 +897,7 @@ var _ApitallyClient = class _ApitallyClient {
       this.logger.debug("Sending startup data to Apitally Hub");
       const payload = {
         instance_uuid: this.instanceUuid,
-        message_uuid: (0, import_crypto3.randomUUID)(),
+        message_uuid: (0, import_crypto5.randomUUID)(),
         ...this.startupData
       };
       try {
@@ -517,7 +919,7 @@ var _ApitallyClient = class _ApitallyClient {
     const newPayload = {
       time_offset: 0,
       instance_uuid: this.instanceUuid,
-      message_uuid: (0, import_crypto3.randomUUID)(),
+      message_uuid: (0, import_crypto5.randomUUID)(),
       requests: this.requestCounter.getAndResetRequests(),
       validation_errors: this.validationErrorCounter.getAndResetValidationErrors(),
       server_errors: this.serverErrorCounter.getAndResetServerErrors(),
@@ -536,8 +938,7 @@ var _ApitallyClient = class _ApitallyClient {
           const timeOffset = Date.now() - time;
           if (timeOffset <= MAX_QUEUE_TIME) {
             if (i > 0) {
-              const waitMs = 100 + Math.random() * 200;
-              await new Promise((resolve) => setTimeout(resolve, waitMs));
+              await this.randomDelay();
             }
             payload.time_offset = timeOffset / 1e3;
             await this.sendData("sync", payload);
@@ -556,6 +957,52 @@ var _ApitallyClient = class _ApitallyClient {
       }
     }
   }
+  async sendLogData() {
+    this.logger.debug("Sending request log data to Apitally Hub");
+    await this.requestLogger.rotateFile();
+    const fetchWithRetry = (0, import_fetch_retry.default)(fetch, {
+      retries: 3,
+      retryDelay: 1e3,
+      retryOn: [
+        408,
+        429,
+        500,
+        502,
+        503,
+        504
+      ]
+    });
+    let i = 0;
+    let logFile;
+    while (logFile = this.requestLogger.getFile()) {
+      if (i > 0) {
+        await this.randomDelay();
+      }
+      try {
+        const response = await fetchWithRetry(`${this.getHubUrlPrefix()}log?uuid=${logFile.uuid}`, {
+          method: "POST",
+          body: await logFile.getContent()
+        });
+        if (response.status === 402 && response.headers.has("Retry-After")) {
+          const retryAfter = parseInt(response.headers.get("Retry-After") ?? "0");
+          if (retryAfter > 0) {
+            this.requestLogger.suspendUntil = Date.now() + retryAfter * 1e3;
+            this.requestLogger.clear();
+            return;
+          }
+        }
+        if (!response.ok) {
+          throw new HTTPError(response);
+        }
+        logFile.delete();
+      } catch (error) {
+        this.requestLogger.retryFileLater(logFile);
+        break;
+      }
+      i++;
+      if (i >= 10) break;
+    }
+  }
   handleHubError(error) {
     if (error instanceof HTTPError) {
       if (error.response.status === 404) {
@@ -570,6 +1017,10 @@ var _ApitallyClient = class _ApitallyClient {
     }
     return false;
   }
+  async randomDelay() {
+    const delay = 100 + Math.random() * 200;
+    await new Promise((resolve) => setTimeout(resolve, delay));
+  }
 };
 __name(_ApitallyClient, "ApitallyClient");
 __publicField(_ApitallyClient, "instance");
@@ -740,18 +1191,18 @@ var getMiddleware = /* @__PURE__ */ __name((app, client) => {
     }
     try {
       const startTime = import_perf_hooks.performance.now();
-      const originalJson = res.json;
-      res.json = (body) => {
+      const originalSend = res.send;
+      res.send = (body) => {
         res.locals.body = body;
-        return originalJson.call(res, body);
+        return originalSend.call(res, body);
       };
       res.on("finish", () => {
         try {
+          const responseTime = import_perf_hooks.performance.now() - startTime;
           const path = getRoutePath(req);
+          const consumer = getConsumer(req);
+          client.consumerRegistry.addOrUpdateConsumer(consumer);
           if (path) {
-            const responseTime = import_perf_hooks.performance.now() - startTime;
-            const consumer = getConsumer(req);
-            client.consumerRegistry.addOrUpdateConsumer(consumer);
             client.requestCounter.addRequest({
               consumer: consumer == null ? void 0 : consumer.identifier,
               method: req.method,
@@ -762,24 +1213,31 @@ var getMiddleware = /* @__PURE__ */ __name((app, client) => {
               responseSize: res.get("content-length")
             });
             if ((res.statusCode === 400 || res.statusCode === 422) && res.locals.body) {
-              const validationErrors = [];
-              if (validationErrors.length === 0) {
-                validationErrors.push(...extractExpressValidatorErrors(res.locals.body));
-              }
-              if (validationErrors.length === 0) {
-                validationErrors.push(...extractCelebrateErrors(res.locals.body));
+              let jsonBody;
+              try {
+                jsonBody = JSON.parse(res.locals.body);
+              } catch {
               }
-              if (validationErrors.length === 0) {
-                validationErrors.push(...extractNestValidationErrors(res.locals.body));
-              }
-              validationErrors.forEach((error) => {
-                client.validationErrorCounter.addValidationError({
-                  consumer: consumer == null ? void 0 : consumer.identifier,
-                  method: req.method,
-                  path: req.route.path,
-                  ...error
+              if (jsonBody) {
+                const validationErrors = [];
+                if (validationErrors.length === 0) {
+                  validationErrors.push(...extractExpressValidatorErrors(jsonBody));
+                }
+                if (validationErrors.length === 0) {
+                  validationErrors.push(...extractCelebrateErrors(jsonBody));
+                }
+                if (validationErrors.length === 0) {
+                  validationErrors.push(...extractNestValidationErrors(jsonBody));
+                }
+                validationErrors.forEach((error) => {
+                  client.validationErrorCounter.addValidationError({
+                    consumer: consumer == null ? void 0 : consumer.identifier,
+                    method: req.method,
+                    path: req.route.path,
+                    ...error
+                  });
                 });
-              });
+              }
             }
             if (res.statusCode === 500 && res.locals.serverError) {
               const serverError = res.locals.serverError;
@@ -793,6 +1251,24 @@ var getMiddleware = /* @__PURE__ */ __name((app, client) => {
               });
             }
           }
+          if (client.requestLogger.enabled) {
+            client.requestLogger.logRequest({
+              timestamp: Date.now() / 1e3,
+              method: req.method,
+              path,
+              url: `${req.protocol}://${req.host}${req.originalUrl}`,
+              headers: convertHeaders(req.headers),
+              size: Number(req.get("content-length")),
+              consumer: consumer == null ? void 0 : consumer.identifier,
+              body: convertBody(req.body, req.get("content-type"))
+            }, {
+              statusCode: res.statusCode,
+              responseTime: responseTime / 1e3,
+              headers: convertHeaders(res.getHeaders()),
+              size: Number(res.get("content-length")),
+              body: convertBody(res.locals.body, res.get("content-type"))
+            });
+          }
         } catch (error) {
           client.logger.error("Error while logging request in Apitally middleware.", {
             request: req,