apitally 0.10.0 → 0.11.1

package/dist/fastify/index.cjs

@@ -41,7 +41,7 @@ module.exports = __toCommonJS(fastify_exports);
 var import_fastify_plugin = __toESM(require("fastify-plugin"), 1);
 
 // src/common/client.ts
-var import_crypto3 = require("crypto");
+var import_crypto5 = require("crypto");
 var import_fetch_retry = __toESM(require("fetch-retry"), 1);
 
 // src/common/consumerRegistry.ts
@@ -210,8 +210,405 @@ var _RequestCounter = class _RequestCounter {
 __name(_RequestCounter, "RequestCounter");
 var RequestCounter = _RequestCounter;
 
-// src/common/serverErrorCounter.ts
+// src/common/requestLogger.ts
+var import_buffer2 = require("buffer");
+var import_crypto2 = require("crypto");
+var import_fs2 = require("fs");
+var import_os2 = require("os");
+var import_path2 = require("path");
+
+// src/common/tempGzipFile.ts
+var import_buffer = require("buffer");
 var import_crypto = require("crypto");
+var import_fs = require("fs");
+var import_os = require("os");
+var import_path = require("path");
+var import_zlib = require("zlib");
+var _TempGzipFile = class _TempGzipFile {
+  uuid;
+  filePath;
+  gzip;
+  writeStream;
+  readyPromise;
+  constructor() {
+    this.uuid = (0, import_crypto.randomUUID)();
+    this.filePath = (0, import_path.join)((0, import_os.tmpdir)(), `apitally-${this.uuid}.gz`);
+    this.writeStream = (0, import_fs.createWriteStream)(this.filePath);
+    this.readyPromise = new Promise((resolve, reject) => {
+      this.writeStream.once("ready", resolve);
+      this.writeStream.once("error", reject);
+    });
+    this.gzip = (0, import_zlib.createGzip)();
+    this.gzip.pipe(this.writeStream);
+  }
+  get size() {
+    return this.writeStream.bytesWritten;
+  }
+  async writeLine(data) {
+    await this.readyPromise;
+    return new Promise((resolve, reject) => {
+      this.gzip.write(import_buffer.Buffer.concat([
+        data,
+        import_buffer.Buffer.from("\n")
+      ]), (error) => {
+        if (error) {
+          reject(error);
+        } else {
+          resolve();
+        }
+      });
+    });
+  }
+  async getContent() {
+    return new Promise((resolve, reject) => {
+      (0, import_fs.readFile)(this.filePath, (error, data) => {
+        if (error) {
+          reject(error);
+        } else {
+          resolve(data);
+        }
+      });
+    });
+  }
+  async close() {
+    return new Promise((resolve) => {
+      this.gzip.end(() => {
+        resolve();
+      });
+    });
+  }
+  async delete() {
+    await this.close();
+    (0, import_fs.unlinkSync)(this.filePath);
+  }
+};
+__name(_TempGzipFile, "TempGzipFile");
+var TempGzipFile = _TempGzipFile;
+
+// src/common/requestLogger.ts
+var MAX_BODY_SIZE = 5e4;
+var MAX_FILE_SIZE = 1e6;
+var MAX_FILES = 50;
+var MAX_PENDING_WRITES = 100;
+var BODY_TOO_LARGE = import_buffer2.Buffer.from("<body too large>");
+var BODY_MASKED = import_buffer2.Buffer.from("<masked>");
+var MASKED = "******";
+var ALLOWED_CONTENT_TYPES = [
+  "application/json",
+  "text/plain"
+];
+var EXCLUDE_PATH_PATTERNS = [
+  /\/_?healthz?$/i,
+  /\/_?health[_-]?checks?$/i,
+  /\/_?heart[_-]?beats?$/i,
+  /\/ping$/i,
+  /\/ready$/i,
+  /\/live$/i
+];
+var EXCLUDE_USER_AGENT_PATTERNS = [
+  /health[-_ ]?check/i,
+  /microsoft-azure-application-lb/i,
+  /googlehc/i,
+  /kube-probe/i
+];
+var MASK_QUERY_PARAM_PATTERNS = [
+  /auth/i,
+  /api-?key/i,
+  /secret/i,
+  /token/i,
+  /password/i,
+  /pwd/i
+];
+var MASK_HEADER_PATTERNS = [
+  /auth/i,
+  /api-?key/i,
+  /secret/i,
+  /token/i,
+  /cookie/i
+];
+var DEFAULT_CONFIG = {
+  enabled: false,
+  logQueryParams: true,
+  logRequestHeaders: false,
+  logRequestBody: false,
+  logResponseHeaders: true,
+  logResponseBody: false,
+  maskQueryParams: [],
+  maskHeaders: [],
+  excludePaths: []
+};
+var _RequestLogger = class _RequestLogger {
+  config;
+  enabled;
+  suspendUntil = null;
+  pendingWrites = [];
+  currentFile = null;
+  files = [];
+  maintainIntervalId;
+  constructor(config) {
+    this.config = {
+      ...DEFAULT_CONFIG,
+      ...config
+    };
+    this.enabled = this.config.enabled && checkWritableFs();
+    if (this.enabled) {
+      this.maintainIntervalId = setInterval(() => {
+        this.maintain();
+      }, 1e3);
+    }
+  }
+  shouldExcludePath(urlPath) {
+    const patterns = [
+      ...this.config.excludePaths,
+      ...EXCLUDE_PATH_PATTERNS
+    ];
+    return matchPatterns(urlPath, patterns);
+  }
+  shouldExcludeUserAgent(userAgent) {
+    return userAgent ? matchPatterns(userAgent, EXCLUDE_USER_AGENT_PATTERNS) : false;
+  }
+  shouldMaskQueryParam(name) {
+    const patterns = [
+      ...this.config.maskQueryParams,
+      ...MASK_QUERY_PARAM_PATTERNS
+    ];
+    return matchPatterns(name, patterns);
+  }
+  shouldMaskHeader(name) {
+    const patterns = [
+      ...this.config.maskHeaders,
+      ...MASK_HEADER_PATTERNS
+    ];
+    return matchPatterns(name, patterns);
+  }
+  hasSupportedContentType(headers) {
+    var _a2;
+    const contentType = (_a2 = headers.find(([k]) => k.toLowerCase() === "content-type")) == null ? void 0 : _a2[1];
+    return contentType !== void 0 && ALLOWED_CONTENT_TYPES.some((t) => contentType.startsWith(t));
+  }
+  maskQueryParams(search) {
+    const params = new URLSearchParams(search);
+    for (const [key] of params) {
+      if (this.shouldMaskQueryParam(key)) {
+        params.set(key, MASKED);
+      }
+    }
+    return params.toString();
+  }
+  maskHeaders(headers) {
+    return headers.map(([k, v]) => [
+      k,
+      this.shouldMaskHeader(k) ? MASKED : v
+    ]);
+  }
+  logRequest(request, response) {
+    var _a2, _b, _c;
+    if (!this.enabled || this.suspendUntil !== null) return;
+    const url = new URL(request.url);
+    const path = request.path ?? url.pathname;
+    const userAgent = (_a2 = request.headers.find(([k]) => k.toLowerCase() === "user-agent")) == null ? void 0 : _a2[1];
+    if (this.shouldExcludePath(path) || this.shouldExcludeUserAgent(userAgent) || (((_c = (_b = this.config).excludeCallback) == null ? void 0 : _c.call(_b, request, response)) ?? false)) {
+      return;
+    }
+    url.search = this.config.logQueryParams ? this.maskQueryParams(url.search) : "";
+    request.url = url.toString();
+    request.headers = this.config.logRequestHeaders ? this.maskHeaders(request.headers) : [];
+    response.headers = this.config.logResponseHeaders ? this.maskHeaders(response.headers) : [];
+    if (!this.config.logRequestBody || !this.hasSupportedContentType(request.headers)) {
+      request.body = void 0;
+    } else if (request.body) {
+      if (request.body.length > MAX_BODY_SIZE) {
+        request.body = BODY_TOO_LARGE;
+      } else if (this.config.maskRequestBodyCallback) {
+        try {
+          request.body = this.config.maskRequestBodyCallback(request) ?? BODY_MASKED;
+          if (request.body.length > MAX_BODY_SIZE) {
+            request.body = BODY_TOO_LARGE;
+          }
+        } catch {
+          request.body = void 0;
+        }
+      }
+    }
+    if (!this.config.logResponseBody || !this.hasSupportedContentType(response.headers)) {
+      response.body = void 0;
+    } else if (response.body) {
+      if (response.body.length > MAX_BODY_SIZE) {
+        response.body = BODY_TOO_LARGE;
+      } else if (this.config.maskResponseBodyCallback) {
+        try {
+          response.body = this.config.maskResponseBodyCallback(request, response) ?? BODY_MASKED;
+          if (response.body.length > MAX_BODY_SIZE) {
+            response.body = BODY_TOO_LARGE;
+          }
+        } catch {
+          response.body = void 0;
+        }
+      }
+    }
+    const item = {
+      uuid: (0, import_crypto2.randomUUID)(),
+      request: skipEmptyValues(request),
+      response: skipEmptyValues(response)
+    };
+    [
+      item.request.body,
+      item.response.body
+    ].forEach((body) => {
+      if (body) {
+        body.toJSON = function() {
+          return this.toString("base64");
+        };
+      }
+    });
+    this.pendingWrites.push(JSON.stringify(item));
+    if (this.pendingWrites.length > MAX_PENDING_WRITES) {
+      this.pendingWrites.shift();
+    }
+  }
+  async writeToFile() {
+    if (!this.enabled || this.pendingWrites.length === 0) {
+      return;
+    }
+    if (!this.currentFile) {
+      this.currentFile = new TempGzipFile();
+    }
+    while (this.pendingWrites.length > 0) {
+      const item = this.pendingWrites.shift();
+      if (item) {
+        await this.currentFile.writeLine(import_buffer2.Buffer.from(item));
+      }
+    }
+  }
+  getFile() {
+    return this.files.shift();
+  }
+  retryFileLater(file) {
+    this.files.unshift(file);
+  }
+  async rotateFile() {
+    if (this.currentFile) {
+      await this.currentFile.close();
+      this.files.push(this.currentFile);
+      this.currentFile = null;
+    }
+  }
+  async maintain() {
+    await this.writeToFile();
+    if (this.currentFile && this.currentFile.size > MAX_FILE_SIZE) {
+      await this.rotateFile();
+    }
+    while (this.files.length > MAX_FILES) {
+      const file = this.files.shift();
+      file == null ? void 0 : file.delete();
+    }
+    if (this.suspendUntil !== null && this.suspendUntil < Date.now()) {
+      this.suspendUntil = null;
+    }
+  }
+  async clear() {
+    this.pendingWrites = [];
+    await this.rotateFile();
+    this.files.forEach((file) => {
+      file.delete();
+    });
+    this.files = [];
+  }
+  async close() {
+    this.enabled = false;
+    await this.clear();
+    if (this.maintainIntervalId) {
+      clearInterval(this.maintainIntervalId);
+    }
+  }
+};
+__name(_RequestLogger, "RequestLogger");
+var RequestLogger = _RequestLogger;
+function convertHeaders(headers) {
+  if (headers instanceof Headers) {
+    return Array.from(headers.entries());
+  }
+  return Object.entries(headers).flatMap(([key, value]) => {
+    if (value === void 0) {
+      return [];
+    }
+    if (Array.isArray(value)) {
+      return value.map((v) => [
+        key,
+        v
+      ]);
+    }
+    return [
+      [
+        key,
+        value.toString()
+      ]
+    ];
+  });
+}
+__name(convertHeaders, "convertHeaders");
+function convertBody(body, contentType) {
+  if (!body || !contentType) {
+    return;
+  }
+  try {
+    if (contentType.startsWith("application/json")) {
+      if (isValidJsonString(body)) {
+        return import_buffer2.Buffer.from(body);
+      } else {
+        return import_buffer2.Buffer.from(JSON.stringify(body));
+      }
+    }
+    if (contentType.startsWith("text/") && typeof body === "string") {
+      return import_buffer2.Buffer.from(body);
+    }
+  } catch (error) {
+    return;
+  }
+}
+__name(convertBody, "convertBody");
+function isValidJsonString(body) {
+  if (typeof body !== "string") {
+    return false;
+  }
+  try {
+    JSON.parse(body);
+    return true;
+  } catch {
+    return false;
+  }
+}
+__name(isValidJsonString, "isValidJsonString");
+function matchPatterns(value, patterns) {
+  return patterns.some((pattern) => {
+    return pattern.test(value);
+  });
+}
+__name(matchPatterns, "matchPatterns");
+function skipEmptyValues(data) {
+  return Object.fromEntries(Object.entries(data).filter(([_, v]) => {
+    if (v == null || Number.isNaN(v)) return false;
+    if (Array.isArray(v) || import_buffer2.Buffer.isBuffer(v) || typeof v === "string") {
+      return v.length > 0;
+    }
+    return true;
+  }));
+}
+__name(skipEmptyValues, "skipEmptyValues");
+function checkWritableFs() {
+  try {
+    const testPath = (0, import_path2.join)((0, import_os2.tmpdir)(), `apitally-${(0, import_crypto2.randomUUID)()}`);
+    (0, import_fs2.writeFileSync)(testPath, "test");
+    (0, import_fs2.unlinkSync)(testPath);
+    return true;
+  } catch (error) {
+    return false;
+  }
+}
+__name(checkWritableFs, "checkWritableFs");
+
+// src/common/serverErrorCounter.ts
+var import_crypto3 = require("crypto");
 var MAX_MSG_LENGTH = 2048;
 var MAX_STACKTRACE_LENGTH = 65536;
 var _ServerErrorCounter = class _ServerErrorCounter {
@@ -263,7 +660,7 @@ var _ServerErrorCounter = class _ServerErrorCounter {
       serverError.msg.trim(),
       serverError.traceback.trim()
     ].join("|");
-    return (0, import_crypto.createHash)("md5").update(hashInput).digest("hex");
+    return (0, import_crypto3.createHash)("md5").update(hashInput).digest("hex");
   }
   getTruncatedMessage(msg) {
     msg = msg.trim();
@@ -309,7 +706,7 @@ __name(_ServerErrorCounter, "ServerErrorCounter");
 var ServerErrorCounter = _ServerErrorCounter;
 
 // src/common/validationErrorCounter.ts
-var import_crypto2 = require("crypto");
+var import_crypto4 = require("crypto");
 var _ValidationErrorCounter = class _ValidationErrorCounter {
   errorCounts;
   errorDetails;
@@ -353,7 +750,7 @@ var _ValidationErrorCounter = class _ValidationErrorCounter {
       validationError.msg.trim(),
       validationError.type
     ].join("|");
-    return (0, import_crypto2.createHash)("md5").update(hashInput).digest("hex");
+    return (0, import_crypto4.createHash)("md5").update(hashInput).digest("hex");
   }
 };
 __name(_ValidationErrorCounter, "ValidationErrorCounter");
@@ -382,11 +779,12 @@ var _ApitallyClient = class _ApitallyClient {
   startupData;
   startupDataSent = false;
   requestCounter;
+  requestLogger;
   validationErrorCounter;
   serverErrorCounter;
   consumerRegistry;
   logger;
-  constructor({ clientId, env = "dev", logger }) {
+  constructor({ clientId, env = "dev", requestLoggingConfig, logger }) {
     if (_ApitallyClient.instance) {
       throw new Error("Apitally client is already initialized");
     }
@@ -399,9 +797,10 @@ var _ApitallyClient = class _ApitallyClient {
     _ApitallyClient.instance = this;
     this.clientId = clientId;
     this.env = env;
-    this.instanceUuid = (0, import_crypto3.randomUUID)();
+    this.instanceUuid = (0, import_crypto5.randomUUID)();
     this.syncDataQueue = [];
     this.requestCounter = new RequestCounter();
+    this.requestLogger = new RequestLogger(requestLoggingConfig);
     this.validationErrorCounter = new ValidationErrorCounter();
     this.serverErrorCounter = new ServerErrorCounter();
     this.consumerRegistry = new ConsumerRegistry();
@@ -423,6 +822,8 @@ var _ApitallyClient = class _ApitallyClient {
   async handleShutdown() {
     this.stopSync();
     await this.sendSyncData();
+    await this.sendLogData();
+    await this.requestLogger.close();
     _ApitallyClient.instance = void 0;
   }
   getHubUrlPrefix() {
@@ -469,7 +870,8 @@ var _ApitallyClient = class _ApitallyClient {
   async sync() {
     try {
       const promises = [
-        this.sendSyncData()
+        this.sendSyncData(),
+        this.sendLogData()
       ];
       if (!this.startupDataSent) {
         promises.push(this.sendStartupData());
@@ -497,7 +899,7 @@ var _ApitallyClient = class _ApitallyClient {
       this.logger.debug("Sending startup data to Apitally Hub");
       const payload = {
         instance_uuid: this.instanceUuid,
-        message_uuid: (0, import_crypto3.randomUUID)(),
+        message_uuid: (0, import_crypto5.randomUUID)(),
         ...this.startupData
       };
       try {
@@ -519,7 +921,7 @@ var _ApitallyClient = class _ApitallyClient {
     const newPayload = {
       time_offset: 0,
       instance_uuid: this.instanceUuid,
-      message_uuid: (0, import_crypto3.randomUUID)(),
+      message_uuid: (0, import_crypto5.randomUUID)(),
       requests: this.requestCounter.getAndResetRequests(),
       validation_errors: this.validationErrorCounter.getAndResetValidationErrors(),
       server_errors: this.serverErrorCounter.getAndResetServerErrors(),
@@ -538,8 +940,7 @@ var _ApitallyClient = class _ApitallyClient {
           const timeOffset = Date.now() - time;
           if (timeOffset <= MAX_QUEUE_TIME) {
             if (i > 0) {
-              const waitMs = 100 + Math.random() * 200;
-              await new Promise((resolve) => setTimeout(resolve, waitMs));
+              await this.randomDelay();
             }
             payload.time_offset = timeOffset / 1e3;
             await this.sendData("sync", payload);
@@ -558,6 +959,52 @@ var _ApitallyClient = class _ApitallyClient {
       }
     }
   }
+  async sendLogData() {
+    this.logger.debug("Sending request log data to Apitally Hub");
+    await this.requestLogger.rotateFile();
+    const fetchWithRetry = (0, import_fetch_retry.default)(fetch, {
+      retries: 3,
+      retryDelay: 1e3,
+      retryOn: [
+        408,
+        429,
+        500,
+        502,
+        503,
+        504
+      ]
+    });
+    let i = 0;
+    let logFile;
+    while (logFile = this.requestLogger.getFile()) {
+      if (i > 0) {
+        await this.randomDelay();
+      }
+      try {
+        const response = await fetchWithRetry(`${this.getHubUrlPrefix()}log?uuid=${logFile.uuid}`, {
+          method: "POST",
+          body: await logFile.getContent()
+        });
+        if (response.status === 402 && response.headers.has("Retry-After")) {
+          const retryAfter = parseInt(response.headers.get("Retry-After") ?? "0");
+          if (retryAfter > 0) {
+            this.requestLogger.suspendUntil = Date.now() + retryAfter * 1e3;
+            this.requestLogger.clear();
+            return;
+          }
+        }
+        if (!response.ok) {
+          throw new HTTPError(response);
+        }
+        logFile.delete();
+      } catch (error) {
+        this.requestLogger.retryFileLater(logFile);
+        break;
+      }
+      i++;
+      if (i >= 10) break;
+    }
+  }
   handleHubError(error) {
     if (error instanceof HTTPError) {
       if (error.response.status === 404) {
@@ -572,6 +1019,10 @@ var _ApitallyClient = class _ApitallyClient {
     }
     return false;
   }
+  async randomDelay() {
+    const delay = 100 + Math.random() * 200;
+    await new Promise((resolve) => setTimeout(resolve, delay));
+  }
 };
 __name(_ApitallyClient, "ApitallyClient");
 __publicField(_ApitallyClient, "instance");
@@ -625,10 +1076,7 @@ var apitallyPlugin = /* @__PURE__ */ __name(async (fastify, config) => {
     await client.handleShutdown();
   });
   fastify.addHook("onSend", (request, reply, payload, done) => {
-    try {
-      reply.payload = JSON.parse(payload);
-    } catch (error) {
-    }
+    reply.payload = payload;
     done();
   });
   fastify.addHook("onError", (request, reply, error, done) => {
@@ -638,6 +1086,7 @@ var apitallyPlugin = /* @__PURE__ */ __name(async (fastify, config) => {
     done();
   });
   fastify.addHook("onResponse", (request, reply, done) => {
+    var _a2;
     if (request.method.toUpperCase() !== "OPTIONS") {
       const consumer = getConsumer(request);
       const path = "routeOptions" in request ? request.routeOptions.url : request.routerPath;
@@ -646,26 +1095,33 @@ var apitallyPlugin = /* @__PURE__ */ __name(async (fastify, config) => {
       if (Array.isArray(responseSize)) {
         responseSize = responseSize[0];
       }
+      const responseTime = getResponseTime(reply);
       client.consumerRegistry.addOrUpdateConsumer(consumer);
       client.requestCounter.addRequest({
         consumer: consumer == null ? void 0 : consumer.identifier,
         method: request.method,
         path,
         statusCode: reply.statusCode,
-        responseTime: getResponseTime(reply),
+        responseTime,
         requestSize,
         responseSize
       });
-      if ((reply.statusCode === 400 || reply.statusCode === 422) && reply.payload && (!reply.payload.code || reply.payload.code === "FST_ERR_VALIDATION") && typeof reply.payload.message === "string") {
-        const validationErrors = extractAjvErrors(reply.payload.message);
-        validationErrors.forEach((error) => {
-          client.validationErrorCounter.addValidationError({
-            consumer: consumer == null ? void 0 : consumer.identifier,
-            method: request.method,
-            path,
-            ...error
-          });
-        });
+      if ((reply.statusCode === 400 || reply.statusCode === 422) && reply.payload) {
+        try {
+          const parsedPayload = JSON.parse(reply.payload);
+          if ((!parsedPayload.code || parsedPayload.code === "FST_ERR_VALIDATION") && typeof parsedPayload.message === "string") {
+            const validationErrors = extractAjvErrors(parsedPayload.message);
+            validationErrors.forEach((error) => {
+              client.validationErrorCounter.addValidationError({
+                consumer: consumer == null ? void 0 : consumer.identifier,
+                method: request.method,
+                path,
+                ...error
+              });
+            });
+          }
+        } catch (error) {
+        }
       }
       if (reply.statusCode === 500 && reply.serverError) {
         client.serverErrorCounter.addServerError({
@@ -677,6 +1133,24 @@ var apitallyPlugin = /* @__PURE__ */ __name(async (fastify, config) => {
           traceback: reply.serverError.stack || ""
         });
       }
+      if (client.requestLogger.enabled) {
+        client.requestLogger.logRequest({
+          timestamp: Date.now() / 1e3,
+          method: request.method,
+          path,
+          url: `${request.protocol}://${request.host ?? request.hostname}${request.originalUrl ?? request.url}`,
+          headers: convertHeaders(request.headers),
+          size: Number(requestSize),
+          consumer: consumer == null ? void 0 : consumer.identifier,
+          body: convertBody(request.body, request.headers["content-type"])
+        }, {
+          statusCode: reply.statusCode,
+          responseTime: responseTime / 1e3,
+          headers: convertHeaders(reply.getHeaders()),
+          size: Number(responseSize),
+          body: convertBody(reply.payload, (_a2 = reply.getHeader("content-type")) == null ? void 0 : _a2.toString())
+        });
+      }
     }
     done();
   });