apitally 0.10.0 → 0.11.1

package/dist/hono/middleware.cjs

@@ -40,7 +40,7 @@ var import_handler = require("hono/utils/handler");
 var import_perf_hooks = require("perf_hooks");
 
 // src/common/client.ts
-var import_crypto3 = require("crypto");
+var import_crypto5 = require("crypto");
 var import_fetch_retry = __toESM(require("fetch-retry"), 1);
 
 // src/common/consumerRegistry.ts
@@ -209,8 +209,373 @@ var _RequestCounter = class _RequestCounter {
 __name(_RequestCounter, "RequestCounter");
 var RequestCounter = _RequestCounter;
 
-// src/common/serverErrorCounter.ts
+// src/common/requestLogger.ts
+var import_buffer2 = require("buffer");
+var import_crypto2 = require("crypto");
+var import_fs2 = require("fs");
+var import_os2 = require("os");
+var import_path2 = require("path");
+
+// src/common/tempGzipFile.ts
+var import_buffer = require("buffer");
 var import_crypto = require("crypto");
+var import_fs = require("fs");
+var import_os = require("os");
+var import_path = require("path");
+var import_zlib = require("zlib");
+var _TempGzipFile = class _TempGzipFile {
+  uuid;
+  filePath;
+  gzip;
+  writeStream;
+  readyPromise;
+  constructor() {
+    this.uuid = (0, import_crypto.randomUUID)();
+    this.filePath = (0, import_path.join)((0, import_os.tmpdir)(), `apitally-${this.uuid}.gz`);
+    this.writeStream = (0, import_fs.createWriteStream)(this.filePath);
+    this.readyPromise = new Promise((resolve, reject) => {
+      this.writeStream.once("ready", resolve);
+      this.writeStream.once("error", reject);
+    });
+    this.gzip = (0, import_zlib.createGzip)();
+    this.gzip.pipe(this.writeStream);
+  }
+  get size() {
+    return this.writeStream.bytesWritten;
+  }
+  async writeLine(data) {
+    await this.readyPromise;
+    return new Promise((resolve, reject) => {
+      this.gzip.write(import_buffer.Buffer.concat([
+        data,
+        import_buffer.Buffer.from("\n")
+      ]), (error) => {
+        if (error) {
+          reject(error);
+        } else {
+          resolve();
+        }
+      });
+    });
+  }
+  async getContent() {
+    return new Promise((resolve, reject) => {
+      (0, import_fs.readFile)(this.filePath, (error, data) => {
+        if (error) {
+          reject(error);
+        } else {
+          resolve(data);
+        }
+      });
+    });
+  }
+  async close() {
+    return new Promise((resolve) => {
+      this.gzip.end(() => {
+        resolve();
+      });
+    });
+  }
+  async delete() {
+    await this.close();
+    (0, import_fs.unlinkSync)(this.filePath);
+  }
+};
+__name(_TempGzipFile, "TempGzipFile");
+var TempGzipFile = _TempGzipFile;
+
+// src/common/requestLogger.ts
+var MAX_BODY_SIZE = 5e4;
+var MAX_FILE_SIZE = 1e6;
+var MAX_FILES = 50;
+var MAX_PENDING_WRITES = 100;
+var BODY_TOO_LARGE = import_buffer2.Buffer.from("<body too large>");
+var BODY_MASKED = import_buffer2.Buffer.from("<masked>");
+var MASKED = "******";
+var ALLOWED_CONTENT_TYPES = [
+  "application/json",
+  "text/plain"
+];
+var EXCLUDE_PATH_PATTERNS = [
+  /\/_?healthz?$/i,
+  /\/_?health[_-]?checks?$/i,
+  /\/_?heart[_-]?beats?$/i,
+  /\/ping$/i,
+  /\/ready$/i,
+  /\/live$/i
+];
+var EXCLUDE_USER_AGENT_PATTERNS = [
+  /health[-_ ]?check/i,
+  /microsoft-azure-application-lb/i,
+  /googlehc/i,
+  /kube-probe/i
+];
+var MASK_QUERY_PARAM_PATTERNS = [
+  /auth/i,
+  /api-?key/i,
+  /secret/i,
+  /token/i,
+  /password/i,
+  /pwd/i
+];
+var MASK_HEADER_PATTERNS = [
+  /auth/i,
+  /api-?key/i,
+  /secret/i,
+  /token/i,
+  /cookie/i
+];
+var DEFAULT_CONFIG = {
+  enabled: false,
+  logQueryParams: true,
+  logRequestHeaders: false,
+  logRequestBody: false,
+  logResponseHeaders: true,
+  logResponseBody: false,
+  maskQueryParams: [],
+  maskHeaders: [],
+  excludePaths: []
+};
+var _RequestLogger = class _RequestLogger {
+  config;
+  enabled;
+  suspendUntil = null;
+  pendingWrites = [];
+  currentFile = null;
+  files = [];
+  maintainIntervalId;
+  constructor(config) {
+    this.config = {
+      ...DEFAULT_CONFIG,
+      ...config
+    };
+    this.enabled = this.config.enabled && checkWritableFs();
+    if (this.enabled) {
+      this.maintainIntervalId = setInterval(() => {
+        this.maintain();
+      }, 1e3);
+    }
+  }
+  shouldExcludePath(urlPath) {
+    const patterns = [
+      ...this.config.excludePaths,
+      ...EXCLUDE_PATH_PATTERNS
+    ];
+    return matchPatterns(urlPath, patterns);
+  }
+  shouldExcludeUserAgent(userAgent) {
+    return userAgent ? matchPatterns(userAgent, EXCLUDE_USER_AGENT_PATTERNS) : false;
+  }
+  shouldMaskQueryParam(name) {
+    const patterns = [
+      ...this.config.maskQueryParams,
+      ...MASK_QUERY_PARAM_PATTERNS
+    ];
+    return matchPatterns(name, patterns);
+  }
+  shouldMaskHeader(name) {
+    const patterns = [
+      ...this.config.maskHeaders,
+      ...MASK_HEADER_PATTERNS
+    ];
+    return matchPatterns(name, patterns);
+  }
+  hasSupportedContentType(headers) {
+    var _a2;
+    const contentType = (_a2 = headers.find(([k]) => k.toLowerCase() === "content-type")) == null ? void 0 : _a2[1];
+    return contentType !== void 0 && ALLOWED_CONTENT_TYPES.some((t) => contentType.startsWith(t));
+  }
+  maskQueryParams(search) {
+    const params = new URLSearchParams(search);
+    for (const [key] of params) {
+      if (this.shouldMaskQueryParam(key)) {
+        params.set(key, MASKED);
+      }
+    }
+    return params.toString();
+  }
+  maskHeaders(headers) {
+    return headers.map(([k, v]) => [
+      k,
+      this.shouldMaskHeader(k) ? MASKED : v
+    ]);
+  }
+  logRequest(request, response) {
+    var _a2, _b, _c;
+    if (!this.enabled || this.suspendUntil !== null) return;
+    const url = new URL(request.url);
+    const path = request.path ?? url.pathname;
+    const userAgent = (_a2 = request.headers.find(([k]) => k.toLowerCase() === "user-agent")) == null ? void 0 : _a2[1];
+    if (this.shouldExcludePath(path) || this.shouldExcludeUserAgent(userAgent) || (((_c = (_b = this.config).excludeCallback) == null ? void 0 : _c.call(_b, request, response)) ?? false)) {
+      return;
+    }
+    url.search = this.config.logQueryParams ? this.maskQueryParams(url.search) : "";
+    request.url = url.toString();
+    request.headers = this.config.logRequestHeaders ? this.maskHeaders(request.headers) : [];
+    response.headers = this.config.logResponseHeaders ? this.maskHeaders(response.headers) : [];
+    if (!this.config.logRequestBody || !this.hasSupportedContentType(request.headers)) {
+      request.body = void 0;
+    } else if (request.body) {
+      if (request.body.length > MAX_BODY_SIZE) {
+        request.body = BODY_TOO_LARGE;
+      } else if (this.config.maskRequestBodyCallback) {
+        try {
+          request.body = this.config.maskRequestBodyCallback(request) ?? BODY_MASKED;
+          if (request.body.length > MAX_BODY_SIZE) {
+            request.body = BODY_TOO_LARGE;
+          }
+        } catch {
+          request.body = void 0;
+        }
+      }
+    }
+    if (!this.config.logResponseBody || !this.hasSupportedContentType(response.headers)) {
+      response.body = void 0;
+    } else if (response.body) {
+      if (response.body.length > MAX_BODY_SIZE) {
+        response.body = BODY_TOO_LARGE;
+      } else if (this.config.maskResponseBodyCallback) {
+        try {
+          response.body = this.config.maskResponseBodyCallback(request, response) ?? BODY_MASKED;
+          if (response.body.length > MAX_BODY_SIZE) {
+            response.body = BODY_TOO_LARGE;
+          }
+        } catch {
+          response.body = void 0;
+        }
+      }
+    }
+    const item = {
+      uuid: (0, import_crypto2.randomUUID)(),
+      request: skipEmptyValues(request),
+      response: skipEmptyValues(response)
+    };
+    [
+      item.request.body,
+      item.response.body
+    ].forEach((body) => {
+      if (body) {
+        body.toJSON = function() {
+          return this.toString("base64");
+        };
+      }
+    });
+    this.pendingWrites.push(JSON.stringify(item));
+    if (this.pendingWrites.length > MAX_PENDING_WRITES) {
+      this.pendingWrites.shift();
+    }
+  }
+  async writeToFile() {
+    if (!this.enabled || this.pendingWrites.length === 0) {
+      return;
+    }
+    if (!this.currentFile) {
+      this.currentFile = new TempGzipFile();
+    }
+    while (this.pendingWrites.length > 0) {
+      const item = this.pendingWrites.shift();
+      if (item) {
+        await this.currentFile.writeLine(import_buffer2.Buffer.from(item));
+      }
+    }
+  }
+  getFile() {
+    return this.files.shift();
+  }
+  retryFileLater(file) {
+    this.files.unshift(file);
+  }
+  async rotateFile() {
+    if (this.currentFile) {
+      await this.currentFile.close();
+      this.files.push(this.currentFile);
+      this.currentFile = null;
+    }
+  }
+  async maintain() {
+    await this.writeToFile();
+    if (this.currentFile && this.currentFile.size > MAX_FILE_SIZE) {
+      await this.rotateFile();
+    }
+    while (this.files.length > MAX_FILES) {
+      const file = this.files.shift();
+      file == null ? void 0 : file.delete();
+    }
+    if (this.suspendUntil !== null && this.suspendUntil < Date.now()) {
+      this.suspendUntil = null;
+    }
+  }
+  async clear() {
+    this.pendingWrites = [];
+    await this.rotateFile();
+    this.files.forEach((file) => {
+      file.delete();
+    });
+    this.files = [];
+  }
+  async close() {
+    this.enabled = false;
+    await this.clear();
+    if (this.maintainIntervalId) {
+      clearInterval(this.maintainIntervalId);
+    }
+  }
+};
+__name(_RequestLogger, "RequestLogger");
+var RequestLogger = _RequestLogger;
+function convertHeaders(headers) {
+  if (headers instanceof Headers) {
+    return Array.from(headers.entries());
+  }
+  return Object.entries(headers).flatMap(([key, value]) => {
+    if (value === void 0) {
+      return [];
+    }
+    if (Array.isArray(value)) {
+      return value.map((v) => [
+        key,
+        v
+      ]);
+    }
+    return [
+      [
+        key,
+        value.toString()
+      ]
+    ];
+  });
+}
+__name(convertHeaders, "convertHeaders");
+function matchPatterns(value, patterns) {
+  return patterns.some((pattern) => {
+    return pattern.test(value);
+  });
+}
+__name(matchPatterns, "matchPatterns");
+function skipEmptyValues(data) {
+  return Object.fromEntries(Object.entries(data).filter(([_, v]) => {
+    if (v == null || Number.isNaN(v)) return false;
+    if (Array.isArray(v) || import_buffer2.Buffer.isBuffer(v) || typeof v === "string") {
+      return v.length > 0;
+    }
+    return true;
+  }));
+}
+__name(skipEmptyValues, "skipEmptyValues");
+function checkWritableFs() {
+  try {
+    const testPath = (0, import_path2.join)((0, import_os2.tmpdir)(), `apitally-${(0, import_crypto2.randomUUID)()}`);
+    (0, import_fs2.writeFileSync)(testPath, "test");
+    (0, import_fs2.unlinkSync)(testPath);
+    return true;
+  } catch (error) {
+    return false;
+  }
+}
+__name(checkWritableFs, "checkWritableFs");
+
+// src/common/serverErrorCounter.ts
+var import_crypto3 = require("crypto");
 var MAX_MSG_LENGTH = 2048;
 var MAX_STACKTRACE_LENGTH = 65536;
 var _ServerErrorCounter = class _ServerErrorCounter {
@@ -262,7 +627,7 @@ var _ServerErrorCounter = class _ServerErrorCounter {
       serverError.msg.trim(),
       serverError.traceback.trim()
     ].join("|");
-    return (0, import_crypto.createHash)("md5").update(hashInput).digest("hex");
+    return (0, import_crypto3.createHash)("md5").update(hashInput).digest("hex");
   }
   getTruncatedMessage(msg) {
     msg = msg.trim();
@@ -308,7 +673,7 @@ __name(_ServerErrorCounter, "ServerErrorCounter");
 var ServerErrorCounter = _ServerErrorCounter;
 
 // src/common/validationErrorCounter.ts
-var import_crypto2 = require("crypto");
+var import_crypto4 = require("crypto");
 var _ValidationErrorCounter = class _ValidationErrorCounter {
   errorCounts;
   errorDetails;
@@ -352,7 +717,7 @@ var _ValidationErrorCounter = class _ValidationErrorCounter {
       validationError.msg.trim(),
       validationError.type
     ].join("|");
-    return (0, import_crypto2.createHash)("md5").update(hashInput).digest("hex");
+    return (0, import_crypto4.createHash)("md5").update(hashInput).digest("hex");
   }
 };
 __name(_ValidationErrorCounter, "ValidationErrorCounter");
@@ -381,11 +746,12 @@ var _ApitallyClient = class _ApitallyClient {
   startupData;
   startupDataSent = false;
   requestCounter;
+  requestLogger;
   validationErrorCounter;
   serverErrorCounter;
   consumerRegistry;
   logger;
-  constructor({ clientId, env = "dev", logger }) {
+  constructor({ clientId, env = "dev", requestLoggingConfig, logger }) {
     if (_ApitallyClient.instance) {
       throw new Error("Apitally client is already initialized");
     }
@@ -398,9 +764,10 @@ var _ApitallyClient = class _ApitallyClient {
     _ApitallyClient.instance = this;
     this.clientId = clientId;
     this.env = env;
-    this.instanceUuid = (0, import_crypto3.randomUUID)();
+    this.instanceUuid = (0, import_crypto5.randomUUID)();
     this.syncDataQueue = [];
     this.requestCounter = new RequestCounter();
+    this.requestLogger = new RequestLogger(requestLoggingConfig);
     this.validationErrorCounter = new ValidationErrorCounter();
     this.serverErrorCounter = new ServerErrorCounter();
     this.consumerRegistry = new ConsumerRegistry();
@@ -422,6 +789,8 @@ var _ApitallyClient = class _ApitallyClient {
   async handleShutdown() {
     this.stopSync();
     await this.sendSyncData();
+    await this.sendLogData();
+    await this.requestLogger.close();
     _ApitallyClient.instance = void 0;
   }
   getHubUrlPrefix() {
@@ -468,7 +837,8 @@ var _ApitallyClient = class _ApitallyClient {
   async sync() {
     try {
       const promises = [
-        this.sendSyncData()
+        this.sendSyncData(),
+        this.sendLogData()
       ];
       if (!this.startupDataSent) {
         promises.push(this.sendStartupData());
@@ -496,7 +866,7 @@ var _ApitallyClient = class _ApitallyClient {
       this.logger.debug("Sending startup data to Apitally Hub");
       const payload = {
         instance_uuid: this.instanceUuid,
-        message_uuid: (0, import_crypto3.randomUUID)(),
+        message_uuid: (0, import_crypto5.randomUUID)(),
         ...this.startupData
       };
       try {
@@ -518,7 +888,7 @@ var _ApitallyClient = class _ApitallyClient {
     const newPayload = {
       time_offset: 0,
       instance_uuid: this.instanceUuid,
-      message_uuid: (0, import_crypto3.randomUUID)(),
+      message_uuid: (0, import_crypto5.randomUUID)(),
       requests: this.requestCounter.getAndResetRequests(),
       validation_errors: this.validationErrorCounter.getAndResetValidationErrors(),
       server_errors: this.serverErrorCounter.getAndResetServerErrors(),
@@ -537,8 +907,7 @@ var _ApitallyClient = class _ApitallyClient {
           const timeOffset = Date.now() - time;
           if (timeOffset <= MAX_QUEUE_TIME) {
             if (i > 0) {
-              const waitMs = 100 + Math.random() * 200;
-              await new Promise((resolve) => setTimeout(resolve, waitMs));
+              await this.randomDelay();
             }
             payload.time_offset = timeOffset / 1e3;
             await this.sendData("sync", payload);
@@ -557,6 +926,52 @@ var _ApitallyClient = class _ApitallyClient {
       }
     }
   }
+  async sendLogData() {
+    this.logger.debug("Sending request log data to Apitally Hub");
+    await this.requestLogger.rotateFile();
+    const fetchWithRetry = (0, import_fetch_retry.default)(fetch, {
+      retries: 3,
+      retryDelay: 1e3,
+      retryOn: [
+        408,
+        429,
+        500,
+        502,
+        503,
+        504
+      ]
+    });
+    let i = 0;
+    let logFile;
+    while (logFile = this.requestLogger.getFile()) {
+      if (i > 0) {
+        await this.randomDelay();
+      }
+      try {
+        const response = await fetchWithRetry(`${this.getHubUrlPrefix()}log?uuid=${logFile.uuid}`, {
+          method: "POST",
+          body: await logFile.getContent()
+        });
+        if (response.status === 402 && response.headers.has("Retry-After")) {
+          const retryAfter = parseInt(response.headers.get("Retry-After") ?? "0");
+          if (retryAfter > 0) {
+            this.requestLogger.suspendUntil = Date.now() + retryAfter * 1e3;
+            this.requestLogger.clear();
+            return;
+          }
+        }
+        if (!response.ok) {
+          throw new HTTPError(response);
+        }
+        logFile.delete();
+      } catch (error) {
+        this.requestLogger.retryFileLater(logFile);
+        break;
+      }
+      i++;
+      if (i >= 10) break;
+    }
+  }
   handleHubError(error) {
     if (error instanceof HTTPError) {
       if (error.response.status === 404) {
@@ -571,6 +986,10 @@ var _ApitallyClient = class _ApitallyClient {
     }
     return false;
   }
+  async randomDelay() {
+    const delay = 100 + Math.random() * 200;
+    await new Promise((resolve) => setTimeout(resolve, delay));
+  }
 };
 __name(_ApitallyClient, "ApitallyClient");
 __publicField(_ApitallyClient, "instance");
@@ -604,13 +1023,13 @@ var useApitally = /* @__PURE__ */ __name((app, config) => {
   }, 1e3);
 }, "useApitally");
 var getMiddleware = /* @__PURE__ */ __name((client) => {
-  const zodInstalled = getPackageVersion("zod") !== null;
   return async (c, next) => {
     const startTime = import_perf_hooks.performance.now();
     await next();
     let response;
     const responseTime = import_perf_hooks.performance.now() - startTime;
     const [responseSize, newResponse] = await measureResponseSize(c.res);
+    const requestSize = c.req.header("Content-Length");
     const consumer = getConsumer(c);
     client.consumerRegistry.addOrUpdateConsumer(consumer);
     client.requestCounter.addRequest({
@@ -619,11 +1038,11 @@ var getMiddleware = /* @__PURE__ */ __name((client) => {
       path: c.req.routePath,
       statusCode: c.res.status,
       responseTime,
-      requestSize: c.req.header("Content-Length"),
+      requestSize,
       responseSize
     });
     response = newResponse;
-    if (c.res.status === 400 && zodInstalled) {
+    if (c.res.status === 400) {
       const [responseJson, newResponse2] = await getResponseJson(response);
       const validationErrors = extractZodErrors(responseJson);
       validationErrors.forEach((error) => {
@@ -646,6 +1065,34 @@ var getMiddleware = /* @__PURE__ */ __name((client) => {
         traceback: c.error.stack || ""
       });
     }
+    if (client.requestLogger.enabled) {
+      let requestBody;
+      let responseBody;
+      let newResponse2 = response;
+      if (client.requestLogger.config.logRequestBody) {
+        requestBody = Buffer.from(await c.req.arrayBuffer());
+      }
+      if (client.requestLogger.config.logResponseBody) {
+        [responseBody, newResponse2] = await getResponseBody(response);
+        response = newResponse2;
+      }
+      client.requestLogger.logRequest({
+        timestamp: Date.now() / 1e3,
+        method: c.req.method,
+        path: c.req.routePath,
+        url: c.req.url,
+        headers: convertHeaders(c.req.header()),
+        size: Number(requestSize),
+        consumer: consumer == null ? void 0 : consumer.identifier,
+        body: requestBody
+      }, {
+        statusCode: c.res.status,
+        responseTime: responseTime / 1e3,
+        headers: convertHeaders(c.res.headers),
+        size: responseSize,
+        body: responseBody
+      });
+    }
     c.res = response;
   };
 }, "getMiddleware");
@@ -675,6 +1122,14 @@ var measureResponseSize = /* @__PURE__ */ __name(async (response) => {
     newResponse1
   ];
 }, "measureResponseSize");
+var getResponseBody = /* @__PURE__ */ __name(async (response) => {
+  const [newResponse1, newResponse2] = await teeResponse(response);
+  const responseBuffer = Buffer.from(await newResponse2.arrayBuffer());
+  return [
+    responseBuffer,
+    newResponse1
+  ];
+}, "getResponseBody");
 var getResponseJson = /* @__PURE__ */ __name(async (response) => {
   const contentType = response.headers.get("content-type");
   if (contentType && contentType.includes("application/json")) {