apitally 0.10.0 → 0.11.1

package/dist/koa/middleware.js

@@ -10,7 +10,7 @@ var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require
 var __publicField = (obj, key, value) => __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
 
 // src/common/client.ts
-import { randomUUID } from "crypto";
+import { randomUUID as randomUUID3 } from "crypto";
 import fetchRetry from "fetch-retry";
 
 // src/common/consumerRegistry.ts
@@ -179,6 +179,403 @@ var _RequestCounter = class _RequestCounter {
 __name(_RequestCounter, "RequestCounter");
 var RequestCounter = _RequestCounter;
 
+// src/common/requestLogger.ts
+import { Buffer as Buffer3 } from "buffer";
+import { randomUUID as randomUUID2 } from "crypto";
+import { unlinkSync as unlinkSync2, writeFileSync } from "fs";
+import { tmpdir as tmpdir2 } from "os";
+import { join as join2 } from "path";
+
+// src/common/tempGzipFile.ts
+import { Buffer as Buffer2 } from "buffer";
+import { randomUUID } from "crypto";
+import { createWriteStream, readFile, unlinkSync } from "fs";
+import { tmpdir } from "os";
+import { join } from "path";
+import { createGzip } from "zlib";
+var _TempGzipFile = class _TempGzipFile {
+  uuid;
+  filePath;
+  gzip;
+  writeStream;
+  readyPromise;
+  constructor() {
+    this.uuid = randomUUID();
+    this.filePath = join(tmpdir(), `apitally-${this.uuid}.gz`);
+    this.writeStream = createWriteStream(this.filePath);
+    this.readyPromise = new Promise((resolve, reject) => {
+      this.writeStream.once("ready", resolve);
+      this.writeStream.once("error", reject);
+    });
+    this.gzip = createGzip();
+    this.gzip.pipe(this.writeStream);
+  }
+  get size() {
+    return this.writeStream.bytesWritten;
+  }
+  async writeLine(data) {
+    await this.readyPromise;
+    return new Promise((resolve, reject) => {
+      this.gzip.write(Buffer2.concat([
+        data,
+        Buffer2.from("\n")
+      ]), (error) => {
+        if (error) {
+          reject(error);
+        } else {
+          resolve();
+        }
+      });
+    });
+  }
+  async getContent() {
+    return new Promise((resolve, reject) => {
+      readFile(this.filePath, (error, data) => {
+        if (error) {
+          reject(error);
+        } else {
+          resolve(data);
+        }
+      });
+    });
+  }
+  async close() {
+    return new Promise((resolve) => {
+      this.gzip.end(() => {
+        resolve();
+      });
+    });
+  }
+  async delete() {
+    await this.close();
+    unlinkSync(this.filePath);
+  }
+};
+__name(_TempGzipFile, "TempGzipFile");
+var TempGzipFile = _TempGzipFile;
+
+// src/common/requestLogger.ts
+var MAX_BODY_SIZE = 5e4;
+var MAX_FILE_SIZE = 1e6;
+var MAX_FILES = 50;
+var MAX_PENDING_WRITES = 100;
+var BODY_TOO_LARGE = Buffer3.from("<body too large>");
+var BODY_MASKED = Buffer3.from("<masked>");
+var MASKED = "******";
+var ALLOWED_CONTENT_TYPES = [
+  "application/json",
+  "text/plain"
+];
+var EXCLUDE_PATH_PATTERNS = [
+  /\/_?healthz?$/i,
+  /\/_?health[_-]?checks?$/i,
+  /\/_?heart[_-]?beats?$/i,
+  /\/ping$/i,
+  /\/ready$/i,
+  /\/live$/i
+];
+var EXCLUDE_USER_AGENT_PATTERNS = [
+  /health[-_ ]?check/i,
+  /microsoft-azure-application-lb/i,
+  /googlehc/i,
+  /kube-probe/i
+];
+var MASK_QUERY_PARAM_PATTERNS = [
+  /auth/i,
+  /api-?key/i,
+  /secret/i,
+  /token/i,
+  /password/i,
+  /pwd/i
+];
+var MASK_HEADER_PATTERNS = [
+  /auth/i,
+  /api-?key/i,
+  /secret/i,
+  /token/i,
+  /cookie/i
+];
+var DEFAULT_CONFIG = {
+  enabled: false,
+  logQueryParams: true,
+  logRequestHeaders: false,
+  logRequestBody: false,
+  logResponseHeaders: true,
+  logResponseBody: false,
+  maskQueryParams: [],
+  maskHeaders: [],
+  excludePaths: []
+};
+var _RequestLogger = class _RequestLogger {
+  config;
+  enabled;
+  suspendUntil = null;
+  pendingWrites = [];
+  currentFile = null;
+  files = [];
+  maintainIntervalId;
+  constructor(config) {
+    this.config = {
+      ...DEFAULT_CONFIG,
+      ...config
+    };
+    this.enabled = this.config.enabled && checkWritableFs();
+    if (this.enabled) {
+      this.maintainIntervalId = setInterval(() => {
+        this.maintain();
+      }, 1e3);
+    }
+  }
+  shouldExcludePath(urlPath) {
+    const patterns = [
+      ...this.config.excludePaths,
+      ...EXCLUDE_PATH_PATTERNS
+    ];
+    return matchPatterns(urlPath, patterns);
+  }
+  shouldExcludeUserAgent(userAgent) {
+    return userAgent ? matchPatterns(userAgent, EXCLUDE_USER_AGENT_PATTERNS) : false;
+  }
+  shouldMaskQueryParam(name) {
+    const patterns = [
+      ...this.config.maskQueryParams,
+      ...MASK_QUERY_PARAM_PATTERNS
+    ];
+    return matchPatterns(name, patterns);
+  }
+  shouldMaskHeader(name) {
+    const patterns = [
+      ...this.config.maskHeaders,
+      ...MASK_HEADER_PATTERNS
+    ];
+    return matchPatterns(name, patterns);
+  }
+  hasSupportedContentType(headers) {
+    var _a2;
+    const contentType = (_a2 = headers.find(([k]) => k.toLowerCase() === "content-type")) == null ? void 0 : _a2[1];
+    return contentType !== void 0 && ALLOWED_CONTENT_TYPES.some((t) => contentType.startsWith(t));
+  }
+  maskQueryParams(search) {
+    const params = new URLSearchParams(search);
+    for (const [key] of params) {
+      if (this.shouldMaskQueryParam(key)) {
+        params.set(key, MASKED);
+      }
+    }
+    return params.toString();
+  }
+  maskHeaders(headers) {
+    return headers.map(([k, v]) => [
+      k,
+      this.shouldMaskHeader(k) ? MASKED : v
+    ]);
+  }
+  logRequest(request, response) {
+    var _a2, _b, _c;
+    if (!this.enabled || this.suspendUntil !== null) return;
+    const url = new URL(request.url);
+    const path = request.path ?? url.pathname;
+    const userAgent = (_a2 = request.headers.find(([k]) => k.toLowerCase() === "user-agent")) == null ? void 0 : _a2[1];
+    if (this.shouldExcludePath(path) || this.shouldExcludeUserAgent(userAgent) || (((_c = (_b = this.config).excludeCallback) == null ? void 0 : _c.call(_b, request, response)) ?? false)) {
+      return;
+    }
+    url.search = this.config.logQueryParams ? this.maskQueryParams(url.search) : "";
+    request.url = url.toString();
+    request.headers = this.config.logRequestHeaders ? this.maskHeaders(request.headers) : [];
+    response.headers = this.config.logResponseHeaders ? this.maskHeaders(response.headers) : [];
+    if (!this.config.logRequestBody || !this.hasSupportedContentType(request.headers)) {
+      request.body = void 0;
+    } else if (request.body) {
+      if (request.body.length > MAX_BODY_SIZE) {
+        request.body = BODY_TOO_LARGE;
+      } else if (this.config.maskRequestBodyCallback) {
+        try {
+          request.body = this.config.maskRequestBodyCallback(request) ?? BODY_MASKED;
+          if (request.body.length > MAX_BODY_SIZE) {
+            request.body = BODY_TOO_LARGE;
+          }
+        } catch {
+          request.body = void 0;
+        }
+      }
+    }
+    if (!this.config.logResponseBody || !this.hasSupportedContentType(response.headers)) {
+      response.body = void 0;
+    } else if (response.body) {
+      if (response.body.length > MAX_BODY_SIZE) {
+        response.body = BODY_TOO_LARGE;
+      } else if (this.config.maskResponseBodyCallback) {
+        try {
+          response.body = this.config.maskResponseBodyCallback(request, response) ?? BODY_MASKED;
+          if (response.body.length > MAX_BODY_SIZE) {
+            response.body = BODY_TOO_LARGE;
+          }
+        } catch {
+          response.body = void 0;
+        }
+      }
+    }
+    const item = {
+      uuid: randomUUID2(),
+      request: skipEmptyValues(request),
+      response: skipEmptyValues(response)
+    };
+    [
+      item.request.body,
+      item.response.body
+    ].forEach((body) => {
+      if (body) {
+        body.toJSON = function() {
+          return this.toString("base64");
+        };
+      }
+    });
+    this.pendingWrites.push(JSON.stringify(item));
+    if (this.pendingWrites.length > MAX_PENDING_WRITES) {
+      this.pendingWrites.shift();
+    }
+  }
+  async writeToFile() {
+    if (!this.enabled || this.pendingWrites.length === 0) {
+      return;
+    }
+    if (!this.currentFile) {
+      this.currentFile = new TempGzipFile();
+    }
+    while (this.pendingWrites.length > 0) {
+      const item = this.pendingWrites.shift();
+      if (item) {
+        await this.currentFile.writeLine(Buffer3.from(item));
+      }
+    }
+  }
+  getFile() {
+    return this.files.shift();
+  }
+  retryFileLater(file) {
+    this.files.unshift(file);
+  }
+  async rotateFile() {
+    if (this.currentFile) {
+      await this.currentFile.close();
+      this.files.push(this.currentFile);
+      this.currentFile = null;
+    }
+  }
+  async maintain() {
+    await this.writeToFile();
+    if (this.currentFile && this.currentFile.size > MAX_FILE_SIZE) {
+      await this.rotateFile();
+    }
+    while (this.files.length > MAX_FILES) {
+      const file = this.files.shift();
+      file == null ? void 0 : file.delete();
+    }
+    if (this.suspendUntil !== null && this.suspendUntil < Date.now()) {
+      this.suspendUntil = null;
+    }
+  }
+  async clear() {
+    this.pendingWrites = [];
+    await this.rotateFile();
+    this.files.forEach((file) => {
+      file.delete();
+    });
+    this.files = [];
+  }
+  async close() {
+    this.enabled = false;
+    await this.clear();
+    if (this.maintainIntervalId) {
+      clearInterval(this.maintainIntervalId);
+    }
+  }
+};
+__name(_RequestLogger, "RequestLogger");
+var RequestLogger = _RequestLogger;
+function convertHeaders(headers) {
+  if (headers instanceof Headers) {
+    return Array.from(headers.entries());
+  }
+  return Object.entries(headers).flatMap(([key, value]) => {
+    if (value === void 0) {
+      return [];
+    }
+    if (Array.isArray(value)) {
+      return value.map((v) => [
+        key,
+        v
+      ]);
+    }
+    return [
+      [
+        key,
+        value.toString()
+      ]
+    ];
+  });
+}
+__name(convertHeaders, "convertHeaders");
+function convertBody(body, contentType) {
+  if (!body || !contentType) {
+    return;
+  }
+  try {
+    if (contentType.startsWith("application/json")) {
+      if (isValidJsonString(body)) {
+        return Buffer3.from(body);
+      } else {
+        return Buffer3.from(JSON.stringify(body));
+      }
+    }
+    if (contentType.startsWith("text/") && typeof body === "string") {
+      return Buffer3.from(body);
+    }
+  } catch (error) {
+    return;
+  }
+}
+__name(convertBody, "convertBody");
+function isValidJsonString(body) {
+  if (typeof body !== "string") {
+    return false;
+  }
+  try {
+    JSON.parse(body);
+    return true;
+  } catch {
+    return false;
+  }
+}
+__name(isValidJsonString, "isValidJsonString");
+function matchPatterns(value, patterns) {
+  return patterns.some((pattern) => {
+    return pattern.test(value);
+  });
+}
+__name(matchPatterns, "matchPatterns");
+function skipEmptyValues(data) {
+  return Object.fromEntries(Object.entries(data).filter(([_, v]) => {
+    if (v == null || Number.isNaN(v)) return false;
+    if (Array.isArray(v) || Buffer3.isBuffer(v) || typeof v === "string") {
+      return v.length > 0;
+    }
+    return true;
+  }));
+}
+__name(skipEmptyValues, "skipEmptyValues");
+function checkWritableFs() {
+  try {
+    const testPath = join2(tmpdir2(), `apitally-${randomUUID2()}`);
+    writeFileSync(testPath, "test");
+    unlinkSync2(testPath);
+    return true;
+  } catch (error) {
+    return false;
+  }
+}
+__name(checkWritableFs, "checkWritableFs");
+
 // src/common/serverErrorCounter.ts
 import { createHash } from "crypto";
 var MAX_MSG_LENGTH = 2048;
@@ -351,11 +748,12 @@ var _ApitallyClient = class _ApitallyClient {
   startupData;
   startupDataSent = false;
   requestCounter;
+  requestLogger;
   validationErrorCounter;
   serverErrorCounter;
   consumerRegistry;
   logger;
-  constructor({ clientId, env = "dev", logger }) {
+  constructor({ clientId, env = "dev", requestLoggingConfig, logger }) {
     if (_ApitallyClient.instance) {
       throw new Error("Apitally client is already initialized");
     }
@@ -368,9 +766,10 @@ var _ApitallyClient = class _ApitallyClient {
     _ApitallyClient.instance = this;
     this.clientId = clientId;
     this.env = env;
-    this.instanceUuid = randomUUID();
+    this.instanceUuid = randomUUID3();
     this.syncDataQueue = [];
     this.requestCounter = new RequestCounter();
+    this.requestLogger = new RequestLogger(requestLoggingConfig);
     this.validationErrorCounter = new ValidationErrorCounter();
     this.serverErrorCounter = new ServerErrorCounter();
     this.consumerRegistry = new ConsumerRegistry();
@@ -392,6 +791,8 @@ var _ApitallyClient = class _ApitallyClient {
   async handleShutdown() {
     this.stopSync();
     await this.sendSyncData();
+    await this.sendLogData();
+    await this.requestLogger.close();
     _ApitallyClient.instance = void 0;
   }
   getHubUrlPrefix() {
@@ -438,7 +839,8 @@ var _ApitallyClient = class _ApitallyClient {
   async sync() {
     try {
       const promises = [
-        this.sendSyncData()
+        this.sendSyncData(),
+        this.sendLogData()
       ];
       if (!this.startupDataSent) {
         promises.push(this.sendStartupData());
@@ -466,7 +868,7 @@ var _ApitallyClient = class _ApitallyClient {
       this.logger.debug("Sending startup data to Apitally Hub");
       const payload = {
         instance_uuid: this.instanceUuid,
-        message_uuid: randomUUID(),
+        message_uuid: randomUUID3(),
         ...this.startupData
       };
       try {
@@ -488,7 +890,7 @@ var _ApitallyClient = class _ApitallyClient {
     const newPayload = {
       time_offset: 0,
       instance_uuid: this.instanceUuid,
-      message_uuid: randomUUID(),
+      message_uuid: randomUUID3(),
       requests: this.requestCounter.getAndResetRequests(),
       validation_errors: this.validationErrorCounter.getAndResetValidationErrors(),
       server_errors: this.serverErrorCounter.getAndResetServerErrors(),
@@ -507,8 +909,7 @@ var _ApitallyClient = class _ApitallyClient {
           const timeOffset = Date.now() - time;
           if (timeOffset <= MAX_QUEUE_TIME) {
             if (i > 0) {
-              const waitMs = 100 + Math.random() * 200;
-              await new Promise((resolve) => setTimeout(resolve, waitMs));
+              await this.randomDelay();
             }
             payload.time_offset = timeOffset / 1e3;
             await this.sendData("sync", payload);
@@ -527,6 +928,52 @@ var _ApitallyClient = class _ApitallyClient {
       }
     }
   }
+  async sendLogData() {
+    this.logger.debug("Sending request log data to Apitally Hub");
+    await this.requestLogger.rotateFile();
+    const fetchWithRetry = fetchRetry(fetch, {
+      retries: 3,
+      retryDelay: 1e3,
+      retryOn: [
+        408,
+        429,
+        500,
+        502,
+        503,
+        504
+      ]
+    });
+    let i = 0;
+    let logFile;
+    while (logFile = this.requestLogger.getFile()) {
+      if (i > 0) {
+        await this.randomDelay();
+      }
+      try {
+        const response = await fetchWithRetry(`${this.getHubUrlPrefix()}log?uuid=${logFile.uuid}`, {
+          method: "POST",
+          body: await logFile.getContent()
+        });
+        if (response.status === 402 && response.headers.has("Retry-After")) {
+          const retryAfter = parseInt(response.headers.get("Retry-After") ?? "0");
+          if (retryAfter > 0) {
+            this.requestLogger.suspendUntil = Date.now() + retryAfter * 1e3;
+            this.requestLogger.clear();
+            return;
+          }
+        }
+        if (!response.ok) {
+          throw new HTTPError(response);
+        }
+        logFile.delete();
+      } catch (error) {
+        this.requestLogger.retryFileLater(logFile);
+        break;
+      }
+      i++;
+      if (i >= 10) break;
+    }
+  }
   handleHubError(error) {
     if (error instanceof HTTPError) {
       if (error.response.status === 404) {
@@ -541,6 +988,10 @@ var _ApitallyClient = class _ApitallyClient {
     }
     return false;
   }
+  async randomDelay() {
+    const delay = 100 + Math.random() * 200;
+    await new Promise((resolve) => setTimeout(resolve, delay));
+  }
 };
 __name(_ApitallyClient, "ApitallyClient");
 __publicField(_ApitallyClient, "instance");
@@ -595,19 +1046,20 @@ var getMiddleware = /* @__PURE__ */ __name((client) => {
       }
       throw error;
     } finally {
+      const responseTime = performance.now() - startTime;
+      const consumer = getConsumer(ctx);
+      client.consumerRegistry.addOrUpdateConsumer(consumer);
       if (!path) {
         path = getPath(ctx);
       }
       if (path) {
         try {
-          const consumer = getConsumer(ctx);
-          client.consumerRegistry.addOrUpdateConsumer(consumer);
           client.requestCounter.addRequest({
             consumer: consumer == null ? void 0 : consumer.identifier,
             method: ctx.request.method,
             path,
             statusCode: statusCode || ctx.response.status,
-            responseTime: performance.now() - startTime,
+            responseTime,
             requestSize: ctx.request.length,
             responseSize: ctx.response.length
           });
@@ -618,6 +1070,24 @@ var getMiddleware = /* @__PURE__ */ __name((client) => {
           });
         }
       }
+      if (client.requestLogger.enabled) {
+        client.requestLogger.logRequest({
+          timestamp: Date.now() / 1e3,
+          method: ctx.request.method,
+          path,
+          url: ctx.request.href,
+          headers: convertHeaders(ctx.request.headers),
+          size: Number(ctx.request.headers["content-length"]),
+          consumer: consumer == null ? void 0 : consumer.identifier,
+          body: convertBody(ctx.request.body, ctx.request.get("content-type"))
+        }, {
+          statusCode: statusCode || ctx.response.status,
+          responseTime: responseTime / 1e3,
+          headers: convertHeaders(ctx.response.headers),
+          size: Number(ctx.response.headers["content-length"]),
+          body: convertBody(ctx.response.body, ctx.response.get("content-type"))
+        });
+      }
     }
   };
 }, "getMiddleware");