npm - apitally - Versions diffs - 0.10.0 → 0.11.0 - Mend

apitally 0.10.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (85) hide show

package/README.md +1 -2
package/dist/common/client.cjs +398 -12
package/dist/common/client.cjs.map +1 -1
package/dist/common/client.d.cts +8 -1
package/dist/common/client.d.ts +8 -1
package/dist/common/client.js +394 -8
package/dist/common/client.js.map +1 -1
package/dist/common/consumerRegistry.d.cts +4 -0
package/dist/common/consumerRegistry.d.ts +4 -0
package/dist/common/requestCounter.d.cts +4 -0
package/dist/common/requestCounter.d.ts +4 -0
package/dist/common/requestLogger.cjs +419 -0
package/dist/common/requestLogger.cjs.map +1 -0
package/dist/common/requestLogger.d.cts +63 -0
package/dist/common/requestLogger.d.ts +63 -0
package/dist/common/requestLogger.js +395 -0
package/dist/common/requestLogger.js.map +1 -0
package/dist/common/serverErrorCounter.d.cts +4 -0
package/dist/common/serverErrorCounter.d.ts +4 -0
package/dist/common/tempGzipFile.cjs +93 -0
package/dist/common/tempGzipFile.cjs.map +1 -0
package/dist/common/tempGzipFile.d.cts +17 -0
package/dist/common/tempGzipFile.d.ts +17 -0
package/dist/common/tempGzipFile.js +74 -0
package/dist/common/tempGzipFile.js.map +1 -0
package/dist/common/types.cjs.map +1 -1
package/dist/common/types.d.cts +5 -0
package/dist/common/types.d.ts +5 -0
package/dist/common/validationErrorCounter.d.cts +4 -0
package/dist/common/validationErrorCounter.d.ts +4 -0
package/dist/express/index.cjs +500 -34
package/dist/express/index.cjs.map +1 -1
package/dist/express/index.d.cts +4 -0
package/dist/express/index.d.ts +4 -0
package/dist/express/index.js +496 -30
package/dist/express/index.js.map +1 -1
package/dist/express/middleware.cjs +500 -34
package/dist/express/middleware.cjs.map +1 -1
package/dist/express/middleware.d.cts +4 -0
package/dist/express/middleware.d.ts +4 -0
package/dist/express/middleware.js +496 -30
package/dist/express/middleware.js.map +1 -1
package/dist/fastify/index.cjs +491 -27
package/dist/fastify/index.cjs.map +1 -1
package/dist/fastify/index.d.cts +4 -0
package/dist/fastify/index.d.ts +4 -0
package/dist/fastify/index.js +487 -23
package/dist/fastify/index.js.map +1 -1
package/dist/fastify/plugin.cjs +491 -27
package/dist/fastify/plugin.cjs.map +1 -1
package/dist/fastify/plugin.d.cts +4 -0
package/dist/fastify/plugin.d.ts +4 -0
package/dist/fastify/plugin.js +487 -23
package/dist/fastify/plugin.js.map +1 -1
package/dist/hono/index.cjs +460 -15
package/dist/hono/index.cjs.map +1 -1
package/dist/hono/index.d.cts +4 -0
package/dist/hono/index.d.ts +4 -0
package/dist/hono/index.js +456 -11
package/dist/hono/index.js.map +1 -1
package/dist/hono/middleware.cjs +460 -15
package/dist/hono/middleware.cjs.map +1 -1
package/dist/hono/middleware.d.cts +4 -0
package/dist/hono/middleware.d.ts +4 -0
package/dist/hono/middleware.js +456 -11
package/dist/hono/middleware.js.map +1 -1
package/dist/koa/index.cjs +475 -15
package/dist/koa/index.cjs.map +1 -1
package/dist/koa/index.d.cts +4 -0
package/dist/koa/index.d.ts +4 -0
package/dist/koa/index.js +471 -11
package/dist/koa/index.js.map +1 -1
package/dist/koa/middleware.cjs +475 -15
package/dist/koa/middleware.cjs.map +1 -1
package/dist/koa/middleware.d.cts +4 -0
package/dist/koa/middleware.d.ts +4 -0
package/dist/koa/middleware.js +471 -11
package/dist/koa/middleware.js.map +1 -1
package/dist/nestjs/index.cjs +500 -34
package/dist/nestjs/index.cjs.map +1 -1
package/dist/nestjs/index.d.cts +4 -0
package/dist/nestjs/index.d.ts +4 -0
package/dist/nestjs/index.js +496 -30
package/dist/nestjs/index.js.map +1 -1
package/package.json +8 -2

package/dist/koa/index.js CHANGED Viewed

@@ -10,7 +10,7 @@ var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require
 var __publicField = (obj, key, value) => __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
 // src/common/client.ts
-import { randomUUID } from "crypto";
+import { randomUUID as randomUUID3 } from "crypto";
 import fetchRetry from "fetch-retry";
 // src/common/consumerRegistry.ts
@@ -179,6 +179,393 @@ var _RequestCounter = class _RequestCounter {
 __name(_RequestCounter, "RequestCounter");
 var RequestCounter = _RequestCounter;
+// src/common/requestLogger.ts
+import { Buffer as Buffer3 } from "buffer";
+import { randomUUID as randomUUID2 } from "crypto";
+import { unlinkSync as unlinkSync2, writeFileSync } from "fs";
+import { tmpdir as tmpdir2 } from "os";
+import { join as join2 } from "path";
+// src/common/tempGzipFile.ts
+import { Buffer as Buffer2 } from "buffer";
+import { randomUUID } from "crypto";
+import { createWriteStream, readFile, unlinkSync } from "fs";
+import { tmpdir } from "os";
+import { join } from "path";
+import { createGzip } from "zlib";
+var _TempGzipFile = class _TempGzipFile {
+  uuid;
+  filePath;
+  gzip;
+  writeStream;
+  readyPromise;
+  constructor() {
+    this.uuid = randomUUID();
+    this.filePath = join(tmpdir(), `apitally-${this.uuid}.gz`);
+    this.writeStream = createWriteStream(this.filePath);
+    this.readyPromise = new Promise((resolve, reject) => {
+      this.writeStream.once("ready", resolve);
+      this.writeStream.once("error", reject);
+    });
+    this.gzip = createGzip();
+    this.gzip.pipe(this.writeStream);
+  }
+  get size() {
+    return this.writeStream.bytesWritten;
+  }
+  async writeLine(data) {
+    await this.readyPromise;
+    return new Promise((resolve, reject) => {
+      this.gzip.write(Buffer2.concat([
+        data,
+        Buffer2.from("\n")
+      ]), (error) => {
+        if (error) {
+          reject(error);
+        } else {
+          resolve();
+        }
+      });
+    });
+  }
+  async getContent() {
+    return new Promise((resolve, reject) => {
+      readFile(this.filePath, (error, data) => {
+        if (error) {
+          reject(error);
+        } else {
+          resolve(data);
+        }
+      });
+    });
+  }
+  async close() {
+    return new Promise((resolve) => {
+      this.gzip.end(() => {
+        resolve();
+      });
+    });
+  }
+  async delete() {
+    await this.close();
+    unlinkSync(this.filePath);
+  }
+};
+__name(_TempGzipFile, "TempGzipFile");
+var TempGzipFile = _TempGzipFile;
+// src/common/requestLogger.ts
+var MAX_BODY_SIZE = 5e4;
+var MAX_FILE_SIZE = 1e6;
+var MAX_FILES = 50;
+var MAX_PENDING_WRITES = 100;
+var BODY_TOO_LARGE = Buffer3.from("<body too large>");
+var BODY_MASKED = Buffer3.from("<masked>");
+var MASKED = "******";
+var ALLOWED_CONTENT_TYPES = [
+  "application/json",
+  "text/plain"
+];
+var EXCLUDE_PATH_PATTERNS = [
+  /\/_?healthz?$/,
+  /\/_?health[_-]?checks?$/,
+  /\/_?heart[_-]?beats?$/,
+  /\/ping$/,
+  /\/ready$/,
+  /\/live$/
+];
+var MASK_QUERY_PARAM_PATTERNS = [
+  /auth/i,
+  /api-?key/i,
+  /secret/i,
+  /token/i,
+  /password/i,
+  /pwd/i
+];
+var MASK_HEADER_PATTERNS = [
+  /auth/i,
+  /api-?key/i,
+  /secret/i,
+  /token/i,
+  /cookie/i
+];
+var DEFAULT_CONFIG = {
+  enabled: false,
+  logQueryParams: true,
+  logRequestHeaders: false,
+  logRequestBody: false,
+  logResponseHeaders: true,
+  logResponseBody: false,
+  maskQueryParams: [],
+  maskHeaders: [],
+  excludePaths: []
+};
+var _RequestLogger = class _RequestLogger {
+  config;
+  enabled;
+  suspendUntil = null;
+  pendingWrites = [];
+  currentFile = null;
+  files = [];
+  maintainIntervalId;
+  constructor(config) {
+    this.config = {
+      ...DEFAULT_CONFIG,
+      ...config
+    };
+    this.enabled = this.config.enabled && checkWritableFs();
+    if (this.enabled) {
+      this.maintainIntervalId = setInterval(() => {
+        this.maintain();
+      }, 1e3);
+    }
+  }
+  shouldExcludePath(urlPath) {
+    const patterns = [
+      ...this.config.excludePaths,
+      ...EXCLUDE_PATH_PATTERNS
+    ];
+    return matchPatterns(urlPath, patterns);
+  }
+  shouldMaskQueryParam(name) {
+    const patterns = [
+      ...this.config.maskQueryParams,
+      ...MASK_QUERY_PARAM_PATTERNS
+    ];
+    return matchPatterns(name, patterns);
+  }
+  shouldMaskHeader(name) {
+    const patterns = [
+      ...this.config.maskHeaders,
+      ...MASK_HEADER_PATTERNS
+    ];
+    return matchPatterns(name, patterns);
+  }
+  hasSupportedContentType(headers) {
+    var _a2;
+    const contentType = (_a2 = headers.find(([k]) => k.toLowerCase() === "content-type")) == null ? void 0 : _a2[1];
+    return contentType !== void 0 && ALLOWED_CONTENT_TYPES.some((t) => contentType.startsWith(t));
+  }
+  maskQueryParams(search) {
+    const params = new URLSearchParams(search);
+    for (const [key] of params) {
+      if (this.shouldMaskQueryParam(key)) {
+        params.set(key, MASKED);
+      }
+    }
+    return params.toString();
+  }
+  maskHeaders(headers) {
+    return headers.map(([k, v]) => [
+      k,
+      this.shouldMaskHeader(k) ? MASKED : v
+    ]);
+  }
+  logRequest(request, response) {
+    var _a2, _b;
+    if (!this.enabled || this.suspendUntil !== null) return;
+    const url = new URL(request.url);
+    const path = request.path ?? url.pathname;
+    if (this.shouldExcludePath(path) || (((_b = (_a2 = this.config).excludeCallback) == null ? void 0 : _b.call(_a2, request, response)) ?? false)) {
+      return;
+    }
+    url.search = this.config.logQueryParams ? this.maskQueryParams(url.search) : "";
+    request.url = url.toString();
+    request.headers = this.config.logRequestHeaders ? this.maskHeaders(request.headers) : [];
+    response.headers = this.config.logResponseHeaders ? this.maskHeaders(response.headers) : [];
+    if (!this.config.logRequestBody || !this.hasSupportedContentType(request.headers)) {
+      request.body = void 0;
+    } else if (request.body) {
+      if (request.body.length > MAX_BODY_SIZE) {
+        request.body = BODY_TOO_LARGE;
+      } else if (this.config.maskRequestBodyCallback) {
+        try {
+          request.body = this.config.maskRequestBodyCallback(request) ?? BODY_MASKED;
+          if (request.body.length > MAX_BODY_SIZE) {
+            request.body = BODY_TOO_LARGE;
+          }
+        } catch {
+          request.body = void 0;
+        }
+      }
+    }
+    if (!this.config.logResponseBody || !this.hasSupportedContentType(response.headers)) {
+      response.body = void 0;
+    } else if (response.body) {
+      if (response.body.length > MAX_BODY_SIZE) {
+        response.body = BODY_TOO_LARGE;
+      } else if (this.config.maskResponseBodyCallback) {
+        try {
+          response.body = this.config.maskResponseBodyCallback(request, response) ?? BODY_MASKED;
+          if (response.body.length > MAX_BODY_SIZE) {
+            response.body = BODY_TOO_LARGE;
+          }
+        } catch {
+          response.body = void 0;
+        }
+      }
+    }
+    const item = {
+      uuid: randomUUID2(),
+      request: skipEmptyValues(request),
+      response: skipEmptyValues(response)
+    };
+    [
+      item.request.body,
+      item.response.body
+    ].forEach((body) => {
+      if (body) {
+        body.toJSON = function() {
+          return this.toString("base64");
+        };
+      }
+    });
+    this.pendingWrites.push(JSON.stringify(item));
+    if (this.pendingWrites.length > MAX_PENDING_WRITES) {
+      this.pendingWrites.shift();
+    }
+  }
+  async writeToFile() {
+    if (!this.enabled || this.pendingWrites.length === 0) {
+      return;
+    }
+    if (!this.currentFile) {
+      this.currentFile = new TempGzipFile();
+    }
+    while (this.pendingWrites.length > 0) {
+      const item = this.pendingWrites.shift();
+      if (item) {
+        await this.currentFile.writeLine(Buffer3.from(item));
+      }
+    }
+  }
+  getFile() {
+    return this.files.shift();
+  }
+  retryFileLater(file) {
+    this.files.unshift(file);
+  }
+  async rotateFile() {
+    if (this.currentFile) {
+      await this.currentFile.close();
+      this.files.push(this.currentFile);
+      this.currentFile = null;
+    }
+  }
+  async maintain() {
+    await this.writeToFile();
+    if (this.currentFile && this.currentFile.size > MAX_FILE_SIZE) {
+      await this.rotateFile();
+    }
+    while (this.files.length > MAX_FILES) {
+      const file = this.files.shift();
+      file == null ? void 0 : file.delete();
+    }
+    if (this.suspendUntil !== null && this.suspendUntil < Date.now()) {
+      this.suspendUntil = null;
+    }
+  }
+  async clear() {
+    this.pendingWrites = [];
+    await this.rotateFile();
+    this.files.forEach((file) => {
+      file.delete();
+    });
+    this.files = [];
+  }
+  async close() {
+    this.enabled = false;
+    await this.clear();
+    if (this.maintainIntervalId) {
+      clearInterval(this.maintainIntervalId);
+    }
+  }
+};
+__name(_RequestLogger, "RequestLogger");
+var RequestLogger = _RequestLogger;
+function convertHeaders(headers) {
+  if (headers instanceof Headers) {
+    return Array.from(headers.entries());
+  }
+  return Object.entries(headers).flatMap(([key, value]) => {
+    if (value === void 0) {
+      return [];
+    }
+    if (Array.isArray(value)) {
+      return value.map((v) => [
+        key,
+        v
+      ]);
+    }
+    return [
+      [
+        key,
+        value.toString()
+      ]
+    ];
+  });
+}
+__name(convertHeaders, "convertHeaders");
+function convertBody(body, contentType) {
+  if (!body || !contentType) {
+    return;
+  }
+  try {
+    if (contentType.startsWith("application/json")) {
+      if (isValidJsonString(body)) {
+        return Buffer3.from(body);
+      } else {
+        return Buffer3.from(JSON.stringify(body));
+      }
+    }
+    if (contentType.startsWith("text/") && typeof body === "string") {
+      return Buffer3.from(body);
+    }
+  } catch (error) {
+    return;
+  }
+}
+__name(convertBody, "convertBody");
+function isValidJsonString(body) {
+  if (typeof body !== "string") {
+    return false;
+  }
+  try {
+    JSON.parse(body);
+    return true;
+  } catch {
+    return false;
+  }
+}
+__name(isValidJsonString, "isValidJsonString");
+function matchPatterns(value, patterns) {
+  return patterns.some((pattern) => {
+    return pattern.test(value);
+  });
+}
+__name(matchPatterns, "matchPatterns");
+function skipEmptyValues(data) {
+  return Object.fromEntries(Object.entries(data).filter(([_, v]) => {
+    if (v == null || Number.isNaN(v)) return false;
+    if (Array.isArray(v) || Buffer3.isBuffer(v) || typeof v === "string") {
+      return v.length > 0;
+    }
+    return true;
+  }));
+}
+__name(skipEmptyValues, "skipEmptyValues");
+function checkWritableFs() {
+  try {
+    const testPath = join2(tmpdir2(), `apitally-${randomUUID2()}`);
+    writeFileSync(testPath, "test");
+    unlinkSync2(testPath);
+    return true;
+  } catch (error) {
+    return false;
+  }
+}
+__name(checkWritableFs, "checkWritableFs");
 // src/common/serverErrorCounter.ts
 import { createHash } from "crypto";
 var MAX_MSG_LENGTH = 2048;
@@ -351,11 +738,12 @@ var _ApitallyClient = class _ApitallyClient {
   startupData;
   startupDataSent = false;
   requestCounter;
+  requestLogger;
   validationErrorCounter;
   serverErrorCounter;
   consumerRegistry;
   logger;
-  constructor({ clientId, env = "dev", logger }) {
+  constructor({ clientId, env = "dev", requestLoggingConfig, logger }) {
     if (_ApitallyClient.instance) {
       throw new Error("Apitally client is already initialized");
     }
@@ -368,9 +756,10 @@ var _ApitallyClient = class _ApitallyClient {
     _ApitallyClient.instance = this;
     this.clientId = clientId;
     this.env = env;
-    this.instanceUuid = randomUUID();
+    this.instanceUuid = randomUUID3();
     this.syncDataQueue = [];
     this.requestCounter = new RequestCounter();
+    this.requestLogger = new RequestLogger(requestLoggingConfig);
     this.validationErrorCounter = new ValidationErrorCounter();
     this.serverErrorCounter = new ServerErrorCounter();
     this.consumerRegistry = new ConsumerRegistry();
@@ -392,6 +781,8 @@ var _ApitallyClient = class _ApitallyClient {
   async handleShutdown() {
     this.stopSync();
     await this.sendSyncData();
+    await this.sendLogData();
+    await this.requestLogger.close();
     _ApitallyClient.instance = void 0;
   }
   getHubUrlPrefix() {
@@ -438,7 +829,8 @@ var _ApitallyClient = class _ApitallyClient {
   async sync() {
     try {
       const promises = [
-        this.sendSyncData()
+        this.sendSyncData(),
+        this.sendLogData()
       ];
       if (!this.startupDataSent) {
         promises.push(this.sendStartupData());
@@ -466,7 +858,7 @@ var _ApitallyClient = class _ApitallyClient {
       this.logger.debug("Sending startup data to Apitally Hub");
       const payload = {
         instance_uuid: this.instanceUuid,
-        message_uuid: randomUUID(),
+        message_uuid: randomUUID3(),
         ...this.startupData
       };
       try {
@@ -488,7 +880,7 @@ var _ApitallyClient = class _ApitallyClient {
     const newPayload = {
       time_offset: 0,
       instance_uuid: this.instanceUuid,
-      message_uuid: randomUUID(),
+      message_uuid: randomUUID3(),
       requests: this.requestCounter.getAndResetRequests(),
       validation_errors: this.validationErrorCounter.getAndResetValidationErrors(),
       server_errors: this.serverErrorCounter.getAndResetServerErrors(),
@@ -507,8 +899,7 @@ var _ApitallyClient = class _ApitallyClient {
           const timeOffset = Date.now() - time;
           if (timeOffset <= MAX_QUEUE_TIME) {
             if (i > 0) {
-              const waitMs = 100 + Math.random() * 200;
-              await new Promise((resolve) => setTimeout(resolve, waitMs));
+              await this.randomDelay();
             }
             payload.time_offset = timeOffset / 1e3;
             await this.sendData("sync", payload);
@@ -527,6 +918,52 @@ var _ApitallyClient = class _ApitallyClient {
       }
     }
   }
+  async sendLogData() {
+    this.logger.debug("Sending request log data to Apitally Hub");
+    await this.requestLogger.rotateFile();
+    const fetchWithRetry = fetchRetry(fetch, {
+      retries: 3,
+      retryDelay: 1e3,
+      retryOn: [
+        408,
+        429,
+        500,
+        502,
+        503,
+        504
+      ]
+    });
+    let i = 0;
+    let logFile;
+    while (logFile = this.requestLogger.getFile()) {
+      if (i > 0) {
+        await this.randomDelay();
+      }
+      try {
+        const response = await fetchWithRetry(`${this.getHubUrlPrefix()}log?uuid=${logFile.uuid}`, {
+          method: "POST",
+          body: await logFile.getContent()
+        });
+        if (response.status === 402 && response.headers.has("Retry-After")) {
+          const retryAfter = parseInt(response.headers.get("Retry-After") ?? "0");
+          if (retryAfter > 0) {
+            this.requestLogger.suspendUntil = Date.now() + retryAfter * 1e3;
+            this.requestLogger.clear();
+            return;
+          }
+        }
+        if (!response.ok) {
+          throw new HTTPError(response);
+        }
+        logFile.delete();
+      } catch (error) {
+        this.requestLogger.retryFileLater(logFile);
+        break;
+      }
+      i++;
+      if (i >= 10) break;
+    }
+  }
   handleHubError(error) {
     if (error instanceof HTTPError) {
       if (error.response.status === 404) {
@@ -541,6 +978,10 @@ var _ApitallyClient = class _ApitallyClient {
     }
     return false;
   }
+  async randomDelay() {
+    const delay = 100 + Math.random() * 200;
+    await new Promise((resolve) => setTimeout(resolve, delay));
+  }
 };
 __name(_ApitallyClient, "ApitallyClient");
 __publicField(_ApitallyClient, "instance");
@@ -595,19 +1036,20 @@ var getMiddleware = /* @__PURE__ */ __name((client) => {
       }
       throw error;
     } finally {
+      const responseTime = performance.now() - startTime;
+      const consumer = getConsumer(ctx);
+      client.consumerRegistry.addOrUpdateConsumer(consumer);
       if (!path) {
         path = getPath(ctx);
       }
       if (path) {
         try {
-          const consumer = getConsumer(ctx);
-          client.consumerRegistry.addOrUpdateConsumer(consumer);
           client.requestCounter.addRequest({
             consumer: consumer == null ? void 0 : consumer.identifier,
             method: ctx.request.method,
             path,
             statusCode: statusCode || ctx.response.status,
-            responseTime: performance.now() - startTime,
+            responseTime,
             requestSize: ctx.request.length,
             responseSize: ctx.response.length
           });
@@ -618,6 +1060,24 @@ var getMiddleware = /* @__PURE__ */ __name((client) => {
           });
         }
       }
+      if (client.requestLogger.enabled) {
+        client.requestLogger.logRequest({
+          timestamp: Date.now() / 1e3,
+          method: ctx.request.method,
+          path,
+          url: ctx.request.href,
+          headers: convertHeaders(ctx.request.headers),
+          size: Number(ctx.request.headers["content-length"]),
+          consumer: consumer == null ? void 0 : consumer.identifier,
+          body: convertBody(ctx.request.body, ctx.request.get("content-type"))
+        }, {
+          statusCode: statusCode || ctx.response.status,
+          responseTime: responseTime / 1e3,
+          headers: convertHeaders(ctx.response.headers),
+          size: Number(ctx.response.headers["content-length"]),
+          body: convertBody(ctx.response.body, ctx.response.get("content-type"))
+        });
+      }
     }
   };
 }, "getMiddleware");