@redthreadlabs/tracelog 1.8.0 → 1.10.0

package/lib/apm-client/s3-uploader.js

@@ -11,7 +11,8 @@ const os = require('os');
 const path = require('path');
 const zlib = require('zlib');
 const { createGzip } = require('zlib');
-const { pipeline } = require('stream');
+const { pipeline, Transform, Writable } = require('stream');
+const { StringDecoder } = require('string_decoder');
 const {
   S3Client,
   PutObjectCommand,
@@ -38,6 +39,139 @@ const {
 // > 0, then the literal 'current' for the live file. A host that died
 // mid-interval leaves its final '_current' upload in place, interval intact.
 
+// Every log object gets a tiny JSON sidecar at `<logkey>.meta.json` carrying
+// the facts the gzipped body hides: uncompressed size, record count, and an
+// hourly interval×kind histogram of the records inside. The histogram matters
+// because buffered remote clients (tracelog-client) can land records from a
+// past day in today's file — so a file's nominal interval is a filing label,
+// not a truthful description of its contents. The viewer reads these into its
+// size ledger for deterministic memory/cache accounting and factual rollups,
+// and falls back to estimation for files written before sidecars existed.
+const SIDECAR_VERSION = 1;
+const SIDECAR_SUFFIX = '.meta.json';
+
+function _pad2(n) {
+  return String(n).padStart(2, '0');
+}
+
+/** epoch-ms → UTC hour-bucket label 'YYYY-MM-DDTHH' (matches the viewer). */
+function _hourBucket(ms) {
+  const d = new Date(ms);
+  return (
+    `${d.getUTCFullYear()}-${_pad2(d.getUTCMonth() + 1)}-${_pad2(d.getUTCDate())}` +
+    `T${_pad2(d.getUTCHours())}`
+  );
+}
+
+function _safeSize(p) {
+  try {
+    return fs.statSync(p).size;
+  } catch (e) {
+    return 0;
+  }
+}
+
+/**
+ * Derives a log file's sidecar histogram by parsing its NDJSON lines. The file
+ * on disk is the source of truth: counts come from the exact bytes being
+ * uploaded, so they cannot drift from the object, and a restart (which wipes
+ * any in-memory write-time counters) or an orphaned file from a crashed run is
+ * handled for free — we just re-derive from the file.
+ *
+ * Tolerant by design: an unparseable line is skipped (not a record); a record
+ * with a missing/garbage timestamp is counted as `malformed` rather than
+ * forced into an interval. Append-only safe: addChunk may be fed successive
+ * tails of a growing current file, since every line is newline-terminated so
+ * chunk/offset boundaries always land between lines.
+ */
+class MetaAccumulator {
+  constructor() {
+    this.offset = 0; // bytes consumed so far (for incremental current parsing)
+    this.records = 0;
+    this.malformed = 0;
+    this.intervals = Object.create(null); // { 'YYYY-MM-DDTHH': { kind: count } }
+    this._partial = '';
+  }
+
+  addChunk(text) {
+    if (!text) return;
+    const s = this._partial + text;
+    let start = 0;
+    let nl;
+    while ((nl = s.indexOf('\n', start)) !== -1) {
+      this._addLine(s.slice(start, nl));
+      start = nl + 1;
+    }
+    this._partial = s.slice(start);
+  }
+
+  flushPartial() {
+    if (this._partial) {
+      this._addLine(this._partial);
+      this._partial = '';
+    }
+  }
+
+  _addLine(line) {
+    const t = line.trim();
+    if (!t) return;
+    let obj;
+    try {
+      obj = JSON.parse(t);
+    } catch (e) {
+      return; // corrupt line — not a countable record (the viewer skips it too)
+    }
+    if (!obj || typeof obj !== 'object') return;
+    const kind = Object.keys(obj)[0];
+    if (!kind || kind === 'metadata') return; // the file's metadata line
+    this.records++;
+    const body = obj[kind];
+    const tsUs =
+      body &&
+      typeof body.timestamp === 'number' &&
+      isFinite(body.timestamp) &&
+      body.timestamp > 0
+        ? body.timestamp
+        : 0;
+    if (!tsUs) {
+      this.malformed++;
+      return;
+    }
+    const bucket = _hourBucket(tsUs / 1000); // serialized timestamps are epoch-µs
+    const byKind =
+      this.intervals[bucket] || (this.intervals[bucket] = Object.create(null));
+    byKind[kind] = (byKind[kind] || 0) + 1;
+  }
+
+  /**
+   * The sidecar object for this file. records === malformed + Σ(intervals).
+   *
+   * Keys are emitted in a fixed, sorted order at every level — top-level fields
+   * in schema order, interval buckets and their kinds sorted lexically — so the
+   * same contents always serialize to byte-identical JSON regardless of the
+   * order records arrived in. That makes a sidecar's ETag a reliable
+   * sameness check.
+   */
+  toMeta(interval, bytes, compressed) {
+    const intervals = Object.create(null);
+    for (const hour of Object.keys(this.intervals).sort()) {
+      const src = this.intervals[hour];
+      const sorted = Object.create(null);
+      for (const kind of Object.keys(src).sort()) sorted[kind] = src[kind];
+      intervals[hour] = sorted;
+    }
+    return {
+      v: SIDECAR_VERSION,
+      interval,
+      bytes,
+      compressed,
+      records: this.records,
+      malformed: this.malformed,
+      intervals,
+    };
+  }
+}
+
 class S3Uploader {
   /**
    * @param {Object} opts
@@ -86,6 +220,12 @@ class S3Uploader {
     }
 
     this._pendingUploads = 0;
+
+    // Incremental sidecar accumulators for in-progress current files, keyed by
+    // their S3 key. Each periodic uploadCurrent parses only the newly-appended
+    // bytes; a restart drops this map, so the next upload re-parses the file
+    // from byte 0 once and resumes incremental — the file always wins.
+    this._currentAccs = new Map();
   }
 
   /**
@@ -111,16 +251,28 @@ class S3Uploader {
 
     this._pendingUploads++;
 
+    // Count records while the bytes stream through to gzip — one read pass,
+    // no extra memory. StringDecoder keeps multi-byte chars whole across chunk
+    // boundaries.
+    const acc = new MetaAccumulator();
+    const decoder = new StringDecoder('utf8');
     const readStream = fs.createReadStream(filePath);
+    const counter = new Transform({
+      transform(chunk, enc, cb) {
+        try { acc.addChunk(decoder.write(chunk)); } catch (e) { /* never break upload */ }
+        cb(null, chunk);
+      },
+    });
     const gzip = createGzip();
     const writeStream = fs.createWriteStream(gzPath);
 
-    pipeline(readStream, gzip, writeStream, (err) => {
+    pipeline(readStream, counter, gzip, writeStream, (err) => {
       if (err) {
         this._logError('Failed to gzip %s: %s', filePath, err.message);
         this._pendingUploads--;
         return;
       }
+      try { acc.addChunk(decoder.end()); acc.flushPartial(); } catch (e) { /* ignore */ }
 
       const body = fs.createReadStream(gzPath);
       const command = new PutObjectCommand({
@@ -137,6 +289,7 @@ class S3Uploader {
           if (this._log) {
             this._log.debug('Uploaded completed log to s3://%s/%s', this._bucket, key);
           }
+          this._uploadSidecar(key, vars.interval, _safeSize(filePath), _safeSize(gzPath), acc);
           try { fs.unlinkSync(filePath); } catch (e) { /* ignore */ }
           try { fs.unlinkSync(gzPath); } catch (e) { /* ignore */ }
           this._deleteStaleCurrent(vars);
@@ -160,33 +313,54 @@ class S3Uploader {
 
     this._pendingUploads++;
 
-    const body = fs.createReadStream(filePath);
-    const command = new PutObjectCommand({
-      Bucket: this._bucket,
-      Key: key,
-      Body: body,
-      ContentType: 'application/x-ndjson',
+    // Count records in a streaming pass (no gzip here), then upload the file.
+    const acc = new MetaAccumulator();
+    const decoder = new StringDecoder('utf8');
+    const sink = new Writable({
+      write(chunk, enc, cb) {
+        try { acc.addChunk(decoder.write(chunk)); } catch (e) { /* ignore */ }
+        cb();
+      },
     });
 
-    this._s3
-      .send(command)
-      .then(() => {
-        if (this._log) {
-          this._log.debug('Uploaded completed log to s3://%s/%s', this._bucket, key);
-        }
-        try { fs.unlinkSync(filePath); } catch (e) { /* ignore */ }
-        this._deleteStaleCurrent(vars);
-      })
-      .catch((uploadErr) => {
-        this._logError(
-          'Failed to upload %s to S3: %s',
-          filePath,
-          uploadErr.message,
-        );
-      })
-      .finally(() => {
+    pipeline(fs.createReadStream(filePath), sink, (err) => {
+      if (err) {
+        this._logError('Failed to read %s: %s', filePath, err.message);
         this._pendingUploads--;
+        return;
+      }
+      try { acc.addChunk(decoder.end()); acc.flushPartial(); } catch (e) { /* ignore */ }
+
+      const bytes = _safeSize(filePath);
+      const body = fs.createReadStream(filePath);
+      const command = new PutObjectCommand({
+        Bucket: this._bucket,
+        Key: key,
+        Body: body,
+        ContentType: 'application/x-ndjson',
       });
+
+      this._s3
+        .send(command)
+        .then(() => {
+          if (this._log) {
+            this._log.debug('Uploaded completed log to s3://%s/%s', this._bucket, key);
+          }
+          this._uploadSidecar(key, vars.interval, bytes, bytes, acc);
+          try { fs.unlinkSync(filePath); } catch (e) { /* ignore */ }
+          this._deleteStaleCurrent(vars);
+        })
+        .catch((uploadErr) => {
+          this._logError(
+            'Failed to upload %s to S3: %s',
+            filePath,
+            uploadErr.message,
+          );
+        })
+        .finally(() => {
+          this._pendingUploads--;
+        });
+    });
   }
 
   /**
@@ -200,18 +374,24 @@ class S3Uploader {
       key += '.gz';
     }
 
-    this._pendingUploads++;
-    this._s3
-      .send(new DeleteObjectCommand({ Bucket: this._bucket, Key: key }))
-      .then(() => {
-        if (this._log) {
-          this._log.debug('Deleted stale current log s3://%s/%s', this._bucket, key);
-        }
-      })
-      .catch(() => { /* best-effort */ })
-      .finally(() => {
-        this._pendingUploads--;
-      });
+    // The current snapshot is finalized — stop tracking its incremental meta.
+    this._currentAccs.delete(key);
+
+    // Delete the snapshot object and its sidecar (best-effort, both).
+    for (const k of [key, key + SIDECAR_SUFFIX]) {
+      this._pendingUploads++;
+      this._s3
+        .send(new DeleteObjectCommand({ Bucket: this._bucket, Key: k }))
+        .then(() => {
+          if (this._log) {
+            this._log.debug('Deleted stale current log s3://%s/%s', this._bucket, k);
+          }
+        })
+        .catch(() => { /* best-effort */ })
+        .finally(() => {
+          this._pendingUploads--;
+        });
+    }
   }
 
   /**
@@ -245,6 +425,7 @@ class S3Uploader {
 
   _uploadCurrentGzipped(filePath, rawBody, vars, cb) {
     const key = this._buildKey(vars) + '.gz';
+    const acc = this._currentAcc(key, rawBody);
 
     this._pendingUploads++;
 
@@ -270,6 +451,7 @@ class S3Uploader {
           if (this._log) {
             this._log.debug('Uploaded current log to s3://%s/%s', this._bucket, key);
           }
+          this._uploadSidecar(key, vars.interval, rawBody.length, compressed.length, acc);
         })
         .catch((uploadErr) => {
           this._logError(
@@ -286,6 +468,7 @@ class S3Uploader {
 
   _uploadCurrentRaw(filePath, rawBody, vars, cb) {
     const key = this._buildKey(vars);
+    const acc = this._currentAcc(key, rawBody);
 
     this._pendingUploads++;
 
@@ -302,6 +485,7 @@ class S3Uploader {
         if (this._log) {
           this._log.debug('Uploaded current log to s3://%s/%s', this._bucket, key);
         }
+        this._uploadSidecar(key, vars.interval, rawBody.length, rawBody.length, acc);
       })
       .catch((uploadErr) => {
         this._logError(
@@ -315,6 +499,64 @@ class S3Uploader {
       });
   }
 
+  /**
+   * The incremental sidecar accumulator for an in-progress current file. Parses
+   * only bytes appended since the last upload; re-derives from byte 0 when this
+   * key is first seen (or after a restart drops the map, or if the file ever
+   * shrank). Offsets land on newline boundaries, so tail slices never split a
+   * line or a multi-byte char.
+   * @param {string} key - the current snapshot's S3 key
+   * @param {Buffer} rawBody - the full current file contents
+   */
+  _currentAcc(key, rawBody) {
+    let acc = this._currentAccs.get(key);
+    if (!acc || rawBody.length < acc.offset) {
+      acc = new MetaAccumulator();
+      this._currentAccs.set(key, acc);
+    }
+    if (rawBody.length > acc.offset) {
+      acc.addChunk(rawBody.toString('utf8', acc.offset));
+      acc.offset = rawBody.length;
+    }
+    return acc;
+  }
+
+  /**
+   * Upload the metadata sidecar for a just-uploaded log object. Best-effort and
+   * fully decoupled: a failure here never affects the log upload — the viewer
+   * just falls back to estimating that file's size.
+   */
+  _uploadSidecar(objectKey, interval, bytes, compressed, acc) {
+    let body;
+    try {
+      body = JSON.stringify(acc.toMeta(interval, bytes, compressed));
+    } catch (e) {
+      return; // never let sidecar serialization affect the run
+    }
+
+    this._pendingUploads++;
+    this._s3
+      .send(new PutObjectCommand({
+        Bucket: this._bucket,
+        Key: objectKey + SIDECAR_SUFFIX,
+        Body: body,
+        ContentType: 'application/json',
+      }))
+      .then(() => {
+        if (this._log) {
+          this._log.debug('Uploaded sidecar s3://%s/%s%s', this._bucket, objectKey, SIDECAR_SUFFIX);
+        }
+      })
+      .catch((err) => {
+        if (this._log) {
+          this._log.debug('Sidecar upload failed for %s: %s', objectKey, err.message);
+        }
+      })
+      .finally(() => {
+        this._pendingUploads--;
+      });
+  }
+
   /**
    * Build the S3 key for a log file (see the layout contract above).
    *
@@ -356,4 +598,4 @@ function normalizeHost(hostname) {
   return hostname;
 }
 
-module.exports = { S3Uploader, normalizeHost };
+module.exports = { S3Uploader, normalizeHost, MetaAccumulator };

package/lib/filters/sanitize-field-names.js

@@ -47,44 +47,55 @@ function redactKeysFromPostedFormVariables(body, requestHeaders, regexes) {
 }
 
 /**
- * Redact sensitive fields from a captured request body of any supported
- * content type:
+ * Redact sensitive fields from a captured request body, returning a
+ * structured value wherever the body is structured (since 1.9.0):
  *
- * - `application/x-www-form-urlencoded` — top-level form fields (the
- *   historical Elastic APM behavior).
  * - JSON content types (`application/json`, `application/*+json`, with or
  *   without a charset suffix) — **deep** redaction: any key at any depth
  *   matching the sanitizeFieldNames patterns is replaced, recursing through
- *   nested objects and arrays. Stringified JSON bodies are parsed, redacted,
- *   and re-stringified; strings that fail to parse are returned untouched.
+ *   nested objects and arrays. The result is an **embedded object/array**,
+ *   whether the body arrived parsed or as a JSON string. Strings that fail
+ *   to parse are returned untouched as strings.
+ * - `application/x-www-form-urlencoded` — also returned as an embedded,
+ *   deep-redacted object (extended parsers can nest); historically this
+ *   re-serialized to a query string.
  * - anything else — returned as-is.
  *
  * @param {Object | String} body
  * @param {Object} requestHeaders
  * @param {Array<RegExp>} regexes
- * @returns {Object | String} a copy of the body with redacted fields
+ * @returns {Object | Array | String} redacted body, structured when possible
  */
 function redactKeysFromBody(body, requestHeaders, regexes) {
+  // Operate even without patterns: redactDeep also normalizes circulars and
+  // pathological nesting, which must never reach the serializer.
+  const res = Array.isArray(regexes) ? regexes : [];
   const contentType = String(requestHeaders['content-type'] || '');
-  if (!isJsonContentType(contentType)) {
-    return redactKeysFromPostedFormVariables(body, requestHeaders, regexes);
+  const mime = contentType.split(';')[0].trim().toLowerCase();
+
+  // A body some middleware already parsed is structured data no matter what
+  // the content-type header claims — deep-redact and embed it.
+  if (body !== null && !Buffer.isBuffer(body) && typeof body === 'object') {
+    return redactDeep(body, res, 0, new WeakSet());
   }
-  if (!Array.isArray(regexes)) {
+
+  if (typeof body !== 'string') {
     return body;
   }
 
-  if (body !== null && !Buffer.isBuffer(body) && typeof body === 'object') {
-    return redactDeep(body, regexes, 0, new WeakSet());
+  if (mime === HEADER_FORM_URLENCODED) {
+    // querystring.parse returns a null-prototype object; copy to a plain one
+    return redactDeep({ ...querystring.parse(body) }, res, 0, new WeakSet());
   }
 
-  if (typeof body === 'string') {
+  if (isJsonContentType(contentType)) {
     let parsed;
     try {
       parsed = JSON.parse(body);
     } catch (_err) {
       return body; // claimed JSON but isn't; leave it alone
     }
-    return JSON.stringify(redactDeep(parsed, regexes, 0, new WeakSet()));
+    return redactDeep(parsed, res, 0, new WeakSet());
   }
 
   return body;

package/lib/parsers.js

@@ -12,7 +12,6 @@ const basicAuth = require('basic-auth');
 const getUrlFromRequest = require('original-url');
 const parseHttpHeadersFromReqOrRes = require('http-headers');
 const cookie = require('cookie');
-const stringify = require('fast-safe-stringify');
 
 const REDACTED = require('./constants').REDACTED;
 const {
@@ -97,23 +96,26 @@ function getContextFromRequest(req, conf, type) {
   var haveBody = body && (chunked || contentLength > 0);
 
   if (haveBody) {
+    const bodyContentType = String(req.headers['content-type'] || '');
     if (!captureBody) {
       context.body = '[REDACTED]';
+    } else if (bodyContentType.split(';')[0].trim().toLowerCase().startsWith('multipart/')) {
+      // File uploads: never record the raw multipart body — it can embed
+      // entire file contents and no per-field redaction applies to it.
+      context.body = '[REDACTED: multipart body]';
     } else if (Buffer.isBuffer(body)) {
       context.body = '<Buffer>';
     } else {
       if (typeof body === 'string' && req.bodyIsBase64Encoded === true) {
         body = Buffer.from(body, 'base64').toString('utf8');
       }
-      body = redactKeysFromBody(
+      // Structured bodies (JSON, form-encoded) come back as deep-redacted
+      // objects and are recorded embedded, not stringified (since 1.9.0).
+      context.body = redactKeysFromBody(
         body,
         req.headers,
         conf.sanitizeFieldNamesRegExp,
       );
-      if (typeof body !== 'string') {
-        body = tryJsonStringify(body) || stringify(body);
-      }
-      context.body = body;
     }
   }
 
@@ -211,11 +213,6 @@ function parseUrl(urlStr) {
   return new url.URL(urlStr, 'relative:///');
 }
 
-function tryJsonStringify(obj) {
-  try {
-    return JSON.stringify(obj);
-  } catch (e) {}
-}
 
 module.exports = {
   getContextFromRequest,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redthreadlabs/tracelog",
-  "version": "1.8.0",
+  "version": "1.10.0",
   "description": "Node.js APM instrumentation that writes traces to JSONL files",
   "publishConfig": {
     "access": "public"