@redthreadlabs/tracelog 1.9.0 → 1.11.0

package/lib/apm-client/s3-uploader.js

@@ -11,12 +11,22 @@ const os = require('os');
 const path = require('path');
 const zlib = require('zlib');
 const { createGzip } = require('zlib');
-const { pipeline } = require('stream');
+const { pipeline, Transform, Writable } = require('stream');
+const { StringDecoder } = require('string_decoder');
 const {
   S3Client,
   PutObjectCommand,
   DeleteObjectCommand,
 } = require('@aws-sdk/client-s3');
+// The S3 key layout, the sidecar shape, and the histogram-deriving
+// MetaAccumulator are the shared contract — owned by tracelog-schema so the
+// agent (writer) and the viewer (reader) can never drift.
+const {
+  buildKey,
+  normalizeHost,
+  MetaAccumulator,
+  sidecarKey,
+} = require('@redthreadlabs/tracelog-schema');
 
 // S3 key layout is FIXED (not configurable): it is the contract between
 // tracelog and the in-browser log viewer, which scans the bucket with
@@ -38,6 +48,22 @@ const {
 // > 0, then the literal 'current' for the live file. A host that died
 // mid-interval leaves its final '_current' upload in place, interval intact.
 
+// Every log object gets a tiny JSON sidecar at `<logkey>.meta.json` carrying
+// the facts the gzipped body hides: uncompressed size, record count, and an
+// hourly interval×kind histogram of the records inside. The histogram matters
+// because buffered remote clients (tracelog-client) can land records from a
+// past day in today's file — so a file's nominal interval is a filing label,
+// not a truthful description of its contents. The viewer reads these into its
+// size ledger for deterministic memory/cache accounting and factual rollups,
+// and falls back to estimation for files written before sidecars existed.
+function _safeSize(p) {
+  try {
+    return fs.statSync(p).size;
+  } catch (e) {
+    return 0;
+  }
+}
+
 class S3Uploader {
   /**
    * @param {Object} opts
@@ -86,6 +112,12 @@ class S3Uploader {
     }
 
     this._pendingUploads = 0;
+
+    // Incremental sidecar accumulators for in-progress current files, keyed by
+    // their S3 key. Each periodic uploadCurrent parses only the newly-appended
+    // bytes; a restart drops this map, so the next upload re-parses the file
+    // from byte 0 once and resumes incremental — the file always wins.
+    this._currentAccs = new Map();
   }
 
   /**
@@ -111,16 +143,28 @@ class S3Uploader {
 
     this._pendingUploads++;
 
+    // Count records while the bytes stream through to gzip — one read pass,
+    // no extra memory. StringDecoder keeps multi-byte chars whole across chunk
+    // boundaries.
+    const acc = new MetaAccumulator();
+    const decoder = new StringDecoder('utf8');
     const readStream = fs.createReadStream(filePath);
+    const counter = new Transform({
+      transform(chunk, enc, cb) {
+        try { acc.addChunk(decoder.write(chunk)); } catch (e) { /* never break upload */ }
+        cb(null, chunk);
+      },
+    });
     const gzip = createGzip();
     const writeStream = fs.createWriteStream(gzPath);
 
-    pipeline(readStream, gzip, writeStream, (err) => {
+    pipeline(readStream, counter, gzip, writeStream, (err) => {
       if (err) {
         this._logError('Failed to gzip %s: %s', filePath, err.message);
         this._pendingUploads--;
         return;
       }
+      try { acc.addChunk(decoder.end()); acc.flushPartial(); } catch (e) { /* ignore */ }
 
       const body = fs.createReadStream(gzPath);
       const command = new PutObjectCommand({
@@ -137,6 +181,7 @@ class S3Uploader {
           if (this._log) {
             this._log.debug('Uploaded completed log to s3://%s/%s', this._bucket, key);
           }
+          this._uploadSidecar(key, vars.interval, _safeSize(filePath), _safeSize(gzPath), acc);
           try { fs.unlinkSync(filePath); } catch (e) { /* ignore */ }
           try { fs.unlinkSync(gzPath); } catch (e) { /* ignore */ }
           this._deleteStaleCurrent(vars);
@@ -160,33 +205,54 @@ class S3Uploader {
 
     this._pendingUploads++;
 
-    const body = fs.createReadStream(filePath);
-    const command = new PutObjectCommand({
-      Bucket: this._bucket,
-      Key: key,
-      Body: body,
-      ContentType: 'application/x-ndjson',
+    // Count records in a streaming pass (no gzip here), then upload the file.
+    const acc = new MetaAccumulator();
+    const decoder = new StringDecoder('utf8');
+    const sink = new Writable({
+      write(chunk, enc, cb) {
+        try { acc.addChunk(decoder.write(chunk)); } catch (e) { /* ignore */ }
+        cb();
+      },
     });
 
-    this._s3
-      .send(command)
-      .then(() => {
-        if (this._log) {
-          this._log.debug('Uploaded completed log to s3://%s/%s', this._bucket, key);
-        }
-        try { fs.unlinkSync(filePath); } catch (e) { /* ignore */ }
-        this._deleteStaleCurrent(vars);
-      })
-      .catch((uploadErr) => {
-        this._logError(
-          'Failed to upload %s to S3: %s',
-          filePath,
-          uploadErr.message,
-        );
-      })
-      .finally(() => {
+    pipeline(fs.createReadStream(filePath), sink, (err) => {
+      if (err) {
+        this._logError('Failed to read %s: %s', filePath, err.message);
         this._pendingUploads--;
+        return;
+      }
+      try { acc.addChunk(decoder.end()); acc.flushPartial(); } catch (e) { /* ignore */ }
+
+      const bytes = _safeSize(filePath);
+      const body = fs.createReadStream(filePath);
+      const command = new PutObjectCommand({
+        Bucket: this._bucket,
+        Key: key,
+        Body: body,
+        ContentType: 'application/x-ndjson',
       });
+
+      this._s3
+        .send(command)
+        .then(() => {
+          if (this._log) {
+            this._log.debug('Uploaded completed log to s3://%s/%s', this._bucket, key);
+          }
+          this._uploadSidecar(key, vars.interval, bytes, bytes, acc);
+          try { fs.unlinkSync(filePath); } catch (e) { /* ignore */ }
+          this._deleteStaleCurrent(vars);
+        })
+        .catch((uploadErr) => {
+          this._logError(
+            'Failed to upload %s to S3: %s',
+            filePath,
+            uploadErr.message,
+          );
+        })
+        .finally(() => {
+          this._pendingUploads--;
+        });
+    });
   }
 
   /**
@@ -200,18 +266,24 @@ class S3Uploader {
       key += '.gz';
     }
 
-    this._pendingUploads++;
-    this._s3
-      .send(new DeleteObjectCommand({ Bucket: this._bucket, Key: key }))
-      .then(() => {
-        if (this._log) {
-          this._log.debug('Deleted stale current log s3://%s/%s', this._bucket, key);
-        }
-      })
-      .catch(() => { /* best-effort */ })
-      .finally(() => {
-        this._pendingUploads--;
-      });
+    // The current snapshot is finalized — stop tracking its incremental meta.
+    this._currentAccs.delete(key);
+
+    // Delete the snapshot object and its sidecar (best-effort, both).
+    for (const k of [key, sidecarKey(key)]) {
+      this._pendingUploads++;
+      this._s3
+        .send(new DeleteObjectCommand({ Bucket: this._bucket, Key: k }))
+        .then(() => {
+          if (this._log) {
+            this._log.debug('Deleted stale current log s3://%s/%s', this._bucket, k);
+          }
+        })
+        .catch(() => { /* best-effort */ })
+        .finally(() => {
+          this._pendingUploads--;
+        });
+    }
   }
 
   /**
@@ -245,6 +317,7 @@ class S3Uploader {
 
   _uploadCurrentGzipped(filePath, rawBody, vars, cb) {
     const key = this._buildKey(vars) + '.gz';
+    const acc = this._currentAcc(key, rawBody);
 
     this._pendingUploads++;
 
@@ -270,6 +343,7 @@ class S3Uploader {
           if (this._log) {
             this._log.debug('Uploaded current log to s3://%s/%s', this._bucket, key);
           }
+          this._uploadSidecar(key, vars.interval, rawBody.length, compressed.length, acc);
         })
         .catch((uploadErr) => {
           this._logError(
@@ -286,6 +360,7 @@ class S3Uploader {
 
   _uploadCurrentRaw(filePath, rawBody, vars, cb) {
     const key = this._buildKey(vars);
+    const acc = this._currentAcc(key, rawBody);
 
     this._pendingUploads++;
 
@@ -302,6 +377,7 @@ class S3Uploader {
         if (this._log) {
           this._log.debug('Uploaded current log to s3://%s/%s', this._bucket, key);
         }
+        this._uploadSidecar(key, vars.interval, rawBody.length, rawBody.length, acc);
       })
       .catch((uploadErr) => {
         this._logError(
@@ -315,6 +391,64 @@ class S3Uploader {
       });
   }
 
+  /**
+   * The incremental sidecar accumulator for an in-progress current file. Parses
+   * only bytes appended since the last upload; re-derives from byte 0 when this
+   * key is first seen (or after a restart drops the map, or if the file ever
+   * shrank). Offsets land on newline boundaries, so tail slices never split a
+   * line or a multi-byte char.
+   * @param {string} key - the current snapshot's S3 key
+   * @param {Buffer} rawBody - the full current file contents
+   */
+  _currentAcc(key, rawBody) {
+    let acc = this._currentAccs.get(key);
+    if (!acc || rawBody.length < acc.offset) {
+      acc = new MetaAccumulator();
+      this._currentAccs.set(key, acc);
+    }
+    if (rawBody.length > acc.offset) {
+      acc.addChunk(rawBody.toString('utf8', acc.offset));
+      acc.offset = rawBody.length;
+    }
+    return acc;
+  }
+
+  /**
+   * Upload the metadata sidecar for a just-uploaded log object. Best-effort and
+   * fully decoupled: a failure here never affects the log upload — the viewer
+   * just falls back to estimating that file's size.
+   */
+  _uploadSidecar(objectKey, interval, bytes, compressed, acc) {
+    let body;
+    try {
+      body = JSON.stringify(acc.toMeta(interval, bytes, compressed));
+    } catch (e) {
+      return; // never let sidecar serialization affect the run
+    }
+
+    this._pendingUploads++;
+    this._s3
+      .send(new PutObjectCommand({
+        Bucket: this._bucket,
+        Key: sidecarKey(objectKey),
+        Body: body,
+        ContentType: 'application/json',
+      }))
+      .then(() => {
+        if (this._log) {
+          this._log.debug('Uploaded sidecar s3://%s/%s', this._bucket, sidecarKey(objectKey));
+        }
+      })
+      .catch((err) => {
+        if (this._log) {
+          this._log.debug('Sidecar upload failed for %s: %s', objectKey, err.message);
+        }
+      })
+      .finally(() => {
+        this._pendingUploads--;
+      });
+  }
+
   /**
    * Build the S3 key for a log file (see the layout contract above).
    *
@@ -325,14 +459,8 @@ class S3Uploader {
    *   - {boolean} [current] - True for the live (incomplete) file snapshot
    */
   _buildKey(vars) {
-    let basename = this._host;
-    if (vars.seq > 0) {
-      basename += `_${vars.seq}`;
-    }
-    if (vars.current) {
-      basename += '_current';
-    }
-    return `${vars.channel}/${vars.interval}/${basename}.jsonl`;
+    // the layout is the shared contract; host comes from this uploader
+    return buildKey({ ...vars, host: this._host });
   }
 
   _logError(fmt, ...args) {
@@ -342,18 +470,6 @@ class S3Uploader {
   }
 }
 
-/**
- * Normalize a hostname into the host label used in S3 keys. EC2 internal
- * hostnames (ip-A-B-C-D or ip-A-B-C-D.ec2.internal etc.) become the dotted
- * IP address, which avoids embedding hyphens in the basename; any other
- * hostname is used as-is.
- */
-function normalizeHost(hostname) {
-  const m = /^ip-(\d{1,3})-(\d{1,3})-(\d{1,3})-(\d{1,3})(\..*)?$/.exec(hostname);
-  if (m) {
-    return `${m[1]}.${m[2]}.${m[3]}.${m[4]}`;
-  }
-  return hostname;
-}
-
-module.exports = { S3Uploader, normalizeHost };
+// normalizeHost + MetaAccumulator are re-exported from tracelog-schema (the
+// shared contract) so existing importers keep working.
+module.exports = { S3Uploader, normalizeHost, MetaAccumulator };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redthreadlabs/tracelog",
-  "version": "1.9.0",
+  "version": "1.11.0",
   "description": "Node.js APM instrumentation that writes traces to JSONL files",
   "publishConfig": {
     "access": "public"
@@ -50,6 +50,7 @@
   "dependencies": {
     "@aws-sdk/client-s3": "^3.0.0",
     "@elastic/ecs-pino-format": "^1.5.0",
+    "@redthreadlabs/tracelog-schema": "^0.1.0",
     "after-all-results": "^2.0.0",
     "async-value-promise": "^1.1.1",
     "basic-auth": "^2.0.1",