@redthreadlabs/tracelog 1.10.0 → 1.11.0

package/lib/apm-client/s3-uploader.js

@@ -18,6 +18,15 @@ const {
   PutObjectCommand,
   DeleteObjectCommand,
 } = require('@aws-sdk/client-s3');
+// The S3 key layout, the sidecar shape, and the histogram-deriving
+// MetaAccumulator are the shared contract — owned by tracelog-schema so the
+// agent (writer) and the viewer (reader) can never drift.
+const {
+  buildKey,
+  normalizeHost,
+  MetaAccumulator,
+  sidecarKey,
+} = require('@redthreadlabs/tracelog-schema');
 
 // S3 key layout is FIXED (not configurable): it is the contract between
 // tracelog and the in-browser log viewer, which scans the bucket with
@@ -47,22 +56,6 @@ const {
 // not a truthful description of its contents. The viewer reads these into its
 // size ledger for deterministic memory/cache accounting and factual rollups,
 // and falls back to estimation for files written before sidecars existed.
-const SIDECAR_VERSION = 1;
-const SIDECAR_SUFFIX = '.meta.json';
-
-function _pad2(n) {
-  return String(n).padStart(2, '0');
-}
-
-/** epoch-ms → UTC hour-bucket label 'YYYY-MM-DDTHH' (matches the viewer). */
-function _hourBucket(ms) {
-  const d = new Date(ms);
-  return (
-    `${d.getUTCFullYear()}-${_pad2(d.getUTCMonth() + 1)}-${_pad2(d.getUTCDate())}` +
-    `T${_pad2(d.getUTCHours())}`
-  );
-}
-
 function _safeSize(p) {
   try {
     return fs.statSync(p).size;
@@ -71,107 +64,6 @@ function _safeSize(p) {
   }
 }
 
-/**
- * Derives a log file's sidecar histogram by parsing its NDJSON lines. The file
- * on disk is the source of truth: counts come from the exact bytes being
- * uploaded, so they cannot drift from the object, and a restart (which wipes
- * any in-memory write-time counters) or an orphaned file from a crashed run is
- * handled for free — we just re-derive from the file.
- *
- * Tolerant by design: an unparseable line is skipped (not a record); a record
- * with a missing/garbage timestamp is counted as `malformed` rather than
- * forced into an interval. Append-only safe: addChunk may be fed successive
- * tails of a growing current file, since every line is newline-terminated so
- * chunk/offset boundaries always land between lines.
- */
-class MetaAccumulator {
-  constructor() {
-    this.offset = 0; // bytes consumed so far (for incremental current parsing)
-    this.records = 0;
-    this.malformed = 0;
-    this.intervals = Object.create(null); // { 'YYYY-MM-DDTHH': { kind: count } }
-    this._partial = '';
-  }
-
-  addChunk(text) {
-    if (!text) return;
-    const s = this._partial + text;
-    let start = 0;
-    let nl;
-    while ((nl = s.indexOf('\n', start)) !== -1) {
-      this._addLine(s.slice(start, nl));
-      start = nl + 1;
-    }
-    this._partial = s.slice(start);
-  }
-
-  flushPartial() {
-    if (this._partial) {
-      this._addLine(this._partial);
-      this._partial = '';
-    }
-  }
-
-  _addLine(line) {
-    const t = line.trim();
-    if (!t) return;
-    let obj;
-    try {
-      obj = JSON.parse(t);
-    } catch (e) {
-      return; // corrupt line — not a countable record (the viewer skips it too)
-    }
-    if (!obj || typeof obj !== 'object') return;
-    const kind = Object.keys(obj)[0];
-    if (!kind || kind === 'metadata') return; // the file's metadata line
-    this.records++;
-    const body = obj[kind];
-    const tsUs =
-      body &&
-      typeof body.timestamp === 'number' &&
-      isFinite(body.timestamp) &&
-      body.timestamp > 0
-        ? body.timestamp
-        : 0;
-    if (!tsUs) {
-      this.malformed++;
-      return;
-    }
-    const bucket = _hourBucket(tsUs / 1000); // serialized timestamps are epoch-µs
-    const byKind =
-      this.intervals[bucket] || (this.intervals[bucket] = Object.create(null));
-    byKind[kind] = (byKind[kind] || 0) + 1;
-  }
-
-  /**
-   * The sidecar object for this file. records === malformed + Σ(intervals).
-   *
-   * Keys are emitted in a fixed, sorted order at every level — top-level fields
-   * in schema order, interval buckets and their kinds sorted lexically — so the
-   * same contents always serialize to byte-identical JSON regardless of the
-   * order records arrived in. That makes a sidecar's ETag a reliable
-   * sameness check.
-   */
-  toMeta(interval, bytes, compressed) {
-    const intervals = Object.create(null);
-    for (const hour of Object.keys(this.intervals).sort()) {
-      const src = this.intervals[hour];
-      const sorted = Object.create(null);
-      for (const kind of Object.keys(src).sort()) sorted[kind] = src[kind];
-      intervals[hour] = sorted;
-    }
-    return {
-      v: SIDECAR_VERSION,
-      interval,
-      bytes,
-      compressed,
-      records: this.records,
-      malformed: this.malformed,
-      intervals,
-    };
-  }
-}
-
 class S3Uploader {
   /**
    * @param {Object} opts
@@ -378,7 +270,7 @@ class S3Uploader {
     this._currentAccs.delete(key);
 
     // Delete the snapshot object and its sidecar (best-effort, both).
-    for (const k of [key, key + SIDECAR_SUFFIX]) {
+    for (const k of [key, sidecarKey(key)]) {
       this._pendingUploads++;
       this._s3
         .send(new DeleteObjectCommand({ Bucket: this._bucket, Key: k }))
@@ -538,13 +430,13 @@ class S3Uploader {
     this._s3
       .send(new PutObjectCommand({
         Bucket: this._bucket,
-        Key: objectKey + SIDECAR_SUFFIX,
+        Key: sidecarKey(objectKey),
         Body: body,
         ContentType: 'application/json',
       }))
       .then(() => {
         if (this._log) {
-          this._log.debug('Uploaded sidecar s3://%s/%s%s', this._bucket, objectKey, SIDECAR_SUFFIX);
+          this._log.debug('Uploaded sidecar s3://%s/%s', this._bucket, sidecarKey(objectKey));
         }
       })
       .catch((err) => {
@@ -567,14 +459,8 @@ class S3Uploader {
    *   - {boolean} [current] - True for the live (incomplete) file snapshot
    */
   _buildKey(vars) {
-    let basename = this._host;
-    if (vars.seq > 0) {
-      basename += `_${vars.seq}`;
-    }
-    if (vars.current) {
-      basename += '_current';
-    }
-    return `${vars.channel}/${vars.interval}/${basename}.jsonl`;
+    // the layout is the shared contract; host comes from this uploader
+    return buildKey({ ...vars, host: this._host });
   }
 
   _logError(fmt, ...args) {
@@ -584,18 +470,6 @@ class S3Uploader {
   }
 }
 
-/**
- * Normalize a hostname into the host label used in S3 keys. EC2 internal
- * hostnames (ip-A-B-C-D or ip-A-B-C-D.ec2.internal etc.) become the dotted
- * IP address, which avoids embedding hyphens in the basename; any other
- * hostname is used as-is.
- */
-function normalizeHost(hostname) {
-  const m = /^ip-(\d{1,3})-(\d{1,3})-(\d{1,3})-(\d{1,3})(\..*)?$/.exec(hostname);
-  if (m) {
-    return `${m[1]}.${m[2]}.${m[3]}.${m[4]}`;
-  }
-  return hostname;
-}
-
+// normalizeHost + MetaAccumulator are re-exported from tracelog-schema (the
+// shared contract) so existing importers keep working.
 module.exports = { S3Uploader, normalizeHost, MetaAccumulator };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redthreadlabs/tracelog",
-  "version": "1.10.0",
+  "version": "1.11.0",
   "description": "Node.js APM instrumentation that writes traces to JSONL files",
   "publishConfig": {
     "access": "public"
@@ -50,6 +50,7 @@
   "dependencies": {
     "@aws-sdk/client-s3": "^3.0.0",
     "@elastic/ecs-pino-format": "^1.5.0",
+    "@redthreadlabs/tracelog-schema": "^0.1.0",
     "after-all-results": "^2.0.0",
     "async-value-promise": "^1.1.1",
     "basic-auth": "^2.0.1",