@redthreadlabs/tracelog 1.5.0 → 1.6.0

package/README.md

@@ -19,12 +19,16 @@ require('tracelog').start({
   serviceName: 'my-api',
   serviceVersion: '1.0.0',
   logDir: '/var/log/myapp',
+  defaultChannel: 'server',
   s3Bucket: 'my-traces',
   s3Region: 'us-east-1',
-  s3KeyTemplate: '{serviceName}/{environment}/{date}/{hostname}-{pid}-{timestamp}.jsonl',
 });
 ```
 
+S3 keys follow a fixed layout designed for prefix scans and per-channel
+lifecycle rules: `{channel}/{interval}/{host}[_{seq}][_current].jsonl[.gz]`
+(see CONFIG.md).
+
 Or use the auto-start entry point with environment variables:
 
 ```bash

package/index.d.ts

@@ -318,10 +318,16 @@ declare namespace apm {
     maxLocalRetentionDays?: number;
     maxBufferSize?: number;
 
-    // Tracelog: S3 upload
+    /**
+     * Name of the default channel — the first segment of local filenames
+     * and S3 keys for records not routed elsewhere. Default: 'default'.
+     */
+    defaultChannel?: string;
+
+    // Tracelog: S3 upload. The S3 key layout is fixed:
+    // {channel}/{interval}/{host}[_{seq}][_current].jsonl[.gz]
     s3Bucket?: string;
     s3Region?: string;
-    s3KeyTemplate?: string;
     s3UploadIntervalMs?: number;
     s3GzipCompleted?: boolean;
     s3GzipCurrent?: boolean;

package/lib/apm-client/apm-client.js

@@ -31,9 +31,6 @@ function createApmClient(config, agent) {
   if (config.s3Bucket) {
     s3Uploader = new S3Uploader({
       bucket: config.s3Bucket,
-      keyTemplate:
-        config.s3KeyTemplate ||
-        '{serviceName}/{environment}/{hostname}-{channel}-{interval}.jsonl',
       region: config.s3Region,
       accessKeyId: config.s3AccessKeyId,
       secretAccessKey: config.s3SecretAccessKey,
@@ -41,8 +38,6 @@ function createApmClient(config, agent) {
       s3Client: config.s3Client, // optional: inject a mock for testing
       gzipCompleted: config.s3GzipCompleted,
       gzipCurrent: config.s3GzipCurrent,
-      serviceName: config.serviceName,
-      environment: config.environment,
       logger: config.logger,
     });
   }
@@ -60,6 +55,7 @@ function createApmClient(config, agent) {
     // JSONL file options
     logDir: config.logDir,
     logFilePrefix: config.logFilePrefix,
+    defaultChannel: config.defaultChannel,
     maxFileSize: config.logMaxFileSize,
     flushIntervalMs: config.logFlushIntervalMs,
     rotationSchedule: config.logRotationSchedule,

package/lib/apm-client/channel-writer.js

@@ -170,7 +170,11 @@ class ChannelWriter {
 
   uploadCurrent() {
     if (this._s3Uploader) {
-      this._s3Uploader.uploadCurrent(this._currentFilePath, { channel: this._channel });
+      this._s3Uploader.uploadCurrent(this._currentFilePath, {
+        channel: this._channel,
+        interval: this._currentPeriodLabel,
+        seq: this._currentSeqNum,
+      });
     }
   }
 
@@ -217,7 +221,11 @@ class ChannelWriter {
     this._wroteMetadata = false;
 
     if (this._s3Uploader && fs.existsSync(completedFilePath)) {
-      this._s3Uploader.uploadCompleted(completedFilePath, { channel: this._channel, interval: this._currentPeriodLabel });
+      this._s3Uploader.uploadCompleted(completedFilePath, {
+        channel: this._channel,
+        interval: this._currentPeriodLabel,
+        seq: this._currentSeqNum,
+      });
     }
 
     if (this._maxLocalRetentionDays > 0) {
@@ -317,10 +325,15 @@ class ChannelWriter {
 
         const withoutPrefix = file.slice(prefix.length);
         const withoutExt = withoutPrefix.slice(0, withoutPrefix.length - this._ext.length);
+        const seqMatch = /\.(\d+)$/.exec(withoutExt);
         const periodLabel = withoutExt.replace(/\.\d+$/, '');
 
         if (periodLabel < this._currentPeriodLabel) {
-          this._s3Uploader.uploadCompleted(filePath, { channel: this._channel, interval: periodLabel });
+          this._s3Uploader.uploadCompleted(filePath, {
+            channel: this._channel,
+            interval: periodLabel,
+            seq: seqMatch ? parseInt(seqMatch[1], 10) : 0,
+          });
         }
       }
     } catch (e) { /* ignore */ }

package/lib/apm-client/jsonl-file-client.js

@@ -20,12 +20,13 @@ const DEFAULT_ROTATION_SCHEDULE = 'daily';
 const DEFAULT_MAX_LOCAL_RETENTION_DAYS = 0;
 const DEFAULT_MAX_BUFFER_SIZE = 10000;
 
-const DEFAULT_CHANNEL = 'default';
+const DEFAULT_CHANNEL = 'default';  // used when opts.defaultChannel is not set
 
 /**
  * A channel-aware JSONL file client. Routes records to ChannelWriter instances
  * based on channel name. Each channel gets its own file, buffer, and rotation
- * lifecycle. The default channel is 'default'.
+ * lifecycle. The default channel is named 'default' unless overridden via
+ * opts.defaultChannel.
  */
 class JsonlFileClient extends EventEmitter {
   constructor(opts) {
@@ -33,6 +34,7 @@ class JsonlFileClient extends EventEmitter {
 
     this._baseDir = opts.logDir || process.cwd();
     this._baseName = opts.logFilePrefix || 'tracelog';
+    this._defaultChannel = opts.defaultChannel || DEFAULT_CHANNEL;
     this._clock = opts.clock || (() => new Date());
     this._log = opts.logger || null;
     this._destroyed = false;
@@ -105,7 +107,7 @@ class JsonlFileClient extends EventEmitter {
     }
 
     // Create the default channel eagerly.
-    this._getWriter(DEFAULT_CHANNEL);
+    this._getWriter(this._defaultChannel);
 
     // Start periodic flush of all channels.
     const flushIntervalMs = opts.flushIntervalMs || DEFAULT_FLUSH_INTERVAL_MS;
@@ -173,23 +175,23 @@ class JsonlFileClient extends EventEmitter {
   lambdaRegisterTransaction(trans, awsRequestId) {}
 
   sendTransaction(transaction, cb) {
-    this._getWriter(DEFAULT_CHANNEL).send('transaction', transaction, cb);
+    this._getWriter(this._defaultChannel).send('transaction', transaction, cb);
   }
 
   sendSpan(span, cb) {
-    this._getWriter(DEFAULT_CHANNEL).send('span', span, cb);
+    this._getWriter(this._defaultChannel).send('span', span, cb);
   }
 
   sendError(error, cb) {
-    this._getWriter(DEFAULT_CHANNEL).send('error', error, cb);
+    this._getWriter(this._defaultChannel).send('error', error, cb);
   }
 
   sendMetricSet(metricset, cb) {
-    this._getWriter(DEFAULT_CHANNEL).send('metricset', metricset, cb);
+    this._getWriter(this._defaultChannel).send('metricset', metricset, cb);
   }
 
   sendEvent(event, cb) {
-    this._getWriter(DEFAULT_CHANNEL).send('event', event, cb);
+    this._getWriter(this._defaultChannel).send('event', event, cb);
   }
 
   // --- Channel-routed API ---

package/lib/apm-client/s3-uploader.js

@@ -12,31 +12,52 @@ const path = require('path');
 const zlib = require('zlib');
 const { createGzip } = require('zlib');
 const { pipeline } = require('stream');
-const { S3Client, PutObjectCommand } = require('@aws-sdk/client-s3');
+const {
+  S3Client,
+  PutObjectCommand,
+  DeleteObjectCommand,
+} = require('@aws-sdk/client-s3');
+
+// S3 key layout is FIXED (not configurable): it is the contract between
+// tracelog and the in-browser log viewer, which scans the bucket with
+// prefix listings. Channel comes before interval so that prefix-scoped
+// lifecycle rules work and a date-range scan within a channel is a single
+// lexicographically-ordered listing:
+//
+//   {channel}/{interval}/{host}[_{seq}][_current].jsonl[.gz]
+//
+//   server/2026-06-11/172.31.27.225.jsonl.gz
+//   server/2026-06-11/172.31.27.225_current.jsonl.gz
+//   server/2026-06-11/172.31.27.225_1.jsonl.gz
+//
+// There is no serviceName segment: buckets are per-service/per-env, channel
+// names are the top-level namespace (the default channel's name is set via
+// the `defaultChannel` config), and the service name is recorded in every
+// file's metadata line. The basename is underscore-delimited (hostnames
+// cannot contain underscores): host, then a numeric size-rotation seq when
+// > 0, then the literal 'current' for the live file. A host that died
+// mid-interval leaves its final '_current' upload in place, interval intact.
 
 class S3Uploader {
   /**
    * @param {Object} opts
    * @param {string} opts.bucket - S3 bucket name
-   * @param {string} opts.keyTemplate - S3 key template with {variable} placeholders
    * @param {string} [opts.region] - AWS region
    * @param {string} [opts.accessKeyId] - AWS access key ID
    * @param {string} [opts.secretAccessKey] - AWS secret access key
    * @param {string} [opts.sessionToken] - AWS session token
-   * @param {string} opts.serviceName - Service name for template variables
-   * @param {string} [opts.environment] - Environment for template variables
    * @param {boolean} [opts.gzipCompleted=true] - Gzip completed files before upload
    * @param {boolean} [opts.gzipCurrent=true] - Gzip current files before upload
    * @param {Object} [opts.logger] - Logger instance
    * @param {Object} [opts.s3Client] - S3 client instance (must have a send() method).
    *   Defaults to a real S3Client from @aws-sdk/client-s3. Inject a mock for testing.
    * @param {Function} [opts.clock] - Clock provider for testability. Returns a Date.
+   * @param {string} [opts.host] - Host label override (for testing). Defaults to
+   *   the normalized os.hostname().
    */
   constructor(opts) {
     this._bucket = opts.bucket;
-    this._keyTemplate = opts.keyTemplate;
-    this._serviceName = opts.serviceName || 'unknown';
-    this._environment = opts.environment || 'development';
+    this._host = opts.host || normalizeHost(os.hostname());
     this._gzipCompleted = opts.gzipCompleted !== false;
     this._gzipCurrent = opts.gzipCurrent !== false;
     this._log = opts.logger || null;
@@ -68,9 +89,11 @@ class S3Uploader {
   }
 
   /**
-   * Upload a completed (rotated) file, then delete local on success.
+   * Upload a completed (rotated) file, then delete local on success. Also
+   * deletes the now-superseded '_current' snapshot object for the same
+   * interval/seq, so only hosts that died mid-interval leave one behind.
    * @param {string} filePath - Path to the completed JSONL file
-   * @param {Object} vars - Template variables (channel, interval)
+   * @param {Object} vars - Key variables (channel, interval, seq)
    */
   uploadCompleted(filePath, vars) {
     if (!fs.existsSync(filePath)) return;
@@ -83,7 +106,7 @@ class S3Uploader {
   }
 
   _uploadCompletedGzipped(filePath, vars) {
-    const key = this._resolveKey(vars) + '.gz';
+    const key = this._buildKey(vars) + '.gz';
     const gzPath = filePath + '.gz';
 
     this._pendingUploads++;
@@ -116,6 +139,7 @@ class S3Uploader {
           }
           try { fs.unlinkSync(filePath); } catch (e) { /* ignore */ }
           try { fs.unlinkSync(gzPath); } catch (e) { /* ignore */ }
+          this._deleteStaleCurrent(vars);
         })
         .catch((uploadErr) => {
           this._logError(
@@ -132,7 +156,7 @@ class S3Uploader {
   }
 
   _uploadCompletedRaw(filePath, vars) {
-    const key = this._resolveKey(vars);
+    const key = this._buildKey(vars);
 
     this._pendingUploads++;
 
@@ -151,6 +175,7 @@ class S3Uploader {
           this._log.debug('Uploaded completed log to s3://%s/%s', this._bucket, key);
         }
         try { fs.unlinkSync(filePath); } catch (e) { /* ignore */ }
+        this._deleteStaleCurrent(vars);
       })
       .catch((uploadErr) => {
         this._logError(
@@ -165,17 +190,37 @@ class S3Uploader {
   }
 
   /**
-   * Upload the current (incomplete) file without deletion.
-   * Uses a stable S3 key (based on the filename, not a timestamp) so that
-   * each upload overwrites the previous snapshot instead of creating duplicates.
-   * @param {string} filePath - Path to the current JSONL file
-   * @param {Function} [cb] - Callback when upload completes
+   * Delete the '_current' snapshot object superseded by a successful
+   * completed upload of the same channel/interval/seq. Best-effort:
+   * missing objects and failures are ignored.
    */
+  _deleteStaleCurrent(vars) {
+    let key = this._buildKey({ ...vars, current: true });
+    if (this._gzipCurrent) {
+      key += '.gz';
+    }
+
+    this._pendingUploads++;
+    this._s3
+      .send(new DeleteObjectCommand({ Bucket: this._bucket, Key: key }))
+      .then(() => {
+        if (this._log) {
+          this._log.debug('Deleted stale current log s3://%s/%s', this._bucket, key);
+        }
+      })
+      .catch(() => { /* best-effort */ })
+      .finally(() => {
+        this._pendingUploads--;
+      });
+  }
+
   /**
-   * Upload the current (incomplete) file without deletion.
-   * Uses 'current' as the interval so periodic uploads overwrite the same object.
+   * Upload the current (incomplete) file without deletion. The key keeps
+   * the file's real interval and carries a '_current' basename suffix, so
+   * periodic uploads overwrite the same object, and the snapshot sorts
+   * beside its finalized siblings.
    * @param {string} filePath - Path to the current JSONL file
-   * @param {Object} vars - Template variables (channel); interval is set to 'current'
+   * @param {Object} vars - Key variables (channel, interval, seq)
    * @param {Function} [cb] - Callback when upload completes
    */
   uploadCurrent(filePath, vars, cb) {
@@ -188,7 +233,7 @@ class S3Uploader {
       return;
     }
 
-    const currentVars = { ...vars, interval: 'current' };
+    const currentVars = { ...vars, current: true };
     const rawBody = fs.readFileSync(filePath);
 
     if (this._gzipCurrent) {
@@ -199,7 +244,7 @@ class S3Uploader {
   }
 
   _uploadCurrentGzipped(filePath, rawBody, vars, cb) {
-    const key = this._resolveKey(vars) + '.gz';
+    const key = this._buildKey(vars) + '.gz';
 
     this._pendingUploads++;
 
@@ -240,7 +285,7 @@ class S3Uploader {
   }
 
   _uploadCurrentRaw(filePath, rawBody, vars, cb) {
-    const key = this._resolveKey(vars);
+    const key = this._buildKey(vars);
 
     this._pendingUploads++;
 
@@ -271,23 +316,23 @@ class S3Uploader {
   }
 
   /**
-   * Resolve an S3 key from the template.
+   * Build the S3 key for a log file (see the layout contract above).
    *
-   * @param {Object} vars - Template variables:
+   * @param {Object} vars - Key variables:
    *   - {string} channel - Channel name (e.g. 'server', 'client')
-   *   - {string} interval - Period label (e.g. '2026-03-17') or 'current'
+   *   - {string} interval - Period label (e.g. '2026-03-17')
+   *   - {number} [seq] - Size-rotation sequence number within the interval
+   *   - {boolean} [current] - True for the live (incomplete) file snapshot
    */
-  _resolveKey(vars) {
-    const allVars = {
-      serviceName: this._serviceName,
-      environment: this._environment,
-      hostname: os.hostname(),
-      ...vars,
-    };
-
-    return this._keyTemplate.replace(/\{(\w+)\}/g, (match, name) => {
-      return allVars[name] !== undefined ? allVars[name] : match;
-    });
+  _buildKey(vars) {
+    let basename = this._host;
+    if (vars.seq > 0) {
+      basename += `_${vars.seq}`;
+    }
+    if (vars.current) {
+      basename += '_current';
+    }
+    return `${vars.channel}/${vars.interval}/${basename}.jsonl`;
   }
 
   _logError(fmt, ...args) {
@@ -297,12 +342,18 @@ class S3Uploader {
   }
 }
 
-function _pad2(n) {
-  return String(n).padStart(2, '0');
-}
-
-function _fmtDate(d) {
-  return `${d.getFullYear()}-${_pad2(d.getMonth() + 1)}-${_pad2(d.getDate())}`;
+/**
+ * Normalize a hostname into the host label used in S3 keys. EC2 internal
+ * hostnames (ip-A-B-C-D or ip-A-B-C-D.ec2.internal etc.) become the dotted
+ * IP address, which avoids embedding hyphens in the basename; any other
+ * hostname is used as-is.
+ */
+function normalizeHost(hostname) {
+  const m = /^ip-(\d{1,3})-(\d{1,3})-(\d{1,3})-(\d{1,3})(\..*)?$/.exec(hostname);
+  if (m) {
+    return `${m[1]}.${m[2]}.${m[3]}.${m[4]}`;
+  }
+  return hostname;
 }
 
-module.exports = { S3Uploader };
+module.exports = { S3Uploader, normalizeHost };

package/lib/config/schema.js

@@ -509,7 +509,15 @@ const CONFIG_SCHEMA = [
     defaultValue: 10000,
     envVar: 'TRACELOG_MAX_BUFFER_SIZE',
   },
-  // S3 upload
+  // Channels
+  {
+    name: 'defaultChannel',
+    configType: 'string',
+    defaultValue: 'default',
+    envVar: 'TRACELOG_DEFAULT_CHANNEL',
+  },
+  // S3 upload. The S3 key layout is fixed (the contract with the log
+  // viewer): {channel}/{interval}/{host}[_{seq}][_current].jsonl[.gz]
   {
     name: 's3Bucket',
     configType: 'string',
@@ -522,13 +530,6 @@ const CONFIG_SCHEMA = [
     defaultValue: undefined,
     envVar: 'TRACELOG_S3_REGION',
   },
-  {
-    name: 's3KeyTemplate',
-    configType: 'string',
-    defaultValue:
-      '{serviceName}/{environment}/{date}/{hostname}-{pid}-{timestamp}.jsonl',
-    envVar: 'TRACELOG_S3_KEY_TEMPLATE',
-  },
   {
     name: 's3UploadIntervalMs',
     configType: 'number',

package/lib/instrumentation/index.js

@@ -710,6 +710,14 @@ Instrumentation.prototype.addEndedTransaction = function (transaction) {
       trans: transaction.id,
       trace: transaction.traceId,
     });
+    // The transaction's name is final now, so held spans can be routed
+    // even though the transaction record itself was dropped.
+    this._drainPendingChannelSpans(
+      transaction,
+      agent._channelForTransactionName(
+        transaction._customName || transaction._defaultName,
+      ),
+    );
     return;
   }
 
@@ -718,6 +726,9 @@ Instrumentation.prototype.addEndedTransaction = function (transaction) {
     trace: transaction.traceId,
   });
   const channel = agent._channelForTransactionName(payload.name);
+  // Spans held while this transaction's name was unresolved follow the
+  // same routing decision as the transaction itself.
+  this._drainPendingChannelSpans(transaction, channel);
   if (channel && typeof agent._apmClient.sendToChannel === 'function') {
     agent._apmClient.sendToChannel(channel, 'transaction', payload);
   } else {
@@ -725,6 +736,31 @@ Instrumentation.prototype.addEndedTransaction = function (transaction) {
   }
 };
 
+// Send any span payloads that were held on the transaction because the
+// transaction's name (and so its channel routing) was not yet resolved
+// when the span ended. See `_encodeAndSendSpan`.
+Instrumentation.prototype._drainPendingChannelSpans = function (
+  transaction,
+  channel,
+) {
+  const agent = this._agent;
+  const pending = transaction._pendingChannelSpans;
+  if (!pending) {
+    return;
+  }
+  transaction._pendingChannelSpans = null;
+  if (!agent._apmClient) {
+    return;
+  }
+  for (const payload of pending) {
+    if (channel && typeof agent._apmClient.sendToChannel === 'function') {
+      agent._apmClient.sendToChannel(channel, 'span', payload);
+    } else {
+      agent._apmClient.sendSpan(payload);
+    }
+  }
+};
+
 Instrumentation.prototype.addEndedSpan = function (span) {
   var agent = this._agent;
 
@@ -847,15 +883,38 @@ Instrumentation.prototype._encodeAndSendSpan = function (span) {
         });
         if (agent._apmClient) {
           // Spans follow their transaction's channel. The transaction name
-          // may not be final until the transaction ends, so this routing is
-          // best-effort for spans that end before the route is resolved.
-          const channel = agent._channelForTransactionName(
-            span.transaction && span.transaction.name,
-          );
-          if (channel && typeof agent._apmClient.sendToChannel === 'function') {
-            agent._apmClient.sendToChannel(channel, 'span', payload);
-          } else {
+          // is only meaningful for routing once *resolved* (custom or
+          // framework-set); the `name` getter must not be used here, since
+          // until the transaction ends it falls back to
+          // '<METHOD> unknown route (unnamed)', which would spuriously
+          // match rules aimed at unmatched-route traffic. When routing is
+          // configured and the name is not yet resolved (e.g. Express only
+          // names transactions when the request finishes), hold the span on
+          // its transaction; addEndedTransaction() drains held spans with
+          // the transaction's final routing decision.
+          const trans = span.transaction;
+          const routingEnabled =
+            agent._conf.transactionChannelRules &&
+            agent._conf.transactionChannelRules.length > 0 &&
+            typeof agent._apmClient.sendToChannel === 'function';
+          if (!routingEnabled) {
             agent._apmClient.sendSpan(payload);
+          } else {
+            const resolvedName =
+              trans && (trans._customName || trans._defaultName);
+            if (!resolvedName && trans && !trans.ended) {
+              if (!trans._pendingChannelSpans) {
+                trans._pendingChannelSpans = [];
+              }
+              trans._pendingChannelSpans.push(payload);
+            } else {
+              const channel = agent._channelForTransactionName(resolvedName);
+              if (channel) {
+                agent._apmClient.sendToChannel(channel, 'span', payload);
+              } else {
+                agent._apmClient.sendSpan(payload);
+              }
+            }
           }
         }
       }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redthreadlabs/tracelog",
-  "version": "1.5.0",
+  "version": "1.6.0",
   "description": "Node.js APM instrumentation that writes traces to JSONL files",
   "publishConfig": {
     "access": "public"