@redthreadlabs/tracelog 1.4.0

package/lib/apm-client/apm-client.js

@@ -0,0 +1,107 @@
+/*
+ * Copyright Elasticsearch B.V. and other contributors where applicable.
+ * Copyright Shaxpir Inc. All rights reserved.
+ * Licensed under the BSD 2-Clause License; you may not use this file except in
+ * compliance with the BSD 2-Clause License.
+ */
+
+'use strict';
+
+const { INTAKE_STRING_MAX_SIZE } = require('../constants');
+const { CloudMetadata } = require('../cloud-metadata');
+const { JsonlFileClient } = require('./jsonl-file-client');
+const { NoopApmClient } = require('./noop-apm-client');
+const { S3Uploader } = require('./s3-uploader');
+
+/**
+ * Returns a tracelog client suited for the configuration provided.
+ *
+ * @param {Object} config The agent's configuration
+ * @param {Object} agent The agent instance
+ */
+function createApmClient(config, agent) {
+  if (config.contextPropagationOnly) {
+    return new NoopApmClient();
+  } else if (typeof config.transport === 'function') {
+    return config.transport(config, agent);
+  }
+
+  // Create S3 uploader if a bucket is configured.
+  let s3Uploader = null;
+  if (config.s3Bucket) {
+    s3Uploader = new S3Uploader({
+      bucket: config.s3Bucket,
+      keyTemplate:
+        config.s3KeyTemplate ||
+        '{serviceName}/{environment}/{hostname}-{channel}-{interval}.jsonl',
+      region: config.s3Region,
+      accessKeyId: config.s3AccessKeyId,
+      secretAccessKey: config.s3SecretAccessKey,
+      sessionToken: config.s3SessionToken,
+      s3Client: config.s3Client, // optional: inject a mock for testing
+      gzipCompleted: config.s3GzipCompleted,
+      gzipCurrent: config.s3GzipCurrent,
+      serviceName: config.serviceName,
+      environment: config.environment,
+      logger: config.logger,
+    });
+  }
+
+  const client = new JsonlFileClient({
+    serviceName: config.serviceName,
+    serviceVersion: config.serviceVersion,
+    environment: config.environment,
+    globalLabels: maybePairsToObject(config.globalLabels),
+
+    // Sanitize conf
+    truncateKeywordsAt: INTAKE_STRING_MAX_SIZE,
+    truncateLongFieldsAt: config.longFieldMaxLength,
+
+    // JSONL file options
+    logDir: config.logDir,
+    logFilePrefix: config.logFilePrefix,
+    maxFileSize: config.logMaxFileSize,
+    flushIntervalMs: config.logFlushIntervalMs,
+    rotationSchedule: config.logRotationSchedule,
+    maxLocalRetentionDays: config.maxLocalRetentionDays,
+    maxBufferSize: config.maxBufferSize,
+
+    // S3 upload
+    s3Uploader,
+    s3UploadIntervalMs: config.s3UploadIntervalMs,
+
+    // Cloud metadata
+    cloudMetadataFetcher:
+      config.cloudProvider !== 'none'
+        ? new CloudMetadata(
+            config.cloudProvider || 'auto',
+            agent.logger,
+            config.serviceName,
+          )
+        : null,
+
+    // Logging
+    logger: config.logger,
+  });
+
+  client.on('error', (err) => {
+    agent.logger.error('Tracelog transport error: %s', err.stack);
+  });
+
+  return client;
+}
+
+function maybePairsToObject(pairs) {
+  return pairs ? pairsToObject(pairs) : undefined;
+}
+
+function pairsToObject(pairs) {
+  return pairs.reduce((object, [key, value]) => {
+    object[key] = value;
+    return object;
+  }, {});
+}
+
+module.exports = {
+  createApmClient,
+};

package/lib/apm-client/channel-writer.js

@@ -0,0 +1,334 @@
+/*
+ * Copyright Shaxpir Inc. All rights reserved.
+ * Licensed under the BSD 2-Clause License; you may not use this file except in
+ * compliance with the BSD 2-Clause License.
+ */
+
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const ndjson = require('./ndjson');
+const truncate = require('./truncate');
+
+const DEFAULT_MAX_FILE_SIZE = 100 * 1024 * 1024; // 100MB
+const DEFAULT_MAX_BUFFER_SIZE = 10000;
+
+const ROTATION_SCHEDULES = {
+  hourly: 60 * 60 * 1000,
+  daily: 24 * 60 * 60 * 1000,
+};
+
+/**
+ * A ChannelWriter manages a single JSONL file stream for a named channel.
+ * It handles buffering, rotation (time-based and size-based), metadata
+ * writing, and S3 upload coordination.
+ */
+class ChannelWriter {
+  /**
+   * @param {Object} opts
+   * @param {string} opts.channel - Channel name (e.g. 'server', 'client')
+   * @param {string} opts.baseDir - Output directory
+   * @param {string} opts.baseName - File prefix (e.g. 'tracelog')
+   * @param {Object} opts.truncOpts - Truncation options
+   * @param {Object} opts.metadata - Metadata object to write at start of each file
+   * @param {Object} [opts.metadataFilters] - Metadata filter chain
+   * @param {Object} [opts.s3Uploader] - S3 uploader instance
+   * @param {number} [opts.maxFileSize] - Max file size before rotation
+   * @param {number} [opts.maxBufferSize] - Max buffered lines before dropping oldest
+   * @param {string|number} [opts.rotationSchedule] - 'daily', 'hourly', or ms
+   * @param {number} [opts.maxLocalRetentionDays] - Auto-delete old files after N days
+   * @param {Function} [opts.clock] - Clock provider for testability
+   * @param {Object} [opts.logger] - Logger instance
+   */
+  constructor(opts) {
+    this._channel = opts.channel;
+    this._baseDir = opts.baseDir;
+    this._baseName = `${opts.baseName}-${opts.channel}`;
+    this._ext = '.jsonl';
+
+    this._maxFileSize = opts.maxFileSize || DEFAULT_MAX_FILE_SIZE;
+    this._maxBufferSize = opts.maxBufferSize || DEFAULT_MAX_BUFFER_SIZE;
+    this._maxLocalRetentionDays = opts.maxLocalRetentionDays || 0;
+    this._truncOpts = opts.truncOpts;
+    this._metadata = opts.metadata;
+    this._metadataFilters = opts.metadataFilters;
+    this._extraMetadata = opts.extraMetadata || null;
+    this._s3Uploader = opts.s3Uploader || null;
+    this._clock = opts.clock || (() => new Date());
+    this._log = opts.logger || null;
+
+    // Rotation schedule
+    const schedule = opts.rotationSchedule || 'daily';
+    if (typeof schedule === 'string' && ROTATION_SCHEDULES[schedule]) {
+      this._rotationIntervalMs = ROTATION_SCHEDULES[schedule];
+    } else if (typeof schedule === 'number' && schedule > 0) {
+      this._rotationIntervalMs = schedule;
+    } else {
+      this._rotationIntervalMs = ROTATION_SCHEDULES.daily;
+    }
+
+    this._buffer = [];
+    this._currentFileSize = 0;
+    this._wroteMetadata = false;
+    this._destroyed = false;
+
+    // Track current time period and sequence number for file naming.
+    this._currentPeriodLabel = this._getPeriodLabel(this._clock());
+    this._currentSeqNum = 0;
+    this._currentFilePath = this._buildFilePath(
+      this._currentPeriodLabel,
+      this._currentSeqNum,
+    );
+
+    // Resume existing file if present.
+    this._resumeExistingFile();
+
+    // Upload any orphaned files from previous runs.
+    if (this._s3Uploader) {
+      this._uploadOrphanedFiles();
+    }
+  }
+
+  get channel() {
+    return this._channel;
+  }
+
+  get currentFilePath() {
+    return this._currentFilePath;
+  }
+
+  send(type, data, cb) {
+    if (this._destroyed) {
+      if (cb) process.nextTick(cb);
+      return;
+    }
+
+    try {
+      const truncated = truncate[type]
+        ? truncate[type](data, this._truncOpts)
+        : data;
+
+      const obj = {};
+      obj[type] = truncated;
+      const line = ndjson.serialize(obj);
+
+      if (this._buffer.length >= this._maxBufferSize) {
+        this._buffer.shift();
+      }
+      this._buffer.push(line);
+    } catch (err) {
+      if (this._log) {
+        this._log.error('ChannelWriter[%s] serialize error: %s', this._channel, err.message);
+      }
+    }
+
+    if (cb) {
+      process.nextTick(cb);
+    }
+  }
+
+  flush() {
+    if (this._buffer.length === 0) return;
+
+    try {
+      this._checkTimeRotation();
+
+      const lines = this._buffer;
+      this._buffer = [];
+
+      for (const line of lines) {
+        if (this._currentFileSize >= this._maxFileSize) {
+          this._rotate();
+          this._currentSeqNum++;
+          this._currentFilePath = this._buildFilePath(
+            this._currentPeriodLabel,
+            this._currentSeqNum,
+          );
+        }
+
+        let output = '';
+        if (!this._wroteMetadata) {
+          const metadataLine = this._getMetadataLine();
+          if (metadataLine) {
+            output += metadataLine;
+          }
+          this._wroteMetadata = true;
+        }
+
+        output += line;
+        fs.appendFileSync(this._currentFilePath, output, 'utf8');
+        this._currentFileSize += Buffer.byteLength(output, 'utf8');
+      }
+    } catch (err) {
+      if (this._log) {
+        this._log.error('ChannelWriter[%s] flush error: %s', this._channel, err.message);
+      }
+    }
+  }
+
+  uploadCurrent() {
+    if (this._s3Uploader) {
+      this._s3Uploader.uploadCurrent(this._currentFilePath, { channel: this._channel });
+    }
+  }
+
+  destroy() {
+    if (this._destroyed) return;
+    this._destroyed = true;
+    this.flush();
+    this.uploadCurrent();
+  }
+
+  setExtraMetadata(metadata) {
+    this._extraMetadata = metadata;
+  }
+
+  // --- Internal methods ---
+
+  _getMetadataLine() {
+    let metadata = Object.assign({}, this._metadata);
+    if (this._extraMetadata) {
+      metadata = Object.assign(metadata, this._extraMetadata);
+    }
+    if (this._metadataFilters) {
+      metadata = this._metadataFilters.process(metadata);
+    }
+    if (!metadata) return null;
+    metadata.channel = this._channel;
+    return ndjson.serialize({ metadata });
+  }
+
+  _checkTimeRotation() {
+    const now = this._clock();
+    const periodLabel = this._getPeriodLabel(now);
+    if (periodLabel !== this._currentPeriodLabel) {
+      this._rotate();
+      this._currentPeriodLabel = periodLabel;
+      this._currentSeqNum = 0;
+      this._currentFilePath = this._buildFilePath(periodLabel, 0);
+    }
+  }
+
+  _rotate() {
+    const completedFilePath = this._currentFilePath;
+    this._currentFileSize = 0;
+    this._wroteMetadata = false;
+
+    if (this._s3Uploader && fs.existsSync(completedFilePath)) {
+      this._s3Uploader.uploadCompleted(completedFilePath, { channel: this._channel, interval: this._currentPeriodLabel });
+    }
+
+    if (this._maxLocalRetentionDays > 0) {
+      this._cleanupOldFiles();
+    }
+  }
+
+  _cleanupOldFiles() {
+    try {
+      const cutoff = new Date(
+        this._clock().getTime() -
+          this._maxLocalRetentionDays * 24 * 60 * 60 * 1000,
+      );
+      const cutoffLabel = this._getPeriodLabel(cutoff);
+      const prefix = this._baseName + '-';
+      const files = fs.readdirSync(this._baseDir);
+
+      for (const file of files) {
+        if (!file.startsWith(prefix) || !file.endsWith(this._ext)) continue;
+        const filePath = path.join(this._baseDir, file);
+        if (filePath === this._currentFilePath) continue;
+
+        const withoutPrefix = file.slice(prefix.length);
+        const withoutExt = withoutPrefix.slice(0, withoutPrefix.length - this._ext.length);
+        const periodLabel = withoutExt.replace(/\.\d+$/, '');
+
+        if (periodLabel <= cutoffLabel) {
+          try { fs.unlinkSync(filePath); } catch (e) { /* ignore */ }
+        }
+      }
+    } catch (e) { /* ignore */ }
+  }
+
+  _buildFilePath(periodLabel, seqNum) {
+    const seq = seqNum > 0 ? `.${seqNum}` : '';
+    return path.join(
+      this._baseDir,
+      `${this._baseName}-${periodLabel}${seq}${this._ext}`,
+    );
+  }
+
+  _getPeriodLabel(date) {
+    const y = date.getFullYear();
+    const m = _pad2(date.getMonth() + 1);
+    const d = _pad2(date.getDate());
+
+    if (this._rotationIntervalMs >= 24 * 60 * 60 * 1000) {
+      return `${y}-${m}-${d}`;
+    }
+
+    const h = _pad2(date.getHours());
+    if (this._rotationIntervalMs >= 60 * 60 * 1000) {
+      return `${y}-${m}-${d}T${h}`;
+    }
+
+    const min = date.getMinutes();
+    const intervalMinutes = Math.floor(this._rotationIntervalMs / (60 * 1000));
+    const flooredMin = Math.floor(min / intervalMinutes) * intervalMinutes;
+    return `${y}-${m}-${d}T${h}${_pad2(flooredMin)}`;
+  }
+
+  _resumeExistingFile() {
+    let seq = 0;
+    while (true) {
+      const candidate = this._buildFilePath(this._currentPeriodLabel, seq + 1);
+      if (fs.existsSync(candidate)) {
+        seq = seq + 1;
+      } else {
+        break;
+      }
+    }
+
+    this._currentSeqNum = seq;
+    this._currentFilePath = this._buildFilePath(
+      this._currentPeriodLabel,
+      this._currentSeqNum,
+    );
+
+    try {
+      const stat = fs.statSync(this._currentFilePath);
+      this._currentFileSize = stat.size;
+      this._wroteMetadata = true;
+    } catch (e) {
+      // File doesn't exist yet.
+    }
+  }
+
+  _uploadOrphanedFiles() {
+    try {
+      const prefix = this._baseName + '-';
+      const files = fs.readdirSync(this._baseDir);
+
+      for (const file of files) {
+        if (!file.startsWith(prefix) || !file.endsWith(this._ext)) continue;
+        const filePath = path.join(this._baseDir, file);
+        if (filePath === this._currentFilePath) continue;
+
+        const withoutPrefix = file.slice(prefix.length);
+        const withoutExt = withoutPrefix.slice(0, withoutPrefix.length - this._ext.length);
+        const periodLabel = withoutExt.replace(/\.\d+$/, '');
+
+        if (periodLabel < this._currentPeriodLabel) {
+          this._s3Uploader.uploadCompleted(filePath, { channel: this._channel, interval: periodLabel });
+        }
+      }
+    } catch (e) { /* ignore */ }
+  }
+}
+
+function _pad2(n) {
+  return String(n).padStart(2, '0');
+}
+
+module.exports = { ChannelWriter };

package/lib/apm-client/jsonl-file-client.js

@@ -0,0 +1,241 @@
+/*
+ * Copyright Elasticsearch B.V. and other contributors where applicable.
+ * Copyright Shaxpir Inc. All rights reserved.
+ * Licensed under the BSD 2-Clause License; you may not use this file except in
+ * compliance with the BSD 2-Clause License.
+ */
+
+'use strict';
+
+const EventEmitter = require('events');
+const fs = require('fs');
+const os = require('os');
+
+const Filters = require('object-filter-sequence');
+
+const { ChannelWriter } = require('./channel-writer');
+
+const DEFAULT_FLUSH_INTERVAL_MS = 1000;
+const DEFAULT_ROTATION_SCHEDULE = 'daily';
+const DEFAULT_MAX_LOCAL_RETENTION_DAYS = 0;
+const DEFAULT_MAX_BUFFER_SIZE = 10000;
+
+const DEFAULT_CHANNEL = 'default';
+
+/**
+ * A channel-aware JSONL file client. Routes records to ChannelWriter instances
+ * based on channel name. Each channel gets its own file, buffer, and rotation
+ * lifecycle. The default channel is 'default'.
+ */
+class JsonlFileClient extends EventEmitter {
+  constructor(opts) {
+    super();
+
+    this._baseDir = opts.logDir || process.cwd();
+    this._baseName = opts.logFilePrefix || 'tracelog';
+    this._clock = opts.clock || (() => new Date());
+    this._log = opts.logger || null;
+    this._destroyed = false;
+
+    // Ensure the output directory exists.
+    fs.mkdirSync(this._baseDir, { recursive: true });
+
+    // Shared config for all channel writers.
+    this._writerOpts = {
+      baseDir: this._baseDir,
+      baseName: this._baseName,
+      truncOpts: {
+        truncateKeywordsAt:
+          opts.truncateKeywordsAt != null ? opts.truncateKeywordsAt : 1024,
+        truncateLongFieldsAt:
+          opts.truncateLongFieldsAt != null ? opts.truncateLongFieldsAt : 10000,
+        truncateErrorMessagesAt:
+          opts.truncateErrorMessagesAt != null
+            ? opts.truncateErrorMessagesAt
+            : undefined,
+      },
+      metadata: {
+        service: {
+          name: opts.serviceName || 'unknown',
+          version: opts.serviceVersion || undefined,
+          environment: opts.environment || undefined,
+          agent: { name: 'tracelog', version: require('../../package').version },
+        },
+        process: {
+          pid: process.pid,
+          title: process.title,
+          argv: process.argv,
+        },
+        system: {
+          hostname: os.hostname(),
+          architecture: os.arch(),
+          platform: os.platform(),
+        },
+        ...(opts.globalLabels && { labels: opts.globalLabels }),
+      },
+      s3Uploader: opts.s3Uploader || null,
+      maxFileSize: opts.maxFileSize,
+      maxBufferSize: opts.maxBufferSize || DEFAULT_MAX_BUFFER_SIZE,
+      rotationSchedule: opts.rotationSchedule || DEFAULT_ROTATION_SCHEDULE,
+      maxLocalRetentionDays:
+        opts.maxLocalRetentionDays != null
+          ? opts.maxLocalRetentionDays
+          : DEFAULT_MAX_LOCAL_RETENTION_DAYS,
+      clock: this._clock,
+      logger: this._log,
+    };
+
+    this._metadataFilters = new Filters();
+    this._extraMetadata = null;
+    this._cloudMetadataReady = false;
+
+    // Channel writers, keyed by channel name.
+    this._writers = new Map();
+
+    // Fetch cloud metadata asynchronously if a fetcher is provided.
+    if (opts.cloudMetadataFetcher) {
+      opts.cloudMetadataFetcher.getCloudMetadata((err, cloudMetadata) => {
+        if (!err && cloudMetadata) {
+          this._writerOpts.metadata.cloud = cloudMetadata;
+        }
+        this._cloudMetadataReady = true;
+      });
+    } else {
+      this._cloudMetadataReady = true;
+    }
+
+    // Create the default channel eagerly.
+    this._getWriter(DEFAULT_CHANNEL);
+
+    // Start periodic flush of all channels.
+    const flushIntervalMs = opts.flushIntervalMs || DEFAULT_FLUSH_INTERVAL_MS;
+    this._flushTimer = setInterval(() => {
+      for (const writer of this._writers.values()) {
+        writer.flush();
+      }
+    }, flushIntervalMs);
+    this._flushTimer.unref();
+
+    // Start periodic S3 upload of current files for all channels.
+    this._s3UploadIntervalMs = opts.s3UploadIntervalMs || 0;
+    this._s3UploadTimer = null;
+    if (opts.s3Uploader && this._s3UploadIntervalMs > 0) {
+      this._s3UploadTimer = setInterval(() => {
+        for (const writer of this._writers.values()) {
+          writer.uploadCurrent();
+        }
+      }, this._s3UploadIntervalMs);
+      this._s3UploadTimer.unref();
+    }
+  }
+
+  // --- Channel management ---
+
+  /**
+   * Get or create a ChannelWriter for the given channel name.
+   */
+  _getWriter(channel) {
+    if (!this._writers.has(channel)) {
+      const writer = new ChannelWriter({
+        ...this._writerOpts,
+        channel,
+        metadataFilters: this._metadataFilters,
+        extraMetadata: this._extraMetadata,
+      });
+      this._writers.set(channel, writer);
+    }
+    return this._writers.get(channel);
+  }
+
+  // --- Public API (default channel) ---
+
+  config(opts) {}
+
+  addMetadataFilter(fn) {
+    this._metadataFilters.push(fn);
+  }
+
+  setExtraMetadata(metadata) {
+    this._extraMetadata = metadata;
+    for (const writer of this._writers.values()) {
+      writer.setExtraMetadata(metadata);
+    }
+  }
+
+  supportsKeepingUnsampledTransaction() {
+    return true;
+  }
+
+  lambdaStart() {}
+  lambdaShouldRegisterTransactions() {
+    return true;
+  }
+  lambdaRegisterTransaction(trans, awsRequestId) {}
+
+  sendTransaction(transaction, cb) {
+    this._getWriter(DEFAULT_CHANNEL).send('transaction', transaction, cb);
+  }
+
+  sendSpan(span, cb) {
+    this._getWriter(DEFAULT_CHANNEL).send('span', span, cb);
+  }
+
+  sendError(error, cb) {
+    this._getWriter(DEFAULT_CHANNEL).send('error', error, cb);
+  }
+
+  sendMetricSet(metricset, cb) {
+    this._getWriter(DEFAULT_CHANNEL).send('metricset', metricset, cb);
+  }
+
+  sendEvent(event, cb) {
+    this._getWriter(DEFAULT_CHANNEL).send('event', event, cb);
+  }
+
+  // --- Channel-routed API ---
+
+  sendToChannel(channel, type, data, cb) {
+    this._getWriter(channel).send(type, data, cb);
+  }
+
+  // --- Lifecycle ---
+
+  flush(opts, cb) {
+    if (typeof opts === 'function') {
+      cb = opts;
+      opts = {};
+    } else if (!opts) {
+      opts = {};
+    }
+
+    for (const writer of this._writers.values()) {
+      writer.flush();
+    }
+
+    if (cb) {
+      process.nextTick(cb);
+    }
+  }
+
+  destroy() {
+    if (this._destroyed) return;
+    this._destroyed = true;
+
+    if (this._flushTimer) {
+      clearInterval(this._flushTimer);
+      this._flushTimer = null;
+    }
+    if (this._s3UploadTimer) {
+      clearInterval(this._s3UploadTimer);
+      this._s3UploadTimer = null;
+    }
+
+    for (const writer of this._writers.values()) {
+      writer.destroy();
+    }
+  }
+}
+
+module.exports = {
+  JsonlFileClient,
+};

package/lib/apm-client/ndjson.js

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and other contributors where applicable.
+ * Licensed under the BSD 2-Clause License; you may not use this file except in
+ * compliance with the BSD 2-Clause License.
+ */
+
+'use strict';
+
+const stringify = require('fast-safe-stringify');
+
+exports.serialize = function serialize(obj) {
+  const str = tryJSONStringify(obj) || stringify(obj);
+  return str + '\n';
+};
+
+function tryJSONStringify(obj) {
+  try {
+    return JSON.stringify(obj);
+  } catch (e) {}
+}