@forge-ops/tracker 0.3.0 → 0.9.0

package/src/integrations/express.js

@@ -1,4 +1,4 @@
-import { captureException } from "../index.js";
+import { captureException, runWithUser } from "../index.js";
 
 /**
  * Express error-handling middleware. Register last, after all routes:
@@ -26,3 +26,52 @@ export function forgeOpsTrackerExpressMiddleware(err, req, res, next) {
   });
   next(err);
 }
+
+/**
+ * Automatically identifies the affected user for every error reported for the rest of this
+ * request, when `req.user` is present. Register early, before your routes (and after whatever
+ * middleware actually sets `req.user`, e.g. Passport's own session middleware):
+ *
+ *     app.use(passport.session());
+ *     app.use(forgeOpsTrackerUserContextMiddleware);
+ *     // ...routes...
+ *     app.use(forgeOpsTrackerExpressMiddleware);
+ *
+ * `req.user` is Passport's own convention (the closest thing Express has to a single dominant
+ * auth library, the same role Warden plays for Rails): set once `passport.authenticate()`/session
+ * deserialization succeeds, left `undefined` otherwise, so a plain truthy check is enough, the
+ * same duck-typed "is there a user object at all" check `gems/forge_ops_tracker`'s own Warden
+ * integration does for `env["warden"].user`. A no-op for an app that never set `req.user` at all,
+ * whether that's because nobody's signed in or Passport (or an equivalent) isn't installed.
+ *
+ * Wraps the rest of the request in `runWithUser()` (see that function's own doc comment for why
+ * this SDK has no imperative `setUser()`) rather than setting anything imperatively itself:
+ * confirmed directly, with a real Express app and a real async route handler, that
+ * `AsyncLocalStorage`'s context set by an early middleware like this one does survive all the way
+ * through to a later error-handling middleware (`forgeOpsTrackerExpressMiddleware`, typically
+ * registered last, after every route) in the same request, not just to handlers registered
+ * synchronously right after this one.
+ */
+export function forgeOpsTrackerUserContextMiddleware(req, res, next) {
+  if (!req.user) {
+    next();
+    return;
+  }
+  runWithUser(serializeUser(req.user), next);
+}
+
+function serializeUser(user) {
+  const result = {};
+  if (user.id !== undefined && user.id !== null) {
+    result.id = String(user.id);
+  }
+  if (user.email) {
+    result.email = user.email;
+  }
+  if (user.username) {
+    result.username = user.username;
+  } else if (user.name) {
+    result.username = user.name;
+  }
+  return result;
+}

package/src/integrations/performance.js

@@ -1,10 +1,11 @@
-import { _recordPerformance } from "../index.js";
+import { _recordBreadcrumb, _recordPerformance } from "../index.js";
 
 /**
  * Express performance-tracking middleware. Register wherever is convenient relative to routes
  * (unlike sessionTracking.js's own middleware, this doesn't need to run first: it only reads
  * req.route once the request has already been routed, inside the "finish" listener below, not
- * at request-start):
+ * at request-start), but after forgeOpsTrackerBreadcrumbContextExpressMiddleware (see
+ * integrations/breadcrumbContext.js), so the breadcrumb recorded below has a trail to land in:
  *
  *   app.use(forgeOpsTrackerPerformanceExpressMiddleware);
  *
@@ -17,9 +18,30 @@ import { _recordPerformance } from "../index.js";
  * transactionName is "<HTTP method> <route pattern>", e.g. "GET /users/:id", not the raw URL:
  * req.route.path is Express's own matched route pattern, read here (after routing has actually
  * happened, unlike at request-start where it isn't populated yet), which keeps a distinct user
- * id from exploding into its own separate transaction the way the literal URL would -- the
+ * id from exploding into its own separate transaction the way the literal URL would: the
  * low-cardinality equivalent of a Rails controller#action. Falls back to req.path (the literal,
  * unmatched path) for a request that never matched a route at all (a 404).
+ *
+ * The same res.on("finish") listener also records a "controller" breadcrumb, gated on
+ * trackBreadcrumbs independently of trackPerformance (each call below checks its own flag,
+ * mirroring gems/forge_ops_tracker/lib/forge_ops_tracker/railtie.rb's own
+ * process_action.action_controller subscription, which records both a performance sample and a
+ * breadcrumb from the one event): an app could want the trail without the timing data, or vice
+ * versa. This is currently the only place this client automatically records a breadcrumb at all:
+ * unlike Rails' framework-wide ActiveSupport::Notifications, this client has no automatic
+ * instrumentation source that isn't also this same opt-in middleware, so an app that wants
+ * automatic breadcrumbs (not just manual ones from addBreadcrumb()) needs this middleware
+ * installed regardless of whether it also wants the performance data. There's no query-level
+ * automatic breadcrumb source for the same reason there's no query-level performance
+ * instrumentation in this client at all today (no ORM/DB driver integration exists yet to hang
+ * one off of); this doesn't invent one.
+ *
+ * Deliberately recorded with the request's final status already known (from the same
+ * res.on("finish") firing this middleware already needed for timing), the same "recorded once
+ * the response actually finished" shape as Ruby's/Python's own automatic controller breadcrumb;
+ * see those implementations' own documented gap (a breadcrumb recorded this late can never show
+ * up on that exact same request's own error, only a later one on the same connection/process
+ * could see it) for why that's an accepted trade-off, not an oversight here either.
  */
 export function forgeOpsTrackerPerformanceExpressMiddleware(req, res, next) {
   const start = performance.now();
@@ -28,6 +50,10 @@ export function forgeOpsTrackerPerformanceExpressMiddleware(req, res, next) {
     const durationMs = performance.now() - start;
     const transactionName = `${req.method} ${req.route?.path ?? req.path}`;
     _recordPerformance(transactionName, durationMs);
+    _recordBreadcrumb(transactionName, "controller", res.statusCode >= 500 ? "error" : "info", {
+      status: res.statusCode,
+      duration_ms: Math.round(durationMs * 10) / 10,
+    });
   });
 
   next();

package/src/integrations/tracing.js

@@ -0,0 +1,80 @@
+import { _finishSpanTrace, _runWithSpanTrace } from "../index.js";
+
+/**
+ * Express distributed-tracing middleware: wraps the rest of the request in _runWithSpanTrace
+ * (see that function's own comment in index.js), the same "own file, own registration point"
+ * shape breadcrumbContext.js already established for a genuinely distinct concern (this is
+ * separate from, and independent of, error capture, session tracking, and aggregate performance
+ * timing). Register wherever is convenient relative to routes, the same as
+ * forgeOpsTrackerPerformanceExpressMiddleware, since it only reads req.route once the request has
+ * already been routed, inside the "finish" listener below, not at request-start:
+ *
+ *   app.use(forgeOpsTrackerTracingExpressMiddleware);
+ *
+ * transactionName is built exactly like forgeOpsTrackerPerformanceExpressMiddleware's own
+ * ("<HTTP method> <route pattern>", never the raw URL): this is the same request a performance
+ * sample would already be recorded for, so a trace's own root span name has to match, not invent
+ * a second naming scheme for the same thing.
+ *
+ * The actual send-or-don't decision happens inside _finishSpanTrace, only once the response has
+ * actually finished and the root span's real total duration is known: nothing about a normal,
+ * fast request costs a single byte over the wire, matching gems/forge_ops_tracker's own
+ * Middleware::SpanTracing.
+ */
+export function forgeOpsTrackerTracingExpressMiddleware(req, res, next) {
+  _runWithSpanTrace(() => {
+    const startedAt = new Date();
+    const start = performance.now();
+
+    res.on("finish", () => {
+      const durationMs = performance.now() - start;
+      const transactionName = `${req.method} ${req.route?.path ?? req.path}`;
+      _finishSpanTrace(transactionName, startedAt, durationMs);
+    });
+
+    next();
+  });
+}
+
+/**
+ * Fastify equivalent, added as two separate hooks rather than one function wrapping "the rest of
+ * this request" the way Express's next() allows (see
+ * registerForgeOpsTrackerBreadcrumbContext's own comment in breadcrumbContext.js for why Fastify
+ * needs this shape at all): onRequest establishes the span trace (wrapping `done`, the same
+ * pattern that file already uses) and stashes this request's own start time on `request` itself,
+ * the same "flag stashed on the request object" convention integrations/express.js's own
+ * req._forgeOpsSessionCrashed already established; onResponse, Fastify's own dedicated
+ * "the response has actually been sent" hook, reads that back once the real total duration is
+ * known:
+ *
+ *   registerForgeOpsTrackerTracing(app); // alongside registerForgeOpsTrackerBreadcrumbContext
+ *
+ * transactionName reads request.routeOptions.url, Fastify's own matched route pattern (the
+ * property that replaced the older, now-removed request.routerPath), falling back to routerPath
+ * for an older Fastify version and finally to the literal request.url for a request that never
+ * matched a route at all (a 404), the same "pattern first, literal path as the 404 fallback"
+ * shape the Express integration above already takes.
+ *
+ * @param {import("fastify").FastifyInstance} fastify
+ */
+export function registerForgeOpsTrackerTracing(fastify) {
+  fastify.addHook("onRequest", (request, reply, done) => {
+    _runWithSpanTrace(() => {
+      request._forgeOpsSpanStartedAt = new Date();
+      request._forgeOpsSpanStart = performance.now();
+      done();
+    });
+  });
+
+  fastify.addHook("onResponse", async (request) => {
+    if (request._forgeOpsSpanStart === undefined) {
+      // trackTracing was off (or the client wasn't enabled) when onRequest ran above, so there's
+      // nothing to finish; _finishSpanTrace would no-op anyway (no buffer to read back), but
+      // there's no point computing a duration for nothing.
+      return;
+    }
+    const durationMs = performance.now() - request._forgeOpsSpanStart;
+    const transactionName = `${request.method} ${request.routeOptions?.url ?? request.routerPath ?? request.url}`;
+    _finishSpanTrace(transactionName, request._forgeOpsSpanStartedAt, durationMs);
+  });
+}

package/src/metricBuffer.js

@@ -0,0 +1,117 @@
+/**
+ * Collects individual captureMetric()/captureInfrastructureMetric() calls in-process and periodically
+ * flushes them as one batch, rather than one network call per capture. Unlike performanceFlusher.js
+ * this keeps a *list* of individually meaningful entries instead of summing them into buckets: a
+ * customer's own signup or payment is exactly the kind of thing they will want a genuinely accurate
+ * count/sum of later, so the server stores one row per entry as-is. Ported from
+ * gems/forge_ops_tracker/lib/forge_ops_tracker/metric_buffer.rb and infrastructure_metric_buffer.rb,
+ * which are the same class twice; here it is one class instantiated twice, told which delivery
+ * method and flush interval to use.
+ *
+ * Three deliberate differences from the Ruby buffers:
+ *
+ * - A flush snapshots the first N entries and, on success, removes exactly those N, instead of
+ *   resetting the whole list: an entry recorded while the request is in flight (this is async, so
+ *   that window is real) is kept for the next flush rather than lost.
+ * - The buffer is capped at MAX_ENTRIES, and once full further entries are dropped until a flush
+ *   succeeds: a plan without the feature answers 403 on every flush, and an uncapped buffer would
+ *   then grow for as long as the process lives. Dropping the newest rather than the oldest keeps
+ *   the entries a flush is delivering at the front of the list, which is what makes removing
+ *   exactly those afterward exact.
+ * - A NaN or infinite value is dropped at record time: JSON.stringify turns it into `null`, which
+ *   the server would reject, taking the whole batch with it.
+ *
+ * setInterval(...).unref(), not a real background thread, same reasoning sessionFlusher.js's own
+ * header comment documents. Unlike that flusher this also flushes on "beforeExit" (fired when the
+ * event loop has nothing left, and which does not change how the process exits the way a
+ * SIGTERM/SIGINT listener would): the real use of the infrastructure endpoint is a short-lived cron
+ * script that captures a few readings and lets the process end.
+ */
+export const MAX_ENTRIES = 1000;
+
+export class MetricBuffer {
+  #configuration;
+  #deliver;
+  #intervalMs;
+  #entries = [];
+  #timer = null;
+  #beforeExit = null;
+
+  /**
+   * @param {import("./configuration.js").Configuration} configuration
+   * @param {(entries: Record<string, unknown>[]) => Promise<boolean>} deliver
+   * @param {() => number} intervalMs read fresh each time the timer is created
+   */
+  constructor(configuration, deliver, intervalMs) {
+    this.#configuration = configuration;
+    this.#deliver = deliver;
+    this.#intervalMs = intervalMs;
+  }
+
+  /**
+   * @param {Record<string, unknown>} entry everything but recorded_at
+   * @returns {boolean} whether it was kept
+   */
+  record(entry) {
+    const value = entry.value;
+    if (typeof value !== "number" || !Number.isFinite(value)) {
+      this.#configuration.log("[forge-ops-tracker] dropped a metric with a non-numeric or non-finite value");
+      return false;
+    }
+    if (this.#entries.length >= MAX_ENTRIES) {
+      this.#configuration.log("[forge-ops-tracker] metric buffer full, dropping a metric");
+      return false;
+    }
+
+    this.#ensureStarted();
+    this.#entries.push({ ...entry, recorded_at: `${new Date().toISOString().slice(0, 19)}Z` });
+    return true;
+  }
+
+  /** Delivers everything buffered so far as one batch. A failed delivery keeps every entry, so the next flush's batch just grows. */
+  async flush() {
+    if (this.#entries.length === 0) {
+      return;
+    }
+
+    const snapshot = this.#entries.slice();
+    const delivered = await this.#deliver(snapshot);
+    if (!delivered) {
+      return;
+    }
+
+    // Exactly the entries just delivered: anything recorded while the request was in flight sits
+    // after them and stays for the next flush.
+    this.#entries.splice(0, snapshot.length);
+  }
+
+  /** Stops the timer and the exit hook without delivering anything. */
+  discard() {
+    if (this.#timer !== null) {
+      clearInterval(this.#timer);
+      this.#timer = null;
+    }
+    if (this.#beforeExit !== null) {
+      process.off("beforeExit", this.#beforeExit);
+      this.#beforeExit = null;
+    }
+    this.#entries = [];
+  }
+
+  #ensureStarted() {
+    if (this.#timer !== null) {
+      return;
+    }
+
+    this.#timer = setInterval(() => this.#flushSafely(), this.#intervalMs());
+    this.#timer.unref();
+    this.#beforeExit = () => this.#flushSafely();
+    process.on("beforeExit", this.#beforeExit);
+  }
+
+  #flushSafely() {
+    return this.flush().catch((e) => {
+      this.#configuration.log(`[forge-ops-tracker] metric flush error: ${e.name}: ${e.message}`);
+    });
+  }
+}

package/src/performanceFlusher.js

@@ -1,3 +1,5 @@
+import { bucketFor } from "./histogramBucketer.js";
+
 /**
  * Times requests in-process, bucketed by transactionName (see integrations/performance.js), and
  * periodically flushes each distinct bucket as one small aggregate report, rather than one
@@ -33,19 +35,30 @@ export class PerformanceFlusher {
   record(transactionName, durationMs) {
     this.#ensureTimerStarted();
 
-    const bucket = this.#buckets.get(transactionName) ?? { count: 0, durationSumMs: 0, maxDurationMs: 0 };
+    const bucket = this.#buckets.get(transactionName) ?? { count: 0, durationSumMs: 0, maxDurationMs: 0, histogram: {} };
     bucket.count += 1;
     bucket.durationSumMs += durationMs;
     if (durationMs > bucket.maxDurationMs) {
       bucket.maxDurationMs = durationMs;
     }
+    // The distribution count/sum/max can't reconstruct: see histogramBucketer.js for why the
+    // server approximates a percentile from these bucket counts.
+    const label = bucketFor(durationMs);
+    bucket.histogram[label] = (bucket.histogram[label] ?? 0) + 1;
     this.#buckets.set(transactionName, bucket);
   }
 
   /**
-   * Snapshots and resets the buckets, then delivers them as one batch. A failed delivery keeps
-   * every bucket where it is rather than resetting, so the next flush's batch just grows instead
-   * of losing what was already tallied; same reasoning sessionFlusher.js's own flush() documents.
+   * Snapshots the buckets, then delivers them as one batch. A failed delivery keeps every bucket
+   * where it is rather than resetting, so the next flush's batch just grows instead of losing what
+   * was already tallied; same reasoning sessionFlusher.js's own flush() documents.
+   *
+   * Only exactly what this snapshot delivered is removed afterward, subtracted from whatever is in
+   * each bucket by then, never the whole map reset: record() can run while the delivery is awaited,
+   * so a record for a transaction already in the snapshot, or a brand-new one, can land between the
+   * snapshot and delivery succeeding, and resetting afterward would silently discard it.
+   * maxDurationMs is left as whatever is currently on the bucket, sent or not: a max can't be
+   * "subtracted" back out, and leaving it never overstates the next period's own max.
    */
   async flush() {
     if (this.#buckets.size === 0) {
@@ -53,6 +66,12 @@ export class PerformanceFlusher {
     }
 
     const periodEndedAt = new Date();
+    const sent = new Map(
+      [...this.#buckets.entries()].map(([transactionName, bucket]) => [
+        transactionName,
+        { count: bucket.count, durationSumMs: bucket.durationSumMs, histogram: { ...bucket.histogram } },
+      ]),
+    );
     const samples = [...this.#buckets.entries()].map(([transactionName, bucket]) => ({
       transaction_name: transactionName,
       environment: this.#configuration.environment,
@@ -62,6 +81,7 @@ export class PerformanceFlusher {
       request_count: bucket.count,
       duration_sum_ms: bucket.durationSumMs,
       max_duration_ms: bucket.maxDurationMs,
+      histogram: { ...bucket.histogram },
     }));
 
     const delivered = await this.#client.deliverPerformanceSamples(samples);
@@ -69,7 +89,25 @@ export class PerformanceFlusher {
       return;
     }
 
-    this.#buckets = new Map();
+    for (const [transactionName, deliveredBucket] of sent) {
+      const current = this.#buckets.get(transactionName);
+      if (!current) {
+        continue;
+      }
+      current.count -= deliveredBucket.count;
+      current.durationSumMs = Math.max(current.durationSumMs - deliveredBucket.durationSumMs, 0);
+      for (const [label, count] of Object.entries(deliveredBucket.histogram)) {
+        const remaining = (current.histogram[label] ?? 0) - count;
+        if (remaining > 0) {
+          current.histogram[label] = remaining;
+        } else {
+          delete current.histogram[label];
+        }
+      }
+      if (current.count <= 0) {
+        this.#buckets.delete(transactionName);
+      }
+    }
     this.#periodStartedAt = periodEndedAt;
   }
 

package/src/reporter.js

@@ -20,14 +20,16 @@ export class Reporter {
   /**
    * @param {Error} error
    * @param {Record<string, unknown>} [context]
+   * @param {Record<string, unknown> | null} [user]
+   * @param {Array<Record<string, unknown>>} [breadcrumbs]
    */
-  report(error, context = {}) {
+  report(error, context = {}, user = null, breadcrumbs = []) {
     try {
       if (!this.#configuration.isEnabled()) {
         return;
       }
 
-      const payload = this.#eventBuilder.build(error, context);
+      const payload = this.#eventBuilder.build(error, context, user, breadcrumbs);
       this.#deliveryQueue.push(payload);
     } catch (e) {
       this.#configuration.log(`[forge-ops-tracker] report failed: ${e.name}: ${e.message}`);

package/src/spanBuffer.js

@@ -0,0 +1,89 @@
+import { randomBytes } from "node:crypto";
+
+/**
+ * A short, unique-enough identifier for one span: 8 random bytes as hex, matching
+ * gems/forge_ops_tracker/lib/forge_ops_tracker/span_buffer.rb's own SecureRandom.hex(8). The
+ * server only ever needs these to be unique within one trace's own array (see
+ * Api::V1::SpansController on the Rails side), never a real database id, so this is deliberately
+ * cheap rather than a full UUID.
+ * @returns {string}
+ */
+export function randomSpanId() {
+  return randomBytes(8).toString("hex");
+}
+
+/**
+ * Accumulates one request's own nested call tree in-process: the tracing analog to
+ * BreadcrumbBuffer's per-request trail. Ported from
+ * gems/forge_ops_tracker/lib/forge_ops_tracker/span_buffer.rb, with one deliberate departure: the
+ * Ruby buffer tracks "what's currently open" with a single mutable stack, which is correct for
+ * Ruby's own single-threaded-per-request execution model, but is the wrong tool here. A customer
+ * can easily await two service-layer span() calls concurrently in the same request (Promise.all
+ * rather than one after another); popping one call's own span off a shared stack while the other
+ * is still pending would leave the second one mis-parented under the first instead of under the
+ * request's own root. index.js's own span()/leaf-span recording instead tracks "the currently
+ * open span" with a second, nested AsyncLocalStorage (see spanParentStorage there): each
+ * concurrent branch gets its own correctly-scoped view automatically, the exact property that
+ * storage exists to provide. This class only ever holds what doesn't depend on that: the trace's
+ * own identifiers and the flat list every span, root or leaf, ends up recorded into.
+ */
+export class SpanBuffer {
+  #configuration;
+  traceId;
+  rootSpanId;
+  spans = [];
+  #rootDurationMs = null;
+
+  constructor(configuration) {
+    this.#configuration = configuration;
+    this.traceId = randomBytes(16).toString("hex");
+    this.rootSpanId = randomSpanId();
+  }
+
+  /**
+   * root: true only for the one call that records the request's own root span: forces
+   * parent_span_id null explicitly, the same reasoning span_buffer.rb's own #record documents for
+   * why that can't just be read off of "whatever's currently open" for the root specifically.
+   *
+   * @param {{
+   *   spanId: string,
+   *   parentSpanId?: string | null,
+   *   name: string,
+   *   kind: string,
+   *   startedAt: Date,
+   *   durationMs: number,
+   *   data?: Record<string, unknown>,
+   *   root?: boolean,
+   * }} span
+   */
+  record({ spanId, parentSpanId = null, name, kind, startedAt, durationMs, data = {}, root = false }) {
+    this.spans.push({
+      span_id: spanId,
+      parent_span_id: root ? null : parentSpanId,
+      name,
+      kind,
+      // Date#toISOString() already yields millisecond precision UTC ("...sssZ"), exactly the wire
+      // format the server expects; unlike BreadcrumbBuffer's own timestamp, this one must NOT
+      // strip milliseconds, since a waterfall's own ordering depends on them.
+      started_at: startedAt.toISOString(),
+      duration_ms: durationMs,
+      environment: this.#configuration.environment,
+      release: this.#configuration.release,
+      data,
+    });
+    if (root) {
+      this.#rootDurationMs = durationMs;
+    }
+  }
+
+  /**
+   * null (never sent) until the root span has actually been recorded: a request whose own
+   * tracing integration never got the chance to call back at all has no duration to compare
+   * against a threshold, so it's never mistakenly treated as slow.
+   * @param {number} thresholdMs
+   * @returns {boolean}
+   */
+  isSlow(thresholdMs) {
+    return this.#rootDurationMs !== null && this.#rootDurationMs >= thresholdMs;
+  }
+}