@forge-ops/tracker 0.5.0 → 0.9.0

package/src/index.js

@@ -1,11 +1,15 @@
 import { AsyncLocalStorage } from "node:async_hooks";
+import { BreadcrumbBuffer } from "./breadcrumbBuffer.js";
 import { Client } from "./client.js";
 import { Configuration } from "./configuration.js";
 import { DeliveryQueue } from "./deliveryQueue.js";
 import { EventBuilder } from "./eventBuilder.js";
+import { installHttpTracing, _uninstallHttpTracing } from "./httpTracing.js";
+import { MetricBuffer } from "./metricBuffer.js";
 import { PerformanceFlusher } from "./performanceFlusher.js";
 import { Reporter } from "./reporter.js";
 import { SessionFlusher } from "./sessionFlusher.js";
+import { randomSpanId, SpanBuffer } from "./spanBuffer.js";
 
 export { Configuration };
 
@@ -13,6 +17,9 @@ let configuration = null;
 let reporter = null;
 let sessionFlusher = null;
 let performanceFlusher = null;
+let spanQueue = null;
+let metricBuffer = null;
+let infrastructureMetricBuffer = null;
 let processHandlersInstalled = false;
 let uncaughtExceptionListener = null;
 let unhandledRejectionListener = null;
@@ -28,6 +35,48 @@ let unhandledRejectionListener = null;
 // wrapping function, not a bare setter.
 const userStorage = new AsyncLocalStorage();
 
+// Request-scoped breadcrumb trail storage. A second, independent AsyncLocalStorage instance
+// rather than folding this into userStorage above: distinct concern (an ordered trail, not a
+// single attached value), same "own storage, own wrapping function" shape
+// gems/forge_ops_tracker's own Thread.current[:forge_ops_tracker_breadcrumbs] (separate from
+// Thread.current[:forge_ops_tracker_current_user]) and Python's own separate _breadcrumbs
+// ContextVar (separate from its threading.local .user) both already take.
+//
+// Unlike Ruby's Thread.current and Python's contextvars.ContextVar, AsyncLocalStorage.run()
+// needs no explicit "clear it when the request ends" step (see BreadcrumbContext's own
+// ensure-clear, or _end_breadcrumb_trail's own reset()): once the callback passed to run()
+// finishes (synchronously or, for everything causally descended from it, asynchronously), the
+// store automatically stops being visible to anything outside that call chain, exactly the
+// property AsyncLocalStorage exists to provide. That's why _runWithBreadcrumbTrail below is a
+// single wrapping call, the same shape runWithUser already has, rather than a separate
+// start/end pair the way Ruby's middleware or Python's Django/Flask/Celery hooks need (those
+// frameworks fire two genuinely separate hooks, "request started" and "request ended," with no
+// single function this client could wrap around the whole thing; Express's next()-based
+// middleware chain and a real Fastify onRequest hook wrapping its own `done` callback both do
+// give this client that single continuous call chain to wrap, confirmed directly below, not
+// assumed).
+const breadcrumbStorage = new AsyncLocalStorage();
+
+// Request-scoped span-tracing storage: one SpanBuffer per request (see spanBuffer.js), the same
+// per-request AsyncLocalStorage scoping breadcrumbStorage already uses above and for the identical
+// reason (Node interleaves many concurrent requests' async callbacks on one thread, so anything
+// less than AsyncLocalStorage would leak one request's spans into another's).
+const spanStorage = new AsyncLocalStorage();
+
+// A second, nested AsyncLocalStorage tracks "the currently open span" (a span_id, not a whole
+// object) separately from the buffer itself: span() below and every leaf-span recorder read this
+// to find their own real parent, and span() enters a fresh nested context around its own callback
+// to make itself the parent for anything recorded further inside. This is deliberately NOT a
+// mutable stack on SpanBuffer the way gems/forge_ops_tracker's own Ruby buffer uses (correct for
+// Ruby's single-threaded-per-request execution, wrong here): two span() calls awaited
+// concurrently in the same request (Promise.all rather than one after another) each get their own,
+// correctly-scoped view of "what's currently open" for free, the exact property nested
+// AsyncLocalStorage.run() calls already provide, rather than one call's pop() corrupting the
+// other's still-pending parent. Defaults to the request's own root span id once _runWithSpanTrace
+// below establishes it; genuinely unset outside of that (span()/_recordSpan both treat "no
+// SpanBuffer at all" as the real signal for "not inside a traced request").
+const spanParentStorage = new AsyncLocalStorage();
+
 function getConfiguration() {
   if (configuration === null) {
     configuration = new Configuration();
@@ -61,6 +110,83 @@ function getPerformanceFlusher() {
   return performanceFlusher;
 }
 
+/** The two metric buffers, created together on first use: independent of each other (a script may only ever call one), but cheap enough that creating both is simpler than tracking which. */
+function getMetricBuffers() {
+  if (metricBuffer === null) {
+    const config = getConfiguration();
+    const client = new Client(config);
+    metricBuffer = new MetricBuffer(config, (entries) => client.deliverMetrics(entries), () => config.metricFlushIntervalMs);
+    infrastructureMetricBuffer = new MetricBuffer(
+      config,
+      (entries) => client.deliverInfrastructureMetrics(entries),
+      () => config.infrastructureMetricFlushIntervalMs,
+    );
+  }
+  return { metrics: metricBuffer, infrastructure: infrastructureMetricBuffer };
+}
+
+/**
+ * Records a named business metric (a signup, a payment, anything you want to name), buffered and
+ * flushed periodically as one batch rather than one network call per capture. `value` defaults to
+ * 1 so a bare counter-style call needs no argument; pass one for a real magnitude
+ * (`captureMetric("payment", 49)`); it may be negative (a refund). A no-op when the client isn't
+ * enabled (no DSN, or this environment isn't in `enabledEnvironments`), and a NaN or infinite value
+ * is dropped.
+ *
+ * @param {string} name
+ * @param {number} [value]
+ */
+export function captureMetric(name, value = 1) {
+  const config = getConfiguration();
+  if (!config.isEnabled()) {
+    return;
+  }
+  getMetricBuffers().metrics.record({ metric_name: name, value, environment: config.environment, release: config.release });
+}
+
+/**
+ * Records one infrastructure reading (CPU, memory, disk, anything else a script of yours reads)
+ * from one of your own hosts. `hostname` defaults to `Configuration#serverName`, so a script running
+ * on the box it reports about needs no argument. Same buffered-batch delivery and no-op-when-disabled
+ * contract as captureMetric. A short-lived cron script needs nothing more: the buffer also flushes
+ * when the event loop drains ("beforeExit"); call `await flushMetrics()` if it exits another way
+ * (`process.exit()`).
+ *
+ * @param {string} name
+ * @param {number} value
+ * @param {{ hostname?: string }} [options]
+ */
+export function captureInfrastructureMetric(name, value, options = {}) {
+  const config = getConfiguration();
+  if (!config.isEnabled()) {
+    return;
+  }
+  getMetricBuffers().infrastructure.record({ metric_name: name, value, hostname: options.hostname ?? config.serverName });
+}
+
+/** Delivers every buffered metric and infrastructure reading right now, instead of waiting for the next flush interval. */
+export async function flushMetrics() {
+  if (metricBuffer === null) {
+    return;
+  }
+  await Promise.all([metricBuffer.flush(), infrastructureMetricBuffer.flush()]);
+}
+
+/**
+ * A DeliveryQueue reused for spans (see that class's own comment for why it takes a
+ * deliverMethod/label rather than needing a whole second, near-identical class the way
+ * gems/forge_ops_tracker's own SpanQueue is): one whole trace is one queue entry, delivered
+ * promptly, never batched together with other traces the way PerformanceFlusher/the error
+ * DeliveryQueue's own default usage batch or accumulate over a time window.
+ */
+function getSpanQueue() {
+  if (spanQueue === null) {
+    const config = getConfiguration();
+    spanQueue = new DeliveryQueue(config, new Client(config), { deliverMethod: "deliverSpans", label: "trace" });
+  }
+  return spanQueue;
+}
+
 /**
  * Internal; called by the Express/Fastify session-tracking integrations, never by host app
  * code directly (there's nothing for a caller to decide here beyond what the middleware/hook
@@ -91,6 +217,125 @@ export function _recordPerformance(transactionName, durationMs) {
   getPerformanceFlusher().record(transactionName, durationMs);
 }
 
+/**
+ * Internal; called by the Express/Fastify breadcrumb-context integrations, never by host app
+ * code directly. Gives `callback`'s own async call chain a fresh, empty breadcrumb trail to
+ * accumulate into (via addBreadcrumb() below, or whatever automatic source records into it
+ * alongside its own timing; see integrations/performance.js), the same per-request scoping
+ * gems/forge_ops_tracker's BreadcrumbContext middleware and Python's _start_breadcrumb_trail
+ * both provide for their own concurrency model. A no-op wrapper (still calls callback, just
+ * with no buffer set up) when trackBreadcrumbs is off or the client isn't enabled at all, the
+ * same "skip entirely" gating _recordPerformance/_recordSession already apply to their own
+ * automatic sources; addBreadcrumb() below still works regardless of this, by lazily creating
+ * its own standalone buffer.
+ *
+ * @template T
+ * @param {() => T} callback
+ * @returns {T}
+ */
+export function _runWithBreadcrumbTrail(callback) {
+  const config = getConfiguration();
+  if (!config.trackBreadcrumbs || !config.isEnabled()) {
+    return callback();
+  }
+  return breadcrumbStorage.run(new BreadcrumbBuffer(config), callback);
+}
+
+/**
+ * Internal; called by the Express/Fastify tracing integration (integrations/tracing.js), never
+ * by host app code directly. Gives callback's own async call chain a fresh SpanBuffer, with its
+ * own root span id already established as "currently open" in spanParentStorage (see that
+ * storage's own comment above), so a leaf span or a manual span() call recorded before the root
+ * span itself is ever actually recorded still parents onto it correctly. A no-op wrapper (still
+ * calls callback, just with no buffer set up) when trackTracing is off or the client isn't
+ * enabled at all, the same "skip entirely" gating every other automatic source in this file
+ * already applies to itself.
+ *
+ * @template T
+ * @param {() => T} callback
+ * @returns {T}
+ */
+export function _runWithSpanTrace(callback) {
+  const config = getConfiguration();
+  if (!config.trackTracing || !config.isEnabled()) {
+    return callback();
+  }
+  const buffer = new SpanBuffer(config);
+  return spanStorage.run(buffer, () => spanParentStorage.run(buffer.rootSpanId, callback));
+}
+
+/**
+ * Internal; called by the Express/Fastify tracing integration once the request has actually
+ * finished and the root span's own real total duration is finally known (not any sooner: that
+ * duration is what the slow/fast decision below is actually made against). Records the request's
+ * own root "controller" span, then, only now and entirely client-side, decides whether the whole
+ * trace crossed configuration.traceCaptureThresholdMs and is worth sending at all: a normal, fast
+ * request's buffer is simply discarded here, unsent, the entire reason this feature never costs a
+ * fast request a single byte over the wire. Mirrors
+ * gems/forge_ops_tracker/lib/forge_ops_tracker/middleware/span_tracing.rb's own ensure block.
+ *
+ * @param {string} name
+ * @param {Date} startedAt
+ * @param {number} durationMs
+ */
+export function _finishSpanTrace(name, startedAt, durationMs) {
+  const buffer = spanStorage.getStore();
+  if (!buffer) {
+    return;
+  }
+  buffer.record({ spanId: buffer.rootSpanId, name, kind: "controller", startedAt, durationMs, root: true });
+  if (buffer.isSlow(getConfiguration().traceCaptureThresholdMs)) {
+    getSpanQueue().push({ trace_id: buffer.traceId, spans: buffer.spans });
+  }
+}
+
+/**
+ * Internal; called by whatever this client already automatically times end-to-end with no
+ * children of its own to nest anything under (currently just the outbound HTTP wrapper; see
+ * httpTracing.js), never by host app code directly. A no-op, the same as every other automatic
+ * source, when there's no request-level trace currently open at all (trackTracing off, or
+ * genuinely outside any request the tracing integration ever wrapped): never fabricates a trace
+ * with no request to belong to.
+ *
+ * @param {string} name
+ * @param {string} kind
+ * @param {Date} startedAt
+ * @param {number} durationMs
+ * @param {Record<string, unknown>} [data]
+ */
+export function _recordSpan(name, kind, startedAt, durationMs, data = {}) {
+  const config = getConfiguration();
+  const buffer = spanStorage.getStore();
+  if (!config.trackTracing || !buffer) {
+    return;
+  }
+  const parentSpanId = spanParentStorage.getStore() ?? buffer.rootSpanId;
+  buffer.record({ spanId: randomSpanId(), parentSpanId, name, kind, startedAt, durationMs, data });
+}
+
+/**
+ * Internal; called by whatever this client already automatically times (currently just the
+ * Express controller/route lifecycle; see integrations/performance.js), never by host app code
+ * directly, same reasoning as _recordPerformance above. Independent of trackPerformance: an app
+ * could want the trail without the timing data, or vice versa, and each call site already has
+ * to check its own flag regardless, so there's no real cost to keeping the two independent
+ * rather than tying breadcrumbs to whether performance monitoring happens to be on (mirrors
+ * gems/forge_ops_tracker/lib/forge_ops_tracker/railtie.rb's own reasoning for the same design
+ * choice).
+ *
+ * @param {string} message
+ * @param {string} category
+ * @param {string} [level]
+ * @param {Record<string, unknown>} [data]
+ */
+export function _recordBreadcrumb(message, category, level = "info", data = {}) {
+  const config = getConfiguration();
+  if (!config.trackBreadcrumbs || !config.isEnabled()) {
+    return;
+  }
+  addBreadcrumb(message, { category, level, data });
+}
+
 /**
  * Configure the client. Call once at startup.
  *
@@ -112,6 +357,13 @@ export function init(options = {}) {
     installProcessLevelHandlers();
   }
 
+  // Independent of installProcessHandlers above (that flag is only about the two process-level
+  // listeners): installed unconditionally, every init() call, the same way
+  // gems/forge_ops_tracker's own Railtie always applies Net::HTTP.prepend(Timing) regardless of
+  // config, checking configuration.trackTracing fresh on every actual outbound call instead (see
+  // _recordSpan above), never at install time.
+  installHttpTracing(_recordSpan);
+
   return config;
 }
 
@@ -125,7 +377,42 @@ export function init(options = {}) {
  * @param {Record<string, unknown> | null} [user]
  */
 export function captureException(error, context = {}, user = null) {
-  getReporter().report(error, context, user ?? userStorage.getStore()?.user ?? null);
+  const breadcrumbs = breadcrumbStorage.getStore()?.all() ?? [];
+  getReporter().report(error, context, user ?? userStorage.getStore()?.user ?? null, breadcrumbs);
+}
+
+/**
+ * Adds one breadcrumb to the current request's trail (see _runWithBreadcrumbTrail above for how
+ * a request gets its own buffer to start with), or, outside a request entirely (a plain script,
+ * a worker with no breadcrumb-context wrapping installed at all), this call's own standalone
+ * trail for the rest of the process's root async context. Lazily creates a buffer the first
+ * time it's needed on whatever AsyncLocalStorage context is currently active, the same "works
+ * standalone, no specific setup required" shape gems/forge_ops_tracker's own
+ * ForgeOpsTracker.add_breadcrumb and Python's forge_ops_tracker.add_breadcrumb both already have.
+ *
+ * Uses breadcrumbStorage.enterWith() rather than .run() for that lazy-creation path
+ * specifically: unlike _runWithBreadcrumbTrail above, there's no callback here to wrap (this
+ * function returns void, called from arbitrary places in host app code), so .run() has nothing
+ * to scope the new buffer to. enterWith() sets the store for the remainder of whatever async
+ * chain is currently executing, which is exactly the "persists for the rest of this thread"
+ * behavior Ruby's Thread.current[:x] ||= ... and Python's ContextVar.set() both give their own
+ * manual API for free; the one deliberate difference is test isolation, see
+ * _resetForTesting's own comment below for why that needs an explicit disable().
+ *
+ * Works whether or not trackBreadcrumbs is on: that flag only gates the automatic sources
+ * _recordBreadcrumb above records, never this manual call, matching
+ * gems/forge_ops_tracker's own ForgeOpsTracker.add_breadcrumb.
+ *
+ * @param {string} message
+ * @param {{ category?: string, level?: string, data?: Record<string, unknown> }} [options]
+ */
+export function addBreadcrumb(message, options = {}) {
+  let buffer = breadcrumbStorage.getStore();
+  if (buffer === undefined) {
+    buffer = new BreadcrumbBuffer(getConfiguration());
+    breadcrumbStorage.enterWith(buffer);
+  }
+  buffer.add(message, options);
 }
 
 /**
@@ -149,6 +436,68 @@ export function runWithUser(user, callback) {
   return userStorage.run({ user }, callback);
 }
 
+/**
+ * Wraps a block of your own service-layer code as a manual span, e.g.
+ *
+ *     await forgeOpsTracker.span("PaymentService.charge", async () => { ... });
+ *
+ * There's no automatic way to detect "this is a logically distinct service layer" the way a
+ * database query or an outbound HTTP call already has a real instrumentation hook to extend, the
+ * same gap gems/forge_ops_tracker's own ForgeOpsTracker.span exists to fill. Works with a sync or
+ * async callback transparently, the same way runWithUser() above does: callback's own return
+ * value (or settled promise) is what this returns, with duration measured either immediately for
+ * a sync callback or once its promise settles for an async one, never forcing a synchronous
+ * caller through an extra microtask the way an always-`async function span` would.
+ *
+ * Outside any request-level trace (trackTracing off, or genuinely outside a request the
+ * Express/Fastify tracing integration ever wrapped), just runs callback and records nothing:
+ * unlike addBreadcrumb, this does NOT lazily create its own standalone buffer, matching
+ * gems/forge_ops_tracker's own ForgeOpsTracker.span exactly, since a lone span with no
+ * request-level trace around it, and no middleware left running to ever flush it, would just
+ * accumulate in memory forever with nothing to ever send it, worse than not recording it at all.
+ *
+ * @template T
+ * @param {string} name
+ * @param {() => T} callback
+ * @param {{ kind?: string, data?: Record<string, unknown> }} [options]
+ * @returns {T}
+ */
+export function span(name, callback, { kind = "service", data = {} } = {}) {
+  const buffer = spanStorage.getStore();
+  if (!buffer) {
+    return callback();
+  }
+
+  const spanId = randomSpanId();
+  const parentSpanId = spanParentStorage.getStore() ?? buffer.rootSpanId;
+  const startedAt = new Date();
+  const start = performance.now();
+  const finish = () => {
+    buffer.record({ spanId, parentSpanId, name, kind, startedAt, durationMs: performance.now() - start, data });
+  };
+
+  try {
+    const result = spanParentStorage.run(spanId, callback);
+    if (result && typeof result.then === "function") {
+      return result.then(
+        (value) => {
+          finish();
+          return value;
+        },
+        (error) => {
+          finish();
+          throw error;
+        },
+      );
+    }
+    finish();
+    return result;
+  } catch (error) {
+    finish();
+    throw error;
+  }
+}
+
 /**
  * Reports anything that would otherwise crash the process outright (a
  * plain script, an unhandled promise rejection) with no further wiring:
@@ -199,7 +548,28 @@ export function _resetForTesting() {
   reporter = null;
   sessionFlusher = null;
   performanceFlusher = null;
+  spanQueue = null;
+  metricBuffer?.discard();
+  infrastructureMetricBuffer?.discard();
+  metricBuffer = null;
+  infrastructureMetricBuffer = null;
   processHandlersInstalled = false;
   uncaughtExceptionListener = null;
   unhandledRejectionListener = null;
+  // disable(), not just leaving it alone: a test that calls addBreadcrumb() standalone (outside
+  // any _runWithBreadcrumbTrail wrapping, exercising the lazy-creation path on purpose) sets the
+  // store via enterWith() on whatever async context Node's test runner happens to be executing
+  // tests from, which, unlike a scoped run() call, has no natural point where it unwinds on its
+  // own. Without this, that leftover store could otherwise bleed into a later, unrelated test
+  // sharing the same root context. disable() drops it and leaves the instance perfectly usable
+  // again for the next run()/enterWith() call, confirmed directly, not assumed.
+  breadcrumbStorage.disable();
+  // Re-installed fresh on the very next init() call in whatever test runs next: without this,
+  // http.request/https.request would stay wrapped by a closure over a *previous* test's own
+  // _recordSpan reference forever (installHttpTracing's own guard only ever installs once per
+  // process otherwise), which happens to still work correctly here since _recordSpan itself reads
+  // module state fresh on every call rather than closing over anything test-specific, but relying
+  // on that instead of actually restoring the original functions would be fragile, not a genuine
+  // guarantee.
+  _uninstallHttpTracing();
 }

package/src/integrations/breadcrumbContext.js

@@ -0,0 +1,70 @@
+import { _runWithBreadcrumbTrail } from "../index.js";
+
+/**
+ * Gives every request a fresh, empty breadcrumb trail to accumulate into (via
+ * addBreadcrumb(), or whatever automatic source records into it alongside its own timing; see
+ * integrations/performance.js), so one request's trail never bleeds into another's, the same
+ * concern gems/forge_ops_tracker's own Middleware::BreadcrumbContext and Python's
+ * _start_breadcrumb_trail/_end_breadcrumb_trail both solve for their own concurrency model. A
+ * sibling to the session-tracking/user-context/performance integrations below, not folded into
+ * any of them: distinct concern, same "own file, own registration point" shape every integration
+ * in this client already takes.
+ *
+ * Register FIRST, before forgeOpsTrackerUserContextMiddleware and
+ * forgeOpsTrackerPerformanceExpressMiddleware (and, for the same reason, before
+ * forgeOpsTrackerSessionTrackingExpressMiddleware too): every one of those can record a
+ * breadcrumb of its own or run host app code that calls addBreadcrumb(), so the trail has to
+ * already exist by the time any of them runs, exactly why gems/forge_ops_tracker's own
+ * initializer ordering comment documents the identical requirement for its Rack middleware.
+ *
+ *   app.use(forgeOpsTrackerBreadcrumbContextExpressMiddleware); // first, before everything else
+ *   app.use(forgeOpsTrackerSessionTrackingExpressMiddleware);
+ *   app.use(forgeOpsTrackerPerformanceExpressMiddleware);
+ *   app.use(forgeOpsTrackerUserContextMiddleware);
+ *   // ...routes...
+ *   app.use(forgeOpsTrackerExpressMiddleware); // still last, after all routes
+ *
+ * Wraps `next` in _runWithBreadcrumbTrail (see that function's own doc comment in index.js for
+ * why this needs no explicit "clear" step the way Ruby's ensure or Python's finally do): confirmed
+ * directly, with a real Express app and a real async route handler, that a breadcrumb recorded
+ * from deep inside an awaited route handler (or from the performance middleware's own
+ * res.on("finish") listener, which fires later still) is visible to
+ * forgeOpsTrackerExpressMiddleware's own captureException() call at the end of the same request,
+ * the same "AsyncLocalStorage survives all the way through, not just synchronously" property
+ * forgeOpsTrackerUserContextMiddleware's own comment already confirms for the user-context case.
+ */
+export function forgeOpsTrackerBreadcrumbContextExpressMiddleware(req, res, next) {
+  _runWithBreadcrumbTrail(next);
+}
+
+/**
+ * Fastify equivalent, added as an onRequest hook (Fastify's earliest request-lifecycle hook,
+ * firing before routing and before any other hook this client registers) rather than exported as
+ * a plain function to call directly, mirroring registerForgeOpsTracker in integrations/fastify.js:
+ *
+ *   registerForgeOpsTrackerBreadcrumbContext(app); // first, before registerForgeOpsTracker
+ *
+ * A real design question Ruby's/Python's implementations don't answer for Node, since Fastify has
+ * no single function this client could wrap the way Express's next() or Ruby's app.call(env) both
+ * let a middleware wrap "the rest of this request" in one call: Fastify's request lifecycle is a
+ * sequence of independently-invoked hooks instead. The fix is _runWithBreadcrumbTrail(done): an
+ * onRequest hook declared with the callback-style (request, reply, done) signature, wrapping done
+ * itself (Fastify's own continuation, which advances to the next hook/handler) in
+ * AsyncLocalStorage.run(). Node's async_hooks propagate context by causality, not by lexical
+ * nesting, so the context established here is visible to everything Fastify goes on to invoke as
+ * a consequence of calling done(): the route handler, and (should it throw) the onError hook this
+ * client's own integrations/fastify.js registers. Confirmed directly against a real Fastify app,
+ * a real async route handler, and two genuinely concurrent, interleaved requests (one slower than
+ * the other, so their event-loop turns actually interleave, not just run back-to-back): each
+ * request's own trail stayed correctly isolated from the other's, and each one's onError hook saw
+ * only its own request's breadcrumbs, never the other's, exactly the property AsyncLocalStorage
+ * exists to provide and exactly what needed confirming for a hook-based framework rather than
+ * assuming it carries over unchanged from the Express case above.
+ *
+ * @param {import("fastify").FastifyInstance} fastify
+ */
+export function registerForgeOpsTrackerBreadcrumbContext(fastify) {
+  fastify.addHook("onRequest", (request, reply, done) => {
+    _runWithBreadcrumbTrail(done);
+  });
+}

package/src/integrations/performance.js

@@ -1,10 +1,11 @@
-import { _recordPerformance } from "../index.js";
+import { _recordBreadcrumb, _recordPerformance } from "../index.js";
 
 /**
  * Express performance-tracking middleware. Register wherever is convenient relative to routes
  * (unlike sessionTracking.js's own middleware, this doesn't need to run first: it only reads
  * req.route once the request has already been routed, inside the "finish" listener below, not
- * at request-start):
+ * at request-start), but after forgeOpsTrackerBreadcrumbContextExpressMiddleware (see
+ * integrations/breadcrumbContext.js), so the breadcrumb recorded below has a trail to land in:
  *
  *   app.use(forgeOpsTrackerPerformanceExpressMiddleware);
  *
@@ -17,9 +18,30 @@ import { _recordPerformance } from "../index.js";
  * transactionName is "<HTTP method> <route pattern>", e.g. "GET /users/:id", not the raw URL:
  * req.route.path is Express's own matched route pattern, read here (after routing has actually
  * happened, unlike at request-start where it isn't populated yet), which keeps a distinct user
- * id from exploding into its own separate transaction the way the literal URL would -- the
+ * id from exploding into its own separate transaction the way the literal URL would: the
  * low-cardinality equivalent of a Rails controller#action. Falls back to req.path (the literal,
  * unmatched path) for a request that never matched a route at all (a 404).
+ *
+ * The same res.on("finish") listener also records a "controller" breadcrumb, gated on
+ * trackBreadcrumbs independently of trackPerformance (each call below checks its own flag,
+ * mirroring gems/forge_ops_tracker/lib/forge_ops_tracker/railtie.rb's own
+ * process_action.action_controller subscription, which records both a performance sample and a
+ * breadcrumb from the one event): an app could want the trail without the timing data, or vice
+ * versa. This is currently the only place this client automatically records a breadcrumb at all:
+ * unlike Rails' framework-wide ActiveSupport::Notifications, this client has no automatic
+ * instrumentation source that isn't also this same opt-in middleware, so an app that wants
+ * automatic breadcrumbs (not just manual ones from addBreadcrumb()) needs this middleware
+ * installed regardless of whether it also wants the performance data. There's no query-level
+ * automatic breadcrumb source for the same reason there's no query-level performance
+ * instrumentation in this client at all today (no ORM/DB driver integration exists yet to hang
+ * one off of); this doesn't invent one.
+ *
+ * Deliberately recorded with the request's final status already known (from the same
+ * res.on("finish") firing this middleware already needed for timing), the same "recorded once
+ * the response actually finished" shape as Ruby's/Python's own automatic controller breadcrumb;
+ * see those implementations' own documented gap (a breadcrumb recorded this late can never show
+ * up on that exact same request's own error, only a later one on the same connection/process
+ * could see it) for why that's an accepted trade-off, not an oversight here either.
  */
 export function forgeOpsTrackerPerformanceExpressMiddleware(req, res, next) {
   const start = performance.now();
@@ -28,6 +50,10 @@ export function forgeOpsTrackerPerformanceExpressMiddleware(req, res, next) {
     const durationMs = performance.now() - start;
     const transactionName = `${req.method} ${req.route?.path ?? req.path}`;
     _recordPerformance(transactionName, durationMs);
+    _recordBreadcrumb(transactionName, "controller", res.statusCode >= 500 ? "error" : "info", {
+      status: res.statusCode,
+      duration_ms: Math.round(durationMs * 10) / 10,
+    });
   });
 
   next();

package/src/integrations/tracing.js

@@ -0,0 +1,80 @@
+import { _finishSpanTrace, _runWithSpanTrace } from "../index.js";
+
+/**
+ * Express distributed-tracing middleware: wraps the rest of the request in _runWithSpanTrace
+ * (see that function's own comment in index.js), the same "own file, own registration point"
+ * shape breadcrumbContext.js already established for a genuinely distinct concern (this is
+ * separate from, and independent of, error capture, session tracking, and aggregate performance
+ * timing). Register wherever is convenient relative to routes, the same as
+ * forgeOpsTrackerPerformanceExpressMiddleware, since it only reads req.route once the request has
+ * already been routed, inside the "finish" listener below, not at request-start:
+ *
+ *   app.use(forgeOpsTrackerTracingExpressMiddleware);
+ *
+ * transactionName is built exactly like forgeOpsTrackerPerformanceExpressMiddleware's own
+ * ("<HTTP method> <route pattern>", never the raw URL): this is the same request a performance
+ * sample would already be recorded for, so a trace's own root span name has to match, not invent
+ * a second naming scheme for the same thing.
+ *
+ * The actual send-or-don't decision happens inside _finishSpanTrace, only once the response has
+ * actually finished and the root span's real total duration is known: nothing about a normal,
+ * fast request costs a single byte over the wire, matching gems/forge_ops_tracker's own
+ * Middleware::SpanTracing.
+ */
+export function forgeOpsTrackerTracingExpressMiddleware(req, res, next) {
+  _runWithSpanTrace(() => {
+    const startedAt = new Date();
+    const start = performance.now();
+
+    res.on("finish", () => {
+      const durationMs = performance.now() - start;
+      const transactionName = `${req.method} ${req.route?.path ?? req.path}`;
+      _finishSpanTrace(transactionName, startedAt, durationMs);
+    });
+
+    next();
+  });
+}
+
+/**
+ * Fastify equivalent, added as two separate hooks rather than one function wrapping "the rest of
+ * this request" the way Express's next() allows (see
+ * registerForgeOpsTrackerBreadcrumbContext's own comment in breadcrumbContext.js for why Fastify
+ * needs this shape at all): onRequest establishes the span trace (wrapping `done`, the same
+ * pattern that file already uses) and stashes this request's own start time on `request` itself,
+ * the same "flag stashed on the request object" convention integrations/express.js's own
+ * req._forgeOpsSessionCrashed already established; onResponse, Fastify's own dedicated
+ * "the response has actually been sent" hook, reads that back once the real total duration is
+ * known:
+ *
+ *   registerForgeOpsTrackerTracing(app); // alongside registerForgeOpsTrackerBreadcrumbContext
+ *
+ * transactionName reads request.routeOptions.url, Fastify's own matched route pattern (the
+ * property that replaced the older, now-removed request.routerPath), falling back to routerPath
+ * for an older Fastify version and finally to the literal request.url for a request that never
+ * matched a route at all (a 404), the same "pattern first, literal path as the 404 fallback"
+ * shape the Express integration above already takes.
+ *
+ * @param {import("fastify").FastifyInstance} fastify
+ */
+export function registerForgeOpsTrackerTracing(fastify) {
+  fastify.addHook("onRequest", (request, reply, done) => {
+    _runWithSpanTrace(() => {
+      request._forgeOpsSpanStartedAt = new Date();
+      request._forgeOpsSpanStart = performance.now();
+      done();
+    });
+  });
+
+  fastify.addHook("onResponse", async (request) => {
+    if (request._forgeOpsSpanStart === undefined) {
+      // trackTracing was off (or the client wasn't enabled) when onRequest ran above, so there's
+      // nothing to finish; _finishSpanTrace would no-op anyway (no buffer to read back), but
+      // there's no point computing a duration for nothing.
+      return;
+    }
+    const durationMs = performance.now() - request._forgeOpsSpanStart;
+    const transactionName = `${request.method} ${request.routeOptions?.url ?? request.routerPath ?? request.url}`;
+    _finishSpanTrace(transactionName, request._forgeOpsSpanStartedAt, durationMs);
+  });
+}