@fougere/observability 0.9.2-alpha.0 → 0.9.3-alpha.0

package/src/index.ts

@@ -6,8 +6,24 @@ import { parseTraceparent, traceparentOf, randomHex, type SpanContext } from './
 /** A step while it runs. */
 interface Running extends SpanContext {
   frond: string | undefined;
+  /**
+   * When this step began, on both clocks — carried so what runs UNDER it is dated in ITS
+   * frame rather than on a clock of its own. `Date.now()` resolves to the millisecond and a
+   * whole call can take a fraction of one, so a statement dating itself lands anywhere:
+   * measured on SigNoz 2026-09-13, six queries drawn 17 µs to the LEFT of the call that made
+   * them, then all six stacked on its first instant. Offset from the monotonic clock, they
+   * fall where they ran.
+   */
+  startedAt: number;
+  start: number;
 }
 
+/**
+ * What a span is a span OF. An operation was dispatched and has an address; a statement ran
+ * underneath one and has a table. A reader that counts operations has to say which it takes.
+ */
+export type SpanKind = 'operation' | 'statement';
+
 /** A step that has finished, and what it did. */
 export interface FinishedSpan extends SpanContext {
   parentId: string | undefined;
@@ -18,17 +34,80 @@ export interface FinishedSpan extends SpanContext {
   callerFrond: string | undefined;
   /** Which frond owned the op — the deployment unit, so the first thing a reader groups by. */
   frond: string | undefined;
+  kind: SpanKind;
   entity: string;
   operation: string;
   /** When it started, in epoch milliseconds — an INSTANT, not an offset. */
   startedAt: number;
   ms: number;
+  /**
+   * `ms` minus what its OBSERVED children account for — the time this step spent itself. An op
+   * that delegates everything reports a duration and almost no self, which is how a report
+   * names the step that is actually slow instead of the one that waited for it.
+   *
+   * Never below zero, and that is not a guard: children running TOGETHER — the four queries
+   * behind one `Promise.all` — add up to more than the step that holds them. What is
+   * measured is the time nothing else accounts for, and there is no less of it than none.
+   *
+   * A child that finishes AFTER its parent is not deducted either: nothing was observed when
+   * the subtraction ran. The figure stays true, it is only cleaner than it should be.
+   */
+  selfMs: number;
+  /** How many statements ran under this step — one per row is the shape of an N+1. */
+  statements: number;
   /** The FougereError code when it refused, absent when it answered. */
   error: string | undefined;
 }
 
 export type SpanSink = (span: FinishedSpan) => void;
 
+/** A statement as its producer describes it — never the values it was given. */
+export interface Statement {
+  /** What it read or wrote: the table, or the source when the statement names none. */
+  subject: string;
+  /** `select`, `insert`, `update`, `delete`. */
+  verb: string;
+  ms: number;
+  failed: boolean;
+}
+
+/** What an app's tracer answers for. */
+export interface Tracing {
+  /** One span per operation — handed to `app.use`. */
+  middleware: AppMiddleware;
+  /** A statement that ran under the operation in flight: its time charged upward, and its own span when asked. */
+  statement(ran: Statement): void;
+}
+
+export interface TracingOptions {
+  /**
+   * A span of its OWN for every statement, and not only the time charged to the operation
+   * above it. Off by default, and the default is not prudence: `selfMs` and `statements` are
+   * counted either way, so what this adds is WHICH queries ran — a diagnosis, against a
+   * volume that multiplies by however many rows the page held. A backend charges per span.
+   *
+   * The pair splits the way profiling splits from monitoring: the count DETECTS, and it runs
+   * always; the detail EXPLAINS, and it is turned on over the operation the count named.
+   */
+  spanPerStatement?: boolean;
+  /**
+   * What share of the operations that stay in this process are traced, `0` to `1`.
+   *
+   * A rate is a BUDGET — how much a backend is worth per day — and no reading of the code
+   * answers it, so it is the operator's to set. What the code does answer is which spans the
+   * budget may not touch: an operation that crosses a process produces the one signal nothing
+   * else carries, since the difference between the caller's span and the callee's IS the wire
+   * cost, and neither process can measure it alone. So a rate below one thins what is already
+   * described by its own histogram, and never what only a trace can show.
+   *
+   * Default `1`, which is what a process did before this existed. The gain is not the default:
+   * it is that turning the rate down no longer drops the traces that matter most.
+   */
+  sample?: number;
+  /** How far an operation's work goes, by `entity.op` — the boot reads it off the model. */
+  hopsOf?: (entity: string, operation: string) => number;
+}
+
 /** The step running here and now. */
 
 /** The step running right now, if any. */
@@ -68,15 +147,33 @@ export function registerFlush(send: () => Promise<void>): () => void {
 }
 
 /**
- * The middleware, and it is HANDED who takes the spans it finishes.
+ * The tracer of ONE app, and it is HANDED who takes the spans it finishes.
  *
  * The list belongs to the app the middleware was installed on, never to the process: a
  * discarded app went on feeding the takers of the app that replaced it, so every metric
  * counted twice. A module-level array made that a leak you had to remember to undo; a list
  * held beside the app cannot outlive it. Read at every end, so a taker added later counts.
+ *
+ * Two doors and not one, because a statement is not dispatched: nothing calls a middleware
+ * around a query. What both need is the same table — the time a step's children accounted
+ * for — so the two sit in one closure rather than reaching for each other.
+ *
+ * Documented: [observability](https://fougere.dev/docs/infra/observability).
  */
-export function trace(takers: readonly SpanSink[]): AppMiddleware {
-  return (ctx, next) => {
+export function tracing(takers: readonly SpanSink[], options: TracingOptions = {}): Tracing {
+  /** What a step's children have accounted for, by the step they ran under. */
+  const charged = new Map<string, { ms: number; statements: number }>();
+  const rate = options.sample ?? 1;
+  const hopsOf = options.hopsOf ?? (() => 0);
+
+  /**
+   * Whether a root span is kept. An operation that leaves the process always is — the budget is
+   * for the ones a histogram already describes.
+   */
+  const keeps = (entity: string, operation: string) =>
+    rate >= 1 || hopsOf(entity, operation) > 0 || Math.random() < rate;
+
+  const middleware: AppMiddleware = (ctx, next) => {
     if (takers.length === 0) return next();
     // An op that CARRIES a line is not a call this process made: counting it puts the
     // delivery of a log line in the saturation figure, and spanning it puts a line about
@@ -88,11 +185,20 @@ export function trace(takers: readonly SpanSink[]): AppMiddleware {
     const inherited = parseTraceparent(ctx.invocation?.trace);
     const ambient = traceContext.current<Running>();
     const parent = inherited ?? ambient;
+    // Both, and they are not the same measurement: the wall clock says WHEN so two
+    // processes land on one timeline, the monotonic one says HOW LONG without being
+    // moved by an NTP correction mid-call.
+    const startedAt = Date.now();
+    const start = performance.now();
     const span: Running = {
       traceId: parent?.traceId ?? randomHex(16),
       spanId: randomHex(8),
-      sampled: parent?.sampled ?? true,
+      // A trace is whole or it is nothing: a call under a sampled parent is sampled, whatever
+      // this process would have decided on its own. Only a root is decided here.
+      sampled: parent?.sampled ?? keeps(ctx.entity, ctx.operation),
       frond: ctx.frond,
+      startedAt,
+      start,
     };
     // An edge exists only when the parent is IN this process and belongs to another frond.
     // `inherited` won means the parent is across a wire, and it did not name its frond.
@@ -105,23 +211,26 @@ export function trace(takers: readonly SpanSink[]): AppMiddleware {
     // carries the invocation.
     if (ctx.invocation) ctx.invocation = { ...ctx.invocation, trace: traceparentOf(span) };
 
-    // Both, and they are not the same measurement: the wall clock says WHEN so two
-    // processes land on one timeline, the monotonic one says HOW LONG without being
-    // moved by an NTP correction mid-call.
-    const startedAt = Date.now();
-    const start = performance.now();
     active += 1;
     const finish = (error: string | undefined) => {
       active -= 1;
+      const ms = performance.now() - start;
+      // Read and dropped in one gesture: a step ends once, and what its children owed it
+      // is of no use to anyone after that.
+      const owed = charged.get(span.spanId);
+      charged.delete(span.spanId);
       const done: FinishedSpan = {
         ...span,
         parentId: parent?.spanId,
         callerFrond,
         frond: ctx.frond,
+        kind: 'operation',
         entity: ctx.entity,
         operation: ctx.operation,
         startedAt,
-        ms: performance.now() - start,
+        ms,
+        selfMs: Math.max(0, ms - (owed?.ms ?? 0)),
+        statements: owed?.statements ?? 0,
         error,
       };
       // A taker that throws is a broken exporter, never a broken call.
@@ -139,6 +248,70 @@ export function trace(takers: readonly SpanSink[]): AppMiddleware {
       }
     });
   };
+
+  return {
+    middleware,
+
+    statement(ran: Statement): void {
+      if (takers.length === 0) return;
+      // A statement with no operation above it — a migration, a seed — belongs to no step,
+      // and inventing a trace for it would put a root span in the viewer per row planted.
+      const under = traceContext.current<Running>();
+      if (!under) return;
+
+      const owed = charged.get(under.spanId) ?? { ms: 0, statements: 0 };
+      owed.ms += ran.ms;
+      owed.statements += 1;
+      charged.set(under.spanId, owed);
+
+      // The op above now reports what it waited for and how many times, which is the whole
+      // of what a dashboard reads. WHICH queries ran leaves only when someone asks.
+      if (!options.spanPerStatement) return;
+
+      const done: FinishedSpan = {
+        traceId: under.traceId,
+        spanId: randomHex(8),
+        sampled: under.sampled,
+        parentId: under.spanId,
+        callerFrond: undefined,
+        frond: under.frond,
+        kind: 'statement',
+        entity: ran.subject,
+        operation: ran.verb,
+        startedAt: under.startedAt + Math.max(0, performance.now() - under.start - ran.ms),
+        ms: ran.ms,
+        selfMs: ran.ms,
+        statements: 0,
+        error: ran.failed ? 'STATEMENT_FAILED' : undefined,
+      };
+      for (const take of takers) { try { take(done); } catch { /* observing never refuses */ } }
+    },
+  };
+}
+
+/**
+ * The join this package exists to make: a statement runs SYNCHRONOUSLY inside the async
+ * context `tracing()` opened around the operation, so the sink finds the step it belongs to
+ * without the app ever holding its sources — which is what made the two ends of `onQuery`
+ * unable to see each other.
+ *
+ * `@fougere/adapter-sql` is optional, and dynamic for the reason `@fougere/calls` states:
+ * this package declares no dependency on it. An app on another storage observes no
+ * statement and every operation still reports its own time.
+ */
+export async function statementsUnder(tracer: Tracing): Promise<() => void> {
+  try {
+    const { onQuery } = await import('@fougere/adapter-sql');
+
+    return onQuery((event) => tracer.statement({
+      subject: event.subject,
+      verb: event.verb,
+      ms: event.ms,
+      failed: event.failed,
+    }));
+  } catch {
+    return () => {};
+  }
 }
 
 function codeOf(err: unknown): string {

package/src/metrics.ts

@@ -1,12 +1,21 @@
 /** The four signals every service is judged on, derived from the span that already exists. */
-import type { App, Edge, FrondPlacement, TopologyReport } from '@fougere/core';
+import { declaredTopologyOf, type App, type Edge, type FrondPlacement, type TopologyReport } from '@fougere/core';
 
 export type { Edge, FrondPlacement, TopologyReport } from '@fougere/core';
 import { activeCalls, type FinishedSpan, type SpanSink } from './index.js';
 
 /**
- * Bucket bounds in SECONDS, the OpenTelemetry recommendation for request durations.
- * Dense under 100 ms because that is where a healthy op lives, and open above 10 s.
+ * Bucket bounds in SECONDS, and the operator's to choose — this is the default, not the rule.
+ *
+ * The OpenTelemetry recommendation for request durations: dense under 100 ms because that is
+ * where a healthy op lives, and open above 10 s. A system whose ops all cross two processes has
+ * its floor elsewhere and wastes half these buckets.
+ *
+ * Deliberately NOT derived per operation, though `reach` could: an explicit-bucket histogram only
+ * aggregates across series that share its bounds, so a panel asking for the p95 of the whole
+ * service — which sums over every op — would stop meaning anything, silently. One list per
+ * process is what keeps that reading honest. The number that IS per-op is the load threshold,
+ * where nothing aggregates.
  */
 const BOUNDS = [0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10];
 
@@ -19,6 +28,11 @@ interface Bucketed {
   sum: number;
   /** One more than the bounds: the last holds everything above the highest bound. */
   buckets: number[];
+  /** The same measurement over `selfMs` — what the op did rather than what it waited for. */
+  selfSum: number;
+  selfBuckets: number[];
+  /** Every statement these calls ran. Against `count`, it is statements per call. */
+  statements: number;
 }
 
 export interface Metrics {
@@ -45,7 +59,7 @@ export interface MetricsSnapshot {
  * `app` is optional and only feeds the topology: what fronds this process found, and which of them
  * run elsewhere.
  */
-export function metrics(app?: App): Metrics {
+export function metrics(app?: App, bounds: readonly number[] = BOUNDS): Metrics {
   const since = Date.now();
   const series = new Map<string, Bucketed>();
   /** Every frond this process has actually CALLED — the other half of the topology. */
@@ -55,6 +69,11 @@ export function metrics(app?: App): Metrics {
 
   return {
     sink: (span: FinishedSpan) => {
+      // The one reader that has to choose: a statement is a step, not a call this process
+      // answered. Counted here it would publish `select.post` as an operation nothing
+      // serves, and every topology and saturation figure would count it too.
+      if (span.kind !== 'operation') return;
+
       const key = `${span.entity}\0${span.operation}\0${span.error ?? ''}`;
       let row = series.get(key);
       if (!row) {
@@ -65,7 +84,10 @@ export function metrics(app?: App): Metrics {
           error: span.error,
           count: 0,
           sum: 0,
-          buckets: new Array(BOUNDS.length + 1).fill(0),
+          buckets: new Array(bounds.length + 1).fill(0),
+          selfSum: 0,
+          selfBuckets: new Array(bounds.length + 1).fill(0),
+          statements: 0,
         };
         series.set(key, row);
       }
@@ -80,13 +102,18 @@ export function metrics(app?: App): Metrics {
       const seconds = span.ms / 1000;
       row.count += 1;
       row.sum += seconds;
-      row.buckets[bucketOf(seconds)] += 1;
+      row.buckets[bucketOf(seconds, bounds)] += 1;
+
+      const self = span.selfMs / 1000;
+      row.selfSum += self;
+      row.selfBuckets[bucketOf(self, bounds)] += 1;
+      row.statements += span.statements;
     },
     snapshot: () => ({
       since,
       series: [...series.values()],
       active: activeCalls(),
-      bounds: BOUNDS,
+      bounds: [...bounds],
       topology: topologyOf(app, seen),
       edges: [...edges.values()],
     }),
@@ -100,8 +127,14 @@ export function metrics(app?: App): Metrics {
 function topologyOf(app: App | undefined, seen: Set<string>): FrondPlacement[] {
   // What the app SERVES: a frond an extension brought is instrumentation, and reporting it
   // would describe this package to itself.
+  //
+  // A frond named in `remotes:` is scanned when its code sits in the same project, and stays
+  // in `app.fronds` — the boot says `declared remote — not hosted locally` and keeps it. This
+  // half reports what RUNS here, so it is not one of them: it used to answer `local` for a
+  // frond every call reached over HTTP. Not a config-derived node either — it is named below
+  // only once it has answered, like any other remote.
   const local = new Map((app?.fronds ?? [])
-    .filter((frond) => !frond.brought)
+    .filter((frond) => !frond.brought && !app?.remotes[frond.name])
     .map((frond) => [frond.name, frond] as const));
 
   const here: FrondPlacement[] = [...local.values()].map((frond) => ({
@@ -121,16 +154,23 @@ function topologyOf(app: App | undefined, seen: Set<string>): FrondPlacement[] {
 }
 
 /** The first bound this duration does not exceed, or the overflow bucket. */
-function bucketOf(seconds: number): number {
-  for (let i = 0; i < BOUNDS.length; i++) if (seconds <= BOUNDS[i]) return i;
-  return BOUNDS.length;
+function bucketOf(seconds: number, bounds: readonly number[]): number {
+  for (let i = 0; i < bounds.length; i++) if (seconds <= bounds[i]!) return i;
+
+  return bounds.length;
 }
 
-/** Serve the topology on `rpc.topology` — read from inside the process it describes. */
+/**
+ * Serve the topology on `rpc.topology` — read from inside the process it describes.
+ *
+ * The declared half is read at every call rather than once here: `reloadFougere()` builds the app
+ * again, and a config read at boot would keep answering for the app that was released.
+ */
 export function serveTopology(app: App, measured: Metrics): void {
   app.serveRpc('topology', (): TopologyReport => {
     const { since, active, topology, edges } = measured.snapshot();
-    return { since, active, fronds: topology, edges };
+
+    return { since, active, fronds: topology, edges, declared: declaredTopologyOf(app) };
   });
 }
 
@@ -139,6 +179,14 @@ export function metricsPayload(service: string, snapshot: MetricsSnapshot) {
   const since = `${snapshot.since * 1e6}`;
   const now = `${Date.now() * 1e6}`;
   const attr = (key: string, value: string) => ({ key, value: { stringValue: value } });
+  /** What an operation's three series are all sliced by — said once, so they stay comparable. */
+  const dimensions = (row: Bucketed) => [
+    ...(row.frond ? [attr('fougere.frond', row.frond)] : []),
+    attr('fougere.entity', row.entity),
+    attr('fougere.operation', row.operation),
+    attr('fougere.outcome', row.error ? 'error' : 'ok'),
+    ...(row.error ? [attr('fougere.error.code', row.error)] : []),
+  ];
 
   return {
     resourceMetrics: [
@@ -158,13 +206,7 @@ export function metricsPayload(service: string, snapshot: MetricsSnapshot) {
                   // only temporality Prometheus reads without a collector in between.
                   aggregationTemporality: 2,
                   dataPoints: snapshot.series.map((row) => ({
-                    attributes: [
-                      ...(row.frond ? [attr('fougere.frond', row.frond)] : []),
-                      attr('fougere.entity', row.entity),
-                      attr('fougere.operation', row.operation),
-                      attr('fougere.outcome', row.error ? 'error' : 'ok'),
-                      ...(row.error ? [attr('fougere.error.code', row.error)] : []),
-                    ],
+                    attributes: dimensions(row),
                     startTimeUnixNano: since,
                     timeUnixNano: now,
                     count: `${row.count}`,
@@ -174,6 +216,40 @@ export function metricsPayload(service: string, snapshot: MetricsSnapshot) {
                   })),
                 },
               },
+              {
+                name: 'fougere.operation.self',
+                description: 'How long an operation took on its own, with what it waited for taken out.',
+                unit: 's',
+                histogram: {
+                  aggregationTemporality: 2,
+                  dataPoints: snapshot.series.map((row) => ({
+                    attributes: dimensions(row),
+                    startTimeUnixNano: since,
+                    timeUnixNano: now,
+                    count: `${row.count}`,
+                    sum: row.selfSum,
+                    bucketCounts: row.selfBuckets.map((n) => `${n}`),
+                    explicitBounds: snapshot.bounds,
+                  })),
+                },
+              },
+              {
+                name: 'fougere.operation.statements',
+                description: 'Statements run under an operation. Against its count, statements per call.',
+                unit: '{statement}',
+                sum: {
+                  aggregationTemporality: 2,
+                  isMonotonic: true,
+                  dataPoints: snapshot.series
+                    .filter((row) => row.statements > 0)
+                    .map((row) => ({
+                      attributes: dimensions(row),
+                      startTimeUnixNano: since,
+                      timeUnixNano: now,
+                      asInt: `${row.statements}`,
+                    })),
+                },
+              },
               {
                 name: 'fougere.operations.active',
                 description: 'Operations running right now — the saturation signal.',

package/src/otlp.ts

@@ -32,6 +32,10 @@ export interface OtlpExporter {
 const OK = 1;
 const ERROR = 2;
 
+/** OTLP span kinds, of the six only these two are ours. */
+const INTERNAL = 1;
+const CLIENT = 3;
+
 export function otlp(options: OtlpOptions): OtlpExporter {
   const url = options.url ?? 'http://localhost:4318/v1/traces';
   const traces = Endpoint.at(url, options.onError);
@@ -75,7 +79,10 @@ function payload(service: string, spans: FinishedSpan[]) {
               spanId: span.spanId,
               ...(span.parentId ? { parentSpanId: span.parentId } : {}),
               name: `${span.entity}.${span.operation}`,
-              kind: 1,
+              // 1 INTERNAL, 3 CLIENT: a statement left this process for an engine, and a
+              // viewer draws the two differently. `selfMs` is NOT sent — a collector
+              // derives it from the tree it already holds.
+              kind: span.kind === 'statement' ? CLIENT : INTERNAL,
               startTimeUnixNano: nanos(span.startedAt),
               endTimeUnixNano: nanos(span.startedAt + span.ms),
               status: span.error ? { code: ERROR, message: span.error } : { code: OK },