@fougere/observability 0.9.2-alpha.0 → 0.10.0-alpha.0

package/src/extension.ts

@@ -2,11 +2,10 @@
 import { frond, loggerMiddleware, Logger, LogLine, type App, type Extension, type LogSink } from '@fougere/core';
 import ExportHandler from './ExportHandler.js';
 import { traceContext } from '#trace-context';
-import { registerFlush } from './index.js';
-import { trace, type SpanSink } from './index.js';
-import { metrics, serveTopology } from './metrics.js';
-import { otlp } from './otlp.js';
-import { logs } from './logs.js';
+import { registerFlush, statementsUnder, tracing, type SpanSink } from './index.js';
+import { metrics, serveTopology } from './metrics/Metrics.js';
+import { otlp } from './otlp/OtlpExporter.js';
+import { logs } from './logs/LogExporter.js';
 
 export interface ObservabilityOptions {
   /** What a dashboard groups this process by, and the name a log line carries. */
@@ -23,6 +22,45 @@ export interface ObservabilityOptions {
    * extension took.
    */
   onSpan?: SpanSink;
+  /**
+   * A span of its own for every statement, beside the count every operation already carries.
+   * Off by default — see `TracingOptions.spanPerStatement`.
+   */
+  spanPerStatement?: boolean;
+  /**
+   * What share of the operations that stay in this process are traced, `0` to `1`. Default `1`.
+   *
+   * A rate is a budget, which no reading of the code answers. What the code does answer is
+   * which spans a budget may not touch — see `TracingOptions.sample`.
+   */
+  sample?: number;
+  /**
+   * Bucket bounds in SECONDS for the duration histograms. Default: the OpenTelemetry
+   * recommendation for request durations, which assumes a healthy op lives under 100 ms.
+   *
+   * One list for the whole process, deliberately — see `metrics.ts`.
+   */
+  bounds?: readonly number[];
+}
+
+/**
+ * How far each operation's work goes, read off the model ONCE at boot.
+ *
+ * `EffectiveOperation.reach` is resolved when the app is built, so the middleware does not
+ * recompute it per call — it looks one name up. An address the model does not hold answers zero,
+ * which is what `rpc` and a brought frond's own ops are.
+ */
+function hopsIn(app: App): (entity: string, operation: string) => number {
+  const hops = new Map<string, number>();
+  for (const frond of app.fronds) {
+    for (const handler of frond.handlers) {
+      for (const [name, op] of app.operationsFor(handler.address) ?? []) {
+        hops.set(`${handler.address}.${name}`, op.reach.hops);
+      }
+    }
+  }
+
+  return (entity, operation) => hops.get(`${entity}.${operation}`) ?? 0;
 }
 
 /** Observe this process — one member of the ascent. */
@@ -55,17 +93,23 @@ export function observability(options: ObservabilityOptions = {}): Extension {
       }],
     })],
 
-    up(app: App) {
+    async up(app: App) {
       // Held across the ascent: the exporter exists only when `otlp` is declared, and the
       // handler resolves this either way.
       app.container.registerValue('LogExport', exporting);
       const undo: (() => void | Promise<void>)[] = [];
       undoing.set(app, undo);
-      // Order matters: `trace()` opens the span that every log line written inside the
+      // Order matters: `tracing()` opens the span that every log line written inside the
       // call will carry. Installed the other way round, the lines leave uncorrelated.
       const spans: SpanSink[] = options.onSpan ? [options.onSpan] : [];
       takers.set(app, spans);
-      app.use(trace(spans));
+      const tracer = tracing(spans, {
+        spanPerStatement: options.spanPerStatement ?? false,
+        ...(options.sample === undefined ? {} : { sample: options.sample }),
+        hopsOf: hopsIn(app),
+      });
+      app.use(tracer.middleware);
+      undo.push(await statementsUnder(tracer));
       // The app's own logger, named — NOT `new Logger(service)`: a logger built here has
       // no `Carry`, so its lines printed and announced nothing. 20 on the console, 0 in
       // the ring, measured on `demos/observability`.
@@ -84,7 +128,7 @@ export function observability(options: ObservabilityOptions = {}): Extension {
         );
       }
 
-      const measured = metrics(app);
+      const measured = metrics(app, options.bounds);
       spans.push(measured.sink);
       serveTopology(app, measured);
 

package/src/index.ts

@@ -6,8 +6,24 @@ import { parseTraceparent, traceparentOf, randomHex, type SpanContext } from './
 /** A step while it runs. */
 interface Running extends SpanContext {
   frond: string | undefined;
+  /**
+   * When this step began, on both clocks — carried so what runs UNDER it is dated in ITS
+   * frame rather than on a clock of its own. `Date.now()` resolves to the millisecond and a
+   * whole call can take a fraction of one, so a statement dating itself lands anywhere:
+   * measured on SigNoz 2026-09-13, six queries drawn 17 µs to the LEFT of the call that made
+   * them, then all six stacked on its first instant. Offset from the monotonic clock, they
+   * fall where they ran.
+   */
+  startedAt: number;
+  start: number;
 }
 
+/**
+ * What a span is a span OF. An operation was dispatched and has an address; a statement ran
+ * underneath one and has a table. A reader that counts operations has to say which it takes.
+ */
+export type SpanKind = 'operation' | 'statement';
+
 /** A step that has finished, and what it did. */
 export interface FinishedSpan extends SpanContext {
   parentId: string | undefined;
@@ -18,17 +34,80 @@ export interface FinishedSpan extends SpanContext {
   callerFrond: string | undefined;
   /** Which frond owned the op — the deployment unit, so the first thing a reader groups by. */
   frond: string | undefined;
+  kind: SpanKind;
   entity: string;
   operation: string;
   /** When it started, in epoch milliseconds — an INSTANT, not an offset. */
   startedAt: number;
   ms: number;
+  /**
+   * `ms` minus what its OBSERVED children account for — the time this step spent itself. An op
+   * that delegates everything reports a duration and almost no self, which is how a report
+   * names the step that is actually slow instead of the one that waited for it.
+   *
+   * Never below zero, and that is not a guard: children running TOGETHER — the four queries
+   * behind one `Promise.all` — add up to more than the step that holds them. What is
+   * measured is the time nothing else accounts for, and there is no less of it than none.
+   *
+   * A child that finishes AFTER its parent is not deducted either: nothing was observed when
+   * the subtraction ran. The figure stays true, it is only cleaner than it should be.
+   */
+  selfMs: number;
+  /** How many statements ran under this step — one per row is the shape of an N+1. */
+  statements: number;
   /** The FougereError code when it refused, absent when it answered. */
   error: string | undefined;
 }
 
 export type SpanSink = (span: FinishedSpan) => void;
 
+/** A statement as its producer describes it — never the values it was given. */
+export interface Statement {
+  /** What it read or wrote: the table, or the source when the statement names none. */
+  subject: string;
+  /** `select`, `insert`, `update`, `delete`. */
+  verb: string;
+  ms: number;
+  failed: boolean;
+}
+
+/** What an app's tracer answers for. */
+export interface Tracing {
+  /** One span per operation — handed to `app.use`. */
+  middleware: AppMiddleware;
+  /** A statement that ran under the operation in flight: its time charged upward, and its own span when asked. */
+  statement(ran: Statement): void;
+}
+
+export interface TracingOptions {
+  /**
+   * A span of its OWN for every statement, and not only the time charged to the operation
+   * above it. Off by default, and the default is not prudence: `selfMs` and `statements` are
+   * counted either way, so what this adds is WHICH queries ran — a diagnosis, against a
+   * volume that multiplies by however many rows the page held. A backend charges per span.
+   *
+   * The pair splits the way profiling splits from monitoring: the count DETECTS, and it runs
+   * always; the detail EXPLAINS, and it is turned on over the operation the count named.
+   */
+  spanPerStatement?: boolean;
+  /**
+   * What share of the operations that stay in this process are traced, `0` to `1`.
+   *
+   * A rate is a BUDGET — how much a backend is worth per day — and no reading of the code
+   * answers it, so it is the operator's to set. What the code does answer is which spans the
+   * budget may not touch: an operation that crosses a process produces the one signal nothing
+   * else carries, since the difference between the caller's span and the callee's IS the wire
+   * cost, and neither process can measure it alone. So a rate below one thins what is already
+   * described by its own histogram, and never what only a trace can show.
+   *
+   * Default `1`, which is what a process did before this existed. The gain is not the default:
+   * it is that turning the rate down no longer drops the traces that matter most.
+   */
+  sample?: number;
+  /** How far an operation's work goes, by `entity.op` — the boot reads it off the model. */
+  hopsOf?: (entity: string, operation: string) => number;
+}
+
 /** The step running here and now. */
 
 /** The step running right now, if any. */
@@ -68,15 +147,33 @@ export function registerFlush(send: () => Promise<void>): () => void {
 }
 
 /**
- * The middleware, and it is HANDED who takes the spans it finishes.
+ * The tracer of ONE app, and it is HANDED who takes the spans it finishes.
  *
  * The list belongs to the app the middleware was installed on, never to the process: a
  * discarded app went on feeding the takers of the app that replaced it, so every metric
  * counted twice. A module-level array made that a leak you had to remember to undo; a list
  * held beside the app cannot outlive it. Read at every end, so a taker added later counts.
+ *
+ * Two facades and not one, because a statement is not dispatched: nothing calls a middleware
+ * around a query. What both need is the same table — the time a step's children accounted
+ * for — so the two sit in one closure rather than reaching for each other.
+ *
+ * Documented: [observability](https://fougere.dev/docs/infra/observability).
  */
-export function trace(takers: readonly SpanSink[]): AppMiddleware {
-  return (ctx, next) => {
+export function tracing(takers: readonly SpanSink[], options: TracingOptions = {}): Tracing {
+  /** What a step's children have accounted for, by the step they ran under. */
+  const charged = new Map<string, { ms: number; statements: number }>();
+  const rate = options.sample ?? 1;
+  const hopsOf = options.hopsOf ?? (() => 0);
+
+  /**
+   * Whether a root span is kept. An operation that leaves the process always is — the budget is
+   * for the ones a histogram already describes.
+   */
+  const keeps = (entity: string, operation: string) =>
+    rate >= 1 || hopsOf(entity, operation) > 0 || Math.random() < rate;
+
+  const middleware: AppMiddleware = (ctx, next) => {
     if (takers.length === 0) return next();
     // An op that CARRIES a line is not a call this process made: counting it puts the
     // delivery of a log line in the saturation figure, and spanning it puts a line about
@@ -88,11 +185,20 @@ export function trace(takers: readonly SpanSink[]): AppMiddleware {
     const inherited = parseTraceparent(ctx.invocation?.trace);
     const ambient = traceContext.current<Running>();
     const parent = inherited ?? ambient;
+    // Both, and they are not the same measurement: the wall clock says WHEN so two
+    // processes land on one timeline, the monotonic one says HOW LONG without being
+    // moved by an NTP correction mid-call.
+    const startedAt = Date.now();
+    const start = performance.now();
     const span: Running = {
       traceId: parent?.traceId ?? randomHex(16),
       spanId: randomHex(8),
-      sampled: parent?.sampled ?? true,
+      // A trace is whole or it is nothing: a call under a sampled parent is sampled, whatever
+      // this process would have decided on its own. Only a root is decided here.
+      sampled: parent?.sampled ?? keeps(ctx.entity, ctx.operation),
       frond: ctx.frond,
+      startedAt,
+      start,
     };
     // An edge exists only when the parent is IN this process and belongs to another frond.
     // `inherited` won means the parent is across a wire, and it did not name its frond.
@@ -105,23 +211,26 @@ export function trace(takers: readonly SpanSink[]): AppMiddleware {
     // carries the invocation.
     if (ctx.invocation) ctx.invocation = { ...ctx.invocation, trace: traceparentOf(span) };
 
-    // Both, and they are not the same measurement: the wall clock says WHEN so two
-    // processes land on one timeline, the monotonic one says HOW LONG without being
-    // moved by an NTP correction mid-call.
-    const startedAt = Date.now();
-    const start = performance.now();
     active += 1;
     const finish = (error: string | undefined) => {
       active -= 1;
+      const ms = performance.now() - start;
+      // Read and dropped in one gesture: a step ends once, and what its children owed it
+      // is of no use to anyone after that.
+      const owed = charged.get(span.spanId);
+      charged.delete(span.spanId);
       const done: FinishedSpan = {
         ...span,
         parentId: parent?.spanId,
         callerFrond,
         frond: ctx.frond,
+        kind: 'operation',
         entity: ctx.entity,
         operation: ctx.operation,
         startedAt,
-        ms: performance.now() - start,
+        ms,
+        selfMs: Math.max(0, ms - (owed?.ms ?? 0)),
+        statements: owed?.statements ?? 0,
         error,
       };
       // A taker that throws is a broken exporter, never a broken call.
@@ -139,6 +248,70 @@ export function trace(takers: readonly SpanSink[]): AppMiddleware {
       }
     });
   };
+
+  return {
+    middleware,
+
+    statement(ran: Statement): void {
+      if (takers.length === 0) return;
+      // A statement with no operation above it — a migration, a seed — belongs to no step,
+      // and inventing a trace for it would put a root span in the viewer per row planted.
+      const under = traceContext.current<Running>();
+      if (!under) return;
+
+      const owed = charged.get(under.spanId) ?? { ms: 0, statements: 0 };
+      owed.ms += ran.ms;
+      owed.statements += 1;
+      charged.set(under.spanId, owed);
+
+      // The op above now reports what it waited for and how many times, which is the whole
+      // of what a dashboard reads. WHICH queries ran leaves only when someone asks.
+      if (!options.spanPerStatement) return;
+
+      const done: FinishedSpan = {
+        traceId: under.traceId,
+        spanId: randomHex(8),
+        sampled: under.sampled,
+        parentId: under.spanId,
+        callerFrond: undefined,
+        frond: under.frond,
+        kind: 'statement',
+        entity: ran.subject,
+        operation: ran.verb,
+        startedAt: under.startedAt + Math.max(0, performance.now() - under.start - ran.ms),
+        ms: ran.ms,
+        selfMs: ran.ms,
+        statements: 0,
+        error: ran.failed ? 'STATEMENT_FAILED' : undefined,
+      };
+      for (const take of takers) { try { take(done); } catch { /* observing never refuses */ } }
+    },
+  };
+}
+
+/**
+ * The join this package exists to make: a statement runs SYNCHRONOUSLY inside the async
+ * context `tracing()` opened around the operation, so the sink finds the step it belongs to
+ * without the app ever holding its sources — which is what made the two ends of `onQuery`
+ * unable to see each other.
+ *
+ * `@fougere/adapter-sql` is optional, and dynamic for the reason `@fougere/calls` states:
+ * this package declares no dependency on it. An app on another storage observes no
+ * statement and every operation still reports its own time.
+ */
+export async function statementsUnder(tracer: Tracing): Promise<() => void> {
+  try {
+    const { onQuery } = await import('@fougere/adapter-sql');
+
+    return onQuery((event) => tracer.statement({
+      subject: event.subject,
+      verb: event.verb,
+      ms: event.ms,
+      failed: event.failed,
+    }));
+  } catch {
+    return () => {};
+  }
 }
 
 function codeOf(err: unknown): string {
@@ -146,8 +319,10 @@ function codeOf(err: unknown): string {
   return typeof code === 'string' ? code : ((err as Error)?.name ?? 'error');
 }
 
-export { otlp } from './otlp.js';
-export { metrics } from './metrics.js';
-export type { Metrics, TopologyReport, FrondPlacement, Edge } from './metrics.js';
-export { logs } from './logs.js';
+export { otlp } from './otlp/OtlpExporter.js';
+export { metrics } from './metrics/Metrics.js';
+export type { Metrics } from './metrics/Metrics.js';
+export type { TopologyReport } from '@fougere/core';
+export type { FrondPlacement, Edge } from '@fougere/core';
+export { logs } from './logs/LogExporter.js';
 export { observability } from './extension.js';

package/src/logs/CapturedLog.ts

@@ -0,0 +1,7 @@
+import type { LogRecord } from '@fougere/core';
+
+/** A record, plus what it could only be told at the moment it was written. */
+export interface CapturedLog extends LogRecord {
+  traceId: string | undefined;
+  spanId: string | undefined;
+}

package/src/{logs.ts → logs/LogExporter.ts}

@@ -1,8 +1,9 @@
-/** Log export — the third signal, and the only one whose value is entirely in its links. */
-import { Beat } from './Beat.js';
-import { Endpoint } from './Endpoint.js';
-import { currentSpan } from './index.js';
+import { Beat } from '../Beat.js';
+import { Endpoint } from '../Endpoint.js';
+import { currentSpan } from '../index.js';
 import type { LogRecord } from '@fougere/core';
+import type { CapturedLog } from './CapturedLog.js';
+import type { LogsOptions } from './LogsOptions.js';
 
 /** OTLP severity numbers — the scale is 1–24, these are the canonical rungs. */
 const SEVERITY: Record<string, { number: number; text: string }> = {
@@ -12,12 +13,6 @@ const SEVERITY: Record<string, { number: number; text: string }> = {
   error: { number: 17, text: 'ERROR' },
 };
 
-/** A record, plus what it could only be told at the moment it was written. */
-export interface CapturedLog extends LogRecord {
-  traceId: string | undefined;
-  spanId: string | undefined;
-}
-
 export interface LogExporter {
   /** Hand to `onLog`. */
   sink: (record: LogRecord) => void;
@@ -27,19 +22,6 @@ export interface LogExporter {
   stop(): Promise<void>;
 }
 
-export interface LogsOptions {
-  /** Which service these lines belong to. */
-  service: string;
-  /** Collector endpoint. Default: the OTLP/HTTP convention on localhost. */
-  url?: string;
-  /** How often a batch leaves. Default: every second. */
-  flushMs?: number;
-  /** Told when a batch could not be sent. Default: silence. */
-  onError?: (err: unknown) => void;
-  /** Drop anything below this level before it leaves the process. */
-  minimum?: 'debug' | 'info' | 'warn' | 'error';
-}
-
 export function logs(options: LogsOptions): LogExporter {
   const collector = Endpoint.at(options.url ?? 'http://localhost:4318/v1/logs', options.onError);
   const floor = options.minimum ? SEVERITY[options.minimum].number : 0;

package/src/logs/LogsOptions.ts

@@ -0,0 +1,12 @@
+export interface LogsOptions {
+  /** Which service these lines belong to. */
+  service: string;
+  /** Collector endpoint. Default: the OTLP/HTTP convention on localhost. */
+  url?: string;
+  /** How often a batch leaves. Default: every second. */
+  flushMs?: number;
+  /** Told when a batch could not be sent. Default: silence. */
+  onError?: (err: unknown) => void;
+  /** Drop anything below this level before it leaves the process. */
+  minimum?: 'debug' | 'info' | 'warn' | 'error';
+}

package/src/metrics/Bucketed.ts

@@ -0,0 +1,15 @@
+export interface Bucketed {
+  frond: string | undefined;
+  entity: string;
+  operation: string;
+  error: string | undefined;
+  count: number;
+  sum: number;
+  /** One more than the bounds: the last holds everything above the highest bound. */
+  buckets: number[];
+  /** The same measurement over `selfMs` — what the op did rather than what it waited for. */
+  selfSum: number;
+  selfBuckets: number[];
+  /** Every statement these calls ran. Against `count`, it is statements per call. */
+  statements: number;
+}