@cosmicdrift/kumiko-framework 0.1.0

package/src/observability/recording-tracer.ts

@@ -0,0 +1,198 @@
+import { observabilityContext } from "./context";
+import { generateSpanId, generateTraceId } from "./ids";
+import { redactAttributes, redactValue, shouldRedactAttribute } from "./sensitive-filter";
+import type {
+  SensitiveFilterConfig,
+  SerializedTraceContext,
+  Span,
+  SpanAttributes,
+  SpanAttributeValue,
+  SpanKind,
+  SpanStatus,
+  StartSpanOptions,
+  Tracer,
+} from "./types";
+
+// A RecordedSpan is the internal representation that provider emitters
+// (console, otlp, test-collector) operate on. Every field is plain data —
+// no references to the tracer or provider — so recording is cheap and the
+// data can be serialized.
+export type RecordedSpan = {
+  readonly traceId: string;
+  readonly spanId: string;
+  readonly parentSpanId: string | undefined;
+  readonly name: string;
+  readonly kind: SpanKind;
+  readonly startTime: number;
+  endTime: number | undefined;
+  attributes: Record<string, SpanAttributeValue>;
+  status: SpanStatus;
+  statusMessage: string | undefined;
+  exception: { readonly name: string; readonly message: string } | undefined;
+};
+
+type SpanEndHandler = (span: RecordedSpan) => void;
+
+class RecordingSpan implements Span {
+  readonly traceId: string;
+  readonly spanId: string;
+  readonly parentSpanId: string | undefined;
+  readonly name: string;
+  private readonly record: RecordedSpan;
+  private readonly sensitiveConfig: SensitiveFilterConfig;
+  private readonly onEnd: SpanEndHandler;
+  private _ended = false;
+
+  constructor(args: {
+    record: RecordedSpan;
+    sensitiveConfig: SensitiveFilterConfig;
+    onEnd: SpanEndHandler;
+  }) {
+    this.record = args.record;
+    this.sensitiveConfig = args.sensitiveConfig;
+    this.onEnd = args.onEnd;
+    this.traceId = args.record.traceId;
+    this.spanId = args.record.spanId;
+    this.parentSpanId = args.record.parentSpanId;
+    this.name = args.record.name;
+  }
+
+  setAttribute(key: string, value: SpanAttributeValue): void {
+    // skip: span ended — mutations after end() would race the emitted snapshot
+    if (this._ended) return;
+    this.record.attributes[key] = shouldRedactAttribute(key, this.sensitiveConfig)
+      ? redactValue(value)
+      : value;
+  }
+
+  setAttributes(attrs: SpanAttributes): void {
+    // skip: span ended — see setAttribute comment
+    if (this._ended) return;
+    const safe = redactAttributes(attrs, this.sensitiveConfig);
+    for (const [k, v] of Object.entries(safe)) {
+      this.record.attributes[k] = v;
+    }
+  }
+
+  setStatus(status: SpanStatus, message?: string): void {
+    // skip: span ended — status is already part of the emitted snapshot
+    if (this._ended) return;
+    this.record.status = status;
+    this.record.statusMessage = message;
+  }
+
+  recordException(error: Error): void {
+    // skip: span ended — exception is already part of the emitted snapshot
+    if (this._ended) return;
+    this.record.exception = { name: error.name, message: error.message };
+  }
+
+  end(endTime?: number): void {
+    // skip: double-end — onEnd should fire exactly once per span
+    if (this._ended) return;
+    this._ended = true;
+    this.record.endTime = endTime ?? performance.now();
+    this.onEnd(this.record);
+  }
+
+  get ended(): boolean {
+    return this._ended;
+  }
+}
+
+export type RecordingTracerOptions = {
+  readonly sensitiveConfig: SensitiveFilterConfig;
+  // Called once per span after end(). Emitters (console, otlp) hook here.
+  readonly onSpanEnd: SpanEndHandler;
+};
+
+export class RecordingTracer implements Tracer {
+  private readonly sensitiveConfig: SensitiveFilterConfig;
+  private readonly onSpanEnd: SpanEndHandler;
+
+  constructor(opts: RecordingTracerOptions) {
+    this.sensitiveConfig = opts.sensitiveConfig;
+    this.onSpanEnd = opts.onSpanEnd;
+  }
+
+  startSpan(name: string, options?: StartSpanOptions): Span {
+    // Parent resolution: explicit parent (Span or SerializedTraceContext —
+    // both carry traceId+spanId so a uniform read works) or the ALS active
+    // span. A missing parent starts a new trace.
+    const explicitParent = options?.parent;
+    const active = explicitParent ?? this.getActiveSpan();
+    const traceId = active?.traceId ?? generateTraceId();
+    const parentSpanId = active?.spanId;
+
+    const record: RecordedSpan = {
+      traceId,
+      spanId: generateSpanId(),
+      parentSpanId,
+      name,
+      kind: options?.kind ?? "internal",
+      startTime: options?.startTime ?? performance.now(),
+      endTime: undefined,
+      attributes: options?.attributes
+        ? redactAttributes(options.attributes, this.sensitiveConfig)
+        : {},
+      status: "unset",
+      statusMessage: undefined,
+      exception: undefined,
+    };
+    return new RecordingSpan({
+      record,
+      sensitiveConfig: this.sensitiveConfig,
+      onEnd: this.onSpanEnd,
+    });
+  }
+
+  async withSpan<T>(
+    name: string,
+    optionsOrFn: StartSpanOptions | ((span: Span) => Promise<T>),
+    fn?: (span: Span) => Promise<T>,
+  ): Promise<T> {
+    const options = typeof optionsOrFn === "function" ? {} : (optionsOrFn ?? {});
+    const actualFn = typeof optionsOrFn === "function" ? optionsOrFn : fn;
+    if (!actualFn) {
+      throw new Error("withSpan called without callback");
+    }
+    const span = this.startSpan(name, options);
+    try {
+      return await observabilityContext.run({ activeSpan: span }, () => actualFn(span));
+    } catch (error) {
+      if (error instanceof Error) {
+        span.recordException(error);
+        span.setStatus("error", error.message);
+      } else {
+        span.setStatus("error", String(error));
+      }
+      throw error;
+    } finally {
+      if (!span.ended) span.end();
+    }
+  }
+
+  getActiveSpan(): Span | undefined {
+    return observabilityContext.getActiveSpan();
+  }
+
+  /**
+   * @deprecated Prefer `startSpan(name, { parent: context })`. Retained as a
+   *   thin alias for call-sites that pre-date the unified StartSpanOptions.
+   */
+  startSpanFromContext(
+    name: string,
+    context: SerializedTraceContext,
+    options?: StartSpanOptions,
+  ): Span {
+    return this.startSpan(name, { ...options, parent: context });
+  }
+}
+
+// Helper to serialize an active Span into the cross-process format.
+export function serializeSpanContext(span: Span): SerializedTraceContext {
+  return {
+    traceId: span.traceId,
+    spanId: span.spanId,
+  };
+}

package/src/observability/redis-wrapper.ts

@@ -0,0 +1,132 @@
+import type Redis from "ioredis";
+import type { Tracer } from "./types";
+
+// List of Redis commands we want to trace. Everything else (connection
+// methods like `on`, `off`, `disconnect`, `duplicate`, plus pipeline/multi)
+// falls through unwrapped. Keeping the list explicit avoids accidentally
+// wrapping things that aren't commands — `connect` is a Promise, `on` is
+// a listener registration — and keeps hot paths cheap.
+const TRACKED_COMMANDS = new Set<string>([
+  // Strings / keys
+  "get",
+  "set",
+  "del",
+  "exists",
+  "expire",
+  "ttl",
+  "incr",
+  "decr",
+  "keys",
+  "scan",
+  "mget",
+  "mset",
+  // Hashes
+  "hget",
+  "hset",
+  "hmget",
+  "hmset",
+  "hgetall",
+  "hdel",
+  "hexists",
+  // Lists
+  "lpush",
+  "rpush",
+  "lpop",
+  "rpop",
+  "llen",
+  "lrange",
+  // Sets
+  "sadd",
+  "srem",
+  "smembers",
+  "sismember",
+  // Sorted sets
+  "zadd",
+  "zrange",
+  "zrem",
+  "zrangebyscore",
+  // Streams
+  "xadd",
+  "xread",
+  "xreadgroup",
+  "xack",
+  "xlen",
+  "xgroup",
+  "xdel",
+  "xrange",
+  "xpending",
+  // Pub/Sub
+  "publish",
+  "subscribe",
+  "unsubscribe",
+  "psubscribe",
+  "punsubscribe",
+  // Scripting
+  "eval",
+  "evalsha",
+]);
+
+// Extract a redaction-safe key pattern from the first command argument.
+// Actual keys often include user-generated fragments (ids, session tokens)
+// that shouldn't leak into traces — we replace everything after the second
+// `:` segment with `*`. Known namespace conventions (colon-separated) keep
+// enough signal for grouping without leaking values.
+function extractKeyPattern(arg: unknown): string | undefined {
+  if (typeof arg !== "string") return undefined;
+  const parts = arg.split(":");
+  if (parts.length <= 2) return arg;
+  return `${parts.slice(0, 2).join(":")}:*`;
+}
+
+// Wrap an ioredis client in an observability-aware proxy. Traced commands
+// emit `redis.cmd` spans; everything else is passed through so pipeline,
+// transaction, and connection APIs keep working.
+export function wrapRedisClient(client: Redis, tracer: Tracer): Redis {
+  return new Proxy(client, {
+    get(target, prop, _receiver) {
+      // Symbols (e.g. internal queue) pass through unchanged.
+      if (typeof prop !== "string") {
+        return Reflect.get(target, prop, target);
+      }
+
+      const original = Reflect.get(target, prop, target);
+
+      if (prop === "duplicate") {
+        // Preserve wrapping on duplicated connections.
+        return (...args: unknown[]) => {
+          // @cast-boundary engine-bridge — Reflect.get returns unknown, narrow to ioredis-method
+          const dup = (original as (...args: unknown[]) => Redis).apply(target, args);
+          return wrapRedisClient(dup, tracer);
+        };
+      }
+
+      if (typeof original !== "function" || !TRACKED_COMMANDS.has(prop)) {
+        // Pass-through for non-command methods. Bind to target so `this`
+        // inside ioredis internals still works.
+        if (typeof original === "function") {
+          // @cast-boundary engine-bridge — Reflect.get returns unknown, narrow to ioredis-method
+          return (original as (...args: unknown[]) => unknown).bind(target);
+        }
+        return original;
+      }
+
+      return function wrappedCommand(this: unknown, ...args: unknown[]) {
+        const keyPattern = extractKeyPattern(args[0]);
+        return tracer.withSpan(
+          "redis.cmd",
+          {
+            kind: "client",
+            attributes: {
+              "redis.command": prop,
+              ...(keyPattern ? { "redis.key_pattern": keyPattern } : {}),
+            },
+          },
+          async () => {
+            // @cast-boundary engine-bridge — Reflect.get returns unknown, narrow to ioredis-method
+            return (original as (...args: unknown[]) => unknown).apply(target, args);
+          },
+        );
+      };
+    },
+  });
+}

package/src/observability/sensitive-filter.ts

@@ -0,0 +1,108 @@
+import type { SensitiveFilterConfig, SpanAttributeValue } from "./types";
+
+export const REDACTED = "[REDACTED]";
+
+// Defaults are intentionally conservative — easier to add a header to the
+// list than to explain a PII leak. All matching is case-insensitive.
+export const DEFAULT_SENSITIVE_CONFIG: SensitiveFilterConfig = {
+  redactedHeaders: ["authorization", "cookie", "set-cookie", "x-api-key", "proxy-authorization"],
+  redactedQueryParams: [
+    "token",
+    "access_token",
+    "refresh_token",
+    "password",
+    "secret",
+    "api_key",
+    "apikey",
+  ],
+  redactedAttributeKeyPatterns: [
+    /password/i,
+    /secret/i,
+    /token/i,
+    /apikey/i,
+    /privatekey/i,
+    /credential/i,
+    /session/i,
+  ],
+};
+
+export function mergeSensitiveConfig(
+  override: Partial<SensitiveFilterConfig> | undefined,
+): SensitiveFilterConfig {
+  if (!override) return DEFAULT_SENSITIVE_CONFIG;
+  return {
+    redactedHeaders: override.redactedHeaders ?? DEFAULT_SENSITIVE_CONFIG.redactedHeaders,
+    redactedQueryParams:
+      override.redactedQueryParams ?? DEFAULT_SENSITIVE_CONFIG.redactedQueryParams,
+    redactedAttributeKeyPatterns:
+      override.redactedAttributeKeyPatterns ??
+      DEFAULT_SENSITIVE_CONFIG.redactedAttributeKeyPatterns,
+  };
+}
+
+function lowercaseSet(names: readonly string[]): ReadonlySet<string> {
+  return new Set(names.map((n) => n.toLowerCase()));
+}
+
+export function redactHeaders(
+  headers: Readonly<Record<string, string>>,
+  config: SensitiveFilterConfig,
+): Record<string, string> {
+  const redacted = lowercaseSet(config.redactedHeaders);
+  const out: Record<string, string> = {};
+  for (const [key, value] of Object.entries(headers)) {
+    out[key] = redacted.has(key.toLowerCase()) ? REDACTED : value;
+  }
+  return out;
+}
+
+// Redact sensitive query params in a URL or query string. Returns a new URL
+// string with the same shape — path, fragment, and ordering preserved.
+export function redactQueryString(input: string, config: SensitiveFilterConfig): string {
+  const redacted = lowercaseSet(config.redactedQueryParams);
+  // URL wants absolute; handle both absolute and path-only forms.
+  const hasScheme = /^[a-z][a-z0-9+\-.]*:/i.test(input);
+  const base = hasScheme ? undefined : "http://_internal";
+  const url = base ? new URL(input, base) : new URL(input);
+  for (const key of Array.from(url.searchParams.keys())) {
+    if (redacted.has(key.toLowerCase())) {
+      url.searchParams.set(key, REDACTED);
+    }
+  }
+  if (!hasScheme) {
+    return `${url.pathname}${url.search}${url.hash}`;
+  }
+  return url.toString();
+}
+
+// Check a single attribute key against the redaction patterns.
+// Used by the Span implementation when setAttribute is called.
+export function shouldRedactAttribute(key: string, config: SensitiveFilterConfig): boolean {
+  for (const pattern of config.redactedAttributeKeyPatterns) {
+    if (pattern.test(key)) return true;
+  }
+  return false;
+}
+
+// Produce a type-preserving redacted value: numbers become 0, booleans become
+// false, strings become "[REDACTED]". Keeps downstream consumers (exporters,
+// dashboards) from having to deal with type drift — a histogram bucket that
+// expected a number never suddenly sees a string.
+export function redactValue(value: SpanAttributeValue): SpanAttributeValue {
+  if (typeof value === "number") return 0;
+  if (typeof value === "boolean") return false;
+  return REDACTED;
+}
+
+// Filter a full attribute map, replacing matching keys with a type-preserving
+// redacted value.
+export function redactAttributes(
+  attrs: Readonly<Record<string, SpanAttributeValue>>,
+  config: SensitiveFilterConfig,
+): Record<string, SpanAttributeValue> {
+  const out: Record<string, SpanAttributeValue> = {};
+  for (const [key, value] of Object.entries(attrs)) {
+    out[key] = shouldRedactAttribute(key, config) ? redactValue(value) : value;
+  }
+  return out;
+}

package/src/observability/standard-metrics.ts

@@ -0,0 +1,213 @@
+import type { Meter, MetricDefinition } from "./types";
+
+// Framework-level metrics registered automatically at boot. Names match the
+// Prometheus + OTel-friendly shape documented in observability-naming.md.
+// Feature code never emits these — the Framework wires them from the HTTP
+// middleware, dispatcher, and DB wrapper.
+
+export const STANDARD_METRIC_DEFS: readonly MetricDefinition[] = [
+  {
+    name: "kumiko_http_requests_total",
+    type: "counter",
+    description: "HTTP requests counted by route, method, and status.",
+    labels: ["route", "method", "status"],
+  },
+  {
+    name: "kumiko_http_request_duration_seconds",
+    type: "histogram",
+    description: "HTTP request latency in seconds.",
+    labels: ["route", "method"],
+    buckets: [0.005, 0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],
+  },
+  {
+    name: "kumiko_dispatcher_handler_duration_seconds",
+    type: "histogram",
+    description: "Dispatcher handler latency in seconds.",
+    labels: ["handler", "success"],
+    buckets: [0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5],
+  },
+  {
+    name: "kumiko_dispatcher_handler_errors_total",
+    type: "counter",
+    description: "Dispatcher handler errors by class.",
+    labels: ["handler", "error_class"],
+  },
+  {
+    name: "kumiko_db_query_duration_seconds",
+    type: "histogram",
+    description: "DB query latency in seconds.",
+    labels: ["operation", "table"],
+    buckets: [0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1],
+  },
+  // Projection-rebuild duration. Only emitted when a rebuild runs (ops-op),
+  // not continuously. Lag metric (continuous, live projections) is skipped
+  // for now — apply is synchronous, so lag is definitionally 0; a meaningful
+  // lag counter lands with async-apply in a future sprint.
+  {
+    name: "kumiko_projection_rebuild_duration_seconds",
+    type: "histogram",
+    description: "Duration of a full projection rebuild in seconds.",
+    labels: ["projection", "success"],
+    buckets: [0.01, 0.05, 0.1, 0.5, 1, 5, 30, 120, 600],
+  },
+  {
+    name: "kumiko_projection_rebuild_events_total",
+    type: "counter",
+    description: "Events replayed during a projection rebuild.",
+    labels: ["projection"],
+  },
+  // Event-dispatcher per-consumer metrics. Lag is the primary ops signal:
+  // how many events sit between the consumer's cursor and events-head.
+  // A growing lag means the consumer can't keep up; zero means fully caught up.
+  // instance_id labels each shard: SHARED_INSTANCE_SENTINEL for shared
+  // consumers (one cursor globally), or the process-local instanceId for
+  // per-instance consumers (one cursor per dispatcher). Without this,
+  // Prometheus collapses per-instance shards into last-writer-wins and
+  // per-instance lag is invisible to alerting.
+  {
+    name: "kumiko_event_consumer_lag_events",
+    type: "gauge",
+    description: "Number of events between the consumer's cursor and the events head.",
+    labels: ["consumer", "instance_id"],
+  },
+  {
+    name: "kumiko_event_consumer_events_processed_total",
+    type: "counter",
+    description: "Events successfully delivered to a consumer.",
+    labels: ["consumer", "instance_id"],
+  },
+  {
+    name: "kumiko_event_consumer_events_failed_total",
+    type: "counter",
+    description:
+      "Event deliveries that threw. Repeated failures on the same event lead to dead-letter.",
+    labels: ["consumer", "instance_id"],
+  },
+  // LISTEN-subscription health. 1 = active, 0 = dropped. Drops to 0 while
+  // the dispatcher is running signal that delivery latency has regressed
+  // from sub-millisecond (LISTEN) to pollIntervalMs (timer fallback); ops
+  // should look at it the moment downstream latency SLOs start slipping.
+  {
+    name: "kumiko_event_dispatcher_listen_connected",
+    type: "gauge",
+    description:
+      "1 if the event-dispatcher holds an active PG LISTEN subscription on the events channel, 0 otherwise.",
+    labels: [],
+  },
+] as const;
+
+export function registerStandardMetrics(meter: Meter): void {
+  for (const def of STANDARD_METRIC_DEFS) {
+    // Guard: if already registered (e.g. buildServer called twice with the
+    // same meter instance — rare outside of hot-reload scenarios), skip.
+    if (meter.definitions().has(def.name)) continue;
+    meter.registerMetric(def);
+  }
+}
+
+// Emit helpers — read-only surface for Auto-Instrumentation call sites.
+// Using these instead of raw meter.counter("...").inc() keeps the metric
+// names centralised.
+
+export function emitHttpRequest(
+  meter: Meter,
+  labels: { readonly route: string; readonly method: string; readonly status: number },
+  durationSeconds: number,
+): void {
+  meter.counter("kumiko_http_requests_total").inc(1, {
+    route: labels.route,
+    method: labels.method,
+    status: String(labels.status),
+  });
+  meter
+    .histogram("kumiko_http_request_duration_seconds")
+    .observe(durationSeconds, { route: labels.route, method: labels.method });
+}
+
+export function emitDispatcherHandler(
+  meter: Meter,
+  labels: { readonly handler: string; readonly success: boolean },
+  durationSeconds: number,
+): void {
+  meter.histogram("kumiko_dispatcher_handler_duration_seconds").observe(durationSeconds, {
+    handler: labels.handler,
+    success: String(labels.success),
+  });
+}
+
+export function emitDispatcherError(
+  meter: Meter,
+  labels: { readonly handler: string; readonly errorClass: string },
+): void {
+  meter.counter("kumiko_dispatcher_handler_errors_total").inc(1, {
+    handler: labels.handler,
+    error_class: labels.errorClass,
+  });
+}
+
+export function emitDbQuery(
+  meter: Meter,
+  labels: { readonly operation: string; readonly table: string },
+  durationSeconds: number,
+): void {
+  meter.histogram("kumiko_db_query_duration_seconds").observe(durationSeconds, {
+    operation: labels.operation,
+    table: labels.table,
+  });
+}
+
+export function emitProjectionRebuild(
+  meter: Meter,
+  labels: { readonly projection: string; readonly success: boolean },
+  durationSeconds: number,
+  eventsReplayed: number,
+): void {
+  meter.histogram("kumiko_projection_rebuild_duration_seconds").observe(durationSeconds, {
+    projection: labels.projection,
+    success: String(labels.success),
+  });
+  meter
+    .counter("kumiko_projection_rebuild_events_total")
+    .inc(eventsReplayed, { projection: labels.projection });
+}
+
+// Per-shard consumer labels. `instance_id` is the reserved sentinel
+// (SHARED_INSTANCE_SENTINEL) for shared-delivery consumers, and the
+// concrete dispatcher-instanceId for per-instance shards. Labelling by
+// both keeps Prometheus series distinct across instances so per-instance
+// lag is visible — without it, scraped gauges collapse last-writer-wins
+// and a single slow shard is invisible to alerting.
+export function emitEventConsumerLag(
+  meter: Meter,
+  labels: { readonly consumer: string; readonly instanceId: string },
+  lagEvents: number,
+): void {
+  meter.gauge("kumiko_event_consumer_lag_events").set(lagEvents, {
+    consumer: labels.consumer,
+    instance_id: labels.instanceId,
+  });
+}
+
+export function emitEventDispatcherListenConnected(meter: Meter, connected: boolean): void {
+  meter.gauge("kumiko_event_dispatcher_listen_connected").set(connected ? 1 : 0);
+}
+
+export function emitEventConsumerPassOutcome(
+  meter: Meter,
+  labels: { readonly consumer: string; readonly instanceId: string },
+  processed: number,
+  failed: number,
+): void {
+  if (processed > 0) {
+    meter.counter("kumiko_event_consumer_events_processed_total").inc(processed, {
+      consumer: labels.consumer,
+      instance_id: labels.instanceId,
+    });
+  }
+  if (failed > 0) {
+    meter.counter("kumiko_event_consumer_events_failed_total").inc(failed, {
+      consumer: labels.consumer,
+      instance_id: labels.instanceId,
+    });
+  }
+}

package/src/observability/types/index.ts

@@ -0,0 +1,29 @@
+// Barrel for observability types. Split into span/metric/provider files so
+// each module stays focused; consumers still import from "./types".
+
+export type {
+  Counter,
+  Gauge,
+  Histogram,
+  Meter,
+  MetricDefinition,
+  MetricLabels,
+  MetricsHandle,
+  MetricType,
+} from "./metric";
+export type {
+  ObservabilityOptions,
+  ObservabilityProvider,
+  SamplingConfig,
+  SensitiveFilterConfig,
+} from "./provider";
+export type {
+  SerializedTraceContext,
+  Span,
+  SpanAttributes,
+  SpanAttributeValue,
+  SpanKind,
+  SpanStatus,
+  StartSpanOptions,
+  Tracer,
+} from "./span";