@cosmicdrift/kumiko-framework 1.0.0 → 2.0.0

package/src/observability/standard-metrics.ts

@@ -23,7 +23,7 @@ export const STANDARD_METRIC_DEFS: readonly MetricDefinition[] = [
     name: "kumiko_dispatcher_handler_duration_seconds",
     type: "histogram",
     description: "Dispatcher handler latency in seconds.",
-    labels: ["handler", "success"],
+    labels: ["handler", "success", "outcome"],
     buckets: [0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5],
   },
   {
@@ -83,6 +83,20 @@ export const STANDARD_METRIC_DEFS: readonly MetricDefinition[] = [
       "Event deliveries that threw. Repeated failures on the same event lead to dead-letter.",
     labels: ["consumer", "instance_id"],
   },
+  // Fires once per (consumer, instance) transition into "stuck dead" —
+  // the rearm-cooldown/rearm-budget in event-dispatcher-delivery.ts gave up
+  // and the consumer now needs an operator (restartConsumer()) to recover.
+  // Without this, "dead + auto-rearm exhausted" is only visible via a
+  // getConsumerState poll — ops has no push signal that a consumer crossed
+  // from "will self-heal" to "needs a human". Deliberately a counter, not a
+  // gauge: emitted once per transition (event-dispatcher.ts tracks which
+  // consumers already fired this), not once per still-dead poll pass.
+  {
+    name: "kumiko_event_consumer_rearm_exhausted_total",
+    type: "counter",
+    description: "Consumer transitioned to dead with its auto-rearm budget exhausted.",
+    labels: ["consumer", "instance_id"],
+  },
   // LISTEN-subscription health. 1 = active, 0 = dropped. Drops to 0 while
   // the dispatcher is running signal that delivery latency has regressed
   // from sub-millisecond (LISTEN) to pollIntervalMs (timer fallback); ops
@@ -94,6 +108,21 @@ export const STANDARD_METRIC_DEFS: readonly MetricDefinition[] = [
       "1 if the event-dispatcher holds an active PG LISTEN subscription on the events channel, 0 otherwise.",
     labels: [],
   },
+  // BullMQ backlog per lane + state (waiting/active/delayed/failed/…). Only
+  // the process that owns a lane's consumerLane polls it (job-runner.ts
+  // start()) — the count is a global Redis-backed value, no per-instance
+  // label needed. A growing "waiting" count means the lane's worker can't
+  // keep up with dispatch; "failed" trending up means jobs are erroring
+  // silently. Motivated by apps that run every job on the "api" lane
+  // (no dedicated worker consumer) — fanout/reconcile jobs then share
+  // Redis-queue capacity with request-handling, and contention is
+  // otherwise invisible until requests slow down.
+  {
+    name: "kumiko_job_queue_depth",
+    type: "gauge",
+    description: "BullMQ job counts per lane and state.",
+    labels: ["lane", "state"],
+  },
 ] as const;
 
 export function registerStandardMetrics(meter: Meter): void {
@@ -126,12 +155,18 @@ export function emitHttpRequest(
 
 export function emitDispatcherHandler(
   meter: Meter,
-  labels: { readonly handler: string; readonly success: boolean },
+  labels: {
+    readonly handler: string;
+    readonly success: boolean;
+    /** completed = drained normally; aborted = consumer walked away (SSE). */
+    readonly outcome?: "completed" | "aborted";
+  },
   durationSeconds: number,
 ): void {
   meter.histogram("kumiko_dispatcher_handler_duration_seconds").observe(durationSeconds, {
     handler: labels.handler,
     success: String(labels.success),
+    outcome: labels.outcome ?? "completed",
   });
 }
 
@@ -188,6 +223,19 @@ export function emitEventConsumerLag(
   });
 }
 
+// Caller (event-dispatcher.ts) is responsible for calling this exactly
+// once per (consumer, instanceId) transition into exhausted-dead — not
+// once per poll pass while it stays dead.
+export function emitEventConsumerRearmExhausted(
+  meter: Meter,
+  labels: { readonly consumer: string; readonly instanceId: string },
+): void {
+  meter.counter("kumiko_event_consumer_rearm_exhausted_total").inc(1, {
+    consumer: labels.consumer,
+    instance_id: labels.instanceId,
+  });
+}
+
 export function emitEventDispatcherListenConnected(meter: Meter, connected: boolean): void {
   meter.gauge("kumiko_event_dispatcher_listen_connected").set(connected ? 1 : 0);
 }
@@ -211,3 +259,17 @@ export function emitEventConsumerPassOutcome(
     });
   }
 }
+
+// counts: BullMQ's Queue.getJobCounts() result, filtered by the caller to
+// backlog-relevant states (waiting/active/delayed/failed/paused) — "completed"
+// grows unboundedly without removeOnComplete and would make the gauge series
+// monotonically increasing, useless as a backlog signal.
+export function emitJobQueueDepth(
+  meter: Meter,
+  lane: string,
+  counts: Readonly<Record<string, number>>,
+): void {
+  for (const [state, count] of Object.entries(counts)) {
+    meter.gauge("kumiko_job_queue_depth").set(count, { lane, state });
+  }
+}

package/src/observability/types/index.ts

@@ -1,29 +1 @@
-// Barrel for observability types. Split into span/metric/provider files so
-// each module stays focused; consumers still import from "./types".
-
-export type {
-  Counter,
-  Gauge,
-  Histogram,
-  Meter,
-  MetricDefinition,
-  MetricLabels,
-  MetricsHandle,
-  MetricType,
-} from "./metric";
-export type {
-  ObservabilityOptions,
-  ObservabilityProvider,
-  SamplingConfig,
-  SensitiveFilterConfig,
-} from "./provider";
-export type {
-  SerializedTraceContext,
-  Span,
-  SpanAttributes,
-  SpanAttributeValue,
-  SpanKind,
-  SpanStatus,
-  StartSpanOptions,
-  Tracer,
-} from "./span";
+export type * from "@cosmicdrift/kumiko-types/observability";

package/src/observability/types/metric.ts

@@ -1,56 +1 @@
-// Metric + Meter contract. Feature code interacts through ctx.metrics
-// (MetricsHandle); the framework wires Counter/Histogram/Gauge behind that.
-
-export type MetricLabels = Record<string, string | number | boolean>;
-
-export type MetricType = "counter" | "histogram" | "gauge";
-
-export type MetricDefinition = {
-  // Fully-qualified name (with kumiko_<feature>_ prefix already applied).
-  readonly name: string;
-  readonly type: MetricType;
-  readonly description?: string;
-  // Declared label keys. Inc/observe calls with unknown label keys throw.
-  readonly labels?: readonly string[];
-  // Buckets only for histogram. If omitted, provider-default is used.
-  readonly buckets?: readonly number[];
-  readonly unit?: string;
-  // If true, the framework auto-injects tenant_id from the active ctx.
-  // Default false — adding tenant_id multiplies cardinality by tenant count.
-  readonly tenantLabel?: boolean;
-};
-
-export interface Counter {
-  inc(value?: number, labels?: MetricLabels): void;
-}
-
-export interface Histogram {
-  observe(value: number, labels?: MetricLabels): void;
-}
-
-export interface Gauge {
-  set(value: number, labels?: MetricLabels): void;
-  inc(value?: number, labels?: MetricLabels): void;
-  dec(value?: number, labels?: MetricLabels): void;
-}
-
-export interface Meter {
-  // Called once per metric during boot. Duplicate names throw.
-  registerMetric(def: MetricDefinition): void;
-  // Lookup by fully-qualified name. Unknown names throw — typed access only.
-  counter(name: string): Counter;
-  histogram(name: string): Histogram;
-  gauge(name: string): Gauge;
-  // List of registered definitions — used by ctx.metrics to validate labels.
-  definitions(): ReadonlyMap<string, MetricDefinition>;
-}
-
-// Public handle for feature code — ctx.metrics.
-// Name resolution uses the fully-qualified name (kumiko_<feature>_<short>).
-// The registrar resolves the short name from the calling feature context
-// at boot time; at handler time the map is ready.
-export interface MetricsHandle {
-  inc(name: string, labels?: MetricLabels, value?: number): void;
-  observe(name: string, value: number, labels?: MetricLabels): void;
-  set(name: string, value: number, labels?: MetricLabels): void;
-}
+export type * from "@cosmicdrift/kumiko-types/observability/metric";

package/src/observability/types/provider.ts

@@ -1,32 +1 @@
-// Observability provider contract — the "plug" that implementations (noop,
-// console, otlp, prometheus) fulfil. Configuration types that consumers
-// (buildServer, setupTestStack) hand in also live here.
-
-import type { Meter } from "./metric";
-import type { Tracer } from "./span";
-
-export type SamplingConfig = {
-  // Base sampling rate 0..1. Default 1 in v1 (sample everything).
-  readonly tracing?: number;
-  readonly alwaysOnError?: boolean;
-  readonly alwaysOnSlow?: { readonly thresholdMs: number };
-};
-
-export type SensitiveFilterConfig = {
-  readonly redactedHeaders: readonly string[];
-  readonly redactedQueryParams: readonly string[];
-  readonly redactedAttributeKeyPatterns: readonly RegExp[];
-};
-
-export type ObservabilityOptions = {
-  readonly sampling?: SamplingConfig;
-  readonly sensitiveFilter?: Partial<SensitiveFilterConfig>;
-};
-
-export interface ObservabilityProvider {
-  readonly name: string;
-  readonly tracer: Tracer;
-  readonly meter: Meter;
-  // Graceful flush. Called from framework lifecycle shutdown.
-  shutdown(): Promise<void>;
-}
+export type * from "@cosmicdrift/kumiko-types/observability/provider";

package/src/observability/types/span.ts

@@ -1,64 +1 @@
-// Span + Tracer contract. Provider implementations (noop, recording → console/otlp)
-// all speak this — everything else (middleware, dispatcher, db, redis, jobs)
-// is provider-agnostic.
-
-export type SpanAttributeValue = string | number | boolean;
-export type SpanAttributes = Record<string, SpanAttributeValue>;
-
-export type SpanStatus = "unset" | "ok" | "error";
-
-export type SpanKind = "internal" | "server" | "client" | "producer" | "consumer";
-
-// Serialized form of a trace context — what we pass across process boundaries
-// (outbox row, BullMQ job payload). Matches W3C trace-context spec loosely,
-// but staying minimal — a full W3C parser is v2.
-export type SerializedTraceContext = {
-  readonly traceId: string;
-  readonly spanId: string;
-  readonly baggage?: Readonly<Record<string, string>>;
-};
-
-export type StartSpanOptions = {
-  // Either a live Span (normal in-process parent) or a serialized trace
-  // context (cross-process — outbox row, job payload). Both carry traceId
-  // and spanId, so the tracer reads them uniformly. Omitted → fall back to
-  // the AsyncLocalStorage active span.
-  readonly parent?: Span | SerializedTraceContext;
-  readonly attributes?: SpanAttributes;
-  readonly kind?: SpanKind;
-  readonly startTime?: number;
-};
-
-export interface Span {
-  readonly traceId: string;
-  readonly spanId: string;
-  readonly parentSpanId: string | undefined;
-  readonly name: string;
-  setAttribute(key: string, value: SpanAttributeValue): void;
-  setAttributes(attrs: SpanAttributes): void;
-  setStatus(status: SpanStatus, message?: string): void;
-  recordException(error: Error): void;
-  end(endTime?: number): void;
-  // Whether end() has been called. Idempotency guard for auto-wrappers.
-  readonly ended: boolean;
-}
-
-export interface Tracer {
-  startSpan(name: string, options?: StartSpanOptions): Span;
-  // Runs fn inside the span context (AsyncLocalStorage), ends the span
-  // automatically — including on thrown errors, where the error is recorded
-  // and status set to "error" before re-throwing.
-  withSpan<T>(
-    name: string,
-    optionsOrFn: StartSpanOptions | ((span: Span) => Promise<T>),
-    fn?: (span: Span) => Promise<T>,
-  ): Promise<T>;
-  // Current active span from AsyncLocalStorage, or undefined.
-  getActiveSpan(): Span | undefined;
-  // @deprecated Prefer `startSpan(name, { parent: context })`.
-  startSpanFromContext(
-    name: string,
-    context: SerializedTraceContext,
-    options?: StartSpanOptions,
-  ): Span;
-}
+export type * from "@cosmicdrift/kumiko-types/observability/span";

package/src/pipeline/__tests__/ctx-bridge.integration.test.ts

@@ -131,14 +131,14 @@ const bridgeFeature = defineFeature("ctxbridge", (r) => {
   );
 
   // afterCommit hook on bag — fires once per outer commit.
-  r.entityHook("postSave", bag, async (result) => {
+  r.hook("postSave", { allOf: bag }, async (result) => {
     afterCommitLog.push(`bag:${result.data["label"]}`);
   });
 
   // afterCommit hook on secret — the entity targeted by the nested writeAs.
   // Proves: (a) hook fires exactly once per successful writeAs, (b) hook
   // does NOT fire when the outer transaction rolls back.
-  r.entityHook("postSave", secret, async (result) => {
+  r.hook("postSave", { allOf: secret }, async (result) => {
     afterCommitLog.push(`secret:${result.data["owner"]}`);
   });
 });