wide_events 0.1.0

data/lib/wide_event/registry.rb

@@ -0,0 +1,69 @@
+require "yaml"
+
+module WideEvent
+  # The attribute schema: every wide-event attribute is declared in a YAML
+  # registry (gem defaults merged with the app's registry.yml). Keys
+  # containing `*` are globs for dynamic attributes (feature_flag.*).
+  # In strict mode the accumulator reports every key it sees via #track;
+  # unregistered keys collect in #violations for the test suite to assert on.
+  class Registry
+    DEFAULTS_PATH = File.expand_path("registry/defaults.yml", __dir__)
+
+    class << self
+      def defaults
+        new(load_file(DEFAULTS_PATH))
+      end
+
+      def load(path)
+        new(load_file(DEFAULTS_PATH).merge(load_file(path)))
+      end
+
+      private
+
+      def load_file(path)
+        YAML.safe_load_file(path) || {}
+      end
+    end
+
+    attr_reader :entries
+
+    def initialize(entries)
+      @entries = entries
+      @exact = entries.keys.reject { |k| k.include?("*") }.to_set
+      @globs = entries.keys.select { |k| k.include?("*") }
+      @violations = Set.new
+      @mutex = Mutex.new
+    end
+
+    def registered?(key)
+      @exact.include?(key) || @globs.any? { |glob| File.fnmatch(glob, key) }
+    end
+
+    def track(key)
+      return if registered?(key)
+      @mutex.synchronize { @violations << key }
+      nil
+    end
+
+    def violations
+      @violations.to_a
+    end
+
+    def validate
+      entries.filter_map do |key, entry|
+        unless entry.is_a?(Hash) && entry["type"]
+          "#{key}: entry must be a mapping with at least a `type` field"
+        end
+      end
+    end
+
+    def to_markdown
+      lines = [ "| Attribute | Type | Set by | PII | Notes |", "|---|---|---|---|---|" ]
+      entries.each do |key, entry|
+        entry = {} unless entry.is_a?(Hash)
+        lines << "| `#{key}` | #{Array(entry["type"]).join(", ")} | #{entry["set_by"]} | #{entry["pii"]} | #{entry["notes"]} |"
+      end
+      lines.join("\n") + "\n"
+    end
+  end
+end

data/lib/wide_event/sinks/log_line.rb

@@ -0,0 +1,27 @@
+require "json"
+require "logger"
+
+module WideEvent
+  module Sinks
+    # Emits the wide event as one JSON log line: the canonical-log-line
+    # deployment for apps that don't run tracing.
+    class LogLine
+      def initialize(logger = nil)
+        @logger = logger
+      end
+
+      def flush(attrs)
+        line = { "timestamp" => Time.now.utc.iso8601(3) }.merge(WideEvent.sanitize(attrs))
+        logger.info(JSON.generate(line))
+      end
+
+      private
+
+      def logger
+        @logger ||= WideEvent.config.logger ||
+          (defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger) ||
+          Logger.new($stdout)
+      end
+    end
+  end
+end

data/lib/wide_event/sinks/memory.rb

@@ -0,0 +1,20 @@
+module WideEvent
+  module Sinks
+    # Captures flushed wide events in memory, for tests.
+    class Memory
+      attr_reader :events
+
+      def initialize
+        @events = []
+      end
+
+      def flush(attrs)
+        @events << attrs.dup
+      end
+
+      def clear
+        @events.clear
+      end
+    end
+  end
+end

data/lib/wide_event/sinks/otel_span.rb

@@ -0,0 +1,12 @@
+module WideEvent
+  module Sinks
+    # Writes the wide event onto the current OTel span: the root span of
+    # the unit of work, since flush happens as the request/job unwinds.
+    class OtelSpan
+      def flush(attrs)
+        return unless defined?(OpenTelemetry::Trace)
+        WideEvent.flush_hash(attrs, OpenTelemetry::Trace.current_span)
+      end
+    end
+  end
+end

data/lib/wide_event/span_counter_processor.rb

@@ -0,0 +1,29 @@
+module WideEvent
+  # Rolls finished dependency child spans up into the active wide event's
+  # stats.* attributes: what makes "this request ran 742 queries" visible
+  # without opening a trace. Instrumentation scopes map to dependency names
+  # via config.span_scopes. Registered by the railtie when OTel is active.
+  class SpanCounterProcessor
+    def on_start(span, parent_context); end
+
+    def on_finish(span)
+      return unless WideEvent.active?
+
+      dep = WideEvent.config.span_scopes[span.instrumentation_scope&.name]
+      return unless dep
+
+      duration_ms = (span.end_timestamp - span.start_timestamp) / 1_000_000.0
+      WideEvent.count(dep, duration_ms.round(2))
+    rescue StandardError => e
+      WideEvent.handle_error(e, "span counter")
+    end
+
+    def force_flush(timeout: nil)
+      OpenTelemetry::SDK::Trace::Export::SUCCESS
+    end
+
+    def shutdown(timeout: nil)
+      OpenTelemetry::SDK::Trace::Export::SUCCESS
+    end
+  end
+end

data/lib/wide_event/subscribers.rb

@@ -0,0 +1,40 @@
+module WideEvent
+  # Copies measurements Rails already makes into the active wide event via
+  # ActiveSupport::Notifications. Subscribed once by the railtie.
+  module Subscribers
+    def self.subscribe!
+      return if @subscribed
+
+      @subscribed = true
+
+      ActiveSupport::Notifications.subscribe("process_action.action_controller") do |event|
+        next unless WideEvent.active?
+
+        payload = event.payload
+        WideEvent.set("http.route.controller" => "#{payload[:controller]}##{payload[:action]}")
+        WideEvent.set("db.duration_ms" => payload[:db_runtime].round(2)) if payload[:db_runtime]
+        WideEvent.set("view.duration_ms" => payload[:view_runtime].round(2)) if payload[:view_runtime]
+      rescue StandardError => e
+        WideEvent.handle_error(e, "process_action")
+      end
+
+      ActiveSupport::Notifications.subscribe("cache_read.active_support") do |event|
+        next unless WideEvent.active?
+
+        prefix = event.payload[:key].to_s.split(%r{[/:]}).first.to_s.gsub(/[^a-z0-9_]+/i, "_")[0, 30].downcase
+        next if prefix.empty?
+
+        key = "cache.#{prefix}"
+        attrs = WideEvent.peek
+        # Cap distinct cache namespaces, but keep refreshing ones already
+        # tracked: otherwise a prefix freezes at whatever value it had
+        # when the cap-th distinct namespace appeared.
+        next if !attrs.key?(key) && attrs.keys.count { |k| k.start_with?("cache.") } >= WideEvent.config.max_cache_attrs
+
+        WideEvent.set(key => event.payload[:hit] ? true : false)
+      rescue StandardError => e
+        WideEvent.handle_error(e, "cache_read")
+      end
+    end
+  end
+end

data/lib/wide_event/tasks/registry.rake

@@ -0,0 +1,26 @@
+namespace :wide_events do
+  namespace :registry do
+    desc "Validate the wide-event attribute registry"
+    task check: :environment do
+      registry = WideEvent.config.registry
+      abort("No wide-event registry at #{WideEvent.config.registry_path.inspect}: run `bin/rails generate wide_events:install`") if registry.nil?
+      errors = registry.validate
+      abort("wide-event registry invalid:\n  #{errors.join("\n  ")}") unless errors.empty?
+      puts "wide-event registry OK (#{registry.entries.size} entries)"
+    end
+
+    desc "Generate the human-readable registry doc (default: docs/wide-events-registry.md)"
+    task docs: :environment do
+      registry = WideEvent.config.registry
+      abort("No wide-event registry at #{WideEvent.config.registry_path.inspect}: run `bin/rails generate wide_events:install`") if registry.nil?
+      path = ENV["WIDE_EVENTS_DOCS_PATH"] || "docs/wide-events-registry.md"
+      require "fileutils"
+      FileUtils.mkdir_p(File.dirname(path))
+      header = "# Wide-events attribute registry\n\n" \
+               "Generated from `#{WideEvent.config.registry_path}` by `rake wide_events:registry:docs`. " \
+               "Do not edit by hand.\n\n"
+      File.write(path, header + registry.to_markdown)
+      puts "wrote #{path}"
+    end
+  end
+end

data/lib/wide_event/test_helper.rb

@@ -0,0 +1,55 @@
+module WideEvent
+  # Minitest assertions for wide-event instrumentation. Require from your
+  # test helper and include in ActiveSupport::TestCase:
+  #
+  #   require "wide_event/test_helper"
+  #   class ActiveSupport::TestCase
+  #     include WideEvent::TestHelper
+  #   end
+  module TestHelper
+    # Runs the block inside an open wide event and asserts on the
+    # accumulated attributes. Bare string arguments assert presence;
+    # keyword-style pairs assert equality. Returns the attributes for
+    # further assertions.
+    #
+    #   assert_wide_event("pdf_render.duration_ms", "report.format" => "pdf") do
+    #     report.render
+    #   end
+    def assert_wide_event(*keys, **pairs, &block)
+      attrs = WideEvent.with do |store|
+        block.call
+        store.dup
+      end
+      keys.each do |key|
+        assert attrs.key?(key), "expected wide event to have #{key.inspect}, got keys #{attrs.keys.inspect}"
+      end
+      pairs.each do |key, value|
+        assert_equal value, attrs[key], "wide event attribute #{key.inspect}"
+      end
+      attrs
+    end
+
+    # Swaps the sink for an in-memory one for the duration of the block and
+    # returns every wide event flushed inside it. Use around code that runs
+    # its own unit of work (jobs, full request dispatch).
+    def capture_wide_events(&block)
+      original = WideEvent.config.sink
+      sink = Sinks::Memory.new
+      WideEvent.config.sink = sink
+      block.call
+      sink.events
+    ensure
+      WideEvent.config.sink = original
+    end
+
+    # Fails if strict mode saw any attribute that isn't declared in the
+    # registry. Call from a dedicated test (or a teardown hook) with
+    # config.strict = true to make the registry an enforced schema.
+    def assert_registered_wide_event_attributes
+      registry = WideEvent.config.registry
+      violations = registry ? registry.violations : []
+      assert violations.empty?,
+        "unregistered wide-event attributes #{violations.sort.inspect} - declare them in #{WideEvent.config.registry_path}"
+    end
+  end
+end

data/lib/wide_event/version.rb

@@ -0,0 +1,3 @@
+module WideEvent
+  VERSION = "0.1.0"
+end

data/lib/wide_event.rb

@@ -0,0 +1,188 @@
+require "time"
+require "active_support"
+require "active_support/isolated_execution_state"
+require "active_support/core_ext/object/blank"
+
+require "wide_event/version"
+require "wide_event/configuration"
+require "wide_event/registry"
+require "wide_event/sinks/otel_span"
+require "wide_event/sinks/log_line"
+require "wide_event/sinks/memory"
+require "wide_event/middleware"
+require "wide_event/span_counter_processor"
+require "wide_event/subscribers"
+require "wide_event/job_instrumentation"
+
+# Wide-event accumulator: one flat attribute hash per unit of work (HTTP
+# request or job execution), flushed to the configured sink by
+# WideEvent::Middleware / WideEvent::JobInstrumentation. Safe no-op when no
+# unit of work is open.
+module WideEvent
+  KEY = :wide_event_attributes
+  PROCESS_START = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+  class << self
+    def config
+      @config ||= Configuration.new
+    end
+
+    def configure
+      yield config
+      config
+    end
+
+    def reset_configuration!
+      @config = Configuration.new
+    end
+
+    def enabled?
+      config.enabled
+    end
+
+    def with
+      previous = store
+      current = {}
+      ActiveSupport::IsolatedExecutionState[KEY] = current
+      yield current
+    ensure
+      ActiveSupport::IsolatedExecutionState[KEY] = previous
+    end
+
+    def active?
+      !store.nil?
+    end
+
+    def set(attrs)
+      registry_check(attrs.keys)
+      store&.merge!(attrs)
+      nil
+    rescue StandardError => e
+      handle_error(e, "set")
+      nil
+    end
+
+    def count(dep, duration_ms)
+      s = store
+      return if s.nil?
+      registry_check([ "stats.#{dep}_count", "stats.#{dep}_duration_ms" ])
+      s["stats.#{dep}_count"] = s.fetch("stats.#{dep}_count", 0) + 1
+      s["stats.#{dep}_duration_ms"] = (s.fetch("stats.#{dep}_duration_ms", 0.0) + duration_ms).round(2)
+      nil
+    rescue StandardError => e
+      handle_error(e, "count")
+      nil
+    end
+
+    def phase(name)
+      start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      yield
+    ensure
+      elapsed_ms = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start) * 1000.0
+      begin
+        set("#{name}.duration_ms" => elapsed_ms.round(2))
+      rescue StandardError => e
+        handle_error(e, "phase")
+        nil
+      end
+    end
+
+    def error!(slug:, exception: nil, expected: false)
+      attrs = { "error" => true, "exception.slug" => slug, "exception.expected" => expected }
+      if exception
+        attrs["exception.type"] = exception.class.name
+        attrs["exception.message"] = exception.message.to_s[0, 500]
+      end
+      set(attrs)
+    rescue StandardError => e
+      handle_error(e, "error!")
+      nil
+    end
+
+    def attributes
+      store&.dup
+    end
+
+    # Read-only view of the live store for hot-path handlers that would
+    # otherwise dup per event (e.g. every cache read). Callers must not
+    # mutate: writes go through set/count/error!.
+    def peek
+      store
+    end
+
+    def uptime_attributes
+      uptime = Process.clock_gettime(Process::CLOCK_MONOTONIC) - PROCESS_START
+      { "uptime_sec" => uptime.round, "uptime_sec_log10" => Math.log10([ uptime, 1 ].max).round(3) }
+    end
+
+    # Hands a finished wide event to the configured sink. Never raises into
+    # the caller.
+    def flush(attrs)
+      return if attrs.nil? || attrs.empty?
+      config.resolved_sink&.flush(attrs)
+      nil
+    rescue StandardError => e
+      handle_error(e, "flush")
+      nil
+    end
+
+    # Wires the pieces that need the host's OpenTelemetry SDK configured:
+    # notification subscribers and, for the OTel sink, the span counter on
+    # the global tracer provider. Called by the railtie after the app's
+    # initializers; call it manually from a non-Rails setup.
+    def install!
+      Subscribers.subscribe!
+      if config.sink == :otel && defined?(OpenTelemetry) &&
+         OpenTelemetry.respond_to?(:tracer_provider) &&
+         OpenTelemetry.tracer_provider.respond_to?(:add_span_processor)
+        OpenTelemetry.tracer_provider.add_span_processor(SpanCounterProcessor.new)
+      end
+      nil
+    end
+
+    # Sanitize and write a wide-event hash onto an OTel span. `span` may be
+    # nil (OTel inactive). Never raises into the caller.
+    def flush_hash(hash, span)
+      return if hash.nil? || hash.empty? || span.nil?
+      span.add_attributes(sanitize(hash))
+    rescue StandardError => e
+      handle_error(e, "flush")
+      nil
+    end
+
+    def sanitize(attrs)
+      attrs.each_with_object({}) do |(k, v), out|
+        key = k.to_s
+        case v
+        when String, Integer, Float, true, false then out[key] = v
+        when Symbol then out[key] = v.to_s
+        when Time, DateTime, ActiveSupport::TimeWithZone then out[key] = v.to_time.utc.iso8601(3)
+        when nil then nil # drop
+        else out[key] = v.to_s[0, 300]
+        end
+      end
+    end
+
+    # Telemetry must never raise into app code: every public entry point
+    # funnels its rescues here, and a broken handler is itself swallowed.
+    def handle_error(exception, message)
+      config.error_handler&.call(exception, message)
+      nil
+    rescue StandardError
+      nil
+    end
+
+    private
+
+    def store
+      ActiveSupport::IsolatedExecutionState[KEY]
+    end
+
+    def registry_check(keys)
+      return unless config.strict
+      registry = config.registry
+      return if registry.nil?
+      keys.each { |key| registry.track(key.to_s) }
+    end
+  end
+end

data/lib/wide_events.rb

@@ -0,0 +1,3 @@
+require "wide_event"
+
+require "wide_event/railtie" if defined?(Rails::Railtie)

data/skills/debugging-with-wide-events/SKILL.md

@@ -0,0 +1,73 @@
+---
+name: debugging-with-wide-events
+description: Use when investigating production behavior by querying wide events (a slow route, an error spike, a suspicious deploy, "why is this happening for account X"). Covers the data model, the symptom-to-query workflow, and standing queries worth running.
+---
+
+# Debugging with wide events
+
+Every request and job execution is one row: the OTel root span with
+`main = true`, carrying route, user/account ids, build SHA, status, error
+state, per-dependency query counts, cache hits, feature flags, phase
+timings, and whatever domain attributes the app set. Cross-request analysis
+never needs child spans: filter `main = true` and aggregate.
+
+## Where the rows live
+
+- **ClickHouse (HyperDX/ClickStack default schema):** the `otel_traces`
+  table; wide-event attributes are in the `SpanAttributes` map, resource
+  attributes (build, host, environment) in `ResourceAttributes`. Adapt table
+  and column names to the backend at hand.
+- **Log sink deployments:** one JSON object per line; query with your log
+  store's JSON operators.
+
+Filter by environment when staging and production share storage.
+
+## Symptom to query
+
+Start from the unit of work, not from logs. One GROUP BY usually replaces a
+cross-tool investigation.
+
+"The reports route is slow":
+
+```sql
+SELECT SpanAttributes['user.account.id'] AS account,
+       ResourceAttributes['service.version'] AS build,
+       count() AS requests,
+       quantile(0.5)(Duration/1e6) AS p50_ms,
+       avg(toFloat64OrZero(SpanAttributes['stats.postgres_query_count'])) AS avg_queries
+FROM otel_traces
+WHERE SpanAttributes['main'] = 'true'
+  AND SpanAttributes['http.route.controller'] = 'ReportsController#index'
+  AND Timestamp > now() - INTERVAL 1 DAY
+GROUP BY account, build
+ORDER BY p50_ms DESC
+LIMIT 20
+```
+
+"Did the deploy change anything": group the affected route by
+`ResourceAttributes['service.version']` and compare p50/p95 and
+`stats.postgres_query_count` across builds.
+
+"Why did this specific request fail": filter by
+`SpanAttributes['http.request.id']` and read every attribute of the one row
+before reaching for logs or the trace waterfall.
+
+## Standing queries
+
+- **Uninstrumented failures**: `error = 'true' AND SpanAttributes['exception.slug'] = ''`:
+  failures nobody wrote a rescue for. Each row is a candidate `error!` call;
+  fixing them is normal instrumentation work (see the
+  instrumenting-wide-events skill).
+- **Expected-error drift**: count by `exception.slug` week over week; a
+  slug trending up is a quiet regression.
+- **Query-count outliers**: max `stats.postgres_query_count` per route; the
+  top entries are N+1s with names and account ids attached.
+- **Cold starts**: correlate latency with `uptime_sec_log10` to separate
+  deploy warm-up from real regressions.
+
+## Verifying your own fix
+
+After a deploy, don't ask a human to check a dashboard. Query the affected
+route/job filtered to the new `service.version` and compare against the
+prior build: error rate, p50, query count, and the specific attribute your
+change was supposed to move. State the numbers in your summary.

data/skills/instrumenting-wide-events/SKILL.md

@@ -0,0 +1,93 @@
+---
+name: instrumenting-wide-events
+description: Use when adding or changing app code that should be observable in production: new features, slow paths, rescue blocks, background jobs. Covers the WideEvent API (set/phase/error!), attribute naming conventions, the registry workflow, and test assertions.
+---
+
+# Instrumenting wide events
+
+Every HTTP request and job execution in this app emits one flat telemetry
+event: the OTel root span, marked `main = true` (or one JSON log line with
+the `:log` sink). The gem records the generic attributes (route, status,
+timings, query counts, cache hits). Your job is the attributes only the app
+knows: the domain nouns.
+
+## The API
+
+All calls are safe no-ops outside a request/job and never raise into app
+code: instrument freely.
+
+```ruby
+# Flat attributes, merged into the current event
+WideEvent.set("report.id" => report.id, "report.format" => "pdf")
+
+# Time a block into <name>.duration_ms (returns the block's value)
+WideEvent.phase("pdf_render") { render_pdf }
+
+# Mark a handled failure in a rescue block
+rescue Api::ClientError => e
+  WideEvent.error!(slug: "err-vendor-user-not-found", exception: e, expected: true)
+```
+
+`error!` slugs are static kebab-case string literals, unique across the app,
+prefixed `err-`. Never interpolate into a slug. Unhandled exceptions get
+`error: true` with NO slug automatically: that absence is the standing
+"instrument this rescue" query, so don't fake slugs for paths you didn't
+handle.
+
+## Naming conventions
+
+- Flat keys, dot namespaces, snake_case leaves: `report.page_count`, not `reportPageCount` or nested hashes.
+- Durations end in `_duration_ms` (float milliseconds). Counts end in `_count`.
+- Booleans read as assertions: `report.cached`, not `report.cache_status`.
+- Timestamps are RFC 3339 strings (pass a Time; the gem serializes it).
+- Sizes and counts over raw content: `note_chars: 2140`, never the note text.
+
+## PII policy
+
+Opaque ids (user id, account id) are fine. Names, emails, free-text user
+input, and request params are not. If an attribute could quote user input,
+mark it `pii: review` in the registry and expect it to be scrubbed or
+excluded from forwarding.
+
+## The registry is the contract
+
+Every attribute you set must be declared in `config/wide_event/registry.yml`
+in the same change:
+
+```yaml
+report.format:
+  type: string
+  set_by: ReportsController#create
+  pii: none
+  notes: pdf or csv
+```
+
+Keys containing `*` are globs for dynamic families (`feature_flag.*`). With
+`config.strict = true` in the test environment, the suite records undeclared
+attributes; `assert_registered_wide_event_attributes` fails on them. Run
+`rake wide_events:registry:docs` if this app generates the human-readable
+registry doc, and never rename a shipped attribute without a reason -
+renames orphan historical data.
+
+## Prove it in tests
+
+```ruby
+require "wide_event/test_helper"   # include WideEvent::TestHelper
+
+test "report rendering is instrumented" do
+  assert_wide_event("pdf_render.duration_ms", "report.format" => "pdf") do
+    Report.new(format: "pdf").render
+  end
+end
+```
+
+For code that runs its own unit of work (jobs, full request dispatch), use
+`capture_wide_events { ... }` and assert on the returned events.
+
+## Checklist for any instrumentation change
+
+1. Attributes follow the naming conventions above.
+2. New rescue blocks that swallow errors call `error!` with a unique static slug.
+3. Every new attribute has a registry entry in the same change.
+4. A test asserts the attribute is set (`assert_wide_event`).
+5. No PII beyond opaque ids; anything questionable is flagged `pii: review`.