rails_observatory 0.1.0

data/app/views/rails_observatory/errors/show.html.erb

@@ -0,0 +1,193 @@
+<% content_for(:title) do %>
+  <%= @error.class_name %>
+  <span class="_location">
+    in <%= @error.location.classify %>
+  </span>
+<% end %>
+<% content_for(:hide_duration, 'true') %>
+<% content_for(:main_css_class, 'layout-two-column') %>
+
+<style>
+
+
+  ._location {
+    font-weight: 200;
+  }
+
+
+  .hll {
+    background-color: #49483e;
+    /*text-underline-style: wave;*/
+    /*text-decoration: underline;*/
+
+    & pre > span {
+      text-decoration-line: underline;
+      text-decoration-style: wavy;
+      text-decoration-color: var(--red);
+    }
+  }
+
+  .highlight {
+    /*background-color: var(--surface-card);*/
+    width: 100%;
+    overflow: auto;
+
+    & table {
+      border-spacing: 0;
+      width: 100%;
+    }
+
+    & tr td:first-child {
+      padding-inline: .25rem;
+      position: sticky;
+      left: 0;
+      background-color: #161b22;
+
+      .hll & {
+        background-color: #49483e;
+      }
+    }
+  }
+
+  .backtrace-line {
+    padding: .5rem 1rem;
+    display: flex;
+    flex-direction: column;
+    border-top: 1px solid var(--divider);
+    gap: 1rem;
+    color: var(--black-secondary);
+
+    &.--application-frame {
+      & summary::after {
+        content: 'Application Frame';
+        font-size: .75rem;
+        text-align: right;
+        color: color-mix(in oklab, #0c8be8 5%, var(--white));
+        padding: .1rem .35rem;
+        border-radius: .35rem;
+        font-weight: 400;
+        /* text gradient fill, blue to white */
+        background: color-mix(in oklab, #0c8be8 80%, var(--black));
+        border: 1px solid color-mix(in oklab, #0c8be8 60%, var(--white));
+        /*background-clip: text;*/
+
+      }
+    }
+
+    & > :not(summary) {
+      margin-top: 1rem;
+    }
+
+    summary {
+      display: flex;
+      align-items: center;
+      gap: .5rem;
+      color: var(--white);
+
+      & > :not(svg) {
+        flex-grow: 1;
+      }
+
+      & > svg {
+        flex-shrink: 0;
+      }
+    }
+
+    ._path {
+      white-space: nowrap;
+      overflow: hidden;
+      text-overflow: ellipsis;
+    }
+
+
+  }
+
+  .framework-trace-chunk {
+    border-top: 1px solid var(--divider);
+    font-style: italic;
+    color: var(--black-secondary);
+
+
+    &[open] {
+
+      color: color-mix(in oklab, var(--black-secondary) 40%, var(--white));
+    }
+
+    & > summary {
+      &:hover {
+        /*background-color: var(--surface-active);*/
+      }
+
+      padding: .5rem 1rem;
+      display: flex;
+      gap: .5rem;
+    }
+  }
+</style>
+
+<div class="layout-two-column-side-panel side-panel">
+  <dl>
+    <dt>Occurrences</dt>
+    <dd class="_occurrence_count"><%= @count %></dd>
+    <dt>Last Seen</dt>
+    <dd title="<%= @error.time %>"><%= time_ago_in_words(@error.time) %> ago</dd>
+
+    <dt>Past 24 Hours</dt>
+    <dd>
+      <%= render "sparkline", type: "bar", series: @past_24_hours %>
+    </dd>
+
+    <dt>Past 7 Days</dt>
+    <dd>
+      <%= render "sparkline", type: "bar", series: @past_7_days %>
+    </dd>
+
+    <dt>Past 30 Days</dt>
+    <dd><%= render "sparkline", type: "bar", series: @past_30_days %></dd>
+  </dl>
+</div>
+
+<div class="layout-two-column-main">
+  <div style="padding: 1rem;padding-top:0;padding-bottom:2rem;">
+  <h2><%= @error.message %></h2>
+  <% @error.causes.each do |cause| %>
+    <div style="color: var(--black-secondary);text-transform: uppercase; font-weight: var(--font-weight-bold);">caused by:</div>
+    <h2><%= cause["class_name"] %></h2>
+    <%= cause['message'] %>
+  <% end %>
+  </div>
+
+  <div class="card">
+    <h2>Trace</h2>
+    <% @error.trace.slice_when { _1["is_application_trace"] != _2["is_application_trace"] }.each do |trace_chunk| %>
+      <% if !trace_chunk.first["is_application_trace"] %>
+        <details class="framework-trace-chunk">
+          <summary>
+            <svg class="arrow-right" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" width="16">
+              <path stroke-linecap="round" stroke-linejoin="round" d="M8.25 4.5l7.5 7.5-7.5 7.5" />
+            </svg>
+            Plus <%= trace_chunk.length %> Framework frames
+          </summary>
+      <% end %>
+      <% trace_chunk.each do |trace_line| %>
+        <details class="backtrace-line <%= '--application-frame' if trace_line["is_application_trace"] %>" <%= 'open' if trace_line["is_application_trace"] %>>
+          <summary>
+            <svg class="arrow-right" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" width="16">
+              <path stroke-linecap="round" stroke-linejoin="round" d="M8.25 4.5l7.5 7.5-7.5 7.5" />
+            </svg>
+            <span class="_path"><%= trace_line["trace"] %></span>
+          </summary>
+          <div class="highlight">
+            <%#= trace_line %>
+            <%== highlight_source_extract(@error.source_extracts[trace_line["id"]]) %>
+          </div>
+        </details>
+      <% end %>
+      <% if !trace_chunk.first["is_application_trace"] %>
+        </details>
+      <% end %>
+    <% end %>
+  </div>
+
+
+</div>

data/app/views/rails_observatory/jobs/_table_chart.html.erb

@@ -0,0 +1,29 @@
+<% hashes = metrics.map { |name, agg| RailsObservatory::TimeSeries.where(name:).group(group_by).send(agg) } %>
+
+<section>
+  <h2><%= name %></h2>
+
+  <div class="table-chart">
+    <table>
+      <thead>
+      <tr>
+        <th><%= group_by %></th>
+        <% metrics.keys.each do |k| %>
+          <th><%= k %></th>
+          <% end %>
+      </tr>
+      </thead>
+      <tbody>
+      <% hashes.first.each do |label, val| %>
+        <tr>
+          <td><%= label %></td>
+          <td><%= val %></td>
+          <% hashes.slice(1..).each do |h| %>
+            <td><%= h[label] %></td>
+          <% end %>
+        </tr>
+      <% end %>
+      </tbody>
+    </table>
+  </div>
+</section>

data/app/views/rails_observatory/jobs/index.html.erb

@@ -0,0 +1,20 @@
+<% content_for(:title, 'Jobs') %>
+
+<%#= render 'chart', name: 'Latency', series: @latency_series, type: 'area', aggregate_using: :avg %>
+<div style="padding: 2rem;">
+<%= render 'chart', name: 'Performed', series: series_for(name: 'job.count', aggregate_using: :sum), type: 'bar' %>
+</div>
+
+<%= render 'table_chart', name: 'By Queue', group_by: :queue_name, metrics: { "job.count": :sum, "job.queue_latency": :last } %>
+<%= render 'table_chart', name: 'By Job', group_by: :job_class, metrics: { "job.count": :sum, latency: :avg, "job.error_count": :sum } %>
+
+<section>
+  <h2>Recent Jobs</h2>
+  <%= render 'events_table',
+             events: @recent_jobs,
+             fields: [:time, :job_id, :job_class, :duration],
+             formatters: {
+               time: ->(time, event) { link_to Time.at(time).to_fs('%H:%M:%S'), job_path(event.job_id) },
+             }
+  %>
+</section>

data/app/views/rails_observatory/jobs/show.html.erb

@@ -0,0 +1,8 @@
+<% content_for :title do %>
+  <%= @job.job_class %><br/>
+  <span style="font-size: 1rem; font-weight: 200;"><%= time_ago_in_words(Time.at(@job.time)) %> ago
+    | <%= @job.duration.round(2) %>ms</span>
+<% end %>
+<% content_for(:hide_duration, 'true') %>
+
+<%= render 'trace', model: @job, events: @events %>

data/app/views/rails_observatory/logs/index.html.erb

@@ -0,0 +1,18 @@
+<% content_for(:title, 'Logs') %>
+
+<style>
+  @scope {
+    :scope {
+      display: grid;
+      gap: 2rem;
+      grid-template-columns: minmax(400px, 1fr);
+    }
+  }
+</style>
+
+<%= render 'chart', type: 'bar', series: @logs_by_type, name: 'Log Volume' %>
+
+<div class="card">
+  <h2>Recent Logs</h2>
+  <%= render 'events_table', events: @recent_logs, fields: [:timestamp, :level, :message, :request_id] %>
+</div>

data/app/views/rails_observatory/mailers/index.html.erb

@@ -0,0 +1,11 @@
+<% content_for(:title, 'Mailers') %>
+<section>
+  <h2>Email Delivered</h2>
+
+  <%= render 'events_table', events: @deliveries, fields: [
+    :time, :to, :from, :subject, :view], formatters: {
+    time: ->(time, event) { Time.at(time).to_fs('%H:%M:%S') },
+    view: ->(view, event) { link_to 'View', preview_mail_path(event.message_id) }
+  }
+  %>
+</section>

data/app/views/rails_observatory/mailers/show.html.erb

@@ -0,0 +1,10 @@
+<%= content_for :title, 'Mail' %>
+<% content_for(:hide_duration, 'true') %>
+
+<div class="card">
+  <h2>Mail Preview</h2>
+  <div>To: <%= @mail.to.join(', ') %></div>
+  <div>From: <%= @mail.from.join(', ') %></div>
+  <div>Subject: <%= @mail.subject %></div>
+  <%== @mail.body %>
+</div>

data/app/views/rails_observatory/requests/_text_gauge.html.erb

@@ -0,0 +1,4 @@
+<text-gauge>
+  <text-gauge-title><%= title %></text-gauge-title>
+  <text-gauge-value><%= value %><span style="color: var(--black-secondary);font-size:1.5rem"><%= local_assigns[:unit] %></span></text-gauge-value>
+</text-gauge>

data/app/views/rails_observatory/requests/index.html.erb

@@ -0,0 +1,56 @@
+<% content_for(:title, params[:controller_action].presence || 'Requests') %>
+<% content_for(:subtitle, 'Requests') if params[:controller_action] %>
+<% content_for(:main_css_class, 'layout-requests_index') %>
+<div class="layout-requests_index-glance" style="display:flex;justify-content: space-evenly">
+  <% request_count = series_value(name: "request.count", aggregate_using: :sum) %>
+  <% if request_count.to_i > 0 %>
+    <%= render 'text_gauge', title: 'Requests', value: series_value(name: "request.count", aggregate_using: :sum) %>
+    <%= render 'text_gauge', title: 'Throughput', value: series_value(name: "request.count", aggregate_using: :sum).fdiv(duration / 60).round(2), unit: 'rpm' %>
+  <% end %>
+</div>
+<div class="layout-requests_index-chart">
+  <%= render 'chart', name: 'Requests', type: 'bar', series: series_for(name: 'request.count', aggregate_using: :sum, action: params[:controller_action]) %>
+  <%= render 'chart', name: 'Latency', type: 'area', series: series_for(name: 'request.latency', aggregate_using: :avg, action: params[:controller_action]) %>
+
+<%#= render 'chart', name: "Latency Breakdown", series: @latency_composition, type: 'area' %>
+<%#= render 'chart', name: 'Errors', series: @errors, type: 'bar', palette: 'palette7' %>
+</div>
+<% if params[:controller_action].blank? %>
+  <section class="layout-requests_index-by_controller">
+    <h2>By Controller Action</h2>
+    <div class="table-chart ">
+      <table>
+        <thead>
+        <tr>
+          <th>Controller Action</th>
+          <th>Requests</th>
+          <th>Avg Latency</th>
+        </tr>
+        </thead>
+        <tbody>
+        <% @count_by_controller.each do |series| %>
+          <tr>
+            <td><%= link_to series.labels[:action].classify, requests_path(controller_action: series.labels[:action].to_param), class: 'table-chart-row-action' %></td>
+            <td><%= series.data.dig(0, 1) %></td>
+            <td><%= @latency_by_controller[series.labels[:action]]&.data&.dig(0, 1).to_f.round(2) %> ms</td>
+          </tr>
+        <% end %>
+
+        <!-- More people... -->
+        </tbody>
+      </table>
+    </div>
+  </section>
+<% end %>
+<section class="layout-requests_index-events">
+  <h2>Recent Requests</h2>
+
+
+  <%= render 'events_table', events: @events, fields: [
+    :time, :http_method, :action, :route_pattern, :duration, :status,
+  ], formatters: {
+    time: ->(time, event) { link_to Time.at(time).to_fs('%H:%M:%S'), request_path(event.request_id) },
+  } %>
+</section>
+
+

data/app/views/rails_observatory/requests/show.html.erb

@@ -0,0 +1,16 @@
+<% content_for :title do %>
+  <span style="font-weight:200"><%= @request.http_method %></span> <%= @request.name.classify %>
+  <svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" height="18">
+    <path stroke-linecap="round" stroke-linejoin="round" d="M17.25 8.25 21 12m0 0-3.75 3.75M21 12H3" />
+  </svg>
+  <span class="status-<%= 'success' if @request.status.to_s.start_with?('2') %>"><%= @request.status %> <%= Rack::Utils::HTTP_STATUS_CODES[@request.status] %></span>
+  <br>
+  <span style="font-size: 1rem; font-weight: 200;"><%= time_ago_in_words(Time.at(@request.time)) %> ago
+    | <%= @request.duration.round(2) %>ms
+    | <%= @request.path %></span>
+<% end %>
+<% content_for(:hide_duration, 'true') %>
+
+<% content_for(:main_css_class, 'layout-events-breakdown') %>
+<%= render 'trace', model: @request, events: @events %>
+

data/config/routes.rb

@@ -0,0 +1,7 @@
+RailsObservatory::Engine.routes.draw do
+  resources :requests, only: [:index, :show]
+  resources :jobs, only: [:index, :show]
+  resources :mailers, only: [:index, :show]
+  resources :errors, only: [:index, :show]
+  root to: "requests#index"
+end

data/lib/rails_observatory/action_mailer_subscriber.rb

@@ -0,0 +1,14 @@
+require_relative './models/mail_delivery'
+module RailsObservatory
+  class ActionMailerSubscriber < ActiveSupport::Subscriber
+    attach_to :action_mailer
+
+    def deliver(event)
+      event.payload => {mail:, mailer:, to:, from:, subject:, message_id:}
+      MailDelivery.new(mail:, mailer:, to:, from:, subject:, message_id:, time: Time.now.to_f, duration: event.duration).save
+
+      TimeSeries.record_occurrence("mailer.delivery_count", labels: {mailer:})
+    end
+
+  end
+end

data/lib/rails_observatory/engine.rb

@@ -0,0 +1,49 @@
+require 'redis-client'
+
+require_relative './action_mailer_subscriber'
+module RailsObservatory
+  class Engine < ::Rails::Engine
+    isolate_namespace RailsObservatory
+
+    # This middleware is specific to the engine and will not be included in the application stack
+    middleware.use(Rack::Static, urls: ["/assets"], root: config.root.join("public"))
+
+    config.rails_observatory = ActiveSupport::OrderedOptions.new
+    config.rails_observatory.redis = { host: "localhost", port: 6379, db: 0, pool_size: ENV["RAILS_MAX_THREADS"] || 3 }
+
+    initializer "rails_observatory.redis" do |app|
+      require_relative './redis/logging_middleware'
+      require_relative './redis/redis_client_instrumentation'
+      app.config.rails_observatory.redis => pool_size:, **redis_config
+      redis_config = RedisClient.config(**redis_config.merge(middlewares: [RedisClientInstrumentation]))
+      app.config.rails_observatory.redis = redis_config.new_pool(timeout: 0.5, size: pool_size)
+    end
+
+    initializer "rails_observatory.middleware" do |app|
+      require_relative './middleware'
+
+      # Middleware is not instrumented UNLESS there's a subscriber listening.
+      # By instantiating the collector, we ensure that the InstrumentationProxy is used
+      EventCollector.instance
+
+      app.middleware.unshift(Middleware)
+    end
+
+    initializer "rails_observatory.active_job_instrumentation" do
+      require_relative './models/job_trace'
+      ActiveSupport.on_load(:active_job) do |active_job|
+        require_relative './railties/active_job_instrumentation'
+        active_job.include(Railties::ActiveJobInstrumentation)
+      end
+    end
+
+    initializer "rails_observatory.logger" do |app|
+      require_relative './log_collector'
+      Rails.logger.broadcast_to(LogCollector.new)
+    end
+
+    initializer "rails_observatory.mailer_instrumentation" do |app|
+      config.action_mailer.preview_paths << "#{config.root}/lib/rails_observatory/mailer_previews"
+    end
+  end
+end

data/lib/rails_observatory/event_collector.rb

@@ -0,0 +1,43 @@
+module RailsObservatory
+  class EventCollector
+    include Singleton
+
+    COLLECTOR_LIST_KEY = :ro_collector
+
+    def initialize
+      @subscriber ||= ActiveSupport::Notifications.subscribe(/\A[^!]/, self)
+    end
+
+    def call(event)
+      return if ActiveSupport::IsolatedExecutionState[COLLECTOR_LIST_KEY].blank?
+
+      ActiveSupport::IsolatedExecutionState[COLLECTOR_LIST_KEY].each do |key|
+        if (events = ActiveSupport::IsolatedExecutionState[key])
+          events << event
+        end
+      end
+
+    end
+
+    def generate_collector_key
+      "collector:#{Object.new.object_id}"
+    end
+
+
+    def collect_events
+      events = []
+      key = generate_collector_key
+      ActiveSupport::IsolatedExecutionState[COLLECTOR_LIST_KEY] ||= []
+      ActiveSupport::IsolatedExecutionState[COLLECTOR_LIST_KEY] << key
+      ActiveSupport::IsolatedExecutionState[key] = events
+      yield
+      events
+    rescue Exception => e
+      e.instance_variable_set(:@_trace_events, events)
+      raise
+    ensure
+      ActiveSupport::IsolatedExecutionState[COLLECTOR_LIST_KEY].delete(key)
+      ActiveSupport::IsolatedExecutionState.delete(key)
+    end
+  end
+end

data/lib/rails_observatory/log_collector.rb

@@ -0,0 +1,46 @@
+module RailsObservatory
+  class LogCollector < ::Logger
+
+    KEY = :observatory_logs
+
+    def initialize(*args, **kwargs)
+      _, *rest = args
+      super(nil, *rest, **kwargs)
+    end
+
+    def self.collect_logs
+      logs = []
+      ActiveSupport::IsolatedExecutionState[KEY] = logs
+      yield
+      logs
+    ensure
+      ActiveSupport::IsolatedExecutionState.delete(KEY)
+    end
+
+    def add(severity, message = nil, progname = nil, &block)
+      if (logs = ActiveSupport::IsolatedExecutionState[KEY])
+        severity ||= UNKNOWN
+        return true if severity < level
+        progname = @progname if progname.nil?
+        if message.nil?
+          if block_given?
+            message = yield
+          else
+            message = progname
+            progname = @progname
+          end
+        end
+        logs << { severity:, message:, progname:, time: Time.now.to_f }
+      end
+    end
+
+    alias log add
+
+    def <<(message)
+      if (logs = ActiveSupport::IsolatedExecutionState[KEY])
+        logs << { severity: UNKNOWN, message:, progname: nil, time: Time.now.to_f }
+      end
+    end
+
+  end
+end

data/lib/rails_observatory/mailer_previews/delivered_mail_preview.rb

@@ -0,0 +1,9 @@
+# Preview all emails at http://localhost:3000/rails/mailers/new_user_mailer
+class DeliveredMailPreview < ActionMailer::Preview
+
+  def preview
+    mail_delivery = RailsObservatory::MailDelivery.find(params[:message_id])
+    Mail.new(mail_delivery.mail)
+  end
+
+end

data/lib/rails_observatory/middleware.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require "active_support/notifications"
+require_relative './event_collector'
+require_relative './models/request_trace'
+require_relative './models/error'
+require_relative './serializers/serializer'
+require 'zlib'
+require 'benchmark'
+
+module RailsObservatory
+  class Middleware
+
+    def initialize(app)
+      @app = app
+    end
+
+    def collect_events_and_logs
+      logs = []
+      events = EventCollector.instance.collect_events do
+        logs = LogCollector.collect_logs do
+          yield
+        end
+      end
+      [events, logs]
+    end
+
+    def call(env)
+      start_at = Time.now
+      request = ActionDispatch::Request.new(env)
+
+      start_at_mono = Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_millisecond)
+      response = nil
+      events, logs = collect_events_and_logs do
+        response = @app.call(env)
+      end
+
+      controller_action = "#{request.params[:controller]}##{request.params[:action]}"
+
+      if (event = events.find { _1.name == 'process_action.action_controller' && _1.payload[:exception_object] })
+        error = Error.new(exception: event.payload[:exception_object], location: controller_action, time: Time.now)
+        error.save
+        TimeSeries.record_occurrence("error.count", labels: { fingerprint: error.fingerprint })
+      end
+
+      return response if request.params[:controller].blank? || request.params[:controller] =~ /rails_observatory/
+
+      status, headers, body = response
+      body = ::Rack::BodyProxy.new(body) do
+        duration = (Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_millisecond) - start_at_mono)
+        RequestTrace.new(
+          request_id: request.request_id,
+          status:,
+          http_method: request.method,
+          route_pattern: request.route_uri_pattern,
+          action: controller_action,
+          error: events.any? { _1.payload[:exception] },
+          format: request.format,
+          duration:,
+          time: start_at.to_f,
+          path: request.path,
+          events: events.map { Serializer.serialize(_1) },
+          logs:
+        ).save
+        labels = { action: controller_action, format: request.format, status:, http_method: request.method }
+        TimeSeries.record_occurrence("request.count", labels:)
+        TimeSeries.record_occurrence("request.error_count", labels:) if status >= 500
+        TimeSeries.record_timing("request.latency", duration, labels:)
+      rescue => e
+        puts e
+        puts e.backtrace.join("\n")
+      end
+
+      [status, headers, body]
+    end
+  end
+end