ruby_llm-monitoring 0.1.0

data/app/controllers/ruby_llm/monitoring/alerts_controller.rb

@@ -0,0 +1,7 @@
+module RubyLLM::Monitoring
+  class AlertsController < ApplicationController
+    def index
+      @alert_rules = RubyLLM::Monitoring.alert_rules
+    end
+  end
+end

data/app/controllers/ruby_llm/monitoring/application_controller.rb

@@ -0,0 +1,12 @@
+module RubyLLM
+  module Monitoring
+    class ApplicationController < ActionController::Base
+      private
+
+      def filter_param
+        {}
+      end
+      helper_method :filter_param
+    end
+  end
+end

data/app/controllers/ruby_llm/monitoring/metrics_controller.rb

@@ -0,0 +1,91 @@
+module RubyLLM::Monitoring
+  class MetricsController < ApplicationController
+    before_action :set_resolution
+    before_action :set_time_range
+
+    def index
+      @events = Event.group(:provider, :model)
+        .group_by_minute(:created_at, range: @time_range, n: @resolution.in_minutes.to_i)
+
+      error_count_by_time = @events.where.not(exception_class: nil).size
+
+      @metrics = [
+        build_metric_series(title: "Throughput", data: aggregate_metric(@events.count)),
+        build_metric_series(title: "Cost", data: aggregate_metric(@events.sum(:cost)), unit: "money"),
+        build_metric_series(
+          title: "Response time",
+          data: aggregate_metric(@events.average(:duration), default_value: 0),
+          unit: "ms"
+        ),
+        build_metric_series(
+          title: "Errors",
+          data: aggregate_metric(error_count_by_time, default_value: 0),
+          unit: "number"
+        )
+      ]
+
+      @totals_by_provider = Event.where(created_at: @time_range)
+        .group(:provider, :model)
+        .select(
+          :provider,
+          :model,
+          "COUNT(*) as requests",
+          "SUM(cost) as cost",
+          "AVG(duration) as avg_response_time"
+        ).to_a
+
+      total_requests = @totals_by_provider.sum(&:requests)
+      error_count = error_count_by_time.values.sum
+
+      @totals = {
+        requests: total_requests,
+        cost: @totals_by_provider.sum { |r| r.cost.to_f },
+        avg_response_time: @totals_by_provider.any? ? @totals_by_provider.sum do |r|
+          r.avg_response_time.to_f * r.requests
+        end / total_requests : nil,
+        error_rate: total_requests.positive? ? (error_count.to_f / total_requests * 100).round(1) : 0
+      }
+    end
+
+    private
+
+    def aggregate_metric(aggregated_data, default_value: nil)
+      aggregated_data
+        .group_by { |(provider, model, _), _| [ provider, model ] }
+        .transform_values do |entries|
+          entries.map do |(_, _, timestamp), value|
+            [ timestamp.to_i * 1000, value || default_value ]
+          end
+        end
+    end
+
+    def build_metric_series(title:, data:, unit: nil)
+      {
+        title: title,
+        unit: unit,
+        series: data.map { |k, v| { name: k.join("/"), data: v } }
+      }.compact
+    end
+
+    def filter_param
+      {
+        filter: {
+          created_at_start: @created_at_start,
+          created_at_end: @created_at_end,
+          resolution: @resolution
+        }
+      }.compact
+    end
+
+    def set_resolution
+      @resolution = params.dig(:filter, :resolution).try(:to_i).try(:minutes) || 1.minute
+    end
+
+    def set_time_range
+      @created_at_start = params.dig(:filter, :created_at_start).try(:in_time_zone) || 2.hours.ago
+      @created_at_end = params.dig(:filter, :created_at_end).try(:in_time_zone)
+
+      @time_range = @created_at_start..@created_at_end
+    end
+  end
+end

data/app/helpers/ruby_llm/monitoring/alerts_helper.rb

@@ -0,0 +1,4 @@
+module RubyLLM::Monitoring
+  module AlertsHelper
+  end
+end

data/app/helpers/ruby_llm/monitoring/application_helper.rb

@@ -0,0 +1,6 @@
+module RubyLLM
+  module Monitoring
+    module ApplicationHelper
+    end
+  end
+end

data/app/helpers/ruby_llm/monitoring/metrics_helper.rb

@@ -0,0 +1,7 @@
+module RubyLLM::Monitoring
+  module MetricsHelper
+    def resolution_options
+      { "1 minute" => 1, "10 minutes" => 10, "1 hour" => 60 }
+    end
+  end
+end

data/app/javascript/ruby_llm/monitoring/application.js

@@ -0,0 +1,5 @@
+import "@hotwired/turbo-rails"
+import "controllers"
+
+import ApexCharts from "apexcharts"
+window.ApexCharts = ApexCharts

data/app/javascript/ruby_llm/monitoring/controllers/application.js

@@ -0,0 +1,11 @@
+import { Application } from "@hotwired/stimulus"
+import AutoSubmit from "@stimulus-components/auto-submit"
+
+const application = Application.start()
+application.register("auto-submit", AutoSubmit)
+
+// Configure Stimulus development experience
+application.debug = false
+window.Stimulus   = application
+
+export { application }

data/app/javascript/ruby_llm/monitoring/controllers/chart_controller.js

@@ -0,0 +1,93 @@
+import { Controller } from "@hotwired/stimulus"
+
+export default class extends Controller {
+  initialize() {
+    this.title = this.element.dataset.title
+    this.min = parseInt(this.element.dataset.min)
+    this.series = JSON.parse(this.element.dataset.series)
+    this.formatter = this.element.dataset.formatter
+  }
+
+  connect() {
+    this.chart = new ApexCharts(this.element, {
+      chart: {
+        id: this.title,
+        group: "metrics",
+        height: "200px",
+        type: "area",
+        toolbar: {
+          tools: {
+            download: false
+          }
+        },
+        events: {
+          rendered: () => this.syncAll()
+        }
+      },
+      colors: ["#3b82f6", "#4ade80", "#facc15", "#dc2626", "#7c3aed"],
+      dataLabels: {
+        enabled: false
+      },
+      onDatasetHover: {
+        highlightDataSeries: true,
+      },
+      series: this.series,
+      stroke: {
+        width: 1
+      },
+      title: {
+        text: this.title
+      },
+      tooltip: {
+        shared: true,
+        x: {
+          format: "dd/MM/yyyy HH:mm:ss"
+        },
+        y: {
+          formatter: this.formatterFunction()
+        }
+      },
+      xaxis: {
+        type: "datetime",
+        min: this.min
+      },
+      yaxis: {
+        labels: { show: false }
+      }
+    })
+
+    this.chart.render()
+  }
+
+  syncAll() {
+    if (typeof ApexCharts !== "undefined") {
+      ApexCharts.exec("metrics", "syncAll")
+    }
+  }
+
+  formatterFunction() {
+    let f = function(value) {
+      return value
+    }
+
+    if (this.formatter === "percentage") {
+      f = function(value) {
+        return `${value || 0}%`
+      }
+    }
+
+    if (this.formatter === "ms") {
+      f = function(value) {
+        return `${value.toFixed(2)} ms`
+      }
+    }
+
+    if (this.formatter === "money") {
+      f = function(value) {
+        return `$${value}`
+      }
+    }
+
+    return f
+  }
+}

data/app/javascript/ruby_llm/monitoring/controllers/index.js

@@ -0,0 +1,4 @@
+import { eagerLoadControllersFrom } from "@hotwired/stimulus-loading"
+import { application } from "controllers/application"
+
+eagerLoadControllersFrom("controllers", application)

data/app/jobs/ruby_llm/monitoring/application_job.rb

@@ -0,0 +1,6 @@
+module RubyLLM
+  module Monitoring
+    class ApplicationJob < ActiveJob::Base
+    end
+  end
+end

data/app/mailers/ruby_llm/monitoring/alert_mailer.rb

@@ -0,0 +1,13 @@
+module RubyLLM::Monitoring
+  class AlertMailer < ApplicationMailer
+    def alert_notification
+      @body = params[:body]
+
+      mail(
+        to: params[:to],
+        from: params[:from] || default_params[:from],
+        subject: params[:subject]
+      )
+    end
+  end
+end

data/app/mailers/ruby_llm/monitoring/application_mailer.rb

@@ -0,0 +1,8 @@
+module RubyLLM
+  module Monitoring
+    class ApplicationMailer < ActionMailer::Base
+      default from: "from@example.com"
+      layout "mailer"
+    end
+  end
+end

data/app/models/concerns/ruby_llm/monitoring/alertable.rb

@@ -0,0 +1,53 @@
+module RubyLLM
+  module Monitoring
+    module Alertable
+      extend ActiveSupport::Concern
+
+      included do
+        after_create_commit :evaluate_alert_rules
+      end
+
+      private
+
+      def cooldown(rule_id, custom_cooldown)
+        ttl = custom_cooldown || RubyLLM::Monitoring.alert_cooldown
+
+        Rails.cache.write("ruby_llm-monitoring/#{rule_id}", Time.current + ttl, expires_in: ttl)
+      end
+
+      def cooling_down?(rule_id)
+        Rails.cache.exist?("ruby_llm-monitoring/#{rule_id}")
+      end
+
+      def dispatch_to_channels(alert_rule)
+        alert_rule[:channels].each do |channel_name|
+          channel = RubyLLM::Monitoring.channel_registry.fetch(channel_name)
+          channel_config = RubyLLM::Monitoring.channels[channel_name] || {}
+          channel.deliver(alert_rule[:message], channel_config)
+        rescue => e
+          Rails.logger.error "[RubyLLM::Monitoring] Failed to deliver alert to #{channel_name}: #{e.message}"
+        end
+      end
+
+      def evaluate_alert_rules
+        RubyLLM::Monitoring.alert_rules.each_with_index do |alert_rule, index|
+          rule_id = "rule_#{index}"
+
+          next unless valid_alert_rule?(alert_rule)
+          next if cooling_down?(rule_id)
+
+          events = self.class.where(created_at: alert_rule[:time_range].call)
+
+          if alert_rule[:rule].call(events)
+            cooldown(rule_id, alert_rule[:cooldown])
+            dispatch_to_channels(alert_rule)
+          end
+        end
+      end
+
+      def valid_alert_rule?(alert_rule)
+        alert_rule[:time_range].respond_to?(:call) && alert_rule[:rule].respond_to?(:call) && alert_rule[:channels].is_a?(Array) && alert_rule[:message].is_a?(Hash)
+      end
+    end
+  end
+end

data/app/models/ruby_llm/monitoring/application_record.rb

@@ -0,0 +1,7 @@
+module RubyLLM
+  module Monitoring
+    class ApplicationRecord < ::ActiveRecord::Base
+      self.abstract_class = true
+    end
+  end
+end

data/app/models/ruby_llm/monitoring/event.rb

@@ -0,0 +1,22 @@
+module RubyLLM::Monitoring
+  class Event < ApplicationRecord
+    include Alertable
+
+    before_validation :set_cost
+
+    private
+
+    def set_cost
+      model, provider = RubyLLM.models.resolve payload["model"], provider: payload["provider"]
+
+      self.cost = if provider.local? || [ payload["input_tokens"], payload["output_tokens"] ].all?(nil)
+        0.0
+      else
+        input_cost = payload["input_tokens"] / 1_000_000.0 * model.input_price_per_million
+        output_cost = payload["output_tokens"] / 1_000_000.0 * model.output_price_per_million
+
+        input_cost + output_cost
+      end
+    end
+  end
+end

data/app/views/layouts/ruby_llm/monitoring/application.html.erb

@@ -0,0 +1,29 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>RubyLLM Monitoring - <%= content_for :title %></title>
+  <%= csrf_meta_tags %>
+  <%= csp_meta_tag %>
+
+  <meta name="viewport" content="width=device-width,initial-scale=1">
+  <meta name="turbo-cache-control" content="no-cache">
+
+  <%= yield :head %>
+
+  <%= stylesheet_link_tag "ruby_llm/monitoring/bulma.min" %>
+  <%= stylesheet_link_tag "ruby_llm/monitoring/application", media: "all" %>
+  <%= javascript_importmap_tags importmap: RubyLLM::Monitoring.importmap %>
+</head>
+<body>
+
+  <section class="section">
+    <div class="container">
+      <%= render "ruby_llm/monitoring/application/flashes" %>
+      <%= render "ruby_llm/monitoring/application/nav" %>
+      <%= render "ruby_llm/monitoring/application/tabs" %>
+      <%= yield %>
+    </div>
+  </section>
+
+</body>
+</html>

data/app/views/layouts/ruby_llm/monitoring/mailer.html.erb

@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+    <style>
+      /* Email styles need to be inline */
+    </style>
+  </head>
+
+  <body>
+    <%= yield %>
+  </body>
+</html>

data/app/views/layouts/ruby_llm/monitoring/mailer.text.erb

@@ -0,0 +1 @@
+<%= yield %>

data/app/views/ruby_llm/monitoring/alert_mailer/alert_notification.html.erb

@@ -0,0 +1 @@
+<p><%= @body %></p>

data/app/views/ruby_llm/monitoring/alert_mailer/alert_notification.text.erb

@@ -0,0 +1 @@
+<%= @body %>

data/app/views/ruby_llm/monitoring/alerts/_alert.html.erb

@@ -0,0 +1,2 @@
+<div id="<%= dom_id alert %>">
+</div>

data/app/views/ruby_llm/monitoring/alerts/index.html.erb

@@ -0,0 +1,26 @@
+<% content_for :title, "Alerts" %>
+
+<% if @alert_rules.any? %>
+  <table class="table is-hoverable is-fullwidth">
+    <thead>
+      <tr>
+        <th>Time range</th>
+        <th>Channels</th>
+        <th>Message</th>
+        <th>Status</th>
+      </tr>
+    </thead>
+    <tbody>
+      <% @alert_rules.each do |alert_rule| %>
+        <tr>
+          <td><%= time_range = alert_rule[:time_range].call %></td>
+          <td><%= alert_rule[:channels].to_sentence %></td>
+          <td><%= alert_rule[:message][:text] %></td>
+          <td><%= alert_rule[:rule].call(RubyLLM::Monitoring::Event.where(created_at: time_range)) ? "🟢" : "🔴" %></td>
+        </tr>
+      <% end %>
+    </tbody>
+  </table>
+<% else %>
+  <div class="mt-6 has-text-centered is-size-4 has-text-grey">There are no alert rules</div>
+<% end %>

data/app/views/ruby_llm/monitoring/application/_flashes.html.erb

@@ -0,0 +1,9 @@
+<% flash.each do |name, message| -%>
+  <% message_class = { notice: "is-success", alert: "is-danger" }[name.to_sym] %>
+
+  <article class="message <%= message_class %>">
+    <div class="message-body">
+      <%= message %>
+    </div>
+  </article>
+<% end -%>

data/app/views/ruby_llm/monitoring/application/_nav.html.erb

@@ -0,0 +1,12 @@
+<nav class="navbar" role="navigation" aria-label="main navigation">
+  <div class="navbar-menu is-active mb-4">
+    <div class="navbar-start">
+      <% if defined?(main_app.root_path) %>
+        <%= link_to "Back to main app", main_app.root_path %>
+      <% end %>
+    </div>
+
+    <div class="navbar-end">
+    </div>
+  </div>
+</nav>

data/app/views/ruby_llm/monitoring/application/_pagination.html.erb

@@ -0,0 +1,7 @@
+<nav class="buttons is-right" role="navigation" aria-label="pagination">
+  <span class="mr-3"><%= page.index %> / <%= page.pages_count || "..." %></span>
+  <%= link_to "↞", url_for(page: 1), class: "pagination-previous", disabled: page.first? %>
+  <%= link_to "Previous page", url_for(page: page.previous_index), class: "pagination-previous", disabled: page.first? %>
+  <%= link_to "Next page", url_for(page: page.next_index), class: "pagination-next", disabled: page.last? %>
+  <%= link_to "↠", url_for(page: page.pages_count), class: "pagination-next", disabled: page.last? %>
+</nav>

data/app/views/ruby_llm/monitoring/application/_tabs.html.erb

@@ -0,0 +1,6 @@
+<div class="tabs is-boxed">
+  <ul>
+    <li class="<%= "is-active" if controller_name == "metrics" %>"><%= link_to "Metrics", root_path %></li>
+    <li class="<%= "is-active" if controller_name == "alerts" %>"><%= link_to "Alerts", alerts_path %></li>
+  </ul>
+</div>

data/app/views/ruby_llm/monitoring/metrics/_filters.html.erb

@@ -0,0 +1,25 @@
+<div class="level">
+  <div class="level-left">
+    <%= form_with scope: :filter, url: metrics_path, method: :get, data: {controller: "auto-submit"} do |form| %>
+      <div class="field is-grouped">
+        <div>
+          <%= form.label :created_at_start, class: "label" %>
+          <%= form.datetime_field :created_at_start, value: filter_param.dig(:filter, :created_at_start), class: "input", data: {action: "input->auto-submit#submit"} %>
+        </div>
+        <div>
+          <%= form.label :created_at_end, class: "label" %>
+          <%= form.datetime_field :created_at_end, value: filter_param.dig(:filter, :created_at_end), class: "input", data: {action: "input->auto-submit#submit"} %>
+        </div>
+        <div>
+          <%= form.label :resolution, class: "label" %>
+          <div class="select">
+            <%= form.select :resolution, resolution_options, {selected: filter_param.dig(:filter, :resolution).in_minutes.to_i}, data: {action: "auto-submit#submit"} %>
+          </div>
+        </div>
+        <div class="is-align-self-flex-end">
+          <%= link_to "Clear", clear_path, class: "button" %>
+        </div>
+      </div>
+    <% end %>
+  </div>
+</div>

data/app/views/ruby_llm/monitoring/metrics/_totals.html.erb

@@ -0,0 +1,51 @@
+<div class="columns is-multiline mb-5">
+  <div class="column is-3">
+    <div class="box has-text-centered">
+      <p class="heading">Requests</p>
+      <p class="title"><%= number_with_delimiter(@totals[:requests]) %></p>
+    </div>
+  </div>
+  <div class="column is-3">
+    <div class="box has-text-centered">
+      <p class="heading">Cost</p>
+      <p class="title"><%= number_to_currency(@totals[:cost]) %></p>
+    </div>
+  </div>
+  <div class="column is-3">
+    <div class="box has-text-centered">
+      <p class="heading">Avg Response Time</p>
+      <p class="title"><%= @totals[:avg_response_time] ? "#{@totals[:avg_response_time].round(1)}ms" : "N/A" %></p>
+    </div>
+  </div>
+  <div class="column is-3">
+    <div class="box has-text-centered">
+      <p class="heading">Error Rate</p>
+      <p class="title"><%= @totals[:error_rate] %>%</p>
+    </div>
+  </div>
+</div>
+
+<% if @totals_by_provider.any? %>
+  <div class="box mb-5">
+    <table class="table is-fullwidth is-striped">
+      <thead>
+        <tr>
+          <th>Provider/Model</th>
+          <th class="has-text-right">Requests</th>
+          <th class="has-text-right">Cost</th>
+          <th class="has-text-right">Avg Response Time</th>
+        </tr>
+      </thead>
+      <tbody>
+        <% @totals_by_provider.each do |row| %>
+          <tr>
+            <td><%= row.provider %>/<%= row.model %></td>
+            <td class="has-text-right"><%= number_with_delimiter(row.requests) %></td>
+            <td class="has-text-right"><%= number_to_currency(row.cost || 0) %></td>
+            <td class="has-text-right"><%= row.avg_response_time ? "#{row.avg_response_time.round(1)}ms" : "N/A" %></td>
+          </tr>
+        <% end %>
+      </tbody>
+    </table>
+  </div>
+<% end %>

data/app/views/ruby_llm/monitoring/metrics/index.html.erb

@@ -0,0 +1,16 @@
+<% content_for :title, "Metrics" %>
+
+<%= render "filters", clear_path: metrics_path %>
+
+<%= render "totals" %>
+
+<% @metrics.each do |metric| %>
+  <div
+    data-controller="chart"
+    data-title="<%= metric[:title] %>"
+    data-min="<%= @created_at_start.to_i.in_milliseconds %>"
+    data-formatter="<%= metric[:unit] %>"
+    data-series="<%= metric[:series].to_json %>"
+  >
+  </div>
+<% end %>

data/config/importmap.rb

@@ -0,0 +1,8 @@
+pin "@hotwired/turbo-rails", to: "turbo.min.js", preload: true
+pin "@hotwired/stimulus", to: "stimulus.min.js", preload: true
+pin "@hotwired/stimulus-loading", to: "stimulus-loading.js", preload: true
+pin "@stimulus-components/auto-submit", to: "https://ga.jspm.io/npm:@stimulus-components/auto-submit@6.0.0/dist/stimulus-auto-submit.mjs"
+pin "apexcharts", to: "https://ga.jspm.io/npm:apexcharts@4.3.0/dist/apexcharts.common.js"
+
+pin "application", to: "ruby_llm/monitoring/application.js", preload: true
+pin_all_from RubyLLM::Monitoring::Engine.root.join("app/javascript/ruby_llm/monitoring/controllers"), under: "controllers", to: "ruby_llm/monitoring/controllers"

data/config/routes.rb

@@ -0,0 +1,6 @@
+RubyLLM::Monitoring::Engine.routes.draw do
+  resources :alerts, only: :index
+  resources :metrics, only: :index
+
+  root to: "metrics#index"
+end