builder_apm 0.2.0

data/app/views/builder_apm/slow_requests/index.html.erb

@@ -0,0 +1,21 @@
+<%= render 'builder_apm/shared/header' %>
+
+<table>
+    <thead>
+        <tr>
+        <th class="sortable" data-field="start_time">Time</th>
+        <th class="sortable" data-field="controller">Controller#Action</th>
+        <th class="sortable" data-field="status">Status</th>
+        <th class="sortable" data-field="real_duration_time">Duration (ms)</th>
+        <th class="sortable" data-field="calc_db_runtime">DB Runtime (ms)</th>
+        <th class="sortable" data-field="view_runtime">View Runtime (ms)</th>
+        </tr>
+    </thead>
+    <tbody>
+        <!-- Table content will be populated here by JavaScript -->
+    </tbody>
+</table>
+
+<%= render 'builder_apm/js/slow_requests' %>
+
+<%= render 'builder_apm/shared/footer' %>

data/app/views/builder_apm/wip/index.html.erb

@@ -0,0 +1,5 @@
+<%= render 'builder_apm/shared/header' %>
+<div class="image-container">
+    <img src="https://cdn.discordapp.com/attachments/1098245214516297798/1131953429615485020/dripster82_web_page_under_construction_funncy_image_cartoonish_49e72072-6e1e-46ec-a597-2a53bda5eac0.png" />
+</div>
+<%= render 'builder_apm/shared/footer' %>

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "builder_apm"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/builder_apm.gemspec

@@ -0,0 +1,23 @@
+require_relative 'lib/builder_apm/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "builder_apm"
+  spec.version       = BuilderApm::VERSION
+  spec.authors       = ["Paul Ketelle"]
+  spec.email         = ["paul.ketelle@builder.ai"]
+
+  spec.summary       = %q{Write a short summary, because RubyGems requires one.}
+  spec.description   = %q{Write a longer description or delete this line.}
+  # spec.homepage      = "http://www.google.com"
+  spec.license       = "MIT"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
+  spec.add_dependency 'rails', '>= 4.0', '< 8'
+  spec.add_dependency "redis", "~> 4.5"
+  
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+end

data/config/routes.rb

@@ -0,0 +1,12 @@
+BuilderApm::Engine.routes.draw do
+    root to: redirect('/builder_apm/dashboard')
+    get 'dashboard', to: 'dashboard#index'
+    get 'request_data', to: 'request_data#index', defaults: { format: 'json' }
+    get 'recent_requests', to: 'recent_requests#index'
+    get 'request_details', to: 'request_details#index'
+    get 'errors_500', to: 'error_requests#index'
+    get 'slow_requests', to: 'slow_requests#index'
+    get 'request_analysis', to: 'request_analysis#index'
+    get 'n_plus_one', to: 'n_plus_one#index'
+
+end

data/lib/builder_apm/configuration.rb

@@ -0,0 +1,15 @@
+module BuilderApm
+  class Configuration
+    attr_accessor :redis_url
+    attr_accessor :enable_controller_profiler
+    attr_accessor :enable_active_record_profiler
+    attr_accessor :enable_methods_profiler
+
+    def initialize
+      @redis_url = 'redis://localhost:6379/0'
+      @enable_controller_profiler = true
+      @enable_active_record_profiler = true
+      @enable_methods_profiler = true
+    end
+  end
+end

data/lib/builder_apm/controllers/instrumenter.rb

@@ -0,0 +1,88 @@
+module BuilderApm
+  module Controllers
+    class Instrumenter
+
+      def start
+        ActiveSupport::Notifications.subscribe 'start_processing.action_controller' do |*args|
+          event = event_from_args(*args)
+          process_start(event)
+        end
+
+        ActiveSupport::Notifications.subscribe 'process_action.action_controller' do |*args|
+          event = event_from_args(*args)
+          process_action(event)
+        end
+      end
+
+      def process_start(event)
+        return if event.payload[:controller].nil? || event.payload[:controller].start_with?("BuilderApm::")
+
+        uuid = event.payload[:headers].env['action_dispatch.request_id']
+        Thread.current[:request_id] = uuid
+      end
+
+      def process_action(event)
+        return if event.payload[:controller].start_with?("BuilderApm::")
+        
+        request_id = Thread.current[:request_id]
+        Thread.current['request_data'] = extract_data_from_event(event)
+      ensure
+        clean_up_thread_values
+      end
+
+      private
+
+      def event_from_args(*args)
+        data = args.extract_options!
+        ActiveSupport::Notifications::Event.new(*args, data)
+      end
+
+      def extract_data_from_event(event)
+        data = {
+          request_id: Thread.current[:request_id],
+          controller: event.payload[:controller],
+          action: event.payload[:action],
+          params: event.payload[:params].except(:controller, :action),
+          path: event.payload[:path],
+          format: event.payload[:format],
+          method: event.payload[:method],
+          controller_line: file_and_line_number(event.payload[:controller], event.payload[:action]),
+          status: event.payload[:status],
+          start_time: event.time, 
+          end_time: event.end, 
+          duration: event.duration, 
+          db_runtime: event.payload[:db_runtime], 
+          view_runtime: event.payload[:view_runtime],
+          stack: Thread.current[:stack]
+        }
+        if event.payload[:exception]
+          exception_class, exception_message = event.payload[:exception]
+          data[:exception_class] = exception_class
+          data[:exception_message] = exception_message
+          data[:exception_backtrace] = event.payload[:exception_object].backtrace #.select { |line| line.start_with?(Rails.root.to_s) }
+        end
+
+        data
+      end
+
+      def clean_up_thread_values
+        Thread.current[:sql_event_id] = nil
+        Thread.current[:request_id] = nil
+        Thread.current[:stack] = nil
+      end
+
+      def file_and_line_number(controller, action)
+        begin
+          controller_class = controller.constantize
+          method_info = controller_class.instance_method(action)
+          absolute_path = method_info.source_location.first
+          line_number = method_info.source_location.last
+          relative_path = Pathname.new(absolute_path).relative_path_from(Rails.root)
+          "#{relative_path}:#{line_number}"
+        rescue NameError
+          "???"
+        end
+      end
+    end
+  end
+end

data/lib/builder_apm/engine.rb

@@ -0,0 +1,17 @@
+module BuilderApm
+  class Engine < ::Rails::Engine
+    isolate_namespace BuilderApm
+
+    initializer 'builder_apm.start' do |app|
+      BuilderApm::Controllers::Instrumenter.new.start if BuilderApm.configuration.enable_controller_profiler
+      BuilderApm::Models::Instrumenter.start if BuilderApm.configuration.enable_active_record_profiler
+      BuilderApm::Methods::Instrumenter.new.start if BuilderApm.configuration.enable_methods_profiler
+    end
+
+    config.after_initialize do
+      Rails.application.routes.append do
+        mount BuilderApm::Engine => '/builder_apm'
+      end
+    end
+  end
+end

data/lib/builder_apm/methods/instrumenter.rb

@@ -0,0 +1,79 @@
+module BuilderApm
+  module Methods
+    class Instrumenter
+      def initialize(root_path: Rails.root.to_s)
+        @root_path = root_path
+        @call_times = {}
+      end
+
+      def start
+        @trace = setup_trace
+        @trace.enable
+      end
+
+      def stop
+        @trace.disable unless @trace.nil?
+      end
+
+      def setup_trace
+        me = self
+        TracePoint.new(:call, :return) do |tp|
+          me.process_trace_point(tp) if me.valid_trace_point?(tp)
+        end
+      end
+
+      def valid_trace_point?(tp)
+        !Thread.current[:request_id].nil? && tp.path.start_with?(@root_path)
+      end
+
+      def process_trace_point(tp)
+        if tp.event == :call
+          process_call_event(tp)
+        elsif tp.event == :return
+          process_return_event(tp)
+        end
+      end
+
+      private
+
+      def process_call_event(tp)
+        method_id = "#{tp.defined_class}##{tp.method_id}"
+        @call_times[method_id] = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        caller_info = caller_locations(4,1).first
+        calling_file_path = caller_info.absolute_path
+        calling_line_number = caller_info.lineno
+
+        method_call = { 
+          method: method_id, 
+          method_line: "#{tp.path.gsub(@root_path, '')}:#{tp.lineno}",
+          triggering_line: "#{calling_file_path.gsub(@root_path, '')}:#{calling_line_number}", 
+          children: [], 
+          start_time: Time.now.to_f * 1000, 
+          sql_events: [] 
+        }
+
+        (Thread.current[:stack] ||= []).push(method_call)
+      end
+
+      def process_return_event(tp)
+        method_id = "#{tp.defined_class}##{tp.method_id}"
+        
+        if @call_times.key?(method_id)
+          elapsed_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @call_times[method_id]
+          elapsed_time_in_ms = (elapsed_time * 1000).round(3)
+          @call_times.delete(method_id)
+
+          method_call = (Thread.current[:stack] ||= []).pop
+          method_call[:end_time] = Time.now.to_f * 1000
+          method_call[:duration] = elapsed_time_in_ms
+
+          if Thread.current[:stack]&.any?
+            Thread.current[:stack].last[:children].push(method_call) 
+          else 
+            Thread.current[:stack].push(method_call) 
+          end
+        end
+      end
+    end
+  end
+end

data/lib/builder_apm/middleware/timing.rb

@@ -0,0 +1,56 @@
+module BuilderApm
+  module Middleware
+    class Timing
+      def initialize(app, redis_client: BuilderApm::RedisClient.client)
+        @redis_client = redis_client
+        @app = app
+      end
+
+      def call(env)
+        request_id = env["action_dispatch.request_id"]
+        Thread.current["request_id"] = request_id
+        start_time = Time.now.to_f * 1000
+
+        begin
+          result = @app.call(env)
+        rescue => e
+          handle_exception(e, start_time, request_id)
+          raise e
+        end
+
+        end_time = Time.now.to_f * 1000
+        handle_timing(start_time, end_time, request_id)
+
+        result
+      end
+
+      private
+
+      def handle_timing(start_time, end_time, request_id)
+        duration = end_time - start_time;
+        data = Thread.current['request_data']
+
+        if data
+          data[:real_start_time] = start_time 
+          data[:real_end_time] = end_time
+          data[:real_duration_time] = end_time - start_time
+          Thread.current['request_data'] = nil
+          
+          @redis_client.zadd("builder_apm:timestamps", end_time, request_id)
+          @redis_client.set("builder_apm:Request:#{data[:request_id]}", data.to_json)
+        end
+      end
+
+      def handle_exception(e, start_time, request_id)
+        end_time = Time.now.to_f * 1000
+
+        data = Thread.current['request_data'] || {}
+        data[:exception_class] = e.class.to_s
+        data[:exception_message] = e.message
+        data[:exception_backtrace] = e.backtrace
+
+        handle_timing(start_time, end_time, request_id)
+      end
+    end
+  end
+end

data/lib/builder_apm/models/instrumenter.rb

@@ -0,0 +1,82 @@
+module BuilderApm
+  module Models
+    class Instrumenter
+      def self.start
+        new.subscribe_to_notifications
+      end
+
+      def subscribe_to_notifications
+        ActiveSupport::Notifications.subscribe('sql.active_record') { |*args| handle_sql_active_record(*args) }
+        ActiveSupport::Notifications.subscribe('instantiation.active_record') { |*args| handle_instantiation_active_record(*args) }
+      end
+
+      private
+
+      def handle_sql_active_record(*args)
+        data = args.extract_options!
+        event = ActiveSupport::Notifications::Event.new(*args, data)
+        name = event.payload[:name]
+        return if name == "SCHEMA" || Thread.current[:request_id].nil?
+
+        triggering_line = determine_triggering_line(caller)
+
+        sql_query_data = build_sql_query_data(event, triggering_line)
+        store_sql_query_data(sql_query_data)
+      end
+
+      def handle_instantiation_active_record(*args)
+        data = args.extract_options!
+        event = ActiveSupport::Notifications::Event.new(*args, data)
+
+        update_last_sql_query_data_with_instantiation_info(event)
+      end
+
+      def determine_triggering_line(call_stack)
+        app_stack = call_stack.select { |line| line.include?(Rails.root.to_s) }
+        app_stack.first.to_s.gsub(Rails.root.to_s, '')
+      end
+
+      def build_sql_query_data(event, triggering_line)
+        {
+          request_id: Thread.current[:request_id],
+          sql_id: SecureRandom.uuid,
+          sql: event.payload[:sql],
+          params: event.payload[:binds].map { |a| a.value },
+          triggering_line: triggering_line,
+          name: event.payload[:name],
+          cached: event.payload[:cached] || false,
+          start_time: event.time, 
+          end_time: event.end, 
+          duration: event.duration,
+          record_count: 0,
+          class_name: ''
+        }
+      end
+
+      def store_sql_query_data(sql_query_data)
+        Thread.current[:sql_event_id] = sql_query_data[:sql_id]
+
+        if Thread.current[:stack]&.any?
+          Thread.current[:stack].last[:sql_events].push(sql_query_data)
+        else
+          (Thread.current[:stack] ||= []).push({sql_events: [sql_query_data], children: []})
+        end
+      end
+
+      def update_last_sql_query_data_with_instantiation_info(event)
+        stack = Thread.current[:stack]
+        request_id = Thread.current[:request_id]
+
+        return if stack.nil? || stack.empty? || request_id.nil?
+
+        last_sql = Thread.current[:stack].last[:sql_events].pop
+        if last_sql[:sql_id] == Thread.current[:sql_event_id]
+          last_sql[:record_count] = event.payload[:record_count]
+          last_sql[:class_name] = event.payload[:class_name]
+        end
+        Thread.current[:stack].last[:sql_events].push(last_sql)
+        Thread.current[:sql_event_id] = nil
+      end      
+    end
+  end
+end

data/lib/builder_apm/railtie.rb

@@ -0,0 +1,9 @@
+# lib/builder_apm/railtie.rb
+module BuilderApm
+    class Railtie < ::Rails::Railtie
+      initializer 'builder_apm.insert_middleware' do |app|
+        app.config.middleware.use BuilderApm::Middleware::Timing
+      end
+    end
+  end
+  

data/lib/builder_apm/redis_client.rb

@@ -0,0 +1,11 @@
+require 'redis'
+
+module BuilderApm
+  module RedisClient
+    def self.client
+      @client ||= begin
+        Redis.new(url: BuilderApm.configuration.redis_url)
+      end
+    end
+  end
+end

data/lib/builder_apm/version.rb

@@ -0,0 +1,3 @@
+module BuilderApm
+  VERSION = "0.2.0"
+end

data/lib/builder_apm.rb

@@ -0,0 +1,22 @@
+require "builder_apm/version"
+require 'builder_apm/controllers/instrumenter'
+require 'builder_apm/models/instrumenter'
+require 'builder_apm/methods/instrumenter'
+require 'builder_apm/middleware/timing'
+require 'builder_apm/configuration'
+require 'builder_apm/engine'
+require 'builder_apm/redis_client'
+require 'builder_apm/railtie'
+
+module BuilderApm 
+  def self.configure
+    yield(configuration)
+  end
+
+  def self.configuration
+    @configuration ||= Configuration.new
+  end
+
+  class Error < StandardError; end
+  # Your code goes here...
+end

data/lib/generators/builder_apm/install_generator.rb

@@ -0,0 +1,21 @@
+module BuilderApm
+  module Generators
+    class InstallGenerator < Rails::Generators::Base
+      include Rails::Generators::Migration
+      source_root File.expand_path('templates', __dir__)
+      
+      def self.next_migration_number(dirname)
+        Time.now.utc.strftime("%Y%m%d%H%M%S")
+      end
+
+      def copy_migrations
+        migration_template "create_builder_apm_requests.rb", "db/migrate/create_builder_apm_requests.rb"
+        migration_template "create_builder_apm_sql_queries.rb", "db/migrate/create_builder_apm_sql_queries.rb"
+      end
+
+      def copy_initializer_file
+        copy_file 'builder_apm_config.rb', 'config/initializers/builder_apm.rb'
+      end
+    end
+  end
+end

data/lib/generators/builder_apm/templates/builder_apm_config.rb

@@ -0,0 +1,6 @@
+BuilderApm.configure do |config|
+  # config.redis_url = 'redis://localhost:6379/0'
+  # config.enable_controller_profiler = true
+  # config.enable_active_record_profiler = true
+  # config.enable_methods_profiler = true
+end

data/lib/generators/builder_apm/templates/create_builder_apm_requests.rb

@@ -0,0 +1,21 @@
+class CreateBuilderApmRequests < ActiveRecord::Migration[6.0]
+  def change
+    create_table :builder_apm_requests do |t|
+      t.string :request_url
+      t.string :request_method
+      t.string :controller
+      t.string :action
+      t.text :params
+      t.integer :status
+      t.string :format
+      t.string :file_and_line_number
+      t.float :duration
+      t.float :view_runtime
+      t.float :db_runtime
+      t.datetime :start_time
+      t.datetime :end_time
+
+      t.timestamps
+    end
+  end
+end

data/lib/generators/builder_apm/templates/create_builder_apm_sql_queries.rb

@@ -0,0 +1,17 @@
+class CreateBuilderApmSqlQueries < ActiveRecord::Migration[6.0]
+  def change
+    create_table :builder_apm_sql_queries do |t|
+      t.string :event_uuid
+      t.text :sql
+      t.text :params
+      t.float :duration
+      t.integer :result_count
+      t.string :class_name
+      t.boolean :rails_cached
+      t.datetime :start_time
+      t.datetime :end_time
+
+      t.timestamps
+    end
+  end
+end