activetracker 0.4.0

data/lib/active_tracker/plugin.rb

@@ -0,0 +1,4 @@
+module ActiveTracker
+  module Plugin
+  end
+end

data/lib/active_tracker/plugin/base.rb

@@ -0,0 +1,9 @@
+module ActiveTracker
+  module Plugin
+    class Base
+      def self.nav_url
+        "/" + ActiveTracker::Configuration.mountpoint + "/" + resources_name.to_s
+      end
+    end
+  end
+end

data/lib/active_tracker/plugin/exception.rb

@@ -0,0 +1,113 @@
+module ActiveTracker
+  module Plugin
+    class Exception < Base
+      def self.register
+        Rails.application.middleware.insert_after BetterErrors::Middleware, ActiveTracker::ExceptionCapturer
+
+        @@registered = true
+      end
+
+      def self.registered?
+        @@registered rescue false
+      end
+
+      def self.resources_name
+        :exceptions
+      end
+
+      def self.statistics
+        ret = []
+        @exceptions = ActiveTracker::Model.all("Exception")
+        num_exceptions = @exceptions.count
+        exceptions_last_day = @exceptions.select {|e| e.log_at >= 1.day.ago}.count
+        exceptions_last_30_minutes = @exceptions.select {|e| e.log_at >= 30.minutes.ago}.count
+
+        ret << {plugin: self, label: "Last 24 hours", value: exceptions_last_day}
+        if exceptions_last_30_minutes == 0
+          ret << {plugin: self, label: "Last 30 min", value: exceptions_last_30_minutes}
+        else
+          ret << {plugin: self, label: "Last 30 min", value: exceptions_last_30_minutes, error: true}
+        end
+
+        ret
+      end
+
+      def self.nav_svg
+        svg = <<~EOF
+          <svg width="16" height="16" viewBox="0 0 16 16" class="fill-current" xmlns="http://www.w3.org/2000/svg">
+          <path d="M13.7656 2.76562L12.1406 4.39062L12.9688 5.21875C13.2625 5.5125 13.2625 5.9875 12.9688 6.27812L12.425 6.82188C12.7937 7.6375 13 8.54375 13 9.49687C13 13.0875 10.0906 15.9969 6.5 15.9969C2.90937 15.9969 0 13.0906 0 9.5C0 5.90938 2.90937 3 6.5 3C7.45312 3 8.35938 3.20625 9.175 3.575L9.71875 3.03125C10.0125 2.7375 10.4875 2.7375 10.7781 3.03125L11.6062 3.85938L13.2312 2.23438L13.7656 2.76562ZM15.625 1.875H14.875C14.6687 1.875 14.5 2.04375 14.5 2.25C14.5 2.45625 14.6687 2.625 14.875 2.625H15.625C15.8313 2.625 16 2.45625 16 2.25C16 2.04375 15.8313 1.875 15.625 1.875ZM13.75 0C13.5437 0 13.375 0.16875 13.375 0.375V1.125C13.375 1.33125 13.5437 1.5 13.75 1.5C13.9563 1.5 14.125 1.33125 14.125 1.125V0.375C14.125 0.16875 13.9563 0 13.75 0ZM14.8094 1.71875L15.3406 1.1875C15.4875 1.04062 15.4875 0.803125 15.3406 0.65625C15.1937 0.509375 14.9563 0.509375 14.8094 0.65625L14.2781 1.1875C14.1312 1.33438 14.1312 1.57187 14.2781 1.71875C14.4281 1.86563 14.6656 1.86563 14.8094 1.71875ZM12.6906 1.71875C12.8375 1.86563 13.075 1.86563 13.2219 1.71875C13.3688 1.57187 13.3688 1.33438 13.2219 1.1875L12.6906 0.65625C12.5437 0.509375 12.3063 0.509375 12.1594 0.65625C12.0125 0.803125 12.0125 1.04062 12.1594 1.1875L12.6906 1.71875ZM14.8094 2.78125C14.6625 2.63438 14.425 2.63438 14.2781 2.78125C14.1312 2.92812 14.1312 3.16563 14.2781 3.3125L14.8094 3.84375C14.9563 3.99062 15.1937 3.99062 15.3406 3.84375C15.4875 3.69688 15.4875 3.45937 15.3406 3.3125L14.8094 2.78125ZM3.5 8.5C3.5 7.39687 4.39687 6.5 5.5 6.5C5.775 6.5 6 6.275 6 6C6 5.725 5.775 5.5 5.5 5.5C3.84687 5.5 2.5 6.84688 2.5 8.5C2.5 8.775 2.725 9 3 9C3.275 9 3.5 8.775 3.5 8.5Z"/>
+          </svg>
+        EOF
+        svg.html_safe
+      end
+
+      def self.nav_title
+        "Exceptions"
+      end
+
+      def self.filters=(value)
+        @filters = value
+      end
+
+      def self.filters
+        @filters ||= []
+      end
+
+      def self.exception_capture(class_name, message, backtrace)
+        return if filter_exception?(class_name)
+
+        tags = {
+          class_name: class_name,
+          backtrace_hash: Digest::SHA2.hexdigest(backtrace.first.to_s),
+        }
+
+        ActiveTracker::Model.find_or_create("Exception", tags:tags, data_type: "full") do |obj|
+          if obj.persisted
+            ActiveTracker::Model.delete(obj.key)
+          end
+          obj.data ||= {}
+          obj.data["count"] = (obj.data["count"] || 0) + 1
+          # Enough for most git commits to be referenced
+          # so should be fine for exception hashes within an application
+          obj.id = "E" + Digest::SHA2.hexdigest(tags.inspect)[0,8]
+          obj.expiry = 7.days
+          obj.log_at = Time.now
+
+          obj.data["backtrace"] = backtrace
+          obj.data["message"] = message
+
+          obj.data["at_requests"] ||= []
+          if ActiveTracker::Plugin::Request.registered?
+            id = ActiveTracker::Plugin::Request.current_tags[:id] rescue nil
+            obj.data["at_requests"].prepend(id) if id.present?
+            obj.data["at_requests"] = obj.data["at_requests"][0,20]
+            ActiveTracker::Plugin::Request.current_tags[:at_exceptions] ||= []
+            ActiveTracker::Plugin::Request.current_tags[:at_exceptions] << obj.id
+          end
+        end
+      end
+
+      def self.filter_exception?(class_name)
+        ActiveTracker::Plugin::Exception.filters.each do |filter|
+          if filter.is_a?(Regexp)
+            if filter.match(class_name)
+              return true
+            end
+          elsif filter.is_a?(String)
+            if class_name == filter
+              return true
+            end
+          elsif filter.is_a?(Exception)
+            if class_name == filter.class.name
+              return true
+            end
+          end
+        end
+
+        false
+      end
+
+
+    end
+  end
+end

data/lib/active_tracker/plugin/query.rb

@@ -0,0 +1,128 @@
+require 'digest'
+
+module ActiveTracker
+  module Plugin
+    class Query < Base
+      def self.register
+        ActiveSupport::Notifications.subscribe "sql.active_record" do |*args|
+          event = ActiveSupport::Notifications::Event.new(*args)
+          capture_query(event)
+        end
+
+        @@registered = true
+      end
+
+      def self.registered?
+        @@registered rescue false
+      end
+
+      def self.resources_name
+        :queries
+      end
+
+      def self.nav_svg
+        svg = <<~EOF
+        <svg aria-hidden="true" focusable="false" data-prefix="fas" data-icon="database" class="fill-current" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512" width="16" height="16"><path d="M448 73.143v45.714C448 159.143 347.667 192 224 192S0 159.143 0 118.857V73.143C0 32.857 100.333 0 224 0s224 32.857 224 73.143zM448 176v102.857C448 319.143 347.667 352 224 352S0 319.143 0 278.857V176c48.125 33.143 136.208 48.572 224 48.572S399.874 209.143 448 176zm0 160v102.857C448 479.143 347.667 512 224 512S0 479.143 0 438.857V336c48.125 33.143 136.208 48.572 224 48.572S399.874 369.143 448 336z"></path></svg>
+        EOF
+        svg.html_safe
+      end
+
+      def self.nav_title
+        "Queries"
+      end
+
+      def self.statistics
+        ret = []
+        queries = ActiveTracker::Model.all("Query")
+        queries = queries.select {|e| e.log_at >= 60.minutes.ago}
+
+        num_queries = 0
+        slow_queries = 0
+        total_duration = 0
+
+        queries.each do |query|
+          actual_query = ActiveTracker::Model.find(query.key)
+          next unless actual_query
+          slow_queries += actual_query.count if actual_query.last_duration > self.min_slow_duration_ms
+          num_queries += actual_query.count
+          total_duration += actual_query.last_duration * actual_query.count
+        end
+
+        ret << {plugin: self, label: "Queries/hour", value: num_queries}
+        if slow_queries == 0
+          ret << {plugin: self, label: "Slow queries/hour", value: slow_queries}
+        else
+          ret << {plugin: self, label: "Slow queries/hour", value: slow_queries, error: true}
+        end
+        ret << {plugin: self, label: "Avg time/query", value: "%.2fms" % (total_duration/num_queries)} if num_queries > 0
+
+        ret
+      end
+
+      def self.filters=(value)
+        @filters = value
+      end
+
+      def self.filters
+        @filters ||= ["SCHEMA", /^$/]
+      end
+
+      def self.min_slow_duration_ms=(value)
+        @min_slow_duration_ms = value
+      end
+
+      def self.min_slow_duration_ms
+        @min_slow_duration_ms ||= 100
+      end
+
+      def self.capture_query(event)
+        tags = {
+          sql: event.payload[:sql],
+          name: event.payload[:name],
+        }
+
+        return if filter_query?(tags)
+
+        ActiveTracker::Model.find_or_create("Query", tags:tags, data_type: "full") do |obj|
+          if obj.persisted
+            ActiveTracker::Model.delete(obj.key)
+          end
+          obj.data ||= {}
+          obj.data["last_duration"] = event.duration
+          obj.data["count"] = (obj.data["count"] || 0) + 1
+          # Enough for most git commits to be referenced
+          # so should be fine for SQL query hashes within an application
+          obj.id = "Q" + Digest::SHA2.hexdigest(tags.inspect)[0,8]
+          obj.expiry = 7.days
+          obj.log_at = Time.now
+
+          obj.data["at_requests"] ||= []
+          if ActiveTracker::Plugin::Request.registered?
+            id = ActiveTracker::Plugin::Request.current_tags[:id] rescue nil
+            obj.data["at_requests"].prepend(id) if id.present?
+            obj.data["at_requests"] = obj.data["at_requests"][0,20]
+            ActiveTracker::Plugin::Request.current_tags[:at_queries] ||= []
+            ActiveTracker::Plugin::Request.current_tags[:at_queries] << obj.id
+          end
+        end
+      end
+
+      def self.filter_query?(details)
+        ActiveTracker::Plugin::Query.filters.each do |filter|
+          if filter.is_a?(Regexp)
+            if filter.match(details[:sql] || "") || filter.match(details[:name] || "")
+              return true
+            end
+          else
+            if (details[:sql] || "").include?(filter) || (details[:name] || "").include?(filter)
+              return true
+            end
+          end
+        end
+
+        false
+      end
+
+    end
+  end
+end

data/lib/active_tracker/plugin/request.rb

@@ -0,0 +1,163 @@
+module ActiveTracker
+  module Plugin
+    class Request < Base
+      def self.register
+        @logger = ActiveTracker::RailsLogger.new
+        @logger.level = Rails.logger.level
+        Rails.logger.extend(ActiveSupport::Logger.broadcast(@logger))
+
+        ActiveSupport::Notifications.subscribe "start_processing.action_controller" do |event|
+          clear_context
+          @logger.reset
+          tag_current(id: SecureRandom.uuid)
+          @output = ""
+        end
+
+        ActiveSupport::Notifications.subscribe "process_action.action_controller" do |*args|
+          event = ActiveSupport::Notifications::Event.new(*args)
+          request_processed(event)
+        end
+
+        Rails.application.middleware.insert_before Rack::Sendfile, ActiveTracker::OutputCapturer
+
+        @@registered = true
+      end
+
+      def self.registered?
+        @@registered rescue false
+      end
+
+      def self.resources_name
+        :requests
+      end
+
+      def self.nav_svg
+        svg = <<~EOF
+          <svg width="16" height="16" viewBox="0 0 16 16" class="fill-current" xmlns="http://www.w3.org/2000/svg">
+          <path d="M16 1.5V10.5C16 11.3281 15.3281 12 14.5 12H13V5.5C13 4.12187 11.8781 3 10.5 3H4V1.5C4 0.671875 4.67188 0 5.5 0H14.5C15.3281 0 16 0.671875 16 1.5ZM12 5.5V14.5C12 15.3281 11.3281 16 10.5 16H1.5C0.671875 16 0 15.3281 0 14.5V5.5C0 4.67188 0.671875 4 1.5 4H10.5C11.3281 4 12 4.67188 12 5.5ZM9.875 6.375C9.875 6.16875 9.70625 6 9.5 6H2.375C2.16875 6 2 6.16875 2 6.375V8H9.875V6.375Z" />
+          </svg>
+        EOF
+        svg.html_safe
+      end
+
+      def self.nav_title
+        "Requests"
+      end
+
+      def self.statistics
+        ret = []
+        @requests = ActiveTracker::Model.all("Request")
+        @requests = @requests.select {|e| e.log_at >= 60.minutes.ago}
+
+        num_requests = @requests.count
+
+        percentage_error = 0
+        avg_milliseconds = 0
+        if num_requests > 0
+          num_errors = @requests.map {|r| r.tags[:status][0]}.select {|s| s=="4" || s=="5"}.count
+          percentage_error = num_errors / @requests.count.to_f * 100.0
+          avg_milliseconds = @requests.map {|r| r.tags[:duration].to_i}.sum / num_requests
+        end
+
+
+        ret << {plugin: self, label: "Requests/hour", value: num_requests}
+        if percentage_error < 1.0
+          ret << {plugin: self, label: "Error percentage", value: "%.1f%%" % percentage_error}
+        else
+          ret << {plugin: self, label: "Error percentage", value: "%.1f%%" % percentage_error, error: true}
+        end
+        ret << {plugin: self, label: "Avg time/request", value: "#{avg_milliseconds}ms"} if avg_milliseconds
+
+        ret
+      end
+
+      def self.filters=(value)
+        @filters = value
+      end
+
+      def self.filters
+        @filters ||= ["/#{ActiveTracker::Configuration.mountpoint}"]
+      end
+
+      def self.clear_context
+        @tags = {}
+        @redactions = []
+      end
+
+      def self.tag_current(tags = {})
+        @tags = current_tags.merge(tags)
+      end
+
+      def self.current_tags
+        @tags || {}
+      end
+
+      def self.redact(value)
+        @redactions << value
+      end
+
+      def self.app_name
+        ENV["APP_NAME"] || Rails.application.class.parent.to_s
+      end
+
+      def self.request_processed(event)
+        tag_current status: event.payload[:status]
+        tag_current duration: "#{@duration.to_i}ms"
+        tag_current url: event.payload[:path]
+        tag_current method: event.payload[:method]
+        tag_current app: app_name
+      end
+
+      def self.output_capture(output)
+        @output = output
+      end
+
+      def self.record_duration(duration)
+        return if ActiveTracker::Plugin::Request.current_tags[:url] && filter_request?(ActiveTracker::Plugin::Request.current_tags[:url])
+
+        @duration = duration
+        log = @logger.lines[0, 65535] rescue ""
+
+        _, status, duration = (@logger&.lines || "").force_encoding("UTF-8").match(/Completed (\d+) .*? in (\d+)ms/m).to_a
+        tag_current status: status
+        tag_current duration: "#{duration.to_i}ms"
+
+        log = apply_redactions(log)
+        @output = apply_redactions(@output)
+
+        ActiveTracker::Model.save("Request", {log: log, output: @output},
+          tags: ActiveTracker::Plugin::Request.current_tags,
+          data_type: "full",
+          expiry: 7.days,
+          log_at: Time.now
+        ) if ActiveTracker::Plugin::Request.current_tags.any? && ActiveTracker::Plugin::Request.current_tags[:id].present?
+
+        clear_context
+      end
+
+      def self.filter_request?(path)
+        ActiveTracker::Plugin::Request.filters.each do |filter|
+          if filter.is_a?(Regexp)
+            if filter.match(path)
+              return true
+            end
+          else
+            if path.start_with?(filter)
+              return true
+            end
+          end
+        end
+
+        false
+      end
+
+      def self.apply_redactions(value)
+        (@redactions || []).each do |redaction|
+          value = value.gsub(redaction, "[REDACTED]")
+        end
+        value
+      end
+
+    end
+  end
+end

data/lib/active_tracker/rails_logger.rb

@@ -0,0 +1,120 @@
+require "active_support/core_ext/time/conversions"
+require "active_support/core_ext/object/blank"
+require "action_dispatch/http/request"
+require "rack/body_proxy"
+
+module ActiveTracker
+  class RailsLogger
+    def initialize
+      reset
+    end
+
+    def reset
+      @lines = ""
+    end
+
+    def lines
+      @lines
+    end
+
+    def add(severity, message = nil, progname = nil)
+      severity ||= UNKNOWN
+      return true if severity < level
+
+      if message.nil?
+        if block_given?
+          message = yield
+        else
+          message = progname
+        end
+      end
+
+      @lines << "#{message}\n"
+      true
+    end
+    alias log add
+
+    def debug?; @level <= DEBUG; end
+    def debug!; self.level = DEBUG; end
+    def info?; @level <= INFO; end
+    def info!; self.level = INFO; end
+    def warn?; @level <= WARN; end
+    def warn!; self.level = WARN; end
+    def error?; @level <= ERROR; end
+    def error!; self.level = ERROR; end
+    def fatal?; @level <= FATAL; end
+    def fatal!; self.level = FATAL; end
+
+    def debug(message)
+      message = yield if block_given?
+      return if message.blank?
+      add(DEBUG, message)
+    end
+
+    def info(message = "")
+      message = yield if block_given?
+      return if message.blank?
+      add(INFO, message)
+
+      # matches = message.match(/Completed (\d+) .*? in (\d+)ms/)
+      # if matches
+      #   @status = matches[1]
+      #   @time_taken = matches[2]
+      # end
+      # @lines << message if info? && @recording
+    end
+
+    def warn(message = "")
+      message = yield if block_given?
+      return if message.blank?
+      add(WARN, message)
+    end
+
+    def error(message = "")
+      message = yield if block_given?
+      return if message.blank?
+      add(ERROR, message)
+    end
+
+    def fatal(message = "")
+      message = yield if block_given?
+      return if message.blank?
+      add(FATAL, message)
+    end
+
+    def level
+      @level
+    end
+
+    def level=(severity)
+      if severity.is_a?(Integer)
+        @level = severity
+      else
+        case severity.to_s.downcase
+        when 'debug'
+          @level = DEBUG
+        when 'info'
+          @level = INFO
+        when 'warn'
+          @level = WARN
+        when 'error'
+          @level = ERROR
+        when 'fatal'
+          @level = FATAL
+        when 'unknown'
+          @level = UNKNOWN
+        else
+          raise ArgumentError, "invalid log level: #{severity}"
+        end
+      end
+    end
+
+    def formatter
+      ActiveSupport::Logger::SimpleFormatter
+    end
+
+    def silence(*args)
+      yield self
+    end
+  end
+end