active_record-sql_analyzer 0.0.5

data/lib/active_record/sql_analyzer/analyzer.rb

@@ -0,0 +1,45 @@
+module ActiveRecord
+  module SqlAnalyzer
+    class Analyzer
+      attr_reader :options
+
+      def initialize
+        @options = {}
+      end
+
+      def [](key)
+        @options[key]
+      end
+
+      # Tables to watch for this analyzer
+      def tables(names)
+        unless names.is_a?(Array)
+          raise ArgumentError, "Names of tables must be an array"
+        end
+
+        @options[:table_regex] = /\A\s*((SELECT|DELETE).*(FROM|JOIN)|(INSERT\s+INTO|UPDATE))\s+`?(#{names.join('|')})`?/i
+      end
+
+      # Logger class to use for recording data
+      def logger(klass)
+        @options[:logger] = klass
+      end
+
+      # How to tag the data
+      def name(name)
+        if !name.is_a?(String) && name !~ /\A([a-z0-9A-Z_]+)\z/
+          raise ArgumentError, "Name for this analyzer can only contain [a-z0-9A-Z_] characters"
+        end
+
+        @options[:name] = name
+      end
+
+      def setup
+        @options[:logger_instance] ||= (@options[:logger] || RedactedLogger).new(
+          SqlAnalyzer.config[:logger_root_path],
+          @options[:name]
+        )
+      end
+    end
+  end
+end

data/lib/active_record/sql_analyzer/background_processor.rb

@@ -0,0 +1,54 @@
+require "set"
+
+module ActiveRecord
+  module SqlAnalyzer
+    class BackgroundProcessor
+      def initialize
+        @queue = Queue.new
+      end
+
+      def <<(event)
+        processor_thread
+        @queue << event
+      end
+
+      private
+
+      MUTEX = Mutex.new
+
+      def process_queue
+        event = @queue.pop
+
+        event[:caller] = SqlAnalyzer.config[:backtrace_filter_proc].call(event[:caller])
+        event[:sql] = SqlAnalyzer.config[:sql_redactor_complex_proc].call(event[:sql].dup)
+
+        logger = event.delete(:logger)
+        logger.filter_event(event)
+        logger.log(event)
+      end
+
+      def processor_thread
+        # Avoid grabbing a mutex unless we really need to
+        return if @thread && @thread.alive?
+
+        MUTEX.synchronize do
+          # Double check to avoid a race condition
+          return if @thread && @thread.alive?
+
+          @thread = Thread.new do
+            Rails.logger.info "[SQL-Analyzer] Starting background query thread id #{Thread.current.object_id} in pid #{Process.pid}"
+
+            begin
+              loop do
+                process_queue
+              end
+            rescue => ex
+              Rails.logger.warn "[SQL-Analyzer] Exception in thread #{Thread.current.object_id}: #{ex.class}, #{ex.message}"
+              Rails.logger.warn "[SQL-Analyzer] #{ex.backtrace.join(", ")}"
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_record/sql_analyzer/backtrace_filter.rb

@@ -0,0 +1,38 @@
+require "pathname"
+
+# This is a bit complex but can't be avoided since otherwise we have to log 5000000 backtrace lines
+module ActiveRecord
+  module SqlAnalyzer
+    class BacktraceFilter
+      def self.library_paths
+        @library_paths ||= begin
+          paths = Gem.path + Gem.path.map { |f| File.realpath(f) }
+          paths << "(eval):"
+          paths << RbConfig::CONFIG.fetch('libdir')
+          paths
+        end
+      end
+
+      def self.rails_root_regex
+        @rails_root ||= %r{^#{Regexp.escape(Rails.root.to_s)}}
+      end
+
+      def self.proc
+        @proc ||= Proc.new do |lines|
+          filtered = []
+          lines.each do |line|
+            unless library_paths.any? { |path| line.include?(path) }
+              if line =~ rails_root_regex
+                filtered << Pathname.new(line).relative_path_from(Rails.root).to_s
+              else
+                filtered << line
+              end
+            end
+          end
+
+          filtered
+        end
+      end
+    end
+  end
+end

data/lib/active_record/sql_analyzer/cli.rb

@@ -0,0 +1,74 @@
+module ActiveRecord
+  module SqlAnalyzer
+    class CLI
+      attr_reader :options, :processor
+
+      def initialize
+        @options = {
+          concurrency: 6
+        }
+      end
+
+      def processor
+        @processor ||= ActiveRecord::SqlAnalyzer::CLIProcessor.new(options[:concurrency])
+      end
+
+      def run
+        definition_logs = Dir["#{options[:log_dir]}/*_definitions.log*"].map do |path|
+          [File.basename(path).gsub(/_definitions\.log.*/, ""), path]
+        end
+
+        if definition_logs.empty?
+          raise ArgumentError, "Cannot find any log files in '#{options[:log_dir]}'"
+        end
+
+        # Process the definition logs
+        processor.run_definition(definition_logs)
+
+        # Process the usage logs
+        usage_logs = Dir["#{options[:log_dir]}/{#{definition_logs.map(&:first).uniq.join(",")}}.log*"].map do |path|
+          [File.basename(path).split(".", 2).first, path]
+        end
+
+        processor.run_usage(usage_logs)
+
+        processor.dump(options[:dest_dir])
+      end
+
+      def parse_options(args)
+        opts = OptionParser.new
+
+        opts.on("--log-dir [DIR]", String, "Directory that logs are in") do |val|
+          unless Dir.exist?(val)
+            raise ArgumentError, "log directory '#{val}' does not exist"
+          end
+
+          options[:log_dir] = val
+        end
+
+        opts.on("--dest-dir [DIR]", String, "Directory to dump logs to") do |val|
+          unless Dir.exist?(val)
+            raise ArgumentError, "dest directory '#{val}' does not exist"
+          end
+
+          options[:dest_dir] = val
+        end
+
+        opts.on("-c", "--concurrency", Integer, "How many threads to use for processing log files") do |val|
+          if val <= 0
+            raise ArgumentError, "Concurrency must be >0"
+          end
+
+          options[:concurrency] = val
+        end
+
+        opts.on_tail("-h", "--help") do
+          puts opts
+          exit
+        end
+
+        opts.parse(args)
+      end
+    end
+  end
+end

data/lib/active_record/sql_analyzer/cli_processor.rb

@@ -0,0 +1,122 @@
+module ActiveRecord
+  module SqlAnalyzer
+    class CLIProcessor
+      attr_reader :concurrency, :definitions
+
+      def initialize(concurrency)
+        @concurrency = concurrency
+        @definitions = {}
+      end
+
+      def self.process_queue(queue)
+        local_data = {}
+
+        while !queue.empty? do
+          prefix, path = queue.pop
+          local_data[prefix] ||= {}
+
+          File.open(path, "r") do |io|
+            while !io.eof? do
+              yield local_data[prefix], io.readline.strip
+            end
+          end
+        end
+
+        local_data
+
+      rescue => ex
+        puts "#{ex.class}: #{ex.message}"
+        puts ex.backtrace
+        raise
+      end
+
+      def run_definition(logs)
+        queue = Queue.new
+        logs.each { |l| queue << l }
+
+        # Spin up threads to start processing the queue
+        threads = concurrency.times.map do
+          Thread.new(queue) do |t_queue|
+            # Create a local copy of each definitions then merge them in
+            CLIProcessor.process_queue(t_queue) do |local_definitions, line|
+              line.strip!
+
+              unless line == ""
+                sha, event = line.split("|", 2)
+                local_definitions[sha] = JSON.parse(event)
+              end
+            end
+          end
+        end
+
+        # Merge everything
+        threads.each do |thread|
+          thread.value.each do |prefix, data|
+            definitions[prefix] ||= {}
+            definitions[prefix].merge!(data)
+          end
+        end
+      end
+
+      def run_usage(logs)
+        queue = Queue.new
+        logs.each { |l| queue << l }
+
+        # Spin up threads to start processing the queue
+        threads = concurrency.times.map do
+          Thread.new(queue) do |t_queue|
+            # Create a local copy of the usage for each SHA then merge it in at the end
+            CLIProcessor.process_queue(t_queue) do |local_usage, line|
+              line.strip!
+
+              unless line == ""
+                last_called, sha = line.split("|", 2)
+                last_called = Time.at(last_called.to_i).utc
+
+                local_usage[sha] ||= {"count" => 0}
+                local_usage[sha]["count"] += 1
+
+                if !local_usage[sha]["last_called"] || local_usage[sha]["last_called"] < last_called
+                  local_usage[sha]["last_called"] = last_called
+                end
+              end
+            end
+          end
+        end
+
+        # Merge everything
+        threads.each do |thread|
+          thread.value.each do |prefix, data|
+            definitions[prefix] ||= {}
+
+            data.each do |sha, usage|
+              definition = definitions[prefix][sha]
+              unless definition
+                puts "Undefined event '#{sha}'"
+                next
+              end
+
+              definition["count"] ||= 0
+              definition["count"] += usage["count"]
+
+              if !definition["last_called"] || definition["last_called"] < usage["last_called"]
+                definition["last_called"] = usage["last_called"]
+              end
+            end
+          end
+        end
+      end
+
+      def dump(dest_dir)
+        definitions.each do |prefix, data|
+          path = "#{dest_dir}/#{prefix}_#{Time.now.strftime("%Y-%m-%d")}.log"
+          puts "Writing logs to '#{path}' (#{data.length} queries)"
+
+          File.open(path, "w+") do |io|
+            io.write(data.to_json)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_record/sql_analyzer/compact_logger.rb

@@ -0,0 +1,31 @@
+require "digest"
+
+module ActiveRecord
+  module SqlAnalyzer
+    class CompactLogger < Logger
+      attr_reader :logged_shas, :definition_log_file
+
+      def initialize(*)
+        super
+
+        @logged_shas = Set.new
+        @definition_log_file = File.open("#{log_root}/#{log_prefix}_definitions.log", "a+")
+      end
+
+      def log(event)
+        sha = Digest::MD5.hexdigest(event.to_s)
+        unless logged_shas.include?(sha)
+          definition_log_file.puts("#{sha}|#{event.to_json}")
+          logged_shas << sha
+        end
+
+        log_file.puts("#{Time.now.to_i}|#{sha}")
+      end
+
+      def close
+        @definition_log_file.close rescue nil
+        super
+      end
+    end
+  end
+end

data/lib/active_record/sql_analyzer/configuration.rb

@@ -0,0 +1,174 @@
+module ActiveRecord
+  module SqlAnalyzer
+    class Configuration
+      attr_reader :options
+
+      def initialize
+        @options = {}
+        setup_defaults
+      end
+
+      # Setup a custom proc that filters out lines before passing them to the loggers.
+      # By default, this attempts to filter out all non-app code lines.
+      def backtrace_filter_proc(proc)
+        check_proc(proc, 1, "the backtrace lines")
+        @options[:backtrace_filter_proc] = proc
+      end
+
+      # Setup a new analyzer for monitoring tables
+      #
+      # add_analyzer(
+      #   name: 'users',
+      #   tables: %w(users permissions),
+      #   logger: ActiveRecord::SqlAnalyzer::RedactedLogger
+      # )
+      #
+      # Will setup an analyzer that looks at the tables users and permissions
+      # when it finds relevant data, it passes it through the `RedactedLogger` class.
+      # When calling the proc passed to `log_sample_proc`, it will use the name
+      # `users` to help identify it, as well as when logging to disk.
+      #
+      def add_analyzer(result)
+        analyzer = Analyzer.new
+        analyzer.name(result[:name])
+        analyzer.tables(result[:tables])
+        analyzer.logger(result[:logger])
+        analyzer.setup
+
+        @options[:analyzers] << analyzer
+        analyzer
+      end
+
+      # Root path where all logs go.
+      # Defaults to `Rails.root.join('log')`
+      def logger_root_path(path)
+        unless Dir.exist?(path)
+          raise ArgumentError, "Path '#{path}' is not a directory"
+        end
+
+        @options[:logger_root_path] = path
+      end
+
+      # Set a proc that determines whether or not to log a single event.
+      # This must be set to log anything, and controls how many SQL queries you look at.
+      #
+      # Proc.new { |name| true }
+      #
+      # Will log everything no matter what
+      #
+      # Proc.new do |name|
+      #   rand(1..100) <= 50
+      # end
+      #
+      # Will only log 50% of queries.
+      #
+      # You can hook this into something like Redis to allow dynamic control of the ratio
+      # without having to redeploy/restart your application.
+      #
+      def log_sample_proc(proc)
+        check_proc(proc, 1, "the analyzer name")
+        @options[:should_log_sample_proc] = proc
+      end
+
+      # For hooking in more complicated redactions beyond a simple find/replace.
+      def complex_sql_redactor_proc(proc)
+        check_proc(proc, 1, "the SQL statement")
+        @options[:sql_redactor_complex_proc] = proc
+      end
+
+      # Additional redactors to filter out data in SQL you don't want logged
+      def add_sql_redactors(list)
+        @options[:sql_redactors].concat(create_redactors(list))
+      end
+
+      # Backtrace redactors filter out data in the backtrace
+      # useful if you want to get rid of lines numbers
+      def add_backtrace_redactors(list)
+        @options[:backtrace_redactors].concat(create_redactors(list))
+      end
+
+      # If the first line in the backtrace matches the regex given, we switch to
+      # ambiguous tracing mode for that call where we log more of the backtrace.
+      #
+      # As an example, if you find you're only getting middleware, you could use:
+      #
+      # %r{\Aapp/middleware/query_string_sanitizer\.rb:\d+:in `call'\z}
+      #
+      # Which would log up to ambiguous_backtrace_lines (default 3) total lines,
+      # rather than the default 1.
+      def add_ambiguous_tracers(list)
+        list.each do |row|
+          unless row.is_a?(Regexp)
+            raise ArgumentError, "Tracing filters must be a Regexp to match on"
+          end
+        end
+
+        @options[:ambiguous_tracers].concat(list)
+      end
+
+      # How many total lines to log when the caller is ambiguous
+      def ambiguous_backtrace_lines(lines)
+        if !lines.is_a?(Fixnum)
+          raise ArgumentError, "Lines must be a Fixnum"
+        elsif lines <= 1
+          raise ArgumentError, "Lines cannot be <= 1"
+        end
+
+        @options[:ambiguous_backtrace_lines] = lines
+      end
+
+      def [](key)
+        @options[key]
+      end
+
+      private
+
+      def check_proc(proc, arity, msg)
+        if !proc.is_a?(Proc)
+          raise ArgumentError, "You must pass a proc"
+        elsif proc.arity != 1
+          raise ArgumentError, "Proc must accept 1 argument for #{msg}"
+        end
+      end
+
+      def create_redactors(list)
+        list.map do |redact|
+          if redact.length != 2
+            raise ArgumentError, "Redactor row should only have two entries"
+          elsif !redact.first.is_a?(Regexp)
+            raise ArgumentError, "First value in pair must be a Regexp to match on"
+          elsif !redact.last.is_a?(String)
+            raise ArgumentError, "Last value in pair must be a String to replace with"
+          end
+
+          Redactor.new(*redact)
+        end
+      end
+
+      def setup_defaults
+        @options[:sql_redactors] = [
+          Redactor.new(/\n/, " "),
+          Redactor.new(/\s+/, " "),
+          Redactor.new(/(\s|\b|`)(=|!=|>=|>|<=|<) ?(BINARY )?-?\d+(\.\d+)?/, " = [REDACTED]"),
+          Redactor.new(/(\s|\b|`)(=|!=|>=|>|<=|<) ?(BINARY )?x?'[^']*'/, " = '[REDACTED]'"),
+          Redactor.new(/VALUES \(.+\)$/, "VALUES ([REDACTED])"),
+          Redactor.new(/IN \([^)]+\)/, "IN ([REDACTED])"),
+          Redactor.new(/BETWEEN '[^']*' AND '[^']*'/, "BETWEEN '[REDACTED]' AND '[REDACTED]'"),
+          Redactor.new(/LIKE '[^'\\]*(?:\\.[^'\\]*)*'/, "LIKE '[REDACTED]'"),
+          Redactor.new(/ LIMIT \d+/, ""),
+          Redactor.new(/ OFFSET \d+/, ""),
+          Redactor.new(/INSERT INTO (`?\w+`?) \([^)]+\)/, 'INSERT INTO \1 ([COLUMNS])'),
+        ]
+
+        @options[:should_log_sample_proc] = Proc.new { |_name| false }
+        @options[:sql_redactor_complex_proc] = Proc.new { |sql| sql }
+        @options[:backtrace_redactors] = []
+        @options[:ambiguous_tracers] = []
+        @options[:ambiguous_backtrace_lines] = 3
+        @options[:analyzers] = []
+        @options[:logger_root_path] = Rails.root.join('log')
+        @options[:backtrace_filter_proc] = BacktraceFilter.proc
+      end
+    end
+  end
+end