active_record-sql_analyzer 0.0.5

data/lib/active_record/sql_analyzer/logger.rb

@@ -0,0 +1,29 @@
+module ActiveRecord
+  module SqlAnalyzer
+    class Logger
+      attr_reader :log_file, :log_prefix, :log_root, :config
+
+      def initialize(log_root, log_prefix)
+        @log_prefix = log_prefix
+        @log_root = log_root
+        @config = SqlAnalyzer.config
+
+        @log_file = File.open("#{log_root}/#{log_prefix}.log", "a")
+      end
+
+      # Log the raw event data directly to disk
+      def log(event)
+        log_file.puts(event.to_json)
+      end
+
+      # Further redact or remove any other information from an event
+      def filter_event(event)
+        event
+      end
+
+      def close
+        @log_file.close rescue nil
+      end
+    end
+  end
+end

data/lib/active_record/sql_analyzer/monkeypatches/query.rb

@@ -0,0 +1,35 @@
+
+module ActiveRecord
+  module SqlAnalyzer
+    module Monkeypatches
+      module Query
+        def execute(sql, *args)
+          return super unless SqlAnalyzer.config
+
+          safe_sql = nil
+
+          SqlAnalyzer.config[:analyzers].each do |analyzer|
+            if SqlAnalyzer.config[:should_log_sample_proc].call(analyzer[:name])
+              # This is here rather than above intentionally.
+              # We assume we're not going to be analyzing 100% of queries and want to only re-encode
+              # when it's actually relevant.
+              safe_sql ||= sql.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
+
+              if safe_sql =~ analyzer[:table_regex]
+                SqlAnalyzer.background_processor << {
+                  sql: safe_sql,
+                  caller: caller,
+                  logger: analyzer[:logger_instance],
+                  tag: Thread.current[:_ar_analyzer_tag],
+                  request_path: Thread.current[:_ar_analyzer_request_path]
+                }
+              end
+            end
+          end
+
+          super
+        end
+      end
+    end
+  end
+end

data/lib/active_record/sql_analyzer/monkeypatches/tagger.rb

@@ -0,0 +1,24 @@
+module ActiveRecord
+  module SqlAnalyzer
+    module Monkeypatches
+      module Tagger
+        def initialize(*)
+          super
+          @_ar_analyzer_tag = nil
+        end
+
+        def with_tag(name)
+          @_ar_analyzer_tag = name
+          self
+        end
+
+        def exec_queries
+          Thread.current[:_ar_analyzer_tag] ||= @_ar_analyzer_tag
+          super
+        ensure
+          Thread.current[:_ar_analyzer_tag] = nil if @_ar_analyzer_tag
+        end
+      end
+    end
+  end
+end

data/lib/active_record/sql_analyzer/redacted_logger.rb

@@ -0,0 +1,22 @@
+module ActiveRecord
+  module SqlAnalyzer
+    class RedactedLogger < CompactLogger
+      def filter_event(event)
+        # Determine if we're doing extended tracing or only the first
+        if config[:ambiguous_tracers].any? { |regex| event[:caller].first =~ regex }
+          event[:caller] = event[:caller][0, config[:ambiguous_backtrace_lines]].join(", ")
+        else
+          event[:caller] = event[:caller].first
+        end
+
+        config[:backtrace_redactors].each do |redactor|
+          event[:caller].gsub!(redactor.search, redactor.replace)
+        end
+
+        config[:sql_redactors].each do |redactor|
+          event[:sql].gsub!(redactor.search, redactor.replace)
+        end
+      end
+    end
+  end
+end

data/lib/active_record/sql_analyzer/redactor.rb

@@ -0,0 +1,5 @@
+module ActiveRecord
+  module SqlAnalyzer
+    Redactor = Struct.new(:search, :replace)
+  end
+end

data/lib/active_record/sql_analyzer/version.rb

@@ -0,0 +1,5 @@
+module ActiveRecord
+  module SqlAnalyzer
+    VERSION = '0.0.5'
+  end
+end

data/spec/active_record/sql_analyzer/analyzer_spec.rb

@@ -0,0 +1,33 @@
+RSpec.describe ActiveRecord::SqlAnalyzer::Analyzer do
+  let(:analyzer) do
+    described_class.new.tap do |instance|
+      instance.tables %w(foo bar)
+    end
+  end
+
+  context "table regex" do
+    let(:regex) { analyzer[:table_regex] }
+
+    it "matches" do
+      expect("SELECT * FROM foo").to match(regex)
+      expect("DELETE FROM foo").to match(regex)
+      expect("INSERT INTO bar (a, b, c) VALUES (1, 2, 3)").to match(regex)
+      expect("UPDATE bar SET a=b WHERE id=1").to match(regex)
+    end
+
+    it "matches with complex queries" do
+      expect("SELECT * FROM apple JOIN foo").to match(regex)
+      expect("SELECT * FROM apple LEFT JOIN foo").to match(regex)
+      expect("SELECT * FROM apple WHERE id = (SELECT * FROM foo)").to match(regex)
+    end
+
+    it "does not match" do
+      expect("SELECT * FROM apple WHERE id='foo'").to_not match(regex)
+      expect("SELECT foo FROM apple WHERE id='bar'").to_not match(regex)
+
+      expect("DELETE FROM apple").to_not match(regex)
+      expect("INSERT INTO apple (a, b, c) VALUES (1, 2, 3)").to_not match(regex)
+      expect("UPDATE apple SET a=b WHERE id=1").to_not match(regex)
+    end
+  end
+end

data/spec/active_record/sql_analyzer/background_processor_spec.rb

@@ -0,0 +1,44 @@
+RSpec.describe ActiveRecord::SqlAnalyzer::BackgroundProcessor do
+  include WaitForPop
+
+  let(:instance) { described_class.new }
+
+  let(:event) { {caller: "CALLER", sql: "SQL", logger: logger} }
+
+  let(:logger) do
+    Class.new do
+      def self.events
+        @events ||= []
+      end
+
+      def self.filter_event(*)
+      end
+
+      def self.log(event)
+        events << event
+        sleep 2
+      end
+    end
+  end
+
+  before do
+    ActiveRecord::SqlAnalyzer.configure do |c|
+      c.backtrace_filter_proc Proc.new { |lines| "BFP #{lines}" }
+      c.complex_sql_redactor_proc Proc.new { |sql| "CSRP #{sql}" }
+    end
+  end
+
+  it "processes in the background" do
+    instance << event
+    wait_for_pop
+
+    expect(logger.events).to eq(
+      [
+        {
+          caller: "BFP CALLER",
+          sql: "CSRP SQL"
+        }
+      ]
+    )
+  end
+end

data/spec/active_record/sql_analyzer/backtrace_filter_spec.rb

@@ -0,0 +1,28 @@
+RSpec.describe ActiveRecord::SqlAnalyzer::BacktraceFilter do
+  before do
+    ActiveRecord::SqlAnalyzer.configure { |_c| }
+  end
+
+  it "filters non-app paths" do
+    lines = ActiveRecord::SqlAnalyzer.config[:backtrace_filter_proc].call(
+      [
+        "foo/bar:1 in 'method'",
+        "#{Gem.path.first}:4231 in 'method'",
+        "foo/bar:2 in 'method'",
+        "#{File.realpath(Gem.path.first)}:9531 in 'method'",
+        "foo/bar:3 in 'method'",
+        "(eval):1234 in 'method'",
+        "foo/bar:4 in 'method'"
+      ]
+    )
+
+    expect(lines).to eq(
+      [
+        "foo/bar:1 in 'method'",
+        "foo/bar:2 in 'method'",
+        "foo/bar:3 in 'method'",
+        "foo/bar:4 in 'method'"
+      ]
+    )
+  end
+end

data/spec/active_record/sql_analyzer/cli_processor_spec.rb

@@ -0,0 +1,120 @@
+require_relative "../../../lib/active_record/sql_analyzer/cli_processor"
+
+RSpec.describe ActiveRecord::SqlAnalyzer::CLIProcessor do
+  let(:tmp_dir) { Dir.mktmpdir }
+  after { FileUtils.remove_entry(tmp_dir) }
+
+  def write_logs(prefix, *events)
+    started_at = Time.utc(2015, 1, 1, 0)
+
+    events.each_slice(2).each_with_index do |lines, index|
+      logger = ActiveRecord::SqlAnalyzer::CompactLogger.new(
+        tmp_dir,
+        "#{prefix}_#{index}"
+      )
+
+      lines.each do |event|
+        started_at += 3600
+
+        Timecop.freeze(started_at) do
+          logger.log(event)
+        end
+      end
+
+      logger.close
+    end
+  end
+
+  let(:instance) { described_class.new(2) }
+
+  before do
+    write_logs(:foo,
+      {sql: "F-SQL1", caller: "CALLER1", tag: true},
+      {sql: "F-SQL2", caller: "CALLER2", tag: true},
+      {sql: "F-SQL1", caller: "CALLER1", tag: true},
+      {sql: "F-SQL3", caller: "CALLER3", tag: true},
+      {sql: "F-SQL2", caller: "CALLER2", tag: true}
+    )
+
+    write_logs(:bar,
+      {sql: "B-SQL1", caller: "CALLER1"},
+      {sql: "B-SQL2", caller: "CALLER2"},
+      {sql: "B-SQL3", caller: "CALLER3"},
+      {sql: "B-SQL2", caller: "CALLER2"},
+      {sql: "B-SQL1", caller: "CALLER1"}
+    )
+  end
+
+  subject(:process) do
+    instance.run_definition(
+      Dir["#{tmp_dir}/*_*_definitions.log"].map do |path|
+        [File.basename(path).split("_", 2).first, path]
+      end
+    )
+
+    instance.run_usage(
+      Dir["#{tmp_dir}/*_*.log"].map do |path|
+        next if path =~ /definitions\.log$/
+        [File.basename(path).split("_", 2).first, path]
+      end.compact
+    )
+
+    instance.definitions
+  end
+
+  it "processes logs" do
+    logs = process
+
+    expect(logs["foo"].length).to eq(3)
+
+    queries = logs["foo"].values.sort_by { |row| row["sql"] }
+
+    expect(queries[0]["sql"]).to eq("F-SQL1")
+    expect(queries[0]["caller"]).to eq("CALLER1")
+    expect(queries[0]["count"]).to eq(2)
+    expect(queries[0]["last_called"]).to eq(Time.utc(2015, 1, 1, 3))
+    expect(queries[0]["tag"]).to eq(true)
+
+    expect(queries[1]["sql"]).to eq("F-SQL2")
+    expect(queries[1]["caller"]).to eq("CALLER2")
+    expect(queries[1]["count"]).to eq(2)
+    expect(queries[1]["last_called"]).to eq(Time.utc(2015, 1, 1, 5))
+    expect(queries[1]["tag"]).to eq(true)
+
+    expect(queries[2]["sql"]).to eq("F-SQL3")
+    expect(queries[2]["caller"]).to eq("CALLER3")
+    expect(queries[2]["count"]).to eq(1)
+    expect(queries[2]["last_called"]).to eq(Time.utc(2015, 1, 1, 4))
+    expect(queries[2]["tag"]).to eq(true)
+
+    expect(logs["bar"].length).to eq(3)
+
+    queries = logs["bar"].values.sort_by { |row| row["sql"] }
+
+    expect(queries[0]["sql"]).to eq("B-SQL1")
+    expect(queries[0]["caller"]).to eq("CALLER1")
+    expect(queries[0]["count"]).to eq(2)
+    expect(queries[0]["last_called"]).to eq(Time.utc(2015, 1, 1, 5))
+    expect(queries[0]["tag"]).to eq(nil)
+
+    expect(queries[1]["sql"]).to eq("B-SQL2")
+    expect(queries[1]["caller"]).to eq("CALLER2")
+    expect(queries[1]["count"]).to eq(2)
+    expect(queries[1]["last_called"]).to eq(Time.utc(2015, 1, 1, 4))
+    expect(queries[1]["tag"]).to eq(nil)
+
+    expect(queries[2]["sql"]).to eq("B-SQL3")
+    expect(queries[2]["caller"]).to eq("CALLER3")
+    expect(queries[2]["count"]).to eq(1)
+    expect(queries[2]["last_called"]).to eq(Time.utc(2015, 1, 1, 3))
+    expect(queries[2]["tag"]).to eq(nil)
+  end
+
+  it "dumps to disk" do
+    process
+    instance.dump(tmp_dir)
+
+    paths = Dir["#{tmp_dir}/{foo,bar}_#{Time.now.strftime("%Y-%m-%d")}.log"]
+    expect(paths.length).to eq(2)
+  end
+end

data/spec/active_record/sql_analyzer/cli_spec.rb

@@ -0,0 +1,66 @@
+require "fileutils"
+require_relative "../../../lib/active_record/sql_analyzer/cli"
+require_relative "../../../lib/active_record/sql_analyzer/cli_processor"
+
+RSpec.describe ActiveRecord::SqlAnalyzer::CLI do
+  let(:tmp_dir) { Dir.mktmpdir }
+  after { FileUtils.remove_entry(tmp_dir) }
+
+  let(:instance) do
+    cli = described_class.new
+    cli.parse_options(["--log-dir", tmp_dir, "--dest-dir", tmp_dir])
+    cli
+  end
+
+  before do
+    %w(foo bar).each do |prefix|
+      3.times do |i|
+        FileUtils.touch("#{tmp_dir}/#{prefix}.log.#{i}")
+        FileUtils.touch("#{tmp_dir}/#{prefix}_definitions.log.#{i}")
+      end
+    end
+  end
+
+  # I'm sorry
+  it "parses logs and starts the processor" do
+    expect(instance.processor).to receive(:run_definition) do |paths|
+      expect(paths.length).to eq(6)
+
+      prefixes = paths[0, 3].map(&:first).uniq
+      logs = paths[0, 3].map(&:last)
+      expect(prefixes.length).to eq(1)
+      expect(logs).to include(/#{prefixes.first}_definitions\.log\.0/)
+      expect(logs).to include(/#{prefixes.first}_definitions\.log\.1/)
+      expect(logs).to include(/#{prefixes.first}_definitions\.log\.2/)
+
+      prefixes = paths[3, 6].map(&:first).uniq
+      logs = paths[3, 6].map(&:last)
+      expect(prefixes.length).to eq(1)
+      expect(logs).to include(/#{prefixes.first}_definitions\.log\.0/)
+      expect(logs).to include(/#{prefixes.first}_definitions\.log\.1/)
+      expect(logs).to include(/#{prefixes.first}_definitions\.log\.2/)
+    end
+
+    expect(instance.processor).to receive(:run_usage) do |paths|
+      expect(paths.length).to eq(6)
+
+      prefixes = paths[0, 3].map(&:first).uniq
+      logs = paths[0, 3].map(&:last)
+      expect(prefixes.length).to eq(1)
+      expect(logs).to include(/#{prefixes.first}\.log\.0/)
+      expect(logs).to include(/#{prefixes.first}\.log\.1/)
+      expect(logs).to include(/#{prefixes.first}\.log\.2/)
+
+      prefixes = paths[3, 6].map(&:first).uniq
+      logs = paths[3, 6].map(&:last)
+      expect(prefixes.length).to eq(1)
+      expect(logs).to include(/#{prefixes.first}\.log\.0/)
+      expect(logs).to include(/#{prefixes.first}\.log\.1/)
+      expect(logs).to include(/#{prefixes.first}\.log\.2/)
+    end
+
+    expect(instance.processor).to receive(:dump).with(tmp_dir)
+
+    instance.run
+  end
+end

data/spec/active_record/sql_analyzer/end_to_end_spec.rb

@@ -0,0 +1,121 @@
+RSpec.describe "End to End" do
+  include WaitForPop
+
+  let(:tmp_dir) { Dir.mktmpdir }
+  after { FileUtils.remove_entry(tmp_dir) }
+
+  let(:log_path) { "#{tmp_dir}/test_tag.log" }
+  let(:log_data) { File.read(log_path) }
+
+  let(:log_hash) do
+    hash = {}
+    log_data.split("\n").each do |line|
+      sha = line.split("|", 2).last
+      hash[sha] ||= 0
+      hash[sha] += 1
+    end
+    hash
+  end
+
+  let(:log_reverse_hash) do
+    Hash[log_hash.map(&:reverse)]
+  end
+
+  let(:log_def_path) { "#{tmp_dir}/test_tag_definitions.log" }
+  let(:log_def_data) { File.read(log_def_path) }
+  let(:log_def_hash) do
+    hash = {}
+    log_def_data.split("\n").each do |line|
+      sha, event = line.split("|", 2)
+      hash[sha] = JSON.parse(event)
+    end
+
+    hash
+  end
+
+  before do
+    ActiveRecord::SqlAnalyzer.configure do |c|
+      c.logger_root_path tmp_dir
+      c.log_sample_proc Proc.new { |_name| true }
+
+      c.add_analyzer(
+        name: :test_tag,
+        tables: %w(matching_table)
+      )
+    end
+
+    ActiveRecord::SqlAnalyzer.config[:analyzers].each do |analyzer|
+      analyzer[:logger_instance].definition_log_file.sync = true
+      analyzer[:logger_instance].log_file.sync = true
+    end
+  end
+
+  def execute(sql)
+    DBConnection.connection.execute(sql)
+    wait_for_pop
+  end
+
+  it "does not log with a non-matching table" do
+    execute "SELECT * FROM nonmatching_table"
+
+    expect(log_data).to eq("")
+    expect(log_def_data).to eq("")
+  end
+
+  it "logs with a matching + non-matching table in one query" do
+    execute "SELECT nmt.id FROM nonmatching_table AS nmt JOIN matching_table AS mt ON mt.id = nmt.id WHERE mt.id = 1234"
+    execute "SELECT nmt.id FROM nonmatching_table AS nmt JOIN matching_table AS mt ON mt.id = nmt.id WHERE mt.id = 4321"
+    execute "SELECT nmt.id FROM nonmatching_table AS nmt JOIN matching_table AS mt ON mt.id = nmt.id WHERE mt.test_string = 'abc'"
+
+    expect(log_hash.length).to eq(2)
+
+    id_sha = log_reverse_hash[2]
+    str_sha = log_reverse_hash[1]
+
+    expect(log_def_hash.length).to eq(2)
+
+    expect(log_def_hash[id_sha]["sql"]).to include("mt.id = [REDACTED]")
+    expect(log_def_hash[str_sha]["sql"]).to include("mt.test_string = '[REDACTED]'")
+  end
+
+  it "logs with only a matching table in a query" do
+    execute "SELECT * FROM matching_table WHERE id = 1234"
+    execute "SELECT * FROM matching_table WHERE id = 4321"
+    execute "SELECT * FROM matching_table WHERE test_string = 'abc'"
+
+    expect(log_hash.length).to eq(2)
+
+    id_sha = log_reverse_hash[2]
+    str_sha = log_reverse_hash[1]
+
+    expect(log_def_hash.length).to eq(2)
+
+    expect(log_def_hash[id_sha]["sql"]).to include("id = [REDACTED]")
+    expect(log_def_hash[str_sha]["sql"]).to include("test_string = '[REDACTED]'")
+  end
+
+  it "handles invalid UTF-8" do
+    execute "SELECT * FROM matching_table WHERE test_string = '\xe5'"
+    execute "SELECT * FROM matching_table WHERE test_string = 'foobar'"
+
+    expect(log_reverse_hash.first.first).to eq(2)
+
+    sha = log_reverse_hash[2]
+    expect(log_def_hash[sha]["sql"]).to include("test_string = '[REDACTED]'")
+  end
+
+  context "when sampling is disabled" do
+    before do
+      ActiveRecord::SqlAnalyzer.configure do |c|
+        c.log_sample_proc Proc.new { |_name| false }
+      end
+    end
+
+    it "does not log" do
+      execute "SELECT * FROM matching_table"
+
+      expect(log_data).to eq("")
+      expect(log_def_data).to eq("")
+    end
+  end
+end