active_record-sql_analyzer 0.0.8 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d12fe56729a7e42b10fdb9b5aadd08ed74f23941
-  data.tar.gz: b3608149a379ef161b541ffc1307cc263ad7750a
+  metadata.gz: 48238f31887e49755dbac5b74348f9f580dc0b65
+  data.tar.gz: 7c875b3aa9dfdb7e4449f12893ff1ddf2f7effff
 SHA512:
-  metadata.gz: 18c45d31d25f34452e3244ec8c7605a766214c241724d8aaf8e2f4835a9fc78c6a52f6f43611b3ca08547b141e9e7b53633c8eeafd97cabfbac8a71e71c3c437
-  data.tar.gz: 7343c8e1d69445cb1c4be107cba3cb128c3dc6f96d7636ac7c0f50c55c17ba295e8b7511a99b75f96d249ad3848ea275f66ab09fb6703f8f387e5e8e624b3ffa
+  metadata.gz: ceaaefc3ea471c547e81c4ce227a7fb1b1b13e6d61ea5caffd6d1391b1802fe35af2f5948d7624f1c376e75bfc0c4a89b8be6efb61667ddb5e1faac0bc34e33b
+  data.tar.gz: a89a8a5ae0d6e38c08123a64b0e1a2ec63b8fcfd8641e5eaff55507c122717177c97fc84ab50f605e08ee3a0954ef98a711abc8fa6188253b76ae2ff359cd05c

data/Gemfile

@@ -7,6 +7,7 @@ group :test do
   gem 'rspec', '~> 3.4'
   gem 'rubocop', '~> 0.30'
   gem 'timecop', '~> 0.8'
+  gem 'sql-parser', git: 'https://github.com/nerdrew/sql-parser.git'
 end
 
 gemspec

data/lib/active_record/sql_analyzer/configuration.rb

@@ -146,19 +146,19 @@ module ActiveRecord
       end
 
       def setup_defaults
-        quotedValuePattern = "'[^'\\\\]*(?:\\\\.[^'\\\\]*)*'"
+        quotedValuePattern = %{('([^\\\\']|\\\\.|'')*'|"([^\\\\"]|\\\\.|"")*")}
         @options[:sql_redactors] = [
           Redactor.new(/\n/, " "),
           Redactor.new(/\s+/, " "),
-          Redactor.new(/(\s|\b|`)(=|!=|>=|>|<=|<) ?(BINARY )?-?\d+(\.\d+)?/i, " = [REDACTED]"),
+          Redactor.new(/IN \([^)]+\)/i, "IN ('[REDACTED]')"),
+          Redactor.new(/(\s|\b|`)(=|!=|>=|>|<=|<) ?(BINARY )?-?\d+(\.\d+)?/i, " = '[REDACTED]'"),
           Redactor.new(/(\s|\b|`)(=|!=|>=|>|<=|<) ?(BINARY )?x?#{quotedValuePattern}/i, " = '[REDACTED]'"),
-          Redactor.new(/VALUES \(.+\)$/i, "VALUES ([REDACTED])"),
-          Redactor.new(/IN \([^)]+\)/i, "IN ([REDACTED])"),
+          Redactor.new(/VALUES \(.+\)$/i, "VALUES ('[REDACTED]')"),
           Redactor.new(/BETWEEN #{quotedValuePattern} AND #{quotedValuePattern}/i, "BETWEEN '[REDACTED]' AND '[REDACTED]'"),
-          Redactor.new(/LIKE #{quotedValuePattern}/i, "LIKE '[REDACTED]'"), 
+          Redactor.new(/LIKE #{quotedValuePattern}/i, "LIKE '[REDACTED]'"),
           Redactor.new(/ LIMIT \d+/i, ""),
           Redactor.new(/ OFFSET \d+/i, ""),
-          Redactor.new(/INSERT INTO (`?\w+`?) \([^)]+\)/i, 'INSERT INTO \1 ([COLUMNS])'),
+          Redactor.new(/INSERT INTO (`?\w+`?) \([^)]+\)/i, "INSERT INTO \\1 (REDACTED_COLUMNS)"),
         ]
 
         @options[:should_log_sample_proc] = Proc.new { |_name| false }

data/lib/active_record/sql_analyzer/monkeypatches/query.rb

@@ -1,3 +1,4 @@
+require 'securerandom'
 
 module ActiveRecord
   module SqlAnalyzer
@@ -9,7 +10,7 @@ module ActiveRecord
           safe_sql = nil
 
           SqlAnalyzer.config[:analyzers].each do |analyzer|
-            if SqlAnalyzer.config[:should_log_sample_proc].call(analyzer[:name])
+            if _query_analyzer_private_should_sample_query(analyzer[:name])
               # This is here rather than above intentionally.
               # We assume we're not going to be analyzing 100% of queries and want to only re-encode
               # when it's actually relevant.
@@ -21,7 +22,8 @@ module ActiveRecord
                   caller: caller,
                   logger: analyzer[:logger_instance],
                   tag: Thread.current[:_ar_analyzer_tag],
-                  request_path: Thread.current[:_ar_analyzer_request_path]
+                  request_path: Thread.current[:_ar_analyzer_request_path],
+                  transaction: @_query_analyzer_private_transaction_uuid
                 }
               end
             end
@@ -29,6 +31,42 @@ module ActiveRecord
 
           super
         end
+
+        # Whether or not we should sample the query. If it's part of a transaction, check whether we
+        # are sampling the transaction. Otherwise, run the sample proc.
+        def _query_analyzer_private_should_sample_query(analyzer_name)
+          if @_query_analyzer_private_transaction_uuid.present?
+            @_query_analyzer_private_should_sample_transaction[analyzer_name]
+          else
+            SqlAnalyzer.config[:should_log_sample_proc].call(analyzer_name)
+          end
+        end
+
+        # Calculate hash of analyzer_name -> boolean representing whether we should sample the
+        # whole transaction for this analyzer.
+        def _query_analyzer_private_calculate_sampling_for_all_analyzers
+          SqlAnalyzer.config[:analyzers].map do |analyzer|
+            [analyzer[:name], SqlAnalyzer.config[:should_log_sample_proc].call(analyzer[:name])]
+          end.to_h
+        end
+
+        def transaction(requires_new: nil, isolation: nil, joinable: true)
+          must_clear_uuid = false
+
+          if @_query_analyzer_private_transaction_uuid.nil? then
+            must_clear_uuid = true
+            @_query_analyzer_private_transaction_uuid = SecureRandom.uuid
+            @_query_analyzer_private_should_sample_transaction =
+               _query_analyzer_private_calculate_sampling_for_all_analyzers
+          end
+
+          super
+        ensure
+          if must_clear_uuid
+            @_query_analyzer_private_transaction_uuid = nil
+            @_query_analyzer_private_should_sample_transaction = nil
+          end
+        end
       end
     end
   end

data/lib/active_record/sql_analyzer/version.rb

@@ -1,5 +1,5 @@
 module ActiveRecord
   module SqlAnalyzer
-    VERSION = '0.0.8'
+    VERSION = '0.1.0'
   end
 end

data/spec/active_record/sql_analyzer/end_to_end_spec.rb

@@ -74,7 +74,7 @@ RSpec.describe "End to End" do
 
     expect(log_def_hash.length).to eq(2)
 
-    expect(log_def_hash[id_sha]["sql"]).to include("mt.id = [REDACTED]")
+    expect(log_def_hash[id_sha]["sql"]).to include("mt.id = '[REDACTED]'")
     expect(log_def_hash[str_sha]["sql"]).to include("mt.test_string = '[REDACTED]'")
   end
 
@@ -90,7 +90,7 @@ RSpec.describe "End to End" do
 
     expect(log_def_hash.length).to eq(2)
 
-    expect(log_def_hash[id_sha]["sql"]).to include("id = [REDACTED]")
+    expect(log_def_hash[id_sha]["sql"]).to include("id = '[REDACTED]'")
     expect(log_def_hash[str_sha]["sql"]).to include("test_string = '[REDACTED]'")
   end
 
@@ -104,6 +104,102 @@ RSpec.describe "End to End" do
     expect(log_def_hash[sha]["sql"]).to include("test_string = '[REDACTED]'")
   end
 
+  it "logs multiple queries in a transaction correctly" do
+    DBConnection.connection.transaction do
+      4.times { execute "SELECT * FROM matching_table WHERE id = 4321" }
+      3.times { execute "SELECT * FROM matching_table WHERE test_string = 'abc'" }
+    end
+
+    DBConnection.connection.transaction do
+      2.times { execute "SELECT * FROM matching_table WHERE test_string LIKE 'abc'" }
+      execute "SELECT * FROM matching_table WHERE id > 2 and id < 4"
+    end
+
+    id_eq_sha = log_reverse_hash[4]
+    str_eq_sha = log_reverse_hash[3]
+    str_like_sha = log_reverse_hash[2]
+    id_gt_sha = log_reverse_hash[1]
+
+    expect(log_def_hash[id_gt_sha]["sql"]).to include("id = [REDACTED] and id = [REDACTED]")
+    expect(log_def_hash[str_like_sha]["sql"]).to include("test_string LIKE '[REDACTED]'")
+    expect(log_def_hash[id_gt_sha]["transaction"]).to eq(log_def_hash[str_like_sha]["transaction"])
+
+    expect(log_def_hash[id_eq_sha]["sql"]).to include("id = [REDACTED]")
+    expect(log_def_hash[str_eq_sha]["sql"]).to include("test_string = '[REDACTED]'")
+    expect(log_def_hash[str_eq_sha]["transaction"]).to eq(log_def_hash[id_eq_sha]["transaction"])
+
+    expect(log_def_hash[str_like_sha]["transaction"]).to_not eq(log_def_hash[id_eq_sha]["transaction"])
+  end
+
+  it "Logs nested transactions correctly" do
+    DBConnection.connection.transaction do
+      execute "SELECT * FROM matching_table WHERE id = 4321"
+      DBConnection.connection.transaction do
+        2.times { execute "SELECT * FROM matching_table WHERE test_string = 'abc'" }
+      end
+    end
+
+
+    id_sha = log_reverse_hash[1]
+    str_sha = log_reverse_hash[2]
+
+    expect(log_def_hash[id_sha]["transaction"]).to eq(log_def_hash[str_sha]["transaction"])
+    expect(log_def_hash[str_sha]["transaction"]).not_to be_nil
+  end
+
+  it "Logs empty transactions correctly" do
+    DBConnection.connection.transaction do
+      execute "SELECT * FROM matching_table WHERE id = 4321"
+    end
+
+    2.times { execute "SELECT * FROM matching_table WHERE test_string = 'abc'" }
+
+    id_sha = log_reverse_hash[1]
+    str_sha = log_reverse_hash[2]
+
+    expect(log_def_hash[id_sha]["sql"]).to include("id = [REDACTED]")
+    expect(log_def_hash[str_sha]["sql"]).to include("test_string = '[REDACTED]'")
+
+    expect(log_def_hash[id_sha]["transaction"]).not_to be_nil
+    expect(log_def_hash[str_sha]["transaction"]).to be_nil
+  end
+
+  context "Selectively sampling" do
+    before do
+      ActiveRecord::SqlAnalyzer.configure do |c|
+        times_called = 0
+        # Return true every other call, starting with the first call
+        c.log_sample_proc Proc.new { |_name| (times_called += 1) % 2 == 1 }
+      end
+    end
+
+    it "Samples some but not other selects" do
+      execute "SELECT * FROM matching_table WHERE id = 1"
+      execute "SELECT * FROM matching_table WHERE test_string = 'abc'"
+
+      expect(log_def_hash.size).to eq(1)
+      expect(log_def_hash.map { |_hash, query| query['sql'] }).to eq([
+        "SELECT * FROM matching_table WHERE id = [REDACTED]"])
+    end
+
+    it "Samples some but not other whole transactions" do
+      DBConnection.connection.transaction do
+        execute "SELECT * FROM matching_table WHERE id = 1"
+        execute "SELECT * FROM matching_table WHERE test_string = 'abc'"
+      end
+
+      DBConnection.connection.transaction do
+        execute "SELECT * FROM matching_table WHERE id = 1 and test_string = 'abc'"
+        execute "SELECT * FROM matching_table WHERE id > 4 and id < 8"
+      end
+
+      expect(log_def_hash.size).to eq(2)
+      expect(Set.new(log_def_hash.map { |_hash, query| query['sql'] })).to eq(Set.new([
+        "SELECT * FROM matching_table WHERE id = [REDACTED]",
+        "SELECT * FROM matching_table WHERE test_string = '[REDACTED]'"]))
+    end
+  end
+
   context "when sampling is disabled" do
     before do
       ActiveRecord::SqlAnalyzer.configure do |c|

data/spec/active_record/sql_analyzer/redacted_logger_spec.rb

@@ -1,5 +1,9 @@
+require "spec_helper"
+require "sql-parser"
+
 RSpec.describe ActiveRecord::SqlAnalyzer::RedactedLogger do
   let(:tmp_dir) { Dir.mktmpdir }
+  let(:parser) { SQLParser::Parser.new }
   after { FileUtils.remove_entry(tmp_dir) }
 
   context "#filter_event" do
@@ -17,6 +21,13 @@ RSpec.describe ActiveRecord::SqlAnalyzer::RedactedLogger do
           [/erb_erb_[0-9]+_[0-9]+/, ""]
         ]
       end
+      # All raw SQL should be valid :)
+      expect { parser.scan_str(event[:sql]) }.not_to raise_exception if event[:sql].present?
+    end
+
+    after do
+      # All redacted SQL should be valid
+      expect { parser.scan_str(filter_event[:sql]) }.not_to raise_exception if filter_event[:sql].present?
     end
 
     context "ambiguous backtraces" do
@@ -101,7 +112,7 @@ RSpec.describe ActiveRecord::SqlAnalyzer::RedactedLogger do
 
       it "redacts" do
         expect(filter_event[:sql]).to eq("SELECT * FROM foo WHERE name LIKE '[REDACTED]'")
-      end      
+      end
     end
 
     context "sql" do
@@ -113,7 +124,7 @@ RSpec.describe ActiveRecord::SqlAnalyzer::RedactedLogger do
       end
 
       it "redacts" do
-        expect(filter_event[:sql]).to eq("SELECT * FROM foo WHERE id = [REDACTED]")
+        expect(filter_event[:sql]).to eq("SELECT * FROM foo WHERE id = '[REDACTED]'")
       end
     end
 
@@ -121,12 +132,12 @@ RSpec.describe ActiveRecord::SqlAnalyzer::RedactedLogger do
       let(:event) do
         {
           caller: [""],
-          sql: "SELECT * FROM foo WHERE name LIKE 'A \\'quoted\\' value.'"
+          sql: %{SELECT * FROM foo WHERE name LIKE 'A \\'quoted\\' value.' OR name LIKE "another ""quoted"" \\"value\\""}
         }
       end
 
       it "redacts" do
-        expect(filter_event[:sql]).to eq("SELECT * FROM foo WHERE name LIKE '[REDACTED]'")
+        expect(filter_event[:sql]).to eq("SELECT * FROM foo WHERE name LIKE '[REDACTED]' OR name LIKE '[REDACTED]'")
       end
     end
 
@@ -147,12 +158,12 @@ RSpec.describe ActiveRecord::SqlAnalyzer::RedactedLogger do
       let(:event) do
         {
           caller: [""],
-          sql: "SELECT * FROM foo WHERE name IN ('A \\'quoted\\' value.')"
+          sql: "SELECT * FROM foo WHERE name IN ('A ''quoted'' value.')"
         }
       end
 
       it "redacts" do
-        expect(filter_event[:sql]).to eq("SELECT * FROM foo WHERE name IN ([REDACTED])")
+        expect(filter_event[:sql]).to eq("SELECT * FROM foo WHERE name IN ('[REDACTED]')")
       end
     end
 
@@ -160,20 +171,20 @@ RSpec.describe ActiveRecord::SqlAnalyzer::RedactedLogger do
       let(:event) do
         {
           caller: [""],
-          sql: "SELECT * FROM foo WHERE name IN ('A \\\'quoted\\\' value.')"
+          sql: %{SELECT * FROM foo WHERE name IN ('A ''quoted'' value.', "another ""quoted"" \\"value\\"")}
         }
       end
 
       it "redacts" do
-        expect(filter_event[:sql]).to eq("SELECT * FROM foo WHERE name IN ([REDACTED])")
+        expect(filter_event[:sql]).to eq("SELECT * FROM foo WHERE name IN ('[REDACTED]')")
       end
     end
 
-    context "between quoted" do
+    context "between strings" do
       let(:event) do
         {
           caller: [""],
-          sql: "SELECT * FROM foo WHERE name BETWEEN 'A \\'quoted\\' value.' AND 'Another value'"
+          sql: "SELECT * FROM foo WHERE name BETWEEN 'A value.' AND 'Another value'"
         }
       end
 
@@ -182,11 +193,11 @@ RSpec.describe ActiveRecord::SqlAnalyzer::RedactedLogger do
       end
     end
 
-    context "between escaped and quoted" do
+    context "between strings with escaped quotes" do
       let(:event) do
         {
           caller: [""],
-          sql: "SELECT * FROM foo WHERE name BETWEEN 'A \\\'quoted\\\' value.' AND 'Another value'"
+          sql: "SELECT * FROM foo WHERE name BETWEEN 'A ''quoted'' value.' AND 'Another \\'value\\''"
         }
       end
 
@@ -194,5 +205,31 @@ RSpec.describe ActiveRecord::SqlAnalyzer::RedactedLogger do
         expect(filter_event[:sql]).to eq("SELECT * FROM foo WHERE name BETWEEN '[REDACTED]' AND '[REDACTED]'")
       end
     end
+
+    context "in with = and other where clauses" do
+      let(:event) do
+        {
+          caller: [""],
+          sql: "SELECT * FROM foo WHERE name IN ('value=') AND name = 'value'"
+        }
+      end
+
+      it "redacts" do
+        expect(filter_event[:sql]).to eq("SELECT * FROM foo WHERE name IN ('[REDACTED]') AND name = '[REDACTED]'")
+      end
+    end
+
+    context "insert" do
+      let(:event) do
+        {
+          caller: [""],
+          sql: "INSERT INTO `boom` (`bam`, `foo`) VALUES ('howdy', 'dowdy')"
+        }
+      end
+
+      it "redacts" do
+        expect(filter_event[:sql]).to eq("INSERT INTO `boom` (REDACTED_COLUMNS) VALUES ('[REDACTED]')")
+      end
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: active_record-sql_analyzer
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.1.0
 platform: ruby
 authors:
 - Zachary Anker
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-05-09 00:00:00.000000000 Z
+date: 2016-08-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -110,7 +110,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Logs a subset of ActiveRecord queries and dumps them for analyses.
@@ -127,4 +127,3 @@ test_files:
 - spec/support/stub_logger.rb
 - spec/support/stub_rails.rb
 - spec/support/wait_for_pop.rb
-has_rdoc: