active_record-sql_analyzer 0.2.2 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 36199dbc04d9d0e4e4a9cf5c936676acb546e091
-  data.tar.gz: 36d99d4c4c1bc6e897c28c2e83903e831fb70249
+  metadata.gz: 77bac67b331f9db59ff2dbc8f57ec2bca04505c0
+  data.tar.gz: 870a62bd36006b4098d8e88a3a3af1cd3fa310b2
 SHA512:
-  metadata.gz: 1e87e40d478a83c12e849df385cf4d98d4275d62793c25ad2253b24cd8a5b28608945fa1890c95d946d1400e6c0327457a16d85a8b9e5079bd19c2e21b335913
-  data.tar.gz: 528b59ab97b3c4225a371cd8fd056e46cfa8253a2df5f9b404c9dda3ce02c2e4c1b1d1c090945729da9adde9033c7678adcdb9d703af3fe4413c62bcfb99afc5
+  metadata.gz: 88f4ef3bbe57d34b89194e045e2845f4fa4bd9d5800e47943b8e9b2561fc2cb9025a870d976fc36e29c43564bc98c3c3baa7497ea6bd5ae2e894b505205ab3bd
+  data.tar.gz: 7042bf53a81900be2c19a3bfa4e5564d2c2ca0d9fa23d3af22838e6322dd584e460db80236a5f35f1a755efba7a72781516d7c33ac9c7c3c24d2660518126cd8

data/.rubocop.yml

@@ -0,0 +1,12 @@
+AllCops:
+  DisplayCopNames: true
+  TargetRubyVersion: 2.3
+  Exclude:
+    - 'bin/*'
+    - 'spec/support/**/*'
+
+Metrics:
+  Enabled: false
+
+Style:
+  Enabled: false

data/.travis.yml

@@ -1,15 +1,13 @@
 bundler_args: --without development
-env:
-  global:
-    - JRUBY_OPTS="$JRUBY_OPTS --debug"
 language: ruby
 rvm:
-  - 2.1
   - 2.2
-  - 2.3.0
+  - 2.3.3
+  - 2.4.0-preview3
   - ruby-head
 matrix:
   allow_failures:
+    - rvm: 2.4.0-preview3
     - rvm: ruby-head
   fast_finish: true
 sudo: false

data/Gemfile

@@ -1,13 +1,14 @@
 source "https://rubygems.org"
 
+gem "pry-byebug"
 gem "rake"
 
 group :test do
-  gem 'mysql2', '~> 0.4', '>= 0.4.0'
-  gem 'rspec', '~> 3.4'
-  gem 'rubocop', '~> 0.30'
-  gem 'timecop', '~> 0.8'
-  gem 'sql-parser', git: 'https://github.com/nerdrew/sql-parser.git'
+  gem "mysql2", "~> 0.4", ">= 0.4.0"
+  gem "rspec", "~> 3.4"
+  gem "rubocop", "~> 0.30"
+  gem "sql-parser", git: "https://github.com/nerdrew/sql-parser.git"
+  gem "timecop", "~> 0.8"
 end
 
 gemspec

data/Rakefile

@@ -1,6 +1,13 @@
 require 'bundler/gem_tasks'
 
-require 'rspec/core/rake_task'
-RSpec::Core::RakeTask.new
+begin
+  require "rubocop/rake_task"
+  require "rspec/core/rake_task"
 
-task default: 'spec'
+  RuboCop::RakeTask.new
+  RSpec::Core::RakeTask.new(:spec)
+
+  task default: [:rubocop, :spec]
+rescue LoadError
+  warn "rubocop, rspec only available in development"
+end

data/lib/active_record/sql_analyzer/cli_processor.rb

@@ -35,7 +35,7 @@ module ActiveRecord
         logs.each { |l| queue << l }
 
         # Spin up threads to start processing the queue
-        threads = concurrency.times.map do
+        threads = Array.new(concurrency) do
           Thread.new(queue) do |t_queue|
             # Create a local copy of each definitions then merge them in
             CLIProcessor.process_queue(t_queue) do |local_definitions, line|
@@ -63,7 +63,7 @@ module ActiveRecord
         logs.each { |l| queue << l }
 
         # Spin up threads to start processing the queue
-        threads = concurrency.times.map do
+        threads = Array.new(concurrency) do
           Thread.new(queue) do |t_queue|
             # Create a local copy of the usage for each SHA then merge it in at the end
             CLIProcessor.process_queue(t_queue) do |local_usage, line|

data/lib/active_record/sql_analyzer/configuration.rb

@@ -108,7 +108,7 @@ module ActiveRecord
 
       # How many total lines to log when the caller is ambiguous
       def ambiguous_backtrace_lines(lines)
-        if !lines.is_a?(Fixnum)
+        if !lines.is_a?(Integer)
           raise ArgumentError, "Lines must be a Fixnum"
         elsif lines <= 1
           raise ArgumentError, "Lines cannot be <= 1"
@@ -129,7 +129,7 @@ module ActiveRecord
 
       private
 
-      def check_proc(proc, arity, msg)
+      def check_proc(proc, _arity, msg)
         if !proc.is_a?(Proc)
           raise ArgumentError, "You must pass a proc"
         elsif proc.arity != 1

data/lib/active_record/sql_analyzer/monkeypatches/query.rb

@@ -8,23 +8,30 @@ module ActiveRecord
 
         def execute(sql, *args)
           return super unless SqlAnalyzer.config
+          safe_sql = nil
+          query_analyzer_call = nil
 
-          if @_query_analyzer_private_transaction_queue
-            @_query_analyzer_private_transaction_queue << QueryAnalyzerCall.new(sql, caller)
-          else
-            safe_sql = nil
-
-            SqlAnalyzer.config[:analyzers].each do |analyzer|
-              if SqlAnalyzer.config[:should_log_sample_proc].call(analyzer[:name])
-                # This is here rather than above intentionally.
-                # We assume we're not going to be analyzing 100% of queries and want to only re-encode
-                # when it's actually relevant.
-                safe_sql ||= sql.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
-
-                if safe_sql =~ analyzer[:table_regex]
-                  SqlAnalyzer.background_processor <<
-                    _query_analyzer_private_query_stanza([QueryAnalyzerCall.new(safe_sql, caller)], analyzer)
-                end
+          # Record "full" transactions (see below for more information about "full")
+          if @_query_analyzer_private_in_transaction
+            if @_query_analyzer_private_record_transaction
+              safe_sql ||= sql.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
+              query_analyzer_call ||= QueryAnalyzerCall.new(safe_sql, caller)
+              @_query_analyzer_private_transaction_queue << query_analyzer_call
+            end
+          end
+
+          # Record interesting queries
+          SqlAnalyzer.config[:analyzers].each do |analyzer|
+            if SqlAnalyzer.config[:should_log_sample_proc].call(analyzer[:name])
+              # This is here rather than above intentionally.
+              # We assume we're not going to be analyzing 100% of queries and want to only re-encode
+              # when it's actually relevant.
+              safe_sql ||= sql.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
+              query_analyzer_call ||= QueryAnalyzerCall.new(safe_sql, caller)
+
+              if safe_sql =~ analyzer[:table_regex]
+                SqlAnalyzer.background_processor <<
+                _query_analyzer_private_query_stanza([query_analyzer_call], analyzer)
               end
             end
           end
@@ -32,30 +39,62 @@ module ActiveRecord
           super
         end
 
+        def begin_db_transaction
+          @_query_analyzer_private_in_transaction = true
+
+          record_transaction = SqlAnalyzer.config[:analyzers].any? do |analyzer|
+            SqlAnalyzer.config[:should_log_sample_proc].call(analyzer[:name])
+          end
+          if record_transaction
+            @_query_analyzer_private_transaction_queue ||= []
+            @_query_analyzer_private_record_transaction = true
+          else
+            @_query_analyzer_private_record_transaction = nil
+          end
+          super
+        end
+
+        def commit_db_transaction
+          _query_analyzer_private_drain_transaction_queue("COMMIT")
+          super
+        ensure
+          @_query_analyzer_private_in_transaction = false
+        end
+
+        def exec_rollback_db_transaction
+          _query_analyzer_private_drain_transaction_queue("ROLLBACK")
+          super
+        ensure
+          @_query_analyzer_private_in_transaction = false
+        end
+
+        # "private" methods for this monkeypatch
+
         # Drain the transaction query queue. Log the current transaction out to any logger that samples it.
-        def _query_analyzer_private_drain_transaction_queue
+        def _query_analyzer_private_drain_transaction_queue(last_query)
+          return unless @_query_analyzer_private_record_transaction
+
           reencoded_calls = nil
 
           SqlAnalyzer.config[:analyzers].each do |analyzer|
-            if SqlAnalyzer.config[:should_log_sample_proc].call(analyzer[:name])
-              # Again, trying to only re-encode and join strings if the transaction is actually
-              # sampled.
-              reencoded_calls ||= @_query_analyzer_private_transaction_queue.map do |call|
-                QueryAnalyzerCall.new(call.sql.encode(Encoding::UTF_8, invalid: :replace, undef: :replace), call.caller)
-              end
-
-              has_matching_calls = reencoded_calls.any? { |call| call.sql =~ analyzer[:table_regex] }
+            reencoded_calls ||= @_query_analyzer_private_transaction_queue << QueryAnalyzerCall.new(last_query, caller)
 
-              if has_matching_calls
-                matching_calls = reencoded_calls.select do |call|
-                  (call.sql =~ /^(BEGIN|COMMIT|ROLLBACK|UPDATE|INSERT|DELETE)/) || (call.sql =~ analyzer[:table_regex])
-                end
+            has_matching_calls = reencoded_calls.any? { |call| call.sql =~ analyzer[:table_regex] }
 
-                SqlAnalyzer.background_processor <<
-                  _query_analyzer_private_query_stanza(matching_calls, analyzer)
+            # Record "full" transactions
+            # Record all INSERT, UPDATE, and DELETE
+            # Record all queries that match the analyzer's table_regex
+            if has_matching_calls
+              matching_calls = reencoded_calls.select do |call|
+                (call.sql =~ /^(BEGIN|COMMIT|ROLLBACK|UPDATE|INSERT|DELETE)/) || (call.sql =~ analyzer[:table_regex])
               end
+
+              SqlAnalyzer.background_processor << _query_analyzer_private_query_stanza(matching_calls, analyzer)
             end
           end
+
+          @_query_analyzer_private_transaction_queue.clear
+          @_query_analyzer_private_record_transaction = nil
         end
 
         # Helper method to construct the event for a query or transaction.
@@ -70,24 +109,6 @@ module ActiveRecord
             request_path: Thread.current[:_ar_analyzer_request_path],
           }
         end
-
-        def transaction(requires_new: nil, isolation: nil, joinable: true)
-          must_clear_transaction = false
-
-          if SqlAnalyzer.config[:consolidate_transactions]
-            if @_query_analyzer_private_transaction_queue.nil?
-              must_clear_transaction = true
-              @_query_analyzer_private_transaction_queue = []
-            end
-          end
-
-          super
-        ensure
-          if must_clear_transaction
-            _query_analyzer_private_drain_transaction_queue
-            @_query_analyzer_private_transaction_queue = nil
-          end
-        end
       end
     end
   end

data/lib/active_record/sql_analyzer/redacted_logger.rb

@@ -22,10 +22,10 @@ module ActiveRecord
 
       def filter_caller(kaller)
         kaller = if config[:ambiguous_tracers].any? { |regex| kaller.first =~ regex }
-          kaller[0, config[:ambiguous_backtrace_lines]].join(", ")
-        else
-          kaller.first
-        end
+                   kaller[0, config[:ambiguous_backtrace_lines]].join(", ")
+                 else
+                   kaller.first
+                 end
 
         return '' unless kaller
 

data/lib/active_record/sql_analyzer/version.rb

@@ -1,5 +1,5 @@
 module ActiveRecord
   module SqlAnalyzer
-    VERSION = '0.2.2'
+    VERSION = '0.2.3'
   end
 end

data/spec/active_record/sql_analyzer/end_to_end_spec.rb

@@ -216,6 +216,36 @@ RSpec.describe "End to End" do
       "COMMIT;")
   end
 
+  context "ActiveRecord generated transactions" do
+    before do
+      stub_const("Matching", Class.new(ActiveRecord::Base) do
+        self.table_name = "matching_table"
+
+        after_commit { self.class.last }
+      end)
+
+      stub_const("NonMatching", Class.new(ActiveRecord::Base) do
+        self.table_name = "nonmatching_table"
+
+        after_commit { self.class.last }
+      end)
+    end
+
+    it "Logs the matching statements in the transaction and logs after_commit hooks outside the transaction" do
+      Matching.transaction do
+        Matching.create!
+        NonMatching.create
+        NonMatching.last
+      end
+      NonMatching.create
+
+      expect(log_def_hash.map { |_hash, data| data["sql"] }).to match([
+        "INSERT INTO `matching_table` VALUES ()",
+        "BEGIN; INSERT INTO `matching_table` VALUES (); INSERT INTO `nonmatching_table` VALUES (); COMMIT;",
+        "SELECT `matching_table`.* FROM `matching_table` ORDER BY `matching_table`.`id` DESC"
+      ])
+    end
+  end
 
   it "Logs mixed matching-nonmatching with inserts correctly" do
     transaction do
@@ -278,15 +308,14 @@ RSpec.describe "End to End" do
         execute "SELECT * FROM matching_table WHERE id > 4 and id < 8"
       end
 
-
-      transaction_executed_once_sha = log_reverse_hash[1]
-
-      expect(log_def_hash.size).to eq(1)
-      expect(log_def_hash[transaction_executed_once_sha]["sql"]).to eq(
-        "BEGIN; " \
-        "SELECT * FROM matching_table WHERE id = '[REDACTED]'; " \
-        "SELECT * FROM matching_table WHERE test_string = '[REDACTED]'; " \
-        "COMMIT;")
+      expect(log_def_hash.map { |_hash, data| data["sql"] }).to match([
+        "SELECT * FROM matching_table WHERE id = '[REDACTED]'",
+        "BEGIN; "\
+        "SELECT * FROM matching_table WHERE id = '[REDACTED]'; "\
+        "SELECT * FROM matching_table WHERE test_string = '[REDACTED]'; "\
+        "COMMIT;",
+        "SELECT * FROM matching_table WHERE id = '[REDACTED]' and id = '[REDACTED]'"
+      ])
     end
   end
 

data/spec/support/db_connection.rb

@@ -19,6 +19,7 @@ class DBConnection
   end
 
   def self.setup_db
+    ActiveRecord::Base.raise_in_transactional_callbacks = true
     conn = ActiveRecord::Base.establish_connection(configuration)
     conn.connection.execute <<-SQL
       CREATE DATABASE IF NOT EXISTS ar_sql_analyzer_test

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: active_record-sql_analyzer
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
 platform: ruby
 authors:
 - Zachary Anker
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-19 00:00:00.000000000 Z
+date: 2016-12-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -56,6 +56,7 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".rspec"
+- ".rubocop.yml"
 - ".travis.yml"
 - CONTRIBUTING.md
 - Gemfile
@@ -110,7 +111,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.6
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Logs a subset of ActiveRecord queries and dumps them for analyses.