activerecord-instrumentation 0.3.0.jlauer2 → 0.4.0.jlauer1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a4923138123e7321b389d97d69503b1fa5db021f5f50438209dbfe06f3942aa7
-  data.tar.gz: f31bd6a40eedfdf3f10dfbfb147a9ac0e15a23f7382dd04b09e9bd6265f4aab0
+  metadata.gz: 48fc834ad963b8c31ea09c9974ca1d024a692a71592a20e75f0079c9a2c93203
+  data.tar.gz: 6be927a6854e55963f0fa1ab1ca0f5aa05c44e2e75d4eb8bebe7b3a459a00c9a
 SHA512:
-  metadata.gz: 8bb7a590213347f0ea7f9da3e42addd4abd2d790fcfea5becba2acc964fc1188a46373dae31277dcc3f8eb9fade5faa24620cbafbd3031b84f153d6c14d8fa00
-  data.tar.gz: 19998dd7f54661abb0d0e0530e962d95482c48543fabfc50f39c03f529fee299a48c24283ce2e49c6d190b82b820688584dbd95a6fb86b1a53b592c1ddf87b94
+  metadata.gz: f4980f5d2f6d3c2081c42b1c5a2761fa10470b820ac75db0d1eda27e2b85fc4d973ddf901b69acc0aa43d0ab09e688934f7b847c6ed2c354431fa73c4e76d2ab
+  data.tar.gz: 2be68549771581ee9e470bcc2664036206fbaa973d6adecdfef4b7f03993c3b2b9b8b45e170ed11da4293453925c3bdebd38192c00937edaa2ac555b15c4de63

data/.rubocop.yml

@@ -22,7 +22,11 @@
       - "./spec/*_helper*.rb"
       - "./tasks/ci.rake"
       - "Gemfile"
-  
+
+  Layout/LineLength:
+    Exclude:
+      - "./**/*_spec.rb"
+
   Metrics/ModuleLength:
     Exclude:
       - "./**/*_spec*.rb"
@@ -31,5 +35,5 @@
     Exclude:
       - "./**/*_spec*.rb"
 
-  Rails/ApplicationRecord:
+  Rails:
     Enabled: false

data/CHANGELOG.md

@@ -1,7 +1,10 @@
 Changelog
 =========
 
-## 0.3.0.jlauer2 04/22/2020
+## 0.4.0.jlauer1 04/24/2020
+  * Add SQL sanitizers
+
+## 0.3.0 04/22/2020
   * Set up build pipeline with circleci and gem-publisher
   * Fixed linting issues
   * Renamed gem to `activerecord-instrumentation`

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    activerecord-instrumentation (0.3.0.jlauer2)
+    activerecord-instrumentation (0.4.0.jlauer1)
       activerecord (~> 6.0)
       opentracing (~> 0.5)
 

data/lib/active_record/open_tracing.rb

@@ -5,11 +5,13 @@ require "opentracing"
 
 require "active_record/open_tracing/version"
 require "active_record/open_tracing/processor"
+require "active_record/open_tracing/sql_sanitizer"
 
 module ActiveRecord
   module OpenTracing
-    def self.instrument(tracer: ::OpenTracing.global_tracer)
-      processor = Processor.new(tracer)
+    def self.instrument(tracer: ::OpenTracing.global_tracer, sanitizer: nil)
+      sql_sanitizer = sanitizer && SqlSanitizer.build_sanitizer(sanitizer)
+      processor = Processor.new(tracer, sanitizer: sql_sanitizer)
 
       ActiveSupport::Notifications.subscribe("sql.active_record") do |*args|
         processor.call(*args)

data/lib/active_record/open_tracing/processor.rb

@@ -8,12 +8,15 @@ module ActiveRecord
       SPAN_KIND = "client"
       DB_TYPE = "sql"
 
-      def initialize(tracer)
+      attr_reader :tracer, :sanitizer
+
+      def initialize(tracer, sanitizer: nil)
         @tracer = tracer
+        @sanitizer = sanitizer
       end
 
       def call(_event_name, start, finish, _id, payload)
-        span = @tracer.start_span(
+        span = tracer.start_span(
           payload[:name] || DEFAULT_OPERATION_NAME,
           start_time: start,
           tags: tags_for_payload(payload)
@@ -45,12 +48,16 @@ module ActiveRecord
           "span.kind" => SPAN_KIND,
           "db.instance" => db_instance,
           "db.cached" => payload.fetch(:cached, false),
-          "db.statement" => payload.fetch(:sql).squish,
+          "db.statement" => sanitize_sql(payload.fetch(:sql).squish),
           "db.type" => DB_TYPE,
           "peer.address" => db_address
         }
       end
 
+      def sanitize_sql(sql)
+        sanitizer ? sanitizer.sanitize(sql) : sql
+      end
+
       def db_instance
         @db_instance ||= db_config.fetch(:database)
       end

data/lib/active_record/open_tracing/sql_sanitizer.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require "active_record/open_tracing/sql_sanitizer/base"
+require "active_record/open_tracing/sql_sanitizer/mysql"
+require "active_record/open_tracing/sql_sanitizer/postgres"
+require "active_record/open_tracing/sql_sanitizer/sql_server"
+require "active_record/open_tracing/sql_sanitizer/sqlite"
+require "active_record/open_tracing/sql_sanitizer/regexes"
+
+module ActiveRecord
+  module OpenTracing
+    module SqlSanitizer
+      KLASSES = {
+        mysql: Mysql,
+        postgres: Postgres,
+        sql_server: SqlServer,
+        sqlite: Sqlite
+      }.freeze
+
+      class << self
+        def build_sanitizer(sanitizer_name)
+          sanitizer_klass(sanitizer_name).build
+        end
+
+        private
+
+        def sanitizer_klass(sanitizer_name)
+          key = KLASSES.keys.detect do |name|
+            sanitizer_name.to_sym == name
+          end || (raise NameError, "Unknown sanitizer #{sanitizer_name.inspect}")
+
+          KLASSES.fetch(key)
+        end
+      end
+    end
+  end
+end

data/lib/active_record/open_tracing/sql_sanitizer/base.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module OpenTracing
+    module SqlSanitizer
+      class Base
+        require "active_record/open_tracing/sql_sanitizer/regexes"
+        include ActiveRecord::OpenTracing::SqlSanitizer::Regexes
+
+        def sanitize(sql)
+          scrubbed = scrub(sql.dup)
+          apply_substitutions(scrubbed)
+        end
+
+        private
+
+        def substitutions
+          raise NotImplementedError
+        end
+
+        def apply_substitutions(str)
+          substitutions.inject(str.dup) do |memo, (regex, replacement)|
+            if replacement.respond_to?(:call)
+              memo.gsub(regex, &replacement)
+            else
+              memo.gsub(regex, replacement)
+            end
+          end.strip
+        end
+
+        def encodings?(encodings = %w[UTF-8 binary])
+          encodings.all? do |enc|
+            begin
+              Encoding.find(enc)
+            rescue StandardError
+              false
+            end
+          end
+        end
+
+        MAX_SQL_LENGTH = 16384
+
+        def scrub(str)
+          # safeguard - don't sanitize or scrub large SQL statements
+          return "" if !str.is_a?(String) || str.length > MAX_SQL_LENGTH
+
+          # Whatever encoding it is, it is valid and we can operate on it
+          return str if str.valid_encoding?
+
+          # Prefer scrub over convert
+          if str.respond_to?(:scrub)
+            str.scrub("_")
+          elsif encodings?(%w[UTF-8 binary])
+            str.encode("UTF-8", "binary", invalid: :replace, undef: :replace, replace: "_")
+          else
+            # Unable to scrub invalid sql encoding, returning empty string
+            ""
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_record/open_tracing/sql_sanitizer/mysql.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module OpenTracing
+    module SqlSanitizer
+      class Mysql < Base
+        def substitutions
+          [
+            [MYSQL_VAR_INTERPOLATION, ""],
+            [MYSQL_REMOVE_SINGLE_QUOTE_STRINGS, "?"],
+            [MYSQL_REMOVE_DOUBLE_QUOTE_STRINGS, "?"],
+            [MYSQL_REMOVE_INTEGERS, "?"],
+            [MYSQL_IN_CLAUSE, "IN (?)"],
+            [MULTIPLE_QUESTIONS, "?"]
+          ]
+        end
+      end
+    end
+  end
+end

data/lib/active_record/open_tracing/sql_sanitizer/postgres.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module OpenTracing
+    module SqlSanitizer
+      class Postgres < Base
+        def substitutions
+          [
+            [PSQL_PLACEHOLDER, "?"],
+            [PSQL_VAR_INTERPOLATION, ""],
+            [PSQL_AFTER_WHERE, ->(c) { c.gsub(PSQL_REMOVE_STRINGS, "?") }],
+            [PSQL_REMOVE_INTEGERS, "?"],
+            [PSQL_IN_CLAUSE, "IN (?)"],
+            [MULTIPLE_SPACES, " "]
+          ]
+        end
+      end
+    end
+  end
+end

data/lib/active_record/open_tracing/sql_sanitizer/regexes.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module OpenTracing
+    module SqlSanitizer
+      module Regexes
+        MULTIPLE_SPACES    = /\s+/.freeze
+        MULTIPLE_QUESTIONS = /\?(,\?)+/.freeze
+
+        PSQL_VAR_INTERPOLATION = /\[\[.*\]\]\s*$/.freeze
+        PSQL_REMOVE_STRINGS = /'(?:[^']|'')*'/.freeze
+        PSQL_REMOVE_INTEGERS = /(?<!LIMIT )\b\d+\b/.freeze
+        PSQL_PLACEHOLDER = /\$\d+/.freeze
+        PSQL_IN_CLAUSE = /IN\s+\(\?[^\)]*\)/.freeze
+        PSQL_AFTER_WHERE = /(?:WHERE\s+).*?(?:SELECT|$)/i.freeze
+
+        MYSQL_VAR_INTERPOLATION = /\[\[.*\]\]\s*$/.freeze
+        MYSQL_REMOVE_INTEGERS = /(?<!LIMIT )\b\d+\b/.freeze
+        MYSQL_REMOVE_SINGLE_QUOTE_STRINGS = /'(?:\\'|[^']|'')*'/.freeze
+        MYSQL_REMOVE_DOUBLE_QUOTE_STRINGS = /"(?:\\"|[^"]|"")*"/.freeze
+        MYSQL_IN_CLAUSE = /IN\s+\(\?[^\)]*\)/.freeze
+
+        SQLITE_VAR_INTERPOLATION = /\[\[.*\]\]\s*$/.freeze
+        SQLITE_REMOVE_STRINGS = /'(?:[^']|'')*'/.freeze
+        SQLITE_REMOVE_INTEGERS = /(?<!LIMIT )\b\d+\b/.freeze
+
+        SQLSERVER_EXECUTESQL = /EXEC sp_executesql N'(.*?)'.*/.freeze
+        SQLSERVER_REMOVE_INTEGERS = /(?<!LIMIT )\b(?<!@)\d+\b/.freeze
+        SQLSERVER_IN_CLAUSE = /IN\s+\(\?[^\)]*\)/.freeze
+      end
+    end
+  end
+end

data/lib/active_record/open_tracing/sql_sanitizer/sql_server.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module OpenTracing
+    module SqlSanitizer
+      class SqlServer < Base
+        def substitutions
+          [
+            [SQLSERVER_EXECUTESQL, '\1'],
+            [SQLSERVER_REMOVE_INTEGERS, "?"],
+            [SQLSERVER_IN_CLAUSE, "IN (?)"]
+          ]
+        end
+      end
+    end
+  end
+end

data/lib/active_record/open_tracing/sql_sanitizer/sqlite.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module OpenTracing
+    module SqlSanitizer
+      class Sqlite < Base
+        def substitutions
+          [
+            [SQLITE_VAR_INTERPOLATION, ""],
+            [SQLITE_REMOVE_STRINGS, "?"],
+            [SQLITE_REMOVE_INTEGERS, "?"],
+            [MULTIPLE_SPACES, " "]
+          ]
+        end
+      end
+    end
+  end
+end

data/lib/active_record/open_tracing/version.rb

@@ -2,6 +2,6 @@
 
 module ActiveRecord
   module OpenTracing
-    VERSION = "0.3.0.jlauer2"
+    VERSION = "0.4.0.jlauer1"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: activerecord-instrumentation
 version: !ruby/object:Gem::Version
-  version: 0.3.0.jlauer2
+  version: 0.4.0.jlauer1
 platform: ruby
 authors:
 - SaleMove TechMovers
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-22 00:00:00.000000000 Z
+date: 2020-04-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -200,6 +200,13 @@ files:
 - activerecord-instrumentation.gemspec
 - lib/active_record/open_tracing.rb
 - lib/active_record/open_tracing/processor.rb
+- lib/active_record/open_tracing/sql_sanitizer.rb
+- lib/active_record/open_tracing/sql_sanitizer/base.rb
+- lib/active_record/open_tracing/sql_sanitizer/mysql.rb
+- lib/active_record/open_tracing/sql_sanitizer/postgres.rb
+- lib/active_record/open_tracing/sql_sanitizer/regexes.rb
+- lib/active_record/open_tracing/sql_sanitizer/sql_server.rb
+- lib/active_record/open_tracing/sql_sanitizer/sqlite.rb
 - lib/active_record/open_tracing/version.rb
 homepage: https://github.com/doximity/ruby-activerecord-opentracing
 licenses: