forge_ops_tracker 0.10.2 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9b32a031e24fb613c219aee3bf01f0517043b05b4d1788d7ece456096d7561a0
-  data.tar.gz: 39c7f16920425d01862607b6d3f85b957c694ab009ce8ffd010f33714632a3a9
+  metadata.gz: af67d3b6d07322ccfd80d56256e5e600c264f60394058acab51b7bfc660435fa
+  data.tar.gz: 2716944aed397bb03c754900b2e34f3abc49247b43b6a97b411cec6b6a3ccbc2
 SHA512:
-  metadata.gz: 315fb4765179a298da43dd2beabe68970f257dee7b4b1f3938bcc6efa72ac403b8dbb0e39b8d16dfcaf19b76696aed961270dc88fd92731b05ae1610b3d4a223
-  data.tar.gz: f2907bae8015b4093346a30369cc3da38c8d36f4198ae4935cb1e2d2cea2e6baed44d5fcab4a8278029f228353d991275316fd54a9e08bd3116519b529f09159
+  metadata.gz: 2952a5218507cfceb871821b1f79add47d255b45577ee4bc67a782b574e3d41135d1f971d966846666c3bbcc1de6e86e3c3c7671e8a5ec04b31d9dabf4af0cc4
+  data.tar.gz: 32efc755d0d05fca0dd4f4354d00808caf135da2c0a22ad2b9d417017a3e73406b6d1b879da3e5862724340d52222fec9d1fbbfebc45efee58101e824b105373

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## 0.11.0
+
+- Database errors now say where to look. When an error comes from a database call
+  (`ActiveRecord::StatementInvalid`, or an exception of yours raised from one), the event carries the
+  names of the stored procedure or function and the tables or views its SQL touched, so ForgeOps can
+  show them on the issue. On by default (`config.capture_sql_objects`); names are identifiers, never
+  values. New opt-in `config.capture_sql_statement` (default false) also sends the statement itself,
+  with every string and number replaced by `?`. Each project has its own server-side setting that
+  can stop the statement being stored regardless of this flag; the names are still kept.
+
 ## 0.10.2
 
 - Documentation only: the README and package description now describe ForgeOps as a hosted service, link to getforgeops.net, and show the real host in the connection string example instead of a placeholder. No code changes.

data/README.md

@@ -187,6 +187,29 @@ ForgeOpsTracker.configure do |config|
 end
 ```
 
+## Database errors
+
+When an error comes from a database call (`ActiveRecord::StatementInvalid`, or your own exception
+raised from one), the event carries the names of the stored procedure or function and the tables or
+views its SQL touched, so the issue tells you where to start looking. This is on by default and
+sends identifiers only, never values. A view and a table are written the same way in SQL, so both
+show as tables/views; the database's own error message usually settles which it was.
+
+To also send the SQL statement itself, opt in. Every string and number is replaced by `?` before it
+leaves your process (`WHERE email = 'a@b.co' AND id = 42` is sent as `WHERE email = ? AND id = ?`),
+and ForgeOps masks it again on arrival:
+
+```ruby
+ForgeOpsTracker.configure do |config|
+  config.capture_sql_statement = true # default false
+  config.capture_sql_objects = false  # default true; false stops even the names
+end
+```
+
+Each ForgeOps project also has its own "Capture the SQL behind database errors" setting. Turn it off
+there and the statement is never stored for that project, whatever this flag says; the names are
+still kept.
+
 ## Session tracking (release health)
 
 By default, every request is counted as a session: crash-free unless an unhandled exception

data/lib/forge_ops_tracker/configuration.rb

@@ -8,6 +8,7 @@ module ForgeOpsTracker
     attr_accessor :dsn, :environment, :release, :server_name, :app_root, :logger
     attr_accessor :enabled_environments, :queue_size, :open_timeout, :read_timeout, :scrub_pii
     attr_accessor :capture_source_context
+    attr_accessor :capture_sql_objects, :capture_sql_statement
     attr_accessor :track_sessions, :session_flush_interval
     attr_accessor :track_performance, :performance_flush_interval
     attr_accessor :track_current_user
@@ -45,6 +46,15 @@ module ForgeOpsTracker
       # off here too if this host app never wants that disk read attempted
       # in the first place.
       @capture_source_context = true
+      # When an error comes from a database call (ActiveRecord::StatementInvalid and anything that
+      # wraps one), send the names of the stored procedure, table and view its SQL touched, so an
+      # issue says where to start looking. Names are identifiers, never values, which is why this
+      # defaults on. capture_sql_statement is the separate, opt-in step of also sending the
+      # statement itself, with every string and number replaced by "?"; off by default because
+      # even a masked statement describes the customer's schema, and ForgeOps' own per-project
+      # setting is what durably governs whether the server stores it. See SqlStatement.
+      @capture_sql_objects = true
+      @capture_sql_statement = false
       # Auto-instruments every request the moment the gem loads, the same "on unless you turn it
       # off" default error tracking itself already has; nothing else in this gem is opt-in. See
       # ForgeOpsTracker::Middleware::SessionTracking for what this actually wraps.

data/lib/forge_ops_tracker/event_builder.rb

@@ -42,6 +42,7 @@ module ForgeOpsTracker
       }
       payload[:user] = user if user && !user.empty?
       payload[:breadcrumbs] = breadcrumbs if breadcrumbs && !breadcrumbs.empty?
+      attach_sql(payload, error)
       scrub(payload)
     end
 
@@ -66,10 +67,25 @@ module ForgeOpsTracker
           context: PiiScrubber.scrub(payload[:context]),
           tags: PiiScrubber.scrub(payload[:tags])
         )
+        scrubbed[:sql_statement] = PiiScrubber.scrub(payload[:sql_statement]) if payload.key?(:sql_statement)
         scrubbed[:breadcrumbs] = PiiScrubber.scrub(payload[:breadcrumbs]) if payload.key?(:breadcrumbs)
         scrubbed
       end
 
+      # See SqlStatement for what's read off the error and how it's masked. The statement itself
+      # only goes out when capture_sql_statement is on; the extracted names go out on their own
+      # (capture_sql_objects) so an issue can still name the procedure or view involved.
+      def attach_sql(payload, error)
+        return unless configuration.capture_sql_objects || configuration.capture_sql_statement
+
+        masked = SqlStatement.mask(SqlStatement.find_in(error))
+        return unless masked
+
+        objects = SqlStatement.objects(masked)
+        payload[:sql_objects] = objects if objects && configuration.capture_sql_objects
+        payload[:sql_statement] = masked if configuration.capture_sql_statement
+      end
+
       def backtrace_frames(error)
         Array(error.backtrace).first(MAX_FRAMES).filter_map { |line| parse_backtrace_line(line) }
       end

data/lib/forge_ops_tracker/sql_statement.rb

@@ -0,0 +1,95 @@
+module ForgeOpsTracker
+  # Finds the SQL behind a database error and reduces it to something safe to send: the names of
+  # the stored procedures, tables and views it touched, and (only if configuration.
+  # capture_sql_statement is on) the statement itself with every string and number replaced by
+  # "?". Ported from the server's own SqlStatementMasker/SqlObjectExtractor, same rules; the
+  # server applies them again on arrival, so a difference here can only ever mean less is
+  # masked client-side, never that something unmasked gets stored.
+  #
+  # Deliberately a single pass over a few patterns, not a SQL parser; see the server's own
+  # comments for the reasoning behind each choice, which apply unchanged here.
+  module SqlStatement
+    MASK = "?".freeze
+    MAX_LENGTH = 4_000
+    MAX_NAMES = 10
+    MAX_NAME_LENGTH = 200
+    MAX_CAUSE_DEPTH = 5
+
+    LITERAL = /
+      '(?:[^']|'')*(?:'|\z)                      # string literal, or one cut off by truncation
+      | (?<tag>\$[A-Za-z_]*\$).*?(?:\k<tag>|\z)  # dollar-quoted string
+      | (?<![\w$.])\d+(?:\.\d+)?(?!\w)           # number, not part of an identifier or placeholder
+    /mx
+
+    PART = /(?:[\w$#@]+|"[^"]+"|\[[^\]]+\]|`[^`]+`)/
+    NAME = /#{PART}(?:\.#{PART})*/
+    OPERATIONS = %w[SELECT INSERT UPDATE DELETE MERGE WITH CALL EXEC EXECUTE CREATE ALTER DROP TRUNCATE].freeze
+    PROCEDURE_CALL = /\b(?:CALL|EXEC(?:UTE)?|PERFORM)\s+(?!IMMEDIATE\b|FUNCTION\b|PROCEDURE\b)(#{NAME})/i
+    RELATION = /\b(FROM|JOIN|INTO|UPDATE|TABLE)\s+(#{NAME})(\s*\()?/i
+    SELECT_FUNCTION = /\A\s*SELECT\s+(#{NAME})\s*\(/i
+    BUILTINS = %w[
+      count sum min max avg now coalesce nullif lower upper length concat cast date_trunc
+      current_timestamp current_date row_number rank json_build_object json_agg array_agg
+    ].freeze
+    FROM_INSIDE_FUNCTION = /\b(?:EXTRACT|SUBSTRING|TRIM|OVERLAY)\s*\([^()]*\)/i
+    KEYWORDS_NOT_NAMES = %w[select set values where lateral only unnest generate_series].freeze
+
+    module_function
+
+    # The raw statement off the error itself or, for an app that wraps a database error in its
+    # own exception, off whatever it was raised from. ActiveRecord::StatementInvalid#sql is the
+    # one source this reads; nothing else in Ruby exposes the statement on the exception.
+    def find_in(error)
+      depth = 0
+      while error && depth < MAX_CAUSE_DEPTH
+        sql = error.sql if error.respond_to?(:sql)
+        return sql.to_s if sql.is_a?(String) && !sql.strip.empty?
+
+        error = error.cause
+        depth += 1
+      end
+      nil
+    end
+
+    def mask(statement)
+      return nil if statement.nil? || statement.strip.empty?
+
+      masked = statement.gsub(LITERAL, MASK)
+      masked.length > MAX_LENGTH ? "#{masked[0, MAX_LENGTH]}..." : masked
+    end
+
+    # Takes an already-masked statement (so a keyword inside a string value can't be mistaken for
+    # SQL). Returns nil when nothing recognizable was found.
+    def objects(masked)
+      return nil if masked.nil? || masked.strip.empty?
+
+      sql = masked.gsub(FROM_INSIDE_FUNCTION, " ")
+      procedures = sql.scan(PROCEDURE_CALL).flatten
+      relations = []
+
+      sql.scan(RELATION) do |keyword, name, paren|
+        next if KEYWORDS_NOT_NAMES.include?(name.downcase)
+
+        function_call = paren && %w[FROM JOIN].include?(keyword.upcase)
+        (function_call ? procedures : relations) << name
+      end
+
+      function = sql[SELECT_FUNCTION, 1]
+      procedures << function if function && !BUILTINS.include?(function.downcase) && !sql.match?(/\bFROM\b/i)
+
+      operation = sql[/\A\s*(\w+)/, 1].to_s.upcase
+      result = {
+        operation: (operation if OPERATIONS.include?(operation)),
+        procedures: clean(procedures),
+        relations: clean(relations)
+      }.compact
+      result[:procedures].empty? && result[:relations].empty? && !result.key?(:operation) ? nil : result
+    end
+
+    def clean(names)
+      names.map { |name| name.to_s.strip[0, MAX_NAME_LENGTH] }
+           .select { |name| name.match?(/\A#{NAME}\z/) }
+           .uniq.first(MAX_NAMES)
+    end
+  end
+end

data/lib/forge_ops_tracker/version.rb

@@ -1,3 +1,3 @@
 module ForgeOpsTracker
-  VERSION = "0.10.2"
+  VERSION = "0.11.0"
 end

data/lib/forge_ops_tracker.rb

@@ -2,6 +2,7 @@ require "securerandom"
 require "forge_ops_tracker/version"
 require "forge_ops_tracker/configuration"
 require "forge_ops_tracker/pii_scrubber"
+require "forge_ops_tracker/sql_statement"
 require "forge_ops_tracker/event_builder"
 require "forge_ops_tracker/client"
 require "forge_ops_tracker/delivery_queue"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: forge_ops_tracker
 version: !ruby/object:Gem::Version
-  version: 0.10.2
+  version: 0.11.0
 platform: ruby
 authors:
 - ForgeOps
@@ -117,6 +117,7 @@ files:
 - lib/forge_ops_tracker/session_flusher.rb
 - lib/forge_ops_tracker/span_buffer.rb
 - lib/forge_ops_tracker/span_queue.rb
+- lib/forge_ops_tracker/sql_statement.rb
 - lib/forge_ops_tracker/version.rb
 homepage: https://getforgeops.net
 licenses: