influxdb-rails 1.0.0.beta1 → 1.0.0.beta2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 68333c3962daf87c05e1ff1b221accc5854e4c7d671f0d0441a5c789b2322cee
-  data.tar.gz: 56017328de99c335fd4c590fe80c2d9de1b9132f39a7d3ac6d07b87b551597f9
+  metadata.gz: e7b5a411b14d5a465a1a4c503498d8c642de1c818213dfe3e437a1177eefa5d1
+  data.tar.gz: f50dc54acb96ac4e9b758aa5be109ffd40a1396b4e43844800fc670496c1d76c
 SHA512:
-  metadata.gz: e6c6087522e487aaefabe377698d2275270070c0e9d70eb7110fd90000f2160fc03b127879e39481ca0ff7f1f6bc5010075c4d9fe64bd30ecebd9b04b7e98df6
-  data.tar.gz: baf2150e6140c85119da580cb89b06204a0372277c6adf7c57f15b89c214e41d80f05e0b47447df939eecc26dfb27115787750d92a22ae3ced38a047448f97bc
+  metadata.gz: 123a265d9418ff25a10b469cb798e9ed004af8a679fde861c40f81c5f96a20aea192576d10dfed877293afa6c59412cccb40d0b7fd14e52461aaa76c6712c092
+  data.tar.gz: 6189e83dfce1d8812c0ce8553f3095ee0be21608f71db0375edc1e6aab4cfaf646675d481ed53144056437b853df71a9e9085009fea936384a167bdac41d0433

data/.gitignore

@@ -2,6 +2,7 @@
 .rvmrc
 *.gem
 Gemfile.lock
+Gemfile.local
 gemfiles/*.lock
 pkg/*
 spec/support/rails*/log/*

data/.rspec

@@ -1,2 +1,3 @@
 --format progress
 --color
+--order rand

data/.rubocop.yml

@@ -8,6 +8,7 @@ AllCops:
   - 'bin/**/*'
   - 'smoke/**/*'
   - 'Gemfile'
+  - 'vendor/bundle/**/*'
   DisplayCopNames: true
   StyleGuideCopsOnly: false
   TargetRubyVersion: 2.3

data/.travis.yml

@@ -32,4 +32,4 @@ addons:
     packages:
       - haveged
       - libgmp-dev
-script: bundle exec rake spec
+script: bundle exec rake $TEST_TASK

data/CHANGELOG.md

@@ -3,6 +3,14 @@
 For the full commit log, [see here](https://github.com/influxdata/influxdb-rails/commits/master).
 
 
+## v1.0.0.beta2, released 2018-12-07
+
+- Added `tags_middleware` config option (#47, @ @Kukunin)
+- Removed path tag from metrics (introduced with #50), because it
+  potentially produces "exceed tag value limit" (#54, @ChrisBr)
+- Added render instrumentation (#53, @ChrisBr)
+- Added SQL instrumentation (#55, @ChrisBr)
+
 ## v1.0.0.beta1, released 2018-11-22
 
 - Added app name to the measurement's tag sets (#44, @stefanhorning)

data/Gemfile

@@ -1,3 +1,9 @@
 source "https://rubygems.org"
 
 gemspec
+
+local_gemfile = 'Gemfile.local'
+
+if File.exist?(local_gemfile)
+  eval(File.read(local_gemfile)) # rubocop:disable Lint/Eval
+end

data/README.md

@@ -11,9 +11,10 @@
 Automatically instrument your Ruby on Rails applications and write the
 metrics directly into [InfluxDB](http://influxdb.org/).
 
-This gem is designed for Rails 4.0+, Ruby 2.2+ and InfluxDB 0.9+.
+This gem is designed for Rails 4.2+, Ruby 2.3+ and InfluxDB 0.9+.
 
-## Install
+
+## Installation
 
 ```
 $ [sudo] gem install influxdb-rails
@@ -21,6 +22,7 @@ $ [sudo] gem install influxdb-rails
 
 Or add it to your `Gemfile`, etc.
 
+
 ## Usage
 
 To get things set up, just create an initializer:
@@ -48,9 +50,15 @@ InfluxDB::Rails.configure do |config|
   # config.max_delay = 300
   # config.time_precision = 'ms'
 
+  # config.tags_middleware = ->(tags) { tags }
+
   # config.series_name_for_controller_runtimes = "rails.controller"
   # config.series_name_for_view_runtimes       = "rails.view"
   # config.series_name_for_db_runtimes         = "rails.db"
+  # config.series_name_for_render_template     = "rails.render_template"
+  # config.series_name_for_render_partial      = "rails.render_partial"
+  # config.series_name_for_render_collection   = "rails.render_collection"
+  # config.series_name_for_sql                 = nil
   # config.series_name_for_exceptions          = "rails.exceptions"
   # config.series_name_for_instrumentation     = "instrumentation"
 
@@ -64,8 +72,21 @@ To see all default values, take a look into `InfluxDB::Rails::Configuration::DEF
 defined in `lib/influxdb/rails/configuration.rb`
 
 Out of the box, you'll automatically get reporting of your controller,
-view, and db runtimes for each request. You can also call through to the
-underlying `InfluxDB::Client` object to write arbitrary data like this:
+view, and db runtimes and rendering of template, partial and collection for each request.
+Reporting of SQL queries is disabled by default because it is still in experimental mode
+and currently requires String parsing which might cause performance issues on query
+intensive applications. You can enable it by setting the `series_name_for_sql`
+configuration.
+
+It is possible to disable the rendering series by setting the series_name to nil.
+
+```ruby
+  # config.series_name_for_render_template     = nil
+  # config.series_name_for_render_partial      = nil
+  # config.series_name_for_render_collection   = nil
+```
+
+You can also call through to the underlying `InfluxDB::Client` object to write arbitrary data like this:
 
 ``` ruby
 InfluxDB::Rails.client.write_point "events",
@@ -76,6 +97,53 @@ InfluxDB::Rails.client.write_point "events",
 Additional documentation for `InfluxDB::Client` lives in the
 [influxdb-ruby](http://github.com/influxdata/influxdb-ruby) repo.
 
+
+### Tags
+
+You can modify the tags sent to InfluxDB by defining a middleware, which
+receives the current tag set (`Hash` with `Symbol` keys and `String`
+values) as argument and returns a hash in the same form. The middleware
+can be any object, as long it responds to `#call` (like a `Proc`):
+
+```ruby
+InfluxDB::Rails.configure do |config|
+  config.tags_middleware = lambda do |tags|
+    tags.merge(env: Rails.env)
+  end
+end
+```
+
+By default, the following tags are sent for *non-exception series*
+(`rails.controller`, `rails.view`, `rails.db` and `instrumentation`):
+
+```ruby
+{
+  method:   "#{payload[:controller]}##{payload[:action]}",
+  server:   Socket.gethostname,
+  app_name: configuration.application_name,
+}
+```
+
+and for the exceptions (series name `rails.exceptions`):
+
+```ruby
+{
+  application_name:   InfluxDB::Rails.configuration.application_name,
+  application_root:   InfluxDB::Rails.configuration.application_root,
+  framework:          InfluxDB::Rails.configuration.framework,
+  framework_version:  InfluxDB::Rails.configuration.framework_version,
+  language:           "Ruby",
+  language_version:   "#{RUBY_VERSION}-p#{RUBY_PATCHLEVEL}",
+  custom_data:        @custom_data,
+  class:              @exception.class.to_s,
+  method:             "#{@controller}##{@action}",
+  filename:           File.basename(@backtrace.lines.first.try(:file)),
+  server:             Socket.gethostname,
+  status:             "open",
+}
+```
+
+
 ## Frequently Asked Questions
 
 
@@ -157,6 +225,7 @@ bundle
 bundle exec rake
 ```
 
+
 ## Contributing
 
 - Fork this repository on GitHub.
@@ -165,11 +234,8 @@ bundle exec rake
   - Add an entry in the `CHANGELOG.md` in the "unreleased" section on top.
 - Run the tests:
   - Either run them manually:
-    ```sh
-    for gemfile in gemfiles/Gemfile.rails-*.x; do     \
-      BUNDLE_GEMFILE=$gemfile bundle install --quiet; \
-      BUNDLE_GEMFILE=$gemfile bundle exec rspec;      \
-    done
+    ```console
+    $ rake test:all
     ```
   - or wait for [Travis][travis-pr] to pick up your changes, *after*
     you made a pull request.

data/Rakefile

@@ -2,28 +2,27 @@ require "bundler/gem_tasks"
 require "rubocop/rake_task"
 RuboCop::RakeTask.new
 
-begin
-  targeted_files = ARGV.drop(1)
-  file_pattern = targeted_files.empty? ? "spec/**/*_spec.rb" : targeted_files
+targeted_files = ARGV.drop(1)
+file_pattern = targeted_files.empty? ? "spec/**/*_spec.rb" : targeted_files
 
-  require "rspec/core"
-  require "rspec/core/rake_task"
+require "rspec/core"
+require "rspec/core/rake_task"
 
-  RSpec::Core::RakeTask.new(:spec) do |t|
-    t.pattern = FileList[file_pattern]
-  end
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.pattern = FileList[file_pattern]
+end
 
-  RSpec.configure do |config|
-    config.color = true
-    config.formatter = :documentation
-  end
-rescue LoadError
-  require "spec/rake/spectask"
+RSpec.configure do |config|
+  config.color = true
+  config.formatter = :documentation
+end
 
-  puts file_pattern
+task default: %i[spec rubocop]
 
-  Spec::Rake::SpecTask.new(:spec) do |t|
-    t.pattern = FileList[file_pattern]
-    t.spec_opts += ["--color"]
+task "test:all" => :default do
+  Dir.glob("gemfiles/Gemfile.rails-*.x") do |gemfile|
+    puts RSpec::Core::Formatters::ConsoleCodes.wrap(gemfile, :cyan)
+    sh({ "BUNDLE_GEMFILE" => gemfile }, "bundle", "install", "--quiet", "--retry=2", "--jobs=2")
+    sh({ "BUNDLE_GEMFILE" => gemfile }, "bundle", "exec", "rspec")
   end
 end

data/influxdb-rails.gemspec

@@ -19,9 +19,10 @@ Gem::Specification.new do |spec|
 
   spec.required_ruby_version = ">= 2.3.0"
 
-  spec.add_runtime_dependency "influxdb", "~> 0.5.0"
-  spec.add_runtime_dependency "railties", "> 3"
+  spec.add_runtime_dependency "influxdb", "~> 0.6", ">= 0.6.4"
+  spec.add_runtime_dependency "railties", ">= 4.2"
 
+  spec.add_development_dependency "activerecord"
   spec.add_development_dependency "bundler", ">= 1.0.0"
   spec.add_development_dependency "fakeweb"
   spec.add_development_dependency "rake"
@@ -29,5 +30,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rspec"
   spec.add_development_dependency "rspec-rails", ">= 3.0.0"
   spec.add_development_dependency "rubocop", "~> 0.60.0"
+  spec.add_development_dependency "sqlite3"
   spec.add_development_dependency "tzinfo"
 end

data/lib/influxdb/rails/configuration.rb

@@ -21,7 +21,12 @@ module InfluxDB
       attr_accessor :series_name_for_db_runtimes
       attr_accessor :series_name_for_exceptions
       attr_accessor :series_name_for_instrumentation
+      attr_accessor :series_name_for_render_template
+      attr_accessor :series_name_for_render_partial
+      attr_accessor :series_name_for_render_collection
+      attr_accessor :series_name_for_sql
 
+      attr_accessor :tags_middleware
       attr_accessor :rails_app_name
 
       attr_accessor :application_name
@@ -65,7 +70,12 @@ module InfluxDB
         series_name_for_db_runtimes:         "rails.db".freeze,
         series_name_for_exceptions:          "rails.exceptions".freeze,
         series_name_for_instrumentation:     "instrumentation".freeze,
+        series_name_for_render_template:     "rails.render_template".freeze,
+        series_name_for_render_partial:      "rails.render_partial".freeze,
+        series_name_for_render_collection:   "rails.render_collection".freeze,
+        series_name_for_sql:                 nil,
 
+        tags_middleware:                     ->(tags) { tags },
         rails_app_name:                      nil,
 
         ignored_exceptions:                  %w[
@@ -124,7 +134,12 @@ module InfluxDB
         @series_name_for_db_runtimes          = DEFAULTS[:series_name_for_db_runtimes]
         @series_name_for_exceptions           = DEFAULTS[:series_name_for_exceptions]
         @series_name_for_instrumentation      = DEFAULTS[:series_name_for_instrumentation]
+        @series_name_for_render_template      = DEFAULTS[:series_name_for_render_template]
+        @series_name_for_render_partial       = DEFAULTS[:series_name_for_render_partial]
+        @series_name_for_render_collection    = DEFAULTS[:series_name_for_render_collection]
+        @series_name_for_sql                  = DEFAULTS[:series_name_for_sql]
 
+        @tags_middleware = DEFAULTS[:tags_middleware]
         @rails_app_name = DEFAULTS[:rails_app_name]
 
         @ignored_exceptions           = DEFAULTS[:ignored_exceptions].dup

data/lib/influxdb/rails/instrumentation.rb

@@ -1,7 +1,7 @@
 module InfluxDB
   module Rails
     module Instrumentation # rubocop:disable Style/Documentation
-      def benchmark_for_instrumentationn # rubocop:disable Metrics/MethodLength
+      def benchmark_for_instrumentation # rubocop:disable Metrics/MethodLength
         start = Time.now
         yield
 
@@ -13,10 +13,10 @@ module InfluxDB
           values: {
             value: ((Time.now - start) * 1000).ceil,
           },
-          tags:   {
+          tags:   configuration.tags_middleware.call(
             method: "#{controller_name}##{action_name}",
-            server: Socket.gethostname,
-          }
+            server: Socket.gethostname
+          )
       end
 
       def self.included(base)

data/lib/influxdb/rails/middleware/render_subscriber.rb

@@ -0,0 +1,20 @@
+require "influxdb/rails/middleware/simple_subscriber"
+
+module InfluxDB
+  module Rails
+    module Middleware
+      class RenderSubscriber < SimpleSubscriber # :nodoc:
+        private
+
+        def tags(payload)
+          {
+            location:   location,
+            filename:   payload[:identifier],
+            count:      payload[:count],
+            cache_hits: payload[:cache_hits],
+          }.reject { |_, value| value.blank? }
+        end
+      end
+    end
+  end
+end

data/lib/influxdb/rails/middleware/request_subscriber.rb

@@ -0,0 +1,51 @@
+require "influxdb/rails/middleware/subscriber"
+
+module InfluxDB
+  module Rails
+    module Middleware
+      class RequestSubscriber < Subscriber # :nodoc:
+        def call(_name, start, finish, _id, payload) # rubocop:disable Metrics/MethodLength
+          return unless enabled?
+
+          ts = InfluxDB.convert_timestamp(finish.utc, configuration.time_precision)
+          tags = tags(payload)
+          begin
+            series(payload, start, finish).each do |series_name, value|
+              InfluxDB::Rails.client.write_point \
+                series_name,
+                values:    { value: value },
+                tags:      tags,
+                timestamp: ts
+            end
+          rescue StandardError => e
+            log :error, "[InfluxDB::Rails] Unable to write points: #{e.message}"
+          ensure
+            Thread.current[:_influxdb_rails_controller] = nil
+            Thread.current[:_influxdb_rails_action]     = nil
+          end
+        end
+
+        private
+
+        def series(payload, start, finish)
+          {
+            configuration.series_name_for_controller_runtimes => ((finish - start) * 1000).ceil,
+            configuration.series_name_for_view_runtimes       => (payload[:view_runtime] || 0).ceil,
+            configuration.series_name_for_db_runtimes         => (payload[:db_runtime] || 0).ceil,
+          }
+        end
+
+        def tags(payload)
+          configuration.tags_middleware.call({
+            method:      "#{payload[:controller]}##{payload[:action]}",
+            status:      payload[:status],
+            format:      payload[:format],
+            http_method: payload[:method],
+            server:      Socket.gethostname,
+            app_name:    configuration.application_name,
+          }.reject { |_, value| value.nil? })
+        end
+      end
+    end
+  end
+end

data/lib/influxdb/rails/middleware/simple_subscriber.rb

@@ -0,0 +1,46 @@
+require "influxdb/rails/middleware/subscriber"
+
+module InfluxDB
+  module Rails
+    module Middleware
+      # Subscriber acts as base class for different *Subscriber classes,
+      # which are intended as ActiveSupport::Notifications.subscribe
+      # consumers.
+      class SimpleSubscriber < Subscriber
+        attr_reader :series_name
+
+        def initialize(configuration, series_name)
+          super(configuration)
+          @series_name = series_name
+        end
+
+        def call(_name, started, finished, _unique_id, payload)
+          return unless enabled?
+
+          begin
+            InfluxDB::Rails.client.write_point series_name,
+                                               values:    values(started, finished, payload),
+                                               tags:      tags(payload),
+                                               timestamp: timestamp(finished.utc)
+          rescue StandardError => e
+            log :error, "[InfluxDB::Rails] Unable to write points: #{e.message}"
+          end
+        end
+
+        private
+
+        def values(started, finished, _payload)
+          { value: ((finished - started) * 1000).ceil }
+        end
+
+        def timestamp(finished)
+          InfluxDB.convert_timestamp(finished.utc, configuration.time_precision)
+        end
+
+        def enabled?
+          super && series_name.present?
+        end
+      end
+    end
+  end
+end

data/lib/influxdb/rails/middleware/sql_subscriber.rb

@@ -0,0 +1,32 @@
+require "influxdb/rails/middleware/simple_subscriber"
+require "influxdb/rails/sql/query"
+
+module InfluxDB
+  module Rails
+    module Middleware
+      class SqlSubscriber < SimpleSubscriber # :nodoc:
+        def call(_name, started, finished, _unique_id, payload)
+          return unless InfluxDB::Rails::Sql::Query.new(payload).track?
+
+          super
+        end
+
+        private
+
+        def values(started, finished, payload)
+          super.merge(sql: InfluxDB::Rails::Sql::Normalizer.new(payload[:sql]).perform)
+        end
+
+        def tags(payload)
+          query = InfluxDB::Rails::Sql::Query.new(payload)
+          {
+            location:   location,
+            operation:  query.operation,
+            class_name: query.class_name,
+            name:       query.name,
+          }
+        end
+      end
+    end
+  end
+end

data/lib/influxdb/rails/middleware/subscriber.rb

@@ -0,0 +1,38 @@
+require "influxdb/rails/logger"
+
+module InfluxDB
+  module Rails
+    module Middleware
+      # Subscriber acts as base class for different *Subscriber classes,
+      # which are intended as ActiveSupport::Notifications.subscribe
+      # consumers.
+      class Subscriber
+        include InfluxDB::Rails::Logger
+
+        attr_reader :configuration
+
+        def initialize(configuration)
+          @configuration = configuration
+        end
+
+        def call(*)
+          raise NotImplementedError, "must be implemented in subclass"
+        end
+
+        private
+
+        def enabled?
+          configuration.instrumentation_enabled? &&
+            !configuration.ignore_current_environment?
+        end
+
+        def location
+          [
+            Thread.current[:_influxdb_rails_controller],
+            Thread.current[:_influxdb_rails_action],
+          ].reject(&:blank?).join("#")
+        end
+      end
+    end
+  end
+end

data/lib/influxdb/rails/railtie.rb

@@ -8,7 +8,7 @@ module InfluxDB
         app.config.middleware.insert 0, InfluxDB::Rails::Rack
       end
 
-      config.after_initialize do
+      config.after_initialize do # rubocop:disable Metrics/BlockLength
         InfluxDB::Rails.configure(true, &:load_rails_defaults)
 
         ActiveSupport.on_load(:action_controller) do
@@ -22,19 +22,27 @@ module InfluxDB
         ::ActionDispatch::DebugExceptions.prepend InfluxDB::Rails::Middleware::HijackRenderException
 
         if defined?(ActiveSupport::Notifications)
-          listen = lambda do |name, start, finish, id, payload|
-            c = InfluxDB::Rails.configuration
-
-            if c.instrumentation_enabled? && !c.ignore_current_environment?
-              begin
-                InfluxDB::Rails.handle_action_controller_metrics(name, start, finish, id, payload)
-              rescue StandardError => e
-                c.logger.error "[InfluxDB::Rails] Failed writing points to InfluxDB: #{e.message}"
-              end
-            end
+          cache = lambda do |_, _, _, _, payload|
+            Thread.current[:_influxdb_rails_controller] = payload[:controller]
+            Thread.current[:_influxdb_rails_action]     = payload[:action]
           end
+          ActiveSupport::Notifications.subscribe "start_processing.action_controller", &cache
 
-          ActiveSupport::Notifications.subscribe "process_action.action_controller", &listen
+          c = InfluxDB::Rails.configuration
+          requests = Middleware::RequestSubscriber.new(c)
+          ActiveSupport::Notifications.subscribe "process_action.action_controller", requests
+
+          templates = Middleware::RenderSubscriber.new(c, c.series_name_for_render_template)
+          ActiveSupport::Notifications.subscribe "render_template.action_view", templates
+
+          partials = Middleware::RenderSubscriber.new(c, c.series_name_for_render_partial)
+          ActiveSupport::Notifications.subscribe "render_partial.action_view", partials
+
+          collections = Middleware::RenderSubscriber.new(c, c.series_name_for_render_collection)
+          ActiveSupport::Notifications.subscribe "render_collection.action_view", collections
+
+          sql = Middleware::SqlSubscriber.new(c, c.series_name_for_sql)
+          ActiveSupport::Notifications.subscribe "sql.active_record", sql
         end
       end
     end

data/lib/influxdb/rails/sql/normalizer.rb

@@ -0,0 +1,27 @@
+module InfluxDB
+  module Rails
+    module Sql
+      class Normalizer # :nodoc:
+        def initialize(query)
+          @query = query.dup
+        end
+
+        def perform
+          query.squish!
+          query.gsub!(/(\s(=|>|<|>=|<=|<>|!=)\s)('[^']+'|[\$\+\-\w\.]+)/, '\1xxx')
+          query.gsub!(/(\sIN\s)\([^\(\)]+\)/i, '\1(xxx)')
+          regex = /(\sBETWEEN\s)('[^']+'|[\+\-\w\.]+)(\sAND\s)('[^']+'|[\+\-\w\.]+)/i
+          query.gsub!(regex, '\1xxx\3xxx')
+          query.gsub!(/(\sVALUES\s)\(.+\)/i, '\1(xxx)')
+          query.gsub!(/(\s(LIKE|ILIKE|SIMILAR TO|NOT SIMILAR TO)\s)('[^']+')/i, '\1xxx')
+          query.gsub!(/(\s(LIMIT|OFFSET)\s)(\d+)/i, '\1xxx')
+          query
+        end
+
+        private
+
+        attr_reader :query
+      end
+    end
+  end
+end

data/lib/influxdb/rails/sql/query.rb

@@ -0,0 +1,30 @@
+require "influxdb/rails/sql/normalizer"
+
+module InfluxDB
+  module Rails
+    module Sql
+      class Query # :nodoc:
+        attr_reader :query, :name
+
+        TRACKED_SQL_COMMANDS = %w[SELECT INSERT UPDATE DELETE].freeze
+
+        def initialize(payload)
+          @query = payload[:sql].to_s.dup
+          @name = payload[:name].to_s.dup
+        end
+
+        def operation
+          query.split.first.upcase
+        end
+
+        def class_name
+          name.split.first
+        end
+
+        def track?
+          @track ||= query.start_with?(*TRACKED_SQL_COMMANDS)
+        end
+      end
+    end
+  end
+end

data/lib/influxdb/rails/version.rb

@@ -1,5 +1,5 @@
 module InfluxDB
   module Rails
-    VERSION = "1.0.0.beta1".freeze
+    VERSION = "1.0.0.beta2".freeze
   end
 end