stasher 0.1.0

data/.gitattributes

@@ -0,0 +1,11 @@
+*       text=auto eol=lf
+
+*.cmd   text eol=crlf
+
+*.png   binary
+*.jpg   binary
+*.gif   binary
+*.ico	binary
+*.ttf 	binary
+*.woff	binary
+*.eot	binary

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format documentation

data/.ruby-gemset

@@ -0,0 +1 @@
+stasher

data/Gemfile

@@ -0,0 +1,15 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in stasher.gemspec
+gemspec
+
+group :test do
+  gem 'rb-fsevent'
+  gem 'guard'
+  gem 'guard-rspec'
+  gem 'growl'
+  gem 'factory_girl'
+  gem 'simplecov', :platforms => :mri_19, :require => false
+  gem 'rcov', :platforms => :mri_18
+  gem 'rails', "~> #{ENV["RAILS_VERSION"] || "3.2.0"}"
+end

data/Guardfile

@@ -0,0 +1,11 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+interactor :simple
+
+guard 'rspec' do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^spec/factories/.+\.rb$}) { "spec" }
+  watch(%r{^spec/support/.+\.rb$}) { "spec" }
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Chris Micacchi
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,101 @@
+# Stasher
+
+This gem is a heavy modification of [Logstasher](https://github.com/shadabahmed/logstasher), which was 
+inspired from [LogRage](https://github.com/roidrage/lograge).  It adds the same request logging for logstash as
+Logstasher, but separates out request and response log entries, and adds a modified Ruby Logger instance to allow
+you to send all of your logging to logstash.
+
+## About stasher
+
+This gem logs to a separate log file named `logstash_<environment>.log`.  It provides two facilities:
+ * Request and response logging (ala Logstasher and LogRage)
+ * Redirection of the Rails logger, with request-scoped parameters
+
+Before **stasher** :
+
+```
+Started GET "/login" for 10.109.10.135 at 2013-04-30 08:59:01 -0400
+Processing by SessionsController#new as HTML
+  Rendered sessions/new.html.haml within layouts/application (4.3ms)
+  Rendered shared/_javascript.html.haml (0.6ms)
+  Rendered shared/_flashes.html.haml (0.2ms)
+  Rendered shared/_header.html.haml (52.9ms)
+  Rendered shared/_title.html.haml (0.2ms)
+  Rendered shared/_footer.html.haml (0.2ms)
+  Banner Load  SELECT `banners`.* FROM `banners` WHERE `banner`.`active` = 1 ORDER BY created_at DESC
+Found 3 banners to display on the login page
+Completed 200 OK in 532ms (Views: 62.4ms | ActiveRecord: 0.0ms | ND API: 0.0ms)
+```
+
+After **stasher**:
+
+```
+{"@source":"rails://localhost/my-app","@tags":["request"],"@fields":{"method":"GET","path":"/login","format":"html","controller":"sessions"
+,"action":"login","ip":"127.0.0.1",params:{},"uuid":"e81ecd178ed3b591099f4d489760dfb6","user":"shadab_ahmed@abc.com",
+"site":"internal"},"@timestamp":"2013-04-30T13:00:46.354500+00:00"}
+{"@source":"rails://localhost/my-app","@tags":["sql"],"@fields":{"name":"Banner Load","sql":"SELECT `banners`.* FROM `banners` WHERE `banner`.`active` = 1 ORDER BY created_at DESC","uuid":"e81ecd178ed3b591099f4d489760dfb6"},"@timestamp":"2013-04-30T13:00:46.362300+00:00"}
+{"@source":"rails://localhost/my-app","@tags":["log","debug"],"@fields":{"severity":"DEBUG","uuid":"e81ecd178ed3b591099f4d489760dfb6"},"@message":"Found 3 banners to display on the login page","@timestamp":"2013-04-30T13:00:46.353400+00:00"}
+{"@source":"rails://localhost/my-app","@tags":["response"],"@fields":{"method":"GET","path":"/login","format":"html","controller":"sessions"
+,"action":"login","status":200,"duration":28.34,"view":25.96,"db":0.88,"ip":"127.0.0.1","uuid":"e81ecd178ed3b591099f4d489760dfb6","user":"shadab_ahmed@abc.com",
+"site":"internal"},"@timestamp":"2013-04-30T13:00:46.354500+00:00"}
+```
+
+By default, the older format rails request logs are disabled, though you can enable them.
+
+All events logged within a Rack request will include the request's UUID, allowing you to follow individual requests through the logs.
+
+## Installation
+
+In your Gemfile:
+
+    gem 'stasher'
+
+### Configure your `<environment>.rb` e.g. `development.rb`
+
+    # Enable the logstasher logs for the current environment and set the log level
+    config.stasher.enabled = true
+    config.stasher.log_level = :debug
+
+    # This line is optional if you do not want to suppress app logs in your <environment>.log
+    # config.stasher.suppress_app_log = false
+
+    # This line causes the Rails logger to be redirected to logstash as well
+    config.stasher.redirect_logger = true
+
+    # To prevent logging of SQL into logstash, remove :active_record from this line
+    config.stasher.attach_to = [ :action_controller, :active_record ]
+
+## Adding custom fields to the log
+
+Since some fields are very specific to your application for e.g. *user_name*, so it is left 
+up to you to add them. Here's how to add those fields to the logs:
+
+    # Create a file - config/initializers/stasher.rb
+
+    if Stasher.enabled
+      Stasher.add_custom_fields do |fields|
+        # This block is run in application_controller context, 
+        # so you have access to all controller methods
+        fields[:user] = current_user && current_user.mail
+        fields[:site] = request.path =~ /^\/api/ ? 'api' : 'user'        
+      end
+    end
+
+## Versions
+All versions require Rails 3.2.x and higher and Ruby 1.9.2+. This code has not been tested on Rails 4 and Ruby 2.0
+
+## Development
+ - Run tests - `rake`
+ - Generate test coverage report - `rake coverage`. Coverage report path - coverage/index.html
+
+## License
+
+Released under MIT license.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,8 @@
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+
+task :default => :spec
+
+RSpec::Core::RakeTask.new('spec') do |spec|
+  spec.pattern = "./spec/**/*_spec.rb"
+end

data/lib/stasher.rb

@@ -0,0 +1,137 @@
+require "stasher/version"
+require 'stasher/log_subscriber'
+require 'stasher/current_scope'
+require 'stasher/logger'
+require 'active_support/core_ext/module/attribute_accessors'
+require 'active_support/core_ext/string/inflections'
+require 'active_support/ordered_options'
+
+module Stasher
+    # Logger for the logstash logs
+    mattr_accessor :logger, :enabled, :source
+
+    def self.remove_existing_log_subscriptions
+      ActiveSupport::LogSubscriber.log_subscribers.each do |subscriber|
+        case subscriber
+          when ActionView::LogSubscriber
+            unsubscribe(:action_view, subscriber)
+          when ActiveRecord::LogSubscriber
+            unsubscribe(:active_record, subscriber)
+          when ActionController::LogSubscriber
+            unsubscribe(:action_controller, subscriber)
+        end
+      end
+    end
+
+    def self.unsubscribe(component, subscriber)
+      events = subscriber.public_methods(false).reject{ |method| method.to_s == 'call' }
+      events.each do |event|
+        ActiveSupport::Notifications.notifier.listeners_for("#{event}.#{component}").each do |listener|
+          if listener.instance_variable_get('@delegate') == subscriber
+            ActiveSupport::Notifications.unsubscribe listener
+          end
+        end
+      end
+    end
+
+    def self.add_default_fields_to_scope(scope, request)
+      scope[:uuid] = request.uuid
+    end
+
+    def self.add_custom_fields(&block)
+      ActionController::Metal.send(:define_method, :stasher_add_custom_fields_to_scope, &block)
+    end
+
+    def self.setup(app)
+      app.config.action_dispatch.rack_cache[:verbose] = false if app.config.action_dispatch.rack_cache
+
+      # Compose source
+      self.source = "rails://#{hostname}/#{app.class.name.deconstantize.underscore}"
+
+      # Initialize & set up instrumentation
+      require 'stasher/rails_ext/action_controller/metal/instrumentation'
+      require 'logstash/event'      
+      self.suppress_app_logs(app) if app.config.stasher.suppress_app_log
+
+      # Redirect Rails' logger if requested
+      Rails.logger = Stasher::Logger.new  if app.config.stasher.redirect_logger
+
+      # Subscribe to configured events
+      app.config.stasher.attach_to.each do |target|
+        Stasher::LogSubscriber.attach_to target
+      end
+
+      # Initialize internal logger
+      self.logger = app.config.stasher.logger || Logger.new("#{Rails.root}/log/logstash_#{Rails.env}.log")
+      level = ::Logger.const_get(app.config.stasher.log_level.to_s.upcase) if app.config.stasher.log_level
+      self.logger.level = level || Logger::WARN
+
+      self.enabled = true
+    end
+
+    def self.suppress_app_logs(app)   
+      require 'stasher/rails_ext/rack/logger'
+      Stasher.remove_existing_log_subscriptions
+
+      # Disable ANSI colorization
+      app.config.colorize_logging = false
+    end
+
+    def self.format_exception(type_name, message, backtrace)
+      {
+        :exception => { 
+          :name => type_name,
+          :message => message,
+          :backtrace => backtrace
+        }
+      }
+    end
+
+    def self.log(severity, msg)
+      if self.logger && self.logger.send("#{severity.to_s.downcase}?")
+        data = {
+          :severity => severity.upcase
+        }
+        tags = ['log']
+
+        if msg.is_a? Exception
+          data.merge! self.format_exception(msg.class.name, msg.message, msg.backtrace.join("\n"))
+          msg = "#{msg.class.name}: #{msg.message}"
+          tags << 'exception'
+        else        
+          # Strip ANSI codes from the message
+          msg.gsub!(/\u001B\[[0-9;]+m/, '')
+        end
+
+        return true if msg.empty?
+        data.merge! CurrentScope.fields
+
+        tags << severity.downcase
+
+        event = LogStash::Event.new(
+          '@fields' => data, 
+          '@tags' => tags,
+          '@message' => msg,
+          '@source' => Stasher.source)
+        self.logger << event.to_json + "\n"
+      end
+    end
+
+    def self.hostname
+      require 'socket'
+
+      Socket.gethostname
+    end
+
+    class << self
+      %w( fatal error warn info debug unknown ).each do |severity|
+        eval <<-EOM, nil, __FILE__, __LINE__ + 1
+          def #{severity}(msg)
+            self.log(:#{severity}, msg)
+          end
+        EOM
+      end
+    end
+end
+
+require 'stasher/railtie' if defined?(Rails)

data/lib/stasher/current_scope.rb

@@ -0,0 +1,21 @@
+module Stasher
+  module CurrentScope
+    ##
+    # Gets the hash of fields in the current scope
+    def self.fields
+      Thread.current[:stasher_fields] ||= {}
+    end
+
+    ##
+    # Gets the hash of fields in the current scope
+    def self.fields=(values)
+      Thread.current[:stasher_fields] = values
+    end
+
+    ##
+    # Clears the current scope
+    def self.clear!
+      Thread.current[:stasher_fields] = nil
+    end
+  end
+end

data/lib/stasher/log_subscriber.rb

@@ -0,0 +1,128 @@
+require 'active_support/core_ext/class/attribute'
+require 'active_support/log_subscriber'
+
+module Stasher
+  class LogSubscriber < ActiveSupport::LogSubscriber
+    def start_processing(ev)
+      # Initialize the scope at the start of the request
+      payload = ev.payload
+
+      data      = extract_request(payload)
+      data.merge! extract_current_scope
+
+      log_event 'request', data
+    end
+
+    def process_action(ev)
+      payload = ev.payload
+
+      data      = extract_request(payload)
+      data.merge! extract_status(payload)
+      data.merge! runtimes(ev)
+      data.merge! extract_exception(payload)
+      data.merge! extract_current_scope
+
+      log_event 'response', data do |event|
+        event.tags << 'exception' if payload[:exception]
+      end
+
+      # Clear the scope at the end of the request
+      Stasher::CurrentScope.clear!
+    end
+    
+    def sql(ev)
+      payload = ev.payload
+
+      return if 'SCHEMA' == payload[:name]
+      return if payload[:name].blank?
+      return if payload[:name] =~ /ActiveRecord::SessionStore/
+
+      data      = extract_sql(payload)
+      data.merge! runtimes(ev)
+      data.merge! extract_current_scope
+
+      log_event 'sql', data
+    end
+
+    def redirect_to(ev)
+      Stasher::CurrentScope.fields[:location] = ev.payload[:location]
+    end
+
+    private
+
+    def log_event(type, data)
+      event = LogStash::Event.new('@fields' => data, '@tags' => [type], '@source' => Stasher.source)
+      yield(event) if block_given?
+      Stasher.logger << event.to_json + "\n"
+    end
+
+    def extract_sql(payload)
+      {
+        :name => payload[:name],
+        :sql => payload[:sql].squeeze(' '),
+      }
+    end
+
+    def extract_request(payload)
+      {
+        :method => payload[:method],
+        :ip => payload[:ip],
+        :params => extract_parms(payload),
+        :path => extract_path(payload),
+        :format => extract_format(payload),
+        :controller => payload[:params]['controller'],
+        :action => payload[:params]['action']
+      }
+    end
+
+    def extract_parms(payload)
+      payload[:params].except(*ActionController::LogSubscriber::INTERNAL_PARAMS) if payload.include?(:params)
+    end
+
+    def extract_path(payload)
+      payload[:path].split("?").first
+    end
+
+    def extract_format(payload)
+      if ::ActionPack::VERSION::MAJOR == 3 && ::ActionPack::VERSION::MINOR == 0
+        payload[:formats].first
+      else
+        payload[:format]
+      end
+    end
+
+    def extract_status(payload)
+      if payload[:status]
+        { :status => payload[:status].to_i }
+      else
+        { :status => 0 }
+      end
+    end
+
+    def runtimes(event)
+      {
+        :duration => event.duration,
+        :view => event.payload[:view_runtime],
+        :db => event.payload[:db_runtime]
+      }.inject({}) do |runtimes, (name, runtime)|
+        runtimes[name] = runtime.to_f.round(2) if runtime
+        runtimes
+      end
+    end
+
+    def extract_current_scope
+      CurrentScope.fields
+    end
+
+    # Monkey patching to enable exception logging
+    def extract_exception(payload)
+      if payload[:exception]
+        exception, message = payload[:exception]
+
+        Stasher.format_exception(exception, message, $!.backtrace.join("\n"))
+      else
+        {}
+      end
+    end
+  end
+end