rails-logstasher 0.1.0

data/lib/rails_logstasher/event.rb

@@ -0,0 +1,44 @@
+module RailsLogstasher
+
+  # Basically a wrapper for a LogStash event that keeps track of if it was created from a rack
+  # middle-ware or not. This is important when it comes to deciding when to write the log
+  class Event
+
+    extend Forwardable
+    def_delegators :@logstash_event, :fields, :message=, :source=, :type=, :tags, :to_json
+
+    def initialize(logger, rack = false)
+      @logger = logger
+      @rack = rack
+      @logstash_event = LogStash::Event.new
+    end
+
+    def write(rack = false)
+      if @rack
+        @logger.info self if rack
+      else
+        @logger.info self
+      end
+    end
+
+    def add_tags_to_logger(request, tags)
+      tag_hash = []
+      if tags
+        tags.each do |tag|
+          case tag
+          when Symbol
+            tag_hash << {tag.to_s => request.send(tag) }
+          when Proc
+            tag_hash << tag.call(request)
+          else
+            tag_hash << tag
+          end
+        end
+      end
+
+      @logger.push_request_tags(tag_hash)
+    end
+
+  end
+
+end

data/lib/rails_logstasher/logger.rb

@@ -0,0 +1,55 @@
+require 'logger'
+
+module RailsLogstasher
+
+  # Based on the ActiveSupport::Logger (Formerly known as BufferedLogger)
+  class Logger < ::Logger
+    # Broadcasts logs to multiple loggers.
+    def self.broadcast(logger) # :nodoc:
+      Module.new do
+        define_method(:add) do |*args, &block|
+          logger.add(*args, &block)
+          super(*args, &block)
+        end
+
+        define_method(:<<) do |x|
+          logger << x
+          super(x)
+        end
+
+        define_method(:close) do
+          logger.close
+          super()
+        end
+
+        define_method(:progname=) do |name|
+          logger.progname = name
+          super(name)
+        end
+
+        define_method(:formatter=) do |formatter|
+          logger.formatter = formatter
+          super(formatter)
+        end
+
+        define_method(:level=) do |level|
+          logger.level = level
+          super(level)
+        end
+      end
+    end
+
+    def initialize(*args)
+      super
+      @formatter = SimpleFormatter.new
+    end
+
+    # Simple formatter which only displays the message.
+    class SimpleFormatter < ::Logger::Formatter
+      # This method is invoked when a log event occurs
+      def call(severity, timestamp, progname, msg)
+        "#{String === msg ? msg : msg.inspect}\n"
+      end
+    end
+  end
+end

data/lib/rails_logstasher/rack/logger.rb

@@ -0,0 +1,53 @@
+module RailsLogstasher
+
+  module Rack
+
+    class Logger
+
+      def initialize(app, tags = nil)
+        @app, @tags = app, tags.presence
+      end
+
+      def call(env)
+
+        t1 = Time.now
+        request = ActionDispatch::Request.new(env)
+
+        event = RailsLogstasher::Event.new(Rails.logger, true)
+        event.message = "#{request.request_method} #{request.filtered_path} for #{request.ip}"
+        event.fields['client_ip'] = request.ip
+        event.fields['method'] = request.request_method
+        event.fields['path'] = request.filtered_path
+        #TODO Should really move this into the base logger
+        event.source = "http://#{Socket.gethostname}#{request.filtered_path}"
+
+        event.add_tags_to_logger(request, @tags) if @tags
+
+        RailsLogstasher.log_entries[Thread.current] = event
+
+        status, headers, response = @app.call(env)
+        [status, headers, response]
+
+      ensure
+        if event
+          event.fields['total_duration'] = Time.now - t1
+          event.fields['status'] = status
+
+          ['rendering','sql'].each do |type|
+            if event.fields[type] && !event.fields[type].empty?
+              duration = event.fields[type].inject(0) {|result, local_event| result += local_event[:duration].to_f }
+              event.fields["#{type}_duration"] = duration
+            end
+          end
+
+          event.write(true)
+        end
+
+        RailsLogstasher.log_entries[Thread.current] = nil
+      end
+
+    end
+
+  end
+
+end

data/lib/rails_logstasher/railtie.rb

@@ -0,0 +1,70 @@
+require 'rails_logstasher/core_ext/object/blank'
+require 'rails_logstasher/action_controller/log_subscriber'
+require 'rails_logstasher/action_view/log_subscriber'
+require 'rails_logstasher/active_record/log_subscriber' if defined?(ActiveRecord)
+require 'rails_logstasher/active_resource/log_subscriber' if defined?(ActiveResource)
+
+module RailsLogstasher
+
+  # Railtie to hook RailsLogstasher into Rails
+  #
+  # This Railtie hooks RailsLogstasher into Rails by adding middleware and loggers as well as
+  # adding a completely new set of LogSubscribers which parallel the default rails ones but
+  # are JSON based rather than string based
+  class Railtie < Rails::Railtie
+
+    initializer "rails_logstasher.swap_rack_logger_middleware" do |app|
+      app.middleware.swap(Rails::Rack::Logger, RailsLogstasher::Rack::Logger, app.config.log_tags)
+    end
+
+    # Silence the asset logger. This has to be done in a before_initialize block because
+    # the initializer is too late. (There might be a better part of the boot process for
+    # this, keep an eye out)
+    config.before_initialize do |app|
+      app.config.assets.logger = false
+
+      if app.config.logger.nil? && Rails.logger.class == ActiveSupport::TaggedLogging
+        raise IncompatibleLogger, "Please replace the default rails logger (See the " +
+                                  "Configuration section of the RailsLogstasher README)"
+      end
+
+      # Take the current logger and replace it with itself wrapped by the
+      # RailsLogstasher::TaggedLogging class
+      app.config.log_type = 'rails' unless app.config.respond_to? :log_type
+      app.config.logger = RailsLogstasher::TaggedLogging.new(app.config.logger, app.config.log_type)
+    end
+
+
+    # We need to do the following in an after_initialize block to make sure we get all the
+    # subscribers. Ideally rails would allow us the ability to stop the LogSubscribers from
+    # registering themselves using a config option.
+    config.after_initialize do
+
+      # Kludge the removal of the default LogSubscribers for the moment. We will use the rails_logstasher
+      # LogSubscribers (since they subscribe to the same hooks in the public methods) to create
+      # a list of hooks we want to unsubscribe current subscribers from.
+      modules = ["ActionController", "ActionView"]
+      modules << "ActiveRecord" if defined?(ActiveRecord)
+      modules << "ActiveResource" if defined?(ActiveResource)
+
+      notifier = ActiveSupport::Notifications.notifier
+
+      modules.each do |mod|
+        "RailsLogstasher::#{mod}::LogSubscriber".constantize.instance_methods(false).each do |method|
+          notifier.listeners_for("#{method}.#{mod.underscore}").each do |subscriber|
+            ActiveSupport::Notifications.unsubscribe subscriber
+          end
+        end
+      end
+
+      # We then subscribe using the rails_logstasher versions of the default rails LogSubscribers
+      RailsLogstasher::ActionController::LogSubscriber.attach_to :action_controller
+      RailsLogstasher::ActionView::LogSubscriber.attach_to :action_view
+      RailsLogstasher::ActiveRecord::LogSubscriber.attach_to :active_record if defined?(ActiveRecord)
+      RailsLogstasher::ActiveResource::LogSubscriber.attach_to :active_resource if defined?(ActiveResource)
+
+    end
+
+  end
+
+end

data/lib/rails_logstasher/tagged_logging.rb

@@ -0,0 +1,108 @@
+require 'rails_logstasher/core_ext/object/blank'
+require 'logger'
+require 'rails_logstasher/logger'
+
+module RailsLogstasher
+  # Wraps any standard Logger object to provide tagging capabilities.
+  #
+  # logger = RailsLogstasher::TaggedLogging.new(Logger.new(STDOUT))
+  # logger.tagged('BCX') { logger.info 'Stuff' } # Adds BCX to the @tags array and "Stuff" to the @message
+  # logger.tagged('BCX', "Jason") { logger.info 'Stuff' } # Adds 'BCX' and 'Jason' to the @tags array and "Stuff"
+  # to the @message
+  # logger.tagged('BCX') { logger.tagged('Jason') { logger.info 'Stuff' } } # Adds 'BCX' and 'Jason' to the @tags
+  # array and "Stuff" to the @message
+  #
+  # This is used by the default Rails.logger when the RailsLogstasher gem is added to a rails application
+  # to make it easy to stamp JSON logs with subdomains, request ids, and anything else
+  # to aid debugging of multi-user production applications.
+  module TaggedLogging
+    module Formatter # :nodoc:
+      # This method is invoked when a log event occurs.
+      def call(severity, timestamp, progname, msg)
+        @entry = nil
+        if msg.class == RailsLogstasher::Event
+          @entry = msg
+        else
+          @entry = RailsLogstasher::Event.new(Rails.logger)
+          @entry.message = msg
+        end
+        @entry.fields['severity'] = severity
+        @entry.type = @log_type
+        process_tags(current_tags)
+        process_tags(current_request_tags)
+        #TODO Should we do anything with progname? What about source?
+        super(severity, timestamp, progname, @entry.to_json)
+      end
+
+      def tagged(*tags)
+        new_tags = push_tags(*tags)
+        yield self
+      ensure
+        pop_tags(new_tags.size)
+      end
+
+      def push_tags(*tags)
+        tags.flatten.reject(&:blank?).tap do |new_tags|
+          current_tags.concat new_tags
+        end
+      end
+
+      def push_request_tags(tags)
+        Thread.current[:activesupport_tagged_logging_request_tags] = tags
+      end
+
+      def process_tags(tags)
+        tags.each do |tag|
+          if tag.class == Hash
+            tag.each_pair do |k,v|
+              @entry.fields[k] = v
+            end
+          else
+            @entry.tags << tag
+          end
+        end
+      end
+
+      def pop_tags(size = 1)
+        current_tags.pop size
+      end
+
+      def clear_tags!
+        current_request_tags.clear
+        current_tags.clear
+      end
+
+      def current_tags
+        Thread.current[:activesupport_tagged_logging_tags] ||= []
+      end
+
+      def current_request_tags
+        Thread.current[:activesupport_tagged_logging_request_tags] ||= []
+      end
+
+      def log_type=(log_type)
+        @log_type = log_type
+      end
+
+    end
+
+    def self.new(logger, log_type)
+      # Ensure we set a default formatter so we aren't extending nil!
+      logger.formatter ||= ActiveSupport::Logger::SimpleFormatter.new
+      logger.formatter.extend Formatter
+      logger.formatter.log_type = log_type
+      logger.extend(self)
+    end
+
+    delegate :push_tags, :push_request_tags, :pop_tags, :clear_tags!, :log_type=, to: :formatter
+
+    def tagged(*tags)
+      formatter.tagged(*tags) { yield self }
+    end
+
+    def flush
+      clear_tags!
+      super if defined?(super)
+    end
+  end
+end

data/lib/rails_logstasher/version.rb

@@ -0,0 +1,3 @@
+module RailsLogstasher
+  VERSION = "0.1.0"
+end

data/test/action_controller/log_subscriber_test.rb

@@ -0,0 +1,165 @@
+require "active_support/log_subscriber/test_helper"
+require "test_helper"
+
+class ACLogSubscriberTest < ActionController::TestCase
+  tests LogSubscribersController
+  include ActiveSupport::LogSubscriber::TestHelper
+
+  def setup
+    super
+
+    @cache_path = File.expand_path('../temp/test_cache', File.dirname(__FILE__))
+    ActionController::Base.page_cache_directory = @cache_path
+    @controller.cache_store = :file_store, @cache_path
+
+    RailsLogstasher::ActionController::LogSubscriber.attach_to :action_controller
+    RailsLogstasher.log_entries[Thread.current] = LogStash::Event.new
+    @log_entry = RailsLogstasher.log_entries[Thread.current]
+  end
+
+  def teardown
+    super
+    ActiveSupport::LogSubscriber.log_subscribers.clear
+    FileUtils.rm_rf(@cache_path)
+  end
+
+  def set_logger(logger)
+    ActionController::Base.logger = logger
+  end
+
+  def test_start_processing
+    get :show, {:test => 'test'}
+    wait
+
+    assert_equal "LogSubscribersController", @log_entry.fields['controller']
+    assert_equal "show", @log_entry.fields['action']
+    assert_equal "html", @log_entry.fields['format']
+  end
+
+
+  def test_halted_callback
+    get :never_executed
+    wait
+
+    assert_equal ":redirector" ,@log_entry.fields['halted_callback']
+  end
+
+  def test_process_action
+    get :show
+    wait
+
+    assert_present @log_entry.fields['controller_duration']
+  end
+
+  def test_process_action_without_parameters
+    get :show
+    wait
+
+    assert_blank @log_entry.fields['parameters']
+  end
+
+  def test_process_action_with_parameters
+    get :show, :id => '10'
+    wait
+
+    assert_equal '10', @log_entry.fields['parameters']['id']
+  end
+
+  def test_process_action_with_wrapped_parameters
+    @request.env['CONTENT_TYPE'] = 'application/json'
+    post :show, :id => '10', :name => 'jose'
+    wait
+
+    assert_equal '10', @log_entry.fields['parameters']['id']
+    assert_equal 'jose', @log_entry.fields['parameters']['name']
+  end
+
+  def test_process_action_with_filter_parameters
+    @request.env["action_dispatch.parameter_filter"] = [:lifo, :amount]
+
+    get :show, :lifo => 'Pratik', :amount => '420', :step => '1'
+    wait
+
+    params = @log_entry.fields['parameters']
+    assert_equal '[FILTERED]', params['amount']
+    assert_equal '[FILTERED]', params['lifo']
+    assert_equal '1', params['step']
+  end
+
+  def test_redirect_to
+    get :redirector
+    wait
+
+    assert_equal 'http://foo.bar/', @log_entry.fields['redirect_to']
+  end
+
+
+  def test_send_data
+    get :data_sender
+    wait
+
+    assert_equal 'file.txt', @log_entry.fields['send_data']
+    assert_present @log_entry.fields['send_data_duration']
+  end
+
+
+  def test_send_file
+    get :file_sender
+    wait
+
+    assert_match 'test/dummy/public/favicon.ico', @log_entry.fields['send_file']
+    assert_present @log_entry.fields['send_file_duration']
+  end
+
+  def test_with_fragment_cache
+    @controller.config.perform_caching = true
+    get :with_fragment_cache
+    wait
+
+    assert_present @log_entry.fields['cache']
+
+    assert_match('Read fragment', @log_entry.fields['cache'].first['type'])
+    assert_match('views/foo', @log_entry.fields['cache'].first['key_or_path'])
+
+    assert_match('Write fragment', @log_entry.fields['cache'].last['type'])
+    assert_match('views/foo', @log_entry.fields['cache'].last['key_or_path'])
+  ensure
+    LogSubscribersController.config.perform_caching = true
+  end
+
+
+  def test_with_fragment_cache_and_percent_in_key
+    @controller.config.perform_caching = true
+    get :with_fragment_cache_and_percent_in_key
+    wait
+
+    assert_present @log_entry.fields['cache']
+
+    assert_match('Read fragment', @log_entry.fields['cache'].first['type'])
+    assert_match('views/foo', @log_entry.fields['cache'].first['key_or_path'])
+
+    assert_match('Write fragment', @log_entry.fields['cache'].last['type'])
+    assert_match('views/foo', @log_entry.fields['cache'].last['key_or_path'])
+  ensure
+    LogSubscribersController.config.perform_caching = true
+  end
+
+=begin TODO Figure out why this last test fails.
+  def test_with_page_cache
+    @controller.config.perform_caching = true
+    get :with_page_cache
+    wait
+
+   assert_present @log_entry.fields['cache']
+
+    assert_match('Write page', @log_entry.fields['cache'][1]['type'])
+    assert_match('index.html', @log_entry.fields['cache'][1]['key_or_path'])
+  ensure
+    @controller.config.perform_caching = true
+  end
+=end
+
+  def logs
+    @logs ||= @logger.logged(:info)
+  end
+end