log2json 0.1.9 → 0.1.10

data/bin/redis2es

@@ -41,11 +41,11 @@ const(:LOG_ENCODING, 'UTF-8')
 const(:LOG_INDEX_NAME, 'log2json-%Y.%m.%d')
 
 # max number of log records allowed in the queue.
-const(:FLUSH_SIZE, 200)
+const(:FLUSH_SIZE, 800)
 
 # flush the queue roughly every FLUSH_TIMEOUT seconds.
 # This value must be >= 2 and it must be a multiple of 2.
-const(:FLUSH_TIMEOUT, 60)
+const(:FLUSH_TIMEOUT, 30)
 if FLUSH_TIMEOUT < 2 or FLUSH_TIMEOUT % 2 != 0
   STDERR.write("Invalid FLUSH_TIMEOUT=#{FLUSH_TIMEOUT}\n")
   exit 1
@@ -75,16 +75,16 @@ HTTP_LOG.level = Logger::WARN
 def flush_queue
   if not @queue.empty?
     req = Net::HTTP::Post.new('/_bulk')
-    req.body = @queue.join("\n") + "\n"
+    req.body = @queue.join("\n")
     response = nil
     begin
       response = @@http.request(req)
     ensure
       if response.nil? or response.code != '200'
         LOG.error(response.body) if not response.nil?
-        #FIXME: might be a good idea to push the undelivered log records to another queue in redis.
         LOG.warn("Failed sending bulk request(#{@queue.size} records) to ES! Logging the request body instead.")
         LOG.info("Failed request body:\n"+req.body)
+        #FIXME: write the queue to another file. This would allow us to resend these records on startup.
       end
     end
     @queue.clear()
@@ -107,8 +107,12 @@ def enqueue(logstr)
   log = JSON.load(logstr.force_encoding(LOG_ENCODING))
 
   # add header for each entry according to http://www.elasticsearch.org/guide/reference/api/bulk/
-  @queue << {"index" => {"_index" => es_index(log["@timestamp"]), "_type" => log["@type"]}}.to_json
-  @queue << log.to_json
+  @queue << (
+    {"index" => {"_index" => es_index(log["@timestamp"]), "_type" => log["@type"]}}.to_json <<
+    "\n" <<
+    log.to_json <<
+    "\n"
+  )
 end
 
 def load_redis_script

data/lib/log2json/railslogger.rb

@@ -1,95 +1,59 @@
 # Use it like this in your config/environments/{staging,production}.rb:
 #
-#   config.logger = ::Log2Json::create_custom_ralis_logger('/tmp/alternative.log', config)
+#   require 'log2json/railslogger'
+#   config.logger = ::Log2Json::create_custom_ralis_logger(config)
 #
 # Also, in unicorn.rb, add it like this:
 #
-#   logger ::Log2Json::create_custom_unicorn_logger('/tmp/alternative.log', self)
+#   if %w(staging production).include?(ENV['RAILS_ENV'])
+#     require 'log2json/railslogger'
+#     logger ::Log2Json::create_custom_unicorn_logger(self)
+#   end
 #
 
 require 'logger'
-    
+
 module Log2Json
 
-  def self.create_custom_logger(to_path)
-    logger = ::Logger.new(to_path)
-    logger.formatter = proc do |severity, datetime, progname, msg|
+  def self.log_formatter
+    proc do |severity, datetime, progname, msg|
       "#{datetime.strftime('%Y-%m-%dT%H:%M:%S%z')}: [#{severity}] #{$$} #{msg.gsub(/\n/, '#012')}\n"
+      # Note: Following rsyslog's convention, all newlines are converted to '#012'.
     end
-    logger
   end
 
-  # Create a custom logger that's just like the default Rails logger but
-  # additionally logs to another file that has its own formatting for easier
-  # parsing by a log2json log monitoring script.
+  # Create a custom logger that uses its own formatting for easier parsing
+  # by a log2json log monitoring script.
   #
-  def self.create_custom_rails_logger(to_path, config)
-    # Do what railties' bootstrap.rb does to initialize a default logger for a Rails app.
-    path = config.paths["log"].first
-    unless File.exist? File.dirname path
-      FileUtils.mkdir_p File.dirname path
+  def self.create_custom_rails_logger(config)
+    if config.respond_to?(:paths)  # assume rails 3
+      path = config.paths["log"].first
+      unless File.exist? File.dirname path
+        FileUtils.mkdir_p File.dirname path
+      end
+      config.colorize_logging = false
+    else                           # assume rails 2
+      path = config.log_path
+      config.active_record.colorize_logging = false 
+    end
+    logger = ::Logger.new(path)
+    logger.formatter = ::Log2Json::log_formatter
+    if defined?(ActiveSupport::TaggedLogging)
+      ActiveSupport::TaggedLogging.new(logger)
+    else
+      logger
     end
-    f = File.open path, 'a'
-    f.binmode
-    f.sync = true # make sure every write flushes
-
-    logger = ActiveSupport::TaggedLogging.new(
-      ActiveSupport::BufferedLogger.new(f)
-    )
-    logger.level = ActiveSupport::BufferedLogger.const_get(config.log_level.to_s.upcase)
-
-    logger.extend(::Log2Json::Logger.broadcast(
-        ::Log2Json::create_custom_logger(to_path)))
   end
 
   # Simiar to the custom rails logger, but for unicorn.
   #
-  def self.create_custom_unicorn_logger(to_path, config)
+  def self.create_custom_unicorn_logger(config)
     logger = ::Logger.new(config.set[:stderr_path])
-    logger.extend(::Log2Json::Logger.broadcast(
-        ::Log2Json::create_custom_logger(to_path)))
+    logger.formatter = ::Log2Json::log_formatter
+    logger
   end
 
 
-  # Code stolen from activesupport-4.0.0
-  class Logger < ::Logger
-
-    # Broadcasts logs to multiple loggers.
-    def self.broadcast(logger) # :nodoc:
-      Module.new do
-        define_method(:add) do |*args, &block|
-          logger.add(*args, &block)
-          super(*args, &block)
-        end
-
-        define_method(:<<) do |x|
-          logger << x
-          super(x)
-        end
-
-        define_method(:close) do
-          logger.close
-          super()
-        end
-
-        define_method(:progname=) do |name|
-          logger.progname = name
-          super(name)
-        end
-
-        define_method(:formatter=) do |formatter|
-          logger.formatter = formatter
-          super(formatter)
-        end
-
-        define_method(:level=) do |level|
-          logger.level = level
-          super(level)
-        end
-      end
-    end
-  end
-
 end
 
 

data/log2json-loggers.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
   s.name        = 'log2json-loggers'
-  s.version     = '0.1.5'
-  s.summary     = "Custom loggers for Rails and Unicorn that log to another file in addition to what their default loggers do."
+  s.version     = '0.1.9'
+  s.summary     = "Custom loggers for Rails and Unicorn that use log2json's single-line log format."
   s.description = IO.read(File.join(File.dirname(__FILE__), 'README'))
   s.authors     = ['Jack Kuan']
   s.email       = 'jack.kuan@thescore.com'

data/log2json.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name        = 'log2json'
-  s.version     = '0.1.9'
+  s.version     = '0.1.10'
   s.summary     = "Read, filter and ship logs. ie, poor man's roll-your-own, light-weight logstash replacement."
   s.description = IO.read(File.join(File.dirname(__FILE__), 'README'))
   s.authors     = ['Jack Kuan']

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: log2json
 version: !ruby/object:Gem::Version
-  version: 0.1.9
+  version: 0.1.10
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-01 00:00:00.000000000 Z
+date: 2013-10-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jls-grok