log2json-loggers 0.1.5

data/lib/log2json/railslogger.rb

@@ -0,0 +1,96 @@
+# Use it like this in your config/environments/{staging,production}.rb:
+#
+#   config.logger = ::Log2Json::create_custom_ralis_logger('/tmp/alternative.log', config)
+#
+# Also, in unicorn.rb, add it like this:
+#
+#   logger ::Log2Json::create_custom_unicorn_logger('/tmp/alternative.log', self)
+#
+
+require 'logger'
+    
+module Log2Json
+
+  def self.create_custom_logger(to_path)
+    logger = ::Logger.new(to_path)
+    logger.formatter = proc do |severity, datetime, progname, msg|
+      "#{datetime.strftime('%Y-%m-%dT%H:%M:%S%z')}: [#{severity}] #{$$} #{msg.gsub(/\n/, '#012')}\n"
+    end
+    logger
+  end
+
+  # Create a custom logger that's just like the default Rails logger but
+  # additionally logs to another file that has its own formatting for easier
+  # parsing by a log2json log monitoring script.
+  #
+  def self.create_custom_rails_logger(to_path, config)
+    # Do what railties' bootstrap.rb does to initialize a default logger for a Rails app.
+    path = config.paths["log"].first
+    unless File.exist? File.dirname path
+      FileUtils.mkdir_p File.dirname path
+    end
+    f = File.open path, 'a'
+    f.binmode
+    f.sync = true # make sure every write flushes
+
+    logger = ActiveSupport::TaggedLogging.new(
+      ActiveSupport::BufferedLogger.new(f)
+    )
+    logger.level = ActiveSupport::BufferedLogger.const_get(config.log_level.to_s.upcase)
+
+    logger.extend(::Log2Json::Logger.broadcast(
+        ::Log2Json::create_custom_logger(to_path)))
+  end
+
+  # Simiar to the custom rails logger, but for unicorn.
+  #
+  def self.create_custom_unicorn_logger(to_path, config)
+    logger = ::Logger.new(config.set[:stderr_path])
+    logger.extend(::Log2Json::Logger.broadcast(
+        ::Log2Json::create_custom_logger(to_path)))
+  end
+
+
+  # Code stolen from activesupport-4.0.0
+  class Logger < ::Logger
+
+    # Broadcasts logs to multiple loggers.
+    def self.broadcast(logger) # :nodoc:
+      Module.new do
+        define_method(:add) do |*args, &block|
+          logger.add(*args, &block)
+          super(*args, &block)
+        end
+
+        define_method(:<<) do |x|
+          logger << x
+          super(x)
+        end
+
+        define_method(:close) do
+          logger.close
+          super()
+        end
+
+        define_method(:progname=) do |name|
+          logger.progname = name
+          super(name)
+        end
+
+        define_method(:formatter=) do |formatter|
+          logger.formatter = formatter
+          super(formatter)
+        end
+
+        define_method(:level=) do |level|
+          logger.level = level
+          super(level)
+        end
+      end
+    end
+  end
+
+end
+
+
+

metadata

@@ -0,0 +1,128 @@
+--- !ruby/object:Gem::Specification 
+name: log2json-loggers
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 5
+  version: 0.1.5
+platform: ruby
+authors: 
+- Jack Kuan
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2013-09-13 00:00:00 -04:00
+default_executable: 
+dependencies: []
+
+description: |+
+  Log2json lets you read, filter and send logs as JSON objects via Unix pipes.
+  It is inspired by Logstash, and is meant to be compatible with it at the JSON
+  event/record level so that it can easily work with Kibana. 
+  
+  Reading logs is done via a shell script(eg, `tail`) running in its own process.
+  You then configure(see the `syslog2json` or the `nginxlog2json` script for
+  examples) and run your filters in Ruby using the `Log2Json` module and its
+  contained helper classes.
+  
+  `Log2Json` reads from stdin the logs(one log record per line), parses the log
+  lines into JSON records, and then serializes and writes the records to stdout,
+  which then can be piped to another process for processing or sending it to
+  somewhere else.
+  
+  Currently, Log2json ships with a `tail-log` script that can be run as the input
+  process. It's the same as using the Linux `tail` utility with the `-v -F`
+  options except that it also tracks the positions(as the numbers of lines read
+  from the beginning of the files) in a few files in the file system so that if the
+  input process is interrupted, it can continue reading from where it left off
+  next time if the files had been followed. This feature is similar to the sincedb
+  feature in Logstash's file input.
+  
+  Note: If you don't need the tracking feature(ie, you are fine with always
+  tailling from the end of file with `-v -F -n0`), then you can just use the `tail`
+  utility that comes with your Linux distribution.(Or more specifically, the
+  `tail` from the GNU coreutils). Other versions of the `tail` utility may also
+  work, but are not tested. The input protocol expected by Log2json is very
+  simple and documented in the source code.
+  
+  ** The `tail-log` script uses a patched version of `tail` from the GNU coreutils
+     package. A binary of the `tail` utility compiled for Ubuntu 12.04 LTS is
+     included with the Log2json gem. If the binary doesn't work for your
+     distribution, then you'll need to get GNU coreutils-8.13, apply the patch(it
+     can be found in the src/ directory of the installed gem), and then replace
+     the bin/tail binary in the directory of the installed gem with your version
+     of the binary. ** 
+  
+  P.S. If you know of a way to configure and compile ONLY the tail program in
+       coreutils, please let me know! The reason I'm not building tail post gem
+       installation is that it takes too long to configure && make because that
+       actually builds every utilties in coreutils.
+  
+  
+  For shipping logs to Redis, there's the `lines2redis` script that can be used as
+  the output process in the pipe. For shipping logs from Redis to ElasticSearch,
+  Log2json provides a `redis2es` script.
+  
+  Finally here's an example of Log2json in action:
+  
+  From a client machine:
+  
+    tail-log /var/log/{sys,mail}log /var/log/{kern,auth}.log | syslog2json |
+          queue=jsonlogs \
+          flush_size=20 \
+          flush_interval=30 \
+          lines2redis host.to.redis.server 6379 0  # use redis DB 0
+  
+  
+  On the Redis server:
+  
+    redis_queue=jsonlogs redis2es host.to.es.server
+  
+  
+  
+  
+  
+email: jack.kuan@thescore.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/log2json/railslogger.rb
+has_rdoc: true
+homepage: 
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.6
+signing_key: 
+specification_version: 3
+summary: Custom loggers for Rails and Unicorn that log to another file in addition to what their default loggers do.
+test_files: []
+