log2json 0.1.21 → 0.1.22

data/Gemfile

@@ -1,2 +1,2 @@
 source "https://rubygems.org"
-gemspec
+gemspec :name => 'log2json'

data/Gemfile.lock

@@ -1,21 +1,21 @@
 PATH
   remote: .
   specs:
-    log2json (0.1.0)
+    log2json (0.1.22)
       jls-grok (~> 0.10.10)
-      persistent_http (~> 1.0.5)
+      persistent_http (~> 1.0.6)
       redis (~> 3.0.2)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    cabin (0.5.0)
+    cabin (0.6.1)
     gene_pool (1.3.0)
-    jls-grok (0.10.10)
-      cabin (~> 0.5.0)
-    persistent_http (1.0.5)
+    jls-grok (0.10.12)
+      cabin (>= 0.6.0)
+    persistent_http (1.0.6)
       gene_pool (>= 1.3)
-    redis (3.0.3)
+    redis (3.0.6)
 
 PLATFORMS
   ruby

data/lib/log2json.rb

@@ -1,6 +1,8 @@
 require 'json'
 require 'grok-pure'
 
+require 'log2json/mlogbuffer'
+
 module Log2Json
 
 def self.main(filters, opts={}, &block)
@@ -13,7 +15,7 @@ def self.main(filters, opts={}, &block)
       key = name.to_s.downcase
       config[name] = ENV[key] if ENV.member?(key)
     end
-    spitter = ::Log2Json::Spitter.new(STDIN, ENV['type'], config)
+    spitter = ::Log2Json::Spitter.new(STDIN, ENV['type'], config.merge(opts))
   end
   if block.nil?
     block = proc do |rec|
@@ -63,7 +65,8 @@ class Spitter
     UTC_TIMESTAMP_FORMAT: "%FT%T.%6NZ",
     SOURCE_SEPERATOR_REGEX: Regexp.new("^==> (.+) <=="),
     # because /.../ screws up syntax highlighting in vim so I use Regexp.new(...)
-    
+
+    KEEP_EMTPY_LINES: false,
     TAGS: '',
     FIELDS: '',
   }
@@ -96,7 +99,7 @@ class Spitter
     @input.each_line do |line|
       line.force_encoding(options[:LOG_INPUT_ENCODING])
       line.chomp!
-      next if line.empty?
+      next if line.empty? unless @options[:KEEP_EMPTY_LINES]
       if line =~ options[:SOURCE_SEPERATOR_REGEX]
         @source_path = $1
         next

data/lib/log2json/mlogbuffer.rb

@@ -0,0 +1,134 @@
+require 'logger'
+
+module Log2Json
+
+
+class MultilineLogBuffer
+
+  LOG = Logger.new(STDERR)
+
+  # This class workaround the flaw that for some applications a log record may consist
+  # of mulitple lines without an end-of-record marker. 
+  #
+  # Example 1:
+  #
+  # [ERROR:12980 2013-11-05 09:46:29] IO_ENCODER
+  # Traceback (most recent call last):
+  #   File "/opt/score_feeder/feeds/rotowire/worker.py", line 59, in process_queue
+  #     (errorlog, rs), processed = method(xmldoc), True
+  #   File "/opt/score_feeder/feeds/__init__.py", line 206, in handle
+  #     results = fn(*args, **kwargs)
+  #   File "/opt/score_feeder/feeds/__init__.py", line 229, in handle
+  #     results = fn(*args, **kwargs)
+  #   File "/opt/score_feeder/feeds/__init__.py", line 141, in handle
+  #     results = fn(*args, **kwargs)
+  #   File "/opt/score_feeder/feeds/__init__.py", line 261, in handle
+  #     results = fn(*args, **kwargs)
+  #   File "/opt/score_feeder/feeds/__init__.py", line 174, in handle
+  #     results = fn(*args, **kwargs)
+  #   File "/opt/score_feeder/feeds/rotowire/leagues/nfl/player_news.py", line 40, in NFL_PLAYER_NEWS
+  #     snippet = xml_parse(xml_serialize(snippet))
+  #   File "/opt/score_feeder/lib/utils.py", line 239, in xml_serialize
+  #     xmlstr = lxml.etree.tostring(node).strip()
+  #   File "lxml.etree.pyx", line 3119, in lxml.etree.tostring (src/lxml/lxml.etree.c:64447)
+  #   File "serializer.pxi", line 132, in lxml.etree._tostring (src/lxml/lxml.etree.c:101602)
+  #   File "serializer.pxi", line 192, in lxml.etree._raiseSerialisationError (src/lxml/lxml.etree.c:102217)
+  # SerialisationError: IO_ENCODER
+  #
+  # Example 2:
+  #
+  # [INFO:7091 2013-11-05 09:32:46] [job 6, pri 0, req 0] NBA_MOVEMENTS finished. [c 1, u 0, d 1]
+  #     No players that match global id [2] and name [Abdul-Wahad, Tariq]. Skipping.
+  #     league: nba, method: NBA_MOVEMENTS, filename: TPnBh5movements_basketball_ab.xml
+  #
+  #     No players that match global id [3] and name [Abdur-Rahim, Shareef]. Skipping.
+  #     league: nba, method: NBA_MOVEMENTS, filename: TPnBh5movements_basketball_ab.xml
+  #
+  #     No players that match global id [3358] and name [Abromaitis, Tim]. Skipping.
+  #     league: nba, method: NBA_MOVEMENTS, filename: TPnBh5movements_basketball_ab.xml#
+  #
+  # This problem is addressed by accumulating log lines in the buffer until another start-of-log-
+  # record line appears. In the case that a multi-line log record is the last record, it will
+  # be flushed after a timeout. This assumes that even for a multi-line log record, it's often
+  # that all the lines are written out by the application together as one log record.
+  # 
+
+  STDOUT_LOCK = Mutex.new
+  def self.write_log_record(rec)
+    if not rec.nil?
+      STDOUT_LOCK.synchronize do
+        STDOUT.write(rec.to_json << "\n")
+        STDOUT.flush()
+      end
+    end
+  end
+
+  def initialize(timeout=5, &block)
+    @lock = Mutex.new
+    @flush_timeout = timeout
+
+    # buffer to aggregate a multi-line log record
+    # The first item will be a log record and the rest will
+    # be lines that are part of the @message field of the log record.
+    @log_buffer = []
+
+    @last_popped_record = nil
+
+    if not block.nil?
+      Thread.new do
+        begin
+          loop do 
+            sleep @flush_timeout
+            rec = pop_push
+            block.call(rec) if not rec.nil?
+          end
+        rescue => e
+          LOG.error(e.backtrace.join("\n"))
+          retry
+        end
+      end
+    end
+  end
+
+  # push a log record or part of its message to the buffer.
+  def push_part(part)
+    @lock.synchronize do
+#      LOG.debug("pushing part: #{part}") unless @log_buffer.empty?
+      @log_buffer << part unless @log_buffer.empty?
+      # Note: we skip pushing any partial lines before there's a log record.
+    end
+  end
+
+  # pops the log record off the buffer and then optionally push a log record into the buffer.
+  # returns the popped record.
+  def pop_push(in_record=nil)
+    rec = nil
+    @lock.synchronize do
+      rec = @log_buffer.shift() 
+      unless rec.nil?
+        if not rec.is_a?(Hash)
+          # the only way this can happen is that a timeout happened
+          # and the buffered multi-line log record got popped off before 
+          # all its lines were pushed to the buffer.
+          @log_buffer.unshift(rec)
+          rec = @last_popped_record
+        end
+        rest = @log_buffer.join("\n")
+        if not rest.empty?
+          rec['@message'] += "\n" + rest
+          rec['@tags'] << 'multiline'
+        end
+        @log_buffer.clear()
+        @last_popped_record = rec
+      end
+      @log_buffer << in_record unless in_record.nil?
+    end
+#    LOG.debug("Popped #{rec}")
+#    LOG.debug("Pushed #{in_record}")
+    rec
+  end
+
+end # end MultilineLogBuffer
+
+
+end # end Log2Json module

data/log2json.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name        = 'log2json'
-  s.version     = '0.1.21'
+  s.version     = '0.1.22'
   s.summary     = "Read, filter and ship logs. ie, poor man's roll-your-own, light-weight logstash replacement."
   s.description = IO.read(File.join(File.dirname(__FILE__), 'README'))
   s.authors     = ['Jack Kuan']

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: log2json
 version: !ruby/object:Gem::Version
-  version: 0.1.21
+  version: 0.1.22
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-11-07 00:00:00.000000000 Z
+date: 2013-11-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jls-grok
@@ -125,6 +125,7 @@ files:
 - lib/log2json/filters/base.patterns
 - lib/log2json/filters/nginx_access.rb
 - lib/log2json/filters/syslog.rb
+- lib/log2json/mlogbuffer.rb
 - lib/log2json/railslogger.rb
 - log2json-loggers.gemspec
 - log2json.gemspec