log2json 0.1.19 → 0.1.20

data/bin/tail-log.sh

@@ -65,9 +65,13 @@ build_tail_args() {
     fi
 
     # at this point, no last position was recorded in the SINCEDB for fpath,
-    # in this case we'd tail from the end of file.
-
-    TAIL_ARGS[$((i++))]="-n+$(( nlines + 1 ))"
+    # in this case we'd tail from the end of file unless the FROM_START env
+    # var is set.
+    if [ "$FROM_START" ]; then
+        TAIL_ARGS[$((i++))]="-n+$FROM_START"
+    else
+        TAIL_ARGS[$((i++))]="-n+$(( nlines + 1 ))"
+    fi
     # Note: we can't just ask tail to seek to the end here(ie, with -n0) since
     #       then we'd lose track of the line count.
     # Note: if fpath doesn't exist yet, then the above evaluates to "-n+1", which

data/lib/log2json.rb

@@ -3,7 +3,7 @@ require 'grok-pure'
 
 module Log2Json
 
-def self.main(filters, opts={})
+def self.main(filters, opts={}, &block)
   output = opts[:output] || STDOUT
   spitter = opts[:spitter]
   if spitter.nil?
@@ -15,15 +15,23 @@ def self.main(filters, opts={})
     end
     spitter = ::Log2Json::Spitter.new(STDIN, ENV['type'], config)
   end
+  if block.nil?
+    block = proc do |rec|
+      if not rec['@timestamp'].nil?
+        output.write(rec.to_json() << "\n")
+        output.flush()
+        # NOTE: Ruby's built-in json module, by default, doesn't output any
+        #       literal newline characters while serializing. So using
+        #       newlines as json record separator is fine here.
+      end
+    end
+  end
   spitter.each_record do |rec|
-    filters[rec['@type']].each { |f| f.filter(rec) }
-    if ! rec['@timestamp'].nil?
-      output.write(rec.to_json() << "\n")
-      output.flush()
-      # NOTE: Ruby's built-in json module, by default, doesn't output any
-      #       literal newline characters while serializing. So using
-      #       newlines as json record separator is fine here.
+    filters[rec['@type']].each do |f|
+      rec = f.filter(rec)
+      break if rec.nil?
     end
+    block.call(rec) unless rec.nil?
   end
 end
 
@@ -156,9 +164,7 @@ class GrokFilter
   # Any '@' key-values configured for this filter will also
   # be added to the record after merging the captured groups.
   #
-  # Return the record at the end if there's a match else return nil.
-  # If the '@timestamp' attribute is removed from a record then the record will
-  # be dropped.
+  # Return the record at the end.
   def filter(record)
     grok, match = @pile.match(record['@message'])
     if match
@@ -195,13 +201,10 @@ class GrokFilter
       end
       (fields['filtered_by'] ||= []) << name
       if @filter_block
-        @filter_block.call(record)
-      else
-        record
+        record = @filter_block.call(record)
       end
-    else
-      nil
     end
+    record
   end
 end  # end class GrokFilter
 

data/log2json.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name        = 'log2json'
-  s.version     = '0.1.19'
+  s.version     = '0.1.20'
   s.summary     = "Read, filter and ship logs. ie, poor man's roll-your-own, light-weight logstash replacement."
   s.description = IO.read(File.join(File.dirname(__FILE__), 'README'))
   s.authors     = ['Jack Kuan']

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: log2json
 version: !ruby/object:Gem::Version
-  version: 0.1.19
+  version: 0.1.20
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-30 00:00:00.000000000 Z
+date: 2013-11-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jls-grok