log2json 0.1.22 → 0.1.23

data/bin/redis2es

@@ -52,6 +52,10 @@ if FLUSH_TIMEOUT < 2 or FLUSH_TIMEOUT % 2 != 0
   exit 1
 end
 
+# If the distance(in seconds) between a log record's timestamp and now is
+# great than this value then the record will be dropped.
+const(:DROP_OFF_TIME, 3600*24*7)  # default is 1 week away from now.
+
 # path to the redo log file that stores the log records that this process
 # failed to send to ES.
 const(:REDO_LOG, File.join(Dir.getwd, "redo.log"))
@@ -151,10 +155,12 @@ def handle_redo_log
   FileUtils.rm(REDO_LOG)
 end
 
+
 # Determines the name of the index in ElasticSearch from the given log record's timestamp.
 def es_index(tstamp)
   begin 
     t = DateTime.parse(tstamp)
+    return nil if (Time.now - t.to_time).abs > DROP_OFF_TIME
   rescue ArgumentError
     LOG.warn("Failed parsing timestamp: #{tstamp}")
     t = DateTime.now
@@ -174,11 +180,14 @@ def enqueue(logstr)
     type, tstamp = log["@type"], log["@timestamp"]
   end
 
-  # add header for each entry according to http://www.elasticsearch.org/guide/reference/api/bulk/
-  @queue << (
-    {"index" => {"_index" => es_index(tstamp), "_type" => type}}.to_json << "\n" <<
-    logstr << "\n"
-  )
+  index_name = es_index(tstamp)
+  if not index_name.nil?
+    # add header for each entry according to http://www.elasticsearch.org/guide/reference/api/bulk/
+    @queue << (
+      {"index" => {"_index" => index_name, "_type" => type}}.to_json << "\n" <<
+      logstr << "\n"
+    )
+  end
 end
 
 def load_redis_script

data/log2json.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name        = 'log2json'
-  s.version     = '0.1.22'
+  s.version     = '0.1.23'
   s.summary     = "Read, filter and ship logs. ie, poor man's roll-your-own, light-weight logstash replacement."
   s.description = IO.read(File.join(File.dirname(__FILE__), 'README'))
   s.authors     = ['Jack Kuan']

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: log2json
 version: !ruby/object:Gem::Version
-  version: 0.1.22
+  version: 0.1.23
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-11-15 00:00:00.000000000 Z
+date: 2014-03-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jls-grok