log2json 0.1.5 → 0.1.6

data/bin/redis2es

@@ -111,24 +111,53 @@ def enqueue(logstr)
   @queue << log.to_json
 end
 
+def load_redis_script
+  # totaly stolen from logstash
+  redis_script = <<EOF
+  local i = tonumber(ARGV[1])
+  local res = {}
+  local length = redis.call('llen',KEYS[1])
+  if length < i then i = length end
+  while (i > 0) do
+    local item = redis.call("lpop", KEYS[1])
+    if (not item) then
+      break
+    end
+    table.insert(res, item)
+    i = i-1
+  end
+  return res
+EOF
+  @redis_script_sha = @redis.script(:load, redis_script)
+end
+
 def main
+  load_redis_script()
   time_start = Time.now
   loop do
     # wait for input from the redis queue
     ret = @redis.blpop(REDIS_QUEUE, timeout: FLUSH_TIMEOUT/2)
     enqueue(ret[1]) if ret != nil
 
-    # try to queue up to FLUSH_SIZE 
-    while @queue.size < FLUSH_SIZE do
-      # Logstash's redis input actually uses a Lua script to do the lpop in one request,
-      # but let's keep it simple and stupid first here.
-      body = @redis.lpop(REDIS_QUEUE)
-      break if body.nil?
-      enqueue(body)
+    # try to queue up to FLUSH_SIZE.
+    begin
+      length = FLUSH_SIZE - @queue.length
+      @redis.evalsha(@redis_script_sha, [REDIS_QUEUE], length).each do |item|
+        enqueue(item)
+      end
+    rescue Redis::CommandError => e
+      #Note: redis script reload logic and code are copied from logstash.
+      if e.to_s =~ /NOSCRIPT/ then
+        @logger.warn("Redis may have been restarted, reloading redis batch EVAL script", :exception => e);
+        load_redis_script()
+        retry
+      else
+        raise e
+      end
     end
 
     # flush when the queue is full or when time is up.
-    if @queue.size == FLUSH_SIZE or (Time.now - time_start) >= FLUSH_TIMEOUT
+    if @queue.size >= FLUSH_SIZE or (Time.now - time_start) >= FLUSH_TIMEOUT
       time_start = Time.now  # reset timer upon a flush or timeout
       flush_queue()
     end

data/bin/tail-log.sh

@@ -27,6 +27,9 @@ IFS=$OIFS
 SINCEDB_DIR=${SINCEDB_DIR:-~/.tail-log}
 mkdir -p "$SINCEDB_DIR" || true
 
+# Whether to tail from the end instead of from the first line when there's record
+# of the file in SINCEDB.
+SINCE_EOF=${SINCE_EOF:-""}
 
 # Helper to build the arguments to tail.
 # Specifically, we expect the use of GNU tail as found in GNU coreutils.
@@ -51,11 +54,13 @@ build_tail_args() {
         continue
       fi
     fi
-    TAIL_ARGS[$((i++))]="-n+$(($(wc -l "$fpath" | cut -d' ' -f1) + 1))"
-    # Note: we can't just ask tail to seek to the end here(ie, with -n0) since
-    #       then we'd lose track of the line count.
-    # Note: if fpath doesn't exist yet, then the above evaluates to "-n+1", which
-    #       is fine.
+    if [ ! "$SINCE_EOF" ]; then
+        TAIL_ARGS[$((i++))]="-n+$(($(wc -l "$fpath" | cut -d' ' -f1) + 1))"
+        # Note: we can't just ask tail to seek to the end here(ie, with -n0) since
+        #       then we'd lose track of the line count.
+        # Note: if fpath doesn't exist yet, then the above evaluates to "-n+1", which
+        #       is fine.
+    fi
     TAIL_ARGS[$((i++))]=$fpath
   done
 }

data/log2json.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name        = 'log2json'
-  s.version     = '0.1.5'
+  s.version     = '0.1.6'
   s.summary     = "Read, filter and ship logs. ie, poor man's roll-your-own, light-weight logstash replacement."
   s.description = IO.read(File.join(File.dirname(__FILE__), 'README'))
   s.authors     = ['Jack Kuan']

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 1
-  - 5
-  version: 0.1.5
+  - 6
+  version: 0.1.6
 platform: ruby
 authors: 
 - Jack Kuan
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2013-09-13 00:00:00 -04:00
+date: 2013-09-17 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency