log2json 0.1.23 → 0.1.24

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e3e168378db61e52503c7a5642fe792573beffed
+  data.tar.gz: c60bb1c1af3213c9f6c2cd049060da0aabb88aa0
+SHA512:
+  metadata.gz: 0fc5c89b98cbff2bfcb8ebb6d2af702e3be88fa09e4c8e655ef700d34907ef57867ccdbb6f46f4a7bde140bdd82fd6c1df314d10341942ddcae5d29cbccb2396
+  data.tar.gz: 2ca8e93b7e69d28fa69c05ca8706133eef204e12e5703300f7c690916d01cff925ff2b47812c416bdc0c99cc33dbe0e25b90c1e72d3de886bde4cd83d5f3363a

data/bin/tail-log.sh

@@ -10,9 +10,9 @@ set -e
 #     it will also print "==> file.name <== [event]" to stdout whenever a file truncation or a new file is
 #     detected. [event] will be one of "[new_file]" or "[truncated]".
 #
-#   - It allows the use of multiple -n options. Each -n will apply to the files specified on the
-#     command line in order.(ie, first -n N corresponds to the first file, etc.)
-#     If there are more files listed than the number of -n options, then the last -n applies to the
+#   - It allows the use of multiple -n options. Each -n+N will apply to the files specified on the
+#     command line in order.(ie, first -n+N corresponds to the first file, etc.)
+#     If there are more files listed than the number of -n+N options, then the last -n+N applies to the
 #     rest of the files.
 #
 TAIL=$(
@@ -35,6 +35,8 @@ IFS=$OIFS
 SINCEDB_DIR=${SINCEDB_DIR:-~/.tail-log}
 mkdir -p "$SINCEDB_DIR" || true
 
+min() { if [[ $1 -lt $2 ]]; then echo $1; else echo $2; fi; }
+
 # Helper to build the arguments to tail.
 # Specifically, we expect the use of GNU tail as found in GNU coreutils.
 # It allows us to follow(with -F) files across rotations or truncations.
@@ -52,13 +54,11 @@ build_tail_args() {
       read line < "$sincedb_path"
       t=($line)
       # if inode number is unchanged and the current file size is not smaller
-      # then we start tailing from 1 + the line number recorded in the sincedb.
+      # then we start tailing from the number of lines read(t[2]), as recorded in the sincedb,
+      # or from the last line, whichever is smaller, +1.
       if [[ ${t[0]} == $(stat -c "%i" "$fpath") &&
-            ${t[1]} -le $(stat -c "%s" "$fpath") &&
-            ${t[2]} -le "$nlines" ]]; then
-        TAIL_ARGS[$((i++))]="-n+$((t[2] + 1))"
-        # tail -n+N means start tailing from the N-th line of the file
-        # and we're even allowed to specify different -n+N for different files!
+            ${t[1]} -le $(stat -c "%s" "$fpath") ]]; then
+        TAIL_ARGS[$((i++))]="-n+$(min $((t[2]+1)) $nlines)"
         TAIL_ARGS[$((i++))]=$fpath
         continue
       fi

data/bin/track-tails

@@ -27,33 +27,72 @@ def commit
   return if @fmap.nil?
   @fmap.each do |fpath, t|
     sincedb_path = "#{@sincedb_dir}/#{fpath}.since"
+    begin
+      fstat = File.stat(fpath)
+      t[1] = fstat.size
+    rescue
+      # empty
+    end
     FileUtils.mkdir_p(File.dirname(sincedb_path))
     IO.write(sincedb_path, "#{t.join(' ')}\n")
   end
 end
 at_exit(&method(:commit))
 
+
+last_line = nil  # the previous non-header output line from tail.
+fpath = nil
 # Note: You probably want to set the locale env var: LC_CTYPE=en_US.UTF-8
 while line = gets()
   if line =~ /^==> (.+) <==(?: \[(.+)\])?$/
-    fpath = $1
-    if @fmap[fpath].nil? or $2 =~ /^new_file$|^truncated$/
+
+    last_fpath = fpath
+    fpath = $1; event = $2
+    if @fmap[fpath].nil? or event =~ /^new_file$|^truncated$/
       fstat = File.stat(fpath)
       @fmap[fpath] = [fstat.ino, fstat.size, 0]
     end
-    if line =~ /<==$/
-      STDOUT.write(line); STDOUT.flush()
-      # Note: we don't print those '==> ... <== [....]' header lines that
-      #       got added by our patch to tail because they might not be followed
-      #       by lines from the indicating files.(eg, when a new empty file appears)
+
+    # When tracking multiple files, each output of a '==> ... <==' line actually first 
+    # prints an empty line first, so that has to be taken into account.
+    if not last_fpath.nil? and event.nil? \
+       and last_line == "\n"
+      # NOTE: also check last_line because it's possible to have this at the beginning:
+      # ==> afile.txt <== [new_file]
+      # ==> afile.txt <==
+      # ...first line of log...
+      #
+      last_stat = @fmap[last_fpath]
+      last_stat[2] -= 1 if not last_stat.nil?
     end
+    #FIXME: it might be possible, although pretty pretty rare, that we got interrupted
+    #       before we had the chance toadjust the -1 to the number of lines read.
+    #       In such case, we'll record one more than the number of lines read!
+    #       This might lead to skipping one log line by tail-log.sh.
+
+    #NOTE: regardless of the kinds of header lines, we always output them because of this
+    #      special case:
+    # 
+    # echo 11111 >> file1.txt; echo 22222 >> file2.txt
+    # tail -vF file1.txt file2.txt
+    # rm file1.txt file2.txt
+    # echo 222222222 >> file2.txt
+    # echo 111111111 >> file1.txt
+    # 
+    # the last 111111111 won't have a '==> file1.txt <==' header line before it but only
+    # our '==> file1.txt <== [new_file]' header!
+    #
+    STDOUT.write(line); STDOUT.flush()
     next
   end
+
   STDOUT.write(line); STDOUT.flush()
   @fmap[fpath][2] += 1
   # Note: In the case of interruption, there's a chance that the line count is
   #       one line behind the number of log lines written to stdout. This is
   #       OK since we'd rather output a duplicate log line rather than miss
   #       one.
+  #
+  last_line = line
 end
 

data/log2json.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name        = 'log2json'
-  s.version     = '0.1.23'
+  s.version     = '0.1.24'
   s.summary     = "Read, filter and ship logs. ie, poor man's roll-your-own, light-weight logstash replacement."
   s.description = IO.read(File.join(File.dirname(__FILE__), 'README'))
   s.authors     = ['Jack Kuan']

metadata

@@ -1,20 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: log2json
 version: !ruby/object:Gem::Version
-  version: 0.1.23
-  prerelease: 
+  version: 0.1.24
 platform: ruby
 authors:
 - Jack Kuan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-03 00:00:00.000000000 Z
+date: 2014-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jls-grok
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -22,7 +20,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -30,7 +27,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -38,7 +34,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -46,7 +41,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: persistent_http
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -54,12 +48,11 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: 1.0.6
-description: ! "Log2json lets you read, filter and send logs as JSON objects via Unix
+description: "Log2json lets you read, filter and send logs as JSON objects via Unix
   pipes.\nIt is inspired by Logstash, and is meant to be compatible with it at the
   JSON\nevent/record level so that it can easily work with Kibana. \n\nReading logs
   is done via a shell script(eg, `tail`) running in its own process.\nYou then configure(see
@@ -133,27 +126,26 @@ files:
 - src/tail.c
 homepage: 
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.3.0
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Read, filter and ship logs. ie, poor man's roll-your-own, light-weight logstash
   replacement.
 test_files: []