log2json 0.1.8 → 0.1.9

data/README

@@ -28,17 +28,17 @@ work, but are not tested. The input protocol expected by Log2json is very
 simple and documented in the source code.
 
 ** The `tail-log` script uses a patched version of `tail` from the GNU coreutils
-   package. A binary of the `tail` utility compiled for Ubuntu 12.04 LTS is
-   included with the Log2json gem. If the binary doesn't work for your
-   distribution, then you'll need to get GNU coreutils-8.13, apply the patch(it
-   can be found in the src/ directory of the installed gem), and then replace
-   the bin/tail binary in the directory of the installed gem with your version
-   of the binary. ** 
+package. A binary of the `tail` utility compiled for Ubuntu 12.04 LTS is
+included with the Log2json gem. If the binary doesn't work for your
+distribution, then you'll need to get GNU coreutils-8.13, apply the patch(it
+can be found in the src/ directory of the installed gem), and then replace
+the bin/tail binary in the directory of the installed gem with your version
+of the binary. ** 
 
 P.S. If you know of a way to configure and compile ONLY the tail program in
-     coreutils, please let me know! The reason I'm not building tail post gem
-     installation is that it takes too long to configure && make because that
-     actually builds every utilties in coreutils.
+coreutils, please let me know! The reason I'm not building tail post gem
+installation is that it takes too long to configure && make because that
+actually builds every utilties in coreutils.
 
 
 For shipping logs to Redis, there's the `lines2redis` script that can be used as
@@ -61,6 +61,9 @@ On the Redis server:
   redis_queue=jsonlogs redis2es host.to.es.server
 
 
+Resources that help writing log2json filters:
 
-
+  - look at log2json.rb source and example filters
+  - http://grokdebug.herokuapp.com/
+  - http://www.ruby-doc.org/stdlib-1.9.3/libdoc/date/rdoc/DateTime.html#method-i-strftime
 

data/bin/track-tails

@@ -41,7 +41,12 @@ while line = gets()
       fstat = File.stat(fpath)
       @fmap[fpath] = [fstat.ino, fstat.size, 0]
     end
-    STDOUT.write(line); STDOUT.flush()
+    if line =~ /<==$/
+      STDOUT.write(line); STDOUT.flush()
+      # Note: we don't print those '==> ... <== [....]' header lines that
+      #       got added by our patch to tail because they might not be followed
+      #       by lines from the indicating files.(eg, when a new empty file appears)
+    end
     next
   end
   STDOUT.write(line); STDOUT.flush()

data/log2json-loggers.gemspec

@@ -0,0 +1,9 @@
+Gem::Specification.new do |s|
+  s.name        = 'log2json-loggers'
+  s.version     = '0.1.5'
+  s.summary     = "Custom loggers for Rails and Unicorn that log to another file in addition to what their default loggers do."
+  s.description = IO.read(File.join(File.dirname(__FILE__), 'README'))
+  s.authors     = ['Jack Kuan']
+  s.email       = 'jack.kuan@thescore.com'
+  s.files       = ['lib/log2json/railslogger.rb']
+end

data/log2json.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name        = 'log2json'
-  s.version     = '0.1.8'
+  s.version     = '0.1.9'
   s.summary     = "Read, filter and ship logs. ie, poor man's roll-your-own, light-weight logstash replacement."
   s.description = IO.read(File.join(File.dirname(__FILE__), 'README'))
   s.authors     = ['Jack Kuan']

metadata

@@ -1,133 +1,105 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: log2json
-version: !ruby/object:Gem::Version 
-  prerelease: false
-  segments: 
-  - 0
-  - 1
-  - 8
-  version: 0.1.8
+version: !ruby/object:Gem::Version
+  version: 0.1.9
+  prerelease: 
 platform: ruby
-authors: 
+authors:
 - Jack Kuan
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2013-09-17 00:00:00 -04:00
-default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2013-10-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: jls-grok
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    requirements: 
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 0
-        - 10
-        - 10
+      - !ruby/object:Gem::Version
         version: 0.10.10
   type: :runtime
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: redis
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
-    requirements: 
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 3
-        - 0
-        - 2
+      - !ruby/object:Gem::Version
+        version: 0.10.10
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
         version: 3.0.2
   type: :runtime
-  version_requirements: *id002
-- !ruby/object:Gem::Dependency 
-  name: persistent_http
   prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
-    requirements: 
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 1
-        - 0
-        - 5
+      - !ruby/object:Gem::Version
+        version: 3.0.2
+- !ruby/object:Gem::Dependency
+  name: persistent_http
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
         version: 1.0.5
   type: :runtime
-  version_requirements: *id003
-description: |+
-  Log2json lets you read, filter and send logs as JSON objects via Unix pipes.
-  It is inspired by Logstash, and is meant to be compatible with it at the JSON
-  event/record level so that it can easily work with Kibana. 
-  
-  Reading logs is done via a shell script(eg, `tail`) running in its own process.
-  You then configure(see the `syslog2json` or the `nginxlog2json` script for
-  examples) and run your filters in Ruby using the `Log2Json` module and its
-  contained helper classes.
-  
-  `Log2Json` reads from stdin the logs(one log record per line), parses the log
-  lines into JSON records, and then serializes and writes the records to stdout,
-  which then can be piped to another process for processing or sending it to
-  somewhere else.
-  
-  Currently, Log2json ships with a `tail-log` script that can be run as the input
-  process. It's the same as using the Linux `tail` utility with the `-v -F`
-  options except that it also tracks the positions(as the numbers of lines read
-  from the beginning of the files) in a few files in the file system so that if the
-  input process is interrupted, it can continue reading from where it left off
-  next time if the files had been followed. This feature is similar to the sincedb
-  feature in Logstash's file input.
-  
-  Note: If you don't need the tracking feature(ie, you are fine with always
-  tailling from the end of file with `-v -F -n0`), then you can just use the `tail`
-  utility that comes with your Linux distribution.(Or more specifically, the
-  `tail` from the GNU coreutils). Other versions of the `tail` utility may also
-  work, but are not tested. The input protocol expected by Log2json is very
-  simple and documented in the source code.
-  
-  ** The `tail-log` script uses a patched version of `tail` from the GNU coreutils
-     package. A binary of the `tail` utility compiled for Ubuntu 12.04 LTS is
-     included with the Log2json gem. If the binary doesn't work for your
-     distribution, then you'll need to get GNU coreutils-8.13, apply the patch(it
-     can be found in the src/ directory of the installed gem), and then replace
-     the bin/tail binary in the directory of the installed gem with your version
-     of the binary. ** 
-  
-  P.S. If you know of a way to configure and compile ONLY the tail program in
-       coreutils, please let me know! The reason I'm not building tail post gem
-       installation is that it takes too long to configure && make because that
-       actually builds every utilties in coreutils.
-  
-  
-  For shipping logs to Redis, there's the `lines2redis` script that can be used as
-  the output process in the pipe. For shipping logs from Redis to ElasticSearch,
-  Log2json provides a `redis2es` script.
-  
-  Finally here's an example of Log2json in action:
-  
-  From a client machine:
-  
-    tail-log /var/log/{sys,mail}log /var/log/{kern,auth}.log | syslog2json |
-          queue=jsonlogs \
-          flush_size=20 \
-          flush_interval=30 \
-          lines2redis host.to.redis.server 6379 0  # use redis DB 0
-  
-  
-  On the Redis server:
-  
-    redis_queue=jsonlogs redis2es host.to.es.server
-  
-  
-  
-  
-  
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.5
+description: ! "Log2json lets you read, filter and send logs as JSON objects via Unix
+  pipes.\nIt is inspired by Logstash, and is meant to be compatible with it at the
+  JSON\nevent/record level so that it can easily work with Kibana. \n\nReading logs
+  is done via a shell script(eg, `tail`) running in its own process.\nYou then configure(see
+  the `syslog2json` or the `nginxlog2json` script for\nexamples) and run your filters
+  in Ruby using the `Log2Json` module and its\ncontained helper classes.\n\n`Log2Json`
+  reads from stdin the logs(one log record per line), parses the log\nlines into JSON
+  records, and then serializes and writes the records to stdout,\nwhich then can be
+  piped to another process for processing or sending it to\nsomewhere else.\n\nCurrently,
+  Log2json ships with a `tail-log` script that can be run as the input\nprocess. It's
+  the same as using the Linux `tail` utility with the `-v -F`\noptions except that
+  it also tracks the positions(as the numbers of lines read\nfrom the beginning of
+  the files) in a few files in the file system so that if the\ninput process is interrupted,
+  it can continue reading from where it left off\nnext time if the files had been
+  followed. This feature is similar to the sincedb\nfeature in Logstash's file input.\n\nNote:
+  If you don't need the tracking feature(ie, you are fine with always\ntailling from
+  the end of file with `-v -F -n0`), then you can just use the `tail`\nutility that
+  comes with your Linux distribution.(Or more specifically, the\n`tail` from the GNU
+  coreutils). Other versions of the `tail` utility may also\nwork, but are not tested.
+  The input protocol expected by Log2json is very\nsimple and documented in the source
+  code.\n\n** The `tail-log` script uses a patched version of `tail` from the GNU
+  coreutils\npackage. A binary of the `tail` utility compiled for Ubuntu 12.04 LTS
+  is\nincluded with the Log2json gem. If the binary doesn't work for your\ndistribution,
+  then you'll need to get GNU coreutils-8.13, apply the patch(it\ncan be found in
+  the src/ directory of the installed gem), and then replace\nthe bin/tail binary
+  in the directory of the installed gem with your version\nof the binary. ** \n\nP.S.
+  If you know of a way to configure and compile ONLY the tail program in\ncoreutils,
+  please let me know! The reason I'm not building tail post gem\ninstallation is that
+  it takes too long to configure && make because that\nactually builds every utilties
+  in coreutils.\n\n\nFor shipping logs to Redis, there's the `lines2redis` script
+  that can be used as\nthe output process in the pipe. For shipping logs from Redis
+  to ElasticSearch,\nLog2json provides a `redis2es` script.\n\nFinally here's an example
+  of Log2json in action:\n\nFrom a client machine:\n\n  tail-log /var/log/{sys,mail}log
+  /var/log/{kern,auth}.log | syslog2json |\n        queue=jsonlogs \\\n        flush_size=20
+  \\\n        flush_interval=30 \\\n        lines2redis host.to.redis.server 6379
+  0  # use redis DB 0\n\n\nOn the Redis server:\n\n  redis_queue=jsonlogs redis2es
+  host.to.es.server\n\n\nResources that help writing log2json filters:\n\n  - look
+  at log2json.rb source and example filters\n  - http://grokdebug.herokuapp.com/\n
+  \ - http://www.ruby-doc.org/stdlib-1.9.3/libdoc/date/rdoc/DateTime.html#method-i-strftime\n\n"
 email: jack.kuan@thescore.com
-executables: 
+executables:
 - tail-log
 - track-tails
 - syslog2json
@@ -135,10 +107,8 @@ executables:
 - lines2redis
 - redis2es
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
+files:
 - Gemfile
 - Gemfile.lock
 - README
@@ -155,38 +125,34 @@ files:
 - lib/log2json/filters/nginx_access.rb
 - lib/log2json/filters/syslog.rb
 - lib/log2json/railslogger.rb
+- log2json-loggers.gemspec
 - log2json.gemspec
 - src/coreutils-8.13_tail.patch
 - src/tail.c
-has_rdoc: true
 homepage: 
 licenses: []
-
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      segments: 
-      - 0
-      version: "0"
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.3.6
+rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
-summary: Read, filter and ship logs. ie, poor man's roll-your-own, light-weight logstash replacement.
+summary: Read, filter and ship logs. ie, poor man's roll-your-own, light-weight logstash
+  replacement.
 test_files: []
-
+has_rdoc: