franz 1.6.5 → 1.6.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 15ec73cb6037c16c59a295c5d1e17160edfcff2b
-  data.tar.gz: 0e68faaa4df6f3d52d4f563c288aeca74b7cb72c
+  metadata.gz: b85070420e513988180b780d2a8a38d1b9814d34
+  data.tar.gz: 6737e0f12e74f791afcfb829647baa5046af5f4d
 SHA512:
-  metadata.gz: 7d4156f5ace594a8d7ed5846efe1d05f1b2eb23c38038bd526516dc9b351e6e4fe3bfcc2e90a79f6f36ef6b67625c51531ecf8dc359eb542a9d8995f21206794
-  data.tar.gz: 884ff2d8cae0b911310efb98d1203d2b28685c1b15c8f7e756c96deeade3ac41079a8c75bdc2b59c3880e9f529b072fb1c8ec66852a746f0e82b48b95506ef2c
+  metadata.gz: ef9f70ca09c8f20e4f56887ec557c1b34a5b8406b1b109a2bea69c10b7e4f896aa9a7f0ad57a7ba7dbd693801741bb60d6905636cff3aaead4583cdebe0a0d79
+  data.tar.gz: 38892366770b0622e58591d48c5b0b16ed2d4d58c350d8917602626dd6e089c64370b67c92a810805c436541927200cbf9218d63b5e0d1ae834ce6b98bb6d011

data/Readme.md

@@ -118,17 +118,28 @@ It's kinda like a JSON version of the Logstash config language:
       "output": {
         "rabbitmq": {
 
-          // Must be a consistently-hashed exchange
+          // Must be a consistently-hashed exchange!
           "exchange": {
             "name": "logs"
           },
 
-          // See Bunny docs for connection configuration
+          // See Bunny docs for connection configuration:
+          // http://rubybunny.info/articles/connecting.html
+          // http://rubybunny.info/articles/tls.html
           "connection": {
+            "port": 5672,
             "host": "localhost",
             "vhost": "/logs",
             "user": "logs",
             "pass": "logs"
+
+            // Sample TLS (SSL) attributes:
+            // "port": 5671,
+            // "tls": true,
+            // "tls_cert": "/path/to/client.cert",
+            // "tls_key": "/path/to/client.key",
+            // "tls_ca_certificates": [ "/path/to/cacert.pem" ],
+            // "verify_peer": true
           }
         }
       }

data/VERSION

@@ -1 +1 @@
-1.6.5
+1.6.6

data/lib/franz/agg.rb

@@ -97,6 +97,14 @@ module Franz
         return
       end
 
+      unless @ic.keep? path, message
+        log.trace \
+          event: 'unkept',
+          file: path,
+          message: message
+        return
+      end
+
       t = @ic.type path
       if t.nil?
         log.trace \

data/lib/franz/input.rb

@@ -178,7 +178,7 @@ module Franz
       File.open(path, 'w') { |f| f.write Marshal.dump(state) }
       old_checkpoints.pop # Keep last two checkpoints
       old_checkpoints.map { |c| FileUtils.rm c }
-      log.info \
+      log.debug \
         event: 'input checkpoint saved'
     end
 

data/lib/franz/input_config.rb

@@ -4,7 +4,12 @@ module Franz
 
     def initialize configs
       @configs = configs
+      @configs.map! do |c|
+        normalized_config c
+      end
       @types = Hash.new
+      @drop  = Hash.new
+      @keep  = Hash.new
     end
 
     def config path
@@ -13,52 +18,85 @@ module Franz
     end
 
     def json? path
-      begin
-        return config(path)[:json?]
-      rescue
-        return false
-      end
+      config(path)[:json?]
+    rescue
+      false
+    end
+
+    def keep? path, message
+      patterns = keeps_for(path)
+      return true if patterns.nil?
+      return true if patterns.empty?
+      apply_patterns patterns, message
     end
 
     def drop? path, message
-      begin
-        drop = config(path)[:drop]
-      rescue
-        return true # No config found, drop it!
-      end
-      if drop
-        drop = drop.is_a?(Array) ? drop : [ drop ]
-        drop.each do |pattern|
-          return true if message =~ pattern
-        end
-      end
-      return false
+      patterns = drops_for(path)
+      return true if patterns.nil?
+      return false if patterns.empty?
+      apply_patterns patterns, message
     end
 
     def type path
-      begin
-        @types.fetch path
-      rescue KeyError
-        configs.each do |config|
-          type = config[:type] if config[:includes].any? { |glob|
-            included = File.fnmatch? glob, path
-            excludes = !config[:excludes].nil?
-            excluded = excludes && config[:excludes].any? { |exlude|
-              File.fnmatch? exlude, File::basename(path)
-            }
-            included && !excluded
+      @types.fetch path
+    rescue KeyError
+      configs.each do |config|
+        type = config[:type] if config[:includes].any? { |glob|
+          included = File.fnmatch? glob, path
+          excludes = !config[:excludes].nil?
+          excluded = excludes && config[:excludes].any? { |exlude|
+            File.fnmatch? exlude, File::basename(path)
           }
-          unless type.nil?
-            @types[path] = type
-            return type
-          end
+          included && !excluded
+        }
+        unless type.nil?
+          @types[path] = type
+          return type
         end
-        log.warn \
-          event: 'type unknown',
-          file: path
-        @types[path] = nil
-        return nil
       end
+      log.warn \
+        event: 'type unknown',
+        file: path
+      @types[path] = nil
+    end
+
+
+  private
+    def normalized_config config
+      config[:keep] = realize_regexps config[:keep]
+      config[:drop] = realize_regexps config[:drop]
+      config
     end
+
+    def realize_regexps ps
+      return [] if ps.nil?
+      ps = ps.is_a?(Array) ? ps : [ ps ]
+      ps.map do |pattern|
+        Regexp.new pattern
+      end
+    end
+
+    def apply_patterns patterns, message
+      return true if patterns.nil?
+      patterns.each do |pattern|
+        return true if message =~ pattern
+      end
+      return false
+    end
+
+    def drops_for path
+      patterns_for path, :drop, @drop
+    end
+
+    def keeps_for path
+      patterns_for path, :keep, @keep
+    end
+
+    def patterns_for path, kind, memo
+      memo.fetch path
+    rescue KeyError
+      memo[path] = config(path)[kind]
+    end
+
   end
 end

data/test/test_franz_agg.rb

@@ -74,6 +74,37 @@ class TestFranzAgg < MiniTest::Test
     assert seqs[path] == 2 # should be two lines
   end
 
+  def test_handles_singular_keep
+    sample = "keep this\nbut not this\n"
+    tmp    = tempfile %w[ test1 .log ]
+    tmp.write sample
+    tmp.flush
+    tmp.close
+    start_agg keep: /^keep/
+    sleep 3
+    seqs = stop_agg
+    path = realpath tmp.path
+    assert seqs.include?(path)
+    assert_equal sample.lines.first.strip, @agg_events.shift[:message]
+    assert seqs[path] == 1 # should be one line
+  end
+
+  def test_handles_plural_keep
+    sample = "keep this\nbut not this\noh this too\nreally\n"
+    tmp    = tempfile %w[ test1 .log ]
+    tmp.write sample
+    tmp.flush
+    tmp.close
+    start_agg keep: [ /^keep/, /^oh/ ]
+    sleep 5
+    seqs = stop_agg
+    path = realpath tmp.path
+    assert seqs.include?(path)
+    assert_equal sample.lines[0].strip, @agg_events.shift[:message]
+    assert_equal sample.lines[2].strip, @agg_events.shift[:message]
+    assert seqs[path] == 2 # should be two lines
+  end
+
 private
   def tempfile prefix=nil
     Tempfile.new prefix, @tmpdir

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: franz
 version: !ruby/object:Gem::Version
-  version: 1.6.5
+  version: 1.6.6
 platform: ruby
 authors:
 - Sean Clemmer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-26 00:00:00.000000000 Z
+date: 2015-02-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: slog