franz 1.4.30 → 1.4.31

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 08543292f273da97e51b5c379e49a26a2f46c83a
-  data.tar.gz: 66a27eb5ae87d5b056a742cf6a10324a21571f7f
+  metadata.gz: 65feda5ab2c2a00503e281b1ad629b84a21eace2
+  data.tar.gz: 29d29bf3d409225e8b2b67a546993ba0d153543a
 SHA512:
-  metadata.gz: b991a836e9051e23ac2fcb91dd6a6425c9b51e4d41261909ce8ed24f8237105318a41d32efcc4d4ab0397be008a551517bf7b9b337c89454e15ec682d49a880a
-  data.tar.gz: 7cbdb22203703c16cb562bfbf7364ddb3abb412701053ef9e5d4a2f8a251efe1775149b88dad0488cd32b112950c851e6ea5bcb32cc27de0a1c238da1fb76d00
+  metadata.gz: 56306a42aa00a1afe1d1fb810e93618c54ea2f88854501e43eb67fa1e9bfcdce8b18a7df45eb7de6cf204b2202c41060846fbebb8851dacde99931f7cee94cfa
+  data.tar.gz: 952836ce6516ea8de4d347c444c1e8013f100600cb0f99b1ca4b5fda1e86ab38b8d788999b9d3683d87e191e0a8944ff44c9f37326e9a09be630ca1f70f8e483

data/VERSION

@@ -1 +1 @@
-1.4.30
+1.4.31

data/lib/franz/agg.rb

@@ -4,6 +4,7 @@ require 'socket'
 require 'pathname'
 
 require_relative 'sash'
+require_relative 'input_config'
 
 module Franz
 
@@ -18,14 +19,15 @@ module Franz
     # Start a new Agg thread in the background.
     #
     # @param [Hash] opts options for the aggregator
-    # @option opts [Array<Hash>] :configs ([]) file input configuration
+    # @option opts [InputConfig] :input_config shared Franz configuration
     # @option opts [Queue] :tail_events ([]) "input" queue from Tail
     # @option opts [Queue] :agg_events ([]) "output" queue
     # @option opts [Integer] :flush_interval (5) seconds between flushes
     # @option opts [Hash<Path,Fixnum>] :seqs ({}) internal "seqs" state
     # @option opts [Logger] :logger (Logger.new(STDOUT)) logger to use
     def initialize opts={}
-      @configs     = opts[:configs]     || []
+      @ic = opts[:input_config] || raise('No input_config specified')
+
       @tail_events = opts[:tail_events] || []
       @agg_events  = opts[:agg_events]  || []
 
@@ -34,7 +36,6 @@ module Franz
       @seqs           = opts[:seqs]           || Hash.new
       @logger         = opts[:logger]         || Logger.new(STDOUT)
 
-      @types  = Hash.new
       @lock   = Hash.new { |h,k| h[k] = Mutex.new }
       @buffer = Franz::Sash.new
       @stop   = false
@@ -53,7 +54,6 @@ module Franz
 
       log.debug \
         event: 'agg started',
-        configs: @configs,
         tail_events: @tail_events,
         agg_events: @agg_events
     end
@@ -76,58 +76,16 @@ module Franz
     end
 
   private
-    attr_reader :configs, :tail_events, :agg_events, :flush_interval, :seqs, :types, :lock, :buffer
+    attr_reader :tail_events, :agg_events, :flush_interval, :lock, :buffer
 
     def log ; @logger end
 
-    def type path
-      begin
-        @types.fetch path
-      rescue KeyError
-        configs.each do |config|
-          type = config[:type] if config[:includes].any? { |glob|
-            included = File.fnmatch? glob, path
-            excludes = !config[:excludes].nil?
-            excluded = excludes && config[:excludes].any? { |exlude|
-              File.fnmatch? exlude, File::basename(path)
-            }
-            included && !excluded
-          }
-          unless type.nil?
-            @types[path] = type
-            return type
-          end
-        end
-        log.warn \
-          event: 'type unknown',
-          path: path
-        @types[path] = nil
-        return nil
-      end
-    end
-
-    def config path
-      t = type(path)
-      configs.select { |c| c[:type] == t }.shift
-    end
-
     def seq path
       seqs[path] = seqs.fetch(path, 0) + 1
     end
 
-    def drop? path, message
-      drop = config(path)[:drop]
-      if drop
-        drop = drop.is_a?(Array) ? drop : [ drop ]
-        drop.each do |pattern|
-          return true if message =~ pattern
-        end
-      end
-      return false
-    end
-
     def enqueue path, message
-      if drop? path, message
+      if @ic.drop? path, message
         log.trace \
           event: 'dropped',
           path: path,
@@ -135,7 +93,7 @@ module Franz
         return
       end
 
-      t = type path
+      t = @ic.type path
       if t.nil?
         log.trace \
           event: 'enqueue skipped',
@@ -158,7 +116,7 @@ module Franz
       log.trace \
         event: 'capture',
         raw: event
-      multiline = config(event[:path])[:multiline] rescue nil
+      multiline = @ic.config(event[:path])[:multiline] rescue nil
       if multiline.nil?
         enqueue event[:path], event[:line] unless event[:line].empty?
       else

data/lib/franz/discover.rb

@@ -11,25 +11,25 @@ class Franz::Discover
   # Start a new Discover thread in the background.
   #
   # @param [Hash] opts options for the discovery
-  # @option opts [Array<Hash>] :configs ([]) file input configuration
+  # @option opts [InputConfig] :input_config shared Franz configuration
   # @option opts [Queue] :discoveries ([]) "output" queue of discovered paths
   # @option opts [Queue] :deletions ([]) "input" queue of deleted paths
   # @option opts [Integer] :discover_interval (5) seconds between discover rounds
   # @option opts [Array<Path>] :known ([]) internal "known" state
   # @option opts [Logger] :logger (Logger.new(STDOUT)) logger to use
   def initialize opts={}
-    @configs     = opts[:configs]     || []
+    @ic = opts[:input_config] || raise('No input_config specified')
+
     @discoveries = opts[:discoveries] || []
     @deletions   = opts[:deletions]   || []
 
     @discover_interval = opts[:discover_interval] || 30
-    @ignore_before     = opts[:ignore_before]     || 0
     @known             = opts[:known]             || []
     @logger            = opts[:logger]            || Logger.new(STDOUT)
 
     @known = Set.new(@known)
 
-    @configs = configs.map do |config|
+    @configs = @ic.configs.map do |config|
       config[:includes] ||= []
       config[:excludes] ||= []
       config
@@ -61,11 +61,9 @@ class Franz::Discover
 
     log.debug \
       event: 'discover started',
-      configs: configs,
       discoveries: discoveries,
       deletions: deletions,
-      discover_interval: discover_interval,
-      ignore_before: ignore_before
+      discover_interval: discover_interval
   end
 
   # Stop the Discover thread. Effectively only once.
@@ -85,7 +83,7 @@ class Franz::Discover
   end
 
 private
-  attr_reader :configs, :discoveries, :deletions, :discover_interval, :known, :ignore_before
+  attr_reader :configs, :discoveries, :deletions, :discover_interval, :known
 
   def log ; @logger end
 

data/lib/franz/input.rb

@@ -67,8 +67,6 @@ module Franz
           checkpoint: last_checkpoint_path
       end
 
-      full_state = state.nil? ? nil : state.dup
-
       state = state || {}
       known = state.keys
       stats, cursors, seqs = {}, {}, {}
@@ -85,45 +83,45 @@ module Franz
       watch_events = possibly_bounded_queue opts[:input][:watch_bound]
       tail_events  = possibly_bounded_queue opts[:input][:tail_bound]
 
+      ic = InputConfig.new opts[:input][:configs]
+
       @disover = Franz::Discover.new \
+        input_config: ic,
         discoveries: discoveries,
         deletions: deletions,
-        configs: opts[:input][:configs],
         discover_interval: opts[:input][:discover_interval],
         ignore_before: opts[:input][:ignore_before],
         logger: opts[:logger],
-        known: known,
-        full_state: full_state
+        known: known
 
       @watch = Franz::Watch.new \
+        input_config: ic,
         discoveries: discoveries,
         deletions: deletions,
         watch_events: watch_events,
         watch_interval: opts[:input][:watch_interval],
         play_catchup?: opts[:input][:play_catchup?],
         logger: opts[:logger],
-        stats: stats,
-        full_state: full_state
+        stats: stats
 
       @tail = Franz::Tail.new \
+        input_config: ic,
         watch_events: watch_events,
         tail_events: tail_events,
         block_size: opts[:input][:block_size],
         line_limit: opts[:input][:line_limit],
         read_limit: opts[:input][:read_limit],
         logger: opts[:logger],
-        cursors: cursors,
-        full_state: full_state
+        cursors: cursors
 
       @agg = Franz::Agg.new \
-        configs: opts[:input][:configs],
+        input_config: ic,
         tail_events: tail_events,
         agg_events: opts[:output],
         flush_interval: opts[:input][:flush_interval],
         buffer_limit: opts[:input][:buffer_limit],
         logger: opts[:logger],
-        seqs: seqs,
-        full_state: full_state
+        seqs: seqs
 
       @stop = false
       @t = Thread.new do

data/lib/franz/input_config.rb

@@ -0,0 +1,56 @@
+module Franz
+  class InputConfig
+    attr_reader :configs
+
+    def initialize configs
+      @configs = configs
+      @types = Hash.new
+    end
+
+    def config path
+      t = type(path)
+      configs.select { |c| c[:type] == t }.shift
+    end
+
+    def drop? path, message
+      begin
+        drop = config(path)[:drop]
+      rescue
+        return true # No config found, drop it!
+      end
+      if drop
+        drop = drop.is_a?(Array) ? drop : [ drop ]
+        drop.each do |pattern|
+          return true if message =~ pattern
+        end
+      end
+      return false
+    end
+
+    def type path
+      begin
+        @types.fetch path
+      rescue KeyError
+        configs.each do |config|
+          type = config[:type] if config[:includes].any? { |glob|
+            included = File.fnmatch? glob, path
+            excludes = !config[:excludes].nil?
+            excluded = excludes && config[:excludes].any? { |exlude|
+              File.fnmatch? exlude, File::basename(path)
+            }
+            included && !excluded
+          }
+          unless type.nil?
+            @types[path] = type
+            return type
+          end
+        end
+        log.warn \
+          event: 'type unknown',
+          path: path
+        @types[path] = nil
+        return nil
+      end
+    end
+  end
+end

data/lib/franz/tail.rb

@@ -8,12 +8,19 @@ module Franz
   # Tail receives low-level file events from a Watch and handles the actual
   # reading of files, providing a stream of lines.
   class Tail
+    ERR_NIL_READ = 1
+    ERR_INVALID_EVENT = 2
+    ERR_INCOMPLETE_READ = 3
+
     attr_reader :cursors
 
     # Start a new Tail thread in the background.
     #
     # @param opts [Hash] a complex Hash for tail configuration
+    # @option opts [InputConfig] :input_config shared Franz configuration
     def initialize opts={}
+      @ic = opts[:input_config] || raise('No input_config specified')
+
       @watch_events = opts[:watch_events] || []
       @tail_events  = opts[:tail_events]  || []
 
@@ -67,6 +74,16 @@ module Franz
       @cursors[path] ||= 0
       spread = size - @cursors[path]
 
+      if spread <= 0
+        log.trace \
+          event: 'ignoring read',
+          path: path,
+          size: size,
+          cursor: @cursors[path],
+          spread: spread
+        return
+      end
+
       if spread > @read_limit
         log.trace \
           event: 'large read',
@@ -112,7 +129,7 @@ module Franz
             size: size,
             cursor: @cursors[path],
             spread: (size - @cursors[path])
-          exit 2
+          exit ERR_NIL_READ
         end
 
         @cursors[path] += size
@@ -125,7 +142,7 @@ module Franz
           size: size,
           cursor: @cursors[path],
           spread: (size - @cursors[path])
-        exit 2
+        exit ERR_INCOMPLETE_READ
       end
     end
 
@@ -152,7 +169,7 @@ module Franz
         close event[:path]
       else
         log.fatal event: 'invalid event', raw: event
-        exit 2
+        exit ERR_INVALID_EVENT
       end
       return event[:path]
     end

data/lib/franz/watch.rb

@@ -12,6 +12,7 @@ module Franz
     # Start a new Watch thread in the background.
     #
     # @param [Hash] opts options for the watch
+    # @option opts [InputConfig] :input_config shared Franz configuration
     # @option opts [Queue] :discoveries ([]) "input" queue of discovered paths
     # @option opts [Queue] :deletions ([]) "output" queue of deleted paths
     # @option opts [Queue] :watch_events ([]) "output" queue of file events
@@ -19,6 +20,8 @@ module Franz
     # @option opts [Hash<Path,State>] :stats ([]) internal "stats" state
     # @option opts [Logger] :logger (Logger.new(STDOUT)) logger to use
     def initialize opts={}
+      @ic = opts[:input_config] || raise('No input_config specified')
+
       @discoveries  = opts[:discoveries]  || []
       @deletions    = opts[:deletions]    || []
       @watch_events = opts[:watch_events] || []
@@ -32,7 +35,7 @@ module Franz
       if @play_catchup
         log.debug event: 'play catchup'
         stats.keys.each do |path|
-          stats[path][:size] = opts[:full_state][path][:cursor] || 0
+          stats[path][:size] = 0
         end
       end
 
@@ -97,10 +100,6 @@ module Franz
       deleted = []
 
       stats.keys.each do |path|
-        # Hacks for logs we've removed
-        next if File.basename(path) =~ /^rtpstat/
-        next if File.basename(path) == 'zuora.log'
-
         old_stat = stats[path]
         stat = stat_for path
         stats[path] = stat

data/lib/franz.rb

@@ -2,6 +2,7 @@ require_relative 'franz/agg'
 require_relative 'franz/config'
 require_relative 'franz/discover'
 require_relative 'franz/input'
+require_relative 'franz/input_config'
 require_relative 'franz/logger'
 require_relative 'franz/metadata'
 require_relative 'franz/output'

data/test/test_franz_agg.rb

@@ -83,21 +83,24 @@ private
   end
 
   def start_agg config, opts={}
-    configs = [{
+    @configs = [{
       type: :test,
       includes: [ "#{@tmpdir}/*.log", "#{realpath @tmpdir}/*.log" ],
       excludes: [ "#{@tmpdir}/exclude*" ]
     }.merge(config)]
 
+    @ic = Franz::InputConfig.new @configs
+
     @discover = Franz::Discover.new({
+      input_config: @ic,
       discover_interval: 1,
       discoveries: @discoveries,
       deletions: @deletions,
-      logger: @logger,
-      configs: configs
+      logger: @logger
     }.deep_merge!(opts))
 
     @watch = Franz::Watch.new({
+      input_config: @ic,
       watch_interval: 1,
       watch_events: @watch_events,
       discoveries: @discoveries,
@@ -106,6 +109,7 @@ private
     }.deep_merge!(opts))
 
     @tail = Franz::Tail.new({
+      input_config: @ic,
       eviction_interval: 1,
       watch_events: @watch_events,
       tail_events: @tail_events,
@@ -113,7 +117,7 @@ private
     }.deep_merge!(opts))
 
     @agg = Franz::Agg.new({
-      configs: configs,
+      input_config: @ic,
       flush_interval: 2,
       tail_events: @tail_events,
       agg_events: @agg_events,

data/test/test_franz_discover.rb

@@ -65,15 +65,19 @@ private
   end
 
   def start_discovery opts={}
+    @configs = [{
+      includes: [ "#{@tmpdir}/*.log" ],
+      excludes: [ "#{@tmpdir}/exclude*" ]
+    }]
+
+    @ic = Franz::InputConfig.new @configs
+
     @discover = Franz::Discover.new({
+      input_config: @ic,
       discover_interval: 1,
       discoveries: @discoveries,
       deletions: @deletions,
-      logger: @logger,
-      configs: [{
-        includes: [ "#{@tmpdir}/*.log" ],
-        excludes: [ "#{@tmpdir}/exclude*" ]
-      }]
+      logger: @logger
     }.deep_merge!(opts))
   end
 

data/test/test_franz_tail.rb

@@ -97,18 +97,23 @@ private
   end
 
   def start_tail opts={}
+    @configs = [{
+      includes: [ "#{@tmpdir}/*.log" ],
+      excludes: [ "#{@tmpdir}/exclude*" ]
+    }]
+
+    @ic = Franz::InputConfig.new @configs
+
     @discover = Franz::Discover.new({
+      input_config: @ic,
       discover_interval: 1,
       discoveries: @discoveries,
       deletions: @deletions,
-      logger: @logger,
-      configs: [{
-        includes: [ "#{@tmpdir}/*.log" ],
-        excludes: [ "#{@tmpdir}/exclude*" ]
-      }]
+      logger: @logger
     }.deep_merge!(opts))
 
     @watch = Franz::Watch.new({
+      input_config: @ic,
       watch_interval: 1,
       watch_events: @watch_events,
       discoveries: @discoveries,
@@ -117,6 +122,7 @@ private
     }.deep_merge!(opts))
 
     @tail = Franz::Tail.new({
+      input_config: @ic,
       eviction_interval: 1,
       watch_events: @watch_events,
       tail_events: @tail_events,

data/test/test_franz_watch.rb

@@ -117,18 +117,23 @@ private
   end
 
   def start_watch opts={}
+    @configs = [{
+      includes: [ "#{@tmpdir}/*.log" ],
+      excludes: [ "#{@tmpdir}/exclude*" ]
+    }]
+
+    @ic = Franz::InputConfig.new @configs
+
     @discover = Franz::Discover.new({
+      input_config: @ic,
       discover_interval: 1,
       discoveries: @discoveries,
       deletions: @deletions,
-      logger: @logger,
-      configs: [{
-        includes: [ "#{@tmpdir}/*.log" ],
-        excludes: [ "#{@tmpdir}/exclude*" ]
-      }]
+      logger: @logger
     })
 
     @watch = Franz::Watch.new({
+      input_config: @ic,
       watch_interval: 1,
       watch_events: @queue,
       discoveries: @discoveries,

data/test/test_performance.rb

@@ -82,21 +82,24 @@ private
   end
 
   def start_agg opts={}
-    configs = [{
+    @configs = [{
       type: :test,
       includes: [ "#{@tmpdir}/*.log" ],
       excludes: [ "#{@tmpdir}/exclude*" ]
     }]
 
+    @ic = Franz::InputConfig.new @configs
+
     @discover = Franz::Discover.new({
+      input_config: @ic,
       discover_interval: @discover_interval,
       discoveries: @discoveries,
       deletions: @deletions,
-      logger: @logger,
-      configs: configs
+      logger: @logger
     }.deep_merge!(opts))
 
     @watch = Franz::Watch.new({
+      input_config: @ic,
       watch_interval: @watch_interval,
       watch_events: @watch_events,
       discoveries: @discoveries,
@@ -105,6 +108,7 @@ private
     }.deep_merge!(opts))
 
     @tail = Franz::Tail.new({
+      input_config: @ic,
       eviction_interval: @eviction_interval,
       watch_events: @watch_events,
       tail_events: @tail_events,
@@ -112,7 +116,7 @@ private
     }.deep_merge!(opts))
 
     @agg = Franz::Agg.new({
-      configs: configs,
+      input_config: @ic,
       flush_interval: @flush_interval,
       tail_events: @tail_events,
       agg_events: @agg_events,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: franz
 version: !ruby/object:Gem::Version
-  version: 1.4.30
+  version: 1.4.31
 platform: ruby
 authors:
 - Sean Clemmer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-28 00:00:00.000000000 Z
+date: 2014-11-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny
@@ -100,6 +100,7 @@ files:
 - lib/franz/config.rb
 - lib/franz/discover.rb
 - lib/franz/input.rb
+- lib/franz/input_config.rb
 - lib/franz/logger.rb
 - lib/franz/metadata.rb
 - lib/franz/output.rb
@@ -130,7 +131,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.2
 signing_key: 
 specification_version: 4
 summary: Aggregate log file events and send them elsewhere