franz 1.2.7 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 56734aa5910e2ab4f20bca8e21ae5cc76bbcfb9d
-  data.tar.gz: 7bb3ea267a09bdeb68b1e0320daacd04c760890a
+  metadata.gz: 9f4c78fdbc67c94815f54786007c9762f1fdfe06
+  data.tar.gz: ed630055c0636262bae059d8574b4eb66f5cdb0c
 SHA512:
-  metadata.gz: 58ceaffc0dc31e5fc7ea08c77d38c32eb7ace575f000182bb629d0c69888b4e7539e50e83d8a08cbb84e6d15b68b1cd5a03e55ed75f89ed0f35a225cdae79444
-  data.tar.gz: 80c24a9410ea0e4404541f4eecbe2a87becec61fe193758f346a1699d7db145eeba80fde2e79a598df7427421d3ed995a7ad54b77c9a9628bef7146d8a3358a1
+  metadata.gz: 317124bed5cc429c1380d0f46fe97fffdb77e521cf9c9d501fcc1c6d1e5d3ed32c7eaa1b5384c6a818c0373c66bdbda7477f381b0daf8eb4e0845d1bc9c453a9
+  data.tar.gz: 7162d593cac7d9b60663ff608f2c278ab0657f4bdd2ad14f98ce12f1237a977c6f9836659500cec43c35dd9c4be76b1eb27f9e81c038a81a8eb7b009cff5f77f

data/VERSION

@@ -1 +1 @@
-1.2.7
+1.3.0

data/bin/franz

@@ -46,7 +46,7 @@ logger = Franz::Logger.new opts[:debug], opts[:trace], opts[:log]
 
 io_bound = config[:output][:bound] || 10_000
 
-begin
+# begin
   io = SizedQueue.new io_bound
 
   # Now we'll connect to our output, RabbitMQ. This creates a new thread in the
@@ -69,13 +69,13 @@ begin
   # so we'll have to wait here in main or else we'll just exit.
   fout.join
 
-rescue SignalException => e
-  logger.fatal "#{e.inspect} #{$!}\n\t#{$@ * "\n\t"}"
-rescue SystemExit, Interrupt => e
-  logger.fatal "#{e.inspect} #{$!}\n\t#{$@ * "\n\t"}"
-ensure
-  logger.info 'Draining. This may take a while...'
-  fin.stop
-  fin.checkpoint
-  logger.info 'Bye!'
-end
+# rescue SignalException => e
+#   logger.fatal "#{e.inspect} #{$!}\n\t#{$@ * "\n\t"}"
+# rescue SystemExit, Interrupt => e
+#   logger.fatal "#{e.inspect} #{$!}\n\t#{$@ * "\n\t"}"
+# ensure
+#   logger.info 'Draining. This may take a while...'
+#   fin.stop
+#   fin.checkpoint
+#   logger.info 'Bye!'
+# end

data/lib/franz/agg.rb

@@ -25,9 +25,10 @@ module Franz
     # @option opts [Hash<Path,Fixnum>] :seqs ({}) internal "seqs" state
     # @option opts [Logger] :logger (Logger.new(STDOUT)) logger to use
     def initialize opts={}
-      @configs        = opts[:configs]        || Array.new
-      @tail_events    = opts[:tail_events]    || []
-      @agg_events     = opts[:agg_events]     || []
+      @configs     = opts[:configs]     || []
+      @tail_events = opts[:tail_events] || []
+      @agg_events  = opts[:agg_events]  || []
+
       @flush_interval = opts[:flush_interval] || 10
       @seqs           = opts[:seqs]           || Hash.new
       @logger         = opts[:logger]         || Logger.new(STDOUT)
@@ -37,21 +38,20 @@ module Franz
       @buffer = Franz::Sash.new
       @stop   = false
 
+      log.debug 'agg: configs=%s tail_events=%s agg_events=%s' % [
+        @configs, @tail_events, @agg_events
+      ]
+
       @t1 = Thread.new do
-        log.debug 'starting agg-flush'
         until @stop
           flush
           sleep flush_interval
         end
-        sleep flush_interval
-        flush
+        flush true
       end
 
       @t2 = Thread.new do
-        log.debug 'starting agg-capture'
-        until @stop
-          capture
-        end
+        capture until @stop
       end
 
       log.debug 'started agg'
@@ -141,11 +141,11 @@ module Franz
       end
     end
 
-    def flush
+    def flush force=false
       lock.synchronize do
         started = Time.now
         buffer.keys.each do |path|
-          if started - buffer.mtime(path) >= flush_interval
+          if started - buffer.mtime(path) >= flush_interval || force
             log.trace 'flushing path=%s' % path.inspect
             buffered = buffer.remove(path)
             lines    = buffered.map { |e| e[:line] }.join("\n")

data/lib/franz/discover.rb

@@ -16,9 +16,10 @@ class Franz::Discover
   # @option opts [Array<Path>] :known ([]) internal "known" state
   # @option opts [Logger] :logger (Logger.new(STDOUT)) logger to use
   def initialize opts={}
-    @configs           = opts[:configs]           || []
-    @discoveries       = opts[:discoveries]       || []
-    @deletions         = opts[:deletions]         || []
+    @configs     = opts[:configs]     || []
+    @discoveries = opts[:discoveries] || []
+    @deletions   = opts[:deletions]   || []
+
     @discover_interval = opts[:discover_interval] || 30
     @ignore_before     = opts[:ignore_before]     || 0
     @known             = opts[:known]             || []
@@ -32,8 +33,11 @@ class Franz::Discover
 
     @stop = false
 
+    log.debug 'discover: configs=%s discoveries=%s deletions=%s' % [
+      @configs, @discoveries, @deletions
+    ]
+
     @thread = Thread.new do
-      log.debug 'starting dicover-delete'
       until @stop
         until deletions.empty?
           d = deletions.pop
@@ -58,7 +62,7 @@ class Franz::Discover
   def stop
     return state if @stop
     @stop = true
-    @thread.join
+    @thread.kill
     log.debug 'stopped discover'
     return state
   end
@@ -92,17 +96,18 @@ private
   end
 
   def expand glob
-    dir_glob = File.dirname(glob)
-    file_glob = File.basename(glob)
-    files = []
-    Dir.glob(dir_glob).each do |dir|
-      next unless File::directory?(dir)
-      Dir.foreach(dir) do |fname|
-        next if fname == '.' || fname == '..'
-        next unless File.fnmatch?(file_glob, fname)
-        files << File.join(dir, fname)
-      end
-    end
-    files
+    Dir[glob]
+    # dir_glob = File.dirname(glob)
+    # file_glob = File.basename(glob)
+    # files = []
+    # Dir.glob(dir_glob).each do |dir|
+    #   next unless File::directory?(dir)
+    #   Dir.foreach(dir) do |fname|
+    #     next if fname == '.' || fname == '..'
+    #     next unless File.fnmatch?(file_glob, fname)
+    #     files << File.join(dir, fname)
+    #   end
+    # end
+    # files
   end
 end

data/lib/franz/input.rb

@@ -5,7 +5,6 @@ require 'deep_merge'
 
 require_relative 'agg'
 require_relative 'tail'
-require_relative 'tail_pool'
 require_relative 'watch'
 require_relative 'discover'
 
@@ -31,7 +30,6 @@ module Franz
         output: [],
         input: {
           ignore_before: 0,
-          tail_pool_size: 10,
           discover_bound: 10_000,
           watch_bound: 1_000,
           tail_bound: 1_000,
@@ -49,6 +47,8 @@ module Franz
       @checkpoint_path     = opts[:checkpoint].sub('*', '%d')
       @checkpoint_glob     = opts[:checkpoint]
 
+      log.debug 'input: opts=%s' % JSON::pretty_generate(opts)
+
       # The checkpoint contains a Marshalled Hash with a compact representation of
       # stateful inputs to various Franz streaming classes (e.g. the "known" option
       # to Franz::Discover). This state file is generated automatically every time
@@ -162,10 +162,14 @@ module Franz
     def checkpoint
       old_checkpoints = Dir[@checkpoint_glob].sort_by { |p| File.mtime p }
       path = @checkpoint_path % Time.now
-      File.open(path, 'w') { |f| f.write Marshal.dump(state) }
-      old_checkpoints.pop # Keep last two checkpoints
-      old_checkpoints.map { |c| FileUtils.rm c }
-      log.info 'Wrote %s' % path.inspect
+      begin
+        File.open(path, 'w') { |f| f.write Marshal.dump(state) }
+        old_checkpoints.pop # Keep last two checkpoints
+        old_checkpoints.map { |c| FileUtils.rm c }
+        log.info 'Wrote %s' % path.inspect
+      rescue Errno::EMFILE
+        log.warn 'Could not write checkpoint (too many open files)'
+      end
     end
 
   private

data/lib/franz/logger.rb

@@ -33,32 +33,49 @@ module Franz
     # @param debug [Boolean] enable DEBUG level logs
     # @param out [File] output destination for logs
     def initialize debug=false, trace=false, out=nil
+      colorize = out.nil?
       out ||= $stdout
       super out
-      colorize
+      format colorize
       @trace = true if trace
       self.level = ::Logger::INFO
       self.level = ::Logger::DEBUG if debug
     end
 
   private
-    def colorize
+    def format colorize
+      short_format = "%s\n"
+      long_format  = "%s [%s] %s -- %s\n"
+
       self.formatter = proc do |severity, datetime, _, message|
-        if level == 1
-          message.to_s.colorize(
-            color: SEVERITY_COLORS[severity.to_s][0],
-            background: SEVERITY_COLORS[severity.to_s][1]
-          ) + "\n"
-        else
-          "%s [%s] %s -- %s\n".colorize(
-            color: SEVERITY_COLORS[severity.to_s][0],
-            background: SEVERITY_COLORS[severity.to_s][1]
-          ) % [
-            severity,
-            datetime.iso8601(6),
-            File::basename(caller[4]),
-            message
-          ]
+        if colorize
+          if level == 1
+            short_format.colorize(
+              color: SEVERITY_COLORS[severity.to_s][0],
+              background: SEVERITY_COLORS[severity.to_s][1]
+            ) % message
+          else
+            long_format.colorize(
+              color: SEVERITY_COLORS[severity.to_s][0],
+              background: SEVERITY_COLORS[severity.to_s][1]
+            ) % [
+              severity,
+              datetime.iso8601(6),
+              File::basename(caller[4]),
+              message
+            ]
+          end
+        else # plain
+          if level == 1
+            short_format % message
+          else
+            long_format % [
+              severity,
+              datetime.iso8601(6),
+              File::basename(caller[4]),
+              message
+            ]
+          end
         end
       end
     end

data/lib/franz/output.rb

@@ -34,7 +34,11 @@ module Franz
 
       @logger = opts[:logger]
 
-      rabbit = Bunny.new opts[:output][:connection]
+      rabbit = Bunny.new opts[:output][:connection].merge \
+        automatically_recover: true,
+        threaded: true,
+        heartbeat: 90
+
       rabbit.start
 
       channel  = rabbit.create_channel

data/lib/franz/tail.rb

@@ -14,14 +14,19 @@ module Franz
     #
     # @param opts [Hash] a complex Hash for tail configuration
     def initialize opts={}
-      @watch_events      = opts[:watch_events]      || []
-      @tail_events       = opts[:tail_events]       || []
+      @watch_events = opts[:watch_events] || []
+      @tail_events  = opts[:tail_events]  || []
+
       @eviction_interval = opts[:eviction_interval] || 60
       @block_size        = opts[:block_size]        || 32_768 # 32 KiB
       @spread_size       = opts[:spread_size]       || 98_304 # 96 KiB
       @cursors           = opts[:cursors]           || Hash.new
       @logger            = opts[:logger]            || Logger.new(STDOUT)
 
+      log.debug 'tail: watch_events=%s tail_events=%s' % [
+        @watch_events, @tail_events
+      ]
+
       @buffer  = Hash.new { |h, k| h[k] = BufferedTokenizer.new }
       @file    = Hash.new
       @changed = Hash.new
@@ -29,44 +34,21 @@ module Franz
       @stop    = false
 
       @evict_thread = Thread.new do
-        log.debug 'starting tail-evict'
         until @stop
           evict
           sleep eviction_interval
         end
-        sleep eviction_interval
-        evict
-      end
-
-      @backlog  = Hash.new { |h, k| h[k] = Array.new }
-      @incoming = Hash.new { |h, k| h[k] = SizedQueue.new 10_000 }
-
-      @watch_thread = Thread.new do
-        log.debug 'starting tail-watch'
-        until @stop
-          e = watch_events.shift
-          @incoming[e[:path]].push e
-        end
+        evict true
       end
 
       @tail_thread = Thread.new do
         until @stop
-          had_event = false
-
-          paths = (@backlog.keys + @incoming.keys).uniq.shuffle
-
-          paths.each do |path|
-            event = @backlog[path].shift
-            begin
-              event = @incoming[path].shift(true)
-            rescue ThreadError
-              next
-            end if event.nil?
-            had_event = true
-            handle event
+          if @file.size >= OPEN_FILE_LIMIT
+            log.debug 'Sleeping until file descriptors become available...'
+            sleep 5
+          else
+            handle(watch_events.shift)
           end
-
-          sleep 0.05 unless had_event
         end
       end
 
@@ -79,9 +61,9 @@ module Franz
     def stop
       return state if @stop
       @stop = true
-      @watch_thread.kill
-      @evict_thread.join
-      @tail_thread.join
+      @watch_thread.kill rescue nil
+      @evict_thread.kill rescue nil
+      @tail_thread.kill  rescue nil
       log.debug 'stopped tail'
       return state
     end
@@ -97,6 +79,11 @@ module Franz
     def log ; @logger end
 
     def open path
+      if file.size > OPEN_FILE_LIMIT
+        log.fatal 'Absolutely too many open files!'
+        raise Errno::EMFILE
+      end
+
       return true unless file[path].nil?
       pos = @cursors.include?(path) ? @cursors[path] : 0
       begin
@@ -104,10 +91,14 @@ module Franz
         file[path].sysseek pos, IO::SEEK_SET
         @cursors[path] = pos
         @changed[path] = Time.now.to_i
+      rescue Errno::EMFILE
+        log.debug 'skipping: path=%s (too many open files)' % path.inspect
+        return false
       rescue Errno::ENOENT
+        log.debug 'skipping: path=%s (file does not exist)' % path.inspect
         return false
       end
-      log.debug 'opened: path=%s' % path.inspect
+      log.trace 'opened: path=%s' % path.inspect
       return true
     end
 
@@ -123,11 +114,6 @@ module Franz
           break if file[path].pos >= size
         end
 
-        if bytes_read >= @spread_size
-          @backlog[path].push name: :appended, path: path, size: size
-          break
-        end
-
         begin
           data = file[path].sysread @block_size
           buffer[path].extract(data).each do |line|
@@ -143,7 +129,10 @@ module Franz
         bytes_read += @cursors[path] - last_pos
       end
 
-      log.trace 'read: path=%s size=%s' % [ path.inspect, size.inspect ]
+      log.trace 'read: path=%s size=%s cursor=%s' % [
+        path.inspect, size.inspect, @cursors[path].inspect
+      ]
+
       @changed[path] = Time.now.to_i
       @reading.delete path
     end
@@ -157,13 +146,14 @@ module Franz
       log.debug 'closed: path=%s' % path.inspect
     end
 
-    def evict
+    def evict force=false
+      cutoff = Time.now.to_i - eviction_interval
       file.keys.each do |path|
-        next if @reading[path]
-        next unless @changed[path] < Time.now.to_i - eviction_interval
-        next unless file.include? path
-        next unless @incoming[path].empty?
-        next unless @backlog[path].empty?
+        unless force
+          next if @reading[path]
+          next unless @changed[path] < cutoff
+          next unless file.include? path
+        end
         file.delete(path).close
         log.debug 'evicted: path=%s' % path.inspect
       end

data/lib/franz/watch.rb

@@ -17,9 +17,10 @@ module Franz
     # @option opts [Hash<Path,State>] :stats ([]) internal "stats" state
     # @option opts [Logger] :logger (Logger.new(STDOUT)) logger to use
     def initialize opts={}
-      @discoveries    = opts[:discoveries]    || []
-      @deletions      = opts[:deletions]      || []
-      @watch_events   = opts[:watch_events]   || []
+      @discoveries  = opts[:discoveries]  || []
+      @deletions    = opts[:deletions]    || []
+      @watch_events = opts[:watch_events] || []
+
       @watch_interval = opts[:watch_interval] || 10
       @stats          = opts[:stats]          || Hash.new
       @logger         = opts[:logger]         || Logger.new(STDOUT)
@@ -31,12 +32,14 @@ module Franz
 
       @stop = false
 
+      log.debug 'watch: discoveries=%s deletions=%s watch_events=%s' % [
+        @discoveries, @deletions, @watch_events
+      ]
+
       @thread = Thread.new do
-        log.debug 'starting watch-discover'
         until @stop
           until discoveries.empty?
-            d = discoveries.pop
-            @stats[d] = nil
+            @stats[discoveries.shift] = nil
           end
           watch.each do |deleted|
             @stats.delete deleted
@@ -55,7 +58,7 @@ module Franz
     def stop
       return state if @stop
       @stop = true
-      @thread.join
+      @thread.kill
       log.debug 'stopped watch'
       return state
     end

data/lib/franz.rb

@@ -1,3 +1,6 @@
+open_file_limit = `ulimit -n`.strip.to_i - 256
+OPEN_FILE_LIMIT = open_file_limit <= 0 ? 256 : open_file_limit
+
 require_relative 'franz/agg'
 require_relative 'franz/config'
 require_relative 'franz/discover'
@@ -6,5 +9,4 @@ require_relative 'franz/logger'
 require_relative 'franz/metadata'
 require_relative 'franz/output'
 require_relative 'franz/tail'
-require_relative 'franz/tail_pool'
 require_relative 'franz/watch'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: franz
 version: !ruby/object:Gem::Version
-  version: 1.2.7
+  version: 1.3.0
 platform: ruby
 authors:
 - Sean Clemmer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-30 00:00:00.000000000 Z
+date: 2014-09-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny
@@ -119,7 +119,6 @@ files:
 - lib/franz/output.rb
 - lib/franz/sash.rb
 - lib/franz/tail.rb
-- lib/franz/tail_pool.rb
 - lib/franz/watch.rb
 - test/test_franz_agg.rb
 - test/test_franz_discover.rb

data/lib/franz/tail_pool.rb

@@ -1,68 +0,0 @@
-require 'thread'
-require 'logger'
-
-require 'consistent_hashing'
-
-module Franz
-
-  # TailPool creates a consistenly-hashed pool of Tails.
-  class TailPool
-    # Start a new TailPool thread in the background.
-    #
-    # @param opts [Hash] a complex Hash for configuration
-    def initialize opts={}
-      @size         = opts[:size]         || 5
-      @watch_events = opts[:watch_events] || []
-      @tail_events  = opts[:tail_events]  || []
-      @size         = opts[:size]         || 5
-      @logger       = opts[:logger]       || Logger.new(STDOUT)
-
-      @tails  = []
-      @ring   = ConsistentHashing::Ring.new
-      @events = Hash.new { |h, k| h[k] = SizedQueue.new 10_000 }
-
-      @size.times do |i|
-        log.debug 'starting tail_pool-tail #%d' %  i
-        @ring << @events[i]
-        @tails << Franz::Tail.new(opts.merge({
-          watch_events: @events[i],
-          tail_events: @tail_events
-        }))
-      end
-
-      @stop = false
-
-      @in_thread = Thread.new do
-        log.debug 'starting tail_pool-watch'
-        until @stop
-          e = @watch_events.shift
-          q = @ring.node_for e[:path]
-          q.push e
-        end
-      end
-
-      log.debug 'started tail_pool'
-    end
-
-    # Stop the TailPool thread. Effectively only once.
-    #
-    # @return [Hash] internal "cursors" state
-    def stop
-      return state if @stop
-      @stop = true
-      @tails.map(&:stop)
-      log.debug 'stopped tail_pool'
-      return state
-    end
-
-    # Return the internal "cursors" state
-    def state
-      @tails.map(&:state).reduce(&:merge)
-    end
-
-  private
-    attr_reader :watch_events, :tail_events, :size
-
-    def log ; @logger end
-  end
-end