franz 1.3.1 → 1.4.14

data/lib/franz/tail.rb

@@ -1,7 +1,7 @@
 require 'thread'
 require 'logger'
 
-require 'buftok'
+require 'eventmachine'
 
 module Franz
 
@@ -17,42 +17,23 @@ module Franz
       @watch_events = opts[:watch_events] || []
       @tail_events  = opts[:tail_events]  || []
 
-      @eviction_interval = opts[:eviction_interval] || 60
-      @block_size        = opts[:block_size]        || 32_768 # 32 KiB
-      @spread_size       = opts[:spread_size]       || 98_304 # 96 KiB
-      @cursors           = opts[:cursors]           || Hash.new
-      @logger            = opts[:logger]            || Logger.new(STDOUT)
-
-      log.debug 'tail: watch_events=%s tail_events=%s' % [
-        @watch_events, @tail_events
-      ]
-
-      @buffer  = Hash.new { |h, k| h[k] = BufferedTokenizer.new }
-      @file    = Hash.new
-      @changed = Hash.new
-      @reading = Hash.new
-      @stop    = false
-
-      @evict_thread = Thread.new do
-        until @stop
-          evict
-          sleep eviction_interval
-        end
-        evict true
-      end
+      @line_limit = opts[:line_limit]  || 10_240 # 10 KiB
+      @block_size = opts[:block_size]  || 32_768 # 32 KiB
+      @cursors    = opts[:cursors]     || Hash.new
+      @logger     = opts[:logger]      || Logger.new(STDOUT)
+
+      @buffer = Hash.new { |h, k| h[k] = BufferedTokenizer.new("\n", @line_limit) }
+      @stop   = false
 
       @tail_thread = Thread.new do
-        until @stop
-          if @file.size >= OPEN_FILE_LIMIT
-            log.debug 'Sleeping until file descriptors become available...'
-            sleep 5
-          else
-            handle(watch_events.shift)
-          end
-        end
+        handle(watch_events.shift) until @stop
       end
 
-      log.debug 'started tail'
+      log.debug \
+        event: 'tail started',
+        watch_events: watch_events,
+        tail_events: tail_events,
+        block_size: block_size
     end
 
     # Stop the Tail thread. Effectively only once.
@@ -62,9 +43,8 @@ module Franz
       return state if @stop
       @stop = true
       @watch_thread.kill rescue nil
-      @evict_thread.kill rescue nil
       @tail_thread.kill  rescue nil
-      log.debug 'stopped tail'
+      log.debug event: 'tail stopped'
       return state
     end
 
@@ -74,93 +54,51 @@ module Franz
     end
 
   private
-    attr_reader :watch_events, :tail_events, :eviction_interval, :block_size, :cursors, :file, :buffer, :changed, :reading
+    attr_reader :watch_events, :tail_events, :block_size, :cursors, :buffer, :reading
 
     def log ; @logger end
 
-    def open path
-      if file.size > OPEN_FILE_LIMIT
-        log.fatal 'Absolutely too many open files!'
-        raise Errno::EMFILE
-      end
-
-      return true unless file[path].nil?
-      pos = @cursors.include?(path) ? @cursors[path] : 0
-      begin
-        file[path] = File.open(path)
-        file[path].sysseek pos, IO::SEEK_SET
-        @cursors[path] = pos
-        @changed[path] = Time.now.to_i
-      rescue Errno::EMFILE
-        log.debug 'skipping: path=%s (too many open files)' % path.inspect
-        return false
-      rescue Errno::ENOENT
-        log.debug 'skipping: path=%s (file does not exist)' % path.inspect
-        return false
-      end
-      log.trace 'opened: path=%s' % path.inspect
-      return true
-    end
-
     def read path, size
-      @reading[path] = true
-
-      bytes_read = 0
+      log.trace \
+        event: 'read',
+        path: path,
+        size: size
+      @cursors[path] ||= 0
       loop do
-        begin
-          break if file[path].pos >= size
-        rescue NoMethodError
-          break unless open(path)
-          break if file[path].pos >= size
-        end
+        break if @cursors[path] >= size
 
         begin
-          data = file[path].sysread @block_size
+          data = IO::read path, @block_size, @cursors[path]
           buffer[path].extract(data).each do |line|
-            log.trace 'captured: path=%s line=%s' % [ path, line ]
+            size = line.bytesize
+            if size > @line_limit
+              log.fatal \
+                event: 'killed',
+                reason: 'line overflow',
+                path: path,
+                size: size,
+                limit: @line_limit,
+                pid: $$
+              exit(2)
+            end
             tail_events.push path: path, line: line
           end
-        rescue EOFError, Errno::ENOENT
+          @cursors[path] += data.bytesize
+        rescue EOFError, Errno::ENOENT, NoMethodError
           # we're done here
         end
-
-        last_pos = @cursors[path]
-        @cursors[path] = file[path].pos
-        bytes_read += @cursors[path] - last_pos
       end
-
-      log.trace 'read: path=%s size=%s cursor=%s' % [
-        path.inspect, size.inspect, @cursors[path].inspect
-      ]
-
-      @changed[path] = Time.now.to_i
-      @reading.delete path
     end
 
     def close path
-      @reading[path] = true # prevent evict from interrupting
-      file.delete(path).close if file.include? path
-      @cursors.delete(path)
-      @changed.delete(path)
-      @reading.delete(path)
-      log.debug 'closed: path=%s' % path.inspect
-    end
-
-    def evict force=false
-      cutoff = Time.now.to_i - eviction_interval
-      file.keys.each do |path|
-        unless force
-          next if @reading[path]
-          next unless @changed[path] < cutoff
-          next unless file.include? path
-        end
-        file.delete(path).close
-        log.debug 'evicted: path=%s' % path.inspect
-      end
+      log.trace event: 'close', path: path
+      @cursors[path] = 0
     end
 
     def handle event
-      log.trace 'handle: event=%s' % event.inspect
+      log.trace \
+        event: 'handle',
+        raw: event
       case event[:name]
       when :created
       when :replaced
@@ -176,6 +114,7 @@ module Franz
       else
         raise 'invalid event'
       end
+      return event[:path]
     end
   end
 end

data/lib/franz/watch.rb

@@ -1,3 +1,5 @@
+require 'set'
+
 require 'logger'
 
 module Franz
@@ -21,35 +23,57 @@ module Franz
       @deletions    = opts[:deletions]    || []
       @watch_events = opts[:watch_events] || []
 
+      # Check if files older than STALE_INTERVAL have been updated every
+      # SKIP_INTERVAL seconds. Useful if you've got tons of old files.
+      @stale_interval = opts[:stale_interval] || 900 # 15 minutes
+      @skip_interval  = opts[:skip_interval]  || 120 # 2 minutes
+
+      @play_catchup   = opts[:play_catchup?].nil? ? true : opts[:play_catchup?]
       @watch_interval = opts[:watch_interval] || 10
       @stats          = opts[:stats]          || Hash.new
       @logger         = opts[:logger]         || Logger.new(STDOUT)
 
-      # Need to resend old events to make sure Tail catches up
-      stats.each do |path, old_stat|
-        watch_events.push name: :appended, path: path, size: old_stat[:size]
+      @num_skipped = 0
+
+      # Make sure we're up-to-date by rewinding our old stats to our cursors
+      if @play_catchup
+        log.debug event: 'play catchup'
+        stats.keys.each do |path|
+          stats[path][:size] = opts[:full_state][path][:cursor] || 0
+        end
       end
 
       @stop = false
-
-      log.debug 'watch: discoveries=%s deletions=%s watch_events=%s' % [
-        @discoveries, @deletions, @watch_events
-      ]
-
       @thread = Thread.new do
+        stale_updated = Time.now - @stale_interval
+
         until @stop
+          started = Time.now
           until discoveries.empty?
             @stats[discoveries.shift] = nil
           end
-          watch.each do |deleted|
+
+          skip_stale = true
+          if stale_updated < started - @skip_interval
+            skip_stale = false
+            stale_updated = Time.now
+          end
+
+          watch(skip_stale).each do |deleted|
             @stats.delete deleted
             deletions.push deleted
           end
+
           sleep watch_interval
         end
       end
 
-      log.debug 'started watch'
+      log.debug \
+        event: 'watch started',
+        discoveries: discoveries,
+        deletions: deletions,
+        watch_events: watch_events,
+        watch_interval: watch_interval
     end
 
     # Stop the Watch thread. Effectively only once.
@@ -59,7 +83,7 @@ module Franz
       return state if @stop
       @stop = true
       @thread.kill
-      log.debug 'stopped watch'
+      log.debug event: 'watch stopped'
       return state
     end
 
@@ -74,21 +98,38 @@ module Franz
     def log ; @logger end
 
     def enqueue name, path, size=nil
-      log.trace 'enqueue: name=%s path=%s size=%s' % [
-        name.inspect, path.inspect, size.inspect
-      ]
+      log.trace \
+        event: 'enqueue',
+        name: name,
+        path: path,
+        size: size
       watch_events.push name: name, path: path, size: size
     end
 
-    def watch
+    def watch skip_stale=true
+      log.debug \
+        event: 'watch',
+        skip_stale: skip_stale
       deleted = []
+      skip_past = Time.now - @stale_interval
+
       stats.keys.each do |path|
-        old_stat    = stats[path]
-        stat        = stat_for path
+        # Hacks for logs we've removed
+        next if File.basename(path) =~ /^rtpstat/
+        next if File.basename(path) == 'zuora.log'
+
+        old_stat = stats[path]
+
+        next if skip_stale \
+        && old_stat \
+        && old_stat[:mtime] \
+        && old_stat[:mtime] < skip_past
+
+        stat = stat_for path
         stats[path] = stat
 
         if file_created? old_stat, stat
-          enqueue :created, path
+          # enqueue :created, path
         elsif file_deleted? old_stat, stat
           enqueue :deleted, path
           deleted << path
@@ -119,7 +160,8 @@ module Franz
             maj: stat.dev_major,
             min: stat.dev_minor
           },
-          size: stat.size
+          size: stat.size,
+          mtime: stat.mtime
         }
       rescue Errno::ENOENT
         nil

data/lib/franz.rb

@@ -1,6 +1,3 @@
-open_file_limit = Process.getrlimit(:NOFILE).first
-OPEN_FILE_LIMIT = open_file_limit <= 0 ? 256 : open_file_limit
-
 require_relative 'franz/agg'
 require_relative 'franz/config'
 require_relative 'franz/discover'

data/test/test_franz_agg.rb

@@ -33,7 +33,7 @@ class TestFranzAgg < MiniTest::Test
     tmp.write sample
     tmp.flush
     tmp.close
-    start_agg
+    start_agg multiline: /^multiline/
     sleep 3
     seqs = stop_agg
     path = realpath tmp.path
@@ -42,6 +42,37 @@ class TestFranzAgg < MiniTest::Test
     assert seqs[path] == 1 # should be one line
   end
 
+  def test_handles_singular_drop
+    sample = "drop this\nbut not this\n"
+    tmp    = tempfile %w[ test1 .log ]
+    tmp.write sample
+    tmp.flush
+    tmp.close
+    start_agg drop: /^drop/
+    sleep 3
+    seqs = stop_agg
+    path = realpath tmp.path
+    assert seqs.include?(path)
+    assert_equal sample.lines.last.strip, @agg_events.shift[:message]
+    assert seqs[path] == 1 # should be one line
+  end
+
+  def test_handles_plural_drop
+    sample = "drop this\nbut not this\nignore this too\nreally\n"
+    tmp    = tempfile %w[ test1 .log ]
+    tmp.write sample
+    tmp.flush
+    tmp.close
+    start_agg drop: [ /^drop/, /^ignore/ ]
+    sleep 5
+    seqs = stop_agg
+    path = realpath tmp.path
+    assert seqs.include?(path)
+    assert_equal sample.lines[1].strip, @agg_events.shift[:message]
+    assert_equal sample.lines[3].strip, @agg_events.shift[:message]
+    assert seqs[path] == 2 # should be two lines
+  end
+
 private
   def tempfile prefix=nil
     Tempfile.new prefix, @tmpdir
@@ -51,13 +82,12 @@ private
     Pathname.new(path).realpath.to_s
   end
 
-  def start_agg opts={}
+  def start_agg config, opts={}
     configs = [{
       type: :test,
-      multiline: /^multiline/,
       includes: [ "#{@tmpdir}/*.log", "#{realpath @tmpdir}/*.log" ],
       excludes: [ "#{@tmpdir}/exclude*" ]
-    }]
+    }.merge(config)]
 
     @discover = Franz::Discover.new({
       discover_interval: 1,

data/test/test_franz_discover.rb

@@ -24,23 +24,23 @@ class TestFranzDiscover < MiniTest::Test
   end
 
   def test_discovers_existing_file
-    tmp = tempfile %w[ test .log ]
+    tmp = tempfile %w[ test1 .log ]
     start_discovery known: []
-    sleep 0.001 # Time to discover
+    sleep 2 # Time to discover
     known = stop_discovery
     assert known.include?(tmp.path)
   end
 
   def test_discovers_new_file
     start_discovery known: []
-    tmp = tempfile %w[ test .log ]
-    sleep 1 # Time to discover
+    tmp = tempfile %w[ test2 .log ]
+    sleep 3 # Time to discover
     known = stop_discovery
     assert known.include?(tmp.path)
   end
 
   def test_deletes_deleted_file
-    tmp = tempfile %w[ test .log ]
+    tmp = tempfile %w[ test3 .log ]
     start_discovery known: []
     # at this point, we know Discover has already picked up tmp
     delete tmp.path
@@ -50,11 +50,11 @@ class TestFranzDiscover < MiniTest::Test
   end
 
   def test_deletes_unknown_file
-    tmp = tempfile %w[ test .log ]
+    tmp = tempfile %w[ test4 .log ]
     delete tmp.path
     # tmp never exists as far as Discover is aware
     start_discovery known: []
-    sleep 0.001
+    sleep 2
     known = stop_discovery
     assert !known.include?(tmp.path)
   end

data/test/test_franz_tail.rb

@@ -25,6 +25,27 @@ class TestFranzTail < MiniTest::Test
     FileUtils.rm_rf @tmpdir
   end
 
+  def test_handles_reading_after_deletion
+    sample = "Hello, world!\n"
+    start_tail
+    tmp = tempfile %w[ test4 .log ]
+    path = tmp.path
+    tmp.write sample
+    tmp.flush
+    tmp.close
+    sleep 1
+    FileUtils.rm_rf path
+    sleep 2
+    File.open(path, 'w') do |f|
+      f.write sample
+      f.flush
+    end
+    sleep 4
+    cursors = stop_tail
+    assert cursors.include?(tmp.path)
+    assert cursors[tmp.path] == sample.length
+  end
+
   def test_handles_existing_file
     sample = "Hello, world!\n"
     tmp = tempfile %w[ test1 .log ]
@@ -57,7 +78,7 @@ class TestFranzTail < MiniTest::Test
     eviction_interval = 2
     start_tail eviction_interval: eviction_interval
     sleep 0
-    tmp = tempfile %w[ test2 .log ]
+    tmp = tempfile %w[ test3 .log ]
     tmp.write sample
     tmp.flush
     sleep eviction_interval / 2
@@ -70,29 +91,6 @@ class TestFranzTail < MiniTest::Test
     assert cursors[tmp.path] == sample.length * 2
   end
 
-  def test_handles_reading_after_deletion
-    sample = "Hello, world!\n"
-    eviction_interval = 2
-    start_tail eviction_interval: eviction_interval
-    sleep 0
-    tmp = tempfile %w[ test2 .log ]
-    path = tmp.path
-    tmp.write sample
-    tmp.flush
-    tmp.close
-    sleep eviction_interval / 2
-    FileUtils.rm_rf path
-    sleep eviction_interval
-    File.open(path, 'w') do |f|
-      f.write sample
-      f.flush
-    end
-    sleep eviction_interval * 2
-    cursors = stop_tail
-    assert cursors.include?(tmp.path)
-    assert cursors[tmp.path] == sample.length
-  end
-
 private
   def tempfile prefix=nil
     Tempfile.new prefix, @tmpdir

data/test/test_franz_watch.rb

@@ -25,7 +25,7 @@ class TestFranzWatch < MiniTest::Test
   end
 
   def test_handles_existing_file
-    tmp = tempfile %w[ test .log ]
+    tmp = tempfile %w[ test1 .log ]
     start_watch
     sleep 2
     stats = stop_watch
@@ -35,7 +35,7 @@ class TestFranzWatch < MiniTest::Test
 
   def test_handles_existing_file_with_content
     content = "Hello, world!\n"
-    tmp = tempfile %w[ test .log ]
+    tmp = tempfile %w[ test2 .log ]
     tmp.write content
     tmp.flush
     start_watch
@@ -47,7 +47,7 @@ class TestFranzWatch < MiniTest::Test
 
   def test_handles_new_file
     start_watch
-    tmp = tempfile %w[ test .log ]
+    tmp = tempfile %w[ test3 .log ]
     sleep 3
     stats = stop_watch
     assert stats.include?(tmp.path)
@@ -57,7 +57,7 @@ class TestFranzWatch < MiniTest::Test
   def test_handles_new_file_with_content
     start_watch
     content = "Hello, world!\n"
-    tmp = tempfile %w[ test .log ]
+    tmp = tempfile %w[ test4 .log ]
     tmp.write content
     tmp.flush
     sleep 3
@@ -70,7 +70,7 @@ class TestFranzWatch < MiniTest::Test
     long_content = "Hello, world!\n"
     short_content = "Bye!\n"
 
-    tmp = tempfile %w[ test .log ]
+    tmp = tempfile %w[ test5 .log ]
     tmp.write long_content
     tmp.flush
 
@@ -91,12 +91,12 @@ class TestFranzWatch < MiniTest::Test
     content1 = "Hello, world!\n"
     content2 = "Bye!\n"
 
-    tmp1 = tempfile %w[ test .log ]
+    tmp1 = tempfile %w[ test6 .log ]
     tmp1.write content1
     tmp1.flush
     tmp1.close
 
-    tmp2 = tempfile %w[ exclude .log ]
+    tmp2 = tempfile %w[ exclude6 .log ]
     tmp2.write content2
     tmp2.flush
     tmp2.close

data/test/test_performance.rb

@@ -0,0 +1,126 @@
+require 'thread'
+require 'tmpdir'
+require 'tempfile'
+require 'fileutils'
+require 'pathname'
+require 'minitest/autorun'
+
+require 'deep_merge'
+
+require_relative '../lib/franz'
+
+Thread.abort_on_exception = true
+
+class TestPerformance < MiniTest::Test
+  def setup
+    @ulimit = Process.getrlimit(:NOFILE).first
+
+    @discover_interval = 2
+    @watch_interval    = 2
+    @eviction_interval = 2
+    @flush_interval    = 2
+
+    @tmpdir       = File.join(Dir.pwd, 'tmp')
+    @discoveries  = Queue.new
+    @deletions    = Queue.new
+    @watch_events = Queue.new
+    @tail_events  = Queue.new
+    @agg_events   = Queue.new
+    @logger       = Logger.new STDERR
+    @logger.level = Logger::WARN
+
+    FileUtils.rm_rf @tmpdir
+    FileUtils.mkdir_p @tmpdir
+  end
+
+  def teardown
+    # nop
+  end
+
+  def test_handles_too_many_files_for_ulimit
+    sample = "Why, hello there, World! How lovely to see you this morning."
+    num_files = @ulimit * 2
+    num_lines_per_file = 100
+
+    paths = []
+    num_files.times do |i|
+      path = File.join(@tmpdir, "test.#{i}.log")
+      File.open(path, 'w') do |f|
+        num_lines_per_file.times do
+          f.puts sample
+        end
+      end
+      paths << path
+    end
+
+    num_events = num_files * num_lines_per_file
+
+    start_agg
+    started = Time.now
+    until @agg_events.size == num_events
+      sleep 1
+    end
+    seqs = stop_agg
+    elapsed = Time.now - started
+
+    @logger.fatal('%ds elapsed' % elapsed)
+    @logger.fatal('%d events' % num_events)
+    @logger.fatal('%f events/s' % ( (1.0 * num_events) / (1.0 * elapsed) ))
+    assert_equal(paths.size, seqs.keys.size)
+    assert_equal(paths.size * num_lines_per_file, @agg_events.size)
+  end
+
+
+
+private
+  def tempfile prefix=nil
+    Tempfile.new prefix, @tmpdir
+  end
+
+  def realpath path
+    Pathname.new(path).realpath.to_s.gsub(/^\/private/, '')
+  end
+
+  def start_agg opts={}
+    configs = [{
+      type: :test,
+      includes: [ "#{@tmpdir}/*.log" ],
+      excludes: [ "#{@tmpdir}/exclude*" ]
+    }]
+
+    @discover = Franz::Discover.new({
+      discover_interval: @discover_interval,
+      discoveries: @discoveries,
+      deletions: @deletions,
+      logger: @logger,
+      configs: configs
+    }.deep_merge!(opts))
+
+    @watch = Franz::Watch.new({
+      watch_interval: @watch_interval,
+      watch_events: @watch_events,
+      discoveries: @discoveries,
+      deletions: @deletions,
+      logger: @logger
+    }.deep_merge!(opts))
+
+    @tail = Franz::Tail.new({
+      eviction_interval: @eviction_interval,
+      watch_events: @watch_events,
+      tail_events: @tail_events,
+      logger: @logger
+    }.deep_merge!(opts))
+
+    @agg = Franz::Agg.new({
+      configs: configs,
+      flush_interval: @flush_interval,
+      tail_events: @tail_events,
+      agg_events: @agg_events,
+      logger: @logger
+    }.deep_merge!(opts))
+  end
+
+  def stop_agg
+    @agg.stop
+  end
+end