franz 1.2.7

data/test/test_franz_agg.rb

@@ -0,0 +1,97 @@
+require 'thread'
+require 'tmpdir'
+require 'tempfile'
+require 'fileutils'
+require 'pathname'
+require 'minitest/autorun'
+
+require 'deep_merge'
+
+require_relative '../lib/franz'
+
+Thread.abort_on_exception = true
+
+class TestFranzAgg < MiniTest::Test
+  def setup
+    @tmpdir       = Dir.mktmpdir
+    @discoveries  = Queue.new
+    @deletions    = Queue.new
+    @watch_events = Queue.new
+    @tail_events  = Queue.new
+    @agg_events   = Queue.new
+    @logger       = Logger.new STDERR
+    @logger.level = Logger::WARN
+  end
+
+  def teardown
+    FileUtils.rm_rf @tmpdir
+  end
+
+  def test_handles_multiline
+    sample = "multiline this\nand this should be included\n"
+    tmp    = tempfile %w[ test1 .log ]
+    tmp.write sample
+    tmp.flush
+    tmp.close
+    start_agg
+    sleep 3
+    seqs = stop_agg
+    path = realpath tmp.path
+    assert seqs.include?(path)
+    assert_equal sample.strip, @agg_events.shift[:message]
+    assert seqs[path] == 1 # should be one line
+  end
+
+private
+  def tempfile prefix=nil
+    Tempfile.new prefix, @tmpdir
+  end
+
+  def realpath path
+    Pathname.new(path).realpath.to_s
+  end
+
+  def start_agg opts={}
+    configs = [{
+      type: :test,
+      multiline: /^multiline/,
+      includes: [ "#{@tmpdir}/*.log", "#{realpath @tmpdir}/*.log" ],
+      excludes: [ "#{@tmpdir}/exclude*" ]
+    }]
+
+    @discover = Franz::Discover.new({
+      discover_interval: 1,
+      discoveries: @discoveries,
+      deletions: @deletions,
+      logger: @logger,
+      configs: configs
+    }.deep_merge!(opts))
+
+    @watch = Franz::Watch.new({
+      watch_interval: 1,
+      watch_events: @watch_events,
+      discoveries: @discoveries,
+      deletions: @deletions,
+      logger: @logger
+    }.deep_merge!(opts))
+
+    @tail = Franz::Tail.new({
+      eviction_interval: 1,
+      watch_events: @watch_events,
+      tail_events: @tail_events,
+      logger: @logger
+    }.deep_merge!(opts))
+
+    @agg = Franz::Agg.new({
+      configs: configs,
+      flush_interval: 2,
+      tail_events: @tail_events,
+      agg_events: @agg_events,
+      logger: @logger
+    }.deep_merge!(opts))
+  end
+
+  def stop_agg
+    @agg.stop
+  end
+end

data/test/test_franz_discover.rb

@@ -0,0 +1,88 @@
+require 'thread'
+require 'tmpdir'
+require 'tempfile'
+require 'fileutils'
+require 'minitest/autorun'
+
+require 'deep_merge'
+
+require_relative '../lib/franz'
+
+Thread.abort_on_exception = true
+
+class TestFranzDiscover < MiniTest::Test
+  def setup
+    @tmpdir       = Dir.mktmpdir
+    @discoveries  = Queue.new
+    @deletions    = Queue.new
+    @logger       = Logger.new STDERR
+    @logger.level = Logger::WARN
+  end
+
+  def teardown
+    FileUtils.rm_rf @tmpdir
+  end
+
+  def test_discovers_existing_file
+    tmp = tempfile %w[ test .log ]
+    start_discovery known: []
+    sleep 0.001 # Time to discover
+    known = stop_discovery
+    assert known.include?(tmp.path)
+  end
+
+  def test_discovers_new_file
+    start_discovery known: []
+    tmp = tempfile %w[ test .log ]
+    sleep 1 # Time to discover
+    known = stop_discovery
+    assert known.include?(tmp.path)
+  end
+
+  def test_deletes_deleted_file
+    tmp = tempfile %w[ test .log ]
+    start_discovery known: []
+    # at this point, we know Discover has already picked up tmp
+    delete tmp.path
+    sleep 2
+    known = stop_discovery
+    assert !known.include?(tmp.path)
+  end
+
+  def test_deletes_unknown_file
+    tmp = tempfile %w[ test .log ]
+    delete tmp.path
+    # tmp never exists as far as Discover is aware
+    start_discovery known: []
+    sleep 0.001
+    known = stop_discovery
+    assert !known.include?(tmp.path)
+  end
+
+private
+  def tempfile prefix=nil
+    Tempfile.new prefix, @tmpdir
+  end
+
+  def start_discovery opts={}
+    @discover = Franz::Discover.new({
+      discover_interval: 1,
+      discoveries: @discoveries,
+      deletions: @deletions,
+      logger: @logger,
+      configs: [{
+        includes: [ "#{@tmpdir}/*.log" ],
+        excludes: [ "#{@tmpdir}/exclude*" ]
+      }]
+    }.deep_merge!(opts))
+  end
+
+  def stop_discovery
+    @discover.stop
+  end
+
+  def delete path
+    FileUtils.rm_rf path
+    @deletions.push path
+  end
+end

data/test/test_franz_tail.rb

@@ -0,0 +1,132 @@
+require 'thread'
+require 'tmpdir'
+require 'tempfile'
+require 'fileutils'
+require 'minitest/autorun'
+
+require 'deep_merge'
+
+require_relative '../lib/franz'
+
+Thread.abort_on_exception = true
+
+class TestFranzTail < MiniTest::Test
+  def setup
+    @tmpdir       = Dir.mktmpdir
+    @discoveries  = Queue.new
+    @deletions    = Queue.new
+    @watch_events = Queue.new
+    @tail_events  = Queue.new
+    @logger       = Logger.new STDERR
+    @logger.level = Logger::WARN
+  end
+
+  def teardown
+    FileUtils.rm_rf @tmpdir
+  end
+
+  def test_handles_existing_file
+    sample = "Hello, world!\n"
+    tmp = tempfile %w[ test1 .log ]
+    tmp.write sample
+    tmp.flush
+    tmp.close
+    start_tail
+    sleep 3
+    cursors = stop_tail
+    assert cursors.include?(tmp.path)
+    assert cursors[tmp.path] == sample.length
+  end
+
+  def test_handles_new_file
+    sample = "Hello, world!\n"
+    start_tail
+    sleep 0
+    tmp = tempfile %w[ test2 .log ]
+    tmp.write sample
+    tmp.flush
+    tmp.close
+    sleep 3
+    cursors = stop_tail
+    assert cursors.include?(tmp.path)
+    assert cursors[tmp.path] == sample.length
+  end
+
+  def test_handles_reading_after_eviction
+    sample = "Hello, world!\n"
+    eviction_interval = 2
+    start_tail eviction_interval: eviction_interval
+    sleep 0
+    tmp = tempfile %w[ test2 .log ]
+    tmp.write sample
+    tmp.flush
+    sleep eviction_interval / 2
+    tmp.write sample
+    tmp.flush
+    tmp.close
+    sleep eviction_interval * 2
+    cursors = stop_tail
+    assert cursors.include?(tmp.path)
+    assert cursors[tmp.path] == sample.length * 2
+  end
+
+  def test_handles_reading_after_deletion
+    sample = "Hello, world!\n"
+    eviction_interval = 2
+    start_tail eviction_interval: eviction_interval
+    sleep 0
+    tmp = tempfile %w[ test2 .log ]
+    path = tmp.path
+    tmp.write sample
+    tmp.flush
+    tmp.close
+    sleep eviction_interval / 2
+    FileUtils.rm_rf path
+    sleep eviction_interval
+    File.open(path, 'w') do |f|
+      f.write sample
+      f.flush
+    end
+    sleep eviction_interval * 2
+    cursors = stop_tail
+    assert cursors.include?(tmp.path)
+    assert cursors[tmp.path] == sample.length
+  end
+
+private
+  def tempfile prefix=nil
+    Tempfile.new prefix, @tmpdir
+  end
+
+  def start_tail opts={}
+    @discover = Franz::Discover.new({
+      discover_interval: 1,
+      discoveries: @discoveries,
+      deletions: @deletions,
+      logger: @logger,
+      configs: [{
+        includes: [ "#{@tmpdir}/*.log" ],
+        excludes: [ "#{@tmpdir}/exclude*" ]
+      }]
+    }.deep_merge!(opts))
+
+    @watch = Franz::Watch.new({
+      watch_interval: 1,
+      watch_events: @watch_events,
+      discoveries: @discoveries,
+      deletions: @deletions,
+      logger: @logger
+    }.deep_merge!(opts))
+
+    @tail = Franz::Tail.new({
+      eviction_interval: 1,
+      watch_events: @watch_events,
+      tail_events: @tail_events,
+      logger: @logger
+    }.deep_merge!(opts))
+  end
+
+  def stop_tail
+    @tail.stop
+  end
+end

data/test/test_franz_watch.rb

@@ -0,0 +1,144 @@
+require 'thread'
+require 'tmpdir'
+require 'tempfile'
+require 'fileutils'
+require 'minitest/autorun'
+
+require 'deep_merge'
+
+require_relative '../lib/franz'
+
+Thread.abort_on_exception = true
+
+class TestFranzWatch < MiniTest::Test
+  def setup
+    @tmpdir       = Dir.mktmpdir
+    @discoveries  = Queue.new
+    @deletions    = Queue.new
+    @queue        = Queue.new
+    @logger       = Logger.new STDERR
+    @logger.level = Logger::WARN
+  end
+
+  def teardown
+    FileUtils.rm_rf @tmpdir
+  end
+
+  def test_handles_existing_file
+    tmp = tempfile %w[ test .log ]
+    start_watch
+    sleep 2
+    stats = stop_watch
+    assert stats.include?(tmp.path)
+    assert stats[tmp.path][:size] == 0
+  end
+
+  def test_handles_existing_file_with_content
+    content = "Hello, world!\n"
+    tmp = tempfile %w[ test .log ]
+    tmp.write content
+    tmp.flush
+    start_watch
+    sleep 2
+    stats = stop_watch
+    assert stats.include?(tmp.path)
+    assert stats[tmp.path][:size] == content.length
+  end
+
+  def test_handles_new_file
+    start_watch
+    tmp = tempfile %w[ test .log ]
+    sleep 3
+    stats = stop_watch
+    assert stats.include?(tmp.path)
+    assert stats[tmp.path][:size] == 0
+  end
+
+  def test_handles_new_file_with_content
+    start_watch
+    content = "Hello, world!\n"
+    tmp = tempfile %w[ test .log ]
+    tmp.write content
+    tmp.flush
+    sleep 3
+    stats = stop_watch
+    assert stats.include?(tmp.path)
+    assert stats[tmp.path][:size] == content.length
+  end
+
+  def test_handles_file_truncated
+    long_content = "Hello, world!\n"
+    short_content = "Bye!\n"
+
+    tmp = tempfile %w[ test .log ]
+    tmp.write long_content
+    tmp.flush
+
+    start_watch
+    sleep 2
+    tmp.rewind
+    tmp.truncate 0
+    tmp.write short_content
+    tmp.flush
+    sleep 3
+
+    stats = stop_watch
+    assert stats.include?(tmp.path)
+    assert stats[tmp.path][:size] == short_content.length
+  end
+
+  def test_handles_file_replaced
+    content1 = "Hello, world!\n"
+    content2 = "Bye!\n"
+
+    tmp1 = tempfile %w[ test .log ]
+    tmp1.write content1
+    tmp1.flush
+    tmp1.close
+
+    tmp2 = tempfile %w[ exclude .log ]
+    tmp2.write content2
+    tmp2.flush
+    tmp2.close
+
+    start_watch
+    sleep 2
+    FileUtils.ln_sf tmp2.path, tmp1.path
+    sleep 3
+
+    stats = stop_watch
+    assert stats.include?(tmp1.path)
+    assert stats[tmp1.path][:size] == content2.length
+  end
+
+private
+  def tempfile prefix=nil
+    Tempfile.new(prefix, @tmpdir)
+  end
+
+  def start_watch opts={}
+    @discover = Franz::Discover.new({
+      discover_interval: 1,
+      discoveries: @discoveries,
+      deletions: @deletions,
+      logger: @logger,
+      configs: [{
+        includes: [ "#{@tmpdir}/*.log" ],
+        excludes: [ "#{@tmpdir}/exclude*" ]
+      }]
+    })
+
+    @watch = Franz::Watch.new({
+      watch_interval: 1,
+      watch_events: @queue,
+      discoveries: @discoveries,
+      deletions: @deletions,
+      logger: @logger,
+      stats: Hash.new
+    }.deep_merge!(opts))
+  end
+
+  def stop_watch
+    @watch.stop
+  end
+end