fluent-plugin-tail-ex-rotate 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a77ea0c7caa116f06886673d73d83d9b822b1b5b
+  data.tar.gz: 12e56f4a6dd248b086b35b73db277f1400b6a8a7
+SHA512:
+  metadata.gz: 6209fbe36cde82650e465dc817a5fafa81614163f2d35ee7912e5329fbad818a1eac96b29a8cc769ccc6f68f5fdb66e78b438cdedbff707ceea0b71a10006f16
+  data.tar.gz: dda0d444b1744ff0fc49185526e2d39ed880bd37b1250d7094a36f22afdb39981eee4e9b5cea9c24bf360fd795eb7f62e955db391018010c067a1468c04d207f

data/COPYING

@@ -0,0 +1,14 @@
+Copyright (C) 2015 DWANGO Co., Ltd.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org/'
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,40 @@
+PATH
+  remote: .
+  specs:
+    fluent-plugin-tail-ex-rotate (0.0.1)
+      fluentd
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    cool.io (1.2.4)
+    fluentd (0.12.2)
+      cool.io (>= 1.2.2, < 2.0.0)
+      http_parser.rb (>= 0.5.1, < 0.7.0)
+      json (>= 1.4.3)
+      msgpack (>= 0.5.4, < 0.6.0)
+      sigdump (~> 0.2.2)
+      string-scrub (>= 0.0.3)
+      tzinfo (>= 1.0.0)
+      tzinfo-data (>= 1.0.0)
+      yajl-ruby (~> 1.0)
+    http_parser.rb (0.6.0)
+    json (1.8.1)
+    msgpack (0.5.9)
+    rake (10.4.2)
+    sigdump (0.2.2)
+    string-scrub (0.0.5)
+    thread_safe (0.3.4)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+    tzinfo-data (1.2014.10)
+      tzinfo (>= 1.0.0)
+    yajl-ruby (1.2.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.3)
+  fluent-plugin-tail-ex-rotate!
+  rake (>= 0.9.2)

data/NOTICE

@@ -0,0 +1,5 @@
+This product contains a part of fluentd source code:
+  * LICENSE:
+    * https://github.com/fluent/fluentd/blob/master/COPYING
+  * HOMEPAGE:
+    * https://github.com/fluent/fluentd

data/README.md

@@ -0,0 +1,30 @@
+# fluent-plugin-tail-ex-rotate
+===================================
+
+fluent-plugin-tail-ex-rotate is a fluentd plugin to extend file lotation time.
+
+## Install
+
+```sh
+gem install fluent-plugin-tail-ex-rotate
+```
+
+## Setting
+
+The setting of fluent-plugin-tail-ex-rotate is roughly the same with in_tail built-in plugin.
+A defference between fluent-plugin-tail-ex-rotate and in_tail built-in plugin setting is "expand_rotate_time" parameter
+
+"expand_rotate_time" parameter should be set number of second.
+
+For example,
+```xml
+<source>
+  type tail_ex_rotate
+  path /var/log/apache2/access_log
+  tag internal.eapi_nicolive_process_time
+  pos_file /var/log/td-agent/httpd-access.log.pos
+  time_format %d/%b/%Y:%H:%M:%S %z
+  expand_rotate_time 18000
+  types process_time:integer
+</source>
+```

data/Rakefile

@@ -0,0 +1,33 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require 'fileutils'
+require 'rake/testtask'
+require 'rake/clean'
+
+task :test => [:base_test]
+
+desc 'Run test_unit based test'
+Rake::TestTask.new(:base_test) do |t|
+  # To run test for only one file (or file path pattern)
+  #  $ bundle exec rake base_test TEST=test/test_specified_path.rb
+  #  $ bundle exec rake base_test TEST=test/test_*.rb
+  t.libs << "test"
+  t.test_files = Dir["test/**/test_*.rb"].sort
+  t.verbose = true
+  #t.warning = true
+end
+
+task :parallel_test do
+  FileUtils.rm_rf('./test/tmp')
+  sh("parallel_test ./test/*.rb ./test/plugin/*.rb")
+  FileUtils.rm_rf('./test/tmp')
+end
+
+desc 'Run test with simplecov'
+task :coverage do |t|
+  ENV['SIMPLE_COV'] = '1'
+  Rake::Task["test"].invoke
+end
+
+task :default => [:test, :build]

data/fluent-plugin-tail-ex-rotate.gemspec

@@ -0,0 +1,25 @@
+require File.expand_path('../lib/fluent/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name          = "fluent-plugin-tail-ex-rotate"
+  gem.version       = "0.0.1"
+
+  gem.authors       = ["Yuta Mizushima"]
+  gem.email         = ["yuta_mizushima@dwango.co.jp"]
+  gem.description   = %q{Extension of in_tail plugin to customize log rotate timing.}
+  gem.summary       = %q{customize log rotate timing}
+  gem.homepage      = "http://github.o-in.dwango.co.jp/Yuta-Mizushima/namasasy_setup/tree/master/node/fluent-plugin-tail-ex-rotate"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+  gem.has_rdoc = false
+
+  gem.required_ruby_version = '>= 1.9.3'
+
+  gem.add_runtime_dependency("fluentd")
+
+  gem.add_development_dependency("bundler", "~> 1.3")
+  gem.add_development_dependency("rake", [">= 0.9.2"])
+end

data/lib/fluent/plugin/in_tail_ex_rotate.rb

@@ -0,0 +1,23 @@
+require 'fluent/plugin/in_tail'
+
+module Fluent
+  class TailExRotateInput < Fluent::NewTailInput
+    Fluent::Plugin.register_input('tail_ex_rotate', self)
+    config_param :expand_rotate_time, :time, :default => 0
+
+    def expand_paths
+      date = Time.now - @expand_rotate_time
+      paths = []
+      @paths.each { |path|
+        path = date.strftime(path)
+        if path.include?('*')
+          paths += Dir.glob(path)
+        else
+          # When file is not created yet, Dir.glob returns an empty array. So just add when path is static.
+          paths << path
+        end
+      }
+      paths
+    end
+  end
+end

data/lib/fluent/version.rb

@@ -0,0 +1,21 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+
+  VERSION = '0.12.2'
+
+end

data/test/helper.rb

@@ -0,0 +1,56 @@
+# simplecov must be loaded before any of target code
+if ENV['SIMPLE_COV']
+  require 'simplecov'
+  if defined?(SimpleCov::SourceFile)
+    mod = SimpleCov::SourceFile
+    def mod.new(*args, &block)
+      m = allocate
+      m.instance_eval do
+        begin
+          initialize(*args, &block)
+        rescue Encoding::UndefinedConversionError
+          @src = "".force_encoding('UTF-8')
+        end
+      end
+      m
+    end
+  end
+  unless SimpleCov.running
+    SimpleCov.start do
+      add_filter '/test/'
+      add_filter '/gems/'
+    end
+  end
+end
+
+require 'rr'
+require 'test/unit'
+require 'test/unit/rr'
+require 'fileutils'
+require 'fluent/log'
+require 'fluent/test'
+
+unless defined?(Test::Unit::AssertionFailedError)
+  class Test::Unit::AssertionFailedError < StandardError
+  end
+end
+
+def unused_port
+  s = TCPServer.open(0)
+  port = s.addr[1]
+  s.close
+  port
+end
+
+def ipv6_enabled?
+  require 'socket'
+
+  begin
+    TCPServer.open("::1", 0)
+    true
+  rescue
+    false
+  end
+end
+
+$log = Fluent::Log.new(Fluent::Test::DummyLogDevice.new, Fluent::Log::LEVEL_WARN)

data/test/plugin/test_in_tail_ex_rotate.rb

@@ -0,0 +1,560 @@
+require_relative '../helper'
+require 'fluent/test'
+require 'net/http'
+require 'flexmock'
+require 'fluent/plugin/in_tail_ex_rotate'
+
+class TailExRotateInputTest < Test::Unit::TestCase
+  include FlexMock::TestCase
+
+  def setup
+    Fluent::Test.setup
+    FileUtils.rm_rf(TMP_DIR)
+    FileUtils.mkdir_p(TMP_DIR)
+  end
+
+  TMP_DIR = File.dirname(__FILE__) + "/../tmp/tail#{ENV['TEST_ENV_NUMBER']}"
+
+  CONFIG = %[
+    path #{TMP_DIR}/tail.txt
+    tag t1
+    rotate_wait 2s
+    expand_rotate_time 18000s
+  ]
+  COMMON_CONFIG = CONFIG + %[
+    pos_file #{TMP_DIR}/tail.pos
+  ]
+  CONFIG_READ_FROM_HEAD = %[
+    read_from_head true
+  ]
+  SINGLE_LINE_CONFIG = %[
+    format /(?<message>.*)/
+  ]
+
+  def create_driver(conf = SINGLE_LINE_CONFIG, use_common_conf = true)
+    config = use_common_conf ? COMMON_CONFIG + conf : conf
+    Fluent::Test::InputTestDriver.new(Fluent::TailExRotateInput).configure(config)
+  end
+
+  def test_configure
+    d = create_driver
+    assert_equal ["#{TMP_DIR}/tail.txt"], d.instance.paths
+    assert_equal "t1", d.instance.tag
+    assert_equal 2, d.instance.rotate_wait
+    assert_equal "#{TMP_DIR}/tail.pos", d.instance.pos_file
+    assert_equal 18000, d.instance.expand_rotate_time
+  end
+
+  # TODO: Should using more better approach instead of sleep wait
+
+  def test_emit
+    File.open("#{TMP_DIR}/tail.txt", "w") {|f|
+      f.puts "test1"
+      f.puts "test2"
+    }
+
+    d = create_driver
+
+    d.run do
+      sleep 1
+
+      File.open("#{TMP_DIR}/tail.txt", "a") {|f|
+        f.puts "test3"
+        f.puts "test4"
+      }
+      sleep 1
+    end
+
+    emits = d.emits
+    assert_equal(true, emits.length > 0)
+    assert_equal({"message" => "test3"}, emits[0][2])
+    assert_equal({"message" => "test4"}, emits[1][2])
+  end
+
+  def test_emit_with_read_from_head
+    File.open("#{TMP_DIR}/tail.txt", "w") {|f|
+      f.puts "test1"
+      f.puts "test2"
+    }
+
+    d = create_driver(CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG)
+
+    d.run do
+      sleep 1
+
+      File.open("#{TMP_DIR}/tail.txt", "a") {|f|
+        f.puts "test3"
+        f.puts "test4"
+      }
+      sleep 1
+    end
+
+    emits = d.emits
+    assert(emits.length > 0)
+    assert_equal({"message" => "test1"}, emits[0][2])
+    assert_equal({"message" => "test2"}, emits[1][2])
+    assert_equal({"message" => "test3"}, emits[2][2])
+    assert_equal({"message" => "test4"}, emits[3][2])
+  end
+
+  def test_rotate_file
+    emits = sub_test_rotate_file(SINGLE_LINE_CONFIG)
+    assert_equal(4, emits.length)
+    assert_equal({"message" => "test3"}, emits[0][2])
+    assert_equal({"message" => "test4"}, emits[1][2])
+    assert_equal({"message" => "test5"}, emits[2][2])
+    assert_equal({"message" => "test6"}, emits[3][2])
+  end
+
+  def test_rotate_file_with_read_from_head
+    emits = sub_test_rotate_file(CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG)
+    assert_equal(6, emits.length)
+    assert_equal({"message" => "test1"}, emits[0][2])
+    assert_equal({"message" => "test2"}, emits[1][2])
+    assert_equal({"message" => "test3"}, emits[2][2])
+    assert_equal({"message" => "test4"}, emits[3][2])
+    assert_equal({"message" => "test5"}, emits[4][2])
+    assert_equal({"message" => "test6"}, emits[5][2])
+  end
+
+  def test_rotate_file_with_write_old
+    emits = sub_test_rotate_file(SINGLE_LINE_CONFIG) { |rotated_file|
+      File.open("#{TMP_DIR}/tail.txt", "w") { |f| }
+      rotated_file.puts "test7"
+      rotated_file.puts "test8"
+      rotated_file.flush
+
+      sleep 1
+      File.open("#{TMP_DIR}/tail.txt", "a") { |f|
+        f.puts "test5"
+        f.puts "test6"
+      }
+    }
+    assert_equal(6, emits.length)
+    assert_equal({"message" => "test3"}, emits[0][2])
+    assert_equal({"message" => "test4"}, emits[1][2])
+    assert_equal({"message" => "test7"}, emits[2][2])
+    assert_equal({"message" => "test8"}, emits[3][2])
+    assert_equal({"message" => "test5"}, emits[4][2])
+    assert_equal({"message" => "test6"}, emits[5][2])
+  end
+
+  def test_rotate_file_with_write_old_and_no_new_file
+    emits = sub_test_rotate_file(SINGLE_LINE_CONFIG) { |rotated_file|
+      rotated_file.puts "test7"
+      rotated_file.puts "test8"
+      rotated_file.flush
+    }
+    assert_equal(4, emits.length)
+    assert_equal({"message" => "test3"}, emits[0][2])
+    assert_equal({"message" => "test4"}, emits[1][2])
+    assert_equal({"message" => "test7"}, emits[2][2])
+    assert_equal({"message" => "test8"}, emits[3][2])
+  end
+
+  def sub_test_rotate_file(config = nil)
+    file = File.open("#{TMP_DIR}/tail.txt", "w")
+    file.puts "test1"
+    file.puts "test2"
+    file.flush
+
+    d = create_driver(config)
+    d.run do
+      sleep 1
+
+      file.puts "test3"
+      file.puts "test4"
+      file.flush
+      sleep 1
+
+      FileUtils.mv("#{TMP_DIR}/tail.txt", "#{TMP_DIR}/tail2.txt")
+      if block_given?
+        yield file
+        sleep 1
+      else
+        sleep 1
+        File.open("#{TMP_DIR}/tail.txt", "w") { |f| }
+        sleep 1
+
+        File.open("#{TMP_DIR}/tail.txt", "a") { |f|
+          f.puts "test5"
+          f.puts "test6"
+        }
+        sleep 1
+      end
+    end
+
+    d.run do
+      sleep 1
+    end
+
+    d.emits
+  ensure
+    file.close
+  end
+
+  def test_lf
+    File.open("#{TMP_DIR}/tail.txt", "w") {|f| }
+
+    d = create_driver
+
+    d.run do
+      File.open("#{TMP_DIR}/tail.txt", "a") {|f|
+        f.print "test3"
+      }
+      sleep 1
+
+      File.open("#{TMP_DIR}/tail.txt", "a") {|f|
+        f.puts "test4"
+      }
+      sleep 1
+    end
+
+    emits = d.emits
+    assert_equal(true, emits.length > 0)
+    assert_equal({"message" => "test3test4"}, emits[0][2])
+  end
+
+  def test_whitespace
+    File.open("#{TMP_DIR}/tail.txt", "w") {|f| }
+
+    d = create_driver
+
+    d.run do
+      sleep 1
+
+      File.open("#{TMP_DIR}/tail.txt", "a") {|f|
+        f.puts "    "		# 4 spaces
+        f.puts "    4 spaces"
+        f.puts "4 spaces    "
+        f.puts "	"	# tab
+        f.puts "	tab"
+        f.puts "tab	"
+      }
+      sleep 1
+    end
+
+    emits = d.emits
+    assert_equal(true, emits.length > 0)
+    assert_equal({"message" => "    "}, emits[0][2])
+    assert_equal({"message" => "    4 spaces"}, emits[1][2])
+    assert_equal({"message" => "4 spaces    "}, emits[2][2])
+    assert_equal({"message" => "	"}, emits[3][2])
+    assert_equal({"message" => "	tab"}, emits[4][2])
+    assert_equal({"message" => "tab	"}, emits[5][2])
+  end
+
+  # multiline mode test
+
+  def test_multiline
+    File.open("#{TMP_DIR}/tail.txt", "w") { |f| }
+
+    d = create_driver %[
+      format multiline
+      format1 /^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/
+      format_firstline /^[s]/
+    ]
+    d.run do
+      File.open("#{TMP_DIR}/tail.txt", "a") { |f|
+        f.puts "f test1"
+        f.puts "s test2"
+        f.puts "f test3"
+        f.puts "f test4"
+        f.puts "s test5"
+        f.puts "s test6"
+        f.puts "f test7"
+        f.puts "s test8"
+      }
+      sleep 1
+    end
+
+    emits = d.emits
+    assert(emits.length > 0)
+    assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, emits[0][2])
+    assert_equal({"message1" => "test5"}, emits[1][2])
+    assert_equal({"message1" => "test6", "message2" => "test7"}, emits[2][2])
+    assert_equal({"message1" => "test8"}, emits[3][2])
+  end
+
+  def test_multiline_with_multiple_formats
+    File.open("#{TMP_DIR}/tail.txt", "w") { |f| }
+
+    d = create_driver %[
+      format multiline
+      format1 /^s (?<message1>[^\\n]+)\\n?/
+      format2 /(f (?<message2>[^\\n]+)\\n?)?/
+      format3 /(f (?<message3>.*))?/
+      format_firstline /^[s]/
+    ]
+    d.run do
+      File.open("#{TMP_DIR}/tail.txt", "a") { |f|
+        f.puts "f test1"
+        f.puts "s test2"
+        f.puts "f test3"
+        f.puts "f test4"
+        f.puts "s test5"
+        f.puts "s test6"
+        f.puts "f test7"
+        f.puts "s test8"
+      }
+      sleep 1
+    end
+
+    emits = d.emits
+    assert(emits.length > 0)
+    assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, emits[0][2])
+    assert_equal({"message1" => "test5"}, emits[1][2])
+    assert_equal({"message1" => "test6", "message2" => "test7"}, emits[2][2])
+    assert_equal({"message1" => "test8"}, emits[3][2])
+  end
+
+  def test_multilinelog_with_multiple_paths
+    files = ["#{TMP_DIR}/tail1.txt", "#{TMP_DIR}/tail2.txt"]
+    files.each { |file| File.open(file, "w") { |f| } }
+
+    d = create_driver(%[
+      path #{files[0]},#{files[1]}
+      tag t1
+      format multiline
+      format1 /^[s|f] (?<message>.*)/
+      format_firstline /^[s]/
+    ], false)
+    d.run do
+      files.each do |file|
+        File.open(file, 'a') { |f|
+          f.puts "f #{file} line should be ignored"
+          f.puts "s test1"
+          f.puts "f test2"
+          f.puts "f test3"
+          f.puts "s test4"
+        }
+      end
+      sleep 1
+    end
+
+    emits = d.emits
+    assert_equal({"message" => "test1\nf test2\nf test3"}, emits[0][2])
+    assert_equal({"message" => "test1\nf test2\nf test3"}, emits[1][2])
+    # "test4" events are here because these events are flushed at shutdown phase
+    assert_equal({"message" => "test4"}, emits[2][2])
+    assert_equal({"message" => "test4"}, emits[3][2])
+  end
+
+  def test_multiline_without_firstline
+    File.open("#{TMP_DIR}/tail.txt", "w") { |f| }
+
+    d = create_driver %[
+      format multiline
+      format1 /(?<var1>foo \\d)\\n/
+      format2 /(?<var2>bar \\d)\\n/
+      format3 /(?<var3>baz \\d)/
+    ]
+    d.run do
+      File.open("#{TMP_DIR}/tail.txt", "a") { |f|
+        f.puts "foo 1"
+        f.puts "bar 1"
+        f.puts "baz 1"
+        f.puts "foo 2"
+        f.puts "bar 2"
+        f.puts "baz 2"
+      }
+      sleep 1
+    end
+
+    emits = d.emits
+    assert_equal(2, emits.length)
+    assert_equal({"var1" => "foo 1", "var2" => "bar 1", "var3" => "baz 1"}, emits[0][2])
+    assert_equal({"var1" => "foo 2", "var2" => "bar 2", "var3" => "baz 2"}, emits[1][2])
+  end
+
+  # * path test
+  # TODO: Clean up tests
+  EX_RORATE_WAIT = 0
+
+  EX_CONFIG = %[
+    tag tail
+    path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
+    format none
+    pos_file #{TMP_DIR}/tail.pos
+    read_from_head true
+    refresh_interval 30
+    rotate_wait #{EX_RORATE_WAIT}s
+  ]
+  EX_PATHS = [
+    'test/plugin/data/2010/01/20100102-030405.log',
+    'test/plugin/data/log/foo/bar.log',
+    'test/plugin/data/log/test.log'
+  ]
+
+  EX_CONFIG_FOR_ROTATE = %[
+    tag tail
+    path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
+    format none
+    pos_file #{TMP_DIR}/tail.pos
+    read_from_head true
+    refresh_interval 30
+    rotate_wait #{EX_RORATE_WAIT}s
+    expand_rotate_time 18000s
+  ]
+
+  EX_PATHS_FOR_ROATE = [
+    'test/plugin/data/2014/12/20141224-000000.log',
+    'test/plugin/data/log/foo/bar.log',
+    'test/plugin/data/log/test.log'
+  ]
+
+  def test_expand_paths
+    plugin = create_driver(EX_CONFIG, false).instance
+    flexstub(Time) do |timeclass|
+      timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 2, 3, 4, 5))
+      assert_equal EX_PATHS, plugin.expand_paths.sort
+    end
+  end
+
+  def test_expand_paths_for_ex_rotate
+    plugin = create_driver(EX_CONFIG_FOR_ROTATE, false).instance
+    flexstub(Time) do |timeclass|
+      timeclass.should_receive(:now).with_no_args.and_return(Time.new(2014, 12, 24, 5, 0, 0))
+      assert_equal EX_PATHS_FOR_ROATE, plugin.expand_paths.sort
+    end
+  end
+
+  def test_refresh_watchers
+    plugin = create_driver(EX_CONFIG, false).instance
+    sio = StringIO.new
+    plugin.instance_eval do
+      @pf = Fluent::TailExRotateInput::PositionFile.parse(sio)
+      @loop = Coolio::Loop.new
+   end
+
+    flexstub(Time) do |timeclass|
+      timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 2, 3, 4, 5), Time.new(2010, 1, 2, 3, 4, 6), Time.new(2010, 1, 2, 3, 4, 7))
+
+      flexstub(Fluent::TailExRotateInput::TailWatcher) do |watcherclass|
+        EX_PATHS.each do |path|
+          watcherclass.should_receive(:new).with(path, EX_RORATE_WAIT, Fluent::TailExRotateInput::FilePositionEntry, any, true, any, any).once.and_return do
+            flexmock('TailWatcher') { |watcher|
+              watcher.should_receive(:attach).once
+              watcher.should_receive(:unwatched=).zero_or_more_times
+              watcher.should_receive(:line_buffer).zero_or_more_times
+            }
+          end
+        end
+        plugin.refresh_watchers
+      end
+
+      plugin.instance_eval do
+        @tails['test/plugin/data/2010/01/20100102-030405.log'].should_receive(:close).zero_or_more_times
+      end
+
+      flexstub(Fluent::TailExRotateInput::TailWatcher) do |watcherclass|
+        watcherclass.should_receive(:new).with('test/plugin/data/2010/01/20100102-030406.log', EX_RORATE_WAIT, Fluent::TailExRotateInput::FilePositionEntry, any, true, any, any).once.and_return do
+          flexmock('TailWatcher') do |watcher|
+            watcher.should_receive(:attach).once
+            watcher.should_receive(:unwatched=).zero_or_more_times
+            watcher.should_receive(:line_buffer).zero_or_more_times
+          end
+        end
+        plugin.refresh_watchers
+      end
+
+      flexstub(Fluent::TailExRotateInput::TailWatcher) do |watcherclass|
+        watcherclass.should_receive(:new).never
+        plugin.refresh_watchers
+      end
+    end
+  end
+
+  DummyWatcher = Struct.new("DummyWatcher", :tag)
+
+  def test_receive_lines
+    plugin = create_driver(EX_CONFIG, false).instance
+    flexstub(plugin.router) do |engineclass|
+      engineclass.should_receive(:emit_stream).with('tail', any).once
+      plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
+    end
+
+    config = %[
+      tag pre.*
+      path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
+      format none
+      read_from_head true
+    ]
+    plugin = create_driver(config, false).instance
+    flexstub(plugin.router) do |engineclass|
+      engineclass.should_receive(:emit_stream).with('pre.foo.bar.log', any).once
+      plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
+    end
+
+    config = %[
+      tag *.post
+      path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
+      format none
+      read_from_head true
+    ]
+    plugin = create_driver(config, false).instance
+    flexstub(plugin.router) do |engineclass|
+      engineclass.should_receive(:emit_stream).with('foo.bar.log.post', any).once
+      plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
+    end
+
+    config = %[
+      tag pre.*.post
+      path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
+      format none
+      read_from_head true
+    ]
+    plugin = create_driver(config, false).instance
+    flexstub(plugin.router) do |engineclass|
+      engineclass.should_receive(:emit_stream).with('pre.foo.bar.log.post', any).once
+      plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
+    end
+
+    config = %[
+      tag pre.*.post*ignore
+      path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
+      format none
+      read_from_head true
+    ]
+    plugin = create_driver(config, false).instance
+    flexstub(plugin.router) do |engineclass|
+      engineclass.should_receive(:emit_stream).with('pre.foo.bar.log.post', any).once
+      plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
+    end
+  end
+
+  # Ensure that no fatal exception is raised when a file is missing and that
+  # files that do exist are still tailed as expected.
+  def test_missing_file
+    File.open("#{TMP_DIR}/tail.txt", "w") {|f|
+      f.puts "test1"
+      f.puts "test2"
+    }
+
+    # Try two different configs - one with read_from_head and one without,
+    # since their interactions with the filesystem differ.
+    config1 = %[
+      tag t1
+      path #{TMP_DIR}/non_existent_file.txt,#{TMP_DIR}/tail.txt
+      format none
+      rotate_wait 2s
+      pos_file #{TMP_DIR}/tail.pos
+    ]
+    config2 = config1 + '  read_from_head true'
+    [config1, config2].each do |config|
+      d = create_driver(config, false)
+      d.run do
+        sleep 1
+        File.open("#{TMP_DIR}/tail.txt", "a") {|f|
+          f.puts "test3"
+          f.puts "test4"
+        }
+        sleep 1
+      end
+      emits = d.emits
+      assert_equal(2, emits.length)
+      assert_equal({"message" => "test3"}, emits[0][2])
+      assert_equal({"message" => "test4"}, emits[1][2])
+    end
+  end
+end

metadata

@@ -0,0 +1,112 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-tail-ex-rotate
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Yuta Mizushima
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-01-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+description: Extension of in_tail plugin to customize log rotate timing.
+email:
+- yuta_mizushima@dwango.co.jp
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- COPYING
+- Gemfile
+- Gemfile.lock
+- NOTICE
+- README.md
+- Rakefile
+- fluent-plugin-tail-ex-rotate.gemspec
+- lib/fluent/plugin/in_tail_ex_rotate.rb
+- lib/fluent/version.rb
+- pkg/fluent-plugin-tail-ex-rotate-0.0.1.gem
+- test/helper.rb
+- test/plugin/data/2010/01/20100102-030405.log
+- test/plugin/data/2010/01/20100102-030406.log
+- test/plugin/data/2014/12/20141224-000000.log
+- test/plugin/data/log/bar
+- test/plugin/data/log/foo/bar.log
+- test/plugin/data/log/test.log
+- test/plugin/test_in_tail_ex_rotate.rb
+homepage: http://github.o-in.dwango.co.jp/Yuta-Mizushima/namasasy_setup/tree/master/node/fluent-plugin-tail-ex-rotate
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 1.9.3
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: customize log rotate timing
+test_files:
+- test/helper.rb
+- test/plugin/data/2010/01/20100102-030405.log
+- test/plugin/data/2010/01/20100102-030406.log
+- test/plugin/data/2014/12/20141224-000000.log
+- test/plugin/data/log/bar
+- test/plugin/data/log/foo/bar.log
+- test/plugin/data/log/test.log
+- test/plugin/test_in_tail_ex_rotate.rb
+has_rdoc: false