fluent-plugin-tail-ex-rotate 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7913c120169c4dc922939f9c9b1c1c9d83421a06
-  data.tar.gz: d51a1a1ff4866a3ea6679da54de21d09b163b9f1
+  metadata.gz: 41366d67f88d73008c13f8ccaba644c8a0275015
+  data.tar.gz: fe145dc47313f4198d4d454570f2a94c99f49484
 SHA512:
-  metadata.gz: 3ee2cf5fc8db5e2c40f90b63d52374ecdd5f0e9c2addafe93f85b55a48b22b6736f2a36c504a6341586d4ea6c092d110de23257da18e7be7a0ebee230c62ab88
-  data.tar.gz: 290412a2f09490984e18a97cb1c6d8e383c888ae5baa3cb04ef7a5052c0fa0d966874ea4d31b17da648c58c8f578b0dbdcf787cfd17de704a6d14597a8fbea73
+  metadata.gz: e9b342459226d0778124e4c027f982b4ee7b915f8d986f76f94c2d9eb292e275b8b667fd7bcce8d421a62c87c5b9250d5c1d3cdb56becc0ceec20e81118ac489
+  data.tar.gz: 1e5855ef0238b44210cb1c20ee24fe0ead7653bde7052a0f06c770e462e432ed110b2562d9ce320471013586b7480c7b2df78d1f99a0d3f8dbb5e28b7e467e45

data/.travis.yml

@@ -0,0 +1,3 @@
+language: ruby
+rvm:
+  - 1.9.3

data/README.md

@@ -1,7 +1,7 @@
 # fluent-plugin-tail-ex-rotate
-===================================
+[![Build Status](https://travis-ci.org/dwango/fluent-plugin-tail-ex-rotate.png?branch=master)](https://travis-ci.org/dwango/fluent-plugin-tail-ex-rotate)
 
-fluent-plugin-tail-ex-rotate is a fluentd plugin to extend file lotation time.
+fluent-plugin-tail-ex-rotate is a fluentd plugin to delay file lotation time.
 
 ## Install
 
@@ -12,25 +12,28 @@ gem install fluent-plugin-tail-ex-rotate
 ## Test
 
 ```sh
+bundle install
 rake test
 ```
 
 ## Setting
 
 The setting of fluent-plugin-tail-ex-rotate is roughly the same with in_tail built-in plugin.
-A defference between fluent-plugin-tail-ex-rotate and in_tail built-in plugin setting is "expand_rotate_time" parameter
+Difference between fluent-plugin-tail-ex-rotate and in_tail built-in plugin setting are :expand_rotate_time attribute.
 
-"expand_rotate_time" parameter should be set number of second.
+:expand_rotate_time attribute must be set number of seconds.
 
 For example,
 ```xml
 <source>
   type tail_ex_rotate
-  path /var/log/apache2/access_log
-  tag internal.eapi_nicolive_process_time
+  path /var/log/httpd/access_log.%Y%m%d
+  tag access_process_time
   pos_file /var/log/td-agent/httpd-access.log.pos
   time_format %d/%b/%Y:%H:%M:%S %z
-  expand_rotate_time 18000
+  expand_rotate_time 18000s
   types process_time:integer
 </source>
 ```
+
+Daily file lotation starts with 05:00 beacause of this setting.

data/fluent-plugin-tail-ex-rotate.gemspec

@@ -2,7 +2,7 @@ require File.expand_path('../lib/fluent/version', __FILE__)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-tail-ex-rotate"
-  gem.version       = "0.0.3"
+  gem.version       = "0.0.4"
 
   gem.authors       = ["Yuta Mizushima"]
   gem.email         = ["yuta_mizushima@dwango.co.jp"]
@@ -18,8 +18,10 @@ Gem::Specification.new do |gem|
 
   gem.required_ruby_version = '>= 1.9.3'
 
-  gem.add_runtime_dependency("fluentd")
+  gem.add_runtime_dependency("fluentd", [">= 0.12.16"])
 
   gem.add_development_dependency("bundler", "~> 1.3")
+  gem.add_development_dependency("test-unit-rr")
+  gem.add_development_dependency("flexmock")
   gem.add_development_dependency("rake", [">= 0.9.2"])
 end

data/lib/fluent/plugin/in_tail_ex_rotate.rb

@@ -8,16 +8,25 @@ module Fluent
     def expand_paths
       date = Time.now - @expand_rotate_time
       paths = []
+
+      excluded = @exclude_path.map { |path| path = date.strftime(path); path.include?('*') ? Dir.glob(path) : path }.flatten.uniq
       @paths.each { |path|
         path = date.strftime(path)
         if path.include?('*')
-          paths += Dir.glob(path)
+          paths += Dir.glob(path).select { |p|
+            if File.readable?(p)
+              true
+            else
+              log.warn "#{p} unreadable. It is excluded and would be examined next time."
+              false
+            end
+          }
         else
           # When file is not created yet, Dir.glob returns an empty array. So just add when path is static.
           paths << path
         end
       }
-      paths
+      paths - excluded
     end
   end
 end

data/test/helper.rb

@@ -1,4 +1,4 @@
-# simplecov must be loaded before any of target code
+# simplecov must be loaded before any of target code is loaded
 if ENV['SIMPLE_COV']
   require 'simplecov'
   if defined?(SimpleCov::SourceFile)

data/test/plugin/test_in_tail_ex_rotate.rb

@@ -2,6 +2,7 @@ require_relative '../helper'
 require 'fluent/test'
 require 'net/http'
 require 'flexmock'
+require 'flexmock/test_unit'
 require 'fluent/plugin/in_tail_ex_rotate'
 
 class TailExRotateInputTest < Test::Unit::TestCase
@@ -31,6 +32,43 @@ class TailExRotateInputTest < Test::Unit::TestCase
     format /(?<message>.*)/
   ]
 
+  # * path test
+  # TODO: Clean up tests
+  EX_RORATE_WAIT = 0
+
+  EX_CONFIG = %[
+    tag tail
+    path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
+    format none
+    pos_file #{TMP_DIR}/tail.pos
+    read_from_head true
+    refresh_interval 30
+    rotate_wait #{EX_RORATE_WAIT}s
+  ]
+  EX_PATHS = [
+    'test/plugin/data/2010/01/20100102-030405.log',
+    'test/plugin/data/log/foo/bar.log',
+    'test/plugin/data/log/test.log'
+  ]
+
+
+  EX_CONFIG_FOR_ROTATE = %[
+    tag tail
+    path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
+    format none
+    pos_file #{TMP_DIR}/tail.pos
+    read_from_head true
+    refresh_interval 30
+    rotate_wait #{EX_RORATE_WAIT}s
+    expand_rotate_time 18000s
+  ]
+
+  EX_PATHS_FOR_ROATE = [
+    'test/plugin/data/2014/12/20141224-000000.log',
+    'test/plugin/data/log/foo/bar.log',
+    'test/plugin/data/log/test.log'
+  ]
+
   def create_driver(conf = SINGLE_LINE_CONFIG, use_common_conf = true)
     config = use_common_conf ? COMMON_CONFIG + conf : conf
     Fluent::Test::InputTestDriver.new(Fluent::TailExRotateInput).configure(config)
@@ -42,7 +80,7 @@ class TailExRotateInputTest < Test::Unit::TestCase
     assert_equal "t1", d.instance.tag
     assert_equal 2, d.instance.rotate_wait
     assert_equal "#{TMP_DIR}/tail.pos", d.instance.pos_file
-    assert_equal 18000, d.instance.expand_rotate_time
+    assert_equal 1000, d.instance.read_lines_limit
   end
 
   # TODO: Should using more better approach instead of sleep wait
@@ -69,6 +107,30 @@ class TailExRotateInputTest < Test::Unit::TestCase
     assert_equal(true, emits.length > 0)
     assert_equal({"message" => "test3"}, emits[0][2])
     assert_equal({"message" => "test4"}, emits[1][2])
+    assert_equal(1, d.emit_streams.size)
+  end
+
+  data('1' => [1, 2], '10' => [10, 1])
+  def test_emit_with_read_lines_limit(data)
+    limit, num_emits = data
+    d = create_driver(CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG + "read_lines_limit #{limit}")
+    msg = 'test' * 500 # in_tail reads 2048 bytes at once.
+
+    d.run do
+      sleep 1
+
+      File.open("#{TMP_DIR}/tail.txt", "a") {|f|
+        f.puts msg
+        f.puts msg
+      }
+      sleep 1
+    end
+
+    emits = d.emits
+    assert_equal(true, emits.length > 0)
+    assert_equal({"message" => msg}, emits[0][2])
+    assert_equal({"message" => msg}, emits[1][2])
+    assert_equal(num_emits, d.emit_streams.size)
   end
 
   def test_emit_with_read_from_head
@@ -367,56 +429,17 @@ class TailExRotateInputTest < Test::Unit::TestCase
     assert_equal({"var1" => "foo 2", "var2" => "bar 2", "var3" => "baz 2"}, emits[1][2])
   end
 
-  # * path test
-  # TODO: Clean up tests
-  EX_RORATE_WAIT = 0
-
-  EX_CONFIG = %[
-    tag tail
-    path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
-    format none
-    pos_file #{TMP_DIR}/tail.pos
-    read_from_head true
-    refresh_interval 30
-    rotate_wait #{EX_RORATE_WAIT}s
-  ]
-  EX_PATHS = [
-    'test/plugin/data/2010/01/20100102-030405.log',
-    'test/plugin/data/log/foo/bar.log',
-    'test/plugin/data/log/test.log'
-  ]
-
-  EX_CONFIG_FOR_ROTATE = %[
-    tag tail
-    path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
-    format none
-    pos_file #{TMP_DIR}/tail.pos
-    read_from_head true
-    refresh_interval 30
-    rotate_wait #{EX_RORATE_WAIT}s
-    expand_rotate_time 18000s
-  ]
-
-  EX_PATHS_FOR_ROATE = [
-    'test/plugin/data/2014/12/20141224-000000.log',
-    'test/plugin/data/log/foo/bar.log',
-    'test/plugin/data/log/test.log'
-  ]
-
   def test_expand_paths
     plugin = create_driver(EX_CONFIG, false).instance
     flexstub(Time) do |timeclass|
       timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 2, 3, 4, 5))
       assert_equal EX_PATHS, plugin.expand_paths.sort
     end
-  end
 
-  def test_expand_paths_for_ex_rotate
-    plugin = create_driver(EX_CONFIG_FOR_ROTATE, false).instance
-    flexstub(Time) do |timeclass|
-      timeclass.should_receive(:now).with_no_args.and_return(Time.new(2014, 12, 24, 5, 0, 0))
-      assert_equal EX_PATHS_FOR_ROATE, plugin.expand_paths.sort
-    end
+    # Test exclusion
+    exclude_config = EX_CONFIG + "  exclude_path [\"#{EX_PATHS.last}\"]"
+    plugin = create_driver(exclude_config, false).instance
+    assert_equal EX_PATHS - [EX_PATHS.last], plugin.expand_paths.sort
   end
 
   def test_refresh_watchers
@@ -432,7 +455,7 @@ class TailExRotateInputTest < Test::Unit::TestCase
 
       flexstub(Fluent::TailExRotateInput::TailWatcher) do |watcherclass|
         EX_PATHS.each do |path|
-          watcherclass.should_receive(:new).with(path, EX_RORATE_WAIT, Fluent::TailExRotateInput::FilePositionEntry, any, true, any, any).once.and_return do
+          watcherclass.should_receive(:new).with(path, EX_RORATE_WAIT, Fluent::TailExRotateInput::FilePositionEntry, any, true, 1000, any, any).once.and_return do
             flexmock('TailWatcher') { |watcher|
               watcher.should_receive(:attach).once
               watcher.should_receive(:unwatched=).zero_or_more_times
@@ -448,7 +471,7 @@ class TailExRotateInputTest < Test::Unit::TestCase
       end
 
       flexstub(Fluent::TailExRotateInput::TailWatcher) do |watcherclass|
-        watcherclass.should_receive(:new).with('test/plugin/data/2010/01/20100102-030406.log', EX_RORATE_WAIT, Fluent::TailExRotateInput::FilePositionEntry, any, true, any, any).once.and_return do
+        watcherclass.should_receive(:new).with('test/plugin/data/2010/01/20100102-030406.log', EX_RORATE_WAIT, Fluent::TailExRotateInput::FilePositionEntry, any, true, 1000, any, any).once.and_return do
           flexmock('TailWatcher') do |watcher|
             watcher.should_receive(:attach).once
             watcher.should_receive(:unwatched=).zero_or_more_times
@@ -557,4 +580,12 @@ class TailExRotateInputTest < Test::Unit::TestCase
       assert_equal({"message" => "test4"}, emits[1][2])
     end
   end
+
+  def test_expand_paths_for_ex_rotate
+    plugin = create_driver(EX_CONFIG_FOR_ROTATE, false).instance
+    flexstub(Time) do |timeclass|
+      timeclass.should_receive(:now).with_no_args.and_return(Time.new(2014, 12, 24, 5, 0, 0))
+      assert_equal EX_PATHS_FOR_ROATE, plugin.expand_paths.sort
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-tail-ex-rotate
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Yuta Mizushima
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-23 00:00:00.000000000 Z
+date: 2015-12-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.12.16
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.12.16
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -38,6 +38,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: test-unit-rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: flexmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -59,6 +87,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".travis.yml"
 - COPYING
 - Gemfile
 - NOTICE
@@ -94,7 +123,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
 summary: customize log rotate timing
@@ -107,4 +136,3 @@ test_files:
 - test/plugin/data/log/foo/bar.log
 - test/plugin/data/log/test.log
 - test/plugin/test_in_tail_ex_rotate.rb
-has_rdoc: false