fluent-plugin-tail-ex-rotate 0.0.3 → 0.0.4

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 7913c120169c4dc922939f9c9b1c1c9d83421a06
4
- data.tar.gz: d51a1a1ff4866a3ea6679da54de21d09b163b9f1
3
+ metadata.gz: 41366d67f88d73008c13f8ccaba644c8a0275015
4
+ data.tar.gz: fe145dc47313f4198d4d454570f2a94c99f49484
5
5
  SHA512:
6
- metadata.gz: 3ee2cf5fc8db5e2c40f90b63d52374ecdd5f0e9c2addafe93f85b55a48b22b6736f2a36c504a6341586d4ea6c092d110de23257da18e7be7a0ebee230c62ab88
7
- data.tar.gz: 290412a2f09490984e18a97cb1c6d8e383c888ae5baa3cb04ef7a5052c0fa0d966874ea4d31b17da648c58c8f578b0dbdcf787cfd17de704a6d14597a8fbea73
6
+ metadata.gz: e9b342459226d0778124e4c027f982b4ee7b915f8d986f76f94c2d9eb292e275b8b667fd7bcce8d421a62c87c5b9250d5c1d3cdb56becc0ceec20e81118ac489
7
+ data.tar.gz: 1e5855ef0238b44210cb1c20ee24fe0ead7653bde7052a0f06c770e462e432ed110b2562d9ce320471013586b7480c7b2df78d1f99a0d3f8dbb5e28b7e467e45
@@ -0,0 +1,3 @@
1
+ language: ruby
2
+ rvm:
3
+ - 1.9.3
data/README.md CHANGED
@@ -1,7 +1,7 @@
1
1
  # fluent-plugin-tail-ex-rotate
2
- ===================================
2
+ [![Build Status](https://travis-ci.org/dwango/fluent-plugin-tail-ex-rotate.png?branch=master)](https://travis-ci.org/dwango/fluent-plugin-tail-ex-rotate)
3
3
 
4
- fluent-plugin-tail-ex-rotate is a fluentd plugin to extend file lotation time.
4
+ fluent-plugin-tail-ex-rotate is a fluentd plugin to delay file lotation time.
5
5
 
6
6
  ## Install
7
7
 
@@ -12,25 +12,28 @@ gem install fluent-plugin-tail-ex-rotate
12
12
  ## Test
13
13
 
14
14
  ```sh
15
+ bundle install
15
16
  rake test
16
17
  ```
17
18
 
18
19
  ## Setting
19
20
 
20
21
  The setting of fluent-plugin-tail-ex-rotate is roughly the same with in_tail built-in plugin.
21
- A defference between fluent-plugin-tail-ex-rotate and in_tail built-in plugin setting is "expand_rotate_time" parameter
22
+ Difference between fluent-plugin-tail-ex-rotate and in_tail built-in plugin setting are :expand_rotate_time attribute.
22
23
 
23
- "expand_rotate_time" parameter should be set number of second.
24
+ :expand_rotate_time attribute must be set number of seconds.
24
25
 
25
26
  For example,
26
27
  ```xml
27
28
  <source>
28
29
  type tail_ex_rotate
29
- path /var/log/apache2/access_log
30
- tag internal.eapi_nicolive_process_time
30
+ path /var/log/httpd/access_log.%Y%m%d
31
+ tag access_process_time
31
32
  pos_file /var/log/td-agent/httpd-access.log.pos
32
33
  time_format %d/%b/%Y:%H:%M:%S %z
33
- expand_rotate_time 18000
34
+ expand_rotate_time 18000s
34
35
  types process_time:integer
35
36
  </source>
36
37
  ```
38
+
39
+ Daily file lotation starts with 05:00 beacause of this setting.
@@ -2,7 +2,7 @@ require File.expand_path('../lib/fluent/version', __FILE__)
2
2
 
3
3
  Gem::Specification.new do |gem|
4
4
  gem.name = "fluent-plugin-tail-ex-rotate"
5
- gem.version = "0.0.3"
5
+ gem.version = "0.0.4"
6
6
 
7
7
  gem.authors = ["Yuta Mizushima"]
8
8
  gem.email = ["yuta_mizushima@dwango.co.jp"]
@@ -18,8 +18,10 @@ Gem::Specification.new do |gem|
18
18
 
19
19
  gem.required_ruby_version = '>= 1.9.3'
20
20
 
21
- gem.add_runtime_dependency("fluentd")
21
+ gem.add_runtime_dependency("fluentd", [">= 0.12.16"])
22
22
 
23
23
  gem.add_development_dependency("bundler", "~> 1.3")
24
+ gem.add_development_dependency("test-unit-rr")
25
+ gem.add_development_dependency("flexmock")
24
26
  gem.add_development_dependency("rake", [">= 0.9.2"])
25
27
  end
@@ -8,16 +8,25 @@ module Fluent
8
8
  def expand_paths
9
9
  date = Time.now - @expand_rotate_time
10
10
  paths = []
11
+
12
+ excluded = @exclude_path.map { |path| path = date.strftime(path); path.include?('*') ? Dir.glob(path) : path }.flatten.uniq
11
13
  @paths.each { |path|
12
14
  path = date.strftime(path)
13
15
  if path.include?('*')
14
- paths += Dir.glob(path)
16
+ paths += Dir.glob(path).select { |p|
17
+ if File.readable?(p)
18
+ true
19
+ else
20
+ log.warn "#{p} unreadable. It is excluded and would be examined next time."
21
+ false
22
+ end
23
+ }
15
24
  else
16
25
  # When file is not created yet, Dir.glob returns an empty array. So just add when path is static.
17
26
  paths << path
18
27
  end
19
28
  }
20
- paths
29
+ paths - excluded
21
30
  end
22
31
  end
23
32
  end
@@ -1,4 +1,4 @@
1
- # simplecov must be loaded before any of target code
1
+ # simplecov must be loaded before any of target code is loaded
2
2
  if ENV['SIMPLE_COV']
3
3
  require 'simplecov'
4
4
  if defined?(SimpleCov::SourceFile)
@@ -2,6 +2,7 @@ require_relative '../helper'
2
2
  require 'fluent/test'
3
3
  require 'net/http'
4
4
  require 'flexmock'
5
+ require 'flexmock/test_unit'
5
6
  require 'fluent/plugin/in_tail_ex_rotate'
6
7
 
7
8
  class TailExRotateInputTest < Test::Unit::TestCase
@@ -31,6 +32,43 @@ class TailExRotateInputTest < Test::Unit::TestCase
31
32
  format /(?<message>.*)/
32
33
  ]
33
34
 
35
+ # * path test
36
+ # TODO: Clean up tests
37
+ EX_RORATE_WAIT = 0
38
+
39
+ EX_CONFIG = %[
40
+ tag tail
41
+ path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
42
+ format none
43
+ pos_file #{TMP_DIR}/tail.pos
44
+ read_from_head true
45
+ refresh_interval 30
46
+ rotate_wait #{EX_RORATE_WAIT}s
47
+ ]
48
+ EX_PATHS = [
49
+ 'test/plugin/data/2010/01/20100102-030405.log',
50
+ 'test/plugin/data/log/foo/bar.log',
51
+ 'test/plugin/data/log/test.log'
52
+ ]
53
+
54
+
55
+ EX_CONFIG_FOR_ROTATE = %[
56
+ tag tail
57
+ path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
58
+ format none
59
+ pos_file #{TMP_DIR}/tail.pos
60
+ read_from_head true
61
+ refresh_interval 30
62
+ rotate_wait #{EX_RORATE_WAIT}s
63
+ expand_rotate_time 18000s
64
+ ]
65
+
66
+ EX_PATHS_FOR_ROATE = [
67
+ 'test/plugin/data/2014/12/20141224-000000.log',
68
+ 'test/plugin/data/log/foo/bar.log',
69
+ 'test/plugin/data/log/test.log'
70
+ ]
71
+
34
72
  def create_driver(conf = SINGLE_LINE_CONFIG, use_common_conf = true)
35
73
  config = use_common_conf ? COMMON_CONFIG + conf : conf
36
74
  Fluent::Test::InputTestDriver.new(Fluent::TailExRotateInput).configure(config)
@@ -42,7 +80,7 @@ class TailExRotateInputTest < Test::Unit::TestCase
42
80
  assert_equal "t1", d.instance.tag
43
81
  assert_equal 2, d.instance.rotate_wait
44
82
  assert_equal "#{TMP_DIR}/tail.pos", d.instance.pos_file
45
- assert_equal 18000, d.instance.expand_rotate_time
83
+ assert_equal 1000, d.instance.read_lines_limit
46
84
  end
47
85
 
48
86
  # TODO: Should using more better approach instead of sleep wait
@@ -69,6 +107,30 @@ class TailExRotateInputTest < Test::Unit::TestCase
69
107
  assert_equal(true, emits.length > 0)
70
108
  assert_equal({"message" => "test3"}, emits[0][2])
71
109
  assert_equal({"message" => "test4"}, emits[1][2])
110
+ assert_equal(1, d.emit_streams.size)
111
+ end
112
+
113
+ data('1' => [1, 2], '10' => [10, 1])
114
+ def test_emit_with_read_lines_limit(data)
115
+ limit, num_emits = data
116
+ d = create_driver(CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG + "read_lines_limit #{limit}")
117
+ msg = 'test' * 500 # in_tail reads 2048 bytes at once.
118
+
119
+ d.run do
120
+ sleep 1
121
+
122
+ File.open("#{TMP_DIR}/tail.txt", "a") {|f|
123
+ f.puts msg
124
+ f.puts msg
125
+ }
126
+ sleep 1
127
+ end
128
+
129
+ emits = d.emits
130
+ assert_equal(true, emits.length > 0)
131
+ assert_equal({"message" => msg}, emits[0][2])
132
+ assert_equal({"message" => msg}, emits[1][2])
133
+ assert_equal(num_emits, d.emit_streams.size)
72
134
  end
73
135
 
74
136
  def test_emit_with_read_from_head
@@ -367,56 +429,17 @@ class TailExRotateInputTest < Test::Unit::TestCase
367
429
  assert_equal({"var1" => "foo 2", "var2" => "bar 2", "var3" => "baz 2"}, emits[1][2])
368
430
  end
369
431
 
370
- # * path test
371
- # TODO: Clean up tests
372
- EX_RORATE_WAIT = 0
373
-
374
- EX_CONFIG = %[
375
- tag tail
376
- path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
377
- format none
378
- pos_file #{TMP_DIR}/tail.pos
379
- read_from_head true
380
- refresh_interval 30
381
- rotate_wait #{EX_RORATE_WAIT}s
382
- ]
383
- EX_PATHS = [
384
- 'test/plugin/data/2010/01/20100102-030405.log',
385
- 'test/plugin/data/log/foo/bar.log',
386
- 'test/plugin/data/log/test.log'
387
- ]
388
-
389
- EX_CONFIG_FOR_ROTATE = %[
390
- tag tail
391
- path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
392
- format none
393
- pos_file #{TMP_DIR}/tail.pos
394
- read_from_head true
395
- refresh_interval 30
396
- rotate_wait #{EX_RORATE_WAIT}s
397
- expand_rotate_time 18000s
398
- ]
399
-
400
- EX_PATHS_FOR_ROATE = [
401
- 'test/plugin/data/2014/12/20141224-000000.log',
402
- 'test/plugin/data/log/foo/bar.log',
403
- 'test/plugin/data/log/test.log'
404
- ]
405
-
406
432
  def test_expand_paths
407
433
  plugin = create_driver(EX_CONFIG, false).instance
408
434
  flexstub(Time) do |timeclass|
409
435
  timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 2, 3, 4, 5))
410
436
  assert_equal EX_PATHS, plugin.expand_paths.sort
411
437
  end
412
- end
413
438
 
414
- def test_expand_paths_for_ex_rotate
415
- plugin = create_driver(EX_CONFIG_FOR_ROTATE, false).instance
416
- flexstub(Time) do |timeclass|
417
- timeclass.should_receive(:now).with_no_args.and_return(Time.new(2014, 12, 24, 5, 0, 0))
418
- assert_equal EX_PATHS_FOR_ROATE, plugin.expand_paths.sort
419
- end
439
+ # Test exclusion
440
+ exclude_config = EX_CONFIG + " exclude_path [\"#{EX_PATHS.last}\"]"
441
+ plugin = create_driver(exclude_config, false).instance
442
+ assert_equal EX_PATHS - [EX_PATHS.last], plugin.expand_paths.sort
420
443
  end
421
444
 
422
445
  def test_refresh_watchers
@@ -432,7 +455,7 @@ class TailExRotateInputTest < Test::Unit::TestCase
432
455
 
433
456
  flexstub(Fluent::TailExRotateInput::TailWatcher) do |watcherclass|
434
457
  EX_PATHS.each do |path|
435
- watcherclass.should_receive(:new).with(path, EX_RORATE_WAIT, Fluent::TailExRotateInput::FilePositionEntry, any, true, any, any).once.and_return do
458
+ watcherclass.should_receive(:new).with(path, EX_RORATE_WAIT, Fluent::TailExRotateInput::FilePositionEntry, any, true, 1000, any, any).once.and_return do
436
459
  flexmock('TailWatcher') { |watcher|
437
460
  watcher.should_receive(:attach).once
438
461
  watcher.should_receive(:unwatched=).zero_or_more_times
@@ -448,7 +471,7 @@ class TailExRotateInputTest < Test::Unit::TestCase
448
471
  end
449
472
 
450
473
  flexstub(Fluent::TailExRotateInput::TailWatcher) do |watcherclass|
451
- watcherclass.should_receive(:new).with('test/plugin/data/2010/01/20100102-030406.log', EX_RORATE_WAIT, Fluent::TailExRotateInput::FilePositionEntry, any, true, any, any).once.and_return do
474
+ watcherclass.should_receive(:new).with('test/plugin/data/2010/01/20100102-030406.log', EX_RORATE_WAIT, Fluent::TailExRotateInput::FilePositionEntry, any, true, 1000, any, any).once.and_return do
452
475
  flexmock('TailWatcher') do |watcher|
453
476
  watcher.should_receive(:attach).once
454
477
  watcher.should_receive(:unwatched=).zero_or_more_times
@@ -557,4 +580,12 @@ class TailExRotateInputTest < Test::Unit::TestCase
557
580
  assert_equal({"message" => "test4"}, emits[1][2])
558
581
  end
559
582
  end
583
+
584
+ def test_expand_paths_for_ex_rotate
585
+ plugin = create_driver(EX_CONFIG_FOR_ROTATE, false).instance
586
+ flexstub(Time) do |timeclass|
587
+ timeclass.should_receive(:now).with_no_args.and_return(Time.new(2014, 12, 24, 5, 0, 0))
588
+ assert_equal EX_PATHS_FOR_ROATE, plugin.expand_paths.sort
589
+ end
590
+ end
560
591
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-tail-ex-rotate
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.3
4
+ version: 0.0.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Yuta Mizushima
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-01-23 00:00:00.000000000 Z
11
+ date: 2015-12-11 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -16,14 +16,14 @@ dependencies:
16
16
  requirements:
17
17
  - - ">="
18
18
  - !ruby/object:Gem::Version
19
- version: '0'
19
+ version: 0.12.16
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - ">="
25
25
  - !ruby/object:Gem::Version
26
- version: '0'
26
+ version: 0.12.16
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: bundler
29
29
  requirement: !ruby/object:Gem::Requirement
@@ -38,6 +38,34 @@ dependencies:
38
38
  - - "~>"
39
39
  - !ruby/object:Gem::Version
40
40
  version: '1.3'
41
+ - !ruby/object:Gem::Dependency
42
+ name: test-unit-rr
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: flexmock
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
41
69
  - !ruby/object:Gem::Dependency
42
70
  name: rake
43
71
  requirement: !ruby/object:Gem::Requirement
@@ -59,6 +87,7 @@ executables: []
59
87
  extensions: []
60
88
  extra_rdoc_files: []
61
89
  files:
90
+ - ".travis.yml"
62
91
  - COPYING
63
92
  - Gemfile
64
93
  - NOTICE
@@ -94,7 +123,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
94
123
  version: '0'
95
124
  requirements: []
96
125
  rubyforge_project:
97
- rubygems_version: 2.2.2
126
+ rubygems_version: 2.4.5.1
98
127
  signing_key:
99
128
  specification_version: 4
100
129
  summary: customize log rotate timing
@@ -107,4 +136,3 @@ test_files:
107
136
  - test/plugin/data/log/foo/bar.log
108
137
  - test/plugin/data/log/test.log
109
138
  - test/plugin/test_in_tail_ex_rotate.rb
110
- has_rdoc: false