fluent-plugin-tail-ex-rotate 0.0.3 → 0.0.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +3 -0
- data/README.md +10 -7
- data/fluent-plugin-tail-ex-rotate.gemspec +4 -2
- data/lib/fluent/plugin/in_tail_ex_rotate.rb +11 -2
- data/test/helper.rb +1 -1
- data/test/plugin/test_in_tail_ex_rotate.rb +77 -46
- metadata +34 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 41366d67f88d73008c13f8ccaba644c8a0275015
|
4
|
+
data.tar.gz: fe145dc47313f4198d4d454570f2a94c99f49484
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e9b342459226d0778124e4c027f982b4ee7b915f8d986f76f94c2d9eb292e275b8b667fd7bcce8d421a62c87c5b9250d5c1d3cdb56becc0ceec20e81118ac489
|
7
|
+
data.tar.gz: 1e5855ef0238b44210cb1c20ee24fe0ead7653bde7052a0f06c770e462e432ed110b2562d9ce320471013586b7480c7b2df78d1f99a0d3f8dbb5e28b7e467e45
|
data/.travis.yml
ADDED
data/README.md
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
# fluent-plugin-tail-ex-rotate
|
2
|
-
|
2
|
+
[![Build Status](https://travis-ci.org/dwango/fluent-plugin-tail-ex-rotate.png?branch=master)](https://travis-ci.org/dwango/fluent-plugin-tail-ex-rotate)
|
3
3
|
|
4
|
-
fluent-plugin-tail-ex-rotate is a fluentd plugin to
|
4
|
+
fluent-plugin-tail-ex-rotate is a fluentd plugin to delay file lotation time.
|
5
5
|
|
6
6
|
## Install
|
7
7
|
|
@@ -12,25 +12,28 @@ gem install fluent-plugin-tail-ex-rotate
|
|
12
12
|
## Test
|
13
13
|
|
14
14
|
```sh
|
15
|
+
bundle install
|
15
16
|
rake test
|
16
17
|
```
|
17
18
|
|
18
19
|
## Setting
|
19
20
|
|
20
21
|
The setting of fluent-plugin-tail-ex-rotate is roughly the same with in_tail built-in plugin.
|
21
|
-
|
22
|
+
Difference between fluent-plugin-tail-ex-rotate and in_tail built-in plugin setting are :expand_rotate_time attribute.
|
22
23
|
|
23
|
-
|
24
|
+
:expand_rotate_time attribute must be set number of seconds.
|
24
25
|
|
25
26
|
For example,
|
26
27
|
```xml
|
27
28
|
<source>
|
28
29
|
type tail_ex_rotate
|
29
|
-
path /var/log/
|
30
|
-
tag
|
30
|
+
path /var/log/httpd/access_log.%Y%m%d
|
31
|
+
tag access_process_time
|
31
32
|
pos_file /var/log/td-agent/httpd-access.log.pos
|
32
33
|
time_format %d/%b/%Y:%H:%M:%S %z
|
33
|
-
expand_rotate_time
|
34
|
+
expand_rotate_time 18000s
|
34
35
|
types process_time:integer
|
35
36
|
</source>
|
36
37
|
```
|
38
|
+
|
39
|
+
Daily file lotation starts with 05:00 beacause of this setting.
|
@@ -2,7 +2,7 @@ require File.expand_path('../lib/fluent/version', __FILE__)
|
|
2
2
|
|
3
3
|
Gem::Specification.new do |gem|
|
4
4
|
gem.name = "fluent-plugin-tail-ex-rotate"
|
5
|
-
gem.version = "0.0.
|
5
|
+
gem.version = "0.0.4"
|
6
6
|
|
7
7
|
gem.authors = ["Yuta Mizushima"]
|
8
8
|
gem.email = ["yuta_mizushima@dwango.co.jp"]
|
@@ -18,8 +18,10 @@ Gem::Specification.new do |gem|
|
|
18
18
|
|
19
19
|
gem.required_ruby_version = '>= 1.9.3'
|
20
20
|
|
21
|
-
gem.add_runtime_dependency("fluentd")
|
21
|
+
gem.add_runtime_dependency("fluentd", [">= 0.12.16"])
|
22
22
|
|
23
23
|
gem.add_development_dependency("bundler", "~> 1.3")
|
24
|
+
gem.add_development_dependency("test-unit-rr")
|
25
|
+
gem.add_development_dependency("flexmock")
|
24
26
|
gem.add_development_dependency("rake", [">= 0.9.2"])
|
25
27
|
end
|
@@ -8,16 +8,25 @@ module Fluent
|
|
8
8
|
def expand_paths
|
9
9
|
date = Time.now - @expand_rotate_time
|
10
10
|
paths = []
|
11
|
+
|
12
|
+
excluded = @exclude_path.map { |path| path = date.strftime(path); path.include?('*') ? Dir.glob(path) : path }.flatten.uniq
|
11
13
|
@paths.each { |path|
|
12
14
|
path = date.strftime(path)
|
13
15
|
if path.include?('*')
|
14
|
-
paths += Dir.glob(path)
|
16
|
+
paths += Dir.glob(path).select { |p|
|
17
|
+
if File.readable?(p)
|
18
|
+
true
|
19
|
+
else
|
20
|
+
log.warn "#{p} unreadable. It is excluded and would be examined next time."
|
21
|
+
false
|
22
|
+
end
|
23
|
+
}
|
15
24
|
else
|
16
25
|
# When file is not created yet, Dir.glob returns an empty array. So just add when path is static.
|
17
26
|
paths << path
|
18
27
|
end
|
19
28
|
}
|
20
|
-
paths
|
29
|
+
paths - excluded
|
21
30
|
end
|
22
31
|
end
|
23
32
|
end
|
data/test/helper.rb
CHANGED
@@ -2,6 +2,7 @@ require_relative '../helper'
|
|
2
2
|
require 'fluent/test'
|
3
3
|
require 'net/http'
|
4
4
|
require 'flexmock'
|
5
|
+
require 'flexmock/test_unit'
|
5
6
|
require 'fluent/plugin/in_tail_ex_rotate'
|
6
7
|
|
7
8
|
class TailExRotateInputTest < Test::Unit::TestCase
|
@@ -31,6 +32,43 @@ class TailExRotateInputTest < Test::Unit::TestCase
|
|
31
32
|
format /(?<message>.*)/
|
32
33
|
]
|
33
34
|
|
35
|
+
# * path test
|
36
|
+
# TODO: Clean up tests
|
37
|
+
EX_RORATE_WAIT = 0
|
38
|
+
|
39
|
+
EX_CONFIG = %[
|
40
|
+
tag tail
|
41
|
+
path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
|
42
|
+
format none
|
43
|
+
pos_file #{TMP_DIR}/tail.pos
|
44
|
+
read_from_head true
|
45
|
+
refresh_interval 30
|
46
|
+
rotate_wait #{EX_RORATE_WAIT}s
|
47
|
+
]
|
48
|
+
EX_PATHS = [
|
49
|
+
'test/plugin/data/2010/01/20100102-030405.log',
|
50
|
+
'test/plugin/data/log/foo/bar.log',
|
51
|
+
'test/plugin/data/log/test.log'
|
52
|
+
]
|
53
|
+
|
54
|
+
|
55
|
+
EX_CONFIG_FOR_ROTATE = %[
|
56
|
+
tag tail
|
57
|
+
path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
|
58
|
+
format none
|
59
|
+
pos_file #{TMP_DIR}/tail.pos
|
60
|
+
read_from_head true
|
61
|
+
refresh_interval 30
|
62
|
+
rotate_wait #{EX_RORATE_WAIT}s
|
63
|
+
expand_rotate_time 18000s
|
64
|
+
]
|
65
|
+
|
66
|
+
EX_PATHS_FOR_ROATE = [
|
67
|
+
'test/plugin/data/2014/12/20141224-000000.log',
|
68
|
+
'test/plugin/data/log/foo/bar.log',
|
69
|
+
'test/plugin/data/log/test.log'
|
70
|
+
]
|
71
|
+
|
34
72
|
def create_driver(conf = SINGLE_LINE_CONFIG, use_common_conf = true)
|
35
73
|
config = use_common_conf ? COMMON_CONFIG + conf : conf
|
36
74
|
Fluent::Test::InputTestDriver.new(Fluent::TailExRotateInput).configure(config)
|
@@ -42,7 +80,7 @@ class TailExRotateInputTest < Test::Unit::TestCase
|
|
42
80
|
assert_equal "t1", d.instance.tag
|
43
81
|
assert_equal 2, d.instance.rotate_wait
|
44
82
|
assert_equal "#{TMP_DIR}/tail.pos", d.instance.pos_file
|
45
|
-
assert_equal
|
83
|
+
assert_equal 1000, d.instance.read_lines_limit
|
46
84
|
end
|
47
85
|
|
48
86
|
# TODO: Should using more better approach instead of sleep wait
|
@@ -69,6 +107,30 @@ class TailExRotateInputTest < Test::Unit::TestCase
|
|
69
107
|
assert_equal(true, emits.length > 0)
|
70
108
|
assert_equal({"message" => "test3"}, emits[0][2])
|
71
109
|
assert_equal({"message" => "test4"}, emits[1][2])
|
110
|
+
assert_equal(1, d.emit_streams.size)
|
111
|
+
end
|
112
|
+
|
113
|
+
data('1' => [1, 2], '10' => [10, 1])
|
114
|
+
def test_emit_with_read_lines_limit(data)
|
115
|
+
limit, num_emits = data
|
116
|
+
d = create_driver(CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG + "read_lines_limit #{limit}")
|
117
|
+
msg = 'test' * 500 # in_tail reads 2048 bytes at once.
|
118
|
+
|
119
|
+
d.run do
|
120
|
+
sleep 1
|
121
|
+
|
122
|
+
File.open("#{TMP_DIR}/tail.txt", "a") {|f|
|
123
|
+
f.puts msg
|
124
|
+
f.puts msg
|
125
|
+
}
|
126
|
+
sleep 1
|
127
|
+
end
|
128
|
+
|
129
|
+
emits = d.emits
|
130
|
+
assert_equal(true, emits.length > 0)
|
131
|
+
assert_equal({"message" => msg}, emits[0][2])
|
132
|
+
assert_equal({"message" => msg}, emits[1][2])
|
133
|
+
assert_equal(num_emits, d.emit_streams.size)
|
72
134
|
end
|
73
135
|
|
74
136
|
def test_emit_with_read_from_head
|
@@ -367,56 +429,17 @@ class TailExRotateInputTest < Test::Unit::TestCase
|
|
367
429
|
assert_equal({"var1" => "foo 2", "var2" => "bar 2", "var3" => "baz 2"}, emits[1][2])
|
368
430
|
end
|
369
431
|
|
370
|
-
# * path test
|
371
|
-
# TODO: Clean up tests
|
372
|
-
EX_RORATE_WAIT = 0
|
373
|
-
|
374
|
-
EX_CONFIG = %[
|
375
|
-
tag tail
|
376
|
-
path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
|
377
|
-
format none
|
378
|
-
pos_file #{TMP_DIR}/tail.pos
|
379
|
-
read_from_head true
|
380
|
-
refresh_interval 30
|
381
|
-
rotate_wait #{EX_RORATE_WAIT}s
|
382
|
-
]
|
383
|
-
EX_PATHS = [
|
384
|
-
'test/plugin/data/2010/01/20100102-030405.log',
|
385
|
-
'test/plugin/data/log/foo/bar.log',
|
386
|
-
'test/plugin/data/log/test.log'
|
387
|
-
]
|
388
|
-
|
389
|
-
EX_CONFIG_FOR_ROTATE = %[
|
390
|
-
tag tail
|
391
|
-
path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
|
392
|
-
format none
|
393
|
-
pos_file #{TMP_DIR}/tail.pos
|
394
|
-
read_from_head true
|
395
|
-
refresh_interval 30
|
396
|
-
rotate_wait #{EX_RORATE_WAIT}s
|
397
|
-
expand_rotate_time 18000s
|
398
|
-
]
|
399
|
-
|
400
|
-
EX_PATHS_FOR_ROATE = [
|
401
|
-
'test/plugin/data/2014/12/20141224-000000.log',
|
402
|
-
'test/plugin/data/log/foo/bar.log',
|
403
|
-
'test/plugin/data/log/test.log'
|
404
|
-
]
|
405
|
-
|
406
432
|
def test_expand_paths
|
407
433
|
plugin = create_driver(EX_CONFIG, false).instance
|
408
434
|
flexstub(Time) do |timeclass|
|
409
435
|
timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 2, 3, 4, 5))
|
410
436
|
assert_equal EX_PATHS, plugin.expand_paths.sort
|
411
437
|
end
|
412
|
-
end
|
413
438
|
|
414
|
-
|
415
|
-
|
416
|
-
|
417
|
-
|
418
|
-
assert_equal EX_PATHS_FOR_ROATE, plugin.expand_paths.sort
|
419
|
-
end
|
439
|
+
# Test exclusion
|
440
|
+
exclude_config = EX_CONFIG + " exclude_path [\"#{EX_PATHS.last}\"]"
|
441
|
+
plugin = create_driver(exclude_config, false).instance
|
442
|
+
assert_equal EX_PATHS - [EX_PATHS.last], plugin.expand_paths.sort
|
420
443
|
end
|
421
444
|
|
422
445
|
def test_refresh_watchers
|
@@ -432,7 +455,7 @@ class TailExRotateInputTest < Test::Unit::TestCase
|
|
432
455
|
|
433
456
|
flexstub(Fluent::TailExRotateInput::TailWatcher) do |watcherclass|
|
434
457
|
EX_PATHS.each do |path|
|
435
|
-
watcherclass.should_receive(:new).with(path, EX_RORATE_WAIT, Fluent::TailExRotateInput::FilePositionEntry, any, true, any, any).once.and_return do
|
458
|
+
watcherclass.should_receive(:new).with(path, EX_RORATE_WAIT, Fluent::TailExRotateInput::FilePositionEntry, any, true, 1000, any, any).once.and_return do
|
436
459
|
flexmock('TailWatcher') { |watcher|
|
437
460
|
watcher.should_receive(:attach).once
|
438
461
|
watcher.should_receive(:unwatched=).zero_or_more_times
|
@@ -448,7 +471,7 @@ class TailExRotateInputTest < Test::Unit::TestCase
|
|
448
471
|
end
|
449
472
|
|
450
473
|
flexstub(Fluent::TailExRotateInput::TailWatcher) do |watcherclass|
|
451
|
-
watcherclass.should_receive(:new).with('test/plugin/data/2010/01/20100102-030406.log', EX_RORATE_WAIT, Fluent::TailExRotateInput::FilePositionEntry, any, true, any, any).once.and_return do
|
474
|
+
watcherclass.should_receive(:new).with('test/plugin/data/2010/01/20100102-030406.log', EX_RORATE_WAIT, Fluent::TailExRotateInput::FilePositionEntry, any, true, 1000, any, any).once.and_return do
|
452
475
|
flexmock('TailWatcher') do |watcher|
|
453
476
|
watcher.should_receive(:attach).once
|
454
477
|
watcher.should_receive(:unwatched=).zero_or_more_times
|
@@ -557,4 +580,12 @@ class TailExRotateInputTest < Test::Unit::TestCase
|
|
557
580
|
assert_equal({"message" => "test4"}, emits[1][2])
|
558
581
|
end
|
559
582
|
end
|
583
|
+
|
584
|
+
def test_expand_paths_for_ex_rotate
|
585
|
+
plugin = create_driver(EX_CONFIG_FOR_ROTATE, false).instance
|
586
|
+
flexstub(Time) do |timeclass|
|
587
|
+
timeclass.should_receive(:now).with_no_args.and_return(Time.new(2014, 12, 24, 5, 0, 0))
|
588
|
+
assert_equal EX_PATHS_FOR_ROATE, plugin.expand_paths.sort
|
589
|
+
end
|
590
|
+
end
|
560
591
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-tail-ex-rotate
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yuta Mizushima
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-12-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -16,14 +16,14 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version:
|
19
|
+
version: 0.12.16
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version:
|
26
|
+
version: 0.12.16
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: bundler
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -38,6 +38,34 @@ dependencies:
|
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '1.3'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: test-unit-rr
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: flexmock
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
41
69
|
- !ruby/object:Gem::Dependency
|
42
70
|
name: rake
|
43
71
|
requirement: !ruby/object:Gem::Requirement
|
@@ -59,6 +87,7 @@ executables: []
|
|
59
87
|
extensions: []
|
60
88
|
extra_rdoc_files: []
|
61
89
|
files:
|
90
|
+
- ".travis.yml"
|
62
91
|
- COPYING
|
63
92
|
- Gemfile
|
64
93
|
- NOTICE
|
@@ -94,7 +123,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
94
123
|
version: '0'
|
95
124
|
requirements: []
|
96
125
|
rubyforge_project:
|
97
|
-
rubygems_version: 2.
|
126
|
+
rubygems_version: 2.4.5.1
|
98
127
|
signing_key:
|
99
128
|
specification_version: 4
|
100
129
|
summary: customize log rotate timing
|
@@ -107,4 +136,3 @@ test_files:
|
|
107
136
|
- test/plugin/data/log/foo/bar.log
|
108
137
|
- test/plugin/data/log/test.log
|
109
138
|
- test/plugin/test_in_tail_ex_rotate.rb
|
110
|
-
has_rdoc: false
|