fluent-plugin-tail-ex-rotate 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/COPYING +14 -0
- data/Gemfile +3 -0
- data/Gemfile.lock +40 -0
- data/NOTICE +5 -0
- data/README.md +30 -0
- data/Rakefile +33 -0
- data/fluent-plugin-tail-ex-rotate.gemspec +25 -0
- data/lib/fluent/plugin/in_tail_ex_rotate.rb +23 -0
- data/lib/fluent/version.rb +21 -0
- data/pkg/fluent-plugin-tail-ex-rotate-0.0.1.gem +0 -0
- data/test/helper.rb +56 -0
- data/test/plugin/data/2010/01/20100102-030405.log +0 -0
- data/test/plugin/data/2010/01/20100102-030406.log +0 -0
- data/test/plugin/data/2014/12/20141224-000000.log +0 -0
- data/test/plugin/data/log/bar +0 -0
- data/test/plugin/data/log/foo/bar.log +0 -0
- data/test/plugin/data/log/test.log +0 -0
- data/test/plugin/test_in_tail_ex_rotate.rb +560 -0
- metadata +112 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: a77ea0c7caa116f06886673d73d83d9b822b1b5b
|
4
|
+
data.tar.gz: 12e56f4a6dd248b086b35b73db277f1400b6a8a7
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 6209fbe36cde82650e465dc817a5fafa81614163f2d35ee7912e5329fbad818a1eac96b29a8cc769ccc6f68f5fdb66e78b438cdedbff707ceea0b71a10006f16
|
7
|
+
data.tar.gz: dda0d444b1744ff0fc49185526e2d39ed880bd37b1250d7094a36f22afdb39981eee4e9b5cea9c24bf360fd795eb7f62e955db391018010c067a1468c04d207f
|
data/COPYING
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
Copyright (C) 2015 DWANGO Co., Ltd.
|
2
|
+
|
3
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
you may not use this file except in compliance with the License.
|
5
|
+
You may obtain a copy of the License at
|
6
|
+
|
7
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
|
9
|
+
Unless required by applicable law or agreed to in writing, software
|
10
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
See the License for the specific language governing permissions and
|
13
|
+
limitations under the License.
|
14
|
+
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,40 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
fluent-plugin-tail-ex-rotate (0.0.1)
|
5
|
+
fluentd
|
6
|
+
|
7
|
+
GEM
|
8
|
+
remote: https://rubygems.org/
|
9
|
+
specs:
|
10
|
+
cool.io (1.2.4)
|
11
|
+
fluentd (0.12.2)
|
12
|
+
cool.io (>= 1.2.2, < 2.0.0)
|
13
|
+
http_parser.rb (>= 0.5.1, < 0.7.0)
|
14
|
+
json (>= 1.4.3)
|
15
|
+
msgpack (>= 0.5.4, < 0.6.0)
|
16
|
+
sigdump (~> 0.2.2)
|
17
|
+
string-scrub (>= 0.0.3)
|
18
|
+
tzinfo (>= 1.0.0)
|
19
|
+
tzinfo-data (>= 1.0.0)
|
20
|
+
yajl-ruby (~> 1.0)
|
21
|
+
http_parser.rb (0.6.0)
|
22
|
+
json (1.8.1)
|
23
|
+
msgpack (0.5.9)
|
24
|
+
rake (10.4.2)
|
25
|
+
sigdump (0.2.2)
|
26
|
+
string-scrub (0.0.5)
|
27
|
+
thread_safe (0.3.4)
|
28
|
+
tzinfo (1.2.2)
|
29
|
+
thread_safe (~> 0.1)
|
30
|
+
tzinfo-data (1.2014.10)
|
31
|
+
tzinfo (>= 1.0.0)
|
32
|
+
yajl-ruby (1.2.1)
|
33
|
+
|
34
|
+
PLATFORMS
|
35
|
+
ruby
|
36
|
+
|
37
|
+
DEPENDENCIES
|
38
|
+
bundler (~> 1.3)
|
39
|
+
fluent-plugin-tail-ex-rotate!
|
40
|
+
rake (>= 0.9.2)
|
data/NOTICE
ADDED
data/README.md
ADDED
@@ -0,0 +1,30 @@
|
|
1
|
+
# fluent-plugin-tail-ex-rotate
|
2
|
+
===================================
|
3
|
+
|
4
|
+
fluent-plugin-tail-ex-rotate is a fluentd plugin to extend file lotation time.
|
5
|
+
|
6
|
+
## Install
|
7
|
+
|
8
|
+
```sh
|
9
|
+
gem install fluent-plugin-tail-ex-rotate
|
10
|
+
```
|
11
|
+
|
12
|
+
## Setting
|
13
|
+
|
14
|
+
The setting of fluent-plugin-tail-ex-rotate is roughly the same with in_tail built-in plugin.
|
15
|
+
A defference between fluent-plugin-tail-ex-rotate and in_tail built-in plugin setting is "expand_rotate_time" parameter
|
16
|
+
|
17
|
+
"expand_rotate_time" parameter should be set number of second.
|
18
|
+
|
19
|
+
For example,
|
20
|
+
```xml
|
21
|
+
<source>
|
22
|
+
type tail_ex_rotate
|
23
|
+
path /var/log/apache2/access_log
|
24
|
+
tag internal.eapi_nicolive_process_time
|
25
|
+
pos_file /var/log/td-agent/httpd-access.log.pos
|
26
|
+
time_format %d/%b/%Y:%H:%M:%S %z
|
27
|
+
expand_rotate_time 18000
|
28
|
+
types process_time:integer
|
29
|
+
</source>
|
30
|
+
```
|
data/Rakefile
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
#!/usr/bin/env rake
|
2
|
+
require "bundler/gem_tasks"
|
3
|
+
|
4
|
+
require 'fileutils'
|
5
|
+
require 'rake/testtask'
|
6
|
+
require 'rake/clean'
|
7
|
+
|
8
|
+
task :test => [:base_test]
|
9
|
+
|
10
|
+
desc 'Run test_unit based test'
|
11
|
+
Rake::TestTask.new(:base_test) do |t|
|
12
|
+
# To run test for only one file (or file path pattern)
|
13
|
+
# $ bundle exec rake base_test TEST=test/test_specified_path.rb
|
14
|
+
# $ bundle exec rake base_test TEST=test/test_*.rb
|
15
|
+
t.libs << "test"
|
16
|
+
t.test_files = Dir["test/**/test_*.rb"].sort
|
17
|
+
t.verbose = true
|
18
|
+
#t.warning = true
|
19
|
+
end
|
20
|
+
|
21
|
+
task :parallel_test do
|
22
|
+
FileUtils.rm_rf('./test/tmp')
|
23
|
+
sh("parallel_test ./test/*.rb ./test/plugin/*.rb")
|
24
|
+
FileUtils.rm_rf('./test/tmp')
|
25
|
+
end
|
26
|
+
|
27
|
+
desc 'Run test with simplecov'
|
28
|
+
task :coverage do |t|
|
29
|
+
ENV['SIMPLE_COV'] = '1'
|
30
|
+
Rake::Task["test"].invoke
|
31
|
+
end
|
32
|
+
|
33
|
+
task :default => [:test, :build]
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require File.expand_path('../lib/fluent/version', __FILE__)
|
2
|
+
|
3
|
+
Gem::Specification.new do |gem|
|
4
|
+
gem.name = "fluent-plugin-tail-ex-rotate"
|
5
|
+
gem.version = "0.0.1"
|
6
|
+
|
7
|
+
gem.authors = ["Yuta Mizushima"]
|
8
|
+
gem.email = ["yuta_mizushima@dwango.co.jp"]
|
9
|
+
gem.description = %q{Extension of in_tail plugin to customize log rotate timing.}
|
10
|
+
gem.summary = %q{customize log rotate timing}
|
11
|
+
gem.homepage = "http://github.o-in.dwango.co.jp/Yuta-Mizushima/namasasy_setup/tree/master/node/fluent-plugin-tail-ex-rotate"
|
12
|
+
|
13
|
+
gem.files = `git ls-files`.split($\)
|
14
|
+
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
15
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
16
|
+
gem.require_paths = ["lib"]
|
17
|
+
gem.has_rdoc = false
|
18
|
+
|
19
|
+
gem.required_ruby_version = '>= 1.9.3'
|
20
|
+
|
21
|
+
gem.add_runtime_dependency("fluentd")
|
22
|
+
|
23
|
+
gem.add_development_dependency("bundler", "~> 1.3")
|
24
|
+
gem.add_development_dependency("rake", [">= 0.9.2"])
|
25
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'fluent/plugin/in_tail'
|
2
|
+
|
3
|
+
module Fluent
|
4
|
+
class TailExRotateInput < Fluent::NewTailInput
|
5
|
+
Fluent::Plugin.register_input('tail_ex_rotate', self)
|
6
|
+
config_param :expand_rotate_time, :time, :default => 0
|
7
|
+
|
8
|
+
def expand_paths
|
9
|
+
date = Time.now - @expand_rotate_time
|
10
|
+
paths = []
|
11
|
+
@paths.each { |path|
|
12
|
+
path = date.strftime(path)
|
13
|
+
if path.include?('*')
|
14
|
+
paths += Dir.glob(path)
|
15
|
+
else
|
16
|
+
# When file is not created yet, Dir.glob returns an empty array. So just add when path is static.
|
17
|
+
paths << path
|
18
|
+
end
|
19
|
+
}
|
20
|
+
paths
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
module Fluent
|
18
|
+
|
19
|
+
VERSION = '0.12.2'
|
20
|
+
|
21
|
+
end
|
Binary file
|
data/test/helper.rb
ADDED
@@ -0,0 +1,56 @@
|
|
1
|
+
# simplecov must be loaded before any of target code
|
2
|
+
if ENV['SIMPLE_COV']
|
3
|
+
require 'simplecov'
|
4
|
+
if defined?(SimpleCov::SourceFile)
|
5
|
+
mod = SimpleCov::SourceFile
|
6
|
+
def mod.new(*args, &block)
|
7
|
+
m = allocate
|
8
|
+
m.instance_eval do
|
9
|
+
begin
|
10
|
+
initialize(*args, &block)
|
11
|
+
rescue Encoding::UndefinedConversionError
|
12
|
+
@src = "".force_encoding('UTF-8')
|
13
|
+
end
|
14
|
+
end
|
15
|
+
m
|
16
|
+
end
|
17
|
+
end
|
18
|
+
unless SimpleCov.running
|
19
|
+
SimpleCov.start do
|
20
|
+
add_filter '/test/'
|
21
|
+
add_filter '/gems/'
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
require 'rr'
|
27
|
+
require 'test/unit'
|
28
|
+
require 'test/unit/rr'
|
29
|
+
require 'fileutils'
|
30
|
+
require 'fluent/log'
|
31
|
+
require 'fluent/test'
|
32
|
+
|
33
|
+
unless defined?(Test::Unit::AssertionFailedError)
|
34
|
+
class Test::Unit::AssertionFailedError < StandardError
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def unused_port
|
39
|
+
s = TCPServer.open(0)
|
40
|
+
port = s.addr[1]
|
41
|
+
s.close
|
42
|
+
port
|
43
|
+
end
|
44
|
+
|
45
|
+
def ipv6_enabled?
|
46
|
+
require 'socket'
|
47
|
+
|
48
|
+
begin
|
49
|
+
TCPServer.open("::1", 0)
|
50
|
+
true
|
51
|
+
rescue
|
52
|
+
false
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
$log = Fluent::Log.new(Fluent::Test::DummyLogDevice.new, Fluent::Log::LEVEL_WARN)
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
@@ -0,0 +1,560 @@
|
|
1
|
+
require_relative '../helper'
|
2
|
+
require 'fluent/test'
|
3
|
+
require 'net/http'
|
4
|
+
require 'flexmock'
|
5
|
+
require 'fluent/plugin/in_tail_ex_rotate'
|
6
|
+
|
7
|
+
class TailExRotateInputTest < Test::Unit::TestCase
|
8
|
+
include FlexMock::TestCase
|
9
|
+
|
10
|
+
def setup
|
11
|
+
Fluent::Test.setup
|
12
|
+
FileUtils.rm_rf(TMP_DIR)
|
13
|
+
FileUtils.mkdir_p(TMP_DIR)
|
14
|
+
end
|
15
|
+
|
16
|
+
TMP_DIR = File.dirname(__FILE__) + "/../tmp/tail#{ENV['TEST_ENV_NUMBER']}"
|
17
|
+
|
18
|
+
CONFIG = %[
|
19
|
+
path #{TMP_DIR}/tail.txt
|
20
|
+
tag t1
|
21
|
+
rotate_wait 2s
|
22
|
+
expand_rotate_time 18000s
|
23
|
+
]
|
24
|
+
COMMON_CONFIG = CONFIG + %[
|
25
|
+
pos_file #{TMP_DIR}/tail.pos
|
26
|
+
]
|
27
|
+
CONFIG_READ_FROM_HEAD = %[
|
28
|
+
read_from_head true
|
29
|
+
]
|
30
|
+
SINGLE_LINE_CONFIG = %[
|
31
|
+
format /(?<message>.*)/
|
32
|
+
]
|
33
|
+
|
34
|
+
def create_driver(conf = SINGLE_LINE_CONFIG, use_common_conf = true)
|
35
|
+
config = use_common_conf ? COMMON_CONFIG + conf : conf
|
36
|
+
Fluent::Test::InputTestDriver.new(Fluent::TailExRotateInput).configure(config)
|
37
|
+
end
|
38
|
+
|
39
|
+
def test_configure
|
40
|
+
d = create_driver
|
41
|
+
assert_equal ["#{TMP_DIR}/tail.txt"], d.instance.paths
|
42
|
+
assert_equal "t1", d.instance.tag
|
43
|
+
assert_equal 2, d.instance.rotate_wait
|
44
|
+
assert_equal "#{TMP_DIR}/tail.pos", d.instance.pos_file
|
45
|
+
assert_equal 18000, d.instance.expand_rotate_time
|
46
|
+
end
|
47
|
+
|
48
|
+
# TODO: Should using more better approach instead of sleep wait
|
49
|
+
|
50
|
+
def test_emit
|
51
|
+
File.open("#{TMP_DIR}/tail.txt", "w") {|f|
|
52
|
+
f.puts "test1"
|
53
|
+
f.puts "test2"
|
54
|
+
}
|
55
|
+
|
56
|
+
d = create_driver
|
57
|
+
|
58
|
+
d.run do
|
59
|
+
sleep 1
|
60
|
+
|
61
|
+
File.open("#{TMP_DIR}/tail.txt", "a") {|f|
|
62
|
+
f.puts "test3"
|
63
|
+
f.puts "test4"
|
64
|
+
}
|
65
|
+
sleep 1
|
66
|
+
end
|
67
|
+
|
68
|
+
emits = d.emits
|
69
|
+
assert_equal(true, emits.length > 0)
|
70
|
+
assert_equal({"message" => "test3"}, emits[0][2])
|
71
|
+
assert_equal({"message" => "test4"}, emits[1][2])
|
72
|
+
end
|
73
|
+
|
74
|
+
def test_emit_with_read_from_head
|
75
|
+
File.open("#{TMP_DIR}/tail.txt", "w") {|f|
|
76
|
+
f.puts "test1"
|
77
|
+
f.puts "test2"
|
78
|
+
}
|
79
|
+
|
80
|
+
d = create_driver(CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG)
|
81
|
+
|
82
|
+
d.run do
|
83
|
+
sleep 1
|
84
|
+
|
85
|
+
File.open("#{TMP_DIR}/tail.txt", "a") {|f|
|
86
|
+
f.puts "test3"
|
87
|
+
f.puts "test4"
|
88
|
+
}
|
89
|
+
sleep 1
|
90
|
+
end
|
91
|
+
|
92
|
+
emits = d.emits
|
93
|
+
assert(emits.length > 0)
|
94
|
+
assert_equal({"message" => "test1"}, emits[0][2])
|
95
|
+
assert_equal({"message" => "test2"}, emits[1][2])
|
96
|
+
assert_equal({"message" => "test3"}, emits[2][2])
|
97
|
+
assert_equal({"message" => "test4"}, emits[3][2])
|
98
|
+
end
|
99
|
+
|
100
|
+
def test_rotate_file
|
101
|
+
emits = sub_test_rotate_file(SINGLE_LINE_CONFIG)
|
102
|
+
assert_equal(4, emits.length)
|
103
|
+
assert_equal({"message" => "test3"}, emits[0][2])
|
104
|
+
assert_equal({"message" => "test4"}, emits[1][2])
|
105
|
+
assert_equal({"message" => "test5"}, emits[2][2])
|
106
|
+
assert_equal({"message" => "test6"}, emits[3][2])
|
107
|
+
end
|
108
|
+
|
109
|
+
def test_rotate_file_with_read_from_head
|
110
|
+
emits = sub_test_rotate_file(CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG)
|
111
|
+
assert_equal(6, emits.length)
|
112
|
+
assert_equal({"message" => "test1"}, emits[0][2])
|
113
|
+
assert_equal({"message" => "test2"}, emits[1][2])
|
114
|
+
assert_equal({"message" => "test3"}, emits[2][2])
|
115
|
+
assert_equal({"message" => "test4"}, emits[3][2])
|
116
|
+
assert_equal({"message" => "test5"}, emits[4][2])
|
117
|
+
assert_equal({"message" => "test6"}, emits[5][2])
|
118
|
+
end
|
119
|
+
|
120
|
+
def test_rotate_file_with_write_old
|
121
|
+
emits = sub_test_rotate_file(SINGLE_LINE_CONFIG) { |rotated_file|
|
122
|
+
File.open("#{TMP_DIR}/tail.txt", "w") { |f| }
|
123
|
+
rotated_file.puts "test7"
|
124
|
+
rotated_file.puts "test8"
|
125
|
+
rotated_file.flush
|
126
|
+
|
127
|
+
sleep 1
|
128
|
+
File.open("#{TMP_DIR}/tail.txt", "a") { |f|
|
129
|
+
f.puts "test5"
|
130
|
+
f.puts "test6"
|
131
|
+
}
|
132
|
+
}
|
133
|
+
assert_equal(6, emits.length)
|
134
|
+
assert_equal({"message" => "test3"}, emits[0][2])
|
135
|
+
assert_equal({"message" => "test4"}, emits[1][2])
|
136
|
+
assert_equal({"message" => "test7"}, emits[2][2])
|
137
|
+
assert_equal({"message" => "test8"}, emits[3][2])
|
138
|
+
assert_equal({"message" => "test5"}, emits[4][2])
|
139
|
+
assert_equal({"message" => "test6"}, emits[5][2])
|
140
|
+
end
|
141
|
+
|
142
|
+
def test_rotate_file_with_write_old_and_no_new_file
|
143
|
+
emits = sub_test_rotate_file(SINGLE_LINE_CONFIG) { |rotated_file|
|
144
|
+
rotated_file.puts "test7"
|
145
|
+
rotated_file.puts "test8"
|
146
|
+
rotated_file.flush
|
147
|
+
}
|
148
|
+
assert_equal(4, emits.length)
|
149
|
+
assert_equal({"message" => "test3"}, emits[0][2])
|
150
|
+
assert_equal({"message" => "test4"}, emits[1][2])
|
151
|
+
assert_equal({"message" => "test7"}, emits[2][2])
|
152
|
+
assert_equal({"message" => "test8"}, emits[3][2])
|
153
|
+
end
|
154
|
+
|
155
|
+
def sub_test_rotate_file(config = nil)
|
156
|
+
file = File.open("#{TMP_DIR}/tail.txt", "w")
|
157
|
+
file.puts "test1"
|
158
|
+
file.puts "test2"
|
159
|
+
file.flush
|
160
|
+
|
161
|
+
d = create_driver(config)
|
162
|
+
d.run do
|
163
|
+
sleep 1
|
164
|
+
|
165
|
+
file.puts "test3"
|
166
|
+
file.puts "test4"
|
167
|
+
file.flush
|
168
|
+
sleep 1
|
169
|
+
|
170
|
+
FileUtils.mv("#{TMP_DIR}/tail.txt", "#{TMP_DIR}/tail2.txt")
|
171
|
+
if block_given?
|
172
|
+
yield file
|
173
|
+
sleep 1
|
174
|
+
else
|
175
|
+
sleep 1
|
176
|
+
File.open("#{TMP_DIR}/tail.txt", "w") { |f| }
|
177
|
+
sleep 1
|
178
|
+
|
179
|
+
File.open("#{TMP_DIR}/tail.txt", "a") { |f|
|
180
|
+
f.puts "test5"
|
181
|
+
f.puts "test6"
|
182
|
+
}
|
183
|
+
sleep 1
|
184
|
+
end
|
185
|
+
end
|
186
|
+
|
187
|
+
d.run do
|
188
|
+
sleep 1
|
189
|
+
end
|
190
|
+
|
191
|
+
d.emits
|
192
|
+
ensure
|
193
|
+
file.close
|
194
|
+
end
|
195
|
+
|
196
|
+
def test_lf
|
197
|
+
File.open("#{TMP_DIR}/tail.txt", "w") {|f| }
|
198
|
+
|
199
|
+
d = create_driver
|
200
|
+
|
201
|
+
d.run do
|
202
|
+
File.open("#{TMP_DIR}/tail.txt", "a") {|f|
|
203
|
+
f.print "test3"
|
204
|
+
}
|
205
|
+
sleep 1
|
206
|
+
|
207
|
+
File.open("#{TMP_DIR}/tail.txt", "a") {|f|
|
208
|
+
f.puts "test4"
|
209
|
+
}
|
210
|
+
sleep 1
|
211
|
+
end
|
212
|
+
|
213
|
+
emits = d.emits
|
214
|
+
assert_equal(true, emits.length > 0)
|
215
|
+
assert_equal({"message" => "test3test4"}, emits[0][2])
|
216
|
+
end
|
217
|
+
|
218
|
+
def test_whitespace
|
219
|
+
File.open("#{TMP_DIR}/tail.txt", "w") {|f| }
|
220
|
+
|
221
|
+
d = create_driver
|
222
|
+
|
223
|
+
d.run do
|
224
|
+
sleep 1
|
225
|
+
|
226
|
+
File.open("#{TMP_DIR}/tail.txt", "a") {|f|
|
227
|
+
f.puts " " # 4 spaces
|
228
|
+
f.puts " 4 spaces"
|
229
|
+
f.puts "4 spaces "
|
230
|
+
f.puts " " # tab
|
231
|
+
f.puts " tab"
|
232
|
+
f.puts "tab "
|
233
|
+
}
|
234
|
+
sleep 1
|
235
|
+
end
|
236
|
+
|
237
|
+
emits = d.emits
|
238
|
+
assert_equal(true, emits.length > 0)
|
239
|
+
assert_equal({"message" => " "}, emits[0][2])
|
240
|
+
assert_equal({"message" => " 4 spaces"}, emits[1][2])
|
241
|
+
assert_equal({"message" => "4 spaces "}, emits[2][2])
|
242
|
+
assert_equal({"message" => " "}, emits[3][2])
|
243
|
+
assert_equal({"message" => " tab"}, emits[4][2])
|
244
|
+
assert_equal({"message" => "tab "}, emits[5][2])
|
245
|
+
end
|
246
|
+
|
247
|
+
# multiline mode test
|
248
|
+
|
249
|
+
def test_multiline
|
250
|
+
File.open("#{TMP_DIR}/tail.txt", "w") { |f| }
|
251
|
+
|
252
|
+
d = create_driver %[
|
253
|
+
format multiline
|
254
|
+
format1 /^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/
|
255
|
+
format_firstline /^[s]/
|
256
|
+
]
|
257
|
+
d.run do
|
258
|
+
File.open("#{TMP_DIR}/tail.txt", "a") { |f|
|
259
|
+
f.puts "f test1"
|
260
|
+
f.puts "s test2"
|
261
|
+
f.puts "f test3"
|
262
|
+
f.puts "f test4"
|
263
|
+
f.puts "s test5"
|
264
|
+
f.puts "s test6"
|
265
|
+
f.puts "f test7"
|
266
|
+
f.puts "s test8"
|
267
|
+
}
|
268
|
+
sleep 1
|
269
|
+
end
|
270
|
+
|
271
|
+
emits = d.emits
|
272
|
+
assert(emits.length > 0)
|
273
|
+
assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, emits[0][2])
|
274
|
+
assert_equal({"message1" => "test5"}, emits[1][2])
|
275
|
+
assert_equal({"message1" => "test6", "message2" => "test7"}, emits[2][2])
|
276
|
+
assert_equal({"message1" => "test8"}, emits[3][2])
|
277
|
+
end
|
278
|
+
|
279
|
+
def test_multiline_with_multiple_formats
|
280
|
+
File.open("#{TMP_DIR}/tail.txt", "w") { |f| }
|
281
|
+
|
282
|
+
d = create_driver %[
|
283
|
+
format multiline
|
284
|
+
format1 /^s (?<message1>[^\\n]+)\\n?/
|
285
|
+
format2 /(f (?<message2>[^\\n]+)\\n?)?/
|
286
|
+
format3 /(f (?<message3>.*))?/
|
287
|
+
format_firstline /^[s]/
|
288
|
+
]
|
289
|
+
d.run do
|
290
|
+
File.open("#{TMP_DIR}/tail.txt", "a") { |f|
|
291
|
+
f.puts "f test1"
|
292
|
+
f.puts "s test2"
|
293
|
+
f.puts "f test3"
|
294
|
+
f.puts "f test4"
|
295
|
+
f.puts "s test5"
|
296
|
+
f.puts "s test6"
|
297
|
+
f.puts "f test7"
|
298
|
+
f.puts "s test8"
|
299
|
+
}
|
300
|
+
sleep 1
|
301
|
+
end
|
302
|
+
|
303
|
+
emits = d.emits
|
304
|
+
assert(emits.length > 0)
|
305
|
+
assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, emits[0][2])
|
306
|
+
assert_equal({"message1" => "test5"}, emits[1][2])
|
307
|
+
assert_equal({"message1" => "test6", "message2" => "test7"}, emits[2][2])
|
308
|
+
assert_equal({"message1" => "test8"}, emits[3][2])
|
309
|
+
end
|
310
|
+
|
311
|
+
def test_multilinelog_with_multiple_paths
|
312
|
+
files = ["#{TMP_DIR}/tail1.txt", "#{TMP_DIR}/tail2.txt"]
|
313
|
+
files.each { |file| File.open(file, "w") { |f| } }
|
314
|
+
|
315
|
+
d = create_driver(%[
|
316
|
+
path #{files[0]},#{files[1]}
|
317
|
+
tag t1
|
318
|
+
format multiline
|
319
|
+
format1 /^[s|f] (?<message>.*)/
|
320
|
+
format_firstline /^[s]/
|
321
|
+
], false)
|
322
|
+
d.run do
|
323
|
+
files.each do |file|
|
324
|
+
File.open(file, 'a') { |f|
|
325
|
+
f.puts "f #{file} line should be ignored"
|
326
|
+
f.puts "s test1"
|
327
|
+
f.puts "f test2"
|
328
|
+
f.puts "f test3"
|
329
|
+
f.puts "s test4"
|
330
|
+
}
|
331
|
+
end
|
332
|
+
sleep 1
|
333
|
+
end
|
334
|
+
|
335
|
+
emits = d.emits
|
336
|
+
assert_equal({"message" => "test1\nf test2\nf test3"}, emits[0][2])
|
337
|
+
assert_equal({"message" => "test1\nf test2\nf test3"}, emits[1][2])
|
338
|
+
# "test4" events are here because these events are flushed at shutdown phase
|
339
|
+
assert_equal({"message" => "test4"}, emits[2][2])
|
340
|
+
assert_equal({"message" => "test4"}, emits[3][2])
|
341
|
+
end
|
342
|
+
|
343
|
+
def test_multiline_without_firstline
|
344
|
+
File.open("#{TMP_DIR}/tail.txt", "w") { |f| }
|
345
|
+
|
346
|
+
d = create_driver %[
|
347
|
+
format multiline
|
348
|
+
format1 /(?<var1>foo \\d)\\n/
|
349
|
+
format2 /(?<var2>bar \\d)\\n/
|
350
|
+
format3 /(?<var3>baz \\d)/
|
351
|
+
]
|
352
|
+
d.run do
|
353
|
+
File.open("#{TMP_DIR}/tail.txt", "a") { |f|
|
354
|
+
f.puts "foo 1"
|
355
|
+
f.puts "bar 1"
|
356
|
+
f.puts "baz 1"
|
357
|
+
f.puts "foo 2"
|
358
|
+
f.puts "bar 2"
|
359
|
+
f.puts "baz 2"
|
360
|
+
}
|
361
|
+
sleep 1
|
362
|
+
end
|
363
|
+
|
364
|
+
emits = d.emits
|
365
|
+
assert_equal(2, emits.length)
|
366
|
+
assert_equal({"var1" => "foo 1", "var2" => "bar 1", "var3" => "baz 1"}, emits[0][2])
|
367
|
+
assert_equal({"var1" => "foo 2", "var2" => "bar 2", "var3" => "baz 2"}, emits[1][2])
|
368
|
+
end
|
369
|
+
|
370
|
+
# * path test
|
371
|
+
# TODO: Clean up tests
|
372
|
+
EX_RORATE_WAIT = 0
|
373
|
+
|
374
|
+
EX_CONFIG = %[
|
375
|
+
tag tail
|
376
|
+
path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
|
377
|
+
format none
|
378
|
+
pos_file #{TMP_DIR}/tail.pos
|
379
|
+
read_from_head true
|
380
|
+
refresh_interval 30
|
381
|
+
rotate_wait #{EX_RORATE_WAIT}s
|
382
|
+
]
|
383
|
+
EX_PATHS = [
|
384
|
+
'test/plugin/data/2010/01/20100102-030405.log',
|
385
|
+
'test/plugin/data/log/foo/bar.log',
|
386
|
+
'test/plugin/data/log/test.log'
|
387
|
+
]
|
388
|
+
|
389
|
+
EX_CONFIG_FOR_ROTATE = %[
|
390
|
+
tag tail
|
391
|
+
path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
|
392
|
+
format none
|
393
|
+
pos_file #{TMP_DIR}/tail.pos
|
394
|
+
read_from_head true
|
395
|
+
refresh_interval 30
|
396
|
+
rotate_wait #{EX_RORATE_WAIT}s
|
397
|
+
expand_rotate_time 18000s
|
398
|
+
]
|
399
|
+
|
400
|
+
EX_PATHS_FOR_ROATE = [
|
401
|
+
'test/plugin/data/2014/12/20141224-000000.log',
|
402
|
+
'test/plugin/data/log/foo/bar.log',
|
403
|
+
'test/plugin/data/log/test.log'
|
404
|
+
]
|
405
|
+
|
406
|
+
def test_expand_paths
|
407
|
+
plugin = create_driver(EX_CONFIG, false).instance
|
408
|
+
flexstub(Time) do |timeclass|
|
409
|
+
timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 2, 3, 4, 5))
|
410
|
+
assert_equal EX_PATHS, plugin.expand_paths.sort
|
411
|
+
end
|
412
|
+
end
|
413
|
+
|
414
|
+
def test_expand_paths_for_ex_rotate
|
415
|
+
plugin = create_driver(EX_CONFIG_FOR_ROTATE, false).instance
|
416
|
+
flexstub(Time) do |timeclass|
|
417
|
+
timeclass.should_receive(:now).with_no_args.and_return(Time.new(2014, 12, 24, 5, 0, 0))
|
418
|
+
assert_equal EX_PATHS_FOR_ROATE, plugin.expand_paths.sort
|
419
|
+
end
|
420
|
+
end
|
421
|
+
|
422
|
+
def test_refresh_watchers
|
423
|
+
plugin = create_driver(EX_CONFIG, false).instance
|
424
|
+
sio = StringIO.new
|
425
|
+
plugin.instance_eval do
|
426
|
+
@pf = Fluent::TailExRotateInput::PositionFile.parse(sio)
|
427
|
+
@loop = Coolio::Loop.new
|
428
|
+
end
|
429
|
+
|
430
|
+
flexstub(Time) do |timeclass|
|
431
|
+
timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 2, 3, 4, 5), Time.new(2010, 1, 2, 3, 4, 6), Time.new(2010, 1, 2, 3, 4, 7))
|
432
|
+
|
433
|
+
flexstub(Fluent::TailExRotateInput::TailWatcher) do |watcherclass|
|
434
|
+
EX_PATHS.each do |path|
|
435
|
+
watcherclass.should_receive(:new).with(path, EX_RORATE_WAIT, Fluent::TailExRotateInput::FilePositionEntry, any, true, any, any).once.and_return do
|
436
|
+
flexmock('TailWatcher') { |watcher|
|
437
|
+
watcher.should_receive(:attach).once
|
438
|
+
watcher.should_receive(:unwatched=).zero_or_more_times
|
439
|
+
watcher.should_receive(:line_buffer).zero_or_more_times
|
440
|
+
}
|
441
|
+
end
|
442
|
+
end
|
443
|
+
plugin.refresh_watchers
|
444
|
+
end
|
445
|
+
|
446
|
+
plugin.instance_eval do
|
447
|
+
@tails['test/plugin/data/2010/01/20100102-030405.log'].should_receive(:close).zero_or_more_times
|
448
|
+
end
|
449
|
+
|
450
|
+
flexstub(Fluent::TailExRotateInput::TailWatcher) do |watcherclass|
|
451
|
+
watcherclass.should_receive(:new).with('test/plugin/data/2010/01/20100102-030406.log', EX_RORATE_WAIT, Fluent::TailExRotateInput::FilePositionEntry, any, true, any, any).once.and_return do
|
452
|
+
flexmock('TailWatcher') do |watcher|
|
453
|
+
watcher.should_receive(:attach).once
|
454
|
+
watcher.should_receive(:unwatched=).zero_or_more_times
|
455
|
+
watcher.should_receive(:line_buffer).zero_or_more_times
|
456
|
+
end
|
457
|
+
end
|
458
|
+
plugin.refresh_watchers
|
459
|
+
end
|
460
|
+
|
461
|
+
flexstub(Fluent::TailExRotateInput::TailWatcher) do |watcherclass|
|
462
|
+
watcherclass.should_receive(:new).never
|
463
|
+
plugin.refresh_watchers
|
464
|
+
end
|
465
|
+
end
|
466
|
+
end
|
467
|
+
|
468
|
+
DummyWatcher = Struct.new("DummyWatcher", :tag)
|
469
|
+
|
470
|
+
def test_receive_lines
|
471
|
+
plugin = create_driver(EX_CONFIG, false).instance
|
472
|
+
flexstub(plugin.router) do |engineclass|
|
473
|
+
engineclass.should_receive(:emit_stream).with('tail', any).once
|
474
|
+
plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
|
475
|
+
end
|
476
|
+
|
477
|
+
config = %[
|
478
|
+
tag pre.*
|
479
|
+
path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
|
480
|
+
format none
|
481
|
+
read_from_head true
|
482
|
+
]
|
483
|
+
plugin = create_driver(config, false).instance
|
484
|
+
flexstub(plugin.router) do |engineclass|
|
485
|
+
engineclass.should_receive(:emit_stream).with('pre.foo.bar.log', any).once
|
486
|
+
plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
|
487
|
+
end
|
488
|
+
|
489
|
+
config = %[
|
490
|
+
tag *.post
|
491
|
+
path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
|
492
|
+
format none
|
493
|
+
read_from_head true
|
494
|
+
]
|
495
|
+
plugin = create_driver(config, false).instance
|
496
|
+
flexstub(plugin.router) do |engineclass|
|
497
|
+
engineclass.should_receive(:emit_stream).with('foo.bar.log.post', any).once
|
498
|
+
plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
|
499
|
+
end
|
500
|
+
|
501
|
+
config = %[
|
502
|
+
tag pre.*.post
|
503
|
+
path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
|
504
|
+
format none
|
505
|
+
read_from_head true
|
506
|
+
]
|
507
|
+
plugin = create_driver(config, false).instance
|
508
|
+
flexstub(plugin.router) do |engineclass|
|
509
|
+
engineclass.should_receive(:emit_stream).with('pre.foo.bar.log.post', any).once
|
510
|
+
plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
|
511
|
+
end
|
512
|
+
|
513
|
+
config = %[
|
514
|
+
tag pre.*.post*ignore
|
515
|
+
path test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log
|
516
|
+
format none
|
517
|
+
read_from_head true
|
518
|
+
]
|
519
|
+
plugin = create_driver(config, false).instance
|
520
|
+
flexstub(plugin.router) do |engineclass|
|
521
|
+
engineclass.should_receive(:emit_stream).with('pre.foo.bar.log.post', any).once
|
522
|
+
plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
|
523
|
+
end
|
524
|
+
end
|
525
|
+
|
526
|
+
# Ensure that no fatal exception is raised when a file is missing and that
|
527
|
+
# files that do exist are still tailed as expected.
|
528
|
+
def test_missing_file
|
529
|
+
File.open("#{TMP_DIR}/tail.txt", "w") {|f|
|
530
|
+
f.puts "test1"
|
531
|
+
f.puts "test2"
|
532
|
+
}
|
533
|
+
|
534
|
+
# Try two different configs - one with read_from_head and one without,
|
535
|
+
# since their interactions with the filesystem differ.
|
536
|
+
config1 = %[
|
537
|
+
tag t1
|
538
|
+
path #{TMP_DIR}/non_existent_file.txt,#{TMP_DIR}/tail.txt
|
539
|
+
format none
|
540
|
+
rotate_wait 2s
|
541
|
+
pos_file #{TMP_DIR}/tail.pos
|
542
|
+
]
|
543
|
+
config2 = config1 + ' read_from_head true'
|
544
|
+
[config1, config2].each do |config|
|
545
|
+
d = create_driver(config, false)
|
546
|
+
d.run do
|
547
|
+
sleep 1
|
548
|
+
File.open("#{TMP_DIR}/tail.txt", "a") {|f|
|
549
|
+
f.puts "test3"
|
550
|
+
f.puts "test4"
|
551
|
+
}
|
552
|
+
sleep 1
|
553
|
+
end
|
554
|
+
emits = d.emits
|
555
|
+
assert_equal(2, emits.length)
|
556
|
+
assert_equal({"message" => "test3"}, emits[0][2])
|
557
|
+
assert_equal({"message" => "test4"}, emits[1][2])
|
558
|
+
end
|
559
|
+
end
|
560
|
+
end
|
metadata
ADDED
@@ -0,0 +1,112 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-tail-ex-rotate
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Yuta Mizushima
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2015-01-23 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: fluentd
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: bundler
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '1.3'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '1.3'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rake
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 0.9.2
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 0.9.2
|
55
|
+
description: Extension of in_tail plugin to customize log rotate timing.
|
56
|
+
email:
|
57
|
+
- yuta_mizushima@dwango.co.jp
|
58
|
+
executables: []
|
59
|
+
extensions: []
|
60
|
+
extra_rdoc_files: []
|
61
|
+
files:
|
62
|
+
- COPYING
|
63
|
+
- Gemfile
|
64
|
+
- Gemfile.lock
|
65
|
+
- NOTICE
|
66
|
+
- README.md
|
67
|
+
- Rakefile
|
68
|
+
- fluent-plugin-tail-ex-rotate.gemspec
|
69
|
+
- lib/fluent/plugin/in_tail_ex_rotate.rb
|
70
|
+
- lib/fluent/version.rb
|
71
|
+
- pkg/fluent-plugin-tail-ex-rotate-0.0.1.gem
|
72
|
+
- test/helper.rb
|
73
|
+
- test/plugin/data/2010/01/20100102-030405.log
|
74
|
+
- test/plugin/data/2010/01/20100102-030406.log
|
75
|
+
- test/plugin/data/2014/12/20141224-000000.log
|
76
|
+
- test/plugin/data/log/bar
|
77
|
+
- test/plugin/data/log/foo/bar.log
|
78
|
+
- test/plugin/data/log/test.log
|
79
|
+
- test/plugin/test_in_tail_ex_rotate.rb
|
80
|
+
homepage: http://github.o-in.dwango.co.jp/Yuta-Mizushima/namasasy_setup/tree/master/node/fluent-plugin-tail-ex-rotate
|
81
|
+
licenses: []
|
82
|
+
metadata: {}
|
83
|
+
post_install_message:
|
84
|
+
rdoc_options: []
|
85
|
+
require_paths:
|
86
|
+
- lib
|
87
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
88
|
+
requirements:
|
89
|
+
- - ">="
|
90
|
+
- !ruby/object:Gem::Version
|
91
|
+
version: 1.9.3
|
92
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
97
|
+
requirements: []
|
98
|
+
rubyforge_project:
|
99
|
+
rubygems_version: 2.2.2
|
100
|
+
signing_key:
|
101
|
+
specification_version: 4
|
102
|
+
summary: customize log rotate timing
|
103
|
+
test_files:
|
104
|
+
- test/helper.rb
|
105
|
+
- test/plugin/data/2010/01/20100102-030405.log
|
106
|
+
- test/plugin/data/2010/01/20100102-030406.log
|
107
|
+
- test/plugin/data/2014/12/20141224-000000.log
|
108
|
+
- test/plugin/data/log/bar
|
109
|
+
- test/plugin/data/log/foo/bar.log
|
110
|
+
- test/plugin/data/log/test.log
|
111
|
+
- test/plugin/test_in_tail_ex_rotate.rb
|
112
|
+
has_rdoc: false
|