fluentd 0.10.16 → 0.10.17
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- data/ChangeLog +10 -0
- data/VERSION +1 -1
- data/lib/fluent/mixin.rb +26 -1
- data/lib/fluent/plugin/in_tail.rb +15 -6
- data/lib/fluent/version.rb +1 -1
- data/test/mixin.rb +104 -0
- data/test/plugin/in_tail.rb +79 -0
- metadata +22 -20
data/ChangeLog
CHANGED
@@ -1,4 +1,14 @@
|
|
1
1
|
|
2
|
+
Release 0.10.17 - 2012/04/02
|
3
|
+
|
4
|
+
* in_tail: don't ignore last line without LF [#44]
|
5
|
+
* Added HandleTagNameMixin
|
6
|
+
* Removed automake/conf style build support
|
7
|
+
Maintainig automake/conf style files are complex, but no user. We've decided
|
8
|
+
not to have those files anymore. Otherwise it's still used at rpm/deb packaging
|
9
|
+
scripts.
|
10
|
+
|
11
|
+
|
2
12
|
Release 0.10.16 - 2012/03/26
|
3
13
|
|
4
14
|
* Added in_gc plugin
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.10.
|
1
|
+
0.10.17
|
data/lib/fluent/mixin.rb
CHANGED
@@ -100,6 +100,32 @@ module RecordFilterMixin
|
|
100
100
|
end
|
101
101
|
end
|
102
102
|
|
103
|
+
module HandleTagNameMixin
|
104
|
+
include RecordFilterMixin
|
105
|
+
|
106
|
+
attr_accessor :remove_tag_prefix, :remove_tag_suffix, :add_tag_prefix, :add_tag_suffix
|
107
|
+
def configure(conf)
|
108
|
+
super
|
109
|
+
if remove_tag_prefix = conf['remove_tag_prefix']
|
110
|
+
@remove_tag_prefix = Regexp.new('^' + Regexp.escape(remove_tag_prefix))
|
111
|
+
end
|
112
|
+
|
113
|
+
if remove_tag_suffix = conf['remove_tag_suffix']
|
114
|
+
@remove_tag_suffix = Regexp.new(Regexp.escape(remove_tag_suffix) + '$')
|
115
|
+
end
|
116
|
+
|
117
|
+
@add_tag_prefix = conf['add_tag_prefix']
|
118
|
+
@add_tag_suffix = conf['add_tag_suffix']
|
119
|
+
end
|
120
|
+
|
121
|
+
def filter_record(tag, time, record)
|
122
|
+
tag.sub!(@remove_tag_prefix, '') if @remove_tag_prefix
|
123
|
+
tag.sub!(@remove_tag_suffix, '') if @remove_tag_suffix
|
124
|
+
tag.insert(0, @add_tag_prefix) if @add_tag_prefix
|
125
|
+
tag << @add_tag_suffix if @add_tag_suffix
|
126
|
+
super(tag, time, record)
|
127
|
+
end
|
128
|
+
end
|
103
129
|
|
104
130
|
module SetTimeKeyMixin
|
105
131
|
include RecordFilterMixin
|
@@ -152,7 +178,6 @@ module SetTimeKeyMixin
|
|
152
178
|
end
|
153
179
|
end
|
154
180
|
|
155
|
-
|
156
181
|
module SetTagKeyMixin
|
157
182
|
include RecordFilterMixin
|
158
183
|
|
@@ -31,6 +31,8 @@ class TailInput < Input
|
|
31
31
|
config_param :rotate_wait, :time, :default => 5
|
32
32
|
config_param :pos_file, :string, :default => nil
|
33
33
|
|
34
|
+
attr_reader :paths
|
35
|
+
|
34
36
|
def configure(conf)
|
35
37
|
super
|
36
38
|
|
@@ -260,6 +262,7 @@ class TailInput < Input
|
|
260
262
|
@pe = pe
|
261
263
|
@receive_lines = receive_lines
|
262
264
|
@buffer = ''
|
265
|
+
@iobuf = ''
|
263
266
|
end
|
264
267
|
|
265
268
|
attr_reader :io
|
@@ -267,18 +270,24 @@ class TailInput < Input
|
|
267
270
|
def on_notify
|
268
271
|
lines = []
|
269
272
|
|
270
|
-
while
|
271
|
-
|
272
|
-
|
273
|
-
|
273
|
+
while true
|
274
|
+
begin
|
275
|
+
if @buffer.empty?
|
276
|
+
@io.read_nonblock(2048, @buffer)
|
277
|
+
else
|
278
|
+
@buffer << @io.read_nonblock(2048, @iobuf)
|
279
|
+
end
|
280
|
+
while line = @buffer.slice!(/.*?\n/m)
|
281
|
+
lines << line
|
282
|
+
end
|
283
|
+
rescue EOFError
|
274
284
|
break
|
275
285
|
end
|
276
|
-
lines << line
|
277
286
|
end
|
278
287
|
|
279
288
|
unless lines.empty?
|
280
|
-
@pe.update_pos(@pos)
|
281
289
|
@receive_lines.call(lines)
|
290
|
+
@pe.update_pos(@io.pos - @buffer.bytesize)
|
282
291
|
end
|
283
292
|
rescue
|
284
293
|
$log.error $!.to_s
|
data/lib/fluent/version.rb
CHANGED
data/test/mixin.rb
CHANGED
@@ -155,4 +155,108 @@ module MixinTest
|
|
155
155
|
d.run
|
156
156
|
end
|
157
157
|
end
|
158
|
+
|
159
|
+
class HandleTagMixinTest < Test::Unit::TestCase
|
160
|
+
include Utils
|
161
|
+
|
162
|
+
def test_add_tag_prefix
|
163
|
+
format_check({
|
164
|
+
'a' => 1
|
165
|
+
}, 'tag_prefix.test')
|
166
|
+
|
167
|
+
d = create_driver(Fluent::HandleTagNameMixin, %[
|
168
|
+
add_tag_prefix tag_prefix.
|
169
|
+
include_tag_key true
|
170
|
+
])
|
171
|
+
|
172
|
+
d.emit({'a' => 1})
|
173
|
+
d.run
|
174
|
+
end
|
175
|
+
|
176
|
+
def test_add_tag_suffix
|
177
|
+
format_check({
|
178
|
+
'a' => 1
|
179
|
+
}, 'test.test_suffix')
|
180
|
+
|
181
|
+
d = create_driver(Fluent::HandleTagNameMixin, %[
|
182
|
+
add_tag_suffix .test_suffix
|
183
|
+
include_tag_key true
|
184
|
+
])
|
185
|
+
|
186
|
+
d.emit({'a' => 1})
|
187
|
+
d.run
|
188
|
+
end
|
189
|
+
|
190
|
+
def test_remove_tag_prefix
|
191
|
+
format_check({
|
192
|
+
'a' => 1
|
193
|
+
}, 'st')
|
194
|
+
|
195
|
+
d = create_driver(Fluent::HandleTagNameMixin, %[
|
196
|
+
remove_tag_prefix te
|
197
|
+
include_tag_key true
|
198
|
+
])
|
199
|
+
|
200
|
+
d.emit({'a' => 1})
|
201
|
+
d.run
|
202
|
+
end
|
203
|
+
|
204
|
+
def test_remove_tag_suffix
|
205
|
+
format_check({
|
206
|
+
'a' => 1
|
207
|
+
}, 'te')
|
208
|
+
|
209
|
+
d = create_driver(Fluent::HandleTagNameMixin, %[
|
210
|
+
remove_tag_suffix st
|
211
|
+
include_tag_key true
|
212
|
+
])
|
213
|
+
|
214
|
+
d.emit({'a' => 1})
|
215
|
+
d.run
|
216
|
+
end
|
217
|
+
|
218
|
+
def test_mix_tag_handle
|
219
|
+
format_check({
|
220
|
+
'a' => 1
|
221
|
+
}, 'prefix.t')
|
222
|
+
|
223
|
+
d = create_driver(Fluent::HandleTagNameMixin, %[
|
224
|
+
remove_tag_prefix tes
|
225
|
+
add_tag_prefix prefix.
|
226
|
+
])
|
227
|
+
|
228
|
+
d.emit({'a' => 1})
|
229
|
+
d.run
|
230
|
+
end
|
231
|
+
|
232
|
+
def test_with_set_tag_key_mixin
|
233
|
+
format_check({
|
234
|
+
'tag' => 'tag_prefix.test',
|
235
|
+
'a' => 1
|
236
|
+
}, 'tag_prefix.test')
|
237
|
+
|
238
|
+
d = create_driver([Fluent::SetTagKeyMixin, Fluent::HandleTagNameMixin], %[
|
239
|
+
add_tag_prefix tag_prefix.
|
240
|
+
include_tag_key true
|
241
|
+
])
|
242
|
+
|
243
|
+
d.emit({'a' => 1})
|
244
|
+
d.run
|
245
|
+
end
|
246
|
+
|
247
|
+
def test_with_set_tag_key_mixin_include_order_reverse
|
248
|
+
format_check({
|
249
|
+
'tag' => 'tag_prefix.test',
|
250
|
+
'a' => 1
|
251
|
+
}, 'tag_prefix.test')
|
252
|
+
|
253
|
+
d = create_driver([Fluent::HandleTagNameMixin, Fluent::SetTagKeyMixin], %[
|
254
|
+
add_tag_prefix tag_prefix.
|
255
|
+
include_tag_key true
|
256
|
+
])
|
257
|
+
|
258
|
+
d.emit({'a' => 1})
|
259
|
+
d.run
|
260
|
+
end
|
261
|
+
end
|
158
262
|
end
|
@@ -0,0 +1,79 @@
|
|
1
|
+
require 'fluent/test'
|
2
|
+
require 'net/http'
|
3
|
+
|
4
|
+
class TailInputTest < Test::Unit::TestCase
|
5
|
+
def setup
|
6
|
+
Fluent::Test.setup
|
7
|
+
FileUtils.rm_rf(TMP_DIR)
|
8
|
+
FileUtils.mkdir_p(TMP_DIR)
|
9
|
+
end
|
10
|
+
|
11
|
+
TMP_DIR = File.dirname(__FILE__) + "/../tmp"
|
12
|
+
|
13
|
+
CONFIG = %[
|
14
|
+
path #{TMP_DIR}/tail.txt
|
15
|
+
tag t1
|
16
|
+
rotate_wait 2s
|
17
|
+
pos_file #{TMP_DIR}/tail.pos
|
18
|
+
format /(?<message>.*)/
|
19
|
+
]
|
20
|
+
|
21
|
+
def create_driver(conf=CONFIG)
|
22
|
+
Fluent::Test::InputTestDriver.new(Fluent::TailInput).configure(conf)
|
23
|
+
end
|
24
|
+
|
25
|
+
def test_configure
|
26
|
+
d = create_driver
|
27
|
+
assert_equal ["#{TMP_DIR}/tail.txt"], d.instance.paths
|
28
|
+
assert_equal "t1", d.instance.tag
|
29
|
+
assert_equal 2, d.instance.rotate_wait
|
30
|
+
assert_equal "#{TMP_DIR}/tail.pos", d.instance.pos_file
|
31
|
+
end
|
32
|
+
|
33
|
+
def test_emit
|
34
|
+
File.open("#{TMP_DIR}/tail.txt", "w") {|f|
|
35
|
+
f.puts "test1"
|
36
|
+
f.puts "test2"
|
37
|
+
}
|
38
|
+
|
39
|
+
d = create_driver
|
40
|
+
|
41
|
+
d.run do
|
42
|
+
sleep 1
|
43
|
+
|
44
|
+
File.open("#{TMP_DIR}/tail.txt", "a") {|f|
|
45
|
+
f.puts "test3"
|
46
|
+
f.puts "test4"
|
47
|
+
}
|
48
|
+
sleep 1
|
49
|
+
end
|
50
|
+
|
51
|
+
emits = d.emits
|
52
|
+
assert_equal(true, emits.length > 0)
|
53
|
+
assert_equal({"message"=>"test3"}, emits[0][2])
|
54
|
+
assert_equal({"message"=>"test4"}, emits[1][2])
|
55
|
+
end
|
56
|
+
|
57
|
+
def test_lf
|
58
|
+
File.open("#{TMP_DIR}/tail.txt", "w") {|f| }
|
59
|
+
|
60
|
+
d = create_driver
|
61
|
+
|
62
|
+
d.run do
|
63
|
+
File.open("#{TMP_DIR}/tail.txt", "a") {|f|
|
64
|
+
f.print "test3"
|
65
|
+
}
|
66
|
+
sleep 1
|
67
|
+
|
68
|
+
File.open("#{TMP_DIR}/tail.txt", "a") {|f|
|
69
|
+
f.puts "test4"
|
70
|
+
}
|
71
|
+
sleep 1
|
72
|
+
end
|
73
|
+
|
74
|
+
emits = d.emits
|
75
|
+
assert_equal(true, emits.length > 0)
|
76
|
+
assert_equal({"message"=>"test3test4"}, emits[0][2])
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluentd
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.10.
|
4
|
+
version: 0.10.17
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,11 +9,11 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-
|
12
|
+
date: 2012-04-02 00:00:00.000000000Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: msgpack
|
16
|
-
requirement: &
|
16
|
+
requirement: &70102411888840 !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - ~>
|
@@ -21,10 +21,10 @@ dependencies:
|
|
21
21
|
version: 0.4.4
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
|
-
version_requirements: *
|
24
|
+
version_requirements: *70102411888840
|
25
25
|
- !ruby/object:Gem::Dependency
|
26
26
|
name: json
|
27
|
-
requirement: &
|
27
|
+
requirement: &70102411888180 !ruby/object:Gem::Requirement
|
28
28
|
none: false
|
29
29
|
requirements:
|
30
30
|
- - ! '>='
|
@@ -32,10 +32,10 @@ dependencies:
|
|
32
32
|
version: 1.4.3
|
33
33
|
type: :runtime
|
34
34
|
prerelease: false
|
35
|
-
version_requirements: *
|
35
|
+
version_requirements: *70102411888180
|
36
36
|
- !ruby/object:Gem::Dependency
|
37
37
|
name: yajl-ruby
|
38
|
-
requirement: &
|
38
|
+
requirement: &70102411887560 !ruby/object:Gem::Requirement
|
39
39
|
none: false
|
40
40
|
requirements:
|
41
41
|
- - ~>
|
@@ -43,10 +43,10 @@ dependencies:
|
|
43
43
|
version: '1.0'
|
44
44
|
type: :runtime
|
45
45
|
prerelease: false
|
46
|
-
version_requirements: *
|
46
|
+
version_requirements: *70102411887560
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
48
|
name: cool.io
|
49
|
-
requirement: &
|
49
|
+
requirement: &70102411886940 !ruby/object:Gem::Requirement
|
50
50
|
none: false
|
51
51
|
requirements:
|
52
52
|
- - ~>
|
@@ -54,10 +54,10 @@ dependencies:
|
|
54
54
|
version: 1.1.0
|
55
55
|
type: :runtime
|
56
56
|
prerelease: false
|
57
|
-
version_requirements: *
|
57
|
+
version_requirements: *70102411886940
|
58
58
|
- !ruby/object:Gem::Dependency
|
59
59
|
name: http_parser.rb
|
60
|
-
requirement: &
|
60
|
+
requirement: &70102411886280 !ruby/object:Gem::Requirement
|
61
61
|
none: false
|
62
62
|
requirements:
|
63
63
|
- - ~>
|
@@ -65,10 +65,10 @@ dependencies:
|
|
65
65
|
version: 0.5.1
|
66
66
|
type: :runtime
|
67
67
|
prerelease: false
|
68
|
-
version_requirements: *
|
68
|
+
version_requirements: *70102411886280
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: rake
|
71
|
-
requirement: &
|
71
|
+
requirement: &70102411885660 !ruby/object:Gem::Requirement
|
72
72
|
none: false
|
73
73
|
requirements:
|
74
74
|
- - ! '>='
|
@@ -76,10 +76,10 @@ dependencies:
|
|
76
76
|
version: 0.9.2
|
77
77
|
type: :development
|
78
78
|
prerelease: false
|
79
|
-
version_requirements: *
|
79
|
+
version_requirements: *70102411885660
|
80
80
|
- !ruby/object:Gem::Dependency
|
81
81
|
name: rr
|
82
|
-
requirement: &
|
82
|
+
requirement: &70102411885040 !ruby/object:Gem::Requirement
|
83
83
|
none: false
|
84
84
|
requirements:
|
85
85
|
- - ! '>='
|
@@ -87,10 +87,10 @@ dependencies:
|
|
87
87
|
version: 1.0.0
|
88
88
|
type: :development
|
89
89
|
prerelease: false
|
90
|
-
version_requirements: *
|
90
|
+
version_requirements: *70102411885040
|
91
91
|
- !ruby/object:Gem::Dependency
|
92
92
|
name: timecop
|
93
|
-
requirement: &
|
93
|
+
requirement: &70102411884420 !ruby/object:Gem::Requirement
|
94
94
|
none: false
|
95
95
|
requirements:
|
96
96
|
- - ! '>='
|
@@ -98,10 +98,10 @@ dependencies:
|
|
98
98
|
version: 0.3.0
|
99
99
|
type: :development
|
100
100
|
prerelease: false
|
101
|
-
version_requirements: *
|
101
|
+
version_requirements: *70102411884420
|
102
102
|
- !ruby/object:Gem::Dependency
|
103
103
|
name: jeweler
|
104
|
-
requirement: &
|
104
|
+
requirement: &70102411883800 !ruby/object:Gem::Requirement
|
105
105
|
none: false
|
106
106
|
requirements:
|
107
107
|
- - ! '>='
|
@@ -109,7 +109,7 @@ dependencies:
|
|
109
109
|
version: 1.0.0
|
110
110
|
type: :development
|
111
111
|
prerelease: false
|
112
|
-
version_requirements: *
|
112
|
+
version_requirements: *70102411883800
|
113
113
|
description:
|
114
114
|
email: frsyuki@gmail.com
|
115
115
|
executables:
|
@@ -185,6 +185,7 @@ files:
|
|
185
185
|
- test/plugin/in_http.rb
|
186
186
|
- test/plugin/in_object_space.rb
|
187
187
|
- test/plugin/in_stream.rb
|
188
|
+
- test/plugin/in_tail.rb
|
188
189
|
- test/plugin/out_copy.rb
|
189
190
|
- test/plugin/out_exec.rb
|
190
191
|
- test/plugin/out_exec_filter.rb
|
@@ -229,6 +230,7 @@ test_files:
|
|
229
230
|
- test/plugin/in_http.rb
|
230
231
|
- test/plugin/in_object_space.rb
|
231
232
|
- test/plugin/in_stream.rb
|
233
|
+
- test/plugin/in_tail.rb
|
232
234
|
- test/plugin/out_copy.rb
|
233
235
|
- test/plugin/out_exec.rb
|
234
236
|
- test/plugin/out_exec_filter.rb
|