fluentd 0.10.12 → 0.10.13
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- data/ChangeLog +6 -1
- data/VERSION +1 -1
- data/lib/fluent/engine.rb +1 -1
- data/lib/fluent/output.rb +5 -0
- data/lib/fluent/plugin/in_tail.rb +189 -168
- data/lib/fluent/version.rb +1 -1
- metadata +14 -14
data/ChangeLog
CHANGED
@@ -1,10 +1,15 @@
|
|
1
1
|
|
2
|
+
Release 0.10.13 - 2012/02/21
|
3
|
+
|
4
|
+
* Rewrote in_tail
|
5
|
+
* Fixed SIGUSR1 handler to force flush logs
|
6
|
+
|
7
|
+
|
2
8
|
Release 0.10.12 - 2012/02/13
|
3
9
|
|
4
10
|
* Engine shows warnings when emitted record doesn't match any outputs
|
5
11
|
* in_tail is rewritten to follow symbolic links correctly
|
6
12
|
* out_forward uses independent default value as 'hard_timeout' parameter
|
7
|
-
*
|
8
13
|
|
9
14
|
|
10
15
|
Release 0.10.11 - 2012/02/10
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.10.
|
1
|
+
0.10.13
|
data/lib/fluent/engine.rb
CHANGED
data/lib/fluent/output.rb
CHANGED
@@ -58,15 +58,20 @@ class TailInput < Input
|
|
58
58
|
|
59
59
|
def start
|
60
60
|
@loop = Coolio::Loop.new
|
61
|
-
@
|
61
|
+
@tails = @paths.map {|path|
|
62
62
|
pe = @pf ? @pf[path] : NullPositionEntry.instance
|
63
|
-
|
63
|
+
TailWatcher.new(path, @rotate_wait, pe, &method(:receive_lines))
|
64
|
+
}
|
65
|
+
@tails.each {|tail|
|
66
|
+
tail.attach(@loop)
|
64
67
|
}
|
65
68
|
@thread = Thread.new(&method(:run))
|
66
69
|
end
|
67
70
|
|
68
71
|
def shutdown
|
69
|
-
@
|
72
|
+
@tails.each {|tail|
|
73
|
+
tail.close
|
74
|
+
}
|
70
75
|
@loop.stop
|
71
76
|
@thread.join
|
72
77
|
@pf_file.close if @pf_file
|
@@ -103,229 +108,245 @@ class TailInput < Input
|
|
103
108
|
return @parser.parse(line)
|
104
109
|
end
|
105
110
|
|
106
|
-
class
|
107
|
-
def initialize(
|
108
|
-
@loop = loop
|
111
|
+
class TailWatcher
|
112
|
+
def initialize(path, rotate_wait, pe, &receive_lines)
|
109
113
|
@path = path
|
110
114
|
@rotate_wait = rotate_wait
|
111
|
-
@pe = pe
|
115
|
+
@pe = pe || NullPositionEntry.instance
|
112
116
|
@receive_lines = receive_lines
|
113
117
|
|
114
118
|
@rotate_queue = []
|
115
|
-
@rotate_timer = nil
|
116
|
-
@io_handler = nil
|
117
119
|
|
118
|
-
@
|
119
|
-
@
|
120
|
-
@rh.on_rotate = method(:on_rotate)
|
121
|
-
@rh.attach(@loop)
|
122
|
-
end
|
120
|
+
@timer_trigger = TimerWatcher.new(1, true, &method(:on_notify))
|
121
|
+
@stat_trigger = StatWatcher.new(path, &method(:on_notify))
|
123
122
|
|
124
|
-
|
125
|
-
|
126
|
-
$log.info "detected rotation of #{@path}; waiting #{@rotate_wait} seconds"
|
127
|
-
@rotate_queue.push(io)
|
128
|
-
|
129
|
-
# start rotate_timer
|
130
|
-
unless @rotate_timer
|
131
|
-
@rotate_timer = RotateTimer.new(@rotate_wait, method(:on_rotate_timer))
|
132
|
-
@rotate_timer.attach(@loop)
|
133
|
-
end
|
123
|
+
@rotate_handler = RotateHandler.new(path, &method(:on_rotate))
|
124
|
+
@io_handler = nil
|
134
125
|
end
|
135
126
|
|
136
|
-
def
|
137
|
-
|
138
|
-
|
127
|
+
def attach(loop)
|
128
|
+
@timer_trigger.attach(loop)
|
129
|
+
@stat_trigger.attach(loop)
|
130
|
+
on_notify
|
139
131
|
end
|
140
132
|
|
141
|
-
def
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
if @io_handler
|
146
|
-
@io_handler.close
|
147
|
-
@io_handler = nil
|
148
|
-
end
|
149
|
-
io_handler.attach(@loop)
|
150
|
-
@io_handler = io_handler
|
151
|
-
@rotate_queue.shift
|
152
|
-
|
153
|
-
if @rotate_queue.empty?
|
154
|
-
@rotate_timer.detach if @rotate_timer
|
155
|
-
@rotate_timer = nil
|
156
|
-
end
|
133
|
+
def detach
|
134
|
+
@timer_trigger.detach if @timer_trigger.attached?
|
135
|
+
@stat_trigger.detach if @stat_trigger.attached?
|
157
136
|
end
|
158
137
|
|
159
|
-
def
|
160
|
-
@rotate_queue.reject! {|
|
161
|
-
io.close
|
138
|
+
def close
|
139
|
+
@rotate_queue.reject! {|req|
|
140
|
+
req.io.close
|
162
141
|
true
|
163
142
|
}
|
164
|
-
|
165
|
-
@io_handler.close
|
166
|
-
@io_handler = nil
|
167
|
-
end
|
168
|
-
if @rotate_timer
|
169
|
-
@rotate_timer.detach
|
170
|
-
@rotate_timer = nil
|
171
|
-
end
|
143
|
+
detach
|
172
144
|
end
|
173
|
-
end
|
174
145
|
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
@
|
179
|
-
@fsize = 0
|
180
|
-
@on_rotate = on_rotate
|
181
|
-
@path = path
|
182
|
-
@stat_watcher = Stat.new(self, @path)
|
183
|
-
@timer_watcher = Timer.new(self, 1)
|
184
|
-
end
|
146
|
+
def on_notify
|
147
|
+
@rotate_handler.on_notify
|
148
|
+
return unless @io_handler
|
149
|
+
@io_handler.on_notify
|
185
150
|
|
186
|
-
|
151
|
+
# proceeds rotate queue
|
152
|
+
return if @rotate_queue.empty?
|
153
|
+
@rotate_queue.first.tick
|
187
154
|
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
@
|
195
|
-
|
155
|
+
while @rotate_queue.first.ready?
|
156
|
+
if io = @rotate_queue.first.io
|
157
|
+
io_handler = IOHandler.new(io, @pe, &@receive_lines)
|
158
|
+
else
|
159
|
+
io_handler = NullIOHandler.new
|
160
|
+
end
|
161
|
+
@io_handler.close
|
162
|
+
@io_handler = io_handler
|
163
|
+
@rotate_queue.shift
|
164
|
+
break if @rotate_queue.empty?
|
196
165
|
end
|
166
|
+
end
|
197
167
|
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
@
|
206
|
-
|
168
|
+
def on_rotate(io)
|
169
|
+
if @io_handler == nil
|
170
|
+
if io
|
171
|
+
# first time
|
172
|
+
stat = io.stat
|
173
|
+
fsize = stat.size
|
174
|
+
inode = stat.ino
|
175
|
+
if inode == @pe.read_inode
|
176
|
+
# seek to the saved position
|
177
|
+
pos = @pe.read_pos
|
178
|
+
else
|
179
|
+
# seek to the end of the file.
|
180
|
+
# logs never duplicate but may be lost if fluentd is down.
|
181
|
+
pos = fsize
|
182
|
+
@pe.update(inode, pos)
|
183
|
+
end
|
184
|
+
io.seek(pos)
|
185
|
+
|
186
|
+
@io_handler = IOHandler.new(io, @pe, &@receive_lines)
|
187
|
+
else
|
188
|
+
@io_handler = NullIOHandler.new
|
207
189
|
end
|
208
190
|
|
209
|
-
|
210
|
-
@
|
211
|
-
|
212
|
-
|
191
|
+
else
|
192
|
+
if io && @rotate_queue.find {|req| req.io == io }
|
193
|
+
return
|
194
|
+
end
|
195
|
+
last_io = @rotate_queue.empty? ? @io_handler.io : @rotate_queue.last.io
|
196
|
+
if last_io == nil
|
197
|
+
$log.info "detected rotation of #{@path}"
|
198
|
+
# rotate imeediately if previous file is nil
|
199
|
+
wait = 0
|
200
|
+
else
|
201
|
+
$log.info "detected rotation of #{@path}; waiting #{@rotate_wait} seconds"
|
202
|
+
wait = @rotate_wait
|
203
|
+
wait -= @rotate_queue.first.wait unless @rotate_queue.empty?
|
204
|
+
end
|
205
|
+
@rotate_queue << RotationRequest.new(io, wait)
|
213
206
|
end
|
214
|
-
|
215
|
-
rescue
|
216
|
-
$log.error $!.to_s
|
217
|
-
$log.error_backtrace
|
218
|
-
end
|
219
|
-
|
220
|
-
def attach(loop)
|
221
|
-
@stat_watcher.attach(loop)
|
222
|
-
@timer_watcher.attach(loop)
|
223
207
|
end
|
224
208
|
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
209
|
+
class TimerWatcher < Coolio::TimerWatcher
|
210
|
+
def initialize(interval, repeat, &callback)
|
211
|
+
@callback = callback
|
212
|
+
super(interval, repeat)
|
213
|
+
end
|
229
214
|
|
230
|
-
|
231
|
-
|
215
|
+
def on_timer
|
216
|
+
@callback.call
|
217
|
+
rescue
|
218
|
+
# TODO log?
|
219
|
+
$log.error $!.to_s
|
220
|
+
$log.error_backtrace
|
221
|
+
end
|
232
222
|
end
|
233
223
|
|
234
|
-
class
|
235
|
-
def initialize(
|
236
|
-
@
|
224
|
+
class StatWatcher < Coolio::StatWatcher
|
225
|
+
def initialize(path, &callback)
|
226
|
+
@callback = callback
|
237
227
|
super(path)
|
238
228
|
end
|
239
229
|
|
240
230
|
def on_change(prev, cur)
|
241
|
-
@
|
231
|
+
@callback.call
|
232
|
+
rescue
|
233
|
+
# TODO log?
|
234
|
+
$log.error $!.to_s
|
235
|
+
$log.error_backtrace
|
242
236
|
end
|
243
237
|
end
|
244
238
|
|
245
|
-
class
|
246
|
-
def initialize(
|
247
|
-
@
|
248
|
-
|
239
|
+
class RotationRequest
|
240
|
+
def initialize(io, wait)
|
241
|
+
@io = io
|
242
|
+
@wait = wait
|
249
243
|
end
|
250
244
|
|
251
|
-
|
252
|
-
|
245
|
+
attr_reader :io
|
246
|
+
|
247
|
+
def tick
|
248
|
+
@wait -= 1
|
253
249
|
end
|
254
|
-
end
|
255
|
-
end
|
256
250
|
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
@callback = callback
|
251
|
+
def ready?
|
252
|
+
@wait <= 0
|
253
|
+
end
|
261
254
|
end
|
262
255
|
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
|
256
|
+
class IOHandler
|
257
|
+
def initialize(io, pe, &receive_lines)
|
258
|
+
$log.info "following tail of #{io.path}"
|
259
|
+
@io = io
|
260
|
+
@pe = pe
|
261
|
+
@receive_lines = receive_lines
|
262
|
+
@buffer = ''
|
263
|
+
end
|
269
264
|
|
270
|
-
|
271
|
-
def initialize(io, start_pos, pe, receive_lines)
|
272
|
-
$log.info "following tail of #{io.path}"
|
273
|
-
@io = io
|
274
|
-
@pe = pe
|
275
|
-
@receive_lines = receive_lines
|
265
|
+
attr_reader :io
|
276
266
|
|
277
|
-
|
278
|
-
|
279
|
-
@pos = start_pos
|
267
|
+
def on_notify
|
268
|
+
lines = []
|
280
269
|
|
281
|
-
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
|
288
|
-
@pos = @pe.read_pos
|
289
|
-
else
|
290
|
-
# seek to the end of the file.
|
291
|
-
# logs never duplicate but may be lost if fluentd is down.
|
292
|
-
@pos = fsize
|
293
|
-
@pe.update(inode, @pos)
|
270
|
+
while line = @io.gets
|
271
|
+
@buffer << line
|
272
|
+
@pos = @io.pos
|
273
|
+
unless @buffer[@buffer.length-1] == ?\n
|
274
|
+
break
|
275
|
+
end
|
276
|
+
lines << line
|
294
277
|
end
|
295
|
-
end
|
296
278
|
|
297
|
-
|
279
|
+
unless lines.empty?
|
280
|
+
@pe.update_pos(@pos)
|
281
|
+
@receive_lines.call(lines)
|
282
|
+
end
|
283
|
+
rescue
|
284
|
+
$log.error $!.to_s
|
285
|
+
$log.error_backtrace
|
286
|
+
close
|
287
|
+
end
|
298
288
|
|
299
|
-
|
300
|
-
|
289
|
+
def close
|
290
|
+
@io.close unless @io.closed?
|
291
|
+
end
|
301
292
|
end
|
302
293
|
|
303
|
-
|
304
|
-
|
294
|
+
class NullIOHandler
|
295
|
+
def initialize
|
296
|
+
end
|
305
297
|
|
306
|
-
|
307
|
-
@buffer << line
|
308
|
-
@pos = @io.pos
|
309
|
-
unless @buffer[@buffer.length-1] == ?\n
|
310
|
-
break
|
311
|
-
end
|
312
|
-
lines << line
|
298
|
+
def io
|
313
299
|
end
|
314
300
|
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
close
|
301
|
+
def on_notify
|
302
|
+
end
|
303
|
+
|
304
|
+
def close
|
305
|
+
end
|
321
306
|
end
|
322
307
|
|
323
|
-
|
324
|
-
|
325
|
-
|
308
|
+
class RotateHandler
|
309
|
+
def initialize(path, &on_rotate)
|
310
|
+
@path = path
|
311
|
+
@inode = nil
|
312
|
+
@fsize = -1 # first
|
313
|
+
@on_rotate = on_rotate
|
314
|
+
@path = path
|
315
|
+
end
|
316
|
+
|
317
|
+
def on_notify
|
318
|
+
begin
|
319
|
+
io = File.open(@path)
|
320
|
+
stat = io.stat
|
321
|
+
inode = stat.ino
|
322
|
+
fsize = stat.size
|
323
|
+
rescue Errno::ENOENT
|
324
|
+
# moved or deleted
|
325
|
+
inode = nil
|
326
|
+
fsize = 0
|
327
|
+
end
|
328
|
+
|
329
|
+
begin
|
330
|
+
if @inode != inode || fsize < @fsize
|
331
|
+
# rotated or truncated
|
332
|
+
@on_rotate.call(io)
|
333
|
+
io = nil
|
334
|
+
end
|
335
|
+
|
336
|
+
@inode = inode
|
337
|
+
@fsize = fsize
|
338
|
+
ensure
|
339
|
+
io.close if io
|
340
|
+
end
|
341
|
+
|
342
|
+
rescue
|
343
|
+
$log.error $!.to_s
|
344
|
+
$log.error_backtrace
|
345
|
+
end
|
326
346
|
end
|
327
347
|
end
|
328
348
|
|
349
|
+
|
329
350
|
class PositionFile
|
330
351
|
def initialize(file, map, last_pos)
|
331
352
|
@file = file
|
data/lib/fluent/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluentd
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.10.
|
4
|
+
version: 0.10.13
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,11 +9,11 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-02-
|
12
|
+
date: 2012-02-22 00:00:00.000000000Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: msgpack
|
16
|
-
requirement: &
|
16
|
+
requirement: &70115974571520 !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - ~>
|
@@ -21,10 +21,10 @@ dependencies:
|
|
21
21
|
version: 0.4.4
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
|
-
version_requirements: *
|
24
|
+
version_requirements: *70115974571520
|
25
25
|
- !ruby/object:Gem::Dependency
|
26
26
|
name: json
|
27
|
-
requirement: &
|
27
|
+
requirement: &70115974563160 !ruby/object:Gem::Requirement
|
28
28
|
none: false
|
29
29
|
requirements:
|
30
30
|
- - ! '>='
|
@@ -32,10 +32,10 @@ dependencies:
|
|
32
32
|
version: 1.4.3
|
33
33
|
type: :runtime
|
34
34
|
prerelease: false
|
35
|
-
version_requirements: *
|
35
|
+
version_requirements: *70115974563160
|
36
36
|
- !ruby/object:Gem::Dependency
|
37
37
|
name: yajl-ruby
|
38
|
-
requirement: &
|
38
|
+
requirement: &70115974560860 !ruby/object:Gem::Requirement
|
39
39
|
none: false
|
40
40
|
requirements:
|
41
41
|
- - ~>
|
@@ -43,10 +43,10 @@ dependencies:
|
|
43
43
|
version: 1.0.0
|
44
44
|
type: :runtime
|
45
45
|
prerelease: false
|
46
|
-
version_requirements: *
|
46
|
+
version_requirements: *70115974560860
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
48
|
name: cool.io
|
49
|
-
requirement: &
|
49
|
+
requirement: &70115974558780 !ruby/object:Gem::Requirement
|
50
50
|
none: false
|
51
51
|
requirements:
|
52
52
|
- - ~>
|
@@ -54,10 +54,10 @@ dependencies:
|
|
54
54
|
version: 1.1.0
|
55
55
|
type: :runtime
|
56
56
|
prerelease: false
|
57
|
-
version_requirements: *
|
57
|
+
version_requirements: *70115974558780
|
58
58
|
- !ruby/object:Gem::Dependency
|
59
59
|
name: http_parser.rb
|
60
|
-
requirement: &
|
60
|
+
requirement: &70115974553640 !ruby/object:Gem::Requirement
|
61
61
|
none: false
|
62
62
|
requirements:
|
63
63
|
- - ~>
|
@@ -65,10 +65,10 @@ dependencies:
|
|
65
65
|
version: 0.5.1
|
66
66
|
type: :runtime
|
67
67
|
prerelease: false
|
68
|
-
version_requirements: *
|
68
|
+
version_requirements: *70115974553640
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: rake
|
71
|
-
requirement: &
|
71
|
+
requirement: &70115974549200 !ruby/object:Gem::Requirement
|
72
72
|
none: false
|
73
73
|
requirements:
|
74
74
|
- - ! '>='
|
@@ -76,7 +76,7 @@ dependencies:
|
|
76
76
|
version: 0.9.2
|
77
77
|
type: :development
|
78
78
|
prerelease: false
|
79
|
-
version_requirements: *
|
79
|
+
version_requirements: *70115974549200
|
80
80
|
description:
|
81
81
|
email: frsyuki@gmail.com
|
82
82
|
executables:
|