logstash-input-file 4.2.2 → 4.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/lib/filewatch/read_mode/handlers/base.rb +7 -8
- data/lib/filewatch/sincedb_collection.rb +22 -25
- data/lib/filewatch/sincedb_record_serializer.rb +5 -11
- data/lib/filewatch/tail_mode/handlers/base.rb +32 -23
- data/lib/filewatch/tail_mode/handlers/delete.rb +1 -1
- data/lib/filewatch/tail_mode/handlers/shrink.rb +2 -3
- data/lib/filewatch/tail_mode/handlers/unignore.rb +4 -4
- data/lib/jars/filewatch-1.0.1.jar +0 -0
- data/lib/logstash/inputs/file.rb +2 -2
- data/lib/logstash/inputs/file_listener.rb +3 -14
- data/logstash-input-file.gemspec +1 -1
- data/spec/filewatch/sincedb_record_serializer_spec.rb +6 -2
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b9475678fd2320f61bf61a3e34884ea6512072934e96f2d16c08a4055ad5f8a9
|
4
|
+
data.tar.gz: ca48090438f9ba71d20cbaf24111fa479fabe2279b3704d092aa4df36c2680e9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 97b642eed6db061be8fc620ff66e4b89e5898fd23bad138407509db20294ec013bba3a4dfe10dfc4587bdea2c09f8f99cd468db38d46a5a82028be5c85730801
|
7
|
+
data.tar.gz: b1583d047f9c042133584f3a82673b661c1eb59f7586a7c39df807a5ec391785dad2230df8561e339be673ef3f799c65e1282eac369459fccbbc491227882121
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,6 @@
|
|
1
|
+
## 4.2.3
|
2
|
+
- Refactor: improve debug logging (log catched exceptions) [#280](https://github.com/logstash-plugins/logstash-input-file/pull/280)
|
3
|
+
|
1
4
|
## 4.2.2
|
2
5
|
- Fix: sincedb_clean_after not being respected [#276](https://github.com/logstash-plugins/logstash-input-file/pull/276)
|
3
6
|
|
@@ -19,7 +19,7 @@ module FileWatch module ReadMode module Handlers
|
|
19
19
|
end
|
20
20
|
|
21
21
|
def handle(watched_file)
|
22
|
-
logger.trace("handling:
|
22
|
+
logger.trace? && logger.trace("handling:", :path => watched_file.path)
|
23
23
|
unless watched_file.has_listener?
|
24
24
|
watched_file.set_listener(@observer)
|
25
25
|
end
|
@@ -41,14 +41,14 @@ module FileWatch module ReadMode module Handlers
|
|
41
41
|
# don't emit this message too often. if a file that we can't
|
42
42
|
# read is changing a lot, we'll try to open it more often, and spam the logs.
|
43
43
|
now = Time.now.to_i
|
44
|
-
logger.trace("opening OPEN_WARN_INTERVAL is '#{OPEN_WARN_INTERVAL}'")
|
44
|
+
logger.trace? && logger.trace("opening OPEN_WARN_INTERVAL is '#{OPEN_WARN_INTERVAL}'")
|
45
45
|
if watched_file.last_open_warning_at.nil? || now - watched_file.last_open_warning_at > OPEN_WARN_INTERVAL
|
46
46
|
backtrace = e.backtrace
|
47
47
|
backtrace = backtrace.take(3) if backtrace && !logger.debug?
|
48
48
|
logger.warn("failed to open", :path => watched_file.path, :exception => e.class, :message => e.message, :backtrace => backtrace)
|
49
49
|
watched_file.last_open_warning_at = now
|
50
50
|
else
|
51
|
-
logger.trace("suppressed warning (failed to open)", :path => watched_file.path, :exception => e.class, :message => e.message)
|
51
|
+
logger.trace? && logger.trace("suppressed warning (failed to open)", :path => watched_file.path, :exception => e.class, :message => e.message)
|
52
52
|
end
|
53
53
|
watched_file.watch # set it back to watch so we can try it again
|
54
54
|
end
|
@@ -67,8 +67,7 @@ module FileWatch module ReadMode module Handlers
|
|
67
67
|
elsif sincedb_value.watched_file == watched_file
|
68
68
|
update_existing_sincedb_collection_value(watched_file, sincedb_value)
|
69
69
|
else
|
70
|
-
|
71
|
-
logger.trace(msg)
|
70
|
+
logger.trace? && logger.trace("add_or_update_sincedb_collection: the found sincedb_value has a watched_file - this is a rename, switching inode to this watched file")
|
72
71
|
existing_watched_file = sincedb_value.watched_file
|
73
72
|
if existing_watched_file.nil?
|
74
73
|
sincedb_value.set_watched_file(watched_file)
|
@@ -77,7 +76,7 @@ module FileWatch module ReadMode module Handlers
|
|
77
76
|
watched_file.update_bytes_read(sincedb_value.position)
|
78
77
|
else
|
79
78
|
sincedb_value.set_watched_file(watched_file)
|
80
|
-
logger.trace("add_or_update_sincedb_collection: switching from", :watched_file => watched_file.details)
|
79
|
+
logger.trace? && logger.trace("add_or_update_sincedb_collection: switching from", :watched_file => watched_file.details)
|
81
80
|
watched_file.rotate_from(existing_watched_file)
|
82
81
|
end
|
83
82
|
|
@@ -86,7 +85,7 @@ module FileWatch module ReadMode module Handlers
|
|
86
85
|
end
|
87
86
|
|
88
87
|
def update_existing_sincedb_collection_value(watched_file, sincedb_value)
|
89
|
-
logger.trace("update_existing_sincedb_collection_value: #{watched_file.path}, last value #{sincedb_value.position}, cur size #{watched_file.last_stat_size}")
|
88
|
+
logger.trace? && logger.trace("update_existing_sincedb_collection_value: #{watched_file.path}, last value #{sincedb_value.position}, cur size #{watched_file.last_stat_size}")
|
90
89
|
# sincedb_value is the source of truth
|
91
90
|
watched_file.update_bytes_read(sincedb_value.position)
|
92
91
|
end
|
@@ -94,7 +93,7 @@ module FileWatch module ReadMode module Handlers
|
|
94
93
|
def add_new_value_sincedb_collection(watched_file)
|
95
94
|
sincedb_value = SincedbValue.new(0)
|
96
95
|
sincedb_value.set_watched_file(watched_file)
|
97
|
-
logger.trace("add_new_value_sincedb_collection:", :path => watched_file.path, :position => sincedb_value.position)
|
96
|
+
logger.trace? && logger.trace("add_new_value_sincedb_collection:", :path => watched_file.path, :position => sincedb_value.position)
|
98
97
|
sincedb_collection.set(watched_file.sincedb_key, sincedb_value)
|
99
98
|
end
|
100
99
|
end
|
@@ -47,16 +47,16 @@ module FileWatch
|
|
47
47
|
@time_sdb_opened = Time.now.to_f
|
48
48
|
begin
|
49
49
|
path.open do |file|
|
50
|
-
logger.
|
50
|
+
logger.debug("open: reading from #{path}")
|
51
51
|
@serializer.deserialize(file) do |key, value|
|
52
|
-
logger.trace("open: importing
|
52
|
+
logger.trace? && logger.trace("open: importing #{key.inspect} => #{value.inspect}")
|
53
53
|
set_key_value(key, value)
|
54
54
|
end
|
55
55
|
end
|
56
56
|
logger.trace("open: count of keys read: #{@sincedb.keys.size}")
|
57
57
|
rescue => e
|
58
58
|
#No existing sincedb to load
|
59
|
-
logger.
|
59
|
+
logger.debug("open: error opening #{path}", :exception => e.class, :message => e.message)
|
60
60
|
end
|
61
61
|
end
|
62
62
|
|
@@ -68,35 +68,32 @@ module FileWatch
|
|
68
68
|
# and due to the window handling of many files
|
69
69
|
# this file may not be opened in this session.
|
70
70
|
# a new value will be added when the file is opened
|
71
|
-
logger.trace("associate: unmatched")
|
71
|
+
logger.trace("associate: unmatched", :filename => watched_file.filename)
|
72
72
|
return true
|
73
73
|
end
|
74
74
|
logger.trace? && logger.trace("associate: found sincedb record", :filename => watched_file.filename,
|
75
75
|
:sincedb_key => watched_file.sincedb_key, :sincedb_value => sincedb_value)
|
76
|
-
if sincedb_value.watched_file.nil?
|
77
|
-
# not associated
|
76
|
+
if sincedb_value.watched_file.nil? # not associated
|
78
77
|
if sincedb_value.path_in_sincedb.nil?
|
79
78
|
handle_association(sincedb_value, watched_file)
|
80
|
-
logger.trace("associate: inode matched but no path in sincedb")
|
79
|
+
logger.trace? && logger.trace("associate: inode matched but no path in sincedb", :filename => watched_file.filename)
|
81
80
|
return true
|
82
81
|
end
|
83
82
|
if sincedb_value.path_in_sincedb == watched_file.path
|
84
|
-
# the path on disk is the same as discovered path
|
85
|
-
# and the inode is the same.
|
83
|
+
# the path on disk is the same as discovered path and the inode is the same.
|
86
84
|
handle_association(sincedb_value, watched_file)
|
87
|
-
logger.trace("associate: inode and path matched")
|
85
|
+
logger.trace? && logger.trace("associate: inode and path matched", :filename => watched_file.filename)
|
88
86
|
return true
|
89
87
|
end
|
90
|
-
# the path on disk is different from discovered unassociated path
|
91
|
-
# but they have the same key (inode)
|
88
|
+
# the path on disk is different from discovered unassociated path but they have the same key (inode)
|
92
89
|
# treat as a new file, a new value will be added when the file is opened
|
93
90
|
sincedb_value.clear_watched_file
|
94
91
|
delete(watched_file.sincedb_key)
|
95
|
-
logger.trace("associate: matched but allocated to another")
|
92
|
+
logger.trace? && logger.trace("associate: matched but allocated to another", :filename => watched_file.filename)
|
96
93
|
return true
|
97
94
|
end
|
98
95
|
if sincedb_value.watched_file.equal?(watched_file) # pointer equals
|
99
|
-
logger.trace("associate: already associated")
|
96
|
+
logger.trace? && logger.trace("associate: already associated", :filename => watched_file.filename)
|
100
97
|
return true
|
101
98
|
end
|
102
99
|
# sincedb_value.watched_file is not this discovered watched_file but they have the same key (inode)
|
@@ -107,7 +104,7 @@ module FileWatch
|
|
107
104
|
# after the original is deleted
|
108
105
|
# are not yet in the delete phase, let this play out
|
109
106
|
existing_watched_file = sincedb_value.watched_file
|
110
|
-
logger.trace? && logger.trace("
|
107
|
+
logger.trace? && logger.trace("associate: found sincedb_value has a watched_file - this is a rename",
|
111
108
|
:this_watched_file => watched_file.details, :existing_watched_file => existing_watched_file.details)
|
112
109
|
watched_file.rotation_in_progress
|
113
110
|
true
|
@@ -197,43 +194,43 @@ module FileWatch
|
|
197
194
|
watched_file.initial_completed
|
198
195
|
if watched_file.all_read?
|
199
196
|
watched_file.ignore
|
200
|
-
logger.trace? && logger.trace("handle_association fully read, ignoring
|
197
|
+
logger.trace? && logger.trace("handle_association fully read, ignoring", :watched_file => watched_file.details, :sincedb_value => sincedb_value)
|
201
198
|
end
|
202
199
|
end
|
203
200
|
|
204
201
|
def set_key_value(key, value)
|
205
202
|
if @time_sdb_opened < value.last_changed_at_expires(@settings.sincedb_expiry_duration)
|
206
|
-
logger.trace("open: setting #{key.inspect} to #{value.inspect}")
|
207
203
|
set(key, value)
|
208
204
|
else
|
209
|
-
logger.
|
205
|
+
logger.debug("set_key_value: record has expired, skipping: #{key.inspect} => #{value.inspect}")
|
210
206
|
end
|
211
207
|
end
|
212
208
|
|
213
209
|
def sincedb_write(time = Time.now)
|
214
|
-
logger.trace("sincedb_write: #{path} (time = #{time})")
|
210
|
+
logger.trace? && logger.trace("sincedb_write: #{path} (time = #{time})")
|
215
211
|
begin
|
216
|
-
@write_method.call(time)
|
217
|
-
|
212
|
+
expired_keys = @write_method.call(time)
|
213
|
+
expired_keys.each do |key|
|
218
214
|
@sincedb[key].unset_watched_file
|
219
215
|
delete(key)
|
220
216
|
logger.trace? && logger.trace("sincedb_write: cleaned", :key => key)
|
221
217
|
end
|
222
218
|
@sincedb_last_write = time.to_i
|
223
219
|
@write_requested = false
|
224
|
-
rescue Errno::EACCES
|
225
|
-
# no file handles free perhaps
|
226
|
-
#
|
227
|
-
logger.trace("sincedb_write: #{path} error: #{$!}")
|
220
|
+
rescue Errno::EACCES => e
|
221
|
+
# no file handles free perhaps - maybe it will work next time
|
222
|
+
logger.debug("sincedb_write: #{path} error:", :exception => e.class, :message => e.message)
|
228
223
|
end
|
229
224
|
end
|
230
225
|
|
226
|
+
# @return expired keys
|
231
227
|
def atomic_write(time)
|
232
228
|
FileHelper.write_atomically(@full_path) do |io|
|
233
229
|
@serializer.serialize(@sincedb, io, time.to_f)
|
234
230
|
end
|
235
231
|
end
|
236
232
|
|
233
|
+
# @return expired keys
|
237
234
|
def non_atomic_write(time)
|
238
235
|
IO.open(IO.sysopen(@full_path, "w+")) do |io|
|
239
236
|
@serializer.serialize(@sincedb, io, time.to_f)
|
@@ -3,30 +3,25 @@
|
|
3
3
|
module FileWatch
|
4
4
|
class SincedbRecordSerializer
|
5
5
|
|
6
|
-
attr_reader :expired_keys
|
7
|
-
|
8
6
|
def self.days_to_seconds(days)
|
9
7
|
(24 * 3600) * days.to_f
|
10
8
|
end
|
11
9
|
|
12
10
|
def initialize(sincedb_value_expiry)
|
13
11
|
@sincedb_value_expiry = sincedb_value_expiry
|
14
|
-
@expired_keys = []
|
15
|
-
end
|
16
|
-
|
17
|
-
def update_sincedb_value_expiry_from_days(days)
|
18
|
-
@sincedb_value_expiry = SincedbRecordSerializer.days_to_seconds(days)
|
19
12
|
end
|
20
13
|
|
14
|
+
# @return Array expired keys (ones that were not written to the file)
|
21
15
|
def serialize(db, io, as_of = Time.now.to_f)
|
22
|
-
|
16
|
+
expired_keys = []
|
23
17
|
db.each do |key, value|
|
24
18
|
if as_of > value.last_changed_at_expires(@sincedb_value_expiry)
|
25
|
-
|
19
|
+
expired_keys << key
|
26
20
|
next
|
27
21
|
end
|
28
22
|
io.write(serialize_record(key, value))
|
29
23
|
end
|
24
|
+
expired_keys
|
30
25
|
end
|
31
26
|
|
32
27
|
def deserialize(io)
|
@@ -36,8 +31,7 @@ module FileWatch
|
|
36
31
|
end
|
37
32
|
|
38
33
|
def serialize_record(k, v)
|
39
|
-
# effectively InodeStruct#to_s SincedbValue#to_s
|
40
|
-
"#{k} #{v}\n"
|
34
|
+
"#{k} #{v}\n" # effectively InodeStruct#to_s SincedbValue#to_s
|
41
35
|
end
|
42
36
|
|
43
37
|
def deserialize_record(record)
|
@@ -18,7 +18,7 @@ module FileWatch module TailMode module Handlers
|
|
18
18
|
end
|
19
19
|
|
20
20
|
def handle(watched_file)
|
21
|
-
logger.trace("handling:
|
21
|
+
logger.trace? && logger.trace("handling:", :path => watched_file.path)
|
22
22
|
unless watched_file.has_listener?
|
23
23
|
watched_file.set_listener(@observer)
|
24
24
|
end
|
@@ -37,7 +37,7 @@ module FileWatch module TailMode module Handlers
|
|
37
37
|
|
38
38
|
def controlled_read(watched_file, loop_control)
|
39
39
|
changed = false
|
40
|
-
logger.trace(
|
40
|
+
logger.trace? && logger.trace(__method__.to_s, :iterations => loop_control.count, :amount => loop_control.size, :filename => watched_file.filename)
|
41
41
|
# from a real config (has 102 file inputs)
|
42
42
|
# -- This cfg creates a file input for every log file to create a dedicated file pointer and read all file simultaneously
|
43
43
|
# -- If we put all log files in one file input glob we will have indexing delay, because Logstash waits until the first file becomes EOF
|
@@ -48,7 +48,7 @@ module FileWatch module TailMode module Handlers
|
|
48
48
|
loop_control.count.times do
|
49
49
|
break if quit?
|
50
50
|
begin
|
51
|
-
logger.debug("
|
51
|
+
logger.debug? && logger.debug("#{__method__} get chunk")
|
52
52
|
result = watched_file.read_extract_lines(loop_control.size) # expect BufferExtractResult
|
53
53
|
logger.trace(result.warning, result.additional) unless result.warning.empty?
|
54
54
|
changed = true
|
@@ -57,40 +57,42 @@ module FileWatch module TailMode module Handlers
|
|
57
57
|
# sincedb position is now independent from the watched_file bytes_read
|
58
58
|
sincedb_collection.increment(watched_file.sincedb_key, line.bytesize + @settings.delimiter_byte_size)
|
59
59
|
end
|
60
|
-
rescue EOFError
|
60
|
+
rescue EOFError => e
|
61
61
|
# it only makes sense to signal EOF in "read" mode not "tail"
|
62
|
+
logger.debug(__method__.to_s, exception_details(watched_file.path, e, false))
|
62
63
|
loop_control.flag_read_error
|
63
64
|
break
|
64
|
-
rescue Errno::EWOULDBLOCK, Errno::EINTR
|
65
|
+
rescue Errno::EWOULDBLOCK, Errno::EINTR => e
|
66
|
+
logger.debug(__method__.to_s, exception_details(watched_file.path, e, false))
|
65
67
|
watched_file.listener.error
|
66
68
|
loop_control.flag_read_error
|
67
69
|
break
|
68
70
|
rescue => e
|
69
|
-
logger.error("
|
71
|
+
logger.error("#{__method__} general error reading", exception_details(watched_file.path, e))
|
70
72
|
watched_file.listener.error
|
71
73
|
loop_control.flag_read_error
|
72
74
|
break
|
73
75
|
end
|
74
76
|
end
|
75
|
-
logger.debug("
|
77
|
+
logger.debug("#{__method__} stopped loop due quit") if quit?
|
76
78
|
sincedb_collection.request_disk_flush if changed
|
77
79
|
end
|
78
80
|
|
79
81
|
def open_file(watched_file)
|
80
82
|
return true if watched_file.file_open?
|
81
|
-
logger.trace("
|
83
|
+
logger.trace? && logger.trace("open_file", :filename => watched_file.filename)
|
82
84
|
begin
|
83
85
|
watched_file.open
|
84
|
-
rescue
|
86
|
+
rescue => e
|
85
87
|
# don't emit this message too often. if a file that we can't
|
86
88
|
# read is changing a lot, we'll try to open it more often, and spam the logs.
|
87
89
|
now = Time.now.to_i
|
88
|
-
logger.trace("open_file OPEN_WARN_INTERVAL is '#{OPEN_WARN_INTERVAL}'")
|
90
|
+
logger.trace? && logger.trace("open_file OPEN_WARN_INTERVAL is '#{OPEN_WARN_INTERVAL}'")
|
89
91
|
if watched_file.last_open_warning_at.nil? || now - watched_file.last_open_warning_at > OPEN_WARN_INTERVAL
|
90
|
-
logger.warn("failed to open
|
92
|
+
logger.warn("failed to open file", exception_details(watched_file.path, e))
|
91
93
|
watched_file.last_open_warning_at = now
|
92
94
|
else
|
93
|
-
logger.
|
95
|
+
logger.debug("open_file suppressed warning `failed to open file`", exception_details(watched_file.path, e, false))
|
94
96
|
end
|
95
97
|
watched_file.watch # set it back to watch so we can try it again
|
96
98
|
else
|
@@ -108,26 +110,22 @@ module FileWatch module TailMode module Handlers
|
|
108
110
|
update_existing_sincedb_collection_value(watched_file, sincedb_value)
|
109
111
|
watched_file.initial_completed
|
110
112
|
else
|
111
|
-
|
112
|
-
|
113
|
-
"sincedb key" => watched_file.sincedb_key,
|
114
|
-
"sincedb value" => sincedb_value
|
115
|
-
)
|
113
|
+
logger.trace? && logger.trace("add_or_update_sincedb_collection: found sincedb record",
|
114
|
+
:sincedb_key => watched_file.sincedb_key, :sincedb_value => sincedb_value)
|
116
115
|
# detected a rotation, Discoverer can't handle this because this watched file is not a new discovery.
|
117
116
|
# we must handle it here, by transferring state and have the sincedb value track this watched file
|
118
117
|
# rotate_as_file and rotate_from will switch the sincedb key to the inode that the path is now pointing to
|
119
118
|
# and pickup the sincedb_value from before.
|
120
|
-
|
121
|
-
logger.trace(msg)
|
119
|
+
logger.debug("add_or_update_sincedb_collection: the found sincedb_value has a watched_file - this is a rename, switching inode to this watched file")
|
122
120
|
existing_watched_file = sincedb_value.watched_file
|
123
121
|
if existing_watched_file.nil?
|
124
122
|
sincedb_value.set_watched_file(watched_file)
|
125
|
-
logger.trace("add_or_update_sincedb_collection: switching as new file")
|
123
|
+
logger.trace? && logger.trace("add_or_update_sincedb_collection: switching as new file")
|
126
124
|
watched_file.rotate_as_file
|
127
125
|
watched_file.update_bytes_read(sincedb_value.position)
|
128
126
|
else
|
129
127
|
sincedb_value.set_watched_file(watched_file)
|
130
|
-
logger.trace("add_or_update_sincedb_collection: switching from
|
128
|
+
logger.trace? && logger.trace("add_or_update_sincedb_collection: switching from:", :watched_file => watched_file.details)
|
131
129
|
watched_file.rotate_from(existing_watched_file)
|
132
130
|
end
|
133
131
|
end
|
@@ -135,13 +133,15 @@ module FileWatch module TailMode module Handlers
|
|
135
133
|
end
|
136
134
|
|
137
135
|
def update_existing_sincedb_collection_value(watched_file, sincedb_value)
|
138
|
-
logger.trace("update_existing_sincedb_collection_value
|
136
|
+
logger.trace? && logger.trace("update_existing_sincedb_collection_value", :position => sincedb_value.position,
|
137
|
+
:filename => watched_file.filename, :last_stat_size => watched_file.last_stat_size)
|
139
138
|
update_existing_specifically(watched_file, sincedb_value)
|
140
139
|
end
|
141
140
|
|
142
141
|
def add_new_value_sincedb_collection(watched_file)
|
143
142
|
sincedb_value = get_new_value_specifically(watched_file)
|
144
|
-
logger.trace("add_new_value_sincedb_collection",
|
143
|
+
logger.trace? && logger.trace("add_new_value_sincedb_collection", :position => sincedb_value.position,
|
144
|
+
:watched_file => watched_file.details)
|
145
145
|
sincedb_collection.set(watched_file.sincedb_key, sincedb_value)
|
146
146
|
sincedb_value
|
147
147
|
end
|
@@ -153,5 +153,14 @@ module FileWatch module TailMode module Handlers
|
|
153
153
|
watched_file.update_bytes_read(position)
|
154
154
|
value
|
155
155
|
end
|
156
|
+
|
157
|
+
private
|
158
|
+
|
159
|
+
def exception_details(path, e, trace = true)
|
160
|
+
details = { :path => path, :exception => e.class, :message => e.message }
|
161
|
+
details[:backtrace] = e.backtrace if trace && logger.debug?
|
162
|
+
details
|
163
|
+
end
|
164
|
+
|
156
165
|
end
|
157
166
|
end end end
|
@@ -7,7 +7,7 @@ module FileWatch module TailMode module Handlers
|
|
7
7
|
# TODO consider trying to find the renamed file - it will have the same inode.
|
8
8
|
# Needs a rotate scheme rename hint from user e.g. "<name>-YYYY-MM-DD-N.<ext>" or "<name>.<ext>.N"
|
9
9
|
# send the found content to the same listener (stream identity)
|
10
|
-
logger.trace(
|
10
|
+
logger.trace? && logger.trace(__method__.to_s, :path => watched_file.path, :watched_file => watched_file.details)
|
11
11
|
if watched_file.bytes_unread > 0
|
12
12
|
logger.warn(DATA_LOSS_WARNING, :path => watched_file.path, :unread_bytes => watched_file.bytes_unread)
|
13
13
|
end
|
@@ -14,11 +14,10 @@ module FileWatch module TailMode module Handlers
|
|
14
14
|
end
|
15
15
|
|
16
16
|
def update_existing_specifically(watched_file, sincedb_value)
|
17
|
-
# we have a match but size is smaller
|
18
|
-
# set all to zero
|
17
|
+
# we have a match but size is smaller - set all to zero
|
19
18
|
watched_file.reset_bytes_unread
|
20
19
|
sincedb_value.update_position(0)
|
21
|
-
logger.trace("update_existing_specifically: was truncated seeking to beginning",
|
20
|
+
logger.trace? && logger.trace("update_existing_specifically: was truncated seeking to beginning", :watched_file => watched_file.details, :sincedb_value => sincedb_value)
|
22
21
|
end
|
23
22
|
end
|
24
23
|
end end end
|
@@ -13,9 +13,9 @@ module FileWatch module TailMode module Handlers
|
|
13
13
|
# for file initially ignored their bytes_read was set to stat.size
|
14
14
|
# use this value not the `start_new_files_at` for the position
|
15
15
|
# logger.trace("get_new_value_specifically", "watched_file" => watched_file.inspect)
|
16
|
-
SincedbValue.new(watched_file.bytes_read).tap do |
|
17
|
-
|
18
|
-
logger.trace("
|
16
|
+
SincedbValue.new(watched_file.bytes_read).tap do |sincedb_value|
|
17
|
+
sincedb_value.set_watched_file(watched_file)
|
18
|
+
logger.trace? && logger.trace("get_new_value_specifically: unignore", :watched_file => watched_file.details, :sincedb_value => sincedb_value)
|
19
19
|
end
|
20
20
|
end
|
21
21
|
|
@@ -26,7 +26,7 @@ module FileWatch module TailMode module Handlers
|
|
26
26
|
# we will handle grow or shrink
|
27
27
|
# for now we seek to where we were before the file got ignored (grow)
|
28
28
|
# or to the start (shrink)
|
29
|
-
logger.trace("
|
29
|
+
logger.trace? && logger.trace("update_existing_specifically: unignore", :watched_file => watched_file.details, :sincedb_value => sincedb_value)
|
30
30
|
position = 0
|
31
31
|
if watched_file.shrunk?
|
32
32
|
watched_file.update_bytes_read(0)
|
Binary file
|
data/lib/logstash/inputs/file.rb
CHANGED
@@ -377,12 +377,12 @@ class File < LogStash::Inputs::Base
|
|
377
377
|
def handle_deletable_path(path)
|
378
378
|
return if tail_mode?
|
379
379
|
return if @completed_file_handlers.empty?
|
380
|
+
@logger.debug? && @logger.debug(__method__.to_s, :path => path)
|
380
381
|
@completed_file_handlers.each { |handler| handler.handle(path) }
|
381
382
|
end
|
382
383
|
|
383
384
|
def log_line_received(path, line)
|
384
|
-
|
385
|
-
@logger.debug("Received line", :path => path, :text => line)
|
385
|
+
@logger.debug? && @logger.debug("Received line", :path => path, :text => line)
|
386
386
|
end
|
387
387
|
|
388
388
|
def stop
|
@@ -7,9 +7,9 @@ module LogStash module Inputs
|
|
7
7
|
class FileListener
|
8
8
|
attr_reader :input, :path, :data
|
9
9
|
# construct with link back to the input plugin instance.
|
10
|
-
def initialize(path, input)
|
10
|
+
def initialize(path, input, data = nil)
|
11
11
|
@path, @input = path, input
|
12
|
-
@data =
|
12
|
+
@data = data
|
13
13
|
end
|
14
14
|
|
15
15
|
def opened
|
@@ -36,7 +36,7 @@ module LogStash module Inputs
|
|
36
36
|
def accept(data)
|
37
37
|
# and push transient data filled dup listener downstream
|
38
38
|
input.log_line_received(path, data)
|
39
|
-
input.codec.accept(
|
39
|
+
input.codec.accept(self.class.new(path, input, data))
|
40
40
|
end
|
41
41
|
|
42
42
|
def process_event(event)
|
@@ -45,17 +45,6 @@ module LogStash module Inputs
|
|
45
45
|
input.post_process_this(event)
|
46
46
|
end
|
47
47
|
|
48
|
-
def add_state(data)
|
49
|
-
@data = data
|
50
|
-
self
|
51
|
-
end
|
52
|
-
|
53
|
-
private
|
54
|
-
|
55
|
-
# duplicate and add state for downstream
|
56
|
-
def dup_adding_state(line)
|
57
|
-
self.class.new(path, input).add_state(line)
|
58
|
-
end
|
59
48
|
end
|
60
49
|
|
61
50
|
class FlushableListener < FileListener
|
data/logstash-input-file.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-input-file'
|
4
|
-
s.version = '4.2.
|
4
|
+
s.version = '4.2.3'
|
5
5
|
s.licenses = ['Apache-2.0']
|
6
6
|
s.summary = "Streams events from files"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
@@ -9,7 +9,9 @@ module FileWatch
|
|
9
9
|
let(:io) { StringIO.new }
|
10
10
|
let(:db) { Hash.new }
|
11
11
|
|
12
|
-
|
12
|
+
let(:sincedb_value_expiry) { SincedbRecordSerializer.days_to_seconds(14) }
|
13
|
+
|
14
|
+
subject { SincedbRecordSerializer.new(sincedb_value_expiry) }
|
13
15
|
|
14
16
|
context "deserialize from IO" do
|
15
17
|
it 'reads V1 records' do
|
@@ -82,8 +84,10 @@ module FileWatch
|
|
82
84
|
end
|
83
85
|
|
84
86
|
context "given a non default `sincedb_clean_after`" do
|
87
|
+
|
88
|
+
let(:sincedb_value_expiry) { SincedbRecordSerializer.days_to_seconds(2) }
|
89
|
+
|
85
90
|
it "does not write expired db entries to an IO object" do
|
86
|
-
subject.update_sincedb_value_expiry_from_days(2)
|
87
91
|
one_day_ago = Time.now.to_f - (1.0*24*3600)
|
88
92
|
three_days_ago = one_day_ago - (2.0*24*3600)
|
89
93
|
db[InodeStruct.new("42424242", 2, 5)] = SincedbValue.new(42, one_day_ago)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-file
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 4.2.
|
4
|
+
version: 4.2.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-12-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|