logstash-input-file 4.2.2 → 4.2.3

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: f8c11df0b47e51ceb5ebd37b3d16eb28acfef360aa976d4155752fffca3c0161
4
- data.tar.gz: 297b2410e81e2f4228f59e459e486269238820b82f94a577068c53c87b9f2ea7
3
+ metadata.gz: b9475678fd2320f61bf61a3e34884ea6512072934e96f2d16c08a4055ad5f8a9
4
+ data.tar.gz: ca48090438f9ba71d20cbaf24111fa479fabe2279b3704d092aa4df36c2680e9
5
5
  SHA512:
6
- metadata.gz: 639fbc4dd7b2f02977923726f997e8ecf2d57862ac6cf20dd284f2a1b55e0532eb0137d1b81be200afc10bc5a8f84c625e74211bb685a45099c4660b36ad15eb
7
- data.tar.gz: ecee35a5081b61038e3f9052cbb447269d1a5def40f47fbbb1cb2f88f0ee509e98995f9441e7cf37329d505ee095087e58e6740856d21253ef59b1d40a25fa10
6
+ metadata.gz: 97b642eed6db061be8fc620ff66e4b89e5898fd23bad138407509db20294ec013bba3a4dfe10dfc4587bdea2c09f8f99cd468db38d46a5a82028be5c85730801
7
+ data.tar.gz: b1583d047f9c042133584f3a82673b661c1eb59f7586a7c39df807a5ec391785dad2230df8561e339be673ef3f799c65e1282eac369459fccbbc491227882121
@@ -1,3 +1,6 @@
1
+ ## 4.2.3
2
+ - Refactor: improve debug logging (log catched exceptions) [#280](https://github.com/logstash-plugins/logstash-input-file/pull/280)
3
+
1
4
  ## 4.2.2
2
5
  - Fix: sincedb_clean_after not being respected [#276](https://github.com/logstash-plugins/logstash-input-file/pull/276)
3
6
 
@@ -19,7 +19,7 @@ module FileWatch module ReadMode module Handlers
19
19
  end
20
20
 
21
21
  def handle(watched_file)
22
- logger.trace("handling: #{watched_file.path}")
22
+ logger.trace? && logger.trace("handling:", :path => watched_file.path)
23
23
  unless watched_file.has_listener?
24
24
  watched_file.set_listener(@observer)
25
25
  end
@@ -41,14 +41,14 @@ module FileWatch module ReadMode module Handlers
41
41
  # don't emit this message too often. if a file that we can't
42
42
  # read is changing a lot, we'll try to open it more often, and spam the logs.
43
43
  now = Time.now.to_i
44
- logger.trace("opening OPEN_WARN_INTERVAL is '#{OPEN_WARN_INTERVAL}'")
44
+ logger.trace? && logger.trace("opening OPEN_WARN_INTERVAL is '#{OPEN_WARN_INTERVAL}'")
45
45
  if watched_file.last_open_warning_at.nil? || now - watched_file.last_open_warning_at > OPEN_WARN_INTERVAL
46
46
  backtrace = e.backtrace
47
47
  backtrace = backtrace.take(3) if backtrace && !logger.debug?
48
48
  logger.warn("failed to open", :path => watched_file.path, :exception => e.class, :message => e.message, :backtrace => backtrace)
49
49
  watched_file.last_open_warning_at = now
50
50
  else
51
- logger.trace("suppressed warning (failed to open)", :path => watched_file.path, :exception => e.class, :message => e.message)
51
+ logger.trace? && logger.trace("suppressed warning (failed to open)", :path => watched_file.path, :exception => e.class, :message => e.message)
52
52
  end
53
53
  watched_file.watch # set it back to watch so we can try it again
54
54
  end
@@ -67,8 +67,7 @@ module FileWatch module ReadMode module Handlers
67
67
  elsif sincedb_value.watched_file == watched_file
68
68
  update_existing_sincedb_collection_value(watched_file, sincedb_value)
69
69
  else
70
- msg = "add_or_update_sincedb_collection: the found sincedb_value has a watched_file - this is a rename, switching inode to this watched file"
71
- logger.trace(msg)
70
+ logger.trace? && logger.trace("add_or_update_sincedb_collection: the found sincedb_value has a watched_file - this is a rename, switching inode to this watched file")
72
71
  existing_watched_file = sincedb_value.watched_file
73
72
  if existing_watched_file.nil?
74
73
  sincedb_value.set_watched_file(watched_file)
@@ -77,7 +76,7 @@ module FileWatch module ReadMode module Handlers
77
76
  watched_file.update_bytes_read(sincedb_value.position)
78
77
  else
79
78
  sincedb_value.set_watched_file(watched_file)
80
- logger.trace("add_or_update_sincedb_collection: switching from", :watched_file => watched_file.details)
79
+ logger.trace? && logger.trace("add_or_update_sincedb_collection: switching from", :watched_file => watched_file.details)
81
80
  watched_file.rotate_from(existing_watched_file)
82
81
  end
83
82
 
@@ -86,7 +85,7 @@ module FileWatch module ReadMode module Handlers
86
85
  end
87
86
 
88
87
  def update_existing_sincedb_collection_value(watched_file, sincedb_value)
89
- logger.trace("update_existing_sincedb_collection_value: #{watched_file.path}, last value #{sincedb_value.position}, cur size #{watched_file.last_stat_size}")
88
+ logger.trace? && logger.trace("update_existing_sincedb_collection_value: #{watched_file.path}, last value #{sincedb_value.position}, cur size #{watched_file.last_stat_size}")
90
89
  # sincedb_value is the source of truth
91
90
  watched_file.update_bytes_read(sincedb_value.position)
92
91
  end
@@ -94,7 +93,7 @@ module FileWatch module ReadMode module Handlers
94
93
  def add_new_value_sincedb_collection(watched_file)
95
94
  sincedb_value = SincedbValue.new(0)
96
95
  sincedb_value.set_watched_file(watched_file)
97
- logger.trace("add_new_value_sincedb_collection:", :path => watched_file.path, :position => sincedb_value.position)
96
+ logger.trace? && logger.trace("add_new_value_sincedb_collection:", :path => watched_file.path, :position => sincedb_value.position)
98
97
  sincedb_collection.set(watched_file.sincedb_key, sincedb_value)
99
98
  end
100
99
  end
@@ -47,16 +47,16 @@ module FileWatch
47
47
  @time_sdb_opened = Time.now.to_f
48
48
  begin
49
49
  path.open do |file|
50
- logger.trace("open: reading from #{path}")
50
+ logger.debug("open: reading from #{path}")
51
51
  @serializer.deserialize(file) do |key, value|
52
- logger.trace("open: importing ... '#{key}' => '#{value}'")
52
+ logger.trace? && logger.trace("open: importing #{key.inspect} => #{value.inspect}")
53
53
  set_key_value(key, value)
54
54
  end
55
55
  end
56
56
  logger.trace("open: count of keys read: #{@sincedb.keys.size}")
57
57
  rescue => e
58
58
  #No existing sincedb to load
59
- logger.trace("open: error:", :path => path, :exception => e.class, :message => e.message)
59
+ logger.debug("open: error opening #{path}", :exception => e.class, :message => e.message)
60
60
  end
61
61
  end
62
62
 
@@ -68,35 +68,32 @@ module FileWatch
68
68
  # and due to the window handling of many files
69
69
  # this file may not be opened in this session.
70
70
  # a new value will be added when the file is opened
71
- logger.trace("associate: unmatched")
71
+ logger.trace("associate: unmatched", :filename => watched_file.filename)
72
72
  return true
73
73
  end
74
74
  logger.trace? && logger.trace("associate: found sincedb record", :filename => watched_file.filename,
75
75
  :sincedb_key => watched_file.sincedb_key, :sincedb_value => sincedb_value)
76
- if sincedb_value.watched_file.nil?
77
- # not associated
76
+ if sincedb_value.watched_file.nil? # not associated
78
77
  if sincedb_value.path_in_sincedb.nil?
79
78
  handle_association(sincedb_value, watched_file)
80
- logger.trace("associate: inode matched but no path in sincedb")
79
+ logger.trace? && logger.trace("associate: inode matched but no path in sincedb", :filename => watched_file.filename)
81
80
  return true
82
81
  end
83
82
  if sincedb_value.path_in_sincedb == watched_file.path
84
- # the path on disk is the same as discovered path
85
- # and the inode is the same.
83
+ # the path on disk is the same as discovered path and the inode is the same.
86
84
  handle_association(sincedb_value, watched_file)
87
- logger.trace("associate: inode and path matched")
85
+ logger.trace? && logger.trace("associate: inode and path matched", :filename => watched_file.filename)
88
86
  return true
89
87
  end
90
- # the path on disk is different from discovered unassociated path
91
- # but they have the same key (inode)
88
+ # the path on disk is different from discovered unassociated path but they have the same key (inode)
92
89
  # treat as a new file, a new value will be added when the file is opened
93
90
  sincedb_value.clear_watched_file
94
91
  delete(watched_file.sincedb_key)
95
- logger.trace("associate: matched but allocated to another")
92
+ logger.trace? && logger.trace("associate: matched but allocated to another", :filename => watched_file.filename)
96
93
  return true
97
94
  end
98
95
  if sincedb_value.watched_file.equal?(watched_file) # pointer equals
99
- logger.trace("associate: already associated")
96
+ logger.trace? && logger.trace("associate: already associated", :filename => watched_file.filename)
100
97
  return true
101
98
  end
102
99
  # sincedb_value.watched_file is not this discovered watched_file but they have the same key (inode)
@@ -107,7 +104,7 @@ module FileWatch
107
104
  # after the original is deleted
108
105
  # are not yet in the delete phase, let this play out
109
106
  existing_watched_file = sincedb_value.watched_file
110
- logger.trace? && logger.trace("----------------- >> associate: the found sincedb_value has a watched_file - this is a rename",
107
+ logger.trace? && logger.trace("associate: found sincedb_value has a watched_file - this is a rename",
111
108
  :this_watched_file => watched_file.details, :existing_watched_file => existing_watched_file.details)
112
109
  watched_file.rotation_in_progress
113
110
  true
@@ -197,43 +194,43 @@ module FileWatch
197
194
  watched_file.initial_completed
198
195
  if watched_file.all_read?
199
196
  watched_file.ignore
200
- logger.trace? && logger.trace("handle_association fully read, ignoring.....", :watched_file => watched_file.details, :sincedb_value => sincedb_value)
197
+ logger.trace? && logger.trace("handle_association fully read, ignoring", :watched_file => watched_file.details, :sincedb_value => sincedb_value)
201
198
  end
202
199
  end
203
200
 
204
201
  def set_key_value(key, value)
205
202
  if @time_sdb_opened < value.last_changed_at_expires(@settings.sincedb_expiry_duration)
206
- logger.trace("open: setting #{key.inspect} to #{value.inspect}")
207
203
  set(key, value)
208
204
  else
209
- logger.trace("open: record has expired, skipping: #{key.inspect} #{value.inspect}")
205
+ logger.debug("set_key_value: record has expired, skipping: #{key.inspect} => #{value.inspect}")
210
206
  end
211
207
  end
212
208
 
213
209
  def sincedb_write(time = Time.now)
214
- logger.trace("sincedb_write: #{path} (time = #{time})")
210
+ logger.trace? && logger.trace("sincedb_write: #{path} (time = #{time})")
215
211
  begin
216
- @write_method.call(time)
217
- @serializer.expired_keys.each do |key|
212
+ expired_keys = @write_method.call(time)
213
+ expired_keys.each do |key|
218
214
  @sincedb[key].unset_watched_file
219
215
  delete(key)
220
216
  logger.trace? && logger.trace("sincedb_write: cleaned", :key => key)
221
217
  end
222
218
  @sincedb_last_write = time.to_i
223
219
  @write_requested = false
224
- rescue Errno::EACCES
225
- # no file handles free perhaps
226
- # maybe it will work next time
227
- logger.trace("sincedb_write: #{path} error: #{$!}")
220
+ rescue Errno::EACCES => e
221
+ # no file handles free perhaps - maybe it will work next time
222
+ logger.debug("sincedb_write: #{path} error:", :exception => e.class, :message => e.message)
228
223
  end
229
224
  end
230
225
 
226
+ # @return expired keys
231
227
  def atomic_write(time)
232
228
  FileHelper.write_atomically(@full_path) do |io|
233
229
  @serializer.serialize(@sincedb, io, time.to_f)
234
230
  end
235
231
  end
236
232
 
233
+ # @return expired keys
237
234
  def non_atomic_write(time)
238
235
  IO.open(IO.sysopen(@full_path, "w+")) do |io|
239
236
  @serializer.serialize(@sincedb, io, time.to_f)
@@ -3,30 +3,25 @@
3
3
  module FileWatch
4
4
  class SincedbRecordSerializer
5
5
 
6
- attr_reader :expired_keys
7
-
8
6
  def self.days_to_seconds(days)
9
7
  (24 * 3600) * days.to_f
10
8
  end
11
9
 
12
10
  def initialize(sincedb_value_expiry)
13
11
  @sincedb_value_expiry = sincedb_value_expiry
14
- @expired_keys = []
15
- end
16
-
17
- def update_sincedb_value_expiry_from_days(days)
18
- @sincedb_value_expiry = SincedbRecordSerializer.days_to_seconds(days)
19
12
  end
20
13
 
14
+ # @return Array expired keys (ones that were not written to the file)
21
15
  def serialize(db, io, as_of = Time.now.to_f)
22
- @expired_keys.clear
16
+ expired_keys = []
23
17
  db.each do |key, value|
24
18
  if as_of > value.last_changed_at_expires(@sincedb_value_expiry)
25
- @expired_keys << key
19
+ expired_keys << key
26
20
  next
27
21
  end
28
22
  io.write(serialize_record(key, value))
29
23
  end
24
+ expired_keys
30
25
  end
31
26
 
32
27
  def deserialize(io)
@@ -36,8 +31,7 @@ module FileWatch
36
31
  end
37
32
 
38
33
  def serialize_record(k, v)
39
- # effectively InodeStruct#to_s SincedbValue#to_s
40
- "#{k} #{v}\n"
34
+ "#{k} #{v}\n" # effectively InodeStruct#to_s SincedbValue#to_s
41
35
  end
42
36
 
43
37
  def deserialize_record(record)
@@ -18,7 +18,7 @@ module FileWatch module TailMode module Handlers
18
18
  end
19
19
 
20
20
  def handle(watched_file)
21
- logger.trace("handling: #{watched_file.filename}")
21
+ logger.trace? && logger.trace("handling:", :path => watched_file.path)
22
22
  unless watched_file.has_listener?
23
23
  watched_file.set_listener(@observer)
24
24
  end
@@ -37,7 +37,7 @@ module FileWatch module TailMode module Handlers
37
37
 
38
38
  def controlled_read(watched_file, loop_control)
39
39
  changed = false
40
- logger.trace("reading...", "iterations" => loop_control.count, "amount" => loop_control.size, "filename" => watched_file.filename)
40
+ logger.trace? && logger.trace(__method__.to_s, :iterations => loop_control.count, :amount => loop_control.size, :filename => watched_file.filename)
41
41
  # from a real config (has 102 file inputs)
42
42
  # -- This cfg creates a file input for every log file to create a dedicated file pointer and read all file simultaneously
43
43
  # -- If we put all log files in one file input glob we will have indexing delay, because Logstash waits until the first file becomes EOF
@@ -48,7 +48,7 @@ module FileWatch module TailMode module Handlers
48
48
  loop_control.count.times do
49
49
  break if quit?
50
50
  begin
51
- logger.debug("read_to_eof: get chunk")
51
+ logger.debug? && logger.debug("#{__method__} get chunk")
52
52
  result = watched_file.read_extract_lines(loop_control.size) # expect BufferExtractResult
53
53
  logger.trace(result.warning, result.additional) unless result.warning.empty?
54
54
  changed = true
@@ -57,40 +57,42 @@ module FileWatch module TailMode module Handlers
57
57
  # sincedb position is now independent from the watched_file bytes_read
58
58
  sincedb_collection.increment(watched_file.sincedb_key, line.bytesize + @settings.delimiter_byte_size)
59
59
  end
60
- rescue EOFError
60
+ rescue EOFError => e
61
61
  # it only makes sense to signal EOF in "read" mode not "tail"
62
+ logger.debug(__method__.to_s, exception_details(watched_file.path, e, false))
62
63
  loop_control.flag_read_error
63
64
  break
64
- rescue Errno::EWOULDBLOCK, Errno::EINTR
65
+ rescue Errno::EWOULDBLOCK, Errno::EINTR => e
66
+ logger.debug(__method__.to_s, exception_details(watched_file.path, e, false))
65
67
  watched_file.listener.error
66
68
  loop_control.flag_read_error
67
69
  break
68
70
  rescue => e
69
- logger.error("read_to_eof: general error reading #{watched_file.path}", "error" => e.inspect, "backtrace" => e.backtrace.take(4))
71
+ logger.error("#{__method__} general error reading", exception_details(watched_file.path, e))
70
72
  watched_file.listener.error
71
73
  loop_control.flag_read_error
72
74
  break
73
75
  end
74
76
  end
75
- logger.debug("read_to_eof: exit due to quit") if quit?
77
+ logger.debug("#{__method__} stopped loop due quit") if quit?
76
78
  sincedb_collection.request_disk_flush if changed
77
79
  end
78
80
 
79
81
  def open_file(watched_file)
80
82
  return true if watched_file.file_open?
81
- logger.trace("opening #{watched_file.filename}")
83
+ logger.trace? && logger.trace("open_file", :filename => watched_file.filename)
82
84
  begin
83
85
  watched_file.open
84
- rescue
86
+ rescue => e
85
87
  # don't emit this message too often. if a file that we can't
86
88
  # read is changing a lot, we'll try to open it more often, and spam the logs.
87
89
  now = Time.now.to_i
88
- logger.trace("open_file OPEN_WARN_INTERVAL is '#{OPEN_WARN_INTERVAL}'")
90
+ logger.trace? && logger.trace("open_file OPEN_WARN_INTERVAL is '#{OPEN_WARN_INTERVAL}'")
89
91
  if watched_file.last_open_warning_at.nil? || now - watched_file.last_open_warning_at > OPEN_WARN_INTERVAL
90
- logger.warn("failed to open #{watched_file.path}: #{$!.inspect}, #{$!.backtrace.take(3)}")
92
+ logger.warn("failed to open file", exception_details(watched_file.path, e))
91
93
  watched_file.last_open_warning_at = now
92
94
  else
93
- logger.trace("suppressed warning for `failed to open` #{watched_file.path}: #{$!.inspect}")
95
+ logger.debug("open_file suppressed warning `failed to open file`", exception_details(watched_file.path, e, false))
94
96
  end
95
97
  watched_file.watch # set it back to watch so we can try it again
96
98
  else
@@ -108,26 +110,22 @@ module FileWatch module TailMode module Handlers
108
110
  update_existing_sincedb_collection_value(watched_file, sincedb_value)
109
111
  watched_file.initial_completed
110
112
  else
111
- msg = "add_or_update_sincedb_collection: found sincedb record"
112
- logger.trace(msg,
113
- "sincedb key" => watched_file.sincedb_key,
114
- "sincedb value" => sincedb_value
115
- )
113
+ logger.trace? && logger.trace("add_or_update_sincedb_collection: found sincedb record",
114
+ :sincedb_key => watched_file.sincedb_key, :sincedb_value => sincedb_value)
116
115
  # detected a rotation, Discoverer can't handle this because this watched file is not a new discovery.
117
116
  # we must handle it here, by transferring state and have the sincedb value track this watched file
118
117
  # rotate_as_file and rotate_from will switch the sincedb key to the inode that the path is now pointing to
119
118
  # and pickup the sincedb_value from before.
120
- msg = "add_or_update_sincedb_collection: the found sincedb_value has a watched_file - this is a rename, switching inode to this watched file"
121
- logger.trace(msg)
119
+ logger.debug("add_or_update_sincedb_collection: the found sincedb_value has a watched_file - this is a rename, switching inode to this watched file")
122
120
  existing_watched_file = sincedb_value.watched_file
123
121
  if existing_watched_file.nil?
124
122
  sincedb_value.set_watched_file(watched_file)
125
- logger.trace("add_or_update_sincedb_collection: switching as new file")
123
+ logger.trace? && logger.trace("add_or_update_sincedb_collection: switching as new file")
126
124
  watched_file.rotate_as_file
127
125
  watched_file.update_bytes_read(sincedb_value.position)
128
126
  else
129
127
  sincedb_value.set_watched_file(watched_file)
130
- logger.trace("add_or_update_sincedb_collection: switching from...", "watched_file details" => watched_file.details)
128
+ logger.trace? && logger.trace("add_or_update_sincedb_collection: switching from:", :watched_file => watched_file.details)
131
129
  watched_file.rotate_from(existing_watched_file)
132
130
  end
133
131
  end
@@ -135,13 +133,15 @@ module FileWatch module TailMode module Handlers
135
133
  end
136
134
 
137
135
  def update_existing_sincedb_collection_value(watched_file, sincedb_value)
138
- logger.trace("update_existing_sincedb_collection_value: #{watched_file.filename}, last value #{sincedb_value.position}, cur size #{watched_file.last_stat_size}")
136
+ logger.trace? && logger.trace("update_existing_sincedb_collection_value", :position => sincedb_value.position,
137
+ :filename => watched_file.filename, :last_stat_size => watched_file.last_stat_size)
139
138
  update_existing_specifically(watched_file, sincedb_value)
140
139
  end
141
140
 
142
141
  def add_new_value_sincedb_collection(watched_file)
143
142
  sincedb_value = get_new_value_specifically(watched_file)
144
- logger.trace("add_new_value_sincedb_collection", "position" => sincedb_value.position, "watched_file details" => watched_file.details)
143
+ logger.trace? && logger.trace("add_new_value_sincedb_collection", :position => sincedb_value.position,
144
+ :watched_file => watched_file.details)
145
145
  sincedb_collection.set(watched_file.sincedb_key, sincedb_value)
146
146
  sincedb_value
147
147
  end
@@ -153,5 +153,14 @@ module FileWatch module TailMode module Handlers
153
153
  watched_file.update_bytes_read(position)
154
154
  value
155
155
  end
156
+
157
+ private
158
+
159
+ def exception_details(path, e, trace = true)
160
+ details = { :path => path, :exception => e.class, :message => e.message }
161
+ details[:backtrace] = e.backtrace if trace && logger.debug?
162
+ details
163
+ end
164
+
156
165
  end
157
166
  end end end
@@ -7,7 +7,7 @@ module FileWatch module TailMode module Handlers
7
7
  # TODO consider trying to find the renamed file - it will have the same inode.
8
8
  # Needs a rotate scheme rename hint from user e.g. "<name>-YYYY-MM-DD-N.<ext>" or "<name>.<ext>.N"
9
9
  # send the found content to the same listener (stream identity)
10
- logger.trace("delete", :path => watched_file.path, :watched_file => watched_file.details)
10
+ logger.trace? && logger.trace(__method__.to_s, :path => watched_file.path, :watched_file => watched_file.details)
11
11
  if watched_file.bytes_unread > 0
12
12
  logger.warn(DATA_LOSS_WARNING, :path => watched_file.path, :unread_bytes => watched_file.bytes_unread)
13
13
  end
@@ -14,11 +14,10 @@ module FileWatch module TailMode module Handlers
14
14
  end
15
15
 
16
16
  def update_existing_specifically(watched_file, sincedb_value)
17
- # we have a match but size is smaller
18
- # set all to zero
17
+ # we have a match but size is smaller - set all to zero
19
18
  watched_file.reset_bytes_unread
20
19
  sincedb_value.update_position(0)
21
- logger.trace("update_existing_specifically: was truncated seeking to beginning", "watched file" => watched_file.details, "sincedb value" => sincedb_value)
20
+ logger.trace? && logger.trace("update_existing_specifically: was truncated seeking to beginning", :watched_file => watched_file.details, :sincedb_value => sincedb_value)
22
21
  end
23
22
  end
24
23
  end end end
@@ -13,9 +13,9 @@ module FileWatch module TailMode module Handlers
13
13
  # for file initially ignored their bytes_read was set to stat.size
14
14
  # use this value not the `start_new_files_at` for the position
15
15
  # logger.trace("get_new_value_specifically", "watched_file" => watched_file.inspect)
16
- SincedbValue.new(watched_file.bytes_read).tap do |val|
17
- val.set_watched_file(watched_file)
18
- logger.trace("-------------------- >>>>> get_new_value_specifically: unignore", "watched file" => watched_file.details, "sincedb value" => val)
16
+ SincedbValue.new(watched_file.bytes_read).tap do |sincedb_value|
17
+ sincedb_value.set_watched_file(watched_file)
18
+ logger.trace? && logger.trace("get_new_value_specifically: unignore", :watched_file => watched_file.details, :sincedb_value => sincedb_value)
19
19
  end
20
20
  end
21
21
 
@@ -26,7 +26,7 @@ module FileWatch module TailMode module Handlers
26
26
  # we will handle grow or shrink
27
27
  # for now we seek to where we were before the file got ignored (grow)
28
28
  # or to the start (shrink)
29
- logger.trace("-------------------- >>>>> update_existing_specifically: unignore", "watched file" => watched_file.details, "sincedb value" => sincedb_value)
29
+ logger.trace? && logger.trace("update_existing_specifically: unignore", :watched_file => watched_file.details, :sincedb_value => sincedb_value)
30
30
  position = 0
31
31
  if watched_file.shrunk?
32
32
  watched_file.update_bytes_read(0)
Binary file
@@ -377,12 +377,12 @@ class File < LogStash::Inputs::Base
377
377
  def handle_deletable_path(path)
378
378
  return if tail_mode?
379
379
  return if @completed_file_handlers.empty?
380
+ @logger.debug? && @logger.debug(__method__.to_s, :path => path)
380
381
  @completed_file_handlers.each { |handler| handler.handle(path) }
381
382
  end
382
383
 
383
384
  def log_line_received(path, line)
384
- return unless @logger.debug?
385
- @logger.debug("Received line", :path => path, :text => line)
385
+ @logger.debug? && @logger.debug("Received line", :path => path, :text => line)
386
386
  end
387
387
 
388
388
  def stop
@@ -7,9 +7,9 @@ module LogStash module Inputs
7
7
  class FileListener
8
8
  attr_reader :input, :path, :data
9
9
  # construct with link back to the input plugin instance.
10
- def initialize(path, input)
10
+ def initialize(path, input, data = nil)
11
11
  @path, @input = path, input
12
- @data = nil
12
+ @data = data
13
13
  end
14
14
 
15
15
  def opened
@@ -36,7 +36,7 @@ module LogStash module Inputs
36
36
  def accept(data)
37
37
  # and push transient data filled dup listener downstream
38
38
  input.log_line_received(path, data)
39
- input.codec.accept(dup_adding_state(data))
39
+ input.codec.accept(self.class.new(path, input, data))
40
40
  end
41
41
 
42
42
  def process_event(event)
@@ -45,17 +45,6 @@ module LogStash module Inputs
45
45
  input.post_process_this(event)
46
46
  end
47
47
 
48
- def add_state(data)
49
- @data = data
50
- self
51
- end
52
-
53
- private
54
-
55
- # duplicate and add state for downstream
56
- def dup_adding_state(line)
57
- self.class.new(path, input).add_state(line)
58
- end
59
48
  end
60
49
 
61
50
  class FlushableListener < FileListener
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-input-file'
4
- s.version = '4.2.2'
4
+ s.version = '4.2.3'
5
5
  s.licenses = ['Apache-2.0']
6
6
  s.summary = "Streams events from files"
7
7
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -9,7 +9,9 @@ module FileWatch
9
9
  let(:io) { StringIO.new }
10
10
  let(:db) { Hash.new }
11
11
 
12
- subject { SincedbRecordSerializer.new(SincedbRecordSerializer.days_to_seconds(14)) }
12
+ let(:sincedb_value_expiry) { SincedbRecordSerializer.days_to_seconds(14) }
13
+
14
+ subject { SincedbRecordSerializer.new(sincedb_value_expiry) }
13
15
 
14
16
  context "deserialize from IO" do
15
17
  it 'reads V1 records' do
@@ -82,8 +84,10 @@ module FileWatch
82
84
  end
83
85
 
84
86
  context "given a non default `sincedb_clean_after`" do
87
+
88
+ let(:sincedb_value_expiry) { SincedbRecordSerializer.days_to_seconds(2) }
89
+
85
90
  it "does not write expired db entries to an IO object" do
86
- subject.update_sincedb_value_expiry_from_days(2)
87
91
  one_day_ago = Time.now.to_f - (1.0*24*3600)
88
92
  three_days_ago = one_day_ago - (2.0*24*3600)
89
93
  db[InodeStruct.new("42424242", 2, 5)] = SincedbValue.new(42, one_day_ago)
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-input-file
3
3
  version: !ruby/object:Gem::Version
4
- version: 4.2.2
4
+ version: 4.2.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-09-25 00:00:00.000000000 Z
11
+ date: 2020-12-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement