logstash-input-file 4.2.2 → 4.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: f8c11df0b47e51ceb5ebd37b3d16eb28acfef360aa976d4155752fffca3c0161
4
- data.tar.gz: 297b2410e81e2f4228f59e459e486269238820b82f94a577068c53c87b9f2ea7
3
+ metadata.gz: b9475678fd2320f61bf61a3e34884ea6512072934e96f2d16c08a4055ad5f8a9
4
+ data.tar.gz: ca48090438f9ba71d20cbaf24111fa479fabe2279b3704d092aa4df36c2680e9
5
5
  SHA512:
6
- metadata.gz: 639fbc4dd7b2f02977923726f997e8ecf2d57862ac6cf20dd284f2a1b55e0532eb0137d1b81be200afc10bc5a8f84c625e74211bb685a45099c4660b36ad15eb
7
- data.tar.gz: ecee35a5081b61038e3f9052cbb447269d1a5def40f47fbbb1cb2f88f0ee509e98995f9441e7cf37329d505ee095087e58e6740856d21253ef59b1d40a25fa10
6
+ metadata.gz: 97b642eed6db061be8fc620ff66e4b89e5898fd23bad138407509db20294ec013bba3a4dfe10dfc4587bdea2c09f8f99cd468db38d46a5a82028be5c85730801
7
+ data.tar.gz: b1583d047f9c042133584f3a82673b661c1eb59f7586a7c39df807a5ec391785dad2230df8561e339be673ef3f799c65e1282eac369459fccbbc491227882121
@@ -1,3 +1,6 @@
1
+ ## 4.2.3
2
+ - Refactor: improve debug logging (log catched exceptions) [#280](https://github.com/logstash-plugins/logstash-input-file/pull/280)
3
+
1
4
  ## 4.2.2
2
5
  - Fix: sincedb_clean_after not being respected [#276](https://github.com/logstash-plugins/logstash-input-file/pull/276)
3
6
 
@@ -19,7 +19,7 @@ module FileWatch module ReadMode module Handlers
19
19
  end
20
20
 
21
21
  def handle(watched_file)
22
- logger.trace("handling: #{watched_file.path}")
22
+ logger.trace? && logger.trace("handling:", :path => watched_file.path)
23
23
  unless watched_file.has_listener?
24
24
  watched_file.set_listener(@observer)
25
25
  end
@@ -41,14 +41,14 @@ module FileWatch module ReadMode module Handlers
41
41
  # don't emit this message too often. if a file that we can't
42
42
  # read is changing a lot, we'll try to open it more often, and spam the logs.
43
43
  now = Time.now.to_i
44
- logger.trace("opening OPEN_WARN_INTERVAL is '#{OPEN_WARN_INTERVAL}'")
44
+ logger.trace? && logger.trace("opening OPEN_WARN_INTERVAL is '#{OPEN_WARN_INTERVAL}'")
45
45
  if watched_file.last_open_warning_at.nil? || now - watched_file.last_open_warning_at > OPEN_WARN_INTERVAL
46
46
  backtrace = e.backtrace
47
47
  backtrace = backtrace.take(3) if backtrace && !logger.debug?
48
48
  logger.warn("failed to open", :path => watched_file.path, :exception => e.class, :message => e.message, :backtrace => backtrace)
49
49
  watched_file.last_open_warning_at = now
50
50
  else
51
- logger.trace("suppressed warning (failed to open)", :path => watched_file.path, :exception => e.class, :message => e.message)
51
+ logger.trace? && logger.trace("suppressed warning (failed to open)", :path => watched_file.path, :exception => e.class, :message => e.message)
52
52
  end
53
53
  watched_file.watch # set it back to watch so we can try it again
54
54
  end
@@ -67,8 +67,7 @@ module FileWatch module ReadMode module Handlers
67
67
  elsif sincedb_value.watched_file == watched_file
68
68
  update_existing_sincedb_collection_value(watched_file, sincedb_value)
69
69
  else
70
- msg = "add_or_update_sincedb_collection: the found sincedb_value has a watched_file - this is a rename, switching inode to this watched file"
71
- logger.trace(msg)
70
+ logger.trace? && logger.trace("add_or_update_sincedb_collection: the found sincedb_value has a watched_file - this is a rename, switching inode to this watched file")
72
71
  existing_watched_file = sincedb_value.watched_file
73
72
  if existing_watched_file.nil?
74
73
  sincedb_value.set_watched_file(watched_file)
@@ -77,7 +76,7 @@ module FileWatch module ReadMode module Handlers
77
76
  watched_file.update_bytes_read(sincedb_value.position)
78
77
  else
79
78
  sincedb_value.set_watched_file(watched_file)
80
- logger.trace("add_or_update_sincedb_collection: switching from", :watched_file => watched_file.details)
79
+ logger.trace? && logger.trace("add_or_update_sincedb_collection: switching from", :watched_file => watched_file.details)
81
80
  watched_file.rotate_from(existing_watched_file)
82
81
  end
83
82
 
@@ -86,7 +85,7 @@ module FileWatch module ReadMode module Handlers
86
85
  end
87
86
 
88
87
  def update_existing_sincedb_collection_value(watched_file, sincedb_value)
89
- logger.trace("update_existing_sincedb_collection_value: #{watched_file.path}, last value #{sincedb_value.position}, cur size #{watched_file.last_stat_size}")
88
+ logger.trace? && logger.trace("update_existing_sincedb_collection_value: #{watched_file.path}, last value #{sincedb_value.position}, cur size #{watched_file.last_stat_size}")
90
89
  # sincedb_value is the source of truth
91
90
  watched_file.update_bytes_read(sincedb_value.position)
92
91
  end
@@ -94,7 +93,7 @@ module FileWatch module ReadMode module Handlers
94
93
  def add_new_value_sincedb_collection(watched_file)
95
94
  sincedb_value = SincedbValue.new(0)
96
95
  sincedb_value.set_watched_file(watched_file)
97
- logger.trace("add_new_value_sincedb_collection:", :path => watched_file.path, :position => sincedb_value.position)
96
+ logger.trace? && logger.trace("add_new_value_sincedb_collection:", :path => watched_file.path, :position => sincedb_value.position)
98
97
  sincedb_collection.set(watched_file.sincedb_key, sincedb_value)
99
98
  end
100
99
  end
@@ -47,16 +47,16 @@ module FileWatch
47
47
  @time_sdb_opened = Time.now.to_f
48
48
  begin
49
49
  path.open do |file|
50
- logger.trace("open: reading from #{path}")
50
+ logger.debug("open: reading from #{path}")
51
51
  @serializer.deserialize(file) do |key, value|
52
- logger.trace("open: importing ... '#{key}' => '#{value}'")
52
+ logger.trace? && logger.trace("open: importing #{key.inspect} => #{value.inspect}")
53
53
  set_key_value(key, value)
54
54
  end
55
55
  end
56
56
  logger.trace("open: count of keys read: #{@sincedb.keys.size}")
57
57
  rescue => e
58
58
  #No existing sincedb to load
59
- logger.trace("open: error:", :path => path, :exception => e.class, :message => e.message)
59
+ logger.debug("open: error opening #{path}", :exception => e.class, :message => e.message)
60
60
  end
61
61
  end
62
62
 
@@ -68,35 +68,32 @@ module FileWatch
68
68
  # and due to the window handling of many files
69
69
  # this file may not be opened in this session.
70
70
  # a new value will be added when the file is opened
71
- logger.trace("associate: unmatched")
71
+ logger.trace("associate: unmatched", :filename => watched_file.filename)
72
72
  return true
73
73
  end
74
74
  logger.trace? && logger.trace("associate: found sincedb record", :filename => watched_file.filename,
75
75
  :sincedb_key => watched_file.sincedb_key, :sincedb_value => sincedb_value)
76
- if sincedb_value.watched_file.nil?
77
- # not associated
76
+ if sincedb_value.watched_file.nil? # not associated
78
77
  if sincedb_value.path_in_sincedb.nil?
79
78
  handle_association(sincedb_value, watched_file)
80
- logger.trace("associate: inode matched but no path in sincedb")
79
+ logger.trace? && logger.trace("associate: inode matched but no path in sincedb", :filename => watched_file.filename)
81
80
  return true
82
81
  end
83
82
  if sincedb_value.path_in_sincedb == watched_file.path
84
- # the path on disk is the same as discovered path
85
- # and the inode is the same.
83
+ # the path on disk is the same as discovered path and the inode is the same.
86
84
  handle_association(sincedb_value, watched_file)
87
- logger.trace("associate: inode and path matched")
85
+ logger.trace? && logger.trace("associate: inode and path matched", :filename => watched_file.filename)
88
86
  return true
89
87
  end
90
- # the path on disk is different from discovered unassociated path
91
- # but they have the same key (inode)
88
+ # the path on disk is different from discovered unassociated path but they have the same key (inode)
92
89
  # treat as a new file, a new value will be added when the file is opened
93
90
  sincedb_value.clear_watched_file
94
91
  delete(watched_file.sincedb_key)
95
- logger.trace("associate: matched but allocated to another")
92
+ logger.trace? && logger.trace("associate: matched but allocated to another", :filename => watched_file.filename)
96
93
  return true
97
94
  end
98
95
  if sincedb_value.watched_file.equal?(watched_file) # pointer equals
99
- logger.trace("associate: already associated")
96
+ logger.trace? && logger.trace("associate: already associated", :filename => watched_file.filename)
100
97
  return true
101
98
  end
102
99
  # sincedb_value.watched_file is not this discovered watched_file but they have the same key (inode)
@@ -107,7 +104,7 @@ module FileWatch
107
104
  # after the original is deleted
108
105
  # are not yet in the delete phase, let this play out
109
106
  existing_watched_file = sincedb_value.watched_file
110
- logger.trace? && logger.trace("----------------- >> associate: the found sincedb_value has a watched_file - this is a rename",
107
+ logger.trace? && logger.trace("associate: found sincedb_value has a watched_file - this is a rename",
111
108
  :this_watched_file => watched_file.details, :existing_watched_file => existing_watched_file.details)
112
109
  watched_file.rotation_in_progress
113
110
  true
@@ -197,43 +194,43 @@ module FileWatch
197
194
  watched_file.initial_completed
198
195
  if watched_file.all_read?
199
196
  watched_file.ignore
200
- logger.trace? && logger.trace("handle_association fully read, ignoring.....", :watched_file => watched_file.details, :sincedb_value => sincedb_value)
197
+ logger.trace? && logger.trace("handle_association fully read, ignoring", :watched_file => watched_file.details, :sincedb_value => sincedb_value)
201
198
  end
202
199
  end
203
200
 
204
201
  def set_key_value(key, value)
205
202
  if @time_sdb_opened < value.last_changed_at_expires(@settings.sincedb_expiry_duration)
206
- logger.trace("open: setting #{key.inspect} to #{value.inspect}")
207
203
  set(key, value)
208
204
  else
209
- logger.trace("open: record has expired, skipping: #{key.inspect} #{value.inspect}")
205
+ logger.debug("set_key_value: record has expired, skipping: #{key.inspect} => #{value.inspect}")
210
206
  end
211
207
  end
212
208
 
213
209
  def sincedb_write(time = Time.now)
214
- logger.trace("sincedb_write: #{path} (time = #{time})")
210
+ logger.trace? && logger.trace("sincedb_write: #{path} (time = #{time})")
215
211
  begin
216
- @write_method.call(time)
217
- @serializer.expired_keys.each do |key|
212
+ expired_keys = @write_method.call(time)
213
+ expired_keys.each do |key|
218
214
  @sincedb[key].unset_watched_file
219
215
  delete(key)
220
216
  logger.trace? && logger.trace("sincedb_write: cleaned", :key => key)
221
217
  end
222
218
  @sincedb_last_write = time.to_i
223
219
  @write_requested = false
224
- rescue Errno::EACCES
225
- # no file handles free perhaps
226
- # maybe it will work next time
227
- logger.trace("sincedb_write: #{path} error: #{$!}")
220
+ rescue Errno::EACCES => e
221
+ # no file handles free perhaps - maybe it will work next time
222
+ logger.debug("sincedb_write: #{path} error:", :exception => e.class, :message => e.message)
228
223
  end
229
224
  end
230
225
 
226
+ # @return expired keys
231
227
  def atomic_write(time)
232
228
  FileHelper.write_atomically(@full_path) do |io|
233
229
  @serializer.serialize(@sincedb, io, time.to_f)
234
230
  end
235
231
  end
236
232
 
233
+ # @return expired keys
237
234
  def non_atomic_write(time)
238
235
  IO.open(IO.sysopen(@full_path, "w+")) do |io|
239
236
  @serializer.serialize(@sincedb, io, time.to_f)
@@ -3,30 +3,25 @@
3
3
  module FileWatch
4
4
  class SincedbRecordSerializer
5
5
 
6
- attr_reader :expired_keys
7
-
8
6
  def self.days_to_seconds(days)
9
7
  (24 * 3600) * days.to_f
10
8
  end
11
9
 
12
10
  def initialize(sincedb_value_expiry)
13
11
  @sincedb_value_expiry = sincedb_value_expiry
14
- @expired_keys = []
15
- end
16
-
17
- def update_sincedb_value_expiry_from_days(days)
18
- @sincedb_value_expiry = SincedbRecordSerializer.days_to_seconds(days)
19
12
  end
20
13
 
14
+ # @return Array expired keys (ones that were not written to the file)
21
15
  def serialize(db, io, as_of = Time.now.to_f)
22
- @expired_keys.clear
16
+ expired_keys = []
23
17
  db.each do |key, value|
24
18
  if as_of > value.last_changed_at_expires(@sincedb_value_expiry)
25
- @expired_keys << key
19
+ expired_keys << key
26
20
  next
27
21
  end
28
22
  io.write(serialize_record(key, value))
29
23
  end
24
+ expired_keys
30
25
  end
31
26
 
32
27
  def deserialize(io)
@@ -36,8 +31,7 @@ module FileWatch
36
31
  end
37
32
 
38
33
  def serialize_record(k, v)
39
- # effectively InodeStruct#to_s SincedbValue#to_s
40
- "#{k} #{v}\n"
34
+ "#{k} #{v}\n" # effectively InodeStruct#to_s SincedbValue#to_s
41
35
  end
42
36
 
43
37
  def deserialize_record(record)
@@ -18,7 +18,7 @@ module FileWatch module TailMode module Handlers
18
18
  end
19
19
 
20
20
  def handle(watched_file)
21
- logger.trace("handling: #{watched_file.filename}")
21
+ logger.trace? && logger.trace("handling:", :path => watched_file.path)
22
22
  unless watched_file.has_listener?
23
23
  watched_file.set_listener(@observer)
24
24
  end
@@ -37,7 +37,7 @@ module FileWatch module TailMode module Handlers
37
37
 
38
38
  def controlled_read(watched_file, loop_control)
39
39
  changed = false
40
- logger.trace("reading...", "iterations" => loop_control.count, "amount" => loop_control.size, "filename" => watched_file.filename)
40
+ logger.trace? && logger.trace(__method__.to_s, :iterations => loop_control.count, :amount => loop_control.size, :filename => watched_file.filename)
41
41
  # from a real config (has 102 file inputs)
42
42
  # -- This cfg creates a file input for every log file to create a dedicated file pointer and read all file simultaneously
43
43
  # -- If we put all log files in one file input glob we will have indexing delay, because Logstash waits until the first file becomes EOF
@@ -48,7 +48,7 @@ module FileWatch module TailMode module Handlers
48
48
  loop_control.count.times do
49
49
  break if quit?
50
50
  begin
51
- logger.debug("read_to_eof: get chunk")
51
+ logger.debug? && logger.debug("#{__method__} get chunk")
52
52
  result = watched_file.read_extract_lines(loop_control.size) # expect BufferExtractResult
53
53
  logger.trace(result.warning, result.additional) unless result.warning.empty?
54
54
  changed = true
@@ -57,40 +57,42 @@ module FileWatch module TailMode module Handlers
57
57
  # sincedb position is now independent from the watched_file bytes_read
58
58
  sincedb_collection.increment(watched_file.sincedb_key, line.bytesize + @settings.delimiter_byte_size)
59
59
  end
60
- rescue EOFError
60
+ rescue EOFError => e
61
61
  # it only makes sense to signal EOF in "read" mode not "tail"
62
+ logger.debug(__method__.to_s, exception_details(watched_file.path, e, false))
62
63
  loop_control.flag_read_error
63
64
  break
64
- rescue Errno::EWOULDBLOCK, Errno::EINTR
65
+ rescue Errno::EWOULDBLOCK, Errno::EINTR => e
66
+ logger.debug(__method__.to_s, exception_details(watched_file.path, e, false))
65
67
  watched_file.listener.error
66
68
  loop_control.flag_read_error
67
69
  break
68
70
  rescue => e
69
- logger.error("read_to_eof: general error reading #{watched_file.path}", "error" => e.inspect, "backtrace" => e.backtrace.take(4))
71
+ logger.error("#{__method__} general error reading", exception_details(watched_file.path, e))
70
72
  watched_file.listener.error
71
73
  loop_control.flag_read_error
72
74
  break
73
75
  end
74
76
  end
75
- logger.debug("read_to_eof: exit due to quit") if quit?
77
+ logger.debug("#{__method__} stopped loop due quit") if quit?
76
78
  sincedb_collection.request_disk_flush if changed
77
79
  end
78
80
 
79
81
  def open_file(watched_file)
80
82
  return true if watched_file.file_open?
81
- logger.trace("opening #{watched_file.filename}")
83
+ logger.trace? && logger.trace("open_file", :filename => watched_file.filename)
82
84
  begin
83
85
  watched_file.open
84
- rescue
86
+ rescue => e
85
87
  # don't emit this message too often. if a file that we can't
86
88
  # read is changing a lot, we'll try to open it more often, and spam the logs.
87
89
  now = Time.now.to_i
88
- logger.trace("open_file OPEN_WARN_INTERVAL is '#{OPEN_WARN_INTERVAL}'")
90
+ logger.trace? && logger.trace("open_file OPEN_WARN_INTERVAL is '#{OPEN_WARN_INTERVAL}'")
89
91
  if watched_file.last_open_warning_at.nil? || now - watched_file.last_open_warning_at > OPEN_WARN_INTERVAL
90
- logger.warn("failed to open #{watched_file.path}: #{$!.inspect}, #{$!.backtrace.take(3)}")
92
+ logger.warn("failed to open file", exception_details(watched_file.path, e))
91
93
  watched_file.last_open_warning_at = now
92
94
  else
93
- logger.trace("suppressed warning for `failed to open` #{watched_file.path}: #{$!.inspect}")
95
+ logger.debug("open_file suppressed warning `failed to open file`", exception_details(watched_file.path, e, false))
94
96
  end
95
97
  watched_file.watch # set it back to watch so we can try it again
96
98
  else
@@ -108,26 +110,22 @@ module FileWatch module TailMode module Handlers
108
110
  update_existing_sincedb_collection_value(watched_file, sincedb_value)
109
111
  watched_file.initial_completed
110
112
  else
111
- msg = "add_or_update_sincedb_collection: found sincedb record"
112
- logger.trace(msg,
113
- "sincedb key" => watched_file.sincedb_key,
114
- "sincedb value" => sincedb_value
115
- )
113
+ logger.trace? && logger.trace("add_or_update_sincedb_collection: found sincedb record",
114
+ :sincedb_key => watched_file.sincedb_key, :sincedb_value => sincedb_value)
116
115
  # detected a rotation, Discoverer can't handle this because this watched file is not a new discovery.
117
116
  # we must handle it here, by transferring state and have the sincedb value track this watched file
118
117
  # rotate_as_file and rotate_from will switch the sincedb key to the inode that the path is now pointing to
119
118
  # and pickup the sincedb_value from before.
120
- msg = "add_or_update_sincedb_collection: the found sincedb_value has a watched_file - this is a rename, switching inode to this watched file"
121
- logger.trace(msg)
119
+ logger.debug("add_or_update_sincedb_collection: the found sincedb_value has a watched_file - this is a rename, switching inode to this watched file")
122
120
  existing_watched_file = sincedb_value.watched_file
123
121
  if existing_watched_file.nil?
124
122
  sincedb_value.set_watched_file(watched_file)
125
- logger.trace("add_or_update_sincedb_collection: switching as new file")
123
+ logger.trace? && logger.trace("add_or_update_sincedb_collection: switching as new file")
126
124
  watched_file.rotate_as_file
127
125
  watched_file.update_bytes_read(sincedb_value.position)
128
126
  else
129
127
  sincedb_value.set_watched_file(watched_file)
130
- logger.trace("add_or_update_sincedb_collection: switching from...", "watched_file details" => watched_file.details)
128
+ logger.trace? && logger.trace("add_or_update_sincedb_collection: switching from:", :watched_file => watched_file.details)
131
129
  watched_file.rotate_from(existing_watched_file)
132
130
  end
133
131
  end
@@ -135,13 +133,15 @@ module FileWatch module TailMode module Handlers
135
133
  end
136
134
 
137
135
  def update_existing_sincedb_collection_value(watched_file, sincedb_value)
138
- logger.trace("update_existing_sincedb_collection_value: #{watched_file.filename}, last value #{sincedb_value.position}, cur size #{watched_file.last_stat_size}")
136
+ logger.trace? && logger.trace("update_existing_sincedb_collection_value", :position => sincedb_value.position,
137
+ :filename => watched_file.filename, :last_stat_size => watched_file.last_stat_size)
139
138
  update_existing_specifically(watched_file, sincedb_value)
140
139
  end
141
140
 
142
141
  def add_new_value_sincedb_collection(watched_file)
143
142
  sincedb_value = get_new_value_specifically(watched_file)
144
- logger.trace("add_new_value_sincedb_collection", "position" => sincedb_value.position, "watched_file details" => watched_file.details)
143
+ logger.trace? && logger.trace("add_new_value_sincedb_collection", :position => sincedb_value.position,
144
+ :watched_file => watched_file.details)
145
145
  sincedb_collection.set(watched_file.sincedb_key, sincedb_value)
146
146
  sincedb_value
147
147
  end
@@ -153,5 +153,14 @@ module FileWatch module TailMode module Handlers
153
153
  watched_file.update_bytes_read(position)
154
154
  value
155
155
  end
156
+
157
+ private
158
+
159
+ def exception_details(path, e, trace = true)
160
+ details = { :path => path, :exception => e.class, :message => e.message }
161
+ details[:backtrace] = e.backtrace if trace && logger.debug?
162
+ details
163
+ end
164
+
156
165
  end
157
166
  end end end
@@ -7,7 +7,7 @@ module FileWatch module TailMode module Handlers
7
7
  # TODO consider trying to find the renamed file - it will have the same inode.
8
8
  # Needs a rotate scheme rename hint from user e.g. "<name>-YYYY-MM-DD-N.<ext>" or "<name>.<ext>.N"
9
9
  # send the found content to the same listener (stream identity)
10
- logger.trace("delete", :path => watched_file.path, :watched_file => watched_file.details)
10
+ logger.trace? && logger.trace(__method__.to_s, :path => watched_file.path, :watched_file => watched_file.details)
11
11
  if watched_file.bytes_unread > 0
12
12
  logger.warn(DATA_LOSS_WARNING, :path => watched_file.path, :unread_bytes => watched_file.bytes_unread)
13
13
  end
@@ -14,11 +14,10 @@ module FileWatch module TailMode module Handlers
14
14
  end
15
15
 
16
16
  def update_existing_specifically(watched_file, sincedb_value)
17
- # we have a match but size is smaller
18
- # set all to zero
17
+ # we have a match but size is smaller - set all to zero
19
18
  watched_file.reset_bytes_unread
20
19
  sincedb_value.update_position(0)
21
- logger.trace("update_existing_specifically: was truncated seeking to beginning", "watched file" => watched_file.details, "sincedb value" => sincedb_value)
20
+ logger.trace? && logger.trace("update_existing_specifically: was truncated seeking to beginning", :watched_file => watched_file.details, :sincedb_value => sincedb_value)
22
21
  end
23
22
  end
24
23
  end end end
@@ -13,9 +13,9 @@ module FileWatch module TailMode module Handlers
13
13
  # for file initially ignored their bytes_read was set to stat.size
14
14
  # use this value not the `start_new_files_at` for the position
15
15
  # logger.trace("get_new_value_specifically", "watched_file" => watched_file.inspect)
16
- SincedbValue.new(watched_file.bytes_read).tap do |val|
17
- val.set_watched_file(watched_file)
18
- logger.trace("-------------------- >>>>> get_new_value_specifically: unignore", "watched file" => watched_file.details, "sincedb value" => val)
16
+ SincedbValue.new(watched_file.bytes_read).tap do |sincedb_value|
17
+ sincedb_value.set_watched_file(watched_file)
18
+ logger.trace? && logger.trace("get_new_value_specifically: unignore", :watched_file => watched_file.details, :sincedb_value => sincedb_value)
19
19
  end
20
20
  end
21
21
 
@@ -26,7 +26,7 @@ module FileWatch module TailMode module Handlers
26
26
  # we will handle grow or shrink
27
27
  # for now we seek to where we were before the file got ignored (grow)
28
28
  # or to the start (shrink)
29
- logger.trace("-------------------- >>>>> update_existing_specifically: unignore", "watched file" => watched_file.details, "sincedb value" => sincedb_value)
29
+ logger.trace? && logger.trace("update_existing_specifically: unignore", :watched_file => watched_file.details, :sincedb_value => sincedb_value)
30
30
  position = 0
31
31
  if watched_file.shrunk?
32
32
  watched_file.update_bytes_read(0)
Binary file
@@ -377,12 +377,12 @@ class File < LogStash::Inputs::Base
377
377
  def handle_deletable_path(path)
378
378
  return if tail_mode?
379
379
  return if @completed_file_handlers.empty?
380
+ @logger.debug? && @logger.debug(__method__.to_s, :path => path)
380
381
  @completed_file_handlers.each { |handler| handler.handle(path) }
381
382
  end
382
383
 
383
384
  def log_line_received(path, line)
384
- return unless @logger.debug?
385
- @logger.debug("Received line", :path => path, :text => line)
385
+ @logger.debug? && @logger.debug("Received line", :path => path, :text => line)
386
386
  end
387
387
 
388
388
  def stop
@@ -7,9 +7,9 @@ module LogStash module Inputs
7
7
  class FileListener
8
8
  attr_reader :input, :path, :data
9
9
  # construct with link back to the input plugin instance.
10
- def initialize(path, input)
10
+ def initialize(path, input, data = nil)
11
11
  @path, @input = path, input
12
- @data = nil
12
+ @data = data
13
13
  end
14
14
 
15
15
  def opened
@@ -36,7 +36,7 @@ module LogStash module Inputs
36
36
  def accept(data)
37
37
  # and push transient data filled dup listener downstream
38
38
  input.log_line_received(path, data)
39
- input.codec.accept(dup_adding_state(data))
39
+ input.codec.accept(self.class.new(path, input, data))
40
40
  end
41
41
 
42
42
  def process_event(event)
@@ -45,17 +45,6 @@ module LogStash module Inputs
45
45
  input.post_process_this(event)
46
46
  end
47
47
 
48
- def add_state(data)
49
- @data = data
50
- self
51
- end
52
-
53
- private
54
-
55
- # duplicate and add state for downstream
56
- def dup_adding_state(line)
57
- self.class.new(path, input).add_state(line)
58
- end
59
48
  end
60
49
 
61
50
  class FlushableListener < FileListener
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-input-file'
4
- s.version = '4.2.2'
4
+ s.version = '4.2.3'
5
5
  s.licenses = ['Apache-2.0']
6
6
  s.summary = "Streams events from files"
7
7
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -9,7 +9,9 @@ module FileWatch
9
9
  let(:io) { StringIO.new }
10
10
  let(:db) { Hash.new }
11
11
 
12
- subject { SincedbRecordSerializer.new(SincedbRecordSerializer.days_to_seconds(14)) }
12
+ let(:sincedb_value_expiry) { SincedbRecordSerializer.days_to_seconds(14) }
13
+
14
+ subject { SincedbRecordSerializer.new(sincedb_value_expiry) }
13
15
 
14
16
  context "deserialize from IO" do
15
17
  it 'reads V1 records' do
@@ -82,8 +84,10 @@ module FileWatch
82
84
  end
83
85
 
84
86
  context "given a non default `sincedb_clean_after`" do
87
+
88
+ let(:sincedb_value_expiry) { SincedbRecordSerializer.days_to_seconds(2) }
89
+
85
90
  it "does not write expired db entries to an IO object" do
86
- subject.update_sincedb_value_expiry_from_days(2)
87
91
  one_day_ago = Time.now.to_f - (1.0*24*3600)
88
92
  three_days_ago = one_day_ago - (2.0*24*3600)
89
93
  db[InodeStruct.new("42424242", 2, 5)] = SincedbValue.new(42, one_day_ago)
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-input-file
3
3
  version: !ruby/object:Gem::Version
4
- version: 4.2.2
4
+ version: 4.2.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-09-25 00:00:00.000000000 Z
11
+ date: 2020-12-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement