logstash-input-file 4.2.1 → 4.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: a3a31addcd028920e2e1206d56c6fbab1b5492ebfeef0d688a431c88685ec8b9
4
- data.tar.gz: 0c07d4dd1c1c821c5ef9fcbbac68375ba04ea4dd399116c6ce016ad1c102b076
3
+ metadata.gz: f8c11df0b47e51ceb5ebd37b3d16eb28acfef360aa976d4155752fffca3c0161
4
+ data.tar.gz: 297b2410e81e2f4228f59e459e486269238820b82f94a577068c53c87b9f2ea7
5
5
  SHA512:
6
- metadata.gz: a1d95cd6b07d0076c3917e816c97ae832d3085b4ccc1029eb3325249a287b3f80e7ed41bfa5870ce9edb5af6ef5c98f8217a918fd0d3088e6ec25f985da1096a
7
- data.tar.gz: 2fafafe12fc3dbab4921029ae02e0e0949f4254f8fc50810de0c80cd49ab4158cacff9028d90c8d34dc28e17eabd7e6bc437ba2f28f0566f8bca9250b5733330
6
+ metadata.gz: 639fbc4dd7b2f02977923726f997e8ecf2d57862ac6cf20dd284f2a1b55e0532eb0137d1b81be200afc10bc5a8f84c625e74211bb685a45099c4660b36ad15eb
7
+ data.tar.gz: ecee35a5081b61038e3f9052cbb447269d1a5def40f47fbbb1cb2f88f0ee509e98995f9441e7cf37329d505ee095087e58e6740856d21253ef59b1d40a25fa10
@@ -1,3 +1,6 @@
1
+ ## 4.2.2
2
+ - Fix: sincedb_clean_after not being respected [#276](https://github.com/logstash-plugins/logstash-input-file/pull/276)
3
+
1
4
  ## 4.2.1
2
5
  - Fix: skip sincedb eviction if read mode completion deletes file during flush [#273](https://github.com/logstash-plugins/logstash-input-file/pull/273)
3
6
 
@@ -83,15 +83,5 @@ module FileWatch
83
83
  # sincedb_write("shutting down")
84
84
  end
85
85
 
86
- # close_file(path) is to be used by external code
87
- # when it knows that it is completely done with a file.
88
- # Other files or folders may still be being watched.
89
- # Caution, once unwatched, a file can't be watched again
90
- # unless a new instance of this class begins watching again.
91
- # The sysadmin should rename, move or delete the file.
92
- def close_file(path)
93
- @watch.unwatch(path)
94
- sincedb_write
95
- end
96
86
  end
97
87
  end
@@ -180,17 +180,17 @@ module FileWatch
180
180
  get(key).watched_file.nil?
181
181
  end
182
182
 
183
- private
184
-
185
183
  def flush_at_interval
186
- now = Time.now.to_i
187
- delta = now - @sincedb_last_write
184
+ now = Time.now
185
+ delta = now.to_i - @sincedb_last_write
188
186
  if delta >= @settings.sincedb_write_interval
189
187
  logger.debug("writing sincedb (delta since last write = #{delta})")
190
188
  sincedb_write(now)
191
189
  end
192
190
  end
193
191
 
192
+ private
193
+
194
194
  def handle_association(sincedb_value, watched_file)
195
195
  watched_file.update_bytes_read(sincedb_value.position)
196
196
  sincedb_value.set_watched_file(watched_file)
@@ -210,33 +210,33 @@ module FileWatch
210
210
  end
211
211
  end
212
212
 
213
- def sincedb_write(time = Time.now.to_i)
214
- logger.trace("sincedb_write: to: #{path}")
213
+ def sincedb_write(time = Time.now)
214
+ logger.trace("sincedb_write: #{path} (time = #{time})")
215
215
  begin
216
- @write_method.call
216
+ @write_method.call(time)
217
217
  @serializer.expired_keys.each do |key|
218
218
  @sincedb[key].unset_watched_file
219
219
  delete(key)
220
220
  logger.trace? && logger.trace("sincedb_write: cleaned", :key => key)
221
221
  end
222
- @sincedb_last_write = time
222
+ @sincedb_last_write = time.to_i
223
223
  @write_requested = false
224
224
  rescue Errno::EACCES
225
225
  # no file handles free perhaps
226
226
  # maybe it will work next time
227
- logger.trace("sincedb_write: error: #{path}: #{$!}")
227
+ logger.trace("sincedb_write: #{path} error: #{$!}")
228
228
  end
229
229
  end
230
230
 
231
- def atomic_write
231
+ def atomic_write(time)
232
232
  FileHelper.write_atomically(@full_path) do |io|
233
- @serializer.serialize(@sincedb, io)
233
+ @serializer.serialize(@sincedb, io, time.to_f)
234
234
  end
235
235
  end
236
236
 
237
- def non_atomic_write
237
+ def non_atomic_write(time)
238
238
  IO.open(IO.sysopen(@full_path, "w+")) do |io|
239
- @serializer.serialize(@sincedb, io)
239
+ @serializer.serialize(@sincedb, io, time.to_f)
240
240
  end
241
241
  end
242
242
  end
@@ -51,7 +51,10 @@ module FileWatch
51
51
  glob = 0
52
52
  end
53
53
  break if quit?
54
+ # NOTE: maybe the plugin should validate stat_interval <= sincedb_write_interval <= sincedb_clean_after
54
55
  sleep(@settings.stat_interval)
56
+ # we need to check potential expired keys (sincedb_clean_after) periodically
57
+ sincedb_collection.flush_at_interval
55
58
  end
56
59
  sincedb_collection.write_if_requested # does nothing if no requests to write were lodged.
57
60
  @watched_files_collection.close_all
Binary file
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-input-file'
4
- s.version = '4.2.1'
4
+ s.version = '4.2.2'
5
5
  s.licenses = ['Apache-2.0']
6
6
  s.summary = "Streams events from files"
7
7
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -301,25 +301,69 @@ describe LogStash::Inputs::File do
301
301
  watched_files = plugin.watcher.watch.watched_files_collection
302
302
  expect( watched_files ).to be_empty
303
303
  end
304
+ end
304
305
 
305
- private
306
+ describe 'sincedb cleanup' do
306
307
 
307
- def wait_for_start_processing(run_thread, timeout: 1.0)
308
- begin
309
- Timeout.timeout(timeout) do
310
- sleep(0.01) while run_thread.status != 'sleep'
311
- sleep(timeout) unless plugin.queue
312
- end
313
- rescue Timeout::Error
314
- raise "plugin did not start processing (timeout: #{timeout})" unless plugin.queue
315
- else
316
- raise "plugin did not start processing" unless plugin.queue
308
+ let(:options) do
309
+ super.merge(
310
+ 'sincedb_path' => sincedb_path,
311
+ 'sincedb_clean_after' => '1.0 seconds',
312
+ 'sincedb_write_interval' => 0.25,
313
+ 'stat_interval' => 0.1,
314
+ )
315
+ end
316
+
317
+ let(:sincedb_path) { "#{temp_directory}/.sincedb" }
318
+
319
+ let(:sample_file) { File.join(temp_directory, "sample.txt") }
320
+
321
+ before do
322
+ plugin.register
323
+ @run_thread = Thread.new(plugin) do |plugin|
324
+ Thread.current.abort_on_exception = true
325
+ plugin.run queue
317
326
  end
327
+
328
+ File.open(sample_file, 'w') { |fd| fd.write("line1\nline2\n") }
329
+
330
+ wait_for_start_processing(@run_thread)
318
331
  end
319
332
 
320
- def wait_for_file_removal(path, timeout: 3 * interval)
321
- wait(timeout).for { File.exist?(path) }.to be_falsey
333
+ after { plugin.stop }
334
+
335
+ it 'cleans up sincedb entry' do
336
+ wait_for_file_removal(sample_file) # watched discovery
337
+
338
+ sincedb_content = File.read(sincedb_path).strip
339
+ expect( sincedb_content ).to_not be_empty
340
+
341
+ Stud.try(3.times) do
342
+ sleep(1.5) # > sincedb_clean_after
343
+
344
+ sincedb_content = File.read(sincedb_path).strip
345
+ expect( sincedb_content ).to be_empty
346
+ end
322
347
  end
323
348
 
324
349
  end
350
+
351
+ private
352
+
353
+ def wait_for_start_processing(run_thread, timeout: 1.0)
354
+ begin
355
+ Timeout.timeout(timeout) do
356
+ sleep(0.01) while run_thread.status != 'sleep'
357
+ sleep(timeout) unless plugin.queue
358
+ end
359
+ rescue Timeout::Error
360
+ raise "plugin did not start processing (timeout: #{timeout})" unless plugin.queue
361
+ else
362
+ raise "plugin did not start processing" unless plugin.queue
363
+ end
364
+ end
365
+
366
+ def wait_for_file_removal(path, timeout: 3 * interval)
367
+ wait(timeout).for { File.exist?(path) }.to be_falsey
368
+ end
325
369
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-input-file
3
3
  version: !ruby/object:Gem::Version
4
- version: 4.2.1
4
+ version: 4.2.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-05-14 00:00:00.000000000 Z
11
+ date: 2020-09-25 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement