logstash-input-file 4.2.1 → 4.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/lib/filewatch/observing_base.rb +0 -10
- data/lib/filewatch/sincedb_collection.rb +13 -13
- data/lib/filewatch/watch.rb +3 -0
- data/lib/jars/filewatch-1.0.1.jar +0 -0
- data/logstash-input-file.gemspec +1 -1
- data/spec/inputs/file_read_spec.rb +57 -13
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f8c11df0b47e51ceb5ebd37b3d16eb28acfef360aa976d4155752fffca3c0161
|
4
|
+
data.tar.gz: 297b2410e81e2f4228f59e459e486269238820b82f94a577068c53c87b9f2ea7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 639fbc4dd7b2f02977923726f997e8ecf2d57862ac6cf20dd284f2a1b55e0532eb0137d1b81be200afc10bc5a8f84c625e74211bb685a45099c4660b36ad15eb
|
7
|
+
data.tar.gz: ecee35a5081b61038e3f9052cbb447269d1a5def40f47fbbb1cb2f88f0ee509e98995f9441e7cf37329d505ee095087e58e6740856d21253ef59b1d40a25fa10
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,6 @@
|
|
1
|
+
## 4.2.2
|
2
|
+
- Fix: sincedb_clean_after not being respected [#276](https://github.com/logstash-plugins/logstash-input-file/pull/276)
|
3
|
+
|
1
4
|
## 4.2.1
|
2
5
|
- Fix: skip sincedb eviction if read mode completion deletes file during flush [#273](https://github.com/logstash-plugins/logstash-input-file/pull/273)
|
3
6
|
|
@@ -83,15 +83,5 @@ module FileWatch
|
|
83
83
|
# sincedb_write("shutting down")
|
84
84
|
end
|
85
85
|
|
86
|
-
# close_file(path) is to be used by external code
|
87
|
-
# when it knows that it is completely done with a file.
|
88
|
-
# Other files or folders may still be being watched.
|
89
|
-
# Caution, once unwatched, a file can't be watched again
|
90
|
-
# unless a new instance of this class begins watching again.
|
91
|
-
# The sysadmin should rename, move or delete the file.
|
92
|
-
def close_file(path)
|
93
|
-
@watch.unwatch(path)
|
94
|
-
sincedb_write
|
95
|
-
end
|
96
86
|
end
|
97
87
|
end
|
@@ -180,17 +180,17 @@ module FileWatch
|
|
180
180
|
get(key).watched_file.nil?
|
181
181
|
end
|
182
182
|
|
183
|
-
private
|
184
|
-
|
185
183
|
def flush_at_interval
|
186
|
-
now = Time.now
|
187
|
-
delta = now - @sincedb_last_write
|
184
|
+
now = Time.now
|
185
|
+
delta = now.to_i - @sincedb_last_write
|
188
186
|
if delta >= @settings.sincedb_write_interval
|
189
187
|
logger.debug("writing sincedb (delta since last write = #{delta})")
|
190
188
|
sincedb_write(now)
|
191
189
|
end
|
192
190
|
end
|
193
191
|
|
192
|
+
private
|
193
|
+
|
194
194
|
def handle_association(sincedb_value, watched_file)
|
195
195
|
watched_file.update_bytes_read(sincedb_value.position)
|
196
196
|
sincedb_value.set_watched_file(watched_file)
|
@@ -210,33 +210,33 @@ module FileWatch
|
|
210
210
|
end
|
211
211
|
end
|
212
212
|
|
213
|
-
def sincedb_write(time = Time.now
|
214
|
-
logger.trace("sincedb_write:
|
213
|
+
def sincedb_write(time = Time.now)
|
214
|
+
logger.trace("sincedb_write: #{path} (time = #{time})")
|
215
215
|
begin
|
216
|
-
@write_method.call
|
216
|
+
@write_method.call(time)
|
217
217
|
@serializer.expired_keys.each do |key|
|
218
218
|
@sincedb[key].unset_watched_file
|
219
219
|
delete(key)
|
220
220
|
logger.trace? && logger.trace("sincedb_write: cleaned", :key => key)
|
221
221
|
end
|
222
|
-
@sincedb_last_write = time
|
222
|
+
@sincedb_last_write = time.to_i
|
223
223
|
@write_requested = false
|
224
224
|
rescue Errno::EACCES
|
225
225
|
# no file handles free perhaps
|
226
226
|
# maybe it will work next time
|
227
|
-
logger.trace("sincedb_write:
|
227
|
+
logger.trace("sincedb_write: #{path} error: #{$!}")
|
228
228
|
end
|
229
229
|
end
|
230
230
|
|
231
|
-
def atomic_write
|
231
|
+
def atomic_write(time)
|
232
232
|
FileHelper.write_atomically(@full_path) do |io|
|
233
|
-
@serializer.serialize(@sincedb, io)
|
233
|
+
@serializer.serialize(@sincedb, io, time.to_f)
|
234
234
|
end
|
235
235
|
end
|
236
236
|
|
237
|
-
def non_atomic_write
|
237
|
+
def non_atomic_write(time)
|
238
238
|
IO.open(IO.sysopen(@full_path, "w+")) do |io|
|
239
|
-
@serializer.serialize(@sincedb, io)
|
239
|
+
@serializer.serialize(@sincedb, io, time.to_f)
|
240
240
|
end
|
241
241
|
end
|
242
242
|
end
|
data/lib/filewatch/watch.rb
CHANGED
@@ -51,7 +51,10 @@ module FileWatch
|
|
51
51
|
glob = 0
|
52
52
|
end
|
53
53
|
break if quit?
|
54
|
+
# NOTE: maybe the plugin should validate stat_interval <= sincedb_write_interval <= sincedb_clean_after
|
54
55
|
sleep(@settings.stat_interval)
|
56
|
+
# we need to check potential expired keys (sincedb_clean_after) periodically
|
57
|
+
sincedb_collection.flush_at_interval
|
55
58
|
end
|
56
59
|
sincedb_collection.write_if_requested # does nothing if no requests to write were lodged.
|
57
60
|
@watched_files_collection.close_all
|
Binary file
|
data/logstash-input-file.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-input-file'
|
4
|
-
s.version = '4.2.
|
4
|
+
s.version = '4.2.2'
|
5
5
|
s.licenses = ['Apache-2.0']
|
6
6
|
s.summary = "Streams events from files"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
@@ -301,25 +301,69 @@ describe LogStash::Inputs::File do
|
|
301
301
|
watched_files = plugin.watcher.watch.watched_files_collection
|
302
302
|
expect( watched_files ).to be_empty
|
303
303
|
end
|
304
|
+
end
|
304
305
|
|
305
|
-
|
306
|
+
describe 'sincedb cleanup' do
|
306
307
|
|
307
|
-
|
308
|
-
|
309
|
-
|
310
|
-
|
311
|
-
|
312
|
-
|
313
|
-
|
314
|
-
|
315
|
-
|
316
|
-
|
308
|
+
let(:options) do
|
309
|
+
super.merge(
|
310
|
+
'sincedb_path' => sincedb_path,
|
311
|
+
'sincedb_clean_after' => '1.0 seconds',
|
312
|
+
'sincedb_write_interval' => 0.25,
|
313
|
+
'stat_interval' => 0.1,
|
314
|
+
)
|
315
|
+
end
|
316
|
+
|
317
|
+
let(:sincedb_path) { "#{temp_directory}/.sincedb" }
|
318
|
+
|
319
|
+
let(:sample_file) { File.join(temp_directory, "sample.txt") }
|
320
|
+
|
321
|
+
before do
|
322
|
+
plugin.register
|
323
|
+
@run_thread = Thread.new(plugin) do |plugin|
|
324
|
+
Thread.current.abort_on_exception = true
|
325
|
+
plugin.run queue
|
317
326
|
end
|
327
|
+
|
328
|
+
File.open(sample_file, 'w') { |fd| fd.write("line1\nline2\n") }
|
329
|
+
|
330
|
+
wait_for_start_processing(@run_thread)
|
318
331
|
end
|
319
332
|
|
320
|
-
|
321
|
-
|
333
|
+
after { plugin.stop }
|
334
|
+
|
335
|
+
it 'cleans up sincedb entry' do
|
336
|
+
wait_for_file_removal(sample_file) # watched discovery
|
337
|
+
|
338
|
+
sincedb_content = File.read(sincedb_path).strip
|
339
|
+
expect( sincedb_content ).to_not be_empty
|
340
|
+
|
341
|
+
Stud.try(3.times) do
|
342
|
+
sleep(1.5) # > sincedb_clean_after
|
343
|
+
|
344
|
+
sincedb_content = File.read(sincedb_path).strip
|
345
|
+
expect( sincedb_content ).to be_empty
|
346
|
+
end
|
322
347
|
end
|
323
348
|
|
324
349
|
end
|
350
|
+
|
351
|
+
private
|
352
|
+
|
353
|
+
def wait_for_start_processing(run_thread, timeout: 1.0)
|
354
|
+
begin
|
355
|
+
Timeout.timeout(timeout) do
|
356
|
+
sleep(0.01) while run_thread.status != 'sleep'
|
357
|
+
sleep(timeout) unless plugin.queue
|
358
|
+
end
|
359
|
+
rescue Timeout::Error
|
360
|
+
raise "plugin did not start processing (timeout: #{timeout})" unless plugin.queue
|
361
|
+
else
|
362
|
+
raise "plugin did not start processing" unless plugin.queue
|
363
|
+
end
|
364
|
+
end
|
365
|
+
|
366
|
+
def wait_for_file_removal(path, timeout: 3 * interval)
|
367
|
+
wait(timeout).for { File.exist?(path) }.to be_falsey
|
368
|
+
end
|
325
369
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-file
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 4.2.
|
4
|
+
version: 4.2.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-09-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|