logstash-input-file 4.4.0 → 4.4.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +10 -0
- data/docs/index.asciidoc +14 -4
- data/lib/filewatch/read_mode/handlers/read_file.rb +10 -0
- data/lib/jars/filewatch-1.0.1.jar +0 -0
- data/logstash-input-file.gemspec +1 -1
- data/spec/filewatch/read_mode_handlers_read_file_spec.rb +40 -0
- data/spec/filewatch/spec_helper.rb +2 -0
- data/spec/helpers/spec_helper.rb +8 -2
- data/spec/inputs/file_tail_spec.rb +173 -138
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 26577b41ecc7118c3f4ec2dba011fe730e44d4a74c0f45b19875ba331922bb24
|
4
|
+
data.tar.gz: e799e9a5e149662f3e1ffa4e902bb8c789d8d5eb7f1b90c1776445571b358a1b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b8ce5e67051bb9d04947d3b3e72475f2929cecf475a5aa1196556fb63fcd37c44e931ebdff1e47fa21f01030bf2a5b15a6301658dbc249633b613c681b942fd9
|
7
|
+
data.tar.gz: 44dc2403a6ca1dcc5c0660404e4419d5acd1f020d376cd3be8fc7d46533335c10ba10e6c0510eb9c4afa3685b281e5a4a8cfb201ac2882c500f42b3bd54681ef
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,13 @@
|
|
1
|
+
## 4.4.3
|
2
|
+
- Fixes read mode to restart the read from reference stored in sincedb in case the file wasn't completely consumed. [#307](https://github.com/logstash-plugins/logstash-input-file/pull/307)
|
3
|
+
|
4
|
+
## 4.4.2
|
5
|
+
- Doc: Fix attribute by removing extra character [#310](https://github.com/logstash-plugins/logstash-input-file/pull/310)
|
6
|
+
|
7
|
+
## 4.4.1
|
8
|
+
- Fix: update to Gradle 7 [#305](https://github.com/logstash-plugins/logstash-input-file/pull/305)
|
9
|
+
- [DOC] Add version attributes to doc source file [#308](https://github.com/logstash-plugins/logstash-input-file/pull/308)
|
10
|
+
|
1
11
|
## 4.4.0
|
2
12
|
- Add support for ECS v8 [#301](https://github.com/logstash-plugins/logstash-input-file/pull/301)
|
3
13
|
|
data/docs/index.asciidoc
CHANGED
@@ -9,6 +9,11 @@ START - GENERATED VARIABLES, DO NOT EDIT!
|
|
9
9
|
:release_date: %RELEASE_DATE%
|
10
10
|
:changelog_url: %CHANGELOG_URL%
|
11
11
|
:include_path: ../../../../logstash/docs/include
|
12
|
+
|
13
|
+
ifeval::["{versioned_docs}"=="true"]
|
14
|
+
:branch: %BRANCH%
|
15
|
+
:ecs_version: %ECS_VERSION%
|
16
|
+
endif::[]
|
12
17
|
///////////////////////////////////////////
|
13
18
|
END - GENERATED VARIABLES, DO NOT EDIT!
|
14
19
|
///////////////////////////////////////////
|
@@ -82,7 +87,7 @@ was not ideal and a dedicated Read mode is an improvement.
|
|
82
87
|
==== Compatibility with the Elastic Common Schema (ECS)
|
83
88
|
|
84
89
|
This plugin adds metadata about event's source, and can be configured to do so
|
85
|
-
in an {
|
90
|
+
in an https://www.elastic.co/guide/en/ecs/{ecs_version}/index.html[ECS-compatible] way with <<plugins-{type}s-{plugin}-ecs_compatibility>>.
|
86
91
|
This metadata is added after the event has been decoded by the appropriate codec,
|
87
92
|
and will never overwrite existing values.
|
88
93
|
|
@@ -270,7 +275,7 @@ In practice, this will be the best case because the time taken to read new conte
|
|
270
275
|
** Otherwise, the default value is `disabled`.
|
271
276
|
|
272
277
|
Controls this plugin's compatibility with the
|
273
|
-
{
|
278
|
+
https://www.elastic.co/guide/en/ecs/{ecs_version}/index.html[Elastic Common Schema (ECS)].
|
274
279
|
|
275
280
|
[id="plugins-{type}s-{plugin}-exclude"]
|
276
281
|
===== `exclude`
|
@@ -426,7 +431,7 @@ of `/var/log` will be done for all `*.log` files.
|
|
426
431
|
Paths must be absolute and cannot be relative.
|
427
432
|
|
428
433
|
You may also configure multiple paths. See an example
|
429
|
-
on the {
|
434
|
+
on the https://www.elastic.co/guide/en/logstash/{branch}/configuration-file-structure.html#array[Logstash configuration page].
|
430
435
|
|
431
436
|
[id="plugins-{type}s-{plugin}-sincedb_clean_after"]
|
432
437
|
===== `sincedb_clean_after`
|
@@ -439,7 +444,7 @@ The sincedb record now has a last active timestamp associated with it.
|
|
439
444
|
If no changes are detected in a tracked file in the last N days its sincedb
|
440
445
|
tracking record expires and will not be persisted.
|
441
446
|
This option helps protect against the inode recycling problem.
|
442
|
-
Filebeat has
|
447
|
+
Filebeat has an https://www.elastic.co/guide/en/beats/filebeat/{branch}/inode-reuse-issue.html[FAQ about inode recycling].
|
443
448
|
|
444
449
|
[id="plugins-{type}s-{plugin}-sincedb_path"]
|
445
450
|
===== `sincedb_path`
|
@@ -533,4 +538,9 @@ Supported values: `us` `usec` `usecs`, e.g. "600 us", "800 usec", "900 usecs"
|
|
533
538
|
[NOTE]
|
534
539
|
`micro` `micros` and `microseconds` are not supported
|
535
540
|
|
541
|
+
ifeval::["{versioned_docs}"=="true"]
|
542
|
+
:branch: current
|
543
|
+
:ecs_version: current
|
544
|
+
endif::[]
|
545
|
+
|
536
546
|
:default_codec!:
|
@@ -2,9 +2,19 @@
|
|
2
2
|
|
3
3
|
module FileWatch module ReadMode module Handlers
|
4
4
|
class ReadFile < Base
|
5
|
+
|
6
|
+
# seek file to which ever is furthest: either current bytes read or sincedb position
|
7
|
+
private
|
8
|
+
def seek_to_furthest_position(watched_file)
|
9
|
+
previous_pos = sincedb_collection.find(watched_file).position
|
10
|
+
watched_file.file_seek([watched_file.bytes_read, previous_pos].max)
|
11
|
+
end
|
12
|
+
|
13
|
+
public
|
5
14
|
def handle_specifically(watched_file)
|
6
15
|
if open_file(watched_file)
|
7
16
|
add_or_update_sincedb_collection(watched_file) unless sincedb_collection.member?(watched_file.sincedb_key)
|
17
|
+
seek_to_furthest_position(watched_file)
|
8
18
|
loop do
|
9
19
|
break if quit?
|
10
20
|
loop_control = watched_file.loop_control_adjusted_for_stat_size
|
Binary file
|
data/logstash-input-file.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-input-file'
|
4
|
-
s.version = '4.4.
|
4
|
+
s.version = '4.4.3'
|
5
5
|
s.licenses = ['Apache-2.0']
|
6
6
|
s.summary = "Streams events from files"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
@@ -36,5 +36,45 @@ module FileWatch
|
|
36
36
|
processor.read_file(watched_file)
|
37
37
|
end
|
38
38
|
end
|
39
|
+
|
40
|
+
context "when restart from existing sincedb" do
|
41
|
+
let(:settings) do
|
42
|
+
Settings.from_options(
|
43
|
+
:sincedb_write_interval => 0,
|
44
|
+
:sincedb_path => File::NULL,
|
45
|
+
:file_chunk_size => 10
|
46
|
+
)
|
47
|
+
end
|
48
|
+
|
49
|
+
let(:processor) { double("fake processor") }
|
50
|
+
let(:observer) { TestObserver.new }
|
51
|
+
let(:watch) { double("watch") }
|
52
|
+
|
53
|
+
before(:each) {
|
54
|
+
allow(watch).to receive(:quit?).and_return(false)#.and_return(false).and_return(true)
|
55
|
+
allow(processor).to receive(:watch).and_return(watch)
|
56
|
+
}
|
57
|
+
|
58
|
+
it "read from where it left" do
|
59
|
+
listener = observer.listener_for(Pathname.new(pathname).to_path)
|
60
|
+
sut = ReadMode::Handlers::ReadFile.new(processor, sdb_collection, observer, settings)
|
61
|
+
|
62
|
+
# simulate a previous partial read of the file
|
63
|
+
sincedb_value = SincedbValue.new(0)
|
64
|
+
sincedb_value.set_watched_file(watched_file)
|
65
|
+
sdb_collection.set(watched_file.sincedb_key, sincedb_value)
|
66
|
+
|
67
|
+
|
68
|
+
# simulate a consumption of first line, (size + newline) bytes
|
69
|
+
sdb_collection.increment(watched_file.sincedb_key, File.readlines(pathname)[0].size + 2)
|
70
|
+
|
71
|
+
# exercise
|
72
|
+
sut.handle(watched_file)
|
73
|
+
|
74
|
+
# verify
|
75
|
+
expect(listener.lines.size).to eq(1)
|
76
|
+
expect(listener.lines[0]).to start_with("2010-03-12 23:51:21 SEA4 192.0.2.222 play 3914 OK")
|
77
|
+
end
|
78
|
+
end
|
39
79
|
end
|
40
80
|
end
|
data/spec/helpers/spec_helper.rb
CHANGED
@@ -31,7 +31,13 @@ module FileInput
|
|
31
31
|
|
32
32
|
def trace_for(symbol)
|
33
33
|
params = @tracer.map {|k,v| k == symbol ? v : nil}.compact
|
34
|
-
params.empty?
|
34
|
+
if params.empty?
|
35
|
+
false
|
36
|
+
else
|
37
|
+
# merge all params with same key
|
38
|
+
# there could be multiple instances of same call, e.g. [[:accept, true], [:auto_flush, true], [:close, true], [:auto_flush, true]]
|
39
|
+
params.reduce {|b1, b2| b1 and b2}
|
40
|
+
end
|
35
41
|
end
|
36
42
|
|
37
43
|
def clear
|
@@ -57,7 +63,7 @@ module FileInput
|
|
57
63
|
@tracer.push [:close, true]
|
58
64
|
end
|
59
65
|
def clone
|
60
|
-
self
|
66
|
+
self
|
61
67
|
end
|
62
68
|
end
|
63
69
|
end
|
@@ -157,39 +157,49 @@ describe LogStash::Inputs::File do
|
|
157
157
|
end
|
158
158
|
end
|
159
159
|
|
160
|
-
context "running the input twice" do
|
161
|
-
|
162
|
-
it "should read old files" do
|
163
|
-
conf = <<-CONFIG
|
164
|
-
input {
|
165
|
-
file {
|
166
|
-
type => "blah"
|
167
|
-
path => "#{path_path}"
|
168
|
-
start_position => "beginning"
|
169
|
-
codec => "json"
|
170
|
-
}
|
171
|
-
}
|
172
|
-
CONFIG
|
160
|
+
context "running the input twice", :ecs_compatibility_support do
|
161
|
+
ecs_compatibility_matrix(:disabled, :v1, :v8 => :v1) do |ecs_select|
|
173
162
|
|
174
|
-
|
175
|
-
|
176
|
-
fd.puts('{"my_field": "my_val"}')
|
177
|
-
fd.fsync
|
163
|
+
before(:each) do
|
164
|
+
allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
|
178
165
|
end
|
179
|
-
|
180
|
-
|
166
|
+
|
167
|
+
let(:file_path_target_field ) { ecs_select[disabled: "path", v1: '[log][file][path]'] }
|
168
|
+
let(:source_host_target_field) { ecs_select[disabled: "host", v1: '[host][name]'] }
|
169
|
+
|
170
|
+
let(:name) { "D" }
|
171
|
+
it "should read old files" do
|
172
|
+
conf = <<-CONFIG
|
173
|
+
input {
|
174
|
+
file {
|
175
|
+
type => "blah"
|
176
|
+
path => "#{path_path}"
|
177
|
+
start_position => "beginning"
|
178
|
+
codec => "json"
|
179
|
+
}
|
180
|
+
}
|
181
|
+
CONFIG
|
181
182
|
|
182
|
-
|
183
|
-
|
183
|
+
File.open(tmpfile_path, "w") do |fd|
|
184
|
+
fd.puts('{"path": "my_path", "host": "my_host"}')
|
185
|
+
fd.puts('{"my_field": "my_val"}')
|
186
|
+
fd.fsync
|
187
|
+
end
|
188
|
+
# arbitrary old file (2 days)
|
189
|
+
FileInput.make_file_older(tmpfile_path, 48 * 60 * 60)
|
190
|
+
|
191
|
+
events = input(conf) do |pipeline, queue|
|
192
|
+
2.times.collect { queue.pop }
|
193
|
+
end
|
194
|
+
existing_path_index, added_path_index = "my_val" == events[0].get("my_field") ? [1,0] : [0,1]
|
195
|
+
expect(events[existing_path_index].get("path")).to eq "my_path"
|
196
|
+
expect(events[existing_path_index].get("host")).to eq "my_host"
|
197
|
+
expect(events[existing_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
|
198
|
+
|
199
|
+
expect(events[added_path_index].get(file_path_target_field)).to eq "#{tmpfile_path}"
|
200
|
+
expect(events[added_path_index].get(source_host_target_field)).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
|
201
|
+
expect(events[added_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
|
184
202
|
end
|
185
|
-
existing_path_index, added_path_index = "my_val" == events[0].get("my_field") ? [1,0] : [0,1]
|
186
|
-
expect(events[existing_path_index].get("path")).to eq "my_path"
|
187
|
-
expect(events[existing_path_index].get("host")).to eq "my_host"
|
188
|
-
expect(events[existing_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
|
189
|
-
|
190
|
-
expect(events[added_path_index].get("path")).to eq "#{tmpfile_path}"
|
191
|
-
expect(events[added_path_index].get("host")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
|
192
|
-
expect(events[added_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
|
193
203
|
end
|
194
204
|
end
|
195
205
|
|
@@ -233,54 +243,62 @@ describe LogStash::Inputs::File do
|
|
233
243
|
FileUtils.rm_rf(sincedb_path)
|
234
244
|
end
|
235
245
|
|
236
|
-
context "when data exists and then more data is appended" do
|
237
|
-
|
246
|
+
context "when data exists and then more data is appended", :ecs_compatibility_support do
|
247
|
+
ecs_compatibility_matrix(:disabled, :v1, :v8 => :v1) do |ecs_select|
|
238
248
|
|
239
|
-
|
240
|
-
|
241
|
-
fd.puts("ignore me 1")
|
242
|
-
fd.puts("ignore me 2")
|
243
|
-
fd.fsync
|
249
|
+
before(:each) do
|
250
|
+
allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
|
244
251
|
end
|
245
|
-
mlconf.update("pattern" => "^\s", "what" => "previous")
|
246
|
-
conf.update("type" => "blah",
|
247
|
-
"path" => path_path,
|
248
|
-
"sincedb_path" => sincedb_path,
|
249
|
-
"stat_interval" => 0.1,
|
250
|
-
"codec" => mlcodec,
|
251
|
-
"delimiter" => TEST_FILE_DELIMITER)
|
252
|
-
end
|
253
252
|
|
254
|
-
|
255
|
-
subject.
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
end
|
263
|
-
.then("quit") do
|
264
|
-
subject.stop
|
265
|
-
end
|
266
|
-
.then("wait for flushed event") do
|
267
|
-
wait(0.75).for{events.size}.to eq(2)
|
253
|
+
let(:file_path_target_field ) { ecs_select[disabled: "path", v1: '[log][file][path]'] }
|
254
|
+
subject { described_class.new(conf) }
|
255
|
+
|
256
|
+
before do
|
257
|
+
File.open(tmpfile_path, "w") do |fd|
|
258
|
+
fd.puts("ignore me 1")
|
259
|
+
fd.puts("ignore me 2")
|
260
|
+
fd.fsync
|
268
261
|
end
|
262
|
+
mlconf.update("pattern" => "^\s", "what" => "previous")
|
263
|
+
conf.update("type" => "blah",
|
264
|
+
"path" => path_path,
|
265
|
+
"sincedb_path" => sincedb_path,
|
266
|
+
"stat_interval" => 0.1,
|
267
|
+
"codec" => mlcodec,
|
268
|
+
"delimiter" => TEST_FILE_DELIMITER)
|
269
|
+
end
|
269
270
|
|
270
|
-
|
271
|
-
|
271
|
+
it "reads the appended data only" do
|
272
|
+
subject.register
|
273
|
+
actions = RSpec::Sequencing
|
274
|
+
.run_after(1, "append two lines after delay") do
|
275
|
+
File.open(tmpfile_path, "a") { |fd| fd.puts("hello"); fd.puts("world") }
|
276
|
+
end
|
277
|
+
.then("wait for one event") do
|
278
|
+
wait(0.75).for{events.size}.to eq(1)
|
279
|
+
end
|
280
|
+
.then("quit") do
|
281
|
+
subject.stop
|
282
|
+
end
|
283
|
+
.then("wait for flushed event") do
|
284
|
+
wait(0.75).for{events.size}.to eq(2)
|
285
|
+
end
|
272
286
|
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
|
283
|
-
|
287
|
+
subject.run(events)
|
288
|
+
actions.assert_no_errors
|
289
|
+
|
290
|
+
event1 = events[0]
|
291
|
+
expect(event1).not_to be_nil
|
292
|
+
expect(event1.get(file_path_target_field)).to eq tmpfile_path
|
293
|
+
expect(event1.get("[@metadata][path]")).to eq tmpfile_path
|
294
|
+
expect(event1.get("message")).to eq "hello"
|
295
|
+
|
296
|
+
event2 = events[1]
|
297
|
+
expect(event2).not_to be_nil
|
298
|
+
expect(event2.get(file_path_target_field)).to eq tmpfile_path
|
299
|
+
expect(event2.get("[@metadata][path]")).to eq tmpfile_path
|
300
|
+
expect(event2.get("message")).to eq "world"
|
301
|
+
end
|
284
302
|
end
|
285
303
|
end
|
286
304
|
|
@@ -311,15 +329,23 @@ describe LogStash::Inputs::File do
|
|
311
329
|
.then_after(0.1, "identity is mapped") do
|
312
330
|
wait(0.75).for{subject.codec.identity_map[tmpfile_path]}.not_to be_nil, "identity is not mapped"
|
313
331
|
end
|
314
|
-
.then("wait
|
315
|
-
wait(0.75).for{
|
332
|
+
.then("wait accept") do
|
333
|
+
wait(0.75).for {
|
334
|
+
subject.codec.identity_map[tmpfile_path].codec.trace_for(:accept)
|
335
|
+
}.to eq(true), "accept didn't"
|
316
336
|
end
|
317
|
-
.then("
|
337
|
+
.then("request a stop") do
|
338
|
+
# without this the subject.run doesn't invokes the #exit_flush which is the only @codec.flush_mapped invocation
|
318
339
|
subject.stop
|
319
340
|
end
|
341
|
+
.then("wait for auto_flush") do
|
342
|
+
wait(2).for {
|
343
|
+
subject.codec.identity_map[tmpfile_path].codec.trace_for(:auto_flush)
|
344
|
+
}.to eq(true), "autoflush didn't"
|
345
|
+
end
|
320
346
|
subject.run(events)
|
321
347
|
actions.assert_no_errors
|
322
|
-
expect(subject.codec.identity_map[tmpfile_path].codec.trace_for(:accept)).to eq(
|
348
|
+
expect(subject.codec.identity_map[tmpfile_path].codec.trace_for(:accept)).to eq(true)
|
323
349
|
end
|
324
350
|
end
|
325
351
|
|
@@ -356,74 +382,50 @@ describe LogStash::Inputs::File do
|
|
356
382
|
end
|
357
383
|
end
|
358
384
|
|
359
|
-
context "when wildcard path and a multiline codec is specified" do
|
360
|
-
|
361
|
-
let(:suffix) { "J" }
|
362
|
-
let(:tmpfile_path2) { ::File.join(tmpdir_path, "K.txt") }
|
363
|
-
before do
|
364
|
-
mlconf.update("pattern" => "^\s", "what" => "previous")
|
365
|
-
conf.update(
|
366
|
-
"type" => "blah",
|
367
|
-
"path" => path_path,
|
368
|
-
"start_position" => "beginning",
|
369
|
-
"sincedb_path" => sincedb_path,
|
370
|
-
"stat_interval" => 0.05,
|
371
|
-
"codec" => mlcodec,
|
372
|
-
"file_sort_by" => "path",
|
373
|
-
"delimiter" => TEST_FILE_DELIMITER)
|
385
|
+
context "when wildcard path and a multiline codec is specified", :ecs_compatibility_support do
|
386
|
+
ecs_compatibility_matrix(:disabled, :v1, :v8 => :v1) do |ecs_select|
|
374
387
|
|
375
|
-
|
376
|
-
|
388
|
+
before(:each) do
|
389
|
+
allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
|
390
|
+
end
|
377
391
|
|
378
|
-
|
379
|
-
subject
|
380
|
-
actions = RSpec::Sequencing
|
381
|
-
.run_after(0.1, "create files") do
|
382
|
-
File.open(tmpfile_path, "wb") do |fd|
|
383
|
-
fd.puts("line1.1-of-J")
|
384
|
-
fd.puts(" line1.2-of-J")
|
385
|
-
fd.puts(" line1.3-of-J")
|
386
|
-
end
|
387
|
-
File.open(tmpfile_path2, "wb") do |fd|
|
388
|
-
fd.puts("line1.1-of-K")
|
389
|
-
fd.puts(" line1.2-of-K")
|
390
|
-
fd.puts(" line1.3-of-K")
|
391
|
-
end
|
392
|
-
end
|
393
|
-
.then("assert both files are mapped as identities and stop") do
|
394
|
-
wait(2).for {subject.codec.identity_count}.to eq(2), "both files are not mapped as identities"
|
395
|
-
end
|
396
|
-
.then("stop") do
|
397
|
-
subject.stop
|
398
|
-
end
|
399
|
-
subject.run(events)
|
400
|
-
# wait for actions to complete
|
401
|
-
actions.assert_no_errors
|
402
|
-
expect(events.size).to eq(2)
|
403
|
-
e1, e2 = events
|
404
|
-
e1_message = e1.get("message")
|
405
|
-
e2_message = e2.get("message")
|
406
|
-
|
407
|
-
expect(e1.get("path")).to match(/J.txt/)
|
408
|
-
expect(e2.get("path")).to match(/K.txt/)
|
409
|
-
expect(e1_message).to eq("line1.1-of-J#{TEST_FILE_DELIMITER} line1.2-of-J#{TEST_FILE_DELIMITER} line1.3-of-J")
|
410
|
-
expect(e2_message).to eq("line1.1-of-K#{TEST_FILE_DELIMITER} line1.2-of-K#{TEST_FILE_DELIMITER} line1.3-of-K")
|
411
|
-
end
|
392
|
+
let(:file_path_target_field ) { ecs_select[disabled: "path", v1: '[log][file][path]'] }
|
412
393
|
|
413
|
-
|
414
|
-
let(:
|
415
|
-
let(:
|
416
|
-
|
394
|
+
subject { described_class.new(conf) }
|
395
|
+
let(:suffix) { "J" }
|
396
|
+
let(:tmpfile_path2) { ::File.join(tmpdir_path, "K.txt") }
|
397
|
+
before do
|
398
|
+
mlconf.update("pattern" => "^\s", "what" => "previous")
|
399
|
+
conf.update(
|
400
|
+
"type" => "blah",
|
401
|
+
"path" => path_path,
|
402
|
+
"start_position" => "beginning",
|
403
|
+
"sincedb_path" => sincedb_path,
|
404
|
+
"stat_interval" => 0.05,
|
405
|
+
"codec" => mlcodec,
|
406
|
+
"file_sort_by" => "path",
|
407
|
+
"delimiter" => TEST_FILE_DELIMITER)
|
408
|
+
|
409
|
+
subject.register
|
410
|
+
end
|
411
|
+
|
412
|
+
it "collects separate multiple line events from each file" do
|
413
|
+
subject
|
417
414
|
actions = RSpec::Sequencing
|
418
415
|
.run_after(0.1, "create files") do
|
419
416
|
File.open(tmpfile_path, "wb") do |fd|
|
420
|
-
fd.puts("line1.1-of-
|
421
|
-
fd.puts(" line1.2-of-
|
422
|
-
fd.puts(" line1.3-of-
|
417
|
+
fd.puts("line1.1-of-J")
|
418
|
+
fd.puts(" line1.2-of-J")
|
419
|
+
fd.puts(" line1.3-of-J")
|
420
|
+
end
|
421
|
+
File.open(tmpfile_path2, "wb") do |fd|
|
422
|
+
fd.puts("line1.1-of-K")
|
423
|
+
fd.puts(" line1.2-of-K")
|
424
|
+
fd.puts(" line1.3-of-K")
|
423
425
|
end
|
424
426
|
end
|
425
|
-
.then("
|
426
|
-
wait(2).for{
|
427
|
+
.then("assert both files are mapped as identities and stop") do
|
428
|
+
wait(2).for {subject.codec.identity_count}.to eq(2), "both files are not mapped as identities"
|
427
429
|
end
|
428
430
|
.then("stop") do
|
429
431
|
subject.stop
|
@@ -431,10 +433,43 @@ describe LogStash::Inputs::File do
|
|
431
433
|
subject.run(events)
|
432
434
|
# wait for actions to complete
|
433
435
|
actions.assert_no_errors
|
434
|
-
|
436
|
+
expect(events.size).to eq(2)
|
437
|
+
e1, e2 = events
|
435
438
|
e1_message = e1.get("message")
|
436
|
-
|
437
|
-
|
439
|
+
e2_message = e2.get("message")
|
440
|
+
|
441
|
+
expect(e1.get(file_path_target_field)).to match(/J.txt/)
|
442
|
+
expect(e2.get(file_path_target_field)).to match(/K.txt/)
|
443
|
+
expect(e1_message).to eq("line1.1-of-J#{TEST_FILE_DELIMITER} line1.2-of-J#{TEST_FILE_DELIMITER} line1.3-of-J")
|
444
|
+
expect(e2_message).to eq("line1.1-of-K#{TEST_FILE_DELIMITER} line1.2-of-K#{TEST_FILE_DELIMITER} line1.3-of-K")
|
445
|
+
end
|
446
|
+
|
447
|
+
context "if auto_flush is enabled on the multiline codec" do
|
448
|
+
let(:mlconf) { { "auto_flush_interval" => 0.5 } }
|
449
|
+
let(:suffix) { "M" }
|
450
|
+
it "an event is generated via auto_flush" do
|
451
|
+
actions = RSpec::Sequencing
|
452
|
+
.run_after(0.1, "create files") do
|
453
|
+
File.open(tmpfile_path, "wb") do |fd|
|
454
|
+
fd.puts("line1.1-of-a")
|
455
|
+
fd.puts(" line1.2-of-a")
|
456
|
+
fd.puts(" line1.3-of-a")
|
457
|
+
end
|
458
|
+
end
|
459
|
+
.then("wait for auto_flush") do
|
460
|
+
wait(2).for{events.size}.to eq(1), "events size is not 1"
|
461
|
+
end
|
462
|
+
.then("stop") do
|
463
|
+
subject.stop
|
464
|
+
end
|
465
|
+
subject.run(events)
|
466
|
+
# wait for actions to complete
|
467
|
+
actions.assert_no_errors
|
468
|
+
e1 = events.first
|
469
|
+
e1_message = e1.get("message")
|
470
|
+
expect(e1_message).to eq("line1.1-of-a#{TEST_FILE_DELIMITER} line1.2-of-a#{TEST_FILE_DELIMITER} line1.3-of-a")
|
471
|
+
expect(e1.get(file_path_target_field)).to match(/M.txt$/)
|
472
|
+
end
|
438
473
|
end
|
439
474
|
end
|
440
475
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-file
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 4.4.
|
4
|
+
version: 4.4.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-06-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|