logstash-input-file 4.2.4 → 4.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/docs/index.asciidoc +30 -0
- data/lib/jars/filewatch-1.0.1.jar +0 -0
- data/lib/logstash/inputs/file.rb +22 -1
- data/lib/logstash/inputs/file_listener.rb +0 -1
- data/logstash-input-file.gemspec +2 -1
- data/spec/inputs/file_tail_spec.rb +48 -28
- metadata +16 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: '0389c6d97f2957a0701da71d3cbcc7865ed51745e2eb49bb0a6327097865a921'
|
4
|
+
data.tar.gz: 7bec1b1e810008641a3286406c7402e82dcb62d6132b5f39cfa035c2ec168829
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3f83c7fcac2202d6380f7e54ecf231530ea09adf8e7019cef5d5935a2c024d85161b4749e7d1f355f8b38b59d26d57d95035c89abe0cb6b555edf0391b81475d
|
7
|
+
data.tar.gz: 0d939d35f02541ca57ac6a1735d6c008ba985ed977842f78a9ad95011e3146d8c2e3a939020354532d53b7975c5a572ffa9702f9ce82a36a4af235c50b6efaca
|
data/CHANGELOG.md
CHANGED
data/docs/index.asciidoc
CHANGED
@@ -78,6 +78,21 @@ Read mode also allows for an action to take place after processing the file comp
|
|
78
78
|
In the past attempts to simulate a Read mode while still assuming infinite streams
|
79
79
|
was not ideal and a dedicated Read mode is an improvement.
|
80
80
|
|
81
|
+
[id="plugins-{type}s-{plugin}-ecs"]
|
82
|
+
==== Compatibility with the Elastic Common Schema (ECS)
|
83
|
+
|
84
|
+
This plugin adds metadata about event's source, and can be configured to do so
|
85
|
+
in an {ecs-ref}[ECS-compatible] way with <<plugins-{type}s-{plugin}-ecs_compatibility>>.
|
86
|
+
This metadata is added after the event has been decoded by the appropriate codec,
|
87
|
+
and will never overwrite existing values.
|
88
|
+
|
89
|
+
|========
|
90
|
+
| ECS Disabled | ECS v1 | Description
|
91
|
+
|
92
|
+
| `host` | `[host][name]` | The name of the {ls} host that processed the event
|
93
|
+
| `path` | `[log][file][path]` | The full path to the log file from which the event originates
|
94
|
+
|========
|
95
|
+
|
81
96
|
==== Tracking of current position in watched files
|
82
97
|
|
83
98
|
The plugin keeps track of the current position in each file by
|
@@ -168,6 +183,7 @@ see <<plugins-{type}s-{plugin}-string_duration,string_duration>> for the details
|
|
168
183
|
| <<plugins-{type}s-{plugin}-close_older>> |<<number,number>> or <<plugins-{type}s-{plugin}-string_duration,string_duration>>|No
|
169
184
|
| <<plugins-{type}s-{plugin}-delimiter>> |<<string,string>>|No
|
170
185
|
| <<plugins-{type}s-{plugin}-discover_interval>> |<<number,number>>|No
|
186
|
+
| <<plugins-{type}s-{plugin}-ecs_compatibility>> |<<string,string>>|No
|
171
187
|
| <<plugins-{type}s-{plugin}-exclude>> |<<array,array>>|No
|
172
188
|
| <<plugins-{type}s-{plugin}-exit_after_read>> |<<boolean,boolean>>|No
|
173
189
|
| <<plugins-{type}s-{plugin}-file_chunk_count>> |<<number,number>>|No
|
@@ -242,6 +258,20 @@ This value is a multiple to `stat_interval`, e.g. if `stat_interval` is "500 ms"
|
|
242
258
|
files could be discovered every 15 X 500 milliseconds - 7.5 seconds.
|
243
259
|
In practice, this will be the best case because the time taken to read new content needs to be factored in.
|
244
260
|
|
261
|
+
[id="plugins-{type}s-{plugin}-ecs_compatibility"]
|
262
|
+
===== `ecs_compatibility`
|
263
|
+
|
264
|
+
* Value type is <<string,string>>
|
265
|
+
* Supported values are:
|
266
|
+
** `disabled`: sets non-ECS metadata on event (such as top-level `host`, `path`)
|
267
|
+
** `v1`: sets ECS-compatible metadata on event (such as `[host][name]`, `[log][file][path]`)
|
268
|
+
* Default value depends on which version of Logstash is running:
|
269
|
+
** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default
|
270
|
+
** Otherwise, the default value is `disabled`.
|
271
|
+
|
272
|
+
Controls this plugin's compatibility with the
|
273
|
+
{ecs-ref}[Elastic Common Schema (ECS)].
|
274
|
+
|
245
275
|
[id="plugins-{type}s-{plugin}-exclude"]
|
246
276
|
===== `exclude`
|
247
277
|
|
Binary file
|
data/lib/logstash/inputs/file.rb
CHANGED
@@ -2,6 +2,7 @@
|
|
2
2
|
require "logstash/namespace"
|
3
3
|
require "logstash/inputs/base"
|
4
4
|
require "logstash/codecs/identity_map_codec"
|
5
|
+
require 'logstash/plugin_mixins/ecs_compatibility_support'
|
5
6
|
|
6
7
|
require "pathname"
|
7
8
|
require "socket" # for Socket.gethostname
|
@@ -88,6 +89,8 @@ module LogStash module Inputs
|
|
88
89
|
class File < LogStash::Inputs::Base
|
89
90
|
config_name "file"
|
90
91
|
|
92
|
+
include PluginMixins::ECSCompatibilitySupport(:disabled, :v1)
|
93
|
+
|
91
94
|
# The path(s) to the file(s) to use as an input.
|
92
95
|
# You can use filename patterns here, such as `/var/log/*.log`.
|
93
96
|
# If you use a pattern like `/var/log/**/*.log`, a recursive search
|
@@ -325,6 +328,9 @@ class File < LogStash::Inputs::Base
|
|
325
328
|
@codec = LogStash::Codecs::IdentityMapCodec.new(@codec)
|
326
329
|
@completely_stopped = Concurrent::AtomicBoolean.new
|
327
330
|
@queue = Concurrent::AtomicReference.new
|
331
|
+
|
332
|
+
@source_host_field = ecs_select[disabled: 'host', v1:'[host][name]']
|
333
|
+
@source_path_field = ecs_select[disabled: 'path', v1:'[log][file][path]']
|
328
334
|
end # def register
|
329
335
|
|
330
336
|
def completely_stopped?
|
@@ -369,7 +375,11 @@ class File < LogStash::Inputs::Base
|
|
369
375
|
|
370
376
|
def post_process_this(event)
|
371
377
|
event.set("[@metadata][host]", @host)
|
372
|
-
event
|
378
|
+
attempt_set(event, @source_host_field, @host)
|
379
|
+
|
380
|
+
source_path = event.get('[@metadata][path]') and
|
381
|
+
attempt_set(event, @source_path_field, source_path)
|
382
|
+
|
373
383
|
decorate(event)
|
374
384
|
@queue.get << event
|
375
385
|
end
|
@@ -407,6 +417,17 @@ class File < LogStash::Inputs::Base
|
|
407
417
|
end
|
408
418
|
end
|
409
419
|
|
420
|
+
# Attempt to set an event's field to the provided value
|
421
|
+
# without overwriting an existing value or producing an error
|
422
|
+
def attempt_set(event, field_reference, value)
|
423
|
+
return false if event.include?(field_reference)
|
424
|
+
|
425
|
+
event.set(field_reference, value)
|
426
|
+
rescue => e
|
427
|
+
logger.trace("failed to set #{field_reference} to `#{value}`", :exception => e.message)
|
428
|
+
false
|
429
|
+
end
|
430
|
+
|
410
431
|
def build_sincedb_base_from_env
|
411
432
|
# This section is going to be deprecated eventually, as path.data will be
|
412
433
|
# the default, not an environment variable (SINCEDB_DIR or LOGSTASH_HOME)
|
data/logstash-input-file.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-input-file'
|
4
|
-
s.version = '4.
|
4
|
+
s.version = '4.3.0'
|
5
5
|
s.licenses = ['Apache-2.0']
|
6
6
|
s.summary = "Streams events from files"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
@@ -33,6 +33,7 @@ Gem::Specification.new do |s|
|
|
33
33
|
|
34
34
|
s.add_runtime_dependency 'concurrent-ruby', '~> 1.0'
|
35
35
|
s.add_runtime_dependency 'logstash-codec-multiline', ['~> 3.0']
|
36
|
+
s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.1'
|
36
37
|
|
37
38
|
s.add_development_dependency 'stud', ['~> 0.0.19']
|
38
39
|
s.add_development_dependency 'logstash-devutils'
|
@@ -3,7 +3,9 @@
|
|
3
3
|
require "helpers/spec_helper"
|
4
4
|
require "logstash/devutils/rspec/shared_examples"
|
5
5
|
require "logstash/inputs/file"
|
6
|
+
require "logstash/plugin_mixins/ecs_compatibility_support/spec_helper"
|
6
7
|
|
8
|
+
require "json"
|
7
9
|
require "tempfile"
|
8
10
|
require "stud/temporary"
|
9
11
|
require "logstash/codecs/multiline"
|
@@ -99,41 +101,59 @@ describe LogStash::Inputs::File do
|
|
99
101
|
end
|
100
102
|
end
|
101
103
|
|
102
|
-
context "when path and host fields exist" do
|
103
|
-
let(:name) { "C" }
|
104
|
-
it "should not overwrite them" do
|
105
|
-
conf = <<-CONFIG
|
106
|
-
input {
|
107
|
-
file {
|
108
|
-
type => "blah"
|
109
|
-
path => "#{path_path}"
|
110
|
-
start_position => "beginning"
|
111
|
-
sincedb_path => "#{sincedb_path}"
|
112
|
-
delimiter => "#{TEST_FILE_DELIMITER}"
|
113
|
-
codec => "json"
|
114
|
-
}
|
115
|
-
}
|
116
|
-
CONFIG
|
117
104
|
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
105
|
+
context "when path and host fields exist", :ecs_compatibility_support do
|
106
|
+
ecs_compatibility_matrix(:disabled, :v1) do |ecs_select|
|
107
|
+
|
108
|
+
before(:each) do
|
109
|
+
allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
|
122
110
|
end
|
123
111
|
|
124
|
-
|
125
|
-
|
112
|
+
let(:file_path_target_field ) { ecs_select[disabled: "path", v1: '[log][file][path]'] }
|
113
|
+
let(:source_host_target_field) { ecs_select[disabled: "host", v1: '[host][name]'] }
|
114
|
+
|
115
|
+
let(:event_with_existing) do
|
116
|
+
LogStash::Event.new.tap do |e|
|
117
|
+
e.set(file_path_target_field, 'my_path')
|
118
|
+
e.set(source_host_target_field, 'my_host')
|
119
|
+
end.to_hash
|
126
120
|
end
|
127
121
|
|
128
|
-
|
122
|
+
let(:name) { "C" }
|
123
|
+
it "should not overwrite them" do
|
124
|
+
conf = <<-CONFIG
|
125
|
+
input {
|
126
|
+
file {
|
127
|
+
type => "blah"
|
128
|
+
path => "#{path_path}"
|
129
|
+
start_position => "beginning"
|
130
|
+
sincedb_path => "#{sincedb_path}"
|
131
|
+
delimiter => "#{TEST_FILE_DELIMITER}"
|
132
|
+
codec => "json"
|
133
|
+
}
|
134
|
+
}
|
135
|
+
CONFIG
|
129
136
|
|
130
|
-
|
131
|
-
|
132
|
-
|
137
|
+
File.open(tmpfile_path, "w") do |fd|
|
138
|
+
fd.puts(event_with_existing.to_json)
|
139
|
+
fd.puts('{"my_field": "my_val"}')
|
140
|
+
fd.fsync
|
141
|
+
end
|
133
142
|
|
134
|
-
|
135
|
-
|
136
|
-
|
143
|
+
events = input(conf) do |pipeline, queue|
|
144
|
+
2.times.collect { queue.pop }
|
145
|
+
end
|
146
|
+
|
147
|
+
existing_path_index, added_path_index = "my_val" == events[0].get("my_field") ? [1,0] : [0,1]
|
148
|
+
|
149
|
+
expect(events[existing_path_index].get(file_path_target_field)).to eq "my_path"
|
150
|
+
expect(events[existing_path_index].get(source_host_target_field)).to eq "my_host"
|
151
|
+
expect(events[existing_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
|
152
|
+
|
153
|
+
expect(events[added_path_index].get(file_path_target_field)).to eq "#{tmpfile_path}"
|
154
|
+
expect(events[added_path_index].get(source_host_target_field)).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
|
155
|
+
expect(events[added_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
|
156
|
+
end
|
137
157
|
end
|
138
158
|
end
|
139
159
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-file
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 4.
|
4
|
+
version: 4.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-04-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -86,6 +86,20 @@ dependencies:
|
|
86
86
|
- - "~>"
|
87
87
|
- !ruby/object:Gem::Version
|
88
88
|
version: '3.0'
|
89
|
+
- !ruby/object:Gem::Dependency
|
90
|
+
requirement: !ruby/object:Gem::Requirement
|
91
|
+
requirements:
|
92
|
+
- - "~>"
|
93
|
+
- !ruby/object:Gem::Version
|
94
|
+
version: '1.1'
|
95
|
+
name: logstash-mixin-ecs_compatibility_support
|
96
|
+
prerelease: false
|
97
|
+
type: :runtime
|
98
|
+
version_requirements: !ruby/object:Gem::Requirement
|
99
|
+
requirements:
|
100
|
+
- - "~>"
|
101
|
+
- !ruby/object:Gem::Version
|
102
|
+
version: '1.1'
|
89
103
|
- !ruby/object:Gem::Dependency
|
90
104
|
requirement: !ruby/object:Gem::Requirement
|
91
105
|
requirements:
|