logstash-input-file 2.0.2 → 2.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +11 -1
- data/CONTRIBUTORS +1 -0
- data/README.md +3 -0
- data/lib/logstash/inputs/file.rb +24 -11
- data/logstash-input-file.gemspec +2 -1
- data/spec/inputs/file_spec.rb +71 -6
- metadata +42 -28
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5377f5857709a60359d657a8007d6ab0b3f87f4d
|
4
|
+
data.tar.gz: 1af3cfea2b4135413c3cb347e61b7c0ff89985cc
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2bd62efe503f31050170ebc8f17455f8c108bb0e0430a7ee1718250831292a90bf218bec728a418aa0cb3644cd587f50dadb35ce51ed6b5256e353bfe90d143b
|
7
|
+
data.tar.gz: 69ecdc00558c2b9da3a1d7e989e1a4936e0c24ab4a7f1c420bc64ba2dbd460f6751014821f71e25263b57dbd8e4598e637688233eee9233587ac73ceb58d776c
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,15 @@
|
|
1
|
+
## 2.0.3
|
2
|
+
- Implement Stream Identity mapping of codecs: distinct codecs will collect input per stream identity (filename)
|
3
|
+
|
4
|
+
## 2.0.2
|
5
|
+
- Change LS core dependency version
|
6
|
+
- Add CI badge
|
7
|
+
|
8
|
+
## 2.0.1
|
9
|
+
- Change LS core dependency version
|
10
|
+
|
1
11
|
## 2.0.0
|
2
|
-
- Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
|
12
|
+
- Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
|
3
13
|
instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
|
4
14
|
- Dependency on logstash-core update to 2.0
|
5
15
|
|
data/CONTRIBUTORS
CHANGED
@@ -17,6 +17,7 @@ Contributors:
|
|
17
17
|
* Tejay Cardon (tejaycar)
|
18
18
|
* elliot moore (em295)
|
19
19
|
* yjpa7145
|
20
|
+
* Guy Boertje (guyboertje)
|
20
21
|
|
21
22
|
Note: If you've sent us patches, bug reports, or otherwise contributed to
|
22
23
|
Logstash, and you aren't on the list above and want to be, please let us know
|
data/README.md
CHANGED
@@ -1,5 +1,8 @@
|
|
1
1
|
# Logstash Plugin
|
2
2
|
|
3
|
+
[](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Inputs/job/logstash-plugin-input-file-unit/)
|
5
|
+
|
3
6
|
This is a plugin for [Logstash](https://github.com/elastic/logstash).
|
4
7
|
|
5
8
|
It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
|
data/lib/logstash/inputs/file.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
# encoding: utf-8
|
2
|
-
require "logstash/inputs/base"
|
3
2
|
require "logstash/namespace"
|
3
|
+
require "logstash/inputs/base"
|
4
|
+
require "logstash/codecs/identity_map_codec"
|
4
5
|
|
5
6
|
require "pathname"
|
6
7
|
require "socket" # for Socket.gethostname
|
@@ -181,27 +182,39 @@ class LogStash::Inputs::File < LogStash::Inputs::Base
|
|
181
182
|
if @start_position == "beginning"
|
182
183
|
@tail_config[:start_new_files_at] = :beginning
|
183
184
|
end
|
185
|
+
|
186
|
+
@codec = LogStash::Codecs::IdentityMapCodec.new(@codec)
|
184
187
|
end # def register
|
185
188
|
|
186
|
-
public
|
187
189
|
def run(queue)
|
188
190
|
@tail = FileWatch::Tail.new(@tail_config)
|
189
191
|
@tail.logger = @logger
|
190
192
|
@path.each { |path| @tail.tail(path) }
|
191
|
-
|
192
193
|
@tail.subscribe do |path, line|
|
193
|
-
|
194
|
-
@codec.decode(line) do |event|
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
queue << event
|
194
|
+
log_line_received(path, line)
|
195
|
+
@codec.decode(line, path) do |event|
|
196
|
+
# path is the identity
|
197
|
+
# Note: this block is cached in the
|
198
|
+
# identity_map_codec for use when
|
199
|
+
# buffered lines are flushed.
|
200
|
+
queue << add_path_meta(event, path)
|
200
201
|
end
|
201
202
|
end
|
202
203
|
end # def run
|
203
204
|
|
204
|
-
|
205
|
+
def log_line_received(path, line)
|
206
|
+
return if !@logger.debug?
|
207
|
+
@logger.debug("Received line", :path => path, :text => line)
|
208
|
+
end
|
209
|
+
|
210
|
+
def add_path_meta(event, path)
|
211
|
+
event["[@metadata][path]"] = path
|
212
|
+
event["host"] = @host if !event.include?("host")
|
213
|
+
event["path"] = path if !event.include?("path")
|
214
|
+
decorate(event)
|
215
|
+
event
|
216
|
+
end
|
217
|
+
|
205
218
|
def stop
|
206
219
|
@tail.quit if @tail # _sincedb_write is called implicitly
|
207
220
|
end
|
data/logstash-input-file.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-input-file'
|
4
|
-
s.version = '2.0.
|
4
|
+
s.version = '2.0.3'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Stream events from files."
|
7
7
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|
@@ -25,6 +25,7 @@ Gem::Specification.new do |s|
|
|
25
25
|
s.add_runtime_dependency 'logstash-codec-plain'
|
26
26
|
s.add_runtime_dependency 'addressable'
|
27
27
|
s.add_runtime_dependency 'filewatch', ['>= 0.6.5', '~> 0.6']
|
28
|
+
s.add_runtime_dependency 'logstash-codec-multiline', ['~> 2.0.3']
|
28
29
|
|
29
30
|
s.add_development_dependency 'stud', ['~> 0.0.19']
|
30
31
|
s.add_development_dependency 'logstash-devutils'
|
data/spec/inputs/file_spec.rb
CHANGED
@@ -5,9 +5,9 @@ require "tempfile"
|
|
5
5
|
require "stud/temporary"
|
6
6
|
require "logstash/inputs/file"
|
7
7
|
|
8
|
-
|
8
|
+
FILE_DELIMITER = LogStash::Environment.windows? ? "\r\n" : "\n"
|
9
9
|
|
10
|
-
|
10
|
+
describe LogStash::Inputs::File do
|
11
11
|
|
12
12
|
it_behaves_like "an interruptible input plugin" do
|
13
13
|
let(:config) do
|
@@ -28,7 +28,7 @@ describe LogStash::Inputs::File do
|
|
28
28
|
type => "blah"
|
29
29
|
path => "#{tmpfile_path}"
|
30
30
|
sincedb_path => "#{sincedb_path}"
|
31
|
-
delimiter => "#{
|
31
|
+
delimiter => "#{FILE_DELIMITER}"
|
32
32
|
}
|
33
33
|
}
|
34
34
|
CONFIG
|
@@ -82,7 +82,7 @@ describe LogStash::Inputs::File do
|
|
82
82
|
path => "#{tmpfile_path}"
|
83
83
|
start_position => "beginning"
|
84
84
|
sincedb_path => "#{sincedb_path}"
|
85
|
-
delimiter => "#{
|
85
|
+
delimiter => "#{FILE_DELIMITER}"
|
86
86
|
}
|
87
87
|
}
|
88
88
|
CONFIG
|
@@ -111,7 +111,7 @@ describe LogStash::Inputs::File do
|
|
111
111
|
path => "#{tmpfile_path}"
|
112
112
|
start_position => "beginning"
|
113
113
|
sincedb_path => "#{sincedb_path}"
|
114
|
-
delimiter => "#{
|
114
|
+
delimiter => "#{FILE_DELIMITER}"
|
115
115
|
}
|
116
116
|
}
|
117
117
|
CONFIG
|
@@ -154,7 +154,7 @@ describe LogStash::Inputs::File do
|
|
154
154
|
path => "#{tmpfile_path}"
|
155
155
|
start_position => "beginning"
|
156
156
|
sincedb_path => "#{sincedb_path}"
|
157
|
-
delimiter => "#{
|
157
|
+
delimiter => "#{FILE_DELIMITER}"
|
158
158
|
codec => "json"
|
159
159
|
}
|
160
160
|
}
|
@@ -189,4 +189,69 @@ describe LogStash::Inputs::File do
|
|
189
189
|
expect { subject.register }.to raise_error(ArgumentError)
|
190
190
|
end
|
191
191
|
end
|
192
|
+
|
193
|
+
context "when wildcard path and a multiline codec is specified" do
|
194
|
+
let(:tmpdir_path) { Stud::Temporary.directory }
|
195
|
+
let(:sincedb_path) { Stud::Temporary.pathname }
|
196
|
+
let(:conf) do
|
197
|
+
<<-CONFIG
|
198
|
+
input {
|
199
|
+
file {
|
200
|
+
type => "blah"
|
201
|
+
path => "#{tmpdir_path}/*.log"
|
202
|
+
start_position => "beginning"
|
203
|
+
sincedb_path => "#{sincedb_path}"
|
204
|
+
delimiter => "#{FILE_DELIMITER}"
|
205
|
+
codec => multiline { pattern => "^\s" what => previous }
|
206
|
+
}
|
207
|
+
}
|
208
|
+
CONFIG
|
209
|
+
end
|
210
|
+
|
211
|
+
let(:writer_proc) do
|
212
|
+
-> do
|
213
|
+
File.open("#{tmpdir_path}/a.log", "a") do |fd|
|
214
|
+
fd.puts("line1.1-of-a")
|
215
|
+
fd.puts(" line1.2-of-a")
|
216
|
+
fd.puts(" line1.3-of-a")
|
217
|
+
fd.puts("line2.1-of-a")
|
218
|
+
end
|
219
|
+
|
220
|
+
File.open("#{tmpdir_path}/z.log", "a") do |fd|
|
221
|
+
fd.puts("line1.1-of-z")
|
222
|
+
fd.puts(" line1.2-of-z")
|
223
|
+
fd.puts(" line1.3-of-z")
|
224
|
+
fd.puts("line2.1-of-z")
|
225
|
+
end
|
226
|
+
end
|
227
|
+
end
|
228
|
+
|
229
|
+
after do
|
230
|
+
FileUtils.rm_rf(tmpdir_path)
|
231
|
+
end
|
232
|
+
|
233
|
+
let(:event_count) { 2 }
|
234
|
+
|
235
|
+
it "collects separate multiple line events from each file" do
|
236
|
+
writer_proc.call
|
237
|
+
|
238
|
+
events = input(conf) do |pipeline, queue|
|
239
|
+
queue.size.times.collect { queue.pop }
|
240
|
+
end
|
241
|
+
|
242
|
+
expect(events.size).to eq(event_count)
|
243
|
+
|
244
|
+
e1_message = events[0]["message"]
|
245
|
+
e2_message = events[1]["message"]
|
246
|
+
|
247
|
+
# can't assume File A will be read first
|
248
|
+
if e1_message.start_with?('line1.1-of-z')
|
249
|
+
expect(e1_message).to eq("line1.1-of-z#{FILE_DELIMITER} line1.2-of-z#{FILE_DELIMITER} line1.3-of-z")
|
250
|
+
expect(e2_message).to eq("line1.1-of-a#{FILE_DELIMITER} line1.2-of-a#{FILE_DELIMITER} line1.3-of-a")
|
251
|
+
else
|
252
|
+
expect(e1_message).to eq("line1.1-of-a#{FILE_DELIMITER} line1.2-of-a#{FILE_DELIMITER} line1.3-of-a")
|
253
|
+
expect(e2_message).to eq("line1.1-of-z#{FILE_DELIMITER} line1.2-of-z#{FILE_DELIMITER} line1.3-of-z")
|
254
|
+
end
|
255
|
+
end
|
256
|
+
end
|
192
257
|
end
|
metadata
CHANGED
@@ -1,17 +1,18 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-file
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0.
|
4
|
+
version: 2.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-11-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
|
14
|
+
name: logstash-core
|
15
|
+
version_requirements: !ruby/object:Gem::Requirement
|
15
16
|
requirements:
|
16
17
|
- - '>='
|
17
18
|
- !ruby/object:Gem::Version
|
@@ -19,10 +20,7 @@ dependencies:
|
|
19
20
|
- - <
|
20
21
|
- !ruby/object:Gem::Version
|
21
22
|
version: 3.0.0
|
22
|
-
|
23
|
-
prerelease: false
|
24
|
-
type: :runtime
|
25
|
-
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirement: !ruby/object:Gem::Requirement
|
26
24
|
requirements:
|
27
25
|
- - '>='
|
28
26
|
- !ruby/object:Gem::Version
|
@@ -30,35 +28,46 @@ dependencies:
|
|
30
28
|
- - <
|
31
29
|
- !ruby/object:Gem::Version
|
32
30
|
version: 3.0.0
|
31
|
+
prerelease: false
|
32
|
+
type: :runtime
|
33
33
|
- !ruby/object:Gem::Dependency
|
34
|
+
name: logstash-codec-plain
|
35
|
+
version_requirements: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - '>='
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: '0'
|
34
40
|
requirement: !ruby/object:Gem::Requirement
|
35
41
|
requirements:
|
36
42
|
- - '>='
|
37
43
|
- !ruby/object:Gem::Version
|
38
44
|
version: '0'
|
39
|
-
name: logstash-codec-plain
|
40
45
|
prerelease: false
|
41
46
|
type: :runtime
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: addressable
|
42
49
|
version_requirements: !ruby/object:Gem::Requirement
|
43
50
|
requirements:
|
44
51
|
- - '>='
|
45
52
|
- !ruby/object:Gem::Version
|
46
53
|
version: '0'
|
47
|
-
- !ruby/object:Gem::Dependency
|
48
54
|
requirement: !ruby/object:Gem::Requirement
|
49
55
|
requirements:
|
50
56
|
- - '>='
|
51
57
|
- !ruby/object:Gem::Version
|
52
58
|
version: '0'
|
53
|
-
name: addressable
|
54
59
|
prerelease: false
|
55
60
|
type: :runtime
|
61
|
+
- !ruby/object:Gem::Dependency
|
62
|
+
name: filewatch
|
56
63
|
version_requirements: !ruby/object:Gem::Requirement
|
57
64
|
requirements:
|
58
65
|
- - '>='
|
59
66
|
- !ruby/object:Gem::Version
|
60
|
-
version:
|
61
|
-
-
|
67
|
+
version: 0.6.5
|
68
|
+
- - ~>
|
69
|
+
- !ruby/object:Gem::Version
|
70
|
+
version: '0.6'
|
62
71
|
requirement: !ruby/object:Gem::Requirement
|
63
72
|
requirements:
|
64
73
|
- - '>='
|
@@ -67,59 +76,64 @@ dependencies:
|
|
67
76
|
- - ~>
|
68
77
|
- !ruby/object:Gem::Version
|
69
78
|
version: '0.6'
|
70
|
-
name: filewatch
|
71
79
|
prerelease: false
|
72
80
|
type: :runtime
|
81
|
+
- !ruby/object:Gem::Dependency
|
82
|
+
name: logstash-codec-multiline
|
73
83
|
version_requirements: !ruby/object:Gem::Requirement
|
74
84
|
requirements:
|
75
|
-
- - '>='
|
76
|
-
- !ruby/object:Gem::Version
|
77
|
-
version: 0.6.5
|
78
85
|
- - ~>
|
79
86
|
- !ruby/object:Gem::Version
|
80
|
-
version:
|
81
|
-
- !ruby/object:Gem::Dependency
|
87
|
+
version: 2.0.3
|
82
88
|
requirement: !ruby/object:Gem::Requirement
|
83
89
|
requirements:
|
84
90
|
- - ~>
|
85
91
|
- !ruby/object:Gem::Version
|
86
|
-
version:
|
87
|
-
name: stud
|
92
|
+
version: 2.0.3
|
88
93
|
prerelease: false
|
89
|
-
type: :
|
94
|
+
type: :runtime
|
95
|
+
- !ruby/object:Gem::Dependency
|
96
|
+
name: stud
|
90
97
|
version_requirements: !ruby/object:Gem::Requirement
|
91
98
|
requirements:
|
92
99
|
- - ~>
|
93
100
|
- !ruby/object:Gem::Version
|
94
101
|
version: 0.0.19
|
95
|
-
- !ruby/object:Gem::Dependency
|
96
102
|
requirement: !ruby/object:Gem::Requirement
|
97
103
|
requirements:
|
98
|
-
- -
|
104
|
+
- - ~>
|
99
105
|
- !ruby/object:Gem::Version
|
100
|
-
version:
|
101
|
-
name: logstash-devutils
|
106
|
+
version: 0.0.19
|
102
107
|
prerelease: false
|
103
108
|
type: :development
|
109
|
+
- !ruby/object:Gem::Dependency
|
110
|
+
name: logstash-devutils
|
104
111
|
version_requirements: !ruby/object:Gem::Requirement
|
105
112
|
requirements:
|
106
113
|
- - '>='
|
107
114
|
- !ruby/object:Gem::Version
|
108
115
|
version: '0'
|
109
|
-
- !ruby/object:Gem::Dependency
|
110
116
|
requirement: !ruby/object:Gem::Requirement
|
111
117
|
requirements:
|
112
118
|
- - '>='
|
113
119
|
- !ruby/object:Gem::Version
|
114
120
|
version: '0'
|
115
|
-
name: logstash-codec-json
|
116
121
|
prerelease: false
|
117
122
|
type: :development
|
123
|
+
- !ruby/object:Gem::Dependency
|
124
|
+
name: logstash-codec-json
|
118
125
|
version_requirements: !ruby/object:Gem::Requirement
|
119
126
|
requirements:
|
120
127
|
- - '>='
|
121
128
|
- !ruby/object:Gem::Version
|
122
129
|
version: '0'
|
130
|
+
requirement: !ruby/object:Gem::Requirement
|
131
|
+
requirements:
|
132
|
+
- - '>='
|
133
|
+
- !ruby/object:Gem::Version
|
134
|
+
version: '0'
|
135
|
+
prerelease: false
|
136
|
+
type: :development
|
123
137
|
description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
|
124
138
|
email: info@elastic.co
|
125
139
|
executables: []
|
@@ -157,7 +171,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
157
171
|
version: '0'
|
158
172
|
requirements: []
|
159
173
|
rubyforge_project:
|
160
|
-
rubygems_version: 2.4.
|
174
|
+
rubygems_version: 2.4.5
|
161
175
|
signing_key:
|
162
176
|
specification_version: 4
|
163
177
|
summary: Stream events from files.
|