logstash-input-file 2.0.2 → 2.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 68859f099301705b718870b36fe7bf1f1347573f
4
- data.tar.gz: 5916cfae7c42b57ddbd274b5ae3321a58e7d4eca
3
+ metadata.gz: 5377f5857709a60359d657a8007d6ab0b3f87f4d
4
+ data.tar.gz: 1af3cfea2b4135413c3cb347e61b7c0ff89985cc
5
5
  SHA512:
6
- metadata.gz: 6bec077461e23f681a0bbd852edaa61015a7df065938e1f1b43842c1a3d662fd30eaa31bf4c42e43c276835d0144c4d65a247ac803dfa656878f367a502b4bcb
7
- data.tar.gz: ca84de7eb86d11f34cc1540b5efe41a76bdbde907a637d66c96b97267d445843f86515f171ce2365085d9f86c7c850f14528c1226f103ed6b10c7bff97c739cc
6
+ metadata.gz: 2bd62efe503f31050170ebc8f17455f8c108bb0e0430a7ee1718250831292a90bf218bec728a418aa0cb3644cd587f50dadb35ce51ed6b5256e353bfe90d143b
7
+ data.tar.gz: 69ecdc00558c2b9da3a1d7e989e1a4936e0c24ab4a7f1c420bc64ba2dbd460f6751014821f71e25263b57dbd8e4598e637688233eee9233587ac73ceb58d776c
@@ -1,5 +1,15 @@
1
+ ## 2.0.3
2
+ - Implement Stream Identity mapping of codecs: distinct codecs will collect input per stream identity (filename)
3
+
4
+ ## 2.0.2
5
+ - Change LS core dependency version
6
+ - Add CI badge
7
+
8
+ ## 2.0.1
9
+ - Change LS core dependency version
10
+
1
11
  ## 2.0.0
2
- - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
12
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
3
13
  instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
4
14
  - Dependency on logstash-core update to 2.0
5
15
 
@@ -17,6 +17,7 @@ Contributors:
17
17
  * Tejay Cardon (tejaycar)
18
18
  * elliot moore (em295)
19
19
  * yjpa7145
20
+ * Guy Boertje (guyboertje)
20
21
 
21
22
  Note: If you've sent us patches, bug reports, or otherwise contributed to
22
23
  Logstash, and you aren't on the list above and want to be, please let us know
data/README.md CHANGED
@@ -1,5 +1,8 @@
1
1
  # Logstash Plugin
2
2
 
3
+ [![Build
4
+ Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Inputs/job/logstash-plugin-input-file-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Inputs/job/logstash-plugin-input-file-unit/)
5
+
3
6
  This is a plugin for [Logstash](https://github.com/elastic/logstash).
4
7
 
5
8
  It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
@@ -1,6 +1,7 @@
1
1
  # encoding: utf-8
2
- require "logstash/inputs/base"
3
2
  require "logstash/namespace"
3
+ require "logstash/inputs/base"
4
+ require "logstash/codecs/identity_map_codec"
4
5
 
5
6
  require "pathname"
6
7
  require "socket" # for Socket.gethostname
@@ -181,27 +182,39 @@ class LogStash::Inputs::File < LogStash::Inputs::Base
181
182
  if @start_position == "beginning"
182
183
  @tail_config[:start_new_files_at] = :beginning
183
184
  end
185
+
186
+ @codec = LogStash::Codecs::IdentityMapCodec.new(@codec)
184
187
  end # def register
185
188
 
186
- public
187
189
  def run(queue)
188
190
  @tail = FileWatch::Tail.new(@tail_config)
189
191
  @tail.logger = @logger
190
192
  @path.each { |path| @tail.tail(path) }
191
-
192
193
  @tail.subscribe do |path, line|
193
- @logger.debug? && @logger.debug("Received line", :path => path, :text => line)
194
- @codec.decode(line) do |event|
195
- event["[@metadata][path]"] = path
196
- event["host"] = @host if !event.include?("host")
197
- event["path"] = path if !event.include?("path")
198
- decorate(event)
199
- queue << event
194
+ log_line_received(path, line)
195
+ @codec.decode(line, path) do |event|
196
+ # path is the identity
197
+ # Note: this block is cached in the
198
+ # identity_map_codec for use when
199
+ # buffered lines are flushed.
200
+ queue << add_path_meta(event, path)
200
201
  end
201
202
  end
202
203
  end # def run
203
204
 
204
- public
205
+ def log_line_received(path, line)
206
+ return if !@logger.debug?
207
+ @logger.debug("Received line", :path => path, :text => line)
208
+ end
209
+
210
+ def add_path_meta(event, path)
211
+ event["[@metadata][path]"] = path
212
+ event["host"] = @host if !event.include?("host")
213
+ event["path"] = path if !event.include?("path")
214
+ decorate(event)
215
+ event
216
+ end
217
+
205
218
  def stop
206
219
  @tail.quit if @tail # _sincedb_write is called implicitly
207
220
  end
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-input-file'
4
- s.version = '2.0.2'
4
+ s.version = '2.0.3'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = "Stream events from files."
7
7
  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -25,6 +25,7 @@ Gem::Specification.new do |s|
25
25
  s.add_runtime_dependency 'logstash-codec-plain'
26
26
  s.add_runtime_dependency 'addressable'
27
27
  s.add_runtime_dependency 'filewatch', ['>= 0.6.5', '~> 0.6']
28
+ s.add_runtime_dependency 'logstash-codec-multiline', ['~> 2.0.3']
28
29
 
29
30
  s.add_development_dependency 'stud', ['~> 0.0.19']
30
31
  s.add_development_dependency 'logstash-devutils'
@@ -5,9 +5,9 @@ require "tempfile"
5
5
  require "stud/temporary"
6
6
  require "logstash/inputs/file"
7
7
 
8
- describe LogStash::Inputs::File do
8
+ FILE_DELIMITER = LogStash::Environment.windows? ? "\r\n" : "\n"
9
9
 
10
- delimiter = (LogStash::Environment.windows? ? "\r\n" : "\n")
10
+ describe LogStash::Inputs::File do
11
11
 
12
12
  it_behaves_like "an interruptible input plugin" do
13
13
  let(:config) do
@@ -28,7 +28,7 @@ describe LogStash::Inputs::File do
28
28
  type => "blah"
29
29
  path => "#{tmpfile_path}"
30
30
  sincedb_path => "#{sincedb_path}"
31
- delimiter => "#{delimiter}"
31
+ delimiter => "#{FILE_DELIMITER}"
32
32
  }
33
33
  }
34
34
  CONFIG
@@ -82,7 +82,7 @@ describe LogStash::Inputs::File do
82
82
  path => "#{tmpfile_path}"
83
83
  start_position => "beginning"
84
84
  sincedb_path => "#{sincedb_path}"
85
- delimiter => "#{delimiter}"
85
+ delimiter => "#{FILE_DELIMITER}"
86
86
  }
87
87
  }
88
88
  CONFIG
@@ -111,7 +111,7 @@ describe LogStash::Inputs::File do
111
111
  path => "#{tmpfile_path}"
112
112
  start_position => "beginning"
113
113
  sincedb_path => "#{sincedb_path}"
114
- delimiter => "#{delimiter}"
114
+ delimiter => "#{FILE_DELIMITER}"
115
115
  }
116
116
  }
117
117
  CONFIG
@@ -154,7 +154,7 @@ describe LogStash::Inputs::File do
154
154
  path => "#{tmpfile_path}"
155
155
  start_position => "beginning"
156
156
  sincedb_path => "#{sincedb_path}"
157
- delimiter => "#{delimiter}"
157
+ delimiter => "#{FILE_DELIMITER}"
158
158
  codec => "json"
159
159
  }
160
160
  }
@@ -189,4 +189,69 @@ describe LogStash::Inputs::File do
189
189
  expect { subject.register }.to raise_error(ArgumentError)
190
190
  end
191
191
  end
192
+
193
+ context "when wildcard path and a multiline codec is specified" do
194
+ let(:tmpdir_path) { Stud::Temporary.directory }
195
+ let(:sincedb_path) { Stud::Temporary.pathname }
196
+ let(:conf) do
197
+ <<-CONFIG
198
+ input {
199
+ file {
200
+ type => "blah"
201
+ path => "#{tmpdir_path}/*.log"
202
+ start_position => "beginning"
203
+ sincedb_path => "#{sincedb_path}"
204
+ delimiter => "#{FILE_DELIMITER}"
205
+ codec => multiline { pattern => "^\s" what => previous }
206
+ }
207
+ }
208
+ CONFIG
209
+ end
210
+
211
+ let(:writer_proc) do
212
+ -> do
213
+ File.open("#{tmpdir_path}/a.log", "a") do |fd|
214
+ fd.puts("line1.1-of-a")
215
+ fd.puts(" line1.2-of-a")
216
+ fd.puts(" line1.3-of-a")
217
+ fd.puts("line2.1-of-a")
218
+ end
219
+
220
+ File.open("#{tmpdir_path}/z.log", "a") do |fd|
221
+ fd.puts("line1.1-of-z")
222
+ fd.puts(" line1.2-of-z")
223
+ fd.puts(" line1.3-of-z")
224
+ fd.puts("line2.1-of-z")
225
+ end
226
+ end
227
+ end
228
+
229
+ after do
230
+ FileUtils.rm_rf(tmpdir_path)
231
+ end
232
+
233
+ let(:event_count) { 2 }
234
+
235
+ it "collects separate multiple line events from each file" do
236
+ writer_proc.call
237
+
238
+ events = input(conf) do |pipeline, queue|
239
+ queue.size.times.collect { queue.pop }
240
+ end
241
+
242
+ expect(events.size).to eq(event_count)
243
+
244
+ e1_message = events[0]["message"]
245
+ e2_message = events[1]["message"]
246
+
247
+ # can't assume File A will be read first
248
+ if e1_message.start_with?('line1.1-of-z')
249
+ expect(e1_message).to eq("line1.1-of-z#{FILE_DELIMITER} line1.2-of-z#{FILE_DELIMITER} line1.3-of-z")
250
+ expect(e2_message).to eq("line1.1-of-a#{FILE_DELIMITER} line1.2-of-a#{FILE_DELIMITER} line1.3-of-a")
251
+ else
252
+ expect(e1_message).to eq("line1.1-of-a#{FILE_DELIMITER} line1.2-of-a#{FILE_DELIMITER} line1.3-of-a")
253
+ expect(e2_message).to eq("line1.1-of-z#{FILE_DELIMITER} line1.2-of-z#{FILE_DELIMITER} line1.3-of-z")
254
+ end
255
+ end
256
+ end
192
257
  end
metadata CHANGED
@@ -1,17 +1,18 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-input-file
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.2
4
+ version: 2.0.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-10-14 00:00:00.000000000 Z
11
+ date: 2015-11-17 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- requirement: !ruby/object:Gem::Requirement
14
+ name: logstash-core
15
+ version_requirements: !ruby/object:Gem::Requirement
15
16
  requirements:
16
17
  - - '>='
17
18
  - !ruby/object:Gem::Version
@@ -19,10 +20,7 @@ dependencies:
19
20
  - - <
20
21
  - !ruby/object:Gem::Version
21
22
  version: 3.0.0
22
- name: logstash-core
23
- prerelease: false
24
- type: :runtime
25
- version_requirements: !ruby/object:Gem::Requirement
23
+ requirement: !ruby/object:Gem::Requirement
26
24
  requirements:
27
25
  - - '>='
28
26
  - !ruby/object:Gem::Version
@@ -30,35 +28,46 @@ dependencies:
30
28
  - - <
31
29
  - !ruby/object:Gem::Version
32
30
  version: 3.0.0
31
+ prerelease: false
32
+ type: :runtime
33
33
  - !ruby/object:Gem::Dependency
34
+ name: logstash-codec-plain
35
+ version_requirements: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - '>='
38
+ - !ruby/object:Gem::Version
39
+ version: '0'
34
40
  requirement: !ruby/object:Gem::Requirement
35
41
  requirements:
36
42
  - - '>='
37
43
  - !ruby/object:Gem::Version
38
44
  version: '0'
39
- name: logstash-codec-plain
40
45
  prerelease: false
41
46
  type: :runtime
47
+ - !ruby/object:Gem::Dependency
48
+ name: addressable
42
49
  version_requirements: !ruby/object:Gem::Requirement
43
50
  requirements:
44
51
  - - '>='
45
52
  - !ruby/object:Gem::Version
46
53
  version: '0'
47
- - !ruby/object:Gem::Dependency
48
54
  requirement: !ruby/object:Gem::Requirement
49
55
  requirements:
50
56
  - - '>='
51
57
  - !ruby/object:Gem::Version
52
58
  version: '0'
53
- name: addressable
54
59
  prerelease: false
55
60
  type: :runtime
61
+ - !ruby/object:Gem::Dependency
62
+ name: filewatch
56
63
  version_requirements: !ruby/object:Gem::Requirement
57
64
  requirements:
58
65
  - - '>='
59
66
  - !ruby/object:Gem::Version
60
- version: '0'
61
- - !ruby/object:Gem::Dependency
67
+ version: 0.6.5
68
+ - - ~>
69
+ - !ruby/object:Gem::Version
70
+ version: '0.6'
62
71
  requirement: !ruby/object:Gem::Requirement
63
72
  requirements:
64
73
  - - '>='
@@ -67,59 +76,64 @@ dependencies:
67
76
  - - ~>
68
77
  - !ruby/object:Gem::Version
69
78
  version: '0.6'
70
- name: filewatch
71
79
  prerelease: false
72
80
  type: :runtime
81
+ - !ruby/object:Gem::Dependency
82
+ name: logstash-codec-multiline
73
83
  version_requirements: !ruby/object:Gem::Requirement
74
84
  requirements:
75
- - - '>='
76
- - !ruby/object:Gem::Version
77
- version: 0.6.5
78
85
  - - ~>
79
86
  - !ruby/object:Gem::Version
80
- version: '0.6'
81
- - !ruby/object:Gem::Dependency
87
+ version: 2.0.3
82
88
  requirement: !ruby/object:Gem::Requirement
83
89
  requirements:
84
90
  - - ~>
85
91
  - !ruby/object:Gem::Version
86
- version: 0.0.19
87
- name: stud
92
+ version: 2.0.3
88
93
  prerelease: false
89
- type: :development
94
+ type: :runtime
95
+ - !ruby/object:Gem::Dependency
96
+ name: stud
90
97
  version_requirements: !ruby/object:Gem::Requirement
91
98
  requirements:
92
99
  - - ~>
93
100
  - !ruby/object:Gem::Version
94
101
  version: 0.0.19
95
- - !ruby/object:Gem::Dependency
96
102
  requirement: !ruby/object:Gem::Requirement
97
103
  requirements:
98
- - - '>='
104
+ - - ~>
99
105
  - !ruby/object:Gem::Version
100
- version: '0'
101
- name: logstash-devutils
106
+ version: 0.0.19
102
107
  prerelease: false
103
108
  type: :development
109
+ - !ruby/object:Gem::Dependency
110
+ name: logstash-devutils
104
111
  version_requirements: !ruby/object:Gem::Requirement
105
112
  requirements:
106
113
  - - '>='
107
114
  - !ruby/object:Gem::Version
108
115
  version: '0'
109
- - !ruby/object:Gem::Dependency
110
116
  requirement: !ruby/object:Gem::Requirement
111
117
  requirements:
112
118
  - - '>='
113
119
  - !ruby/object:Gem::Version
114
120
  version: '0'
115
- name: logstash-codec-json
116
121
  prerelease: false
117
122
  type: :development
123
+ - !ruby/object:Gem::Dependency
124
+ name: logstash-codec-json
118
125
  version_requirements: !ruby/object:Gem::Requirement
119
126
  requirements:
120
127
  - - '>='
121
128
  - !ruby/object:Gem::Version
122
129
  version: '0'
130
+ requirement: !ruby/object:Gem::Requirement
131
+ requirements:
132
+ - - '>='
133
+ - !ruby/object:Gem::Version
134
+ version: '0'
135
+ prerelease: false
136
+ type: :development
123
137
  description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
124
138
  email: info@elastic.co
125
139
  executables: []
@@ -157,7 +171,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
157
171
  version: '0'
158
172
  requirements: []
159
173
  rubyforge_project:
160
- rubygems_version: 2.4.8
174
+ rubygems_version: 2.4.5
161
175
  signing_key:
162
176
  specification_version: 4
163
177
  summary: Stream events from files.