logstash-input-s3 2.0.6 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +2 -0
- data/Gemfile +2 -0
- data/LICENSE +1 -1
- data/README.md +12 -3
- data/lib/logstash/inputs/s3.rb +5 -5
- data/logstash-input-s3.gemspec +3 -3
- data/spec/inputs/s3_spec.rb +2 -2
- metadata +26 -24
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 05a4b10636fe50d3eb875cc1812243b665f0863a
|
4
|
+
data.tar.gz: 34ab5b06c255265127e2a57bac4d7f722a16076a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 11cdc47bb3b33b8f34c299a89a429fe439aca9d3ddc3a1e618ffb762895db283431cb125f1cb9033ce774c7e666616e5145dfff24e9908e2bb46a7df5111f5f8
|
7
|
+
data.tar.gz: c2daad5e00304afde37f2fd32fc859bb48b8214a146e5b807dec1f146a8f3652e24d22fee64a5593dc8f70c7d2fda3db66f3e2bdd4bac39dc2b4855345220a54
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
## 3.0.0
|
2
|
+
- Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
|
1
3
|
# 2.0.6
|
2
4
|
- Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
|
3
5
|
# 2.0.5
|
data/Gemfile
CHANGED
data/LICENSE
CHANGED
data/README.md
CHANGED
@@ -1,7 +1,6 @@
|
|
1
1
|
# Logstash Plugin
|
2
2
|
|
3
|
-
[](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Inputs/job/logstash-plugin-input-s3-unit/)
|
3
|
+
[](https://travis-ci.org/logstash-plugins/logstash-input-s3)
|
5
4
|
|
6
5
|
This is a plugin for [Logstash](https://github.com/elastic/logstash).
|
7
6
|
|
@@ -74,7 +73,12 @@ gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
|
|
74
73
|
```
|
75
74
|
- Install plugin
|
76
75
|
```sh
|
76
|
+
# Logstash 2.3 and higher
|
77
|
+
bin/logstash-plugin install --no-verify
|
78
|
+
|
79
|
+
# Prior to Logstash 2.3
|
77
80
|
bin/plugin install --no-verify
|
81
|
+
|
78
82
|
```
|
79
83
|
- Run Logstash with your plugin
|
80
84
|
```sh
|
@@ -92,7 +96,12 @@ gem build logstash-filter-awesome.gemspec
|
|
92
96
|
```
|
93
97
|
- Install the plugin from the Logstash home
|
94
98
|
```sh
|
95
|
-
|
99
|
+
# Logstash 2.3 and higher
|
100
|
+
bin/logstash-plugin install --no-verify
|
101
|
+
|
102
|
+
# Prior to Logstash 2.3
|
103
|
+
bin/plugin install --no-verify
|
104
|
+
|
96
105
|
```
|
97
106
|
- Start Logstash and proceed to test the plugin
|
98
107
|
|
data/lib/logstash/inputs/s3.rb
CHANGED
@@ -207,8 +207,8 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
|
|
207
207
|
else
|
208
208
|
decorate(event)
|
209
209
|
|
210
|
-
event
|
211
|
-
event
|
210
|
+
event.set("cloudfront_version", metadata[:cloudfront_version]) unless metadata[:cloudfront_version].nil?
|
211
|
+
event.set("cloudfront_fields", metadata[:cloudfront_fields]) unless metadata[:cloudfront_fields].nil?
|
212
212
|
|
213
213
|
queue << event
|
214
214
|
end
|
@@ -220,8 +220,8 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
|
|
220
220
|
|
221
221
|
private
|
222
222
|
def event_is_metadata?(event)
|
223
|
-
return false if event
|
224
|
-
line = event
|
223
|
+
return false if event.get("message").nil?
|
224
|
+
line = event.get("message")
|
225
225
|
version_metadata?(line) || fields_metadata?(line)
|
226
226
|
end
|
227
227
|
|
@@ -237,7 +237,7 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
|
|
237
237
|
|
238
238
|
private
|
239
239
|
def update_metadata(metadata, event)
|
240
|
-
line = event
|
240
|
+
line = event.get('message').strip
|
241
241
|
|
242
242
|
if version_metadata?(line)
|
243
243
|
metadata[:cloudfront_version] = line.split(/#Version: (.+)/).last
|
data/logstash-input-s3.gemspec
CHANGED
@@ -1,10 +1,10 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-input-s3'
|
4
|
-
s.version = '
|
4
|
+
s.version = '3.0.0'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Stream events from files from a S3 bucket."
|
7
|
-
s.description = "This gem is a
|
7
|
+
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
8
8
|
s.authors = ["Elastic"]
|
9
9
|
s.email = 'info@elastic.co'
|
10
10
|
s.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"
|
@@ -20,7 +20,7 @@ Gem::Specification.new do |s|
|
|
20
20
|
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
|
21
21
|
|
22
22
|
# Gem dependencies
|
23
|
-
s.add_runtime_dependency "logstash-core-plugin-api", "~>
|
23
|
+
s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
|
24
24
|
s.add_runtime_dependency 'logstash-mixin-aws'
|
25
25
|
s.add_runtime_dependency 'stud', '~> 0.0.18'
|
26
26
|
|
data/spec/inputs/s3_spec.rb
CHANGED
@@ -299,8 +299,8 @@ describe LogStash::Inputs::S3 do
|
|
299
299
|
events = fetch_events(config)
|
300
300
|
|
301
301
|
events.each do |event|
|
302
|
-
expect(event
|
303
|
-
expect(event
|
302
|
+
expect(event.get('cloudfront_fields')).to eq('date time x-edge-location c-ip x-event sc-bytes x-cf-status x-cf-client-id cs-uri-stem cs-uri-query c-referrer x-page-url c-user-agent x-sname x-sname-query x-file-ext x-sid')
|
303
|
+
expect(event.get('cloudfront_version')).to eq('1.0')
|
304
304
|
end
|
305
305
|
end
|
306
306
|
|
metadata
CHANGED
@@ -1,114 +1,116 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-s3
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 3.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-05-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
+
name: logstash-core-plugin-api
|
14
15
|
requirement: !ruby/object:Gem::Requirement
|
15
16
|
requirements:
|
16
17
|
- - "~>"
|
17
18
|
- !ruby/object:Gem::Version
|
18
|
-
version: '
|
19
|
-
name: logstash-core-plugin-api
|
20
|
-
prerelease: false
|
19
|
+
version: '2.0'
|
21
20
|
type: :runtime
|
21
|
+
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '
|
26
|
+
version: '2.0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
|
+
name: logstash-mixin-aws
|
28
29
|
requirement: !ruby/object:Gem::Requirement
|
29
30
|
requirements:
|
30
31
|
- - ">="
|
31
32
|
- !ruby/object:Gem::Version
|
32
33
|
version: '0'
|
33
|
-
name: logstash-mixin-aws
|
34
|
-
prerelease: false
|
35
34
|
type: :runtime
|
35
|
+
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '0'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
|
+
name: stud
|
42
43
|
requirement: !ruby/object:Gem::Requirement
|
43
44
|
requirements:
|
44
45
|
- - "~>"
|
45
46
|
- !ruby/object:Gem::Version
|
46
47
|
version: 0.0.18
|
47
|
-
name: stud
|
48
|
-
prerelease: false
|
49
48
|
type: :runtime
|
49
|
+
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: 0.0.18
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
|
+
name: logstash-devutils
|
56
57
|
requirement: !ruby/object:Gem::Requirement
|
57
58
|
requirements:
|
58
59
|
- - ">="
|
59
60
|
- !ruby/object:Gem::Version
|
60
61
|
version: '0'
|
61
|
-
name: logstash-devutils
|
62
|
-
prerelease: false
|
63
62
|
type: :development
|
63
|
+
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
66
|
- - ">="
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '0'
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
|
+
name: simplecov
|
70
71
|
requirement: !ruby/object:Gem::Requirement
|
71
72
|
requirements:
|
72
73
|
- - ">="
|
73
74
|
- !ruby/object:Gem::Version
|
74
75
|
version: '0'
|
75
|
-
name: simplecov
|
76
|
-
prerelease: false
|
77
76
|
type: :development
|
77
|
+
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
80
|
- - ">="
|
81
81
|
- !ruby/object:Gem::Version
|
82
82
|
version: '0'
|
83
83
|
- !ruby/object:Gem::Dependency
|
84
|
+
name: coveralls
|
84
85
|
requirement: !ruby/object:Gem::Requirement
|
85
86
|
requirements:
|
86
87
|
- - ">="
|
87
88
|
- !ruby/object:Gem::Version
|
88
89
|
version: '0'
|
89
|
-
name: coveralls
|
90
|
-
prerelease: false
|
91
90
|
type: :development
|
91
|
+
prerelease: false
|
92
92
|
version_requirements: !ruby/object:Gem::Requirement
|
93
93
|
requirements:
|
94
94
|
- - ">="
|
95
95
|
- !ruby/object:Gem::Version
|
96
96
|
version: '0'
|
97
97
|
- !ruby/object:Gem::Dependency
|
98
|
+
name: logstash-codec-json
|
98
99
|
requirement: !ruby/object:Gem::Requirement
|
99
100
|
requirements:
|
100
101
|
- - ">="
|
101
102
|
- !ruby/object:Gem::Version
|
102
103
|
version: '0'
|
103
|
-
name: logstash-codec-json
|
104
|
-
prerelease: false
|
105
104
|
type: :development
|
105
|
+
prerelease: false
|
106
106
|
version_requirements: !ruby/object:Gem::Requirement
|
107
107
|
requirements:
|
108
108
|
- - ">="
|
109
109
|
- !ruby/object:Gem::Version
|
110
110
|
version: '0'
|
111
|
-
description: This gem is a
|
111
|
+
description: This gem is a Logstash plugin required to be installed on top of the
|
112
|
+
Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
|
113
|
+
gem is not a stand-alone program
|
112
114
|
email: info@elastic.co
|
113
115
|
executables: []
|
114
116
|
extensions: []
|
@@ -137,7 +139,7 @@ licenses:
|
|
137
139
|
metadata:
|
138
140
|
logstash_plugin: 'true'
|
139
141
|
logstash_group: input
|
140
|
-
post_install_message:
|
142
|
+
post_install_message:
|
141
143
|
rdoc_options: []
|
142
144
|
require_paths:
|
143
145
|
- lib
|
@@ -152,9 +154,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
152
154
|
- !ruby/object:Gem::Version
|
153
155
|
version: '0'
|
154
156
|
requirements: []
|
155
|
-
rubyforge_project:
|
156
|
-
rubygems_version: 2.
|
157
|
-
signing_key:
|
157
|
+
rubyforge_project:
|
158
|
+
rubygems_version: 2.5.1
|
159
|
+
signing_key:
|
158
160
|
specification_version: 4
|
159
161
|
summary: Stream events from files from a S3 bucket.
|
160
162
|
test_files:
|