logstash-input-s3 3.0.1 → 3.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +18 -6
- data/lib/logstash/inputs/s3.rb +24 -81
- data/logstash-input-s3.gemspec +3 -2
- data/spec/inputs/s3_spec.rb +37 -59
- data/spec/support/helpers.rb +9 -9
- metadata +13 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b975c1700f9a6aab421e175f94d1bce38b9d9e01
|
4
|
+
data.tar.gz: 1ebe414e3ba68ae1d4e9d3477d9ec9663f13dc10
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5477fd22a31ab077044f93c089a9b466ac692b9c3cfdf6156fcfc7b70135d7881b34ea314c54aa7fc1566b46e045f1bc0f714acd554e6971318db4a2e468fda1
|
7
|
+
data.tar.gz: 44adae47471d9ad5a495b63bb155110cb86df5d0167fc94562e5b6e35c8f20d069727c166ac8bee438e4f22573410d25e247e010ff9398cff8554c3c15ec52cc
|
data/CHANGELOG.md
CHANGED
@@ -1,15 +1,27 @@
|
|
1
|
+
## 3.1.1
|
2
|
+
- Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
|
3
|
+
|
4
|
+
## 3.1.0
|
5
|
+
- breaking,config: Remove deprecated config `credentials` and `region_endpoint`. Please use AWS mixin.
|
6
|
+
|
1
7
|
## 3.0.1
|
2
|
-
|
8
|
+
- Republish all the gems under jruby.
|
9
|
+
|
3
10
|
## 3.0.0
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
11
|
+
- Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
|
12
|
+
|
13
|
+
## 2.0.6
|
14
|
+
- Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
|
15
|
+
|
16
|
+
## 2.0.5
|
17
|
+
- New dependency requirements for logstash-core for the 5.0 release
|
18
|
+
|
9
19
|
## 2.0.4
|
10
20
|
- Fix for Error: No Such Key problem when deleting
|
21
|
+
|
11
22
|
## 2.0.3
|
12
23
|
- Do not raise an exception if the sincedb file is empty, instead return the current time #66
|
24
|
+
|
13
25
|
## 2.0.0
|
14
26
|
- Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
|
15
27
|
instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
|
data/lib/logstash/inputs/s3.rb
CHANGED
@@ -12,27 +12,15 @@ require "stud/temporary"
|
|
12
12
|
# Each line from each file generates an event.
|
13
13
|
# Files ending in `.gz` are handled as gzip'ed files.
|
14
14
|
class LogStash::Inputs::S3 < LogStash::Inputs::Base
|
15
|
-
include LogStash::PluginMixins::AwsConfig
|
15
|
+
include LogStash::PluginMixins::AwsConfig::V2
|
16
16
|
|
17
17
|
config_name "s3"
|
18
18
|
|
19
19
|
default :codec, "plain"
|
20
20
|
|
21
|
-
# DEPRECATED: The credentials of the AWS account used to access the bucket.
|
22
|
-
# Credentials can be specified:
|
23
|
-
# - As an ["id","secret"] array
|
24
|
-
# - As a path to a file containing AWS_ACCESS_KEY_ID=... and AWS_SECRET_ACCESS_KEY=...
|
25
|
-
# - In the environment, if not set (using variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY)
|
26
|
-
config :credentials, :validate => :array, :default => [], :deprecated => "This only exists to be backwards compatible. This plugin now uses the AwsConfig from PluginMixins"
|
27
|
-
|
28
21
|
# The name of the S3 bucket.
|
29
22
|
config :bucket, :validate => :string, :required => true
|
30
23
|
|
31
|
-
# The AWS region for your bucket.
|
32
|
-
config :region_endpoint, :validate => ["us-east-1", "us-west-1", "us-west-2",
|
33
|
-
"eu-west-1", "ap-southeast-1", "ap-southeast-2",
|
34
|
-
"ap-northeast-1", "sa-east-1", "us-gov-west-1"], :deprecated => "This only exists to be backwards compatible. This plugin now uses the AwsConfig from PluginMixins"
|
35
|
-
|
36
24
|
# If specified, the prefix of filenames in the bucket must match (not a regexp)
|
37
25
|
config :prefix, :validate => :string, :default => nil
|
38
26
|
|
@@ -71,20 +59,20 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
|
|
71
59
|
def register
|
72
60
|
require "fileutils"
|
73
61
|
require "digest/md5"
|
74
|
-
require "aws-sdk"
|
75
|
-
|
76
|
-
@region = get_region
|
62
|
+
require "aws-sdk-resources"
|
77
63
|
|
78
64
|
@logger.info("Registering s3 input", :bucket => @bucket, :region => @region)
|
79
65
|
|
80
66
|
s3 = get_s3object
|
81
67
|
|
82
|
-
@s3bucket = s3.
|
68
|
+
@s3bucket = s3.bucket(@bucket)
|
83
69
|
|
84
70
|
unless @backup_to_bucket.nil?
|
85
|
-
@backup_bucket = s3.
|
86
|
-
|
87
|
-
s3.
|
71
|
+
@backup_bucket = s3.bucket(@backup_to_bucket)
|
72
|
+
begin
|
73
|
+
s3.client.head_bucket({ :bucket => @backup_to_bucket})
|
74
|
+
rescue Aws::S3::Errors::NoSuchBucket
|
75
|
+
s3.create_bucket({ :bucket => @backup_to_bucket})
|
88
76
|
end
|
89
77
|
end
|
90
78
|
|
@@ -93,7 +81,7 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
|
|
93
81
|
end
|
94
82
|
|
95
83
|
FileUtils.mkdir_p(@temporary_directory) unless Dir.exist?(@temporary_directory)
|
96
|
-
end
|
84
|
+
end
|
97
85
|
|
98
86
|
public
|
99
87
|
def run(queue)
|
@@ -107,13 +95,14 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
|
|
107
95
|
def list_new_files
|
108
96
|
objects = {}
|
109
97
|
|
110
|
-
@s3bucket.objects
|
98
|
+
@s3bucket.objects(:prefix => @prefix).each do |log|
|
111
99
|
@logger.debug("S3 input: Found key", :key => log.key)
|
112
100
|
|
113
101
|
unless ignore_filename?(log.key)
|
114
102
|
if sincedb.newer?(log.last_modified)
|
115
103
|
objects[log.key] = log.last_modified
|
116
104
|
@logger.debug("S3 input: Adding to objects[]", :key => log.key)
|
105
|
+
@logger.debug("objects[] length is: ", :length => objects.length)
|
117
106
|
end
|
118
107
|
end
|
119
108
|
end
|
@@ -121,13 +110,12 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
|
|
121
110
|
end # def fetch_new_files
|
122
111
|
|
123
112
|
public
|
124
|
-
def backup_to_bucket(object
|
113
|
+
def backup_to_bucket(object)
|
125
114
|
unless @backup_to_bucket.nil?
|
126
|
-
backup_key = "#{@backup_add_prefix}#{key}"
|
115
|
+
backup_key = "#{@backup_add_prefix}#{object.key}"
|
116
|
+
@backup_bucket.object(backup_key).copy_from(:copy_source => "#{object.bucket_name}/#{object.key}")
|
127
117
|
if @delete
|
128
|
-
object.
|
129
|
-
else
|
130
|
-
object.copy_to(backup_key, :bucket => @backup_bucket)
|
118
|
+
object.delete()
|
131
119
|
end
|
132
120
|
end
|
133
121
|
end
|
@@ -155,22 +143,12 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
|
|
155
143
|
|
156
144
|
public
|
157
145
|
def stop
|
158
|
-
# @current_thread is initialized in the `#run` method,
|
159
|
-
# this variable is needed because the `#stop` is a called in another thread
|
146
|
+
# @current_thread is initialized in the `#run` method,
|
147
|
+
# this variable is needed because the `#stop` is a called in another thread
|
160
148
|
# than the `#run` method and requiring us to call stop! with a explicit thread.
|
161
149
|
Stud.stop!(@current_thread)
|
162
150
|
end
|
163
151
|
|
164
|
-
public
|
165
|
-
def aws_service_endpoint(region)
|
166
|
-
region_to_use = get_region
|
167
|
-
|
168
|
-
return {
|
169
|
-
:s3_endpoint => region_to_use == 'us-east-1' ?
|
170
|
-
's3.amazonaws.com' : "s3-#{region_to_use}.amazonaws.com"
|
171
|
-
}
|
172
|
-
end
|
173
|
-
|
174
152
|
private
|
175
153
|
|
176
154
|
# Read the content of the local file
|
@@ -180,7 +158,6 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
|
|
180
158
|
# @return [Boolean] True if the file was completely read, false otherwise.
|
181
159
|
def process_local_log(queue, filename)
|
182
160
|
@logger.debug('Processing file', :filename => filename)
|
183
|
-
|
184
161
|
metadata = {}
|
185
162
|
# Currently codecs operates on bytes instead of stream.
|
186
163
|
# So all IO stuff: decompression, reading need to be done in the actual
|
@@ -301,6 +278,8 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
|
|
301
278
|
def ignore_filename?(filename)
|
302
279
|
if @prefix == filename
|
303
280
|
return true
|
281
|
+
elsif filename.end_with?("/")
|
282
|
+
return true
|
304
283
|
elsif (@backup_add_prefix && @backup_to_bucket == @bucket && filename =~ /^#{backup_add_prefix}/)
|
305
284
|
return true
|
306
285
|
elsif @exclude_pattern.nil?
|
@@ -314,14 +293,13 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
|
|
314
293
|
|
315
294
|
private
|
316
295
|
def process_log(queue, key)
|
317
|
-
object = @s3bucket.
|
296
|
+
object = @s3bucket.object(key)
|
318
297
|
|
319
298
|
filename = File.join(temporary_directory, File.basename(key))
|
320
|
-
|
321
299
|
if download_remote_file(object, filename)
|
322
300
|
if process_local_log(queue, filename)
|
323
301
|
lastmod = object.last_modified
|
324
|
-
backup_to_bucket(object
|
302
|
+
backup_to_bucket(object)
|
325
303
|
backup_to_dir(filename)
|
326
304
|
delete_file_from_bucket(object)
|
327
305
|
FileUtils.remove_entry_secure(filename, true)
|
@@ -340,13 +318,10 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
|
|
340
318
|
# @return [Boolean] True if the file was completely downloaded
|
341
319
|
def download_remote_file(remote_object, local_filename)
|
342
320
|
completed = false
|
343
|
-
|
344
321
|
@logger.debug("S3 input: Download remote file", :remote_key => remote_object.key, :local_filename => local_filename)
|
345
322
|
File.open(local_filename, 'wb') do |s3file|
|
346
|
-
|
347
|
-
|
348
|
-
s3file.write(chunk)
|
349
|
-
end
|
323
|
+
return completed if stop?
|
324
|
+
remote_object.get(:response_target => s3file)
|
350
325
|
end
|
351
326
|
completed = true
|
352
327
|
|
@@ -360,41 +335,9 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
|
|
360
335
|
end
|
361
336
|
end
|
362
337
|
|
363
|
-
private
|
364
|
-
def get_region
|
365
|
-
# TODO: (ph) Deprecated, it will be removed
|
366
|
-
if @region_endpoint
|
367
|
-
@region_endpoint
|
368
|
-
else
|
369
|
-
@region
|
370
|
-
end
|
371
|
-
end
|
372
|
-
|
373
338
|
private
|
374
339
|
def get_s3object
|
375
|
-
|
376
|
-
if @credentials.length == 1
|
377
|
-
File.open(@credentials[0]) { |f| f.each do |line|
|
378
|
-
unless (/^\#/.match(line))
|
379
|
-
if(/\s*=\s*/.match(line))
|
380
|
-
param, value = line.split('=', 2)
|
381
|
-
param = param.chomp().strip()
|
382
|
-
value = value.chomp().strip()
|
383
|
-
if param.eql?('AWS_ACCESS_KEY_ID')
|
384
|
-
@access_key_id = value
|
385
|
-
elsif param.eql?('AWS_SECRET_ACCESS_KEY')
|
386
|
-
@secret_access_key = value
|
387
|
-
end
|
388
|
-
end
|
389
|
-
end
|
390
|
-
end
|
391
|
-
}
|
392
|
-
elsif @credentials.length == 2
|
393
|
-
@access_key_id = @credentials[0]
|
394
|
-
@secret_access_key = @credentials[1]
|
395
|
-
end
|
396
|
-
|
397
|
-
s3 = AWS::S3.new(aws_options_hash)
|
340
|
+
s3 = Aws::S3::Resource.new(aws_options_hash)
|
398
341
|
end
|
399
342
|
|
400
343
|
private
|
data/logstash-input-s3.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-input-s3'
|
4
|
-
s.version = '3.
|
4
|
+
s.version = '3.1.1'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Stream events from files from a S3 bucket."
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
@@ -20,9 +20,10 @@ Gem::Specification.new do |s|
|
|
20
20
|
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
|
21
21
|
|
22
22
|
# Gem dependencies
|
23
|
-
s.add_runtime_dependency "logstash-core-plugin-api", "
|
23
|
+
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
|
24
24
|
s.add_runtime_dependency 'logstash-mixin-aws'
|
25
25
|
s.add_runtime_dependency 'stud', '~> 0.0.18'
|
26
|
+
# s.add_runtime_dependency 'aws-sdk-resources', '>= 2.0.33'
|
26
27
|
|
27
28
|
s.add_development_dependency 'logstash-devutils'
|
28
29
|
s.add_development_dependency 'simplecov'
|
data/spec/inputs/s3_spec.rb
CHANGED
@@ -2,6 +2,7 @@
|
|
2
2
|
require "logstash/devutils/rspec/spec_helper"
|
3
3
|
require "logstash/inputs/s3"
|
4
4
|
require "logstash/errors"
|
5
|
+
require "aws-sdk-resources"
|
5
6
|
require_relative "../support/helpers"
|
6
7
|
require "stud/temporary"
|
7
8
|
require "aws-sdk"
|
@@ -11,6 +12,7 @@ describe LogStash::Inputs::S3 do
|
|
11
12
|
let(:temporary_directory) { Stud::Temporary.pathname }
|
12
13
|
let(:sincedb_path) { Stud::Temporary.pathname }
|
13
14
|
let(:day) { 3600 * 24 }
|
15
|
+
let(:creds) { Aws::Credentials.new('1234', 'secret') }
|
14
16
|
let(:config) {
|
15
17
|
{
|
16
18
|
"access_key_id" => "1234",
|
@@ -21,9 +23,10 @@ describe LogStash::Inputs::S3 do
|
|
21
23
|
}
|
22
24
|
}
|
23
25
|
|
26
|
+
|
24
27
|
before do
|
25
28
|
FileUtils.mkdir_p(sincedb_path)
|
26
|
-
|
29
|
+
Aws.config[:stub_responses] = true
|
27
30
|
Thread.abort_on_exception = true
|
28
31
|
end
|
29
32
|
|
@@ -52,27 +55,6 @@ describe LogStash::Inputs::S3 do
|
|
52
55
|
describe '#get_s3object' do
|
53
56
|
subject { LogStash::Inputs::S3.new(settings) }
|
54
57
|
|
55
|
-
context 'with deprecated credentials option' do
|
56
|
-
let(:settings) {
|
57
|
-
{
|
58
|
-
"credentials" => ["1234", "secret"],
|
59
|
-
"proxy_uri" => "http://example.com",
|
60
|
-
"bucket" => "logstash-test",
|
61
|
-
}
|
62
|
-
}
|
63
|
-
|
64
|
-
it 'should instantiate AWS::S3 clients with a proxy set' do
|
65
|
-
expect(AWS::S3).to receive(:new).with({
|
66
|
-
:access_key_id => "1234",
|
67
|
-
:secret_access_key => "secret",
|
68
|
-
:proxy_uri => 'http://example.com',
|
69
|
-
:use_ssl => subject.use_ssl,
|
70
|
-
}.merge(subject.aws_service_endpoint(subject.region)))
|
71
|
-
|
72
|
-
subject.send(:get_s3object)
|
73
|
-
end
|
74
|
-
end
|
75
|
-
|
76
58
|
context 'with modern access key options' do
|
77
59
|
let(:settings) {
|
78
60
|
{
|
@@ -84,13 +66,11 @@ describe LogStash::Inputs::S3 do
|
|
84
66
|
}
|
85
67
|
|
86
68
|
it 'should instantiate AWS::S3 clients with a proxy set' do
|
87
|
-
expect(
|
88
|
-
:
|
89
|
-
:
|
90
|
-
:
|
91
|
-
|
92
|
-
}.merge(subject.aws_service_endpoint(subject.region)))
|
93
|
-
|
69
|
+
expect(Aws::S3::Resource).to receive(:new).with({
|
70
|
+
:credentials => kind_of(Aws::Credentials),
|
71
|
+
:http_proxy => 'http://example.com',
|
72
|
+
:region => subject.region
|
73
|
+
})
|
94
74
|
|
95
75
|
subject.send(:get_s3object)
|
96
76
|
end
|
@@ -98,7 +78,7 @@ describe LogStash::Inputs::S3 do
|
|
98
78
|
end
|
99
79
|
|
100
80
|
describe "#list_new_files" do
|
101
|
-
before { allow_any_instance_of(
|
81
|
+
before { allow_any_instance_of(Aws::S3::Bucket).to receive(:objects) { objects_list } }
|
102
82
|
|
103
83
|
let!(:present_object) { double(:key => 'this-should-be-present', :last_modified => Time.now) }
|
104
84
|
let(:objects_list) {
|
@@ -128,7 +108,7 @@ describe LogStash::Inputs::S3 do
|
|
128
108
|
present_object
|
129
109
|
]
|
130
110
|
|
131
|
-
allow_any_instance_of(
|
111
|
+
allow_any_instance_of(Aws::S3::Bucket).to receive(:objects) { objects_list }
|
132
112
|
|
133
113
|
plugin = LogStash::Inputs::S3.new(config.merge({ 'backup_add_prefix' => 'mybackup',
|
134
114
|
'backup_to_bucket' => config['bucket']}))
|
@@ -154,7 +134,7 @@ describe LogStash::Inputs::S3 do
|
|
154
134
|
present_object
|
155
135
|
]
|
156
136
|
|
157
|
-
allow_any_instance_of(
|
137
|
+
allow_any_instance_of(Aws::S3::Bucket).to receive(:objects).with(:prefix => prefix) { objects_list }
|
158
138
|
|
159
139
|
plugin = LogStash::Inputs::S3.new(config.merge({ 'prefix' => prefix }))
|
160
140
|
plugin.register
|
@@ -168,7 +148,7 @@ describe LogStash::Inputs::S3 do
|
|
168
148
|
double(:key => 'TWO_DAYS_AGO', :last_modified => Time.now - 2 * day)
|
169
149
|
]
|
170
150
|
|
171
|
-
allow_any_instance_of(
|
151
|
+
allow_any_instance_of(Aws::S3::Bucket).to receive(:objects) { objects }
|
172
152
|
|
173
153
|
|
174
154
|
plugin = LogStash::Inputs::S3.new(config)
|
@@ -181,20 +161,22 @@ describe LogStash::Inputs::S3 do
|
|
181
161
|
plugin = LogStash::Inputs::S3.new(config.merge({ "backup_to_bucket" => "mybackup"}))
|
182
162
|
plugin.register
|
183
163
|
|
184
|
-
s3object =
|
185
|
-
|
164
|
+
s3object = Aws::S3::Object.new('mybucket', 'testkey')
|
165
|
+
expect_any_instance_of(Aws::S3::Object).to receive(:copy_from).with(:copy_source => "mybucket/testkey")
|
166
|
+
expect(s3object).to_not receive(:delete)
|
186
167
|
|
187
|
-
plugin.backup_to_bucket(s3object
|
168
|
+
plugin.backup_to_bucket(s3object)
|
188
169
|
end
|
189
170
|
|
190
|
-
it 'should
|
171
|
+
it 'should copy to another s3 bucket when deleting the original file' do
|
191
172
|
plugin = LogStash::Inputs::S3.new(config.merge({ "backup_to_bucket" => "mybackup", "delete" => true }))
|
192
173
|
plugin.register
|
193
174
|
|
194
|
-
s3object =
|
195
|
-
|
175
|
+
s3object = Aws::S3::Object.new('mybucket', 'testkey')
|
176
|
+
expect_any_instance_of(Aws::S3::Object).to receive(:copy_from).with(:copy_source => "mybucket/testkey")
|
177
|
+
expect(s3object).to receive(:delete)
|
196
178
|
|
197
|
-
plugin.backup_to_bucket(s3object
|
179
|
+
plugin.backup_to_bucket(s3object)
|
198
180
|
end
|
199
181
|
|
200
182
|
it 'should add the specified prefix to the backup file' do
|
@@ -202,10 +184,11 @@ describe LogStash::Inputs::S3 do
|
|
202
184
|
"backup_add_prefix" => 'backup-' }))
|
203
185
|
plugin.register
|
204
186
|
|
205
|
-
s3object =
|
206
|
-
|
187
|
+
s3object = Aws::S3::Object.new('mybucket', 'testkey')
|
188
|
+
expect_any_instance_of(Aws::S3::Object).to receive(:copy_from).with(:copy_source => "mybucket/testkey")
|
189
|
+
expect(s3object).to_not receive(:delete)
|
207
190
|
|
208
|
-
plugin.backup_to_bucket(s3object
|
191
|
+
plugin.backup_to_bucket(s3object)
|
209
192
|
end
|
210
193
|
end
|
211
194
|
|
@@ -222,19 +205,6 @@ describe LogStash::Inputs::S3 do
|
|
222
205
|
end
|
223
206
|
end
|
224
207
|
end
|
225
|
-
|
226
|
-
it 'should accepts a list of credentials for the aws-sdk, this is deprecated' do
|
227
|
-
Stud::Temporary.directory do |tmp_directory|
|
228
|
-
old_credentials_config = {
|
229
|
-
"credentials" => ['1234', 'secret'],
|
230
|
-
"backup_to_dir" => tmp_directory,
|
231
|
-
"bucket" => "logstash-test"
|
232
|
-
}
|
233
|
-
|
234
|
-
plugin = LogStash::Inputs::S3.new(old_credentials_config)
|
235
|
-
expect{ plugin.register }.not_to raise_error
|
236
|
-
end
|
237
|
-
end
|
238
208
|
end
|
239
209
|
|
240
210
|
shared_examples "generated events" do
|
@@ -252,11 +222,19 @@ describe LogStash::Inputs::S3 do
|
|
252
222
|
context 'when working with logs' do
|
253
223
|
let(:objects) { [log] }
|
254
224
|
let(:log) { double(:key => 'uncompressed.log', :last_modified => Time.now - 2 * day) }
|
225
|
+
let(:data) { File.read(log_file) }
|
255
226
|
|
256
227
|
before do
|
257
|
-
|
258
|
-
|
259
|
-
|
228
|
+
Aws.config[:s3] = {
|
229
|
+
stub_responses: {
|
230
|
+
get_object: { body: data }
|
231
|
+
}
|
232
|
+
}
|
233
|
+
allow_any_instance_of(Aws::S3::Bucket).to receive(:objects) { objects }
|
234
|
+
allow_any_instance_of(Aws::S3::Bucket).to receive(:object).with(log.key) { log }
|
235
|
+
expect(log).to receive(:get).with(instance_of(Hash)) do |arg|
|
236
|
+
File.open(arg[:response_target], 'wb') { |s3file| s3file.write(data) }
|
237
|
+
end
|
260
238
|
end
|
261
239
|
|
262
240
|
context "when event doesn't have a `message` field" do
|
data/spec/support/helpers.rb
CHANGED
@@ -8,28 +8,28 @@ end
|
|
8
8
|
|
9
9
|
# delete_files(prefix)
|
10
10
|
def upload_file(local_file, remote_name)
|
11
|
-
bucket = s3object.
|
11
|
+
bucket = s3object.bucket(ENV['AWS_LOGSTASH_TEST_BUCKET'])
|
12
12
|
file = File.expand_path(File.join(File.dirname(__FILE__), local_file))
|
13
|
-
bucket.
|
13
|
+
bucket.object(remote_name).upload_file(file)
|
14
14
|
end
|
15
15
|
|
16
16
|
def delete_remote_files(prefix)
|
17
|
-
bucket = s3object.
|
18
|
-
bucket.objects
|
17
|
+
bucket = s3object.bucket(ENV['AWS_LOGSTASH_TEST_BUCKET'])
|
18
|
+
bucket.objects(:prefix => prefix).each { |object| object.delete }
|
19
19
|
end
|
20
20
|
|
21
21
|
def list_remote_files(prefix, target_bucket = ENV['AWS_LOGSTASH_TEST_BUCKET'])
|
22
|
-
bucket = s3object.
|
23
|
-
bucket.objects
|
22
|
+
bucket = s3object.bucket(target_bucket)
|
23
|
+
bucket.objects(:prefix => prefix).collect(&:key)
|
24
24
|
end
|
25
25
|
|
26
26
|
def delete_bucket(name)
|
27
|
-
s3object.
|
28
|
-
s3object.
|
27
|
+
s3object.bucket(name).objects.map(&:delete)
|
28
|
+
s3object.bucket(name).delete
|
29
29
|
end
|
30
30
|
|
31
31
|
def s3object
|
32
|
-
|
32
|
+
Aws::S3::Resource.new
|
33
33
|
end
|
34
34
|
|
35
35
|
class TestInfiniteS3Object
|
metadata
CHANGED
@@ -1,29 +1,35 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-s3
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-07-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
15
15
|
requirements:
|
16
|
-
- - "
|
16
|
+
- - ">="
|
17
|
+
- !ruby/object:Gem::Version
|
18
|
+
version: '1.60'
|
19
|
+
- - "<="
|
17
20
|
- !ruby/object:Gem::Version
|
18
|
-
version: '2.
|
21
|
+
version: '2.99'
|
19
22
|
name: logstash-core-plugin-api
|
20
23
|
prerelease: false
|
21
24
|
type: :runtime
|
22
25
|
version_requirements: !ruby/object:Gem::Requirement
|
23
26
|
requirements:
|
24
|
-
- - "
|
27
|
+
- - ">="
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '1.60'
|
30
|
+
- - "<="
|
25
31
|
- !ruby/object:Gem::Version
|
26
|
-
version: '2.
|
32
|
+
version: '2.99'
|
27
33
|
- !ruby/object:Gem::Dependency
|
28
34
|
requirement: !ruby/object:Gem::Requirement
|
29
35
|
requirements:
|
@@ -153,7 +159,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
153
159
|
version: '0'
|
154
160
|
requirements: []
|
155
161
|
rubyforge_project:
|
156
|
-
rubygems_version: 2.
|
162
|
+
rubygems_version: 2.6.3
|
157
163
|
signing_key:
|
158
164
|
specification_version: 4
|
159
165
|
summary: Stream events from files from a S3 bucket.
|