logstash-input-s3-test 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,17 @@
1
+ # encoding: utf-8
2
+ require "logstash/devutils/rspec/spec_helper"
3
+ require "logstash/inputs/s3"
4
+ require "stud/temporary"
5
+ require "fileutils"
6
+
7
+ describe LogStash::Inputs::S3::SinceDB::File do
8
+ let(:file) { Stud::Temporary.file.path }
9
+ subject { LogStash::Inputs::S3::SinceDB::File.new(file) }
10
+ before do
11
+ FileUtils.touch(file)
12
+ end
13
+
14
+ it "doesnt raise an exception if the file is empty" do
15
+ expect { subject.read }.not_to raise_error
16
+ end
17
+ end
@@ -0,0 +1,61 @@
1
+ require "logstash/devutils/rspec/spec_helper"
2
+ require "logstash/inputs/s3"
3
+ require "aws-sdk-s3"
4
+ require "fileutils"
5
+ require_relative "../support/helpers"
6
+
7
+ describe LogStash::Inputs::S3, :integration => true, :s3 => true do
8
+ before do
9
+ Thread.abort_on_exception = true
10
+
11
+ upload_file('../fixtures/uncompressed.log' , "#{prefix}uncompressed_1.log")
12
+ upload_file('../fixtures/compressed.log.gz', "#{prefix}compressed_1.log.gz")
13
+ end
14
+
15
+ after do
16
+ delete_remote_files(prefix)
17
+ FileUtils.rm_rf(temporary_directory)
18
+ delete_remote_files(backup_prefix)
19
+ end
20
+
21
+ let(:temporary_directory) { Stud::Temporary.directory }
22
+ let(:prefix) { 'logstash-s3-input-prefix/' }
23
+
24
+ let(:minimal_settings) { { "access_key_id" => ENV['AWS_ACCESS_KEY_ID'],
25
+ "secret_access_key" => ENV['AWS_SECRET_ACCESS_KEY'],
26
+ "bucket" => ENV['AWS_LOGSTASH_TEST_BUCKET'],
27
+ "region" => ENV["AWS_REGION"] || "us-east-1",
28
+ "prefix" => prefix,
29
+ "temporary_directory" => temporary_directory } }
30
+ let(:backup_prefix) { "backup/" }
31
+
32
+ it "support prefix to scope the remote files" do
33
+ events = fetch_events(minimal_settings)
34
+ expect(events.size).to eq(4)
35
+ end
36
+
37
+
38
+ it "add a prefix to the file" do
39
+ fetch_events(minimal_settings.merge({ "backup_to_bucket" => ENV["AWS_LOGSTASH_TEST_BUCKET"],
40
+ "backup_add_prefix" => backup_prefix }))
41
+ expect(list_remote_files(backup_prefix).size).to eq(2)
42
+ end
43
+
44
+ it "allow you to backup to a local directory" do
45
+ Stud::Temporary.directory do |backup_dir|
46
+ fetch_events(minimal_settings.merge({ "backup_to_dir" => backup_dir }))
47
+ expect(Dir.glob(File.join(backup_dir, "*")).size).to eq(2)
48
+ end
49
+ end
50
+
51
+ context "remote backup" do
52
+ it "another bucket" do
53
+ fetch_events(minimal_settings.merge({ "backup_to_bucket" => "logstash-s3-input-backup"}))
54
+ expect(list_remote_files("", "logstash-s3-input-backup").size).to eq(2)
55
+ end
56
+
57
+ after do
58
+ delete_bucket("logstash-s3-input-backup")
59
+ end
60
+ end
61
+ end
@@ -0,0 +1,45 @@
1
+ def fetch_events(settings)
2
+ queue = []
3
+ s3 = LogStash::Inputs::S3.new(settings)
4
+ s3.register
5
+ s3.process_files(queue)
6
+ queue
7
+ end
8
+
9
+ # delete_files(prefix)
10
+ def upload_file(local_file, remote_name)
11
+ bucket = s3object.bucket(ENV['AWS_LOGSTASH_TEST_BUCKET'])
12
+ file = File.expand_path(File.join(File.dirname(__FILE__), local_file))
13
+ bucket.object(remote_name).upload_file(file)
14
+ end
15
+
16
+ def delete_remote_files(prefix)
17
+ bucket = s3object.bucket(ENV['AWS_LOGSTASH_TEST_BUCKET'])
18
+ bucket.objects(:prefix => prefix).each { |object| object.delete }
19
+ end
20
+
21
+ def list_remote_files(prefix, target_bucket = ENV['AWS_LOGSTASH_TEST_BUCKET'])
22
+ bucket = s3object.bucket(target_bucket)
23
+ bucket.objects(:prefix => prefix).collect(&:key)
24
+ end
25
+
26
+ def delete_bucket(name)
27
+ s3object.bucket(name).objects.map(&:delete)
28
+ s3object.bucket(name).delete
29
+ end
30
+
31
+ def s3object
32
+ Aws::S3::Resource.new
33
+ end
34
+
35
+ class TestInfiniteS3Object
36
+ def each
37
+ counter = 1
38
+
39
+ loop do
40
+ yield "awesome-#{counter}"
41
+ counter +=1
42
+ end
43
+ end
44
+ end
45
+
metadata ADDED
@@ -0,0 +1,189 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: logstash-input-s3-test
3
+ version: !ruby/object:Gem::Version
4
+ version: 4.0.0
5
+ platform: ruby
6
+ authors:
7
+ - Sukhbir
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2020-12-03 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: logstash-core-plugin-api
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: 2.1.12
20
+ - - "<="
21
+ - !ruby/object:Gem::Version
22
+ version: '2.99'
23
+ type: :runtime
24
+ prerelease: false
25
+ version_requirements: !ruby/object:Gem::Requirement
26
+ requirements:
27
+ - - ">="
28
+ - !ruby/object:Gem::Version
29
+ version: 2.1.12
30
+ - - "<="
31
+ - !ruby/object:Gem::Version
32
+ version: '2.99'
33
+ - !ruby/object:Gem::Dependency
34
+ name: stud
35
+ requirement: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - "~>"
38
+ - !ruby/object:Gem::Version
39
+ version: 0.0.18
40
+ type: :runtime
41
+ prerelease: false
42
+ version_requirements: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - "~>"
45
+ - !ruby/object:Gem::Version
46
+ version: 0.0.18
47
+ - !ruby/object:Gem::Dependency
48
+ name: aws-sdk-s3
49
+ requirement: !ruby/object:Gem::Requirement
50
+ requirements:
51
+ - - "~>"
52
+ - !ruby/object:Gem::Version
53
+ version: '1'
54
+ type: :runtime
55
+ prerelease: false
56
+ version_requirements: !ruby/object:Gem::Requirement
57
+ requirements:
58
+ - - "~>"
59
+ - !ruby/object:Gem::Version
60
+ version: '1'
61
+ - !ruby/object:Gem::Dependency
62
+ name: logstash-devutils
63
+ requirement: !ruby/object:Gem::Requirement
64
+ requirements:
65
+ - - ">="
66
+ - !ruby/object:Gem::Version
67
+ version: '0'
68
+ type: :development
69
+ prerelease: false
70
+ version_requirements: !ruby/object:Gem::Requirement
71
+ requirements:
72
+ - - ">="
73
+ - !ruby/object:Gem::Version
74
+ version: '0'
75
+ - !ruby/object:Gem::Dependency
76
+ name: logstash-codec-json
77
+ requirement: !ruby/object:Gem::Requirement
78
+ requirements:
79
+ - - ">="
80
+ - !ruby/object:Gem::Version
81
+ version: '0'
82
+ type: :development
83
+ prerelease: false
84
+ version_requirements: !ruby/object:Gem::Requirement
85
+ requirements:
86
+ - - ">="
87
+ - !ruby/object:Gem::Version
88
+ version: '0'
89
+ - !ruby/object:Gem::Dependency
90
+ name: logstash-codec-plain
91
+ requirement: !ruby/object:Gem::Requirement
92
+ requirements:
93
+ - - ">="
94
+ - !ruby/object:Gem::Version
95
+ version: '0'
96
+ type: :development
97
+ prerelease: false
98
+ version_requirements: !ruby/object:Gem::Requirement
99
+ requirements:
100
+ - - ">="
101
+ - !ruby/object:Gem::Version
102
+ version: '0'
103
+ - !ruby/object:Gem::Dependency
104
+ name: logstash-codec-multiline
105
+ requirement: !ruby/object:Gem::Requirement
106
+ requirements:
107
+ - - ">="
108
+ - !ruby/object:Gem::Version
109
+ version: '0'
110
+ type: :development
111
+ prerelease: false
112
+ version_requirements: !ruby/object:Gem::Requirement
113
+ requirements:
114
+ - - ">="
115
+ - !ruby/object:Gem::Version
116
+ version: '0'
117
+ description: This gem is a Logstash plugin required to be installed on top of the
118
+ Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
119
+ gem is not a stand-alone program
120
+ email: sukhbir947@gmail.com
121
+ executables: []
122
+ extensions: []
123
+ extra_rdoc_files: []
124
+ files:
125
+ - CHANGELOG.md
126
+ - CONTRIBUTORS
127
+ - Gemfile
128
+ - LICENSE
129
+ - NOTICE.TXT
130
+ - README.md
131
+ - docs/index.asciidoc
132
+ - lib/logstash/inputs/s3.rb
133
+ - lib/logstash/inputs/s3/patch.rb
134
+ - logstash-input-s3.gemspec
135
+ - spec/fixtures/cloudfront.log
136
+ - spec/fixtures/compressed.log.gee.zip
137
+ - spec/fixtures/compressed.log.gz
138
+ - spec/fixtures/compressed.log.gzip
139
+ - spec/fixtures/invalid_utf8.gbk.log
140
+ - spec/fixtures/json.log
141
+ - spec/fixtures/json_with_message.log
142
+ - spec/fixtures/multiline.log
143
+ - spec/fixtures/multiple_compressed_streams.gz
144
+ - spec/fixtures/uncompressed.log
145
+ - spec/inputs/s3_spec.rb
146
+ - spec/inputs/sincedb_spec.rb
147
+ - spec/integration/s3_spec.rb
148
+ - spec/support/helpers.rb
149
+ homepage: http://www.elastic.co/guide/en/logstash/current/index.html
150
+ licenses:
151
+ - Apache-2.0
152
+ metadata:
153
+ logstash_plugin: 'true'
154
+ logstash_group: input
155
+ post_install_message:
156
+ rdoc_options: []
157
+ require_paths:
158
+ - lib
159
+ required_ruby_version: !ruby/object:Gem::Requirement
160
+ requirements:
161
+ - - ">="
162
+ - !ruby/object:Gem::Version
163
+ version: '0'
164
+ required_rubygems_version: !ruby/object:Gem::Requirement
165
+ requirements:
166
+ - - ">="
167
+ - !ruby/object:Gem::Version
168
+ version: '0'
169
+ requirements: []
170
+ rubyforge_project:
171
+ rubygems_version: 2.6.14
172
+ signing_key:
173
+ specification_version: 4
174
+ summary: Streams events from files in a S3 bucket
175
+ test_files:
176
+ - spec/fixtures/cloudfront.log
177
+ - spec/fixtures/compressed.log.gee.zip
178
+ - spec/fixtures/compressed.log.gz
179
+ - spec/fixtures/compressed.log.gzip
180
+ - spec/fixtures/invalid_utf8.gbk.log
181
+ - spec/fixtures/json.log
182
+ - spec/fixtures/json_with_message.log
183
+ - spec/fixtures/multiline.log
184
+ - spec/fixtures/multiple_compressed_streams.gz
185
+ - spec/fixtures/uncompressed.log
186
+ - spec/inputs/s3_spec.rb
187
+ - spec/inputs/sincedb_spec.rb
188
+ - spec/integration/s3_spec.rb
189
+ - spec/support/helpers.rb