logstash-input-s3-cos 3.8.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,67 @@
1
+ require "logstash/devutils/rspec/spec_helper"
2
+ require "logstash/inputs/s3"
3
+ require "aws-sdk"
4
+ require "fileutils"
5
+ require_relative "../support/helpers"
6
+
7
+ describe LogStash::Inputs::S3, :integration => true, :s3 => true do
8
+ before do
9
+ Thread.abort_on_exception = true
10
+
11
+ upload_file('../fixtures/uncompressed.log' , "#{prefix}uncompressed_1.log")
12
+ upload_file('../fixtures/compressed.log.gz', "#{prefix}compressed_1.log.gz")
13
+ sleep(LogStash::Inputs::S3::CUTOFF_SECOND + 1)
14
+ end
15
+
16
+ after do
17
+ delete_remote_files(prefix)
18
+ FileUtils.rm_rf(temporary_directory)
19
+ delete_remote_files(backup_prefix)
20
+ end
21
+
22
+ let(:temporary_directory) { Stud::Temporary.directory }
23
+ let(:prefix) { 'logstash-s3-input-prefix/' }
24
+
25
+ let(:minimal_settings) { { "access_key_id" => ENV['AWS_ACCESS_KEY_ID'],
26
+ "secret_access_key" => ENV['AWS_SECRET_ACCESS_KEY'],
27
+ "bucket" => ENV['AWS_LOGSTASH_TEST_BUCKET'],
28
+ "region" => ENV["AWS_REGION"] || "us-east-1",
29
+ "prefix" => prefix,
30
+ "temporary_directory" => temporary_directory } }
31
+ let(:backup_prefix) { "backup/" }
32
+ let(:backup_bucket) { "logstash-s3-input-backup" }
33
+
34
+ it "support prefix to scope the remote files" do
35
+ events = fetch_events(minimal_settings)
36
+ expect(events.size).to eq(4)
37
+ end
38
+
39
+
40
+ it "add a prefix to the file" do
41
+ fetch_events(minimal_settings.merge({ "backup_to_bucket" => ENV["AWS_LOGSTASH_TEST_BUCKET"],
42
+ "backup_add_prefix" => backup_prefix }))
43
+ expect(list_remote_files(backup_prefix).size).to eq(2)
44
+ end
45
+
46
+ it "allow you to backup to a local directory" do
47
+ Stud::Temporary.directory do |backup_dir|
48
+ fetch_events(minimal_settings.merge({ "backup_to_dir" => backup_dir }))
49
+ expect(Dir.glob(File.join(backup_dir, "*")).size).to eq(2)
50
+ end
51
+ end
52
+
53
+ context "remote backup" do
54
+ before do
55
+ create_bucket(backup_bucket)
56
+ end
57
+
58
+ it "another bucket" do
59
+ fetch_events(minimal_settings.merge({ "backup_to_bucket" => backup_bucket}))
60
+ expect(list_remote_files("", backup_bucket).size).to eq(2)
61
+ end
62
+
63
+ after do
64
+ delete_bucket(backup_bucket)
65
+ end
66
+ end
67
+ end
@@ -0,0 +1,52 @@
1
+ def fetch_events(settings)
2
+ queue = []
3
+ s3 = LogStash::Inputs::S3.new(settings)
4
+ s3.register
5
+ s3.process_files(queue)
6
+ queue
7
+ end
8
+
9
+ # delete_files(prefix)
10
+ def upload_file(local_file, remote_name)
11
+ bucket = s3object.bucket(ENV['AWS_LOGSTASH_TEST_BUCKET'])
12
+ file = File.expand_path(File.join(File.dirname(__FILE__), local_file))
13
+ bucket.object(remote_name).upload_file(file)
14
+ end
15
+
16
+ def delete_remote_files(prefix)
17
+ bucket = s3object.bucket(ENV['AWS_LOGSTASH_TEST_BUCKET'])
18
+ bucket.objects(:prefix => prefix).each { |object| object.delete }
19
+ end
20
+
21
+ def list_remote_files(prefix, target_bucket = ENV['AWS_LOGSTASH_TEST_BUCKET'])
22
+ bucket = s3object.bucket(target_bucket)
23
+ bucket.objects(:prefix => prefix).collect(&:key)
24
+ end
25
+
26
+ def create_bucket(name)
27
+ s3object.bucket(name).create
28
+ end
29
+
30
+ def delete_bucket(name)
31
+ s3object.bucket(name).objects.map(&:delete)
32
+ s3object.bucket(name).delete
33
+ end
34
+
35
+ def s3object
36
+ Aws::S3::Resource.new
37
+ end
38
+
39
+ class TestInfiniteS3Object
40
+ def initialize(s3_obj)
41
+ @s3_obj = s3_obj
42
+ end
43
+
44
+ def each
45
+ counter = 1
46
+
47
+ loop do
48
+ yield @s3_obj
49
+ counter +=1
50
+ end
51
+ end
52
+ end
metadata ADDED
@@ -0,0 +1,174 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: logstash-input-s3-cos
3
+ version: !ruby/object:Gem::Version
4
+ version: 3.8.5
5
+ platform: ruby
6
+ authors:
7
+ - Elastic
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2023-03-09 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ requirement: !ruby/object:Gem::Requirement
15
+ requirements:
16
+ - - ">="
17
+ - !ruby/object:Gem::Version
18
+ version: 2.1.12
19
+ - - "<="
20
+ - !ruby/object:Gem::Version
21
+ version: '2.99'
22
+ name: logstash-core-plugin-api
23
+ prerelease: false
24
+ type: :runtime
25
+ version_requirements: !ruby/object:Gem::Requirement
26
+ requirements:
27
+ - - ">="
28
+ - !ruby/object:Gem::Version
29
+ version: 2.1.12
30
+ - - "<="
31
+ - !ruby/object:Gem::Version
32
+ version: '2.99'
33
+ - !ruby/object:Gem::Dependency
34
+ requirement: !ruby/object:Gem::Requirement
35
+ requirements:
36
+ - - ">="
37
+ - !ruby/object:Gem::Version
38
+ version: '0'
39
+ name: logstash-mixin-aws
40
+ prerelease: false
41
+ type: :runtime
42
+ version_requirements: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - ">="
45
+ - !ruby/object:Gem::Version
46
+ version: '0'
47
+ - !ruby/object:Gem::Dependency
48
+ requirement: !ruby/object:Gem::Requirement
49
+ requirements:
50
+ - - "~>"
51
+ - !ruby/object:Gem::Version
52
+ version: 0.0.18
53
+ name: stud
54
+ prerelease: false
55
+ type: :runtime
56
+ version_requirements: !ruby/object:Gem::Requirement
57
+ requirements:
58
+ - - "~>"
59
+ - !ruby/object:Gem::Version
60
+ version: 0.0.18
61
+ - !ruby/object:Gem::Dependency
62
+ requirement: !ruby/object:Gem::Requirement
63
+ requirements:
64
+ - - ">="
65
+ - !ruby/object:Gem::Version
66
+ version: '0'
67
+ name: logstash-devutils
68
+ prerelease: false
69
+ type: :development
70
+ version_requirements: !ruby/object:Gem::Requirement
71
+ requirements:
72
+ - - ">="
73
+ - !ruby/object:Gem::Version
74
+ version: '0'
75
+ - !ruby/object:Gem::Dependency
76
+ requirement: !ruby/object:Gem::Requirement
77
+ requirements:
78
+ - - ">="
79
+ - !ruby/object:Gem::Version
80
+ version: '0'
81
+ name: logstash-codec-json
82
+ prerelease: false
83
+ type: :development
84
+ version_requirements: !ruby/object:Gem::Requirement
85
+ requirements:
86
+ - - ">="
87
+ - !ruby/object:Gem::Version
88
+ version: '0'
89
+ - !ruby/object:Gem::Dependency
90
+ requirement: !ruby/object:Gem::Requirement
91
+ requirements:
92
+ - - ">="
93
+ - !ruby/object:Gem::Version
94
+ version: '0'
95
+ name: logstash-codec-multiline
96
+ prerelease: false
97
+ type: :development
98
+ version_requirements: !ruby/object:Gem::Requirement
99
+ requirements:
100
+ - - ">="
101
+ - !ruby/object:Gem::Version
102
+ version: '0'
103
+ description: This gem is a Logstash plugin required to be installed on top of the
104
+ Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
105
+ gem is not a stand-alone program
106
+ email: info@elastic.co
107
+ executables: []
108
+ extensions: []
109
+ extra_rdoc_files: []
110
+ files:
111
+ - CHANGELOG.md
112
+ - CONTRIBUTORS
113
+ - Gemfile
114
+ - LICENSE
115
+ - NOTICE.TXT
116
+ - README.md
117
+ - docs/index.asciidoc
118
+ - lib/logstash/inputs/s3.rb
119
+ - lib/logstash/inputs/s3/patch.rb
120
+ - logstash-input-s3.gemspec
121
+ - spec/fixtures/cloudfront.log
122
+ - spec/fixtures/compressed.log.gee.zip
123
+ - spec/fixtures/compressed.log.gz
124
+ - spec/fixtures/compressed.log.gzip
125
+ - spec/fixtures/invalid_utf8.gbk.log
126
+ - spec/fixtures/json.log
127
+ - spec/fixtures/json_with_message.log
128
+ - spec/fixtures/multiline.log
129
+ - spec/fixtures/multiple_compressed_streams.gz
130
+ - spec/fixtures/uncompressed.log
131
+ - spec/inputs/s3_spec.rb
132
+ - spec/inputs/sincedb_spec.rb
133
+ - spec/integration/s3_spec.rb
134
+ - spec/support/helpers.rb
135
+ homepage: http://www.elastic.co/guide/en/logstash/current/index.html
136
+ licenses:
137
+ - Apache-2.0
138
+ metadata:
139
+ logstash_plugin: 'true'
140
+ logstash_group: input
141
+ post_install_message:
142
+ rdoc_options: []
143
+ require_paths:
144
+ - lib
145
+ required_ruby_version: !ruby/object:Gem::Requirement
146
+ requirements:
147
+ - - ">="
148
+ - !ruby/object:Gem::Version
149
+ version: '0'
150
+ required_rubygems_version: !ruby/object:Gem::Requirement
151
+ requirements:
152
+ - - ">="
153
+ - !ruby/object:Gem::Version
154
+ version: '0'
155
+ requirements: []
156
+ rubygems_version: 3.3.25
157
+ signing_key:
158
+ specification_version: 4
159
+ summary: Streams events from files in a S3 bucket
160
+ test_files:
161
+ - spec/fixtures/cloudfront.log
162
+ - spec/fixtures/compressed.log.gee.zip
163
+ - spec/fixtures/compressed.log.gz
164
+ - spec/fixtures/compressed.log.gzip
165
+ - spec/fixtures/invalid_utf8.gbk.log
166
+ - spec/fixtures/json.log
167
+ - spec/fixtures/json_with_message.log
168
+ - spec/fixtures/multiline.log
169
+ - spec/fixtures/multiple_compressed_streams.gz
170
+ - spec/fixtures/uncompressed.log
171
+ - spec/inputs/s3_spec.rb
172
+ - spec/inputs/sincedb_spec.rb
173
+ - spec/integration/s3_spec.rb
174
+ - spec/support/helpers.rb