logstash-input-s3-test 4.0.0

data/spec/inputs/sincedb_spec.rb

@@ -0,0 +1,17 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/inputs/s3"
+require "stud/temporary"
+require "fileutils"
+
+describe LogStash::Inputs::S3::SinceDB::File do
+  let(:file) { Stud::Temporary.file.path }
+  subject { LogStash::Inputs::S3::SinceDB::File.new(file) }
+  before do
+    FileUtils.touch(file)
+  end
+
+  it "doesnt raise an exception if the file is empty" do
+    expect { subject.read }.not_to raise_error
+  end
+end

data/spec/integration/s3_spec.rb

@@ -0,0 +1,61 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/inputs/s3"
+require "aws-sdk-s3"
+require "fileutils"
+require_relative "../support/helpers"
+
+describe LogStash::Inputs::S3, :integration => true, :s3 => true do
+  before do
+    Thread.abort_on_exception = true
+
+    upload_file('../fixtures/uncompressed.log' , "#{prefix}uncompressed_1.log")
+    upload_file('../fixtures/compressed.log.gz', "#{prefix}compressed_1.log.gz")
+  end
+
+  after do
+    delete_remote_files(prefix)
+    FileUtils.rm_rf(temporary_directory)
+    delete_remote_files(backup_prefix)
+  end
+  
+  let(:temporary_directory) { Stud::Temporary.directory }
+  let(:prefix)  { 'logstash-s3-input-prefix/' }
+  
+  let(:minimal_settings)  {  { "access_key_id" => ENV['AWS_ACCESS_KEY_ID'],
+                               "secret_access_key" => ENV['AWS_SECRET_ACCESS_KEY'],
+                               "bucket" => ENV['AWS_LOGSTASH_TEST_BUCKET'],
+                               "region" => ENV["AWS_REGION"] || "us-east-1",
+                               "prefix" => prefix,
+                               "temporary_directory" => temporary_directory } }
+  let(:backup_prefix) { "backup/" }
+
+  it "support prefix to scope the remote files" do
+    events = fetch_events(minimal_settings)
+    expect(events.size).to eq(4)
+  end
+
+
+  it "add a prefix to the file" do
+    fetch_events(minimal_settings.merge({ "backup_to_bucket" => ENV["AWS_LOGSTASH_TEST_BUCKET"],
+                                                   "backup_add_prefix" => backup_prefix }))
+    expect(list_remote_files(backup_prefix).size).to eq(2)
+  end
+
+  it "allow you to backup to a local directory" do
+    Stud::Temporary.directory do |backup_dir|
+      fetch_events(minimal_settings.merge({ "backup_to_dir" => backup_dir }))
+      expect(Dir.glob(File.join(backup_dir, "*")).size).to eq(2)
+    end
+  end
+
+  context "remote backup" do
+    it "another bucket" do
+      fetch_events(minimal_settings.merge({ "backup_to_bucket" => "logstash-s3-input-backup"}))
+      expect(list_remote_files("", "logstash-s3-input-backup").size).to eq(2)
+    end
+
+    after do
+      delete_bucket("logstash-s3-input-backup")
+    end
+  end
+end

data/spec/support/helpers.rb

@@ -0,0 +1,45 @@
+def fetch_events(settings)
+  queue = []
+  s3 = LogStash::Inputs::S3.new(settings)
+  s3.register
+  s3.process_files(queue)
+  queue
+end
+
+# delete_files(prefix)
+def upload_file(local_file, remote_name)
+  bucket = s3object.bucket(ENV['AWS_LOGSTASH_TEST_BUCKET'])
+  file = File.expand_path(File.join(File.dirname(__FILE__), local_file))
+  bucket.object(remote_name).upload_file(file)
+end
+
+def delete_remote_files(prefix)
+  bucket = s3object.bucket(ENV['AWS_LOGSTASH_TEST_BUCKET'])
+  bucket.objects(:prefix => prefix).each { |object| object.delete }
+end
+
+def list_remote_files(prefix, target_bucket = ENV['AWS_LOGSTASH_TEST_BUCKET'])
+  bucket = s3object.bucket(target_bucket)
+  bucket.objects(:prefix => prefix).collect(&:key)
+end
+
+def delete_bucket(name)
+  s3object.bucket(name).objects.map(&:delete)
+  s3object.bucket(name).delete
+end
+
+def s3object
+  Aws::S3::Resource.new
+end
+
+class TestInfiniteS3Object
+  def each
+    counter = 1
+
+    loop do
+      yield "awesome-#{counter}"
+      counter +=1
+    end
+  end
+end
+

metadata

@@ -0,0 +1,189 @@
+--- !ruby/object:Gem::Specification
+name: logstash-input-s3-test
+version: !ruby/object:Gem::Version
+  version: 4.0.0
+platform: ruby
+authors:
+- Sukhbir
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-12-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: logstash-core-plugin-api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.12
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.12
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+- !ruby/object:Gem::Dependency
+  name: stud
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.18
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.18
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-multiline
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a Logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
+  gem is not a stand-alone program
+email: sukhbir947@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- CONTRIBUTORS
+- Gemfile
+- LICENSE
+- NOTICE.TXT
+- README.md
+- docs/index.asciidoc
+- lib/logstash/inputs/s3.rb
+- lib/logstash/inputs/s3/patch.rb
+- logstash-input-s3.gemspec
+- spec/fixtures/cloudfront.log
+- spec/fixtures/compressed.log.gee.zip
+- spec/fixtures/compressed.log.gz
+- spec/fixtures/compressed.log.gzip
+- spec/fixtures/invalid_utf8.gbk.log
+- spec/fixtures/json.log
+- spec/fixtures/json_with_message.log
+- spec/fixtures/multiline.log
+- spec/fixtures/multiple_compressed_streams.gz
+- spec/fixtures/uncompressed.log
+- spec/inputs/s3_spec.rb
+- spec/inputs/sincedb_spec.rb
+- spec/integration/s3_spec.rb
+- spec/support/helpers.rb
+homepage: http://www.elastic.co/guide/en/logstash/current/index.html
+licenses:
+- Apache-2.0
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: input
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.14
+signing_key: 
+specification_version: 4
+summary: Streams events from files in a S3 bucket
+test_files:
+- spec/fixtures/cloudfront.log
+- spec/fixtures/compressed.log.gee.zip
+- spec/fixtures/compressed.log.gz
+- spec/fixtures/compressed.log.gzip
+- spec/fixtures/invalid_utf8.gbk.log
+- spec/fixtures/json.log
+- spec/fixtures/json_with_message.log
+- spec/fixtures/multiline.log
+- spec/fixtures/multiple_compressed_streams.gz
+- spec/fixtures/uncompressed.log
+- spec/inputs/s3_spec.rb
+- spec/inputs/sincedb_spec.rb
+- spec/integration/s3_spec.rb
+- spec/support/helpers.rb