logstash-input-s3-cloudian 1.0.0

data/spec/integration/s3_spec.rb

@@ -0,0 +1,67 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/inputs/s3"
+require "aws-sdk"
+require "fileutils"
+require_relative "../support/helpers"
+
+describe LogStash::Inputs::S3, :integration => true, :s3 => true do
+  before do
+    Thread.abort_on_exception = true
+
+    upload_file('../fixtures/uncompressed.log' , "#{prefix}uncompressed_1.log")
+    upload_file('../fixtures/compressed.log.gz', "#{prefix}compressed_1.log.gz")
+    sleep(LogStash::Inputs::S3::CUTOFF_SECOND + 1)
+  end
+
+  after do
+    delete_remote_files(prefix)
+    FileUtils.rm_rf(temporary_directory)
+    delete_remote_files(backup_prefix)
+  end
+  
+  let(:temporary_directory) { Stud::Temporary.directory }
+  let(:prefix)  { 'logstash-s3-input-prefix/' }
+  
+  let(:minimal_settings)  {  { "access_key_id" => ENV['AWS_ACCESS_KEY_ID'],
+                               "secret_access_key" => ENV['AWS_SECRET_ACCESS_KEY'],
+                               "bucket" => ENV['AWS_LOGSTASH_TEST_BUCKET'],
+                               "region" => ENV["AWS_REGION"] || "us-east-1",
+                               "prefix" => prefix,
+                               "temporary_directory" => temporary_directory } }
+  let(:backup_prefix) { "backup/" }
+  let(:backup_bucket) { "logstash-s3-input-backup" }
+
+  it "support prefix to scope the remote files" do
+    events = fetch_events(minimal_settings)
+    expect(events.size).to eq(4)
+  end
+
+
+  it "add a prefix to the file" do
+    fetch_events(minimal_settings.merge({ "backup_to_bucket" => ENV["AWS_LOGSTASH_TEST_BUCKET"],
+                                                   "backup_add_prefix" => backup_prefix }))
+    expect(list_remote_files(backup_prefix).size).to eq(2)
+  end
+
+  it "allow you to backup to a local directory" do
+    Stud::Temporary.directory do |backup_dir|
+      fetch_events(minimal_settings.merge({ "backup_to_dir" => backup_dir }))
+      expect(Dir.glob(File.join(backup_dir, "*")).size).to eq(2)
+    end
+  end
+
+  context "remote backup" do
+    before do
+      create_bucket(backup_bucket)
+    end
+
+    it "another bucket" do
+      fetch_events(minimal_settings.merge({ "backup_to_bucket" => backup_bucket}))
+      expect(list_remote_files("", backup_bucket).size).to eq(2)
+    end
+
+    after do
+      delete_bucket(backup_bucket)
+    end
+  end
+end

data/spec/support/helpers.rb

@@ -0,0 +1,52 @@
+def fetch_events(settings)
+  queue = []
+  s3 = LogStash::Inputs::S3.new(settings)
+  s3.register
+  s3.process_files(queue)
+  queue
+end
+
+# delete_files(prefix)
+def upload_file(local_file, remote_name)
+  bucket = s3object.bucket(ENV['AWS_LOGSTASH_TEST_BUCKET'])
+  file = File.expand_path(File.join(File.dirname(__FILE__), local_file))
+  bucket.object(remote_name).upload_file(file)
+end
+
+def delete_remote_files(prefix)
+  bucket = s3object.bucket(ENV['AWS_LOGSTASH_TEST_BUCKET'])
+  bucket.objects(:prefix => prefix).each { |object| object.delete }
+end
+
+def list_remote_files(prefix, target_bucket = ENV['AWS_LOGSTASH_TEST_BUCKET'])
+  bucket = s3object.bucket(target_bucket)
+  bucket.objects(:prefix => prefix).collect(&:key)
+end
+
+def create_bucket(name)
+  s3object.bucket(name).create
+end
+
+def delete_bucket(name)
+  s3object.bucket(name).objects.map(&:delete)
+  s3object.bucket(name).delete
+end
+
+def s3object
+  Aws::S3::Resource.new
+end
+
+class TestInfiniteS3Object
+  def initialize(s3_obj)
+    @s3_obj = s3_obj
+  end
+
+  def each
+    counter = 1
+
+    loop do
+      yield @s3_obj
+      counter +=1
+    end
+  end
+end

metadata

@@ -0,0 +1,188 @@
+--- !ruby/object:Gem::Specification
+name: logstash-input-s3-cloudian
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- dm.belousov
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2022-03-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: logstash-core-plugin-api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.12
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.12
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+- !ruby/object:Gem::Dependency
+  name: logstash-mixin-aws
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.1.0
+- !ruby/object:Gem::Dependency
+  name: stud
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.18
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.18
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-multiline
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: logstash-mixin-ecs_compatibility_support
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+description: This gem is a Logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
+  gem is not a stand-alone program
+email: dmitriy.belousov@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- CONTRIBUTORS
+- Gemfile
+- LICENSE
+- NOTICE.TXT
+- README.md
+- docs/index.asciidoc
+- lib/logstash/inputs/s3.rb
+- lib/logstash/inputs/s3/patch.rb
+- logstash-input-s3.gemspec
+- spec/fixtures/cloudfront.log
+- spec/fixtures/compressed.log.gee.zip
+- spec/fixtures/compressed.log.gz
+- spec/fixtures/compressed.log.gzip
+- spec/fixtures/invalid_utf8.gbk.log
+- spec/fixtures/json.log
+- spec/fixtures/json_with_message.log
+- spec/fixtures/multiline.log
+- spec/fixtures/multiple_compressed_streams.gz
+- spec/fixtures/uncompressed.log
+- spec/inputs/s3_spec.rb
+- spec/inputs/sincedb_spec.rb
+- spec/integration/s3_spec.rb
+- spec/support/helpers.rb
+homepage: http://www.elastic.co/guide/en/logstash/current/index.html
+licenses:
+- Apache-2.0
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: input
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.2
+signing_key: 
+specification_version: 4
+summary: Streams events from files in a S3 bucket
+test_files:
+- spec/fixtures/cloudfront.log
+- spec/fixtures/compressed.log.gee.zip
+- spec/fixtures/compressed.log.gz
+- spec/fixtures/compressed.log.gzip
+- spec/fixtures/invalid_utf8.gbk.log
+- spec/fixtures/json.log
+- spec/fixtures/json_with_message.log
+- spec/fixtures/multiline.log
+- spec/fixtures/multiple_compressed_streams.gz
+- spec/fixtures/uncompressed.log
+- spec/inputs/s3_spec.rb
+- spec/inputs/sincedb_spec.rb
+- spec/integration/s3_spec.rb
+- spec/support/helpers.rb