logstash-output-swift 0.1

data/spec/outputs/swift/temporary_file_spec.rb

@@ -0,0 +1,47 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/swift/temporary_file"
+require "stud/temporary"
+require "fileutils"
+require "securerandom"
+
+describe LogStash::Outputs::Swift::TemporaryFile do
+  let(:content) { "hello world" }
+  let(:key) { "foo" }
+  let(:uuid) { SecureRandom.uuid }
+  let(:temporary_file) { ::File.open(::File.join(temporary_directory, uuid, key), "w+") }
+  let(:temporary_directory) {  Stud::Temporary.directory }
+
+  before :each do
+    FileUtils.mkdir_p(::File.join(temporary_directory, uuid))
+  end
+
+  subject { described_class.new(key, temporary_file, temporary_directory) }
+
+  it "returns the key of the file" do
+    expect(subject.key).to eq(key)
+  end
+
+  it "saves content to a file" do
+    subject.write(content)
+    subject.close
+    expect(File.read(subject.path).strip).to eq(content)
+  end
+
+  it "deletes a file" do
+    expect(File.exist?(subject.path)).to be_truthy
+    subject.delete!
+    expect(File.exist?(subject.path)).to be_falsey
+  end
+
+  it "successfully delete a file already closed" do
+    subject.close
+    expect(File.exist?(subject.path)).to be_truthy
+    subject.delete!
+    expect(File.exist?(subject.path)).to be_falsey
+  end
+
+  it "returns the creation time" do
+    expect(subject.ctime).to be < Time.now + 0.5
+  end
+end

data/spec/outputs/swift/time_rotation_policy_spec.rb

@@ -0,0 +1,60 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/swift/time_rotation_policy"
+require "logstash/outputs/swift/temporary_file"
+
+describe LogStash::Outputs::Swift::TimeRotationPolicy do
+  subject { described_class.new(max_time) }
+
+  let(:max_time) { 1 }
+  let(:temporary_directory) {  Stud::Temporary.directory }
+  let(:temporary_file) { Stud::Temporary.file }
+  let(:name) { "foobar" }
+  let(:content) { "hello" * 1000 }
+  let(:file) { LogStash::Outputs::Swift::TemporaryFile.new(name, temporary_file, temporary_directory) }
+
+  it "raises an exception if the `file_time` is set to 0" do
+    expect { described_class.new(0) }.to raise_error(LogStash::ConfigurationError, /`time_file` need to be greather than 0/)
+  end
+
+  it "raises an exception if the `file_time` is < 0" do
+    expect { described_class.new(-100) }.to raise_error(LogStash::ConfigurationError, /`time_file` need to be greather than 0/)
+  end
+
+  context "when the size of the file is superior to 0" do
+    before :each do
+      file.write(content)
+      file.fsync
+    end
+
+    it "returns true if the file old enough" do
+      allow(file).to receive(:ctime).and_return(Time.now - (max_time * 2 * 60))
+      expect(subject.rotate?(file)).to be_truthy
+    end
+
+    it "returns false is not old enough" do
+      expect(subject.rotate?(file)).to be_falsey
+    end
+  end
+
+  context "When the size of the file is 0" do
+    it "returns false if the file old enough" do
+      allow(file).to receive(:ctime).and_return(Time.now - (max_time * 2 * 60))
+      expect(subject.rotate?(file)).to be_falsey
+    end
+
+    it "returns false is not old enough" do
+      expect(subject.rotate?(file)).to be_falsey
+    end
+  end
+
+  context "#needs_periodic?" do
+    it "return false" do
+      expect(subject.needs_periodic?).to be_truthy
+    end
+  end
+
+  it "convert minute into seconds" do
+    expect(subject.time_file).to eq(60)
+  end
+end

data/spec/outputs/swift/uploader_spec.rb

@@ -0,0 +1,49 @@
+# Encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/swift/uploader"
+require "logstash/outputs/swift/temporary_file"
+require "fog/openstack"
+require "stud/temporary"
+
+describe LogStash::Outputs::Swift::Uploader do
+  let(:logger) { spy(:logger ) }
+  let(:max_upload_workers) { 1 }
+  let(:bucket_name) { "foobar-bucket" }
+
+  let(:service) { Fog::Storage::OpenStack.new }
+  let(:container) { service.directories.get(bucket_name)  }
+
+  let(:temporary_directory) { Stud::Temporary.pathname }
+  let(:temporary_file) { Stud::Temporary.file }
+  let(:key) { "foobar" }
+  let(:upload_options) { {} }
+  let(:threadpool) do
+    Concurrent::ThreadPoolExecutor.new({
+                                         :min_threads => 1,
+                                         :max_threads => 8,
+                                         :max_queue => 1,
+                                         :fallback_policy => :caller_runs
+                                       })
+  end
+
+  let(:file) do
+    f = LogStash::Outputs::Swift::TemporaryFile.new(key, temporary_file, temporary_directory)
+    f.write("random content")
+    f.fsync
+    f
+  end
+
+  subject { described_class.new(container, logger, threadpool) }
+
+  it "upload file to the swift container" do
+    expect { subject.upload(file) }.not_to raise_error
+  end
+
+  it "execute a callback when the upload is complete" do
+    callback = proc { |f| }
+
+    expect(callback).to receive(:call).with(file)
+    subject.upload(file, { :on_complete => callback })
+  end
+
+end

data/spec/outputs/swift/writable_directory_validator_spec.rb

@@ -0,0 +1,40 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/swift/writable_directory_validator"
+require "stud/temporary"
+
+describe LogStash::Outputs::Swift::WritableDirectoryValidator do
+  let(:temporary_directory) {  File.join(Stud::Temporary.directory, Time.now.to_i.to_s) }
+
+  subject { described_class }
+
+  context "when the directory doesn't exists" do
+    it "creates the directory" do
+      expect(Dir.exist?(temporary_directory)).to be_falsey
+      expect(subject.valid?(temporary_directory)).to be_truthy
+      expect(Dir.exist?(temporary_directory)).to be_truthy
+    end
+  end
+
+  context "when the directory exist" do
+    before do
+      FileUtils.mkdir_p(temporary_directory)
+    end
+
+    it "doesn't change the directory" do
+      expect(Dir.exist?(temporary_directory)).to be_truthy
+      expect(subject.valid?(temporary_directory)).to be_truthy
+      expect(Dir.exist?(temporary_directory)).to be_truthy
+    end
+  end
+
+  it "return false if the directory is not writable" do
+    expect(::File).to receive(:writable?).with(temporary_directory).and_return(false)
+    expect(subject.valid?(temporary_directory)).to be_falsey
+  end
+
+  it "return true if the directory is writable" do
+    expect(::File).to receive(:writable?).with(temporary_directory).and_return(true)
+    expect(subject.valid?(temporary_directory)).to be_truthy
+  end
+end

data/spec/outputs/swift/write_container_permission_validator_spec.rb

@@ -0,0 +1,38 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/swift/write_container_permission_validator"
+require "fog/openstack"
+
+describe LogStash::Outputs::Swift::WriteContainerPermissionValidator do
+  let(:logger) { spy(:logger ) }
+  let(:bucket_name) { "foobar" }
+  let(:obj) { double("swift_object") }
+
+  let(:service) { Fog::Storage::OpenStack.new }
+  let(:container) { service.directories.get(bucket_name)  }
+
+  let(:upload_options) { {} }
+
+  subject { described_class.new(logger) }
+
+  context "when permissions are sufficient" do
+    it "returns true" do
+      expect(container).to receive_message_chain(:files, :create).with(any_args).and_return(obj)
+      expect(obj).to receive(:delete).and_return(true)
+      expect(subject.valid?(container)).to be_truthy
+    end
+
+    it "hides delete errors" do
+      expect(container).to receive_message_chain(:files, :create).with(any_args).and_return(obj)
+      expect(obj).to receive(:delete).and_raise(StandardError)
+      expect(subject.valid?(container)).to be_truthy
+    end
+  end
+
+  context "when permission aren't sufficient" do
+    it "returns false" do
+      expect(container).to receive_message_chain(:files, :create).with(any_args).and_raise(StandardError)
+      expect(subject.valid?(container)).to be_falsey
+    end
+  end
+end

data/spec/outputs/swift_spec.rb

@@ -0,0 +1,76 @@
+# encoding: utf-8
+require "logstash/outputs/swift"
+require "logstash/event"
+require "logstash/codecs/line"
+require "stud/temporary"
+require "fog/openstack"
+
+describe LogStash::Outputs::Swift do
+  let(:prefix) { "super/%{server}" }
+  let(:bucket_name) { "mybucket" }
+  let(:options) { { "container" => bucket_name,
+                    "prefix" => prefix,
+                    "restore" => false,
+                    "username" => "access_key_id",
+                    "api_key" => "secret_access_key",
+                    "auth_url" => "secret_access_key",
+                    "project_name" => "secret_access_key",
+                    "domain_name" => "secret_access_key"
+  } }
+  let(:service) { Fog::Storage::OpenStack.new }
+  let(:mock_container) { service.directories.get(bucket_name)  }
+  let(:event) { LogStash::Event.new({ "server" => "overwatch" }) }
+  let(:event_encoded) { "super hype" }
+  let(:events_and_encoded) { { event => event_encoded } }
+
+  subject { described_class.new(options) }
+
+  before do
+    allow(subject).to receive(:container_resource).and_return(mock_container)
+    allow_any_instance_of(LogStash::Outputs::Swift::WriteContainerPermissionValidator).to receive(:valid?).with(mock_container).and_return(true)
+  end
+
+  context "#register configuration validation" do
+    describe "temporary directory" do
+      let(:temporary_directory) { Stud::Temporary.pathname }
+      let(:options) { super.merge({ "temporary_directory" => temporary_directory }) }
+
+      it "creates the directory when it doesn't exist" do
+        expect(Dir.exist?(temporary_directory)).to be_falsey
+        subject.register
+        expect(Dir.exist?(temporary_directory)).to be_truthy
+      end
+
+      it "raises an error if we cannot write to the directory" do
+        expect(LogStash::Outputs::Swift::WritableDirectoryValidator).to receive(:valid?).with(temporary_directory).and_return(false)
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+      end
+    end
+
+    it "validates the prefix" do
+      swift = described_class.new(options.merge({ "prefix" => "`no\><^" }))
+      expect { swift.register }.to raise_error(LogStash::ConfigurationError)
+    end
+
+    it "allow to not validate credentials" do
+      swift = described_class.new(options.merge({"validate_credentials_on_root_bucket" => false}))
+      expect_any_instance_of(LogStash::Outputs::Swift::WriteContainerPermissionValidator).not_to receive(:valid?).with(any_args)
+      swift.register
+    end
+  end
+
+  context "receiving events" do
+    before do
+      subject.register
+    end
+
+    after do
+      subject.close
+    end
+
+    it "uses `Event#sprintf` for the prefix" do
+      expect(event).to receive(:sprintf).with(prefix).and_return("super/overwatch")
+      subject.multi_receive_encoded(events_and_encoded)
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,48 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require_relative "supports/helpers"
+require "logstash/logging/logger"
+require "fog/openstack"
+
+LogStash::Logging::Logger::configure_logging("debug") if ENV["DEBUG"]
+
+module Fog
+  module Storage
+    class OpenStack
+
+      class Mock
+
+        def put_object(container, object, data, options = {}, &block)
+          response = Excon::Response.new
+          response.status = 201
+          response.body = ""
+          response
+        end
+
+        def put_container(name, options = {})
+          response = Excon::Response.new
+          response.status = 201
+          response.body = ""
+          response
+        end
+
+        def get_container(container, options = {})
+          response = Excon::Response.new
+          response.status = 201
+          response.body = []
+          response
+        end
+
+        def get_containers(options = {})
+          response = Excon::Response.new
+          response.status = 201
+          response.body = []
+          response
+        end
+      end
+
+    end
+  end
+end
+
+Fog.mock!

data/spec/supports/helpers.rb

@@ -0,0 +1,38 @@
+# encoding: utf-8
+shared_context "setup plugin" do
+  let(:temporary_directory) { Stud::Temporary.pathname }
+
+  let(:bucket) { ENV["AWS_LOGSTASH_TEST_BUCKET"] }
+  let(:access_key_id) {  ENV["AWS_ACCESS_KEY_ID"] }
+  let(:secret_access_key) { ENV["AWS_SECRET_ACCESS_KEY"] }
+  let(:size_file) { 100 }
+  let(:time_file) { 100 }
+  let(:tags) { [] }
+  let(:prefix) { "home" }
+  let(:region) { ENV['AWS_REGION'] }
+
+  let(:main_options) do
+    {
+      "bucket" => bucket,
+      "prefix" => prefix,
+      "temporary_directory" => temporary_directory,
+      "access_key_id" => access_key_id,
+      "secret_access_key" => secret_access_key,
+      "size_file" => size_file,
+      "time_file" => time_file,
+      "region" => region,
+      "tags" => []
+    }
+  end
+
+  let(:client_credentials) { Aws::Credentials.new(access_key_id, secret_access_key) }
+  let(:bucket_resource) { Aws::S3::Bucket.new(bucket, { :credentials => client_credentials, :region => region }) }
+
+  subject { LogStash::Outputs::S3.new(options) }
+end
+
+def clean_remote_files(prefix = "")
+  bucket_resource.objects(:prefix => prefix).each do |object|
+    object.delete
+  end
+end

metadata

@@ -0,0 +1,212 @@
+--- !ruby/object:Gem::Specification
+name: logstash-output-swift
+version: !ruby/object:Gem::Version
+  version: '0.1'
+platform: ruby
+authors:
+- kakoni
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-12-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: logstash-core-plugin-api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+- !ruby/object:Gem::Dependency
+  name: fog-openstack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: stud
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.22
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.22
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: logstash-input-generator
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-line
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a Logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
+  gem is not a stand-alone program
+email: kakoni@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- CONTRIBUTORS
+- DEVELOPER.md
+- Gemfile
+- LICENSE
+- NOTICE.TXT
+- README.md
+- lib/logstash/outputs/swift.rb
+- lib/logstash/outputs/swift/file_repository.rb
+- lib/logstash/outputs/swift/path_validator.rb
+- lib/logstash/outputs/swift/size_and_time_rotation_policy.rb
+- lib/logstash/outputs/swift/size_rotation_policy.rb
+- lib/logstash/outputs/swift/temporary_file.rb
+- lib/logstash/outputs/swift/temporary_file_factory.rb
+- lib/logstash/outputs/swift/time_rotation_policy.rb
+- lib/logstash/outputs/swift/uploader.rb
+- lib/logstash/outputs/swift/writable_directory_validator.rb
+- lib/logstash/outputs/swift/write_container_permission_validator.rb
+- logstash-output-swift.gemspec
+- spec/integration/dynamic_prefix_spec.rb
+- spec/integration/gzip_file_spec.rb
+- spec/integration/gzip_size_rotation_spec.rb
+- spec/integration/restore_from_crash_spec.rb
+- spec/integration/size_rotation_spec.rb
+- spec/integration/stress_test_spec.rb
+- spec/integration/time_based_rotation_with_constant_write_spec.rb
+- spec/integration/time_based_rotation_with_stale_write_spec.rb
+- spec/integration/upload_current_file_on_shutdown_spec.rb
+- spec/outputs/swift/file_repository_spec.rb
+- spec/outputs/swift/size_and_time_rotation_policy_spec.rb
+- spec/outputs/swift/size_rotation_policy_spec.rb
+- spec/outputs/swift/temporary_file_factory_spec.rb
+- spec/outputs/swift/temporary_file_spec.rb
+- spec/outputs/swift/time_rotation_policy_spec.rb
+- spec/outputs/swift/uploader_spec.rb
+- spec/outputs/swift/writable_directory_validator_spec.rb
+- spec/outputs/swift/write_container_permission_validator_spec.rb
+- spec/outputs/swift_spec.rb
+- spec/spec_helper.rb
+- spec/supports/helpers.rb
+homepage: 
+licenses:
+- Apache-2.0
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: output
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.13
+signing_key: 
+specification_version: 4
+summary: Sends Logstash events to the Openstack Swift Service
+test_files:
+- spec/integration/dynamic_prefix_spec.rb
+- spec/integration/gzip_file_spec.rb
+- spec/integration/gzip_size_rotation_spec.rb
+- spec/integration/restore_from_crash_spec.rb
+- spec/integration/size_rotation_spec.rb
+- spec/integration/stress_test_spec.rb
+- spec/integration/time_based_rotation_with_constant_write_spec.rb
+- spec/integration/time_based_rotation_with_stale_write_spec.rb
+- spec/integration/upload_current_file_on_shutdown_spec.rb
+- spec/outputs/swift/file_repository_spec.rb
+- spec/outputs/swift/size_and_time_rotation_policy_spec.rb
+- spec/outputs/swift/size_rotation_policy_spec.rb
+- spec/outputs/swift/temporary_file_factory_spec.rb
+- spec/outputs/swift/temporary_file_spec.rb
+- spec/outputs/swift/time_rotation_policy_spec.rb
+- spec/outputs/swift/uploader_spec.rb
+- spec/outputs/swift/writable_directory_validator_spec.rb
+- spec/outputs/swift/write_container_permission_validator_spec.rb
+- spec/outputs/swift_spec.rb
+- spec/spec_helper.rb
+- spec/supports/helpers.rb