logstash-output-swift 0.1

data/lib/logstash/outputs/swift/writable_directory_validator.rb

@@ -0,0 +1,17 @@
+# encoding: utf-8
+module LogStash
+  module Outputs
+    class Swift
+      class WritableDirectoryValidator
+        def self.valid?(path)
+          begin
+            FileUtils.mkdir_p(path) unless Dir.exist?(path)
+            ::File.writable?(path)
+          rescue
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/swift/write_container_permission_validator.rb

@@ -0,0 +1,52 @@
+# encoding: utf-8
+require "stud/temporary"
+require "socket"
+require "fileutils"
+
+module LogStash
+  module Outputs
+    class Swift
+      class WriteContainerPermissionValidator
+        attr_reader :logger
+
+        def initialize(logger)
+          @logger = logger
+        end
+
+        def valid?(container_resource)
+          begin
+            upload_test_file(container_resource)
+            true
+          rescue StandardError => e
+            logger.error("Error validating bucket write permissions!",
+              :message => e.message,
+              :class => e.class.name,
+              :backtrace => e.backtrace
+              )
+            false
+          end
+        end
+
+        private
+        def upload_test_file(container_resource)
+          generated_at = Time.now
+
+          key = "logstash-programmatic-access-test-object-#{generated_at}"
+          content = "Logstash permission check on #{generated_at}, by #{Socket.gethostname}"
+
+          begin
+            obj = container_resource.files.create(key: key, body: content)
+
+            begin
+              obj.destroy
+            rescue
+              # Try to remove the files on the remote bucket,
+              # but don't raise any errors if that doesn't work.
+              # since we only really need `putobject`.
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/logstash-output-swift.gemspec

@@ -0,0 +1,28 @@
+Gem::Specification.new do |s|
+  s.name            = 'logstash-output-swift'
+  s.version         = '0.1'
+  s.licenses        = ['Apache-2.0']
+  s.summary         = "Sends Logstash events to the Openstack Swift Service"
+  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
+  s.authors         = ["kakoni"]
+  s.email           = "kakoni@gmail.com"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency 'fog-openstack'
+  s.add_runtime_dependency "concurrent-ruby"
+  s.add_runtime_dependency 'stud', '~> 0.0.22'
+  s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'logstash-input-generator'
+  s.add_development_dependency 'logstash-codec-line'
+end

data/spec/integration/dynamic_prefix_spec.rb

@@ -0,0 +1,92 @@
+# encoding: utf-8
+require_relative "../spec_helper"
+require "logstash/outputs/swift"
+require "logstash/codecs/line"
+require "stud/temporary"
+
+describe "Dynamic Prefix", :integration => true do
+  include_context "setup plugin"
+
+  let(:options) { main_options.merge({ "rotation_strategy" => "size" }) }
+  let(:sandbox) { "test" }
+
+  before do
+    clean_remote_files(sandbox)
+    subject.register
+    subject.multi_receive_encoded(batch)
+    subject.close
+  end
+
+  context "With field string" do
+    let(:prefix) { "/#{sandbox}/%{server}/%{language}" }
+    let(:batch) do
+      b = {}
+      e1 = LogStash::Event.new({ "server" => "es1", "language" => "ruby"})
+      b[e1] = "es1-ruby"
+      e2 = LogStash::Event.new({ "server" => "es2", "language" => "java"})
+      b[e2] = "es2-ruby"
+      b
+    end
+
+    it "creates a specific quantity of files" do
+      expect(bucket_resource.objects(:prefix => sandbox).count).to eq(batch.size)
+    end
+
+    it "creates specific keys" do
+      re = Regexp.union(/^es1\/ruby\/ls.s3.sashimi/, /^es2\/java\/ls.s3.sashimi/)
+
+      bucket_resource.objects(:prefix => sandbox) do |obj|
+        expect(obj.key).to match(re)
+      end
+    end
+
+    it "Persists all events" do
+      download_directory = Stud::Temporary.pathname
+
+      FileUtils.rm_rf(download_directory)
+      FileUtils.mkdir_p(download_directory)
+
+      counter = 0
+      bucket_resource.objects(:prefix => sandbox).each do |object|
+        target = File.join(download_directory, "#{counter}.txt")
+        object.get(:response_target => target)
+        counter += 1
+      end
+      expect(Dir.glob(File.join(download_directory, "**", "*.txt")).inject(0) { |sum, f| sum + IO.readlines(f).size }).to eq(batch.size)
+    end
+  end
+
+  context "with unsupported char" do
+    let(:prefix) { "/#{sandbox}/%{server}/%{language}" }
+    let(:batch) do
+      b = {}
+      e1 = LogStash::Event.new({ "server" => "e>s1", "language" => "ruby"})
+      b[e1] = "es2-ruby"
+      b
+    end
+
+    it "convert them to underscore" do
+      re = Regexp.union(/^e_s1\/ruby\/ls.s3.sashimi/)
+
+      bucket_resource.objects(:prefix => sandbox) do |obj|
+        expect(obj.key).to match(re)
+      end
+    end
+  end
+
+  context "with dates" do
+    let(:prefix) { "/#{sandbox}/%{+YYYY-MM-d}" }
+
+    let(:batch) do
+      b = {}
+      e1 = LogStash::Event.new({ "server" => "e>s1", "language" => "ruby"})
+      b[e1] = "es2-ruby"
+      b
+    end
+
+    it "creates dated path" do
+      re = /^#{sandbox}\/\d{4}-\d{2}-\d{1,2}\/ls\.s3\./
+      expect(bucket_resource.objects(:prefix => sandbox).first.key).to match(re)
+    end
+  end
+end

data/spec/integration/gzip_file_spec.rb

@@ -0,0 +1,62 @@
+# encoding: utf-8
+require_relative "../spec_helper"
+require "logstash/outputs/swift"
+require "logstash/codecs/line"
+require "stud/temporary"
+
+describe "Gzip File Time rotation with constant write", :integration => true do
+  include_context "setup plugin"
+
+  let(:time_file) { 0.004 }
+  let(:options) { main_options.merge({ "encoding" => "gzip",
+                                       "rotation_strategy" => "time" }) }
+  let(:number_of_events) { 5000 }
+  let(:batch_size) { 125 }
+  let(:event_encoded) { "Hello world" }
+  let(:batch) do
+    b = {}
+    number_of_events.times do
+      event = LogStash::Event.new({ "message" => event_encoded })
+      b[event] = "#{event_encoded}\n"
+    end
+    b
+  end
+  let(:minimum_number_of_time_rotation) { 3 }
+  let(:batch_step) { (number_of_events / minimum_number_of_time_rotation).ceil }
+
+  before do
+    clean_remote_files(prefix)
+    subject.register
+
+    # simulate batch read/write
+    batch.each_slice(batch_step) do |batch_time|
+      batch_time.each_slice(batch_size) do |smaller_batch|
+        subject.multi_receive_encoded(smaller_batch)
+      end
+      sleep(1)
+    end
+
+    subject.close
+  end
+
+  it "creates multiples files" do
+    # using close will upload the current file
+    expect(bucket_resource.objects(:prefix => prefix).count).to be_between(minimum_number_of_time_rotation, minimum_number_of_time_rotation + 1).inclusive
+  end
+
+  it "Persists all events" do
+    download_directory = Stud::Temporary.pathname
+
+    FileUtils.rm_rf(download_directory)
+    FileUtils.mkdir_p(download_directory)
+
+    counter = 0
+    bucket_resource.objects(:prefix => prefix).each do |object|
+      target = File.join(download_directory, "#{counter}.gz")
+      object.get(:response_target => target)
+      counter += 1
+    end
+
+    expect(Dir.glob(File.join(download_directory, "**", "*.gz")).inject(0) { |sum, f| sum + Zlib::GzipReader.new(File.open(f)).readlines.size }).to eq(number_of_events)
+  end
+end

data/spec/integration/gzip_size_rotation_spec.rb

@@ -0,0 +1,63 @@
+# encoding: utf-8
+require_relative "../spec_helper"
+require "logstash/outputs/swift"
+require "logstash/codecs/line"
+require "stud/temporary"
+
+describe "Gzip Size rotation", :integration => true do
+  include_context "setup plugin"
+
+  let(:document_size) { 20 * 1024 } # in bytes
+
+  let(:options) do
+    main_options.merge({
+      "encoding" => "gzip",
+      "size_file" => document_size,
+      "rotation_strategy" => "size" })
+  end
+
+  let(:number_of_events) { 1_000_000 }
+  let(:batch_size) { 125 }
+  let(:event_encoded) { "Hello world" * 20 }
+  let(:batch) do
+    b = {}
+    batch_size.times do
+      event = LogStash::Event.new({ "message" => event_encoded })
+      b[event] = "#{event_encoded}\n"
+    end
+    b
+  end
+  let(:number_of_files) { number_of_events / 50000 }
+
+  before do
+    clean_remote_files(prefix)
+    subject.register
+    (number_of_events/batch_size).times do
+      subject.multi_receive_encoded(batch)
+    end
+    subject.close
+  end
+
+  it "Rotates the files based on size" do
+    f = bucket_resource.objects(:prefix => prefix).first
+    expect(f.size).to be_between(document_size, document_size * 2).inclusive
+  end
+
+  it "Persists all events" do
+    download_directory = Stud::Temporary.pathname
+
+    FileUtils.rm_rf(download_directory)
+    FileUtils.mkdir_p(download_directory)
+
+    counter = 0
+    bucket_resource.objects(:prefix => prefix).each do |object|
+      target = File.join(download_directory, "#{counter}.txt.gz")
+      object.get(:response_target => target)
+      counter += 1
+    end
+
+    expect(Dir.glob(File.join(download_directory, "**", "*.gz")).inject(0) do |sum, f|
+      sum + Zlib::GzipReader.new(File.open(f)).readlines.size
+    end).to eq(number_of_events)
+  end
+end

data/spec/integration/restore_from_crash_spec.rb

@@ -0,0 +1,39 @@
+# encoding: utf-8
+require_relative "../spec_helper"
+require "logstash/outputs/swift"
+require "logstash/codecs/line"
+require "stud/temporary"
+
+describe "Restore from crash", :integration => true do
+  include_context "setup plugin"
+
+  let(:options) { main_options.merge({ "restore" => true, "canned_acl" => "public-read-write" }) }
+
+  let(:number_of_files) { 5 }
+  let(:dummy_content) { "foobar\n" * 100 }
+
+  before do
+    clean_remote_files(prefix)
+    # Use the S3 factory to create mutliples files with dummy content
+    factory = LogStash::Outputs::S3::TemporaryFileFactory.new(prefix, tags, "none", temporary_directory)
+
+    # Creating a factory always create a file
+    factory.current.write(dummy_content)
+    factory.current.fsync
+
+    (number_of_files - 1).times do
+      factory.rotate!
+      factory.current.write(dummy_content)
+      factory.current.fsync
+    end
+  end
+
+  it "uploads the file to the bucket" do
+    subject.register
+    try(20) do
+      expect(bucket_resource.objects(:prefix => prefix).count).to eq(number_of_files)
+      expect(Dir.glob(File.join(temporary_directory, "*")).size).to eq(0)
+      expect(bucket_resource.objects(:prefix => prefix).first.acl.grants.collect(&:permission)).to include("READ", "WRITE")
+    end
+  end
+end

data/spec/integration/size_rotation_spec.rb

@@ -0,0 +1,59 @@
+# encoding: utf-8
+require_relative "../spec_helper"
+require "logstash/outputs/swift"
+require "logstash/codecs/line"
+require "stud/temporary"
+
+describe "Size rotation", :integration => true do
+  include_context "setup plugin"
+
+  let(:event_size) { "Hello world".bytesize }
+  let(:size_file) { batch_size * event_size * 2 }
+  let(:options) { main_options.merge({ "rotation_strategy" => "size", "size_file" => size_file }) }
+  let(:number_of_events) { 5000 }
+  let(:batch_size) { 125 }
+  let(:event_encoded) { "Hello world" }
+  let(:batch) do
+    b = {}
+    number_of_events.times do
+      event = LogStash::Event.new({ "message" => event_encoded })
+      b[event] = "#{event_encoded}\n"
+    end
+    b
+  end
+  let(:number_of_files) { number_of_events * event_size / size_file }
+
+  before do
+    clean_remote_files(prefix)
+    subject.register
+    batch.each_slice(batch_size) do |smaller_batch|
+      subject.multi_receive_encoded(smaller_batch)
+    end
+    subject.close
+  end
+
+  it "creates a specific quantity of files" do
+    expect(bucket_resource.objects(:prefix => prefix).count).to eq(number_of_files)
+  end
+
+  it "Rotates the files based on size" do
+    bucket_resource.objects(:prefix => prefix).each do |f|
+      expect(f.size).to be_between(size_file, size_file * 2).inclusive
+    end
+  end
+
+  it "Persists all events" do
+    download_directory = Stud::Temporary.pathname
+
+    FileUtils.rm_rf(download_directory)
+    FileUtils.mkdir_p(download_directory)
+
+    counter = 0
+    bucket_resource.objects(:prefix => prefix).each do |object|
+      target = File.join(download_directory, "#{counter}.txt")
+      object.get(:response_target => target)
+      counter += 1
+    end
+    expect(Dir.glob(File.join(download_directory, "**", "*.txt")).inject(0) { |sum, f| sum + IO.readlines(f).size }).to eq(number_of_events)
+  end
+end

data/spec/integration/stress_test_spec.rb

@@ -0,0 +1,60 @@
+# encoding: utf-8
+require_relative "../spec_helper"
+require "logstash/outputs/swift"
+require "logstash/codecs/line"
+require "stud/temporary"
+
+describe "Upload current file on shutdown", :integration => true, :slow => true do
+  include_context "setup plugin"
+  let(:stress_time) { ENV["RUNTIME"] || 1 * 60 }
+  let(:options) { main_options }
+
+  let(:time_file) { 15 }
+  let(:batch_size) { 125 }
+  let(:event_encoded) { "Hello world" }
+  let(:batch) do
+    b = {}
+    batch_size.times do
+      event = LogStash::Event.new({ "message" => event_encoded })
+      b[event] = "#{event_encoded}\n"
+    end
+    b
+  end
+  let(:workers) { 3 }
+
+  it "Persists all events" do
+    started_at = Time.now
+    events_sent = {}
+
+    clean_remote_files(prefix)
+    subject.register
+
+    workers.times do
+      Thread.new do
+        events_sent[Thread.current] = 0
+
+        while Time.now - started_at < stress_time
+          subject.multi_receive_encoded(batch)
+          events_sent[Thread.current] += batch_size
+        end
+      end
+    end
+
+    sleep(1) while Time.now - started_at < stress_time
+
+    subject.close
+
+    download_directory = Stud::Temporary.pathname
+
+    FileUtils.rm_rf(download_directory)
+    FileUtils.mkdir_p(download_directory)
+
+    counter = 0
+    bucket_resource.objects(:prefix => prefix).each do |object|
+      target = File.join(download_directory, "#{counter}.txt")
+      object.get(:response_target => target)
+      counter += 1
+    end
+    expect(Dir.glob(File.join(download_directory, "**", "*.txt")).inject(0) { |sum, f| sum + IO.readlines(f).size }).to eq(events_sent.values.inject(0, :+))
+  end
+end