logstash-integration-aws 0.1.0.pre

data/spec/integration/restore_from_crash_spec.rb

@@ -0,0 +1,67 @@
+# encoding: utf-8
+require_relative "../spec_helper"
+require "logstash/outputs/s3"
+require "logstash/codecs/line"
+require "stud/temporary"
+
+describe "Restore from crash", :integration => true do
+  include_context "setup plugin"
+
+  let(:options) { main_options.merge({ "restore" => true, "canned_acl" => "public-read-write" }) }
+
+  let(:number_of_files) { 5 }
+  let(:dummy_content) { "foobar\n" * 100 }
+  let(:factory) { LogStash::Outputs::S3::TemporaryFileFactory.new(prefix, tags, "none", temporary_directory)}
+
+  before do
+    clean_remote_files(prefix)
+  end
+
+
+  context 'with a non-empty tempfile' do
+    before do
+      # Creating a factory always create a file
+      factory.current.write(dummy_content)
+      factory.current.fsync
+
+      (number_of_files - 1).times do
+        factory.rotate!
+        factory.current.write(dummy_content)
+        factory.current.fsync
+      end
+    end
+    it "uploads the file to the bucket" do
+      subject.register
+      try(20) do
+        expect(bucket_resource.objects(:prefix => prefix).count).to eq(number_of_files)
+        expect(Dir.glob(File.join(temporary_directory, "*")).size).to eq(0)
+        expect(bucket_resource.objects(:prefix => prefix).first.acl.grants.collect(&:permission)).to include("READ", "WRITE")
+      end
+    end
+  end
+
+  context 'with an empty tempfile' do
+    before do
+      factory.current
+      factory.rotate!
+    end
+
+    it "should remove the temporary file" do
+      expect(Dir.glob(::File.join(temporary_directory, "**", "*")).size).to be > 0
+      subject.register
+      puts Dir.glob(::File.join(temporary_directory, "**", "*"))
+      expect(Dir.glob(::File.join(temporary_directory, "**", "*")).size).to eq(0)
+    end
+
+    it "should not upload the file to the bucket" do
+      expect(bucket_resource.objects(:prefix => prefix).count).to eq(0)
+      expect(Dir.glob(::File.join(temporary_directory, "**", "*")).size).to be > 0
+      subject.register
+
+      # Sleep to give enough time for plugin upload to s3 if it attempts to upload empty temporary file to S3
+      sleep 5
+      expect(bucket_resource.objects(:prefix => prefix).count).to eq(0)
+    end
+  end
+end
+

data/spec/integration/s3_spec.rb

@@ -0,0 +1,66 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/inputs/s3"
+require "fileutils"
+require_relative "../support/helpers"
+
+describe LogStash::Inputs::S3, :integration => true, :s3 => true do
+  before do
+    Thread.abort_on_exception = true
+
+    upload_file('../fixtures/uncompressed.log' , "#{prefix}uncompressed_1.log")
+    upload_file('../fixtures/compressed.log.gz', "#{prefix}compressed_1.log.gz")
+    sleep(LogStash::Inputs::S3::CUTOFF_SECOND + 1)
+  end
+
+  after do
+    delete_remote_files(prefix)
+    FileUtils.rm_rf(temporary_directory)
+    delete_remote_files(backup_prefix)
+  end
+  
+  let(:temporary_directory) { Stud::Temporary.directory }
+  let(:prefix)  { 'logstash-s3-input-prefix/' }
+  
+  let(:minimal_settings)  {  { "access_key_id" => ENV['AWS_ACCESS_KEY_ID'],
+                               "secret_access_key" => ENV['AWS_SECRET_ACCESS_KEY'],
+                               "bucket" => ENV['AWS_LOGSTASH_TEST_BUCKET'],
+                               "region" => ENV["AWS_REGION"] || "us-east-1",
+                               "prefix" => prefix,
+                               "temporary_directory" => temporary_directory } }
+  let(:backup_prefix) { "backup/" }
+  let(:backup_bucket) { "logstash-s3-input-backup" }
+
+  it "support prefix to scope the remote files" do
+    events = fetch_events(minimal_settings)
+    expect(events.size).to eq(4)
+  end
+
+
+  it "add a prefix to the file" do
+    fetch_events(minimal_settings.merge({ "backup_to_bucket" => ENV["AWS_LOGSTASH_TEST_BUCKET"],
+                                                   "backup_add_prefix" => backup_prefix }))
+    expect(list_remote_files(backup_prefix).size).to eq(2)
+  end
+
+  it "allow you to backup to a local directory" do
+    Stud::Temporary.directory do |backup_dir|
+      fetch_events(minimal_settings.merge({ "backup_to_dir" => backup_dir }))
+      expect(Dir.glob(File.join(backup_dir, "*")).size).to eq(2)
+    end
+  end
+
+  context "remote backup" do
+    before do
+      create_bucket(backup_bucket)
+    end
+
+    it "another bucket" do
+      fetch_events(minimal_settings.merge({ "backup_to_bucket" => backup_bucket}))
+      expect(list_remote_files("", backup_bucket).size).to eq(2)
+    end
+
+    after do
+      delete_bucket(backup_bucket)
+    end
+  end
+end

data/spec/integration/size_rotation_spec.rb

@@ -0,0 +1,59 @@
+# encoding: utf-8
+require_relative "../spec_helper"
+require "logstash/outputs/s3"
+require "logstash/codecs/line"
+require "stud/temporary"
+
+describe "Size rotation", :integration => true do
+  include_context "setup plugin"
+
+  let(:event_size) { "Hello world".bytesize }
+  let(:size_file) { batch_size * event_size * 2 }
+  let(:options) { main_options.merge({ "rotation_strategy" => "size", "size_file" => size_file }) }
+  let(:number_of_events) { 5000 }
+  let(:batch_size) { 125 }
+  let(:event_encoded) { "Hello world" }
+  let(:batch) do
+    b = {}
+    number_of_events.times do
+      event = LogStash::Event.new({ "message" => event_encoded })
+      b[event] = "#{event_encoded}\n"
+    end
+    b
+  end
+  let(:number_of_files) { number_of_events * event_size / size_file }
+
+  before do
+    clean_remote_files(prefix)
+    subject.register
+    batch.each_slice(batch_size) do |smaller_batch|
+      subject.multi_receive_encoded(smaller_batch)
+    end
+    subject.close
+  end
+
+  it "creates a specific quantity of files" do
+    expect(bucket_resource.objects(:prefix => prefix).count).to eq(number_of_files)
+  end
+
+  it "Rotates the files based on size" do
+    bucket_resource.objects(:prefix => prefix).each do |f|
+      expect(f.size).to be_between(size_file, size_file * 2).inclusive
+    end
+  end
+
+  it "Persists all events" do
+    download_directory = Stud::Temporary.pathname
+
+    FileUtils.rm_rf(download_directory)
+    FileUtils.mkdir_p(download_directory)
+
+    counter = 0
+    bucket_resource.objects(:prefix => prefix).each do |object|
+      target = File.join(download_directory, "#{counter}.txt")
+      object.get(:response_target => target)
+      counter += 1
+    end
+    expect(Dir.glob(File.join(download_directory, "**", "*.txt")).inject(0) { |sum, f| sum + IO.readlines(f).size }).to eq(number_of_events)
+  end
+end

data/spec/integration/sqs_spec.rb

@@ -0,0 +1,110 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/inputs/sqs"
+require "logstash/event"
+require "logstash/json"
+require_relative "../support/helpers"
+require "thread"
+
+Thread.abort_on_exception = true
+
+describe "LogStash::Inputs::SQS integration", :integration => true do
+  let(:decoded_message) { { "drstrange" => "is-he-really-that-strange" } }
+  let(:encoded_message) { LogStash::Json.dump(decoded_message) }
+  let(:queue) { Queue.new }
+
+  let(:input) { LogStash::Inputs::SQS.new(options) }
+
+  context "with invalid credentials" do
+    let(:options) do
+      {
+        "queue" => "do-not-exist",
+        "access_key_id" => "bad_access",
+        "secret_access_key" => "bad_secret_key",
+        "region" => ENV["AWS_REGION"]
+      }
+    end
+
+    subject { input }
+
+    it "raises a Configuration error if the credentials are bad" do
+      expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+    end
+  end
+
+  context "with valid credentials" do
+    let(:options) do
+      {
+        "queue" => ENV["SQS_QUEUE_NAME"],
+        "access_key_id" => ENV['AWS_ACCESS_KEY_ID'],
+        "secret_access_key" => ENV['AWS_SECRET_ACCESS_KEY'],
+        "region" => ENV["AWS_REGION"]
+      }
+    end
+
+    before :each do
+      push_sqs_event(encoded_message)
+      input.register
+      @server = Thread.new { input.run(queue) }
+    end
+
+    after do
+      @server.kill
+    end
+
+    subject { queue.pop }
+
+    it "creates logstash events" do
+      expect(subject["drstrange"]).to eq(decoded_message["drstrange"])
+    end
+
+    context "when the optionals fields are not specified" do
+      let(:id_field) { "my_id_field" }
+      let(:md5_field) { "my_md5_field" }
+      let(:sent_timestamp_field) { "my_sent_timestamp_field" }
+
+      it "add the `message_id`" do
+        expect(subject[id_field]).to be_nil
+      end
+
+      it "add the `md5_of_body`" do
+        expect(subject[md5_field]).to be_nil
+      end
+
+      it "add the `sent_timestamp`" do
+        expect(subject[sent_timestamp_field]).to be_nil
+      end
+
+    end
+
+    context "when the optionals fields are specified" do
+      let(:id_field) { "my_id_field" }
+      let(:md5_field) { "my_md5_field" }
+      let(:sent_timestamp_field) { "my_sent_timestamp_field" }
+
+      let(:options) do
+        {
+          "queue" => ENV["SQS_QUEUE_NAME"],
+          "access_key_id" => ENV['AWS_ACCESS_KEY_ID'],
+          "secret_access_key" => ENV['AWS_SECRET_ACCESS_KEY'],
+          "region" => ENV["AWS_REGION"],
+          "id_field" => id_field,
+          "md5_field" => md5_field,
+          "sent_timestamp_field" => sent_timestamp_field
+        }
+      end
+
+      it "add the `message_id`" do
+        expect(subject[id_field]).not_to be_nil
+      end
+
+      it "add the `md5_of_body`" do
+        expect(subject[md5_field]).not_to be_nil
+      end
+
+      it "add the `sent_timestamp`" do
+        expect(subject[sent_timestamp_field]).not_to be_nil
+      end
+    end
+  end
+end

data/spec/integration/stress_test_spec.rb

@@ -0,0 +1,60 @@
+# encoding: utf-8
+require_relative "../spec_helper"
+require "logstash/outputs/s3"
+require "logstash/codecs/line"
+require "stud/temporary"
+
+describe "Upload current file on shutdown", :integration => true, :slow => true do
+  include_context "setup plugin"
+  let(:stress_time) { ENV["RUNTIME"] || 1 * 60 }
+  let(:options) { main_options }
+
+  let(:time_file) { 15 }
+  let(:batch_size) { 125 }
+  let(:event_encoded) { "Hello world" }
+  let(:batch) do
+    b = {}
+    batch_size.times do
+      event = LogStash::Event.new({ "message" => event_encoded })
+      b[event] = "#{event_encoded}\n"
+    end
+    b
+  end
+  let(:workers) { 3 }
+
+  it "Persists all events" do
+    started_at = Time.now
+    events_sent = {}
+
+    clean_remote_files(prefix)
+    subject.register
+
+    workers.times do
+      Thread.new do
+        events_sent[Thread.current] = 0
+
+        while Time.now - started_at < stress_time
+          subject.multi_receive_encoded(batch)
+          events_sent[Thread.current] += batch_size
+        end
+      end
+    end
+
+    sleep(1) while Time.now - started_at < stress_time
+
+    subject.close
+
+    download_directory = Stud::Temporary.pathname
+
+    FileUtils.rm_rf(download_directory)
+    FileUtils.mkdir_p(download_directory)
+
+    counter = 0
+    bucket_resource.objects(:prefix => prefix).each do |object|
+      target = File.join(download_directory, "#{counter}.txt")
+      object.get(:response_target => target)
+      counter += 1
+    end
+    expect(Dir.glob(File.join(download_directory, "**", "*.txt")).inject(0) { |sum, f| sum + IO.readlines(f).size }).to eq(events_sent.values.inject(0, :+))
+  end
+end

data/spec/integration/time_based_rotation_with_constant_write_spec.rb

@@ -0,0 +1,60 @@
+# encoding: utf-8
+require_relative "../spec_helper"
+require "logstash/outputs/s3"
+require "logstash/codecs/line"
+require "stud/temporary"
+
+describe "File Time rotation with constant write", :integration => true do
+  include_context "setup plugin"
+
+  let(:time_file) { 0.004 }
+  let(:options) { main_options.merge({ "rotation_strategy" => "time" }) }
+  let(:number_of_events) { 5000 }
+  let(:batch_size) { 125 }
+  let(:event_encoded) { "Hello world" }
+  let(:batch) do
+    b = {}
+    number_of_events.times do
+      event = LogStash::Event.new({ "message" => event_encoded })
+      b[event] = "#{event_encoded}\n"
+    end
+    b
+  end
+  let(:minimum_number_of_time_rotation) { 3 }
+  let(:batch_step) { (number_of_events / minimum_number_of_time_rotation).ceil }
+
+  before do
+    clean_remote_files(prefix)
+    subject.register
+
+    # simulate batch read/write
+    batch.each_slice(batch_step) do |batch_time|
+      batch_time.each_slice(batch_size) do |smaller_batch|
+        subject.multi_receive_encoded(smaller_batch)
+      end
+      sleep(1)
+    end
+
+    subject.close
+  end
+
+  it "creates multiples files" do
+    # using close will upload the current file
+    expect(bucket_resource.objects(:prefix => prefix).count).to be_between(minimum_number_of_time_rotation, minimum_number_of_time_rotation + 1).inclusive
+  end
+
+  it "Persists all events" do
+    download_directory = Stud::Temporary.pathname
+
+    FileUtils.rm_rf(download_directory)
+    FileUtils.mkdir_p(download_directory)
+
+    counter = 0
+    bucket_resource.objects(:prefix => prefix).each do |object|
+      target = File.join(download_directory, "#{counter}.txt")
+      object.get(:response_target => target)
+      counter += 1
+    end
+    expect(Dir.glob(File.join(download_directory, "**", "*.txt")).inject(0) { |sum, f| sum + IO.readlines(f).size }).to eq(number_of_events)
+  end
+end

data/spec/integration/time_based_rotation_with_stale_write_spec.rb

@@ -0,0 +1,64 @@
+# encoding: utf-8
+require_relative "../spec_helper"
+require "logstash/outputs/s3"
+require "logstash/codecs/line"
+require "stud/temporary"
+
+describe "File Time rotation with stale write", :integration => true do
+  include_context "setup plugin"
+
+  let(:time_file) { 0.0004 }
+  let(:options) { main_options.merge({ "rotation_strategy" => "time" }) }
+  let(:number_of_events) { 5000 }
+  let(:batch_size) { 125 }
+  let(:event_encoded) { "Hello world" }
+  let(:batch) do
+    b = {}
+    number_of_events.times do
+      event = LogStash::Event.new({ "message" => event_encoded })
+      b[event] = "#{event_encoded}\n"
+    end
+    b
+  end
+
+  before do
+    stub_const('LogStash::Outputs::S3::PERIODIC_CHECK_INTERVAL_IN_SECONDS', 1)
+    clean_remote_files(prefix)
+    subject.register
+    subject.multi_receive_encoded(batch)
+    sleep(5) # the periodic check should have kick in
+  end
+
+  after do
+    subject.close
+  end
+
+  it "create one file" do
+    # using close will upload the current file
+    try(20) do
+      expect(bucket_resource.objects(:prefix => prefix).count).to eq(1)
+    end
+  end
+
+  it "Persists all events" do
+    download_directory = Stud::Temporary.pathname
+
+    FileUtils.rm_rf(download_directory)
+    FileUtils.mkdir_p(download_directory)
+
+    counter = 0
+    bucket_resource.objects(:prefix => prefix).each do |object|
+      target = File.join(download_directory, "#{counter}.txt")
+      object.get(:response_target => target)
+      counter += 1
+    end
+
+    try(20) do
+      expect(Dir.glob(File.join(download_directory, "**", "*.txt")).inject(0) { |sum, f| sum + IO.readlines(f).size }).to eq(number_of_events)
+    end
+
+    try(10) do
+      expect(Dir.glob(File.join(temporary_directory, "**", "*.txt")).size).to eq(1) # we should only have 1 file left, since we did a rotation
+    end
+  end
+end

data/spec/integration/upload_current_file_on_shutdown_spec.rb

@@ -0,0 +1,51 @@
+# encoding: utf-8
+require_relative "../spec_helper"
+require "logstash/outputs/s3"
+require "logstash/codecs/line"
+require "stud/temporary"
+
+describe "Upload current file on shutdown", :integration => true do
+  include_context "setup plugin"
+  let(:options) { main_options }
+
+  let(:size_file) { 1000000 }
+  let(:time_file) { 100000 }
+  let(:number_of_events) { 5000 }
+  let(:batch_size) { 125 }
+  let(:event_encoded) { "Hello world" }
+  let(:batch) do
+    b = {}
+    number_of_events.times do
+      event = LogStash::Event.new({ "message" => event_encoded })
+      b[event] = "#{event_encoded}\n"
+    end
+    b
+  end
+
+  before do
+    clean_remote_files(prefix)
+    subject.register
+    subject.multi_receive_encoded(batch)
+    subject.close
+  end
+
+  it "creates a specific quantity of files" do
+    # Since we have really big value of time_file and size_file
+    expect(bucket_resource.objects(:prefix => prefix).count).to eq(1)
+  end
+
+  it "Persists all events" do
+    download_directory = Stud::Temporary.pathname
+
+    FileUtils.rm_rf(download_directory)
+    FileUtils.mkdir_p(download_directory)
+
+    counter = 0
+    bucket_resource.objects(:prefix => prefix).each do |object|
+      target = File.join(download_directory, "#{counter}.txt")
+      object.get(:response_target => target)
+      counter += 1
+    end
+    expect(Dir.glob(File.join(download_directory, "**", "*.txt")).inject(0) { |sum, f| sum + IO.readlines(f).size }).to eq(number_of_events)
+  end
+end

data/spec/outputs/cloudwatch_spec.rb

@@ -0,0 +1,38 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/cloudwatch"
+
+describe "outputs/cloudwatch" do
+  
+  let(:config) { { 'metricname' => 'foo' } }
+
+  subject(:plugin) { LogStash::Outputs::CloudWatch.new(config) }
+
+  it "should register" do
+    expect { plugin.register }.to_not raise_error
+  end
+
+  it "should respond correctly to a receive call" do
+    plugin.register
+    event = LogStash::Event.new
+    expect { plugin.receive(event) }.to_not raise_error
+  end
+
+  context 'with queue_size' do
+
+    let(:queue_size) { 100 }
+
+    let(:config) { super().merge('queue_size' => queue_size) }
+
+    it "triggers job ahead of time" do
+      plugin.register
+      event_queue = plugin.event_queue
+      allow( event_queue ).to receive(:length).and_return queue_size # emulate full queue
+      expect( plugin ).to receive(:publish)
+
+      event = LogStash::Event.new
+      plugin.receive(event)
+      sleep 1.0 # allow scheduler to kick in
+    end
+
+  end
+end