logstash-output-cassandra 0.9.0

data/logstash-output-cassandra.gemspec

@@ -0,0 +1,35 @@
+Gem::Specification.new do |s|
+
+  s.name            = 'logstash-output-cassandra'
+  s.version         = '0.9.0'
+  s.licenses        = [ 'Apache License (2.0)' ]
+  s.summary         = 'Store events into Cassandra'
+  s.description     = 'This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program'
+  s.authors         = [ 'PerimeterX' ]
+  s.email           = [ 'elad@perimeterx.com' ]
+  s.homepage        = 'https://github.com/PerimeterX/logstash-output-cassandra'
+  s.require_paths   = [ 'lib' ]
+
+  # Files
+  s.files = Dir[ 'lib/**/*', 'spec/**/*', 'vendor/**/*', '*.gemspec', '*.md', 'CONTRIBUTORS', 'Gemfile', 'LICENSE', 'NOTICE.TXT' ]
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { 'logstash_plugin' => 'true', 'logstash_group' => 'output' }
+
+  # Gem dependencies
+  s.add_runtime_dependency 'concurrent-ruby'
+  s.add_runtime_dependency 'logstash-core', '>= 2.0.0', '< 3.0.0'
+  s.add_runtime_dependency 'cassandra-driver', '>= 2.0.0', '< 3.0.0'
+  s.add_development_dependency 'cabin', ['~> 0.6']
+  s.add_development_dependency 'longshoreman'
+  s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'logstash-codec-plain'
+  s.add_development_dependency 'simplecov'
+  s.add_development_dependency 'simplecov-rcov'
+  s.add_development_dependency 'unparser', '0.2.4'
+  s.add_development_dependency 'metric_fu'
+  s.add_development_dependency 'coveralls'
+  s.add_development_dependency 'gems'
+end

data/spec/cassandra_spec_helper.rb

@@ -0,0 +1,14 @@
+# encoding: utf-8
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/event'
+require 'simplecov'
+require 'simplecov-rcov'
+
+SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter.new([
+  SimpleCov::Formatter::HTMLFormatter,
+  SimpleCov::Formatter::RcovFormatter
+])
+
+SimpleCov.start do
+  add_filter '/spec/'
+end

data/spec/integration/outputs/cassandra_spec.rb

@@ -0,0 +1,115 @@
+# encoding: utf-8
+require_relative './integration_helper'
+require 'logstash/outputs/cassandra'
+
+module Helper
+  def self.get_assert_timestamp_equallity
+    Proc.new do |expect, row, type_to_test|
+      expect.call(row['value_column'].to_s).to(eq(Time.at(type_to_test[:value]).to_s))
+    end
+  end
+
+  def self.get_assert_set_equallity
+    Proc.new do |expect, row, type_to_test|
+      set_from_cassandra = row['value_column']
+      original_value = type_to_test[:value]
+      expect.call(set_from_cassandra.size).to(eq(original_value.size))
+      set_from_cassandra.to_a.each { |item|
+        expect.call(original_value).to(include(item.to_s))
+      }
+    end
+  end
+end
+
+describe 'client create actions', :docker => true do
+  before(:each) do
+    get_session.execute("CREATE KEYSPACE test WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 };")
+  end
+
+  after(:each) do
+    get_session.execute('DROP KEYSPACE test;')
+  end
+
+  def get_sut
+    options = {
+      'hosts' => [get_host_ip],
+      'port' => get_port,
+      'keyspace' => 'test',
+      'table' => '%{[cassandra_table]}',
+      'username' => 'cassandra',
+      'password' => 'cassandra',
+      'filter_transform_event_key' => 'cassandra_filter'
+    }
+    sut = LogStash::Outputs::CassandraOutput.new(options)
+    return sut
+  end
+
+  def create_table(type_to_test)
+    get_session.execute("
+      CREATE TABLE test.simple(
+        idish_column text,
+        value_column #{type_to_test[:type]},
+        PRIMARY KEY  (idish_column)
+      );")
+  end
+
+  def build_event(type_to_test)
+    options = {
+        'cassandra_table' => 'simple',
+        'idish_field' => 'some text',
+        'value_field' => type_to_test[:value],
+        'cassandra_filter' => [
+            { 'event_key' => 'idish_field', 'column_name' => 'idish_column' },
+            { 'event_key' => 'value_field', 'column_name' => 'value_column', 'cassandra_type' => type_to_test[:type] }
+        ]
+    }
+    LogStash::Event.new(options)
+  end
+
+  def assert_proper_insert(type_to_test)
+    result = get_session.execute('SELECT * FROM test.simple')
+    expect(result.size).to((eq(1)))
+    result.each { |row|
+      expect(row['idish_column']).to(eq('some text'))
+      if type_to_test.has_key?(:assert_override)
+        expect_proc = Proc.new do |value|
+            return expect(value)
+        end
+        type_to_test[:assert_override].call(expect_proc, row, type_to_test)
+      else
+        expect(row['value_column'].to_s).to(eq(type_to_test[:value].to_s))
+      end
+    }
+  end
+
+  [
+      { type: 'timestamp', value: 1457606758, assert_override: Helper::get_assert_timestamp_equallity() },
+      { type: 'inet', value: '192.168.99.100' },
+      { type: 'float', value: '10.050000190734863' },
+      { type: 'varchar', value: 'some chars' },
+      { type: 'text', value: 'some text' },
+      { type: 'blob', value: 'a blob' },
+      { type: 'ascii', value: 'some ascii' },
+      { type: 'bigint', value: '123456789' },
+      { type: 'int', value: '12345' },
+      { type: 'varint', value: '12345678' },
+      { type: 'boolean', value: 'true' },
+      { type: 'decimal', value: '0.1015E2' },
+      { type: 'double', value: '200.54' },
+      { type: 'timeuuid', value: 'd2177dd0-eaa2-11de-a572-001b779c76e3' },
+      { type: 'set<timeuuid>',
+        value: %w(d2177dd0-eaa2-11de-a572-001b779c76e3 d2177dd0-eaa2-11de-a572-001b779c76e4 d2177dd0-eaa2-11de-a572-001b779c76e5), assert_override: Helper::get_assert_set_equallity }
+  ].each { |type_to_test|
+    it "properly inserts data of type #{type_to_test[:type]}" do
+      create_table(type_to_test)
+      sut = get_sut
+      sut.register
+      event = build_event(type_to_test)
+
+      sut.receive(event)
+      sut.flush
+
+      assert_proper_insert(type_to_test)
+    end
+  }
+end

data/spec/integration/outputs/integration_helper.rb

@@ -0,0 +1,91 @@
+# encoding: utf-8
+require_relative '../../cassandra_spec_helper'
+require 'longshoreman'
+require 'cassandra'
+
+CONTAINER_NAME = "logstash-output-cassandra-#{rand(999).to_s}"
+CONTAINER_IMAGE = 'cassandra'
+CONTAINER_TAG = '2.2'
+
+module CassandraHelper
+  def get_host_ip
+    Longshoreman.new.get_host_ip
+  end
+
+  def get_port
+    container = Longshoreman::Container.new
+    container.get(CONTAINER_NAME)
+    container.rport(9042)
+  end
+
+  def get_session
+    cluster = ::Cassandra.cluster(
+        username: 'cassandra',
+        password: 'cassandra',
+        port: get_port,
+        hosts: [get_host_ip]
+    )
+    cluster.connect
+  end
+end
+
+
+RSpec.configure do |config|
+  config.include CassandraHelper
+
+  # this :all hook gets run before every describe block that is tagged with :integration => true.
+  config.before(:all, :docker => true) do
+    # check if container exists already before creating new one.
+    begin
+      ls = Longshoreman::new
+      ls.container.get(CONTAINER_NAME)
+    rescue Docker::Error::NotFoundError
+      create_retry = 0
+      begin
+        Longshoreman.new("#{CONTAINER_IMAGE}:#{CONTAINER_TAG}", CONTAINER_NAME, {
+            'HostConfig' => {
+                'PublishAllPorts' => true
+            }
+        })
+        connect_retry = 0
+        begin
+          get_session
+        rescue ::Cassandra::Errors::NoHostsAvailable
+          # retry connecting for a minute
+          connect_retry += 1
+          if connect_retry <= 60
+            sleep(1)
+            retry
+          else
+            raise
+          end
+        end
+      rescue Docker::Error::NotFoundError
+        # try to pull the image once if it does not exist
+        create_retry += 1
+        if create_retry <= 1
+          Longshoreman.pull_image(CONTAINER_IMAGE, CONTAINER_TAG)
+          retry
+        else
+          raise
+        end
+      end
+    end
+  end
+
+  # we want to do a final cleanup after all :integration runs,
+  # but we don't want to clean up before the last block.
+  # This is a final blind check to see if the ES docker container is running and
+  # needs to be cleaned up. If no container can be found and/or docker is not
+  # running on the system, we do nothing.
+  config.after(:suite) do
+    # only cleanup docker container if system has docker and the container is running
+    begin
+      ls = Longshoreman::new
+      ls.container.get(CONTAINER_NAME)
+      ls.cleanup
+    rescue Docker::Error::NotFoundError, Excon::Errors::SocketError
+      # do nothing
+    end
+  end
+end

data/spec/unit/outputs/backoff_retry_policy_spec.rb

@@ -0,0 +1,131 @@
+# encoding: utf-8
+require_relative '../../cassandra_spec_helper'
+require 'logstash/outputs/cassandra/backoff_retry_policy'
+
+RSpec.describe ::Cassandra::Retry::Policies::Backoff do
+  let(:sut) { ::Cassandra::Retry::Policies::Backoff }
+  let(:linear_backoff) {
+    logger = double
+    allow(logger).to(receive(:error))
+    {
+      'logger' => logger,
+      'backoff_type' => '*',
+      'backoff_size' => 5,
+      'retry_limit' => 10
+    }
+  }
+  let(:exponential_backoff) {
+    linear_backoff.merge({
+      'backoff_type' => '**',
+      'backoff_size' => 2,
+      'retry_limit' => 10
+    })
+  }
+
+  describe '#retry_with_backoff' do
+    describe 'retry limit not reached' do
+      it 'decides to try again with the same consistency level' do
+        sut_instance = sut.new(linear_backoff)
+
+        decision = sut_instance.retry_with_backoff({ :retries => 0, :consistency => :one })
+
+        expect(decision).to(be_an_instance_of(::Cassandra::Retry::Decisions::Retry))
+        expect(decision.consistency).to(be(:one))
+      end
+
+      it 'waits _before_ retrying' do
+        sut_instance = sut.new(linear_backoff)
+        expect(Kernel).to(receive(:sleep))
+
+        sut_instance.retry_with_backoff({ :retries => 0 })
+      end
+
+      it 'allows for an infinite amount of retries if configured with -1 as the retry limit' do
+        sut_instance = sut.new(linear_backoff.merge({ 'retry_limit' => -1 }))
+        expect(Kernel).to(receive(:sleep))
+
+        sut_instance.retry_with_backoff({ :retries => 1000000 })
+      end
+
+      it 'allows for exponential backoffs' do
+        sut_instance = sut.new(exponential_backoff)
+        test_retry = exponential_backoff['retry_limit'] - 1
+        expect(Kernel).to(receive(:sleep).with(exponential_backoff['backoff_size'] ** test_retry))
+
+        sut_instance.retry_with_backoff({ :retries => test_retry })
+      end
+
+      it 'allows for linear backoffs' do
+        sut_instance = sut.new(linear_backoff)
+        test_retry = exponential_backoff['retry_limit'] - 1
+        expect(Kernel).to(receive(:sleep).with(linear_backoff['backoff_size'] * test_retry))
+
+        sut_instance.retry_with_backoff({ :retries => test_retry })
+      end
+
+      it 'fails for unknown backoff types' do
+        sut_instance = sut.new(linear_backoff.merge({ 'backoff_type' => '^' }))
+
+        expect { sut_instance.retry_with_backoff({ :retries => 0}) }.to raise_error ArgumentError
+      end
+    end
+
+    describe 'retry limit reached' do
+      it 'decides to reraise' do
+        sut_instance = sut.new(linear_backoff)
+
+        decision = sut_instance.retry_with_backoff({ :retries => linear_backoff['retry_limit'] + 1 })
+
+        expect(decision).to(be_an_instance_of(::Cassandra::Retry::Decisions::Reraise))
+      end
+
+      it 'does not wait' do
+        sut_instance = sut.new(linear_backoff)
+
+        expect(Kernel).not_to(receive(:sleep))
+
+        sut_instance.retry_with_backoff({ :retries => linear_backoff['retry_limit'] + 1 })
+      end
+    end
+  end
+
+  [
+      {
+          :method_name=> 'read_timeout',
+          :expected_opts => { :statement => 'statement', :consistency => :one, :required => 1, :received => 0,
+                              :retrieved => false, :retries => 0 },
+          :call_args => ['statement', :one, 1, 0, false, 0]
+      },
+      {
+          :method_name=> 'write_timeout',
+          :expected_opts => { :statement => 'statement', :consistency => :one, :type => :prepared,
+                              :required => 1, :received => 2, :retries => 5 },
+          :call_args => ['statement', :one, :prepared, 1, 2, 5]
+      },
+      {
+          :method_name=> 'unavailable',
+          :expected_opts => { :statement => 'statement', :consistency => :one, :required => 3,
+                              :alive => 2, :retries => 4},
+          :call_args => ['statement', :one, 3, 2, 4]
+      }
+  ].each { |use_case|
+    describe '#{use_case[:method_name]}' do
+      it 'properly calls #retry_with_backoff' do
+        sut_instance = sut.new(linear_backoff)
+        expect(sut_instance).to(receive(:retry_with_backoff).with(use_case[:expected_opts]))
+
+        sut_instance.send(use_case[:method_name], *use_case[:call_args])
+      end
+
+      it 'returns the decision it got' do
+        sut_instance = sut.new(linear_backoff)
+        expected_result = double
+        expect(sut_instance).to(receive(:retry_with_backoff).and_return(expected_result))
+
+        result = sut_instance.send(use_case[:method_name], *use_case[:call_args])
+
+        expect(result).to(be(expected_result))
+      end
+    end
+  }
+end

data/spec/unit/outputs/buffer_spec.rb

@@ -0,0 +1,119 @@
+# encoding: utf-8
+require "logstash/outputs/cassandra/buffer"
+require "cabin"
+
+describe LogStash::Outputs::Cassandra::Buffer do
+  class OperationTarget # Used to track buffer flushesn
+    attr_reader :buffer, :buffer_history, :receive_count
+    def initialize
+      @buffer = nil
+      @buffer_history = []
+      @receive_count = 0
+    end
+
+    def receive(buffer)
+      @receive_count += 1
+      @buffer_history << buffer.clone
+      @buffer = buffer
+    end
+  end
+
+  let(:logger) { Cabin::Channel.get }
+  let(:max_size) { 10 }
+  let(:flush_interval) { 2 }
+  # Used to track flush count
+  let(:operation_target) { OperationTarget.new() }
+  let(:operation) { proc {|buffer| operation_target.receive(buffer) } }
+  subject(:buffer){ LogStash::Outputs::Cassandra::Buffer.new(logger, max_size, flush_interval, &operation) }
+
+  after(:each) do
+    buffer.stop(do_flush=false)
+  end
+
+  it "should initialize cleanly" do
+    expect(buffer).to be_a(LogStash::Outputs::Cassandra::Buffer)
+  end
+
+  shared_examples("a buffer with two items inside") do
+    it "should add a pushed item to the buffer" do
+      buffer.synchronize do |data|
+        expect(data).to include(item1)
+        expect(data).to include(item2)
+      end
+    end
+
+    describe "interval flushing" do
+      before do
+        sleep flush_interval + 1
+      end
+
+      it "should flush the buffer after the interval has passed" do
+        expect(operation_target.receive_count).to eql(1)
+      end
+
+      it "should clear the buffer after a successful flush" do
+        expect(operation_target.buffer).to eql([])
+      end
+    end
+
+    describe "interval flushing a stopped buffer" do
+      before do
+        buffer.stop(do_flush=false)
+        sleep flush_interval + 1
+      end
+
+      it "should not flush if the buffer is stopped" do
+        expect(operation_target.receive_count).to eql(0)
+      end
+    end
+  end
+
+  describe "with a buffer push" do
+    let(:item1) { "foo" }
+    let(:item2) { "bar" }
+
+    describe "a buffer with two items pushed to it separately" do
+      before do
+        buffer << item1
+        buffer << item2
+      end
+
+      include_examples("a buffer with two items inside")
+    end
+
+    describe "a buffer with two items pushed to it in one operation" do
+      before do
+        buffer.push_multi([item1, item2])
+      end
+
+      include_examples("a buffer with two items inside")
+    end
+  end
+
+  describe "with an empty buffer" do
+    it "should not perform an operation if the buffer is empty" do
+      buffer.flush
+      expect(operation_target.receive_count).to eql(0)
+    end
+  end
+
+  describe "flushing with an operation that raises an error" do
+    class TestError < StandardError; end
+    let(:operation) { proc {|buffer| raise TestError, "A test" } }
+    let(:item) { double("item") }
+
+    before do
+      buffer << item
+    end
+
+    it "should raise an exception" do
+      expect { buffer.flush }.to raise_error(TestError)
+    end
+
+    it "should not clear the buffer" do
+      expect do
+        buffer.flush rescue TestError
+      end.not_to change(buffer, :contents)
+    end
+  end
+end