logstash-output-cassandra 0.9.0

data/spec/unit/outputs/cassandra_spec.rb

@@ -0,0 +1,5 @@
+# encoding: utf-8
+require_relative "../../cassandra_spec_helper"
+require "logstash/outputs/cassandra"
+
+# TODO: add unit tests for the main cassandra output class

data/spec/unit/outputs/event_parser_spec.rb

@@ -0,0 +1,304 @@
+# encoding: utf-8
+require_relative '../../cassandra_spec_helper'
+require 'logstash/outputs/cassandra/event_parser'
+
+RSpec.describe LogStash::Outputs::Cassandra::EventParser do
+  let(:sut) { LogStash::Outputs::Cassandra::EventParser }
+  let(:default_opts) {
+    logger = double
+    allow(logger).to(receive(:debug))
+    return {
+      'logger' => logger,
+      'table' => 'dummy',
+      'filter_transform_event_key' => nil,
+      'filter_transform' => [],
+      'hints' => {},
+      'ignore_bad_values' => false
+    }
+  }
+  let(:sample_event) { LogStash::Event.new('message' => 'sample message here') }
+
+  describe 'table name parsing' do
+    it 'leaves regular table names unchanged' do
+      sut_instance = sut.new(default_opts.update({ 'table' => 'simple' }))
+
+      action = sut_instance.parse(sample_event)
+
+      expect(action['table']).to(eq('simple'))
+    end
+
+    it 'allows for string expansion in table names' do
+      sut_instance = sut.new(default_opts.update({ 'table' => '%{[a_field]}' }))
+      sample_event['a_field'] = 'a_value'
+
+      action = sut_instance.parse(sample_event)
+
+      expect(action['table']).to(eq('a_value'))
+    end
+  end
+
+  describe 'filter transforms' do
+    describe 'from config' do
+      describe 'malformed configurations' do
+        it 'fails if the transform has no event_data setting' do
+          expect { sut.new(default_opts.update({ 'filter_transform' => [{ 'column_name' => '' }] })) }.to raise_error(/item is incorrectly configured/)
+        end
+
+        it 'fails if the transform has no column_name setting' do
+          expect { sut.new(default_opts.update({ 'filter_transform' => [{ 'event_key' => '' }] })) }.to raise_error(/item is incorrectly configured/)
+        end
+      end
+
+      describe 'properly configured' do
+        it 'maps the event key to the column' do
+          sut_instance = sut.new(default_opts.update({ 'filter_transform' => [{ 'event_key' => 'a_field', 'column_name' => 'a_column' }] }))
+          sample_event['a_field'] = 'a_value'
+
+          action = sut_instance.parse(sample_event)
+
+          expect(action['data']['a_column']).to(eq('a_value'))
+        end
+
+        it 'works with multiple filter transforms' do
+          sut_instance = sut.new(default_opts.update({ 'filter_transform' => [{ 'event_key' => 'a_field', 'column_name' => 'a_column' }, { 'event_key' => 'another_field', 'column_name' => 'a_different_column' }] }))
+          sample_event['a_field'] = 'a_value'
+          sample_event['another_field'] = 'a_second_value'
+
+          action = sut_instance.parse(sample_event)
+
+          expect(action['data']['a_column']).to(eq('a_value'))
+          expect(action['data']['a_different_column']).to(eq('a_second_value'))
+        end
+
+        it 'allows for string expansion in event keys' do
+          sut_instance = sut.new(default_opts.update({ 'filter_transform' => [{ 'event_key' => '%{[pointer_to_another_field]}', 'column_name' => 'a_column' }] }))
+          sample_event['pointer_to_another_field'] = 'another_field'
+          sample_event['another_field'] = 'a_value'
+
+          action = sut_instance.parse(sample_event)
+
+          expect(action['data']['a_column']).to(eq('a_value'))
+        end
+
+        it 'allows for string expansion only filters for things like date string formats' do
+          sut_instance = sut.new(default_opts.update({ 'filter_transform' => [{ 'event_key' => '%{+yyyyMMddHHmm}', 'expansion_only' => true, 'column_name' => 'a_column' }] }))
+          expected_value = Time.now.getutc.strftime('%Y%m%d%H%M')
+
+          action = sut_instance.parse(sample_event)
+
+          expect(action['data']['a_column']).to(eq(expected_value))
+        end
+
+        it 'allows for string expansion in column names' do
+          sut_instance = sut.new(default_opts.update({ 'filter_transform' => [{ 'event_key' => 'a_field', 'column_name' => '%{[pointer_to_another_field]}' }] }))
+          sample_event['a_field'] = 'a_value'
+          sample_event['pointer_to_another_field'] = 'a_different_column'
+
+          action = sut_instance.parse(sample_event)
+
+          expect(action['data']['a_different_column']).to(eq('a_value'))
+        end
+
+      end
+
+      describe 'cassandra type mapping' do
+        [
+          { :name => 'timestamp', :type => ::Cassandra::Types::Timestamp, :value => Time::parse('1979-07-27 00:00:00 +0300') },
+          { :name => 'timestamp', :type => ::Cassandra::Types::Timestamp, :value => '1982-05-04 00:00:00 +0300', expected: Time::parse('1982-05-04 00:00:00 +0300') },
+          { :name => 'timestamp', :type => ::Cassandra::Types::Timestamp, :value => 1457606758, expected: Time.at(1457606758) },
+          { :name => 'inet',      :type => ::Cassandra::Types::Inet,      :value => '0.0.0.0' },
+          { :name => 'float',     :type => ::Cassandra::Types::Float,     :value => '10.15' },
+          { :name => 'varchar',   :type => ::Cassandra::Types::Varchar,   :value => 'a varchar' },
+          { :name => 'text',      :type => ::Cassandra::Types::Text,      :value => 'some text' },
+          { :name => 'blob',      :type => ::Cassandra::Types::Blob,      :value => '12345678' },
+          { :name => 'ascii',     :type => ::Cassandra::Types::Ascii,     :value => 'some ascii' },
+          { :name => 'bigint',    :type => ::Cassandra::Types::Bigint,    :value => '100' },
+          { :name => 'counter',   :type => ::Cassandra::Types::Counter,   :value => '15' },
+          { :name => 'int',       :type => ::Cassandra::Types::Int,       :value => '123' },
+          { :name => 'varint',    :type => ::Cassandra::Types::Varint,    :value => '345' },
+          { :name => 'boolean',   :type => ::Cassandra::Types::Boolean,   :value => 'true' },
+          { :name => 'decimal',   :type => ::Cassandra::Types::Decimal,   :value => '0.12E2' },
+          { :name => 'double',    :type => ::Cassandra::Types::Double,    :value => '123.65' },
+          { :name => 'timeuuid',  :type => ::Cassandra::Types::Timeuuid,  :value => '00000000-0000-0000-0000-000000000000' }
+        ].each { |mapping|
+          # NOTE: this is not the best test there is, but it is the best / simplest I could think of :/
+          it "properly maps #{mapping[:name]} to #{mapping[:type]}" do
+            sut_instance = sut.new(default_opts.update({ 'filter_transform' => [{ 'event_key' => 'a_field', 'column_name' => 'a_column', 'cassandra_type' => mapping[:name] }] }))
+            sample_event['a_field'] = mapping[:value]
+
+            action = sut_instance.parse(sample_event)
+
+            expected_value = mapping.has_key?(:expected) ? mapping[:expected] : mapping[:value]
+            expect(action['data']['a_column'].to_s).to(eq(expected_value.to_s))
+          end
+        }
+
+        it 'properly maps sets to their specific set types' do
+          sut_instance = sut.new(default_opts.update({ 'filter_transform' => [{ 'event_key' => 'a_field', 'column_name' => 'a_column', 'cassandra_type' => 'set<int>' }] }))
+          original_value = [ 1, 2, 3 ]
+          sample_event['a_field'] = original_value
+
+          action = sut_instance.parse(sample_event)
+
+          expect(action['data']['a_column'].to_a).to(eq(original_value))
+        end
+
+        it 'properly maps sets to their specific set types for type which also require actual conversion' do
+          sut_instance = sut.new(default_opts.update({ 'filter_transform' => [{ 'event_key' => 'a_field', 'column_name' => 'a_column', 'cassandra_type' => 'set<timeuuid>' }] }))
+          original_value = %w(00000000-0000-0000-0000-000000000000 00000000-0000-0000-0000-000000000001 00000000-0000-0000-0000-000000000002)
+          sample_event['a_field'] = original_value
+
+          action = sut_instance.parse(sample_event)
+
+          expect(action['data']['a_column'].size).to(eq(original_value.size))
+          action['data']['a_column'].to_a.each { |item|
+            expect(original_value).to(include(item.to_s))
+          }
+        end
+
+        it 'allows for string expansion in cassandra types' do
+          sut_instance = sut.new(default_opts.update({ 'filter_transform' => [{ 'event_key' => 'a_field', 'column_name' => 'a_column', 'cassandra_type' => '%{[pointer_to_a_field]}' }] }))
+          sample_event['a_field'] = '123'
+          sample_event['pointer_to_a_field'] = 'int'
+
+          action = sut_instance.parse(sample_event)
+
+          expect(action['data']['a_column']).to(eq(123))
+        end
+
+        it 'fails in case of an unknown type' do
+          options = default_opts.update({ 'filter_transform' => [{ 'event_key' => 'a_field', 'column_name' => 'a_column', 'cassandra_type' => 'what?!' }] })
+          sut_instance = sut.new(options)
+          sample_event['a_field'] = 'a_value'
+          expect(options['logger']).to(receive(:error))
+
+          expect { sut_instance.parse(sample_event) }.to raise_error(/Cannot convert/)
+        end
+      end
+    end
+
+    describe 'from event' do
+      it 'obtains the filter transform from the event if defined' do
+        sut_instance = sut.new(default_opts.update({ 'filter_transform_event_key' => 'an_event_filter' }))
+        sample_event['a_field'] = 'a_value'
+        sample_event['an_event_filter'] = [{ 'event_key' => 'a_field', 'column_name' => 'a_column' }]
+
+        action = sut_instance.parse(sample_event)
+
+        expect(action['data']['a_column']).to(eq('a_value'))
+      end
+
+      it 'obtains the filter transform from the event even when it is in the metadata' do
+        sut_instance = sut.new(default_opts.update({ 'filter_transform_event_key' => '[@metadata][the_filter]' }))
+        sample_event['a_field'] = 'a_value'
+        sample_event['@metadata'] = { 'the_filter' => [{ 'event_key' => 'a_field', 'column_name' => 'a_column' }] }
+
+        action = sut_instance.parse(sample_event)
+
+        expect(action['data']['a_column']).to(eq('a_value'))
+      end
+    end
+  end
+
+  describe 'hints' do
+    it 'removes fields starting with @' do
+      sut_instance = sut.new(default_opts.update({ 'hints' => {} }))
+      sample_event['leave'] = 'a_value'
+      sample_event['@remove'] = 'another_value'
+
+      action = sut_instance.parse(sample_event)
+
+      expect(action['data']['leave']).to(eq('a_value'))
+      expect(action['data']).not_to(include('@remove'))
+    end
+
+    it 'does not attempt to change items with no hints' do
+      sut_instance = sut.new(default_opts.update({ 'hints' => {} }))
+      expected_value = [ 1, 2, 3 ]
+      sample_event['no_hint_here'] = expected_value
+
+      action = sut_instance.parse(sample_event)
+
+      expect(action['data']['no_hint_here']).to(equal(expected_value))
+    end
+
+    it 'converts items with hints' do
+      sut_instance = sut.new(default_opts.update({ 'hints' => { 'a_set' => 'set<int>', 'an_int' => 'int' } }))
+      original_set = [ 1, 2, 3 ]
+      sample_event['a_set'] = original_set
+      sample_event['an_int'] = '123'
+
+      action = sut_instance.parse(sample_event)
+
+      expect(action['data']['a_set']).to(be_a(Set))
+      expect(action['data']['a_set'].to_a).to(eql(original_set))
+      expect(action['data']['an_int']).to(eql(123))
+    end
+
+    it 'fails for unknown hint types' do
+      options = default_opts.update({ 'hints' => { 'a_field' => 'not_a_real_type' } })
+      sut_instance = sut.new(options)
+      expect(options['logger']).to(receive(:error))
+
+      sample_event['a_field'] = 'a value'
+
+      expect { sut_instance.parse(sample_event) }.to raise_error(/Cannot convert/)
+    end
+
+    it 'fails for unsuccessful hint conversion' do
+      options = default_opts.update({ 'hints' => { 'a_field' => 'int' } })
+      expect(options['logger']).to(receive(:error))
+
+      sut_instance = sut.new(options)
+
+      sample_event['a_field'] = 'i am not an int!!!'
+      expect { sut_instance.parse(sample_event) }.to raise_error(/Cannot convert/)
+    end
+  end
+
+  describe 'ignore_bad_values is turned on' do
+    [
+        { :name => 'timestamp', :value => 'i dont have to_time',      :expected => Time::parse('1970-01-01 00:00:00 +0000') },
+        { :name => 'inet',      :value => 'i am not an inet address', :expected => '0.0.0.0' },
+        { :name => 'float',     :value => 'i am not a float',         :expected => 0.0 },
+        { :name => 'bigint',    :value => 'i am not a bigint',        :expected => 0 },
+        { :name => 'counter',   :value => 'i am not a counter',       :expected => 0 },
+        { :name => 'int',       :value => 'i am not a int',           :expected => 0 },
+        { :name => 'varint',    :value => 'i am not a varint',        :expected => 0 },
+        { :name => 'double',    :value => 'i am not a double',        :expected => 0.0 },
+        { :name => 'timeuuid',  :value => 'i am not a timeuuid',      :expected => '00000000-0000-0000-0000-000000000000' }
+    ].each { |mapping|
+      # NOTE: this is not the best test there is, but it is the best / simplest I could think of :/
+      it "properly defaults #{mapping[:name]}" do
+        options = default_opts.update({ 'ignore_bad_values' => true, 'hints' => { 'a_field' => mapping[:name] } })
+        expect(options['logger']).to(receive(:warn))
+        sut_instance = sut.new(options)
+        sample_event['a_field'] = mapping[:value]
+
+        action = sut_instance.parse(sample_event)
+
+        expect(action['data']['a_field'].to_s).to(eq(mapping[:expected].to_s))
+      end
+    }
+
+    it 'properly default sets' do
+      options = default_opts.update({ 'ignore_bad_values' => true, 'hints' => { 'a_field' => 'set<float>' } })
+      expect(options['logger']).to(receive(:warn))
+      sut_instance = sut.new(options)
+      sample_event['a_field'] = 'i am not a set'
+
+      action = sut_instance.parse(sample_event)
+
+      expect(action['data']['a_field'].size).to(be(0))
+      expect(action['data']['a_field']).to(be_an_instance_of(Set))
+    end
+
+    it 'raises an ArgumentError in case we try to default a type we dont know' do
+      options = default_opts.update({ 'ignore_bad_values' => true, 'hints' => { 'a_field' => 'map<float>' } })
+      sut_instance = sut.new(options)
+      sample_event['a_field'] = 'i am not a set'
+
+      expect { sut_instance.parse(sample_event) }.to raise_error ArgumentError
+    end
+  end
+end

data/spec/unit/outputs/safe_submitter_spec.rb

@@ -0,0 +1,201 @@
+# encoding: utf-8
+require_relative '../../cassandra_spec_helper'
+require 'logstash/outputs/cassandra/safe_submitter'
+
+RSpec.describe LogStash::Outputs::Cassandra::SafeSubmitter do
+  let(:sut) { LogStash::Outputs::Cassandra::SafeSubmitter }
+  let(:default_options) {
+    logger = double
+    allow(logger).to(receive(:debug))
+    allow(logger).to(receive(:error))
+    {
+      'logger' => logger,
+      'cassandra' => double,
+      'username' => 'a user',
+      'password' => 'a password',
+      'protocol_version' => 3,
+      'hosts' => 'some host',
+      'port' => 9042,
+      'consistency' => 'one',
+      'request_timeout' => 10,
+      'retry_policy' => { 'type' => 'default' },
+      'concrete_retry_policy' => ::Cassandra::Retry::Policies::Default,
+      'keyspace' => 'the final frontier'
+    }
+  }
+
+  def setup_session_double(options)
+    session_double = double
+    cluster_double = double
+    expect(cluster_double).to(receive(:connect)).with(options['keyspace']).and_return(session_double)
+    expect(options['cassandra']).to(receive(:cluster).with(
+      username: options['username'],
+      password: options['password'],
+      protocol_version: options['protocol_version'],
+      hosts: options['hosts'],
+      port: options['port'],
+      consistency: options['consistency'].to_sym,
+      timeout: options['request_timeout'],
+      retry_policy: options['concrete_retry_policy'],
+      logger: options['logger']
+    )).and_return(cluster_double)
+    return { :session_double => session_double }
+  end
+
+  describe 'init' do
+    it 'properly inits the cassandra session' do
+      setup_session_double(default_options)
+
+      sut.new(default_options)
+    end
+
+    [
+        { :setting => { 'type' => 'default' },                 :concrete_retry_policy => ::Cassandra::Retry::Policies::Default },
+        { :setting => { 'type' => 'downgrading_consistency' }, :concrete_retry_policy => ::Cassandra::Retry::Policies::DowngradingConsistency },
+        { :setting => { 'type' => 'failthrough' },             :concrete_retry_policy => ::Cassandra::Retry::Policies::Fallthrough },
+        { :setting => { 'type' => 'backoff', 'backoff_type' => '**', 'backoff_size' => 2, 'retry_limit' => 10 },
+                                                               :concrete_retry_policy => ::Cassandra::Retry::Policies::Backoff }
+    ].each { |mapping|
+      it "supports the #{mapping[:concrete_retry_policy]} retry policy by passing #{mapping[:setting]['type']} as the retry_policy" do
+        options = default_options.merge({ 'retry_policy' => mapping[:setting], 'concrete_retry_policy' => mapping[:concrete_retry_policy] })
+        setup_session_double(options)
+
+        sut.new(options)
+      end
+    }
+
+    it 'properly initializes the backoff retry policy' do
+      retry_policy_config = { 'type' => 'backoff', 'backoff_type' => '**', 'backoff_size' => 2, 'retry_limit' => 10 }
+      expected_policy = double
+      options = default_options.merge({ 'retry_policy' => retry_policy_config, 'concrete_retry_policy' => expected_policy })
+      expect(::Cassandra::Retry::Policies::Backoff).to(receive(:new).with({
+        'backoff_type' => options['retry_policy']['backoff_type'], 'backoff_size' => options['retry_policy']['backoff_size'],
+        'retry_limit' => options['retry_policy']['retry_limit'], 'logger' =>  options['logger']}).and_return(expected_policy))
+      setup_session_double(options)
+
+      sut.new(options)
+    end
+
+    it 'fails if the retry policy is unknown' do
+      options = default_options.merge({ 'retry_policy' => 'bad policy' })
+
+      expect { sut.new(options) }.to(raise_error(ArgumentError))
+    end
+  end
+
+  describe 'execution' do
+    let(:one_action) {{
+      'table' => 'a_table',
+      'data' => {
+        'a_column' => 'a_value',
+        'another_column' => 'another_value'
+      }
+    }}
+    let(:expected_query_for_one_action) { "INSERT INTO a_table (a_column, another_column)\nVALUES (?, ?)" }
+    let(:another_action) {{
+      'table' => 'another_table',
+      'data' => {
+          'a_column' => 'a_value',
+          'another_column' => 'another_value',
+          'a_third_column' => 'another_value'
+      }
+    }}
+    let(:expected_query_for_another_action) { "INSERT INTO another_table (a_column, another_column, a_third_column)\nVALUES (?, ?, ?)" }
+
+    def generate_future_double
+      future_double = double
+      expect(future_double).to(receive(:join))
+      expect(future_double).to(receive(:on_failure))
+      return future_double
+    end
+
+    it 'prepares and executes the query' do
+      doubles = setup_session_double(default_options)
+      expect(doubles[:session_double]).to(receive(:prepare).with(expected_query_for_one_action)).and_return('eureka')
+      expect(doubles[:session_double]).to(receive(:execute_async).with('eureka', :arguments => one_action['data'].values)).and_return(generate_future_double)
+      sut_instance = sut.new(default_options)
+
+      sut_instance.submit([one_action])
+    end
+
+    it 'caches the generated query' do
+      doubles = setup_session_double(default_options)
+      expect(doubles[:session_double]).to(receive(:prepare).with(expected_query_for_one_action).once).and_return('eureka')
+      2.times {
+        expect(doubles[:session_double]).to(receive(:execute_async).with('eureka', :arguments => one_action['data'].values)).and_return(generate_future_double)
+      }
+      sut_instance = sut.new(default_options)
+
+      sut_instance.submit([one_action, one_action])
+    end
+
+    it 'does not confuse between a new query and cached queries' do
+      doubles = setup_session_double(default_options)
+      expect(doubles[:session_double]).to(receive(:prepare).with(expected_query_for_one_action).once).and_return('eureka')
+      expect(doubles[:session_double]).to(receive(:prepare).with(expected_query_for_another_action).once).and_return('great scott')
+      expect(doubles[:session_double]).to(receive(:execute_async).with('eureka', :arguments => one_action['data'].values)).and_return(generate_future_double)
+      expect(doubles[:session_double]).to(receive(:execute_async).with('great scott', :arguments => another_action['data'].values)).and_return(generate_future_double)
+      sut_instance = sut.new(default_options)
+
+      sut_instance.submit([one_action, another_action])
+    end
+
+    it 'logs and skips failed query preps' do
+      setup_session_double(default_options)
+      sut_instance = sut.new(default_options)
+      expect(sut_instance).to(receive(:get_query).and_raise(ArgumentError))
+      expect(default_options['logger']).to(receive(:error))
+
+      expect { sut_instance.submit([one_action]) }.to_not raise_error
+    end
+
+    it 'logs and skips queries which failed during send' do
+      setup_session_double(default_options)
+      sut_instance = sut.new(default_options)
+      expect(sut_instance).to(receive(:get_query).and_return(double))
+      expect(sut_instance).to(receive(:execute_async).and_raise(ArgumentError))
+      expect(default_options['logger']).to(receive(:error))
+
+      expect { sut_instance.submit([one_action]) }.to_not raise_error
+    end
+
+    it 'does not retry queries which failed to execute in case the retry policy is not backoff' do
+      doubles = setup_session_double(default_options)
+      expect(doubles[:session_double]).to(receive(:prepare).and_return('great scott'))
+      # setup a fail once execution
+      fail_on_join_future = Object.new
+      def fail_on_join_future.on_failure(&block)
+        @block = block
+      end
+      def fail_on_join_future.join
+        @block.call('oh boy...')
+      end
+      expect(doubles[:session_double]).to(receive(:execute_async).with('great scott', :arguments => another_action['data'].values).once).and_return(fail_on_join_future)
+      sut_instance = sut.new(default_options)
+
+      sut_instance.submit([another_action])
+    end
+
+    it 'retries queries which failed to execute' do
+      options = default_options.merge({ 'retry_policy' => { 'type' => 'backoff', 'backoff_type' => '**', 'backoff_size' => 2, 'retry_limit' => 10 },
+                                        'concrete_retry_policy' => ::Cassandra::Retry::Policies::Backoff })
+      doubles = setup_session_double(options)
+      expect(doubles[:session_double]).to(receive(:prepare).and_return('eureka'))
+      expect(doubles[:session_double]).to(receive(:prepare).and_return('great scott'))
+      expect(doubles[:session_double]).to(receive(:execute_async).with('eureka', :arguments => one_action['data'].values)).and_return(generate_future_double)
+      # setup a fail once execution
+      fail_on_join_future = Object.new
+      def fail_on_join_future.on_failure(&block)
+        @block = block
+      end
+      def fail_on_join_future.join
+        @block.call('oh boy...')
+      end
+      expect(doubles[:session_double]).to(receive(:execute_async).with('great scott', :arguments => another_action['data'].values)).and_return(fail_on_join_future)
+      expect(doubles[:session_double]).to(receive(:execute_async).with('great scott', :arguments => another_action['data'].values)).and_return(generate_future_double)
+      sut_instance = sut.new(options)
+
+      sut_instance.submit([one_action, another_action])
+    end
+  end
+end