logstash-output-cassandra 0.9.0

data/lib/logstash/outputs/cassandra/backoff_retry_policy.rb

@@ -0,0 +1,65 @@
+# encoding: utf-8
+require 'cassandra'
+
+module Cassandra
+  module Retry
+    module Policies
+      # This is a version of the default retry policy (https://github.com/datastax/ruby-driver/blob/v2.1.5/lib/cassandra/retry/policies/default.rb)
+      # with backoff retry configuration options
+      class Backoff
+        include ::Cassandra::Retry::Policy
+
+        def initialize(opts)
+          @logger = opts['logger']
+          @backoff_type = opts['backoff_type']
+          @backoff_size = opts['backoff_size']
+          @retry_limit = opts['retry_limit']
+        end
+
+        def read_timeout(statement, consistency, required, received, retrieved, retries)
+          retry_with_backoff({ :statement => statement, :consistency => consistency, :required => required,
+                               :received => received, :retrieved => retrieved, :retries => retries })
+        end
+
+        def write_timeout(statement, consistency, type, required, received, retries)
+          retry_with_backoff({ :statement => statement, :consistency => consistency, :type => type,
+                               :required => required, :received => received, :retries => retries })
+        end
+
+        def unavailable(statement, consistency, required, alive, retries)
+          retry_with_backoff({ :statement => statement, :consistency => consistency, :required => required,
+                               :alive => alive, :retries => retries })
+        end
+
+        def retry_with_backoff(opts)
+          if @retry_limit > -1 && opts[:retries] > @retry_limit
+            @logger.error('backoff retries exhausted', :opts => opts)
+            return reraise
+          end
+
+          @logger.error('activating backoff wait', :opts => opts)
+          backoff_wait_before_next_retry(opts[:retries])
+
+          try_again(opts[:consistency])
+        end
+
+        private
+        def backoff_wait_before_next_retry(retries)
+          backoff_wait_time = calculate_backoff_wait_time(retries)
+          Kernel::sleep(backoff_wait_time)
+        end
+
+        def calculate_backoff_wait_time(retries)
+          case @backoff_type
+            when '**'
+              return @backoff_size ** retries
+            when '*'
+              return @backoff_size * retries
+            else
+              raise ArgumentError, "unknown backoff type #{@backoff_type}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/cassandra/buffer.rb

@@ -0,0 +1,125 @@
+# encoding: utf-8
+require "concurrent"
+java_import java.util.concurrent.locks.ReentrantLock
+
+module LogStash; module Outputs; module Cassandra
+  class Buffer
+    def initialize(logger, max_size, flush_interval, &block)
+      @logger = logger
+      # You need to aquire this for anything modifying state generally
+      @operations_mutex = Mutex.new
+      @operations_lock = java.util.concurrent.locks.ReentrantLock.new
+
+      @stopping = Concurrent::AtomicBoolean.new(false)
+      @max_size = max_size
+      @submit_proc = block
+
+      @buffer = []
+
+      @last_flush = Time.now
+      @flush_interval = flush_interval
+      @flush_thread = spawn_interval_flusher
+    end
+
+    def push(item)
+      synchronize do |buffer|
+        push_unsafe(item)
+      end
+    end
+    alias_method :<<, :push
+
+    # Push multiple items onto the buffer in a single operation
+    def push_multi(items)
+      raise ArgumentError, "push multi takes an array!, not an #{items.class}!" unless items.is_a?(Array)
+      synchronize do |buffer|
+        items.each {|item| push_unsafe(item) }
+      end
+    end
+
+    def flush
+      synchronize { flush_unsafe }
+    end
+
+    def stop(do_flush=true,wait_complete=true)
+      return if stopping?
+      @stopping.make_true
+
+      # No need to acquire a lock in this case
+      return if !do_flush && !wait_complete
+
+      synchronize do
+        flush_unsafe if do_flush
+        @flush_thread.join if wait_complete
+      end
+    end
+
+    def contents
+      synchronize {|buffer| buffer}
+    end
+
+    # For externally operating on the buffer contents
+    # this takes a block and will yield the internal buffer and executes
+    # the block in a synchronized block from the internal mutex
+    def synchronize
+      @operations_mutex.synchronize { yield(@buffer) }
+    end
+
+    # These methods are private for various reasons, chief among them threadsafety!
+    # Many require the @operations_mutex to be locked to be safe
+    private
+
+    def push_unsafe(item)
+      @buffer << item
+      if @buffer.size >= @max_size
+        flush_unsafe
+      end
+    end
+
+    def spawn_interval_flusher
+      Thread.new do
+        loop do
+          sleep 0.2
+          break if stopping?
+          synchronize { interval_flush }
+        end
+      end
+    end
+
+    def interval_flush
+      if last_flush_seconds_ago >= @flush_interval
+        begin
+          @logger.debug? && @logger.debug("Flushing buffer at interval",
+                                        :instance => self.inspect,
+                                        :interval => @flush_interval)
+          flush_unsafe
+        rescue StandardError => e
+          @logger.warn("Error flushing buffer at interval!",
+                       :instance => self.inspect,
+                       :message => e.message,
+                       :class => e.class.name,
+                       :backtrace => e.backtrace
+          )
+        rescue Exception => e
+          @logger.warn("Exception flushing buffer at interval!", :error => e.message, :class => e.class.name)
+        end
+      end
+    end
+
+    def flush_unsafe
+      if @buffer.size > 0
+        @submit_proc.call(@buffer)
+        @buffer.clear
+      end
+
+      @last_flush = Time.now # This must always be set to ensure correct timer behavior
+    end
+
+    def last_flush_seconds_ago
+      Time.now - @last_flush
+    end
+
+    def stopping?
+      @stopping.true?
+    end
+  end
+end end end

data/lib/logstash/outputs/cassandra/event_parser.rb

@@ -0,0 +1,161 @@
+# encoding: utf-8
+require 'time'
+require 'cassandra'
+
+module LogStash; module Outputs; module Cassandra
+  # Responsible for accepting events from the pipeline and returning actions for the SafeSubmitter
+  class EventParser
+    def initialize(options)
+      @logger = options['logger']
+      @table = options['table']
+      @filter_transform_event_key = options['filter_transform_event_key']
+      assert_filter_transform_structure(options['filter_transform']) if options['filter_transform']
+      @filter_transform = options['filter_transform']
+      @hints = options['hints']
+      @ignore_bad_values = options['ignore_bad_values']
+    end
+
+    def parse(event)
+      action = {}
+      action['table'] = event.sprintf(@table)
+      filter_transform = get_filter_transform(event)
+      if filter_transform
+        action['data'] = {}
+        filter_transform.each { |filter|
+          add_event_value_from_filter_to_action(event, filter, action)
+        }
+      else
+        add_event_data_using_configured_hints(event, action)
+      end
+
+      @logger.debug('event parsed to action', :action => action)
+      action
+    end
+
+    private
+    def get_filter_transform(event)
+      filter_transform = nil
+      if @filter_transform_event_key
+        filter_transform = event[@filter_transform_event_key]
+        assert_filter_transform_structure(filter_transform)
+      elsif @filter_transform.length > 0
+        filter_transform = @filter_transform
+      end
+      filter_transform
+    end
+
+    def assert_filter_transform_structure(filter_transform)
+      filter_transform.each { |item|
+        if !item.has_key?('event_key') || !item.has_key?('column_name')
+          raise ArgumentError, "item is incorrectly configured in filter_transform:\nitem => #{item}\nfilter_transform => #{filter_transform}"
+        end
+      }
+    end
+
+    def add_event_value_from_filter_to_action(event, filter, action)
+      event_data = event.sprintf(filter['event_key'])
+      unless filter.fetch('expansion_only', false)
+        event_data = event[event_data]
+      end
+      if filter.has_key?('cassandra_type')
+        cassandra_type = event.sprintf(filter['cassandra_type'])
+        event_data = convert_value_to_cassandra_type_or_default_if_configured(event_data, cassandra_type)
+      end
+      column_name = event.sprintf(filter['column_name'])
+      action['data'][column_name] = event_data
+    end
+
+    def add_event_data_using_configured_hints(event, action)
+      action_data = event.to_hash
+      # Filter out @timestamp, @version, etc to be able to use elasticsearch input plugin directly
+      action_data.reject!{|key| %r{^@} =~ key}
+      @hints.each do |event_key, cassandra_type|
+        if action_data.has_key?(event_key)
+          action_data[event_key] = convert_value_to_cassandra_type_or_default_if_configured(action_data[event_key], cassandra_type)
+        end
+      end
+      action['data'] = action_data
+    end
+
+    def convert_value_to_cassandra_type_or_default_if_configured(event_data, cassandra_type)
+      typed_event_data = nil
+      begin
+        typed_event_data = convert_value_to_cassandra_type(event_data, cassandra_type)
+      rescue Exception => e
+        error_message = "Cannot convert `value (`#{event_data}`) to `#{cassandra_type}` type"
+        if @ignore_bad_values
+          case cassandra_type
+            when 'float', 'int', 'varint', 'bigint', 'double', 'counter', 'timestamp'
+              typed_event_data = convert_value_to_cassandra_type(0, cassandra_type)
+            when 'timeuuid'
+              typed_event_data = convert_value_to_cassandra_type('00000000-0000-0000-0000-000000000000', cassandra_type)
+            when 'inet'
+              typed_event_data = convert_value_to_cassandra_type('0.0.0.0', cassandra_type)
+            when /^set<.*>$/
+              typed_event_data = convert_value_to_cassandra_type([], cassandra_type)
+            else
+              raise ArgumentError, "unable to provide a default value for type #{event_data}"
+          end
+          @logger.warn(error_message, :exception => e, :backtrace => e.backtrace)
+        else
+          @logger.error(error_message, :exception => e, :backtrace => e.backtrace)
+          raise error_message
+        end
+      end
+      typed_event_data
+    end
+
+    def convert_value_to_cassandra_type(event_data, cassandra_type)
+      case cassandra_type
+        when 'timestamp'
+          converted_value = event_data
+          if converted_value.is_a?(Numeric)
+            converted_value = Time.at(converted_value)
+          elsif converted_value.respond_to?(:to_s)
+            converted_value = Time::parse(event_data.to_s)
+          end
+          return ::Cassandra::Types::Timestamp.new(converted_value)
+        when 'inet'
+          return ::Cassandra::Types::Inet.new(event_data)
+        when 'float'
+          return ::Cassandra::Types::Float.new(event_data)
+        when 'varchar'
+          return ::Cassandra::Types::Varchar.new(event_data)
+        when 'text'
+          return ::Cassandra::Types::Text.new(event_data)
+        when 'blob'
+          return ::Cassandra::Types::Blob.new(event_data)
+        when 'ascii'
+          return ::Cassandra::Types::Ascii.new(event_data)
+        when 'bigint'
+          return ::Cassandra::Types::Bigint.new(event_data)
+        when 'counter'
+          return ::Cassandra::Types::Counter.new(event_data)
+        when 'int'
+          return ::Cassandra::Types::Int.new(event_data)
+        when 'varint'
+          return ::Cassandra::Types::Varint.new(event_data)
+        when 'boolean'
+          return ::Cassandra::Types::Boolean.new(event_data)
+        when 'decimal'
+          return ::Cassandra::Types::Decimal.new(event_data)
+        when 'double'
+          return ::Cassandra::Types::Double.new(event_data)
+        when 'timeuuid'
+          return ::Cassandra::Types::Timeuuid.new(event_data)
+        when /^set<(.*)>$/
+          # convert each value
+          # then add all to an array and convert to set
+          converted_items = ::Set.new
+          set_type = $1
+          event_data.each { |item|
+            converted_item = convert_value_to_cassandra_type(item, set_type)
+            converted_items.add(converted_item)
+          }
+          return converted_items
+        else
+          raise "Unknown cassandra_type #{name}"
+      end
+    end
+  end
+end end end

data/lib/logstash/outputs/cassandra/safe_submitter.rb

@@ -0,0 +1,118 @@
+# encoding: utf-8
+require 'thread'
+require 'cassandra'
+require 'logstash/outputs/cassandra/backoff_retry_policy'
+
+module LogStash; module Outputs; module Cassandra
+  # Responsible for submitting parsed actions to cassandra (with or without a retry mechanism)
+  class SafeSubmitter
+    def initialize(options)
+      @statement_cache = {}
+      @logger = options['logger']
+      setup_cassandra_session(options)
+    end
+
+    def submit(actions)
+      queries = prepare_queries(actions)
+      execute_queries_with_retries(queries)
+    end
+
+    private
+    def setup_cassandra_session(options)
+      @retry_policy = get_retry_policy(options['retry_policy'])
+      @consistency = options['consistency'].to_sym
+      cluster = options['cassandra'].cluster(
+        username: options['username'],
+        password: options['password'],
+        protocol_version: options['protocol_version'],
+        hosts: options['hosts'],
+        port: options['port'],
+        consistency: @consistency,
+        timeout: options['request_timeout'],
+        retry_policy: @retry_policy,
+        logger: options['logger']
+      )
+      @session = cluster.connect(options['keyspace'])
+    end
+
+    def get_retry_policy(retry_policy)
+      case retry_policy['type']
+        when 'default'
+          return ::Cassandra::Retry::Policies::Default.new
+        when 'downgrading_consistency'
+          return ::Cassandra::Retry::Policies::DowngradingConsistency.new
+        when 'failthrough'
+          return ::Cassandra::Retry::Policies::Fallthrough.new
+        when 'backoff'
+          return ::Cassandra::Retry::Policies::Backoff.new({
+            'backoff_type' => retry_policy['backoff_type'], 'backoff_size' => retry_policy['backoff_size'],
+            'retry_limit' => retry_policy['retry_limit'], 'logger' => @logger
+          })
+        else
+          raise ArgumentError, "unknown retry policy type: #{retry_policy['type']}"
+      end
+    end
+
+    def prepare_queries(actions)
+      remaining_queries = Queue.new
+      actions.each do |action|
+        begin
+          query = get_query(action)
+          remaining_queries << { :query => query, :arguments => action['data'].values }
+        rescue Exception => e
+          @logger.error('Failed to prepare query', :action => action, :exception => e, :backtrace => e.backtrace)
+        end
+      end
+      remaining_queries
+    end
+
+    def get_query(action)
+      @logger.debug('generating query for action', :action => action)
+      action_data = action['data']
+      query =
+"INSERT INTO #{action['table']} (#{action_data.keys.join(', ')})
+VALUES (#{('?' * action_data.keys.count).split(//) * ', '})"
+      unless @statement_cache.has_key?(query)
+        @logger.debug('preparing new query', :query => query)
+        @statement_cache[query] = @session.prepare(query)
+      end
+      @statement_cache[query]
+    end
+
+    def execute_queries_with_retries(queries)
+      retries = 0
+      while queries.length > 0
+        execute_queries(queries, retries)
+        retries += 1
+      end
+    end
+
+    def execute_queries(queries, retries)
+      futures = []
+      while queries.length > 0
+        query = queries.pop
+        begin
+          future = execute_async(query, retries, queries)
+          futures << future
+        rescue Exception => e
+          @logger.error('Failed to send query', :query => query, :exception => e, :backtrace => e.backtrace)
+        end
+      end
+      futures.each(&:join)
+    end
+
+    def execute_async(query, retries, queries)
+      future = @session.execute_async(query[:query], arguments: query[:arguments])
+      future.on_failure { |error|
+        @logger.error('Failed to execute query', :query => query, :error => error)
+        if @retry_policy.is_a?(::Cassandra::Retry::Policies::Backoff)
+          decision = @retry_policy.retry_with_backoff({ :retries => retries, :consistency => @consistency })
+          if decision.is_a?(::Cassandra::Retry::Decisions::Retry)
+            queries << query
+          end
+        end
+      }
+      future
+    end
+  end
+end end end