logstash-output-cassandra-v5 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 5a1152690c197d94b35534062a0fd5d017db79e699a0bc30cd97b39f04ddf8e3
+  data.tar.gz: 8ca6fc3784f815ff823df65ef73e9e8b3f99df3b2eacd50e31b3c08536a8f3fd
+SHA512:
+  metadata.gz: 9e83ee3db069423a55c37d76bcd121454f979e08ba15de74907662836d57e2ed9197c36807334eacfe2279e326271b1344e72d8f45390c8c5d7379b680322d47
+  data.tar.gz: 1e7549cb53ab649f6c2d0f01c35f52bd61914e708dc59e41eaf3165730ac9a775978e7f4460a296f08373eba27522cf5069aed939acdf600fbb1c3b84401240b

data/CONTRIBUTORS

@@ -0,0 +1,10 @@
+The following is a list of people who have contributed (in chronological order) ideas, code, bug
+reports, or in general have helped this plugin along its way.
+
+Contributors:
+* Luis Burbano Ulloa (lburbanoulloa)
+* Oleg Tokarev (otokarev)
+* Elad Amit (eladamitpxi, amitelad7)
+* Valentin Fischer (valentinul)
+* tansinee
+* Pitsanu Swangpheaw (roongr2k7)

data/Gemfile

@@ -0,0 +1,4 @@
+#ruby=jruby
+#ruby-gemset=logstash-output-cassandra-v5
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2016 PerimeterX <http://www.perimeterx.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,146 @@
+# Logstash Cassandra Output Plugin v5
+
+This is a plugin for [Logstash](https://github.com/elastic/logstash).
+
+It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
+
+It was originally a fork of the [logstash-output-cassandra](https://github.com/PerimeterX/logstash-output-cassandra
+
+This version update the plugin to work with Cassandra v5, and fix an error when inserting UUId fields.
+
+## Usage
+
+<pre><code>
+output {
+    cassandra {
+        # List of Cassandra hostname(s) or IP-address(es)
+        hosts => [ "cass-01", "cass-02" ]
+
+        # The port cassandra is listening to
+        port => 9042
+
+        # The protocol version to use with cassandra
+        protocol_version => 4
+
+        # Cassandra consistency level.
+        # Options: "any", "one", "two", "three", "quorum", "all", "local_quorum", "each_quorum", "serial", "local_serial", "local_one"
+        # Default: "one"
+        consistency => 'any'
+
+        # The keyspace to use
+        keyspace => "a_ks"
+
+        # The table to use (event level processing (e.g. %{[key]}) is supported)
+        table => "%{[@metadata][cassandra_table]}"
+
+        # Username
+        username => "cassandra"
+
+        # Password
+        password => "cassandra"
+
+        # An optional hints hash which will be used in case filter_transform or filter_transform_event_key are not in use
+        # It is used to trigger a forced type casting to the cassandra driver types in
+        # the form of a hash from column name to type name in the following manner:
+        hints => {
+            id => "int"
+            at => "timestamp"
+            resellerId => "int"
+            errno => "int"
+            duration => "float"
+            ip => "inet"
+        }
+
+        # The retry policy to use (the default is the default retry policy)
+        # the hash requires the name of the policy and the params it requires
+        # The available policy names are:
+        # * default => retry once if needed / possible
+        # * downgrading_consistency => retry once with a best guess lowered consistency
+        # * failthrough => fail immediately (i.e. no retries)
+        # * backoff => a version of the default retry policy but with configurable backoff retries
+        # The backoff options are as follows:
+        # * backoff_type => either * or ** for linear and exponential backoffs respectively
+        # * backoff_size => the left operand for the backoff type in seconds
+        # * retry_limit => the maximum amount of retries to allow per query
+        # example:
+        # using { "type" => "backoff" "backoff_type" => "**" "backoff_size" => 2 "retry_limit" => 10 } will perform 10 retries with the following wait times: 1, 2, 4, 8, 16, ... 1024
+        # NOTE: there is an underlying assumption that the insert query is idempotent !!!
+        # NOTE: when the backoff retry policy is used, it will also be used to handle pure client timeouts and not just ones coming from the coordinator
+        retry_policy => { "type" => "default" }
+
+        # The command execution timeout
+        request_timeout => 1
+
+        # Ignore bad values
+        ignore_bad_values => false
+
+        # In Logstashes >= 2.2 this setting defines the maximum sized bulk request Logstash will make
+        # You you may want to increase this to be in line with your pipeline's batch size.
+        # If you specify a number larger than the batch size of your pipeline it will have no effect,
+        # save for the case where a filter increases the size of an inflight batch by outputting
+        # events.
+        #
+        # In Logstashes <= 2.1 this plugin uses its own internal buffer of events.
+        # This config option sets that size. In these older logstashes this size may
+        # have a significant impact on heap usage, whereas in 2.2+ it will never increase it.
+        # To make efficient bulk API calls, we will buffer a certain number of
+        # events before flushing that out to Cassandra. This setting
+        # controls how many events will be buffered before sending a batch
+        # of events. Increasing the `flush_size` has an effect on Logstash's heap size.
+        # Remember to also increase the heap size using `LS_HEAP_SIZE` if you are sending big commands
+        # or have increased the `flush_size` to a higher value.
+        flush_size => 500
+
+        # The amount of time since last flush before a flush is forced.
+        #
+        # This setting helps ensure slow event rates don't get stuck in Logstash.
+        # For example, if your `flush_size` is 100, and you have received 10 events,
+        # and it has been more than `idle_flush_time` seconds since the last flush,
+        # Logstash will flush those 10 events automatically.
+        #
+        # This helps keep both fast and slow log streams moving along in
+        # near-real-time.
+        idle_flush_time => 1
+    }
+}
+</code></pre>
+
+## Running Plugin in Logstash
+### Run in a local Logstash clone
+
+Edit Logstash Gemfile and add the local plugin path, for example:
+```
+gem "logstash-output-cassandra", :path => "/your/local/logstash-output-cassandra"
+```
+And install by executing:
+```
+bin/plugin install --no-verify
+```
+
+Or install plugin from RubyGems:
+```
+bin/plugin install logstash-output-cassandra
+```
+
+And then run Logstash with the plugin:
+```
+bin/logstash -e 'output {cassandra {}}'
+```
+
+### Run in an installed Logstash
+
+You can use the same method to run your plugin in an installed Logstash by editing its Gemfile and pointing the :path to your local plugin development directory or you can build the gem and install it using:
+
+Build your plugin gem
+```
+gem build logstash-output-cassandra.gemspec
+```
+Install the plugin from the Logstash home
+```
+bin/plugin install /your/local/plugin/logstash-output-cassandra.gem
+```
+Run Logstash with the plugin
+```
+bin/logstash -e 'output {cassandra {}}'
+```
+

data/lib/logstash/outputs/cassandra/backoff_retry_policy.rb

@@ -0,0 +1,65 @@
+# encoding: utf-8
+require 'cassandra'
+
+module Cassandra
+  module Retry
+    module Policies
+      # This is a version of the default retry policy (https://github.com/datastax/ruby-driver/blob/v2.1.5/lib/cassandra/retry/policies/default.rb)
+      # with backoff retry configuration options
+      class Backoff
+        include ::Cassandra::Retry::Policy
+
+        def initialize(opts)
+          @logger = opts['logger']
+          @backoff_type = opts['backoff_type']
+          @backoff_size = opts['backoff_size']
+          @retry_limit = opts['retry_limit']
+        end
+
+        def read_timeout(statement, consistency, required, received, retrieved, retries)
+          retry_with_backoff({ :statement => statement, :consistency => consistency, :required => required,
+                               :received => received, :retrieved => retrieved, :retries => retries })
+        end
+
+        def write_timeout(statement, consistency, type, required, received, retries)
+          retry_with_backoff({ :statement => statement, :consistency => consistency, :type => type,
+                               :required => required, :received => received, :retries => retries })
+        end
+
+        def unavailable(statement, consistency, required, alive, retries)
+          retry_with_backoff({ :statement => statement, :consistency => consistency, :required => required,
+                               :alive => alive, :retries => retries })
+        end
+
+        def retry_with_backoff(opts)
+          if @retry_limit > -1 && opts[:retries] > @retry_limit
+            @logger.error('backoff retries exhausted', :opts => opts)
+            return reraise
+          end
+
+          @logger.error('activating backoff wait', :opts => opts)
+          backoff_wait_before_next_retry(opts[:retries])
+
+          try_again(opts[:consistency])
+        end
+
+        private
+        def backoff_wait_before_next_retry(retries)
+          backoff_wait_time = calculate_backoff_wait_time(retries)
+          Kernel::sleep(backoff_wait_time)
+        end
+
+        def calculate_backoff_wait_time(retries)
+          case @backoff_type
+            when '**'
+              return @backoff_size ** retries
+            when '*'
+              return @backoff_size * retries
+            else
+              raise ArgumentError, "unknown backoff type #{@backoff_type}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/cassandra/buffer.rb

@@ -0,0 +1,125 @@
+# encoding: utf-8
+require "concurrent"
+java_import java.util.concurrent.locks.ReentrantLock
+
+module LogStash; module Outputs; module Cassandra
+  class Buffer
+    def initialize(logger, max_size, flush_interval, &block)
+      @logger = logger
+      # You need to aquire this for anything modifying state generally
+      @operations_mutex = Mutex.new
+      @operations_lock = java.util.concurrent.locks.ReentrantLock.new
+
+      @stopping = Concurrent::AtomicBoolean.new(false)
+      @max_size = max_size
+      @submit_proc = block
+
+      @buffer = []
+
+      @last_flush = Time.now
+      @flush_interval = flush_interval
+      @flush_thread = spawn_interval_flusher
+    end
+
+    def push(item)
+      synchronize do |buffer|
+        push_unsafe(item)
+      end
+    end
+    alias_method :<<, :push
+
+    # Push multiple items onto the buffer in a single operation
+    def push_multi(items)
+      raise ArgumentError, "push multi takes an array!, not an #{items.class}!" unless items.is_a?(Array)
+      synchronize do |buffer|
+        items.each {|item| push_unsafe(item) }
+      end
+    end
+
+    def flush
+      synchronize { flush_unsafe }
+    end
+
+    def stop(do_flush=true,wait_complete=true)
+      return if stopping?
+      @stopping.make_true
+
+      # No need to acquire a lock in this case
+      return if !do_flush && !wait_complete
+
+      synchronize do
+        flush_unsafe if do_flush
+        @flush_thread.join if wait_complete
+      end
+    end
+
+    def contents
+      synchronize {|buffer| buffer}
+    end
+
+    # For externally operating on the buffer contents
+    # this takes a block and will yield the internal buffer and executes
+    # the block in a synchronized block from the internal mutex
+    def synchronize
+      @operations_mutex.synchronize { yield(@buffer) }
+    end
+
+    # These methods are private for various reasons, chief among them threadsafety!
+    # Many require the @operations_mutex to be locked to be safe
+    private
+
+    def push_unsafe(item)
+      @buffer << item
+      if @buffer.size >= @max_size
+        flush_unsafe
+      end
+    end
+
+    def spawn_interval_flusher
+      Thread.new do
+        loop do
+          sleep 0.2
+          break if stopping?
+          synchronize { interval_flush }
+        end
+      end
+    end
+
+    def interval_flush
+      if last_flush_seconds_ago >= @flush_interval
+        begin
+          @logger.debug? && @logger.debug("Flushing buffer at interval",
+                                        :instance => self.inspect,
+                                        :interval => @flush_interval)
+          flush_unsafe
+        rescue StandardError => e
+          @logger.warn("Error flushing buffer at interval!",
+                       :instance => self.inspect,
+                       :message => e.message,
+                       :class => e.class.name,
+                       :backtrace => e.backtrace
+          )
+        rescue Exception => e
+          @logger.warn("Exception flushing buffer at interval!", :error => e.message, :class => e.class.name)
+        end
+      end
+    end
+
+    def flush_unsafe
+      if @buffer.size > 0
+        @submit_proc.call(@buffer)
+        @buffer.clear
+      end
+
+      @last_flush = Time.now # This must always be set to ensure correct timer behavior
+    end
+
+    def last_flush_seconds_ago
+      Time.now - @last_flush
+    end
+
+    def stopping?
+      @stopping.true?
+    end
+  end
+end end end

data/lib/logstash/outputs/cassandra/event_parser.rb

@@ -0,0 +1,164 @@
+# encoding: utf-8
+require 'time'
+require 'cassandra'
+
+module LogStash; module Outputs; module Cassandra
+  # Responsible for accepting events from the pipeline and returning actions for the SafeSubmitter
+  class EventParser
+    def initialize(options)
+      @logger = options['logger']
+      @table = options['table']
+      @filter_transform_event_key = options['filter_transform_event_key']
+      assert_filter_transform_structure(options['filter_transform']) if options['filter_transform']
+      @filter_transform = options['filter_transform']
+      @hints = options['hints']
+      @ignore_bad_values = options['ignore_bad_values']
+    end
+
+    def parse(event)
+      action = {}
+      begin
+        action['table'] = event.sprintf(@table)
+        filter_transform = get_filter_transform(event)
+        if filter_transform
+          action['data'] = {}
+          filter_transform.each { |filter|
+            add_event_value_from_filter_to_action(event, filter, action)
+          }
+        else
+          add_event_data_using_configured_hints(event, action)
+        end
+        @logger.debug('event parsed to action', :action => action)
+      rescue Exception => e
+        @logger.error('failed parsing event', :event => event, :error => e)
+        action = nil
+      end
+      action
+    end
+
+    private
+    def get_filter_transform(event)
+      filter_transform = nil
+      if @filter_transform_event_key
+        filter_transform = event.get(@filter_transform_event_key)
+        assert_filter_transform_structure(filter_transform)
+      elsif @filter_transform.length > 0
+        filter_transform = @filter_transform
+      end
+      filter_transform
+    end
+
+    def assert_filter_transform_structure(filter_transform)
+      filter_transform.each { |item|
+        if !item.has_key?('event_key') || !item.has_key?('column_name')
+          raise ArgumentError, "item is incorrectly configured in filter_transform:\nitem => #{item}\nfilter_transform => #{filter_transform}"
+        end
+      }
+    end
+
+    def add_event_value_from_filter_to_action(event, filter, action)
+      event_data = event.sprintf(filter['event_key'])
+      unless filter.fetch('expansion_only', false)
+        event_data = event.get(event_data)
+      end
+      if filter.has_key?('cassandra_type')
+        cassandra_type = event.sprintf(filter['cassandra_type'])
+        event_data = convert_value_to_cassandra_type_or_default_if_configured(event_data, cassandra_type)
+      end
+      column_name = event.sprintf(filter['column_name'])
+      action['data'][column_name] = event_data
+    end
+
+    def add_event_data_using_configured_hints(event, action)
+      action_data = event.to_hash.reject { |key| %r{^@} =~ key }
+      
+      @hints.each do |event_key, cassandra_type|
+        if action_data.has_key?(event_key)
+          action_data[event_key] = convert_value_to_cassandra_type_or_default_if_configured(action_data[event_key], cassandra_type)
+        end
+      end
+      action['data'] = action_data
+    end
+
+    def convert_value_to_cassandra_type_or_default_if_configured(event_data, cassandra_type)
+      typed_event_data = nil
+      begin
+        typed_event_data = convert_value_to_cassandra_type(event_data, cassandra_type)
+      rescue Exception => e
+        error_message = "Cannot convert `value (`#{event_data}`) to `#{cassandra_type}` type"
+        if @ignore_bad_values
+          case cassandra_type
+            when 'float', 'int', 'varint', 'bigint', 'double', 'counter', 'timestamp'
+              typed_event_data = convert_value_to_cassandra_type(0, cassandra_type)
+            when 'uuid', 'timeuuid'
+              typed_event_data = convert_value_to_cassandra_type('00000000-0000-0000-0000-000000000000', cassandra_type)
+            when 'inet'
+              typed_event_data = convert_value_to_cassandra_type('0.0.0.0', cassandra_type)
+            when /^set<.*>$/
+              typed_event_data = convert_value_to_cassandra_type([], cassandra_type)
+            else
+              raise ArgumentError, "unable to provide a default value for type #{event_data}"
+          end
+          @logger.warn(error_message, :exception => e, :backtrace => e.backtrace)
+        else
+          @logger.error(error_message, :exception => e, :backtrace => e.backtrace)
+          raise error_message
+        end
+      end
+      typed_event_data
+    end
+
+    def convert_value_to_cassandra_type(event_data, cassandra_type)
+      case cassandra_type
+        when 'timestamp'
+          converted_value = event_data
+          if converted_value.is_a?(Numeric)
+            converted_value = Time.at(converted_value)
+          elsif converted_value.respond_to?(:to_s)
+            converted_value = Time::parse(event_data.to_s)
+          end
+          return ::Cassandra::Types::Timestamp.new(converted_value)
+        when 'inet'
+          return ::Cassandra::Types::Inet.new(event_data)
+        when 'float'
+          return ::Cassandra::Types::Float.new(event_data)
+        when 'text'
+          return ::Cassandra::Types::Text.new(event_data)
+        when 'blob'
+          return ::Cassandra::Types::Blob.new(event_data)
+        when 'ascii'
+          return ::Cassandra::Types::Ascii.new(event_data)
+        when 'bigint'
+          return ::Cassandra::Types::Bigint.new(event_data)
+        when 'counter'
+          return ::Cassandra::Types::Counter.new(event_data)
+        when 'int'
+          return ::Cassandra::Types::Int.new(event_data)
+        when 'varint'
+          return ::Cassandra::Types::Varint.new(event_data)
+        when 'boolean'
+          return ::Cassandra::Types::Boolean.new(event_data)
+        when 'decimal'
+          return ::Cassandra::Types::Decimal.new(event_data)
+        when 'double'
+          return ::Cassandra::Types::Double.new(event_data)
+        when 'timeuuid'
+          return ::Cassandra::Types::Timeuuid.new(event_data)
+        when 'uuid'
+          return ::Cassandra::Types::Uuid.new(event_data)
+        when /^set<(.*)>$/
+          # convert each value
+          # then add all to an array and convert to set
+          converted_items = ::Set.new
+          set_type = $1
+          event_data.each { |item|
+            converted_item = convert_value_to_cassandra_type(item, set_type)
+            converted_items.add(converted_item)
+          }
+          return converted_items
+        else
+          raise "Unknown cassandra_type #{cassandra_type}"
+      end
+    end
+  end
+end end end

data/lib/logstash/outputs/cassandra/safe_submitter.rb

@@ -0,0 +1,121 @@
+# encoding: utf-8
+require 'thread'
+require 'cassandra'
+require 'logstash/outputs/cassandra/backoff_retry_policy'
+require 'logstash/outputs/cassandra/schema_fetcher_version_patch'
+
+module LogStash; module Outputs; module Cassandra
+  # Responsible for submitting parsed actions to cassandra (with or without a retry mechanism)
+  class SafeSubmitter
+    def initialize(options)
+      @statement_cache = {}
+      @logger = options['logger']
+      setup_cassandra_session(options)
+    end
+
+    def submit(actions)
+      queries = prepare_queries(actions)
+      execute_queries_with_retries(queries)
+    end
+
+    private
+    def setup_cassandra_session(options)
+      @retry_policy = get_retry_policy(options['retry_policy'])
+      @consistency = options['consistency'].to_sym
+      cluster = options['cassandra'].cluster(
+        username: options['username'],
+        password: options['password'],
+        protocol_version: options['protocol_version'],
+        hosts: options['hosts'],
+        port: options['port'],
+        consistency: @consistency,
+        timeout: options['request_timeout'],
+        retry_policy: @retry_policy,
+        logger: options['logger']
+      )
+      @session = cluster.connect(options['keyspace'])
+    end
+
+    def get_retry_policy(retry_policy)
+      case retry_policy['type']
+        when 'default'
+          return ::Cassandra::Retry::Policies::Default.new
+        when 'downgrading_consistency'
+          return ::Cassandra::Retry::Policies::DowngradingConsistency.new
+        when 'failthrough'
+          return ::Cassandra::Retry::Policies::Fallthrough.new
+        when 'backoff'
+          return ::Cassandra::Retry::Policies::Backoff.new({
+            'backoff_type' => retry_policy['backoff_type'], 'backoff_size' => retry_policy['backoff_size'],
+            'retry_limit' => retry_policy['retry_limit'], 'logger' => @logger
+          })
+        else
+          raise ArgumentError, "unknown retry policy type: #{retry_policy['type']}"
+      end
+    end
+
+    def prepare_queries(actions)
+      remaining_queries = Queue.new
+      actions.each do |action|
+        begin
+          if action
+            query = get_query(action)
+            remaining_queries << { :query => query, :arguments => action['data'].values }
+          end
+        rescue Exception => e
+          @logger.error('Failed to prepare query', :action => action, :exception => e, :backtrace => e.backtrace)
+        end
+      end
+      remaining_queries
+    end
+
+    def get_query(action)
+      @logger.debug('generating query for action', :action => action)
+      action_data = action['data']
+      query =
+"INSERT INTO #{action['table']} (#{action_data.keys.join(', ')})
+VALUES (#{('?' * action_data.keys.count).split(//) * ', '})"
+      unless @statement_cache.has_key?(query)
+        @logger.debug('preparing new query', :query => query)
+        @statement_cache[query] = @session.prepare(query)
+      end
+      @statement_cache[query]
+    end
+
+    def execute_queries_with_retries(queries)
+      retries = 0
+      while queries.length > 0
+        execute_queries(queries, retries)
+        retries += 1
+      end
+    end
+
+    def execute_queries(queries, retries)
+      futures = []
+      while queries.length > 0
+        query = queries.pop
+        begin
+          future = execute_async(query, retries, queries)
+          futures << future
+        rescue Exception => e
+          @logger.error('Failed to send query', :query => query, :exception => e, :backtrace => e.backtrace)
+        end
+      end
+      futures.each(&:join)
+    end
+
+    def execute_async(query, retries, queries)
+      future = @session.execute_async(query[:query], arguments: query[:arguments])
+      future.on_failure { |error|
+        @logger.error('Failed to execute query', :query => query, :error => error)
+        if @retry_policy.is_a?(::Cassandra::Retry::Policies::Backoff)
+          decision = @retry_policy.retry_with_backoff({ :retries => retries, :consistency => @consistency })
+          if decision.is_a?(::Cassandra::Retry::Decisions::Retry)
+            queries << query
+          end
+        end
+      }
+      future
+    end
+  end
+end end end

data/lib/logstash/outputs/cassandra/schema_fetcher_version_patch.rb

@@ -0,0 +1,23 @@
+# encoding: utf-8
+require 'cassandra'
+
+# cassandra-driver 3.2.5 (last released in 2020) only registers schema fetchers for
+# Cassandra release versions '1.2', '2.0', '2.1', '2.2', '3.' and '4.' (see
+# Cassandra::Driver#create_schema_fetcher_picker). Anything else, e.g. Cassandra 5.x,
+# makes the control connection fail with:
+#   Cassandra::Errors::ClientError: unsupported release version "5.0.8".
+# The system_schema.* tables that the '3.'/'4.' fetcher (V3_0_x) reads from have not
+# changed in a way that breaks it for Cassandra 5.x, so it's safe to reuse it here.
+module LogStash; module Outputs; module Cassandra
+  module SchemaFetcherVersionPatch
+    def create_schema_fetcher_picker
+      picker = super
+      picker.when('5.') do
+        ::Cassandra::Cluster::Schema::Fetchers::V3_0_x.new(schema_cql_type_parser, cluster_schema)
+      end
+      picker
+    end
+  end
+end; end; end
+
+::Cassandra::Driver.prepend(LogStash::Outputs::Cassandra::SchemaFetcherVersionPatch)