logstash-output-elasticsearch 2.1.2-java → 2.1.4-java

data/lib/logstash/outputs/elasticsearch/buffer.rb

@@ -0,0 +1,124 @@
+require 'concurrent'
+java_import java.util.concurrent.locks.ReentrantLock
+
+module LogStash; module Outputs; class ElasticSearch
+  class Buffer
+    def initialize(logger, max_size, flush_interval, &block)
+      @logger = logger
+      # You need to aquire this for anything modifying state generally
+      @operations_mutex = Mutex.new
+      @operations_lock = java.util.concurrent.locks.ReentrantLock.new
+
+      @stopping = Concurrent::AtomicBoolean.new(false)
+      @max_size = max_size
+      @submit_proc = block
+
+      @buffer = []
+
+      @last_flush = Time.now
+      @flush_interval = flush_interval
+      @flush_thread = spawn_interval_flusher
+    end
+
+    def push(item)
+      synchronize do |buffer|
+        push_unsafe(item)
+      end
+    end
+    alias_method :<<, :push
+
+    # Push multiple items onto the buffer in a single operation
+    def push_multi(items)
+      raise ArgumentError, "push multi takes an array!, not an #{items.class}!" unless items.is_a?(Array)
+      synchronize do |buffer|
+        items.each {|item| push_unsafe(item) }
+      end
+    end
+
+    def flush
+      synchronize { flush_unsafe }
+    end
+
+    def stop(do_flush=true,wait_complete=true)
+      return if stopping?
+      @stopping.make_true
+
+      # No need to acquire a lock in this case
+      return if !do_flush && !wait_complete
+
+      synchronize do
+        flush_unsafe if do_flush
+        @flush_thread.join if wait_complete
+      end
+    end
+
+    def contents
+      synchronize {|buffer| buffer}
+    end
+
+    # For externally operating on the buffer contents
+    # this takes a block and will yield the internal buffer and executes
+    # the block in a synchronized block from the internal mutex
+    def synchronize
+      @operations_mutex.synchronize { yield(@buffer) }
+    end
+
+    # These methods are private for various reasons, chief among them threadsafety!
+    # Many require the @operations_mutex to be locked to be safe
+    private
+
+    def push_unsafe(item)
+      @buffer << item
+      if @buffer.size >= @max_size
+        flush_unsafe
+      end
+    end
+
+    def spawn_interval_flusher
+      Thread.new do
+        loop do
+          sleep 0.2
+          break if stopping?
+          synchronize { interval_flush }
+        end
+      end
+    end
+
+    def interval_flush
+      if last_flush_seconds_ago >= @flush_interval
+        begin
+          @logger.info? && @logger.info("Flushing buffer at interval",
+                                        :instance => self.inspect,
+                                        :interval => @flush_interval)
+          flush_unsafe
+        rescue StandardError => e
+          @logger.warn("Error flushing buffer at interval!",
+                       :instance => self.inspect,
+                       :message => e.message,
+                       :class => e.class.name,
+                       :backtrace => e.backtrace
+          )
+        rescue Exception => e
+          @logger.warn("Exception flushing buffer at interval!", :error => e.message, :class => e.class.name)
+        end
+      end
+    end
+
+    def flush_unsafe
+      if @buffer.size > 0
+        @submit_proc.call(@buffer)
+        @buffer.clear
+      end
+
+      @last_flush = Time.now # This must always be set to ensure correct timer behavior
+    end
+
+    def last_flush_seconds_ago
+      Time.now - @last_flush
+    end
+
+    def stopping?
+      @stopping.true?
+    end
+  end
+end end end

data/lib/logstash/outputs/elasticsearch/common.rb

@@ -0,0 +1,173 @@
+require "logstash/outputs/elasticsearch/template_manager"
+require "logstash/outputs/elasticsearch/buffer"
+
+module LogStash; module Outputs; class ElasticSearch;
+  module Common
+    attr_reader :client, :hosts
+
+    RETRYABLE_CODES = [409, 429, 503]
+    SUCCESS_CODES = [200, 201]
+
+    def register
+      @stopping = Concurrent::AtomicBoolean.new(false)
+      setup_hosts # properly sets @hosts
+      build_client
+      install_template
+      setup_buffer_and_handler
+
+      @logger.info("New Elasticsearch output", :class => self.class.name, :hosts => @hosts)
+    end
+
+    def receive(event)
+      @buffer << event_action_tuple(event)
+    end
+
+    # Receive an array of events and immediately attempt to index them (no buffering)
+    def multi_receive(events)
+      retrying_submit(events.map {|e| event_action_tuple(e) })
+    end
+
+    # Convert the event into a 3-tuple of action, params, and event
+    def event_action_tuple(event)
+      params = event_action_params(event)
+      action = event.sprintf(@action)
+      [action, params, event]
+    end
+
+    def flush
+      @buffer.flush
+    end
+
+    def setup_hosts
+      @hosts = Array(@hosts)
+      if @hosts.empty?
+        @logger.info("No 'host' set in elasticsearch output. Defaulting to localhost")
+        @hosts.replace(["localhost"])
+      end
+    end
+
+    def install_template
+      TemplateManager.install_template(self)
+    end
+
+    def setup_buffer_and_handler
+      @buffer = ::LogStash::Outputs::ElasticSearch::Buffer.new(@logger, @flush_size, @idle_flush_time) do |actions|
+        retrying_submit(actions)
+      end
+    end
+
+    def retrying_submit(actions)
+      retries_left = @max_retries+1 # +1 for the first attempt
+
+      # Initially we submit the full list of actions
+      submit_actions = actions
+
+      while submit_actions && submit_actions.length > 0 && retries_left > 0
+        return if !submit_actions || submit_actions.empty? # If everything's a success we move along
+        # We retry with whatever is didn't succeed
+        begin
+          submit_actions = submit(submit_actions)
+        rescue => e
+          @logger.warn("Encountered an unexpected error submitting a bulk request!",
+                       :message => e.message,
+                       :class => e.class.name,
+                       :backtrace => e.backtrace)
+        end
+        retries_left -= 1
+      end
+    end
+
+    def submit(actions)
+      es_actions = actions.map { |a, doc, event| [a, doc, event.to_hash]}
+
+      bulk_response = safe_bulk(es_actions,actions)
+
+      # If there are no errors, we're done here!
+      return unless bulk_response["errors"]
+
+      actions_to_retry = []
+      bulk_response["items"].each_with_index do |response,idx|
+        action_type, action_props = response.first
+        status = action_props["status"]
+        action = actions[idx]
+
+        if SUCCESS_CODES.include?(status)
+          next
+        elsif RETRYABLE_CODES.include?(status)
+          @logger.warn "retrying failed action with response code: #{status}"
+          actions_to_retry << action
+        else
+          @logger.warn "Failed action. ", status: status, action: action, response: response
+        end
+      end
+
+      actions_to_retry
+    end
+
+    # get the action parameters for the given event
+    def event_action_params(event)
+      type = get_event_type(event)
+
+      params = {
+        :_id => @document_id ? event.sprintf(@document_id) : nil,
+        :_index => event.sprintf(@index),
+        :_type => type,
+        :_routing => @routing ? event.sprintf(@routing) : nil
+      }
+
+      params[:_upsert] = LogStash::Json.load(event.sprintf(@upsert)) if @action == 'update' && @upsert != ""
+      params
+    end
+
+    # Determine the correct value for the 'type' field for the given event
+    def get_event_type(event)
+      # Set the 'type' value for the index.
+      type = if @document_type
+               event.sprintf(@document_type)
+             else
+               event["type"] || "logs"
+             end
+
+      if !(type.is_a?(String) || type.is_a?(Numeric))
+        @logger.warn("Bad event type! Non-string/integer type value set!", :type_class => type.class, :type_value => type.to_s, :event => event)
+      end
+
+      type.to_s
+    end
+
+    # Rescue retryable errors during bulk submission
+    def safe_bulk(es_actions,actions)
+      @client.bulk(es_actions)
+    rescue Manticore::SocketException, Manticore::SocketTimeout => e
+      # If we can't even connect to the server let's just print out the URL (:hosts is actually a URL)
+      # and let the user sort it out from there
+      @logger.error(
+        "Attempted to send a bulk request to Elasticsearch configured at '#{@client.client_options[:hosts]}',"+
+          " but Elasticsearch appears to be unreachable or down!",
+        :client_config => @client.client_options,
+        :error_message => e.message,
+        :class => e.class.name
+      )
+      @logger.debug("Failed actions for last bad bulk request!", :actions => actions)
+
+      # We retry until there are no errors! Errors should all go to the retry queue
+      sleep @retry_max_interval
+      retry unless @stopping.true?
+    rescue => e
+      # For all other errors print out full connection issues
+      @logger.error(
+        "Attempted to send a bulk request to Elasticsearch configured at '#{@client.client_options[:hosts]}'," +
+          " but an error occurred and it failed! Are you sure you can reach elasticsearch from this machine using " +
+          "the configuration provided?",
+        :client_config => @client.client_options,
+        :error_message => e.message,
+        :error_class => e.class.name,
+        :backtrace => e.backtrace
+      )
+
+      @logger.debug("Failed actions for last bad bulk request!", :actions => actions)
+
+      raise e
+    end
+  end
+end; end; end

data/lib/logstash/outputs/elasticsearch/common_configs.rb

@@ -0,0 +1,113 @@
+module LogStash; module Outputs; class ElasticSearch
+  module CommonConfigs
+    def self.included(mod)
+      # The index to write events to. This can be dynamic using the `%{foo}` syntax.
+      # The default value will partition your indices by day so you can more easily
+      # delete old data or only search specific date ranges.
+      # Indexes may not contain uppercase characters.
+      # For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}
+      mod.config :index, :validate => :string, :default => "logstash-%{+YYYY.MM.dd}"
+
+      # The index type to write events to. Generally you should try to write only
+      # similar events to the same 'type'. String expansion `%{foo}` works here.
+      #
+      # Deprecated in favor of `docoument_type` field.
+      mod.config :index_type, :validate => :string, :obsolete => "Please use the 'document_type' setting instead. It has the same effect, but is more appropriately named."
+
+      # The document type to write events to. Generally you should try to write only
+      # similar events to the same 'type'. String expansion `%{foo}` works here.
+      # Unless you set 'document_type', the event 'type' will be used if it exists
+      # otherwise the document type will be assigned the value of 'logs'
+      mod.config :document_type, :validate => :string
+
+      # Starting in Logstash 1.3 (unless you set option `manage_template` to false)
+      # a default mapping template for Elasticsearch will be applied, if you do not
+      # already have one set to match the index pattern defined (default of
+      # `logstash-%{+YYYY.MM.dd}`), minus any variables.  For example, in this case
+      # the template will be applied to all indices starting with `logstash-*`
+      #
+      # If you have dynamic templating (e.g. creating indices based on field names)
+      # then you should set `manage_template` to false and use the REST API to upload
+      # your templates manually.
+      mod.config :manage_template, :validate => :boolean, :default => true
+
+      # This configuration option defines how the template is named inside Elasticsearch.
+      # Note that if you have used the template management features and subsequently
+      # change this, you will need to prune the old template manually, e.g.
+      #
+      # `curl -XDELETE <http://localhost:9200/_template/OldTemplateName?pretty>`
+      #
+      # where `OldTemplateName` is whatever the former setting was.
+      mod.config :template_name, :validate => :string, :default => "logstash"
+
+      # You can set the path to your own template here, if you so desire.
+      # If not set, the included template will be used.
+      mod.config :template, :validate => :path
+
+      # Overwrite the current template with whatever is configured
+      # in the `template` and `template_name` directives.
+      mod.config :template_overwrite, :validate => :boolean, :default => false
+
+      # The document ID for the index. Useful for overwriting existing entries in
+      # Elasticsearch with the same ID.
+      mod.config :document_id, :validate => :string
+
+      # A routing override to be applied to all processed events.
+      # This can be dynamic using the `%{foo}` syntax.
+      mod.config :routing, :validate => :string
+
+      # Sets the host(s) of the remote instance. If given an array it will load balance requests across the hosts specified in the `hosts` parameter.
+      # Remember the `http` protocol uses the http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-http.html#modules-http[http] address (eg. 9200, not 9300).
+      #     `"127.0.0.1"`
+      #     `["127.0.0.1:9200","127.0.0.2:9200"]`
+      # It is important to exclude http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html[dedicated master nodes] from the `hosts` list
+      # to prevent LS from sending bulk requests to the master nodes.  So this parameter should only reference either data or client nodes in Elasticsearch.
+      mod.config :hosts, :validate => :array, :default => ["127.0.0.1"]
+
+      mod.config :host, :obsolete => "Please use the 'hosts' setting instead. You can specify multiple entries separated by comma in 'host:port' format."
+
+      # The port setting is obsolete.  Please use the 'hosts' setting instead.
+      # Hosts entries can be in "host:port" format.
+      mod.config :port, :obsolete => "Please use the 'hosts' setting instead. Hosts entries can be in 'host:port' format."
+
+      # This plugin uses the bulk index API for improved indexing performance.
+      # To make efficient bulk API calls, we will buffer a certain number of
+      # events before flushing that out to Elasticsearch. This setting
+      # controls how many events will be buffered before sending a batch
+      # of events. Increasing the `flush_size` has an effect on Logstash's heap size.
+      # Remember to also increase the heap size using `LS_HEAP_SIZE` if you are sending big documents
+      # or have increased the `flush_size` to a higher value.
+      mod.config :flush_size, :validate => :number, :default => 500
+
+      # The amount of time since last flush before a flush is forced.
+      #
+      # This setting helps ensure slow event rates don't get stuck in Logstash.
+      # For example, if your `flush_size` is 100, and you have received 10 events,
+      # and it has been more than `idle_flush_time` seconds since the last flush,
+      # Logstash will flush those 10 events automatically.
+      #
+      # This helps keep both fast and slow log streams moving along in
+      # near-real-time.
+      mod.config :idle_flush_time, :validate => :number, :default => 1
+
+      # Set upsert content for update mode.s
+      # Create a new document with this parameter as json string if `document_id` doesn't exists
+      mod.config :upsert, :validate => :string, :default => ""
+
+      # Enable `doc_as_upsert` for update mode.
+      # Create a new document with source if `document_id` doesn't exist in Elasticsearch
+      mod.config :doc_as_upsert, :validate => :boolean, :default => false
+
+      # Set max retry for each event. The total time spent blocked on retries will be
+      # (max_retries * retry_max_interval). This may vary a bit if Elasticsearch is very slow to respond
+      mod.config :max_retries, :validate => :number, :default => 3
+
+      # Set max interval between bulk retries.
+      mod.config :retry_max_interval, :validate => :number, :default => 2
+
+      # DEPRECATED This setting no longer does anything. If you need to change the number of retries in flight
+      # try increasing the total number of workers to better handle this.
+      mod.config :retry_max_items, :validate => :number, :default => 500, :deprecated => true
+    end
+  end
+end end end

data/lib/logstash/outputs/elasticsearch/elasticsearch-template.json

@@ -5,37 +5,91 @@
   },
   "mappings" : {
     "_default_" : {
-       "_all" : {"enabled" : true, "omit_norms" : true},
-       "dynamic_templates" : [ {
-         "message_field" : {
-           "match" : "message",
-           "match_mapping_type" : "string",
-           "mapping" : {
-             "type" : "string", "index" : "analyzed", "omit_norms" : true
-           }
-         }
-       }, {
-         "string_fields" : {
-           "match" : "*",
-           "match_mapping_type" : "string",
-           "mapping" : {
-             "type" : "string", "index" : "analyzed", "omit_norms" : true,
-               "fields" : {
-                 "raw" : {"type": "string", "index" : "not_analyzed", "ignore_above" : 256}
-               }
-           }
-         }
-       } ],
-       "properties" : {
-         "@version": { "type": "string", "index": "not_analyzed" },
-         "geoip"  : {
-           "type" : "object",
-             "dynamic": true,
-             "properties" : {
-               "location" : { "type" : "geo_point" }
-             }
-         }
-       }
+      "_all" : {"enabled" : true, "omit_norms" : true},
+      "dynamic_templates" : [ {
+        "message_field" : {
+          "match" : "message",
+          "match_mapping_type" : "string",
+          "mapping" : {
+            "type" : "string", "index" : "analyzed", "omit_norms" : true,
+            "fielddata" : { "format" : "disabled" }
+          }
+        }
+      }, {
+        "string_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "string",
+          "mapping" : {
+            "type" : "string", "index" : "analyzed", "omit_norms" : true,
+            "fielddata" : { "format" : "disabled" },
+            "fields" : {
+              "raw" : {"type": "string", "index" : "not_analyzed", "doc_values" : true, "ignore_above" : 256}
+            }
+          }
+        }
+      }, {
+        "float_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "float",
+          "mapping" : { "type" : "float", "doc_values" : true }
+        }
+      }, {
+        "double_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "double",
+          "mapping" : { "type" : "double", "doc_values" : true }
+        }
+      }, {
+        "byte_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "byte",
+          "mapping" : { "type" : "byte", "doc_values" : true }
+        }
+      }, {
+        "short_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "short",
+          "mapping" : { "type" : "short", "doc_values" : true }
+        }
+      }, {
+        "integer_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "integer",
+          "mapping" : { "type" : "integer", "doc_values" : true }
+        }
+      }, {
+        "long_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "long",
+          "mapping" : { "type" : "long", "doc_values" : true }
+        }
+      }, {
+        "date_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "date",
+          "mapping" : { "type" : "date", "doc_values" : true }
+        }
+      }, {
+        "geo_point_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "geo_point",
+          "mapping" : { "type" : "geo_point", "doc_values" : true }
+        }
+      } ],
+      "properties" : {
+        "@timestamp": { "type": "date", "doc_values" : true },
+        "@version": { "type": "string", "index": "not_analyzed", "doc_values" : true },
+        "geoip"  : {
+          "type" : "object",
+          "dynamic": true,
+          "properties" : {
+            "ip": { "type": "ip", "doc_values" : true },
+            "location" : { "type" : "geo_point", "doc_values" : true },
+            "latitude" : { "type" : "float", "doc_values" : true },
+            "longitude" : { "type" : "float", "doc_values" : true }
+          }
+        }
+      }
     }
   }
 }