logstash-output-elasticsearch 0.1.6 → 3.0.0

data/lib/logstash/outputs/elasticsearch/buffer.rb

@@ -0,0 +1,124 @@
+require 'concurrent'
+java_import java.util.concurrent.locks.ReentrantLock
+
+module LogStash; module Outputs; class ElasticSearch
+  class Buffer
+    def initialize(logger, max_size, flush_interval, &block)
+      @logger = logger
+      # You need to aquire this for anything modifying state generally
+      @operations_mutex = Mutex.new
+      @operations_lock = java.util.concurrent.locks.ReentrantLock.new
+
+      @stopping = Concurrent::AtomicBoolean.new(false)
+      @max_size = max_size
+      @submit_proc = block
+
+      @buffer = []
+
+      @last_flush = Time.now
+      @flush_interval = flush_interval
+      @flush_thread = spawn_interval_flusher
+    end
+
+    def push(item)
+      synchronize do |buffer|
+        push_unsafe(item)
+      end
+    end
+    alias_method :<<, :push
+
+    # Push multiple items onto the buffer in a single operation
+    def push_multi(items)
+      raise ArgumentError, "push multi takes an array!, not an #{items.class}!" unless items.is_a?(Array)
+      synchronize do |buffer|
+        items.each {|item| push_unsafe(item) }
+      end
+    end
+
+    def flush
+      synchronize { flush_unsafe }
+    end
+
+    def stop(do_flush=true,wait_complete=true)
+      return if stopping?
+      @stopping.make_true
+
+      # No need to acquire a lock in this case
+      return if !do_flush && !wait_complete
+
+      synchronize do
+        flush_unsafe if do_flush
+        @flush_thread.join if wait_complete
+      end
+    end
+
+    def contents
+      synchronize {|buffer| buffer}
+    end
+
+    # For externally operating on the buffer contents
+    # this takes a block and will yield the internal buffer and executes
+    # the block in a synchronized block from the internal mutex
+    def synchronize
+      @operations_mutex.synchronize { yield(@buffer) }
+    end
+
+    # These methods are private for various reasons, chief among them threadsafety!
+    # Many require the @operations_mutex to be locked to be safe
+    private
+
+    def push_unsafe(item)
+      @buffer << item
+      if @buffer.size >= @max_size
+        flush_unsafe
+      end
+    end
+
+    def spawn_interval_flusher
+      Thread.new do
+        loop do
+          sleep 0.2
+          break if stopping?
+          synchronize { interval_flush }
+        end
+      end
+    end
+
+    def interval_flush
+      if last_flush_seconds_ago >= @flush_interval
+        begin
+          @logger.debug? && @logger.debug("Flushing buffer at interval",
+                                        :instance => self.inspect,
+                                        :interval => @flush_interval)
+          flush_unsafe
+        rescue StandardError => e
+          @logger.warn("Error flushing buffer at interval!",
+                       :instance => self.inspect,
+                       :message => e.message,
+                       :class => e.class.name,
+                       :backtrace => e.backtrace
+          )
+        rescue Exception => e
+          @logger.warn("Exception flushing buffer at interval!", :error => e.message, :class => e.class.name)
+        end
+      end
+    end
+
+    def flush_unsafe
+      if @buffer.size > 0
+        @submit_proc.call(@buffer)
+        @buffer.clear
+      end
+
+      @last_flush = Time.now # This must always be set to ensure correct timer behavior
+    end
+
+    def last_flush_seconds_ago
+      Time.now - @last_flush
+    end
+
+    def stopping?
+      @stopping.true?
+    end
+  end
+end end end

data/lib/logstash/outputs/elasticsearch/common.rb

@@ -0,0 +1,205 @@
+require "logstash/outputs/elasticsearch/template_manager"
+require "logstash/outputs/elasticsearch/buffer"
+
+module LogStash; module Outputs; class ElasticSearch;
+  module Common
+    attr_reader :client, :hosts
+
+    RETRYABLE_CODES = [429, 503]
+    SUCCESS_CODES = [200, 201]
+
+    def register
+      @stopping = Concurrent::AtomicBoolean.new(false)
+      setup_hosts # properly sets @hosts
+      build_client
+      install_template
+      setup_buffer_and_handler
+      check_action_validity
+
+      @logger.info("New Elasticsearch output", :class => self.class.name, :hosts => @hosts)
+    end
+
+    def receive(event)
+      @buffer << event_action_tuple(event)
+    end
+
+    # Receive an array of events and immediately attempt to index them (no buffering)
+    def multi_receive(events)
+      events.each_slice(@flush_size) do |slice|
+        retrying_submit(slice.map {|e| event_action_tuple(e) })
+      end
+    end
+
+    # Convert the event into a 3-tuple of action, params, and event
+    def event_action_tuple(event)
+      params = event_action_params(event)
+      action = event.sprintf(@action)
+      [action, params, event]
+    end
+
+    def flush
+      @buffer.flush
+    end
+
+    def setup_hosts
+      @hosts = Array(@hosts)
+      if @hosts.empty?
+        @logger.info("No 'host' set in elasticsearch output. Defaulting to localhost")
+        @hosts.replace(["localhost"])
+      end
+    end
+
+    def install_template
+      TemplateManager.install_template(self)
+    end
+
+    def setup_buffer_and_handler
+      @buffer = ::LogStash::Outputs::ElasticSearch::Buffer.new(@logger, @flush_size, @idle_flush_time) do |actions|
+        retrying_submit(actions)
+      end
+    end
+
+    def check_action_validity
+      raise LogStash::ConfigurationError, "No action specified!" unless @action
+
+      # If we're using string interpolation, we're good!
+      return if @action =~ /%{.+}/
+      return if valid_actions.include?(@action)
+
+      raise LogStash::ConfigurationError, "Action '#{@action}' is invalid! Pick one of #{valid_actions} or use a sprintf style statement"
+    end
+
+    # To be overidden by the -java version
+    VALID_HTTP_ACTIONS=["index", "delete", "create", "update"]
+    def valid_actions
+      VALID_HTTP_ACTIONS
+    end
+
+    def retrying_submit(actions)
+      # Initially we submit the full list of actions
+      submit_actions = actions
+
+      while submit_actions && submit_actions.length > 0
+        return if !submit_actions || submit_actions.empty? # If everything's a success we move along
+        # We retry with whatever is didn't succeed
+        begin
+          submit_actions = submit(submit_actions)
+        rescue => e
+          @logger.warn("Encountered an unexpected error submitting a bulk request! Will retry.",
+                       :message => e.message,
+                       :class => e.class.name,
+                       :backtrace => e.backtrace)
+        end
+
+        sleep @retry_max_interval if submit_actions && submit_actions.length > 0
+      end
+    end
+
+    def submit(actions)
+      es_actions = actions.map { |a, doc, event| [a, doc, event.to_hash]}
+
+      bulk_response = safe_bulk(es_actions,actions)
+
+      # If there are no errors, we're done here!
+      return unless bulk_response["errors"]
+
+      actions_to_retry = []
+      bulk_response["items"].each_with_index do |response,idx|
+        action_type, action_props = response.first
+        status = action_props["status"]
+        error  = action_props["error"]
+        action = actions[idx]
+
+        if SUCCESS_CODES.include?(status)
+          next
+        elsif RETRYABLE_CODES.include?(status)
+          @logger.info "retrying failed action with response code: #{status} (#{error})"
+          actions_to_retry << action
+        else
+          @logger.warn "Failed action. ", status: status, action: action, response: response
+        end
+      end
+
+      actions_to_retry
+    end
+
+    # get the action parameters for the given event
+    def event_action_params(event)
+      type = get_event_type(event)
+
+      params = {
+        :_id => @document_id ? event.sprintf(@document_id) : nil,
+        :_index => event.sprintf(@index),
+        :_type => type,
+        :_routing => @routing ? event.sprintf(@routing) : nil
+      }
+
+      if @pipeline
+        params[:pipeline] = @pipeline
+      end
+
+     if @parent
+        params[:parent] = event.sprintf(@parent)
+      end
+
+      if @action == 'update'
+        params[:_upsert] = LogStash::Json.load(event.sprintf(@upsert)) if @upsert != ""
+        params[:_script] = event.sprintf(@script) if @script != ""
+        params[:_retry_on_conflict] = @retry_on_conflict
+      end
+
+      params
+    end
+
+    # Determine the correct value for the 'type' field for the given event
+    def get_event_type(event)
+      # Set the 'type' value for the index.
+      type = if @document_type
+               event.sprintf(@document_type)
+             else
+               event.get("type") || "logs"
+             end
+
+      if !(type.is_a?(String) || type.is_a?(Numeric))
+        @logger.warn("Bad event type! Non-string/integer type value set!", :type_class => type.class, :type_value => type.to_s, :event => event)
+      end
+
+      type.to_s
+    end
+
+    # Rescue retryable errors during bulk submission
+    def safe_bulk(es_actions,actions)
+      @client.bulk(es_actions)
+    rescue Manticore::SocketException, Manticore::SocketTimeout => e
+      # If we can't even connect to the server let's just print out the URL (:hosts is actually a URL)
+      # and let the user sort it out from there
+      @logger.error(
+        "Attempted to send a bulk request to Elasticsearch configured at '#{@client.client_options[:hosts]}',"+
+          " but Elasticsearch appears to be unreachable or down!",
+        :error_message => e.message,
+        :class => e.class.name,
+        :client_config => @client.client_options,
+      )
+      @logger.debug("Failed actions for last bad bulk request!", :actions => actions)
+
+      # We retry until there are no errors! Errors should all go to the retry queue
+      sleep @retry_max_interval
+      retry unless @stopping.true?
+    rescue => e
+      # For all other errors print out full connection issues
+      @logger.error(
+        "Attempted to send a bulk request to Elasticsearch configured at '#{@client.client_options[:hosts]}'," +
+          " but an error occurred and it failed! Are you sure you can reach elasticsearch from this machine using " +
+          "the configuration provided?",
+        :error_message => e.message,
+        :error_class => e.class.name,
+        :backtrace => e.backtrace,
+        :client_config => @client.client_options,
+      )
+
+      @logger.debug("Failed actions for last bad bulk request!", :actions => actions)
+
+      raise e
+    end
+  end
+end; end; end

data/lib/logstash/outputs/elasticsearch/common_configs.rb

@@ -0,0 +1,164 @@
+module LogStash; module Outputs; class ElasticSearch
+  module CommonConfigs
+    def self.included(mod)
+      # The index to write events to. This can be dynamic using the `%{foo}` syntax.
+      # The default value will partition your indices by day so you can more easily
+      # delete old data or only search specific date ranges.
+      # Indexes may not contain uppercase characters.
+      # For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}.
+      # LS uses Joda to format the index pattern from event timestamp. 
+      # Joda formats are defined http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[here]. 
+      mod.config :index, :validate => :string, :default => "logstash-%{+YYYY.MM.dd}"
+
+      # The index type to write events to. Generally you should try to write only
+      # similar events to the same 'type'. String expansion `%{foo}` works here.
+      #
+      # Deprecated in favor of `docoument_type` field.
+      mod.config :index_type, :validate => :string, :obsolete => "Please use the 'document_type' setting instead. It has the same effect, but is more appropriately named."
+
+      # The document type to write events to. Generally you should try to write only
+      # similar events to the same 'type'. String expansion `%{foo}` works here.
+      # Unless you set 'document_type', the event 'type' will be used if it exists
+      # otherwise the document type will be assigned the value of 'logs'
+      mod.config :document_type, :validate => :string
+
+      # Starting in Logstash 1.3 (unless you set option `manage_template` to false)
+      # a default mapping template for Elasticsearch will be applied, if you do not
+      # already have one set to match the index pattern defined (default of
+      # `logstash-%{+YYYY.MM.dd}`), minus any variables.  For example, in this case
+      # the template will be applied to all indices starting with `logstash-*`
+      #
+      # If you have dynamic templating (e.g. creating indices based on field names)
+      # then you should set `manage_template` to false and use the REST API to upload
+      # your templates manually.
+      mod.config :manage_template, :validate => :boolean, :default => true
+
+      # This configuration option defines how the template is named inside Elasticsearch.
+      # Note that if you have used the template management features and subsequently
+      # change this, you will need to prune the old template manually, e.g.
+      #
+      # `curl -XDELETE <http://localhost:9200/_template/OldTemplateName?pretty>`
+      #
+      # where `OldTemplateName` is whatever the former setting was.
+      mod.config :template_name, :validate => :string, :default => "logstash"
+
+      # You can set the path to your own template here, if you so desire.
+      # If not set, the included template will be used.
+      mod.config :template, :validate => :path
+
+      # The template_overwrite option will always overwrite the indicated template
+      # in Elasticsearch with either the one indicated by template or the included one.
+      # This option is set to false by default. If you always want to stay up to date
+      # with the template provided by Logstash, this option could be very useful to you.
+      # Likewise, if you have your own template file managed by puppet, for example, and
+      # you wanted to be able to update it regularly, this option could help there as well.
+      #
+      # Please note that if you are using your own customized version of the Logstash
+      # template (logstash), setting this to true will make Logstash to overwrite
+      # the "logstash" template (i.e. removing all customized settings)
+      mod.config :template_overwrite, :validate => :boolean, :default => false
+
+      # The document ID for the index. Useful for overwriting existing entries in
+      # Elasticsearch with the same ID.
+      mod.config :document_id, :validate => :string
+
+      # A routing override to be applied to all processed events.
+      # This can be dynamic using the `%{foo}` syntax.
+      mod.config :routing, :validate => :string
+
+      # For child documents, ID of the associated parent.
+      # This can be dynamic using the `%{foo}` syntax.
+      mod.config :parent, :validate => :string, :default => nil
+
+      # Sets the host(s) of the remote instance. If given an array it will load balance requests across the hosts specified in the `hosts` parameter.
+      # Remember the `http` protocol uses the http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-http.html#modules-http[http] address (eg. 9200, not 9300).
+      #     `"127.0.0.1"`
+      #     `["127.0.0.1:9200","127.0.0.2:9200"]`
+      #     `["http://127.0.0.1"]`
+      #     `["https://127.0.0.1:9200"]`
+      #     `["https://127.0.0.1:9200/mypath"]` (If using a proxy on a subpath)
+      # It is important to exclude http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html[dedicated master nodes] from the `hosts` list
+      # to prevent LS from sending bulk requests to the master nodes.  So this parameter should only reference either data or client nodes in Elasticsearch.
+      mod.config :hosts, :validate => :array, :default => ["127.0.0.1"]
+
+      mod.config :host, :obsolete => "Please use the 'hosts' setting instead. You can specify multiple entries separated by comma in 'host:port' format."
+
+      # The port setting is obsolete.  Please use the 'hosts' setting instead.
+      # Hosts entries can be in "host:port" format.
+      mod.config :port, :obsolete => "Please use the 'hosts' setting instead. Hosts entries can be in 'host:port' format."
+
+      # This plugin uses the bulk index API for improved indexing performance.
+      # In Logstashes >= 2.2 this setting defines the maximum sized bulk request Logstash will make
+      # You you may want to increase this to be in line with your pipeline's batch size.
+      # If you specify a number larger than the batch size of your pipeline it will have no effect,
+      # save for the case where a filter increases the size of an inflight batch by outputting
+      # events.
+      #
+      # In Logstashes <= 2.1 this plugin uses its own internal buffer of events.
+      # This config option sets that size. In these older logstashes this size may
+      # have a significant impact on heap usage, whereas in 2.2+ it will never increase it.
+      # To make efficient bulk API calls, we will buffer a certain number of
+      # events before flushing that out to Elasticsearch. This setting
+      # controls how many events will be buffered before sending a batch
+      # of events. Increasing the `flush_size` has an effect on Logstash's heap size.
+      # Remember to also increase the heap size using `LS_HEAP_SIZE` if you are sending big documents
+      # or have increased the `flush_size` to a higher value.
+      mod.config :flush_size, :validate => :number, :default => 500
+
+      # The amount of time since last flush before a flush is forced.
+      #
+      # This setting helps ensure slow event rates don't get stuck in Logstash.
+      # For example, if your `flush_size` is 100, and you have received 10 events,
+      # and it has been more than `idle_flush_time` seconds since the last flush,
+      # Logstash will flush those 10 events automatically.
+      #
+      # This helps keep both fast and slow log streams moving along in
+      # near-real-time.
+      mod.config :idle_flush_time, :validate => :number, :default => 1
+
+      # Set upsert content for update mode.s
+      # Create a new document with this parameter as json string if `document_id` doesn't exists
+      mod.config :upsert, :validate => :string, :default => ""
+
+      # Enable `doc_as_upsert` for update mode.
+      # Create a new document with source if `document_id` doesn't exist in Elasticsearch
+      mod.config :doc_as_upsert, :validate => :boolean, :default => false
+
+      # DEPRECATED This setting no longer does anything. It will be marked obsolete in a future version.
+      mod.config :max_retries, :validate => :number, :default => 3
+
+      # Set script name for scripted update mode
+      mod.config :script, :validate => :string, :default => ""
+
+      # Define the type of script referenced by "script" variable
+      #  inline : "script" contains inline script
+      #  indexed : "script" contains the name of script directly indexed in elasticsearch
+      #  file    : "script" contains the name of script stored in elasticseach's config directory
+      mod.config :script_type, :validate => ["inline", 'indexed', "file"], :default => ["inline"]
+
+      # Set the language of the used script
+      mod.config :script_lang, :validate => :string, :default => ""
+
+      # Set variable name passed to script (scripted update)
+      mod.config :script_var_name, :validate => :string, :default => "event"
+
+      # if enabled, script is in charge of creating non-existent document (scripted update)
+      mod.config :scripted_upsert, :validate => :boolean, :default => false
+
+      # Set max interval between bulk retries.
+      mod.config :retry_max_interval, :validate => :number, :default => 2
+
+      # DEPRECATED This setting no longer does anything. If you need to change the number of retries in flight
+      # try increasing the total number of workers to better handle this.
+      mod.config :retry_max_items, :validate => :number, :default => 500, :deprecated => true
+
+      # The number of times Elasticsearch should internally retry an update/upserted document
+      # See the https://www.elastic.co/guide/en/elasticsearch/guide/current/partial-updates.html[partial updates]
+      # for more info
+      mod.config :retry_on_conflict, :validate => :number, :default => 1
+
+      # Set which ingest pipeline you wish to execute for an event
+      mod.config :pipeline, :validate => :string, :default => nil
+    end
+  end
+end end end