logstash-output-elasticsearch 2.1.2-java → 2.1.4-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7e2d8a90cde0226696b3a4175335fd83f7ad1679
-  data.tar.gz: 2dd0bdbf2575a60a70c6ebd6a62beb4a5d771c47
+  metadata.gz: 83b2c74b37b6266cc97c1801321c858302faf85d
+  data.tar.gz: f47adea8d8172e63fd433e2e58135dd6657bff07
 SHA512:
-  metadata.gz: ae2d14fea30e7d2c9c8acd7855c17cc4f1751dd06f82b89c7f36819951507bab49f516b9f646c646bdf1ebf93d24b078d5389aa4040309f557b452f9a25ae5fa
-  data.tar.gz: ce3b54cf43ac86b23a918ee6e8199997185f4f4ba97dc7f3c7830b89ed6f061b7b2706c753778a44d526826ec1b3b569ca62ea87da68b7dc1d883caa5f659c15
+  metadata.gz: 0db5b0f53f6611fd0b7a5587d4d4a2764d849c02a984dea4dcf735c4f72986085d03acc0dc5ccd08fdc4817c01509aacc2dc98f52ddbb0dbfd74d297bf158d23
+  data.tar.gz: 07cc0da34006d0c84e756d54ea4938df4fbaa822c7c5a284770b50ad040e8270f5fe8ea021a743777fac00e83f0a9c918114a5ab0b145161a2a753b2d7eaa834

data/CHANGELOG.md

@@ -1,3 +1,16 @@
+## 2.1.4
+ - Improved the default template to disable fielddata on analyzed string fields. #309
+ - Dependend on logstash-core 2.0.0 released version, rather than RC1
+
+## 2.1.3
+ - Improved the default template to use doc_values wherever possible.
+ - Template contains example mappings for every numeric type. You must map your
+   own fields to make use of anything other than long and double.
+
+## 2.1.2
+ - Fixed dependencies (#280)
+ - Fixed an RSpec test (#281)
+
 ## 2.1.1
  - Made host config obsolete.
 
@@ -7,7 +20,7 @@
    firewall, or load balancer issues.
 
 ## 2.0.0
- - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
    instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
  - Dependency on logstash-core update to 2.0
 
@@ -20,7 +33,7 @@
 ## 2.0.0-beta
  - Only support HTTP Protocol
  - Removed support for node and transport protocols (now in logstash-output-elasticsearch_java)
- 
+
 ## 1.0.7
  - Add update API support
 
@@ -31,7 +44,7 @@
  - Update to Elasticsearch 1.7
 
 ## 1.0.3
- - Add HTTP proxy support 
+ - Add HTTP proxy support
 
 ## 1.0.2
  - Upgrade Manticore HTTP Client

data/Gemfile

@@ -1,3 +1,3 @@
 source 'https://rubygems.org'
 
-gemspec
+gemspec

data/README.md

@@ -1,5 +1,7 @@
 # Logstash Plugin
 
+ATTN: This plugin is undergoing a [major refactor](https://github.com/logstash-plugins/logstash-output-elasticsearch/issues/291)! Any refactors started now may need to be rebased/refactored before they are accepted!
+
 [![Build
 Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Outputs/job/logstash-plugin-output-elasticsearch-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Outputs/job/logstash-plugin-output-elasticsearch-unit/)
 

data/lib/logstash/outputs/elasticsearch.rb

@@ -8,7 +8,6 @@ require "stud/buffer"
 require "socket" # for Socket.gethostname
 require "thread" # for safe queueing
 require "uri" # for escaping user input
-require "logstash/outputs/elasticsearch/http_client"
 
 # This plugin is the recommended method of storing logs in Elasticsearch.
 # If you plan on using the Kibana web interface, you'll want to use this output.
@@ -38,104 +37,30 @@ require "logstash/outputs/elasticsearch/http_client"
 # - Events from the retry queue are submitted again when the queue reaches its max size or when
 #   the max interval time is reached. The max interval time is configurable via :retry_max_interval.
 # - Events which are not retryable or have reached their max retry count are logged to stderr.
+#
+# ==== DNS Caching
+#
+# This plugin uses the JVM to lookup DNS entries and is subject to the value of https://docs.oracle.com/javase/7/docs/technotes/guides/net/properties.html[networkaddress.cache.ttl],
+# a global setting for the JVM.
+#
+# As an example, to set your DNS TTL to 1 second you would set
+# the `LS_JAVA_OPTS` environment variable to `-Dnetwordaddress.cache.ttl=1`.
+#
+# Keep in mind that a connection with keepalive enabled will
+# not reevaluate its DNS value while the keepalive is in effect.
 class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
-  attr_reader :client
-
-  include Stud::Buffer
-  RETRYABLE_CODES = [409, 429, 503]
-  SUCCESS_CODES = [200, 201]
-
-  config_name "elasticsearch"
-
-  # The index to write events to. This can be dynamic using the `%{foo}` syntax.
-  # The default value will partition your indices by day so you can more easily
-  # delete old data or only search specific date ranges.
-  # Indexes may not contain uppercase characters.
-  # For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}
-  config :index, :validate => :string, :default => "logstash-%{+YYYY.MM.dd}"
-
-  # The index type to write events to. Generally you should try to write only
-  # similar events to the same 'type'. String expansion `%{foo}` works here.
-  #
-  # Deprecated in favor of `document_type` field.
-  config :index_type, :validate => :string, :deprecated => "Please use the 'document_type' setting instead. It has the same effect, but is more appropriately named."
-
-  # The document type to write events to. Generally you should try to write only
-  # similar events to the same 'type'. String expansion `%{foo}` works here.
-  # Unless you set 'document_type', the event 'type' will be used if it exists
-  # otherwise the document type will be assigned the value of 'logs'
-  config :document_type, :validate => :string
-
-  # Starting in Logstash 1.3 (unless you set option `manage_template` to false)
-  # a default mapping template for Elasticsearch will be applied, if you do not
-  # already have one set to match the index pattern defined (default of
-  # `logstash-%{+YYYY.MM.dd}`), minus any variables.  For example, in this case
-  # the template will be applied to all indices starting with `logstash-*`
-  #
-  # If you have dynamic templating (e.g. creating indices based on field names)
-  # then you should set `manage_template` to false and use the REST API to upload
-  # your templates manually.
-  config :manage_template, :validate => :boolean, :default => true
-
-  # This configuration option defines how the template is named inside Elasticsearch.
-  # Note that if you have used the template management features and subsequently
-  # change this, you will need to prune the old template manually, e.g.
-  #
-  # `curl -XDELETE <http://localhost:9200/_template/OldTemplateName?pretty>`
-  #
-  # where `OldTemplateName` is whatever the former setting was.
-  config :template_name, :validate => :string, :default => "logstash"
-
-  # You can set the path to your own template here, if you so desire.
-  # If not set, the included template will be used.
-  config :template, :validate => :path
-
-  # Overwrite the current template with whatever is configured
-  # in the `template` and `template_name` directives.
-  config :template_overwrite, :validate => :boolean, :default => false
-
-  # The document ID for the index. Useful for overwriting existing entries in
-  # Elasticsearch with the same ID.
-  config :document_id, :validate => :string
-
-  # A routing override to be applied to all processed events.
-  # This can be dynamic using the `%{foo}` syntax.
-  config :routing, :validate => :string
-
-  # Sets the host(s) of the remote instance. If given an array it will load balance requests across the hosts specified in the `hosts` parameter.
-  # Remember the `http` protocol uses the http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-http.html#modules-http[http] address (eg. 9200, not 9300).
-  #     `"127.0.0.1"`
-  #     `["127.0.0.1:9200","127.0.0.2:9200"]`
-  # It is important to exclude http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html[dedicated master nodes] from the `hosts` list
-  # to prevent LS from sending bulk requests to the master nodes.  So this parameter should only reference either data or client nodes in Elasticsearch.
-
-  config :hosts, :validate => :array
+  require "logstash/outputs/elasticsearch/http_client"
+  require "logstash/outputs/elasticsearch/http_client_builder"
+  require "logstash/outputs/elasticsearch/common_configs"
+  require "logstash/outputs/elasticsearch/common"
 
-  config :host, :obsolete => "Please use the 'hosts' setting instead. You can specify multiple entries separated by comma in 'host:port' format."
+  # Protocol agnostic (i.e. non-http, non-java specific) configs go here
+  include(LogStash::Outputs::ElasticSearch::CommonConfigs)
 
-  # The port setting is obsolete.  Please use the 'hosts' setting instead.
-  # Hosts entries can be in "host:port" format.
-  config :port, :obsolete => "Please use the 'hosts' setting instead. Hosts entries can be in 'host:port' format."
+  # Protocol agnostic methods
+  include(LogStash::Outputs::ElasticSearch::Common)
 
-  # This plugin uses the bulk index API for improved indexing performance.
-  # To make efficient bulk API calls, we will buffer a certain number of
-  # events before flushing that out to Elasticsearch. This setting
-  # controls how many events will be buffered before sending a batch
-  # of events. Increasing the `flush_size` has an effect on Logstash's heap size.
-  # Remember to also increase the heap size using `LS_HEAP_SIZE` if you are sending big documents
-  # or have increased the `flush_size` to a higher value.
-  config :flush_size, :validate => :number, :default => 500
-
-  # The amount of time since last flush before a flush is forced.
-  #
-  # This setting helps ensure slow event rates don't get stuck in Logstash.
-  # For example, if your `flush_size` is 100, and you have received 10 events,
-  # and it has been more than `idle_flush_time` seconds since the last flush,
-  # Logstash will flush those 10 events automatically.
-  #
-  # This helps keep both fast and slow log streams moving along in
-  # near-real-time.
-  config :idle_flush_time, :validate => :number, :default => 1
+  config_name "elasticsearch"
 
   # The Elasticsearch action to perform. Valid actions are:
   #
@@ -192,339 +117,24 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # How long to wait, in seconds, between sniffing attempts
   config :sniffing_delay, :validate => :number, :default => 5
 
-  # Set max retry for each event
-  config :max_retries, :validate => :number, :default => 3
-
-  # Set retry policy for events that failed to send
-  config :retry_max_items, :validate => :number, :default => 5000
-
-  # Set max interval between bulk retries
-  config :retry_max_interval, :validate => :number, :default => 5
-
   # Set the address of a forward HTTP proxy.
   # Can be either a string, such as `http://localhost:123` or a hash in the form
   # of `{host: 'proxy.org' port: 80 scheme: 'http'}`.
   # Note, this is NOT a SOCKS proxy, but a plain HTTP proxy
   config :proxy
 
-  # Enable `doc_as_upsert` for update mode.
-  # Create a new document with source if `document_id` doesn't exist in Elasticsearch
-  config :doc_as_upsert, :validate => :boolean, :default => false
-
-  # Set upsert content for update mode.
-  # Create a new document with this parameter as json string if `document_id` doesn't exists
-  config :upsert, :validate => :string, :default => ""
-
   # Set the timeout for network operations and requests sent Elasticsearch. If
   # a timeout occurs, the request will be retried.
   config :timeout, :validate => :number
 
-  public
-  def register
-    @hosts = Array(@hosts)
-    # retry-specific variables
-    @retry_flush_mutex = Mutex.new
-    @retry_close_requested = Concurrent::AtomicBoolean.new(false)
-    # needs flushing when interval
-    @retry_queue_needs_flushing = ConditionVariable.new
-    @retry_queue_not_full = ConditionVariable.new
-    @retry_queue = Queue.new
-    @submit_mutex = Mutex.new
-
-    client_settings = {}
-    common_options = {
-      :client_settings => client_settings,
-      :sniffing => @sniffing,
-      :sniffing_delay => @sniffing_delay
-    }
-
-    common_options[:timeout] = @timeout if @timeout
-    client_settings[:path] = "/#{@path}/".gsub(/\/+/, "/") # Normalize slashes
-    @logger.debug? && @logger.debug("Normalizing http path", :path => @path, :normalized => client_settings[:path])
-
-    if @hosts.nil? || @hosts.empty?
-      @logger.info("No 'host' set in elasticsearch output. Defaulting to localhost")
-      @hosts = ["localhost"]
-    end
-
-    client_settings.merge! setup_ssl()
-    client_settings.merge! setup_proxy()
-    common_options.merge! setup_basic_auth()
-
-    # Update API setup
-    update_options = {
-      :upsert => @upsert,
-      :doc_as_upsert => @doc_as_upsert
-    }
-    common_options.merge! update_options if @action == 'update'
-
-    @client = LogStash::Outputs::Elasticsearch::HttpClient.new(
-      common_options.merge(:hosts => @hosts, :logger => @logger)
-    )
-
-    if @manage_template
-      begin
-        @logger.info("Automatic template management enabled", :manage_template => @manage_template.to_s)
-        @client.template_install(@template_name, get_template, @template_overwrite)
-      rescue => e
-        @logger.error("Failed to install template: #{e.message}")
-      end
-    end
-
-    @logger.info("New Elasticsearch output", :hosts => @hosts)
-
-    @client_idx = 0
-
-    buffer_initialize(
-      :max_items => @flush_size,
-      :max_interval => @idle_flush_time,
-      :logger => @logger
-    )
-
-    @retry_timer_thread = Thread.new do
-      loop do
-        sleep(@retry_max_interval)
-        @retry_flush_mutex.synchronize { @retry_queue_needs_flushing.signal }
-      end
-    end
-
-    @retry_thread = Thread.new do
-      while @retry_close_requested.false?
-        @retry_flush_mutex.synchronize { @retry_queue_needs_flushing.wait(@retry_flush_mutex) }
-        retry_flush
-      end
-    end
-  end # def register
-
-  public
-  def get_template
-    if @template.nil?
-      @template = ::File.expand_path('elasticsearch/elasticsearch-template.json', ::File.dirname(__FILE__))
-      if !File.exists?(@template)
-        raise "You must specify 'template => ...' in your elasticsearch output (I looked for '#{@template}')"
-      end
-    end
-    template_json = IO.read(@template).gsub(/\n/,'')
-    template = LogStash::Json.load(template_json)
-    @logger.info("Using mapping template", :template => template)
-    return template
-  end # def get_template
-
-  public
-  def receive(event)
-
-    # block until we have not maxed out our
-    # retry queue. This is applying back-pressure
-    # to slow down the receive-rate
-    @retry_flush_mutex.synchronize {
-      @retry_queue_not_full.wait(@retry_flush_mutex) while @retry_queue.size > @retry_max_items
-    }
-
-    event['@metadata']['retry_count'] = 0
-
-    # Set the 'type' value for the index.
-    type = if @document_type
-             event.sprintf(@document_type)
-           elsif @index_type # deprecated
-             event.sprintf(@index_type)
-           else
-             event["type"] || "logs"
-           end
-
-    params = {
-      :_id => @document_id ? event.sprintf(@document_id) : nil,
-      :_index => event.sprintf(@index),
-      :_type => type,
-      :_routing => @routing ? event.sprintf(@routing) : nil
-    }
-
-    params[:_upsert] = LogStash::Json.load(event.sprintf(@upsert)) if @action == 'update' && @upsert != ""
-
-    buffer_receive([event.sprintf(@action), params, event])
-  end # def receive
-
-  public
-  # The submit method can be called from both the
-  # Stud::Buffer flush thread and from our own retry thread.
-  def submit(actions)
-    @submit_mutex.synchronize do
-      es_actions = actions.map { |a, doc, event| [a, doc, event.to_hash] }
-
-      bulk_response = @client.bulk(es_actions)
-
-      next unless bulk_response["errors"]
-
-      actions_to_retry = []
-
-      bulk_response["items"].each_with_index do |resp,idx|
-        action_type, action_props = resp.first
-
-        status = action_props["status"]
-        action = actions[idx]
-
-        if RETRYABLE_CODES.include?(status)
-          @logger.warn "retrying failed action with response code: #{status}"
-          actions_to_retry << action
-        elsif not SUCCESS_CODES.include?(status)
-          @logger.warn "Failed action. ", status: status, action: action, response: resp
-        end
-      end
-
-      retry_push(actions_to_retry) unless actions_to_retry.empty?
-    end
+  def build_client
+    @client = ::LogStash::Outputs::ElasticSearch::HttpClientBuilder.build(@logger, @hosts, params)
   end
 
-  # When there are exceptions raised upon submission, we raise an exception so that
-  # Stud::Buffer will retry to flush
-  public
-  def flush(actions, close = false)
-    begin
-      submit(actions)
-    rescue Manticore::SocketException => e
-      # If we can't even connect to the server let's just print out the URL (:hosts is actually a URL)
-      # and let the user sort it out from there
-      @logger.error(
-        "Attempted to send a bulk request to Elasticsearch configured at '#{@client.client_options[:hosts]}',"+
-          " but Elasticsearch appears to be unreachable or down!",
-        :client_config => @client.client_options,
-        :error_message => e.message
-      )
-      @logger.debug("Failed actions for last bad bulk request!", :actions => actions)
-    rescue => e
-      # For all other errors print out full connection issues
-      @logger.error(
-        "Attempted to send a bulk request to Elasticsearch configured at '#{@client.client_options[:hosts]}'," +
-            " but an error occurred and it failed! Are you sure you can reach elasticsearch from this machine using " +
-          "the configuration provided?",
-        :client_config => @client.client_options,
-        :error_message => e.message,
-        :error_class => e.class.name,
-        :backtrace => e.backtrace
-      )
-
-      @logger.debug("Failed actions for last bad bulk request!", :actions => actions)
-
-      raise e
-    end
-  end # def flush
-
-  public
   def close
+    @stopping.make_true
     @client.stop_sniffing!
-
-    @retry_close_requested.make_true
-    # First, make sure retry_timer_thread is stopped
-    # to ensure we do not signal a retry based on
-    # the retry interval.
-    Thread.kill(@retry_timer_thread)
-    @retry_timer_thread.join
-    # Signal flushing in the case that #retry_flush is in
-    # the process of waiting for a signal.
-    @retry_flush_mutex.synchronize { @retry_queue_needs_flushing.signal }
-    # Now, #retry_flush is ensured to not be in a state of
-    # waiting and can be safely joined into the main thread
-    # for further final execution of an in-process remaining call.
-    @retry_thread.join
-
-    # execute any final actions along with a proceeding retry for any
-    # final actions that did not succeed.
-    buffer_flush(:final => true)
-    retry_flush
-  end
-
-  private
-  def setup_proxy
-    return {} unless @proxy
-
-    # Symbolize keys
-    proxy = if @proxy.is_a?(Hash)
-              Hash[@proxy.map {|k,v| [k.to_sym, v]}]
-            elsif @proxy.is_a?(String)
-              @proxy
-            else
-              raise LogStash::ConfigurationError, "Expected 'proxy' to be a string or hash, not '#{@proxy}''!"
-            end
-
-    return {:proxy => proxy}
-  end
-
-  private
-  def setup_ssl
-    return {} unless @ssl
-
-    if @cacert && @truststore
-      raise(LogStash::ConfigurationError, "Use either \"cacert\" or \"truststore\" when configuring the CA certificate") if @truststore
-    end
-
-    ssl_options = {}
-
-    if @cacert
-      ssl_options[:ca_file] = @cacert
-    elsif @truststore
-      ssl_options[:truststore_password] = @truststore_password.value if @truststore_password
-    end
-
-    ssl_options[:truststore] = @truststore if @truststore
-    if @keystore
-      ssl_options[:keystore] = @keystore
-      ssl_options[:keystore_password] = @keystore_password.value if @keystore_password
-    end
-    if @ssl_certificate_verification == false
-      @logger.warn [
-        "** WARNING ** Detected UNSAFE options in elasticsearch output configuration!",
-        "** WARNING ** You have enabled encryption but DISABLED certificate verification.",
-        "** WARNING ** To make sure your data is secure change :ssl_certificate_verification to true"
-      ].join("\n")
-      ssl_options[:verify] = false
-    end
-    { ssl: ssl_options }
-  end
-
-  private
-  def setup_basic_auth
-    return {} unless @user && @password
-
-    {
-      :user => ::URI.escape(@user, "@:"),
-      :password => ::URI.escape(@password.value, "@:")
-    }
-  end
-
-  private
-  # in charge of submitting any actions in @retry_queue that need to be
-  # retried
-  #
-  # This method is not called concurrently. It is only called by @retry_thread
-  # and once that thread is ended during the close process, a final call
-  # to this method is done upon close in the main thread.
-  def retry_flush()
-    unless @retry_queue.empty?
-      buffer = @retry_queue.size.times.map do
-        next_action, next_doc, next_event = @retry_queue.pop
-        next_event['@metadata']['retry_count'] += 1
-
-        if next_event['@metadata']['retry_count'] > @max_retries
-          @logger.error "too many attempts at sending event. dropping: #{next_event}"
-          nil
-        else
-          [next_action, next_doc, next_event]
-        end
-      end.compact
-
-      submit(buffer) unless buffer.empty?
-    end
-
-    @retry_flush_mutex.synchronize {
-      @retry_queue_not_full.signal if @retry_queue.size < @retry_max_items
-    }
-  end
-
-  private
-  def retry_push(actions)
-    Array(actions).each{|action| @retry_queue << action}
-    @retry_flush_mutex.synchronize {
-      @retry_queue_needs_flushing.signal if @retry_queue.size >= @retry_max_items
-    }
+    @buffer.stop
   end
 
   @@plugins = Gem::Specification.find_all{|spec| spec.name =~ /logstash-output-elasticsearch-/ }