logstash-output-elasticsearch_java 2.0.2 → 2.1.1.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a5467c0eb6987bbc644c21942c60480d27521b2a
-  data.tar.gz: 5bbd2ada48fffbfa2594875177982b9245c5b233
+  metadata.gz: 5cebb6798c2db7392cf7a8a7957e37b7290306d5
+  data.tar.gz: ad5f55d2a1b7de142b35c7b69262f7d501867b0f
 SHA512:
-  metadata.gz: 4ebcd346bdd78df297791ec1e2c2f65f80894f8996f53eee0439717fd9a80ffc5acbeb2bbb25bb390c87a55115a786316b2ae02f6c29e38bbd16f9a7a60907bb
-  data.tar.gz: f3b04dade1f983bb2bf358ae80a9d08dc622c7b5504aa519bbb6bf46f5c18c420d217fb0c63396e5d9c5a52ff5e30c413338bea73057fa930c8ccf7acb438528
+  metadata.gz: 220a4b40f8bf0294b3e2ecdba3d7647fa72fbb1e1f0d7e0ecf22ad27c823aefeef3dd584227193e0e24781b881a4545227c70691f458bcecc695aafc47d63740
+  data.tar.gz: b7db119540b418fb3602be6e86909eb4675cc340e56b546bd2b4e8bfa22b0c61f9ec7cdbe42b65a3a1b243b6d271c3402e15805dea1cc2d7d3bc3bb1e679f1c6

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 2.1.1
+ - Support Elasticsearch 2.1.0
+ - Fix issue where transport client w/o port specified breaks
+## 2.1.0
+ - Port option no longer allowed
+ - Majority of code now shared with logstash-output-elasticsearch
 ## 2.0.2
  - Update loading of addons to conform with new naming scheme (replaced - with _)
 ## 2.0.1

data/Gemfile

@@ -1,3 +1,3 @@
 source 'https://rubygems.org'
 
-gemspec
+gemspec

data/lib/logstash/outputs/elasticsearch_java.rb

@@ -4,11 +4,11 @@ require "logstash/environment"
 require "logstash/outputs/base"
 require "logstash/json"
 require "concurrent"
-require "stud/buffer"
 require "socket" # for Socket.gethostname
 require "thread" # for safe queueing
 require "uri" # for escaping user input
 require "logstash/outputs/elasticsearch_java/protocol"
+require "logstash/outputs/elasticsearch"
 
 # This output lets you store logs in Elasticsearch using the native 'node' and 'transport'
 # protocols. It is highly recommended to use the regular 'logstash-output-elasticsearch' output
@@ -67,66 +67,25 @@ require "logstash/outputs/elasticsearch_java/protocol"
 class LogStash::Outputs::ElasticSearchJava < LogStash::Outputs::Base
   attr_reader :client
 
-  include Stud::Buffer
+  include LogStash::Outputs::ElasticSearch::CommonConfigs
+  include LogStash::Outputs::ElasticSearch::Common
+
   RETRYABLE_CODES = [409, 429, 503]
   SUCCESS_CODES = [200, 201]
 
   config_name "elasticsearch_java"
 
-  # The index to write events to. This can be dynamic using the `%{foo}` syntax.
-  # The default value will partition your indices by day so you can more easily
-  # delete old data or only search specific date ranges.
-  # Indexes may not contain uppercase characters.
-  # For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}
-  config :index, :validate => :string, :default => "logstash-%{+YYYY.MM.dd}"
-
-  # The index type to write events to. Generally you should try to write only
-  # similar events to the same 'type'. String expansion `%{foo}` works here.
-  # 
-  # Deprecated in favor of `document_type` field.
-  config :index_type, :validate => :string, :deprecated => "Please use the 'document_type' setting instead. It has the same effect, but is more appropriately named."
-
-  # The document type to write events to. Generally you should try to write only
-  # similar events to the same 'type'. String expansion `%{foo}` works here.
-  # Unless you set 'document_type', the event 'type' will be used if it exists 
-  # otherwise the document type will be assigned the value of 'logs'
-  config :document_type, :validate => :string
-
-  # Starting in Logstash 1.3 (unless you set option `manage_template` to false)
-  # a default mapping template for Elasticsearch will be applied, if you do not
-  # already have one set to match the index pattern defined (default of
-  # `logstash-%{+YYYY.MM.dd}`), minus any variables.  For example, in this case
-  # the template will be applied to all indices starting with `logstash-*`
+  # The Elasticsearch action to perform. Valid actions are:
   #
-  # If you have dynamic templating (e.g. creating indices based on field names)
-  # then you should set `manage_template` to false and use the REST API to upload
-  # your templates manually.
-  config :manage_template, :validate => :boolean, :default => true
-
-  # This configuration option defines how the template is named inside Elasticsearch.
-  # Note that if you have used the template management features and subsequently
-  # change this, you will need to prune the old template manually, e.g.
-  #
-  # `curl -XDELETE <http://localhost:9200/_template/OldTemplateName?pretty>`
+  # - index: indexes a document (an event from Logstash).
+  # - delete: deletes a document by id (An id is required for this action)
+  # - create: indexes a document, fails if a document by that id already exists in the index.
+  # - update: updates a document by id. Update has a special case where you can upsert -- update a
+  #   document if not already present. See the `upsert` option
+  # - create_unless_exists: create the document unless it already exists, in which case do nothing.
   #
-  # where `OldTemplateName` is whatever the former setting was.
-  config :template_name, :validate => :string, :default => "logstash"
-
-  # You can set the path to your own template here, if you so desire.
-  # If not set, the included template will be used.
-  config :template, :validate => :path
-
-  # Overwrite the current template with whatever is configured
-  # in the `template` and `template_name` directives.
-  config :template_overwrite, :validate => :boolean, :default => false
-
-  # The document ID for the index. Useful for overwriting existing entries in
-  # Elasticsearch with the same ID.
-  config :document_id, :validate => :string
-
-  # A routing override to be applied to all processed events.
-  # This can be dynamic using the `%{foo}` syntax.
-  config :routing, :validate => :string
+  # For more details on actions, check out the http://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html[Elasticsearch bulk API documentation]
+  config :action, :validate => %w(index delete create update create_unless_exists), :default => "index"
 
   # The name of your cluster if you set it on the Elasticsearch side. Useful
   # for discovery when using `node` or `transport` protocols.
@@ -134,53 +93,6 @@ class LogStash::Outputs::ElasticSearchJava < LogStash::Outputs::Base
   # Equivalent to the Elasticsearch option 'cluster.name'
   config :cluster, :validate => :string
 
-  # For the `node` protocol, if you do not specify `host`, it will attempt to use
-  # multicast discovery to connect to Elasticsearch.  If http://www.elastic.co/guide/en/elasticsearch/guide/current/_important_configuration_changes.html#_prefer_unicast_over_multicast[multicast is disabled] in Elasticsearch, 
-  # you must include the hostname or IP address of the host(s) to use for Elasticsearch unicast discovery.
-  # Remember the `node` protocol uses the http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-transport.html#modules-transport[transport] address (eg. 9300, not 9200).
-  #     `"127.0.0.1"`
-  #     `["127.0.0.1:9300","127.0.0.2:9300"]`
-  # When setting hosts for `node` protocol, it is important to confirm that at least one non-client
-  # node is listed in the `host` list.  Also keep in mind that the `host` parameter when used with 
-  # the `node` protocol is for *discovery purposes only* (not for load balancing).  When multiple hosts 
-  # are specified, it will contact the first host to see if it can use it to discover the cluster.  If not, 
-  # then it will contact the second host in the list and so forth. With the `node` protocol, 
-  # Logstash will join the Elasticsearch cluster as a node client (which has a copy of the cluster
-  # state) and this node client is the one that will automatically handle the load balancing of requests 
-  # across data nodes in the cluster.  
-  # If you are looking for a high availability setup, our recommendation is to use the `transport` protocol (below), 
-  # set up multiple http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html[client nodes] and list the client nodes in the `host` parameter.
-  # 
-  # For the `transport` protocol, it will load balance requests across the hosts specified in the `host` parameter.
-  # Remember the `transport` protocol uses the http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-transport.html#modules-transport[transport] address (eg. 9300, not 9200).
-  #     `"127.0.0.1"`
-  #     `["127.0.0.1:9300","127.0.0.2:9300"]`
-  # There is also a `sniffing` option (see below) that can be used with the transport protocol to instruct it to use the host to sniff for
-  # "alive" nodes in the cluster and automatically use it as the hosts list (but will skip the dedicated master nodes).  
-  # If you do not use the sniffing option, it is important to exclude http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html[dedicated master nodes] from the `host` list
-  # to prevent Logstash from sending bulk requests to the master nodes. So this parameter should only reference either data or client nodes.
-  #
-  # For the `http` protocol, it will load balance requests across the hosts specified in the `host` parameter.
-  # Remember the `http` protocol uses the http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-http.html#modules-http[http] address (eg. 9200, not 9300).
-  #     `"127.0.0.1"`
-  #     `["127.0.0.1:9200","127.0.0.2:9200"]`
-  # It is important to exclude http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html[dedicated master nodes] from the `host` list
-  # to prevent LS from sending bulk requests to the master nodes.  So this parameter should only reference either data or client nodes.
-  config :hosts, :validate => :array, :default => ["127.0.0.1"]
-
-  # The port for Elasticsearch transport to use.
-  #
-  # If you do not set this, the following defaults are used:
-  # * `protocol => transport` - port 9300-9305
-  # * `protocol => node` - port 9300-9305
-  config :port, :validate => :string, :default => "9300-9305"
-
-  # The name/address of the host to bind to for Elasticsearch clustering. Equivalent to the Elasticsearch option 'network.host'
-  # option.
-  # This MUST be set for either protocol to work (node or transport)! The internal Elasticsearch node
-  # will bind to this ip. This ip MUST be reachable by all nodes in the Elasticsearch cluster
-  config :network_host, :validate => :string, :required => true
-
   # This sets the local port to bind to. Equivalent to the Elasticsrearch option 'transport.tcp.port'
   config :transport_tcp_port, :validate => :number
 
@@ -193,24 +105,6 @@ class LogStash::Outputs::ElasticSearchJava < LogStash::Outputs::Base
   # By default, this is generated internally by the ES client.
   config :node_name, :validate => :string
 
-  # This plugin uses the bulk index api for improved indexing performance.
-  # To make efficient bulk api calls, we will buffer a certain number of
-  # events before flushing that out to Elasticsearch. This setting
-  # controls how many events will be buffered before sending a batch
-  # of events.
-  config :flush_size, :validate => :number, :default => 500
-
-  # The amount of time since last flush before a flush is forced.
-  #
-  # This setting helps ensure slow event rates don't get stuck in Logstash.
-  # For example, if your `flush_size` is 100, and you have received 10 events,
-  # and it has been more than `idle_flush_time` seconds since the last flush,
-  # Logstash will flush those 10 events automatically.
-  #
-  # This helps keep both fast and slow log streams moving along in
-  # near-real-time.
-  config :idle_flush_time, :validate => :number, :default => 1
-
   # Choose the protocol used to talk to Elasticsearch.
   #
   # The 'node' protocol (default) will connect to the cluster as a normal Elasticsearch
@@ -231,132 +125,16 @@ class LogStash::Outputs::ElasticSearchJava < LogStash::Outputs::Base
   # All protocols will use bulk requests when talking to Elasticsearch.
   config :protocol, :validate => [ "node", "transport"], :default => "transport"
 
-  # The Elasticsearch action to perform. Valid actions are: `index`, `delete`.
-  #
-  # Use of this setting *REQUIRES* you also configure the `document_id` setting
-  # because `delete` actions all require a document id.
-  #
-  # What does each action do?
-  #
-  # - index: indexes a document (an event from Logstash).
-  # - delete: deletes a document by id
-  # - create: indexes a document, fails if a document by that id already exists in the index.
-  # - update: updates a document by id
-  # following action is not supported by HTTP protocol
-  # - create_unless_exists: creates a document, fails if no id is provided
-  #
-  # For more details on actions, check out the http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-bulk.html[Elasticsearch bulk API documentation]
-  config :action, :validate => :string, :default => "index"
-
-  # Validate the server's certificate
-  # Disabling this severely compromises security
-  # For more information read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf
-  config :ssl_certificate_verification, :validate => :boolean, :default => true
-
-  # The .cer or .pem file to validate the server's certificate
-  config :cacert, :validate => :path
-
-  # The JKS truststore to validate the server's certificate
-  # Use either `:truststore` or `:cacert`
-  config :truststore, :validate => :path
-
-  # Set the truststore password
-  config :truststore_password, :validate => :password
-
-  # The keystore used to present a certificate to the server
-  # It can be either .jks or .p12
-  config :keystore, :validate => :path
-
-  # Set the truststore password
-  config :keystore_password, :validate => :password
-
   # Enable cluster sniffing (transport only).
   # Asks host for the list of all cluster nodes and adds them to the hosts list
   # Equivalent to the Elasticsearch option 'client.transport.sniff'
   config :sniffing, :validate => :boolean, :default => false
 
-  # Set max retry for each event
-  config :max_retries, :validate => :number, :default => 3
-
-  # Set retry policy for events that failed to send
-  config :retry_max_items, :validate => :number, :default => 5000
-
-  # Set max interval between bulk retries
-  config :retry_max_interval, :validate => :number, :default => 5
-
-  # Enable doc_as_upsert for update mode
-  # create a new document with source if document_id doesn't exists
-  config :doc_as_upsert, :validate => :boolean, :default => false
-
-  # Set upsert content for update mode
-  # create a new document with this parameter as json string if document_id doesn't exists
-  config :upsert, :validate => :string, :default => ""
-
-  public
-  def register
-    @submit_mutex = Mutex.new
-    # retry-specific variables
-    @retry_flush_mutex = Mutex.new
-    @retry_teardown_requested = Concurrent::AtomicBoolean.new(false)
-    # needs flushing when interval
-    @retry_queue_needs_flushing = ConditionVariable.new
-    @retry_queue_not_full = ConditionVariable.new
-    @retry_queue = Queue.new
-
-
-    if @protocol =='node' && !@network_host
-      raise LogStash::ConfigurationError, "network_host MUST be set if the 'node' protocol is in use! If this is set incorrectly Logstash will hang attempting to connect!"
-    end
-
-    if (@hosts.nil? || @hosts.empty?) && @protocol != "node" # node can use zen discovery
-      @logger.info("No 'hosts' set in elasticsearch output. Defaulting to localhost")
-      @hosts = ["localhost"]
-    end
-
-    client_class = case @protocol
-      when "transport"
-        LogStash::Outputs::ElasticSearchJavaPlugins::Protocols::TransportClient
-      when "node"
-        LogStash::Outputs::ElasticSearchJavaPlugins::Protocols::NodeClient
-    end
-
-    @client = client_class.new(client_options)
-
-    if @manage_template
-      begin
-        @logger.info("Automatic template management enabled", :manage_template => @manage_template.to_s)
-        client.template_install(@template_name, get_template, @template_overwrite)
-      rescue => e
-        @logger.error("Failed to install template",
-                      :message => e.message,
-                      :error_class => e.class.name,
-                      )
-      end
-    end
-
-    @logger.info("New Elasticsearch output", :cluster => @cluster,
-                 :hosts => @host, :port => @port, :protocol => @protocol)
-
-    buffer_initialize(
-      :max_items => @flush_size,
-      :max_interval => @idle_flush_time,
-      :logger => @logger
-    )
-
-    @retry_timer_thread = Thread.new do
-      loop do
-        sleep(@retry_max_interval)
-        @retry_flush_mutex.synchronize { @retry_queue_needs_flushing.signal }
-      end
-    end
-
-    @retry_thread = Thread.new do
-      while @retry_teardown_requested.false?
-        @retry_flush_mutex.synchronize { @retry_queue_needs_flushing.wait(@retry_flush_mutex) }
-        retry_flush
-      end
-    end
-  end # def register
+  # The name/address of the host to bind to for Elasticsearch clustering. Equivalent to the Elasticsearch option 'network.host'
+  # option.
+  # This MUST be set for either protocol to work (node or transport)! The internal Elasticsearch node
+  # will bind to this ip. This ip MUST be reachable by all nodes in the Elasticsearch cluster
+  config :network_host, :validate => :string, :required => true
 
   def client_options
     client_settings = {}
@@ -371,16 +149,10 @@ class LogStash::Outputs::ElasticSearchJava < LogStash::Outputs::Base
       client_settings["node.name"] = "logstash-#{Socket.gethostname}-#{$$}-#{object_id}"
     end
 
-    @@plugins.each do |plugin|
-      name = plugin.name.split('_')[-1]
-      client_settings.merge!(LogStash::Outputs::ElasticSearchJava.const_get(name.capitalize).create_client_config(self))
-    end
-
-    common_options = {
+    options = {
       :protocol => @protocol,
       :client_settings => client_settings,
-      :hosts => @hosts,
-      :port => @port
+      :hosts => @hosts
     }
 
     # Update API setup
@@ -388,155 +160,34 @@ class LogStash::Outputs::ElasticSearchJava < LogStash::Outputs::Base
       :upsert => @upsert,
       :doc_as_upsert => @doc_as_upsert
     }
-    common_options.merge! update_options if @action == 'update'
+    options.merge! update_options if @action == 'update'
 
-    common_options
+    options
   end
 
-
-  public
-  def get_template
-    if @template.nil?
-      @template = ::File.expand_path('elasticsearch_java/elasticsearch-template.json', ::File.dirname(__FILE__))
-      if !File.exists?(@template)
-        raise "You must specify 'template => ...' in your elasticsearch output (I looked for '#{@template}')"
-      end
-    end
-    template_json = IO.read(@template).gsub(/\n/,'')
-    template = LogStash::Json.load(template_json)
-    @logger.info("Using mapping template", :template => template)
-    return template
-  end # def get_template
-
-  public
-  def receive(event)
-    
-
-    # block until we have not maxed out our 
-    # retry queue. This is applying back-pressure
-    # to slow down the receive-rate
-    @retry_flush_mutex.synchronize {
-      @retry_queue_not_full.wait(@retry_flush_mutex) while @retry_queue.size > @retry_max_items
-    }
-
-    event['@metadata']['retry_count'] = 0
-
-    # Set the 'type' value for the index.
-    type = if @document_type
-             event.sprintf(@document_type)
-           elsif @index_type # deprecated
-             event.sprintf(@index_type)
-           else
-             event["type"] || "logs"
-           end
-
-    params = {
-      :_id => @document_id ? event.sprintf(@document_id) : nil,
-      :_index => event.sprintf(@index),
-      :_type => type,
-      :_routing => @routing ? event.sprintf(@routing) : nil
-    }
-    
-    params[:_upsert] = LogStash::Json.load(event.sprintf(@upsert)) if @action == 'update' && @upsert != ""
-
-    buffer_receive([event.sprintf(@action), params, event])
-  end # def receive
-
-  public
-  # The submit method can be called from both the
-  # Stud::Buffer flush thread and from our own retry thread.
-  def submit(actions)
-    es_actions = actions.map { |a, doc, event| [a, doc, event.to_hash] }
-    @submit_mutex.lock
-    begin
-      bulk_response = @client.bulk(es_actions)
-    ensure
-      @submit_mutex.unlock
-    end
-    if bulk_response["errors"]
-      actions_with_responses = actions.zip(bulk_response['statuses'])
-      actions_to_retry = []
-      actions_with_responses.each do |action, resp_code|
-        if RETRYABLE_CODES.include?(resp_code)
-          @logger.warn "retrying failed action with response code: #{resp_code}"
-          actions_to_retry << action
-        elsif not SUCCESS_CODES.include?(resp_code)
-          @logger.warn "failed action with response of #{resp_code}, dropping action: #{action}"
-        end
-      end
-      retry_push(actions_to_retry) unless actions_to_retry.empty?
-    end
+  def build_client
+    @client = client_class.new(client_options)
   end
 
-  # When there are exceptions raised upon submission, we raise an exception so that
-  # Stud::Buffer will retry to flush
-  public
-  def flush(actions, teardown = false)
-    begin
-      submit(actions)
-    rescue => e
-      @logger.error "Got error to send bulk of actions: #{e.message}"
-      raise e
-    end
-  end # def flush
-
-  public
   def close
-    @retry_teardown_requested.make_true
-    # First, make sure retry_timer_thread is stopped
-    # to ensure we do not signal a retry based on 
-    # the retry interval.
-    Thread.kill(@retry_timer_thread)
-    @retry_timer_thread.join
-    # Signal flushing in the case that #retry_flush is in 
-    # the process of waiting for a signal.
-    @retry_flush_mutex.synchronize { @retry_queue_needs_flushing.signal }
-    # Now, #retry_flush is ensured to not be in a state of 
-    # waiting and can be safely joined into the main thread
-    # for further final execution of an in-process remaining call.
-    @retry_thread.join
-
-    # execute any final actions along with a proceeding retry for any 
-    # final actions that did not succeed.
-    buffer_flush(:final => true)
-    retry_flush
+    @stopping.make_true
+    @buffer.stop
   end
 
-  private
-  # in charge of submitting any actions in @retry_queue that need to be 
-  # retried
-  #
-  # This method is not called concurrently. It is only called by @retry_thread
-  # and once that thread is ended during the teardown process, a final call 
-  # to this method is done upon teardown in the main thread.
-  def retry_flush()
-    unless @retry_queue.empty?
-      buffer = @retry_queue.size.times.map do
-        next_action, next_doc, next_event = @retry_queue.pop
-        next_event['@metadata']['retry_count'] += 1
-
-        if next_event['@metadata']['retry_count'] > @max_retries
-          @logger.error "too many attempts at sending event. dropping: #{next_event}"
-          nil
-        else
-          [next_action, next_doc, next_event]
-        end
-      end.compact
-
-      submit(buffer) unless buffer.empty?
+  def get_plugin_options
+    @@plugins.each do |plugin|
+      name = plugin.name.split('-')[-1]
+      client_settings.merge!(LogStash::Outputs::ElasticSearchJava.const_get(name.capitalize).create_client_config(self))
     end
-
-    @retry_flush_mutex.synchronize {
-      @retry_queue_not_full.signal if @retry_queue.size < @retry_max_items
-    }
   end
 
-  private
-  def retry_push(actions)
-    Array(actions).each{|action| @retry_queue << action}
-    @retry_flush_mutex.synchronize {
-      @retry_queue_needs_flushing.signal if @retry_queue.size >= @retry_max_items
-    }
+  def client_class
+    case @protocol
+      when "transport"
+        LogStash::Outputs::ElasticSearchJavaPlugins::Protocols::TransportClient
+      when "node"
+        LogStash::Outputs::ElasticSearchJavaPlugins::Protocols::NodeClient
+    end
   end
 
   @@plugins = Gem::Specification.find_all{|spec| spec.name =~ /logstash-output-elasticsearch_java_/ }