logstash-output-elasticsearch 0.1.1 → 0.1.5

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    YTQ0NzIzNzc2MmVkMTYxM2Y5OTBhNTExZjQ4Nzc5ZGJlNDU5MDQwYg==
+    OTFjZmRiNDk2YjMzNGE4NzVmYTgyNGMzYzAxNGUxZjY3ZDEwMTE4MQ==
   data.tar.gz: !binary |-
-    NGQxYTgwYzRlMjhkMDBkYWFjODM4ZDAyNjhmNWE2NDZjOWY1ZDliZA==
+    ZTcyNTg2NzA5ZDQ3NGRmZDM4NGE1ZmJlNzllOWI3MTAxMzgwNjllMQ==
 SHA512:
   metadata.gz: !binary |-
-    NTJlMWQzNDU2OTMwZGMzZDMyNjAwYjE5M2M0NjQ2ZWQ5YmQ4OGRlMDIxODEy
-    MDY0NTI4NTU2ZWE0NDdlMmU4MTY1NjAwM2IwODAxZTUwZGMxN2IyNDdhZWI3
-    MWU5MGYyYmNjYWMyYzA5OTQzM2JjMzZhNTk3Njg4NThkZTJjYjE=
+    MmIxMGUyZjAxYTE5NmUzZGIzZWRmNDc0NmMyOWVlMTQ2YTQxMjA2ZmQzZDg2
+    M2FlMWVjMGRmNTQwMjdjZjM3NzY5Y2UxMTk1ODVlM2NhOTBlZmIwMzA4ZjE4
+    YWNjYjdlMzgxYmUyMTMwN2UwMzI2MWEzNjI3OWMyNWM4ZDRlYmE=
   data.tar.gz: !binary |-
-    YTQ1YjU1MDNmOTliMzgwYTU4YmNjYTdlMmFjNjQyNTBiODcxMTk5NWU5ZGJh
-    MjNkMDZlYTBlM2RkOTdjNTBjNDI5MDI4MDIzMTNjNzBjZjQxMzE1ZTk0YTc3
-    Y2Q3MjgxNmE2NjFmOTQxMjdkYWE3ZWU3NjMwNzgwNzU3MjY0ZGU=
+    YjZmYWQzZjMxMGUwODFkZDMzYWY4OTQ2Zjc1NmZiMDRiODUwYWVmMmIyNDNl
+    YjMwMjIyMzg2YjA5M2MwNTdlNDdiMmZlOTA0OTZkNGRkZGNmYTFkYTFjNGUz
+    YTQ4YjQ1ZWFiZjQzZTU5Y2I5MzI1ZWJlZjVhNTlhMDU5NzZmNDI=

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+Gemfile.lock
+.bundle
+vendor

data/Gemfile

@@ -0,0 +1,4 @@
+source 'http://rubygems.org'
+gem 'rake'
+gem 'gem_publisher'
+gem 'archive-tar-minitar'

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2012-2014 Elasticsearch <http://www.elasticsearch.org>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/Rakefile

@@ -0,0 +1,6 @@
+@files=[]
+
+task :default do
+  system("rake -T")
+end
+

data/lib/logstash/outputs/elasticsearch.rb

@@ -5,20 +5,21 @@ require "logstash/outputs/base"
 require "logstash/json"
 require "stud/buffer"
 require "socket" # for Socket.gethostname
+require "uri" # for escaping user input
+require 'logstash-output-elasticsearch_jars.rb'
 
 # This output lets you store logs in Elasticsearch and is the most recommended
 # output for Logstash. If you plan on using the Kibana web interface, you'll
 # need to use this output.
 #
 #   *VERSION NOTE*: Your Elasticsearch cluster must be running Elasticsearch
-#   %ELASTICSEARCH_VERSION%. If you use any other version of Elasticsearch,
-#   you should set `protocol => http` in this plugin.
+#   1.0.0 or later.
 #
 # If you want to set other Elasticsearch options that are not exposed directly
 # as configuration options, there are two methods:
 #
 # * Create an `elasticsearch.yml` file in the $PWD of the Logstash process
-# * Pass in es.* java properties (java -Des.node.foo= or ruby -J-Des.node.foo=)
+# * Pass in es.* java properties (`java -Des.node.foo=` or `ruby -J-Des.node.foo=`)
 #
 # With the default `protocol` setting ("node"), this plugin will join your
 # Elasticsearch cluster as a client node, so it will show up in Elasticsearch's
@@ -28,10 +29,6 @@ require "socket" # for Socket.gethostname
 #
 # ## Operational Notes
 #
-# Template management requires Elasticsearch version 0.90.7 or later. If you
-# are using a version older than this, please upgrade. You will receive
-# more benefits than just template management!
-#
 # If using the default `protocol` setting ("node"), your firewalls might need
 # to permit port 9300 in *both* directions (from Logstash to Elasticsearch, and
 # Elasticsearch to Logstash)
@@ -41,32 +38,34 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   config_name "elasticsearch"
   milestone 3
 
-  # The index to write events to. This can be dynamic using the %{foo} syntax.
+  # The index to write events to. This can be dynamic using the `%{foo}` syntax.
   # The default value will partition your indices by day so you can more easily
   # delete old data or only search specific date ranges.
   # Indexes may not contain uppercase characters.
   config :index, :validate => :string, :default => "logstash-%{+YYYY.MM.dd}"
 
   # The index type to write events to. Generally you should try to write only
-  # similar events to the same 'type'. String expansion '%{foo}' works here.
+  # similar events to the same 'type'. String expansion `%{foo}` works here.
   config :index_type, :validate => :string
 
-  # Starting in Logstash 1.3 (unless you set option "manage_template" to false)
+  # Starting in Logstash 1.3 (unless you set option `manage_template` to false)
   # a default mapping template for Elasticsearch will be applied, if you do not
   # already have one set to match the index pattern defined (default of
-  # "logstash-%{+YYYY.MM.dd}"), minus any variables.  For example, in this case
-  # the template will be applied to all indices starting with logstash-*
+  # `logstash-%{+YYYY.MM.dd}`), minus any variables.  For example, in this case
+  # the template will be applied to all indices starting with `logstash-*`
   #
   # If you have dynamic templating (e.g. creating indices based on field names)
-  # then you should set "manage_template" to false and use the REST API to upload
+  # then you should set `manage_template` to false and use the REST API to upload
   # your templates manually.
   config :manage_template, :validate => :boolean, :default => true
 
   # This configuration option defines how the template is named inside Elasticsearch.
   # Note that if you have used the template management features and subsequently
   # change this, you will need to prune the old template manually, e.g.
-  # curl -XDELETE <http://localhost:9200/_template/OldTemplateName?pretty>
-  # where OldTemplateName is whatever the former setting was.
+  #
+  # `curl -XDELETE <http://localhost:9200/_template/OldTemplateName?pretty>`
+  #
+  # where `OldTemplateName` is whatever the former setting was.
   config :template_name, :validate => :string, :default => "logstash"
 
   # You can set the path to your own template here, if you so desire.
@@ -74,7 +73,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   config :template, :validate => :path
 
   # Overwrite the current template with whatever is configured
-  # in the template and template_name directives.
+  # in the `template` and `template_name` directives.
   config :template_overwrite, :validate => :boolean, :default => false
 
   # The document ID for the index. Useful for overwriting existing entries in
@@ -88,7 +87,10 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # The hostname or IP address of the host to use for Elasticsearch unicast discovery
   # This is only required if the normal multicast/cluster discovery stuff won't
   # work in your environment.
-  config :host, :validate => :string
+  #
+  #     `"127.0.0.1"`
+  #     `["127.0.0.1:9300","127.0.0.2:9300"]`
+  config :host, :validate => :array
 
   # The port for Elasticsearch transport to use.
   #
@@ -173,24 +175,34 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   #
   # What does each action do?
   #
-  # - index: indexes a document (an event from logstash).
+  # - index: indexes a document (an event from Logstash).
   # - delete: deletes a document by id
   #
   # For more details on actions, check out the [Elasticsearch bulk API documentation](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-bulk.html)
   config :action, :validate => :string, :default => "index"
 
+  # Username and password (HTTP only)
+  config :user, :validate => :string
+  config :password, :validate => :password
+
+  # SSL Configurations (HTTP only)
+  #
+  # Enable SSL
+  config :ssl, :validate => :boolean, :default => false
+
+  # The .cer or .pem file to validate the server's certificate
+  config :cacert, :validate => :path
+
+  # The JKS truststore to validate the server's certificate
+  # Use either `:truststore` or `:cacert`
+  config :truststore, :validate => :path
+
+  # Set the truststore password
+  config :truststore_password, :validate => :password
+
   public
   def register
     client_settings = {}
-    client_settings["cluster.name"] = @cluster if @cluster
-    client_settings["network.host"] = @bind_host if @bind_host
-    client_settings["transport.tcp.port"] = @bind_port if @bind_port
-
-    if @node_name
-      client_settings["node.name"] = @node_name
-    else
-      client_settings["node.name"] = "logstash-#{Socket.gethostname}-#{$$}-#{object_id}"
-    end
 
     if @protocol.nil?
       @protocol = LogStash::Environment.jruby? ? "node" : "http"
@@ -199,10 +211,21 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     if ["node", "transport"].include?(@protocol)
       # Node or TransportClient; requires JRuby
       raise(LogStash::PluginLoadingError, "This configuration requires JRuby. If you are not using JRuby, you must set 'protocol' to 'http'. For example: output { elasticsearch { protocol => \"http\" } }") unless LogStash::Environment.jruby?
-      LogStash::Environment.load_elasticsearch_jars!
 
-      # setup log4j properties for Elasticsearch
-      LogStash::Logger.setup_log4j(@logger)
+      client_settings["cluster.name"] = @cluster if @cluster
+      client_settings["network.host"] = @bind_host if @bind_host
+      client_settings["transport.tcp.port"] = @bind_port if @bind_port
+ 
+      if @node_name
+        client_settings["node.name"] = @node_name
+      else
+        client_settings["node.name"] = "logstash-#{Socket.gethostname}-#{$$}-#{object_id}"
+      end
+
+      @@plugins.each do |plugin|
+        name = plugin.name.split('-')[-1]
+        client_settings.merge!(LogStash::Outputs::ElasticSearch.const_get(name.capitalize).create_client_config(self))
+      end
     end
 
     require "logstash/outputs/elasticsearch/protocol"
@@ -216,49 +239,102 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
 
     if @host.nil? && @protocol == "http"
       @logger.info("No 'host' set in elasticsearch output. Defaulting to localhost")
-      @host = "localhost"
+      @host = ["localhost"]
     end
 
-    options = {
-      :host => @host,
-      :port => @port,
+    if @ssl
+      if @protocol == "http"
+        @protocol = "https"
+        if @cacert && @truststore
+          raise(LogStash::ConfigurationError, "Use either \"cacert\" or \"truststore\" when configuring the CA certificate") if @truststore
+        end
+        ssl_options = {}
+        if @cacert then
+          @truststore, ssl_options[:truststore_password] = generate_jks @cacert
+        elsif @truststore
+          ssl_options[:truststore_password] = @truststore_password.value if @truststore_password
+        end
+        ssl_options[:truststore] = @truststore
+        client_settings[:ssl] = ssl_options
+      else
+        raise(LogStash::ConfigurationError, "SSL is not supported for '#{@protocol}'. Change the protocol to 'http' if you need SSL.")
+      end
+    end
+
+    common_options = {
+      :protocol => @protocol,
       :client_settings => client_settings
     }
 
+    if @user && @password
+      if @protocol =~ /http/
+        common_options[:user] = ::URI.escape(@user, "@:")
+        common_options[:password] = ::URI.escape(@password.value, "@:")
+      else
+        raise(LogStash::ConfigurationError, "User and password parameters are not supported for '#{@protocol}'. Change the protocol to 'http' if you need them.")
+      end
+    end
 
     client_class = case @protocol
       when "transport"
         LogStash::Outputs::Elasticsearch::Protocols::TransportClient
       when "node"
         LogStash::Outputs::Elasticsearch::Protocols::NodeClient
-      when "http"
+      when /http/
         LogStash::Outputs::Elasticsearch::Protocols::HTTPClient
     end
 
     if @embedded
       raise(LogStash::ConfigurationError, "The 'embedded => true' setting is only valid for the elasticsearch output under JRuby. You are running #{RUBY_DESCRIPTION}") unless LogStash::Environment.jruby?
-      LogStash::Environment.load_elasticsearch_jars!
+#      LogStash::Environment.load_elasticsearch_jars!
 
       # Default @host with embedded to localhost. This should help avoid
       # newbies tripping on ubuntu and other distros that have a default
       # firewall that blocks multicast.
-      @host ||= "localhost"
+      @host ||= ["localhost"]
 
       # Start Elasticsearch local.
       start_local_elasticsearch
     end
 
-    @client = client_class.new(options)
+    @client = Array.new
+
+    if protocol == "node" or @host.nil? # if @protocol is "node" or @host is not set
+      options = {
+          :host => @host,
+          :port => @port,
+      }.merge(common_options)
+      @client << client_class.new(options)
+    else # if @protocol in ["transport","http"]
+      @host.each do |host|
+          (_host,_port) = host.split ":"
+          options = {
+            :host => _host,
+            :port => _port || @port,
+          }.merge(common_options)
+          @logger.info "Create client to elasticsearch server on #{_host}:#{_port}"
+          @client << client_class.new(options)
+      end # @host.each
+    end
+
+    if @manage_template
+      for client in @client
+          begin
+            @logger.info("Automatic template management enabled", :manage_template => @manage_template.to_s)
+            client.template_install(@template_name, get_template, @template_overwrite)
+            break
+          rescue => e
+            @logger.error("Failed to install template: #{e.message}")
+          end
+      end # for @client loop
+    end # if @manage_templates
 
     @logger.info("New Elasticsearch output", :cluster => @cluster,
                  :host => @host, :port => @port, :embedded => @embedded,
                  :protocol => @protocol)
 
-
-    if @manage_template
-      @logger.info("Automatic template management enabled", :manage_template => @manage_template.to_s)
-      @client.template_install(@template_name, get_template, @template_overwrite)
-    end # if @manage_templates
+    @client_idx = 0
+    @current_client = @client[@client_idx]
 
     buffer_initialize(
       :max_items => @flush_size,
@@ -267,10 +343,17 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     )
   end # def register
 
+  protected
+  def shift_client
+    @client_idx = (@client_idx+1) % @client.length
+    @current_client = @client[@client_idx]
+    @logger.debug? and @logger.debug("Switched current elasticsearch client to ##{@client_idx} at #{@host[@client_idx]}")
+  end
+
   public
   def get_template
     if @template.nil?
-      @template = LogStash::Environment.plugin_path("outputs/elasticsearch/elasticsearch-template.json")
+      @template = ::File.expand_path('elasticsearch/elasticsearch-template.json', ::File.dirname(__FILE__))
       if !File.exists?(@template)
         raise "You must specify 'template => ...' in your elasticsearch output (I looked for '#{@template}')"
       end
@@ -295,6 +378,29 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     @embedded_elasticsearch.start
   end # def start_local_elasticsearch
 
+  private
+  def generate_jks cert_path
+
+    require 'securerandom'
+    require 'tempfile'
+    require 'java'
+    import java.io.FileInputStream
+    import java.io.FileOutputStream
+    import java.security.KeyStore
+    import java.security.cert.CertificateFactory
+
+    jks = java.io.File.createTempFile("cert", ".jks")
+
+    ks = KeyStore.getInstance "JKS"
+    ks.load nil, nil
+    cf = CertificateFactory.getInstance "X.509"
+    cert = cf.generateCertificate FileInputStream.new(cert_path)
+    ks.setCertificateEntry "cacert", cert
+    pwd = SecureRandom.urlsafe_base64(9)
+    ks.store FileOutputStream.new(jks), pwd.to_java.toCharArray
+    [jks.path, pwd]
+  end
+
   public
   def receive(event)
     return unless output?(event)
@@ -313,7 +419,18 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   end # def receive
 
   def flush(actions, teardown=false)
-    @client.bulk(actions)
+    begin
+      @logger.debug? and @logger.debug "Sending bulk of actions to client[#{@client_idx}]: #{@host[@client_idx]}"
+      @current_client.bulk(actions)
+    rescue => e
+      @logger.error "Got error to send bulk of actions to elasticsearch server at #{@host[@client_idx]} : #{e.message}"
+      raise e
+    ensure
+      unless @protocol == "node"
+          @logger.debug? and @logger.debug "Shifting current elasticsearch client"
+          shift_client
+      end
+    end
     # TODO(sissel): Handle errors. Since bulk requests could mostly succeed
     # (aka partially fail), we need to figure out what documents need to be
     # retried.
@@ -322,7 +439,17 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   end # def flush
 
   def teardown
+    if @cacert # remove temporary jks store created from the cacert
+      File.delete(@truststore)
+    end
     buffer_flush(:final => true)
   end
 
+  @@plugins = Gem::Specification.find_all{|spec| spec.name =~ /logstash-output-elasticsearch-/ }
+
+  @@plugins.each do |plugin|
+    name = plugin.name.split('-')[-1]
+    require "logstash/outputs/elasticsearch/#{name}"
+  end
+
 end # class LogStash::Outputs::Elasticsearch

data/lib/logstash/outputs/elasticsearch/protocol.rb

@@ -51,37 +51,33 @@ module LogStash::Outputs::Elasticsearch
       }
 
       def initialize(options={})
-        require "ftw"
         super
         require "elasticsearch" # gem 'elasticsearch-ruby'
+        # manticore http transport
+        require "elasticsearch/transport/transport/http/manticore"
         @options = DEFAULT_OPTIONS.merge(options)
         @client = client
       end
 
       def build_client(options)
-        client = Elasticsearch::Client.new(
-          :host => [options[:host], options[:port]].join(":")
-        )
-
-        # Use FTW to do indexing requests, for now, until we
-        # can identify and resolve performance problems of elasticsearch-ruby
-        @bulk_url = "http://#{options[:host]}:#{options[:port]}/_bulk"
-        @agent = FTW::Agent.new
-
-        return client
-      end
-
-      if ENV["BULK"] == "esruby"
-        def bulk(actions)
-          bulk_esruby(actions)
-        end
-      else
-        def bulk(actions)
-          bulk_ftw(actions)
+        uri = "#{options[:protocol]}://#{options[:host]}:#{options[:port]}"
+
+        client_options = {
+          :host => [uri],
+          :transport_options => options[:client_settings]
+        }
+        client_options[:transport_class] = ::Elasticsearch::Transport::Transport::HTTP::Manticore
+        client_options[:ssl] = client_options[:transport_options].delete(:ssl)
+
+        if options[:user] && options[:password] then
+          token = Base64.strict_encode64(options[:user] + ":" + options[:password])
+          client_options[:headers] = { "Authorization" => "Basic #{token}" }
         end
+
+        Elasticsearch::Client.new client_options
       end
 
-      def bulk_esruby(actions)
+      def bulk(actions)
         @client.bulk(:body => actions.collect do |action, args, source|
           if source
             next [ { action => args }, source ]
@@ -89,44 +85,7 @@ module LogStash::Outputs::Elasticsearch
             next { action => args }
           end
         end.flatten)
-      end # def bulk_esruby
-
-      # Avoid creating a new string for newline every time
-      NEWLINE = "\n".freeze
-      def bulk_ftw(actions)
-        body = actions.collect do |action, args, source|
-          header = { action => args }
-          if source
-            next [ LogStash::Json.dump(header), NEWLINE, LogStash::Json.dump(source), NEWLINE ]
-          else
-            next [ LogStash::Json.dump(header), NEWLINE ]
-          end
-        end.flatten.join("")
-        begin
-          response = @agent.post!(@bulk_url, :body => body)
-        rescue EOFError
-          @logger.warn("EOF while writing request or reading response header from elasticsearch", :host => @host, :port => @port)
-          raise
-        end
-
-        # Consume the body for error checking
-        # This will also free up the connection for reuse.
-        response_body = ""
-        begin
-          response.read_body { |chunk| response_body += chunk }
-        rescue EOFError
-          @logger.warn("EOF while reading response body from elasticsearch",
-                       :url => @bulk_url)
-          raise
-        end
-
-        if response.status != 200
-          @logger.error("Error writing (bulk) to elasticsearch",
-                        :response => response, :response_body => response_body,
-                        :request_body => body)
-          raise "Non-OK response code from Elasticsearch: #{response.status}"
-        end
-      end # def bulk_ftw
+      end # def bulk
 
       def template_exists?(name)
         @client.indices.get_template(:name => name)
@@ -181,16 +140,40 @@ module LogStash::Outputs::Elasticsearch
       end
 
       def hosts(options)
-        if options[:port].to_s =~ /^\d+-\d+$/
-          # port ranges are 'host[port1-port2]' according to
-          # http://www.elasticsearch.org/guide/reference/modules/discovery/zen/
-          # However, it seems to only query the first port.
-          # So generate our own list of unicast hosts to scan.
-          range = Range.new(*options[:port].split("-"))
-          return range.collect { |p| "#{options[:host]}:#{p}" }.join(",")
+        # http://www.elasticsearch.org/guide/reference/modules/discovery/zen/
+        result = Array.new
+        if options[:host].class == Array
+          options[:host].each do |host|
+            if host.to_s =~ /^.+:.+$/
+              # For host in format: host:port, ignore options[:port]
+              result << host
+            else
+              if options[:port].to_s =~ /^\d+-\d+$/
+                # port ranges are 'host[port1-port2]'
+                result << Range.new(*options[:port].split("-")).collect { |p| "#{host}:#{p}" }
+              else
+                result << "#{host}:#{options[:port]}"
+              end
+            end
+          end
         else
-          return "#{options[:host]}:#{options[:port]}"
+          if options[:host].to_s =~ /^.+:.+$/
+            # For host in format: host:port, ignore options[:port]
+            result << options[:host]
+          else
+            if options[:port].to_s =~ /^\d+-\d+$/
+              # port ranges are 'host[port1-port2]' according to
+              # http://www.elasticsearch.org/guide/reference/modules/discovery/zen/
+              # However, it seems to only query the first port.
+              # So generate our own list of unicast hosts to scan.
+              range = Range.new(*options[:port].split("-"))
+              result << range.collect { |p| "#{options[:host]}:#{p}" }
+            else
+              result << "#{options[:host]}:#{options[:port]}"
+            end
+          end
         end
+        result.flatten.join(",")
       end # def hosts
 
       def build_client(options)
@@ -268,4 +251,3 @@ module LogStash::Outputs::Elasticsearch
     class Delete; end
   end
 end
-

data/logstash-output-elasticsearch.gemspec

@@ -1,33 +1,37 @@
 Gem::Specification.new do |s|
- 
+
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '0.1.1'
+  s.version         = '0.1.5'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Logstash Output to Elasticsearch"
   s.description     = "Output events to elasticsearch"
   s.authors         = ["Elasticsearch"]
-  s.email           = 'rubycoder@example.com'
+  s.email           = 'richard.pijnenburg@elasticsearch.com'
   s.homepage        = "http://logstash.net/"
   s.require_paths = ["lib"]
-  
+
   # Files
   s.files = `git ls-files`.split($\)
-  
+
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
-  
+
   # Special flag to let us know this is actually a logstash plugin
-  s.metadata = { "logstash_plugin" => "true" }
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
 
   # Jar dependencies
-  s.requirements << "jar 'org.elasticsearch:elasticsearch', '1.2.2'"
+  s.requirements << "jar 'org.elasticsearch:elasticsearch', '1.3.1'"
 
   # Gem dependencies
-  s.add_runtime_dependency 'elasticsearch'
+  s.add_runtime_dependency 'elasticsearch', ['~> 1.0.6']
   s.add_runtime_dependency 'stud'
   s.add_runtime_dependency 'cabin', ['>=0.6.0']
-  s.add_runtime_dependency 'ftw', ['~> 0.0.39']
+  s.add_runtime_dependency 'ftw', ['~> 0.0.40']
   s.add_runtime_dependency 'logstash', '>= 1.4.0', '< 2.0.0'
-  s.add_runtime_dependency 'jar-dependencies', ['~> 0.0.6'] 
-  
+  s.add_runtime_dependency 'jar-dependencies'
+
+  if RUBY_PLATFORM == 'java'
+    gem.add_runtime_dependency "manticore"
+  end
 end
+

data/rakelib/publish.rake

@@ -0,0 +1,9 @@
+require "gem_publisher"
+
+desc "Publish gem to RubyGems.org"
+task :publish_gem do |t|
+  gem_file = Dir.glob(File.expand_path('../*.gemspec',File.dirname(__FILE__))).first
+  gem = GemPublisher.publish_if_updated(gem_file, :rubygems)
+  puts "Published #{gem}" if gem
+end
+

data/rakelib/vendor.rake

@@ -0,0 +1,169 @@
+require "net/http"
+require "uri"
+require "digest/sha1"
+
+def vendor(*args)
+  return File.join("vendor", *args)
+end
+
+directory "vendor/" => ["vendor"] do |task, args|
+  mkdir task.name
+end
+
+def fetch(url, sha1, output)
+
+  puts "Downloading #{url}"
+  actual_sha1 = download(url, output)
+
+  if actual_sha1 != sha1
+    fail "SHA1 does not match (expected '#{sha1}' but got '#{actual_sha1}')"
+  end
+end # def fetch
+
+def file_fetch(url, sha1)
+  filename = File.basename( URI(url).path )
+  output = "vendor/#{filename}"
+  task output => [ "vendor/" ] do
+    begin
+      actual_sha1 = file_sha1(output)
+      if actual_sha1 != sha1
+        fetch(url, sha1, output)
+      end
+    rescue Errno::ENOENT
+      fetch(url, sha1, output)
+    end
+  end.invoke
+
+  return output
+end
+
+def file_sha1(path)
+  digest = Digest::SHA1.new
+  fd = File.new(path, "r")
+  while true
+    begin
+      digest << fd.sysread(16384)
+    rescue EOFError
+      break
+    end
+  end
+  return digest.hexdigest
+ensure
+  fd.close if fd
+end
+
+def download(url, output)
+  uri = URI(url)
+  digest = Digest::SHA1.new
+  tmp = "#{output}.tmp"
+  Net::HTTP.start(uri.host, uri.port, :use_ssl => (uri.scheme == "https")) do |http|
+    request = Net::HTTP::Get.new(uri.path)
+    http.request(request) do |response|
+      fail "HTTP fetch failed for #{url}. #{response}" if [200, 301].include?(response.code)
+      size = (response["content-length"].to_i || -1).to_f
+      count = 0
+      File.open(tmp, "w") do |fd|
+        response.read_body do |chunk|
+          fd.write(chunk)
+          digest << chunk
+          if size > 0 && $stdout.tty?
+            count += chunk.bytesize
+            $stdout.write(sprintf("\r%0.2f%%", count/size * 100))
+          end
+        end
+      end
+      $stdout.write("\r      \r") if $stdout.tty?
+    end
+  end
+
+  File.rename(tmp, output)
+
+  return digest.hexdigest
+rescue SocketError => e
+  puts "Failure while downloading #{url}: #{e}"
+  raise
+ensure
+  File.unlink(tmp) if File.exist?(tmp)
+end # def download
+
+def untar(tarball, &block)
+  require "archive/tar/minitar"
+  tgz = Zlib::GzipReader.new(File.open(tarball))
+  # Pull out typesdb
+  tar = Archive::Tar::Minitar::Input.open(tgz)
+  tar.each do |entry|
+    path = block.call(entry)
+    next if path.nil?
+    parent = File.dirname(path)
+    
+    mkdir_p parent unless File.directory?(parent)
+
+    # Skip this file if the output file is the same size
+    if entry.directory?
+      mkdir path unless File.directory?(path)
+    else
+      entry_mode = entry.instance_eval { @mode } & 0777
+      if File.exists?(path)
+        stat = File.stat(path)
+        # TODO(sissel): Submit a patch to archive-tar-minitar upstream to
+        # expose headers in the entry.
+        entry_size = entry.instance_eval { @size }
+        # If file sizes are same, skip writing.
+        next if stat.size == entry_size && (stat.mode & 0777) == entry_mode
+      end
+      puts "Extracting #{entry.full_name} from #{tarball} #{entry_mode.to_s(8)}"
+      File.open(path, "w") do |fd|
+        # eof? check lets us skip empty files. Necessary because the API provided by
+        # Archive::Tar::Minitar::Reader::EntryStream only mostly acts like an
+        # IO object. Something about empty files in this EntryStream causes
+        # IO.copy_stream to throw "can't convert nil into String" on JRuby
+        # TODO(sissel): File a bug about this.
+        while !entry.eof?
+          chunk = entry.read(16384)
+          fd.write(chunk)
+        end
+          #IO.copy_stream(entry, fd)
+      end
+      File.chmod(entry_mode, path)
+    end
+  end
+  tar.close
+  File.unlink(tarball) if File.file?(tarball)
+end # def untar
+
+def ungz(file)
+
+  outpath = file.gsub('.gz', '')
+  tgz = Zlib::GzipReader.new(File.open(file))
+  begin
+    File.open(outpath, "w") do |out|
+      IO::copy_stream(tgz, out)
+    end
+    File.unlink(file)
+  rescue
+    File.unlink(outpath) if File.file?(outpath)
+   raise
+  end
+  tgz.close
+end
+
+desc "Process any vendor files required for this plugin"
+task "vendor" do |task, args|
+
+  @files.each do |file| 
+    download = file_fetch(file['url'], file['sha1'])
+    if download =~ /.tar.gz/
+      prefix = download.gsub('.tar.gz', '').gsub('vendor/', '')
+      untar(download) do |entry|
+        if !file['files'].nil?
+          next unless file['files'].include?(entry.full_name.gsub(prefix, ''))
+          out = entry.full_name.split("/").last
+        end
+        File.join('vendor', out)
+      end
+    elsif download =~ /.gz/
+      ungz(download)
+    end
+  end
+
+end

data/spec/outputs/elasticsearch.rb

@@ -1,10 +1,9 @@
-require "test_utils"
+require "spec_helper"
 require "ftw"
 require "logstash/plugin"
 require "logstash/json"
 
 describe "outputs/elasticsearch" do
-  extend LogStash::RSpec
 
   it "should register" do
     output = LogStash::Plugin.lookup("output", "elasticsearch").new("embedded" => "false", "protocol" => "transport", "manage_template" => "false")
@@ -346,4 +345,145 @@ describe "outputs/elasticsearch" do
       end
     end
   end
+
+  describe "elasticsearch protocol" do
+    # ElasticSearch related jars
+#LogStash::Environment.load_elasticsearch_jars!
+    # Load elasticsearch protocol
+    require "logstash/outputs/elasticsearch/protocol"
+
+    describe "elasticsearch node client" do
+      # Test ElasticSearch Node Client
+      # Reference: http://www.elasticsearch.org/guide/reference/modules/discovery/zen/
+
+      it "should support hosts in both string and array" do
+        # Because we defined *hosts* method in NodeClient as private,
+        # we use *obj.send :method,[args...]* to call method *hosts*
+        client = LogStash::Outputs::Elasticsearch::Protocols::NodeClient.new
+
+        # Node client should support host in string
+        # Case 1: default :host in string
+        insist { client.send :hosts, :host => "host",:port => 9300 } == "host:9300"
+        # Case 2: :port =~ /^\d+_\d+$/
+        insist { client.send :hosts, :host => "host",:port => "9300-9302"} == "host:9300,host:9301,host:9302"
+        # Case 3: :host =~ /^.+:.+$/
+        insist { client.send :hosts, :host => "host:9303",:port => 9300 } == "host:9303"
+        # Case 4:  :host =~ /^.+:.+$/ and :port =~ /^\d+_\d+$/
+        insist { client.send :hosts, :host => "host:9303",:port => "9300-9302"} == "host:9303"
+
+        # Node client should support host in array
+        # Case 5: :host in array with single item
+        insist { client.send :hosts, :host => ["host"],:port => 9300 } == ("host:9300")
+        # Case 6: :host in array with more than one items
+        insist { client.send :hosts, :host => ["host1","host2"],:port => 9300 } == "host1:9300,host2:9300"
+        # Case 7: :host in array with more than one items and :port =~ /^\d+_\d+$/
+        insist { client.send :hosts, :host => ["host1","host2"],:port => "9300-9302" } == "host1:9300,host1:9301,host1:9302,host2:9300,host2:9301,host2:9302"
+        # Case 8: :host in array with more than one items and some :host =~ /^.+:.+$/
+        insist { client.send :hosts, :host => ["host1","host2:9303"],:port => 9300 } == "host1:9300,host2:9303"
+        # Case 9: :host in array with more than one items, :port =~ /^\d+_\d+$/ and some :host =~ /^.+:.+$/
+        insist { client.send :hosts, :host => ["host1","host2:9303"],:port => "9300-9302" } == "host1:9300,host1:9301,host1:9302,host2:9303"
+      end
+    end
+  end
+
+  describe "Authentication option" do
+    ["node", "transport"].each do |protocol|
+      context "with protocol => #{protocol}" do
+        subject do
+          require "logstash/outputs/elasticsearch"
+          settings = {
+            "protocol" => protocol,
+            "node_name" => "logstash",
+            "cluster" => "elasticsearch",
+            "host" => "node01",
+            "user" => "test",
+            "password" => "test"
+          }
+          next LogStash::Outputs::ElasticSearch.new(settings)
+        end
+
+        it "should fail in register" do
+          expect {subject.register}.to raise_error
+        end
+      end
+    end
+  end
+
+  describe "SSL option" do
+    ["node", "transport"].each do |protocol|
+      context "with protocol => #{protocol}" do
+        subject do
+          require "logstash/outputs/elasticsearch"
+          settings = {
+            "protocol" => protocol,
+            "node_name" => "logstash",
+            "cluster" => "elasticsearch",
+            "host" => "node01",
+            "ssl" => true
+          }
+          next LogStash::Outputs::ElasticSearch.new(settings)
+        end
+
+        it "should fail in register" do
+          expect {subject.register}.to raise_error
+        end
+      end
+    end
+  end
+
+  describe "send messages to ElasticSearch using HTTPS", :elasticsearch_secure => true do
+    subject do
+      require "logstash/outputs/elasticsearch"
+      settings = {
+        "protocol" => "http",
+        "node_name" => "logstash",
+        "cluster" => "elasticsearch",
+        "host" => "node01",
+        "user" => "user",
+        "password" => "changeme",
+        "ssl" => true,
+        "cacert" => "/tmp/ca/certs/cacert.pem",
+        # or
+        #"truststore" => "/tmp/ca/truststore.jks",
+        #"truststore_password" => "testeteste"
+      }
+      next LogStash::Outputs::ElasticSearch.new(settings)
+    end
+
+    before :each do
+      subject.register
+    end
+
+    it "sends events to ES" do
+      expect {
+        subject.receive(LogStash::Event.new("message" => "sample message here"))
+        subject.buffer_flush(:final => true)
+      }.to_not raise_error
+    end
+  end
+
+  describe "connect using HTTP Authentication", :elasticsearch_secure => true do
+    subject do
+      require "logstash/outputs/elasticsearch"
+      settings = {
+        "protocol" => "http",
+        "cluster" => "elasticsearch",
+        "host" => "node01",
+        "user" => "user",
+        "password" => "changeme",
+      }
+      next LogStash::Outputs::ElasticSearch.new(settings)
+    end
+
+    before :each do
+      subject.register
+    end
+
+    it "sends events to ES" do
+      expect {
+        subject.receive(LogStash::Event.new("message" => "sample message here"))
+        subject.buffer_flush(:final => true)
+      }.to_not raise_error
+    end
+  end
 end

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.5
 platform: ruby
 authors:
 - Elasticsearch
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-04 00:00:00.000000000 Z
+date: 2014-11-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: elasticsearch
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.6
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.6
 - !ruby/object:Gem::Dependency
   name: stud
   requirement: !ruby/object:Gem::Requirement
@@ -58,14 +58,14 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.0.39
+        version: 0.0.40
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.0.39
+        version: 0.0.40
 - !ruby/object:Gem::Dependency
   name: logstash
   requirement: !ruby/object:Gem::Requirement
@@ -90,32 +90,39 @@ dependencies:
   name: jar-dependencies
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.0.6
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.0.6
+        version: '0'
 description: Output events to elasticsearch
-email: rubycoder@example.com
+email: richard.pijnenburg@elasticsearch.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- .gitignore
+- Gemfile
+- LICENSE
+- Rakefile
 - lib/logstash/outputs/elasticsearch.rb
 - lib/logstash/outputs/elasticsearch/elasticsearch-template.json
 - lib/logstash/outputs/elasticsearch/protocol.rb
 - logstash-output-elasticsearch.gemspec
+- rakelib/publish.rake
+- rakelib/vendor.rake
 - spec/outputs/elasticsearch.rb
 homepage: http://logstash.net/
 licenses:
 - Apache License (2.0)
 metadata:
   logstash_plugin: 'true'
+  logstash_group: output
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -131,12 +138,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements:
-- jar 'org.elasticsearch:elasticsearch', '1.2.2'
+- jar 'org.elasticsearch:elasticsearch', '1.3.1'
 rubyforge_project: 
-rubygems_version: 2.3.0
+rubygems_version: 2.4.1
 signing_key: 
 specification_version: 4
 summary: Logstash Output to Elasticsearch
 test_files:
 - spec/outputs/elasticsearch.rb
-has_rdoc: