logstash-output-elasticsearch 1.1.0-java → 2.0.0.beta4-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 21f30a9fb4fae09d20c43211c4a47a38a99892d0
-  data.tar.gz: c8a82c32a338ecc12109b251efab824707770429
+  metadata.gz: f3ed085b4e52cb51518cbcb5329a304fad4a33de
+  data.tar.gz: 028096af0d9e7e78c244e1234671e7424bfeefb6
 SHA512:
-  metadata.gz: 1db3b07141dd557c7fcd34dfcfe64fb7705c9c776faf5a9455c6417be00cd23b20e2e267dbcbdbd7477d3c54ac80ea30a65cfe69482722c98ad3505d934ef97b
-  data.tar.gz: cc2c42fd9b8af5caab9b6b15d957d988c7f0c748da737d9449abcd8202ef6f41ac8148c7f77c32a5b92641e43b3a334654f636a0541230f21375aab5820d4c67
+  metadata.gz: bd51e7fa9bcbcb2abd77f0cc8ddffba9231204a02c229e4a3ec041f9d5b88bed41c9752669807c18896030627dc70acf8af08c877cad9573ddca383992774bc3
+  data.tar.gz: 6556604b6d5303874ae953a1ddc5c74409f523811e888a67ebd00b7f1fd3a4a54c14bf51aa9f37dab5e309f71f28f19c692beeffe2b5d3c104215016c4376043

data/.gitignore

@@ -3,3 +3,4 @@ Gemfile.lock
 .bundle
 .idea
 *~
+.ruby-version

data/CHANGELOG.md

@@ -1,6 +1,13 @@
-## 1.1.0
-  - Add timeout setting.
-  
+## 2.0.0-beta2
+ - Massive internal refactor of client handling
+ - Background HTTP sniffing support
+ - Reduced bulk request size to 500 from 5000 (better memory utilization)
+ - Removed 'host' config option. Now use 'hosts'
+
+## 2.0.0-beta
+ - Only support HTTP Protocol
+ - Removed support for node and transport protocols (now in logstash-output-elasticsearch_java)
+ 
 ## 1.0.7
  - Add update API support
 

data/README.md

@@ -1,15 +1,15 @@
 # Logstash Plugin
 
-This is a plugin for [Logstash](https://github.com/elasticsearch/logstash).
+This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
 
 ## Documentation
 
-Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elasticsearch.org/guide/en/logstash/current/).
+Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elastic.co/guide/en/logstash/current/).
 
 - For formatting code or config example, you can use the asciidoc `[source,ruby]` directive
-- For more asciidoc formatting tips, see the excellent reference here https://github.com/elasticsearch/docs#asciidoc-guide
+- For more asciidoc formatting tips, see the excellent reference here https://github.com/elastic/docs#asciidoc-guide
 
 ## Need Help?
 
@@ -95,4 +95,4 @@ Programming is not a required skill. Whatever you've seen about open source and
 
 It is more important to the community that you are able to contribute.
 
-For more information about contributing, see the [CONTRIBUTING](https://github.com/elasticsearch/logstash/blob/master/CONTRIBUTING.md) file.
+For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -0,0 +1,144 @@
+require "logstash/outputs/elasticsearch"
+require "cabin"
+require "base64"
+require "elasticsearch"
+require "elasticsearch/transport/transport/http/manticore"
+
+module LogStash::Outputs::Elasticsearch
+  class HttpClient
+    attr_reader :client, :options, :client_options, :sniffer_thread
+    DEFAULT_OPTIONS = {
+      :port => 9200
+    }
+
+    def initialize(options={})
+      @logger = Cabin::Channel.get
+      @options = DEFAULT_OPTIONS.merge(options)
+      @client = build_client(@options)
+      start_sniffing!
+    end
+
+    def template_install(name, template, force=false)
+      if template_exists?(name) && !force
+        @logger.debug("Found existing Elasticsearch template. Skipping template management", :name => name)
+        return
+      end
+      template_put(name, template)
+    end
+
+    def bulk(actions)
+      bulk_body = actions.collect do |action, args, source|
+        if action == 'update'
+          if args[:_id]
+            source = { 'doc' => source }
+            if @options[:doc_as_upsert]
+              source['doc_as_upsert'] = true
+            else
+              source['upsert'] = args[:_upsert] if args[:_upsert]
+            end
+          else
+            raise(LogStash::ConfigurationError, "Specifying action => 'update' without a document '_id' is not supported.")
+          end
+        end
+
+        args.delete(:_upsert)
+
+        if source
+          next [ { action => args }, source ]
+        else
+          next { action => args }
+        end
+      end.flatten
+
+      bulk_response = @client.bulk(:body => bulk_body)
+
+      self.class.normalize_bulk_response(bulk_response)
+    end
+
+    def start_sniffing!
+      if options[:sniffing]
+        @sniffer_thread = Thread.new do
+          loop do
+            sniff!
+            sleep (options[:sniffing_delay].to_f || 30)
+          end
+        end
+      end
+    end
+
+    def stop_sniffing!
+      @sniffer_thread.kill() if @sniffer_thread
+    end
+
+    def sniff!
+      client.transport.reload_connections! if options[:sniffing]
+    rescue StandardError => e
+      @logger.error("Error while sniffing connection",
+                    :message => e.message,
+                    :class => e.class.name)
+    end
+
+    private
+
+    def build_client(options)
+      hosts = options[:hosts] || ["127.0.0.1"]
+      port = options[:port] || 9200
+      client_settings = options[:client_settings] || {}
+
+      uris = hosts.map do |host|
+        "http://#{host}:#{port}#{client_settings[:path]}"
+      end
+
+      @client_options = {
+        :hosts => uris,
+        :ssl => client_settings[:ssl],
+        :transport_options => {  # manticore settings so we
+          :socket_timeout => 0,  # do not timeout socket reads
+          :request_timeout => 0,  # and requests
+          :proxy => client_settings[:proxy]
+        },
+        :transport_class => ::Elasticsearch::Transport::Transport::HTTP::Manticore
+      }
+
+      if options[:user] && options[:password] then
+        token = Base64.strict_encode64(options[:user] + ":" + options[:password])
+        @client_options[:headers] = { "Authorization" => "Basic #{token}" }
+      end
+
+      Elasticsearch::Client.new(client_options)
+    end
+
+    def self.normalize_bulk_response(bulk_response)
+      if bulk_response["errors"]
+        # The structure of the response from the REST Bulk API is follows:
+        # {"took"=>74, "errors"=>true, "items"=>[{"create"=>{"_index"=>"logstash-2014.11.17",
+        #                                                    "_type"=>"logs",
+        #                                                    "_id"=>"AUxTS2C55Jrgi-hC6rQF",
+        #                                                    "_version"=>1,
+        #                                                    "status"=>400,
+        #                                                    "error"=>"MapperParsingException[failed to parse]..."}}]}
+        # where each `item` is a hash of {OPTYPE => Hash[]}. calling first, will retrieve
+        # this hash as a single array with two elements, where the value is the second element (i.first[1])
+        # then the status of that item is retrieved.
+        {
+          "errors" => true,
+          "statuses" => bulk_response["items"].map { |i| i.first[1]['status'] },
+          "details" => bulk_response["items"].select {|i| i.first[1]["error"] }.map {|i| i.first[1]}
+        }
+      else
+        {"errors" => false}
+      end
+    end
+
+    def template_exists?(name)
+      @client.indices.get_template(:name => name)
+      return true
+    rescue Elasticsearch::Transport::Transport::Errors::NotFound
+      return false
+    end
+
+    def template_put(name, template)
+      @client.indices.put_template(:name => name, :body => template)
+    end
+  end
+end