logstash-output-elasticsearch 0.1.6 → 3.0.0

data/lib/logstash/outputs/elasticsearch/elasticsearch-template.json

@@ -5,30 +5,42 @@
   },
   "mappings" : {
     "_default_" : {
-       "_all" : {"enabled" : true},
-       "dynamic_templates" : [ {
-         "string_fields" : {
-           "match" : "*",
-           "match_mapping_type" : "string",
-           "mapping" : {
-             "type" : "string", "index" : "analyzed", "omit_norms" : true,
-               "fields" : {
-                 "raw" : {"type": "string", "index" : "not_analyzed", "ignore_above" : 256}
-               }
-           }
-         }
-       } ],
-       "properties" : {
-         "@version": { "type": "string", "index": "not_analyzed" },
-         "geoip"  : {
-           "type" : "object",
-             "dynamic": true,
-             "path": "full",
-             "properties" : {
-               "location" : { "type" : "geo_point" }
-             }
-         }
-       }
+      "_all" : {"enabled" : true, "omit_norms" : true},
+      "dynamic_templates" : [ {
+        "message_field" : {
+          "match" : "message",
+          "match_mapping_type" : "string",
+          "mapping" : {
+            "type" : "string", "index" : "analyzed", "omit_norms" : true,
+            "fielddata" : { "format" : "disabled" }
+          }
+        }
+      }, {
+        "string_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "string",
+          "mapping" : {
+            "type" : "string", "index" : "analyzed", "omit_norms" : true,
+            "fielddata" : { "format" : "disabled" },
+            "fields" : {
+              "raw" : {"type": "string", "index" : "not_analyzed", "ignore_above" : 256}
+            }
+          }
+        }
+      } ],
+      "properties" : {
+        "@timestamp": { "type": "date" },
+        "@version": { "type": "string", "index": "not_analyzed" },
+        "geoip"  : {
+          "dynamic": true,
+          "properties" : {
+            "ip": { "type": "ip" },
+            "location" : { "type" : "geo_point" },
+            "latitude" : { "type" : "float" },
+            "longitude" : { "type" : "float" }
+          }
+        }
+      }
     }
   }
 }

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -0,0 +1,236 @@
+require "logstash/outputs/elasticsearch"
+require "cabin"
+require "base64"
+require "elasticsearch"
+require "elasticsearch/transport/transport/http/manticore"
+
+module LogStash; module Outputs; class ElasticSearch;
+  class HttpClient
+    attr_reader :client, :options, :client_options, :sniffer_thread
+    # This is here in case we use DEFAULT_OPTIONS in the future
+    # DEFAULT_OPTIONS = {
+    #   :setting => value
+    # }
+
+    def initialize(options={})
+      @logger = options[:logger]
+      # Again, in case we use DEFAULT_OPTIONS in the future, uncomment this.
+      # @options = DEFAULT_OPTIONS.merge(options)
+      @options = options
+      @client = build_client(@options)
+      # mutex to prevent requests and sniffing to access the
+      # connection pool at the same time
+      @request_mutex = Mutex.new
+      start_sniffing!
+    end
+
+    def template_install(name, template, force=false)
+      @request_mutex.synchronize do
+        if template_exists?(name) && !force
+          @logger.debug("Found existing Elasticsearch template. Skipping template management", :name => name)
+          return
+        end
+        template_put(name, template)
+      end
+    end
+
+    def bulk(actions)
+      @request_mutex.synchronize { non_threadsafe_bulk(actions) }
+    end
+
+    def non_threadsafe_bulk(actions)
+      return if actions.empty?
+      bulk_body = actions.collect do |action, args, source|
+        args, source = update_action_builder(args, source) if action == 'update'
+
+        if source && action != 'delete'
+          next [ { action => args.merge({ :data => source }) } ]
+        else
+          next { action => args }
+        end
+      end.flatten
+
+      @client.bulk(:body => bulk_body)
+    end
+
+    def start_sniffing!
+      if options[:sniffing]
+        @sniffer_thread = Thread.new do
+          loop do
+            @request_mutex.synchronize { sniff! }
+            sleep (options[:sniffing_delay].to_f || 30)
+          end
+        end
+      end
+    end
+
+    def stop_sniffing!
+      @sniffer_thread.kill() if @sniffer_thread
+    end
+
+    def sniff!
+      client.transport.reload_connections! if options[:sniffing]
+      hosts_by_name = client.transport.hosts.map {|h| h["name"]}.sort
+      @logger.debug({"count" => hosts_by_name.count, "hosts" => hosts_by_name})
+    rescue StandardError => e
+      @logger.error("Error while sniffing connection",
+                    :message => e.message,
+                    :class => e.class.name,
+                    :backtrace => e.backtrace)
+    end
+
+    private
+
+    # Builds a client and returns an Elasticsearch::Client
+    #
+    # The `options` is a hash where the following symbol keys have meaning:
+    #
+    # * `:hosts` - array of String. Set a list of hosts to use for communication.
+    # * `:port` - number. set the port to use to communicate with Elasticsearch
+    # * `:user` - String. The user to use for authentication.
+    # * `:password` - String. The password to use for authentication.
+    # * `:timeout` - Float. A duration value, in seconds, after which a socket
+    #    operation or request will be aborted if not yet successfull
+    # * `:client_settings` - a hash; see below for keys.
+    #
+    # The `client_settings` key is a has that can contain other settings:
+    #
+    # * `:ssl` - Boolean. Enable or disable SSL/TLS.
+    # * `:proxy` - String. Choose a HTTP HTTProxy to use.
+    # * `:path` - String. The leading path for prefixing Elasticsearch
+    #   requests. This is sometimes used if you are proxying Elasticsearch access
+    #   through a special http path, such as using mod_rewrite.
+    def build_client(options)
+      hosts = options[:hosts] || ["127.0.0.1"]
+      client_settings = options[:client_settings] || {}
+      timeout = options[:timeout] || 0
+
+      host_ssl_opt = client_settings[:ssl].nil? ? nil : client_settings[:ssl][:enabled]
+      urls = hosts.map {|host| host_to_url(host, host_ssl_opt, client_settings[:path])}
+
+      @client_options = {
+        :hosts => urls,
+        :ssl => client_settings[:ssl],
+        :transport_options => {
+          :socket_timeout => timeout,
+          :request_timeout => timeout,
+          :proxy => client_settings[:proxy]
+        },
+        :transport_class => ::Elasticsearch::Transport::Transport::HTTP::Manticore
+      }
+
+      if options[:user] && options[:password] then
+        token = Base64.strict_encode64(options[:user] + ":" + options[:password])
+        @client_options[:headers] = { "Authorization" => "Basic #{token}" }
+      end
+
+      @logger.debug? && @logger.debug("Elasticsearch HTTP client options", client_options)
+
+      Elasticsearch::Client.new(client_options)
+    end
+
+    HOSTNAME_PORT_REGEX=/\A(?<hostname>([A-Za-z0-9\.\-]+)|\[[0-9A-Fa-f\:]+\])(:(?<port>\d+))?\Z/
+    URL_REGEX=/\A#{URI::regexp(['http', 'https'])}\z/
+    # Parse a configuration host to a normalized URL
+    def host_to_url(host, ssl=nil, path=nil)
+      explicit_scheme = case ssl
+                        when true
+                          "https"
+                        when false
+                          "http"
+                        when nil
+                          nil
+                        else
+                          raise ArgumentError, "Unexpected SSL value!"
+                        end
+
+      # Ensure path starts with a /
+      if path && path[0] != '/'
+        path = "/#{path}"
+      end
+
+      url = nil
+      if host =~ URL_REGEX
+        url = URI.parse(host)
+
+        # Please note that the ssl == nil case is different! If you didn't make an explicit
+        # choice we don't complain!
+        if url.scheme == "http" && ssl == true
+          raise LogStash::ConfigurationError, "You specified a plain 'http' URL '#{host}' but set 'ssl' to true! Aborting!"
+        elsif url.scheme == "https" && ssl == false
+          raise LogStash::ConfigurationError, "You have explicitly disabled SSL but passed in an https URL '#{host}'! Aborting!"
+        end
+
+        url.scheme = explicit_scheme if explicit_scheme
+      elsif (match_results = HOSTNAME_PORT_REGEX.match(host))
+        hostname = match_results["hostname"]
+        port = match_results["port"] || 9200
+        url = URI.parse("#{explicit_scheme || 'http'}://#{hostname}:#{port}")
+      else
+        raise LogStash::ConfigurationError, "Host '#{host}' was specified, but is not valid! Use either a full URL or a hostname:port string!"
+      end
+
+      if path && url.path && url.path != "/" && url.path != ''
+        raise LogStash::ConfigurationError, "A path '#{url.path}' has been explicitly specified in the url '#{url}', but you also specified a path of '#{path}'. This is probably a mistake, please remove one setting."
+      end
+
+      if path
+        url.path = path  # The URI library cannot stringify if it holds a nil
+      end
+
+      if url.password || url.user
+        raise LogStash::ConfigurationError, "We do not support setting the user password in the URL directly as " +
+          "this may be logged to disk thus leaking credentials. Use the 'user' and 'password' options respectively"
+      end
+
+      url.to_s
+    end
+
+    def template_exists?(name)
+      @client.indices.get_template(:name => name)
+      return true
+    rescue Elasticsearch::Transport::Transport::Errors::NotFound
+      return false
+    end
+
+    def template_put(name, template)
+      @client.indices.put_template(:name => name, :body => template)
+    end
+
+    # Build a bulk item for an elasticsearch update action
+    def update_action_builder(args, source)
+      if args[:_script]
+        # Use the event as a hash from your script with variable name defined
+        # by script_var_name (default: "event")
+        # Ex: event["@timestamp"]
+        source_orig = source
+        source = { 'script' => {'params' => { @options[:script_var_name] => source_orig }} }
+        if @options[:scripted_upsert]
+          source['scripted_upsert'] = true
+          source['upsert'] = {}
+        elsif @options[:doc_as_upsert]
+          source['upsert'] = source_orig
+        else
+          source['upsert'] = args.delete(:_upsert) if args[:_upsert]
+        end
+        case @options[:script_type]
+        when 'indexed'
+          source['script']['id'] = args.delete(:_script)
+        when 'file'
+          source['script']['file'] = args.delete(:_script)
+        when 'inline'
+          source['script']['inline'] = args.delete(:_script)
+        end
+        source['script']['lang'] = @options[:script_lang] if @options[:script_lang] != ''
+      else
+        source = { 'doc' => source }
+        if @options[:doc_as_upsert]
+          source['doc_as_upsert'] = true
+        else
+          source['upsert'] = args.delete(:_upsert) if args[:_upsert]
+        end
+      end
+      [args, source]
+    end
+  end
+end end end

data/lib/logstash/outputs/elasticsearch/http_client_builder.rb

@@ -0,0 +1,106 @@
+module LogStash; module Outputs; class ElasticSearch;
+  module HttpClientBuilder
+    def self.build(logger, hosts, params)
+      client_settings = {}
+
+      common_options = {
+        :client_settings => client_settings,
+        :sniffing => params["sniffing"],
+        :sniffing_delay => params["sniffing_delay"]
+      }
+
+      common_options[:timeout] = params["timeout"] if params["timeout"]
+      client_settings[:path] = "/#{params["path"]}/".gsub(/\/+/, "/") # Normalize slashes
+      logger.debug? && logger.debug("Normalizing http path", :path => params["path"], :normalized => client_settings[:path])
+
+      client_settings.merge! setup_ssl(logger, params)
+      client_settings.merge! setup_proxy(logger, params)
+      common_options.merge! setup_basic_auth(logger, params)
+
+      # Update API setup
+      raise( Logstash::ConfigurationError,
+        "doc_as_upsert and scripted_upsert are mutually exclusive."
+      ) if params["doc_as_upsert"] and params["scripted_upsert"]
+
+      raise(
+        LogStash::ConfigurationError,
+        "Specifying action => 'update' needs a document_id."
+      ) if params['action'] == 'update' and params.fetch('document_id', '') == ''
+
+      # Update API setup
+      update_options = {
+        :doc_as_upsert => params["doc_as_upsert"],
+        :script_var_name => params["script_var_name"],
+        :script_type => params["script_type"],
+        :script_lang => params["script_lang"],
+        :scripted_upsert => params["scripted_upsert"]
+      }
+      common_options.merge! update_options if params["action"] == 'update'
+
+      LogStash::Outputs::ElasticSearch::HttpClient.new(
+        common_options.merge(:hosts => hosts, :logger => logger)
+      )
+    end
+
+    def self.setup_proxy(logger, params)
+      proxy = params["proxy"]
+      return {} unless proxy
+
+      # Symbolize keys
+      proxy = if proxy.is_a?(Hash)
+                Hash[proxy.map {|k,v| [k.to_sym, v]}]
+              elsif proxy.is_a?(String)
+                proxy
+              else
+                raise LogStash::ConfigurationError, "Expected 'proxy' to be a string or hash, not '#{proxy}''!"
+              end
+
+      return {:proxy => proxy}
+    end
+
+    def self.setup_ssl(logger, params)
+      return {} if params["ssl"].nil?
+      return {:ssl => {:enabled => false}} if params["ssl"] == false
+
+      cacert, truststore, truststore_password, keystore, keystore_password =
+        params.values_at('cacert', 'truststore', 'truststore_password', 'keystore', 'keystore_password')
+
+      if cacert && truststore
+        raise(LogStash::ConfigurationError, "Use either \"cacert\" or \"truststore\" when configuring the CA certificate") if truststore
+      end
+
+      ssl_options = {:enabled => true}
+
+      if cacert
+        ssl_options[:ca_file] = cacert
+      elsif truststore
+        ssl_options[:truststore_password] = truststore_password.value if truststore_password
+      end
+
+      ssl_options[:truststore] = truststore if truststore
+      if keystore
+        ssl_options[:keystore] = keystore
+        ssl_options[:keystore_password] = keystore_password.value if keystore_password
+      end
+      if !params["ssl_certificate_verification"]
+        logger.warn [
+                       "** WARNING ** Detected UNSAFE options in elasticsearch output configuration!",
+                       "** WARNING ** You have enabled encryption but DISABLED certificate verification.",
+                       "** WARNING ** To make sure your data is secure change :ssl_certificate_verification to true"
+                     ].join("\n")
+        ssl_options[:verify] = false
+      end
+      { ssl: ssl_options }
+    end
+
+    def self.setup_basic_auth(logger, params)
+      user, password = params["user"], params["password"]
+      return {} unless user && password
+
+      {
+        :user => user,
+        :password => password.value
+      }
+    end
+  end
+end; end; end

data/lib/logstash/outputs/elasticsearch/template_manager.rb

@@ -0,0 +1,35 @@
+module LogStash; module Outputs; class ElasticSearch
+  class TemplateManager
+    # To be mixed into the elasticsearch plugin base
+    def self.install_template(plugin)
+      return unless plugin.manage_template
+      plugin.logger.info("Using mapping template from", :path => plugin.template)
+      template = get_template(plugin.template)
+      plugin.logger.info("Attempting to install template", :manage_template => template)
+      install(plugin.client, plugin.template_name, template, plugin.template_overwrite)
+    rescue => e
+      plugin.logger.error("Failed to install template.", :message => e.message, :class => e.class.name)
+    end
+
+    private
+
+    def self.get_template(path)
+      template_path = path || default_template_path
+      read_template_file(template_path)
+    end
+
+    def self.install(client, template_name, template, template_overwrite)
+      client.template_install(template_name, template, template_overwrite)
+    end
+
+    def self.default_template_path
+      ::File.expand_path('elasticsearch-template.json', ::File.dirname(__FILE__))
+    end
+
+    def self.read_template_file(template_path)
+      raise ArgumentError, "Template file '#{@template_path}' could not be found!" unless ::File.exists?(template_path)
+      template_data = ::IO.read(template_path)
+      LogStash::Json.load(template_data)
+    end
+  end
+end end end

data/logstash-output-elasticsearch.gemspec

@@ -1,17 +1,17 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '0.1.6'
-  s.licenses        = ['Apache License (2.0)']
+  s.version         = '3.0.0'
+  s.licenses        = ['apache-2.0']
   s.summary         = "Logstash Output to Elasticsearch"
-  s.description     = "Output events to elasticsearch"
-  s.authors         = ["Elasticsearch"]
-  s.email           = 'richard.pijnenburg@elasticsearch.com'
+  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
+  s.authors         = ["Elastic"]
+  s.email           = 'info@elastic.co'
   s.homepage        = "http://logstash.net/"
   s.require_paths = ["lib"]
 
   # Files
-  s.files = `git ls-files`.split($\)
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
@@ -19,20 +19,22 @@ Gem::Specification.new do |s|
   # Special flag to let us know this is actually a logstash plugin
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
 
-  # Jar dependencies
-  s.requirements << "jar 'org.elasticsearch:elasticsearch', '1.4.0'"
-
   # Gem dependencies
-  s.add_runtime_dependency 'elasticsearch', ['>= 1.0.6', '~> 1.0']
+  s.add_runtime_dependency 'concurrent-ruby'
+  s.add_runtime_dependency 'elasticsearch', ['>= 1.0.13', '~> 1.0']
   s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']
   s.add_runtime_dependency 'cabin', ['~> 0.6']
-  s.add_runtime_dependency 'logstash', '>= 1.4.0', '< 2.0.0'
-  s.add_runtime_dependency 'jar-dependencies'
+  s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
 
-  s.add_development_dependency 'ftw', ['>= 0.0.40', '~> 0']
+  s.add_development_dependency 'ftw', '~> 0.0.42'
+  s.add_development_dependency 'logstash-codec-plain'
 
   if RUBY_PLATFORM == 'java'
-    s.add_runtime_dependency "manticore", '~> 0.3'
+    s.platform = RUBY_PLATFORM
+    s.add_runtime_dependency "manticore", '>= 0.5.4', '< 1.0.0'
   end
-end
 
+  s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'longshoreman'
+  s.add_development_dependency 'flores'
+end