logstash-output-elasticsearch 0.1.6 → 3.0.0

data/spec/unit/outputs/elasticsearch_ssl_spec.rb

@@ -0,0 +1,55 @@
+require_relative "../../../spec/es_spec_helper"
+require 'stud/temporary'
+
+describe "SSL option" do
+  context "when using ssl without cert verification" do
+    subject do
+      require "logstash/outputs/elasticsearch"
+      settings = {
+        "hosts" => "node01",
+        "ssl" => true,
+        "ssl_certificate_verification" => false
+      }
+      next LogStash::Outputs::ElasticSearch.new(settings)
+    end
+
+    it "should pass the flag to the ES client" do
+      expect(::Elasticsearch::Client).to receive(:new) do |args|
+        expect(args[:ssl]).to eq(:enabled => true, :verify => false)
+      end
+      subject.register
+    end
+
+    it "print a warning" do
+      expect(subject.logger).to receive(:warn)
+      subject.register
+    end
+  end
+
+  context "when using ssl with client certificates" do
+    let(:keystore_path) { Stud::Temporary.file.path }
+
+    after :each do
+      File.delete(keystore_path)
+    end
+
+    subject do
+      require "logstash/outputs/elasticsearch"
+      settings = {
+        "hosts" => "node01",
+        "ssl" => true,
+        "keystore" => keystore_path,
+        "keystore_password" => "test"
+      }
+      next LogStash::Outputs::ElasticSearch.new(settings)
+    end
+
+    it "should pass the keystore parameters to the ES client" do
+      expect(::Elasticsearch::Client).to receive(:new) do |args|
+        expect(args[:ssl]).to include(:keystore => keystore_path, :keystore_password => "test")
+      end
+      subject.register
+    end
+
+  end
+end

metadata

@@ -1,143 +1,212 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 3.0.0
 platform: ruby
 authors:
-- Elasticsearch
+- Elastic
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-15 00:00:00.000000000 Z
+date: 2016-05-06 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: elasticsearch
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.6
-    - - ~>
+        version: 1.0.13
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.6
-    - - ~>
+        version: 1.0.13
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
   name: stud
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.0.17
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.0.17
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.0'
 - !ruby/object:Gem::Dependency
   name: cabin
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.6'
 - !ruby/object:Gem::Dependency
-  name: logstash
+  name: logstash-core-plugin-api
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.4.0
-    - - <
-      - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.4.0
-    - - <
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: ftw
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 0.0.42
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.42
 - !ruby/object:Gem::Dependency
-  name: jar-dependencies
+  name: logstash-codec-plain
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: ftw
+  name: logstash-devutils
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.0.40
-    - - ~>
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: longshoreman
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: flores
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.0.40
-    - - ~>
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: Output events to elasticsearch
-email: richard.pijnenburg@elasticsearch.com
+description: This gem is a Logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
+  gem is not a stand-alone program
+email: info@elastic.co
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- CHANGELOG.md
+- CONTRIBUTORS
 - Gemfile
 - LICENSE
-- Rakefile
+- NOTICE.TXT
+- README.md
 - lib/logstash/outputs/elasticsearch.rb
+- lib/logstash/outputs/elasticsearch/buffer.rb
+- lib/logstash/outputs/elasticsearch/common.rb
+- lib/logstash/outputs/elasticsearch/common_configs.rb
 - lib/logstash/outputs/elasticsearch/elasticsearch-template.json
-- lib/logstash/outputs/elasticsearch/protocol.rb
+- lib/logstash/outputs/elasticsearch/http_client.rb
+- lib/logstash/outputs/elasticsearch/http_client_builder.rb
+- lib/logstash/outputs/elasticsearch/template_manager.rb
 - logstash-output-elasticsearch.gemspec
-- rakelib/publish.rake
-- rakelib/vendor.rake
-- spec/outputs/elasticsearch.rb
+- spec/es_spec_helper.rb
+- spec/fixtures/scripts/scripted_update.groovy
+- spec/fixtures/scripts/scripted_update_nested.groovy
+- spec/fixtures/scripts/scripted_upsert.groovy
+- spec/integration/outputs/create_spec.rb
+- spec/integration/outputs/index_spec.rb
+- spec/integration/outputs/parent_spec.rb
+- spec/integration/outputs/pipeline_spec.rb
+- spec/integration/outputs/retry_spec.rb
+- spec/integration/outputs/routing_spec.rb
+- spec/integration/outputs/secure_spec.rb
+- spec/integration/outputs/templates_spec.rb
+- spec/integration/outputs/update_spec.rb
+- spec/unit/buffer_spec.rb
+- spec/unit/http_client_builder_spec.rb
+- spec/unit/outputs/elasticsearch/http_client_spec.rb
+- spec/unit/outputs/elasticsearch_proxy_spec.rb
+- spec/unit/outputs/elasticsearch_spec.rb
+- spec/unit/outputs/elasticsearch_ssl_spec.rb
 homepage: http://logstash.net/
 licenses:
-- Apache License (2.0)
+- apache-2.0
 metadata:
   logstash_plugin: 'true'
   logstash_group: output
@@ -147,20 +216,37 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
-requirements:
-- jar 'org.elasticsearch:elasticsearch', '1.4.0'
+requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.1
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Logstash Output to Elasticsearch
 test_files:
-- spec/outputs/elasticsearch.rb
+- spec/es_spec_helper.rb
+- spec/fixtures/scripts/scripted_update.groovy
+- spec/fixtures/scripts/scripted_update_nested.groovy
+- spec/fixtures/scripts/scripted_upsert.groovy
+- spec/integration/outputs/create_spec.rb
+- spec/integration/outputs/index_spec.rb
+- spec/integration/outputs/parent_spec.rb
+- spec/integration/outputs/pipeline_spec.rb
+- spec/integration/outputs/retry_spec.rb
+- spec/integration/outputs/routing_spec.rb
+- spec/integration/outputs/secure_spec.rb
+- spec/integration/outputs/templates_spec.rb
+- spec/integration/outputs/update_spec.rb
+- spec/unit/buffer_spec.rb
+- spec/unit/http_client_builder_spec.rb
+- spec/unit/outputs/elasticsearch/http_client_spec.rb
+- spec/unit/outputs/elasticsearch_proxy_spec.rb
+- spec/unit/outputs/elasticsearch_spec.rb
+- spec/unit/outputs/elasticsearch_ssl_spec.rb

data/.gitignore

@@ -1,4 +0,0 @@
-*.gem
-Gemfile.lock
-.bundle
-vendor

data/Rakefile

@@ -1,6 +0,0 @@
-@files=[]
-
-task :default do
-  system("rake -T")
-end
-

data/lib/logstash/outputs/elasticsearch/protocol.rb

@@ -1,253 +0,0 @@
-require "logstash/outputs/elasticsearch"
-require "cabin"
-
-module LogStash::Outputs::Elasticsearch
-  module Protocols
-    class Base
-      private
-      def initialize(options={})
-        # host(s), port, cluster
-        @logger = Cabin::Channel.get
-      end
-
-      def client
-        return @client if @client
-        @client = build_client(@options)
-        return @client
-      end # def client
-
-
-      def template_install(name, template, force=false)
-        if template_exists?(name) && !force
-          @logger.debug("Found existing Elasticsearch template. Skipping template management", :name => name)
-          return
-        end
-        template_put(name, template)
-      end
-
-      # Do a bulk request with the given actions.
-      #
-      # 'actions' is expected to be an array of bulk requests as string json
-      # values.
-      #
-      # Each 'action' becomes a single line in the bulk api call. For more
-      # details on the format of each.
-      def bulk(actions)
-        raise NotImplemented, "You must implement this yourself"
-        # bulk([
-        # '{ "index" : { "_index" : "test", "_type" : "type1", "_id" : "1" } }',
-        # '{ "field1" : "value1" }'
-        #])
-      end
-
-      public(:initialize, :template_install)
-    end
-
-    class HTTPClient < Base
-      private
-
-      DEFAULT_OPTIONS = {
-        :port => 9200
-      }
-
-      def initialize(options={})
-        super
-        require "elasticsearch" # gem 'elasticsearch-ruby'
-        # manticore http transport
-        require "elasticsearch/transport/transport/http/manticore"
-        @options = DEFAULT_OPTIONS.merge(options)
-        @client = client
-      end
-
-      def build_client(options)
-        uri = "#{options[:protocol]}://#{options[:host]}:#{options[:port]}"
-
-        client_options = {
-          :host => [uri],
-          :transport_options => options[:client_settings]
-        }
-        client_options[:transport_class] = ::Elasticsearch::Transport::Transport::HTTP::Manticore
-        client_options[:ssl] = client_options[:transport_options].delete(:ssl)
-
-        if options[:user] && options[:password] then
-          token = Base64.strict_encode64(options[:user] + ":" + options[:password])
-          client_options[:headers] = { "Authorization" => "Basic #{token}" }
-        end
-
-        Elasticsearch::Client.new client_options
-      end
-
-      def bulk(actions)
-        @client.bulk(:body => actions.collect do |action, args, source|
-          if source
-            next [ { action => args }, source ]
-          else
-            next { action => args }
-          end
-        end.flatten)
-      end # def bulk
-
-      def template_exists?(name)
-        @client.indices.get_template(:name => name)
-        return true
-      rescue Elasticsearch::Transport::Transport::Errors::NotFound
-        return false
-      end # def template_exists?
-
-      def template_put(name, template)
-        @client.indices.put_template(:name => name, :body => template)
-      end # template_put
-
-      public(:bulk)
-    end # class HTTPClient
-
-    class NodeClient < Base
-      private
-
-      DEFAULT_OPTIONS = {
-        :port => 9300,
-      }
-
-      def initialize(options={})
-        super
-        require "java"
-        @options = DEFAULT_OPTIONS.merge(options)
-        setup(@options)
-        @client = client
-      end # def initialize
-
-      def settings
-        return @settings
-      end
-
-      def setup(options={})
-        @settings = org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder
-        if options[:host]
-          @settings.put("discovery.zen.ping.multicast.enabled", false)
-          @settings.put("discovery.zen.ping.unicast.hosts", hosts(options))
-        end
-
-        @settings.put("node.client", true)
-        @settings.put("http.enabled", false)
-
-        if options[:client_settings]
-          options[:client_settings].each do |key, value|
-            @settings.put(key, value)
-          end
-        end
-
-        return @settings
-      end
-
-      def hosts(options)
-        # http://www.elasticsearch.org/guide/reference/modules/discovery/zen/
-        result = Array.new
-        if options[:host].class == Array
-          options[:host].each do |host|
-            if host.to_s =~ /^.+:.+$/
-              # For host in format: host:port, ignore options[:port]
-              result << host
-            else
-              if options[:port].to_s =~ /^\d+-\d+$/
-                # port ranges are 'host[port1-port2]'
-                result << Range.new(*options[:port].split("-")).collect { |p| "#{host}:#{p}" }
-              else
-                result << "#{host}:#{options[:port]}"
-              end
-            end
-          end
-        else
-          if options[:host].to_s =~ /^.+:.+$/
-            # For host in format: host:port, ignore options[:port]
-            result << options[:host]
-          else
-            if options[:port].to_s =~ /^\d+-\d+$/
-              # port ranges are 'host[port1-port2]' according to
-              # http://www.elasticsearch.org/guide/reference/modules/discovery/zen/
-              # However, it seems to only query the first port.
-              # So generate our own list of unicast hosts to scan.
-              range = Range.new(*options[:port].split("-"))
-              result << range.collect { |p| "#{options[:host]}:#{p}" }
-            else
-              result << "#{options[:host]}:#{options[:port]}"
-            end
-          end
-        end
-        result.flatten.join(",")
-      end # def hosts
-
-      def build_client(options)
-        nodebuilder = org.elasticsearch.node.NodeBuilder.nodeBuilder
-        return nodebuilder.settings(@settings).node.client
-      end # def build_client
-
-      def bulk(actions)
-        # Actions an array of [ action, action_metadata, source ]
-        prep = @client.prepareBulk
-        actions.each do |action, args, source|
-          prep.add(build_request(action, args, source))
-        end
-        response = prep.execute.actionGet()
-
-        # TODO(sissel): What format should the response be in?
-      end # def bulk
-
-      def build_request(action, args, source)
-        case action
-          when "index"
-            request = org.elasticsearch.action.index.IndexRequest.new(args[:_index])
-            request.id(args[:_id]) if args[:_id]
-            request.source(source)
-          when "delete"
-            request = org.elasticsearch.action.delete.DeleteRequest.new(args[:_index])
-            request.id(args[:_id])
-          #when "update"
-          #when "create"
-        end # case action
-
-        request.type(args[:_type]) if args[:_type]
-        return request
-      end # def build_request
-
-      def template_exists?(name)
-        request = org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesRequestBuilder.new(@client.admin.indices, name)
-        response = request.get
-        return !response.getIndexTemplates.isEmpty
-      end # def template_exists?
-
-      def template_put(name, template)
-        request = org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequestBuilder.new(@client.admin.indices, name)
-        request.setSource(LogStash::Json.dump(template))
-
-        # execute the request and get the response, if it fails, we'll get an exception.
-        request.get
-      end # template_put
-
-      public(:initialize, :bulk)
-    end # class NodeClient
-
-    class TransportClient < NodeClient
-      private
-      def build_client(options)
-        client = org.elasticsearch.client.transport.TransportClient.new(settings.build)
-
-        if options[:host]
-          client.addTransportAddress(
-            org.elasticsearch.common.transport.InetSocketTransportAddress.new(
-              options[:host], options[:port].to_i
-            )
-          )
-        end
-
-        return client
-      end # def build_client
-    end # class TransportClient
-  end # module Protocols
-
-  module Requests
-    class GetIndexTemplates; end
-    class Bulk; end
-    class Index; end
-    class Delete; end
-  end
-end