logstash-output-amazon_es 0.1.3 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e48c468ce47cae67a17ebe6d49b5fe63f7cba5ae
-  data.tar.gz: 41533252d079d85996d9d1e93fb1575000e1121b
+  metadata.gz: 91df4bac9097a2019a5ff1d37a7bf34ed1e868e3
+  data.tar.gz: 0966b9abe9df86ab821c223a636c20e0a5ba4131
 SHA512:
-  metadata.gz: a667121c3bc20fe640af90a8bac57c65e29b69433141948684521a0b939d6ff7ed66864cb1cd4d8e6eccfa92eb8fb39853e4b9f379e3de79aaeb65a35025927e
-  data.tar.gz: fba50c632a5d6bdddaf5382eb1ce9907ca7ecc9b11a525a4af81b84df663fc808cf6a3f1ebe79bb57bc32dbfc087c4fc48e3fc6c75f8bcef6f44ba1aa3339cae
+  metadata.gz: 8a1394030e59655508565ad916c1a4b2cf63db64a688f709d04fd243cc66ca90474c870a6331971b9eff2278c9ac5a38e7e2a6f0125718a67b08e15ccc5aee2d
+  data.tar.gz: 0c88842929c77097e10bb0eaa8d8cd07efbf3e7c391faa4436cfb93fca4526e1deaf06b8b918ecf11d363085d198581c7dcacd0bf1d08cbeefcdd8a4836778b2

data/README.md

@@ -10,12 +10,12 @@ It is fully free and fully open source. The license is Apache 2.0, meaning you a
 One command installation
 `bin/plugin install logstash-output-amazon_es`
 
-While we are in the process of getting this plugin fully integrated within logstash to make installation simpler, 
+While we are in the process of getting this plugin fully integrated within logstash to make installation simpler,
 if above does not work, or you would like to patch code here is a workaround to install this plugin within your logstash:
 
 1. Check out/clone this code from github
 2. Build plugin using - `gem build logstash-output-amazon_es.gemspec` ( this works with jruby and rubygem versions > 1.9)
-3. Install plugin using `<logstash-home>/bin/plugin install logstash-output-amazon_es-0.1.0-java.gem` (or the non java variant)
+3. Install plugin using `<logstash-home>/bin/plugin install logstash-output-amazon_es-0.2.0-java.gem` (or the non java variant)
 
 ## Configuration for Amazon Elasticsearch Output plugin
 
@@ -28,15 +28,15 @@ An example configuration:
 	        hosts => ["foo.us-east-1.es.amazonaws.com"]
 	        region => "us-east-1"
 			aws_access_key_id => 'ACCESS_KEY' (Will be made optional in next release to support instance profiles)
-			aws_secret_access_key => 'SECRET_KEY' 
+			aws_secret_access_key => 'SECRET_KEY'
 			index => "production-logs-%{+YYYY.MM.dd}"
 		}
 	}
-  
+
 * Required Parameters
 	* hosts (array of string) - Amazon Elasticsearch domain endpoint. eg ["foo.us-east-1.es.amazonaws.com"]
     * region (string, :default => "us-east-1") - region where the domain is located
-    
+
 * Optional Parameters
 	* Credential parameters
 		* aws_access_key_id, :validate => :string - Optional AWS Access key
@@ -62,7 +62,8 @@ An example configuration:
 		This helps keep both fast and slow log streams moving along in near-real-time.
 	* template (path) - You can set the path to your own template here, if you so desire. If not set, the included template will be used.
 	* template_name (string, default => "logstash") - defines how the template is named inside Elasticsearch
-	* port (string, default 80) - Amazon Elasticsearch Service listens on port 80, if you have a custom proxy fronting elasticsearch, this parameter may need tweaking.
+	* port (string, default 443) - Amazon Elasticsearch Service listens on port 443 - https (default) and 80 - http. Tweak this for custom proxy.
+	* protocol (string, default https) - The protocol used to connect to the Amazon Elasticsearch Service
 
 ## Documentation
 
@@ -108,7 +109,7 @@ bundle exec rspec
 Dependencies: [Docker](http://docker.com)
 
 Before the test suite is run, we will load and run an
-Elasticsearch instance within a docker container. This container 
+Elasticsearch instance within a docker container. This container
 will be cleaned up when suite has finished.
 
 ```sh

data/lib/logstash/outputs/amazon_es.rb

@@ -14,11 +14,11 @@ require "uri"
 #
 #
 # The configuration and experience is similar to logstash-output-elasticsearch plugin and we have added Signature V4 support for the same
-# Some of the default configurations like connection timeouts have been tuned for optimal performance with Amazon Elasticsearch 
+# Some of the default configurations like connection timeouts have been tuned for optimal performance with Amazon Elasticsearch
 #
 # ==== Retry Policy
 #
-# This plugin uses the same retry policy as logstash-output-elasticsearch, It uses bulk API to optimize its  
+# This plugin uses the same retry policy as logstash-output-elasticsearch, It uses bulk API to optimize its
 # imports into Elasticsearch.. These requests may experience either partial or total failures.
 # Events are retried if they fail due to either a network error or the status codes
 # 429 (the server is busy), 409 (Version Conflict), or 503 (temporary overloading/maintenance).
@@ -53,13 +53,13 @@ class LogStash::Outputs::AmazonES < LogStash::Outputs::Base
 
   # The index type to write events to. Generally you should try to write only
   # similar events to the same 'type'. String expansion `%{foo}` works here.
-  # 
+  #
   # Deprecated in favor of `document_type` field.
   config :index_type, :validate => :string, :deprecated => "Please use the 'document_type' setting instead. It has the same effect, but is more appropriately named."
 
   # The document type to write events to. Generally you should try to write only
   # similar events to the same 'type'. String expansion `%{foo}` works here.
-  # Unless you set 'document_type', the event 'type' will be used if it exists 
+  # Unless you set 'document_type', the event 'type' will be used if it exists
   # otherwise the document type will be assigned the value of 'logs'
   config :document_type, :validate => :string
 
@@ -100,28 +100,31 @@ class LogStash::Outputs::AmazonES < LogStash::Outputs::Base
   config :routing, :validate => :string
 
 
-# Set the endpoint of your Amazon Elasticsearch domain. This will always be array of size 1 
+# Set the endpoint of your Amazon Elasticsearch domain. This will always be array of size 1
 #     ["foo.us-east-1.es.amazonaws.com"]
   config :hosts, :validate => :array
 
   # You can set the remote port as part of the host, or explicitly here as well
-  config :port, :validate => :string, :default => 80
-  
+  config :port, :validate => :string, :default => 443
+
+  # Sets the protocol thats used to connect to elastisearch
+  config :protocol, :validate => :string, :default => "https"
+
   #Signing specific details
   config :region, :validate => :string, :default => "us-east-1"
-  
-  # aws_access_key_id and aws_secret_access_key are currently needed for this plugin to work right. 
+
+  # aws_access_key_id and aws_secret_access_key are currently needed for this plugin to work right.
   # Subsequent versions will have the credential resolution logic as follows:
   #
-  # - User passed aws_access_key_id and aws_secret_access_key in aes configuration 
-  # - Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY 
-  #   (RECOMMENDED since they are recognized by all the AWS SDKs and CLI except for .NET), 
+  # - User passed aws_access_key_id and aws_secret_access_key in aes configuration
+  # - Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
+  #   (RECOMMENDED since they are recognized by all the AWS SDKs and CLI except for .NET),
   #   or AWS_ACCESS_KEY and AWS_SECRET_KEY (only recognized by Java SDK)
   # - Credential profiles file at the default location (~/.aws/credentials) shared by all AWS SDKs and the AWS CLI
   # - Instance profile credentials delivered through the Amazon EC2 metadata service
   config :aws_access_key_id, :validate => :string
   config :aws_secret_access_key, :validate => :string
-  
+
 
   # This plugin uses the bulk index api for improved indexing performance.
   # To make efficient bulk api calls, we will buffer a certain number of
@@ -224,7 +227,7 @@ class LogStash::Outputs::AmazonES < LogStash::Outputs::Base
     common_options.merge! update_options if @action == 'update'
 
     @client = LogStash::Outputs::AES::HttpClient.new(
-      common_options.merge(:hosts => @hosts, :port => @port, :region => @region, :aws_access_key_id => @aws_access_key_id, :aws_secret_access_key => @aws_secret_access_key)
+      common_options.merge(:hosts => @hosts, :port => @port, :region => @region, :aws_access_key_id => @aws_access_key_id, :aws_secret_access_key => @aws_secret_access_key,:protocol => @protocol)
     )
 
     if @manage_template
@@ -279,7 +282,7 @@ class LogStash::Outputs::AmazonES < LogStash::Outputs::Base
   def receive(event)
     return unless output?(event)
 
-    # block until we have not maxed out our 
+    # block until we have not maxed out our
     # retry queue. This is applying back-pressure
     # to slow down the receive-rate
     @retry_flush_mutex.synchronize {
@@ -303,7 +306,7 @@ class LogStash::Outputs::AmazonES < LogStash::Outputs::Base
       :_type => type,
       :_routing => @routing ? event.sprintf(@routing) : nil
     }
-    
+
     params[:_upsert] = LogStash::Json.load(event.sprintf(@upsert)) if @action == 'update' && @upsert != ""
 
     buffer_receive([event.sprintf(@action), params, event])
@@ -380,19 +383,19 @@ class LogStash::Outputs::AmazonES < LogStash::Outputs::Base
 
     @retry_teardown_requested.make_true
     # First, make sure retry_timer_thread is stopped
-    # to ensure we do not signal a retry based on 
+    # to ensure we do not signal a retry based on
     # the retry interval.
     Thread.kill(@retry_timer_thread)
     @retry_timer_thread.join
-    # Signal flushing in the case that #retry_flush is in 
+    # Signal flushing in the case that #retry_flush is in
     # the process of waiting for a signal.
     @retry_flush_mutex.synchronize { @retry_queue_needs_flushing.signal }
-    # Now, #retry_flush is ensured to not be in a state of 
+    # Now, #retry_flush is ensured to not be in a state of
     # waiting and can be safely joined into the main thread
     # for further final execution of an in-process remaining call.
     @retry_thread.join
 
-    # execute any final actions along with a proceeding retry for any 
+    # execute any final actions along with a proceeding retry for any
     # final actions that did not succeed.
     buffer_flush(:final => true)
     retry_flush
@@ -426,11 +429,11 @@ class LogStash::Outputs::AmazonES < LogStash::Outputs::Base
 
 
   private
-  # in charge of submitting any actions in @retry_queue that need to be 
+  # in charge of submitting any actions in @retry_queue that need to be
   # retried
   #
   # This method is not called concurrently. It is only called by @retry_thread
-  # and once that thread is ended during the teardown process, a final call 
+  # and once that thread is ended during the teardown process, a final call
   # to this method is done upon teardown in the main thread.
   def retry_flush()
     unless @retry_queue.empty?

data/lib/logstash/outputs/amazon_es/http_client.rb

@@ -59,10 +59,11 @@ module LogStash::Outputs::AES
     def build_client(options)
       hosts = options[:hosts]
       port = options[:port]
+      protocol = options[:protocol]
       client_settings = options[:client_settings] || {}
 
       uris = hosts.map do |host|
-        "http://#{host}:#{port}#{client_settings[:path]}".gsub(/[\/]+$/,'')
+        "#{protocol}://#{host}:#{port}#{client_settings[:path]}".gsub(/[\/]+$/,'')
       end
 
       @client_options = {
@@ -72,7 +73,7 @@ module LogStash::Outputs::AES
         :aws_secret_access_key => options[:aws_secret_access_key],
         :transport_options => {
           :request => {:open_timeout => 0, :timeout => 60},  # ELB timeouts are set at 60
-          :proxy => client_settings[:proxy],          
+          :proxy => client_settings[:proxy],
         },
         :transport_class => Elasticsearch::Transport::Transport::HTTP::AWS
       }

data/logstash-output-amazon_es.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-amazon_es'
-  s.version         = '0.1.3'
+  s.version         = '0.2.0'
   s.licenses        = ['apache-2.0']
   s.summary         = "Logstash Output to Amazon Elasticsearch Service"
   s.description     = "Output events to Amazon Elasticsearch Service with V4 signing"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-amazon_es
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.0
 platform: ruby
 authors:
 - Amazon
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-03 00:00:00.000000000 Z
+date: 2015-11-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby