fluent-plugin-aws-elasticsearch-service 0.1.6 → 0.1.7.pre.alpha

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 336a3781840865fe97e3706eb663486e97c36b54
-  data.tar.gz: ecba7aa13b0569e3aa9940905c988320f38cabbe
+  metadata.gz: 2600c216583522a9225c296e0e08b69e83ab43b0
+  data.tar.gz: 3290c1a3dfef7268c864fdfc8537bcc9b98701c1
 SHA512:
-  metadata.gz: dc057bb9614641abbfe7058ba9b73523718dcf8ec41ac28ae7ddec31b0715590ebf1ecaf4dc82af6d8b93b52d3bf4ec3c3d671c9def22d7876cc939a08e63e54
-  data.tar.gz: 6215725de5851401a57545687022105c0119ab2b3798feee7a7ca107d73b207d6594fa48b23b7f7d32c5019571ca646f31595ffcf3aaf5fca553c5e0207db3a0
+  metadata.gz: ce5905d8c5681741da685cdf6cbae33becac308e63222f6a3461c4c730fadcd05e07ae372af8022d04a2c72ef2c4bde3f336fa9250a67b9e6920bfd516bebb92
+  data.tar.gz: 2381c6e58f8248368067040637d8c857dfd00a7b41aeb9641498b353e4582e554e40298b7085abf93fb858d0e53dd7a9991f5d934eae94d17dbc59de7dd947f0

data/README.md

@@ -90,7 +90,7 @@ Additionally, you can use an STS assumed role as the authenticating factor and i
  <endpoint>
     url https://CLUSTER_ENDPOINT_URL
     region eu-west-1
-    assume_role_arn arn:aws:sts::ACCOUNT:assumed-role/ROLE
+    assume_role_arn arn:aws:sts::ACCOUNT:role/ROLE
     assume_role_session_name SESSION_ID # Defaults to fluentd if omitted
   </endpoint>
 ```
@@ -126,6 +126,14 @@ You'll need to ensure that the environment in which the fluentd plugin runs has
 }
 ```
 
+## Troubleshooting
+
+* "Elasticsearch::Transport::Transport::Errors::Forbidden" error="[403]" even after verifying the access keys/roles/policies.
+   * Ensure you don't have a trailing slash on the endpoint URL in your fluentd configuration file (see CLUSTER_ENDPOINT_URL above).
+
+*  "ElasticsearchIllegalArgumentException[explicit index in bulk is not allowed]"
+   * Check that `rest.action.multi.allow_explicit` is set true on your Amazon ES domain (verify in the console - there's a bug in Terraform, https://github.com/hashicorp/terraform/issues/3980).
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/fluent-plugin-aws-elasticsearch-service.gemspec

@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-aws-elasticsearch-service"
-  spec.version       = "0.1.6"
+  spec.version       = "0.1.7-alpha"
   spec.authors       = ["atomita"]
   spec.email         = ["sleeping.cait.sith+gh@gmail.com"]
 

data/lib/fluent/plugin/out_aws-elasticsearch-service.rb

@@ -17,9 +17,16 @@ module Fluent
       config_param :access_key_id, :string, :default => ""
       config_param :secret_access_key, :string, :default => ""
       config_param :assume_role_arn, :string, :default => nil
+      config_param :ecs_container_credentials_relative_uri, :string, :default => nil #Set with AWS_CONTAINER_CREDENTIALS_RELATIVE_URI environment variable value
       config_param :assume_role_session_name, :string, :default => "fluentd"
     end
 
+    # here overrides default value of reload_connections to false because
+    # AWS Elasticsearch Service doesn't return addresses of nodes and Elasticsearch client
+    # fails to reload connections properly. This ends up "temporarily failed to flush the buffer"
+    # error repeating forever. See this discussion for details:
+    # https://discuss.elastic.co/t/elasitcsearch-ruby-raises-cannot-get-new-connection-from-pool-error/36252
+    config_set_default :reload_connections, false
 
     #
     # @override
@@ -63,10 +70,15 @@ module Fluent
           credentials = Aws::Credentials.new opts[:access_key_id], opts[:secret_access_key]
         else
           if opts[:assume_role_arn].nil?
-            credentials = Aws::SharedCredentials.new({
-                             retries: 2
-                          }).credentials
-            credentials ||= Aws::InstanceProfileCredentials.new.credentials
+            if opts[:ecs_container_credentials_relative_uri].nil?
+              credentials = Aws::SharedCredentials.new({retries: 2}).credentials
+              credentials ||= Aws::InstanceProfileCredentials.new.credentials
+              credentials ||= Aws::ECSCredentials.new.credentials
+            else
+              credentials = Aws::ECSCredentials.new({
+                credential_path: opts[:ecs_container_credentials_relative_uri]
+              }).credentials
+            end
           else
             credentials = sts_credential_provider({
                             role_arn: opts[:assume_role_arn],
@@ -80,7 +92,7 @@ module Fluent
       end
       def calback.inspect
         credentials = self.call
-        "#<#{credentials.class.name} access_key_id=#{credentials.access_key_id.inspect}>"
+        credentials.credentials.inspect
       end
       calback
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-aws-elasticsearch-service
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.7.pre.alpha
 platform: ruby
 authors:
 - atomita
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-06-23 00:00:00.000000000 Z
+date: 2017-10-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -143,12 +143,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: Output plugin to post to "Amazon Elasticsearch Service".