fluent-plugin-aws-elasticsearch-service 0.1.4 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 804bbbcf1d607ae465fa39f132a6b97a843d1673
-  data.tar.gz: 780c3494851839ab063e2f63d1d86cf0f25a26df
+  metadata.gz: 336a3781840865fe97e3706eb663486e97c36b54
+  data.tar.gz: ecba7aa13b0569e3aa9940905c988320f38cabbe
 SHA512:
-  metadata.gz: a2ca9588aec45f2945c6f6c822a61bc7791c9fa712b35addae722478f1bd49b7ef081c2c93e59b01d38be84b16f7a6f498cf6d9d2dd05be09c45e6df856bbd5c
-  data.tar.gz: f908b30536317c870756531b26848e6834cc42313f521f8e119da967fa586776a3ecf7f88fbc345eb98a6475aa16bf3c685466260c67468fa67249de1bc9668e
+  metadata.gz: dc057bb9614641abbfe7058ba9b73523718dcf8ec41ac28ae7ddec31b0715590ebf1ecaf4dc82af6d8b93b52d3bf4ec3c3d671c9def22d7876cc939a08e63e54
+  data.tar.gz: 6215725de5851401a57545687022105c0119ab2b3798feee7a7ca107d73b207d6594fa48b23b7f7d32c5019571ca646f31595ffcf3aaf5fca553c5e0207db3a0

data/Gemfile

@@ -5,4 +5,4 @@ gemspec
 
 gem 'fluent-plugin-elasticsearch', '~> 1.0', require: false
 gem 'aws-sdk', '~> 2', require: false
-gem 'faraday_middleware-aws-signers-v4', '>= 0.1.0, < 0.1.2', require: false
+gem 'faraday_middleware-aws-signers-v4', '>= 0.1.0', '< 0.1.2', require: false

data/README.md

@@ -16,38 +16,115 @@ In your fluentd configration, use `type aws-elasticsearch-service`.
 
 example:
 
-```rb
-source do
-  type :tail
-  format :apache
+```ruby
+<source>
+  type tail
+  format apache
   time_format "%d/%b/%Y:%T %z"
   path "/var/log/nginx/access.log"
   pos_file "/var/log/td-agent/nginx.access.pos"
   tag "es.nginx.access"
-end
+</source>
 
-match ("es.**") do
+<match es.**>
   type "aws-elasticsearch-service"
   type_name "access_log"
   logstash_format true
   include_tag_key true
   tag_key "@log_name"
-  flush_interval "10s"
-
-  endpoint do
-    url "YOUR_ENDPOINT_URL"
-    region "YOUR_ENDPOINT_REAGION"
-  end
-  # endpoint do
-  #   url "https://search-xxxxxxxx.ap-northeast-1.es.amazonaws.com"
-  #   region "ap-northeast-1"
-  #   access_key_id "yyyyyyyyy"      # optional
-  #   secret_access_key "zzzzzzzzz"  # optional
-  # end
-end
+  flush_interval 1s
+
+  <endpoint>
+    url https://CLUSTER_ENDPOINT_URL
+    region eu-west-1
+    # access_key_id "secret"
+    # secret_access_key "seekret"
+  </endpoint>
+</match>
+```
+
+## IAM
+If you do not wish to use credentials in your configuration via the `access_key_id` and `secret_access_key` options you should use IAM policies.
+
+The first step is to assign an IAM instance role `ROLE` to your EC2 instances. Name it appropriately. The role should contain no policy: we're using the possession of the role as the authenticating factor and placing the policy against the ES cluster.
+
+You should then configure a policy for the ES cluster policy thus, with appropriate substitutions for the capitalized terms:
+
+```json
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": {
+        "AWS": "arn:aws:iam::ACCOUNT:role/ROLE"
+      },
+      "Action": "es:*",
+      "Resource": "arn:aws:es:eu-west-1:ACCOUNT:domain/ES_DOMAIN/*"
+    },
+    {
+      "Effect": "Allow",
+      "Principal": {
+        "AWS": "*"
+      },
+      "Action": "es:*",
+      "Resource": "arn:aws:es:eu-west-1:ACCOUNT:domain/ES_DOMAIN/*",
+      "Condition": {
+        "IpAddress": {
+          "aws:SourceIp": [
+            "1.2.3.4/32",
+            "5.6.7.8/32"
+          ]
+        }
+      }
+    }
+  ]
+}
 ```
 
-If you use the "IAM User", please copy "access_key_id" and "secret_access_key" from a "IAM User".
+This will allow your fluentd hosts (by virtue of the possession of the role) and any traffic coming from the specified IP addresses (you querying Kibana) to access the various endpoints. Whilst not ideally secure (both the fluentd and Kibana boxes should ideally be restricted to the verbs they require) it should allow you to get up and ingesting logs without anything getting in your way, before you tighten down the policy.
+
+Additionally, you can use an STS assumed role as the authenticating factor and instruct the plugin to assume this role. This is useful for cross-account access and when assigning a standard role is not possible. The endpoint configuration looks like:
+
+```ruby
+ <endpoint>
+    url https://CLUSTER_ENDPOINT_URL
+    region eu-west-1
+    assume_role_arn arn:aws:sts::ACCOUNT:assumed-role/ROLE
+    assume_role_session_name SESSION_ID # Defaults to fluentd if omitted
+  </endpoint>
+```
+
+The policy attached to your AWS Elasticsearch cluster then becomes something like:
+
+```json
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": {
+        "AWS": "arn:aws:sts::ACCOUNT:assumed-role/ROLE/SESSION_ID"
+      },
+      "Action": "es:*",
+      "Resource": "arn:aws:es:eu-west-1:ACCOUNT:domain/ES_DOMAIN/*"
+    }
+  ]
+}
+```
+
+You'll need to ensure that the environment in which the fluentd plugin runs has the capability to assume this role, by attaching a policy something like this to the instance profile:
+
+```json
+{
+    "Version": "2012-10-17",
+    "Statement": {
+        "Effect": "Allow",
+        "Action": "sts:AssumeRole",
+        "Resource": "arn:aws:iam::ACCOUNT:role/ROLE"
+    }
+}
+```
 
 ## Development
 

data/fluent-plugin-aws-elasticsearch-service.gemspec

@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-aws-elasticsearch-service"
-  spec.version       = "0.1.4"
+  spec.version       = "0.1.6"
   spec.authors       = ["atomita"]
   spec.email         = ["sleeping.cait.sith+gh@gmail.com"]
 

data/lib/fluent/plugin/out_aws-elasticsearch-service.rb

@@ -16,6 +16,8 @@ module Fluent
       config_param :url, :string
       config_param :access_key_id, :string, :default => ""
       config_param :secret_access_key, :string, :default => ""
+      config_param :assume_role_arn, :string, :default => nil
+      config_param :assume_role_session_name, :string, :default => "fluentd"
     end
 
 
@@ -24,7 +26,7 @@ module Fluent
     #
     def get_connection_options
       raise "`endpoint` require." if @endpoint.empty?
-      
+
       hosts =
         begin
           @endpoint.map do |ep|
@@ -35,14 +37,14 @@ module Fluent
             end
 
             host[:aws_elasticsearch_service] = {
-              :credentials => credentials(ep[:access_key_id], ep[:secret_access_key]),
+              :credentials => credentials(ep),
               :region => ep[:region]
             }
-            
+
             host
           end
         end
-      
+
       {
         hosts: hosts
       }
@@ -54,14 +56,26 @@ module Fluent
     #
     # get AWS Credentials
     #
-    def credentials(access_key, secret_key)
+    def credentials(opts)
       calback = lambda do
         credentials = nil
-        if access_key.empty? or secret_key.empty?
-          credentials   = Aws::InstanceProfileCredentials.new.credentials
-          credentials ||= Aws::SharedCredentials.new.credentials
+        unless opts[:access_key_id].empty? or opts[:secret_access_key].empty?
+          credentials = Aws::Credentials.new opts[:access_key_id], opts[:secret_access_key]
+        else
+          if opts[:assume_role_arn].nil?
+            credentials = Aws::SharedCredentials.new({
+                             retries: 2
+                          }).credentials
+            credentials ||= Aws::InstanceProfileCredentials.new.credentials
+          else
+            credentials = sts_credential_provider({
+                            role_arn: opts[:assume_role_arn],
+                            role_session_name: opts[:assume_role_session_name],
+                            region: opts[:region]
+                          }).credentials
+          end
         end
-        credentials ||= Aws::Credentials.new access_key, secret_key
+        raise "No valid AWS credentials found." unless credentials.set?
         credentials
       end
       def calback.inspect
@@ -71,6 +85,11 @@ module Fluent
       calback
     end
 
+    def sts_credential_provider(opts)
+      # AssumeRoleCredentials is an auto-refreshing credential provider
+      @sts ||= Aws::AssumeRoleCredentials.new(opts)
+    end
+
   end
 
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-aws-elasticsearch-service
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.6
 platform: ruby
 authors:
 - atomita
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-11-20 00:00:00.000000000 Z
+date: 2016-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler