fluent-plugin-pearson-aws-elasticsearch-service 1.2.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: efe09c9dd697cef05228641478c5192e654dd3097862f3a7ac4e296935753ae7
+  data.tar.gz: c10f4f1034f97d86dff26b0b4be6cd4255ebfec5cd10bc52193d2e5d66567149
+SHA512:
+  metadata.gz: 8acb5041bd470c6c5dd7379a56b3eb1e5afe865bbba5c0441d188e0f990864bcc157eb70dd23635c7e8b90e0a391cd01bc5347bc528e146da268f58be528d89a
+  data.tar.gz: 46762972477e21cd47fedadfb283d7db80cb88935d4aa34468049583fd4cdfccbef5636708e2b6f3a469f658caa896c3a0f85bf9a512b795e16558c743a2dca8

data/.gitignore

@@ -0,0 +1,35 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalisation:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/Gemfile

@@ -0,0 +1,8 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-aws-elasticsearch-service.gemspec
+gemspec
+
+gem 'fluent-plugin-elasticsearch', "~> 2.11.0", require: false
+gem 'aws-sdk-core', '~> 3', require: false
+gem 'faraday_middleware-aws-sigv4', '>= 0.2.4', '< 0.3.0', require: false

data/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 @
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

data/README.md

@@ -0,0 +1,151 @@
+# Fluent::AwsElasticsearchServiceOutput
+
+This output plugin to post to "Amazon Elasticsearch Service".
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'fluent-plugin-aws-elasticsearch-service'
+```
+
+## Usage
+
+In your fluentd configration, use `type aws-elasticsearch-service`.
+
+example:
+
+```ruby
+<source>
+  type tail
+  format apache
+  time_format "%d/%b/%Y:%T %z"
+  path "/var/log/nginx/access.log"
+  pos_file "/var/log/td-agent/nginx.access.pos"
+  tag "es.nginx.access"
+</source>
+
+<match es.**>
+  type "aws-elasticsearch-service"
+  type_name "access_log"
+  logstash_format true
+  include_tag_key true
+  tag_key "@log_name"
+  flush_interval 1s
+
+  <endpoint>
+    url https://CLUSTER_ENDPOINT_URL
+    region eu-west-1
+    # access_key_id "secret"
+    # secret_access_key "seekret"
+  </endpoint>
+</match>
+```
+
+## IAM
+If you do not wish to use credentials in your configuration via the `access_key_id` and `secret_access_key` options you should use IAM policies.
+
+The first step is to assign an IAM instance role `ROLE` to your EC2 instances. Name it appropriately. The role should contain no policy: we're using the possession of the role as the authenticating factor and placing the policy against the ES cluster.
+
+You should then configure a policy for the ES cluster policy thus, with appropriate substitutions for the capitalized terms:
+
+```json
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": {
+        "AWS": "arn:aws:iam::ACCOUNT:role/ROLE"
+      },
+      "Action": "es:*",
+      "Resource": "arn:aws:es:eu-west-1:ACCOUNT:domain/ES_DOMAIN/*"
+    },
+    {
+      "Effect": "Allow",
+      "Principal": {
+        "AWS": "*"
+      },
+      "Action": "es:*",
+      "Resource": "arn:aws:es:eu-west-1:ACCOUNT:domain/ES_DOMAIN/*",
+      "Condition": {
+        "IpAddress": {
+          "aws:SourceIp": [
+            "1.2.3.4/32",
+            "5.6.7.8/32"
+          ]
+        }
+      }
+    }
+  ]
+}
+```
+
+This will allow your fluentd hosts (by virtue of the possession of the role) and any traffic coming from the specified IP addresses (you querying Kibana) to access the various endpoints. Whilst not ideally secure (both the fluentd and Kibana boxes should ideally be restricted to the verbs they require) it should allow you to get up and ingesting logs without anything getting in your way, before you tighten down the policy.
+
+Additionally, you can use an STS assumed role as the authenticating factor and instruct the plugin to assume this role. This is useful for cross-account access and when assigning a standard role is not possible. The endpoint configuration looks like:
+
+```ruby
+ <endpoint>
+    url https://CLUSTER_ENDPOINT_URL
+    region eu-west-1
+    assume_role_arn arn:aws:sts::ACCOUNT:role/ROLE
+    assume_role_session_name SESSION_ID # Defaults to fluentd if omitted
+  </endpoint>
+```
+
+The policy attached to your AWS Elasticsearch cluster then becomes something like:
+
+```json
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": {
+        "AWS": "arn:aws:sts::ACCOUNT:assumed-role/ROLE/SESSION_ID"
+      },
+      "Action": "es:*",
+      "Resource": "arn:aws:es:eu-west-1:ACCOUNT:domain/ES_DOMAIN/*"
+    }
+  ]
+}
+```
+
+You'll need to ensure that the environment in which the fluentd plugin runs has the capability to assume this role, by attaching a policy something like this to the instance profile:
+
+```json
+{
+    "Version": "2012-10-17",
+    "Statement": {
+        "Effect": "Allow",
+        "Action": "sts:AssumeRole",
+        "Resource": "arn:aws:iam::ACCOUNT:role/ROLE"
+    }
+}
+```
+
+## Troubleshooting
+
+* "Elasticsearch::Transport::Transport::Errors::Forbidden" error="[403]" even after verifying the access keys/roles/policies.
+   * Ensure you don't have a trailing slash on the endpoint URL in your fluentd configuration file (see CLUSTER_ENDPOINT_URL above).
+
+*  "ElasticsearchIllegalArgumentException[explicit index in bulk is not allowed]"
+   * Check that `rest.action.multi.allow_explicit` is set true on your Amazon ES domain (verify in the console - there's a bug in Terraform, https://github.com/hashicorp/terraform/issues/3980).
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/atomita/fluent-plugin-aws-elasticsearch-service. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](contributor-covenant.org) code of conduct.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-pearson-aws-elasticsearch-service.gemspec

@@ -0,0 +1,31 @@
+# -*- encoding: utf-8 -*-
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-pearson-aws-elasticsearch-service"
+  spec.version       = "1.2.2"
+  spec.authors       = ["gamunu"]
+  spec.email         = ["gamunu.balagalla@gmail.com"]
+
+  spec.summary       = %q{Output plugin to post to "Amazon Elasticsearch Service".}
+  spec.description   = %q{this is a Output plugin. Post to "Amazon Elasticsearch Service".}
+  spec.homepage      = "https://github.com/pearsontechnology/fluent-plugin-aws-elasticsearch-service"
+  spec.license       = "MIT"
+
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 2.0"
+  spec.add_development_dependency "rake", "~> 12.3.0"
+  spec.add_development_dependency "rspec", "~> 3.8"
+  spec.add_development_dependency "test-unit", "~> 3.0"
+  spec.add_runtime_dependency "fluentd", ">= 0.14"
+  spec.add_runtime_dependency "fluent-plugin-elasticsearch", "~> 2.11.0"
+  spec.add_runtime_dependency "aws-sdk-core", "~> 3"
+  spec.add_runtime_dependency "faraday_middleware-aws-sigv4", ">= 0.2.4", "< 0.3.0"
+end

data/lib/fluent/plugin/out_aws-elasticsearch-service.rb

@@ -0,0 +1,208 @@
+# -*- encoding: utf-8 -*-
+
+require 'rubygems'
+require 'fluent/plugin/out_elasticsearch'
+require 'aws-sdk-core'
+require 'faraday_middleware/aws_sigv4'
+
+
+module Fluent::Plugin
+  class AwsElasticsearchServiceOutput < ElasticsearchOutput
+
+    Fluent::Plugin.register_output('aws-elasticsearch-service', self)
+
+    config_section :endpoint do
+      config_param :region, :string
+      config_param :url, :string
+      config_param :access_key_id, :string, :default => ""
+      config_param :secret_access_key, :string, :default => ""
+      config_param :assume_role_arn, :string, :default => nil
+      config_param :ecs_container_credentials_relative_uri, :string, :default => nil #Set with AWS_CONTAINER_CREDENTIALS_RELATIVE_URI environment variable value
+      config_param :assume_role_session_name, :string, :default => "fluentd"
+    end
+
+    # here overrides default value of reload_connections to false because
+    # AWS Elasticsearch Service doesn't return addresses of nodes and Elasticsearch client
+    # fails to reload connections properly. This ends up "temporarily failed to flush the buffer"
+    # error repeating forever. See this discussion for details:
+    # https://discuss.elastic.co/t/elasitcsearch-ruby-raises-cannot-get-new-connection-from-pool-error/36252
+    config_set_default :reload_connections, false
+
+    #
+    # @override
+    #
+    def get_connection_options
+      raise "`endpoint` require." if @endpoint.empty?
+
+      hosts =
+        begin
+          @endpoint.map do |ep|
+            uri = URI(ep[:url])
+            host = %w(user password path).inject(host: uri.host, port: uri.port, scheme: uri.scheme) do |hash, key|
+              hash[key.to_sym] = uri.public_send(key) unless uri.public_send(key).nil? || uri.public_send(key) == ''
+              hash
+            end
+
+            host[:aws_elasticsearch_service] = {
+              :credentials => credentials(ep),
+              :region => ep[:region]
+            }
+
+            host
+          end
+        end
+
+      {
+        hosts: hosts
+      }
+    end
+
+    def write(chunk)
+      super
+    end
+
+
+    private
+
+    #
+    # get AWS Credentials
+    #
+    def credentials(opts)
+      calback = lambda do
+        credentials = nil
+        unless opts[:access_key_id].empty? or opts[:secret_access_key].empty?
+          credentials = Aws::Credentials.new opts[:access_key_id], opts[:secret_access_key]
+        else
+          if opts[:assume_role_arn].nil?
+            if opts[:ecs_container_credentials_relative_uri].nil?
+              credentials = Aws::SharedCredentials.new({retries: 2}).credentials
+              credentials ||= Aws::InstanceProfileCredentials.new.credentials
+              credentials ||= Aws::ECSCredentials.new.credentials
+            else
+              credentials = Aws::ECSCredentials.new({
+                credential_path: opts[:ecs_container_credentials_relative_uri]
+              }).credentials
+            end
+          else
+            credentials = sts_credential_provider({
+                            role_arn: opts[:assume_role_arn],
+                            role_session_name: opts[:assume_role_session_name],
+                            region: opts[:region]
+                          }).credentials
+          end
+        end
+        raise "No valid AWS credentials found." unless credentials.set?
+        credentials
+      end
+      def calback.inspect
+        credentials = self.call
+        credentials.credentials.inspect
+      end
+      calback
+    end
+
+    def sts_credential_provider(opts)
+      # AssumeRoleCredentials is an auto-refreshing credential provider
+      @sts ||= Aws::AssumeRoleCredentials.new(opts)
+    end
+
+  end
+
+
+  #
+  # monkey patch
+  #
+  class ElasticsearchOutput
+    module Elasticsearch
+
+      module Client
+        include ::Elasticsearch::Client
+        extend self
+      end
+
+      module Transport
+        module Transport
+          module HTTP
+            class Faraday < ::Elasticsearch::Transport::Transport::HTTP::Faraday
+
+              alias :__build_connections_origin_from_aws_elasticsearch_service_output :__build_connections
+
+              # Builds and returns a collection of connections.
+              #
+              # @return [Connections::Collection]
+              # @override
+              #
+              def __build_connections
+                ::Elasticsearch::Transport::Transport::Connections::Collection.new(
+                  :connections => hosts.map { |host|
+                    host[:protocol]   = host[:scheme] || DEFAULT_PROTOCOL
+                    host[:port]     ||= DEFAULT_PORT
+                    url               = __full_url(host)
+
+                    ::Elasticsearch::Transport::Transport::Connections::Connection.new(
+                      :host => host,
+                      :connection => ::Faraday::Connection.new(
+                        url,
+                        (options[:transport_options] || {}),
+                        &__aws_elasticsearch_service_setting(host, &@block)
+                      ),
+                      :options => host[:connection_options]
+                    )
+                  },
+                  :selector_class => options[:selector_class],
+                  :selector => options[:selector]
+                )
+              end
+
+              def __aws_elasticsearch_service_setting(host, &block)
+                lambda do |faraday|
+                  if host[:aws_elasticsearch_service]
+                    faraday.request :aws_sigv4,
+                                    credentials: host[:aws_elasticsearch_service][:credentials],
+                                    service_name: 'es',
+                                    region: host[:aws_elasticsearch_service][:region]
+                  end
+                  block.call faraday
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+
+  end
+end
+
+
+#
+# monkey patch
+#
+class FaradayMiddleware::AwsSigV4
+
+  alias :initialize_origin_from_aws_elasticsearch_service_output :initialize
+
+  def initialize(app, options = nil)
+    super(app)
+
+    credentials = options.fetch(:credentials)
+    service_name = options.fetch(:service_name)
+    region = options.fetch(:region)
+    @signer =
+      begin
+        if credentials.is_a?(Proc)
+          signer = lambda do
+            Aws::Sigv4::Signer.new(service: service_name, region: region, credentials: credentials.call)
+          end
+          def signer.sign_request(req)
+            self.call.sign_request(req)
+          end
+          signer
+        else
+          Aws::Sigv4::Signer.new(service: service_name, region: region, credentials: credentials)
+        end
+      end
+
+    @net_http = app.is_a?(Faraday::Adapter::NetHttp)
+  end
+end

metadata

@@ -0,0 +1,168 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-pearson-aws-elasticsearch-service
+version: !ruby/object:Gem::Version
+  version: 1.2.2
+platform: ruby
+authors:
+- gamunu
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2019-01-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 12.3.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 12.3.0
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.8'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.14'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.14'
+- !ruby/object:Gem::Dependency
+  name: fluent-plugin-elasticsearch
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.11.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.11.0
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  name: faraday_middleware-aws-sigv4
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.4
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.4
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+description: this is a Output plugin. Post to "Amazon Elasticsearch Service".
+email:
+- gamunu.balagalla@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- fluent-plugin-pearson-aws-elasticsearch-service.gemspec
+- lib/fluent/plugin/out_aws-elasticsearch-service.rb
+homepage: https://github.com/pearsontechnology/fluent-plugin-aws-elasticsearch-service
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.1
+signing_key: 
+specification_version: 4
+summary: Output plugin to post to "Amazon Elasticsearch Service".
+test_files: []