fluent-plugin-aws-elasticsearch-service 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7e67fc954f730743d5a89c67f29a060e92764498
+  data.tar.gz: 7beb32b73861d1ca4614ae303026ad311c6baddf
+SHA512:
+  metadata.gz: bfbb04a89975ba7c06b515a2b1b9d4357eac2ab6c4512d7d44af0d7cda00b8680d3d346c29a4414cbe3f9b948609940b99b24cf0369c5718d19e58f5fe3cf3ee
+  data.tar.gz: 070a07ea3bb3589fee8d8b5f359e53774557ac5849e869b9454dbcb418c976eee54cce1721a92ac79d2eb4ca7a47d90850bdc11c9ac7811ab76538205f550c33

data/.gitignore

@@ -0,0 +1,35 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalisation:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/Gemfile

@@ -0,0 +1,8 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-aws-elasticsearch-service.gemspec
+gemspec
+
+gem 'fluent-plugin-elasticsearch', '~> 1.0', require: false
+gem 'aws-sdk', '~> 2', require: false
+gem 'faraday_middleware-aws-signers-v4', require: false

data/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 @
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

data/README.md

@@ -0,0 +1,58 @@
+# Fluent::AwsElasticsearchServiceOutput
+
+This output plugin to post to "Amazon Elasticsearch Service".
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'fluent-plugin-aws-elasticsearch-service'
+```
+
+## Usage
+
+In your fluentd configration, use `type aws-elasticsearch-service`.
+
+example:
+
+```rb
+source do
+  type :tail
+  format :apache
+  time_format "%d/%b/%Y:%T %z"
+  path "/var/log/nginx/access.log"
+  pos_file "/var/log/td-agent/nginx.access.pos"
+  tag "es.nginx.access"
+end
+
+match ("es.**") do
+  type "aws-elasticsearch-service"
+  type_name "access_log"
+  logstash_format true
+  include_tag_key true
+  tag_key "@log_name"
+  flush_interval "10s"
+
+  endpoint do
+    url "YOUR_ENDPOINT_URL"
+    region "ap-northeast-1"
+  end
+end
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/atomita/fluent-plugin-aws-elasticsearch-service. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](contributor-covenant.org) code of conduct.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-aws-elasticsearch-service.gemspec

@@ -0,0 +1,30 @@
+# -*- encoding: utf-8 -*-
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-aws-elasticsearch-service"
+  spec.version       = "0.1.0"
+  spec.authors       = ["atomita"]
+  spec.email         = ["sleeping.cait.sith+gh@gmail.com"]
+
+  spec.summary       = %q{Output plugin to post to "Amazon Elasticsearch Service".}
+  spec.description   = %q{this is a Output plugin. Post to "Amazon Elasticsearch Service".}
+  spec.homepage      = "https://github.com/atomita/fluent-plugin-aws-elasticsearch-service"
+  spec.license       = "MIT"
+
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_runtime_dependency "fluentd", "~> 0"
+  spec.add_runtime_dependency "fluent-plugin-elasticsearch", "~> 1.0"
+  spec.add_runtime_dependency "aws-sdk", "~> 2"
+  spec.add_runtime_dependency "faraday_middleware-aws-signers-v4", "~> 0"
+end

data/lib/fluent/plugin/out_aws-elasticsearch-service.rb

@@ -0,0 +1,122 @@
+# -*- encoding: utf-8 -*-
+
+require 'rubygems'
+require 'fluent/plugin/out_elasticsearch'
+require 'aws-sdk'
+require 'faraday_middleware'
+require 'faraday_middleware/aws_signers_v4'
+
+
+module Fluent
+  class AwsElasticsearchServiceOutput < ElasticsearchOutput
+
+    Plugin.register_output('aws-elasticsearch-service', self)
+
+    config_section :endpoint do
+      config_param :region, :string
+      config_param :url, :string
+      config_param :access_key_id, :string, :default => ""
+      config_param :secret_access_key, :string, :default => ""
+    end
+
+
+    #
+    # @override
+    #
+    def get_connection_options
+      raise "`endpoint` require." if @endpoint.empty?
+      
+      hosts =
+        begin
+          @endpoint.map do |ep|
+            uri = URI(ep[:url])
+            host = %w(user password path).inject(host: uri.host, port: uri.port, scheme: uri.scheme) do |hash, key|
+              hash[key.to_sym] = uri.public_send(key) unless uri.public_send(key).nil? || uri.public_send(key) == ''
+              hash
+            end
+
+            host[:aws_elasticsearch_service] = {
+              :credentials => credentials(ep[:access_key_id], ep[:secret_access_key]),
+              :region => ep[:region]
+            }
+            
+            host
+          end
+        end
+      
+      {
+        hosts: hosts
+      }
+    end
+
+
+    private
+
+    #
+    # get AWS Credentials
+    #
+    def credentials(access_key, secret_key)
+      @credentials ||= {}
+      @credentials[access_key] ||= {}
+      @credentials[access_key][secret_key] ||=
+        begin
+          credentials = nil
+          if access_key.empty? or secret_key.empty?
+            credentials   = Aws::InstanceProfileCredentials.new.credentials
+            credentials ||= Aws::SharedCredentials.new.credentials
+          end
+          credentials ||= Aws::Credentials.new access_key, secret_key
+          credentials
+        end
+    end
+
+
+    #
+    # monkey patch
+    #
+    class ElasticsearchOutput::Elasticsearch::Transport::Transport::HTTP::Faraday
+
+      alias :__build_connections_origin_from_aws_elasticsearch_service_output :__build_connections
+
+      # Builds and returns a collection of connections.
+      #
+      # @return [Connections::Collection]
+      # @override
+      #
+      def __build_connections
+        Elasticsearch::Transport::Transport::Connections::Collection.new(
+          :connections => hosts.map { |host|
+            host[:protocol]   = host[:scheme] || DEFAULT_PROTOCOL
+            host[:port]     ||= DEFAULT_PORT
+            url               = __full_url(host)
+
+            Elasticsearch::Transport::Transport::Connections::Connection.new(
+              :host => host,
+              :connection => ::Faraday::Connection.new(
+                url,
+                (options[:transport_options] || {}),
+                &__aws_elasticsearch_service_setting(host, &@block)
+              ),
+              :options => host[:connection_options]
+            )
+          },
+          :selector_class => options[:selector_class],
+          :selector => options[:selector]
+        )
+      end
+
+      def __aws_elasticsearch_service_setting(host, &block)
+        lambda do |faraday|
+          if host[:aws_elasticsearch_service]
+            faraday.request :aws_signers_v4,
+                            credentials: host[:aws_elasticsearch_service][:credentials],
+                            service_name: 'es',
+                            region: host[:aws_elasticsearch_service][:region]
+          end
+          block.call faraday
+        end
+      end
+    end
+
+  end
+end

metadata

@@ -0,0 +1,149 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-aws-elasticsearch-service
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- atomita
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2015-10-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluent-plugin-elasticsearch
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: faraday_middleware-aws-signers-v4
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+description: this is a Output plugin. Post to "Amazon Elasticsearch Service".
+email:
+- sleeping.cait.sith+gh@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- fluent-plugin-aws-elasticsearch-service.gemspec
+- lib/fluent/plugin/out_aws-elasticsearch-service.rb
+homepage: https://github.com/atomita/fluent-plugin-aws-elasticsearch-service
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Output plugin to post to "Amazon Elasticsearch Service".
+test_files: []