logstash-output-amazon_es 0.1.0

data/lib/logstash/outputs/amazon_es/aws_transport.rb

@@ -0,0 +1,103 @@
+require 'faraday'
+require 'faraday_middleware'
+require 'aws-sdk-core'
+require 'elasticsearch'
+require 'elasticsearch-transport'
+require 'manticore'
+require 'faraday/adapter/manticore'
+
+#
+require 'uri'
+
+require_relative "aws_v4_signer"
+
+
+module Elasticsearch
+  module Transport
+    module Transport
+      module HTTP
+
+        # Transport implementation, which V4 Signs requests using the [_Faraday_](https://rubygems.org/gems/faraday)
+        # library for abstracting the HTTP client.
+        #
+        # @see Transport::Base
+        #
+        class AWS
+          include Elasticsearch::Transport::Transport::Base
+          
+
+          DEFAULT_PORT = 80
+          DEFAULT_PROTOCOL = "http"    
+          
+          CredentialConfig = Struct.new(
+            :access_key_id,
+            :secret_access_key,
+            :session_token,
+            :profile
+          )      
+
+          # Performs the request by invoking {Transport::Base#perform_request} with a block.
+          #
+          # @return [Response]
+          # @see    Transport::Base#perform_request
+          #
+          def perform_request(method, path, params={}, body=nil)
+            super do |connection, url|
+              response = connection.connection.run_request \
+               method.downcase.to_sym,
+                url,
+                ( body ? __convert_to_json(body) : nil ),
+                {}
+
+              Response.new response.status, response.body, response.headers
+            end
+          end
+
+          # Builds and returns a collection of connections.
+          #
+          # @return [Connections::Collection]
+          #
+          def __build_connections
+            region = options[:region]
+            access_key_id = options[:aws_access_key_id] || nil
+            secret_access_key = options[:aws_secret_access_key] || nil
+            session_token = options[:session_token] || nil
+            profile = options[:profile] || 'default'
+            
+            credential_config = CredentialConfig.new(access_key_id, secret_access_key, session_token, profile)
+            credentials = Aws::CredentialProviderChain.new(credential_config).resolve
+
+            Connections::Collection.new \
+              :connections => hosts.map { |host|
+                host[:protocol]   = host[:scheme] || DEFAULT_PROTOCOL
+                host[:port]     ||= DEFAULT_PORT
+                url               = __full_url(host)
+                                  
+                aes_connection = ::Faraday::Connection.new(url,  (options[:transport_options] || {})) do |faraday|
+                  faraday.request :aws_v4_signer,
+                                        credentials: credentials,
+                                        service_name: 'es',
+                                        region: region
+                  faraday.adapter :manticore
+                end
+                
+                Connections::Connection.new \
+                  :host => host,
+                  :connection => aes_connection
+              },
+              :selector_class => options[:selector_class],
+              :selector => options[:selector]
+          end
+
+          # Returns an array of implementation specific connection errors.
+          #
+          # @return [Array]
+          #
+          def host_unreachable_exceptions
+            [::Faraday::Error::ConnectionFailed, ::Faraday::Error::TimeoutError]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/logstash/outputs/amazon_es/aws_v4_signer.rb

@@ -0,0 +1,7 @@
+require 'faraday'
+require_relative 'aws_v4_signer_impl'
+
+module FaradayMiddleware
+  Faraday::Request.register_middleware :aws_v4_signer => lambda { AwsV4Signer }
+end
+

data/lib/logstash/outputs/amazon_es/aws_v4_signer_impl.rb

@@ -0,0 +1,58 @@
+require 'aws-sdk-core'
+require 'faraday'
+
+
+class AwsV4Signer < Faraday::Middleware
+  class Request
+    def initialize(env)
+      @env = env
+    end
+    
+    def headers
+      @env.request_headers
+    end
+    
+    def set_header(header)
+      @env.request_headers = header
+    end
+
+    def body
+      @env.body || ''
+    end
+
+    def endpoint
+      @env.url
+    end
+
+    def http_method
+      @env.method.to_s.upcase
+    end
+  end
+
+  def initialize(app, options = nil)
+    super(app)
+    credentials = options.fetch(:credentials)
+    service_name = options.fetch(:service_name)
+    region = options.fetch(:region)
+    @signer = Aws::Signers::V4.new(credentials, service_name, region)
+    @net_http = app.is_a?(Faraday::Adapter::NetHttp)
+  end
+
+  def call(env)
+    normalize_for_net_http!(env)
+    req = Request.new(env)
+    @signer.sign(req)
+    @app.call(env)
+  end
+
+  private
+  def normalize_for_net_http!(env)
+    return unless @net_http
+
+    if Net::HTTP::HAVE_ZLIB
+      env.request_headers['Accept-Encoding'] ||= 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3'
+    end
+
+    env.request_headers['Accept'] ||= '*/*'
+  end
+end

data/lib/logstash/outputs/amazon_es/elasticsearch-template.json

@@ -0,0 +1,41 @@
+{
+  "template" : "logstash-*",
+  "settings" : {
+    "index.refresh_interval" : "5s"
+  },
+  "mappings" : {
+    "_default_" : {
+       "_all" : {"enabled" : true, "omit_norms" : true},
+       "dynamic_templates" : [ {
+         "message_field" : {
+           "match" : "message",
+           "match_mapping_type" : "string",
+           "mapping" : {
+             "type" : "string", "index" : "analyzed", "omit_norms" : true
+           }
+         }
+       }, {
+         "string_fields" : {
+           "match" : "*",
+           "match_mapping_type" : "string",
+           "mapping" : {
+             "type" : "string", "index" : "analyzed", "omit_norms" : true,
+               "fields" : {
+                 "raw" : {"type": "string", "index" : "not_analyzed", "ignore_above" : 256}
+               }
+           }
+         }
+       } ],
+       "properties" : {
+         "@version": { "type": "string", "index": "not_analyzed" },
+         "geoip"  : {
+           "type" : "object",
+             "dynamic": true,
+             "properties" : {
+               "location" : { "type" : "geo_point" }
+             }
+         }
+       }
+    }
+  }
+}

data/lib/logstash/outputs/amazon_es/http_client.rb

@@ -0,0 +1,117 @@
+require "logstash/outputs/amazon_es"
+require "cabin"
+require "base64"
+require "elasticsearch"
+
+require_relative "aws_transport"
+
+module LogStash::Outputs::AES
+  class HttpClient
+    attr_reader :client, :options, :client_options
+    DEFAULT_OPTIONS = {
+      :port => 80
+    }
+
+    def initialize(options={})
+      @logger = Cabin::Channel.get
+      @options = DEFAULT_OPTIONS.merge(options)
+      @client = build_client(@options)
+    end
+
+    def template_install(name, template, force=false)
+      if template_exists?(name) && !force
+        @logger.debug("Found existing Elasticsearch template. Skipping template management", :name => name)
+        return
+      end
+      template_put(name, template)
+    end
+
+    def bulk(actions)
+      bulk_body = actions.collect do |action, args, source|
+        if action == 'update'
+          if args[:_id]
+            source = { 'doc' => source }
+            if @options[:doc_as_upsert]
+              source['doc_as_upsert'] = true
+            else
+              source['upsert'] = args[:_upsert] if args[:_upsert]
+            end
+          else
+            raise(LogStash::ConfigurationError, "Specifying action => 'update' without a document '_id' is not supported.")
+          end
+        end
+
+        args.delete(:_upsert)
+
+        if source
+          next [ { action => args }, source ]
+        else
+          next { action => args }
+        end
+      end.flatten
+
+      bulk_response = @client.bulk(:body => bulk_body)
+
+      self.class.normalize_bulk_response(bulk_response)
+    end
+
+    private
+    def build_client(options)
+      hosts = options[:hosts]
+      port = options[:port]
+      client_settings = options[:client_settings] || {}
+
+      uris = hosts.map do |host|
+        "http://#{host}:#{port}#{client_settings[:path]}".gsub(/[\/]+$/,'')
+      end
+
+      @client_options = {
+        :hosts => uris,
+        :region => options[:region],
+        :aws_access_key_id => options[:aws_access_key_id],
+        :aws_secret_access_key => options[:aws_secret_access_key],
+        :transport_options => {
+          :request => {:open_timeout => 0, :timeout => 60},  # ELB timeouts are set at 60
+          :proxy => client_settings[:proxy],          
+        },
+        :transport_class => Elasticsearch::Transport::Transport::HTTP::AWS
+      }
+
+      if options[:user] && options[:password] then
+        token = Base64.strict_encode64(options[:user] + ":" + options[:password])
+        @client_options[:headers] = { "Authorization" => "Basic #{token}" }
+      end
+
+      Elasticsearch::Client.new(client_options)
+    end
+
+    def self.normalize_bulk_response(bulk_response)
+      if bulk_response["errors"]
+        # The structure of the response from the REST Bulk API is follows:
+        # {"took"=>74, "errors"=>true, "items"=>[{"create"=>{"_index"=>"logstash-2014.11.17",
+        #                                                    "_type"=>"logs",
+        #                                                    "_id"=>"AUxTS2C55Jrgi-hC6rQF",
+        #                                                    "_version"=>1,
+        #                                                    "status"=>400,
+        #                                                    "error"=>"MapperParsingException[failed to parse]..."}}]}
+        # where each `item` is a hash of {OPTYPE => Hash[]}. calling first, will retrieve
+        # this hash as a single array with two elements, where the value is the second element (i.first[1])
+        # then the status of that item is retrieved.
+        {"errors" => true, "statuses" => bulk_response["items"].map { |i| i.first[1]['status'] }}
+      else
+        {"errors" => false}
+      end
+    end
+
+    def template_exists?(name)
+      @client.indices.get_template(:name => name)
+      return true
+    rescue Elasticsearch::Transport::Transport::Errors::NotFound
+      return false
+    end
+
+    def template_put(name, template)
+      @client.indices.put_template(:name => name, :body => template)
+    end
+  end
+end

data/logstash-output-amazon_es.gemspec

@@ -0,0 +1,42 @@
+Gem::Specification.new do |s|
+
+  s.name            = 'logstash-output-amazon_es'
+  s.version         = '0.1.0'
+  s.licenses        = ['apache-2.0']
+  s.summary         = "Logstash Output to Amazon Elasticsearch Service"
+  s.description     = "Output events to Amazon Elasticsearch Service with V4 signing"
+  s.authors         = ["Amazon"]
+  s.email           = 'feedback-prod-elasticsearch@amazon.com'
+  s.homepage        = "http://logstash.net/"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
+
+  # Gem dependencies
+  s.add_runtime_dependency 'concurrent-ruby'
+  s.add_runtime_dependency 'elasticsearch', ['>= 1.0.10', '~> 1.0']
+  s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']
+  s.add_runtime_dependency 'cabin', ['~> 0.6']
+  s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0'
+  s.add_runtime_dependency "aws-sdk", ['>= 2.1.14', '~> 2.1']
+  s.add_runtime_dependency "faraday", '~> 0.9.1'
+  s.add_runtime_dependency "faraday_middleware", '~> 0.10.0'
+
+  s.add_development_dependency 'ftw', '~> 0.0.42'
+  s.add_development_dependency 'logstash-input-generator'
+
+  if RUBY_PLATFORM == 'java'
+    s.platform = RUBY_PLATFORM
+    s.add_runtime_dependency "manticore", '~> 0.4.2'
+  end
+
+  s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'longshoreman'
+end

data/spec/amazon_es_spec_helper.rb

@@ -0,0 +1,69 @@
+require "logstash/devutils/rspec/spec_helper"
+require "ftw"
+require "logstash/plugin"
+require "logstash/json"
+require "stud/try"
+require "longshoreman"
+
+CONTAINER_NAME = "logstash-output-amazon-es-#{rand(999).to_s}"
+CONTAINER_IMAGE = "elasticsearch"
+CONTAINER_TAG = "1.6"
+
+DOCKER_INTEGRATION = ENV["DOCKER_INTEGRATION"]
+
+module ESHelper
+  def get_host
+    DOCKER_INTEGRATION ? Longshoreman.new.get_host_ip : "127.0.0.1"
+  end
+
+  def get_port
+    return 9200 unless DOCKER_INTEGRATION
+
+    container = Longshoreman::Container.new
+    container.get(CONTAINER_NAME)
+    container.rport(9200)
+  end
+
+  def get_client
+    Elasticsearch::Client.new(:host => "#{get_host}:#{get_port}")
+  end
+end
+
+
+RSpec.configure do |config|
+  config.include ESHelper
+
+
+  if DOCKER_INTEGRATION
+    # this :all hook gets run before every describe block that is tagged with :integration => true.
+    config.before(:all, :integration => true) do
+
+
+      # check if container exists already before creating new one.
+      begin
+        ls = Longshoreman::new
+        ls.container.get(CONTAINER_NAME)
+      rescue Docker::Error::NotFoundError
+        Longshoreman.new("#{CONTAINER_IMAGE}:#{CONTAINER_TAG}", CONTAINER_NAME)
+        # TODO(talevy): verify ES is running instead of static timeout
+        sleep 10
+      end
+    end
+
+    # we want to do a final cleanup after all :integration runs,
+    # but we don't want to clean up before the last block.
+    # This is a final blind check to see if the ES docker container is running and
+    # needs to be cleaned up. If no container can be found and/or docker is not
+    # running on the system, we do nothing.
+    config.after(:suite) do
+      # only cleanup docker container if system has docker and the container is running
+      begin
+        ls = Longshoreman::new
+        ls.container.get(CONTAINER_NAME)
+        ls.cleanup
+      rescue Docker::Error::NotFoundError, Excon::Errors::SocketError
+        # do nothing
+      end
+    end
+  end
+end

data/spec/unit/outputs/amazon_es_spec.rb

@@ -0,0 +1,50 @@
+require_relative "../../../spec/amazon_es_spec_helper"
+
+describe "outputs/amazon_es" do
+  describe "http client create" do
+    require "logstash/outputs/amazon_es"
+    require "elasticsearch"
+
+    let(:options) {
+      {
+        "index" => "my-index",
+        "hosts" => "localhost",
+        "path" => "some-path"
+      }
+    }
+
+    let(:eso) {LogStash::Outputs::AmazonES.new(options)}
+
+    let(:manticore_host) {
+      eso.client.send(:client).transport.options[:hosts].first
+    }
+
+    around(:each) do |block|
+      thread = eso.register
+      block.call()
+      thread.kill()
+    end
+
+    describe "with path" do
+      it "should properly create a URI with the path" do
+        expect(eso.path).to eql(options["path"])
+      end
+
+
+      it "should properly set the path on the HTTP client" do
+        expect(manticore_host).to include("/" + options["path"])
+      end
+
+      context "with extra slashes" do
+        let(:path) { "/slashed-path/ "}
+        let(:eso) {
+          LogStash::Outputs::AmazonES.new(options.merge("path" => "/some-path/"))
+        }
+
+        it "should properly set the path on the HTTP client without adding slashes" do
+          expect(manticore_host).to include(options["path"])
+        end
+      end
+    end
+  end
+end