logstash-output-elasticsearch 0.1.0 → 0.1.1

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    M2UwZjRiN2E5NDZkZDU0NmUyYjMwNDk4ZWFjMGUxYmY1Y2IwZjMzYQ==
+    YTQ0NzIzNzc2MmVkMTYxM2Y5OTBhNTExZjQ4Nzc5ZGJlNDU5MDQwYg==
   data.tar.gz: !binary |-
-    MWMxZmIyNjg2NTY1MzkwNDc0ZWE2OTg5MmQ1MWQ3M2QwZDdjM2RkMQ==
+    NGQxYTgwYzRlMjhkMDBkYWFjODM4ZDAyNjhmNWE2NDZjOWY1ZDliZA==
 SHA512:
   metadata.gz: !binary |-
-    YWEzNmU0ZWMxOTQxNTNjZThkOTkyZDU2ZTBmZjc5M2RhZDc1MjQ1ZDRkYjgx
-    MmY4NGZlN2I3MTY0ZDhkZjNhMzM0YjQ0MzM4OWNhMTU0ZDZkNGU2MzdhNWYz
-    MmNhZGVhZTViOWExNGU3NWVjMGViYjJkMzRhOGRiNDliZjcxNzk=
+    NTJlMWQzNDU2OTMwZGMzZDMyNjAwYjE5M2M0NjQ2ZWQ5YmQ4OGRlMDIxODEy
+    MDY0NTI4NTU2ZWE0NDdlMmU4MTY1NjAwM2IwODAxZTUwZGMxN2IyNDdhZWI3
+    MWU5MGYyYmNjYWMyYzA5OTQzM2JjMzZhNTk3Njg4NThkZTJjYjE=
   data.tar.gz: !binary |-
-    OGE1ODUxZDU3OTgyODM4ZTI2Njk3YTMyMGVmY2JkNzZmNDNjYmU1NWI5ZDg0
-    ODZjYWYxZjhjZDU4ODRhNDY4YWU5NzZjNDE0NTkyNTk1ODkyZjVjOTExMWRl
-    NjM4MmFiYzlhMjM3MmY2ZTA1ZDBlN2EyYThkNjgwMjQwZjQ5ZTg=
+    YTQ1YjU1MDNmOTliMzgwYTU4YmNjYTdlMmFjNjQyNTBiODcxMTk5NWU5ZGJh
+    MjNkMDZlYTBlM2RkOTdjNTBjNDI5MDI4MDIzMTNjNzBjZjQxMzE1ZTk0YTc3
+    Y2Q3MjgxNmE2NjFmOTQxMjdkYWE3ZWU3NjMwNzgwNzU3MjY0ZGU=

data/lib/logstash/outputs/elasticsearch.rb

@@ -0,0 +1,328 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/environment"
+require "logstash/outputs/base"
+require "logstash/json"
+require "stud/buffer"
+require "socket" # for Socket.gethostname
+
+# This output lets you store logs in Elasticsearch and is the most recommended
+# output for Logstash. If you plan on using the Kibana web interface, you'll
+# need to use this output.
+#
+#   *VERSION NOTE*: Your Elasticsearch cluster must be running Elasticsearch
+#   %ELASTICSEARCH_VERSION%. If you use any other version of Elasticsearch,
+#   you should set `protocol => http` in this plugin.
+#
+# If you want to set other Elasticsearch options that are not exposed directly
+# as configuration options, there are two methods:
+#
+# * Create an `elasticsearch.yml` file in the $PWD of the Logstash process
+# * Pass in es.* java properties (java -Des.node.foo= or ruby -J-Des.node.foo=)
+#
+# With the default `protocol` setting ("node"), this plugin will join your
+# Elasticsearch cluster as a client node, so it will show up in Elasticsearch's
+# cluster status.
+#
+# You can learn more about Elasticsearch at <http://www.elasticsearch.org>
+#
+# ## Operational Notes
+#
+# Template management requires Elasticsearch version 0.90.7 or later. If you
+# are using a version older than this, please upgrade. You will receive
+# more benefits than just template management!
+#
+# If using the default `protocol` setting ("node"), your firewalls might need
+# to permit port 9300 in *both* directions (from Logstash to Elasticsearch, and
+# Elasticsearch to Logstash)
+class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
+  include Stud::Buffer
+
+  config_name "elasticsearch"
+  milestone 3
+
+  # The index to write events to. This can be dynamic using the %{foo} syntax.
+  # The default value will partition your indices by day so you can more easily
+  # delete old data or only search specific date ranges.
+  # Indexes may not contain uppercase characters.
+  config :index, :validate => :string, :default => "logstash-%{+YYYY.MM.dd}"
+
+  # The index type to write events to. Generally you should try to write only
+  # similar events to the same 'type'. String expansion '%{foo}' works here.
+  config :index_type, :validate => :string
+
+  # Starting in Logstash 1.3 (unless you set option "manage_template" to false)
+  # a default mapping template for Elasticsearch will be applied, if you do not
+  # already have one set to match the index pattern defined (default of
+  # "logstash-%{+YYYY.MM.dd}"), minus any variables.  For example, in this case
+  # the template will be applied to all indices starting with logstash-*
+  #
+  # If you have dynamic templating (e.g. creating indices based on field names)
+  # then you should set "manage_template" to false and use the REST API to upload
+  # your templates manually.
+  config :manage_template, :validate => :boolean, :default => true
+
+  # This configuration option defines how the template is named inside Elasticsearch.
+  # Note that if you have used the template management features and subsequently
+  # change this, you will need to prune the old template manually, e.g.
+  # curl -XDELETE <http://localhost:9200/_template/OldTemplateName?pretty>
+  # where OldTemplateName is whatever the former setting was.
+  config :template_name, :validate => :string, :default => "logstash"
+
+  # You can set the path to your own template here, if you so desire.
+  # If not set, the included template will be used.
+  config :template, :validate => :path
+
+  # Overwrite the current template with whatever is configured
+  # in the template and template_name directives.
+  config :template_overwrite, :validate => :boolean, :default => false
+
+  # The document ID for the index. Useful for overwriting existing entries in
+  # Elasticsearch with the same ID.
+  config :document_id, :validate => :string, :default => nil
+
+  # The name of your cluster if you set it on the Elasticsearch side. Useful
+  # for discovery.
+  config :cluster, :validate => :string
+
+  # The hostname or IP address of the host to use for Elasticsearch unicast discovery
+  # This is only required if the normal multicast/cluster discovery stuff won't
+  # work in your environment.
+  config :host, :validate => :string
+
+  # The port for Elasticsearch transport to use.
+  #
+  # If you do not set this, the following defaults are used:
+  # * `protocol => http` - port 9200
+  # * `protocol => transport` - port 9300-9305
+  # * `protocol => node` - port 9300-9305
+  config :port, :validate => :string
+
+  # The name/address of the host to bind to for Elasticsearch clustering
+  config :bind_host, :validate => :string
+
+  # This is only valid for the 'node' protocol.
+  #
+  # The port for the node to listen on.
+  config :bind_port, :validate => :number
+
+  # Run the Elasticsearch server embedded in this process.
+  # This option is useful if you want to run a single Logstash process that
+  # handles log processing and indexing; it saves you from needing to run
+  # a separate Elasticsearch process.
+  config :embedded, :validate => :boolean, :default => false
+
+  # If you are running the embedded Elasticsearch server, you can set the http
+  # port it listens on here; it is not common to need this setting changed from
+  # default.
+  config :embedded_http_port, :validate => :string, :default => "9200-9300"
+
+  # This setting no longer does anything. It exists to keep config validation
+  # from failing. It will be removed in future versions.
+  config :max_inflight_requests, :validate => :number, :default => 50, :deprecated => true
+
+  # The node name Elasticsearch will use when joining a cluster.
+  #
+  # By default, this is generated internally by the ES client.
+  config :node_name, :validate => :string
+
+  # This plugin uses the bulk index api for improved indexing performance.
+  # To make efficient bulk api calls, we will buffer a certain number of
+  # events before flushing that out to Elasticsearch. This setting
+  # controls how many events will be buffered before sending a batch
+  # of events.
+  config :flush_size, :validate => :number, :default => 5000
+
+  # The amount of time since last flush before a flush is forced.
+  #
+  # This setting helps ensure slow event rates don't get stuck in Logstash.
+  # For example, if your `flush_size` is 100, and you have received 10 events,
+  # and it has been more than `idle_flush_time` seconds since the last flush,
+  # Logstash will flush those 10 events automatically.
+  #
+  # This helps keep both fast and slow log streams moving along in
+  # near-real-time.
+  config :idle_flush_time, :validate => :number, :default => 1
+
+  # Choose the protocol used to talk to Elasticsearch.
+  #
+  # The 'node' protocol will connect to the cluster as a normal Elasticsearch
+  # node (but will not store data). This allows you to use things like
+  # multicast discovery. If you use the `node` protocol, you must permit
+  # bidirectional communication on the port 9300 (or whichever port you have
+  # configured).
+  #
+  # The 'transport' protocol will connect to the host you specify and will
+  # not show up as a 'node' in the Elasticsearch cluster. This is useful
+  # in situations where you cannot permit connections outbound from the
+  # Elasticsearch cluster to this Logstash server.
+  #
+  # The 'http' protocol will use the Elasticsearch REST/HTTP interface to talk
+  # to elasticsearch.
+  #
+  # All protocols will use bulk requests when talking to Elasticsearch.
+  #
+  # The default `protocol` setting under java/jruby is "node". The default
+  # `protocol` on non-java rubies is "http"
+  config :protocol, :validate => [ "node", "transport", "http" ]
+
+  # The Elasticsearch action to perform. Valid actions are: `index`, `delete`.
+  #
+  # Use of this setting *REQUIRES* you also configure the `document_id` setting
+  # because `delete` actions all require a document id.
+  #
+  # What does each action do?
+  #
+  # - index: indexes a document (an event from logstash).
+  # - delete: deletes a document by id
+  #
+  # For more details on actions, check out the [Elasticsearch bulk API documentation](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-bulk.html)
+  config :action, :validate => :string, :default => "index"
+
+  public
+  def register
+    client_settings = {}
+    client_settings["cluster.name"] = @cluster if @cluster
+    client_settings["network.host"] = @bind_host if @bind_host
+    client_settings["transport.tcp.port"] = @bind_port if @bind_port
+
+    if @node_name
+      client_settings["node.name"] = @node_name
+    else
+      client_settings["node.name"] = "logstash-#{Socket.gethostname}-#{$$}-#{object_id}"
+    end
+
+    if @protocol.nil?
+      @protocol = LogStash::Environment.jruby? ? "node" : "http"
+    end
+
+    if ["node", "transport"].include?(@protocol)
+      # Node or TransportClient; requires JRuby
+      raise(LogStash::PluginLoadingError, "This configuration requires JRuby. If you are not using JRuby, you must set 'protocol' to 'http'. For example: output { elasticsearch { protocol => \"http\" } }") unless LogStash::Environment.jruby?
+      LogStash::Environment.load_elasticsearch_jars!
+
+      # setup log4j properties for Elasticsearch
+      LogStash::Logger.setup_log4j(@logger)
+    end
+
+    require "logstash/outputs/elasticsearch/protocol"
+
+    if @port.nil?
+      @port = case @protocol
+        when "http"; "9200"
+        when "transport", "node"; "9300-9305"
+      end
+    end
+
+    if @host.nil? && @protocol == "http"
+      @logger.info("No 'host' set in elasticsearch output. Defaulting to localhost")
+      @host = "localhost"
+    end
+
+    options = {
+      :host => @host,
+      :port => @port,
+      :client_settings => client_settings
+    }
+
+
+    client_class = case @protocol
+      when "transport"
+        LogStash::Outputs::Elasticsearch::Protocols::TransportClient
+      when "node"
+        LogStash::Outputs::Elasticsearch::Protocols::NodeClient
+      when "http"
+        LogStash::Outputs::Elasticsearch::Protocols::HTTPClient
+    end
+
+    if @embedded
+      raise(LogStash::ConfigurationError, "The 'embedded => true' setting is only valid for the elasticsearch output under JRuby. You are running #{RUBY_DESCRIPTION}") unless LogStash::Environment.jruby?
+      LogStash::Environment.load_elasticsearch_jars!
+
+      # Default @host with embedded to localhost. This should help avoid
+      # newbies tripping on ubuntu and other distros that have a default
+      # firewall that blocks multicast.
+      @host ||= "localhost"
+
+      # Start Elasticsearch local.
+      start_local_elasticsearch
+    end
+
+    @client = client_class.new(options)
+
+    @logger.info("New Elasticsearch output", :cluster => @cluster,
+                 :host => @host, :port => @port, :embedded => @embedded,
+                 :protocol => @protocol)
+
+
+    if @manage_template
+      @logger.info("Automatic template management enabled", :manage_template => @manage_template.to_s)
+      @client.template_install(@template_name, get_template, @template_overwrite)
+    end # if @manage_templates
+
+    buffer_initialize(
+      :max_items => @flush_size,
+      :max_interval => @idle_flush_time,
+      :logger => @logger
+    )
+  end # def register
+
+  public
+  def get_template
+    if @template.nil?
+      @template = LogStash::Environment.plugin_path("outputs/elasticsearch/elasticsearch-template.json")
+      if !File.exists?(@template)
+        raise "You must specify 'template => ...' in your elasticsearch output (I looked for '#{@template}')"
+      end
+    end
+    template_json = IO.read(@template).gsub(/\n/,'')
+    @logger.info("Using mapping template", :template => template_json)
+    return LogStash::Json.load(template_json)
+  end # def get_template
+
+  protected
+  def start_local_elasticsearch
+    @logger.info("Starting embedded Elasticsearch local node.")
+    builder = org.elasticsearch.node.NodeBuilder.nodeBuilder
+    # Disable 'local only' - LOGSTASH-277
+    #builder.local(true)
+    builder.settings.put("cluster.name", @cluster) if @cluster
+    builder.settings.put("node.name", @node_name) if @node_name
+    builder.settings.put("network.host", @bind_host) if @bind_host
+    builder.settings.put("http.port", @embedded_http_port)
+
+    @embedded_elasticsearch = builder.node
+    @embedded_elasticsearch.start
+  end # def start_local_elasticsearch
+
+  public
+  def receive(event)
+    return unless output?(event)
+
+    # Set the 'type' value for the index.
+    if @index_type
+      type = event.sprintf(@index_type)
+    else
+      type = event["type"] || "logs"
+    end
+
+    index = event.sprintf(@index)
+
+    document_id = @document_id ? event.sprintf(@document_id) : nil
+    buffer_receive([event.sprintf(@action), { :_id => document_id, :_index => index, :_type => type }, event.to_hash])
+  end # def receive
+
+  def flush(actions, teardown=false)
+    @client.bulk(actions)
+    # TODO(sissel): Handle errors. Since bulk requests could mostly succeed
+    # (aka partially fail), we need to figure out what documents need to be
+    # retried.
+    #
+    # In the worst case, a failing flush (exception) will incur a retry from Stud::Buffer.
+  end # def flush
+
+  def teardown
+    buffer_flush(:final => true)
+  end
+
+end # class LogStash::Outputs::Elasticsearch

data/lib/logstash/outputs/elasticsearch/elasticsearch-template.json

@@ -0,0 +1,34 @@
+{
+  "template" : "logstash-*",
+  "settings" : {
+    "index.refresh_interval" : "5s"
+  },
+  "mappings" : {
+    "_default_" : {
+       "_all" : {"enabled" : true},
+       "dynamic_templates" : [ {
+         "string_fields" : {
+           "match" : "*",
+           "match_mapping_type" : "string",
+           "mapping" : {
+             "type" : "string", "index" : "analyzed", "omit_norms" : true,
+               "fields" : {
+                 "raw" : {"type": "string", "index" : "not_analyzed", "ignore_above" : 256}
+               }
+           }
+         }
+       } ],
+       "properties" : {
+         "@version": { "type": "string", "index": "not_analyzed" },
+         "geoip"  : {
+           "type" : "object",
+             "dynamic": true,
+             "path": "full",
+             "properties" : {
+               "location" : { "type" : "geo_point" }
+             }
+         }
+       }
+    }
+  }
+}

data/lib/logstash/outputs/elasticsearch/protocol.rb

@@ -0,0 +1,271 @@
+require "logstash/outputs/elasticsearch"
+require "cabin"
+
+module LogStash::Outputs::Elasticsearch
+  module Protocols
+    class Base
+      private
+      def initialize(options={})
+        # host(s), port, cluster
+        @logger = Cabin::Channel.get
+      end
+
+      def client
+        return @client if @client
+        @client = build_client(@options)
+        return @client
+      end # def client
+
+
+      def template_install(name, template, force=false)
+        if template_exists?(name) && !force
+          @logger.debug("Found existing Elasticsearch template. Skipping template management", :name => name)
+          return
+        end
+        template_put(name, template)
+      end
+
+      # Do a bulk request with the given actions.
+      #
+      # 'actions' is expected to be an array of bulk requests as string json
+      # values.
+      #
+      # Each 'action' becomes a single line in the bulk api call. For more
+      # details on the format of each.
+      def bulk(actions)
+        raise NotImplemented, "You must implement this yourself"
+        # bulk([
+        # '{ "index" : { "_index" : "test", "_type" : "type1", "_id" : "1" } }',
+        # '{ "field1" : "value1" }'
+        #])
+      end
+
+      public(:initialize, :template_install)
+    end
+
+    class HTTPClient < Base
+      private
+
+      DEFAULT_OPTIONS = {
+        :port => 9200
+      }
+
+      def initialize(options={})
+        require "ftw"
+        super
+        require "elasticsearch" # gem 'elasticsearch-ruby'
+        @options = DEFAULT_OPTIONS.merge(options)
+        @client = client
+      end
+
+      def build_client(options)
+        client = Elasticsearch::Client.new(
+          :host => [options[:host], options[:port]].join(":")
+        )
+
+        # Use FTW to do indexing requests, for now, until we
+        # can identify and resolve performance problems of elasticsearch-ruby
+        @bulk_url = "http://#{options[:host]}:#{options[:port]}/_bulk"
+        @agent = FTW::Agent.new
+
+        return client
+      end
+
+      if ENV["BULK"] == "esruby"
+        def bulk(actions)
+          bulk_esruby(actions)
+        end
+      else
+        def bulk(actions)
+          bulk_ftw(actions)
+        end
+      end
+
+      def bulk_esruby(actions)
+        @client.bulk(:body => actions.collect do |action, args, source|
+          if source
+            next [ { action => args }, source ]
+          else
+            next { action => args }
+          end
+        end.flatten)
+      end # def bulk_esruby
+
+      # Avoid creating a new string for newline every time
+      NEWLINE = "\n".freeze
+      def bulk_ftw(actions)
+        body = actions.collect do |action, args, source|
+          header = { action => args }
+          if source
+            next [ LogStash::Json.dump(header), NEWLINE, LogStash::Json.dump(source), NEWLINE ]
+          else
+            next [ LogStash::Json.dump(header), NEWLINE ]
+          end
+        end.flatten.join("")
+        begin
+          response = @agent.post!(@bulk_url, :body => body)
+        rescue EOFError
+          @logger.warn("EOF while writing request or reading response header from elasticsearch", :host => @host, :port => @port)
+          raise
+        end
+
+        # Consume the body for error checking
+        # This will also free up the connection for reuse.
+        response_body = ""
+        begin
+          response.read_body { |chunk| response_body += chunk }
+        rescue EOFError
+          @logger.warn("EOF while reading response body from elasticsearch",
+                       :url => @bulk_url)
+          raise
+        end
+
+        if response.status != 200
+          @logger.error("Error writing (bulk) to elasticsearch",
+                        :response => response, :response_body => response_body,
+                        :request_body => body)
+          raise "Non-OK response code from Elasticsearch: #{response.status}"
+        end
+      end # def bulk_ftw
+
+      def template_exists?(name)
+        @client.indices.get_template(:name => name)
+        return true
+      rescue Elasticsearch::Transport::Transport::Errors::NotFound
+        return false
+      end # def template_exists?
+
+      def template_put(name, template)
+        @client.indices.put_template(:name => name, :body => template)
+      end # template_put
+
+      public(:bulk)
+    end # class HTTPClient
+
+    class NodeClient < Base
+      private
+
+      DEFAULT_OPTIONS = {
+        :port => 9300,
+      }
+
+      def initialize(options={})
+        super
+        require "java"
+        @options = DEFAULT_OPTIONS.merge(options)
+        setup(@options)
+        @client = client
+      end # def initialize
+
+      def settings
+        return @settings
+      end
+
+      def setup(options={})
+        @settings = org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder
+        if options[:host]
+          @settings.put("discovery.zen.ping.multicast.enabled", false)
+          @settings.put("discovery.zen.ping.unicast.hosts", hosts(options))
+        end
+
+        @settings.put("node.client", true)
+        @settings.put("http.enabled", false)
+
+        if options[:client_settings]
+          options[:client_settings].each do |key, value|
+            @settings.put(key, value)
+          end
+        end
+
+        return @settings
+      end
+
+      def hosts(options)
+        if options[:port].to_s =~ /^\d+-\d+$/
+          # port ranges are 'host[port1-port2]' according to
+          # http://www.elasticsearch.org/guide/reference/modules/discovery/zen/
+          # However, it seems to only query the first port.
+          # So generate our own list of unicast hosts to scan.
+          range = Range.new(*options[:port].split("-"))
+          return range.collect { |p| "#{options[:host]}:#{p}" }.join(",")
+        else
+          return "#{options[:host]}:#{options[:port]}"
+        end
+      end # def hosts
+
+      def build_client(options)
+        nodebuilder = org.elasticsearch.node.NodeBuilder.nodeBuilder
+        return nodebuilder.settings(@settings).node.client
+      end # def build_client
+
+      def bulk(actions)
+        # Actions an array of [ action, action_metadata, source ]
+        prep = @client.prepareBulk
+        actions.each do |action, args, source|
+          prep.add(build_request(action, args, source))
+        end
+        response = prep.execute.actionGet()
+
+        # TODO(sissel): What format should the response be in?
+      end # def bulk
+
+      def build_request(action, args, source)
+        case action
+          when "index"
+            request = org.elasticsearch.action.index.IndexRequest.new(args[:_index])
+            request.id(args[:_id]) if args[:_id]
+            request.source(source)
+          when "delete"
+            request = org.elasticsearch.action.delete.DeleteRequest.new(args[:_index])
+            request.id(args[:_id])
+          #when "update"
+          #when "create"
+        end # case action
+
+        request.type(args[:_type]) if args[:_type]
+        return request
+      end # def build_request
+
+      def template_exists?(name)
+        request = org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesRequestBuilder.new(@client.admin.indices, name)
+        response = request.get
+        return !response.getIndexTemplates.isEmpty
+      end # def template_exists?
+
+      def template_put(name, template)
+        request = org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequestBuilder.new(@client.admin.indices, name)
+        request.setSource(LogStash::Json.dump(template))
+
+        # execute the request and get the response, if it fails, we'll get an exception.
+        request.get
+      end # template_put
+
+      public(:initialize, :bulk)
+    end # class NodeClient
+
+    class TransportClient < NodeClient
+      private
+      def build_client(options)
+        client = org.elasticsearch.client.transport.TransportClient.new(settings.build)
+
+        if options[:host]
+          client.addTransportAddress(
+            org.elasticsearch.common.transport.InetSocketTransportAddress.new(
+              options[:host], options[:port].to_i
+            )
+          )
+        end
+
+        return client
+      end # def build_client
+    end # class TransportClient
+  end # module Protocols
+
+  module Requests
+    class GetIndexTemplates; end
+    class Bulk; end
+    class Index; end
+    class Delete; end
+  end
+end
+

data/logstash-output-elasticsearch.gemspec

@@ -0,0 +1,33 @@
+Gem::Specification.new do |s|
+ 
+  s.name            = 'logstash-output-elasticsearch'
+  s.version         = '0.1.1'
+  s.licenses        = ['Apache License (2.0)']
+  s.summary         = "Logstash Output to Elasticsearch"
+  s.description     = "Output events to elasticsearch"
+  s.authors         = ["Elasticsearch"]
+  s.email           = 'rubycoder@example.com'
+  s.homepage        = "http://logstash.net/"
+  s.require_paths = ["lib"]
+  
+  # Files
+  s.files = `git ls-files`.split($\)
+  
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+  
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true" }
+
+  # Jar dependencies
+  s.requirements << "jar 'org.elasticsearch:elasticsearch', '1.2.2'"
+
+  # Gem dependencies
+  s.add_runtime_dependency 'elasticsearch'
+  s.add_runtime_dependency 'stud'
+  s.add_runtime_dependency 'cabin', ['>=0.6.0']
+  s.add_runtime_dependency 'ftw', ['~> 0.0.39']
+  s.add_runtime_dependency 'logstash', '>= 1.4.0', '< 2.0.0'
+  s.add_runtime_dependency 'jar-dependencies', ['~> 0.0.6'] 
+  
+end

data/spec/outputs/elasticsearch.rb

@@ -0,0 +1,349 @@
+require "test_utils"
+require "ftw"
+require "logstash/plugin"
+require "logstash/json"
+
+describe "outputs/elasticsearch" do
+  extend LogStash::RSpec
+
+  it "should register" do
+    output = LogStash::Plugin.lookup("output", "elasticsearch").new("embedded" => "false", "protocol" => "transport", "manage_template" => "false")
+
+    # register will try to load jars and raise if it cannot find jars
+    expect {output.register}.to_not raise_error
+  end
+
+  describe "ship lots of events w/ default index_type", :elasticsearch => true do
+    # Generate a random index name
+    index = 10.times.collect { rand(10).to_s }.join("")
+    type = 10.times.collect { rand(10).to_s }.join("")
+
+    # Write about 10000 events. Add jitter to increase likeliness of finding
+    # boundary-related bugs.
+    event_count = 10000 + rand(500)
+    flush_size = rand(200) + 1
+
+    config <<-CONFIG
+      input {
+        generator {
+          message => "hello world"
+          count => #{event_count}
+          type => "#{type}"
+        }
+      }
+      output {
+        elasticsearch {
+          host => "127.0.0.1"
+          index => "#{index}"
+          flush_size => #{flush_size}
+        }
+      }
+    CONFIG
+
+    agent do
+      # Try a few times to check if we have the correct number of events stored
+      # in ES.
+      #
+      # We try multiple times to allow final agent flushes as well as allowing
+      # elasticsearch to finish processing everything.
+      ftw = FTW::Agent.new
+      ftw.post!("http://localhost:9200/#{index}/_refresh")
+
+      # Wait until all events are available.
+      Stud::try(10.times) do
+        data = ""
+        response = ftw.get!("http://127.0.0.1:9200/#{index}/_count?q=*")
+        response.read_body { |chunk| data << chunk }
+        result = LogStash::Json.load(data)
+        count = result["count"]
+        insist { count } == event_count
+      end
+
+      response = ftw.get!("http://127.0.0.1:9200/#{index}/_search?q=*&size=1000")
+      data = ""
+      response.read_body { |chunk| data << chunk }
+      result = LogStash::Json.load(data)
+      result["hits"]["hits"].each do |doc|
+        # With no 'index_type' set, the document type should be the type
+        # set on the input
+        insist { doc["_type"] } == type
+        insist { doc["_index"] } == index
+        insist { doc["_source"]["message"] } == "hello world"
+      end
+    end
+  end
+
+  describe "testing index_type", :elasticsearch => true do
+    describe "no type value" do
+      # Generate a random index name
+      index = 10.times.collect { rand(10).to_s }.join("")
+      event_count = 100 + rand(100)
+      flush_size = rand(200) + 1
+
+      config <<-CONFIG
+        input {
+          generator {
+            message => "hello world"
+            count => #{event_count}
+          }
+        }
+        output {
+          elasticsearch {
+            host => "127.0.0.1"
+            index => "#{index}"
+            flush_size => #{flush_size}
+          }
+        }
+      CONFIG
+
+      agent do
+        ftw = FTW::Agent.new
+        ftw.post!("http://localhost:9200/#{index}/_refresh")
+
+        # Wait until all events are available.
+        Stud::try(10.times) do
+          data = ""
+          response = ftw.get!("http://127.0.0.1:9200/#{index}/_count?q=*")
+          response.read_body { |chunk| data << chunk }
+          result = LogStash::Json.load(data)
+          count = result["count"]
+          insist { count } == event_count
+        end
+
+        response = ftw.get!("http://127.0.0.1:9200/#{index}/_search?q=*&size=1000")
+        data = ""
+        response.read_body { |chunk| data << chunk }
+        result = LogStash::Json.load(data)
+        result["hits"]["hits"].each do |doc|
+          insist { doc["_type"] } == "logs"
+        end
+      end
+    end
+
+    describe "default event type value" do
+      # Generate a random index name
+      index = 10.times.collect { rand(10).to_s }.join("")
+      event_count = 100 + rand(100)
+      flush_size = rand(200) + 1
+
+      config <<-CONFIG
+        input {
+          generator {
+            message => "hello world"
+            count => #{event_count}
+            type => "generated"
+          }
+        }
+        output {
+          elasticsearch {
+            host => "127.0.0.1"
+            index => "#{index}"
+            flush_size => #{flush_size}
+          }
+        }
+      CONFIG
+
+      agent do
+        ftw = FTW::Agent.new
+        ftw.post!("http://localhost:9200/#{index}/_refresh")
+
+        # Wait until all events are available.
+        Stud::try(10.times) do
+          data = ""
+          response = ftw.get!("http://127.0.0.1:9200/#{index}/_count?q=*")
+          response.read_body { |chunk| data << chunk }
+          result = LogStash::Json.load(data)
+          count = result["count"]
+          insist { count } == event_count
+        end
+
+        response = ftw.get!("http://127.0.0.1:9200/#{index}/_search?q=*&size=1000")
+        data = ""
+        response.read_body { |chunk| data << chunk }
+        result = LogStash::Json.load(data)
+        result["hits"]["hits"].each do |doc|
+          insist { doc["_type"] } == "generated"
+        end
+      end
+    end
+  end
+
+  describe "action => ...", :elasticsearch => true do
+    index_name = 10.times.collect { rand(10).to_s }.join("")
+
+    config <<-CONFIG
+      input {
+        generator {
+          message => "hello world"
+          count => 100
+        }
+      }
+      output {
+        elasticsearch {
+          host => "127.0.0.1"
+          index => "#{index_name}"
+        }
+      }
+    CONFIG
+
+
+    agent do
+      ftw = FTW::Agent.new
+      ftw.post!("http://localhost:9200/#{index_name}/_refresh")
+
+      # Wait until all events are available.
+      Stud::try(10.times) do
+        data = ""
+        response = ftw.get!("http://127.0.0.1:9200/#{index_name}/_count?q=*")
+        response.read_body { |chunk| data << chunk }
+        result = LogStash::Json.load(data)
+        count = result["count"]
+        insist { count } == 100
+      end
+
+      response = ftw.get!("http://127.0.0.1:9200/#{index_name}/_search?q=*&size=1000")
+      data = ""
+      response.read_body { |chunk| data << chunk }
+      result = LogStash::Json.load(data)
+      result["hits"]["hits"].each do |doc|
+        insist { doc["_type"] } == "logs"
+      end
+    end
+
+    describe "default event type value", :elasticsearch => true do
+      # Generate a random index name
+      index = 10.times.collect { rand(10).to_s }.join("")
+      event_count = 100 + rand(100)
+      flush_size = rand(200) + 1
+
+      config <<-CONFIG
+        input {
+          generator {
+            message => "hello world"
+            count => #{event_count}
+            type => "generated"
+          }
+        }
+        output {
+          elasticsearch {
+            host => "127.0.0.1"
+            index => "#{index}"
+            flush_size => #{flush_size}
+          }
+        }
+      CONFIG
+
+      agent do
+        ftw = FTW::Agent.new
+        ftw.post!("http://localhost:9200/#{index}/_refresh")
+
+        # Wait until all events are available.
+        Stud::try(10.times) do
+          data = ""
+          response = ftw.get!("http://127.0.0.1:9200/#{index}/_count?q=*")
+          response.read_body { |chunk| data << chunk }
+          result = LogStash::Json.load(data)
+          count = result["count"]
+          insist { count } == event_count
+        end
+
+        response = ftw.get!("http://127.0.0.1:9200/#{index}/_search?q=*&size=1000")
+        data = ""
+        response.read_body { |chunk| data << chunk }
+        result = LogStash::Json.load(data)
+        result["hits"]["hits"].each do |doc|
+          insist { doc["_type"] } == "generated"
+        end
+      end
+    end
+  end
+
+  describe "index template expected behavior", :elasticsearch => true do
+    ["node", "transport", "http"].each do |protocol|
+      context "with protocol => #{protocol}" do
+        subject do
+          require "logstash/outputs/elasticsearch"
+          settings = {
+            "manage_template" => true,
+            "template_overwrite" => true,
+            "protocol" => protocol,
+            "host" => "localhost"
+          }
+          next LogStash::Outputs::ElasticSearch.new(settings)
+        end
+
+        before :each do
+          # Delete all templates first.
+          require "elasticsearch"
+
+          # Clean ES of data before we start.
+          @es = Elasticsearch::Client.new
+          @es.indices.delete_template(:name => "*")
+
+          # This can fail if there are no indexes, ignore failure.
+          @es.indices.delete(:index => "*") rescue nil
+
+          subject.register
+
+          subject.receive(LogStash::Event.new("message" => "sample message here"))
+          subject.receive(LogStash::Event.new("somevalue" => 100))
+          subject.receive(LogStash::Event.new("somevalue" => 10))
+          subject.receive(LogStash::Event.new("somevalue" => 1))
+          subject.receive(LogStash::Event.new("country" => "us"))
+          subject.receive(LogStash::Event.new("country" => "at"))
+          subject.receive(LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] }))
+          subject.buffer_flush(:final => true)
+          @es.indices.refresh
+
+          # Wait or fail until everything's indexed.
+          Stud::try(20.times) do
+            r = @es.search
+            insist { r["hits"]["total"] } == 7
+          end
+        end
+
+        it "permits phrase searching on string fields" do
+          results = @es.search(:q => "message:\"sample message\"")
+          insist { results["hits"]["total"] } == 1
+          insist { results["hits"]["hits"][0]["_source"]["message"] } == "sample message here"
+        end
+
+        it "numbers dynamically map to a numeric type and permit range queries" do
+          results = @es.search(:q => "somevalue:[5 TO 105]")
+          insist { results["hits"]["total"] } == 2
+
+          values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
+          insist { values }.include?(10)
+          insist { values }.include?(100)
+          reject { values }.include?(1)
+        end
+
+        it "creates .raw field fro any string field which is not_analyzed" do
+          results = @es.search(:q => "message.raw:\"sample message here\"")
+          insist { results["hits"]["total"] } == 1
+          insist { results["hits"]["hits"][0]["_source"]["message"] } == "sample message here"
+
+          # partial or terms should not work.
+          results = @es.search(:q => "message.raw:\"sample\"")
+          insist { results["hits"]["total"] } == 0
+        end
+
+        it "make [geoip][location] a geo_point" do
+          results = @es.search(:body => { "filter" => { "geo_distance" => { "distance" => "1000km", "geoip.location" => { "lat" => 0.5, "lon" => 0.5 } } } })
+          insist { results["hits"]["total"] } == 1
+          insist { results["hits"]["hits"][0]["_source"]["geoip"]["location"] } == [ 0.0, 0.0 ]
+        end
+
+        it "should index stopwords like 'at' " do
+          results = @es.search(:body => { "facets" => { "t" => { "terms" => { "field" => "country" } } } })["facets"]["t"]
+          terms = results["terms"].collect { |t| t["term"] }
+
+          insist { terms }.include?("us")
+
+          # 'at' is a stopword, make sure stopwords are not ignored.
+          insist { terms }.include?("at")
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Elasticsearch
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-16 00:00:00.000000000 Z
+date: 2014-08-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: elasticsearch
@@ -86,12 +86,31 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 2.0.0
+- !ruby/object:Gem::Dependency
+  name: jar-dependencies
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.0.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.0.6
 description: Output events to elasticsearch
 email: rubycoder@example.com
 executables: []
 extensions: []
 extra_rdoc_files: []
-files: []
+files:
+- lib/logstash/outputs/elasticsearch.rb
+- lib/logstash/outputs/elasticsearch/elasticsearch-template.json
+- lib/logstash/outputs/elasticsearch/protocol.rb
+- logstash-output-elasticsearch.gemspec
+- spec/outputs/elasticsearch.rb
 homepage: http://logstash.net/
 licenses:
 - Apache License (2.0)
@@ -118,5 +137,6 @@ rubygems_version: 2.3.0
 signing_key: 
 specification_version: 4
 summary: Logstash Output to Elasticsearch
-test_files: []
+test_files:
+- spec/outputs/elasticsearch.rb
 has_rdoc: