logstash-output-elasticsearch 0.1.5 → 0.1.6

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    OTFjZmRiNDk2YjMzNGE4NzVmYTgyNGMzYzAxNGUxZjY3ZDEwMTE4MQ==
+    N2UwMzczMTE0ZWNmZjFhOGU4Mjk0YjUwZDE5YmVmNDQyMWUzMzg0Mw==
   data.tar.gz: !binary |-
-    ZTcyNTg2NzA5ZDQ3NGRmZDM4NGE1ZmJlNzllOWI3MTAxMzgwNjllMQ==
+    ODJkYzg1NWU5YWEzYmZmOWRiMDFkNjFlYmM0Y2Q2NmZiZGEwNjYyMQ==
 SHA512:
   metadata.gz: !binary |-
-    MmIxMGUyZjAxYTE5NmUzZGIzZWRmNDc0NmMyOWVlMTQ2YTQxMjA2ZmQzZDg2
-    M2FlMWVjMGRmNTQwMjdjZjM3NzY5Y2UxMTk1ODVlM2NhOTBlZmIwMzA4ZjE4
-    YWNjYjdlMzgxYmUyMTMwN2UwMzI2MWEzNjI3OWMyNWM4ZDRlYmE=
+    Y2M3ZDliMjhjYmY4OGM0NjI5ODA2OTYyZWE1MzM2MzE0MmVlZDI1NmE0OTNl
+    YWY0MmEzNzQ2OWUwODc0ZWRkNmI0OWY3MDViOTU3ZjlhMGY5NzFjZDJhNGQz
+    NjdlZWQ4YWM3Nzk1NTI3OWQ1MDEyZmY0MzBkODM0Nzc3YTgzNWM=
   data.tar.gz: !binary |-
-    YjZmYWQzZjMxMGUwODFkZDMzYWY4OTQ2Zjc1NmZiMDRiODUwYWVmMmIyNDNl
-    YjMwMjIyMzg2YjA5M2MwNTdlNDdiMmZlOTA0OTZkNGRkZGNmYTFkYTFjNGUz
-    YTQ4YjQ1ZWFiZjQzZTU5Y2I5MzI1ZWJlZjVhNTlhMDU5NzZmNDI=
+    MjY2Mzc0ZDRiOGEyOGIzMDg2NWMzNGRiMDA2MWYxMDM4YzhiZmZiN2ViZDhk
+    ZDI2ZGE2MzRhMTA1ZDMxZGM2NTQ3YzU1ZjE0NjYzMTU0NTk4OTJhMWFiNTMy
+    MzQzYzc0OGM2NTNkMzJmNmIxY2I4YWY5ODZhYTgzOTAwODUxYWM=

data/lib/logstash/outputs/elasticsearch.rb

@@ -42,6 +42,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # The default value will partition your indices by day so you can more easily
   # delete old data or only search specific date ranges.
   # Indexes may not contain uppercase characters.
+  # For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}
   config :index, :validate => :string, :default => "logstash-%{+YYYY.MM.dd}"
 
   # The index type to write events to. Generally you should try to write only
@@ -178,7 +179,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # - index: indexes a document (an event from Logstash).
   # - delete: deletes a document by id
   #
-  # For more details on actions, check out the [Elasticsearch bulk API documentation](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-bulk.html)
+  # For more details on actions, check out the http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-bulk.html[Elasticsearch bulk API documentation]
   config :action, :validate => :string, :default => "index"
 
   # Username and password (HTTP only)
@@ -200,6 +201,14 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # Set the truststore password
   config :truststore_password, :validate => :password
 
+  # helper function to replace placeholders
+  # in index names to wildcards
+  # example:
+  #    "logs-%{YYYY}" -> "logs-*"
+  def wildcard_substitute(name)
+    name.gsub(/%\{[^}]+\}/, "*")
+  end
+
   public
   def register
     client_settings = {}
@@ -242,38 +251,14 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
       @host = ["localhost"]
     end
 
-    if @ssl
-      if @protocol == "http"
-        @protocol = "https"
-        if @cacert && @truststore
-          raise(LogStash::ConfigurationError, "Use either \"cacert\" or \"truststore\" when configuring the CA certificate") if @truststore
-        end
-        ssl_options = {}
-        if @cacert then
-          @truststore, ssl_options[:truststore_password] = generate_jks @cacert
-        elsif @truststore
-          ssl_options[:truststore_password] = @truststore_password.value if @truststore_password
-        end
-        ssl_options[:truststore] = @truststore
-        client_settings[:ssl] = ssl_options
-      else
-        raise(LogStash::ConfigurationError, "SSL is not supported for '#{@protocol}'. Change the protocol to 'http' if you need SSL.")
-      end
-    end
+    client_settings.merge! setup_ssl()
 
     common_options = {
       :protocol => @protocol,
       :client_settings => client_settings
     }
 
-    if @user && @password
-      if @protocol =~ /http/
-        common_options[:user] = ::URI.escape(@user, "@:")
-        common_options[:password] = ::URI.escape(@password.value, "@:")
-      else
-        raise(LogStash::ConfigurationError, "User and password parameters are not supported for '#{@protocol}'. Change the protocol to 'http' if you need them.")
-      end
-    end
+    common_options.merge! setup_basic_auth()
 
     client_class = case @protocol
       when "transport"
@@ -350,6 +335,40 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     @logger.debug? and @logger.debug("Switched current elasticsearch client to ##{@client_idx} at #{@host[@client_idx]}")
   end
 
+  private
+  def setup_ssl
+    return {} unless @ssl
+    if @protocol != "http"
+      raise(LogStash::ConfigurationError, "SSL is not supported for '#{@protocol}'. Change the protocol to 'http' if you need SSL.")
+    end
+    @protocol = "https"
+    if @cacert && @truststore
+      raise(LogStash::ConfigurationError, "Use either \"cacert\" or \"truststore\" when configuring the CA certificate") if @truststore
+    end
+    ssl_options = {}
+    if @cacert then
+      @truststore, ssl_options[:truststore_password] = generate_jks @cacert
+    elsif @truststore
+      ssl_options[:truststore_password] = @truststore_password.value if @truststore_password
+    end
+    ssl_options[:truststore] = @truststore
+    { ssl: ssl_options }
+  end
+
+  private
+  def setup_basic_auth
+    return {} unless @user && @password
+
+    if @protocol =~ /http/
+      {
+        :user => ::URI.escape(@user, "@:"),
+        :password => ::URI.escape(@password.value, "@:")
+      }
+    else
+      raise(LogStash::ConfigurationError, "User and password parameters are not supported for '#{@protocol}'. Change the protocol to 'http' if you need them.")
+    end
+  end
+
   public
   def get_template
     if @template.nil?
@@ -359,8 +378,10 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
       end
     end
     template_json = IO.read(@template).gsub(/\n/,'')
-    @logger.info("Using mapping template", :template => template_json)
-    return LogStash::Json.load(template_json)
+    template = LogStash::Json.load(template_json)
+    template['template'] = wildcard_substitute(@index)
+    @logger.info("Using mapping template", :template => template)
+    return template
   end # def get_template
 
   protected

data/logstash-output-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '0.1.5'
+  s.version         = '0.1.6'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Logstash Output to Elasticsearch"
   s.description     = "Output events to elasticsearch"
@@ -20,18 +20,19 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
 
   # Jar dependencies
-  s.requirements << "jar 'org.elasticsearch:elasticsearch', '1.3.1'"
+  s.requirements << "jar 'org.elasticsearch:elasticsearch', '1.4.0'"
 
   # Gem dependencies
-  s.add_runtime_dependency 'elasticsearch', ['~> 1.0.6']
-  s.add_runtime_dependency 'stud'
-  s.add_runtime_dependency 'cabin', ['>=0.6.0']
-  s.add_runtime_dependency 'ftw', ['~> 0.0.40']
+  s.add_runtime_dependency 'elasticsearch', ['>= 1.0.6', '~> 1.0']
+  s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']
+  s.add_runtime_dependency 'cabin', ['~> 0.6']
   s.add_runtime_dependency 'logstash', '>= 1.4.0', '< 2.0.0'
   s.add_runtime_dependency 'jar-dependencies'
 
+  s.add_development_dependency 'ftw', ['>= 0.0.40', '~> 0']
+
   if RUBY_PLATFORM == 'java'
-    gem.add_runtime_dependency "manticore"
+    s.add_runtime_dependency "manticore", '~> 0.3'
   end
 end
 

data/spec/outputs/elasticsearch.rb

@@ -12,6 +12,7 @@ describe "outputs/elasticsearch" do
     expect {output.register}.to_not raise_error
   end
 
+
   describe "ship lots of events w/ default index_type", :elasticsearch => true do
     # Generate a random index name
     index = 10.times.collect { rand(10).to_s }.join("")
@@ -257,6 +258,34 @@ describe "outputs/elasticsearch" do
     end
   end
 
+  describe "wildcard substitution in index templates", :todo => true do
+    require "logstash/outputs/elasticsearch"
+
+    let(:template) { '{"template" : "not important, will be updated by :index"}' }
+
+    def settings_with_index(index)
+      return {
+        "manage_template" => true,
+        "template_overwrite" => true,
+        "protocol" => "http",
+        "host" => "localhost",
+        "index" => "#{index}"
+      }
+    end
+
+    it "should substitude placeholders" do
+      IO.stub(:read).with(anything) { template }
+      es_output = LogStash::Outputs::ElasticSearch.new(settings_with_index("index-%{YYYY}"))
+      insist { es_output.get_template['template'] } == "index-*"
+    end
+
+    it "should do nothing to an index with no placeholder" do
+      IO.stub(:read).with(anything) { template }
+      es_output = LogStash::Outputs::ElasticSearch.new(settings_with_index("index"))
+      insist { es_output.get_template['template'] } == "index"
+    end
+  end
+
   describe "index template expected behavior", :elasticsearch => true do
     ["node", "transport", "http"].each do |protocol|
       context "with protocol => #{protocol}" do

metadata

@@ -1,71 +1,69 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.6
 platform: ruby
 authors:
 - Elasticsearch
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-07 00:00:00.000000000 Z
+date: 2014-11-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: elasticsearch
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 1.0.6
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.0.6
-- !ruby/object:Gem::Dependency
-  name: stud
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.6
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
-  name: cabin
+  name: stud
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.6.0
+        version: 0.0.17
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.6.0
+        version: 0.0.17
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.0'
 - !ruby/object:Gem::Dependency
-  name: ftw
+  name: cabin
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.0.40
+        version: '0.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.0.40
+        version: '0.6'
 - !ruby/object:Gem::Dependency
   name: logstash
   requirement: !ruby/object:Gem::Requirement
@@ -100,6 +98,26 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: ftw
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.0.40
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.0.40
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Output events to elasticsearch
 email: richard.pijnenburg@elasticsearch.com
 executables: []
@@ -138,7 +156,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements:
-- jar 'org.elasticsearch:elasticsearch', '1.3.1'
+- jar 'org.elasticsearch:elasticsearch', '1.4.0'
 rubyforge_project: 
 rubygems_version: 2.4.1
 signing_key: