fluent-plugin-elasticsearch 0.1.4 → 0.2.0

data/Gemfile

@@ -2,3 +2,5 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fluent-plugin-elasticsearch.gemspec
 gemspec
+
+gem 'coveralls', require: false

data/History.md

@@ -1,6 +1,12 @@
 Changelog
 =========
 
+0.2.0
+=====
+
+- fix encoding issues with JSON conversion and again when sending to elasticsearch (#19, #21)
+- add logstash_dateformat option (#20)
+
 0.1.4
 =====
 

data/README.md

@@ -1,5 +1,11 @@
 # Fluent::Plugin::Elasticsearch
 
+[![Gem Version](https://badge.fury.io/rb/fluent-plugin-elasticsearch.png)](http://badge.fury.io/rb/fluent-plugin-elasticsearch)
+[![Dependency Status](https://gemnasium.com/uken/guard-sidekiq.png)](https://gemnasium.com/uken/fluent-plugin-elasticsearch)
+[![Build Status](https://travis-ci.org/uken/fluent-plugin-elasticsearch.png?branch=master)](https://travis-ci.org/uken/fluent-plugin-elasticsearch)
+[![Coverage Status](https://coveralls.io/repos/uken/fluent-plugin-elasticsearch/badge.png)](https://coveralls.io/r/uken/fluent-plugin-elasticsearch)
+[![Code Climate](https://codeclimate.com/github/uken/fluent-plugin-elasticsearch.png)](https://codeclimate.com/github/uken/fluent-plugin-elasticsearch)
+
 I wrote this so you can search logs routed through Fluentd.
 
 ## Installation
@@ -32,6 +38,12 @@ logstash_prefix mylogs # defaults to "logstash"
 
 By default, the records inserted into index `logstash-YYMMDD`. This option allows to insert into specified index like `mylogs-YYMMDD`.
 
+```
+logstash_dateformat %Y.%m. # defaults to "%Y.%m.%d"
+```
+
+By default, the records inserted into index `logstash-YYMMDD`. This option allows to insert into specified index like `logstash-YYYYMM` for a monthly index.
+
 ---
 
 ```
@@ -82,7 +94,21 @@ retry_wait 1.0
 num_threads 1
 ```
 
+---
+
+Please consider using [fluent-plugin-forest](https://github.com/tagomoris/fluent-plugin-forest) to send multiple logs to multiple ElasticSearch indices:
 
+```
+<match my.logs.*>
+  type forest
+  subtype elasticsearch
+  remove_prefix my.logs
+  <template>
+    logstash_prefix ${tag}
+    # ...
+  </template>
+</match>
+```
 
 ## Contributing
 

data/fluent-plugin-elasticsearch.gemspec

@@ -3,12 +3,13 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = "fluent-plugin-elasticsearch"
-  s.version       = '0.1.4'
+  s.version       = '0.2.0'
   s.authors       = ["diogo", 'pitr']
   s.email         = ["team@uken.com"]
   s.description   = %q{ElasticSearch output plugin for Fluent event collector}
   s.summary       = s.description
   s.homepage      = "https://github.com/uken/fluent-plugin-elasticsearch"
+  s.license       = 'MIT'
 
   s.files         = `git ls-files`.split($/)
   s.executables   = s.files.grep(%r{^bin/}).map{ |f| File.basename(f) }

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -9,6 +9,7 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
   config_param :port, :integer, :default => 9200
   config_param :logstash_format, :bool, :default => false
   config_param :logstash_prefix, :string, :default => "logstash"
+  config_param :logstash_dateformat, :string, :default => "%Y.%m.%d"
   config_param :type_name, :string, :default => "fluentd"
   config_param :index_name, :string, :default => "fluentd"
   config_param :id_key, :string, :default => nil
@@ -42,7 +43,7 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
     chunk.msgpack_each do |tag, time, record|
       if @logstash_format
         record.merge!({"@timestamp" => Time.at(time).to_datetime.to_s})
-        target_index = "#{@logstash_prefix}-#{Time.at(time).getutc.strftime("%Y.%m.%d")}"
+        target_index = "#{@logstash_prefix}-#{Time.at(time).getutc.strftime("#{@logstash_dateformat}")}"
       else
         target_index = @index_name
       end
@@ -55,13 +56,13 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
       if @id_key && record[@id_key]
         meta['index']['_id'] = record[@id_key]
       end
-      bulk_message << meta.to_json
-      bulk_message << record.to_json
+      bulk_message << Yajl::Encoder.encode(meta)
+      bulk_message << Yajl::Encoder.encode(record)
     end
     bulk_message << ""
 
     http = Net::HTTP.new(@host, @port.to_i)
-    request = Net::HTTP::Post.new("/_bulk")
+    request = Net::HTTP::Post.new('/_bulk', {'content-type' => 'application/json; charset=utf-8'})
     request.body = bulk_message.join("\n")
     http.request(request).value
   end

data/test/plugin/test_out_elasticsearch.rb

@@ -14,7 +14,7 @@ $:.push File.dirname(__FILE__)
 WebMock.disable_net_connect!
 
 class ElasticsearchOutput < Test::Unit::TestCase
-  attr_accessor :index_cmds
+  attr_accessor :index_cmds, :content_type
 
   def setup
     Fluent::Test.setup
@@ -31,6 +31,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def stub_elastic(url="http://localhost:9200/_bulk")
     stub_request(:post, url).with do |req|
+      @content_type = req.headers["Content-Type"]
       @index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
     end
   end
@@ -46,6 +47,13 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal('fluentd', index_cmds.first['index']['_index'])
   end
 
+  def test_wrties_with_proper_content_type
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert_equal("application/json; charset=utf-8", @content_type)
+  end
+
   def test_writes_to_default_type
     stub_elastic
     driver.emit(sample_record)
@@ -123,6 +131,29 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal(logstash_index, index_cmds.first['index']['_index'])
   end
 
+    def test_writes_to_logstash_index_with_specified_dateformat
+    driver.configure("logstash_format true\n")
+    driver.configure("logstash_dateformat %Y.%m\n")
+    time = Time.parse Date.today.to_s
+    logstash_index = "logstash-#{time.getutc.strftime("%Y.%m")}"
+    stub_elastic
+    driver.emit(sample_record, time)
+    driver.run
+    assert_equal(logstash_index, index_cmds.first['index']['_index'])
+  end
+
+  def test_writes_to_logstash_index_with_specified_prefix_and_dateformat
+    driver.configure("logstash_format true\n")
+    driver.configure("logstash_prefix myprefix\n")
+    driver.configure("logstash_dateformat %Y.%m\n")
+    time = Time.parse Date.today.to_s
+    logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m")}"
+    stub_elastic
+    driver.emit(sample_record, time)
+    driver.run
+    assert_equal(logstash_index, index_cmds.first['index']['_index'])
+  end
+
   def test_doesnt_add_logstash_timestamp_by_default
     stub_elastic
     driver.emit(sample_record)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-28 00:00:00.000000000 Z
+date: 2013-12-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -78,7 +78,8 @@ files:
 - test/helper.rb
 - test/plugin/test_out_elasticsearch.rb
 homepage: https://github.com/uken/fluent-plugin-elasticsearch
-licenses: []
+licenses:
+- MIT
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -89,12 +90,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: -1091394761449257025
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: -1091394761449257025
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23
@@ -104,4 +111,3 @@ summary: ElasticSearch output plugin for Fluent event collector
 test_files:
 - test/helper.rb
 - test/plugin/test_out_elasticsearch.rb
-has_rdoc: