fluent-plugin-out-solr 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f134d6f7dcc1c67533c8bf19deb402cab474d64c
-  data.tar.gz: 8433e0d6e1b0bf02dabdf70f9ed592e1c622d6ec
+  metadata.gz: 9fba370cfa9aa095c4d62a31298fb5986d7e82c3
+  data.tar.gz: bc3e47a0617809a5ccee18e9811eb3832eef63f5
 SHA512:
-  metadata.gz: 84430230e21373715cb2b956021119992b488290f65584208850bc613bc4d67f689f1fe44eb357af6579265edc376aec10ada999380cc1f63a675475720285db
-  data.tar.gz: 5097d3937e3966989c3dea2ef74706ff9174b711a1a96727bab1a51b72143d9d9cfa29e4236d445e6b20ffba8a813a9d2aed96a1e003c32390af60b20890bbba
+  metadata.gz: 58b5f2a164b2851f917843e29b70db8fc91d160c887441bb66969011a64727da40b93809d47c8620d32153fbb319a9b807cff48b0affbe3fc05172fd25576e66
+  data.tar.gz: 982239392497f4e9a88e9bfdc5f93ef404fe58c4b7d7e215220c3ba78996e4f02dfcb6f891d3e5daaf10bcfa174d9d159903ccca89a361d9d18d8a721bef2157

data/README.md

@@ -30,15 +30,18 @@ Notice: no relationship with [btigit/fluent-plugin-solr](https://github.com/btig
   include_tag_key true
   tag_key tag
   time_field timestamp
-  use_utc false
+  utc # if you do not want to use localtime
+
   flush_interval 3s
 </match>
 ```
 
-#### core rotation by date
+#### time sliced by date
 
 You should create cores in advance.
 
+See: [Time Sliced Plugin Overview - Buffer Plugin Overview | Fluentd](http://docs.fluentd.org/articles/buffer-plugin-overview#time-sliced-plugin-overview)
+
 ```
 <source>
   type tail
@@ -48,24 +51,35 @@ You should create cores in advance.
 </source>
 
 <match apache.*>
-  type solr
+  type solr_time_sliced
   host localhost
   port 8983
-  use_core_rotation true
-  core_prefix apache
+  core log-%Y%m%d
   include_tag_key true
   tag_key tag
   time_field timestamp
-  use_utc false
+  utc # if you do not want to use localtime
+
   flush_interval 3s
 </match>
 ```
 
 ### solrconfig.xml snippet
 
-See: [UniqueKey - Solr Wiki](https://wiki.apache.org/solr/UniqueKey)
+* See: [UniqueKey - Solr Wiki](https://wiki.apache.org/solr/UniqueKey)
+* fluent-plugin-out-solr doesn't commit. use autoSoftCommit and autoCommit.
+
 
 ```xml
+  <autoCommit>
+    <maxTime>${solr.autoCommit.maxTime:15000}</maxTime>
+    <openSearcher>false</openSearcher>
+  </autoCommit>
+
+  <autoSoftCommit>
+    <maxTime>${solr.autoSoftCommit.maxTime:10}</maxTime>
+  </autoSoftCommit>
+
   <requestHandler name="/update" class="solr.UpdateRequestHandler">
     <lst name="defaults">
       <str name="update.chain">uuid</str>

data/fluent-plugin-out-solr.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-out-solr'
-  s.version       = '0.0.3'
+  s.version       = '0.0.4'
   s.authors       = ['diogo', 'pitr', 'haruyama']
   s.email         = ['team@uken.com', 'haruyama@unixuser.org']
   s.description   = %q{Solr output plugin for Fluent event collector}
@@ -20,5 +20,4 @@ Gem::Specification.new do |s|
 
   s.add_development_dependency 'rake'
   s.add_development_dependency 'webmock'
-  s.add_development_dependency 'timecop'
 end

data/lib/fluent/plugin/out_solr.rb

@@ -1,27 +1,35 @@
 # encoding: UTF-8
-require 'net/http'
-require 'uri'
 
 # Solr output plugin for Fluent
 class Fluent::SolrOutput < Fluent::BufferedOutput
   Fluent::Plugin.register_output('solr', self)
 
+  require 'fluent/plugin/solr_util'
+  include SolrUtil
+
   config_param :host,              :string,  default: 'localhost'
   config_param :port,              :integer, default: 8983
   config_param :core,              :string,  default: 'collection1'
-  config_param :use_core_rotation, :bool,    default: false
-  config_param :core_prefix,       :string,  default: 'core'
   config_param :time_field,        :string,  default: 'timestamp'
-  config_param :use_utc,           :bool,    default: false
 
   include Fluent::SetTagKeyMixin
   config_set_default :include_tag_key, false
 
+  attr_accessor :localtime
   def initialize
+    require 'net/http'
+    require 'uri'
+    require 'time'
     super
+    @localtime = true
   end
 
   def configure(conf)
+    if conf['utc']
+      @localtime = false
+    elsif conf['localtime']
+      @localtime = true
+    end
     super
   end
 
@@ -38,25 +46,6 @@ class Fluent::SolrOutput < Fluent::BufferedOutput
   end
 
   def write(chunk)
-    bulk_messages = Hash.new { |h, k| h[k] = [] }
-
-    chunk.msgpack_each do |tag, unixtime, record|
-      time = Time.at(unixtime)
-      time = time.utc if @use_utc
-      record.merge!(@time_field => time.strftime('%FT%TZ'))
-      record.merge!(@tag_key    => tag) if @include_tag_key
-      if @use_core_rotation
-        bulk_messages[@core_prefix + '-' + time.strftime('%F')] << record
-      else
-        bulk_messages[@core] << record
-      end
-    end
-
-    http = Net::HTTP.new(@host, @port.to_i)
-    bulk_messages.each do |corename, messages|
-      request = Net::HTTP::Post.new('/solr/' + URI.escape(corename) + '/update', 'content-type' => 'application/json; charset=utf-8')
-      request.body = Yajl::Encoder.encode(messages)
-      http.request(request).value
-    end
+    update_core(chunk, @core)
   end
 end

data/lib/fluent/plugin/out_solr_time_sliced.rb

@@ -0,0 +1,63 @@
+# encoding: UTF-8
+
+# Solr output plugin for Fluent
+class Fluent::SolrTimeSlicedOutput < Fluent::TimeSlicedOutput
+  Fluent::Plugin.register_output('solr_time_sliced', self)
+
+  require 'fluent/plugin/solr_util'
+  include SolrUtil
+
+  config_set_default :buffer_type,       'memory'
+  config_set_default :time_slice_format, '%Y%m%d'
+
+  config_param :host,              :string,  default: 'localhost'
+  config_param :port,              :integer, default: 8983
+  config_param :core,              :string,  default: 'log-%Y%m%d'
+  config_param :time_field,        :string,  default: 'timestamp'
+
+  include Fluent::SetTagKeyMixin
+  config_set_default :include_tag_key, false
+
+  attr_accessor :localtime
+
+  def initialize
+    require 'net/http'
+    require 'uri'
+    require 'time'
+    super
+    @localtime = true
+  end
+
+  def configure(conf)
+    if conf['core']
+      if conf['core'].index('%S')
+        conf['time_slice_format'] = '%Y%m%d%H%M%S'
+      elsif conf['core'].index('%M')
+        conf['time_slice_format'] = '%Y%m%d%H%M'
+      elsif conf['core'].index('%H')
+        conf['time_slice_format'] = '%Y%m%d%H'
+      end
+    end
+    super
+  end
+
+  def start
+    super
+  end
+
+  def format(tag, time, record)
+    [tag, time, record].to_msgpack
+  end
+
+  def shutdown
+    super
+  end
+
+  def core_format(chunk_key)
+    Time.strptime(chunk_key, @time_slice_format).strftime(@core)
+  end
+
+  def write(chunk)
+    update_core(chunk, core_format(chunk.key))
+  end
+end

data/lib/fluent/plugin/solr_util.rb

@@ -0,0 +1,21 @@
+# encoding: UTF-8
+
+# Solr utility
+module SolrUtil
+  def update_core(chunk, core)
+    documents = []
+
+    chunk.msgpack_each do |tag, unixtime, record|
+      time = Time.at(unixtime)
+      time = time.utc unless @localtime
+      record.merge!(@time_field => time.strftime('%FT%TZ'))
+      record.merge!(@tag_key    => tag) if @include_tag_key
+      documents << record
+    end
+
+    http = Net::HTTP.new(@host, @port.to_i)
+    request = Net::HTTP::Post.new('/solr/' + URI.escape(core) + '/update', 'content-type' => 'application/json; charset=utf-8')
+    request.body = Yajl::Encoder.encode(documents)
+    http.request(request).value
+  end
+end

data/test/plugin/test_out_solr.rb

@@ -4,7 +4,6 @@ require 'fluent/test'
 require 'fluent/plugin/out_solr'
 
 require 'webmock/test_unit'
-require 'timecop'
 
 require 'helper'
 
@@ -16,7 +15,6 @@ WebMock.disable_net_connect!
 # Solr output test
 class SolrOutput < Test::Unit::TestCase
   attr_accessor :index_cmds, :content_type
-  attr_accessor :index_cmds2
 
   def setup
     Fluent::Test.setup
@@ -38,21 +36,13 @@ class SolrOutput < Test::Unit::TestCase
     end
   end
 
-  def stub_solr2(url = 'http://localhost:8983/solr/collection1/update')
-    stub_request(:post, url).with do |req|
-      @index_cmds2   = JSON.parse(req.body)
-    end
-  end
-
   def stub_solr_unavailable(url = 'http://localhost:8983/solr/collection1/update')
     stub_request(:post, url).to_return(status: [503, 'Service Unavailable'])
   end
 
   def test_writes_to_default_index
     stub_solr
-    Timecop.freeze(Time.local(2013, 12, 20, 19, 0, 0)) do
-      driver.emit(sample_record)
-    end
+    driver.emit(sample_record, Time.local(2013, 12, 20, 19, 0, 0).to_i)
     driver.run
     assert_equal(26,                     @index_cmds[0]['age'])
     assert_equal('42',                   @index_cmds[0]['request_id'])
@@ -116,32 +106,16 @@ class SolrOutput < Test::Unit::TestCase
     assert_equal(@index_cmds[0]['tag'], 'mytag')
   end
 
-  def test_use_utc
-    driver.configure("use_utc true\n")
+  def test_utc
+    driver.configure("utc\n")
     stub_solr
-    Timecop.freeze(Time.local(2013, 12, 20, 19, 0, 0)) do
-      driver.emit(sample_record)
-    end
+    ENV['TZ'] = 'Japan'
+    driver.emit(sample_record, Time.local(2013, 12, 20, 19, 0, 0).to_i)
+    ENV['TZ'] = nil
     driver.run
     assert_equal('2013-12-20T10:00:00Z', @index_cmds[0]['timestamp'])
   end
 
-  def test_use_core_rotation
-    driver.configure("use_core_rotation true\n")
-    driver.configure("core_prefix log\n")
-    stub_solr('http://localhost:8983/solr/log-2013-12-20/update')
-    stub_solr2('http://localhost:8983/solr/log-2013-12-21/update')
-    Timecop.freeze(Time.local(2013, 12, 20, 19, 0, 0)) do
-      driver.emit(sample_record)
-    end
-    Timecop.freeze(Time.local(2013, 12, 21, 19, 0, 0)) do
-      driver.emit(sample_record)
-    end
-    driver.run
-    assert_equal(1,  @index_cmds.count)
-    assert_equal(1,  @index_cmds2.count)
-  end
-
   def test_request_error
     stub_solr_unavailable
     driver.emit(sample_record)

data/test/plugin/test_out_solr_time_sliced.rb

@@ -0,0 +1,168 @@
+require 'test/unit'
+
+require 'fluent/test'
+require 'fluent/plugin/out_solr_time_sliced'
+
+require 'webmock/test_unit'
+
+require 'helper'
+
+$LOAD_PATH.push File.expand_path('../lib', __FILE__)
+$LOAD_PATH.push File.dirname(__FILE__)
+
+WebMock.disable_net_connect!
+
+# SolrTimeSlicedOutput test
+class SolrTimeSlicedOutputTest < Test::Unit::TestCase
+  attr_accessor :index_cmds, :content_type
+  attr_accessor :index_cmds2
+
+  def setup
+    Fluent::Test.setup
+    @driver = nil
+  end
+
+  def driver(tag = 'test', conf = '')
+    @driver ||= Fluent::Test::TimeSlicedOutputTestDriver.new(Fluent::SolrTimeSlicedOutput, tag).configure(conf)
+  end
+
+  def sample_record
+    { 'age' => 26, 'request_id' => '42' }
+  end
+
+  def time
+    Time.local(2013, 12, 21, 17, 30, 0).to_i
+  end
+
+  def time2
+    Time.local(2013, 12, 22, 17, 30, 0).to_i
+  end
+
+  def stub_solr(url = 'http://localhost:8983/solr/log-20131221/update')
+    stub_request(:post, url).with do |req|
+      @content_type = req.headers['Content-Type']
+      @index_cmds   = JSON.parse(req.body)
+    end
+  end
+
+  def stub_solr2(url = 'http://localhost:8983/solr/log-20131222/update')
+    stub_request(:post, url).with do |req|
+      @content_type2 = req.headers['Content-Type']
+      @index_cmds2   = JSON.parse(req.body)
+    end
+  end
+
+  def stub_solr_unavailable(url = 'http://localhost:8983/solr/log-20131221/update')
+    stub_request(:post, url).to_return(status: [503, 'Service Unavailable'])
+  end
+
+  def test_writes_to_default_index
+    stub_solr
+    driver.emit(sample_record, time)
+    driver.run
+    assert_equal(26,                     @index_cmds[0]['age'])
+    assert_equal('42',                   @index_cmds[0]['request_id'])
+    assert_equal('2013-12-21T17:30:00Z', @index_cmds[0]['timestamp'])
+  end
+
+  def test_wrties_with_proper_content_type
+    stub_solr
+    driver.emit(sample_record)
+    driver.run
+    assert_equal('application/json; charset=utf-8', @content_type)
+  end
+
+  def test_writes_to_speficied_core
+    driver.configure("core log-%Y%m%d%H\n")
+    solr_request = stub_solr('http://localhost:8983/solr/log-2013122117/update')
+    driver.emit(sample_record, time)
+    driver.run
+    assert_requested(solr_request)
+  end
+
+  def test_writes_to_speficied_core2
+    driver.configure("core log2-%Y%m%d%H%M\n")
+    solr_request = stub_solr('http://localhost:8983/solr/log2-201312211730/update')
+    driver.emit(sample_record, time)
+    driver.run
+    assert_requested(solr_request)
+  end
+
+  def test_writes_to_speficied_core3
+    driver.configure("core log3-%Y%m%d%H%M%S\n")
+    solr_request = stub_solr('http://localhost:8983/solr/log3-20131221173000/update')
+    driver.emit(sample_record, time)
+    driver.run
+    assert_requested(solr_request)
+  end
+
+  def test_writes_to_speficied_host
+    driver.configure("host 192.168.33.50\n")
+    solr_request = stub_solr('http://192.168.33.50:8983/solr/log-20131221/update')
+    driver.emit(sample_record, time)
+    driver.run
+    assert_requested(solr_request)
+  end
+
+  def test_writes_to_speficied_port
+    driver.configure("port 9201\n")
+    solr_request = stub_solr('http://localhost:9201/solr/log-20131221/update')
+    driver.emit(sample_record, time)
+    driver.run
+    assert_requested(solr_request)
+  end
+
+  def test_emit_multi_records
+    stub_solr
+    driver.emit(sample_record, time)
+    driver.emit(sample_record.merge('age' => 27), time)
+    driver.run
+    assert_equal(2,  @index_cmds.count)
+    assert_equal(26, @index_cmds[0]['age'])
+    assert_equal(27, @index_cmds[1]['age'])
+  end
+
+  def test_doesnt_add_tag_key_by_default
+    stub_solr
+    driver.emit(sample_record)
+    driver.run
+    assert_nil(@index_cmds[0]['tag'])
+  end
+
+  def test_adds_tag_key_when_configured
+    driver('mytag').configure("include_tag_key true\n")
+    stub_solr
+    driver.emit(sample_record, time)
+    driver.run
+    assert(@index_cmds[0].key?('tag'))
+    assert_equal(@index_cmds[0]['tag'], 'mytag')
+  end
+
+  def test_utc
+    driver.configure("utc\n")
+    stub_solr
+    ENV['TZ'] = 'Japan'
+    driver.emit(sample_record, Time.local(2013, 12, 22, 7, 30, 0).to_i)
+    ENV['TZ'] = nil
+    driver.run
+    assert_equal('2013-12-21T22:30:00Z', @index_cmds[0]['timestamp'])
+  end
+
+  def test_emit_records_on_different_days
+    stub_solr
+    stub_solr2
+    driver.emit(sample_record, time)
+    driver.emit(sample_record, time2)
+    driver.run
+    assert_equal(1,  @index_cmds.count)
+    assert_equal(1,  @index_cmds2.count)
+  end
+
+  def test_request_error
+    stub_solr_unavailable
+    driver.emit(sample_record, time)
+    assert_raise(Net::HTTPFatalError) do
+      driver.run
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-out-solr
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - diogo
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-20 00:00:00.000000000 Z
+date: 2013-12-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -54,20 +54,6 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: timecop
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
 description: Solr output plugin for Fluent event collector
 email:
 - team@uken.com
@@ -83,9 +69,12 @@ files:
 - Rakefile
 - fluent-plugin-out-solr.gemspec
 - lib/fluent/plugin/out_solr.rb
+- lib/fluent/plugin/out_solr_time_sliced.rb
+- lib/fluent/plugin/solr_util.rb
 - test/helper.rb
 - test/plugin/.rubocop.yml
 - test/plugin/test_out_solr.rb
+- test/plugin/test_out_solr_time_sliced.rb
 homepage: https://github.com/haruyama/fluent-plugin-out-solr
 licenses:
 - MIT
@@ -114,3 +103,4 @@ test_files:
 - test/helper.rb
 - test/plugin/.rubocop.yml
 - test/plugin/test_out_solr.rb
+- test/plugin/test_out_solr_time_sliced.rb