fluent-plugin-out-solr 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7916f0ec7e50455a7b9e11b836be309b9deffcad
-  data.tar.gz: 6d4b7033396e97c37b3c3005ad006b9cf7d98642
+  metadata.gz: f134d6f7dcc1c67533c8bf19deb402cab474d64c
+  data.tar.gz: 8433e0d6e1b0bf02dabdf70f9ed592e1c622d6ec
 SHA512:
-  metadata.gz: bb946c862e75c7c45fc4dbeef7287f14e258e4c9b891cdc2a5a538530fa92d9801819722f3b2204eef5fbb1fed72c70fbc38f8af47467815fbc971b18f11aee2
-  data.tar.gz: 053c03470d2ba1e7cf52739a33606b62ca0b897e6592553325bbc08b5898ad437daf5812b9d9b1ccfd5ab931aab6f8505413851ff402c80e34105cc17eff137a
+  metadata.gz: 84430230e21373715cb2b956021119992b488290f65584208850bc613bc4d67f689f1fe44eb357af6579265edc376aec10ada999380cc1f63a675475720285db
+  data.tar.gz: 5097d3937e3966989c3dea2ef74706ff9174b711a1a96727bab1a51b72143d9d9cfa29e4236d445e6b20ffba8a813a9d2aed96a1e003c32390af60b20890bbba

data/README.md

@@ -12,6 +12,8 @@ Notice: no relationship with [btigit/fluent-plugin-solr](https://github.com/btig
 
 ### fluent.conf snippet
 
+#### single core
+
 ```
 <source>
   type tail
@@ -27,6 +29,33 @@ Notice: no relationship with [btigit/fluent-plugin-solr](https://github.com/btig
   core collection1
   include_tag_key true
   tag_key tag
+  time_field timestamp
+  use_utc false
+  flush_interval 3s
+</match>
+```
+
+#### core rotation by date
+
+You should create cores in advance.
+
+```
+<source>
+  type tail
+  format apache
+  path /tmp/access.log
+  tag apache.access
+</source>
+
+<match apache.*>
+  type solr
+  host localhost
+  port 8983
+  use_core_rotation true
+  core_prefix apache
+  include_tag_key true
+  tag_key tag
+  time_field timestamp
   use_utc false
   flush_interval 3s
 </match>

data/fluent-plugin-out-solr.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-out-solr'
-  s.version       = '0.0.2'
+  s.version       = '0.0.3'
   s.authors       = ['diogo', 'pitr', 'haruyama']
   s.email         = ['team@uken.com', 'haruyama@unixuser.org']
   s.description   = %q{Solr output plugin for Fluent event collector}
@@ -20,4 +20,5 @@ Gem::Specification.new do |s|
 
   s.add_development_dependency 'rake'
   s.add_development_dependency 'webmock'
+  s.add_development_dependency 'timecop'
 end

data/lib/fluent/plugin/out_solr.rb

@@ -6,11 +6,13 @@ require 'uri'
 class Fluent::SolrOutput < Fluent::BufferedOutput
   Fluent::Plugin.register_output('solr', self)
 
-  config_param :host,       :string,  default: 'localhost'
-  config_param :port,       :integer, default: 8983
-  config_param :core,       :string,  default: 'collection1'
-  config_param :time_field, :string,  default: 'timestamp'
-  config_param :use_utc,    :bool,    default: false
+  config_param :host,              :string,  default: 'localhost'
+  config_param :port,              :integer, default: 8983
+  config_param :core,              :string,  default: 'collection1'
+  config_param :use_core_rotation, :bool,    default: false
+  config_param :core_prefix,       :string,  default: 'core'
+  config_param :time_field,        :string,  default: 'timestamp'
+  config_param :use_utc,           :bool,    default: false
 
   include Fluent::SetTagKeyMixin
   config_set_default :include_tag_key, false
@@ -36,25 +38,25 @@ class Fluent::SolrOutput < Fluent::BufferedOutput
   end
 
   def write(chunk)
-    bulk_message = []
+    bulk_messages = Hash.new { |h, k| h[k] = [] }
 
-    chunk.msgpack_each do |tag, time, record|
-
-      time_string =
-        if @use_utc
-          Time.at(time).utc.strftime('%FT%TZ')
-        else
-          Time.at(time).strftime('%FT%TZ')
-        end
-
-      record.merge!(@time_field => time_string)
+    chunk.msgpack_each do |tag, unixtime, record|
+      time = Time.at(unixtime)
+      time = time.utc if @use_utc
+      record.merge!(@time_field => time.strftime('%FT%TZ'))
       record.merge!(@tag_key    => tag) if @include_tag_key
-      bulk_message << record
+      if @use_core_rotation
+        bulk_messages[@core_prefix + '-' + time.strftime('%F')] << record
+      else
+        bulk_messages[@core] << record
+      end
     end
 
     http = Net::HTTP.new(@host, @port.to_i)
-    request = Net::HTTP::Post.new('/solr/' + URI.escape(@core) + '/update', 'content-type' => 'application/json; charset=utf-8')
-    request.body = Yajl::Encoder.encode(bulk_message)
-    http.request(request).value
+    bulk_messages.each do |corename, messages|
+      request = Net::HTTP::Post.new('/solr/' + URI.escape(corename) + '/update', 'content-type' => 'application/json; charset=utf-8')
+      request.body = Yajl::Encoder.encode(messages)
+      http.request(request).value
+    end
   end
 end

data/test/plugin/test_out_solr.rb

@@ -4,7 +4,7 @@ require 'fluent/test'
 require 'fluent/plugin/out_solr'
 
 require 'webmock/test_unit'
-require 'date'
+require 'timecop'
 
 require 'helper'
 
@@ -16,6 +16,7 @@ WebMock.disable_net_connect!
 # Solr output test
 class SolrOutput < Test::Unit::TestCase
   attr_accessor :index_cmds, :content_type
+  attr_accessor :index_cmds2
 
   def setup
     Fluent::Test.setup
@@ -37,16 +38,25 @@ class SolrOutput < Test::Unit::TestCase
     end
   end
 
+  def stub_solr2(url = 'http://localhost:8983/solr/collection1/update')
+    stub_request(:post, url).with do |req|
+      @index_cmds2   = JSON.parse(req.body)
+    end
+  end
+
   def stub_solr_unavailable(url = 'http://localhost:8983/solr/collection1/update')
     stub_request(:post, url).to_return(status: [503, 'Service Unavailable'])
   end
 
   def test_writes_to_default_index
     stub_solr
-    driver.emit(sample_record)
+    Timecop.freeze(Time.local(2013, 12, 20, 19, 0, 0)) do
+      driver.emit(sample_record)
+    end
     driver.run
-    assert_equal(26,   @index_cmds[0]['age'])
-    assert_equal('42', @index_cmds[0]['request_id'])
+    assert_equal(26,                     @index_cmds[0]['age'])
+    assert_equal('42',                   @index_cmds[0]['request_id'])
+    assert_equal('2013-12-20T19:00:00Z', @index_cmds[0]['timestamp'])
   end
 
   def test_wrties_with_proper_content_type
@@ -85,7 +95,7 @@ class SolrOutput < Test::Unit::TestCase
     driver.emit(sample_record)
     driver.emit(sample_record.merge('age' => 27))
     driver.run
-    assert_equal(2, index_cmds.count)
+    assert_equal(2,  @index_cmds.count)
     assert_equal(26, @index_cmds[0]['age'])
     assert_equal(27, @index_cmds[1]['age'])
   end
@@ -94,7 +104,7 @@ class SolrOutput < Test::Unit::TestCase
     stub_solr
     driver.emit(sample_record)
     driver.run
-    assert_nil(index_cmds[0]['tag'])
+    assert_nil(@index_cmds[0]['tag'])
   end
 
   def test_adds_tag_key_when_configured
@@ -102,8 +112,34 @@ class SolrOutput < Test::Unit::TestCase
     stub_solr
     driver.emit(sample_record)
     driver.run
-    assert(index_cmds[0].key?('tag'))
-    assert_equal(index_cmds[0]['tag'], 'mytag')
+    assert(@index_cmds[0].key?('tag'))
+    assert_equal(@index_cmds[0]['tag'], 'mytag')
+  end
+
+  def test_use_utc
+    driver.configure("use_utc true\n")
+    stub_solr
+    Timecop.freeze(Time.local(2013, 12, 20, 19, 0, 0)) do
+      driver.emit(sample_record)
+    end
+    driver.run
+    assert_equal('2013-12-20T10:00:00Z', @index_cmds[0]['timestamp'])
+  end
+
+  def test_use_core_rotation
+    driver.configure("use_core_rotation true\n")
+    driver.configure("core_prefix log\n")
+    stub_solr('http://localhost:8983/solr/log-2013-12-20/update')
+    stub_solr2('http://localhost:8983/solr/log-2013-12-21/update')
+    Timecop.freeze(Time.local(2013, 12, 20, 19, 0, 0)) do
+      driver.emit(sample_record)
+    end
+    Timecop.freeze(Time.local(2013, 12, 21, 19, 0, 0)) do
+      driver.emit(sample_record)
+    end
+    driver.run
+    assert_equal(1,  @index_cmds.count)
+    assert_equal(1,  @index_cmds2.count)
   end
 
   def test_request_error

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-out-solr
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - diogo
@@ -54,6 +54,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Solr output plugin for Fluent event collector
 email:
 - team@uken.com