fluent-plugin-out-solr 0.0.4 → 0.0.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +4 -2
- data/fluent-plugin-out-solr.gemspec +1 -1
- data/lib/fluent/plugin/out_solr.rb +3 -8
- data/lib/fluent/plugin/out_solr_time_sliced.rb +3 -9
- data/lib/fluent/plugin/solr_config_common.rb +17 -0
- data/lib/fluent/plugin/solr_util.rb +5 -1
- data/test/plugin/test_out_solr.rb +8 -0
- data/test/plugin/test_out_solr_time_sliced.rb +20 -2
- metadata +4 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 084cfbcff140583951af85f95686ce9b1cce218a
|
4
|
+
data.tar.gz: b76871e6609e4b256ae5a7c2e1657bf5a29c894f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f1bb5193a19f3689a040114fa4d515ad19f5b7cbc3779139632bbeadb57355b5b1db604c9d192d52131904ad30f3c63aa09ffa13722fee37498da4a8ba6ca2f8
|
7
|
+
data.tar.gz: 68b6b924c039f2b5c77f6254c5ee2c60907265a6974141b4f7d14cf1a1c2e205f724b88d7eb73aeb2530d1b5780ccf972cd767c9bd78dfe8fa0b039af3c93e41
|
data/README.md
CHANGED
@@ -30,7 +30,8 @@ Notice: no relationship with [btigit/fluent-plugin-solr](https://github.com/btig
|
|
30
30
|
include_tag_key true
|
31
31
|
tag_key tag
|
32
32
|
time_field timestamp
|
33
|
-
utc # if you do not want to use localtime
|
33
|
+
# utc # if you do not want to use localtime
|
34
|
+
# commit true # if you want to commit explicitly
|
34
35
|
|
35
36
|
flush_interval 3s
|
36
37
|
</match>
|
@@ -58,7 +59,8 @@ See: [Time Sliced Plugin Overview - Buffer Plugin Overview | Fluentd](http://doc
|
|
58
59
|
include_tag_key true
|
59
60
|
tag_key tag
|
60
61
|
time_field timestamp
|
61
|
-
utc # if you do not want to use localtime
|
62
|
+
# utc # if you do not want to use localtime
|
63
|
+
# commit true # if you want to commit explicitly
|
62
64
|
|
63
65
|
flush_interval 3s
|
64
66
|
</match>
|
@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = 'fluent-plugin-out-solr'
|
6
|
-
s.version = '0.0.
|
6
|
+
s.version = '0.0.5'
|
7
7
|
s.authors = ['diogo', 'pitr', 'haruyama']
|
8
8
|
s.email = ['team@uken.com', 'haruyama@unixuser.org']
|
9
9
|
s.description = %q{Solr output plugin for Fluent event collector}
|
@@ -6,16 +6,11 @@ class Fluent::SolrOutput < Fluent::BufferedOutput
|
|
6
6
|
|
7
7
|
require 'fluent/plugin/solr_util'
|
8
8
|
include SolrUtil
|
9
|
+
require 'fluent/plugin/solr_config_common'
|
10
|
+
include SolrConfigCommon
|
9
11
|
|
10
|
-
config_param :
|
11
|
-
config_param :port, :integer, default: 8983
|
12
|
-
config_param :core, :string, default: 'collection1'
|
13
|
-
config_param :time_field, :string, default: 'timestamp'
|
12
|
+
config_param :core, :string, default: 'collection1'
|
14
13
|
|
15
|
-
include Fluent::SetTagKeyMixin
|
16
|
-
config_set_default :include_tag_key, false
|
17
|
-
|
18
|
-
attr_accessor :localtime
|
19
14
|
def initialize
|
20
15
|
require 'net/http'
|
21
16
|
require 'uri'
|
@@ -6,19 +6,13 @@ class Fluent::SolrTimeSlicedOutput < Fluent::TimeSlicedOutput
|
|
6
6
|
|
7
7
|
require 'fluent/plugin/solr_util'
|
8
8
|
include SolrUtil
|
9
|
+
require 'fluent/plugin/solr_config_common'
|
10
|
+
include SolrConfigCommon
|
9
11
|
|
10
12
|
config_set_default :buffer_type, 'memory'
|
11
13
|
config_set_default :time_slice_format, '%Y%m%d'
|
12
14
|
|
13
|
-
config_param :
|
14
|
-
config_param :port, :integer, default: 8983
|
15
|
-
config_param :core, :string, default: 'log-%Y%m%d'
|
16
|
-
config_param :time_field, :string, default: 'timestamp'
|
17
|
-
|
18
|
-
include Fluent::SetTagKeyMixin
|
19
|
-
config_set_default :include_tag_key, false
|
20
|
-
|
21
|
-
attr_accessor :localtime
|
15
|
+
config_param :core, :string, default: 'log-%Y%m%d'
|
22
16
|
|
23
17
|
def initialize
|
24
18
|
require 'net/http'
|
@@ -0,0 +1,17 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
|
3
|
+
require 'fluent/config'
|
4
|
+
|
5
|
+
# Solr Config Common
|
6
|
+
module SolrConfigCommon
|
7
|
+
include Fluent::Configurable
|
8
|
+
config_param :host, :string, default: 'localhost'
|
9
|
+
config_param :port, :integer, default: 8983
|
10
|
+
config_param :time_field, :string, default: 'timestamp'
|
11
|
+
config_param :commit, :bool, default: false
|
12
|
+
|
13
|
+
include Fluent::SetTagKeyMixin
|
14
|
+
config_set_default :include_tag_key, false
|
15
|
+
|
16
|
+
attr_accessor :localtime
|
17
|
+
end
|
@@ -1,5 +1,7 @@
|
|
1
1
|
# encoding: UTF-8
|
2
2
|
|
3
|
+
require 'uri'
|
4
|
+
|
3
5
|
# Solr utility
|
4
6
|
module SolrUtil
|
5
7
|
def update_core(chunk, core)
|
@@ -14,7 +16,9 @@ module SolrUtil
|
|
14
16
|
end
|
15
17
|
|
16
18
|
http = Net::HTTP.new(@host, @port.to_i)
|
17
|
-
|
19
|
+
url = '/solr/' + URI.escape(core) + '/update'
|
20
|
+
url += '?commit=true' if @commit
|
21
|
+
request = Net::HTTP::Post.new(url, 'content-type' => 'application/json; charset=utf-8')
|
18
22
|
request.body = Yajl::Encoder.encode(documents)
|
19
23
|
http.request(request).value
|
20
24
|
end
|
@@ -123,4 +123,12 @@ class SolrOutput < Test::Unit::TestCase
|
|
123
123
|
driver.run
|
124
124
|
end
|
125
125
|
end
|
126
|
+
|
127
|
+
def test_writes_with_commit
|
128
|
+
driver.configure("commit true\n")
|
129
|
+
solr_request = stub_solr('http://localhost:8983/solr/collection1/update?commit=true')
|
130
|
+
driver.emit(sample_record)
|
131
|
+
driver.run
|
132
|
+
assert_requested(solr_request)
|
133
|
+
end
|
126
134
|
end
|
@@ -67,7 +67,7 @@ class SolrTimeSlicedOutputTest < Test::Unit::TestCase
|
|
67
67
|
|
68
68
|
def test_wrties_with_proper_content_type
|
69
69
|
stub_solr
|
70
|
-
driver.emit(sample_record)
|
70
|
+
driver.emit(sample_record, time)
|
71
71
|
driver.run
|
72
72
|
assert_equal('application/json; charset=utf-8', @content_type)
|
73
73
|
end
|
@@ -124,7 +124,7 @@ class SolrTimeSlicedOutputTest < Test::Unit::TestCase
|
|
124
124
|
|
125
125
|
def test_doesnt_add_tag_key_by_default
|
126
126
|
stub_solr
|
127
|
-
driver.emit(sample_record)
|
127
|
+
driver.emit(sample_record, time)
|
128
128
|
driver.run
|
129
129
|
assert_nil(@index_cmds[0]['tag'])
|
130
130
|
end
|
@@ -148,6 +148,16 @@ class SolrTimeSlicedOutputTest < Test::Unit::TestCase
|
|
148
148
|
assert_equal('2013-12-21T22:30:00Z', @index_cmds[0]['timestamp'])
|
149
149
|
end
|
150
150
|
|
151
|
+
def test_utc2
|
152
|
+
driver.configure("utc\n")
|
153
|
+
stub_solr2
|
154
|
+
ENV['TZ'] = 'Europe/Berlin'
|
155
|
+
driver.emit(sample_record, Time.local(2013, 12, 22, 7, 30, 0).to_i)
|
156
|
+
ENV['TZ'] = nil
|
157
|
+
driver.run
|
158
|
+
assert_equal('2013-12-22T06:30:00Z', @index_cmds2[0]['timestamp'])
|
159
|
+
end
|
160
|
+
|
151
161
|
def test_emit_records_on_different_days
|
152
162
|
stub_solr
|
153
163
|
stub_solr2
|
@@ -165,4 +175,12 @@ class SolrTimeSlicedOutputTest < Test::Unit::TestCase
|
|
165
175
|
driver.run
|
166
176
|
end
|
167
177
|
end
|
178
|
+
|
179
|
+
def test_writes_with_commit
|
180
|
+
driver.configure("commit true\n")
|
181
|
+
solr_request = stub_solr('http://localhost:8983/solr/log-20131221/update?commit=true')
|
182
|
+
driver.emit(sample_record, time)
|
183
|
+
driver.run
|
184
|
+
assert_requested(solr_request)
|
185
|
+
end
|
168
186
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-out-solr
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- diogo
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date:
|
13
|
+
date: 2014-01-02 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: fluentd
|
@@ -70,6 +70,7 @@ files:
|
|
70
70
|
- fluent-plugin-out-solr.gemspec
|
71
71
|
- lib/fluent/plugin/out_solr.rb
|
72
72
|
- lib/fluent/plugin/out_solr_time_sliced.rb
|
73
|
+
- lib/fluent/plugin/solr_config_common.rb
|
73
74
|
- lib/fluent/plugin/solr_util.rb
|
74
75
|
- test/helper.rb
|
75
76
|
- test/plugin/.rubocop.yml
|
@@ -95,7 +96,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
95
96
|
version: '0'
|
96
97
|
requirements: []
|
97
98
|
rubyforge_project:
|
98
|
-
rubygems_version: 2.
|
99
|
+
rubygems_version: 2.2.0
|
99
100
|
signing_key:
|
100
101
|
specification_version: 4
|
101
102
|
summary: Solr output plugin for Fluent event collector
|