fluent-plugin-out-solr 0.0.2 → 0.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +29 -0
- data/fluent-plugin-out-solr.gemspec +2 -1
- data/lib/fluent/plugin/out_solr.rb +22 -20
- data/test/plugin/test_out_solr.rb +44 -8
- metadata +15 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f134d6f7dcc1c67533c8bf19deb402cab474d64c
|
4
|
+
data.tar.gz: 8433e0d6e1b0bf02dabdf70f9ed592e1c622d6ec
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 84430230e21373715cb2b956021119992b488290f65584208850bc613bc4d67f689f1fe44eb357af6579265edc376aec10ada999380cc1f63a675475720285db
|
7
|
+
data.tar.gz: 5097d3937e3966989c3dea2ef74706ff9174b711a1a96727bab1a51b72143d9d9cfa29e4236d445e6b20ffba8a813a9d2aed96a1e003c32390af60b20890bbba
|
data/README.md
CHANGED
@@ -12,6 +12,8 @@ Notice: no relationship with [btigit/fluent-plugin-solr](https://github.com/btig
|
|
12
12
|
|
13
13
|
### fluent.conf snippet
|
14
14
|
|
15
|
+
#### single core
|
16
|
+
|
15
17
|
```
|
16
18
|
<source>
|
17
19
|
type tail
|
@@ -27,6 +29,33 @@ Notice: no relationship with [btigit/fluent-plugin-solr](https://github.com/btig
|
|
27
29
|
core collection1
|
28
30
|
include_tag_key true
|
29
31
|
tag_key tag
|
32
|
+
time_field timestamp
|
33
|
+
use_utc false
|
34
|
+
flush_interval 3s
|
35
|
+
</match>
|
36
|
+
```
|
37
|
+
|
38
|
+
#### core rotation by date
|
39
|
+
|
40
|
+
You should create cores in advance.
|
41
|
+
|
42
|
+
```
|
43
|
+
<source>
|
44
|
+
type tail
|
45
|
+
format apache
|
46
|
+
path /tmp/access.log
|
47
|
+
tag apache.access
|
48
|
+
</source>
|
49
|
+
|
50
|
+
<match apache.*>
|
51
|
+
type solr
|
52
|
+
host localhost
|
53
|
+
port 8983
|
54
|
+
use_core_rotation true
|
55
|
+
core_prefix apache
|
56
|
+
include_tag_key true
|
57
|
+
tag_key tag
|
58
|
+
time_field timestamp
|
30
59
|
use_utc false
|
31
60
|
flush_interval 3s
|
32
61
|
</match>
|
@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = 'fluent-plugin-out-solr'
|
6
|
-
s.version = '0.0.
|
6
|
+
s.version = '0.0.3'
|
7
7
|
s.authors = ['diogo', 'pitr', 'haruyama']
|
8
8
|
s.email = ['team@uken.com', 'haruyama@unixuser.org']
|
9
9
|
s.description = %q{Solr output plugin for Fluent event collector}
|
@@ -20,4 +20,5 @@ Gem::Specification.new do |s|
|
|
20
20
|
|
21
21
|
s.add_development_dependency 'rake'
|
22
22
|
s.add_development_dependency 'webmock'
|
23
|
+
s.add_development_dependency 'timecop'
|
23
24
|
end
|
@@ -6,11 +6,13 @@ require 'uri'
|
|
6
6
|
class Fluent::SolrOutput < Fluent::BufferedOutput
|
7
7
|
Fluent::Plugin.register_output('solr', self)
|
8
8
|
|
9
|
-
config_param :host,
|
10
|
-
config_param :port,
|
11
|
-
config_param :core,
|
12
|
-
config_param :
|
13
|
-
config_param :
|
9
|
+
config_param :host, :string, default: 'localhost'
|
10
|
+
config_param :port, :integer, default: 8983
|
11
|
+
config_param :core, :string, default: 'collection1'
|
12
|
+
config_param :use_core_rotation, :bool, default: false
|
13
|
+
config_param :core_prefix, :string, default: 'core'
|
14
|
+
config_param :time_field, :string, default: 'timestamp'
|
15
|
+
config_param :use_utc, :bool, default: false
|
14
16
|
|
15
17
|
include Fluent::SetTagKeyMixin
|
16
18
|
config_set_default :include_tag_key, false
|
@@ -36,25 +38,25 @@ class Fluent::SolrOutput < Fluent::BufferedOutput
|
|
36
38
|
end
|
37
39
|
|
38
40
|
def write(chunk)
|
39
|
-
|
41
|
+
bulk_messages = Hash.new { |h, k| h[k] = [] }
|
40
42
|
|
41
|
-
chunk.msgpack_each do |tag,
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
Time.at(time).utc.strftime('%FT%TZ')
|
46
|
-
else
|
47
|
-
Time.at(time).strftime('%FT%TZ')
|
48
|
-
end
|
49
|
-
|
50
|
-
record.merge!(@time_field => time_string)
|
43
|
+
chunk.msgpack_each do |tag, unixtime, record|
|
44
|
+
time = Time.at(unixtime)
|
45
|
+
time = time.utc if @use_utc
|
46
|
+
record.merge!(@time_field => time.strftime('%FT%TZ'))
|
51
47
|
record.merge!(@tag_key => tag) if @include_tag_key
|
52
|
-
|
48
|
+
if @use_core_rotation
|
49
|
+
bulk_messages[@core_prefix + '-' + time.strftime('%F')] << record
|
50
|
+
else
|
51
|
+
bulk_messages[@core] << record
|
52
|
+
end
|
53
53
|
end
|
54
54
|
|
55
55
|
http = Net::HTTP.new(@host, @port.to_i)
|
56
|
-
|
57
|
-
|
58
|
-
|
56
|
+
bulk_messages.each do |corename, messages|
|
57
|
+
request = Net::HTTP::Post.new('/solr/' + URI.escape(corename) + '/update', 'content-type' => 'application/json; charset=utf-8')
|
58
|
+
request.body = Yajl::Encoder.encode(messages)
|
59
|
+
http.request(request).value
|
60
|
+
end
|
59
61
|
end
|
60
62
|
end
|
@@ -4,7 +4,7 @@ require 'fluent/test'
|
|
4
4
|
require 'fluent/plugin/out_solr'
|
5
5
|
|
6
6
|
require 'webmock/test_unit'
|
7
|
-
require '
|
7
|
+
require 'timecop'
|
8
8
|
|
9
9
|
require 'helper'
|
10
10
|
|
@@ -16,6 +16,7 @@ WebMock.disable_net_connect!
|
|
16
16
|
# Solr output test
|
17
17
|
class SolrOutput < Test::Unit::TestCase
|
18
18
|
attr_accessor :index_cmds, :content_type
|
19
|
+
attr_accessor :index_cmds2
|
19
20
|
|
20
21
|
def setup
|
21
22
|
Fluent::Test.setup
|
@@ -37,16 +38,25 @@ class SolrOutput < Test::Unit::TestCase
|
|
37
38
|
end
|
38
39
|
end
|
39
40
|
|
41
|
+
def stub_solr2(url = 'http://localhost:8983/solr/collection1/update')
|
42
|
+
stub_request(:post, url).with do |req|
|
43
|
+
@index_cmds2 = JSON.parse(req.body)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
40
47
|
def stub_solr_unavailable(url = 'http://localhost:8983/solr/collection1/update')
|
41
48
|
stub_request(:post, url).to_return(status: [503, 'Service Unavailable'])
|
42
49
|
end
|
43
50
|
|
44
51
|
def test_writes_to_default_index
|
45
52
|
stub_solr
|
46
|
-
|
53
|
+
Timecop.freeze(Time.local(2013, 12, 20, 19, 0, 0)) do
|
54
|
+
driver.emit(sample_record)
|
55
|
+
end
|
47
56
|
driver.run
|
48
|
-
assert_equal(26,
|
49
|
-
assert_equal('42',
|
57
|
+
assert_equal(26, @index_cmds[0]['age'])
|
58
|
+
assert_equal('42', @index_cmds[0]['request_id'])
|
59
|
+
assert_equal('2013-12-20T19:00:00Z', @index_cmds[0]['timestamp'])
|
50
60
|
end
|
51
61
|
|
52
62
|
def test_wrties_with_proper_content_type
|
@@ -85,7 +95,7 @@ class SolrOutput < Test::Unit::TestCase
|
|
85
95
|
driver.emit(sample_record)
|
86
96
|
driver.emit(sample_record.merge('age' => 27))
|
87
97
|
driver.run
|
88
|
-
assert_equal(2,
|
98
|
+
assert_equal(2, @index_cmds.count)
|
89
99
|
assert_equal(26, @index_cmds[0]['age'])
|
90
100
|
assert_equal(27, @index_cmds[1]['age'])
|
91
101
|
end
|
@@ -94,7 +104,7 @@ class SolrOutput < Test::Unit::TestCase
|
|
94
104
|
stub_solr
|
95
105
|
driver.emit(sample_record)
|
96
106
|
driver.run
|
97
|
-
assert_nil(index_cmds[0]['tag'])
|
107
|
+
assert_nil(@index_cmds[0]['tag'])
|
98
108
|
end
|
99
109
|
|
100
110
|
def test_adds_tag_key_when_configured
|
@@ -102,8 +112,34 @@ class SolrOutput < Test::Unit::TestCase
|
|
102
112
|
stub_solr
|
103
113
|
driver.emit(sample_record)
|
104
114
|
driver.run
|
105
|
-
assert(index_cmds[0].key?('tag'))
|
106
|
-
assert_equal(index_cmds[0]['tag'], 'mytag')
|
115
|
+
assert(@index_cmds[0].key?('tag'))
|
116
|
+
assert_equal(@index_cmds[0]['tag'], 'mytag')
|
117
|
+
end
|
118
|
+
|
119
|
+
def test_use_utc
|
120
|
+
driver.configure("use_utc true\n")
|
121
|
+
stub_solr
|
122
|
+
Timecop.freeze(Time.local(2013, 12, 20, 19, 0, 0)) do
|
123
|
+
driver.emit(sample_record)
|
124
|
+
end
|
125
|
+
driver.run
|
126
|
+
assert_equal('2013-12-20T10:00:00Z', @index_cmds[0]['timestamp'])
|
127
|
+
end
|
128
|
+
|
129
|
+
def test_use_core_rotation
|
130
|
+
driver.configure("use_core_rotation true\n")
|
131
|
+
driver.configure("core_prefix log\n")
|
132
|
+
stub_solr('http://localhost:8983/solr/log-2013-12-20/update')
|
133
|
+
stub_solr2('http://localhost:8983/solr/log-2013-12-21/update')
|
134
|
+
Timecop.freeze(Time.local(2013, 12, 20, 19, 0, 0)) do
|
135
|
+
driver.emit(sample_record)
|
136
|
+
end
|
137
|
+
Timecop.freeze(Time.local(2013, 12, 21, 19, 0, 0)) do
|
138
|
+
driver.emit(sample_record)
|
139
|
+
end
|
140
|
+
driver.run
|
141
|
+
assert_equal(1, @index_cmds.count)
|
142
|
+
assert_equal(1, @index_cmds2.count)
|
107
143
|
end
|
108
144
|
|
109
145
|
def test_request_error
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-out-solr
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- diogo
|
@@ -54,6 +54,20 @@ dependencies:
|
|
54
54
|
- - '>='
|
55
55
|
- !ruby/object:Gem::Version
|
56
56
|
version: '0'
|
57
|
+
- !ruby/object:Gem::Dependency
|
58
|
+
name: timecop
|
59
|
+
requirement: !ruby/object:Gem::Requirement
|
60
|
+
requirements:
|
61
|
+
- - '>='
|
62
|
+
- !ruby/object:Gem::Version
|
63
|
+
version: '0'
|
64
|
+
type: :development
|
65
|
+
prerelease: false
|
66
|
+
version_requirements: !ruby/object:Gem::Requirement
|
67
|
+
requirements:
|
68
|
+
- - '>='
|
69
|
+
- !ruby/object:Gem::Version
|
70
|
+
version: '0'
|
57
71
|
description: Solr output plugin for Fluent event collector
|
58
72
|
email:
|
59
73
|
- team@uken.com
|