fluent-plugin-output-solr 0.4.13 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +96 -35
- data/fluent-plugin-output-solr.gemspec +14 -14
- data/fluent.conf +2 -8
- data/lib/fluent/plugin/out_solr.rb +82 -90
- metadata +20 -13
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 411fb997709bdee2ba7bdfacddd331dfacd82c0d
|
4
|
+
data.tar.gz: 481e8990f52260ad3d5ccabb03032ece66c7ad01
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e52a8fc04fd20bb336ddbcad6c7e12bf562180c52565736ba149334ba23afd61c8ceea989776d4ad2abb1d4fe8b37bcd0ac1bb4c989e5f251297e44b0328a776
|
7
|
+
data.tar.gz: 833671aa903c55254b03bf50ed4156f1e9b8a23e17027972a234991f14f2cc1176a6262777315aa4f7612ba7e296e9cff1b589cd01b625dac70a8a474bb24f69
|
data/README.md
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
# Fluent::Plugin::SolrOutput
|
2
2
|
|
3
|
-
This is a [Fluentd](http://fluentd.org/) output plugin for send data to [Apache Solr](http://lucene.apache.org/solr/).
|
3
|
+
This is a [Fluentd](http://fluentd.org/) output plugin for send data to [Apache Solr](http://lucene.apache.org/solr/). It support [SolrCloud](https://cwiki.apache.org/confluence/display/solr/SolrCloud) not only Standalone Solr.
|
4
4
|
|
5
5
|
## Requirements
|
6
6
|
|
7
|
-
| fluent-plugin-output-solr | fluentd | td-agent | ruby
|
8
|
-
| ------------------------- | --------------- | -------- |
|
9
|
-
| 1.x.x |
|
10
|
-
| 0.x.x | ~> 0.12.0 | 2 |
|
7
|
+
| fluent-plugin-output-solr | fluentd | td-agent | ruby |
|
8
|
+
| ------------------------- | --------------- | -------- | ------- |
|
9
|
+
| 1.x.x | \>= 0.14.0, < 2 | 3 | \>= 2.1 |
|
10
|
+
| 0.x.x | ~> 0.12.0 | 2 | \>= 1.9 |
|
11
11
|
|
12
12
|
* The 1.x.x series is developed from this branch (master)
|
13
13
|
* The 0.x.x series (compatible with fluentd v0.12, and td-agent 2) is developed on the [v0.x.x branch](https://github.com/mosuka/fluent-plugin-output-solr/tree/v0.x.x)
|
@@ -32,12 +32,12 @@ $ rake install
|
|
32
32
|
|
33
33
|
## Config parameters
|
34
34
|
|
35
|
-
###
|
35
|
+
### url
|
36
36
|
|
37
|
-
The Solr
|
37
|
+
The Solr server url (for example http://localhost:8983/solr/collection1).
|
38
38
|
|
39
39
|
```
|
40
|
-
|
40
|
+
url http://localhost:8983/solr/collection1
|
41
41
|
```
|
42
42
|
|
43
43
|
### zk_host
|
@@ -50,50 +50,42 @@ zk_host localhost:2181/solr
|
|
50
50
|
|
51
51
|
### collection
|
52
52
|
|
53
|
-
The
|
53
|
+
The SolrCloud collection name (default collection1).
|
54
54
|
|
55
55
|
```
|
56
56
|
collection collection1
|
57
57
|
```
|
58
58
|
|
59
|
-
###
|
60
|
-
|
61
|
-
Ignore undefined fields in the Solr schema.xml.
|
62
|
-
|
63
|
-
```
|
64
|
-
ignore_undefined_fields false
|
65
|
-
```
|
66
|
-
|
67
|
-
### tag_field
|
59
|
+
### defined_fields
|
68
60
|
|
69
|
-
|
61
|
+
The defined fields in the Solr schema.xml. If omitted, it will get fields via Solr Schema API.
|
70
62
|
|
71
63
|
```
|
72
|
-
|
64
|
+
defined_fields ["id", "title"]
|
73
65
|
```
|
74
66
|
|
75
|
-
###
|
67
|
+
### ignore_undefined_fields
|
76
68
|
|
77
|
-
|
69
|
+
Ignore undefined fields in the Solr schema.xml.
|
78
70
|
|
79
71
|
```
|
80
|
-
|
72
|
+
ignore_undefined_fields false
|
81
73
|
```
|
82
74
|
|
83
|
-
###
|
75
|
+
### unique_key_field
|
84
76
|
|
85
|
-
|
77
|
+
A field name of unique key in the Solr schema.xml. If omitted, it will get unique key via Solr Schema API.
|
86
78
|
|
87
79
|
```
|
88
|
-
|
80
|
+
unique_key_field id
|
89
81
|
```
|
90
82
|
|
91
|
-
###
|
83
|
+
### timestamp_field
|
92
84
|
|
93
|
-
|
85
|
+
A field name of event timestamp in the Solr schema.xml (default event_timestamp).
|
94
86
|
|
95
87
|
```
|
96
|
-
|
88
|
+
timestamp_field event_timestamp
|
97
89
|
```
|
98
90
|
|
99
91
|
### flush_size
|
@@ -119,11 +111,8 @@ commit_with_flush true
|
|
119
111
|
<match something.logs>
|
120
112
|
@type solr
|
121
113
|
|
122
|
-
# The Solr
|
123
|
-
|
124
|
-
|
125
|
-
# The Solr collection/core name (default collection1).
|
126
|
-
collection collection1
|
114
|
+
# The Solr server url (for example http://localhost:8983/solr/collection1).
|
115
|
+
url http://localhost:8983/solr/collection1
|
127
116
|
</match>
|
128
117
|
```
|
129
118
|
|
@@ -135,11 +124,83 @@ commit_with_flush true
|
|
135
124
|
# The ZooKeeper connection string that SolrCloud refers to (for example localhost:2181/solr).
|
136
125
|
zk_host localhost:2181/solr
|
137
126
|
|
138
|
-
# The
|
127
|
+
# The SolrCloud collection name (default collection1).
|
139
128
|
collection collection1
|
140
129
|
</match>
|
141
130
|
```
|
142
131
|
|
132
|
+
## Solr setup examples
|
133
|
+
|
134
|
+
### How to setup Standalone Solr using data-driven schemaless mode.
|
135
|
+
|
136
|
+
1.Download and install Solr
|
137
|
+
|
138
|
+
```sh
|
139
|
+
$ mkdir $HOME/solr
|
140
|
+
$ cd $HOME/solr
|
141
|
+
$ wget https://archive.apache.org/dist/lucene/solr/5.4.0/solr-5.4.0.tgz
|
142
|
+
$ tar zxvf solr-5.4.0.tgz
|
143
|
+
$ cd solr-5.4.0
|
144
|
+
```
|
145
|
+
|
146
|
+
2.Start standalone Solr
|
147
|
+
|
148
|
+
```sh
|
149
|
+
$ ./bin/solr start -p 8983 -s server/solr
|
150
|
+
```
|
151
|
+
|
152
|
+
3.Create core
|
153
|
+
|
154
|
+
```sh
|
155
|
+
$ ./bin/solr create -c collection1 -d server/solr/configsets/data_driven_schema_configs -n collection1_configs
|
156
|
+
```
|
157
|
+
|
158
|
+
### How to setup SolrCloud using data-driven schemaless mode (shards=1 and replicationfactor=2).
|
159
|
+
|
160
|
+
1.Download and install ZooKeeper
|
161
|
+
|
162
|
+
```sh
|
163
|
+
$ mkdir $HOME/zookeeper
|
164
|
+
$ cd $HOME/zookeeper
|
165
|
+
$ wget https://archive.apache.org/dist/zookeeper/zookeeper-3.4.6/zookeeper-3.4.6.tar.gz
|
166
|
+
$ tar zxvf zookeeper-3.4.6.tar.gz
|
167
|
+
$ cd zookeeper-3.4.6
|
168
|
+
$ cp -p ./conf/zoo_sample.cfg ./conf/zoo.cfg
|
169
|
+
```
|
170
|
+
|
171
|
+
2.Start standalone ZooKeeper
|
172
|
+
|
173
|
+
```sh
|
174
|
+
$ ./bin/zkServer.sh start
|
175
|
+
```
|
176
|
+
|
177
|
+
3.Download an install Solr
|
178
|
+
|
179
|
+
```sh
|
180
|
+
$ mkdir $HOME/solr
|
181
|
+
$ cd $HOME/solr
|
182
|
+
$ wget https://archive.apache.org/dist/lucene/solr/5.4.0/solr-5.4.0.tgz
|
183
|
+
$ tar zxvf solr-5.4.0.tgz
|
184
|
+
$ cd solr-5.4.0
|
185
|
+
$ ./server/scripts/cloud-scripts/zkcli.sh -zkhost localhost:2181 -cmd clear /solr
|
186
|
+
$ ./server/scripts/cloud-scripts/zkcli.sh -zkhost localhost:2181 -cmd makepath /solr
|
187
|
+
$ cp -pr server/solr server/solr1
|
188
|
+
$ cp -pr server/solr server/solr2
|
189
|
+
```
|
190
|
+
|
191
|
+
4.Start SolrCloud
|
192
|
+
|
193
|
+
```sh
|
194
|
+
$ ./bin/solr start -h localhost -p 8983 -z localhost:2181/solr -s server/solr1
|
195
|
+
$ ./bin/solr start -h localhost -p 8985 -z localhost:2181/solr -s server/solr2
|
196
|
+
```
|
197
|
+
|
198
|
+
5.Create collection
|
199
|
+
|
200
|
+
```sh
|
201
|
+
$ ./bin/solr create -c collection1 -d server/solr1/configsets/data_driven_schema_configs -n collection1_configs -shards 1 -replicationFactor 2
|
202
|
+
```
|
203
|
+
|
143
204
|
## Development
|
144
205
|
|
145
206
|
After checking out the repo, run `bundle install` to install dependencies. Then, run `rake test` to run the tests.
|
@@ -3,28 +3,28 @@ lib = File.expand_path('../lib', __FILE__)
|
|
3
3
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
4
|
|
5
5
|
Gem::Specification.new do |spec|
|
6
|
-
spec.name =
|
7
|
-
spec.version =
|
8
|
-
spec.authors = [
|
9
|
-
spec.email = [
|
6
|
+
spec.name = "fluent-plugin-output-solr"
|
7
|
+
spec.version = "1.0.0"
|
8
|
+
spec.authors = ["Minoru Osuka"]
|
9
|
+
spec.email = ["minoru.osuka@gmail.com"]
|
10
10
|
|
11
|
-
spec.summary =
|
12
|
-
spec.description =
|
13
|
-
spec.homepage =
|
11
|
+
spec.summary = "Fluent output plugin for sending data to Apache Solr."
|
12
|
+
spec.description = "Fluent output plugin for sending data to Apache Solr. It support SolrCloud not only Standalone Solr."
|
13
|
+
spec.homepage = "https://github.com/mosuka/fluent-plugin-output-solr"
|
14
14
|
|
15
|
-
spec.license =
|
15
|
+
spec.license = "Apache-2.0"
|
16
16
|
|
17
17
|
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
18
|
-
spec.bindir =
|
18
|
+
spec.bindir = "exe"
|
19
19
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
20
|
-
spec.require_paths = [
|
20
|
+
spec.require_paths = ["lib"]
|
21
21
|
|
22
|
-
spec.add_runtime_dependency 'fluentd', '
|
22
|
+
spec.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 2']
|
23
23
|
spec.add_runtime_dependency 'rsolr-cloud', '~> 1.1.0'
|
24
|
-
spec.add_runtime_dependency 'rsolr', '~> 1.
|
25
|
-
spec.add_runtime_dependency 'zk', '~> 1.9.
|
24
|
+
spec.add_runtime_dependency 'rsolr', '~> 1.0.12'
|
25
|
+
spec.add_runtime_dependency 'zk', '~> 1.9.5'
|
26
26
|
|
27
|
-
spec.add_development_dependency 'bundler', '~> 1.
|
27
|
+
spec.add_development_dependency 'bundler', '~> 1.14.6'
|
28
28
|
spec.add_development_dependency 'rake', '~> 11.1.2'
|
29
29
|
spec.add_development_dependency 'test-unit', '~> 3.1.5'
|
30
30
|
spec.add_development_dependency 'minitest', '~> 5.8.3'
|
data/fluent.conf
CHANGED
@@ -15,17 +15,11 @@
|
|
15
15
|
|
16
16
|
<match messages>
|
17
17
|
@type solr
|
18
|
-
|
19
|
-
|
18
|
+
# url http://localhost:8983/solr/collection1
|
19
|
+
zk_host localhost:2181/solr
|
20
20
|
collection collection1
|
21
21
|
ignore_undefined_fields false
|
22
|
-
tag_field tag
|
23
|
-
time_field time
|
24
|
-
time_format %FT%TZ
|
25
|
-
millisecond true
|
26
22
|
flush_size 100
|
27
|
-
commit_with_flush true
|
28
|
-
|
29
23
|
buffer_type memory
|
30
24
|
buffer_queue_limit 64m
|
31
25
|
buffer_chunk_limit 8m
|
@@ -2,54 +2,47 @@ require 'securerandom'
|
|
2
2
|
require 'rsolr'
|
3
3
|
require 'zk'
|
4
4
|
require 'rsolr/cloud'
|
5
|
+
require 'fluent/plugin/output'
|
5
6
|
|
6
|
-
module Fluent
|
7
|
-
class SolrOutput <
|
7
|
+
module Fluent::Plugin
|
8
|
+
class SolrOutput < Output
|
8
9
|
Fluent::Plugin.register_output('solr', self)
|
9
10
|
|
10
|
-
|
11
|
-
|
12
|
-
DEFAULT_TAG_FIELD = 'tag'
|
13
|
-
|
14
|
-
DEFAULT_TIME_FIELD = 'time'
|
15
|
-
DEFAULT_TIME_FORMAT = '%FT%TZ'
|
16
|
-
DEFAULT_MILLISECOND = false
|
11
|
+
helpers :inject, :compat_parameters
|
17
12
|
|
13
|
+
DEFAULT_COLLECTION = 'collection1'
|
18
14
|
DEFAULT_IGNORE_UNDEFINED_FIELDS = false
|
19
|
-
|
15
|
+
DEFAULT_TAG_FIELD = 'tag'
|
16
|
+
DEFAULT_TIMESTAMP_FIELD = 'event_timestamp'
|
20
17
|
DEFAULT_FLUSH_SIZE = 100
|
18
|
+
DEFAULT_BUFFER_TYPE = "memory"
|
21
19
|
DEFAULT_COMMIT_WITH_FLUSH = true
|
22
20
|
|
23
21
|
MODE_STANDALONE = 'Standalone'
|
24
22
|
MODE_SOLRCLOUD = 'SolrCloud'
|
25
23
|
|
26
|
-
include Fluent::SetTagKeyMixin
|
27
24
|
config_set_default :include_tag_key, false
|
28
|
-
|
29
|
-
include Fluent::SetTimeKeyMixin
|
30
25
|
config_set_default :include_time_key, false
|
31
26
|
|
32
|
-
config_param :
|
33
|
-
:desc => 'The Solr
|
27
|
+
config_param :url, :string, :default => nil,
|
28
|
+
:desc => 'The Solr server url (for example http://localhost:8983/solr/collection1).'
|
34
29
|
|
35
30
|
config_param :zk_host, :string, :default => nil,
|
36
31
|
:desc => 'The ZooKeeper connection string that SolrCloud refers to (for example localhost:2181/solr).'
|
37
|
-
|
38
32
|
config_param :collection, :string, :default => DEFAULT_COLLECTION,
|
39
|
-
:desc => 'The
|
33
|
+
:desc => 'The SolrCloud collection name (default collection1).'
|
40
34
|
|
35
|
+
config_param :defined_fields, :array, :default => nil,
|
36
|
+
:desc => 'The defined fields in the Solr schema.xml. If omitted, it will get fields via Solr Schema API.'
|
41
37
|
config_param :ignore_undefined_fields, :bool, :default => DEFAULT_IGNORE_UNDEFINED_FIELDS,
|
42
38
|
:desc => 'Ignore undefined fields in the Solr schema.xml.'
|
43
39
|
|
40
|
+
config_param :unique_key_field, :string, :default => nil,
|
41
|
+
:desc => 'A field name of unique key in the Solr schema.xml. If omitted, it will get unique key via Solr Schema API.'
|
44
42
|
config_param :tag_field, :string, :default => DEFAULT_TAG_FIELD,
|
45
|
-
:desc => 'A field name of fluentd tag in the Solr schema.xml (default
|
46
|
-
|
47
|
-
|
48
|
-
:desc => 'A field name of event timestamp in the Solr schema.xml (default time).'
|
49
|
-
config_param :time_format, :string, :default => DEFAULT_TIME_FORMAT,
|
50
|
-
:desc => 'The format of the time field (default %d/%b/%Y:%H:%M:%S %z).'
|
51
|
-
config_param :millisecond, :bool, :default => DEFAULT_MILLISECOND,
|
52
|
-
:desc => 'Output millisecond to Solr (default false).'
|
43
|
+
:desc => 'A field name of fluentd tag in the Solr schema.xml (default event_timestamp).'
|
44
|
+
config_param :timestamp_field, :string, :default => DEFAULT_TIMESTAMP_FIELD,
|
45
|
+
:desc => 'A field name of event timestamp in the Solr schema.xml (default event_timestamp).'
|
53
46
|
|
54
47
|
config_param :flush_size, :integer, :default => DEFAULT_FLUSH_SIZE,
|
55
48
|
:desc => 'A number of events to queue up before writing to Solr (default 100).'
|
@@ -57,11 +50,17 @@ module Fluent
|
|
57
50
|
config_param :commit_with_flush, :bool, :default => DEFAULT_COMMIT_WITH_FLUSH,
|
58
51
|
:desc => 'Send commit command to Solr with flush (default true).'
|
59
52
|
|
53
|
+
config_section :buffer do
|
54
|
+
config_set_default :@type, DEFAULT_BUFFER_TYPE
|
55
|
+
config_set_default :chunk_keys, ['tag']
|
56
|
+
end
|
57
|
+
|
60
58
|
def initialize
|
61
59
|
super
|
62
60
|
end
|
63
61
|
|
64
62
|
def configure(conf)
|
63
|
+
compat_parameters_convert(conf, :inject)
|
65
64
|
super
|
66
65
|
end
|
67
66
|
|
@@ -69,7 +68,7 @@ module Fluent
|
|
69
68
|
super
|
70
69
|
|
71
70
|
@mode = nil
|
72
|
-
if ! @
|
71
|
+
if ! @url.nil? then
|
73
72
|
@mode = MODE_STANDALONE
|
74
73
|
elsif ! @zk_host.nil?
|
75
74
|
@mode = MODE_SOLRCLOUD
|
@@ -79,7 +78,7 @@ module Fluent
|
|
79
78
|
@zk = nil
|
80
79
|
|
81
80
|
if @mode == MODE_STANDALONE then
|
82
|
-
@solr = RSolr.connect :url => @
|
81
|
+
@solr = RSolr.connect :url => @url
|
83
82
|
elsif @mode == MODE_SOLRCLOUD then
|
84
83
|
@zk = ZK.new(@zk_host)
|
85
84
|
cloud_connection = RSolr::Cloud::Connection.new(@zk)
|
@@ -96,120 +95,113 @@ module Fluent
|
|
96
95
|
end
|
97
96
|
|
98
97
|
def format(tag, time, record)
|
99
|
-
[
|
98
|
+
[time, record].to_msgpack
|
99
|
+
end
|
100
|
+
|
101
|
+
def formatted_to_msgpack_binary
|
102
|
+
true
|
103
|
+
end
|
104
|
+
|
105
|
+
def multi_workers_ready?
|
106
|
+
true
|
100
107
|
end
|
101
108
|
|
102
109
|
def write(chunk)
|
103
110
|
documents = []
|
104
111
|
|
105
|
-
|
106
|
-
|
112
|
+
@fields = @defined_fields.nil? ? get_fields : @defined_fields
|
113
|
+
@unique_key = @unique_key_field.nil? ? get_unique_key : @unique_key_field
|
114
|
+
tag = chunk.metadata.tag
|
115
|
+
chunk.msgpack_each do |time, record|
|
116
|
+
record = inject_values_to_record(tag, time, record)
|
107
117
|
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
chunk.msgpack_each do |tag, time, record|
|
112
|
-
# Set unique key and value
|
113
|
-
unless record.has_key?(unique_key) then
|
114
|
-
record.merge!({unique_key => SecureRandom.uuid})
|
118
|
+
unless record.has_key?(@unique_key) then
|
119
|
+
record.merge!({@unique_key => SecureRandom.uuid})
|
115
120
|
end
|
116
121
|
|
117
|
-
# Set Fluentd tag to Solr tag field
|
118
122
|
unless record.has_key?(@tag_field) then
|
119
123
|
record.merge!({@tag_field => tag})
|
120
124
|
end
|
121
125
|
|
122
|
-
|
123
|
-
tmp_time = Time.at(time).utc
|
124
|
-
if record.has_key?(@time_field) then
|
125
|
-
# Parsing the time field in the record by the specified format.
|
126
|
+
if record.has_key?(@timestamp_field) then
|
126
127
|
begin
|
127
|
-
|
128
|
-
|
129
|
-
|
128
|
+
event_timestamp_dt = DateTime.strptime(record[@timestamp_field], "%d/%b/%Y:%H:%M:%S %z").to_s
|
129
|
+
record.merge!({@timestamp_field => Time.parse(event_timestamp_dt.to_s).utc.strftime('%FT%TZ')})
|
130
|
+
rescue
|
131
|
+
record.merge!({@timestamp_field => Time.at(time).utc.strftime('%FT%TZ')})
|
130
132
|
end
|
131
|
-
end
|
132
|
-
if @millisecond then
|
133
|
-
record.merge!({@time_field => '%s.%03dZ' % [tmp_time.strftime('%FT%T'), tmp_time.usec / 1000.0]})
|
134
133
|
else
|
135
|
-
record.merge!({@
|
134
|
+
record.merge!({@timestamp_field => Time.at(time).utc.strftime('%FT%TZ')})
|
136
135
|
end
|
137
136
|
|
138
|
-
# Ignore undefined fields
|
139
137
|
if @ignore_undefined_fields then
|
140
138
|
record.each_key do |key|
|
141
|
-
unless fields.include?(key) then
|
139
|
+
unless @fields.include?(key) then
|
142
140
|
record.delete(key)
|
143
141
|
end
|
144
142
|
end
|
145
143
|
end
|
146
144
|
|
147
|
-
# Add record to documents
|
148
145
|
documents << record
|
149
146
|
|
150
|
-
# Update when flash size is reached
|
151
147
|
if documents.count >= @flush_size
|
152
148
|
update documents
|
153
149
|
documents.clear
|
154
150
|
end
|
155
151
|
end
|
156
152
|
|
157
|
-
# Update remaining documents
|
158
153
|
update documents unless documents.empty?
|
159
154
|
end
|
160
155
|
|
161
156
|
def update(documents)
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
log.debug "Sent #{documents.count} document(s) to Solr"
|
169
|
-
rescue Exception
|
170
|
-
log.warn "An error occurred while sending #{documents.count} document(s) to Solr"
|
157
|
+
if @mode == MODE_STANDALONE then
|
158
|
+
@solr.add documents, :params => {:commit => @commit_with_flush}
|
159
|
+
log.debug "Added %d document(s) to Solr" % documents.count
|
160
|
+
elsif @mode == MODE_SOLRCLOUD then
|
161
|
+
@solr.add documents, collection: @collection, :params => {:commit => @commit_with_flush}
|
162
|
+
log.debug "Update: Added %d document(s) to Solr" % documents.count
|
171
163
|
end
|
164
|
+
rescue Exception => e
|
165
|
+
log.warn "Update: An error occurred while indexing: #{e.message}"
|
172
166
|
end
|
173
167
|
|
174
168
|
def get_unique_key
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
response =
|
179
|
-
|
180
|
-
|
181
|
-
elsif @mode == MODE_SOLRCLOUD then
|
182
|
-
response = @solr.get 'schema/uniquekey', collection: @collection
|
183
|
-
end
|
184
|
-
unique_key = response['uniqueKey']
|
185
|
-
log.debug "Unique key: #{unique_key}"
|
186
|
-
rescue Exception
|
187
|
-
log.warn 'An error occurred while getting unique key'
|
169
|
+
response = nil
|
170
|
+
|
171
|
+
if @mode == MODE_STANDALONE then
|
172
|
+
response = @solr.get 'schema/uniquekey'
|
173
|
+
elsif @mode == MODE_SOLRCLOUD then
|
174
|
+
response = @solr.get 'schema/uniquekey', collection: @collection
|
188
175
|
end
|
189
176
|
|
177
|
+
unique_key = response['uniqueKey']
|
178
|
+
log.debug "Unique key: #{unique_key}"
|
179
|
+
|
190
180
|
return unique_key
|
181
|
+
|
182
|
+
rescue Exception => e
|
183
|
+
log.warn "Unique key: #{e.message}"
|
191
184
|
end
|
192
185
|
|
193
186
|
def get_fields
|
194
|
-
|
187
|
+
response = nil
|
195
188
|
|
196
|
-
|
197
|
-
response =
|
189
|
+
if @mode == MODE_STANDALONE then
|
190
|
+
response = @solr.get 'schema/fields'
|
191
|
+
elsif @mode == MODE_SOLRCLOUD then
|
192
|
+
response = @solr.get 'schema/fields', collection: @collection
|
193
|
+
end
|
198
194
|
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
response = @solr.get 'schema/fields', collection: @collection
|
203
|
-
end
|
204
|
-
response['fields'].each do |field|
|
205
|
-
fields.push(field['name'])
|
206
|
-
end
|
207
|
-
log.debug "Fields: #{fields}"
|
208
|
-
rescue Exception
|
209
|
-
log.warn 'An error occurred while getting fields'
|
195
|
+
fields = []
|
196
|
+
response['fields'].each do |field|
|
197
|
+
fields.push(field['name'])
|
210
198
|
end
|
199
|
+
log.debug "Fields: #{fields}"
|
211
200
|
|
212
201
|
return fields
|
202
|
+
|
203
|
+
rescue Exception => e
|
204
|
+
log.warn "Fields: #{e.message}"
|
213
205
|
end
|
214
206
|
end
|
215
207
|
end
|
metadata
CHANGED
@@ -1,29 +1,35 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-output-solr
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 1.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Minoru Osuka
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-03-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - "
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 0.14.0
|
20
|
+
- - "<"
|
18
21
|
- !ruby/object:Gem::Version
|
19
|
-
version:
|
22
|
+
version: '2'
|
20
23
|
type: :runtime
|
21
24
|
prerelease: false
|
22
25
|
version_requirements: !ruby/object:Gem::Requirement
|
23
26
|
requirements:
|
24
|
-
- - "
|
27
|
+
- - ">="
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: 0.14.0
|
30
|
+
- - "<"
|
25
31
|
- !ruby/object:Gem::Version
|
26
|
-
version:
|
32
|
+
version: '2'
|
27
33
|
- !ruby/object:Gem::Dependency
|
28
34
|
name: rsolr-cloud
|
29
35
|
requirement: !ruby/object:Gem::Requirement
|
@@ -44,42 +50,42 @@ dependencies:
|
|
44
50
|
requirements:
|
45
51
|
- - "~>"
|
46
52
|
- !ruby/object:Gem::Version
|
47
|
-
version: 1.
|
53
|
+
version: 1.0.12
|
48
54
|
type: :runtime
|
49
55
|
prerelease: false
|
50
56
|
version_requirements: !ruby/object:Gem::Requirement
|
51
57
|
requirements:
|
52
58
|
- - "~>"
|
53
59
|
- !ruby/object:Gem::Version
|
54
|
-
version: 1.
|
60
|
+
version: 1.0.12
|
55
61
|
- !ruby/object:Gem::Dependency
|
56
62
|
name: zk
|
57
63
|
requirement: !ruby/object:Gem::Requirement
|
58
64
|
requirements:
|
59
65
|
- - "~>"
|
60
66
|
- !ruby/object:Gem::Version
|
61
|
-
version: 1.9.
|
67
|
+
version: 1.9.5
|
62
68
|
type: :runtime
|
63
69
|
prerelease: false
|
64
70
|
version_requirements: !ruby/object:Gem::Requirement
|
65
71
|
requirements:
|
66
72
|
- - "~>"
|
67
73
|
- !ruby/object:Gem::Version
|
68
|
-
version: 1.9.
|
74
|
+
version: 1.9.5
|
69
75
|
- !ruby/object:Gem::Dependency
|
70
76
|
name: bundler
|
71
77
|
requirement: !ruby/object:Gem::Requirement
|
72
78
|
requirements:
|
73
79
|
- - "~>"
|
74
80
|
- !ruby/object:Gem::Version
|
75
|
-
version: 1.
|
81
|
+
version: 1.14.6
|
76
82
|
type: :development
|
77
83
|
prerelease: false
|
78
84
|
version_requirements: !ruby/object:Gem::Requirement
|
79
85
|
requirements:
|
80
86
|
- - "~>"
|
81
87
|
- !ruby/object:Gem::Version
|
82
|
-
version: 1.
|
88
|
+
version: 1.14.6
|
83
89
|
- !ruby/object:Gem::Dependency
|
84
90
|
name: rake
|
85
91
|
requirement: !ruby/object:Gem::Requirement
|
@@ -150,7 +156,8 @@ dependencies:
|
|
150
156
|
- - "~>"
|
151
157
|
- !ruby/object:Gem::Version
|
152
158
|
version: 1.1.8
|
153
|
-
description: Fluent output plugin for sending data to Apache Solr.
|
159
|
+
description: Fluent output plugin for sending data to Apache Solr. It support SolrCloud
|
160
|
+
not only Standalone Solr.
|
154
161
|
email:
|
155
162
|
- minoru.osuka@gmail.com
|
156
163
|
executables: []
|