fluent-plugin-output-solr 0.4.13 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +96 -35
- data/fluent-plugin-output-solr.gemspec +14 -14
- data/fluent.conf +2 -8
- data/lib/fluent/plugin/out_solr.rb +82 -90
- metadata +20 -13
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 411fb997709bdee2ba7bdfacddd331dfacd82c0d
|
4
|
+
data.tar.gz: 481e8990f52260ad3d5ccabb03032ece66c7ad01
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e52a8fc04fd20bb336ddbcad6c7e12bf562180c52565736ba149334ba23afd61c8ceea989776d4ad2abb1d4fe8b37bcd0ac1bb4c989e5f251297e44b0328a776
|
7
|
+
data.tar.gz: 833671aa903c55254b03bf50ed4156f1e9b8a23e17027972a234991f14f2cc1176a6262777315aa4f7612ba7e296e9cff1b589cd01b625dac70a8a474bb24f69
|
data/README.md
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
# Fluent::Plugin::SolrOutput
|
2
2
|
|
3
|
-
This is a [Fluentd](http://fluentd.org/) output plugin for send data to [Apache Solr](http://lucene.apache.org/solr/).
|
3
|
+
This is a [Fluentd](http://fluentd.org/) output plugin for send data to [Apache Solr](http://lucene.apache.org/solr/). It support [SolrCloud](https://cwiki.apache.org/confluence/display/solr/SolrCloud) not only Standalone Solr.
|
4
4
|
|
5
5
|
## Requirements
|
6
6
|
|
7
|
-
| fluent-plugin-output-solr | fluentd | td-agent | ruby
|
8
|
-
| ------------------------- | --------------- | -------- |
|
9
|
-
| 1.x.x |
|
10
|
-
| 0.x.x | ~> 0.12.0 | 2 |
|
7
|
+
| fluent-plugin-output-solr | fluentd | td-agent | ruby |
|
8
|
+
| ------------------------- | --------------- | -------- | ------- |
|
9
|
+
| 1.x.x | \>= 0.14.0, < 2 | 3 | \>= 2.1 |
|
10
|
+
| 0.x.x | ~> 0.12.0 | 2 | \>= 1.9 |
|
11
11
|
|
12
12
|
* The 1.x.x series is developed from this branch (master)
|
13
13
|
* The 0.x.x series (compatible with fluentd v0.12, and td-agent 2) is developed on the [v0.x.x branch](https://github.com/mosuka/fluent-plugin-output-solr/tree/v0.x.x)
|
@@ -32,12 +32,12 @@ $ rake install
|
|
32
32
|
|
33
33
|
## Config parameters
|
34
34
|
|
35
|
-
###
|
35
|
+
### url
|
36
36
|
|
37
|
-
The Solr
|
37
|
+
The Solr server url (for example http://localhost:8983/solr/collection1).
|
38
38
|
|
39
39
|
```
|
40
|
-
|
40
|
+
url http://localhost:8983/solr/collection1
|
41
41
|
```
|
42
42
|
|
43
43
|
### zk_host
|
@@ -50,50 +50,42 @@ zk_host localhost:2181/solr
|
|
50
50
|
|
51
51
|
### collection
|
52
52
|
|
53
|
-
The
|
53
|
+
The SolrCloud collection name (default collection1).
|
54
54
|
|
55
55
|
```
|
56
56
|
collection collection1
|
57
57
|
```
|
58
58
|
|
59
|
-
###
|
60
|
-
|
61
|
-
Ignore undefined fields in the Solr schema.xml.
|
62
|
-
|
63
|
-
```
|
64
|
-
ignore_undefined_fields false
|
65
|
-
```
|
66
|
-
|
67
|
-
### tag_field
|
59
|
+
### defined_fields
|
68
60
|
|
69
|
-
|
61
|
+
The defined fields in the Solr schema.xml. If omitted, it will get fields via Solr Schema API.
|
70
62
|
|
71
63
|
```
|
72
|
-
|
64
|
+
defined_fields ["id", "title"]
|
73
65
|
```
|
74
66
|
|
75
|
-
###
|
67
|
+
### ignore_undefined_fields
|
76
68
|
|
77
|
-
|
69
|
+
Ignore undefined fields in the Solr schema.xml.
|
78
70
|
|
79
71
|
```
|
80
|
-
|
72
|
+
ignore_undefined_fields false
|
81
73
|
```
|
82
74
|
|
83
|
-
###
|
75
|
+
### unique_key_field
|
84
76
|
|
85
|
-
|
77
|
+
A field name of unique key in the Solr schema.xml. If omitted, it will get unique key via Solr Schema API.
|
86
78
|
|
87
79
|
```
|
88
|
-
|
80
|
+
unique_key_field id
|
89
81
|
```
|
90
82
|
|
91
|
-
###
|
83
|
+
### timestamp_field
|
92
84
|
|
93
|
-
|
85
|
+
A field name of event timestamp in the Solr schema.xml (default event_timestamp).
|
94
86
|
|
95
87
|
```
|
96
|
-
|
88
|
+
timestamp_field event_timestamp
|
97
89
|
```
|
98
90
|
|
99
91
|
### flush_size
|
@@ -119,11 +111,8 @@ commit_with_flush true
|
|
119
111
|
<match something.logs>
|
120
112
|
@type solr
|
121
113
|
|
122
|
-
# The Solr
|
123
|
-
|
124
|
-
|
125
|
-
# The Solr collection/core name (default collection1).
|
126
|
-
collection collection1
|
114
|
+
# The Solr server url (for example http://localhost:8983/solr/collection1).
|
115
|
+
url http://localhost:8983/solr/collection1
|
127
116
|
</match>
|
128
117
|
```
|
129
118
|
|
@@ -135,11 +124,83 @@ commit_with_flush true
|
|
135
124
|
# The ZooKeeper connection string that SolrCloud refers to (for example localhost:2181/solr).
|
136
125
|
zk_host localhost:2181/solr
|
137
126
|
|
138
|
-
# The
|
127
|
+
# The SolrCloud collection name (default collection1).
|
139
128
|
collection collection1
|
140
129
|
</match>
|
141
130
|
```
|
142
131
|
|
132
|
+
## Solr setup examples
|
133
|
+
|
134
|
+
### How to setup Standalone Solr using data-driven schemaless mode.
|
135
|
+
|
136
|
+
1.Download and install Solr
|
137
|
+
|
138
|
+
```sh
|
139
|
+
$ mkdir $HOME/solr
|
140
|
+
$ cd $HOME/solr
|
141
|
+
$ wget https://archive.apache.org/dist/lucene/solr/5.4.0/solr-5.4.0.tgz
|
142
|
+
$ tar zxvf solr-5.4.0.tgz
|
143
|
+
$ cd solr-5.4.0
|
144
|
+
```
|
145
|
+
|
146
|
+
2.Start standalone Solr
|
147
|
+
|
148
|
+
```sh
|
149
|
+
$ ./bin/solr start -p 8983 -s server/solr
|
150
|
+
```
|
151
|
+
|
152
|
+
3.Create core
|
153
|
+
|
154
|
+
```sh
|
155
|
+
$ ./bin/solr create -c collection1 -d server/solr/configsets/data_driven_schema_configs -n collection1_configs
|
156
|
+
```
|
157
|
+
|
158
|
+
### How to setup SolrCloud using data-driven schemaless mode (shards=1 and replicationfactor=2).
|
159
|
+
|
160
|
+
1.Download and install ZooKeeper
|
161
|
+
|
162
|
+
```sh
|
163
|
+
$ mkdir $HOME/zookeeper
|
164
|
+
$ cd $HOME/zookeeper
|
165
|
+
$ wget https://archive.apache.org/dist/zookeeper/zookeeper-3.4.6/zookeeper-3.4.6.tar.gz
|
166
|
+
$ tar zxvf zookeeper-3.4.6.tar.gz
|
167
|
+
$ cd zookeeper-3.4.6
|
168
|
+
$ cp -p ./conf/zoo_sample.cfg ./conf/zoo.cfg
|
169
|
+
```
|
170
|
+
|
171
|
+
2.Start standalone ZooKeeper
|
172
|
+
|
173
|
+
```sh
|
174
|
+
$ ./bin/zkServer.sh start
|
175
|
+
```
|
176
|
+
|
177
|
+
3.Download an install Solr
|
178
|
+
|
179
|
+
```sh
|
180
|
+
$ mkdir $HOME/solr
|
181
|
+
$ cd $HOME/solr
|
182
|
+
$ wget https://archive.apache.org/dist/lucene/solr/5.4.0/solr-5.4.0.tgz
|
183
|
+
$ tar zxvf solr-5.4.0.tgz
|
184
|
+
$ cd solr-5.4.0
|
185
|
+
$ ./server/scripts/cloud-scripts/zkcli.sh -zkhost localhost:2181 -cmd clear /solr
|
186
|
+
$ ./server/scripts/cloud-scripts/zkcli.sh -zkhost localhost:2181 -cmd makepath /solr
|
187
|
+
$ cp -pr server/solr server/solr1
|
188
|
+
$ cp -pr server/solr server/solr2
|
189
|
+
```
|
190
|
+
|
191
|
+
4.Start SolrCloud
|
192
|
+
|
193
|
+
```sh
|
194
|
+
$ ./bin/solr start -h localhost -p 8983 -z localhost:2181/solr -s server/solr1
|
195
|
+
$ ./bin/solr start -h localhost -p 8985 -z localhost:2181/solr -s server/solr2
|
196
|
+
```
|
197
|
+
|
198
|
+
5.Create collection
|
199
|
+
|
200
|
+
```sh
|
201
|
+
$ ./bin/solr create -c collection1 -d server/solr1/configsets/data_driven_schema_configs -n collection1_configs -shards 1 -replicationFactor 2
|
202
|
+
```
|
203
|
+
|
143
204
|
## Development
|
144
205
|
|
145
206
|
After checking out the repo, run `bundle install` to install dependencies. Then, run `rake test` to run the tests.
|
@@ -3,28 +3,28 @@ lib = File.expand_path('../lib', __FILE__)
|
|
3
3
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
4
|
|
5
5
|
Gem::Specification.new do |spec|
|
6
|
-
spec.name =
|
7
|
-
spec.version =
|
8
|
-
spec.authors = [
|
9
|
-
spec.email = [
|
6
|
+
spec.name = "fluent-plugin-output-solr"
|
7
|
+
spec.version = "1.0.0"
|
8
|
+
spec.authors = ["Minoru Osuka"]
|
9
|
+
spec.email = ["minoru.osuka@gmail.com"]
|
10
10
|
|
11
|
-
spec.summary =
|
12
|
-
spec.description =
|
13
|
-
spec.homepage =
|
11
|
+
spec.summary = "Fluent output plugin for sending data to Apache Solr."
|
12
|
+
spec.description = "Fluent output plugin for sending data to Apache Solr. It support SolrCloud not only Standalone Solr."
|
13
|
+
spec.homepage = "https://github.com/mosuka/fluent-plugin-output-solr"
|
14
14
|
|
15
|
-
spec.license =
|
15
|
+
spec.license = "Apache-2.0"
|
16
16
|
|
17
17
|
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
18
|
-
spec.bindir =
|
18
|
+
spec.bindir = "exe"
|
19
19
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
20
|
-
spec.require_paths = [
|
20
|
+
spec.require_paths = ["lib"]
|
21
21
|
|
22
|
-
spec.add_runtime_dependency 'fluentd', '
|
22
|
+
spec.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 2']
|
23
23
|
spec.add_runtime_dependency 'rsolr-cloud', '~> 1.1.0'
|
24
|
-
spec.add_runtime_dependency 'rsolr', '~> 1.
|
25
|
-
spec.add_runtime_dependency 'zk', '~> 1.9.
|
24
|
+
spec.add_runtime_dependency 'rsolr', '~> 1.0.12'
|
25
|
+
spec.add_runtime_dependency 'zk', '~> 1.9.5'
|
26
26
|
|
27
|
-
spec.add_development_dependency 'bundler', '~> 1.
|
27
|
+
spec.add_development_dependency 'bundler', '~> 1.14.6'
|
28
28
|
spec.add_development_dependency 'rake', '~> 11.1.2'
|
29
29
|
spec.add_development_dependency 'test-unit', '~> 3.1.5'
|
30
30
|
spec.add_development_dependency 'minitest', '~> 5.8.3'
|
data/fluent.conf
CHANGED
@@ -15,17 +15,11 @@
|
|
15
15
|
|
16
16
|
<match messages>
|
17
17
|
@type solr
|
18
|
-
|
19
|
-
|
18
|
+
# url http://localhost:8983/solr/collection1
|
19
|
+
zk_host localhost:2181/solr
|
20
20
|
collection collection1
|
21
21
|
ignore_undefined_fields false
|
22
|
-
tag_field tag
|
23
|
-
time_field time
|
24
|
-
time_format %FT%TZ
|
25
|
-
millisecond true
|
26
22
|
flush_size 100
|
27
|
-
commit_with_flush true
|
28
|
-
|
29
23
|
buffer_type memory
|
30
24
|
buffer_queue_limit 64m
|
31
25
|
buffer_chunk_limit 8m
|
@@ -2,54 +2,47 @@ require 'securerandom'
|
|
2
2
|
require 'rsolr'
|
3
3
|
require 'zk'
|
4
4
|
require 'rsolr/cloud'
|
5
|
+
require 'fluent/plugin/output'
|
5
6
|
|
6
|
-
module Fluent
|
7
|
-
class SolrOutput <
|
7
|
+
module Fluent::Plugin
|
8
|
+
class SolrOutput < Output
|
8
9
|
Fluent::Plugin.register_output('solr', self)
|
9
10
|
|
10
|
-
|
11
|
-
|
12
|
-
DEFAULT_TAG_FIELD = 'tag'
|
13
|
-
|
14
|
-
DEFAULT_TIME_FIELD = 'time'
|
15
|
-
DEFAULT_TIME_FORMAT = '%FT%TZ'
|
16
|
-
DEFAULT_MILLISECOND = false
|
11
|
+
helpers :inject, :compat_parameters
|
17
12
|
|
13
|
+
DEFAULT_COLLECTION = 'collection1'
|
18
14
|
DEFAULT_IGNORE_UNDEFINED_FIELDS = false
|
19
|
-
|
15
|
+
DEFAULT_TAG_FIELD = 'tag'
|
16
|
+
DEFAULT_TIMESTAMP_FIELD = 'event_timestamp'
|
20
17
|
DEFAULT_FLUSH_SIZE = 100
|
18
|
+
DEFAULT_BUFFER_TYPE = "memory"
|
21
19
|
DEFAULT_COMMIT_WITH_FLUSH = true
|
22
20
|
|
23
21
|
MODE_STANDALONE = 'Standalone'
|
24
22
|
MODE_SOLRCLOUD = 'SolrCloud'
|
25
23
|
|
26
|
-
include Fluent::SetTagKeyMixin
|
27
24
|
config_set_default :include_tag_key, false
|
28
|
-
|
29
|
-
include Fluent::SetTimeKeyMixin
|
30
25
|
config_set_default :include_time_key, false
|
31
26
|
|
32
|
-
config_param :
|
33
|
-
:desc => 'The Solr
|
27
|
+
config_param :url, :string, :default => nil,
|
28
|
+
:desc => 'The Solr server url (for example http://localhost:8983/solr/collection1).'
|
34
29
|
|
35
30
|
config_param :zk_host, :string, :default => nil,
|
36
31
|
:desc => 'The ZooKeeper connection string that SolrCloud refers to (for example localhost:2181/solr).'
|
37
|
-
|
38
32
|
config_param :collection, :string, :default => DEFAULT_COLLECTION,
|
39
|
-
:desc => 'The
|
33
|
+
:desc => 'The SolrCloud collection name (default collection1).'
|
40
34
|
|
35
|
+
config_param :defined_fields, :array, :default => nil,
|
36
|
+
:desc => 'The defined fields in the Solr schema.xml. If omitted, it will get fields via Solr Schema API.'
|
41
37
|
config_param :ignore_undefined_fields, :bool, :default => DEFAULT_IGNORE_UNDEFINED_FIELDS,
|
42
38
|
:desc => 'Ignore undefined fields in the Solr schema.xml.'
|
43
39
|
|
40
|
+
config_param :unique_key_field, :string, :default => nil,
|
41
|
+
:desc => 'A field name of unique key in the Solr schema.xml. If omitted, it will get unique key via Solr Schema API.'
|
44
42
|
config_param :tag_field, :string, :default => DEFAULT_TAG_FIELD,
|
45
|
-
:desc => 'A field name of fluentd tag in the Solr schema.xml (default
|
46
|
-
|
47
|
-
|
48
|
-
:desc => 'A field name of event timestamp in the Solr schema.xml (default time).'
|
49
|
-
config_param :time_format, :string, :default => DEFAULT_TIME_FORMAT,
|
50
|
-
:desc => 'The format of the time field (default %d/%b/%Y:%H:%M:%S %z).'
|
51
|
-
config_param :millisecond, :bool, :default => DEFAULT_MILLISECOND,
|
52
|
-
:desc => 'Output millisecond to Solr (default false).'
|
43
|
+
:desc => 'A field name of fluentd tag in the Solr schema.xml (default event_timestamp).'
|
44
|
+
config_param :timestamp_field, :string, :default => DEFAULT_TIMESTAMP_FIELD,
|
45
|
+
:desc => 'A field name of event timestamp in the Solr schema.xml (default event_timestamp).'
|
53
46
|
|
54
47
|
config_param :flush_size, :integer, :default => DEFAULT_FLUSH_SIZE,
|
55
48
|
:desc => 'A number of events to queue up before writing to Solr (default 100).'
|
@@ -57,11 +50,17 @@ module Fluent
|
|
57
50
|
config_param :commit_with_flush, :bool, :default => DEFAULT_COMMIT_WITH_FLUSH,
|
58
51
|
:desc => 'Send commit command to Solr with flush (default true).'
|
59
52
|
|
53
|
+
config_section :buffer do
|
54
|
+
config_set_default :@type, DEFAULT_BUFFER_TYPE
|
55
|
+
config_set_default :chunk_keys, ['tag']
|
56
|
+
end
|
57
|
+
|
60
58
|
def initialize
|
61
59
|
super
|
62
60
|
end
|
63
61
|
|
64
62
|
def configure(conf)
|
63
|
+
compat_parameters_convert(conf, :inject)
|
65
64
|
super
|
66
65
|
end
|
67
66
|
|
@@ -69,7 +68,7 @@ module Fluent
|
|
69
68
|
super
|
70
69
|
|
71
70
|
@mode = nil
|
72
|
-
if ! @
|
71
|
+
if ! @url.nil? then
|
73
72
|
@mode = MODE_STANDALONE
|
74
73
|
elsif ! @zk_host.nil?
|
75
74
|
@mode = MODE_SOLRCLOUD
|
@@ -79,7 +78,7 @@ module Fluent
|
|
79
78
|
@zk = nil
|
80
79
|
|
81
80
|
if @mode == MODE_STANDALONE then
|
82
|
-
@solr = RSolr.connect :url => @
|
81
|
+
@solr = RSolr.connect :url => @url
|
83
82
|
elsif @mode == MODE_SOLRCLOUD then
|
84
83
|
@zk = ZK.new(@zk_host)
|
85
84
|
cloud_connection = RSolr::Cloud::Connection.new(@zk)
|
@@ -96,120 +95,113 @@ module Fluent
|
|
96
95
|
end
|
97
96
|
|
98
97
|
def format(tag, time, record)
|
99
|
-
[
|
98
|
+
[time, record].to_msgpack
|
99
|
+
end
|
100
|
+
|
101
|
+
def formatted_to_msgpack_binary
|
102
|
+
true
|
103
|
+
end
|
104
|
+
|
105
|
+
def multi_workers_ready?
|
106
|
+
true
|
100
107
|
end
|
101
108
|
|
102
109
|
def write(chunk)
|
103
110
|
documents = []
|
104
111
|
|
105
|
-
|
106
|
-
|
112
|
+
@fields = @defined_fields.nil? ? get_fields : @defined_fields
|
113
|
+
@unique_key = @unique_key_field.nil? ? get_unique_key : @unique_key_field
|
114
|
+
tag = chunk.metadata.tag
|
115
|
+
chunk.msgpack_each do |time, record|
|
116
|
+
record = inject_values_to_record(tag, time, record)
|
107
117
|
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
chunk.msgpack_each do |tag, time, record|
|
112
|
-
# Set unique key and value
|
113
|
-
unless record.has_key?(unique_key) then
|
114
|
-
record.merge!({unique_key => SecureRandom.uuid})
|
118
|
+
unless record.has_key?(@unique_key) then
|
119
|
+
record.merge!({@unique_key => SecureRandom.uuid})
|
115
120
|
end
|
116
121
|
|
117
|
-
# Set Fluentd tag to Solr tag field
|
118
122
|
unless record.has_key?(@tag_field) then
|
119
123
|
record.merge!({@tag_field => tag})
|
120
124
|
end
|
121
125
|
|
122
|
-
|
123
|
-
tmp_time = Time.at(time).utc
|
124
|
-
if record.has_key?(@time_field) then
|
125
|
-
# Parsing the time field in the record by the specified format.
|
126
|
+
if record.has_key?(@timestamp_field) then
|
126
127
|
begin
|
127
|
-
|
128
|
-
|
129
|
-
|
128
|
+
event_timestamp_dt = DateTime.strptime(record[@timestamp_field], "%d/%b/%Y:%H:%M:%S %z").to_s
|
129
|
+
record.merge!({@timestamp_field => Time.parse(event_timestamp_dt.to_s).utc.strftime('%FT%TZ')})
|
130
|
+
rescue
|
131
|
+
record.merge!({@timestamp_field => Time.at(time).utc.strftime('%FT%TZ')})
|
130
132
|
end
|
131
|
-
end
|
132
|
-
if @millisecond then
|
133
|
-
record.merge!({@time_field => '%s.%03dZ' % [tmp_time.strftime('%FT%T'), tmp_time.usec / 1000.0]})
|
134
133
|
else
|
135
|
-
record.merge!({@
|
134
|
+
record.merge!({@timestamp_field => Time.at(time).utc.strftime('%FT%TZ')})
|
136
135
|
end
|
137
136
|
|
138
|
-
# Ignore undefined fields
|
139
137
|
if @ignore_undefined_fields then
|
140
138
|
record.each_key do |key|
|
141
|
-
unless fields.include?(key) then
|
139
|
+
unless @fields.include?(key) then
|
142
140
|
record.delete(key)
|
143
141
|
end
|
144
142
|
end
|
145
143
|
end
|
146
144
|
|
147
|
-
# Add record to documents
|
148
145
|
documents << record
|
149
146
|
|
150
|
-
# Update when flash size is reached
|
151
147
|
if documents.count >= @flush_size
|
152
148
|
update documents
|
153
149
|
documents.clear
|
154
150
|
end
|
155
151
|
end
|
156
152
|
|
157
|
-
# Update remaining documents
|
158
153
|
update documents unless documents.empty?
|
159
154
|
end
|
160
155
|
|
161
156
|
def update(documents)
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
log.debug "Sent #{documents.count} document(s) to Solr"
|
169
|
-
rescue Exception
|
170
|
-
log.warn "An error occurred while sending #{documents.count} document(s) to Solr"
|
157
|
+
if @mode == MODE_STANDALONE then
|
158
|
+
@solr.add documents, :params => {:commit => @commit_with_flush}
|
159
|
+
log.debug "Added %d document(s) to Solr" % documents.count
|
160
|
+
elsif @mode == MODE_SOLRCLOUD then
|
161
|
+
@solr.add documents, collection: @collection, :params => {:commit => @commit_with_flush}
|
162
|
+
log.debug "Update: Added %d document(s) to Solr" % documents.count
|
171
163
|
end
|
164
|
+
rescue Exception => e
|
165
|
+
log.warn "Update: An error occurred while indexing: #{e.message}"
|
172
166
|
end
|
173
167
|
|
174
168
|
def get_unique_key
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
response =
|
179
|
-
|
180
|
-
|
181
|
-
elsif @mode == MODE_SOLRCLOUD then
|
182
|
-
response = @solr.get 'schema/uniquekey', collection: @collection
|
183
|
-
end
|
184
|
-
unique_key = response['uniqueKey']
|
185
|
-
log.debug "Unique key: #{unique_key}"
|
186
|
-
rescue Exception
|
187
|
-
log.warn 'An error occurred while getting unique key'
|
169
|
+
response = nil
|
170
|
+
|
171
|
+
if @mode == MODE_STANDALONE then
|
172
|
+
response = @solr.get 'schema/uniquekey'
|
173
|
+
elsif @mode == MODE_SOLRCLOUD then
|
174
|
+
response = @solr.get 'schema/uniquekey', collection: @collection
|
188
175
|
end
|
189
176
|
|
177
|
+
unique_key = response['uniqueKey']
|
178
|
+
log.debug "Unique key: #{unique_key}"
|
179
|
+
|
190
180
|
return unique_key
|
181
|
+
|
182
|
+
rescue Exception => e
|
183
|
+
log.warn "Unique key: #{e.message}"
|
191
184
|
end
|
192
185
|
|
193
186
|
def get_fields
|
194
|
-
|
187
|
+
response = nil
|
195
188
|
|
196
|
-
|
197
|
-
response =
|
189
|
+
if @mode == MODE_STANDALONE then
|
190
|
+
response = @solr.get 'schema/fields'
|
191
|
+
elsif @mode == MODE_SOLRCLOUD then
|
192
|
+
response = @solr.get 'schema/fields', collection: @collection
|
193
|
+
end
|
198
194
|
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
response = @solr.get 'schema/fields', collection: @collection
|
203
|
-
end
|
204
|
-
response['fields'].each do |field|
|
205
|
-
fields.push(field['name'])
|
206
|
-
end
|
207
|
-
log.debug "Fields: #{fields}"
|
208
|
-
rescue Exception
|
209
|
-
log.warn 'An error occurred while getting fields'
|
195
|
+
fields = []
|
196
|
+
response['fields'].each do |field|
|
197
|
+
fields.push(field['name'])
|
210
198
|
end
|
199
|
+
log.debug "Fields: #{fields}"
|
211
200
|
|
212
201
|
return fields
|
202
|
+
|
203
|
+
rescue Exception => e
|
204
|
+
log.warn "Fields: #{e.message}"
|
213
205
|
end
|
214
206
|
end
|
215
207
|
end
|
metadata
CHANGED
@@ -1,29 +1,35 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-output-solr
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 1.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Minoru Osuka
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-03-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - "
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 0.14.0
|
20
|
+
- - "<"
|
18
21
|
- !ruby/object:Gem::Version
|
19
|
-
version:
|
22
|
+
version: '2'
|
20
23
|
type: :runtime
|
21
24
|
prerelease: false
|
22
25
|
version_requirements: !ruby/object:Gem::Requirement
|
23
26
|
requirements:
|
24
|
-
- - "
|
27
|
+
- - ">="
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: 0.14.0
|
30
|
+
- - "<"
|
25
31
|
- !ruby/object:Gem::Version
|
26
|
-
version:
|
32
|
+
version: '2'
|
27
33
|
- !ruby/object:Gem::Dependency
|
28
34
|
name: rsolr-cloud
|
29
35
|
requirement: !ruby/object:Gem::Requirement
|
@@ -44,42 +50,42 @@ dependencies:
|
|
44
50
|
requirements:
|
45
51
|
- - "~>"
|
46
52
|
- !ruby/object:Gem::Version
|
47
|
-
version: 1.
|
53
|
+
version: 1.0.12
|
48
54
|
type: :runtime
|
49
55
|
prerelease: false
|
50
56
|
version_requirements: !ruby/object:Gem::Requirement
|
51
57
|
requirements:
|
52
58
|
- - "~>"
|
53
59
|
- !ruby/object:Gem::Version
|
54
|
-
version: 1.
|
60
|
+
version: 1.0.12
|
55
61
|
- !ruby/object:Gem::Dependency
|
56
62
|
name: zk
|
57
63
|
requirement: !ruby/object:Gem::Requirement
|
58
64
|
requirements:
|
59
65
|
- - "~>"
|
60
66
|
- !ruby/object:Gem::Version
|
61
|
-
version: 1.9.
|
67
|
+
version: 1.9.5
|
62
68
|
type: :runtime
|
63
69
|
prerelease: false
|
64
70
|
version_requirements: !ruby/object:Gem::Requirement
|
65
71
|
requirements:
|
66
72
|
- - "~>"
|
67
73
|
- !ruby/object:Gem::Version
|
68
|
-
version: 1.9.
|
74
|
+
version: 1.9.5
|
69
75
|
- !ruby/object:Gem::Dependency
|
70
76
|
name: bundler
|
71
77
|
requirement: !ruby/object:Gem::Requirement
|
72
78
|
requirements:
|
73
79
|
- - "~>"
|
74
80
|
- !ruby/object:Gem::Version
|
75
|
-
version: 1.
|
81
|
+
version: 1.14.6
|
76
82
|
type: :development
|
77
83
|
prerelease: false
|
78
84
|
version_requirements: !ruby/object:Gem::Requirement
|
79
85
|
requirements:
|
80
86
|
- - "~>"
|
81
87
|
- !ruby/object:Gem::Version
|
82
|
-
version: 1.
|
88
|
+
version: 1.14.6
|
83
89
|
- !ruby/object:Gem::Dependency
|
84
90
|
name: rake
|
85
91
|
requirement: !ruby/object:Gem::Requirement
|
@@ -150,7 +156,8 @@ dependencies:
|
|
150
156
|
- - "~>"
|
151
157
|
- !ruby/object:Gem::Version
|
152
158
|
version: 1.1.8
|
153
|
-
description: Fluent output plugin for sending data to Apache Solr.
|
159
|
+
description: Fluent output plugin for sending data to Apache Solr. It support SolrCloud
|
160
|
+
not only Standalone Solr.
|
154
161
|
email:
|
155
162
|
- minoru.osuka@gmail.com
|
156
163
|
executables: []
|