logstash-output-custom-solr 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/CHANGELOG.md +11 -0
- data/CONTRIBUTORS +10 -0
- data/DEVELOPER.md +2 -0
- data/Gemfile +2 -0
- data/LICENSE +13 -0
- data/README.md +249 -0
- data/lib/logstash/outputs/solr.rb +167 -0
- data/logstash-output-custom-solr.gemspec +31 -0
- data/spec/files/collections/collection1/state.json +14 -0
- data/spec/outputs/solr_spec.rb +229 -0
- metadata +190 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 71fad4671bc6f96e8852ad45c9abcf69c5cedc76b1acf2901b0817889ec5029f
|
4
|
+
data.tar.gz: e7370ff6cea53d407d8272e07c6349cf25913080d2bff4fa36368c2b69254118
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 9c5cfac43debf01dddb3e80bed0e09020509f25e63c35643d705238a153a64c61187bb0467c92427920ac1f127484e0431b2e6fc756e90c5189ccb1bafa3632a
|
7
|
+
data.tar.gz: 76420cb872fbe00a2a18d587206e4f738967c39fd2e52b92779304cf8dfa45b47b160ea90143779fbe264cb02d73ca169d448f660f07675bd2ea58000f5c2fa2
|
data/CHANGELOG.md
ADDED
data/CONTRIBUTORS
ADDED
@@ -0,0 +1,10 @@
|
|
1
|
+
The following is a list of people who have contributed ideas, code, bug
|
2
|
+
reports, or in general have helped logstash along its way.
|
3
|
+
|
4
|
+
Contributors:
|
5
|
+
* Minoru Osuka (minoru.osuka@gmail.com)
|
6
|
+
|
7
|
+
Note: If you've sent us patches, bug reports, or otherwise contributed to
|
8
|
+
Logstash, and you aren't on the list above and want to be, please let us know
|
9
|
+
and we'll make sure you're here. Contributions from folks like you are what make
|
10
|
+
open source awesome.
|
data/DEVELOPER.md
ADDED
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
Copyright (c) 2012–2015 Elasticsearch <http://www.elastic.co>
|
2
|
+
|
3
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
you may not use this file except in compliance with the License.
|
5
|
+
You may obtain a copy of the License at
|
6
|
+
|
7
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
|
9
|
+
Unless required by applicable law or agreed to in writing, software
|
10
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
See the License for the specific language governing permissions and
|
13
|
+
limitations under the License.
|
data/README.md
ADDED
@@ -0,0 +1,249 @@
|
|
1
|
+
# Logstash Plugin
|
2
|
+
|
3
|
+
This is a plugin for [Logstash](https://github.com/elastic/logstash). It support [SolrCloud](https://cwiki.apache.org/confluence/display/solr/SolrCloud) not only Standalone Solr.
|
4
|
+
|
5
|
+
It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
|
6
|
+
|
7
|
+
## Config parameters
|
8
|
+
|
9
|
+
### url
|
10
|
+
|
11
|
+
The Solr server url (for example http://localhost:8983/solr/collection1).
|
12
|
+
|
13
|
+
```
|
14
|
+
url http://localhost:8983/solr/collection1
|
15
|
+
```
|
16
|
+
|
17
|
+
### zk_host
|
18
|
+
|
19
|
+
The ZooKeeper connection string that SolrCloud refers to (for example localhost:2181/solr).
|
20
|
+
|
21
|
+
```
|
22
|
+
zk_host localhost:2181/solr
|
23
|
+
```
|
24
|
+
|
25
|
+
### collection
|
26
|
+
|
27
|
+
The SolrCloud collection name (default collection1).
|
28
|
+
|
29
|
+
```
|
30
|
+
collection collection1
|
31
|
+
```
|
32
|
+
|
33
|
+
### defined_fields
|
34
|
+
|
35
|
+
The defined fields in the Solr schema.xml. If omitted, it will get fields via Solr Schema API.
|
36
|
+
|
37
|
+
```
|
38
|
+
defined_fields ["id", "title"]
|
39
|
+
```
|
40
|
+
|
41
|
+
### ignore_undefined_fields
|
42
|
+
|
43
|
+
Ignore undefined fields in the Solr schema.xml.
|
44
|
+
|
45
|
+
```
|
46
|
+
ignore_undefined_fields false
|
47
|
+
```
|
48
|
+
|
49
|
+
### unique_key_field
|
50
|
+
|
51
|
+
A field name of unique key in the Solr schema.xml. If omitted, it will get unique key via Solr Schema API.
|
52
|
+
|
53
|
+
```
|
54
|
+
unique_key_field id
|
55
|
+
```
|
56
|
+
|
57
|
+
### timestamp_field
|
58
|
+
|
59
|
+
A field name of event timestamp in the Solr schema.xml (default event_timestamp).
|
60
|
+
|
61
|
+
```
|
62
|
+
timestamp_field event_timestamp
|
63
|
+
```
|
64
|
+
|
65
|
+
### flush_size
|
66
|
+
|
67
|
+
A number of events to queue up before writing to Solr (default 100).
|
68
|
+
|
69
|
+
```
|
70
|
+
flush_size 100
|
71
|
+
```
|
72
|
+
|
73
|
+
## Plugin setup examples
|
74
|
+
|
75
|
+
### Sent to standalone Solr using data-driven schemaless mode.
|
76
|
+
```
|
77
|
+
output {
|
78
|
+
solr {
|
79
|
+
url => "http://localhost:8983/solr/collection1"
|
80
|
+
}
|
81
|
+
}
|
82
|
+
|
83
|
+
```
|
84
|
+
|
85
|
+
### Sent to SolrCloud using data-driven schemaless mode.
|
86
|
+
```
|
87
|
+
output {
|
88
|
+
solr {
|
89
|
+
zk_host => "localhost:2181/solr"
|
90
|
+
collection => "collection1"
|
91
|
+
}
|
92
|
+
}
|
93
|
+
```
|
94
|
+
|
95
|
+
## Solr setup examples
|
96
|
+
|
97
|
+
### How to setup Standalone Solr using data-driven schemaless mode.
|
98
|
+
|
99
|
+
1.Download and install Solr
|
100
|
+
|
101
|
+
```sh
|
102
|
+
$ mkdir $HOME/solr
|
103
|
+
$ cd $HOME/solr
|
104
|
+
$ wget https://archive.apache.org/dist/lucene/solr/5.4.0/solr-5.4.0.tgz
|
105
|
+
$ tar zxvf solr-5.4.0.tgz
|
106
|
+
$ cd solr-5.4.0
|
107
|
+
```
|
108
|
+
|
109
|
+
2.Start standalone Solr
|
110
|
+
|
111
|
+
```sh
|
112
|
+
$ ./bin/solr start -p 8983 -s server/solr
|
113
|
+
```
|
114
|
+
|
115
|
+
3.Create core
|
116
|
+
|
117
|
+
```sh
|
118
|
+
$ ./bin/solr create -c collection1 -d server/solr/configsets/data_driven_schema_configs -n collection1_configs
|
119
|
+
```
|
120
|
+
|
121
|
+
### How to setup SolrCloud using data-driven schemaless mode (shards=1 and replicationfactor=2).
|
122
|
+
|
123
|
+
1.Download and install ZooKeeper
|
124
|
+
|
125
|
+
```sh
|
126
|
+
$ mkdir $HOME/zookeeper
|
127
|
+
$ cd $HOME/zookeeper
|
128
|
+
$ wget https://archive.apache.org/dist/zookeeper/zookeeper-3.4.6/zookeeper-3.4.6.tar.gz
|
129
|
+
$ tar zxvf zookeeper-3.4.6.tar.gz
|
130
|
+
$ cd zookeeper-3.4.6
|
131
|
+
$ cp -p ./conf/zoo_sample.cfg ./conf/zoo.cfg
|
132
|
+
```
|
133
|
+
|
134
|
+
2.Start standalone ZooKeeper
|
135
|
+
|
136
|
+
```sh
|
137
|
+
$ ./bin/zkServer.sh start
|
138
|
+
```
|
139
|
+
|
140
|
+
3.Download an install Solr
|
141
|
+
|
142
|
+
```sh
|
143
|
+
$ mkdir $HOME/solr
|
144
|
+
$ cd $HOME/solr
|
145
|
+
$ wget https://archive.apache.org/dist/lucene/solr/5.4.0/solr-5.4.0.tgz
|
146
|
+
$ tar zxvf solr-5.4.0.tgz
|
147
|
+
$ cd solr-5.4.0
|
148
|
+
$ ./server/scripts/cloud-scripts/zkcli.sh -zkhost localhost:2181 -cmd clear /solr
|
149
|
+
$ ./server/scripts/cloud-scripts/zkcli.sh -zkhost localhost:2181 -cmd makepath /solr
|
150
|
+
$ cp -pr server/solr server/solr1
|
151
|
+
$ cp -pr server/solr server/solr2
|
152
|
+
```
|
153
|
+
|
154
|
+
4.Start SolrCloud
|
155
|
+
|
156
|
+
```sh
|
157
|
+
$ ./bin/solr start -h localhost -p 8983 -z localhost:2181/solr -s server/solr1
|
158
|
+
$ ./bin/solr start -h localhost -p 8985 -z localhost:2181/solr -s server/solr2
|
159
|
+
```
|
160
|
+
|
161
|
+
5.Create collection
|
162
|
+
|
163
|
+
```sh
|
164
|
+
$ ./bin/solr create -c collection1 -d server/solr1/configsets/data_driven_schema_configs -n collection1_configs -shards 1 -replicationFactor 2
|
165
|
+
```
|
166
|
+
|
167
|
+
## Need Help?
|
168
|
+
|
169
|
+
Need help? Try #logstash on freenode IRC or the [https://discuss.elastic.co/c/logstash](https://discuss.elastic.co/c/logstash) discussion forum.
|
170
|
+
|
171
|
+
## Developing
|
172
|
+
|
173
|
+
### 1. Plugin Developement and Testing
|
174
|
+
|
175
|
+
#### Code
|
176
|
+
- To get started, you'll need JRuby with the Bundler gem installed.
|
177
|
+
|
178
|
+
- Create a new plugin or clone and existing from the GitHub [logstash-plugins](https://github.com/logstash-plugins) organization. We also provide [example plugins](https://github.com/logstash-plugins?query=example).
|
179
|
+
|
180
|
+
- Install dependencies
|
181
|
+
```sh
|
182
|
+
bundle install
|
183
|
+
```
|
184
|
+
|
185
|
+
#### Test
|
186
|
+
|
187
|
+
- Update your dependencies
|
188
|
+
|
189
|
+
```sh
|
190
|
+
bundle install
|
191
|
+
```
|
192
|
+
|
193
|
+
- Run tests
|
194
|
+
|
195
|
+
```sh
|
196
|
+
bundle exec rspec
|
197
|
+
```
|
198
|
+
|
199
|
+
### 2. Running your unpublished Plugin in Logstash
|
200
|
+
|
201
|
+
#### 2.1 Run in a local Logstash clone
|
202
|
+
|
203
|
+
- Edit Logstash `Gemfile` and add the local plugin path, for example:
|
204
|
+
|
205
|
+
```ruby
|
206
|
+
gem "logstash-output-solr", :path => "/your/local/logstash-output-solr"
|
207
|
+
```
|
208
|
+
|
209
|
+
- Install plugin
|
210
|
+
|
211
|
+
```sh
|
212
|
+
bin/plugin install --no-verify
|
213
|
+
```
|
214
|
+
|
215
|
+
- Run Logstash with your plugin
|
216
|
+
|
217
|
+
```sh
|
218
|
+
bin/logstash -e 'output {solr {}}'
|
219
|
+
```
|
220
|
+
|
221
|
+
At this point any modifications to the plugin code will be applied to this local Logstash setup. After modifying the plugin, simply rerun Logstash.
|
222
|
+
|
223
|
+
#### 2.2 Run in an installed Logstash
|
224
|
+
|
225
|
+
You can use the same **2.1** method to run your plugin in an installed Logstash by editing its `Gemfile` and pointing the `:path` to your local plugin development directory or you can build the gem and install it using:
|
226
|
+
|
227
|
+
- Build your plugin gem
|
228
|
+
|
229
|
+
```sh
|
230
|
+
gem build logstash-output-solr.gemspec
|
231
|
+
```
|
232
|
+
|
233
|
+
- Install the plugin from the Logstash home
|
234
|
+
|
235
|
+
```sh
|
236
|
+
bin/plugin install /your/local/plugin/logstash-output-solr.gem
|
237
|
+
```
|
238
|
+
|
239
|
+
- Start Logstash and proceed to test the plugin
|
240
|
+
|
241
|
+
## Contributing
|
242
|
+
|
243
|
+
All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin.
|
244
|
+
|
245
|
+
Programming is not a required skill. Whatever you've seen about open source and maintainers or community members saying "send patches or die" - you will not see that here.
|
246
|
+
|
247
|
+
It is more important to the community that you are able to contribute.
|
248
|
+
|
249
|
+
For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.
|
@@ -0,0 +1,167 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/outputs/base"
|
3
|
+
require "logstash/namespace"
|
4
|
+
|
5
|
+
require 'securerandom'
|
6
|
+
require "stud/buffer"
|
7
|
+
require 'rsolr'
|
8
|
+
require 'zk'
|
9
|
+
require 'rsolr/cloud'
|
10
|
+
|
11
|
+
# An Solr output that send data to Apache Solr.
|
12
|
+
class LogStash::Outputs::Solr < LogStash::Outputs::Base
|
13
|
+
config_name "solr"
|
14
|
+
|
15
|
+
include Stud::Buffer
|
16
|
+
|
17
|
+
# The Solr server url (for example http://localhost:8983/solr/collection1).
|
18
|
+
config :url, :validate => :string, :default => nil
|
19
|
+
|
20
|
+
# The ZooKeeper connection string that SolrCloud refers to (for example localhost:2181/solr).
|
21
|
+
config :zk_host, :validate => :string, :default => nil
|
22
|
+
# The SolrCloud collection name.
|
23
|
+
config :collection, :validate => :string, :default => 'collection1'
|
24
|
+
|
25
|
+
# The defined fields in the Solr schema.xml. If omitted, it will get fields via Solr Schema API.
|
26
|
+
config :defined_fields, :validate => :array, :default => nil
|
27
|
+
# Ignore undefined fields in the Solr schema.xml.
|
28
|
+
config :ignore_undefined_fields, :validate => :boolean, :default => false
|
29
|
+
|
30
|
+
# A field name of unique key in the Solr schema.xml. If omitted, it will get unique key via Solr Schema API.
|
31
|
+
config :unique_key_field, :validate => :string, :default => nil
|
32
|
+
# A field name of event timestamp in the Solr schema.xml (default event_timestamp).
|
33
|
+
config :timestamp_field, :validate => :string, :default => 'event_timestamp'
|
34
|
+
|
35
|
+
# The batch size used in update.
|
36
|
+
config :flush_size, :validate => :number, :default => 100
|
37
|
+
|
38
|
+
# The batch size used in update.
|
39
|
+
config :idle_flush_time, :validate => :number, :default => 10
|
40
|
+
|
41
|
+
MODE_STANDALONE = 'Standalone'
|
42
|
+
MODE_SOLRCLOUD = 'SolrCloud'
|
43
|
+
|
44
|
+
public
|
45
|
+
def register
|
46
|
+
@mode = nil
|
47
|
+
if ! @url.nil? then
|
48
|
+
@mode = MODE_STANDALONE
|
49
|
+
elsif ! @zk_host.nil?
|
50
|
+
@mode = MODE_SOLRCLOUD
|
51
|
+
end
|
52
|
+
|
53
|
+
@solr = nil
|
54
|
+
@zk = nil
|
55
|
+
|
56
|
+
if @mode == MODE_STANDALONE then
|
57
|
+
@solr = RSolr.connect :url => @url
|
58
|
+
elsif @mode == MODE_SOLRCLOUD then
|
59
|
+
@zk = ZK.new(@zk_host)
|
60
|
+
cloud_connection = RSolr::Cloud::Connection.new(@zk)
|
61
|
+
@solr = RSolr::Client.new(cloud_connection, read_timeout: 60, open_timeout: 60)
|
62
|
+
end
|
63
|
+
|
64
|
+
buffer_initialize(
|
65
|
+
:max_items => @flush_size,
|
66
|
+
:max_interval => @idle_flush_time,
|
67
|
+
:logger => @logger
|
68
|
+
)
|
69
|
+
end # def register
|
70
|
+
|
71
|
+
public
|
72
|
+
def receive(event)
|
73
|
+
buffer_receive(event)
|
74
|
+
end # def event
|
75
|
+
|
76
|
+
public
|
77
|
+
def flush(events, close=false)
|
78
|
+
documents = []
|
79
|
+
|
80
|
+
@fields = @defined_fields.nil? || @defined_fields.empty? ? get_fields : @defined_fields
|
81
|
+
|
82
|
+
@unique_key = @unique_key_field.nil? ? get_unique_key : @unique_key_field
|
83
|
+
|
84
|
+
events.each do |event|
|
85
|
+
document = event.to_hash()
|
86
|
+
|
87
|
+
unless document.has_key?(@unique_key) then
|
88
|
+
document.merge!({@unique_key => SecureRandom.uuid})
|
89
|
+
end
|
90
|
+
|
91
|
+
unless document.has_key?(@timestamp_field) then
|
92
|
+
document.merge!({@timestamp_field => document['@timestamp']})
|
93
|
+
end
|
94
|
+
|
95
|
+
if @ignore_undefined_fields then
|
96
|
+
document.each_key do |key|
|
97
|
+
unless @fields.include?(key) then
|
98
|
+
document.delete(key)
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
@logger.info 'Record: %s' % document.inspect
|
104
|
+
|
105
|
+
documents.push(document)
|
106
|
+
end
|
107
|
+
|
108
|
+
if @mode == MODE_STANDALONE then
|
109
|
+
@solr.add documents, :params => {:commit => true}
|
110
|
+
@logger.info 'Added %d document(s) to Solr' % documents.count
|
111
|
+
elsif @mode == MODE_SOLRCLOUD then
|
112
|
+
@solr.add documents, collection: @collection, :params => {:commit => true}
|
113
|
+
@logger.info 'Added %d document(s) to Solr' % documents.count
|
114
|
+
end
|
115
|
+
|
116
|
+
rescue Exception => e
|
117
|
+
@logger.warn('An error occurred while indexing: #{e.message}')
|
118
|
+
end # def flush
|
119
|
+
|
120
|
+
public
|
121
|
+
def close
|
122
|
+
unless @zk.nil? then
|
123
|
+
@zk.close
|
124
|
+
end
|
125
|
+
end # def close
|
126
|
+
|
127
|
+
private
|
128
|
+
def get_unique_key
|
129
|
+
response = nil
|
130
|
+
|
131
|
+
if @mode == MODE_STANDALONE then
|
132
|
+
response = @solr.get 'schema/uniquekey'
|
133
|
+
elsif @mode == MODE_SOLRCLOUD then
|
134
|
+
response = @solr.get 'schema/uniquekey', collection: @collection
|
135
|
+
end
|
136
|
+
|
137
|
+
unique_key = response['uniqueKey']
|
138
|
+
@logger.info 'Unique key: #{unique_key}'
|
139
|
+
|
140
|
+
return unique_key
|
141
|
+
|
142
|
+
rescue Exception => e
|
143
|
+
@logger.warn 'Unique key: #{e.message}'
|
144
|
+
end # def get_unique_key
|
145
|
+
|
146
|
+
private
|
147
|
+
def get_fields
|
148
|
+
response = nil
|
149
|
+
|
150
|
+
if @mode == MODE_STANDALONE then
|
151
|
+
response = @solr.get 'schema/fields'
|
152
|
+
elsif @mode == MODE_SOLRCLOUD then
|
153
|
+
response = @solr.get 'schema/fields', collection: @collection
|
154
|
+
end
|
155
|
+
|
156
|
+
fields = []
|
157
|
+
response['fields'].each do |field|
|
158
|
+
fields.push(field['name'])
|
159
|
+
end
|
160
|
+
@logger.info 'Fields: #{fields}'
|
161
|
+
|
162
|
+
return fields
|
163
|
+
|
164
|
+
rescue Exception => e
|
165
|
+
@logger.warn 'Fields: #{e.message}'
|
166
|
+
end # def get_fields
|
167
|
+
end # class LogStash::Outputs::Solr
|
@@ -0,0 +1,31 @@
|
|
1
|
+
Gem::Specification.new do |s|
|
2
|
+
s.name = 'logstash-output-custom-solr'
|
3
|
+
s.version = "0.1.0"
|
4
|
+
s.licenses = ["Apache License (2.0)"]
|
5
|
+
s.summary = "Logstash output plugin for sending data to Solr."
|
6
|
+
s.description = "Logstash output plugin for sending data to Solr. It support SolrCloud not only Standalone Solr."
|
7
|
+
s.authors = ["Minoru Osuka"]
|
8
|
+
s.email = "minoru.osuka@gmail.com"
|
9
|
+
s.homepage = "https://github.com/mosuka/logstash-output-solr"
|
10
|
+
s.require_paths = ["lib"]
|
11
|
+
|
12
|
+
# Files
|
13
|
+
s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE']
|
14
|
+
# Tests
|
15
|
+
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
16
|
+
|
17
|
+
# Special flag to let us know this is actually a logstash plugin
|
18
|
+
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
|
19
|
+
|
20
|
+
# Gem dependencies
|
21
|
+
s.add_runtime_dependency "logstash-core", ">= 2.0.0", "<= 2.99"
|
22
|
+
s.add_runtime_dependency "logstash-codec-plain"
|
23
|
+
s.add_runtime_dependency 'rsolr', '~> 1.1.2'
|
24
|
+
s.add_runtime_dependency 'zk', '~> 1.9.6'
|
25
|
+
s.add_runtime_dependency 'rsolr-cloud', '~> 1.1.0'
|
26
|
+
s.add_runtime_dependency 'stud', '~> 0.0.22'
|
27
|
+
|
28
|
+
s.add_development_dependency "logstash-devutils", '~> 0.0.18'
|
29
|
+
s.add_development_dependency 'rake', '~> 10.5.0'
|
30
|
+
s.add_development_dependency 'zk-server', '~> 1.1.8'
|
31
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
{"collection1":{
|
2
|
+
"replicationFactor":"1",
|
3
|
+
"router":{"name":"compositeId"},
|
4
|
+
"maxShardsPerNode":"1",
|
5
|
+
"autoAddReplicas":"false",
|
6
|
+
"shards":{"shard1":{
|
7
|
+
"range":"80000000-7fffffff",
|
8
|
+
"state":"active",
|
9
|
+
"replicas":{"core_node1":{
|
10
|
+
"core":"collection1_shard1_replica1",
|
11
|
+
"base_url":"http://localhost:8983/solr",
|
12
|
+
"node_name":"localhost:8983_solr",
|
13
|
+
"state":"active",
|
14
|
+
"leader":"true"}}}}}}
|
@@ -0,0 +1,229 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/devutils/rspec/spec_helper"
|
3
|
+
require "logstash/outputs/solr"
|
4
|
+
require "logstash/codecs/plain"
|
5
|
+
require "logstash/event"
|
6
|
+
|
7
|
+
require 'zk-server'
|
8
|
+
require 'zk'
|
9
|
+
|
10
|
+
describe LogStash::Outputs::Solr do
|
11
|
+
|
12
|
+
before do
|
13
|
+
@zk_server = nil
|
14
|
+
end
|
15
|
+
|
16
|
+
describe 'configuration' do
|
17
|
+
let(:config) {
|
18
|
+
{
|
19
|
+
'url' => 'http://localhost:8983/solr/collection1',
|
20
|
+
'zk_host' => 'localhost:2181/solr',
|
21
|
+
'collection' => 'collection1',
|
22
|
+
'defined_fields' => ['id', 'title'],
|
23
|
+
'ignore_undefined_fields' => true,
|
24
|
+
'unique_key_field' => 'id',
|
25
|
+
'timestamp_field' => 'event_timestamp',
|
26
|
+
'flush_size' => 100
|
27
|
+
}
|
28
|
+
}
|
29
|
+
|
30
|
+
it 'url' do
|
31
|
+
output = LogStash::Outputs::Solr.new(config)
|
32
|
+
expect(output.config['url']).to eq('http://localhost:8983/solr/collection1')
|
33
|
+
end
|
34
|
+
|
35
|
+
it 'zk_host' do
|
36
|
+
output = LogStash::Outputs::Solr.new(config)
|
37
|
+
expect(output.config['zk_host']).to eq('localhost:2181/solr')
|
38
|
+
end
|
39
|
+
|
40
|
+
it 'collection' do
|
41
|
+
output = LogStash::Outputs::Solr.new(config)
|
42
|
+
expect(output.config['collection']).to eq('collection1')
|
43
|
+
end
|
44
|
+
|
45
|
+
it 'defined_fields' do
|
46
|
+
output = LogStash::Outputs::Solr.new(config)
|
47
|
+
expect(output.config['defined_fields']).to eq(['id', 'title'])
|
48
|
+
end
|
49
|
+
|
50
|
+
it 'ignore_undefined_fields' do
|
51
|
+
output = LogStash::Outputs::Solr.new(config)
|
52
|
+
expect(output.config['ignore_undefined_fields']).to eq(true)
|
53
|
+
end
|
54
|
+
|
55
|
+
it 'unique_key_field' do
|
56
|
+
output = LogStash::Outputs::Solr.new(config)
|
57
|
+
expect(output.config['unique_key_field']).to eq('id')
|
58
|
+
end
|
59
|
+
|
60
|
+
it 'timestamp_field' do
|
61
|
+
output = LogStash::Outputs::Solr.new(config)
|
62
|
+
expect(output.config['timestamp_field']).to eq('event_timestamp')
|
63
|
+
end
|
64
|
+
|
65
|
+
it 'flush_size' do
|
66
|
+
output = LogStash::Outputs::Solr.new(config)
|
67
|
+
expect(output.config['flush_size']).to eq(100)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
describe 'register_standalone' do
|
72
|
+
let(:config) {
|
73
|
+
{
|
74
|
+
'url' => 'http://localhost:8983/solr/collection1',
|
75
|
+
'defined_fields' => ['id', 'title'],
|
76
|
+
'ignore_undefined_fields' => true,
|
77
|
+
'unique_key_field' => 'id',
|
78
|
+
'timestamp_field' => 'event_timestamp',
|
79
|
+
'flush_size' => 100
|
80
|
+
}
|
81
|
+
}
|
82
|
+
|
83
|
+
it 'mode' do
|
84
|
+
output = LogStash::Outputs::Solr.new(config)
|
85
|
+
output.register
|
86
|
+
|
87
|
+
mode = output.instance_variable_get('@mode')
|
88
|
+
expect(mode).to eq('Standalone')
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
describe 'register_solrcloud' do
|
93
|
+
before do
|
94
|
+
start_zookeeper
|
95
|
+
end
|
96
|
+
|
97
|
+
after do
|
98
|
+
stop_zookeeper
|
99
|
+
end
|
100
|
+
|
101
|
+
let(:config) {
|
102
|
+
{
|
103
|
+
'zk_host' => 'localhost:3292/solr',
|
104
|
+
'collection' => 'collection1',
|
105
|
+
'defined_fields' => ['id', 'title'],
|
106
|
+
'ignore_undefined_fields' => true,
|
107
|
+
'unique_key_field' => 'id',
|
108
|
+
'timestamp_field' => 'event_timestamp',
|
109
|
+
'flush_size' => 100
|
110
|
+
}
|
111
|
+
}
|
112
|
+
|
113
|
+
it 'mode' do
|
114
|
+
output = LogStash::Outputs::Solr.new(config)
|
115
|
+
output.register
|
116
|
+
|
117
|
+
mode = output.instance_variable_get('@mode')
|
118
|
+
expect(mode).to eq('SolrCloud')
|
119
|
+
end
|
120
|
+
end
|
121
|
+
|
122
|
+
describe 'receive_standalone' do
|
123
|
+
let(:config) {
|
124
|
+
{
|
125
|
+
'url' => 'http://localhost:8983/solr/collection1',
|
126
|
+
'defined_fields' => ['id', 'title'],
|
127
|
+
'ignore_undefined_fields' => true,
|
128
|
+
'unique_key_field' => 'id',
|
129
|
+
'timestamp_field' => 'event_timestamp',
|
130
|
+
'flush_size' => 100
|
131
|
+
}
|
132
|
+
}
|
133
|
+
|
134
|
+
let(:sample_record) {
|
135
|
+
{
|
136
|
+
'id' => 'change.me',
|
137
|
+
'title' => 'change.me'
|
138
|
+
}
|
139
|
+
}
|
140
|
+
|
141
|
+
it 'receive' do
|
142
|
+
output = LogStash::Outputs::Solr.new(config)
|
143
|
+
output.register
|
144
|
+
|
145
|
+
output.receive(sample_record)
|
146
|
+
end
|
147
|
+
end
|
148
|
+
|
149
|
+
describe 'receive_solrcloud' do
|
150
|
+
before do
|
151
|
+
start_zookeeper
|
152
|
+
end
|
153
|
+
|
154
|
+
after do
|
155
|
+
stop_zookeeper
|
156
|
+
end
|
157
|
+
|
158
|
+
let(:config) {
|
159
|
+
{
|
160
|
+
'zk_host' => 'localhost:3292/solr',
|
161
|
+
'collection' => 'collection1',
|
162
|
+
'defined_fields' => ['id', 'title'],
|
163
|
+
'ignore_undefined_fields' => true,
|
164
|
+
'unique_key_field' => 'id',
|
165
|
+
'timestamp_field' => 'event_timestamp',
|
166
|
+
'flush_size' => 100
|
167
|
+
}
|
168
|
+
}
|
169
|
+
|
170
|
+
let(:sample_record) {
|
171
|
+
{
|
172
|
+
'id' => 'change.me',
|
173
|
+
'title' => 'change.me'
|
174
|
+
}
|
175
|
+
}
|
176
|
+
|
177
|
+
it 'receive' do
|
178
|
+
output = LogStash::Outputs::Solr.new(config)
|
179
|
+
output.register
|
180
|
+
|
181
|
+
output.receive(sample_record)
|
182
|
+
end
|
183
|
+
end
|
184
|
+
|
185
|
+
def start_zookeeper
|
186
|
+
@zk_server = ZK::Server.new do |config|
|
187
|
+
config.client_port = 3292
|
188
|
+
config.enable_jmx = true
|
189
|
+
config.force_sync = false
|
190
|
+
end
|
191
|
+
|
192
|
+
@zk_server.run
|
193
|
+
|
194
|
+
zk = ZK.new('localhost:3292')
|
195
|
+
delete_nodes(zk, '/solr')
|
196
|
+
create_nodes(zk, '/solr/live_nodes')
|
197
|
+
create_nodes(zk, '/solr/collections')
|
198
|
+
['localhost:8983_solr'].each do |node|
|
199
|
+
zk.create("/solr/live_nodes/#{node}", '', mode: :ephemeral)
|
200
|
+
end
|
201
|
+
['collection1'].each do |collection|
|
202
|
+
zk.create("/solr/collections/#{collection}")
|
203
|
+
json = File.read("spec/files/collections/#{collection}/state.json")
|
204
|
+
zk.create("/solr/collections/#{collection}/state.json", json, mode: :ephemeral)
|
205
|
+
end
|
206
|
+
end
|
207
|
+
|
208
|
+
def stop_zookeeper
|
209
|
+
@zk_server.shutdown
|
210
|
+
end
|
211
|
+
|
212
|
+
def delete_nodes(zk, path)
|
213
|
+
zk.children(path).each do |node|
|
214
|
+
delete_nodes(zk, File.join(path, node))
|
215
|
+
end
|
216
|
+
zk.delete(path)
|
217
|
+
rescue ZK::Exceptions::NoNode
|
218
|
+
end
|
219
|
+
|
220
|
+
def create_nodes(zk, path)
|
221
|
+
parent_path = File.dirname(path)
|
222
|
+
unless zk.exists?(parent_path, :watch => true) then
|
223
|
+
create_nodes(zk, parent_path)
|
224
|
+
end
|
225
|
+
zk.create(path)
|
226
|
+
end
|
227
|
+
end
|
228
|
+
|
229
|
+
|
metadata
ADDED
@@ -0,0 +1,190 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: logstash-output-custom-solr
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Minoru Osuka
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2018-05-11 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
15
|
+
requirements:
|
16
|
+
- - ">="
|
17
|
+
- !ruby/object:Gem::Version
|
18
|
+
version: 2.0.0
|
19
|
+
- - "<="
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '2.99'
|
22
|
+
name: logstash-core
|
23
|
+
prerelease: false
|
24
|
+
type: :runtime
|
25
|
+
version_requirements: !ruby/object:Gem::Requirement
|
26
|
+
requirements:
|
27
|
+
- - ">="
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: 2.0.0
|
30
|
+
- - "<="
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '2.99'
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
requirement: !ruby/object:Gem::Requirement
|
35
|
+
requirements:
|
36
|
+
- - ">="
|
37
|
+
- !ruby/object:Gem::Version
|
38
|
+
version: '0'
|
39
|
+
name: logstash-codec-plain
|
40
|
+
prerelease: false
|
41
|
+
type: :runtime
|
42
|
+
version_requirements: !ruby/object:Gem::Requirement
|
43
|
+
requirements:
|
44
|
+
- - ">="
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: '0'
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
requirements:
|
50
|
+
- - "~>"
|
51
|
+
- !ruby/object:Gem::Version
|
52
|
+
version: 1.1.2
|
53
|
+
name: rsolr
|
54
|
+
prerelease: false
|
55
|
+
type: :runtime
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
requirements:
|
58
|
+
- - "~>"
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
version: 1.1.2
|
61
|
+
- !ruby/object:Gem::Dependency
|
62
|
+
requirement: !ruby/object:Gem::Requirement
|
63
|
+
requirements:
|
64
|
+
- - "~>"
|
65
|
+
- !ruby/object:Gem::Version
|
66
|
+
version: 1.9.6
|
67
|
+
name: zk
|
68
|
+
prerelease: false
|
69
|
+
type: :runtime
|
70
|
+
version_requirements: !ruby/object:Gem::Requirement
|
71
|
+
requirements:
|
72
|
+
- - "~>"
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: 1.9.6
|
75
|
+
- !ruby/object:Gem::Dependency
|
76
|
+
requirement: !ruby/object:Gem::Requirement
|
77
|
+
requirements:
|
78
|
+
- - "~>"
|
79
|
+
- !ruby/object:Gem::Version
|
80
|
+
version: 1.1.0
|
81
|
+
name: rsolr-cloud
|
82
|
+
prerelease: false
|
83
|
+
type: :runtime
|
84
|
+
version_requirements: !ruby/object:Gem::Requirement
|
85
|
+
requirements:
|
86
|
+
- - "~>"
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: 1.1.0
|
89
|
+
- !ruby/object:Gem::Dependency
|
90
|
+
requirement: !ruby/object:Gem::Requirement
|
91
|
+
requirements:
|
92
|
+
- - "~>"
|
93
|
+
- !ruby/object:Gem::Version
|
94
|
+
version: 0.0.22
|
95
|
+
name: stud
|
96
|
+
prerelease: false
|
97
|
+
type: :runtime
|
98
|
+
version_requirements: !ruby/object:Gem::Requirement
|
99
|
+
requirements:
|
100
|
+
- - "~>"
|
101
|
+
- !ruby/object:Gem::Version
|
102
|
+
version: 0.0.22
|
103
|
+
- !ruby/object:Gem::Dependency
|
104
|
+
requirement: !ruby/object:Gem::Requirement
|
105
|
+
requirements:
|
106
|
+
- - "~>"
|
107
|
+
- !ruby/object:Gem::Version
|
108
|
+
version: 0.0.18
|
109
|
+
name: logstash-devutils
|
110
|
+
prerelease: false
|
111
|
+
type: :development
|
112
|
+
version_requirements: !ruby/object:Gem::Requirement
|
113
|
+
requirements:
|
114
|
+
- - "~>"
|
115
|
+
- !ruby/object:Gem::Version
|
116
|
+
version: 0.0.18
|
117
|
+
- !ruby/object:Gem::Dependency
|
118
|
+
requirement: !ruby/object:Gem::Requirement
|
119
|
+
requirements:
|
120
|
+
- - "~>"
|
121
|
+
- !ruby/object:Gem::Version
|
122
|
+
version: 10.5.0
|
123
|
+
name: rake
|
124
|
+
prerelease: false
|
125
|
+
type: :development
|
126
|
+
version_requirements: !ruby/object:Gem::Requirement
|
127
|
+
requirements:
|
128
|
+
- - "~>"
|
129
|
+
- !ruby/object:Gem::Version
|
130
|
+
version: 10.5.0
|
131
|
+
- !ruby/object:Gem::Dependency
|
132
|
+
requirement: !ruby/object:Gem::Requirement
|
133
|
+
requirements:
|
134
|
+
- - "~>"
|
135
|
+
- !ruby/object:Gem::Version
|
136
|
+
version: 1.1.8
|
137
|
+
name: zk-server
|
138
|
+
prerelease: false
|
139
|
+
type: :development
|
140
|
+
version_requirements: !ruby/object:Gem::Requirement
|
141
|
+
requirements:
|
142
|
+
- - "~>"
|
143
|
+
- !ruby/object:Gem::Version
|
144
|
+
version: 1.1.8
|
145
|
+
description: Logstash output plugin for sending data to Solr. It support SolrCloud
|
146
|
+
not only Standalone Solr.
|
147
|
+
email: minoru.osuka@gmail.com
|
148
|
+
executables: []
|
149
|
+
extensions: []
|
150
|
+
extra_rdoc_files: []
|
151
|
+
files:
|
152
|
+
- CHANGELOG.md
|
153
|
+
- CONTRIBUTORS
|
154
|
+
- DEVELOPER.md
|
155
|
+
- Gemfile
|
156
|
+
- LICENSE
|
157
|
+
- README.md
|
158
|
+
- lib/logstash/outputs/solr.rb
|
159
|
+
- logstash-output-custom-solr.gemspec
|
160
|
+
- spec/files/collections/collection1/state.json
|
161
|
+
- spec/outputs/solr_spec.rb
|
162
|
+
homepage: https://github.com/mosuka/logstash-output-solr
|
163
|
+
licenses:
|
164
|
+
- Apache License (2.0)
|
165
|
+
metadata:
|
166
|
+
logstash_plugin: 'true'
|
167
|
+
logstash_group: output
|
168
|
+
post_install_message:
|
169
|
+
rdoc_options: []
|
170
|
+
require_paths:
|
171
|
+
- lib
|
172
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
173
|
+
requirements:
|
174
|
+
- - ">="
|
175
|
+
- !ruby/object:Gem::Version
|
176
|
+
version: '0'
|
177
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
178
|
+
requirements:
|
179
|
+
- - ">="
|
180
|
+
- !ruby/object:Gem::Version
|
181
|
+
version: '0'
|
182
|
+
requirements: []
|
183
|
+
rubyforge_project:
|
184
|
+
rubygems_version: 2.6.13
|
185
|
+
signing_key:
|
186
|
+
specification_version: 4
|
187
|
+
summary: Logstash output plugin for sending data to Solr.
|
188
|
+
test_files:
|
189
|
+
- spec/files/collections/collection1/state.json
|
190
|
+
- spec/outputs/solr_spec.rb
|