logstash-output-custom-solr 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 71fad4671bc6f96e8852ad45c9abcf69c5cedc76b1acf2901b0817889ec5029f
4
+ data.tar.gz: e7370ff6cea53d407d8272e07c6349cf25913080d2bff4fa36368c2b69254118
5
+ SHA512:
6
+ metadata.gz: 9c5cfac43debf01dddb3e80bed0e09020509f25e63c35643d705238a153a64c61187bb0467c92427920ac1f127484e0431b2e6fc756e90c5189ccb1bafa3632a
7
+ data.tar.gz: 76420cb872fbe00a2a18d587206e4f738967c39fd2e52b92779304cf8dfa45b47b160ea90143779fbe264cb02d73ca169d448f660f07675bd2ea58000f5c2fa2
data/CHANGELOG.md ADDED
@@ -0,0 +1,11 @@
1
+ ## 0.3.0
2
+ - Update rsolr-cloud.
3
+
4
+ ## 0.2.0
5
+ - Bug fix.
6
+
7
+ ## 0.2.0
8
+ - Supports SolrCloud.
9
+
10
+ ## 0.1.0
11
+ - First release.
data/CONTRIBUTORS ADDED
@@ -0,0 +1,10 @@
1
+ The following is a list of people who have contributed ideas, code, bug
2
+ reports, or in general have helped logstash along its way.
3
+
4
+ Contributors:
5
+ * Minoru Osuka (minoru.osuka@gmail.com)
6
+
7
+ Note: If you've sent us patches, bug reports, or otherwise contributed to
8
+ Logstash, and you aren't on the list above and want to be, please let us know
9
+ and we'll make sure you're here. Contributions from folks like you are what make
10
+ open source awesome.
data/DEVELOPER.md ADDED
@@ -0,0 +1,2 @@
1
+ # logstash-output-solr
2
+ Solr output plugin.
data/Gemfile ADDED
@@ -0,0 +1,2 @@
1
+ source 'https://rubygems.org'
2
+ gemspec
data/LICENSE ADDED
@@ -0,0 +1,13 @@
1
+ Copyright (c) 2012–2015 Elasticsearch <http://www.elastic.co>
2
+
3
+ Licensed under the Apache License, Version 2.0 (the "License");
4
+ you may not use this file except in compliance with the License.
5
+ You may obtain a copy of the License at
6
+
7
+ http://www.apache.org/licenses/LICENSE-2.0
8
+
9
+ Unless required by applicable law or agreed to in writing, software
10
+ distributed under the License is distributed on an "AS IS" BASIS,
11
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ See the License for the specific language governing permissions and
13
+ limitations under the License.
data/README.md ADDED
@@ -0,0 +1,249 @@
1
+ # Logstash Plugin
2
+
3
+ This is a plugin for [Logstash](https://github.com/elastic/logstash). It support [SolrCloud](https://cwiki.apache.org/confluence/display/solr/SolrCloud) not only Standalone Solr.
4
+
5
+ It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
6
+
7
+ ## Config parameters
8
+
9
+ ### url
10
+
11
+ The Solr server url (for example http://localhost:8983/solr/collection1).
12
+
13
+ ```
14
+ url http://localhost:8983/solr/collection1
15
+ ```
16
+
17
+ ### zk_host
18
+
19
+ The ZooKeeper connection string that SolrCloud refers to (for example localhost:2181/solr).
20
+
21
+ ```
22
+ zk_host localhost:2181/solr
23
+ ```
24
+
25
+ ### collection
26
+
27
+ The SolrCloud collection name (default collection1).
28
+
29
+ ```
30
+ collection collection1
31
+ ```
32
+
33
+ ### defined_fields
34
+
35
+ The defined fields in the Solr schema.xml. If omitted, it will get fields via Solr Schema API.
36
+
37
+ ```
38
+ defined_fields ["id", "title"]
39
+ ```
40
+
41
+ ### ignore_undefined_fields
42
+
43
+ Ignore undefined fields in the Solr schema.xml.
44
+
45
+ ```
46
+ ignore_undefined_fields false
47
+ ```
48
+
49
+ ### unique_key_field
50
+
51
+ A field name of unique key in the Solr schema.xml. If omitted, it will get unique key via Solr Schema API.
52
+
53
+ ```
54
+ unique_key_field id
55
+ ```
56
+
57
+ ### timestamp_field
58
+
59
+ A field name of event timestamp in the Solr schema.xml (default event_timestamp).
60
+
61
+ ```
62
+ timestamp_field event_timestamp
63
+ ```
64
+
65
+ ### flush_size
66
+
67
+ A number of events to queue up before writing to Solr (default 100).
68
+
69
+ ```
70
+ flush_size 100
71
+ ```
72
+
73
+ ## Plugin setup examples
74
+
75
+ ### Sent to standalone Solr using data-driven schemaless mode.
76
+ ```
77
+ output {
78
+ solr {
79
+ url => "http://localhost:8983/solr/collection1"
80
+ }
81
+ }
82
+
83
+ ```
84
+
85
+ ### Sent to SolrCloud using data-driven schemaless mode.
86
+ ```
87
+ output {
88
+ solr {
89
+ zk_host => "localhost:2181/solr"
90
+ collection => "collection1"
91
+ }
92
+ }
93
+ ```
94
+
95
+ ## Solr setup examples
96
+
97
+ ### How to setup Standalone Solr using data-driven schemaless mode.
98
+
99
+ 1.Download and install Solr
100
+
101
+ ```sh
102
+ $ mkdir $HOME/solr
103
+ $ cd $HOME/solr
104
+ $ wget https://archive.apache.org/dist/lucene/solr/5.4.0/solr-5.4.0.tgz
105
+ $ tar zxvf solr-5.4.0.tgz
106
+ $ cd solr-5.4.0
107
+ ```
108
+
109
+ 2.Start standalone Solr
110
+
111
+ ```sh
112
+ $ ./bin/solr start -p 8983 -s server/solr
113
+ ```
114
+
115
+ 3.Create core
116
+
117
+ ```sh
118
+ $ ./bin/solr create -c collection1 -d server/solr/configsets/data_driven_schema_configs -n collection1_configs
119
+ ```
120
+
121
+ ### How to setup SolrCloud using data-driven schemaless mode (shards=1 and replicationfactor=2).
122
+
123
+ 1.Download and install ZooKeeper
124
+
125
+ ```sh
126
+ $ mkdir $HOME/zookeeper
127
+ $ cd $HOME/zookeeper
128
+ $ wget https://archive.apache.org/dist/zookeeper/zookeeper-3.4.6/zookeeper-3.4.6.tar.gz
129
+ $ tar zxvf zookeeper-3.4.6.tar.gz
130
+ $ cd zookeeper-3.4.6
131
+ $ cp -p ./conf/zoo_sample.cfg ./conf/zoo.cfg
132
+ ```
133
+
134
+ 2.Start standalone ZooKeeper
135
+
136
+ ```sh
137
+ $ ./bin/zkServer.sh start
138
+ ```
139
+
140
+ 3.Download an install Solr
141
+
142
+ ```sh
143
+ $ mkdir $HOME/solr
144
+ $ cd $HOME/solr
145
+ $ wget https://archive.apache.org/dist/lucene/solr/5.4.0/solr-5.4.0.tgz
146
+ $ tar zxvf solr-5.4.0.tgz
147
+ $ cd solr-5.4.0
148
+ $ ./server/scripts/cloud-scripts/zkcli.sh -zkhost localhost:2181 -cmd clear /solr
149
+ $ ./server/scripts/cloud-scripts/zkcli.sh -zkhost localhost:2181 -cmd makepath /solr
150
+ $ cp -pr server/solr server/solr1
151
+ $ cp -pr server/solr server/solr2
152
+ ```
153
+
154
+ 4.Start SolrCloud
155
+
156
+ ```sh
157
+ $ ./bin/solr start -h localhost -p 8983 -z localhost:2181/solr -s server/solr1
158
+ $ ./bin/solr start -h localhost -p 8985 -z localhost:2181/solr -s server/solr2
159
+ ```
160
+
161
+ 5.Create collection
162
+
163
+ ```sh
164
+ $ ./bin/solr create -c collection1 -d server/solr1/configsets/data_driven_schema_configs -n collection1_configs -shards 1 -replicationFactor 2
165
+ ```
166
+
167
+ ## Need Help?
168
+
169
+ Need help? Try #logstash on freenode IRC or the [https://discuss.elastic.co/c/logstash](https://discuss.elastic.co/c/logstash) discussion forum.
170
+
171
+ ## Developing
172
+
173
+ ### 1. Plugin Developement and Testing
174
+
175
+ #### Code
176
+ - To get started, you'll need JRuby with the Bundler gem installed.
177
+
178
+ - Create a new plugin or clone and existing from the GitHub [logstash-plugins](https://github.com/logstash-plugins) organization. We also provide [example plugins](https://github.com/logstash-plugins?query=example).
179
+
180
+ - Install dependencies
181
+ ```sh
182
+ bundle install
183
+ ```
184
+
185
+ #### Test
186
+
187
+ - Update your dependencies
188
+
189
+ ```sh
190
+ bundle install
191
+ ```
192
+
193
+ - Run tests
194
+
195
+ ```sh
196
+ bundle exec rspec
197
+ ```
198
+
199
+ ### 2. Running your unpublished Plugin in Logstash
200
+
201
+ #### 2.1 Run in a local Logstash clone
202
+
203
+ - Edit Logstash `Gemfile` and add the local plugin path, for example:
204
+
205
+ ```ruby
206
+ gem "logstash-output-solr", :path => "/your/local/logstash-output-solr"
207
+ ```
208
+
209
+ - Install plugin
210
+
211
+ ```sh
212
+ bin/plugin install --no-verify
213
+ ```
214
+
215
+ - Run Logstash with your plugin
216
+
217
+ ```sh
218
+ bin/logstash -e 'output {solr {}}'
219
+ ```
220
+
221
+ At this point any modifications to the plugin code will be applied to this local Logstash setup. After modifying the plugin, simply rerun Logstash.
222
+
223
+ #### 2.2 Run in an installed Logstash
224
+
225
+ You can use the same **2.1** method to run your plugin in an installed Logstash by editing its `Gemfile` and pointing the `:path` to your local plugin development directory or you can build the gem and install it using:
226
+
227
+ - Build your plugin gem
228
+
229
+ ```sh
230
+ gem build logstash-output-solr.gemspec
231
+ ```
232
+
233
+ - Install the plugin from the Logstash home
234
+
235
+ ```sh
236
+ bin/plugin install /your/local/plugin/logstash-output-solr.gem
237
+ ```
238
+
239
+ - Start Logstash and proceed to test the plugin
240
+
241
+ ## Contributing
242
+
243
+ All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin.
244
+
245
+ Programming is not a required skill. Whatever you've seen about open source and maintainers or community members saying "send patches or die" - you will not see that here.
246
+
247
+ It is more important to the community that you are able to contribute.
248
+
249
+ For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.
@@ -0,0 +1,167 @@
1
+ # encoding: utf-8
2
+ require "logstash/outputs/base"
3
+ require "logstash/namespace"
4
+
5
+ require 'securerandom'
6
+ require "stud/buffer"
7
+ require 'rsolr'
8
+ require 'zk'
9
+ require 'rsolr/cloud'
10
+
11
+ # An Solr output that send data to Apache Solr.
12
+ class LogStash::Outputs::Solr < LogStash::Outputs::Base
13
+ config_name "solr"
14
+
15
+ include Stud::Buffer
16
+
17
+ # The Solr server url (for example http://localhost:8983/solr/collection1).
18
+ config :url, :validate => :string, :default => nil
19
+
20
+ # The ZooKeeper connection string that SolrCloud refers to (for example localhost:2181/solr).
21
+ config :zk_host, :validate => :string, :default => nil
22
+ # The SolrCloud collection name.
23
+ config :collection, :validate => :string, :default => 'collection1'
24
+
25
+ # The defined fields in the Solr schema.xml. If omitted, it will get fields via Solr Schema API.
26
+ config :defined_fields, :validate => :array, :default => nil
27
+ # Ignore undefined fields in the Solr schema.xml.
28
+ config :ignore_undefined_fields, :validate => :boolean, :default => false
29
+
30
+ # A field name of unique key in the Solr schema.xml. If omitted, it will get unique key via Solr Schema API.
31
+ config :unique_key_field, :validate => :string, :default => nil
32
+ # A field name of event timestamp in the Solr schema.xml (default event_timestamp).
33
+ config :timestamp_field, :validate => :string, :default => 'event_timestamp'
34
+
35
+ # The batch size used in update.
36
+ config :flush_size, :validate => :number, :default => 100
37
+
38
+ # The batch size used in update.
39
+ config :idle_flush_time, :validate => :number, :default => 10
40
+
41
+ MODE_STANDALONE = 'Standalone'
42
+ MODE_SOLRCLOUD = 'SolrCloud'
43
+
44
+ public
45
+ def register
46
+ @mode = nil
47
+ if ! @url.nil? then
48
+ @mode = MODE_STANDALONE
49
+ elsif ! @zk_host.nil?
50
+ @mode = MODE_SOLRCLOUD
51
+ end
52
+
53
+ @solr = nil
54
+ @zk = nil
55
+
56
+ if @mode == MODE_STANDALONE then
57
+ @solr = RSolr.connect :url => @url
58
+ elsif @mode == MODE_SOLRCLOUD then
59
+ @zk = ZK.new(@zk_host)
60
+ cloud_connection = RSolr::Cloud::Connection.new(@zk)
61
+ @solr = RSolr::Client.new(cloud_connection, read_timeout: 60, open_timeout: 60)
62
+ end
63
+
64
+ buffer_initialize(
65
+ :max_items => @flush_size,
66
+ :max_interval => @idle_flush_time,
67
+ :logger => @logger
68
+ )
69
+ end # def register
70
+
71
+ public
72
+ def receive(event)
73
+ buffer_receive(event)
74
+ end # def event
75
+
76
+ public
77
+ def flush(events, close=false)
78
+ documents = []
79
+
80
+ @fields = @defined_fields.nil? || @defined_fields.empty? ? get_fields : @defined_fields
81
+
82
+ @unique_key = @unique_key_field.nil? ? get_unique_key : @unique_key_field
83
+
84
+ events.each do |event|
85
+ document = event.to_hash()
86
+
87
+ unless document.has_key?(@unique_key) then
88
+ document.merge!({@unique_key => SecureRandom.uuid})
89
+ end
90
+
91
+ unless document.has_key?(@timestamp_field) then
92
+ document.merge!({@timestamp_field => document['@timestamp']})
93
+ end
94
+
95
+ if @ignore_undefined_fields then
96
+ document.each_key do |key|
97
+ unless @fields.include?(key) then
98
+ document.delete(key)
99
+ end
100
+ end
101
+ end
102
+
103
+ @logger.info 'Record: %s' % document.inspect
104
+
105
+ documents.push(document)
106
+ end
107
+
108
+ if @mode == MODE_STANDALONE then
109
+ @solr.add documents, :params => {:commit => true}
110
+ @logger.info 'Added %d document(s) to Solr' % documents.count
111
+ elsif @mode == MODE_SOLRCLOUD then
112
+ @solr.add documents, collection: @collection, :params => {:commit => true}
113
+ @logger.info 'Added %d document(s) to Solr' % documents.count
114
+ end
115
+
116
+ rescue Exception => e
117
+ @logger.warn('An error occurred while indexing: #{e.message}')
118
+ end # def flush
119
+
120
+ public
121
+ def close
122
+ unless @zk.nil? then
123
+ @zk.close
124
+ end
125
+ end # def close
126
+
127
+ private
128
+ def get_unique_key
129
+ response = nil
130
+
131
+ if @mode == MODE_STANDALONE then
132
+ response = @solr.get 'schema/uniquekey'
133
+ elsif @mode == MODE_SOLRCLOUD then
134
+ response = @solr.get 'schema/uniquekey', collection: @collection
135
+ end
136
+
137
+ unique_key = response['uniqueKey']
138
+ @logger.info 'Unique key: #{unique_key}'
139
+
140
+ return unique_key
141
+
142
+ rescue Exception => e
143
+ @logger.warn 'Unique key: #{e.message}'
144
+ end # def get_unique_key
145
+
146
+ private
147
+ def get_fields
148
+ response = nil
149
+
150
+ if @mode == MODE_STANDALONE then
151
+ response = @solr.get 'schema/fields'
152
+ elsif @mode == MODE_SOLRCLOUD then
153
+ response = @solr.get 'schema/fields', collection: @collection
154
+ end
155
+
156
+ fields = []
157
+ response['fields'].each do |field|
158
+ fields.push(field['name'])
159
+ end
160
+ @logger.info 'Fields: #{fields}'
161
+
162
+ return fields
163
+
164
+ rescue Exception => e
165
+ @logger.warn 'Fields: #{e.message}'
166
+ end # def get_fields
167
+ end # class LogStash::Outputs::Solr
@@ -0,0 +1,31 @@
1
+ Gem::Specification.new do |s|
2
+ s.name = 'logstash-output-custom-solr'
3
+ s.version = "0.1.0"
4
+ s.licenses = ["Apache License (2.0)"]
5
+ s.summary = "Logstash output plugin for sending data to Solr."
6
+ s.description = "Logstash output plugin for sending data to Solr. It support SolrCloud not only Standalone Solr."
7
+ s.authors = ["Minoru Osuka"]
8
+ s.email = "minoru.osuka@gmail.com"
9
+ s.homepage = "https://github.com/mosuka/logstash-output-solr"
10
+ s.require_paths = ["lib"]
11
+
12
+ # Files
13
+ s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE']
14
+ # Tests
15
+ s.test_files = s.files.grep(%r{^(test|spec|features)/})
16
+
17
+ # Special flag to let us know this is actually a logstash plugin
18
+ s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
19
+
20
+ # Gem dependencies
21
+ s.add_runtime_dependency "logstash-core", ">= 2.0.0", "<= 2.99"
22
+ s.add_runtime_dependency "logstash-codec-plain"
23
+ s.add_runtime_dependency 'rsolr', '~> 1.1.2'
24
+ s.add_runtime_dependency 'zk', '~> 1.9.6'
25
+ s.add_runtime_dependency 'rsolr-cloud', '~> 1.1.0'
26
+ s.add_runtime_dependency 'stud', '~> 0.0.22'
27
+
28
+ s.add_development_dependency "logstash-devutils", '~> 0.0.18'
29
+ s.add_development_dependency 'rake', '~> 10.5.0'
30
+ s.add_development_dependency 'zk-server', '~> 1.1.8'
31
+ end
@@ -0,0 +1,14 @@
1
+ {"collection1":{
2
+ "replicationFactor":"1",
3
+ "router":{"name":"compositeId"},
4
+ "maxShardsPerNode":"1",
5
+ "autoAddReplicas":"false",
6
+ "shards":{"shard1":{
7
+ "range":"80000000-7fffffff",
8
+ "state":"active",
9
+ "replicas":{"core_node1":{
10
+ "core":"collection1_shard1_replica1",
11
+ "base_url":"http://localhost:8983/solr",
12
+ "node_name":"localhost:8983_solr",
13
+ "state":"active",
14
+ "leader":"true"}}}}}}
@@ -0,0 +1,229 @@
1
+ # encoding: utf-8
2
+ require "logstash/devutils/rspec/spec_helper"
3
+ require "logstash/outputs/solr"
4
+ require "logstash/codecs/plain"
5
+ require "logstash/event"
6
+
7
+ require 'zk-server'
8
+ require 'zk'
9
+
10
+ describe LogStash::Outputs::Solr do
11
+
12
+ before do
13
+ @zk_server = nil
14
+ end
15
+
16
+ describe 'configuration' do
17
+ let(:config) {
18
+ {
19
+ 'url' => 'http://localhost:8983/solr/collection1',
20
+ 'zk_host' => 'localhost:2181/solr',
21
+ 'collection' => 'collection1',
22
+ 'defined_fields' => ['id', 'title'],
23
+ 'ignore_undefined_fields' => true,
24
+ 'unique_key_field' => 'id',
25
+ 'timestamp_field' => 'event_timestamp',
26
+ 'flush_size' => 100
27
+ }
28
+ }
29
+
30
+ it 'url' do
31
+ output = LogStash::Outputs::Solr.new(config)
32
+ expect(output.config['url']).to eq('http://localhost:8983/solr/collection1')
33
+ end
34
+
35
+ it 'zk_host' do
36
+ output = LogStash::Outputs::Solr.new(config)
37
+ expect(output.config['zk_host']).to eq('localhost:2181/solr')
38
+ end
39
+
40
+ it 'collection' do
41
+ output = LogStash::Outputs::Solr.new(config)
42
+ expect(output.config['collection']).to eq('collection1')
43
+ end
44
+
45
+ it 'defined_fields' do
46
+ output = LogStash::Outputs::Solr.new(config)
47
+ expect(output.config['defined_fields']).to eq(['id', 'title'])
48
+ end
49
+
50
+ it 'ignore_undefined_fields' do
51
+ output = LogStash::Outputs::Solr.new(config)
52
+ expect(output.config['ignore_undefined_fields']).to eq(true)
53
+ end
54
+
55
+ it 'unique_key_field' do
56
+ output = LogStash::Outputs::Solr.new(config)
57
+ expect(output.config['unique_key_field']).to eq('id')
58
+ end
59
+
60
+ it 'timestamp_field' do
61
+ output = LogStash::Outputs::Solr.new(config)
62
+ expect(output.config['timestamp_field']).to eq('event_timestamp')
63
+ end
64
+
65
+ it 'flush_size' do
66
+ output = LogStash::Outputs::Solr.new(config)
67
+ expect(output.config['flush_size']).to eq(100)
68
+ end
69
+ end
70
+
71
+ describe 'register_standalone' do
72
+ let(:config) {
73
+ {
74
+ 'url' => 'http://localhost:8983/solr/collection1',
75
+ 'defined_fields' => ['id', 'title'],
76
+ 'ignore_undefined_fields' => true,
77
+ 'unique_key_field' => 'id',
78
+ 'timestamp_field' => 'event_timestamp',
79
+ 'flush_size' => 100
80
+ }
81
+ }
82
+
83
+ it 'mode' do
84
+ output = LogStash::Outputs::Solr.new(config)
85
+ output.register
86
+
87
+ mode = output.instance_variable_get('@mode')
88
+ expect(mode).to eq('Standalone')
89
+ end
90
+ end
91
+
92
+ describe 'register_solrcloud' do
93
+ before do
94
+ start_zookeeper
95
+ end
96
+
97
+ after do
98
+ stop_zookeeper
99
+ end
100
+
101
+ let(:config) {
102
+ {
103
+ 'zk_host' => 'localhost:3292/solr',
104
+ 'collection' => 'collection1',
105
+ 'defined_fields' => ['id', 'title'],
106
+ 'ignore_undefined_fields' => true,
107
+ 'unique_key_field' => 'id',
108
+ 'timestamp_field' => 'event_timestamp',
109
+ 'flush_size' => 100
110
+ }
111
+ }
112
+
113
+ it 'mode' do
114
+ output = LogStash::Outputs::Solr.new(config)
115
+ output.register
116
+
117
+ mode = output.instance_variable_get('@mode')
118
+ expect(mode).to eq('SolrCloud')
119
+ end
120
+ end
121
+
122
+ describe 'receive_standalone' do
123
+ let(:config) {
124
+ {
125
+ 'url' => 'http://localhost:8983/solr/collection1',
126
+ 'defined_fields' => ['id', 'title'],
127
+ 'ignore_undefined_fields' => true,
128
+ 'unique_key_field' => 'id',
129
+ 'timestamp_field' => 'event_timestamp',
130
+ 'flush_size' => 100
131
+ }
132
+ }
133
+
134
+ let(:sample_record) {
135
+ {
136
+ 'id' => 'change.me',
137
+ 'title' => 'change.me'
138
+ }
139
+ }
140
+
141
+ it 'receive' do
142
+ output = LogStash::Outputs::Solr.new(config)
143
+ output.register
144
+
145
+ output.receive(sample_record)
146
+ end
147
+ end
148
+
149
+ describe 'receive_solrcloud' do
150
+ before do
151
+ start_zookeeper
152
+ end
153
+
154
+ after do
155
+ stop_zookeeper
156
+ end
157
+
158
+ let(:config) {
159
+ {
160
+ 'zk_host' => 'localhost:3292/solr',
161
+ 'collection' => 'collection1',
162
+ 'defined_fields' => ['id', 'title'],
163
+ 'ignore_undefined_fields' => true,
164
+ 'unique_key_field' => 'id',
165
+ 'timestamp_field' => 'event_timestamp',
166
+ 'flush_size' => 100
167
+ }
168
+ }
169
+
170
+ let(:sample_record) {
171
+ {
172
+ 'id' => 'change.me',
173
+ 'title' => 'change.me'
174
+ }
175
+ }
176
+
177
+ it 'receive' do
178
+ output = LogStash::Outputs::Solr.new(config)
179
+ output.register
180
+
181
+ output.receive(sample_record)
182
+ end
183
+ end
184
+
185
+ def start_zookeeper
186
+ @zk_server = ZK::Server.new do |config|
187
+ config.client_port = 3292
188
+ config.enable_jmx = true
189
+ config.force_sync = false
190
+ end
191
+
192
+ @zk_server.run
193
+
194
+ zk = ZK.new('localhost:3292')
195
+ delete_nodes(zk, '/solr')
196
+ create_nodes(zk, '/solr/live_nodes')
197
+ create_nodes(zk, '/solr/collections')
198
+ ['localhost:8983_solr'].each do |node|
199
+ zk.create("/solr/live_nodes/#{node}", '', mode: :ephemeral)
200
+ end
201
+ ['collection1'].each do |collection|
202
+ zk.create("/solr/collections/#{collection}")
203
+ json = File.read("spec/files/collections/#{collection}/state.json")
204
+ zk.create("/solr/collections/#{collection}/state.json", json, mode: :ephemeral)
205
+ end
206
+ end
207
+
208
+ def stop_zookeeper
209
+ @zk_server.shutdown
210
+ end
211
+
212
+ def delete_nodes(zk, path)
213
+ zk.children(path).each do |node|
214
+ delete_nodes(zk, File.join(path, node))
215
+ end
216
+ zk.delete(path)
217
+ rescue ZK::Exceptions::NoNode
218
+ end
219
+
220
+ def create_nodes(zk, path)
221
+ parent_path = File.dirname(path)
222
+ unless zk.exists?(parent_path, :watch => true) then
223
+ create_nodes(zk, parent_path)
224
+ end
225
+ zk.create(path)
226
+ end
227
+ end
228
+
229
+
metadata ADDED
@@ -0,0 +1,190 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: logstash-output-custom-solr
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Minoru Osuka
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2018-05-11 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ requirement: !ruby/object:Gem::Requirement
15
+ requirements:
16
+ - - ">="
17
+ - !ruby/object:Gem::Version
18
+ version: 2.0.0
19
+ - - "<="
20
+ - !ruby/object:Gem::Version
21
+ version: '2.99'
22
+ name: logstash-core
23
+ prerelease: false
24
+ type: :runtime
25
+ version_requirements: !ruby/object:Gem::Requirement
26
+ requirements:
27
+ - - ">="
28
+ - !ruby/object:Gem::Version
29
+ version: 2.0.0
30
+ - - "<="
31
+ - !ruby/object:Gem::Version
32
+ version: '2.99'
33
+ - !ruby/object:Gem::Dependency
34
+ requirement: !ruby/object:Gem::Requirement
35
+ requirements:
36
+ - - ">="
37
+ - !ruby/object:Gem::Version
38
+ version: '0'
39
+ name: logstash-codec-plain
40
+ prerelease: false
41
+ type: :runtime
42
+ version_requirements: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - ">="
45
+ - !ruby/object:Gem::Version
46
+ version: '0'
47
+ - !ruby/object:Gem::Dependency
48
+ requirement: !ruby/object:Gem::Requirement
49
+ requirements:
50
+ - - "~>"
51
+ - !ruby/object:Gem::Version
52
+ version: 1.1.2
53
+ name: rsolr
54
+ prerelease: false
55
+ type: :runtime
56
+ version_requirements: !ruby/object:Gem::Requirement
57
+ requirements:
58
+ - - "~>"
59
+ - !ruby/object:Gem::Version
60
+ version: 1.1.2
61
+ - !ruby/object:Gem::Dependency
62
+ requirement: !ruby/object:Gem::Requirement
63
+ requirements:
64
+ - - "~>"
65
+ - !ruby/object:Gem::Version
66
+ version: 1.9.6
67
+ name: zk
68
+ prerelease: false
69
+ type: :runtime
70
+ version_requirements: !ruby/object:Gem::Requirement
71
+ requirements:
72
+ - - "~>"
73
+ - !ruby/object:Gem::Version
74
+ version: 1.9.6
75
+ - !ruby/object:Gem::Dependency
76
+ requirement: !ruby/object:Gem::Requirement
77
+ requirements:
78
+ - - "~>"
79
+ - !ruby/object:Gem::Version
80
+ version: 1.1.0
81
+ name: rsolr-cloud
82
+ prerelease: false
83
+ type: :runtime
84
+ version_requirements: !ruby/object:Gem::Requirement
85
+ requirements:
86
+ - - "~>"
87
+ - !ruby/object:Gem::Version
88
+ version: 1.1.0
89
+ - !ruby/object:Gem::Dependency
90
+ requirement: !ruby/object:Gem::Requirement
91
+ requirements:
92
+ - - "~>"
93
+ - !ruby/object:Gem::Version
94
+ version: 0.0.22
95
+ name: stud
96
+ prerelease: false
97
+ type: :runtime
98
+ version_requirements: !ruby/object:Gem::Requirement
99
+ requirements:
100
+ - - "~>"
101
+ - !ruby/object:Gem::Version
102
+ version: 0.0.22
103
+ - !ruby/object:Gem::Dependency
104
+ requirement: !ruby/object:Gem::Requirement
105
+ requirements:
106
+ - - "~>"
107
+ - !ruby/object:Gem::Version
108
+ version: 0.0.18
109
+ name: logstash-devutils
110
+ prerelease: false
111
+ type: :development
112
+ version_requirements: !ruby/object:Gem::Requirement
113
+ requirements:
114
+ - - "~>"
115
+ - !ruby/object:Gem::Version
116
+ version: 0.0.18
117
+ - !ruby/object:Gem::Dependency
118
+ requirement: !ruby/object:Gem::Requirement
119
+ requirements:
120
+ - - "~>"
121
+ - !ruby/object:Gem::Version
122
+ version: 10.5.0
123
+ name: rake
124
+ prerelease: false
125
+ type: :development
126
+ version_requirements: !ruby/object:Gem::Requirement
127
+ requirements:
128
+ - - "~>"
129
+ - !ruby/object:Gem::Version
130
+ version: 10.5.0
131
+ - !ruby/object:Gem::Dependency
132
+ requirement: !ruby/object:Gem::Requirement
133
+ requirements:
134
+ - - "~>"
135
+ - !ruby/object:Gem::Version
136
+ version: 1.1.8
137
+ name: zk-server
138
+ prerelease: false
139
+ type: :development
140
+ version_requirements: !ruby/object:Gem::Requirement
141
+ requirements:
142
+ - - "~>"
143
+ - !ruby/object:Gem::Version
144
+ version: 1.1.8
145
+ description: Logstash output plugin for sending data to Solr. It support SolrCloud
146
+ not only Standalone Solr.
147
+ email: minoru.osuka@gmail.com
148
+ executables: []
149
+ extensions: []
150
+ extra_rdoc_files: []
151
+ files:
152
+ - CHANGELOG.md
153
+ - CONTRIBUTORS
154
+ - DEVELOPER.md
155
+ - Gemfile
156
+ - LICENSE
157
+ - README.md
158
+ - lib/logstash/outputs/solr.rb
159
+ - logstash-output-custom-solr.gemspec
160
+ - spec/files/collections/collection1/state.json
161
+ - spec/outputs/solr_spec.rb
162
+ homepage: https://github.com/mosuka/logstash-output-solr
163
+ licenses:
164
+ - Apache License (2.0)
165
+ metadata:
166
+ logstash_plugin: 'true'
167
+ logstash_group: output
168
+ post_install_message:
169
+ rdoc_options: []
170
+ require_paths:
171
+ - lib
172
+ required_ruby_version: !ruby/object:Gem::Requirement
173
+ requirements:
174
+ - - ">="
175
+ - !ruby/object:Gem::Version
176
+ version: '0'
177
+ required_rubygems_version: !ruby/object:Gem::Requirement
178
+ requirements:
179
+ - - ">="
180
+ - !ruby/object:Gem::Version
181
+ version: '0'
182
+ requirements: []
183
+ rubyforge_project:
184
+ rubygems_version: 2.6.13
185
+ signing_key:
186
+ specification_version: 4
187
+ summary: Logstash output plugin for sending data to Solr.
188
+ test_files:
189
+ - spec/files/collections/collection1/state.json
190
+ - spec/outputs/solr_spec.rb