logstash-output-custom-solr 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 71fad4671bc6f96e8852ad45c9abcf69c5cedc76b1acf2901b0817889ec5029f
4
+ data.tar.gz: e7370ff6cea53d407d8272e07c6349cf25913080d2bff4fa36368c2b69254118
5
+ SHA512:
6
+ metadata.gz: 9c5cfac43debf01dddb3e80bed0e09020509f25e63c35643d705238a153a64c61187bb0467c92427920ac1f127484e0431b2e6fc756e90c5189ccb1bafa3632a
7
+ data.tar.gz: 76420cb872fbe00a2a18d587206e4f738967c39fd2e52b92779304cf8dfa45b47b160ea90143779fbe264cb02d73ca169d448f660f07675bd2ea58000f5c2fa2
data/CHANGELOG.md ADDED
@@ -0,0 +1,11 @@
1
+ ## 0.3.0
2
+ - Update rsolr-cloud.
3
+
4
+ ## 0.2.0
5
+ - Bug fix.
6
+
7
+ ## 0.2.0
8
+ - Supports SolrCloud.
9
+
10
+ ## 0.1.0
11
+ - First release.
data/CONTRIBUTORS ADDED
@@ -0,0 +1,10 @@
1
+ The following is a list of people who have contributed ideas, code, bug
2
+ reports, or in general have helped logstash along its way.
3
+
4
+ Contributors:
5
+ * Minoru Osuka (minoru.osuka@gmail.com)
6
+
7
+ Note: If you've sent us patches, bug reports, or otherwise contributed to
8
+ Logstash, and you aren't on the list above and want to be, please let us know
9
+ and we'll make sure you're here. Contributions from folks like you are what make
10
+ open source awesome.
data/DEVELOPER.md ADDED
@@ -0,0 +1,2 @@
1
+ # logstash-output-solr
2
+ Solr output plugin.
data/Gemfile ADDED
@@ -0,0 +1,2 @@
1
+ source 'https://rubygems.org'
2
+ gemspec
data/LICENSE ADDED
@@ -0,0 +1,13 @@
1
+ Copyright (c) 2012–2015 Elasticsearch <http://www.elastic.co>
2
+
3
+ Licensed under the Apache License, Version 2.0 (the "License");
4
+ you may not use this file except in compliance with the License.
5
+ You may obtain a copy of the License at
6
+
7
+ http://www.apache.org/licenses/LICENSE-2.0
8
+
9
+ Unless required by applicable law or agreed to in writing, software
10
+ distributed under the License is distributed on an "AS IS" BASIS,
11
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ See the License for the specific language governing permissions and
13
+ limitations under the License.
data/README.md ADDED
@@ -0,0 +1,249 @@
1
+ # Logstash Plugin
2
+
3
+ This is a plugin for [Logstash](https://github.com/elastic/logstash). It support [SolrCloud](https://cwiki.apache.org/confluence/display/solr/SolrCloud) not only Standalone Solr.
4
+
5
+ It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
6
+
7
+ ## Config parameters
8
+
9
+ ### url
10
+
11
+ The Solr server url (for example http://localhost:8983/solr/collection1).
12
+
13
+ ```
14
+ url http://localhost:8983/solr/collection1
15
+ ```
16
+
17
+ ### zk_host
18
+
19
+ The ZooKeeper connection string that SolrCloud refers to (for example localhost:2181/solr).
20
+
21
+ ```
22
+ zk_host localhost:2181/solr
23
+ ```
24
+
25
+ ### collection
26
+
27
+ The SolrCloud collection name (default collection1).
28
+
29
+ ```
30
+ collection collection1
31
+ ```
32
+
33
+ ### defined_fields
34
+
35
+ The defined fields in the Solr schema.xml. If omitted, it will get fields via Solr Schema API.
36
+
37
+ ```
38
+ defined_fields ["id", "title"]
39
+ ```
40
+
41
+ ### ignore_undefined_fields
42
+
43
+ Ignore undefined fields in the Solr schema.xml.
44
+
45
+ ```
46
+ ignore_undefined_fields false
47
+ ```
48
+
49
+ ### unique_key_field
50
+
51
+ A field name of unique key in the Solr schema.xml. If omitted, it will get unique key via Solr Schema API.
52
+
53
+ ```
54
+ unique_key_field id
55
+ ```
56
+
57
+ ### timestamp_field
58
+
59
+ A field name of event timestamp in the Solr schema.xml (default event_timestamp).
60
+
61
+ ```
62
+ timestamp_field event_timestamp
63
+ ```
64
+
65
+ ### flush_size
66
+
67
+ A number of events to queue up before writing to Solr (default 100).
68
+
69
+ ```
70
+ flush_size 100
71
+ ```
72
+
73
+ ## Plugin setup examples
74
+
75
+ ### Sent to standalone Solr using data-driven schemaless mode.
76
+ ```
77
+ output {
78
+ solr {
79
+ url => "http://localhost:8983/solr/collection1"
80
+ }
81
+ }
82
+
83
+ ```
84
+
85
+ ### Sent to SolrCloud using data-driven schemaless mode.
86
+ ```
87
+ output {
88
+ solr {
89
+ zk_host => "localhost:2181/solr"
90
+ collection => "collection1"
91
+ }
92
+ }
93
+ ```
94
+
95
+ ## Solr setup examples
96
+
97
+ ### How to setup Standalone Solr using data-driven schemaless mode.
98
+
99
+ 1.Download and install Solr
100
+
101
+ ```sh
102
+ $ mkdir $HOME/solr
103
+ $ cd $HOME/solr
104
+ $ wget https://archive.apache.org/dist/lucene/solr/5.4.0/solr-5.4.0.tgz
105
+ $ tar zxvf solr-5.4.0.tgz
106
+ $ cd solr-5.4.0
107
+ ```
108
+
109
+ 2.Start standalone Solr
110
+
111
+ ```sh
112
+ $ ./bin/solr start -p 8983 -s server/solr
113
+ ```
114
+
115
+ 3.Create core
116
+
117
+ ```sh
118
+ $ ./bin/solr create -c collection1 -d server/solr/configsets/data_driven_schema_configs -n collection1_configs
119
+ ```
120
+
121
+ ### How to setup SolrCloud using data-driven schemaless mode (shards=1 and replicationfactor=2).
122
+
123
+ 1.Download and install ZooKeeper
124
+
125
+ ```sh
126
+ $ mkdir $HOME/zookeeper
127
+ $ cd $HOME/zookeeper
128
+ $ wget https://archive.apache.org/dist/zookeeper/zookeeper-3.4.6/zookeeper-3.4.6.tar.gz
129
+ $ tar zxvf zookeeper-3.4.6.tar.gz
130
+ $ cd zookeeper-3.4.6
131
+ $ cp -p ./conf/zoo_sample.cfg ./conf/zoo.cfg
132
+ ```
133
+
134
+ 2.Start standalone ZooKeeper
135
+
136
+ ```sh
137
+ $ ./bin/zkServer.sh start
138
+ ```
139
+
140
+ 3.Download an install Solr
141
+
142
+ ```sh
143
+ $ mkdir $HOME/solr
144
+ $ cd $HOME/solr
145
+ $ wget https://archive.apache.org/dist/lucene/solr/5.4.0/solr-5.4.0.tgz
146
+ $ tar zxvf solr-5.4.0.tgz
147
+ $ cd solr-5.4.0
148
+ $ ./server/scripts/cloud-scripts/zkcli.sh -zkhost localhost:2181 -cmd clear /solr
149
+ $ ./server/scripts/cloud-scripts/zkcli.sh -zkhost localhost:2181 -cmd makepath /solr
150
+ $ cp -pr server/solr server/solr1
151
+ $ cp -pr server/solr server/solr2
152
+ ```
153
+
154
+ 4.Start SolrCloud
155
+
156
+ ```sh
157
+ $ ./bin/solr start -h localhost -p 8983 -z localhost:2181/solr -s server/solr1
158
+ $ ./bin/solr start -h localhost -p 8985 -z localhost:2181/solr -s server/solr2
159
+ ```
160
+
161
+ 5.Create collection
162
+
163
+ ```sh
164
+ $ ./bin/solr create -c collection1 -d server/solr1/configsets/data_driven_schema_configs -n collection1_configs -shards 1 -replicationFactor 2
165
+ ```
166
+
167
+ ## Need Help?
168
+
169
+ Need help? Try #logstash on freenode IRC or the [https://discuss.elastic.co/c/logstash](https://discuss.elastic.co/c/logstash) discussion forum.
170
+
171
+ ## Developing
172
+
173
+ ### 1. Plugin Developement and Testing
174
+
175
+ #### Code
176
+ - To get started, you'll need JRuby with the Bundler gem installed.
177
+
178
+ - Create a new plugin or clone and existing from the GitHub [logstash-plugins](https://github.com/logstash-plugins) organization. We also provide [example plugins](https://github.com/logstash-plugins?query=example).
179
+
180
+ - Install dependencies
181
+ ```sh
182
+ bundle install
183
+ ```
184
+
185
+ #### Test
186
+
187
+ - Update your dependencies
188
+
189
+ ```sh
190
+ bundle install
191
+ ```
192
+
193
+ - Run tests
194
+
195
+ ```sh
196
+ bundle exec rspec
197
+ ```
198
+
199
+ ### 2. Running your unpublished Plugin in Logstash
200
+
201
+ #### 2.1 Run in a local Logstash clone
202
+
203
+ - Edit Logstash `Gemfile` and add the local plugin path, for example:
204
+
205
+ ```ruby
206
+ gem "logstash-output-solr", :path => "/your/local/logstash-output-solr"
207
+ ```
208
+
209
+ - Install plugin
210
+
211
+ ```sh
212
+ bin/plugin install --no-verify
213
+ ```
214
+
215
+ - Run Logstash with your plugin
216
+
217
+ ```sh
218
+ bin/logstash -e 'output {solr {}}'
219
+ ```
220
+
221
+ At this point any modifications to the plugin code will be applied to this local Logstash setup. After modifying the plugin, simply rerun Logstash.
222
+
223
+ #### 2.2 Run in an installed Logstash
224
+
225
+ You can use the same **2.1** method to run your plugin in an installed Logstash by editing its `Gemfile` and pointing the `:path` to your local plugin development directory or you can build the gem and install it using:
226
+
227
+ - Build your plugin gem
228
+
229
+ ```sh
230
+ gem build logstash-output-solr.gemspec
231
+ ```
232
+
233
+ - Install the plugin from the Logstash home
234
+
235
+ ```sh
236
+ bin/plugin install /your/local/plugin/logstash-output-solr.gem
237
+ ```
238
+
239
+ - Start Logstash and proceed to test the plugin
240
+
241
+ ## Contributing
242
+
243
+ All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin.
244
+
245
+ Programming is not a required skill. Whatever you've seen about open source and maintainers or community members saying "send patches or die" - you will not see that here.
246
+
247
+ It is more important to the community that you are able to contribute.
248
+
249
+ For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.
@@ -0,0 +1,167 @@
1
+ # encoding: utf-8
2
+ require "logstash/outputs/base"
3
+ require "logstash/namespace"
4
+
5
+ require 'securerandom'
6
+ require "stud/buffer"
7
+ require 'rsolr'
8
+ require 'zk'
9
+ require 'rsolr/cloud'
10
+
11
+ # An Solr output that send data to Apache Solr.
12
+ class LogStash::Outputs::Solr < LogStash::Outputs::Base
13
+ config_name "solr"
14
+
15
+ include Stud::Buffer
16
+
17
+ # The Solr server url (for example http://localhost:8983/solr/collection1).
18
+ config :url, :validate => :string, :default => nil
19
+
20
+ # The ZooKeeper connection string that SolrCloud refers to (for example localhost:2181/solr).
21
+ config :zk_host, :validate => :string, :default => nil
22
+ # The SolrCloud collection name.
23
+ config :collection, :validate => :string, :default => 'collection1'
24
+
25
+ # The defined fields in the Solr schema.xml. If omitted, it will get fields via Solr Schema API.
26
+ config :defined_fields, :validate => :array, :default => nil
27
+ # Ignore undefined fields in the Solr schema.xml.
28
+ config :ignore_undefined_fields, :validate => :boolean, :default => false
29
+
30
+ # A field name of unique key in the Solr schema.xml. If omitted, it will get unique key via Solr Schema API.
31
+ config :unique_key_field, :validate => :string, :default => nil
32
+ # A field name of event timestamp in the Solr schema.xml (default event_timestamp).
33
+ config :timestamp_field, :validate => :string, :default => 'event_timestamp'
34
+
35
+ # The batch size used in update.
36
+ config :flush_size, :validate => :number, :default => 100
37
+
38
+ # The batch size used in update.
39
+ config :idle_flush_time, :validate => :number, :default => 10
40
+
41
+ MODE_STANDALONE = 'Standalone'
42
+ MODE_SOLRCLOUD = 'SolrCloud'
43
+
44
+ public
45
+ def register
46
+ @mode = nil
47
+ if ! @url.nil? then
48
+ @mode = MODE_STANDALONE
49
+ elsif ! @zk_host.nil?
50
+ @mode = MODE_SOLRCLOUD
51
+ end
52
+
53
+ @solr = nil
54
+ @zk = nil
55
+
56
+ if @mode == MODE_STANDALONE then
57
+ @solr = RSolr.connect :url => @url
58
+ elsif @mode == MODE_SOLRCLOUD then
59
+ @zk = ZK.new(@zk_host)
60
+ cloud_connection = RSolr::Cloud::Connection.new(@zk)
61
+ @solr = RSolr::Client.new(cloud_connection, read_timeout: 60, open_timeout: 60)
62
+ end
63
+
64
+ buffer_initialize(
65
+ :max_items => @flush_size,
66
+ :max_interval => @idle_flush_time,
67
+ :logger => @logger
68
+ )
69
+ end # def register
70
+
71
+ public
72
+ def receive(event)
73
+ buffer_receive(event)
74
+ end # def event
75
+
76
+ public
77
+ def flush(events, close=false)
78
+ documents = []
79
+
80
+ @fields = @defined_fields.nil? || @defined_fields.empty? ? get_fields : @defined_fields
81
+
82
+ @unique_key = @unique_key_field.nil? ? get_unique_key : @unique_key_field
83
+
84
+ events.each do |event|
85
+ document = event.to_hash()
86
+
87
+ unless document.has_key?(@unique_key) then
88
+ document.merge!({@unique_key => SecureRandom.uuid})
89
+ end
90
+
91
+ unless document.has_key?(@timestamp_field) then
92
+ document.merge!({@timestamp_field => document['@timestamp']})
93
+ end
94
+
95
+ if @ignore_undefined_fields then
96
+ document.each_key do |key|
97
+ unless @fields.include?(key) then
98
+ document.delete(key)
99
+ end
100
+ end
101
+ end
102
+
103
+ @logger.info 'Record: %s' % document.inspect
104
+
105
+ documents.push(document)
106
+ end
107
+
108
+ if @mode == MODE_STANDALONE then
109
+ @solr.add documents, :params => {:commit => true}
110
+ @logger.info 'Added %d document(s) to Solr' % documents.count
111
+ elsif @mode == MODE_SOLRCLOUD then
112
+ @solr.add documents, collection: @collection, :params => {:commit => true}
113
+ @logger.info 'Added %d document(s) to Solr' % documents.count
114
+ end
115
+
116
+ rescue Exception => e
117
+ @logger.warn('An error occurred while indexing: #{e.message}')
118
+ end # def flush
119
+
120
+ public
121
+ def close
122
+ unless @zk.nil? then
123
+ @zk.close
124
+ end
125
+ end # def close
126
+
127
+ private
128
+ def get_unique_key
129
+ response = nil
130
+
131
+ if @mode == MODE_STANDALONE then
132
+ response = @solr.get 'schema/uniquekey'
133
+ elsif @mode == MODE_SOLRCLOUD then
134
+ response = @solr.get 'schema/uniquekey', collection: @collection
135
+ end
136
+
137
+ unique_key = response['uniqueKey']
138
+ @logger.info 'Unique key: #{unique_key}'
139
+
140
+ return unique_key
141
+
142
+ rescue Exception => e
143
+ @logger.warn 'Unique key: #{e.message}'
144
+ end # def get_unique_key
145
+
146
+ private
147
+ def get_fields
148
+ response = nil
149
+
150
+ if @mode == MODE_STANDALONE then
151
+ response = @solr.get 'schema/fields'
152
+ elsif @mode == MODE_SOLRCLOUD then
153
+ response = @solr.get 'schema/fields', collection: @collection
154
+ end
155
+
156
+ fields = []
157
+ response['fields'].each do |field|
158
+ fields.push(field['name'])
159
+ end
160
+ @logger.info 'Fields: #{fields}'
161
+
162
+ return fields
163
+
164
+ rescue Exception => e
165
+ @logger.warn 'Fields: #{e.message}'
166
+ end # def get_fields
167
+ end # class LogStash::Outputs::Solr
@@ -0,0 +1,31 @@
1
+ Gem::Specification.new do |s|
2
+ s.name = 'logstash-output-custom-solr'
3
+ s.version = "0.1.0"
4
+ s.licenses = ["Apache License (2.0)"]
5
+ s.summary = "Logstash output plugin for sending data to Solr."
6
+ s.description = "Logstash output plugin for sending data to Solr. It support SolrCloud not only Standalone Solr."
7
+ s.authors = ["Minoru Osuka"]
8
+ s.email = "minoru.osuka@gmail.com"
9
+ s.homepage = "https://github.com/mosuka/logstash-output-solr"
10
+ s.require_paths = ["lib"]
11
+
12
+ # Files
13
+ s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE']
14
+ # Tests
15
+ s.test_files = s.files.grep(%r{^(test|spec|features)/})
16
+
17
+ # Special flag to let us know this is actually a logstash plugin
18
+ s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
19
+
20
+ # Gem dependencies
21
+ s.add_runtime_dependency "logstash-core", ">= 2.0.0", "<= 2.99"
22
+ s.add_runtime_dependency "logstash-codec-plain"
23
+ s.add_runtime_dependency 'rsolr', '~> 1.1.2'
24
+ s.add_runtime_dependency 'zk', '~> 1.9.6'
25
+ s.add_runtime_dependency 'rsolr-cloud', '~> 1.1.0'
26
+ s.add_runtime_dependency 'stud', '~> 0.0.22'
27
+
28
+ s.add_development_dependency "logstash-devutils", '~> 0.0.18'
29
+ s.add_development_dependency 'rake', '~> 10.5.0'
30
+ s.add_development_dependency 'zk-server', '~> 1.1.8'
31
+ end
@@ -0,0 +1,14 @@
1
+ {"collection1":{
2
+ "replicationFactor":"1",
3
+ "router":{"name":"compositeId"},
4
+ "maxShardsPerNode":"1",
5
+ "autoAddReplicas":"false",
6
+ "shards":{"shard1":{
7
+ "range":"80000000-7fffffff",
8
+ "state":"active",
9
+ "replicas":{"core_node1":{
10
+ "core":"collection1_shard1_replica1",
11
+ "base_url":"http://localhost:8983/solr",
12
+ "node_name":"localhost:8983_solr",
13
+ "state":"active",
14
+ "leader":"true"}}}}}}
@@ -0,0 +1,229 @@
1
+ # encoding: utf-8
2
+ require "logstash/devutils/rspec/spec_helper"
3
+ require "logstash/outputs/solr"
4
+ require "logstash/codecs/plain"
5
+ require "logstash/event"
6
+
7
+ require 'zk-server'
8
+ require 'zk'
9
+
10
+ describe LogStash::Outputs::Solr do
11
+
12
+ before do
13
+ @zk_server = nil
14
+ end
15
+
16
+ describe 'configuration' do
17
+ let(:config) {
18
+ {
19
+ 'url' => 'http://localhost:8983/solr/collection1',
20
+ 'zk_host' => 'localhost:2181/solr',
21
+ 'collection' => 'collection1',
22
+ 'defined_fields' => ['id', 'title'],
23
+ 'ignore_undefined_fields' => true,
24
+ 'unique_key_field' => 'id',
25
+ 'timestamp_field' => 'event_timestamp',
26
+ 'flush_size' => 100
27
+ }
28
+ }
29
+
30
+ it 'url' do
31
+ output = LogStash::Outputs::Solr.new(config)
32
+ expect(output.config['url']).to eq('http://localhost:8983/solr/collection1')
33
+ end
34
+
35
+ it 'zk_host' do
36
+ output = LogStash::Outputs::Solr.new(config)
37
+ expect(output.config['zk_host']).to eq('localhost:2181/solr')
38
+ end
39
+
40
+ it 'collection' do
41
+ output = LogStash::Outputs::Solr.new(config)
42
+ expect(output.config['collection']).to eq('collection1')
43
+ end
44
+
45
+ it 'defined_fields' do
46
+ output = LogStash::Outputs::Solr.new(config)
47
+ expect(output.config['defined_fields']).to eq(['id', 'title'])
48
+ end
49
+
50
+ it 'ignore_undefined_fields' do
51
+ output = LogStash::Outputs::Solr.new(config)
52
+ expect(output.config['ignore_undefined_fields']).to eq(true)
53
+ end
54
+
55
+ it 'unique_key_field' do
56
+ output = LogStash::Outputs::Solr.new(config)
57
+ expect(output.config['unique_key_field']).to eq('id')
58
+ end
59
+
60
+ it 'timestamp_field' do
61
+ output = LogStash::Outputs::Solr.new(config)
62
+ expect(output.config['timestamp_field']).to eq('event_timestamp')
63
+ end
64
+
65
+ it 'flush_size' do
66
+ output = LogStash::Outputs::Solr.new(config)
67
+ expect(output.config['flush_size']).to eq(100)
68
+ end
69
+ end
70
+
71
+ describe 'register_standalone' do
72
+ let(:config) {
73
+ {
74
+ 'url' => 'http://localhost:8983/solr/collection1',
75
+ 'defined_fields' => ['id', 'title'],
76
+ 'ignore_undefined_fields' => true,
77
+ 'unique_key_field' => 'id',
78
+ 'timestamp_field' => 'event_timestamp',
79
+ 'flush_size' => 100
80
+ }
81
+ }
82
+
83
+ it 'mode' do
84
+ output = LogStash::Outputs::Solr.new(config)
85
+ output.register
86
+
87
+ mode = output.instance_variable_get('@mode')
88
+ expect(mode).to eq('Standalone')
89
+ end
90
+ end
91
+
92
+ describe 'register_solrcloud' do
93
+ before do
94
+ start_zookeeper
95
+ end
96
+
97
+ after do
98
+ stop_zookeeper
99
+ end
100
+
101
+ let(:config) {
102
+ {
103
+ 'zk_host' => 'localhost:3292/solr',
104
+ 'collection' => 'collection1',
105
+ 'defined_fields' => ['id', 'title'],
106
+ 'ignore_undefined_fields' => true,
107
+ 'unique_key_field' => 'id',
108
+ 'timestamp_field' => 'event_timestamp',
109
+ 'flush_size' => 100
110
+ }
111
+ }
112
+
113
+ it 'mode' do
114
+ output = LogStash::Outputs::Solr.new(config)
115
+ output.register
116
+
117
+ mode = output.instance_variable_get('@mode')
118
+ expect(mode).to eq('SolrCloud')
119
+ end
120
+ end
121
+
122
+ describe 'receive_standalone' do
123
+ let(:config) {
124
+ {
125
+ 'url' => 'http://localhost:8983/solr/collection1',
126
+ 'defined_fields' => ['id', 'title'],
127
+ 'ignore_undefined_fields' => true,
128
+ 'unique_key_field' => 'id',
129
+ 'timestamp_field' => 'event_timestamp',
130
+ 'flush_size' => 100
131
+ }
132
+ }
133
+
134
+ let(:sample_record) {
135
+ {
136
+ 'id' => 'change.me',
137
+ 'title' => 'change.me'
138
+ }
139
+ }
140
+
141
+ it 'receive' do
142
+ output = LogStash::Outputs::Solr.new(config)
143
+ output.register
144
+
145
+ output.receive(sample_record)
146
+ end
147
+ end
148
+
149
+ describe 'receive_solrcloud' do
150
+ before do
151
+ start_zookeeper
152
+ end
153
+
154
+ after do
155
+ stop_zookeeper
156
+ end
157
+
158
+ let(:config) {
159
+ {
160
+ 'zk_host' => 'localhost:3292/solr',
161
+ 'collection' => 'collection1',
162
+ 'defined_fields' => ['id', 'title'],
163
+ 'ignore_undefined_fields' => true,
164
+ 'unique_key_field' => 'id',
165
+ 'timestamp_field' => 'event_timestamp',
166
+ 'flush_size' => 100
167
+ }
168
+ }
169
+
170
+ let(:sample_record) {
171
+ {
172
+ 'id' => 'change.me',
173
+ 'title' => 'change.me'
174
+ }
175
+ }
176
+
177
+ it 'receive' do
178
+ output = LogStash::Outputs::Solr.new(config)
179
+ output.register
180
+
181
+ output.receive(sample_record)
182
+ end
183
+ end
184
+
185
+ def start_zookeeper
186
+ @zk_server = ZK::Server.new do |config|
187
+ config.client_port = 3292
188
+ config.enable_jmx = true
189
+ config.force_sync = false
190
+ end
191
+
192
+ @zk_server.run
193
+
194
+ zk = ZK.new('localhost:3292')
195
+ delete_nodes(zk, '/solr')
196
+ create_nodes(zk, '/solr/live_nodes')
197
+ create_nodes(zk, '/solr/collections')
198
+ ['localhost:8983_solr'].each do |node|
199
+ zk.create("/solr/live_nodes/#{node}", '', mode: :ephemeral)
200
+ end
201
+ ['collection1'].each do |collection|
202
+ zk.create("/solr/collections/#{collection}")
203
+ json = File.read("spec/files/collections/#{collection}/state.json")
204
+ zk.create("/solr/collections/#{collection}/state.json", json, mode: :ephemeral)
205
+ end
206
+ end
207
+
208
+ def stop_zookeeper
209
+ @zk_server.shutdown
210
+ end
211
+
212
+ def delete_nodes(zk, path)
213
+ zk.children(path).each do |node|
214
+ delete_nodes(zk, File.join(path, node))
215
+ end
216
+ zk.delete(path)
217
+ rescue ZK::Exceptions::NoNode
218
+ end
219
+
220
+ def create_nodes(zk, path)
221
+ parent_path = File.dirname(path)
222
+ unless zk.exists?(parent_path, :watch => true) then
223
+ create_nodes(zk, parent_path)
224
+ end
225
+ zk.create(path)
226
+ end
227
+ end
228
+
229
+
metadata ADDED
@@ -0,0 +1,190 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: logstash-output-custom-solr
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Minoru Osuka
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2018-05-11 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ requirement: !ruby/object:Gem::Requirement
15
+ requirements:
16
+ - - ">="
17
+ - !ruby/object:Gem::Version
18
+ version: 2.0.0
19
+ - - "<="
20
+ - !ruby/object:Gem::Version
21
+ version: '2.99'
22
+ name: logstash-core
23
+ prerelease: false
24
+ type: :runtime
25
+ version_requirements: !ruby/object:Gem::Requirement
26
+ requirements:
27
+ - - ">="
28
+ - !ruby/object:Gem::Version
29
+ version: 2.0.0
30
+ - - "<="
31
+ - !ruby/object:Gem::Version
32
+ version: '2.99'
33
+ - !ruby/object:Gem::Dependency
34
+ requirement: !ruby/object:Gem::Requirement
35
+ requirements:
36
+ - - ">="
37
+ - !ruby/object:Gem::Version
38
+ version: '0'
39
+ name: logstash-codec-plain
40
+ prerelease: false
41
+ type: :runtime
42
+ version_requirements: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - ">="
45
+ - !ruby/object:Gem::Version
46
+ version: '0'
47
+ - !ruby/object:Gem::Dependency
48
+ requirement: !ruby/object:Gem::Requirement
49
+ requirements:
50
+ - - "~>"
51
+ - !ruby/object:Gem::Version
52
+ version: 1.1.2
53
+ name: rsolr
54
+ prerelease: false
55
+ type: :runtime
56
+ version_requirements: !ruby/object:Gem::Requirement
57
+ requirements:
58
+ - - "~>"
59
+ - !ruby/object:Gem::Version
60
+ version: 1.1.2
61
+ - !ruby/object:Gem::Dependency
62
+ requirement: !ruby/object:Gem::Requirement
63
+ requirements:
64
+ - - "~>"
65
+ - !ruby/object:Gem::Version
66
+ version: 1.9.6
67
+ name: zk
68
+ prerelease: false
69
+ type: :runtime
70
+ version_requirements: !ruby/object:Gem::Requirement
71
+ requirements:
72
+ - - "~>"
73
+ - !ruby/object:Gem::Version
74
+ version: 1.9.6
75
+ - !ruby/object:Gem::Dependency
76
+ requirement: !ruby/object:Gem::Requirement
77
+ requirements:
78
+ - - "~>"
79
+ - !ruby/object:Gem::Version
80
+ version: 1.1.0
81
+ name: rsolr-cloud
82
+ prerelease: false
83
+ type: :runtime
84
+ version_requirements: !ruby/object:Gem::Requirement
85
+ requirements:
86
+ - - "~>"
87
+ - !ruby/object:Gem::Version
88
+ version: 1.1.0
89
+ - !ruby/object:Gem::Dependency
90
+ requirement: !ruby/object:Gem::Requirement
91
+ requirements:
92
+ - - "~>"
93
+ - !ruby/object:Gem::Version
94
+ version: 0.0.22
95
+ name: stud
96
+ prerelease: false
97
+ type: :runtime
98
+ version_requirements: !ruby/object:Gem::Requirement
99
+ requirements:
100
+ - - "~>"
101
+ - !ruby/object:Gem::Version
102
+ version: 0.0.22
103
+ - !ruby/object:Gem::Dependency
104
+ requirement: !ruby/object:Gem::Requirement
105
+ requirements:
106
+ - - "~>"
107
+ - !ruby/object:Gem::Version
108
+ version: 0.0.18
109
+ name: logstash-devutils
110
+ prerelease: false
111
+ type: :development
112
+ version_requirements: !ruby/object:Gem::Requirement
113
+ requirements:
114
+ - - "~>"
115
+ - !ruby/object:Gem::Version
116
+ version: 0.0.18
117
+ - !ruby/object:Gem::Dependency
118
+ requirement: !ruby/object:Gem::Requirement
119
+ requirements:
120
+ - - "~>"
121
+ - !ruby/object:Gem::Version
122
+ version: 10.5.0
123
+ name: rake
124
+ prerelease: false
125
+ type: :development
126
+ version_requirements: !ruby/object:Gem::Requirement
127
+ requirements:
128
+ - - "~>"
129
+ - !ruby/object:Gem::Version
130
+ version: 10.5.0
131
+ - !ruby/object:Gem::Dependency
132
+ requirement: !ruby/object:Gem::Requirement
133
+ requirements:
134
+ - - "~>"
135
+ - !ruby/object:Gem::Version
136
+ version: 1.1.8
137
+ name: zk-server
138
+ prerelease: false
139
+ type: :development
140
+ version_requirements: !ruby/object:Gem::Requirement
141
+ requirements:
142
+ - - "~>"
143
+ - !ruby/object:Gem::Version
144
+ version: 1.1.8
145
+ description: Logstash output plugin for sending data to Solr. It support SolrCloud
146
+ not only Standalone Solr.
147
+ email: minoru.osuka@gmail.com
148
+ executables: []
149
+ extensions: []
150
+ extra_rdoc_files: []
151
+ files:
152
+ - CHANGELOG.md
153
+ - CONTRIBUTORS
154
+ - DEVELOPER.md
155
+ - Gemfile
156
+ - LICENSE
157
+ - README.md
158
+ - lib/logstash/outputs/solr.rb
159
+ - logstash-output-custom-solr.gemspec
160
+ - spec/files/collections/collection1/state.json
161
+ - spec/outputs/solr_spec.rb
162
+ homepage: https://github.com/mosuka/logstash-output-solr
163
+ licenses:
164
+ - Apache License (2.0)
165
+ metadata:
166
+ logstash_plugin: 'true'
167
+ logstash_group: output
168
+ post_install_message:
169
+ rdoc_options: []
170
+ require_paths:
171
+ - lib
172
+ required_ruby_version: !ruby/object:Gem::Requirement
173
+ requirements:
174
+ - - ">="
175
+ - !ruby/object:Gem::Version
176
+ version: '0'
177
+ required_rubygems_version: !ruby/object:Gem::Requirement
178
+ requirements:
179
+ - - ">="
180
+ - !ruby/object:Gem::Version
181
+ version: '0'
182
+ requirements: []
183
+ rubyforge_project:
184
+ rubygems_version: 2.6.13
185
+ signing_key:
186
+ specification_version: 4
187
+ summary: Logstash output plugin for sending data to Solr.
188
+ test_files:
189
+ - spec/files/collections/collection1/state.json
190
+ - spec/outputs/solr_spec.rb