solrb 0.2.3 → 0.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/reviewdog.yml +20 -0
- data/.github/workflows/tests.yaml +37 -0
- data/.gitignore +2 -0
- data/.rubocop.yml +10 -6
- data/README.md +47 -0
- data/lib/solr.rb +21 -1
- data/lib/solr/cloud/configuration.rb +2 -1
- data/lib/solr/cloud/helper_methods.rb +1 -1
- data/lib/solr/commands.rb +3 -2
- data/lib/solr/commit/request.rb +15 -5
- data/lib/solr/configuration.rb +27 -5
- data/lib/solr/connection.rb +6 -4
- data/lib/solr/data_import/request.rb +7 -1
- data/lib/solr/delete/request.rb +7 -1
- data/lib/solr/errors/solr_connection_failed_error.rb +3 -2
- data/lib/solr/helper_methods.rb +11 -0
- data/lib/solr/indexing/request.rb +7 -1
- data/lib/solr/master_slave/configuration.rb +38 -0
- data/lib/solr/master_slave/helper_methods.rb +25 -0
- data/lib/solr/master_slave/nodes_gray_list/disabled.rb +21 -0
- data/lib/solr/master_slave/nodes_gray_list/in_memory.rb +47 -0
- data/lib/solr/query/handler.rb +8 -7
- data/lib/solr/query/http_request_builder.rb +6 -12
- data/lib/solr/query/request.rb +19 -2
- data/lib/solr/query/request/boosting/dictionary_boost_function.rb +4 -3
- data/lib/solr/query/request/facet.rb +5 -2
- data/lib/solr/request/cloud/first_shard_leader_node_selection_strategy.rb +27 -0
- data/lib/solr/request/cloud/leader_node_selection_strategy.rb +21 -0
- data/lib/solr/request/default_node_selection_strategy.rb +2 -1
- data/lib/solr/request/master_slave/master_node_selection_strategy.rb +12 -0
- data/lib/solr/request/node_selection_strategy.rb +6 -0
- data/lib/solr/request/runner.rb +18 -11
- data/lib/solr/response/parser.rb +1 -1
- data/lib/solr/support/url_helper.rb +8 -3
- data/lib/solr/version.rb +1 -1
- data/solrb.gemspec +2 -3
- metadata +25 -35
- data/.circleci/config.yml +0 -69
- data/.circleci/run-with-local-config.sh +0 -7
- data/Gemfile.lock +0 -83
- data/lib/solr/request/first_shard_leader_node_selection_strategy.rb +0 -24
- data/lib/solr/request/leader_node_selection_strategy.rb +0 -18
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ea33f82de9a9b3bb71a87821df6be65e39e1156704ed713c556752781257012a
|
4
|
+
data.tar.gz: 8b06c57da1f0948858a1ea97c8adc9e318e53f5288549eb80236d1e1c6e54a88
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c7b5cd5683b89b389d9d39a8e026a0483a2f9daa6c435a967bc93723150d8fddfc3f804496fece068fb06d9fd2732892fbe9305ca2b3876180f8d8433f270a36
|
7
|
+
data.tar.gz: bf7ca512022d6edbf347c6d0d2697aa90b1d1fc610d6dd8674cc5d78c30ce7e513f463538877857624ae002826e0fa0c3582712c53cd46835c77170a0e960ada
|
@@ -0,0 +1,20 @@
|
|
1
|
+
---
|
2
|
+
name: reviewdog
|
3
|
+
'on': [push]
|
4
|
+
jobs:
|
5
|
+
rubocop:
|
6
|
+
name: runner / rubocop
|
7
|
+
runs-on: ubuntu-latest
|
8
|
+
steps:
|
9
|
+
- uses: actions/checkout@v1
|
10
|
+
- uses: ruby/setup-ruby@v1
|
11
|
+
with:
|
12
|
+
ruby-version: 2.5.1
|
13
|
+
bundler-cache: true
|
14
|
+
- uses: reviewdog/action-rubocop@v1
|
15
|
+
with:
|
16
|
+
github_token: ${{ secrets.GITHUB_TOKEN }}
|
17
|
+
rubocop_version: '0.80.1'
|
18
|
+
rubocop_extensions: ''
|
19
|
+
reporter: github-check
|
20
|
+
fail_on_error: true
|
@@ -0,0 +1,37 @@
|
|
1
|
+
---
|
2
|
+
name: Tests
|
3
|
+
'on': [push]
|
4
|
+
|
5
|
+
env:
|
6
|
+
SOLR_URL: http://localhost:8983/solr/test-core
|
7
|
+
|
8
|
+
jobs:
|
9
|
+
rspec:
|
10
|
+
name: Rspec on Ruby ${{ matrix.ruby }}
|
11
|
+
runs-on: ubuntu-latest
|
12
|
+
strategy:
|
13
|
+
matrix:
|
14
|
+
ruby: ['2.5', '2.6', '2.7', '3.0']
|
15
|
+
services:
|
16
|
+
solr:
|
17
|
+
image: solr:7.4.0
|
18
|
+
ports:
|
19
|
+
- 8983:8983
|
20
|
+
steps:
|
21
|
+
- uses: actions/checkout@v2
|
22
|
+
|
23
|
+
- uses: ruby/setup-ruby@v1
|
24
|
+
with:
|
25
|
+
ruby-version: ${{ matrix.ruby }}
|
26
|
+
bundler-cache: true
|
27
|
+
|
28
|
+
- name: Create a test core
|
29
|
+
run: |
|
30
|
+
curl 'http://localhost:8983/solr/admin/cores?action=CREATE&name=test-core&configSet=_default'
|
31
|
+
|
32
|
+
- name: Disable field type guessing
|
33
|
+
run: |
|
34
|
+
curl http://localhost:8983/solr/test-core/config -d '{"set-user-property": {"update.autoCreateFields":"false"}}'
|
35
|
+
|
36
|
+
- name: Rspec
|
37
|
+
run: bundle exec rspec --format progress
|
data/.gitignore
CHANGED
data/.rubocop.yml
CHANGED
@@ -1,19 +1,20 @@
|
|
1
1
|
AllCops:
|
2
2
|
Exclude:
|
3
|
-
- db/schema.rb
|
4
|
-
- node_modules/**/*
|
5
3
|
- bin/*
|
4
|
+
- vendor/**/*
|
5
|
+
- pkg/**/*
|
6
6
|
TargetRubyVersion: '2.5'
|
7
|
-
|
7
|
+
DisabledByDefault: true
|
8
|
+
Layout/LineLength:
|
8
9
|
Max: 120
|
9
|
-
Documentation:
|
10
|
+
Style/Documentation:
|
10
11
|
Enabled: false
|
11
|
-
FrozenStringLiteralComment:
|
12
|
+
Style/FrozenStringLiteralComment:
|
12
13
|
Enabled: false
|
13
14
|
Metrics/MethodLength:
|
14
15
|
Max: 20
|
15
16
|
Metrics/AbcSize:
|
16
|
-
|
17
|
+
Enabled: false
|
17
18
|
# Removing Blocks Cop for RSpec
|
18
19
|
Style/BlockDelimiters:
|
19
20
|
Exclude:
|
@@ -28,3 +29,6 @@ Metrics/BlockLength:
|
|
28
29
|
- '**/*.rake'
|
29
30
|
- 'spec/**/*'
|
30
31
|
- 'config/environments/**/*'
|
32
|
+
Layout/SpaceInsideHashLiteralBraces:
|
33
|
+
Exclude:
|
34
|
+
- 'spec/**/*'
|
data/README.md
CHANGED
@@ -15,6 +15,8 @@ Object-Oriented approach to Solr in Ruby.
|
|
15
15
|
* [Single core configuration](#single-core-configuration)
|
16
16
|
* [Multiple core configuration](#multiple-core-configuration)
|
17
17
|
* [Solr Cloud](#solr-cloud)
|
18
|
+
* [Master-slave](#master-slave)
|
19
|
+
* [Gray list](#gray-list)
|
18
20
|
* [Basic Authentication](#basic-authentication)
|
19
21
|
* [Indexing](#indexing)
|
20
22
|
* [Querying](#querying)
|
@@ -164,6 +166,51 @@ on_worker_boot do
|
|
164
166
|
end
|
165
167
|
```
|
166
168
|
|
169
|
+
## Master-slave
|
170
|
+
|
171
|
+
To enable master-slave mode you must define a master url and slave url on solr config block.
|
172
|
+
In solr master-slave mode you don't need to provide a solr url (`config.url` or `ENV['SOLR_URL']`).
|
173
|
+
|
174
|
+
```ruby
|
175
|
+
Solr.configure do |config|
|
176
|
+
config.master_url = 'localhost:8983'
|
177
|
+
config.slave_url = 'localhost:8984'
|
178
|
+
# Disable select queries from master:
|
179
|
+
config.disable_read_from_master = true
|
180
|
+
# Specify Gray-list service
|
181
|
+
config.nodes_gray_list = Solr::MasterSlave::NodesGrayList::InMemory.new
|
182
|
+
end
|
183
|
+
```
|
184
|
+
|
185
|
+
If you are using puma web server in clustered mode you must call `enable_master_slave!` on `on_worker_boot`
|
186
|
+
callback to make each puma worker connect with zookeeper.
|
187
|
+
|
188
|
+
|
189
|
+
```ruby
|
190
|
+
on_worker_boot do
|
191
|
+
Solr.enable_master_slave!
|
192
|
+
end
|
193
|
+
```
|
194
|
+
|
195
|
+
### Gray list
|
196
|
+
|
197
|
+
Solrb provides two built-in services:
|
198
|
+
- `Solr::MasterSlave::NodesGrayList::Disabled` — Disabled service (default). Just does nothing.
|
199
|
+
- `Solr::MasterSlave::NodesGrayList::InMemory` — In memory service. It stores failed URLs in an instance variable, so it's not shared across threads/servers. URLs will be marked as "gray" for 5 minutes, but if all URLs are gray, the policy will try to send requests to these URLs earlier.
|
200
|
+
|
201
|
+
You are able to implement your own services with corresponding API.
|
202
|
+
|
203
|
+
## Force node URL
|
204
|
+
|
205
|
+
You can force solrb to use a specific node URL with the `with_node_url` method:
|
206
|
+
|
207
|
+
```ruby
|
208
|
+
Solr.with_node_url('http://localhost:9000') do
|
209
|
+
Solr::Query::Request.new(search_term: 'example', query_fields: query_fields).run
|
210
|
+
end
|
211
|
+
```
|
212
|
+
|
213
|
+
|
167
214
|
## Basic Authentication
|
168
215
|
|
169
216
|
Basic authentication is supported by solrb. You can enable it by providing `auth_user` and `auth_password`
|
data/lib/solr.rb
CHANGED
@@ -16,14 +16,19 @@ require 'solr/indexing/document'
|
|
16
16
|
require 'solr/indexing/request'
|
17
17
|
|
18
18
|
require 'solr/cloud/helper_methods'
|
19
|
+
require 'solr/master_slave/helper_methods'
|
20
|
+
require 'solr/helper_methods'
|
19
21
|
require 'solr/commands'
|
20
22
|
|
21
23
|
module Solr
|
22
24
|
class << self
|
23
25
|
include Solr::Commands
|
24
26
|
include Solr::Cloud::HelperMethods
|
27
|
+
include Solr::MasterSlave::HelperMethods
|
28
|
+
include Solr::HelperMethods
|
25
29
|
|
26
30
|
CURRENT_CORE_CONFIG_VARIABLE_NAME = :solrb_current_core_config
|
31
|
+
SOLR_NODE_URL_OVERRIDE_CONFIG = :solrb_node_url_override_config
|
27
32
|
|
28
33
|
attr_accessor :configuration
|
29
34
|
|
@@ -32,7 +37,11 @@ module Solr
|
|
32
37
|
def configure
|
33
38
|
yield configuration
|
34
39
|
configuration.validate!
|
35
|
-
|
40
|
+
if configuration.zookeeper_url
|
41
|
+
enable_solr_cloud!
|
42
|
+
elsif configuration.master_url
|
43
|
+
enable_master_slave!
|
44
|
+
end
|
36
45
|
configuration
|
37
46
|
end
|
38
47
|
|
@@ -49,6 +58,17 @@ module Solr
|
|
49
58
|
Thread.current[CURRENT_CORE_CONFIG_VARIABLE_NAME] = old_core_config
|
50
59
|
end
|
51
60
|
|
61
|
+
def with_node_url(url)
|
62
|
+
Thread.current[SOLR_NODE_URL_OVERRIDE_CONFIG] = url
|
63
|
+
yield
|
64
|
+
ensure
|
65
|
+
Thread.current[SOLR_NODE_URL_OVERRIDE_CONFIG] = nil
|
66
|
+
end
|
67
|
+
|
68
|
+
def node_url_override
|
69
|
+
Thread.current[SOLR_NODE_URL_OVERRIDE_CONFIG]
|
70
|
+
end
|
71
|
+
|
52
72
|
def solr_url(path = '')
|
53
73
|
Solr::Support::UrlHelper.solr_url(path)
|
54
74
|
end
|
@@ -18,7 +18,8 @@ module Solr
|
|
18
18
|
end
|
19
19
|
|
20
20
|
def build_zookeeper_connection
|
21
|
-
|
21
|
+
zookeeper_url = zookeeper_url.is_a?(Array) ? zookeeper_url.join(',') : zookeeper_url
|
22
|
+
Solr::Cloud::ZookeeperConnection.new(zookeeper_url: zookeeper_url,
|
22
23
|
zookeeper_auth_user: zookeeper_auth_user,
|
23
24
|
zookeeper_auth_password: zookeeper_auth_password)
|
24
25
|
end
|
data/lib/solr/commands.rb
CHANGED
@@ -5,8 +5,9 @@ require 'solr/data_import/request'
|
|
5
5
|
|
6
6
|
module Solr
|
7
7
|
module Commands
|
8
|
-
def commit(open_searcher: true, runner_options: nil)
|
9
|
-
Solr::Commit::Request.new.run(
|
8
|
+
def commit(open_searcher: true, optimize: false, runner_options: nil)
|
9
|
+
Solr::Commit::Request.new.run(optimize: optimize,
|
10
|
+
open_searcher: open_searcher,
|
10
11
|
runner_options: runner_options)
|
11
12
|
end
|
12
13
|
|
data/lib/solr/commit/request.rb
CHANGED
@@ -3,8 +3,8 @@ module Solr
|
|
3
3
|
class Request
|
4
4
|
PATH = '/update'.freeze
|
5
5
|
|
6
|
-
def run(open_searcher: true, runner_options: nil)
|
7
|
-
http_request = build_http_request(open_searcher)
|
6
|
+
def run(open_searcher: true, optimize: false, runner_options: nil)
|
7
|
+
http_request = build_http_request(open_searcher, optimize)
|
8
8
|
options = default_runner_options.merge(runner_options || {})
|
9
9
|
Solr::Request::Runner.call(request: http_request, **options)
|
10
10
|
end
|
@@ -12,12 +12,22 @@ module Solr
|
|
12
12
|
private
|
13
13
|
|
14
14
|
def default_runner_options
|
15
|
-
|
15
|
+
if Solr.cloud_enabled?
|
16
|
+
{ node_selection_strategy: Solr::Request::Cloud::LeaderNodeSelectionStrategy }
|
17
|
+
elsif Solr.master_slave_enabled?
|
18
|
+
{ node_selection_strategy: Solr::Request::MasterSlave::MasterNodeSelectionStrategy }
|
19
|
+
else
|
20
|
+
{}
|
21
|
+
end
|
16
22
|
end
|
17
23
|
|
18
|
-
def build_http_request(open_searcher)
|
24
|
+
def build_http_request(open_searcher, optimize)
|
19
25
|
Solr::Request::HttpRequest.new(path: PATH,
|
20
|
-
url_params: {
|
26
|
+
url_params: {
|
27
|
+
commit: true,
|
28
|
+
optimize: optimize,
|
29
|
+
openSearcher: open_searcher
|
30
|
+
},
|
21
31
|
method: :post)
|
22
32
|
end
|
23
33
|
end
|
data/lib/solr/configuration.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
require 'logger'
|
1
2
|
require 'solr/core_configuration/dynamic_field'
|
2
3
|
require 'solr/core_configuration/field'
|
3
4
|
require 'solr/core_configuration/core_config'
|
@@ -10,21 +11,29 @@ module Solr
|
|
10
11
|
class Configuration
|
11
12
|
extend Forwardable
|
12
13
|
|
13
|
-
delegate [:zookeeper_url, :zookeeper_url=, :zookeeper_auth_user=,
|
14
|
+
delegate [:zookeeper_url, :zookeeper_url=, :zookeeper_auth_user=,
|
15
|
+
:zookeeper_auth_password=] => :@cloud_configuration
|
16
|
+
delegate [:master_url, :master_url=, :slave_url, :slave_url=, :disable_read_from_master,
|
17
|
+
:disable_read_from_master=, :nodes_gray_list, :nodes_gray_list=] => :@master_slave_configuration
|
14
18
|
|
15
19
|
SOLRB_USER_AGENT_HEADER = { user_agent: "Solrb v#{Solr::VERSION}" }.freeze
|
16
20
|
|
17
21
|
attr_accessor :cores, :test_connection, :auth_user, :auth_password
|
18
22
|
|
19
|
-
attr_reader :url, :faraday_options, :cloud_configuration
|
23
|
+
attr_reader :url, :faraday_options, :faraday_configuration, :cloud_configuration,
|
24
|
+
:master_slave_configuration
|
25
|
+
|
26
|
+
attr_writer :url, :logger
|
20
27
|
|
21
28
|
def initialize
|
22
29
|
@faraday_options = {
|
23
30
|
request: { timeout: 2, open_timeout: 8 },
|
24
31
|
headers: SOLRB_USER_AGENT_HEADER
|
25
32
|
}
|
33
|
+
@faraday_configuration = nil
|
26
34
|
@cores = {}
|
27
35
|
@cloud_configuration = Solr::Cloud::Configuration.new
|
36
|
+
@master_slave_configuration = Solr::MasterSlave::Configuration.new
|
28
37
|
end
|
29
38
|
|
30
39
|
def faraday_options=(options)
|
@@ -33,8 +42,8 @@ module Solr
|
|
33
42
|
@faraday_options = options
|
34
43
|
end
|
35
44
|
|
36
|
-
def
|
37
|
-
@
|
45
|
+
def faraday_configure(&block)
|
46
|
+
@faraday_configuration = block
|
38
47
|
end
|
39
48
|
|
40
49
|
def core_config_by_name(core)
|
@@ -87,9 +96,22 @@ module Solr
|
|
87
96
|
end
|
88
97
|
|
89
98
|
def validate!
|
90
|
-
if !(url ||
|
99
|
+
if !(url ||
|
100
|
+
@cloud_configuration.zookeeper_url ||
|
101
|
+
(@master_slave_configuration.master_url && @master_slave_configuration.slave_url) ||
|
102
|
+
ENV['SOLR_URL'])
|
91
103
|
raise Solr::Errors::SolrUrlNotDefinedError
|
92
104
|
end
|
93
105
|
end
|
106
|
+
|
107
|
+
def logger
|
108
|
+
@logger || null_logger
|
109
|
+
end
|
110
|
+
|
111
|
+
private
|
112
|
+
|
113
|
+
def null_logger
|
114
|
+
@null_logger ||= ::Logger.new(IO::NULL)
|
115
|
+
end
|
94
116
|
end
|
95
117
|
end
|
data/lib/solr/connection.rb
CHANGED
@@ -3,9 +3,11 @@ module Solr
|
|
3
3
|
class Connection
|
4
4
|
INSTRUMENT_KEY = 'request.solrb'.freeze
|
5
5
|
|
6
|
-
def initialize(url, faraday_options: Solr.configuration.faraday_options
|
6
|
+
def initialize(url, faraday_options: Solr.configuration.faraday_options,
|
7
|
+
faraday_configuration: Solr.configuration.faraday_configuration)
|
7
8
|
# Allow mock the connection for testing
|
8
|
-
@raw_connection = Solr.configuration.test_connection
|
9
|
+
@raw_connection = Solr.configuration.test_connection
|
10
|
+
@raw_connection ||= build_faraday_connection(url, faraday_options, faraday_configuration)
|
9
11
|
freeze
|
10
12
|
end
|
11
13
|
|
@@ -29,8 +31,8 @@ module Solr
|
|
29
31
|
|
30
32
|
private
|
31
33
|
|
32
|
-
def build_faraday_connection(url, faraday_options)
|
33
|
-
connection = Faraday.new(url, faraday_options)
|
34
|
+
def build_faraday_connection(url, faraday_options, faraday_configuration)
|
35
|
+
connection = Faraday.new(url, faraday_options, &faraday_configuration)
|
34
36
|
if Solr.configuration.auth_user && Solr.configuration.auth_password
|
35
37
|
connection.basic_auth(Solr.configuration.auth_user, Solr.configuration.auth_password)
|
36
38
|
end
|
@@ -21,7 +21,13 @@ module Solr
|
|
21
21
|
private
|
22
22
|
|
23
23
|
def default_runner_options
|
24
|
-
|
24
|
+
if Solr.cloud_enabled?
|
25
|
+
{ node_selection_strategy: Solr::Request::Cloud::LeaderNodeSelectionStrategy }
|
26
|
+
elsif Solr.master_slave_enabled?
|
27
|
+
{ node_selection_strategy: Solr::Request::MasterSlave::MasterNodeSelectionStrategy }
|
28
|
+
else
|
29
|
+
{}
|
30
|
+
end
|
25
31
|
end
|
26
32
|
end
|
27
33
|
end
|
data/lib/solr/delete/request.rb
CHANGED
@@ -21,7 +21,13 @@ module Solr
|
|
21
21
|
private
|
22
22
|
|
23
23
|
def default_runner_options
|
24
|
-
|
24
|
+
if Solr.cloud_enabled?
|
25
|
+
{ node_selection_strategy: Solr::Request::Cloud::LeaderNodeSelectionStrategy }
|
26
|
+
elsif Solr.master_slave_enabled?
|
27
|
+
{ node_selection_strategy: Solr::Request::MasterSlave::MasterNodeSelectionStrategy }
|
28
|
+
else
|
29
|
+
{}
|
30
|
+
end
|
25
31
|
end
|
26
32
|
|
27
33
|
def build_http_request(commit)
|
@@ -1,10 +1,11 @@
|
|
1
1
|
module Solr
|
2
2
|
module Errors
|
3
3
|
class SolrConnectionFailedError < StandardError
|
4
|
-
def initialize(
|
4
|
+
def initialize(solr_url_errors)
|
5
|
+
urls_message = solr_url_errors.map { |url, error| "[#{url}] #{error}" }.join("\n")
|
5
6
|
message = <<~MESSAGE
|
6
7
|
Could not connect to any available solr instance:
|
7
|
-
#{
|
8
|
+
#{urls_message}
|
8
9
|
MESSAGE
|
9
10
|
super(message)
|
10
11
|
end
|