solrb 0.2.3 → 0.2.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/reviewdog.yml +20 -0
  3. data/.github/workflows/tests.yaml +37 -0
  4. data/.gitignore +2 -0
  5. data/.rubocop.yml +10 -6
  6. data/README.md +47 -0
  7. data/lib/solr.rb +21 -1
  8. data/lib/solr/cloud/configuration.rb +2 -1
  9. data/lib/solr/cloud/helper_methods.rb +1 -1
  10. data/lib/solr/commands.rb +3 -2
  11. data/lib/solr/commit/request.rb +15 -5
  12. data/lib/solr/configuration.rb +27 -5
  13. data/lib/solr/connection.rb +6 -4
  14. data/lib/solr/data_import/request.rb +7 -1
  15. data/lib/solr/delete/request.rb +7 -1
  16. data/lib/solr/errors/solr_connection_failed_error.rb +3 -2
  17. data/lib/solr/helper_methods.rb +11 -0
  18. data/lib/solr/indexing/request.rb +7 -1
  19. data/lib/solr/master_slave/configuration.rb +38 -0
  20. data/lib/solr/master_slave/helper_methods.rb +25 -0
  21. data/lib/solr/master_slave/nodes_gray_list/disabled.rb +21 -0
  22. data/lib/solr/master_slave/nodes_gray_list/in_memory.rb +47 -0
  23. data/lib/solr/query/handler.rb +8 -7
  24. data/lib/solr/query/http_request_builder.rb +6 -12
  25. data/lib/solr/query/request.rb +19 -2
  26. data/lib/solr/query/request/boosting/dictionary_boost_function.rb +4 -3
  27. data/lib/solr/query/request/facet.rb +5 -2
  28. data/lib/solr/request/cloud/first_shard_leader_node_selection_strategy.rb +27 -0
  29. data/lib/solr/request/cloud/leader_node_selection_strategy.rb +21 -0
  30. data/lib/solr/request/default_node_selection_strategy.rb +2 -1
  31. data/lib/solr/request/master_slave/master_node_selection_strategy.rb +12 -0
  32. data/lib/solr/request/node_selection_strategy.rb +6 -0
  33. data/lib/solr/request/runner.rb +18 -11
  34. data/lib/solr/response/parser.rb +1 -1
  35. data/lib/solr/support/url_helper.rb +8 -3
  36. data/lib/solr/version.rb +1 -1
  37. data/solrb.gemspec +2 -3
  38. metadata +25 -35
  39. data/.circleci/config.yml +0 -69
  40. data/.circleci/run-with-local-config.sh +0 -7
  41. data/Gemfile.lock +0 -83
  42. data/lib/solr/request/first_shard_leader_node_selection_strategy.rb +0 -24
  43. data/lib/solr/request/leader_node_selection_strategy.rb +0 -18
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: e9ae1ed7f5931da9955de331a59ca5c10598882b38e137136464a136c9b9b079
4
- data.tar.gz: 91b33fad87d4bc4b286d591a0b2c91d1c51a63e59e9a53f9cc8e8891592fba87
3
+ metadata.gz: ea33f82de9a9b3bb71a87821df6be65e39e1156704ed713c556752781257012a
4
+ data.tar.gz: 8b06c57da1f0948858a1ea97c8adc9e318e53f5288549eb80236d1e1c6e54a88
5
5
  SHA512:
6
- metadata.gz: 1539610c599f2493f8b3d6f0fd0c9b926b69032a2bffc0b0519aaf73c0e5175d63196657893452ce11b147a45243e648c683983049e0f685503be21ec6d60506
7
- data.tar.gz: 8d059163628f2c9220a467d6cada61e7b26cb088cc2db82d6b02f14836875f6089e5eb001de0d9dea8f3a74f92eb498bd26ac84a54e85901b7feefadc17b7b62
6
+ metadata.gz: c7b5cd5683b89b389d9d39a8e026a0483a2f9daa6c435a967bc93723150d8fddfc3f804496fece068fb06d9fd2732892fbe9305ca2b3876180f8d8433f270a36
7
+ data.tar.gz: bf7ca512022d6edbf347c6d0d2697aa90b1d1fc610d6dd8674cc5d78c30ce7e513f463538877857624ae002826e0fa0c3582712c53cd46835c77170a0e960ada
@@ -0,0 +1,20 @@
1
+ ---
2
+ name: reviewdog
3
+ 'on': [push]
4
+ jobs:
5
+ rubocop:
6
+ name: runner / rubocop
7
+ runs-on: ubuntu-latest
8
+ steps:
9
+ - uses: actions/checkout@v1
10
+ - uses: ruby/setup-ruby@v1
11
+ with:
12
+ ruby-version: 2.5.1
13
+ bundler-cache: true
14
+ - uses: reviewdog/action-rubocop@v1
15
+ with:
16
+ github_token: ${{ secrets.GITHUB_TOKEN }}
17
+ rubocop_version: '0.80.1'
18
+ rubocop_extensions: ''
19
+ reporter: github-check
20
+ fail_on_error: true
@@ -0,0 +1,37 @@
1
+ ---
2
+ name: Tests
3
+ 'on': [push]
4
+
5
+ env:
6
+ SOLR_URL: http://localhost:8983/solr/test-core
7
+
8
+ jobs:
9
+ rspec:
10
+ name: Rspec on Ruby ${{ matrix.ruby }}
11
+ runs-on: ubuntu-latest
12
+ strategy:
13
+ matrix:
14
+ ruby: ['2.5', '2.6', '2.7', '3.0']
15
+ services:
16
+ solr:
17
+ image: solr:7.4.0
18
+ ports:
19
+ - 8983:8983
20
+ steps:
21
+ - uses: actions/checkout@v2
22
+
23
+ - uses: ruby/setup-ruby@v1
24
+ with:
25
+ ruby-version: ${{ matrix.ruby }}
26
+ bundler-cache: true
27
+
28
+ - name: Create a test core
29
+ run: |
30
+ curl 'http://localhost:8983/solr/admin/cores?action=CREATE&name=test-core&configSet=_default'
31
+
32
+ - name: Disable field type guessing
33
+ run: |
34
+ curl http://localhost:8983/solr/test-core/config -d '{"set-user-property": {"update.autoCreateFields":"false"}}'
35
+
36
+ - name: Rspec
37
+ run: bundle exec rspec --format progress
data/.gitignore CHANGED
@@ -10,3 +10,5 @@
10
10
  # rspec failure tracking
11
11
  .rspec_status
12
12
  .ruby-version
13
+
14
+ Gemfile.lock
data/.rubocop.yml CHANGED
@@ -1,19 +1,20 @@
1
1
  AllCops:
2
2
  Exclude:
3
- - db/schema.rb
4
- - node_modules/**/*
5
3
  - bin/*
4
+ - vendor/**/*
5
+ - pkg/**/*
6
6
  TargetRubyVersion: '2.5'
7
- Metrics/LineLength:
7
+ DisabledByDefault: true
8
+ Layout/LineLength:
8
9
  Max: 120
9
- Documentation:
10
+ Style/Documentation:
10
11
  Enabled: false
11
- FrozenStringLiteralComment:
12
+ Style/FrozenStringLiteralComment:
12
13
  Enabled: false
13
14
  Metrics/MethodLength:
14
15
  Max: 20
15
16
  Metrics/AbcSize:
16
- Max: 20
17
+ Enabled: false
17
18
  # Removing Blocks Cop for RSpec
18
19
  Style/BlockDelimiters:
19
20
  Exclude:
@@ -28,3 +29,6 @@ Metrics/BlockLength:
28
29
  - '**/*.rake'
29
30
  - 'spec/**/*'
30
31
  - 'config/environments/**/*'
32
+ Layout/SpaceInsideHashLiteralBraces:
33
+ Exclude:
34
+ - 'spec/**/*'
data/README.md CHANGED
@@ -15,6 +15,8 @@ Object-Oriented approach to Solr in Ruby.
15
15
  * [Single core configuration](#single-core-configuration)
16
16
  * [Multiple core configuration](#multiple-core-configuration)
17
17
  * [Solr Cloud](#solr-cloud)
18
+ * [Master-slave](#master-slave)
19
+ * [Gray list](#gray-list)
18
20
  * [Basic Authentication](#basic-authentication)
19
21
  * [Indexing](#indexing)
20
22
  * [Querying](#querying)
@@ -164,6 +166,51 @@ on_worker_boot do
164
166
  end
165
167
  ```
166
168
 
169
+ ## Master-slave
170
+
171
+ To enable master-slave mode you must define a master url and slave url on solr config block.
172
+ In solr master-slave mode you don't need to provide a solr url (`config.url` or `ENV['SOLR_URL']`).
173
+
174
+ ```ruby
175
+ Solr.configure do |config|
176
+ config.master_url = 'localhost:8983'
177
+ config.slave_url = 'localhost:8984'
178
+ # Disable select queries from master:
179
+ config.disable_read_from_master = true
180
+ # Specify Gray-list service
181
+ config.nodes_gray_list = Solr::MasterSlave::NodesGrayList::InMemory.new
182
+ end
183
+ ```
184
+
185
+ If you are using puma web server in clustered mode you must call `enable_master_slave!` on `on_worker_boot`
186
+ callback to make each puma worker connect with zookeeper.
187
+
188
+
189
+ ```ruby
190
+ on_worker_boot do
191
+ Solr.enable_master_slave!
192
+ end
193
+ ```
194
+
195
+ ### Gray list
196
+
197
+ Solrb provides two built-in services:
198
+ - `Solr::MasterSlave::NodesGrayList::Disabled` — Disabled service (default). Just does nothing.
199
+ - `Solr::MasterSlave::NodesGrayList::InMemory` — In memory service. It stores failed URLs in an instance variable, so it's not shared across threads/servers. URLs will be marked as "gray" for 5 minutes, but if all URLs are gray, the policy will try to send requests to these URLs earlier.
200
+
201
+ You are able to implement your own services with corresponding API.
202
+
203
+ ## Force node URL
204
+
205
+ You can force solrb to use a specific node URL with the `with_node_url` method:
206
+
207
+ ```ruby
208
+ Solr.with_node_url('http://localhost:9000') do
209
+ Solr::Query::Request.new(search_term: 'example', query_fields: query_fields).run
210
+ end
211
+ ```
212
+
213
+
167
214
  ## Basic Authentication
168
215
 
169
216
  Basic authentication is supported by solrb. You can enable it by providing `auth_user` and `auth_password`
data/lib/solr.rb CHANGED
@@ -16,14 +16,19 @@ require 'solr/indexing/document'
16
16
  require 'solr/indexing/request'
17
17
 
18
18
  require 'solr/cloud/helper_methods'
19
+ require 'solr/master_slave/helper_methods'
20
+ require 'solr/helper_methods'
19
21
  require 'solr/commands'
20
22
 
21
23
  module Solr
22
24
  class << self
23
25
  include Solr::Commands
24
26
  include Solr::Cloud::HelperMethods
27
+ include Solr::MasterSlave::HelperMethods
28
+ include Solr::HelperMethods
25
29
 
26
30
  CURRENT_CORE_CONFIG_VARIABLE_NAME = :solrb_current_core_config
31
+ SOLR_NODE_URL_OVERRIDE_CONFIG = :solrb_node_url_override_config
27
32
 
28
33
  attr_accessor :configuration
29
34
 
@@ -32,7 +37,11 @@ module Solr
32
37
  def configure
33
38
  yield configuration
34
39
  configuration.validate!
35
- enable_solr_cloud! unless configuration.zookeeper_url.nil?
40
+ if configuration.zookeeper_url
41
+ enable_solr_cloud!
42
+ elsif configuration.master_url
43
+ enable_master_slave!
44
+ end
36
45
  configuration
37
46
  end
38
47
 
@@ -49,6 +58,17 @@ module Solr
49
58
  Thread.current[CURRENT_CORE_CONFIG_VARIABLE_NAME] = old_core_config
50
59
  end
51
60
 
61
+ def with_node_url(url)
62
+ Thread.current[SOLR_NODE_URL_OVERRIDE_CONFIG] = url
63
+ yield
64
+ ensure
65
+ Thread.current[SOLR_NODE_URL_OVERRIDE_CONFIG] = nil
66
+ end
67
+
68
+ def node_url_override
69
+ Thread.current[SOLR_NODE_URL_OVERRIDE_CONFIG]
70
+ end
71
+
52
72
  def solr_url(path = '')
53
73
  Solr::Support::UrlHelper.solr_url(path)
54
74
  end
@@ -18,7 +18,8 @@ module Solr
18
18
  end
19
19
 
20
20
  def build_zookeeper_connection
21
- Solr::Cloud::ZookeeperConnection.new(zookeeper_url: zookeeper_url.is_a?(Array) ? zookeeper_url.join(',') : zookeeper_url,
21
+ zookeeper_url = zookeeper_url.is_a?(Array) ? zookeeper_url.join(',') : zookeeper_url
22
+ Solr::Cloud::ZookeeperConnection.new(zookeeper_url: zookeeper_url,
22
23
  zookeeper_auth_user: zookeeper_auth_user,
23
24
  zookeeper_auth_password: zookeeper_auth_password)
24
25
  end
@@ -3,7 +3,7 @@ require 'solr/cloud/configuration'
3
3
  module Solr
4
4
  module Cloud
5
5
  module HelperMethods
6
- def active_nodes_for(collection:)
6
+ def cloud_active_nodes_for(collection:)
7
7
  collections_state_manager.active_nodes_for(collection: collection)
8
8
  end
9
9
 
data/lib/solr/commands.rb CHANGED
@@ -5,8 +5,9 @@ require 'solr/data_import/request'
5
5
 
6
6
  module Solr
7
7
  module Commands
8
- def commit(open_searcher: true, runner_options: nil)
9
- Solr::Commit::Request.new.run(open_searcher: open_searcher,
8
+ def commit(open_searcher: true, optimize: false, runner_options: nil)
9
+ Solr::Commit::Request.new.run(optimize: optimize,
10
+ open_searcher: open_searcher,
10
11
  runner_options: runner_options)
11
12
  end
12
13
 
@@ -3,8 +3,8 @@ module Solr
3
3
  class Request
4
4
  PATH = '/update'.freeze
5
5
 
6
- def run(open_searcher: true, runner_options: nil)
7
- http_request = build_http_request(open_searcher)
6
+ def run(open_searcher: true, optimize: false, runner_options: nil)
7
+ http_request = build_http_request(open_searcher, optimize)
8
8
  options = default_runner_options.merge(runner_options || {})
9
9
  Solr::Request::Runner.call(request: http_request, **options)
10
10
  end
@@ -12,12 +12,22 @@ module Solr
12
12
  private
13
13
 
14
14
  def default_runner_options
15
- { node_selection_strategy: Solr::Request::LeaderNodeSelectionStrategy }
15
+ if Solr.cloud_enabled?
16
+ { node_selection_strategy: Solr::Request::Cloud::LeaderNodeSelectionStrategy }
17
+ elsif Solr.master_slave_enabled?
18
+ { node_selection_strategy: Solr::Request::MasterSlave::MasterNodeSelectionStrategy }
19
+ else
20
+ {}
21
+ end
16
22
  end
17
23
 
18
- def build_http_request(open_searcher)
24
+ def build_http_request(open_searcher, optimize)
19
25
  Solr::Request::HttpRequest.new(path: PATH,
20
- url_params: { commit: true, openSearcher: open_searcher },
26
+ url_params: {
27
+ commit: true,
28
+ optimize: optimize,
29
+ openSearcher: open_searcher
30
+ },
21
31
  method: :post)
22
32
  end
23
33
  end
@@ -1,3 +1,4 @@
1
+ require 'logger'
1
2
  require 'solr/core_configuration/dynamic_field'
2
3
  require 'solr/core_configuration/field'
3
4
  require 'solr/core_configuration/core_config'
@@ -10,21 +11,29 @@ module Solr
10
11
  class Configuration
11
12
  extend Forwardable
12
13
 
13
- delegate [:zookeeper_url, :zookeeper_url=, :zookeeper_auth_user=, :zookeeper_auth_password=] => :@cloud_configuration
14
+ delegate [:zookeeper_url, :zookeeper_url=, :zookeeper_auth_user=,
15
+ :zookeeper_auth_password=] => :@cloud_configuration
16
+ delegate [:master_url, :master_url=, :slave_url, :slave_url=, :disable_read_from_master,
17
+ :disable_read_from_master=, :nodes_gray_list, :nodes_gray_list=] => :@master_slave_configuration
14
18
 
15
19
  SOLRB_USER_AGENT_HEADER = { user_agent: "Solrb v#{Solr::VERSION}" }.freeze
16
20
 
17
21
  attr_accessor :cores, :test_connection, :auth_user, :auth_password
18
22
 
19
- attr_reader :url, :faraday_options, :cloud_configuration
23
+ attr_reader :url, :faraday_options, :faraday_configuration, :cloud_configuration,
24
+ :master_slave_configuration
25
+
26
+ attr_writer :url, :logger
20
27
 
21
28
  def initialize
22
29
  @faraday_options = {
23
30
  request: { timeout: 2, open_timeout: 8 },
24
31
  headers: SOLRB_USER_AGENT_HEADER
25
32
  }
33
+ @faraday_configuration = nil
26
34
  @cores = {}
27
35
  @cloud_configuration = Solr::Cloud::Configuration.new
36
+ @master_slave_configuration = Solr::MasterSlave::Configuration.new
28
37
  end
29
38
 
30
39
  def faraday_options=(options)
@@ -33,8 +42,8 @@ module Solr
33
42
  @faraday_options = options
34
43
  end
35
44
 
36
- def url=(value)
37
- @url = value
45
+ def faraday_configure(&block)
46
+ @faraday_configuration = block
38
47
  end
39
48
 
40
49
  def core_config_by_name(core)
@@ -87,9 +96,22 @@ module Solr
87
96
  end
88
97
 
89
98
  def validate!
90
- if !(url || @cloud_configuration.zookeeper_url || ENV['SOLR_URL'])
99
+ if !(url ||
100
+ @cloud_configuration.zookeeper_url ||
101
+ (@master_slave_configuration.master_url && @master_slave_configuration.slave_url) ||
102
+ ENV['SOLR_URL'])
91
103
  raise Solr::Errors::SolrUrlNotDefinedError
92
104
  end
93
105
  end
106
+
107
+ def logger
108
+ @logger || null_logger
109
+ end
110
+
111
+ private
112
+
113
+ def null_logger
114
+ @null_logger ||= ::Logger.new(IO::NULL)
115
+ end
94
116
  end
95
117
  end
@@ -3,9 +3,11 @@ module Solr
3
3
  class Connection
4
4
  INSTRUMENT_KEY = 'request.solrb'.freeze
5
5
 
6
- def initialize(url, faraday_options: Solr.configuration.faraday_options)
6
+ def initialize(url, faraday_options: Solr.configuration.faraday_options,
7
+ faraday_configuration: Solr.configuration.faraday_configuration)
7
8
  # Allow mock the connection for testing
8
- @raw_connection = Solr.configuration.test_connection || build_faraday_connection(url, faraday_options)
9
+ @raw_connection = Solr.configuration.test_connection
10
+ @raw_connection ||= build_faraday_connection(url, faraday_options, faraday_configuration)
9
11
  freeze
10
12
  end
11
13
 
@@ -29,8 +31,8 @@ module Solr
29
31
 
30
32
  private
31
33
 
32
- def build_faraday_connection(url, faraday_options)
33
- connection = Faraday.new(url, faraday_options)
34
+ def build_faraday_connection(url, faraday_options, faraday_configuration)
35
+ connection = Faraday.new(url, faraday_options, &faraday_configuration)
34
36
  if Solr.configuration.auth_user && Solr.configuration.auth_password
35
37
  connection.basic_auth(Solr.configuration.auth_user, Solr.configuration.auth_password)
36
38
  end
@@ -21,7 +21,13 @@ module Solr
21
21
  private
22
22
 
23
23
  def default_runner_options
24
- { node_selection_strategy: Solr::Request::LeaderNodeSelectionStrategy }
24
+ if Solr.cloud_enabled?
25
+ { node_selection_strategy: Solr::Request::Cloud::LeaderNodeSelectionStrategy }
26
+ elsif Solr.master_slave_enabled?
27
+ { node_selection_strategy: Solr::Request::MasterSlave::MasterNodeSelectionStrategy }
28
+ else
29
+ {}
30
+ end
25
31
  end
26
32
  end
27
33
  end
@@ -21,7 +21,13 @@ module Solr
21
21
  private
22
22
 
23
23
  def default_runner_options
24
- { node_selection_strategy: Solr::Request::LeaderNodeSelectionStrategy }
24
+ if Solr.cloud_enabled?
25
+ { node_selection_strategy: Solr::Request::Cloud::LeaderNodeSelectionStrategy }
26
+ elsif Solr.master_slave_enabled?
27
+ { node_selection_strategy: Solr::Request::MasterSlave::MasterNodeSelectionStrategy }
28
+ else
29
+ {}
30
+ end
25
31
  end
26
32
 
27
33
  def build_http_request(commit)
@@ -1,10 +1,11 @@
1
1
  module Solr
2
2
  module Errors
3
3
  class SolrConnectionFailedError < StandardError
4
- def initialize(solr_urls)
4
+ def initialize(solr_url_errors)
5
+ urls_message = solr_url_errors.map { |url, error| "[#{url}] #{error}" }.join("\n")
5
6
  message = <<~MESSAGE
6
7
  Could not connect to any available solr instance:
7
- #{solr_urls.join(', ')}
8
+ #{urls_message}
8
9
  MESSAGE
9
10
  super(message)
10
11
  end
@@ -0,0 +1,11 @@
1
+ module Solr
2
+ module HelperMethods
3
+ def active_nodes_for(collection:)
4
+ if cloud_enabled?
5
+ cloud_active_nodes_for(collection: collection)
6
+ elsif master_slave_enabled?
7
+ master_slave_active_nodes_for(collection: collection)
8
+ end
9
+ end
10
+ end
11
+ end