thinking-sphinx 3.3.0 → 3.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -0
  3. data/.travis.yml +29 -20
  4. data/Appraisals +9 -5
  5. data/Gemfile +8 -3
  6. data/HISTORY +24 -0
  7. data/README.textile +5 -4
  8. data/bin/console +14 -0
  9. data/bin/literals +9 -0
  10. data/bin/loadsphinx +38 -0
  11. data/lib/thinking_sphinx.rb +15 -2
  12. data/lib/thinking_sphinx/active_record/callbacks/delta_callbacks.rb +2 -3
  13. data/lib/thinking_sphinx/active_record/callbacks/update_callbacks.rb +11 -1
  14. data/lib/thinking_sphinx/active_record/index.rb +1 -1
  15. data/lib/thinking_sphinx/active_record/join_association.rb +3 -1
  16. data/lib/thinking_sphinx/active_record/log_subscriber.rb +5 -0
  17. data/lib/thinking_sphinx/active_record/sql_source.rb +1 -1
  18. data/lib/thinking_sphinx/attribute_types.rb +70 -0
  19. data/lib/thinking_sphinx/commands/base.rb +41 -0
  20. data/lib/thinking_sphinx/commands/configure.rb +13 -0
  21. data/lib/thinking_sphinx/commands/index.rb +11 -0
  22. data/lib/thinking_sphinx/commands/start_attached.rb +20 -0
  23. data/lib/thinking_sphinx/commands/start_detached.rb +19 -0
  24. data/lib/thinking_sphinx/commands/stop.rb +22 -0
  25. data/lib/thinking_sphinx/configuration.rb +36 -28
  26. data/lib/thinking_sphinx/configuration/minimum_fields.rb +11 -8
  27. data/lib/thinking_sphinx/connection.rb +5 -122
  28. data/lib/thinking_sphinx/connection/client.rb +48 -0
  29. data/lib/thinking_sphinx/connection/jruby.rb +53 -0
  30. data/lib/thinking_sphinx/connection/mri.rb +28 -0
  31. data/lib/thinking_sphinx/core/index.rb +11 -0
  32. data/lib/thinking_sphinx/deletion.rb +6 -2
  33. data/lib/thinking_sphinx/deltas/default_delta.rb +1 -1
  34. data/lib/thinking_sphinx/deltas/delete_job.rb +14 -4
  35. data/lib/thinking_sphinx/distributed/index.rb +10 -0
  36. data/lib/thinking_sphinx/errors.rb +1 -1
  37. data/lib/thinking_sphinx/index_set.rb +14 -2
  38. data/lib/thinking_sphinx/interfaces/daemon.rb +32 -0
  39. data/lib/thinking_sphinx/interfaces/real_time.rb +41 -0
  40. data/lib/thinking_sphinx/interfaces/sql.rb +41 -0
  41. data/lib/thinking_sphinx/middlewares.rb +5 -3
  42. data/lib/thinking_sphinx/middlewares/active_record_translator.rb +13 -6
  43. data/lib/thinking_sphinx/middlewares/attribute_typer.rb +48 -0
  44. data/lib/thinking_sphinx/middlewares/valid_options.rb +23 -0
  45. data/lib/thinking_sphinx/rake_interface.rb +10 -124
  46. data/lib/thinking_sphinx/search.rb +11 -0
  47. data/lib/thinking_sphinx/search/query.rb +7 -1
  48. data/lib/thinking_sphinx/tasks.rb +80 -21
  49. data/lib/thinking_sphinx/with_output.rb +11 -0
  50. data/spec/acceptance/connection_spec.rb +4 -4
  51. data/spec/acceptance/searching_within_a_model_spec.rb +7 -0
  52. data/spec/acceptance/specifying_sql_spec.rb +26 -8
  53. data/spec/acceptance/sql_deltas_spec.rb +12 -0
  54. data/spec/internal/app/indices/album_index.rb +3 -0
  55. data/spec/internal/app/models/album.rb +19 -0
  56. data/spec/internal/db/schema.rb +8 -0
  57. data/spec/spec_helper.rb +4 -0
  58. data/spec/support/json_column.rb +5 -1
  59. data/spec/thinking_sphinx/active_record/callbacks/update_callbacks_spec.rb +5 -1
  60. data/spec/thinking_sphinx/active_record/sql_source_spec.rb +6 -0
  61. data/spec/thinking_sphinx/attribute_types_spec.rb +50 -0
  62. data/spec/thinking_sphinx/commands/configure_spec.rb +29 -0
  63. data/spec/thinking_sphinx/commands/index_spec.rb +26 -0
  64. data/spec/thinking_sphinx/commands/start_detached_spec.rb +55 -0
  65. data/spec/thinking_sphinx/commands/stop_spec.rb +54 -0
  66. data/spec/thinking_sphinx/configuration/minimum_fields_spec.rb +36 -0
  67. data/spec/thinking_sphinx/deletion_spec.rb +2 -5
  68. data/spec/thinking_sphinx/deltas/default_delta_spec.rb +1 -1
  69. data/spec/thinking_sphinx/errors_spec.rb +7 -0
  70. data/spec/thinking_sphinx/index_set_spec.rb +30 -7
  71. data/spec/thinking_sphinx/interfaces/daemon_spec.rb +52 -0
  72. data/spec/thinking_sphinx/interfaces/real_time_spec.rb +109 -0
  73. data/spec/thinking_sphinx/interfaces/sql_spec.rb +98 -0
  74. data/spec/thinking_sphinx/middlewares/attribute_typer_spec.rb +42 -0
  75. data/spec/thinking_sphinx/middlewares/valid_options_spec.rb +49 -0
  76. data/spec/thinking_sphinx/rake_interface_spec.rb +13 -246
  77. data/spec/thinking_sphinx/search/query_spec.rb +7 -0
  78. data/thinking-sphinx.gemspec +5 -4
  79. metadata +72 -16
  80. data/gemfiles/.gitignore +0 -1
  81. data/gemfiles/rails_3_2.gemfile +0 -13
  82. data/gemfiles/rails_4_0.gemfile +0 -13
  83. data/gemfiles/rails_4_1.gemfile +0 -13
  84. data/gemfiles/rails_4_2.gemfile +0 -13
  85. data/gemfiles/rails_5_0.gemfile +0 -12
@@ -41,6 +41,18 @@ describe 'SQL delta indexing', :live => true do
41
41
  expect(Book.search('Harry')).to be_empty
42
42
  end
43
43
 
44
+ it "does not match on old values with alternative ids" do
45
+ album = Album.create :name => 'Eternal Nightcap', :artist => 'The Whitloms'
46
+ index
47
+
48
+ expect(Album.search('Whitloms').to_a).to eq([album])
49
+
50
+ album.reload.update_attributes(:artist => 'The Whitlams')
51
+ sleep 0.25
52
+
53
+ expect(Book.search('Whitloms')).to be_empty
54
+ end
55
+
44
56
  it "automatically indexes new records of subclasses" do
45
57
  book = Hardcover.create(
46
58
  :title => 'American Gods', :author => 'Neil Gaiman'
@@ -0,0 +1,3 @@
1
+ ThinkingSphinx::Index.define :album, :with => :active_record, :primary_key => :integer_id, :delta => true do
2
+ indexes name, artist
3
+ end
@@ -0,0 +1,19 @@
1
+ class Album < ActiveRecord::Base
2
+ self.primary_key = :id
3
+
4
+ before_validation :set_id, :on => :create
5
+ before_validation :set_integer_id, :on => :create
6
+
7
+ validates :id, :presence => true, :uniqueness => true
8
+ validates :integer_id, :presence => true, :uniqueness => true
9
+
10
+ private
11
+
12
+ def set_id
13
+ self.id = (Album.maximum(:id) || "a").next
14
+ end
15
+
16
+ def set_integer_id
17
+ self.integer_id = (Album.maximum(:integer_id) || 0) + 1
18
+ end
19
+ end
@@ -4,6 +4,14 @@ ActiveRecord::Schema.define do
4
4
  t.timestamps null: false
5
5
  end
6
6
 
7
+ create_table(:albums, :force => true, :id => false) do |t|
8
+ t.string :id
9
+ t.integer :integer_id
10
+ t.string :name
11
+ t.string :artist
12
+ t.boolean :delta, :default => true, :null => false
13
+ end
14
+
7
15
  create_table(:animals, :force => true) do |t|
8
16
  t.string :name
9
17
  t.string :type
data/spec/spec_helper.rb CHANGED
@@ -18,4 +18,8 @@ RSpec.configure do |config|
18
18
  # enable filtering for examples
19
19
  config.filter_run :wip => nil
20
20
  config.run_all_when_everything_filtered = true
21
+
22
+ config.around :each, :live do |example|
23
+ example.run_with_retry :retry => 3
24
+ end
21
25
  end
@@ -6,7 +6,7 @@ class JSONColumn
6
6
  end
7
7
 
8
8
  def call
9
- sphinx? && postgresql? && column?
9
+ ruby? && sphinx? && postgresql? && column?
10
10
  end
11
11
 
12
12
  private
@@ -27,6 +27,10 @@ class JSONColumn
27
27
  ENV['DATABASE'] == 'postgresql'
28
28
  end
29
29
 
30
+ def ruby?
31
+ RUBY_PLATFORM != 'java'
32
+ end
33
+
30
34
  def sphinx?
31
35
  ENV['SPHINX_VERSION'].nil? || ENV['SPHINX_VERSION'].to_f > 2.0
32
36
  end
@@ -40,7 +40,11 @@ describe ThinkingSphinx::ActiveRecord::Callbacks::UpdateCallbacks do
40
40
  double(:name => 'baz', :updateable? => false)
41
41
  ])
42
42
 
43
- allow(instance).to receive_messages :changed => ['bar_column', 'baz'], :bar_column => 7
43
+ allow(instance).to receive_messages(
44
+ :changed => ['bar_column', 'baz'],
45
+ :bar_column => 7,
46
+ :saved_changes => {'bar_column' => [1, 2], 'baz' => [3, 4]}
47
+ )
44
48
  end
45
49
 
46
50
  it "does not send any updates to Sphinx if updates are disabled" do
@@ -159,6 +159,12 @@ describe ThinkingSphinx::ActiveRecord::SQLSource do
159
159
  expect(source.options[:utf8?]).to be_truthy
160
160
  end
161
161
 
162
+ it "sets utf8? to true if the database encoding starts with utf8" do
163
+ db_config[:encoding] = 'utf8mb4'
164
+
165
+ expect(source.options[:utf8?]).to be_truthy
166
+ end
167
+
162
168
  describe "#primary key" do
163
169
  let(:model) { double('model', :connection => connection,
164
170
  :name => 'User', :column_names => [], :inheritance_column => 'type') }
@@ -0,0 +1,50 @@
1
+ require 'spec_helper'
2
+
3
+ RSpec.describe ThinkingSphinx::AttributeTypes do
4
+ let(:configuration) {
5
+ double('configuration', :configuration_file => 'sphinx.conf')
6
+ }
7
+
8
+ before :each do
9
+ allow(ThinkingSphinx::Configuration).to receive(:instance).
10
+ and_return(configuration)
11
+
12
+ allow(File).to receive(:exist?).with('sphinx.conf').and_return(true)
13
+ allow(File).to receive(:read).with('sphinx.conf').and_return(<<-CONF)
14
+ index plain_index
15
+ {
16
+ source = plain_source
17
+ }
18
+
19
+ source plain_source
20
+ {
21
+ type = mysql
22
+ sql_attr_uint = customer_id
23
+ sql_attr_float = price
24
+ sql_attr_multi = uint comment_ids from field
25
+ }
26
+
27
+ index rt_index
28
+ {
29
+ type = rt
30
+ rt_attr_uint = user_id
31
+ rt_attr_multi = comment_ids
32
+ }
33
+ CONF
34
+ end
35
+
36
+ it 'returns an empty hash if no configuration file exists' do
37
+ allow(File).to receive(:exist?).with('sphinx.conf').and_return(false)
38
+
39
+ expect(ThinkingSphinx::AttributeTypes.new.call).to eq({})
40
+ end
41
+
42
+ it 'returns all known attributes' do
43
+ expect(ThinkingSphinx::AttributeTypes.new.call).to eq({
44
+ 'customer_id' => [:uint],
45
+ 'price' => [:float],
46
+ 'comment_ids' => [:uint],
47
+ 'user_id' => [:uint]
48
+ })
49
+ end
50
+ end
@@ -0,0 +1,29 @@
1
+ require 'spec_helper'
2
+
3
+ RSpec.describe ThinkingSphinx::Commands::Configure do
4
+ let(:command) { ThinkingSphinx::Commands::Configure.new(
5
+ configuration, {}, stream
6
+ ) }
7
+ let(:configuration) { double 'configuration' }
8
+ let(:stream) { double :puts => nil }
9
+
10
+ before :each do
11
+ allow(configuration).to receive_messages(
12
+ :configuration_file => '/path/to/foo.conf',
13
+ :render_to_file => true
14
+ )
15
+ end
16
+
17
+ it "renders the configuration to a file" do
18
+ expect(configuration).to receive(:render_to_file)
19
+
20
+ command.call
21
+ end
22
+
23
+ it "prints a message stating the file is being generated" do
24
+ expect(stream).to receive(:puts).
25
+ with('Generating configuration to /path/to/foo.conf')
26
+
27
+ command.call
28
+ end
29
+ end
@@ -0,0 +1,26 @@
1
+ require 'spec_helper'
2
+
3
+ RSpec.describe ThinkingSphinx::Commands::Index do
4
+ let(:command) { ThinkingSphinx::Commands::Index.new(
5
+ configuration, {:verbose => true}, stream
6
+ ) }
7
+ let(:configuration) { double 'configuration', :controller => controller }
8
+ let(:controller) { double 'controller', :index => true }
9
+ let(:stream) { double :puts => nil }
10
+
11
+ it "indexes all indices verbosely" do
12
+ expect(controller).to receive(:index).with(:verbose => true)
13
+
14
+ command.call
15
+ end
16
+
17
+ it "does not index verbosely if requested" do
18
+ command = ThinkingSphinx::Commands::Index.new(
19
+ configuration, {:verbose => false}, stream
20
+ )
21
+
22
+ expect(controller).to receive(:index).with(:verbose => false)
23
+
24
+ command.call
25
+ end
26
+ end
@@ -0,0 +1,55 @@
1
+ require 'spec_helper'
2
+
3
+ RSpec.describe ThinkingSphinx::Commands::StartDetached do
4
+ let(:command) {
5
+ ThinkingSphinx::Commands::StartDetached.new(configuration, {}, stream)
6
+ }
7
+ let(:configuration) { double 'configuration', :controller => controller }
8
+ let(:controller) { double 'controller', :start => result, :pid => 101 }
9
+ let(:result) { double 'result', :command => 'start', :status => 1,
10
+ :output => '' }
11
+ let(:stream) { double :puts => nil }
12
+
13
+ before :each do
14
+ allow(controller).to receive(:running?).and_return(true)
15
+ allow(configuration).to receive_messages(
16
+ :indices_location => 'my/index/files',
17
+ :searchd => double(:log => '/path/to/log')
18
+ )
19
+ allow(command).to receive(:exit).and_return(true)
20
+
21
+ allow(FileUtils).to receive_messages :mkdir_p => true
22
+ end
23
+
24
+ it "creates the index files directory" do
25
+ expect(FileUtils).to receive(:mkdir_p).with('my/index/files')
26
+
27
+ command.call
28
+ end
29
+
30
+ it "starts the daemon" do
31
+ expect(controller).to receive(:start)
32
+
33
+ command.call
34
+ end
35
+
36
+ it "prints a success message if the daemon has started" do
37
+ allow(controller).to receive(:running?).and_return(true)
38
+
39
+ expect(stream).to receive(:puts).
40
+ with('Started searchd successfully (pid: 101).')
41
+
42
+ command.call
43
+ end
44
+
45
+ it "prints a failure message if the daemon does not start" do
46
+ allow(controller).to receive(:running?).and_return(false)
47
+ allow(command).to receive(:exit)
48
+
49
+ expect(stream).to receive(:puts) do |string|
50
+ expect(string).to match('The Sphinx start command failed')
51
+ end
52
+
53
+ command.call
54
+ end
55
+ end
@@ -0,0 +1,54 @@
1
+ require 'spec_helper'
2
+
3
+ RSpec.describe ThinkingSphinx::Commands::Stop do
4
+ let(:command) {
5
+ ThinkingSphinx::Commands::Stop.new(configuration, {}, stream)
6
+ }
7
+ let(:configuration) { double 'configuration', :controller => controller }
8
+ let(:controller) { double 'controller', :stop => true, :pid => 101 }
9
+ let(:stream) { double :puts => nil }
10
+
11
+ before :each do
12
+ allow(controller).to receive(:running?).and_return(true, true, false)
13
+ end
14
+
15
+ it "prints a message if the daemon is not already running" do
16
+ allow(controller).to receive_messages :running? => false
17
+
18
+ expect(stream).to receive(:puts).with('searchd is not currently running.').
19
+ and_return(nil)
20
+ expect(stream).to_not receive(:puts).
21
+ with('"Stopped searchd daemon (pid: ).')
22
+
23
+ command.call
24
+ end
25
+
26
+ it "does not try to stop the daemon if it's not running" do
27
+ allow(controller).to receive_messages :running? => false
28
+
29
+ expect(controller).to_not receive(:stop)
30
+
31
+ command.call
32
+ end
33
+
34
+ it "stops the daemon" do
35
+ expect(controller).to receive(:stop)
36
+
37
+ command.call
38
+ end
39
+
40
+ it "prints a message informing the daemon has stopped" do
41
+ expect(stream).to receive(:puts).with('Stopped searchd daemon (pid: 101).')
42
+
43
+ command.call
44
+ end
45
+
46
+ it "should retry stopping the daemon until it stops" do
47
+ allow(controller).to receive(:running?).
48
+ and_return(true, true, true, false)
49
+
50
+ expect(controller).to receive(:stop).twice
51
+
52
+ command.call
53
+ end
54
+ end
@@ -0,0 +1,36 @@
1
+ require 'spec_helper'
2
+
3
+ RSpec.describe ThinkingSphinx::Configuration::MinimumFields do
4
+ let(:indices) { [index_a, index_b] }
5
+ let(:index_a) { double 'Index A', :model => model_a, :type => 'plain',
6
+ :sources => [double(:fields => [field_a1, field_a2])] }
7
+ let(:index_b) { double 'Index B', :model => model_a, :type => 'rt',
8
+ :fields => [field_b1, field_b2] }
9
+ let(:field_a1) { double :name => 'sphinx_internal_class_name' }
10
+ let(:field_a2) { double :name => 'name' }
11
+ let(:field_b1) { double :name => 'sphinx_internal_class_name' }
12
+ let(:field_b2) { double :name => 'name' }
13
+ let(:model_a) { double :inheritance_column => 'type' }
14
+ let(:model_b) { double :inheritance_column => 'type' }
15
+ let(:subject) { ThinkingSphinx::Configuration::MinimumFields.new indices }
16
+
17
+ it 'removes the class name fields when no index models have type columns' do
18
+ allow(model_a).to receive(:column_names).and_return(['id', 'name'])
19
+ allow(model_b).to receive(:column_names).and_return(['id', 'name'])
20
+
21
+ subject.reconcile
22
+
23
+ expect(index_a.sources.first.fields).to eq([field_a2])
24
+ expect(index_b.fields).to eq([field_b2])
25
+ end
26
+
27
+ it 'keeps the class name fields when one index model has a type column' do
28
+ allow(model_a).to receive(:column_names).and_return(['id', 'name', 'type'])
29
+ allow(model_b).to receive(:column_names).and_return(['id', 'name'])
30
+
31
+ subject.reconcile
32
+
33
+ expect(index_a.sources.first.fields).to eq([field_a1, field_a2])
34
+ expect(index_b.fields).to eq([field_b1, field_b2])
35
+ end
36
+ end
@@ -13,11 +13,8 @@ describe ThinkingSphinx::Deletion do
13
13
 
14
14
  context 'index is SQL-backed' do
15
15
  it "updates the deleted flag to false" do
16
- expect(connection).to receive(:execute).with <<-SQL
17
- UPDATE foo_core
18
- SET sphinx_deleted = 1
19
- WHERE id IN (14)
20
- SQL
16
+ expect(connection).to receive(:execute).
17
+ with('UPDATE foo_core SET sphinx_deleted = 1 WHERE id IN (14)')
21
18
 
22
19
  ThinkingSphinx::Deletion.perform index, 7
23
20
  end
@@ -21,7 +21,7 @@ describe ThinkingSphinx::Deltas::DefaultDelta do
21
21
  describe '#delete' do
22
22
  let(:connection) { double('connection', :execute => nil) }
23
23
  let(:index) { double('index', :name => 'foo_core',
24
- :document_id_for_key => 14) }
24
+ :document_id_for_instance => 14) }
25
25
  let(:instance) { double('instance', :id => 7) }
26
26
 
27
27
  before :each do
@@ -19,6 +19,13 @@ describe ThinkingSphinx::SphinxError do
19
19
  to be_a(ThinkingSphinx::ParseError)
20
20
  end
21
21
 
22
+ it "translates 'query is non-computable' errors" do
23
+ allow(error).to receive_messages :message => 'index model_core: query is non-computable (single NOT operator)'
24
+
25
+ expect(ThinkingSphinx::SphinxError.new_from_mysql(error)).
26
+ to be_a(ThinkingSphinx::ParseError)
27
+ end
28
+
22
29
  it "translates query errors" do
23
30
  allow(error).to receive_messages :message => 'index foo: query error: something is wrong'
24
31
 
@@ -15,9 +15,15 @@ describe ThinkingSphinx::IndexSet do
15
15
  stub_const 'ActiveRecord::Base', ar_base
16
16
  end
17
17
 
18
- def class_double(name, *superclasses)
18
+ def class_double(name, methods = {}, *superclasses)
19
19
  klass = double 'class', :name => name, :class => Class
20
- allow(klass).to receive_messages :ancestors => ([klass] + superclasses + [ar_base])
20
+
21
+ allow(klass).to receive_messages(
22
+ :ancestors => ([klass] + superclasses + [ar_base]),
23
+ :inheritance_column => :type
24
+ )
25
+ allow(klass).to receive_messages(methods)
26
+
21
27
  klass
22
28
  end
23
29
 
@@ -47,25 +53,42 @@ describe ThinkingSphinx::IndexSet do
47
53
  double(:reference => :page, :distributed? => false)
48
54
  ]
49
55
 
50
- options[:classes] = [class_double('Article')]
56
+ options[:classes] = [class_double('Article', :column_names => [])]
51
57
 
52
58
  expect(set.to_a.length).to eq(1)
53
59
  end
54
60
 
55
- it "requests indices for any superclasses" do
61
+ it "requests indices for any STI superclasses" do
56
62
  configuration.indices.replace [
57
63
  double(:reference => :article, :distributed? => false),
58
64
  double(:reference => :opinion_article, :distributed? => false),
59
65
  double(:reference => :page, :distributed? => false)
60
66
  ]
61
67
 
62
- options[:classes] = [
63
- class_double('OpinionArticle', class_double('Article'))
64
- ]
68
+ article = class_double('Article', :column_names => [:type])
69
+ opinion = class_double('OpinionArticle', {:column_names => [:type]},
70
+ article)
71
+
72
+ options[:classes] = [opinion]
65
73
 
66
74
  expect(set.to_a.length).to eq(2)
67
75
  end
68
76
 
77
+ it "does not use MTI superclasses" do
78
+ configuration.indices.replace [
79
+ double(:reference => :article, :distributed? => false),
80
+ double(:reference => :opinion_article, :distributed? => false),
81
+ double(:reference => :page, :distributed? => false)
82
+ ]
83
+
84
+ article = class_double('Article', :column_names => [])
85
+ opinion = class_double('OpinionArticle', {:column_names => []}, article)
86
+
87
+ options[:classes] = [opinion]
88
+
89
+ expect(set.to_a.length).to eq(1)
90
+ end
91
+
69
92
  it "uses named indices if names are provided" do
70
93
  article_core = double('index', :name => 'article_core')
71
94
  user_core = double('index', :name => 'user_core')