ultrasphinx 1.6 → 1.6.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. data.tar.gz.sig +0 -0
  2. data/CHANGELOG +5 -1
  3. data/Manifest +47 -7
  4. data/README +4 -4
  5. data/TODO +1 -0
  6. data/examples/default.base +6 -2
  7. data/lib/ultrasphinx.rb +1 -1
  8. data/lib/ultrasphinx/configure.rb +53 -28
  9. data/lib/ultrasphinx/fields.rb +16 -13
  10. data/lib/ultrasphinx/postgresql/concat_ws.sql +35 -0
  11. data/lib/ultrasphinx/postgresql/crc32.sql +7 -0
  12. data/lib/ultrasphinx/postgresql/group_concat.sql +25 -0
  13. data/lib/ultrasphinx/{hex_to_int.sql → postgresql/hex_to_int.sql} +0 -0
  14. data/lib/ultrasphinx/postgresql/language.sql +1 -0
  15. data/lib/ultrasphinx/postgresql/unix_timestamp.sql +12 -0
  16. data/lib/ultrasphinx/search/internals.rb +42 -16
  17. data/lib/ultrasphinx/ultrasphinx.rb +23 -12
  18. data/test/integration/app/app/models/person/user.rb +1 -1
  19. data/test/integration/app/config/database.yml +9 -13
  20. data/test/integration/app/config/ultrasphinx/development.conf +6 -6
  21. data/test/integration/app/config/ultrasphinx/development.conf.canonical +6 -6
  22. data/test/integration/app/db/schema.rb +9 -2
  23. data/test/integration/search_test.rb +16 -6
  24. data/test/setup.rb +5 -1
  25. data/test/ts.multi +2 -0
  26. data/ultrasphinx.gemspec +5 -5
  27. data/vendor/riddle/{MIT-LICENSE → MIT-LICENCE} +0 -0
  28. data/vendor/riddle/README +60 -0
  29. data/vendor/riddle/Rakefile +25 -0
  30. data/vendor/riddle/{riddle.rb → lib/riddle.rb} +3 -0
  31. data/vendor/riddle/{riddle → lib/riddle}/client.rb +73 -4
  32. data/vendor/riddle/{riddle → lib/riddle}/client/filter.rb +0 -0
  33. data/vendor/riddle/{riddle → lib/riddle}/client/message.rb +2 -0
  34. data/vendor/riddle/{riddle → lib/riddle}/client/response.rb +0 -0
  35. data/vendor/riddle/spec/fixtures/data/anchor.bin +0 -0
  36. data/vendor/riddle/spec/fixtures/data/any.bin +0 -0
  37. data/vendor/riddle/spec/fixtures/data/boolean.bin +0 -0
  38. data/vendor/riddle/spec/fixtures/data/distinct.bin +0 -0
  39. data/vendor/riddle/spec/fixtures/data/filter.bin +0 -0
  40. data/vendor/riddle/spec/fixtures/data/filter_array.bin +0 -0
  41. data/vendor/riddle/spec/fixtures/data/filter_array_exclude.bin +0 -0
  42. data/vendor/riddle/spec/fixtures/data/filter_floats.bin +0 -0
  43. data/vendor/riddle/spec/fixtures/data/filter_floats_exclude.bin +0 -0
  44. data/vendor/riddle/spec/fixtures/data/filter_floats_range.bin +0 -0
  45. data/vendor/riddle/spec/fixtures/data/filter_range.bin +0 -0
  46. data/vendor/riddle/spec/fixtures/data/filter_range_exclude.bin +0 -0
  47. data/vendor/riddle/spec/fixtures/data/group.bin +0 -0
  48. data/vendor/riddle/spec/fixtures/data/index.bin +0 -0
  49. data/vendor/riddle/spec/fixtures/data/phrase.bin +0 -0
  50. data/vendor/riddle/spec/fixtures/data/simple.bin +0 -0
  51. data/vendor/riddle/spec/fixtures/data/sort.bin +0 -0
  52. data/vendor/riddle/spec/fixtures/data/update_simple.bin +0 -0
  53. data/vendor/riddle/spec/fixtures/data/weights.bin +0 -0
  54. data/vendor/riddle/spec/fixtures/sphinx/configuration.erb +38 -0
  55. data/vendor/riddle/spec/fixtures/sql/conf.example.yml +3 -0
  56. data/vendor/riddle/spec/fixtures/sql/data.sql +25000 -0
  57. data/vendor/riddle/spec/fixtures/sql/structure.sql +16 -0
  58. data/vendor/riddle/spec/functional/excerpt_spec.rb +102 -0
  59. data/vendor/riddle/spec/functional/search_spec.rb +69 -0
  60. data/vendor/riddle/spec/functional/update_spec.rb +41 -0
  61. data/vendor/riddle/spec/spec_helper.rb +25 -0
  62. data/vendor/riddle/spec/sphinx_helper.rb +91 -0
  63. data/vendor/riddle/spec/unit/client_spec.rb +140 -0
  64. data/vendor/riddle/spec/unit/filter_spec.rb +33 -0
  65. data/vendor/riddle/spec/unit/message_spec.rb +63 -0
  66. data/vendor/riddle/spec/unit/response_spec.rb +64 -0
  67. metadata +95 -55
  68. metadata.gz.sig +0 -0
@@ -0,0 +1,16 @@
1
+ DROP TABLE `people`;
2
+
3
+ CREATE TABLE `people` (
4
+ `id` int(11) NOT NULL auto_increment,
5
+ `first_name` varchar(50) NOT NULL,
6
+ `middle_initial` varchar(10) NOT NULL,
7
+ `last_name` varchar(50) NOT NULL,
8
+ `gender` varchar(10) NOT NULL,
9
+ `street_address` varchar(200) NOT NULL,
10
+ `city` varchar(100) NOT NULL,
11
+ `state` varchar(100) NOT NULL,
12
+ `postcode` varchar(10) NOT NULL,
13
+ `email` varchar(100) NOT NULL,
14
+ `birthday` datetime NOT NULL,
15
+ PRIMARY KEY (`id`)
16
+ ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
@@ -0,0 +1,102 @@
1
+ require 'spec/spec_helper'
2
+
3
+ describe "Sphinx Excepts" do
4
+ before :each do
5
+ @client = Riddle::Client.new("localhost", 3313)
6
+ end
7
+
8
+ it "should highlight a single word multiple times in a document" do
9
+ @client.excerpts(
10
+ :index => "people",
11
+ :words => "Mary",
12
+ :docs => ["Mary, Mary, quite contrary."]
13
+ ).should == [
14
+ '<span class="match">Mary</span>, <span class="match">Mary</span>, quite contrary.'
15
+ ]
16
+ end
17
+
18
+ it "should use specified word markers" do
19
+ @client.excerpts(
20
+ :index => "people",
21
+ :words => "Mary",
22
+ :docs => ["Mary, Mary, quite contrary."],
23
+ :before_match => "<em>",
24
+ :after_match => "</em>"
25
+ ).should == [
26
+ "<em>Mary</em>, <em>Mary</em>, quite contrary."
27
+ ]
28
+ end
29
+
30
+ it "should separate matches that are far apart by an ellipsis by default" do
31
+ @client.excerpts(
32
+ :index => "people",
33
+ :words => "Pat",
34
+ :docs => [
35
+ <<-SENTENCE
36
+ This is a really long sentence written by Pat. It has to be over 256
37
+ characters long, between keywords. But what is the keyword? Well, I
38
+ can't tell you just yet... wait patiently until we've hit the 256 mark.
39
+ It'll take a bit longer than you think. We're probably just hitting the
40
+ 200 mark at this point. But I think we've now arrived - so I can tell
41
+ you what the keyword is. I bet you're really interested in finding out,
42
+ yeah? Excerpts are particularly riveting. This keyword, however, is
43
+ not. It's just my name: Pat.
44
+ SENTENCE
45
+ ],
46
+ :before_match => "<em>",
47
+ :after_match => "</em>"
48
+ ).should == [
49
+ <<-SENTENCE
50
+ This is a really long sentence written by <em>Pat</em>. It has to be over 256
51
+ characters long, between keywords. But what is the keyword? &#8230; interested in finding out,
52
+ yeah? Excerpts are particularly riveting. This keyword, however, is
53
+ not. It's just my name: <em>Pat</em>.
54
+ SENTENCE
55
+ ]
56
+ end
57
+
58
+ it "should use the provided separator" do
59
+ @client.excerpts(
60
+ :index => "people",
61
+ :words => "Pat",
62
+ :docs => [
63
+ <<-SENTENCE
64
+ This is a really long sentence written by Pat. It has to be over 256
65
+ characters long, between keywords. But what is the keyword? Well, I
66
+ can't tell you just yet... wait patiently until we've hit the 256 mark.
67
+ It'll take a bit longer than you think. We're probably just hitting the
68
+ 200 mark at this point. But I think we've now arrived - so I can tell
69
+ you what the keyword is. I bet you're really interested in finding out,
70
+ yeah? Excerpts are particularly riveting. This keyword, however, is
71
+ not. It's just my name: Pat.
72
+ SENTENCE
73
+ ],
74
+ :before_match => "<em>",
75
+ :after_match => "</em>",
76
+ :chunk_separator => " --- "
77
+ ).should == [
78
+ <<-SENTENCE
79
+ This is a really long sentence written by <em>Pat</em>. It has to be over 256
80
+ characters long, between keywords. But what is the keyword? --- interested in finding out,
81
+ yeah? Excerpts are particularly riveting. This keyword, however, is
82
+ not. It's just my name: <em>Pat</em>.
83
+ SENTENCE
84
+ ]
85
+ end
86
+
87
+ it "should return multiple results for multiple documents" do
88
+ @client.excerpts(
89
+ :index => "people",
90
+ :words => "Mary",
91
+ :docs => [
92
+ "Mary, Mary, quite contrary.",
93
+ "The epithet \"Bloody Mary\" is associated with a number of historical and fictional women, most notably Queen Mary I of England"
94
+ ],
95
+ :before_match => "<em>",
96
+ :after_match => "</em>"
97
+ ).should == [
98
+ "<em>Mary</em>, <em>Mary</em>, quite contrary.",
99
+ "The epithet \"Bloody <em>Mary</em>\" is associated with a number of historical and fictional women, most notably Queen <em>Mary</em> I of England"
100
+ ]
101
+ end
102
+ end
@@ -0,0 +1,69 @@
1
+ require 'spec/spec_helper'
2
+
3
+ describe "Sphinx Searches" do
4
+ before :each do
5
+ @client = Riddle::Client.new("localhost", 3313)
6
+ end
7
+
8
+ it "should return a single hash if a single query" do
9
+ @client.query("smith").should be_kind_of(Hash)
10
+ end
11
+
12
+ it "should return an array of hashs if multiple queries are run" do
13
+ @client.append_query "smith"
14
+ @client.append_query "jones"
15
+ results = @client.run
16
+ results.should be_kind_of(Array)
17
+ results.each { |result| result.should be_kind_of(Hash) }
18
+ end
19
+
20
+ it "should return an array of matches" do
21
+ matches = @client.query("smith")[:matches]
22
+ matches.should be_kind_of(Array)
23
+ matches.each { |match| match.should be_kind_of(Hash) }
24
+ end
25
+
26
+ it "should return an array of string fields" do
27
+ fields = @client.query("smith")[:fields]
28
+ fields.should be_kind_of(Array)
29
+ fields.each { |field| field.should be_kind_of(String) }
30
+ end
31
+
32
+ it "should return an array of attribute names" do
33
+ attributes = @client.query("smith")[:attribute_names]
34
+ attributes.should be_kind_of(Array)
35
+ attributes.each { |a| a.should be_kind_of(String) }
36
+ end
37
+
38
+ it "should return a hash of attributes" do
39
+ attributes = @client.query("smith")[:attributes]
40
+ attributes.should be_kind_of(Hash)
41
+ attributes.each do |key,value|
42
+ key.should be_kind_of(String)
43
+ value.should be_kind_of(Integer)
44
+ end
45
+ end
46
+
47
+ it "should return the total number of results returned" do
48
+ @client.query("smith")[:total].should be_kind_of(Integer)
49
+ end
50
+
51
+ it "should return the total number of results available" do
52
+ @client.query("smith")[:total_found].should be_kind_of(Integer)
53
+ end
54
+
55
+ it "should return the time taken for the query as a float" do
56
+ @client.query("smith")[:time].should be_kind_of(Float)
57
+ end
58
+
59
+ it "should return a hash of the words from the query, with the number of documents and the number of hits" do
60
+ words = @client.query("smith victoria")[:words]
61
+ words.should be_kind_of(Hash)
62
+ words.each do |word,hash|
63
+ word.should be_kind_of(String)
64
+ hash.should be_kind_of(Hash)
65
+ hash[:docs].should be_kind_of(Integer)
66
+ hash[:hits].should be_kind_of(Integer)
67
+ end
68
+ end
69
+ end
@@ -0,0 +1,41 @@
1
+ require 'spec/spec_helper'
2
+
3
+ describe "Sphinx Updates" do
4
+ before :each do
5
+ @client = Riddle::Client.new("localhost", 3313)
6
+ end
7
+
8
+ it "should update a single record appropriately" do
9
+ # check existing birthday
10
+ result = @client.query("Ellie K Ford")
11
+ result[:matches].should_not be_empty
12
+ result[:matches].length.should == 1
13
+ ellie = result[:matches].first
14
+ ellie[:attributes]["birthday"].should == Time.local(1970, 1, 23).to_i
15
+
16
+ # make Ellie younger by 6 years
17
+ @client.update("people", ["birthday"], {ellie[:doc] => [Time.local(1976, 1, 23).to_i]})
18
+
19
+ # check attribute's value
20
+ result = @client.query("Ellie K Ford")
21
+ result[:matches].should_not be_empty
22
+ result[:matches].length.should == 1
23
+ ellie = result[:matches].first
24
+ ellie[:attributes]["birthday"].should == Time.local(1976, 1, 23).to_i
25
+ end
26
+
27
+ it "should update multiple records appropriately" do
28
+ result = @client.query("Steele")
29
+ pairs = {}
30
+ result[:matches].each do |match|
31
+ pairs[match[:doc]] = [match[:attributes]["birthday"] + (365*24*60*60)]
32
+ end
33
+
34
+ @client.update "people", ["birthday"], pairs
35
+
36
+ result = @client.query("Steele")
37
+ result[:matches].each do |match|
38
+ match[:attributes]["birthday"].should == pairs[match[:doc]].first
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,25 @@
1
+ require 'riddle'
2
+ require 'spec/sphinx_helper'
3
+
4
+ Spec::Runner.configure do |config|
5
+ sphinx = SphinxHelper.new
6
+ sphinx.setup_mysql
7
+ sphinx.generate_configuration
8
+ sphinx.index
9
+
10
+ config.before :all do
11
+ sphinx.start
12
+ end
13
+
14
+ # config.before :each do
15
+ # sphinx.reset
16
+ # end
17
+
18
+ config.after :all do
19
+ sphinx.stop
20
+ end
21
+ end
22
+
23
+ def query_contents(key)
24
+ open("spec/fixtures/data/#{key.to_s}.bin") { |f| f.read }
25
+ end
@@ -0,0 +1,91 @@
1
+ require 'mysql'
2
+ require 'erb'
3
+
4
+ class SphinxHelper
5
+ attr_accessor :host, :username, :password
6
+ attr_reader :path
7
+
8
+ def initialize
9
+ @host = "localhost"
10
+ @username = "anonymous"
11
+ @password = ""
12
+
13
+ if File.exist?("spec/fixtures/sql/conf.yml")
14
+ config = YAML.load(File.open("spec/fixtures/sql/conf.yml"))
15
+ @host = config["host"]
16
+ @username = config["username"]
17
+ @password = config["password"]
18
+ end
19
+
20
+ @path = File.expand_path(File.dirname(__FILE__))
21
+ end
22
+
23
+ def setup_mysql
24
+ server = Mysql.new @host, @username, @password
25
+
26
+ unless server.list_dbs.include?("riddle_sphinx_spec")
27
+ server.create_db "riddle_sphinx_spec"
28
+ end
29
+
30
+ server.query "USE riddle_sphinx_spec;"
31
+
32
+ structure = File.open("spec/fixtures/sql/structure.sql") { |f| f.read }
33
+ # Block ensures multiple statements can be run
34
+ server.query(structure) { }
35
+ data = File.open("spec/fixtures/sql/data.sql") { |f|
36
+ while line = f.gets
37
+ server.query line
38
+ end
39
+ }
40
+
41
+ server.close
42
+ end
43
+
44
+ def reset
45
+ setup_mysql
46
+ index
47
+ end
48
+
49
+ def generate_configuration
50
+ template = File.open("spec/fixtures/sphinx/configuration.erb") { |f| f.read }
51
+ File.open("spec/fixtures/sphinx/spec.conf", "w") { |f|
52
+ f.puts ERB.new(template).result(binding)
53
+ }
54
+ end
55
+
56
+ def index
57
+ cmd = "indexer --config #{@path}/fixtures/sphinx/spec.conf --all"
58
+ cmd << " --rotate" if running?
59
+ `#{cmd}`
60
+ end
61
+
62
+ def start
63
+ return if running?
64
+
65
+ cmd = "searchd --config #{@path}/fixtures/sphinx/spec.conf"
66
+ `#{cmd}`
67
+
68
+ sleep(1)
69
+
70
+ unless running?
71
+ puts "Failed to start searchd daemon. Check fixtures/sphinx/searchd.log."
72
+ end
73
+ end
74
+
75
+ def stop
76
+ return unless running?
77
+ `kill #{pid}`
78
+ end
79
+
80
+ def pid
81
+ if File.exists?("#{@path}/fixtures/sphinx/searchd.pid")
82
+ `cat #{@path}/fixtures/sphinx/searchd.pid`[/\d+/]
83
+ else
84
+ nil
85
+ end
86
+ end
87
+
88
+ def running?
89
+ pid && `ps #{pid} | wc -l`.to_i > 1
90
+ end
91
+ end
@@ -0,0 +1,140 @@
1
+ require 'spec/spec_helper'
2
+
3
+ describe Riddle::Client do
4
+ it "should have the same keys for both commands and versions" do
5
+ Riddle::Client::Commands.keys.should == Riddle::Client::Versions.keys
6
+ end
7
+
8
+ it "should default to localhost as the server" do
9
+ Riddle::Client.new.server.should == "localhost"
10
+ end
11
+
12
+ it "should default to port 3312" do
13
+ Riddle::Client.new.port.should == 3312
14
+ end
15
+
16
+ it "should translate anchor arguments correctly" do
17
+ client = Riddle::Client.new
18
+ client.set_anchor "latitude", 10.0, "longtitude", 95.0
19
+ client.anchor.should == {
20
+ :latitude_attribute => "latitude",
21
+ :latitude => 10.0,
22
+ :longtitude_attribute => "longtitude",
23
+ :longtitude => 95.0
24
+ }
25
+ end
26
+
27
+ it "should add queries to the queue" do
28
+ client = Riddle::Client.new
29
+ client.queue.should be_empty
30
+ client.append_query "spec"
31
+ client.queue.should_not be_empty
32
+ end
33
+
34
+ it "should build a basic search message correctly" do
35
+ client = Riddle::Client.new
36
+ client.append_query "test "
37
+ client.queue.first.should == query_contents(:simple)
38
+ end
39
+
40
+ it "should build a message with a specified index correctly" do
41
+ client = Riddle::Client.new
42
+ client.append_query "test ", "edition"
43
+ client.queue.first.should == query_contents(:index)
44
+ end
45
+
46
+ it "should build a message using match mode :any correctly" do
47
+ client = Riddle::Client.new
48
+ client.match_mode = :any
49
+ client.append_query "test this "
50
+ client.queue.first.should == query_contents(:any)
51
+ end
52
+
53
+ it "should build a message using sort by correctly" do
54
+ client = Riddle::Client.new
55
+ client.sort_by = 'id'
56
+ client.sort_mode = :extended
57
+ client.append_query "testing "
58
+ client.queue.first.should == query_contents(:sort)
59
+ end
60
+
61
+ it "should build a message using match mode :boolean correctly" do
62
+ client = Riddle::Client.new
63
+ client.match_mode = :boolean
64
+ client.append_query "test "
65
+ client.queue.first.should == query_contents(:boolean)
66
+ end
67
+
68
+ it "should build a message using match mode :phrase correctly" do
69
+ client = Riddle::Client.new
70
+ client.match_mode = :phrase
71
+ client.append_query "testing this "
72
+ client.queue.first.should == query_contents(:phrase)
73
+ end
74
+
75
+ it "should build a message with a filter correctly" do
76
+ client = Riddle::Client.new
77
+ client.filters << Riddle::Client::Filter.new("id", [10, 100, 1000])
78
+ client.append_query "test "
79
+ client.queue.first.should == query_contents(:filter)
80
+ end
81
+
82
+ it "should build a message with group values correctly" do
83
+ client = Riddle::Client.new
84
+ client.group_by = "id"
85
+ client.group_function = :attr
86
+ client.group_clause = "id"
87
+ client.append_query "test "
88
+ client.queue.first.should == query_contents(:group)
89
+ end
90
+
91
+ it "should build a message with group distinct value correctly" do
92
+ client = Riddle::Client.new
93
+ client.group_distinct = "id"
94
+ client.append_query "test "
95
+ client.queue.first.should == query_contents(:distinct)
96
+ end
97
+
98
+ it "should build a message with weights correctly" do
99
+ client = Riddle::Client.new
100
+ client.weights = [100, 1]
101
+ client.append_query "test "
102
+ client.queue.first.should == query_contents(:weights)
103
+ end
104
+
105
+ it "should build a message with an anchor correctly" do
106
+ client = Riddle::Client.new
107
+ client.set_anchor "latitude", 10.0, "longtitude", 95.0
108
+ client.append_query "test "
109
+ client.queue.first.should == query_contents(:anchor)
110
+ end
111
+
112
+ it "should keep multiple messages in the queue" do
113
+ client = Riddle::Client.new
114
+ client.weights = [100, 1]
115
+ client.append_query "test "
116
+ client.append_query "test "
117
+ client.queue.length.should == 2
118
+ client.queue.each { |item| item.should == query_contents(:weights) }
119
+ end
120
+
121
+ it "should keep multiple messages in the queue with different params" do
122
+ client = Riddle::Client.new
123
+ client.weights = [100, 1]
124
+ client.append_query "test "
125
+ client.weights = []
126
+ client.append_query "test ", "edition"
127
+ client.queue.first.should == query_contents(:weights)
128
+ client.queue.last.should == query_contents(:index)
129
+ end
130
+
131
+ it "should build a basic update message correctly" do
132
+ client = Riddle::Client.new
133
+ client.send(
134
+ :update_message,
135
+ "people",
136
+ ["birthday"],
137
+ {1 => [191163600]}
138
+ ).should == query_contents(:update_simple)
139
+ end
140
+ end