ultrasphinx 1.6 → 1.6.7

Sign up to get free protection for your applications and to get access to all the features.
Files changed (68) hide show
  1. data.tar.gz.sig +0 -0
  2. data/CHANGELOG +5 -1
  3. data/Manifest +47 -7
  4. data/README +4 -4
  5. data/TODO +1 -0
  6. data/examples/default.base +6 -2
  7. data/lib/ultrasphinx.rb +1 -1
  8. data/lib/ultrasphinx/configure.rb +53 -28
  9. data/lib/ultrasphinx/fields.rb +16 -13
  10. data/lib/ultrasphinx/postgresql/concat_ws.sql +35 -0
  11. data/lib/ultrasphinx/postgresql/crc32.sql +7 -0
  12. data/lib/ultrasphinx/postgresql/group_concat.sql +25 -0
  13. data/lib/ultrasphinx/{hex_to_int.sql → postgresql/hex_to_int.sql} +0 -0
  14. data/lib/ultrasphinx/postgresql/language.sql +1 -0
  15. data/lib/ultrasphinx/postgresql/unix_timestamp.sql +12 -0
  16. data/lib/ultrasphinx/search/internals.rb +42 -16
  17. data/lib/ultrasphinx/ultrasphinx.rb +23 -12
  18. data/test/integration/app/app/models/person/user.rb +1 -1
  19. data/test/integration/app/config/database.yml +9 -13
  20. data/test/integration/app/config/ultrasphinx/development.conf +6 -6
  21. data/test/integration/app/config/ultrasphinx/development.conf.canonical +6 -6
  22. data/test/integration/app/db/schema.rb +9 -2
  23. data/test/integration/search_test.rb +16 -6
  24. data/test/setup.rb +5 -1
  25. data/test/ts.multi +2 -0
  26. data/ultrasphinx.gemspec +5 -5
  27. data/vendor/riddle/{MIT-LICENSE → MIT-LICENCE} +0 -0
  28. data/vendor/riddle/README +60 -0
  29. data/vendor/riddle/Rakefile +25 -0
  30. data/vendor/riddle/{riddle.rb → lib/riddle.rb} +3 -0
  31. data/vendor/riddle/{riddle → lib/riddle}/client.rb +73 -4
  32. data/vendor/riddle/{riddle → lib/riddle}/client/filter.rb +0 -0
  33. data/vendor/riddle/{riddle → lib/riddle}/client/message.rb +2 -0
  34. data/vendor/riddle/{riddle → lib/riddle}/client/response.rb +0 -0
  35. data/vendor/riddle/spec/fixtures/data/anchor.bin +0 -0
  36. data/vendor/riddle/spec/fixtures/data/any.bin +0 -0
  37. data/vendor/riddle/spec/fixtures/data/boolean.bin +0 -0
  38. data/vendor/riddle/spec/fixtures/data/distinct.bin +0 -0
  39. data/vendor/riddle/spec/fixtures/data/filter.bin +0 -0
  40. data/vendor/riddle/spec/fixtures/data/filter_array.bin +0 -0
  41. data/vendor/riddle/spec/fixtures/data/filter_array_exclude.bin +0 -0
  42. data/vendor/riddle/spec/fixtures/data/filter_floats.bin +0 -0
  43. data/vendor/riddle/spec/fixtures/data/filter_floats_exclude.bin +0 -0
  44. data/vendor/riddle/spec/fixtures/data/filter_floats_range.bin +0 -0
  45. data/vendor/riddle/spec/fixtures/data/filter_range.bin +0 -0
  46. data/vendor/riddle/spec/fixtures/data/filter_range_exclude.bin +0 -0
  47. data/vendor/riddle/spec/fixtures/data/group.bin +0 -0
  48. data/vendor/riddle/spec/fixtures/data/index.bin +0 -0
  49. data/vendor/riddle/spec/fixtures/data/phrase.bin +0 -0
  50. data/vendor/riddle/spec/fixtures/data/simple.bin +0 -0
  51. data/vendor/riddle/spec/fixtures/data/sort.bin +0 -0
  52. data/vendor/riddle/spec/fixtures/data/update_simple.bin +0 -0
  53. data/vendor/riddle/spec/fixtures/data/weights.bin +0 -0
  54. data/vendor/riddle/spec/fixtures/sphinx/configuration.erb +38 -0
  55. data/vendor/riddle/spec/fixtures/sql/conf.example.yml +3 -0
  56. data/vendor/riddle/spec/fixtures/sql/data.sql +25000 -0
  57. data/vendor/riddle/spec/fixtures/sql/structure.sql +16 -0
  58. data/vendor/riddle/spec/functional/excerpt_spec.rb +102 -0
  59. data/vendor/riddle/spec/functional/search_spec.rb +69 -0
  60. data/vendor/riddle/spec/functional/update_spec.rb +41 -0
  61. data/vendor/riddle/spec/spec_helper.rb +25 -0
  62. data/vendor/riddle/spec/sphinx_helper.rb +91 -0
  63. data/vendor/riddle/spec/unit/client_spec.rb +140 -0
  64. data/vendor/riddle/spec/unit/filter_spec.rb +33 -0
  65. data/vendor/riddle/spec/unit/message_spec.rb +63 -0
  66. data/vendor/riddle/spec/unit/response_spec.rb +64 -0
  67. metadata +95 -55
  68. metadata.gz.sig +0 -0
@@ -0,0 +1,16 @@
1
+ DROP TABLE `people`;
2
+
3
+ CREATE TABLE `people` (
4
+ `id` int(11) NOT NULL auto_increment,
5
+ `first_name` varchar(50) NOT NULL,
6
+ `middle_initial` varchar(10) NOT NULL,
7
+ `last_name` varchar(50) NOT NULL,
8
+ `gender` varchar(10) NOT NULL,
9
+ `street_address` varchar(200) NOT NULL,
10
+ `city` varchar(100) NOT NULL,
11
+ `state` varchar(100) NOT NULL,
12
+ `postcode` varchar(10) NOT NULL,
13
+ `email` varchar(100) NOT NULL,
14
+ `birthday` datetime NOT NULL,
15
+ PRIMARY KEY (`id`)
16
+ ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
@@ -0,0 +1,102 @@
1
+ require 'spec/spec_helper'
2
+
3
+ describe "Sphinx Excepts" do
4
+ before :each do
5
+ @client = Riddle::Client.new("localhost", 3313)
6
+ end
7
+
8
+ it "should highlight a single word multiple times in a document" do
9
+ @client.excerpts(
10
+ :index => "people",
11
+ :words => "Mary",
12
+ :docs => ["Mary, Mary, quite contrary."]
13
+ ).should == [
14
+ '<span class="match">Mary</span>, <span class="match">Mary</span>, quite contrary.'
15
+ ]
16
+ end
17
+
18
+ it "should use specified word markers" do
19
+ @client.excerpts(
20
+ :index => "people",
21
+ :words => "Mary",
22
+ :docs => ["Mary, Mary, quite contrary."],
23
+ :before_match => "<em>",
24
+ :after_match => "</em>"
25
+ ).should == [
26
+ "<em>Mary</em>, <em>Mary</em>, quite contrary."
27
+ ]
28
+ end
29
+
30
+ it "should separate matches that are far apart by an ellipsis by default" do
31
+ @client.excerpts(
32
+ :index => "people",
33
+ :words => "Pat",
34
+ :docs => [
35
+ <<-SENTENCE
36
+ This is a really long sentence written by Pat. It has to be over 256
37
+ characters long, between keywords. But what is the keyword? Well, I
38
+ can't tell you just yet... wait patiently until we've hit the 256 mark.
39
+ It'll take a bit longer than you think. We're probably just hitting the
40
+ 200 mark at this point. But I think we've now arrived - so I can tell
41
+ you what the keyword is. I bet you're really interested in finding out,
42
+ yeah? Excerpts are particularly riveting. This keyword, however, is
43
+ not. It's just my name: Pat.
44
+ SENTENCE
45
+ ],
46
+ :before_match => "<em>",
47
+ :after_match => "</em>"
48
+ ).should == [
49
+ <<-SENTENCE
50
+ This is a really long sentence written by <em>Pat</em>. It has to be over 256
51
+ characters long, between keywords. But what is the keyword? &#8230; interested in finding out,
52
+ yeah? Excerpts are particularly riveting. This keyword, however, is
53
+ not. It's just my name: <em>Pat</em>.
54
+ SENTENCE
55
+ ]
56
+ end
57
+
58
+ it "should use the provided separator" do
59
+ @client.excerpts(
60
+ :index => "people",
61
+ :words => "Pat",
62
+ :docs => [
63
+ <<-SENTENCE
64
+ This is a really long sentence written by Pat. It has to be over 256
65
+ characters long, between keywords. But what is the keyword? Well, I
66
+ can't tell you just yet... wait patiently until we've hit the 256 mark.
67
+ It'll take a bit longer than you think. We're probably just hitting the
68
+ 200 mark at this point. But I think we've now arrived - so I can tell
69
+ you what the keyword is. I bet you're really interested in finding out,
70
+ yeah? Excerpts are particularly riveting. This keyword, however, is
71
+ not. It's just my name: Pat.
72
+ SENTENCE
73
+ ],
74
+ :before_match => "<em>",
75
+ :after_match => "</em>",
76
+ :chunk_separator => " --- "
77
+ ).should == [
78
+ <<-SENTENCE
79
+ This is a really long sentence written by <em>Pat</em>. It has to be over 256
80
+ characters long, between keywords. But what is the keyword? --- interested in finding out,
81
+ yeah? Excerpts are particularly riveting. This keyword, however, is
82
+ not. It's just my name: <em>Pat</em>.
83
+ SENTENCE
84
+ ]
85
+ end
86
+
87
+ it "should return multiple results for multiple documents" do
88
+ @client.excerpts(
89
+ :index => "people",
90
+ :words => "Mary",
91
+ :docs => [
92
+ "Mary, Mary, quite contrary.",
93
+ "The epithet \"Bloody Mary\" is associated with a number of historical and fictional women, most notably Queen Mary I of England"
94
+ ],
95
+ :before_match => "<em>",
96
+ :after_match => "</em>"
97
+ ).should == [
98
+ "<em>Mary</em>, <em>Mary</em>, quite contrary.",
99
+ "The epithet \"Bloody <em>Mary</em>\" is associated with a number of historical and fictional women, most notably Queen <em>Mary</em> I of England"
100
+ ]
101
+ end
102
+ end
@@ -0,0 +1,69 @@
1
+ require 'spec/spec_helper'
2
+
3
+ describe "Sphinx Searches" do
4
+ before :each do
5
+ @client = Riddle::Client.new("localhost", 3313)
6
+ end
7
+
8
+ it "should return a single hash if a single query" do
9
+ @client.query("smith").should be_kind_of(Hash)
10
+ end
11
+
12
+ it "should return an array of hashs if multiple queries are run" do
13
+ @client.append_query "smith"
14
+ @client.append_query "jones"
15
+ results = @client.run
16
+ results.should be_kind_of(Array)
17
+ results.each { |result| result.should be_kind_of(Hash) }
18
+ end
19
+
20
+ it "should return an array of matches" do
21
+ matches = @client.query("smith")[:matches]
22
+ matches.should be_kind_of(Array)
23
+ matches.each { |match| match.should be_kind_of(Hash) }
24
+ end
25
+
26
+ it "should return an array of string fields" do
27
+ fields = @client.query("smith")[:fields]
28
+ fields.should be_kind_of(Array)
29
+ fields.each { |field| field.should be_kind_of(String) }
30
+ end
31
+
32
+ it "should return an array of attribute names" do
33
+ attributes = @client.query("smith")[:attribute_names]
34
+ attributes.should be_kind_of(Array)
35
+ attributes.each { |a| a.should be_kind_of(String) }
36
+ end
37
+
38
+ it "should return a hash of attributes" do
39
+ attributes = @client.query("smith")[:attributes]
40
+ attributes.should be_kind_of(Hash)
41
+ attributes.each do |key,value|
42
+ key.should be_kind_of(String)
43
+ value.should be_kind_of(Integer)
44
+ end
45
+ end
46
+
47
+ it "should return the total number of results returned" do
48
+ @client.query("smith")[:total].should be_kind_of(Integer)
49
+ end
50
+
51
+ it "should return the total number of results available" do
52
+ @client.query("smith")[:total_found].should be_kind_of(Integer)
53
+ end
54
+
55
+ it "should return the time taken for the query as a float" do
56
+ @client.query("smith")[:time].should be_kind_of(Float)
57
+ end
58
+
59
+ it "should return a hash of the words from the query, with the number of documents and the number of hits" do
60
+ words = @client.query("smith victoria")[:words]
61
+ words.should be_kind_of(Hash)
62
+ words.each do |word,hash|
63
+ word.should be_kind_of(String)
64
+ hash.should be_kind_of(Hash)
65
+ hash[:docs].should be_kind_of(Integer)
66
+ hash[:hits].should be_kind_of(Integer)
67
+ end
68
+ end
69
+ end
@@ -0,0 +1,41 @@
1
+ require 'spec/spec_helper'
2
+
3
+ describe "Sphinx Updates" do
4
+ before :each do
5
+ @client = Riddle::Client.new("localhost", 3313)
6
+ end
7
+
8
+ it "should update a single record appropriately" do
9
+ # check existing birthday
10
+ result = @client.query("Ellie K Ford")
11
+ result[:matches].should_not be_empty
12
+ result[:matches].length.should == 1
13
+ ellie = result[:matches].first
14
+ ellie[:attributes]["birthday"].should == Time.local(1970, 1, 23).to_i
15
+
16
+ # make Ellie younger by 6 years
17
+ @client.update("people", ["birthday"], {ellie[:doc] => [Time.local(1976, 1, 23).to_i]})
18
+
19
+ # check attribute's value
20
+ result = @client.query("Ellie K Ford")
21
+ result[:matches].should_not be_empty
22
+ result[:matches].length.should == 1
23
+ ellie = result[:matches].first
24
+ ellie[:attributes]["birthday"].should == Time.local(1976, 1, 23).to_i
25
+ end
26
+
27
+ it "should update multiple records appropriately" do
28
+ result = @client.query("Steele")
29
+ pairs = {}
30
+ result[:matches].each do |match|
31
+ pairs[match[:doc]] = [match[:attributes]["birthday"] + (365*24*60*60)]
32
+ end
33
+
34
+ @client.update "people", ["birthday"], pairs
35
+
36
+ result = @client.query("Steele")
37
+ result[:matches].each do |match|
38
+ match[:attributes]["birthday"].should == pairs[match[:doc]].first
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,25 @@
1
+ require 'riddle'
2
+ require 'spec/sphinx_helper'
3
+
4
+ Spec::Runner.configure do |config|
5
+ sphinx = SphinxHelper.new
6
+ sphinx.setup_mysql
7
+ sphinx.generate_configuration
8
+ sphinx.index
9
+
10
+ config.before :all do
11
+ sphinx.start
12
+ end
13
+
14
+ # config.before :each do
15
+ # sphinx.reset
16
+ # end
17
+
18
+ config.after :all do
19
+ sphinx.stop
20
+ end
21
+ end
22
+
23
+ def query_contents(key)
24
+ open("spec/fixtures/data/#{key.to_s}.bin") { |f| f.read }
25
+ end
@@ -0,0 +1,91 @@
1
+ require 'mysql'
2
+ require 'erb'
3
+
4
+ class SphinxHelper
5
+ attr_accessor :host, :username, :password
6
+ attr_reader :path
7
+
8
+ def initialize
9
+ @host = "localhost"
10
+ @username = "anonymous"
11
+ @password = ""
12
+
13
+ if File.exist?("spec/fixtures/sql/conf.yml")
14
+ config = YAML.load(File.open("spec/fixtures/sql/conf.yml"))
15
+ @host = config["host"]
16
+ @username = config["username"]
17
+ @password = config["password"]
18
+ end
19
+
20
+ @path = File.expand_path(File.dirname(__FILE__))
21
+ end
22
+
23
+ def setup_mysql
24
+ server = Mysql.new @host, @username, @password
25
+
26
+ unless server.list_dbs.include?("riddle_sphinx_spec")
27
+ server.create_db "riddle_sphinx_spec"
28
+ end
29
+
30
+ server.query "USE riddle_sphinx_spec;"
31
+
32
+ structure = File.open("spec/fixtures/sql/structure.sql") { |f| f.read }
33
+ # Block ensures multiple statements can be run
34
+ server.query(structure) { }
35
+ data = File.open("spec/fixtures/sql/data.sql") { |f|
36
+ while line = f.gets
37
+ server.query line
38
+ end
39
+ }
40
+
41
+ server.close
42
+ end
43
+
44
+ def reset
45
+ setup_mysql
46
+ index
47
+ end
48
+
49
+ def generate_configuration
50
+ template = File.open("spec/fixtures/sphinx/configuration.erb") { |f| f.read }
51
+ File.open("spec/fixtures/sphinx/spec.conf", "w") { |f|
52
+ f.puts ERB.new(template).result(binding)
53
+ }
54
+ end
55
+
56
+ def index
57
+ cmd = "indexer --config #{@path}/fixtures/sphinx/spec.conf --all"
58
+ cmd << " --rotate" if running?
59
+ `#{cmd}`
60
+ end
61
+
62
+ def start
63
+ return if running?
64
+
65
+ cmd = "searchd --config #{@path}/fixtures/sphinx/spec.conf"
66
+ `#{cmd}`
67
+
68
+ sleep(1)
69
+
70
+ unless running?
71
+ puts "Failed to start searchd daemon. Check fixtures/sphinx/searchd.log."
72
+ end
73
+ end
74
+
75
+ def stop
76
+ return unless running?
77
+ `kill #{pid}`
78
+ end
79
+
80
+ def pid
81
+ if File.exists?("#{@path}/fixtures/sphinx/searchd.pid")
82
+ `cat #{@path}/fixtures/sphinx/searchd.pid`[/\d+/]
83
+ else
84
+ nil
85
+ end
86
+ end
87
+
88
+ def running?
89
+ pid && `ps #{pid} | wc -l`.to_i > 1
90
+ end
91
+ end
@@ -0,0 +1,140 @@
1
+ require 'spec/spec_helper'
2
+
3
+ describe Riddle::Client do
4
+ it "should have the same keys for both commands and versions" do
5
+ Riddle::Client::Commands.keys.should == Riddle::Client::Versions.keys
6
+ end
7
+
8
+ it "should default to localhost as the server" do
9
+ Riddle::Client.new.server.should == "localhost"
10
+ end
11
+
12
+ it "should default to port 3312" do
13
+ Riddle::Client.new.port.should == 3312
14
+ end
15
+
16
+ it "should translate anchor arguments correctly" do
17
+ client = Riddle::Client.new
18
+ client.set_anchor "latitude", 10.0, "longtitude", 95.0
19
+ client.anchor.should == {
20
+ :latitude_attribute => "latitude",
21
+ :latitude => 10.0,
22
+ :longtitude_attribute => "longtitude",
23
+ :longtitude => 95.0
24
+ }
25
+ end
26
+
27
+ it "should add queries to the queue" do
28
+ client = Riddle::Client.new
29
+ client.queue.should be_empty
30
+ client.append_query "spec"
31
+ client.queue.should_not be_empty
32
+ end
33
+
34
+ it "should build a basic search message correctly" do
35
+ client = Riddle::Client.new
36
+ client.append_query "test "
37
+ client.queue.first.should == query_contents(:simple)
38
+ end
39
+
40
+ it "should build a message with a specified index correctly" do
41
+ client = Riddle::Client.new
42
+ client.append_query "test ", "edition"
43
+ client.queue.first.should == query_contents(:index)
44
+ end
45
+
46
+ it "should build a message using match mode :any correctly" do
47
+ client = Riddle::Client.new
48
+ client.match_mode = :any
49
+ client.append_query "test this "
50
+ client.queue.first.should == query_contents(:any)
51
+ end
52
+
53
+ it "should build a message using sort by correctly" do
54
+ client = Riddle::Client.new
55
+ client.sort_by = 'id'
56
+ client.sort_mode = :extended
57
+ client.append_query "testing "
58
+ client.queue.first.should == query_contents(:sort)
59
+ end
60
+
61
+ it "should build a message using match mode :boolean correctly" do
62
+ client = Riddle::Client.new
63
+ client.match_mode = :boolean
64
+ client.append_query "test "
65
+ client.queue.first.should == query_contents(:boolean)
66
+ end
67
+
68
+ it "should build a message using match mode :phrase correctly" do
69
+ client = Riddle::Client.new
70
+ client.match_mode = :phrase
71
+ client.append_query "testing this "
72
+ client.queue.first.should == query_contents(:phrase)
73
+ end
74
+
75
+ it "should build a message with a filter correctly" do
76
+ client = Riddle::Client.new
77
+ client.filters << Riddle::Client::Filter.new("id", [10, 100, 1000])
78
+ client.append_query "test "
79
+ client.queue.first.should == query_contents(:filter)
80
+ end
81
+
82
+ it "should build a message with group values correctly" do
83
+ client = Riddle::Client.new
84
+ client.group_by = "id"
85
+ client.group_function = :attr
86
+ client.group_clause = "id"
87
+ client.append_query "test "
88
+ client.queue.first.should == query_contents(:group)
89
+ end
90
+
91
+ it "should build a message with group distinct value correctly" do
92
+ client = Riddle::Client.new
93
+ client.group_distinct = "id"
94
+ client.append_query "test "
95
+ client.queue.first.should == query_contents(:distinct)
96
+ end
97
+
98
+ it "should build a message with weights correctly" do
99
+ client = Riddle::Client.new
100
+ client.weights = [100, 1]
101
+ client.append_query "test "
102
+ client.queue.first.should == query_contents(:weights)
103
+ end
104
+
105
+ it "should build a message with an anchor correctly" do
106
+ client = Riddle::Client.new
107
+ client.set_anchor "latitude", 10.0, "longtitude", 95.0
108
+ client.append_query "test "
109
+ client.queue.first.should == query_contents(:anchor)
110
+ end
111
+
112
+ it "should keep multiple messages in the queue" do
113
+ client = Riddle::Client.new
114
+ client.weights = [100, 1]
115
+ client.append_query "test "
116
+ client.append_query "test "
117
+ client.queue.length.should == 2
118
+ client.queue.each { |item| item.should == query_contents(:weights) }
119
+ end
120
+
121
+ it "should keep multiple messages in the queue with different params" do
122
+ client = Riddle::Client.new
123
+ client.weights = [100, 1]
124
+ client.append_query "test "
125
+ client.weights = []
126
+ client.append_query "test ", "edition"
127
+ client.queue.first.should == query_contents(:weights)
128
+ client.queue.last.should == query_contents(:index)
129
+ end
130
+
131
+ it "should build a basic update message correctly" do
132
+ client = Riddle::Client.new
133
+ client.send(
134
+ :update_message,
135
+ "people",
136
+ ["birthday"],
137
+ {1 => [191163600]}
138
+ ).should == query_contents(:update_simple)
139
+ end
140
+ end