dm-sphinx-adapter 0.5 → 0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/History.txt +5 -0
- data/Manifest.txt +12 -19
- data/README.txt +20 -41
- data/Rakefile +2 -3
- data/dm-sphinx-adapter.gemspec +6 -9
- data/lib/dm-sphinx-adapter.rb +14 -11
- data/lib/dm-sphinx-adapter/adapter.rb +36 -62
- data/lib/dm-sphinx-adapter/attribute.rb +48 -3
- data/lib/riddle.rb +28 -0
- data/lib/riddle/client.rb +619 -0
- data/lib/riddle/client/filter.rb +53 -0
- data/lib/riddle/client/message.rb +65 -0
- data/lib/riddle/client/response.rb +84 -0
- data/test/files/model.rb +23 -0
- data/test/files/mysql5.sphinx.conf +97 -0
- data/test/files/mysql5.sql +26 -0
- data/test/helper.rb +51 -0
- data/test/test_adapter.rb +74 -28
- data/test/test_attribute.rb +36 -0
- data/test/test_index.rb +30 -0
- data/test/test_query.rb +47 -32
- data/test/test_resource.rb +17 -0
- metadata +18 -41
- data/lib/dm-sphinx-adapter/client.rb +0 -109
- data/lib/dm-sphinx-adapter/config.rb +0 -122
- data/lib/dm-sphinx-adapter/config_parser.rb +0 -71
- data/test/files/dm_sphinx_adapter_test.sql +0 -21
- data/test/files/resource_explicit.rb +0 -25
- data/test/files/resource_resource.rb +0 -19
- data/test/files/resource_searchable.rb +0 -16
- data/test/files/resource_storage_name.rb +0 -11
- data/test/files/resource_vanilla.rb +0 -7
- data/test/files/sphinx.conf +0 -78
- data/test/test_adapter_explicit.rb +0 -48
- data/test/test_adapter_resource.rb +0 -25
- data/test/test_adapter_searchable.rb +0 -23
- data/test/test_adapter_vanilla.rb +0 -46
- data/test/test_client.rb +0 -31
- data/test/test_config.rb +0 -75
- data/test/test_config_parser.rb +0 -29
- data/test/test_type_attribute.rb +0 -8
- data/test/test_type_index.rb +0 -8
@@ -0,0 +1,53 @@
|
|
1
|
+
module Riddle
|
2
|
+
class Client
|
3
|
+
# Used for querying Sphinx.
|
4
|
+
class Filter
|
5
|
+
attr_accessor :attribute, :values, :exclude
|
6
|
+
|
7
|
+
# Attribute name, values (which can be an array or a range), and whether
|
8
|
+
# the filter should be exclusive.
|
9
|
+
def initialize(attribute, values, exclude=false)
|
10
|
+
@attribute, @values, @exclude = attribute, values, exclude
|
11
|
+
end
|
12
|
+
|
13
|
+
def exclude?
|
14
|
+
self.exclude
|
15
|
+
end
|
16
|
+
|
17
|
+
# Returns the message for this filter to send to the Sphinx service
|
18
|
+
def query_message
|
19
|
+
message = Message.new
|
20
|
+
|
21
|
+
message.append_string self.attribute.to_s
|
22
|
+
case self.values
|
23
|
+
when Range
|
24
|
+
if self.values.first.is_a?(Float) && self.values.last.is_a?(Float)
|
25
|
+
message.append_int FilterTypes[:float_range]
|
26
|
+
message.append_floats self.values.first, self.values.last
|
27
|
+
else
|
28
|
+
message.append_int FilterTypes[:range]
|
29
|
+
message.append_ints self.values.first, self.values.last
|
30
|
+
end
|
31
|
+
when Array
|
32
|
+
message.append_int FilterTypes[:values]
|
33
|
+
message.append_int self.values.length
|
34
|
+
# using to_f is a hack from the php client - to workaround 32bit
|
35
|
+
# signed ints on x32 platforms
|
36
|
+
message.append_ints *self.values.collect { |val|
|
37
|
+
case val
|
38
|
+
when TrueClass
|
39
|
+
1.0
|
40
|
+
when FalseClass
|
41
|
+
0.0
|
42
|
+
else
|
43
|
+
val.to_f
|
44
|
+
end
|
45
|
+
}
|
46
|
+
end
|
47
|
+
message.append_int self.exclude? ? 1 : 0
|
48
|
+
|
49
|
+
message.to_s
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
@@ -0,0 +1,65 @@
|
|
1
|
+
module Riddle
|
2
|
+
class Client
|
3
|
+
# This class takes care of the translation of ints, strings and arrays to
|
4
|
+
# the format required by the Sphinx service.
|
5
|
+
class Message
|
6
|
+
def initialize
|
7
|
+
@message = ""
|
8
|
+
@size_method = @message.respond_to?(:bytesize) ? :bytesize : :length
|
9
|
+
end
|
10
|
+
|
11
|
+
# Append raw data (only use if you know what you're doing)
|
12
|
+
def append(*args)
|
13
|
+
return if args.length == 0
|
14
|
+
|
15
|
+
args.each { |arg| @message << arg }
|
16
|
+
end
|
17
|
+
|
18
|
+
# Append a string's length, then the string itself
|
19
|
+
def append_string(str)
|
20
|
+
@message << [str.send(@size_method)].pack('N') + str
|
21
|
+
end
|
22
|
+
|
23
|
+
# Append an integer
|
24
|
+
def append_int(int)
|
25
|
+
@message << [int].pack('N')
|
26
|
+
end
|
27
|
+
|
28
|
+
def append_64bit_int(int)
|
29
|
+
@message << [int >> 32, int & 0xFFFFFFFF].pack('NN')
|
30
|
+
end
|
31
|
+
|
32
|
+
# Append a float
|
33
|
+
def append_float(float)
|
34
|
+
@message << [float].pack('f').unpack('L*').pack("N")
|
35
|
+
end
|
36
|
+
|
37
|
+
# Append multiple integers
|
38
|
+
def append_ints(*ints)
|
39
|
+
ints.each { |int| append_int(int) }
|
40
|
+
end
|
41
|
+
|
42
|
+
def append_64bit_ints(*ints)
|
43
|
+
ints.each { |int| append_64bit_int(int) }
|
44
|
+
end
|
45
|
+
|
46
|
+
# Append multiple floats
|
47
|
+
def append_floats(*floats)
|
48
|
+
floats.each { |float| append_float(float) }
|
49
|
+
end
|
50
|
+
|
51
|
+
# Append an array of strings - first appends the length of the array,
|
52
|
+
# then each item's length and value.
|
53
|
+
def append_array(array)
|
54
|
+
append_int(array.length)
|
55
|
+
|
56
|
+
array.each { |item| append_string(item) }
|
57
|
+
end
|
58
|
+
|
59
|
+
# Returns the entire message
|
60
|
+
def to_s
|
61
|
+
@message
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
@@ -0,0 +1,84 @@
|
|
1
|
+
module Riddle
|
2
|
+
class Client
|
3
|
+
# Used to interrogate responses from the Sphinx daemon. Keep in mind none
|
4
|
+
# of the methods here check whether the data they're grabbing are what the
|
5
|
+
# user expects - it just assumes the user knows what the data stream is
|
6
|
+
# made up of.
|
7
|
+
class Response
|
8
|
+
# Create with the data to interpret
|
9
|
+
def initialize(str)
|
10
|
+
@str = str
|
11
|
+
@marker = 0
|
12
|
+
end
|
13
|
+
|
14
|
+
# Return the next string value in the stream
|
15
|
+
def next
|
16
|
+
len = next_int
|
17
|
+
result = @str[@marker, len]
|
18
|
+
@marker += len
|
19
|
+
|
20
|
+
return result
|
21
|
+
end
|
22
|
+
|
23
|
+
# Return the next integer value from the stream
|
24
|
+
def next_int
|
25
|
+
int = @str[@marker, 4].unpack('N*').first
|
26
|
+
@marker += 4
|
27
|
+
|
28
|
+
return int
|
29
|
+
end
|
30
|
+
|
31
|
+
def next_64bit_int
|
32
|
+
high, low = @str[@marker, 8].unpack('N*N*')[0..1]
|
33
|
+
@marker += 8
|
34
|
+
|
35
|
+
return (high << 32) + low
|
36
|
+
end
|
37
|
+
|
38
|
+
# Return the next float value from the stream
|
39
|
+
def next_float
|
40
|
+
float = @str[@marker, 4].unpack('N*').pack('L').unpack('f*').first
|
41
|
+
@marker += 4
|
42
|
+
|
43
|
+
return float
|
44
|
+
end
|
45
|
+
|
46
|
+
# Returns an array of string items
|
47
|
+
def next_array
|
48
|
+
count = next_int
|
49
|
+
items = []
|
50
|
+
for i in 0...count
|
51
|
+
items << self.next
|
52
|
+
end
|
53
|
+
|
54
|
+
return items
|
55
|
+
end
|
56
|
+
|
57
|
+
# Returns an array of int items
|
58
|
+
def next_int_array
|
59
|
+
count = next_int
|
60
|
+
items = []
|
61
|
+
for i in 0...count
|
62
|
+
items << self.next_int
|
63
|
+
end
|
64
|
+
|
65
|
+
return items
|
66
|
+
end
|
67
|
+
|
68
|
+
def next_float_array
|
69
|
+
count = next_int
|
70
|
+
items = []
|
71
|
+
for i in 0...count
|
72
|
+
items << self.next_float
|
73
|
+
end
|
74
|
+
|
75
|
+
return items
|
76
|
+
end
|
77
|
+
|
78
|
+
# Returns the length of the streamed data
|
79
|
+
def length
|
80
|
+
@str.length
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
data/test/files/model.rb
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
class Item
|
2
|
+
include DataMapper::SphinxResource
|
3
|
+
property :id, Serial
|
4
|
+
property :t_string, String
|
5
|
+
property :t_text, Text, :lazy => false
|
6
|
+
property :t_decimal, BigDecimal
|
7
|
+
property :t_float, Float
|
8
|
+
property :t_integer, Integer
|
9
|
+
property :t_datetime, DateTime
|
10
|
+
|
11
|
+
repository(:search) do
|
12
|
+
properties(:search).clear
|
13
|
+
property :id, Serial
|
14
|
+
property :t_string, String
|
15
|
+
|
16
|
+
attribute :t_text, Text, :lazy => false
|
17
|
+
attribute :t_decimal, BigDecimal
|
18
|
+
attribute :t_float, Float
|
19
|
+
attribute :t_integer, Integer
|
20
|
+
attribute :t_datetime, DateTime
|
21
|
+
end
|
22
|
+
end # Item
|
23
|
+
|
@@ -0,0 +1,97 @@
|
|
1
|
+
# searchd and indexer must be run from the root directory of this lib.
|
2
|
+
|
3
|
+
indexer
|
4
|
+
{
|
5
|
+
mem_limit = 64M
|
6
|
+
}
|
7
|
+
|
8
|
+
searchd
|
9
|
+
{
|
10
|
+
address = localhost
|
11
|
+
port = 3312
|
12
|
+
log = test/files/tmp/sphinx.log
|
13
|
+
query_log = test/files/tmp/sphinx.query.log
|
14
|
+
read_timeout = 5
|
15
|
+
pid_file = test/files/tmp/sphinx.pid
|
16
|
+
max_matches = 1000
|
17
|
+
}
|
18
|
+
|
19
|
+
source items
|
20
|
+
{
|
21
|
+
type = mysql
|
22
|
+
sql_host = localhost
|
23
|
+
sql_user = root
|
24
|
+
sql_pass =
|
25
|
+
sql_db = dm_sphinx_adapter_test
|
26
|
+
|
27
|
+
sql_query_pre = set names utf8
|
28
|
+
sql_query_pre = \
|
29
|
+
replace into delta (name, updated_on) ( \
|
30
|
+
select 'items', t_datetime \
|
31
|
+
from items \
|
32
|
+
order by t_datetime desc \
|
33
|
+
limit 1\
|
34
|
+
)
|
35
|
+
sql_query_info = select * from items where id = $id
|
36
|
+
|
37
|
+
sql_query_pre = set names utf8
|
38
|
+
sql_query = \
|
39
|
+
select \
|
40
|
+
id, \
|
41
|
+
t_string, \
|
42
|
+
t_text, \
|
43
|
+
t_decimal, \
|
44
|
+
t_float, \
|
45
|
+
t_integer, \
|
46
|
+
unix_timestamp(t_datetime) as t_datetime \
|
47
|
+
from items \
|
48
|
+
where t_datetime <= ( \
|
49
|
+
select updated_on \
|
50
|
+
from delta \
|
51
|
+
where name = 'items' \
|
52
|
+
)
|
53
|
+
|
54
|
+
sql_attr_float = t_decimal
|
55
|
+
sql_attr_float = t_float
|
56
|
+
sql_attr_uint = t_integer
|
57
|
+
sql_attr_timestamp = t_datetime
|
58
|
+
}
|
59
|
+
|
60
|
+
source items_delta : items {
|
61
|
+
sql_query_pre = set names utf8
|
62
|
+
sql_query_pre =
|
63
|
+
sql_query = \
|
64
|
+
select \
|
65
|
+
id, \
|
66
|
+
t_string, \
|
67
|
+
t_text, \
|
68
|
+
t_decimal, \
|
69
|
+
t_float, \
|
70
|
+
t_integer, \
|
71
|
+
unix_timestamp(t_datetime) as t_datetime \
|
72
|
+
from items \
|
73
|
+
where t_datetime > ( \
|
74
|
+
select updated_on \
|
75
|
+
from delta \
|
76
|
+
where name = 'items' \
|
77
|
+
)
|
78
|
+
}
|
79
|
+
|
80
|
+
index items_main
|
81
|
+
{
|
82
|
+
source = items
|
83
|
+
path = test/files/tmp/items_main
|
84
|
+
}
|
85
|
+
|
86
|
+
index items_delta : items_main
|
87
|
+
{
|
88
|
+
source = items_delta
|
89
|
+
path = test/files/tmp/items_delta
|
90
|
+
}
|
91
|
+
|
92
|
+
index items
|
93
|
+
{
|
94
|
+
type = distributed
|
95
|
+
local = items_main
|
96
|
+
local = items_delta
|
97
|
+
}
|
@@ -0,0 +1,26 @@
|
|
1
|
+
drop table if exists delta;
|
2
|
+
create table delta (
|
3
|
+
name varchar(50) not null,
|
4
|
+
updated_on datetime,
|
5
|
+
primary key (name)
|
6
|
+
) engine=innodb default charset=utf8;
|
7
|
+
|
8
|
+
insert into delta (name, updated_on) values
|
9
|
+
('items', now());
|
10
|
+
|
11
|
+
drop table if exists items;
|
12
|
+
create table items (
|
13
|
+
id int(11) not null auto_increment,
|
14
|
+
t_string varchar(50),
|
15
|
+
t_text text,
|
16
|
+
t_decimal decimal(30,10),
|
17
|
+
t_float float,
|
18
|
+
t_integer int,
|
19
|
+
t_datetime datetime,
|
20
|
+
primary key (id)
|
21
|
+
) engine=innodb default charset=utf8;
|
22
|
+
|
23
|
+
insert into items (t_string, t_text, t_decimal, t_float, t_integer, t_datetime) values
|
24
|
+
('one', 'text one!', '10.50', '100.50', '1000', now()),
|
25
|
+
('two', 'text two!', '20.50', '200.50', '2000', now()),
|
26
|
+
('three', 'text three!', '30.50', '300.50', '3000', now());
|
data/test/helper.rb
ADDED
@@ -0,0 +1,51 @@
|
|
1
|
+
$VERBOSE = false # Shitloads of warnings in dm :(
|
2
|
+
require 'rubygems'
|
3
|
+
require 'extlib'
|
4
|
+
require 'extlib/hook'
|
5
|
+
require 'pathname'
|
6
|
+
require 'shoulda'
|
7
|
+
require 'test/unit'
|
8
|
+
|
9
|
+
base = Pathname.new(__FILE__).dirname + '..'
|
10
|
+
%w{lib test}.each{|p| $:.unshift base + p}
|
11
|
+
|
12
|
+
require 'dm-sphinx-adapter'
|
13
|
+
|
14
|
+
# Sphinx runner.
|
15
|
+
Dir.chdir(base)
|
16
|
+
config = base + 'test' + 'files' + 'mysql5.sphinx.conf'
|
17
|
+
begin
|
18
|
+
TCPSocket.new('localhost', '3312')
|
19
|
+
rescue
|
20
|
+
puts 'Starting Sphinx...'
|
21
|
+
system("searchd --config #{config}") || exit
|
22
|
+
system('ps aux | grep searchd')
|
23
|
+
end
|
24
|
+
|
25
|
+
class Test::Unit::TestCase
|
26
|
+
include Extlib::Hook
|
27
|
+
|
28
|
+
before :setup do
|
29
|
+
files = Pathname.new(__FILE__).dirname + 'files'
|
30
|
+
|
31
|
+
mysql = `mysql5 dm_sphinx_adapter_test < #{files + 'mysql5.sql'} 2>&1`
|
32
|
+
raise %{Re-create database failed:\n #{mysql}} unless mysql.blank?
|
33
|
+
|
34
|
+
indexer = `indexer --config #{files + 'mysql5.sphinx.conf'} --all --rotate`
|
35
|
+
raise %{Re-create index failed:\n #{indexer}} if indexer =~ /error|fatal/i
|
36
|
+
|
37
|
+
DataMapper.setup(:default, :adapter => 'mysql', :database => 'dm_sphinx_adapter_test')
|
38
|
+
sleep 1; # Give sphinx a chance to catch up before test runs.
|
39
|
+
end
|
40
|
+
|
41
|
+
# after :teardown do
|
42
|
+
def teardown
|
43
|
+
descendants = DataMapper::Resource.descendants.dup.to_a
|
44
|
+
while model = descendants.shift
|
45
|
+
descendants.concat(model.descendants) if model.respond_to?(:descendants)
|
46
|
+
Object.send(:remove_const, model.name.to_sym)
|
47
|
+
DataMapper::Resource.descendants.delete(model)
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
data/test/test_adapter.rb
CHANGED
@@ -1,38 +1,84 @@
|
|
1
|
-
require '
|
2
|
-
require 'test/unit'
|
3
|
-
|
4
|
-
# DataMapper::Logger.new(STDOUT, :debug)
|
1
|
+
require File.join(File.dirname(__FILE__), 'helper')
|
5
2
|
|
6
3
|
class TestAdapter < Test::Unit::TestCase
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
4
|
+
context 'DM::A::Sphinx::Adapter class' do
|
5
|
+
setup do
|
6
|
+
DataMapper.setup(:adapter, :adapter => 'sphinx')
|
7
|
+
load File.join(File.dirname(__FILE__), 'files', 'model.rb')
|
8
|
+
@it = repository(:adapter)
|
9
|
+
@resource = Item
|
12
10
|
end
|
13
11
|
|
14
|
-
|
12
|
+
context '#create' do
|
13
|
+
should 'should return zero records created' do
|
14
|
+
assert_equal 0, @it.create(create_resource)
|
15
|
+
end
|
16
|
+
end
|
15
17
|
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
18
|
+
context '#delete' do
|
19
|
+
should 'should return zero records deleted' do
|
20
|
+
assert_equal 0, @it.delete(create_resource)
|
21
|
+
end
|
22
|
+
end
|
21
23
|
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
24
|
+
context '#read_many' do
|
25
|
+
context 'conditions' do
|
26
|
+
should 'return all objects when nil' do
|
27
|
+
assert_equal [{:id => 1}, {:id => 2}, {:id => 3}], @it.read_many(query)
|
28
|
+
end
|
27
29
|
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
30
|
+
should 'return subset of objects for conditions' do
|
31
|
+
assert_equal [{:id => 2}], @it.read_many(query(:t_string => 'two'))
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
context 'offsets' do
|
36
|
+
should 'be able to offset the objects' do
|
37
|
+
assert_equal [{:id => 1}, {:id => 2}, {:id => 3}], @it.read_many(query(:offset => 0))
|
38
|
+
assert_equal [{:id => 2}, {:id => 3}], @it.read_many(query(:offset => 1))
|
39
|
+
assert_equal [], @it.read_many(query(:offset => 3))
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
context 'limits' do
|
44
|
+
should 'be able to limit the objects' do
|
45
|
+
assert_equal [{:id => 1}], @it.read_many(query(:limit => 1))
|
46
|
+
assert_equal [{:id => 1}, {:id => 2}], @it.read_many(query(:limit => 2))
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
33
50
|
|
34
|
-
|
35
|
-
|
36
|
-
|
51
|
+
context '#read_one' do
|
52
|
+
should 'return the first object of a #read_many' do
|
53
|
+
assert_equal @it.read_many(query).first, @it.read_one(query)
|
54
|
+
|
55
|
+
query = query(:t_string => 'two')
|
56
|
+
assert_equal @it.read_many(query).first, @it.read_one(query)
|
57
|
+
end
|
58
|
+
end
|
37
59
|
end
|
60
|
+
|
61
|
+
protected
|
62
|
+
def query(conditions = {})
|
63
|
+
DataMapper::Query.new(repository(:adapter), @resource, conditions)
|
64
|
+
end
|
65
|
+
|
66
|
+
def resource(options = {})
|
67
|
+
now = Time.now
|
68
|
+
attributes = {
|
69
|
+
:t_string => now.to_s,
|
70
|
+
:t_text => "text #{now.to_s}!",
|
71
|
+
:t_decimal => now.to_i * 0.001,
|
72
|
+
:t_float => now.to_i * 0.0001,
|
73
|
+
:t_integer => now.to_i,
|
74
|
+
:t_datetime => now
|
75
|
+
}.update(options)
|
76
|
+
@resource.new(attributes)
|
77
|
+
end
|
78
|
+
|
79
|
+
def create_resource(options = {})
|
80
|
+
repository(:adapter) do
|
81
|
+
@resource.create(resource(options).attributes.except(:id))
|
82
|
+
end
|
83
|
+
end
|
38
84
|
end
|