dm-sphinx-adapter 0.5 → 0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/History.txt +5 -0
- data/Manifest.txt +12 -19
- data/README.txt +20 -41
- data/Rakefile +2 -3
- data/dm-sphinx-adapter.gemspec +6 -9
- data/lib/dm-sphinx-adapter.rb +14 -11
- data/lib/dm-sphinx-adapter/adapter.rb +36 -62
- data/lib/dm-sphinx-adapter/attribute.rb +48 -3
- data/lib/riddle.rb +28 -0
- data/lib/riddle/client.rb +619 -0
- data/lib/riddle/client/filter.rb +53 -0
- data/lib/riddle/client/message.rb +65 -0
- data/lib/riddle/client/response.rb +84 -0
- data/test/files/model.rb +23 -0
- data/test/files/mysql5.sphinx.conf +97 -0
- data/test/files/mysql5.sql +26 -0
- data/test/helper.rb +51 -0
- data/test/test_adapter.rb +74 -28
- data/test/test_attribute.rb +36 -0
- data/test/test_index.rb +30 -0
- data/test/test_query.rb +47 -32
- data/test/test_resource.rb +17 -0
- metadata +18 -41
- data/lib/dm-sphinx-adapter/client.rb +0 -109
- data/lib/dm-sphinx-adapter/config.rb +0 -122
- data/lib/dm-sphinx-adapter/config_parser.rb +0 -71
- data/test/files/dm_sphinx_adapter_test.sql +0 -21
- data/test/files/resource_explicit.rb +0 -25
- data/test/files/resource_resource.rb +0 -19
- data/test/files/resource_searchable.rb +0 -16
- data/test/files/resource_storage_name.rb +0 -11
- data/test/files/resource_vanilla.rb +0 -7
- data/test/files/sphinx.conf +0 -78
- data/test/test_adapter_explicit.rb +0 -48
- data/test/test_adapter_resource.rb +0 -25
- data/test/test_adapter_searchable.rb +0 -23
- data/test/test_adapter_vanilla.rb +0 -46
- data/test/test_client.rb +0 -31
- data/test/test_config.rb +0 -75
- data/test/test_config_parser.rb +0 -29
- data/test/test_type_attribute.rb +0 -8
- data/test/test_type_index.rb +0 -8
@@ -1,71 +0,0 @@
|
|
1
|
-
require 'rubygems'
|
2
|
-
|
3
|
-
gem 'extlib', '~> 0.9.7'
|
4
|
-
require 'extlib'
|
5
|
-
|
6
|
-
require 'pathname'
|
7
|
-
require 'strscan'
|
8
|
-
|
9
|
-
module DataMapper
|
10
|
-
module Adapters
|
11
|
-
module Sphinx
|
12
|
-
module ConfigParser
|
13
|
-
extend Extlib::Assertions
|
14
|
-
|
15
|
-
# Parse a sphinx config file and return searchd options as a hash.
|
16
|
-
#
|
17
|
-
# ==== Raises
|
18
|
-
# RuntimeError:: On parsing errors.
|
19
|
-
#
|
20
|
-
# ==== Parameters
|
21
|
-
# path<String>:: Sphinx configuration file.
|
22
|
-
#
|
23
|
-
# ==== Returns
|
24
|
-
# Hash:: Searchd options.
|
25
|
-
def self.parse(path)
|
26
|
-
assert_kind_of 'path', path, Pathname, String
|
27
|
-
|
28
|
-
config = Pathname(path).read
|
29
|
-
config.gsub!(/\r\n|\r|\n/, "\n") # Everything in \n
|
30
|
-
config.gsub!(/\s*\\\n\s*/, ' ') # Remove unixy line wraps.
|
31
|
-
|
32
|
-
blocks(StringScanner.new(config), out = [])
|
33
|
-
out.find{|c| c['type'] =~ /searchd/i} || {}
|
34
|
-
end
|
35
|
-
|
36
|
-
protected
|
37
|
-
def self.blocks(conf, out = []) #:nodoc:
|
38
|
-
if conf.scan(/\#[^\n]*\n/) || conf.scan(/\s+/)
|
39
|
-
blocks(conf, out)
|
40
|
-
elsif conf.scan(/indexer|searchd|source|index/i)
|
41
|
-
out << group = {'type' => conf.matched}
|
42
|
-
if conf.matched =~ /^(?:index|source)$/i
|
43
|
-
conf.scan(/\s* ([\w_\-]+) (?:\s*:\s*([\w_\-]+))? \s*/x) or raise "Expected #{group[:type]} name."
|
44
|
-
group['name'] = conf[1]
|
45
|
-
group['ancestor'] = conf[2]
|
46
|
-
end
|
47
|
-
conf.scan(/\s*\{/) or raise %q{Expected '\{'.}
|
48
|
-
pairs(conf, kv = {})
|
49
|
-
group.merge!(kv)
|
50
|
-
conf.scan(/\s*\}/) or raise %q{Expected '\}'.}
|
51
|
-
blocks(conf, out)
|
52
|
-
else
|
53
|
-
raise "Unknown near: #{conf.peek(30)}" unless conf.eos?
|
54
|
-
end
|
55
|
-
end
|
56
|
-
|
57
|
-
def self.pairs(conf, out = {}) #:nodoc:
|
58
|
-
if conf.scan(/\#[^\n]*\n/) || conf.scan(/\s+/)
|
59
|
-
pairs(conf, out)
|
60
|
-
elsif conf.scan(/[\w_-]+/)
|
61
|
-
key = conf.matched
|
62
|
-
conf.scan(/\s*=/) or raise %q{Expected '='.}
|
63
|
-
out[key] = conf.scan(/[^\n]*\n/).strip
|
64
|
-
pairs(conf, out)
|
65
|
-
end
|
66
|
-
end
|
67
|
-
end # ConfigParser
|
68
|
-
end # Sphinx
|
69
|
-
end # Adapters
|
70
|
-
end # DataMapper
|
71
|
-
|
@@ -1,21 +0,0 @@
|
|
1
|
-
drop table if exists delta;
|
2
|
-
create table delta (
|
3
|
-
name varchar(50) not null,
|
4
|
-
updated_on datetime,
|
5
|
-
primary key (name)
|
6
|
-
) engine=innodb default charset=utf8;
|
7
|
-
|
8
|
-
drop table if exists items;
|
9
|
-
create table items (
|
10
|
-
id int(11) not null auto_increment,
|
11
|
-
name varchar(50) not null,
|
12
|
-
likes text not null,
|
13
|
-
updated_on datetime,
|
14
|
-
primary key (id),
|
15
|
-
index (updated_on)
|
16
|
-
) engine=innodb default charset=utf8;
|
17
|
-
|
18
|
-
insert into items (name, likes, updated_on) values
|
19
|
-
('one', 'I really like one!', now()),
|
20
|
-
('two', 'I really like two!', now()),
|
21
|
-
('three', 'I really like three!', now());
|
@@ -1,25 +0,0 @@
|
|
1
|
-
require 'rubygems'
|
2
|
-
require 'dm-is-searchable'
|
3
|
-
|
4
|
-
class Explicit
|
5
|
-
include DataMapper::Resource
|
6
|
-
include DataMapper::SphinxResource
|
7
|
-
|
8
|
-
property :id, Serial
|
9
|
-
property :name, String
|
10
|
-
property :likes, Text, :lazy => false
|
11
|
-
property :updated_on, DateTime
|
12
|
-
|
13
|
-
is :searchable
|
14
|
-
repository(:search) do
|
15
|
-
properties(:search).clear
|
16
|
-
index :items
|
17
|
-
index :items_delta, :delta => true
|
18
|
-
property :name, String
|
19
|
-
attribute :updated_on, DateTime
|
20
|
-
end
|
21
|
-
|
22
|
-
def self.default_storage_name
|
23
|
-
'item'
|
24
|
-
end
|
25
|
-
end # Explicit
|
@@ -1,19 +0,0 @@
|
|
1
|
-
require 'rubygems'
|
2
|
-
require 'dm-is-searchable'
|
3
|
-
|
4
|
-
class Resource
|
5
|
-
include DataMapper::Resource
|
6
|
-
include DataMapper::SphinxResource
|
7
|
-
|
8
|
-
property :id, Serial
|
9
|
-
property :name, String
|
10
|
-
property :likes, Text, :lazy => false
|
11
|
-
property :updated_on, DateTime
|
12
|
-
|
13
|
-
is :searchable
|
14
|
-
|
15
|
-
def self.default_storage_name
|
16
|
-
'item'
|
17
|
-
end
|
18
|
-
end # Resource
|
19
|
-
|
@@ -1,16 +0,0 @@
|
|
1
|
-
require 'rubygems'
|
2
|
-
require 'dm-is-searchable'
|
3
|
-
|
4
|
-
class Searchable
|
5
|
-
include DataMapper::Resource
|
6
|
-
property :id, Serial
|
7
|
-
property :name, String
|
8
|
-
property :likes, Text, :lazy => false
|
9
|
-
property :updated_on, DateTime
|
10
|
-
|
11
|
-
is :searchable
|
12
|
-
|
13
|
-
def self.default_storage_name
|
14
|
-
'item'
|
15
|
-
end
|
16
|
-
end # Searchable
|
data/test/files/sphinx.conf
DELETED
@@ -1,78 +0,0 @@
|
|
1
|
-
# searchd and indexer must be run from the root directory of this lib.
|
2
|
-
|
3
|
-
indexer
|
4
|
-
{
|
5
|
-
mem_limit = 64M
|
6
|
-
}
|
7
|
-
|
8
|
-
searchd
|
9
|
-
{
|
10
|
-
address = localhost
|
11
|
-
port = 3312
|
12
|
-
log = test/var/sphinx.log
|
13
|
-
query_log = test/var/sphinx.query.log
|
14
|
-
read_timeout = 5
|
15
|
-
pid_file = test/var/sphinx.pid
|
16
|
-
max_matches = 1000
|
17
|
-
}
|
18
|
-
|
19
|
-
source items
|
20
|
-
{
|
21
|
-
type = mysql
|
22
|
-
sql_host = localhost
|
23
|
-
sql_user = root
|
24
|
-
sql_pass =
|
25
|
-
sql_db = dm_sphinx_adapter_test
|
26
|
-
|
27
|
-
sql_query_pre = set names utf8
|
28
|
-
sql_query_pre = \
|
29
|
-
replace into delta (name, updated_on) ( \
|
30
|
-
select 'items', updated_on \
|
31
|
-
from items \
|
32
|
-
order by updated_on desc \
|
33
|
-
limit 1\
|
34
|
-
)
|
35
|
-
|
36
|
-
sql_query = \
|
37
|
-
select id, name, likes, unix_timestamp(updated_on) as updated_on \
|
38
|
-
from items \
|
39
|
-
where updated_on <= ( \
|
40
|
-
select updated_on \
|
41
|
-
from delta \
|
42
|
-
where name = 'items'\
|
43
|
-
)
|
44
|
-
|
45
|
-
sql_query_info = select * from items where id = $id
|
46
|
-
sql_attr_timestamp = updated_on
|
47
|
-
}
|
48
|
-
|
49
|
-
source items_delta : items
|
50
|
-
{
|
51
|
-
sql_query_pre = set names utf8
|
52
|
-
sql_query = \
|
53
|
-
select id, name, likes, unix_timestamp(updated_on) as updated_on \
|
54
|
-
from items \
|
55
|
-
where updated_on > ( \
|
56
|
-
select updated_on \
|
57
|
-
from delta \
|
58
|
-
where name = 'items'\
|
59
|
-
)
|
60
|
-
}
|
61
|
-
|
62
|
-
index items
|
63
|
-
{
|
64
|
-
source = items
|
65
|
-
path = test/var/sphinx/items
|
66
|
-
}
|
67
|
-
|
68
|
-
index items_delta : items
|
69
|
-
{
|
70
|
-
source = items_delta
|
71
|
-
path = test/var/sphinx/items_delta
|
72
|
-
}
|
73
|
-
|
74
|
-
index vanillas
|
75
|
-
{
|
76
|
-
type = distributed
|
77
|
-
local = items
|
78
|
-
}
|
@@ -1,48 +0,0 @@
|
|
1
|
-
require 'test_adapter'
|
2
|
-
require 'files/resource_explicit'
|
3
|
-
|
4
|
-
class TestAdapterExplicit < TestAdapter
|
5
|
-
def setup
|
6
|
-
super
|
7
|
-
DataMapper.setup(:search, :adapter => 'sphinx', :config => @config, :managed => true)
|
8
|
-
end
|
9
|
-
|
10
|
-
def teardown
|
11
|
-
DataMapper.repository(:search).adapter.client.stop
|
12
|
-
super
|
13
|
-
end
|
14
|
-
|
15
|
-
def test_initialize
|
16
|
-
assert_nothing_raised{ Explicit.new }
|
17
|
-
end
|
18
|
-
|
19
|
-
def test_search_properties
|
20
|
-
assert_equal Explicit.all, Explicit.search
|
21
|
-
assert_equal [Explicit.first(:id => 2)], Explicit.search(:name => 'two')
|
22
|
-
end
|
23
|
-
|
24
|
-
def test_search_delta
|
25
|
-
resource = Explicit.create(:name => 'four', :likes => 'chicken', :updated_on => Time.now)
|
26
|
-
DataMapper.repository(:search).adapter.client.index('items_delta')
|
27
|
-
assert_equal [resource], Explicit.search(:name => 'four')
|
28
|
-
end
|
29
|
-
|
30
|
-
def test_search_attribute_timestamp
|
31
|
-
time = Time.now
|
32
|
-
resource = Explicit.create(:name => 'four', :likes => 'chicken', :updated_on => time)
|
33
|
-
DataMapper.repository(:search).adapter.client.index('items_delta')
|
34
|
-
|
35
|
-
assert_equal [resource], Explicit.search(:updated_on => time.to_i)
|
36
|
-
assert_equal [resource], Explicit.search(:updated_on => (time .. time + 1))
|
37
|
-
assert_equal [], Explicit.search(:updated_on => (time - 60 * 60))
|
38
|
-
assert_equal [], Explicit.search(:updated_on => (time + 60 * 60))
|
39
|
-
end
|
40
|
-
|
41
|
-
def test_search_attribute_boolean
|
42
|
-
# TODO:
|
43
|
-
end
|
44
|
-
|
45
|
-
def test_search_attribute_integer
|
46
|
-
# TODO
|
47
|
-
end
|
48
|
-
end # TestAdapterExplicit
|
@@ -1,25 +0,0 @@
|
|
1
|
-
require 'test_adapter'
|
2
|
-
require 'files/resource_resource'
|
3
|
-
|
4
|
-
class TestAdapterResource < TestAdapter
|
5
|
-
def setup
|
6
|
-
super
|
7
|
-
DataMapper.setup(:search, :adapter => 'sphinx', :config => @config, :managed => true)
|
8
|
-
end
|
9
|
-
|
10
|
-
def teardown
|
11
|
-
DataMapper.repository(:search).adapter.client.stop
|
12
|
-
super
|
13
|
-
end
|
14
|
-
|
15
|
-
def test_initialize
|
16
|
-
assert_nothing_raised{ Resource.new }
|
17
|
-
end
|
18
|
-
|
19
|
-
def test_search_properties
|
20
|
-
assert_equal Resource.all, Resource.search
|
21
|
-
assert_equal [Resource.first(:id => 2)], Resource.search(:name => 'two')
|
22
|
-
assert_equal [Resource.first(:id => 2)], Resource.search(:conditions => ['two'])
|
23
|
-
end
|
24
|
-
end # TestAdapterResource
|
25
|
-
|
@@ -1,23 +0,0 @@
|
|
1
|
-
require 'test_adapter'
|
2
|
-
require 'files/resource_searchable'
|
3
|
-
|
4
|
-
class TestAdapterSearchable < TestAdapter
|
5
|
-
def setup
|
6
|
-
super
|
7
|
-
DataMapper.setup(:search, :adapter => 'sphinx', :config => @config, :managed => true)
|
8
|
-
end
|
9
|
-
|
10
|
-
def teardown
|
11
|
-
DataMapper.repository(:search).adapter.client.stop
|
12
|
-
super
|
13
|
-
end
|
14
|
-
|
15
|
-
def test_initialize
|
16
|
-
assert_nothing_raised{ Searchable.new }
|
17
|
-
end
|
18
|
-
|
19
|
-
def test_search
|
20
|
-
assert_equal Searchable.all, Searchable.search
|
21
|
-
assert_equal [Searchable.first(:id => 2)], Searchable.search(:name => 'two')
|
22
|
-
end
|
23
|
-
end # TestAdapterSearchable
|
@@ -1,46 +0,0 @@
|
|
1
|
-
require 'test_adapter'
|
2
|
-
require 'files/resource_vanilla'
|
3
|
-
require 'files/resource_storage_name'
|
4
|
-
|
5
|
-
class TestAdapterVanilla < TestAdapter
|
6
|
-
def setup
|
7
|
-
super
|
8
|
-
DataMapper.setup(:default, :adapter => 'sphinx', :config => @config, :managed => true)
|
9
|
-
end
|
10
|
-
|
11
|
-
def teardown
|
12
|
-
DataMapper.repository(:default).adapter.client.stop
|
13
|
-
super
|
14
|
-
end
|
15
|
-
|
16
|
-
def test_initialize
|
17
|
-
assert_nothing_raised{ Vanilla.new }
|
18
|
-
assert_nothing_raised{ StorageName.new }
|
19
|
-
end
|
20
|
-
|
21
|
-
def test_all
|
22
|
-
assert_equal [{:id => 1}, {:id => 2}, {:id => 3}], Vanilla.all
|
23
|
-
assert_equal [{:id => 1}], Vanilla.all(:name => 'one')
|
24
|
-
end
|
25
|
-
|
26
|
-
def test_all_limit
|
27
|
-
assert_equal [{:id => 1}], Vanilla.all(:limit => 1)
|
28
|
-
assert_equal [{:id => 1}, {:id => 2}], Vanilla.all(:limit => 2)
|
29
|
-
end
|
30
|
-
|
31
|
-
def test_all_offset
|
32
|
-
assert_equal [{:id => 1}, {:id => 2}, {:id => 3}], Vanilla.all(:offset => 0)
|
33
|
-
assert_equal [{:id => 2}, {:id => 3}], Vanilla.all(:offset => 1)
|
34
|
-
assert_equal [], Vanilla.all(:offset => 3)
|
35
|
-
end
|
36
|
-
|
37
|
-
def test_first
|
38
|
-
assert_equal({:id => 1}, Vanilla.first(:name => 'one'))
|
39
|
-
assert_nil Vanilla.first(:name => 'missing')
|
40
|
-
end
|
41
|
-
|
42
|
-
def test_storage_name
|
43
|
-
assert_equal Vanilla.all, StorageName.all
|
44
|
-
assert_equal Vanilla.first, StorageName.first
|
45
|
-
end
|
46
|
-
end
|
data/test/test_client.rb
DELETED
@@ -1,31 +0,0 @@
|
|
1
|
-
require 'test_adapter'
|
2
|
-
|
3
|
-
class TestClient < TestAdapter
|
4
|
-
def test_initialize
|
5
|
-
assert_nothing_raised { DataMapper::Adapters::Sphinx::Client.new(@config) }
|
6
|
-
end
|
7
|
-
|
8
|
-
def test_index
|
9
|
-
client = DataMapper::Adapters::Sphinx::Client.new(@config)
|
10
|
-
assert_nothing_raised{ client.index }
|
11
|
-
assert_nothing_raised{ client.index 'items' }
|
12
|
-
assert_nothing_raised{ client.index '*' }
|
13
|
-
assert_nothing_raised{ client.index ['items', 'items_delta'] }
|
14
|
-
end
|
15
|
-
|
16
|
-
def test_managed_initialize
|
17
|
-
assert_nothing_raised { DataMapper::Adapters::Sphinx::ManagedClient.new(@config) }
|
18
|
-
end
|
19
|
-
|
20
|
-
def test_search
|
21
|
-
begin
|
22
|
-
client = DataMapper::Adapters::Sphinx::ManagedClient.new(@config)
|
23
|
-
client.index
|
24
|
-
assert match = client.search('two')
|
25
|
-
assert_equal 1, match[:total]
|
26
|
-
assert_equal 2, match[:matches][0][:doc]
|
27
|
-
ensure
|
28
|
-
client.stop
|
29
|
-
end
|
30
|
-
end
|
31
|
-
end # TestClient
|
data/test/test_config.rb
DELETED
@@ -1,75 +0,0 @@
|
|
1
|
-
require 'test/unit'
|
2
|
-
require 'dm-sphinx-adapter'
|
3
|
-
|
4
|
-
class TestConfig < Test::Unit::TestCase
|
5
|
-
def setup
|
6
|
-
base = Pathname(__FILE__).dirname.expand_path
|
7
|
-
@config = base / 'files' / 'sphinx.conf'
|
8
|
-
@log = base / 'var' / 'sphinx.log'
|
9
|
-
end
|
10
|
-
|
11
|
-
def test_initialize
|
12
|
-
assert_nothing_raised{ config }
|
13
|
-
assert_nothing_raised{ config({}) }
|
14
|
-
assert_nothing_raised{ config(:config => nil) }
|
15
|
-
assert_nothing_raised{ config(:config => 'blah') }
|
16
|
-
assert_nothing_raised{ config(:config => @log) }
|
17
|
-
end
|
18
|
-
|
19
|
-
def test_initalize_defaults
|
20
|
-
assert c = config({})
|
21
|
-
assert_equal '0.0.0.0', c.address
|
22
|
-
assert_equal 3312, c.port
|
23
|
-
assert_equal 'searchd.log', c.log
|
24
|
-
assert_raise(RuntimeError){ c.pid_file }
|
25
|
-
assert_nil c.config
|
26
|
-
end
|
27
|
-
|
28
|
-
def test_initalize_options_hash
|
29
|
-
assert c = config(
|
30
|
-
:host => 'options',
|
31
|
-
:port => 1234,
|
32
|
-
:log => 'log.log',
|
33
|
-
:pid_file => 'pid.pid'
|
34
|
-
)
|
35
|
-
|
36
|
-
assert_equal 'options', c.address
|
37
|
-
assert_equal 1234, c.port
|
38
|
-
assert_equal 'log.log', c.log
|
39
|
-
assert_equal 'pid.pid', c.pid_file
|
40
|
-
assert_nil c.config
|
41
|
-
end
|
42
|
-
|
43
|
-
def test_initialize_options_string
|
44
|
-
assert c = config("sphinx://options:1234")
|
45
|
-
assert_equal 'options', c.address
|
46
|
-
assert_equal 1234, c.port
|
47
|
-
assert_equal 'searchd.log', c.log
|
48
|
-
assert_raise(RuntimeError){ c.pid_file }
|
49
|
-
assert_nil c.config
|
50
|
-
end
|
51
|
-
|
52
|
-
def test_initialize_config
|
53
|
-
assert c = config(:config => @config)
|
54
|
-
assert_equal 'localhost', c.address
|
55
|
-
assert_equal '3312', c.port
|
56
|
-
assert_equal 'test/var/sphinx.log', c.log
|
57
|
-
assert_equal 'test/var/sphinx.pid', c.pid_file
|
58
|
-
assert_equal @config, c.config
|
59
|
-
end
|
60
|
-
|
61
|
-
def test_initialize_database_hash
|
62
|
-
assert c = config(:database => @config)
|
63
|
-
assert_equal @config, c.config
|
64
|
-
end
|
65
|
-
|
66
|
-
def test_initialize_database_string
|
67
|
-
assert c = config("sphinx://localhost/test/files/sphinx.conf")
|
68
|
-
assert_equal @config, c.config
|
69
|
-
end
|
70
|
-
|
71
|
-
protected
|
72
|
-
def config(options = {:config => @config})
|
73
|
-
DataMapper::Adapters::Sphinx::Config.new(options)
|
74
|
-
end
|
75
|
-
end # TestConfig
|