dm-sphinx-adapter 0.5 → 0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. data/History.txt +5 -0
  2. data/Manifest.txt +12 -19
  3. data/README.txt +20 -41
  4. data/Rakefile +2 -3
  5. data/dm-sphinx-adapter.gemspec +6 -9
  6. data/lib/dm-sphinx-adapter.rb +14 -11
  7. data/lib/dm-sphinx-adapter/adapter.rb +36 -62
  8. data/lib/dm-sphinx-adapter/attribute.rb +48 -3
  9. data/lib/riddle.rb +28 -0
  10. data/lib/riddle/client.rb +619 -0
  11. data/lib/riddle/client/filter.rb +53 -0
  12. data/lib/riddle/client/message.rb +65 -0
  13. data/lib/riddle/client/response.rb +84 -0
  14. data/test/files/model.rb +23 -0
  15. data/test/files/mysql5.sphinx.conf +97 -0
  16. data/test/files/mysql5.sql +26 -0
  17. data/test/helper.rb +51 -0
  18. data/test/test_adapter.rb +74 -28
  19. data/test/test_attribute.rb +36 -0
  20. data/test/test_index.rb +30 -0
  21. data/test/test_query.rb +47 -32
  22. data/test/test_resource.rb +17 -0
  23. metadata +18 -41
  24. data/lib/dm-sphinx-adapter/client.rb +0 -109
  25. data/lib/dm-sphinx-adapter/config.rb +0 -122
  26. data/lib/dm-sphinx-adapter/config_parser.rb +0 -71
  27. data/test/files/dm_sphinx_adapter_test.sql +0 -21
  28. data/test/files/resource_explicit.rb +0 -25
  29. data/test/files/resource_resource.rb +0 -19
  30. data/test/files/resource_searchable.rb +0 -16
  31. data/test/files/resource_storage_name.rb +0 -11
  32. data/test/files/resource_vanilla.rb +0 -7
  33. data/test/files/sphinx.conf +0 -78
  34. data/test/test_adapter_explicit.rb +0 -48
  35. data/test/test_adapter_resource.rb +0 -25
  36. data/test/test_adapter_searchable.rb +0 -23
  37. data/test/test_adapter_vanilla.rb +0 -46
  38. data/test/test_client.rb +0 -31
  39. data/test/test_config.rb +0 -75
  40. data/test/test_config_parser.rb +0 -29
  41. data/test/test_type_attribute.rb +0 -8
  42. data/test/test_type_index.rb +0 -8
@@ -0,0 +1,53 @@
1
+ module Riddle
2
+ class Client
3
+ # Used for querying Sphinx.
4
+ class Filter
5
+ attr_accessor :attribute, :values, :exclude
6
+
7
+ # Attribute name, values (which can be an array or a range), and whether
8
+ # the filter should be exclusive.
9
+ def initialize(attribute, values, exclude=false)
10
+ @attribute, @values, @exclude = attribute, values, exclude
11
+ end
12
+
13
+ def exclude?
14
+ self.exclude
15
+ end
16
+
17
+ # Returns the message for this filter to send to the Sphinx service
18
+ def query_message
19
+ message = Message.new
20
+
21
+ message.append_string self.attribute.to_s
22
+ case self.values
23
+ when Range
24
+ if self.values.first.is_a?(Float) && self.values.last.is_a?(Float)
25
+ message.append_int FilterTypes[:float_range]
26
+ message.append_floats self.values.first, self.values.last
27
+ else
28
+ message.append_int FilterTypes[:range]
29
+ message.append_ints self.values.first, self.values.last
30
+ end
31
+ when Array
32
+ message.append_int FilterTypes[:values]
33
+ message.append_int self.values.length
34
+ # using to_f is a hack from the php client - to workaround 32bit
35
+ # signed ints on x32 platforms
36
+ message.append_ints *self.values.collect { |val|
37
+ case val
38
+ when TrueClass
39
+ 1.0
40
+ when FalseClass
41
+ 0.0
42
+ else
43
+ val.to_f
44
+ end
45
+ }
46
+ end
47
+ message.append_int self.exclude? ? 1 : 0
48
+
49
+ message.to_s
50
+ end
51
+ end
52
+ end
53
+ end
@@ -0,0 +1,65 @@
1
+ module Riddle
2
+ class Client
3
+ # This class takes care of the translation of ints, strings and arrays to
4
+ # the format required by the Sphinx service.
5
+ class Message
6
+ def initialize
7
+ @message = ""
8
+ @size_method = @message.respond_to?(:bytesize) ? :bytesize : :length
9
+ end
10
+
11
+ # Append raw data (only use if you know what you're doing)
12
+ def append(*args)
13
+ return if args.length == 0
14
+
15
+ args.each { |arg| @message << arg }
16
+ end
17
+
18
+ # Append a string's length, then the string itself
19
+ def append_string(str)
20
+ @message << [str.send(@size_method)].pack('N') + str
21
+ end
22
+
23
+ # Append an integer
24
+ def append_int(int)
25
+ @message << [int].pack('N')
26
+ end
27
+
28
+ def append_64bit_int(int)
29
+ @message << [int >> 32, int & 0xFFFFFFFF].pack('NN')
30
+ end
31
+
32
+ # Append a float
33
+ def append_float(float)
34
+ @message << [float].pack('f').unpack('L*').pack("N")
35
+ end
36
+
37
+ # Append multiple integers
38
+ def append_ints(*ints)
39
+ ints.each { |int| append_int(int) }
40
+ end
41
+
42
+ def append_64bit_ints(*ints)
43
+ ints.each { |int| append_64bit_int(int) }
44
+ end
45
+
46
+ # Append multiple floats
47
+ def append_floats(*floats)
48
+ floats.each { |float| append_float(float) }
49
+ end
50
+
51
+ # Append an array of strings - first appends the length of the array,
52
+ # then each item's length and value.
53
+ def append_array(array)
54
+ append_int(array.length)
55
+
56
+ array.each { |item| append_string(item) }
57
+ end
58
+
59
+ # Returns the entire message
60
+ def to_s
61
+ @message
62
+ end
63
+ end
64
+ end
65
+ end
@@ -0,0 +1,84 @@
1
+ module Riddle
2
+ class Client
3
+ # Used to interrogate responses from the Sphinx daemon. Keep in mind none
4
+ # of the methods here check whether the data they're grabbing are what the
5
+ # user expects - it just assumes the user knows what the data stream is
6
+ # made up of.
7
+ class Response
8
+ # Create with the data to interpret
9
+ def initialize(str)
10
+ @str = str
11
+ @marker = 0
12
+ end
13
+
14
+ # Return the next string value in the stream
15
+ def next
16
+ len = next_int
17
+ result = @str[@marker, len]
18
+ @marker += len
19
+
20
+ return result
21
+ end
22
+
23
+ # Return the next integer value from the stream
24
+ def next_int
25
+ int = @str[@marker, 4].unpack('N*').first
26
+ @marker += 4
27
+
28
+ return int
29
+ end
30
+
31
+ def next_64bit_int
32
+ high, low = @str[@marker, 8].unpack('N*N*')[0..1]
33
+ @marker += 8
34
+
35
+ return (high << 32) + low
36
+ end
37
+
38
+ # Return the next float value from the stream
39
+ def next_float
40
+ float = @str[@marker, 4].unpack('N*').pack('L').unpack('f*').first
41
+ @marker += 4
42
+
43
+ return float
44
+ end
45
+
46
+ # Returns an array of string items
47
+ def next_array
48
+ count = next_int
49
+ items = []
50
+ for i in 0...count
51
+ items << self.next
52
+ end
53
+
54
+ return items
55
+ end
56
+
57
+ # Returns an array of int items
58
+ def next_int_array
59
+ count = next_int
60
+ items = []
61
+ for i in 0...count
62
+ items << self.next_int
63
+ end
64
+
65
+ return items
66
+ end
67
+
68
+ def next_float_array
69
+ count = next_int
70
+ items = []
71
+ for i in 0...count
72
+ items << self.next_float
73
+ end
74
+
75
+ return items
76
+ end
77
+
78
+ # Returns the length of the streamed data
79
+ def length
80
+ @str.length
81
+ end
82
+ end
83
+ end
84
+ end
@@ -0,0 +1,23 @@
1
+ class Item
2
+ include DataMapper::SphinxResource
3
+ property :id, Serial
4
+ property :t_string, String
5
+ property :t_text, Text, :lazy => false
6
+ property :t_decimal, BigDecimal
7
+ property :t_float, Float
8
+ property :t_integer, Integer
9
+ property :t_datetime, DateTime
10
+
11
+ repository(:search) do
12
+ properties(:search).clear
13
+ property :id, Serial
14
+ property :t_string, String
15
+
16
+ attribute :t_text, Text, :lazy => false
17
+ attribute :t_decimal, BigDecimal
18
+ attribute :t_float, Float
19
+ attribute :t_integer, Integer
20
+ attribute :t_datetime, DateTime
21
+ end
22
+ end # Item
23
+
@@ -0,0 +1,97 @@
1
+ # searchd and indexer must be run from the root directory of this lib.
2
+
3
+ indexer
4
+ {
5
+ mem_limit = 64M
6
+ }
7
+
8
+ searchd
9
+ {
10
+ address = localhost
11
+ port = 3312
12
+ log = test/files/tmp/sphinx.log
13
+ query_log = test/files/tmp/sphinx.query.log
14
+ read_timeout = 5
15
+ pid_file = test/files/tmp/sphinx.pid
16
+ max_matches = 1000
17
+ }
18
+
19
+ source items
20
+ {
21
+ type = mysql
22
+ sql_host = localhost
23
+ sql_user = root
24
+ sql_pass =
25
+ sql_db = dm_sphinx_adapter_test
26
+
27
+ sql_query_pre = set names utf8
28
+ sql_query_pre = \
29
+ replace into delta (name, updated_on) ( \
30
+ select 'items', t_datetime \
31
+ from items \
32
+ order by t_datetime desc \
33
+ limit 1\
34
+ )
35
+ sql_query_info = select * from items where id = $id
36
+
37
+ sql_query_pre = set names utf8
38
+ sql_query = \
39
+ select \
40
+ id, \
41
+ t_string, \
42
+ t_text, \
43
+ t_decimal, \
44
+ t_float, \
45
+ t_integer, \
46
+ unix_timestamp(t_datetime) as t_datetime \
47
+ from items \
48
+ where t_datetime <= ( \
49
+ select updated_on \
50
+ from delta \
51
+ where name = 'items' \
52
+ )
53
+
54
+ sql_attr_float = t_decimal
55
+ sql_attr_float = t_float
56
+ sql_attr_uint = t_integer
57
+ sql_attr_timestamp = t_datetime
58
+ }
59
+
60
+ source items_delta : items {
61
+ sql_query_pre = set names utf8
62
+ sql_query_pre =
63
+ sql_query = \
64
+ select \
65
+ id, \
66
+ t_string, \
67
+ t_text, \
68
+ t_decimal, \
69
+ t_float, \
70
+ t_integer, \
71
+ unix_timestamp(t_datetime) as t_datetime \
72
+ from items \
73
+ where t_datetime > ( \
74
+ select updated_on \
75
+ from delta \
76
+ where name = 'items' \
77
+ )
78
+ }
79
+
80
+ index items_main
81
+ {
82
+ source = items
83
+ path = test/files/tmp/items_main
84
+ }
85
+
86
+ index items_delta : items_main
87
+ {
88
+ source = items_delta
89
+ path = test/files/tmp/items_delta
90
+ }
91
+
92
+ index items
93
+ {
94
+ type = distributed
95
+ local = items_main
96
+ local = items_delta
97
+ }
@@ -0,0 +1,26 @@
1
+ drop table if exists delta;
2
+ create table delta (
3
+ name varchar(50) not null,
4
+ updated_on datetime,
5
+ primary key (name)
6
+ ) engine=innodb default charset=utf8;
7
+
8
+ insert into delta (name, updated_on) values
9
+ ('items', now());
10
+
11
+ drop table if exists items;
12
+ create table items (
13
+ id int(11) not null auto_increment,
14
+ t_string varchar(50),
15
+ t_text text,
16
+ t_decimal decimal(30,10),
17
+ t_float float,
18
+ t_integer int,
19
+ t_datetime datetime,
20
+ primary key (id)
21
+ ) engine=innodb default charset=utf8;
22
+
23
+ insert into items (t_string, t_text, t_decimal, t_float, t_integer, t_datetime) values
24
+ ('one', 'text one!', '10.50', '100.50', '1000', now()),
25
+ ('two', 'text two!', '20.50', '200.50', '2000', now()),
26
+ ('three', 'text three!', '30.50', '300.50', '3000', now());
data/test/helper.rb ADDED
@@ -0,0 +1,51 @@
1
+ $VERBOSE = false # Shitloads of warnings in dm :(
2
+ require 'rubygems'
3
+ require 'extlib'
4
+ require 'extlib/hook'
5
+ require 'pathname'
6
+ require 'shoulda'
7
+ require 'test/unit'
8
+
9
+ base = Pathname.new(__FILE__).dirname + '..'
10
+ %w{lib test}.each{|p| $:.unshift base + p}
11
+
12
+ require 'dm-sphinx-adapter'
13
+
14
+ # Sphinx runner.
15
+ Dir.chdir(base)
16
+ config = base + 'test' + 'files' + 'mysql5.sphinx.conf'
17
+ begin
18
+ TCPSocket.new('localhost', '3312')
19
+ rescue
20
+ puts 'Starting Sphinx...'
21
+ system("searchd --config #{config}") || exit
22
+ system('ps aux | grep searchd')
23
+ end
24
+
25
+ class Test::Unit::TestCase
26
+ include Extlib::Hook
27
+
28
+ before :setup do
29
+ files = Pathname.new(__FILE__).dirname + 'files'
30
+
31
+ mysql = `mysql5 dm_sphinx_adapter_test < #{files + 'mysql5.sql'} 2>&1`
32
+ raise %{Re-create database failed:\n #{mysql}} unless mysql.blank?
33
+
34
+ indexer = `indexer --config #{files + 'mysql5.sphinx.conf'} --all --rotate`
35
+ raise %{Re-create index failed:\n #{indexer}} if indexer =~ /error|fatal/i
36
+
37
+ DataMapper.setup(:default, :adapter => 'mysql', :database => 'dm_sphinx_adapter_test')
38
+ sleep 1; # Give sphinx a chance to catch up before test runs.
39
+ end
40
+
41
+ # after :teardown do
42
+ def teardown
43
+ descendants = DataMapper::Resource.descendants.dup.to_a
44
+ while model = descendants.shift
45
+ descendants.concat(model.descendants) if model.respond_to?(:descendants)
46
+ Object.send(:remove_const, model.name.to_sym)
47
+ DataMapper::Resource.descendants.delete(model)
48
+ end
49
+ end
50
+ end
51
+
data/test/test_adapter.rb CHANGED
@@ -1,38 +1,84 @@
1
- require 'dm-sphinx-adapter'
2
- require 'test/unit'
3
-
4
- # DataMapper::Logger.new(STDOUT, :debug)
1
+ require File.join(File.dirname(__FILE__), 'helper')
5
2
 
6
3
  class TestAdapter < Test::Unit::TestCase
7
- def setup
8
- # TODO: A little too brutal even by my standards.
9
- Dir.chdir(File.join(File.dirname(__FILE__), 'files')) do
10
- system 'mysql -u root dm_sphinx_adapter_test < dm_sphinx_adapter_test.sql' \
11
- or raise %q{Tests require the dm_sphinx_adapter_test database.}
4
+ context 'DM::A::Sphinx::Adapter class' do
5
+ setup do
6
+ DataMapper.setup(:adapter, :adapter => 'sphinx')
7
+ load File.join(File.dirname(__FILE__), 'files', 'model.rb')
8
+ @it = repository(:adapter)
9
+ @resource = Item
12
10
  end
13
11
 
14
- DataMapper.setup(:default, 'mysql://localhost/dm_sphinx_adapter_test')
12
+ context '#create' do
13
+ should 'should return zero records created' do
14
+ assert_equal 0, @it.create(create_resource)
15
+ end
16
+ end
15
17
 
16
- @config = Pathname.new(__FILE__).dirname.expand_path / 'files' / 'sphinx.conf'
17
- @client = DataMapper::Adapters::Sphinx::ManagedClient.new(:config => @config)
18
- @client.index
19
- sleep 1
20
- end
18
+ context '#delete' do
19
+ should 'should return zero records deleted' do
20
+ assert_equal 0, @it.delete(create_resource)
21
+ end
22
+ end
21
23
 
22
- def test_unmanaged_setup
23
- assert DataMapper.setup(:sphinx, :adapter => 'sphinx')
24
- assert_kind_of DataMapper::Adapters::SphinxAdapter, repository(:sphinx).adapter
25
- assert_kind_of DataMapper::Adapters::Sphinx::Client, repository(:sphinx).adapter.client
26
- end
24
+ context '#read_many' do
25
+ context 'conditions' do
26
+ should 'return all objects when nil' do
27
+ assert_equal [{:id => 1}, {:id => 2}, {:id => 3}], @it.read_many(query)
28
+ end
27
29
 
28
- def test_managed_setup
29
- assert DataMapper.setup(:sphinx, :adapter => 'sphinx', :config => @config, :managed => true)
30
- assert_kind_of DataMapper::Adapters::SphinxAdapter, repository(:sphinx).adapter
31
- assert_kind_of DataMapper::Adapters::Sphinx::ManagedClient, repository(:sphinx).adapter.client
32
- end
30
+ should 'return subset of objects for conditions' do
31
+ assert_equal [{:id => 2}], @it.read_many(query(:t_string => 'two'))
32
+ end
33
+ end
34
+
35
+ context 'offsets' do
36
+ should 'be able to offset the objects' do
37
+ assert_equal [{:id => 1}, {:id => 2}, {:id => 3}], @it.read_many(query(:offset => 0))
38
+ assert_equal [{:id => 2}, {:id => 3}], @it.read_many(query(:offset => 1))
39
+ assert_equal [], @it.read_many(query(:offset => 3))
40
+ end
41
+ end
42
+
43
+ context 'limits' do
44
+ should 'be able to limit the objects' do
45
+ assert_equal [{:id => 1}], @it.read_many(query(:limit => 1))
46
+ assert_equal [{:id => 1}, {:id => 2}], @it.read_many(query(:limit => 2))
47
+ end
48
+ end
49
+ end
33
50
 
34
- def teardown
35
- @client.stop
36
- sleep 1
51
+ context '#read_one' do
52
+ should 'return the first object of a #read_many' do
53
+ assert_equal @it.read_many(query).first, @it.read_one(query)
54
+
55
+ query = query(:t_string => 'two')
56
+ assert_equal @it.read_many(query).first, @it.read_one(query)
57
+ end
58
+ end
37
59
  end
60
+
61
+ protected
62
+ def query(conditions = {})
63
+ DataMapper::Query.new(repository(:adapter), @resource, conditions)
64
+ end
65
+
66
+ def resource(options = {})
67
+ now = Time.now
68
+ attributes = {
69
+ :t_string => now.to_s,
70
+ :t_text => "text #{now.to_s}!",
71
+ :t_decimal => now.to_i * 0.001,
72
+ :t_float => now.to_i * 0.0001,
73
+ :t_integer => now.to_i,
74
+ :t_datetime => now
75
+ }.update(options)
76
+ @resource.new(attributes)
77
+ end
78
+
79
+ def create_resource(options = {})
80
+ repository(:adapter) do
81
+ @resource.create(resource(options).attributes.except(:id))
82
+ end
83
+ end
38
84
  end