dm-sphinx-adapter 0.5 → 0.6

data/lib/riddle/client/filter.rb

@@ -0,0 +1,53 @@
+module Riddle
+  class Client
+    # Used for querying Sphinx.
+    class Filter
+      attr_accessor :attribute, :values, :exclude
+      
+      # Attribute name, values (which can be an array or a range), and whether
+      # the filter should be exclusive.
+      def initialize(attribute, values, exclude=false)
+        @attribute, @values, @exclude = attribute, values, exclude
+      end
+      
+      def exclude?
+        self.exclude
+      end
+      
+      # Returns the message for this filter to send to the Sphinx service
+      def query_message
+        message = Message.new
+        
+        message.append_string self.attribute.to_s
+        case self.values
+        when Range
+          if self.values.first.is_a?(Float) && self.values.last.is_a?(Float)
+            message.append_int FilterTypes[:float_range]
+            message.append_floats self.values.first, self.values.last
+          else
+            message.append_int FilterTypes[:range]
+            message.append_ints self.values.first, self.values.last
+          end
+        when Array
+          message.append_int FilterTypes[:values]
+          message.append_int self.values.length
+          # using to_f is a hack from the php client - to workaround 32bit
+          # signed ints on x32 platforms
+          message.append_ints *self.values.collect { |val|
+            case val
+            when TrueClass
+              1.0
+            when FalseClass
+              0.0
+            else
+              val.to_f
+            end
+          }
+        end
+        message.append_int self.exclude? ? 1 : 0
+        
+        message.to_s
+      end
+    end
+  end
+end

data/lib/riddle/client/message.rb

@@ -0,0 +1,65 @@
+module Riddle
+  class Client
+    # This class takes care of the translation of ints, strings and arrays to
+    # the format required by the Sphinx service.
+    class Message
+      def initialize
+        @message = ""
+        @size_method = @message.respond_to?(:bytesize) ? :bytesize : :length
+      end
+      
+      # Append raw data (only use if you know what you're doing)
+      def append(*args)
+        return if args.length == 0
+        
+        args.each { |arg| @message << arg }
+      end
+      
+      # Append a string's length, then the string itself
+      def append_string(str)
+        @message << [str.send(@size_method)].pack('N') + str
+      end
+      
+      # Append an integer
+      def append_int(int)
+        @message << [int].pack('N')
+      end
+      
+      def append_64bit_int(int)
+        @message << [int >> 32, int & 0xFFFFFFFF].pack('NN')
+      end
+      
+      # Append a float
+      def append_float(float)
+        @message << [float].pack('f').unpack('L*').pack("N")
+      end
+      
+      # Append multiple integers
+      def append_ints(*ints)
+        ints.each { |int| append_int(int) }
+      end
+      
+      def append_64bit_ints(*ints)
+        ints.each { |int| append_64bit_int(int) }
+      end
+      
+      # Append multiple floats
+      def append_floats(*floats)
+        floats.each { |float| append_float(float) }
+      end
+      
+      # Append an array of strings - first appends the length of the array,
+      # then each item's length and value.
+      def append_array(array)
+        append_int(array.length)
+        
+        array.each { |item| append_string(item) }
+      end
+      
+      # Returns the entire message
+      def to_s
+        @message
+      end
+    end
+  end
+end

data/lib/riddle/client/response.rb

@@ -0,0 +1,84 @@
+module Riddle
+  class Client
+    # Used to interrogate responses from the Sphinx daemon. Keep in mind none
+    # of the methods here check whether the data they're grabbing are what the
+    # user expects - it just assumes the user knows what the data stream is
+    # made up of.
+    class Response
+      # Create with the data to interpret
+      def initialize(str)
+        @str = str
+        @marker = 0
+      end
+      
+      # Return the next string value in the stream
+      def next
+        len = next_int
+        result = @str[@marker, len]
+        @marker += len
+        
+        return result
+      end
+      
+      # Return the next integer value from the stream
+      def next_int
+        int = @str[@marker, 4].unpack('N*').first
+        @marker += 4
+        
+        return int
+      end
+      
+      def next_64bit_int
+        high, low = @str[@marker, 8].unpack('N*N*')[0..1]
+        @marker += 8
+        
+        return (high << 32) + low
+      end
+      
+      # Return the next float value from the stream
+      def next_float
+        float = @str[@marker, 4].unpack('N*').pack('L').unpack('f*').first
+        @marker += 4
+        
+        return float
+      end
+      
+      # Returns an array of string items
+      def next_array
+        count = next_int
+        items = []
+        for i in 0...count
+          items << self.next
+        end
+        
+        return items
+      end
+      
+      # Returns an array of int items
+      def next_int_array
+        count = next_int
+        items = []
+        for i in 0...count
+          items << self.next_int
+        end
+        
+        return items
+      end
+      
+      def next_float_array
+        count = next_int
+        items = []
+        for i in 0...count
+          items << self.next_float
+        end
+        
+        return items
+      end
+      
+      # Returns the length of the streamed data
+      def length
+        @str.length
+      end
+    end
+  end
+end

data/test/files/model.rb

@@ -0,0 +1,23 @@
+class Item
+  include DataMapper::SphinxResource
+  property :id,         Serial
+  property :t_string,   String
+  property :t_text,     Text, :lazy => false
+  property :t_decimal,  BigDecimal
+  property :t_float,    Float
+  property :t_integer,  Integer
+  property :t_datetime, DateTime
+
+  repository(:search) do
+    properties(:search).clear
+    property :id,         Serial
+    property :t_string,   String
+
+    attribute :t_text,     Text, :lazy => false
+    attribute :t_decimal,  BigDecimal
+    attribute :t_float,    Float
+    attribute :t_integer,  Integer
+    attribute :t_datetime, DateTime
+  end
+end # Item
+

data/test/files/mysql5.sphinx.conf

@@ -0,0 +1,97 @@
+# searchd and indexer must be run from the root directory of this lib.
+
+indexer
+{
+  mem_limit = 64M
+}
+
+searchd
+{
+  address      = localhost
+  port         = 3312
+  log          = test/files/tmp/sphinx.log
+  query_log    = test/files/tmp/sphinx.query.log
+  read_timeout = 5
+  pid_file     = test/files/tmp/sphinx.pid
+  max_matches  = 1000
+}
+
+source items
+{
+  type     = mysql
+  sql_host = localhost
+  sql_user = root
+  sql_pass =
+  sql_db   = dm_sphinx_adapter_test
+
+  sql_query_pre = set names utf8
+  sql_query_pre = \
+    replace into delta (name, updated_on) ( \
+      select 'items', t_datetime \
+      from items \
+      order by t_datetime desc \
+      limit 1\
+    )
+  sql_query_info = select * from items where id = $id
+
+  sql_query_pre = set names utf8
+  sql_query = \
+    select \
+      id, \
+      t_string, \
+      t_text, \
+      t_decimal, \
+      t_float, \
+      t_integer, \
+      unix_timestamp(t_datetime) as t_datetime \
+    from items \
+    where t_datetime <= ( \
+      select updated_on \
+      from delta \
+      where name = 'items' \
+    )
+
+  sql_attr_float     = t_decimal
+  sql_attr_float     = t_float
+  sql_attr_uint      = t_integer
+  sql_attr_timestamp = t_datetime
+}
+
+source items_delta : items {
+  sql_query_pre = set names utf8
+  sql_query_pre =
+  sql_query     = \
+    select \
+      id, \
+      t_string, \
+      t_text, \
+      t_decimal, \
+      t_float, \
+      t_integer, \
+      unix_timestamp(t_datetime) as t_datetime \
+    from items \
+    where t_datetime > ( \
+      select updated_on \
+      from delta \
+      where name = 'items' \
+    )
+}
+
+index items_main
+{
+  source = items
+  path   = test/files/tmp/items_main
+}
+
+index items_delta : items_main
+{
+  source = items_delta
+  path   = test/files/tmp/items_delta
+}
+
+index items
+{
+  type  = distributed
+  local = items_main
+  local = items_delta
+}

data/test/files/mysql5.sql

@@ -0,0 +1,26 @@
+drop table if exists delta;
+create table delta (
+  name varchar(50) not null,
+  updated_on datetime,
+  primary key (name)
+) engine=innodb default charset=utf8;
+
+insert into delta (name, updated_on) values
+  ('items', now());
+
+drop table if exists items;
+create table items (
+  id int(11) not null auto_increment,
+  t_string   varchar(50),
+  t_text     text,
+  t_decimal  decimal(30,10),
+  t_float    float,
+  t_integer  int,
+  t_datetime datetime,
+  primary key (id)
+) engine=innodb default charset=utf8;
+
+insert into items (t_string, t_text, t_decimal, t_float, t_integer, t_datetime) values
+  ('one',   'text one!',   '10.50', '100.50', '1000', now()),
+  ('two',   'text two!',   '20.50', '200.50', '2000', now()),
+  ('three', 'text three!', '30.50', '300.50', '3000', now());

data/test/helper.rb

@@ -0,0 +1,51 @@
+$VERBOSE = false # Shitloads of warnings in dm :(
+require 'rubygems'
+require 'extlib'
+require 'extlib/hook'
+require 'pathname'
+require 'shoulda'
+require 'test/unit'
+
+base = Pathname.new(__FILE__).dirname + '..'
+%w{lib test}.each{|p| $:.unshift base + p}
+
+require 'dm-sphinx-adapter'
+
+# Sphinx runner.
+Dir.chdir(base)
+config = base + 'test' + 'files' + 'mysql5.sphinx.conf'
+begin
+  TCPSocket.new('localhost', '3312')
+rescue
+  puts 'Starting Sphinx...'
+  system("searchd --config #{config}") || exit
+  system('ps aux | grep searchd')
+end
+
+class Test::Unit::TestCase
+  include Extlib::Hook
+
+  before :setup do
+    files = Pathname.new(__FILE__).dirname + 'files'
+
+    mysql = `mysql5 dm_sphinx_adapter_test < #{files + 'mysql5.sql'} 2>&1`
+    raise %{Re-create database failed:\n #{mysql}} unless mysql.blank?
+
+    indexer = `indexer --config #{files + 'mysql5.sphinx.conf'} --all --rotate`
+    raise %{Re-create index failed:\n #{indexer}} if indexer =~ /error|fatal/i
+
+    DataMapper.setup(:default, :adapter => 'mysql', :database => 'dm_sphinx_adapter_test')
+    sleep 1; # Give sphinx a chance to catch up before test runs.
+  end
+
+  # after :teardown do
+  def teardown
+    descendants = DataMapper::Resource.descendants.dup.to_a
+    while model = descendants.shift
+      descendants.concat(model.descendants) if model.respond_to?(:descendants)
+      Object.send(:remove_const, model.name.to_sym)
+      DataMapper::Resource.descendants.delete(model)
+    end
+  end
+end
+

data/test/test_adapter.rb

@@ -1,38 +1,84 @@
-require 'dm-sphinx-adapter'
-require 'test/unit'
-
-# DataMapper::Logger.new(STDOUT, :debug)
+require File.join(File.dirname(__FILE__), 'helper')
 
 class TestAdapter < Test::Unit::TestCase
-  def setup
-    # TODO: A little too brutal even by my standards.
-    Dir.chdir(File.join(File.dirname(__FILE__), 'files')) do
-      system 'mysql -u root dm_sphinx_adapter_test < dm_sphinx_adapter_test.sql' \
-        or raise %q{Tests require the dm_sphinx_adapter_test database.}
+  context 'DM::A::Sphinx::Adapter class' do
+    setup do
+      DataMapper.setup(:adapter, :adapter => 'sphinx')
+      load File.join(File.dirname(__FILE__), 'files', 'model.rb')
+      @it       = repository(:adapter)
+      @resource = Item
     end
 
-    DataMapper.setup(:default, 'mysql://localhost/dm_sphinx_adapter_test')
+    context '#create' do
+      should 'should return zero records created' do
+        assert_equal 0, @it.create(create_resource)
+      end
+    end
 
-    @config = Pathname.new(__FILE__).dirname.expand_path / 'files' / 'sphinx.conf'
-    @client = DataMapper::Adapters::Sphinx::ManagedClient.new(:config => @config)
-    @client.index
-    sleep 1
-  end
+    context '#delete' do
+      should 'should return zero records deleted' do
+        assert_equal 0, @it.delete(create_resource)
+      end
+    end
 
-  def test_unmanaged_setup
-    assert DataMapper.setup(:sphinx, :adapter => 'sphinx')
-    assert_kind_of DataMapper::Adapters::SphinxAdapter, repository(:sphinx).adapter
-    assert_kind_of DataMapper::Adapters::Sphinx::Client, repository(:sphinx).adapter.client
-  end
+    context '#read_many' do
+      context 'conditions' do
+        should 'return all objects when nil' do
+          assert_equal [{:id => 1}, {:id => 2}, {:id => 3}], @it.read_many(query)
+        end
 
-  def test_managed_setup
-    assert DataMapper.setup(:sphinx, :adapter => 'sphinx', :config => @config, :managed => true)
-    assert_kind_of DataMapper::Adapters::SphinxAdapter, repository(:sphinx).adapter
-    assert_kind_of DataMapper::Adapters::Sphinx::ManagedClient, repository(:sphinx).adapter.client
-  end
+        should 'return subset of objects for conditions' do
+          assert_equal [{:id => 2}], @it.read_many(query(:t_string => 'two'))
+        end
+      end
+
+      context 'offsets' do
+        should 'be able to offset the objects' do
+          assert_equal [{:id => 1}, {:id => 2}, {:id => 3}], @it.read_many(query(:offset => 0))
+          assert_equal [{:id => 2}, {:id => 3}], @it.read_many(query(:offset => 1))
+          assert_equal [], @it.read_many(query(:offset => 3))
+        end
+      end
+
+      context 'limits' do
+        should 'be able to limit the objects' do
+          assert_equal [{:id => 1}], @it.read_many(query(:limit => 1))
+          assert_equal [{:id => 1}, {:id => 2}], @it.read_many(query(:limit => 2))
+        end
+      end
+    end
 
-  def teardown
-    @client.stop
-    sleep 1
+    context '#read_one' do
+      should 'return the first object of a #read_many' do
+        assert_equal @it.read_many(query).first, @it.read_one(query)
+
+        query = query(:t_string => 'two')
+        assert_equal @it.read_many(query).first, @it.read_one(query)
+      end
+    end
   end
+
+  protected
+    def query(conditions = {})
+      DataMapper::Query.new(repository(:adapter), @resource, conditions)
+    end
+
+    def resource(options = {})
+      now = Time.now
+      attributes = {
+        :t_string   => now.to_s,
+        :t_text     => "text #{now.to_s}!",
+        :t_decimal  => now.to_i * 0.001,
+        :t_float    => now.to_i * 0.0001,
+        :t_integer  => now.to_i,
+        :t_datetime => now
+      }.update(options)
+      @resource.new(attributes)
+    end
+
+    def create_resource(options = {})
+      repository(:adapter) do
+        @resource.create(resource(options).attributes.except(:id))
+      end
+    end
 end