bdimcheff-dm-sphinx-adapter 0.8.0

data/lib/riddle/client/filter.rb

@@ -0,0 +1,53 @@
+module Riddle
+  class Client
+    # Used for querying Sphinx.
+    class Filter
+      attr_accessor :attribute, :values, :exclude
+      
+      # Attribute name, values (which can be an array or a range), and whether
+      # the filter should be exclusive.
+      def initialize(attribute, values, exclude=false)
+        @attribute, @values, @exclude = attribute, values, exclude
+      end
+      
+      def exclude?
+        self.exclude
+      end
+      
+      # Returns the message for this filter to send to the Sphinx service
+      def query_message
+        message = Message.new
+        
+        message.append_string self.attribute.to_s
+        case self.values
+        when Range
+          if self.values.first.is_a?(Float) && self.values.last.is_a?(Float)
+            message.append_int FilterTypes[:float_range]
+            message.append_floats self.values.first, self.values.last
+          else
+            message.append_int FilterTypes[:range]
+            message.append_ints self.values.first, self.values.last
+          end
+        when Array
+          message.append_int FilterTypes[:values]
+          message.append_int self.values.length
+          # using to_f is a hack from the php client - to workaround 32bit
+          # signed ints on x32 platforms
+          message.append_ints *self.values.collect { |val|
+            case val
+            when TrueClass
+              1.0
+            when FalseClass
+              0.0
+            else
+              val.to_f
+            end
+          }
+        end
+        message.append_int self.exclude? ? 1 : 0
+        
+        message.to_s
+      end
+    end
+  end
+end

data/lib/riddle/client/message.rb

@@ -0,0 +1,65 @@
+module Riddle
+  class Client
+    # This class takes care of the translation of ints, strings and arrays to
+    # the format required by the Sphinx service.
+    class Message
+      def initialize
+        @message = ""
+        @size_method = @message.respond_to?(:bytesize) ? :bytesize : :length
+      end
+      
+      # Append raw data (only use if you know what you're doing)
+      def append(*args)
+        return if args.length == 0
+        
+        args.each { |arg| @message << arg }
+      end
+      
+      # Append a string's length, then the string itself
+      def append_string(str)
+        @message << [str.send(@size_method)].pack('N') + str
+      end
+      
+      # Append an integer
+      def append_int(int)
+        @message << [int].pack('N')
+      end
+      
+      def append_64bit_int(int)
+        @message << [int >> 32, int & 0xFFFFFFFF].pack('NN')
+      end
+      
+      # Append a float
+      def append_float(float)
+        @message << [float].pack('f').unpack('L*').pack("N")
+      end
+      
+      # Append multiple integers
+      def append_ints(*ints)
+        ints.each { |int| append_int(int) }
+      end
+      
+      def append_64bit_ints(*ints)
+        ints.each { |int| append_64bit_int(int) }
+      end
+      
+      # Append multiple floats
+      def append_floats(*floats)
+        floats.each { |float| append_float(float) }
+      end
+      
+      # Append an array of strings - first appends the length of the array,
+      # then each item's length and value.
+      def append_array(array)
+        append_int(array.length)
+        
+        array.each { |item| append_string(item) }
+      end
+      
+      # Returns the entire message
+      def to_s
+        @message
+      end
+    end
+  end
+end

data/lib/riddle/client/response.rb

@@ -0,0 +1,84 @@
+module Riddle
+  class Client
+    # Used to interrogate responses from the Sphinx daemon. Keep in mind none
+    # of the methods here check whether the data they're grabbing are what the
+    # user expects - it just assumes the user knows what the data stream is
+    # made up of.
+    class Response
+      # Create with the data to interpret
+      def initialize(str)
+        @str = str
+        @marker = 0
+      end
+      
+      # Return the next string value in the stream
+      def next
+        len = next_int
+        result = @str[@marker, len]
+        @marker += len
+        
+        return result
+      end
+      
+      # Return the next integer value from the stream
+      def next_int
+        int = @str[@marker, 4].unpack('N*').first
+        @marker += 4
+        
+        return int
+      end
+      
+      def next_64bit_int
+        high, low = @str[@marker, 8].unpack('N*N*')[0..1]
+        @marker += 8
+        
+        return (high << 32) + low
+      end
+      
+      # Return the next float value from the stream
+      def next_float
+        float = @str[@marker, 4].unpack('N*').pack('L').unpack('f*').first
+        @marker += 4
+        
+        return float
+      end
+      
+      # Returns an array of string items
+      def next_array
+        count = next_int
+        items = []
+        for i in 0...count
+          items << self.next
+        end
+        
+        return items
+      end
+      
+      # Returns an array of int items
+      def next_int_array
+        count = next_int
+        items = []
+        for i in 0...count
+          items << self.next_int
+        end
+        
+        return items
+      end
+      
+      def next_float_array
+        count = next_int
+        items = []
+        for i in 0...count
+          items << self.next_float
+        end
+        
+        return items
+      end
+      
+      # Returns the length of the streamed data
+      def length
+        @str.length
+      end
+    end
+  end
+end

data/test/files/model.rb

@@ -0,0 +1,24 @@
+class Item
+  include DataMapper::Resource
+  include DataMapper::SphinxResource
+  property :id,         Serial
+  property :t_string,   String
+  property :t_text,     Text, :lazy => false
+  property :t_decimal,  BigDecimal
+  property :t_float,    Float
+  property :t_integer,  Integer
+  property :t_datetime, DateTime
+
+  repository(:search) do
+    properties(:search).clear
+    property :id,         Serial
+    property :t_string,   String
+
+    attribute :t_text,     Text, :lazy => false
+    attribute :t_decimal,  BigDecimal
+    attribute :t_float,    Float
+    attribute :t_integer,  Integer
+    attribute :t_datetime, DateTime
+  end
+end # Item
+

data/test/files/source.xml

@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="utf-8"?>
+<sphinx:docset xmlns:sphinx="sphinx">
+  <sphinx:schema>
+    <sphinx:field name="t_string"/>
+    <sphinx:field name="t_text"/>
+    <sphinx:attr name="t_decimal"  type="float"/>
+    <sphinx:attr name="t_float"    type="float"/>
+    <sphinx:attr name="t_integer"  type="int"/>
+    <sphinx:attr name="t_datetime" type="timestamp"/>
+  </sphinx:schema>
+
+  <sphinx:document id="1">
+    <t_string><![CDATA[one]]></t_string>
+    <t_text><![CDATA[text one!]]></t_text>
+    <t_decimal><![CDATA[10.5]]></t_decimal>
+    <t_float><![CDATA[100.5]]></t_float>
+    <t_integer><![CDATA[1000]]></t_integer>
+    <t_datetime><![CDATA[1235183682]]></t_datetime>
+  </sphinx:document>
+
+  <sphinx:document id="2">
+    <t_string><![CDATA[two]]></t_string>
+    <t_text><![CDATA[text two!]]></t_text>
+    <t_decimal><![CDATA[20.5]]></t_decimal>
+    <t_float><![CDATA[200.5]]></t_float>
+    <t_integer><![CDATA[2000]]></t_integer>
+    <t_datetime><![CDATA[1235183682]]></t_datetime>
+  </sphinx:document>
+
+  <sphinx:document id="3">
+    <t_string><![CDATA[three]]></t_string>
+    <t_text><![CDATA[text three!]]></t_text>
+    <t_decimal><![CDATA[30.5]]></t_decimal>
+    <t_float><![CDATA[300.5]]></t_float>
+    <t_integer><![CDATA[3000]]></t_integer>
+    <t_datetime><![CDATA[1235183682]]></t_datetime>
+  </sphinx:document>
+</sphinx:docset>
+

data/test/files/sphinx.conf

@@ -0,0 +1,36 @@
+# searchd and indexer must be run from the root directory of this lib.
+
+indexer
+{
+  mem_limit = 64M
+}
+
+searchd
+{
+  address      = localhost
+  port         = 3312
+  log          = test/files/tmp/sphinx.log
+  query_log    = test/files/tmp/sphinx.query.log
+  read_timeout = 5
+  pid_file     = test/files/tmp/sphinx.pid
+  max_matches  = 1000
+}
+
+source items
+{
+  type            = xmlpipe2
+  xmlpipe_command = cat test/files/source.xml
+}
+
+index items_main
+{
+  source       = items
+  charset_type = utf-8
+  path         = test/files/tmp/items
+}
+
+index items
+{
+  type  = distributed
+  local = items_main
+}

data/test/files/test_xmlpipe2.xml

@@ -0,0 +1 @@
+<?xml version="1.0" encoding="UTF-8"?><sphinx:docset xmlns:sphinx="sphinx"><sphinx:schema><sphinx:field name="id"/><sphinx:field name="t_string"/><sphinx:attr type="str2ordinal" name="t_text"/><sphinx:attr type="float" name="t_decimal"/><sphinx:attr type="float" name="t_float"/><sphinx:attr type="int" name="t_integer"/><sphinx:attr type="timestamp" name="t_datetime"/></sphinx:schema><sphinx:document id="1"><id><![CDATA[1]]></id><t_string><![CDATA[one]]></t_string><t_text><![CDATA[text one!]]></t_text><t_decimal><![CDATA[0.01]]></t_decimal><t_float><![CDATA[0.0001]]></t_float><t_integer><![CDATA[1]]></t_integer><t_datetime><![CDATA[1235914716]]></t_datetime></sphinx:document></sphinx:docset>

data/test/helper.rb

@@ -0,0 +1,46 @@
+$VERBOSE = false # Shitloads of warnings in dm :(
+require 'rubygems'
+require 'extlib'
+require 'extlib/hook'
+require 'pathname'
+require 'test/unit'
+require 'shoulda'
+
+base  = Pathname.new(__FILE__).dirname + '..'
+files = base + 'test' + 'files'
+%w{lib test}.each{|p| $:.unshift base + p}
+
+require 'dm-sphinx-adapter'
+
+# Sphinx runner.
+Dir.chdir(base)
+begin
+  TCPSocket.new('localhost', '3312')
+rescue
+  puts 'Starting Sphinx...'
+  system("searchd --config #{files + 'sphinx.conf'}") || exit
+  system('ps aux | grep searchd')
+end
+
+indexer = `indexer --config #{files + 'sphinx.conf'} --all --rotate`
+raise %{Re-create index failed:\n #{indexer}} if indexer =~ /error|fatal/i
+sleep 1
+
+# :default is unused at the moment.
+DataMapper.setup(:default, :adapter => 'in_memory', :database => 'dm_sphinx_adapter_test')
+DataMapper.setup(:search,  :adapter => 'sphinx')
+
+class Test::Unit::TestCase
+  include Extlib::Hook
+
+  # after :teardown do
+  def teardown
+    descendants = DataMapper::Resource.descendants.dup.to_a
+    while model = descendants.shift
+      descendants.concat(model.descendants) if model.respond_to?(:descendants)
+      Object.send(:remove_const, model.name.to_sym)
+      DataMapper::Resource.descendants.delete(model)
+    end
+  end
+end
+

data/test/test_adapter.rb

@@ -0,0 +1,68 @@
+require File.join(File.dirname(__FILE__), 'helper')
+
+class TestAdapter < Test::Unit::TestCase
+  context 'DM::A::Sphinx::Adapter' do
+    setup do
+      load File.join(File.dirname(__FILE__), 'files', 'model.rb')
+      @it       = repository(:search)
+      @resource = Item
+    end
+
+    context 'class' do
+      should 'use default field naming convention' do
+        assert_equal(
+          DataMapper::NamingConventions::Field::Underscored,
+          @it.adapter.field_naming_convention
+        )
+      end
+
+      should 'use default resource naming convention' do
+        assert_equal(
+          DataMapper::NamingConventions::Resource::UnderscoredAndPluralized,
+          @it.adapter.resource_naming_convention
+        )
+      end
+    end
+
+    context '#read_many' do
+      context 'conditions' do
+        should 'return all objects when nil' do
+          assert_equal [1, 2, 3], @it.read_many(query).map{|d| d[:id]}
+        end
+
+        should 'return subset of objects for conditions' do
+          assert_equal [2], @it.read_many(query(:t_string => 'two')).map{|d| d[:id]}
+        end
+      end
+
+      context 'offsets' do
+        should 'be able to offset the objects' do
+          assert_equal [1, 2, 3], @it.read_many(query(:offset => 0)).map{|d| d[:id]}
+          assert_equal [2, 3], @it.read_many(query(:offset => 1)).map{|d| d[:id]}
+          assert_equal [], @it.read_many(query(:offset => 3))
+        end
+      end
+
+      context 'limits' do
+        should 'be able to limit the objects' do
+          assert_equal [1], @it.read_many(query(:limit => 1)).map{|d| d[:id]}
+          assert_equal [1, 2], @it.read_many(query(:limit => 2)).map{|d| d[:id]}
+        end
+      end
+    end
+
+    context '#read_one' do
+      should 'return the first object of a #read_many' do
+        assert_equal @it.read_many(query).first, @it.read_one(query)
+
+        query = query(:t_string => 'two')
+        assert_equal @it.read_many(query).first, @it.read_one(query)
+      end
+    end
+  end
+
+  protected
+    def query(conditions = {})
+      DataMapper::Query.new(repository(:search), @resource, conditions)
+    end
+end

data/test/test_attribute.rb

@@ -0,0 +1,36 @@
+require File.join(File.dirname(__FILE__), 'helper')
+
+class TestAttribute < Test::Unit::TestCase
+  context 'DM::A::Sphinx::Attribute instance' do
+    should 'typecast DateTime to Integer'
+    should 'typecast Date to Integer'
+    should 'typecast Time to Integer'
+    should 'typecast BigDecimal to Float'
+  end
+
+  context 'DM::A::Sphinx::Resource#attribute class method' do
+    setup do
+      class ::Resource
+        include DataMapper::SphinxResource
+      end
+    end
+
+    DataMapper::Adapters::Sphinx::Attribute::TYPES.each do |type|
+      should "accept a #{type} type" do
+        assert_nothing_raised do
+          Resource.class_eval do
+            attribute :name, type
+          end
+        end
+      end
+    end
+
+    should 'raise ArgumentError for unsupported type' do
+      assert_raise(ArgumentError) do
+        Resource.class_eval do
+          attribute :name, Test::Unit::TestCase
+        end
+      end
+    end
+  end
+end