sphinx 0.9.9.2117

data/lib/sphinx/constants.rb

@@ -0,0 +1,179 @@
+module Sphinx
+  # Contains all constants need by Sphinx API.
+  #
+  module Constants
+    #=================================================================
+    # Known searchd commands
+    #=================================================================
+
+    # search command
+    # @private
+    SEARCHD_COMMAND_SEARCH     = 0
+    # excerpt command
+    # @private
+    SEARCHD_COMMAND_EXCERPT    = 1
+    # update command
+    # @private
+    SEARCHD_COMMAND_UPDATE     = 2
+    # keywords command
+    # @private
+    SEARCHD_COMMAND_KEYWORDS   = 3
+    # persist command
+    # @private
+    SEARCHD_COMMAND_PERSIST    = 4
+    # status command
+    # @private
+    SEARCHD_COMMAND_STATUS     = 5
+    # query command
+    # @private
+    SEARCHD_COMMAND_QUERY      = 6
+
+    #=================================================================
+    # Current client-side command implementation versions
+    #=================================================================
+
+    # search command version
+    # @private
+    VER_COMMAND_SEARCH     = 0x116
+    # excerpt command version
+    # @private
+    VER_COMMAND_EXCERPT    = 0x100
+    # update command version
+    # @private
+    VER_COMMAND_UPDATE     = 0x102
+    # keywords command version
+    # @private
+    VER_COMMAND_KEYWORDS   = 0x100
+    # persist command version
+    # @private
+    VER_COMMAND_PERSIST    = 0x000
+    # status command version
+    # @private
+    VER_COMMAND_STATUS     = 0x100
+    # query command version
+    # @private
+    VER_COMMAND_QUERY      = 0x100
+
+    #=================================================================
+    # Known searchd status codes
+    #=================================================================
+
+    # general success, command-specific reply follows
+    # @private
+    SEARCHD_OK      = 0
+    # general failure, command-specific reply may follow
+    # @private
+    SEARCHD_ERROR   = 1
+    # temporaty failure, client should retry later
+    # @private
+    SEARCHD_RETRY   = 2
+    # general success, warning message and command-specific reply follow
+    # @private
+    SEARCHD_WARNING = 3
+
+    #=================================================================
+    # Known match modes
+    #=================================================================
+
+    # match all query words
+    SPH_MATCH_ALL       = 0
+    # match any query word
+    SPH_MATCH_ANY       = 1
+    # match this exact phrase
+    SPH_MATCH_PHRASE    = 2
+    # match this boolean query
+    SPH_MATCH_BOOLEAN   = 3
+    # match this extended query
+    SPH_MATCH_EXTENDED  = 4
+    # match all document IDs w/o fulltext query, apply filters
+    SPH_MATCH_FULLSCAN  = 5
+    # extended engine V2 (TEMPORARY, WILL BE REMOVED IN 0.9.8-RELEASE)
+    SPH_MATCH_EXTENDED2 = 6
+
+    #=================================================================
+    # Known ranking modes (ext2 only)
+    #=================================================================
+
+    # default mode, phrase proximity major factor and BM25 minor one
+    SPH_RANK_PROXIMITY_BM25 = 0
+    # statistical mode, BM25 ranking only (faster but worse quality)
+    SPH_RANK_BM25           = 1
+    # no ranking, all matches get a weight of 1
+    SPH_RANK_NONE           = 2
+    # simple word-count weighting, rank is a weighted sum of per-field keyword occurence counts
+    SPH_RANK_WORDCOUNT      = 3
+    # phrase proximity
+    SPH_RANK_PROXIMITY      = 4
+    # emulate old match-any weighting
+    SPH_RANK_MATCHANY       = 5
+    # sets bits where there were matches
+    SPH_RANK_FIELDMASK      = 6
+
+    #=================================================================
+    # Known sort modes
+    #=================================================================
+
+    # sort by document relevance desc, then by date
+    SPH_SORT_RELEVANCE     = 0
+    # sort by document date desc, then by relevance desc
+    SPH_SORT_ATTR_DESC     = 1
+    # sort by document date asc, then by relevance desc
+    SPH_SORT_ATTR_ASC      = 2
+    # sort by time segments (hour/day/week/etc) desc, then by relevance desc
+    SPH_SORT_TIME_SEGMENTS = 3
+    # sort by SQL-like expression (eg. "@relevance DESC, price ASC, @id DESC")
+    SPH_SORT_EXTENDED      = 4
+    # sort by arithmetic expression in descending order (eg. "@id + max(@weight,1000)*boost + log(price)")
+    SPH_SORT_EXPR          = 5
+
+    #=================================================================
+    # Known filter types
+    #=================================================================
+
+    # filter by integer values set
+    SPH_FILTER_VALUES      = 0
+    # filter by integer range
+    SPH_FILTER_RANGE       = 1
+    # filter by float range
+    SPH_FILTER_FLOATRANGE  = 2
+
+    #=================================================================
+    # Known attribute types
+    #=================================================================
+
+    # this attr is just an integer
+    SPH_ATTR_INTEGER   = 1
+    # this attr is a timestamp
+    SPH_ATTR_TIMESTAMP = 2
+    # this attr is an ordinal string number (integer at search time,
+    # specially handled at indexing time)
+    SPH_ATTR_ORDINAL   = 3
+    # this attr is a boolean bit field
+    SPH_ATTR_BOOL      = 4
+    # this attr is a float
+    SPH_ATTR_FLOAT     = 5
+    # signed 64-bit integer
+    SPH_ATTR_BIGINT    = 6
+    # this attr has multiple values (0 or more)
+    SPH_ATTR_MULTI     = 0x40000000
+
+    #=================================================================
+    # Known grouping functions
+    #=================================================================
+
+    # group by day
+    SPH_GROUPBY_DAY      = 0
+    # group by week
+    SPH_GROUPBY_WEEK     = 1
+    # group by month
+    SPH_GROUPBY_MONTH    = 2
+    # group by year
+    SPH_GROUPBY_YEAR     = 3
+    # group by attribute value
+    SPH_GROUPBY_ATTR     = 4
+    # group by sequential attrs pair
+    SPH_GROUPBY_ATTRPAIR = 5
+  end
+
+  include Constants
+end

data/lib/sphinx/indifferent_access.rb

@@ -0,0 +1,152 @@
+module Sphinx
+  begin
+    HashWithIndifferentAccess = ::HashWithIndifferentAccess
+  rescue NameError
+    # This class has dubious semantics and we only have it so that
+    # people can write params[:key] instead of params['key']
+    # and they get the same value for both keys.
+    #
+    # This is part of Rails' ActiveSupport project. If you are
+    # using Rails, this class will not be used.
+    #
+    class HashWithIndifferentAccess < Hash
+      def initialize(constructor = {})
+        if constructor.is_a?(Hash)
+          super()
+          update(constructor)
+        else
+          super(constructor)
+        end
+      end
+
+      def default(key = nil)
+        if key.is_a?(Symbol) && include?(key = key.to_s)
+          self[key]
+        else
+          super
+        end
+      end
+
+      alias_method :regular_writer, :[]= unless method_defined?(:regular_writer)
+      alias_method :regular_update, :update unless method_defined?(:regular_update)
+
+      # Assigns a new value to the hash:
+      #
+      #   hash = HashWithIndifferentAccess.new
+      #   hash[:key] = "value"
+      #
+      def []=(key, value)
+        regular_writer(convert_key(key), convert_value(value))
+      end
+
+      # Updates the instantized hash with values from the second:
+      # 
+      #   hash_1 = HashWithIndifferentAccess.new
+      #   hash_1[:key] = "value"
+      # 
+      #   hash_2 = HashWithIndifferentAccess.new
+      #   hash_2[:key] = "New Value!"
+      # 
+      #   hash_1.update(hash_2) # => {"key"=>"New Value!"}
+      # 
+      def update(other_hash)
+        other_hash.each_pair { |key, value| regular_writer(convert_key(key), convert_value(value)) }
+        self
+      end
+
+      alias_method :merge!, :update
+
+      # Checks the hash for a key matching the argument passed in:
+      #
+      #   hash = HashWithIndifferentAccess.new
+      #   hash["key"] = "value"
+      #   hash.key? :key  # => true
+      #   hash.key? "key" # => true
+      #
+      def key?(key)
+        super(convert_key(key))
+      end
+
+      alias_method :include?, :key?
+      alias_method :has_key?, :key?
+      alias_method :member?, :key?
+
+      # Fetches the value for the specified key, same as doing hash[key]
+      def fetch(key, *extras)
+        super(convert_key(key), *extras)
+      end
+
+      # Returns an array of the values at the specified indices:
+      #
+      #   hash = HashWithIndifferentAccess.new
+      #   hash[:a] = "x"
+      #   hash[:b] = "y"
+      #   hash.values_at("a", "b") # => ["x", "y"]
+      #
+      def values_at(*indices)
+        indices.collect {|key| self[convert_key(key)]}
+      end
+
+      # Returns an exact copy of the hash.
+      def dup
+        HashWithIndifferentAccess.new(self)
+      end
+
+      # Merges the instantized and the specified hashes together, giving precedence to the values from the second hash
+      # Does not overwrite the existing hash.
+      def merge(hash)
+        self.dup.update(hash)
+      end
+
+      # Performs the opposite of merge, with the keys and values from the first hash taking precedence over the second.
+      # This overloaded definition prevents returning a regular hash, if reverse_merge is called on a HashWithDifferentAccess.
+      def reverse_merge(other_hash)
+        super other_hash.with_indifferent_access
+      end
+
+      # Removes a specified key from the hash.
+      def delete(key)
+        super(convert_key(key))
+      end
+
+      def stringify_keys!; self end
+      def symbolize_keys!; self end
+      def to_options!; self end
+
+      # Convert to a Hash with String keys.
+      def to_hash
+        Hash.new(default).merge(self)
+      end
+
+      protected
+        def convert_key(key)
+          key.kind_of?(Symbol) ? key.to_s : key
+        end
+
+        def convert_value(value)
+          case value
+          when Hash
+            value.with_indifferent_access
+          when Array
+            value.collect { |e| e.is_a?(Hash) ? e.with_indifferent_access : e }
+          else
+            value
+          end
+        end
+    end
+
+    module HashIndifferentAccess #:nodoc:
+      def with_indifferent_access
+        hash = HashWithIndifferentAccess.new(self)
+        hash.default = self.default
+        hash
+      end
+    end
+
+    class ::Hash #:nodoc:
+      unless respond_to?(:with_indifferent_access)
+        include Sphinx::HashIndifferentAccess
+      end
+    end
+  end
+end

data/lib/sphinx/request.rb

@@ -0,0 +1,121 @@
+module Sphinx
+  # Pack ints, floats, strings, and arrays to internal representation
+  # needed by Sphinx search engine.
+  #
+  class Request
+    # Initialize new empty request.
+    #
+    def initialize
+      @request = ''
+    end
+
+    # Put int(s) to request.
+    #
+    # @param [Integer] ints one or more integers to put to the request.
+    # @return [Request] self.
+    #
+    # @example
+    #   request.put_int 10
+    #   request.put_int 10, 20, 30
+    #   request.put_int *[10, 20, 30]
+    #
+    def put_int(*ints)
+      ints.each { |i| @request << [i].pack('N') }
+      self
+    end
+
+    # Put 64-bit int(s) to request.
+    #
+    # @param [Integer] ints one or more 64-bit integers to put to the request.
+    # @return [Request] self.
+    #
+    # @example
+    #   request.put_int64 10
+    #   request.put_int64 10, 20, 30
+    #   request.put_int64 *[10, 20, 30]
+    #
+    def put_int64(*ints)
+      ints.each { |i| @request << [i].pack('q').reverse }#[i >> 32, i & ((1 << 32) - 1)].pack('NN') }
+      self
+    end
+
+    # Put strings to request.
+    #
+    # @param [String] strings one or more strings to put to the request.
+    # @return [Request] self.
+    #
+    # @example
+    #   request.put_string 'str1'
+    #   request.put_string 'str1', 'str2', 'str3'
+    #   request.put_string *['str1', 'str2', 'str3']
+    #
+    def put_string(*strings)
+      strings.each { |s| @request << [s.length].pack('N') + s }
+      self
+    end
+
+    # Put float(s) to request.
+    #
+    # @param [Integer, Float] floats one or more floats to put to the request.
+    # @return [Request] self.
+    #
+    # @example
+    #   request.put_float 10
+    #   request.put_float 10, 20, 30
+    #   request.put_float *[10, 20, 30]
+    #
+    def put_float(*floats)
+      floats.each do |f|
+        t1 = [f.to_f].pack('f') # machine order
+        t2 = t1.unpack('L*').first # int in machine order
+        @request << [t2].pack('N')
+      end
+      self
+    end
+
+    # Put array of ints to request.
+    #
+    # @param [Array<Integer>] arr an array of integers to put to the request.
+    # @return [Request] self.
+    #
+    # @example
+    #   request.put_int_array [10]
+    #   request.put_int_array [10, 20, 30]
+    #
+    def put_int_array(arr)
+      put_int arr.length, *arr
+      self
+    end
+
+    # Put array of 64-bit ints to request.
+    #
+    # @param [Array<Integer>] arr an array of integers to put to the request.
+    # @return [Request] self.
+    #
+    # @example
+    #   request.put_int64_array [10]
+    #   request.put_int64_array [10, 20, 30]
+    #
+    def put_int64_array(arr)
+      put_int(arr.length)
+      put_int64(*arr)
+      self
+    end
+
+    # Returns the serialized request.
+    #
+    # @return [String] serialized request.
+    #
+    def to_s
+      @request
+    end
+
+    # Returns CRC32 of the serialized request.
+    #
+    # @return [Integer] CRC32 of the serialized request.
+    #
+    def crc32
+      Zlib.crc32(@request)
+    end
+  end
+end

data/lib/sphinx/response.rb

@@ -0,0 +1,71 @@
+module Sphinx
+  # Unpack internal Sphinx representation of ints, floats, strings, and arrays.
+  # needed by Sphinx search engine.
+  #
+  # @private
+  class Response
+    # Initialize new request.
+    def initialize(response)
+      @response = response
+      @position = 0
+      @size = response.length
+    end
+    
+    # Gets current stream position.
+    def position
+      @position
+    end
+    
+    # Gets response size.
+    def size
+      @size
+    end
+    
+    # Returns <tt>true</tt> when response stream is out.
+    def eof?
+      @position >= @size
+    end
+
+    # Get int from stream.
+    def get_int
+      raise EOFError if @position + 4 > @size
+      value = @response[@position, 4].unpack('N*').first
+      @position += 4
+      return value
+    end
+
+    # Get 64-bit int from stream.
+    def get_int64
+      raise EOFError if @position + 8 > @size
+      hi, lo = @response[@position, 8].unpack('N*N*')
+      @position += 8
+      return (hi << 32) + lo
+    end
+
+    # Get array of <tt>count</tt> ints from stream.
+    def get_ints(count)
+      length = 4 * count
+      raise EOFError if @position + length > @size
+      values = @response[@position, length].unpack('N*' * count)
+      @position += length
+      return values
+    end
+    
+    # Get string from stream.
+    def get_string
+      length = get_int
+      raise EOFError if @position + length > @size
+      value = length > 0 ? @response[@position, length] : ''
+      @position += length
+      return value
+    end
+    
+    # Get float from stream.
+    def get_float
+      raise EOFError if @position + 4 > @size
+      uval = @response[@position, 4].unpack('N*').first;
+      @position += 4
+      return ([uval].pack('L')).unpack('f*').first
+    end
+  end
+end

data/lib/sphinx/server.rb

@@ -0,0 +1,170 @@
+# Represents an instance of searchd server.
+#
+# @private
+class Sphinx::Server
+  # The host the Sphinx server is running on
+  attr_reader :host
+
+  # The port the Sphinx server is listening on
+  attr_reader :port
+
+  # The path to UNIX socket where Sphinx server is running on
+  attr_reader :path
+
+  # Creates a new instance of +Server+.
+  #
+  # Parameters:
+  # * +sphinx+ -- an instance of <tt>Sphinx::Client</tt>.
+  # * +host+ -- name of host where search is running (if +path+ is not specified).
+  # * +port+ -- searchd port (if +path+ is not specified).
+  # * +path+ -- an absolute path to the UNIX socket.
+  #
+  def initialize(sphinx, host, port, path)
+    @sphinx = sphinx
+    @host = host
+    @port = port
+    @path = path
+
+    @socket = nil
+  end
+
+  # Gets the opened socket to the server.
+  #
+  # You can pass a block to make any connection establishing related things,
+  # like protocol version interchange. They will be treated as a part of
+  # connection routine, so connection timeout will include them.
+  #
+  # In case of connection error, +SphinxConnectError+ exception will be raised.
+  #
+  # Method returns opened socket, so do not forget to close it using +free_socket+
+  # method. Make sure you will close socket in case of any emergency.
+  #
+  def get_socket(&block)
+    if persistent?
+      yield @socket
+      @socket
+    else
+      socket = nil
+      Sphinx::safe_execute(@sphinx.timeout) do
+        socket = establish_connection
+
+        # Do custom initialization
+        yield socket if block_given?
+      end
+      socket
+    end
+  rescue SocketError, SystemCallError, IOError, EOFError, ::Timeout::Error, ::Errno::EPIPE => e
+    # Close previously opened socket (in case of it has been really opened)
+    free_socket(socket, true)
+
+    error = "connection to #{to_s} failed ("
+    if e.kind_of?(SystemCallError)
+      error << "errno=#{e.class::Errno}, "
+    end
+    error << "msg=#{e.message})"
+    raise Sphinx::SphinxConnectError, error
+  end
+
+  # Closes previously opened socket.
+  #
+  # Pass socket retrieved with +get_socket+ method when finished work. It does
+  # not close persistent sockets, but if really you need to do it, pass +true+
+  # as +force+ parameter value.
+  #
+  def free_socket(socket, force = false)
+    # Socket has not been open
+    return false if socket.nil?
+
+    # Do we try to close persistent socket?
+    if socket == @socket
+      # do not close it if not forced
+      if force
+        @socket.close unless @socket.closed?
+        @socket = nil
+        true
+      else
+        false
+      end
+    else
+      # Just close this socket
+      socket.close unless socket.closed?
+      true
+    end
+  end
+
+  # Makes specified socket persistent.
+  #
+  # Previous persistent socket will be closed as well.
+  def make_persistent!(socket)
+    unless socket == @socket
+      close_persistent!
+      @socket = socket
+    end
+    @socket
+  end
+
+  # Closes persistent socket.
+  def close_persistent!
+    free_socket(@socket, true)
+  end
+
+  # Gets a value indicating whether server has persistent socket associated.
+  def persistent?
+    !@socket.nil?
+  end
+
+  # Returns a string representation of the sphinx server object.
+  #
+  def to_s
+    @path || "#{@host}:#{@port}"
+  end
+
+  # Returns a string representation of the sphinx server object.
+  #
+  def inspect
+    "<Sphinx::Server: %s%s>" % [to_s, @socket ? '; persistent' : '']
+  end
+
+  private
+
+    # This is internal method which establishes a connection to a configured server.
+    #
+    # Method configures various socket options (like TCP_NODELAY), and
+    # sets socket timeouts.
+    #
+    # It does not close socket on any failure, please do it from calling code!
+    #
+    def establish_connection
+      if @path
+        sock = UNIXSocket.new(@path)
+      else
+        sock = TCPSocket.new(@host, @port)
+      end
+
+      io = Sphinx::BufferedIO.new(sock)
+      io.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
+      if @sphinx.reqtimeout > 0
+        io.read_timeout = @sphinx.reqtimeout
+
+        # This is a part of memcache-client library.
+        #
+        # Getting reports from several customers, including 37signals,
+        # that the non-blocking timeouts in 1.7.5 don't seem to be reliable.
+        # It can't hurt to set the underlying socket timeout also, if possible.
+        secs = Integer(@sphinx.reqtimeout)
+        usecs = Integer((@sphinx.reqtimeout - secs) * 1_000_000)
+        optval = [secs, usecs].pack("l_2")
+        begin
+          io.setsockopt Socket::SOL_SOCKET, Socket::SO_RCVTIMEO, optval
+          io.setsockopt Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, optval
+        rescue Exception => ex
+          # Solaris, for one, does not like/support socket timeouts.
+          @sphinx.logger.warn { "[sphinx] Unable to use raw socket timeouts: #{ex.class.name}: #{ex.message}" } if @sphinx.logger
+        end
+      else
+        io.read_timeout = false
+      end
+
+      io
+    end
+end