riddle 0.9.8.1231.0 → 0.9.8.1533.10

data/README.textile

@@ -0,0 +1,89 @@
+This client has been written to interface with "Sphinx":http://sphinxsearch.com/. It is written by
+"Pat Allan":http://freelancing-gods.com, and has been influenced by both Dmytro Shteflyuk's Ruby
+client and the original PHP client - credit where credit's due, after all.
+
+It does not follow the same syntax as those two, though (not much point writing this otherwise) -
+opting for a more Ruby-like structure.
+
+The easiest way to install is to grab the gem from GitHub:
+
+  sudo gem install freelancing-god-riddle --source http://gems.github.com/
+
+However, if you're so inclined, you can grab sourcecode.
+
+  git clone git://github.com/freelancing-god/thinking-sphinx.git
+
+If you're looking for old versions - for older versions of Sphinx - you'll want to peruse the tagged versions in the old subversion repository
+
+  svn co http://rails-oceania.googlecode.com/svn/patallan/riddle/tags riddle-tags
+
+Please note that at the time of writing, the following versions are supported (if you get the appropriate tag):
+
+* 0.9.8-r871
+* 0.9.8-r909
+* 0.9.8-r985
+* 0.9.8-r1065
+* 0.9.8-r1112
+* 0.9.8-rc1 (gem version: 0.9.8.1198)
+* 0.9.8-rc2 (gem version: 0.9.8.1231)
+* 0.9.8 (gem version: 0.9.8.1371)
+
+To get started, just instantiate a Client object:
+
+  client = Riddle::Client.new # defaults to localhost and port 3312
+  client = Riddle::Client.new "sphinxserver.domain.tld", 3333 # custom settings
+  
+And then set the parameters to what you want, before running a query:
+
+  client.match_mode = :extended
+  client.query "Pat Allan @state Victoria"
+  
+The results from a query are similar to the other clients - but here's the details. It's a hash with
+the following keys:
+
+* :matches
+* :fields
+* :attributes
+* :attribute_names
+* :words
+* :total
+* :total_found
+* :time
+* :status
+* :warning (if appropriate)
+* :error (if appropriate)
+
+The key @:matches@ returns an array of hashes - the actual search results. Each hash has the
+document id (@:doc@), the result weighting (@:weight@), and a hash of the attributes for
+the document (@:attributes@).
+
+The @:fields@ and @:attribute_names@ keys return list of fields and attributes for the
+documents. The key @:attributes@ will return a hash of attribute name and type pairs, and
+@:words@ returns a hash of hashes representing the words from the search, with the number of
+documents and hits for each, along the lines of:
+
+  results[:words]["Pat"] #=> {:docs => 12, :hits => 15}
+  
+@:total@, @:total_found@ and @:time@ return the number of matches available, the
+total number of matches (which may be greater than the maximum available), and the time in milliseconds
+that the query took to run.
+
+@:status@ is the error code for the query - and if there was a related warning, it will be under
+the @:warning@ key. Fatal errors will be described under @:error@.
+
+If you've installed the gem and wondering why there's no tests - check out the svn version. I've kept the specs out of the gem as I have a decent amount of test data in there, which really isn't needed unless you want to submit patches.
+
+h2. Contributors
+
+Thanks to the following people who have contributed to Riddle in some shape or form:
+
+* Andrew Aksyonoff
+* Brad Greenlee
+* Lachie Cox
+* Jeremy Seitz
+* Mark Lane
+* Xavier Noria
+* Henrik Nye
+* Kristopher Chambers
+* Rob Anderton
+* Dylan Egan

data/lib/riddle.rb

@@ -1,8 +1,9 @@
 require 'socket'
+require 'timeout'
+
 require 'riddle/client'
-require 'riddle/client/filter'
-require 'riddle/client/message'
-require 'riddle/client/response'
+require 'riddle/configuration'
+require 'riddle/controller'
 
 module Riddle #:nodoc:
   class ConnectionError < StandardError #:nodoc:
@@ -14,12 +15,16 @@ module Riddle #:nodoc:
     Tiny    = 8
     # Revision number for RubyForge's sake, taken from what Sphinx
     # outputs to the command line.
-    Rev     = 1231
+    Rev     = 1533
     # Release number to mark my own fixes, beyond feature parity with
     # Sphinx itself.
-    Release = 0
+    Release = 10
     
-    String      = [Major, Minor, Tiny].join('.') + "rc2"
+    String      = [Major, Minor, Tiny].join('.')
     GemVersion  = [Major, Minor, Tiny, Rev, Release].join('.')
   end
+  
+  def self.escape(string)
+    string.gsub(/[\(\)\|\-!@~"&\/]/) { |char| "\\#{char}" }
+  end
 end

data/lib/riddle/client.rb

@@ -1,3 +1,7 @@
+require 'riddle/client/filter'
+require 'riddle/client/message'
+require 'riddle/client/response'
+
 module Riddle
   class VersionError < StandardError;  end
   class ResponseError < StandardError; end
@@ -100,7 +104,7 @@ module Riddle
       :match_mode, :sort_mode, :sort_by, :weights, :id_range, :filters,
       :group_by, :group_function, :group_clause, :group_distinct, :cut_off,
       :retry_count, :retry_delay, :anchor, :index_weights, :rank_mode,
-      :max_query_time, :field_weights
+      :max_query_time, :field_weights, :timeout
     attr_reader :queue
     
     # Can instantiate with a specific server and port - otherwise it assumes
@@ -110,6 +114,13 @@ module Riddle
       @server = server || "localhost"
       @port   = port   || 3312
       
+      reset
+      
+      @queue = []
+    end
+    
+    # Reset attributes and settings to defaults.
+    def reset
       # defaults
       @offset         = 0
       @limit          = 20
@@ -134,8 +145,7 @@ module Riddle
       @max_query_time = 0
       # string keys are field names, integer values are weightings
       @field_weights  = {}
-      
-      @queue = []
+      @timeout        = 0
     end
     
     # Set the geo-anchor point - with the names of the attributes that contain
@@ -384,8 +394,28 @@ module Riddle
     # Connects to the Sphinx daemon, and yields a socket to use. The socket is
     # closed at the end of the block.
     def connect(&block)
-      socket = TCPSocket.new @server, @port
-      
+      socket = nil
+      if @timeout == 0
+        socket = initialise_connection
+      else
+        begin
+          Timeout.timeout(@timeout) { socket = initialise_connection }
+        rescue Timeout::Error
+          raise Riddle::ConnectionError,
+            "Connection to #{@server} on #{@port} timed out after #{@timeout} seconds"
+        end
+      end
+
+      begin
+        yield socket
+      ensure
+        socket.close
+      end
+    end
+    
+    def initialise_connection
+      socket = initialise_socket
+
       # Checking version
       version = socket.recv(4).unpack('N*').first
       if version < 1
@@ -396,11 +426,20 @@ module Riddle
       # Send version
       socket.send [1].pack('N'), 0
       
+      socket
+    end
+    
+    def initialise_socket
+      tries = 0
       begin
-        yield socket
-      ensure
-        socket.close
+        socket = TCPSocket.new @server, @port
+      rescue Errno::ECONNREFUSED => e
+        retry if (tries += 1) < 5
+        raise Riddle::ConnectionError,
+          "Connection to #{@server} on #{@port} failed. #{e.message}"
       end
+      
+      socket
     end
     
     # Send a collection of messages, for a command type (eg, search, excerpts,
@@ -411,6 +450,9 @@ module Riddle
       version  = 0
       length   = 0
       message  = Array(messages).join("")
+      if message.respond_to?(:force_encoding)
+        message = message.force_encoding('ASCII-8BIT')
+      end
           
       connect do |socket|
         case command
@@ -430,7 +472,7 @@ module Riddle
         header = socket.recv(8)
         status, version, length = header.unpack('n2N')
         
-        while response.length < length
+        while response.length < (length || 0)
           part = socket.recv(length - response.length)
           response << part if part
         end
@@ -507,7 +549,7 @@ module Riddle
       # Per Index Weights
       message.append_int @index_weights.length
       @index_weights.each do |key,val|
-        message.append_string key
+        message.append_string key.to_s
         message.append_int val
       end
       
@@ -517,7 +559,7 @@ module Riddle
       # Per Field Weights
       message.append_int @field_weights.length
       @field_weights.each do |key,val|
-        message.append_string key
+        message.append_string key.to_s
         message.append_int val
       end
       

data/lib/riddle/client/filter.rb

@@ -18,7 +18,7 @@ module Riddle
       def query_message
         message = Message.new
         
-        message.append_string self.attribute
+        message.append_string self.attribute.to_s
         case self.values
         when Range
           if self.values.first.is_a?(Float) && self.values.last.is_a?(Float)
@@ -33,7 +33,16 @@ module Riddle
           message.append_int self.values.length
           # using to_f is a hack from the php client - to workaround 32bit
           # signed ints on x32 platforms
-          message.append_ints *self.values.collect { |val| val.to_f }
+          message.append_ints *self.values.collect { |val|
+            case val
+            when TrueClass
+              1.0
+            when FalseClass
+              0.0
+            else
+              val.to_f
+            end
+          }
         end
         message.append_int self.exclude? ? 1 : 0
         

data/lib/riddle/client/message.rb

@@ -10,14 +10,15 @@ module Riddle
       
       # Append raw data (only use if you know what you're doing)
       def append(*args)
-        return if args.length == 0
-        
         args.each { |arg| @message << arg }
       end
       
       # Append a string's length, then the string itself
       def append_string(str)
-        @message << [str.send(@size_method)].pack('N') + str
+        string = str.respond_to?(:force_encoding) ?
+          str.dup.force_encoding('ASCII-8BIT') : str
+        
+        @message << [string.send(@size_method)].pack('N') + string
       end
       
       # Append an integer

data/lib/riddle/configuration.rb

@@ -0,0 +1,33 @@
+require 'riddle/configuration/section'
+
+require 'riddle/configuration/distributed_index'
+require 'riddle/configuration/index'
+require 'riddle/configuration/indexer'
+require 'riddle/configuration/remote_index'
+require 'riddle/configuration/searchd'
+require 'riddle/configuration/source'
+require 'riddle/configuration/sql_source'
+require 'riddle/configuration/xml_source'
+
+module Riddle
+  class Configuration
+    class ConfigurationError < StandardError #:nodoc:
+    end
+    
+    attr_reader :indexes, :searchd
+    attr_accessor :indexer
+    
+    def initialize
+      @indexer = Riddle::Configuration::Indexer.new
+      @searchd = Riddle::Configuration::Searchd.new
+      @indexes = []
+    end
+    
+    def render
+      (
+        [@indexer.render, @searchd.render] +
+        @indexes.collect { |index| index.render }
+      ).join("\n")
+    end
+  end
+end

data/lib/riddle/configuration/distributed_index.rb

@@ -0,0 +1,48 @@
+module Riddle
+  class Configuration
+    class DistributedIndex < Riddle::Configuration::Section
+      self.settings = [:type, :local, :agent, :agent_connect_timeout,
+        :agent_query_timeout]
+      
+      attr_accessor :name, :local_indexes, :remote_indexes,
+        :agent_connect_timeout, :agent_query_timeout
+      
+      def initialize(name)
+        @name           = name
+        @local_indexes  = []
+        @remote_indexes = []
+      end
+      
+      def type
+        "distributed"
+      end
+      
+      def local
+        self.local_indexes
+      end
+      
+      def agent
+        agents = remote_indexes.collect { |index| index.remote }.uniq
+        agents.collect { |agent|
+          agent + ":" + remote_indexes.select { |index|
+            index.remote == agent
+          }.collect { |index| index.name }.join(",")
+        }
+      end
+      
+      def render
+        raise ConfigurationError unless valid?
+        
+        (
+          ["index #{name}", "{"] +
+          settings_body +
+          ["}", ""]
+        ).join("\n")
+      end
+      
+      def valid?
+        @local_indexes.length > 0 || @remote_indexes.length > 0
+      end
+    end
+  end
+end

data/lib/riddle/configuration/index.rb

@@ -0,0 +1,142 @@
+module Riddle
+  class Configuration
+    class Index < Riddle::Configuration::Section
+      self.settings = [:source, :path, :docinfo, :mlock, :morphology,
+        :stopwords, :wordforms, :exceptions, :min_word_len, :charset_type,
+        :charset_table, :ignore_chars, :min_prefix_len, :min_infix_len,
+        :prefix_fields, :infix_fields, :enable_star, :ngram_len, :ngram_chars,
+        :phrase_boundary, :phrase_boundary_step, :html_strip,
+        :html_index_attrs, :html_remove_elements, :preopen]
+      
+      attr_accessor :name, :parent, :sources, :path, :docinfo, :mlock,
+        :morphologies, :stopword_files, :wordform_files, :exception_files,
+        :min_word_len, :charset_type, :charset_table, :ignore_characters,
+        :min_prefix_len, :min_infix_len, :prefix_field_names,
+        :infix_field_names, :enable_star, :ngram_len, :ngram_characters,
+        :phrase_boundaries, :phrase_boundary_step, :html_strip,
+        :html_index_attrs, :html_remove_element_tags, :preopen
+      
+      def initialize(name, *sources)
+        @name                     = name
+        @sources                  = sources
+        @morphologies             = []
+        @stopword_files           = []
+        @wordform_files           = []
+        @exception_files          = []
+        @ignore_characters        = []
+        @prefix_field_names       = []
+        @infix_field_names        = []
+        @ngram_characters         = []
+        @phrase_boundaries        = []
+        @html_remove_element_tags = []
+      end
+      
+      def source
+        @sources.collect { |s| s.name }
+      end
+      
+      def morphology
+        nil_join @morphologies, ", "
+      end
+      
+      def morphology=(morphology)
+        @morphologies = nil_split morphology, /,\s?/
+      end
+      
+      def stopwords
+        nil_join @stopword_files, " "
+      end
+      
+      def stopwords=(stopwords)
+        @stopword_files = nil_split stopwords, ' '
+      end
+      
+      def wordforms
+        nil_join @wordform_files, " "
+      end
+      
+      def wordforms=(wordforms)
+        @wordform_files = nil_split wordforms, ' '
+      end
+      
+      def exceptions
+        nil_join @exception_files, " "
+      end
+      
+      def exceptions=(exceptions)
+        @exception_files = nil_split exceptions, ' '
+      end
+      
+      def ignore_chars
+        nil_join @ignore_characters, ", "
+      end
+      
+      def ignore_chars=(ignore_chars)
+        @ignore_characters = nil_split ignore_chars, /,\s?/
+      end
+      
+      def prefix_fields
+        nil_join @prefix_field_names, ", "
+      end
+      
+      def infix_fields
+        nil_join @infix_field_names, ", "
+      end
+      
+      def ngram_chars
+        nil_join @ngram_characters, ", "
+      end
+      
+      def ngram_chars=(ngram_chars)
+        @ngram_characters = nil_split ngram_chars, /,\s?/
+      end
+      
+      def phrase_boundary
+        nil_join @phrase_boundaries, ", "
+      end
+      
+      def phrase_boundary=(phrase_boundary)
+        @phrase_boundaries = nil_split phrase_boundary, /,\s?/
+      end
+      
+      def html_remove_elements
+        nil_join @html_remove_element_tags, ", "
+      end
+      
+      def html_remove_elements=(html_remove_elements)
+        @html_remove_element_tags = nil_split html_remove_elements, /,\s?/
+      end
+      
+      def render
+        raise ConfigurationError, "#{@name} #{@sources.inspect} #{@path} #{@parent}" unless valid?
+        
+        inherited_name = "#{name}"
+        inherited_name << " : #{parent}" if parent
+        (
+          @sources.collect { |s| s.render } +
+          ["index #{inherited_name}", "{"] +
+          settings_body +
+          ["}", ""]
+        ).join("\n")
+      end
+      
+      def valid?
+        (!@name.nil?) && (!( @sources.length == 0 || @path.nil? ) || !@parent.nil?)
+      end
+      
+      private
+      
+      def nil_split(string, pattern)
+        (string || "").split(pattern)
+      end
+      
+      def nil_join(array, delimiter)
+        if array.length == 0
+          nil
+        else
+          array.join(delimiter)
+        end
+      end
+    end
+  end
+end