warp-thinking-sphinx 1.2.12

data/vendor/riddle/lib/riddle/client/filter.rb

@@ -0,0 +1,53 @@
+module Riddle
+  class Client
+    # Used for querying Sphinx.
+    class Filter
+      attr_accessor :attribute, :values, :exclude
+      
+      # Attribute name, values (which can be an array or a range), and whether
+      # the filter should be exclusive.
+      def initialize(attribute, values, exclude=false)
+        @attribute, @values, @exclude = attribute, values, exclude
+      end
+      
+      def exclude?
+        self.exclude
+      end
+      
+      # Returns the message for this filter to send to the Sphinx service
+      def query_message
+        message = Message.new
+        
+        message.append_string self.attribute.to_s
+        case self.values
+        when Range
+          if self.values.first.is_a?(Float) && self.values.last.is_a?(Float)
+            message.append_int FilterTypes[:float_range]
+            message.append_floats self.values.first, self.values.last
+          else
+            message.append_int FilterTypes[:range]
+            message.append_ints self.values.first, self.values.last
+          end
+        when Array
+          message.append_int FilterTypes[:values]
+          message.append_int self.values.length
+          # using to_f is a hack from the php client - to workaround 32bit
+          # signed ints on x32 platforms
+          message.append_ints *self.values.collect { |val|
+            case val
+            when TrueClass
+              1.0
+            when FalseClass
+              0.0
+            else
+              val.to_f
+            end
+          }
+        end
+        message.append_int self.exclude? ? 1 : 0
+        
+        message.to_s
+      end
+    end
+  end
+end

data/vendor/riddle/lib/riddle/client/message.rb

@@ -0,0 +1,66 @@
+module Riddle
+  class Client
+    # This class takes care of the translation of ints, strings and arrays to
+    # the format required by the Sphinx service.
+    class Message
+      def initialize
+        @message = ""
+        @size_method = @message.respond_to?(:bytesize) ? :bytesize : :length
+      end
+      
+      # Append raw data (only use if you know what you're doing)
+      def append(*args)
+        args.each { |arg| @message << arg }
+      end
+      
+      # Append a string's length, then the string itself
+      def append_string(str)
+        string = str.respond_to?(:force_encoding) ?
+          str.dup.force_encoding('ASCII-8BIT') : str
+        
+        @message << [string.send(@size_method)].pack('N') + string
+      end
+      
+      # Append an integer
+      def append_int(int)
+        @message << [int].pack('N')
+      end
+      
+      def append_64bit_int(int)
+        @message << [int >> 32, int & 0xFFFFFFFF].pack('NN')
+      end
+      
+      # Append a float
+      def append_float(float)
+        @message << [float].pack('f').unpack('L*').pack("N")
+      end
+      
+      # Append multiple integers
+      def append_ints(*ints)
+        ints.each { |int| append_int(int) }
+      end
+      
+      def append_64bit_ints(*ints)
+        ints.each { |int| append_64bit_int(int) }
+      end
+      
+      # Append multiple floats
+      def append_floats(*floats)
+        floats.each { |float| append_float(float) }
+      end
+      
+      # Append an array of strings - first appends the length of the array,
+      # then each item's length and value.
+      def append_array(array)
+        append_int(array.length)
+        
+        array.each { |item| append_string(item) }
+      end
+      
+      # Returns the entire message
+      def to_s
+        @message
+      end
+    end
+  end
+end

data/vendor/riddle/lib/riddle/client/response.rb

@@ -0,0 +1,84 @@
+module Riddle
+  class Client
+    # Used to interrogate responses from the Sphinx daemon. Keep in mind none
+    # of the methods here check whether the data they're grabbing are what the
+    # user expects - it just assumes the user knows what the data stream is
+    # made up of.
+    class Response
+      # Create with the data to interpret
+      def initialize(str)
+        @str = str
+        @marker = 0
+      end
+      
+      # Return the next string value in the stream
+      def next
+        len = next_int
+        result = @str[@marker, len]
+        @marker += len
+        
+        return result
+      end
+      
+      # Return the next integer value from the stream
+      def next_int
+        int = @str[@marker, 4].unpack('N*').first
+        @marker += 4
+        
+        return int
+      end
+      
+      def next_64bit_int
+        high, low = @str[@marker, 8].unpack('N*N*')[0..1]
+        @marker += 8
+        
+        return (high << 32) + low
+      end
+      
+      # Return the next float value from the stream
+      def next_float
+        float = @str[@marker, 4].unpack('N*').pack('L').unpack('f*').first
+        @marker += 4
+        
+        return float
+      end
+      
+      # Returns an array of string items
+      def next_array
+        count = next_int
+        items = []
+        for i in 0...count
+          items << self.next
+        end
+        
+        return items
+      end
+      
+      # Returns an array of int items
+      def next_int_array
+        count = next_int
+        items = []
+        for i in 0...count
+          items << self.next_int
+        end
+        
+        return items
+      end
+      
+      def next_float_array
+        count = next_int
+        items = []
+        for i in 0...count
+          items << self.next_float
+        end
+        
+        return items
+      end
+      
+      # Returns the length of the streamed data
+      def length
+        @str.length
+      end
+    end
+  end
+end

data/vendor/riddle/lib/riddle/configuration.rb

@@ -0,0 +1,33 @@
+require 'riddle/configuration/section'
+
+require 'riddle/configuration/distributed_index'
+require 'riddle/configuration/index'
+require 'riddle/configuration/indexer'
+require 'riddle/configuration/remote_index'
+require 'riddle/configuration/searchd'
+require 'riddle/configuration/source'
+require 'riddle/configuration/sql_source'
+require 'riddle/configuration/xml_source'
+
+module Riddle
+  class Configuration
+    class ConfigurationError < StandardError #:nodoc:
+    end
+    
+    attr_reader :indexes, :searchd
+    attr_accessor :indexer
+    
+    def initialize
+      @indexer = Riddle::Configuration::Indexer.new
+      @searchd = Riddle::Configuration::Searchd.new
+      @indexes = []
+    end
+    
+    def render
+      (
+        [@indexer.render, @searchd.render] +
+        @indexes.collect { |index| index.render }
+      ).join("\n")
+    end
+  end
+end

data/vendor/riddle/lib/riddle/configuration/distributed_index.rb

@@ -0,0 +1,48 @@
+module Riddle
+  class Configuration
+    class DistributedIndex < Riddle::Configuration::Section
+      self.settings = [:type, :local, :agent, :agent_connect_timeout,
+        :agent_query_timeout]
+      
+      attr_accessor :name, :local_indexes, :remote_indexes,
+        :agent_connect_timeout, :agent_query_timeout
+      
+      def initialize(name)
+        @name           = name
+        @local_indexes  = []
+        @remote_indexes = []
+      end
+      
+      def type
+        "distributed"
+      end
+      
+      def local
+        self.local_indexes
+      end
+      
+      def agent
+        agents = remote_indexes.collect { |index| index.remote }.uniq
+        agents.collect { |agent|
+          agent + ":" + remote_indexes.select { |index|
+            index.remote == agent
+          }.collect { |index| index.name }.join(",")
+        }
+      end
+      
+      def render
+        raise ConfigurationError unless valid?
+        
+        (
+          ["index #{name}", "{"] +
+          settings_body +
+          ["}", ""]
+        ).join("\n")
+      end
+      
+      def valid?
+        @local_indexes.length > 0 || @remote_indexes.length > 0
+      end
+    end
+  end
+end

data/vendor/riddle/lib/riddle/configuration/index.rb

@@ -0,0 +1,142 @@
+module Riddle
+  class Configuration
+    class Index < Riddle::Configuration::Section
+      self.settings = [:source, :path, :docinfo, :mlock, :morphology,
+        :stopwords, :wordforms, :exceptions, :min_word_len, :charset_type,
+        :charset_table, :ignore_chars, :min_prefix_len, :min_infix_len,
+        :prefix_fields, :infix_fields, :enable_star, :ngram_len, :ngram_chars,
+        :phrase_boundary, :phrase_boundary_step, :html_strip,
+        :html_index_attrs, :html_remove_elements, :preopen]
+      
+      attr_accessor :name, :parent, :sources, :path, :docinfo, :mlock,
+        :morphologies, :stopword_files, :wordform_files, :exception_files,
+        :min_word_len, :charset_type, :charset_table, :ignore_characters,
+        :min_prefix_len, :min_infix_len, :prefix_field_names,
+        :infix_field_names, :enable_star, :ngram_len, :ngram_characters,
+        :phrase_boundaries, :phrase_boundary_step, :html_strip,
+        :html_index_attrs, :html_remove_element_tags, :preopen
+      
+      def initialize(name, *sources)
+        @name                     = name
+        @sources                  = sources
+        @morphologies             = []
+        @stopword_files           = []
+        @wordform_files           = []
+        @exception_files          = []
+        @ignore_characters        = []
+        @prefix_field_names       = []
+        @infix_field_names        = []
+        @ngram_characters         = []
+        @phrase_boundaries        = []
+        @html_remove_element_tags = []
+      end
+      
+      def source
+        @sources.collect { |s| s.name }
+      end
+      
+      def morphology
+        nil_join @morphologies, ", "
+      end
+      
+      def morphology=(morphology)
+        @morphologies = nil_split morphology, /,\s?/
+      end
+      
+      def stopwords
+        nil_join @stopword_files, " "
+      end
+      
+      def stopwords=(stopwords)
+        @stopword_files = nil_split stopwords, ' '
+      end
+      
+      def wordforms
+        nil_join @wordform_files, " "
+      end
+      
+      def wordforms=(wordforms)
+        @wordform_files = nil_split wordforms, ' '
+      end
+      
+      def exceptions
+        nil_join @exception_files, " "
+      end
+      
+      def exceptions=(exceptions)
+        @exception_files = nil_split exceptions, ' '
+      end
+      
+      def ignore_chars
+        nil_join @ignore_characters, ", "
+      end
+      
+      def ignore_chars=(ignore_chars)
+        @ignore_characters = nil_split ignore_chars, /,\s?/
+      end
+      
+      def prefix_fields
+        nil_join @prefix_field_names, ", "
+      end
+      
+      def infix_fields
+        nil_join @infix_field_names, ", "
+      end
+      
+      def ngram_chars
+        nil_join @ngram_characters, ", "
+      end
+      
+      def ngram_chars=(ngram_chars)
+        @ngram_characters = nil_split ngram_chars, /,\s?/
+      end
+      
+      def phrase_boundary
+        nil_join @phrase_boundaries, ", "
+      end
+      
+      def phrase_boundary=(phrase_boundary)
+        @phrase_boundaries = nil_split phrase_boundary, /,\s?/
+      end
+      
+      def html_remove_elements
+        nil_join @html_remove_element_tags, ", "
+      end
+      
+      def html_remove_elements=(html_remove_elements)
+        @html_remove_element_tags = nil_split html_remove_elements, /,\s?/
+      end
+      
+      def render
+        raise ConfigurationError, "#{@name} #{@sources.inspect} #{@path} #{@parent}" unless valid?
+        
+        inherited_name = "#{name}"
+        inherited_name << " : #{parent}" if parent
+        (
+          @sources.collect { |s| s.render } +
+          ["index #{inherited_name}", "{"] +
+          settings_body +
+          ["}", ""]
+        ).join("\n")
+      end
+      
+      def valid?
+        (!@name.nil?) && (!( @sources.length == 0 || @path.nil? ) || !@parent.nil?)
+      end
+      
+      private
+      
+      def nil_split(string, pattern)
+        (string || "").split(pattern)
+      end
+      
+      def nil_join(array, delimiter)
+        if array.length == 0
+          nil
+        else
+          array.join(delimiter)
+        end
+      end
+    end
+  end
+end

data/vendor/riddle/lib/riddle/configuration/indexer.rb

@@ -0,0 +1,19 @@
+module Riddle
+  class Configuration
+    class Indexer < Riddle::Configuration::Section
+      self.settings = [:mem_limit, :max_iops, :max_iosize]
+      
+      attr_accessor *self.settings
+            
+      def render
+        raise ConfigurationError unless valid?
+        
+        (
+          ["indexer", "{"] +
+          settings_body +
+          ["}", ""]
+        ).join("\n")
+      end
+    end
+  end
+end