dpickett-thinking-sphinx 1.1.4

data/lib/thinking_sphinx/configuration.rb

@@ -0,0 +1,236 @@
+require 'erb'
+require 'singleton'
+
+module ThinkingSphinx
+  # This class both keeps track of the configuration settings for Sphinx and
+  # also generates the resulting file for Sphinx to use.
+  # 
+  # Here are the default settings, relative to RAILS_ROOT where relevant:
+  #
+  # config file::           config/#{environment}.sphinx.conf
+  # searchd log file::      log/searchd.log
+  # query log file::        log/searchd.query.log
+  # pid file::              log/searchd.#{environment}.pid
+  # searchd files::         db/sphinx/#{environment}/
+  # address::               127.0.0.1
+  # port::                  3312
+  # allow star::            false
+  # min prefix length::     1
+  # min infix length::      1
+  # mem limit::             64M
+  # max matches::           1000
+  # morphology::            stem_en
+  # charset type::          utf-8
+  # charset table::         nil
+  # ignore chars::          nil
+  # html strip::            false
+  # html remove elements::  ''
+  #
+  # If you want to change these settings, create a YAML file at
+  # config/sphinx.yml with settings for each environment, in a similar
+  # fashion to database.yml - using the following keys: config_file,
+  # searchd_log_file, query_log_file, pid_file, searchd_file_path, port,
+  # allow_star, enable_star, min_prefix_len, min_infix_len, mem_limit,
+  # max_matches, # morphology, charset_type, charset_table, ignore_chars,
+  # html_strip, # html_remove_elements. I think you've got the idea.
+  # 
+  # Each setting in the YAML file is optional - so only put in the ones you
+  # want to change.
+  #
+  # Keep in mind, if for some particular reason you're using a version of
+  # Sphinx older than 0.9.8 r871 (that's prior to the proper 0.9.8 release),
+  # don't set allow_star to true.
+  # 
+  class Configuration
+    include Singleton
+    
+    SourceOptions = %w( mysql_connect_flags sql_range_step sql_query_pre
+      sql_query_post sql_ranged_throttle sql_query_post_index )
+    
+    IndexOptions  = %w( charset_table charset_type docinfo enable_star
+      exceptions html_index_attrs html_remove_elements html_strip ignore_chars
+      min_infix_len min_prefix_len min_word_len mlock morphology ngram_chars
+      ngram_len phrase_boundary phrase_boundary_step preopen stopwords
+      wordforms )
+        
+    attr_accessor :config_file, :searchd_log_file, :query_log_file,
+      :pid_file, :searchd_file_path, :address, :port, :allow_star,
+      :database_yml_file, :app_root, :bin_path, :model_directories
+    
+    attr_accessor :source_options, :index_options
+    
+    attr_reader :environment, :configuration
+    
+    # Load in the configuration settings - this will look for config/sphinx.yml
+    # and parse it according to the current environment.
+    # 
+    def initialize(app_root = Dir.pwd)
+      self.reset
+    end
+    
+    def reset
+      self.app_root          = RAILS_ROOT if defined?(RAILS_ROOT)
+      self.app_root          = Merb.root  if defined?(Merb)
+      self.app_root        ||= app_root
+      
+      @configuration = Riddle::Configuration.new
+      @configuration.searchd.address    = "127.0.0.1"
+      @configuration.searchd.port       = 3312
+      @configuration.searchd.pid_file   = "#{self.app_root}/log/searchd.#{environment}.pid"
+      @configuration.searchd.log        = "#{self.app_root}/log/searchd.log"
+      @configuration.searchd.query_log  = "#{self.app_root}/log/searchd.query.log"
+      
+      self.database_yml_file    = "#{self.app_root}/config/database.yml"
+      self.config_file          = "#{self.app_root}/config/#{environment}.sphinx.conf"
+      self.searchd_file_path    = "#{self.app_root}/db/sphinx/#{environment}"
+      self.allow_star           = false
+      self.bin_path             = ""
+      self.model_directories    = ["#{app_root}/app/models/"]
+      
+      self.source_options  = {}
+      self.index_options   = {
+        :charset_type => "utf-8",
+        :morphology   => "stem_en"
+      }
+            
+      parse_config
+      
+      self
+    end
+    
+    def self.environment
+      @@environment ||= (
+        defined?(Merb) ? Merb.environment : ENV['RAILS_ENV']
+      ) || "development"
+    end
+    
+    def environment
+      self.class.environment
+    end
+    
+    def controller
+      @controller ||= Riddle::Controller.new(@configuration, self.config_file)
+    end
+    
+    # Generate the config file for Sphinx by using all the settings defined and
+    # looping through all the models with indexes to build the relevant
+    # indexer and searchd configuration, and sources and indexes details.
+    #
+    def build(file_path=nil)
+      load_models
+      file_path ||= "#{self.config_file}"
+      
+      @configuration.indexes.clear
+      
+      ThinkingSphinx.indexed_models.each_with_index do |model, model_index|
+        @configuration.indexes.concat model.constantize.to_riddle(model_index)
+      end
+      
+      open(file_path, "w") do |file|
+        file.write @configuration.render
+      end
+    end
+    
+    # Make sure all models are loaded - without reloading any that
+    # ActiveRecord::Base is already aware of (otherwise we start to hit some
+    # messy dependencies issues).
+    # 
+    def load_models
+      self.model_directories.each do |base|
+        Dir["#{base}**/*.rb"].each do |file|
+          model_name = file.gsub(/^#{base}([\w_\/\\]+)\.rb/, '\1')
+        
+          next if model_name.nil?
+          next if ::ActiveRecord::Base.send(:subclasses).detect { |model|
+            model.name == model_name
+          }
+        
+          begin
+            model_name.camelize.constantize
+          rescue LoadError
+            model_name.gsub!(/.*[\/\\]/, '').nil? ? next : retry
+          rescue NameError
+            next
+          end
+        end
+      end
+    end
+    
+    def address
+      @configuration.searchd.address
+    end
+    
+    def address=(address)
+      @configuration.searchd.address = address
+    end
+    
+    def port
+      @configuration.searchd.port
+    end
+    
+    def port=(port)
+      @configuration.searchd.port = port
+    end
+    
+    def pid_file
+      @configuration.searchd.pid_file
+    end
+    
+    def pid_file=(pid_file)
+      @configuration.searchd.pid_file = pid_file
+    end
+    
+    def searchd_log_file
+      @configuration.searchd.log
+    end
+    
+    def searchd_log_file=(file)
+      @configuration.searchd.log = file
+    end
+    
+    def query_log_file
+      @configuration.searchd.query_log
+    end
+    
+    def query_log_file=(file)
+      @configuration.searchd.query_log = file
+    end
+    
+    private
+    
+    # Parse the config/sphinx.yml file - if it exists - then use the attribute
+    # accessors to set the appropriate values. Nothing too clever.
+    # 
+    def parse_config
+      path = "#{app_root}/config/sphinx.yml"
+      return unless File.exists?(path)
+      
+      conf = YAML::load(ERB.new(IO.read(path)).result)[environment]
+      
+      conf.each do |key,value|
+        self.send("#{key}=", value) if self.methods.include?("#{key}=")
+        
+        set_sphinx_setting self.source_options, key, value, SourceOptions
+        set_sphinx_setting self.index_options,  key, value, IndexOptions
+        set_sphinx_setting @configuration.searchd, key, value
+        set_sphinx_setting @configuration.indexer, key, value
+      end unless conf.nil?
+      
+      self.bin_path += '/' unless self.bin_path.blank?
+      
+      if self.allow_star
+        self.index_options[:enable_star]    = true
+        self.index_options[:min_prefix_len] = 1
+      end
+    end
+    
+    def set_sphinx_setting(object, key, value, allowed = {})
+      if object.is_a?(Hash)
+        object[key.to_sym] = value if allowed.include?(key.to_s)
+      else
+        object.send("#{key}=", value) if object.methods.include?("#{key}")
+        send("#{key}=", value) if self.methods.include?("#{key}")
+      end
+    end
+  end
+end

data/lib/thinking_sphinx/core/string.rb

@@ -0,0 +1,22 @@
+module ThinkingSphinx
+  module Core
+    module String
+      
+      def to_crc32
+        result = 0xFFFFFFFF
+        self.each_byte do |byte|
+          result ^= byte
+          8.times do
+            result = (result >> 1) ^ (0xEDB88320 * (result & 1))
+          end
+        end
+        result ^ 0xFFFFFFFF
+      end
+      
+    end
+  end
+end
+
+class String
+  include ThinkingSphinx::Core::String
+end

data/lib/thinking_sphinx/deltas/datetime_delta.rb

@@ -0,0 +1,50 @@
+module ThinkingSphinx
+  module Deltas
+    class DatetimeDelta < ThinkingSphinx::Deltas::DefaultDelta
+      attr_accessor :column, :threshold
+      
+      def initialize(index, options)
+        @index      = index
+        @column     = options.delete(:delta_column) || :updated_at
+        @threshold  = options.delete(:threshold)    || 1.day
+      end
+      
+      def index(model, instance = nil)
+        # do nothing
+        true
+      end
+      
+      def delayed_index(model)
+        config = ThinkingSphinx::Configuration.instance
+        rotate = ThinkingSphinx.sphinx_running? ? "--rotate" : ""
+        
+        output = `#{config.bin_path}indexer --config #{config.config_file} #{rotate} #{delta_index_name model}`
+        output += `#{config.bin_path}indexer --config #{config.config_file} #{rotate} --merge #{core_index_name model} #{delta_index_name model} --merge-dst-range sphinx_deleted 0 0`
+        puts output unless ThinkingSphinx.suppress_delta_output?
+        
+        true
+      end
+            
+      def toggle(instance)
+        # do nothing
+      end
+      
+      def toggled(instance)
+        instance.send(@column) > @threshold.ago
+      end
+      
+      def reset_query(model)
+        nil
+      end
+      
+      def clause(model, toggled)
+        if toggled
+          "#{model.quoted_table_name}.#{@index.quote_column(@column.to_s)}" +
+          " > #{adapter.time_difference(@threshold)}"
+        else
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/thinking_sphinx/deltas/default_delta.rb

@@ -0,0 +1,65 @@
+module ThinkingSphinx
+  module Deltas
+    class DefaultDelta
+      attr_accessor :column
+      
+      def initialize(index, options)
+        @index  = index
+        @column = options.delete(:delta_column) || :delta
+      end
+      
+      def index(model, instance = nil)
+        return true unless ThinkingSphinx.updates_enabled? &&
+          ThinkingSphinx.deltas_enabled?
+        
+        config = ThinkingSphinx::Configuration.instance
+        client = Riddle::Client.new config.address, config.port
+        
+        client.update(
+          core_index_name(model),
+          ['sphinx_deleted'],
+          {instance.sphinx_document_id => [1]}
+        ) if instance && ThinkingSphinx.sphinx_running? && instance.in_core_index?
+        
+        output = `#{config.bin_path}indexer --config #{config.config_file} --rotate #{delta_index_name model}`
+        puts output unless ThinkingSphinx.suppress_delta_output?
+        
+        true
+      end
+      
+      def toggle(instance)
+        instance.delta = true
+      end
+      
+      def toggled(instance)
+        instance.delta
+      end
+      
+      def reset_query(model)
+        "UPDATE #{model.quoted_table_name} SET " +
+        "#{@index.quote_column(@column.to_s)} = #{adapter.boolean(false)}"
+      end
+      
+      def clause(model, toggled)
+        "#{model.quoted_table_name}.#{@index.quote_column(@column.to_s)}" +
+        " = #{adapter.boolean(toggled)}"
+      end
+      
+      protected
+      
+      def core_index_name(model)
+        "#{model.source_of_sphinx_index.name.underscore.tr(':/\\', '_')}_core"
+      end
+      
+      def delta_index_name(model)
+        "#{model.source_of_sphinx_index.name.underscore.tr(':/\\', '_')}_delta"
+      end  
+      
+      private
+      
+      def adapter
+        @adapter = @index.model.sphinx_database_adapter
+      end
+    end
+  end
+end

data/lib/thinking_sphinx/deltas/delayed_delta/delta_job.rb

@@ -0,0 +1,24 @@
+module ThinkingSphinx
+  module Deltas
+    class DeltaJob
+      attr_accessor :index
+      
+      def initialize(index)
+        @index = index
+      end
+      
+      def perform
+        return true unless ThinkingSphinx.updates_enabled? &&
+          ThinkingSphinx.deltas_enabled?
+        
+        config = ThinkingSphinx::Configuration.instance
+        client = Riddle::Client.new config.address, config.port
+        
+        output = `#{config.bin_path}indexer --config #{config.config_file} --rotate #{index}`
+        puts output unless ThinkingSphinx.suppress_delta_output?
+        
+        true
+      end
+    end
+  end
+end

data/lib/thinking_sphinx/deltas/delayed_delta/flag_as_deleted_job.rb

@@ -0,0 +1,27 @@
+module ThinkingSphinx
+  module Deltas
+    class FlagAsDeletedJob
+      attr_accessor :index, :document_id
+      
+      def initialize(index, document_id)
+        @index, @document_id = index, document_id
+      end
+      
+      def perform
+        return true unless ThinkingSphinx.updates_enabled?
+        
+        config = ThinkingSphinx::Configuration.instance
+        client = Riddle::Client.new config.address, config.port
+        
+        client.update(
+          @index,
+          ['sphinx_deleted'],
+          {@document_id => [1]}
+        ) if ThinkingSphinx.sphinx_running? &&
+          ThinkingSphinx::Search.search_for_id(@document_id, @index)
+        
+        true
+      end
+    end
+  end
+end

data/lib/thinking_sphinx/deltas/delayed_delta/job.rb

@@ -0,0 +1,26 @@
+module ThinkingSphinx
+  module Deltas
+    class Job < Delayed::Job
+      def self.enqueue(object, priority = 0)
+        super unless duplicates_exist(object)
+      end
+      
+      def self.cancel_thinking_sphinx_jobs
+        if connection.tables.include?("delayed_jobs")
+          delete_all("handler LIKE '--- !ruby/object:ThinkingSphinx::Deltas::%'")
+        end
+      end
+
+      private
+
+      def self.duplicates_exist(object)
+        count(
+          :conditions => {
+            :handler    => object.to_yaml,
+            :locked_at  => nil
+          }
+        ) > 0
+      end
+    end
+  end
+end

data/lib/thinking_sphinx/deltas/delayed_delta.rb

@@ -0,0 +1,25 @@
+require 'delayed/job'
+
+require 'thinking_sphinx/deltas/delayed_delta/delta_job'
+require 'thinking_sphinx/deltas/delayed_delta/flag_as_deleted_job'
+require 'thinking_sphinx/deltas/delayed_delta/job'
+
+module ThinkingSphinx
+  module Deltas
+    class DelayedDelta < ThinkingSphinx::Deltas::DefaultDelta
+      def index(model, instance = nil)
+        ThinkingSphinx::Deltas::Job.enqueue(
+          ThinkingSphinx::Deltas::DeltaJob.new(delta_index_name(model))
+        )
+        
+        Delayed::Job.enqueue(
+          ThinkingSphinx::Deltas::FlagAsDeletedJob.new(
+            core_index_name(model), instance.sphinx_document_id
+          )
+        ) if instance
+        
+        true
+      end
+    end
+  end
+end

data/lib/thinking_sphinx/deltas.rb

@@ -0,0 +1,22 @@
+require 'thinking_sphinx/deltas/default_delta'
+require 'thinking_sphinx/deltas/delayed_delta'
+require 'thinking_sphinx/deltas/datetime_delta'
+
+module ThinkingSphinx
+  module Deltas
+    def self.parse(index, options)
+      case options.delete(:delta)
+      when TrueClass, :default
+        DefaultDelta.new index, options
+      when :delayed
+        DelayedDelta.new index, options
+      when :datetime
+        DatetimeDelta.new index, options
+      when FalseClass, nil
+        nil
+      else
+        raise "Unknown delta type"
+      end
+    end
+  end
+end

data/lib/thinking_sphinx/facet.rb

@@ -0,0 +1,58 @@
+module ThinkingSphinx
+  class Facet
+    attr_reader :reference
+    
+    def initialize(reference)
+      @reference = reference
+      
+      if reference.columns.length != 1
+        raise "Can't translate Facets on multiple-column field or attribute"
+      end
+    end
+    
+    def name
+      reference.unique_name
+    end
+    
+    def attribute_name
+      @attribute_name ||= case @reference
+      when Attribute
+        @reference.unique_name.to_s
+      when Field
+        @reference.unique_name.to_s + "_sort"
+      end
+    end
+    
+    def value(object, attribute_value)
+      return translate(object, attribute_value) if @reference.is_a?(Field)
+      
+      case @reference.type
+      when :string, :multi
+        translate(object, attribute_value)
+      when :datetime
+        Time.at(attribute_value)
+      when :boolean
+        attribute_value > 0
+      else
+        attribute_value
+      end
+    end
+    
+    def to_s
+      name
+    end
+    
+    protected
+    
+    def translate(object, attribute_value)
+      column.__stack.each { |method|
+        object = object.send(method)
+      }
+      object.send(column.__name)
+    end
+    
+    def column
+      @reference.columns.first
+    end
+  end
+end

data/lib/thinking_sphinx/facet_collection.rb

@@ -0,0 +1,45 @@
+module ThinkingSphinx
+  class FacetCollection < Hash
+    attr_accessor :arguments
+    
+    def initialize(arguments)
+      @arguments        = arguments.clone
+      @attribute_values = {}
+      @facets           = []
+    end
+    
+    def add_from_results(facet, results)
+      self[facet.name]              ||= {}
+      @attribute_values[facet.name] = {}
+      @facets << facet
+      
+      results.each_with_groupby_and_count { |result, group, count|
+        facet_value = facet.value(result, group)
+        
+        self[facet.name][facet_value]              ||= 0
+        self[facet.name][facet_value]               += count
+        @attribute_values[facet.name][facet_value] ||= group
+      }
+    end
+    
+    def for(hash = {})
+      arguments        = @arguments.clone
+      options          = arguments.extract_options!
+      options[:with] ||= {}
+      
+      hash.each do |key, value|
+        attrib = facet_for_key(key).attribute_name
+        options[:with][attrib] = @attribute_values[key][value]
+      end
+      
+      arguments << options
+      ThinkingSphinx::Search.search *arguments
+    end
+    
+    private
+    
+    def facet_for_key(key)
+      @facets.detect { |facet| facet.name == key }
+    end
+  end
+end