initforthe-thinking-sphinx 1.1.21

data/lib/thinking_sphinx/search/facets.rb

@@ -0,0 +1,104 @@
+module ThinkingSphinx
+  class Search
+    module Facets
+      # Model.facets *args
+      # ThinkingSphinx::Search.facets *args
+      # ThinkingSphinx::Search.facets *args, :all_attributes  => true
+      # ThinkingSphinx::Search.facets *args, :class_facet     => false
+      # 
+      def facets(*args)
+        options = args.extract_options!
+        
+        if options[:class]
+          facets_for_model options[:class], args, options
+        else
+          facets_for_all_models args, options
+        end
+      end
+      
+      private
+      
+      def facets_for_model(klass, args, options)
+        hash    = ThinkingSphinx::FacetCollection.new args + [options]
+        options = options.clone.merge! facet_query_options
+        
+        facets  = klass.sphinx_facets
+        facets  = Array(options.delete(:facets)).collect { |name|
+          klass.sphinx_facets.detect { |facet| facet.name.to_s == name.to_s }
+        }.compact if options[:facets]
+        
+        facets.inject(hash) do |hash, facet|
+          unless facet.name == :class && !options[:class_facet]
+            options[:group_by]    = facet.attribute_name
+            hash.add_from_results facet, search(*(args + [options]))
+          end
+          
+          hash
+        end
+      end
+      
+      def facets_for_all_models(args, options)
+        options = GlobalFacetOptions.merge(options)
+        hash    = ThinkingSphinx::FacetCollection.new args + [options]
+        options = options.merge! facet_query_options
+        
+        facet_names(options).inject(hash) do |hash, name|
+          options[:group_by] = name
+          hash.add_from_results name, search(*(args + [options]))
+          hash
+        end
+      end
+      
+      def facet_query_options
+        config = ThinkingSphinx::Configuration.instance
+        max    = config.configuration.searchd.max_matches || 1000
+        
+        {
+          :group_function => :attr,
+          :limit          => max,
+          :max_matches    => max,
+          :page           => 1
+        }
+      end
+      
+      def facet_classes(options)
+        options[:classes] || ThinkingSphinx.indexed_models.collect { |model|
+          model.constantize
+        }
+      end
+      
+      def facet_names(options)
+        classes = facet_classes(options)
+        names   = options[:all_attributes] ?
+          facet_names_for_all_classes(classes) :
+          facet_names_common_to_all_classes(classes)
+        
+        names.delete "class_crc" unless options[:class_facet]
+        names
+      end
+      
+      def facet_names_for_all_classes(classes)
+        all_facets = classes.collect { |klass| klass.sphinx_facets }.flatten
+        
+        all_facets.group_by { |facet|
+          facet.name
+        }.collect { |name, facets|
+          if facets.collect { |facet| facet.type }.uniq.length > 1
+            raise "Facet #{name} exists in more than one model with different types"
+          end
+          facets.first.attribute_name
+        }
+      end
+      
+      def facet_names_common_to_all_classes(classes)
+        facet_names_for_all_classes(classes).select { |name|
+          classes.all? { |klass|
+            klass.sphinx_facets.detect { |facet|
+              facet.attribute_name == name
+            }
+          }
+        }
+      end
+    end
+  end
+end

data/lib/thinking_sphinx/source.rb

@@ -0,0 +1,150 @@
+require 'thinking_sphinx/source/internal_properties'
+require 'thinking_sphinx/source/sql'
+
+module ThinkingSphinx
+  class Source
+    include ThinkingSphinx::Source::InternalProperties
+    include ThinkingSphinx::Source::SQL
+    
+    attr_accessor :model, :fields, :attributes, :conditions, :groupings,
+      :options
+    attr_reader :base
+    
+    def initialize(index, options = {})
+      @index        = index
+      @model        = index.model
+      @fields       = []
+      @attributes   = []
+      @conditions   = []
+      @groupings    = []
+      @options      = options
+      @associations = {}
+      
+      @base = ::ActiveRecord::Associations::ClassMethods::JoinDependency.new(
+        @model, [], nil
+      )
+      
+      unless @model.descends_from_active_record?
+        stored_class = @model.store_full_sti_class ? @model.name : @model.name.demodulize
+        @conditions << "#{@model.quoted_table_name}.#{quote_column(@model.inheritance_column)} = '#{stored_class}'"
+      end
+      
+      add_internal_attributes_and_facets
+    end
+    
+    def name
+      @model.name.underscore.tr(':/\\', '_')
+    end
+    
+    def to_riddle_for_core(offset, index)
+      source = Riddle::Configuration::SQLSource.new(
+        "#{name}_core_#{index}", adapter.sphinx_identifier
+      )
+      
+      set_source_database_settings  source
+      set_source_attributes         source, offset
+      set_source_sql                source, offset
+      set_source_settings           source
+      
+      source
+    end
+    
+    def to_riddle_for_delta(offset, index)
+      source = Riddle::Configuration::SQLSource.new(
+        "#{name}_delta_#{index}", adapter.sphinx_identifier
+      )
+      source.parent = "#{name}_core_#{index}"
+      
+      set_source_database_settings  source
+      set_source_attributes         source, offset
+      set_source_sql                source, offset, true
+      
+      source
+    end
+    
+    def delta?
+      !@index.delta_object.nil?
+    end
+    
+    # Gets the association stack for a specific key.
+    # 
+    def association(key)
+      @associations[key] ||= Association.children(@model, key)
+    end
+    
+    private
+    
+    def adapter
+      @adapter ||= @model.sphinx_database_adapter
+    end
+    
+    def set_source_database_settings(source)
+      config = @model.connection.instance_variable_get(:@config)
+      
+      source.sql_host = config[:host]           || "localhost"
+      source.sql_user = config[:username]       || config[:user] || ""
+      source.sql_pass = (config[:password].to_s || "").gsub('#', '\#')
+      source.sql_db   = config[:database]
+      source.sql_port = config[:port]
+      source.sql_sock = config[:socket]
+    end
+    
+    def set_source_attributes(source, offset)
+      attributes.each do |attrib|
+        source.send(attrib.type_to_config) << attrib.config_value(offset)
+      end
+    end
+    
+    def set_source_sql(source, offset, delta = false)
+      source.sql_query        = to_sql(:offset => offset, :delta => delta).gsub(/\n/, ' ')
+      source.sql_query_range  = to_sql_query_range(:delta => delta)
+      source.sql_query_info   = to_sql_query_info(offset)
+      
+      source.sql_query_pre += send(!delta ? :sql_query_pre_for_core : :sql_query_pre_for_delta)
+      
+      if @index.local_options[:group_concat_max_len]
+        source.sql_query_pre << "SET SESSION group_concat_max_len = #{@index.local_options[:group_concat_max_len]}"
+      end
+      
+      source.sql_query_pre += [adapter.utf8_query_pre].compact if utf8?
+    end
+    
+    def set_source_settings(source)
+      config = ThinkingSphinx::Configuration.instance
+      config.source_options.each do |key, value|
+        source.send("#{key}=".to_sym, value)
+      end
+      
+      source_options = ThinkingSphinx::Configuration::SourceOptions
+      @options.each do |key, value|
+        if source_options.include?(key.to_s) && !value.nil?
+          source.send("#{key}=".to_sym, value)
+        end
+      end
+    end
+    
+    # Returns all associations used amongst all the fields and attributes.
+    # This includes all associations between the model and what the actual
+    # columns are from.
+    # 
+    def all_associations
+      @all_associations ||= (
+        # field associations
+        @fields.collect { |field|
+          field.associations.values
+        }.flatten +
+        # attribute associations
+        @attributes.collect { |attrib|
+          attrib.associations.values if attrib.include_as_association?
+        }.compact.flatten
+      ).uniq.collect { |assoc|
+        # get ancestors as well as column-level associations
+        assoc.ancestors
+      }.flatten.uniq
+    end
+    
+    def utf8?
+      @index.options[:charset_type] == "utf-8"
+    end
+  end
+end

data/lib/thinking_sphinx/source/internal_properties.rb

@@ -0,0 +1,46 @@
+module ThinkingSphinx
+  class Source
+    module InternalProperties
+      def add_internal_attributes_and_facets
+        add_internal_attribute :sphinx_internal_id, :integer, @model.primary_key.to_sym
+        add_internal_attribute :class_crc,          :integer, crc_column, true
+        add_internal_attribute :subclass_crcs,      :multi,   subclasses_to_s
+        add_internal_attribute :sphinx_deleted,     :integer, "0"
+
+        add_internal_facet :class_crc
+      end
+
+      def add_internal_attribute(name, type, contents, facet = false)
+        return unless attribute_by_alias(name).nil?
+
+        Attribute.new(self,
+          ThinkingSphinx::Index::FauxColumn.new(contents),
+          :type   => type,
+          :as     => name,
+          :facet  => facet,
+          :admin  => true
+        )
+      end
+
+      def add_internal_facet(name)
+        return unless facet_by_alias(name).nil?
+
+        @model.sphinx_facets << ClassFacet.new(attribute_by_alias(name))
+      end
+
+      def attribute_by_alias(attr_alias)
+        @attributes.detect { |attrib| attrib.alias == attr_alias }
+      end
+
+      def facet_by_alias(name)
+        @model.sphinx_facets.detect { |facet| facet.name == name }
+      end
+
+      def subclasses_to_s
+        "'" + (@model.send(:subclasses).collect { |klass|
+          klass.to_crc32.to_s
+        } << @model.to_crc32.to_s).join(",") + "'"
+      end
+    end
+  end
+end

data/lib/thinking_sphinx/source/sql.rb

@@ -0,0 +1,126 @@
+module ThinkingSphinx
+  class Source
+    module SQL
+      # Generates the big SQL statement to get the data back for all the fields
+      # and attributes, using all the relevant association joins. If you want
+      # the version filtered for delta values, send through :delta => true in the
+      # options. Won't do much though if the index isn't set up to support a
+      # delta sibling.
+      # 
+      # Examples:
+      # 
+      #   source.to_sql
+      #   source.to_sql(:delta => true)
+      #
+      def to_sql(options={})
+        sql = <<-SQL
+SELECT #{ sql_select_clause options[:offset] }
+FROM #{ @model.quoted_table_name }
+  #{ all_associations.collect { |assoc| assoc.to_sql }.join(' ') }
+#{ sql_where_clause(options) }
+GROUP BY #{ sql_group_clause }
+        SQL
+
+        sql += " ORDER BY NULL" if adapter.sphinx_identifier == "mysql"
+        sql
+      end
+
+      # Simple helper method for the query range SQL - which is a statement that
+      # returns minimum and maximum id values. These can be filtered by delta -
+      # so pass in :delta => true to get the delta version of the SQL.
+      # 
+      def to_sql_query_range(options={})
+        return nil if @index.options[:disable_range]
+        
+        min_statement = adapter.convert_nulls(
+          "MIN(#{quote_column(@model.primary_key)})", 1
+        )
+        max_statement = adapter.convert_nulls(
+          "MAX(#{quote_column(@model.primary_key)})", 1
+        )
+
+        sql = "SELECT #{min_statement}, #{max_statement} " +
+              "FROM #{@model.quoted_table_name} "
+        if self.delta? && !@index.delta_object.clause(@model, options[:delta]).blank?
+          sql << "WHERE #{@index.delta_object.clause(@model, options[:delta])}"
+        end
+
+        sql
+      end
+
+      # Simple helper method for the query info SQL - which is a statement that
+      # returns the single row for a corresponding id.
+      # 
+      def to_sql_query_info(offset)
+        "SELECT * FROM #{@model.quoted_table_name} WHERE " +
+        "#{quote_column(@model.primary_key)} = (($id - #{offset}) / #{ThinkingSphinx.indexed_models.size})"
+      end
+
+      def sql_select_clause(offset)
+        unique_id_expr = ThinkingSphinx.unique_id_expression(offset)
+
+        (
+          ["#{@model.quoted_table_name}.#{quote_column(@model.primary_key)} #{unique_id_expr} AS #{quote_column(@model.primary_key)} "] + 
+          @fields.collect     { |field|     field.to_select_sql     } +
+          @attributes.collect { |attribute| attribute.to_select_sql }
+        ).compact.join(", ")
+      end
+
+      def sql_where_clause(options)
+        logic = []
+        logic += [
+          "#{@model.quoted_table_name}.#{quote_column(@model.primary_key)} >= $start",
+          "#{@model.quoted_table_name}.#{quote_column(@model.primary_key)} <= $end"
+        ] unless @index.options[:disable_range]
+
+        if self.delta? && !@index.delta_object.clause(@model, options[:delta]).blank?
+          logic << "#{@index.delta_object.clause(@model, options[:delta])}"
+        end
+
+        logic += (@conditions || [])
+        logic.empty? ? "" : "WHERE #{logic.join(' AND ')}"
+      end
+
+      def sql_group_clause
+        internal_groupings = []
+        if @model.column_names.include?(@model.inheritance_column)
+           internal_groupings << "#{@model.quoted_table_name}.#{quote_column(@model.inheritance_column)}"
+        end
+
+        (
+          ["#{@model.quoted_table_name}.#{quote_column(@model.primary_key)}"] + 
+          @fields.collect     { |field|     field.to_group_sql     }.compact +
+          @attributes.collect { |attribute| attribute.to_group_sql }.compact +
+          @groupings + internal_groupings
+        ).join(", ")
+      end
+
+      def sql_query_pre_for_core
+        if self.delta? && !@index.delta_object.reset_query(@model).blank?
+          [@index.delta_object.reset_query(@model)]
+        else
+          []
+        end
+      end
+
+      def sql_query_pre_for_delta
+        [""]
+      end
+
+      def quote_column(column)
+        @model.connection.quote_column_name(column)
+      end
+
+      def crc_column
+        if @model.column_names.include?(@model.inheritance_column)
+          adapter.cast_to_unsigned(adapter.convert_nulls(
+            adapter.crc(adapter.quote_with_table(@model.inheritance_column), true),
+            @model.to_crc32
+          ))
+        else
+          @model.to_crc32.to_s
+        end
+      end
+    end
+  end
+end

data/lib/thinking_sphinx/tasks.rb

@@ -0,0 +1,162 @@
+require 'fileutils'
+
+namespace :thinking_sphinx do
+  task :app_env do
+    if defined?(RAILS_ROOT)
+      Rake::Task[:environment].invoke
+      Rails.configuration.cache_classes = false
+    end
+    
+    Rake::Task[:merb_env].invoke    if defined?(Merb)
+  end
+  
+  desc "Output the current Thinking Sphinx version"
+  task :version => :app_env do
+    puts "Thinking Sphinx v" + ThinkingSphinx::Version::String
+  end
+  
+  desc "Stop if running, then start a Sphinx searchd daemon using Thinking Sphinx's settings"
+  task :running_start => :app_env do
+    Rake::Task["thinking_sphinx:stop"].invoke if sphinx_running?
+    Rake::Task["thinking_sphinx:start"].invoke
+  end
+  
+  desc "Start a Sphinx searchd daemon using Thinking Sphinx's settings"
+  task :start => :app_env do
+    config = ThinkingSphinx::Configuration.instance
+    
+    FileUtils.mkdir_p config.searchd_file_path
+    raise RuntimeError, "searchd is already running." if sphinx_running?
+    
+    Dir["#{config.searchd_file_path}/*.spl"].each { |file| File.delete(file) }
+
+    system! "#{config.bin_path}#{config.searchd_binary_name} --pidfile --config #{config.config_file}"
+    
+    sleep(2)
+    
+    if sphinx_running?
+      puts "Started successfully (pid #{sphinx_pid})."
+    else
+      puts "Failed to start searchd daemon. Check #{config.searchd_log_file}"
+    end
+  end
+  
+  desc "Stop Sphinx using Thinking Sphinx's settings"
+  task :stop => :app_env do
+    raise RuntimeError, "searchd is not running." unless sphinx_running?
+    config = ThinkingSphinx::Configuration.instance
+    pid    = sphinx_pid
+    system! "#{config.bin_path}#{searchd_binary_name} --stop --config #{config.config_file}"
+    puts "Stopped search daemon (pid #{pid})."
+  end
+  
+  desc "Restart Sphinx"
+  task :restart => [:app_env, :stop, :start]
+  
+  desc "Generate the Sphinx configuration file using Thinking Sphinx's settings"
+  task :configure => :app_env do
+    config = ThinkingSphinx::Configuration.instance
+    puts "Generating Configuration to #{config.config_file}"
+    config.build
+  end
+  
+  desc "Index data for Sphinx using Thinking Sphinx's settings"
+  task :index => :app_env do
+    ThinkingSphinx::Deltas::Job.cancel_thinking_sphinx_jobs
+    
+    config = ThinkingSphinx::Configuration.instance
+    unless ENV["INDEX_ONLY"] == "true"
+      puts "Generating Configuration to #{config.config_file}"
+      config.build
+    end
+        
+    FileUtils.mkdir_p config.searchd_file_path
+    cmd = "#{config.bin_path}#{indexer_binary_name} --config #{config.config_file} --all"
+    cmd << " --rotate" if sphinx_running?
+    
+    system! cmd
+  end
+  
+  desc "Stop Sphinx (if it's running), rebuild the indexes, and start Sphinx"
+  task :rebuild => :app_env do
+    Rake::Task["thinking_sphinx:stop"].invoke if sphinx_running?
+    Rake::Task["thinking_sphinx:index"].invoke
+    Rake::Task["thinking_sphinx:start"].invoke
+  end
+  
+  namespace :index do
+    task :delta => :app_env do
+      ThinkingSphinx.indexed_models.select { |model|
+        model.constantize.sphinx_indexes.any? { |index| index.delta? }
+      }.each do |model|
+        model.constantize.sphinx_indexes.select { |index|
+          index.delta? && index.delta_object.respond_to?(:delayed_index)
+        }.each { |index|
+          index.delta_object.delayed_index(index.model)
+        }
+      end
+    end
+  end
+  
+  desc "Process stored delta index requests"
+  task :delayed_delta => :app_env do
+    require 'delayed/worker'
+    
+    Delayed::Worker.new(
+      :min_priority => ENV['MIN_PRIORITY'],
+      :max_priority => ENV['MAX_PRIORITY']
+    ).start
+  end
+end
+
+namespace :ts do
+  desc "Output the current Thinking Sphinx version"
+  task :version => "thinking_sphinx:version"
+  desc "Stop if running, then start a Sphinx searchd daemon using Thinking Sphinx's settings"
+  task :run     => "thinking_sphinx:running_start"
+  desc "Start a Sphinx searchd daemon using Thinking Sphinx's settings"
+  task :start   => "thinking_sphinx:start"
+  desc "Stop Sphinx using Thinking Sphinx's settings"
+  task :stop    => "thinking_sphinx:stop"
+  desc "Index data for Sphinx using Thinking Sphinx's settings"
+  task :in      => "thinking_sphinx:index"
+  namespace :in do
+    desc "Index Thinking Sphinx datetime delta indexes"
+    task :delta => "thinking_sphinx:index:delta"
+  end
+  task :index   => "thinking_sphinx:index"
+  desc "Restart Sphinx"
+  task :restart => "thinking_sphinx:restart"
+  desc "Generate the Sphinx configuration file using Thinking Sphinx's settings"
+  task :conf    => "thinking_sphinx:configure"
+  desc "Generate the Sphinx configuration file using Thinking Sphinx's settings"
+  task :config  => "thinking_sphinx:configure"
+  desc "Stop Sphinx (if it's running), rebuild the indexes, and start Sphinx"
+  task :rebuild => "thinking_sphinx:rebuild"
+  desc "Process stored delta index requests"
+  task :dd      => "thinking_sphinx:delayed_delta"
+end
+
+def sphinx_pid
+  ThinkingSphinx.sphinx_pid
+end
+
+def sphinx_running?
+  ThinkingSphinx.sphinx_running?
+end
+
+# a fail-fast, hopefully helpful version of system
+def system!(cmd)
+  unless system(cmd)
+    raise <<-SYSTEM_CALL_FAILED
+The following command failed:
+  #{cmd}
+
+This could be caused by a PATH issue in the environment of cron/passenger/etc. Your current PATH:
+  #{ENV['PATH']}
+You can set the path to your indexer and searchd binaries using the bin_path property in config/sphinx.yml:
+  production:
+    bin_path: '/usr/local/bin'
+SYSTEM_CALL_FAILED
+  end
+end