dpickett-thinking-sphinx 1.1.4 → 1.1.12

data/lib/thinking_sphinx/configuration.rb

@@ -31,8 +31,10 @@ module ThinkingSphinx
   # fashion to database.yml - using the following keys: config_file,
   # searchd_log_file, query_log_file, pid_file, searchd_file_path, port,
   # allow_star, enable_star, min_prefix_len, min_infix_len, mem_limit,
-  # max_matches, # morphology, charset_type, charset_table, ignore_chars,
-  # html_strip, # html_remove_elements. I think you've got the idea.
+  # max_matches, morphology, charset_type, charset_table, ignore_chars,
+  # html_strip, html_remove_elements, delayed_job_priority.
+  #
+  # I think you've got the idea.
   # 
   # Each setting in the YAML file is optional - so only put in the ones you
   # want to change.
@@ -55,7 +57,8 @@ module ThinkingSphinx
         
     attr_accessor :config_file, :searchd_log_file, :query_log_file,
       :pid_file, :searchd_file_path, :address, :port, :allow_star,
-      :database_yml_file, :app_root, :bin_path, :model_directories
+      :database_yml_file, :app_root, :bin_path, :model_directories,
+      :delayed_job_priority
     
     attr_accessor :source_options, :index_options
     
@@ -85,7 +88,9 @@ module ThinkingSphinx
       self.searchd_file_path    = "#{self.app_root}/db/sphinx/#{environment}"
       self.allow_star           = false
       self.bin_path             = ""
-      self.model_directories    = ["#{app_root}/app/models/"]
+      self.model_directories    = ["#{app_root}/app/models/"] +
+        Dir.glob("#{app_root}/vendor/plugins/*/app/models/")
+      self.delayed_job_priority = 0
       
       self.source_options  = {}
       self.index_options   = {

data/lib/thinking_sphinx/core/string.rb

@@ -1,18 +1,11 @@
+require 'zlib'
+
 module ThinkingSphinx
   module Core
     module String
-      
       def to_crc32
-        result = 0xFFFFFFFF
-        self.each_byte do |byte|
-          result ^= byte
-          8.times do
-            result = (result >> 1) ^ (0xEDB88320 * (result & 1))
-          end
-        end
-        result ^ 0xFFFFFFFF
+        Zlib.crc32 self
       end
-      
     end
   end
 end

data/lib/thinking_sphinx/deltas/default_delta.rb

@@ -11,18 +11,20 @@ module ThinkingSphinx
       def index(model, instance = nil)
         return true unless ThinkingSphinx.updates_enabled? &&
           ThinkingSphinx.deltas_enabled?
+        return true if instance && !toggled(instance)
         
         config = ThinkingSphinx::Configuration.instance
         client = Riddle::Client.new config.address, config.port
+        rotate = ThinkingSphinx.sphinx_running? ? "--rotate" : ""
+        
+        output = `#{config.bin_path}indexer --config #{config.config_file} #{rotate} #{delta_index_name model}`
+        puts(output) unless ThinkingSphinx.suppress_delta_output?
         
         client.update(
           core_index_name(model),
           ['sphinx_deleted'],
           {instance.sphinx_document_id => [1]}
-        ) if instance && ThinkingSphinx.sphinx_running? && instance.in_core_index?
-        
-        output = `#{config.bin_path}indexer --config #{config.config_file} --rotate #{delta_index_name model}`
-        puts output unless ThinkingSphinx.suppress_delta_output?
+        ) if instance && ThinkingSphinx.sphinx_running? && instance.in_both_indexes?
         
         true
       end
@@ -37,7 +39,8 @@ module ThinkingSphinx
       
       def reset_query(model)
         "UPDATE #{model.quoted_table_name} SET " +
-        "#{@index.quote_column(@column.to_s)} = #{adapter.boolean(false)}"
+        "#{@index.quote_column(@column.to_s)} = #{adapter.boolean(false)} " +
+        "WHERE #{@index.quote_column(@column.to_s)} = #{adapter.boolean(true)}"
       end
       
       def clause(model, toggled)

data/lib/thinking_sphinx/deltas/delayed_delta.rb

@@ -9,13 +9,15 @@ module ThinkingSphinx
     class DelayedDelta < ThinkingSphinx::Deltas::DefaultDelta
       def index(model, instance = nil)
         ThinkingSphinx::Deltas::Job.enqueue(
-          ThinkingSphinx::Deltas::DeltaJob.new(delta_index_name(model))
+          ThinkingSphinx::Deltas::DeltaJob.new(delta_index_name(model)),
+          ThinkingSphinx::Configuration.instance.delayed_job_priority
         )
         
         Delayed::Job.enqueue(
           ThinkingSphinx::Deltas::FlagAsDeletedJob.new(
             core_index_name(model), instance.sphinx_document_id
-          )
+          ),
+          ThinkingSphinx::Configuration.instance.delayed_job_priority
         ) if instance
         
         true

data/lib/thinking_sphinx/deltas.rb

@@ -5,7 +5,8 @@ require 'thinking_sphinx/deltas/datetime_delta'
 module ThinkingSphinx
   module Deltas
     def self.parse(index, options)
-      case options.delete(:delta)
+      delta_option = options.delete(:delta)
+      case delta_option
       when TrueClass, :default
         DefaultDelta.new index, options
       when :delayed
@@ -15,7 +16,11 @@ module ThinkingSphinx
       when FalseClass, nil
         nil
       else
-        raise "Unknown delta type"
+        if delta_option.ancestors.include?(ThinkingSphinx::Deltas::DefaultDelta)
+          delta_option.new index, options
+        else
+          raise "Unknown delta type"
+        end
       end
     end
   end

data/lib/thinking_sphinx/deploy/capistrano.rb

@@ -0,0 +1,80 @@
+namespace :thinking_sphinx do
+  namespace :install do
+    desc "Install Sphinx by source"
+    task :sphinx do
+      with_postgres = false
+      run "which pg_config" do |channel, stream, data|
+        with_postgres = !(data.nil? || data == "")
+      end
+      
+      args = []
+      if with_postgres
+        run "pg_config --pkgincludedir" do |channel, stream, data|
+          args << "--with-pgsql=#{data}"
+        end
+      end
+      
+      commands = <<-CMD
+      wget -q http://www.sphinxsearch.com/downloads/sphinx-0.9.8.1.tar.gz >> sphinx.log
+      tar xzvf sphinx-0.9.8.1.tar.gz
+      cd sphinx-0.9.8.1
+      ./configure #{args.join(" ")}
+      make
+      sudo make install
+      rm -rf sphinx-0.9.8.1 sphinx-0.9.8.1.tar.gz
+      CMD
+      run commands.split(/\n\s+/).join(" && ")
+    end
+    
+    desc "Install Thinking Sphinx as a gem from GitHub"
+    task :ts do
+      sudo "gem install freelancing-god-thinking-sphinx --source http://gems.github.com"
+    end
+  end
+  
+  desc "Generate the Sphinx configuration file"
+  task :configure do
+    rake "thinking_sphinx:configure"
+  end
+  
+  desc "Index data"
+  task :index do
+    rake "thinking_sphinx:index"
+  end
+  
+  desc "Start the Sphinx daemon"
+  task :start do
+    configure
+    rake "thinking_sphinx:start"
+  end
+  
+  desc "Stop the Sphinx daemon"
+  task :stop do
+    configure
+    rake "thinking_sphinx:stop"
+  end
+  
+  desc "Stop and then start the Sphinx daemon"
+  task :restart do
+    stop
+    start
+  end
+  
+  desc "Stop, re-index and then start the Sphinx daemon"
+  task :rebuild do
+    stop
+    index
+    start
+  end
+  
+  desc "Add the shared folder for sphinx files for the production environment"
+  task :shared_sphinx_folder, :roles => :web do
+    sudo "mkdir -p #{shared_path}/db/sphinx/production"
+  end
+
+  def rake(*tasks)
+    tasks.each do |t|
+      run "cd #{current_path} && rake #{t} RAILS_ENV=production"
+    end
+  end
+end

data/lib/thinking_sphinx/facet.rb

@@ -9,25 +9,38 @@ module ThinkingSphinx
         raise "Can't translate Facets on multiple-column field or attribute"
       end
     end
+
+    def self.name_for(facet)
+      case facet
+      when Facet
+        facet.name
+      when String, Symbol
+        facet.to_s.gsub(/(_facet|_crc)$/,'').to_sym
+      end
+    end
     
     def name
       reference.unique_name
     end
+
+    def self.attribute_name_for(name)
+      name.to_s == 'class' ? 'class_crc' : "#{name}_facet"
+    end
     
     def attribute_name
-      @attribute_name ||= case @reference
-      when Attribute
-        @reference.unique_name.to_s
-      when Field
-        @reference.unique_name.to_s + "_sort"
-      end
+      # @attribute_name ||= case @reference
+      # when Attribute
+      #   @reference.unique_name.to_s
+      # when Field
+      @attribute_name ||= @reference.unique_name.to_s + "_facet"
+      # end
     end
     
     def value(object, attribute_value)
       return translate(object, attribute_value) if @reference.is_a?(Field)
       
       case @reference.type
-      when :string, :multi
+      when :string
         translate(object, attribute_value)
       when :datetime
         Time.at(attribute_value)
@@ -42,7 +55,7 @@ module ThinkingSphinx
       name
     end
     
-    protected
+    private
     
     def translate(object, attribute_value)
       column.__stack.each { |method|
@@ -55,4 +68,4 @@ module ThinkingSphinx
       @reference.columns.first
     end
   end
-end
+end

data/lib/thinking_sphinx/facet_collection.rb

@@ -5,20 +5,26 @@ module ThinkingSphinx
     def initialize(arguments)
       @arguments        = arguments.clone
       @attribute_values = {}
-      @facets           = []
+      @facet_names           = []
     end
     
     def add_from_results(facet, results)
-      self[facet.name]              ||= {}
-      @attribute_values[facet.name] = {}
-      @facets << facet
+      name = ThinkingSphinx::Facet.name_for(facet)
+
+      self[name]              ||= {}
+      @attribute_values[name] ||= {}
+      @facet_names << name
+
+      return if results.empty?
+
+      facet = facet_from_object(results.first, facet) if facet.is_a?(String)
       
       results.each_with_groupby_and_count { |result, group, count|
         facet_value = facet.value(result, group)
         
-        self[facet.name][facet_value]              ||= 0
-        self[facet.name][facet_value]               += count
-        @attribute_values[facet.name][facet_value] ||= group
+        self[name][facet_value]              ||= 0
+        self[name][facet_value]              += count
+        @attribute_values[name][facet_value]  = group
       }
     end
     
@@ -28,8 +34,8 @@ module ThinkingSphinx
       options[:with] ||= {}
       
       hash.each do |key, value|
-        attrib = facet_for_key(key).attribute_name
-        options[:with][attrib] = @attribute_values[key][value]
+        attrib = ThinkingSphinx::Facet.attribute_name_for(key)
+        options[:with][attrib] = underlying_value key, value
       end
       
       arguments << options
@@ -38,8 +44,17 @@ module ThinkingSphinx
     
     private
     
-    def facet_for_key(key)
-      @facets.detect { |facet| facet.name == key }
+    def underlying_value(key, value)
+      case value
+      when Array
+        value.collect { |item| underlying_value(key, item) }
+      else
+        @attribute_values[key][value]
+      end
+    end
+    
+    def facet_from_object(object, name)
+      object.sphinx_facets.detect { |facet| facet.attribute_name == name }
     end
   end
-end
+end

data/lib/thinking_sphinx/field.rb

@@ -7,9 +7,8 @@ module ThinkingSphinx
   # generate SQL statements, you'll need to set the base model, and all the
   # associations. Which can get messy. Use Index.link!, it really helps.
   # 
-  class Field
-    attr_accessor :alias, :columns, :sortable, :associations, :model, :infixes,
-      :prefixes, :faceted
+  class Field < ThinkingSphinx::Property
+    attr_accessor :sortable, :infixes, :prefixes
     
     # To create a new field, you'll need to pass in either a single Column
     # or an array of them, and some (optional) options. The columns are
@@ -54,16 +53,11 @@ module ThinkingSphinx
     #   )
     # 
     def initialize(columns, options = {})
-      @columns      = Array(columns)
-      @associations = {}
-
-      raise "Cannot define a field with no columns. Maybe you are trying to index a field with a reserved name (id, name). You can fix this error by using a symbol rather than a bare name (:id instead of id)." if @columns.empty? || @columns.any? { |column| !column.respond_to?(:__stack) }
+      super
       
-      @alias    = options[:as]
       @sortable = options[:sortable] || false
       @infixes  = options[:infixes]  || false
       @prefixes = options[:prefixes] || false
-      @faceted  = options[:facet]    || false
     end
     
     # Get the part of the SELECT clause related to this field. Don't forget
@@ -77,96 +71,10 @@ module ThinkingSphinx
         column_with_prefix(column)
       }.join(', ')
       
-      clause = adapter.concatenate(clause) if concat_ws?
+      clause = adapter.concatenate(clause)       if concat_ws?
       clause = adapter.group_concatenate(clause) if is_many?
       
       "#{adapter.cast_to_string clause } AS #{quote_column(unique_name)}"
     end
-    
-    # Get the part of the GROUP BY clause related to this field - if one is
-    # needed. If not, all you'll get back is nil. The latter will happen if
-    # there's multiple data values (read: a has_many or has_and_belongs_to_many
-    # association).
-    #
-    def to_group_sql
-      case
-      when is_many?, ThinkingSphinx.use_group_by_shortcut?
-        nil
-      else
-        @columns.collect { |column|
-          column_with_prefix(column)
-        }
-      end
-    end
-    
-    # Returns the unique name of the field - which is either the alias of
-    # the field, or the name of the only column - if there is only one. If
-    # there isn't, there should be an alias. Else things probably won't work.
-    # Consider yourself warned.
-    #
-    def unique_name
-      if @columns.length == 1
-        @alias || @columns.first.__name
-      else
-        @alias
-      end
-    end
-    
-    def to_facet
-      return nil unless @faceted
-      
-      ThinkingSphinx::Facet.new(self)
-    end
-    
-    private
-    
-    def adapter
-      @adapter ||= @model.sphinx_database_adapter
-    end
-    
-    def quote_column(column)
-      @model.connection.quote_column_name(column)
-    end
-    
-    # Indication of whether the columns should be concatenated with a space
-    # between each value. True if there's either multiple sources or multiple
-    # associations.
-    # 
-    def concat_ws?
-      @columns.length > 1 || multiple_associations?
-    end
-        
-    # Checks whether any column requires multiple associations (which only
-    # happens for polymorphic situations).
-    # 
-    def multiple_associations?
-      associations.any? { |col,assocs| assocs.length > 1 }
-    end
-    
-    # Builds a column reference tied to the appropriate associations. This
-    # dives into the associations hash and their corresponding joins to
-    # figure out how to correctly reference a column in SQL.
-    #
-    def column_with_prefix(column)
-      if column.is_string?
-        column.__name
-      elsif associations[column].empty?
-        "#{@model.quoted_table_name}.#{quote_column(column.__name)}"
-      else
-        associations[column].collect { |assoc|
-          assoc.has_column?(column.__name) ?
-          "#{@model.connection.quote_table_name(assoc.join.aliased_table_name)}" + 
-          ".#{quote_column(column.__name)}" :
-          nil
-        }.compact.join(', ')
-      end
-    end
-    
-    # Could there be more than one value related to the parent record? If so,
-    # then this will return true. If not, false. It's that simple.
-    #
-    def is_many?
-      associations.values.flatten.any? { |assoc| assoc.is_many? }
-    end
   end
 end

data/lib/thinking_sphinx/index/builder.rb

@@ -87,22 +87,16 @@ module ThinkingSphinx
         def indexes(*args)
           options = args.extract_options!
           args.each do |columns|
-            fields << Field.new(FauxColumn.coerce(columns), options)
+            field = Field.new(FauxColumn.coerce(columns), options)
+            fields << field
             
-            if fields.last.sortable || fields.last.faceted
-              attributes << Attribute.new(
-                fields.last.columns.collect { |col| col.clone },
-                options.merge(
-                  :type => :string,
-                  :as => fields.last.unique_name.to_s.concat("_sort").to_sym
-                ).except(:facet)
-              )
-            end
+            add_sort_attribute field, options   if field.sortable
+            add_facet_attribute field, options  if field.faceted
           end
         end
         alias_method :field,    :indexes
         alias_method :includes, :indexes
-        
+                
         # This is the method to add attributes to your index (hence why it is
         # aliased as 'attribute'). The syntax is the same as #indexes, so use
         # that as starting point, but keep in mind the following points.
@@ -142,7 +136,10 @@ module ThinkingSphinx
         def has(*args)
           options = args.extract_options!
           args.each do |columns|
-            attributes << Attribute.new(FauxColumn.coerce(columns), options)
+            attribute = Attribute.new(FauxColumn.coerce(columns), options)
+            attributes << attribute
+            
+            add_facet_attribute attribute, options if attribute.faceted
           end
         end
         alias_method :attribute, :has
@@ -152,7 +149,10 @@ module ThinkingSphinx
           options[:facet] = true
           
           args.each do |columns|
-            attributes << Attribute.new(FauxColumn.coerce(columns), options)
+            attribute = Attribute.new(FauxColumn.coerce(columns), options)
+            attributes << attribute
+            
+            add_facet_attribute attribute, options
           end
         end
         
@@ -200,7 +200,17 @@ module ThinkingSphinx
         # 
         # Please don't forget to add a boolean field named 'delta' to your
         # model's database table if enabling the delta index for it.
+        # Valid options for the delta property are:
         # 
+        # true
+        # false
+        # :default
+        # :delayed
+        # :datetime
+        # 
+        # You can also extend ThinkingSphinx::Deltas::DefaultDelta to implement 
+        # your own handling for delta indexing.
+        
         def set_property(*args)
           options = args.extract_options!
           if options.empty?
@@ -224,8 +234,29 @@ module ThinkingSphinx
         # 
         # Example: indexes assoc(:properties).column
         # 
-        def assoc(assoc)
-          FauxColumn.new(method)
+        def assoc(assoc, *args)
+          FauxColumn.new(assoc, *args)
+        end
+        
+        private
+        
+        def add_sort_attribute(field, options)
+          add_internal_attribute field, options, "_sort"
+        end
+        
+        def add_facet_attribute(resource, options)
+          add_internal_attribute resource, options, "_facet", true
+        end
+        
+        def add_internal_attribute(resource, options, suffix, crc = false)
+          @attributes << Attribute.new(
+            resource.columns.collect { |col| col.clone },
+            options.merge(
+              :type => resource.is_a?(Field) ? :string : options[:type],
+              :as   => resource.unique_name.to_s.concat(suffix).to_sym,
+              :crc  => crc
+            ).except(:facet)
+          )
         end
       end
     end