onemorecloud-websolr-rails 1.1.1

data/lib/class_methods.rb

@@ -0,0 +1,239 @@
+require File.dirname(__FILE__) + '/common_methods'
+require File.dirname(__FILE__) + '/parser_methods'
+
+module ActsAsSolr #:nodoc:
+
+  module ClassMethods
+    include CommonMethods
+    include ParserMethods
+    
+    # Finds instances of a model. Terms are ANDed by default, can be overwritten 
+    # by using OR between terms
+    # 
+    # Here's a sample (untested) code for your controller:
+    # 
+    #  def search
+    #    results = Book.find_by_solr params[:query]
+    #  end
+    # 
+    # You can also search for specific fields by searching for 'field:value'
+    # 
+    # ====options:
+    # offset:: - The first document to be retrieved (offset)
+    # limit:: - The number of rows per page
+    # order:: - Orders (sort by) the result set using a given criteria:
+    #
+    #             Book.find_by_solr 'ruby', :order => 'description asc'
+    # 
+    # field_types:: This option is deprecated and will be obsolete by version 1.0.
+    #               There's no need to specify the :field_types anymore when doing a 
+    #               search in a model that specifies a field type for a field. The field 
+    #               types are automatically traced back when they're included.
+    # 
+    #                 class Electronic < ActiveRecord::Base
+    #                   acts_as_solr :fields => [{:price => :range_float}]
+    #                 end
+    # 
+    # facets:: This option argument accepts the following arguments:
+    #          fields:: The fields to be included in the faceted search (Solr's facet.field)
+    #          query:: The queries to be included in the faceted search (Solr's facet.query)
+    #          zeros:: Display facets with count of zero. (true|false)
+    #          sort:: Sorts the faceted resuls by highest to lowest count. (true|false)
+    #          browse:: This is where the 'drill-down' of the facets work. Accepts an array of
+    #                   fields in the format "facet_field:term"
+    #          mincount:: Replacement for zeros (it has been deprecated in Solr). Specifies the
+    #                     minimum count necessary for a facet field to be returned. (Solr's
+    #                     facet.mincount) Overrides :zeros if it is specified. Default is 0.
+    #
+    #          dates:: Run date faceted queries using the following arguments:
+    #            fields:: The fields to be included in the faceted date search (Solr's facet.date).
+    #                     It may be either a String/Symbol or Hash. If it's a hash the options are the
+    #                     same as date_facets minus the fields option (i.e., :start:, :end, :gap, :other,
+    #                     :between). These options if provided will override the base options.
+    #                     (Solr's f.<field_name>.date.<key>=<value>).
+    #            start:: The lower bound for the first date range for all Date Faceting. Required if
+    #                    :fields is present
+    #            end:: The upper bound for the last date range for all Date Faceting. Required if
+    #                  :fields is prsent
+    #            gap:: The size of each date range expressed as an interval to be added to the lower
+    #                  bound using the DateMathParser syntax.  Required if :fields is prsent
+    #            hardend:: A Boolean parameter instructing Solr what do do in the event that
+    #                      facet.date.gap does not divide evenly between facet.date.start and facet.date.end.
+    #            other:: This param indicates that in addition to the counts for each date range
+    #                    constraint between facet.date.start and facet.date.end, other counds should be
+    #                    calculated. May specify more then one in an Array. The possible options are:
+    #              before:: - all records with lower bound less than start
+    #              after:: - all records with upper bound greater than end
+    #              between:: - all records with field values between start and end
+    #              none:: - compute no other bounds (useful in per field assignment)
+    #              all:: - shortcut for before, after, and between
+    #            filter:: Similar to :query option provided by :facets, in that accepts an array of
+    #                     of date queries to limit results. Can not be used as a part of a :field hash.
+    #                     This is the only option that can be used if :fields is not present.
+    # 
+    # Example:
+    # 
+    #   Electronic.find_by_solr "memory", :facets => {:zeros => false, :sort => true,
+    #                                                 :query => ["price:[* TO 200]",
+    #                                                            "price:[200 TO 500]",
+    #                                                            "price:[500 TO *]"],
+    #                                                 :fields => [:category, :manufacturer],
+    #                                                 :browse => ["category:Memory","manufacturer:Someone"]}
+    # 
+    #
+    # Examples of date faceting:
+    #
+    #  basic:
+    #    Electronic.find_by_solr "memory", :facets => {:dates => {:fields => [:updated_at, :created_at],
+    #      :start => 'NOW-10YEARS/DAY', :end => 'NOW/DAY', :gap => '+2YEARS', :other => :before}}
+    #
+    #  advanced:
+    #    Electronic.find_by_solr "memory", :facets => {:dates => {:fields => [:updated_at,
+    #    {:created_at => {:start => 'NOW-20YEARS/DAY', :end => 'NOW-10YEARS/DAY', :other => [:before, :after]}
+    #    }], :start => 'NOW-10YEARS/DAY', :end => 'NOW/DAY', :other => :before, :filter =>
+    #    ["created_at:[NOW-10YEARS/DAY TO NOW/DAY]", "updated_at:[NOW-1YEAR/DAY TO NOW/DAY]"]}}
+    #
+    #  filter only:
+    #    Electronic.find_by_solr "memory", :facets => {:dates => {:filter => "updated_at:[NOW-1YEAR/DAY TO NOW/DAY]"}}
+    #
+    #
+    #
+    # scores:: If set to true this will return the score as a 'solr_score' attribute
+    #          for each one of the instances found. Does not currently work with find_id_by_solr
+    # 
+    #            books = Book.find_by_solr 'ruby OR splinter', :scores => true
+    #            books.records.first.solr_score
+    #            => 1.21321397
+    #            books.records.last.solr_score
+    #            => 0.12321548
+    # 
+    # lazy:: If set to true the search will return objects that will touch the database when you ask for one
+    #        of their attributes for the first time. Useful when you're using fragment caching based solely on
+    #        types and ids.
+    #
+    def find_by_solr(query, options={})
+      data = parse_query(query, options)
+      return parse_results(data, options) if data
+    end
+    
+    # Finds instances of a model and returns an array with the ids:
+    #  Book.find_id_by_solr "rails" => [1,4,7]
+    # The options accepted are the same as find_by_solr
+    # 
+    def find_id_by_solr(query, options={})
+      data = parse_query(query, options)
+      return parse_results(data, {:format => :ids}) if data
+    end
+    
+    # This method can be used to execute a search across multiple models:
+    #   Book.multi_solr_search "Napoleon OR Tom", :models => [Movie]
+    # 
+    # ====options:
+    # Accepts the same options as find_by_solr plus:
+    # models:: The additional models you'd like to include in the search
+    # results_format:: Specify the format of the results found
+    #                  :objects :: Will return an array with the results being objects (default). Example:
+    #                               Book.multi_solr_search "Napoleon OR Tom", :models => [Movie], :results_format => :objects
+    #                  :ids :: Will return an array with the ids of each entry found. Example:
+    #                           Book.multi_solr_search "Napoleon OR Tom", :models => [Movie], :results_format => :ids
+    #                           => [{"id" => "Movie:1"},{"id" => Book:1}]
+    #                          Where the value of each array is as Model:instance_id
+    # scores:: If set to true this will return the score as a 'solr_score' attribute
+    #          for each one of the instances found. Does not currently work with find_id_by_solr
+    # 
+    #            books = Book.multi_solr_search 'ruby OR splinter', :scores => true
+    #            books.records.first.solr_score
+    #            => 1.21321397
+    #            books.records.last.solr_score
+    #            => 0.12321548
+    # 
+    def multi_solr_search(query, options = {})
+      models = multi_model_suffix(options)
+      options.update(:results_format => :objects) unless options[:results_format]
+      data = parse_query(query, options, models)
+      
+      if data.nil? or data.total_hits == 0
+        return SearchResults.new(:docs => [], :total => 0)
+      end
+
+      result = find_multi_search_objects(data, options)
+      if options[:scores] and options[:results_format] == :objects
+        add_scores(result, data) 
+      end
+      SearchResults.new :docs => result, :total => data.total_hits
+    end
+
+    def find_multi_search_objects(data, options)
+      result = []
+      if options[:results_format] == :objects
+        data.hits.each do |doc| 
+          k = doc.fetch('id').first.to_s.split(':')
+          result << k[0].constantize.find_by_id(k[1])
+        end
+      elsif options[:results_format] == :ids
+        data.hits.each{|doc| result << {"id" => doc.values.pop.to_s}}
+      end
+      result
+    end
+    
+    def multi_model_suffix(options)
+      models = "AND (#{solr_configuration[:type_field]}:#{self.name}"
+      models << " OR " + options[:models].collect {|m| "#{solr_configuration[:type_field]}:" + m.to_s}.join(" OR ") if options[:models].is_a?(Array)
+      models << ")"
+    end
+    
+    # returns the total number of documents found in the query specified:
+    #  Book.count_by_solr 'rails' => 3
+    # 
+    def count_by_solr(query, options = {})        
+      data = parse_query(query, options)
+      data.total_hits
+    end
+            
+    # It's used to rebuild the Solr index for a specific model. 
+    #  Book.rebuild_solr_index
+    # 
+    # If batch_size is greater than 0, adds will be done in batches.
+    # NOTE: If using sqlserver, be sure to use a finder with an explicit order.
+    # Non-edge versions of rails do not handle pagination correctly for sqlserver
+    # without an order clause.
+    # 
+    # If a finder block is given, it will be called to retrieve the items to index.
+    # This can be very useful for things such as updating based on conditions or
+    # using eager loading for indexed associations.
+    def rebuild_solr_index(batch_size=0, &finder)
+      finder ||= lambda { |ar, options| ar.find(:all, options.merge({:order => self.primary_key})) }
+      start_time = Time.now
+
+      if batch_size > 0
+        items_processed = 0
+        limit = batch_size
+        offset = 0
+        begin
+          iteration_start = Time.now
+          items = finder.call(self, {:limit => limit, :offset => offset})
+          add_batch = items.collect { |content| content.to_solr_doc }
+    
+          if items.size > 0
+            solr_add add_batch
+            solr_commit
+          end
+    
+          items_processed += items.size
+          last_id = items.last.id if items.last
+          time_so_far = Time.now - start_time
+          iteration_time = Time.now - iteration_start         
+          logger.info "#{Process.pid}: #{items_processed} items for #{self.name} have been batch added to index in #{'%.3f' % time_so_far}s at #{'%.3f' % (items_processed / time_so_far)} items/sec (#{'%.3f' % (items.size / iteration_time)} items/sec for the last batch). Last id: #{last_id}"
+          offset += items.size
+        end while items.nil? || items.size > 0
+      else
+        items = finder.call(self, {})
+        items.each { |content| content.solr_save }
+        items_processed = items.size
+      end
+      solr_optimize
+      logger.info items_processed > 0 ? "Index for #{self.name} has been rebuilt" : "Nothing to index for #{self.name}"
+    end
+  end
+  
+end

data/lib/common_methods.rb

@@ -0,0 +1,89 @@
+module ActsAsSolr #:nodoc:
+  
+  module CommonMethods
+    
+    # Converts field types into Solr types
+    def get_solr_field_type(field_type)
+      if field_type.is_a?(Symbol)
+        case field_type
+          when :float
+            return "f"
+          when :integer
+            return "i"
+          when :boolean
+            return "b"
+          when :string
+            return "s"
+          when :date
+            return "d"
+          when :range_float
+            return "rf"
+          when :range_integer
+            return "ri"
+          when :facet
+            return "facet"
+          when :text
+            return "t"
+        else
+          raise "Unknown field_type symbol: #{field_type}"
+        end
+      elsif field_type.is_a?(String)
+        return field_type
+      else
+        raise "Unknown field_type class: #{field_type.class}: #{field_type}"
+      end
+    end
+    
+    # Sets a default value when value being set is nil.
+    def set_value_if_nil(field_type)
+      case field_type
+        when "b", :boolean
+          return "false"
+        when "s", "t", "d", :date, :string, :text
+          return ""
+        when "f", "rf", :float, :range_float
+          return 0.00
+        when "i", "ri", :integer, :range_integer
+          return 0
+      else
+        return ""
+      end
+    end
+    
+    # Sends an add command to Solr
+    def solr_add(add_xml)
+      ActsAsSolr::Post.execute(Solr::Request::AddDocument.new(add_xml))
+    end
+    
+    # Sends the delete command to Solr
+    def solr_delete(solr_ids)
+      ActsAsSolr::Post.execute(Solr::Request::Delete.new(:id => solr_ids))
+    end
+    
+    # Sends the commit command to Solr
+    def solr_commit
+      ActsAsSolr::Post.execute(Solr::Request::Commit.new)
+    end
+    
+    # Optimizes the Solr index. Solr says:
+    # 
+    # Optimizations can take nearly ten minutes to run. 
+    # We are presuming optimizations should be run once following large 
+    # batch-like updates to the collection and/or once a day.
+    # 
+    # One of the solutions for this would be to create a cron job that 
+    # runs every day at midnight and optmizes the index:
+    #   0 0 * * * /your_rails_dir/script/runner -e production "Model.solr_optimize"
+    # 
+    def solr_optimize
+      ActsAsSolr::Post.execute(Solr::Request::Optimize.new)
+    end
+    
+    # Returns the id for the given instance
+    def record_id(object)
+      eval "object.#{object.class.primary_key}"
+    end
+    
+  end
+  
+end

data/lib/deprecation.rb

@@ -0,0 +1,61 @@
+module ActsAsSolr #:nodoc:
+  
+  class Post
+    def initialize(body, mode = :search)
+      @body = body
+      @mode = mode
+      puts "The method ActsAsSolr::Post.new(body, mode).execute_post is depracated. " +
+           "Use ActsAsSolr::Post.execute(body, mode) instead!"
+    end
+    
+    def execute_post
+      ActsAsSolr::Post.execute(@body, @mode)
+    end    
+  end
+  
+  module ClassMethods
+    def find_with_facet(query, options={})
+      Deprecation.plog "The method find_with_facet is deprecated. Use find_by_solr instead, passing the " +
+                       "arguments the same way you used to do with find_with_facet."
+      find_by_solr(query, options)
+    end
+  end
+  
+  class Deprecation
+    # Validates the options passed during query
+    def self.validate_query options={}
+      if options[:field_types]
+        plog "The option :field_types for searching is deprecated. " +
+             "The field types are automatically traced back when you specify a field type in your model."
+      end
+      if options[:sort_by]
+        plog "The option :sort_by is deprecated, use :order instead!"
+        options[:order] ||= options[:sort_by]
+      end
+      if options[:start]
+        plog "The option :start is deprecated, use :offset instead!"
+        options[:offset] ||= options[:start]
+      end
+      if options[:rows]
+        plog "The option :rows is deprecated, use :limit instead!"
+        options[:limit] ||= options[:rows]
+      end
+    end
+    
+    # Validates the options passed during indexing
+    def self.validate_index options={}
+      if options[:background]
+        plog "The :background option is being deprecated. There are better and more efficient " +
+             "ways to handle delayed saving of your records."
+      end
+    end
+    
+    # This will print the text to stdout and log the text
+    # if rails logger is available
+    def self.plog text
+      puts text
+      RAILS_DEFAULT_LOGGER.warn text if defined? RAILS_DEFAULT_LOGGER
+    end
+  end
+  
+end

data/lib/instance_methods.rb

@@ -0,0 +1,166 @@
+module ActsAsSolr #:nodoc:
+  
+  module InstanceMethods
+
+    # Solr id is <class.name>:<id> to be unique across all models
+    def solr_id
+      "#{self.class.name}:#{record_id(self)}"
+    end
+
+    # saves to the Solr index
+    def solr_save
+      return true if indexing_disabled?
+      if evaluate_condition(:if, self) 
+        logger.debug "solr_save: #{self.class.name} : #{record_id(self)}"
+        solr_add to_solr_doc
+        solr_commit if configuration[:auto_commit]
+        true
+      else
+        solr_destroy
+      end
+    end
+
+    def indexing_disabled?
+      evaluate_condition(:offline, self) || !configuration[:if]
+    end
+
+    # remove from index
+    def solr_destroy
+      return true if indexing_disabled?
+      logger.debug "solr_destroy: #{self.class.name} : #{record_id(self)}"
+      solr_delete solr_id
+      solr_commit if configuration[:auto_commit]
+      true
+    end
+
+    # convert instance to Solr document
+    def to_solr_doc
+      logger.debug "to_solr_doc: creating doc for class: #{self.class.name}, id: #{record_id(self)}"
+      doc = Solr::Document.new
+      doc.boost = validate_boost(configuration[:boost]) if configuration[:boost]
+      
+      doc << {:id => solr_id,
+              solr_configuration[:type_field] => self.class.name,
+              solr_configuration[:primary_key_field] => record_id(self).to_s}
+
+      # iterate through the fields and add them to the document,
+      configuration[:solr_fields].each do |field_name, options|
+        #field_type = configuration[:facets] && configuration[:facets].include?(field) ? :facet : :text
+        
+        field_boost = options[:boost] || solr_configuration[:default_boost]
+        field_type = get_solr_field_type(options[:type])
+        solr_name = options[:as] || field_name
+        
+        value = self.send("#{field_name}_for_solr")
+        value = set_value_if_nil(field_type) if value.to_s == ""
+        
+        # add the field to the document, but only if it's not the id field
+        # or the type field (from single table inheritance), since these
+        # fields have already been added above.
+        if field_name.to_s != self.class.primary_key and field_name.to_s != "type"
+          suffix = get_solr_field_type(field_type)
+          # This next line ensures that e.g. nil dates are excluded from the 
+          # document, since they choke Solr. Also ignores e.g. empty strings, 
+          # but these can't be searched for anyway: 
+          # http://www.mail-archive.com/solr-dev@lucene.apache.org/msg05423.html
+          next if value.nil? || value.to_s.strip.empty?
+          [value].flatten.each do |v|
+            v = set_value_if_nil(suffix) if value.to_s == ""
+            field = Solr::Field.new("#{solr_name}_#{suffix}" => ERB::Util.html_escape(v.to_s))
+            field.boost = validate_boost(field_boost)
+            doc << field
+          end
+        end
+      end
+      
+      add_includes(doc)
+      logger.debug doc.to_xml
+      doc
+    end
+    
+    private
+    def add_includes(doc)
+      if configuration[:solr_includes].respond_to?(:each)
+        configuration[:solr_includes].each do |association, options|
+          data = options[:multivalued] ? [] : ""
+          field_name = options[:as] || association.to_s.singularize
+          field_type = get_solr_field_type(options[:type])
+          field_boost = options[:boost] || solr_configuration[:default_boost]
+          suffix = get_solr_field_type(field_type)
+          case self.class.reflect_on_association(association).macro
+          when :has_many, :has_and_belongs_to_many
+            records = self.send(association).to_a
+            unless records.empty?
+              records.each {|r| data << include_value(r, options)}
+              [data].flatten.each do |value|
+                field = Solr::Field.new("#{field_name}_#{suffix}" => value)
+                field.boost = validate_boost(field_boost)
+                doc << field
+              end
+            end
+          when :has_one, :belongs_to
+            record = self.send(association)
+            unless record.nil?
+              doc["#{field_name}_#{suffix}"] = include_value(record, options)
+            end
+          end
+        end
+      end
+    end
+    
+    def include_value(record, options)
+      if options[:using].is_a? Proc
+        options[:using].call(record)
+      elsif options[:using].is_a? Symbol
+        record.send(options[:using])
+      else
+        record.attributes.inject([]){|k,v| k << "#{v.first}=#{ERB::Util.html_escape(v.last)}"}.join(" ")
+      end
+    end
+
+    def validate_boost(boost)
+      boost_value = case boost
+      when Float
+        return solr_configuration[:default_boost] if boost < 0
+        boost
+      when Proc
+        boost.call(self)
+      when Symbol
+        if self.respond_to?(boost)
+          self.send(boost)
+        end
+      end
+      
+      boost_value || solr_configuration[:default_boost]
+    end
+    
+    def condition_block?(condition)
+      condition.respond_to?("call") && (condition.arity == 1 || condition.arity == -1)
+    end
+    
+    def evaluate_condition(which_condition, field)
+      condition = configuration[which_condition]
+      case condition
+        when Symbol
+          field.send(condition)
+        when String
+          eval(condition, binding)
+        when FalseClass, NilClass
+          false
+        when TrueClass
+          true
+        else
+          if condition_block?(condition)
+            condition.call(field)
+          else
+            raise(
+              ArgumentError,
+              "The :#{which_condition} option has to be either a symbol, string (to be eval'ed), proc/method, true/false, or " +
+              "class implementing a static validation method"
+            )
+          end
+        end
+    end
+    
+  end
+end