watson-acts_as_ferret 0.4.8.2

data/lib/instance_methods.rb

@@ -0,0 +1,172 @@
+module ActsAsFerret #:nodoc:
+
+  module InstanceMethods
+    include ResultAttributes
+
+    # Returns an array of strings with the matches highlighted. The +query+ can
+    # either be a String or a Ferret::Search::Query object.
+    # 
+    # === Options
+    #
+    # field::            field to take the content from. This field has 
+    #                    to have it's content stored in the index 
+    #                    (:store => :yes in your call to aaf). If not
+    #                    given, all stored fields are searched, and the
+    #                    highlighted content found in all of them is returned.
+    #                    set :highlight => :no in the field options to
+    #                    avoid highlighting of contents from a :stored field.
+    # excerpt_length::   Default: 150. Length of excerpt to show. Highlighted
+    #                    terms will be in the centre of the excerpt.
+    # num_excerpts::     Default: 2. Number of excerpts to return.
+    # pre_tag::          Default: "<em>". Tag to place to the left of the
+    #                    match.  
+    # post_tag::         Default: "</em>". This tag should close the
+    #                    +:pre_tag+.
+    # ellipsis::         Default: "...". This is the string that is appended
+    #                    at the beginning and end of excerpts (unless the
+    #                    excerpt hits the start or end of the field. You'll
+    #                    probably want to change this to a Unicode elipsis
+    #                    character.
+    def highlight(query, options = {})
+      self.class.highlight(self.ferret_key, query, options)
+    end
+    
+    # re-eneable ferret indexing for this instance after a call to #disable_ferret
+    def enable_ferret  
+      @ferret_disabled = nil 
+    end
+    alias ferret_enable enable_ferret  # compatibility
+    
+    # returns true if ferret indexing is enabled for this record.
+    #
+    # The optional is_bulk_index parameter will be true if the method is called
+    # by rebuild_index or bulk_index, and false otherwise.
+    #
+    # If is_bulk_index is true, the class level ferret_enabled state will be
+    # ignored by this method (per-instance ferret_enabled checks however will 
+    # take place, so if you override this method to forbid indexing of certain 
+    # records you're still safe).
+    def ferret_enabled?(is_bulk_index = false)
+      @ferret_disabled.nil? && (is_bulk_index || self.class.ferret_enabled?) && (aaf_configuration[:if].nil? || aaf_configuration[:if].call(self))
+    end
+
+    # Returns the analyzer to use when adding this record to the index.
+    #
+    # Override to return a specific analyzer for any record that is to be
+    # indexed, i.e. specify a different analyzer based on language. Returns nil
+    # by default so the global analyzer (specified with the acts_as_ferret
+    # call) is used.
+    def ferret_analyzer
+      nil
+    end
+
+    # Disable Ferret for this record for a specified amount of time. ::once will 
+    # disable Ferret for the next call to #save (this is the default), ::always 
+    # will do so for all subsequent calls. 
+    #
+    # Note that this will turn off only the create and update hooks, but not the 
+    # destroy hook. I think that's reasonable, if you think the opposite, please 
+    # tell me.
+    #
+    # To manually trigger reindexing of a record after you're finished modifying 
+    # it, you can call #ferret_update directly instead of #save (remember to
+    # enable ferret again before).
+    #
+    # When given a block, this will be executed without any ferret indexing of 
+    # this object taking place. The optional argument in this case can be used 
+    # to indicate if the object should be indexed after executing the block
+    # (::index_when_finished). Automatic Ferret indexing of this object will be 
+    # turned on after the block has been executed. If passed ::index_when_true, 
+    # the index will only be updated if the block evaluated not to false or nil.
+    #
+    def disable_ferret(option = :once)
+      if block_given?
+        @ferret_disabled = :always
+        result = yield
+        ferret_enable
+        ferret_update if option == :index_when_finished || (option == :index_when_true && result)
+        result
+      elsif [:once, :always].include?(option)
+        @ferret_disabled = option
+      else
+        raise ArgumentError.new("Invalid Argument #{option}")
+      end
+    end
+
+    # add to index
+    def ferret_create
+      if ferret_enabled?
+        logger.debug "ferret_create/update: #{self.ferret_key}"
+        self.class.aaf_index << self
+      else
+        ferret_enable if @ferret_disabled == :once
+      end
+      true # signal success to AR
+    end
+    alias :ferret_update :ferret_create
+    
+
+    # remove from index
+    def ferret_destroy
+      logger.debug "ferret_destroy: #{self.ferret_key}"
+      begin
+        self.class.aaf_index.remove self.ferret_key
+      rescue
+        logger.warn("Could not find indexed value for this object: #{$!}\n#{$!.backtrace}")
+      end
+      true # signal success to AR
+    end
+    
+    def ferret_key
+      "#{self.class.name}-#{self.send self.class.primary_key}" unless new_record?
+    end
+    
+    # turn this instance into a ferret document (which basically is a hash of
+    # fieldname => value pairs)
+    def to_doc
+      logger.debug "creating doc for class: #{self.ferret_key}"
+      Ferret::Document.new.tap do |doc|
+        # store the id and class name of each item, and the unique key used for identifying the record
+        # even in multi-class indexes.
+        doc[:key] = self.ferret_key
+        doc[:id] = self.id.to_s
+        doc[:class_name] = self.class.name
+      
+        # iterate through the fields and add them to the document
+        aaf_configuration[:defined_fields].each_pair do |field, config|
+          doc[field] = self.send("#{field}_to_ferret") unless config[:ignore]
+        end
+        if aaf_configuration[:boost]
+          if self.respond_to?(aaf_configuration[:boost])
+            boost = self.send aaf_configuration[:boost]
+            doc.boost = boost.to_i if boost
+          else
+            logger.error "boost option should point to an instance method: #{aaf_configuration[:boost]}"
+          end
+        end
+      end
+    end
+
+    def document_number
+      self.class.aaf_index.document_number(self.ferret_key)
+    end
+
+    def query_for_record
+      self.class.aaf_index.query_for_record(self.ferret_key)
+    end
+
+    def content_for_field_name(field, via = field, dynamic_boost = nil)
+      field_data = (respond_to?(via) ? send(via) : instance_variable_get("@#{via}"))
+      field_data = (field_data.is_a?(Array) ? field_data.map{|d| d.to_s} : field_data.to_s)
+      # field_data = self.send(via) || self.instance_variable_get("@#{via}")
+      if (dynamic_boost && boost_value = self.send(dynamic_boost))
+        field_data = Ferret::Field.new(field_data)
+        field_data.boost = boost_value.to_i
+      end
+      field_data
+    end
+
+
+  end
+
+end

data/lib/local_index.rb

@@ -0,0 +1,202 @@
+module ActsAsFerret
+  class LocalIndex < AbstractIndex
+    include MoreLikeThis::IndexMethods
+
+    def initialize(index_name)
+      super
+      ensure_index_exists
+    end
+
+    def reopen!
+      logger.debug "reopening index at #{index_definition[:ferret][:path]}"
+      close
+      ferret_index
+    end
+
+    # The 'real' Ferret Index instance
+    def ferret_index
+      ensure_index_exists
+      (@ferret_index ||= Ferret::Index::Index.new(index_definition[:ferret])).tap do |idx|
+        idx.batch_size = index_definition[:reindex_batch_size]
+        idx.logger = logger
+      end
+    end
+
+    # Checks for the presence of a segments file in the index directory
+    # Rebuilds the index if none exists.
+    def ensure_index_exists
+      #logger.debug "LocalIndex: ensure_index_exists at #{index_definition[:index_dir]}"
+      unless File.file? "#{index_definition[:index_dir]}/segments"
+        ActsAsFerret::ensure_directory(index_definition[:index_dir])
+        rebuild_index 
+      end
+    end
+
+    # Closes the underlying index instance
+    def close
+      @ferret_index.close if @ferret_index
+    rescue StandardError 
+      # is raised when index already closed
+    ensure
+      @ferret_index = nil
+    end
+
+    # rebuilds the index from all records of the model classes associated with this index
+    def rebuild_index
+      models = index_definition[:registered_models]
+      logger.debug "rebuild index with models: #{models.inspect}"
+      close
+      index = Ferret::Index::Index.new(index_definition[:ferret].dup.update(:auto_flush  => false, 
+                                                                            :field_infos => ActsAsFerret::field_infos(index_definition),
+                                                                            :create      => true))
+      index.batch_size = index_definition[:reindex_batch_size]
+      index.logger = logger
+      index.index_models models
+      reopen!
+    end
+
+    def bulk_index(class_name, ids, options)
+      ferret_index.bulk_index(class_name.constantize, ids, options)
+    end
+
+    # Parses the given query string into a Ferret Query object.
+    def process_query(query, options = {})
+      return query unless String === query
+      ferret_index.synchronize do
+        if options[:analyzer]
+          # use per-query analyzer if present
+          qp = Ferret::QueryParser.new ferret_index.instance_variable_get('@options').merge(options)
+          reader = ferret_index.reader
+          qp.fields =
+              reader.fields unless options[:all_fields] || options[:fields]
+          qp.tokenized_fields =
+              reader.tokenized_fields unless options[:tokenized_fields]
+          return qp.parse query
+        else
+          return ferret_index.process_query(query)
+        end
+      end
+    end
+
+    # Total number of hits for the given query. 
+    def total_hits(query, options = {})
+      ferret_index.search(process_query(query, options), options).total_hits
+    end
+
+    def searcher
+      ferret_index
+    end
+
+
+    ######################################
+    # methods working on a single record
+    # called from instance_methods, here to simplify interfacing with the
+    # remote ferret server
+    # TODO having to pass id and class_name around like this isn't nice
+    ######################################
+
+    # add record to index
+    # record may be the full AR object, a Ferret document instance or a Hash
+    def add(record, analyzer = nil)
+      unless Hash === record || Ferret::Document === record
+        analyzer = record.ferret_analyzer
+        record = record.to_doc 
+      end
+      ferret_index.add_document(record, analyzer)
+    end
+    alias << add
+
+    # delete record from index
+    def remove(key)
+      ferret_index.delete key
+    end
+
+    # highlight search terms for the record with the given id.
+    def highlight(key, query, options = {})
+      logger.debug("highlight: #{key} query: #{query}")
+      options.reverse_merge! :num_excerpts => 2, :pre_tag => '<em>', :post_tag => '</em>'
+      highlights = []
+      ferret_index.synchronize do
+        doc_num = document_number(key)
+
+        if options[:field]
+          highlights << ferret_index.highlight(query, doc_num, options)
+        else
+          query = process_query(query) # process only once
+          index_definition[:ferret_fields].each_pair do |field, config|
+            next if config[:store] == :no || config[:highlight] == :no
+            options[:field] = field
+            highlights << ferret_index.highlight(query, doc_num, options)
+          end
+        end
+      end
+      return highlights.compact.flatten[0..options[:num_excerpts]-1]
+    end
+
+    # retrieves the ferret document number of the record with the given key.
+    def document_number(key)
+      docnum = ferret_index.doc_number(key)
+      # hits = ferret_index.search query_for_record(key)
+      # return hits.hits.first.doc if hits.total_hits == 1
+      raise "cannot determine document number for record #{key}" if docnum.nil?
+      docnum
+    end
+
+    # build a ferret query matching only the record with the given id
+    # the class name only needs to be given in case of a shared index configuration
+    def query_for_record(key)
+      return Ferret::Search::TermQuery.new(:key, key.to_s)
+      # if shared?
+      #   raise InvalidArgumentError.new("shared index needs class_name argument") if class_name.nil?
+      #   Ferret::Search::BooleanQuery.new.tap do |bq|
+      #     bq.add_query(Ferret::Search::TermQuery.new(:id,         id.to_s),    :must)
+      #     bq.add_query(Ferret::Search::TermQuery.new(:class_name, class_name), :must)
+      #   end
+      # else
+      #   Ferret::Search::TermQuery.new(:id, id.to_s)
+      # end
+    end
+
+
+    # retrieves stored fields from index definition in case the fields to retrieve 
+    # haven't been specified with the :lazy option
+    def determine_stored_fields(options = {})
+      stored_fields = options[:lazy]
+      if stored_fields && !(Array === stored_fields)
+        stored_fields = index_definition[:ferret_fields].select { |field, config| config[:store] == :yes }.map(&:first)
+      end
+      logger.debug "stored_fields: #{stored_fields.inspect}"
+      return stored_fields
+    end
+
+    # loads data for fields declared as :lazy from the Ferret document
+    def extract_stored_fields(doc, stored_fields) 
+      data = {} 
+      unless stored_fields.nil?
+        logger.debug "extracting stored fields #{stored_fields.inspect} from document #{doc[:class_name]} / #{doc[:id]}"
+        fields = index_definition[:ferret_fields] 
+        stored_fields.each do |field|
+          if field_cfg = fields[field]
+            data[field_cfg[:via]] = doc[field]
+          end
+        end
+        logger.debug "done: #{data.inspect}"
+      end
+      return data 
+    end
+
+    protected
+
+    # returns a MultiIndex instance operating on a MultiReader
+    #def multi_index(model_classes)
+    #  model_classes.map!(&:constantize) if String === model_classes.first
+    #  model_classes.sort! { |a, b| a.name <=> b.name }
+    #  key = model_classes.inject("") { |s, clazz| s + clazz.name }
+    #  multi_config = index_definition[:ferret].dup
+    #  multi_config.delete :default_field  # we don't want the default field list of *this* class for multi_searching
+    #  ActsAsFerret::multi_indexes[key] ||= MultiIndex.new(model_classes, multi_config)
+    #end
+ 
+  end
+
+end

data/lib/more_like_this.rb

@@ -0,0 +1,217 @@
+module ActsAsFerret #:nodoc:
+
+    module MoreLikeThis
+
+      module InstanceMethods
+
+        # returns other instances of this class, which have similar contents
+        # like this one. Basically works like this: find out n most interesting
+        # (i.e. characteristic) terms from this document, and then build a
+        # query from those which is run against the whole index. Which terms
+        # are interesting is decided on variour criteria which can be
+        # influenced by the given options. 
+        #
+        # The algorithm used here is a quite straight port of the MoreLikeThis class
+        # from Apache Lucene.
+        #
+        # options are:
+        # :field_names : Array of field names to use for similarity search (mandatory)
+        # :min_term_freq => 2,  # Ignore terms with less than this frequency in the source doc.
+        # :min_doc_freq => 5,   # Ignore words which do not occur in at least this many docs
+        # :min_word_length => nil, # Ignore words shorter than this length (longer words tend to 
+        #                            be more characteristic for the document they occur in).
+        # :max_word_length => nil, # Ignore words if greater than this len.
+        # :max_query_terms => 25,  # maximum number of terms in the query built
+        # :max_num_tokens => 5000, # maximum number of tokens to examine in a single field
+        # :boost => false,         # when true, a boost according to the relative score of 
+        #                            a term is applied to this Term's TermQuery.
+        # :similarity => 'DefaultAAFSimilarity'   # the similarity implementation to use (the default 
+        #                                           equals Ferret's internal similarity implementation)
+        # :analyzer => 'Ferret::Analysis::StandardAnalyzer' # class name of the analyzer to use
+        # :append_to_query => nil # proc taking a query object as argument, which will be called after generating the query. can be used to further manipulate the query used to find related documents, i.e. to constrain the search to a given class in single table inheritance scenarios
+        # ferret_options : Ferret options handed over to find_with_ferret (i.e. for limits and sorting)
+        # ar_options : options handed over to find_with_ferret for AR scoping
+        def more_like_this(options = {}, ferret_options = {}, ar_options = {})
+          options = {
+            :field_names => nil,  # Default field names
+            :min_term_freq => 2,  # Ignore terms with less than this frequency in the source doc.
+            :min_doc_freq => 5,   # Ignore words which do not occur in at least this many docs
+            :min_word_length => 0, # Ignore words if less than this len. Default is not to ignore any words.
+            :max_word_length => 0, # Ignore words if greater than this len. Default is not to ignore any words.
+            :max_query_terms => 25,  # maximum number of terms in the query built
+            :max_num_tokens => 5000, # maximum number of tokens to analyze when analyzing contents
+            :boost => false,      
+            :similarity => 'ActsAsFerret::MoreLikeThis::DefaultAAFSimilarity',  # class name of the similarity implementation to use
+            :analyzer => 'Ferret::Analysis::StandardAnalyzer', # class name of the analyzer to use
+            :append_to_query => nil,
+            :base_class => self.class # base class to use for querying, useful in STI scenarios where BaseClass.find_with_ferret can be used to retrieve results from other classes, too
+          }.update(options)
+          #index.search_each('id:*') do |doc, score|
+          #  puts "#{doc} == #{index[doc][:description]}"
+          #end
+          clazz = options[:base_class]
+          options[:base_class] = clazz.name
+          query = clazz.aaf_index.build_more_like_this_query(self.ferret_key, self.id, options)
+          options[:append_to_query].call(query) if options[:append_to_query]
+          clazz.find_with_ferret(query, ferret_options, ar_options)
+        end
+
+      end
+
+      module IndexMethods
+
+        # TODO to allow morelikethis for unsaved records, we have to give the
+        # unsaved record's data to this method. check how this will work out
+        # via drb...
+        def build_more_like_this_query(key, id, options)
+          [:similarity, :analyzer].each { |sym| options[sym] = options[sym].constantize.new }
+          ferret_index.synchronize do # avoid that concurrent writes close our reader
+            ferret_index.send(:ensure_reader_open)
+            reader = ferret_index.send(:reader)
+            term_freq_map = retrieve_terms(key, id, reader, options)
+            priority_queue = create_queue(term_freq_map, reader, options)
+            create_query(key, priority_queue, options)
+          end
+        end
+
+        protected
+        
+        def create_query(key, priority_queue, options={})
+          query = Ferret::Search::BooleanQuery.new
+          qterms = 0
+          best_score = nil
+          while(cur = priority_queue.pop)
+            term_query = Ferret::Search::TermQuery.new(cur.field, cur.word)
+            
+            if options[:boost]
+              # boost term according to relative score
+              # TODO untested
+              best_score ||= cur.score
+              term_query.boost = cur.score / best_score
+            end
+            begin
+              query.add_query(term_query, :should) 
+            rescue Ferret::Search::BooleanQuery::TooManyClauses
+              break
+            end
+            qterms += 1
+            break if options[:max_query_terms] > 0 && qterms >= options[:max_query_terms]
+          end
+          # exclude the original record 
+          query.add_query(query_for_record(key), :must_not)
+          return query
+        end
+
+        
+
+        # creates a term/term_frequency map for terms from the fields
+        # given in options[:field_names]
+        def retrieve_terms(key, id, reader, options)
+          raise "more_like_this atm only works on saved records" if key.nil?
+          document_number = document_number(key) rescue nil
+          field_names = options[:field_names]
+          max_num_tokens = options[:max_num_tokens]
+          term_freq_map = Hash.new(0)
+          doc = nil
+          record = nil
+          field_names.each do |field|
+            #puts "field: #{field}"
+            term_freq_vector = reader.term_vector(document_number, field) if document_number
+            #if false
+            if term_freq_vector
+              # use stored term vector
+              # puts 'using stored term vector'
+              term_freq_vector.terms.each do |term|
+                term_freq_map[term.text] += term.positions.size unless noise_word?(term.text, options)
+              end
+            else
+              # puts 'no stored term vector'
+              # no term vector stored, but we have stored the contents in the index
+              # -> extract terms from there
+              content = nil
+              if document_number
+                doc = reader[document_number]
+                content = doc[field]
+              end
+              unless content
+                # no term vector, no stored content, so try content from this instance
+                record ||= options[:base_class].constantize.find(id)
+                content = record.content_for_field_name(field.to_s)
+              end
+              puts "have doc: #{doc[:id]} with #{field} == #{content}"
+              token_count = 0
+              
+              ts = options[:analyzer].token_stream(field, content)
+              while token = ts.next
+                break if (token_count+=1) > max_num_tokens
+                next if noise_word?(token.text, options)
+                term_freq_map[token.text] += 1
+              end
+            end
+          end
+          term_freq_map
+        end
+
+        # create an ordered(by score) list of word,fieldname,score 
+        # structures
+        def create_queue(term_freq_map, reader, options)
+          pq = Array.new(term_freq_map.size)
+          
+          similarity = options[:similarity]
+          num_docs = reader.num_docs
+          term_freq_map.each_pair do |word, tf|
+            # filter out words that don't occur enough times in the source
+            next if options[:min_term_freq] && tf < options[:min_term_freq]
+            
+            # go through all the fields and find the largest document frequency
+            top_field = options[:field_names].first
+            doc_freq = 0
+            options[:field_names].each do |field_name| 
+              freq = reader.doc_freq(field_name, word)
+              if freq > doc_freq 
+                top_field = field_name
+                doc_freq = freq
+              end
+            end
+            # filter out words that don't occur in enough docs
+            next if options[:min_doc_freq] && doc_freq < options[:min_doc_freq]
+            next if doc_freq == 0 # index update problem ?
+            
+            idf = similarity.idf(doc_freq, num_docs)
+            score = tf * idf
+            pq << FrequencyQueueItem.new(word, top_field, score)
+          end
+          pq.compact!
+          pq.sort! { |a,b| a.score<=>b.score }
+          return pq
+        end
+        
+        def noise_word?(text, options)
+          len = text.length
+          (
+            (options[:min_word_length] > 0 && len < options[:min_word_length]) ||
+            (options[:max_word_length] > 0 && len > options[:max_word_length]) ||
+            (options[:stop_words] && options.include?(text))
+          )
+        end
+
+      end
+
+      class DefaultAAFSimilarity
+        def idf(doc_freq, num_docs)
+          return 0.0 if num_docs == 0
+          return Math.log(num_docs.to_f/(doc_freq+1)) + 1.0
+        end
+      end
+
+
+      class FrequencyQueueItem
+        attr_reader :word, :field, :score
+        def initialize(word, field, score)
+          @word = word; @field = field; @score = score
+        end
+      end
+
+    end
+end
+