acts_as_ferret 0.3.1 → 0.4.0

data/lib/more_like_this.rb

@@ -1,15 +1,8 @@
-module FerretMixin
-  module Acts #:nodoc:
-    module ARFerret #:nodoc:
+module ActsAsFerret #:nodoc:
 
-      module MoreLikeThis
+    module MoreLikeThis
 
-        class DefaultAAFSimilarity
-          def idf(doc_freq, num_docs)
-            return 0.0 if num_docs == 0
-            return Math.log(num_docs.to_f/(doc_freq+1)) + 1.0
-          end
-        end
+      module InstanceMethods
 
         # returns other instances of this class, which have similar contents
         # like this one. Basically works like this: find out n most interesting
@@ -25,18 +18,16 @@ module FerretMixin
         # :field_names : Array of field names to use for similarity search (mandatory)
         # :min_term_freq => 2,  # Ignore terms with less than this frequency in the source doc.
         # :min_doc_freq => 5,   # Ignore words which do not occur in at least this many docs
-        # :min_word_length => nil, # Ignore words if less than this len (longer
-        # words tend to be more characteristic for the document they occur in).
+        # :min_word_length => nil, # Ignore words shorter than this length (longer words tend to 
+        #                            be more characteristic for the document they occur in).
         # :max_word_length => nil, # Ignore words if greater than this len.
         # :max_query_terms => 25,  # maximum number of terms in the query built
-        # :max_num_tokens => 5000, # maximum number of tokens to examine in a
-        # single field
-        # :boost => false,         # when true, a boost according to the
-        # relative score of a term is applied to this Term's TermQuery.
-        # :similarity => Ferret::Search::Similarity.default, # the similarity
-        # implementation to use
-        # :analyzer => Ferret::Analysis::StandardAnalyzer.new # the analyzer to
-        # use
+        # :max_num_tokens => 5000, # maximum number of tokens to examine in a single field
+        # :boost => false,         # when true, a boost according to the relative score of 
+        #                            a term is applied to this Term's TermQuery.
+        # :similarity => 'DefaultAAFSimilarity'   # the similarity implementation to use (the default 
+        #                                           equals Ferret's internal similarity implementation)
+        # :analyzer => 'Ferret::Analysis::StandardAnalyzer' # class name of the analyzer to use
         # :append_to_query => nil # proc taking a query object as argument, which will be called after generating the query. can be used to further manipulate the query used to find related documents, i.e. to constrain the search to a given class in single table inheritance scenarios
         # find_options : options handed over to find_by_contents
         def more_like_this(options = {}, find_options = {})
@@ -49,30 +40,39 @@ module FerretMixin
             :max_query_terms => 25,  # maximum number of terms in the query built
             :max_num_tokens => 5000, # maximum number of tokens to analyze when analyzing contents
             :boost => false,      
-            :similarity => DefaultAAFSimilarity.new,
-            :analyzer => Ferret::Analysis::StandardAnalyzer.new,
+            :similarity => 'ActsAsFerret::MoreLikeThis::DefaultAAFSimilarity',  # class name of the similarity implementation to use
+            :analyzer => 'Ferret::Analysis::StandardAnalyzer', # class name of the analyzer to use
             :append_to_query => nil,
             :base_class => self.class # base class to use for querying, useful in STI scenarios where BaseClass.find_by_contents can be used to retrieve results from other classes, too
           }.update(options)
-          index = self.class.ferret_index
           #index.search_each('id:*') do |doc, score|
           #  puts "#{doc} == #{index[doc][:description]}"
           #end
-          index.synchronize do # avoid that concurrent writes close our reader
-            index.send(:ensure_reader_open)
-            reader = index.send(:reader)
-            doc_number = self.document_number
-            term_freq_map = retrieve_terms(document_number, reader, options)
+          clazz = options[:base_class]
+          options[:base_class] = clazz.name
+          query = clazz.aaf_index.build_more_like_this_query(self.id, self.class.name, options)
+          options[:append_to_query].call(query) if options[:append_to_query]
+          clazz.find_by_contents(query, find_options)
+        end
+
+      end
+
+      module IndexMethods
+
+        def build_more_like_this_query(id, class_name, options)
+          [:similarity, :analyzer].each { |sym| options[sym] = options[sym].constantize.new }
+          ferret_index.synchronize do # avoid that concurrent writes close our reader
+            ferret_index.send(:ensure_reader_open)
+            reader = ferret_index.send(:reader)
+            term_freq_map = retrieve_terms(id, class_name, reader, options)
             priority_queue = create_queue(term_freq_map, reader, options)
-            query = create_query(priority_queue, options)
-            logger.debug "morelikethis-query: #{query}"
-            options[:append_to_query].call(query) if options[:append_to_query]
-            options[:base_class].find_by_contents(query, find_options)
+            create_query(id, class_name, priority_queue, options)
           end
         end
 
+        protected
         
-        def create_query(priority_queue, options={})
+        def create_query(id, class_name, priority_queue, options={})
           query = Ferret::Search::BooleanQuery.new
           qterms = 0
           best_score = nil
@@ -93,8 +93,8 @@ module FerretMixin
             qterms += 1
             break if options[:max_query_terms] > 0 && qterms >= options[:max_query_terms]
           end
-          # exclude ourselves
-          query.add_query(Ferret::Search::TermQuery.new(:id, self.id.to_s), :must_not)
+          # exclude the original record 
+          query.add_query(query_for_record(id, class_name), :must_not)
           return query
         end
 
@@ -102,11 +102,13 @@ module FerretMixin
 
         # creates a term/term_frequency map for terms from the fields
         # given in options[:field_names]
-        def retrieve_terms(doc_number, reader, options)
+        def retrieve_terms(id, class_name, reader, options)
+          document_number = document_number(id, class_name)
           field_names = options[:field_names]
           max_num_tokens = options[:max_num_tokens]
           term_freq_map = Hash.new(0)
           doc = nil
+          record = nil
           field_names.each do |field|
             #puts "field: #{field}"
             term_freq_vector = reader.term_vector(document_number, field)
@@ -125,7 +127,8 @@ module FerretMixin
               content = doc[field]
               unless content
                 # no term vector, no stored content, so try content from this instance
-                content = content_for_field_name(field.to_s)
+                record ||= options[:base_class].constantize.find(id)
+                content = record.content_for_field_name(field.to_s)
               end
               puts "have doc: #{doc[:id]} with #{field} == #{content}"
               token_count = 0
@@ -184,12 +187,16 @@ module FerretMixin
           )
         end
 
-        def content_for_field_name(field)
-          self[field] || self.instance_variable_get("@#{field.to_s}".to_sym) || self.send(field.to_sym)
-        end
+      end
 
+      class DefaultAAFSimilarity
+        def idf(doc_freq, num_docs)
+          return 0.0 if num_docs == 0
+          return Math.log(num_docs.to_f/(doc_freq+1)) + 1.0
+        end
       end
 
+
       class FrequencyQueueItem
         attr_reader :word, :field, :score
         def initialize(word, field, score)
@@ -198,6 +205,5 @@ module FerretMixin
       end
 
     end
-  end
 end
 

data/lib/multi_index.rb

@@ -1,15 +1,14 @@
-module FerretMixin
-  module Acts #:nodoc:
-    module ARFerret #:nodoc:
-      # not threadsafe
+module ActsAsFerret #:nodoc:
+  
+      # this class is not threadsafe
       class MultiIndex
         
-        # todo: check for necessary index rebuilds in this place, too
-        # idea - each class gets a create_reader method that does this
         def initialize(model_classes, options = {})
           @model_classes = model_classes
+          # ensure all models indexes exist
+          @model_classes.each { |m| m.aaf_index.ensure_index_exists }
           default_fields = @model_classes.inject([]) do |fields, c| 
-            fields + c.ferret_configuration[:default_field] 
+            fields + c.aaf_configuration[:ferret][:default_field] 
           end
           @options = { 
             :default_field => default_fields
@@ -68,9 +67,9 @@ module FerretMixin
             unless latest?
               @sub_readers = @model_classes.map { |clazz| 
                 begin
-                  reader = Ferret::Index::IndexReader.new(clazz.class_index_dir)
+                  reader = Ferret::Index::IndexReader.new(clazz.aaf_configuration[:index_dir])
                 rescue Exception
-                  puts "error opening #{clazz.class_index_dir}: #{$!}"
+                  puts "error opening #{clazz.aaf_configuration[:index_dir]}: #{$!}"
                 end
                 reader
               }
@@ -82,6 +81,4 @@ module FerretMixin
 
       end # of class MultiIndex
 
-    end
-  end
 end

data/lib/remote_index.rb

@@ -0,0 +1,50 @@
+require 'drb'
+module ActsAsFerret
+
+  # This index implementation connects to a remote ferret server instance. It
+  # basically forwards all calls to the remote server.
+  class RemoteIndex < AbstractIndex
+
+    def initialize(config)
+      @config = config
+      @ferret_config = config[:ferret]
+      @server = DRbObject.new(nil, config[:remote])
+    end
+
+    def method_missing(method_name, *args)
+      args.unshift model_class_name
+      @server.send(method_name, *args)
+    end
+
+    def find_id_by_contents(q, options = {}, &proc)
+      total_hits, results = @server.find_id_by_contents(model_class_name, q, options)
+      block_given? ? yield_results(total_hits, results, &proc) : [ total_hits, results ]
+    end
+
+    def id_multi_search(query, models, options, &proc)
+      total_hits, results = @server.id_multi_search(model_class_name, query, models, options)
+      block_given? ? yield_results(total_hits, results, &proc) : [ total_hits, results ]
+    end
+
+    # add record to index
+    def add(record)
+      @server.add record.class.name, record.to_doc
+    end
+    alias << add
+
+    private
+
+    def yield_results(total_hits, results)
+      results.each do |result|
+        yield result[:model], result[:id], result[:score], result[:data]
+      end
+      total_hits
+    end
+
+    def model_class_name
+      @config[:class_name]
+    end
+
+  end
+
+end

data/lib/shared_index.rb

@@ -0,0 +1,14 @@
+module ActsAsFerret
+  
+  class SharedIndex < LocalIndex
+
+    # build a ferret query matching only the record with the given id and class
+    def query_for_record(id, class_name)
+      returning bq = Ferret::Search::BooleanQuery.new do
+        bq.add_query(Ferret::Search::TermQuery.new(:id,         id.to_s),    :must)
+        bq.add_query(Ferret::Search::TermQuery.new(:class_name, class_name), :must)
+      end
+    end
+
+  end
+end

data/lib/shared_index_class_methods.rb

@@ -0,0 +1,90 @@
+module ActsAsFerret
+  
+  # class methods for classes using acts_as_ferret :single_index => true
+  module SharedIndexClassMethods
+
+    def find_id_by_contents(q, options = {}, &block)
+      # add class name scoping to query if necessary
+      unless options[:models] == :all # search needs to be restricted by one or more class names
+        options[:models] ||= [] 
+        # add this class to the list of given models
+        options[:models] << self unless options[:models].include?(self)
+        # keep original query 
+        original_query = q
+        
+        if original_query.is_a? String
+          model_query = options[:models].map(&:name).join '|'
+          q << %{ +class_name:"#{model_query}"}
+        else
+          q = Ferret::Search::BooleanQuery.new
+          q.add_query(original_query, :must)
+          model_query = Ferret::Search::BooleanQuery.new
+          options[:models].each do |model|
+            model_query.add_query(Ferret::Search::TermQuery.new(:class_name, model.name), :should)
+          end
+          q.add_query(model_query, :must)
+        end
+      end
+      options.delete :models
+      
+      super(q, options, &block)
+    end
+
+    # Overrides the standard find_by_contents for searching a shared index.
+    #
+    # please note that records from different models will be fetched in
+    # separate sql calls, so any sql order_by clause given with 
+    # find_options[:order] will be ignored.
+    def find_by_contents(q, options = {}, find_options = {})
+      if order = find_options.delete(:order)
+        logger.warn "using a shared index, so ignoring order_by clause #{order}"
+      end
+      total_hits, result = find_records_lazy_or_not q, options, find_options
+      # sort so results have the same order they had when originally retrieved
+      # from ferret
+      return SearchResults.new(result, total_hits)
+    end
+
+    protected
+
+    def ar_find_by_contents(q, options = {}, find_options = {})
+      total_hits, id_arrays = collect_results(q, options)
+      result = retrieve_records(id_arrays, find_options)
+      result.sort! { |a, b| id_arrays[a.class.name][a.id.to_s].first <=> id_arrays[b.class.name][b.id.to_s].first }
+      [ total_hits, result ]
+    end
+
+    def collect_results(q, options = {})
+      id_arrays = {}
+      # get object ids for index hits
+      rank = 0
+      total_hits = find_id_by_contents(q, options) do |model, id, score, data|
+        id_arrays[model] ||= {}
+        # store result rank and score
+        id_arrays[model][id] = [ rank += 1, score ]
+      end
+      [ total_hits, id_arrays ]
+    end
+
+    
+    # determine all field names in the shared index
+    # TODO unused
+#    def single_index_field_names(models)
+#      @single_index_field_names ||= (
+#          searcher = Ferret::Search::Searcher.new(class_index_dir)
+#          if searcher.reader.respond_to?(:get_field_names)
+#            (searcher.reader.send(:get_field_names) - ['id', 'class_name']).to_a
+#          else
+#            puts <<-END
+#unable to retrieve field names for class #{self.name}, please 
+#consider naming all indexed fields in your call to acts_as_ferret!
+#            END
+#            models.map { |m| m.content_columns.map { |col| col.name } }.flatten
+#          end
+#      )
+#
+#    end
+ 
+  end
+end
+

data/rakefile

@@ -1,7 +1,8 @@
-# rakefile for acts_as_ferret.
+# rakefile for acts_as_ferret. 
 # use to create a gem or generate rdoc api documentation.
 #
-# heavily based on the one from the acts_as_searchable plugin.
+# RELEASE creation:
+# rake release REL=x.y.z 
 
 require 'rake'
 require 'rake/rdoctask'
@@ -10,16 +11,21 @@ require 'rake/gempackagetask'
 require 'rake/testtask'
 require 'rake/contrib/rubyforgepublisher'
 
+def announce(msg='')
+  STDERR.puts msg
+end
+
+
 PKG_NAME      = 'acts_as_ferret'
-PKG_VERSION   = '0.3.1'
+PKG_VERSION   = ENV['REL']
 PKG_FILE_NAME = "#{PKG_NAME}-#{PKG_VERSION}"
-RUBY_FORGE_PROJECT = 'actsasferret'
-RUBY_FORGE_USER    = 'jkraemer'
+RUBYFORGE_PROJECT = 'actsasferret'
+RUBYFORGE_USER    = 'jkraemer'
 
 desc 'Default: run unit tests.'
 task :default => :test
 
-desc 'Test the acts_as_searchable plugin.'
+desc 'Test the acts_as_ferret plugin.'
 Rake::TestTask.new(:test) do |t|
   t.libs << 'lib'
   t.pattern = 'test/**/*_test.rb'
@@ -37,155 +43,90 @@ Rake::RDocTask.new(:rdoc) do |rdoc|
   rdoc.rdoc_files.include('lib/**/*.rb')
 end
 
-spec = Gem::Specification.new do |s|
-  s.name            = PKG_NAME
-  s.version         = PKG_VERSION
-  s.platform        = Gem::Platform::RUBY
-  s.summary         = "acts_as_ferret - Ferret based full text search for any ActiveRecord model"
-  s.files  = Dir.glob('**/*', File::FNM_DOTMATCH).reject do |f| 
-     [ /\.$/, /sqlite$/, /\.log$/, /^pkg/, /\.svn/,
-     /\~$/, /\/\._/, /\/#/ ].any? {|regex| f =~ regex }
-  end
-  #s.files           = FileList["{lib,test}/**/*"].to_a + %w(README MIT-LICENSE CHANGELOG)
- # s.files.delete  ...    
-  s.require_path    = 'lib'
-  s.autorequire     = 'acts_as_ferret'
-  s.has_rdoc        = true
- # s.test_files      = Dir['test/**/*_test.rb']
-  s.author          = "Jens Kraemer"
-  s.email           = "jk@jkraemer.net"
-  s.homepage        = "http://projects.jkraemer.net/acts_as_ferret"
-end
-
-Rake::GemPackageTask.new(spec) do |pkg|
-  pkg.need_tar = true
-end
-
 desc "Publish the API documentation"
 task :pdoc => [:rdoc] do
-  Rake::RubyForgePublisher.new(RUBY_FORGE_PROJECT, RUBY_FORGE_USER).upload
+  Rake::RubyForgePublisher.new(RUBYFORGE_PROJECT, RUBYFORGE_USER).upload
 end
 
-desc 'Publish the gem and API docs'
-task :publish => [:pdoc, :rubyforge_upload]
-
-desc "Publish the release files to RubyForge."
-task :rubyforge_upload => :package do
-  files = %w(gem tgz).map { |ext| "pkg/#{PKG_FILE_NAME}.#{ext}" }
-
-  if RUBY_FORGE_PROJECT then
-    require 'net/http'
-    require 'open-uri'
-
-    project_uri = "http://rubyforge.org/projects/#{RUBY_FORGE_PROJECT}/"
-    project_data = open(project_uri) { |data| data.read }
-    group_id = project_data[/[?&]group_id=(\d+)/, 1]
-    raise "Couldn't get group id" unless group_id
-
-    # This echos password to shell which is a bit sucky
-    if ENV["RUBY_FORGE_PASSWORD"]
-      password = ENV["RUBY_FORGE_PASSWORD"]
-    else
-      print "#{RUBY_FORGE_USER}@rubyforge.org's password: "
-      password = STDIN.gets.chomp
+if PKG_VERSION
+  spec = Gem::Specification.new do |s|
+    s.name            = PKG_NAME
+    s.version         = PKG_VERSION
+    s.platform        = Gem::Platform::RUBY
+    s.summary         = "acts_as_ferret - Ferret based full text search for any ActiveRecord model"
+    s.files  = Dir.glob('**/*', File::FNM_DOTMATCH).reject do |f| 
+      [ /\.$/, /sqlite$/, /\.log$/, /^pkg/, /\.svn/, /\.swp$/, /^html/, /\~$/, /\/\._/, /\/#/ ].any? {|regex| f =~ regex }
     end
+    #s.files           = FileList["{lib,test}/**/*"].to_a + %w(README MIT-LICENSE CHANGELOG)
+  # s.files.delete  ...    
+    s.require_path    = 'lib'
+    s.autorequire     = 'acts_as_ferret'
+    s.has_rdoc        = true
+  # s.test_files      = Dir['test/**/*_test.rb']
+    s.author          = "Jens Kraemer"
+    s.email           = "jk@jkraemer.net"
+    s.homepage        = "http://projects.jkraemer.net/acts_as_ferret"
+  end
 
-    login_response = Net::HTTP.start("rubyforge.org", 80) do |http|
-      data = [
-        "login=Login",
-        "form_loginname=#{RUBY_FORGE_USER}",
-        "form_pw=#{password}"
-      ].join("&")
-
-      headers = { 'Content-Type' => 'application/x-www-form-urlencoded' }
-      
-      http.post("/account/login.php", data, headers)
-    end
-
-    cookie = login_response["set-cookie"]
-    raise "Login failed" unless cookie
-    headers = { "Cookie" => cookie }
-
-    release_uri = "http://rubyforge.org/frs/admin/?group_id=#{group_id}"
-    release_data = open(release_uri, headers) { |data| data.read }
-    package_id = release_data[/[?&]package_id=(\d+)/, 1]
-    raise "Couldn't get package id" unless package_id
-
-    first_file = true
-    release_id = ""
-
-    files.each do |filename|
-      basename  = File.basename(filename)
-      file_ext  = File.extname(filename)
-      file_data = File.open(filename, "rb") { |file| file.read }
-
-      puts "Releasing #{basename}..."
-
-      release_response = Net::HTTP.start("rubyforge.org", 80) do |http|
-        release_date = Time.now.strftime("%Y-%m-%d %H:%M")
-        type_map = {
-          ".zip"    => "3000",
-          ".tgz"    => "3110",
-          ".gz"     => "3110",
-          ".gem"    => "1400"
-        }; type_map.default = "9999"
-        type = type_map[file_ext]
-        boundary = "rubyqMY6QN9bp6e4kS21H4y0zxcvoor"
-
-        query_hash = if first_file then
-          {
-            "group_id" => group_id,
-            "package_id" => package_id,
-            "release_name" => PKG_FILE_NAME,
-            "release_date" => release_date,
-            "type_id" => type,
-            "processor_id" => "8000", # Any
-            "release_notes" => "",
-            "release_changes" => "",
-            "preformatted" => "1",
-            "submit" => "1"
-          }
-        else
-          {
-            "group_id" => group_id,
-            "release_id" => release_id,
-            "package_id" => package_id,
-            "step2" => "1",
-            "type_id" => type,
-            "processor_id" => "8000", # Any
-            "submit" => "Add This File"
-          }
-        end
-
-        data = [
-          "--" + boundary,
-          "Content-Disposition: form-data; name=\"userfile\"; filename=\"#{basename}\"",
-          "Content-Type: application/octet-stream",
-          "Content-Transfer-Encoding: binary",
-          "", file_data, "",
-          query_hash.collect do |name, value|
-            [ "--" + boundary,
-              "Content-Disposition: form-data; name='#{name}'",
-              "", value, "" ]
-          end
-          ].flatten.join("\x0D\x0A")
-
-        release_headers = headers.merge(
-          "Content-Type" => "multipart/form-data; boundary=#{boundary}"
-        )
-
-        target = first_file ? "/frs/admin/qrs.php" : "/frs/admin/editrelease.php"
-        http.post(target, data, release_headers)
-      end
+  package_task = Rake::GemPackageTask.new(spec) do |pkg|
+    pkg.need_tar = true
+  end
 
-      if first_file then
-        release_id = release_response.body[/release_id=(\d+)/, 1]
-        raise("Couldn't get release id") unless release_id
+  # Validate that everything is ready to go for a release.
+  task :prerelease do
+    announce 
+    announce "**************************************************************"
+    announce "* Making RubyGem Release #{PKG_VERSION}"
+    announce "**************************************************************"
+    announce  
+    # Are all source files checked in?
+    if ENV['RELTEST']
+      announce "Release Task Testing, skipping checked-in file test"
+    else
+      announce "Pulling in svn..."
+      `svk pull .`
+      announce "Checking for unchecked-in files..."
+      data = `svk st`
+      unless data =~ /^$/
+        fail "SVK status is not clean ... do you have unchecked-in files?"
       end
-
-      first_file = false
+      announce "No outstanding checkins found ... OK"
+      announce "Pushing to svn..."
+      `svk push .`
     end
   end
+  
+  
+  desc "tag the new release"
+  task :tag => [ :prerelease ] do
+    reltag = "REL_#{PKG_VERSION.gsub(/\./, '_')}"
+    reltag << ENV['REUSE'].gsub(/\./, '_') if ENV['REUSE']
+    announce "Tagging with [#{PKG_VERSION}]"
+    if ENV['RELTEST']
+      announce "Release Task Testing, skipping tagging"
+    else
+      `svn copy -m 'tagging version #{PKG_VERSION}' svn://projects.jkraemer.net/acts_as_ferret/trunk/plugin svn://projects.jkraemer.net/acts_as_ferret/tags/#{PKG_VERSION}`
+      `svn del -m 'remove old stable' svn://projects.jkraemer.net/acts_as_ferret/tags/stable`
+      `svn copy -m 'tagging version #{PKG_VERSION} as stable' svn://projects.jkraemer.net/acts_as_ferret/tags/#{PKG_VERSION} svn://projects.jkraemer.net/acts_as_ferret/tags/stable`
+    end
+  end
+  
+  # Upload release to rubyforge
+  desc "Upload release to rubyforge"
+  #task :prel => [ :tag, :prerelease, :package ] do
+  task :prel => [ :package ] do
+  #task :prel do
+    `rubyforge login`
+    release_command = "rubyforge add_release #{RUBYFORGE_PROJECT} #{PKG_NAME} '#{PKG_VERSION}' pkg/#{PKG_NAME}-#{PKG_VERSION}.gem"
+    puts release_command
+    system(release_command)
+    `rubyforge config #{RUBYFORGE_PROJECT}`
+    release_command = "rubyforge add_file #{RUBYFORGE_PROJECT} #{PKG_NAME} '#{PKG_VERSION}' pkg/#{PKG_NAME}-#{PKG_VERSION}.tgz"
+    puts release_command
+    system(release_command)
+  end
+  
+  desc 'Publish the gem and API docs'
+  task :release => [:pdoc, :prel ]
+  
 end
-
-