refinerycms 0.9.0

data/vendor/plugins/acts_as_indexed/lib/search_index.rb

@@ -0,0 +1,329 @@
+# ActsAsIndexed
+# Copyright (c) 2007 Douglas F Shearer.
+# http://douglasfshearer.com
+# Distributed under the MIT license as included with this plugin.
+
+module Foo #:nodoc:
+  module Acts #:nodoc:
+    module Indexed #:nodoc:
+      class SearchIndex
+
+        # root:: Location of index on filesystem.
+        # index_depth:: Degree of index partitioning.
+        # fields:: Fields or instance methods of ActiveRecord model to be indexed.
+        # min_word_size:: Smallest query term that will be run through search.
+        def initialize(root, index_depth, fields, min_word_size)
+          @root = root
+          @fields = fields
+          @index_depth = index_depth
+          @atoms = {}
+          @min_word_size = min_word_size
+          @records_size = exists? ? load_record_size : 0
+        end
+
+        # Adds +record+ to the index.
+        def add_record(record)
+          condensed_record = condense_record(record)
+          load_atoms(condensed_record)
+          add_occurences(condensed_record,record.id)
+          @records_size += 1
+        end
+
+        # Adds multiple records to the index. Accepts an array of +records+.
+        def add_records(records)
+          records.each do |r|
+            condensed_record = condense_record(r)
+            load_atoms(condensed_record)
+            add_occurences(condensed_record,r.id)
+            @records_size += 1
+          end
+        end
+
+        # Removes +record+ from the index.
+        def remove_record(record)
+          atoms = condense_record(record)
+          load_atoms(atoms)
+          atoms.each do |a|
+            @atoms[a].remove_record(record.id) if @atoms.has_key?(a)
+            @records_size -= 1
+            #p "removing #{record.id} from #{a}"
+          end
+        end
+
+        def update_record(record_new, record_old)
+          # Work out which atoms have modifications.
+          # Minimises loading and saving of partitions.
+          old_atoms = condense_record(record_old)
+          new_atoms = condense_record(record_new)
+
+          # Remove the old version from the appropriate atoms.
+          load_atoms(old_atoms)
+          old_atoms.each do |a|
+            @atoms[a].remove_record(record_new.id) if @atoms.has_key?(a)
+          end
+
+          # Add the new version to the appropriate atoms.
+          load_atoms(new_atoms)
+          # TODO: Make a version of this method that takes the
+          # atomised version of the record.
+          add_occurences(new_atoms, record_new.id)
+        end
+
+        # Saves the current index partitions to the filesystem.
+        def save
+          prepare
+          atoms_sorted = {}
+          @atoms.each do |atom_name, records|
+            e_p = encoded_prefix(atom_name)
+            atoms_sorted[e_p] = {} if !atoms_sorted.has_key?(e_p)
+            atoms_sorted[e_p][atom_name] = records
+          end
+          atoms_sorted.each do |e_p, atoms|
+            #p "Saving #{e_p}."
+            File.open(File.join(@root + [e_p.to_s]),'w+') do |f|
+              Marshal.dump(atoms,f)
+            end
+          end
+          save_record_size
+        end
+
+        # Deletes the current model's index from the filesystem.
+        #--
+        # TODO: Write a public method that will delete all indexes.
+        def destroy
+          FileUtils.rm_rf(@root)
+          true
+        end
+
+        # Returns an array of IDs for records matching +query+.
+        def search(query)
+          load_atoms(cleanup_atoms(query))
+          return [] if query.nil?
+          queries = parse_query(query.dup)
+          positive = run_queries(queries[:positive])
+          positive_quoted = run_quoted_queries(queries[:positive_quoted])
+          negative = run_queries(queries[:negative])
+          negative_quoted = run_quoted_queries(queries[:negative_quoted])
+          
+          if !queries[:positive].empty? && !queries[:positive_quoted].empty?
+            p = positive.delete_if{ |r_id,w| !positive_quoted.include?(r_id) }
+            pq = positive_quoted.delete_if{ |r_id,w| !positive.include?(r_id) }
+            results = p.merge(pq) { |r_id,old_val,new_val| old_val + new_val}
+          elsif !queries[:positive].empty?
+            results = positive
+          else
+            results = positive_quoted
+          end
+          
+          negative_results = (negative.keys + negative_quoted.keys)
+          results.delete_if { |r_id, w| negative_results.include?(r_id) }
+          #p results
+          results
+        end
+
+        # Returns true if the index root exists on the FS.
+        #--
+        # TODO: Make a private method called 'root_exists?' which checks for the root directory.
+        def exists?
+          File.exists?(File.join(@root))
+        end
+
+        private
+
+        # Gets the size file from the index.
+        def load_record_size
+          File.open(File.join(@root + ['size'])) do |f|
+            return (Marshal.load(f))
+          end
+        end
+
+        # Saves the size to the size file.
+        def save_record_size
+          File.open(File.join(@root + ['size']),'w+') do |f|
+            Marshal.dump(@records_size,f)
+          end
+        end
+
+        # Returns true if the given atom is present.
+        def include_atom?(atom)
+          @atoms.has_key?(atom)
+        end
+
+        # Returns true if all the given atoms are present.
+        def include_atoms?(atoms_arr)
+          atoms_arr.each do |a|
+            return false if !include_atom?(a)
+          end
+          true
+        end
+
+        # Returns true if the given record is present.
+        def include_record?(record_id)
+          @atoms.each do |atomname, atom|
+            return true if atom.include_record?(record_id)
+          end
+        end
+
+        def add_atom(atom)
+          @atoms[atom] = SearchAtom.new if !include_atom?(atom)
+        end
+
+        def add_occurences(condensed_record,record_id)
+          condensed_record.each_with_index do |atom, i|
+            add_atom(atom)
+            @atoms[atom].add_position(record_id, i)
+            #p "adding #{record.id} to #{atom}"
+          end
+        end
+
+        def encoded_prefix(atom)
+          prefix = atom[0,@index_depth]
+          if !@prefix_cache || !@prefix_cache.has_key?(prefix)
+            @prefix_cache = {} if !@prefix_cache
+            len = atom.length
+            if len > 1
+              @prefix_cache[prefix] = prefix.split(//).map{|c| encode_character(c)}.join('_')
+            else
+              @prefix_cache[prefix] = encode_character(atom)
+            end
+          end
+          @prefix_cache[prefix]
+        end
+        
+        # Allows compatibility with 1.8.6 which has no ord method.
+        def encode_character(char)
+          if @@has_ord ||= char.respond_to?(:ord)
+            char.ord.to_s
+          else
+            char[0]
+          end
+        end
+
+        def parse_query(s)
+
+          # Find -"foo bar".
+          negative_quoted = []
+          while neg_quoted = s.slice!(/-\"[^\"]*\"/)
+            negative_quoted << cleanup_atoms(neg_quoted)
+          end
+
+          # Find "foo bar".
+          positive_quoted = []
+          while pos_quoted = s.slice!(/\"[^\"]*\"/)
+            positive_quoted << cleanup_atoms(pos_quoted)
+          end
+
+          # Find -foo.
+          negative = []
+          while neg = s.slice!(/-[\S]*/)
+            negative << cleanup_atoms(neg).first
+          end
+
+          # Find +foo
+          positive = []
+          while pos = s.slice!(/\+[\S]*/)
+            positive << cleanup_atoms(pos).first
+          end
+
+          # Find all other terms.
+          positive += cleanup_atoms(s,true)
+
+          return {:negative_quoted => negative_quoted, :positive_quoted => positive_quoted, :negative => negative, :positive => positive}
+        end
+
+        def run_queries(atoms)
+          results = {}
+          atoms.uniq.each do |atom|
+            interim_results = {}
+            if include_atom?(atom)
+              
+              interim_results = @atoms[atom].weightings(@records_size)
+            end
+            if results.empty?
+              results = interim_results
+            else
+              rr = {}
+              interim_results.each do |r,w|
+                rr[r] = w + results[r] if results[r]
+              end
+              results = rr
+            end
+          end
+          #p results
+          results
+        end
+
+        def run_quoted_queries(quoted_atoms)
+          results = {}
+          quoted_atoms.each do |quoted_atom|
+            interim_results = {}
+            # Check the index contains all the required atoms.
+            # match_atom = first_word_atom
+            # for each of the others
+            #   return atom containing records + positions where current atom is preceded by following atom.
+            # end
+            # return records from final atom.
+            next if !include_atoms?(quoted_atom)
+            matches = @atoms[quoted_atom.first]
+            quoted_atom[1..-1].each do |atom_name|
+              matches = @atoms[atom_name].preceded_by(matches)
+            end
+            #results += matches.record_ids
+            
+            interim_results = matches.weightings(@records_size)
+            if results.empty?
+              results = interim_results
+            else
+              rr = {}
+              interim_results.each do |r,w|
+                rr[r] = w + results[r] if results[r]
+              end
+              #p results.class
+              results = rr
+            end
+            
+          end
+          return results
+        end
+
+        def load_atoms(atoms)
+          # Remove duplicates
+          # Remove atoms already in index.
+          # Calculate prefixes.
+          # Remove duplicates
+          atoms.uniq.reject{|a| include_atom?(a)}.collect{|a| encoded_prefix(a)}.uniq.each do |name|
+            if File.exists?(File.join(@root + [name.to_s]))
+              File.open(File.join(@root + [name.to_s])) do |f|
+                @atoms.merge!(Marshal.load(f))
+              end
+            end
+          end
+        end
+
+        def prepare
+          # Makes the RAILS_ROOT/index directory
+          Dir.mkdir(File.join(@root[0,2])) if !File.exists?(File.join(@root[0,2]))
+          # Makes the RAILS_ROOT/index/ENVIRONMENT directory
+          Dir.mkdir(File.join(@root[0,3])) if !File.exists?(File.join(@root[0,3]))
+          # Makes the RAILS_ROOT/index/ENVIRONMENT/CLASS directory
+          Dir.mkdir(File.join(@root)) if !File.exists?(File.join(@root))
+        end
+
+        def cleanup_atoms(s, limit_size=false, min_size = @min_word_size || 3)
+          atoms = s.downcase.gsub(/\W/,' ').squeeze(' ').split
+          return atoms if !limit_size
+          atoms.reject{|w| w.size < min_size}
+        end
+
+        def condense_record(record)
+          record_condensed = ''
+          @fields.each do |f|
+            record_condensed += ' ' + record.send(f).to_s if record.send(f)
+          end
+          cleanup_atoms(record_condensed)
+        end
+
+      end
+    end
+  end
+end

data/vendor/plugins/acts_as_indexed/lib/will_paginate_search.rb

@@ -0,0 +1,29 @@
+# WillPaginateSearch
+# Copyright (c) 2007 Douglas F Shearer.
+# http://douglasfshearer.com
+
+module WillPaginate
+
+  module Finder
+
+    module ClassMethods
+
+
+      def paginate_search(query, options)
+
+        page, per_page, total_entries = wp_parse_options(options)
+        
+        total_entries ||= find_with_index(query,{},{:ids_only => true}).size
+
+        returning WillPaginate::Collection.new(page, per_page, total_entries) do |pager|
+          options.update :offset => pager.offset, :limit => pager.per_page
+
+          options = options.delete_if {|key, value| [:page, :per_page].include?(key) }
+
+          pager.replace find_with_index(query, options)
+        end
+      end
+
+    end
+  end
+end

data/vendor/plugins/acts_as_indexed/test/abstract_unit.rb

@@ -0,0 +1,47 @@
+require 'test/unit'
+require 'fileutils'
+require 'rubygems'
+require 'active_record'
+require 'active_record/fixtures'
+require File.dirname(__FILE__) + '/../lib/acts_as_indexed'
+
+# Mock out the required environment variables.
+RAILS_ENV = 'test'
+RAILS_ROOT = Dir.pwd
+
+ActiveRecord::Base.logger = Logger.new(File.dirname(__FILE__) + '/test.log')
+ActiveRecord::Base.configurations = YAML::load(IO.read(File.dirname(__FILE__) + '/database.yml'))
+ActiveRecord::Base.establish_connection(ENV['DB'] || 'sqlite3')
+
+# Load Schema
+load(File.dirname(__FILE__) + '/schema.rb')
+
+# Load model.
+$LOAD_PATH.unshift(File.dirname(__FILE__) + '/fixtures/')
+
+class ActiveSupport::TestCase #:nodoc:
+  include ActiveRecord::TestFixtures
+  self.fixture_path = File.dirname(__FILE__) + '/fixtures/'
+  self.use_transactional_fixtures = true
+  self.use_instantiated_fixtures  = false
+  
+  def destroy_index
+    FileUtils::rm_rf(index_loc) if File.exists?(index_loc)
+    assert !File.exists?(index_loc)
+    true
+  end
+  
+  def build_index
+    # Makes a query to invoke the index build.
+    assert_equal [], Post.find_with_index('badger')
+    assert File.exists?(index_loc)
+    true
+  end
+  
+  protected
+  
+  def index_loc
+    File.join(RAILS_ROOT,'index')
+  end
+  
+end

data/vendor/plugins/acts_as_indexed/test/acts_as_indexed_test.rb

@@ -0,0 +1,102 @@
+require File.dirname(__FILE__) + '/abstract_unit'
+
+class ActsAsIndexedTest < ActiveSupport::TestCase
+  fixtures :posts
+
+  def setup
+    destroy_index
+  end
+
+  def teardown
+    destroy_index
+  end
+
+  def test_adds_to_index
+    original_post_count = Post.count
+    assert_equal [], Post.find_with_index('badger')
+    p = Post.new(:title => 'badger', :body => 'Thousands of them!')
+    assert p.save
+    assert_equal original_post_count+1, Post.count
+    assert_equal [p.id], Post.find_with_index('badger',{},{:ids_only => true})
+  end
+
+  def test_removes_from_index
+    original_post_count = Post.count
+    assert_equal [posts(:wikipedia_article_4).id], Post.find_with_index('album',{},{:ids_only => true})
+    assert Post.find(posts(:wikipedia_article_4).id).destroy
+    assert_equal [], Post.find_with_index('album',{},{:ids_only => true})
+    assert_equal original_post_count-1, Post.count
+  end
+
+  def test_search_returns_posts
+    Post.find_with_index('album').each do |p|
+      assert_equal Post, p.class
+    end
+  end
+
+  def test_search_returns_post_ids
+    Post.find_with_index('album',{},{:ids_only => true}).each do |pid|
+      assert p = Post.find(pid)
+      assert_equal Post, p.class
+    end
+  end
+
+  # After a portion of a record has been removed
+  # the portion removes should no longer be in the index.
+  def test_updates_index
+    p = Post.create(:title => 'A special title', :body => 'foo bar bla bla bla')
+    assert Post.find_with_index('title',{},{:ids_only => true}).include?(p.id)
+    p.update_attributes(:title => 'No longer special')
+    assert !Post.find_with_index('title',{},{:ids_only => true}).include?(p.id)
+  end
+
+  def test_simple_queries
+    assert_equal [5, 6],  Post.find_with_index('ship',{},{:ids_only => true}).sort
+    assert_equal [6],  Post.find_with_index('foo',{},{:ids_only => true})
+    assert_equal [6],  Post.find_with_index('foo ship',{},{:ids_only => true})
+    assert_equal [6],  Post.find_with_index('ship foo',{},{:ids_only => true})
+  end
+
+  def test_negative_queries
+    assert_equal [5, 6],  Post.find_with_index('crane',{},{:ids_only => true}).sort
+    assert_equal [5],  Post.find_with_index('crane -foo',{},{:ids_only => true})
+    assert_equal [5],  Post.find_with_index('-foo crane',{},{:ids_only => true})
+    assert_equal [],  Post.find_with_index('-foo',{},{:ids_only => true}) #Edgecase
+  end
+
+  def test_quoted_queries
+    assert_equal [5],  Post.find_with_index('"crane ship"',{},{:ids_only => true})
+    assert_equal [6],  Post.find_with_index('"crane big"',{},{:ids_only => true})
+    assert_equal [],  Post.find_with_index('foo "crane ship"',{},{:ids_only => true})
+    assert_equal [],  Post.find_with_index('"crane badger"',{},{:ids_only => true})
+  end
+
+  def test_negative_quoted_queries
+    assert_equal [6],  Post.find_with_index('crane -"crane ship"',{},{:ids_only => true})
+    assert_equal [],  Post.find_with_index('-"crane big"',{},{:ids_only => true}) # Edgecase
+  end
+
+  def test_find_options
+    all_results = Post.find_with_index('crane',{},{:ids_only => true})
+    first_result = Post.find_with_index('crane',{:limit => 1})
+
+    assert_equal 1, first_result.size
+    assert_equal all_results.first, first_result.first.id
+
+    second_result = Post.find_with_index('crane',{:limit => 1, :offset => 1})
+    assert_equal 1, second_result.size
+    assert_equal all_results[1], second_result.first.id
+
+  end
+
+  # When a atom already in a record is duplicated, it removes 
+  # all records with that same atom from the index.
+  def test_update_record_bug
+    assert_equal 2, Post.find_with_index('crane',{},{:ids_only => true}).size
+    p = Post.find(6)
+    assert p.update_attributes(:body => p.body + ' crane')
+    assert_equal 2, Post.find_with_index('crane',{},{:ids_only => true}).size
+    assert_equal 2, Post.find_with_index('ship',{},{:ids_only => true}).size
+  end
+  
+end