xapit 0.1.0

data/lib/xapit/indexers/classic_indexer.rb

@@ -0,0 +1,27 @@
+module Xapit
+  class ClassicIndexer < AbstractIndexer
+    def index_text_attributes(member, document)
+      term_generator.document = document
+      @blueprint.text_attributes.each do |name, options|
+        content = member.send(name).to_s
+        if options[:proc]
+          index_terms(options[:proc].call(content).reject(&:blank?).map(&:to_s).map(&:downcase), document)
+        else
+          term_generator.index_text(content)
+        end
+      end
+    end
+    
+    def term_generator
+      @term_generator ||= create_term_generator
+    end
+    
+    def create_term_generator
+      term_generator = Xapian::TermGenerator.new
+      term_generator.set_flags(Xapian::TermGenerator::FLAG_SPELLING, 0) if Config.spelling?
+      term_generator.database = database
+      term_generator.stemmer = Xapian::Stem.new(Config.stemming)
+      term_generator
+    end
+  end
+end

data/lib/xapit/indexers/simple_indexer.rb

@@ -0,0 +1,31 @@
+module Xapit
+  class SimpleIndexer < AbstractIndexer
+    def index_text_attributes(member, document)
+      @blueprint.text_attributes.map do |name, options|
+        terms_for_attribute(member, name, options).each do |term|
+          document.add_term(term, options[:weight] || 1)
+          database.add_spelling(term) if Config.spelling?
+        end
+      end
+    end
+    
+    def terms_for_attribute(member, name, options)
+      terms_for_attribute_without_stemming(member, name, options).map do |term|
+        [term, "Z#{stemmer.call(term)}"]
+      end.flatten
+    end
+    
+    def terms_for_attribute_without_stemming(member, name, options)
+      content = member.send(name).to_s
+      if options[:proc]
+        options[:proc].call(content).reject(&:blank?).map(&:to_s).map(&:downcase)
+      else
+        content.scan(/\w+/u).map(&:downcase)
+      end
+    end
+    
+    def stemmer
+      @stemmer ||= Xapian::Stem.new(Config.stemming)
+    end
+  end
+end

data/lib/xapit/membership.rb

@@ -0,0 +1,103 @@
+module Xapit
+  # Use "include Xapit::Membership" on a class to allow xapian searching on it. This is automatically included
+  # in ActiveRecord::Base so you do not need to do anything there.
+  module Membership
+    def self.included(base)
+      base.extend ClassMethods
+    end
+    
+    module ClassMethods
+      # Simply call "xapit" on a class and pass a block to define the indexed attributes.
+      # 
+      #   class Article < ActiveRecord::Base
+      #     xapit do |index|
+      #       index.text :name, :content
+      #       index.field :category_id
+      #       index.facet :author_name, "Author"
+      #       index.sortable :id, :category_id
+      #     end
+      #   end
+      # 
+      # First we index "name" and "content" attributes for full text searching. The "category_id" field is indexed for :conditions searching. The "author_name" is indexed as a facet with "Author" being the display name of the facet. See the facets section below for details. Finally the "id" and "category_id" attributes are indexed as sortable attributes so they can be included in the :order option in a search.
+      # 
+      # Because the indexing happens in Ruby these attributes do no have to be database columns. They can be simple Ruby methods. For example, the "author_name" attribute mentioned above can be defined like this.
+      # 
+      #   def author_name
+      #     author.name
+      #   end
+      # 
+      # This way you can create a completely custom facet by simply defining your own method
+      # 
+      # You can also pass any find options to the xapit method to determine what gets indexed and improve performance with eager loading or a different batch size.
+      # 
+      #   xapit(:batch_size => 100, :include => :author, :conditions => { :visible => true })
+      #
+      # If you pass in a block you can customize how the text words will be devided (instead of by simply white space).
+      # 
+      #   xapit do |index|
+      #     index.text(:keywords) { |words| words.split(', ') }
+      #   end
+      # 
+      # You can specify a :weight option to give a text attribute more importance. This will cause search terms matching
+      # that attribute to have a higher rank. The default weight is 1. Decimal (0.5) weight values are not supported.
+      #
+      #   index.text :name, :weight => 10
+      # 
+      def xapit(*args)
+        @xapit_index_blueprint = IndexBlueprint.new(self, *args)
+        yield(@xapit_index_blueprint)
+        include AdditionalMethods
+      end
+    end
+    
+    module AdditionalMethods
+      def self.included(base)
+        base.extend ClassMethods
+        base.send(:attr_accessor, :xapit_relevance) # is there a better way to do this?
+      end
+      
+      # Find similar records to the given model. It takes the same arguments as Membership::AdditionalMethods::ClassMethods#search to further narrow down the results.
+      def search_similar(*args)
+        Collection.search_similar(self, *args)
+      end
+    
+      module ClassMethods
+        # Used to perform a search on a model.
+        #   
+        #   # perform a simple full text search
+        #   @articles = Article.search("phone")
+        #   
+        #   # add pagination if you're using will_paginate
+        #   @articles = Article.search("phone", :per_page => 10, :page => params[:page])
+        #   
+        #   # search based on indexed fields
+        #   @articles = Article.search("phone", :conditions => { :category_id => params[:category_id] })
+        #   
+        #   # manually sort based on any number of indexed fields, sort defaults to most relevant
+        #   @articles = Article.search("phone", :order => [:category_id, :id], :descending => true)
+        #
+        #   # basic boolean matching is supported
+        #   @articles = Article.search("phone or fax not email")
+        #
+        #   # no need to specify first query string when searching all records
+        #   @articles = Article.search(:conditions => { :category_id => params[:category_id] })
+        #
+        def search(*args)
+          Collection.new(self, *args)
+        end
+      
+        # The Xapit::IndexBlueprint object used for this class.
+        def xapit_index_blueprint
+          @xapit_index_blueprint
+        end
+      
+        # Finds a Xapit::FacetBlueprint for the given attribute.
+        def xapit_facet_blueprint(attribute)
+          result = xapit_index_blueprint.facets.detect { |f| f.attribute.to_s == attribute.to_s }
+          raise "Unable to find facet blueprint for #{attribute} on #{name}" if result.nil?
+          result
+        end
+      end
+    end
+  end
+end

data/lib/xapit/query.rb

@@ -0,0 +1,62 @@
+module Xapit
+  # This class wraps a Xapian::Query for convenience purposes. You will likely not need to use
+  # this class unless you are trying to query the Xapian database directly.
+  # You may be looking for Xapit::Collection instead.
+  class Query
+    attr_reader :default_options, :xapian_query
+    
+    def initialize(query)
+      @xapian_query = build_xapian_query(query)
+      @default_options = { :offset => 0, :sort_descending => false }
+    end
+    
+    def and_query(query)
+      @xapian_query = Xapian::Query.new(Xapian::Query::OP_AND, @xapian_query, build_xapian_query(query)) unless query.blank?
+      self
+    end
+    
+    def or_query(query)
+      @xapian_query = Xapian::Query.new(Xapian::Query::OP_OR, @xapian_query, build_xapian_query(query)) unless query.blank?
+      self
+    end
+    
+    def not_query(query)
+      @xapian_query = Xapian::Query.new(Xapian::Query::OP_AND_NOT, @xapian_query, build_xapian_query(query)) unless query.blank?
+      self
+    end
+    
+    def matchset(options = {})
+      options.reverse_merge!(default_options)
+      enquire = Xapian::Enquire.new(Config.database)
+      if options[:sort_by_values]
+        sorter = Xapian::MultiValueSorter.new
+        options[:sort_by_values].each do |sort_value|
+          sorter.add(sort_value, !!options[:sort_descending])
+        end
+        enquire.set_sort_by_key_then_relevance(sorter)
+      end
+      enquire.collapse_key = options[:collapse_key] if options[:collapse_key]
+      enquire.query = @xapian_query
+      enquire.mset(options[:offset], options[:limit])
+    end
+    
+    def matches(options = {})
+      matchset(options).matches
+    end
+    
+    def count
+      # a bit of a hack to get more accurate count estimate
+      matchset(:limit => Config.database.doccount).matches_estimated
+    end
+    
+    private
+    
+    def build_xapian_query(query)
+      if query.kind_of? Xapian::Query
+        query
+      else
+        Xapian::Query.new(Xapian::Query::OP_AND, [query].flatten)
+      end
+    end
+  end
+end

data/lib/xapit/query_parsers/abstract_query_parser.rb

@@ -0,0 +1,115 @@
+module Xapit
+  class AbstractQueryParser
+    attr_reader :member_class
+    attr_writer :base_query
+    
+    def initialize(*args)
+      @options = args.extract_options!
+      @member_class = args[0]
+      @search_text = args[1].to_s
+    end
+    
+    def query
+      if (@search_text.split + condition_terms + facet_terms).empty?
+        base_query
+      else
+        @query ||= base_query.and_query(xapian_query_from_text(@search_text)).and_query(condition_terms + facet_terms)
+      end
+    end
+    
+    def current_page
+      @options[:page] ? @options[:page].to_i : 1
+    end
+    
+    def per_page
+      @options[:per_page] ? @options[:per_page].to_i : 20
+    end
+    
+    def offset
+      per_page*(current_page-1)
+    end
+    
+    def sort_by_values
+      if @options[:order] && @member_class
+        index = @member_class.xapit_index_blueprint
+        if @options[:order].kind_of? Array
+          @options[:order].map do |attribute|
+            index.sortable_position_for(attribute)
+          end
+        else
+          [index.sortable_position_for(@options[:order])]
+        end
+      end
+    end
+    
+    def base_query
+      @base_query ||= initial_query
+    end
+    
+    def initial_query
+      query = Query.new(Xapian::Query.new(Xapian::Query::OP_OR, initial_query_strings))
+      query.default_options[:offset] = offset
+      query.default_options[:limit] = per_page
+      query.default_options[:sort_by_values] = sort_by_values
+      query.default_options[:sort_descending] = @options[:descending]
+      query
+    end
+    
+    def initial_query_strings
+      if classes.empty?
+        [""]
+      else
+        classes.map { |klass| "C#{klass.name}" }
+      end
+    end
+    
+    def classes
+      (@options[:classes] || [@member_class]).compact
+    end
+    
+    def condition_terms
+      if @options[:conditions]
+        @options[:conditions].map do |name, value|
+          if value.kind_of? Time
+            value = value.to_i
+          elsif value.kind_of? Date
+            value = value.to_time.to_i
+          end
+          "X#{name}-#{value.to_s.downcase}"
+        end
+      else
+        []
+      end
+    end
+    
+    def facet_terms
+      if @options[:facets]
+        facet_identifiers.map do |identifier|
+          "F#{identifier}"
+        end
+      else
+        []
+      end
+    end
+    
+    def facet_identifiers
+      @options[:facets].kind_of?(String) ? @options[:facets].split('-') : (@options[:facets] || [])
+    end
+    
+    def spelling_suggestion
+      raise "Spelling has been disabled. Enable spelling in Xapit::Config.setup." unless Config.spelling?
+      if @search_text.downcase.scan(/\w+/).all? { |term| Config.database.get_spelling_suggestion(term).empty? }
+        nil
+      else
+        @search_text.downcase.gsub(/\w+/) do |term|
+          suggestion = Config.database.get_spelling_suggestion(term)
+          if suggestion.blank?
+            term
+          else
+            suggestion
+          end
+        end
+      end
+    end
+  end
+end

data/lib/xapit/query_parsers/classic_query_parser.rb

@@ -0,0 +1,19 @@
+module Xapit
+  class ClassicQueryParser < AbstractQueryParser
+    def xapian_query_from_text(text)
+      xapian_parser.parse_query(text)
+    end
+    
+    def xapian_parser
+      @xapian_parser ||= build_xapian_parser
+    end
+    
+    def build_xapian_parser
+      parser = Xapian::QueryParser.new
+      parser.stemmer = Xapian::Stem.new(Config.stemming)
+      parser.stemming_strategy = Xapian::QueryParser::STEM_SOME
+      parser.default_op = Xapian::Query::OP_AND
+      parser
+    end
+  end
+end

data/lib/xapit/query_parsers/simple_query_parser.rb

@@ -0,0 +1,75 @@
+module Xapit
+  class SimpleQueryParser < AbstractQueryParser
+    # REFACTORME this is a bit complex for one method...
+    def xapian_query(instructions = nil)
+      instructions ||= parsed
+      instructions = [:add, instructions] if instructions.kind_of? String
+      operator = (instructions.first == :or ? Xapian::Query::OP_OR : Xapian::Query::OP_AND)
+      words = instructions[1..-1].select { |i| i.kind_of? String }
+      query = Xapian::Query.new(operator, words) unless words.empty?
+      instructions[1..-1].select { |i| i.kind_of? Array }.each do |sub_instructions|
+        if sub_instructions.first == :not
+          sub_operator = Xapian::Query::OP_AND_NOT
+        else
+          sub_operator = operator
+        end
+        if query
+          query = Xapian::Query.new(sub_operator, query, xapian_query(sub_instructions))
+        else
+          query = xapian_query(sub_instructions)
+        end
+      end
+      query
+    end
+    
+    def parsed
+      parse(@search_text.downcase)
+    end
+    
+    def xapian_query_from_text(text)
+      xapian_query(parse(text.downcase))
+    end
+    
+    private
+    
+    
+    def parse(text)
+      if text.kind_of? Array
+        [:and, *text]
+      else
+        text = text.strip
+        if text =~ /\sor\s/ui
+          [:or, *text.split(/\s+or\s+/ui).map { |t| parse(t) }]
+        elsif text =~ /\s+/u
+          words = text.scan(/(?:\bnot\s+)?[^\s]+/ui)
+          words.map! do |word|
+            if Config.stemming
+              if word =~ /^not\s/ui
+                [:not, "Z" + stemmer.call(word.sub(/^not\s+/ui, ''))]
+              else
+                "Z" + stemmer.call(word)
+              end
+            else
+              if word =~ /^not\s/ui
+                [:not, word.sub(/^not\s+/ui, '')]
+              else
+                word
+              end
+            end
+          end
+          [:and, *words]
+        else
+          if Config.stemming && !text.blank?
+            "Z" + stemmer.call(text)
+          else
+            text
+          end
+        end
+      end
+    end
+    
+    def stemmer
+      @stemmer ||= Xapian::Stem.new(Config.stemming)
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,15 @@
+require 'rubygems'
+require 'spec'
+require 'active_support'
+require 'fileutils'
+require File.dirname(__FILE__) + '/../lib/xapit'
+require File.dirname(__FILE__) + '/xapit_member'
+
+Spec::Runner.configure do |config|
+  config.mock_with :rr
+  config.before(:each) do
+    Xapit::Config.setup(:database_path => File.dirname(__FILE__) + '/tmp/xapiandb')
+    Xapit::Config.remove_database
+    XapitMember.delete_all
+  end
+end