xapit 0.1.0

data/lib/xapit/config.rb

@@ -0,0 +1,83 @@
+module Xapit
+  # Singleton class for storing Xapit configuration settings. Currently this only includes the database path.
+  class Config
+    class << self
+      attr_reader :options
+      
+      # Setup configuration options. The following options are supported.
+      # 
+      # <tt>:database_path</tt>:     Where the database is stored.
+      # <tt>:stemming</tt>:          The language to use for stemming, defaults to "english".
+      # <tt>:spelling</tt>:          True or false to enable/disable spelling, defaults to true.
+      # <tt>:indexer</tt>:           Class to handle the indexing, defaults to SimpleIndexer.
+      # <tt>:query_parser</tt>:      Class to handle the parsing, defaults to ClassicQueryParser.
+      # <tt>:breadcrumb_facets</tt>: Use breadcrumb mode for applied facets. See Collection#applied_facet_options for details.
+      #
+      def setup(options = {})
+        if @options && options[:database_path] != @options[:database_path]
+          @database = nil
+          @writable_database = nil
+        end
+        @options = options.reverse_merge(default_options)
+      end
+      
+      def default_options
+        {
+          :indexer => SimpleIndexer,
+          :query_parser => ClassicQueryParser,
+          :spelling => true,
+          :stemming => "english"
+        }
+      end
+      
+      # See if setup options are already set.
+      def setup?
+        @options
+      end
+      
+      # The configured path to the database.
+      def path
+        @options[:database_path]
+      end
+      
+      def query_parser
+        @options[:query_parser]
+      end
+      
+      def indexer
+        @options[:indexer]
+      end
+      
+      def spelling?
+        @options[:spelling]
+      end
+      
+      def stemming
+        @options[:stemming]
+      end
+      
+      def breadcrumb_facets?
+        @options[:breadcrumb_facets]
+      end
+      
+      # Fetch Xapian::Database object at configured path. Database is stored in memory.
+      def database
+        @writable_database || (@database ||= Xapian::Database.new(path))
+      end
+      
+      # Fetch Xapian::WritableDatabase object at configured path. Database is stored in memory.
+      # Creates the database directory if needed.
+      def writable_database
+        FileUtils.mkdir_p(File.dirname(path)) unless File.exist?(File.dirname(path))
+        @writable_database ||= Xapian::WritableDatabase.new(path, Xapian::DB_CREATE_OR_OPEN)
+      end
+      
+      # Removes the configured database file and clears the stored one in memory.
+      def remove_database # this can be a bit dangers, maybe do some checking here first?
+        FileUtils.rm_rf(path) if File.exist? path
+        @database = nil
+        @writable_database = nil
+      end
+    end
+  end
+end

data/lib/xapit/facet.rb

@@ -0,0 +1,59 @@
+module Xapit
+  # Facets allow users to further filter the result set based on certain attributes.
+  # You should fetch facets by calling "facets" on a Xapit::Collection search result.
+  # 
+  # <% for facet in @articles.facets %>
+  #   <%= facet.name %>
+  #   <% for option in facet.options %>
+  #     <%= link_to option.name, :overwrite_params => { :facets => option } %>
+  #     (<%= option.count %>)
+  #   <% end %>
+  # <% end %>
+  #
+  # See Xapit::FacetBlueprint for details on how to index a facet.
+  class Facet
+    attr_accessor :existing_facet_identifiers
+    
+    def initialize(blueprint, query, existing_facet_identifiers)
+      @blueprint = blueprint
+      @query = query.dup
+      @existing_facet_identifiers = existing_facet_identifiers
+    end
+    
+    # Xapit::FacetOption objects for this facet. This only lists the ones which match the current query.
+    def options
+      matching_identifiers.map do |identifier, count|
+        option = FacetOption.find(identifier)
+        option.count = count
+        option.existing_facet_identifiers = @existing_facet_identifiers
+        option
+      end.sort_by(&:name)
+    end
+    
+    def matching_identifiers
+      result = {}
+      matches.each do |match|
+        class_name, id = match.document.data.split('-')
+        record = class_name.constantize.find(id)
+        @blueprint.identifiers_for(record).each do |identifier|
+          unless existing_facet_identifiers.include? identifier
+            result[identifier] ||= 0
+            result[identifier] += (match.collapse_count + 1)
+          end
+        end
+      end
+      result
+    end
+    
+    # The name of the facet. See Xapit::FacetBlueprint for details.
+    def name
+      @blueprint.name
+    end
+    
+    private
+    
+    def matches
+      @query.matches(:offset => 0, :limit => 1000, :collapse_key => @blueprint.position)
+    end
+  end
+end

data/lib/xapit/facet_blueprint.rb

@@ -0,0 +1,59 @@
+module Xapit
+  # A facet blueprint keeps track of the settings for indexing a given facet. You can specify a custom
+  # name for a given facet by providing a second argument when defining.
+  #
+  #   xapit do |index|
+  #     index.facet :category_name, "Category"
+  #   end
+  #
+  # Multiple facet values are supported for a single record. All you need to do is return an array of
+  # values instead of a single string.
+  # 
+  #   def category_names
+  #     categories.map(&:name) # => ["Toys", "Clothing"]
+  #   end
+  # 
+  class FacetBlueprint
+    attr_reader :member_class
+    attr_reader :position
+    attr_reader :attribute
+    
+    def initialize(member_class, position, attribute, custom_name = nil)
+      @member_class = member_class
+      @position = position
+      @attribute = attribute
+      @custom_name = custom_name
+    end
+    
+    def identifiers_for(member)
+      values_for(member).map do |value|
+        Digest::SHA1.hexdigest(@attribute.to_s + value)[0..6]
+      end
+    end
+    
+    # The name of the facet. This will return the custom name if given while setting up the index,
+    # or default to humanizing the attribute name.
+    def name
+      @custom_name || @attribute.to_s.humanize
+    end
+    
+    def save_facet_options_for(member)
+      values_for(member).map do |value|
+        option = FacetOption.new(member.class.name, @attribute.to_s, value)
+        option.save
+      end
+    end
+    
+    private
+    
+    def values_for(member)
+      value = member.send(@attribute)
+      if value.kind_of? Array
+        value.map(&:to_s).reject(&:empty?)
+      else
+        [value].map(&:to_s).reject(&:empty?)
+      end
+    end
+  end
+end
+

data/lib/xapit/facet_option.rb

@@ -0,0 +1,56 @@
+module Xapit
+  # A facet option is a specific value or choice for a facet. See Xapit::Facet for details on how to use it.
+  class FacetOption
+    attr_accessor :facet, :name, :existing_facet_identifiers, :count
+    
+    # Fetch a facet option given an id.
+    def self.find(id)
+      match = Query.new("Q#{name}-#{id}").matches(:offset => 0, :limit => 1).first
+      if match.nil?
+        raise "Unable to find facet option for #{id}."
+      else
+        class_name, facet_attribute, name = match.document.data.split('|||')
+        new(class_name.to_s, facet_attribute.to_s, name.to_s)
+      end
+    end
+    
+    # See if the given facet option exists with this id.
+    def self.exist?(id)
+      Query.new("Q#{name}-#{id}").count >= 1
+    end
+    
+    def initialize(class_name, facet_attribute, name)
+      @facet = class_name.constantize.xapit_facet_blueprint(facet_attribute) if class_name && facet_attribute
+      @name = name
+    end
+    
+    def identifier
+      Digest::SHA1.hexdigest(facet.attribute.to_s + name)[0..6]
+    end
+    
+    # Saves the given facet option to the database if it hasn't been already.
+    def save
+      unless self.class.exist?(identifier)
+        doc = Xapian::Document.new
+        doc.data = [facet.member_class.name, facet.attribute, name].join("|||")
+        doc.add_term("Q#{self.class.name}-#{identifier}")
+        Xapit::Config.writable_database.add_document(doc)
+      end
+    end
+    
+    # Converts the facet to be used in a URL. It adds to the existing ones for convenience.
+    # If this facet option is currently selected, then this will return all selected facets except
+    # this one. This conveniently allows you to use this as both an "add this facet" and "remove this facet" link.
+    def to_param
+      if existing_facet_identifiers.include? identifier
+        if Xapit::Config.breadcrumb_facets?
+          existing_facet_identifiers[0..existing_facet_identifiers.index(identifier)].join('-')
+        else
+          (existing_facet_identifiers - [identifier]).join('-')
+        end
+      else
+        (existing_facet_identifiers + [identifier]).join('-')
+      end
+    end
+  end
+end

data/lib/xapit/index_blueprint.rb

@@ -0,0 +1,117 @@
+module Xapit
+  # This is the object used in the block of the xapit method in Xapit::Membership. It keeps track of the
+  # index settings for a given class. It also provides some indexing functionality.
+  class IndexBlueprint
+    attr_reader :text_attributes
+    attr_reader :field_attributes
+    attr_reader :sortable_attributes
+    attr_reader :facets
+    
+    # Indexes all classes known to have an index blueprint defined.
+    def self.index_all
+      load_models
+      @@instances.each do |member_class, blueprint|
+        yield(member_class) if block_given?
+        blueprint.index_all
+      end
+    end
+    
+    def initialize(member_class, *args)
+      @member_class = member_class
+      @args = args
+      @text_attributes = {}
+      @field_attributes = []
+      @sortable_attributes = []
+      @facets = []
+      @@instances ||= {}
+      @@instances[member_class] = self # TODO make this thread safe
+      @indexer = SimpleIndexer.new(self)
+    end
+    
+    # Adds a text attribute. Each word in the text will be indexed as a separate term allowing full text searching.
+    # Text terms are what is searched by the primary string in a search query.
+    #
+    #   Article.search("kite")
+    #
+    # You can specify a :weight option to give a text attribute more importance. This will cause search terms matching
+    # that attribute to have a higher rank. The default weight is 1. Decimal (0.5) weight values are not supported.
+    #
+    #   index.text :name, :weight => 10
+    # 
+    def text(*attributes, &proc)
+      options = attributes.extract_options!
+      options[:proc] ||= proc
+      attributes.each do |attribute|
+        @text_attributes[attribute] = options
+      end
+    end
+    
+    # Adds a field attribute. Field terms are not split by word so it is not designed for full text search.
+    # Instead you can filter through a field using the :conditions hash in a search query.
+    #
+    #   Article.search("", :conditions => { :priority => 5 })
+    # 
+    # Multiple field values are supported if the given attribute is an array.
+    # 
+    #   def priority
+    #     [3, 5] # will match priority search for 3 or 5
+    #   end
+    # 
+    def field(*attributes)
+      @field_attributes += attributes
+    end
+    
+    # Adds a facet attribute. See Xapit::FacetBlueprint and Xapit::Facet for details.
+    def facet(*args, &block)
+      @facets << FacetBlueprint.new(@member_class, @facets.size, *args, &block)
+    end
+    
+    # Adds a sortable attribute for use with the :order option in a search call.
+    def sortable(*attributes)
+      @sortable_attributes += attributes
+    end
+    
+    # Indexes all records of this blueprint class. It does this using the ".find_each" method on the member class.
+    # You will likely want to call Xapit::Config.remove_database before this.
+    def index_all
+      @member_class.find_each(*@args) do |member|
+        @indexer.add_member(member)
+      end
+    end
+    
+    def sortable_position_for(sortable_attribute)
+      index = sortable_attributes.map(&:to_s).index(sortable_attribute.to_s)
+      raise "Unable to find indexed sortable attribute \"#{sortable_attribute}\" in #{@member_class} sortable attributes: #{sortable_attributes.inspect}" if index.nil?
+      index + facets.size
+    end
+    
+    private
+    
+    # Make sure all models are loaded - without reloading any that
+    # ActiveRecord::Base is already aware of (otherwise we start to hit some
+    # messy dependencies issues).
+    # 
+    # Taken from thinking-sphinx
+    def self.load_models
+      if defined? Rails
+        base = "#{Rails.root}/app/models/"
+        Dir["#{base}**/*.rb"].each do |file|
+          model_name = file.gsub(/^#{base}([\w_\/\\]+)\.rb/, '\1')
+      
+          next if model_name.nil?
+          next if ::ActiveRecord::Base.send(:subclasses).detect { |model|
+            model.name == model_name
+          }
+      
+          begin
+            model_name.camelize.constantize
+          rescue LoadError
+            model_name.gsub!(/.*[\/\\]/, '').nil? ? next : retry
+          rescue NameError
+            next
+          end
+        end
+      end
+    end
+  end
+end

data/lib/xapit/indexers/abstract_indexer.rb

@@ -0,0 +1,101 @@
+module Xapit
+  class AbstractIndexer
+    def initialize(blueprint)
+      @blueprint = blueprint
+    end
+    
+    def add_member(member)
+      database.add_document(document_for(member))
+    end
+    
+    def document_for(member)
+      document = Xapian::Document.new
+      document.data = "#{member.class}-#{member.id}"
+      index_text_attributes(member, document)
+      index_terms(other_terms(member), document)
+      values(member).each_with_index do |value, index|
+        document.add_value(index, value)
+      end
+      save_facet_options_for(member)
+      document
+    end
+    
+    def index_terms(terms, document)
+      terms.each do |term|
+        document.add_term(term)
+        database.add_spelling(term) if Config.spelling?
+      end
+    end
+    
+    def index_text_attributes(member, document)
+      # to be overridden by subclass
+    end
+    
+    def other_terms(member)
+      base_terms(member) + field_terms(member) + facet_terms(member)
+    end
+    
+    def base_terms(member)
+      ["C#{member.class}", "Q#{member.class}-#{member.id}"]
+    end
+    
+    def field_terms(member)
+      @blueprint.field_attributes.map do |name|
+        [member.send(name)].flatten.map do |value|
+          if value.kind_of? Time
+            value = value.to_i
+          elsif value.kind_of? Date
+            value = value.to_time.to_i
+          end
+          "X#{name}-#{value.to_s.downcase}"
+        end
+      end.flatten
+    end
+    
+    def facet_terms(member)
+      @blueprint.facets.map do |facet|
+        facet.identifiers_for(member).map { |id| "F#{id}" }
+      end.flatten
+    end
+    
+    # used primarily by search similar functionality
+    def text_terms(member) # REFACTORME some duplicaiton with simple indexer
+      @blueprint.text_attributes.map do |name, options|
+        content = member.send(name).to_s
+        if options[:proc]
+          options[:proc].call(content).reject(&:blank?).map(&:to_s).map(&:downcase)
+        else
+          content.scan(/\w+/u).map(&:downcase)
+        end
+      end.flatten
+    end
+    
+    def values(member)
+      facet_values(member) + sortable_values(member)
+    end
+    
+    def sortable_values(member)
+      @blueprint.sortable_attributes.map do |sortable|
+        member.send(sortable).to_s.downcase
+      end
+    end
+    
+    def facet_values(member)
+      @blueprint.facets.map do |facet|
+        facet.identifiers_for(member).join("-")
+      end
+    end
+    
+    def save_facet_options_for(member)
+      @blueprint.facets.each do |facet|
+        facet.save_facet_options_for(member)
+      end
+    end
+    
+    private
+    
+    def database
+      Config.writable_database
+    end
+  end
+end