xapit 0.1.0

data/features/finding.feature

@@ -0,0 +1,119 @@
+Background:
+  Given an empty database at "tmp/xapiandatabase"
+
+Scenario: Query Matching No Records
+  Given indexed records named "John, Jane"
+  When I query for "Sam"
+  Then I should find 0 records
+
+Scenario: Query Matching One Record
+  Given indexed records named "John, Jane"
+  When I query for "John"
+  Then I should find record named "John"
+
+Scenario: Query Matching Two Records
+  Given indexed records named "John Smith, Jane Smith, John Smithsonian"
+  When I query for "Smith"
+  Then I should find records named "John Smith, Jane Smith"
+
+Scenario: Query Field Matching
+  Given the following indexed records
+    | name | age |
+    | John | 23  |
+    | Jane | 17  |
+    | Jack | 17  |
+  When I query "age" matching "17"
+  Then I should find records named "Jane, Jack"
+
+Scenario: Query for Page 1
+  Given 3 indexed records
+  When I query page 1 at 2 per page
+  Then I should find 2 records
+
+Scenario: Query for Page 2
+  Given 3 indexed records
+  When I query page 2 at 2 per page
+  Then I should find 1 record
+  And I should have 3 records total
+
+Scenario: Query for One Facet
+  Given the following indexed records
+    | name | age |
+    | John | 23  |
+    | Jane | 17  |
+    | Jack | 17  |
+  When I query facets "0c93ee1"
+  Then I should find records named "Jane, Jack"
+
+
+Scenario: Query for Two Facets
+  Given the following indexed records
+    | name | age |
+    | John | 23  |
+    | Jane | 17  |
+    | Jack | 17  |
+  When I query facets "0c93ee1-078661c"
+  Then I should find records named "Jane"
+
+Scenario: Query for All Records Class Agnostic
+  Given indexed records named "John, Jane"
+  When I query for "John" on Xapit
+  Then I should find 1 record
+
+Scenario: Query Matching Or Query
+  Given indexed records named "John, Jane, Jacob"
+  When I query for "Jane OR John"
+  Then I should find records named "John, Jane"
+
+Scenario: Query Matching Not Query
+  Given indexed records named "John Smith, John Johnson"
+  When I query for "John NOT Smith"
+  Then I should find records named "John Johnson"
+
+Scenario: Query for Facets with Keywords
+  Given the following indexed records
+    | name | age |
+    | John | 23  |
+    | Jane | 17  |
+    | Jack | 17  |
+  When I query "Jane" with facets "0c93ee1"
+  Then I should find record named "Jane"
+
+Scenario: Query for All Records Sorted by Name
+  Given indexed records named "Zebra, Apple, Banana"
+  When I query "" sorted by name
+  Then I should find records named "Apple, Banana, Zebra"
+
+Scenario: Query for All Records Sorted by Age then Name
+  Given the following indexed records
+    | name   | age |
+    | Banana | 23  |
+    | Zebra  | 17  |
+    | Apple  | 17  |
+  When I query "" sorted by age, name
+  Then I should find records named "Apple, Zebra, Banana"
+
+Scenario: Query for All Records Sorted by Name Descending
+  Given indexed records named "Zebra, Apple, Banana"
+  When I query "" sorted by name descending
+  Then I should find records named "Zebra, Banana, Apple"
+
+Scenario: Spelling suggestion
+  Given indexed records named "Zebra, Apple, Bike"
+  When I query for "zerba bike aple"
+  Then I should have "zebra bike apple" as a spelling suggestion
+
+Scenario: Match similar words with stemming
+  Given indexed records named "flies, fly, glider"
+  When I query for "flying"
+  Then I should find records named "flies, fly"
+
+Scenario: Find similar records
+  Given indexed records named "Jason John Smith, John Doe, Jason Smith, Jacob Johnson"
+  When I query for similar records for "Jason John Smith"
+  Then I should find records named "Jason Smith, John Doe"
+
+Scenario: Unicode characters in search
+  Given indexed records named "über cool, uber hot"
+  When I query for "über"
+  Then I should find records named "über cool"

data/features/indexing.feature

@@ -0,0 +1,41 @@
+Scenario: Save xapian database on index
+  Given no file exists at "tmp/xapiandb"
+  And I configured the database to be saved at "tmp/xapiandb"
+  And 3 records
+  When I index the database
+  Then I should find a directory at "tmp/xapiandb"
+
+Scenario: Fetch all records which are indexed
+  Given an empty database at "tmp/xapiandb"
+  And records named "John, Jane, Joe"
+  When I index the database
+  And I query for ""
+  Then I should find records named "John, Jane, Joe"
+
+Scenario: Split indexed text fields differently
+  Given an empty database at "tmp/xapiandb"
+  And records named "JohnXSmith, JaneXSmith, JoeXBlack"
+  When I index the database splitting name by "X"
+  And I query for "Smith"
+  Then I should find records named "JohnXSmith, JaneXSmith"
+
+Scenario: Index Multiple Field Values Separately
+  Given an empty database at "tmp/xapiandatabase"
+  And the following indexed records
+    | name | age    |
+    | John | 17, 16 |
+    | Jack | 17     |
+    | Jane | 16     |
+  When I query "age" matching "16"
+  Then I should find records named "Jane, John"
+
+Scenario: Index Weighted Attributes
+  Given an empty database at "tmp/xapiandatabase"
+  And the following indexed records with "name" weighted by "10"
+    | name | description |
+    | foo  | bar         |
+    | bar  | foo         |
+  When I query for "bar"
+  Then I should find records named "bar, foo"
+  When I query for "foo"
+  Then I should find records named "foo, bar"

data/features/step_definitions/common_steps.rb

@@ -0,0 +1,7 @@
+Given /^no file exists at "([^\"]*)"$/ do |path|
+  FileUtils.rm_rf(File.dirname(__FILE__) + "/../../#{path}") if File.exist? path
+end
+
+Then /^I should find a directory at "([^\"]*)"$/ do |path|
+  File.exist?(File.dirname(__FILE__) + "/../../#{path}").should be_true
+end

data/features/step_definitions/xapit_steps.rb

@@ -0,0 +1,117 @@
+Given /^I configured the database to be saved at "([^\"]*)"$/ do |path|
+  Xapit::Config.setup(:database_path => File.dirname(__FILE__) + "/../../#{path}")
+end
+
+Given /^an empty database at "([^\"]*)"$/ do |path|
+  Xapit::Config.setup(:database_path => File.dirname(__FILE__) + "/../../#{path}")
+  Xapit::Config.remove_database
+  XapitMember.delete_all
+end
+
+Given /^(indexed )?records? named "([^\"]*)"$/ do |indexed, joined_names|
+  records = joined_names.split(', ').map { |name| {:name => name} }
+  create_records(records, indexed)
+end
+
+Given /^([0-9]+) (indexed )?records?$/ do |num, indexed|
+  create_records([:name => "foo"]*num.to_i, indexed)
+end
+
+Given /^the following indexed records$/ do |records_table|
+  create_records(records_table.hashes)
+end
+
+Given /^the following indexed records with "([^\"]*)" weighted by "([^\"]*)"$/ do |weight_name, weight_value, records_table|
+  create_records(records_table.hashes) do |index, attribute|
+    if attribute.to_s == weight_name
+      index.text attribute, :weight => weight_value.to_i
+    else
+      index.text attribute
+    end
+  end
+end
+
+When /^I index the database$/ do
+  Xapit.index_all
+end
+
+When /^I index the database splitting name by "([^\"]*)"$/ do |divider|
+  XapitMember.xapit do |index|
+    index.text(:name) { |name| name.split(divider) }
+  end
+  Xapit.index_all
+end
+
+When /^I query for "([^\"]*)"$/ do |query|
+  @records = XapitMember.search(query)
+end
+
+When /^I query for "([^\"]*)" on Xapit$/ do |query|
+  @records = Xapit.search(query)
+end
+
+When /^I query "([^\"]*)" with facets "([^\"]*)"$/ do |keywords, facets|
+  @records = XapitMember.search(keywords, :facets => facets)
+end
+
+Then /^I should find records? named "([^\"]*)"$/ do |joined_names|
+  @records.map(&:name).join(", ").should == joined_names
+end
+
+Then /^I should find ([0-9]+) records?$/ do |num|
+  @records.should have(num.to_i).records
+end
+
+Then /^I should have ([0-9]+) records? total$/ do |num|
+  @records.total_entries.should == num.to_i
+end
+
+When /^I query "([^\"]*)" matching "([^\"]*)"$/ do |field, value|
+  @records = XapitMember.search("", :conditions => { field.to_sym => value })
+end
+
+When /^I query page ([0-9]+) at ([0-9]+) per page$/ do |page, per_page|
+  @records = XapitMember.search("", :page => page, :per_page => per_page.to_i)
+end
+
+When /^I query facets "([^\"]*)"$/ do |facets|
+  @records = XapitMember.search("", :facets => facets)
+end
+
+When /^I query "([^\"]*)" sorted by (.*?)( descending)?$/ do |keywords, sort, descending|
+  @records = XapitMember.search("", :order => sort.split(', '), :descending => descending)
+end
+
+When /^I query for similar records for "([^\"]*)"$/ do |keywords|
+  @records = XapitMember.search(keywords).first.search_similar
+end
+
+Then /^I should have the following facets$/ do |facets_table|
+  result = []
+  @records.facets.each do |facet|
+    facet.options.each do |option|
+      result << {
+        "facet" => facet.name,
+        "option" => option.name,
+        "count" => option.count.to_s
+      }
+    end
+  end
+  result.should == facets_table.hashes # this is somewhat fragile because it depends on order of hash result
+end
+
+Then /^I should have the following applied facets$/ do |facets_table|
+  result = []
+  @records.applied_facet_options.each do |option|
+    result << {
+      "facet" => option.facet.name,
+      "option" => option.name
+    }
+  end
+  result.should == facets_table.hashes # this is somewhat fragile because it depends on order of hash result
+end
+
+Then /^I should have "([^\"]*)" as a spelling suggestion$/ do |term|
+  @records.spelling_suggestion.should == term
+end
+

data/features/support/env.rb

@@ -0,0 +1,7 @@
+require 'cucumber'
+require 'spec'
+require 'active_support'
+require 'fileutils'
+
+require File.dirname(__FILE__) + '/../../lib/xapit'
+require File.dirname(__FILE__) + '/../../spec/xapit_member'

data/features/support/xapit_helpers.rb

@@ -0,0 +1,27 @@
+module XapitHelpers
+  def create_records(records,  perform_index = true)
+    Xapit::Config.remove_database
+    XapitMember.delete_all
+    XapitMember.xapit do |index|
+      records.first.keys.each do |attribute|
+        if block_given?
+          yield(index, attribute)
+        else
+          index.text attribute
+          index.field attribute
+          index.facet attribute
+          index.sortable attribute
+        end
+      end
+    end
+    records.each do |attributes|
+      attributes.each do |key, value|
+        attributes[key] = value.split(', ') if value.include? ', '
+      end
+      XapitMember.new(attributes.symbolize_keys)
+    end
+    Xapit.index_all if perform_index
+  end
+end
+
+World(XapitHelpers)

data/init.rb

@@ -0,0 +1,3 @@
+ActiveRecord::Base.class_eval do
+  include Xapit::Membership
+end

data/install.rb

@@ -0,0 +1,9 @@
+path = "#{Rails.root}/config/initializers/setup_xapit.rb"
+unless File.exist? path
+  puts "Adding setup_xapit.rb initializer."
+  File.open(path, "w") do |f|
+    f.write <<-EOS
+Xapit::Config.setup(:database_path => "\#{Rails.root}/db/xapiandb")
+EOS
+  end
+end

data/lib/xapit.rb

@@ -0,0 +1,39 @@
+require 'digest/sha1'
+require 'rubygems'
+require 'xapian'
+
+# Looking for more documentation? A good place to start is Xapit::Membership
+module Xapit
+  
+  # Index all membership classes with xapit defined. Delegates to Xapit::IndexBlueprint.
+  # You will likely want to call Xapit::Config.remove_database before this.
+  def self.index_all(*args, &block)
+    IndexBlueprint.index_all(*args, &block)
+  end
+  
+  # Used to perform a search on all indexed models. The returned collection can
+  # contain instances of different classes which were indexed.
+  # 
+  #   # perform a simple full text search
+  #   @records = Xapit.search("phone")
+  # 
+  # See Xapit::Membership for details on search options.
+  def self.search(*args)
+    Collection.new(nil, *args)
+  end
+end
+
+require File.dirname(__FILE__) + '/xapit/membership'
+require File.dirname(__FILE__) + '/xapit/index_blueprint'
+require File.dirname(__FILE__) + '/xapit/collection'
+require File.dirname(__FILE__) + '/xapit/config'
+require File.dirname(__FILE__) + '/xapit/facet_blueprint'
+require File.dirname(__FILE__) + '/xapit/facet'
+require File.dirname(__FILE__) + '/xapit/facet_option'
+require File.dirname(__FILE__) + '/xapit/query'
+require File.dirname(__FILE__) + '/xapit/query_parsers/abstract_query_parser'
+require File.dirname(__FILE__) + '/xapit/query_parsers/simple_query_parser'
+require File.dirname(__FILE__) + '/xapit/query_parsers/classic_query_parser'
+require File.dirname(__FILE__) + '/xapit/indexers/abstract_indexer'
+require File.dirname(__FILE__) + '/xapit/indexers/simple_indexer'
+require File.dirname(__FILE__) + '/xapit/indexers/classic_indexer'

data/lib/xapit/collection.rb

@@ -0,0 +1,165 @@
+module Xapit
+  # This is the object which is returned when performing a search. It behaves like an array, so you do not need
+  # to worry about fetching the results separately. Just loop through this collection.
+  # 
+  # The results are lazy loading, meaning it does not perform the query on the database until it has to.
+  # This allows you to string queries onto one another.
+  #
+  #   Article.search("kite").search("sky") # only performs one query
+  #
+  # This class is compatible with will_paginate; you can pass it to the will_paginate helper in the view.
+  class Collection
+    NON_DELEGATE_METHODS = %w(nil? send object_id class extend size paginate first last empty? respond_to?).to_set
+    [].methods.each do |m|
+      delegate m, :to => :results unless m =~ /^__/ || NON_DELEGATE_METHODS.include?(m.to_s)
+    end
+    
+    def self.search_similar(member, *args)
+      collection = new(member.class, *args)
+      indexer = SimpleIndexer.new(member.class.xapit_index_blueprint)
+      terms = indexer.text_terms(member) + indexer.field_terms(member)
+      collection.base_query.and_query(Xapian::Query.new(Xapian::Query::OP_OR, terms))
+      collection.base_query.not_query("Q#{member.class}-#{member.id}")
+      collection
+    end
+    
+    def initialize(*args)
+      @query_parser = Config.query_parser.new(*args)
+    end
+    
+    # Returns an array of results. You should not need to call this directly because most methods are 
+    # automatically delegated to this array.
+    def results
+      @results ||= fetch_results
+    end
+    
+    # The number of total records found despite any pagination settings.
+    def size
+      @query_parser.query.count
+    end
+    alias_method :total_entries, :size
+    
+    # Returns true if no results are found.
+    def empty?
+      @results ? @results.empty? : size.zero?
+    end
+    
+    # The first record in the result set.
+    def first
+      fetch_results(:offset => 0, :limit => 1).first
+    end
+    
+    # The last record in the result set.
+    def last
+      fetch_results(:offset => size-1, :limit => 1).last
+    end
+    
+    # Perform another search on this one, inheriting all options already passed.
+    # See Xapit::Membership for search options.
+    def search(keywords, options = {})
+      collection = Collection.new(@query_parser.member_class, keywords, options)
+      collection.base_query = @query_parser.query
+      collection
+    end
+    
+    def base_query=(base_query)
+      @query_parser.base_query = base_query
+    end
+    
+    def base_query
+      @query_parser.base_query
+    end
+    
+    # The page number we are currently on.
+    def current_page
+      @query_parser.current_page
+    end
+    
+    # How many records to display on each page, defaults to 20. Sets with :per_page option when performing search.
+    def per_page
+      @query_parser.per_page
+    end
+    
+    # The offset for the current page
+    def offset
+      @query_parser.offset
+    end
+    
+    # Total number of pages with found results.
+    def total_pages
+      (total_entries / per_page.to_f).ceil
+    end
+    
+    # The previous page number. Returns nil if on first page.
+    def previous_page
+      current_page > 1 ? (current_page - 1) : nil
+    end
+    
+    # The next page number. Returns nil if on last page.
+    def next_page
+      current_page < total_pages ? (current_page + 1): nil
+    end
+    
+    # Xapit::Facet objects matching this search query. See class for details.
+    def facets
+      all_facets.select do |facet|
+        facet.options.size > 1
+      end
+    end
+    
+    # Xapit::FacetOption objects which are currently applied to search (through :facets option). Use this to
+    # display the facets which are currently applied.
+    #
+    #   <% for option in @articles.applied_facet_options %>
+    #     <%=h option.name %>
+    #     <%= link_to "remove", :overwrite_params => { :facets => option } %>
+    #   <% end %>
+    #
+    # If you set :breadcrumb_facets option to true in Config#setup the link will drop leftover facets
+    # instead of removing the current one. This makes it easy to add a breadcrumb style interface.
+    #
+    #   Config.setup(:breadcrumb_facets => true)
+    #   <% for option in @articles.applied_facet_options %>
+    #     <%= link_to h(option.name), :overwrite_params => { :facets => option } %> &gt;
+    #   <% end %>
+    #
+    def applied_facet_options
+      @query_parser.facet_identifiers.map do |identifier|
+        option = FacetOption.find(identifier)
+        option.existing_facet_identifiers = @query_parser.facet_identifiers
+        option
+      end
+    end
+    
+    # Includes a suggested variation of a term which will get many more results. Returns nil if no suggestion.
+    # 
+    #   <% if @articles.spelling_suggestion %>
+    #     Did you mean <%= link_to h(@articles.spelling_suggestion), :overwrite_params => { :keywords => @articles.spelling_suggestion } %>?
+    #   <% end %>
+    # 
+    def spelling_suggestion
+      @query_parser.spelling_suggestion
+    end
+    
+    private
+    
+    def all_facets
+      @query_parser.member_class.xapit_index_blueprint.facets.map do |facet_blueprint|
+        Facet.new(facet_blueprint, @query_parser.query, @query_parser.facet_identifiers)
+      end
+    end
+    
+    def matchset(options = {})
+      @query_parser.query.matchset(options)
+    end
+    
+    def fetch_results(options = {})
+      matchset(options).matches.map do |match|
+        class_name, id = match.document.data.split('-')
+        member = class_name.constantize.find(id)
+        member.xapit_relevance = match.percent
+        member
+      end
+    end
+  end
+end