xapit 0.1.0

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2008 Ryan Bates
+ 
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+ 
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+ 
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Manifest

@@ -0,0 +1,178 @@
+features/facets.feature
+features/finding.feature
+features/indexing.feature
+features/step_definitions/common_steps.rb
+features/step_definitions/xapit_steps.rb
+features/support/env.rb
+features/support/xapit_helpers.rb
+init.rb
+install.rb
+lib/xapit/collection.rb
+lib/xapit/config.rb
+lib/xapit/facet.rb
+lib/xapit/facet_blueprint.rb
+lib/xapit/facet_option.rb
+lib/xapit/index_blueprint.rb
+lib/xapit/indexers/abstract_indexer.rb
+lib/xapit/indexers/classic_indexer.rb
+lib/xapit/indexers/simple_indexer.rb
+lib/xapit/membership.rb
+lib/xapit/query.rb
+lib/xapit/query_parsers/abstract_query_parser.rb
+lib/xapit/query_parsers/classic_query_parser.rb
+lib/xapit/query_parsers/simple_query_parser.rb
+lib/xapit.rb
+LICENSE
+Rakefile
+README.rdoc
+spec/spec_helper.rb
+spec/tmp/xapdb/flintlock
+spec/tmp/xapdb/iamflint
+spec/tmp/xapdb/postlist.baseA
+spec/tmp/xapdb/postlist.baseB
+spec/tmp/xapdb/postlist.DB
+spec/tmp/xapdb/record.baseA
+spec/tmp/xapdb/record.baseB
+spec/tmp/xapdb/record.DB
+spec/tmp/xapdb/spelling.baseA
+spec/tmp/xapdb/spelling.baseB
+spec/tmp/xapdb/spelling.DB
+spec/tmp/xapdb/termlist.baseA
+spec/tmp/xapdb/termlist.baseB
+spec/tmp/xapdb/termlist.DB
+spec/tmp/xapian_db/flintlock
+spec/tmp/xapian_db/iamflint
+spec/tmp/xapian_db/postlist.baseA
+spec/tmp/xapian_db/postlist.DB
+spec/tmp/xapian_db/record.baseA
+spec/tmp/xapian_db/record.DB
+spec/tmp/xapian_db/termlist.baseA
+spec/tmp/xapian_db/termlist.DB
+spec/tmp/xapiandab/flintlock
+spec/tmp/xapiandab/iamflint
+spec/tmp/xapiandab/postlist.baseA
+spec/tmp/xapiandab/postlist.baseB
+spec/tmp/xapiandab/postlist.DB
+spec/tmp/xapiandab/record.baseA
+spec/tmp/xapiandab/record.baseB
+spec/tmp/xapiandab/record.DB
+spec/tmp/xapiandab/spelling.baseA
+spec/tmp/xapiandab/spelling.baseB
+spec/tmp/xapiandab/spelling.DB
+spec/tmp/xapiandab/termlist.baseA
+spec/tmp/xapiandab/termlist.baseB
+spec/tmp/xapiandab/termlist.DB
+spec/tmp/xapiandatab/flintlock
+spec/tmp/xapiandatab/iamflint
+spec/tmp/xapiandatab/postlist.baseA
+spec/tmp/xapiandatab/postlist.baseB
+spec/tmp/xapiandatab/postlist.DB
+spec/tmp/xapiandatab/record.baseA
+spec/tmp/xapiandatab/record.baseB
+spec/tmp/xapiandatab/record.DB
+spec/tmp/xapiandatab/spelling.baseA
+spec/tmp/xapiandatab/spelling.baseB
+spec/tmp/xapiandatab/spelling.DB
+spec/tmp/xapiandatab/termlist.baseA
+spec/tmp/xapiandatab/termlist.baseB
+spec/tmp/xapiandatab/termlist.DB
+spec/tmp/xapiandataba/flintlock
+spec/tmp/xapiandataba/iamflint
+spec/tmp/xapiandataba/postlist.baseA
+spec/tmp/xapiandataba/postlist.baseB
+spec/tmp/xapiandataba/postlist.DB
+spec/tmp/xapiandataba/record.baseA
+spec/tmp/xapiandataba/record.baseB
+spec/tmp/xapiandataba/record.DB
+spec/tmp/xapiandataba/spelling.baseA
+spec/tmp/xapiandataba/spelling.baseB
+spec/tmp/xapiandataba/spelling.DB
+spec/tmp/xapiandataba/termlist.baseA
+spec/tmp/xapiandataba/termlist.baseB
+spec/tmp/xapiandataba/termlist.DB
+spec/tmp/xapiandatabas/flintlock
+spec/tmp/xapiandatabas/iamflint
+spec/tmp/xapiandatabas/postlist.baseA
+spec/tmp/xapiandatabas/postlist.DB
+spec/tmp/xapiandatabas/record.baseA
+spec/tmp/xapiandatabas/record.DB
+spec/tmp/xapiandatabas/termlist.baseA
+spec/tmp/xapiandatabas/termlist.DB
+spec/tmp/xapiandatb/flintlock
+spec/tmp/xapiandatb/iamflint
+spec/tmp/xapiandatb/postlist.baseA
+spec/tmp/xapiandatb/postlist.baseB
+spec/tmp/xapiandatb/postlist.DB
+spec/tmp/xapiandatb/record.baseA
+spec/tmp/xapiandatb/record.baseB
+spec/tmp/xapiandatb/record.DB
+spec/tmp/xapiandatb/spelling.baseA
+spec/tmp/xapiandatb/spelling.baseB
+spec/tmp/xapiandatb/spelling.DB
+spec/tmp/xapiandatb/termlist.baseA
+spec/tmp/xapiandatb/termlist.baseB
+spec/tmp/xapiandatb/termlist.DB
+spec/tmp/xapiandbase/flintlock
+spec/tmp/xapiandbase/iamflint
+spec/tmp/xapiandbase/postlist.baseA
+spec/tmp/xapiandbase/postlist.baseB
+spec/tmp/xapiandbase/postlist.DB
+spec/tmp/xapiandbase/record.baseA
+spec/tmp/xapiandbase/record.baseB
+spec/tmp/xapiandbase/record.DB
+spec/tmp/xapiandbase/spelling.baseA
+spec/tmp/xapiandbase/spelling.baseB
+spec/tmp/xapiandbase/spelling.DB
+spec/tmp/xapiandbase/termlist.baseA
+spec/tmp/xapiandbase/termlist.baseB
+spec/tmp/xapiandbase/termlist.DB
+spec/xapit/collection_spec.rb
+spec/xapit/config_spec.rb
+spec/xapit/facet_blueprint_spec.rb
+spec/xapit/facet_option_spec.rb
+spec/xapit/facet_spec.rb
+spec/xapit/index_blueprint_spec.rb
+spec/xapit/indexers/abstract_indexer_spec.rb
+spec/xapit/indexers/classic_indexer_spec.rb
+spec/xapit/indexers/simple_indexer_spec.rb
+spec/xapit/membership_spec.rb
+spec/xapit/query_parsers/abstract_query_parser_spec.rb
+spec/xapit/query_parsers/classic_query_parser_spec.rb
+spec/xapit/query_parsers/simple_query_parser_spec.rb
+spec/xapit/query_spec.rb
+spec/xapit_member.rb
+tasks/spec.rb
+tasks/xapit.rake
+tmp/xapiandatabase/flintlock
+tmp/xapiandatabase/iamflint
+tmp/xapiandatabase/postlist.baseA
+tmp/xapiandatabase/postlist.baseB
+tmp/xapiandatabase/postlist.DB
+tmp/xapiandatabase/record.baseA
+tmp/xapiandatabase/record.baseB
+tmp/xapiandatabase/record.DB
+tmp/xapiandatabase/spelling.baseA
+tmp/xapiandatabase/spelling.baseB
+tmp/xapiandatabase/spelling.DB
+tmp/xapiandatabase/termlist.baseA
+tmp/xapiandatabase/termlist.baseB
+tmp/xapiandatabase/termlist.DB
+tmp/xapiandatabase/value.baseB
+tmp/xapiandb/flintlock
+tmp/xapiandb/iamflint
+tmp/xapiandb/postlist.baseA
+tmp/xapiandb/postlist.baseB
+tmp/xapiandb/postlist.DB
+tmp/xapiandb/record.baseA
+tmp/xapiandb/record.baseB
+tmp/xapiandb/record.DB
+tmp/xapiandb/spelling.baseA
+tmp/xapiandb/spelling.baseB
+tmp/xapiandb/spelling.DB
+tmp/xapiandb/termlist.baseA
+tmp/xapiandb/termlist.baseB
+tmp/xapiandb/termlist.DB
+tmp/xapiandb/value.baseB
+TODO
+uninstall.rb
+Manifest

data/README.rdoc

@@ -0,0 +1,183 @@
+= Xapit
+
+Xapit (pronounced "zap it") is a high level interface for working with a Xapian database.
+
+Note: This project is early in development and the API is subject to change.
+
+
+== Install
+
+If you haven't already, first install Xapian and the Xapian Bindings for Ruby.
+http://wiki.github.com/Overbryd/acts_as_xapian/installation
+
+To install as a Rails plugin, run this command.
+
+  script/plugin install git://github.com/ryanb/xapit.git
+
+Note: this will soon also be available as a Ruby gem, but not yet.
+
+
+== Setup
+
+Simply call "xapit" in the model and pass a block to define the indexed attributes.
+
+  class Article < ActiveRecord::Base
+    xapit do |index|
+      index.text :name, :content
+      index.field :category_id
+      index.facet :author_name, "Author"
+      index.sortable :id, :category_id
+    end
+  end
+
+First we index "name" and "content" attributes for full text searching. The "category_id" field is indexed for :conditions searching. The "author_name" is indexed as a facet with "Author" being the display name of the facet. See the facets section below for details. Finally the "id" and "category_id" attributes are indexed as sortable attributes so they can be included in the :order option in a search.
+
+Because the indexing happens in Ruby these attributes do no have to be database columns. They can be simple Ruby methods. For example, the "author_name" attribute mentioned above can be defined like this.
+
+  def author_name
+    author.name
+  end
+
+This way you can create a completely custom facet by simply defining your own method. Multiple facet options or field values per record are supported if you return an array.
+
+  def author_names
+    authors.map(&:name) # => ["John", "Bob"]
+  end
+
+Finally, you can pass any find options to the xapit method to determine what gets indexed or improve performance with eager loading or a different batch size.
+
+  xapit(:batch_size => 100, :include => :author, :conditions => { :visible => true })
+
+You can specify a :weight option to give a text attribute more importance. This will cause search terms matching that attribute to have a higher rank. The default weight is 1. Decimal (0.5) weight values are not supported.
+
+  index.text :name, :weight => 10
+
+
+== Index
+
+To perform the indexing, run the xapit:index rake task.
+
+  rake xapit:index
+
+It can also be triggered through Ruby code using this command.
+
+  Xapit::Config.remove_database
+  Xapit.index_all
+
+
+== Search
+
+You can then perform a search on the model.
+  
+  # perform a simple full text search
+  @articles = Article.search("phone")
+  
+  # add pagination if you're using will_paginate
+  @articles = Article.search("phone", :per_page => 10, :page => params[:page])
+  
+  # search based on indexed fields
+  @articles = Article.search("phone", :conditions => { :category_id => params[:category_id] })
+  
+  # manually sort based on any number of indexed fields, sort defaults to most relevant
+  @articles = Article.search("phone", :order => [:category_id, :id], :descending => true)
+  
+  # basic boolean matching is supported
+  @articles = Article.search("phone OR fax NOT email")
+
+
+You can also search all indexed models through Xapit.search.
+
+  # search all indexed models
+  @records = Xapit.search("phone")
+
+
+== Results
+
+Simply iterate through the returned set to display the results.
+
+  <% for article in @articles %>
+    <%= article.name %>
+    <%= article.xapit_relevance %>
+  <% end %>
+
+The "xapit_relevance" holds a percentage (between 0 and 100) determining how relevant the given document is to the user's search query.
+
+
+== Spelling
+
+If the searched term isn't found, but it is similar to another term then it will show up as a spelling suggestion.
+
+  <% if @articles.spelling_suggestion %>
+    Did you mean <%= link_to h(@articles.spelling_suggestion), :overwrite_params => { :keywords => @articles.spelling_suggestion } %>?
+  <% end %>
+
+
+== Facets
+
+Facets allow you to further filter the result set based on certain attributes.
+
+  <% for facet in @articles.facets %>
+    <%= facet.name %>
+    <% for option in facet.options %>
+      <%= link_to option.name, :overwrite_params => { :facets => option } %>
+      (<%= option.count %>)
+    <% end %>
+  <% end %>
+
+The to_param method is defined on option to return an identifier which will be passed through the URL. Use this in the search.
+
+  Article.search("phone", :facets => params[:facets])
+
+You can also list the applied facets along with a remove link.
+
+  <% for option in @articles.applied_facet_options %>
+    <%=h option.name %>
+    <%= link_to "remove", :overwrite_params => { :facets => option } %>
+  <% end %>
+
+
+== Config
+
+When installing Xapit as a Rails plugin, an initializer file is automatically created to setup. It looks like this.
+
+  Xapit::Config.setup(:database_path => "#{Rails.root}/db/xapiandb")
+
+There are many other options you can pass into here. This is a more advanced configuration setting which changes the stemming language, disables spelling, and changes the indexer and parser to a classic variation. The classic ones use Xapian's built-in term generator and query parser instead of the ones offered by Xapit.
+
+  Xapit::Config.setup(
+    :database_path => "#{Rails.root}/db/external/xapiandb",
+    :spelling => false,
+    :stemming => "german",
+    :indexer => ClassicIndexer,
+    :query_parser => ClassicQueryParser
+  )
+
+
+== Outside of Rails
+
+Xapit can be used outside of Rails too. You'll just need to include the membership module:
+
+  class Product # not Active Record
+    include Xapit::Membership
+  
+    xapit # ...
+  end
+
+This class is expected to respond to "find_each" (Product.find_each) which is used to iterate through the records when indexing and "find" for fetching a record by "id".
+
+
+== Bug Reports
+
+If you have found a bug to report or a feature to request, please add it to the GitHub issue tracker if it is not there already.
+
+http://github.com/ryanb/xapit/issues/
+
+
+== Development
+
+This project can be found on github at the following URL.
+
+http://github.com/ryanb/xapit
+
+If you would like to contribute to this project, please fork the 
+repository and send me a pull request.

data/Rakefile

@@ -0,0 +1,15 @@
+require 'rubygems'
+require 'rake'
+require 'echoe'
+
+Echoe.new('xapit', '0.1.0') do |p|
+  p.summary        = "Ruby library for interacting with Xapian, a full text search engine."
+  p.description    = "Ruby library for interacting with Xapian, a full text search engine."
+  p.url            = "http://github.com/ryanb/xapit"
+  p.author         = 'Ryan Bates'
+  p.email          = "ryan (at) railscasts (dot) com"
+  p.ignore_pattern = ["tmp/*"]
+  p.development_dependencies = []
+end
+
+Dir["#{File.dirname(__FILE__)}/tasks/*.rb"].sort.each { |ext| load ext }

data/TODO

@@ -0,0 +1,23 @@
+Indexing
++ re-indexing support for new/changed models (?)
++ auto-reload xapian database in memory if database recently indexed
+
+Finding
+? support non-ascii characters in find (å, ä, ö)
++ range in conditions
++ support "age:17" condition matching in query
++ add classes option to Xapit.search to limit which classes are included
++ support @collection.facets on Xapit.search (multiple classes)
++ support :order option for Xapit.search (multiple classes)
+
+Facets
++ sort facet options
++ handle unique id collision gracefully
+
+Performance
+- benchmark indexing
+- multiple id finds in a single query
+
+Other
+- turn into Ruby Gem with clear instructions on requirements
++ remove dependency on active_support

data/features/facets.feature

@@ -0,0 +1,51 @@
+Background:
+  Given an empty database at "tmp/xapiandatabase"
+
+Scenario: List All Facets
+  Given the following indexed records
+    | name | age |
+    | John | 23  |
+    | John | 17  |
+    | Jack | 17  |
+  When I query for ""
+  Then I should have the following facets
+    | facet | option | count |
+    | Name  | Jack   | 1     |
+    | Name  | John   | 2     |
+    | Age   | 17     | 2     |
+    | Age   | 23     | 1     |
+
+Scenario: List Matching Facets
+  Given the following indexed records
+    | name | age |
+    | John | 23  |
+    | John | 17  |
+    | Jack | 17  |
+  When I query for "John"
+  Then I should have the following facets
+    | facet | option | count |
+    | Age   | 17     | 1     |
+    | Age   | 23     | 1     |
+
+Scenario: List Applied Facets
+  Given the following indexed records
+    | name | age |
+    | John | 23  |
+    | Jane | 17  |
+    | Jack | 17  |
+  When I query facets "0c93ee1-078661c"
+  Then I should have the following applied facets
+    | facet | option |
+    | Age   | 17     |
+    | Name  | Jane   |
+
+Scenario: List Multiple Facets Applied to One Record
+  Given the following indexed records
+    | name       |
+    | John, Jack |
+    | John       |
+  When I query for ""
+  Then I should have the following facets
+    | facet | option | count |
+    | Name  | Jack   | 1     |
+    | Name  | John   | 2     |