xapian-fu 0.2

data/LICENSE

@@ -0,0 +1,21 @@
+xapian_fu is released under the MIT License.
+
+Copyright (c) 2009 John Leach
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of the acts_as_xapian software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including without
+limitation the rights to use, copy, modify, merge, publish, distribute,
+sublicense, and/or sell copies of the Software, and to permit persons to whom
+the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,39 @@
+= Xapian Fu
+
+XapianFu is a Ruby library for working with
+{Xapian}[http://xapian.org/] databases.  It builds on the GPL licensed
+Xapian Ruby bindings but provides an interface more in-line with "The
+Ruby Way"(tm).
+
+== Example
+
+Create a database, add 3 documents to it and then search and retrieve 
+them.
+
+  db = XapianDb.new(:dir => 'example.db', :create => true,
+                    :store => [:title, :year])
+  db << { :title => 'Brokeback Mountain', :year => 2005 }
+  db << { :title => 'Cold Mountain', :year => 2004 }
+  db << { :title => 'Yes Man', :year => 2008 }
+  db.search("mountain").each do |match|
+    puts match.fields[:title]
+  end
+
+== ActiveRecord Example
+
+You could use it with something like ActiveRecord to index database 
+records:
+
+  db = XapianDb.new(:dir => 'posts.db', :create => true,
+                    :store => :id)
+  Post.all.each { db << p.attributes }
+  db.search("custard").collect do |doc|
+    Post.find(doc.id)
+  end
+
+= More Info
+
+Author::  John Leach  (mailto:john@johnleach.co.uk)
+Copyright:: Copyright (c) 2009 John Leach
+License:: GPL v2
+Github:: http://github.com/johnl/xapian-fu/tree/master

data/examples/ar_query.rb

@@ -0,0 +1,35 @@
+#!/usr/bin/ruby
+
+require 'rubygems'
+require 'benchmark'
+require 'lib/xapian_fu'
+include XapianFu
+require 'active_record'
+
+ActiveRecord::Base.establish_connection(
+   :adapter  => "mysql",
+   :host     => "localhost",
+   :username => "john",
+   :password => "john",
+   :database => "john_fametastic_dev" )
+
+class WpPost < ActiveRecord::Base
+  set_primary_key :ID
+end
+
+#puts WpPost.new.attributes.keys.join(' ')
+db = XapianDb.new(:dir => 'ar_spider.db')
+
+results = nil
+bm = Benchmark.measure do
+  results = db.search(ARGV.join(' '))
+end
+
+posts = WpPost.find(results.collect { |r| r.id })
+
+puts "Weight\tTitle"
+posts.each_with_index do |p,i|
+  puts "%.3f\t#{p.post_title}" % results[i].weight
+end
+
+puts "Search took %.5f seconds" % bm.total

data/examples/ar_spider.rb

@@ -0,0 +1,37 @@
+#!/usr/bin/ruby
+
+require 'rubygems'
+require 'benchmark'
+require 'lib/xapian_fu'
+include XapianFu
+require 'active_record'
+
+ActiveRecord::Base.establish_connection(
+   :adapter  => "mysql",
+   :host     => "localhost",
+   :username => "john",
+   :password => "john",
+   :database => "john_fametastic_dev" )
+
+class WpPost < ActiveRecord::Base
+  set_primary_key :ID
+end
+
+#puts WpPost.new.attributes.keys.join(' ')
+db = XapianDb.new(:dir => 'ar_spider.db', :overwrite => true)
+
+count = 0
+indexing_time = 0.0
+WpPost.find_in_batches do |posts|
+  db.transaction do
+    puts "Indexing wp_posts #{count} to #{count += posts.size}"
+    posts.each do |post|
+      bm = Benchmark.measure do
+        db << XapianDoc.new(post.attributes.merge({ :id => post.id }))
+      end
+      indexing_time += bm.total
+    end
+  end
+  indexing_time += Benchmark.measure { db.flush }.total
+end
+puts "%i documents took %.4f seconds. %.2f per second" % [count, indexing_time, count / indexing_time]

data/examples/query.rb

@@ -0,0 +1,16 @@
+#!/usr/bin/ruby
+#
+require 'rubygems'
+require 'benchmark'
+require 'lib/xapian_fu'
+
+query_string = ARGV.join(" ")
+db = XapianFu::XapianDb.new(:dir => 'spider.db')
+results = nil
+bm = Benchmark.measure { results = db.search(query_string) }
+puts "Weight\tFilename"
+results.each do |result|
+  puts "%.2f\t%s" % [result.weight, result.fields[:filename]]
+end
+puts "Search took %.5f seconds" %  bm.total
+

data/examples/spider.rb

@@ -0,0 +1,28 @@
+#!/usr/bin/ruby
+
+require 'rubygems'
+require 'benchmark'
+require 'lib/xapian_fu'
+
+db = XapianFu::XapianDb.new(:dir => 'spider.db', :store => :filename, 
+                            :overwrite => true)
+
+base_path = ARGV[0] || '.'
+
+docs = 0
+indexing_time = 0.0
+Dir.glob(File.join(base_path, "/**/*")) do |filename|
+  next unless File.file?(filename)
+  next unless filename =~ /\.(txt|doc|README|c|h|rb|py|note|xml)$/i
+  puts "Indexing #{filename}"
+  text = File.open(filename) { |f| f.read(10 * 1024) }
+  bm = Benchmark.measure do
+    db << XapianFu::XapianDoc.new({:text => text, :filename => filename, 
+                                    :filesize => File.size(filename) })
+  end
+  indexing_time += bm.total
+  docs += 1
+  break if docs == 10000
+end
+indexing_time += Benchmark.measure { db.flush }.total
+puts "#{docs} docs indexed in #{indexing_time} seconds (#{docs / indexing_time} docs per second)"

data/lib/xapian_fu.rb

@@ -0,0 +1,3 @@
+$:.unshift File.join(File.dirname(__FILE__), 'xapian_fu')
+require 'xapian_db'
+require 'xapian_doc'

data/lib/xapian_fu/xapian_db.rb

@@ -0,0 +1,193 @@
+module XapianFu
+  class XapianFuError < StandardError ; end
+
+  require 'xapian'
+  require 'xapian_doc'
+  require 'thread'
+
+  class ConcurrencyError < XapianFuError ; end
+  class DocNotFound < XapianFuError ; end
+
+  class XapianDb
+    attr_reader :dir, :db_flag, :query_parser
+    attr_reader :store_fields, :store_values
+
+    def initialize( options = { } )
+      @dir = options[:dir]
+      @db_flag = Xapian::DB_OPEN
+      @db_flag = Xapian::DB_CREATE_OR_OPEN if options[:create]
+      @db_flag = Xapian::DB_CREATE_OR_OVERWRITE if options[:overwrite]
+      @store_fields = Array.new(1, options[:store]).compact
+      @store_values = Array.new(1, options[:sortable]).compact
+      @store_values += Array.new(1, options[:collapsible]).compact
+      rw.flush if options[:create]
+      @tx_mutex = Mutex.new
+    end
+
+    # Return the writable Xapian database
+    def rw
+      @rw ||= setup_rw_db
+    end
+
+    # Return the read-only Xapian database
+    def ro
+      @ro ||= setup_ro_db
+    end
+
+    # Return the number of docs in the Xapian database
+    def size
+      ro.doccount
+    end
+
+    # Return the XapianDocumentsAccessor for this database
+    def documents
+      @documents_accessor ||= XapianDocumentsAccessor.new(self)
+    end
+
+    # Add a document to the index. A document can be just a hash, the
+    # keys representing field names and their values the data to be
+    # indexed.  Or it can be a XapianDoc, or any object with a to_s method.
+    # 
+    # If the document object reponds to the method :data, whatever it
+    # returns is marshalled and stored in the  Xapian database.  Any
+    # arbitrary data up to Xmeg can be stored here.
+    #
+    # Currently, all fields are stored in the database. This will
+    # change to store only those fields requested to be stored.
+    def add_doc(doc)
+      doc = XapianDoc.new(doc) unless doc.is_a? XapianDoc
+      doc.db = self
+      xdoc = doc.to_xapian_document
+      tg = Xapian::TermGenerator.new
+      tg.database = rw
+      tg.document = xdoc
+      tg.index_text( doc.text )
+      if doc.id
+        rw.replace_document(doc.id, xdoc)
+      else
+        doc.id = rw.add_document(xdoc)
+      end
+      doc
+    end
+    alias_method "<<", :add_doc
+
+    # Conduct a search on the Xapian database, returning an array of 
+    # XapianDoc objects for the matches
+    def search(q, options = {})
+      defaults = { :page => 1, :per_page => 10, :reverse => false }
+      options = defaults.merge(options)
+      page = options[:page].to_i rescue 1
+      page = page > 1 ? page - 1 : 0
+      per_page = options[:per_page].to_i rescue 10
+      offset = page * per_page
+      query = query_parser.parse_query(q, Xapian::QueryParser::FLAG_WILDCARD && Xapian::QueryParser::FLAG_LOVEHATE)
+      if options[:order]
+        enquiry.sort_by_value!(options[:order].to_s.hash, options[:reverse])
+      end
+      if options[:collapse]
+        enquiry.collapse_key = options[:collapse].to_s.hash
+      end
+      enquiry.query = query
+      enquiry.mset(offset, per_page).matches.collect { |m| XapianDoc.new(m) }
+    end
+
+    # Run the given block in a XapianDB transaction.  Any changes to the 
+    # Xapian database made in the block will be atomically committed at the end.
+    # 
+    # If an exception is raised by the block, all changes are discarded and the
+    # exception re-raised.
+    # 
+    # Xapian does not support multiple concurrent transactions on the
+    # same Xapian database. Any attempts at this will be serialized by
+    # XapianFu, which is not perfect but probably better than just
+    # kicking up an exception.
+    #
+    def transaction
+      @tx_mutex.synchronize do
+        rw.begin_transaction
+        yield
+        rw.commit_transaction
+      end
+    rescue Exception => e
+      rw.cancel_transaction
+      raise e
+    end
+
+    # Flush any changes to disk and reopen the read-only database.
+    # Raises ConcurrencyError if a transaction is in process
+    def flush
+      raise ConcurrencyError if @tx_mutex.locked?
+      rw.flush
+      ro.reopen
+    end
+
+    def query_parser
+      unless @query_parser
+        @query_parser = Xapian::QueryParser.new
+        @query_parser.database = ro
+      end
+      @query_parser
+    end 
+
+    def enquiry
+      @enquiry ||= Xapian::Enquire.new(ro)
+    end
+
+    private
+
+    def setup_rw_db
+      if dir
+        @rw = Xapian::WritableDatabase.new(dir, db_flag)
+      else
+        # In memory database
+        @rw = Xapian::inmemory_open
+      end
+    end
+
+    def setup_ro_db
+      if dir
+        @ro = Xapian::Database.new(dir)
+      else
+        # In memory db
+        @ro = rw
+      end
+    end
+
+    #
+    class XapianDocumentsAccessor
+      def initialize(xdb)
+        @xdb = xdb
+      end
+
+      # Return the document with the given id from the
+      # database. Raises a XapianFu::DocNotFoundError exception 
+      # if it doesn't exist.
+      def find(doc_id)
+        xdoc = @xdb.ro.document(doc_id)
+        XapianDoc.new(xdoc)
+      rescue RuntimeError => e
+        raise e.to_s =~ /^DocNotFoundError/ ? XapianFu::DocNotFound : e
+      end
+
+      # Return the document with the given id from the database or nil
+      # if it doesn't exist
+      def [](doc_id)
+        find(doc_id)
+      rescue XapianFu::DocNotFound
+        nil
+      end
+
+      # Delete the given document from the database and return the
+      # document id, or nil if it doesn't exist
+      def delete(doc)
+        if doc.respond_to?(:to_i)
+          @xdb.rw.delete_document(doc.to_i)
+          doc.to_i
+        end
+      rescue RuntimeError => e
+        raise e unless e.to_s =~ /^DocNotFoundError/
+      end
+    end
+  end
+
+end

data/lib/xapian_fu/xapian_doc.rb

@@ -0,0 +1,117 @@
+module XapianFu
+  
+  class XapianDbNotSet < XapianFuError ; end
+  class XapianDocNotSet < XapianFuError ; end
+  class XapianTypeError < XapianFuError ; end
+  
+  class XapianDoc
+    attr_reader :fields, :data, :weight, :match
+    attr_reader :xapian_document
+    attr_accessor :id, :db
+
+    # Expects a Xapian::Document, a Hash-like object, or anything that
+    # with a to_s method.  Anything else raises a XapianTypeError.
+    # Options can be <tt>:weight</tt> to set the search weight or
+    # <tt>:data</tt> to set some additional data to be stored with the
+    # record in the database.
+    def initialize(doc, options = {})
+      @fields = {}
+      if doc.is_a? Xapian::Match
+        match = doc
+        doc = match.document
+        @match = match
+        @weight = @match.weight
+      end
+
+      # Handle initialisation from a Xapian::Document, which is
+      # usually a search result from a Xapian database
+      if doc.is_a?(Xapian::Document)
+        @xapian_document = doc
+        @id = doc.docid
+        begin
+          xdoc_data = Marshal::load(doc.data) unless doc.data.empty?
+        rescue ArgumentError
+          @data = nil
+        end
+        if xdoc_data.is_a? Hash
+          @data = xdoc_data.delete(:__data)
+          @fields = xdoc_data
+        else
+          @data = xdoc_data
+        end
+      # Handle initialisation from a hash-like object
+      elsif doc.respond_to?("[]") and doc.respond_to?(:has_key?)
+        @fields = doc
+        @id = doc[:id] if doc.has_key?(:id)
+      # Handle initialisation from anything else that can be coerced
+      # into a string
+      elsif doc.respond_to? :to_s
+        @fields = { :content => doc.to_s }
+      else
+        raise XapianTypeError, "Can't handle indexing a '#{doc.class}' object"
+      end
+      @weight = options[:weight] if options[:weight]
+      @data = options[:data] if options[:data]
+    end
+
+    # Retrieve the given Xapianvalue from the XapianDb.  <tt>vkey</tt>
+    # can be a symbol or string, in which case it's hashed to get an
+    # integer value number. Or you can give the integer value number
+    # if you know it.
+    def get_value(vkey)
+      raise XapianDocNotSet unless @xapian_document
+      vkey = vkey.to_s.hash unless vkey.is_a? Integer
+      @xapian_document.value(vkey)
+    end
+
+    # Return a list of terms that the db has for this document.
+    def terms
+      raise XapianFu::XapianDbNotSet unless db      
+      db.ro.termlist(id) if db.respond_to?(:ro) and db.ro and id
+    end
+
+    # Return a Xapian::Document ready for putting into a Xapian
+    # database. Requires that the db attribute has been set up.
+    def to_xapian_document
+      raise XapianFu::XapianDbNotSet unless db
+      xdoc = Xapian::Document.new
+      add_stored_fields_to_xapian_doc(xdoc)
+      add_stored_values_to_xapian_doc(xdoc)
+      xdoc
+    end
+
+    # Return text for indexing from the fields
+    def text
+      fields.keys.collect { |key| fields[key].to_s }.join(' ')
+    end
+    
+    def ==(b)
+      if b.is_a?(XapianDoc)
+        id == b.id
+      else
+        super(b)
+      end
+    end
+    
+    def inspect
+      "<#{self.class.to_s} id=#{id}>"
+    end
+    
+    private
+    
+    def add_stored_fields_to_xapian_doc(xdoc)
+      stored_fields = fields.reject { |k,v| ! db.store_fields.include? k }
+      stored_fields[:__data] = data if data
+      xdoc.data = Marshal.dump(stored_fields) unless stored_fields.empty?
+      xdoc
+    end
+    
+    def add_stored_values_to_xapian_doc(xdoc)
+      stored_values = fields.reject { |k,v| ! db.store_values.include? k }
+      stored_values.each do |k,v|
+        xdoc.add_value(k.to_s.hash, v.to_s)
+      end
+      xdoc
+    end
+  end
+end

data/spec/spec.opts

@@ -0,0 +1,5 @@
+--colour
+--format n
+--loadby mtime
+--reverse
+

data/spec/xapian_db_spec.rb

@@ -0,0 +1,295 @@
+require 'xapian'
+require 'lib/xapian_fu.rb'
+include XapianFu
+require 'fileutils'
+
+# Will be deleted
+tmp_dir = '/tmp/xapian_fu_test.db'
+
+describe XapianDb do
+  before do
+    FileUtils.rm_rf tmp_dir if File.exists?(tmp_dir)
+  end
+    
+  it "should make an in-memory database by default" do
+    xdb = XapianDb.new
+    xdb.ro.should be_a_kind_of(Xapian::Database)
+    xdb.rw.should === xdb.ro
+  end
+  
+  it "should make an on-disk database when given a :dir option" do
+    xdb = XapianDb.new(:dir => tmp_dir, :create => true)
+    File.exists?(tmp_dir).should be_true
+    xdb.should respond_to(:dir)
+    xdb.dir.should == tmp_dir
+    xdb.rw.should be_a_kind_of(Xapian::WritableDatabase)
+    xdb.ro.should be_a_kind_of(Xapian::Database)
+  end
+
+  it "should flush documents to the index when flush is called" do
+    xdb = XapianDb.new(:dir => tmp_dir, :create => true)
+    xdb.size.should == 0
+    xdb << "Once upon a time"
+    xdb.size.should == 0
+    xdb.flush
+    xdb.size.should == 1
+  end
+
+  it "should support transactions" do
+    xdb = XapianDb.new(:dir => tmp_dir, :create => true)
+    xdb << "Once upon a time"
+    xdb.transaction do
+      xdb << "Once upon a time"
+      xdb.size.should == 1
+    end
+    xdb.flush
+    xdb.size.should == 2
+  end
+
+  it "should serialize attempts at concurrent transactions" do
+    xdb = XapianDb.new(:dir => tmp_dir, :create => true)
+    thread = Thread.new do
+      xdb.transaction do
+        sleep 0.1
+        xdb << "Once upon a time"
+        sleep 0.1
+        xdb << "Once upon a time"
+      end
+    end
+    xdb.transaction do
+      xdb << "Once upon a time"
+      sleep 0.1
+      xdb << "Once upon a time"
+    end
+    thread.join
+    xdb.flush
+    xdb.size.should == 4
+  end
+
+  it "should abort a transaction on an exception" do
+    xdb = XapianDb.new(:dir => tmp_dir, :create => true)
+    xdb << "Once upon a time"
+    begin
+      xdb.transaction do
+        xdb << "Once upon a time"
+        raise StandardError
+      end
+    rescue StandardError
+    end
+    xdb.flush
+    xdb.size.should == 1
+  end
+
+  it "should index a XapianDoc" do
+    xdb = XapianDb.new
+    xdb << XapianDoc.new({ :text => "once upon a time", :title => "A story" })
+    xdb.flush
+    xdb.size.should == 1
+  end
+
+  it "should index a Hash" do
+    xdb = XapianDb.new
+    xdb << { :text => "once upon a time", :title => "A story" }
+    xdb.flush
+    xdb.size.should == 1
+  end
+
+  it "should index a string" do
+    xdb = XapianDb.new
+    xdb << "once upon a time"
+    xdb.size.should == 1
+    xdb << XapianDoc.new("once upon a time")
+    xdb.size.should == 2
+  end
+
+  it "should raise a XapianFu::DocNotFound error on find if the document doesn't exist" do
+    xdb = XapianDb.new
+    xdb << "once upon a time"
+    xdb.flush
+    lambda { xdb.documents.find(10) }.should raise_error XapianFu::DocNotFound
+  end
+
+  it "should retrieve documents with the find method" do
+    xdb = XapianDb.new
+    xdb << "Once upon a time"
+    xdb.flush
+    xdb.documents.find(1).should be_a_kind_of(XapianDoc)
+  end
+
+  it "should retrieve documents like an array and return a XapianDoc" do
+    xdb = XapianDb.new
+    xdb << "once upon a time"
+    xdb.flush
+    xdb.documents[1].should be_a_kind_of(XapianDoc)
+  end
+
+  it "should provide the id of retrieved documents" do
+    xdb = XapianDb.new
+    xdb << "once upon a time"
+    xdb.documents[1].id.should == 1
+  end
+
+  it "should store data in the database" do
+    xdb = XapianDb.new
+    xdb << XapianDoc.new({ :text => "once upon a time" }, :data => { :thing => 0xdeadbeef })
+    xdb.size.should == 1
+    doc = xdb.documents[1]
+    doc.data.should == { :thing => 0xdeadbeef }
+  end
+
+  it "should return a XapianDoc with an id after indexing" do
+    xdb = XapianDb.new
+    doc = XapianDoc.new("once upon a time")
+    doc.id.should == nil
+    new_doc = xdb << doc
+    new_doc.id.should == 1
+  end
+
+  it "should replace docs that already have an id when adding to the db" do
+    xdb = XapianDb.new
+    doc = xdb << XapianDoc.new("Once upon a time")
+    xdb.flush
+    xdb.size.should == 1
+    doc.id.should == 1
+    updated_doc = xdb << doc
+    xdb.flush
+    xdb.size.should == 1
+    updated_doc.id.should == doc.id
+  end
+
+  it "should delete docs by id" do
+    xdb = XapianDb.new
+    doc = xdb << XapianDoc.new("Once upon a time")
+    xdb.flush
+    xdb.size.should == 1
+    xdb.documents.delete(doc.id).should == 1
+    xdb.flush
+    xdb.size.should == 0
+  end
+
+  it "should handle being asked to delete docs that don't exist in the db" do
+    xdb = XapianDb.new
+    doc = xdb << XapianDoc.new("Once upon a time")
+    xdb.flush
+    xdb.documents.delete(100000).should == nil
+  end
+
+  it "should add new docs with the given id" do
+    xdb = XapianDb.new
+    doc = xdb << XapianDoc.new(:id => 0xbeef, :title => "Once upon a time")
+    xdb.flush
+    xdb.documents[0xbeef].id.should == 0xbeef
+    doc.id.should == 0xbeef
+  end
+
+  it "should tokenize strings" do
+    xdb = XapianDb.new
+    doc = xdb << XapianDoc.new("once upon a time")
+    doc.terms.should be_a_kind_of Array
+    doc.terms.last.should be_a_kind_of Xapian::Term
+    doc.terms.last.term.should == "upon"
+  end
+
+  it "should tokenize a hash" do
+    xdb = XapianDb.new
+    doc = xdb << XapianDoc.new(:title => 'once upon a time')
+    doc.terms.should be_a_kind_of Array
+    doc.terms.last.should be_a_kind_of Xapian::Term
+    doc.terms.last.term.should == "upon"
+  end
+
+  it "should return a list of XapianDocs with the weight and match set when returning search results" do
+    xdb = XapianDb.new
+    xdb << XapianDoc.new(:title => 'once upon a time')
+    xdb << XapianDoc.new(:title => 'three little pings')
+    results = xdb.search("pings")
+    results.should be_a_kind_of Array
+    results.size.should == 1
+    results.first.should be_a_kind_of XapianDoc
+    results.first.match.should be_a_kind_of Xapian::Match
+    results.first.weight.should be_a_kind_of Float
+  end
+
+  it "should support searching with :page and :per_page options" do
+    xdb = XapianDb.new
+    content = "word"
+    200.times { xdb << XapianDoc.new(content) }
+    xdb.size.should == 200
+    results = xdb.search(content, :page => 1, :per_page => 12)
+    results.first.id.should == 1
+    results.size.should == 12
+    results = xdb.search(content, :page => 5, :per_page => 18)
+    results.first.id.should == 18 * 4 + 1
+    results.size.should == 18
+    results = xdb.search(content, :page => 100, :per_page => 12)
+    results.size.should == 0
+  end
+
+  it "should store no fields by default" do
+    xdb = XapianDb.new
+    xdb << XapianDoc.new(:title => "Once upon a time")
+    xdb.flush
+    xdb.documents.find(1).fields[:title].should be_nil
+  end
+
+  it "should store fields declared as to be stored" do
+    xdb = XapianDb.new(:store => :title)
+    xdb << XapianDoc.new(:title => "Once upon a time", :author => "Jim Jones")
+    xdb.flush
+    doc = xdb.documents.find(1)
+    doc.fields[:title].should == "Once upon a time"
+    doc.fields[:author].should be_nil
+  end
+  
+  it "should store values declared as to be sortable" do
+    xdb = XapianDb.new(:sortable => :created_at)
+    time = Time.now
+    xdb << XapianDoc.new(:created_at => time.to_i.to_s, :author => "Jim Jones")
+    xdb.flush
+    doc = xdb.documents.find(1)
+    doc.get_value(:created_at).should == time.to_i.to_s
+  end
+  
+  it "should store values declared as to be collapsible" do
+    xdb = XapianDb.new(:collapsible => :group_id)
+    xdb << XapianDoc.new(:group_id => "666", :author => "Jim Jones")
+    xdb.flush
+    doc = xdb.documents.find(1)
+    doc.get_value(:group_id).should == "666"
+  end
+
+  describe "search results sort order" do
+    before(:each) do
+      @xdb = XapianDb.new(:sortable => :number)
+      @expected_results = []
+      @expected_results << (@xdb << XapianDoc.new(:words => "cow dog cat", :number => 1))
+      @expected_results << (@xdb << XapianDoc.new(:words => "cow dog", :number => 3))
+      @expected_results << (@xdb << XapianDoc.new(:words => "cow", :number => 2))
+    end
+    
+    it "should be by relevance by default" do
+      results = @xdb.search("cow dog cat")
+      results.should == @expected_results
+    end
+    
+    it "should be by the value specified in descending numerical order" do
+      results = @xdb.search("cow dog cat", :order => :number)
+      results.should == @expected_results.sort_by { |r| r.fields[:number] }
+    end
+    
+    it "should be reversed when the reverse option is set to true" do
+      results = @xdb.search("cow dog cat", :order => :number, :reverse => true)
+      results.should == @expected_results.sort_by { |r| r.fields[:number] }.reverse
+    end
+  end
+  
+  it "should collapse results by the value specified by the :collapse option" do
+    xdb = XapianDb.new(:collapsible => :group)
+    alpha1 = xdb << XapianDoc.new(:words => "cow dog cat", :group => "alpha")
+    alpha2 = xdb << XapianDoc.new(:words => "cow dog", :group => "alpha")
+    beta1  = xdb << XapianDoc.new(:words => "cow", :group => "beta")
+    results = xdb.search("cow dog cat", :collapse => :group)
+    results.should == [alpha1, beta1]
+  end
+  
+end

data/spec/xapian_doc_spec.rb

@@ -0,0 +1,16 @@
+require 'xapian'
+require 'lib/xapian_fu.rb'
+include XapianFu
+require 'fileutils'
+
+describe XapianDoc do
+
+  it "should be equal to other XapianDoc objects with the same id" do
+    XapianDoc.new(:id => 666).should == XapianDoc.new(:id => 666)
+  end
+  
+  it "should not be equal to other XapianDoc objects with different ids" do
+    XapianDoc.new(:id => 666).should_not == XapianDoc.new(:id => 667)
+  end
+
+end

metadata

@@ -0,0 +1,69 @@
+--- !ruby/object:Gem::Specification 
+name: xapian-fu
+version: !ruby/object:Gem::Version 
+  version: "0.2"
+platform: ruby
+authors: 
+- John Leach
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-06-20 00:00:00 +01:00
+default_executable: 
+dependencies: []
+
+description: A library to provide a more Ruby-like interface to the Xapian search engine.
+email: john@johnleach.co.uk
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+- LICENSE
+files: 
+- lib/xapian_fu.rb
+- lib/xapian_fu
+- lib/xapian_fu/xapian_db.rb
+- lib/xapian_fu/xapian_doc.rb
+- examples/ar_spider.rb
+- examples/query.rb
+- examples/spider.rb
+- examples/ar_query.rb
+- README.rdoc
+- LICENSE
+has_rdoc: true
+homepage: http://github.com/johnl/xapian-fu/tree/master
+post_install_message: 
+rdoc_options: 
+- --title
+- Xapian Fu
+- --main
+- README.rdoc
+- --line-numbers
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: xapian-fu
+rubygems_version: 1.3.1
+signing_key: 
+specification_version: 2
+summary: A Ruby interface to the Xapian search engine
+test_files: 
+- spec/xapian_doc_spec.rb
+- spec/xapian_db_spec.rb
+- spec/spec.opts