xapian-fu 1.0.1 → 1.1.0

data/README.rdoc

@@ -32,6 +32,8 @@ added to a Xapian database.
 Create a database, add 3 documents to it and then search and retrieve
 them.
 
+  require 'xapian-fu'
+  include XapianFu
   db = XapianDb.new(:dir => 'example.db', :create => true,
                     :store => [:title, :year])
   db << { :title => 'Brokeback Mountain', :year => 2005 }

data/examples/spider.rb

@@ -8,7 +8,7 @@ require 'rubygems'
 require 'benchmark'
 require 'lib/xapian_fu'
 
-db = XapianFu::XapianDb.new(:dir => 'spider.db', :store => [:filename, :filesize],
+db = XapianFu::XapianDb.new(:store => [:filename, :filesize],
                             :overwrite => true)
 
 base_path = ARGV[0] || '.'
@@ -31,12 +31,12 @@ while dir = index_queue.shift
       next
     end
     next unless File.file?(filename)
-    next unless filename =~ /(txt|doc|README|c|h|pl|sh|rb|py|note|xml)$/i
+    next unless filename =~ /(txt|doc|README|c|cpp|h|pl|sh|rb|py|note|xml)$/i
     file_count += 1
 
     # Read the first 10k of data
     text = File.open(filename) { |f| f.read(10 * 1024) }
-    file_data += text.size
+    file_data += text.to_s.size
     # Index the data, filename and filesize
     bm = Benchmark.measure do
       db << {

data/lib/xapian-fu.rb

@@ -0,0 +1 @@
+require 'xapian_fu'

data/lib/xapian_fu/xapian_db.rb

@@ -196,7 +196,7 @@ module XapianFu #:nodoc:
       enquiry = Xapian::Enquire.new(ro)
       setup_ordering(enquiry, options[:order], options[:reverse])
       if options[:collapse]
-        enquiry.collapse_key = options[:collapse].to_s.hash
+        enquiry.collapse_key = XapianDocValueAccessor.value_key(options[:collapse])
       end
       enquiry.query = query
       ResultSet.new(:mset => enquiry.mset(offset, per_page), :current_page => page + 1,
@@ -268,7 +268,7 @@ module XapianFu #:nodoc:
         enquiry.sort_by_value!((1 << 32)-1, reverse)
         enquiry.docid_order = reverse ? Xapian::Enquire::DESCENDING : Xapian::Enquire::ASCENDING
       elsif order.is_a? String or order.is_a? Symbol
-        enquiry.sort_by_value!(order.to_s.hash, reverse)
+        enquiry.sort_by_value!(XapianDocValueAccessor.value_key(order), reverse)
       else
         enquiry.sort_by_relevance!
       end

data/lib/xapian_fu/xapian_doc_value_accessor.rb

@@ -1,55 +1,73 @@
+require 'zlib'
+
 class Integer #:nodoc:
-  def to_xapian_fu_storage_value
-    [self].pack("l")
+  def self.to_xapian_fu_storage_value(value)
+    [value].pack("G")
   end
 
   def self.from_xapian_fu_storage_value(value)
-    value.unpack("l").first
+    value.unpack("G").first.truncate rescue nil
   end
 end
 
 class Bignum #:nodoc:
-  def to_xapian_fu_storage_value
-    [self].pack("G")
+  def self.to_xapian_fu_storage_value(value)
+    if value > 0x1fffffffffffff or value < -0x1fffffffffffff
+      raise XapianFu::ValueOutOfBounds
+    end
+    [value].pack("G")
   end
 
   def self.from_xapian_fu_storage_value(value)
-    value.unpack("G").first
+    value.unpack("G").first.truncate rescue nil
+  end
+end
+
+class Fixnum #:nodoc:
+  def self.to_xapian_fu_storage_value(value)
+    [value].pack("G")
+  end
+
+  def self.from_xapian_fu_storage_value(value)
+    value.unpack("G").first.truncate rescue nil
   end
 end
 
 class Float #:nodoc:
-  def to_xapian_fu_storage_value
-    [self].pack("G")
+  def self.to_xapian_fu_storage_value(value)
+    [value].pack("G")
   end
 
   def self.from_xapian_fu_storage_value(value)
-    value.unpack("G").first
+    value.unpack("G").first rescue nil
   end
 end
 
 class Time #:nodoc:
-  def to_xapian_fu_storage_value
-    [self.utc.to_f].pack("G")
+  def self.to_xapian_fu_storage_value(value)
+    [value.utc.to_f].pack("G")
   end
 
   def self.from_xapian_fu_storage_value(value)
-    Time.at(value.unpack("G").first)
+    Time.at(value.unpack("G").first) rescue nil
   end
 end
 
 class Date #:nodoc:
-  def to_xapian_fu_storage_value
-    to_s
+  def self.to_xapian_fu_storage_value(value)
+    value.to_s
   end
 
   def self.from_xapian_fu_storage_value(value)
-    self.parse(value)
+    self.parse(value) rescue nil
   end
 end
 
 module XapianFu #:nodoc:
   
+  class ValueOutOfBounds < XapianFuError
+  end
+  
   # A XapianDocValueAccessor is used to provide the XapianDoc#values
   # interface to read and write field values to a XapianDb.  It is
   # usually set up by a XapianDoc so you shouldn't need to set up your
@@ -67,12 +85,12 @@ module XapianFu #:nodoc:
     # <tt>from_xapian_fu_storage_value</tt> class method on retrieval.
     def store(key, value, type = nil)
       type = @doc.db.fields[key] if type.nil? and @doc.db
-      if type and value.is_a?(type) and value.respond_to?(:to_xapian_fu_storage_value)
-        converted_value = value.to_xapian_fu_storage_value
+      if type and type.respond_to?(:to_xapian_fu_storage_value)
+        converted_value = type.to_xapian_fu_storage_value(value)
       else
         converted_value = value.to_s
       end
-      @doc.xapian_document.add_value(value_key(key), converted_value)
+      @doc.xapian_document.add_value(XapianDocValueAccessor.value_key(key), converted_value)
       value
     end
     alias_method "[]=", :store
@@ -92,7 +110,7 @@ module XapianFu #:nodoc:
     # Due to the design of Xapian, if the value does not exist then an
     # empty string is returned.
     def fetch(key, type = nil)
-      value = @doc.xapian_document.value(value_key(key))
+      value = @doc.xapian_document.value(XapianDocValueAccessor.value_key(key))
       type = @doc.db.fields[key] if type.nil? and @doc.db
       if type and type.respond_to?(:from_xapian_fu_storage_value)
         type.from_xapian_fu_storage_value(value)
@@ -110,16 +128,14 @@ module XapianFu #:nodoc:
     # Remove the value with the given key from the XapianDoc and return it
     def delete(key)
       value = fetch(key)
-      @doc.xapian_document.remove_value(value_key(key))
+      @doc.xapian_document.remove_value(XapianDocValueAccessor.value_key(key))
       value
     end
 
-    private
-
     # Convert the given key to an integer that can be used as a Xapian
     # value number
-    def value_key(key)
-      key.is_a?(Integer) ? key : key.to_s.hash
+    def self.value_key(key)
+      (key.is_a?(Integer) ? key : Zlib.crc32(key.to_s))
     end    
   end
 end

data/lib/xapian_fu/xapian_documents_accessor.rb

@@ -70,10 +70,10 @@ module XapianFu
         find(@xdb.ro.lastdocid) rescue nil
       else
         # for other values, we do a search ordered by that key in descening order
-        query = Xapian::Query.new(Xapian::Query::OP_VALUE_GE, key.to_s.hash, "0")
+        query = Xapian::Query.new(Xapian::Query::OP_VALUE_GE, XapianDocValueAccessor.value_key(key), "0")
         e = Xapian::Enquire.new(@xdb.ro)
         e.query = query
-        e.sort_by_value!(key.to_s.hash)
+        e.sort_by_value!(XapianDocValueAccessor.value_key(key))
         r = e.mset(0, 1).matches.first
         find(r.docid) rescue nil
       end

data/spec/build_db_for_value_testing.rb

@@ -0,0 +1,24 @@
+#!/usr/bin/env ruby
+#
+
+require 'date'
+require File.join(File.dirname(__FILE__), '../lib/xapian_fu')
+include XapianFu
+require File.join(File.dirname(__FILE__), 'fixtures/film_data')
+require 'fileutils'
+
+
+db_name = [RUBY_PLATFORM, RUBY_VERSION].join('~')
+db_path = File.join(File.dirname(__FILE__), 'fixtures/film_data')
+FileUtils.mkdir_p db_path
+db = XapianDb.new(:dir => File.join(db_path, db_name), :create => true,
+									:fields => {
+															:title => { :type => String, :store => true }, 
+															:released_on => { :type => Date, :store => true }, 
+															:revenue => { :type => Integer, :store => true }})
+
+
+
+FILM_DATA.each { |film| db << film }
+
+

data/spec/fixtures/film_data.rb

@@ -0,0 +1,5 @@
+
+FILM_DATA = []
+FILM_DATA << { :title => "Cold Mountain", :released_on => Date.parse("2003/12/25"), :revenue => 173_013_509 }
+FILM_DATA << { :title => "Brokeback Mountain", :released_on => Date.parse("2005/12/16"), :revenue => 178_062_759 }
+FILM_DATA << { :title => "Yes Man", :released_on => Date.parse("2008/12/26"), :revenue => 301_455_191 }

data/spec/stopper_factory_spec.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 require 'xapian'
 require 'lib/xapian_fu.rb'
 include XapianFu

data/spec/xapian_doc_spec.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 require 'xapian'
 require 'lib/xapian_fu.rb'
 include XapianFu
@@ -109,7 +110,7 @@ describe XapianDoc do
       :french => { "majestueusement" => "majestu" },
       :swedish => { "kloekornas" => "kloek" },
       :danish => { "indvendingerne" => "indvending" },
-      :russian => { "падшую", "падш" },
+      :russian => { "падшую" => "падш" },
       :italian => { "propagamento" => "propag" }
     }
     stems.keys.each do |lang|

data/spec/xapian_doc_value_accessor_spec.rb

@@ -2,9 +2,16 @@ require 'xapian'
 require 'lib/xapian_fu.rb'
 include XapianFu
 require 'fileutils'
+require 'fixtures/film_data'
 
 describe XapianDocValueAccessor do
 
+  describe "value_key" do
+    it "should return the crc32 of the given string" do
+      XapianDocValueAccessor.value_key("louisa").should == 4040578532
+    end
+  end
+
   it "should store and fetch values like a hash" do
     values = XapianDocValueAccessor.new(XapianDoc.new(nil))
     values.store(:city, "Leeds").should == "Leeds"
@@ -19,25 +26,38 @@ describe XapianDocValueAccessor do
     lambda { values[:city] = "London" }.should change(doc.xapian_document, :values_count).by(1)
   end
 
-  it "should store fields defined as Fixnum as packed Long" do
+  it "should store fields defined as Fixnum as packed double-precision float, network byte order" do
     xdb = XapianDb.new(:fields => { :number => { :type => Fixnum, :store => true } })
-    [-83883, 256532, 0, 0x3fffffff].each do |number|
+    [-83883, 256532, 0, 0xffffff].each do |number|
       doc = xdb.documents.new(:number => number)
       doc.values.store(:number, number, Fixnum).should == number
       doc.values.fetch(:number, Fixnum).should == number
-      doc.to_xapian_document.values.first.value.should == [number].pack("l")
+      doc.to_xapian_document.values.first.value.should == [number].pack("G")
     end
   end  
 
-  it "should store fields defined as Bignum as packed Double-precision float, big-endian byte order" do
+  it "should store fields defined as Bignum as packed double-precision float, network byte order" do
     xdb = XapianDb.new(:fields => { :number => { :type => Bignum, :store => true } })
-    doc = xdb.documents.new(:number => 0x3fffffffffffff)
-    doc.values.store(:number, 0x3fffffffffffff).should == 0x3fffffffffffff
-    doc.values.fetch(:number).should == 0x3fffffffffffff
-    doc.to_xapian_document.values.first.value.should == [0x3fffffffffffff].pack("G")
+    [
+     (-0x1fffffffffffff..-0x1fffffffffffff + 10).to_a, 
+     (0x1fffffffffffff-10..0x1fffffffffffff).to_a
+    ].flatten.each do |number|
+      doc = xdb.documents.new(:number => number)
+      doc.values.store(:number, number, Bignum).should == number
+      doc.values.fetch(:number, Bignum).should == number
+      doc.to_xapian_document.values.first.value.should == [number].pack("G")
+    end
+  end
+
+  it "should raise an error when attempting to store Bignum values bigger or smaller than can be stored" do
+    xdb = XapianDb.new(:fields => { :number => { :type => Bignum, :store => true } })
+    [-(0x1fffffffffffff+1), 0x1fffffffffffff+1].each do |number|
+      doc = xdb.documents.new(:number => number)
+      lambda {  doc.values.store(:number, number, Bignum) }.should raise_error XapianFu::ValueOutOfBounds
+    end
   end
   
-  it "should store fields defined as Float as packed Double-precision float, big-endian byte order" do
+  it "should store fields defined as Float as packed double-precision float, network byte order" do
     xdb = XapianDb.new(:fields => { :number => { :type => Float, :store => true } })
     [-0.303393984588383833, 8.448488388488384, 1.0].each do |number|
       doc = xdb.documents.new(:number => number)
@@ -47,12 +67,12 @@ describe XapianDocValueAccessor do
     end      
   end
   
-  it "should store fields defined as Time in UTC as packed Double-precision float, big-endian byte order" do
+  it "should store fields defined as Time in UTC as packed double-precision float, network byte order" do
     xdb = XapianDb.new(:fields => { :created_at => { :type => Time, :store => true }})
     time = Time.now
     doc = xdb.documents.new(:created_at => time)
     doc.values.store(:created_at, time).should == time
-    doc.values.fetch(:created_at).should == time
+    doc.values.fetch(:created_at).should be_close(time, 0.0001) # ignore milliseconds
     doc.to_xapian_document.values.first.value.should == [time.utc.to_f].pack("G")
   end
 
@@ -87,6 +107,25 @@ describe XapianDocValueAccessor do
     doc.values.delete(:city).should == "London"
   end
 
+  film_data_path = File.join(File.dirname(__FILE__), "fixtures/film_data")
+  Dir.foreach(film_data_path) do |db_path|
+    next unless db_path =~ /.+~.+/
+    it "should read stored values from databases created by #{db_path}" do      
+      db = XapianDb.new(:dir => File.join(film_data_path, db_path),
+                        :fields => { 
+                          :title => { :type => String, :store => true },
+                          :released_on => { :type => Date, :store => true }, 
+                          :revenue => { :type => Integer, :store => true }
+                        })
+      FILM_DATA.size.times do |i|
+        doc = db.documents[i+1]
+        [:title, :released_on, :revenue].each do |field|
+          doc.values[field].should === FILM_DATA[i][field]  
+        end
+      end
+    end
+  end
+  
 end
 
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: xapian-fu
 version: !ruby/object:Gem::Version 
-  version: 1.0.1
+  version: 1.1.0
 platform: ruby
 authors: 
 - John Leach
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-09-09 00:00:00 +01:00
+date: 2009-12-13 00:00:00 +00:00
 default_executable: 
 dependencies: []
 
@@ -23,6 +23,7 @@ extra_rdoc_files:
 - README.rdoc
 - LICENSE
 files: 
+- lib/xapian-fu.rb
 - lib/xapian_fu.rb
 - lib/xapian_fu/result_set.rb
 - lib/xapian_fu/stopper_factory.rb
@@ -87,7 +88,60 @@ summary: A Ruby interface to the Xapian search engine
 test_files: 
 - spec/stopper_factory_spec.rb
 - spec/xapian_doc_spec.rb
+- spec/build_db_for_value_testing.rb
 - spec/xapian_db_spec.rb
+- spec/fixtures/film_data.rb
+- spec/fixtures/film_data/x86_64-linux~1.8.7/position.baseA
+- spec/fixtures/film_data/x86_64-linux~1.8.7/flintlock
+- spec/fixtures/film_data/x86_64-linux~1.8.7/record.baseA
+- spec/fixtures/film_data/x86_64-linux~1.8.7/termlist.baseB
+- spec/fixtures/film_data/x86_64-linux~1.8.7/position.baseB
+- spec/fixtures/film_data/x86_64-linux~1.8.7/position.DB
+- spec/fixtures/film_data/x86_64-linux~1.8.7/record.baseB
+- spec/fixtures/film_data/x86_64-linux~1.8.7/termlist.DB
+- spec/fixtures/film_data/x86_64-linux~1.8.7/postlist.baseB
+- spec/fixtures/film_data/x86_64-linux~1.8.7/record.DB
+- spec/fixtures/film_data/x86_64-linux~1.8.7/iamflint
+- spec/fixtures/film_data/x86_64-linux~1.8.7/postlist.baseA
+- spec/fixtures/film_data/x86_64-linux~1.8.7/value.DB
+- spec/fixtures/film_data/x86_64-linux~1.8.7/termlist.baseA
+- spec/fixtures/film_data/x86_64-linux~1.8.7/postlist.DB
+- spec/fixtures/film_data/x86_64-linux~1.8.7/value.baseA
+- spec/fixtures/film_data/x86_64-linux~1.8.7/value.baseB
+- spec/fixtures/film_data/x86_64-linux~1.9.1/position.baseA
+- spec/fixtures/film_data/x86_64-linux~1.9.1/flintlock
+- spec/fixtures/film_data/x86_64-linux~1.9.1/record.baseA
+- spec/fixtures/film_data/x86_64-linux~1.9.1/termlist.baseB
+- spec/fixtures/film_data/x86_64-linux~1.9.1/position.baseB
+- spec/fixtures/film_data/x86_64-linux~1.9.1/position.DB
+- spec/fixtures/film_data/x86_64-linux~1.9.1/record.baseB
+- spec/fixtures/film_data/x86_64-linux~1.9.1/termlist.DB
+- spec/fixtures/film_data/x86_64-linux~1.9.1/postlist.baseB
+- spec/fixtures/film_data/x86_64-linux~1.9.1/record.DB
+- spec/fixtures/film_data/x86_64-linux~1.9.1/iamflint
+- spec/fixtures/film_data/x86_64-linux~1.9.1/postlist.baseA
+- spec/fixtures/film_data/x86_64-linux~1.9.1/value.DB
+- spec/fixtures/film_data/x86_64-linux~1.9.1/termlist.baseA
+- spec/fixtures/film_data/x86_64-linux~1.9.1/postlist.DB
+- spec/fixtures/film_data/x86_64-linux~1.9.1/value.baseA
+- spec/fixtures/film_data/x86_64-linux~1.9.1/value.baseB
+- spec/fixtures/film_data/i486-linux~1.8.7/position.baseA
+- spec/fixtures/film_data/i486-linux~1.8.7/flintlock
+- spec/fixtures/film_data/i486-linux~1.8.7/record.baseA
+- spec/fixtures/film_data/i486-linux~1.8.7/termlist.baseB
+- spec/fixtures/film_data/i486-linux~1.8.7/position.baseB
+- spec/fixtures/film_data/i486-linux~1.8.7/position.DB
+- spec/fixtures/film_data/i486-linux~1.8.7/record.baseB
+- spec/fixtures/film_data/i486-linux~1.8.7/termlist.DB
+- spec/fixtures/film_data/i486-linux~1.8.7/postlist.baseB
+- spec/fixtures/film_data/i486-linux~1.8.7/record.DB
+- spec/fixtures/film_data/i486-linux~1.8.7/iamflint
+- spec/fixtures/film_data/i486-linux~1.8.7/postlist.baseA
+- spec/fixtures/film_data/i486-linux~1.8.7/value.DB
+- spec/fixtures/film_data/i486-linux~1.8.7/termlist.baseA
+- spec/fixtures/film_data/i486-linux~1.8.7/postlist.DB
+- spec/fixtures/film_data/i486-linux~1.8.7/value.baseA
+- spec/fixtures/film_data/i486-linux~1.8.7/value.baseB
 - spec/query_parser_spec.rb
 - spec/xapian_doc_value_accessor_spec.rb
 - spec/spec.opts