xapian-fu 1.1.2 → 1.2

data/CHANGELOG.rdoc

@@ -1,3 +1,9 @@
+=== 1.2 (5th July 2011)
+
+* Support for synonyms (Damian Janowski)
+* Fix some Ruby 1.9.2 issues, particularly reading utf8 stop files
+* Fixed specs to work with both xapian 1.0 and 1.2
+
 === 1.1.2 (26th August 2010)
 
 * Spelling correction support

data/examples/spider.rb

@@ -8,7 +8,7 @@ require 'rubygems'
 require 'benchmark'
 require 'lib/xapian_fu'
 
-db = XapianFu::XapianDb.new(:store => [:filename, :filesize],
+db = XapianFu::XapianDb.new(:dir => 'spider.db', :store => [:filename, :filesize],
                             :overwrite => true)
 
 base_path = ARGV[0] || '.'
@@ -31,12 +31,12 @@ while dir = index_queue.shift
       next
     end
     next unless File.file?(filename)
-    next unless filename =~ /(txt|doc|README|c|cpp|h|pl|sh|rb|py|note|xml)$/i
+    next unless filename =~ /(txt|doc|README|c|h|pl|sh|rb|py|note|xml)$/i
     file_count += 1
 
     # Read the first 10k of data
     text = File.open(filename) { |f| f.read(10 * 1024) }
-    file_data += text.to_s.size
+    file_data += text.size
     # Index the data, filename and filesize
     bm = Benchmark.measure do
       db << {

data/lib/xapian_fu/query_parser.rb

@@ -120,7 +120,7 @@ module XapianFu #:nodoc:
       if @flags
         @flags
       else
-        valid_flags = [:boolean, :boolean_anycase, :wildcards, :lovehate, :spelling, :pure_not]
+        valid_flags = [:boolean, :boolean_anycase, :wildcards, :lovehate, :spelling, :pure_not, :synonyms]
         @flags = valid_flags.delete_if { |vf| not @options[vf] }
       end
     end
@@ -135,6 +135,7 @@ module XapianFu #:nodoc:
       qflags |= Xapian::QueryParser::FLAG_LOVEHATE if flags.include?(:lovehate)
       qflags |= Xapian::QueryParser::FLAG_SPELLING_CORRECTION if flags.include?(:spelling)
       qflags |= Xapian::QueryParser::FLAG_PURE_NOT if flags.include?(:pure_not)
+      qflags |= Xapian::QueryParser::FLAG_AUTO_SYNONYMS if flags.include?(:synonyms)
       qflags
     end
 

data/lib/xapian_fu/stopper_factory.rb

@@ -32,7 +32,10 @@ module XapianFu
     def self.stop_words_for(lang)
       raise UnsupportedStopperLanguage, lang.to_s unless File.exists?(stop_words_filename(lang))
       words = []
-      open(stop_words_filename(lang), "r") do |f|
+      # Open files with correct encoding in Ruby 1.9
+      open_args = [stop_words_filename(lang), "r"]
+      open_args << { :encoding => "UTF-8" } if String.new.respond_to? :encoding
+      open(*open_args) do |f|
         while line = f.readline rescue nil
           words << line.split(" ", 2).first.downcase.strip  unless line =~ /^ +|^$|^\|/
         end

data/lib/xapian_fu/xapian_db.rb

@@ -170,6 +170,19 @@ module XapianFu #:nodoc:
     end
     alias_method "<<", :add_doc
 
+    # Add a synonym to the database.
+    #
+    # If you want to search with synonym support, remember to add
+    # the option:
+    #
+    #   db.search("foo", :synonyms => true)
+    #
+    # Note that in-memory databases don't support synonyms.
+    #
+    def add_synonym(term, synonym)
+      rw.add_synonym(term, synonym)
+    end
+
     # Conduct a search on the Xapian database, returning an array of
     # XapianFu::XapianDoc objects for the matches wrapped in a
     # XapianFu::ResultSet.

data/spec/query_parser_spec.rb

@@ -1,5 +1,5 @@
 require 'xapian'
-require 'lib/xapian_fu.rb'
+require File.expand_path('../lib/xapian_fu.rb', File.dirname(__FILE__))
 include XapianFu
 
 describe QueryParser do

data/spec/stopper_factory_spec.rb

@@ -1,6 +1,6 @@
 # encoding: utf-8
 require 'xapian'
-require 'lib/xapian_fu.rb'
+require File.expand_path('../lib/xapian_fu.rb', File.dirname(__FILE__))
 include XapianFu
 require 'fileutils'
 
@@ -33,7 +33,7 @@ describe StopperFactory do
       words.should include 'and'
       words.should include "they're"
     end
-    
+
     %w(danish dutch english finnish french german hungarian italian norwegian portuguese russian spanish swedish).each do |lang|
       describe lang do
         it "should return an array of words" do
@@ -49,7 +49,7 @@ describe StopperFactory do
       end
     end
 
-  
+
     it "should raise a UnsupportedStopperLanguage error if there is no data for the given language" do
       Proc.new { StopperFactory.stop_words_for(:no_existy) }.should raise_error UnsupportedStopperLanguage
     end

data/spec/xapian_db_spec.rb

@@ -1,5 +1,5 @@
 require 'xapian'
-require 'lib/xapian_fu.rb'
+require File.expand_path('../lib/xapian_fu.rb', File.dirname(__FILE__))
 include XapianFu
 require 'fileutils'
 require 'date'
@@ -28,14 +28,14 @@ describe XapianDb do
       xdb.rw.should be_a_kind_of(Xapian::WritableDatabase)
       xdb.ro.should be_a_kind_of(Xapian::Database)
     end
-    
+
   end
 
   it "should lazily create the on-disk database when rw is used" do
     xdb = XapianDb.new(:dir => tmp_dir, :create => true)
     File.exists?(tmp_dir).should be_false
     xdb.rw
-    File.exists?(tmp_dir).should be_true    
+    File.exists?(tmp_dir).should be_true
   end
 
   it "should flush documents to the index when flush is called" do
@@ -201,7 +201,7 @@ describe XapianDb do
       xdb << XapianDoc.new("once upon a time")
       xdb.size.should == 2
     end
-    
+
   end
 
   describe "search" do
@@ -277,10 +277,10 @@ describe XapianDb do
     it "should provide a corrected spelling string by default" do
       xdb = XapianDb.new(:dir => tmp_dir + 'corrected_spelling', :create => true,
                          :overwrite => true)
-      xdb << "there is a mouse in this house"
+      xdb << "there is a mouse in this building"
       xdb.flush
-      results = xdb.search("there was a moose at our house")
-      results.corrected_query.should == "there was a mouse at our house"
+      results = xdb.search("there was a moose at our building")
+      results.corrected_query.should == "there was a mouse at our building"
     end
 
     it "should not provide corrected spellings when disabled" do
@@ -293,11 +293,17 @@ describe XapianDb do
     end
 
 
-    it "should do phrase matching by default when then :default_op option is :phrase"
+    it "should do phrase matching by default when then :default_op option is :phrase" do
+      pending
+    end
 
-    it "should do AND_MAYBE matching by default when the :default_op option is :and_maybe"
+    it "should do AND_MAYBE matching by default when the :default_op option is :and_maybe" do
+      pending
+    end
 
-    it "should do PURE_NOT matching by default when the :default_op option is :pure_not"
+    it "should do PURE_NOT matching by default when the :default_op option is :pure_not" do
+      pending
+    end
 
     it "should page results when given the :page and :per_page options" do
       xdb = XapianDb.new
@@ -356,6 +362,22 @@ describe XapianDb do
       xdb.search("john").should == [john,katherine,louisa]
       xdb.search("john -name:john").should == [katherine,louisa]
     end
+
+    it "should recognize synonyms" do
+      xdb = XapianDb.new(:dir => tmp_dir + 'synonyms', :create => true,
+                         :fields => [:name], :overwrite => true)
+
+      xdb << {:name => "john"}
+      xdb.flush
+
+      xdb.search("jon", :synonyms => true).should be_empty
+
+      xdb.add_synonym("jon", "john")
+      xdb.flush
+
+      xdb.search("jon").should be_empty
+      xdb.search("jon", :synonyms => true).should_not be_empty
+    end
   end
 
   describe "add_doc" do
@@ -533,9 +555,9 @@ describe XapianDb do
       xdb = XapianDb.new(:fields => { :name => String, :title => String })
       xdb.unindexed_fields.should == []
     end
-    
+
     it "should return fields defined as not indexed in the fields option" do
-      xdb = XapianDb.new(:fields => { 
+      xdb = XapianDb.new(:fields => {
                            :name => { :type => String, :index => false },
                            :title => String })
       xdb.unindexed_fields.should include :name

data/spec/xapian_doc_spec.rb

@@ -1,6 +1,6 @@
 # encoding: utf-8
 require 'xapian'
-require 'lib/xapian_fu.rb'
+require File.expand_path('../lib/xapian_fu.rb', File.dirname(__FILE__))
 include XapianFu
 require 'fileutils'
 
@@ -9,7 +9,7 @@ describe XapianDoc do
   it "should be equal to other XapianDoc objects with the same id" do
     XapianDoc.new(:id => 666).should == XapianDoc.new(:id => 666)
   end
-  
+
   it "should not be equal to other XapianDoc objects with different ids" do
     XapianDoc.new(:id => 666).should_not == XapianDoc.new(:id => 667)
   end
@@ -44,7 +44,7 @@ describe XapianDoc do
       xdoc.terms.last.should be_a_kind_of Xapian::Term
       xdoc.terms.last.term.should == "upon"
     end
-    
+
     it "should tokenize the fields of a hash separately" do
       xdb = XapianDb.new
       xdoc = xdb.documents.new({ :text => "once upon a time", :title => "A story" }).to_xapian_document
@@ -73,7 +73,7 @@ describe XapianDoc do
       terms = xdoc.terms.collect { |t| t.term }
       terms.should include time.utc.strftime("%Y%m%d%H%M%S")
     end
-    
+
     it "should convert DateTime instances to a useful format when tokenizing" do
       datetime = DateTime.now
       xdb = XapianDb.new
@@ -118,8 +118,8 @@ describe XapianDoc do
         it "should stem #{lang.to_s.capitalize} words when the :stemmer option is set to :#{lang}" do
           xdb = XapianDb.new
           xdoc = xdb.documents.new(word, :stemmer => lang).to_xapian_document
-          terms = xdoc.terms.collect { |t| t.term }
-          terms.should include 'Z'+stem
+          terms = xdoc.terms.collect { |t| t.term.respond_to?(:force_encoding) ? t.term.force_encoding("UTF-8") : t.term }
+          terms.should include 'Z' + stem
         end
       end
     end
@@ -130,7 +130,7 @@ describe XapianDoc do
       terms = xdoc.terms.collect { |t| t.term if t.term =~ /^Z/ }.compact
       terms.should be_empty
     end
-    
+
     it "should not stem english stop words by default" do
       xdb = XapianDb.new
       xdoc = xdb.documents.new("And they made a cake", :stemmer => :english).to_xapian_document
@@ -139,14 +139,14 @@ describe XapianDoc do
       terms.should_not include 'Za'
       terms.should include 'Zcake'
     end
-    
+
     it "should allow setting the stopper on initialisation" do
       xdb = XapianDb.new(:stopper => :english)
       xdoc = xdb.documents.new("And they made a cake", :stopper => :french)
       xdoc.stopper.call("ayantes").should == true
       xdoc.stopper.call("and").should == false
     end
-    
+
     it "should not stop words when stopper is set to false" do
       xdb = XapianDb.new
       xdoc = xdb.documents.new("And they made a cake", :stopper => false).to_xapian_document
@@ -160,7 +160,7 @@ describe XapianDoc do
       terms = xdoc.terms.collect { |t| t.term }
       terms.should_not include 'Zи'
       terms.should_not include 'Zони'
-      terms.should include 'Zcake'      
+      terms.should include 'Zcake'
     end
   end
 
@@ -180,7 +180,7 @@ describe XapianDoc do
       xdoc = xdb.documents.new("stink and bones", :language => :english, :stemmer => :french)
       xdoc.stemmer.call("contournait").should == "contourn"
     end
-     
+
   end
 
   describe "stopper" do
@@ -188,7 +188,7 @@ describe XapianDoc do
       xdb = XapianDb.new(:language => :french)
       xdoc = xdb.documents.new("stink and bones")
       xdoc.stopper.call("avec").should == true
-    end    
+    end
     it "should return a stopper for the document language, overriding the db" do
       xdb = XapianDb.new(:language => :english)
       xdoc = xdb.documents.new("stink and bones", :language => :french)
@@ -198,7 +198,7 @@ describe XapianDoc do
       xdb = XapianDb.new(:language => :german)
       xdoc = xdb.documents.new("stink and bones", :language => :english, :stopper => :french)
       xdoc.stopper.call("avec").should == true
-    end    
+    end
   end
 
 end

data/spec/xapian_doc_value_accessor_spec.rb

@@ -1,5 +1,5 @@
 require 'xapian'
-require 'lib/xapian_fu.rb'
+require File.expand_path('../lib/xapian_fu.rb', File.dirname(__FILE__))
 include XapianFu
 require 'fileutils'
 require 'fixtures/film_data'
@@ -34,12 +34,12 @@ describe XapianDocValueAccessor do
       doc.values.fetch(:number, Fixnum).should == number
       doc.to_xapian_document.values.first.value.should == [number].pack("G")
     end
-  end  
+  end
 
   it "should store fields defined as Bignum as packed double-precision float, network byte order" do
     xdb = XapianDb.new(:fields => { :number => { :type => Bignum, :store => true } })
     [
-     (-0x1fffffffffffff..-0x1fffffffffffff + 10).to_a, 
+     (-0x1fffffffffffff..-0x1fffffffffffff + 10).to_a,
      (0x1fffffffffffff-10..0x1fffffffffffff).to_a
     ].flatten.each do |number|
       doc = xdb.documents.new(:number => number)
@@ -56,7 +56,7 @@ describe XapianDocValueAccessor do
       lambda {  doc.values.store(:number, number, Bignum) }.should raise_error XapianFu::ValueOutOfBounds
     end
   end
-  
+
   it "should store fields defined as Float as packed double-precision float, network byte order" do
     xdb = XapianDb.new(:fields => { :number => { :type => Float, :store => true } })
     [-0.303393984588383833, 8.448488388488384, 1.0].each do |number|
@@ -64,9 +64,9 @@ describe XapianDocValueAccessor do
       doc.values.store(:number, number).should == number
       doc.values.fetch(:number).should == number
       doc.to_xapian_document.values.first.value.should == [number].pack("G")
-    end      
+    end
   end
-  
+
   it "should store fields defined as Time in UTC as packed double-precision float, network byte order" do
     xdb = XapianDb.new(:fields => { :created_at => { :type => Time, :store => true }})
     time = Time.now
@@ -110,22 +110,22 @@ describe XapianDocValueAccessor do
   film_data_path = File.join(File.dirname(__FILE__), "fixtures/film_data")
   Dir.foreach(film_data_path) do |db_path|
     next unless db_path =~ /.+~.+/
-    it "should read stored values from databases created by #{db_path}" do      
+    it "should read stored values from databases created by #{db_path}" do
       db = XapianDb.new(:dir => File.join(film_data_path, db_path),
-                        :fields => { 
+                        :fields => {
                           :title => { :type => String, :store => true },
-                          :released_on => { :type => Date, :store => true }, 
+                          :released_on => { :type => Date, :store => true },
                           :revenue => { :type => Integer, :store => true }
                         })
       FILM_DATA.size.times do |i|
         doc = db.documents[i+1]
         [:title, :released_on, :revenue].each do |field|
-          doc.values[field].should === FILM_DATA[i][field]  
+          doc.values[field].should === FILM_DATA[i][field]
         end
       end
     end
   end
-  
+
 end
 
 

metadata

@@ -1,13 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: xapian-fu
 version: !ruby/object:Gem::Version 
-  hash: 23
-  prerelease: false
+  hash: 11
+  prerelease: 
   segments: 
   - 1
-  - 1
   - 2
-  version: 1.1.2
+  version: "1.2"
 platform: ruby
 authors: 
 - John Leach
@@ -15,10 +14,24 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-08-26 00:00:00 +01:00
-default_executable: 
-dependencies: []
-
+date: 2011-07-05 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - "="
+      - !ruby/object:Gem::Version 
+        hash: 27
+        segments: 
+        - 1
+        - 3
+        - 0
+        version: 1.3.0
+  type: :development
+  version_requirements: *id001
 description: A library to provide a more Ruby-like interface to the Xapian search engine.
 email: john@johnleach.co.uk
 executables: []
@@ -120,7 +133,6 @@ files:
 - spec/build_db_for_value_testing.rb
 - spec/query_parser_spec.rb
 - spec/spec.opts
-has_rdoc: true
 homepage: http://github.com/johnl/xapian-fu
 licenses: []
 
@@ -154,7 +166,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: xapian-fu
-rubygems_version: 1.3.7
+rubygems_version: 1.7.2
 signing_key: 
 specification_version: 3
 summary: A Ruby interface to the Xapian search engine
@@ -218,3 +230,4 @@ test_files:
 - spec/build_db_for_value_testing.rb
 - spec/query_parser_spec.rb
 - spec/spec.opts
+has_rdoc: