ferret 0.9.1 → 0.9.2

data/test/unit/index/tc_index.rb

@@ -1,6 +1,5 @@
 require File.dirname(__FILE__) + "/../../test_helper"
 
-
 class IndexTest < Test::Unit::TestCase
   include Ferret::Index
   include Ferret::Search
@@ -146,6 +145,7 @@ class IndexTest < Test::Unit::TestCase
 
   def test_fs_index
     fs_path = File.expand_path(File.join(File.dirname(__FILE__), '../../temp/fsdir'))
+
     Dir[File.join(fs_path, "*")].each {|path| begin File.delete(path) rescue nil end}
     assert_raise(StandardError) do
       Index.new(:path => fs_path,
@@ -171,6 +171,7 @@ class IndexTest < Test::Unit::TestCase
 
   def test_fs_index_is_persistant
     fs_path = File.expand_path(File.join(File.dirname(__FILE__), '../../temp/fsdir'))
+
     Dir[File.join(fs_path, "*")].each {|path| begin File.delete(path) rescue nil end}
     data = [
       {"def_field" => "one two", :id => "me"},
@@ -195,6 +196,7 @@ class IndexTest < Test::Unit::TestCase
 
   def test_key_used_for_id_field
     fs_path = File.expand_path(File.join(File.dirname(__FILE__), '../../temp/fsdir'))
+
     Dir[File.join(fs_path, "*")].each {|path| begin File.delete(path) rescue nil end}
     data = [
       {:my_id => "one two", :id => "me"},
@@ -301,6 +303,7 @@ class IndexTest < Test::Unit::TestCase
     index = Index.new(:default_field => "f")
     data.each {|doc| index << doc }
     fs_path = File.expand_path(File.join(File.dirname(__FILE__), '../../temp/fsdir'))
+
     index.persist(fs_path, true)
     assert_equal(3, index.size)
     assert_equal("zero", index[0]["f"])
@@ -585,4 +588,40 @@ class IndexTest < Test::Unit::TestCase
     index1.close
     index2.close
   end
+
+  def test_doc_specific_analyzer
+    index = Index.new
+    index.add_document("abc", Ferret::Analysis::Analyzer.new)
+    assert_equal(1, index.size)
+  end
+
+
+  def test_adding_empty_term_vectors
+    index = Index.new()
+    doc = Document.new
+
+    # Note: Adding keywords to either field1 or field2 gets rid of the error
+
+    doc << Field.new('field1', '',
+            Field::Store::NO,
+            Field::Index::TOKENIZED,
+            Field::TermVector::YES)
+
+    doc << Field.new('field2', '',
+            Field::Store::NO,
+            Field::Index::TOKENIZED,
+            Field::TermVector::YES)
+
+    # Note: keywords in this un-term-vector-stored field don't help the situation
+
+    doc << Field.new('field3', 'foo bar baz',
+            Field::Store::YES,
+            Field::Index::TOKENIZED,
+            Field::TermVector::NO)
+
+    index << doc
+
+    index.flush
+    index.close
+  end
 end

data/test/unit/index/tc_term.rb

@@ -17,4 +17,11 @@ class TermTest < Test::Unit::TestCase
     term4.set!("field3", "text3")
     assert_not_equal(term1, term4)
   end
+
+  def test_non_strings()
+    t = Term.new(2345, 3)
+    t = Term.new(:symbol, :symbol)
+    t.set!(:symbol, :symbol)
+    t.set!(234, 23462346)
+  end
 end

data/test/unit/index/th_doc.rb

@@ -240,5 +240,13 @@ module IndexTestHelper
     end
     return docs
   end
+
+  def IndexTestHelper.explain (query, searcher, field)
+    top_docs = searcher.search(query)
+    top_docs.score_docs.each { |sd|
+      puts "\nDoc #{sd.doc}: #{searcher.doc(sd.doc)[field]}\n#{searcher.explain(query, sd.doc).to_s}\n"
+    }
+  end
+
 end
 

data/test/unit/query_parser/tc_query_parser.rb

@@ -8,10 +8,10 @@ class QueryParserTest < Test::Unit::TestCase
       ['', ''],
       ['word', 'word'],
       ['field:word', 'field:word'],
-      ['"word1 word2 word3"', '"word word word"'],
-      ['"word1 2342 word3"', '"word word"'],
+      ['"word1 word2 word#"', '"word1 word2 word"'],
+      ['"word1 %%% word3"', '"word1 word3"'],
       ['field:"one two three"', 'field:"one two three"'],
-      ['field:"one 222 three"', 'field:"one three"'],
+      ['field:"one %%% three"', 'field:"one three"'],
       ['field:"one <> three"', 'field:"one <> three"'],
       ['field:"one <> three <>"', 'field:"one <> three"'],
       ['field:"one <> <> <> three <>"', 'field:"one <> <> <> three"'],
@@ -104,7 +104,9 @@ class QueryParserTest < Test::Unit::TestCase
                                      :analyzer => Ferret::Analysis::StandardAnalyzer.new)
     pairs = [
       ['key:1234', 'key:1234'],
-      ['key:(1234)', 'key:1234']
+      ['key:(1234 and Dave)', 'key:1234 key:dave'],
+      ['key:(1234)', 'key:1234'],
+      ['and the but they with', '']
     ]
       
     pairs.each do |query_str, expected|

data/test/unit/search/rtc_sort_field.rb

@@ -4,11 +4,11 @@ class SortFieldTest < Test::Unit::TestCase
   include Ferret::Search
 
   def test_params()
-    assert_equal("score",  SortField::SortType::SCORE.to_s)
-    assert_equal("doc",    SortField::SortType::DOC.to_s)
-    assert_equal("auto",   SortField::SortType::AUTO.to_s)
-    assert_equal("string", SortField::SortType::STRING.to_s)
-    assert_equal("int",    SortField::SortType::INTEGER.to_s)
-    assert_equal("float",  SortField::SortType::FLOAT.to_s)
+    assert_equal("SCORE",   SortField::SortType::SCORE.to_s)
+    assert_equal("DOC",     SortField::SortType::DOC.to_s)
+    assert_equal("auto",    SortField::SortType::AUTO.to_s)
+    assert_equal("string",  SortField::SortType::STRING.to_s)
+    assert_equal("integer", SortField::SortType::INTEGER.to_s)
+    assert_equal("float",   SortField::SortType::FLOAT.to_s)
   end
 end

data/test/unit/search/tc_index_searcher.rb

@@ -59,6 +59,7 @@ class IndexSearcherTest < Test::Unit::TestCase
     assert_equal(18, @is.max_doc)
     assert_equal("20050930", @is.doc(0).values(:date))
     assert_equal("cat1/sub2/subsub2", @is.doc(4)[:cat])
+    assert_equal("20051012", @is.doc(12)[:date])
   end
 
   def test_term_query
@@ -129,6 +130,13 @@ class IndexSearcherTest < Test::Unit::TestCase
     bq.add_query(tq2, BooleanClause::Occur::SHOULD)
     bq.add_query(tq3, BooleanClause::Occur::SHOULD)
     check_hits(bq, [1,2,3,4,6,8,11,14])
+
+    bq = BooleanQuery.new()
+    bc1 = BooleanClause.new(tq2, BooleanClause::Occur::SHOULD)
+    bc2 = BooleanClause.new(tq3, BooleanClause::Occur::SHOULD)
+    bq.add_clause(bc1)
+    bq.add_clause(bc2)
+    check_hits(bq, [1,2,3,4,6,8,11,14])
   end
 
   def test_phrase_query()

data/test/unit/search/tc_multi_searcher.rb

@@ -0,0 +1,275 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+require File.join(File.dirname(__FILE__), "tc_index_searcher.rb")
+
+# make sure a MultiSearcher searching only one index
+# passes all the IndexSearcher tests
+class SimpleMultiSearcherTest < IndexSearcherTest
+  alias :old_setup :setup 
+  def setup()
+    old_setup
+    @multi = MultiSearcher.new([IndexSearcher.new(@dir)])
+  end
+end
+
+
+# checks query results of a multisearcher searching two indexes
+# against those of a single indexsearcher searching the same
+# set of documents
+class MultiSearcherTest < Test::Unit::TestCase
+  include Ferret::Document
+  include Ferret::Search
+  include Ferret::Store
+  include Ferret::Analysis
+  include Ferret::Index
+
+  def prepare_search_docs(data)
+    docs = []
+    data.each_with_index do |fields, i|
+      doc = Document.new()
+      fields.each_pair do |field, text|
+        doc << Field.new(field, text, Field::Store::YES, Field::Index::TOKENIZED, Field::TermVector::NO, false)
+      end
+      docs << doc
+    end
+    return docs
+  end
+
+  def prepare_documents
+    @documents = prepare_search_docs([
+      {"date" => "20050930", "field" => "word1",
+        "cat" => "cat1/"},
+      {"date" => "20051001", "field" => "word1 word2 the quick brown fox",
+        "cat" => "cat1/sub1"},
+      {"date" => "20051002", "field" => "word1 word3",
+        "cat" => "cat1/sub1/subsub1"},
+      {"date" => "20051003", "field" => "word1 word3",
+        "cat" => "cat1/sub2"},
+      {"date" => "20051004", "field" => "word1 word2",
+        "cat" => "cat1/sub2/subsub2"},
+      {"date" => "20051005", "field" => "word1",
+        "cat" => "cat2/sub1"},
+      {"date" => "20051006", "field" => "word1 word3",
+        "cat" => "cat2/sub1"},
+      {"date" => "20051007", "field" => "word1",
+        "cat" => "cat2/sub1"},
+      {"date" => "20051008", "field" => "word1 word2 word3 the fast brown fox",
+        "cat" => "cat2/sub1"}
+    ])
+    @documents2 = prepare_search_docs([
+      {"date" => "20051009", "field" => "word1",
+        "cat" => "cat3/sub1"},
+      {"date" => "20051010", "field" => "word1",
+        "cat" => "cat3/sub1"},
+      {"date" => "20051011", "field" => "word1 word3 the quick red fox",
+        "cat" => "cat3/sub1"},
+      {"date" => "20051012", "field" => "word1",
+        "cat" => "cat3/sub1"},
+      {"date" => "20051013", "field" => "word1",
+        "cat" => "cat1/sub2"},
+      {"date" => "20051014", "field" => "word1 word3 the quick hairy fox",
+        "cat" => "cat1/sub1"},
+      {"date" => "20051015", "field" => "word1",
+        "cat" => "cat1/sub2/subsub1"},
+      {"date" => "20051016",
+        "field" => "word1 the quick fox is brown and hairy and a little red",
+        "cat" => "cat1/sub1/subsub2"},
+      {"date" => "20051017", "field" => "word1 the brown fox is quick and red",
+        "cat" => "cat1/"}
+    ]) 
+  end
+
+  def setup()
+    prepare_documents
+    # create MultiSearcher from two seperate searchers
+    dir1 = RAMDirectory.new()
+    iw1 = IndexWriter.new(dir1, :analyzer => WhiteSpaceAnalyzer.new(), :create => true)
+    @documents.each { |doc| iw1 << doc }
+    iw1.close()
+    
+    dir2 = RAMDirectory.new()
+    iw2 = IndexWriter.new(dir2, :analyzer => WhiteSpaceAnalyzer.new(), :create => true)
+    @documents2.each { |doc| iw2 << doc }
+    iw2.close()
+    @multi = Ferret::Search::MultiSearcher.new([IndexSearcher.new(dir1), IndexSearcher.new(dir2)])
+
+    # create single searcher
+    dir = RAMDirectory.new
+    iw = IndexWriter.new(dir, :analyzer => WhiteSpaceAnalyzer.new(), :create => true)
+    @documents.each { |doc| iw << doc }
+    @documents2.each { |doc| iw << doc }
+    iw.close
+    @single = IndexSearcher.new(dir)
+
+    @query_parser = Ferret::QueryParser.new(['date', 'field', 'cat'], :analyzer => WhiteSpaceAnalyzer.new())
+  end
+
+  def tear_down()
+    @multi.close
+    @single.close
+  end
+
+  def check_hits(query, debug_field=nil)
+    query = @query_parser.parse(query) if (query.is_a? String)
+    multi_docs = @multi.search(query)
+    single_docs = @single.search(query)
+    IndexTestHelper.explain(query, @single, debug_field) if debug_field
+    IndexTestHelper.explain(query, @multi, debug_field) if debug_field
+    assert_equal(single_docs.score_docs.size, multi_docs.score_docs.size, 'hit count')
+    assert_equal(single_docs.total_hits, multi_docs.total_hits, 'hit count')
+    
+    multi_docs.score_docs.each_with_index { |sd, id|
+      assert_equal(single_docs.score_docs[id].doc, sd.doc)
+      assert_equal(single_docs.score_docs[id].score, sd.score)
+    }
+  end
+
+  def test_get_doc()
+    assert_equal(18, @multi.max_doc)
+    assert_equal("20050930", @multi.doc(0).values(:date))
+    assert_equal("cat1/sub2/subsub2", @multi.doc(4)[:cat])
+    assert_equal("20051012", @multi.doc(12)[:date])
+    assert_equal(18, @single.max_doc)
+    assert_equal("20050930", @single.doc(0).values(:date))
+    assert_equal("cat1/sub2/subsub2", @single.doc(4)[:cat])
+    assert_equal("20051012", @single.doc(12)[:date])
+  end
+
+  def test_term_query
+    tq = TermQuery.new(Term.new("field", "word2"));
+    tq.boost = 100
+    check_hits(tq)
+
+    tq = TermQuery.new(Term.new("field", "2342"));
+    check_hits(tq)
+
+    tq = TermQuery.new(Term.new("field", ""));
+    check_hits(tq)
+
+    tq = TermQuery.new(Term.new("field", "word1"));
+    check_hits(tq)
+  end
+
+
+  def test_boolean_query
+    bq = BooleanQuery.new()
+    tq1 = TermQuery.new(Term.new("field", "word1"))
+    tq2 = TermQuery.new(Term.new("field", "word3"))
+    bq.add_query(tq1, BooleanClause::Occur::MUST)
+    bq.add_query(tq2, BooleanClause::Occur::MUST)
+    check_hits(bq)
+
+    tq3 = TermQuery.new(Term.new("field", "word2"))
+    bq.add_query(tq3, BooleanClause::Occur::SHOULD)
+    check_hits(bq)
+
+    bq = BooleanQuery.new()
+    bq.add_query(tq2, BooleanClause::Occur::MUST)
+    bq.add_query(tq3, BooleanClause::Occur::MUST_NOT)
+    check_hits(bq)
+
+    bq = BooleanQuery.new()
+    bq.add_query(tq2, BooleanClause::Occur::MUST_NOT)
+    check_hits(bq)
+
+    bq = BooleanQuery.new()
+    bq.add_query(tq2, BooleanClause::Occur::SHOULD)
+    bq.add_query(tq3, BooleanClause::Occur::SHOULD)
+    check_hits(bq)
+  end
+
+  def test_phrase_query()
+    pq = PhraseQuery.new()
+    t1 = Term.new("field", "quick")
+    t2 = Term.new("field", "brown")
+    t3 = Term.new("field", "fox")
+    pq << t1 << t2 << t3
+    check_hits(pq)
+
+    pq = PhraseQuery.new()
+    pq << t1
+    pq.add(t3, 2)
+    check_hits(pq)
+
+    pq.slop = 1
+    check_hits(pq)
+
+    pq.slop = 4
+    check_hits(pq)
+  end
+
+  def test_range_query()
+    rq = RangeQuery.new("date", "20051006", "20051010", true, true)
+    check_hits(rq)
+
+    rq = RangeQuery.new("date", "20051006", "20051010", false, true)
+    check_hits(rq)
+
+    rq = RangeQuery.new("date", "20051006", "20051010", true, false)
+    check_hits(rq)
+
+    rq = RangeQuery.new("date", "20051006", "20051010", false, false)
+    check_hits(rq)
+
+    rq = RangeQuery.new("date", nil, "20051003", false, true)
+    check_hits(rq)
+
+    rq = RangeQuery.new("date", nil, "20051003", false, false)
+    check_hits(rq)
+
+    rq = RangeQuery.new_less("date", "20051003", true)
+    check_hits(rq)
+
+    rq = RangeQuery.new_less("date", "20051003", false)
+    check_hits(rq)
+
+    rq = RangeQuery.new("date", "20051014", nil, true, false)
+    check_hits(rq)
+
+    rq = RangeQuery.new("date", "20051014", nil, false, false)
+    check_hits(rq)
+
+    rq = RangeQuery.new_more("date", "20051014", true)
+    check_hits(rq)
+
+    rq = RangeQuery.new_more("date", "20051014", false)
+    check_hits(rq)
+  end
+
+  def test_prefix_query()
+    t = Term.new("cat", "cat1")
+    pq = PrefixQuery.new(t)
+    check_hits(pq)
+
+    t.text = "cat1/sub2"
+    pq = PrefixQuery.new(t)
+    check_hits(pq)
+  end
+
+  def test_wildcard_query()
+    t = Term.new("cat", "cat1*")
+    wq = WildcardQuery.new(t)
+    check_hits(wq)
+
+    t.text = "cat1*/su??ub2"
+    wq = WildcardQuery.new(t)
+    check_hits(wq)
+  end
+
+  def test_multi_phrase_query()
+    t11 = Term.new("field", "quick")
+    t12 = Term.new("field", "fast")
+    t21 = Term.new("field", "brown")
+    t22 = Term.new("field", "red")
+    t23 = Term.new("field", "hairy")
+    t3 = Term.new("field", "fox")
+
+    mpq = MultiPhraseQuery.new()
+    mpq << [t11, t12]
+    mpq << [t21, t22, t23]
+    mpq << t3
+    check_hits(mpq)
+
+    mpq.slop = 4
+    check_hits(mpq)
+  end
+end

data/test/unit/search/tc_multi_searcher2.rb

@@ -0,0 +1,126 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+# Tests the multisearcher by comparing it's results
+# with those returned by an IndexSearcher.
+# Taken from TestMultiSearcherRanking.java of Lucene
+class MultiSearcher2Test < Test::Unit::TestCase
+  include Ferret::Document
+  include Ferret::Search
+  include Ferret::Store
+  include Ferret::Analysis
+  include Ferret::Index
+
+  FIELD_NAME = 'body'
+
+  def test_one_Term_query
+    check_query 'three'
+  end
+
+  def test_two_term_query
+    check_query 'three foo'
+    # as of 2006/03/11 these fail in Java Lucene as 
+    # well, hits are returned in slightly different order. 
+    #check_query '+pizza +blue*', :body
+    #check_query '+pizza blue*', :body
+    #check_query 'pizza blue*', :body
+  end
+
+  def test_prefix_query
+    check_query 'multi*'
+  end
+
+  def test_fuzzy_query
+    check_query 'multiThree~'
+  end
+
+  def test_range_query
+    check_query '{multiA multiP}'
+  end
+  
+  # fails (query parse error)
+  #def test_multi_phrase_query
+  #  check_query '"blueberry pi*"'
+  #end
+  
+  def test_nomatch_query
+    check_query '+three +nomatch'
+  end
+  
+  # this yields differing scores, but doesn't work in 
+  # Java Lucene either
+  #def test_term_repeated_query
+  #  check_query 'multi* multi* foo'
+  #end
+  
+  
+  def check_query(query_str, debug_field=nil)
+    @parser ||= Ferret::QueryParser.new(FIELD_NAME, :analyzer => @analyzer)
+    query = @parser.parse(query_str)
+    puts "Query: #{query}" if debug_field
+    IndexTestHelper.explain(query, @multi, debug_field) if debug_field
+    IndexTestHelper.explain(query, @single, debug_field) if debug_field
+    multi_hits = @multi.search(query)
+    single_hits = @single.search(query)
+    assert_equal single_hits.size, multi_hits.size, "hit count differs"
+    multi_hits.score_docs.each_with_index { |multi_sd, i|
+      single_sd = single_hits.score_docs[i]
+      doc_multi = @multi.doc(multi_sd.doc)
+      doc_single = @single.doc(single_sd.doc)
+      assert_equal single_sd.score, multi_sd.score, "score differs in result #{i}"
+      assert_equal doc_single[FIELD_NAME], doc_multi[FIELD_NAME], "field values differ in result #{i}"
+    }
+  end
+
+  def setup()
+    @analyzer = WhiteSpaceAnalyzer.new()
+    # create MultiSearcher from two seperate searchers
+    d1 = RAMDirectory.new()
+    iw1 = IndexWriter.new(d1, :analyzer => @analyzer, :create => true)
+    add_collection1(iw1)
+    iw1.close()
+    
+    d2 = RAMDirectory.new()
+    iw2 = IndexWriter.new(d2, :analyzer => @analyzer, :create => true)
+    add_collection2(iw2)
+    iw2.close()
+    @multi = MultiSearcher.new([IndexSearcher.new(d1), IndexSearcher.new(d2)])
+
+    # create IndexSearcher which contains all documents
+    d = RAMDirectory.new()
+    iw = IndexWriter.new(d, :analyzer => @analyzer, :create => true)
+    add_collection1(iw)
+    add_collection2(iw)
+    iw.close()
+    @single = IndexSearcher.new(d)
+  end
+
+  def tear_down()
+    @multi.close
+    @single.close
+  end
+
+  def add(value, iw)
+    d = Document.new
+    d << Field.new(FIELD_NAME, value, Field::Store::YES, Field::Index::TOKENIZED)
+    iw << d
+  end
+
+  def add_collection1(iw)
+    add("one blah three", iw)
+    add("one foo three multiOne", iw)
+    add("one foobar three multiThree", iw)
+    add("blueberry pie", iw)
+    add("blueberry strudel", iw)
+    add("blueberry pizza", iw)
+  end
+  def add_collection2(iw)
+    add("two blah three", iw)
+    add("two foo xxx multiTwo", iw)
+    add("two foobar xxx multiThreee", iw)
+    add("blueberry chewing gum", iw)
+    add("bluebird pizza", iw)
+    add("bluebird foobar pizza", iw)
+    add("piccadilly circus", iw)
+  end
+
+end