sdsykes-ferret 0.11.6.19

data/test/unit/search/tc_index_searcher.rb

@@ -0,0 +1,61 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+require File.dirname(__FILE__) + "/tm_searcher"
+
+class SearcherTest < Test::Unit::TestCase
+  include Ferret::Search
+  include Ferret::Store
+  include Ferret::Analysis
+  include Ferret::Index
+
+  include SearcherTests
+
+  def setup()
+    @dir = RAMDirectory.new()
+    iw = IndexWriter.new(:dir => @dir,
+                         :analyzer => WhiteSpaceAnalyzer.new(),
+                         :create => true)
+    @documents = IndexTestHelper::SEARCH_TEST_DOCS
+    @documents.each { |doc| iw << doc }
+    iw.close()
+    @searcher = Searcher.new(@dir)
+  end
+
+  def teardown()
+    @searcher.close
+    @dir.close()
+  end
+
+  def get_docs(hits)
+    docs = []
+    hits.each do |hit|
+      docs << hit.doc
+    end
+    docs
+  end
+
+  def check_hits(query, expected, top=nil, total_hits=nil)
+    options = {}
+    options[:limit] = expected.size + 1 if (expected.size > 10) 
+    top_docs = @searcher.search(query, options)
+    assert_equal(expected.length, top_docs.hits.size)
+    assert_equal(top, top_docs.hits[0].doc) if top
+    if total_hits
+      assert_equal(total_hits, top_docs.total_hits)
+    else
+      assert_equal(expected.length, top_docs.total_hits)
+    end
+    top_docs.hits.each do |score_doc|
+      assert(expected.include?(score_doc.doc),
+             "#{score_doc.doc} was found unexpectedly")
+      assert(score_doc.score.approx_eql?(@searcher.explain(query, score_doc.doc).score), 
+        "Scores(#{score_doc.score} != #{@searcher.explain(query, score_doc.doc).score})")
+    end
+  end
+
+  def test_get_doc()
+    assert_equal(18, @searcher.max_doc)
+    assert_equal("20050930", @searcher.get_document(0)[:date])
+    assert_equal("cat1/sub2/subsub2", @searcher.get_document(4)[:category])
+    assert_equal("20051012", @searcher.get_document(12)[:date])
+  end
+end

data/test/unit/search/tc_multi_searcher.rb

@@ -0,0 +1,128 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+require File.join(File.dirname(__FILE__), "tc_index_searcher.rb")
+
+# make sure a MultiSearcher searching only one index
+# passes all the Searcher tests
+class SimpleMultiSearcherTest < SearcherTest
+  alias :old_setup :setup 
+  def setup()
+    old_setup
+    @searcher = MultiSearcher.new([Searcher.new(@dir)])
+  end
+end
+
+
+# checks query results of a multisearcher searching two indexes
+# against those of a single indexsearcher searching the same
+# set of documents
+class MultiSearcherTest < Test::Unit::TestCase
+  include Ferret::Search
+  include Ferret::Store
+  include Ferret::Analysis
+  include Ferret::Index
+
+  include SearcherTests
+
+  DOCUMENTS1 = [
+    {"date" => "20050930", :field => "word1",
+      "cat" => "cat1/"},
+    {"date" => "20051001", :field => "word1 word2 the quick brown fox",
+      "cat" => "cat1/sub1"},
+    {"date" => "20051002", :field => "word1 word3",
+      "cat" => "cat1/sub1/subsub1"},
+    {"date" => "20051003", :field => "word1 word3",
+      "cat" => "cat1/sub2"},
+    {"date" => "20051004", :field => "word1 word2",
+      "cat" => "cat1/sub2/subsub2"},
+    {"date" => "20051005", :field => "word1",
+      "cat" => "cat2/sub1"},
+    {"date" => "20051006", :field => "word1 word3",
+      "cat" => "cat2/sub1"},
+    {"date" => "20051007", :field => "word1",
+      "cat" => "cat2/sub1"},
+    {"date" => "20051008", :field => "word1 word2 word3 the fast brown fox",
+      "cat" => "cat2/sub1"}
+  ]
+
+  DOCUMENTS2 = [
+    {"date" => "20051009", :field => "word1",
+      "cat" => "cat3/sub1"},
+    {"date" => "20051010", :field => "word1",
+      "cat" => "cat3/sub1"},
+    {"date" => "20051011", :field => "word1 word3 the quick red fox",
+      "cat" => "cat3/sub1"},
+    {"date" => "20051012", :field => "word1",
+      "cat" => "cat3/sub1"},
+    {"date" => "20051013", :field => "word1",
+      "cat" => "cat1/sub2"},
+    {"date" => "20051014", :field => "word1 word3 the quick hairy fox",
+      "cat" => "cat1/sub1"},
+    {"date" => "20051015", :field => "word1",
+      "cat" => "cat1/sub2/subsub1"},
+    {"date" => "20051016",
+      :field => "word1 the quick fox is brown and hairy and a little red",
+      "cat" => "cat1/sub1/subsub2"},
+    {"date" => "20051017", :field => "word1 the brown fox is quick and red",
+      "cat" => "cat1/"}
+  ]
+
+  def setup()
+    # create MultiSearcher from two seperate searchers
+    dir1 = RAMDirectory.new()
+    iw1 = IndexWriter.new(:dir => dir1,
+                          :analyzer => WhiteSpaceAnalyzer.new(),
+                          :create => true)
+    DOCUMENTS1.each { |doc| iw1 << doc }
+    iw1.close()
+    
+    dir2 = RAMDirectory.new()
+    iw2 = IndexWriter.new(:dir => dir2,
+                          :analyzer => WhiteSpaceAnalyzer.new(),
+                          :create => true)
+    DOCUMENTS2.each { |doc| iw2 << doc }
+    iw2.close()
+    @searcher = Ferret::Search::MultiSearcher.new([Searcher.new(dir1),
+                                                   Searcher.new(dir2)])
+
+    # create single searcher
+    dir = RAMDirectory.new
+    iw = IndexWriter.new(:dir => dir,
+                         :analyzer => WhiteSpaceAnalyzer.new(),
+                         :create => true)
+    DOCUMENTS1.each { |doc| iw << doc }
+    DOCUMENTS2.each { |doc| iw << doc }
+    iw.close
+    @single = Searcher.new(dir)
+
+    #@query_parser = Ferret::QueryParser.new([:date, :field, :cat], :analyzer => WhiteSpaceAnalyzer.new())
+  end
+
+  def teardown()
+    @searcher.close
+    @single.close
+  end
+
+  def check_hits(query, ignore1, ignore2 = nil, ignore3 = nil)
+    multi_docs = @searcher.search(query)
+    single_docs = @single.search(query)
+    assert_equal(single_docs.hits.size, multi_docs.hits.size, 'hit count')
+    assert_equal(single_docs.total_hits, multi_docs.total_hits, 'hit count')
+    
+    multi_docs.hits.each_with_index { |sd, id|
+      assert_equal(single_docs.hits[id].doc, sd.doc)
+      assert(single_docs.hits[id].score.approx_eql?(sd.score), 
+             "#{single_docs.hits[id]} != #{sd.score}")
+    }
+  end
+
+  def test_get_doc()
+    assert_equal(18, @searcher.max_doc)
+    assert_equal("20050930", @searcher.get_document(0)[:date])
+    assert_equal("cat1/sub2/subsub2", @searcher[4][:cat])
+    assert_equal("20051012", @searcher.get_document(12)[:date])
+    assert_equal(18, @single.max_doc)
+    assert_equal("20050930", @single.get_document(0)[:date])
+    assert_equal("cat1/sub2/subsub2", @single[4][:cat])
+    assert_equal("20051012", @single.get_document(12)[:date])
+  end
+end

data/test/unit/search/tc_multiple_search_requests.rb

@@ -0,0 +1,58 @@
+require File.dirname(__FILE__) + "/../../test_helper" 
+
+class MultipleSearchRequestsTest < Test::Unit::TestCase 
+  include Ferret::Search 
+  include Ferret::Store 
+  include Ferret::Analysis 
+  include Ferret::Index 
+
+  def setup() 
+    dpath = File.expand_path(File.join(File.dirname(__FILE__), 
+                       '../../temp/fsdir')) 
+    fs_dir = Ferret::Store::FSDirectory.new(dpath, true) 
+
+    iw = IndexWriter.new(:dir => fs_dir, :create => true, :key => [:id]) 
+    1000.times do |x| 
+      doc = {:id => x}
+      iw << doc 
+    end 
+    iw.close() 
+    fs_dir.close() 
+
+    @ix = Index.new(:path => dpath, :create => true, :key => [:id]) 
+  end 
+
+  def tear_down() 
+    @ix.close 
+  end 
+
+  def test_repeated_queries_segmentation_fault 
+    1000.times do |x| 
+      bq = BooleanQuery.new() 
+      tq1 = TermQuery.new(:id, 1) 
+      tq2 = TermQuery.new(:another_id, 1)
+      bq.add_query(tq1, :must) 
+      bq.add_query(tq2, :must) 
+      top_docs = @ix.search(bq) 
+    end 
+  end 
+
+  def test_repeated_queries_bus_error 
+    1000.times do |x| 
+      bq = BooleanQuery.new() 
+      tq1 = TermQuery.new(:id, '1')
+      tq2 = TermQuery.new(:another_id, '1')
+      tq3 = TermQuery.new(:yet_another_id, '1')
+      tq4 = TermQuery.new(:still_another_id, '1')
+      tq5 = TermQuery.new(:one_more_id, '1')
+      tq6 = TermQuery.new(:and_another_id, '1')
+      bq.add_query(tq1, :must) 
+      bq.add_query(tq2, :must) 
+      bq.add_query(tq3, :must) 
+      bq.add_query(tq4, :must) 
+      bq.add_query(tq5, :must) 
+      bq.add_query(tq6, :must) 
+      top_docs = @ix.search(bq) 
+    end 
+  end 
+end 

data/test/unit/search/tc_search_and_sort.rb

@@ -0,0 +1,179 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+class SearchAndSortTest < Test::Unit::TestCase
+  include Ferret::Search
+  include Ferret::Store
+  include Ferret::Analysis
+  include Ferret::Index
+
+  def setup()
+    @dir = RAMDirectory.new()
+    iw = IndexWriter.new(:dir => @dir,
+                         :analyzer => WhiteSpaceAnalyzer.new(),
+                         :create => true, :min_merge_docs => 3)
+    [                                                                      # len mod
+      {:x => "findall", :string => "a", :int => "6", :float => "0.01"},    #  4   0
+      {:x => "findall", :string => "c", :int => "5", :float => "0.1"},     #  3   3
+      {:x => "findall", :string => "e", :int => "2", :float => "0.001"},   #  5   1
+      {:x => "findall", :string => "g", :int => "1", :float => "1.0"},     #  3   3
+      {:x => "findall", :string => nil, :int => "3", :float => "0.0001"},  #  6   2
+      {:x => "findall", :string => "",  :int => "4", :float => "10.0"},    #  4   0
+      {:x => "findall", :string => "h", :int => "5", :float => "0.00001"}, #  7   3
+      {:x => "findall", :string => "f", :int => "2", :float => "100.0"},   #  5   1
+      {:x => "findall", :string => "d", :int => "3", :float => "1000.0"},  #  6   2
+      {:x => "findall", :string => "b", :int => "4", :float => "0.000001"} #  8   0
+    ].each do |doc|
+      doc.extend(Ferret::BoostMixin)
+      doc.boost = doc[:float].to_f
+      iw << doc
+    end
+    iw.close
+  end
+
+  def teardown()
+    @dir.close()
+  end
+
+  def do_test_top_docs(is, query, expected, sort = nil)
+    top_docs = is.search(query, {:sort => sort})
+    top_docs.total_hits.times do |i|
+      assert_equal(expected[i], top_docs.hits[i].doc)
+    end
+
+    # test sorting works for smaller ranged query
+    offset = 3
+    limit = 3
+    top_docs = is.search(query, {:sort => sort,
+                                 :offset => offset,
+                                 :limit => limit})
+    limit.times do |i|
+      assert_equal(expected[offset + i], top_docs.hits[i].doc)
+    end
+  end
+
+  def test_sort_field_to_s()
+    assert_equal("<SCORE>", SortField::SCORE.to_s);
+    sf = SortField.new("MyScore",
+                       {:type => :score,
+                        :reverse => true})
+    assert_equal("MyScore:<SCORE>!", sf.to_s)
+    assert_equal("<DOC>", SortField::DOC_ID.to_s);
+    sf = SortField.new("MyDoc",
+                       {:type => :doc_id,
+                        :reverse => true})
+    assert_equal("MyDoc:<DOC>!", sf.to_s)
+    sf = SortField.new(:date,
+                       {:type => :integer})
+    assert_equal("date:<integer>", sf.to_s)
+    sf = SortField.new(:date,
+                       {:type => :integer,
+                        :reverse => true})
+    assert_equal("date:<integer>!", sf.to_s)
+    sf = SortField.new(:price,
+                       {:type => :float})
+    assert_equal("price:<float>", sf.to_s)
+    sf = SortField.new(:price,
+                       {:type => :float,
+                        :reverse => true})
+    assert_equal("price:<float>!", sf.to_s)
+    sf = SortField.new(:content,
+                       {:type => :string})
+    assert_equal("content:<string>", sf.to_s)
+    sf = SortField.new(:content,
+                       {:type => :string,
+                        :reverse => true})
+    assert_equal("content:<string>!", sf.to_s)
+    sf = SortField.new(:auto_field,
+                       {:type => :auto})
+    assert_equal("auto_field:<auto>", sf.to_s)
+    sf = SortField.new(:auto_field,
+                       {:type => :auto,
+                        :reverse => true})
+    assert_equal("auto_field:<auto>!", sf.to_s)
+  end
+  
+  def test_sort_to_s()
+    sort = Sort.new
+    assert_equal("Sort[<SCORE>, <DOC>]", sort.to_s)
+    sf = SortField.new(:auto_field,
+                       {:type => :auto,
+                        :reverse => true})
+    sort = Sort.new([sf, SortField::SCORE, SortField::DOC_ID])
+    assert_equal("Sort[auto_field:<auto>!, <SCORE>, <DOC>]", sort.to_s)
+    sort = Sort.new([:one, :two, SortField::DOC_ID])
+    assert_equal("Sort[one:<auto>, two:<auto>, <DOC>]", sort.to_s)
+    sort = Sort.new([:one, :two])
+    assert_equal("Sort[one:<auto>, two:<auto>, <DOC>]", sort.to_s)
+  end
+
+
+  def test_sorts()
+    is = Searcher.new(@dir)
+    q = TermQuery.new(:x, "findall")
+    do_test_top_docs(is, q, [8,7,5,3,1,0,2,4,6,9])
+    do_test_top_docs(is, q, [8,7,5,3,1,0,2,4,6,9], Sort::RELEVANCE)
+    do_test_top_docs(is, q, [8,7,5,3,1,0,2,4,6,9], [SortField::SCORE])
+    do_test_top_docs(is, q, [0,1,2,3,4,5,6,7,8,9], Sort::INDEX_ORDER)
+    do_test_top_docs(is, q, [0,1,2,3,4,5,6,7,8,9], [SortField::DOC_ID])
+
+    ## int
+    sf_int = SortField.new(:int, {:type => :integer, :reverse => true})
+    do_test_top_docs(is, q, [0,1,6,5,9,4,8,2,7,3], [sf_int])
+    do_test_top_docs(is, q, [0,1,6,5,9,4,8,2,7,3], "int DESC")
+    do_test_top_docs(is, q, [0,1,6,5,9,8,4,7,2,3], [sf_int, SortField::SCORE])
+    do_test_top_docs(is, q, [0,1,6,5,9,8,4,7,2,3], "int DESC, SCORE")
+    sf_int = SortField.new(:int, {:type => :integer})
+    do_test_top_docs(is, q, [3,2,7,4,8,5,9,1,6,0], [sf_int])
+    do_test_top_docs(is, q, [3,2,7,4,8,5,9,1,6,0], "int")
+
+    ## byte
+    do_test_top_docs(is, q, [3,2,7,4,8,5,9,1,6,0],
+                     SortField.new(:int, :type => :byte))
+    do_test_top_docs(is, q, [0,1,6,5,9,4,8,2,7,3],
+                     [SortField.new(:int, :type => :byte, :reverse => true)])
+    
+
+    ## float
+    sf_float = SortField.new(:float, {:type => :float, :reverse => true})
+    do_test_top_docs(is, q, [8,7,5,3,1,0,2,4,6,9],
+                     Sort.new([sf_float, SortField::SCORE]))
+    do_test_top_docs(is, q, [8,7,5,3,1,0,2,4,6,9], "float DESC, SCORE")
+    sf_float = SortField.new(:float, {:type => :float})
+    do_test_top_docs(is, q, [9,6,4,2,0,1,3,5,7,8],
+                     Sort.new([sf_float, SortField::SCORE]))
+    do_test_top_docs(is, q, [9,6,4,2,0,1,3,5,7,8], "float, SCORE")
+
+    ## str
+    sf_str = SortField.new(:string, {:type => :string})
+    do_test_top_docs(is, q, [0,9,1,8,2,7,3,6,5,4], [sf_str, SortField::SCORE])
+    do_test_top_docs(is, q, [0,9,1,8,2,7,3,6,4,5], "string")
+
+    ## auto
+    do_test_top_docs(is, q, [0,9,1,8,2,7,3,6,4,5], Sort.new(:string))
+    do_test_top_docs(is, q, [3,2,7,4,8,5,9,1,6,0], Sort.new([:int]))
+    do_test_top_docs(is, q, [9,6,4,2,0,1,3,5,7,8], Sort.new(:float))
+    do_test_top_docs(is, q, [9,6,4,2,0,1,3,5,7,8], :float)
+    do_test_top_docs(is, q, [8,7,5,3,1,0,2,4,6,9], Sort.new(:float, true))
+    do_test_top_docs(is, q, [0,6,1,5,9,4,8,7,2,3], Sort.new([:int, :string], true))
+    do_test_top_docs(is, q, [0,6,1,5,9,4,8,7,2,3], "int DESC, string DESC")
+    do_test_top_docs(is, q, [3,2,7,8,4,9,5,1,6,0], Sort.new([:int, :string]))
+    do_test_top_docs(is, q, [3,2,7,8,4,9,5,1,6,0], [:int, :string])
+    do_test_top_docs(is, q, [3,2,7,8,4,9,5,1,6,0], "int, string")
+  end
+
+  #LENGTH = SortField::SortType.new("length", lambda{|str| str.length})
+  #LENGTH_MODULO = SortField::SortType.new("length_mod", lambda{|str| str.length},
+  #                                        lambda{|i, j| (i%4) <=> (j%4)})
+  #def test_special_sorts
+  #  is = IndexSearcher.new(@dir)
+  #  q = TermQuery.new(Term.new(:x, "findall"))
+  #  sf = SortField.new(:float, {:type => LENGTH, :reverse => true})
+  #  do_test_top_docs(is, q, [9,6,4,8,2,7,0,5,1,3], [sf])
+  #  sf = SortField.new(:float, {:type => LENGTH_MODULO, :reverse => true})
+  #  do_test_top_docs(is, q, [1,3,6,4,8,2,7,0,5,9], [sf])
+  #  sf = SortField.new(:float, {:type => LENGTH,
+  #                               :reverse => true,
+  #                               :comparator => lambda{|i,j| (j%4) <=> (i%4)}})
+  #  do_test_top_docs(is, q, [0,5,9,2,7,4,8,1,3,6], [sf])
+  #end
+end

data/test/unit/search/tc_sort.rb

@@ -0,0 +1,49 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+class SortTest < Test::Unit::TestCase
+  include Ferret::Search
+
+  def test_basic()
+    s = Sort::RELEVANCE
+    assert_equal(2, s.fields.size)
+    assert_equal(SortField::SCORE, s.fields[0])
+    assert_equal(SortField::DOC_ID, s.fields[1])
+
+    s = Sort::INDEX_ORDER
+    assert_equal(1, s.fields.size)
+    assert_equal(SortField::DOC_ID, s.fields[0])
+  end
+
+  def test_string_init()
+    s = Sort.new(:field)
+    assert_equal(2, s.fields.size)
+    assert_equal(:auto, s.fields[0].type)
+    assert_equal(:field, s.fields[0].name)
+    assert_equal(SortField::DOC_ID, s.fields[1])
+
+    s = Sort.new([:field1, :field2, :field3])
+    assert_equal(4, s.fields.size)
+    assert_equal(:auto, s.fields[0].type)
+    assert_equal(:field1, s.fields[0].name)
+    assert_equal(:auto, s.fields[1].type)
+    assert_equal(:field2, s.fields[1].name)
+    assert_equal(:auto, s.fields[2].type)
+    assert_equal(:field3, s.fields[2].name)
+    assert_equal(SortField::DOC_ID, s.fields[3])
+  end
+
+  def test_multi_fields()
+    sf1 = SortField.new(:field, {:type => :integer,
+                                 :reverse => true})
+    sf2 = SortField::SCORE
+    sf3 = SortField::DOC_ID
+    s = Sort.new([sf1, sf2, sf3])
+
+    assert_equal(3, s.fields.size)
+    assert_equal(:integer, s.fields[0].type)
+    assert_equal(:field, s.fields[0].name)
+    assert(s.fields[0].reverse?)
+    assert_equal(SortField::SCORE, s.fields[1])
+    assert_equal(SortField::DOC_ID, s.fields[2])
+  end
+end