ferret 0.1.0

data/test/unit/query_parser/tc_query_parser.rb

@@ -0,0 +1,84 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+class QueryParserTest < Test::Unit::TestCase
+
+
+  def setup()
+    @parser = Ferret::QueryParser.new("xxx")
+  end
+
+  def test_strings()
+    pairs = [
+      ['word', 'word'],
+      ['field:word', 'field:word'],
+      ['"word1 word2 word3"', '"word word word"'],
+      ['"word1 2342 word3"', '"word word"'],
+      ['field:"one two three"', 'field:"one two three"'],
+      ['field:"one 222 three"', 'field:"one three"'],
+      ['field:"one <> three"', 'field:"one <> three"'],
+      ['field:"one <> three <>"', 'field:"one <> three"'],
+      ['field:"one <> <> <> three <>"', 'field:"one <> <> <> three"'],
+      ['field:"one <> <> <> three|four|five <>"', 'field:"one <> <> <> three|four|five"'],
+      ['field:"one|two three|four|five six|seven"', 'field:"one|two three|four|five six|seven"'],
+      ['[aaa bbb]', '[aaa bbb]'],
+      ['{aaa bbb]', '{aaa bbb]'],
+      ['field:[aaa bbb}', 'field:[aaa bbb}'],
+      ['{aaa bbb}', '{aaa bbb}'],
+      ['{aaa|', '{aaa|'],
+      ['[aaa|', '[aaa|'],
+      ['field:|aaa}', 'field:|aaa}'],
+      ['|aaa]', '|aaa]'],
+      ['>aaa', '{aaa|'],
+      ['>=aaa', '[aaa|'],
+      ['<aaa', '|aaa}'],
+      ['field:<=aaa', 'field:|aaa]'],
+      ['REQ one REQ two', '+one +two'],
+      ['REQ one two', '+one two'],
+      ['one REQ two', 'one +two'],
+      ['+one +two', '+one +two'],
+      ['+one two', '+one two'],
+      ['one +two', 'one +two'],
+      ['-one -two', '-one -two'],
+      ['-one two', '-one two'],
+      ['one -two', 'one -two'],
+      ['!one !two', '-one -two'],
+      ['!one two', '-one two'],
+      ['one !two', 'one -two'],
+      ['NOT one NOT two', '-one -two'],
+      ['NOT one two', '-one two'],
+      ['one NOT two', 'one -two'],
+      ['one two', 'one two'],
+      ['one OR two', 'one two'],
+      ['one AND two', '+one +two'],
+      ['one two AND three', 'one two +three'],
+      ['one two OR three', 'one two three'],
+      ['one (two AND three)', 'one (+two +three)'],
+      ['one AND (two OR three)', '+one +(two three)'],
+      ['field:(one AND (two OR three))', '+field:one +(field:two field:three)'],
+      ['one AND (two OR [aaa vvv})', '+one +(two [aaa vvv})'],
+      ['one AND (one:two OR two:three) AND four', '+one +(one:two two:three) +four'],
+      ['one^1.23', 'one^1.23'],
+      ['(one AND two)^100.23', '(+one +two)^100.23'],
+      ['field:(one AND two)^100.23', '(+field:one +field:two)^100.23'],
+      ['field:(one AND [aaa bbb]^23.3)^100.23', '(+field:one +field:[aaa bbb]^23.3)^100.23'],
+      ['(REQ field:"one two three")^23', 'field:"one two three"^23.0'],
+      ['asdf~0.2', 'asdf~0.2'],
+      ['field:asdf~0.2', 'field:asdf~0.2'],
+      ['asdf~0.2^100.0', 'asdf~0.2^100.0'],
+      ['field:asdf~0.2^0.1', 'field:asdf~0.2^0.1'],
+      ['field:"asdf <> asdf|asdf"~4', 'field:"asdf <> asdf|asdf"~4'],
+      ['"one two three four five"~5', '"one two three four five"~5'],
+      ['ab?de', 'ab?de'],
+      ['ab*de', 'ab*de'],
+      ['asdf?*?asd*dsf?asfd*asdf?', 'asdf?*?asd*dsf?asfd*asdf?'],
+      ['field:a* AND field:(b*)', '+field:a* +field:b*'],
+      ['field:abc~ AND field:(b*)', '+field:abc~0.5 +field:b*'],
+      ['asdf?*?asd*dsf?asfd*asdf?^20.0', 'asdf?*?asd*dsf?asfd*asdf?^20.0']
+    ]
+
+      
+    pairs.each do |pair|
+      assert_equal(pair[1], @parser.parse(pair[0]).to_s(@parser.default_field))
+    end
+  end
+end

data/test/unit/search/tc_filter.rb

@@ -0,0 +1,113 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+
+class FilterTest < Test::Unit::TestCase
+  include Ferret::Document
+  include Ferret::Search
+  include Ferret::Analysis
+  include Ferret::Index
+
+  def add_doc(hash, writer)
+    doc = Document.new()
+    hash.each_pair do |field, text|
+      doc << Field.new(field, text, Field::Store::NO, Field::Index::UNTOKENIZED)
+    end
+    writer << doc
+  end
+
+  def setup()
+    @dir = Ferret::Store::RAMDirectory.new()
+    iw = IndexWriter.new(@dir,
+                         :analyzer => WhiteSpaceAnalyzer.new(),
+                         :create => true)
+    docs = [ 
+      {"int"=>"0","date"=>"20040601","switch"=>"on"},
+      {"int"=>"1","date"=>"20041001","switch"=>"off"},
+      {"int"=>"2","date"=>"20051101","switch"=>"on"},
+      {"int"=>"3","date"=>"20041201","switch"=>"off"},
+      {"int"=>"4","date"=>"20051101","switch"=>"on"},
+      {"int"=>"5","date"=>"20041201","switch"=>"off"},
+      {"int"=>"6","date"=>"20050101","switch"=>"on"},
+      {"int"=>"7","date"=>"20040701","switch"=>"off"},
+      {"int"=>"8","date"=>"20050301","switch"=>"on"},
+      {"int"=>"9","date"=>"20050401","switch"=>"off"}
+    ]
+    docs.each {|doc| add_doc(doc, iw)}
+    iw.close
+  end
+
+  def tear_down()
+    @dir.close()
+  end
+
+  def do_test_top_docs(is, query, expected, filter)
+    top_docs = is.search(query, {:filter => filter})
+    #puts top_docs
+    assert_equal(expected.size, top_docs.score_docs.size)
+    top_docs.total_hits.times do |i|
+      assert_equal(expected[i], top_docs.score_docs[i].doc)
+    end
+  end
+
+  def test_range_filter
+    is = IndexSearcher.new(@dir)
+    q = MatchAllDocsQuery.new()
+    rf = RangeFilter.new("int", "2", "6", true, true)
+    do_test_top_docs(is, q, [2,3,4,5,6], rf)
+    rf = RangeFilter.new("int", "2", "6", true, false)
+    do_test_top_docs(is, q, [2,3,4,5], rf)
+    rf = RangeFilter.new("int", "2", "6", false, true)
+    do_test_top_docs(is, q, [3,4,5,6], rf)
+    rf = RangeFilter.new("int", "2", "6", false, false)
+    do_test_top_docs(is, q, [3,4,5], rf)
+    rf = RangeFilter.new_more("int", "6")
+    do_test_top_docs(is, q, [6,7,8,9], rf)
+    rf = RangeFilter.new_more("int", "6", false)
+    do_test_top_docs(is, q, [7,8,9], rf)
+    rf = RangeFilter.new_less("int", "2")
+    do_test_top_docs(is, q, [0,1,2], rf)
+    rf = RangeFilter.new_less("int", "2", false)
+    do_test_top_docs(is, q, [0,1], rf)
+  end
+
+  def test_range_filter_errors
+    assert_raise(ArgumentError) {f = RangeFilter.new("", "asd", nil, false, true)}
+    assert_raise(ArgumentError) {f = RangeFilter.new("", nil, "asd", true, false)}
+    assert_raise(ArgumentError) {f = RangeFilter.new("", "ac", "ab", false, false)}
+    assert_raise(ArgumentError) {f = RangeFilter.new("", nil, nil, false, false)}
+  end
+
+  def test_query_filter()
+    is = IndexSearcher.new(@dir)
+    q = MatchAllDocsQuery.new()
+    qf = QueryFilter.new(TermQuery.new(Term.new("switch", "on")))
+    do_test_top_docs(is, q, [0,2,4,6,8], qf)
+    # test again to test caching doesn't break it
+    do_test_top_docs(is, q, [0,2,4,6,8], qf)
+    qf = QueryFilter.new(TermQuery.new(Term.new("switch", "off")))
+    do_test_top_docs(is, q, [1,3,5,7,9], qf)
+  end
+
+  def test_caching_wrapper_filter
+    is = IndexSearcher.new(@dir)
+    q = MatchAllDocsQuery.new()
+    rf = RangeFilter.new("int", "2", "6", true, true)
+    cf = CachingWrapperFilter.new(rf)
+    #puts "about to test cache"
+    do_test_top_docs(is, q, [2,3,4,5,6], cf)
+    do_test_top_docs(is, q, [2,3,4,5,6], cf)
+    #puts "finished_testing_cache"
+  end
+
+  def test_filtered_query
+    is = IndexSearcher.new(@dir)
+    q = MatchAllDocsQuery.new()
+    rf = RangeFilter.new("int", "2", "6", true, true)
+    rq = FilteredQuery.new(q, rf)
+    qf = QueryFilter.new(TermQuery.new(Term.new("switch", "on")))
+    do_test_top_docs(is, rq, [2,4,6], qf)
+    query = FilteredQuery.new(rq, qf)
+    rf2 = RangeFilter.new_more("int", "3")
+    do_test_top_docs(is, query, [4,6], rf2)
+  end
+end

data/test/unit/search/tc_fuzzy_query.rb

@@ -0,0 +1,136 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+class FuzzyQueryTest < Test::Unit::TestCase
+  include Ferret::Document
+  include Ferret::Search
+  include Ferret::Store
+  include Ferret::Analysis
+  include Ferret::Index
+
+  def add_doc(text, writer)
+    doc = Document.new()
+    doc << Field.new("field", text, Field::Store::NO, Field::Index::TOKENIZED)
+    writer << doc
+  end
+
+  def setup()
+    @dir = RAMDirectory.new()
+  end
+
+  def tear_down()
+    @dir.close()
+  end
+
+  def do_test_top_docs(is, query, expected)
+    top_docs = is.search(query)
+    assert_equal(expected.length, top_docs.total_hits,
+                "expected #{expected.length} hits but got #{top_docs.total_hits}")
+    assert_equal(expected.length, top_docs.score_docs.size)
+    top_docs.total_hits.times do |i|
+      assert_equal(expected[i], top_docs.score_docs[i].doc)
+    end
+  end
+
+  def do_prefix_test(is, text, prefix, expected)
+    fq = FuzzyQuery.new(Term.new("field", text), FuzzyQuery.default_min_similarity, prefix)
+    #puts is.explain(fq, 0)
+    #puts is.explain(fq, 1)
+    do_test_top_docs(is, fq, expected)
+  end
+
+  def test_fuzziness()
+    iw = IndexWriter.new(@dir, :analyzer => WhiteSpaceAnalyzer.new(), :create => true)
+    add_doc("aaaaa", iw)
+    add_doc("aaaab", iw)
+    add_doc("aaabb", iw)
+    add_doc("aabbb", iw)
+    add_doc("abbbb", iw)
+    add_doc("bbbbb", iw)
+    add_doc("ddddd", iw)
+    #iw.optimize()
+    iw.close()
+
+
+    is = IndexSearcher.new(@dir)
+
+    fq = FuzzyQuery.new(Term.new("field", "aaaaa"), FuzzyQuery.default_min_similarity, 5)
+
+    do_prefix_test(is, "aaaaa", 0, [0,1,2])
+    do_prefix_test(is, "aaaaa", 1, [0,1,2])
+    do_prefix_test(is, "aaaaa", 2, [0,1,2])
+    do_prefix_test(is, "aaaaa", 3, [0,1,2])
+    do_prefix_test(is, "aaaaa", 4, [0,1])
+    do_prefix_test(is, "aaaaa", 5, [0])
+    do_prefix_test(is, "aaaaa", 6, [0])
+
+    do_prefix_test(is, "xxxxx", 0, [])
+
+    do_prefix_test(is, "aaccc", 0, [])
+
+    do_prefix_test(is, "aaaac", 0, [0,1,2])
+    do_prefix_test(is, "aaaac", 1, [0,1,2])
+    do_prefix_test(is, "aaaac", 2, [0,1,2])
+    do_prefix_test(is, "aaaac", 3, [0,1,2])
+    do_prefix_test(is, "aaaac", 4, [0,1])
+    do_prefix_test(is, "aaaac", 5, [])
+
+    do_prefix_test(is, "ddddX", 0, [6])
+    do_prefix_test(is, "ddddX", 1, [6])
+    do_prefix_test(is, "ddddX", 2, [6])
+    do_prefix_test(is, "ddddX", 3, [6])
+    do_prefix_test(is, "ddddX", 4, [6])
+    do_prefix_test(is, "ddddX", 5, [])
+    
+    fq = FuzzyQuery.new(Term.new("anotherfield", "ddddX"), FuzzyQuery.default_min_similarity, 0)
+    top_docs = is.search(fq)
+    assert_equal(0, top_docs.total_hits)
+
+    is.close()
+  end
+
+  def test_fuzziness_long()
+    iw = IndexWriter.new(@dir, :analyzer => WhiteSpaceAnalyzer.new(), :create => true)
+    add_doc("aaaaaaa", iw)
+    add_doc("segment", iw)
+    iw.optimize()
+    iw.close()
+    is = IndexSearcher.new(@dir)
+
+    # not similar enough:
+    do_prefix_test(is, "xxxxx", 0, [])
+
+    # edit distance to "aaaaaaa" = 3, this matches because the string is longer than
+    # in testDefaultFuzziness so a bigger difference is allowed:
+    do_prefix_test(is, "aaaaccc", 0, [0])
+    
+    # now with prefix
+    do_prefix_test(is, "aaaaccc", 1, [0])
+    do_prefix_test(is, "aaaaccc", 4, [0])
+    do_prefix_test(is, "aaaaccc", 5, [])
+
+    # no match, more than half of the characters is wrong:
+    do_prefix_test(is, "aaacccc", 0, [])
+    
+    # now with prefix
+    do_prefix_test(is, "aaacccc", 1, [])
+
+    # "student" and "stellent" are indeed similar to "segment" by default:
+    do_prefix_test(is, "student", 0, [1])
+    do_prefix_test(is, "stellent", 0, [1])
+
+    # now with prefix
+    do_prefix_test(is, "student", 2, [])
+    do_prefix_test(is, "stellent", 2, [])
+
+    # "student" doesn't match anymore thanks to increased minimum similarity:
+    fq = FuzzyQuery.new(Term.new("field", "student"), 0.6, 0)
+    top_docs = is.search(fq)
+    assert_equal(0, top_docs.total_hits)
+
+    assert_raise(ArgumentError) {fq = FuzzyQuery.new(Term.new("f", "s"), 1.1)}
+    assert_raise(ArgumentError) {fq = FuzzyQuery.new(Term.new("f", "s"), -0.1)}
+
+    is.close()
+  end
+  
+end

data/test/unit/search/tc_index_searcher.rb

@@ -0,0 +1,188 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+class IndexSearcherTest < Test::Unit::TestCase
+  include Ferret::Document
+  include Ferret::Search
+  include Ferret::Store
+  include Ferret::Analysis
+  include Ferret::Index
+
+  def setup()
+    @dir = RAMDirectory.new()
+    iw = IndexWriter.new(@dir, :analyzer => WhiteSpaceAnalyzer.new(), :create => true)
+    @documents = IndexTestHelper.prepare_search_docs()
+    @documents.each { |doc| iw << doc; }
+    iw.close()
+    @is = IndexSearcher.new(@dir)
+  end
+
+  def tear_down()
+    @is.close
+    @dir.close()
+  end
+
+  def get_docs(score_docs)
+    docs = []
+    score_docs.each do |score_doc|
+      docs << score_doc.doc
+    end
+    docs
+  end
+
+  def check_hits(query, expected, top=nil, total_hits=nil)
+    top_docs = @is.search(query)
+    assert_equal(expected.length, top_docs.score_docs.size)
+    assert_equal(top, top_docs.score_docs[0].doc) if top
+    if total_hits
+      assert_equal(total_hits, top_docs.total_hits)
+    else
+      assert_equal(expected.length, top_docs.total_hits)
+    end
+    top_docs.score_docs.each do |score_doc|
+      assert(expected.include?(score_doc.doc),
+             "#{score_doc.doc} was found unexpectedly")
+      assert(score_doc.score =~ @is.explain(query, score_doc.doc).value, 
+        "Scores(#{score_doc.score} != #{@is.explain(query, score_doc.doc).value})")
+    end
+  end
+
+  def test_term_query
+    tq = TermQuery.new(Term.new("field", "word2"));
+    tq.boost = 100
+    check_hits(tq, [1,4,8])
+
+    tq = TermQuery.new(Term.new("field", "word1"));
+    top_docs = @is.search(tq)
+    #puts top_docs.score_docs
+    assert_equal(@documents.size, top_docs.total_hits)
+    assert_equal(10, top_docs.score_docs.size)
+    top_docs = @is.search(tq, {:num_docs => 20})
+    assert_equal(@documents.size, top_docs.score_docs.size)
+  end
+
+  def test_boolean_query
+    bq = BooleanQuery.new()
+    tq1 = TermQuery.new(Term.new("field", "word1"))
+    tq2 = TermQuery.new(Term.new("field", "word3"))
+    bq.add_query(tq1, BooleanClause::Occur::MUST)
+    bq.add_query(tq2, BooleanClause::Occur::MUST)
+    check_hits(bq, [2,3,6,8,11,14], 14)
+
+    tq3 = TermQuery.new(Term.new("field", "word2"))
+    bq.add_query(tq3, BooleanClause::Occur::SHOULD)
+    check_hits(bq, [2,3,6,8,11,14], 8)
+
+    bq = BooleanQuery.new()
+    bq.add_query(tq2, BooleanClause::Occur::MUST)
+    bq.add_query(tq3, BooleanClause::Occur::MUST_NOT)
+    check_hits(bq, [2,3,6,11,14])
+
+    bq = BooleanQuery.new()
+    bq.add_query(tq2, BooleanClause::Occur::MUST_NOT)
+    check_hits(bq, [])
+
+    bq = BooleanQuery.new()
+    bq.add_query(tq2, BooleanClause::Occur::SHOULD)
+    bq.add_query(tq3, BooleanClause::Occur::SHOULD)
+    check_hits(bq, [1,2,3,4,6,8,11,14])
+  end
+
+  def test_phrase_query()
+    pq = PhraseQuery.new()
+    t1 = Term.new("field", "quick")
+    t2 = Term.new("field", "brown")
+    t3 = Term.new("field", "fox")
+    pq << t1 << t2 << t3
+    check_hits(pq, [1])
+
+    pq.slop = 4
+    check_hits(pq, [1,16,17])
+
+    pq = PhraseQuery.new()
+    pq << t1
+    pq.add(t3, 2)
+    check_hits(pq, [1,11,14])
+
+    pq.slop = 1
+    check_hits(pq, [1,11,14,16])
+
+    pq.slop = 4
+    check_hits(pq, [1,11,14,16,17])
+  end
+
+  def test_range_query()
+    rq = RangeQuery.new("date", "20051006", "20051010", true, true)
+    check_hits(rq, [6,7,8,9,10])
+
+    rq = RangeQuery.new("date", "20051006", "20051010", false, true)
+    check_hits(rq, [7,8,9,10])
+
+    rq = RangeQuery.new("date", "20051006", "20051010", true, false)
+    check_hits(rq, [6,7,8,9])
+
+    rq = RangeQuery.new("date", "20051006", "20051010", false, false)
+    check_hits(rq, [7,8,9])
+
+    rq = RangeQuery.new("date", nil, "20051003", false, true)
+    check_hits(rq, [0,1,2,3])
+
+    rq = RangeQuery.new("date", nil, "20051003", false, false)
+    check_hits(rq, [0,1,2])
+
+    rq = RangeQuery.new_less("date", "20051003", true)
+    check_hits(rq, [0,1,2,3])
+
+    rq = RangeQuery.new_less("date", "20051003", false)
+    check_hits(rq, [0,1,2])
+
+    rq = RangeQuery.new("date", "20051014", nil, true, false)
+    check_hits(rq, [14,15,16,17])
+
+    rq = RangeQuery.new("date", "20051014", nil, false, false)
+    check_hits(rq, [15,16,17])
+
+    rq = RangeQuery.new_more("date", "20051014", true)
+    check_hits(rq, [14,15,16,17])
+
+    rq = RangeQuery.new_more("date", "20051014", false)
+    check_hits(rq, [15,16,17])
+  end
+
+  def test_prefix_query()
+    t = Term.new("cat", "cat1")
+    pq = PrefixQuery.new(t)
+    check_hits(pq, [0, 1, 2, 3, 4, 13, 14, 15, 16, 17])
+
+    t.text = "cat1/sub2"
+    pq = PrefixQuery.new(t)
+    check_hits(pq, [3, 4, 13, 15])
+  end
+
+  def test_wildcard_query()
+    t = Term.new("cat", "cat1*")
+    wq = WildcardQuery.new(t)
+    check_hits(wq, [0, 1, 2, 3, 4, 13, 14, 15, 16, 17])
+
+    t.text = "cat1*/su??ub2"
+    wq = WildcardQuery.new(t)
+    check_hits(wq, [4, 16])
+  end
+
+  def test_prefix_query()
+    t11 = Term.new("field", "quick")
+    t12 = Term.new("field", "fast")
+    t21 = Term.new("field", "brown")
+    t22 = Term.new("field", "red")
+    t23 = Term.new("field", "hairy")
+    t3 = Term.new("field", "fox")
+
+    mpq = MultiPhraseQuery.new()
+    mpq << [t11, t12]
+    mpq << [t21, t22, t23]
+    mpq << t3
+    check_hits(mpq, [1, 8, 11, 14])
+
+    mpq.slop = 4
+    check_hits(mpq, [1, 8, 11, 14, 16, 17])
+  end
+end