jk-ferret 0.11.8.2

data/test/unit/query_parser/tc_query_parser.rb

@@ -0,0 +1,238 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+class QueryParserTest < Test::Unit::TestCase
+  include Ferret::Analysis
+
+  def test_strings()
+    parser = Ferret::QueryParser.new(:default_field => "xxx",
+                                     :fields => ["xxx", "field", "f1", "f2"],
+                                     :tokenized_fields => ["xxx", "f1", "f2"])
+    pairs = [
+      ['', ''],
+      ['*:word', 'word field:word f1:word f2:word'],
+      ['word', 'word'],
+      ['field:word', 'field:word'],
+      ['"word1 word2 word#"', '"word1 word2 word"'],
+      ['"word1 %%% word3"', '"word1 <> word3"~1'],
+      ['field:"one two three"', 'field:"one two three"'],
+      ['field:"one %%% three"', 'field:"one %%% three"'],
+      ['f1:"one %%% three"', 'f1:"one <> three"~1'],
+      ['field:"one <> three"', 'field:"one <> three"'],
+      ['field:"one <> three <>"', 'field:"one <> three"'],
+      ['field:"one <> <> <> three <>"', 'field:"one <> <> <> three"'],
+      ['field:"one <> 222 <> three|four|five <>"', 'field:"one <> 222 <> three|four|five"'],
+      ['field:"on1|tw2 THREE|four|five six|seven"', 'field:"on1|tw2 THREE|four|five six|seven"'],
+      ['field:"testing|trucks"', 'field:"testing|trucks"'],
+      ['[aaa bbb]', '[aaa bbb]'],
+      ['{aaa bbb]', '{aaa bbb]'],
+      ['field:[aaa bbb}', 'field:[aaa bbb}'],
+      ['{aaa bbb}', '{aaa bbb}'],
+      ['{aaa>', '{aaa>'],
+      ['[aaa>', '[aaa>'],
+      ['field:<a\ aa}', 'field:<a aa}'],
+      ['<aaa]', '<aaa]'],
+      ['>aaa', '{aaa>'],
+      ['>=aaa', '[aaa>'],
+      ['<aaa', '<aaa}'],
+      ['[A>', '[a>'],
+      ['field:<=aaa', 'field:<aaa]'],
+      ['REQ one REQ two', '+one +two'],
+      ['REQ one two', '+one two'],
+      ['one REQ two', 'one +two'],
+      ['+one +two', '+one +two'],
+      ['+one two', '+one two'],
+      ['one +two', 'one +two'],
+      ['-one -two', '-one -two'],
+      ['-one two', '-one two'],
+      ['one -two', 'one -two'],
+      ['!one !two', '-one -two'],
+      ['!one two', '-one two'],
+      ['one !two', 'one -two'],
+      ['NOT one NOT two', '-one -two'],
+      ['NOT one two', '-one two'],
+      ['one NOT two', 'one -two'],
+      ['NOT two', '-two +*'],
+      ['one two', 'one two'],
+      ['one OR two', 'one two'],
+      ['one AND two', '+one +two'],
+      ['one two AND three', 'one two +three'],
+      ['one two OR three', 'one two three'],
+      ['one (two AND three)', 'one (+two +three)'],
+      ['one AND (two OR three)', '+one +(two three)'],
+      ['field:(one AND (two OR three))', '+field:one +(field:two field:three)'],
+      ['one AND (two OR [aaa vvv})', '+one +(two [aaa vvv})'],
+      ['one AND (f1:two OR f2:three) AND four', '+one +(f1:two f2:three) +four'],
+      ['one^1.23', 'one^1.23'],
+      ['(one AND two)^100.23', '(+one +two)^100.23'],
+      ['field:(one AND two)^100.23', '(+field:one +field:two)^100.23'],
+      ['field:(one AND [aaa bbb]^23.3)^100.23', '(+field:one +field:[aaa bbb]^23.3)^100.23'],
+      ['(REQ field:"one two three")^23', 'field:"one two three"^23.0'],
+      ['asdf~0.2', 'asdf~0.2'],
+      ['field:asdf~0.2', 'field:asdf~0.2'],
+      ['asdf~0.2^100.0', 'asdf~0.2^100.0'],
+      ['field:asdf~0.2^0.1', 'field:asdf~0.2^0.1'],
+      ['field:"asdf <> asdf|asdf"~4', 'field:"asdf <> asdf|asdf"~4'],
+      ['"one two three four five"~5', '"one two three four five"~5'],
+      ['ab?de', 'ab?de'],
+      ['ab*de', 'ab*de'],
+      ['asdf?*?asd*dsf?asfd*asdf?', 'asdf?*?asd*dsf?asfd*asdf?'],
+      ['field:a* AND field:(b*)', '+field:a* +field:b*'],
+      ['field:abc~ AND field:(b*)', '+field:abc~ +field:b*'],
+      ['asdf?*?asd*dsf?asfd*asdf?^20.0', 'asdf?*?asd*dsf?asfd*asdf?^20.0'],
+
+      ['*:xxx', 'xxx field:xxx f1:xxx f2:xxx'],
+      ['f1|f2:xxx', 'f1:xxx f2:xxx'],
+
+      ['*:asd~0.2', 'asd~0.2 field:asd~0.2 f1:asd~0.2 f2:asd~0.2'],
+      ['f1|f2:asd~0.2', 'f1:asd~0.2 f2:asd~0.2'],
+
+      ['*:a?d*^20.0', '(a?d* field:a?d* f1:a?d* f2:a?d*)^20.0'],
+      ['f1|f2:a?d*^20.0', '(f1:a?d* f2:a?d*)^20.0'],
+
+      ['*:"asdf <> xxx|yyy"', '"asdf <> xxx|yyy" field:"asdf <> xxx|yyy" f1:"asdf <> xxx|yyy" f2:"asdf <> xxx|yyy"'],
+      ['f1|f2:"asdf <> xxx|yyy"', 'f1:"asdf <> xxx|yyy" f2:"asdf <> xxx|yyy"'],
+      ['f1|f2:"asdf <> do|yyy"', 'f1:"asdf <> yyy" f2:"asdf <> yyy"'],
+      ['f1|f2:"do|cat"', 'f1:cat f2:cat'],
+
+      ['*:[bbb xxx]', '[bbb xxx] field:[bbb xxx] f1:[bbb xxx] f2:[bbb xxx]'],
+      ['f1|f2:[bbb xxx]', 'f1:[bbb xxx] f2:[bbb xxx]'],
+
+      ['*:(xxx AND bbb)', '+(xxx field:xxx f1:xxx f2:xxx) +(bbb field:bbb f1:bbb f2:bbb)'],
+      ['f1|f2:(xxx AND bbb)', '+(f1:xxx f2:xxx) +(f1:bbb f2:bbb)'],
+      ['asdf?*?asd*dsf?asfd*asdf?^20.0', 'asdf?*?asd*dsf?asfd*asdf?^20.0'],
+      ['"onewordphrase"', 'onewordphrase'],
+      ["who'd", "who'd"]
+    ]
+      
+    pairs.each do |query_str, expected|
+      assert_equal(expected, parser.parse(query_str).to_s("xxx"))
+    end
+  end
+
+  def test_qp_with_standard_analyzer()
+    parser = Ferret::QueryParser.new(:default_field => "xxx",
+                                     :fields => ["xxx", "key"],
+                                     :analyzer => StandardAnalyzer.new)
+    pairs = [
+      ['key:1234', 'key:1234'],
+      ['key:(1234 and Dave)', 'key:1234 key:dave'],
+      ['key:(1234)', 'key:1234'],
+      ['and the but they with', '']
+    ]
+      
+    pairs.each do |query_str, expected|
+      assert_equal(expected, parser.parse(query_str).to_s("xxx"))
+    end
+
+  end
+
+  def test_qp_changing_fields()
+    parser = Ferret::QueryParser.new(:default_field => "xxx",
+                                     :fields => ["xxx", "key"],
+                                     :analyzer => WhiteSpaceAnalyzer.new)
+    assert_equal('word key:word', parser.parse("*:word").to_s("xxx"))
+
+    parser.fields = ["xxx", "one", "two", "three"]
+    assert_equal('word one:word two:word three:word',
+                 parser.parse("*:word").to_s("xxx"))
+    assert_equal('three:word four:word',
+                 parser.parse("three:word four:word").to_s("xxx"))
+  end
+
+  def test_qp_allow_any_field()
+    parser = Ferret::QueryParser.new(:default_field => "xxx",
+                                     :fields => ["xxx", "key"],
+                                     :analyzer => WhiteSpaceAnalyzer.new,
+                                     :validate_fields => true)
+
+    assert_equal('key:word',
+                 parser.parse("key:word song:word").to_s("xxx"))
+    assert_equal('word key:word', parser.parse("*:word").to_s("xxx"))
+
+
+    parser = Ferret::QueryParser.new(:default_field => "xxx",
+                                     :fields => ["xxx", "key"],
+                                     :analyzer => WhiteSpaceAnalyzer.new)
+
+    assert_equal('key:word song:word',
+                 parser.parse("key:word song:word").to_s("xxx"))
+    assert_equal('word key:word', parser.parse("*:word").to_s("xxx"))
+  end
+  
+  def do_test_query_parse_exception_raised(str)
+    parser = Ferret::QueryParser.new(:default_field => "xxx",
+                                     :fields => ["f1", "f2", "f3"],
+                                     :handle_parse_errors => false)
+    assert_raise(Ferret::QueryParser::QueryParseException,
+                 str + " should have failed") do
+      parser.parse(str)
+    end
+  end
+
+  def test_or_default
+    parser = Ferret::QueryParser.new(:default_field => :*,
+                                     :fields => [:x, :y],
+                                     :or_default => false,
+                                     :analyzer => StandardAnalyzer.new)
+    pairs = [
+      ['word', 'x:word y:word'],
+      ['word1 word2', '+(x:word1 y:word1) +(x:word2 y:word2)']
+    ]
+      
+    pairs.each do |query_str, expected|
+      assert_equal(expected, parser.parse(query_str).to_s(""))
+    end
+  end
+
+  def test_prefix_query
+    parser = Ferret::QueryParser.new(:default_field => "xxx",
+                                     :fields => ["xxx"],
+                                     :analyzer => StandardAnalyzer.new)
+    assert_equal(Ferret::Search::PrefixQuery, parser.parse("asdg*").class)
+    assert_equal(Ferret::Search::WildcardQuery, parser.parse("a?dg*").class)
+    assert_equal(Ferret::Search::WildcardQuery, parser.parse("a*dg*").class)
+    assert_equal(Ferret::Search::WildcardQuery, parser.parse("adg*c").class)
+  end
+  
+  def test_bad_queries
+    parser = Ferret::QueryParser.new(:default_field => "xxx",
+                                     :fields => ["f1", "f2"])
+
+    pairs = [
+      ['::*word', 'word'],
+      ['::*&)(*^&*(', ''],
+      ['::*&one)(*two(*&"', '"one two"~1'],
+      [':', ''],
+      ['[, ]', ''],
+      ['{, }', ''],
+      ['!', ''],
+      ['+', ''],
+      ['~', ''],
+      ['^', ''],
+      ['-', ''],
+      ['|', ''],
+      ['<, >', ''],
+      ['=', ''],
+      ['<script>', 'script']
+    ]
+      
+    pairs.each do |query_str, expected|
+      do_test_query_parse_exception_raised(query_str)
+      assert_equal(expected, parser.parse(query_str).to_s("xxx"))
+    end
+  end
+
+  def test_use_keywords_switch
+    analyzer = LetterAnalyzer.new
+    parser = Ferret::QueryParser.new(:analyzer => analyzer,
+                                     :default_field => "xxx")
+    assert_equal("+www (+xxx +yyy) -zzz",
+                 parser.parse("REQ www (xxx AND yyy) OR NOT zzz").to_s("xxx"))
+
+    parser = Ferret::QueryParser.new(:analyzer => analyzer,
+                                     :default_field => "xxx",
+                                     :use_keywords => false)
+    assert_equal("req www (xxx and yyy) or not zzz",
+                 parser.parse("REQ www (xxx AND yyy) OR NOT zzz").to_s("xxx"))
+  end
+end

data/test/unit/search/tc_filter.rb

@@ -0,0 +1,156 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+require 'date'
+
+
+class FilterTest < Test::Unit::TestCase
+  include Ferret::Search
+  include Ferret::Analysis
+  include Ferret::Index
+
+  def setup()
+    @dir = Ferret::Store::RAMDirectory.new()
+    iw = IndexWriter.new(:dir => @dir,
+                         :analyzer => WhiteSpaceAnalyzer.new(),
+                         :create => true)
+    [ 
+      {:int => "0", :date => "20040601", :switch => "on"},
+      {:int => "1", :date => "20041001", :switch => "off"},
+      {:int => "2", :date => "20051101", :switch => "on"},
+      {:int => "3", :date => "20041201", :switch => "off"},
+      {:int => "4", :date => "20051101", :switch => "on"},
+      {:int => "5", :date => "20041201", :switch => "off"},
+      {:int => "6", :date => "20050101", :switch => "on"},
+      {:int => "7", :date => "20040701", :switch => "off"},
+      {:int => "8", :date => "20050301", :switch => "on"},
+      {:int => "9", :date => "20050401", :switch => "off"}
+    ].each {|doc| iw << doc}
+    iw.close
+  end
+
+  def teardown()
+    @dir.close()
+  end
+
+  def do_test_top_docs(searcher, query, expected, filter)
+    top_docs = searcher.search(query, {:filter => filter})
+    #puts top_docs
+    assert_equal(expected.size, top_docs.hits.size)
+    top_docs.total_hits.times do |i|
+      assert_equal(expected[i], top_docs.hits[i].doc)
+    end
+  end
+
+  def test_range_filter
+    searcher = Searcher.new(@dir)
+    q = MatchAllQuery.new()
+    rf = RangeFilter.new(:int, :>= => "2", :<= => "6")
+    do_test_top_docs(searcher, q, [2,3,4,5,6], rf)
+    rf = RangeFilter.new(:int, :>= => "2", :< => "6")
+    do_test_top_docs(searcher, q, [2,3,4,5], rf)
+    rf = RangeFilter.new(:int, :> => "2", :<= => "6")
+    do_test_top_docs(searcher, q, [3,4,5,6], rf)
+    rf = RangeFilter.new(:int, :> => "2", :< => "6")
+    do_test_top_docs(searcher, q, [3,4,5], rf)
+    rf = RangeFilter.new(:int, :>= => "6")
+    do_test_top_docs(searcher, q, [6,7,8,9], rf)
+    rf = RangeFilter.new(:int, :> => "6")
+    do_test_top_docs(searcher, q, [7,8,9], rf)
+    rf = RangeFilter.new(:int, :<= => "2")
+    do_test_top_docs(searcher, q, [0,1,2], rf)
+    rf = RangeFilter.new(:int, :< => "2")
+    do_test_top_docs(searcher, q, [0,1], rf)
+
+    bits = rf.bits(searcher.reader)
+    assert(bits[0])
+    assert(bits[1])
+    assert(!bits[2])
+    assert(!bits[3])
+    assert(!bits[4])
+  end
+
+  def test_range_filter_errors
+    assert_raise(ArgumentError) {f = RangeFilter.new(:f, :> => "b", :< => "a")}
+    assert_raise(ArgumentError) {f = RangeFilter.new(:f, :include_lower => true)}
+    assert_raise(ArgumentError) {f = RangeFilter.new(:f, :include_upper => true)}
+  end
+
+  def test_query_filter()
+    searcher = Searcher.new(@dir)
+    q = MatchAllQuery.new()
+    qf = QueryFilter.new(TermQuery.new(:switch, "on"))
+    do_test_top_docs(searcher, q, [0,2,4,6,8], qf)
+    # test again to test caching doesn't break it
+    do_test_top_docs(searcher, q, [0,2,4,6,8], qf)
+    qf = QueryFilter.new(TermQuery.new(:switch, "off"))
+    do_test_top_docs(searcher, q, [1,3,5,7,9], qf)
+
+    bits = qf.bits(searcher.reader)
+    assert(bits[1])
+    assert(bits[3])
+    assert(bits[5])
+    assert(bits[7])
+    assert(bits[9])
+    assert(!bits[0])
+    assert(!bits[2])
+    assert(!bits[4])
+    assert(!bits[6])
+    assert(!bits[8])
+  end
+
+  def test_filtered_query
+    searcher = Searcher.new(@dir)
+    q = MatchAllQuery.new()
+    rf = RangeFilter.new(:int, :>= => "2", :<= => "6")
+    rq = FilteredQuery.new(q, rf)
+    qf = QueryFilter.new(TermQuery.new(:switch, "on"))
+    do_test_top_docs(searcher, rq, [2,4,6], qf)
+    query = FilteredQuery.new(rq, qf)
+    rf2 = RangeFilter.new(:int, :>= => "3")
+    do_test_top_docs(searcher, query, [4,6], rf2)
+  end
+
+  class CustomFilter
+    def bits(ir)
+      bv = Ferret::Utils::BitVector.new
+      bv[0] = bv[2] = bv[4] = true
+      bv
+    end
+  end
+
+  def test_custom_filter
+    searcher = Searcher.new(@dir)
+    q = MatchAllQuery.new
+    filt = CustomFilter.new
+    do_test_top_docs(searcher, q, [0, 2, 4], filt)
+  end
+
+  def test_filter_proc
+    searcher = Searcher.new(@dir)
+    q = MatchAllQuery.new()
+    filter_proc = lambda {|doc, score, s| (s[doc][:int] % 2) == 0}
+    top_docs = searcher.search(q, :filter_proc => filter_proc)
+    top_docs.hits.each do |hit|
+      assert_equal(0, searcher[hit.doc][:int] % 2)
+    end
+  end
+
+  def test_score_modifying_filter_proc
+    searcher = Searcher.new(@dir)
+    q = MatchAllQuery.new()
+    start_date = Date.parse('2008-02-08')
+    date_half_life_50 = lambda do |doc, score, s|
+      days = (start_date - Date.parse(s[doc][:date], '%Y%m%d')).to_i
+      1.0 / (2.0 ** (days.to_f / 50.0))
+    end
+    top_docs = searcher.search(q, :filter_proc => date_half_life_50)
+    docs = top_docs.hits.collect {|hit| hit.doc}
+    assert_equal(docs, [2,4,9,8,6,3,5,1,7,0])
+    rev_date_half_life_50 = lambda do |doc, score, s|
+      days = (start_date - Date.parse(s[doc][:date], '%Y%m%d')).to_i
+      1.0 - 1.0 / (2.0 ** (days.to_f / 50.0))
+    end
+    top_docs = searcher.search(q, :filter_proc => rev_date_half_life_50)
+    docs = top_docs.hits.collect {|hit| hit.doc}
+    assert_equal(docs, [0,7,1,3,5,6,8,9,2,4])
+  end
+end

data/test/unit/search/tc_fuzzy_query.rb

@@ -0,0 +1,147 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+class FuzzyQueryTest < Test::Unit::TestCase
+  include Ferret::Search
+  include Ferret::Store
+  include Ferret::Analysis
+  include Ferret::Index
+
+  def add_doc(text, writer)
+    writer << {:field => text}
+  end
+
+  def setup()
+    @dir = RAMDirectory.new()
+  end
+
+  def teardown()
+    @dir.close()
+  end
+
+  def do_test_top_docs(is, query, expected)
+    top_docs = is.search(query)
+    assert_equal(expected.length, top_docs.total_hits,
+                "expected #{expected.length} hits but got #{top_docs.total_hits}")
+    assert_equal(expected.length, top_docs.hits.size)
+    top_docs.total_hits.times do |i|
+      assert_equal(expected[i], top_docs.hits[i].doc)
+    end
+  end
+
+  def do_prefix_test(is, text, prefix, expected)
+    fq = FuzzyQuery.new(:field, text, :prefix_length => prefix)
+    #puts is.explain(fq, 0)
+    #puts is.explain(fq, 1)
+    do_test_top_docs(is, fq, expected)
+  end
+
+  def test_fuzziness()
+    iw = IndexWriter.new(:dir => @dir,
+                         :analyzer => WhiteSpaceAnalyzer.new(),
+                         :create => true)
+    add_doc("aaaaa", iw)
+    add_doc("aaaab", iw)
+    add_doc("aaabb", iw)
+    add_doc("aabbb", iw)
+    add_doc("abbbb", iw)
+    add_doc("bbbbb", iw)
+    add_doc("ddddd", iw)
+    add_doc("ddddddddddddddddddddd", iw) # test max_distances problem
+    add_doc("aaaaaaaaaaaaaaaaaaaaaaa", iw) # test max_distances problem
+    #iw.optimize()
+    iw.close()
+
+
+    is = Searcher.new(@dir)
+
+    fq = FuzzyQuery.new(:field, "aaaaa", :prefix_length => 5)
+
+    do_prefix_test(is, "aaaaaaaaaaaaaaaaaaaaaa", 1, [8])
+    do_prefix_test(is, "aaaaa", 0, [0,1,2])
+    do_prefix_test(is, "aaaaa", 1, [0,1,2])
+    do_prefix_test(is, "aaaaa", 2, [0,1,2])
+    do_prefix_test(is, "aaaaa", 3, [0,1,2])
+    do_prefix_test(is, "aaaaa", 4, [0,1])
+    do_prefix_test(is, "aaaaa", 5, [0])
+    do_prefix_test(is, "aaaaa", 6, [0])
+
+    do_prefix_test(is, "xxxxx", 0, [])
+
+    do_prefix_test(is, "aaccc", 0, [])
+
+    do_prefix_test(is, "aaaac", 0, [0,1,2])
+    do_prefix_test(is, "aaaac", 1, [0,1,2])
+    do_prefix_test(is, "aaaac", 2, [0,1,2])
+    do_prefix_test(is, "aaaac", 3, [0,1,2])
+    do_prefix_test(is, "aaaac", 4, [0,1])
+    do_prefix_test(is, "aaaac", 5, [])
+
+    do_prefix_test(is, "ddddX", 0, [6])
+    do_prefix_test(is, "ddddX", 1, [6])
+    do_prefix_test(is, "ddddX", 2, [6])
+    do_prefix_test(is, "ddddX", 3, [6])
+    do_prefix_test(is, "ddddX", 4, [6])
+    do_prefix_test(is, "ddddX", 5, [])
+    
+    fq = FuzzyQuery.new(:anotherfield, "ddddX", :prefix_length => 0)
+    top_docs = is.search(fq)
+    assert_equal(0, top_docs.total_hits)
+
+    is.close()
+  end
+
+  def test_fuzziness_long()
+    iw = IndexWriter.new(:dir => @dir,
+                         :analyzer => WhiteSpaceAnalyzer.new(),
+                         :create => true)
+    add_doc("aaaaaaa", iw)
+    add_doc("segment", iw)
+    iw.optimize()
+    iw.close()
+    is = Searcher.new(@dir)
+
+    # not similar enough:
+    do_prefix_test(is, "xxxxx", 0, [])
+
+    # edit distance to "aaaaaaa" = 3, this matches because the string is longer than
+    # in testDefaultFuzziness so a bigger difference is allowed:
+    do_prefix_test(is, "aaaaccc", 0, [0])
+    
+    # now with prefix
+    do_prefix_test(is, "aaaaccc", 1, [0])
+    do_prefix_test(is, "aaaaccc", 4, [0])
+    do_prefix_test(is, "aaaaccc", 5, [])
+
+    # no match, more than half of the characters is wrong:
+    do_prefix_test(is, "aaacccc", 0, [])
+    
+    # now with prefix
+    do_prefix_test(is, "aaacccc", 1, [])
+
+    # "student" and "stellent" are indeed similar to "segment" by default:
+    do_prefix_test(is, "student", 0, [1])
+    do_prefix_test(is, "stellent", 0, [1])
+
+    # now with prefix
+    do_prefix_test(is, "student", 2, [])
+    do_prefix_test(is, "stellent", 2, [])
+
+    # "student" doesn't match anymore thanks to increased minimum similarity:
+    fq = FuzzyQuery.new(:field, "student",
+                        :min_similarity => 0.6,
+                        :prefix_length => 0)
+
+    top_docs = is.search(fq)
+    assert_equal(0, top_docs.total_hits)
+
+    assert_raise(ArgumentError) do
+      fq = FuzzyQuery.new(:f, "s", :min_similarity => 1.1)
+    end
+    assert_raise(ArgumentError) do
+      fq = FuzzyQuery.new(:f, "s", :min_similarity => -0.1)
+    end
+
+    is.close()
+  end
+  
+end