ferret 0.1.0

data/test/unit/search/tc_search_and_sort.rb

@@ -0,0 +1,98 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+class SearchAndSortTest < Test::Unit::TestCase
+  include Ferret::Document
+  include Ferret::Search
+  include Ferret::Store
+  include Ferret::Analysis
+  include Ferret::Index
+
+  def add_doc(hash, writer)
+    doc = Document.new()
+    hash.each_pair do |field, text|
+      doc << Field.new(field, text, Field::Store::NO, Field::Index::UNTOKENIZED)
+    end
+    doc.boost = hash["float"].to_f
+    writer << doc
+  end
+
+  def setup()
+    @dir = RAMDirectory.new()
+    iw = IndexWriter.new(@dir, :analyzer => WhiteSpaceAnalyzer.new(), :create => true)
+    docs = [                                                             # len mod
+      {"search"=>"findall","string"=>"a","int"=>"6","float"=>"0.01"},    #  4   0
+      {"search"=>"findall","string"=>"c","int"=>"5","float"=>"0.1"},     #  3   3
+      {"search"=>"findall","string"=>"e","int"=>"2","float"=>"0.001"},   #  5   1
+      {"search"=>"findall","string"=>"g","int"=>"1","float"=>"1.0"},     #  3   3
+      {"search"=>"findall","string"=>"i","int"=>"3","float"=>"0.0001"},  #  6   2
+      {"search"=>"findall","string"=>"j","int"=>"4","float"=>"10.0"},    #  4   0
+      {"search"=>"findall","string"=>"h","int"=>"5","float"=>"0.00001"}, #  7   3
+      {"search"=>"findall","string"=>"f","int"=>"2","float"=>"100.0"},   #  5   1
+      {"search"=>"findall","string"=>"d","int"=>"3","float"=>"1000.0"},  #  6   2
+      {"search"=>"findall","string"=>"b","int"=>"4","float"=>"0.000001"} #  8   0
+    ]
+    docs.each {|doc| add_doc(doc, iw)}
+    iw.close
+  end
+
+  def tear_down()
+    @dir.close()
+  end
+
+  def do_test_top_docs(is, query, expected, sort = nil)
+    top_docs = is.search(query, {:sort => sort})
+    top_docs.total_hits.times do |i|
+      assert_equal(expected[i], top_docs.score_docs[i].doc)
+    end
+  end
+
+  def test_sorts()
+    is = IndexSearcher.new(@dir)
+    q = TermQuery.new(Term.new("search", "findall"))
+    do_test_top_docs(is, q, [8,7,5,3,1,0,2,4,6,9])
+    do_test_top_docs(is, q, [8,7,5,3,1,0,2,4,6,9], Sort::RELEVANCE)
+    do_test_top_docs(is, q, [8,7,5,3,1,0,2,4,6,9], [SortField::FIELD_SCORE])
+    do_test_top_docs(is, q, [0,1,2,3,4,5,6,7,8,9], Sort::INDEX_ORDER)
+    do_test_top_docs(is, q, [0,1,2,3,4,5,6,7,8,9], [SortField::FIELD_DOC])
+
+    ## int
+    sf_int = SortField.new("int", {:sort_type => SortField::SortType::INT})
+    do_test_top_docs(is, q, [0,1,6,5,9,4,8,2,7,3], [sf_int])
+    do_test_top_docs(is, q, [0,1,6,5,9,8,4,7,2,3], [sf_int, SortField::FIELD_SCORE])
+    sf_int = SortField.new("int", {:sort_type => SortField::SortType::INT, :reverse => true})
+    do_test_top_docs(is, q, [3,2,7,4,8,5,9,1,6,0], [sf_int])
+
+    ## float
+    sf_float = SortField.new("float", {:sort_type => SortField::SortType::FLOAT})
+    do_test_top_docs(is, q, [8,7,5,3,1,0,2,4,6,9], Sort.new([sf_float, SortField::FIELD_SCORE]))
+    sf_float = SortField.new("float", {:sort_type => SortField::SortType::FLOAT, :reverse => true})
+    do_test_top_docs(is, q, [9,6,4,2,0,1,3,5,7,8], Sort.new([sf_float, SortField::FIELD_SCORE]))
+
+    ## str
+    sf_str = SortField.new("string", {:sort_type => SortField::SortType::STRING})
+    do_test_top_docs(is, q, [0,9,1,8,2,7,3,6,4,5], [sf_str, SortField::FIELD_SCORE])
+
+    ## auto
+    do_test_top_docs(is, q, [0,9,1,8,2,7,3,6,4,5], Sort.new("string"))
+    do_test_top_docs(is, q, [0,1,6,5,9,4,8,2,7,3], Sort.new(["int"]))
+    do_test_top_docs(is, q, [8,7,5,3,1,0,2,4,6,9], Sort.new("float"))
+    do_test_top_docs(is, q, [9,6,4,2,0,1,3,5,7,8], Sort.new("float", true))
+    do_test_top_docs(is, q, [0,1,6,9,5,8,4,2,7,3], Sort.new(["int", "string"]))
+    do_test_top_docs(is, q, [3,7,2,4,8,5,9,6,1,0], Sort.new(["int", "string"], true))
+  end
+
+  LENGTH = SortField::SortType.new("length", lambda{|str| str.length})
+  LENGTH_MODULO = SortField::SortType.new("length_mod", lambda{|str| str.length},
+                                          lambda{|i, j| (i%4) <=> (j%4)})
+  def test_special_sorts
+    is = IndexSearcher.new(@dir)
+    q = TermQuery.new(Term.new("search", "findall"))
+    sf = SortField.new("float", {:sort_type => LENGTH})
+    do_test_top_docs(is, q, [9,6,4,8,2,7,0,5,1,3], [sf])
+    sf = SortField.new("float", {:sort_type => LENGTH_MODULO})
+    do_test_top_docs(is, q, [1,3,6,4,8,2,7,0,5,9], [sf])
+    sf = SortField.new("float", {:sort_type => LENGTH,
+                                 :comparator => lambda{|i,j| (j%4) <=> (i%4)}})
+    do_test_top_docs(is, q, [0,5,9,2,7,4,8,1,3,6], [sf])
+  end
+end

data/test/unit/search/tc_similarity.rb

@@ -0,0 +1,37 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+
+class SimilarityTest < Test::Unit::TestCase
+  include Ferret::Search
+  include Ferret::Index
+
+  def test_byte_float_conversion()
+    256.times do |i|
+      assert_equal(i, Similarity.float_to_byte(Similarity.byte_to_float(i)))
+      assert_equal(Similarity.byte_to_float(i), Similarity::NORM_TABLE[i])
+      assert_equal(i, Similarity.encode_norm(Similarity.decode_norm(i)))
+    end
+  end
+
+  def test_default_similarity
+    dsim = DefaultSimilarity.new()
+    assert_equal(1.0/4, dsim.length_norm("field", 16))
+    assert_equal(1.0/4, dsim.query_norm(16))
+    assert_equal(3.0, dsim.tf(9))
+    assert_equal(1.0/10, dsim.sloppy_freq(9))
+    assert_equal(1.0, dsim.idf(9, 10))
+    assert_equal(4.0, dsim.coord(12, 3))
+    searcher = Object.new
+    def searcher.doc_freq(term) 9 end
+    def searcher.max_doc() 10 end
+    term = Term.new("field", "text")
+    assert_equal(1.0, dsim.idf_term(term, searcher))
+    terms = [
+      Term.new("field1", "text1"),
+      Term.new("field1", "text2"),
+      Term.new("field2", "text3"),
+      Term.new("field2", "text4")
+    ]
+    assert_equal(4.0, dsim.idf_phrase(terms, searcher))
+  end
+end

data/test/unit/search/tc_sort.rb

@@ -0,0 +1,48 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+class SortTest < Test::Unit::TestCase
+  include Ferret::Search
+
+  def test_basic()
+    s = Sort::RELEVANCE
+    assert_equal(2, s.fields.size)
+    assert_equal(SortField::FIELD_SCORE, s.fields[0])
+    assert_equal(SortField::FIELD_DOC, s.fields[1])
+
+    s = Sort::INDEX_ORDER
+    assert_equal(1, s.fields.size)
+    assert_equal(SortField::FIELD_DOC, s.fields[0])
+  end
+
+  def test_string_init()
+    s = Sort.new("field")
+    assert_equal(2, s.fields.size)
+    assert_equal(SortField::SortType::AUTO, s.fields[0].sort_type)
+    assert_equal("field", s.fields[0].name)
+    assert_equal(SortField::FIELD_DOC, s.fields[1])
+
+    s = Sort.new(["field1", "field2", "field3"])
+    assert_equal(3, s.fields.size)
+    assert_equal(SortField::SortType::AUTO, s.fields[0].sort_type)
+    assert_equal("field1", s.fields[0].name)
+    assert_equal(SortField::SortType::AUTO, s.fields[1].sort_type)
+    assert_equal("field2", s.fields[1].name)
+    assert_equal(SortField::SortType::AUTO, s.fields[2].sort_type)
+    assert_equal("field3", s.fields[2].name)
+  end
+
+  def test_multi_fields()
+    sf1 = SortField.new("field", {:sort_type => SortField::SortType::INT,
+                                  :reverse => true})
+    sf2 = SortField::FIELD_SCORE
+    sf3 = SortField::FIELD_DOC
+    s = Sort.new([sf1, sf2, sf3])
+
+    assert_equal(3, s.fields.size)
+    assert_equal(SortField::SortType::INT, s.fields[0].sort_type)
+    assert_equal("field", s.fields[0].name)
+    assert(s.fields[0].reverse?)
+    assert_equal(SortField::FIELD_SCORE, s.fields[1])
+    assert_equal(SortField::FIELD_DOC, s.fields[2])
+  end
+end

data/test/unit/search/tc_sort_field.rb

@@ -0,0 +1,27 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+class SortFieldTest < Test::Unit::TestCase
+  include Ferret::Search
+
+  def test_field_score()
+    fs = SortField::FIELD_SCORE
+    assert_equal(SortField::SortType::SCORE, fs.sort_type)
+    assert_nil(fs.name)
+    assert(!fs.reverse?, "FIELD_SCORE should not be reverse")
+    assert_nil(fs.comparator)
+  end
+
+  def test_field_doc()
+    fs = SortField::FIELD_DOC
+    assert_equal(SortField::SortType::DOC, fs.sort_type)
+    assert_nil(fs.name)
+    assert(!fs.reverse?, "FIELD_DOC should not be reverse")
+    assert_nil(fs.comparator)
+  end
+
+  def test_error_raised()
+    assert_raise(ArgumentError) {
+      fs = SortField.new(nil, {:sort_type => SortField::SortType::INT})
+    }
+  end
+end

data/test/unit/search/tc_spans.rb

@@ -0,0 +1,153 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+
+class SpansBasicTest < Test::Unit::TestCase
+  include Ferret::Document
+  include Ferret::Search
+  include Ferret::Store
+  include Ferret::Index
+  include Ferret::Search::Spans
+  include Ferret::Analysis
+
+  def setup()
+    data = [
+      "start finish one two three four five six seven",
+      "start one finish two three four five six seven",
+      "start one two finish three four five six seven",
+      "start one two three finish four five six seven",
+      "start one two three four finish five six seven",
+      "start one two three four five finish six seven",
+      "start one two three four five six finish seven eight",
+      "start one two three four five six seven finish eight nine",
+      "start one two three four five six finish seven eight",
+      "start one two three four five finish six seven",
+      "start one two three four finish five six seven",
+      "start one two three finish four five six seven",
+      "start one two finish three four five six seven",
+      "start one finish two three four five six seven",
+      "start finish one two three four five six seven",
+      "start start  one two three four five six seven",
+      "finish start one two three four five six seven",
+      "finish one start two three four five six seven",
+      "finish one two start three four five six seven",
+      "finish one two three start four five six seven",
+      "finish one two three four start five six seven",
+      "finish one two three four five start six seven",
+      "finish one two three four five six start seven eight",
+      "finish one two three four five six seven start eight nine",
+      "finish one two three four five six start seven eight",
+      "finish one two three four five start six seven",
+      "finish one two three four start five six seven",
+      "finish one two three start four five six seven",
+      "finish one two start three four five six seven",
+      "finish one start two three four five six seven",
+      "finish start one two three four five six seven"
+    ]
+    @dir = RAMDirectory.new
+    iw = IndexWriter.new(@dir, :analyzer => WhiteSpaceAnalyzer.new(), :create => true)
+    data.each do |line|
+      doc = Document.new()
+      doc << Field.new("field", line, Field::Store::NO, Field::Index::TOKENIZED)
+      iw << doc
+    end
+
+    iw.close()
+
+    @is = IndexSearcher.new(@dir)
+  end
+
+  def tear_down()
+    @iw.close
+    @dir.close
+  end
+
+  def number_split(i)
+    if (i < 10)
+      return "<#{i}>"
+    elsif (i < 100)
+      return "<#{((i/10)*10)}> <#{i%10}>"
+    else
+      return "<#{((i/100)*100)}> <#{(((i%100)/10)*10)}> <#{i%10}>"
+    end
+  end
+
+  def check_hits(query, expected, test_explain = false, top=nil)
+    top_docs = @is.search(query, {:num_docs => expected.length})
+    assert_equal(expected.length, top_docs.score_docs.size)
+    assert_equal(top, top_docs.score_docs[0].doc) if top
+    assert_equal(expected.length, top_docs.total_hits)
+    top_docs.score_docs.each do |score_doc|
+      assert(expected.include?(score_doc.doc),
+             "#{score_doc.doc} was found unexpectedly")
+      if test_explain
+        assert(score_doc.score =~ @is.explain(query, score_doc.doc).value, 
+          "Scores(#{score_doc.score} != #{@is.explain(query, score_doc.doc).value})")
+      end
+    end
+  end
+
+  def test_span_term_query()
+    tq = SpanTermQuery.new(Term.new("field", "nine"))
+    check_hits(tq, [7,23], true)
+    tq = SpanTermQuery.new(Term.new("field", "eight"))
+    check_hits(tq, [6,7,8,22,23,24])
+  end
+
+  def test_span_near_query()
+    tq1 = SpanTermQuery.new(Term.new("field", "start"))
+    tq2 = SpanTermQuery.new(Term.new("field", "finish"))
+    q = SpanNearQuery.new([tq1, tq2], 0, true)
+    check_hits(q, [0,14], true)
+    q = SpanNearQuery.new([tq1, tq2], 0, false)
+    check_hits(q, [0,14,16,30], true)
+    q = SpanNearQuery.new([tq1, tq2], 1, true)
+    check_hits(q, [0,1,13,14])
+    q = SpanNearQuery.new([tq1, tq2], 1, false)
+    check_hits(q, [0,1,13,14,16,17,29,30])
+    q = SpanNearQuery.new([tq1, tq2], 4, true)
+    check_hits(q, [0,1,2,3,4,10,11,12,13,14])
+    q = SpanNearQuery.new([tq1, tq2], 4, false)
+    check_hits(q, [0,1,2,3,4,10,11,12,13,14,16,17,18,19,20,26,27,28,29,30])
+  end
+
+  def test_span_not_query()
+    tq1 = SpanTermQuery.new(Term.new("field", "start"))
+    tq2 = SpanTermQuery.new(Term.new("field", "finish"))
+    tq3 = SpanTermQuery.new(Term.new("field", "two"))
+    tq4 = SpanTermQuery.new(Term.new("field", "five"))
+    nearq1 = SpanNearQuery.new([tq1, tq2], 4, true)
+    nearq2 = SpanNearQuery.new([tq3, tq4], 4, true)
+    q = SpanNotQuery.new(nearq1, nearq2)
+    check_hits(q, [0,1,13,14], true)
+    nearq1 = SpanNearQuery.new([tq1, tq2], 4, false)
+    q = SpanNotQuery.new(nearq1, nearq2)
+    check_hits(q, [0,1,13,14,16,17,29,30])
+    nearq1 = SpanNearQuery.new([tq1, tq3], 4, true)
+    nearq2 = SpanNearQuery.new([tq2, tq4], 8, false)
+    q = SpanNotQuery.new(nearq1, nearq2)
+    check_hits(q, [2,3,4,5,6,7,8,9,10,11,12,15])
+  end
+
+  def test_span_first_query()
+    finish_first = [16,17,18,19,20,21,22,23,24,25,26,27,28,29,30]
+    tq = SpanTermQuery.new(Term.new("field", "finish"))
+    q = SpanFirstQuery.new(tq, 1)
+    check_hits(q, finish_first, true)
+    q = SpanFirstQuery.new(tq, 5)
+    check_hits(q, [0,1,2,3,11,12,13,14]+finish_first, false)
+  end
+
+  def test_span_or_query_query()
+    tq1 = SpanTermQuery.new(Term.new("field", "start"))
+    tq2 = SpanTermQuery.new(Term.new("field", "finish"))
+    tq3 = SpanTermQuery.new(Term.new("field", "five"))
+    nearq1 = SpanNearQuery.new([tq1, tq2], 1, true)
+    nearq2 = SpanNearQuery.new([tq2, tq3], 0, false)
+    q = SpanOrQuery.new([nearq1, nearq2])
+    check_hits(q, [0,1,4,5,9,10,13,14], false)
+    nearq1 = SpanNearQuery.new([tq1, tq2], 0, false)
+    nearq2 = SpanNearQuery.new([tq2, tq3], 1, false)
+    q = SpanOrQuery.new([nearq1, nearq2])
+    check_hits(q, [0,3,4,5,6,8,9,10,11,14,16,30], false)
+  end
+end

data/test/unit/store/tc_fs_store.rb

@@ -0,0 +1,84 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+require File.dirname(__FILE__) + "/tm_store"
+require File.dirname(__FILE__) + "/tm_store_lock"
+
+module Ferret::Store
+
+  class FSDirectory
+    def FSDirectory.directory_cache
+      @@Directories
+    end
+
+    def ref_count
+      @ref_count
+    end
+
+    def get_lock_prefix
+      lock_prefix
+    end
+  end
+end
+
+class FSStoreTest < Test::Unit::TestCase
+  include Ferret::Store
+  include StoreTest
+  include StoreLockTest
+  def setup
+    @dpath = File.join(File.dirname(__FILE__),
+                       '../../temp/fsdir')
+    @dir = FSDirectory.get_directory(@dpath, true)
+  end
+
+  def teardown
+    @dir.refresh()
+    @dir.close()
+  end
+
+  def test_cache
+    dir_path = File.join(File.dirname(__FILE__),
+                         '/../../temp/cachetest')
+    assert(! FSDirectory.directory_cache[dir_path],
+           "this directory should not be cached yet") 
+    @dir1 = FSDirectory.get_directory(dir_path, true)
+    assert(FSDirectory.directory_cache[dir_path],
+           "this directory should now be cached") 
+    assert_equal(@dir1.ref_count, 1,
+                 "There is one reference so the refcount should now be 1")
+    @dir2 = FSDirectory.get_directory(dir_path, true)
+    assert(@dir1 === @dir2,
+           "The directory should be cached so the same directory object should have been returned")
+    assert_equal(@dir1.ref_count, 2,
+                 "There are two references so the refcount should now be 2")
+    @dir1.close
+    assert(FSDirectory.directory_cache[dir_path],
+           "this directory shouldn't have been removed yet") 
+    assert_equal(@dir2.ref_count, 1,
+                 "There is one reference so the refcount should now be 1")
+    @dir2.close
+    assert(! FSDirectory.directory_cache[dir_path],
+           "this directory should have been removed from the cache") 
+  end
+
+  def test_fslock
+    name = "lfile"
+    lock_file_path = @dpath + "/" + @dir.get_lock_prefix() + name
+    if File.exists?(lock_file_path) then
+      File.delete(lock_file_path)
+    end
+    assert(! File.exists?(lock_file_path),
+           "There should be no lock file")
+    lock = @dir.make_lock(name)
+    assert(! File.exists?(lock_file_path),
+           "There should still be no lock file")
+    assert(! @dir.exists?(lock_file_path),
+           "The lock should be hidden by the FSDirectories directory scan")
+    assert(! lock.locked?, "lock shouldn't be locked yet")
+    lock.obtain
+    assert(lock.locked?, "lock should now be locked")
+    assert(File.exists?(lock_file_path),
+           "A lock file should have been created")
+    lock.release
+    assert(! lock.locked?, "lock should be freed again")
+    assert(! File.exists?(lock_file_path), "The lock file should have been deleted")
+  end
+end