ferret 0.9.6 → 0.10.0

data/test/unit/analysis/data/wordfile

@@ -1,6 +0,0 @@
-and
-to
-it
-the
-there
-their

data/test/unit/analysis/rtc_letter_tokenizer.rb

@@ -1,20 +0,0 @@
-require File.dirname(__FILE__) + "/../../test_helper"
-
-class LetterTokenizerTest < Test::Unit::TestCase
-  include Ferret::Analysis
-  include Ferret::Utils::StringHelper
-
-  def test_lettertokenizer()
-    input = StringReader.new('DBalmain@gmail.com is My e-mail 523@#$ address. 23#@$')
-    t = LetterTokenizer.new(input)
-    assert_equal(Token.new("DBalmain", 0, 8), t.next())
-    assert_equal(Token.new("gmail", 9, 14), t.next())
-    assert_equal(Token.new("com", 15, 18), t.next())
-    assert_equal(Token.new("is", 19, 21), t.next())
-    assert_equal(Token.new("My", 22, 24), t.next())
-    assert_equal(Token.new("e", 25, 26), t.next())
-    assert_equal(Token.new("mail", 27, 31), t.next())
-    assert_equal(Token.new("address", 39, 46), t.next())
-    assert(! t.next())
-  end
-end

data/test/unit/analysis/rtc_lower_case_filter.rb

@@ -1,20 +0,0 @@
-require File.dirname(__FILE__) + "/../../test_helper"
-
-class LowerCaseFilterTest < Test::Unit::TestCase
-  include Ferret::Analysis
-  include Ferret::Utils::StringHelper
-
-  def test_lowercasefilter()
-    input = StringReader.new('DBalmain@gmail.com is My E-Mail 52   #$ ADDRESS. 23#@$')
-    t = LowerCaseFilter.new(WhiteSpaceTokenizer.new(input))
-    assert_equal(Token.new('dbalmain@gmail.com', 0, 18), t.next())
-    assert_equal(Token.new('is', 19, 21), t.next())
-    assert_equal(Token.new('my', 22, 24), t.next())
-    assert_equal(Token.new('e-mail', 25, 31), t.next())
-    assert_equal(Token.new('52', 32, 34), t.next())
-    assert_equal(Token.new('#$', 37, 39), t.next())
-    assert_equal(Token.new('address.', 40, 48), t.next())
-    assert_equal(Token.new('23#@$', 49, 54), t.next())
-    assert(! t.next())
-  end
-end

data/test/unit/analysis/rtc_lower_case_tokenizer.rb

@@ -1,27 +0,0 @@
-require File.dirname(__FILE__) + "/../../test_helper"
-
-class LowerCaseTokenizerTest < Test::Unit::TestCase
-  include Ferret::Analysis
-  include Ferret::Utils::StringHelper
-
-  def test_normalize()
-    lt = LowerCaseTokenizer.new(StringReader.new(""))
-    assert_equal('!', lt.__send__(:normalize,"!"))
-    assert_equal('r', lt.__send__(:normalize,"r"))
-    assert_equal('r', lt.__send__(:normalize,"R"))
-  end
-
-  def test_lowercase_tokenizer()
-    input = StringReader.new('DBalmain@gmail.com is My E-Mail 523@#$ ADDRESS. 23#@$')
-    t = LowerCaseTokenizer.new(input)
-    assert_equal(Token.new("dbalmain", 0, 8), t.next())
-    assert_equal(Token.new("gmail", 9, 14), t.next())
-    assert_equal(Token.new("com", 15, 18), t.next())
-    assert_equal(Token.new("is", 19, 21), t.next())
-    assert_equal(Token.new("my", 22, 24), t.next())
-    assert_equal(Token.new("e", 25, 26), t.next())
-    assert_equal(Token.new("mail", 27, 31), t.next())
-    assert_equal(Token.new("address", 39, 46), t.next())
-    assert(! t.next())
-  end
-end

data/test/unit/analysis/rtc_per_field_analyzer_wrapper.rb

@@ -1,39 +0,0 @@
-require File.dirname(__FILE__) + "/../../test_helper"
-
-class PerFieldAnalyzerWrapperTest < Test::Unit::TestCase
-  include Ferret::Analysis
-  include Ferret::Utils::StringHelper
-  def test_perfieldanalyzerwrapper()
-    aw = PerFieldAnalyzerWrapper.new(Analyzer.new())
-    aw.add_analyzer("abstract", WhiteSpaceAnalyzer.new())
-    aw.add_analyzer("body", StopAnalyzer.new(['is', 'my', 'address']))
-    input = StringReader.new('DBalmain@gmail.com is My e-mail ADDRESS')
-    t = aw.token_stream("title", input)
-    assert_equal(Token.new("dbalmain", 0, 8), t.next())
-    assert_equal(Token.new("gmail", 9, 14), t.next())
-    assert_equal(Token.new("com", 15, 18), t.next())
-    assert_equal(Token.new("is", 19, 21), t.next())
-    assert_equal(Token.new("my", 22, 24), t.next())
-    assert_equal(Token.new("e", 25, 26), t.next())
-    assert_equal(Token.new("mail", 27, 31), t.next())
-    assert_equal(Token.new("address", 32, 39), t.next())
-    assert(! t.next())
-    input.reset()
-    t = aw.token_stream("abstract", input)
-    assert_equal(Token.new('DBalmain@gmail.com', 0, 18), t.next())
-    assert_equal(Token.new('is', 19, 21), t.next())
-    assert_equal(Token.new('My', 22, 24), t.next())
-    assert_equal(Token.new('e-mail', 25, 31), t.next())
-    assert_equal(Token.new("ADDRESS", 32, 39), t.next())
-    if ( token = t.next()): puts token.text end
-    assert(! t.next())
-    input.reset()
-    t = aw.token_stream("body", input)
-    assert_equal(Token.new("dbalmain", 0, 8), t.next())
-    assert_equal(Token.new("gmail", 9, 14), t.next())
-    assert_equal(Token.new("com", 15, 18), t.next())
-    assert_equal(Token.new("e", 25, 26), t.next())
-    assert_equal(Token.new("mail", 27, 31), t.next())
-    assert(! t.next())
-  end
-end

data/test/unit/analysis/rtc_porter_stem_filter.rb

@@ -1,16 +0,0 @@
-require File.dirname(__FILE__) + "/../../test_helper"
-
-class PorterStemFilterTest < Test::Unit::TestCase
-  include Ferret::Analysis
-  include Ferret::Utils::StringHelper
-
-  def test_porterstempfilter()
-    input = StringReader.new('breath Breathes BreatHed BREATHING')
-    t = PorterStemFilter.new(LowerCaseFilter.new(WhiteSpaceTokenizer.new(input)))
-    assert_equal(Token.new('breath', 0, 6), t.next())
-    assert_equal(Token.new('breath', 7, 15), t.next())
-    assert_equal(Token.new('breath', 16, 24), t.next())
-    assert_equal(Token.new('breath', 25, 34), t.next())
-    assert(! t.next())
-  end
-end

data/test/unit/analysis/rtc_standard_analyzer.rb

@@ -1,20 +0,0 @@
-require File.dirname(__FILE__) + "/../../test_helper"
-
-class StandardAnalyzerTest < Test::Unit::TestCase
-  include Ferret::Utils::StringHelper
-  include Ferret::Analysis
-
-  def test_standard_analyzer()
-    input = StringReader.new('D.Ba_l-n@gma-l.com AB&Sons Toys\'r\'us you\'re she\'s, #$%^$%*& job@dot I.B.M. the an AnD THEIR')
-    sa = StandardAnalyzer.new()
-    t = sa.token_stream("field", input)
-    assert_equal(Token.new("d.ba_l-n@gma-l.com", 0, 18), t.next())
-    assert_equal(Token.new("ab&sons", 19, 26), t.next())
-    assert_equal(Token.new("toys'r'us", 27, 36), t.next())
-    assert_equal(Token.new("you're", 37, 43), t.next())
-    assert_equal(Token.new("she", 44, 49), t.next())
-    assert_equal(Token.new("job@dot", 60, 67), t.next())
-    assert_equal(Token.new("ibm", 68, 74), t.next())
-    assert(! t.next())
-  end
-end

data/test/unit/analysis/rtc_standard_tokenizer.rb

@@ -1,20 +0,0 @@
-require File.dirname(__FILE__) + "/../../test_helper"
-
-class StandardTokenizerTest < Test::Unit::TestCase
-  include Ferret::Analysis
-  include Ferret::Utils::StringHelper
-
-  def test_lettertokenizer()
-    input = StringReader.new('DBalmain@gmail.com is My e-mail 523@#$ address. 23#@$')
-    t = StandardTokenizer.new(input)
-    assert_equal(Token.new("DBalmain@gmail.com", 0, 18), t.next())
-    assert_equal(Token.new("is", 19, 21), t.next())
-    assert_equal(Token.new("My", 22, 24), t.next())
-    assert_equal(Token.new("e", 25, 26), t.next())
-    assert_equal(Token.new("mail", 27, 31), t.next())
-    assert_equal(Token.new("523", 32, 35), t.next())
-    assert_equal(Token.new("address", 39, 46), t.next())
-    assert_equal(Token.new("23", 48, 50), t.next())
-    assert(! t.next())
-  end
-end

data/test/unit/analysis/rtc_stop_analyzer.rb

@@ -1,20 +0,0 @@
-require File.dirname(__FILE__) + "/../../test_helper"
-
-class StopAnalyzerTest < Test::Unit::TestCase
-  include Ferret::Analysis
-  include Ferret::Utils::StringHelper
-
-  def test_stopanalyzer()
-    input = StringReader.new('The Quick AND the DEAD the and to it there their')
-    a = StopAnalyzer.new()
-    t = a.token_stream("field name", input)
-    assert_equal(Token.new('quick', 4, 9), t.next())
-    assert_equal(Token.new('dead', 18, 22), t.next())
-    assert(! t.next())
-    input = StringReader.new("David Balmain")
-    a = StopAnalyzer.new(["david"])
-    t = a.token_stream("field name", input)
-    assert_equal(Token.new('balmain', 6, 13), t.next())
-    assert(! t.next())
-  end
-end

data/test/unit/analysis/rtc_stop_filter.rb

@@ -1,14 +0,0 @@
-require File.dirname(__FILE__) + "/../../test_helper"
-
-class StopFilterTest < Test::Unit::TestCase
-  include Ferret::Analysis
-  include Ferret::Utils::StringHelper
-
-  def test_stopfilter()
-    input = StringReader.new('The Quick AND the DEAD the and to it there their')
-    t = StopFilter.new_with_file(LowerCaseTokenizer.new(input), File.dirname(__FILE__) + '/data/wordfile')
-    assert_equal(Token.new('quick', 4, 9), t.next())
-    assert_equal(Token.new('dead', 18, 22), t.next())
-    assert(! t.next())
-  end
-end

data/test/unit/analysis/rtc_white_space_analyzer.rb

@@ -1,21 +0,0 @@
-require File.dirname(__FILE__) + "/../../test_helper"
-
-class WhiteSpaceAnalyzerTest < Test::Unit::TestCase
-  include Ferret::Analysis
-  include Ferret::Utils::StringHelper
-
-  def test_whitespaceanalyzer()
-    input = StringReader.new('DBalmain@gmail.com is My e-mail 52   #$ address. 23#@$')
-    a = WhiteSpaceAnalyzer.new()
-    t = a.token_stream("field", input)
-    assert_equal(Token.new('DBalmain@gmail.com', 0, 18), t.next())
-    assert_equal(Token.new('is', 19, 21), t.next())
-    assert_equal(Token.new('My', 22, 24), t.next())
-    assert_equal(Token.new('e-mail', 25, 31), t.next())
-    assert_equal(Token.new('52', 32, 34), t.next())
-    assert_equal(Token.new('#$', 37, 39), t.next())
-    assert_equal(Token.new('address.', 40, 48), t.next())
-    assert_equal(Token.new('23#@$', 49, 54), t.next())
-    assert(! t.next())
-  end
-end

data/test/unit/analysis/rtc_white_space_tokenizer.rb

@@ -1,20 +0,0 @@
-require File.dirname(__FILE__) + "/../../test_helper"
-
-class WhiteSpaceTokenizerTest < Test::Unit::TestCase
-  include Ferret::Analysis
-  include Ferret::Utils::StringHelper
-
-  def test_whitespacetokenizer()
-    input = StringReader.new('DBalmain@gmail.com is My e-mail 52   #$ address. 23#@$')
-    t = WhiteSpaceTokenizer.new(input)
-    assert_equal(Token.new('DBalmain@gmail.com', 0, 18), t.next())
-    assert_equal(Token.new('is', 19, 21), t.next())
-    assert_equal(Token.new('My', 22, 24), t.next())
-    assert_equal(Token.new('e-mail', 25, 31), t.next())
-    assert_equal(Token.new('52', 32, 34), t.next())
-    assert_equal(Token.new('#$', 37, 39), t.next())
-    assert_equal(Token.new('address.', 40, 48), t.next())
-    assert_equal(Token.new('23#@$', 49, 54), t.next())
-    assert(! t.next())
-  end
-end

data/test/unit/analysis/rtc_word_list_loader.rb

@@ -1,32 +0,0 @@
-require File.dirname(__FILE__) + "/../../test_helper"
-
-class WordListLoaderTest < Test::Unit::TestCase
-  include Ferret::Analysis
-  def test_word_set_from_file()
-    wl = WordListLoader.word_set_from_file(File.dirname(__FILE__) + '/data/wordfile')
-    assert_equal(6, wl.size())
-    assert(wl.member?('and'))
-    assert(wl.member?('to'))
-    assert(wl.member?('it'))
-    assert(wl.member?('the'))
-    assert(wl.member?('there'))
-    assert(wl.member?('their'))
-    assert(!wl.member?('horse'))
-    assert(!wl.member?('judo'))
-    assert(!wl.member?('dairy'))
-  end
-
-  def test_word_set_from_array()
-    wl = WordListLoader.word_set_from_array(['and','to','it','the','there','their'])
-    assert_equal(6, wl.size())
-    assert(wl.member?('and'))
-    assert(wl.member?('to'))
-    assert(wl.member?('it'))
-    assert(wl.member?('the'))
-    assert(wl.member?('there'))
-    assert(wl.member?('their'))
-    assert(!wl.member?('horse'))
-    assert(!wl.member?('judo'))
-    assert(!wl.member?('dairy'))
-  end
-end

data/test/unit/analysis/tc_token.rb

@@ -1,25 +0,0 @@
-require File.dirname(__FILE__) + "/../../test_helper"
-
-class TokenTest < Test::Unit::TestCase
-  include Ferret::Analysis
-
-  def test_token()
-    tk1 = Token.new("DBalmain", 1, 8, 5, "token")
-    assert_equal(tk1, Token.new("DBalmain", 1, 8))
-    assert_not_equal(tk1, Token.new("DBalmain", 0, 8))
-    assert_not_equal(tk1, Token.new("DBalmain", 1, 9))
-    assert_not_equal(tk1, Token.new("Dbalmain", 1, 8))
-    assert(tk1 < Token.new("CBalmain", 2, 7))
-    assert(tk1 > Token.new("EBalmain", 0, 9))
-    assert(tk1 < Token.new("CBalmain", 1, 9))
-    assert(tk1 > Token.new("EBalmain", 1, 7))
-    assert(tk1 < Token.new("EBalmain", 1, 8))
-    assert(tk1 > Token.new("CBalmain", 1, 8))
-    assert_equal("DBalmain", tk1.text)
-    tk1.text = "Hello"
-    assert_equal("Hello", tk1.text)
-    assert_equal(1, tk1.start_offset)
-    assert_equal(8, tk1.end_offset)
-    assert_equal(5, tk1.pos_inc)
-  end
-end

data/test/unit/document/rtc_field.rb

@@ -1,28 +0,0 @@
-require File.dirname(__FILE__) + "/../../test_helper"
-
-
-class FieldTest < Test::Unit::TestCase
-  include Ferret::Document
-  include Ferret::Utils
-
-  def test_store()
-    assert_equal("COMPRESS", Field::Store::COMPRESS.to_s)
-    assert_equal("YES", Field::Store::YES.to_s)
-    assert_equal("NO", Field::Store::NO.to_s)
-  end
-
-  def test_index()
-    assert_equal("TOKENIZED", Field::Index::TOKENIZED.to_s)
-    assert_equal("UNTOKENIZED", Field::Index::UNTOKENIZED.to_s)
-    assert_equal("NO", Field::Index::NO.to_s)
-    assert_equal("NO_NORMS", Field::Index::NO_NORMS.to_s)
-  end
-
-  def test_term_vector()
-    assert_equal("YES", Field::TermVector::YES.to_s)
-    assert_equal("NO", Field::TermVector::NO.to_s)
-    assert_equal("WITH_POSITIONS", Field::TermVector::WITH_POSITIONS.to_s)
-    assert_equal("WITH_OFFSETS", Field::TermVector::WITH_OFFSETS.to_s)
-    assert_equal("WITH_POSITIONS_OFFSETS", Field::TermVector::WITH_POSITIONS_OFFSETS.to_s)
-  end
-end

data/test/unit/document/tc_document.rb

@@ -1,47 +0,0 @@
-require File.dirname(__FILE__) + "/../../test_helper"
-
-class DocumentTest < Test::Unit::TestCase
-  include Ferret::Document
-  def test_document()
-    doc = Document.new()
-    f11 = Field.new("field1", "value1", Field::Store::YES, Field::Index::NO)
-    f12 = Field.new("field1", "value2", Field::Store::YES, Field::Index::NO)
-    f13 = Field.new("field1", "value3", Field::Store::YES, Field::Index::NO)
-    f21 = Field.new("field2", "value1", Field::Store::YES, Field::Index::NO)
-    doc.add_field(f11)
-    doc.add_field(f12)
-    doc.add_field(f13)
-    doc.add_field(f21)
-    assert_equal(3, doc.fields("field1").size)
-    assert_equal(1, doc.fields("field2").size)
-    field = doc.remove_field("field1")
-    assert_equal(2, doc.fields("field1").size)
-    assert_equal(f11, field)
-    assert_equal("value2 value3", doc.values("field1"))
-    doc.remove_fields("field1")
-    assert_equal(nil, doc.field("field1"))
-  end
-
-  def test_binary_string()
-    tmp = []
-    256.times {|i| tmp[i] = i}
-    bin1 = tmp.pack("c*")
-    tmp = []
-    56.times {|i| tmp[i] = i}
-    bin2 = tmp.pack("c*")
-    doc = Document.new()
-    fs1 = Field.new("field1", "value1", Field::Store::YES, Field::Index::NO)
-    fs2 = Field.new("field1", "value2", Field::Store::YES, Field::Index::NO)
-    fb1 = Field.new_binary_field("field1", bin1, Field::Store::YES)
-    fb2 = Field.new_binary_field("field1", bin2, Field::Store::YES)
-
-    doc.add_field(fs1)
-    doc.add_field(fs2)
-    doc.add_field(fb1)
-    doc.add_field(fb2)
-
-    assert_equal(4, doc.fields("field1").size)
-    assert_equal("value1 value2", doc.values("field1").strip)
-    assert_equal([bin1, bin2], doc.binaries("field1"))
-  end
-end

data/test/unit/document/tc_field.rb

@@ -1,98 +0,0 @@
-require File.dirname(__FILE__) + "/../../test_helper"
-
-
-class FieldTest < Test::Unit::TestCase
-  include Ferret::Document
-  include Ferret::Utils
-
-  def test_store()
-    assert_not_nil(Field::Store::COMPRESS)
-    assert_not_nil(Field::Store::YES)
-    assert_not_nil(Field::Store::NO)
-  end
-
-  def test_index()
-    assert_not_nil(Field::Index::TOKENIZED)
-    assert_not_nil(Field::Index::UNTOKENIZED)
-    assert_not_nil(Field::Index::NO)
-    assert_not_nil(Field::Index::NO_NORMS)
-  end
-
-  def test_term_vector()
-    assert_not_nil(Field::TermVector::YES)
-    assert_not_nil(Field::TermVector::NO)
-    assert_not_nil(Field::TermVector::WITH_POSITIONS)
-    assert_not_nil(Field::TermVector::WITH_OFFSETS)
-    assert_not_nil(Field::TermVector::WITH_POSITIONS_OFFSETS)
-  end
-
-  def test_standard_field()
-    f = Field.new("name", "value", Field::Store::COMPRESS, Field::Index::TOKENIZED)
-    assert_equal("name", f.name)
-    assert_equal("value", f.data)
-    assert_equal(true, f.stored?)
-    assert_equal(true, f.compressed?)
-    assert_equal(true, f.indexed?)
-    assert_equal(true, f.tokenized?)
-    assert_equal(false, f.store_term_vector?)
-    assert_equal(false, f.store_offsets?)
-    assert_equal(false, f.store_positions?)
-    assert_equal(false, f.omit_norms?)
-    assert_equal(false, f.binary?)
-    assert_equal("stored/compressed,indexed,tokenized,<name:value>", f.to_s)
-    f.data = "183"
-    f.boost = 0.001
-    assert_equal("183", f.data)
-    assert(0.001 =~ f.boost)
-  end
-
-  def test_set_store()
-    f = Field.new("name", "", Field::Store::COMPRESS, Field::Index::TOKENIZED)
-    f.store = Field::Store::NO
-    assert_equal(false, f.stored?)
-    assert_equal(false, f.compressed?)
-    assert_equal("indexed,tokenized,<name:>", f.to_s)
-  end
-
-  def test_set_index()
-    f = Field.new("name", "value", Field::Store::COMPRESS, Field::Index::TOKENIZED)
-    f.index = Field::Index::NO
-    assert_equal(false, f.indexed?)
-    assert_equal(false, f.tokenized?)
-    assert_equal(false, f.omit_norms?)
-    assert_equal("stored/compressed,<name:value>", f.to_s)
-    f.index = Field::Index::NO_NORMS
-    assert_equal(true, f.indexed?)
-    assert_equal(false, f.tokenized?)
-    assert_equal(true, f.omit_norms?)
-    assert_equal("stored/compressed,indexed,omit_norms,<name:value>", f.to_s)
-  end
-
-  def test_set_term_vector()
-    f = Field.new("name", "value", Field::Store::COMPRESS, Field::Index::TOKENIZED)
-    f.term_vector = Field::TermVector::WITH_POSITIONS_OFFSETS
-    assert_equal(true, f.store_term_vector?)
-    assert_equal(true, f.store_offsets?)
-    assert_equal(true, f.store_positions?)
-    assert_equal("stored/compressed,indexed,tokenized,store_term_vector,store_offsets,store_positions,<name:value>", f.to_s)
-  end
-
-  def test_new_binary_field()
-    tmp = []
-    256.times {|i| tmp[i] = i}
-    bin = tmp.pack("c*")
-    f = Field.new_binary_field("name", bin, Field::Store::YES)
-    assert_equal("name", f.name)
-    assert_equal(bin, f.data)
-    assert_equal(true, f.stored?)
-    assert_equal(false, f.compressed?)
-    assert_equal(false, f.indexed?)
-    assert_equal(false, f.tokenized?)
-    assert_equal(false, f.store_term_vector?)
-    assert_equal(false, f.store_offsets?)
-    assert_equal(false, f.store_positions?)
-    assert_equal(false, f.omit_norms?)
-    assert_equal(true, f.binary?)
-    assert_equal("stored/uncompressed,binary,<name:=bin_data=>", f.to_s)
-  end
-end