ferret 0.1.0

data/test/benchmark/tb_ram_store.rb

@@ -0,0 +1,76 @@
+require File.dirname(__FILE__) + "/../test_helper"
+require 'benchmark'
+
+class RAMStoreTest < Test::Unit::TestCase
+  def setup
+    @dir = Ferret::Store::RAMDirectory.new
+  end
+
+  def teardown
+    @dir.close()
+  end
+
+  def test_rw_bytes
+    bytes = [0x34, 0x87, 0xF9, 0xEA, 0x00, 0xFF]
+    rw_test(bytes, "byte")
+  end
+
+  def test_rw_ints
+    ints = [-2147483648, 2147483647, -1, 0]
+    rw_test(ints, "int")
+  end
+
+  def test_rw_longs
+    longs = [-9223372036854775808, 9223372036854775807, -1, 0]
+    rw_test(longs, "long")
+  end
+
+  def test_rw_uints
+    uints = [0xffffffff, 100000, 0]
+    rw_test(uints, "uint")
+  end
+
+  def test_rw_ulongs
+    ulongs = [0xffffffffffffffff, 100000000000000, 0]
+    rw_test(ulongs, "ulong")
+  end
+
+  def test_rw_vints
+    vints = [ 0xF8DC843342FE3484234987FE98AB987C897D214D123D123458EFBE2E238BACDEB9878790ABCDEF123DEF23988B89C,
+              0x0000000000000000000000000000000000000000,
+              0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF]
+    rw_test(vints, "vint")
+  end
+
+  def test_rw_vlongs
+    vlongs = [ 0xF8DC843342FE3484234987FE98AB987C897D214D123D123458EFBE2E238BACDEB9878790ABCDEF123DEF23988B89C,
+              0x0000000000000000000000000000000000000000,
+              0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF]
+    rw_test(vlongs, "vlong")
+  end
+
+  def test_rw_strings
+    strings = ['This is a ruby ferret test string ~!@#$%^&*()`123456790-=\)_+|', 'This is another string. I\'ll make this one a little longer than the last one. But I guess we need a few shorter ones too.', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', 'ten']
+    rw_test(strings, "string")
+  end
+
+  # this test fills up the output stream so that the buffer will have to be
+  # written a few times. It then uses seek to make sure that it works
+  # correctly
+
+  def rw_test(values, type)
+    puts "\nrw_#{type} test"
+    Benchmark.bmbm do |x|
+      x.report("write") do
+        ostream = @dir.create_output("rw_#{type}.test")
+        1000.times {values.each { |b| ostream.__send__("write_" + type, b) }}
+        ostream.close
+      end
+      x.report("read") do
+        istream = @dir.open_input("rw_#{type}.test")
+        1000.times {values.each { |b| assert_equal(b, istream.__send__("read_" + type), "#{type} should be equal") }}
+        istream.close
+      end
+    end
+  end
+end

data/test/benchmark/tb_rw_vint.rb

@@ -0,0 +1,26 @@
+$:.unshift File.join(File.dirname(__FILE__), '../../lib')
+
+require 'ferret'
+
+vints = [ 9223372036854775807,
+          0x00,
+          0xFFFFFFFFFFFFFFFF]
+t = Time.new
+10.times do
+  dpath = File.join(File.dirname(__FILE__),
+                       'fsdir')
+  dir = Ferret::Store::FSDirectory.get_directory(dpath, true)
+
+  100.times do
+    ostream = dir.create_output("rw_vint.test")
+    300.times { |i| ostream.write_vint(vints[i%3]) }
+    ostream.close
+    istream = dir.open_input("rw_vint.test")
+    300.times { istream.read_vint }
+    istream.close
+  end
+
+  dir.close
+end
+
+puts "took #{Time.new - t} seconds"

data/test/longrunning/tc_numbertools.rb

@@ -0,0 +1,60 @@
+require File.dirname(__FILE__) + "/../../../test_helper"
+
+
+class NumberToolsTest < Test::Unit::TestCase
+  include Lucene::Document
+  def test_near_zero()
+    10.times() do |i|
+      10.times() { |j| subtest_two_longs(i, j) }
+    end
+  end
+
+  def test_max()
+    # make sure the constants convert to their equivelents
+    assert_equal(NumberTools::LONG_MAX_VALUE, NumberTools.s_to_long(NumberTools::MAX_STRING_VALUE))
+    assert_equal(NumberTools::MAX_STRING_VALUE, NumberTools.long_to_s(NumberTools::LONG_MAX_VALUE))
+    # test near MAX, too
+
+    NumberTools::LONG_MAX_VALUE.downto(NumberTools::LONG_MAX_VALUE - 100) do |l|
+      subtest_two_longs(l, l - 1)
+    end
+  end
+
+  def test_min()
+    # make sure the constants convert to their equivelents
+    assert_equal(NumberTools::LONG_MIN_VALUE, NumberTools.s_to_long(NumberTools::MIN_STRING_VALUE))
+    assert_equal(NumberTools::MIN_STRING_VALUE, NumberTools.long_to_s(NumberTools::LONG_MIN_VALUE))
+
+    # test near MIN, too
+    NumberTools::LONG_MIN_VALUE.upto(NumberTools::LONG_MIN_VALUE + 100) do |l|
+      subtest_two_longs(l, l + 1)
+    end
+  end
+
+  def subtest_two_longs(i, j)
+    # convert to strings
+    a = NumberTools.long_to_s(i)
+    b = NumberTools.long_to_s(j)
+
+    # are they the right length?
+    assert_equal(NumberTools::STR_SIZE, a.length())
+    assert_equal(NumberTools::STR_SIZE, b.length())
+
+    # are they the right order?
+    if (i < j)
+        assert(a < b)
+    elsif (i > j)
+        assert(a > b)
+    else
+        assert_equal(a, b)
+    end
+
+    # can we convert them back to longs?
+    i2 = NumberTools.s_to_long(a)
+    j2 = NumberTools.s_to_long(b)
+
+    assert_equal(i, i2)
+    assert_equal(j, j2)
+  end
+
+end

data/test/longrunning/tm_store.rb

@@ -0,0 +1,19 @@
+module StoreTest
+  # declare dir so inheritors can access it.
+  def test_modified_full
+    # difficult to test this one but as file mtime is only stored to the
+    # nearest second. We can assume this test will happen in less than one
+    # second. (I hope)
+    time = Time.new.to_i
+    @dir.touch('mtime_test')
+    time_before = @dir.modified('mtime_test').to_i
+    assert(time_before - time <= 2, "test that mtime is approximately equal to the system time when the file was touched")
+    # wait until the time ticks over one second.
+    time = Time.new while (time.to_i == time_before)
+    time_before_again = @dir.modified('mtime_test').to_i
+    assert_equal(time_before, time_before_again, "the modified time shouldn't change")
+    @dir.touch('mtime_test')
+    time_after = @dir.modified('mtime_test').to_i
+    assert(time_before < time_after, "the modified time should now be greater")
+  end
+end

data/test/test_all.rb

@@ -0,0 +1,9 @@
+$:.unshift File.dirname(__FILE__)
+
+require 'unit/ts_analysis.rb'
+require 'unit/ts_document.rb'
+require 'unit/ts_index.rb'
+require 'unit/ts_query_parser.rb'
+require 'unit/ts_search.rb'
+require 'unit/ts_store.rb'
+require 'unit/ts_utils.rb'

data/test/test_helper.rb

@@ -0,0 +1,6 @@
+$:.unshift File.join(File.dirname(__FILE__), '../lib')
+$:.unshift File.join(File.dirname(__FILE__), '../ext')
+
+require 'test/unit'
+require 'ferret'
+require 'test/unit/index/th_doc'

data/test/unit/analysis/tc_analyzer.rb

@@ -0,0 +1,21 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+class AnalyzerTest < Test::Unit::TestCase
+  include Ferret::Analysis
+  include Ferret::Utils::StringHelper
+
+  def test_analyzer()
+    input = StringReader.new('DBalmain@gmail.com is My E-Mail 523@#$ ADDRESS. 23#@$')
+    a = Analyzer.new()
+    t = a.token_stream("fieldname", input)
+    assert_equal(Token.new("dbalmain", 0, 8), t.next())
+    assert_equal(Token.new("gmail", 9, 14), t.next())
+    assert_equal(Token.new("com", 15, 18), t.next())
+    assert_equal(Token.new("is", 19, 21), t.next())
+    assert_equal(Token.new("my", 22, 24), t.next())
+    assert_equal(Token.new("e", 25, 26), t.next())
+    assert_equal(Token.new("mail", 27, 31), t.next())
+    assert_equal(Token.new("address", 39, 46), t.next())
+    assert(! t.next())
+  end
+end

data/test/unit/analysis/tc_letter_tokenizer.rb

@@ -0,0 +1,20 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+class LetterTokenizerTest < Test::Unit::TestCase
+  include Ferret::Analysis
+  include Ferret::Utils::StringHelper
+
+  def test_lettertokenizer()
+    input = StringReader.new('DBalmain@gmail.com is My e-mail 523@#$ address. 23#@$')
+    t = LetterTokenizer.new(input)
+    assert_equal(Token.new("DBalmain", 0, 8), t.next())
+    assert_equal(Token.new("gmail", 9, 14), t.next())
+    assert_equal(Token.new("com", 15, 18), t.next())
+    assert_equal(Token.new("is", 19, 21), t.next())
+    assert_equal(Token.new("My", 22, 24), t.next())
+    assert_equal(Token.new("e", 25, 26), t.next())
+    assert_equal(Token.new("mail", 27, 31), t.next())
+    assert_equal(Token.new("address", 39, 46), t.next())
+    assert(! t.next())
+  end
+end

data/test/unit/analysis/tc_lower_case_filter.rb

@@ -0,0 +1,20 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+class LowerCaseFilterTest < Test::Unit::TestCase
+  include Ferret::Analysis
+  include Ferret::Utils::StringHelper
+
+  def test_lowercasefilter()
+    input = StringReader.new('DBalmain@gmail.com is My E-Mail 52   #$ ADDRESS. 23#@$')
+    t = LowerCaseFilter.new(WhiteSpaceTokenizer.new(input))
+    assert_equal(Token.new('dbalmain@gmail.com', 0, 18), t.next())
+    assert_equal(Token.new('is', 19, 21), t.next())
+    assert_equal(Token.new('my', 22, 24), t.next())
+    assert_equal(Token.new('e-mail', 25, 31), t.next())
+    assert_equal(Token.new('52', 32, 34), t.next())
+    assert_equal(Token.new('#$', 37, 39), t.next())
+    assert_equal(Token.new('address.', 40, 48), t.next())
+    assert_equal(Token.new('23#@$', 49, 54), t.next())
+    assert(! t.next())
+  end
+end

data/test/unit/analysis/tc_lower_case_tokenizer.rb

@@ -0,0 +1,27 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+class LowerCaseTokenizerTest < Test::Unit::TestCase
+  include Ferret::Analysis
+  include Ferret::Utils::StringHelper
+
+  def test_normalize()
+    lt = LowerCaseTokenizer.new(StringReader.new(""))
+    assert_equal('!', lt.__send__(:normalize,"!"))
+    assert_equal('r', lt.__send__(:normalize,"r"))
+    assert_equal('r', lt.__send__(:normalize,"R"))
+  end
+
+  def test_lowercase_tokenizer()
+    input = StringReader.new('DBalmain@gmail.com is My E-Mail 523@#$ ADDRESS. 23#@$')
+    t = LowerCaseTokenizer.new(input)
+    assert_equal(Token.new("dbalmain", 0, 8), t.next())
+    assert_equal(Token.new("gmail", 9, 14), t.next())
+    assert_equal(Token.new("com", 15, 18), t.next())
+    assert_equal(Token.new("is", 19, 21), t.next())
+    assert_equal(Token.new("my", 22, 24), t.next())
+    assert_equal(Token.new("e", 25, 26), t.next())
+    assert_equal(Token.new("mail", 27, 31), t.next())
+    assert_equal(Token.new("address", 39, 46), t.next())
+    assert(! t.next())
+  end
+end

data/test/unit/analysis/tc_per_field_analyzer_wrapper.rb

@@ -0,0 +1,39 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+class PerFieldAnalyzerWrapperTest < Test::Unit::TestCase
+  include Ferret::Analysis
+  include Ferret::Utils::StringHelper
+  def test_perfieldanalyzerwrapper()
+    aw = PerFieldAnalyzerWrapper.new(Analyzer.new())
+    aw.add_analyzer("abstract", WhiteSpaceAnalyzer.new())
+    aw.add_analyzer("body", StopAnalyzer.new(['is', 'my', 'address']))
+    input = StringReader.new('DBalmain@gmail.com is My e-mail ADDRESS')
+    t = aw.token_stream("title", input)
+    assert_equal(Token.new("dbalmain", 0, 8), t.next())
+    assert_equal(Token.new("gmail", 9, 14), t.next())
+    assert_equal(Token.new("com", 15, 18), t.next())
+    assert_equal(Token.new("is", 19, 21), t.next())
+    assert_equal(Token.new("my", 22, 24), t.next())
+    assert_equal(Token.new("e", 25, 26), t.next())
+    assert_equal(Token.new("mail", 27, 31), t.next())
+    assert_equal(Token.new("address", 32, 39), t.next())
+    assert(! t.next())
+    input.reset()
+    t = aw.token_stream("abstract", input)
+    assert_equal(Token.new('DBalmain@gmail.com', 0, 18), t.next())
+    assert_equal(Token.new('is', 19, 21), t.next())
+    assert_equal(Token.new('My', 22, 24), t.next())
+    assert_equal(Token.new('e-mail', 25, 31), t.next())
+    assert_equal(Token.new("ADDRESS", 32, 39), t.next())
+    if ( token = t.next()): puts token.term_text end
+    assert(! t.next())
+    input.reset()
+    t = aw.token_stream("body", input)
+    assert_equal(Token.new("dbalmain", 0, 8), t.next())
+    assert_equal(Token.new("gmail", 9, 14), t.next())
+    assert_equal(Token.new("com", 15, 18), t.next())
+    assert_equal(Token.new("e", 25, 26), t.next())
+    assert_equal(Token.new("mail", 27, 31), t.next())
+    assert(! t.next())
+  end
+end

data/test/unit/analysis/tc_porter_stem_filter.rb

@@ -0,0 +1,16 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+class PorterStemFilterTest < Test::Unit::TestCase
+  include Ferret::Analysis
+  include Ferret::Utils::StringHelper
+
+  def test_porterstempfilter()
+    input = StringReader.new('breath Breathes BreatHed BREATHING')
+    t = PorterStemFilter.new(LowerCaseFilter.new(WhiteSpaceTokenizer.new(input)))
+    assert_equal(Token.new('breath', 0, 6), t.next())
+    assert_equal(Token.new('breath', 7, 15), t.next())
+    assert_equal(Token.new('breath', 16, 24), t.next())
+    assert_equal(Token.new('breath', 25, 34), t.next())
+    assert(! t.next())
+  end
+end

data/test/unit/analysis/tc_standard_analyzer.rb

@@ -0,0 +1,20 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+class StandardAnalyzerTest < Test::Unit::TestCase
+  include Ferret::Utils::StringHelper
+  include Ferret::Analysis
+
+  def test_lettertokenizer()
+    input = StringReader.new('D.Ba_l-n@gma-l.com AB&Sons Toys\'r\'us you\'re she\'s, #$%^$%*& job@dot I.B.M. the an AnD THEIR')
+    sa = StandardAnalyzer.new()
+    t = sa.token_stream("field", input)
+    assert_equal(Token.new("d.ba_l-n@gma-l.com", 0, 18), t.next())
+    assert_equal(Token.new("ab&sons", 19, 26), t.next())
+    assert_equal(Token.new("toys'r'us", 27, 36), t.next())
+    assert_equal(Token.new("you're", 37, 43), t.next())
+    assert_equal(Token.new("she", 44, 49), t.next())
+    assert_equal(Token.new("job@dot", 60, 67), t.next())
+    assert_equal(Token.new("ibm", 68, 74), t.next())
+    assert(! t.next())
+  end
+end

data/test/unit/analysis/tc_standard_tokenizer.rb

@@ -0,0 +1,20 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+class StandardTokenizerTest < Test::Unit::TestCase
+  include Ferret::Analysis
+  include Ferret::Utils::StringHelper
+
+  def test_lettertokenizer()
+    input = StringReader.new('DBalmain@gmail.com is My e-mail 523@#$ address. 23#@$')
+    t = StandardTokenizer.new(input)
+    assert_equal(Token.new("DBalmain@gmail.com", 0, 18), t.next())
+    assert_equal(Token.new("is", 19, 21), t.next())
+    assert_equal(Token.new("My", 22, 24), t.next())
+    assert_equal(Token.new("e", 25, 26), t.next())
+    assert_equal(Token.new("mail", 27, 31), t.next())
+    assert_equal(Token.new("523", 32, 35), t.next())
+    assert_equal(Token.new("address", 39, 46), t.next())
+    assert_equal(Token.new("23", 48, 50), t.next())
+    assert(! t.next())
+  end
+end

data/test/unit/analysis/tc_stop_analyzer.rb

@@ -0,0 +1,20 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+class StopAnalyzerTest < Test::Unit::TestCase
+  include Ferret::Analysis
+  include Ferret::Utils::StringHelper
+
+  def test_stopanalyzer()
+    input = StringReader.new('The Quick AND the DEAD the and to it there their')
+    a = StopAnalyzer.new()
+    t = a.token_stream("field name", input)
+    assert_equal(Token.new('quick', 4, 9), t.next())
+    assert_equal(Token.new('dead', 18, 22), t.next())
+    assert(! t.next())
+    input = StringReader.new("David Balmain")
+    a = StopAnalyzer.new(["david"])
+    t = a.token_stream("field name", input)
+    assert_equal(Token.new('balmain', 6, 13), t.next())
+    assert(! t.next())
+  end
+end

data/test/unit/analysis/tc_stop_filter.rb

@@ -0,0 +1,14 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+class StopFilterTest < Test::Unit::TestCase
+  include Ferret::Analysis
+  include Ferret::Utils::StringHelper
+
+  def test_stopfilter()
+    input = StringReader.new('The Quick AND the DEAD the and to it there their')
+    t = StopFilter.new_with_file(LowerCaseTokenizer.new(input), File.dirname(__FILE__) + '/data/wordfile')
+    assert_equal(Token.new('quick', 4, 9), t.next())
+    assert_equal(Token.new('dead', 18, 22), t.next())
+    assert(! t.next())
+  end
+end

data/test/unit/analysis/tc_white_space_analyzer.rb

@@ -0,0 +1,21 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+class WhiteSpaceAnalyzerTest < Test::Unit::TestCase
+  include Ferret::Analysis
+  include Ferret::Utils::StringHelper
+
+  def test_whitespaceanalyzer()
+    input = StringReader.new('DBalmain@gmail.com is My e-mail 52   #$ address. 23#@$')
+    a = WhiteSpaceAnalyzer.new()
+    t = a.token_stream("field", input)
+    assert_equal(Token.new('DBalmain@gmail.com', 0, 18), t.next())
+    assert_equal(Token.new('is', 19, 21), t.next())
+    assert_equal(Token.new('My', 22, 24), t.next())
+    assert_equal(Token.new('e-mail', 25, 31), t.next())
+    assert_equal(Token.new('52', 32, 34), t.next())
+    assert_equal(Token.new('#$', 37, 39), t.next())
+    assert_equal(Token.new('address.', 40, 48), t.next())
+    assert_equal(Token.new('23#@$', 49, 54), t.next())
+    assert(! t.next())
+  end
+end