ferret 0.1.0

data/test/unit/utils/tc_number_tools.rb

@@ -0,0 +1,59 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+
+class NumberToolsTest < Test::Unit::TestCase
+  include Ferret::Utils
+
+  def test_near_zero()
+    2.times() do |i|
+      2.times() { |j| subtest_two_longs(i, j) }
+    end
+  end
+
+  def test_max()
+    # make sure the constants convert to their equivelents
+    assert_equal(NumberTools::LONG_MAX_VALUE, NumberTools.s_to_long(NumberTools::MAX_STRING_VALUE))
+    assert_equal(NumberTools::MAX_STRING_VALUE, NumberTools.long_to_s(NumberTools::LONG_MAX_VALUE))
+    # test near MAX, too
+
+    l = NumberTools::LONG_MAX_VALUE
+    subtest_two_longs(l, l - 1)
+  end
+
+  def test_min()
+    # make sure the constants convert to their equivelents
+    assert_equal(NumberTools::LONG_MIN_VALUE, NumberTools.s_to_long(NumberTools::MIN_STRING_VALUE))
+    assert_equal(NumberTools::MIN_STRING_VALUE, NumberTools.long_to_s(NumberTools::LONG_MIN_VALUE))
+
+    # test near MIN, too
+    l = NumberTools::LONG_MIN_VALUE
+    subtest_two_longs(l, l + 1)
+  end
+
+  def subtest_two_longs(i, j)
+    # convert to strings
+    a = NumberTools.long_to_s(i)
+    b = NumberTools.long_to_s(j)
+
+    # are they the right length?
+    assert_equal(NumberTools::STR_SIZE, a.length())
+    assert_equal(NumberTools::STR_SIZE, b.length())
+
+    # are they the right order?
+    if (i < j)
+        assert(a < b)
+    elsif (i > j)
+        assert(a > b)
+    else
+        assert_equal(a, b)
+    end
+
+    # can we convert them back to longs?
+    i2 = NumberTools.s_to_long(a)
+    j2 = NumberTools.s_to_long(b)
+
+    assert_equal(i, i2)
+    assert_equal(j, j2)
+  end
+
+end

data/test/unit/utils/tc_parameter.rb

@@ -0,0 +1,40 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+
+class ParameterTest < Test::Unit::TestCase
+  include Ferret::Utils
+
+  class Param1 < Parameter
+    VAL1 = Param1.new("VAL")
+  end
+
+  class Param2 < Parameter
+    VAL1 = Param2.new("VAL")
+  end
+
+  def test_parameter_cmp()
+    assert_raise(ArgumentError) do
+      class <<Param1
+         v = Param1.new("VAL")
+      end
+    end
+    assert_raise(ArgumentError) do
+      class <<Param1
+         v = Param2.new("VAL")
+      end
+    end
+      
+    p1 = Param1::VAL1
+    p2 = Param1::VAL1
+    p3 = Param2::VAL1
+    assert_equal(p1, p2)
+    assert_not_equal(p1, p3)
+  end
+
+  def test_marshalling()
+    p1 = Param1::VAL1
+    data = Marshal.dump(p1)
+    p2 = Marshal.load(data)
+    assert_equal(p1, p2)
+  end
+end

data/test/unit/utils/tc_priority_queue.rb

@@ -0,0 +1,62 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+
+class PriorityQueueTest < Test::Unit::TestCase
+  include Ferret::Utils
+
+  PQ_STRESS_SIZE = 1000
+
+  def test_pq()
+    pq = PriorityQueue.new(4)
+    assert_equal(0, pq.size)
+    pq.push("bword")
+    assert_equal(1, pq.size)
+    assert_equal("bword", pq.top)
+    pq.push("cword")
+    assert_equal(2, pq.size)
+    assert_equal("bword", pq.top)
+    pq.push("aword")
+    assert_equal(3, pq.size)
+    assert_equal("aword", pq.top)
+    pq.push("dword")
+    assert_equal(4, pq.size)
+    assert_equal("aword", pq.top)
+    assert_equal("aword", pq.pop())
+    assert_equal(3, pq.size)
+    assert_equal("bword", pq.pop())
+    assert_equal(2, pq.size)
+    assert_equal("cword", pq.pop())
+    assert_equal(1, pq.size)
+    assert_equal("dword", pq.pop())
+    assert_equal(0, pq.size)
+  end
+
+  def test_pq_clear()
+    pq = PriorityQueue.new(3)
+    pq.push("word1")
+    pq.push("word2")
+    pq.push("word3")
+    assert_equal(3, pq.size)
+    pq.clear()
+    assert_equal(0, pq.size)
+  end
+
+
+  #define PQ_STRESS_SIZE 1000
+  def test_stress_pq()
+    pq = PriorityQueue.new(PQ_STRESS_SIZE)
+    PQ_STRESS_SIZE.times do
+      pq.push("<#{(rand * PQ_STRESS_SIZE).to_i}>")
+    end
+
+    prev = pq.pop()
+    (PQ_STRESS_SIZE - 1).times do
+      curr = pq.pop()
+      if (prev > curr)
+        assert(prev <= curr, "previous #{prev} should be less than current #{curr}")
+      end
+      prev = curr
+    end
+    pq.clear()
+  end
+end

data/test/unit/utils/tc_string_helper.rb

@@ -0,0 +1,21 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+
+class StringHelperTest < Test::Unit::TestCase
+  include Ferret::Utils
+
+  def test_string_difference()
+    assert_equal(3, StringHelper.string_difference("David", "Dave"))
+    assert_equal(0, StringHelper.string_difference("David", "Erik"))
+    assert_equal(4, StringHelper.string_difference("book", "bookworm"))
+  end
+
+  def test_string_reader
+    sr = StringHelper::StringReader.new("TestString")
+    assert_equal("T", sr.read(1))
+    assert_equal("es", sr.read(2))
+    assert_equal("tStr", sr.read(4))
+    assert_equal("ing", sr.read(100))
+    assert_nil(sr.read(100))
+  end
+end

data/test/unit/utils/tc_weak_key_hash.rb

@@ -0,0 +1,25 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+
+class WeakKeyHashTest < Test::Unit::TestCase
+  include Ferret::Utils
+
+  def test_marshalling()
+    w = WeakKeyHash.new()
+    a = []
+    10.times {|i| a[i] = "#{i}"; w[a[i]] = i }
+    10.times {|i| assert_equal(i, w[a[i]]) }
+    10.times {|i| a[i] = nil; }
+    #puts w
+    
+    # this is a hack to get the GC to collect the last ref created above
+    x = WeakKeyHash.new()
+    10.times {|i| a[i] = "#{i}"; x[a[i]] = i }
+
+    assert_equal(10, w.size)
+    GC.start
+    #puts w.size
+    #puts w
+    assert(0, w.size)
+  end
+end

metadata

@@ -0,0 +1,251 @@
+--- !ruby/object:Gem::Specification 
+rubygems_version: 0.8.11
+specification_version: 1
+name: ferret
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+date: 2005-10-19 00:00:00 +09:00
+summary: Ruby indexing library.
+require_paths: 
+  - lib
+email: dbalmain@gmail.com
+homepage: http://ferret.davebalmain.com
+rubyforge_project: ferret
+description: Ferret is a port of the Java Lucene project. It is a powerful indexing and search library.
+autorequire: 
+default_executable: 
+bindir: bin
+has_rdoc: true
+required_ruby_version: !ruby/object:Gem::Version::Requirement 
+  requirements: 
+    - 
+      - ">"
+      - !ruby/object:Gem::Version 
+        version: 0.0.0
+  version: 
+platform: ruby
+signing_key: 
+cert_chain: 
+authors: 
+  - David Balmain
+files: 
+  - setup.rb
+  - Rakefile
+  - README
+  - MIT-LICENSE
+  - TODO
+  - TUTORIAL
+  - ext/index_io.c
+  - ext/term_buffer.c
+  - ext/ram_directory.c
+  - ext/priority_queue.c
+  - ext/string_helper.c
+  - ext/segment_merge_queue.c
+  - ext/extconf.rb
+  - ext/ferret.c
+  - ext/term.c
+  - ext/ferret.h
+  - ext/util.c
+  - ext/tags
+  - lib/ferret.rb
+  - lib/ferret/analysis.rb
+  - lib/ferret/document.rb
+  - lib/ferret/index.rb
+  - lib/ferret/search.rb
+  - lib/ferret/query_parser.rb
+  - lib/ferret/stemmers.rb
+  - lib/ferret/store.rb
+  - lib/ferret/utils.rb
+  - lib/ferret/analysis/standard_tokenizer.rb
+  - lib/ferret/analysis/word_list_loader.rb
+  - lib/ferret/analysis/tokenizers.rb
+  - lib/ferret/analysis/token.rb
+  - lib/ferret/analysis/analyzers.rb
+  - lib/ferret/analysis/token_filters.rb
+  - lib/ferret/analysis/token_stream.rb
+  - lib/ferret/document/document.rb
+  - lib/ferret/document/field.rb
+  - lib/ferret/index/fields_io.rb
+  - lib/ferret/index/index.rb
+  - lib/ferret/index/segment_merge_queue.rb
+  - lib/ferret/index/segment_term_enum.rb
+  - lib/ferret/index/term_info.rb
+  - lib/ferret/index/field_infos.rb
+  - lib/ferret/index/segment_reader.rb
+  - lib/ferret/index/index_file_names.rb
+  - lib/ferret/index/index_reader.rb
+  - lib/ferret/index/term_infos_io.rb
+  - lib/ferret/index/term_enum.rb
+  - lib/ferret/index/term.rb
+  - lib/ferret/index/segment_merge_info.rb
+  - lib/ferret/index/segment_merger.rb
+  - lib/ferret/index/segment_infos.rb
+  - lib/ferret/index/term_doc_enum.rb
+  - lib/ferret/index/term_buffer.rb
+  - lib/ferret/index/multi_reader.rb
+  - lib/ferret/index/term_vectors_io.rb
+  - lib/ferret/index/index_writer.rb
+  - lib/ferret/index/term_vector_offset_info.rb
+  - lib/ferret/index/segment_term_vector.rb
+  - lib/ferret/index/document_writer.rb
+  - lib/ferret/index/compound_file_io.rb
+  - lib/ferret/index/multiple_term_doc_pos_enum.rb
+  - lib/ferret/search/query.rb
+  - lib/ferret/search/non_matching_scorer.rb
+  - lib/ferret/search/top_field_docs.rb
+  - lib/ferret/search/weight.rb
+  - lib/ferret/search/field_doc.rb
+  - lib/ferret/search/hit_queue.rb
+  - lib/ferret/search/sort.rb
+  - lib/ferret/search/disjunction_sum_scorer.rb
+  - lib/ferret/search/boolean_scorer.rb
+  - lib/ferret/search/boolean_query.rb
+  - lib/ferret/search/field_sorted_hit_queue.rb
+  - lib/ferret/search/conjunction_scorer.rb
+  - lib/ferret/search/top_docs.rb
+  - lib/ferret/search/req_opt_sum_scorer.rb
+  - lib/ferret/search/boolean_clause.rb
+  - lib/ferret/search/score_doc_comparator.rb
+  - lib/ferret/search/req_excl_scorer.rb
+  - lib/ferret/search/explanation.rb
+  - lib/ferret/search/index_searcher.rb
+  - lib/ferret/search/term_scorer.rb
+  - lib/ferret/search/filter.rb
+  - lib/ferret/search/similarity.rb
+  - lib/ferret/search/fuzzy_term_enum.rb
+  - lib/ferret/search/sort_field.rb
+  - lib/ferret/search/hit_collector.rb
+  - lib/ferret/search/term_query.rb
+  - lib/ferret/search/scorer.rb
+  - lib/ferret/search/score_doc.rb
+  - lib/ferret/search/prefix_query.rb
+  - lib/ferret/search/fuzzy_query.rb
+  - lib/ferret/search/spans.rb
+  - lib/ferret/search/caching_wrapper_filter.rb
+  - lib/ferret/search/filtered_term_enum.rb
+  - lib/ferret/search/multi_term_query.rb
+  - lib/ferret/search/query_filter.rb
+  - lib/ferret/search/wildcard_query.rb
+  - lib/ferret/search/wildcard_term_enum.rb
+  - lib/ferret/search/match_all_docs_query.rb
+  - lib/ferret/search/phrase_query.rb
+  - lib/ferret/search/exact_phrase_scorer.rb
+  - lib/ferret/search/sloppy_phrase_scorer.rb
+  - lib/ferret/search/phrase_positions.rb
+  - lib/ferret/search/phrase_scorer.rb
+  - lib/ferret/search/range_query.rb
+  - lib/ferret/search/multi_phrase_query.rb
+  - lib/ferret/search/filtered_query.rb
+  - lib/ferret/search/sort_comparator.rb
+  - lib/ferret/search/range_filter.rb
+  - lib/ferret/search/field_cache.rb
+  - lib/ferret/search/spans/near_spans_enum.rb
+  - lib/ferret/search/spans/span_first_query.rb
+  - lib/ferret/search/spans/spans_enum.rb
+  - lib/ferret/search/spans/span_not_query.rb
+  - lib/ferret/search/spans/span_near_query.rb
+  - lib/ferret/search/spans/span_or_query.rb
+  - lib/ferret/search/spans/span_term_query.rb
+  - lib/ferret/search/spans/span_query.rb
+  - lib/ferret/search/spans/span_scorer.rb
+  - lib/ferret/search/spans/span_weight.rb
+  - lib/ferret/query_parser/query_parser.tab.rb
+  - lib/ferret/stemmers/porter_stemmer.rb
+  - lib/ferret/store/index_io.rb
+  - lib/ferret/store/directory.rb
+  - lib/ferret/store/buffered_index_io.rb
+  - lib/ferret/store/fs_store.rb
+  - lib/ferret/store/ram_store.rb
+  - lib/ferret/utils/priority_queue.rb
+  - lib/ferret/utils/bit_vector.rb
+  - lib/ferret/utils/parameter.rb
+  - lib/ferret/utils/weak_key_hash.rb
+  - lib/ferret/utils/string_helper.rb
+  - lib/ferret/utils/number_tools.rb
+  - lib/ferret/utils/date_tools.rb
+  - test/test_helper.rb
+  - test/test_all.rb
+  - test/unit/ts_document.rb
+  - test/unit/ts_index.rb
+  - test/unit/ts_store.rb
+  - test/unit/ts_analysis.rb
+  - test/unit/ts_utils.rb
+  - test/unit/ts_search.rb
+  - test/unit/ts_query_parser.rb
+  - test/unit/utils/tc_bit_vector.rb
+  - test/unit/utils/tc_string_helper.rb
+  - test/unit/utils/tc_priority_queue.rb
+  - test/unit/utils/tc_number_tools.rb
+  - test/unit/utils/tc_date_tools.rb
+  - test/unit/utils/tc_parameter.rb
+  - test/unit/utils/tc_weak_key_hash.rb
+  - test/unit/analysis/tc_lower_case_tokenizer.rb
+  - test/unit/analysis/tc_lower_case_filter.rb
+  - test/unit/analysis/tc_porter_stem_filter.rb
+  - test/unit/analysis/tc_letter_tokenizer.rb
+  - test/unit/analysis/tc_standard_tokenizer.rb
+  - test/unit/analysis/tc_stop_analyzer.rb
+  - test/unit/analysis/tc_stop_filter.rb
+  - test/unit/analysis/tc_word_list_loader.rb
+  - test/unit/analysis/tc_per_field_analyzer_wrapper.rb
+  - test/unit/analysis/tc_white_space_analyzer.rb
+  - test/unit/analysis/tc_analyzer.rb
+  - test/unit/analysis/tc_standard_analyzer.rb
+  - test/unit/analysis/tc_white_space_tokenizer.rb
+  - test/unit/index/tc_segment_term_docs.rb
+  - test/unit/index/tc_term_infos_io.rb
+  - test/unit/index/tc_fields_io.rb
+  - test/unit/index/tc_term_buffer.rb
+  - test/unit/index/tc_segment_infos.rb
+  - test/unit/index/tc_segment_term_vector.rb
+  - test/unit/index/tc_term_vectors_io.rb
+  - test/unit/index/tc_index_writer.rb
+  - test/unit/index/tc_term.rb
+  - test/unit/index/th_doc.rb
+  - test/unit/index/tc_compound_file_io.rb
+  - test/unit/index/tc_term_vector_offset_info.rb
+  - test/unit/index/tc_segment_term_enum.rb
+  - test/unit/index/tc_field_infos.rb
+  - test/unit/index/tc_term_info.rb
+  - test/unit/index/tc_index.rb
+  - test/unit/index/tc_index_reader.rb
+  - test/unit/index/tc_multiple_term_doc_pos_enum.rb
+  - test/unit/store/tc_ram_store.rb
+  - test/unit/store/tm_store_lock.rb
+  - test/unit/store/tc_fs_store.rb
+  - test/unit/store/tm_store.rb
+  - test/unit/document/tc_document.rb
+  - test/unit/document/tc_field.rb
+  - test/unit/query_parser/tc_query_parser.rb
+  - test/unit/search/tc_sort.rb
+  - test/unit/search/tc_similarity.rb
+  - test/unit/search/tc_index_searcher.rb
+  - test/unit/search/tc_fuzzy_query.rb
+  - test/unit/search/tc_search_and_sort.rb
+  - test/unit/search/tc_sort_field.rb
+  - test/unit/search/tc_filter.rb
+  - test/unit/search/tc_spans.rb
+  - test/longrunning/tc_numbertools.rb
+  - test/longrunning/tm_store.rb
+  - test/benchmark/tb_rw_vint.rb
+  - test/benchmark/tb_ram_store.rb
+  - rake_utils/code_statistics.rb
+test_files: []
+rdoc_options: 
+  - "--title"
+  - "Ferret -- Ruby Indexer"
+  - "--main"
+  - README
+  - "--line-numbers"
+  - TUTORIAL
+  - TODO
+extra_rdoc_files: 
+  - README
+  - TODO
+  - TUTORIAL
+  - MIT-LICENSE
+executables: []
+extensions: 
+  - ext/extconf.rb
+requirements: []
+dependencies: []