ferret 0.11.6 → 0.11.8.4

data/test/unit/index/tc_index.rb

@@ -353,7 +353,7 @@ class IndexTest < Test::Unit::TestCase
     assert_equal(0, top_docs.hits.size)
 
     iw = IndexWriter.new(:path => fs_path, :analyzer => WhiteSpaceAnalyzer.new)
-    iw << {:f, "content3"}
+    iw << {:f => "content3"}
     iw.close()
 
     top_docs = index.search("content3")
@@ -462,6 +462,95 @@ class IndexTest < Test::Unit::TestCase
     index.close
   end
 
+  def test_index_key_batch0
+    data = {
+      "0" => {:id => "0", :val => "one"},
+      "0" => {:id => "0", :val => "two"},
+      "1" =>{:id => "1", :val => "three"},
+      "1" => {:id => "1", :val => "four"},
+    }
+
+    index = Index.new(:analyzer => WhiteSpaceAnalyzer.new,
+                      :key => :id)
+    index.batch_update data
+    assert_equal(2, index.size)
+    index.close
+  end
+
+  def test_index_key_batch1
+    data0 = {
+      "0" => {:id => "0", :val => "one"},
+      "0" => {:id => "0", :val => "two"},
+      "1" =>{:id => "1", :val => "three"},
+      "2" => {:id => "1", :val => "four"},
+    }
+
+    data1 = {
+      "0" => {:id => "0", :val => "one"},
+      "3" => {:id => "3", :val => "two"},
+      "2" =>{:id => "2", :val => "three"},
+      "1" => {:id => "1", :val => "four"},
+      "4" => {:id => "4", :val => "four"},
+    }
+
+    index = Index.new(:analyzer => WhiteSpaceAnalyzer.new,
+                      :key => :id)
+    index.batch_update data0
+    assert_equal(3, index.size)
+    index.batch_update data1
+    assert_equal(5, index.size)
+    index.close
+  end
+
+  def test_index_key_delete_batch0
+    data0 = {
+      "0" => {:id => "0", :val => "one"},
+      "0" => {:id => "0", :val => "two"},
+      "1" =>{:id => "1", :val => "three"},
+      "2" => {:id => "2", :val => "four"},
+      "0" => {:id => "0", :val => "four"},
+    }
+
+    data1 = ["0", "1"];
+
+    index = Index.new(:analyzer => WhiteSpaceAnalyzer.new, :key => :id)
+    index.batch_update data0
+
+    assert_equal("four", index["0"][:val])
+    assert_equal("three", index["1"][:val])
+    assert_equal("four", index["2"][:val])
+
+    assert_equal(3, index.size)
+    index.delete data1
+    assert_equal(1, index.size)
+    assert_equal("four", index["2"][:val])
+
+    index.close
+  end
+
+  def test_index_key_delete_batch0
+    index = Index.new(:analyzer => WhiteSpaceAnalyzer.new)
+    1000.times {|i| index << {:id => "#{i}", :content => "content #{i}"}}
+    assert_equal(1000, index.size)
+    assert_equal("content 876", index['876'][:content])
+
+    new_docs = Array.new(1000) {|i| {:id => i, :content => "#{i} > content"}}
+    index.batch_update(new_docs)
+    assert_equal(1000, index.size)
+    assert_equal("128 > content", index['128'][:content])
+
+    new_docs = Array.new(1000) {|i| {:id => i.to_s, :content => "_(#{i})_"}}
+    index.batch_update(new_docs)
+    assert_equal(1000, index.size)
+    assert_equal("_(287)_", index['287'][:content])
+
+    new_docs = {}
+    1000.times {|i| new_docs[i.to_s] = {:id => i, :content => "Hash(#{i})"}}
+    index.batch_update(new_docs)
+    assert_equal(1000, index.size)
+    assert_equal("Hash(78)", index['78'][:content])
+  end
+
   def test_index_multi_key
     index = Index.new(:analyzer => WhiteSpaceAnalyzer.new,
                       :key => [:id, :table])
@@ -555,19 +644,23 @@ class IndexTest < Test::Unit::TestCase
 
     index.close
   end
-
+  
+  # this test has been corrected to work as intended
+  # it now fails the same way on both 1.8 and 1.9 -- sds
   def test_auto_flush
     fs_path = File.expand_path(File.join(File.dirname(__FILE__), '../../temp/fsdir'))
     Dir[File.join(fs_path, "*")].each {|path| begin File.delete(path) rescue nil end}
 
-    data = %q(one two three four five six seven eight nine ten eleven twelve)
+    data = %w(one two three four five six seven eight nine ten eleven twelve)
     index1 = Index.new(:path => fs_path, :auto_flush => true, :key => :id)
-    index1 << "zero"
+    index1 << {:id => 0, :content => "zero"}
     index2 = Index.new(:path => fs_path, :auto_flush => true)
     begin
+      n = 1
       data.each do |datum|
-        index1 << {:id => datum[0], :content => datum}
-        index2 << {:id => datum[0], :content => datum}
+        index1 << {:id => n, :content => datum}
+        index2 << {:id => n, :content => datum}
+        n += 1
       end
       5.times do |i|
         index1.delete(i)
@@ -593,9 +686,9 @@ class IndexTest < Test::Unit::TestCase
 
     # Note: Adding keywords to either field1 or field2 gets rid of the error
 
-    index << {:field1, ''}
-    index << {:field2, ''}
-    index << {:field3, 'foo bar baz'}
+    index << {:field1 => ''}
+    index << {:field2 => ''}
+    index << {:field3 => 'foo bar baz'}
 
     index.flush
     index.close
@@ -644,7 +737,7 @@ class IndexTest < Test::Unit::TestCase
   end 
 
   def test_wildcard
-    i = nil
+    j = nil
     Ferret::I.new do |i|
       i << "one"
       assert_equal(1, i.search("*").total_hits)
@@ -654,8 +747,9 @@ class IndexTest < Test::Unit::TestCase
       assert_equal(3, i.search("*").total_hits)
       assert_equal(3, i.search("id:*").total_hits)
       assert_equal(2, i.search('id:?*').total_hits)
+      j = i
     end
-    assert_raise(StandardError) {i.close} 
+    assert_raise(StandardError) {j.close} 
   end
 
   def check_highlight(index, q, excerpt_length, num_excerpts, expected, field = :field)
@@ -759,4 +853,15 @@ class IndexTest < Test::Unit::TestCase
     index.query_delete('id:one')
     assert_equal(20, index.size)
   end
+
+  def test_query_update_delete_more_than_ten
+    index = Ferret::I.new
+    20.times {|i| index << {:id => i, :find => 'match', :change => 'one'} }
+
+    assert_equal(20, index.search('find:match').total_hits)
+    index.query_update('find:match', {:change => 'two'})
+    assert_equal(20, index.search('find:match AND change:two').total_hits)
+    index.query_delete('find:match')
+    assert_equal(0, index.size)
+  end
 end

data/test/unit/index/tc_index_reader.rb

@@ -191,10 +191,10 @@ module IndexReaderCommon
   def do_test_term_vectors()
     expected_tv = TermVector.new(:body,
       [
-        TVTerm.new("word1", [2, 4, 7]),
-        TVTerm.new("word2", [3]),
-        TVTerm.new("word3", [0, 5, 8, 9]),
-        TVTerm.new("word4", [1, 6])
+        TVTerm.new("word1", 3, [2, 4, 7]),
+        TVTerm.new("word2", 1, [3]),
+        TVTerm.new("word3", 4, [0, 5, 8, 9]),
+        TVTerm.new("word4", 2, [1, 6])
       ],
       [*(0...10)].collect {|i| TVOffsets.new(i*6, (i+1)*6 - 1)})
 
@@ -209,13 +209,13 @@ module IndexReaderCommon
     
     tv = tvs[:author]
     assert_equal(:author, tv.field)
-    assert_equal([TVTerm.new("Leo", [0]), TVTerm.new("Tolstoy", [1])], tv.terms)
+    assert_equal([TVTerm.new("Leo", 1, [0]), TVTerm.new("Tolstoy", 1, [1])], tv.terms)
     assert(tv.offsets.nil?)
 
 
     tv = tvs[:title]
     assert_equal(:title, tv.field)
-    assert_equal([TVTerm.new("War And Peace", nil)], tv.terms)
+    assert_equal([TVTerm.new("War And Peace", 1, nil)], tv.terms)
     assert_equal([TVOffsets.new(0, 13)], tv.offsets)
   end
    
@@ -254,19 +254,19 @@ module IndexReaderCommon
 
     norms = @ir.norms(:text)
 
-    assert_equal(202, norms[ 3])
-    assert_equal( 20, norms[25])
-    assert_equal(200, norms[50])
-    assert_equal(155, norms[63])
+    assert_equal(202, norms.bytes.to_a[ 3])
+    assert_equal( 20, norms.bytes.to_a[25])
+    assert_equal(200, norms.bytes.to_a[50])
+    assert_equal(155, norms.bytes.to_a[63])
 
     norms = @ir.norms(:title)
-    assert_equal(1, norms[3])
+    assert_equal(1, norms.bytes.to_a[3])
 
     norms = @ir.norms(:body)
-    assert_equal(12, norms[3])
+    assert_equal(12, norms.bytes.to_a[3])
 
     norms = @ir.norms(:author)
-    assert_equal(145, norms[3])
+    assert_equal(145, norms.bytes.to_a[3])
 
     norms = @ir.norms(:year)
     # TODO: this returns two possible results depending on whether it is 
@@ -277,10 +277,10 @@ module IndexReaderCommon
 
     norms = " " * 164
     @ir.get_norms_into(:text, norms, 100)
-    assert_equal(202, norms[103])
-    assert_equal( 20, norms[125])
-    assert_equal(200, norms[150])
-    assert_equal(155, norms[163])
+    assert_equal(202, norms.bytes.to_a[103])
+    assert_equal( 20, norms.bytes.to_a[125])
+    assert_equal(200, norms.bytes.to_a[150])
+    assert_equal(155, norms.bytes.to_a[163])
 
     @ir.commit()
 
@@ -290,10 +290,10 @@ module IndexReaderCommon
 
     norms = " " * 164
     ir2.get_norms_into(:text, norms, 100)
-    assert_equal(202, norms[103])
-    assert_equal( 20, norms[125])
-    assert_equal(200, norms[150])
-    assert_equal(155, norms[163])
+    assert_equal(202, norms.bytes.to_a[103])
+    assert_equal( 20, norms.bytes.to_a[125])
+    assert_equal(200, norms.bytes.to_a[150])
+    assert_equal(155, norms.bytes.to_a[163])
     ir2.close()
   end
 
@@ -608,10 +608,10 @@ class IndexReaderTest < Test::Unit::TestCase
   def do_test_term_vectors(ir)
     expected_tv = TermVector.new(:body,
       [
-        TVTerm.new("word1", [2, 4, 7]),
-        TVTerm.new("word2", [3]),
-        TVTerm.new("word3", [0, 5, 8, 9]),
-        TVTerm.new("word4", [1, 6])
+        TVTerm.new("word1", 3, [2, 4, 7]),
+        TVTerm.new("word2", 1, [3]),
+        TVTerm.new("word3", 4, [0, 5, 8, 9]),
+        TVTerm.new("word4", 2, [1, 6])
       ],
       [*(0...10)].collect {|i| TVOffsets.new(i*6, (i+1)*6 - 1)})
 
@@ -626,13 +626,13 @@ class IndexReaderTest < Test::Unit::TestCase
     
     tv = tvs[:author]
     assert_equal(:author, tv.field)
-    assert_equal([TVTerm.new("Leo", [0]), TVTerm.new("Tolstoy", [1])], tv.terms)
+    assert_equal([TVTerm.new("Leo", 1, [0]), TVTerm.new("Tolstoy", 1, [1])], tv.terms)
     assert(tv.offsets.nil?)
 
 
     tv = tvs[:title]
     assert_equal(:title, tv.field)
-    assert_equal([TVTerm.new("War And Peace", nil)], tv.terms)
+    assert_equal([TVTerm.new("War And Peace", 1, nil)], tv.terms)
     assert_equal([TVOffsets.new(0, 13)], tv.offsets)
   end
 

data/test/unit/index/tc_index_writer.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 require File.dirname(__FILE__) + "/../../test_helper"
 
 
@@ -57,6 +59,14 @@ class IndexWriterTest < Test::Unit::TestCase
     iw.close()
   end
 
+  def test_adding_long_url
+      iw = IndexWriter.new(:dir => @dir,
+                           :default_field => 'content')
+      iw << {:content => "http://" + 'x' * 255}
+      # The following line will cause a segfault prior to 0.11.6
+      iw << {:content => "http://" + 'x' * 1_000_000}
+  end
+
   private
 
   WORDS = [

data/test/unit/index/th_doc.rb

@@ -281,32 +281,49 @@ module IndexTestHelper
   def self.prepare_search_docs
     i = 1
     [
-      ["20050930", "cat1/",             "word1"                               ],
-      ["20051001", "cat1/sub1",         "word1 word2 the quick brown fox"     ],
-      ["20051002", "cat1/sub1/subsub1", "word1 word3"                         ],
-      ["20051003", "cat1/sub2",         "word1 word3"                         ],
-      ["20051004", "cat1/sub2/subsub2", "word1 word2"                         ],
-      ["20051005", "cat2/sub1",         "word1"                               ],
-      ["20051006", "cat2/sub1",         "word1 word3"                         ],
-      ["20051007", "cat2/sub1",         "word1"                               ],
-      ["20051008", "cat2/sub1",         "word1 word2 word3 the fast brown fox"],
-      ["20051009", "cat3/sub1",         "word1"                               ],
-      ["20051010", "cat3/sub1",         "word1"                               ],
-      ["20051011", "cat3/sub1",         "word1 word3 the quick red fox"       ],
-      ["20051012", "cat3/sub1",         "word1"                               ],
-      ["20051013", "cat1/sub2",         "word1"                               ],
-      ["20051014", "cat1/sub1",         "word1 word3 the quick hairy fox"     ],
-      ["20051015", "cat1/sub2/subsub1", "word1"                               ],
-      ["20051016", "cat1/sub1/subsub2",
-        "word1 the quick fox is brown and hairy and a little red"             ],
-      ["20051017", "cat1/",
-        "word1 the brown fox is quick and red"                                ]
-    ].map do |date, category, field| 
+      ["20050930", "cat1/",             0.123,
+        "word1"                                                             ],
+      ["20051001", "cat1/sub1",         0.954,
+        "word1 word2 the quick brown fox"                                   ],
+      ["20051002", "cat1/sub1/subsub1", 908.125,
+        "word1 word3"                                                       ],
+      ["20051003", "cat1/sub2",         3999,
+        "word1 word3"                                                       ],
+      ["20051004", "cat1/sub2/subsub2", "+.3412",
+        "word1 word2"                                                       ],
+      ["20051005", "cat2/sub1",         -1.298,
+        "word1"                                                             ],
+      ["20051006", "cat2/sub1",         "2",
+        "word1 word3"                                                       ],
+      ["20051007", "cat2/sub1",         "+8.894",
+        "word1"                                                             ],
+      ["20051008", "cat2/sub1",         "+21235.2135",
+        "word1 word2 word3 the fast brown fox"                              ],
+      ["20051009", "cat3/sub1",         "10.0",
+        "word1"                                                             ],
+      ["20051010", "cat3/sub1",         1,
+        "word1"                                                             ],
+      ["20051011", "cat3/sub1",         -12518419,
+        "word1 word3 the quick red fox"                                     ],
+      ["20051012", "cat3/sub1",         "10",
+        "word1"                                                             ],
+      ["20051013", "cat1/sub2",         "15682954",
+        "word1"                                                             ],
+      ["20051014", "cat1/sub1",         "91239",
+        "word1 word3 the quick hairy fox"                                   ],
+      ["20051015", "cat1/sub2/subsub1", "-.89321",
+        "word1"                                                             ],
+      ["20051016", "cat1/sub1/subsub2", -89,
+        "word1 the quick fox is brown and hairy and a little red"           ],
+      ["20051017", "cat1/",             "-1.0",
+        "word1 the brown fox is quick and red"                              ]
+    ].map do |date, category, number, field| 
       doc = Ferret::Document.new(i)
       i += 1
       doc[:date] = date
       doc[:category] = category
       doc[:field] = field
+      doc[:number] = number
       doc
     end
   end

data/test/unit/search/tc_filter.rb

@@ -1,4 +1,5 @@
 require File.dirname(__FILE__) + "/../../test_helper"
+require 'date'
 
 
 class FilterTest < Test::Unit::TestCase
@@ -39,16 +40,6 @@ class FilterTest < Test::Unit::TestCase
     end
   end
 
-  def test_filter_proc
-    searcher = Searcher.new(@dir)
-    q = MatchAllQuery.new()
-    filter_proc = lambda {|doc, score, s| (s[doc][:int] % 2) == 0}
-    top_docs = searcher.search(q, :filter_proc => filter_proc)
-    top_docs.hits.each do |hit|
-      assert_equal(0, searcher[hit.doc][:int] % 2)
-    end
-  end
-
   def test_range_filter
     searcher = Searcher.new(@dir)
     q = MatchAllQuery.new()
@@ -132,4 +123,34 @@ class FilterTest < Test::Unit::TestCase
     filt = CustomFilter.new
     do_test_top_docs(searcher, q, [0, 2, 4], filt)
   end
+
+  def test_filter_proc
+    searcher = Searcher.new(@dir)
+    q = MatchAllQuery.new()
+    filter_proc = lambda {|doc, score, s| (s[doc][:int] % 2) == 0}
+    top_docs = searcher.search(q, :filter_proc => filter_proc)
+    top_docs.hits.each do |hit|
+      assert_equal(0, searcher[hit.doc][:int] % 2)
+    end
+  end
+
+  def test_score_modifying_filter_proc
+    searcher = Searcher.new(@dir)
+    q = MatchAllQuery.new()
+    start_date = Date.parse('2008-02-08')
+    date_half_life_50 = lambda do |doc, score, s|
+      days = (start_date - Date.parse(s[doc][:date], '%Y%m%d')).to_i
+      1.0 / (2.0 ** (days.to_f / 50.0))
+    end
+    top_docs = searcher.search(q, :filter_proc => date_half_life_50)
+    docs = top_docs.hits.collect {|hit| hit.doc}
+    assert_equal(docs, [2,4,9,8,6,3,5,1,7,0])
+    rev_date_half_life_50 = lambda do |doc, score, s|
+      days = (start_date - Date.parse(s[doc][:date], '%Y%m%d')).to_i
+      1.0 - 1.0 / (2.0 ** (days.to_f / 50.0))
+    end
+    top_docs = searcher.search(q, :filter_proc => rev_date_half_life_50)
+    docs = top_docs.hits.collect {|hit| hit.doc}
+    assert_equal(docs, [0,7,1,3,5,6,8,9,2,4])
+  end
 end

data/test/unit/search/tc_index_searcher.rb

@@ -50,6 +50,12 @@ class SearcherTest < Test::Unit::TestCase
       assert(score_doc.score.approx_eql?(@searcher.explain(query, score_doc.doc).score), 
         "Scores(#{score_doc.score} != #{@searcher.explain(query, score_doc.doc).score})")
     end
+
+    assert_equal(expected.sort, @searcher.scan(query))
+    if expected.size > 5
+        assert_equal(expected[0...5], @searcher.scan(query, :limit => 5))
+        assert_equal(expected[5..-1], @searcher.scan(query, :start_doc => expected[5]))
+    end
   end
 
   def test_get_doc()