ferret 0.9.0 → 0.9.1

data/test/unit/document/rtc_field.rb

@@ -0,0 +1,28 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+
+class FieldTest < Test::Unit::TestCase
+  include Ferret::Document
+  include Ferret::Utils
+
+  def test_store()
+    assert_equal("COMPRESS", Field::Store::COMPRESS.to_s)
+    assert_equal("YES", Field::Store::YES.to_s)
+    assert_equal("NO", Field::Store::NO.to_s)
+  end
+
+  def test_index()
+    assert_equal("TOKENIZED", Field::Index::TOKENIZED.to_s)
+    assert_equal("UNTOKENIZED", Field::Index::UNTOKENIZED.to_s)
+    assert_equal("NO", Field::Index::NO.to_s)
+    assert_equal("NO_NORMS", Field::Index::NO_NORMS.to_s)
+  end
+
+  def test_term_vector()
+    assert_equal("YES", Field::TermVector::YES.to_s)
+    assert_equal("NO", Field::TermVector::NO.to_s)
+    assert_equal("WITH_POSITIONS", Field::TermVector::WITH_POSITIONS.to_s)
+    assert_equal("WITH_OFFSETS", Field::TermVector::WITH_OFFSETS.to_s)
+    assert_equal("WITH_POSITIONS_OFFSETS", Field::TermVector::WITH_POSITIONS_OFFSETS.to_s)
+  end
+end

data/test/unit/document/tc_field.rb

@@ -6,23 +6,93 @@ class FieldTest < Test::Unit::TestCase
   include Ferret::Utils
 
   def test_store()
-    assert_equal("COMPRESS", Field::Store::COMPRESS.to_s)
-    assert_equal("YES", Field::Store::YES.to_s)
-    assert_equal("NO", Field::Store::NO.to_s)
+    assert_not_nil(Field::Store::COMPRESS)
+    assert_not_nil(Field::Store::YES)
+    assert_not_nil(Field::Store::NO)
   end
 
   def test_index()
-    assert_equal("TOKENIZED", Field::Index::TOKENIZED.to_s)
-    assert_equal("UNTOKENIZED", Field::Index::UNTOKENIZED.to_s)
-    assert_equal("NO", Field::Index::NO.to_s)
-    assert_equal("NO_NORMS", Field::Index::NO_NORMS.to_s)
+    assert_not_nil(Field::Index::TOKENIZED)
+    assert_not_nil(Field::Index::UNTOKENIZED)
+    assert_not_nil(Field::Index::NO)
+    assert_not_nil(Field::Index::NO_NORMS)
   end
 
   def test_term_vector()
-    assert_equal("YES", Field::TermVector::YES.to_s)
-    assert_equal("NO", Field::TermVector::NO.to_s)
-    assert_equal("WITH_POSITIONS", Field::TermVector::WITH_POSITIONS.to_s)
-    assert_equal("WITH_OFFSETS", Field::TermVector::WITH_OFFSETS.to_s)
-    assert_equal("WITH_POSITIONS_OFFSETS", Field::TermVector::WITH_POSITIONS_OFFSETS.to_s)
+    assert_not_nil(Field::TermVector::YES)
+    assert_not_nil(Field::TermVector::NO)
+    assert_not_nil(Field::TermVector::WITH_POSITIONS)
+    assert_not_nil(Field::TermVector::WITH_OFFSETS)
+    assert_not_nil(Field::TermVector::WITH_POSITIONS_OFFSETS)
+  end
+
+  def test_standard_field()
+    f = Field.new("name", "value", Field::Store::COMPRESS, Field::Index::TOKENIZED)
+    assert_equal("name", f.name)
+    assert_equal("value", f.data)
+    assert_equal(true, f.stored?)
+    assert_equal(true, f.compressed?)
+    assert_equal(true, f.indexed?)
+    assert_equal(true, f.tokenized?)
+    assert_equal(false, f.store_term_vector?)
+    assert_equal(false, f.store_offsets?)
+    assert_equal(false, f.store_positions?)
+    assert_equal(false, f.omit_norms?)
+    assert_equal(false, f.binary?)
+    assert_equal("stored/compressed,indexed,tokenized,<name:value>", f.to_s)
+    f.data = "183"
+    f.boost = 0.001
+    assert_equal("183", f.data)
+    assert(0.001 =~ f.boost)
+  end
+
+  def test_set_store()
+    f = Field.new("name", "", Field::Store::COMPRESS, Field::Index::TOKENIZED)
+    f.store = Field::Store::NO
+    assert_equal(false, f.stored?)
+    assert_equal(false, f.compressed?)
+    assert_equal("indexed,tokenized,<name:>", f.to_s)
+  end
+
+  def test_set_index()
+    f = Field.new("name", "value", Field::Store::COMPRESS, Field::Index::TOKENIZED)
+    f.index = Field::Index::NO
+    assert_equal(false, f.indexed?)
+    assert_equal(false, f.tokenized?)
+    assert_equal(false, f.omit_norms?)
+    assert_equal("stored/compressed,<name:value>", f.to_s)
+    f.index = Field::Index::NO_NORMS
+    assert_equal(true, f.indexed?)
+    assert_equal(false, f.tokenized?)
+    assert_equal(true, f.omit_norms?)
+    assert_equal("stored/compressed,indexed,omit_norms,<name:value>", f.to_s)
+  end
+
+  def test_set_term_vector()
+    f = Field.new("name", "value", Field::Store::COMPRESS, Field::Index::TOKENIZED)
+    f.term_vector = Field::TermVector::WITH_POSITIONS_OFFSETS
+    assert_equal(true, f.store_term_vector?)
+    assert_equal(true, f.store_offsets?)
+    assert_equal(true, f.store_positions?)
+    assert_equal("stored/compressed,indexed,tokenized,store_term_vector,store_offsets,store_positions,<name:value>", f.to_s)
+  end
+
+  def test_new_binary_field()
+    tmp = []
+    256.times {|i| tmp[i] = i}
+    bin = tmp.pack("c*")
+    f = Field.new_binary_field("name", bin, Field::Store::YES)
+    assert_equal("name", f.name)
+    assert_equal(bin, f.data)
+    assert_equal(true, f.stored?)
+    assert_equal(false, f.compressed?)
+    assert_equal(false, f.indexed?)
+    assert_equal(false, f.tokenized?)
+    assert_equal(false, f.store_term_vector?)
+    assert_equal(false, f.store_offsets?)
+    assert_equal(false, f.store_positions?)
+    assert_equal(false, f.omit_norms?)
+    assert_equal(true, f.binary?)
+    assert_equal("stored/uncompressed,binary,<name:=bin_data=>", f.to_s)
   end
 end

data/test/unit/index/{c_index.rb → tc_index.rb}

@@ -77,7 +77,7 @@ class IndexTest < Test::Unit::TestCase
 
   def do_test_index_with_doc_array(index)
     data = [
-      {"def_field" => "one two multi", :id => "me"},
+      {"def_field" => "one two multi", :id => "myid"},
       {"def_field" => "one", :field2 => "three multi"},
       {"def_field" => "two"},
       {"def_field" => "one", :field2 => "four"},
@@ -125,8 +125,8 @@ class IndexTest < Test::Unit::TestCase
     assert(! index.deleted?(7))
     t = Term.new("field2", "four")
     assert_equal("one", index[t]["def_field"])
-    assert_equal("one two multi", index["me"]["def_field"])
-    index.delete("me")
+    assert_equal("one two multi", index["myid"]["def_field"])
+    index.delete("myid")
     assert(index.deleted?(0))
   end
 
@@ -193,6 +193,26 @@ class IndexTest < Test::Unit::TestCase
     index.close
   end
 
+  def test_key_used_for_id_field
+    fs_path = File.expand_path(File.join(File.dirname(__FILE__), '../../temp/fsdir'))
+    Dir[File.join(fs_path, "*")].each {|path| begin File.delete(path) rescue nil end}
+    data = [
+      {:my_id => "one two", :id => "me"},
+      {:my_id => "one", :field2 => "three"},
+      {:my_id => "two"},
+      {:my_id => "one", :field2 => "four"},
+      {:my_id => "one two"},
+      {:my_id => "two", :field2 => "three", "field3" => "four"},
+      {:my_id => "one"},
+      {:my_id => "two", :field2 => "three", "field3" => "five"}
+    ]
+    index = Index.new(:path => fs_path, :key => :my_id)
+    data.each {|doc| index << doc }
+    assert_equal(3, index.size)
+    assert_equal("three", index[:two][:field2])
+    index.close
+  end
+
   def test_merging_indexes
     data = [
       {"f" => "zero"},
@@ -432,7 +452,7 @@ class IndexTest < Test::Unit::TestCase
       {:id => 1, :val => "four"},
     ]
     index = Index.new(:analyzer => WhiteSpaceAnalyzer.new,
-                      :key => "id")
+                      :key => :id)
     data.each { |doc| index << doc }
     assert_equal(2, index.size)
     assert_equal("two", index[0][:val])
@@ -453,7 +473,7 @@ class IndexTest < Test::Unit::TestCase
       {:id => 1, :table => "product", :product => "backpack"}
     ]
     index = Index.new(:analyzer => WhiteSpaceAnalyzer.new,
-                      :key => ["id", "table"])
+                      :key => [:id, :table])
     data.each { |doc| index << doc }
     assert_equal(4, index.size)
     assert_equal("super tent", index[0][:product])
@@ -476,7 +496,7 @@ class IndexTest < Test::Unit::TestCase
       {:id => 1, :table => "Product", :product => "backpack"}
     ]
     index = Index.new(:analyzer => Analyzer.new,
-                      :key => ["id", "table"])
+                      :key => [:id, :table])
     data.each do |dat|
       doc = Document.new
       dat.each_pair do |key, value|

data/test/unit/query_parser/{c_query_parser.rb → rtc_query_parser.rb}

@@ -3,7 +3,7 @@ require File.dirname(__FILE__) + "/../../test_helper"
 class QueryParserTest < Test::Unit::TestCase
 
   def test_strings()
-    parser = Ferret::QueryParser.new("xxx", :fields => ["xxx", "field", "f1", "f2"])
+    parser = Ferret::QueryParser.new("xxx", :fields => ["f1", "f2", "f3"])
     pairs = [
       ['', ''],
       ['word', 'word'],
@@ -17,7 +17,7 @@ class QueryParserTest < Test::Unit::TestCase
       ['field:"one <> <> <> three <>"', 'field:"one <> <> <> three"'],
       ['field:"one <> <> <> three|four|five <>"', 'field:"one <> <> <> three|four|five"'],
       ['field:"one|two three|four|five six|seven"', 'field:"one|two three|four|five six|seven"'],
-      ['field:"testing|trucks"', 'field:testing field:trucks'],
+      ['contents:"testing|trucks"', 'contents:testing contents:trucks'],
       ['[aaa bbb]', '[aaa bbb]'],
       ['{aaa bbb]', '{aaa bbb]'],
       ['field:[aaa bbb}', 'field:[aaa bbb}'],
@@ -54,7 +54,7 @@ class QueryParserTest < Test::Unit::TestCase
       ['one AND (two OR three)', '+one +(two three)'],
       ['field:(one AND (two OR three))', '+field:one +(field:two field:three)'],
       ['one AND (two OR [aaa vvv})', '+one +(two [aaa vvv})'],
-      ['one AND (f1:two OR f2:three) AND four', '+one +(f1:two f2:three) +four'],
+      ['one AND (one:two OR two:three) AND four', '+one +(one:two two:three) +four'],
       ['one^1.23', 'one^1.23'],
       ['(one AND two)^100.23', '(+one +two)^100.23'],
       ['field:(one AND two)^100.23', '(+field:one +field:two)^100.23'],
@@ -73,42 +73,42 @@ class QueryParserTest < Test::Unit::TestCase
       ['field:abc~ AND field:(b*)', '+field:abc~ +field:b*'],
       ['asdf?*?asd*dsf?asfd*asdf?^20.0', 'asdf?*?asd*dsf?asfd*asdf?^20.0'],
 
-      ['*:xxx', 'xxx field:xxx f1:xxx f2:xxx'],
+      ['*:xxx', 'f1:xxx f2:xxx f3:xxx'],
       ['f1|f2:xxx', 'f1:xxx f2:xxx'],
 
-      ['*:asd~0.2', 'asd~0.2 field:asd~0.2 f1:asd~0.2 f2:asd~0.2'],
+      ['*:asd~0.2', 'f1:asd~0.2 f2:asd~0.2 f3:asd~0.2'],
       ['f1|f2:asd~0.2', 'f1:asd~0.2 f2:asd~0.2'],
 
-      ['*:a?d*^20.0', '(a?d* field:a?d* f1:a?d* f2:a?d*)^20.0'],
+      ['*:a?d*^20.0', '(f1:a?d* f2:a?d* f3:a?d*)^20.0'],
       ['f1|f2:a?d*^20.0', '(f1:a?d* f2:a?d*)^20.0'],
 
-      ['*:"asdf <> xxx|yyy"', '"asdf <> xxx|yyy" field:"asdf <> xxx|yyy" f1:"asdf <> xxx|yyy" f2:"asdf <> xxx|yyy"'],
+      ['*:"asdf <> xxx|yyy"', 'f1:"asdf <> xxx|yyy" f2:"asdf <> xxx|yyy" f3:"asdf <> xxx|yyy"'],
       ['f1|f2:"asdf <> xxx|yyy"', 'f1:"asdf <> xxx|yyy" f2:"asdf <> xxx|yyy"'],
 
-      ['*:[bbb xxx]', '[bbb xxx] field:[bbb xxx] f1:[bbb xxx] f2:[bbb xxx]'],
+      ['*:[bbb xxx]', 'f1:[bbb xxx] f2:[bbb xxx] f3:[bbb xxx]'],
       ['f1|f2:[bbb xxx]', 'f1:[bbb xxx] f2:[bbb xxx]'],
 
-      ['*:(xxx AND bbb)', '+(xxx field:xxx f1:xxx f2:xxx) +(bbb field:bbb f1:bbb f2:bbb)'],
+      ['*:(xxx AND bbb)', '+(f1:xxx f2:xxx f3:xxx) +(f1:bbb f2:bbb f3:bbb)'],
       ['f1|f2:(xxx AND bbb)', '+(f1:xxx f2:xxx) +(f1:bbb f2:bbb)'],
       ['asdf?*?asd*dsf?asfd*asdf?^20.0', 'asdf?*?asd*dsf?asfd*asdf?^20.0'],
       ['"onewordphrase"', 'onewordphrase']
     ]
       
     pairs.each do |query_str, expected|
-      assert_equal(expected, parser.parse(query_str).to_s("xxx"))
+      assert_equal(expected, parser.parse(query_str).to_s(parser.default_field))
     end
   end
 
   def test_qp_with_standard_analyzer()
-    parser = Ferret::QueryParser.new("xxx", :fields => ["xxx", "key"],
+    parser = Ferret::QueryParser.new("xxx", :fields => ["f1", "f2", "f3"],
                                      :analyzer => Ferret::Analysis::StandardAnalyzer.new)
     pairs = [
-      ['key:1234', 'key:1234'],
+    ['key:1234', 'key:1234'],
       ['key:(1234)', 'key:1234']
     ]
       
     pairs.each do |query_str, expected|
-      assert_equal(expected, parser.parse(query_str).to_s("xxx"))
+      assert_equal(expected, parser.parse(query_str).to_s(parser.default_field))
     end
   end
   
@@ -132,7 +132,7 @@ class QueryParserTest < Test::Unit::TestCase
       
     pairs.each do |query_str, expected|
       do_test_query_parse_exception_raised(query_str)
-      assert_equal(expected, parser.parse(query_str).to_s("xxx"))
+      assert_equal(expected, parser.parse(query_str).to_s(parser.default_field))
     end
   end
 end

data/test/unit/query_parser/tc_query_parser.rb

@@ -3,7 +3,7 @@ require File.dirname(__FILE__) + "/../../test_helper"
 class QueryParserTest < Test::Unit::TestCase
 
   def test_strings()
-    parser = Ferret::QueryParser.new("xxx", :fields => ["f1", "f2", "f3"])
+    parser = Ferret::QueryParser.new("xxx", :fields => ["xxx", "field", "f1", "f2"])
     pairs = [
       ['', ''],
       ['word', 'word'],
@@ -17,7 +17,7 @@ class QueryParserTest < Test::Unit::TestCase
       ['field:"one <> <> <> three <>"', 'field:"one <> <> <> three"'],
       ['field:"one <> <> <> three|four|five <>"', 'field:"one <> <> <> three|four|five"'],
       ['field:"one|two three|four|five six|seven"', 'field:"one|two three|four|five six|seven"'],
-      ['contents:"testing|trucks"', 'contents:"testing|trucks"'],
+      ['field:"testing|trucks"', 'field:testing field:trucks'],
       ['[aaa bbb]', '[aaa bbb]'],
       ['{aaa bbb]', '{aaa bbb]'],
       ['field:[aaa bbb}', 'field:[aaa bbb}'],
@@ -54,7 +54,7 @@ class QueryParserTest < Test::Unit::TestCase
       ['one AND (two OR three)', '+one +(two three)'],
       ['field:(one AND (two OR three))', '+field:one +(field:two field:three)'],
       ['one AND (two OR [aaa vvv})', '+one +(two [aaa vvv})'],
-      ['one AND (one:two OR two:three) AND four', '+one +(one:two two:three) +four'],
+      ['one AND (f1:two OR f2:three) AND four', '+one +(f1:two f2:three) +four'],
       ['one^1.23', 'one^1.23'],
       ['(one AND two)^100.23', '(+one +two)^100.23'],
       ['field:(one AND two)^100.23', '(+field:one +field:two)^100.23'],
@@ -70,45 +70,45 @@ class QueryParserTest < Test::Unit::TestCase
       ['ab*de', 'ab*de'],
       ['asdf?*?asd*dsf?asfd*asdf?', 'asdf?*?asd*dsf?asfd*asdf?'],
       ['field:a* AND field:(b*)', '+field:a* +field:b*'],
-      ['field:abc~ AND field:(b*)', '+field:abc~0.5 +field:b*'],
+      ['field:abc~ AND field:(b*)', '+field:abc~ +field:b*'],
       ['asdf?*?asd*dsf?asfd*asdf?^20.0', 'asdf?*?asd*dsf?asfd*asdf?^20.0'],
 
-      ['*:xxx', 'f1:xxx f2:xxx f3:xxx'],
+      ['*:xxx', 'xxx field:xxx f1:xxx f2:xxx'],
       ['f1|f2:xxx', 'f1:xxx f2:xxx'],
 
-      ['*:asd~0.2', 'f1:asd~0.2 f2:asd~0.2 f3:asd~0.2'],
+      ['*:asd~0.2', 'asd~0.2 field:asd~0.2 f1:asd~0.2 f2:asd~0.2'],
       ['f1|f2:asd~0.2', 'f1:asd~0.2 f2:asd~0.2'],
 
-      ['*:a?d*^20.0', '(f1:a?d* f2:a?d* f3:a?d*)^20.0'],
+      ['*:a?d*^20.0', '(a?d* field:a?d* f1:a?d* f2:a?d*)^20.0'],
       ['f1|f2:a?d*^20.0', '(f1:a?d* f2:a?d*)^20.0'],
 
-      ['*:"asdf <> xxx|yyy"', 'f1:"asdf <> xxx|yyy" f2:"asdf <> xxx|yyy" f3:"asdf <> xxx|yyy"'],
+      ['*:"asdf <> xxx|yyy"', '"asdf <> xxx|yyy" field:"asdf <> xxx|yyy" f1:"asdf <> xxx|yyy" f2:"asdf <> xxx|yyy"'],
       ['f1|f2:"asdf <> xxx|yyy"', 'f1:"asdf <> xxx|yyy" f2:"asdf <> xxx|yyy"'],
 
-      ['*:[bbb xxx]', 'f1:[bbb xxx] f2:[bbb xxx] f3:[bbb xxx]'],
+      ['*:[bbb xxx]', '[bbb xxx] field:[bbb xxx] f1:[bbb xxx] f2:[bbb xxx]'],
       ['f1|f2:[bbb xxx]', 'f1:[bbb xxx] f2:[bbb xxx]'],
 
-      ['*:(xxx AND bbb)', '+(f1:xxx f2:xxx f3:xxx) +(f1:bbb f2:bbb f3:bbb)'],
+      ['*:(xxx AND bbb)', '+(xxx field:xxx f1:xxx f2:xxx) +(bbb field:bbb f1:bbb f2:bbb)'],
       ['f1|f2:(xxx AND bbb)', '+(f1:xxx f2:xxx) +(f1:bbb f2:bbb)'],
       ['asdf?*?asd*dsf?asfd*asdf?^20.0', 'asdf?*?asd*dsf?asfd*asdf?^20.0'],
       ['"onewordphrase"', 'onewordphrase']
     ]
       
     pairs.each do |query_str, expected|
-      assert_equal(expected, parser.parse(query_str).to_s(parser.default_field))
+      assert_equal(expected, parser.parse(query_str).to_s("xxx"))
     end
   end
 
   def test_qp_with_standard_analyzer()
-    parser = Ferret::QueryParser.new("xxx", :fields => ["f1", "f2", "f3"],
+    parser = Ferret::QueryParser.new("xxx", :fields => ["xxx", "key"],
                                      :analyzer => Ferret::Analysis::StandardAnalyzer.new)
     pairs = [
-    ['key:1234', 'key:1234'],
+      ['key:1234', 'key:1234'],
       ['key:(1234)', 'key:1234']
     ]
       
     pairs.each do |query_str, expected|
-      assert_equal(expected, parser.parse(query_str).to_s(parser.default_field))
+      assert_equal(expected, parser.parse(query_str).to_s("xxx"))
     end
   end
   
@@ -119,6 +119,14 @@ class QueryParserTest < Test::Unit::TestCase
     end
   end
 
+  def test_prefix_query
+    parser = Ferret::QueryParser.new("xxx", :fields => ["xxx"],
+                                     :analyzer => Ferret::Analysis::StandardAnalyzer.new)
+    assert_equal(Ferret::Search::PrefixQuery, parser.parse("asdg*").class)
+    assert_equal(Ferret::Search::WildcardQuery, parser.parse("a?dg*").class)
+    assert_equal(Ferret::Search::WildcardQuery, parser.parse("a*dg*").class)
+    assert_equal(Ferret::Search::WildcardQuery, parser.parse("adg*c").class)
+  end
   
   def test_bad_queries
     parser = Ferret::QueryParser.new("xxx", :fields => ["f1", "f2"],
@@ -127,12 +135,12 @@ class QueryParserTest < Test::Unit::TestCase
     pairs = [
       ['::*word', 'word'],
       ['()*&)(*^&*(', ''],
-      ['()*&one)(*two(*&"', 'one two']
+      ['()*&one)(*two(*&"', '"one two"']
     ]
       
     pairs.each do |query_str, expected|
       do_test_query_parse_exception_raised(query_str)
-      assert_equal(expected, parser.parse(query_str).to_s(parser.default_field))
+      assert_equal(expected, parser.parse(query_str).to_s("xxx"))
     end
   end
 end

data/test/unit/search/rtc_sort_field.rb

@@ -0,0 +1,14 @@
+require File.dirname(__FILE__) + "/../../test_helper"
+
+class SortFieldTest < Test::Unit::TestCase
+  include Ferret::Search
+
+  def test_params()
+    assert_equal("score",  SortField::SortType::SCORE.to_s)
+    assert_equal("doc",    SortField::SortType::DOC.to_s)
+    assert_equal("auto",   SortField::SortType::AUTO.to_s)
+    assert_equal("string", SortField::SortType::STRING.to_s)
+    assert_equal("int",    SortField::SortType::INTEGER.to_s)
+    assert_equal("float",  SortField::SortType::FLOAT.to_s)
+  end
+end