ferret 0.10.0 → 0.10.1

data/ext/r_index.c

@@ -1799,7 +1799,7 @@ frt_get_lazy_doc(LazyDoc *lazy_doc)
     rb_ivar_set(self, id_data, rdata);
 
     for (i = 0; i < lazy_doc->size; i++) {
-        RARRAY(rfields)->ptr[i] = rb_intern(lazy_doc->fields[i]->name);
+        RARRAY(rfields)->ptr[i] = ID2SYM(rb_intern(lazy_doc->fields[i]->name));
         RARRAY(rfields)->len++;
     }
     rb_ivar_set(self, id_fields, rfields);

data/ext/r_qparser.c

@@ -160,7 +160,7 @@ frt_qp_init(int argc, VALUE *argv, VALUE self)
             qp->wild_lower = RTEST(rval);
         }
         if (Qnil != (rval = rb_hash_aref(roptions, sym_or_default))) {
-            qp->or_default = (FIX2INT(rval) == BC_MUST) ? false : true;
+            qp->or_default = RTEST(rval);
         }
         if (Qnil != (rval = rb_hash_aref(roptions, sym_default_slop))) {
             qp->def_slop = FIX2INT(rval);

data/ext/search.c

@@ -1199,7 +1199,7 @@ static TopDocs *cdfsea_search_w(Searcher *self, Weight *w, int fd, int nd,
                                 Filter *f, Sort *s, filter_ft ff, bool load)
 {
     (void)self; (void)w; (void)fd; (void)nd;
-    (void)f; (void)s; (void)ff, (void)load;
+    (void)f; (void)s; (void)ff; (void)load;
     RAISE(UNSUPPORTED_ERROR, UNSUPPORTED_ERROR_MSG);
     return NULL;
 }
@@ -1208,7 +1208,7 @@ static TopDocs *cdfsea_search(Searcher *self, Query *q, int fd, int nd,
                               Filter *f, Sort *s, filter_ft ff, bool load)
 {
     (void)self; (void)q; (void)fd; (void)nd;
-    (void)f; (void)s; (void)ff, (void)load;
+    (void)f; (void)s; (void)ff; (void)load;
     RAISE(UNSUPPORTED_ERROR, UNSUPPORTED_ERROR_MSG);
     return NULL;
 }

data/lib/ferret/document.rb

@@ -24,8 +24,8 @@ module Ferret
   # A Document is a set of fields.  Each field has a name and an array of
   # textual values. If you are coming from a Lucene background you should note
   # that Fields don't have any properties except for the boost property. You
-  # should use the FieldInfos class to set field properties accross the whole
-  # index instead.
+  # should use the Ferret::Index::FieldInfos class to set field properties
+  # accross the whole index instead.
   # 
   # === Boost
   #
@@ -35,9 +35,9 @@ module Ferret
   # You may, for example, want to boost products that have a higher user
   # rating so that they are more likely to appear in search results.
   #
-  # Note: that fields which are _not_ stored (see FieldInfos) are _not_
-  # available in documents retrieved from the index, e.g. Searcher#doc or
-  # IndexReader#doc.
+  # Note: that fields which are _not_ stored (see Ferret::Index::FieldInfos)
+  # are _not_ available in documents retrieved from the index, e.g.
+  # Ferret::Search::Searcher#doc or Ferret::Index::IndexReader#doc.
   #
   # Note: that modifying a Document retrieved from the index will not modify
   # the document contained within the index. You need to delete the old

data/lib/ferret/index.rb

@@ -199,7 +199,8 @@ module Ferret::Index
           else
             id = doc[@key].to_s
             if id
-              delete(id)
+              ensure_writer_open()
+              @writer.delete(@key, id)
               @writer.commit
             end
           end
@@ -247,7 +248,7 @@ module Ferret::Index
         ensure_searcher_open()
         query = process_query(query)
 
-        @searcher.search_each(query) do |doc, score|
+        @searcher.search_each(query, options) do |doc, score|
           yield doc, score
         end
       end
@@ -575,3 +576,7 @@ module Ferret::Index
 
   end
 end
+
+module Ferret
+  I = Index::Index
+end

data/test/unit/analysis/tc_analyzer.rb

@@ -38,7 +38,7 @@ class AnalyzerTest < Test::Unit::TestCase
     assert_equal(Token.new("ADDRESS", 39, 46), t.next())
     assert(! t.next())
   end
-end
+end if Ferret.locale.downcase.index("utf")
 
 class AsciiLetterAnalyzerTest < Test::Unit::TestCase
   include Ferret::Analysis
@@ -131,7 +131,7 @@ class LetterAnalyzerTest < Test::Unit::TestCase
     assert_equal(Token.new("öîí", 80, 86), t.next)
     assert(! t.next())
   end
-end
+end if Ferret.locale.downcase.index("utf")
 
 class AsciiWhiteSpaceAnalyzerTest < Test::Unit::TestCase
   include Ferret::Analysis
@@ -214,7 +214,7 @@ class WhiteSpaceAnalyzerTest < Test::Unit::TestCase
     assert_equal(Token.new('áägç®êëì¯úøã¬öîí', 55, 86), t.next)
     assert(! t.next())
   end
-end
+end if Ferret.locale.downcase.index("utf")
 
 class AsciiStandardAnalyzerTest < Test::Unit::TestCase
   include Ferret::Analysis
@@ -338,7 +338,7 @@ class StandardAnalyzerTest < Test::Unit::TestCase
     assert_equal(Token.new('öîí', 142, 148), t2.next)
     assert(! t2.next())
   end
-end
+end if Ferret.locale.downcase.index("utf")
 
 class PerFieldAnalyzerTest < Test::Unit::TestCase
   include Ferret::Analysis
@@ -529,4 +529,4 @@ class CustomAnalyzerTest < Test::Unit::TestCase
     assert_equal(Token.new("dêbater", 36, 44), t.next)
     assert(! t.next())
   end
-end
+end if Ferret.locale.downcase.index("utf")

data/test/unit/analysis/tc_token_stream.rb

@@ -108,7 +108,7 @@ class LetterTokenizerTest < Test::Unit::TestCase
     assert_equal(Token.new('öîí', 80, 86), t.next)
     assert(! t.next())
   end
-end
+end if Ferret.locale.downcase.index("utf")
 
 class AsciiWhiteSpaceTokenizerTest < Test::Unit::TestCase
   include Ferret::Analysis
@@ -185,7 +185,7 @@ class WhiteSpaceTokenizerTest < Test::Unit::TestCase
     assert_equal(Token.new('áägç®êëì¯úøã¬öîí', 55, 86), t.next)
     assert(! t.next())
   end
-end
+end if Ferret.locale.downcase.index("utf")
 
 class AsciiStandardTokenizerTest < Test::Unit::TestCase
   include Ferret::Analysis
@@ -267,7 +267,7 @@ class StandardTokenizerTest < Test::Unit::TestCase
     assert_equal(Token.new('öîí', 142, 148), t.next)
     assert(! t.next())
   end
-end
+end if Ferret.locale.downcase.index("utf")
 
 class RegExpTokenizerTest < Test::Unit::TestCase
   include Ferret::Analysis
@@ -397,20 +397,22 @@ class StemFilterTest < Test::Unit::TestCase
     assert_equal(Token.new("DEBate", 23, 31), t.next)
     assert_equal(Token.new("Debat", 32, 39), t.next)
 
-    input = "Dêbate dêbates DÊBATED DÊBATing dêbater";
-    t = StemFilter.new(LowerCaseFilter.new(LetterTokenizer.new(input)), :english)
-    assert_equal(Token.new("dêbate", 0, 7), t.next)
-    assert_equal(Token.new("dêbate", 8, 16), t.next)
-    assert_equal(Token.new("dêbate", 17, 25), t.next)
-    assert_equal(Token.new("dêbate", 26, 35), t.next)
-    assert_equal(Token.new("dêbater", 36, 44), t.next)
-    t = StemFilter.new(LetterTokenizer.new(input), :english)
-    assert_equal(Token.new("Dêbate", 0, 7), t.next)
-    assert_equal(Token.new("dêbate", 8, 16), t.next)
-    assert_equal(Token.new("DÊBATED", 17, 25), t.next)
-    assert_equal(Token.new("DÊBATing", 26, 35), t.next)
-    assert_equal(Token.new("dêbater", 36, 44), t.next)
-    assert(! t.next())
+    if Ferret.locale.downcase.index("utf")
+      input = "Dêbate dêbates DÊBATED DÊBATing dêbater";
+      t = StemFilter.new(LowerCaseFilter.new(LetterTokenizer.new(input)), :english)
+      assert_equal(Token.new("dêbate", 0, 7), t.next)
+      assert_equal(Token.new("dêbate", 8, 16), t.next)
+      assert_equal(Token.new("dêbate", 17, 25), t.next)
+      assert_equal(Token.new("dêbate", 26, 35), t.next)
+      assert_equal(Token.new("dêbater", 36, 44), t.next)
+      t = StemFilter.new(LetterTokenizer.new(input), :english)
+      assert_equal(Token.new("Dêbate", 0, 7), t.next)
+      assert_equal(Token.new("dêbate", 8, 16), t.next)
+      assert_equal(Token.new("DÊBATED", 17, 25), t.next)
+      assert_equal(Token.new("DÊBATing", 26, 35), t.next)
+      assert_equal(Token.new("dêbater", 36, 44), t.next)
+      assert(! t.next())
+    end
   end
 end
 

data/test/unit/index/tc_index.rb

@@ -129,7 +129,7 @@ class IndexTest < Test::Unit::TestCase
   end
 
   def test_ram_index
-    index = Index.new(:default_input_field => :xxx)
+    index = Ferret::I.new(:default_input_field => :xxx)
     do_test_index_with_array(index)
     index.close
 
@@ -555,12 +555,12 @@ class IndexTest < Test::Unit::TestCase
     Dir[File.join(fs_path, "*")].each {|path| begin File.delete(path) rescue nil end}
 
     data = %q(one two three four five six seven eight nine ten eleven twelve)
-    index1 = Index.new(:path => fs_path, :auto_flush => true)
+    index1 = Index.new(:path => fs_path, :auto_flush => true, :key => :id)
     index2 = Index.new(:path => fs_path, :auto_flush => true)
     begin
-      data.each do |doc|
-        index1 << doc
-        index2 << doc
+      data.each do |datum|
+        index1 << {:id => datum[0], :content => datum}
+        index2 << {:id => datum[0], :content => datum}
       end
       5.times do |i|
         index1.delete(i)
@@ -593,4 +593,22 @@ class IndexTest < Test::Unit::TestCase
     index.flush
     index.close
   end
+
+  def test_stopwords
+    field_infos = FieldInfos.new(:store => :no, :term_vector => :no)
+    field_infos.add_field(:id, :store => :yes, :index => :untokenized)
+
+    i = Ferret::Index::Index.new(:or_default => false, :default_search_field => '*')
+ 
+    # adding this additional field to the document leads to failure below
+    # comment out this statement and all tests pass:
+    i << {:id => 1, :content => "Move or shake"}
+
+    hits = i.search 'move nothere shake'
+    assert_equal 0, hits.total_hits
+    hits = i.search 'move shake'
+    assert_equal 1, hits.total_hits
+    hits = i.search 'move or shake'
+    assert_equal 1, hits.total_hits # fails when id field is present
+  end
 end

data/test/unit/index/tc_index_reader.rb

@@ -177,6 +177,7 @@ module IndexReaderCommon
    
   def do_test_get_doc()
     doc = @ir.get_document(3)
+    assert_equal([:year, :body, :title, :author], doc.fields)
     assert_equal(4, doc.fields.size)
     assert_equal(0, doc.size)
     assert_equal([], doc.keys)

data/test/unit/query_parser/tc_query_parser.rb

@@ -160,6 +160,21 @@ class QueryParserTest < Test::Unit::TestCase
     end
   end
 
+  def test_or_default
+    parser = Ferret::QueryParser.new(:default_field => :*,
+                                     :fields => [:x, :y],
+                                     :or_default => false,
+                                     :analyzer => StandardAnalyzer.new)
+    pairs = [
+      ['word', 'x:word y:word'],
+      ['word1 word2', '+(x:word1 y:word1) +(x:word2 y:word2)']
+    ]
+      
+    pairs.each do |query_str, expected|
+      assert_equal(expected, parser.parse(query_str).to_s(""))
+    end
+  end
+
   def test_prefix_query
     parser = Ferret::QueryParser.new(:default_field => "xxx",
                                      :fields => ["xxx"],

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.8.11
 specification_version: 1
 name: ferret
 version: !ruby/object:Gem::Version 
-  version: 0.10.0
-date: 2006-08-22 00:00:00 +09:00
+  version: 0.10.1
+date: 2006-08-26 00:00:00 -04:00
 summary: Ruby indexing library.
 require_paths: 
 - lib
@@ -29,178 +29,177 @@ authors:
 - David Balmain
 files: 
 - setup.rb
-- Rakefile
 - TODO
-- README
+- Rakefile
 - MIT-LICENSE
 - TUTORIAL
-- ext/q_filtered_query.c
-- ext/priorityqueue.c
-- ext/hashset.c
-- ext/helper.c
-- ext/global.c
-- ext/bitvector.c
+- README
+- ext/stem_ISO_8859_1_norwegian.c
+- ext/array.h
+- ext/bitvector.h
+- ext/document.c
+- ext/stem_UTF_8_spanish.h
+- ext/stem_ISO_8859_1_danish.c
+- ext/stem_UTF_8_portuguese.h
+- ext/similarity.h
+- ext/r_search.c
+- ext/hashset.h
+- ext/extconf.rb
+- ext/analysis.h
+- ext/index.h
+- ext/stem_UTF_8_russian.h
+- ext/q_match_all.c
+- ext/libstemmer.h
+- ext/stem_UTF_8_finnish.h
+- ext/stem_ISO_8859_1_german.c
 - ext/hash.c
-- ext/fs_store.c
-- ext/posh.c
-- ext/except.c
-- ext/ram_store.c
+- ext/stem_ISO_8859_1_english.h
+- ext/stem_ISO_8859_1_german.h
 - ext/store.c
-- ext/analysis.c
-- ext/document.c
-- ext/search.c
+- ext/stem_UTF_8_swedish.h
+- ext/q_term.c
 - ext/similarity.c
-- ext/stopwords.c
-- ext/array.c
-- ext/index.c
-- ext/mem_pool.c
+- ext/r_analysis.c
+- ext/stem_UTF_8_portuguese.c
+- ext/stem_ISO_8859_1_norwegian.h
+- ext/helper.c
+- ext/stem_UTF_8_french.h
+- ext/q_span.c
+- ext/utilities.c
+- ext/stem_ISO_8859_1_porter.h
+- ext/ind.c
+- ext/global.h
+- ext/except.h
+- ext/search.c
+- ext/stem_ISO_8859_1_danish.h
+- ext/stem_KOI8_R_russian.c
+- ext/stem_UTF_8_finnish.c
+- ext/stem_ISO_8859_1_french.c
 - ext/compound_io.c
-- ext/q_prefix.c
-- ext/q_range.c
-- ext/q_phrase.c
-- ext/q_term.c
+- ext/ferret.h
+- ext/stem_UTF_8_german.h
+- ext/stem_UTF_8_dutch.c
+- ext/hashset.c
+- ext/defines.h
+- ext/stem_ISO_8859_1_dutch.h
+- ext/stem_ISO_8859_1_swedish.h
 - ext/sort.c
-- ext/q_boolean.c
-- ext/filter.c
+- ext/document.h
+- ext/lang.h
+- ext/stem_UTF_8_porter.c
 - ext/q_const_score.c
-- ext/q_match_all.c
-- ext/q_wildcard.c
+- ext/stem_UTF_8_dutch.h
+- ext/q_prefix.c
 - ext/q_fuzzy.c
-- ext/q_multi_term.c
-- ext/q_parser.c
-- ext/q_span.c
-- ext/ind.c
-- ext/term_vectors.c
-- ext/priorityqueue.h
-- ext/defines.h
 - ext/posh.h
-- ext/store.h
-- ext/hashset.h
-- ext/helper.h
-- ext/global.h
-- ext/bitvector.h
-- ext/document.h
+- ext/stem_UTF_8_italian.h
+- ext/stem_ISO_8859_1_swedish.c
 - ext/hash.h
-- ext/except.h
-- ext/threading.h
-- ext/similarity.h
-- ext/index.h
-- ext/win32.h
-- ext/analysis.h
 - ext/search.h
-- ext/mem_pool.h
-- ext/array.h
-- ext/ind.h
-- ext/lang.h
-- ext/stem_UTF_8_norwegian.c
 - ext/stem_UTF_8_danish.c
-- ext/stem_UTF_8_dutch.c
-- ext/stem_UTF_8_english.c
-- ext/stem_UTF_8_finnish.c
-- ext/stem_UTF_8_french.c
-- ext/stem_UTF_8_german.c
-- ext/stem_UTF_8_italian.c
-- ext/stem_UTF_8_portuguese.c
-- ext/stem_UTF_8_porter.c
-- ext/stem_KOI8_R_russian.c
-- ext/stem_UTF_8_russian.c
-- ext/stem_UTF_8_spanish.c
+- ext/stem_ISO_8859_1_portuguese.h
+- ext/except.c
+- ext/ind.h
+- ext/modules.h
+- ext/win32.h
+- ext/stem_KOI8_R_russian.h
+- ext/q_range.c
 - ext/stem_UTF_8_swedish.c
+- ext/q_boolean.c
+- ext/api.h
+- ext/ram_store.c
+- ext/stem_UTF_8_porter.h
+- ext/inc
+- ext/stem_ISO_8859_1_spanish.h
+- ext/stem_UTF_8_danish.h
+- ext/index.c
 - ext/stem_ISO_8859_1_english.c
-- ext/stem_ISO_8859_1_danish.c
+- ext/q_phrase.c
+- ext/stopwords.c
+- ext/r_qparser.c
+- ext/store.h
+- ext/analysis.c
 - ext/stem_ISO_8859_1_dutch.c
-- ext/stem_ISO_8859_1_finnish.c
-- ext/stem_ISO_8859_1_french.c
-- ext/stem_ISO_8859_1_german.c
-- ext/stem_ISO_8859_1_italian.c
-- ext/stem_ISO_8859_1_norwegian.c
-- ext/stem_ISO_8859_1_porter.c
+- ext/stem_ISO_8859_1_finnish.h
+- ext/threading.h
 - ext/stem_ISO_8859_1_portuguese.c
-- ext/stem_ISO_8859_1_spanish.c
-- ext/stem_ISO_8859_1_swedish.c
-- ext/stem_UTF_8_danish.h
-- ext/stem_UTF_8_dutch.h
-- ext/stem_UTF_8_english.h
-- ext/stem_UTF_8_finnish.h
-- ext/stem_UTF_8_french.h
-- ext/stem_UTF_8_german.h
-- ext/stem_UTF_8_italian.h
+- ext/stem_UTF_8_russian.c
 - ext/stem_UTF_8_norwegian.h
-- ext/stem_UTF_8_porter.h
-- ext/stem_UTF_8_portuguese.h
-- ext/stem_UTF_8_russian.h
-- ext/stem_UTF_8_spanish.h
-- ext/stem_UTF_8_swedish.h
-- ext/stem_KOI8_R_russian.h
-- ext/stem_ISO_8859_1_danish.h
-- ext/stem_ISO_8859_1_dutch.h
-- ext/stem_ISO_8859_1_english.h
-- ext/stem_ISO_8859_1_finnish.h
-- ext/stem_ISO_8859_1_french.h
-- ext/stem_ISO_8859_1_german.h
-- ext/stem_ISO_8859_1_italian.h
-- ext/stem_ISO_8859_1_norwegian.h
-- ext/stem_ISO_8859_1_porter.h
-- ext/stem_ISO_8859_1_portuguese.h
-- ext/stem_ISO_8859_1_spanish.h
-- ext/stem_ISO_8859_1_swedish.h
-- ext/utilities.c
 - ext/api.c
-- ext/api.h
-- ext/header.h
-- ext/libstemmer.c
-- ext/modules.h
-- ext/libstemmer.h
-- ext/extconf.rb
-- ext/inc
-- ext/ferret.h
+- ext/array.c
+- ext/stem_ISO_8859_1_italian.c
+- ext/stem_ISO_8859_1_spanish.c
+- ext/r_store.c
+- ext/stem_UTF_8_german.c
 - ext/ferret.c
-- ext/tags
-- ext/r_analysis.c
+- ext/helper.h
+- ext/q_multi_term.c
+- ext/stem_UTF_8_english.c
+- ext/stem_ISO_8859_1_french.h
+- ext/q_parser.c
+- ext/stem_ISO_8859_1_italian.h
+- ext/stem_ISO_8859_1_porter.c
+- ext/priorityqueue.c
+- ext/stem_UTF_8_italian.c
+- ext/global.c
+- ext/filter.c
+- ext/stem_UTF_8_spanish.c
+- ext/stem_UTF_8_norwegian.c
 - ext/r_utils.c
-- ext/r_store.c
+- ext/stem_UTF_8_english.h
+- ext/stem_ISO_8859_1_finnish.c
+- ext/stem_UTF_8_french.c
+- ext/mem_pool.h
+- ext/posh.c
+- ext/fs_store.c
+- ext/mem_pool.c
+- ext/priorityqueue.h
+- ext/header.h
+- ext/bitvector.c
 - ext/r_index.c
-- ext/r_search.c
-- ext/r_qparser.c
-- ext/inc/threading.h
+- ext/libstemmer.c
+- ext/term_vectors.c
+- ext/q_filtered_query.c
+- ext/q_wildcard.c
 - ext/inc/lang.h
-- lib/ferret.rb
+- ext/inc/threading.h
 - lib/ferret_version.rb
+- lib/ferret.rb
 - lib/ferret/number_tools.rb
 - lib/ferret/index.rb
 - lib/ferret/document.rb
-- test/test_all.rb
 - test/test_helper.rb
-- test/unit/ts_analysis.rb
+- test/test_all.rb
 - test/unit/ts_store.rb
-- test/unit/ts_utils.rb
+- test/unit/ts_analysis.rb
+- test/unit/ts_query_parser.rb
 - test/unit/tc_document.rb
-- test/unit/ts_index.rb
 - test/unit/ts_search.rb
-- test/unit/ts_query_parser.rb
-- test/unit/utils/tc_bit_vector.rb
-- test/unit/utils/tc_priority_queue.rb
-- test/unit/utils/tc_number_tools.rb
-- test/unit/analysis/tc_analyzer.rb
-- test/unit/analysis/tc_token_stream.rb
-- test/unit/store/tm_store_lock.rb
+- test/unit/ts_utils.rb
+- test/unit/ts_index.rb
+- test/unit/store/tm_store.rb
 - test/unit/store/tc_fs_store.rb
+- test/unit/store/tm_store_lock.rb
 - test/unit/store/tc_ram_store.rb
-- test/unit/store/tm_store.rb
-- test/unit/index/tc_index_reader.rb
-- test/unit/index/tc_index_writer.rb
-- test/unit/index/tc_index.rb
-- test/unit/index/th_doc.rb
 - test/unit/search/tc_index_searcher.rb
-- test/unit/search/tc_multi_searcher.rb
 - test/unit/search/tc_fuzzy_query.rb
-- test/unit/search/tc_sort.rb
-- test/unit/search/tc_filter.rb
-- test/unit/search/tc_spans.rb
-- test/unit/search/tc_sort_field.rb
+- test/unit/search/tc_multi_searcher.rb
 - test/unit/search/tc_search_and_sort.rb
 - test/unit/search/tm_searcher.rb
+- test/unit/search/tc_spans.rb
+- test/unit/search/tc_filter.rb
+- test/unit/search/tc_sort.rb
+- test/unit/search/tc_sort_field.rb
+- test/unit/index/tc_index.rb
+- test/unit/index/th_doc.rb
+- test/unit/index/tc_index_writer.rb
+- test/unit/index/tc_index_reader.rb
+- test/unit/analysis/tc_token_stream.rb
+- test/unit/analysis/tc_analyzer.rb
 - test/unit/query_parser/tc_query_parser.rb
+- test/unit/utils/tc_priority_queue.rb
+- test/unit/utils/tc_bit_vector.rb
+- test/unit/utils/tc_number_tools.rb
 test_files: []
 
 rdoc_options: 
@@ -216,12 +215,12 @@ extra_rdoc_files:
 - TODO
 - TUTORIAL
 - MIT-LICENSE
+- ext/r_search.c
 - ext/r_analysis.c
-- ext/r_utils.c
+- ext/r_qparser.c
 - ext/r_store.c
+- ext/r_utils.c
 - ext/r_index.c
-- ext/r_search.c
-- ext/r_qparser.c
 - ext/ferret.c
 executables: []