ferret 0.11.0 → 0.11.1

data/ext/analysis.c

@@ -845,7 +845,7 @@ static Token *std_next(TokenStream *ts)
     char *t;
     char *start = NULL;
     char *num_end = NULL;
-    char token[MAX_WORD_SIZE];
+    char token[MAX_WORD_SIZE + 1];
     int token_i = 0;
     int len;
     bool is_acronym;
@@ -925,6 +925,9 @@ static Token *std_next(TokenStream *ts)
         }
         ts->t = t + len;
         token[len] = 0;
+        Token *k = tk_set(&(CTS(ts)->token), token, len, (int)(start - ts->text),
+               (int)(ts->t - ts->text), 1);
+        return k;
         return tk_set(&(CTS(ts)->token), token, len, (int)(start - ts->text),
                (int)(ts->t - ts->text), 1);
     }
@@ -1174,7 +1177,7 @@ static TokenStream *mf_clone_i(TokenStream *orig_ts)
 
 static Token *mf_next(TokenStream *ts)
 {
-    char buf[MAX_WORD_SIZE];
+    char buf[MAX_WORD_SIZE + 1];
     MultiMapper *mapper = MFilt(ts)->mapper;
     TokenFilter *tf = TkFilt(ts);
     Token *tk = tf->sub_ts->next(tf->sub_ts);
@@ -1299,7 +1302,7 @@ TokenStream *hyphen_filter_new(TokenStream *sub_ts)
 
 Token *mb_lcf_next(TokenStream *ts)
 {
-    wchar_t wbuf[MAX_WORD_SIZE], *wchr;
+    wchar_t wbuf[MAX_WORD_SIZE + 1], *wchr;
     Token *tk = TkFilt(ts)->sub_ts->next(TkFilt(ts)->sub_ts);
 
     if (tk == NULL) {

data/ext/extconf.rb

@@ -6,6 +6,6 @@ if (/mswin/ =~ RUBY_PLATFORM) and ENV['make'].nil?
 else
   require 'mkmf'
   #$CFLAGS += " -fno-common"
-  $CFLAGS += " -fno-common -D_FILE_OFFSET_BITS=64"
+  $CFLAGS += " -fno-stack-protector -fno-common -D_FILE_OFFSET_BITS=64"
   create_makefile("ferret_ext")
 end

data/ext/fs_store.c

@@ -335,7 +335,7 @@ static InStream *fs_open_input(Store *store, const char *filename)
     return is;
 }
 
-#define LOCK_OBTAIN_TIMEOUT 5
+#define LOCK_OBTAIN_TIMEOUT 10
 
 #ifdef RUBY_BINDINGS
 struct timeval rb_time_interval _((VALUE));

data/ext/global.c

@@ -6,7 +6,6 @@
 #include <assert.h>
 #include <math.h>
 #include <ctype.h>
-#include <unistd.h>
 
 const char *EMPTY_STRING = "";
 
@@ -359,14 +358,24 @@ void dummy_free(void *p)
 
 #ifdef FRT_IS_C99
 extern void usleep(unsigned long usec);
+#else
+# ifdef RUBY_BINDINGS
+struct timeval rb_time_interval _((VALUE));
+# else
+#  include <unistd.h>
+# endif
 #endif
 
 extern void micro_sleep(const int micro_seconds)
 {
-#ifdef POSH_OS_WIN32
-    Sleep(micro_seconds / 1000);
+#ifdef RUBY_BINDINGS
+    rb_thread_wait_for(rb_time_interval(rb_float_new((double)micro_seconds/1000000.0)));
 #else
+# ifdef POSH_OS_WIN32
+    Sleep(micro_seconds / 1000);
+# else
     usleep(micro_seconds);
+# endif
 #endif
 }
 

data/ext/index.c

@@ -35,7 +35,7 @@ static void ste_reset(TermEnum *te);
 static char *ste_next(TermEnum *te);
 
 #define FORMAT 0
-#define SEGMENTS_GEN_FILE_NAME "segments.gen"
+#define SEGMENTS_GEN_FILE_NAME "segments"
 #define MAX_EXT_LEN 10
 
 /* *** Must be three characters *** */
@@ -53,7 +53,7 @@ static const char BASE36_DIGITMAP[] = "0123456789abcdefghijklmnopqrstuvwxyz";
 
 static char *u64_to_str36(char *buf, int buf_size, f_u64 u)
 {
-    int i = buf_size--;
+    int i = buf_size - 1;
     buf[i] = '\0';
     for (i--; i >= 0; i--) {
         buf[i] = BASE36_DIGITMAP[u % 36];
@@ -826,7 +826,7 @@ void sis_find_segments_file(Store *store, FindSegmentsFile *fsf,
         }
 
         /* Method 2 (fallback if Method 1 isn't reliable): if the directory
-         * listing seems to be stale, try loading the "segments.gen" file. */
+         * listing seems to be stale, try loading the "segments" file. */
         if (1 == method || (0 == method && last_gen == gen && retry)) {
             method = 1;
             for (i = 0; i < GEN_FILE_RETRY_COUNT; i++) {
@@ -835,7 +835,7 @@ void sis_find_segments_file(Store *store, FindSegmentsFile *fsf,
                     gen_is = store->open_input(store, SEGMENTS_GEN_FILE_NAME);
                 XCATCHALL
                     HANDLED();
-                    /* TODO:LOG "segments.gen open: IO_ERROR"*/
+                    /* TODO:LOG "segments open: IO_ERROR"*/
                 XENDTRY
 
                 if (NULL != gen_is) {

data/ext/q_phrase.c

@@ -498,12 +498,12 @@ static Scorer *phw_scorer(Weight *self, IndexReader *ir)
     if (phq->slop == 0) {       /* optimize exact (common) case */
         phsc = exact_phrase_scorer_new(self, tps, positions, pos_cnt,
                                        self->similarity,
-                                       ir->get_norms(ir, field_num));
+                                       ir_get_norms_i(ir, field_num));
     }
     else {
         phsc = sloppy_phrase_scorer_new(self, tps, positions, pos_cnt,
                                         self->similarity, phq->slop,
-                                        ir->get_norms(ir, field_num));
+                                        ir_get_norms_i(ir, field_num));
     }
     free(tps);
     return phsc;

data/ext/r_index.c

@@ -1205,7 +1205,7 @@ frt_iw_init(int argc, VALUE *argv, VALUE self)
         store = open_ram_store();
         DEREF(store);
     }
-    if (!create && create_if_missing && !store->exists(store, "segments.gen")) {
+    if (!create && create_if_missing && !store->exists(store, "segments")) {
         create = true;
     }
     if (create) {

data/lib/ferret/index.rb

@@ -1,6 +1,20 @@
 require 'monitor'
 
 module Ferret::Index
+  module SynchroLockMixin
+    def synchrolock
+      trys = 5
+      begin
+        synchronize {yield}
+      rescue Ferret::Store::Lock::LockError => e
+        if (trys -= 1) <= 0
+          raise e
+        else
+          retry
+        end
+      end
+    end
+  end
   # This is a simplified interface to the index. See the TUTORIAL for more
   # information on how to use this class.
   class Index
@@ -116,37 +130,34 @@ module Ferret::Index
         @dir = RAMDirectory.new
       end
 
+      @dir.extend(MonitorMixin).extend(SynchroLockMixin)
       options[:dir] = @dir
       options[:lock_retry_time]||= 2
-      @dir.extend(MonitorMixin)
-      @dir.synchronize do
-        @options = options
-        if (!@dir.exists?("segments")) || options[:create]
-          IndexWriter.new(options).close
-        end
-        options[:analyzer]||= Ferret::Analysis::StandardAnalyzer.new
+      @options = options
+      if (!@dir.exists?("segments")) || options[:create]
+        IndexWriter.new(options).close
+      end
+      options[:analyzer]||= Ferret::Analysis::StandardAnalyzer.new
 
-        @searcher = nil
-        @writer = nil
-        @reader = nil
+      @searcher = nil
+      @writer = nil
+      @reader = nil
 
-        @options.delete(:create) # only create the first time if at all
-        @auto_flush = @options[:auto_flush] || false
-        if (@options[:id_field].nil? and
-            @key.is_a?(Symbol))
-          @id_field = @key
-        else
-          @id_field = @options[:id_field] || :id
-        end
-        @default_field = (@options[:default_field]||= :*)
-        @default_input_field = options[:default_input_field] || @id_field
+      @options.delete(:create) # only create the first time if at all
+      @auto_flush = @options[:auto_flush] || false
+      if (@options[:id_field].nil? and @key.is_a?(Symbol))
+        @id_field = @key
+      else
+        @id_field = @options[:id_field] || :id
+      end
+      @default_field = (@options[:default_field]||= :*)
+      @default_input_field = options[:default_input_field] || @id_field
 
-        if @default_input_field.respond_to?(:intern)
-          @default_input_field = @default_input_field.intern
-        end
-        @open = true
-        @qp = nil
+      if @default_input_field.respond_to?(:intern)
+        @default_input_field = @default_input_field.intern
       end
+      @open = true
+      @qp = nil
       if block
         yield self
         self.close
@@ -253,7 +264,8 @@ module Ferret::Index
     # 
     # See FieldInfos for more information on how to set field properties.
     def add_document(doc, analyzer = nil)
-      @dir.synchronize do
+      @dir.synchrolock do
+        ensure_writer_open()
         if doc.is_a?(String) or doc.is_a?(Array)
           doc = {@default_input_field => doc}
         end
@@ -399,7 +411,8 @@ module Ferret::Index
     #
     # id:: The number of the document to delete
     def delete(id)
-      @dir.synchronize do
+      @dir.synchrolock do
+        ensure_writer_open()
         if id.is_a?(String) or id.is_a?(Symbol)
           ensure_writer_open()
           @writer.delete(@id_field, id.to_s)
@@ -420,7 +433,8 @@ module Ferret::Index
     #         string (in which case it is parsed by the standard query parser)
     #         or an actual query object.
     def query_delete(query)
-      @dir.synchronize do
+      @dir.synchrolock do
+        ensure_writer_open()
         ensure_searcher_open()
         query = do_process_query(query)
         @searcher.search_each(query) do |doc, score|
@@ -447,7 +461,8 @@ module Ferret::Index
     #           the :key attribute.
     # new_doc:: The document to replace the old document with
     def update(id, new_doc)
-      @dir.synchronize do
+      @dir.synchrolock do
+        ensure_writer_open()
         delete(id)
         if id.is_a?(String) or id.is_a?(Symbol)
           @writer.commit
@@ -484,7 +499,8 @@ module Ferret::Index
     #     #=> {:id => "28", :title => "My Oh My", :artist => "David Gray"}
     #
     def query_update(query, new_val)
-      @dir.synchronize do
+      @dir.synchrolock do
+        ensure_writer_open()
         ensure_searcher_open()
         docs_to_add = []
         query = do_process_query(query)
@@ -534,7 +550,7 @@ module Ferret::Index
     # optimizes the index. This should only be called when the index will no
     # longer be updated very often, but will be read a lot.
     def optimize()
-      @dir.synchronize do
+      @dir.synchrolock do
         ensure_writer_open()
         @writer.optimize()
         @writer.close()
@@ -562,7 +578,8 @@ module Ferret::Index
     #
     # After this completes, the index is optimized.
     def add_indexes(indexes)
-      @dir.synchronize do
+      @dir.synchrolock do
+        ensure_writer_open()
         indexes = [indexes].flatten   # make sure we have an array
         return if indexes.size == 0 # nothing to do
         if indexes[0].is_a?(Index)
@@ -604,7 +621,7 @@ module Ferret::Index
         elsif directory.is_a?(Ferret::Store::Directory)
           @dir = directory
         end
-        @dir.extend(MonitorMixin)
+        @dir.extend(MonitorMixin).extend(SynchroLockMixin)
         @options[:dir] = @dir
         @options[:create_if_missing] = true
         add_indexes([old_dir])
@@ -646,7 +663,7 @@ module Ferret::Index
     # Returns the field_infos object so that you can add new fields to the
     # index.
     def field_infos
-      @dir.synchronize do
+      @dir.synchrolock do
         ensure_writer_open()
         return @writer.field_infos
       end

data/lib/ferret_version.rb

@@ -1,3 +1,3 @@
 module Ferret
-  VERSION = '0.11.0'
+  VERSION = '0.11.1'
 end

data/test/threading/thread_safety_index_test.rb

@@ -6,8 +6,8 @@ class IndexThreadSafetyTest < Test::Unit::TestCase
   include Ferret::Index
 
   INDEX_DIR = File.expand_path(File.join(File.dirname(__FILE__), "index"))
-  ITERATIONS = 1000
-  NUM_THREADS = 2
+  ITERATIONS = 100
+  NUM_THREADS = 3
   ANALYZER = Ferret::Analysis::StandardAnalyzer.new()
 
   def setup
@@ -35,6 +35,7 @@ class IndexThreadSafetyTest < Test::Unit::TestCase
       else
         do_add_doc(index)
       end
+      index.commit
     end
   end 
 

data/test/unit/index/tc_index_writer.rb

@@ -21,10 +21,10 @@ class IndexWriterTest < Test::Unit::TestCase
     assert(! wlock.locked?)
     assert(! clock.locked?)
     iw = IndexWriter.new(:dir => @dir, :create => true)
-    assert(@dir.exists?("segments.gen"))
+    assert(@dir.exists?("segments"))
     assert(wlock.locked?)
     iw.close()
-    assert(@dir.exists?("segments.gen"))
+    assert(@dir.exists?("segments"))
     assert(! wlock.locked?)
     assert(! clock.locked?)
   end

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.9.0
 specification_version: 1
 name: ferret
 version: !ruby/object:Gem::Version 
-  version: 0.11.0
-date: 2007-02-25 00:00:00 +11:00
+  version: 0.11.1
+date: 2007-02-27 00:00:00 +11:00
 summary: Ruby indexing library.
 require_paths: 
 - lib