ferret 0.10.1 → 0.10.2

data/Rakefile

@@ -34,6 +34,7 @@ EXT_SRC = FileList["../c/src/*.[c]", "../c/include/*.h",
                    "../c/lib/libstemmer_c/runtime/*.[ch]",
                    "../c/lib/libstemmer_c/libstemmer/*.[ch]",
                    "../c/lib/libstemmer_c/include/libstemmer.h"]
+EXT_SRC.exclude('../**/ind.[ch]')
 
 EXT_SRC_DEST = EXT_SRC.map {|fn| File.join("ext", File.basename(fn))}
 SRC = (FileList["ext/*.[ch]"] + EXT_SRC_DEST).uniq
@@ -94,7 +95,12 @@ end
 EXT_SRC.each do |fn|
   dest_fn = File.join("ext", File.basename(fn))
   file dest_fn => fn do |t|
-    ln_s File.join("..", fn), dest_fn
+    begin
+      ln_s File.join("..", fn), dest_fn
+    rescue Exception => e
+      cp File.expand_path(fn), dest_fn
+    end
+
     if fn =~ /stemmer/
       # flatten the directory structure for lib_stemmer
       open(dest_fn) do |in_f|

data/ext/analysis.c

@@ -12,7 +12,7 @@
  *
  ****************************************************************************/
 
-inline Token *tk_set(Token *tk,
+__inline Token *tk_set(Token *tk,
                      char *text, int tlen, int start, int end, int pos_inc)
 {
     if (tlen >= MAX_WORD_SIZE) {
@@ -27,20 +27,20 @@ inline Token *tk_set(Token *tk,
     return tk;
 }
 
-inline Token *tk_set_ts(Token *tk,
+__inline Token *tk_set_ts(Token *tk,
                         char *start, char *end, char *text, int pos_inc)
 {
     return tk_set(tk, start, (int)(end - start),
                   (int)(start - text), (int)(end - text), pos_inc);
 }
 
-inline Token *tk_set_no_len(Token *tk,
+__inline Token *tk_set_no_len(Token *tk,
                             char *text, int start, int end, int pos_inc)
 {
     return tk_set(tk, text, (int)strlen(text), start, end, pos_inc);
 }
 
-inline Token *w_tk_set(Token *tk, wchar_t *text, int start, int end,
+__inline Token *w_tk_set(Token *tk, wchar_t *text, int start, int end,
                        int pos_inc)
 {
     int len = wcstombs(tk->text, text, MAX_WORD_SIZE - 1);
@@ -151,7 +151,7 @@ static TokenStream *cts_new()
 
 #define MBTS(token_stream) ((MultiByteTokenStream *)(token_stream))
 
-inline int mb_next_char(wchar_t *wchr, const char *s, mbstate_t *state)
+__inline int mb_next_char(wchar_t *wchr, const char *s, mbstate_t *state)
 {
     int num_bytes;
     if ((num_bytes = (int)mbrtowc(wchr, s, MB_CUR_MAX, state)) < 0) {
@@ -161,7 +161,7 @@ inline int mb_next_char(wchar_t *wchr, const char *s, mbstate_t *state)
             ZEROSET(state, mbstate_t);
             num_bytes = (int)mbrtowc(wchr, t, MB_CUR_MAX, state);
         } while ((num_bytes < 0) && (*wchr != 0) && (*t != 0));
-        num_bytes += t - s;
+        num_bytes = t - s;
     }
     return num_bytes;
 }
@@ -774,6 +774,7 @@ static bool std_advance_to_start(TokenStream *ts)
 {
     char *t = ts->t;
     while (*t != '\0' && !isalnum(*t)) {
+        if (isnumpunc(*t) && isdigit(t[1])) break;
         t++;
     }
 
@@ -790,7 +791,8 @@ static bool mb_std_advance_to_start(TokenStream *ts)
 
     i = mb_next_char(&wchr, ts->t, &state);
 
-    while (wchr != 0 && !iswalnum(wchr)) {
+    while (wchr != 0 && !iswalpha(wchr) && !isdigit(*(ts->t))) {
+        if (isnumpunc(*ts->t) && isdigit(ts->t[1])) break;
         ts->t += i;
         i = mb_next_char(&wchr, ts->t, &state);
     }
@@ -817,8 +819,9 @@ static Token *std_next(TokenStream *ts)
     }
 
     start = t = ts->t;
-    if (isdigit(*t)) {
-        t += std_get_number(t);
+    if ((isdigit(*t) || isnumpunc(*t))
+        && ((len = std_get_number(t)) > 0)) {
+        t += len;
         ts->t = t;
         tk_set_ts(&(CTS(ts)->token), start, t, ts->text, 1);
     }
@@ -850,8 +853,9 @@ static Token *std_next(TokenStream *ts)
             return tk_set_ts(&(CTS(ts)->token), start, t, ts->text, 1);
         }
 
-        if (isdigit(*t) || isnumpunc(*t)) {      /* possibly a number */
-            num_end = start + std_get_number(start);
+        if ((isdigit(*t) || isnumpunc(*t))       /* possibly a number */
+            && (len = std_get_number(t) > 0)) {
+            num_end = start + len;
             if (!std_tz->is_tok_char(num_end)) { /* won't find a longer token */
                 ts->t = num_end;
                 return tk_set_ts(&(CTS(ts)->token), start, num_end, ts->text, 1);
@@ -908,7 +912,7 @@ static Token *std_next(TokenStream *ts)
             t--;                /* strip trailing punctuation */
         }
 
-        if (t > num_end) {
+        if (num_end == NULL || t > num_end) {
             ts->t = t;
 
             if (is_acronym) {   /* check it is one letter followed by one '.' */
@@ -1132,13 +1136,17 @@ Token *mb_lcf_next(TokenStream *ts)
         return tk;
     }
 
-    mbstowcs(wbuf, tk->text, MAX_WORD_SIZE);
+    if (mbstowcs(wbuf, tk->text, MAX_WORD_SIZE) <= 0) return tk;
     wchr = wbuf;
     while (*wchr != 0) {
         *wchr = towlower(*wchr);
         wchr++;
     }
     tk->len = wcstombs(tk->text, wbuf, MAX_WORD_SIZE);
+    if (tk->len <= 0) {
+        strcpy(tk->text, "BAD_DATA");
+        tk->len = 8;
+    }
     tk->text[tk->len] = '\0';
     return tk;
 }

data/ext/array.c

@@ -16,7 +16,7 @@ void **ary_new_i(int type_size, int init_capa)
     return (void **)&ary[3];
 }
 
-inline void ary_resize_i(void ***ary, int size)
+__inline void ary_resize_i(void ***ary, int size)
 {
     size++;
     if (size >= ary_sz(*ary)) {

data/ext/bitvector.c

@@ -190,7 +190,7 @@ const int NUM_TRAILING_ZEROS[] = {
 /*
  * This method is highly optimized, hence the loop unrolling
  */
-static inline int bv_get_1_offset(f_u32 word)
+static __inline int bv_get_1_offset(f_u32 word)
 {
     if (word & 0xff) {
         return NUM_TRAILING_ZEROS[word & 0xff];
@@ -397,7 +397,7 @@ BitVector *bv_and_x(BitVector *bv1, BitVector *bv2)
     return bv_and_i(bv1, bv1, bv2);
 }
 
-static inline void bv_recapa(BitVector *bv, int new_capa)
+static __inline void bv_recapa(BitVector *bv, int new_capa)
 {
     if (bv->capa < new_capa) {
         REALLOC_N(bv->bits, f_u32, new_capa);

data/ext/defines.h

@@ -30,20 +30,14 @@ typedef posh_i64_t f_i64;
 #endif
 
 #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)
-#define FRT_HAS_INLINE
 #define FRT_IS_C99
 #define FRT_HAS_ISO_VARARGS
 #define FRT_HAS_VARARGS
 #endif
 
 #if defined(__GNUC__) && !defined(__STRICT_ANSI__)
-#define FRT_HAS_INLINE
 #define FRT_HAS_GNUC_VARARGS
 #define FRT_HAS_VARARGS
 #endif
 
-#ifndef FRT_HAS_INLINE
-# define inline
-#endif
-
 #endif

data/ext/except.c

@@ -26,8 +26,8 @@ const char *const FRT_ERROR_TYPES[] = {
 
 char *const UNSUPPORTED_ERROR_MSG = "Unsupported operation";
 char *const EOF_ERROR_MSG = "Read past end of file";
-char xmsg_buffer[2048];
-char xmsg_buffer_final[2048];
+char xmsg_buffer[XMSG_BUFFER_SIZE];
+char xmsg_buffer_final[XMSG_BUFFER_SIZE];
 
 static thread_key_t exception_stack_key;
 static thread_once_t exception_stack_key_once = THREAD_ONCE_INIT;
@@ -48,9 +48,9 @@ void xpush_context(xcontext_t *context)
     context->in_finally = false;
 }
 
-static inline void xraise_context(xcontext_t *context,
-                                  volatile int excode,
-                                  const char *const msg)
+static __inline void xraise_context(xcontext_t *context,
+                                    volatile int excode,
+                                    const char *const msg)
 {
     context->msg = msg;
     context->excode = excode;
@@ -63,7 +63,7 @@ void RAISE(int excode, const char *fmt, ...)
 {
     va_list args;
     va_start(args, fmt);
-    vsprintf(xmsg_buffer, fmt, args);
+    vsnprintf(xmsg_buffer, XMSG_BUFFER_SIZE, fmt, args);
     xraise(excode, xmsg_buffer);
     va_end(args);
 }

data/ext/except.h

@@ -68,7 +68,7 @@
 #define BODY 0
 #define FINALLY 1
 #define EXCEPTION 2
-#define ERROR 2
+#define FERRET_ERROR 2
 #define IO_ERROR 3
 #define ARG_ERROR 4
 #define EOF_ERROR 5
@@ -121,17 +121,21 @@ typedef struct xcontext_t
 
 #define HANDLED() xcontext.handled = 1; /* true */
 
+#define XMSG_BUFFER_SIZE 2048
+
 #ifdef FRT_HAS_ISO_VARARGS
 # define RAISE(excode, ...) do {\
-  sprintf(xmsg_buffer, __VA_ARGS__);\
-  sprintf(xmsg_buffer_final, "Error occured in %s:%d - %s\n\t%s\n",\
+  snprintf(xmsg_buffer, XMSG_BUFFER_SIZE, __VA_ARGS__);\
+  snprintf(xmsg_buffer_final, XMSG_BUFFER_SIZE,\
+          "Error occured in %s:%d - %s\n\t%s\n",\
           __FILE__, __LINE__, __func__, xmsg_buffer);\
   xraise(excode, xmsg_buffer_final);\
 } while (0)
 #elif defined(FRT_HAS_GNUC_VARARGS)
 # define RAISE(excode, args...) do {\
-  sprintf(xmsg_buffer, ##args);\
-  sprintf(xmsg_buffer_final, "Error occured in %s:%d - %s\n\t%s\n",\
+  snprintf(xmsg_buffer, XMSG_BUFFER_SIZE, ##args);\
+  snprintf(xmsg_buffer_final, XMSG_BUFFER_SIZE,\
+          "Error occured in %s:%d - %s\n\t%s\n",\
           __FILE__, __LINE__, __func__, xmsg_buffer);\
   xraise(excode, xmsg_buffer_final);\
 } while (0)
@@ -139,14 +143,14 @@ typedef struct xcontext_t
 #else
 extern void RAISE(int excode, const char *fmt, ...);
 #endif
-#define RAISE_HELL() RAISE(ERROR, "Hell")
+#define RAISE_HELL() RAISE(FERRET_ERROR, "Hell")
 
 
 extern void xraise(int excode, const char *const msg);
 extern void xpush_context(xcontext_t *context);
 extern void xpop_context();
 
-extern char xmsg_buffer[2048];
-extern char xmsg_buffer_final[2048];
+extern char xmsg_buffer[XMSG_BUFFER_SIZE];
+extern char xmsg_buffer_final[XMSG_BUFFER_SIZE];
 
 #endif

data/ext/extconf.rb

@@ -1,6 +1,7 @@
 # extconf.rb for Ferret extensions
 if (/mswin/ =~ RUBY_PLATFORM) and ENV['make'].nil?
   require 'mkmf'
+  $LIBS += " msvcprt.lib"
   create_makefile("ferret_ext")
 else
   require 'mkmf'

data/ext/ferret.c

@@ -15,6 +15,7 @@ ID id_less_than;
 ID id_lt;
 ID id_call;
 ID id_is_directory;
+ID id_close;
 ID id_data;
 
 static ID id_mkdir_p;
@@ -182,6 +183,8 @@ VALUE frt_get_error(const char *err_type)
 #ifdef FRT_HAS_VARARGS
 void vfrt_rb_raise(const char *file, int line_num, const char *func,
                    const char *err_type, const char *fmt, va_list args)
+#else
+void V_FRT_EXIT(const char *err_type, const char *fmt, va_list args)
 #endif
 {
     char buf[FRT_BUF_SIZ];
@@ -250,6 +253,7 @@ void Init_ferret_ext(void)
 
     id_mkdir_p = rb_intern("mkdir_p");
     id_is_directory = rb_intern("directory?");
+    id_close = rb_intern("close");
 
     id_data = rb_intern("@data");
 

data/ext/ferret.h

@@ -12,6 +12,7 @@ extern ID id_less_than;
 extern ID id_lt;
 extern ID id_call;
 extern ID id_is_directory;
+extern ID id_close;
 extern ID id_data;
 
 /* Symbols */

data/ext/fs_store.c

@@ -1,4 +1,5 @@
 #include "store.h"
+#include <time.h>
 #include <sys/types.h>
 #include <fcntl.h>
 #include <sys/stat.h>
@@ -41,7 +42,7 @@ extern int file_is_lock(char *filename);
  */
 static char *join_path(char *buf, const char *base, const char *filename)
 {
-  sprintf(buf, "%s"DIR_SEPARATOR"%s", base, filename);
+  snprintf(buf, MAX_FILE_PATH, "%s"DIR_SEPARATOR"%s", base, filename);
   return buf;
 }
 
@@ -330,6 +331,10 @@ static InStream *fs_open_input(Store *store, const char *filename)
 
 #define LOCK_OBTAIN_TIMEOUT 5
 
+#ifdef RUBY_BINDINGS
+struct timeval rb_time_interval _((VALUE));
+#endif
+
 static int fs_lock_obtain(Lock *lock)
 {
     int f;
@@ -337,9 +342,18 @@ static int fs_lock_obtain(Lock *lock)
     while (((f =
              open(lock->name, O_CREAT | O_EXCL | O_RDWR,
                    S_IRUSR | S_IWUSR)) < 0) && (trys > 0)) {
+
+#ifdef RUBY_BINDINGS
+        rb_thread_wait_for(rb_time_interval(rb_float_new(0.01)));
+#endif
         trys--;
-        /* FIXME:: implement sleep properly to sleep in milliseconds
-         * sleep(1); */
+        /* sleep for 10 milliseconds
+        clock_t start = clock();
+        trys--;
+
+        while (((double)(clock() - start) / CLOCKS_PER_SEC) < 0.01) {
+        }
+        */
     }
     if (f >= 0) {
         close(f);
@@ -375,7 +389,7 @@ static Lock *fs_open_lock(Store *store, char *lockname)
     Lock *lock = ALLOC(Lock);
     char lname[100];
     char path[MAX_FILE_PATH];
-    sprintf(lname, "%s%s.lck", LOCK_PREFIX, lockname);
+    snprintf(lname, 100, "%s%s.lck", LOCK_PREFIX, lockname);
     lock->name = estrdup(join_path(path, store->dir.path, lname));
     lock->store = store;
     lock->obtain = &fs_lock_obtain;

data/ext/global.c

@@ -11,22 +11,22 @@ const char *EMPTY_STRING = "";
 
 bool x_do_logging = false;
 
-inline int min3(int a, int b, int c)
+__inline int min3(int a, int b, int c)
 {
     return MIN3(a, b, c);
 }
 
-inline int min2(int a, int b)
+__inline int min2(int a, int b)
 {
     return MIN(a, b);
 }
 
-inline int max3(int a, int b, int c)
+__inline int max3(int a, int b, int c)
 {
     return MAX3(a, b, c);
 }
 
-inline int max2(int a, int b)
+__inline int max2(int a, int b)
 {
     return MAX(a, b);
 }
@@ -56,13 +56,14 @@ int icmp_risky(const void *p1, const void *p2)
 }
 
 
+#ifndef RUBY_BINDINGS
 /* frt_exit: print error message and exit */
-#ifdef FRT_HAS_VARARGS
+# ifdef FRT_HAS_VARARGS
 void vfrt_exit(const char *file, int line_num, const char *func,
                const char *err_type, const char *fmt, va_list args)
-#else
+# else
 void V_FRT_EXIT(const char *err_type, const char *fmt, va_list args)
-#endif
+# endif
 {
     fflush(stdout);
     fprintf(stderr, "\n");
@@ -70,12 +71,12 @@ void V_FRT_EXIT(const char *err_type, const char *fmt, va_list args)
         fprintf(stderr, "%s: ", progname());
     }
 
-#ifdef FRT_HAS_VARARGS
+# ifdef FRT_HAS_VARARGS
     fprintf(stderr, "%s occured at <%s>:%d in %s\n",
             err_type, file, line_num, func);
-#else
+# else
     fprintf(stderr, "%s occured:\n", err_type);
-#endif
+# endif
     vfprintf(stderr, fmt, args);
 
     if (fmt[0] != '\0' && fmt[strlen(fmt) - 1] == ':') {
@@ -87,22 +88,23 @@ void V_FRT_EXIT(const char *err_type, const char *fmt, va_list args)
 }
 
 
-#ifdef FRT_HAS_VARARGS
+# ifdef FRT_HAS_VARARGS
 void frt_exit(const char *file, int line_num, const char *func,
               const char *err_type, const char *fmt, ...)
-#else
+# else
 void FRT_EXIT(const char *err_type, const char *fmt, ...)
-#endif
+# endif
 {
     va_list args;
     va_start(args, fmt);
-#ifdef FRT_HAS_VARARGS
+# ifdef FRT_HAS_VARARGS
     vfrt_exit(file, line_num, func, err_type, fmt, args);
-#else
+# else
     V_FRT_EXIT(err_type, fmt, args);
-#endif
+# endif
     va_end(args);
 }
+#endif
 
 
 /* weprintf: print error message and don't exit */