ferret 0.11.4 → 0.11.5

data/Rakefile

@@ -55,6 +55,7 @@ task :valgrind do
      "--leak-check=yes --show-reachable=yes -v ruby test/test_all.rb"
   #sh "valgrind --suppressions=ferret_valgrind.supp " +
   #   "--leak-check=yes --show-reachable=yes -v ruby test/unit/index/tc_index_reader.rb"
+  #valgrind --gen-suppressions=yes --suppressions=ferret_valgrind.supp --leak-check=yes --show-reachable=yes -v ruby test/test_all.rb
 end
 
 task :default => :test_all

data/TUTORIAL

@@ -116,7 +116,7 @@ when printing to the console:
     puts highlights
   end
 
-And if you want to highlight a whole document, set :excert_length to :all:
+And if you want to highlight a whole document, set :excerpt_length to :all:
 
   puts index.highlight(query, doc_id,
                        :field => :content,
@@ -175,7 +175,7 @@ you change the data once it is in the index. But you can delete documents so
 the standard way to modify data is to delete it and re-add it again with the
 modifications made. It is important to note that when doing this the documents
 will get a new document number so you should be careful not to use a document
-number after the document has been deleted. Here is an examle of modifying a
+number after the document has been deleted. Here is an example of modifying a
 document;
 
   index << {:title => "Programing Rbuy", :content => "blah blah blah"}
@@ -185,7 +185,7 @@ document;
   doc = index[doc_id]
   index.delete(doc_id)
 
-  # modify doc. It is just a Hash afterall
+  # modify doc. It is just a Hash after all
   doc[:title] = "Programming Ruby"
 
   index << doc

data/ext/analysis.c

@@ -1,6 +1,6 @@
 #include "analysis.h"
 #include "hash.h"
-#include <libstemmer.h>
+#include "libstemmer.h"
 #include <string.h>
 #include <ctype.h>
 #include <wctype.h>
@@ -12,7 +12,7 @@
  *
  ****************************************************************************/
 
-__inline Token *tk_set(Token *tk,
+INLINE Token *tk_set(Token *tk,
                      char *text, int tlen, int start, int end, int pos_inc)
 {
     if (tlen >= MAX_WORD_SIZE) {
@@ -27,20 +27,20 @@ __inline Token *tk_set(Token *tk,
     return tk;
 }
 
-__inline Token *tk_set_ts(Token *tk,
+INLINE Token *tk_set_ts(Token *tk,
                         char *start, char *end, char *text, int pos_inc)
 {
     return tk_set(tk, start, (int)(end - start),
                   (int)(start - text), (int)(end - text), pos_inc);
 }
 
-__inline Token *tk_set_no_len(Token *tk,
+INLINE Token *tk_set_no_len(Token *tk,
                             char *text, int start, int end, int pos_inc)
 {
     return tk_set(tk, text, (int)strlen(text), start, end, pos_inc);
 }
 
-__inline Token *w_tk_set(Token *tk, wchar_t *text, int start, int end,
+INLINE Token *w_tk_set(Token *tk, wchar_t *text, int start, int end,
                        int pos_inc)
 {
     int len = wcstombs(tk->text, text, MAX_WORD_SIZE - 1);
@@ -152,7 +152,7 @@ static TokenStream *cts_new()
 
 #define MBTS(token_stream) ((MultiByteTokenStream *)(token_stream))
 
-__inline int mb_next_char(wchar_t *wchr, const char *s, mbstate_t *state)
+INLINE int mb_next_char(wchar_t *wchr, const char *s, mbstate_t *state)
 {
     int num_bytes;
     if ((num_bytes = (int)mbrtowc(wchr, s, MB_CUR_MAX, state)) < 0) {
@@ -830,7 +830,7 @@ static bool mb_std_advance_to_start(TokenStream *ts)
 
     i = mb_next_char(&wchr, ts->t, &state);
 
-    while (wchr != 0 && !iswalpha(wchr) && !isdigit(*(ts->t))) {
+    while (wchr != 0 && !iswalnum(wchr)) {
         if (isnumpunc(*ts->t) && isdigit(ts->t[1])) break;
         ts->t += i;
         i = mb_next_char(&wchr, ts->t, &state);
@@ -950,11 +950,14 @@ static Token *std_next(TokenStream *ts)
         }
         t++;
     }
-    while (isurlxatpunc(t[-1])) {
+    while (isurlxatpunc(t[-1]) && t > ts->t) {
         t--;                /* strip trailing punctuation */
     }
 
-    if (num_end == NULL || t > num_end) {
+    if (t < ts->t || (num_end != NULL && num_end < ts->t)) {
+        fprintf(stderr, "Warning: encoding error. Please check that you are using the correct locale for your input");
+        return NULL;
+    } else if (num_end == NULL || t > num_end) {
         ts->t = t;
 
         if (is_acronym) {   /* check it is one letter followed by one '.' */

data/ext/array.c

@@ -1,35 +1,35 @@
 #include "array.h"
-#include "global.h"
 #include <string.h>
 
-#define DATA_SZ sizeof(int) * 3
+#define META_CNT ARY_META_CNT
+#define DATA_SZ sizeof(int) * META_CNT
 
 void **ary_new_i(int type_size, int init_capa)
 {
-    int *ary;
+    void **ary;
     if (init_capa <= 0) {
         init_capa = ARY_INIT_CAPA;
     }
-    ary = ((int *)ecalloc(DATA_SZ + init_capa * type_size));
-    ary[0] = type_size;
-    ary[1] = init_capa;
-    return (void **)&ary[3];
+    ary = (void **)&(((int *)ecalloc(DATA_SZ + init_capa * type_size))[META_CNT]);
+    ary_type_size(ary) = type_size;
+    ary_capa(ary) = init_capa;
+    return ary;
 }
 
-__inline void ary_resize_i(void ***ary, int size)
+INLINE void ary_resize_i(void ***ary, int size)
 {
     size++;
     if (size >= ary_sz(*ary)) {
         int capa = ary_capa(*ary);
         if (size >= capa) {
-            int *ary_start = &((int *)*ary)[-3];
+            int *ary_start = &((int *)*ary)[-META_CNT];
             while (size >= capa) {
                 capa <<= 1;
             }
 
             ary_start = (int *)erealloc(ary_start,
                                         DATA_SZ + capa * ary_type_size(*ary));
-            *ary = (void **)&(ary_start[3]);
+            *ary = (void **)&(ary_start[META_CNT]);
             memset(((char *)*ary) + ary_type_size(*ary) * ary_sz(*ary), 0,
                    (capa - ary_sz(*ary)) * ary_type_size(*ary));
             ary_capa(*ary) = capa;

data/ext/array.h

@@ -1,12 +1,19 @@
 #ifndef FRT_ARRAY_H
 #define FRT_ARRAY_H
+#include "global.h"
+
+#if defined POSH_OS_SOLARIS || defined POSH_OS_SUNOS
+# define ARY_META_CNT 4
+#else
+# define ARY_META_CNT 3
+#endif
 
 #define ARY_INIT_CAPA 8
 #define ary_size(ary)      ary_sz(ary)
 #define ary_sz(ary)        (((int *)ary)[-1])
 #define ary_capa(ary)      (((int *)ary)[-2])
 #define ary_type_size(ary) (((int *)ary)[-3])
-#define ary_start(ary)     ((void **)&(((int *)ary)[-3]))
+#define ary_start(ary)     ((void **)&(((int *)ary)[-ARY_META_CNT]))
 #define ary_free(ary)      free(ary_start(ary))
 
 #define ary_new_type_capa(type, init_capa)\

data/ext/bitvector.c

@@ -193,7 +193,7 @@ const int NUM_TRAILING_ZEROS[] = {
 /*
  * This method is highly optimized, hence the loop unrolling
  */
-static __inline int bv_get_1_offset(f_u32 word)
+static INLINE int bv_get_1_offset(f_u32 word)
 {
     if (word & 0xff) {
         return NUM_TRAILING_ZEROS[word & 0xff];
@@ -360,7 +360,7 @@ unsigned long bv_hash(BitVector *bv)
     return hash;
 }
 
-static __inline void bv_recapa(BitVector *bv, int new_capa)
+static INLINE void bv_recapa(BitVector *bv, int new_capa)
 {
     if (bv->capa < new_capa) {
         REALLOC_N(bv->bits, f_u32, new_capa);

data/ext/except.c

@@ -62,7 +62,7 @@ void xpush_context(xcontext_t *context)
     context->in_finally = false;
 }
 
-static __inline void xraise_context(xcontext_t *context,
+static INLINE void xraise_context(xcontext_t *context,
                                     volatile int excode,
                                     const char *const msg)
 {

data/ext/ferret.c

@@ -192,11 +192,11 @@ frt_field(VALUE rfield)
 /*
  * Json Exportation - Loading each LazyDoc and formatting them into json
  * This code is designed to get a VERY FAST json string, the goal was speed,
- * not sexyness.
+ * not sexiness.
  * Jeremie 'ahFeel' BORDIER
  * ahFeel@rift.Fr
  */
-__inline char *
+char *
 json_concat_string(char *s, char *field)
 {
     *(s++) = '"';

data/ext/ferret.h

@@ -65,7 +65,7 @@ extern VALUE frt_hs_to_rb_ary(HashSet *hs);
 extern void *frt_rb_data_ptr(VALUE val);
 extern char * frt_field(VALUE rfield);
 extern VALUE frt_get_term(const char *field, const char *term);
-extern __inline char *json_concat_string(char *s, char *field);
+extern char *json_concat_string(char *s, char *field);
 extern char *rs2s(VALUE rstr);
 extern char *nstrdup(VALUE rstr);
 #define Frt_Make_Struct(klass)\

data/ext/fs_store.c

@@ -51,7 +51,7 @@ static void fs_touch(Store *store, char *filename)
     int f;
     char path[MAX_FILE_PATH];
     join_path(path, store->dir.path, filename);
-    if ((f = creat(path, S_IRUSR | S_IWUSR)) == 0) {
+    if ((f = creat(path, store->file_mode)) == 0) {
         RAISE(IO_ERROR, "couldn't create file %s: <%s>", path,
               strerror(errno));
     }
@@ -257,7 +257,7 @@ static OutStream *fs_new_output(Store *store, const char *filename)
 {
     char path[MAX_FILE_PATH];
     int fd = open(join_path(path, store->dir.path, filename),
-                  O_WRONLY | O_CREAT | O_BINARY, S_IRUSR | S_IWUSR);
+                  O_WRONLY | O_CREAT | O_BINARY, store->file_mode);
     OutStream *os;
     if (fd < 0) {
         RAISE(IO_ERROR, "couldn't create OutStream %s: <%s>",
@@ -430,8 +430,19 @@ static void fs_close_i(Store *store)
 
 static Store *fs_store_new(const char *pathname)
 {
+    struct stat stt;
     Store *new_store = store_new();
 
+    new_store->file_mode = S_IRUSR | S_IWUSR;
+#ifndef POSH_OS_WIN32
+    if (!stat(pathname, &stt) && stt.st_gid == getgid()) {
+        if (stt.st_mode & S_IWGRP) {
+            umask(S_IWOTH);
+        }
+        new_store->file_mode |= stt.st_mode & (S_IRGRP | S_IWGRP);
+    }
+#endif
+
     new_store->dir.path      = estrdup(pathname);
     new_store->touch         = &fs_touch;
     new_store->exists        = &fs_exists;

data/ext/global.c

@@ -11,22 +11,22 @@ const char *EMPTY_STRING = "";
 
 bool x_do_logging = false;
 
-__inline int min3(int a, int b, int c)
+INLINE int min3(int a, int b, int c)
 {
     return MIN3(a, b, c);
 }
 
-__inline int min2(int a, int b)
+INLINE int min2(int a, int b)
 {
     return MIN(a, b);
 }
 
-__inline int max3(int a, int b, int c)
+INLINE int max3(int a, int b, int c)
 {
     return MAX3(a, b, c);
 }
 
-__inline int max2(int a, int b)
+INLINE int max2(int a, int b)
 {
     return MAX(a, b);
 }

data/ext/global.h

@@ -11,6 +11,12 @@
 #define MAX_WORD_SIZE 255
 #define MAX_FILE_PATH 1024
 
+#if defined(__GNUC__)
+# define INLINE __inline__
+#else
+# define INLINE
+#endif
+
 typedef void (*free_ft)(void *key);
 
 #define NELEMS(array) ((int)(sizeof(array)/sizeof(array[0])))

data/ext/hash.c

@@ -61,7 +61,7 @@ typedef HashEntry *(*lookup_ft)(struct HashTable *ht, register const void *key);
  * @param ht the HashTable to do the fast lookup in
  * @param the hashkey we are looking for
  */
-static __inline HashEntry *h_resize_lookup(HashTable *ht,
+static INLINE HashEntry *h_resize_lookup(HashTable *ht,
                                            register const unsigned long hash)
 {
     register unsigned long perturb;

data/ext/helper.c

@@ -1,6 +1,6 @@
 #include "helper.h"
 
-__inline int hlp_string_diff(register const char *const s1,
+int hlp_string_diff(register const char *const s1,
                            register const char *const s2)
 {
     register int i = 0;

data/ext/helper.h

@@ -3,7 +3,7 @@
 
 #include "config.h"
 
-extern __inline int hlp_string_diff(register const char *const s1,
+extern int hlp_string_diff(register const char *const s1,
                                   register const char *const s2);
 extern f_i32 float2int(float f);
 extern float int2float(f_i32 i32);

data/ext/index.c

@@ -206,7 +206,7 @@ HashTable *co_hash_create()
  *
  ****************************************************************************/
 
-__inline void fi_set_store(FieldInfo *fi, int store)
+INLINE void fi_set_store(FieldInfo *fi, int store)
 {
     switch (store) {
         case STORE_NO:
@@ -220,7 +220,7 @@ __inline void fi_set_store(FieldInfo *fi, int store)
     }
 }
 
-__inline void fi_set_index(FieldInfo *fi, int index)
+INLINE void fi_set_index(FieldInfo *fi, int index)
 {
     switch (index) {
         case INDEX_NO:
@@ -241,7 +241,7 @@ __inline void fi_set_index(FieldInfo *fi, int index)
     }
 }
 
-__inline void fi_set_term_vector(FieldInfo *fi, int term_vector)
+INLINE void fi_set_term_vector(FieldInfo *fi, int term_vector)
 {
     switch (term_vector) {
         case TERM_VECTOR_NO:
@@ -466,7 +466,7 @@ static const char *index_str[] = {
     "",
     ":untokenized_omit_norms",
     "",
-    ":yes_omit_norms"
+    ":omit_norms"
 };
 
 static const char *fi_index_str(FieldInfo *fi)
@@ -1375,7 +1375,8 @@ LazyDoc *fr_get_lazy_doc(FieldsReader *fr, int doc_num)
     lazy_doc = lazy_doc_new(stored_cnt, fdt_in);
 
     for (i = 0; i < stored_cnt; i++) {
-        int start = 0, end, data_cnt;
+        off_t start = 0, end;
+        int data_cnt;
         field_num = is_read_vint(fdt_in);
         fi = fr->fis->fields[field_num];
         data_cnt = is_read_vint(fdt_in);
@@ -1449,7 +1450,7 @@ TermVector *fr_read_term_vector(FieldsReader *fr, int field_num)
         if (store_offsets) {
             int num_positions = tv->offset_cnt = is_read_vint(fdt_in);
             Offset *offsets = tv->offsets = ALLOC_N(Offset, num_positions);
-            int offset = 0;
+            off_t offset = 0;
             for (i = 0; i < num_positions; i++) {
                 offsets[i].start = offset += is_read_vint(fdt_in);
                 offsets[i].end = offset += is_read_vint(fdt_in);
@@ -1567,7 +1568,7 @@ void fw_close(FieldsWriter *fw)
     free(fw);
 }
 
-static __inline void save_data(OutStream *fdt_out, char *data, int dlen)
+static INLINE void save_data(OutStream *fdt_out, char *data, int dlen)
 {
     os_write_vint(fdt_out, dlen);
     os_write_bytes(fdt_out, (uchar *)data, dlen);
@@ -1683,8 +1684,8 @@ void fw_add_postings(FieldsWriter *fw,
         int last_end = 0;
         os_write_vint(fdt_out, offset_count);  /* write shared prefix length */
         for (i = 0; i < offset_count; i++) {
-            int start = offsets[i].start;
-            int end = offsets[i].end;
+            off_t start = offsets[i].start;
+            off_t end = offsets[i].end;
             os_write_vint(fdt_out, start - last_end);
             os_write_vint(fdt_out, end - start);
             last_end = end;
@@ -1863,7 +1864,7 @@ void sfi_close(SegmentFieldIndex *sfi)
  * SegmentTermEnum
  ****************************************************************************/
 
-static __inline int term_read(char *buf, InStream *is)
+static INLINE int term_read(char *buf, InStream *is)
 {
     int start = (int)is_read_vint(is);
     int length = (int)is_read_vint(is);
@@ -2297,7 +2298,7 @@ TermInfosReader *tir_open(Store *store,
     return tir;
 }
 
-static __inline TermEnum *tir_enum(TermInfosReader *tir)
+static INLINE TermEnum *tir_enum(TermInfosReader *tir)
 {
     TermEnum *te;
     if (NULL == (te = thread_getspecific(tir->thread_te))) {
@@ -2423,7 +2424,7 @@ TermInfosWriter *tiw_open(Store *store,
     return tiw;
 }
 
-static __inline void tw_write_term(TermWriter *tw,
+static INLINE void tw_write_term(TermWriter *tw,
                                  OutStream *os,
                                  const char *term,
                                  int term_len)
@@ -2499,7 +2500,7 @@ void tiw_add(TermInfosWriter *tiw,
     tw_add(tiw->tis_writer, term, term_len, ti, tiw->skip_interval);
 }
 
-static __inline void tw_reset(TermWriter *tw)
+static INLINE void tw_reset(TermWriter *tw)
 {
     tw->counter = 0;
     tw->last_term = EMPTY_STRING;
@@ -3838,7 +3839,7 @@ void ir_add_cache(IndexReader *ir)
 
 bool ir_is_latest(IndexReader *ir)
 {
-    return (sis_read_current_version(ir->store) == ir->sis->version);
+    return ir->is_latest_i(ir);
 }
 
 /****************************************************************************
@@ -3919,7 +3920,7 @@ typedef struct SegmentReader {
 #define SR(ir) ((SegmentReader *)(ir))
 #define SR_SIZE(ir) (SR(ir)->fr->size)
 
-static __inline FieldsReader *sr_fr(SegmentReader *sr)
+static INLINE FieldsReader *sr_fr(SegmentReader *sr)
 {
     FieldsReader *fr;
 
@@ -3931,12 +3932,12 @@ static __inline FieldsReader *sr_fr(SegmentReader *sr)
     return fr;
 }
 
-static __inline bool sr_is_deleted_i(SegmentReader *sr, int doc_num)
+static INLINE bool sr_is_deleted_i(SegmentReader *sr, int doc_num)
 {
     return (NULL != sr->deleted_docs && bv_get(sr->deleted_docs, doc_num));
 }
 
-static __inline void sr_get_norms_into_i(SegmentReader *sr, int field_num,
+static INLINE void sr_get_norms_into_i(SegmentReader *sr, int field_num,
                                        uchar *buf)
 {
     Norm *norm = h_get_int(sr->norms, field_num);
@@ -3955,7 +3956,7 @@ static __inline void sr_get_norms_into_i(SegmentReader *sr, int field_num,
     }
 }
 
-static __inline uchar *sr_get_norms_i(SegmentReader *sr, int field_num)
+static INLINE uchar *sr_get_norms_i(SegmentReader *sr, int field_num)
 {
     Norm *norm = h_get_int(sr->norms, field_num);
     if (NULL == norm) {                           /* not an indexed field */
@@ -4043,6 +4044,11 @@ static BitVector *bv_read(Store *store, char *name)
     return bv;
 }
 
+static bool sr_is_latest_i(IndexReader *ir)
+{
+    return (sis_read_current_version(ir->store) == ir->sis->version);
+}
+
 static void sr_commit_i(IndexReader *ir)
 {
     SegmentInfo *si = SR(ir)->si;
@@ -4283,6 +4289,7 @@ static IndexReader *sr_setup_i(SegmentReader *sr)
     ir->delete_doc_i        = &sr_delete_doc_i;
     ir->undelete_all_i      = &sr_undelete_all_i;
     ir->set_deleter_i       = &sr_set_deleter_i;
+    ir->is_latest_i         = &sr_is_latest_i;
     ir->commit_i            = &sr_commit_i;
     ir->close_i             = &sr_close_i;
 
@@ -4570,6 +4577,18 @@ static void mr_set_deleter_i(IndexReader *ir, Deleter *deleter)
     }
 }
 
+static bool mr_is_latest_i(IndexReader *ir)
+{
+    int i;
+    const int mr_reader_cnt = MR(ir)->r_cnt;
+    for (i = 0; i < mr_reader_cnt; i++) {
+        if (!ir_is_latest(MR(ir)->sub_readers[i])) {
+            return false;
+        }
+    }
+    return true;
+}
+
 static void mr_commit_i(IndexReader *ir)
 {
     int i;
@@ -4639,6 +4658,7 @@ static IndexReader *mr_new(IndexReader **sub_readers, const int r_cnt)
     ir->delete_doc_i        = &mr_delete_doc_i;
     ir->undelete_all_i      = &mr_undelete_all_i;
     ir->set_deleter_i       = &mr_set_deleter_i;
+    ir->is_latest_i         = &mr_is_latest_i;
     ir->commit_i            = &mr_commit_i;
     ir->close_i             = &mr_close_i;
 
@@ -4799,7 +4819,7 @@ IndexReader *ir_open(Store *store)
  *
  ****************************************************************************/
 
-Offset *offset_new(int start, int end)
+Offset *offset_new(off_t start, off_t end)
 {
     Offset *offset = ALLOC(Offset);
     offset->start = start;
@@ -5177,7 +5197,7 @@ static void dw_add_posting(MemoryPool *mp,
     }
 }
 
-static __inline void dw_add_offsets(DocWriter *dw, int pos, int start, int end)
+static INLINE void dw_add_offsets(DocWriter *dw, int pos, int start, int end)
 {
     if (pos >= dw->offsets_capa) {
         int old_capa = dw->offsets_capa;
@@ -5204,6 +5224,7 @@ HashTable *dw_invert_field(DocWriter *dw,
     int doc_num = dw->doc_num;
     int i;
     const int df_size = df->size;
+    off_t start_offset = 0;
 
     if (fld_inv->is_tokenized) {
         Token *tk;
@@ -5217,7 +5238,9 @@ HashTable *dw_invert_field(DocWriter *dw,
                     pos += tk->pos_inc;
                     dw_add_posting(mp, curr_plists, fld_plists, doc_num,
                                    tk->text, tk->len, pos);
-                    dw_add_offsets(dw, pos, tk->start, tk->end);
+                    dw_add_offsets(dw, pos,
+                                   start_offset + tk->start,
+                                   start_offset + tk->end);
                     if (num_terms++ >= dw->max_field_length) {
                         break;
                     }
@@ -5234,6 +5257,7 @@ HashTable *dw_invert_field(DocWriter *dw,
                 }
             }
             ts_deref(ts);
+            start_offset += df->lengths[i] + 1;
         }
         fld_inv->length = num_terms;
     }
@@ -5250,8 +5274,10 @@ HashTable *dw_invert_field(DocWriter *dw,
             dw_add_posting(mp, curr_plists, fld_plists, doc_num, data_ptr,
                            len, i);
             if (store_offsets) {
-                dw_add_offsets(dw, i, 0, df->lengths[i]);
+                dw_add_offsets(dw, i, start_offset,
+                               start_offset + df->lengths[i]);
             }
+            start_offset += df->lengths[i] + 1;
         }
         fld_inv->length = i;
     }