ferret 0.10.14 → 0.11.0

data/ext/r_analysis.c

@@ -73,7 +73,7 @@ get_stopwords(VALUE rstop_words)
     stop_words[len] = NULL;
     for (i = 0; i < len; i++) {
         rstr = rb_obj_as_string(RARRAY(rstop_words)->ptr[i]);
-        stop_words[i] = RSTRING(rstr)->ptr;
+        stop_words[i] = rs2s(rstr);
     }
     return stop_words;
 }
@@ -131,7 +131,7 @@ frt_set_token(Token *tk, VALUE rt)
     if (rt == Qnil) return NULL;
 
     Data_Get_Struct(rt, RToken, rtk);
-    tk_set(tk, RSTRING(rtk->text)->ptr, RSTRING(rtk->text)->len,
+    tk_set(tk, rs2s(rtk->text), RSTRING(rtk->text)->len,
            rtk->start, rtk->end, rtk->pos_inc);
     return tk;
 }
@@ -216,7 +216,7 @@ frt_token_cmp(VALUE self, VALUE rother)
         } else if (token->end < other->end) {
             cmp = -1;
         } else {
-            cmp = strcmp(RSTRING(token->text)->ptr, RSTRING(other->text)->ptr);
+            cmp = strcmp(rs2s(token->text), rs2s(other->text));
         }
     }
     return INT2FIX(cmp);
@@ -372,7 +372,7 @@ frt_token_to_s(VALUE self)
     char *buf;
     GET_TK(token, self);
     buf = alloca(RSTRING(token->text)->len + 80);
-    sprintf(buf, "token[\"%s\":%d:%d:%d]", RSTRING(token->text)->ptr,
+    sprintf(buf, "token[\"%s\":%d:%d:%d]", rs2s(token->text),
             token->start, token->end, token->pos_inc);
     return rb_str_new2(buf);
 }
@@ -427,7 +427,7 @@ static inline VALUE
 get_wrapped_ts(VALUE self, VALUE rstr, TokenStream *ts)
 {
     StringValue(rstr);
-    ts->reset(ts, RSTRING(rstr)->ptr);
+    ts->reset(ts, rs2s(rstr));
     Frt_Wrap_Struct(self, &frt_ts_mark, &frt_ts_free, ts);
     object_add(&ts->text, rstr);
     object_add(ts, self);
@@ -449,7 +449,7 @@ frt_ts_set_text(VALUE self, VALUE rtext)
     TokenStream *ts; 
     Data_Get_Struct(self, TokenStream, ts);
     StringValue(rtext);
-    ts->reset(ts, RSTRING(rtext)->ptr);
+    ts->reset(ts, rs2s(rtext));
     object_set(&ts->text, rtext);
 
     return rtext;
@@ -703,7 +703,7 @@ rets_next(TokenStream *ts)
     } else {
         VALUE rtok = rb_str_new(rtext->ptr + beg, end - beg);
         rtok = rb_funcall(RETS(ts)->proc, id_call, 1, rtok);
-        return tk_set(&(CachedTS(ts)->token), RSTRING(rtok)->ptr,
+        return tk_set(&(CachedTS(ts)->token), rs2s(rtok),
                       RSTRING(rtok)->len, beg, end, 1);
     }
 }
@@ -977,7 +977,7 @@ static __inline void frt_add_mapping_i(TokenStream *mf, VALUE from, char *to)
 {
     switch (TYPE(from)) {
         case T_STRING:
-            mapping_filter_add(mf, RSTRING(from)->ptr, to);
+            mapping_filter_add(mf, rs2s(from), to);
             break;
         case T_SYMBOL:
             mapping_filter_add(mf, rb_id2name(SYM2ID(from)), to);
@@ -985,7 +985,7 @@ static __inline void frt_add_mapping_i(TokenStream *mf, VALUE from, char *to)
         default:
             rb_raise(rb_eArgError,
                      "cannot map from %s with MappingFilter",
-                     RSTRING(rb_obj_as_string(from))->ptr);
+                     rs2s(rb_obj_as_string(from)));
             break;
     }
 }
@@ -999,7 +999,7 @@ static int frt_add_mappings_i(VALUE key, VALUE value, VALUE arg)
         char *to;
         switch (TYPE(value)) {
             case T_STRING:
-                to = RSTRING(value)->ptr;
+                to = rs2s(value);
                 break;
             case T_SYMBOL:
                 to = rb_id2name(SYM2ID(value));
@@ -1007,7 +1007,7 @@ static int frt_add_mappings_i(VALUE key, VALUE value, VALUE arg)
             default:
                 rb_raise(rb_eArgError,
                          "cannot map to %s with MappingFilter",
-                         RSTRING(rb_obj_as_string(key))->ptr);
+                         rs2s(rb_obj_as_string(key)));
                 break;
         }
         if (TYPE(key) == T_ARRAY) {
@@ -1088,8 +1088,8 @@ frt_stem_filter_init(int argc, VALUE *argv, VALUE self)
     rb_scan_args(argc, argv, "12", &rsub_ts, &ralgorithm, &rcharenc);
     ts = frt_get_cwrapped_rts(rsub_ts);
     switch (argc) {
-        case 3: charenc = RSTRING(rb_obj_as_string(rcharenc))->ptr;
-        case 2: algorithm = RSTRING(rb_obj_as_string(ralgorithm))->ptr;
+        case 3: charenc = rs2s(rb_obj_as_string(rcharenc));
+        case 2: algorithm = rs2s(rb_obj_as_string(ralgorithm));
     }
     ts = stem_filter_new(ts, algorithm, charenc);
     object_add(&(TkFilt(ts)->sub_ts), rsub_ts);
@@ -1198,7 +1198,7 @@ frt_analyzer_token_stream(VALUE self, VALUE rfield, VALUE rstring)
 
     StringValue(rstring);
 
-    ts = a_get_ts(a, frt_field(rfield), RSTRING(rstring)->ptr);
+    ts = a_get_ts(a, frt_field(rfield), rs2s(rstring));
 
     /* Make sure that there is no entry already */
     object_set(&ts->text, rstring);
@@ -1503,7 +1503,7 @@ frt_re_analyzer_token_stream(VALUE self, VALUE rfield, VALUE rtext)
 
     StringValue(rtext);
 
-    ts = a_get_ts(a, frt_field(rfield), RSTRING(rtext)->ptr);
+    ts = a_get_ts(a, frt_field(rfield), rs2s(rtext));
 
     /* Make sure that there is no entry already */
     object_set(&ts->text, rtext);
@@ -1546,7 +1546,7 @@ static VALUE frt_get_locale(VALUE self, VALUE locale)
  */
 static VALUE frt_set_locale(VALUE self, VALUE locale)
 {
-    char *l = ((locale == Qnil) ? NULL : RSTRING(rb_obj_as_string(locale))->ptr);
+    char *l = ((locale == Qnil) ? NULL : rs2s(rb_obj_as_string(locale)));
     frt_locale = setlocale(LC_CTYPE, l);
     return frt_locale ? rb_str_new2(frt_locale) : Qnil;
 }

data/ext/r_index.c

@@ -479,7 +479,7 @@ frt_fis_get(VALUE self, VALUE ridx)
             break;
         default:
             rb_raise(rb_eArgError, "Can't index FieldInfos with %s",
-                     RSTRING(rb_obj_as_string(ridx))->ptr);
+                     rs2s(rb_obj_as_string(ridx)));
             break;
     }
     return rfi;
@@ -599,7 +599,7 @@ frt_fis_create_index(VALUE self, VALUE rdir)
     } else {
         StringValue(rdir);
         frt_create_dir(rdir);
-        store = open_fs_store(RSTRING(rdir)->ptr);
+        store = open_fs_store(rs2s(rdir));
     }
     index_create(store, fis);
     store_deref(store);
@@ -753,6 +753,7 @@ frt_te_each(VALUE self)
     int term_cnt = 0;
     VALUE vals = rb_ary_new2(2);
     RARRAY(vals)->len = 2;
+    rb_mem_clear(RARRAY(vals)->ptr, 2);
 
 
     /* each is being called so there will be no current term */
@@ -947,6 +948,7 @@ frt_tde_each(VALUE self)
     TermDocEnum *tde = (TermDocEnum *)DATA_PTR(self);
     VALUE vals = rb_ary_new2(2);
     RARRAY(vals)->len = 2;
+    rb_mem_clear(RARRAY(vals)->ptr, 2);
 
     while (tde->next(tde)) {
         doc_cnt++;
@@ -1038,10 +1040,10 @@ frt_get_tv_term(TVTerm *tv_term)
         int *positions = tv_term->positions;
         rpositions = rb_ary_new2(freq);
         rpos = RARRAY(rpositions)->ptr;
-        RARRAY(rpositions)->len = freq;
         for (i = 0; i < freq; i++) {
             rpos[i] = INT2FIX(positions[i]);
         }
+        RARRAY(rpositions)->len = freq;
     }
     return rb_struct_new(cTVTerm, rtext, rpositions, NULL);
 }
@@ -1064,10 +1066,10 @@ frt_get_tv(TermVector *tv)
     rfield = ID2SYM(rb_intern(tv->field));
 
     rterms = rb_ary_new2(t_cnt);
-    RARRAY(rterms)->len = t_cnt;
     rts = RARRAY(rterms)->ptr;
     for (i = 0; i < t_cnt; i++) {
         rts[i] = frt_get_tv_term(&terms[i]);
+        RARRAY(rterms)->len++;
     }
 
     if (tv->offsets) {
@@ -1075,9 +1077,9 @@ frt_get_tv(TermVector *tv)
         Offset *offsets = tv->offsets;
         roffsets = rb_ary_new2(o_cnt);
         ros = RARRAY(roffsets)->ptr;
-        RARRAY(roffsets)->len = o_cnt;
         for (i = 0; i < o_cnt; i++) {
             ros[i] = frt_get_tv_offsets(&offsets[i]);
+            RARRAY(roffsets)->len++;
         }
     }
 
@@ -1167,7 +1169,7 @@ frt_iw_init(int argc, VALUE *argv, VALUE self)
         } else if ((rval = rb_hash_aref(roptions, sym_path)) != Qnil) {
             StringValue(rval);
             frt_create_dir(rval);
-            store = open_fs_store(RSTRING(rval)->ptr);
+            store = open_fs_store(rs2s(rval));
             DEREF(store);
         }
         
@@ -1203,7 +1205,7 @@ frt_iw_init(int argc, VALUE *argv, VALUE self)
         store = open_ram_store();
         DEREF(store);
     }
-    if (!create && create_if_missing && !store->exists(store, "segments")) {
+    if (!create && create_if_missing && !store->exists(store, "segments.gen")) {
         create = true;
     }
     if (create) {
@@ -1246,14 +1248,6 @@ frt_iw_get_doc_count(VALUE self)
     return INT2FIX(iw_doc_count(iw));
 }
 
-static char *
-nstrdup(const char *str, int len)
-{
-    char *new = ALLOC_N(char, len + 1);
-    memcpy(new, str, len + 1);
-    return new;
-}
-
 static int
 frt_hash_to_doc_i(VALUE key, VALUE value, VALUE arg)
 {
@@ -1266,7 +1260,7 @@ frt_hash_to_doc_i(VALUE key, VALUE value, VALUE arg)
         DocField *df;
         switch (TYPE(key)) {
             case T_STRING:
-                field = RSTRING(key)->ptr;
+                field = rs2s(key);
                 break;
             case T_SYMBOL:
                 field = rb_id2name(SYM2ID(key));
@@ -1274,7 +1268,7 @@ frt_hash_to_doc_i(VALUE key, VALUE value, VALUE arg)
             default:
                 rb_raise(rb_eArgError,
                          "%s cannot be a key to a field. Field keys must "
-                         " be symbols.", RSTRING(rb_obj_as_string(key))->ptr);
+                         " be symbols.", rs2s(rb_obj_as_string(key)));
                 break;
         }
         if (NULL == (df = doc_get_field(doc, field))) {
@@ -1290,22 +1284,17 @@ frt_hash_to_doc_i(VALUE key, VALUE value, VALUE arg)
                     df->destroy_data = true;
                     for (i = 0; i < RARRAY(value)->len; i++) {
                         val = rb_obj_as_string(RARRAY(value)->ptr[i]);
-                        df_add_data_len(df,
-                                        nstrdup(RSTRING(val)->ptr,
-                                                RSTRING(val)->len),
-                                        RSTRING(val)->len);
+                        df_add_data_len(df, nstrdup(val), RSTRING(val)->len);
                     }
                 }
                 break;
             case T_STRING:
-                df_add_data_len(df, RSTRING(value)->ptr, RSTRING(value)->len);
+                df_add_data_len(df, rs2s(value), RSTRING(value)->len);
                 break;
             default:
                 val = rb_obj_as_string(value);
                 df->destroy_data = true;
-                df_add_data_len(df,
-                                nstrdup(RSTRING(val)->ptr, RSTRING(val)->len),
-                                RSTRING(val)->len);
+                df_add_data_len(df, nstrdup(val), RSTRING(val)->len);
                 break;
         }
         doc_add_field(doc, df);
@@ -1335,10 +1324,7 @@ frt_get_doc(VALUE rdoc)
                 df->destroy_data = true;
                 for (i = 0; i < RARRAY(rdoc)->len; i++) {
                     val = rb_obj_as_string(RARRAY(rdoc)->ptr[i]);
-                    df_add_data_len(df,
-                                    nstrdup(RSTRING(val)->ptr,
-                                            RSTRING(val)->len),
-                                    RSTRING(val)->len);
+                    df_add_data_len(df, nstrdup(val), RSTRING(val)->len);
                 }
                 doc_add_field(doc, df);
             }
@@ -1348,15 +1334,13 @@ frt_get_doc(VALUE rdoc)
             doc_add_field(doc, df);
             break;
         case T_STRING:
-            df = df_add_data_len(df_new("content"), RSTRING(rdoc)->ptr,
+            df = df_add_data_len(df_new("content"), rs2s(rdoc),
                                  RSTRING(rdoc)->len);
             doc_add_field(doc, df);
             break;
         default:
             val = rb_obj_as_string(rdoc);
-            df = df_add_data_len(df_new("content"),
-                                 nstrdup(RSTRING(val)->ptr,
-                                         RSTRING(val)->len),
+            df = df_add_data_len(df_new("content"), nstrdup(val),
                                  RSTRING(val)->len);
             df->destroy_data = true;
             doc_add_field(doc, df);
@@ -1512,6 +1496,19 @@ frt_iw_set_analyzer(VALUE self, VALUE ranalyzer)
     return ranalyzer;
 }
 
+/*
+ *  call-seq:
+ *     index_writer.version -> int
+ *
+ *  Returns the current version of the index writer.
+ */
+static VALUE
+frt_iw_version(VALUE self)
+{
+    IndexWriter *iw = (IndexWriter *)DATA_PTR(self);
+    return ULL2NUM(iw->sis->version);
+}
+
 /*
  *  call-seq:
  *     iw.chunk_size -> number
@@ -1804,7 +1801,7 @@ frt_lzd_default(VALUE self, VALUE rkey)
     char *field = NULL;
     switch (TYPE(rkey)) {
         case T_STRING:
-            field = RSTRING(rkey)->ptr;
+            field = rs2s(rkey);
             rkey = ID2SYM(rb_intern(field));
             break;
         case T_SYMBOL:
@@ -1813,7 +1810,7 @@ frt_lzd_default(VALUE self, VALUE rkey)
         default:
             rb_raise(rb_eArgError,
                      "%s cannot be a key to a field. Field keys must "
-                     " be symbols.", RSTRING(rb_obj_as_string(rkey))->ptr);
+                     " be symbols.", rs2s(rb_obj_as_string(rkey)));
             break;
     }
     return frt_lazy_df_load(self, rkey, h_get(lazy_doc->field_dict, field));
@@ -1960,19 +1957,19 @@ frt_ir_init(VALUE self, VALUE rdir)
                                  "be created from other IndexReaders, "
                                  "Directory objects or file-system paths. "
                                  "Not %s",
-                                 RSTRING(rb_obj_as_string(rdir))->ptr);
+                                 rs2s(rb_obj_as_string(rdir)));
                     }
                     break;
                 case T_STRING:
                     frt_create_dir(rdir);
-                    store = open_fs_store(RSTRING(rdir)->ptr);
+                    store = open_fs_store(rs2s(rdir));
                     DEREF(store);
                     break;
                 default:
                     rb_raise(rb_eArgError, "%s isn't a valid directory "
                              "argument. You should use either a String or "
                              "a Directory",
-                             RSTRING(rb_obj_as_string(rdir))->ptr);
+                             rs2s(rb_obj_as_string(rdir)));
                     break;
             }
             sub_readers[i] = ir_open(store);
@@ -1986,13 +1983,13 @@ frt_ir_init(VALUE self, VALUE rdir)
                 break;
             case T_STRING:
                 frt_create_dir(rdir);
-                store = open_fs_store(RSTRING(rdir)->ptr);
+                store = open_fs_store(rs2s(rdir));
                 DEREF(store);
                 break;
             default:
                 rb_raise(rb_eArgError, "%s isn't a valid directory argument. "
                          "You should use either a String or a Directory",
-                         RSTRING(rb_obj_as_string(rdir))->ptr);
+                         rs2s(rb_obj_as_string(rdir)));
                 break;
         }
         ir = ir_open(store);
@@ -2070,7 +2067,7 @@ frt_ir_get_norms_into(VALUE self, VALUE rfield, VALUE rnorms, VALUE roffset)
     }
 
     ir_get_norms_into(ir, frt_field(rfield),
-                      (uchar *)RSTRING(rnorms)->ptr + offset);
+                      (uchar *)rs2s(rnorms) + offset);
     return rnorms;
 }
 
@@ -2210,8 +2207,8 @@ frt_get_doc_range(IndexReader *ir, int pos, int len, int max)
     ary = rb_ary_new2(len);
     for (i = 0; i < len; i++) {
         RARRAY(ary)->ptr[i] = frt_get_lazy_doc(ir->get_lazy_doc(ir, i + pos));
+        RARRAY(ary)->len++;
     }
-    RARRAY(ary)->len = len;
     return ary;
 }
 
@@ -2525,7 +2522,7 @@ static VALUE
 frt_ir_version(VALUE self)
 {
     IndexReader *ir = (IndexReader *)DATA_PTR(self);
-    return INT2FIX(ir->sis->version);
+    return ULL2NUM(ir->sis->version);
 }
 
 /****************************************************************************
@@ -3126,6 +3123,7 @@ Init_IndexWriter(void)
     rb_define_method(cIndexWriter, "field_infos",   frt_iw_field_infos, 0);
     rb_define_method(cIndexWriter, "analyzer",      frt_iw_get_analyzer, 0);
     rb_define_method(cIndexWriter, "analyzer=",     frt_iw_set_analyzer, 1);
+    rb_define_method(cIndexWriter, "version",       frt_iw_version, 0);
 
     rb_define_method(cIndexWriter, "chunk_size", 
                      frt_iw_get_chunk_size, 0);

data/ext/r_qparser.c

@@ -54,15 +54,15 @@ frt_get_fields(VALUE rfields)
         int i;
         for (i = 0; i < RARRAY(rfields)->len; i++) {
             rval = rb_obj_as_string(RARRAY(rfields)->ptr[i]);
-            hs_add(fields, estrdup(RSTRING(rval)->ptr));
+            hs_add(fields, nstrdup(rval));
         }
     } else {
         rval = rb_obj_as_string(rfields);
-        if (strcmp("*", RSTRING(rval)->ptr) == 0) {
+        if (strcmp("*", rs2s(rval)) == 0) {
             hs_destroy(fields);
             fields = NULL;
         } else {
-            s = str = estrdup(RSTRING(rval)->ptr);
+            s = str = nstrdup(rval);
             while ((p = strchr(s, '|')) != '\0') {
                 *p = '\0';
                 hs_add(fields, estrdup(s));
@@ -84,38 +84,38 @@ frt_get_fields(VALUE rfields)
  *
  *  === Options
  *
- *  :default_field::        Default: "*" (all fields). The default field to
- *                          search when no field is specified in the search
- *                          string. It can also be an array of fields.
- *  :analyzer::             Default: StandardAnalyzer. Analyzer used by the
- *                          query parser to parse query terms
- *  :wild_card_downcase::   Default: true. Specifies whether wild-card queries
- *                          should be downcased or not since they are not
- *                          passed through the parser
- *  :fields::               Default: []. Lets the query parser know what
- *                          fields are available for searching, particularly
- *                          when the "*" is specified as the search field
- *  :tokenized_fields::     Default: :fields. Lets the query parser know which
- *                          fields are tokenized so it knows which fields to
- *                          run the analyzer over.
- *  :validate_fields::      Default: false. Set to true if you want an
- *                          exception to be raised if there is an attempt to
- *                          search a non-existent field
- *  :or_default::           Default: true. Use "OR" as the default boolean
- *                          operator
- *  :default_slop::         Default: 0. Default slop to use in PhraseQuery
- *  :handle_parser_errors:: Default: true. QueryParser will quietly handle all
- *                          parsing errors internally. If you'd like to handle
- *                          them yourself, set this parameter to false.
- *  :clean_string::         Default: true. QueryParser will do a quick
- *                          once-over the query string make sure that quotes
- *                          and brackets match up and special characters are
- *                          escaped
- *  :max_clauses::          Default: 512. the maximum number of clauses
- *                          allowed in boolean queries and the maximum number
- *                          of terms allowed in multi, prefix, wild-card or
- *                          fuzzy queries when those queries are generated by
- *                          rewriting other queries
+ *  :default_field::       Default: "*" (all fields). The default field to
+ *                         search when no field is specified in the search
+ *                         string. It can also be an array of fields.
+ *  :analyzer::            Default: StandardAnalyzer. Analyzer used by the
+ *                         query parser to parse query terms
+ *  :wild_card_downcase::  Default: true. Specifies whether wild-card queries
+ *                         and range queries should be downcased or not since
+ *                         they are not passed through the parser
+ *  :fields::              Default: []. Lets the query parser know what
+ *                         fields are available for searching, particularly
+ *                         when the "*" is specified as the search field
+ *  :tokenized_fields::    Default: :fields. Lets the query parser know which
+ *                         fields are tokenized so it knows which fields to
+ *                         run the analyzer over.
+ *  :validate_fields::     Default: false. Set to true if you want an
+ *                         exception to be raised if there is an attempt to
+ *                         search a non-existent field
+ *  :or_default::          Default: true. Use "OR" as the default boolean
+ *                         operator
+ *  :default_slop::        Default: 0. Default slop to use in PhraseQuery
+ *  :handle_parse_errors:: Default: true. QueryParser will quietly handle all
+ *                         parsing errors internally. If you'd like to handle
+ *                         them yourself, set this parameter to false.
+ *  :clean_string::        Default: true. QueryParser will do a quick
+ *                         once-over the query string make sure that quotes
+ *                         and brackets match up and special characters are
+ *                         escaped
+ *  :max_clauses::         Default: 512. the maximum number of clauses
+ *                         allowed in boolean queries and the maximum number
+ *                         of terms allowed in multi, prefix, wild-card or
+ *                         fuzzy queries when those queries are generated by
+ *                         rewriting other queries
  */                   
 static VALUE
 frt_qp_init(int argc, VALUE *argv, VALUE self)
@@ -163,6 +163,7 @@ frt_qp_init(int argc, VALUE *argv, VALUE self)
     qp = qp_new(all_fields, def_fields, tkz_fields, analyzer);
     qp->allow_any_fields = true;
     qp->clean_str = true;
+    qp->handle_parse_errors = true;
     /* handle options */
     if (argc > 0) {
         if (Qnil != (rval = rb_hash_aref(roptions, sym_handle_parse_errors))) {
@@ -208,7 +209,7 @@ frt_qp_parse(VALUE self, VALUE rstr)
     GET_QP;
     rstr = rb_obj_as_string(rstr);
     TRY
-        rq = frt_get_q(qp_parse(qp, RSTRING(rstr)->ptr));
+        rq = frt_get_q(qp_parse(qp, rs2s(rstr)));
         break;
     default:
         msg = xcontext.msg;