isomorfeus-ferret 0.14.1 → 0.14.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5818fce6d84b9bd4814be3bbed270127e05297dcf85adeebc495c8f334430d88
-  data.tar.gz: 77c9c3246c7777947084b47620d3aeeeb9eb76d7b0a17a4d30a37a38547a54da
+  metadata.gz: 2edd7c4bc27bf507a9003988136eaaa475fed8943eb4fe6a083995d02a9015c1
+  data.tar.gz: d779442eb4d5c2ce83490c0a0a425230701d21fd535f4f9b30970752557888ac
 SHA512:
-  metadata.gz: 59632a0b46b9bd247da0f8b3908654a8027fbcef2aadc897f7681d25b03d4404191d037be323f666ef9bae679c72b135318aa853158e6bf0205b754ec3b2b18f
-  data.tar.gz: 2a037003347c6bca0900bf80410e83f43d397400f37e22f112e6ef6893a568dba29561b12594f803f3b28baee9f5f1ae67595c244d91b7dffa9d06e4e493c891
+  metadata.gz: a3ac194cc68b44fe0600ed79a9016e1dbab3b511d6b341a48c8cee27610d95683488fc5dc6ee10ebe5ed537a7eba4a3f18aaa31c8d27b84d16efc51e4525724d
+  data.tar.gz: 5fc5ffae4692e5f710ff7c01ab8a5b0f465db7d03b6a1ce2bd5d0f6db6c6699fcdb52034c5143cb91699234fc1028140f3bd4d4b07d1f41de80ebf9836f1b4e2

data/ext/isomorfeus_ferret_ext/frb_field_info.c

@@ -19,57 +19,62 @@ static VALUE sym_with_positions_offsets;
 
 extern VALUE sym_boost;
 
-void frb_fi_get_params(VALUE roptions, FrtStoreValue *store, FrtCompressionType *compression, FrtIndexValue *index, FrtTermVectorValue *term_vector, float *boost) {
+void frb_fi_get_params(VALUE roptions, unsigned int *bits, float *boost) {
     VALUE v;
     Check_Type(roptions, T_HASH);
     v = rb_hash_aref(roptions, sym_boost);
-    if (Qnil != v) {
-        *boost = (float)NUM2DBL(v);
-    } else {
-        *boost = 1.0f;
-    }
+    if (Qnil != v) *boost = (float)NUM2DBL(v);
+    else *boost = 1.0f;
+
     v = rb_hash_aref(roptions, sym_store);
     if (Qnil != v) Check_Type(v, T_SYMBOL);
     if (v == sym_no || v == sym_false || v == Qfalse) {
-        *store = FRT_STORE_NO;
+        *bits &= ~FRT_FI_IS_STORED_BM;
     } else if (v == sym_yes || v == sym_true || v == Qtrue) {
-        *store = FRT_STORE_YES;
+        *bits |= FRT_FI_IS_STORED_BM;
     } else if (v == Qnil) {
         /* leave as default */
     } else {
-        rb_raise(rb_eArgError, ":%s isn't a valid argument for :store. Please choose from [:yes, :no]",
-                 rb_id2name(SYM2ID(v)));
+        rb_raise(rb_eArgError, ":%s isn't a valid argument for :store. Please choose from [:yes, :no]", rb_id2name(SYM2ID(v)));
     }
 
     v = rb_hash_aref(roptions, sym_compression);
     if (Qnil != v) Check_Type(v, T_SYMBOL);
     if (v == sym_no || v == sym_false || v == Qfalse) {
-        *compression = FRT_COMPRESSION_NONE;
+        *bits &= ~FRT_FI_IS_COMPRESSED_BM;
+        *bits &= ~FRT_FI_COMPRESSION_BROTLI_BM;
+        *bits &= ~FRT_FI_COMPRESSION_BZ2_BM;
+        *bits &= ~FRT_FI_COMPRESSION_LZ4_BM;
     } else if (v == sym_yes || v == sym_true || v == Qtrue || v == sym_brotli) {
-        *compression = FRT_COMPRESSION_BROTLI;
+        *bits |= FRT_FI_IS_COMPRESSED_BM | FRT_FI_COMPRESSION_BROTLI_BM;
     } else if (v == sym_bz2) {
-        *compression = FRT_COMPRESSION_BZ2;
+        *bits |= FRT_FI_IS_COMPRESSED_BM | FRT_FI_COMPRESSION_BZ2_BM;
     } else if (v == sym_lz4) {
-        *compression = FRT_COMPRESSION_LZ4;
+        *bits |= FRT_FI_IS_COMPRESSED_BM | FRT_FI_COMPRESSION_LZ4_BM;
     } else if (v == Qnil) {
         /* leave as default */
     } else {
-        rb_raise(rb_eArgError, ":%s isn't a valid argument for :compression. Please choose from [:yes, :no, :brotli, :bz2, :lz4]",
-                 rb_id2name(SYM2ID(v)));
+        rb_raise(rb_eArgError, ":%s isn't a valid argument for :compression. Please choose from [:yes, :no, :brotli, :bz2, :lz4]", rb_id2name(SYM2ID(v)));
     }
 
     v = rb_hash_aref(roptions, sym_index);
     if (Qnil != v) Check_Type(v, T_SYMBOL);
     if (v == sym_no || v == sym_false || v == Qfalse) {
-        *index = FRT_INDEX_NO;
+        *bits &= ~FRT_FI_IS_INDEXED_BM;
+        *bits &= ~FRT_FI_IS_TOKENIZED_BM;
+        *bits &= ~FRT_FI_OMIT_NORMS_BM;
     } else if (v == sym_yes || v == sym_true || v == Qtrue) {
-        *index = FRT_INDEX_YES;
+        *bits |= FRT_FI_IS_INDEXED_BM | FRT_FI_IS_TOKENIZED_BM;
+        *bits &= ~FRT_FI_OMIT_NORMS_BM;
     } else if (v == sym_untokenized) {
-        *index = FRT_INDEX_UNTOKENIZED;
+        *bits |= FRT_FI_IS_INDEXED_BM;
+        *bits &= ~FRT_FI_IS_TOKENIZED_BM;
+        *bits &= ~FRT_FI_OMIT_NORMS_BM;
     } else if (v == sym_omit_norms) {
-        *index = FRT_INDEX_YES_OMIT_NORMS;
+        *bits |= FRT_FI_IS_INDEXED_BM | FRT_FI_IS_TOKENIZED_BM | FRT_FI_OMIT_NORMS_BM;
     } else if (v == sym_untokenized_omit_norms) {
-        *index = FRT_INDEX_UNTOKENIZED_OMIT_NORMS;
+        *bits |= FRT_FI_IS_INDEXED_BM | FRT_FI_OMIT_NORMS_BM;
+        *bits &= ~FRT_FI_IS_TOKENIZED_BM;
     } else if (v == Qnil) {
         /* leave as default */
     } else {
@@ -80,18 +85,28 @@ void frb_fi_get_params(VALUE roptions, FrtStoreValue *store, FrtCompressionType
     v = rb_hash_aref(roptions, sym_term_vector);
     if (Qnil != v) Check_Type(v, T_SYMBOL);
     if (v == sym_no || v == sym_false || v == Qfalse) {
-        *term_vector = FRT_TERM_VECTOR_NO;
+        *bits &= ~FRT_FI_STORE_TERM_VECTOR_BM;
+        *bits &= ~FRT_FI_STORE_POSITIONS_BM;
+        *bits &= ~FRT_FI_STORE_OFFSETS_BM;
     } else if (v == sym_yes || v == sym_true || v == Qtrue) {
-        *term_vector = FRT_TERM_VECTOR_YES;
+        *bits |= FRT_FI_STORE_TERM_VECTOR_BM;
+        *bits &= ~FRT_FI_STORE_POSITIONS_BM;
+        *bits &= ~FRT_FI_STORE_OFFSETS_BM;
     } else if (v == sym_with_positions) {
-        *term_vector = FRT_TERM_VECTOR_WITH_POSITIONS;
+        *bits |= FRT_FI_STORE_TERM_VECTOR_BM | FRT_FI_STORE_POSITIONS_BM;
+        *bits &= ~FRT_FI_STORE_OFFSETS_BM;
     } else if (v == sym_with_offsets) {
-        *term_vector = FRT_TERM_VECTOR_WITH_OFFSETS;
+        *bits |= FRT_FI_STORE_TERM_VECTOR_BM | FRT_FI_STORE_OFFSETS_BM;
+        *bits &= ~FRT_FI_STORE_POSITIONS_BM;
     } else if (v == sym_with_positions_offsets) {
-        *term_vector = FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS;
+        *bits |= FRT_FI_STORE_TERM_VECTOR_BM | FRT_FI_STORE_POSITIONS_BM | FRT_FI_STORE_OFFSETS_BM;
     } else if (v == Qnil) {
         /* leave as default */
-        if (*index == FRT_INDEX_NO) *term_vector = FRT_TERM_VECTOR_NO;
+        if ((*bits & FRT_FI_IS_INDEXED_BM) == 0) {
+            *bits &= ~FRT_FI_STORE_TERM_VECTOR_BM;
+            *bits &= ~FRT_FI_STORE_POSITIONS_BM;
+            *bits &= ~FRT_FI_STORE_OFFSETS_BM;
+        }
     } else {
         rb_raise(rb_eArgError, ":%s isn't a valid argument for :term_vector. Please choose from [:no, :yes, "
                  ":with_positions, :with_offsets, :with_positions_offsets]", rb_id2name(SYM2ID(v)));
@@ -150,17 +165,12 @@ static VALUE frb_fi_init(int argc, VALUE *argv, VALUE self) {
     VALUE roptions, rname;
     FrtFieldInfo *fi;
     TypedData_Get_Struct(self, FrtFieldInfo, &frb_field_info_t, fi);
-    FrtStoreValue store = FRT_STORE_YES;
-    FrtCompressionType compression = FRT_COMPRESSION_NONE;
-    FrtIndexValue index = FRT_INDEX_YES;
-    FrtTermVectorValue term_vector = FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS;
+    unsigned int bits = FRT_FI_DEFAULTS_BM;
     float boost = 1.0f;
 
     rb_scan_args(argc, argv, "11", &rname, &roptions);
-    if (argc > 1) {
-        frb_fi_get_params(roptions, &store, &compression, &index, &term_vector, &boost);
-    }
-    fi = frt_fi_init(fi, frb_field(rname), store, compression, index, term_vector);
+    if (argc > 1) frb_fi_get_params(roptions, &bits, &boost);
+    fi = frt_fi_init(fi, frb_field(rname), bits);
     fi->boost = boost;
     fi->rfi = self;
     return self;
@@ -185,7 +195,7 @@ static VALUE frb_fi_name(VALUE self) {
  */
 static VALUE frb_fi_is_stored(VALUE self) {
     FrtFieldInfo *fi = (FrtFieldInfo *)DATA_PTR(self);
-    return fi_is_stored(fi) ? Qtrue : Qfalse;
+    return bits_is_stored(fi->bits) ? Qtrue : Qfalse;
 }
 
 /*
@@ -196,7 +206,7 @@ static VALUE frb_fi_is_stored(VALUE self) {
  */
 static VALUE frb_fi_is_compressed(VALUE self) {
     FrtFieldInfo *fi = (FrtFieldInfo *)DATA_PTR(self);
-    return fi_is_compressed(fi) ? Qtrue : Qfalse;
+    return bits_is_compressed(fi->bits) ? Qtrue : Qfalse;
 }
 
 /*
@@ -207,7 +217,7 @@ static VALUE frb_fi_is_compressed(VALUE self) {
  */
 static VALUE frb_fi_is_indexed(VALUE self) {
     FrtFieldInfo *fi = (FrtFieldInfo *)DATA_PTR(self);
-    return fi_is_indexed(fi) ? Qtrue : Qfalse;
+    return bits_is_indexed(fi->bits) ? Qtrue : Qfalse;
 }
 
 /*
@@ -223,7 +233,7 @@ static VALUE frb_fi_is_indexed(VALUE self) {
  */
 static VALUE frb_fi_is_tokenized(VALUE self) {
     FrtFieldInfo *fi = (FrtFieldInfo *)DATA_PTR(self);
-    return fi_is_tokenized(fi) ? Qtrue : Qfalse;
+    return bits_is_tokenized(fi->bits) ? Qtrue : Qfalse;
 }
 
 /*
@@ -239,7 +249,7 @@ static VALUE frb_fi_is_tokenized(VALUE self) {
  */
 static VALUE frb_fi_omit_norms(VALUE self) {
     FrtFieldInfo *fi = (FrtFieldInfo *)DATA_PTR(self);
-    return fi_omit_norms(fi) ? Qtrue : Qfalse;
+    return bits_omit_norms(fi->bits) ? Qtrue : Qfalse;
 }
 
 /*
@@ -250,7 +260,7 @@ static VALUE frb_fi_omit_norms(VALUE self) {
  */
 static VALUE frb_fi_store_term_vector(VALUE self) {
     FrtFieldInfo *fi = (FrtFieldInfo *)DATA_PTR(self);
-    return fi_store_term_vector(fi) ? Qtrue : Qfalse;
+    return bits_store_term_vector(fi->bits) ? Qtrue : Qfalse;
 }
 
 /*
@@ -261,7 +271,7 @@ static VALUE frb_fi_store_term_vector(VALUE self) {
  */
 static VALUE frb_fi_store_positions(VALUE self) {
     FrtFieldInfo *fi = (FrtFieldInfo *)DATA_PTR(self);
-    return fi_store_positions(fi) ? Qtrue : Qfalse;
+    return bits_store_positions(fi->bits) ? Qtrue : Qfalse;
 }
 
 /*
@@ -272,7 +282,7 @@ static VALUE frb_fi_store_positions(VALUE self) {
  */
 static VALUE frb_fi_store_offsets(VALUE self) {
     FrtFieldInfo *fi = (FrtFieldInfo *)DATA_PTR(self);
-    return fi_store_offsets(fi) ? Qtrue : Qfalse;
+    return bits_store_offsets(fi->bits) ? Qtrue : Qfalse;
 }
 
 /*
@@ -285,7 +295,7 @@ static VALUE frb_fi_store_offsets(VALUE self) {
  */
 static VALUE frb_fi_has_norms(VALUE self) {
     FrtFieldInfo *fi = (FrtFieldInfo *)DATA_PTR(self);
-    return fi_has_norms(fi) ? Qtrue : Qfalse;
+    return bits_has_norms(fi->bits) ? Qtrue : Qfalse;
 }
 
 /*
@@ -326,10 +336,10 @@ static VALUE frb_fi_to_h(VALUE self) {
     bool o;
 
     // :index
-    if (!fi_is_indexed(fi)) val = sym_no;
+    if (!bits_is_indexed(fi->bits)) val = sym_no;
     else {
-      bool t = fi_is_tokenized(fi);
-      o = fi_omit_norms(fi);
+      bool t = bits_is_tokenized(fi->bits);
+      o = bits_omit_norms(fi->bits);
       if (!t && o) val = sym_untokenized_omit_norms;
       else if (t && o) val = sym_omit_norms;
       else if (!t && !o) val = sym_untokenized;
@@ -338,23 +348,23 @@ static VALUE frb_fi_to_h(VALUE self) {
     rb_hash_aset(hash, sym_index, val);
 
     // :store
-    rb_hash_aset(hash, sym_store, fi_is_stored(fi) ? sym_yes : sym_no);
+    rb_hash_aset(hash, sym_store, bits_is_stored(fi->bits) ? sym_yes : sym_no);
 
     // :compress
-    if (!fi_is_compressed(fi)) val = sym_no;
+    if (!bits_is_compressed(fi->bits)) val = sym_no;
     else {
-      if (fi_is_compressed_brotli(fi)) val = sym_brotli;
-      else if (fi_is_compressed_bz2(fi)) val = sym_bz2;
-      else if (fi_is_compressed_lz4(fi)) val = sym_lz4;
+      if (bits_is_compressed_brotli(fi->bits)) val = sym_brotli;
+      else if (bits_is_compressed_bz2(fi->bits)) val = sym_bz2;
+      else if (bits_is_compressed_lz4(fi->bits)) val = sym_lz4;
       else val = sym_yes;
     }
     rb_hash_aset(hash, sym_compression, val);
 
     // :term_vector
-    if (!fi_store_term_vector(fi)) val = sym_no;
+    if (!bits_store_term_vector(fi->bits)) val = sym_no;
     else {
-      bool p = fi_store_positions(fi);
-      o = fi_store_offsets(fi);
+      bool p = bits_store_positions(fi->bits);
+      o = bits_store_offsets(fi->bits);
       if (p && o) val = sym_with_positions_offsets;
       else if (o) val = sym_with_offsets;
       else if (p) val = sym_with_positions;

data/ext/isomorfeus_ferret_ext/frb_index.c

@@ -40,7 +40,7 @@ static ID id_boost;
 
 extern VALUE sym_each;
 extern rb_encoding *utf8_encoding;
-extern void frb_fi_get_params(VALUE roptions, FrtStoreValue *store, FrtCompressionType *compression, FrtIndexValue *index, FrtTermVectorValue *term_vector, float *boost);
+extern void frb_fi_get_params(VALUE roptions, unsigned int *bits, float *boost);
 extern FrtAnalyzer *frb_get_cwrapped_analyzer(VALUE ranalyzer);
 extern VALUE frb_get_analyzer(FrtAnalyzer *a);
 extern VALUE frb_get_field_info(FrtFieldInfo *fi);
@@ -119,17 +119,12 @@ static VALUE frb_fis_init(int argc, VALUE *argv, VALUE self) {
     VALUE roptions;
     FrtFieldInfos *fis;
     TypedData_Get_Struct(self, FrtFieldInfos, &frb_field_infos_t, fis);
-    FrtStoreValue store = FRT_STORE_YES;
-    FrtCompressionType compression = FRT_COMPRESSION_NONE;
-    FrtIndexValue index = FRT_INDEX_YES;
-    FrtTermVectorValue term_vector = FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS;
+    unsigned int bits = FRT_FI_DEFAULTS_BM;
     float boost;
 
     rb_scan_args(argc, argv, "01", &roptions);
-    if (argc > 0) {
-        frb_fi_get_params(roptions, &store, &compression, &index, &term_vector, &boost);
-    }
-    fis = frt_fis_init(fis, store, compression, index, term_vector);
+    if (argc > 0) frb_fi_get_params(roptions, &bits, &boost);
+    fis = frt_fis_init(fis, bits);
     fis->rfis = self;
     return self;
 }
@@ -218,18 +213,15 @@ frb_fis_add_field(int argc, VALUE *argv, VALUE self)
 {
     FrtFieldInfos *fis = (FrtFieldInfos *)DATA_PTR(self);
     FrtFieldInfo *fi;
-    FrtStoreValue store_val = fis->store_val;
-    FrtCompressionType compression = fis->compression;
-    FrtIndexValue index = fis->index;
-    FrtTermVectorValue term_vector = fis->term_vector;
+    unsigned int bits = fis->bits;
     float boost = 1.0f;
     VALUE rname, roptions;
 
     rb_scan_args(argc, argv, "11", &rname, &roptions);
     if (argc > 1) {
-        frb_fi_get_params(roptions, &store_val, &compression, &index, &term_vector, &boost);
+        frb_fi_get_params(roptions, &bits, &boost);
     }
-    fi = frt_fi_new(frb_field(rname), store_val, compression, index, term_vector);
+    fi = frt_fi_new(frb_field(rname), bits);
     fi->boost = boost;
     frt_fis_add_field(fis, fi);
     return self;
@@ -340,7 +332,7 @@ frb_fis_get_tk_fields(VALUE self)
     VALUE rfield_names = rb_ary_new();
     int i;
     for (i = 0; i < fis->size; i++) {
-        if (!fi_is_tokenized(fis->fields[i])) continue;
+        if (!bits_is_tokenized(fis->fields[i]->bits)) continue;
         rb_ary_push(rfield_names, ID2SYM(fis->fields[i]->name));
     }
     return rfield_names;
@@ -1082,7 +1074,7 @@ static VALUE frb_iw_init(int argc, VALUE *argv, VALUE self) {
                 TypedData_Get_Struct(rval, FrtFieldInfos, &frb_field_infos_t, fis);
                 frt_index_create(store, fis);
             } else {
-                fis = frt_fis_new(FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
+                fis = frt_fis_new(FRT_FI_DEFAULTS_BM);
                 frt_index_create(store, fis);
                 frt_fis_deref(fis);
             }
@@ -1132,6 +1124,8 @@ frb_hash_to_doc_i(VALUE key, VALUE value, VALUE arg)
     if (key == Qundef) {
         return ST_CONTINUE;
     } else {
+        int ex_code = 0;
+        const char *msg = NULL;
         FrtDocument *doc = (FrtDocument *)arg;
         ID field = frb_field(key);
         VALUE val;
@@ -1162,7 +1156,15 @@ frb_hash_to_doc_i(VALUE key, VALUE value, VALUE arg)
                 frt_df_add_data_len(df, rstrdup(val), RSTRING_LEN(val), rb_enc_get(val));
                 break;
         }
-        frt_doc_add_field(doc, df);
+        FRT_TRY
+            frt_doc_add_field(doc, df);
+        FRT_XCATCHALL
+            ex_code = xcontext.excode;
+            msg = xcontext.msg;
+            FRT_HANDLED();
+        FRT_XENDTRY
+
+        if (ex_code && msg) { frb_raise(ex_code, msg); }
     }
     return ST_CONTINUE;
 }
@@ -2293,7 +2295,7 @@ frb_ir_tk_fields(VALUE self)
     VALUE rfield_names = rb_ary_new();
     int i;
     for (i = 0; i < fis->size; i++) {
-        if (!fi_is_tokenized(fis->fields[i])) continue;
+        if (!bits_is_tokenized(fis->fields[i]->bits)) continue;
         rb_ary_push(rfield_names, rb_str_new_cstr(rb_id2name(fis->fields[i]->name)));
     }
     return rfield_names;
@@ -2548,7 +2550,7 @@ static void Init_TVTerm(void) {
  *  == Example
  *
  *    tv = index_reader.term_vector(doc_id, :content)
- *    tv_term = tv.find {|tvt| tvt.term = "fox"}
+ *    tv_term = tv.find {|tvt| tvt.term == "fox"}
  *
  *    # get the term frequency
  *    term_freq = tv_term.positions.size

data/ext/isomorfeus_ferret_ext/frt_analysis.c

@@ -66,7 +66,8 @@ static inline int get_cp(char *start, char *end, int *cp_len, rb_encoding *enc)
 
 FrtToken *frt_tk_set(FrtToken *tk, char *text, int tlen, frt_off_t start, frt_off_t end, int pos_inc, rb_encoding *encoding) {
     if (tlen >= FRT_MAX_WORD_SIZE) {
-        tlen = FRT_MAX_WORD_SIZE - 1; // TODO: this may invalidate mbc's
+        char *head_last = rb_enc_left_char_head(text, text + FRT_MAX_WORD_SIZE - 1, text + tlen, encoding);
+        tlen = head_last - text;
     }
 
     if (encoding == utf8_encoding) {
@@ -1031,9 +1032,9 @@ static FrtToken *stemf_next(FrtTokenStream *ts) {
     stemmed = sb_stemmer_stem(stemmer, (sb_symbol *)tk->text, tk->len);
     len = sb_stemmer_length(stemmer);
     if (len >= FRT_MAX_WORD_SIZE) {
-        len = FRT_MAX_WORD_SIZE - 1;
+        char *head_last = rb_enc_left_char_head(tk->text, tk->text + FRT_MAX_WORD_SIZE - 1, tk->text + len, utf8_encoding);
+        len = head_last - tk->text;
     }
-
     memcpy(tk->text, stemmed, len);
     tk->text[len] = '\0';
     tk->len = len;

data/ext/isomorfeus_ferret_ext/frt_global.c

@@ -261,30 +261,30 @@ void frt_register_for_cleanup(void *p, frt_free_ft free_func) {
 void frt_init(int argc, const char *const argv[]) {
     atexit(&frt_hash_finalize);
 
-    utf8_encoding = rb_enc_find("UTF-8");
+    utf8_encoding = rb_utf8_encoding();
     utf8_mbmaxlen = rb_enc_mbmaxlen(utf8_encoding);
     char *p = "'";
     cp_apostrophe = rb_enc_mbc_to_codepoint(p, p + 1, utf8_encoding);
-    p = ".";
-    cp_dot = rb_enc_mbc_to_codepoint(p, p + 1, utf8_encoding);
-    p = ",";
-    cp_comma = rb_enc_mbc_to_codepoint(p, p + 1, utf8_encoding);
-    p = "\\";
-    cp_backslash = rb_enc_mbc_to_codepoint(p, p + 1, utf8_encoding);
-    p = "/";
-    cp_slash = rb_enc_mbc_to_codepoint(p, p + 1, utf8_encoding);
-    p = "_";
-    cp_underscore = rb_enc_mbc_to_codepoint(p, p + 1, utf8_encoding);
-    p = "-";
-    cp_dash = rb_enc_mbc_to_codepoint(p, p + 1, utf8_encoding);
-    p = "\u2010";
-    cp_hyphen = rb_enc_mbc_to_codepoint(p, p + 1, utf8_encoding);
-    p = "@";
-    cp_at = rb_enc_mbc_to_codepoint(p, p + 1, utf8_encoding);
-    p = "&";
-    cp_ampersand = rb_enc_mbc_to_codepoint(p, p + 1, utf8_encoding);
-    p = ":";
-    cp_colon = rb_enc_mbc_to_codepoint(p, p + 1, utf8_encoding);
+    char *q = ".";
+    cp_dot = rb_enc_mbc_to_codepoint(q, q + 1, utf8_encoding);
+    char *r = ",";
+    cp_comma = rb_enc_mbc_to_codepoint(r, r + 1, utf8_encoding);
+    char *s = "\\";
+    cp_backslash = rb_enc_mbc_to_codepoint(s, s + 1, utf8_encoding);
+    char *t = "/";
+    cp_slash = rb_enc_mbc_to_codepoint(t, t + 1, utf8_encoding);
+    char *u = "_";
+    cp_underscore = rb_enc_mbc_to_codepoint(u, u + 1, utf8_encoding);
+    char *v = "-";
+    cp_dash = rb_enc_mbc_to_codepoint(v, v + 1, utf8_encoding);
+    char *w = "\u2010";
+    cp_hyphen = rb_enc_mbc_to_codepoint(w, w + 1, utf8_encoding);
+    char *x = "@";
+    cp_at = rb_enc_mbc_to_codepoint(x, x + 1, utf8_encoding);
+    char *y = "&";
+    cp_ampersand = rb_enc_mbc_to_codepoint(y, y + 1, utf8_encoding);
+    char *z = ":";
+    cp_colon = rb_enc_mbc_to_codepoint(z, z + 1, utf8_encoding);
 
     FRT_SORT_FIELD_SCORE = frt_sort_field_alloc();
     FRT_SORT_FIELD_SCORE->field_index_class = NULL;               /* field_index_class */

data/ext/isomorfeus_ferret_ext/frt_global.h

@@ -13,7 +13,7 @@
 
 #define FRT_MAX_WORD_SIZE 255
 #define FRT_MAX_FILE_PATH 1024
-#define FRT_BUFFER_SIZE 16384
+#define FRT_BUFFER_SIZE 4096
 
 typedef enum {
     FRT_COMPRESSION_NONE = 0,

data/ext/isomorfeus_ferret_ext/frt_ind.c

@@ -54,7 +54,7 @@ FrtIndex *frt_index_new(FrtStore *store, FrtAnalyzer *analyzer, FrtHashSet *def_
     }
 
     if (create) {
-        FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
+        FrtFieldInfos *fis = frt_fis_new(FRT_FI_DEFAULTS_BM);
         frt_index_create(self->store, fis);
         frt_fis_deref(fis);
     }