isomorfeus-ferret 0.12.7 → 0.13.2

data/ext/isomorfeus_ferret_ext/frb_index.c

@@ -1,6 +1,8 @@
 #include "frt_index.h"
 #include "isomorfeus_ferret.h"
-#include <ruby/st.h>
+#include <ruby.h>
+
+#undef close
 
 VALUE mIndex;
 
@@ -41,8 +43,10 @@ static VALUE sym_store;
 static VALUE sym_index;
 static VALUE sym_term_vector;
 
-static VALUE sym_compress;
-static VALUE sym_compressed;
+static VALUE sym_brotli;
+static VALUE sym_bz2;
+static VALUE sym_lz4;
+static VALUE sym_compression;
 
 static VALUE sym_untokenized;
 static VALUE sym_omit_norms;
@@ -52,7 +56,7 @@ static VALUE sym_with_positions;
 static VALUE sym_with_offsets;
 static VALUE sym_with_positions_offsets;
 
-static FrtSymbol fsym_content;
+static ID fsym_content;
 
 static ID id_term;
 static ID id_fields;
@@ -60,6 +64,7 @@ static ID id_fld_num_map;
 static ID id_field_num;
 static ID id_boost;
 
+extern rb_encoding *utf8_encoding;
 extern void frb_set_term(VALUE rterm, FrtTerm *t);
 extern FrtAnalyzer *frb_get_cwrapped_analyzer(VALUE ranalyzer);
 extern VALUE frb_get_analyzer(FrtAnalyzer *a);
@@ -70,20 +75,11 @@ extern VALUE frb_get_analyzer(FrtAnalyzer *a);
  *
  ****************************************************************************/
 
-static void
-frb_fi_free(void *p)
-{
-    object_del(p);
+static void frb_fi_free(void *p) {
     frt_fi_deref((FrtFieldInfo *)p);
 }
 
-static void
-frb_fi_get_params(VALUE roptions,
-                  FrtStoreValue *store,
-                  FrtIndexValue *index,
-                  FrtTermVectorValue *term_vector,
-                  float *boost)
-{
+static void frb_fi_get_params(VALUE roptions, FrtStoreValue *store, FrtCompressionType *compression, FrtIndexValue *index, FrtTermVectorValue *term_vector, float *boost) {
     VALUE v;
     Check_Type(roptions, T_HASH);
     v = rb_hash_aref(roptions, sym_boost);
@@ -98,13 +94,27 @@ frb_fi_get_params(VALUE roptions,
         *store = FRT_STORE_NO;
     } else if (v == sym_yes || v == sym_true || v == Qtrue) {
         *store = FRT_STORE_YES;
-    } else if (v == sym_compress || v == sym_compressed) {
-        *store = FRT_STORE_COMPRESS;
     } else if (v == Qnil) {
         /* leave as default */
     } else {
-        rb_raise(rb_eArgError, ":%s isn't a valid argument for :store."
-                 " Please choose from [:yes, :no, :compressed]",
+        rb_raise(rb_eArgError, ":%s isn't a valid argument for :store. Please choose from [:yes, :no]",
+                 rb_id2name(SYM2ID(v)));
+    }
+
+    v = rb_hash_aref(roptions, sym_compression);
+    if (Qnil != v) Check_Type(v, T_SYMBOL);
+    if (v == sym_no || v == sym_false || v == Qfalse) {
+        *compression = FRT_COMPRESSION_NONE;
+    } else if (v == sym_yes || v == sym_true || v == Qtrue || v == sym_brotli) {
+        *compression = FRT_COMPRESSION_BROTLI;
+    } else if (v == sym_bz2) {
+        *compression = FRT_COMPRESSION_BZ2;
+    } else if (v == sym_lz4) {
+        *compression = FRT_COMPRESSION_LZ4;
+    } else if (v == Qnil) {
+        /* leave as default */
+    } else {
+        rb_raise(rb_eArgError, ":%s isn't a valid argument for :compression. Please choose from [:yes, :no, :brotli, :bz2, :lz4]",
                  rb_id2name(SYM2ID(v)));
     }
 
@@ -123,10 +133,8 @@ frb_fi_get_params(VALUE roptions,
     } else if (v == Qnil) {
         /* leave as default */
     } else {
-        rb_raise(rb_eArgError, ":%s isn't a valid argument for :index."
-                 " Please choose from [:no, :yes, :untokenized, "
-                 ":omit_norms, :untokenized_omit_norms]",
-                 rb_id2name(SYM2ID(v)));
+        rb_raise(rb_eArgError, ":%s isn't a valid argument for :index. Please choose from [:no, :yes, :untokenized, "
+                 ":omit_norms, :untokenized_omit_norms]", rb_id2name(SYM2ID(v)));
     }
 
     v = rb_hash_aref(roptions, sym_term_vector);
@@ -144,28 +152,38 @@ frb_fi_get_params(VALUE roptions,
     } else if (v == Qnil) {
         /* leave as default */
     } else {
-        rb_raise(rb_eArgError, ":%s isn't a valid argument for "
-                 ":term_vector. Please choose from [:no, :yes, "
-                 ":with_positions, :with_offsets, "
-                 ":with_positions_offsets]",
-                 rb_id2name(SYM2ID(v)));
+        rb_raise(rb_eArgError, ":%s isn't a valid argument for :term_vector. Please choose from [:no, :yes, "
+                 ":with_positions, :with_offsets, :with_positions_offsets]", rb_id2name(SYM2ID(v)));
     }
 }
 
-static VALUE
-frb_get_field_info(FrtFieldInfo *fi)
-{
+static size_t frb_fi_size(const void *p) {
+    return sizeof(FrtFieldInfo);
+    (void)p;
+}
 
-    VALUE rfi = Qnil;
+const rb_data_type_t frb_field_info_t = {
+    .wrap_struct_name = "FrbFieldInfo",
+    .function = {
+        .dmark = NULL,
+        .dfree = frb_fi_free,
+        .dsize = frb_fi_size,
+        .dcompact = NULL,
+        .reserved = {0},
+    },
+    .parent = NULL,
+    .data = NULL,
+    .flags = RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+static VALUE frb_get_field_info(FrtFieldInfo *fi) {
     if (fi) {
-        rfi = object_get(fi);
-        if (rfi == Qnil) {
-            rfi = Data_Wrap_Struct(cFieldInfo, NULL, &frb_fi_free, fi);
+        if (fi->rfi == 0 || fi->rfi == Qnil) {
+            fi->rfi = TypedData_Wrap_Struct(cFieldInfo, &frb_field_info_t, fi);
             FRT_REF(fi);
-            object_add(fi, rfi);
         }
     }
-    return rfi;
+    return fi->rfi;
 }
 
 /*
@@ -173,28 +191,32 @@ frb_get_field_info(FrtFieldInfo *fi)
  *     FieldInfo.new(name, options = {}) -> field_info
  *
  *  Create a new FieldInfo object with the name +name+ and the properties
- *  specified in +options+. The available options are [:store, :index,
- *  :term_vector, :boost]. See the description of FieldInfo for more
+ *  specified in +options+. The available options are [:store, :compression,
+ *  :index, :term_vector, :boost]. See the description of FieldInfo for more
  *  information on these properties.
  */
-static VALUE
-frb_fi_init(int argc, VALUE *argv, VALUE self)
-{
+static VALUE frb_fi_alloc(VALUE rclass) {
+    FrtFieldInfo *fi = frt_fi_alloc();
+    return TypedData_Wrap_Struct(rclass, &frb_field_info_t, fi);
+}
+
+static VALUE frb_fi_init(int argc, VALUE *argv, VALUE self) {
     VALUE roptions, rname;
     FrtFieldInfo *fi;
+    TypedData_Get_Struct(self, FrtFieldInfo, &frb_field_info_t, fi);
     FrtStoreValue store = FRT_STORE_YES;
+    FrtCompressionType compression = FRT_COMPRESSION_NONE;
     FrtIndexValue index = FRT_INDEX_YES;
     FrtTermVectorValue term_vector = FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS;
     float boost = 1.0f;
 
     rb_scan_args(argc, argv, "11", &rname, &roptions);
     if (argc > 1) {
-        frb_fi_get_params(roptions, &store, &index, &term_vector, &boost);
+        frb_fi_get_params(roptions, &store, &compression, &index, &term_vector, &boost);
     }
-    fi = frt_fi_new(frb_field(rname), store, index, term_vector);
+    fi = frt_fi_init(fi, frb_field(rname), store, compression, index, term_vector);
     fi->boost = boost;
-    Frt_Wrap_Struct(self, NULL, &frb_fi_free, fi);
-    object_add(fi, self);
+    fi->rfi = self;
     return self;
 }
 
@@ -204,9 +226,7 @@ frb_fi_init(int argc, VALUE *argv, VALUE self)
  *
  *  Return the name of the field
  */
-static VALUE
-frb_fi_name(VALUE self)
-{
+static VALUE frb_fi_name(VALUE self) {
     FrtFieldInfo *fi = (FrtFieldInfo *)DATA_PTR(self);
     return rb_str_new_cstr(rb_id2name(fi->name));
 }
@@ -217,9 +237,7 @@ frb_fi_name(VALUE self)
  *
  *  Return true if the field is stored in the index.
  */
-static VALUE
-frb_fi_is_stored(VALUE self)
-{
+static VALUE frb_fi_is_stored(VALUE self) {
     FrtFieldInfo *fi = (FrtFieldInfo *)DATA_PTR(self);
     return fi_is_stored(fi) ? Qtrue : Qfalse;
 }
@@ -230,9 +248,7 @@ frb_fi_is_stored(VALUE self)
  *
  *  Return true if the field is stored in the index in compressed format.
  */
-static VALUE
-frb_fi_is_compressed(VALUE self)
-{
+static VALUE frb_fi_is_compressed(VALUE self) {
     FrtFieldInfo *fi = (FrtFieldInfo *)DATA_PTR(self);
     return fi_is_compressed(fi) ? Qtrue : Qfalse;
 }
@@ -243,9 +259,7 @@ frb_fi_is_compressed(VALUE self)
  *
  *  Return true if the field is indexed, ie searchable in the index.
  */
-static VALUE
-frb_fi_is_indexed(VALUE self)
-{
+static VALUE frb_fi_is_indexed(VALUE self) {
     FrtFieldInfo *fi = (FrtFieldInfo *)DATA_PTR(self);
     return fi_is_indexed(fi) ? Qtrue : Qfalse;
 }
@@ -261,9 +275,7 @@ frb_fi_is_indexed(VALUE self)
  *
  *  A field can only be tokenized if it is indexed.
  */
-static VALUE
-frb_fi_is_tokenized(VALUE self)
-{
+static VALUE frb_fi_is_tokenized(VALUE self) {
     FrtFieldInfo *fi = (FrtFieldInfo *)DATA_PTR(self);
     return fi_is_tokenized(fi) ? Qtrue : Qfalse;
 }
@@ -279,9 +291,7 @@ frb_fi_is_tokenized(VALUE self)
  *  boost and it will use less memory, especially for indexes which have a
  *  large number of documents.
  */
-static VALUE
-frb_fi_omit_norms(VALUE self)
-{
+static VALUE frb_fi_omit_norms(VALUE self) {
     FrtFieldInfo *fi = (FrtFieldInfo *)DATA_PTR(self);
     return fi_omit_norms(fi) ? Qtrue : Qfalse;
 }
@@ -292,9 +302,7 @@ frb_fi_omit_norms(VALUE self)
  *
  *  Return true if the term-vectors are stored for this field.
  */
-static VALUE
-frb_fi_store_term_vector(VALUE self)
-{
+static VALUE frb_fi_store_term_vector(VALUE self) {
     FrtFieldInfo *fi = (FrtFieldInfo *)DATA_PTR(self);
     return fi_store_term_vector(fi) ? Qtrue : Qfalse;
 }
@@ -305,9 +313,7 @@ frb_fi_store_term_vector(VALUE self)
  *
  *  Return true if positions are stored with the term-vectors for this field.
  */
-static VALUE
-frb_fi_store_positions(VALUE self)
-{
+static VALUE frb_fi_store_positions(VALUE self) {
     FrtFieldInfo *fi = (FrtFieldInfo *)DATA_PTR(self);
     return fi_store_positions(fi) ? Qtrue : Qfalse;
 }
@@ -318,9 +324,7 @@ frb_fi_store_positions(VALUE self)
  *
  *  Return true if offsets are stored with the term-vectors for this field.
  */
-static VALUE
-frb_fi_store_offsets(VALUE self)
-{
+static VALUE frb_fi_store_offsets(VALUE self) {
     FrtFieldInfo *fi = (FrtFieldInfo *)DATA_PTR(self);
     return fi_store_offsets(fi) ? Qtrue : Qfalse;
 }
@@ -333,9 +337,7 @@ frb_fi_store_offsets(VALUE self)
  *
  *    fi.indexed? and not fi.omit_norms?
  */
-static VALUE
-frb_fi_has_norms(VALUE self)
-{
+static VALUE frb_fi_has_norms(VALUE self) {
     FrtFieldInfo *fi = (FrtFieldInfo *)DATA_PTR(self);
     return fi_has_norms(fi) ? Qtrue : Qfalse;
 }
@@ -346,9 +348,7 @@ frb_fi_has_norms(VALUE self)
  *
  *  Return the default boost for this field
  */
-static VALUE
-frb_fi_boost(VALUE self)
-{
+static VALUE frb_fi_boost(VALUE self) {
     FrtFieldInfo *fi = (FrtFieldInfo *)DATA_PTR(self);
     return rb_float_new((double)fi->boost);
 }
@@ -359,9 +359,7 @@ frb_fi_boost(VALUE self)
  *
  *  Return a string representation of the FieldInfo object.
  */
-static VALUE
-frb_fi_to_s(VALUE self)
-{
+static VALUE frb_fi_to_s(VALUE self) {
     FrtFieldInfo *fi = (FrtFieldInfo *)DATA_PTR(self);
     char *fi_s = frt_fi_to_s(fi);
     VALUE rfi_s = rb_str_new2(fi_s);
@@ -375,39 +373,47 @@ frb_fi_to_s(VALUE self)
  *
  ****************************************************************************/
 
-static void
-frb_fis_free(void *p)
-{
-    object_del(p);
+static void frb_fis_free(void *p) {
     frt_fis_deref((FrtFieldInfos *)p);
 }
 
-static void
-frb_fis_mark(void *p)
-{
+static void frb_fis_mark(void *p) {
     int i;
     FrtFieldInfos *fis = (FrtFieldInfos *)p;
 
     for (i = 0; i < fis->size; i++) {
-        frb_gc_mark(fis->fields[i]);
+        if (fis->fields[i]->rfi)
+            rb_gc_mark(fis->fields[i]->rfi);
     }
 }
 
-static VALUE
-frb_get_field_infos(FrtFieldInfos *fis)
-{
+static size_t frb_field_infos_t_size(const void *p) {
+    return sizeof(FrtFieldInfos);
+    (void)p;
+}
 
-    VALUE rfis = Qnil;
+const rb_data_type_t frb_field_infos_t = {
+    .wrap_struct_name = "FrbFieldInfos",
+    .function = {
+        .dmark = frb_fis_mark,
+        .dfree = frb_fis_free,
+        .dsize = frb_field_infos_t_size,
+        .dcompact = NULL,
+        .reserved = {0},
+    },
+    .parent = NULL,
+    .data = NULL,
+    .flags = RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+static VALUE frb_get_field_infos(FrtFieldInfos *fis) {
     if (fis) {
-        rfis = object_get(fis);
-        if (rfis == Qnil) {
-            rfis = Data_Wrap_Struct(cFieldInfos, &frb_fis_mark, &frb_fis_free,
-                                    fis);
+        if (fis->rfis == 0 || fis->rfis == Qnil) {
+            fis->rfis = TypedData_Wrap_Struct(cFieldInfos, &frb_field_infos_t, fis);
             FRT_REF(fis);
-            object_add(fis, rfis);
         }
     }
-    return rfis;
+    return fis->rfis;
 }
 
 /*
@@ -418,23 +424,28 @@ frb_get_field_infos(FrtFieldInfos *fis)
  *  specified in the +default+ hash parameter. See FieldInfo for available
  *  property values.
  */
-static VALUE
-frb_fis_init(int argc, VALUE *argv, VALUE self)
-{
+
+static VALUE frb_fis_alloc(VALUE rclass) {
+    FrtFieldInfos *fis = frt_fis_alloc();
+    return TypedData_Wrap_Struct(rclass, &frb_field_infos_t, fis);
+}
+
+static VALUE frb_fis_init(int argc, VALUE *argv, VALUE self) {
     VALUE roptions;
     FrtFieldInfos *fis;
+    TypedData_Get_Struct(self, FrtFieldInfos, &frb_field_infos_t, fis);
     FrtStoreValue store = FRT_STORE_YES;
+    FrtCompressionType compression = FRT_COMPRESSION_NONE;
     FrtIndexValue index = FRT_INDEX_YES;
     FrtTermVectorValue term_vector = FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS;
     float boost;
 
     rb_scan_args(argc, argv, "01", &roptions);
     if (argc > 0) {
-        frb_fi_get_params(roptions, &store, &index, &term_vector, &boost);
+        frb_fi_get_params(roptions, &store, &compression, &index, &term_vector, &boost);
     }
-    fis = frt_fis_new(store, index, term_vector);
-    Frt_Wrap_Struct(self, &frb_fis_mark, &frb_fis_free, fis);
-    object_add(fis, self);
+    fis = frt_fis_init(fis, store, compression, index, term_vector);
+    fis->rfis = self;
     return self;
 }
 
@@ -445,9 +456,7 @@ frb_fis_init(int argc, VALUE *argv, VALUE self)
  *  Return an array of the FieldInfo objects contained but this FieldInfos
  *  object.
  */
-static VALUE
-frb_fis_to_a(VALUE self)
-{
+static VALUE frb_fis_to_a(VALUE self) {
     FrtFieldInfos *fis = (FrtFieldInfos *)DATA_PTR(self);
     VALUE rary = rb_ary_new();
     int i;
@@ -470,9 +479,7 @@ frb_fis_to_a(VALUE self)
  *    fi = fis[:name]
  *    fi = fis[2]
  */
-static VALUE
-frb_fis_get(VALUE self, VALUE ridx)
-{
+static VALUE frb_fis_get(VALUE self, VALUE ridx) {
     FrtFieldInfos *fis = (FrtFieldInfos *)DATA_PTR(self);
     VALUE rfi = Qnil;
     switch (TYPE(ridx)) {
@@ -511,9 +518,7 @@ frb_fis_get(VALUE self, VALUE ridx)
  *  Add a FieldInfo object. Use the FieldInfos#add_field method where
  *  possible.
  */
-static VALUE
-frb_fis_add(VALUE self, VALUE rfi)
-{
+static VALUE frb_fis_add(VALUE self, VALUE rfi) {
     FrtFieldInfos *fis = (FrtFieldInfos *)DATA_PTR(self);
     FrtFieldInfo *fi = (FrtFieldInfo *)frb_rb_data_ptr(rfi);
     frt_fis_add_field(fis, fi);
@@ -534,6 +539,7 @@ frb_fis_add_field(int argc, VALUE *argv, VALUE self)
     FrtFieldInfos *fis = (FrtFieldInfos *)DATA_PTR(self);
     FrtFieldInfo *fi;
     FrtStoreValue store = fis->store;
+    FrtCompressionType compression = fis->compression;
     FrtIndexValue index = fis->index;
     FrtTermVectorValue term_vector = fis->term_vector;
     float boost = 1.0f;
@@ -541,9 +547,9 @@ frb_fis_add_field(int argc, VALUE *argv, VALUE self)
 
     rb_scan_args(argc, argv, "11", &rname, &roptions);
     if (argc > 1) {
-        frb_fi_get_params(roptions, &store, &index, &term_vector, &boost);
+        frb_fi_get_params(roptions, &store, &compression, &index, &term_vector, &boost);
     }
-    fi = frt_fi_new(frb_field(rname), store, index, term_vector);
+    fi = frt_fi_new(frb_field(rname), store, compression, index, term_vector);
     fi->boost = boost;
     frt_fis_add_field(fis, fi);
     return self;
@@ -670,28 +676,46 @@ frb_fis_get_tk_fields(VALUE self)
  *
  ****************************************************************************/
 
-static void
-frb_te_free(void *p)
-{
+static void frb_te_free(void *p) {
     FrtTermEnum *te = (FrtTermEnum *)p;
     te->close(te);
 }
 
-static VALUE
-frb_te_get_set_term(VALUE self, const char *term)
-{
+static size_t frb_te_size(const void *p) {
+    return sizeof(FrtTermEnum);
+    (void)p;
+}
+
+const rb_data_type_t frb_term_enum_t = {
+    .wrap_struct_name = "FrbTermEnum",
+    .function = {
+        .dmark = NULL,
+        .dfree = frb_te_free,
+        .dsize = frb_te_size,
+        .dcompact = NULL,
+        .reserved = {0},
+    },
+    .parent = NULL,
+    .data = NULL,
+    .flags = RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+static VALUE frb_te_alloc(VALUE rclass) {
+    FrtTermEnum *te = FRT_ALLOC_AND_ZERO(FrtTermEnum);
+    return TypedData_Wrap_Struct(rclass, &frb_term_enum_t, te);
+}
+
+static VALUE frb_te_get_set_term(VALUE self, const char *term) {
     FrtTermEnum *te = (FrtTermEnum *)DATA_PTR(self);
     VALUE str = term ? rb_str_new(term, te->curr_term_len) : Qnil;
     rb_ivar_set(self, id_term, str);
     return str;
 }
 
-static VALUE
-frb_get_te(VALUE rir, FrtTermEnum *te)
-{
+static VALUE frb_get_te(VALUE rir, FrtTermEnum *te) {
     VALUE self = Qnil;
     if (te != NULL) {
-        self = Data_Wrap_Struct(cTermEnum, NULL, &frb_te_free, te);
+        self = TypedData_Wrap_Struct(cTermEnum, &frb_term_enum_t, te);
         frb_te_get_set_term(self, te->curr_term);
         rb_ivar_set(self, id_fld_num_map, rb_ivar_get(rir, id_fld_num_map));
     }
@@ -704,9 +728,7 @@ frb_get_te(VALUE rir, FrtTermEnum *te)
  *
  *  Returns the next term in the enumeration or nil otherwise.
  */
-static VALUE
-frb_te_next(VALUE self)
-{
+static VALUE frb_te_next(VALUE self) {
     FrtTermEnum *te = (FrtTermEnum *)DATA_PTR(self);
     return frb_te_get_set_term(self, te->next(te));
 }
@@ -718,9 +740,7 @@ frb_te_next(VALUE self)
  *  Returns the current term pointed to by the enum. This method should only
  *  be called after a successful call to TermEnum#next.
  */
-static VALUE
-frb_te_term(VALUE self)
-{
+static VALUE frb_te_term(VALUE self) {
     return rb_ivar_get(self, id_term);
 }
 
@@ -732,9 +752,7 @@ frb_te_term(VALUE self)
  *  That is the number of documents that this term appears in. The method
  *  should only be called after a successful call to TermEnum#next.
  */
-static VALUE
-frb_te_doc_freq(VALUE self)
-{
+static VALUE frb_te_doc_freq(VALUE self) {
     FrtTermEnum *te = (FrtTermEnum *)DATA_PTR(self);
     return INT2FIX(te->curr_ti.doc_freq);
 }
@@ -750,9 +768,7 @@ frb_te_doc_freq(VALUE self)
  *
  *  Returns the first term greater than or equal to +target+
  */
-static VALUE
-frb_te_skip_to(VALUE self, VALUE rterm)
-{
+static VALUE frb_te_skip_to(VALUE self, VALUE rterm) {
     FrtTermEnum *te = (FrtTermEnum *)DATA_PTR(self);
     return frb_te_get_set_term(self, te->skip_to(te, rs2s(rterm)));
 }
@@ -764,9 +780,7 @@ frb_te_skip_to(VALUE self, VALUE rterm)
  *  Iterates through all the terms in the field, yielding the term and the
  *  document frequency.
  */
-static VALUE
-frb_te_each(VALUE self)
-{
+static VALUE frb_te_each(VALUE self) {
     FrtTermEnum *te = (FrtTermEnum *)DATA_PTR(self);
     char *term;
     int term_cnt = 0;
@@ -798,9 +812,7 @@ frb_te_each(VALUE self)
  *      do_something()
  *    end
  */
-static VALUE
-frb_te_set_field(VALUE self, VALUE rfield)
-{
+static VALUE frb_te_set_field(VALUE self, VALUE rfield) {
     FrtTermEnum *te = (FrtTermEnum *)DATA_PTR(self);
     int field_num = 0;
     VALUE rfnum_map = rb_ivar_get(self, id_fld_num_map);
@@ -840,9 +852,7 @@ frb_te_set_field(VALUE self, VALUE rfield)
  *    #   ["cantaloupe",12]
  *    # ]
  */
-static VALUE
-frb_te_to_json(int argc, VALUE *argv, VALUE self)
-{
+static VALUE frb_te_to_json(int argc, VALUE *argv, VALUE self) {
     FrtTermEnum *te = (FrtTermEnum *)DATA_PTR(self);
     VALUE rjson;
     char *json, *jp;
@@ -867,8 +877,7 @@ frb_te_to_json(int argc, VALUE *argv, VALUE self)
             *(jp++) = ']';
             *(jp++) = ',';
         }
-    }
-    else {
+    } else {
         while (NULL != (term = te->next(te))) {
             /* enough room for for term after converting " to '"' and frequency
              * plus some extra for good measure */
@@ -904,17 +913,37 @@ frb_te_to_json(int argc, VALUE *argv, VALUE self)
  *
  ****************************************************************************/
 
-static void
-frb_tde_free(void *p)
-{
+static void frb_tde_free(void *p) {
     FrtTermDocEnum *tde = (FrtTermDocEnum *)p;
     tde->close(tde);
 }
 
-static VALUE
-frb_get_tde(VALUE rir, FrtTermDocEnum *tde)
-{
-    VALUE self = Data_Wrap_Struct(cTermDocEnum, NULL, &frb_tde_free, tde);
+static size_t frb_tde_size(const void *p) {
+    return sizeof(FrtTermDocEnum);
+    (void)p;
+}
+
+const rb_data_type_t frb_term_doc_enum_t = {
+    .wrap_struct_name = "FrbTermDocEnum",
+    .function = {
+        .dmark = NULL,
+        .dfree = frb_tde_free,
+        .dsize = frb_tde_size,
+        .dcompact = NULL,
+        .reserved = {0},
+    },
+    .parent = NULL,
+    .data = NULL,
+    .flags = RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+static VALUE frb_tde_alloc(VALUE rclass) {
+    FrtTermDocEnum *tde = FRT_ALLOC_AND_ZERO(FrtTermDocEnum);
+    return TypedData_Wrap_Struct(rclass, &frb_term_doc_enum_t, tde);
+}
+
+static VALUE frb_get_tde(VALUE rir, FrtTermDocEnum *tde) {
+    VALUE self = TypedData_Wrap_Struct(cTermDocEnum, &frb_term_doc_enum_t, tde);
     rb_ivar_set(self, id_fld_num_map, rb_ivar_get(rir, id_fld_num_map));
     return self;
 }
@@ -927,9 +956,7 @@ frb_get_tde(VALUE rir, FrtTermDocEnum *tde)
  *  you can call next or each to skip through the documents and positions of
  *  this particular term.
  */
-static VALUE
-frb_tde_seek(VALUE self, VALUE rfield, VALUE rterm)
-{
+static VALUE frb_tde_seek(VALUE self, VALUE rfield, VALUE rterm) {
     FrtTermDocEnum *tde = (FrtTermDocEnum *)DATA_PTR(self);
     char *term;
     VALUE rfnum_map = rb_ivar_get(self, id_fld_num_map);
@@ -939,8 +966,7 @@ frb_tde_seek(VALUE self, VALUE rfield, VALUE rterm)
     if (rfnum != Qnil) {
         field_num = FIX2INT(rfnum);
     } else {
-        rb_raise(rb_eArgError, "field %s doesn't exist in the index",
-                rb_id2name(frb_field(rfield)));
+        rb_raise(rb_eArgError, "field %s doesn't exist in the index", rb_id2name(frb_field(rfield)));
     }
     tde->seek(tde, field_num, term);
     return self;
@@ -958,9 +984,7 @@ frb_tde_seek(VALUE self, VALUE rfield, VALUE rterm)
  *  However the +seek_term_enum+ method saves an index lookup so should offer
  *  a large performance improvement.
  */
-static VALUE
-frb_tde_seek_te(VALUE self, VALUE rterm_enum)
-{
+static VALUE frb_tde_seek_te(VALUE self, VALUE rterm_enum) {
     FrtTermDocEnum *tde = (FrtTermDocEnum *)DATA_PTR(self);
     FrtTermEnum *te = (FrtTermEnum *)frb_rb_data_ptr(rterm_enum);
     tde->seek_te(tde, te);
@@ -973,9 +997,7 @@ frb_tde_seek_te(VALUE self, VALUE rterm_enum)
  *
  *  Returns the current document number pointed to by the +term_doc_enum+.
  */
-static VALUE
-frb_tde_doc(VALUE self)
-{
+static VALUE frb_tde_doc(VALUE self) {
     FrtTermDocEnum *tde = (FrtTermDocEnum *)DATA_PTR(self);
     return INT2FIX(tde->doc_num(tde));
 }
@@ -987,9 +1009,7 @@ frb_tde_doc(VALUE self)
  *  Returns the frequency of the current document pointed to by the
  *  +term_doc_enum+.
  */
-static VALUE
-frb_tde_freq(VALUE self)
-{
+static VALUE frb_tde_freq(VALUE self) {
     FrtTermDocEnum *tde = (FrtTermDocEnum *)DATA_PTR(self);
     return INT2FIX(tde->freq(tde));
 }
@@ -1001,9 +1021,7 @@ frb_tde_freq(VALUE self)
  *  Move forward to the next document in the enumeration. Returns +true+ if
  *  there is another document or +false+ otherwise.
  */
-static VALUE
-frb_tde_next(VALUE self)
-{
+static VALUE frb_tde_next(VALUE self) {
     FrtTermDocEnum *tde = (FrtTermDocEnum *)DATA_PTR(self);
     return tde->next(tde) ? Qtrue : Qfalse;
 }
@@ -1015,9 +1033,7 @@ frb_tde_next(VALUE self)
  *  Move forward to the next document in the enumeration. Returns +true+ if
  *  there is another document or +false+ otherwise.
  */
-static VALUE
-frb_tde_next_position(VALUE self)
-{
+static VALUE frb_tde_next_position(VALUE self) {
     FrtTermDocEnum *tde = (FrtTermDocEnum *)DATA_PTR(self);
     int pos;
     if (tde->next_position == NULL) {
@@ -1039,9 +1055,7 @@ frb_tde_next_position(VALUE self)
  *  NOTE: this method can only be called once after each seek. If you need to
  *  call +#each+ again then you should call +#seek+ again too.
  */
-static VALUE
-frb_tde_each(VALUE self)
-{
+static VALUE frb_tde_each(VALUE self) {
     int doc_cnt = 0;
     FrtTermDocEnum *tde = (FrtTermDocEnum *)DATA_PTR(self);
     VALUE vals = rb_ary_new2(2);
@@ -1083,9 +1097,7 @@ frb_tde_each(VALUE self)
  *    #   [30,3]
  *    # ]
  */
-static VALUE
-frb_tde_to_json(int argc, VALUE *argv, VALUE self)
-{
+static VALUE frb_tde_to_json(int argc, VALUE *argv, VALUE self) {
     FrtTermDocEnum *tde = (FrtTermDocEnum *)DATA_PTR(self);
     VALUE rjson;
     char *json, *jp;
@@ -1157,9 +1169,7 @@ frb_tde_to_json(int argc, VALUE *argv, VALUE self)
  *      puts "  #{positions.join(', ')}"
  *    end
  */
-static VALUE
-frb_tde_each_position(VALUE self)
-{
+static VALUE frb_tde_each_position(VALUE self) {
     FrtTermDocEnum *tde = (FrtTermDocEnum *)DATA_PTR(self);
     int pos;
     if (tde->next_position == NULL) {
@@ -1180,9 +1190,7 @@ frb_tde_each_position(VALUE self)
  *  Skip to the required document number +target+ and return true if there is
  *  a document >= +target+.
  */
-static VALUE
-frb_tde_skip_to(VALUE self, VALUE rtarget)
-{
+static VALUE frb_tde_skip_to(VALUE self, VALUE rtarget) {
     FrtTermDocEnum *tde = (FrtTermDocEnum *)DATA_PTR(self);
     return tde->skip_to(tde, FIX2INT(rtarget)) ? Qtrue : Qfalse;
 }
@@ -1193,9 +1201,7 @@ frb_tde_skip_to(VALUE self, VALUE rtarget)
  *
  ****************************************************************************/
 
-static VALUE
-frb_get_tv_offsets(FrtOffset *offset)
-{
+static VALUE frb_get_tv_offsets(FrtOffset *offset) {
     return rb_struct_new(cTVOffsets,
                          ULL2NUM((frt_u64)offset->start),
                          ULL2NUM((frt_u64)offset->end),
@@ -1208,14 +1214,13 @@ frb_get_tv_offsets(FrtOffset *offset)
  *
  ****************************************************************************/
 
-static VALUE
-frb_get_tv_term(FrtTVTerm *tv_term)
-{
+static VALUE frb_get_tv_term(FrtTVTerm *tv_term) {
     int i;
     const int freq = tv_term->freq;
     VALUE rtext;
     VALUE rpositions = Qnil;
     rtext = rb_str_new2(tv_term->text);
+    rb_enc_associate(rtext, utf8_encoding);
     if (tv_term->positions) {
         int *positions = tv_term->positions;
         rpositions = rb_ary_new2(freq);
@@ -1232,9 +1237,7 @@ frb_get_tv_term(FrtTVTerm *tv_term)
  *
  ****************************************************************************/
 
-static VALUE
-frb_get_tv(FrtTermVector *tv)
-{
+static VALUE frb_get_tv(FrtTermVector *tv) {
     int i;
     FrtTVTerm *terms = tv->terms;
     const int t_cnt = tv->term_cnt;
@@ -1265,19 +1268,18 @@ frb_get_tv(FrtTermVector *tv)
  *
  ****************************************************************************/
 
-void
-frb_iw_free(void *p)
-{
+void frb_iw_free(void *p) {
     frt_iw_close((FrtIndexWriter *)p);
 }
 
-void
-frb_iw_mark(void *p)
-{
+void frb_iw_mark(void *p) {
     FrtIndexWriter *iw = (FrtIndexWriter *)p;
-    frb_gc_mark(iw->analyzer);
-    frb_gc_mark(iw->store);
-    frb_gc_mark(iw->fis);
+    if (iw->analyzer->ranalyzer)
+        rb_gc_mark(iw->analyzer->ranalyzer);
+    if (iw->store->rstore)
+        rb_gc_mark(iw->store->rstore);
+    if (iw->fis->rfis)
+        rb_gc_mark(iw->fis->rfis);
 }
 
 /*
@@ -1288,11 +1290,11 @@ frb_iw_mark(void *p)
  *  exclusively by the index writer. The garbage collector will do this
  *  automatically if not called explicitly.
  */
-static VALUE
-frb_iw_close(VALUE self)
-{
+static VALUE frb_iw_close(VALUE self) {
     FrtIndexWriter *iw = (FrtIndexWriter *)DATA_PTR(self);
-    Frt_Unwrap_Struct(self);
+    ((struct RData *)(self))->data = NULL;
+    ((struct RData *)(self))->dmark = NULL;
+    ((struct RData *)(self))->dfree = NULL;
     frt_iw_close(iw);
     return Qnil;
 }
@@ -1321,9 +1323,31 @@ frb_iw_close(VALUE self)
  *
  * See FrtIndexWriter for more options.
  */
-static VALUE
-frb_iw_init(int argc, VALUE *argv, VALUE self)
-{
+static size_t frb_index_writer_t_size(const void *p) {
+    return sizeof(FrtIndexWriter);
+    (void)p;
+}
+
+const rb_data_type_t frb_index_writer_t = {
+    .wrap_struct_name = "FrbIndexWriter",
+    .function = {
+        .dmark = frb_iw_mark,
+        .dfree = frb_iw_free,
+        .dsize = frb_index_writer_t_size,
+        .dcompact = NULL,
+        .reserved = {0},
+    },
+    .parent = NULL,
+    .data = NULL,
+    .flags = RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+static VALUE frb_iw_alloc(VALUE rclass) {
+    FrtIndexWriter *iw = frt_iw_alloc();
+    return TypedData_Wrap_Struct(rclass, &frb_index_writer_t, iw);
+}
+
+static VALUE frb_iw_init(int argc, VALUE *argv, VALUE self) {
     VALUE roptions, rval;
     bool create = false;
     bool create_if_missing = true;
@@ -1341,7 +1365,7 @@ frb_iw_init(int argc, VALUE *argv, VALUE self)
             Check_Type(roptions, T_HASH);
 
             if ((rval = rb_hash_aref(roptions, sym_dir)) != Qnil) {
-                Check_Type(rval, T_DATA);
+                // Check_Type(rval, T_DATA);
                 store = DATA_PTR(rval);
             } else if ((rval = rb_hash_aref(roptions, sym_path)) != Qnil) {
                 StringValue(rval);
@@ -1349,17 +1373,9 @@ frb_iw_init(int argc, VALUE *argv, VALUE self)
                 store = frt_open_fs_store(rs2s(rval));
                 FRT_DEREF(store);
             }
-
-            /* Let ruby's garbage collector handle the closing of the store
-            if (!close_dir) {
-            close_dir = RTEST(rb_hash_aref(roptions, sym_close_dir));
-            }
-            */
             /* use_compound_file defaults to true */
             config.use_compound_file =
-                (rb_hash_aref(roptions, sym_use_compound_file) == Qfalse)
-                ? false
-                : true;
+                (rb_hash_aref(roptions, sym_use_compound_file) == Qfalse) ? false : true;
 
             if ((rval = rb_hash_aref(roptions, sym_analyzer)) != Qnil) {
                 analyzer = frb_get_cwrapped_analyzer(rval);
@@ -1379,7 +1395,7 @@ frb_iw_init(int argc, VALUE *argv, VALUE self)
             SET_INT_ATTR(max_field_length);
         }
         if (NULL == store) {
-            store = frt_open_ram_store();
+            store = frt_open_ram_store(NULL);
             FRT_DEREF(store);
         }
         if (!create && create_if_missing && !store->exists(store, "segments")) {
@@ -1388,26 +1404,29 @@ frb_iw_init(int argc, VALUE *argv, VALUE self)
         if (create) {
             FrtFieldInfos *fis;
             if ((rval = rb_hash_aref(roptions, sym_field_infos)) != Qnil) {
-                Data_Get_Struct(rval, FrtFieldInfos, fis);
+                TypedData_Get_Struct(rval, FrtFieldInfos, &frb_field_infos_t, fis);
                 frt_index_create(store, fis);
             } else {
-                fis = frt_fis_new(FRT_STORE_YES, FRT_INDEX_YES,
-                            FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
+                fis = frt_fis_new(FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
                 frt_index_create(store, fis);
                 frt_fis_deref(fis);
             }
         }
 
-        iw = frt_iw_open(store, analyzer, &config);
-
-        Frt_Wrap_Struct(self, &frb_iw_mark, &frb_iw_free, iw);
-    default:
+        TypedData_Get_Struct(self, FrtIndexWriter, &frb_index_writer_t, iw);
+        iw = frt_iw_open(iw, store, analyzer, &config);
+    FRT_XCATCHALL
         ex_code = xcontext.excode;
         msg = xcontext.msg;
         FRT_HANDLED();
     FRT_XENDTRY
 
-    if (ex_code && msg) { frb_raise(ex_code, msg); }
+    if (ex_code && msg) {
+        ((struct RData *)(self))->data = NULL;
+        ((struct RData *)(self))->dmark = NULL;
+        ((struct RData *)(self))->dfree = NULL;
+        frb_raise(ex_code, msg);
+    }
 
     if (rb_block_given_p()) {
         rb_yield(self);
@@ -1439,7 +1458,7 @@ frb_hash_to_doc_i(VALUE key, VALUE value, VALUE arg)
         return ST_CONTINUE;
     } else {
         FrtDocument *doc = (FrtDocument *)arg;
-        FrtSymbol field = frb_field(key);
+        ID field = frb_field(key);
         VALUE val;
         FrtDocField *df;
         if (NULL == (df = frt_doc_get_field(doc, field))) {
@@ -1455,17 +1474,17 @@ frb_hash_to_doc_i(VALUE key, VALUE value, VALUE arg)
                     df->destroy_data = true;
                     for (i = 0; i < RARRAY_LEN(value); i++) {
                         val = rb_obj_as_string(RARRAY_PTR(value)[i]);
-                        frt_df_add_data_len(df, rstrdup(val), RSTRING_LEN(val));
+                        frt_df_add_data_len(df, rstrdup(val), RSTRING_LEN(val), rb_enc_get(val));
                     }
                 }
                 break;
             case T_STRING:
-                frt_df_add_data_len(df, rs2s(value), RSTRING_LEN(value));
+                frt_df_add_data_len(df, rs2s(value), RSTRING_LEN(value), rb_enc_get(value));
                 break;
             default:
                 val = rb_obj_as_string(value);
                 df->destroy_data = true;
-                frt_df_add_data_len(df, rstrdup(val), RSTRING_LEN(val));
+                frt_df_add_data_len(df, rstrdup(val), RSTRING_LEN(val), rb_enc_get(val));
                 break;
         }
         frt_doc_add_field(doc, df);
@@ -1495,25 +1514,23 @@ frb_get_doc(VALUE rdoc)
                 df->destroy_data = true;
                 for (i = 0; i < RARRAY_LEN(rdoc); i++) {
                     val = rb_obj_as_string(RARRAY_PTR(rdoc)[i]);
-                    frt_df_add_data_len(df, rstrdup(val), RSTRING_LEN(val));
+                    frt_df_add_data_len(df, rstrdup(val), RSTRING_LEN(val), rb_enc_get(val));
                 }
                 frt_doc_add_field(doc, df);
             }
             break;
         case T_SYMBOL:
             /* TODO: clean up this ugly cast */
-            df = frt_df_add_data(frt_df_new(fsym_content), (char *)rb_id2name(SYM2ID(rdoc)));
+            df = frt_df_add_data(frt_df_new(fsym_content), (char *)rb_id2name(SYM2ID(rdoc)), rb_enc_get(rdoc));
             frt_doc_add_field(doc, df);
             break;
         case T_STRING:
-            df = frt_df_add_data_len(frt_df_new(fsym_content), rs2s(rdoc),
-                                 RSTRING_LEN(rdoc));
+            df = frt_df_add_data_len(frt_df_new(fsym_content), rs2s(rdoc), RSTRING_LEN(rdoc), rb_enc_get(rdoc));
             frt_doc_add_field(doc, df);
             break;
         default:
             val = rb_obj_as_string(rdoc);
-            df = frt_df_add_data_len(frt_df_new(fsym_content), rstrdup(val),
-                                 RSTRING_LEN(val));
+            df = frt_df_add_data_len(frt_df_new(fsym_content), rstrdup(val), RSTRING_LEN(val), rb_enc_get(val));
             df->destroy_data = true;
             frt_doc_add_field(doc, df);
             break;
@@ -1575,6 +1592,48 @@ frb_iw_commit(VALUE self)
     return self;
 }
 
+/* index reader intermission */
+static VALUE frb_ir_close(VALUE self);
+
+void frb_ir_free(void *p) {
+    frt_ir_close((FrtIndexReader *)p);
+}
+
+void frb_ir_mark(void *p) {
+    FrtIndexReader *ir = (FrtIndexReader *)p;
+    FrtMultiReader *mr = (FrtMultiReader *)p;
+
+    if (ir->type == FRT_MULTI_READER) {
+        int i;
+        for (i = 0; i < mr->r_cnt; i++) {
+            if (mr->sub_readers[i]->rir)
+                rb_gc_mark(mr->sub_readers[i]->rir);
+        }
+    } else {
+        if (ir->store && ir->store->rstore)
+            rb_gc_mark(ir->store->rstore);
+    }
+}
+
+static size_t frb_index_reader_t_size(const void *p) {
+    return sizeof(FrtMultiReader);
+    (void)p;
+}
+
+const rb_data_type_t frb_index_reader_t = {
+    .wrap_struct_name = "FrbIndexReader",
+    .function = {
+        .dmark = frb_ir_mark,
+        .dfree = frb_ir_free,
+        .dsize = frb_index_reader_t_size,
+        .dcompact = NULL,
+        .reserved = {0},
+    },
+    .parent = NULL,
+    .data = NULL,
+    .flags = RUBY_TYPED_FREE_IMMEDIATELY
+};
+
 /*
  *  call-seq:
  *     iw.add_readers(reader_array) -> iw
@@ -1585,9 +1644,7 @@ frb_iw_commit(VALUE self)
  *  machines. Then you can finish by merging all of the indexes into a single
  *  index.
  */
-static VALUE
-frb_iw_add_readers(VALUE self, VALUE rreaders)
-{
+static VALUE frb_iw_add_readers(VALUE self, VALUE rreaders) {
     FrtIndexWriter *iw = (FrtIndexWriter *)DATA_PTR(self);
     int i;
     FrtIndexReader **irs;
@@ -1597,7 +1654,7 @@ frb_iw_add_readers(VALUE self, VALUE rreaders)
     i = RARRAY_LEN(rreaders);
     while (i-- > 0) {
         FrtIndexReader *ir;
-        Data_Get_Struct(RARRAY_PTR(rreaders)[i], FrtIndexReader, ir);
+        TypedData_Get_Struct(RARRAY_PTR(rreaders)[i], FrtIndexReader, &frb_index_reader_t, ir);
         irs[i] = ir;
     }
     frt_iw_add_readers(iw, irs, RARRAY_LEN(rreaders));
@@ -1943,26 +2000,50 @@ frb_iw_set_use_compound_file(VALUE self, VALUE rval)
  *
  ****************************************************************************/
 
-static void
-frb_lzd_data_free(void *p)
-{
+static void frb_lzd_data_free(void *p) {
     frt_lazy_doc_close((FrtLazyDoc *)p);
 }
 
-static VALUE
-frb_lazy_df_load(VALUE self, VALUE rkey, FrtLazyDocField *lazy_df)
-{
+static size_t frb_lazy_doc_size(const void *p) {
+    return sizeof(FrtLazyDoc);
+    (void)p;
+}
+
+const rb_data_type_t frb_lazy_doc_t = {
+    .wrap_struct_name = "FrbLazyDoc",
+    .function = {
+        .dmark = NULL,
+        .dfree = frb_lzd_data_free,
+        .dsize = frb_lazy_doc_size,
+        .dcompact = NULL,
+        .reserved = {0},
+    },
+    .parent = NULL,
+    .data = NULL,
+    .flags = RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+static VALUE frb_lzd_alloc(VALUE klass) {
+    FrtLazyDoc *ld = FRT_ALLOC(FrtLazyDoc);
+    return TypedData_Wrap_Struct(klass, &frb_lazy_doc_t, ld);
+}
+
+static VALUE frb_lazy_df_load(VALUE self, VALUE rkey, FrtLazyDocField *lazy_df) {
     VALUE rdata = Qnil;
     if (lazy_df) {
         if (lazy_df->size == 1) {
             char *data = frt_lazy_df_get_data(lazy_df, 0);
-            rdata = rb_str_new(data, lazy_df->len);
+            rdata = rb_str_new(data, lazy_df->data[0].length);
+            rb_enc_associate(rdata, lazy_df->data[0].encoding);
         } else {
             int i;
+            VALUE rstr;
             rdata = rb_ary_new2(lazy_df->size);
             for (i = 0; i < lazy_df->size; i++) {
                 char *data = frt_lazy_df_get_data(lazy_df, i);
-                rb_ary_store(rdata, i, rb_str_new(data, lazy_df->data[i].length));
+                rstr = rb_str_new(data, lazy_df->data[i].length);
+                rb_enc_associate(rstr, lazy_df->data[i].encoding);
+                rb_ary_store(rdata, i, rstr);
             }
         }
         rb_hash_aset(self, rkey, rdata);
@@ -1977,11 +2058,9 @@ frb_lazy_df_load(VALUE self, VALUE rkey, FrtLazyDocField *lazy_df)
  *  This method is used internally to lazily load fields. You should never
  *  really need to call it yourself.
  */
-static VALUE
-frb_lzd_default(VALUE self, VALUE rkey)
-{
+static VALUE frb_lzd_default(VALUE self, VALUE rkey) {
     FrtLazyDoc *lazy_doc = (FrtLazyDoc *)DATA_PTR(rb_ivar_get(self, id_data));
-    FrtSymbol field = frb_field(rkey);
+    ID field = frb_field(rkey);
     VALUE rfield = ID2SYM(field);
 
     return frb_lazy_df_load(self, rfield, frt_lazy_doc_get(lazy_doc, field));
@@ -1995,9 +2074,7 @@ frb_lzd_default(VALUE self, VALUE rkey)
  *  to access any of these fields in the document the field will be loaded.
  *  Try to access any other field an nil will be returned.
  */
-static VALUE
-frb_lzd_fields(VALUE self)
-{
+static VALUE frb_lzd_fields(VALUE self) {
     return rb_ivar_get(self, id_fields);
 }
 
@@ -2007,9 +2084,7 @@ frb_lzd_fields(VALUE self)
  *
  *  Load all unloaded fields in the document from the index.
  */
-static VALUE
-frb_lzd_load(VALUE self)
-{
+static VALUE frb_lzd_load(VALUE self) {
     FrtLazyDoc *lazy_doc = (FrtLazyDoc *)DATA_PTR(rb_ivar_get(self, id_data));
     int i;
     for (i = 0; i < lazy_doc->size; i++) {
@@ -2019,9 +2094,7 @@ frb_lzd_load(VALUE self)
     return self;
 }
 
-VALUE
-frb_get_lazy_doc(FrtLazyDoc *lazy_doc)
-{
+VALUE frb_get_lazy_doc(FrtLazyDoc *lazy_doc) {
     int i;
     VALUE rfields = rb_ary_new2(lazy_doc->size);
 
@@ -2029,7 +2102,7 @@ frb_get_lazy_doc(FrtLazyDoc *lazy_doc)
     self = rb_hash_new();
     OBJSETUP(self, cLazyDoc, T_HASH);
 
-    rdata = Data_Wrap_Struct(cLazyDocData, NULL, &frb_lzd_data_free, lazy_doc);
+    rdata = TypedData_Wrap_Struct(cLazyDocData, &frb_lazy_doc_t, lazy_doc);
     rb_ivar_set(self, id_data, rdata);
 
     for (i = 0; i < lazy_doc->size; i++) {
@@ -2046,32 +2119,6 @@ frb_get_lazy_doc(FrtLazyDoc *lazy_doc)
  *
  ****************************************************************************/
 
-void
-frb_ir_free(void *p)
-{
-    object_del(p);
-    frt_ir_close((FrtIndexReader *)p);
-}
-
-void
-frb_ir_mark(void *p)
-{
-    FrtIndexReader *ir = (FrtIndexReader *)p;
-    frb_gc_mark(ir->store);
-}
-
-static VALUE frb_ir_close(VALUE self);
-
-void
-frb_mr_mark(void *p)
-{
-    FrtMultiReader *mr = (FrtMultiReader *)p;
-    int i;
-    for (i = 0; i < mr->r_cnt; i++) {
-        frb_gc_mark(mr->sub_readers[i]);
-    }
-}
-
 /*
  *  call-seq:
  *     IndexReader.new(dir) -> index_reader
@@ -2098,9 +2145,15 @@ frb_mr_mark(void *p)
  *
  *    iw = IndexReader.new(["/path/to/index1", "/path/to/index2"])
  */
-static VALUE
-frb_ir_init(VALUE self, VALUE rdir)
-{
+
+static VALUE frb_ir_alloc(VALUE rclass) {
+    // allocate for FrtSegmentReader, the largest of the Frt*Reader structs,
+    // FrtIndexReader is part of it and later on its determined what its going to be
+    FrtIndexReader *ir = (FrtIndexReader *)frt_sr_alloc();
+    return TypedData_Wrap_Struct(rclass, &frb_index_reader_t, ir);
+}
+
+static VALUE frb_ir_init(VALUE self, VALUE rdir) {
     FrtStore *store = NULL;
     FrtIndexReader *ir;
     int i;
@@ -2120,7 +2173,7 @@ frb_ir_init(VALUE self, VALUE rdir)
                 switch (TYPE(rdir)) {
                     case T_DATA:
                         if (CLASS_OF(rdir) == cIndexReader) {
-                            Data_Get_Struct(rdir, FrtIndexReader, sub_readers[i]);
+                            TypedData_Get_Struct(rdir, FrtIndexReader, &frb_index_reader_t, sub_readers[i]);
                             FRT_REF(sub_readers[i]);
                             continue;
                         } else if (RTEST(rb_obj_is_kind_of(rdir, cDirectory))) {
@@ -2145,10 +2198,10 @@ frb_ir_init(VALUE self, VALUE rdir)
                                 rs2s(rb_obj_as_string(rdir)));
                         break;
                 }
-                sub_readers[i] = frt_ir_open(store);
+                sub_readers[i] = frt_ir_open(NULL, store);
             }
-            ir = frt_mr_open(sub_readers, reader_cnt);
-            Frt_Wrap_Struct(self, &frb_mr_mark, &frb_ir_free, ir);
+            TypedData_Get_Struct(self, FrtIndexReader, &frb_index_reader_t, ir);
+            ir = frt_mr_open(ir, sub_readers, reader_cnt);
         } else {
             switch (TYPE(rdir)) {
                 case T_DATA:
@@ -2165,25 +2218,28 @@ frb_ir_init(VALUE self, VALUE rdir)
                             rs2s(rb_obj_as_string(rdir)));
                     break;
             }
-            ir = frt_ir_open(store);
-            Frt_Wrap_Struct(self, &frb_ir_mark, &frb_ir_free, ir);
+            TypedData_Get_Struct(self, FrtIndexReader, &frb_index_reader_t, ir);
+            ir = frt_ir_open(ir, store);
         }
-    default:
+    FRT_XCATCHALL
         ex_code = xcontext.excode;
         msg = xcontext.msg;
         FRT_HANDLED();
     FRT_XENDTRY
 
-    if (ex_code && msg) { frb_raise(ex_code, msg); }
+    if (ex_code && msg) {
+        ((struct RData *)(self))->data = NULL;
+        ((struct RData *)(self))->dmark = NULL;
+        ((struct RData *)(self))->dfree = NULL;
+        frb_raise(ex_code, msg);
+    }
 
-    object_add(ir, self);
+    ir->rir = self;
 
     fis = ir->fis;
     for (i = 0; i < fis->size; i++) {
         FrtFieldInfo *fi = fis->fields[i];
-        rb_hash_aset(rfield_num_map,
-                     ID2SYM(fi->name),
-                     INT2FIX(fi->number));
+        rb_hash_aset(rfield_num_map, ID2SYM(fi->name), INT2FIX(fi->number));
     }
     rb_ivar_set(self, id_fld_num_map, rfield_num_map);
 
@@ -2281,8 +2337,9 @@ static VALUE
 frb_ir_close(VALUE self)
 {
     FrtIndexReader *ir = (FrtIndexReader *)DATA_PTR(self);
-    object_del(ir);
-    Frt_Unwrap_Struct(self);
+    ((struct RData *)(self))->data = NULL;
+    ((struct RData *)(self))->dmark = NULL;
+    ((struct RData *)(self))->dfree = NULL;
     frt_ir_close(ir);
     return self;
 }
@@ -2482,7 +2539,7 @@ frb_ir_term_vector(VALUE self, VALUE rdoc_id, VALUE rfield)
 static void
 frb_add_each_tv(void *key, void *value, void *rtvs)
 {
-    rb_hash_aset((VALUE)rtvs, ID2SYM((FrtSymbol)key), frb_get_tv(value));
+    rb_hash_aset((VALUE)rtvs, ID2SYM((ID)key), frb_get_tv(value));
 }
 
 /*
@@ -2769,9 +2826,14 @@ frb_ir_version(VALUE self)
  *                  |                         | want to highlight matches.
  *                  |                         | or print match excerpts a la
  *                  |                         | Google search.
+ *     -------------|-------------------------|------------------------------
+ *     :compression | :no (default)           | Don't compress stored field
+ *                  |                         |
+ *                  | :brotli                 | Compress field using Brotli
  *                  |                         |
- *                  | :compressed             | Store field in compressed
- *                  |                         | format.
+ *                  | :bz2                    | Compress field using BZip2
+ *                  |                         |
+ *                  | :lz4                    | Compress field using LZ4
  *     -------------|-------------------------|------------------------------
  *     :index       | :no                     | Do not make this field
  *                  |                         | searchable.
@@ -2831,7 +2893,7 @@ frb_ir_version(VALUE self)
  *    fi = FieldInfo.new(:created_on, :index => :untokenized_omit_norms,
  *                       :term_vector => :no)
  *
- *    fi = FieldInfo.new(:image, :store => :compressed, :index => :no,
+ *    fi = FieldInfo.new(:image, :store => :yes, :compression => :brotli, :index => :no,
  *                       :term_vector => :no)
  */
 static void
@@ -2841,8 +2903,11 @@ Init_FieldInfo(void)
     sym_index = ID2SYM(rb_intern("index"));
     sym_term_vector = ID2SYM(rb_intern("term_vector"));
 
-    sym_compress = ID2SYM(rb_intern("compress"));
-    sym_compressed = ID2SYM(rb_intern("compressed"));
+    sym_brotli = ID2SYM(rb_intern("brotli"));
+    sym_bz2 = ID2SYM(rb_intern("bz2"));
+    sym_lz4 = ID2SYM(rb_intern("lz4"));
+    // sym_level = ID2SYM(rb_intern("level"));
+    sym_compression = ID2SYM(rb_intern("compression"));
 
     sym_untokenized = ID2SYM(rb_intern("untokenized"));
     sym_omit_norms = ID2SYM(rb_intern("omit_norms"));
@@ -2853,7 +2918,7 @@ Init_FieldInfo(void)
     sym_with_positions_offsets = ID2SYM(rb_intern("with_positions_offsets"));
 
     cFieldInfo = rb_define_class_under(mIndex, "FieldInfo", rb_cObject);
-    rb_define_alloc_func(cFieldInfo, frb_data_alloc);
+    rb_define_alloc_func(cFieldInfo, frb_fi_alloc);
 
     rb_define_method(cFieldInfo, "initialize",  frb_fi_init, -1);
     rb_define_method(cFieldInfo, "name",        frb_fi_name, 0);
@@ -2897,7 +2962,7 @@ Init_FieldInfo(void)
  *    field_infos.add_field(:created_on, :index => :untokenized_omit_norms,
  *                          :term_vector => :no)
  *
- *    field_infos.add_field(:image, :store => :compressed, :index => :no,
+ *    field_infos.add_field(:image, :store => :yes, :compression => :brotli, :index => :no,
  *                          :term_vector => :no)
  *
  *    field_infos.create_index("/path/to/index")
@@ -2913,13 +2978,11 @@ Init_FieldInfo(void)
  *  along. If you add a document to the index which has fields that the index
  *  doesn't know about then the default properties are used for the new field.
  */
-static void
-Init_FieldInfos(void)
-{
+static void Init_FieldInfos(void) {
     Init_FieldInfo();
 
     cFieldInfos = rb_define_class_under(mIndex, "FieldInfos", rb_cObject);
-    rb_define_alloc_func(cFieldInfos, frb_data_alloc);
+    rb_define_alloc_func(cFieldInfos, frb_fis_alloc);
 
     rb_define_method(cFieldInfos, "initialize", frb_fis_init, -1);
     rb_define_method(cFieldInfos, "to_a",       frb_fis_to_a, 0);
@@ -2958,12 +3021,11 @@ Init_FieldInfos(void)
  *    end
  */
 static void
-Init_TermEnum(void)
-{
+Init_TermEnum(void) {
     id_term = rb_intern("@term");
 
     cTermEnum = rb_define_class_under(mIndex, "TermEnum", rb_cObject);
-    rb_define_alloc_func(cTermEnum, frb_data_alloc);
+    rb_define_alloc_func(cTermEnum, frb_te_alloc);
 
     rb_define_method(cTermEnum, "next?",    frb_te_next, 0);
     rb_define_method(cTermEnum, "term",     frb_te_term, 0);
@@ -3007,14 +3069,12 @@ Init_TermEnum(void)
  *      puts "  #{positions.join(', ')}"
  *    end
  */
-static void
-Init_TermDocEnum(void)
-{
+static void Init_TermDocEnum(void) {
     id_fld_num_map = rb_intern("@field_num_map");
     id_field_num = rb_intern("@field_num");
 
     cTermDocEnum = rb_define_class_under(mIndex, "TermDocEnum", rb_cObject);
-    rb_define_alloc_func(cTermDocEnum, frb_data_alloc);
+    rb_define_alloc_func(cTermDocEnum, frb_tde_alloc);
     rb_define_method(cTermDocEnum, "seek",           frb_tde_seek, 2);
     rb_define_method(cTermDocEnum, "seek_term_enum", frb_tde_seek_te, 1);
     rb_define_method(cTermDocEnum, "doc",            frb_tde_doc, 0);
@@ -3027,10 +3087,6 @@ Init_TermDocEnum(void)
     rb_define_method(cTermDocEnum, "to_json",        frb_tde_to_json, -1);
 }
 
-/* rdochack
-cTermVector = rb_define_class_under(mIndex, "TermVector", rb_cObject);
-*/
-
 /*
  *  Document-class: Ferret::Index::TermVector::TVOffsets
  *
@@ -3047,13 +3103,8 @@ cTermVector = rb_define_class_under(mIndex, "TermVector", rb_cObject);
  *
  *  See the Analysis module for more information on setting the offsets.
  */
-static void
-Init_TVOffsets(void)
-{
+static void Init_TVOffsets(void) {
     const char *tv_offsets_class = "TVOffsets";
-    /* rdochack
-    cTVOffsets = rb_define_class_under(cTermVector, "TVOffsets", rb_cObject);
-    */
     cTVOffsets = rb_struct_define(tv_offsets_class, "start", "end", NULL);
     rb_set_class_path(cTVOffsets, cTermVector, tv_offsets_class);
     rb_const_set(mIndex, rb_intern(tv_offsets_class), cTVOffsets);
@@ -3074,13 +3125,8 @@ Init_TVOffsets(void)
  *    tv_term = tv.find {|tvt| tvt.term = "fox"}
  *    offsets = tv_term.positions.collect {|pos| tv.offsets[pos]}
  */
-static void
-Init_TVTerm(void)
-{
+static void Init_TVTerm(void) {
     const char *tv_term_class = "TVTerm";
-    /* rdochack
-    cTVTerm = rb_define_class_under(cTermVector, "TVTerm", rb_cObject);
-    */
     cTVTerm = rb_struct_define(tv_term_class, "text", "freq", "positions", NULL);
     rb_set_class_path(cTVTerm, cTermVector, tv_term_class);
     rb_const_set(mIndex, rb_intern(tv_term_class), cTVTerm);
@@ -3116,15 +3162,9 @@ Init_TVTerm(void)
  *  particular that you need to store both positions and offsets if you want
  *  to associate offsets with particular terms.
  */
-static void
-Init_TermVector(void)
-{
+static void Init_TermVector(void) {
     const char *tv_class = "TermVector";
-    /* rdochack
-    cTermVector = rb_define_class_under(mIndex, "TermVector", rb_cObject);
-    */
-    cTermVector = rb_struct_define(tv_class,
-                                   "field", "terms", "offsets", NULL);
+    cTermVector = rb_struct_define(tv_class, "field", "terms", "offsets", NULL);
     rb_set_class_path(cTermVector, mIndex, tv_class);
     rb_const_set(mIndex, rb_intern(tv_class), cTermVector);
 
@@ -3245,112 +3285,80 @@ Init_TermVector(void)
  *
  *    index_writer.delete(:id, "/path/to/indexed/file")
  */
-void
-Init_IndexWriter(void)
-{
+void Init_IndexWriter(void) {
     id_boost = rb_intern("boost");
 
-    sym_create              = ID2SYM(rb_intern("create"));
-    sym_create_if_missing   = ID2SYM(rb_intern("create_if_missing"));
-    sym_field_infos         = ID2SYM(rb_intern("field_infos"));
+    sym_create            = ID2SYM(rb_intern("create"));
+    sym_create_if_missing = ID2SYM(rb_intern("create_if_missing"));
+    sym_field_infos       = ID2SYM(rb_intern("field_infos"));
 
-    sym_chunk_size          = ID2SYM(rb_intern("chunk_size"));
-    sym_max_buffer_memory   = ID2SYM(rb_intern("max_buffer_memory"));
-    sym_index_interval      = ID2SYM(rb_intern("term_index_interval"));
-    sym_skip_interval       = ID2SYM(rb_intern("doc_skip_interval"));
-    sym_merge_factor        = ID2SYM(rb_intern("merge_factor"));
-    sym_max_buffered_docs   = ID2SYM(rb_intern("max_buffered_docs"));
-    sym_max_merge_docs      = ID2SYM(rb_intern("max_merge_docs"));
-    sym_max_field_length    = ID2SYM(rb_intern("max_field_length"));
-    sym_use_compound_file   = ID2SYM(rb_intern("use_compound_file"));
+    sym_chunk_size        = ID2SYM(rb_intern("chunk_size"));
+    sym_max_buffer_memory = ID2SYM(rb_intern("max_buffer_memory"));
+    sym_index_interval    = ID2SYM(rb_intern("term_index_interval"));
+    sym_skip_interval     = ID2SYM(rb_intern("doc_skip_interval"));
+    sym_merge_factor      = ID2SYM(rb_intern("merge_factor"));
+    sym_max_buffered_docs = ID2SYM(rb_intern("max_buffered_docs"));
+    sym_max_merge_docs    = ID2SYM(rb_intern("max_merge_docs"));
+    sym_max_field_length  = ID2SYM(rb_intern("max_field_length"));
+    sym_use_compound_file = ID2SYM(rb_intern("use_compound_file"));
 
     cIndexWriter = rb_define_class_under(mIndex, "IndexWriter", rb_cObject);
-    rb_define_alloc_func(cIndexWriter, frb_data_alloc);
+    rb_define_alloc_func(cIndexWriter, frb_iw_alloc);
 
     rb_define_const(cIndexWriter, "WRITE_LOCK_TIMEOUT", INT2FIX(1));
     rb_define_const(cIndexWriter, "COMMIT_LOCK_TIMEOUT", INT2FIX(10));
-    rb_define_const(cIndexWriter, "WRITE_LOCK_NAME",
-                    rb_str_new2(FRT_WRITE_LOCK_NAME));
-    rb_define_const(cIndexWriter, "COMMIT_LOCK_NAME",
-                    rb_str_new2(FRT_COMMIT_LOCK_NAME));
-    rb_define_const(cIndexWriter, "DEFAULT_CHUNK_SIZE",
-                    INT2FIX(frt_default_config.chunk_size));
-    rb_define_const(cIndexWriter, "DEFAULT_MAX_BUFFER_MEMORY",
-                    INT2FIX(frt_default_config.max_buffer_memory));
-    rb_define_const(cIndexWriter, "DEFAULT_TERM_INDEX_INTERVAL",
-                    INT2FIX(frt_default_config.index_interval));
-    rb_define_const(cIndexWriter, "DEFAULT_DOC_SKIP_INTERVAL",
-                    INT2FIX(frt_default_config.skip_interval));
-    rb_define_const(cIndexWriter, "DEFAULT_MERGE_FACTOR",
-                    INT2FIX(frt_default_config.merge_factor));
-    rb_define_const(cIndexWriter, "DEFAULT_MAX_BUFFERED_DOCS",
-                    INT2FIX(frt_default_config.max_buffered_docs));
-    rb_define_const(cIndexWriter, "DEFAULT_MAX_MERGE_DOCS",
-                    INT2FIX(frt_default_config.max_merge_docs));
-    rb_define_const(cIndexWriter, "DEFAULT_MAX_FIELD_LENGTH",
-                    INT2FIX(frt_default_config.max_field_length));
-    rb_define_const(cIndexWriter, "DEFAULT_USE_COMPOUND_FILE",
-                    frt_default_config.use_compound_file ? Qtrue : Qfalse);
-
-    rb_define_method(cIndexWriter, "initialize",    frb_iw_init, -1);
-    rb_define_method(cIndexWriter, "doc_count",     frb_iw_get_doc_count, 0);
-    rb_define_method(cIndexWriter, "close",         frb_iw_close, 0);
-    rb_define_method(cIndexWriter, "add_document",  frb_iw_add_doc, 1);
-    rb_define_method(cIndexWriter, "<<",            frb_iw_add_doc, 1);
-    rb_define_method(cIndexWriter, "optimize",      frb_iw_optimize, 0);
-    rb_define_method(cIndexWriter, "commit",        frb_iw_commit, 0);
-    rb_define_method(cIndexWriter, "add_readers",   frb_iw_add_readers, 1);
-    rb_define_method(cIndexWriter, "delete",        frb_iw_delete, 2);
-    rb_define_method(cIndexWriter, "field_infos",   frb_iw_field_infos, 0);
-    rb_define_method(cIndexWriter, "analyzer",      frb_iw_get_analyzer, 0);
-    rb_define_method(cIndexWriter, "analyzer=",     frb_iw_set_analyzer, 1);
-    rb_define_method(cIndexWriter, "version",       frb_iw_version, 0);
-
-    rb_define_method(cIndexWriter, "chunk_size",
-                     frb_iw_get_chunk_size, 0);
-    rb_define_method(cIndexWriter, "chunk_size=",
-                     frb_iw_set_chunk_size, 1);
-
-    rb_define_method(cIndexWriter, "max_buffer_memory",
-                     frb_iw_get_max_buffer_memory, 0);
-    rb_define_method(cIndexWriter, "max_buffer_memory=",
-                     frb_iw_set_max_buffer_memory, 1);
-
-    rb_define_method(cIndexWriter, "term_index_interval",
-                     frb_iw_get_index_interval, 0);
-    rb_define_method(cIndexWriter, "term_index_interval=",
-                     frb_iw_set_index_interval, 1);
-
-    rb_define_method(cIndexWriter, "doc_skip_interval",
-                     frb_iw_get_skip_interval, 0);
-    rb_define_method(cIndexWriter, "doc_skip_interval=",
-                     frb_iw_set_skip_interval, 1);
-
-    rb_define_method(cIndexWriter, "merge_factor",
-                     frb_iw_get_merge_factor, 0);
-    rb_define_method(cIndexWriter, "merge_factor=",
-                     frb_iw_set_merge_factor, 1);
-
-    rb_define_method(cIndexWriter, "max_buffered_docs",
-                     frb_iw_get_max_buffered_docs, 0);
-    rb_define_method(cIndexWriter, "max_buffered_docs=",
-                     frb_iw_set_max_buffered_docs, 1);
-
-    rb_define_method(cIndexWriter, "max_merge_docs",
-                     frb_iw_get_max_merge_docs, 0);
-    rb_define_method(cIndexWriter, "max_merge_docs=",
-                     frb_iw_set_max_merge_docs, 1);
-
-    rb_define_method(cIndexWriter, "max_field_length",
-                     frb_iw_get_max_field_length, 0);
-    rb_define_method(cIndexWriter, "max_field_length=",
-                     frb_iw_set_max_field_length, 1);
-
-    rb_define_method(cIndexWriter, "use_compound_file",
-                     frb_iw_get_use_compound_file, 0);
-    rb_define_method(cIndexWriter, "use_compound_file=",
-                     frb_iw_set_use_compound_file, 1);
+    rb_define_const(cIndexWriter, "WRITE_LOCK_NAME", rb_str_new2(FRT_WRITE_LOCK_NAME));
+    rb_define_const(cIndexWriter, "COMMIT_LOCK_NAME", rb_str_new2(FRT_COMMIT_LOCK_NAME));
+    rb_define_const(cIndexWriter, "DEFAULT_CHUNK_SIZE", INT2FIX(frt_default_config.chunk_size));
+    rb_define_const(cIndexWriter, "DEFAULT_MAX_BUFFER_MEMORY", INT2FIX(frt_default_config.max_buffer_memory));
+    rb_define_const(cIndexWriter, "DEFAULT_TERM_INDEX_INTERVAL", INT2FIX(frt_default_config.index_interval));
+    rb_define_const(cIndexWriter, "DEFAULT_DOC_SKIP_INTERVAL", INT2FIX(frt_default_config.skip_interval));
+    rb_define_const(cIndexWriter, "DEFAULT_MERGE_FACTOR", INT2FIX(frt_default_config.merge_factor));
+    rb_define_const(cIndexWriter, "DEFAULT_MAX_BUFFERED_DOCS", INT2FIX(frt_default_config.max_buffered_docs));
+    rb_define_const(cIndexWriter, "DEFAULT_MAX_MERGE_DOCS", INT2FIX(frt_default_config.max_merge_docs));
+    rb_define_const(cIndexWriter, "DEFAULT_MAX_FIELD_LENGTH", INT2FIX(frt_default_config.max_field_length));
+    rb_define_const(cIndexWriter, "DEFAULT_USE_COMPOUND_FILE", frt_default_config.use_compound_file ? Qtrue : Qfalse);
+
+    rb_define_method(cIndexWriter, "initialize",   frb_iw_init, -1);
+    rb_define_method(cIndexWriter, "doc_count",    frb_iw_get_doc_count, 0);
+    rb_define_method(cIndexWriter, "close",        frb_iw_close, 0);
+    rb_define_method(cIndexWriter, "add_document", frb_iw_add_doc, 1);
+    rb_define_method(cIndexWriter, "<<",           frb_iw_add_doc, 1);
+    rb_define_method(cIndexWriter, "optimize",     frb_iw_optimize, 0);
+    rb_define_method(cIndexWriter, "commit",       frb_iw_commit, 0);
+    rb_define_method(cIndexWriter, "add_readers",  frb_iw_add_readers, 1);
+    rb_define_method(cIndexWriter, "delete",       frb_iw_delete, 2);
+    rb_define_method(cIndexWriter, "field_infos",  frb_iw_field_infos, 0);
+    rb_define_method(cIndexWriter, "analyzer",     frb_iw_get_analyzer, 0);
+    rb_define_method(cIndexWriter, "analyzer=",    frb_iw_set_analyzer, 1);
+    rb_define_method(cIndexWriter, "version",      frb_iw_version, 0);
+
+    rb_define_method(cIndexWriter, "chunk_size",  frb_iw_get_chunk_size, 0);
+    rb_define_method(cIndexWriter, "chunk_size=", frb_iw_set_chunk_size, 1);
+
+    rb_define_method(cIndexWriter, "max_buffer_memory",  frb_iw_get_max_buffer_memory, 0);
+    rb_define_method(cIndexWriter, "max_buffer_memory=", frb_iw_set_max_buffer_memory, 1);
+
+    rb_define_method(cIndexWriter, "term_index_interval",  frb_iw_get_index_interval, 0);
+    rb_define_method(cIndexWriter, "term_index_interval=", frb_iw_set_index_interval, 1);
+
+    rb_define_method(cIndexWriter, "doc_skip_interval",  frb_iw_get_skip_interval, 0);
+    rb_define_method(cIndexWriter, "doc_skip_interval=", frb_iw_set_skip_interval, 1);
 
+    rb_define_method(cIndexWriter, "merge_factor",  frb_iw_get_merge_factor, 0);
+    rb_define_method(cIndexWriter, "merge_factor=", frb_iw_set_merge_factor, 1);
+
+    rb_define_method(cIndexWriter, "max_buffered_docs",  frb_iw_get_max_buffered_docs, 0);
+    rb_define_method(cIndexWriter, "max_buffered_docs=", frb_iw_set_max_buffered_docs, 1);
+
+    rb_define_method(cIndexWriter, "max_merge_docs",  frb_iw_get_max_merge_docs, 0);
+    rb_define_method(cIndexWriter, "max_merge_docs=", frb_iw_set_max_merge_docs, 1);
+
+    rb_define_method(cIndexWriter, "max_field_length",  frb_iw_get_max_field_length, 0);
+    rb_define_method(cIndexWriter, "max_field_length=", frb_iw_set_max_field_length, 1);
+
+    rb_define_method(cIndexWriter, "use_compound_file",  frb_iw_get_use_compound_file, 0);
+    rb_define_method(cIndexWriter, "use_compound_file=", frb_iw_set_use_compound_file, 1);
 }
 
 /*
@@ -3383,18 +3391,16 @@ Init_IndexWriter(void)
  *    doc.values   #=> ["the title", "the content"]
  *    doc.fields   #=> [:title, :content]
  */
-void
-Init_LazyDoc(void)
-{
+void Init_LazyDoc(void) {
     id_fields = rb_intern("@fields");
 
     cLazyDoc = rb_define_class_under(mIndex, "LazyDoc", rb_cHash);
-    rb_define_method(cLazyDoc, "default",   frb_lzd_default, 1);
-    rb_define_method(cLazyDoc, "load",      frb_lzd_load, 0);
-    rb_define_method(cLazyDoc, "fields",    frb_lzd_fields, 0);
+    rb_define_method(cLazyDoc, "default", frb_lzd_default, 1);
+    rb_define_method(cLazyDoc, "load",    frb_lzd_load, 0);
+    rb_define_method(cLazyDoc, "fields",  frb_lzd_fields, 0);
 
     cLazyDocData = rb_define_class_under(cLazyDoc, "LazyDocData", rb_cObject);
-    rb_define_alloc_func(cLazyDocData, frb_data_alloc);
+    rb_define_alloc_func(cLazyDocData, frb_lzd_alloc);
 }
 
 /*
@@ -3407,41 +3413,39 @@ Init_LazyDoc(void)
  *  index, accessing term-vectors or deleting documents by document id. It is
  *  also used internally by IndexSearcher.
  */
-void
-Init_IndexReader(void)
-{
+void Init_IndexReader(void) {
     cIndexReader = rb_define_class_under(mIndex, "IndexReader", rb_cObject);
-    rb_define_alloc_func(cIndexReader, frb_data_alloc);
-    rb_define_method(cIndexReader, "initialize",    frb_ir_init, 1);
-    rb_define_method(cIndexReader, "set_norm",      frb_ir_set_norm, 3);
-    rb_define_method(cIndexReader, "norms",         frb_ir_norms, 1);
-    rb_define_method(cIndexReader, "get_norms_into",frb_ir_get_norms_into, 3);
-    rb_define_method(cIndexReader, "commit",        frb_ir_commit, 0);
-    rb_define_method(cIndexReader, "close",         frb_ir_close, 0);
-    rb_define_method(cIndexReader, "has_deletions?",frb_ir_has_deletions, 0);
-    rb_define_method(cIndexReader, "delete",        frb_ir_delete, 1);
-    rb_define_method(cIndexReader, "deleted?",      frb_ir_is_deleted, 1);
-    rb_define_method(cIndexReader, "max_doc",       frb_ir_max_doc, 0);
-    rb_define_method(cIndexReader, "num_docs",      frb_ir_num_docs, 0);
-    rb_define_method(cIndexReader, "undelete_all",  frb_ir_undelete_all, 0);
-    rb_define_method(cIndexReader, "latest?",       frb_ir_is_latest, 0);
-    rb_define_method(cIndexReader, "get_document",  frb_ir_get_doc, -1);
-    rb_define_method(cIndexReader, "[]",            frb_ir_get_doc, -1);
-    rb_define_method(cIndexReader, "term_vector",   frb_ir_term_vector, 2);
-    rb_define_method(cIndexReader, "term_vectors",  frb_ir_term_vectors, 1);
-    rb_define_method(cIndexReader, "term_docs",     frb_ir_term_docs, 0);
-    rb_define_method(cIndexReader, "term_positions",frb_ir_term_positions, 0);
-    rb_define_method(cIndexReader, "term_docs_for", frb_ir_term_docs_for, 2);
+    rb_define_alloc_func(cIndexReader, frb_ir_alloc);
+    rb_define_method(cIndexReader, "initialize",     frb_ir_init, 1);
+    rb_define_method(cIndexReader, "set_norm",       frb_ir_set_norm, 3);
+    rb_define_method(cIndexReader, "norms",          frb_ir_norms, 1);
+    rb_define_method(cIndexReader, "get_norms_into", frb_ir_get_norms_into, 3);
+    rb_define_method(cIndexReader, "commit",         frb_ir_commit, 0);
+    rb_define_method(cIndexReader, "close",          frb_ir_close, 0);
+    rb_define_method(cIndexReader, "has_deletions?", frb_ir_has_deletions, 0);
+    rb_define_method(cIndexReader, "delete",         frb_ir_delete, 1);
+    rb_define_method(cIndexReader, "deleted?",       frb_ir_is_deleted, 1);
+    rb_define_method(cIndexReader, "max_doc",        frb_ir_max_doc, 0);
+    rb_define_method(cIndexReader, "num_docs",       frb_ir_num_docs, 0);
+    rb_define_method(cIndexReader, "undelete_all",   frb_ir_undelete_all, 0);
+    rb_define_method(cIndexReader, "latest?",        frb_ir_is_latest, 0);
+    rb_define_method(cIndexReader, "get_document",   frb_ir_get_doc, -1);
+    rb_define_method(cIndexReader, "[]",             frb_ir_get_doc, -1);
+    rb_define_method(cIndexReader, "term_vector",    frb_ir_term_vector, 2);
+    rb_define_method(cIndexReader, "term_vectors",   frb_ir_term_vectors, 1);
+    rb_define_method(cIndexReader, "term_docs",      frb_ir_term_docs, 0);
+    rb_define_method(cIndexReader, "term_positions", frb_ir_term_positions, 0);
+    rb_define_method(cIndexReader, "term_docs_for",  frb_ir_term_docs_for, 2);
     rb_define_method(cIndexReader, "term_positions_for", frb_ir_t_pos_for, 2);
-    rb_define_method(cIndexReader, "doc_freq",      frb_ir_doc_freq, 2);
-    rb_define_method(cIndexReader, "terms",         frb_ir_terms, 1);
-    rb_define_method(cIndexReader, "terms_from",    frb_ir_terms_from, 2);
-    rb_define_method(cIndexReader, "term_count",    frb_ir_term_count, 1);
-    rb_define_method(cIndexReader, "fields",        frb_ir_fields, 0);
-    rb_define_method(cIndexReader, "field_names",   frb_ir_fields, 0);
-    rb_define_method(cIndexReader, "field_infos",   frb_ir_field_infos, 0);
+    rb_define_method(cIndexReader, "doc_freq",       frb_ir_doc_freq, 2);
+    rb_define_method(cIndexReader, "terms",          frb_ir_terms, 1);
+    rb_define_method(cIndexReader, "terms_from",     frb_ir_terms_from, 2);
+    rb_define_method(cIndexReader, "term_count",     frb_ir_term_count, 1);
+    rb_define_method(cIndexReader, "fields",         frb_ir_fields, 0);
+    rb_define_method(cIndexReader, "field_names",    frb_ir_fields, 0);
+    rb_define_method(cIndexReader, "field_infos",    frb_ir_field_infos, 0);
     rb_define_method(cIndexReader, "tokenized_fields", frb_ir_tk_fields, 0);
-    rb_define_method(cIndexReader, "version",       frb_ir_version, 0);
+    rb_define_method(cIndexReader, "version",        frb_ir_version, 0);
 }
 
 /* rdoc hack
@@ -3466,9 +3470,7 @@ extern VALUE mFerret = rb_define_module("Ferret");
  *  building tag clouds, creating more-like-this queries, custom highlighting
  *  etc. They are also useful for index browsers.
  */
-void
-Init_Index(void)
-{
+void Init_Index(void) {
     mIndex = rb_define_module_under(mFerret, "Index");
 
     sym_boost     = ID2SYM(rb_intern("boost"));