isomorfeus-ferret 0.12.7 → 0.13.0

data/ext/isomorfeus_ferret_ext/frb_analysis.c

@@ -1,23 +1,16 @@
-#include <locale.h>
 #include "frt_analysis.h"
 #include "isomorfeus_ferret.h"
+#include <ruby.h>
 #include <ruby/re.h>
-#include <ruby/st.h>
-
-static char *frb_locale = NULL;
 
 static VALUE mAnalysis;
 
 static VALUE cToken;
-static VALUE cAsciiLetterTokenizer;
 static VALUE cLetterTokenizer;
-static VALUE cAsciiWhiteSpaceTokenizer;
 static VALUE cWhiteSpaceTokenizer;
-static VALUE cAsciiStandardTokenizer;
 static VALUE cStandardTokenizer;
 static VALUE cRegExpTokenizer;
 
-static VALUE cAsciiLowerCaseFilter;
 static VALUE cLowerCaseFilter;
 static VALUE cStopFilter;
 static VALUE cMappingFilter;
@@ -25,11 +18,8 @@ static VALUE cHyphenFilter;
 static VALUE cStemFilter;
 
 static VALUE cAnalyzer;
-static VALUE cAsciiLetterAnalyzer;
 static VALUE cLetterAnalyzer;
-static VALUE cAsciiWhiteSpaceAnalyzer;
 static VALUE cWhiteSpaceAnalyzer;
-static VALUE cAsciiStandardAnalyzer;
 static VALUE cStandardAnalyzer;
 static VALUE cPerFieldAnalyzer;
 static VALUE cRegExpAnalyzer;
@@ -47,12 +37,10 @@ static ID id_token_stream;
 
 static VALUE object_space;
 
-extern int ruby_re_search(struct re_pattern_buffer *, const char *, int, int,
-                          int, struct re_registers *);
+extern rb_encoding *utf8_encoding;
+extern int ruby_re_search(struct re_pattern_buffer *, const char *, int, int, int, struct re_registers *);
 
-int
-frb_rb_hash_size(VALUE hash)
-{
+int frb_rb_hash_size(VALUE hash) {
 #ifdef RHASH_SIZE
     return RHASH_SIZE(hash);
 #else
@@ -66,9 +54,7 @@ frb_rb_hash_size(VALUE hash)
  *
  ****************************************************************************/
 
-static char **
-get_stopwords(VALUE rstop_words)
-{
+static char **get_stopwords(VALUE rstop_words) {
     char **stop_words;
     int i, len;
     VALUE rstr;
@@ -89,60 +75,43 @@ get_stopwords(VALUE rstop_words)
  *
  ****************************************************************************/
 
-typedef struct RToken {
-    VALUE text;
-    int start;
-    int end;
-    int pos_inc;
-} RToken;
-
-static void
-frb_token_free(void *p)
-{
+static void frb_token_free(void *p) {
     free(p);
 }
 
-static void
-frb_token_mark(void *p)
-{
-    RToken *token = (RToken *)p;
-    rb_gc_mark(token->text);
-}
-
-static VALUE
-frb_token_alloc(VALUE klass)
-{
-    return Data_Wrap_Struct(klass, &frb_token_mark, &frb_token_free,
-                            ALLOC(RToken));
-}
-
-static VALUE
-get_token(FrtToken *tk)
-{
-    RToken *token = ALLOC(RToken);
-
-    token->text = rb_str_new2(tk->text);
-    token->start = tk->start;
-    token->end = tk->end;
-    token->pos_inc = tk->pos_inc;
-    return Data_Wrap_Struct(cToken, &frb_token_mark, &frb_token_free, token);
-}
-
-FrtToken *
-frb_set_token(FrtToken *tk, VALUE rt)
-{
-    RToken *rtk;
-
-    if (rt == Qnil) return NULL;
-
-    Data_Get_Struct(rt, RToken, rtk);
-    frt_tk_set(tk, rs2s(rtk->text), RSTRING_LEN(rtk->text),
-           rtk->start, rtk->end, rtk->pos_inc);
+static size_t frb_token_size(const void *p) {
+    return sizeof(FrtToken);
+    (void)p;
+}
+
+const rb_data_type_t frb_token_t = {
+    .wrap_struct_name = "FrbToken",
+    .function = {
+        .dmark = NULL,
+        .dfree = frb_token_free,
+        .dsize = frb_token_size,
+        .dcompact = NULL,
+        .reserved = {0},
+    },
+    .parent = NULL,
+    .data = NULL,
+    .flags = RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+static VALUE frb_token_alloc(VALUE rclass) {
+    FrtToken *tk = frt_tk_new();
+    return TypedData_Wrap_Struct(rclass, &frb_token_t, tk);
+}
+
+FrtToken *frb_set_token(FrtToken *tk, VALUE rt) {
+    FrtToken *rtk;
+    if (rt == Qnil)
+        return NULL;
+    TypedData_Get_Struct(rt, FrtToken, &frb_token_t, rtk);
+    frt_tk_set(tk, rtk->text, rtk->len, rtk->start, rtk->end, rtk->pos_inc, utf8_encoding);
     return tk;
 }
 
-#define GET_TK(tk, self) Data_Get_Struct(self, RToken, tk)
-
 /*
  *  call-seq:
  *     Token.new(text, start, end, pos_inc = 1) -> new Token
@@ -174,21 +143,17 @@ frb_set_token(FrtToken *tk, VALUE rt)
  *  pos_inc::    the position increment of a token. See above.
  *  return::     a newly created and assigned Token object
  */
-static VALUE
-frb_token_init(int argc, VALUE *argv, VALUE self)
-{
-    RToken *token;
-    VALUE rtext, rstart, rend, rpos_inc, rtype;
-    GET_TK(token, self);
-    token->pos_inc = 1;
-    switch (rb_scan_args(argc, argv, "32", &rtext, &rstart,
-                         &rend, &rpos_inc, &rtype)) {
-        case 5: /* type gets ignored at this stage */
-        case 4: token->pos_inc = FIX2INT(rpos_inc);
+static VALUE frb_token_init(int argc, VALUE *argv, VALUE self) {
+    FrtToken *tk;
+    char *text;
+    int pos_inc = 1;
+    VALUE rtext, rstart, rend, rpos_inc;
+    switch (rb_scan_args(argc, argv, "31", &rtext, &rstart, &rend, &rpos_inc)) {
+        case 4: pos_inc = FIX2INT(rpos_inc);
     }
-    token->text = rb_obj_as_string(rtext);
-    token->start = FIX2INT(rstart);
-    token->end = FIX2INT(rend);
+    TypedData_Get_Struct(self, FrtToken, &frb_token_t, tk);
+    text = rs2s(rtext);
+    frt_tk_set(tk, text, strlen(text), FIX2INT(rstart), FIX2INT(rend), pos_inc, rb_enc_get(rtext));
     return self;
 }
 
@@ -204,27 +169,11 @@ frb_token_init(int argc, VALUE *argv, VALUE self)
  *  pos_inc=) then, they are sorted by the end offset and then
  *  lexically by the token text.
  */
-static VALUE
-frb_token_cmp(VALUE self, VALUE rother)
-{
-    RToken *token, *other;
-    int cmp;
-    GET_TK(token, self);
-    GET_TK(other, rother);
-    if (token->start > other->start) {
-        cmp = 1;
-    } else if (token->start < other->start) {
-        cmp = -1;
-    } else {
-        if (token->end > other->end) {
-            cmp = 1;
-        } else if (token->end < other->end) {
-            cmp = -1;
-        } else {
-            cmp = strcmp(rs2s(token->text), rs2s(other->text));
-        }
-    }
-    return INT2FIX(cmp);
+static VALUE frb_token_cmp(VALUE self, VALUE rother) {
+    FrtToken *tk, *other;
+    TypedData_Get_Struct(self, FrtToken, &frb_token_t, tk);
+    TypedData_Get_Struct(rother, FrtToken, &frb_token_t, other);
+    return INT2FIX(frt_tk_cmp(tk, other));
 }
 
 /*
@@ -233,12 +182,12 @@ frb_token_cmp(VALUE self, VALUE rother)
  *
  *  Returns the text that this token represents
  */
-static VALUE
-frb_token_get_text(VALUE self)
-{
-    RToken *token;
-    GET_TK(token, self);
-    return token->text;
+static VALUE frb_token_get_text(VALUE self) {
+    FrtToken *tk;
+    TypedData_Get_Struct(self, FrtToken, &frb_token_t, tk);
+    VALUE rtext = rb_str_new2(tk->text);
+    rb_enc_associate(rtext, utf8_encoding);
+    return rtext;
 }
 
 /*
@@ -247,13 +196,12 @@ frb_token_get_text(VALUE self)
  *
  *  Set the text for this token.
  */
-static VALUE
-frb_token_set_text(VALUE self, VALUE rtext)
-{
-    RToken *token;
-    GET_TK(token, self);
-    token->text = rtext;
-    return rtext;
+static VALUE frb_token_set_text(VALUE self, VALUE rtext) {
+    FrtToken *tk;
+    TypedData_Get_Struct(self, FrtToken, &frb_token_t, tk);
+    char *text = rs2s(rtext);
+    frt_tk_set(tk, text, strlen(text), tk->start, tk->end, tk->pos_inc, rb_enc_get(rtext));
+    return frb_token_get_text(self);
 }
 
 /*
@@ -262,12 +210,10 @@ frb_token_set_text(VALUE self, VALUE rtext)
  *
  *  Start byte-position of this token
  */
-static VALUE
-frb_token_get_start_offset(VALUE self)
-{
-    RToken *token;
-    GET_TK(token, self);
-    return INT2FIX(token->start);
+static VALUE frb_token_get_start_offset(VALUE self) {
+    FrtToken *tk;
+    TypedData_Get_Struct(self, FrtToken, &frb_token_t, tk);
+    return INT2FIX(tk->start);
 }
 
 /*
@@ -276,12 +222,10 @@ frb_token_get_start_offset(VALUE self)
  *
  *  End byte-position of this token
  */
-static VALUE
-frb_token_get_end_offset(VALUE self)
-{
-    RToken *token;
-    GET_TK(token, self);
-    return INT2FIX(token->end);
+static VALUE frb_token_get_end_offset(VALUE self) {
+    FrtToken *tk;
+    TypedData_Get_Struct(self, FrtToken, &frb_token_t, tk);
+    return INT2FIX(tk->end);
 }
 
 /*
@@ -290,12 +234,10 @@ frb_token_get_end_offset(VALUE self)
  *
  *  Position Increment for this token
  */
-static VALUE
-frb_token_get_pos_inc(VALUE self)
-{
-    RToken *token;
-    GET_TK(token, self);
-    return INT2FIX(token->pos_inc);
+static VALUE frb_token_get_pos_inc(VALUE self) {
+    FrtToken *tk;
+    TypedData_Get_Struct(self, FrtToken, &frb_token_t, tk);
+    return INT2FIX(tk->pos_inc);
 }
 
 /*
@@ -304,12 +246,10 @@ frb_token_get_pos_inc(VALUE self)
  *
  *  Set start byte-position of this token
  */
-static VALUE
-frb_token_set_start_offset(VALUE self, VALUE rstart)
-{
-    RToken *token;
-    GET_TK(token, self);
-    token->start = FIX2INT(rstart);
+static VALUE frb_token_set_start_offset(VALUE self, VALUE rstart) {
+    FrtToken *tk;
+    TypedData_Get_Struct(self, FrtToken, &frb_token_t, tk);
+    tk->start = FIX2INT(rstart);
     return rstart;
 }
 
@@ -319,12 +259,10 @@ frb_token_set_start_offset(VALUE self, VALUE rstart)
  *
  *  Set end byte-position of this token
  */
-static VALUE
-frb_token_set_end_offset(VALUE self, VALUE rend)
-{
-    RToken *token;
-    GET_TK(token, self);
-    token->end = FIX2INT(rend);
+static VALUE frb_token_set_end_offset(VALUE self, VALUE rend) {
+    FrtToken *tk;
+    TypedData_Get_Struct(self, FrtToken, &frb_token_t, tk);
+    tk->end = FIX2INT(rend);
     return rend;
 }
 
@@ -355,12 +293,10 @@ frb_token_set_end_offset(VALUE self, VALUE rend)
  *    when the terms occur with no intervening stop words.
  *
  */
-static VALUE
-frb_token_set_pos_inc(VALUE self, VALUE rpos_inc)
-{
-    RToken *token;
-    GET_TK(token, self);
-    token->pos_inc = FIX2INT(rpos_inc);
+static VALUE frb_token_set_pos_inc(VALUE self, VALUE rpos_inc) {
+    FrtToken *tk;
+    TypedData_Get_Struct(self, FrtToken, &frb_token_t, tk);
+    tk->pos_inc = FIX2INT(rpos_inc);
     return rpos_inc;
 }
 
@@ -370,16 +306,16 @@ frb_token_set_pos_inc(VALUE self, VALUE rpos_inc)
  *
  *  Return a string representation of the token
  */
-static VALUE
-frb_token_to_s(VALUE self)
-{
-    RToken *token;
+static VALUE frb_token_to_s(VALUE self) {
+    FrtToken *tk;
     char *buf;
-    GET_TK(token, self);
-    buf = alloca(RSTRING_LEN(token->text) + 80);
-    sprintf(buf, "token[\"%s\":%d:%d:%d]", rs2s(token->text),
-            token->start, token->end, token->pos_inc);
-    return rb_str_new2(buf);
+    VALUE rstr;
+    TypedData_Get_Struct(self, FrtToken, &frb_token_t, tk);
+    buf = alloca(strlen(tk->text) + 80);
+    sprintf(buf, "token[\"%s\":%d:%d:%d]", tk->text, (int)tk->start, (int)tk->end, tk->pos_inc);
+    rstr = rb_str_new2(buf);
+    rb_enc_associate(rstr, utf8_encoding);
+    return rstr;
 }
 
 /****************************************************************************
@@ -388,57 +324,77 @@ frb_token_to_s(VALUE self)
  *
  ****************************************************************************/
 
-#define GET_TS(ts, self) Data_Get_Struct(self, FrtTokenStream, ts)
-
-static void
-frb_ts_mark(void *p)
-{
-    FrtTokenStream *ts = (FrtTokenStream *)p;
-    if (ts->text)   frb_gc_mark(&ts->text);
+static void frb_ts_free(void *p) {
+    frt_ts_deref((FrtTokenStream *)p);
 }
 
-static void
-frb_ts_free(FrtTokenStream *ts)
-{
-    if (object_get(&ts->text) != Qnil) {
-        object_del(&ts->text);
-    }
-    object_del(ts);
-    frt_ts_deref(ts);
+static size_t frb_ts_size(const void *p) {
+    return sizeof(FrtTokenStream);
+    (void)p;
 }
 
-static void frb_rets_free(FrtTokenStream *ts);
-static void frb_rets_mark(FrtTokenStream *ts);
+typedef struct RegExpTokenStream {
+    FrtTokenStream super;
+    VALUE regex;
+    VALUE proc;
+    long  curr_ind;
+} RegExpTokenStream;
+
+static void frb_rets_mark(void *p) {
+    RegExpTokenStream *ts = (RegExpTokenStream *)p;
+    rb_gc_mark(ts->regex);
+    rb_gc_mark(ts->proc);
+}
+
+static size_t frb_rets_size(const void *p) {
+    return sizeof(RegExpTokenStream);
+    (void)p;
+}
+
+const rb_data_type_t frb_token_stream_t = {
+    .wrap_struct_name = "FrbTokenStream",
+    .function = {
+        .dmark = NULL,
+        .dfree = frb_ts_free,
+        .dsize = frb_ts_size,
+        .dcompact = NULL,
+        .reserved = {0},
+    },
+    .parent = NULL,
+    .data = NULL,
+    .flags = RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+const rb_data_type_t frb_reg_exp_token_stream_t = {
+    .wrap_struct_name = "FrbRegExpTokenStream",
+    .function = {
+        .dmark = frb_rets_mark,
+        .dfree = frb_ts_free,
+        .dsize = frb_rets_size,
+        .dcompact = NULL,
+        .reserved = {0},
+    },
+    .parent = &frb_token_stream_t,
+    .data = NULL,
+    .flags = RUBY_TYPED_FREE_IMMEDIATELY
+};
+
 static FrtToken *rets_next(FrtTokenStream *ts);
 
-static VALUE
-get_rb_token_stream(FrtTokenStream *ts)
-{
-    VALUE rts = object_get(ts);
-    if (rts == Qnil) {
+static VALUE get_rb_token_stream(FrtTokenStream *ts) {
+    VALUE rts = ts->rts;
+
+    if (rts == 0 || rts == Qnil) {
         if (ts->next == &rets_next) {
-            rts = Data_Wrap_Struct(cTokenStream, &frb_rets_mark,
-                                   &frb_rets_free, ts);
+            rts = TypedData_Wrap_Struct(cTokenStream, &frb_reg_exp_token_stream_t, ts);
         } else {
-            rts = Data_Wrap_Struct(cTokenStream, &frb_ts_mark,
-                                   &frb_ts_free, ts);
+            rts = TypedData_Wrap_Struct(cTokenStream, &frb_token_stream_t, ts);
         }
-        object_add(ts, rts);
+        ts->rts = rts;
     }
     return rts;
 }
 
-static VALUE
-get_wrapped_ts(VALUE self, VALUE rstr, FrtTokenStream *ts)
-{
-    StringValue(rstr);
-    ts->reset(ts, rs2s(rstr));
-    Frt_Wrap_Struct(self, &frb_ts_mark, &frb_ts_free, ts);
-    object_add(&ts->text, rstr);
-    object_add(ts, self);
-    return self;
-}
-
 /*
  *  call-seq:
  *     token_stream.text = text -> text
@@ -448,17 +404,12 @@ get_wrapped_ts(VALUE self, VALUE rstr, FrtTokenStream *ts)
  *
  *      token_stream.text = File.read(file_name)
  */
-static VALUE
-frb_ts_set_text(VALUE self, VALUE rtext)
-{
+static VALUE frb_ts_set_text(VALUE self, VALUE rtext) {
     FrtTokenStream *ts;
-    Data_Get_Struct(self, FrtTokenStream, ts);
+    // TypedData_Get_Struct(self, FrtTokenStream, &frb_token_stream_t, ts);
+    ts = DATA_PTR(self);
     StringValue(rtext);
-    ts->reset(ts, rs2s(rtext));
-
-    /* prevent garbage collection */
-    rb_ivar_set(self, id_text, rtext);
-
+    ts->reset(ts, rs2s(rtext), rb_enc_get(rtext));
     return rtext;
 }
 
@@ -468,17 +419,14 @@ frb_ts_set_text(VALUE self, VALUE rtext)
  *
  *  Return the text that the TokenStream is tokenizing
  */
-static VALUE
-frb_ts_get_text(VALUE self)
-{
+static VALUE frb_ts_get_text(VALUE self) {
     VALUE rtext = Qnil;
     FrtTokenStream *ts;
-    Data_Get_Struct(self, FrtTokenStream, ts);
-    if ((rtext = object_get(&ts->text)) == Qnil) {
-        if (ts->text) {
-            rtext = rb_str_new2(ts->text);
-            object_set(&ts->text, rtext);
-        }
+    // TypedData_Get_Struct(self, FrtTokenStream, &frb_token_stream_t, ts);
+    ts = DATA_PTR(self);
+    if (ts->text) {
+        rtext = rb_str_new2(ts->text);
+        rb_enc_associate(rtext, ts->encoding);
     }
     return rtext;
 }
@@ -490,18 +438,16 @@ frb_ts_get_text(VALUE self)
  *  Return the next token from the TokenStream or nil if there are no more
  *  tokens.
  */
-static VALUE
-frb_ts_next(VALUE self)
-{
-    FrtTokenStream *ts;
-    FrtToken *next;
-    GET_TS(ts, self);
-    next = ts->next(ts);
-    if (next == NULL) {
+static VALUE frb_ts_next(VALUE self) {
+    FrtTokenStream *ts = DATA_PTR(self);
+    FrtToken *next_tk;
+    FrtToken *tk = ts->next(ts);
+    if (tk == NULL) {
         return Qnil;
     }
-
-    return get_token(next);
+    next_tk = frt_tk_new();
+    frt_tk_set(next_tk, tk->text, tk->len, tk->start, tk->end, tk->pos_inc, utf8_encoding);
+    return TypedData_Wrap_Struct(cToken, &frb_token_t, next_tk);
 }
 
 /****************************************************************************
@@ -510,83 +456,73 @@ frb_ts_next(VALUE self)
 
 #define TkFilt(filter) ((FrtTokenFilter *)(filter))
 
-static void
-frb_tf_mark(void *p)
-{
+static void frb_tf_mark(void *p) {
     FrtTokenStream *ts = (FrtTokenStream *)p;
-    if (TkFilt(ts)->sub_ts) {
-        frb_gc_mark(&TkFilt(ts)->sub_ts);
-    }
+    if (TkFilt(ts)->sub_ts->rts)
+        rb_gc_mark(TkFilt(ts)->sub_ts->rts);
 }
 
-static void
-frb_tf_free(FrtTokenStream *ts)
-{
-    if (TkFilt(ts)->sub_ts && (object_get(&TkFilt(ts)->sub_ts) != Qnil)) {
-        object_del(&TkFilt(ts)->sub_ts);
-    }
-    object_del(ts);
-    frt_ts_deref(ts);
+static void frb_tf_free(void *p) {
+    frt_ts_deref((FrtTokenStream *)p);
 }
 
+static size_t frb_tf_size(const void *p) {
+    return sizeof(FrtTokenFilter);
+    (void)p;
+}
+
+const rb_data_type_t frb_token_filter_t = {
+    .wrap_struct_name = "FrbTokenFilter",
+    .function = {
+        .dmark = frb_tf_mark,
+        .dfree = frb_tf_free,
+        .dsize = frb_tf_size,
+        .dcompact = NULL,
+        .reserved = {0},
+    },
+    .parent = NULL,
+    .data = NULL,
+    .flags = RUBY_TYPED_FREE_IMMEDIATELY
+};
 
 /****************************************************************************
- * CWrappedTokenStream
+ * Wrapped TokenStream
  ****************************************************************************/
 
-#define CachedTS(token_stream) ((FrtCachedTokenStream *)(token_stream))
-#define CWTS(token_stream) ((CWrappedTokenStream *)(token_stream))
-
-typedef struct CWrappedTokenStream {
-    FrtCachedTokenStream super;
-    VALUE rts;
-} CWrappedTokenStream;
-
-static void
-cwrts_destroy_i(FrtTokenStream *ts)
-{
-    if (object_get(&ts->text) != Qnil) {
-        object_del(&ts->text);
-    }
+static void cwrts_destroy_i(FrtTokenStream *ts) {
     rb_hash_delete(object_space, ((VALUE)ts)|1);
     free(ts);
 }
 
-static FrtToken *
-cwrts_next(FrtTokenStream *ts)
-{
-    VALUE rtoken = rb_funcall(CWTS(ts)->rts, id_next, 0);
-    return frb_set_token(&(CachedTS(ts)->token), rtoken);
+static FrtToken *cwrts_next(FrtTokenStream *ts) {
+    VALUE rtoken = rb_funcall(ts->rts, id_next, 0);
+    return frb_set_token(&(ts->token), rtoken);
 }
 
-static FrtTokenStream *
-cwrts_reset(FrtTokenStream *ts, char *text)
-{
+static FrtTokenStream *cwrts_reset(FrtTokenStream *ts, char *text, rb_encoding *encoding) {
     ts->t = ts->text = text;
-    rb_funcall(CWTS(ts)->rts, id_reset, 1, rb_str_new2(text));
+    ts->length = strlen(text);
+    ts->encoding = encoding;
+    rb_funcall(ts->rts, id_reset, 1, rb_str_new2(text));
     return ts;
 }
 
-static FrtTokenStream *
-cwrts_clone_i(FrtTokenStream *orig_ts)
-{
-    FrtTokenStream *new_ts = frt_ts_clone_size(orig_ts, sizeof(CWrappedTokenStream));
-    VALUE rts = CWTS(new_ts)->rts = rb_funcall(CWTS(orig_ts)->rts, id_clone, 0);
+static FrtTokenStream *cwrts_clone_i(FrtTokenStream *orig_ts) {
+    FrtTokenStream *new_ts = frt_ts_clone_size(orig_ts, sizeof(FrtTokenStream));
+    VALUE rts = new_ts->rts = rb_funcall(orig_ts->rts, id_clone, 0);
     rb_hash_aset(object_space, ((VALUE)new_ts)|1, rts);
     return new_ts;
 }
 
-static FrtTokenStream *
-frb_get_cwrapped_rts(VALUE rts)
-{
+static FrtTokenStream *frb_get_cwrapped_rts(VALUE rts) {
     FrtTokenStream *ts;
     if (frb_is_cclass(rts) && DATA_PTR(rts)) {
-        GET_TS(ts, rts);
+        // TypedData_Get_Struct(rts, FrtTokenStream, &frb_token_stream_t, ts);
+        ts = DATA_PTR(rts);
         FRT_REF(ts);
-    }
-    else {
-        ts = frt_ts_new(CWrappedTokenStream);
-        CWTS(ts)->rts = rts;
+    } else {
+        ts = frt_ts_new_i(sizeof(FrtTokenStream));
+        ts->rts = rts;
         ts->next = &cwrts_next;
         ts->reset = &cwrts_reset;
         ts->clone_i = &cwrts_clone_i;
@@ -619,43 +555,11 @@ static const char *TOKEN_RE =
     ")";
 static VALUE rtoken_re;
 
-typedef struct RegExpTokenStream {
-    FrtCachedTokenStream super;
-    VALUE rtext;
-    VALUE regex;
-    VALUE proc;
-    long   curr_ind;
-} RegExpTokenStream;
-
-static void
-rets_destroy_i(FrtTokenStream *ts)
-{
-    if (object_get(&ts->text) != Qnil) {
-        object_del(&ts->text);
-    }
+static void rets_destroy_i(FrtTokenStream *ts) {
     rb_hash_delete(object_space, ((VALUE)ts)|1);
     free(ts);
 }
 
-static void
-frb_rets_free(FrtTokenStream *ts)
-{
-    if (object_get(&ts->text) != Qnil) {
-        object_del(&ts->text);
-    }
-    object_del(ts);
-    frt_ts_deref(ts);
-}
-
-static void
-frb_rets_mark(FrtTokenStream *ts)
-{
-    if (ts->text)   frb_gc_mark(&ts->text);
-    rb_gc_mark(RETS(ts)->rtext);
-    rb_gc_mark(RETS(ts)->regex);
-    rb_gc_mark(RETS(ts)->proc);
-}
-
 /*
  *  call-seq:
  *     tokenizer.text = text -> text
@@ -663,17 +567,11 @@ frb_rets_mark(FrtTokenStream *ts)
  *  Set the text to be tokenized by the tokenizer. The tokenizer gets reset to
  *  tokenize the text from the beginning.
  */
-static VALUE
-frb_rets_set_text(VALUE self, VALUE rtext)
-{
+static VALUE frb_rets_set_text(VALUE self, VALUE rtext) {
     FrtTokenStream *ts;
-    GET_TS(ts, self);
-
-    rb_hash_aset(object_space, ((VALUE)ts)|1, rtext);
+    TypedData_Get_Struct(self, FrtTokenStream, &frb_reg_exp_token_stream_t, ts);
     StringValue(rtext);
-    RETS(ts)->rtext = rtext;
-    RETS(ts)->curr_ind = 0;
-
+    ts->reset(ts, rs2s(rtext), rb_enc_get(rtext));
     return rtext;
 }
 
@@ -683,12 +581,12 @@ frb_rets_set_text(VALUE self, VALUE rtext)
  *
  *  Get the text being tokenized by the tokenizer.
  */
-static VALUE
-frb_rets_get_text(VALUE self)
-{
+static VALUE frb_rets_get_text(VALUE self) {
     FrtTokenStream *ts;
-    GET_TS(ts, self);
-    return RETS(ts)->rtext;
+    TypedData_Get_Struct(self, FrtTokenStream, &frb_reg_exp_token_stream_t, ts);
+    VALUE rstr = rb_str_new2(ts->text);
+    rb_enc_associate(rstr, ts->encoding);
+    return rstr;
 }
 
 // partly lifted from ruby 1.9 string.c
@@ -696,92 +594,84 @@ frb_rets_get_text(VALUE self)
 #define BEG(no) regs->beg[no]
 #define END(no) regs->end[no]
 #define STR_ENC_GET(str) rb_enc_from_index(ENCODING_GET(str))
-static VALUE
-  scan_once(VALUE str, VALUE pat, long *start)
-{
-  VALUE match;
-  struct re_registers *regs;
-
-  if (rb_reg_search(pat, str, *start, 0) >= 0) {
-    match = rb_backref_get();
-    regs = RMATCH_REGS(match);
-    if (BEG(0) == END(0)) {
-      rb_encoding *enc = STR_ENC_GET(str);
-      /*
-      * Always consume at least one character of the input string
-       */
-        if (RSTRING_LEN(str) > END(0))
-        *start = END(0)+rb_enc_mbclen(RSTRING_PTR(str)+END(0),
-        RSTRING_END(str), enc);
-      else
-        *start = END(0)+1;
-    }
-    else {
-      *start = END(0);
+static VALUE scan_once(VALUE rstr, VALUE pat, long *start) {
+    VALUE match;
+    struct re_registers *regs;
+    if (rb_reg_search(pat, rstr, *start, 0) >= 0) {
+        match = rb_backref_get();
+        regs = RMATCH_REGS(match);
+        if (BEG(0) == END(0)) {
+            rb_encoding *enc = STR_ENC_GET(rstr);
+            /* Always consume at least one character of the input string */
+            if (RSTRING_LEN(rstr) > END(0))
+                *start = END(0)+rb_enc_mbclen(RSTRING_PTR(rstr)+END(0), RSTRING_END(rstr), enc);
+            else
+                *start = END(0)+1;
+        } else {
+            *start = END(0);
+        }
+        return rb_reg_nth_match(0, match);
     }
-    return rb_reg_nth_match(0, match);
-  }
   return Qnil;
 }
-//
 
-static FrtToken *
-  rets_next(FrtTokenStream *ts)
-{
-  VALUE ret;
-  long rtok_len;
-  int beg, end;
-  Check_Type(RETS(ts)->regex, T_REGEXP);
-  ret = scan_once(RETS(ts)->rtext, RETS(ts)->regex, &(RETS(ts)->curr_ind));
-  if (NIL_P(ret)) return NULL;
-
-  Check_Type(ret, T_STRING);
-  rtok_len = RSTRING_LEN(ret);
-  beg = RETS(ts)->curr_ind - rtok_len;
-  end = RETS(ts)->curr_ind;
-
-  if (NIL_P(RETS(ts)->proc)) {
-    return frt_tk_set(&(CachedTS(ts)->token), rs2s(ret), rtok_len,
-      beg, end, 1);
-  } else {
-    VALUE rtok;
-    rtok = rb_funcall(RETS(ts)->proc, id_call, 1, ret);
-    return frt_tk_set(&(CachedTS(ts)->token), rs2s(rtok),
-      RSTRING_LEN(rtok), beg, end, 1);
-  }
-}
-
-static FrtTokenStream *
-rets_reset(FrtTokenStream *ts, char *text)
-{
-    RETS(ts)->rtext = rb_str_new2(text);
+static FrtToken *rets_next(FrtTokenStream *ts) {
+    VALUE ret;
+    long rtok_len;
+    int beg, end;
+    Check_Type(RETS(ts)->regex, T_REGEXP);
+
+    VALUE rstr = rb_str_new_static(ts->text, ts->length);
+    rb_enc_associate(rstr, ts->encoding);
+
+    ret = scan_once(rstr, RETS(ts)->regex, &(RETS(ts)->curr_ind));
+
+    if (NIL_P(ret))
+        return NULL;
+
+    Check_Type(ret, T_STRING);
+    rtok_len = RSTRING_LEN(ret);
+    beg = RETS(ts)->curr_ind - rtok_len;
+    end = RETS(ts)->curr_ind;
+
+    if (NIL_P(RETS(ts)->proc)) {
+        return frt_tk_set(&(ts->token), rs2s(ret), rtok_len, beg, end, 1, rb_enc_get(ret));
+    } else {
+        VALUE rtok;
+        rtok = rb_funcall(RETS(ts)->proc, id_call, 1, ret);
+        return frt_tk_set(&(ts->token), rs2s(rtok), RSTRING_LEN(rtok), beg, end, 1, rb_enc_get(rtok));
+    }
+}
+
+static FrtTokenStream *rets_reset(FrtTokenStream *ts, char *text, rb_encoding *encoding) {
+    frt_ts_reset(ts, text, encoding);
     RETS(ts)->curr_ind = 0;
     return ts;
 }
 
-static FrtTokenStream *
-rets_clone_i(FrtTokenStream *orig_ts)
-{
+static FrtTokenStream *rets_clone_i(FrtTokenStream *orig_ts) {
     FrtTokenStream *ts = frt_ts_clone_size(orig_ts, sizeof(RegExpTokenStream));
     return ts;
 }
 
-static FrtTokenStream *
-rets_new(VALUE rtext, VALUE regex, VALUE proc)
-{
-    FrtTokenStream *ts = frt_ts_new(RegExpTokenStream);
+FrtTokenStream *rets_alloc(void) {
+    return (FrtTokenStream *)frt_ecalloc(sizeof(RegExpTokenStream));
+}
 
-    if (rtext != Qnil) {
-        rtext = StringValue(rtext);
-        rb_hash_aset(object_space, ((VALUE)ts)|1, rtext);
-    }
+FrtTokenStream *rets_init(FrtTokenStream *ts, VALUE rtext, VALUE regex, VALUE proc) {
+    ts = frt_ts_init(ts);
     ts->reset = &rets_reset;
     ts->next = &rets_next;
     ts->clone_i = &rets_clone_i;
     ts->destroy_i = &rets_destroy_i;
 
-    RETS(ts)->curr_ind = 0;
-    RETS(ts)->rtext = rtext;
+    if (rtext != Qnil) {
+        rtext = StringValue(rtext);
+        ts->reset(ts, rs2s(rtext), rb_enc_get(rtext));
+    } else {
+        RETS(ts)->curr_ind = 0;
+    }
+
     RETS(ts)->proc = proc;
 
     if (NIL_P(regex)) {
@@ -790,10 +680,20 @@ rets_new(VALUE rtext, VALUE regex, VALUE proc)
         Check_Type(regex, T_REGEXP);
         RETS(ts)->regex = regex;
     }
-
     return ts;
 }
 
+static FrtTokenStream *rets_new(VALUE rtext, VALUE regex, VALUE proc) {
+    FrtTokenStream *ts = rets_alloc();
+    return rets_init(ts, rtext, regex, proc);
+}
+
+
+static VALUE frb_reg_exp_tokenizer_alloc(VALUE rclass) {
+    FrtTokenStream *ts = rets_alloc();
+    return TypedData_Wrap_Struct(rclass, &frb_reg_exp_token_stream_t, ts);
+}
+
 /*
  *  call-seq:
  *    RegExpTokenizer.new(input, /[[:alpha:]]+/)
@@ -803,18 +703,13 @@ rets_new(VALUE rtext, VALUE regex, VALUE proc)
  *  input::  text to tokenizer
  *  regexp:: regular expression used to recognize tokens in the input
  */
-static VALUE
-frb_rets_init(int argc, VALUE *argv, VALUE self)
-{
+static VALUE frb_rets_init(int argc, VALUE *argv, VALUE self) {
     VALUE rtext, regex, proc;
-    FrtTokenStream *ts;
-
     rb_scan_args(argc, argv, "11&", &rtext, &regex, &proc);
-
-    ts = rets_new(rtext, regex, proc);
-
-    Frt_Wrap_Struct(self, &frb_rets_mark, &frb_rets_free, ts);
-    object_add(ts, self);
+    FrtTokenStream *ts;
+    TypedData_Get_Struct(self, FrtTokenStream, &frb_reg_exp_token_stream_t, ts);
+    rets_init(ts, rtext, regex, proc);
+    ts->rts = self;
     return self;
 }
 
@@ -822,129 +717,72 @@ frb_rets_init(int argc, VALUE *argv, VALUE self)
  * Tokenizers
  ****************************************************************************/
 
-#define TS_ARGS(dflt) \
-    bool lower;\
-VALUE rlower, rstr;\
-rb_scan_args(argc, argv, "11", &rstr, &rlower);\
-lower = (argc ? RTEST(rlower) : dflt)
-
 /*
  *  call-seq:
- *     AsciiLetterTokenizer.new() -> tokenizer
+ *     LetterTokenizer.new() -> tokenizer
  *
- *  Create a new AsciiLetterTokenizer
+ *  Create a new LetterTokenizer.
  */
-static VALUE
-frb_a_letter_tokenizer_init(VALUE self, VALUE rstr)
-{
-    return get_wrapped_ts(self, rstr, frt_letter_tokenizer_new());
+static VALUE frb_letter_tokenizer_alloc(VALUE rclass) {
+    FrtTokenStream *ts = frt_letter_tokenizer_new();
+    return TypedData_Wrap_Struct(rclass, &frb_token_stream_t, ts);
 }
 
-/*
- *  call-seq:
- *     LetterTokenizer.new(lower = true) -> tokenizer
- *
- *  Create a new LetterTokenizer which optionally downcases tokens. Downcasing
- *  is done according the current locale.
- *
- *  lower:: set to false if you don't wish to downcase tokens
- */
-static VALUE
-frb_letter_tokenizer_init(int argc, VALUE *argv, VALUE self)
-{
-    TS_ARGS(false);
-#if !defined POSH_OS_WIN32 && !defined POSH_OS_WIN64
-    if (!frb_locale) frb_locale = setlocale(LC_CTYPE, "");
-#endif
-    return get_wrapped_ts(self, rstr, frt_mb_letter_tokenizer_new(lower));
+static VALUE frb_letter_tokenizer_init(int argc, VALUE *argv, VALUE self) {
+    VALUE rstr;
+    rb_scan_args(argc, argv, "1", &rstr);
+    FrtTokenStream *ts;
+    TypedData_Get_Struct(self, FrtTokenStream, &frb_token_stream_t, ts);
+    ts->reset(ts, rs2s(rstr), rb_enc_get(rstr));
+    ts->rts = self;
+    return self;
 }
 
 /*
  *  call-seq:
- *     AsciiWhiteSpaceTokenizer.new() -> tokenizer
+ *     WhiteSpaceTokenizer.new -> tokenizer
  *
- *  Create a new AsciiWhiteSpaceTokenizer
+ *  Create a new WhiteSpaceTokenizer.
  */
-static VALUE
-frb_a_whitespace_tokenizer_init(VALUE self, VALUE rstr)
-{
-    return get_wrapped_ts(self, rstr, frt_whitespace_tokenizer_new());
+static VALUE frb_whitespace_tokenizer_alloc(VALUE rclass) {
+    FrtTokenStream *ts = frt_whitespace_tokenizer_new();
+    return TypedData_Wrap_Struct(rclass, &frb_token_stream_t, ts);
 }
 
-/*
- *  call-seq:
- *     WhiteSpaceTokenizer.new(lower = true) -> tokenizer
- *
- *  Create a new WhiteSpaceTokenizer which optionally downcases tokens.
- *  Downcasing is done according the current locale.
- *
- *  lower:: set to false if you don't wish to downcase tokens
- */
-static VALUE
-frb_whitespace_tokenizer_init(int argc, VALUE *argv, VALUE self)
-{
-    TS_ARGS(false);
-#if !defined POSH_OS_WIN32 && !defined POSH_OS_WIN64
-    if (!frb_locale) frb_locale = setlocale(LC_CTYPE, "");
-#endif
-    return get_wrapped_ts(self, rstr, frt_mb_whitespace_tokenizer_new(lower));
+static VALUE frb_whitespace_tokenizer_init(int argc, VALUE *argv, VALUE self) {
+    VALUE rstr;
+    rb_scan_args(argc, argv, "1", &rstr);
+    FrtTokenStream *ts;
+    TypedData_Get_Struct(self, FrtTokenStream, &frb_token_stream_t, ts);
+    ts->reset(ts, rs2s(rstr), rb_enc_get(rstr));
+    ts->rts = self;
+    return self;
 }
 
 /*
  *  call-seq:
- *     AsciiStandardTokenizer.new() -> tokenizer
+ *     StandardTokenizer.new -> tokenizer
  *
- *  Create a new AsciiStandardTokenizer
+ *  Create a new StandardTokenizer.
  */
-static VALUE
-frb_a_standard_tokenizer_init(VALUE self, VALUE rstr)
-{
-    return get_wrapped_ts(self, rstr, frt_standard_tokenizer_new());
+static VALUE frb_standard_tokenizer_alloc(VALUE rclass) {
+    FrtTokenStream *ts = frt_standard_tokenizer_new();
+    return TypedData_Wrap_Struct(rclass, &frb_token_stream_t, ts);
 }
 
-/*
- *  call-seq:
- *     StandardTokenizer.new(lower = true) -> tokenizer
- *
- *  Create a new StandardTokenizer which optionally downcases tokens.
- *  Downcasing is done according the current locale.
- *
- *  lower:: set to false if you don't wish to downcase tokens
- */
-static VALUE
-frb_standard_tokenizer_init(VALUE self, VALUE rstr)
-{
-#if !defined POSH_OS_WIN32 && !defined POSH_OS_WIN64
-    if (!frb_locale) frb_locale = setlocale(LC_CTYPE, "");
-#endif
-    return get_wrapped_ts(self, rstr, frt_mb_standard_tokenizer_new());
+static VALUE frb_standard_tokenizer_init(int argc, VALUE *argv, VALUE self) {
+    VALUE rstr;
+    rb_scan_args(argc, argv, "1", &rstr);
+    FrtTokenStream *ts;
+    TypedData_Get_Struct(self, FrtTokenStream, &frb_token_stream_t, ts);
+    ts->reset(ts, rs2s(rstr), rb_enc_get(rstr));
+    ts->rts = self;
+    return self;
 }
 
 /****************************************************************************
  * Filters
  ****************************************************************************/
-
-
-/*
- *  call-seq:
- *     AsciiLowerCaseFilter.new(token_stream) -> token_stream
- *
- *  Create an AsciiLowerCaseFilter which normalizes a token's text to
- *  lowercase but only for ASCII characters. For other characters use
- *  LowerCaseFilter.
- */
-static VALUE
-frb_a_lowercase_filter_init(VALUE self, VALUE rsub_ts)
-{
-    FrtTokenStream *ts = frb_get_cwrapped_rts(rsub_ts);
-    ts = frt_lowercase_filter_new(ts);
-    object_add(&(TkFilt(ts)->sub_ts), rsub_ts);
-
-    Frt_Wrap_Struct(self, &frb_tf_mark, &frb_tf_free, ts);
-    object_add(ts, self);
-    return self;
-}
-
 /*
  *  call-seq:
  *     LowerCaseFilter.new(token_stream) -> token_stream
@@ -952,18 +790,33 @@ frb_a_lowercase_filter_init(VALUE self, VALUE rsub_ts)
  *  Create an LowerCaseFilter which normalizes a token's text to
  *  lowercase based on the current locale.
  */
-static VALUE
-frb_lowercase_filter_init(VALUE self, VALUE rsub_ts)
-{
-    FrtTokenStream *ts = frb_get_cwrapped_rts(rsub_ts);
-#if !defined POSH_OS_WIN32 && !defined POSH_OS_WIN64
-    if (!frb_locale) frb_locale = setlocale(LC_CTYPE, "");
-#endif
-    ts = frt_mb_lowercase_filter_new(ts);
-    object_add(&(TkFilt(ts)->sub_ts), rsub_ts);
 
-    Frt_Wrap_Struct(self, &frb_tf_mark, &frb_tf_free, ts);
-    object_add(ts, self);
+const rb_data_type_t frb_lowercase_filter_t = {
+    .wrap_struct_name = "FrbLowercaseFilter",
+    .function = {
+        .dmark = frb_tf_mark,
+        .dfree = frb_tf_free,
+        .dsize = frb_tf_size,
+        .dcompact = NULL,
+        .reserved = {0},
+    },
+    .parent = NULL,
+    .data = NULL,
+    .flags = RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+static VALUE frb_lowercase_filter_alloc(VALUE rclass) {
+    FrtTokenStream *tf = frt_lowercase_filter_alloc();
+    return TypedData_Wrap_Struct(rclass, &frb_lowercase_filter_t, tf);
+}
+
+static VALUE frb_lowercase_filter_init(VALUE self, VALUE rsub_ts) {
+    FrtTokenStream *sub_ts = frb_get_cwrapped_rts(rsub_ts);
+    FrtTokenStream *ts;
+    TypedData_Get_Struct(self, FrtTokenStream, &frb_lowercase_filter_t, ts);
+    frt_lowercase_filter_init(ts, sub_ts);
+    TkFilt(ts)->sub_ts->rts = rsub_ts;
+    ts->rts = self;
     return self;
 }
 
@@ -977,15 +830,38 @@ frb_lowercase_filter_init(VALUE self, VALUE rsub_ts)
  *  search for "e-mail", "email" and "mail" will all match. This filter is
  *  used by default by the StandardAnalyzer.
  */
-static VALUE
-frb_hyphen_filter_init(VALUE self, VALUE rsub_ts)
-{
-    FrtTokenStream *ts = frb_get_cwrapped_rts(rsub_ts);
-    ts = frt_hyphen_filter_new(ts);
-    object_add(&(TkFilt(ts)->sub_ts), rsub_ts);
 
-    Frt_Wrap_Struct(self, &frb_tf_mark, &frb_tf_free, ts);
-    object_add(ts, self);
+static size_t frb_hyphen_filter_size(const void *p) {
+    return sizeof(FrtHyphenFilter);
+    (void)p;
+}
+
+const rb_data_type_t frb_hyphen_filter_t = {
+    .wrap_struct_name = "FrbHyphenFilter",
+    .function = {
+        .dmark = frb_tf_mark,
+        .dfree = frb_tf_free,
+        .dsize = frb_hyphen_filter_size,
+        .dcompact = NULL,
+        .reserved = {0},
+    },
+    .parent = NULL,
+    .data = NULL,
+    .flags = RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+static VALUE frb_hyphen_filter_alloc(VALUE rclass) {
+    FrtTokenStream *hf = frt_hyphen_filter_alloc();
+    return TypedData_Wrap_Struct(rclass, &frb_hyphen_filter_t, hf);
+}
+
+static VALUE frb_hyphen_filter_init(VALUE self, VALUE rsub_ts) {
+    FrtTokenStream *sub_ts = frb_get_cwrapped_rts(rsub_ts);
+    FrtTokenStream *ts;
+    TypedData_Get_Struct(self, FrtTokenStream, &frb_hyphen_filter_t, ts);
+    frt_hyphen_filter_init(ts, sub_ts);
+    TkFilt(ts)->sub_ts->rts = rsub_ts;
+    ts->rts = self;
     return self;
 }
 
@@ -1002,31 +878,54 @@ frb_hyphen_filter_init(VALUE self, VALUE rsub_ts)
  *                 defaults to a list of English stop-words. The
  *                 Ferret::Analysis contains a number of stop-word lists.
  */
-static VALUE
-frb_stop_filter_init(int argc, VALUE *argv, VALUE self)
-{
+
+static size_t frb_stop_filter_size(const void *p) {
+    return sizeof(FrtStopFilter);
+    (void)p;
+}
+
+const rb_data_type_t frb_stop_filter_t = {
+    .wrap_struct_name = "FrbStopFilter",
+    .function = {
+        .dmark = frb_tf_mark,
+        .dfree = frb_tf_free,
+        .dsize = frb_stop_filter_size,
+        .dcompact = NULL,
+        .reserved = {0},
+    },
+    .parent = NULL,
+    .data = NULL,
+    .flags = RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+static VALUE frb_stop_filter_alloc(VALUE rclass) {
+    FrtTokenStream *sf = frt_stop_filter_alloc();
+    return TypedData_Wrap_Struct(rclass, &frb_stop_filter_t, sf);
+}
+
+static VALUE frb_stop_filter_init(int argc, VALUE *argv, VALUE self) {
     VALUE rsub_ts, rstop_words;
+    FrtTokenStream *sub_ts;
     FrtTokenStream *ts;
     rb_scan_args(argc, argv, "11", &rsub_ts, &rstop_words);
-    ts = frb_get_cwrapped_rts(rsub_ts);
+    sub_ts = frb_get_cwrapped_rts(rsub_ts);
+
+    TypedData_Get_Struct(self, FrtTokenStream, &frb_stop_filter_t, ts);
     if (rstop_words != Qnil) {
         char **stop_words = get_stopwords(rstop_words);
-        ts = frt_stop_filter_new_with_words(ts, (const char **)stop_words);
-
+        frt_stop_filter_init(ts, sub_ts);
+        frt_stop_filter_set_words(ts, (const char **)stop_words);
         free(stop_words);
     } else {
-        ts = frt_stop_filter_new(ts);
+        frt_stop_filter_init(ts, sub_ts);
     }
-    object_add(&(TkFilt(ts)->sub_ts), rsub_ts);
+    TkFilt(ts)->sub_ts->rts = rsub_ts;
 
-    Frt_Wrap_Struct(self, &frb_tf_mark, &frb_tf_free, ts);
-    object_add(ts, self);
+    ts->rts = self;
     return self;
 }
 
-static void frb_add_mapping_i(FrtTokenStream *mf, VALUE from,
-                                     const char *to)
-{
+static void frb_add_mapping_i(FrtTokenStream *mf, VALUE from, const char *to) {
     switch (TYPE(from)) {
         case T_STRING:
             frt_mapping_filter_add(mf, rs2s(from), to);
@@ -1042,8 +941,7 @@ static void frb_add_mapping_i(FrtTokenStream *mf, VALUE from,
     }
 }
 
-static int frb_add_mappings_i(VALUE key, VALUE value, VALUE arg)
-{
+static int frb_add_mappings_i(VALUE key, VALUE value, VALUE arg) {
     if (key == Qundef) {
         return ST_CONTINUE;
     } else {
@@ -1067,8 +965,7 @@ static int frb_add_mappings_i(VALUE key, VALUE value, VALUE arg)
             for (i = RARRAY_LEN(key) - 1; i >= 0; i--) {
                 frb_add_mapping_i(mf, RARRAY_PTR(key)[i], to);
             }
-        }
-        else {
+        } else {
             frb_add_mapping_i(mf, key, to);
         }
     }
@@ -1100,18 +997,40 @@ static int frb_add_mappings_i(VALUE key, VALUE value, VALUE arg)
  *                                ['è','é','ê','ë','ē','ę'] => 'e'
  *                              })
  */
-static VALUE
-frb_mapping_filter_init(VALUE self, VALUE rsub_ts, VALUE mapping)
-{
+
+static size_t frb_mapping_filter_size(const void *p) {
+    return sizeof(FrtMappingFilter);
+    (void)p;
+}
+
+const rb_data_type_t frb_mapping_filter_t = {
+    .wrap_struct_name = "FrbMappingFilter",
+    .function = {
+        .dmark = frb_tf_mark,
+        .dfree = frb_tf_free,
+        .dsize = frb_mapping_filter_size,
+        .dcompact = NULL,
+        .reserved = {0},
+    },
+    .parent = NULL,
+    .data = NULL,
+    .flags = RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+static VALUE frb_mapping_filter_alloc(VALUE rclass) {
+    FrtTokenStream *mf = frt_mapping_filter_alloc();
+    return TypedData_Wrap_Struct(rclass, &frb_mapping_filter_t, mf);
+}
+
+static VALUE frb_mapping_filter_init(VALUE self, VALUE rsub_ts, VALUE mapping) {
     FrtTokenStream *ts;
-    ts = frb_get_cwrapped_rts(rsub_ts);
-    ts = frt_mapping_filter_new(ts);
+    FrtTokenStream *sub_ts = frb_get_cwrapped_rts(rsub_ts);
+    TypedData_Get_Struct(self, FrtTokenStream, &frb_mapping_filter_t, ts);
+    frt_mapping_filter_init(ts, sub_ts);
     rb_hash_foreach(mapping, frb_add_mappings_i, (VALUE)ts);
     frt_mulmap_compile(((FrtMappingFilter *)ts)->mapper);
-    object_add(&(TkFilt(ts)->sub_ts), rsub_ts);
-
-    Frt_Wrap_Struct(self, &frb_tf_mark, &frb_tf_free, ts);
-    object_add(ts, self);
+    TkFilt(ts)->sub_ts->rts = rsub_ts;
+    ts->rts = self;
     return self;
 }
 
@@ -1128,29 +1047,49 @@ frb_mapping_filter_init(VALUE self, VALUE rsub_ts, VALUE mapping)
  *
  *  token_stream:: TokenStream to be filtered
  *  algorithm::    The algorithm (or language) to use
- *  encoding::     The encoding of the data (default: "UTF-8")
  */
-static VALUE
-frb_stem_filter_init(int argc, VALUE *argv, VALUE self)
-{
-    VALUE rsub_ts, ralgorithm, rcharenc;
+
+static size_t frb_stem_filter_size(const void *p) {
+    return sizeof(FrtStemFilter);
+    (void)p;
+}
+
+const rb_data_type_t frb_stem_filter_t = {
+    .wrap_struct_name = "FrbStemFilter",
+    .function = {
+        .dmark = frb_tf_mark,
+        .dfree = frb_tf_free,
+        .dsize = frb_stem_filter_size,
+        .dcompact = NULL,
+        .reserved = {0},
+    },
+    .parent = NULL,
+    .data = NULL,
+    .flags = RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+static VALUE frb_stem_filter_alloc(VALUE rclass) {
+    FrtTokenStream *sf = frt_stem_filter_alloc();
+    return TypedData_Wrap_Struct(rclass, &frb_stem_filter_t, sf);
+}
+
+static VALUE frb_stem_filter_init(int argc, VALUE *argv, VALUE self) {
+    VALUE rsub_ts, ralgorithm;
     const char *algorithm = "english";
-    char *charenc = NULL;
+    FrtTokenStream *sub_ts;
     FrtTokenStream *ts;
-    rb_scan_args(argc, argv, "12", &rsub_ts, &ralgorithm, &rcharenc);
-    ts = frb_get_cwrapped_rts(rsub_ts);
-    switch (argc) {
-        case 3: charenc = rs2s(rb_obj_as_string(rcharenc));
-        case 2: algorithm = rs2s(rb_obj_as_string(ralgorithm));
-    }
-    ts = frt_stem_filter_new(ts, algorithm, charenc);
-    object_add(&(TkFilt(ts)->sub_ts), rsub_ts);
+    TypedData_Get_Struct(self, FrtTokenStream, &frb_stem_filter_t, ts);
+
+    rb_scan_args(argc, argv, "11", &rsub_ts, &ralgorithm);
+    sub_ts = frb_get_cwrapped_rts(rsub_ts);
+    if (argc == 2)
+        algorithm = rs2s(rb_obj_as_string(ralgorithm));
 
-    Frt_Wrap_Struct(self, &frb_tf_mark, &frb_tf_free, ts);
-    object_add(ts, self);
+    frt_stem_filter_init(ts, sub_ts, algorithm);
+    TkFilt(ts)->sub_ts->rts = rsub_ts;
+    ts->rts = self;
     if (((FrtStemFilter *)ts)->stemmer == NULL) {
-        rb_raise(rb_eArgError, "No stemmer could be found with the encoding "
-                 "%s and the language %s", charenc, algorithm);
+        rb_raise(rb_eArgError, "No stemmer could be found for the %s language.", algorithm);
     }
     return self;
 }
@@ -1165,79 +1104,82 @@ frb_stem_filter_init(int argc, VALUE *argv, VALUE self)
  * CWrappedAnalyzer Methods
  ****************************************************************************/
 
-#define GET_A(a, self) Data_Get_Struct(self, FrtAnalyzer, a)
-
-#define CWA(analyzer) ((CWrappedAnalyzer *)(analyzer))
-typedef struct CWrappedAnalyzer
-{
+typedef struct CWrappedAnalyzer {
     FrtAnalyzer super;
     VALUE ranalyzer;
 } CWrappedAnalyzer;
 
-static void
-cwa_destroy_i(FrtAnalyzer *a)
-{
+#define CWA(analyzer) ((CWrappedAnalyzer *)(analyzer))
+
+static void frb_analyzer_free(void *p) {
+    frt_a_deref((FrtAnalyzer *)p);
+}
+
+static size_t frb_analyzer_size(const void *p) {
+    return sizeof(CWrappedAnalyzer);
+    (void)p;
+}
+
+const rb_data_type_t frb_analyzer_t = {
+    .wrap_struct_name = "FrbAnalyzer",
+    .function = {
+        .dmark = NULL,
+        .dfree = frb_analyzer_free,
+        .dsize = frb_analyzer_size,
+        .dcompact = NULL,
+        .reserved = {0},
+    },
+    .parent = NULL,
+    .data = NULL,
+    .flags = RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+static void cwa_destroy_i(FrtAnalyzer *a) {
     rb_hash_delete(object_space, ((VALUE)a)|1);
     free(a);
 }
 
-static FrtTokenStream *
-cwa_get_ts(FrtAnalyzer *a, FrtSymbol field, char *text)
-{
-    VALUE rts = rb_funcall(CWA(a)->ranalyzer, id_token_stream, 2,
-                           rb_str_new_cstr(rb_id2name(field)), rb_str_new_cstr(text));
+static FrtTokenStream *cwa_get_ts(FrtAnalyzer *a, ID field, char *text, rb_encoding *encoding) {
+    VALUE rstr = rb_str_new_cstr(text);
+    rb_enc_associate(rstr, encoding);
+    VALUE rts = rb_funcall(CWA(a)->ranalyzer, id_token_stream, 2, rb_str_new_cstr(rb_id2name(field)), rstr);
     return frb_get_cwrapped_rts(rts);
 }
 
-FrtAnalyzer *
-frb_get_cwrapped_analyzer(VALUE ranalyzer)
-{
+FrtAnalyzer *frb_get_cwrapped_analyzer(VALUE ranalyzer) {
     FrtAnalyzer *a = NULL;
     if (frb_is_cclass(ranalyzer) && DATA_PTR(ranalyzer)) {
-        Data_Get_Struct(ranalyzer, FrtAnalyzer, a);
+        // TypedData_Get_Struct(ranalyzer, FrtAnalyzer, &frb_analyzer_t, a);
+        a = DATA_PTR(ranalyzer);
         FRT_REF(a);
-    }
-    else {
+    } else {
         a = (FrtAnalyzer *)frt_ecalloc(sizeof(CWrappedAnalyzer));
         a->destroy_i = &cwa_destroy_i;
         a->get_ts    = &cwa_get_ts;
         a->ref_cnt   = 1;
-        ((CWrappedAnalyzer *)a)->ranalyzer = ranalyzer;
+        CWA(a)->ranalyzer = ranalyzer;
         /* prevent from being garbage collected */
         rb_hash_aset(object_space, ((VALUE)a)|1, ranalyzer);
     }
     return a;
 }
 
-static void
-frb_analyzer_free(FrtAnalyzer *a)
-{
-    object_del(a);
-    frt_a_deref(a);
-}
-
-VALUE
-frb_get_analyzer(FrtAnalyzer *a)
-{
+VALUE frb_get_analyzer(FrtAnalyzer *a) {
     VALUE self = Qnil;
     if (a) {
-        self = object_get(a);
-        if (self == Qnil) {
-            self = Data_Wrap_Struct(cAnalyzer, NULL, &frb_analyzer_free, a);
+        self = a->ranalyzer;
+        if (self == 0 || self == Qnil) {
+            self = TypedData_Wrap_Struct(cAnalyzer, &frb_analyzer_t, a);
             FRT_REF(a);
-            object_add(a, self);
+            a->ranalyzer = self;
         }
     }
     return self;
 }
 
-VALUE
-get_rb_ts_from_a(FrtAnalyzer *a, VALUE rfield, VALUE rstring)
-{
-    FrtTokenStream *ts = frt_a_get_ts(a, frb_field(rfield), rs2s(rstring));
-
+VALUE get_rb_ts_from_a(FrtAnalyzer *a, VALUE rfield, VALUE rstring) {
+    FrtTokenStream *ts = frt_a_get_ts(a, frb_field(rfield), rs2s(rstring), rb_enc_get(rstring));
     /* Make sure that there is no entry already */
-    object_set(&ts->text, rstring);
     return get_rb_token_stream(ts);
 }
 
@@ -1252,16 +1194,11 @@ get_rb_ts_from_a(FrtAnalyzer *a, VALUE rfield, VALUE rstring)
  *  field_name:: name of the field to be tokenized
  *  input::      data from the field to be tokenized
  */
-static VALUE
-frb_analyzer_token_stream(VALUE self, VALUE rfield, VALUE rstring)
-{
+static VALUE frb_analyzer_token_stream(VALUE self, VALUE rfield, VALUE rstring) {
     /* NOTE: Any changes made to this method may also need to be applied to
      * frb_re_analyzer_token_stream */
-    FrtAnalyzer *a;
-    GET_A(a, self);
-
+    FrtAnalyzer *a = DATA_PTR(self);
     StringValue(rstring);
-
     return get_rb_ts_from_a(a, rfield, rstring);
 }
 
@@ -1271,27 +1208,24 @@ VALUE rlower;\
 rb_scan_args(argc, argv, "01", &rlower);\
 lower = (argc ? RTEST(rlower) : dflt)
 
-/*
- *  call-seq:
- *     AsciiWhiteSpaceAnalyzer.new(lower = false) -> analyzer
- *
- *  Create a new AsciiWhiteSpaceAnalyzer which downcases tokens by default
- *  but can optionally leave case as is. Lowercasing will only be done to
- *  ASCII characters.
- *
- *  lower:: set to false if you don't want the field's tokens to be downcased
- */
-static VALUE
-frb_a_white_space_analyzer_init(int argc, VALUE *argv, VALUE self)
-{
+static VALUE frb_analyzer_alloc(VALUE rclass) {
+    FrtAnalyzer *a = frt_letter_analyzer_alloc();
+    return TypedData_Wrap_Struct(rclass, &frb_analyzer_t, a);
+}
+
+static VALUE frb_analyzer_init(int argc, VALUE *argv, VALUE self) {
     FrtAnalyzer *a;
-    GET_LOWER(false);
-    a = frt_whitespace_analyzer_new(lower);
-    Frt_Wrap_Struct(self, NULL, &frb_analyzer_free, a);
-    object_add(a, self);
+    GET_LOWER(true);
+    TypedData_Get_Struct(self, FrtAnalyzer, &frb_analyzer_t, a);
+    frt_letter_analyzer_init(a, lower);
+    a->ranalyzer = self;
     return self;
 }
 
+
+/*****************************************************************************/
+/*** WhiteSpaceAnalyzer ******************************************************/
+/*****************************************************************************/
 /*
  *  call-seq:
  *     WhiteSpaceAnalyzer.new(lower = false) -> analyzer
@@ -1302,38 +1236,18 @@ frb_a_white_space_analyzer_init(int argc, VALUE *argv, VALUE self)
  *
  *  lower:: set to false if you don't want the field's tokens to be downcased
  */
-static VALUE
-frb_white_space_analyzer_init(int argc, VALUE *argv, VALUE self)
-{
-    FrtAnalyzer *a;
-    GET_LOWER(false);
-#if !defined POSH_OS_WIN32 && !defined POSH_OS_WIN64
-    if (!frb_locale) frb_locale = setlocale(LC_CTYPE, "");
-#endif
-    a = frt_mb_whitespace_analyzer_new(lower);
-    Frt_Wrap_Struct(self, NULL, &frb_analyzer_free, a);
-    object_add(a, self);
-    return self;
+
+static VALUE frb_whitespace_analyzer_alloc(VALUE rclass) {
+    FrtAnalyzer *a = frt_whitespace_analyzer_alloc();
+    return TypedData_Wrap_Struct(rclass, &frb_analyzer_t, a);
 }
 
-/*
- *  call-seq:
- *     AsciiLetterAnalyzer.new(lower = true) -> analyzer
- *
- *  Create a new AsciiWhiteSpaceAnalyzer which downcases tokens by default
- *  but can optionally leave case as is. Lowercasing will only be done to
- *  ASCII characters.
- *
- *  lower:: set to false if you don't want the field's tokens to be downcased
- */
-static VALUE
-frb_a_letter_analyzer_init(int argc, VALUE *argv, VALUE self)
-{
+static VALUE frb_whitespace_analyzer_init(int argc, VALUE *argv, VALUE self) {
     FrtAnalyzer *a;
-    GET_LOWER(true);
-    a = frt_letter_analyzer_new(lower);
-    Frt_Wrap_Struct(self, NULL, &frb_analyzer_free, a);
-    object_add(a, self);
+    GET_LOWER(false);
+    TypedData_Get_Struct(self, FrtAnalyzer, &frb_analyzer_t, a);
+    frt_whitespace_analyzer_init(a, lower);
+    a->ranalyzer = self;
     return self;
 }
 
@@ -1347,17 +1261,18 @@ frb_a_letter_analyzer_init(int argc, VALUE *argv, VALUE self)
  *
  *  lower:: set to false if you don't want the field's tokens to be downcased
  */
-static VALUE
-frb_letter_analyzer_init(int argc, VALUE *argv, VALUE self)
-{
+
+static VALUE frb_letter_analyzer_alloc(VALUE rclass) {
+    FrtAnalyzer *a = frt_letter_analyzer_alloc();
+    return TypedData_Wrap_Struct(rclass, &frb_analyzer_t, a);
+}
+
+static VALUE frb_letter_analyzer_init(int argc, VALUE *argv, VALUE self) {
     FrtAnalyzer *a;
     GET_LOWER(true);
-#if !defined POSH_OS_WIN32 && !defined POSH_OS_WIN64
-    if (!frb_locale) frb_locale = setlocale(LC_CTYPE, "");
-#endif
-    a = frt_mb_letter_analyzer_new(lower);
-    Frt_Wrap_Struct(self, NULL, &frb_analyzer_free, a);
-    object_add(a, self);
+    TypedData_Get_Struct(self, FrtAnalyzer, &frb_analyzer_t, a);
+    frt_letter_analyzer_init(a, lower);
+    a->ranalyzer = self;
     return self;
 }
 
@@ -1374,39 +1289,6 @@ get_rstopwords(const char **stop_words)
     return rstopwords;
 }
 
-/*
- *  call-seq:
- *     AsciiStandardAnalyzer.new(lower = true, stop_words = FRT_FULL_ENGLISH_STOP_WORDS)
- *     -> analyzer
- *
- *  Create a new AsciiStandardAnalyzer which downcases tokens by default but
- *  can optionally leave case as is. Lowercasing will be done based on the
- *  current locale. You can also set the list of stop-words to be used by the
- *  StopFilter.
- *
- *  lower::      set to false if you don't want the field's tokens to be downcased
- *  stop_words:: list of stop-words to pass to the StopFilter
- */
-static VALUE
-frb_a_standard_analyzer_init(int argc, VALUE *argv, VALUE self)
-{
-    bool lower;
-    VALUE rlower, rstop_words;
-    FrtAnalyzer *a;
-    rb_scan_args(argc, argv, "02", &rstop_words, &rlower);
-    lower = ((rlower == Qnil) ? true : RTEST(rlower));
-    if (rstop_words != Qnil) {
-        char **stop_words = get_stopwords(rstop_words);
-        a = frt_standard_analyzer_new_with_words((const char **)stop_words, lower);
-        free(stop_words);
-    } else {
-        a = frt_standard_analyzer_new(lower);
-    }
-    Frt_Wrap_Struct(self, NULL, &frb_analyzer_free, a);
-    object_add(a, self);
-    return self;
-}
-
 /*
  *  call-seq:
  *     StandardAnalyzer.new(stop_words = FRT_FULL_ENGLISH_STOP_WORDS, lower=true)
@@ -1420,39 +1302,37 @@ frb_a_standard_analyzer_init(int argc, VALUE *argv, VALUE self)
  *  lower::      set to false if you don't want the field's tokens to be downcased
  *  stop_words:: list of stop-words to pass to the StopFilter
  */
-static VALUE
-frb_standard_analyzer_init(int argc, VALUE *argv, VALUE self)
-{
+static VALUE frb_standard_analyzer_alloc(VALUE rclass) {
+    FrtAnalyzer *a = frt_standard_analyzer_alloc();
+    return TypedData_Wrap_Struct(rclass, &frb_analyzer_t, a);
+}
+
+static VALUE frb_standard_analyzer_init(int argc, VALUE *argv, VALUE self) {
     bool lower;
     VALUE rlower, rstop_words;
     FrtAnalyzer *a;
-#if !defined POSH_OS_WIN32 && !defined POSH_OS_WIN64
-    if (!frb_locale) frb_locale = setlocale(LC_CTYPE, "");
-#endif
     rb_scan_args(argc, argv, "02", &rstop_words, &rlower);
     lower = ((rlower == Qnil) ? true : RTEST(rlower));
+    TypedData_Get_Struct(self, FrtAnalyzer, &frb_analyzer_t, a);
     if (rstop_words != Qnil) {
         char **stop_words = get_stopwords(rstop_words);
-        a = frt_mb_standard_analyzer_new_with_words((const char **)stop_words, lower);
+        frt_standard_analyzer_init(a, lower, (const char **)stop_words);
         free(stop_words);
     } else {
-        a = frt_mb_standard_analyzer_new(lower);
+        frt_standard_analyzer_init(a, lower, FRT_FULL_ENGLISH_STOP_WORDS);
     }
-    Frt_Wrap_Struct(self, NULL, &frb_analyzer_free, a);
-    object_add(a, self);
+    a->ranalyzer = self;
     return self;
 }
 
-static void
-frb_h_mark_values_i(void *key, void *value, void *arg)
-{
-    frb_gc_mark(value);
+static void frb_h_mark_values_i(void *key, void *value, void *arg) {
+    if (((FrtAnalyzer *)value)->ranalyzer)
+        rb_gc_mark(((FrtAnalyzer *)value)->ranalyzer);
 }
 
-static void
-frb_pfa_mark(void *p)
-{
-    frb_gc_mark(PFA(p)->default_a);
+static void frb_pfa_mark(void *p) {
+    if (PFA(p)->default_a->ranalyzer)
+        rb_gc_mark(PFA(p)->default_a->ranalyzer);
     frt_h_each(PFA(p)->dict, &frb_h_mark_values_i, NULL);
 }
 
@@ -1468,13 +1348,37 @@ frb_pfa_mark(void *p)
  *  default_analyzer:: analyzer to be used on fields that aren't otherwise
  *                     specified
  */
-static VALUE
-frb_per_field_analyzer_init(VALUE self, VALUE ranalyzer)
-{
+
+static size_t frb_per_field_analyzer_size(const void *p) {
+    return sizeof(FrtPerFieldAnalyzer);
+    (void)p;
+}
+
+const rb_data_type_t frb_per_field_analyzer_t = {
+    .wrap_struct_name = "FrbPerFieldAnalyzer",
+    .function = {
+        .dmark = frb_pfa_mark,
+        .dfree = frb_analyzer_free,
+        .dsize = frb_per_field_analyzer_size,
+        .dcompact = NULL,
+        .reserved = {0},
+    },
+    .parent = NULL,
+    .data = NULL,
+    .flags = RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+static VALUE frb_per_field_analyzer_alloc(VALUE rclass) {
+    FrtAnalyzer *a = frt_per_field_analyzer_alloc();
+    return TypedData_Wrap_Struct(rclass, &frb_per_field_analyzer_t, a);
+}
+
+static VALUE frb_per_field_analyzer_init(VALUE self, VALUE ranalyzer) {
     FrtAnalyzer *def = frb_get_cwrapped_analyzer(ranalyzer);
-    FrtAnalyzer *a = frt_per_field_analyzer_new(def);
-    Frt_Wrap_Struct(self, &frb_pfa_mark, &frb_analyzer_free, a);
-    object_add(a, self);
+    FrtAnalyzer *a;
+    TypedData_Get_Struct(self, FrtAnalyzer, &frb_per_field_analyzer_t, a);
+    frt_per_field_analyzer_init(a, def);
+    a->ranalyzer = self;
     return self;
 }
 
@@ -1489,11 +1393,9 @@ frb_per_field_analyzer_init(VALUE self, VALUE ranalyzer)
  *  field_name:: field we wish to set the analyzer for
  *  analyzer::   analyzer to be used on +field_name+
  */
-static VALUE
-frb_per_field_analyzer_add_field(VALUE self, VALUE rfield, VALUE ranalyzer)
-{
+static VALUE frb_per_field_analyzer_add_field(VALUE self, VALUE rfield, VALUE ranalyzer) {
     FrtAnalyzer *pfa, *a;
-    Data_Get_Struct(self, FrtAnalyzer, pfa);
+    TypedData_Get_Struct(self, FrtAnalyzer, &frb_per_field_analyzer_t, pfa);
     a = frb_get_cwrapped_analyzer(ranalyzer);
 
     frt_pfa_add_field(pfa, frb_field(rfield), a);
@@ -1510,12 +1412,10 @@ frb_per_field_analyzer_add_field(VALUE self, VALUE rfield, VALUE ranalyzer)
  *  field_name:: name of the field to be tokenized
  *  input::      data from the field to be tokenized
  */
-static VALUE
-frb_pfa_analyzer_token_stream(VALUE self, VALUE rfield, VALUE rstring)
-{
+static VALUE frb_pfa_analyzer_token_stream(VALUE self, VALUE rfield, VALUE rstring) {
     FrtAnalyzer *pfa, *a;
-    FrtSymbol field = frb_field(rfield);
-    GET_A(pfa, self);
+    ID field = frb_field(rfield);
+    TypedData_Get_Struct(self, FrtAnalyzer, &frb_per_field_analyzer_t, pfa);
 
     StringValue(rstring);
     a = (FrtAnalyzer *)frt_h_get(PFA(pfa)->dict, (void *)field);
@@ -1523,29 +1423,46 @@ frb_pfa_analyzer_token_stream(VALUE self, VALUE rfield, VALUE rstring)
         a = PFA(pfa)->default_a;
     }
     if (a->get_ts == cwa_get_ts) {
+        VALUE rstr = rb_str_new_cstr(rs2s(rstring));
+        rb_enc_associate(rstr, rb_enc_get(rstring));
         return rb_funcall(CWA(a)->ranalyzer, id_token_stream, 2,
-                          rb_str_new_cstr(rb_id2name(field)), rb_str_new_cstr(rs2s(rstring)));
-    }
-    else {
+                          rb_str_new_cstr(rb_id2name(field)), rstr);
+    } else {
         return get_rb_ts_from_a(a, rfield, rstring);
     }
 }
 
 /*** RegExpAnalyzer ***/
 
-static void
-frb_re_analyzer_mark(FrtAnalyzer *a)
-{
-    frb_gc_mark(a->current_ts);
+static void frb_re_analyzer_mark(void *p) {
+    if (((FrtAnalyzer *)p)->current_ts->rts)
+        rb_gc_mark(((FrtAnalyzer *)p)->current_ts->rts);
 }
 
-static void
-re_analyzer_destroy_i(FrtAnalyzer *a)
-{
+static void re_analyzer_destroy_i(FrtAnalyzer *a) {
     frt_ts_deref(a->current_ts);
     free(a);
 }
 
+const rb_data_type_t frb_reg_exp_analyzer_t = {
+    .wrap_struct_name = "FrbRegExpAnalyzer",
+    .function = {
+        .dmark = frb_re_analyzer_mark,
+        .dfree = frb_analyzer_free,
+        .dsize = frb_analyzer_size,
+        .dcompact = 0,
+        .reserved = {0},
+    },
+    .parent = 0,
+    .data = 0,
+    .flags = RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+static VALUE frb_reg_exp_analyzer_alloc(VALUE rclass) {
+    FrtAnalyzer *a;
+    return TypedData_Make_Struct(rclass, FrtAnalyzer, &frb_reg_exp_analyzer_t, a);
+}
+
 /*
  *  call-seq:
  *     RegExpAnalyzer.new(reg_exp, lower = true) -> analyzer
@@ -1556,27 +1473,26 @@ re_analyzer_destroy_i(FrtAnalyzer *a)
  *  reg_exp:: the token matcher for the tokenizer to use
  *  lower::   set to false if you don't want to downcase the tokens
  */
-static VALUE
-frb_re_analyzer_init(int argc, VALUE *argv, VALUE self)
-{
+static VALUE frb_re_analyzer_init(int argc, VALUE *argv, VALUE self) {
     VALUE lower, rets, regex, proc;
     FrtAnalyzer *a;
     FrtTokenStream *ts;
     rb_scan_args(argc, argv, "02&", &regex, &lower, &proc);
 
     ts = rets_new(Qnil, regex, proc);
-    rets = Data_Wrap_Struct(cRegExpTokenizer, &frb_rets_mark, &frb_rets_free, ts);
-    object_add(ts, rets);
+    rets = TypedData_Wrap_Struct(cRegExpTokenizer, &frb_reg_exp_token_stream_t, ts);
+    ts->rts = rets;
 
     if (lower != Qfalse) {
-        rets = frb_lowercase_filter_init(frb_data_alloc(cLowerCaseFilter), rets);
+        rets = frb_lowercase_filter_init(frb_lowercase_filter_alloc(cLowerCaseFilter), rets);
         ts = DATA_PTR(rets);
+        ts->rts = rets;
     }
     FRT_REF(ts);
 
-    a = frt_analyzer_new(ts, &re_analyzer_destroy_i, NULL);
-    Frt_Wrap_Struct(self, &frb_re_analyzer_mark, &frb_analyzer_free, a);
-    object_add(a, self);
+    TypedData_Get_Struct(self, FrtAnalyzer, &frb_reg_exp_analyzer_t, a);
+    frt_analyzer_init(a, ts, &re_analyzer_destroy_i, NULL);
+    a->ranalyzer = self;
     return self;
 }
 
@@ -1591,63 +1507,16 @@ frb_re_analyzer_init(int argc, VALUE *argv, VALUE self)
  *  field_name:: name of the field to be tokenized
  *  input::      data from the field to be tokenized
  */
-static VALUE
-frb_re_analyzer_token_stream(VALUE self, VALUE rfield, VALUE rtext)
-{
+static VALUE frb_re_analyzer_token_stream(VALUE self, VALUE rfield, VALUE rtext) {
     FrtTokenStream *ts;
     FrtAnalyzer *a;
-    GET_A(a, self);
-
-    StringValue(rtext);
+    TypedData_Get_Struct(self, FrtAnalyzer, &frb_reg_exp_analyzer_t, a);
 
-    ts = frt_a_get_ts(a, frb_field(rfield), rs2s(rtext));
+    ts = frt_a_get_ts(a, frb_field(rfield), rs2s(rtext), rb_enc_get(rtext));
 
-    /* Make sure that there is no entry already */
-    object_set(&ts->text, rtext);
-    if (ts->next == &rets_next) {
-        RETS(ts)->rtext = rtext;
-        rb_hash_aset(object_space, ((VALUE)ts)|1, rtext);
-    }
-    else {
-        RETS(((FrtTokenFilter*)ts)->sub_ts)->rtext = rtext;
-        rb_hash_aset(object_space, ((VALUE)((FrtTokenFilter*)ts)->sub_ts)|1, rtext);
-    }
     return get_rb_token_stream(ts);
 }
 
-/****************************************************************************
- *
- * Locale stuff
- *
- ****************************************************************************/
-
-/*
- *  call-seq:
- *     Ferret.locale -> locale_str
- *
- *  Returns a string corresponding to the locale set. For example;
- *
- *     puts Ferret.locale #=> "en_US.UTF-8"
- */
-static VALUE frb_get_locale(VALUE self)
-{
-    return (frb_locale ? rb_str_new2(frb_locale) : Qnil);
-}
-
-/*
- *  call-seq:
- *     Ferret.locale = "en_US.UTF-8"
- *
- *  Set the global locale. You should use this method to set different locales
- *  when indexing documents with different encodings.
- */
-static VALUE frb_set_locale(VALUE self, VALUE locale)
-{
-    char *l = ((locale == Qnil) ? NULL : rs2s(rb_obj_as_string(locale)));
-    frb_locale = setlocale(LC_CTYPE, l);
-    return frb_locale ? rb_str_new2(frb_locale) : Qnil;
-}
-
 /****************************************************************************
  *
  * Init Functions
@@ -1680,8 +1549,7 @@ static VALUE frb_set_locale(VALUE self, VALUE locale)
  *          equal to @text.length(), as the term text may have been
  *          altered by a stemmer or some other filter.
  */
-static void Init_Token(void)
-{
+static void Init_Token(void) {
     cToken = rb_define_class_under(mAnalysis, "Token", rb_cObject);
     rb_define_alloc_func(cToken, frb_token_alloc);
     rb_include_module(cToken, rb_mComparable);
@@ -1712,8 +1580,7 @@ static void Init_Token(void)
  *  Tokenizer::   a TokenStream whose input is a string
  *  TokenFilter:: a TokenStream whose input is another TokenStream
  */
-static void Init_TokenStream(void)
-{
+static void Init_TokenStream(void) {
     cTokenStream = rb_define_class_under(mAnalysis, "TokenStream", rb_cObject);
     frb_mark_cclass(cTokenStream);
     rb_define_method(cTokenStream, "next", frb_ts_next, 0);
@@ -1721,30 +1588,6 @@ static void Init_TokenStream(void)
     rb_define_method(cTokenStream, "text", frb_ts_get_text, 0);
 }
 
-/*
- *  Document-class: Ferret::Analysis::AsciiLetterTokenizer
- *
- *  == Summary
- *
- *  A LetterTokenizer is a tokenizer that divides text at non-ASCII letters.
- *  That is to say, it defines tokens as maximal strings of adjacent letters,
- *  as defined by the regular expression _/[A-Za-z]+/_.
- *
- *  === Example
- *
- *    "Dave's résumé, at http://www.davebalmain.com/ 1234"
- *      => ["Dave", "s", "r", "sum", "at", "http", "www", "davebalmain", "com"]
- */
-static void Init_AsciiLetterTokenizer(void)
-{
-    cAsciiLetterTokenizer =
-        rb_define_class_under(mAnalysis, "AsciiLetterTokenizer", cTokenStream);
-    frb_mark_cclass(cAsciiLetterTokenizer);
-    rb_define_alloc_func(cAsciiLetterTokenizer, frb_data_alloc);
-    rb_define_method(cAsciiLetterTokenizer, "initialize",
-                     frb_a_letter_tokenizer_init, 1);
-}
-
 /*
  *  Document-class: Ferret::Analysis::LetterTokenizer
  *
@@ -1760,38 +1603,11 @@ static void Init_AsciiLetterTokenizer(void)
  *    "Dave's résumé, at http://www.davebalmain.com/ 1234"
  *      => ["Dave", "s", "résumé", "at", "http", "www", "davebalmain", "com"]
  */
-static void Init_LetterTokenizer(void)
-{
-    cLetterTokenizer =
-        rb_define_class_under(mAnalysis, "LetterTokenizer", cTokenStream);
+static void Init_LetterTokenizer(void) {
+    cLetterTokenizer = rb_define_class_under(mAnalysis, "LetterTokenizer", cTokenStream);
     frb_mark_cclass(cLetterTokenizer);
-    rb_define_alloc_func(cLetterTokenizer, frb_data_alloc);
-    rb_define_method(cLetterTokenizer, "initialize",
-                     frb_letter_tokenizer_init, -1);
-}
-
-/*
- *  Document-class: Ferret::Analysis::AsciiWhiteSpaceTokenizer
- *
- *  == Summary
- *
- *  A WhiteSpaceTokenizer is a tokenizer that divides text at white-space.
- *  Adjacent sequences of non-WhiteSpace characters form tokens.
- *
- *  === Example
- *
- *    "Dave's résumé, at http://www.davebalmain.com/ 1234"
- *      => ["Dave's", "résumé,", "at", "http://www.davebalmain.com", "1234"]
- */
-static void Init_AsciiWhiteSpaceTokenizer(void)
-{
-    cAsciiWhiteSpaceTokenizer =
-        rb_define_class_under(mAnalysis, "AsciiWhiteSpaceTokenizer",
-                              cTokenStream);
-    frb_mark_cclass(cAsciiWhiteSpaceTokenizer);
-    rb_define_alloc_func(cAsciiWhiteSpaceTokenizer, frb_data_alloc);
-    rb_define_method(cAsciiWhiteSpaceTokenizer, "initialize",
-                     frb_a_whitespace_tokenizer_init, 1);
+    rb_define_alloc_func(cLetterTokenizer, frb_letter_tokenizer_alloc);
+    rb_define_method(cLetterTokenizer, "initialize", frb_letter_tokenizer_init, -1);
 }
 
 /*
@@ -1807,38 +1623,11 @@ static void Init_AsciiWhiteSpaceTokenizer(void)
  *    "Dave's résumé, at http://www.davebalmain.com/ 1234"
  *      => ["Dave's", "résumé,", "at", "http://www.davebalmain.com", "1234"]
  */
-static void Init_WhiteSpaceTokenizer(void)
-{
-    cWhiteSpaceTokenizer =
-        rb_define_class_under(mAnalysis, "WhiteSpaceTokenizer", cTokenStream);
+static void Init_WhiteSpaceTokenizer(void) {
+    cWhiteSpaceTokenizer = rb_define_class_under(mAnalysis, "WhiteSpaceTokenizer", cTokenStream);
     frb_mark_cclass(cWhiteSpaceTokenizer);
-    rb_define_alloc_func(cWhiteSpaceTokenizer, frb_data_alloc);
-    rb_define_method(cWhiteSpaceTokenizer, "initialize",
-                     frb_whitespace_tokenizer_init, -1);
-}
-
-/*
- *  Document-class: Ferret::Analysis::AsciiStandardTokenizer
- *
- *  == Summary
- *
- *  The standard tokenizer is an advanced tokenizer which tokenizes most
- *  words correctly as well as tokenizing things like email addresses, web
- *  addresses, phone numbers, etc.
- *
- *  === Example
- *
- *    "Dave's résumé, at http://www.davebalmain.com/ 1234"
- *      => ["Dave's", "r", "sum", "at", "http://www.davebalmain.com", "1234"]
- */
-static void Init_AsciiStandardTokenizer(void)
-{
-    cAsciiStandardTokenizer =
-        rb_define_class_under(mAnalysis, "AsciiStandardTokenizer", cTokenStream);
-    frb_mark_cclass(cAsciiStandardTokenizer);
-    rb_define_alloc_func(cAsciiStandardTokenizer, frb_data_alloc);
-    rb_define_method(cAsciiStandardTokenizer, "initialize",
-                     frb_a_standard_tokenizer_init, 1);
+    rb_define_alloc_func(cWhiteSpaceTokenizer, frb_whitespace_tokenizer_alloc);
+    rb_define_method(cWhiteSpaceTokenizer, "initialize", frb_whitespace_tokenizer_init, -1);
 }
 
 /*
@@ -1855,14 +1644,11 @@ static void Init_AsciiStandardTokenizer(void)
  *    "Dave's résumé, at http://www.davebalmain.com/ 1234"
  *      => ["Dave's", "résumé", "at", "http://www.davebalmain.com", "1234"]
  */
-static void Init_StandardTokenizer(void)
-{
-    cStandardTokenizer =
-        rb_define_class_under(mAnalysis, "StandardTokenizer", cTokenStream);
+static void Init_StandardTokenizer(void) {
+    cStandardTokenizer = rb_define_class_under(mAnalysis, "StandardTokenizer", cTokenStream);
     frb_mark_cclass(cStandardTokenizer);
-    rb_define_alloc_func(cStandardTokenizer, frb_data_alloc);
-    rb_define_method(cStandardTokenizer, "initialize",
-                     frb_standard_tokenizer_init, 1);
+    rb_define_alloc_func(cStandardTokenizer, frb_standard_tokenizer_alloc);
+    rb_define_method(cStandardTokenizer, "initialize", frb_standard_tokenizer_init, -1);
 }
 
 /*
@@ -1885,16 +1671,13 @@ static void Init_StandardTokenizer(void)
  *    "Dave's résumé, at http://www.davebalmain.com/ 1234"
  *      => ["Dave", "s", "résumé", "at", "http", "www", "davebalmain", "com"]
  */
-static void Init_RegExpTokenizer(void)
-{
-    cRegExpTokenizer =
-        rb_define_class_under(mAnalysis, "RegExpTokenizer", cTokenStream);
+static void Init_RegExpTokenizer(void) {
+    cRegExpTokenizer = rb_define_class_under(mAnalysis, "RegExpTokenizer", cTokenStream);
     frb_mark_cclass(cRegExpTokenizer);
     rtoken_re = rb_reg_new(TOKEN_RE, strlen(TOKEN_RE), 0);
     rb_define_const(cRegExpTokenizer, "REGEXP", rtoken_re);
-    rb_define_alloc_func(cRegExpTokenizer, frb_data_alloc);
-    rb_define_method(cRegExpTokenizer, "initialize",
-                     frb_rets_init, -1);
+    rb_define_alloc_func(cRegExpTokenizer, frb_reg_exp_tokenizer_alloc);
+    rb_define_method(cRegExpTokenizer, "initialize", frb_rets_init, -1);
     rb_define_method(cRegExpTokenizer, "text=", frb_rets_set_text, 1);
     rb_define_method(cRegExpTokenizer, "text", frb_rets_get_text, 0);
 }
@@ -1902,30 +1685,6 @@ static void Init_RegExpTokenizer(void)
 /***************/
 /*** Filters ***/
 /***************/
-
-/*
- *  Document-class: Ferret::Analysis::AsciiLowerCaseFilter
- *
- *  == Summary
- *
- *  AsciiLowerCaseFilter normalizes a token's text to lowercase but only for
- *  ASCII characters. For other characters use LowerCaseFilter.
- *
- *  === Example
- *
- *    ["One", "TWO", "three", "RÉSUMÉ"] => ["one", "two", "three", "rÉsumÉ"]
- *
- */
-static void Init_AsciiLowerCaseFilter(void)
-{
-    cAsciiLowerCaseFilter =
-        rb_define_class_under(mAnalysis, "AsciiLowerCaseFilter", cTokenStream);
-    frb_mark_cclass(cAsciiLowerCaseFilter);
-    rb_define_alloc_func(cAsciiLowerCaseFilter, frb_data_alloc);
-    rb_define_method(cAsciiLowerCaseFilter, "initialize",
-                     frb_a_lowercase_filter_init, 1);
-}
-
 /*
  *  Document-class: Ferret::Analysis::LowerCaseFilter
  *
@@ -1939,14 +1698,11 @@ static void Init_AsciiLowerCaseFilter(void)
  *    ["One", "TWO", "three", "RÉSUMÉ"] => ["one", "two", "three", "résumé"]
  *
  */
-static void Init_LowerCaseFilter(void)
-{
-    cLowerCaseFilter =
-        rb_define_class_under(mAnalysis, "LowerCaseFilter", cTokenStream);
+static void Init_LowerCaseFilter(void) {
+    cLowerCaseFilter = rb_define_class_under(mAnalysis, "LowerCaseFilter", cTokenStream);
     frb_mark_cclass(cLowerCaseFilter);
-    rb_define_alloc_func(cLowerCaseFilter, frb_data_alloc);
-    rb_define_method(cLowerCaseFilter, "initialize",
-                     frb_lowercase_filter_init, 1);
+    rb_define_alloc_func(cLowerCaseFilter, frb_lowercase_filter_alloc);
+    rb_define_method(cLowerCaseFilter, "initialize", frb_lowercase_filter_init, 1);
 }
 
 /*
@@ -1964,12 +1720,10 @@ static void Init_LowerCaseFilter(void)
  *    ["e-mail", "set-up"] => ["email", "e", "mail", "setup", "set", "up"]
  *
  */
-static void Init_HyphenFilter(void)
-{
-    cHyphenFilter =
-        rb_define_class_under(mAnalysis, "HyphenFilter", cTokenStream);
+static void Init_HyphenFilter(void) {
+    cHyphenFilter = rb_define_class_under(mAnalysis, "HyphenFilter", cTokenStream);
     frb_mark_cclass(cHyphenFilter);
-    rb_define_alloc_func(cHyphenFilter, frb_data_alloc);
+    rb_define_alloc_func(cHyphenFilter, frb_hyphen_filter_alloc);
     rb_define_method(cHyphenFilter, "initialize", frb_hyphen_filter_init, 1);
 }
 
@@ -2014,14 +1768,11 @@ static void Init_HyphenFilter(void)
  *     }
  *     filt = MappingFilter.new(token_stream, mapping)
  */
-static void Init_MappingFilter(void)
-{
-    cMappingFilter =
-        rb_define_class_under(mAnalysis, "MappingFilter", cTokenStream);
+static void Init_MappingFilter(void) {
+    cMappingFilter = rb_define_class_under(mAnalysis, "MappingFilter", cTokenStream);
     frb_mark_cclass(cMappingFilter);
-    rb_define_alloc_func(cMappingFilter, frb_data_alloc);
-    rb_define_method(cMappingFilter, "initialize",
-                     frb_mapping_filter_init, 2);
+    rb_define_alloc_func(cMappingFilter, frb_mapping_filter_alloc);
+    rb_define_method(cMappingFilter, "initialize", frb_mapping_filter_init, 2);
 }
 
 /*
@@ -2037,14 +1788,11 @@ static void Init_MappingFilter(void)
  *
  *    ["the", "pig", "and", "whistle"] => ["pig", "whistle"]
  */
-static void Init_StopFilter(void)
-{
-    cStopFilter =
-        rb_define_class_under(mAnalysis, "StopFilter", cTokenStream);
+static void Init_StopFilter(void) {
+    cStopFilter = rb_define_class_under(mAnalysis, "StopFilter", cTokenStream);
     frb_mark_cclass(cStopFilter);
-    rb_define_alloc_func(cStopFilter, frb_data_alloc);
-    rb_define_method(cStopFilter, "initialize",
-                     frb_stop_filter_init, -1);
+    rb_define_alloc_func(cStopFilter, frb_stop_filter_alloc);
+    rb_define_method(cStopFilter, "initialize", frb_stop_filter_init, -1);
 }
 
 /*
@@ -2056,28 +1804,41 @@ static void Init_StopFilter(void)
  *  stemming algorithm.  Note: the input to the stemming filter must already
  *  be in lower case, so you will need to use LowerCaseFilter or lowercasing
  *  Tokenizer further down the Tokenizer chain in order for this to work
- *  properly!
- *
- *  === Available algorithms and encodings
- *
- *    Algorithm       Algorithm Pseudonyms       Encoding
- *    ----------------------------------------------------------------
- *     "danish",     | "da", "dan"              | "ISO_8859_1", "UTF_8"
- *     "dutch",      | "dut", "nld"             | "ISO_8859_1", "UTF_8"
- *     "english",    | "en", "eng"              | "ISO_8859_1", "UTF_8"
- *     "finnish",    | "fi", "fin"              | "ISO_8859_1", "UTF_8"
- *     "french",     | "fr", "fra", "fre"       | "ISO_8859_1", "UTF_8"
- *     "german",     | "de", "deu", "ge", "ger" | "ISO_8859_1", "UTF_8"
- *     "hungarian",  | "hu", "hun"              | "ISO_8859_1", "UTF_8"
- *     "italian",    | "it", "ita"              | "ISO_8859_1", "UTF_8"
- *     "norwegian",  | "nl", "no"               | "ISO_8859_1", "UTF_8"
- *     "porter",     |                          | "ISO_8859_1", "UTF_8"
- *     "portuguese", | "por", "pt"              | "ISO_8859_1", "UTF_8"
- *     "romanian",   | "ro", "ron", "rum"       | "ISO_8859_2", "UTF_8"
- *     "russian",    | "ru", "rus"              | "KOI8_R",     "UTF_8"
- *     "spanish",    | "es", "esl"              | "ISO_8859_1", "UTF_8"
- *     "swedish",    | "sv", "swe"              | "ISO_8859_1", "UTF_8"
- *     "turkish",    | "tr", "tur"              |               "UTF_8"
+ *  properly! Stemmers work on UTF-8 encoding only.
+ *
+ *  === Available algorithms
+ *
+ *    Algorithm       Algorithm Pseudonyms
+ *    ------------------------------------------
+ *     "arabic"      | "ar", "ara"
+ *     "armenian"    | "arm", "hy", "hye"
+ *     "basque"      | "baq", "eu", "eus"
+ *     "catalan"     | "ca", "cat"
+ *     "danish"      | "da", "dan"
+ *     "dutch"       | "dut", "nl", "nld"
+ *     "english"     | "en", "eng"
+ *     "finnish"     | "fi", "fin"
+ *     "french"      | "fr", "fra", "fre"
+ *     "german"      | "de", "deu", "ge", "ger"
+ *     "greek"       | "el", "ell", "gre"
+ *     "hindi"       | "hi", "hin"
+ *     "hungarian"   | "hu", "hun"
+ *     "indonesian"  | "id", "ind"
+ *     "italian"     | "it", "ita"
+ *     "irish"       | "ga", "gle"
+ *     "lithuanian"  | "lit"
+ *     "nepali"      | "ne", "nep"
+ *     "norwegian"   | "nl", "no", "nor"
+ *     "porter"      |
+ *     "portuguese"  | "por", "pt"
+ *     "romanian"    | "ro", "ron", "rum"
+ *     "russian"     | "ru", "rus"
+ *     "serbian"     | "sr", "srp"
+ *     "spanish"     | "es", "esl", "spa"
+ *     "swedish"     | "sv", "swe"
+ *     "tamil"       | "ta", "tam"
+ *     "turkish"     | "tr", "tur"
+ *     "yiddish"     | "yi", "yid"
  *
  *
  *  === New Stemmers
@@ -2109,14 +1870,11 @@ static void Init_StopFilter(void)
  *  algorithm::    The algorithm (or language) to use (default: "english")
  *  encoding::     The encoding of the data (default: "UTF-8")
  */
-static void Init_StemFilter(void)
-{
-    cStemFilter =
-        rb_define_class_under(mAnalysis, "StemFilter", cTokenStream);
+static void Init_StemFilter(void) {
+    cStemFilter = rb_define_class_under(mAnalysis, "StemFilter", cTokenStream);
     frb_mark_cclass(cStemFilter);
-    rb_define_alloc_func(cStemFilter, frb_data_alloc);
-    rb_define_method(cStemFilter, "initialize",
-                     frb_stem_filter_init, -1);
+    rb_define_alloc_func(cStemFilter, frb_stem_filter_alloc);
+    rb_define_method(cStemFilter, "initialize", frb_stem_filter_init, -1);
 }
 
 /*************************/
@@ -2153,61 +1911,21 @@ static void Init_StemFilter(void)
  *      end
  *    end
  */
-static void Init_Analyzer(void)
-{
-    cAnalyzer =
-        rb_define_class_under(mAnalysis, "Analyzer", rb_cObject);
+static void Init_Analyzer(void) {
+    cAnalyzer = rb_define_class_under(mAnalysis, "Analyzer", rb_cObject);
     frb_mark_cclass(cAnalyzer);
-    rb_define_alloc_func(cAnalyzer, frb_data_alloc);
-    rb_define_method(cAnalyzer, "initialize", frb_letter_analyzer_init, -1);
+    rb_define_alloc_func(cAnalyzer, frb_analyzer_alloc);
+    rb_define_method(cAnalyzer, "initialize", frb_analyzer_init, -1);
     rb_define_method(cAnalyzer, "token_stream", frb_analyzer_token_stream, 2);
 }
 
-/*
- *  Document-class: Ferret::Analysis::AsciiLetterAnalyzer
- *
- *  == Summary
- *
- *  An AsciiLetterAnalyzer creates a TokenStream that splits the input up into
- *  maximal strings of ASCII characters. If implemented in Ruby it would look
- *  like;
- *
- *    class AsciiLetterAnalyzer
- *      def initialize(lower = true)
- *        @lower = lower
- *      end
- *
- *      def token_stream(field, str)
- *        if @lower
- *          return AsciiLowerCaseFilter.new(AsciiLetterTokenizer.new(str))
- *        else
- *          return AsciiLetterTokenizer.new(str)
- *        end
- *      end
- *    end
- *
- *  As you can see it makes use of the AsciiLetterTokenizer and
- *  AsciiLowerCaseFilter. Note that this tokenizer won't recognize non-ASCII
- *  characters so you should use the LetterAnalyzer is you want to analyze
- *  multi-byte data like "UTF-8".
- */
-static void Init_AsciiLetterAnalyzer(void)
-{
-    cAsciiLetterAnalyzer =
-        rb_define_class_under(mAnalysis, "AsciiLetterAnalyzer", cAnalyzer);
-    frb_mark_cclass(cAsciiLetterAnalyzer);
-    rb_define_alloc_func(cAsciiLetterAnalyzer, frb_data_alloc);
-    rb_define_method(cAsciiLetterAnalyzer, "initialize",
-                     frb_a_letter_analyzer_init, -1);
-}
-
 /*
  *  Document-class: Ferret::Analysis::LetterAnalyzer
  *
  *  == Summary
  *
  *  A LetterAnalyzer creates a TokenStream that splits the input up into
- *  maximal strings of characters as recognized by the current locale. If
+ *  maximal strings of characters as recognized by the current encoding. If
  *  implemented in Ruby it would look like;
  *
  *    class LetterAnalyzer
@@ -2222,51 +1940,11 @@ static void Init_AsciiLetterAnalyzer(void)
  *
  *  As you can see it makes use of the LetterTokenizer.
  */
-static void Init_LetterAnalyzer(void)
-{
-    cLetterAnalyzer =
-        rb_define_class_under(mAnalysis, "LetterAnalyzer", cAnalyzer);
+static void Init_LetterAnalyzer(void) {
+    cLetterAnalyzer = rb_define_class_under(mAnalysis, "LetterAnalyzer", cAnalyzer);
     frb_mark_cclass(cLetterAnalyzer);
-    rb_define_alloc_func(cLetterAnalyzer, frb_data_alloc);
-    rb_define_method(cLetterAnalyzer, "initialize",
-                     frb_letter_analyzer_init, -1);
-}
-
-/*
- *  Document-class: Ferret::Analysis::AsciiWhiteSpaceAnalyzer
- *
- *  == Summary
- *
- *  The AsciiWhiteSpaceAnalyzer recognizes tokens as maximal strings of
- *  non-whitespace characters. If implemented in Ruby the
- *  AsciiWhiteSpaceAnalyzer would look like;
- *
- *    class AsciiWhiteSpaceAnalyzer
- *      def initialize(lower = true)
- *        @lower = lower
- *      end
- *
- *      def token_stream(field, str)
- *        if @lower
- *          return AsciiLowerCaseFilter.new(AsciiWhiteSpaceTokenizer.new(str))
- *        else
- *          return AsciiWhiteSpaceTokenizer.new(str)
- *        end
- *      end
- *    end
- *
- *  As you can see it makes use of the AsciiWhiteSpaceTokenizer. You should
- *  use WhiteSpaceAnalyzer if you want to recognize multibyte encodings such
- *  as "UTF-8".
- */
-static void Init_AsciiWhiteSpaceAnalyzer(void)
-{
-    cAsciiWhiteSpaceAnalyzer =
-        rb_define_class_under(mAnalysis, "AsciiWhiteSpaceAnalyzer", cAnalyzer);
-    frb_mark_cclass(cAsciiWhiteSpaceAnalyzer);
-    rb_define_alloc_func(cAsciiWhiteSpaceAnalyzer, frb_data_alloc);
-    rb_define_method(cAsciiWhiteSpaceAnalyzer, "initialize",
-                     frb_a_white_space_analyzer_init, -1);
+    rb_define_alloc_func(cLetterAnalyzer, frb_letter_analyzer_alloc);
+    rb_define_method(cLetterAnalyzer, "initialize", frb_letter_analyzer_init, -1);
 }
 
 /*
@@ -2290,51 +1968,11 @@ static void Init_AsciiWhiteSpaceAnalyzer(void)
  *
  *  As you can see it makes use of the WhiteSpaceTokenizer.
  */
-static void Init_WhiteSpaceAnalyzer(void)
-{
-    cWhiteSpaceAnalyzer =
-        rb_define_class_under(mAnalysis, "WhiteSpaceAnalyzer", cAnalyzer);
+static void Init_WhiteSpaceAnalyzer(void) {
+    cWhiteSpaceAnalyzer = rb_define_class_under(mAnalysis, "WhiteSpaceAnalyzer", cAnalyzer);
     frb_mark_cclass(cWhiteSpaceAnalyzer);
-    rb_define_alloc_func(cWhiteSpaceAnalyzer, frb_data_alloc);
-    rb_define_method(cWhiteSpaceAnalyzer, "initialize",
-                     frb_white_space_analyzer_init, -1);
-}
-
-/*
- *  Document-class: Ferret::Analysis::AsciiStandardAnalyzer
- *
- *  == Summary
- *
- *  The AsciiStandardAnalyzer is the most advanced of the available
- *  ASCII-analyzers. If it were implemented in Ruby it would look like this;
- *
- *    class AsciiStandardAnalyzer
- *      def initialize(stop_words = FRT_FULL_ENGLISH_STOP_WORDS, lower = true)
- *        @lower = lower
- *        @stop_words = stop_words
- *      end
- *
- *      def token_stream(field, str)
- *        ts = AsciiStandardTokenizer.new(str)
- *        ts = AsciiLowerCaseFilter.new(ts) if @lower
- *        ts = StopFilter.new(ts, @stop_words)
- *        ts = HyphenFilter.new(ts)
- *      end
- *    end
- *
- *  As you can see it makes use of the AsciiStandardTokenizer and you can also
- *  add your own list of stop-words if you wish. Note that this tokenizer
- *  won't recognize non-ASCII characters so you should use the
- *  StandardAnalyzer is you want to analyze multi-byte data like "UTF-8".
- */
-static void Init_AsciiStandardAnalyzer(void)
-{
-    cAsciiStandardAnalyzer =
-        rb_define_class_under(mAnalysis, "AsciiStandardAnalyzer", cAnalyzer);
-    frb_mark_cclass(cAsciiStandardAnalyzer);
-    rb_define_alloc_func(cAsciiStandardAnalyzer, frb_data_alloc);
-    rb_define_method(cAsciiStandardAnalyzer, "initialize",
-                     frb_a_standard_analyzer_init, -1);
+    rb_define_alloc_func(cWhiteSpaceAnalyzer, frb_whitespace_analyzer_alloc);
+    rb_define_method(cWhiteSpaceAnalyzer, "initialize", frb_whitespace_analyzer_init, -1);
 }
 
 /*
@@ -2362,14 +2000,11 @@ static void Init_AsciiStandardAnalyzer(void)
  *  As you can see it makes use of the StandardTokenizer and you can also add
  *  your own list of stopwords if you wish.
  */
-static void Init_StandardAnalyzer(void)
-{
-    cStandardAnalyzer =
-        rb_define_class_under(mAnalysis, "StandardAnalyzer", cAnalyzer);
+static void Init_StandardAnalyzer(void) {
+    cStandardAnalyzer = rb_define_class_under(mAnalysis, "StandardAnalyzer", cAnalyzer);
     frb_mark_cclass(cStandardAnalyzer);
-    rb_define_alloc_func(cStandardAnalyzer, frb_data_alloc);
-    rb_define_method(cStandardAnalyzer, "initialize",
-                     frb_standard_analyzer_init, -1);
+    rb_define_alloc_func(cStandardAnalyzer, frb_standard_analyzer_alloc);
+    rb_define_method(cStandardAnalyzer, "initialize", frb_standard_analyzer_init, -1);
 }
 
 /*
@@ -2392,20 +2027,14 @@ static void Init_StandardAnalyzer(void)
  *    # Use a custom analyzer on the :created_at field
  *    pfa[:created_at] = DateAnalyzer.new
  */
-static void Init_PerFieldAnalyzer(void)
-{
-    cPerFieldAnalyzer =
-        rb_define_class_under(mAnalysis, "PerFieldAnalyzer", cAnalyzer);
+static void Init_PerFieldAnalyzer(void) {
+    cPerFieldAnalyzer = rb_define_class_under(mAnalysis, "PerFieldAnalyzer", cAnalyzer);
     frb_mark_cclass(cPerFieldAnalyzer);
-    rb_define_alloc_func(cPerFieldAnalyzer, frb_data_alloc);
-    rb_define_method(cPerFieldAnalyzer, "initialize",
-                     frb_per_field_analyzer_init, 1);
-    rb_define_method(cPerFieldAnalyzer, "add_field",
-                     frb_per_field_analyzer_add_field, 2);
-    rb_define_method(cPerFieldAnalyzer, "[]=",
-                     frb_per_field_analyzer_add_field, 2);
-    rb_define_method(cPerFieldAnalyzer, "token_stream",
-                     frb_pfa_analyzer_token_stream, 2);
+    rb_define_alloc_func(cPerFieldAnalyzer, frb_per_field_analyzer_alloc);
+    rb_define_method(cPerFieldAnalyzer, "initialize", frb_per_field_analyzer_init, 1);
+    rb_define_method(cPerFieldAnalyzer, "add_field", frb_per_field_analyzer_add_field, 2);
+    rb_define_method(cPerFieldAnalyzer, "[]=", frb_per_field_analyzer_add_field, 2);
+    rb_define_method(cPerFieldAnalyzer, "token_stream", frb_pfa_analyzer_token_stream, 2);
 }
 
 /*
@@ -2435,16 +2064,12 @@ static void Init_PerFieldAnalyzer(void)
  *
  *    csv_analyzer = RegExpAnalyzer.new(/[^,]+/, false)
  */
-static void Init_RegExpAnalyzer(void)
-{
-    cRegExpAnalyzer =
-        rb_define_class_under(mAnalysis, "RegExpAnalyzer", cAnalyzer);
+static void Init_RegExpAnalyzer(void) {
+    cRegExpAnalyzer = rb_define_class_under(mAnalysis, "RegExpAnalyzer", cAnalyzer);
     frb_mark_cclass(cRegExpAnalyzer);
-    rb_define_alloc_func(cRegExpAnalyzer, frb_data_alloc);
-    rb_define_method(cRegExpAnalyzer, "initialize",
-                     frb_re_analyzer_init, -1);
-    rb_define_method(cRegExpAnalyzer, "token_stream",
-                     frb_re_analyzer_token_stream, 2);
+    rb_define_alloc_func(cRegExpAnalyzer, frb_reg_exp_analyzer_alloc);
+    rb_define_method(cRegExpAnalyzer, "initialize", frb_re_analyzer_init, -1);
+    rb_define_method(cRegExpAnalyzer, "token_stream", frb_re_analyzer_token_stream, 2);
 }
 
 /* rdoc hack
@@ -2491,9 +2116,7 @@ extern VALUE mFerret = rb_define_module("Ferret");
  *  offsets of 10 and 19 respectively ("Beginning".length == 9) but Token#text
  *  might be "begin" (after stemming).
  */
-void
-Init_Analysis(void)
-{
+void Init_Analysis(void) {
     mAnalysis = rb_define_module_under(mFerret, "Analysis");
 
     /* TokenStream Methods */
@@ -2508,56 +2131,30 @@ Init_Analysis(void)
     object_space = rb_hash_new();
     rb_define_const(mFerret, "OBJECT_SPACE", object_space);
 
-    /*** * * Locale stuff * * ***/
-    rb_define_singleton_method(mFerret, "locale=", frb_set_locale, 1);
-    rb_define_singleton_method(mFerret, "locale", frb_get_locale, 0);
-
-    rb_define_const(mAnalysis, "ENGLISH_STOP_WORDS",
-                    get_rstopwords(FRT_ENGLISH_STOP_WORDS));
-    rb_define_const(mAnalysis, "FULL_ENGLISH_STOP_WORDS",
-                    get_rstopwords(FRT_FULL_ENGLISH_STOP_WORDS));
-    rb_define_const(mAnalysis, "EXTENDED_ENGLISH_STOP_WORDS",
-                    get_rstopwords(FRT_EXTENDED_ENGLISH_STOP_WORDS));
-    rb_define_const(mAnalysis, "FULL_FRENCH_STOP_WORDS",
-                    get_rstopwords(FRT_FULL_FRENCH_STOP_WORDS));
-    rb_define_const(mAnalysis, "FULL_SPANISH_STOP_WORDS",
-                    get_rstopwords(FRT_FULL_SPANISH_STOP_WORDS));
-    rb_define_const(mAnalysis, "FULL_PORTUGUESE_STOP_WORDS",
-                    get_rstopwords(FRT_FULL_PORTUGUESE_STOP_WORDS));
-    rb_define_const(mAnalysis, "FULL_ITALIAN_STOP_WORDS",
-                    get_rstopwords(FRT_FULL_ITALIAN_STOP_WORDS));
-    rb_define_const(mAnalysis, "FULL_GERMAN_STOP_WORDS",
-                    get_rstopwords(FRT_FULL_GERMAN_STOP_WORDS));
-    rb_define_const(mAnalysis, "FULL_DUTCH_STOP_WORDS",
-                    get_rstopwords(FRT_FULL_DUTCH_STOP_WORDS));
-    rb_define_const(mAnalysis, "FULL_SWEDISH_STOP_WORDS",
-                    get_rstopwords(FRT_FULL_SWEDISH_STOP_WORDS));
-    rb_define_const(mAnalysis, "FULL_NORWEGIAN_STOP_WORDS",
-                    get_rstopwords(FRT_FULL_NORWEGIAN_STOP_WORDS));
-    rb_define_const(mAnalysis, "FULL_DANISH_STOP_WORDS",
-                    get_rstopwords(FRT_FULL_DANISH_STOP_WORDS));
-    rb_define_const(mAnalysis, "FULL_RUSSIAN_STOP_WORDS",
-                    get_rstopwords(FRT_FULL_RUSSIAN_STOP_WORDS));
-    rb_define_const(mAnalysis, "FULL_FINNISH_STOP_WORDS",
-                    get_rstopwords(FRT_FULL_FINNISH_STOP_WORDS));
-    rb_define_const(mAnalysis, "FULL_HUNGARIAN_STOP_WORDS",
-                    get_rstopwords(FRT_FULL_HUNGARIAN_STOP_WORDS));
+    rb_define_const(mAnalysis, "ENGLISH_STOP_WORDS", get_rstopwords(FRT_ENGLISH_STOP_WORDS));
+    rb_define_const(mAnalysis, "FULL_ENGLISH_STOP_WORDS", get_rstopwords(FRT_FULL_ENGLISH_STOP_WORDS));
+    rb_define_const(mAnalysis, "EXTENDED_ENGLISH_STOP_WORDS", get_rstopwords(FRT_EXTENDED_ENGLISH_STOP_WORDS));
+    rb_define_const(mAnalysis, "FULL_FRENCH_STOP_WORDS", get_rstopwords(FRT_FULL_FRENCH_STOP_WORDS));
+    rb_define_const(mAnalysis, "FULL_SPANISH_STOP_WORDS", get_rstopwords(FRT_FULL_SPANISH_STOP_WORDS));
+    rb_define_const(mAnalysis, "FULL_PORTUGUESE_STOP_WORDS", get_rstopwords(FRT_FULL_PORTUGUESE_STOP_WORDS));
+    rb_define_const(mAnalysis, "FULL_ITALIAN_STOP_WORDS", get_rstopwords(FRT_FULL_ITALIAN_STOP_WORDS));
+    rb_define_const(mAnalysis, "FULL_GERMAN_STOP_WORDS", get_rstopwords(FRT_FULL_GERMAN_STOP_WORDS));
+    rb_define_const(mAnalysis, "FULL_DUTCH_STOP_WORDS", get_rstopwords(FRT_FULL_DUTCH_STOP_WORDS));
+    rb_define_const(mAnalysis, "FULL_SWEDISH_STOP_WORDS", get_rstopwords(FRT_FULL_SWEDISH_STOP_WORDS));
+    rb_define_const(mAnalysis, "FULL_NORWEGIAN_STOP_WORDS", get_rstopwords(FRT_FULL_NORWEGIAN_STOP_WORDS));
+    rb_define_const(mAnalysis, "FULL_DANISH_STOP_WORDS", get_rstopwords(FRT_FULL_DANISH_STOP_WORDS));
+    rb_define_const(mAnalysis, "FULL_RUSSIAN_STOP_WORDS", get_rstopwords(FRT_FULL_RUSSIAN_STOP_WORDS));
+    rb_define_const(mAnalysis, "FULL_FINNISH_STOP_WORDS", get_rstopwords(FRT_FULL_FINNISH_STOP_WORDS));
+    rb_define_const(mAnalysis, "FULL_HUNGARIAN_STOP_WORDS", get_rstopwords(FRT_FULL_HUNGARIAN_STOP_WORDS));
 
     Init_Token();
     Init_TokenStream();
 
-    Init_AsciiLetterTokenizer();
     Init_LetterTokenizer();
-
-    Init_AsciiWhiteSpaceTokenizer();
     Init_WhiteSpaceTokenizer();
-
-    Init_AsciiStandardTokenizer();
     Init_StandardTokenizer();
-
     Init_RegExpTokenizer();
 
-    Init_AsciiLowerCaseFilter();
     Init_LowerCaseFilter();
     Init_HyphenFilter();
     Init_StopFilter();
@@ -2565,13 +2162,9 @@ Init_Analysis(void)
     Init_StemFilter();
 
     Init_Analyzer();
-    Init_AsciiLetterAnalyzer();
     Init_LetterAnalyzer();
-    Init_AsciiWhiteSpaceAnalyzer();
     Init_WhiteSpaceAnalyzer();
-    Init_AsciiStandardAnalyzer();
     Init_StandardAnalyzer();
     Init_PerFieldAnalyzer();
     Init_RegExpAnalyzer();
-
 }