isomorfeus-ferret 0.12.7 → 0.13.0

data/ext/isomorfeus_ferret_ext/frt_analysis.c

@@ -1,741 +1,346 @@
 #include <string.h>
 #include <ctype.h>
-#include <wctype.h>
-#include <wchar.h>
 #include "frt_analysis.h"
 #include "frt_hash.h"
 #include "libstemmer.h"
-#include "frt_scanner.h"
 
-/****************************************************************************
- *
- * Token
- *
- ****************************************************************************/
+/*****************************************************************************/
+/*** Helpers *****************************************************************/
+/*****************************************************************************/
 
-FrtToken *frt_tk_set(FrtToken *tk,
-                     char *text, int tlen, off_t start, off_t end, int pos_inc)
-{
+/* initialized in frt_global.c */
+extern rb_encoding *utf8_encoding;
+extern OnigCodePoint cp_apostrophe;
+extern OnigCodePoint cp_dot;
+extern OnigCodePoint cp_comma;
+extern OnigCodePoint cp_backslash;
+extern OnigCodePoint cp_slash;
+extern OnigCodePoint cp_underscore;
+extern OnigCodePoint cp_dash;
+extern OnigCodePoint cp_hyphen;
+extern OnigCodePoint cp_at;
+extern OnigCodePoint cp_ampersand;
+extern OnigCodePoint cp_colon;
+
+static int cp_isnumpunc(OnigCodePoint cp) {
+    return (cp == cp_dot || cp == cp_comma || cp == cp_backslash || cp == cp_slash || cp == cp_underscore || cp == cp_dash);
+}
+
+static int cp_isurlpunc(OnigCodePoint cp) {
+    return (cp == cp_dot || cp == cp_slash || cp == cp_dash || cp == cp_underscore);
+}
+
+static int cp_enc_isurlc(OnigCodePoint cp, rb_encoding *enc) {
+    return (cp_isurlpunc(cp) || rb_enc_isalnum(cp, enc));
+}
+
+static int cp_isurlxatpunc(OnigCodePoint cp) {
+    return (cp == cp_dot || cp == cp_slash || cp == cp_dash || cp == cp_underscore || cp == cp_at);
+}
+
+static int cp_enc_isurlxatc(OnigCodePoint cp, rb_encoding *enc){
+    return (cp_isurlxatpunc(cp) || rb_enc_isalnum(cp, enc));
+}
+
+static bool cp_enc_istok(OnigCodePoint cp, rb_encoding *enc) {
+    if (rb_enc_isspace(cp, enc)) /* most common so check first. */
+        return false;
+    if (rb_enc_isalnum(cp, enc) || cp_isnumpunc(cp) ||
+        cp == cp_ampersand || cp == cp_at || cp == cp_apostrophe || cp == cp_colon) {
+        return true;
+    }
+    return false;
+}
+
+static inline int get_cp(char *start, char *end, int *cp_len, rb_encoding *enc) {
+    if (start >= end) {
+        *cp_len = 0;
+        return 0;
+    }
+    return rb_enc_codepoint_len(start, end, cp_len, enc);
+}
+
+/*****************************************************************************/
+/*** FrtToken ****************************************************************/
+/*****************************************************************************/
+
+FrtToken *frt_tk_set(FrtToken *tk, char *text, int tlen, off_t start, off_t end, int pos_inc, rb_encoding *encoding) {
     if (tlen >= FRT_MAX_WORD_SIZE) {
-        tlen = FRT_MAX_WORD_SIZE - 1;
+        tlen = FRT_MAX_WORD_SIZE - 1; // TODO: this may invalidate mbc's
+    }
+
+    if (encoding == utf8_encoding) {
+        memcpy(tk->text, text, sizeof(char) * tlen);
+    } else {
+        const unsigned char *sp = (unsigned char *)text;
+        unsigned char *dp = (unsigned char *)tk->text;
+        rb_econv_t *ec = rb_econv_open(rb_enc_name(encoding), "UTF-8", RUBY_ECONV_INVALID_REPLACE);
+        assert(ec != NULL);
+        rb_econv_convert(ec, &sp, (unsigned char *)text + tlen, &dp, (unsigned char *)tk->text + FRT_MAX_WORD_SIZE - 1, 0);
+        rb_econv_close(ec);
+        tlen = dp - (unsigned char *)tk->text;
     }
-    memcpy(tk->text, text, sizeof(char) * tlen);
     tk->text[tlen] = '\0';
-    tk->len = tlen;
-    tk->start = start;
-    tk->end = end;
+    tk->len = tlen;    // in bytes in utf8_encoding
+    tk->start = start; // in original encoding
+    tk->end = end;     // in original encoding
     tk->pos_inc = pos_inc;
     return tk;
 }
 
-static FrtToken *frt_tk_set_ts(FrtToken *tk, char *start, char *end,
-                               char *text, int pos_inc)
-{
-    return frt_tk_set(tk, start, (int)(end - start),
-                  (off_t)(start - text), (off_t)(end - text), pos_inc);
+static FrtToken *frt_tk_set_ts(FrtToken *tk, char *start, char *end, char *text, int pos_inc, rb_encoding *encoding) {
+    return frt_tk_set(tk, start, (int)(end - start), (off_t)(start - text), (off_t)(end - text), pos_inc, encoding);
 }
 
-FrtToken *frt_tk_set_no_len(FrtToken *tk,
-                            char *text, off_t start, off_t end, int pos_inc)
-{
-    return frt_tk_set(tk, text, (int)strlen(text), start, end, pos_inc);
+FrtToken *frt_tk_set_no_len(FrtToken *tk, char *text, off_t start, off_t end, int pos_inc, rb_encoding *encoding) {
+    return frt_tk_set(tk, text, (int)strlen(text), start, end, pos_inc, encoding);
 }
 
-static FrtToken *w_tk_set(FrtToken *tk, wchar_t *text, off_t start,
-                              off_t end, int pos_inc)
-{
-    int len = wcstombs(tk->text, text, FRT_MAX_WORD_SIZE - 1);
-    tk->text[len] = '\0';
-    tk->len = len;
-    tk->start = start;
-    tk->end = end;
-    tk->pos_inc = pos_inc;
-    return tk;
-}
-
-int frt_tk_eq(FrtToken *tk1, FrtToken *tk2)
-{
+int frt_tk_eq(FrtToken *tk1, FrtToken *tk2) {
     return (strcmp((char *)tk1->text, (char *)tk2->text) == 0 &&
             tk1->start == tk2->start && tk1->end == tk2->end &&
             tk1->pos_inc == tk2->pos_inc);
 }
 
-int frt_tk_cmp(FrtToken *tk1, FrtToken *tk2)
-{
+int frt_tk_cmp(FrtToken *tk1, FrtToken *tk2) {
     int cmp;
     if (tk1->start > tk2->start) {
         cmp = 1;
-    }
-    else if (tk1->start < tk2->start) {
+    } else if (tk1->start < tk2->start) {
         cmp = -1;
-    }
-    else {
+    } else {
         if (tk1->end > tk2->end) {
             cmp = 1;
-        }
-        else if (tk1->end < tk2->end) {
+        } else if (tk1->end < tk2->end) {
             cmp = -1;
-        }
-        else {
+        } else {
             cmp = strcmp((char *)tk1->text, (char *)tk2->text);
         }
     }
     return cmp;
 }
 
-void frt_tk_destroy(void *p)
-{
+void frt_tk_destroy(void *p) {
     free(p);
 }
 
-FrtToken *frt_tk_new()
-{
+FrtToken *frt_tk_new(void) {
     return FRT_ALLOC(FrtToken);
 }
-/****************************************************************************
- *
- * TokenStream
- *
- ****************************************************************************/
 
-void frt_ts_deref(FrtTokenStream *ts)
-{
-    if (--ts->ref_cnt <= 0) {
+/*****************************************************************************/
+/*** FrtTokenStream **********************************************************/
+/*****************************************************************************/
+
+void frt_ts_deref(FrtTokenStream *ts) {
+    if (--ts->ref_cnt <= 0)
         ts->destroy_i(ts);
-    }
 }
 
-static FrtTokenStream *ts_reset(FrtTokenStream *ts, char *text)
-{
+FrtTokenStream *frt_ts_reset(FrtTokenStream *ts, char *text, rb_encoding *encoding) {
     ts->t = ts->text = text;
+    ts->length = strlen(text);
+    ts->encoding = encoding;
     return ts;
 }
 
-FrtTokenStream *frt_ts_clone_size(FrtTokenStream *orig_ts, size_t size)
-{
+FrtTokenStream *frt_ts_clone_size(FrtTokenStream *orig_ts, size_t size) {
     FrtTokenStream *ts = (FrtTokenStream *)frt_ecalloc(size);
     memcpy(ts, orig_ts, size);
     ts->ref_cnt = 1;
+    ts->rts = 0;
+    ts->rts = Qnil;
     return ts;
 }
 
-FrtTokenStream *frt_ts_new_i(size_t size)
-{
-    FrtTokenStream *ts = (FrtTokenStream *)frt_ecalloc(size);
+FrtTokenStream *frt_ts_alloc_i(size_t size) {
+    return (FrtTokenStream *)frt_ecalloc(size);
+}
 
+FrtTokenStream *frt_ts_init(FrtTokenStream *ts) {
     ts->destroy_i = (void (*)(FrtTokenStream *))&free;
-    ts->reset = &ts_reset;
+    ts->reset = &frt_ts_reset;
     ts->ref_cnt = 1;
-
+    ts->rts = Qnil;
     return ts;
 }
 
-/****************************************************************************
- * CachedTokenStream
- ****************************************************************************/
-
-#define CTS(token_stream) ((FrtCachedTokenStream *)(token_stream))
-
-static FrtTokenStream *cts_clone_i(FrtTokenStream *orig_ts)
-{
-    return frt_ts_clone_size(orig_ts, sizeof(FrtCachedTokenStream));
+FrtTokenStream *frt_ts_new_i(size_t size) {
+    FrtTokenStream *ts = frt_ts_alloc_i(size);
+    return frt_ts_init(ts);
 }
 
-static FrtTokenStream *cts_new()
-{
-    FrtTokenStream *ts = frt_ts_new(FrtCachedTokenStream);
-    ts->clone_i = &cts_clone_i;
-    return ts;
-}
-
-/* * Multi-byte TokenStream * */
-
-#define MBTS(token_stream) ((FrtMultiByteTokenStream *)(token_stream))
-
-static int mb_next_char(wchar_t *wchr, const char *s, mbstate_t *state)
-{
-    int num_bytes;
-    if ((num_bytes = (int)mbrtowc(wchr, s, MB_CUR_MAX, state)) < 0) {
-        const char *t = s;
-        do {
-            t++;
-            FRT_ZEROSET(state, mbstate_t);
-            num_bytes = (int)mbrtowc(wchr, t, MB_CUR_MAX, state);
-        } while ((num_bytes < 0) && (*t != 0));
-        num_bytes = t - s;
-        if (*t == 0) *wchr = 0;
-    }
-    return num_bytes;
-}
+/*****************************************************************************/
+/*** FrtCachedTokenStream ****************************************************/
+/*****************************************************************************/
 
-static FrtTokenStream *mb_ts_reset(FrtTokenStream *ts, char *text)
-{
-    FRT_ZEROSET(&(MBTS(ts)->state), mbstate_t);
-    ts_reset(ts, text);
-    return ts;
+static FrtTokenStream *cts_clone_i(FrtTokenStream *orig_ts) {
+    return frt_ts_clone_size(orig_ts, sizeof(FrtTokenStream));
 }
 
-static FrtTokenStream *mb_ts_clone_i(FrtTokenStream *orig_ts)
-{
-    return frt_ts_clone_size(orig_ts, sizeof(FrtMultiByteTokenStream));
+static FrtTokenStream *frt_cts_alloc(void) {
+    return (FrtTokenStream *)frt_ecalloc(sizeof(FrtTokenStream));
 }
 
-static FrtTokenStream *mb_ts_new()
-{
-    FrtTokenStream *ts = frt_ts_new(FrtMultiByteTokenStream);
-    ts->reset = &mb_ts_reset;
-    ts->clone_i = &mb_ts_clone_i;
+static FrtTokenStream *frt_cts_init(FrtTokenStream *ts) {
+    frt_ts_init(ts);
+    ts->reset   = &frt_ts_reset;
+    ts->clone_i = &cts_clone_i;
     ts->ref_cnt = 1;
     return ts;
 }
 
-/****************************************************************************
- *
- * Analyzer
- *
- ****************************************************************************/
-
-void frt_a_deref(FrtAnalyzer *a)
-{
-    if (--a->ref_cnt <= 0) {
-        a->destroy_i(a);
-    }
-}
-
-static void frt_a_standard_destroy_i(FrtAnalyzer *a)
-{
-    if (a->current_ts) {
-        frt_ts_deref(a->current_ts);
-    }
-    free(a);
-}
-
-static FrtTokenStream *a_standard_get_ts(FrtAnalyzer *a,
-                                      FrtSymbol field,
-                                      char *text)
-{
-    FrtTokenStream *ts;
-    (void)field;
-    ts = frt_ts_clone(a->current_ts);
-    return ts->reset(ts, text);
+static FrtTokenStream *frt_cts_new(void) {
+    FrtTokenStream *ts = frt_cts_alloc();
+    return frt_cts_init(ts);
 }
 
-FrtAnalyzer *frt_analyzer_new(FrtTokenStream *ts,
-                       void (*destroy_i)(FrtAnalyzer *a),
-                       FrtTokenStream *(*get_ts)(FrtAnalyzer *a,
-                                              FrtSymbol field,
-                                              char *text))
-{
-    FrtAnalyzer *a = FRT_ALLOC(FrtAnalyzer);
-    a->current_ts = ts;
-    a->destroy_i = (destroy_i ? destroy_i : &frt_a_standard_destroy_i);
-    a->get_ts = (get_ts ? get_ts : &a_standard_get_ts);
-    a->ref_cnt = 1;
-    return a;
-}
+/*****************************************************************************/
+/*** Tokenizer ***************************************************************/
+/*****************************************************************************/
 
-/****************************************************************************
- *
- * Non
- *
- ****************************************************************************/
+/*****************************************************************************/
+/*** FrtNonTokenizer *********************************************************/
+/*****************************************************************************/
 
-/*
- * NonTokenizer
- */
-static FrtToken *nt_next(FrtTokenStream *ts)
-{
+static FrtToken *nt_next(FrtTokenStream *ts) {
     if (ts->t) {
         size_t len = strlen(ts->t);
         ts->t = NULL;
-
-        return frt_tk_set(&(CTS(ts)->token), ts->text, len, 0, len, 1);
-    }
-    else {
+        return frt_tk_set(&(ts->token), ts->text, len, 0, len, 1, ts->encoding);
+    } else {
         return NULL;
     }
 }
 
-FrtTokenStream *frt_non_tokenizer_new()
-{
-    FrtTokenStream *ts = cts_new();
+FrtTokenStream *frt_non_tokenizer_new(void) {
+    FrtTokenStream *ts = frt_cts_new();
     ts->next = &nt_next;
     return ts;
 }
 
-/*
- * NonAnalyzer
- */
-FrtAnalyzer *frt_non_analyzer_new()
-{
-    return frt_analyzer_new(frt_non_tokenizer_new(), NULL, NULL);
-}
-
-/****************************************************************************
- *
- * Whitespace
- *
- ****************************************************************************/
+/*****************************************************************************/
+/*** FrtWhiteSpaceTokenizer **************************************************/
+/*****************************************************************************/
 
-/*
- * WhitespaceTokenizer
- */
 static FrtToken *wst_next(FrtTokenStream *ts)
 {
-    char *t = ts->t;
-    char *start;
-
-    while (*t != '\0' && isspace(*t)) {
-        t++;
-    }
-
-    if (*t == '\0') {
-        return NULL;
-    }
-
-    start = t;
-    while (*t != '\0' && !isspace(*t)) {
-        t++;
-    }
-
-    ts->t = t;
-    return frt_tk_set_ts(&(CTS(ts)->token), start, t, ts->text, 1);
-}
-
-FrtTokenStream *frt_whitespace_tokenizer_new()
-{
-    FrtTokenStream *ts = cts_new();
-    ts->next = &wst_next;
-    return ts;
-}
-
-/*
- * Multi-byte WhitespaceTokenizer
- */
-static FrtToken *mb_wst_next(FrtTokenStream *ts)
-{
-    int i;
+    int cp_len = 0;
+    OnigCodePoint cp;
+    rb_encoding *enc = ts->encoding;
+    char *end = ts->text + ts->length;
     char *start;
     char *t = ts->t;
-    wchar_t wchr;
-    mbstate_t *state = &(MBTS(ts)->state);
 
-    i = mb_next_char(&wchr, t, state);
-    while (wchr != 0 && iswspace(wchr)) {
-        t += i;
-        i = mb_next_char(&wchr, t, state);
-    }
-    if (wchr == 0) {
+    cp = get_cp(t, end, &cp_len, enc);
+    if (cp < 1)
         return NULL;
-    }
 
-    start = t;
-    t += i;
-    i = mb_next_char(&wchr, t, state);
-    while (wchr != 0 && !iswspace(wchr)) {
-        t += i;
-        i = mb_next_char(&wchr, t, state);
+    while (cp_len > 0 && rb_enc_isspace(cp, enc)) {
+        t += cp_len;
+        cp = get_cp(t, end, &cp_len, enc);
     }
-    ts->t = t;
-    return frt_tk_set_ts(&(CTS(ts)->token), start, t, ts->text, 1);
-}
-
-/*
- * Lowercasing Multi-byte WhitespaceTokenizer
- */
-static FrtToken *mb_wst_next_lc(FrtTokenStream *ts)
-{
-    int i;
-    char *start;
-    char *t = ts->t;
-    wchar_t wchr;
-    wchar_t wbuf[FRT_MAX_WORD_SIZE + 1], *w, *w_end;
-    mbstate_t *state = &(MBTS(ts)->state);
-
-    w = wbuf;
-    w_end = &wbuf[FRT_MAX_WORD_SIZE];
 
-    i = mb_next_char(&wchr, t, state);
-    while (wchr != 0 && iswspace(wchr)) {
-        t += i;
-        i = mb_next_char(&wchr, t, state);
-    }
-    if (wchr == 0) {
+    start = t;
+    if (start >= end)
         return NULL;
-    }
 
-    start = t;
-    t += i;
-    *w++ = towlower(wchr);
-    i = mb_next_char(&wchr, t, state);
-    while (wchr != 0 && !iswspace(wchr)) {
-        if (w < w_end) {
-            *w++ = towlower(wchr);
-        }
-        t += i;
-        i = mb_next_char(&wchr, t, state);
-    }
-    *w = 0;
+    do {
+        t += cp_len;
+        cp = get_cp(t, end, &cp_len, enc);
+    } while (cp_len > 0 && !rb_enc_isspace(cp, enc));
+
     ts->t = t;
-    return w_tk_set(&(CTS(ts)->token), wbuf, (off_t)(start - ts->text),
-                    (off_t)(t - ts->text), 1);
+    return frt_tk_set_ts(&(ts->token), start, t, ts->text, 1, enc);
 }
 
-FrtTokenStream *frt_mb_whitespace_tokenizer_new(bool lowercase)
-{
-    FrtTokenStream *ts = mb_ts_new();
-    ts->next = lowercase ? &mb_wst_next_lc : &mb_wst_next;
-    return ts;
+FrtTokenStream *frt_whitespace_tokenizer_alloc(void) {
+    return frt_cts_alloc();
 }
 
-/*
- * WhitespaceAnalyzers
- */
-FrtAnalyzer *frt_whitespace_analyzer_new(bool lowercase)
-{
-    FrtTokenStream *ts;
-    if (lowercase) {
-        ts = frt_lowercase_filter_new(frt_whitespace_tokenizer_new());
-    }
-    else {
-        ts = frt_whitespace_tokenizer_new();
-    }
-    return frt_analyzer_new(ts, NULL, NULL);
+FrtTokenStream *frt_whitespace_tokenizer_init(FrtTokenStream *ts) {
+    ts = frt_cts_init(ts);
+    ts->next = &wst_next;
+    return ts;
 }
 
-FrtAnalyzer *frt_mb_whitespace_analyzer_new(bool lowercase)
-{
-    return frt_analyzer_new(frt_mb_whitespace_tokenizer_new(lowercase), NULL, NULL);
+FrtTokenStream *frt_whitespace_tokenizer_new(void) {
+    FrtTokenStream *ts = frt_whitespace_tokenizer_alloc();
+    return frt_whitespace_tokenizer_init(ts);
 }
 
-/****************************************************************************
- *
- * Letter
- *
- ****************************************************************************/
+/*****************************************************************************/
+/*** FrtLetterTokenizer ******************************************************/
+/*****************************************************************************/
 
-/*
- * LetterTokenizer
- */
-static FrtToken *lt_next(FrtTokenStream *ts)
-{
+static FrtToken *lt_next(FrtTokenStream *ts) {
+    int cp_len = 0;
+    OnigCodePoint cp;
+    rb_encoding *enc = ts->encoding;
+    char *end = ts->text + ts->length;
     char *start;
     char *t = ts->t;
 
-    while (*t != '\0' && !isalpha(*t)) {
-        t++;
-    }
-
-    if (*t == '\0') {
+    cp = get_cp(t, end, &cp_len, enc);
+    if (cp < 1)
         return NULL;
-    }
 
-    start = t;
-    while (*t != '\0' && isalpha(*t)) {
-        t++;
-    }
-
-    ts->t = t;
-    return frt_tk_set_ts(&(CTS(ts)->token), start, t, ts->text, 1);
-}
-
-FrtTokenStream *frt_letter_tokenizer_new()
-{
-    FrtTokenStream *ts = cts_new();
-    ts->next = &lt_next;
-    return ts;
-}
-
-/*
- * Multi-byte LetterTokenizer
- */
-static FrtToken *mb_lt_next(FrtTokenStream *ts)
-{
-    int i;
-    char *start;
-    char *t = ts->t;
-    wchar_t wchr;
-    mbstate_t *state = &(MBTS(ts)->state);
-
-    i = mb_next_char(&wchr, t, state);
-    while (wchr != 0 && !iswalpha(wchr)) {
-        t += i;
-        i = mb_next_char(&wchr, t, state);
-    }
-
-    if (wchr == 0) {
-        return NULL;
+    while (cp_len > 0 && !rb_enc_isalpha(cp, enc)) {
+        t += cp_len;
+        cp = get_cp(t, end, &cp_len, enc);
     }
 
     start = t;
-    t += i;
-    i = mb_next_char(&wchr, t, state);
-    while (wchr != 0 && iswalpha(wchr)) {
-        t += i;
-        i = mb_next_char(&wchr, t, state);
-    }
-    ts->t = t;
-    return frt_tk_set_ts(&(CTS(ts)->token), start, t, ts->text, 1);
-}
-
-/*
- * Lowercasing Multi-byte LetterTokenizer
- */
-static FrtToken *mb_lt_next_lc(FrtTokenStream *ts)
-{
-    int i;
-    char *start;
-    char *t = ts->t;
-    wchar_t wchr;
-    wchar_t wbuf[FRT_MAX_WORD_SIZE + 1], *w, *w_end;
-    mbstate_t *state = &(MBTS(ts)->state);
-
-    w = wbuf;
-    w_end = &wbuf[FRT_MAX_WORD_SIZE];
-
-    i = mb_next_char(&wchr, t, state);
-    while (wchr != 0 && !iswalpha(wchr)) {
-        t += i;
-        i = mb_next_char(&wchr, t, state);
-    }
-    if (wchr == 0) {
-        return NULL;
-    }
-
-    start = t;
-    t += i;
-    *w++ = towlower(wchr);
-    i = mb_next_char(&wchr, t, state);
-    while (wchr != 0 && iswalpha(wchr)) {
-        if (w < w_end) {
-            *w++ = towlower(wchr);
-        }
-        t += i;
-        i = mb_next_char(&wchr, t, state);
-    }
-    *w = 0;
-    ts->t = t;
-    return w_tk_set(&(CTS(ts)->token), wbuf, (off_t)(start - ts->text),
-                    (off_t)(t - ts->text), 1);
-}
-
-FrtTokenStream *frt_mb_letter_tokenizer_new(bool lowercase)
-{
-    FrtTokenStream *ts = mb_ts_new();
-    ts->next = lowercase ? &mb_lt_next_lc : &mb_lt_next;
-    return ts;
-}
-
-/*
- * LetterAnalyzers
- */
-FrtAnalyzer *frt_letter_analyzer_new(bool lowercase)
-{
-    FrtTokenStream *ts;
-    if (lowercase) {
-        ts = frt_lowercase_filter_new(frt_letter_tokenizer_new());
-    }
-    else {
-        ts = frt_letter_tokenizer_new();
-    }
-    return frt_analyzer_new(ts, NULL, NULL);
-}
-
-FrtAnalyzer *frt_mb_letter_analyzer_new(bool lowercase)
-{
-    return frt_analyzer_new(frt_mb_letter_tokenizer_new(lowercase), NULL, NULL);
-}
-
-/****************************************************************************
- *
- * Standard
- *
- ****************************************************************************/
-
-#define STDTS(token_stream) ((FrtStandardTokenizer *)(token_stream))
-
-/*
- * FrtStandardTokenizer
- */
-static FrtToken *std_next(FrtTokenStream *ts)
-{
-    FrtStandardTokenizer *std_tz = STDTS(ts);
-    const char *start = NULL;
-    const char *end = NULL;
-    int len;
-    FrtToken *tk = &(CTS(ts)->token);
-
-    switch (std_tz->type) {
-        case FRT_STT_ASCII:
-            frt_std_scan(ts->t, tk->text, sizeof(tk->text) - 1,
-                         &start, &end, &len);
-            break;
-        case FRT_STT_MB:
-            frt_std_scan_mb(ts->t, tk->text, sizeof(tk->text) - 1,
-                            &start, &end, &len);
-            break;
-        case FRT_STT_UTF8:
-            frt_std_scan_utf8(ts->t, tk->text, sizeof(tk->text) - 1,
-                              &start, &end, &len);
-            break;
-    }
-
-    if (len == 0)
+    if (start >= end)
         return NULL;
 
-    ts->t       = (char *)end;
-    tk->len     = len;
-    tk->start   = start - ts->text;
-    tk->end     = end   - ts->text;
-    tk->pos_inc = 1;
-    return &(CTS(ts)->token);
-}
+    do {
+        t += cp_len;
+        cp = get_cp(t, end, &cp_len, enc);
+    } while (cp_len > 0 && rb_enc_isalpha(cp, enc));
 
-static FrtTokenStream *std_ts_clone_i(FrtTokenStream *orig_ts)
-{
-    return frt_ts_clone_size(orig_ts, sizeof(FrtStandardTokenizer));
+    ts->t = t;
+    return frt_tk_set_ts(&(ts->token), start, t, ts->text, 1, enc);
 }
 
-static FrtTokenStream *std_ts_new()
-{
-    FrtTokenStream *ts = frt_ts_new(FrtStandardTokenizer);
-
-    ts->clone_i     = &std_ts_clone_i;
-    ts->next        = &std_next;
-
-    return ts;
+FrtTokenStream *frt_letter_tokenizer_alloc(void) {
+    return frt_cts_alloc();
 }
 
-FrtTokenStream *frt_standard_tokenizer_new()
-{
-    FrtTokenStream *ts = std_ts_new();
-    STDTS(ts)->type = FRT_STT_ASCII;
-    return ts;
-}
-
-FrtTokenStream *frt_mb_standard_tokenizer_new()
-{
-    FrtTokenStream *ts = std_ts_new();
-    STDTS(ts)->type = FRT_STT_MB;
+FrtTokenStream *frt_letter_tokenizer_init(FrtTokenStream *ts) {
+    ts = frt_cts_init(ts);
+    ts->next = &lt_next;
     return ts;
 }
 
-FrtTokenStream *frt_utf8_standard_tokenizer_new()
-{
-    FrtTokenStream *ts = std_ts_new();
-    STDTS(ts)->type = FRT_STT_UTF8;
-    return ts;
+FrtTokenStream *frt_letter_tokenizer_new(void) {
+    FrtTokenStream *ts = frt_letter_tokenizer_alloc();
+    return frt_letter_tokenizer_init(ts);
 }
 
-/****************************************************************************
- *
- * LegacyStandard
- *
- ****************************************************************************/
-
-#define LSTDTS(token_stream) ((FrtLegacyStandardTokenizer *)(token_stream))
+/*****************************************************************************/
+/*** FrtStandardTokenizer ****************************************************/
+/*****************************************************************************/
 
-/*
- * LegacyStandardTokenizer
- */
-static int legacy_std_get_alpha(FrtTokenStream *ts, char *token)
-{
-    int i = 0;
+static int std_get_alnum(FrtTokenStream *ts, char *token, OnigCodePoint cp, int *cp_len_p, OnigCodePoint *cp_out_p, rb_encoding *enc) {
+    char *end = ts->text + ts->length;
     char *t = ts->t;
-    while (t[i] != '\0' && isalnum(t[i])) {
-        if (i < FRT_MAX_WORD_SIZE) {
-            token[i] = t[i];
-        }
-        i++;
-    }
-    return i;
-}
-
-static int mb_legacy_std_get_alpha(FrtTokenStream *ts, char *token)
-{
-    char *t = ts->t;
-    wchar_t wchr;
-    int i;
-    mbstate_t state; FRT_ZEROSET(&state, mbstate_t);
+    char *tt = ts->t;
+    int cp_len = *cp_len_p;
 
-    i = mb_next_char(&wchr, t, &state);
-
-    while (wchr != 0 && iswalnum(wchr)) {
-        t += i;
-        i = mb_next_char(&wchr, t, &state);
-    }
-
-    i = (int)(t - ts->t);
-    if (i > FRT_MAX_WORD_SIZE) {
-        i = FRT_MAX_WORD_SIZE - 1;
+    while (cp > 0 && rb_enc_isalnum(cp, enc)) {
+        if ((t - ts->t + cp_len) < FRT_MAX_WORD_SIZE)
+            tt += cp_len;
+        t += cp_len;
+        cp = get_cp(t, end, &cp_len, enc);
     }
-    memcpy(token, ts->t, i);
-    return i;
-}
-
-static int isnumpunc(char c)
-{
-    return (c == '.' || c == ',' || c == '\\' || c == '/' || c == '_'
-            || c == '-');
-}
-
-static int w_isnumpunc(wchar_t c)
-{
-    return (c == L'.' || c == L',' || c == L'\\' || c == L'/' || c == L'_'
-            || c == L'-');
-}
-
-static int isurlpunc(char c)
-{
-    return (c == '.' || c == '/' || c == '-' || c == '_');
-}
-
-static int isurlc(char c)
-{
-    return (c == '.' || c == '/' || c == '-' || c == '_' || isalnum(c));
-}
 
-static int isurlxatpunc(char c)
-{
-    return (c == '.' || c == '/' || c == '-' || c == '_' || c == '@');
-}
-
-static int isurlxatc(char c)
-{
-    return (c == '.' || c == '/' || c == '-' || c == '_' || c == '@'
-            || isalnum(c));
-}
+    memcpy(token, ts->t, tt - ts->t);
+    token[tt - ts->t] = '\0';
 
-static bool legacy_std_is_tok_char(char *c)
-{
-    if (isspace(*c)) {
-        return false;           /* most common so check first. */
-    }
-    if (isalnum(*c) || isnumpunc(*c) || *c == '&' ||
-        *c == '@' || *c == '\'' || *c == ':') {
-        return true;
-    }
-    return false;
-}
-
-static bool mb_legacy_std_is_tok_char(char *t)
-{
-    wchar_t c;
-    mbstate_t state; FRT_ZEROSET(&state, mbstate_t);
-
-    if (((int)mbrtowc(&c, t, MB_CUR_MAX, &state)) < 0) {
-        /* error which we can handle next time round. For now just return
-         * false so that we can return a token */
-        return false;
-    }
-    if (iswspace(c)) {
-        return false;           /* most common so check first. */
-    }
-    if (iswalnum(c) || w_isnumpunc(c) || c == L'&' || c == L'@' || c == L'\''
-        || c == L':') {
-        return true;
-    }
-    return false;
+    *cp_out_p = cp;
+    *cp_len_p = cp_len;
+    return t - ts->t;
 }
 
 /* (alnum)((punc)(alnum))+ where every second sequence of alnum must contain at
@@ -743,242 +348,261 @@ static bool mb_legacy_std_is_tok_char(char *t)
  * (alnum) = [a-zA-Z0-9]
  * (punc) = [_\/.,-]
  */
-static int legacy_std_get_number(char *input)
-{
-    int i = 0;
-    int count = 0;
+static int std_get_number(FrtTokenStream *ts, char *start, char *end, OnigCodePoint cp, int cp_len_a, rb_encoding *enc) {
+
+    OnigCodePoint cp_1 = 0;
+    char *t = start;
+    int cp_len = cp_len_a;
+    int cp_1_len = 0;
     int last_seen_digit = 2;
     int seen_digit = false;
 
-    while (last_seen_digit >= 0) {
-        while ((input[i] != '\0') && isalnum(input[i])) {
-            if ((last_seen_digit < 2) && isdigit(input[i])) {
+    while (cp > 0 && last_seen_digit >= 0) {
+        while ((cp > 0) && rb_enc_isalnum(cp, enc)) {
+            if ((last_seen_digit < 2) && rb_enc_isdigit(cp, enc)) {
                 last_seen_digit = 2;
             }
-            if ((seen_digit == false) && isdigit(input[i])) {
+            if ((seen_digit == false) && rb_enc_isdigit(cp, enc)) {
                 seen_digit = true;
             }
-            i++;
+            t += cp_len;
+            cp = get_cp(t, end, &cp_len, enc);
         }
         last_seen_digit--;
-        if (!isnumpunc(input[i]) || !isalnum(input[i + 1])) {
-
-            if (last_seen_digit >= 0) {
-                count = i;
-            }
+        cp_1 = get_cp(t + cp_len, end, &cp_1_len, enc);
+        if (!cp_isnumpunc(cp) || !rb_enc_isalnum(cp_1, enc)) {
             break;
         }
-        count = i;
-        i++;
+        t += cp_len;
+        cp = cp_1;
+        cp_len = cp_1_len;
     }
     if (seen_digit) {
-        return count;
-    }
-    else {
+        return t - start;
+    } else {
         return 0;
     }
 }
 
-static int legacy_std_get_apostrophe(char *input)
-{
+static int std_get_apostrophe(FrtTokenStream *ts, char *input, OnigCodePoint cp, int *cp_len_p, rb_encoding *enc) {
+    int cp_len = *cp_len_p;
+    char *end = ts->text + ts->length;
     char *t = input;
 
-    while (isalpha(*t) || *t == '\'') {
-        t++;
+    while (cp_len > 0 && (rb_enc_isalpha(cp, enc) || cp == cp_apostrophe)) {
+        t += cp_len;
+        cp = get_cp(t, end, &cp_len, enc);
     }
-
     return (int)(t - input);
 }
 
-static int mb_legacy_std_get_apostrophe(char *input)
-{
-    char *t = input;
-    wchar_t wchr;
-    int i;
-    mbstate_t state; FRT_ZEROSET(&state, mbstate_t);
-
-    i = mb_next_char(&wchr, t, &state);
+static char *std_get_url(FrtTokenStream *ts, char *start, char *end, char *token, int *len, int bufred) {
+    rb_encoding *enc = ts->encoding;
+    OnigCodePoint cp;
+    OnigCodePoint prev_cp = 0;
+    int cp_len = 0;
+    int prev_cp_len = 0;
+    char *t = start;
+    char *tt = start;
 
-    while (iswalpha(wchr) || wchr == L'\'') {
-        t += i;
-        i = mb_next_char(&wchr, t, &state);
-    }
-    return (int)(t - input);
-}
-
-static char *std_get_url(char *input, char *token, int i, int *len)
-{
-    char *next = NULL;
-    while (isurlc(input[i])) {
-        if (isurlpunc(input[i]) && isurlpunc(input[i - 1])) {
+    cp = get_cp(t, end, &cp_len, enc);
+    while (cp > 0 && cp_enc_isurlc(cp, enc)) {
+        if (cp_isurlpunc(cp) && cp_isurlpunc(prev_cp)) {
             break; /* can't have two puncs in a row */
         }
-        if (i < FRT_MAX_WORD_SIZE) {
-            token[i] = input[i];
-        }
-        i++;
+        prev_cp = cp;
+        prev_cp_len = cp_len;
+        t += cp_len;
+        if (((t + cp_len) - start) <= (FRT_MAX_WORD_SIZE - bufred))
+            tt += cp_len;
+        cp = get_cp(t, end, &cp_len, enc);
     }
-    next = input + i;
 
-    /* We don't want to index past the end of the token capacity) */
-    if (i >= FRT_MAX_WORD_SIZE) {
-        i = FRT_MAX_WORD_SIZE - 1;
+    /* strip trailing punc */
+    if (t == tt && cp_isurlpunc(prev_cp)) {
+        tt -= prev_cp_len;
     }
 
-    /* strip trailing puncs */
-    while (isurlpunc(input[i - 1])) {
-        i--;
-    }
-    *len = i;
-    token[i] = '\0';
+    *len = (tt - start) + bufred;
+    memcpy(token, start, tt - start);
+    token[tt - start] = '\0';
 
-    return next;
+    return t;
 }
 
-/* Company names can contain '@' and '&' like AT&T and Excite@Home. Let's
-*/
-static int legacy_std_get_company_name(char *input)
-{
-    int i = 0;
-    while (isalpha(input[i]) || input[i] == '@' || input[i] == '&') {
-        i++;
-    }
+/* Company names can contain '@' and '&' like AT&T and Excite@Home. */
+static int std_get_company_name(FrtTokenStream *ts, char *start, char* end) {
+    rb_encoding *enc = ts->encoding;
+    char * t = start;
+    OnigCodePoint cp;
+    int cp_len = 0;
 
-    return i;
-}
-
-static bool legacy_std_advance_to_start(FrtTokenStream *ts)
-{
-    char *t = ts->t;
-    while (*t != '\0' && !isalnum(*t)) {
-        if (isnumpunc(*t) && isdigit(t[1])) break;
-        t++;
+    cp = get_cp(t, end, &cp_len, enc);
+    while (cp > 0 && (rb_enc_isalpha(cp, enc) || cp == cp_at || cp == cp_ampersand)) {
+        t += cp_len;
+        cp = get_cp(t, end, &cp_len, enc);
     }
 
-    ts->t = t;
-
-    return (*t != '\0');
+    return t - start;
 }
 
-static bool mb_legacy_std_advance_to_start(FrtTokenStream *ts)
-{
-    int i;
-    wchar_t wchr;
-    mbstate_t state; FRT_ZEROSET(&state, mbstate_t);
-
-    i = mb_next_char(&wchr, ts->t, &state);
+static int std_advance_to_start(FrtTokenStream *ts, int *cp_len_p, OnigCodePoint *cp_out_p, rb_encoding *enc) {
+    int cp_len = 0;
+    int cp_next = 0;
+    int cp_len_next = 0;
+    OnigCodePoint cp;
+    char *end = ts->text + ts->length;
+    char *t = ts->t;
 
-    while (wchr != 0 && !iswalnum(wchr)) {
-        if (isnumpunc(*ts->t) && isdigit(ts->t[1])) break;
-        ts->t += i;
-        i = mb_next_char(&wchr, ts->t, &state);
+    cp = get_cp(t, end, &cp_len, enc);
+    while (cp > 0 && !rb_enc_isalnum(cp, enc)) {
+        if (cp_isnumpunc(cp)) {
+            cp_next = get_cp(t + cp_len, end, &cp_len_next, enc);
+            if (cp_next > 0 && rb_enc_isdigit(cp_next, enc))
+                break;
+        }
+        t += cp_len;
+        cp = get_cp(t, end, &cp_len, enc);
     }
-
-    return (wchr != 0);
+    ts->t = t;
+    *cp_out_p = cp;
+    *cp_len_p = cp_len;
+    return (t < end);
 }
 
-static FrtToken *legacy_std_next(FrtTokenStream *ts)
-{
-    FrtLegacyStandardTokenizer *std_tz = LSTDTS(ts);
+static FrtToken *std_next(FrtTokenStream *ts) {
     char *s;
     char *t;
     char *start = NULL;
+    char *end;
     char *num_end = NULL;
     char token[FRT_MAX_WORD_SIZE + 1];
+    OnigCodePoint cp = 0;
+    OnigCodePoint cp_1 = 0;
+    OnigCodePoint cp_2 = 0;
+    OnigCodePoint prev_cp = 0;
+    int cp_len = 0;
+    int cp_1_len = 0;
+    int cp_2_len = 0;
     int token_i = 0;
     int len;
     bool is_acronym;
     bool seen_at_symbol;
+    rb_encoding *enc = ts->encoding;
 
-
-    if (!std_tz->advance_to_start(ts)) {
+    /* advance to start and return first cp and len */
+    if (!std_advance_to_start(ts, &cp_len, &cp, enc))
         return NULL;
-    }
 
+    end = ts->text + ts->length;
     start = t = ts->t;
-    token_i = std_tz->get_alpha(ts, token);
+
+    /* get all alnums */
+    token_i = std_get_alnum(ts, token, cp, &cp_len, &cp, enc);
     t += token_i;
 
-    if (!std_tz->is_tok_char(t)) {
+    if (t >= end && token_i > 0) {
+        ts->t += token_i;
+        return frt_tk_set_ts(&(ts->token), start, t, ts->text, 1, enc);
+    }
+
+    // already got cp and cp_len from get_alnum above
+    // cp = get_cp(t, end, &cp_len, enc);
+    if (cp < 1)
+        return NULL;
+
+    if (!cp_enc_istok(cp, enc)) {
         /* very common case, ie a plain word, so check and return */
-        ts->t = t;
-        return frt_tk_set_ts(&(CTS(ts)->token), start, t, ts->text, 1);
+        ts->t = t + cp_len;
+        return frt_tk_set_ts(&(ts->token), start, t, ts->text, 1, enc);
     }
 
-    if (*t == '\'') {       /* apostrophe case. */
-        t += std_tz->get_apostrophe(t);
+    if (cp == cp_apostrophe) {       /* apostrophe case. */
+        t += std_get_apostrophe(ts, t, cp, &cp_len, enc);
         ts->t = t;
         len = (int)(t - start);
         /* strip possesive */
+        /* TODO: wont work with multibyte */
         if ((t[-1] == 's' || t[-1] == 'S') && t[-2] == '\'') {
             t -= 2;
-            frt_tk_set_ts(&(CTS(ts)->token), start, t, ts->text, 1);
-            CTS(ts)->token.end += 2;
+            frt_tk_set_ts(&(ts->token), start, t, ts->text, 1, enc);
+            ts->token.end += 2;
         }
         else if (t[-1] == '\'') {
             t -= 1;
-            frt_tk_set_ts(&(CTS(ts)->token), start, t, ts->text, 1);
-            CTS(ts)->token.end += 1;
+            frt_tk_set_ts(&(ts->token), start, t, ts->text, 1, enc);
+            ts->token.end += 1;
         }
         else {
-            frt_tk_set_ts(&(CTS(ts)->token), start, t, ts->text, 1);
+            frt_tk_set_ts(&(ts->token), start, t, ts->text, 1, enc);
         }
-
-        return &(CTS(ts)->token);
+        return &(ts->token);
     }
 
-    if (*t == '&') {        /* apostrophe case. */
-        t += legacy_std_get_company_name(t);
+    // already got cp and cp_len from get_alnum above
+    // cp = get_cp(t, end, &cp_len, enc);
+    if (cp == cp_ampersand) {        /* ampersand case. */
+        t += std_get_company_name(ts, t, end);
         ts->t = t;
-        return frt_tk_set_ts(&(CTS(ts)->token), start, t, ts->text, 1);
+        return frt_tk_set_ts(&(ts->token), start, t, ts->text, 1, enc);
     }
 
-    if ((isdigit(*start) || isnumpunc(*start))       /* possibly a number */
-        && ((len = legacy_std_get_number(start)) > 0)) {
+    // already got cp and cp_len from get_alnum above
+    // cp = get_cp(start, end, &cp_len, enc);
+    if ((rb_enc_isdigit(cp, enc) || cp_isnumpunc(cp))
+        && ((len = std_get_number(ts, start, end, cp, cp_len, enc)) > 0)) { /* possibly a number */
         num_end = start + len;
-        if (!std_tz->is_tok_char(num_end)) { /* won't find a longer token */
+        cp = get_cp(num_end, end, &cp_len, enc);
+        if (cp > 0 && !cp_enc_istok(cp, enc)) { /* won't find a longer token */
             ts->t = num_end;
-            return frt_tk_set_ts(&(CTS(ts)->token), start, num_end, ts->text, 1);
+            return frt_tk_set_ts(&(ts->token), start, num_end, ts->text, 1, enc);
         }
         /* else there may be a longer token so check */
     }
 
-    if (t[0] == ':' && t[1] == '/' && t[2] == '/') {
+    // already got cp and cp_len from get_alnum or the last block above
+    // cp = get_cp(t, end, &cp_len, enc);
+    cp_1 = get_cp(t + cp_len, end, &cp_1_len, enc);
+    cp_2 = get_cp(t + cp_len + cp_1_len, end, &cp_2_len, enc);
+    if (cp == cp_colon && cp_1 == cp_slash && cp_2 == cp_slash) {
         /* check for a known url start */
         token[token_i] = '\0';
-        t += 3;
-        token_i += 3;
-        while (*t == '/') {
-            t++;
-        }
-        if (isalpha(*t) &&
-            (memcmp(token, "ftp", 3) == 0 ||
-             memcmp(token, "http", 4) == 0 ||
-             memcmp(token, "https", 5) == 0 ||
-             memcmp(token, "file", 4) == 0)) {
-            ts->t = std_get_url(t, token, 0, &len); /* dispose of first part of the URL */
+        t += cp_len + cp_1_len + cp_2_len;
+        token_i += cp_len + cp_1_len + cp_2_len;
+        cp = get_cp(t, end, &cp_len, enc);
+        while (cp > 0 && cp == cp_slash) {
+            t += cp_len;
+            cp = get_cp(t, end, &cp_len, enc);
         }
-        else {              /* still treat as url but keep the first part */
+        if (rb_enc_isalpha(cp, enc) &&
+               (memcmp(token, "ftp", 3) == 0 ||
+                memcmp(token, "http", 4) == 0 ||
+                memcmp(token, "https", 5) == 0 ||
+                memcmp(token, "file", 4) == 0)) {
+            ts->t = std_get_url(ts, t, end, token, &len, 0); /* dispose of first part of the URL */
+        } else {              /* still treat as url but keep the first part */
             token_i = (int)(t - start);
             memcpy(token, start, token_i * sizeof(char));
-            ts->t = std_get_url(start, token, token_i, &len); /* keep start */
+            ts->t = std_get_url(ts, t, end, token + token_i, &len, token_i); /* keep start */
         }
-        return frt_tk_set(&(CTS(ts)->token), token, len,
+        return frt_tk_set(&(ts->token), token, len,
                       (off_t)(start - ts->text),
-                      (off_t)(ts->t - ts->text), 1);
+                      (off_t)(ts->t - ts->text), 1, enc);
     }
 
     /* now see how long a url we can find. */
     is_acronym = true;
     seen_at_symbol = false;
-    while (isurlxatc(*t)) {
-        if (is_acronym && !isalpha(*t) && (*t != '.')) {
+
+    cp = get_cp(t, end, &cp_len, enc);
+    while (cp_enc_isurlxatc(cp, enc)) {
+        if (is_acronym && !rb_enc_isalpha(cp, enc) && (cp != cp_dot)) {
             is_acronym = false;
         }
-        if (isurlxatpunc(*t) && isurlxatpunc(t[-1])) {
+        if (cp_isurlxatpunc(cp) && cp_isurlxatpunc(prev_cp)) {
             break; /* can't have two punctuation characters in a row */
         }
-        if (*t == '@') {
+        if (cp == cp_at) {
             if (seen_at_symbol) {
                 break; /* we can only have one @ symbol */
             }
@@ -986,10 +610,12 @@ static FrtToken *legacy_std_next(FrtTokenStream *ts)
                 seen_at_symbol = true;
             }
         }
-        t++;
+        prev_cp = cp;
+        t += cp_len;
+        cp = get_cp(t, end, &cp_len, enc);
     }
-    while (isurlxatpunc(t[-1]) && t > ts->t) {
-        t--;                /* strip trailing punctuation */
+    if (cp_isurlxatpunc(prev_cp) && t > ts->t) {
+        t -= cp_len;                /* strip trailing punctuation */
     }
 
     if (t < ts->t || (num_end != NULL && num_end < ts->t)) {
@@ -999,140 +625,119 @@ static FrtToken *legacy_std_next(FrtTokenStream *ts)
         ts->t = t;
 
         if (is_acronym) {   /* check it is one letter followed by one '.' */
-            for (s = start; s < t - 1; s++) {
-                if (isalpha(*s) && (s[1] != '.'))
+            cp_len = 0;
+            for (s = start; s < t - 1; s += cp_len) {
+                cp = get_cp(s, end, &cp_len, enc);
+                cp_1 = get_cp(s + cp_len, end, &cp_1_len, enc);
+                if (rb_enc_isalpha(cp, enc) && (cp_1 != cp_dot))
                     is_acronym = false;
             }
         }
         if (is_acronym) {   /* strip '.'s */
-            for (s = start + token_i; s < t; s++) {
-                if (*s != '.') {
-                    token[token_i] = *s;
-                    token_i++;
+            cp_len = 0;
+            for (s = start + token_i; s < t; s += cp_len) {
+                cp = get_cp(s, end, &cp_len, enc);
+                if (cp > 0 && cp != cp_dot) {
+                    memcpy(token + token_i, s, cp_len);
+                    token_i += cp_len;
                 }
             }
-            frt_tk_set(&(CTS(ts)->token), token, token_i,
+            token[token_i] = '\0';
+            frt_tk_set(&(ts->token), token, token_i,
                    (off_t)(start - ts->text),
-                   (off_t)(t - ts->text), 1);
-        }
-        else { /* just return the url as is */
-            frt_tk_set_ts(&(CTS(ts)->token), start, t, ts->text, 1);
+                   (off_t)(t - ts->text), 1, enc);
+        } else { /* just return the url as is */
+            frt_tk_set_ts(&(ts->token), start, t, ts->text, 1, enc);
         }
-    }
-    else {                  /* return the number */
+    } else {                  /* return the number */
         ts->t = num_end;
-        frt_tk_set_ts(&(CTS(ts)->token), start, num_end, ts->text, 1);
+        frt_tk_set_ts(&(ts->token), start, num_end, ts->text, 1, enc);
     }
-
-    return &(CTS(ts)->token);
+    return &(ts->token);
 }
 
-static FrtTokenStream *legacy_std_ts_clone_i(FrtTokenStream *orig_ts)
-{
-    return frt_ts_clone_size(orig_ts, sizeof(FrtLegacyStandardTokenizer));
+static FrtTokenStream *std_ts_clone_i(FrtTokenStream *orig_ts) {
+    return frt_ts_clone_size(orig_ts, sizeof(FrtTokenStream));
 }
 
-static FrtTokenStream *legacy_std_ts_new()
-{
-    FrtTokenStream *ts = frt_ts_new(FrtLegacyStandardTokenizer);
-
-    ts->clone_i     = &legacy_std_ts_clone_i;
-    ts->next        = &legacy_std_next;
-
-    return ts;
+FrtTokenStream *frt_standard_tokenizer_alloc(void) {
+    return (FrtTokenStream *)frt_ecalloc(sizeof(FrtTokenStream));
 }
 
-FrtTokenStream *frt_legacy_standard_tokenizer_new()
-{
-    FrtTokenStream *ts = legacy_std_ts_new();
-
-    LSTDTS(ts)->advance_to_start = &legacy_std_advance_to_start;
-    LSTDTS(ts)->get_alpha        = &legacy_std_get_alpha;
-    LSTDTS(ts)->is_tok_char      = &legacy_std_is_tok_char;
-    LSTDTS(ts)->get_apostrophe   = &legacy_std_get_apostrophe;
-
+FrtTokenStream *frt_standard_tokenizer_init(FrtTokenStream *ts) {
+    ts = frt_ts_init(ts);
+    ts->clone_i = &std_ts_clone_i;
+    ts->next    = &std_next;
     return ts;
 }
 
-FrtTokenStream *frt_mb_legacy_standard_tokenizer_new()
-{
-    FrtTokenStream *ts = legacy_std_ts_new();
-
-    LSTDTS(ts)->advance_to_start = &mb_legacy_std_advance_to_start;
-    LSTDTS(ts)->get_alpha        = &mb_legacy_std_get_alpha;
-    LSTDTS(ts)->is_tok_char      = &mb_legacy_std_is_tok_char;
-    LSTDTS(ts)->get_apostrophe   = &mb_legacy_std_get_apostrophe;
-
-    return ts;
+FrtTokenStream *frt_standard_tokenizer_new(void) {
+    FrtTokenStream *ts = frt_standard_tokenizer_alloc();
+    return frt_standard_tokenizer_init(ts);
 }
 
-/****************************************************************************
- *
- * Filters
- *
- ****************************************************************************/
+/*****************************************************************************/
+/*** FrtFilters **************************************************************/
+/*****************************************************************************/
 
 #define TkFilt(filter) ((FrtTokenFilter *)(filter))
 
-FrtTokenStream *frt_filter_clone_size(FrtTokenStream *ts, size_t size)
-{
+FrtTokenStream *frt_filter_clone_size(FrtTokenStream *ts, size_t size) {
     FrtTokenStream *ts_new = frt_ts_clone_size(ts, size);
     TkFilt(ts_new)->sub_ts = TkFilt(ts)->sub_ts->clone_i(TkFilt(ts)->sub_ts);
     return ts_new;
 }
 
-static FrtTokenStream *filter_clone_i(FrtTokenStream *ts)
-{
+static FrtTokenStream *filter_clone_i(FrtTokenStream *ts) {
     return frt_filter_clone_size(ts, sizeof(FrtTokenFilter));
 }
 
-static FrtTokenStream *filter_reset(FrtTokenStream *ts, char *text)
-{
-    TkFilt(ts)->sub_ts->reset(TkFilt(ts)->sub_ts, text);
+static FrtTokenStream *filter_reset(FrtTokenStream *ts, char *text, rb_encoding *encoding) {
+    TkFilt(ts)->sub_ts->reset(TkFilt(ts)->sub_ts, text, encoding);
     return ts;
 }
 
-static void filter_destroy_i(FrtTokenStream *ts)
-{
+static void filter_destroy_i(FrtTokenStream *ts) {
     frt_ts_deref(TkFilt(ts)->sub_ts);
     free(ts);
 }
 
-FrtTokenStream *frt_tf_new_i(size_t size, FrtTokenStream *sub_ts)
-{
-    FrtTokenStream *ts     = (FrtTokenStream *)frt_ecalloc(size);
-
-    TkFilt(ts)->sub_ts  = sub_ts;
-
-    ts->clone_i         = &filter_clone_i;
-    ts->destroy_i       = &filter_destroy_i;
-    ts->reset           = &filter_reset;
-    ts->ref_cnt         = 1;
+FrtTokenStream *frt_tf_alloc_i(size_t size) {
+    return (FrtTokenStream *)frt_ecalloc(size);
+}
 
+FrtTokenStream *frt_tf_init(FrtTokenStream *ts, FrtTokenStream *sub_ts) {
+    ts->clone_i        = &filter_clone_i;
+    ts->destroy_i      = &filter_destroy_i;
+    ts->reset          = &filter_reset;
+    ts->ref_cnt        = 1;
+    TkFilt(ts)->sub_ts = sub_ts;
     return ts;
 }
 
-/****************************************************************************
- * FrtStopFilter
- ****************************************************************************/
+FrtTokenStream *frt_tf_new_i(size_t size, FrtTokenStream *sub_ts) {
+    FrtTokenStream *ts = frt_tf_alloc_i(size);
+    return frt_tf_init(ts, sub_ts);
+}
+
+/*****************************************************************************/
+/**** FrtStopFilter **********************************************************/
+/*****************************************************************************/
 
 #define StopFilt(filter) ((FrtStopFilter *)(filter))
 
-static void sf_destroy_i(FrtTokenStream *ts)
-{
+static void sf_destroy_i(FrtTokenStream *ts) {
     frt_h_destroy(StopFilt(ts)->words);
     filter_destroy_i(ts);
 }
 
-static FrtTokenStream *sf_clone_i(FrtTokenStream *orig_ts)
-{
+static FrtTokenStream *sf_clone_i(FrtTokenStream *orig_ts) {
     FrtTokenStream *new_ts = frt_filter_clone_size(orig_ts, sizeof(FrtMappingFilter));
     FRT_REF(StopFilt(new_ts)->words);
     return new_ts;
 }
 
-static FrtToken *sf_next(FrtTokenStream *ts)
-{
+static FrtToken *sf_next(FrtTokenStream *ts) {
     int pos_inc = 0;
     FrtHash *words = StopFilt(ts)->words;
     FrtTokenFilter *tf = TkFilt(ts);
@@ -1150,71 +755,76 @@ static FrtToken *sf_next(FrtTokenStream *ts)
     return tk;
 }
 
-FrtTokenStream *frt_stop_filter_new_with_words_len(FrtTokenStream *sub_ts,
-                                            const char **words, int len)
-{
+FrtTokenStream *frt_stop_filter_alloc(void) {
+    return (FrtTokenStream *)frt_ecalloc(sizeof(FrtStopFilter));
+}
+
+FrtTokenStream *frt_stop_filter_init(FrtTokenStream *ts, FrtTokenStream *sub_ts) {
+    frt_tf_init(ts, sub_ts);
+    ts->next      = &sf_next;
+    ts->destroy_i = &sf_destroy_i;
+    ts->clone_i   = &sf_clone_i;
+    return ts;
+}
+
+void frt_stop_filter_set_words_len(FrtTokenStream *ts, const char **words, int len) {
     int i;
     char *word;
     FrtHash *word_table = frt_h_new_str(&free, (frt_free_ft) NULL);
-    FrtTokenStream *ts = tf_new(FrtStopFilter, sub_ts);
-
     for (i = 0; i < len; i++) {
         word = frt_estrdup(words[i]);
         frt_h_set(word_table, word, word);
     }
     StopFilt(ts)->words = word_table;
-    ts->next            = &sf_next;
-    ts->destroy_i       = &sf_destroy_i;
-    ts->clone_i         = &sf_clone_i;
+}
+
+FrtTokenStream *frt_stop_filter_new_with_words_len(FrtTokenStream *sub_ts, const char **words, int len) {
+    FrtTokenStream *ts = frt_stop_filter_alloc();
+    ts = frt_stop_filter_init(ts, sub_ts);
+    frt_stop_filter_set_words_len(ts, words, len);
     return ts;
 }
 
-FrtTokenStream *frt_stop_filter_new_with_words(FrtTokenStream *sub_ts,
-                                        const char **words)
-{
+void frt_stop_filter_set_words(FrtTokenStream *ts, const char **words) {
     char *word;
     FrtHash *word_table = frt_h_new_str(&free, (frt_free_ft) NULL);
-    FrtTokenStream *ts = tf_new(FrtStopFilter, sub_ts);
-
     while (*words) {
         word = frt_estrdup(*words);
         frt_h_set(word_table, word, word);
         words++;
     }
-
     StopFilt(ts)->words = word_table;
-    ts->next            = &sf_next;
-    ts->destroy_i       = &sf_destroy_i;
-    ts->clone_i         = &sf_clone_i;
+}
+
+FrtTokenStream *frt_stop_filter_new_with_words(FrtTokenStream *sub_ts, const char **words) {
+    FrtTokenStream *ts = frt_stop_filter_alloc();
+    frt_stop_filter_init(ts, sub_ts);
+    frt_stop_filter_set_words(ts, words);
     return ts;
 }
 
-FrtTokenStream *frt_stop_filter_new(FrtTokenStream *ts)
-{
-    return frt_stop_filter_new_with_words(ts, FRT_FULL_ENGLISH_STOP_WORDS);
+FrtTokenStream *frt_stop_filter_new(FrtTokenStream *sub_ts) {
+    return frt_stop_filter_new_with_words(sub_ts, FRT_FULL_ENGLISH_STOP_WORDS);
 }
 
-/****************************************************************************
- * MappingFilter
- ****************************************************************************/
+/*****************************************************************************/
+/*** MappingFilter ***********************************************************/
+/*****************************************************************************/
 
 #define MFilt(filter) ((FrtMappingFilter *)(filter))
 
-static void mf_destroy_i(FrtTokenStream *ts)
-{
+static void mf_destroy_i(FrtTokenStream *ts) {
     frt_mulmap_destroy(MFilt(ts)->mapper);
     filter_destroy_i(ts);
 }
 
-static FrtTokenStream *mf_clone_i(FrtTokenStream *orig_ts)
-{
+static FrtTokenStream *mf_clone_i(FrtTokenStream *orig_ts) {
     FrtTokenStream *new_ts = frt_filter_clone_size(orig_ts, sizeof(FrtMappingFilter));
     FRT_REF(MFilt(new_ts)->mapper);
     return new_ts;
 }
 
-static FrtToken *mf_next(FrtTokenStream *ts)
-{
+static FrtToken *mf_next(FrtTokenStream *ts) {
     char buf[FRT_MAX_WORD_SIZE + 1];
     FrtMultiMapper *mapper = MFilt(ts)->mapper;
     FrtTokenFilter *tf = TkFilt(ts);
@@ -1226,48 +836,53 @@ static FrtToken *mf_next(FrtTokenStream *ts)
     return tk;
 }
 
-static FrtTokenStream *mf_reset(FrtTokenStream *ts, char *text)
-{
+static FrtTokenStream *mf_reset(FrtTokenStream *ts, char *text, rb_encoding *encoding) {
     FrtMultiMapper *mm = MFilt(ts)->mapper;
-    if (mm->d_size == 0) {
+    if (mm->d_size == 0)
         frt_mulmap_compile(MFilt(ts)->mapper);
-    }
-    filter_reset(ts, text);
+    filter_reset(ts, text, encoding);
     return ts;
 }
 
-FrtTokenStream *frt_mapping_filter_new(FrtTokenStream *sub_ts)
-{
-    FrtTokenStream *ts   = tf_new(FrtMappingFilter, sub_ts);
-    MFilt(ts)->mapper = frt_mulmap_new();
-    ts->next          = &mf_next;
-    ts->destroy_i     = &mf_destroy_i;
-    ts->clone_i       = &mf_clone_i;
-    ts->reset         = &mf_reset;
+FrtTokenStream *frt_mapping_filter_alloc(void) {
+    return (FrtTokenStream *)frt_ecalloc(sizeof(FrtMappingFilter));
+}
+
+void frt_mapping_filter_init(FrtTokenStream *ts, FrtTokenStream *sub_ts) {
+    frt_tf_init(ts, sub_ts);
+    ts->next           = &mf_next;
+    ts->destroy_i      = &mf_destroy_i;
+    ts->clone_i        = &mf_clone_i;
+    ts->reset          = &mf_reset;
+    MFilt(ts)->mapper  = frt_mulmap_new();
+}
+
+FrtTokenStream *frt_mapping_filter_new(FrtTokenStream *sub_ts) {
+    FrtTokenStream *ts = frt_mapping_filter_alloc();
+    frt_mapping_filter_init(ts, sub_ts);
     return ts;
 }
 
-FrtTokenStream *frt_mapping_filter_add(FrtTokenStream *ts, const char *pattern,
-                                const char *replacement)
-{
+FrtTokenStream *frt_mapping_filter_add(FrtTokenStream *ts, const char *pattern, const char *replacement) {
     frt_mulmap_add_mapping(MFilt(ts)->mapper, pattern, replacement);
     return ts;
 }
 
-/****************************************************************************
- * HyphenFilter
- ****************************************************************************/
+/*****************************************************************************/
+/*** FrtHyphenFilter *********************************************************/
+/*****************************************************************************/
 
 #define HyphenFilt(filter) ((FrtHyphenFilter *)(filter))
 
-static FrtTokenStream *hf_clone_i(FrtTokenStream *orig_ts)
-{
+static FrtTokenStream *hf_clone_i(FrtTokenStream *orig_ts) {
     FrtTokenStream *new_ts = frt_filter_clone_size(orig_ts, sizeof(FrtHyphenFilter));
     return new_ts;
 }
 
-static FrtToken *hf_next(FrtTokenStream *ts)
-{
+static FrtToken *hf_next(FrtTokenStream *ts) {
+    int cp_len = 0;
+    OnigCodePoint cp;
+    rb_encoding *enc = utf8_encoding;
     FrtHyphenFilter *hf = HyphenFilt(ts);
     FrtTokenFilter *tf = TkFilt(ts);
     FrtToken *tk = hf->tk;
@@ -1282,38 +897,48 @@ static FrtToken *hf_next(FrtTokenStream *ts)
         hf->pos += text_len + 1;
         tk->len = text_len;
         return tk;
-    }
-    else {
-        char *p;
+    } else {
+        char *t;
+        char *end;
+
         bool seen_hyphen = false;
         bool seen_other_punc = false;
         hf->tk = tk = tf->sub_ts->next(tf->sub_ts);
         if (NULL == tk) return NULL;
-        p = tk->text + 1;
-        while (*p) {
-            if (*p == '-') {
+        t = tk->text;
+        end = tk->text + tk->len;
+        get_cp(t, end, &cp_len, enc);
+        t += cp_len; // skip first
+        cp = get_cp(t, end, &cp_len, enc);
+        while (cp > 0) {
+            if (cp == cp_dash || cp == cp_hyphen) {
                 seen_hyphen = true;
-            }
-            else if (!isalpha(*p)) {
+            } else if (!rb_enc_isalpha(cp, enc)) {
                 seen_other_punc = true;
                 break;
             }
-            p++;
+            t += cp_len;
+            cp = get_cp(t, end, &cp_len, enc);
         }
         if (seen_hyphen && !seen_other_punc) {
             char *q = hf->text;
             char *r = tk->text;
-            p = tk->text;
-            while (*p) {
-                if (*p == '-') {
+            t = tk->text;
+            end = tk->text + tk->len;
+            cp = 0;
+            cp = get_cp(t, end, &cp_len, enc);
+            while (cp > 0) {
+                if (cp == cp_dash || cp == cp_hyphen) {
                     *q = '\0';
+                    q++;
+                } else {
+                    memcpy(q, t, cp_len);
+                    if (r!=t) memcpy(r, t, cp_len);
+                    r += cp_len;
+                    q += cp_len;
                 }
-                else {
-                    *r = *q = *p;
-                    r++;
-                }
-                q++;
-                p++;
+                t += cp_len;
+                cp = get_cp(t, end, &cp_len, enc);
             }
             *r = *q = '\0';
             hf->start = tk->start;
@@ -1325,89 +950,76 @@ static FrtToken *hf_next(FrtTokenStream *ts)
     return tk;
 }
 
-FrtTokenStream *frt_hyphen_filter_new(FrtTokenStream *sub_ts)
-{
-    FrtTokenStream *ts = tf_new(FrtHyphenFilter, sub_ts);
-    ts->next        = &hf_next;
-    ts->clone_i     = &hf_clone_i;
+FrtTokenStream *frt_hyphen_filter_alloc(void) {
+    return (FrtTokenStream *)frt_ecalloc(sizeof(FrtHyphenFilter));
+}
+
+FrtTokenStream *frt_hyphen_filter_init(FrtTokenStream *ts, FrtTokenStream *sub_ts) {
+    frt_tf_init(ts, sub_ts);
+    ts->next    = &hf_next;
+    ts->clone_i = &hf_clone_i;
     return ts;
 }
 
-/****************************************************************************
- * LowerCaseFilter
- ****************************************************************************/
+FrtTokenStream *frt_hyphen_filter_new(FrtTokenStream *sub_ts) {
+    FrtTokenStream *ts = frt_hyphen_filter_alloc();
+    return frt_hyphen_filter_init(ts, sub_ts);
+}
 
+/*****************************************************************************/
+/*** FrtLowercaseFilter ******************************************************/
+/*****************************************************************************/
+
+static FrtToken *lcf_next(FrtTokenStream *ts) {
+    int len = 0;
+    OnigCaseFoldType fold_type = ONIGENC_CASE_DOWNCASE;
+    rb_encoding *enc = utf8_encoding; // Token encoding is always UTF-8
+    char buf[FRT_MAX_WORD_SIZE + 20]; // CASE_MAPPING_ADDITIONAL_LENGTH
+    char *buf_end = buf + FRT_MAX_WORD_SIZE + 19;
 
-static FrtToken *mb_lcf_next(FrtTokenStream *ts)
-{
-    wchar_t wbuf[FRT_MAX_WORD_SIZE + 1], *wchr;
     FrtToken *tk = TkFilt(ts)->sub_ts->next(TkFilt(ts)->sub_ts);
-    int x;
-    wbuf[FRT_MAX_WORD_SIZE] = 0;
+    if (tk == NULL) { return tk; }
+    if (tk->len < 1) { return tk; }
 
-    if (tk == NULL) {
-        return tk;
-    }
+    const OnigUChar *t = (const OnigUChar *)tk->text;
+
+    len = enc->case_map(&fold_type, &t, (const OnigUChar *)(tk->text + tk->len), (OnigUChar *)buf, (OnigUChar *)buf_end, enc);
+    tk->len = len;
+    memcpy(tk->text, buf, len);
+    tk->text[len] = '\0';
 
-    if ((x=mbstowcs(wbuf, tk->text, FRT_MAX_WORD_SIZE)) <= 0) return tk;
-    wchr = wbuf;
-    while (*wchr != 0) {
-        *wchr = towlower(*wchr);
-        wchr++;
-    }
-    tk->len = wcstombs(tk->text, wbuf, FRT_MAX_WORD_SIZE);
-    if (tk->len <= 0) {
-        strcpy(tk->text, "BAD_DATA");
-        tk->len = 8;
-    }
-    tk->text[tk->len] = '\0';
     return tk;
 }
 
-FrtTokenStream *frt_mb_lowercase_filter_new(FrtTokenStream *sub_ts)
-{
-    FrtTokenStream *ts = tf_new(FrtTokenFilter, sub_ts);
-    ts->next = &mb_lcf_next;
-    return ts;
+FrtTokenStream *frt_lowercase_filter_alloc(void) {
+    return (FrtTokenStream *)frt_ecalloc(sizeof(FrtTokenFilter));
 }
 
-static FrtToken *lcf_next(FrtTokenStream *ts)
-{
-    int i = 0;
-    FrtToken *tk = TkFilt(ts)->sub_ts->next(TkFilt(ts)->sub_ts);
-    if (tk == NULL) {
-        return tk;
-    }
-    while (tk->text[i] != '\0') {
-        tk->text[i] = tolower(tk->text[i]);
-        i++;
-    }
-    return tk;
+void frt_lowercase_filter_init(FrtTokenStream *ts, FrtTokenStream *sub_ts) {
+    frt_tf_init(ts, sub_ts);
+    ts->next = &lcf_next;
 }
 
-FrtTokenStream *frt_lowercase_filter_new(FrtTokenStream *sub_ts)
-{
-    FrtTokenStream *ts = tf_new(FrtTokenFilter, sub_ts);
-    ts->next = &lcf_next;
+FrtTokenStream *frt_lowercase_filter_new(FrtTokenStream *sub_ts) {
+    FrtTokenStream *ts = frt_lowercase_filter_alloc();
+    frt_lowercase_filter_init(ts, sub_ts);
     return ts;
 }
 
-/****************************************************************************
- * FrtStemFilter
- ****************************************************************************/
+/*****************************************************************************/
+/*** FrtStemFilter ***********************************************************/
+/*****************************************************************************/
 
 #define StemFilt(filter) ((FrtStemFilter *)(filter))
 
-static void stemf_destroy_i(FrtTokenStream *ts)
-{
+static void stemf_destroy_i(FrtTokenStream *ts) {
     sb_stemmer_delete(StemFilt(ts)->stemmer);
     free(StemFilt(ts)->algorithm);
     free(StemFilt(ts)->charenc);
     filter_destroy_i(ts);
 }
 
-static FrtToken *stemf_next(FrtTokenStream *ts)
-{
+static FrtToken *stemf_next(FrtTokenStream *ts) {
     int len;
     const sb_symbol *stemmed;
     struct sb_stemmer *stemmer = StemFilt(ts)->stemmer;
@@ -1428,26 +1040,27 @@ static FrtToken *stemf_next(FrtTokenStream *ts)
     return tk;
 }
 
-static FrtTokenStream *stemf_clone_i(FrtTokenStream *orig_ts)
-{
-    FrtTokenStream *new_ts      = frt_filter_clone_size(orig_ts, sizeof(FrtStemFilter));
-    FrtStemFilter *stemf        = StemFilt(new_ts);
-    FrtStemFilter *orig_stemf   = StemFilt(orig_ts);
-    stemf->stemmer =
-        sb_stemmer_new(orig_stemf->algorithm, orig_stemf->charenc);
-    stemf->algorithm =
-        orig_stemf->algorithm ? frt_estrdup(orig_stemf->algorithm) : NULL;
-    stemf->charenc =
-        orig_stemf->charenc ? frt_estrdup(orig_stemf->charenc) : NULL;
+static FrtTokenStream *stemf_clone_i(FrtTokenStream *orig_ts) {
+    FrtTokenStream *new_ts    = frt_filter_clone_size(orig_ts, sizeof(FrtStemFilter));
+    FrtStemFilter *stemf      = StemFilt(new_ts);
+    FrtStemFilter *orig_stemf = StemFilt(orig_ts);
+    stemf->stemmer = sb_stemmer_new(orig_stemf->algorithm, orig_stemf->charenc);
+    stemf->algorithm = orig_stemf->algorithm ? frt_estrdup(orig_stemf->algorithm) : NULL;
+    stemf->charenc = orig_stemf->charenc ? frt_estrdup(orig_stemf->charenc) : NULL;
     return new_ts;
 }
 
-FrtTokenStream *frt_stem_filter_new(FrtTokenStream *ts, const char *algorithm,
-                             const char *charenc)
-{
-    FrtTokenStream *tf = tf_new(FrtStemFilter, ts);
+FrtTokenStream *frt_stem_filter_alloc(void) {
+    return (FrtTokenStream *)frt_ecalloc(sizeof(FrtStemFilter));
+}
+
+void frt_stem_filter_init(FrtTokenStream *ts, FrtTokenStream *sub_ts, const char *algorithm) {
+    frt_tf_init(ts, sub_ts);
+    ts->next      = &stemf_next;
+    ts->destroy_i = &stemf_destroy_i;
+    ts->clone_i   = &stemf_clone_i;
+
     char *my_algorithm = NULL;
-    char *my_charenc   = NULL;
     char *s = NULL;
 
     if (algorithm) {
@@ -1459,186 +1072,178 @@ FrtTokenStream *frt_stem_filter_new(FrtTokenStream *ts, const char *algorithm,
             *s = tolower(*s);
             s++;
         }
-        StemFilt(tf)->algorithm = my_algorithm;
+        StemFilt(ts)->algorithm = my_algorithm;
     }
 
-    if (charenc) {
-        my_charenc   = frt_estrdup(charenc);
+    StemFilt(ts)->stemmer   = sb_stemmer_new(my_algorithm, "UTF_8");
+}
 
-        /* encodings are uppercase and use '_' instead of '-' */
-        s = my_charenc;
-        while (*s) {
-            *s = (*s == '-') ? '_' : toupper(*s);
-            s++;
-        }
-        StemFilt(tf)->charenc = my_charenc;
-    }
+FrtTokenStream *frt_stem_filter_new(FrtTokenStream *sub_ts, const char *algorithm) {
+    FrtTokenStream *ts = frt_stem_filter_alloc();
+    frt_stem_filter_init(ts, sub_ts, algorithm);
+    return ts;
+}
 
-    StemFilt(tf)->stemmer   = sb_stemmer_new(my_algorithm, my_charenc);
+/*****************************************************************************/
+/*** FrtAnalyzer *************************************************************/
+/*****************************************************************************/
 
-    tf->next = &stemf_next;
-    tf->destroy_i = &stemf_destroy_i;
-    tf->clone_i = &stemf_clone_i;
-    return tf;
+void frt_a_deref(FrtAnalyzer *a) {
+    if (--a->ref_cnt <= 0)
+        a->destroy_i(a);
 }
 
-/****************************************************************************
- *
- * Analyzers
- *
- ****************************************************************************/
+static void frt_a_standard_destroy_i(FrtAnalyzer *a) {
+    if (a->current_ts)
+        frt_ts_deref(a->current_ts);
+    free(a);
+}
 
-/****************************************************************************
- * Standard
- ****************************************************************************/
+static FrtTokenStream *a_standard_get_ts(FrtAnalyzer *a, ID field, char *text, rb_encoding *encoding) {
+    FrtTokenStream *ts;
+    (void)field;
+    ts = frt_ts_clone(a->current_ts);
+    return ts->reset(ts, text, encoding);
+}
 
-FrtAnalyzer *frt_standard_analyzer_new_with_words_len(const char **words, int len,
-                                               bool lowercase)
-{
-    FrtTokenStream *ts = frt_standard_tokenizer_new();
-    if (lowercase) {
-        ts = frt_lowercase_filter_new(ts);
-    }
-    ts = frt_hyphen_filter_new(frt_stop_filter_new_with_words_len(ts, words, len));
-    return frt_analyzer_new(ts, NULL, NULL);
+FrtAnalyzer *frt_analyzer_alloc(void) {
+    return (FrtAnalyzer *) FRT_ALLOC(FrtAnalyzer);
 }
 
-FrtAnalyzer *frt_standard_analyzer_new_with_words(const char **words,
-                                           bool lowercase)
-{
-    FrtTokenStream *ts = frt_standard_tokenizer_new();
-    if (lowercase) {
-        ts = frt_lowercase_filter_new(ts);
-    }
-    ts = frt_hyphen_filter_new(frt_stop_filter_new_with_words(ts, words));
-    return frt_analyzer_new(ts, NULL, NULL);
+void frt_analyzer_init(FrtAnalyzer *a, FrtTokenStream *ts, void (*destroy_i)(FrtAnalyzer *a),
+                       FrtTokenStream *(*get_ts)(FrtAnalyzer *a, ID field, char *text, rb_encoding *encoding)) {
+    a->current_ts = ts;
+    a->destroy_i = (destroy_i ? destroy_i : &frt_a_standard_destroy_i);
+    a->get_ts = (get_ts ? get_ts : &a_standard_get_ts);
+    a->ref_cnt = 1;
+    a->ranalyzer = Qnil;
 }
 
-FrtAnalyzer *frt_mb_standard_analyzer_new_with_words(const char **words,
-                                              bool lowercase)
-{
-    FrtTokenStream *ts = frt_mb_standard_tokenizer_new();
-    if (lowercase) {
-        ts = frt_mb_lowercase_filter_new(ts);
-    }
-    ts = frt_hyphen_filter_new(frt_stop_filter_new_with_words(ts, words));
-    return frt_analyzer_new(ts, NULL, NULL);
+FrtAnalyzer *frt_analyzer_new(FrtTokenStream *ts, void (*destroy_i)(FrtAnalyzer *a),
+                       FrtTokenStream *(*get_ts)(FrtAnalyzer *a, ID field, char *text, rb_encoding *encoding)) {
+    FrtAnalyzer *a = frt_analyzer_alloc();
+    frt_analyzer_init(a, ts, destroy_i, get_ts);
+    return a;
 }
 
-FrtAnalyzer *frt_utf8_standard_analyzer_new_with_words(const char **words,
-                                              bool lowercase)
-{
-    FrtTokenStream *ts = frt_utf8_standard_tokenizer_new();
-    if (lowercase) {
-        ts = frt_mb_lowercase_filter_new(ts);
-    }
-    ts = frt_hyphen_filter_new(frt_stop_filter_new_with_words(ts, words));
-    return frt_analyzer_new(ts, NULL, NULL);
+/*****************************************************************************/
+/*** FrtNonAnalyzer **********************************************************/
+/*****************************************************************************/
+
+FrtAnalyzer *frt_non_analyzer_new(void) {
+    return frt_analyzer_new(frt_non_tokenizer_new(), NULL, NULL);
 }
 
-FrtAnalyzer *frt_standard_analyzer_new(bool lowercase)
-{
-    return frt_standard_analyzer_new_with_words(FRT_FULL_ENGLISH_STOP_WORDS,
-                                            lowercase);
+/*****************************************************************************/
+/*** FrtWhiteSpaceAnalyzer ***************************************************/
+/*****************************************************************************/
+
+FrtAnalyzer *frt_whitespace_analyzer_alloc(void) {
+    return frt_analyzer_alloc();
 }
 
-FrtAnalyzer *frt_mb_standard_analyzer_new(bool lowercase)
-{
-    return frt_mb_standard_analyzer_new_with_words(FRT_FULL_ENGLISH_STOP_WORDS,
-                                               lowercase);
+void frt_whitespace_analyzer_init(FrtAnalyzer *a, bool lowercase) {
+    FrtTokenStream *ts = frt_whitespace_tokenizer_new();
+    if (lowercase)
+        ts = frt_lowercase_filter_new(ts);
+    frt_analyzer_init(a, ts, NULL, NULL);
 }
 
-FrtAnalyzer *frt_utf8_standard_analyzer_new(bool lowercase)
-{
-    return frt_utf8_standard_analyzer_new_with_words(FRT_FULL_ENGLISH_STOP_WORDS,
-                                                 lowercase);
+FrtAnalyzer *frt_whitespace_analyzer_new(bool lowercase) {
+    FrtAnalyzer *a = frt_whitespace_analyzer_alloc();
+    frt_whitespace_analyzer_init(a, lowercase);
+    return a;
 }
 
-/****************************************************************************
- * Legacy
- ****************************************************************************/
+/*****************************************************************************/
+/*** FrtLetterAnalyzer *******************************************************/
+/*****************************************************************************/
 
-FrtAnalyzer *frt_legacy_standard_analyzer_new_with_words(const char **words,
-                                                  bool lowercase)
-{
-    FrtTokenStream *ts = frt_legacy_standard_tokenizer_new();
-    if (lowercase) {
+FrtAnalyzer *frt_letter_analyzer_alloc(void) {
+    return frt_analyzer_alloc();
+}
+
+void frt_letter_analyzer_init(FrtAnalyzer *a, bool lowercase) {
+    FrtTokenStream *ts = frt_letter_tokenizer_new();
+    if (lowercase)
         ts = frt_lowercase_filter_new(ts);
-    }
-    ts = frt_hyphen_filter_new(frt_stop_filter_new_with_words(ts, words));
-    return frt_analyzer_new(ts, NULL, NULL);
+    frt_analyzer_init(a, ts, NULL, NULL);
 }
 
-FrtAnalyzer *frt_mb_legacy_standard_analyzer_new_with_words(const char **words,
-                                                     bool lowercase)
-{
-    FrtTokenStream *ts = frt_mb_legacy_standard_tokenizer_new();
-    if (lowercase) {
-        ts = frt_mb_lowercase_filter_new(ts);
-    }
+FrtAnalyzer *frt_letter_analyzer_new(bool lowercase) {
+    FrtAnalyzer *a = frt_letter_analyzer_alloc();
+    frt_letter_analyzer_init(a, lowercase);
+    return a;
+}
+
+/*****************************************************************************/
+/*** FrtStandardAnalyzer *****************************************************/
+/*****************************************************************************/
+
+FrtAnalyzer *frt_standard_analyzer_alloc(void) {
+    return frt_analyzer_alloc();
+}
+
+void frt_standard_analyzer_init(FrtAnalyzer *a, bool lowercase, const char **words) {
+    FrtTokenStream *ts = frt_standard_tokenizer_new();
+    if (lowercase)
+        ts = frt_lowercase_filter_new(ts);
     ts = frt_hyphen_filter_new(frt_stop_filter_new_with_words(ts, words));
-    return frt_analyzer_new(ts, NULL, NULL);
+    frt_analyzer_init(a, ts, NULL, NULL);
 }
 
-FrtAnalyzer *frt_legacy_standard_analyzer_new(bool lowercase)
-{
-    return frt_legacy_standard_analyzer_new_with_words(FRT_FULL_ENGLISH_STOP_WORDS,
-                                                   lowercase);
+FrtAnalyzer *frt_standard_analyzer_new_with_words(bool lowercase, const char **words) {
+    FrtAnalyzer *a = frt_standard_analyzer_alloc();
+    frt_standard_analyzer_init(a, lowercase, words);
+    return a;
 }
 
-FrtAnalyzer *frt_mb_legacy_standard_analyzer_new(bool lowercase)
-{
-    return frt_mb_legacy_standard_analyzer_new_with_words(FRT_FULL_ENGLISH_STOP_WORDS,
-                                                      lowercase);
+FrtAnalyzer *frt_standard_analyzer_new(bool lowercase) {
+    return frt_standard_analyzer_new_with_words(lowercase, FRT_FULL_ENGLISH_STOP_WORDS);
 }
 
-/****************************************************************************
- *
- * PerFieldAnalyzer
- *
- ****************************************************************************/
+/*****************************************************************************/
+/*** FrtPerFieldAnalyzer *****************************************************/
+/*****************************************************************************/
 
-static void pfa_destroy_i(FrtAnalyzer *self)
-{
+static void pfa_destroy_i(FrtAnalyzer *self) {
     frt_h_destroy(PFA(self)->dict);
 
     frt_a_deref(PFA(self)->default_a);
     free(self);
 }
 
-static FrtTokenStream *pfa_get_ts(FrtAnalyzer *self,
-                               FrtSymbol field, char *text)
-{
+static FrtTokenStream *pfa_get_ts(FrtAnalyzer *self, ID field, char *text, rb_encoding *encoding) {
     FrtAnalyzer *a = (FrtAnalyzer *)frt_h_get(PFA(self)->dict, (void *)field);
-    if (a == NULL) {
+    if (a == NULL)
         a = PFA(self)->default_a;
-    }
-    return frt_a_get_ts(a, field, text);
+    return frt_a_get_ts(a, field, text, encoding);
 }
 
-static void pfa_sub_a_destroy_i(void *p)
-{
+static void pfa_sub_a_destroy_i(void *p) {
     FrtAnalyzer *a = (FrtAnalyzer *) p;
     frt_a_deref(a);
 }
 
-void frt_pfa_add_field(FrtAnalyzer *self,
-                   FrtSymbol field,
-                   FrtAnalyzer *analyzer)
-{
+void frt_pfa_add_field(FrtAnalyzer *self, ID field, FrtAnalyzer *analyzer) {
     frt_h_set(PFA(self)->dict, (void *)field, analyzer);
 }
 
-FrtAnalyzer *frt_per_field_analyzer_new(FrtAnalyzer *default_a)
-{
-    FrtAnalyzer *a = (FrtAnalyzer *)frt_ecalloc(sizeof(FrtPerFieldAnalyzer));
-
-    PFA(a)->default_a = default_a;
-    PFA(a)->dict = frt_h_new_ptr(&pfa_sub_a_destroy_i);
+FrtAnalyzer *frt_per_field_analyzer_alloc(void) {
+    return (FrtAnalyzer *)frt_ecalloc(sizeof(FrtPerFieldAnalyzer));
+}
 
+void frt_per_field_analyzer_init(FrtAnalyzer *a, FrtAnalyzer *default_a) {
     a->destroy_i = &pfa_destroy_i;
-    a->get_ts    = pfa_get_ts;
+    a->get_ts    = &pfa_get_ts;
     a->ref_cnt   = 1;
 
+    PFA(a)->default_a = default_a;
+    PFA(a)->dict = frt_h_new_ptr(&pfa_sub_a_destroy_i);
+}
+
+FrtAnalyzer *frt_per_field_analyzer_new(FrtAnalyzer *default_a) {
+    FrtAnalyzer *a = frt_per_field_analyzer_alloc();
+    frt_per_field_analyzer_init(a, default_a);
     return a;
 }