ferret 0.11.6 → 0.11.8.4

data/ext/q_term.c

@@ -1,5 +1,7 @@
+#include "symbol.h"
 #include <string.h>
 #include "search.h"
+#include "internal.h"
 
 #define TQ(query) ((TermQuery *)(query))
 #define TSc(scorer) ((TermScorer *)(scorer))
@@ -103,7 +105,7 @@ static Explanation *tsc_explain(Scorer *self, int doc_num)
     }
     return expl_new(sim_tf(self->similarity, (float)tf),
                     "tf(term_freq(%s:%s)=%d)",
-                    TQ(query)->field, TQ(query)->term, tf);
+                    S(TQ(query)->field), TQ(query)->term, tf);
 }
 
 static void tsc_destroy(Scorer *self)
@@ -144,9 +146,8 @@ static Scorer *tw_scorer(Weight *self, IndexReader *ir)
 {
     TermQuery *tq = TQ(self->query);
     TermDocEnum *tde = ir_term_docs_for(ir, tq->field, tq->term);
-    if (!tde) {
-        return NULL;
-    }
+    /* ir_term_docs_for should always return a TermDocEnum */
+    assert(NULL != tde);
 
     return tsc_new(self, tde, ir_get_norms(ir, tq->field));
 }
@@ -161,10 +162,9 @@ static Explanation *tw_explain(Weight *self, IndexReader *ir, int doc_num)
     float field_norm;
     Explanation *field_norm_expl;
 
-    char *query_str = self->query->to_s(self->query, "");
+    char *query_str = self->query->to_s(self->query, NULL);
     TermQuery *tq = TQ(self->query);
     char *term = tq->term;
-    char *field = tq->field;
 
     Explanation *expl = expl_new(0.0, "weight(%s in %d), product of:",
                                  query_str, doc_num);
@@ -172,9 +172,9 @@ static Explanation *tw_explain(Weight *self, IndexReader *ir, int doc_num)
     /* We need two of these as it's included in both the query explanation
      * and the field explanation */
     Explanation *idf_expl1 = expl_new(self->idf, "idf(doc_freq=%d)",
-                                      ir_doc_freq(ir, field, term));
+                                      ir_doc_freq(ir, tq->field, term));
     Explanation *idf_expl2 = expl_new(self->idf, "idf(doc_freq=%d)",
-                                      ir_doc_freq(ir, field, term));
+                                      ir_doc_freq(ir, tq->field, term));
 
     /* explain query weight */
     Explanation *query_expl = expl_new(0.0, "query_weight(%s), product of:",
@@ -197,7 +197,7 @@ static Explanation *tw_explain(Weight *self, IndexReader *ir, int doc_num)
 
     /* explain field weight */
     field_expl = expl_new(0.0, "field_weight(%s:%s in %d), product of:",
-                          field, term, doc_num);
+                          S(tq->field), term, doc_num);
 
     scorer = self->scorer(self, ir);
     tf_expl = scorer->explain(scorer, doc_num);
@@ -205,12 +205,12 @@ static Explanation *tw_explain(Weight *self, IndexReader *ir, int doc_num)
     expl_add_detail(field_expl, tf_expl);
     expl_add_detail(field_expl, idf_expl2);
 
-    field_norms = ir_get_norms(ir, field);
+    field_norms = ir_get_norms(ir, tq->field);
     field_norm = (field_norms 
                   ? sim_decode_norm(self->similarity, field_norms[doc_num]) 
                   : (float)0.0);
     field_norm_expl = expl_new(field_norm, "field_norm(field=%s, doc=%d)",
-                               field, doc_num);
+                               S(tq->field), doc_num);
 
     expl_add_detail(field_expl, field_norm_expl);
 
@@ -259,18 +259,18 @@ static Weight *tw_new(Query *query, Searcher *searcher)
 static void tq_destroy(Query *self)
 {
     free(TQ(self)->term);
-    free(TQ(self)->field);
     q_destroy_i(self);
 }
 
-static char *tq_to_s(Query *self, const char *field)
+static char *tq_to_s(Query *self, Symbol default_field)
 {
-    size_t flen = strlen(TQ(self)->field);
+    const char *field = S(TQ(self)->field);
+    size_t flen = strlen(field);
     size_t tlen = strlen(TQ(self)->term);
     char *buffer = ALLOC_N(char, 34 + flen + tlen);
     char *b = buffer;
-    if (strcmp(field, TQ(self)->field) != 0) {
-        memcpy(b, TQ(self)->field, sizeof(char) * flen);
+    if (default_field != TQ(self)->field) {
+        memcpy(b, field, sizeof(char) * flen);
         b[flen] = ':';
         b += flen + 1;
     }
@@ -291,19 +291,19 @@ static void tq_extract_terms(Query *self, HashSet *terms)
 
 static unsigned long tq_hash(Query *self)
 {
-    return str_hash(TQ(self)->term) ^ str_hash(TQ(self)->field);
+    return str_hash(TQ(self)->term) ^ sym_hash(TQ(self)->field);
 }
 
 static int tq_eq(Query *self, Query *o)
 {
     return (strcmp(TQ(self)->term, TQ(o)->term) == 0) 
-        && (strcmp(TQ(self)->field, TQ(o)->field) == 0);
+        && (TQ(self)->field == TQ(o)->field);
 }
 
 static MatchVector *tq_get_matchv_i(Query *self, MatchVector *mv,
                                     TermVector *tv)
 {
-    if (strcmp(tv->field, TQ(self)->field) == 0) {
+    if (tv->field == TQ(self)->field) {
         int i;
         TVTerm *tv_term = tv_get_tv_term(tv, TQ(self)->term);
         if (tv_term) {
@@ -316,11 +316,11 @@ static MatchVector *tq_get_matchv_i(Query *self, MatchVector *mv,
     return mv;
 }
 
-Query *tq_new(const char *field, const char *term)
+Query *tq_new(Symbol field, const char *term)
 {
     Query *self             = q_new(TermQuery);
 
-    TQ(self)->field         = estrdup(field);
+    TQ(self)->field         = field;
     TQ(self)->term          = estrdup(term);
 
     self->type              = TERM_QUERY;

data/ext/q_wildcard.c

@@ -1,5 +1,7 @@
 #include <string.h>
 #include "search.h"
+#include "symbol.h"
+#include "internal.h"
 
 /****************************************************************************
  *
@@ -9,21 +11,17 @@
 
 #define WCQ(query) ((WildCardQuery *)(query))
 
-static char *wcq_to_s(Query *self, const char *current_field) 
+static char *wcq_to_s(Query *self, Symbol default_field)
 {
     char *buffer, *bptr;
-    const char *field = WCQ(self)->field;
+    const char *field_str = S(WCQ(self)->field);
     const char *pattern = WCQ(self)->pattern;
-    size_t flen = strlen(field);
-    size_t plen = strlen(pattern);
-    bptr = buffer = ALLOC_N(char, plen + flen + 35);
+    bptr = buffer = ALLOC_N(char, strlen(pattern) + strlen(field_str) + 35);
 
-    if (strcmp(field, current_field) != 0) {
-        sprintf(bptr, "%s:", field);
-        bptr += flen + 1;
+    if (WCQ(self)->field != default_field) {
+        bptr += sprintf(bptr, "%s:", field_str);
     }
-    sprintf(bptr, "%s", pattern);
-    bptr += plen;
+    bptr += sprintf(bptr, "%s", pattern);
 
     if (self->boost != 1.0) {
         *bptr = '^';
@@ -35,7 +33,7 @@ static char *wcq_to_s(Query *self, const char *current_field)
 
 bool wc_match(const char *pattern, const char *text)
 {
-    const char *p = pattern, *t = text, *xt; 
+    const char *p = pattern, *t = text, *xt;
 
     /* include '\0' as we need to match empty string */
     const char *text_last = t + strlen(t);
@@ -83,18 +81,17 @@ bool wc_match(const char *pattern, const char *text)
 static Query *wcq_rewrite(Query *self, IndexReader *ir)
 {
     Query *q;
-    const char *field = WCQ(self)->field;
     const char *pattern = WCQ(self)->pattern;
     const char *first_star = strchr(pattern, WILD_STRING);
     const char *first_ques = strchr(pattern, WILD_CHAR);
 
     if (NULL == first_star && NULL == first_ques) {
-        q = tq_new(field, pattern);
+        q = tq_new(WCQ(self)->field, pattern);
         q->boost = self->boost;
     }
     else {
-        const int field_num = fis_get_field_num(ir->fis, field);
-        q = multi_tq_new_conf(field, MTQMaxTerms(self), 0.0);
+        const int field_num = fis_get_field_num(ir->fis, WCQ(self)->field);
+        q = multi_tq_new_conf(WCQ(self)->field, MTQMaxTerms(self), 0.0);
 
         if (field_num >= 0) {
             TermEnum *te;
@@ -116,8 +113,8 @@ static Query *wcq_rewrite(Query *self, IndexReader *ir)
             if (te != NULL) {
                 const char *term = te->curr_term;
                 const char *pat_term = term + prefix_len;
-                do { 
-                    if (prefix && strncmp(term, prefix, prefix_len) != 0) {
+                do {
+                    if (prefix[0] && strncmp(term, prefix, prefix_len) != 0) {
                         break;
                     }
 
@@ -135,27 +132,26 @@ static Query *wcq_rewrite(Query *self, IndexReader *ir)
 
 static void wcq_destroy(Query *self)
 {
-    free(WCQ(self)->field);
     free(WCQ(self)->pattern);
     q_destroy_i(self);
 }
 
 static unsigned long wcq_hash(Query *self)
 {
-    return str_hash(WCQ(self)->field) ^ str_hash(WCQ(self)->pattern);
+    return sym_hash(WCQ(self)->field) ^ str_hash(WCQ(self)->pattern);
 }
 
 static int wcq_eq(Query *self, Query *o)
 {
-    return (strcmp(WCQ(self)->pattern, WCQ(o)->pattern) == 0) 
-        && (strcmp(WCQ(self)->field,   WCQ(o)->field) == 0);
+    return (strcmp(WCQ(self)->pattern, WCQ(o)->pattern) == 0)
+        && (WCQ(self)->field == WCQ(o)->field);
 }
 
-Query *wcq_new(const char *field, const char *pattern)
+Query *wcq_new(Symbol field, const char *pattern)
 {
     Query *self = q_new(WildCardQuery);
 
-    WCQ(self)->field        = estrdup(field);
+    WCQ(self)->field        = field;
     WCQ(self)->pattern      = estrdup(pattern);
     MTQMaxTerms(self)       = WILD_CARD_QUERY_MAX_TERMS;
 

data/ext/r_analysis.c

@@ -1,10 +1,19 @@
-#include <regex.h>
+#include "lang.h"
+#ifdef FRT_RUBY_VERSION_1_9
+#  include <ruby/re.h>
+#else
+#  include <regex.h>
+#endif
 #include <locale.h>
-#include <st.h>
+#ifdef FRT_RUBY_VERSION_1_9
+#  include <ruby/st.h>
+#else
+#  include <st.h>
+#endif
 #include "ferret.h"
 #include "analysis.h"
 
-static char *frt_locale = NULL;
+static char *frb_locale = NULL;
 
 static VALUE mAnalysis;
 
@@ -47,13 +56,19 @@ static ID id_token_stream;
 
 static VALUE object_space;
 
+#ifndef FRT_RUBY_VERSION_1_9
 extern int ruby_re_search(struct re_pattern_buffer *, const char *, int, int,
                           int, struct re_registers *);
+#endif
 
 int
-frt_rb_hash_size(VALUE hash)
+frb_rb_hash_size(VALUE hash)
 {
+#ifdef FRT_RUBY_VERSION_1_9
+    return RHASH(hash)->ntbl->num_entries;
+#else
     return RHASH(hash)->tbl->num_entries;
+#endif
 }
 
 /****************************************************************************
@@ -69,11 +84,11 @@ get_stopwords(VALUE rstop_words)
     int i, len;
     VALUE rstr;
     Check_Type(rstop_words, T_ARRAY);
-    len = RARRAY(rstop_words)->len;
-    stop_words = ALLOC_N(char *, RARRAY(rstop_words)->len + 1);
+    len = RARRAY_LEN(rstop_words);
+    stop_words = ALLOC_N(char *, RARRAY_LEN(rstop_words) + 1);
     stop_words[len] = NULL;
     for (i = 0; i < len; i++) {
-        rstr = rb_obj_as_string(RARRAY(rstop_words)->ptr[i]);
+        rstr = rb_obj_as_string(RARRAY_PTR(rstop_words)[i]);
         stop_words[i] = rs2s(rstr);
     }
     return stop_words;
@@ -93,22 +108,22 @@ typedef struct RToken {
 } RToken;
 
 static void
-frt_token_free(void *p)
+frb_token_free(void *p)
 {
     free(p);
 }
 
 static void
-frt_token_mark(void *p)
+frb_token_mark(void *p)
 {
     RToken *token = (RToken *)p;
     rb_gc_mark(token->text);
 }
 
 static VALUE
-frt_token_alloc(VALUE klass)
+frb_token_alloc(VALUE klass)
 {
-    return Data_Wrap_Struct(klass, &frt_token_mark, &frt_token_free,
+    return Data_Wrap_Struct(klass, &frb_token_mark, &frb_token_free,
                             ALLOC(RToken));
 }
 
@@ -121,18 +136,18 @@ get_token(Token *tk)
     token->start = tk->start;
     token->end = tk->end;
     token->pos_inc = tk->pos_inc;
-    return Data_Wrap_Struct(cToken, &frt_token_mark, &frt_token_free, token);
+    return Data_Wrap_Struct(cToken, &frb_token_mark, &frb_token_free, token);
 }
 
 Token *
-frt_set_token(Token *tk, VALUE rt)
+frb_set_token(Token *tk, VALUE rt)
 {
     RToken *rtk;
 
     if (rt == Qnil) return NULL;
 
     Data_Get_Struct(rt, RToken, rtk);
-    tk_set(tk, rs2s(rtk->text), RSTRING(rtk->text)->len,
+    tk_set(tk, rs2s(rtk->text), RSTRING_LEN(rtk->text),
            rtk->start, rtk->end, rtk->pos_inc);
     return tk;
 }
@@ -171,7 +186,7 @@ frt_set_token(Token *tk, VALUE rt)
  *  return::     a newly created and assigned Token object
  */
 static VALUE
-frt_token_init(int argc, VALUE *argv, VALUE self) 
+frb_token_init(int argc, VALUE *argv, VALUE self) 
 {
     RToken *token;
     VALUE rtext, rstart, rend, rpos_inc, rtype;
@@ -201,7 +216,7 @@ frt_token_init(int argc, VALUE *argv, VALUE self)
  *  lexically by the token text.
  */
 static VALUE
-frt_token_cmp(VALUE self, VALUE rother)
+frb_token_cmp(VALUE self, VALUE rother)
 {
     RToken *token, *other;
     int cmp;
@@ -230,7 +245,7 @@ frt_token_cmp(VALUE self, VALUE rother)
  *  Returns the text that this token represents
  */
 static VALUE
-frt_token_get_text(VALUE self)
+frb_token_get_text(VALUE self)
 {
     RToken *token;
     GET_TK(token, self);
@@ -244,7 +259,7 @@ frt_token_get_text(VALUE self)
  *  Set the text for this token.
  */
 static VALUE
-frt_token_set_text(VALUE self, VALUE rtext)
+frb_token_set_text(VALUE self, VALUE rtext)
 {
     RToken *token;
     GET_TK(token, self);
@@ -259,7 +274,7 @@ frt_token_set_text(VALUE self, VALUE rtext)
  *  Start byte-position of this token
  */
 static VALUE
-frt_token_get_start_offset(VALUE self)
+frb_token_get_start_offset(VALUE self)
 {
     RToken *token;
     GET_TK(token, self);
@@ -273,7 +288,7 @@ frt_token_get_start_offset(VALUE self)
  *  End byte-position of this token
  */
 static VALUE
-frt_token_get_end_offset(VALUE self)
+frb_token_get_end_offset(VALUE self)
 {
     RToken *token;
     GET_TK(token, self);
@@ -287,7 +302,7 @@ frt_token_get_end_offset(VALUE self)
  *  Position Increment for this token
  */
 static VALUE
-frt_token_get_pos_inc(VALUE self)
+frb_token_get_pos_inc(VALUE self)
 {
     RToken *token;
     GET_TK(token, self);
@@ -301,7 +316,7 @@ frt_token_get_pos_inc(VALUE self)
  *  Set start byte-position of this token
  */
 static VALUE
-frt_token_set_start_offset(VALUE self, VALUE rstart)
+frb_token_set_start_offset(VALUE self, VALUE rstart)
 {
     RToken *token;
     GET_TK(token, self);
@@ -316,7 +331,7 @@ frt_token_set_start_offset(VALUE self, VALUE rstart)
  *  Set end byte-position of this token
  */
 static VALUE
-frt_token_set_end_offset(VALUE self, VALUE rend)
+frb_token_set_end_offset(VALUE self, VALUE rend)
 {
     RToken *token;
     GET_TK(token, self);
@@ -352,7 +367,7 @@ frt_token_set_end_offset(VALUE self, VALUE rend)
  *  
  */
 static VALUE
-frt_token_set_pos_inc(VALUE self, VALUE rpos_inc)
+frb_token_set_pos_inc(VALUE self, VALUE rpos_inc)
 {
     RToken *token;
     GET_TK(token, self);
@@ -367,12 +382,12 @@ frt_token_set_pos_inc(VALUE self, VALUE rpos_inc)
  *  Return a string representation of the token
  */
 static VALUE
-frt_token_to_s(VALUE self)
+frb_token_to_s(VALUE self)
 {
     RToken *token;
     char *buf;
     GET_TK(token, self);
-    buf = alloca(RSTRING(token->text)->len + 80);
+    buf = alloca(RSTRING_LEN(token->text) + 80);
     sprintf(buf, "token[\"%s\":%d:%d:%d]", rs2s(token->text),
             token->start, token->end, token->pos_inc);
     return rb_str_new2(buf);
@@ -387,14 +402,14 @@ frt_token_to_s(VALUE self)
 #define GET_TS(ts, self) Data_Get_Struct(self, TokenStream, ts)
 
 static void
-frt_ts_mark(void *p)
+frb_ts_mark(void *p)
 {
     TokenStream *ts = (TokenStream *)p;
-    if (ts->text)   frt_gc_mark(&ts->text);
+    if (ts->text)   frb_gc_mark(&ts->text);
 }
 
 static void
-frt_ts_free(TokenStream *ts)
+frb_ts_free(TokenStream *ts)
 {
     if (object_get(&ts->text) != Qnil) {
         object_del(&ts->text);
@@ -403,8 +418,8 @@ frt_ts_free(TokenStream *ts)
     ts_deref(ts);
 }
 
-static void frt_rets_free(TokenStream *ts);
-static void frt_rets_mark(TokenStream *ts);
+static void frb_rets_free(TokenStream *ts);
+static void frb_rets_mark(TokenStream *ts);
 static Token *rets_next(TokenStream *ts);
 
 static VALUE
@@ -413,11 +428,11 @@ get_rb_token_stream(TokenStream *ts)
     VALUE rts = object_get(ts);
     if (rts == Qnil) {
         if (ts->next == &rets_next) {
-            rts = Data_Wrap_Struct(cTokenStream, &frt_rets_mark,
-                                   &frt_rets_free, ts);
+            rts = Data_Wrap_Struct(cTokenStream, &frb_rets_mark,
+                                   &frb_rets_free, ts);
         } else {
-            rts = Data_Wrap_Struct(cTokenStream, &frt_ts_mark,
-                                   &frt_ts_free, ts);
+            rts = Data_Wrap_Struct(cTokenStream, &frb_ts_mark,
+                                   &frb_ts_free, ts);
         }
         object_add(ts, rts);
     }
@@ -429,7 +444,7 @@ get_wrapped_ts(VALUE self, VALUE rstr, TokenStream *ts)
 {
     StringValue(rstr);
     ts->reset(ts, rs2s(rstr));
-    Frt_Wrap_Struct(self, &frt_ts_mark, &frt_ts_free, ts);
+    Frt_Wrap_Struct(self, &frb_ts_mark, &frb_ts_free, ts);
     object_add(&ts->text, rstr);
     object_add(ts, self);
     return self;
@@ -445,7 +460,7 @@ get_wrapped_ts(VALUE self, VALUE rstr, TokenStream *ts)
  *      token_stream.text = File.read(file_name)
  */
 static VALUE
-frt_ts_set_text(VALUE self, VALUE rtext)
+frb_ts_set_text(VALUE self, VALUE rtext)
 {
     TokenStream *ts; 
     Data_Get_Struct(self, TokenStream, ts);
@@ -465,7 +480,7 @@ frt_ts_set_text(VALUE self, VALUE rtext)
  *  Return the text that the TokenStream is tokenizing
  */
 static VALUE
-frt_ts_get_text(VALUE self)
+frb_ts_get_text(VALUE self)
 {
     VALUE rtext = Qnil;
     TokenStream *ts; 
@@ -487,7 +502,7 @@ frt_ts_get_text(VALUE self)
  *  tokens.
  */
 static VALUE
-frt_ts_next(VALUE self)
+frb_ts_next(VALUE self)
 {
     TokenStream *ts;
     Token *next;
@@ -507,16 +522,16 @@ frt_ts_next(VALUE self)
 #define TkFilt(filter) ((TokenFilter *)(filter))
 
 static void
-frt_tf_mark(void *p)
+frb_tf_mark(void *p)
 {
     TokenStream *ts = (TokenStream *)p;
     if (TkFilt(ts)->sub_ts) {
-        frt_gc_mark(&TkFilt(ts)->sub_ts);
+        frb_gc_mark(&TkFilt(ts)->sub_ts);
     }
 }
 
 static void
-frt_tf_free(TokenStream *ts)
+frb_tf_free(TokenStream *ts)
 {
     if (TkFilt(ts)->sub_ts && (object_get(&TkFilt(ts)->sub_ts) != Qnil)) {
         object_del(&TkFilt(ts)->sub_ts);
@@ -545,7 +560,7 @@ cwrts_destroy_i(TokenStream *ts)
         object_del(&ts->text);
     }
     rb_hash_delete(object_space, ((VALUE)ts)|1);
-    /*printf("rb_hash_size = %d\n", frt_rb_hash_size(object_space)); */
+    /*printf("rb_hash_size = %d\n", frb_rb_hash_size(object_space)); */
     free(ts);
 }
 
@@ -553,7 +568,7 @@ static Token *
 cwrts_next(TokenStream *ts)
 {
     VALUE rtoken = rb_funcall(CWTS(ts)->rts, id_next, 0);
-    return frt_set_token(&(CachedTS(ts)->token), rtoken);
+    return frb_set_token(&(CachedTS(ts)->token), rtoken);
 }
 
 static TokenStream *
@@ -574,10 +589,10 @@ cwrts_clone_i(TokenStream *orig_ts)
 }
 
 static TokenStream *
-frt_get_cwrapped_rts(VALUE rts) 
+frb_get_cwrapped_rts(VALUE rts) 
 {
     TokenStream *ts;
-    if (frt_is_cclass(rts) && DATA_PTR(rts)) {
+    if (frb_is_cclass(rts) && DATA_PTR(rts)) {
         GET_TS(ts, rts);
         REF(ts);
     }
@@ -621,7 +636,7 @@ typedef struct RegExpTokenStream {
     VALUE rtext;
     VALUE regex;
     VALUE proc;
-    int   curr_ind;  
+    long   curr_ind;  
 } RegExpTokenStream;
 
 static void
@@ -631,12 +646,12 @@ rets_destroy_i(TokenStream *ts)
         object_del(&ts->text);
     }
     rb_hash_delete(object_space, ((VALUE)ts)|1);
-    /*printf("rb_hash_size = %d\n", frt_rb_hash_size(object_space)); */
+    /*printf("rb_hash_size = %d\n", frb_rb_hash_size(object_space)); */
     free(ts);
 }
 
 static void
-frt_rets_free(TokenStream *ts)
+frb_rets_free(TokenStream *ts)
 {
     if (object_get(&ts->text) != Qnil) {
         object_del(&ts->text);
@@ -646,9 +661,9 @@ frt_rets_free(TokenStream *ts)
 }
 
 static void
-frt_rets_mark(TokenStream *ts)
+frb_rets_mark(TokenStream *ts)
 {
-    if (ts->text)   frt_gc_mark(&ts->text);
+    if (ts->text)   frb_gc_mark(&ts->text);
     rb_gc_mark(RETS(ts)->rtext);
     rb_gc_mark(RETS(ts)->regex);
     rb_gc_mark(RETS(ts)->proc);
@@ -662,7 +677,7 @@ frt_rets_mark(TokenStream *ts)
  *  tokenize the text from the beginning.
  */
 static VALUE
-frt_rets_set_text(VALUE self, VALUE rtext)
+frb_rets_set_text(VALUE self, VALUE rtext)
 {
     TokenStream *ts;
     GET_TS(ts, self);
@@ -682,23 +697,88 @@ frt_rets_set_text(VALUE self, VALUE rtext)
  *  Get the text being tokenized by the tokenizer.
  */
 static VALUE
-frt_rets_get_text(VALUE self)
+frb_rets_get_text(VALUE self)
 {
     TokenStream *ts;
     GET_TS(ts, self);
     return RETS(ts)->rtext;
 }
 
+#ifdef FRT_RUBY_VERSION_1_9
+
+// partly lifted from ruby 1.9 string.c
+#include <ruby/encoding.h>
+#define BEG(no) regs->beg[no]
+#define END(no) regs->end[no]
+#define STR_ENC_GET(str) rb_enc_from_index(ENCODING_GET(str))
+static VALUE
+  scan_once(VALUE str, VALUE pat, long *start)
+{
+  VALUE match;
+  struct re_registers *regs;
+
+  if (rb_reg_search(pat, str, *start, 0) >= 0) {
+    match = rb_backref_get();
+    regs = RMATCH_REGS(match);
+    if (BEG(0) == END(0)) {
+      rb_encoding *enc = STR_ENC_GET(str);
+      /*
+      * Always consume at least one character of the input string
+       */
+        if (RSTRING_LEN(str) > END(0))
+        *start = END(0)+rb_enc_mbclen(RSTRING_PTR(str)+END(0),
+        RSTRING_END(str), enc);
+      else
+        *start = END(0)+1;
+    }
+    else {
+      *start = END(0);
+    }
+    return rb_reg_nth_match(0, match);
+  }
+  return Qnil;
+}
+//
+
+static Token *
+  rets_next(TokenStream *ts)
+{
+  VALUE ret;
+  long rtok_len;
+  int beg, end;
+  Check_Type(RETS(ts)->regex, T_REGEXP);
+  ret = scan_once(RETS(ts)->rtext, RETS(ts)->regex, &(RETS(ts)->curr_ind));
+  if (NIL_P(ret)) return NULL;
+
+  Check_Type(ret, T_STRING);
+  rtok_len = RSTRING_LEN(ret);
+  beg = RETS(ts)->curr_ind - rtok_len;
+  end = RETS(ts)->curr_ind;
+
+  if (NIL_P(RETS(ts)->proc)) {
+    return tk_set(&(CachedTS(ts)->token), rs2s(ret), rtok_len,
+      beg, end, 1);
+  } else {
+    VALUE rtok;
+    rtok = rb_funcall(RETS(ts)->proc, id_call, 1, ret);
+    return tk_set(&(CachedTS(ts)->token), rs2s(rtok),
+      RSTRING_LEN(rtok), beg, end, 1);
+  }
+}
+
+#else
+
 static Token *
 rets_next(TokenStream *ts)
 {
     static struct re_registers regs;
     int ret, beg, end;
-    struct RString *rtext = RSTRING(RETS(ts)->rtext);
+    long rtext_len = RSTRING_LEN(RETS(ts)->rtext);
+    char *rtext_ptr = RSTRING_PTR(RETS(ts)->rtext);
     Check_Type(RETS(ts)->regex, T_REGEXP);
     ret = ruby_re_search(RREGEXP(RETS(ts)->regex)->ptr,
-                         rtext->ptr, rtext->len,
-                         RETS(ts)->curr_ind, rtext->len - RETS(ts)->curr_ind,
+                         rtext_ptr, rtext_len,
+                         RETS(ts)->curr_ind, rtext_len - RETS(ts)->curr_ind,
                          &regs);
 
     if (ret == -2) rb_raise(rb_eStandardError, "regexp buffer overflow");
@@ -707,16 +787,18 @@ rets_next(TokenStream *ts)
     beg = regs.beg[0];
     RETS(ts)->curr_ind = end = regs.end[0];
     if (NIL_P(RETS(ts)->proc)) {
-        return tk_set(&(CachedTS(ts)->token), rtext->ptr + beg, end - beg,
+        return tk_set(&(CachedTS(ts)->token), rtext_ptr + beg, end - beg,
                       beg, end, 1);
     } else {
-        VALUE rtok = rb_str_new(rtext->ptr + beg, end - beg);
+        VALUE rtok = rb_str_new(rtext_ptr + beg, end - beg);
         rtok = rb_funcall(RETS(ts)->proc, id_call, 1, rtok);
         return tk_set(&(CachedTS(ts)->token), rs2s(rtok),
-                      RSTRING(rtok)->len, beg, end, 1);
+                      RSTRING_LEN(rtok), beg, end, 1);
     }
 }
 
+#endif
+
 static TokenStream *
 rets_reset(TokenStream *ts, char *text)
 {
@@ -770,7 +852,7 @@ rets_new(VALUE rtext, VALUE regex, VALUE proc)
  *  regexp:: regular expression used to recognize tokens in the input
  */
 static VALUE
-frt_rets_init(int argc, VALUE *argv, VALUE self) 
+frb_rets_init(int argc, VALUE *argv, VALUE self) 
 {
     VALUE rtext, regex, proc;
     TokenStream *ts;
@@ -779,7 +861,7 @@ frt_rets_init(int argc, VALUE *argv, VALUE self)
 
     ts = rets_new(rtext, regex, proc);
 
-    Frt_Wrap_Struct(self, &frt_rets_mark, &frt_rets_free, ts);
+    Frt_Wrap_Struct(self, &frb_rets_mark, &frb_rets_free, ts);
     object_add(ts, self);
     return self;
 }
@@ -801,7 +883,7 @@ lower = (argc ? RTEST(rlower) : dflt)
  *  Create a new AsciiLetterTokenizer 
  */
 static VALUE
-frt_a_letter_tokenizer_init(VALUE self, VALUE rstr) 
+frb_a_letter_tokenizer_init(VALUE self, VALUE rstr) 
 {
     return get_wrapped_ts(self, rstr, letter_tokenizer_new());
 }
@@ -816,11 +898,11 @@ frt_a_letter_tokenizer_init(VALUE self, VALUE rstr)
  *  lower:: set to false if you don't wish to downcase tokens
  */
 static VALUE
-frt_letter_tokenizer_init(int argc, VALUE *argv, VALUE self) 
+frb_letter_tokenizer_init(int argc, VALUE *argv, VALUE self) 
 {
     TS_ARGS(false);
 #ifndef POSH_OS_WIN32
-    if (!frt_locale) frt_locale = setlocale(LC_CTYPE, "");
+    if (!frb_locale) frb_locale = setlocale(LC_CTYPE, "");
 #endif
     return get_wrapped_ts(self, rstr, mb_letter_tokenizer_new(lower));
 }
@@ -832,7 +914,7 @@ frt_letter_tokenizer_init(int argc, VALUE *argv, VALUE self)
  *  Create a new AsciiWhiteSpaceTokenizer 
  */
 static VALUE
-frt_a_whitespace_tokenizer_init(VALUE self, VALUE rstr) 
+frb_a_whitespace_tokenizer_init(VALUE self, VALUE rstr) 
 {
     return get_wrapped_ts(self, rstr, whitespace_tokenizer_new());
 }
@@ -847,11 +929,11 @@ frt_a_whitespace_tokenizer_init(VALUE self, VALUE rstr)
  *  lower:: set to false if you don't wish to downcase tokens
  */
 static VALUE
-frt_whitespace_tokenizer_init(int argc, VALUE *argv, VALUE self) 
+frb_whitespace_tokenizer_init(int argc, VALUE *argv, VALUE self) 
 {
     TS_ARGS(false);
 #ifndef POSH_OS_WIN32
-    if (!frt_locale) frt_locale = setlocale(LC_CTYPE, "");
+    if (!frb_locale) frb_locale = setlocale(LC_CTYPE, "");
 #endif
     return get_wrapped_ts(self, rstr, mb_whitespace_tokenizer_new(lower));
 }
@@ -863,7 +945,7 @@ frt_whitespace_tokenizer_init(int argc, VALUE *argv, VALUE self)
  *  Create a new AsciiStandardTokenizer 
  */
 static VALUE
-frt_a_standard_tokenizer_init(VALUE self, VALUE rstr) 
+frb_a_standard_tokenizer_init(VALUE self, VALUE rstr) 
 {
     return get_wrapped_ts(self, rstr, standard_tokenizer_new());
 }
@@ -878,10 +960,10 @@ frt_a_standard_tokenizer_init(VALUE self, VALUE rstr)
  *  lower:: set to false if you don't wish to downcase tokens
  */
 static VALUE
-frt_standard_tokenizer_init(VALUE self, VALUE rstr) 
+frb_standard_tokenizer_init(VALUE self, VALUE rstr) 
 {
 #ifndef POSH_OS_WIN32
-    if (!frt_locale) frt_locale = setlocale(LC_CTYPE, "");
+    if (!frb_locale) frb_locale = setlocale(LC_CTYPE, "");
 #endif
     return get_wrapped_ts(self, rstr, mb_standard_tokenizer_new());
 }
@@ -900,13 +982,13 @@ frt_standard_tokenizer_init(VALUE self, VALUE rstr)
  *  LowerCaseFilter.
  */
 static VALUE
-frt_a_lowercase_filter_init(VALUE self, VALUE rsub_ts) 
+frb_a_lowercase_filter_init(VALUE self, VALUE rsub_ts) 
 {
-    TokenStream *ts = frt_get_cwrapped_rts(rsub_ts);
+    TokenStream *ts = frb_get_cwrapped_rts(rsub_ts);
     ts = lowercase_filter_new(ts);
     object_add(&(TkFilt(ts)->sub_ts), rsub_ts);
 
-    Frt_Wrap_Struct(self, &frt_tf_mark, &frt_tf_free, ts);
+    Frt_Wrap_Struct(self, &frb_tf_mark, &frb_tf_free, ts);
     object_add(ts, self);
     return self;
 }
@@ -919,16 +1001,16 @@ frt_a_lowercase_filter_init(VALUE self, VALUE rsub_ts)
  *  lowercase based on the current locale.
  */
 static VALUE
-frt_lowercase_filter_init(VALUE self, VALUE rsub_ts) 
+frb_lowercase_filter_init(VALUE self, VALUE rsub_ts) 
 {
-    TokenStream *ts = frt_get_cwrapped_rts(rsub_ts);
+    TokenStream *ts = frb_get_cwrapped_rts(rsub_ts);
 #ifndef POSH_OS_WIN32
-    if (!frt_locale) frt_locale = setlocale(LC_CTYPE, "");
+    if (!frb_locale) frb_locale = setlocale(LC_CTYPE, "");
 #endif
     ts = mb_lowercase_filter_new(ts);
     object_add(&(TkFilt(ts)->sub_ts), rsub_ts);
 
-    Frt_Wrap_Struct(self, &frt_tf_mark, &frt_tf_free, ts);
+    Frt_Wrap_Struct(self, &frb_tf_mark, &frb_tf_free, ts);
     object_add(ts, self);
     return self;
 }
@@ -944,13 +1026,13 @@ frt_lowercase_filter_init(VALUE self, VALUE rsub_ts)
  *  used by default by the StandardAnalyzer.
  */
 static VALUE
-frt_hyphen_filter_init(VALUE self, VALUE rsub_ts) 
+frb_hyphen_filter_init(VALUE self, VALUE rsub_ts) 
 {
-    TokenStream *ts = frt_get_cwrapped_rts(rsub_ts);
+    TokenStream *ts = frb_get_cwrapped_rts(rsub_ts);
     ts = hyphen_filter_new(ts);
     object_add(&(TkFilt(ts)->sub_ts), rsub_ts);
 
-    Frt_Wrap_Struct(self, &frt_tf_mark, &frt_tf_free, ts);
+    Frt_Wrap_Struct(self, &frb_tf_mark, &frb_tf_free, ts);
     object_add(ts, self);
     return self;
 }
@@ -969,12 +1051,12 @@ frt_hyphen_filter_init(VALUE self, VALUE rsub_ts)
  *                 Ferret::Analysis contains a number of stop-word lists.
  */
 static VALUE
-frt_stop_filter_init(int argc, VALUE *argv, VALUE self) 
+frb_stop_filter_init(int argc, VALUE *argv, VALUE self) 
 {
     VALUE rsub_ts, rstop_words;
     TokenStream *ts;
     rb_scan_args(argc, argv, "11", &rsub_ts, &rstop_words);
-    ts = frt_get_cwrapped_rts(rsub_ts);
+    ts = frb_get_cwrapped_rts(rsub_ts);
     if (rstop_words != Qnil) {
         char **stop_words = get_stopwords(rstop_words);
         ts = stop_filter_new_with_words(ts, (const char **)stop_words);
@@ -985,12 +1067,13 @@ frt_stop_filter_init(int argc, VALUE *argv, VALUE self)
     }
     object_add(&(TkFilt(ts)->sub_ts), rsub_ts);
 
-    Frt_Wrap_Struct(self, &frt_tf_mark, &frt_tf_free, ts);
+    Frt_Wrap_Struct(self, &frb_tf_mark, &frb_tf_free, ts);
     object_add(ts, self);
     return self;
 }
 
-static INLINE void frt_add_mapping_i(TokenStream *mf, VALUE from, char *to)
+static INLINE void frb_add_mapping_i(TokenStream *mf, VALUE from,
+                                     const char *to)
 {
     switch (TYPE(from)) {
         case T_STRING:
@@ -1007,13 +1090,13 @@ static INLINE void frt_add_mapping_i(TokenStream *mf, VALUE from, char *to)
     }
 }
 
-static int frt_add_mappings_i(VALUE key, VALUE value, VALUE arg)
+static int frb_add_mappings_i(VALUE key, VALUE value, VALUE arg)
 {
     if (key == Qundef) {
         return ST_CONTINUE;
     } else {
         TokenStream *mf = (TokenStream *)arg;
-        char *to;
+        const char *to;
         switch (TYPE(value)) {
             case T_STRING:
                 to = rs2s(value);
@@ -1029,12 +1112,12 @@ static int frt_add_mappings_i(VALUE key, VALUE value, VALUE arg)
         }
         if (TYPE(key) == T_ARRAY) {
             int i;
-            for (i = RARRAY(key)->len - 1; i >= 0; i--) {
-                frt_add_mapping_i(mf, RARRAY(key)->ptr[i], to);
+            for (i = RARRAY_LEN(key) - 1; i >= 0; i--) {
+                frb_add_mapping_i(mf, RARRAY_PTR(key)[i], to);
             }
         }
         else {
-            frt_add_mapping_i(mf, key, to);
+            frb_add_mapping_i(mf, key, to);
         }
     }
     return ST_CONTINUE;
@@ -1066,16 +1149,16 @@ static int frt_add_mappings_i(VALUE key, VALUE value, VALUE arg)
  *                              })
  */
 static VALUE
-frt_mapping_filter_init(VALUE self, VALUE rsub_ts, VALUE mapping) 
+frb_mapping_filter_init(VALUE self, VALUE rsub_ts, VALUE mapping) 
 {
     TokenStream *ts;
-    ts = frt_get_cwrapped_rts(rsub_ts);
+    ts = frb_get_cwrapped_rts(rsub_ts);
     ts = mapping_filter_new(ts);
-    rb_hash_foreach(mapping, frt_add_mappings_i, (VALUE)ts);
+    rb_hash_foreach(mapping, frb_add_mappings_i, (VALUE)ts);
     mulmap_compile(((MappingFilter *)ts)->mapper);
     object_add(&(TkFilt(ts)->sub_ts), rsub_ts);
 
-    Frt_Wrap_Struct(self, &frt_tf_mark, &frt_tf_free, ts);
+    Frt_Wrap_Struct(self, &frb_tf_mark, &frb_tf_free, ts);
     object_add(ts, self);
     return self;
 }
@@ -1096,14 +1179,14 @@ frt_mapping_filter_init(VALUE self, VALUE rsub_ts, VALUE mapping)
  *  encoding::     The encoding of the data (default: "UTF-8")
  */
 static VALUE
-frt_stem_filter_init(int argc, VALUE *argv, VALUE self) 
+frb_stem_filter_init(int argc, VALUE *argv, VALUE self) 
 {
     VALUE rsub_ts, ralgorithm, rcharenc;
     char *algorithm = "english";
     char *charenc = NULL;
     TokenStream *ts;
     rb_scan_args(argc, argv, "12", &rsub_ts, &ralgorithm, &rcharenc);
-    ts = frt_get_cwrapped_rts(rsub_ts);
+    ts = frb_get_cwrapped_rts(rsub_ts);
     switch (argc) {
         case 3: charenc = rs2s(rb_obj_as_string(rcharenc));
         case 2: algorithm = rs2s(rb_obj_as_string(ralgorithm));
@@ -1111,8 +1194,12 @@ frt_stem_filter_init(int argc, VALUE *argv, VALUE self)
     ts = stem_filter_new(ts, algorithm, charenc);
     object_add(&(TkFilt(ts)->sub_ts), rsub_ts);
 
-    Frt_Wrap_Struct(self, &frt_tf_mark, &frt_tf_free, ts);
+    Frt_Wrap_Struct(self, &frb_tf_mark, &frb_tf_free, ts);
     object_add(ts, self);
+    if (((StemFilter *)ts)->stemmer == NULL) {
+        rb_raise(rb_eArgError, "No stemmer could be found with the encoding "
+                 "%s and the language %s", charenc, algorithm);
+    }
     return self;
 }
 
@@ -1139,28 +1226,28 @@ static void
 cwa_destroy_i(Analyzer *a)
 {
     rb_hash_delete(object_space, ((VALUE)a)|1);
-    /*printf("rb_hash_size = %d\n", frt_rb_hash_size(object_space)); */
+    /*printf("rb_hash_size = %d\n", frb_rb_hash_size(object_space)); */
     free(a);
 }
 
 static TokenStream *
-cwa_get_ts(Analyzer *a, char *field, char *text)
+cwa_get_ts(Analyzer *a, Symbol field, char *text)
 {
     VALUE rts = rb_funcall(CWA(a)->ranalyzer, id_token_stream, 2,
-                           ID2SYM(rb_intern(field)), rb_str_new2(text));
-    return frt_get_cwrapped_rts(rts);
+                           FSYM2SYM(field), rb_str_new2(text));
+    return frb_get_cwrapped_rts(rts);
 } 
 
 Analyzer *
-frt_get_cwrapped_analyzer(VALUE ranalyzer)
+frb_get_cwrapped_analyzer(VALUE ranalyzer)
 {
     Analyzer *a = NULL;
-    if (frt_is_cclass(ranalyzer) && DATA_PTR(ranalyzer)) {
+    if (frb_is_cclass(ranalyzer) && DATA_PTR(ranalyzer)) {
         Data_Get_Struct(ranalyzer, Analyzer, a);
         REF(a);
     }
     else {
-        a = (Analyzer *)ecalloc(sizeof(CWrappedAnalyzer));
+        a = (Analyzer *)frt_ecalloc(sizeof(CWrappedAnalyzer));
         a->destroy_i = &cwa_destroy_i;
         a->get_ts    = &cwa_get_ts;
         a->ref_cnt   = 1;
@@ -1172,20 +1259,20 @@ frt_get_cwrapped_analyzer(VALUE ranalyzer)
 }
 
 static void
-frt_analyzer_free(Analyzer *a)
+frb_analyzer_free(Analyzer *a)
 {
     object_del(a);
     a_deref(a);
 }
 
 VALUE
-frt_get_analyzer(Analyzer *a)
+frb_get_analyzer(Analyzer *a)
 {
     VALUE self = Qnil;
     if (a) {
         self = object_get(a);
         if (self == Qnil) {
-            self = Data_Wrap_Struct(cAnalyzer, NULL, &frt_analyzer_free, a);
+            self = Data_Wrap_Struct(cAnalyzer, NULL, &frb_analyzer_free, a);
             REF(a);
             object_add(a, self);
         }
@@ -1196,7 +1283,7 @@ frt_get_analyzer(Analyzer *a)
 INLINE VALUE
 get_rb_ts_from_a(Analyzer *a, VALUE rfield, VALUE rstring)
 {
-    TokenStream *ts = a_get_ts(a, frt_field(rfield), rs2s(rstring));
+    TokenStream *ts = a_get_ts(a, frb_field(rfield), rs2s(rstring));
 
     /* Make sure that there is no entry already */
     object_set(&ts->text, rstring);
@@ -1215,10 +1302,10 @@ get_rb_ts_from_a(Analyzer *a, VALUE rfield, VALUE rstring)
  *  input::      data from the field to be tokenized
  */
 static VALUE
-frt_analyzer_token_stream(VALUE self, VALUE rfield, VALUE rstring)
+frb_analyzer_token_stream(VALUE self, VALUE rfield, VALUE rstring)
 {
     /* NOTE: Any changes made to this method may also need to be applied to
-     * frt_re_analyzer_token_stream */
+     * frb_re_analyzer_token_stream */
     Analyzer *a;
     GET_A(a, self);
 
@@ -1244,12 +1331,12 @@ lower = (argc ? RTEST(rlower) : dflt)
  *  lower:: set to false if you don't want the field's tokens to be downcased
  */
 static VALUE
-frt_a_white_space_analyzer_init(int argc, VALUE *argv, VALUE self)
+frb_a_white_space_analyzer_init(int argc, VALUE *argv, VALUE self)
 {
     Analyzer *a;
     GET_LOWER(false);
     a = whitespace_analyzer_new(lower);
-    Frt_Wrap_Struct(self, NULL, &frt_analyzer_free, a);
+    Frt_Wrap_Struct(self, NULL, &frb_analyzer_free, a);
     object_add(a, self);
     return self;
 }
@@ -1265,15 +1352,15 @@ frt_a_white_space_analyzer_init(int argc, VALUE *argv, VALUE self)
  *  lower:: set to false if you don't want the field's tokens to be downcased
  */
 static VALUE
-frt_white_space_analyzer_init(int argc, VALUE *argv, VALUE self)
+frb_white_space_analyzer_init(int argc, VALUE *argv, VALUE self)
 {
     Analyzer *a;
     GET_LOWER(false);
 #ifndef POSH_OS_WIN32
-    if (!frt_locale) frt_locale = setlocale(LC_CTYPE, "");
+    if (!frb_locale) frb_locale = setlocale(LC_CTYPE, "");
 #endif
     a = mb_whitespace_analyzer_new(lower);
-    Frt_Wrap_Struct(self, NULL, &frt_analyzer_free, a);
+    Frt_Wrap_Struct(self, NULL, &frb_analyzer_free, a);
     object_add(a, self);
     return self;
 }
@@ -1289,12 +1376,12 @@ frt_white_space_analyzer_init(int argc, VALUE *argv, VALUE self)
  *  lower:: set to false if you don't want the field's tokens to be downcased
  */
 static VALUE
-frt_a_letter_analyzer_init(int argc, VALUE *argv, VALUE self)
+frb_a_letter_analyzer_init(int argc, VALUE *argv, VALUE self)
 {
     Analyzer *a;
     GET_LOWER(true);
     a = letter_analyzer_new(lower);
-    Frt_Wrap_Struct(self, NULL, &frt_analyzer_free, a);
+    Frt_Wrap_Struct(self, NULL, &frb_analyzer_free, a);
     object_add(a, self);
     return self;
 }
@@ -1310,15 +1397,15 @@ frt_a_letter_analyzer_init(int argc, VALUE *argv, VALUE self)
  *  lower:: set to false if you don't want the field's tokens to be downcased
  */
 static VALUE
-frt_letter_analyzer_init(int argc, VALUE *argv, VALUE self)
+frb_letter_analyzer_init(int argc, VALUE *argv, VALUE self)
 {
     Analyzer *a;
     GET_LOWER(true);
 #ifndef POSH_OS_WIN32
-    if (!frt_locale) frt_locale = setlocale(LC_CTYPE, "");
+    if (!frb_locale) frb_locale = setlocale(LC_CTYPE, "");
 #endif
     a = mb_letter_analyzer_new(lower);
-    Frt_Wrap_Struct(self, NULL, &frt_analyzer_free, a);
+    Frt_Wrap_Struct(self, NULL, &frb_analyzer_free, a);
     object_add(a, self);
     return self;
 }
@@ -1350,7 +1437,7 @@ get_rstopwords(const char **stop_words)
  *  stop_words:: list of stop-words to pass to the StopFilter
  */
 static VALUE
-frt_a_standard_analyzer_init(int argc, VALUE *argv, VALUE self)
+frb_a_standard_analyzer_init(int argc, VALUE *argv, VALUE self)
 {
     bool lower;
     VALUE rlower, rstop_words;
@@ -1364,7 +1451,7 @@ frt_a_standard_analyzer_init(int argc, VALUE *argv, VALUE self)
     } else {
         a = standard_analyzer_new(lower);
     }
-    Frt_Wrap_Struct(self, NULL, &frt_analyzer_free, a);
+    Frt_Wrap_Struct(self, NULL, &frb_analyzer_free, a);
     object_add(a, self);
     return self;
 }
@@ -1383,13 +1470,13 @@ frt_a_standard_analyzer_init(int argc, VALUE *argv, VALUE self)
  *  stop_words:: list of stop-words to pass to the StopFilter
  */
 static VALUE
-frt_standard_analyzer_init(int argc, VALUE *argv, VALUE self)
+frb_standard_analyzer_init(int argc, VALUE *argv, VALUE self)
 {
     bool lower;
     VALUE rlower, rstop_words;
     Analyzer *a;
 #ifndef POSH_OS_WIN32
-    if (!frt_locale) frt_locale = setlocale(LC_CTYPE, "");
+    if (!frb_locale) frb_locale = setlocale(LC_CTYPE, "");
 #endif
     rb_scan_args(argc, argv, "02", &rstop_words, &rlower);
     lower = ((rlower == Qnil) ? true : RTEST(rlower));
@@ -1400,22 +1487,22 @@ frt_standard_analyzer_init(int argc, VALUE *argv, VALUE self)
     } else {
         a = mb_standard_analyzer_new(lower);
     }
-    Frt_Wrap_Struct(self, NULL, &frt_analyzer_free, a);
+    Frt_Wrap_Struct(self, NULL, &frb_analyzer_free, a);
     object_add(a, self);
     return self;
 }
 
 static void
-frt_h_mark_values_i(void *key, void *value, void *arg)
+frb_h_mark_values_i(void *key, void *value, void *arg)
 {
-    frt_gc_mark(value);
+    frb_gc_mark(value);
 }
 
 static void
-frt_pfa_mark(void *p)
+frb_pfa_mark(void *p)
 {
-    frt_gc_mark(PFA(p)->default_a);
-    h_each(PFA(p)->dict, &frt_h_mark_values_i, NULL);
+    frb_gc_mark(PFA(p)->default_a);
+    h_each(PFA(p)->dict, &frb_h_mark_values_i, NULL);
 }
 
 /*** PerFieldAnalyzer ***/
@@ -1431,11 +1518,11 @@ frt_pfa_mark(void *p)
  *                     specified
  */
 static VALUE
-frt_per_field_analyzer_init(VALUE self, VALUE ranalyzer)
+frb_per_field_analyzer_init(VALUE self, VALUE ranalyzer)
 {
-    Analyzer *def = frt_get_cwrapped_analyzer(ranalyzer);
+    Analyzer *def = frb_get_cwrapped_analyzer(ranalyzer);
     Analyzer *a = per_field_analyzer_new(def);
-    Frt_Wrap_Struct(self, &frt_pfa_mark, &frt_analyzer_free, a);
+    Frt_Wrap_Struct(self, &frb_pfa_mark, &frb_analyzer_free, a);
     object_add(a, self);
     return self;
 }
@@ -1452,13 +1539,13 @@ frt_per_field_analyzer_init(VALUE self, VALUE ranalyzer)
  *  analyzer::   analyzer to be used on +field_name+
  */
 static VALUE
-frt_per_field_analyzer_add_field(VALUE self, VALUE rfield, VALUE ranalyzer)
+frb_per_field_analyzer_add_field(VALUE self, VALUE rfield, VALUE ranalyzer)
 {
     Analyzer *pfa, *a;
     Data_Get_Struct(self, Analyzer, pfa);
-    a = frt_get_cwrapped_analyzer(ranalyzer);
+    a = frb_get_cwrapped_analyzer(ranalyzer);
 
-    pfa_add_field(pfa, frt_field(rfield), a);
+    pfa_add_field(pfa, frb_field(rfield), a);
     return self;
 }
 
@@ -1473,10 +1560,10 @@ frt_per_field_analyzer_add_field(VALUE self, VALUE rfield, VALUE ranalyzer)
  *  input::      data from the field to be tokenized
  */
 static VALUE
-frt_pfa_analyzer_token_stream(VALUE self, VALUE rfield, VALUE rstring)
+frb_pfa_analyzer_token_stream(VALUE self, VALUE rfield, VALUE rstring)
 {
     Analyzer *pfa, *a;
-    char *field = frt_field(rfield);
+    Symbol field = frb_field(rfield);
     GET_A(pfa, self);
 
     StringValue(rstring);
@@ -1486,7 +1573,7 @@ frt_pfa_analyzer_token_stream(VALUE self, VALUE rfield, VALUE rstring)
     }
     if (a->get_ts == cwa_get_ts) {
         return rb_funcall(CWA(a)->ranalyzer, id_token_stream, 2,
-                          ID2SYM(rb_intern(field)), rb_str_new2(rs2s(rstring)));
+                          FSYM2SYM(field), rb_str_new2(rs2s(rstring)));
     }
     else {
         return get_rb_ts_from_a(a, rfield, rstring);
@@ -1496,9 +1583,9 @@ frt_pfa_analyzer_token_stream(VALUE self, VALUE rfield, VALUE rstring)
 /*** RegExpAnalyzer ***/
 
 static void
-frt_re_analyzer_mark(Analyzer *a)
+frb_re_analyzer_mark(Analyzer *a)
 {
-    frt_gc_mark(a->current_ts);
+    frb_gc_mark(a->current_ts);
 }
 
 static void
@@ -1519,7 +1606,7 @@ re_analyzer_destroy_i(Analyzer *a)
  *  lower::   set to false if you don't want to downcase the tokens
  */
 static VALUE
-frt_re_analyzer_init(int argc, VALUE *argv, VALUE self)
+frb_re_analyzer_init(int argc, VALUE *argv, VALUE self)
 {
     VALUE lower, rets, regex, proc;
     Analyzer *a;
@@ -1527,17 +1614,17 @@ frt_re_analyzer_init(int argc, VALUE *argv, VALUE self)
     rb_scan_args(argc, argv, "02&", &regex, &lower, &proc);
 
     ts = rets_new(Qnil, regex, proc);
-    rets = Data_Wrap_Struct(cRegExpTokenizer, &frt_rets_mark, &frt_rets_free, ts);
+    rets = Data_Wrap_Struct(cRegExpTokenizer, &frb_rets_mark, &frb_rets_free, ts);
     object_add(ts, rets);
 
     if (lower != Qfalse) {
-        rets = frt_lowercase_filter_init(frt_data_alloc(cLowerCaseFilter), rets);
+        rets = frb_lowercase_filter_init(frb_data_alloc(cLowerCaseFilter), rets);
         ts = DATA_PTR(rets);
     }
     REF(ts);
 
     a = analyzer_new(ts, &re_analyzer_destroy_i, NULL);
-    Frt_Wrap_Struct(self, &frt_re_analyzer_mark, &frt_analyzer_free, a);
+    Frt_Wrap_Struct(self, &frb_re_analyzer_mark, &frb_analyzer_free, a);
     object_add(a, self);
     return self;
 }
@@ -1554,7 +1641,7 @@ frt_re_analyzer_init(int argc, VALUE *argv, VALUE self)
  *  input::      data from the field to be tokenized
  */
 static VALUE
-frt_re_analyzer_token_stream(VALUE self, VALUE rfield, VALUE rtext)
+frb_re_analyzer_token_stream(VALUE self, VALUE rfield, VALUE rtext)
 {
     TokenStream *ts;
     Analyzer *a;
@@ -1562,7 +1649,7 @@ frt_re_analyzer_token_stream(VALUE self, VALUE rfield, VALUE rtext)
 
     StringValue(rtext);
 
-    ts = a_get_ts(a, frt_field(rfield), rs2s(rtext));
+    ts = a_get_ts(a, frb_field(rfield), rs2s(rtext));
 
     /* Make sure that there is no entry already */
     object_set(&ts->text, rtext);
@@ -1591,9 +1678,9 @@ frt_re_analyzer_token_stream(VALUE self, VALUE rfield, VALUE rtext)
  *
  *     puts Ferret.locale #=> "en_US.UTF-8"
  */
-static VALUE frt_get_locale(VALUE self, VALUE locale)
+static VALUE frb_get_locale(VALUE self, VALUE locale)
 {
-    return (frt_locale ? rb_str_new2(frt_locale) : Qnil);
+    return (frb_locale ? rb_str_new2(frb_locale) : Qnil);
 }
 
 /*
@@ -1603,11 +1690,11 @@ static VALUE frt_get_locale(VALUE self, VALUE locale)
  *  Set the global locale. You should use this method to set different locales
  *  when indexing documents with different encodings.
  */
-static VALUE frt_set_locale(VALUE self, VALUE locale)
+static VALUE frb_set_locale(VALUE self, VALUE locale)
 {
     char *l = ((locale == Qnil) ? NULL : rs2s(rb_obj_as_string(locale)));
-    frt_locale = setlocale(LC_CTYPE, l);
-    return frt_locale ? rb_str_new2(frt_locale) : Qnil;
+    frb_locale = setlocale(LC_CTYPE, l);
+    return frb_locale ? rb_str_new2(frb_locale) : Qnil;
 }
 
 /****************************************************************************
@@ -1645,25 +1732,27 @@ static VALUE frt_set_locale(VALUE self, VALUE locale)
 static void Init_Token(void)
 {
     cToken = rb_define_class_under(mAnalysis, "Token", rb_cObject);
-    rb_define_alloc_func(cToken, frt_token_alloc);
+    rb_define_alloc_func(cToken, frb_token_alloc);
     rb_include_module(cToken, rb_mComparable);
 
-    rb_define_method(cToken, "initialize",  frt_token_init, -1);
-    rb_define_method(cToken, "<=>",         frt_token_cmp, 1);
-    rb_define_method(cToken, "text",        frt_token_get_text, 0);
-    rb_define_method(cToken, "text=",       frt_token_set_text, 1);
-    rb_define_method(cToken, "start",       frt_token_get_start_offset, 0);
-    rb_define_method(cToken, "start=",      frt_token_set_start_offset, 1);
-    rb_define_method(cToken, "end",         frt_token_get_end_offset, 0);
-    rb_define_method(cToken, "end=",        frt_token_set_end_offset, 1);
-    rb_define_method(cToken, "pos_inc",     frt_token_get_pos_inc, 0);
-    rb_define_method(cToken, "pos_inc=",    frt_token_set_pos_inc, 1);
-    rb_define_method(cToken, "to_s",        frt_token_to_s, 0);
+    rb_define_method(cToken, "initialize",  frb_token_init, -1);
+    rb_define_method(cToken, "<=>",         frb_token_cmp, 1);
+    rb_define_method(cToken, "text",        frb_token_get_text, 0);
+    rb_define_method(cToken, "text=",       frb_token_set_text, 1);
+    rb_define_method(cToken, "start",       frb_token_get_start_offset, 0);
+    rb_define_method(cToken, "start=",      frb_token_set_start_offset, 1);
+    rb_define_method(cToken, "end",         frb_token_get_end_offset, 0);
+    rb_define_method(cToken, "end=",        frb_token_set_end_offset, 1);
+    rb_define_method(cToken, "pos_inc",     frb_token_get_pos_inc, 0);
+    rb_define_method(cToken, "pos_inc=",    frb_token_set_pos_inc, 1);
+    rb_define_method(cToken, "to_s",        frb_token_to_s, 0);
 }
 
 /*
  *  Document-class: Ferret::Analysis::TokenStream
  *
+ *  == Summary
+ *
  *  A TokenStream enumerates the sequence of tokens, either from
  *  fields of a document or from query text.
  * 
@@ -1675,15 +1764,17 @@ static void Init_Token(void)
 static void Init_TokenStream(void)
 {
     cTokenStream = rb_define_class_under(mAnalysis, "TokenStream", rb_cObject);
-    frt_mark_cclass(cTokenStream);
-    rb_define_method(cTokenStream, "next", frt_ts_next, 0);
-    rb_define_method(cTokenStream, "text=", frt_ts_set_text, 1);
-    rb_define_method(cTokenStream, "text", frt_ts_get_text, 0);
+    frb_mark_cclass(cTokenStream);
+    rb_define_method(cTokenStream, "next", frb_ts_next, 0);
+    rb_define_method(cTokenStream, "text=", frb_ts_set_text, 1);
+    rb_define_method(cTokenStream, "text", frb_ts_get_text, 0);
 }
 
 /*
  *  Document-class: Ferret::Analysis::AsciiLetterTokenizer
  *
+ *  == Summary
+ *
  *  A LetterTokenizer is a tokenizer that divides text at non-ASCII letters.
  *  That is to say, it defines tokens as maximal strings of adjacent letters,
  *  as defined by the regular expression _/[A-Za-z]+/_.
@@ -1697,15 +1788,17 @@ static void Init_AsciiLetterTokenizer(void)
 {
     cAsciiLetterTokenizer =
         rb_define_class_under(mAnalysis, "AsciiLetterTokenizer", cTokenStream);
-    frt_mark_cclass(cAsciiLetterTokenizer);
-    rb_define_alloc_func(cAsciiLetterTokenizer, frt_data_alloc);
+    frb_mark_cclass(cAsciiLetterTokenizer);
+    rb_define_alloc_func(cAsciiLetterTokenizer, frb_data_alloc);
     rb_define_method(cAsciiLetterTokenizer, "initialize",
-                     frt_a_letter_tokenizer_init, 1);
+                     frb_a_letter_tokenizer_init, 1);
 }
 
 /*
  *  Document-class: Ferret::Analysis::LetterTokenizer
  *
+ *  == Summary
+ *
  *  A LetterTokenizer is a tokenizer that divides text at non-letters. That is
  *  to say, it defines tokens as maximal strings of adjacent letters, as
  *  defined by the regular expression _/[[:alpha:]]+/_ where [:alpha] matches
@@ -1720,15 +1813,17 @@ static void Init_LetterTokenizer(void)
 {
     cLetterTokenizer =
         rb_define_class_under(mAnalysis, "LetterTokenizer", cTokenStream);
-    frt_mark_cclass(cLetterTokenizer);
-    rb_define_alloc_func(cLetterTokenizer, frt_data_alloc);
+    frb_mark_cclass(cLetterTokenizer);
+    rb_define_alloc_func(cLetterTokenizer, frb_data_alloc);
     rb_define_method(cLetterTokenizer, "initialize",
-                     frt_letter_tokenizer_init, -1);
+                     frb_letter_tokenizer_init, -1);
 }
 
 /*
  *  Document-class: Ferret::Analysis::AsciiWhiteSpaceTokenizer
  *
+ *  == Summary
+ *
  *  A WhiteSpaceTokenizer is a tokenizer that divides text at white-space.
  *  Adjacent sequences of non-WhiteSpace characters form tokens.
  *
@@ -1742,15 +1837,17 @@ static void Init_AsciiWhiteSpaceTokenizer(void)
     cAsciiWhiteSpaceTokenizer =
         rb_define_class_under(mAnalysis, "AsciiWhiteSpaceTokenizer",
                               cTokenStream);
-    frt_mark_cclass(cAsciiWhiteSpaceTokenizer);
-    rb_define_alloc_func(cAsciiWhiteSpaceTokenizer, frt_data_alloc);
+    frb_mark_cclass(cAsciiWhiteSpaceTokenizer);
+    rb_define_alloc_func(cAsciiWhiteSpaceTokenizer, frb_data_alloc);
     rb_define_method(cAsciiWhiteSpaceTokenizer, "initialize",
-                     frt_a_whitespace_tokenizer_init, 1);
+                     frb_a_whitespace_tokenizer_init, 1);
 }
 
 /*
  *  Document-class: Ferret::Analysis::WhiteSpaceTokenizer
  *
+ *  == Summary
+ *
  *  A WhiteSpaceTokenizer is a tokenizer that divides text at white-space.
  *  Adjacent sequences of non-WhiteSpace characters form tokens.
  *
@@ -1763,15 +1860,17 @@ static void Init_WhiteSpaceTokenizer(void)
 {
     cWhiteSpaceTokenizer =
         rb_define_class_under(mAnalysis, "WhiteSpaceTokenizer", cTokenStream);
-    frt_mark_cclass(cWhiteSpaceTokenizer);
-    rb_define_alloc_func(cWhiteSpaceTokenizer, frt_data_alloc);
+    frb_mark_cclass(cWhiteSpaceTokenizer);
+    rb_define_alloc_func(cWhiteSpaceTokenizer, frb_data_alloc);
     rb_define_method(cWhiteSpaceTokenizer, "initialize",
-                     frt_whitespace_tokenizer_init, -1);
+                     frb_whitespace_tokenizer_init, -1);
 }
 
 /*
  *  Document-class: Ferret::Analysis::AsciiStandardTokenizer
  *
+ *  == Summary
+ *
  *  The standard tokenizer is an advanced tokenizer which tokenizes most
  *  words correctly as well as tokenizing things like email addresses, web
  *  addresses, phone numbers, etc.
@@ -1785,15 +1884,17 @@ static void Init_AsciiStandardTokenizer(void)
 {
     cAsciiStandardTokenizer =
         rb_define_class_under(mAnalysis, "AsciiStandardTokenizer", cTokenStream);
-    frt_mark_cclass(cAsciiStandardTokenizer);
-    rb_define_alloc_func(cAsciiStandardTokenizer, frt_data_alloc);
+    frb_mark_cclass(cAsciiStandardTokenizer);
+    rb_define_alloc_func(cAsciiStandardTokenizer, frb_data_alloc);
     rb_define_method(cAsciiStandardTokenizer, "initialize",
-                     frt_a_standard_tokenizer_init, 1);
+                     frb_a_standard_tokenizer_init, 1);
 }
 
 /*
  *  Document-class: Ferret::Analysis::StandardTokenizer
  *
+ *  == Summary
+ *
  *  The standard tokenizer is an advanced tokenizer which tokenizes most
  *  words correctly as well as tokenizing things like email addresses, web
  *  addresses, phone numbers, etc.
@@ -1807,15 +1908,17 @@ static void Init_StandardTokenizer(void)
 {
     cStandardTokenizer =
         rb_define_class_under(mAnalysis, "StandardTokenizer", cTokenStream);
-    frt_mark_cclass(cStandardTokenizer);
-    rb_define_alloc_func(cStandardTokenizer, frt_data_alloc);
+    frb_mark_cclass(cStandardTokenizer);
+    rb_define_alloc_func(cStandardTokenizer, frb_data_alloc);
     rb_define_method(cStandardTokenizer, "initialize",
-                     frt_standard_tokenizer_init, 1);
+                     frb_standard_tokenizer_init, 1);
 }
 
 /*
  *  Document-class: Ferret::Analysis::RegExpTokenizer
  *
+ *  == Summary
+ *
  *  A tokenizer that recognizes tokens based on a regular expression passed to
  *  the constructor. Most possible tokenizers can be created using this class.
  *
@@ -1835,14 +1938,14 @@ static void Init_RegExpTokenizer(void)
 {
     cRegExpTokenizer =
         rb_define_class_under(mAnalysis, "RegExpTokenizer", cTokenStream);
-    frt_mark_cclass(cRegExpTokenizer);
+    frb_mark_cclass(cRegExpTokenizer);
     rtoken_re = rb_reg_new(TOKEN_RE, strlen(TOKEN_RE), 0);
     rb_define_const(cRegExpTokenizer, "REGEXP", rtoken_re);
-    rb_define_alloc_func(cRegExpTokenizer, frt_data_alloc);
+    rb_define_alloc_func(cRegExpTokenizer, frb_data_alloc);
     rb_define_method(cRegExpTokenizer, "initialize",
-                     frt_rets_init, -1);
-    rb_define_method(cRegExpTokenizer, "text=", frt_rets_set_text, 1);
-    rb_define_method(cRegExpTokenizer, "text", frt_rets_get_text, 0);
+                     frb_rets_init, -1);
+    rb_define_method(cRegExpTokenizer, "text=", frb_rets_set_text, 1);
+    rb_define_method(cRegExpTokenizer, "text", frb_rets_get_text, 0);
 }
 
 /***************/
@@ -1852,6 +1955,8 @@ static void Init_RegExpTokenizer(void)
 /*
  *  Document-class: Ferret::Analysis::AsciiLowerCaseFilter
  *
+ *  == Summary
+ *
  *  AsciiLowerCaseFilter normalizes a token's text to lowercase but only for
  *  ASCII characters. For other characters use LowerCaseFilter.
  *
@@ -1864,15 +1969,17 @@ static void Init_AsciiLowerCaseFilter(void)
 {
     cAsciiLowerCaseFilter =
         rb_define_class_under(mAnalysis, "AsciiLowerCaseFilter", cTokenStream);
-    frt_mark_cclass(cAsciiLowerCaseFilter);
-    rb_define_alloc_func(cAsciiLowerCaseFilter, frt_data_alloc);
+    frb_mark_cclass(cAsciiLowerCaseFilter);
+    rb_define_alloc_func(cAsciiLowerCaseFilter, frb_data_alloc);
     rb_define_method(cAsciiLowerCaseFilter, "initialize",
-                     frt_a_lowercase_filter_init, 1);
+                     frb_a_lowercase_filter_init, 1);
 }
 
 /*
  *  Document-class: Ferret::Analysis::LowerCaseFilter
  *
+ *  == Summary
+ *
  *  LowerCaseFilter normalizes a token's text to lowercase based on the
  *  current locale.
  *
@@ -1885,15 +1992,17 @@ static void Init_LowerCaseFilter(void)
 {
     cLowerCaseFilter =
         rb_define_class_under(mAnalysis, "LowerCaseFilter", cTokenStream);
-    frt_mark_cclass(cLowerCaseFilter);
-    rb_define_alloc_func(cLowerCaseFilter, frt_data_alloc);
+    frb_mark_cclass(cLowerCaseFilter);
+    rb_define_alloc_func(cLowerCaseFilter, frb_data_alloc);
     rb_define_method(cLowerCaseFilter, "initialize",
-                     frt_lowercase_filter_init, 1);
+                     frb_lowercase_filter_init, 1);
 }
 
 /*
  *  Document-class: Ferret::Analysis::HyphenFilter
  *
+ *  == Summary
+ *
  *  HyphenFilter filters hyphenated words by adding both the word concatenated
  *  into a single word and split into multiple words. ie "e-mail" becomes
  *  "email" and "e mail". This way a search for "e-mail", "email" and "mail"
@@ -1908,14 +2017,16 @@ static void Init_HyphenFilter(void)
 {
     cHyphenFilter =
         rb_define_class_under(mAnalysis, "HyphenFilter", cTokenStream);
-    frt_mark_cclass(cHyphenFilter);
-    rb_define_alloc_func(cHyphenFilter, frt_data_alloc);
-    rb_define_method(cHyphenFilter, "initialize", frt_hyphen_filter_init, 1);
+    frb_mark_cclass(cHyphenFilter);
+    rb_define_alloc_func(cHyphenFilter, frb_data_alloc);
+    rb_define_method(cHyphenFilter, "initialize", frb_hyphen_filter_init, 1);
 }
 
 /*
  *  Document-class: Ferret::Analysis::MappingFilter
  *
+ *  == Summary
+ *
  *  A MappingFilter maps strings in tokens. This is usually used to map UTF-8
  *  characters to ASCII characters for easier searching and better search
  *  recall. The mapping is compiled into a Deterministic Finite Automata so it
@@ -1956,15 +2067,17 @@ static void Init_MappingFilter(void)
 {
     cMappingFilter =
         rb_define_class_under(mAnalysis, "MappingFilter", cTokenStream);
-    frt_mark_cclass(cMappingFilter);
-    rb_define_alloc_func(cMappingFilter, frt_data_alloc);
+    frb_mark_cclass(cMappingFilter);
+    rb_define_alloc_func(cMappingFilter, frb_data_alloc);
     rb_define_method(cMappingFilter, "initialize",
-                     frt_mapping_filter_init, 2);
+                     frb_mapping_filter_init, 2);
 }
 
 /*
  *  Document-class: Ferret::Analysis::StopFilter
  *
+ *  == Summary
+ *
  *  A StopFilter filters *stop-words* from a TokenStream. Stop-words are words
  *  that you don't wish to be index. Usually they will be common words like
  *  "the" and "and" although you can specify whichever words you want.
@@ -1977,10 +2090,10 @@ static void Init_StopFilter(void)
 {
     cStopFilter =
         rb_define_class_under(mAnalysis, "StopFilter", cTokenStream);
-    frt_mark_cclass(cStopFilter);
-    rb_define_alloc_func(cStopFilter, frt_data_alloc);
+    frb_mark_cclass(cStopFilter);
+    rb_define_alloc_func(cStopFilter, frb_data_alloc);
     rb_define_method(cStopFilter, "initialize",
-                     frt_stop_filter_init, -1);
+                     frb_stop_filter_init, -1);
 }
 
 /* 
@@ -2004,13 +2117,25 @@ static void Init_StopFilter(void)
  *     "finnish",    | "fi", "fin"              | "ISO_8859_1", "UTF_8"
  *     "french",     | "fr", "fra", "fre"       | "ISO_8859_1", "UTF_8"
  *     "german",     | "de", "deu", "ge", "ger" | "ISO_8859_1", "UTF_8"
+ *     "hungarian",  | "hu", "hun"              | "ISO_8859_1", "UTF_8"
  *     "italian",    | "it", "ita"              | "ISO_8859_1", "UTF_8"
  *     "norwegian",  | "nl", "no"               | "ISO_8859_1", "UTF_8"
  *     "porter",     |                          | "ISO_8859_1", "UTF_8"
  *     "portuguese", | "por", "pt"              | "ISO_8859_1", "UTF_8"
+ *     "romanian",   | "ro", "ron", "rum"       | "ISO_8859_2", "UTF_8"
  *     "russian",    | "ru", "rus"              | "KOI8_R",     "UTF_8"
  *     "spanish",    | "es", "esl"              | "ISO_8859_1", "UTF_8"
  *     "swedish",    | "sv", "swe"              | "ISO_8859_1", "UTF_8"
+ *     "turkish",    | "tr", "tur"              |               "UTF_8"
+ *
+ *
+ *  === New Stemmers
+ *
+ *  The following stemmers have recently benn added. Please try them out;
+ *
+ *    * Hungarian
+ *    * Romanian
+ *    * Turkish
  *
  *  === Example
  *
@@ -2037,10 +2162,10 @@ static void Init_StemFilter(void)
 {
     cStemFilter =
         rb_define_class_under(mAnalysis, "StemFilter", cTokenStream);
-    frt_mark_cclass(cStemFilter);
-    rb_define_alloc_func(cStemFilter, frt_data_alloc);
+    frb_mark_cclass(cStemFilter);
+    rb_define_alloc_func(cStemFilter, frb_data_alloc);
     rb_define_method(cStemFilter, "initialize",
-                     frt_stem_filter_init, -1);
+                     frb_stem_filter_init, -1);
 }
 
 /*************************/
@@ -2081,10 +2206,10 @@ static void Init_Analyzer(void)
 {
     cAnalyzer =
         rb_define_class_under(mAnalysis, "Analyzer", rb_cObject);
-    frt_mark_cclass(cAnalyzer);
-    rb_define_alloc_func(cAnalyzer, frt_data_alloc);
-    rb_define_method(cAnalyzer, "initialize", frt_letter_analyzer_init, -1);
-    rb_define_method(cAnalyzer, "token_stream", frt_analyzer_token_stream, 2);
+    frb_mark_cclass(cAnalyzer);
+    rb_define_alloc_func(cAnalyzer, frb_data_alloc);
+    rb_define_method(cAnalyzer, "initialize", frb_letter_analyzer_init, -1);
+    rb_define_method(cAnalyzer, "token_stream", frb_analyzer_token_stream, 2);
 }
 
 /*
@@ -2119,10 +2244,10 @@ static void Init_AsciiLetterAnalyzer(void)
 {
     cAsciiLetterAnalyzer =
         rb_define_class_under(mAnalysis, "AsciiLetterAnalyzer", cAnalyzer);
-    frt_mark_cclass(cAsciiLetterAnalyzer);
-    rb_define_alloc_func(cAsciiLetterAnalyzer, frt_data_alloc);
+    frb_mark_cclass(cAsciiLetterAnalyzer);
+    rb_define_alloc_func(cAsciiLetterAnalyzer, frb_data_alloc);
     rb_define_method(cAsciiLetterAnalyzer, "initialize",
-                     frt_a_letter_analyzer_init, -1);
+                     frb_a_letter_analyzer_init, -1);
 }
 
 /*
@@ -2150,10 +2275,10 @@ static void Init_LetterAnalyzer(void)
 {
     cLetterAnalyzer =
         rb_define_class_under(mAnalysis, "LetterAnalyzer", cAnalyzer);
-    frt_mark_cclass(cLetterAnalyzer);
-    rb_define_alloc_func(cLetterAnalyzer, frt_data_alloc);
+    frb_mark_cclass(cLetterAnalyzer);
+    rb_define_alloc_func(cLetterAnalyzer, frb_data_alloc);
     rb_define_method(cLetterAnalyzer, "initialize",
-                     frt_letter_analyzer_init, -1);
+                     frb_letter_analyzer_init, -1);
 }
 
 /*
@@ -2187,10 +2312,10 @@ static void Init_AsciiWhiteSpaceAnalyzer(void)
 {
     cAsciiWhiteSpaceAnalyzer =
         rb_define_class_under(mAnalysis, "AsciiWhiteSpaceAnalyzer", cAnalyzer);
-    frt_mark_cclass(cAsciiWhiteSpaceAnalyzer);
-    rb_define_alloc_func(cAsciiWhiteSpaceAnalyzer, frt_data_alloc);
+    frb_mark_cclass(cAsciiWhiteSpaceAnalyzer);
+    rb_define_alloc_func(cAsciiWhiteSpaceAnalyzer, frb_data_alloc);
     rb_define_method(cAsciiWhiteSpaceAnalyzer, "initialize",
-                     frt_a_white_space_analyzer_init, -1);
+                     frb_a_white_space_analyzer_init, -1);
 }
 
 /*
@@ -2218,10 +2343,10 @@ static void Init_WhiteSpaceAnalyzer(void)
 {
     cWhiteSpaceAnalyzer =
         rb_define_class_under(mAnalysis, "WhiteSpaceAnalyzer", cAnalyzer);
-    frt_mark_cclass(cWhiteSpaceAnalyzer);
-    rb_define_alloc_func(cWhiteSpaceAnalyzer, frt_data_alloc);
+    frb_mark_cclass(cWhiteSpaceAnalyzer);
+    rb_define_alloc_func(cWhiteSpaceAnalyzer, frb_data_alloc);
     rb_define_method(cWhiteSpaceAnalyzer, "initialize",
-                     frt_white_space_analyzer_init, -1);
+                     frb_white_space_analyzer_init, -1);
 }
 
 /*
@@ -2255,10 +2380,10 @@ static void Init_AsciiStandardAnalyzer(void)
 {
     cAsciiStandardAnalyzer =
         rb_define_class_under(mAnalysis, "AsciiStandardAnalyzer", cAnalyzer);
-    frt_mark_cclass(cAsciiStandardAnalyzer);
-    rb_define_alloc_func(cAsciiStandardAnalyzer, frt_data_alloc);
+    frb_mark_cclass(cAsciiStandardAnalyzer);
+    rb_define_alloc_func(cAsciiStandardAnalyzer, frb_data_alloc);
     rb_define_method(cAsciiStandardAnalyzer, "initialize",
-                     frt_a_standard_analyzer_init, -1);
+                     frb_a_standard_analyzer_init, -1);
 }
 
 /*
@@ -2290,10 +2415,10 @@ static void Init_StandardAnalyzer(void)
 {
     cStandardAnalyzer =
         rb_define_class_under(mAnalysis, "StandardAnalyzer", cAnalyzer);
-    frt_mark_cclass(cStandardAnalyzer);
-    rb_define_alloc_func(cStandardAnalyzer, frt_data_alloc);
+    frb_mark_cclass(cStandardAnalyzer);
+    rb_define_alloc_func(cStandardAnalyzer, frb_data_alloc);
     rb_define_method(cStandardAnalyzer, "initialize",
-                     frt_standard_analyzer_init, -1);
+                     frb_standard_analyzer_init, -1);
 }
 
 /*
@@ -2320,16 +2445,16 @@ static void Init_PerFieldAnalyzer(void)
 {
     cPerFieldAnalyzer =
         rb_define_class_under(mAnalysis, "PerFieldAnalyzer", cAnalyzer);
-    frt_mark_cclass(cPerFieldAnalyzer);
-    rb_define_alloc_func(cPerFieldAnalyzer, frt_data_alloc);
+    frb_mark_cclass(cPerFieldAnalyzer);
+    rb_define_alloc_func(cPerFieldAnalyzer, frb_data_alloc);
     rb_define_method(cPerFieldAnalyzer, "initialize",
-                     frt_per_field_analyzer_init, 1);
+                     frb_per_field_analyzer_init, 1);
     rb_define_method(cPerFieldAnalyzer, "add_field",
-                     frt_per_field_analyzer_add_field, 2);
+                     frb_per_field_analyzer_add_field, 2);
     rb_define_method(cPerFieldAnalyzer, "[]=",
-                     frt_per_field_analyzer_add_field, 2);
+                     frb_per_field_analyzer_add_field, 2);
     rb_define_method(cPerFieldAnalyzer, "token_stream",
-                     frt_pfa_analyzer_token_stream, 2);
+                     frb_pfa_analyzer_token_stream, 2);
 }
 
 /*
@@ -2363,12 +2488,12 @@ static void Init_RegExpAnalyzer(void)
 {
     cRegExpAnalyzer =
         rb_define_class_under(mAnalysis, "RegExpAnalyzer", cAnalyzer);
-    frt_mark_cclass(cRegExpAnalyzer);
-    rb_define_alloc_func(cRegExpAnalyzer, frt_data_alloc);
+    frb_mark_cclass(cRegExpAnalyzer);
+    rb_define_alloc_func(cRegExpAnalyzer, frb_data_alloc);
     rb_define_method(cRegExpAnalyzer, "initialize",
-                     frt_re_analyzer_init, -1);
+                     frb_re_analyzer_init, -1);
     rb_define_method(cRegExpAnalyzer, "token_stream",
-                     frt_re_analyzer_token_stream, 2);
+                     frb_re_analyzer_token_stream, 2);
 }
 
 /* rdoc hack
@@ -2433,8 +2558,8 @@ Init_Analysis(void)
     rb_define_const(mFerret, "OBJECT_SPACE", object_space);
 
     /*** * * Locale stuff * * ***/
-    rb_define_singleton_method(mFerret, "locale=", frt_set_locale, 1);
-    rb_define_singleton_method(mFerret, "locale", frt_get_locale, 0);
+    rb_define_singleton_method(mFerret, "locale=", frb_set_locale, 1);
+    rb_define_singleton_method(mFerret, "locale", frb_get_locale, 0);
 
     rb_define_const(mAnalysis, "ENGLISH_STOP_WORDS",
                     get_rstopwords(ENGLISH_STOP_WORDS));
@@ -2464,6 +2589,8 @@ Init_Analysis(void)
                     get_rstopwords(FULL_RUSSIAN_STOP_WORDS));
     rb_define_const(mAnalysis, "FULL_FINNISH_STOP_WORDS",
                     get_rstopwords(FULL_FINNISH_STOP_WORDS));
+    rb_define_const(mAnalysis, "FULL_HUNGARIAN_STOP_WORDS",
+                    get_rstopwords(FULL_HUNGARIAN_STOP_WORDS));
 
     Init_Token();
     Init_TokenStream();