ferret 0.9.0 → 0.9.1

data/ext/q_prefix.c

@@ -17,7 +17,7 @@ char *prq_to_s(Query *self, char *field)
 
   if (strcmp(term->field, field) != 0) {
     sprintf(bptr, "%s:", term->field);
-    bptr++;
+    bptr += strlen(bptr);
   }
   sprintf(bptr, "%s*", term->text);
   if (self->boost != 1.0) {
@@ -38,17 +38,20 @@ Query *prq_rewrite(Query *self, IndexReader *ir)
   Query *tq;
   Query *bq = bq_create(true);
 
-  do { 
-    TermBuffer *tb = te->tb_curr;
-    if (!tb || strcmp(tb->field, prefix_field) != 0 ||
-      strncmp(tb->text, prefix_text, prefix_length) != 0) {
-      break;
-    }
-    tq = tq_create(term_create(tb->field, tb->text)); // found a match
-    tq->boost = self->boost;                          // set the boost
-    bq_add_query(bq, tq, BC_SHOULD);                  // add to query
-  } while (te->next(te));
-  te->close(te);
+  TRY
+    do { 
+      TermBuffer *tb = te->tb_curr;
+      if (!tb || strcmp(tb->field, prefix_field) != 0 ||
+        strncmp(tb->text, prefix_text, prefix_length) != 0) {
+        break;
+      }
+      tq = tq_create(term_create(tb->field, tb->text)); // found a match
+      tq->boost = self->boost;                          // set the boost
+      bq_add_query(bq, tq, BC_SHOULD);                  // add to query
+    } while (te->next(te));
+  XFINALLY
+    te->close(te);
+  XENDTRY
 
   if (self->rewritten) self->rewritten->destroy(self->rewritten);
   return self->rewritten = bq;

data/ext/q_range.c

@@ -1,6 +1,11 @@
 #include <string.h>
 #include "search.h"
 
+static char * const NIL_BOUNDS_ERROR_MSG = "At least one value must be non-nil";
+static char * const LOWER_BOUND_ERROR_MSG = "The lower bound must be non-nil to be inclusive";
+static char * const UPPER_BOUND_ERROR_MSG = "The upper bound must be non-nil to be inclusive";
+static char * const BOUND_ORDER_ERROR_MSG = "The lower bound must less than the upper bound";
+        
 /*****************************************************************************
  *
  * Range
@@ -9,51 +14,50 @@
 
 char *range_to_s(Range *range, char *field, float boost)
 {
-  char *buffer, *bptr;
+  char *buffer, *b;
   int flen, llen, ulen;
 
   flen = strlen(range->field);
   llen = range->lower_term ? strlen(range->lower_term) : 0;
   ulen = range->upper_term ? strlen(range->upper_term) : 0;
   buffer = ALLOC_N(char, flen + llen + ulen + 40);
-  bptr = buffer;
+  b = buffer;
 
   if (strcmp(field, range->field)) {
     memcpy(buffer, range->field, flen * sizeof(char));
-    bptr += flen;
-    *bptr = ':';
-    bptr++;
+    b += flen;
+    *b = ':';
+    b++;
   }
 
   if (range->lower_term) {
-    *bptr = range->include_lower ? '[' : '{';
-    bptr++;
-    memcpy(bptr, range->lower_term, llen);
-    bptr += llen;
+    *b = range->include_lower ? '[' : '{';
+    b++;
+    memcpy(b, range->lower_term, llen);
+    b += llen;
   } else {
-    *bptr = '<';
-    bptr++;
+    *b = '<';
+    b++;
   }
 
   if (range->upper_term && range->lower_term) {
-    *bptr = ' '; bptr++;
+    *b = ' '; b++;
   }
 
   if (range->upper_term) {
-    memcpy(bptr, range->upper_term, ulen);
-    bptr += ulen;
-    *bptr = range->include_upper ? ']' : '}';
-    bptr++;
+    memcpy(b, range->upper_term, ulen);
+    b += ulen;
+    *b = range->include_upper ? ']' : '}';
+    b++;
   } else {
-    *bptr = '>';
-    bptr++;
+    *b = '>';
+    b++;
   }
 
-  *bptr = 0;
+  *b = 0;
   if (boost != 1.0) {
-    char dbuf[32];
-    dbl_to_s(dbuf, boost);
-    sprintf(bptr, "^%s", dbuf);
+    *b = '^';
+    dbl_to_s(b + 1, boost);
   }
   return buffer;
 }
@@ -73,15 +77,13 @@ Range *range_create(const char *field, char *lower_term, char *upper_term,
   Range *range; 
 
   if (!lower_term && !upper_term) 
-    eprintf(ARG_ERROR, "At least one value must be non-nil");
+    RAISE(ARG_ERROR, NIL_BOUNDS_ERROR_MSG);
   if (include_lower && !lower_term) 
-    eprintf(ARG_ERROR, "The lower bound must be non-nil to be inclusive");
+    RAISE(ARG_ERROR, LOWER_BOUND_ERROR_MSG);
   if (include_upper && !upper_term) 
-    eprintf(ARG_ERROR, "The upper bound must be non-nil to be inclusive");
+    RAISE(ARG_ERROR, UPPER_BOUND_ERROR_MSG);
   if (upper_term && lower_term && (strcmp(upper_term, lower_term) < 0))
-    eprintf(ARG_ERROR,
-        "The lower bound must less than the upper bound, %s > %s",
-        upper_term, upper_term);
+    RAISE(ARG_ERROR, BOUND_ORDER_ERROR_MSG);
 
   range = ALLOC(Range);
 

data/ext/q_span.c

@@ -39,21 +39,19 @@ Explanation *spanw_explain(Weight *self, IndexReader *ir, int target)
   }
 
   Explanation *expl = expl_create(0.0,
-      epstrdup("weight(%s in %d), product of:",
-        strlen(query_str) + 20,
-        query_str, target));
+      strfmt("weight(%s in %d), product of:", query_str, target));
 
   /* We need two of these as it's included in both the query explanation
    * and the field explanation */
   Explanation *idf_expl1 = expl_create(self->idf,
-      epstrdup("idf(%s: %s)", strlen(field) + df_i, field, doc_freqs));
+      strfmt("idf(%s: %s)", field, doc_freqs));
   Explanation *idf_expl2 = expl_create(self->idf,
-      epstrdup("idf(%s: %s)", strlen(field) + df_i, field, doc_freqs));
+      strfmt("idf(%s: %s)", field, doc_freqs));
   if (terms->size > 0) free(doc_freqs); /* only free if allocated */
 
   /* explain query weight */
   Explanation *query_expl = expl_create(0.0,
-      epstrdup("query_weight(%s), product of:", strlen(query_str), query_str));
+      strfmt("query_weight(%s), product of:", query_str));
 
   if (self->query->boost != 1.0) {
     expl_add_detail(query_expl, expl_create(self->query->boost, estrdup("boost")));
@@ -70,9 +68,7 @@ Explanation *spanw_explain(Weight *self, IndexReader *ir, int target)
 
   /* explain field weight */
   Explanation *field_expl = expl_create(0.0, 
-      epstrdup("field_weight(%s:%s in %d), product of:",
-        strlen(field) + strlen(query_str) + 20,
-        field, query_str, target));
+      strfmt("field_weight(%s:%s in %d), product of:", field, query_str, target));
   free(query_str);
 
   Scorer *scorer = self->scorer(self, ir);
@@ -84,8 +80,7 @@ Explanation *spanw_explain(Weight *self, IndexReader *ir, int target)
   uchar *field_norms = ir->get_norms(ir, field);
   float field_norm = (field_norms ? sim_decode_norm(self->similarity, field_norms[target]) : 0.0);
   Explanation *field_norm_expl = expl_create(field_norm,
-      epstrdup("field_norm(field=%s, doc=%d)",
-        strlen(field) + 20, field, target));
+      strfmt("field_norm(field=%s, doc=%d)", field, target));
   expl_add_detail(field_expl, field_norm_expl);
 
   field_expl->value = tf_expl->value * idf_expl2->value * field_norm_expl->value;
@@ -103,9 +98,7 @@ Explanation *spanw_explain(Weight *self, IndexReader *ir, int target)
 
 char *spanw_to_s(Weight *self)
 {
-  char dbuf[32];
-  dbl_to_s(dbuf, self->value);
-  return epstrdup("SpanWeight(%s)", strlen(dbuf), dbuf);
+  return strfmt("SpanWeight(%f)", self->value);
 }
 
 void spanw_destroy(void *p)
@@ -317,7 +310,7 @@ char *spanfe_to_s(SpanEnum *self)
 {
   char *field = ((SpanQuery *)self->query->data)->field;
   char *query_str = self->query->to_s(self->query, field);
-  char *res = epstrdup("SpanFirstEnum(%s)", strlen(query_str), query_str);
+  char *res = strfmt("SpanFirstEnum(%s)", query_str);
   free(query_str);
   return res;
 }
@@ -888,7 +881,7 @@ char *spanxe_to_s(SpanEnum *self)
 {
   char *field = ((SpanQuery *)self->query->data)->field;
   char *query_str = self->query->to_s(self->query, field);
-  char *res = epstrdup("SpanNotEnum(%s)", strlen(query_str), query_str);
+  char *res = strfmt("SpanNotEnum(%s)", query_str);
   free(query_str);
   return res;
 }
@@ -954,7 +947,7 @@ char *spantq_to_s(Query *self, char *field)
   } else {
     term_str = term_to_s(term);
   }
-  res = epstrdup("span_term(%s)", strlen(term_str), term_str);
+  res = strfmt("span_term(%s)", term_str);
   free(term_str);
   return res;
 }
@@ -1015,7 +1008,7 @@ char *spanfq_to_s(Query *self, char *field)
   SpanFirstQuery *sfq = (SpanFirstQuery *)((SpanQuery *)self->data)->data;
   Query *match = sfq->match;
   char *q_str = match->to_s(match, field);
-  char *res = epstrdup("span_first(%s, %d)", strlen(q_str) + 20, q_str, sfq->end);
+  char *res = strfmt("span_first(%s, %d)", q_str, sfq->end);
   free(q_str);
   return res;
 }
@@ -1372,8 +1365,7 @@ char *spanxq_to_s(Query *self, char *field)
   SpanNotQuery *sxq = (SpanNotQuery *)((SpanQuery *)self->data)->data;
   char *inc_s = sxq->inc->to_s(sxq->inc, field);
   char *exc_s = sxq->exc->to_s(sxq->exc, field);
-  char *res = epstrdup("span_not(inc:<%s>, exc:<%s>)",
-      strlen(inc_s) + strlen(exc_s), inc_s, exc_s);
+  char *res = strfmt("span_not(inc:<%s>, exc:<%s>)", inc_s, exc_s);
   
   free(inc_s);
   free(exc_s);
@@ -1526,7 +1518,7 @@ Explanation *spansc_explain(Scorer *self, int target)
   phrase_freq = (self->doc == target) ? spansc->freq : 0.0;
 
   Explanation *tf_explanation = expl_create(sim_tf(self->similarity, phrase_freq),
-      epstrdup("tf(phrase_freq(%#.5g)", 32, phrase_freq));
+      strfmt("tf(phrase_freq(%f)", phrase_freq));
 
   return tf_explanation;
 }

data/ext/q_term.c

@@ -24,20 +24,18 @@ Explanation *tw_explain(Weight *self, IndexReader *ir, int doc_num)
   char *field_name = term->field;
 
   Explanation *expl = expl_create(0.0,
-      epstrdup("weight(%s in %d), product of:",
-        strlen(query_str) + 20,
-        query_str, doc_num));
+      strfmt("weight(%s in %d), product of:", query_str, doc_num));
 
   // We need two of these as it's included in both the query explanation
   // and the field explanation
   Explanation *idf_expl1 = expl_create(self->idf,
-      epstrdup("idf(doc_freq=%d)", 20, ir->doc_freq(ir, tq->term)));
+      strfmt("idf(doc_freq=%d)", ir->doc_freq(ir, tq->term)));
   Explanation *idf_expl2 = expl_create(self->idf,
-      epstrdup("idf(doc_freq=%d)", 20, ir->doc_freq(ir, tq->term)));
+      strfmt("idf(doc_freq=%d)", ir->doc_freq(ir, tq->term)));
 
   // explain query weight
   Explanation *query_expl = expl_create(0.0,
-      epstrdup("query_weight(%s), product of:", strlen(query_str), query_str));
+      strfmt("query_weight(%s), product of:", query_str));
   free(query_str);
 
   if (self->query->boost != 1.0) {
@@ -55,8 +53,7 @@ Explanation *tw_explain(Weight *self, IndexReader *ir, int doc_num)
 
   // explain field weight
   Explanation *field_expl = expl_create(0.0, 
-      epstrdup("field_weight(%s:%s in %d), product of:",
-        strlen(field_name) + strlen(term->text) + 20,
+      strfmt("field_weight(%s:%s in %d), product of:",
         field_name, term->text, doc_num));
 
   Scorer *scorer = self->scorer(self, ir);
@@ -68,8 +65,7 @@ Explanation *tw_explain(Weight *self, IndexReader *ir, int doc_num)
   uchar *field_norms = ir->get_norms(ir, field_name);
   float field_norm = (field_norms ? sim_decode_norm(self->similarity, field_norms[doc_num]) : 0.0);
   Explanation *field_norm_expl = expl_create(field_norm,
-      epstrdup("field_norm(field=%s, doc=%d)",
-        strlen(field_name) + 20, field_name, doc_num));
+      strfmt("field_norm(field=%s, doc=%d)", field_name, doc_num));
   expl_add_detail(field_expl, field_norm_expl);
 
   field_expl->value = tf_expl->value * idf_expl2->value * field_norm_expl->value;
@@ -87,9 +83,7 @@ Explanation *tw_explain(Weight *self, IndexReader *ir, int doc_num)
 
 char *tw_to_s(Weight *self)
 {
-  char dbuf[32];
-  dbl_to_s(dbuf, self->value);
-  return epstrdup("TermWeight(%#.5g)", strlen(dbuf), dbuf);
+  return strfmt("TermWeight(%f)", self->value);
 }
 
 void tw_destroy(void *p)
@@ -141,19 +135,18 @@ char *tq_to_s(Query *self, char *field)
   int flen = strlen(term->field);
   int tlen = strlen(term->text);
   char *buffer = ALLOC_N(char, 34 + flen + tlen);
-  char *bp = buffer;
+  char *b = buffer;
   if (strcmp(field, term->field) != 0) {
-    memcpy(bp, term->field, sizeof(char) * flen);
-    bp[flen] = ':';
-    bp += flen + 1;
+    memcpy(b, term->field, sizeof(char) * flen);
+    b[flen] = ':';
+    b += flen + 1;
   }
-  memcpy(bp, term->text, tlen);
-  bp += tlen;
-  *bp = 0;
+  memcpy(b, term->text, tlen);
+  b += tlen;
+  *b = 0;
   if (self->boost != 1.0) {
-    char dbuf[32];
-    dbl_to_s(dbuf, self->boost);
-    sprintf(bp, "^%s", dbuf);
+    *b = '^';
+    dbl_to_s(b+1, self->boost);
   }
   return buffer;
 }
@@ -268,9 +261,7 @@ Explanation *tsc_explain(Scorer *self, int doc_num)
   tde->close(tde);
   ts->tde = NULL;
   Explanation *tf_explanation = expl_create(sim_tf(self->similarity, tf),
-      epstrdup("tf(term_freq(%s:%s)=%ld)",
-        strlen(term->field) + strlen(term->text) + 20,
-        term->field, term->text, tf));
+      strfmt("tf(term_freq(%s:%s)=%d)", term->field, term->text, tf));
 
   return tf_explanation;
 }

data/ext/r_analysis.c

@@ -1,17 +1,62 @@
 #include "ferret.h"
 #include "analysis.h"
+#include "locale.h"
 
 static VALUE cToken;
+static VALUE cAsciiLetterTokenizer;
 static VALUE cLetterTokenizer;
+static VALUE cAsciiWhiteSpaceTokenizer;
+static VALUE cWhiteSpaceTokenizer;
+static VALUE cAsciiStandardTokenizer;
+static VALUE cStandardTokenizer;
+
+static VALUE cAsciiLowerCaseFilter;
+static VALUE cLowerCaseFilter;
+static VALUE cStopFilter;
+static VALUE cStemFilter;
 
 static VALUE cAnalyzer;
+static VALUE cAsciiLetterAnalyzer;
 static VALUE cLetterAnalyzer;
+static VALUE cAsciiWhiteSpaceAnalyzer;
 static VALUE cWhiteSpaceAnalyzer;
+static VALUE cAsciiStandardAnalyzer;
 static VALUE cStandardAnalyzer;
+static VALUE cPerFieldAnalyzer;
+
+//static VALUE cRegexAnalyzer;
+static VALUE cTokenStream;
+
+static ID id_next;
+static ID id_reset;
+static ID id_clone;
 
 /****************************************************************************
  *
- * Token Methods
+ * Utility Methods
+ *
+ ****************************************************************************/
+
+static char **
+get_stopwords(VALUE rstop_words)
+{
+  char **stop_words;
+  int i, len;
+  VALUE rstr;
+  Check_Type(rstop_words, T_ARRAY);
+  len = RARRAY(rstop_words)->len;
+  stop_words = ALLOC_N(char *, RARRAY(rstop_words)->len + 1);
+  stop_words[len] = NULL;
+  for (i = 0; i < len; i++) {
+    rstr = rb_obj_as_string(RARRAY(rstop_words)->ptr[i]);
+    stop_words[i] = RSTRING(rstr)->ptr;
+  }
+  return stop_words;
+}
+
+/****************************************************************************
+ *
+ * token methods
  *
  ****************************************************************************/
 
@@ -41,6 +86,31 @@ frt_token_alloc(VALUE klass)
   return Data_Wrap_Struct(klass, &frt_token_mark, &frt_token_free, ALLOC(RToken));
 }
 
+static VALUE
+get_token(Token *tk)
+{
+  RToken *token = ALLOC(RToken);
+
+  token->text = rb_str_new2(tk->text);
+  token->start = tk->start;
+  token->end = tk->end;
+  token->pos_inc = tk->pos_inc;
+  return Data_Wrap_Struct(cToken, &frt_token_mark, &frt_token_free, token);
+}
+
+Token *
+frt_set_token(Token *tk, VALUE rt)
+{
+  RToken *rtk;
+
+  if (rt == Qnil) return NULL;
+
+  Data_Get_Struct(rt, RToken, rtk);
+  tk_set(tk, RSTRING(rtk->text)->ptr, RSTRING(rtk->text)->len,
+      rtk->start, rtk->end, rtk->pos_inc);
+  return tk;
+}
+
 #define GET_TK RToken *token; Data_Get_Struct(self, RToken, token);
 static VALUE
 frt_token_init(int argc, VALUE *argv, VALUE self) 
@@ -129,23 +199,270 @@ frt_token_to_s(VALUE self)
 
 /****************************************************************************
  *
- * Tokenizer Methods
+ * TokenStream Methods
  *
  ****************************************************************************/
 
 static void
-frt_tokenizer_free(void *p)
+frt_ts_mark(void *p)
 {
   TokenStream *ts = (TokenStream *)p;
-  object_del(p);
+  if (ts->text) frt_gc_mark(&ts->text);
+  if (ts->sub_ts) frt_gc_mark(&ts->sub_ts);
+}
+
+static void
+frt_ts_free(void *p)
+{
+  TokenStream *ts = (TokenStream *)p;
+  if (object_get(&ts->text) != Qnil) object_del(&ts->text);
+  if (ts->sub_ts && (object_get(&ts->sub_ts) != Qnil)) object_del(&ts->sub_ts);
+  object_del(ts);
   ts->destroy(ts);
 }
 
 static VALUE
-frt_letter_tokenizer_init(VALUE self, VALUE rstr) 
+get_token_stream(TokenStream *ts)
+{
+  VALUE rts = object_get(ts);
+  if (rts == Qnil) {
+    rts = Data_Wrap_Struct(cTokenStream, &frt_ts_mark, &frt_ts_free, ts);
+    object_add(ts, rts);
+  }
+  return rts;
+}
+
+static inline VALUE
+get_wrapped_ts(VALUE self, VALUE rstr, TokenStream *ts)
 {
-  TokenStream *ts = letter_tokenizer_create();
-  Frt_Wrap_Struct(self, NULL, &frt_tokenizer_free, ts);
+  rstr = rb_obj_as_string(rstr);
+  ts->reset(ts, RSTRING(rstr)->ptr);
+  Frt_Wrap_Struct(self, &frt_ts_mark, &frt_ts_free, ts);
+  object_add(&ts->text, rstr);
+  object_add(ts, self);
+  return self;
+}
+
+static VALUE
+frt_ts_set_text(VALUE self, VALUE rtext)
+{
+  TokenStream *ts; 
+  Data_Get_Struct(self, TokenStream, ts);
+  rtext = rb_obj_as_string(rtext);
+  ts->reset(ts, RSTRING(rtext)->ptr);
+  object_set(&ts->text, rtext);
+
+  return rtext;
+}
+
+static VALUE
+frt_ts_get_text(VALUE self)
+{
+  VALUE rtext = Qnil;
+  TokenStream *ts; 
+  Data_Get_Struct(self, TokenStream, ts);
+  if (ts->text) {
+    if ((rtext = object_get(&ts->text)) == Qnil) {
+      rtext = rb_str_new2(ts->text);
+      object_set(&ts->text, rtext);
+    } 
+  }
+  return rtext;
+}
+
+static VALUE
+frt_ts_next(VALUE self)
+{
+  TokenStream *ts; 
+  Data_Get_Struct(self, TokenStream, ts);
+  Token *next = ts->next(ts);
+  if (next == NULL) {
+    return Qnil;
+  }
+
+  return get_token(next);
+}
+
+/****************************************************************************
+ * CWrappedTokenStream
+ ****************************************************************************/
+
+void cwrts_destroy(void *p)
+{
+  TokenStream *ts = (TokenStream *)p;
+  free(ts->token);
+  free(ts);
+}
+
+Token *cwrts_next(TokenStream *ts)
+{
+  VALUE rts = (VALUE)ts->data;
+  VALUE rtoken = rb_funcall(rts, id_next, 0);
+  return frt_set_token(ts->token, rtoken);
+}
+
+void cwrts_reset(TokenStream *ts, char *text)
+{
+  VALUE rts = (VALUE)ts->data;
+  ts->t = ts->text = text;
+  rb_funcall(rts, id_reset, 1, rb_str_new2(text));
+}
+
+void cwrts_clone_i(TokenStream *orig_ts, TokenStream *new_ts)
+{
+  VALUE rorig_ts = (VALUE)orig_ts->data;
+  new_ts->data = (void *)rb_funcall(rorig_ts, id_clone, 0);
+}
+
+static TokenStream *
+get_cwrapped_rts(VALUE rts, bool *self_destroy) 
+{
+  TokenStream *ts;
+  switch (TYPE(rts)) {
+    case T_DATA:
+      Data_Get_Struct(rts, TokenStream, ts);
+      *self_destroy = true;
+      break;
+    default:
+      ts = ALLOC(TokenStream);
+      ts->token = ALLOC(Token);
+      ts->data = (void *)rts;
+      ts->next = &cwrts_next;
+      ts->reset = &cwrts_reset;
+      ts->clone_i = &cwrts_clone_i;
+      ts->destroy = &cwrts_destroy;
+      ts->sub_ts = NULL;
+      *self_destroy = false;
+      break;
+  }
+  return ts;
+}
+
+/****************************************************************************
+ * Tokenizers
+ ****************************************************************************/
+
+#define TS_ARGS(dflt) \
+  bool lower;\
+  VALUE rlower, rstr;\
+  rb_scan_args(argc, argv, "11", &rstr, &rlower);\
+  lower = (argc ? RTEST(rlower) : dflt)
+
+static VALUE
+frt_a_letter_tokenizer_init(VALUE self, VALUE rstr) 
+{
+  return get_wrapped_ts(self, rstr, letter_tokenizer_create());
+}
+
+static VALUE
+frt_letter_tokenizer_init(int argc, VALUE *argv, VALUE self) 
+{
+  TS_ARGS(false);
+  return get_wrapped_ts(self, rstr, mb_letter_tokenizer_create(lower));
+}
+
+static VALUE
+frt_a_whitespace_tokenizer_init(VALUE self, VALUE rstr) 
+{
+  return get_wrapped_ts(self, rstr, whitespace_tokenizer_create());
+}
+
+static VALUE
+frt_whitespace_tokenizer_init(int argc, VALUE *argv, VALUE self) 
+{
+  TS_ARGS(false);
+  return get_wrapped_ts(self, rstr, mb_whitespace_tokenizer_create(lower));
+}
+
+static VALUE
+frt_a_standard_tokenizer_init(VALUE self, VALUE rstr) 
+{
+  return get_wrapped_ts(self, rstr, standard_tokenizer_create());
+}
+
+static VALUE
+frt_standard_tokenizer_init(VALUE self, VALUE rstr) 
+{
+  return get_wrapped_ts(self, rstr, mb_standard_tokenizer_create());
+}
+
+/****************************************************************************
+ * Filters
+ ****************************************************************************/
+
+
+static VALUE
+frt_a_lowercase_filter_init(VALUE self, VALUE rsub_ts) 
+{
+  bool self_destroy;
+  TokenStream *ts = lowercase_filter_create(
+      get_cwrapped_rts(rsub_ts, &self_destroy));
+  ts->destroy_sub = !self_destroy;
+  object_add(&ts->sub_ts, rsub_ts);
+
+  Frt_Wrap_Struct(self, &frt_ts_mark, &frt_ts_free, ts);
+  object_add(ts, self);
+  return self;
+}
+
+static VALUE
+frt_lowercase_filter_init(VALUE self, VALUE rsub_ts) 
+{
+  bool self_destroy;
+  TokenStream *ts = mb_lowercase_filter_create(
+      get_cwrapped_rts(rsub_ts, &self_destroy));
+  ts->destroy_sub = !self_destroy;
+  object_add(&ts->sub_ts, rsub_ts);
+
+  Frt_Wrap_Struct(self, &frt_ts_mark, &frt_ts_free, ts);
+  object_add(ts, self);
+  return self;
+}
+
+static VALUE
+frt_stop_filter_init(int argc, VALUE *argv, VALUE self) 
+{
+  VALUE rsub_ts, rstop_words;
+  bool self_destroy;
+  TokenStream *ts;
+  rb_scan_args(argc, argv, "11", &rsub_ts, &rstop_words);
+  if (rstop_words != Qnil) {
+    char **stop_words = get_stopwords(rstop_words);
+    ts = stop_filter_create_with_words(
+        get_cwrapped_rts(rsub_ts, &self_destroy), (const char **)stop_words);
+    free(stop_words);
+  } else {
+    ts = stop_filter_create(
+        get_cwrapped_rts(rsub_ts, &self_destroy));
+  }
+  ts->destroy_sub = !self_destroy;
+  object_add(&ts->sub_ts, rsub_ts);
+
+  Frt_Wrap_Struct(self, &frt_ts_mark, &frt_ts_free, ts);
+  object_add(ts, self);
+  return self;
+}
+
+static VALUE
+frt_stem_filter_init(int argc, VALUE *argv, VALUE self) 
+{
+  VALUE rsub_ts, ralgorithm, rcharenc;
+  char *algorithm = "english";
+  char *charenc = NULL;
+  bool self_destroy;
+  TokenStream *ts;
+  rb_scan_args(argc, argv, "12", &rsub_ts, &ralgorithm, &rcharenc);
+  switch (argc) {
+    case 3: charenc = RSTRING(rb_obj_as_string(rcharenc))->ptr;
+    case 2: algorithm = RSTRING(rb_obj_as_string(ralgorithm))->ptr;
+  }
+  ts = stem_filter_create(
+      get_cwrapped_rts(rsub_ts, &self_destroy), algorithm, charenc);
+  ts->destroy_sub = !self_destroy;
+  object_add(&ts->sub_ts, rsub_ts);
+
+  Frt_Wrap_Struct(self, &frt_ts_mark, &frt_ts_free, ts);
+  object_add(ts, self);
   return self;
 }
 
@@ -155,6 +472,28 @@ frt_letter_tokenizer_init(VALUE self, VALUE rstr)
  *
  ****************************************************************************/
 
+Analyzer *get_cwrapped_analyzer(ranalyzer)
+{
+  Analyzer *a = NULL;
+  switch (TYPE(ranalyzer)) {
+    case T_DATA:
+      Data_Get_Struct(ranalyzer, Analyzer, a);
+      break;
+    default:
+      printf("Oh RFuck\n");
+      //ts = ALLOC(TokenStream);
+      //ts->token = ALLOC(Token);
+      //ts->data = (void *)rts;
+      //ts->next = &cwrts_next;
+      //ts->reset = &cwrts_reset;
+      //ts->clone_i = &cwrts_clone_i;
+      //ts->destroy = &cwrts_destroy;
+      //ts->sub_ts = NULL;
+      break;
+  }
+  return a;
+}
+
 static void
 frt_analyzer_free(void *p)
 {
@@ -171,11 +510,53 @@ frt_get_analyzer(Analyzer *a)
   return self;
 }
 
+static VALUE
+frt_analyzer_token_stream(VALUE self, VALUE rfield, VALUE rstring)
+{
+  Analyzer *a = ((struct RData *)(self))->data;
+  rfield = rb_obj_as_string(rfield);
+  rstring = rb_obj_as_string(rstring);
+  
+  TokenStream *ts = a_get_new_ts(a, RSTRING(rfield)->ptr, RSTRING(rstring)->ptr);
+
+  object_set(&ts->text, rstring); // Make sure that there is no entry already
+  return get_token_stream(ts);
+}
+
+#define GET_LOWER(dflt) \
+  bool lower;\
+  VALUE rlower;\
+  rb_scan_args(argc, argv, "01", &rlower);\
+  lower = (argc ? RTEST(rlower) : dflt)
+
+/*** AsciiWhiteSpaceAnalyzer ***/
+static VALUE
+frt_a_white_space_analyzer_init(int argc, VALUE *argv, VALUE self)
+{
+  GET_LOWER(false);
+  Analyzer *a = whitespace_analyzer_create(lower);
+  Frt_Wrap_Struct(self, NULL, &frt_analyzer_free, a);
+  object_add(a, self);
+  return self;
+}
+
 /*** WhiteSpaceAnalyzer ***/
 static VALUE
-frt_white_space_analyzer_init(VALUE self)
+frt_white_space_analyzer_init(int argc, VALUE *argv, VALUE self)
+{
+  GET_LOWER(false);
+  Analyzer *a = mb_whitespace_analyzer_create(lower);
+  Frt_Wrap_Struct(self, NULL, &frt_analyzer_free, a);
+  object_add(a, self);
+  return self;
+}
+
+/*** AsciiLetterAnalyzer ***/
+static VALUE
+frt_a_letter_analyzer_init(int argc, VALUE *argv, VALUE self)
 {
-  Analyzer *a = whitespace_analyzer_create();
+  GET_LOWER(true);
+  Analyzer *a = letter_analyzer_create(lower);
   Frt_Wrap_Struct(self, NULL, &frt_analyzer_free, a);
   object_add(a, self);
   return self;
@@ -183,9 +564,44 @@ frt_white_space_analyzer_init(VALUE self)
 
 /*** LetterAnalyzer ***/
 static VALUE
-frt_letter_analyzer_init(VALUE self)
+frt_letter_analyzer_init(int argc, VALUE *argv, VALUE self)
 {
-  Analyzer *a = letter_analyzer_create();
+  GET_LOWER(true);
+  Analyzer *a = mb_letter_analyzer_create(lower);
+  Frt_Wrap_Struct(self, NULL, &frt_analyzer_free, a);
+  object_add(a, self);
+  return self;
+}
+
+static VALUE
+get_rstopwords(const char **stop_words)
+{
+  char **w = (char **)stop_words;
+  VALUE rstopwords = rb_ary_new();
+
+  while (*w) {
+    rb_ary_push(rstopwords, rb_str_new2(*w));
+    w++;
+  }
+  return rstopwords;
+}
+
+/*** AsciiStandardAnalyzer ***/
+static VALUE
+frt_a_standard_analyzer_init(int argc, VALUE *argv, VALUE self)
+{
+  bool lower;
+  VALUE rlower, rstop_words;
+  Analyzer *a;
+  rb_scan_args(argc, argv, "02", &rlower, &rstop_words);
+  lower = ((rlower == Qnil) ? true : RTEST(rlower));
+  if (rstop_words != Qnil) {
+    char **stop_words = get_stopwords(rstop_words);
+    a = standard_analyzer_create_with_words((const char **)stop_words, lower);
+    free(stop_words);
+  } else {
+    a = standard_analyzer_create(lower);
+  }
   Frt_Wrap_Struct(self, NULL, &frt_analyzer_free, a);
   object_add(a, self);
   return self;
@@ -193,14 +609,98 @@ frt_letter_analyzer_init(VALUE self)
 
 /*** StandardAnalyzer ***/
 static VALUE
-frt_standard_analyzer_init(VALUE self)
+frt_standard_analyzer_init(int argc, VALUE *argv, VALUE self)
 {
-  Analyzer *a = standard_analyzer_create();
+  bool lower;
+  VALUE rlower, rstop_words;
+  Analyzer *a;
+  rb_scan_args(argc, argv, "02", &rstop_words, &rlower);
+  lower = ((rlower == Qnil) ? true : RTEST(rlower));
+  if (rstop_words != Qnil) {
+    char **stop_words = get_stopwords(rstop_words);
+    a = mb_standard_analyzer_create_with_words((const char **)stop_words, lower);
+    free(stop_words);
+  } else {
+    a = mb_standard_analyzer_create(lower);
+  }
   Frt_Wrap_Struct(self, NULL, &frt_analyzer_free, a);
   object_add(a, self);
   return self;
 }
 
+/*** PerFieldAnalyzer ***/
+static VALUE
+frt_per_field_analyzer_init(VALUE self, VALUE ranalyzer)
+{
+  Analyzer *def = get_cwrapped_analyzer(ranalyzer);
+  Analyzer *a = per_field_analyzer_create(def, false);
+  Frt_Wrap_Struct(self, NULL, &frt_analyzer_free, a);
+  object_add(a, self);
+  return self;
+}
+
+static VALUE
+frt_per_field_analyzer_add_field(VALUE self, VALUE rfield, VALUE ranalyzer)
+{
+  Analyzer *pfa, *a;
+  Data_Get_Struct(self, Analyzer, pfa);
+  Data_Get_Struct(ranalyzer, Analyzer, a);
+
+  pfa_add_field(pfa, StringValuePtr(rfield), a);
+  return self;
+}
+
+
+/** RegexAnalyzer **/
+/*
+static VALUE
+frt_regex_analyzer_init(VALUE self)
+{
+  Analyzer *a = regex_analyzer_create();
+  // keine Ahnung warum hier das Makro und nicht Data_Wrap_Struct:
+  Frt_Wrap_Struct(self, NULL, &frt_analyzer_free, a);
+  // wofuer?:
+  object_add(a, self);
+  return self;
+}
+
+// convenience method
+// XXX this sets the locale for the entire program
+static VALUE
+frt_regex_analyzer_token_stream(VALUE self, VALUE field, VALUE string)
+{
+  Analyzer *a =((struct RData *)(self))->data;
+  TokenStream *ts = a->get_ts( a, StringValuePtr(field), StringValuePtr(string) );
+  // already freed via analyzer's free()
+  VALUE token_stream = Data_Wrap_Struct(cTokenStream, NULL, NULL, ts);
+  return token_stream;
+}
+*/
+/** /RegexAnalyzer **/
+
+/** TokenStream **/
+/** /TokenStream **/
+
+/****************************************************************************
+ *
+ * Locale stuff
+ *
+ ****************************************************************************/
+
+static char *frt_locale = NULL;
+
+static VALUE frt_getlocale(VALUE self, VALUE locale)
+{
+  return (frt_locale ? rb_str_new2(frt_locale) : Qnil);
+}
+
+static VALUE frt_setlocale(VALUE self, VALUE locale)
+{
+  char *l = ((locale == Qnil) ? NULL : RSTRING(rb_obj_as_string(locale))->ptr);
+  frt_locale = setlocale(LC_ALL, l);
+  return frt_locale ? rb_str_new2(frt_locale) : Qnil;
+}
+
 /****************************************************************************
  *
  * Init Function
@@ -210,6 +710,18 @@ frt_standard_analyzer_init(VALUE self)
 void
 Init_analysis(void)
 {
+	id_next = rb_intern("next");
+	id_reset = rb_intern("text=");
+	id_clone = rb_intern("clone");
+
+  /*** * * Locale stuff * * ***/
+  frt_locale = setlocale(LC_ALL, "");
+  rb_define_singleton_method(mFerret, "locale=", frt_setlocale, 1);
+  rb_define_singleton_method(mFerret, "locale", frt_getlocale, 0);
+
+  /*********************/
+  /*** * * Token * * ***/
+  /*********************/
   cToken = rb_define_class_under(mAnalysis, "Token", rb_cObject);
   rb_define_alloc_func(cToken, frt_token_alloc);
   rb_include_module(cToken, rb_mComparable);
@@ -223,33 +735,193 @@ Init_analysis(void)
   rb_define_method(cToken, "pos_inc", frt_token_get_pos_inc, 0);
   rb_define_method(cToken, "to_s", frt_token_to_s, 0);
 
+  /****************************/
+  /*** * * TokenStreams * * ***/
+  /****************************/
+
+  cTokenStream = rb_define_class_under(mAnalysis, "TokenStream", rb_cObject);
+  rb_define_method(cTokenStream, "next", frt_ts_next, 0);
+  rb_define_method(cTokenStream, "text=", frt_ts_set_text, 1);
+  rb_define_method(cTokenStream, "text", frt_ts_get_text, 0);
+
+  /******************/
+  /*** Tokenizers ***/
+  /******************/
+
+  /*** * * AsciiLetterTokenizer * * ***/
+  cAsciiLetterTokenizer =
+    rb_define_class_under(mAnalysis, "AsciiLetterTokenizer", cTokenStream);
+  rb_define_alloc_func(cAsciiLetterTokenizer, frt_data_alloc);
+  rb_define_method(cAsciiLetterTokenizer, "initialize",
+      frt_a_letter_tokenizer_init, 1);
+
+  /*** * * LetterTokenizer * * ***/
   cLetterTokenizer =
-    rb_define_class_under(mAnalysis, "LetterTokenizer", rb_cObject);
+    rb_define_class_under(mAnalysis, "LetterTokenizer", cTokenStream);
   rb_define_alloc_func(cLetterTokenizer, frt_data_alloc);
   rb_define_method(cLetterTokenizer, "initialize",
-      frt_letter_tokenizer_init, 1);
+      frt_letter_tokenizer_init, -1);
+
+  /*** * * AsciiWhiteSpaceTokenizer * * ***/
+  cAsciiWhiteSpaceTokenizer =
+    rb_define_class_under(mAnalysis, "AsciiWhiteSpaceTokenizer", cTokenStream);
+  rb_define_alloc_func(cAsciiWhiteSpaceTokenizer, frt_data_alloc);
+  rb_define_method(cAsciiWhiteSpaceTokenizer, "initialize",
+      frt_a_whitespace_tokenizer_init, 1);
+
+  /*** * * WhiteSpaceTokenizer * * ***/
+  cWhiteSpaceTokenizer =
+    rb_define_class_under(mAnalysis, "WhiteSpaceTokenizer", cTokenStream);
+  rb_define_alloc_func(cWhiteSpaceTokenizer, frt_data_alloc);
+  rb_define_method(cWhiteSpaceTokenizer, "initialize",
+      frt_whitespace_tokenizer_init, -1);
+
+  /*** * * AsciiStandardTokenizer * * ***/
+  cAsciiStandardTokenizer =
+    rb_define_class_under(mAnalysis, "AsciiStandardTokenizer", cTokenStream);
+  rb_define_alloc_func(cAsciiStandardTokenizer, frt_data_alloc);
+  rb_define_method(cAsciiStandardTokenizer, "initialize",
+      frt_a_standard_tokenizer_init, 1);
 
+  /*** * * StandardTokenizer * * ***/
+  cStandardTokenizer =
+    rb_define_class_under(mAnalysis, "StandardTokenizer", cTokenStream);
+  rb_define_alloc_func(cStandardTokenizer, frt_data_alloc);
+  rb_define_method(cStandardTokenizer, "initialize",
+      frt_standard_tokenizer_init, 1);
+
+  /***************/
+  /*** Filters ***/
+  /***************/
+  rb_define_const(mAnalysis, "ENGLISH_STOP_WORDS",
+      get_rstopwords(ENGLISH_STOP_WORDS));
+  rb_define_const(mAnalysis, "FULL_ENGLISH_STOP_WORDS",
+      get_rstopwords(FULL_ENGLISH_STOP_WORDS));
+  rb_define_const(mAnalysis, "EXTENDED_ENGLISH_STOP_WORDS",
+      get_rstopwords(EXTENDED_ENGLISH_STOP_WORDS));
+  rb_define_const(mAnalysis, "FULL_FRENCH_STOP_WORDS",
+      get_rstopwords(FULL_FRENCH_STOP_WORDS));
+  rb_define_const(mAnalysis, "FULL_SPANISH_STOP_WORDS",
+      get_rstopwords(FULL_SPANISH_STOP_WORDS));
+  rb_define_const(mAnalysis, "FULL_PORTUGUESE_STOP_WORDS",
+      get_rstopwords(FULL_PORTUGUESE_STOP_WORDS));
+  rb_define_const(mAnalysis, "FULL_ITALIAN_STOP_WORDS",
+      get_rstopwords(FULL_ITALIAN_STOP_WORDS));
+  rb_define_const(mAnalysis, "FULL_GERMAN_STOP_WORDS",
+      get_rstopwords(FULL_GERMAN_STOP_WORDS));
+  rb_define_const(mAnalysis, "FULL_DUTCH_STOP_WORDS",
+      get_rstopwords(FULL_DUTCH_STOP_WORDS));
+  rb_define_const(mAnalysis, "FULL_SWEDISH_STOP_WORDS",
+      get_rstopwords(FULL_SWEDISH_STOP_WORDS));
+  rb_define_const(mAnalysis, "FULL_NORWEGIAN_STOP_WORDS",
+      get_rstopwords(FULL_NORWEGIAN_STOP_WORDS));
+  rb_define_const(mAnalysis, "FULL_DANISH_STOP_WORDS",
+      get_rstopwords(FULL_DANISH_STOP_WORDS));
+  rb_define_const(mAnalysis, "FULL_RUSSIAN_STOP_WORDS",
+      get_rstopwords(FULL_RUSSIAN_STOP_WORDS));
+  rb_define_const(mAnalysis, "FULL_FINNISH_STOP_WORDS",
+      get_rstopwords(FULL_FINNISH_STOP_WORDS));
+  
+  cAsciiLowerCaseFilter =
+    rb_define_class_under(mAnalysis, "AsciiLowerCaseFilter", cTokenStream);
+  rb_define_alloc_func(cAsciiLowerCaseFilter, frt_data_alloc);
+  rb_define_method(cAsciiLowerCaseFilter, "initialize",
+      frt_a_lowercase_filter_init, 1);
+
+  cLowerCaseFilter =
+    rb_define_class_under(mAnalysis, "LowerCaseFilter", cTokenStream);
+  rb_define_alloc_func(cLowerCaseFilter, frt_data_alloc);
+  rb_define_method(cLowerCaseFilter, "initialize",
+      frt_lowercase_filter_init, 1);
+
+  cStopFilter =
+    rb_define_class_under(mAnalysis, "StopFilter", cTokenStream);
+  rb_define_alloc_func(cStopFilter, frt_data_alloc);
+  rb_define_method(cStopFilter, "initialize",
+      frt_stop_filter_init, -1);
+
+  cStemFilter =
+    rb_define_class_under(mAnalysis, "StemFilter", cTokenStream);
+  rb_define_alloc_func(cStemFilter, frt_data_alloc);
+  rb_define_method(cStemFilter, "initialize",
+      frt_stem_filter_init, -1);
+
+
+  /*************************/
+  /*** * * Analyzers * * ***/
+  /*************************/
+
+  /*** * * Analyzer * * ***/
   cAnalyzer =
     rb_define_class_under(mAnalysis, "Analyzer", rb_cObject);
   rb_define_alloc_func(cAnalyzer, frt_data_alloc);
-  rb_define_method(cAnalyzer, "initialize",
-      frt_letter_analyzer_init, 0);
+  rb_define_method(cAnalyzer, "initialize", frt_letter_analyzer_init, -1);
+  rb_define_method(cAnalyzer, "token_stream", frt_analyzer_token_stream, 2);
+
+  /*** * * AsciiLetterAnalyzer * * ***/
+  cAsciiLetterAnalyzer =
+    rb_define_class_under(mAnalysis, "AsciiLetterAnalyzer", cAnalyzer);
+  rb_define_alloc_func(cAsciiLetterAnalyzer, frt_data_alloc);
+  rb_define_method(cAsciiLetterAnalyzer, "initialize",
+      frt_a_letter_analyzer_init, -1);
 
+  /*** * * LetterAnalyzer * * ***/
   cLetterAnalyzer =
     rb_define_class_under(mAnalysis, "LetterAnalyzer", cAnalyzer);
   rb_define_alloc_func(cLetterAnalyzer, frt_data_alloc);
-  rb_define_method(cAnalyzer, "initialize",
-      frt_letter_analyzer_init, 0);
+  rb_define_method(cLetterAnalyzer, "initialize",
+        frt_letter_analyzer_init, -1);
 
+  /*** * * AsciiWhiteSpaceAnalyzer * * ***/
+  cAsciiWhiteSpaceAnalyzer =
+    rb_define_class_under(mAnalysis, "AsciiWhiteSpaceAnalyzer", cAnalyzer);
+  rb_define_alloc_func(cAsciiWhiteSpaceAnalyzer, frt_data_alloc);
+  rb_define_method(cAsciiWhiteSpaceAnalyzer, "initialize",
+      frt_a_white_space_analyzer_init, -1);
+
+  /*** * * WhiteSpaceAnalyzer * * ***/
   cWhiteSpaceAnalyzer =
     rb_define_class_under(mAnalysis, "WhiteSpaceAnalyzer", cAnalyzer);
   rb_define_alloc_func(cWhiteSpaceAnalyzer, frt_data_alloc);
   rb_define_method(cWhiteSpaceAnalyzer, "initialize",
-      frt_white_space_analyzer_init, 0);
+      frt_white_space_analyzer_init, -1);
+
+  /*** * * AsciiStandardAnalyzer * * ***/
+  cAsciiStandardAnalyzer =
+    rb_define_class_under(mAnalysis, "AsciiStandardAnalyzer", cAnalyzer);
+  rb_define_alloc_func(cAsciiStandardAnalyzer, frt_data_alloc);
+  rb_define_method(cAsciiStandardAnalyzer, "initialize",
+      frt_a_standard_analyzer_init, -1);
 
+  /*** * * StandardAnalyzer * * ***/
   cStandardAnalyzer =
     rb_define_class_under(mAnalysis, "StandardAnalyzer", cAnalyzer);
   rb_define_alloc_func(cStandardAnalyzer, frt_data_alloc);
   rb_define_method(cStandardAnalyzer, "initialize",
-      frt_standard_analyzer_init, 0);
+      frt_standard_analyzer_init, -1);
+
+  /*** * * PerFieldAnalyzer * * ***/
+  cPerFieldAnalyzer =
+    rb_define_class_under(mAnalysis, "PerFieldAnalyzer", cAnalyzer);
+  rb_define_alloc_func(cPerFieldAnalyzer, frt_data_alloc);
+  rb_define_method(cPerFieldAnalyzer, "initialize",
+      frt_per_field_analyzer_init, 1);
+  rb_define_method(cPerFieldAnalyzer, "add_field",
+      frt_per_field_analyzer_add_field, 2);
+  rb_define_method(cPerFieldAnalyzer, "[]=",
+      frt_per_field_analyzer_add_field, 2);
+
+  /** RegexAnalyzer **/
+  /*
+  cRegexAnalyzer =
+    rb_define_class_under(mAnalysis, "RegexAnalyzer", cAnalyzer);
+  rb_define_alloc_func(cRegexAnalyzer, frt_data_alloc);
+  rb_define_method(cRegexAnalyzer, "initialize",
+      frt_regex_analyzer_init, 0);
+  rb_define_method(cRegexAnalyzer, "token_stream",
+      frt_regex_analyzer_token_stream, 2);
+  rb_define_method(cRegexAnalyzer, "setlocale",
+      frt_regex_analyzer_setlocale, 1);
+  */
+
 }