ferret 0.9.1 → 0.9.2

data/ext/q_term.c

@@ -18,6 +18,14 @@ Scorer *tw_scorer(Weight *self, IndexReader *ir)
 
 Explanation *tw_explain(Weight *self, IndexReader *ir, int doc_num)
 {
+  Explanation *qnorm_expl;
+  Explanation *field_expl;
+  Scorer *scorer;
+  Explanation *tf_expl;
+  uchar *field_norms;
+  float field_norm;
+  Explanation *field_norm_expl;
+
   char *query_str = self->query->to_s(self->query, "");
   TermQuery *tq = (TermQuery *)self->query->data;
   Term *term = tq->term;
@@ -26,14 +34,14 @@ Explanation *tw_explain(Weight *self, IndexReader *ir, int doc_num)
   Explanation *expl = expl_create(0.0,
       strfmt("weight(%s in %d), product of:", query_str, doc_num));
 
-  // We need two of these as it's included in both the query explanation
-  // and the field explanation
+  /* We need two of these as it's included in both the query explanation
+   * and the field explanation */
   Explanation *idf_expl1 = expl_create(self->idf,
       strfmt("idf(doc_freq=%d)", ir->doc_freq(ir, tq->term)));
   Explanation *idf_expl2 = expl_create(self->idf,
       strfmt("idf(doc_freq=%d)", ir->doc_freq(ir, tq->term)));
 
-  // explain query weight
+  /* explain query weight */
   Explanation *query_expl = expl_create(0.0,
       strfmt("query_weight(%s), product of:", query_str));
   free(query_str);
@@ -44,33 +52,35 @@ Explanation *tw_explain(Weight *self, IndexReader *ir, int doc_num)
 
   expl_add_detail(query_expl, idf_expl1);
 
-  Explanation *qnorm_expl = expl_create(self->qnorm, estrdup("query_norm"));
+  qnorm_expl = expl_create(self->qnorm, estrdup("query_norm"));
   expl_add_detail(query_expl, qnorm_expl);
 
   query_expl->value = self->query->boost * idf_expl1->value * qnorm_expl->value;
 
   expl_add_detail(expl, query_expl);
 
-  // explain field weight
-  Explanation *field_expl = expl_create(0.0, 
+  /* explain field weight */
+  field_expl = expl_create(0.0, 
       strfmt("field_weight(%s:%s in %d), product of:",
         field_name, term->text, doc_num));
 
-  Scorer *scorer = self->scorer(self, ir);
-  Explanation *tf_expl = scorer->explain(scorer, doc_num);
+  scorer = self->scorer(self, ir);
+  tf_expl = scorer->explain(scorer, doc_num);
   scorer->destroy(scorer);
   expl_add_detail(field_expl, tf_expl);
   expl_add_detail(field_expl, idf_expl2);
 
-  uchar *field_norms = ir->get_norms(ir, field_name);
-  float field_norm = (field_norms ? sim_decode_norm(self->similarity, field_norms[doc_num]) : 0.0);
-  Explanation *field_norm_expl = expl_create(field_norm,
+  field_norms = ir->get_norms(ir, field_name);
+  field_norm = (field_norms 
+    ? sim_decode_norm(self->similarity, field_norms[doc_num]) 
+    : (float)0.0);
+  field_norm_expl = expl_create(field_norm,
       strfmt("field_norm(field=%s, doc=%d)", field_name, doc_num));
   expl_add_detail(field_expl, field_norm_expl);
 
   field_expl->value = tf_expl->value * idf_expl2->value * field_norm_expl->value;
 
-  // combine them
+  /* combine them */
   if (query_expl->value == 1.0) {
     expl_destoy(expl);
     return field_expl;
@@ -86,30 +96,18 @@ char *tw_to_s(Weight *self)
   return strfmt("TermWeight(%f)", self->value);
 }
 
-void tw_destroy(void *p)
-{
-  free(p);
-}
-
 Weight *tw_create(Query *query, Searcher *searcher)
 {
-  Weight *self = ALLOC(Weight);
-  ZEROSET(self, Weight, 1);
-  self->get_query = &w_get_query;
-  self->get_value = &w_get_value;
-  self->normalize = &w_normalize;
+  Weight *self = w_create(query);
   self->scorer    = &tw_scorer;
   self->explain   = &tw_explain;
   self->to_s      = &tw_to_s;
-  self->destroy   = &tw_destroy;
   self->sum_of_squared_weights = &w_sum_of_squared_weights;
 
   self->similarity = query->get_similarity(query, searcher);
   self->idf = sim_idf(self->similarity,
       searcher->doc_freq(searcher, ((TermQuery *)query->data)->term),
       searcher->max_doc(searcher)); // compute idf
-  self->query = query;
-  self->value = 0.0;
 
   return self;
 }
@@ -120,20 +118,19 @@ Weight *tw_create(Query *query, Searcher *searcher)
  *
  ***************************************************************************/
 
-void tq_destroy(void *p)
+void tq_destroy(Query *self)
 {
-  Query *q = (Query *)p;
-  TermQuery *tq = q->data;
+  TermQuery *tq = self->data;
   term_destroy(tq->term);
   free(tq);
-  q_destroy(q);
+  q_destroy_i(self);
 }
 
 char *tq_to_s(Query *self, char *field)
 {
   Term *term = ((TermQuery *)self->data)->term;
-  int flen = strlen(term->field);
-  int tlen = strlen(term->text);
+  size_t flen = strlen(term->field);
+  size_t tlen = strlen(term->text);
   char *buffer = ALLOC_N(char, 34 + flen + tlen);
   char *b = buffer;
   if (strcmp(field, term->field) != 0) {
@@ -151,10 +148,21 @@ char *tq_to_s(Query *self, char *field)
   return buffer;
 }
 
-void tq_extract_terms(Query *self, Array *terms)
+static void tq_extract_terms(Query *self, HashSet *terms)
 {
   Term *term = ((TermQuery *)self->data)->term;
-  ary_append(terms, term);
+  hs_add(terms, term_clone(term));
+}
+
+static uint tq_hash(Query *self)
+{
+  return term_hash(((TermQuery *)self->data)->term);
+}
+
+static int tq_eq(Query *self, Query *o)
+{
+  return term_eq(((TermQuery *)self->data)->term,
+              ((TermQuery *)o->data)->term);
 }
 
 Query *tq_create(Term *term)
@@ -164,14 +172,18 @@ Query *tq_create(Term *term)
   tq->term = term;
   self->type = TERM_QUERY;
   self->data = tq;
-  self->create_weight = &tw_create;
   self->extract_terms = &tq_extract_terms;
   self->to_s = &tq_to_s;
-  self->destroy = &tq_destroy;
+  self->hash = &tq_hash;
+  self->eq = &tq_eq;
+
+  self->destroy_i = &tq_destroy;
+  self->create_weight_i = &tw_create;
 
   return self;
 }
 
+
 /***************************************************************************
  *
  * TermScorer
@@ -183,13 +195,13 @@ float tsc_score(Scorer *self)
   TermScorer *ts = (TermScorer *)self->data;
   int freq = ts->freqs[ts->pointer];
   float score;
-  // compute tf(f)*weight
-  if (freq < SCORE_CACHE_SIZE) {    // check cache
-    score = ts->score_cache[freq];  // cache hit
+  /* compute tf(f)*weight */
+  if (freq < SCORE_CACHE_SIZE) {    /* check cache */
+    score = ts->score_cache[freq];  /* cache hit */
   } else {
-    score = sim_tf(self->similarity, freq) * ts->weight_value; // cache miss
+    score = sim_tf(self->similarity, (float)freq) * ts->weight_value; /* cache miss */
   }
-  // normalize for field
+  /* normalize for field */
   score *= sim_decode_norm(self->similarity, ts->norms[self->doc]);
   return score;
 }
@@ -217,8 +229,9 @@ bool tsc_next(Scorer *self)
 bool tsc_skip_to(Scorer *self, int doc_num)
 {
   TermScorer *ts = (TermScorer *)self->data;
-
-  // first scan in cache
+  TermDocEnum *tde = ts->tde;
+  
+  /* first scan in cache */
   while (++(ts->pointer) < ts->pointer_max) {
     if (ts->docs[ts->pointer] >= doc_num) {
       self->doc = ts->docs[ts->pointer];
@@ -226,10 +239,8 @@ bool tsc_skip_to(Scorer *self, int doc_num)
     }
   }
 
-  // not found in cache, seek underlying stream
-  TermDocEnum *tde = ts->tde;
-  bool result = tde->skip_to(tde, doc_num);
-  if (result) {
+  /* not found in cache, seek underlying stream */
+  if (tde->skip_to(tde, doc_num)) {
     ts->pointer_max = 1;
     ts->pointer = 0;
     ts->docs[0] = self->doc = tde->doc_num(tde);
@@ -242,6 +253,7 @@ bool tsc_skip_to(Scorer *self, int doc_num)
 
 Explanation *tsc_explain(Scorer *self, int doc_num)
 {
+  Explanation *tf_explanation;
   TermScorer *ts = (TermScorer *)self->data;
   Query *query = ts->weight->get_query(ts->weight);
   Term *term = ((TermQuery *)query->data)->term;
@@ -260,18 +272,17 @@ Explanation *tsc_explain(Scorer *self, int doc_num)
   }
   tde->close(tde);
   ts->tde = NULL;
-  Explanation *tf_explanation = expl_create(sim_tf(self->similarity, tf),
+  tf_explanation = expl_create(sim_tf(self->similarity, (float)tf),
       strfmt("tf(term_freq(%s:%s)=%d)", term->field, term->text, tf));
 
   return tf_explanation;
 }
 
-void tsc_destroy(void *p)
+void tsc_destroy(Scorer *self)
 {
-  Scorer *self = (Scorer *)p;
   TermScorer *ts = (TermScorer *)self->data;
   if (ts->tde) ts->tde->close(ts->tde);
-  scorer_destroy(p);
+  scorer_destroy_i(self);
 }
 
 Scorer *tsc_create(Weight *weight, TermDocEnum *tde, uchar *norms)
@@ -287,7 +298,7 @@ Scorer *tsc_create(Weight *weight, TermDocEnum *tde, uchar *norms)
   ts->weight_value = weight->value;
 
   for (i = 0; i < SCORE_CACHE_SIZE; i++) {
-    ts->score_cache[i] = sim_tf(self->similarity, i) * ts->weight_value;
+    ts->score_cache[i] = sim_tf(self->similarity, (float)i) * ts->weight_value;
   }
 
   self->score = &tsc_score;

data/ext/q_wildcard.c

@@ -11,8 +11,8 @@ char *wcq_to_s(Query *self, char *field)
 {
   char *buffer, *bptr;
   Term *term = (Term *)self->data;
-  int tlen = strlen(term->text);
-  int flen = strlen(term->field);
+  size_t tlen = strlen(term->text);
+  size_t flen = strlen(term->field);
   bptr = buffer = ALLOC_N(char, tlen + flen + 35);
 
   if (strcmp(term->field, field) != 0) {
@@ -77,8 +77,8 @@ Query *wcq_rewrite(Query *self, IndexReader *ir)
   Term *term = (Term *)self->data;
   char *text = term->text;
   char *field = term->field;
-  char *first_star = index(text, WILD_STRING);
-  char *first_ques = index(text, WILD_CHAR);
+  char *first_star = strrchr(text, WILD_STRING);
+  char *first_ques = strrchr(text, WILD_CHAR);
   if (!first_star && !first_ques) {
     q = tq_create(term_clone(term));
   } else {
@@ -89,7 +89,7 @@ Query *wcq_rewrite(Query *self, IndexReader *ir)
     char *pattern = (first_ques && first_star > first_ques)
       ? first_ques : first_star;
 
-    int prefix_len = pattern - text;
+    int prefix_len = (int)(pattern - text);
 
     prefix_term.field = field;
     prefix_term.text = (char *)EMPTY_STRING;
@@ -120,15 +120,23 @@ Query *wcq_rewrite(Query *self, IndexReader *ir)
     free(prefix);
   }
 
-  if (self->rewritten) self->rewritten->destroy(self->rewritten);
-  return self->rewritten = q;
+  return q;
 }
 
-void wcq_destroy(void *p)
+static void wcq_destroy(Query *self)
 {
-  Query *self = (Query *)p;
   if (self->destroy_all) term_destroy((Term *)self->data);
-  q_destroy(self);
+  q_destroy_i(self);
+}
+
+static uint wcq_hash(Query *self)
+{
+  return term_hash((Term *)self->data);
+}
+
+static int wcq_eq(Query *self, Query *o)
+{
+  return term_eq((Term *)self->data, (Term *)o->data);
 }
 
 Query *wcq_create(Term *term)
@@ -136,11 +144,14 @@ Query *wcq_create(Term *term)
   Query *self = q_create();
 
   self->data = term;
+
   self->type = WILD_CARD_QUERY;
-  self->create_weight = NULL;
-  self->to_s = &wcq_to_s;
   self->rewrite = &wcq_rewrite;
-  self->destroy = &wcq_destroy;
+  self->to_s = &wcq_to_s;
+  self->hash = &wcq_hash;
+  self->eq = &wcq_eq;
+  self->destroy_i = &wcq_destroy;
+  self->create_weight_i = &q_create_weight_unsup;
 
   return self;
 }

data/ext/r_analysis.c

@@ -1,3 +1,4 @@
+#include <regex.h>
 #include "ferret.h"
 #include "analysis.h"
 #include "locale.h"
@@ -9,6 +10,7 @@ static VALUE cAsciiWhiteSpaceTokenizer;
 static VALUE cWhiteSpaceTokenizer;
 static VALUE cAsciiStandardTokenizer;
 static VALUE cStandardTokenizer;
+static VALUE cRegExpTokenizer;
 
 static VALUE cAsciiLowerCaseFilter;
 static VALUE cLowerCaseFilter;
@@ -23,14 +25,25 @@ static VALUE cWhiteSpaceAnalyzer;
 static VALUE cAsciiStandardAnalyzer;
 static VALUE cStandardAnalyzer;
 static VALUE cPerFieldAnalyzer;
+static VALUE cRegExpAnalyzer;
 
 //static VALUE cRegexAnalyzer;
 static VALUE cTokenStream;
 
+/* TokenStream Methods */
 static ID id_next;
 static ID id_reset;
 static ID id_clone;
 
+/* Analyzer Methods */
+static ID id_token_stream;
+
+static VALUE object_space;
+
+extern TokenStream *ts_create();
+extern int ruby_re_search(struct re_pattern_buffer *, const char *, int, int, int,
+		     struct re_registers *);
+
 /****************************************************************************
  *
  * Utility Methods
@@ -111,7 +124,7 @@ frt_set_token(Token *tk, VALUE rt)
   return tk;
 }
 
-#define GET_TK RToken *token; Data_Get_Struct(self, RToken, token);
+#define GET_TK RToken *token = (RToken *)DATA_PTR(self)
 static VALUE
 frt_token_init(int argc, VALUE *argv, VALUE self) 
 {
@@ -212,13 +225,12 @@ frt_ts_mark(void *p)
 }
 
 static void
-frt_ts_free(void *p)
+frt_ts_free(TokenStream *ts)
 {
-  TokenStream *ts = (TokenStream *)p;
   if (object_get(&ts->text) != Qnil) object_del(&ts->text);
   if (ts->sub_ts && (object_get(&ts->sub_ts) != Qnil)) object_del(&ts->sub_ts);
   object_del(ts);
-  ts->destroy(ts);
+  ts_deref(ts);
 }
 
 static VALUE
@@ -273,8 +285,7 @@ frt_ts_get_text(VALUE self)
 static VALUE
 frt_ts_next(VALUE self)
 {
-  TokenStream *ts; 
-  Data_Get_Struct(self, TokenStream, ts);
+  TokenStream *ts = (TokenStream *)DATA_PTR(self); 
   Token *next = ts->next(ts);
   if (next == NULL) {
     return Qnil;
@@ -287,41 +298,45 @@ frt_ts_next(VALUE self)
  * CWrappedTokenStream
  ****************************************************************************/
 
-void cwrts_destroy(void *p)
+static void
+cwrts_destroy(TokenStream *ts)
 {
-  TokenStream *ts = (TokenStream *)p;
+  rb_hash_delete(object_space, LONG2NUM((long)ts->data));
   free(ts->token);
   free(ts);
 }
 
-Token *cwrts_next(TokenStream *ts)
+static Token *
+cwrts_next(TokenStream *ts)
 {
   VALUE rts = (VALUE)ts->data;
   VALUE rtoken = rb_funcall(rts, id_next, 0);
   return frt_set_token(ts->token, rtoken);
 }
 
-void cwrts_reset(TokenStream *ts, char *text)
+static void
+cwrts_reset(TokenStream *ts, char *text)
 {
   VALUE rts = (VALUE)ts->data;
   ts->t = ts->text = text;
   rb_funcall(rts, id_reset, 1, rb_str_new2(text));
 }
 
-void cwrts_clone_i(TokenStream *orig_ts, TokenStream *new_ts)
+static void
+cwrts_clone_i(TokenStream *orig_ts, TokenStream *new_ts)
 {
   VALUE rorig_ts = (VALUE)orig_ts->data;
   new_ts->data = (void *)rb_funcall(rorig_ts, id_clone, 0);
 }
 
 static TokenStream *
-get_cwrapped_rts(VALUE rts, bool *self_destroy) 
+frt_get_cwrapped_rts(VALUE rts) 
 {
   TokenStream *ts;
   switch (TYPE(rts)) {
     case T_DATA:
       Data_Get_Struct(rts, TokenStream, ts);
-      *self_destroy = true;
+      ref(ts);
       break;
     default:
       ts = ALLOC(TokenStream);
@@ -332,12 +347,184 @@ get_cwrapped_rts(VALUE rts, bool *self_destroy)
       ts->clone_i = &cwrts_clone_i;
       ts->destroy = &cwrts_destroy;
       ts->sub_ts = NULL;
-      *self_destroy = false;
+      // prevent from being garbage collected
+      rb_hash_aset(object_space, LONG2NUM(rts), rts);
+      ts->ref_cnt = 1;
       break;
   }
   return ts;
 }
 
+/****************************************************************************
+ * RegExpTokenStream
+ ****************************************************************************/
+
+#define P "[_\\/.,-]"
+#define HASDIGIT "\\w*\\d\\w*"
+#define ALPHA "[-_[:alpha:]]"
+#define ALNUM "[-_[:alnum:]]"
+
+static char *token_re =
+  ALPHA "+(('" ALPHA "+)+|\\.(" ALPHA "\\.)+|"
+  "(@|\\&)\\w+([-.]\\w+)*|:\\/\\/" ALNUM "+([-.\\/]" ALNUM "+)*)?"
+  "|\\w+(([-._]\\w+)*\\@\\w+([-.]\\w+)+"
+    "|" P HASDIGIT "(" P "\\w+" P HASDIGIT ")*(" P "\\w+)?"
+    "|(\\.\\w+)+"
+    "|"
+  ")";
+static VALUE rtoken_re;
+
+typedef struct RegExpTokenStream {
+  VALUE rtext;
+  VALUE regex;
+  VALUE proc;
+  int curr_ind;  
+} RegExpTokenStream;
+
+static void
+rets_destroy(TokenStream *ts)
+{
+  rb_hash_delete(object_space, LONG2NUM((long)object_get(ts)));
+  free(ts->data);
+  free(ts->token);
+  free(ts);
+}
+
+static void
+frt_rets_free(TokenStream *ts)
+{
+  object_del(ts);
+  ts_deref(ts);
+}
+
+static void
+frt_rets_mark(TokenStream *ts)
+{
+  RegExpTokenStream *rets = (RegExpTokenStream *)ts->data;
+  rb_gc_mark(rets->rtext);
+  rb_gc_mark(rets->regex);
+  rb_gc_mark(rets->proc);
+}
+
+static VALUE
+frt_rets_set_text(VALUE self, VALUE rtext)
+{
+  TokenStream *ts;
+  RegExpTokenStream *rets;
+  Data_Get_Struct(self, TokenStream, ts);
+
+  StringValue(rtext);
+  rets = (RegExpTokenStream *)ts->data;
+  rets->rtext = rtext;
+  rets->curr_ind = 0;
+  
+  return rtext;
+}
+
+static VALUE
+frt_rets_get_text(VALUE self)
+{
+  TokenStream *ts;
+  RegExpTokenStream *rets;
+  Data_Get_Struct(self, TokenStream, ts);
+  rets = (RegExpTokenStream *)ts->data;
+  return rets->rtext;
+}
+
+static Token *
+rets_next(TokenStream *ts)
+{
+  static struct re_registers regs;
+  int ret, beg, end;
+  RegExpTokenStream *rets = (RegExpTokenStream *)ts->data;
+  struct RString *rtext = RSTRING(rets->rtext);
+  Check_Type(rets->regex, T_REGEXP);
+  ret = ruby_re_search(RREGEXP(rets->regex)->ptr,
+                 rtext->ptr, rtext->len,
+                 rets->curr_ind, rtext->len - rets->curr_ind,
+                 &regs);
+
+  if (ret == -2) rb_raise(rb_eStandardError, "regexp buffer overflow");
+  if (ret < 0) return NULL; /* not matched */
+  
+  beg = regs.beg[0];
+  rets->curr_ind = end = regs.end[0];
+  if (NIL_P(rets->proc)) {
+    return tk_set(ts->token, rtext->ptr + beg, end - beg, beg, end, 1);
+  } else {
+    VALUE rtok = rb_str_new(rtext->ptr + beg, end - beg);
+    rtok = rb_funcall(rets->proc, id_call, 1, rtok);
+    return tk_set(ts->token, RSTRING(rtok)->ptr, RSTRING(rtok)->len, beg, end, 1);
+  }
+}
+
+static void
+rets_reset(TokenStream *ts, char *text)
+{
+  RegExpTokenStream *rets = (RegExpTokenStream *)ts->data;
+  rets->rtext = rb_str_new2(text);
+  rets->curr_ind = 0;
+}
+
+void
+rets_clone_i(TokenStream *orig_ts, TokenStream *new_ts)
+{
+  RegExpTokenStream *new_rets = ALLOC(RegExpTokenStream);
+  RegExpTokenStream *orig_rets = (RegExpTokenStream *)orig_ts->data;
+  memcpy(new_rets, orig_rets, sizeof(RegExpTokenStream));
+  new_ts->data = new_rets;
+}
+
+static TokenStream *
+rets_create(VALUE rtext, VALUE regex, VALUE proc)
+{
+  RegExpTokenStream *rets;
+  TokenStream *ts;
+
+  if (rtext != Qnil) {
+    rtext = StringValue(rtext);
+  }
+  ts = ts_create();
+  ts->reset = &rets_reset;
+  ts->next = &rets_next;
+  ts->clone_i = &rets_clone_i;
+  ts->destroy = &rets_destroy;
+  ts->ref_cnt = 1;
+
+  rets = ALLOC(RegExpTokenStream);
+  rets->curr_ind = 0;
+  rets->rtext = rtext;
+  rets->proc = proc;
+  if (NIL_P(regex)) {
+    rets->regex = rtoken_re;
+  } else {
+    Check_Type(regex, T_REGEXP);
+    rets->regex = regex;
+  }
+
+  ts->data = rets;
+
+  return ts;
+}
+
+static VALUE
+frt_rets_init(int argc, VALUE *argv, VALUE self) 
+{
+  VALUE rtext, regex, proc;
+  TokenStream *ts;
+
+  rb_scan_args(argc, argv, "11&", &rtext, &regex, &proc);
+
+  ts = rets_create(rtext, regex, proc);
+
+  Frt_Wrap_Struct(self, &frt_rets_mark, &frt_rets_free, ts);
+  object_add(ts, self);
+  /* no need to add to object space as it is going to ruby space
+   * rb_hash_aset(object_space, LONG2NUM((long)self), self);
+   */
+  return self;
+}
+
 /****************************************************************************
  * Tokenizers
  ****************************************************************************/
@@ -394,10 +581,8 @@ frt_standard_tokenizer_init(VALUE self, VALUE rstr)
 static VALUE
 frt_a_lowercase_filter_init(VALUE self, VALUE rsub_ts) 
 {
-  bool self_destroy;
-  TokenStream *ts = lowercase_filter_create(
-      get_cwrapped_rts(rsub_ts, &self_destroy));
-  ts->destroy_sub = !self_destroy;
+  TokenStream *ts = frt_get_cwrapped_rts(rsub_ts);
+  ts = lowercase_filter_create(ts);
   object_add(&ts->sub_ts, rsub_ts);
 
   Frt_Wrap_Struct(self, &frt_ts_mark, &frt_ts_free, ts);
@@ -408,10 +593,8 @@ frt_a_lowercase_filter_init(VALUE self, VALUE rsub_ts)
 static VALUE
 frt_lowercase_filter_init(VALUE self, VALUE rsub_ts) 
 {
-  bool self_destroy;
-  TokenStream *ts = mb_lowercase_filter_create(
-      get_cwrapped_rts(rsub_ts, &self_destroy));
-  ts->destroy_sub = !self_destroy;
+  TokenStream *ts = frt_get_cwrapped_rts(rsub_ts);
+  ts = mb_lowercase_filter_create(ts);
   object_add(&ts->sub_ts, rsub_ts);
 
   Frt_Wrap_Struct(self, &frt_ts_mark, &frt_ts_free, ts);
@@ -423,19 +606,17 @@ static VALUE
 frt_stop_filter_init(int argc, VALUE *argv, VALUE self) 
 {
   VALUE rsub_ts, rstop_words;
-  bool self_destroy;
   TokenStream *ts;
   rb_scan_args(argc, argv, "11", &rsub_ts, &rstop_words);
+  ts = frt_get_cwrapped_rts(rsub_ts);
   if (rstop_words != Qnil) {
     char **stop_words = get_stopwords(rstop_words);
-    ts = stop_filter_create_with_words(
-        get_cwrapped_rts(rsub_ts, &self_destroy), (const char **)stop_words);
+    ts = stop_filter_create_with_words(ts, (const char **)stop_words);
+
     free(stop_words);
   } else {
-    ts = stop_filter_create(
-        get_cwrapped_rts(rsub_ts, &self_destroy));
+    ts = stop_filter_create(ts);
   }
-  ts->destroy_sub = !self_destroy;
   object_add(&ts->sub_ts, rsub_ts);
 
   Frt_Wrap_Struct(self, &frt_ts_mark, &frt_ts_free, ts);
@@ -449,16 +630,14 @@ frt_stem_filter_init(int argc, VALUE *argv, VALUE self)
   VALUE rsub_ts, ralgorithm, rcharenc;
   char *algorithm = "english";
   char *charenc = NULL;
-  bool self_destroy;
   TokenStream *ts;
   rb_scan_args(argc, argv, "12", &rsub_ts, &ralgorithm, &rcharenc);
+  ts = frt_get_cwrapped_rts(rsub_ts);
   switch (argc) {
     case 3: charenc = RSTRING(rb_obj_as_string(rcharenc))->ptr;
     case 2: algorithm = RSTRING(rb_obj_as_string(ralgorithm))->ptr;
   }
-  ts = stem_filter_create(
-      get_cwrapped_rts(rsub_ts, &self_destroy), algorithm, charenc);
-  ts->destroy_sub = !self_destroy;
+  ts = stem_filter_create(ts, algorithm, charenc);
   object_add(&ts->sub_ts, rsub_ts);
 
   Frt_Wrap_Struct(self, &frt_ts_mark, &frt_ts_free, ts);
@@ -472,34 +651,49 @@ frt_stem_filter_init(int argc, VALUE *argv, VALUE self)
  *
  ****************************************************************************/
 
-Analyzer *get_cwrapped_analyzer(ranalyzer)
+/****************************************************************************
+ * CWrappedAnalyzer Methods
+ ****************************************************************************/
+
+static void
+cwa_destroy(Analyzer *a)
+{
+  rb_hash_delete(object_space, LONG2NUM((long)a->data));
+  a_standard_destroy(a);
+}
+
+static TokenStream *
+cwa_get_ts(Analyzer *a, char *field, char *text)
+{
+  VALUE ranalyzer = (VALUE)a->data;
+  VALUE rts = rb_funcall(ranalyzer, id_token_stream, 2,
+      rb_str_new2(field), rb_str_new2(text));
+  return frt_get_cwrapped_rts(rts);
+} 
+
+Analyzer *
+frt_get_cwrapped_analyzer(ranalyzer)
 {
   Analyzer *a = NULL;
   switch (TYPE(ranalyzer)) {
     case T_DATA:
       Data_Get_Struct(ranalyzer, Analyzer, a);
+      ref(a);
       break;
     default:
-      printf("Oh RFuck\n");
-      //ts = ALLOC(TokenStream);
-      //ts->token = ALLOC(Token);
-      //ts->data = (void *)rts;
-      //ts->next = &cwrts_next;
-      //ts->reset = &cwrts_reset;
-      //ts->clone_i = &cwrts_clone_i;
-      //ts->destroy = &cwrts_destroy;
-      //ts->sub_ts = NULL;
+      a = analyzer_create((void *)ranalyzer, NULL, &cwa_destroy, &cwa_get_ts);
+      // prevent from being garbage collected
+      rb_hash_aset(object_space, LONG2NUM(ranalyzer), ranalyzer);
       break;
   }
   return a;
 }
 
 static void
-frt_analyzer_free(void *p)
+frt_analyzer_free(Analyzer *a)
 {
-  Analyzer *a = (Analyzer *)p;
   object_del(a);
-  a->destroy(a);
+  a_deref(a);
 }
 
 VALUE
@@ -513,13 +707,16 @@ frt_get_analyzer(Analyzer *a)
 static VALUE
 frt_analyzer_token_stream(VALUE self, VALUE rfield, VALUE rstring)
 {
-  Analyzer *a = ((struct RData *)(self))->data;
+  TokenStream *ts;
+  Analyzer *a = (Analyzer *)DATA_PTR(self);
+
   rfield = rb_obj_as_string(rfield);
   rstring = rb_obj_as_string(rstring);
   
-  TokenStream *ts = a_get_new_ts(a, RSTRING(rfield)->ptr, RSTRING(rstring)->ptr);
+  ts = a_get_new_ts(a, RSTRING(rfield)->ptr, RSTRING(rstring)->ptr);
 
-  object_set(&ts->text, rstring); // Make sure that there is no entry already
+  /* Make sure that there is no entry already */
+  object_set(&ts->text, rstring);
   return get_token_stream(ts);
 }
 
@@ -533,8 +730,9 @@ frt_analyzer_token_stream(VALUE self, VALUE rfield, VALUE rstring)
 static VALUE
 frt_a_white_space_analyzer_init(int argc, VALUE *argv, VALUE self)
 {
+  Analyzer *a;
   GET_LOWER(false);
-  Analyzer *a = whitespace_analyzer_create(lower);
+  a = whitespace_analyzer_create(lower);
   Frt_Wrap_Struct(self, NULL, &frt_analyzer_free, a);
   object_add(a, self);
   return self;
@@ -544,8 +742,9 @@ frt_a_white_space_analyzer_init(int argc, VALUE *argv, VALUE self)
 static VALUE
 frt_white_space_analyzer_init(int argc, VALUE *argv, VALUE self)
 {
+  Analyzer *a;
   GET_LOWER(false);
-  Analyzer *a = mb_whitespace_analyzer_create(lower);
+  a = mb_whitespace_analyzer_create(lower);
   Frt_Wrap_Struct(self, NULL, &frt_analyzer_free, a);
   object_add(a, self);
   return self;
@@ -555,8 +754,9 @@ frt_white_space_analyzer_init(int argc, VALUE *argv, VALUE self)
 static VALUE
 frt_a_letter_analyzer_init(int argc, VALUE *argv, VALUE self)
 {
+  Analyzer *a;
   GET_LOWER(true);
-  Analyzer *a = letter_analyzer_create(lower);
+  a = letter_analyzer_create(lower);
   Frt_Wrap_Struct(self, NULL, &frt_analyzer_free, a);
   object_add(a, self);
   return self;
@@ -566,8 +766,9 @@ frt_a_letter_analyzer_init(int argc, VALUE *argv, VALUE self)
 static VALUE
 frt_letter_analyzer_init(int argc, VALUE *argv, VALUE self)
 {
+  Analyzer *a;
   GET_LOWER(true);
-  Analyzer *a = mb_letter_analyzer_create(lower);
+  a = mb_letter_analyzer_create(lower);
   Frt_Wrap_Struct(self, NULL, &frt_analyzer_free, a);
   object_add(a, self);
   return self;
@@ -628,13 +829,29 @@ frt_standard_analyzer_init(int argc, VALUE *argv, VALUE self)
   return self;
 }
 
+void
+frt_h_mark_values_i(void *key, void *value, void *arg)
+{
+  frt_gc_mark(value);
+}
+
+void
+frt_pfa_mark(void *p)
+{
+  Analyzer *a = (Analyzer *)p;
+  PerFieldAnalyzer *pfa = (PerFieldAnalyzer *)a->data;
+  frt_gc_mark(pfa->def);
+  h_each(pfa->dict, &frt_h_mark_values_i, NULL);
+}
+
 /*** PerFieldAnalyzer ***/
+
 static VALUE
 frt_per_field_analyzer_init(VALUE self, VALUE ranalyzer)
 {
-  Analyzer *def = get_cwrapped_analyzer(ranalyzer);
-  Analyzer *a = per_field_analyzer_create(def, false);
-  Frt_Wrap_Struct(self, NULL, &frt_analyzer_free, a);
+  Analyzer *def = frt_get_cwrapped_analyzer(ranalyzer);
+  Analyzer *a = per_field_analyzer_create(def);
+  Frt_Wrap_Struct(self, &frt_pfa_mark, &frt_analyzer_free, a);
   object_add(a, self);
   return self;
 }
@@ -644,42 +861,48 @@ frt_per_field_analyzer_add_field(VALUE self, VALUE rfield, VALUE ranalyzer)
 {
   Analyzer *pfa, *a;
   Data_Get_Struct(self, Analyzer, pfa);
-  Data_Get_Struct(ranalyzer, Analyzer, a);
+  a = frt_get_cwrapped_analyzer(ranalyzer);
 
   pfa_add_field(pfa, StringValuePtr(rfield), a);
   return self;
 }
 
+/*** RegExpAnalyzer ***/
 
-/** RegexAnalyzer **/
-/*
-static VALUE
-frt_regex_analyzer_init(VALUE self)
+static void
+frt_re_analyzer_mark(Analyzer *a)
 {
-  Analyzer *a = regex_analyzer_create();
-  // keine Ahnung warum hier das Makro und nicht Data_Wrap_Struct:
-  Frt_Wrap_Struct(self, NULL, &frt_analyzer_free, a);
-  // wofuer?:
-  object_add(a, self);
-  return self;
+  frt_gc_mark(a->current_ts);
 }
 
-// convenience method
-// XXX this sets the locale for the entire program
-static VALUE
-frt_regex_analyzer_token_stream(VALUE self, VALUE field, VALUE string)
+static void
+re_analyzer_destroy(Analyzer *a)
 {
-  Analyzer *a =((struct RData *)(self))->data;
-  TokenStream *ts = a->get_ts( a, StringValuePtr(field), StringValuePtr(string) );
-  // already freed via analyzer's free()
-  VALUE token_stream = Data_Wrap_Struct(cTokenStream, NULL, NULL, ts);
-  return token_stream;
+  free(a->data);
+  a_standard_destroy(a);
 }
-*/
-/** /RegexAnalyzer **/
 
-/** TokenStream **/
-/** /TokenStream **/
+static VALUE
+frt_re_analyzer_init(int argc, VALUE *argv, VALUE self)
+{
+  VALUE lower, rets, regex, proc;
+  Analyzer *a;
+  TokenStream *ts;
+  rb_scan_args(argc, argv, "02&", &regex, &lower, &proc);
+
+  ts = rets_create(Qnil, regex, proc);
+  rets = Data_Wrap_Struct(cRegExpTokenizer, &frt_rets_mark, &frt_rets_free, ts);
+  ref(ts);
+  rb_hash_aset(object_space, LONG2NUM((long)rets), rets);
+  object_add(ts, rets);
+
+  if (lower != Qfalse) ts = mb_lowercase_filter_create(ts);
+
+  a = analyzer_create(NULL, ts, &re_analyzer_destroy, NULL);
+  Frt_Wrap_Struct(self, &frt_re_analyzer_mark, &frt_analyzer_free, a);
+  object_add(a, self);
+  return self;
+}
 
 /****************************************************************************
  *
@@ -710,10 +933,17 @@ static VALUE frt_setlocale(VALUE self, VALUE locale)
 void
 Init_analysis(void)
 {
+  /* TokenStream Methods */
 	id_next = rb_intern("next");
 	id_reset = rb_intern("text=");
 	id_clone = rb_intern("clone");
 
+  /* Analyzer Methods */
+	id_token_stream = rb_intern("token_stream");
+
+  object_space = rb_hash_new();
+  rb_define_const(mFerret, "OBJECT_SPACE", object_space);
+
   /*** * * Locale stuff * * ***/
   frt_locale = setlocale(LC_ALL, "");
   rb_define_singleton_method(mFerret, "locale=", frt_setlocale, 1);
@@ -790,6 +1020,18 @@ Init_analysis(void)
   rb_define_method(cStandardTokenizer, "initialize",
       frt_standard_tokenizer_init, 1);
 
+  /*** * * RegExpTokenizer * * ***/
+  cRegExpTokenizer =
+    rb_define_class_under(mAnalysis, "RegExpTokenizer", cTokenStream);
+  rtoken_re = rb_reg_new(token_re, strlen(token_re), 0);
+  rb_define_const(cRegExpTokenizer, "REGEXP", rtoken_re);
+  rb_define_alloc_func(cRegExpTokenizer, frt_data_alloc);
+  rb_define_method(cRegExpTokenizer, "initialize",
+      frt_rets_init, -1);
+  rb_define_method(cRegExpTokenizer, "next", frt_ts_next, 0);
+  rb_define_method(cRegExpTokenizer, "text=", frt_rets_set_text, 1);
+  rb_define_method(cRegExpTokenizer, "text", frt_rets_get_text, 0);
+
   /***************/
   /*** Filters ***/
   /***************/
@@ -911,7 +1153,13 @@ Init_analysis(void)
   rb_define_method(cPerFieldAnalyzer, "[]=",
       frt_per_field_analyzer_add_field, 2);
 
-  /** RegexAnalyzer **/
+  /*** * * RegexAnalyzer * * ***/
+  cRegExpAnalyzer =
+    rb_define_class_under(mAnalysis, "RegExpAnalyzer", cAnalyzer);
+  rb_define_alloc_func(cRegExpAnalyzer, frt_data_alloc);
+  rb_define_method(cRegExpAnalyzer, "initialize",
+      frt_re_analyzer_init, -1);
+
   /*
   cRegexAnalyzer =
     rb_define_class_under(mAnalysis, "RegexAnalyzer", cAnalyzer);