ferret 0.9.6 → 0.10.0

data/MIT-LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2005 David Balmain
+Copyright (c) 2005-2006 David Balmain
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/README

@@ -8,7 +8,7 @@ search for things in them later.
 == Requirements
 
 * Ruby 1.8
-* (C compiler to build the extension but not required to use Ferret)
+* C compiler to build the extension. Tested with gcc, VC6 and VC2005
 
 == Installation
 
@@ -26,7 +26,7 @@ Run the following;
   $ rake ext
   $ ruby setup.rb config
   $ ruby setup.rb setup
-  # sudo ruby setup.rb install
+  # ruby setup.rb install
   
 These simple steps install ferret in the default location of Ruby libraries.
 You can also install files into your favorite directory by supplying setup.rb
@@ -53,10 +53,8 @@ documentation.
   abilities of Ferret to present your data the best way you see fit.
 
 * Ferret::Document: to find out how to create documents. This part of Ferret
-  is relatively straightforward. The main thing that we haven't gone into here
-  is the use of term vectors. These allow you to store and retrieve the
-  positions and offsets of the data which can be very useful in document
-  comparison amoung other things.  == More information
+  is relatively straightforward. If you know how Strings, Hashes and Arrays work
+  Ferret then you'll be able to create Documents.
 
 * Ferret::QueryParser: if you want to find out more about what you can do with
   Ferret's Query Parser, this is the place to look. The query parser is one
@@ -71,17 +69,8 @@ documentation.
 
 === Performance
 
-Currently Ferret is an order of magnitude slower than Java Lucene which can be
-quite a pain at times. I have written some basic C extensions which may or may
-not have installed when you installed Ferret. These double the speed but still
-leave it a lot slower than the Java version. I have, however, ported the
-indexing part of Java Lucene to C and it is an order of magnitude faster then
-the Java version. Once I'm pretty certain that the API of Ferret has settled
-and won't be changing much, I'll intergrate my C version. So expect to see
-Ferret running faster than Java Lucene some time in the future. If you'd like
-to try cferret and test my claims, let me know (if you haven't already found
-it in my subversion repository). It's not currently portable and will probably
-only run on linux.
+We are unaware of any alternatives that can out-perform Ferret while still
+matching it in features.
 
 == Contact
 
@@ -89,17 +78,16 @@ For bug reports and patches I have set up Trac here;
 
   http://ferret.davebalmain.com/trac
 
-Queries, discussion etc should be addressed to the forum or mailing lists hosted
-at;
+Queries, discussion etc should be addressed to the mailing lists here;
 
   http://rubyforge.org/projects/ferret/
 
-Alternatively you could create a new page for discussion on the wiki at my Trac
-page above. Or, if you're shy, please feel free to email me directly at dbalmain@gmail.com
+Alternatively you could create a new page for discussion on the Ferret wiki;
 
-Of course, since Ferret is almost a straight port of Java Lucene,
-everything said about Lucene at http://jakarta.apache.org/lucene/ should
-be true about Ferret. Apart from the bits about it being in Java.
+  http://ferret.davebalmain.com/trac
+
+Of course, since Ferret was ported from Apache Lucene, most of what you can
+do with Lucene you can also do with Ferret.
 
 == Authors
 

data/Rakefile

@@ -8,8 +8,7 @@ require 'rake'
 require 'rake/testtask'
 require 'rake/rdoctask'
 require 'rake/clean'
-require 'rake_utils/code_statistics'
-require 'lib/rferret'
+require 'ferret_version'
 
 begin
   require 'rubygems'
@@ -29,18 +28,18 @@ def announce(msg='')
   STDERR.puts msg
 end
 
-$VERBOSE = nil
-
 EXT = "ferret_ext.so"
-EXT_SRC = FileList["src/**/*.[ch]"]
-if (/mswin/ =~ RUBY_PLATFORM)
-  EXT_SRC.delete('src/io/nix_io.c')
-end
+EXT_SRC = FileList["../c/src/*.[c]", "../c/include/*.h",
+                   "../c/lib/libstemmer_c/src_c/*.[ch]",
+                   "../c/lib/libstemmer_c/runtime/*.[ch]",
+                   "../c/lib/libstemmer_c/libstemmer/*.[ch]",
+                   "../c/lib/libstemmer_c/include/libstemmer.h"]
 
 EXT_SRC_DEST = EXT_SRC.map {|fn| File.join("ext", File.basename(fn))}
 SRC = (FileList["ext/*.[ch]"] + EXT_SRC_DEST).uniq
 
-CLEAN.include(FileList['**/*.o', '**/*.obj', 'InstalledFiles', '.config'])
+CLEAN.include(FileList['**/*.o', '**/*.obj', 'InstalledFiles',
+                       '.config', 'ext/cferret.c'])
 CLOBBER.include(FileList['**/*.so'], 'ext/Makefile', EXT_SRC_DEST)
 POLISH = Rake::FileList.new.include(FileList['**/*.so'], 'ext/Makefile')
 
@@ -49,69 +48,53 @@ task :polish => [:clean] do
   POLISH.each { |fn| rm_r fn rescue nil }
 end
 
+desc "Run tests with Valgrind"
+task :valgrind do
+  sh "valgrind --gen-suppressions=yes --suppressions=ferret_valgrind.supp " +
+     "--leak-check=yes --show-reachable=yes -v ruby test/test_all.rb"
+  #sh "valgrind --suppressions=ferret_valgrind.supp " +
+  #   "--leak-check=yes --show-reachable=yes -v ruby test/unit/index/tc_index_reader.rb"
+end
+
 task :default => :test_all
-desc "Run all tests"
-task :test_all => [ :test_runits, :test_cunits, :test_functional ]
+#task :default => :ext do
+#  sh "ruby test/unit/index/tc_index.rb"
+#end
 
-desc "Generate API documentation, and show coding stats"
-task :doc => [ :stats, :appdoc ]
+desc "Run all tests"
+task :test_all => [ :test_units ]
 
-desc "run unit tests in test/unit for pure ruby ferret"
-Rake::TestTask.new("test_runits" => :parsers) do |t|
-  t.ruby_opts = ["-r 'lib/rferret'"]
-  t.libs << "test/unit"
-  t.pattern = 'test/unit/ts_*.rb'
-  t.verbose = true
-end
-
-desc "run unit tests in test/unit for C ferret"
-Rake::TestTask.new("test_cunits" => :ext) do |t|
-  t.libs << "test/unit"
-  t.pattern = 'test/unit/ts_*.rb'
-  t.verbose = true
-end
+desc "Generate API documentation"
+task :doc => [ :appdoc ]
 
 desc "run unit tests in test/unit"
-Rake::TestTask.new("test_long") do |t|
-  t.libs << "test"
+Rake::TestTask.new("test_units" => :ext) do |t|
   t.libs << "test/unit"
-  t.test_files = FileList["test/longrunning/tm_store.rb"]
   t.pattern = 'test/unit/t[cs]_*.rb'
+  #t.pattern = 'test/unit/search/tc_index_searcher.rb'
   t.verbose = true
 end
 
-desc "run funtional tests in test/funtional"
-Rake::TestTask.new("test_functional") do |t|
-  t.libs << "test"
-  t.pattern = 'test/funtional/tc_*.rb'
-  t.verbose = true
-end
-
-desc "Report code statistics (KLOCS, etc) from application"
-task :stats do
-  CodeStatistics.new(
-                      ["Ferret", "lib/ferret"],
-                      ["Units", "test/unit"],
-                      ["Units-extended", "test/longrunning"]
-                    ).to_s
-end
-
 desc "Generate documentation for the application"
 rd = Rake::RDocTask.new("appdoc") do |rdoc|
   rdoc.rdoc_dir = 'doc/api'
   rdoc.title    = "Ferret Search Library Documentation"
-  rdoc.options << '--line-numbers --inline-source'
+  rdoc.options << '--line-numbers'
+  rdoc.options << '--inline-source'
+  rdoc.options << '--charset=utf-8'
   rdoc.rdoc_files.include('README')
   rdoc.rdoc_files.include('TODO')
   rdoc.rdoc_files.include('TUTORIAL')
   rdoc.rdoc_files.include('MIT-LICENSE')
   rdoc.rdoc_files.include('lib/**/*.rb')
+  rdoc.rdoc_files.include('ext/r_*.c')
+  rdoc.rdoc_files.include('ext/ferret.c')
 end
 
 EXT_SRC.each do |fn|
   dest_fn = File.join("ext", File.basename(fn))
   file dest_fn => fn do |t|
-    cp fn, dest_fn
+    ln_s File.join("..", fn), dest_fn
     if fn =~ /stemmer/
       # flatten the directory structure for lib_stemmer
       open(dest_fn) do |in_f|
@@ -129,7 +112,7 @@ task :ext => ["ext/#{EXT}"] + SRC
 
 file "ext/#{EXT}" => ["ext/Makefile"] do
   cp "ext/inc/lang.h", "ext/lang.h"
-  cp "ext/inc/except.h", "ext/except.h"
+  cp "ext/inc/threading.h", "ext/threading.h"
   cd "ext"
   if (/mswin/ =~ RUBY_PLATFORM) and ENV['make'].nil?
     sh "nmake"
@@ -140,10 +123,12 @@ file "ext/#{EXT}" => ["ext/Makefile"] do
 end
 
 file "ext/lang.h" => ["ext/inc/lang.h"] do
+  rm_f "ext/lang.h"
   cp "ext/inc/lang.h", "ext/lang.h"
 end
-file "ext/except.h" => ["ext/inc/except.h"] do
-  cp "ext/inc/except.h", "ext/except.h"
+file "ext/threading.h" => ["ext/inc/threading.h"] do
+  rm_f "ext/threading.h"
+  cp "ext/inc/threading.h", "ext/threading.h"
 end
 
 file "ext/Makefile" => SRC do
@@ -175,7 +160,6 @@ PKG_FILES = FileList[
   'Rakefile'
 ]
 PKG_FILES.exclude('**/*.o')
-PKG_FILES.include('ext/termdocs.c')
 PKG_FILES.exclude('**/Makefile')
 PKG_FILES.exclude('ext/ferret_ext.so')
 
@@ -213,6 +197,7 @@ else
     s.require_path = 'lib'                         # Use these for libraries.
     s.autorequire = 'ferret'
 
+
     #s.bindir = "bin"                               # Use these for applications.
     #s.executables = ["rake"]
     #s.default_executable = "rake"
@@ -319,11 +304,10 @@ task :update_version => [:prerelease] do
   else
     announce "Updating Ferret version to #{PKG_VERSION}"
     reversion("lib/ferret.rb")
-    reversion("lib/rferret.rb")
     if ENV['RELTEST']
       announce "Release Task Testing, skipping commiting of new version"
     else
-      sh %{svn ci -m "Updated to version #{PKG_VERSION}" lib/rferret.rb}
+      sh %{svn ci -m "Updated to version #{PKG_VERSION}" lib/ferret.rb}
     end
   end
 end

data/TODO

@@ -1,17 +1,14 @@
-= Ferret Project -- To Do List
-
-Send suggestions for this list to mailto:dbalmain@gmail.com
-
-=== To Do
-
-* Make a dll for people on Windows
-* fix rb_obj_as_string to StringValue()
-* pure ruby ConstantScoreQuery
-
-=== Done
-
-* Add the ability to persist an in memory index to Ferret::Index::Index
-* Add UTF-8 support
-* Multi Field Query
-* Test threading
-* Compile a proper dummy executable
+= TODO
+
+* user defined sorting
+* add field compression
+* Fix highlighting to work for compressed fields
+* Fix highlighting to work for external fields
+* Add Ferret::Index::Index
+
+= Done
+* Add string Sort descripter
+* fix memory bug
+* add MultiReader interface
+* add lexicographical sort (byte sort)
+* Add highlighting

data/ext/analysis.c

@@ -1,90 +1,95 @@
 #include "analysis.h"
 #include "hash.h"
-#include "libstemmer.h"
+#include <libstemmer.h>
 #include <string.h>
 #include <ctype.h>
 #include <wctype.h>
 #include <wchar.h>
 
-
 /****************************************************************************
  *
  * Token
  *
  ****************************************************************************/
 
-Token *tk_create()
+inline Token *tk_set(Token *tk,
+                     char *text, int tlen, int start, int end, int pos_inc)
 {
-  return ALLOC(Token);
-}
-
-void tk_destroy(void *p)
-{
-  free(p);
+    if (tlen >= MAX_WORD_SIZE) {
+        tlen = MAX_WORD_SIZE - 1;
+    }
+    memcpy(tk->text, text, sizeof(char) * tlen);
+    tk->text[tlen] = '\0';
+    tk->len = tlen;
+    tk->start = start;
+    tk->end = end;
+    tk->pos_inc = pos_inc;
+    return tk;
 }
 
-inline Token *tk_set(Token *tk, 
-					 char *text, 
-					 int tlen, 
-					 int start, 
-					 int end, 
-					 int pos_inc)
+inline Token *tk_set_ts(Token *tk,
+                        char *start, char *end, char *text, int pos_inc)
 {
-  if (tlen >= MAX_WORD_SIZE) {
-    tlen = MAX_WORD_SIZE - 1;
-  }
-  memcpy(tk->text, text, sizeof(char) * tlen);
-  tk->text[tlen] = '\0';
-  tk->start = start;
-  tk->end = end;
-  tk->pos_inc = pos_inc;
-  return tk;
+    return tk_set(tk, start, (int)(end - start),
+                  (int)(start - text), (int)(end - text), pos_inc);
 }
 
-inline Token *tk_set_ts(Token *tk, 
-						char *start, 
-						char *end, 
-						char *text, 
-						int pos_inc)
+inline Token *tk_set_no_len(Token *tk,
+                            char *text, int start, int end, int pos_inc)
 {
-  return tk_set(tk, start, (int)(end - start), 
-	  (int)(start - text), (int)(end - text), pos_inc);
+    return tk_set(tk, text, (int)strlen(text), start, end, pos_inc);
 }
 
-inline Token *tk_set_no_len(Token *tk, 
-							char *text, 
-							int start, 
-							int end, 
-							int pos_inc)
+inline Token *w_tk_set(Token *tk, wchar_t *text, int start, int end,
+                       int pos_inc)
 {
-  return tk_set(tk, text, (int)strlen(text), start, end, pos_inc);
+    int len = wcstombs(tk->text, text, MAX_WORD_SIZE - 1);
+    tk->text[len] = '\0';
+    tk->len = len;
+    tk->start = start;
+    tk->end = end;
+    tk->pos_inc = pos_inc;
+    return tk;
 }
 
 int tk_eq(Token *tk1, Token *tk2)
 {
-  return (strcmp((char *)tk1->text, (char *)tk2->text) == 0 &&
-      tk1->start == tk2->start && tk1->end == tk2->end);
+    return (strcmp((char *)tk1->text, (char *)tk2->text) == 0 &&
+            tk1->start == tk2->start && tk1->end == tk2->end);
 }
 
 int tk_cmp(Token *tk1, Token *tk2)
 {
-  int cmp;
-  if (tk1->start > tk2->start) {
-    cmp = 1;
-  } else if (tk1->start < tk2->start) {
-    cmp = -1;
-  } else {
-    if (tk1->end > tk2->end) {
-      cmp = 1;
-    } else if (tk1->end < tk2->end) {
-      cmp = -1;
-    } else {
-      cmp = strcmp((char *)tk1->text, (char *)tk2->text);
+    int cmp;
+    if (tk1->start > tk2->start) {
+        cmp = 1;
+    }
+    else if (tk1->start < tk2->start) {
+        cmp = -1;
     }
-  }
-  return cmp;
+    else {
+        if (tk1->end > tk2->end) {
+            cmp = 1;
+        }
+        else if (tk1->end < tk2->end) {
+            cmp = -1;
+        }
+        else {
+            cmp = strcmp((char *)tk1->text, (char *)tk2->text);
+        }
+    }
+    return cmp;
+}
+
+void tk_destroy(void *p)
+{
+    free(p);
 }
 
+Token *tk_new()
+{
+    return ALLOC(Token);
+}
 
 /****************************************************************************
  *
@@ -92,92 +97,94 @@ int tk_cmp(Token *tk1, Token *tk2)
  *
  ****************************************************************************/
 
-void ts_deref(void *p)
+void ts_deref(TokenStream *ts)
 {
-  TokenStream *ts = (TokenStream *)p;
-  if (--ts->ref_cnt <= 0) ts->destroy(ts);
+    if (--ts->ref_cnt <= 0) {
+        ts->destroy_i(ts);
+    }
 }
 
-void ts_standard_destroy(TokenStream *ts)
+static TokenStream *ts_reset(TokenStream *ts, char *text)
 {
-  tk_destroy(ts->token);
-  free(ts);
+    ts->t = ts->text = text;
+    return ts;
 }
 
-void ts_reset(TokenStream *ts, char *text)
+TokenStream *ts_clone_size(TokenStream *orig_ts, size_t size)
 {
-  ts->t = ts->text = text;
+    TokenStream *ts = (TokenStream *)ecalloc(size);
+    memcpy(ts, orig_ts, size);
+    ts->ref_cnt = 1;
+    return ts;
 }
 
-TokenStream *ts_create()
+TokenStream *ts_new_i(size_t size)
 {
-  TokenStream *ts = ALLOC_AND_ZERO_N(TokenStream, 1);
-  ts->token = tk_create();
-  ts->destroy = &ts_standard_destroy;
-  ts->reset = &ts_reset;
-  ts->ref_cnt = 1;
-  return ts;
+    TokenStream *ts = ecalloc(size);
+
+    ts->destroy_i = (void (*)(TokenStream *))&free;
+    ts->reset = &ts_reset;
+    ts->ref_cnt = 1;
+
+    return ts;
 }
 
-TokenStream *ts_clone(TokenStream *orig_ts)
+/****************************************************************************
+ * CachedTokenStream
+ ****************************************************************************/
+
+#define CTS(token_stream) ((CachedTokenStream *)(token_stream))
+
+static TokenStream *cts_clone_i(TokenStream *orig_ts)
 {
-  TokenStream *ts = ALLOC(TokenStream);
-  memcpy(ts, orig_ts, sizeof(TokenStream));
-  if (orig_ts->token) {
-    ts->token = ALLOC(Token);
-    memcpy(ts->token, orig_ts->token, sizeof(Token));
-  }
-  if (orig_ts->sub_ts) ts->sub_ts = ts_clone(orig_ts->sub_ts);
-  if (orig_ts->clone_i) orig_ts->clone_i(orig_ts, ts);
-  ts->ref_cnt = 1;
-  return ts;
+    return ts_clone_size(orig_ts, sizeof(CachedTokenStream));
 }
 
-/* * Multi-byte TokenStream * */
-static char * const ENC_ERR_MSG = "Error decoding input string. "
-                          "Check that you have the locale set correctly";
-#define MB_NEXT_CHAR \
-  if ((i = (int)mbrtowc(&wchr, t, MB_CUR_MAX, (mbstate_t *)ts->data)) < 0)\
-    RAISE(IO_ERROR, ENC_ERR_MSG)
-
-inline Token *w_tk_set(Token *tk, wchar_t *text, int start, int end, int pos_inc)
+static TokenStream *cts_new()
 {
-  tk->text[wcstombs(tk->text, text, MAX_WORD_SIZE - 1)] = '\0';
-  tk->start = start;
-  tk->end = end;
-  tk->pos_inc = pos_inc;
-  return tk;
+    TokenStream *ts = ts_new(CachedTokenStream);
+    ts->clone_i = &cts_clone_i;
+    return ts;
 }
 
-void mb_ts_standard_destroy(TokenStream *ts)
+/* * Multi-byte TokenStream * */
+
+#define MBTS(token_stream) ((MultiByteTokenStream *)(token_stream))
+
+inline int mb_next_char(wchar_t *wchr, const char *s, mbstate_t *state)
 {
-  tk_destroy(ts->token);
-  free(ts->data);
-  free(ts);
+    int num_bytes;
+    if ((num_bytes = (int)mbrtowc(wchr, s, MB_CUR_MAX, state)) < 0) {
+        const char *t = s;
+        do {
+            t++;
+            ZEROSET(state, mbstate_t);
+            num_bytes = (int)mbrtowc(wchr, t, MB_CUR_MAX, state);
+        } while ((num_bytes < 0) && (*wchr != 0) && (*t != 0));
+        num_bytes += t - s;
+    }
+    return num_bytes;
 }
 
-void mb_ts_reset(TokenStream *ts, char *text)
+static TokenStream *mb_ts_reset(TokenStream *ts, char *text)
 {
-  ZEROSET(ts->data, mbstate_t, 1);
-  ts_reset(ts, text);
+    ZEROSET(&(MBTS(ts)->state), mbstate_t);
+    ts_reset(ts, text);
+    return ts;
 }
 
-void mb_ts_clone_i(TokenStream *orig_ts, TokenStream *new_ts)
+static TokenStream *mb_ts_clone_i(TokenStream *orig_ts)
 {
-  new_ts->data = ALLOC(mbstate_t);
-  memcpy(new_ts->data, orig_ts->data, sizeof(mbstate_t));
+    return ts_clone_size(orig_ts, sizeof(MultiByteTokenStream));
 }
 
-TokenStream *mb_ts_create()
+TokenStream *mb_ts_new()
 {
-  TokenStream *ts = ALLOC_AND_ZERO_N(TokenStream, 1);
-  ts->data = ALLOC(mbstate_t);
-  ts->token = tk_create();
-  ts->destroy = &mb_ts_standard_destroy;
-  ts->reset = &mb_ts_reset;
-  ts->clone_i = &mb_ts_clone_i;
-  ts->ref_cnt = 1;
-  return ts;
+    TokenStream *ts = ts_new(MultiByteTokenStream);
+    ts->reset = &mb_ts_reset;
+    ts->clone_i = &mb_ts_clone_i;
+    ts->ref_cnt = 1;
+    return ts;
 }
 
 /****************************************************************************
@@ -186,35 +193,40 @@ TokenStream *mb_ts_create()
  *
  ****************************************************************************/
 
-void a_deref(void *p)
+void a_deref(Analyzer *a)
 {
-  Analyzer *a = (Analyzer *)p;
-  if (--a->ref_cnt <= 0) a->destroy(a);
+    if (--a->ref_cnt <= 0) {
+        a->destroy_i(a);
+    }
 }
 
-void a_standard_destroy(Analyzer *a)
+static void a_standard_destroy_i(Analyzer *a)
 {
-  if (a->current_ts) ts_deref(a->current_ts);
-  free(a);
+    if (a->current_ts) {
+        ts_deref(a->current_ts);
+    }
+    free(a);
 }
 
-TokenStream *a_standard_get_ts(Analyzer *a, char *field, char *text)
+static TokenStream *a_standard_get_ts(Analyzer *a, char *field, char *text)
 {
-  a->current_ts->reset(a->current_ts, text);
-  return a->current_ts; 
+    TokenStream *ts;
+    (void)field;
+    ts = ts_clone(a->current_ts);
+    return ts->reset(ts, text);
 }
 
-Analyzer *analyzer_create(void *data, TokenStream *ts,
-    void (*destroy)(Analyzer *a),
-    TokenStream *(*get_ts)(Analyzer *a, char *field, char *text))
+Analyzer *analyzer_new(TokenStream *ts,
+                       void (*destroy_i)(Analyzer *a),
+                       TokenStream *(*get_ts)(Analyzer *a, char *field,
+                                              char *text))
 {
-  Analyzer *a = ALLOC(Analyzer);
-  a->data = data;
-  a->current_ts = ts;
-  a->destroy = (destroy ? destroy : &a_standard_destroy);
-  a->get_ts = (get_ts ? get_ts : &a_standard_get_ts);
-  a->ref_cnt = 1;
-  return a;
+    Analyzer *a = ALLOC(Analyzer);
+    a->current_ts = ts;
+    a->destroy_i = (destroy_i ? destroy_i : &a_standard_destroy_i);
+    a->get_ts = (get_ts ? get_ts : &a_standard_get_ts);
+    a->ref_cnt = 1;
+    return a;
 }
 
 /****************************************************************************
@@ -226,120 +238,132 @@ Analyzer *analyzer_create(void *data, TokenStream *ts,
 /*
  * WhitespaceTokenizer
  */
-Token *wst_next(TokenStream *ts)
+static Token *wst_next(TokenStream *ts)
 {
-  char *t = ts->t;
-  char *start;
+    char *t = ts->t;
+    char *start;
 
-  while (*t != '\0' && isspace(*t)) t++;
+    while (*t != '\0' && isspace(*t)) {
+        t++;
+    }
 
-  if (*t == '\0') return NULL;
+    if (*t == '\0') {
+        return NULL;
+    }
 
-  start = t;
-  while (*t != '\0' && !isspace(*t)) t++;
+    start = t;
+    while (*t != '\0' && !isspace(*t)) {
+        t++;
+    }
 
-  ts->t = t;
-  tk_set_ts(ts->token, start, t, ts->text, 1);
-  return ts->token;
+    ts->t = t;
+    return tk_set_ts(&(CTS(ts)->token), start, t, ts->text, 1);
 }
 
-TokenStream *whitespace_tokenizer_create()
+TokenStream *whitespace_tokenizer_new()
 {
-  TokenStream *ts = ts_create();
-  ts->next = &wst_next;
-  return ts;
+    TokenStream *ts = cts_new();
+    ts->next = &wst_next;
+    return ts;
 }
 
 /*
  * Multi-byte WhitespaceTokenizer
  */
-Token *mb_wst_next(TokenStream *ts)
-{
-  int i;
-  char *start;
-  char *t = ts->t;
-  wchar_t wchr;
+static Token *mb_wst_next(TokenStream *ts)
+{
+    int i;
+    char *start;
+    char *t = ts->t;
+    wchar_t wchr;
+    mbstate_t *state = &(MBTS(ts)->state);
+
+    i = mb_next_char(&wchr, t, state);
+    while (wchr != 0 && iswspace(wchr)) {
+        t += i;
+        i = mb_next_char(&wchr, t, state);
+    }
+    if (wchr == 0) {
+        return NULL;
+    }
 
-  MB_NEXT_CHAR;
-  while (wchr != 0 && iswspace(wchr)) {
+    start = t;
     t += i;
-    MB_NEXT_CHAR;
-  }
-  if (wchr == 0) return NULL;
-
-  start = t;
-  t += i;
-  MB_NEXT_CHAR;
-  while (wchr != 0 && !iswspace(wchr)) {
-    t += i;
-    MB_NEXT_CHAR;
-  }
-  tk_set_ts(ts->token, start, t, ts->text, 1);
-  ts->t = t;
-  return ts->token;
+    i = mb_next_char(&wchr, t, state);
+    while (wchr != 0 && !iswspace(wchr)) {
+        t += i;
+        i = mb_next_char(&wchr, t, state);
+    }
+    ts->t = t;
+    return tk_set_ts(&(CTS(ts)->token), start, t, ts->text, 1);
 }
 
 /*
  * Lowercasing Multi-byte WhitespaceTokenizer
  */
-Token *mb_wst_next_lc(TokenStream *ts)
-{
-  int i;
-  char *start;
-  char *t = ts->t;
-  wchar_t wchr;
-  wchar_t wbuf[MAX_WORD_SIZE+1], *w, *w_end;
-
-  w = wbuf;
-  w_end = &wbuf[MAX_WORD_SIZE];
+static Token *mb_wst_next_lc(TokenStream *ts)
+{
+    int i;
+    char *start;
+    char *t = ts->t;
+    wchar_t wchr;
+    wchar_t wbuf[MAX_WORD_SIZE + 1], *w, *w_end;
+    mbstate_t *state = &(MBTS(ts)->state);
+
+    w = wbuf;
+    w_end = &wbuf[MAX_WORD_SIZE];
+
+    i = mb_next_char(&wchr, t, state);
+    while (wchr != 0 && iswspace(wchr)) {
+        t += i;
+        i = mb_next_char(&wchr, t, state);
+    }
+    if (wchr == 0) {
+        return NULL;
+    }
 
-  MB_NEXT_CHAR;
-  while (wchr != 0 && iswspace(wchr)) {
+    start = t;
     t += i;
-    MB_NEXT_CHAR;
-  }
-  if (wchr == 0) return NULL;
-
-  start = t;
-  t += i;
-  *w++ = towlower(wchr);
-  MB_NEXT_CHAR;
-  while (wchr != 0 && !iswspace(wchr)) {
-    if (w < w_end) *w++ = towlower(wchr);
-    t += i;
-    MB_NEXT_CHAR;
-  }
-  *w = 0;
-  w_tk_set(ts->token, wbuf, (int)(start - ts->text), (int)(t - ts->text), 1);
-  ts->t = t;
-  return ts->token;
+    *w++ = towlower(wchr);
+    i = mb_next_char(&wchr, t, state);
+    while (wchr != 0 && !iswspace(wchr)) {
+        if (w < w_end) {
+            *w++ = towlower(wchr);
+        }
+        t += i;
+        i = mb_next_char(&wchr, t, state);
+    }
+    *w = 0;
+    ts->t = t;
+    return w_tk_set(&(CTS(ts)->token), wbuf, (int)(start - ts->text),
+                    (int)(t - ts->text), 1);
 }
 
-TokenStream *mb_whitespace_tokenizer_create(bool lowercase)
+TokenStream *mb_whitespace_tokenizer_new(bool lowercase)
 {
-  TokenStream *ts = mb_ts_create();
-  ts->next = lowercase ? &mb_wst_next_lc : &mb_wst_next;
-  return ts;
+    TokenStream *ts = mb_ts_new();
+    ts->next = lowercase ? &mb_wst_next_lc : &mb_wst_next;
+    return ts;
 }
 
 /*
  * WhitespaceAnalyzers
  */
-Analyzer *whitespace_analyzer_create(bool lowercase)
+Analyzer *whitespace_analyzer_new(bool lowercase)
 {
-  TokenStream *ts;
-  if (lowercase) {
-    ts = lowercase_filter_create(whitespace_tokenizer_create());
-  } else {
-    ts = whitespace_tokenizer_create();
-  }
-  return analyzer_create(NULL, ts, NULL, NULL);
+    TokenStream *ts;
+    if (lowercase) {
+        ts = lowercase_filter_new(whitespace_tokenizer_new());
+    }
+    else {
+        ts = whitespace_tokenizer_new();
+    }
+    return analyzer_new(ts, NULL, NULL);
 }
 
-Analyzer *mb_whitespace_analyzer_create(bool lowercase)
+Analyzer *mb_whitespace_analyzer_new(bool lowercase)
 {
-  return analyzer_create(NULL, mb_whitespace_tokenizer_create(lowercase),
-      NULL, NULL);
+    return analyzer_new(mb_whitespace_tokenizer_new(lowercase), NULL, NULL);
 }
 
 /****************************************************************************
@@ -353,26 +377,31 @@ Analyzer *mb_whitespace_analyzer_create(bool lowercase)
  */
 Token *lt_next(TokenStream *ts)
 {
-  char *start;
-  char *t = ts->t;
+    char *start;
+    char *t = ts->t;
 
-  while (*t != '\0' && !isalpha(*t)) t++;
+    while (*t != '\0' && !isalpha(*t)) {
+        t++;
+    }
 
-  if (*t == '\0') return NULL;
+    if (*t == '\0') {
+        return NULL;
+    }
 
-  start = t;
-  while (*t != '\0' && isalpha(*t)) t++;
+    start = t;
+    while (*t != '\0' && isalpha(*t)) {
+        t++;
+    }
 
-  tk_set_ts(ts->token, start, t, ts->text, 1);
-  ts->t = t;
-  return ts->token;
+    ts->t = t;
+    return tk_set_ts(&(CTS(ts)->token), start, t, ts->text, 1);
 }
 
-TokenStream *letter_tokenizer_create()
+TokenStream *letter_tokenizer_new()
 {
-  TokenStream *ts = ts_create();
-  ts->next = &lt_next;
-  return ts;
+    TokenStream *ts = cts_new();
+    ts->next = &lt_next;
+    return ts;
 }
 
 /*
@@ -380,28 +409,31 @@ TokenStream *letter_tokenizer_create()
  */
 Token *mb_lt_next(TokenStream *ts)
 {
-  int i;
-  char *start;
-  char *t = ts->t;
-  wchar_t wchr;
+    int i;
+    char *start;
+    char *t = ts->t;
+    wchar_t wchr;
+    mbstate_t *state = &(MBTS(ts)->state);
 
-  MB_NEXT_CHAR;
-  while (wchr != 0 && !iswalpha(wchr)) {
-    t += i;
-    MB_NEXT_CHAR;
-  }
-  if (wchr == 0) return NULL;
-
-  start = t;
-  t += i;
-  MB_NEXT_CHAR;
-  while (wchr != 0 && iswalpha(wchr)) {
+    i = mb_next_char(&wchr, t, state);
+    while (wchr != 0 && !iswalpha(wchr)) {
+        t += i;
+        i = mb_next_char(&wchr, t, state);
+    }
+
+    if (wchr == 0) {
+        return NULL;
+    }
+
+    start = t;
     t += i;
-    MB_NEXT_CHAR;
-  }
-  tk_set_ts(ts->token, start, t, ts->text, 1);
-  ts->t = t;
-  return ts->token;
+    i = mb_next_char(&wchr, t, state);
+    while (wchr != 0 && iswalpha(wchr)) {
+        t += i;
+        i = mb_next_char(&wchr, t, state);
+    }
+    ts->t = t;
+    return tk_set_ts(&(CTS(ts)->token), start, t, ts->text, 1);
 }
 
 /*
@@ -409,62 +441,67 @@ Token *mb_lt_next(TokenStream *ts)
  */
 Token *mb_lt_next_lc(TokenStream *ts)
 {
-  int i;
-  char *start;
-  char *t = ts->t;
-  wchar_t wchr;
-  wchar_t wbuf[MAX_WORD_SIZE+1], *w, *w_end;
+    int i;
+    char *start;
+    char *t = ts->t;
+    wchar_t wchr;
+    wchar_t wbuf[MAX_WORD_SIZE + 1], *w, *w_end;
+    mbstate_t *state = &(MBTS(ts)->state);
 
-  w = wbuf;
-  w_end = &wbuf[MAX_WORD_SIZE];
+    w = wbuf;
+    w_end = &wbuf[MAX_WORD_SIZE];
 
-  MB_NEXT_CHAR;
-  while (wchr != 0 && !iswalpha(wchr)) {
-    t += i;
-    MB_NEXT_CHAR;
-  }
-  if (wchr == 0) return NULL;
-
-  start = t;
-  t += i;
-  *w++ = towlower(wchr);
-  MB_NEXT_CHAR;
-  while (wchr != 0 && iswalpha(wchr)) {
-    if (w < w_end) *w++ = towlower(wchr);
+    i = mb_next_char(&wchr, t, state);
+    while (wchr != 0 && !iswalpha(wchr)) {
+        t += i;
+        i = mb_next_char(&wchr, t, state);
+    }
+    if (wchr == 0) {
+        return NULL;
+    }
+
+    start = t;
     t += i;
-    MB_NEXT_CHAR;
-  }
-  *w = 0;
-  w_tk_set(ts->token, wbuf, (int)(start - ts->text), (int)(t - ts->text), 1);
-  ts->t = t;
-  return ts->token;
+    *w++ = towlower(wchr);
+    i = mb_next_char(&wchr, t, state);
+    while (wchr != 0 && iswalpha(wchr)) {
+        if (w < w_end) {
+            *w++ = towlower(wchr);
+        }
+        t += i;
+        i = mb_next_char(&wchr, t, state);
+    }
+    *w = 0;
+    ts->t = t;
+    return w_tk_set(&(CTS(ts)->token), wbuf, (int)(start - ts->text),
+                    (int)(t - ts->text), 1);
 }
 
-TokenStream *mb_letter_tokenizer_create(bool lowercase)
+TokenStream *mb_letter_tokenizer_new(bool lowercase)
 {
-  TokenStream *ts = mb_ts_create();
-  ts->next = lowercase ? &mb_lt_next_lc : &mb_lt_next;
-  return ts;
+    TokenStream *ts = mb_ts_new();
+    ts->next = lowercase ? &mb_lt_next_lc : &mb_lt_next;
+    return ts;
 }
 
 /*
  * LetterAnalyzers
  */
-Analyzer *letter_analyzer_create(bool lowercase)
+Analyzer *letter_analyzer_new(bool lowercase)
 {
-  TokenStream *ts;
-  if (lowercase) {
-    ts = lowercase_filter_create(letter_tokenizer_create());
-  } else {
-    ts = letter_tokenizer_create();
-  }
-  return analyzer_create(NULL, ts, NULL, NULL);
+    TokenStream *ts;
+    if (lowercase) {
+        ts = lowercase_filter_new(letter_tokenizer_new());
+    }
+    else {
+        ts = letter_tokenizer_new();
+    }
+    return analyzer_new(ts, NULL, NULL);
 }
 
-Analyzer *mb_letter_analyzer_create(bool lowercase)
+Analyzer *mb_letter_analyzer_new(bool lowercase)
 {
-  return analyzer_create(NULL,
-      mb_letter_tokenizer_create(lowercase), NULL, NULL);
+    return analyzer_new(mb_letter_tokenizer_new(lowercase), NULL, NULL);
 }
 
 /****************************************************************************
@@ -473,115 +510,146 @@ Analyzer *mb_letter_analyzer_create(bool lowercase)
  *
  ****************************************************************************/
 
+#define STDTS(token_stream) ((StandardTokenizer *)(token_stream))
+
 /*
  * StandardTokenizer
  */
-int std_get_alpha(TokenStream *ts, char *token)
+static int std_get_alpha(TokenStream *ts, char *token)
 {
-  int i = 0;
-  char *t = ts->t;
-  while (t[i] != '\0' && isalpha(t[i])) {
-    if (i < MAX_WORD_SIZE) token[i] = t[i];
-    i++;
-  }
-  return i;
+    int i = 0;
+    char *t = ts->t;
+    while (t[i] != '\0' && isalpha(t[i])) {
+        if (i < MAX_WORD_SIZE) {
+            token[i] = t[i];
+        }
+        i++;
+    }
+    return i;
 }
 
-int mb_std_get_alpha(TokenStream *ts, char *token)
+static int mb_std_get_alpha(TokenStream *ts, char *token)
 {
-  char *t = ts->t;
-  wchar_t w;
-  int i;
-  if ((i = mbtowc(&w, t, MB_CUR_MAX)) < 0) RAISE(IO_ERROR, ENC_ERR_MSG);
-  while (w != 0 && iswalpha(w)) {
-    t += i;
-    if ((i = mbtowc(&w, t, MB_CUR_MAX)) < 0) RAISE(IO_ERROR, ENC_ERR_MSG);
-  }
+    char *t = ts->t;
+    wchar_t wchr;
+    int i;
+    mbstate_t state; ZEROSET(&state, mbstate_t);
+
+    i = mb_next_char(&wchr, t, &state);
 
-  i = (int)(t - ts->t);
-  if (i > MAX_WORD_SIZE) i = MAX_WORD_SIZE - 1;
-  memcpy(token, ts->t, i);
-  return i;
+    while (wchr != 0 && iswalpha(wchr)) {
+        t += i;
+        i = mb_next_char(&wchr, t, &state);
+    }
+
+    i = (int)(t - ts->t);
+    if (i > MAX_WORD_SIZE) {
+        i = MAX_WORD_SIZE - 1;
+    }
+    memcpy(token, ts->t, i);
+    return i;
 }
 
-int std_get_alnum(TokenStream *ts, char *token)
+/*
+static int std_get_alnum(TokenStream *ts, char *token)
 {
-  int i = 0;
-  char *t = ts->t;
-  while (t[i] != '\0' && isalnum(t[i])) {
-    if (i < MAX_WORD_SIZE) token[i] = t[i];
-    i++;
-  }
-  return i;
+    int i = 0;
+    char *t = ts->t;
+    while (t[i] != '\0' && isalnum(t[i])) {
+        if (i < MAX_WORD_SIZE) {
+            token[i] = t[i];
+        }
+        i++;
+    }
+    return i;
 }
 
-int mb_std_get_alnum(char *text, char *token, TokenStream *ts)
+static int mb_std_get_alnum(TokenStream *ts, char *token)
 {
-  char *t = ts->t;
-  wchar_t w;
-  int i;
-  if ((i = mbtowc(&w, t, MB_CUR_MAX)) < 0) RAISE(IO_ERROR, ENC_ERR_MSG);
-  while (w != 0 && iswalnum(w)) {
-    t += i;
-    if ((i = mbtowc(&w, t, MB_CUR_MAX)) < 0) RAISE(IO_ERROR, ENC_ERR_MSG);
-  }
+    char *t = ts->t;
+    wchar_t wchr;
+    int i;
+    mbstate_t state; ZEROSET(&state, mbstate_t);
 
-  i = (int)(t - ts->t);
-  if (i > MAX_WORD_SIZE) i = MAX_WORD_SIZE - 1;
-  memcpy(token, ts->t, i);
-  return i;
+    i = mb_next_char(&wchr, t, &state);
 
+    while (wchr != 0 && iswalnum(wchr)) {
+        t += i;
+        i = mb_next_char(&wchr, t, &state);
+    }
+
+    i = (int)(t - ts->t);
+    if (i > MAX_WORD_SIZE) {
+        i = MAX_WORD_SIZE - 1;
+    }
+    memcpy(token, ts->t, i);
+    return i;
 }
+*/
 
-int isnumpunc(char c)
+static int isnumpunc(char c)
 {
-  return (c == '.' || c == ',' || c == '\\' || c == '/' || c == '_' || c == '-');
+    return (c == '.' || c == ',' || c == '\\' || c == '/' || c == '_'
+            || c == '-');
 }
 
-int w_isnumpunc(wchar_t c)
+static int w_isnumpunc(wchar_t c)
 {
-  return (c == L'.' || c == L',' || c == L'\\' || c == L'/' || c == L'_' || c == L'-');
+    return (c == L'.' || c == L',' || c == L'\\' || c == L'/' || c == L'_'
+            || c == L'-');
 }
 
-int isurlpunc(char c)
+static int isurlpunc(char c)
 {
-  return (c == '.' || c == '/' || c == '-' || c == '_');
+    return (c == '.' || c == '/' || c == '-' || c == '_');
 }
 
-int isurlc(char c)
+static int isurlc(char c)
 {
-  return (c == '.' || c == '/' || c == '-' || c == '_' || isalnum(c));
+    return (c == '.' || c == '/' || c == '-' || c == '_' || isalnum(c));
 }
 
-int isurlxatpunc(char c)
+static int isurlxatpunc(char c)
 {
-  return (c == '.' || c == '/' || c == '-' || c == '_' || c == '@');
+    return (c == '.' || c == '/' || c == '-' || c == '_' || c == '@');
 }
 
-int isurlxatc(char c)
+static int isurlxatc(char c)
 {
-  return (c == '.' || c == '/' || c == '-' || c == '_' || c == '@' || isalnum(c));
+    return (c == '.' || c == '/' || c == '-' || c == '_' || c == '@'
+            || isalnum(c));
 }
 
-bool std_is_tok_char(char *c)
+static bool std_is_tok_char(char *c)
 {
-  if (isspace(*c)) return false; // most common so check first.
-  if (isalnum(*c) || isnumpunc(*c) || *c == '&' ||
-      *c == '@' || *c == '\'' || *c == ':')
-    return true;
-  return false;
+    if (isspace(*c)) {
+        return false;           /* most common so check first. */
+    }
+    if (isalnum(*c) || isnumpunc(*c) || *c == '&' ||
+        *c == '@' || *c == '\'' || *c == ':') {
+        return true;
+    }
+    return false;
 }
 
-bool w_std_is_tok_char(char *t)
+static bool mb_std_is_tok_char(char *t)
 {
-  wchar_t c;
-  if ((mbtowc(&c, t, MB_CUR_MAX)) < 0)
-    RAISE(IO_ERROR, ENC_ERR_MSG);
-  if (iswspace(c)) return false; // most common so check first.
-  if (iswalnum(c) || w_isnumpunc(c) || c == L'&' ||
-      c == L'@' || c == L'\'' || c == L':')
-    return true;
-  return false;
+    wchar_t c;
+    mbstate_t state; ZEROSET(&state, mbstate_t);
+    
+    if (((int)mbrtowc(&c, t, MB_CUR_MAX, &state)) < 0) {
+        /* error which we can handle next time round. For now just return
+         * false so that we can return a token */
+        return false;
+    }
+    if (iswspace(c)) {
+        return false;           /* most common so check first. */
+    }
+    if (iswalnum(c) || w_isnumpunc(c) || c == L'&' || c == L'@' || c == L'\''
+        || c == L':') {
+        return true;
+    }
+    return false;
 }
 
 /* (alnum)((punc)(alnum))+ where every second sequence of alnum must contain at
@@ -589,583 +657,669 @@ bool w_std_is_tok_char(char *t)
  * (alnum) = [a-zA-Z0-9]
  * (punc) = [_\/.,-]
  */
-int std_get_number(char *input)
-{
-  int i = 0;
-  int count = 0;
-  int last_seen_digit = 2;
-  int seen_digit = false;
-
-  while (last_seen_digit >= 0) {
-    while ((input[i] != '\0') && isalnum(input[i])) {
-      if ((last_seen_digit < 2) && isdigit(input[i])) last_seen_digit = 2;
-      if ((seen_digit == false) && isdigit(input[i])) seen_digit = true;
-      i++;
-    }
-    last_seen_digit--;
-    if (!isnumpunc(input[i]) || !isalnum(input[i+1])) {
-      
-      if (last_seen_digit >= 0)
+static int std_get_number(char *input)
+{
+    int i = 0;
+    int count = 0;
+    int last_seen_digit = 2;
+    int seen_digit = false;
+
+    while (last_seen_digit >= 0) {
+        while ((input[i] != '\0') && isalnum(input[i])) {
+            if ((last_seen_digit < 2) && isdigit(input[i])) {
+                last_seen_digit = 2;
+            }
+            if ((seen_digit == false) && isdigit(input[i])) {
+                seen_digit = true;
+            }
+            i++;
+        }
+        last_seen_digit--;
+        if (!isnumpunc(input[i]) || !isalnum(input[i + 1])) {
+
+            if (last_seen_digit >= 0) {
+                count = i;
+            }
+            break;
+        }
         count = i;
-      break;
-    }
-    count = i;
-    i++;
-  }
-  if (seen_digit)
-    return count;
-  else
-    return 0;
+        i++;
+    }
+    if (seen_digit) {
+        return count;
+    }
+    else {
+        return 0;
+    }
 }
 
-int std_get_apostrophe(char *input)
+static int std_get_apostrophe(char *input)
 {
-  char *t = input;
+    char *t = input;
 
-  while (isalpha(*t) || *t == '\'')
-    t++;
+    while (isalpha(*t) || *t == '\'') {
+        t++;
+    }
 
-  return (int)(t - input);
+    return (int)(t - input);
 }
 
-int mb_std_get_apostrophe(char *input)
+static int mb_std_get_apostrophe(char *input)
 {
-  char *t = input;
-  wchar_t w;
-  int i;
+    char *t = input;
+    wchar_t wchr;
+    int i;
+    mbstate_t state; ZEROSET(&state, mbstate_t);
 
-  if ((i = mbtowc(&w, t, MB_CUR_MAX)) < 0) RAISE(IO_ERROR, ENC_ERR_MSG);
-  while (iswalpha(w) || w == L'\'') {
-    t += i;
-    if ((i = mbtowc(&w, t, MB_CUR_MAX)) < 0) RAISE(IO_ERROR, ENC_ERR_MSG);
-  }
-  return (int)(t - input);
+    i = mb_next_char(&wchr, t, &state);
+
+    while (iswalpha(wchr) || wchr == L'\'') {
+        t += i;
+        i = mb_next_char(&wchr, t, &state);
+    }
+    return (int)(t - input);
 }
 
-int std_get_url(char *input, char *token, int i)
+static int std_get_url(char *input, char *token, int i)
 {
-  while (isurlc(input[i])) {
-    if (isurlpunc(input[i]) && isurlpunc(input[i-1]))
-      break; // can't have to puncs in a row
-    if (i < MAX_WORD_SIZE) token[i] = input[i];
-    i++;
-  }
+    while (isurlc(input[i])) {
+        if (isurlpunc(input[i]) && isurlpunc(input[i - 1])) {
+            break;              /* can't have to puncs in a row */
+        }
+        if (i < MAX_WORD_SIZE) {
+            token[i] = input[i];
+        }
+        i++;
+    }
 
-  //strip trailing puncs
-  while (isurlpunc(input[i-1])) i--;
+    /* strip trailing puncs */
+    while (isurlpunc(input[i - 1])) {
+        i--;
+    }
 
-  return i;
+    return i;
 }
 
 /* Company names can contain '@' and '&' like AT&T and Excite@Home. Let's
- */
-int std_get_company_name(char *input)
+*/
+static int std_get_company_name(char *input)
 {
-  int i = 0;
-  while (isalpha(input[i]) || input[i] == '@' || input[i] == '&')
-    i++;
+    int i = 0;
+    while (isalpha(input[i]) || input[i] == '@' || input[i] == '&') {
+        i++;
+    }
 
-  return i;
+    return i;
 }
 
-int mb_std_get_company_name(char *input, TokenStream *ts)
+/*
+static int mb_std_get_company_name(char *input, TokenStream *ts)
 {
-  char *t = input;
-  wchar_t wchr;
-  int i;
+    char *t = input;
+    wchar_t wchr;
+    int i;
+    mbstate_t state; ZEROSET(&state, mbstate_t);
 
-  MB_NEXT_CHAR;
-  while (iswalpha(wchr) || wchr == L'@' || wchr == L'&') {
-    t += i;
-    MB_NEXT_CHAR;
-  }
+    i = mb_next_char(&wchr, t, &state);
+    while (iswalpha(wchr) || wchr == L'@' || wchr == L'&') {
+        t += i;
+        i = mb_next_char(&wchr, t, &state);
+    }
 
-  return (int)(t - input);
+    return (int)(t - input);
 }
+*/
 
-bool std_advance_to_start(TokenStream *ts)
+static bool std_advance_to_start(TokenStream *ts)
 {
-  char *t = ts->t;
-  while (*t != '\0' && !isalnum(*t)) t++;
+    char *t = ts->t;
+    while (*t != '\0' && !isalnum(*t)) {
+        t++;
+    }
 
-  ts->t = t;
+    ts->t = t;
 
-  return (*t != '\0');
+    return (*t != '\0');
 }
 
-bool mb_std_advance_to_start(TokenStream *ts)
+static bool mb_std_advance_to_start(TokenStream *ts)
 {
-  int i;
-  wchar_t w;
+    int i;
+    wchar_t wchr;
+    mbstate_t state; ZEROSET(&state, mbstate_t);
 
-  if ((i = mbtowc(&w, ts->t, MB_CUR_MAX)) < 0) RAISE(IO_ERROR, ENC_ERR_MSG);
-  while (w != 0 && !iswalnum(w)) {
-    ts->t += i;
-    if ((i = mbtowc(&w, ts->t, MB_CUR_MAX)) < 0) RAISE(IO_ERROR, ENC_ERR_MSG);
-  }
+    i = mb_next_char(&wchr, ts->t, &state);
 
-  return (w != 0);
-}
+    while (wchr != 0 && !iswalnum(wchr)) {
+        ts->t += i;
+        i = mb_next_char(&wchr, ts->t, &state);
+    }
 
-typedef struct StandardTokenizer {
-  bool (*advance_to_start)(TokenStream *ts);
-  bool (*is_tok_char)(char *c);
-  int (*get_alpha)(TokenStream *ts, char *token);
-  int (*get_apostrophe)(char *input);
-} StandardTokenizer;
+    return (wchr != 0);
+}
 
-Token *std_next(TokenStream *ts)
+static Token *std_next(TokenStream *ts)
 {
-  StandardTokenizer *std_tz = (StandardTokenizer *)ts->data;
-  char *s;
-  char *t;
-  char *start = NULL;
-  char *num_end = NULL;
-  char token[MAX_WORD_SIZE];
-  int token_i = 0;
-  int len;
-  bool is_acronym;
-  bool seen_at_symbol;
-  
+    StandardTokenizer *std_tz = STDTS(ts);
+    char *s;
+    char *t;
+    char *start = NULL;
+    char *num_end = NULL;
+    char token[MAX_WORD_SIZE];
+    int token_i = 0;
+    int len;
+    bool is_acronym;
+    bool seen_at_symbol;
 
-  if (!std_tz->advance_to_start(ts)) return NULL;
 
-  start = t = ts->t;
-  if (isdigit(*t)) {
-    t += std_get_number(t);
-    ts->t = t;
-    tk_set_ts(ts->token, start, t, ts->text, 1);
-  } else {
-    token_i = std_tz->get_alpha(ts, token);
-    t += token_i;
-
-    if (!std_tz->is_tok_char(t)) {
-      // very common case, ie a plain word, so check and return
-      tk_set_ts(ts->token, start, t, ts->text, 1);
-      ts->t = t;
-      return ts->token;
-    }
-
-    if (*t == '\'') { // apostrophe case. 
-      t += std_tz->get_apostrophe(t);
-      ts->t = t;
-      len = (int)(t - start);
-      // strip possesive
-      if ((t[-1] == 's' || t[-1] == 'S') && t[-2] == '\'') t -= 2;
-
-      tk_set_ts(ts->token, start, t, ts->text, 1);
-      return ts->token;
-    }
-
-    if (*t == '&') { // apostrophe case. 
-      t += std_get_company_name(t);
-      ts->t = t;
-      tk_set_ts(ts->token, start, t, ts->text, 1);
-      return ts->token;
-    }
-
-    if (isdigit(*t) || isnumpunc(*t)) { // possibly a number
-      num_end = start + std_get_number(start);
-      if (!std_tz->is_tok_char(num_end)) { // we won't find a longer token
-        ts->t = num_end;
-        tk_set_ts(ts->token, start, num_end, ts->text, 1);
-        return ts->token;
-      }
-      // else there may be a longer token so check
-    }
-
-    if (t[0] == ':' && t[1] == '/' && t[2] == '/') {
-      // check for a known url start
-      token[token_i] = '\0';
-      t += 3;
-      while (*t == '/') t++;
-      if (isalpha(*t) &&
-          (memcmp(token, "ftp", 3) == 0 ||
-           memcmp(token, "http", 4) == 0 ||
-           memcmp(token, "https", 5) == 0 ||
-           memcmp(token, "file", 4) == 0)) {
-        len = std_get_url(t, token, 0); // dispose of first part of the URL
-      } else { //still treat as url but keep the first part
-        token_i = (int)(t - start);
-        memcpy(token, start, token_i * sizeof(char));
-        len = token_i + std_get_url(t, token, token_i); // keep start
-      }
-      ts->t = t + len;
-      token[len] = 0;
-      tk_set(ts->token, token, len, (int)(start - ts->text), 
-		  (int)(ts->t - ts->text), 1);
-      return ts->token;
-    }
-
-    // now see how long a url we can find.
-    is_acronym = true;
-    seen_at_symbol = false;
-    while (isurlxatc(*t)) {
-      if (is_acronym && !isalpha(*t) && (*t != '.')) {
-        is_acronym = false;
-      }
-      if (isurlxatpunc(*t) && isurlxatpunc(t[-1])) {
-        break; // can't have two punctuation characters in a row
-      }
-      if (*t == '@') {
-        if (seen_at_symbol) {
-          break; // we can only have one @ symbol
-        } else {
-          seen_at_symbol = true;
-        }
-      }
-      t++;
+    if (!std_tz->advance_to_start(ts)) {
+        return NULL;
     }
-    while (isurlxatpunc(t[-1])) t--; // strip trailing punctuation
 
-    if (t > num_end) {
-      ts->t = t;
+    start = t = ts->t;
+    if (isdigit(*t)) {
+        t += std_get_number(t);
+        ts->t = t;
+        tk_set_ts(&(CTS(ts)->token), start, t, ts->text, 1);
+    }
+    else {
+        token_i = std_tz->get_alpha(ts, token);
+        t += token_i;
+
+        if (!std_tz->is_tok_char(t)) {
+            /* very common case, ie a plain word, so check and return */
+            ts->t = t;
+            return tk_set_ts(&(CTS(ts)->token), start, t, ts->text, 1);
+        }
+
+        if (*t == '\'') {       /* apostrophe case. */
+            t += std_tz->get_apostrophe(t);
+            ts->t = t;
+            len = (int)(t - start);
+            /* strip possesive */
+            if ((t[-1] == 's' || t[-1] == 'S') && t[-2] == '\'') {
+                t -= 2;
+            }
 
-      if (is_acronym) { // check that it is one letter followed by one '.'
-        for (s = start; s < t-1; s++) {
-          if (isalpha(*s) && (s[1] != '.')) is_acronym = false;
+            return tk_set_ts(&(CTS(ts)->token), start, t, ts->text, 1);
         }
-      }
-      if (is_acronym) {// strip '.'s
-        for (s = start + token_i; s < t; s++) {
-          if (*s != '.') {
-            token[token_i] = *s;
-            token_i++;
-          }
+
+        if (*t == '&') {        /* apostrophe case. */
+            t += std_get_company_name(t);
+            ts->t = t;
+            return tk_set_ts(&(CTS(ts)->token), start, t, ts->text, 1);
+        }
+
+        if (isdigit(*t) || isnumpunc(*t)) {      /* possibly a number */
+            num_end = start + std_get_number(start);
+            if (!std_tz->is_tok_char(num_end)) { /* won't find a longer token */
+                ts->t = num_end;
+                return tk_set_ts(&(CTS(ts)->token), start, num_end, ts->text, 1);
+            }
+            /* else there may be a longer token so check */
+        }
+
+        if (t[0] == ':' && t[1] == '/' && t[2] == '/') {
+            /* check for a known url start */
+            token[token_i] = '\0';
+            t += 3;
+            while (*t == '/') {
+                t++;
+            }
+            if (isalpha(*t) &&
+                (memcmp(token, "ftp", 3) == 0 ||
+                 memcmp(token, "http", 4) == 0 ||
+                 memcmp(token, "https", 5) == 0 ||
+                 memcmp(token, "file", 4) == 0)) {
+                len = std_get_url(t, token, 0); /* dispose of first part of the URL */
+            }
+            else {              /* still treat as url but keep the first part */
+                token_i = (int)(t - start);
+                memcpy(token, start, token_i * sizeof(char));
+                len = token_i + std_get_url(t, token, token_i); /* keep start */
+            }
+            ts->t = t + len;
+            token[len] = 0;
+            return tk_set(&(CTS(ts)->token), token, len, (int)(start - ts->text),
+                   (int)(ts->t - ts->text), 1);
+        }
+
+        /* now see how long a url we can find. */
+        is_acronym = true;
+        seen_at_symbol = false;
+        while (isurlxatc(*t)) {
+            if (is_acronym && !isalpha(*t) && (*t != '.')) {
+                is_acronym = false;
+            }
+            if (isurlxatpunc(*t) && isurlxatpunc(t[-1])) {
+                break; /* can't have two punctuation characters in a row */
+            }
+            if (*t == '@') {
+                if (seen_at_symbol) {
+                    break; /* we can only have one @ symbol */
+                }
+                else {
+                    seen_at_symbol = true;
+                }
+            }
+            t++;
+        }
+        while (isurlxatpunc(t[-1])) {
+            t--;                /* strip trailing punctuation */
+        }
+
+        if (t > num_end) {
+            ts->t = t;
+
+            if (is_acronym) {   /* check it is one letter followed by one '.' */
+                for (s = start; s < t - 1; s++) {
+                    if (isalpha(*s) && (s[1] != '.'))
+                        is_acronym = false;
+                }
+            }
+            if (is_acronym) {   /* strip '.'s */
+                for (s = start + token_i; s < t; s++) {
+                    if (*s != '.') {
+                        token[token_i] = *s;
+                        token_i++;
+                    }
+                }
+                tk_set(&(CTS(ts)->token), token, token_i,
+                       (int)(start - ts->text),
+                       (int)(t - ts->text), 1);
+            }
+            else { /* just return the url as is */
+                tk_set_ts(&(CTS(ts)->token), start, t, ts->text, 1);
+            }
+        }
+        else {                  /* return the number */
+            ts->t = num_end;
+            tk_set_ts(&(CTS(ts)->token), start, num_end, ts->text, 1);
         }
-        tk_set(ts->token, token, token_i, (int)(start - ts->text), 
-			(int)(t - ts->text), 1);
-      } else { // just return the url as is
-        tk_set_ts(ts->token, start, t, ts->text, 1);
-      }
-    } else { // return the number
-      ts->t = num_end;
-      tk_set_ts(ts->token, start, num_end, ts->text, 1);
     }
-  }
 
-  return ts->token;
+    return &(CTS(ts)->token);
 }
 
-void std_ts_destroy(TokenStream *ts)
+static TokenStream *std_ts_clone_i(TokenStream *orig_ts)
 {
-  free(ts->data);
-  ts_standard_destroy(ts);
+    return ts_clone_size(orig_ts, sizeof(StandardTokenizer));
 }
 
-void std_ts_clone_i(TokenStream *orig_ts, TokenStream *new_ts)
+static TokenStream *std_ts_new()
 {
-  new_ts->data = ALLOC(StandardTokenizer);
-  memcpy(new_ts->data, orig_ts->data, sizeof(StandardTokenizer));
+    TokenStream *ts = ts_new(StandardTokenizer);
+
+    ts->clone_i     = &std_ts_clone_i;
+    ts->next        = &std_next;
+
+    return ts;
 }
 
-TokenStream *standard_tokenizer_create()
+TokenStream *standard_tokenizer_new()
 {
-  TokenStream *ts = ts_create();
+    TokenStream *ts = std_ts_new();
 
-  StandardTokenizer *std_tz = ALLOC(StandardTokenizer);
-  std_tz->advance_to_start = &std_advance_to_start;
-  std_tz->get_alpha = &std_get_alpha;
-  std_tz->is_tok_char = &std_is_tok_char;
-  std_tz->get_apostrophe = &std_get_apostrophe;
+    STDTS(ts)->advance_to_start = &std_advance_to_start;
+    STDTS(ts)->get_alpha        = &std_get_alpha;
+    STDTS(ts)->is_tok_char      = &std_is_tok_char;
+    STDTS(ts)->get_apostrophe   = &std_get_apostrophe;
 
-  ts->data = std_tz;
-  ts->destroy = &std_ts_destroy;
-  ts->clone_i = &std_ts_clone_i;
-  ts->next = &std_next;
-  return ts;
+    return ts;
 }
 
-TokenStream *mb_standard_tokenizer_create()
+TokenStream *mb_standard_tokenizer_new()
 {
-  TokenStream *ts = ts_create();
+    TokenStream *ts = std_ts_new();
 
-  StandardTokenizer *std_tz = ALLOC(StandardTokenizer);
-  std_tz->advance_to_start = &mb_std_advance_to_start;
-  std_tz->get_alpha = &mb_std_get_alpha;
-  std_tz->is_tok_char = &w_std_is_tok_char;
-  std_tz->get_apostrophe = &mb_std_get_apostrophe;
+    STDTS(ts)->advance_to_start = &mb_std_advance_to_start;
+    STDTS(ts)->get_alpha        = &mb_std_get_alpha;
+    STDTS(ts)->is_tok_char      = &mb_std_is_tok_char;
+    STDTS(ts)->get_apostrophe   = &mb_std_get_apostrophe;
 
-  ts->data = std_tz;
-  ts->destroy = &std_ts_destroy;
-  ts->clone_i = &std_ts_clone_i;
-  ts->next = &std_next;
-  return ts;
+    return ts;
 }
 
-void filter_reset(TokenStream *ts, char *text)
+/****************************************************************************
+ *
+ * Filters
+ *
+ ****************************************************************************/
+
+#define TkFilt(filter) ((TokenFilter *)(filter))
+
+TokenStream *filter_clone_size(TokenStream *ts, size_t size)
 {
-  ts->sub_ts->reset(ts->sub_ts, text);
+    TokenStream *ts_new = ts_clone_size(ts, size);
+    TkFilt(ts_new)->sub_ts = TkFilt(ts)->sub_ts->clone_i(TkFilt(ts)->sub_ts);
+    return ts_new;
 }
 
-void filter_destroy(TokenStream *tf)
+static TokenStream *filter_clone_i(TokenStream *ts)
 {
-  ts_deref(tf->sub_ts);
-  if (tf->token != NULL) tk_destroy(tf->token);
-  free(tf);
+    return filter_clone_size(ts, sizeof(TokenFilter));
 }
 
-void sf_destroy(TokenStream *tf)
+static TokenStream *filter_reset(TokenStream *ts, char *text)
+{
+    TkFilt(ts)->sub_ts->reset(TkFilt(ts)->sub_ts, text);
+    return ts;
+}
+
+static void filter_destroy_i(TokenStream *ts)
+{
+    ts_deref(TkFilt(ts)->sub_ts);
+    free(ts);
+}
+
+#define tf_new(type, sub) tf_new_i(sizeof(type), sub)
+TokenStream *tf_new_i(size_t size, TokenStream *sub_ts)
+{
+    TokenStream *ts     = (TokenStream *)ecalloc(size);
+
+    TkFilt(ts)->sub_ts  = sub_ts;
+
+    ts->clone_i         = &filter_clone_i;
+    ts->destroy_i       = &filter_destroy_i;
+    ts->reset           = &filter_reset;
+    ts->ref_cnt         = 1;
+
+    return ts;
+}
+
+/****************************************************************************
+ * StopFilter
+ ****************************************************************************/
+
+#define StopFilt(filter) ((StopFilter *)(filter))
+
+static void sf_destroy_i(TokenStream *ts)
 {
-  HshTable *words = (HshTable *)tf->data;
-  h_destroy(words);
-  filter_destroy(tf);
+    h_destroy(StopFilt(ts)->words);
+    filter_destroy_i(ts);
 }
 
-void sf_clone_i_i(void *key, void *value, void *arg)
+static void sf_clone_i_i(void *key, void *value, void *arg)
 {
-  HshTable *wordtable = (HshTable *)arg;
-  char *w = estrdup(key);
-  h_set(wordtable, w, w);
+    HashTable *word_table = (HashTable *)arg;
+    char *word = estrdup(key);
+    (void)value;
+    h_set(word_table, word, word);
 }
 
-void sf_clone_i(TokenStream *orig_ts, TokenStream *new_ts)
+static TokenStream *sf_clone_i(TokenStream *orig_ts)
 {
-  new_ts->data = h_new_str(&free, NULL);
-  h_each(orig_ts->data, &sf_clone_i_i, new_ts->data);
+    TokenStream *new_ts = filter_clone_size(orig_ts, sizeof(StopFilter));
+    StopFilt(new_ts)->words = h_new_str(&free, NULL);
+    h_each(StopFilt(orig_ts)->words, &sf_clone_i_i, StopFilt(new_ts)->words);
+    return new_ts;
 }
 
-Token *sf_next(TokenStream *tf)
+static Token *sf_next(TokenStream *ts)
 {
-  int pos_inc = 1;
-  HshTable *words = (HshTable *)tf->data;
-  Token *tk = tf->sub_ts->next(tf->sub_ts);
-  while ((tk != NULL) && (h_get(words, tk->text) != NULL)) {
-    tk = tf->sub_ts->next(tf->sub_ts);
-    pos_inc++;
-  }
-  if (tk != NULL) tk->pos_inc = pos_inc;
-  return tk;
+    int pos_inc = 1;
+    HashTable *words = StopFilt(ts)->words;
+    TokenFilter *tf = TkFilt(ts);
+    Token *tk = tf->sub_ts->next(tf->sub_ts);
+
+    while ((tk != NULL) && (h_get(words, tk->text) != NULL)) {
+        tk = tf->sub_ts->next(tf->sub_ts);
+        pos_inc++;
+    }
+
+    if (tk != NULL) {
+        tk->pos_inc = pos_inc;
+    }
+
+    return tk;
 }
 
-TokenStream *stop_filter_create_with_words_len(TokenStream *ts,
-    const char **words, int len)
+TokenStream *stop_filter_new_with_words_len(TokenStream *sub_ts,
+                                            const char **words, int len)
 {
-  int i;
-  char *w;
-  HshTable *wordtable = h_new_str(&free, (free_ft)NULL);
-  TokenStream *tf = ALLOC(TokenStream);
-  tf->sub_ts = ts;
+    int i;
+    char *word;
+    HashTable *word_table = h_new_str(&free, (free_ft) NULL);
+    TokenStream *ts = tf_new(StopFilter, sub_ts);
 
-  for (i = 0; i < len; i++) {
-    w = estrdup(words[i]);
-    h_set(wordtable, w, w);
-  }
-  tf->data = wordtable;
-  tf->token = NULL;
-  tf->next = &sf_next;
-  tf->reset = &filter_reset;
-  tf->destroy = &sf_destroy;
-  tf->clone_i = &sf_clone_i;
-  tf->ref_cnt = 1;
-  return tf;
+    for (i = 0; i < len; i++) {
+        word = estrdup(words[i]);
+        h_set(word_table, word, word);
+    }
+    StopFilt(ts)->words = word_table;
+    ts->next            = &sf_next;
+    ts->destroy_i       = &sf_destroy_i;
+    ts->clone_i         = &sf_clone_i;
+    return ts;
 }
 
-TokenStream *stop_filter_create_with_words(TokenStream *ts, const char **words)
+TokenStream *stop_filter_new_with_words(TokenStream *sub_ts,
+                                        const char **words)
 {
-  char *w;
-  HshTable *wordtable = h_new_str(&free, (free_ft)NULL);
-  TokenStream *tf = ALLOC(TokenStream);
-  tf->sub_ts = ts;
-  while (*words) {
-    w = estrdup(*words);
-    h_set(wordtable, w, w);
-    words++;
-  }
-  tf->data = wordtable;
-  tf->token = NULL;
-  tf->next = &sf_next;
-  tf->reset = &filter_reset;
-  tf->destroy = &sf_destroy;
-  tf->clone_i = &sf_clone_i;
-  tf->ref_cnt = 1;
-  return tf;
+    char *word;
+    HashTable *word_table = h_new_str(&free, (free_ft) NULL);
+    TokenStream *ts = tf_new(StopFilter, sub_ts);
+
+    while (*words) {
+        word = estrdup(*words);
+        h_set(word_table, word, word);
+        words++;
+    }
+
+    StopFilt(ts)->words = word_table;
+    ts->next            = &sf_next;
+    ts->destroy_i       = &sf_destroy_i;
+    ts->clone_i         = &sf_clone_i;
+    return ts;
 }
 
-TokenStream *stop_filter_create(TokenStream *ts)
+TokenStream *stop_filter_new(TokenStream *ts)
 {
-  return stop_filter_create_with_words(ts, FULL_ENGLISH_STOP_WORDS);
+    return stop_filter_new_with_words(ts, FULL_ENGLISH_STOP_WORDS);
 }
 
+/****************************************************************************
+ * LowerCaseFilter
+ ****************************************************************************/
+
+
 Token *mb_lcf_next(TokenStream *ts)
 {
-  wchar_t wbuf[MAX_WORD_SIZE], *w;
-  //mbstate_t state = {0};
-  int i;
-  Token *tk = ts->sub_ts->next(ts->sub_ts);
-  if (tk == NULL) return tk;
+    wchar_t wbuf[MAX_WORD_SIZE], *wchr;
+    Token *tk = TkFilt(ts)->sub_ts->next(TkFilt(ts)->sub_ts);
 
-  i = (int)mbstowcs(wbuf, tk->text, MAX_WORD_SIZE);
-  w = wbuf;
-  while (*w != 0) {
-    *w = towlower(*w);
-    w++;
-  }
-  wcstombs(tk->text, wbuf, MAX_WORD_SIZE);
-  return tk;
+    if (tk == NULL) {
+        return tk;
+    }
+
+    mbstowcs(wbuf, tk->text, MAX_WORD_SIZE);
+    wchr = wbuf;
+    while (*wchr != 0) {
+        *wchr = towlower(*wchr);
+        wchr++;
+    }
+    tk->len = wcstombs(tk->text, wbuf, MAX_WORD_SIZE);
+    tk->text[tk->len] = '\0';
+    return tk;
 }
 
-TokenStream *mb_lowercase_filter_create(TokenStream *ts)
+TokenStream *mb_lowercase_filter_new(TokenStream *sub_ts)
 {
-  TokenStream *tf = ALLOC(TokenStream);
-  tf->token = NULL;
-  tf->next = &mb_lcf_next;
-  tf->reset = &filter_reset;
-  tf->destroy = &filter_destroy;
-  tf->sub_ts = ts;
-  tf->clone_i = NULL;
-  tf->ref_cnt = 1;
-  return tf;
+    TokenStream *ts = tf_new(TokenFilter, sub_ts);
+    ts->next = &mb_lcf_next;
+    return ts;
 }
 
 Token *lcf_next(TokenStream *ts)
 {
-  int i = 0;
-  Token *tk = ts->sub_ts->next(ts->sub_ts);
-  if (tk == NULL) return tk;
-  while (tk->text[i] != '\0') {
-    tk->text[i] = tolower(tk->text[i]);
-    i++;
-  }
-  return tk;
+    int i = 0;
+    Token *tk = TkFilt(ts)->sub_ts->next(TkFilt(ts)->sub_ts);
+    if (tk == NULL) {
+        return tk;
+    }
+    while (tk->text[i] != '\0') {
+        tk->text[i] = tolower(tk->text[i]);
+        i++;
+    }
+    return tk;
 }
 
-TokenStream *lowercase_filter_create(TokenStream *ts)
+TokenStream *lowercase_filter_new(TokenStream *sub_ts)
 {
-  TokenStream *tf = ALLOC(TokenStream);
-  tf->token = NULL;
-  tf->next = &lcf_next;
-  tf->reset = &filter_reset;
-  tf->destroy = &filter_destroy;
-  tf->sub_ts = ts;
-  tf->clone_i = NULL;
-  tf->ref_cnt = 1;
-  return tf;
+    TokenStream *ts = tf_new(TokenFilter, sub_ts);
+    ts->next = &lcf_next;
+    return ts;
 }
 
-typedef struct StemFilter {
-  struct sb_stemmer *stemmer;
-  char *algorithm;
-  char *charenc;
-} StemFilter;
+/****************************************************************************
+ * StemFilter
+ ****************************************************************************/
+
+#define StemFilt(filter) ((StemFilter *)(filter))
 
-void stemf_destroy(TokenStream *tf)
+void stemf_destroy_i(TokenStream *ts)
 {
-  StemFilter *stemf = (StemFilter *)tf->data;
-  sb_stemmer_delete(stemf->stemmer);
-  free(stemf->algorithm);
-  free(stemf->charenc);
-  free(stemf);
-  filter_destroy(tf);
+    sb_stemmer_delete(StemFilt(ts)->stemmer);
+    free(StemFilt(ts)->algorithm);
+    free(StemFilt(ts)->charenc);
+    filter_destroy_i(ts);
 }
 
 Token *stemf_next(TokenStream *ts)
 {
-  int len;
-  const sb_symbol *stemmed;
-  struct sb_stemmer *stemmer = ((StemFilter *)ts->data)->stemmer;
-  Token *tk = ts->sub_ts->next(ts->sub_ts);
-  if (tk == NULL) return tk;
-  stemmed = sb_stemmer_stem(stemmer, (sb_symbol *)tk->text, (int)strlen(tk->text));
-  len = sb_stemmer_length(stemmer);
-  if (len >= MAX_WORD_SIZE) len = MAX_WORD_SIZE - 1;
-  memcpy(tk->text, stemmed, len);
-  tk->text[len] = '\0';
-  return tk;
+    int len;
+    const sb_symbol *stemmed;
+    struct sb_stemmer *stemmer = StemFilt(ts)->stemmer;
+    TokenFilter *tf = TkFilt(ts);
+    Token *tk = tf->sub_ts->next(tf->sub_ts);
+    if (tk == NULL) {
+        return tk;
+    }
+    stemmed = sb_stemmer_stem(stemmer, (sb_symbol *)tk->text, tk->len);
+    len = sb_stemmer_length(stemmer);
+    if (len >= MAX_WORD_SIZE) {
+        len = MAX_WORD_SIZE - 1;
+    }
+
+    memcpy(tk->text, stemmed, len);
+    tk->text[len] = '\0';
+    tk->len = len;
+    return tk;
 }
 
-void stemf_clone_i(TokenStream *orig_ts, TokenStream *new_ts)
+TokenStream *stemf_clone_i(TokenStream *orig_ts)
 {
-  StemFilter *orig_stemf = (StemFilter *)orig_ts->data;
-  StemFilter *stemf = ALLOC(StemFilter);
-  stemf->stemmer = sb_stemmer_new(orig_stemf->algorithm, orig_stemf->charenc);
-  stemf->algorithm = orig_stemf->algorithm ? estrdup(orig_stemf->algorithm) : NULL;
-  stemf->charenc = orig_stemf->charenc ? estrdup(orig_stemf->charenc) : NULL;
-  new_ts->data = stemf;
+    TokenStream *new_ts      = filter_clone_size(orig_ts, sizeof(StemFilter));
+    StemFilter *stemf        = StemFilt(new_ts);
+    StemFilter *orig_stemf   = StemFilt(orig_ts);
+    stemf->stemmer =
+        sb_stemmer_new(orig_stemf->algorithm, orig_stemf->charenc);
+    stemf->algorithm =
+        orig_stemf->algorithm ? estrdup(orig_stemf->algorithm) : NULL;
+    stemf->charenc =
+        orig_stemf->charenc ? estrdup(orig_stemf->charenc) : NULL;
+    return new_ts;
 }
 
-TokenStream *stem_filter_create(TokenStream *ts, const char * algorithm,
-    const char * charenc)
+TokenStream *stem_filter_new(TokenStream *ts, const char *algorithm,
+                             const char *charenc)
 {
-  TokenStream *tf = ALLOC(TokenStream);
-  StemFilter *stemf = ALLOC(StemFilter);
-  stemf->stemmer = sb_stemmer_new(algorithm, charenc);
-  stemf->algorithm = algorithm ? estrdup(algorithm) : NULL;
-  stemf->charenc = charenc ? estrdup(charenc) : NULL;
-  tf->data = stemf;
+    TokenStream *tf = tf_new(StemFilter, ts);
+
+    StemFilt(tf)->stemmer   = sb_stemmer_new(algorithm, charenc);
+    StemFilt(tf)->algorithm = algorithm ? estrdup(algorithm) : NULL;
+    StemFilt(tf)->charenc   = charenc ? estrdup(charenc) : NULL;
 
-  tf->token = NULL;
-  tf->next = &stemf_next;
-  tf->reset = &filter_reset;
-  tf->destroy = &stemf_destroy;
-  tf->clone_i = &stemf_clone_i;
-  tf->sub_ts = ts;
-  tf->ref_cnt = 1;
-  return tf;
+    tf->next = &stemf_next;
+    tf->destroy_i = &stemf_destroy_i;
+    tf->clone_i = &stemf_clone_i;
+    return tf;
 }
 
-Analyzer *standard_analyzer_create_with_words_len(
-    const char **words, int len, bool lowercase)
+/****************************************************************************
+ *
+ * Analyzers
+ *
+ ****************************************************************************/
+
+/****************************************************************************
+ * Standard
+ ****************************************************************************/
+
+Analyzer *standard_analyzer_new_with_words_len(const char **words, int len,
+                                               bool lowercase)
 {
-  TokenStream *ts;
-  if (lowercase) {
-    ts = stop_filter_create_with_words_len(
-        lowercase_filter_create(standard_tokenizer_create()), words, len);
-  } else {
-    ts = stop_filter_create_with_words_len(
-        standard_tokenizer_create(), words, len);
-  }
-  return analyzer_create(NULL, ts, NULL, NULL);
+    TokenStream *ts;
+    if (lowercase) {
+        ts = stop_filter_new_with_words_len(lowercase_filter_new
+                                            (standard_tokenizer_new()),
+                                            words, len);
+    }
+    else {
+        ts = stop_filter_new_with_words_len(standard_tokenizer_new(),
+                                            words, len);
+    }
+    return analyzer_new(ts, NULL, NULL);
 }
 
-Analyzer *standard_analyzer_create_with_words(const char **words, bool lowercase)
+Analyzer *standard_analyzer_new_with_words(const char **words,
+                                           bool lowercase)
 {
-  TokenStream *ts;
-  if (lowercase) {
-    ts = stop_filter_create_with_words(
-        lowercase_filter_create(standard_tokenizer_create()), words);
-  } else {
-    ts = stop_filter_create_with_words(
-        standard_tokenizer_create(), words);
-  }
-  return analyzer_create(NULL, ts, NULL, NULL);
+    TokenStream *ts;
+    if (lowercase) {
+        ts = stop_filter_new_with_words(lowercase_filter_new
+                                        (standard_tokenizer_new()),
+                                        words);
+    }
+    else {
+        ts = stop_filter_new_with_words(standard_tokenizer_new(),
+                                        words);
+    }
+    return analyzer_new(ts, NULL, NULL);
 }
 
-Analyzer *mb_standard_analyzer_create_with_words_len(
-    const char **words, int len, bool lowercase)
+Analyzer *mb_standard_analyzer_new_with_words_len(const char **words,
+                                                  int len, bool lowercase)
 {
-  TokenStream *ts;
-  if (lowercase) {
-    ts = stop_filter_create_with_words_len(
-      mb_lowercase_filter_create(mb_standard_tokenizer_create()), words, len);
-  } else {
-    ts = stop_filter_create_with_words_len(
-        mb_standard_tokenizer_create(), words, len);
-  }
-  return analyzer_create(NULL, ts, NULL, NULL);
+    TokenStream *ts;
+    if (lowercase) {
+        ts = stop_filter_new_with_words_len(mb_lowercase_filter_new
+                                            (mb_standard_tokenizer_new
+                                             ()), words, len);
+    }
+    else {
+        ts = stop_filter_new_with_words_len(mb_standard_tokenizer_new(),
+                                            words, len);
+    }
+    return analyzer_new(ts, NULL, NULL);
 }
 
-Analyzer *mb_standard_analyzer_create_with_words(
-    const char **words, bool lowercase)
+Analyzer *mb_standard_analyzer_new_with_words(const char **words,
+                                              bool lowercase)
 {
-  TokenStream *ts;
-  if (lowercase) {
-    ts = stop_filter_create_with_words(
-      mb_lowercase_filter_create(mb_standard_tokenizer_create()), words);
-  } else {
-    ts = stop_filter_create_with_words(mb_standard_tokenizer_create(), words);
-  }
-  return analyzer_create(NULL, ts, NULL, NULL);
+    TokenStream *ts;
+    if (lowercase) {
+        ts = stop_filter_new_with_words(mb_lowercase_filter_new
+                                        (mb_standard_tokenizer_new()),
+                                        words);
+    }
+    else {
+        ts = stop_filter_new_with_words(mb_standard_tokenizer_new(),
+                                        words);
+    }
+    return analyzer_new(ts, NULL, NULL);
 }
 
-Analyzer *standard_analyzer_create(bool lowercase)
+Analyzer *standard_analyzer_new(bool lowercase)
 {
-  return standard_analyzer_create_with_words(FULL_ENGLISH_STOP_WORDS, lowercase);
+    return standard_analyzer_new_with_words(FULL_ENGLISH_STOP_WORDS,
+                                            lowercase);
 }
 
-Analyzer *mb_standard_analyzer_create(bool lowercase)
+Analyzer *mb_standard_analyzer_new(bool lowercase)
 {
-  return mb_standard_analyzer_create_with_words(FULL_ENGLISH_STOP_WORDS, lowercase);
+    return mb_standard_analyzer_new_with_words(FULL_ENGLISH_STOP_WORDS,
+                                               lowercase);
 }
 
 /****************************************************************************
@@ -1174,58 +1328,63 @@ Analyzer *mb_standard_analyzer_create(bool lowercase)
  *
  ****************************************************************************/
 
-void pfa_destroy(Analyzer *self)
+#define PFA(analyzer) ((PerFieldAnalyzer *)(analyzer))
+void pfa_destroy_i(Analyzer *self)
 {
-  PerFieldAnalyzer *pfa = (PerFieldAnalyzer *)self->data;
-  h_destroy(pfa->dict);
+    h_destroy(PFA(self)->dict);
 
-  a_deref(pfa->def);
-  free(pfa);
-  free(self);
+    a_deref(PFA(self)->default_a);
+    free(self);
 }
 
 TokenStream *pfa_get_ts(Analyzer *self, char *field, char *text)
 {
-  PerFieldAnalyzer *pfa = (PerFieldAnalyzer *)self->data;
-  Analyzer *a = h_get(pfa->dict, field);
-  if (a == NULL) a = pfa->def;
-  return a_get_ts(a, field, text);
+    Analyzer *a = h_get(PFA(self)->dict, field);
+    if (a == NULL) {
+        a = PFA(self)->default_a;
+    }
+    return a_get_ts(a, field, text);
 }
 
-void pfa_sub_a_destroy(void *p)
+void pfa_sub_a_destroy_i(void *p)
 {
-  Analyzer *a = (Analyzer *)p;
-  a_deref(a);
+    Analyzer *a = (Analyzer *) p;
+    a_deref(a);
 }
 
 void pfa_add_field(Analyzer *self, char *field, Analyzer *analyzer)
 {
-  PerFieldAnalyzer *pfa = (PerFieldAnalyzer *)self->data;
-  h_set(pfa->dict, estrdup(field), analyzer);
+    h_set(PFA(self)->dict, estrdup(field), analyzer);
 }
 
-Analyzer *per_field_analyzer_create(Analyzer *def)
+Analyzer *per_field_analyzer_new(Analyzer *default_a)
 {
-  PerFieldAnalyzer *pfa = ALLOC(PerFieldAnalyzer);
-  pfa->def = def;
-  pfa->dict = h_new_str(&free, &pfa_sub_a_destroy);
-  return analyzer_create(pfa, NULL, &pfa_destroy, &pfa_get_ts);
+    Analyzer *a = (Analyzer *)ecalloc(sizeof(PerFieldAnalyzer));
+
+    PFA(a)->default_a = default_a;
+    PFA(a)->dict = h_new_str(&free, &pfa_sub_a_destroy_i);
+
+    a->destroy_i = &pfa_destroy_i;
+    a->get_ts    = pfa_get_ts;
+    
+    return a;
 }
 
 #ifdef ALONE
 int main(int argc, char **argv)
 {
-  char buf[10000];
-  Analyzer *a = standard_analyzer_create(true);
-  TokenStream *ts;
-  Token *tk;
-  while (fgets(buf, 9999, stdin) != NULL) {
-    ts = a->get_ts(a, "hello", buf);
-    while ((tk = ts->next(ts)) != NULL) {
-      printf("<%s:%ld:%ld> ", tk->text, tk->start, tk->end);
-    }
-    printf("\n");
-  }
-  return 0;
+    char buf[10000];
+    Analyzer *a = standard_analyzer_new(true);
+    TokenStream *ts;
+    Token *tk;
+    while (fgets(buf, 9999, stdin) != NULL) {
+        ts = a_get_ts(a, "hello", buf);
+        while ((tk = ts->next(ts)) != NULL) {
+            printf("<%s:%ld:%ld> ", tk->text, tk->start, tk->end);
+        }
+        printf("\n");
+        ts_deref(ts);
+    }
+    return 0;
 }
 #endif