ferret 0.9.0 → 0.9.1

data/Rakefile

@@ -1,6 +1,6 @@
 $:. << 'lib'
 # Some parts of this Rakefile where taken from Jim Weirich's Rakefile for
-# Rake. Other parts where stolen from the David Heinemeier Hansson's Rails
+# Rake. Other parts where taken from the David Heinemeier Hansson's Rails
 # Rakefile. Both are under MIT-LICENSE. Thanks to both for their excellent
 # projects.
 
@@ -32,12 +32,13 @@ end
 $VERBOSE = nil
 
 EXT = "ferret_ext.so"
-EXT_SRC = FileList["src/*/*.[ch]"]
+EXT_SRC = FileList["src/**/*.[ch]"]
+
 EXT_SRC_DEST = EXT_SRC.map {|fn| File.join("ext", File.basename(fn))}
 SRC = (FileList["ext/*.[ch]"] + EXT_SRC_DEST).uniq
 
 CLEAN.include(FileList['**/*.o', 'InstalledFiles', '.config'])
-CLOBBER.include(FileList['**/*.so'], 'ext/Makefile')
+CLOBBER.include(FileList['**/*.so'], 'ext/Makefile', EXT_SRC_DEST)
 
 task :default => :all_tests
 desc "Run all tests"
@@ -57,7 +58,7 @@ end
 desc "run unit tests in test/unit for C ferret"
 Rake::TestTask.new("test_cunits" => :ext) do |t|
   t.libs << "test/unit"
-  t.pattern = 'test/unit/t[cs]_*.rb'
+  t.pattern = 'test/unit/ts_*.rb'
   t.verbose = true
 end
 
@@ -102,6 +103,15 @@ EXT_SRC.each do |fn|
   dest_fn = File.join("ext", File.basename(fn))
   file dest_fn => fn do |t|
     cp fn, dest_fn
+    if fn =~ /stemmer/
+      # flatten the directory structure for lib_stemmer
+      open(dest_fn) do |in_f|
+        open(dest_fn + ".out", "w") do |out_f|
+          in_f.each {|line| out_f.write(line.sub(/(#include ["<])[.a-z_\/]*\//) {"#{$1}"})}
+        end
+      end
+      mv dest_fn + ".out", dest_fn
+    end
   end
 end
 
@@ -110,9 +120,17 @@ task :ext => ["ext/#{EXT}"] + SRC
 
 file "ext/#{EXT}" => ["ext/Makefile"] do
   cp "ext/inc/lang.h", "ext/lang.h"
+  cp "ext/inc/except.h", "ext/except.h"
   sh "cd ext; make"
 end
 
+file "ext/lang.h" => ["ext/inc/lang.h"] do
+  cp "ext/inc/lang.h", "ext/lang.h"
+end
+file "ext/except.h" => ["ext/inc/except.h"] do
+  cp "ext/inc/except.h", "ext/except.h"
+end
+
 file "ext/Makefile" => SRC do
   sh "cd ext; ruby extconf.rb"
 end
@@ -220,7 +238,7 @@ task :repackage => EXT_SRC_DEST
 task :package => EXT_SRC_DEST
 task :tag => [:prerelease]
 task :update_version => [:prerelease]
-task :release => [:tag, :update_version, :package] do
+task :release do #=> [:tag, :update_version, :package] do
   announce 
   announce "**************************************************************"
   announce "* Release #{PKG_VERSION} Complete."

data/TODO

@@ -4,11 +4,12 @@ Send suggestions for this list to mailto:dbalmain@gmail.com
 
 === To Do
 
-* Add the ability to persist an in memory index to Ferret::Index::Index
 * Make a dll for people on Windows
+* pure ruby ConstantScoreQuery
 
 === Done
 
+* Add the ability to persist an in memory index to Ferret::Index::Index
 * Add UTF-8 support
 * Multi Field Query
 * Test threading

data/ext/analysis.c

@@ -1,7 +1,16 @@
 #include <analysis.h>
 #include <string.h>
 #include <ctype.h>
-#include <hash.h>
+#include <wctype.h>
+#include <wchar.h>
+#include "hash.h"
+#include "libstemmer.h"
+
+/****************************************************************************
+ *
+ * Token
+ *
+ ****************************************************************************/
 
 Token *tk_create()
 {
@@ -24,6 +33,11 @@ inline Token *tk_set(Token *tk, char *text, int tlen, int start, int end, int po
   return tk;
 }
 
+inline Token *tk_set_ts(Token *tk, char *start, char *end, char *text, int pos_inc)
+{
+  return tk_set(tk, start, end - start, start - text, end - text, pos_inc);
+}
+
 inline Token *tk_set_no_len(Token *tk, char *text, int start, int end, int pos_inc)
 {
   return tk_set(tk, text, strlen(text), start, end, pos_inc);
@@ -31,11 +45,8 @@ inline Token *tk_set_no_len(Token *tk, char *text, int start, int end, int pos_i
 
 int tk_eq(Token *tk1, Token *tk2)
 {
-  if (strcmp((char *)tk1->text, (char *)tk2->text) == 0 &&
-      tk1->start == tk2->start && tk1->end == tk2->end)
-    return true;
-  else
-    return false;
+  return (strcmp((char *)tk1->text, (char *)tk2->text) == 0 &&
+      tk1->start == tk2->start && tk1->end == tk2->end);
 }
 
 int tk_cmp(Token *tk1, Token *tk2)
@@ -57,46 +68,152 @@ int tk_cmp(Token *tk1, Token *tk2)
   return cmp;
 }
 
+
+/****************************************************************************
+ *
+ * TokenStream
+ *
+ ****************************************************************************/
+
 void ts_standard_destroy(void *p)
 {
   TokenStream *ts = (TokenStream *)p;
   tk_destroy(ts->token);
-  free(p);
+  free(ts);
 }
 
 void ts_reset(TokenStream *ts, char *text)
 {
-  ts->text = text;
-  ts->pos = 0;
+  ts->t = ts->text = text;
 }
 
 TokenStream *ts_create()
 {
   TokenStream *ts = ALLOC(TokenStream);
-  ts->pos = -1;
   ts->text = NULL;
   ts->token = tk_create();
   ts->destroy = &ts_standard_destroy;
   ts->reset = &ts_reset;
+  ts->sub_ts = NULL;
+  ts->clone_i = NULL;
   return ts;
 }
 
+TokenStream *ts_clone(TokenStream *orig_ts)
+{
+  TokenStream *ts = ALLOC(TokenStream);
+  memcpy(ts, orig_ts, sizeof(TokenStream));
+  if (orig_ts->token) {
+    ts->token = ALLOC(Token);
+    memcpy(ts->token, orig_ts->token, sizeof(Token));
+  }
+  if (orig_ts->sub_ts) ts->sub_ts = ts_clone(orig_ts->sub_ts);
+  if (orig_ts->clone_i) orig_ts->clone_i(orig_ts, ts);
+  return ts;
+}
+
+/* * Multi-byte TokenStream * */
+static char * const ENC_ERR_MSG = "Error decoding input string. "
+                          "Check that you have the locale set correctly";
+#define MB_NEXT_CHAR \
+  if ((i = mbrtowc(&wchr, t, MB_CUR_MAX, (mbstate_t *)ts->data)) < 0)\
+    RAISE(IO_ERROR, ENC_ERR_MSG)
+
+inline Token *w_tk_set(Token *tk, wchar_t *text, int start, int end, int pos_inc)
+{
+  tk->text[wcstombs(tk->text, text, MAX_WORD_SIZE - 1)] = '\0';
+  tk->start = start;
+  tk->end = end;
+  tk->pos_inc = pos_inc;
+  return tk;
+}
+
+void mb_ts_standard_destroy(void *p)
+{
+  TokenStream *ts = (TokenStream *)p;
+  tk_destroy(ts->token);
+  free(ts->data);
+  free(ts);
+}
+
+void mb_ts_reset(TokenStream *ts, char *text)
+{
+  ZEROSET(ts->data, mbstate_t, 1);
+  ts_reset(ts, text);
+}
+
+void mb_ts_clone_i(TokenStream *orig_ts, TokenStream *new_ts)
+{
+  new_ts->data = ALLOC(mbstate_t);
+  memcpy(new_ts->data, orig_ts->data, sizeof(mbstate_t));
+}
+
+TokenStream *mb_ts_create()
+{
+  TokenStream *ts = ALLOC(TokenStream);
+  ts->data = ALLOC(mbstate_t);
+  ts->text = NULL;
+  ts->token = tk_create();
+  ts->destroy = &mb_ts_standard_destroy;
+  ts->reset = &mb_ts_reset;
+  ts->clone_i = &mb_ts_clone_i;
+  ts->sub_ts = NULL;
+  return ts;
+}
+
+/****************************************************************************
+ *
+ * Analyzer
+ *
+ ****************************************************************************/
+
+void a_standard_destroy(void *p)
+{
+  Analyzer *a = (Analyzer *)p;
+  ts_destroy(a->current_ts);
+  free(p);
+}
+
+TokenStream *a_standard_get_ts(Analyzer *a, char *field, char *text)
+{
+  a->current_ts->reset(a->current_ts, text);
+  return a->current_ts; 
+}
+
+Analyzer *analyzer_create(void *data, TokenStream *ts, void (*destroy)(void *),
+    TokenStream *(*get_ts)(Analyzer *a, char *field, char *text))
+{
+  Analyzer *a = ALLOC(Analyzer);
+  a->data = data;
+  a->current_ts = ts;
+  a->destroy = (destroy ? destroy : &a_standard_destroy);
+  a->get_ts = (get_ts ? get_ts : &a_standard_get_ts);
+  return a;
+}
+
+/****************************************************************************
+ *
+ * Whitespace
+ *
+ ****************************************************************************/
+
+/*
+ * WhitespaceTokenizer
+ */
 Token *wst_next(TokenStream *ts)
 {
-  int i = ts->pos;
-  int start, end;
-  char *text = ts->text;
+  char *t = ts->t;
+  char *start;
 
-  while (text[i] != '\0' && isspace(text[i]))
-    i++;
-  if (text[i] == '\0')
-    return NULL;
+  while (*t != '\0' && isspace(*t)) t++;
 
-  start = i;
-  while (text[i] != '\0' && !isspace(text[i]))
-    i++;
-  ts->pos = end = i;
-  tk_set(ts->token, text+start, end-start, start, end, 1);
+  if (*t == '\0') return NULL;
+
+  start = t;
+  while (*t != '\0' && !isspace(*t)) t++;
+
+  ts->t = t;
+  tk_set_ts(ts->token, start, t, ts->text, 1);
   return ts->token;
 }
 
@@ -107,22 +224,121 @@ TokenStream *whitespace_tokenizer_create()
   return ts;
 }
 
+/*
+ * Multi-byte WhitespaceTokenizer
+ */
+Token *mb_wst_next(TokenStream *ts)
+{
+  int i;
+  char *start;
+  char *t = ts->t;
+  wchar_t wchr;
+
+  MB_NEXT_CHAR;
+  while (wchr != 0 && iswspace(wchr)) {
+    t += i;
+    MB_NEXT_CHAR;
+  }
+  if (wchr == 0) return NULL;
+
+  start = t;
+  t += i;
+  MB_NEXT_CHAR;
+  while (wchr != 0 && !iswspace(wchr)) {
+    t += i;
+    MB_NEXT_CHAR;
+  }
+  tk_set_ts(ts->token, start, t, ts->text, 1);
+  ts->t = t;
+  return ts->token;
+}
+
+/*
+ * Lowercasing Multi-byte WhitespaceTokenizer
+ */
+Token *mb_wst_next_lc(TokenStream *ts)
+{
+  int i;
+  char *start;
+  char *t = ts->t;
+  wchar_t wchr;
+  wchar_t wbuf[MAX_WORD_SIZE+1], *w, *w_end;
+
+  w = wbuf;
+  w_end = &wbuf[MAX_WORD_SIZE];
+
+  MB_NEXT_CHAR;
+  while (wchr != 0 && iswspace(wchr)) {
+    t += i;
+    MB_NEXT_CHAR;
+  }
+  if (wchr == 0) return NULL;
+
+  start = t;
+  t += i;
+  *w++ = towlower(wchr);
+  MB_NEXT_CHAR;
+  while (wchr != 0 && !iswspace(wchr)) {
+    if (w < w_end) *w++ = towlower(wchr);
+    t += i;
+    MB_NEXT_CHAR;
+  }
+  *w = 0;
+  w_tk_set(ts->token, wbuf, start - ts->text, t - ts->text, 1);
+  ts->t = t;
+  return ts->token;
+}
+
+TokenStream *mb_whitespace_tokenizer_create(bool lowercase)
+{
+  TokenStream *ts = mb_ts_create();
+  ts->next = lowercase ? &mb_wst_next_lc : &mb_wst_next;
+  return ts;
+}
+
+/*
+ * WhitespaceAnalyzers
+ */
+Analyzer *whitespace_analyzer_create(bool lowercase)
+{
+  TokenStream *ts;
+  if (lowercase) {
+    ts = lowercase_filter_create(whitespace_tokenizer_create());
+  } else {
+    ts = whitespace_tokenizer_create();
+  }
+  return analyzer_create(NULL, ts, NULL, NULL);
+}
+
+Analyzer *mb_whitespace_analyzer_create(bool lowercase)
+{
+  return analyzer_create(NULL, mb_whitespace_tokenizer_create(lowercase),
+      NULL, NULL);
+}
+
+/****************************************************************************
+ *
+ * Letter
+ *
+ ****************************************************************************/
+
+/*
+ * LetterTokenizer
+ */
 Token *lt_next(TokenStream *ts)
 {
-  int i = ts->pos;
-  int start, end;
-  char *text = ts->text;
+  char *start;
+  char *t = ts->t;
 
-  while (text[i] != '\0' && !isalpha(text[i]))
-    i++;
-  if (text[i] == '\0')
-    return NULL;
+  while (*t != '\0' && !isalpha(*t)) t++;
 
-  start = i;
-  while (text[i] != '\0' && isalpha(text[i]))
-    i++;
-  ts->pos = end = i;
-  tk_set(ts->token, text+start, end-start, start, end, 1);
+  if (*t == '\0') return NULL;
+
+  start = t;
+  while (*t != '\0' && isalpha(*t)) t++;
+
+  tk_set_ts(ts->token, start, t, ts->text, 1);
+  ts->t = t;
   return ts->token;
 }
 
@@ -133,54 +349,174 @@ TokenStream *letter_tokenizer_create()
   return ts;
 }
 
-void a_standard_destroy(void *p)
+/*
+ * Multi-byte LetterTokenizer
+ */
+Token *mb_lt_next(TokenStream *ts)
 {
-  Analyzer *a = (Analyzer *)p;
-  ts_destroy(a->current_ts);
-  free(p);
+  int i;
+  char *start;
+  char *t = ts->t;
+  wchar_t wchr;
+
+  MB_NEXT_CHAR;
+  while (wchr != 0 && !iswalpha(wchr)) {
+    t += i;
+    MB_NEXT_CHAR;
+  }
+  if (wchr == 0) return NULL;
+
+  start = t;
+  t += i;
+  MB_NEXT_CHAR;
+  while (wchr != 0 && iswalpha(wchr)) {
+    t += i;
+    MB_NEXT_CHAR;
+  }
+  tk_set_ts(ts->token, start, t, ts->text, 1);
+  ts->t = t;
+  return ts->token;
 }
 
-TokenStream *a_standard_get_ts(Analyzer *a, char *field, char *text)
+/*
+ * Lowercasing Multi-byte LetterTokenizer
+ */
+Token *mb_lt_next_lc(TokenStream *ts)
 {
-  a->current_ts->reset(a->current_ts, text);
-  return a->current_ts; 
+  int i;
+  char *start;
+  char *t = ts->t;
+  wchar_t wchr;
+  wchar_t wbuf[MAX_WORD_SIZE+1], *w, *w_end;
+
+  w = wbuf;
+  w_end = &wbuf[MAX_WORD_SIZE];
+
+  MB_NEXT_CHAR;
+  while (wchr != 0 && !iswalpha(wchr)) {
+    t += i;
+    MB_NEXT_CHAR;
+  }
+  if (wchr == 0) return NULL;
+
+  start = t;
+  t += i;
+  *w++ = towlower(wchr);
+  MB_NEXT_CHAR;
+  while (wchr != 0 && iswalpha(wchr)) {
+    if (w < w_end) *w++ = towlower(wchr);
+    t += i;
+    MB_NEXT_CHAR;
+  }
+  *w = 0;
+  w_tk_set(ts->token, wbuf, start - ts->text, t - ts->text, 1);
+  ts->t = t;
+  return ts->token;
 }
 
-Analyzer *whitespace_analyzer_create()
+TokenStream *mb_letter_tokenizer_create(bool lowercase)
 {
-  Analyzer *a = ALLOC(Analyzer);
-  a->data = NULL;
-  a->current_ts = whitespace_tokenizer_create();
-  a->destroy = &a_standard_destroy;
-  a->get_ts = &a_standard_get_ts;
-  return a;
+  TokenStream *ts = mb_ts_create();
+  ts->next = lowercase ? &mb_lt_next_lc : &mb_lt_next;
+  return ts;
 }
 
-int std_get_alpha(char *input, char *token)
+/*
+ * LetterAnalyzers
+ */
+Analyzer *letter_analyzer_create(bool lowercase)
+{
+  TokenStream *ts;
+  if (lowercase) {
+    ts = lowercase_filter_create(letter_tokenizer_create());
+  } else {
+    ts = letter_tokenizer_create();
+  }
+  return analyzer_create(NULL, ts, NULL, NULL);
+}
+
+Analyzer *mb_letter_analyzer_create(bool lowercase)
+{
+  return analyzer_create(NULL,
+      mb_letter_tokenizer_create(lowercase), NULL, NULL);
+}
+
+/****************************************************************************
+ *
+ * Standard
+ *
+ ****************************************************************************/
+
+/*
+ * StandardTokenizer
+ */
+int std_get_alpha(TokenStream *ts, char *token)
 {
   int i = 0;
-  while (input[i] != '\0' && isalpha(input[i])) {
-    token[i] = input[i];
+  char *t = ts->t;
+  while (t[i] != '\0' && isalpha(t[i])) {
+    if (i < MAX_WORD_SIZE) token[i] = t[i];
     i++;
   }
   return i;
 }
 
-int std_get_alnum(char *input, char *token)
+int mb_std_get_alpha(TokenStream *ts, char *token)
+{
+  char *t = ts->t;
+  wchar_t w;
+  int i;
+  if ((i = mbtowc(&w, t, MB_CUR_MAX)) < 0) RAISE(IO_ERROR, ENC_ERR_MSG);
+  while (w != 0 && iswalpha(w)) {
+    t += i;
+    if ((i = mbtowc(&w, t, MB_CUR_MAX)) < 0) RAISE(IO_ERROR, ENC_ERR_MSG);
+  }
+
+  i = t - ts->t;
+  if (i > MAX_WORD_SIZE) i = MAX_WORD_SIZE - 1;
+  memcpy(token, ts->t, i);
+  return i;
+}
+
+int std_get_alnum(TokenStream *ts, char *token)
 {
   int i = 0;
-  while (input[i] != '\0' && isalnum(input[i])) {
-    token[i] = input[i];
+  char *t = ts->t;
+  while (t[i] != '\0' && isalnum(t[i])) {
+    if (i < MAX_WORD_SIZE) token[i] = t[i];
     i++;
   }
   return i;
 }
 
+int mb_std_get_alnum(char *text, char *token, TokenStream *ts)
+{
+  char *t = ts->t;
+  wchar_t w;
+  int i;
+  if ((i = mbtowc(&w, t, MB_CUR_MAX)) < 0) RAISE(IO_ERROR, ENC_ERR_MSG);
+  while (w != 0 && iswalnum(w)) {
+    t += i;
+    if ((i = mbtowc(&w, t, MB_CUR_MAX)) < 0) RAISE(IO_ERROR, ENC_ERR_MSG);
+  }
+
+  i = t - ts->t;
+  if (i > MAX_WORD_SIZE) i = MAX_WORD_SIZE - 1;
+  memcpy(token, ts->t, i);
+  return i;
+
+}
+
 int isnumpunc(char c)
 {
   return (c == '.' || c == ',' || c == '\\' || c == '/' || c == '_' || c == '-');
 }
 
+int w_isnumpunc(wchar_t c)
+{
+  return (c == L'.' || c == L',' || c == L'\\' || c == L'/' || c == L'_' || c == L'-');
+}
+
 int isurlpunc(char c)
 {
   return (c == '.' || c == '/' || c == '-' || c == '_');
@@ -201,11 +537,23 @@ int isurlxatc(char c)
   return (c == '.' || c == '/' || c == '-' || c == '_' || c == '@' || isalnum(c));
 }
 
-int isstdtokchar(char c)
+bool std_is_tok_char(char *c)
 {
-  if (isspace(c)) return false; // most common so check first.
-  if (isalnum(c) || isnumpunc(c) || c == '&' ||
-      c == '@' || c == '\'' || c == ':')
+  if (isspace(*c)) return false; // most common so check first.
+  if (isalnum(*c) || isnumpunc(*c) || *c == '&' ||
+      *c == '@' || *c == '\'' || *c == ':')
+    return true;
+  return false;
+}
+
+bool w_std_is_tok_char(char *t)
+{
+  wchar_t c;
+  if ((mbtowc(&c, t, MB_CUR_MAX)) < 0)
+    RAISE(IO_ERROR, ENC_ERR_MSG);
+  if (iswspace(c)) return false; // most common so check first.
+  if (iswalnum(c) || w_isnumpunc(c) || c == L'&' ||
+      c == L'@' || c == L'\'' || c == L':')
     return true;
   return false;
 }
@@ -246,22 +594,34 @@ int std_get_number(char *input)
 
 int std_get_apostrophe(char *input)
 {
-  int i = 0;
+  char *t = input;
 
-  while (isalpha(input[i]) || input[i] == '\'')
-    i++;
+  while (isalpha(*t) || *t == '\'')
+    t++;
 
-  return i;
+  return t - input;
 }
 
-int std_get_url(char *input, char *token)
+int mb_std_get_apostrophe(char *input)
 {
-  int i = 0;
+  char *t = input;
+  wchar_t w;
+  int i;
 
+  if ((i = mbtowc(&w, t, MB_CUR_MAX)) < 0) RAISE(IO_ERROR, ENC_ERR_MSG);
+  while (iswalpha(w) || w == L'\'') {
+    t += i;
+    if ((i = mbtowc(&w, t, MB_CUR_MAX)) < 0) RAISE(IO_ERROR, ENC_ERR_MSG);
+  }
+  return t - input;
+}
+
+int std_get_url(char *input, char *token, int i)
+{
   while (isurlc(input[i])) {
     if (isurlpunc(input[i]) && isurlpunc(input[i-1]))
       break; // can't have to puncs in a row
-    token[i] = input[i];
+    if (i < MAX_WORD_SIZE) token[i] = input[i];
     i++;
   }
 
@@ -282,148 +642,229 @@ int std_get_company_name(char *input)
   return i;
 }
 
+int mb_std_get_company_name(char *input, TokenStream *ts)
+{
+  char *t = input;
+  wchar_t wchr;
+  int i;
+
+  MB_NEXT_CHAR;
+  while (iswalpha(wchr) || wchr == L'@' || wchr == L'&') {
+    t += i;
+    MB_NEXT_CHAR;
+  }
+
+  return t - input;
+}
+
+bool std_advance_to_start(TokenStream *ts)
+{
+  char *t = ts->t;
+  while (*t != '\0' && !isalnum(*t)) t++;
+
+  ts->t = t;
+
+  return (*t != '\0');
+}
+
+bool mb_std_advance_to_start(TokenStream *ts)
+{
+  int i;
+  wchar_t w;
+
+  if ((i = mbtowc(&w, ts->t, MB_CUR_MAX)) < 0) RAISE(IO_ERROR, ENC_ERR_MSG);
+  while (w != 0 && !iswalnum(w)) {
+    ts->t += i;
+    if ((i = mbtowc(&w, ts->t, MB_CUR_MAX)) < 0) RAISE(IO_ERROR, ENC_ERR_MSG);
+  }
+
+  return (w != 0);
+}
+
+typedef struct StandardTokenizer {
+  bool (*advance_to_start)(TokenStream *ts);
+  bool (*is_tok_char)(char *c);
+  int (*get_alpha)(TokenStream *ts, char *token);
+  int (*get_apostrophe)(char *input);
+} StandardTokenizer;
+
 Token *std_next(TokenStream *ts)
 {
-  int i = ts->pos, j;
-  int start;
-  char *text = ts->text;
+  StandardTokenizer *std_tz = (StandardTokenizer *)ts->data;
+  char *s;
+  char *t;
+  char *start = NULL;
+  char *num_end = NULL;
   char token[MAX_WORD_SIZE];
   int token_i = 0;
   int len;
-  int num_end = 0;
-  int is_acronym;
-  int seen_at_symbol;
+  bool is_acronym;
+  bool seen_at_symbol;
   
-  while (text[i] != '\0' && !isalnum(text[i]))
-    i++;
-  if (text[i] == '\0')
-    return NULL;
-
-  start = i;
-  if (isdigit(text[i])) {
-    i += std_get_number(text + i);
-    ts->pos = i;
-    tk_set(ts->token, text+start, i - start, start, ts->pos, 1);
+
+  if (!std_tz->advance_to_start(ts)) return NULL;
+
+  start = t = ts->t;
+  if (isdigit(*t)) {
+    t += std_get_number(t);
+    ts->t = t;
+    tk_set_ts(ts->token, start, t, ts->text, 1);
   } else {
-    token_i = std_get_alpha(text + i, token);
-    i += token_i;
+    token_i = std_tz->get_alpha(ts, token);
+    t += token_i;
 
-    if (!isstdtokchar(text[i])) {
+    if (!std_tz->is_tok_char(t)) {
       // very common case, ie a plain word, so check and return
-      tk_set(ts->token, text+start, i-start, start, i, 1);
-      ts->pos = i;
+      tk_set_ts(ts->token, start, t, ts->text, 1);
+      ts->t = t;
       return ts->token;
     }
 
-    if (text[i] == '\'') { // apostrophe case. 
-      i += std_get_apostrophe(text + i);
-      ts->pos = i;
-      len = i - start;
+    if (*t == '\'') { // apostrophe case. 
+      t += std_tz->get_apostrophe(t);
+      ts->t = t;
+      len = t - start;
       // strip possesive
-      if ((text[i-1] == 's' || text[i-1] == 'S') && text[i-2] == '\'')
-        len -= 2;
-      tk_set(ts->token, text+start, len, start, i, 1);
+      if ((t[-1] == 's' || t[-1] == 'S') && t[-2] == '\'') t -= 2;
+
+      tk_set_ts(ts->token, start, t, ts->text, 1);
       return ts->token;
     }
-    if (text[i] == '&') { // apostrophe case. 
-      i += std_get_company_name(text + i);
-      ts->pos = i;
-      tk_set(ts->token, text+start, i - start, start, i, 1);
+
+    if (*t == '&') { // apostrophe case. 
+      t += std_get_company_name(t);
+      ts->t = t;
+      tk_set_ts(ts->token, start, t, ts->text, 1);
       return ts->token;
     }
 
-    if (isdigit(text[i]) || isnumpunc(text[i])) { // possibly a number
-      num_end = start + std_get_number(text + start);
-      if (!isstdtokchar(text[num_end])) { // we won't find a longer token
-        ts->pos = num_end;
-        tk_set(ts->token, text+start, num_end-start, start, ts->pos, 1);
+    if (isdigit(*t) || isnumpunc(*t)) { // possibly a number
+      num_end = start + std_get_number(start);
+      if (!std_tz->is_tok_char(num_end)) { // we won't find a longer token
+        ts->t = num_end;
+        tk_set_ts(ts->token, start, num_end, ts->text, 1);
         return ts->token;
       }
       // else there may be a longer token so check
     }
 
-    if (text[i] == ':' && text[i+1] == '/' && text[i+2] == '/') {
+    if (t[0] == ':' && t[1] == '/' && t[2] == '/') {
       // check for a known url start
       token[token_i] = '\0';
-      i += 3;
-      while (text[i] == '/') i++;
-      if (isalpha(text[i]) &&
-          (strcmp(token, "ftp") == 0 ||
-           strcmp(token, "http") == 0 ||
-           strcmp(token, "https") == 0 ||
-           strcmp(token, "file") == 0)) {
-        len = std_get_url(text + i, token); // dispose of first part of the URL
+      t += 3;
+      while (*t == '/') t++;
+      if (isalpha(*t) &&
+          (memcmp(token, "ftp", 3) == 0 ||
+           memcmp(token, "http", 4) == 0 ||
+           memcmp(token, "https", 5) == 0 ||
+           memcmp(token, "file", 4) == 0)) {
+        len = std_get_url(t, token, 0); // dispose of first part of the URL
       } else { //still treat as url but keep the first part
-        token_i = i - start;
-        memcpy(token, text + start, token_i * sizeof(char));
-        len = token_i + std_get_url(text + i, token + token_i); // keep start
+        token_i = t - start;
+        memcpy(token, start, token_i * sizeof(char));
+        len = token_i + std_get_url(t, token, token_i); // keep start
       }
-      ts->pos = i + len;
+      ts->t = t + len;
       token[len] = 0;
-      tk_set(ts->token, token, len, start, ts->pos, 1);
+      tk_set(ts->token, token, len, start - ts->text, ts->t - ts->text, 1);
       return ts->token;
     }
 
-    // now see how int a url we can find.
+    // now see how long a url we can find.
     is_acronym = true;
     seen_at_symbol = false;
-    while (isurlxatc(text[i])) {
-      if (is_acronym && !isalpha(text[i]) && (text[i] != '.')) {
+    while (isurlxatc(*t)) {
+      if (is_acronym && !isalpha(*t) && (*t != '.')) {
         is_acronym = false;
       }
-      if (isurlxatpunc(text[i]) && isurlxatpunc(text[i-1]))
-        break; // can't have to punctuation characters in a row
-      if (text[i] == '@') {
-        if (seen_at_symbol)
+      if (isurlxatpunc(*t) && isurlxatpunc(t[-1])) {
+        break; // can't have two punctuation characters in a row
+      }
+      if (*t == '@') {
+        if (seen_at_symbol) {
           break; // we can only have one @ symbol
-        else
+        } else {
           seen_at_symbol = true;
+        }
       }
-      i++;
+      t++;
     }
-    while (isurlxatpunc(text[i-1])) i--; // strip trailing punctuation
-    if (i > num_end) {
-      ts->pos = i;
+    while (isurlxatpunc(t[-1])) t--; // strip trailing punctuation
+
+    if (t > num_end) {
+      ts->t = t;
 
       if (is_acronym) { // check that it is one letter followed by one '.'
-        for (j = start; j < i-1; j++) {
-          if (isalpha(text[j]) && (text[j+1] != '.')) is_acronym = false;
+        for (s = start; s < t-1; s++) {
+          if (isalpha(*s) && (s[1] != '.')) is_acronym = false;
         }
       }
       if (is_acronym) {// strip '.'s
-        for (j = start + token_i; j < i; j++) {
-          if (text[j] != '.') {
-            token[token_i] = text[j];
+        for (s = start + token_i; s < t; s++) {
+          if (*s != '.') {
+            token[token_i] = *s;
             token_i++;
           }
         }
-        tk_set(ts->token, token, token_i, start, ts->pos, 1);
+        tk_set(ts->token, token, token_i, start - ts->text, t - ts->text, 1);
       } else { // just return the url as is
-        tk_set(ts->token, text+start, i-start, start, ts->pos, 1);
+        tk_set_ts(ts->token, start, t, ts->text, 1);
       }
     } else { // return the number
-      ts->pos = num_end;
-      tk_set(ts->token, text+start, num_end-start, start, ts->pos, 1);
+      ts->t = num_end;
+      tk_set_ts(ts->token, start, num_end, ts->text, 1);
     }
   }
 
   return ts->token;
 }
 
+void std_ts_destroy(void *p)
+{
+  TokenStream *ts = (TokenStream *)p;
+  free(ts->data);
+  ts_standard_destroy(ts);
+}
+
+void std_ts_clone_i(TokenStream *orig_ts, TokenStream *new_ts)
+{
+  new_ts->data = ALLOC(StandardTokenizer);
+  memcpy(new_ts->data, orig_ts->data, sizeof(StandardTokenizer));
+}
+
 TokenStream *standard_tokenizer_create()
 {
   TokenStream *ts = ts_create();
+
+  StandardTokenizer *std_tz = ALLOC(StandardTokenizer);
+  std_tz->advance_to_start = &std_advance_to_start;
+  std_tz->get_alpha = &std_get_alpha;
+  std_tz->is_tok_char = &std_is_tok_char;
+  std_tz->get_apostrophe = &std_get_apostrophe;
+
+  ts->data = std_tz;
+  ts->destroy = &std_ts_destroy;
+  ts->clone_i = &std_ts_clone_i;
   ts->next = &std_next;
   return ts;
 }
 
-const char *ENGLISH_STOP_WORDS[] = {
-  "a", "an", "and", "are", "as", "at", "be", "but", "by",
-  "for", "if", "in", "into", "is", "it",
-  "no", "not", "of", "on", "or", "s", "such",
-  "t", "that", "the", "their", "then", "there", "these",
-  "they", "this", "to", "was", "will", "with"
-};
+TokenStream *mb_standard_tokenizer_create()
+{
+  TokenStream *ts = ts_create();
+
+  StandardTokenizer *std_tz = ALLOC(StandardTokenizer);
+  std_tz->advance_to_start = &mb_std_advance_to_start;
+  std_tz->get_alpha = &mb_std_get_alpha;
+  std_tz->is_tok_char = &w_std_is_tok_char;
+  std_tz->get_apostrophe = &mb_std_get_apostrophe;
+
+  ts->data = std_tz;
+  ts->destroy = &std_ts_destroy;
+  ts->clone_i = &std_ts_clone_i;
+  ts->next = &std_next;
+  return ts;
+}
 
 void filter_reset(TokenStream *ts, char *text)
 {
@@ -432,10 +873,10 @@ void filter_reset(TokenStream *ts, char *text)
 
 void filter_destroy(void *p)
 {
-  TokenStream *ts = (TokenStream *)p;
-  ts->sub_ts->destroy(ts->sub_ts);
-  if (ts->token != NULL) tk_destroy(ts->token);
-  free(ts);
+  TokenStream *tf = (TokenStream *)p;
+  if (tf->destroy_sub) tf->sub_ts->destroy(tf->sub_ts);
+  if (tf->token != NULL) tk_destroy(tf->token);
+  free(tf);
 }
 
 void sf_destroy(void *p)
@@ -445,40 +886,109 @@ void sf_destroy(void *p)
   filter_destroy(p);
 }
 
-Token *sf_next(TokenStream *ts)
+void sf_clone_i_i(void *key, void *value, void *arg)
+{
+  HshTable *wordtable = (HshTable *)arg;
+  char *w = estrdup(key);
+  h_set(wordtable, w, w);
+}
+
+void sf_clone_i(TokenStream *orig_ts, TokenStream *new_ts)
+{
+  new_ts->data = h_new_str(&free, NULL);
+  h_each(orig_ts->data, &sf_clone_i_i, new_ts->data);
+}
+
+Token *sf_next(TokenStream *tf)
 {
   int pos_inc = 1;
-  HshTable *words = (HshTable *)ts->data;
-  Token *tk = ts->sub_ts->next(ts->sub_ts);
+  HshTable *words = (HshTable *)tf->data;
+  Token *tk = tf->sub_ts->next(tf->sub_ts);
   while ((tk != NULL) && (h_get(words, tk->text) != NULL)) {
-    tk = ts->sub_ts->next(ts->sub_ts);
+    tk = tf->sub_ts->next(tf->sub_ts);
     pos_inc++;
   }
   if (tk != NULL) tk->pos_inc = pos_inc;
   return tk;
 }
 
-TokenStream *stop_filter_create_with_words(TokenStream *ts, char **words, int len)
+TokenStream *stop_filter_create_with_words_len(TokenStream *ts,
+    const char **words, int len)
 {
   int i;
+  char *w;
   TokenStream *tf = ALLOC(TokenStream);
   tf->sub_ts = ts;
-  HshTable *wordtable = h_new_str(NULL, NULL);
+  tf->destroy_sub = true;
+  HshTable *wordtable = h_new_str(&free, NULL);
   for (i = 0; i < len; i++) {
-    h_set(wordtable, words[i], words[i]);
+    w = estrdup(words[i]);
+    h_set(wordtable, w, w);
+  }
+  tf->data = wordtable;
+  tf->token = NULL;
+  tf->next = &sf_next;
+  tf->reset = &filter_reset;
+  tf->destroy = &sf_destroy;
+  tf->clone_i = &sf_clone_i;
+  return tf;
+}
+
+TokenStream *stop_filter_create_with_words(TokenStream *ts, const char **words)
+{
+  char *w;
+  TokenStream *tf = ALLOC(TokenStream);
+  tf->sub_ts = ts;
+  tf->destroy_sub = true;
+  HshTable *wordtable = h_new_str(&free, NULL);
+  while (*words) {
+    w = estrdup(*words);
+    h_set(wordtable, w, w);
+    words++;
   }
   tf->data = wordtable;
   tf->token = NULL;
   tf->next = &sf_next;
   tf->reset = &filter_reset;
   tf->destroy = &sf_destroy;
+  tf->clone_i = &sf_clone_i;
   return tf;
 }
 
 TokenStream *stop_filter_create(TokenStream *ts)
 {
-  return stop_filter_create_with_words(ts, 
-      (char **)ENGLISH_STOP_WORDS, NELEMS(ENGLISH_STOP_WORDS));
+  return stop_filter_create_with_words(ts, FULL_ENGLISH_STOP_WORDS);
+}
+
+Token *mb_lcf_next(TokenStream *ts)
+{
+  wchar_t wbuf[MAX_WORD_SIZE], *w;
+  //mbstate_t state = {0};
+  int i;
+  Token *tk = ts->sub_ts->next(ts->sub_ts);
+  if (tk == NULL) return tk;
+
+  i = mbstowcs(wbuf, tk->text, MAX_WORD_SIZE);
+  w = wbuf;
+  while (*w != 0) {
+    *w = towlower(*w);
+    w++;
+  }
+  wcstombs(tk->text, wbuf, MAX_WORD_SIZE);
+  return tk;
+}
+
+TokenStream *mb_lowercase_filter_create(TokenStream *ts)
+{
+  TokenStream *tf = ALLOC(TokenStream);
+  tf->token = NULL;
+  tf->next = &mb_lcf_next;
+  tf->reset = &filter_reset;
+  tf->destroy = &filter_destroy;
+  tf->sub_ts = ts;
+  tf->destroy_sub = true;
+  tf->clone_i = NULL;
+  return tf;
 }
 
 Token *lcf_next(TokenStream *ts)
@@ -501,48 +1011,199 @@ TokenStream *lowercase_filter_create(TokenStream *ts)
   tf->reset = &filter_reset;
   tf->destroy = &filter_destroy;
   tf->sub_ts = ts;
+  tf->destroy_sub = true;
+  tf->clone_i = NULL;
   return tf;
 }
 
-Analyzer *letter_analyzer_create()
+typedef struct StemFilter {
+  struct sb_stemmer *stemmer;
+  char *algorithm;
+  char *charenc;
+} StemFilter;
+
+void stemf_destroy(void *p)
 {
-  Analyzer *a = ALLOC(Analyzer);
-  a->data = NULL;
-  a->current_ts = lowercase_filter_create(letter_tokenizer_create());
-  a->destroy = &a_standard_destroy;
-  a->get_ts = &a_standard_get_ts;
-  return a;
+  TokenStream *ts = (TokenStream *)p;
+  StemFilter *stemf = (StemFilter *)ts->data;
+  sb_stemmer_delete(stemf->stemmer);
+  free(stemf->algorithm);
+  free(stemf->charenc);
+  free(stemf);
+  filter_destroy(ts);
 }
 
+Token *stemf_next(TokenStream *ts)
+{
+  int len;
+  const sb_symbol *stemmed;
+  struct sb_stemmer *stemmer = ((StemFilter *)ts->data)->stemmer;
+  Token *tk = ts->sub_ts->next(ts->sub_ts);
+  if (tk == NULL) return tk;
+  stemmed = sb_stemmer_stem(stemmer, (sb_symbol *)tk->text, strlen(tk->text));
+  len = sb_stemmer_length(stemmer);
+  if (len >= MAX_WORD_SIZE) len = MAX_WORD_SIZE - 1;
+  memcpy(tk->text, stemmed, len);
+  tk->text[len] = '\0';
+  return tk;
+}
 
-Analyzer *standard_analyzer_create_with_words(char **words, int len)
+void stemf_clone_i(TokenStream *orig_ts, TokenStream *new_ts)
 {
-  Analyzer *a = ALLOC(Analyzer);
-  a->data = NULL;
-  a->current_ts = 
-    stop_filter_create_with_words(
+  StemFilter *orig_stemf = (StemFilter *)orig_ts->data;
+  StemFilter *stemf = ALLOC(StemFilter);
+  stemf->stemmer = sb_stemmer_new(orig_stemf->algorithm, orig_stemf->charenc);
+  stemf->algorithm = orig_stemf->algorithm ? estrdup(orig_stemf->algorithm) : NULL;
+  stemf->charenc = orig_stemf->charenc ? estrdup(orig_stemf->charenc) : NULL;
+  new_ts->data = stemf;
+}
+
+TokenStream *stem_filter_create(TokenStream *ts, const char * algorithm,
+    const char * charenc)
+{
+  TokenStream *tf = ALLOC(TokenStream);
+  StemFilter *stemf = ALLOC(StemFilter);
+  stemf->stemmer = sb_stemmer_new(algorithm, charenc);
+  stemf->algorithm = algorithm ? estrdup(algorithm) : NULL;
+  stemf->charenc = charenc ? estrdup(charenc) : NULL;
+  tf->data = stemf;
+
+  tf->token = NULL;
+  tf->next = &stemf_next;
+  tf->reset = &filter_reset;
+  tf->destroy = &stemf_destroy;
+  tf->clone_i = &stemf_clone_i;
+  tf->sub_ts = ts;
+  tf->destroy_sub = true;
+  return tf;
+}
+
+Analyzer *standard_analyzer_create_with_words_len(
+    const char **words, int len, bool lowercase)
+{
+  TokenStream *ts;
+  if (lowercase) {
+    ts = stop_filter_create_with_words_len(
         lowercase_filter_create(standard_tokenizer_create()), words, len);
-  a->destroy = &a_standard_destroy;
-  a->get_ts = &a_standard_get_ts;
-  return a;
+  } else {
+    ts = stop_filter_create_with_words_len(
+        standard_tokenizer_create(), words, len);
+  }
+  return analyzer_create(NULL, ts, NULL, NULL);
+}
+
+Analyzer *standard_analyzer_create_with_words(const char **words, bool lowercase)
+{
+  TokenStream *ts;
+  if (lowercase) {
+    ts = stop_filter_create_with_words(
+        lowercase_filter_create(standard_tokenizer_create()), words);
+  } else {
+    ts = stop_filter_create_with_words(
+        standard_tokenizer_create(), words);
+  }
+  return analyzer_create(NULL, ts, NULL, NULL);
+}
+
+Analyzer *mb_standard_analyzer_create_with_words_len(
+    const char **words, int len, bool lowercase)
+{
+  TokenStream *ts;
+  if (lowercase) {
+    ts = stop_filter_create_with_words_len(
+      mb_lowercase_filter_create(mb_standard_tokenizer_create()), words, len);
+  } else {
+    ts = stop_filter_create_with_words_len(
+        mb_standard_tokenizer_create(), words, len);
+  }
+  return analyzer_create(NULL, ts, NULL, NULL);
+}
+
+Analyzer *mb_standard_analyzer_create_with_words(
+    const char **words, bool lowercase)
+{
+  TokenStream *ts;
+  if (lowercase) {
+    ts = stop_filter_create_with_words(
+      mb_lowercase_filter_create(mb_standard_tokenizer_create()), words);
+  } else {
+    ts = stop_filter_create_with_words(mb_standard_tokenizer_create(), words);
+  }
+  return analyzer_create(NULL, ts, NULL, NULL);
+}
+
+Analyzer *standard_analyzer_create(bool lowercase)
+{
+  return standard_analyzer_create_with_words(FULL_ENGLISH_STOP_WORDS, lowercase);
+}
+
+Analyzer *mb_standard_analyzer_create(bool lowercase)
+{
+  return mb_standard_analyzer_create_with_words(FULL_ENGLISH_STOP_WORDS, lowercase);
+}
+
+/****************************************************************************
+ *
+ * PerFieldAnalyzer
+ *
+ ****************************************************************************/
+
+typedef struct PerFieldAnalyzer {
+  HshTable *dict;
+  Analyzer *def;
+  bool destroy_subs : 1;
+} PerFieldAnalyzer;
+
+void pfa_destroy(void *p)
+{
+  Analyzer *self = (Analyzer *)p;
+  PerFieldAnalyzer *pfa = (PerFieldAnalyzer *)self->data;
+  h_destroy(pfa->dict);
+
+  if (pfa->destroy_subs) a_destroy(pfa->def);
+  free(pfa);
+  free(self);
+}
+
+TokenStream *pfa_get_ts(Analyzer *self, char *field, char *text)
+{
+  PerFieldAnalyzer *pfa = (PerFieldAnalyzer *)self->data;
+  Analyzer *a = h_get(pfa->dict, field);
+  if (a == NULL) a = pfa->def;
+  return a_get_ts(a, field, text);
+}
+
+void pfa_sub_a_destroy(void *p)
+{
+  Analyzer *a = (Analyzer *)p;
+  a->destroy(a);
+}
+
+void pfa_add_field(Analyzer *self, char *field, Analyzer *analyzer)
+{
+  PerFieldAnalyzer *pfa = (PerFieldAnalyzer *)self->data;
+  h_set(pfa->dict, estrdup(field), analyzer);
 }
 
-Analyzer *standard_analyzer_create()
+Analyzer *per_field_analyzer_create(Analyzer *def, bool destroy_subs)
 {
-  return standard_analyzer_create_with_words(
-        (char **)ENGLISH_STOP_WORDS, NELEMS(ENGLISH_STOP_WORDS));
+  PerFieldAnalyzer *pfa = ALLOC(PerFieldAnalyzer);
+  pfa->def = def;
+  pfa->destroy_subs = destroy_subs;
+  pfa->dict = destroy_subs ? h_new_str(&free, &pfa_sub_a_destroy)
+                           : h_new_str(&free, NULL);
+  return analyzer_create(pfa, NULL, &pfa_destroy, &pfa_get_ts);
 }
 
 #ifdef ALONE
 int main(int argc, char **argv)
 {
   char buf[10000];
-  Analyzer *a = standard_analyzer_create();
+  Analyzer *a = standard_analyzer_create(true);
   TokenStream *ts;
   Token *tk;
   while (fgets(buf, 9999, stdin) != NULL) {
     ts = a->get_ts(a, "hello", buf);
-    ts->pos = 0;
     while ((tk = ts->next(ts)) != NULL) {
       printf("<%s:%ld:%ld> ", tk->text, tk->start, tk->end);
     }