ferret 0.11.6 → 0.11.8.4

data/ext/{utilities.c → STEMMER_utilities.c}

@@ -25,8 +25,8 @@ extern void lose_s(symbol * p) {
 }
 
 /*
-   new_p = X_skip_utf8(p, c, lb, l, n); skips n characters forwards from p + c
-   if n +ve, or n characters backwards from p +c - 1 if n -ve. new_p is the new
+   new_p = skip_utf8(p, c, lb, l, n); skips n characters forwards from p + c
+   if n +ve, or n characters backwards from p + c - 1 if n -ve. new_p is the new
    position, or 0 on failure.
 
    -- used to implement hop and next in the utf8 case.
@@ -76,7 +76,7 @@ static int get_utf8(const symbol * p, int c, int l, int * slot) {
     if (b0 < 0xE0 || c == l) {   /* 1110 0000 */
         * slot = (b0 & 0x1F) << 6 | (b1 & 0x3F); return 2;
     }
-    * slot = (b0 & 0xF) << 12 | (b1 & 0x3F) << 6 | (*p & 0x3F); return 3;
+    * slot = (b0 & 0xF) << 12 | (b1 & 0x3F) << 6 | (p[c] & 0x3F); return 3;
 }
 
 static int get_b_utf8(const symbol * p, int c, int lb, int * slot) {
@@ -90,94 +90,126 @@ static int get_b_utf8(const symbol * p, int c, int lb, int * slot) {
     if (b1 >= 0xC0 || c == lb) {   /* 1100 0000 */
         * slot = (b1 & 0x1F) << 6 | (b0 & 0x3F); return 2;
     }
-    * slot = (*p & 0xF) << 12 | (b1 & 0x3F) << 6 | (b0 & 0x3F); return 3;
+    * slot = (p[c] & 0xF) << 12 | (b1 & 0x3F) << 6 | (b0 & 0x3F); return 3;
 }
 
-extern int in_grouping_U(struct SN_env * z, unsigned char * s, int min, int max) {
-    int ch;
-    int w = get_utf8(z->p, z->c, z->l, & ch);
-    unless (w) return 0;
-    if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0) return 0;
-    z->c += w; return 1;
+extern int in_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
+    do {
+	int ch;
+	int w = get_utf8(z->p, z->c, z->l, & ch);
+	unless (w) return -1;
+	if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
+	    return w;
+	z->c += w;
+    } while (repeat);
+    return 0;
 }
 
-extern int in_grouping_b_U(struct SN_env * z, unsigned char * s, int min, int max) {
-    int ch;
-    int w = get_b_utf8(z->p, z->c, z->lb, & ch);
-    unless (w) return 0;
-    if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0) return 0;
-    z->c -= w; return 1;
+extern int in_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
+    do {
+	int ch;
+	int w = get_b_utf8(z->p, z->c, z->lb, & ch);
+	unless (w) return -1;
+	if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
+	    return w;
+	z->c -= w;
+    } while (repeat);
+    return 0;
 }
 
-extern int out_grouping_U(struct SN_env * z, unsigned char * s, int min, int max) {
-    int ch;
-    int w = get_utf8(z->p, z->c, z->l, & ch);
-    unless (w) return 0;
-    unless (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0) return 0;
-    z->c += w; return 1;
+extern int out_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
+    do {
+	int ch;
+	int w = get_utf8(z->p, z->c, z->l, & ch);
+	unless (w) return -1;
+	unless (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
+	    return w;
+	z->c += w;
+    } while (repeat);
+    return 0;
 }
 
-extern int out_grouping_b_U(struct SN_env * z, unsigned char * s, int min, int max) {
-    int ch;
-    int w = get_b_utf8(z->p, z->c, z->lb, & ch);
-    unless (w) return 0;
-    unless (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0) return 0;
-    z->c -= w; return 1;
+extern int out_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
+    do {
+	int ch;
+	int w = get_b_utf8(z->p, z->c, z->lb, & ch);
+	unless (w) return -1;
+	unless (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
+	    return w;
+	z->c -= w;
+    } while (repeat);
+    return 0;
 }
 
 /* Code for character groupings: non-utf8 cases */
 
-extern int in_grouping(struct SN_env * z, unsigned char * s, int min, int max) {
-    int ch;
-    if (z->c >= z->l) return 0;
-    ch = z->p[z->c];
-    if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0) return 0;
-    z->c++; return 1;
+extern int in_grouping(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
+    do {
+	int ch;
+	if (z->c >= z->l) return -1;
+	ch = z->p[z->c];
+	if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
+	    return 1;
+	z->c++;
+    } while (repeat);
+    return 0;
 }
 
-extern int in_grouping_b(struct SN_env * z, unsigned char * s, int min, int max) {
-    int ch;
-    if (z->c <= z->lb) return 0;
-    ch = z->p[z->c - 1];
-    if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0) return 0;
-    z->c--; return 1;
+extern int in_grouping_b(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
+    do {
+	int ch;
+	if (z->c <= z->lb) return -1;
+	ch = z->p[z->c - 1];
+	if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
+	    return 1;
+	z->c--;
+    } while (repeat);
+    return 0;
 }
 
-extern int out_grouping(struct SN_env * z, unsigned char * s, int min, int max) {
-    int ch;
-    if (z->c >= z->l) return 0;
-    ch = z->p[z->c];
-    unless (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0) return 0;
-    z->c++; return 1;
+extern int out_grouping(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
+    do {
+	int ch;
+	if (z->c >= z->l) return -1;
+	ch = z->p[z->c];
+	unless (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
+	    return 1;
+	z->c++;
+    } while (repeat);
+    return 0;
 }
 
-extern int out_grouping_b(struct SN_env * z, unsigned char * s, int min, int max) {
-    int ch;
-    if (z->c <= z->lb) return 0;
-    ch = z->p[z->c - 1];
-    unless (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0) return 0;
-    z->c--; return 1;
+extern int out_grouping_b(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
+    do {
+	int ch;
+	if (z->c <= z->lb) return -1;
+	ch = z->p[z->c - 1];
+	unless (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
+	    return 1;
+	z->c--;
+    } while (repeat);
+    return 0;
 }
 
-extern int eq_s(struct SN_env * z, int s_size, symbol * s) {
+extern int eq_s(struct SN_env * z, int s_size, const symbol * s) {
     if (z->l - z->c < s_size || memcmp(z->p + z->c, s, s_size * sizeof(symbol)) != 0) return 0;
     z->c += s_size; return 1;
 }
 
-extern int eq_s_b(struct SN_env * z, int s_size, symbol * s) {
+extern int eq_s_b(struct SN_env * z, int s_size, const symbol * s) {
     if (z->c - z->lb < s_size || memcmp(z->p + z->c - s_size, s, s_size * sizeof(symbol)) != 0) return 0;
     z->c -= s_size; return 1;
 }
 
-extern int eq_v(struct SN_env * z, symbol * p) {
+extern int eq_v(struct SN_env * z, const symbol * p) {
     return eq_s(z, SIZE(p), p);
 }
 
-extern int eq_v_b(struct SN_env * z, symbol * p) {
+extern int eq_v_b(struct SN_env * z, const symbol * p) {
     return eq_s_b(z, SIZE(p), p);
 }
 
-extern int find_among(struct SN_env * z, struct among * v, int v_size) {
+extern int find_among(struct SN_env * z, const struct among * v, int v_size) {
 
     int i = 0;
     int j = v_size;
@@ -185,7 +217,7 @@ extern int find_among(struct SN_env * z, struct among * v, int v_size) {
     int c = z->c; int l = z->l;
     symbol * q = z->p + c;
 
-    struct among * w;
+    const struct among * w;
 
     int common_i = 0;
     int common_j = 0;
@@ -198,9 +230,9 @@ extern int find_among(struct SN_env * z, struct among * v, int v_size) {
         int common = common_i < common_j ? common_i : common_j; /* smaller */
         w = v + k;
         {
-            int i; for (i = common; i < w->s_size; i++) {
+            int i2; for (i2 = common; i2 < w->s_size; i2++) {
                 if (c + common == l) { diff = -1; break; }
-                diff = q[common] - w->s[i];
+                diff = q[common] - w->s[i2];
                 if (diff != 0) break;
                 common++;
             }
@@ -237,7 +269,7 @@ extern int find_among(struct SN_env * z, struct among * v, int v_size) {
 
 /* find_among_b is for backwards processing. Same comments apply */
 
-extern int find_among_b(struct SN_env * z, struct among * v, int v_size) {
+extern int find_among_b(struct SN_env * z, const struct among * v, int v_size) {
 
     int i = 0;
     int j = v_size;
@@ -245,7 +277,7 @@ extern int find_among_b(struct SN_env * z, struct among * v, int v_size) {
     int c = z->c; int lb = z->lb;
     symbol * q = z->p + c - 1;
 
-    struct among * w;
+    const struct among * w;
 
     int common_i = 0;
     int common_j = 0;
@@ -258,9 +290,9 @@ extern int find_among_b(struct SN_env * z, struct among * v, int v_size) {
         int common = common_i < common_j ? common_i : common_j;
         w = v + k;
         {
-            int i; for (i = w->s_size - 1 - common; i >= 0; i--) {
+            int i2; for (i2 = w->s_size - 1 - common; i2 >= 0; i2--) {
                 if (c - common == lb) { diff = -1; break; }
-                diff = q[- common] - w->s[i];
+                diff = q[- common] - w->s[i2];
                 if (diff != 0) break;
                 common++;
             }
@@ -362,12 +394,12 @@ static int slice_check(struct SN_env * z) {
     return 0;
 }
 
-extern int slice_from_s(struct SN_env * z, int s_size, symbol * s) {
+extern int slice_from_s(struct SN_env * z, int s_size, const symbol * s) {
     if (slice_check(z)) return -1;
     return replace_s(z, z->bra, z->ket, s_size, s, NULL);
 }
 
-extern int slice_from_v(struct SN_env * z, symbol * p) {
+extern int slice_from_v(struct SN_env * z, const symbol * p) {
     return slice_from_s(z, SIZE(p), p);
 }
 
@@ -375,7 +407,7 @@ extern int slice_del(struct SN_env * z) {
     return slice_from_s(z, 0, 0);
 }
 
-extern int insert_s(struct SN_env * z, int bra, int ket, int s_size, symbol * s) {
+extern int insert_s(struct SN_env * z, int bra, int ket, int s_size, const symbol * s) {
     int adjustment;
     if (replace_s(z, bra, ket, s_size, s, &adjustment))
         return -1;
@@ -384,7 +416,7 @@ extern int insert_s(struct SN_env * z, int bra, int ket, int s_size, symbol * s)
     return 0;
 }
 
-extern int insert_v(struct SN_env * z, int bra, int ket, symbol * p) {
+extern int insert_v(struct SN_env * z, int bra, int ket, const symbol * p) {
     int adjustment;
     if (replace_s(z, bra, ket, SIZE(p), p, &adjustment))
         return -1;

data/ext/analysis.c

@@ -5,6 +5,8 @@
 #include <ctype.h>
 #include <wctype.h>
 #include <wchar.h>
+#include "internal.h"
+#include "scanner.h"
 
 /****************************************************************************
  *
@@ -27,8 +29,8 @@ INLINE Token *tk_set(Token *tk,
     return tk;
 }
 
-INLINE Token *tk_set_ts(Token *tk,
-                        char *start, char *end, char *text, int pos_inc)
+static INLINE Token *tk_set_ts(Token *tk, char *start, char *end,
+                               char *text, int pos_inc)
 {
     return tk_set(tk, start, (int)(end - start),
                   (off_t)(start - text), (off_t)(end - text), pos_inc);
@@ -40,8 +42,8 @@ INLINE Token *tk_set_no_len(Token *tk,
     return tk_set(tk, text, (int)strlen(text), start, end, pos_inc);
 }
 
-INLINE Token *w_tk_set(Token *tk, wchar_t *text, off_t start, off_t end,
-                       int pos_inc)
+static INLINE Token *w_tk_set(Token *tk, wchar_t *text, off_t start,
+                              off_t end, int pos_inc)
 {
     int len = wcstombs(tk->text, text, MAX_WORD_SIZE - 1);
     tk->text[len] = '\0';
@@ -121,7 +123,7 @@ TokenStream *ts_clone_size(TokenStream *orig_ts, size_t size)
 
 TokenStream *ts_new_i(size_t size)
 {
-    TokenStream *ts = ecalloc(size);
+    TokenStream *ts = (TokenStream *)ecalloc(size);
 
     ts->destroy_i = (void (*)(TokenStream *))&free;
     ts->reset = &ts_reset;
@@ -152,7 +154,7 @@ static TokenStream *cts_new()
 
 #define MBTS(token_stream) ((MultiByteTokenStream *)(token_stream))
 
-INLINE int mb_next_char(wchar_t *wchr, const char *s, mbstate_t *state)
+static INLINE int mb_next_char(wchar_t *wchr, const char *s, mbstate_t *state)
 {
     int num_bytes;
     if ((num_bytes = (int)mbrtowc(wchr, s, MB_CUR_MAX, state)) < 0) {
@@ -180,7 +182,7 @@ static TokenStream *mb_ts_clone_i(TokenStream *orig_ts)
     return ts_clone_size(orig_ts, sizeof(MultiByteTokenStream));
 }
 
-TokenStream *mb_ts_new()
+static TokenStream *mb_ts_new()
 {
     TokenStream *ts = ts_new(MultiByteTokenStream);
     ts->reset = &mb_ts_reset;
@@ -210,7 +212,9 @@ static void a_standard_destroy_i(Analyzer *a)
     free(a);
 }
 
-static TokenStream *a_standard_get_ts(Analyzer *a, char *field, char *text)
+static TokenStream *a_standard_get_ts(Analyzer *a,
+                                      Symbol field,
+                                      char *text)
 {
     TokenStream *ts;
     (void)field;
@@ -220,7 +224,8 @@ static TokenStream *a_standard_get_ts(Analyzer *a, char *field, char *text)
 
 Analyzer *analyzer_new(TokenStream *ts,
                        void (*destroy_i)(Analyzer *a),
-                       TokenStream *(*get_ts)(Analyzer *a, char *field,
+                       TokenStream *(*get_ts)(Analyzer *a,
+                                              Symbol field,
                                               char *text))
 {
     Analyzer *a = ALLOC(Analyzer);
@@ -414,7 +419,7 @@ Analyzer *mb_whitespace_analyzer_new(bool lowercase)
 /*
  * LetterTokenizer
  */
-Token *lt_next(TokenStream *ts)
+static Token *lt_next(TokenStream *ts)
 {
     char *start;
     char *t = ts->t;
@@ -446,7 +451,7 @@ TokenStream *letter_tokenizer_new()
 /*
  * Multi-byte LetterTokenizer
  */
-Token *mb_lt_next(TokenStream *ts)
+static Token *mb_lt_next(TokenStream *ts)
 {
     int i;
     char *start;
@@ -478,7 +483,7 @@ Token *mb_lt_next(TokenStream *ts)
 /*
  * Lowercasing Multi-byte LetterTokenizer
  */
-Token *mb_lt_next_lc(TokenStream *ts)
+static Token *mb_lt_next_lc(TokenStream *ts)
 {
     int i;
     char *start;
@@ -554,43 +559,88 @@ Analyzer *mb_letter_analyzer_new(bool lowercase)
 /*
  * StandardTokenizer
  */
-static int std_get_alpha(TokenStream *ts, char *token)
+static Token *std_next(TokenStream *ts)
 {
-    int i = 0;
-    char *t = ts->t;
-    while (t[i] != '\0' && isalnum(t[i])) {
-        if (i < MAX_WORD_SIZE) {
-            token[i] = t[i];
-        }
-        i++;
+    StandardTokenizer *std_tz = STDTS(ts);
+    const char *start = NULL;
+    const char *end = NULL;
+    int len;
+    Token *tk = &(CTS(ts)->token);
+
+    switch (std_tz->type) {
+        case STT_ASCII:
+            frt_std_scan(ts->t, tk->text, sizeof(tk->text) - 1,
+                         &start, &end, &len);
+            break;
+        case STT_MB:
+            frt_std_scan_mb(ts->t, tk->text, sizeof(tk->text) - 1,
+                            &start, &end, &len);
+            break;
+        case STT_UTF8:
+            frt_std_scan_utf8(ts->t, tk->text, sizeof(tk->text) - 1,
+                              &start, &end, &len);
+            break;
     }
-    return i;
+
+    if (len == 0)
+        return NULL;
+
+    ts->t       = (char *)end;
+    tk->len     = len;
+    tk->start   = start - ts->text;
+    tk->end     = end   - ts->text;
+    tk->pos_inc = 1;
+    return &(CTS(ts)->token);
 }
 
-static int mb_std_get_alpha(TokenStream *ts, char *token)
+static TokenStream *std_ts_clone_i(TokenStream *orig_ts)
 {
-    char *t = ts->t;
-    wchar_t wchr;
-    int i;
-    mbstate_t state; ZEROSET(&state, mbstate_t);
+    return ts_clone_size(orig_ts, sizeof(StandardTokenizer));
+}
 
-    i = mb_next_char(&wchr, t, &state);
+static TokenStream *std_ts_new()
+{
+    TokenStream *ts = ts_new(StandardTokenizer);
 
-    while (wchr != 0 && iswalnum(wchr)) {
-        t += i;
-        i = mb_next_char(&wchr, t, &state);
-    }
+    ts->clone_i     = &std_ts_clone_i;
+    ts->next        = &std_next;
 
-    i = (int)(t - ts->t);
-    if (i > MAX_WORD_SIZE) {
-        i = MAX_WORD_SIZE - 1;
-    }
-    memcpy(token, ts->t, i);
-    return i;
+    return ts;
+}
+
+TokenStream *standard_tokenizer_new()
+{
+    TokenStream *ts = std_ts_new();
+    STDTS(ts)->type = STT_ASCII;
+    return ts;
+}
+
+TokenStream *mb_standard_tokenizer_new()
+{
+    TokenStream *ts = std_ts_new();
+    STDTS(ts)->type = STT_MB;
+    return ts;
 }
 
+TokenStream *utf8_standard_tokenizer_new()
+{
+    TokenStream *ts = std_ts_new();
+    STDTS(ts)->type = STT_UTF8;
+    return ts;
+}
+
+/****************************************************************************
+ *
+ * LegacyStandard
+ *
+ ****************************************************************************/
+
+#define LSTDTS(token_stream) ((LegacyStandardTokenizer *)(token_stream))
+
 /*
-static int std_get_alnum(TokenStream *ts, char *token)
+ * LegacyStandardTokenizer
+ */
+static int legacy_std_get_alpha(TokenStream *ts, char *token)
 {
     int i = 0;
     char *t = ts->t;
@@ -603,7 +653,7 @@ static int std_get_alnum(TokenStream *ts, char *token)
     return i;
 }
 
-static int mb_std_get_alnum(TokenStream *ts, char *token)
+static int mb_legacy_std_get_alpha(TokenStream *ts, char *token)
 {
     char *t = ts->t;
     wchar_t wchr;
@@ -624,7 +674,6 @@ static int mb_std_get_alnum(TokenStream *ts, char *token)
     memcpy(token, ts->t, i);
     return i;
 }
-*/
 
 static int isnumpunc(char c)
 {
@@ -659,7 +708,7 @@ static int isurlxatc(char c)
             || isalnum(c));
 }
 
-static bool std_is_tok_char(char *c)
+static bool legacy_std_is_tok_char(char *c)
 {
     if (isspace(*c)) {
         return false;           /* most common so check first. */
@@ -671,11 +720,11 @@ static bool std_is_tok_char(char *c)
     return false;
 }
 
-static bool mb_std_is_tok_char(char *t)
+static bool mb_legacy_std_is_tok_char(char *t)
 {
     wchar_t c;
     mbstate_t state; ZEROSET(&state, mbstate_t);
-    
+
     if (((int)mbrtowc(&c, t, MB_CUR_MAX, &state)) < 0) {
         /* error which we can handle next time round. For now just return
          * false so that we can return a token */
@@ -696,7 +745,7 @@ static bool mb_std_is_tok_char(char *t)
  * (alnum) = [a-zA-Z0-9]
  * (punc) = [_\/.,-]
  */
-static int std_get_number(char *input)
+static int legacy_std_get_number(char *input)
 {
     int i = 0;
     int count = 0;
@@ -732,7 +781,7 @@ static int std_get_number(char *input)
     }
 }
 
-static int std_get_apostrophe(char *input)
+static int legacy_std_get_apostrophe(char *input)
 {
     char *t = input;
 
@@ -743,7 +792,7 @@ static int std_get_apostrophe(char *input)
     return (int)(t - input);
 }
 
-static int mb_std_get_apostrophe(char *input)
+static int mb_legacy_std_get_apostrophe(char *input)
 {
     char *t = input;
     wchar_t wchr;
@@ -759,8 +808,9 @@ static int mb_std_get_apostrophe(char *input)
     return (int)(t - input);
 }
 
-static int std_get_url(char *input, char *token, int i)
+static char *std_get_url(char *input, char *token, int i, int *len)
 {
+    char *next = NULL;
     while (isurlc(input[i])) {
         if (isurlpunc(input[i]) && isurlpunc(input[i - 1])) {
             break; /* can't have two puncs in a row */
@@ -770,18 +820,26 @@ static int std_get_url(char *input, char *token, int i)
         }
         i++;
     }
+    next = input + i;
+
+    /* We don't want to index past the end of the token capacity) */
+    if (i >= MAX_WORD_SIZE) {
+        i = MAX_WORD_SIZE - 1;
+    }
 
     /* strip trailing puncs */
     while (isurlpunc(input[i - 1])) {
         i--;
     }
+    *len = i;
+    token[i] = '\0';
 
-    return i;
+    return next;
 }
 
 /* Company names can contain '@' and '&' like AT&T and Excite@Home. Let's
 */
-static int std_get_company_name(char *input)
+static int legacy_std_get_company_name(char *input)
 {
     int i = 0;
     while (isalpha(input[i]) || input[i] == '@' || input[i] == '&') {
@@ -791,25 +849,7 @@ static int std_get_company_name(char *input)
     return i;
 }
 
-/*
-static int mb_std_get_company_name(char *input, TokenStream *ts)
-{
-    char *t = input;
-    wchar_t wchr;
-    int i;
-    mbstate_t state; ZEROSET(&state, mbstate_t);
-
-    i = mb_next_char(&wchr, t, &state);
-    while (iswalpha(wchr) || wchr == L'@' || wchr == L'&') {
-        t += i;
-        i = mb_next_char(&wchr, t, &state);
-    }
-
-    return (int)(t - input);
-}
-*/
-
-static bool std_advance_to_start(TokenStream *ts)
+static bool legacy_std_advance_to_start(TokenStream *ts)
 {
     char *t = ts->t;
     while (*t != '\0' && !isalnum(*t)) {
@@ -822,7 +862,7 @@ static bool std_advance_to_start(TokenStream *ts)
     return (*t != '\0');
 }
 
-static bool mb_std_advance_to_start(TokenStream *ts)
+static bool mb_legacy_std_advance_to_start(TokenStream *ts)
 {
     int i;
     wchar_t wchr;
@@ -839,9 +879,9 @@ static bool mb_std_advance_to_start(TokenStream *ts)
     return (wchr != 0);
 }
 
-static Token *std_next(TokenStream *ts)
+static Token *legacy_std_next(TokenStream *ts)
 {
-    StandardTokenizer *std_tz = STDTS(ts);
+    LegacyStandardTokenizer *std_tz = LSTDTS(ts);
     char *s;
     char *t;
     char *start = NULL;
@@ -890,13 +930,13 @@ static Token *std_next(TokenStream *ts)
     }
 
     if (*t == '&') {        /* apostrophe case. */
-        t += std_get_company_name(t);
+        t += legacy_std_get_company_name(t);
         ts->t = t;
         return tk_set_ts(&(CTS(ts)->token), start, t, ts->text, 1);
     }
 
-    if ((isdigit(*t) || isnumpunc(*t))       /* possibly a number */
-        && (len = std_get_number(t) > 0)) {
+    if ((isdigit(*start) || isnumpunc(*start))       /* possibly a number */
+        && ((len = legacy_std_get_number(start)) > 0)) {
         num_end = start + len;
         if (!std_tz->is_tok_char(num_end)) { /* won't find a longer token */
             ts->t = num_end;
@@ -909,6 +949,7 @@ static Token *std_next(TokenStream *ts)
         /* check for a known url start */
         token[token_i] = '\0';
         t += 3;
+        token_i += 3;
         while (*t == '/') {
             t++;
         }
@@ -917,17 +958,16 @@ static Token *std_next(TokenStream *ts)
              memcmp(token, "http", 4) == 0 ||
              memcmp(token, "https", 5) == 0 ||
              memcmp(token, "file", 4) == 0)) {
-            len = std_get_url(t, token, 0); /* dispose of first part of the URL */
+            ts->t = std_get_url(t, token, 0, &len); /* dispose of first part of the URL */
         }
         else {              /* still treat as url but keep the first part */
             token_i = (int)(t - start);
             memcpy(token, start, token_i * sizeof(char));
-            len = token_i + std_get_url(t, token, token_i); /* keep start */
+            ts->t = std_get_url(start, token, token_i, &len); /* keep start */
         }
-        ts->t = t + len;
-        token[len] = 0;
-        return tk_set(&(CTS(ts)->token), token, len, (off_t)(start - ts->text),
-               (off_t)(ts->t - ts->text), 1);
+        return tk_set(&(CTS(ts)->token), token, len,
+                      (off_t)(start - ts->text),
+                      (off_t)(ts->t - ts->text), 1);
     }
 
     /* now see how long a url we can find. */
@@ -989,41 +1029,41 @@ static Token *std_next(TokenStream *ts)
     return &(CTS(ts)->token);
 }
 
-static TokenStream *std_ts_clone_i(TokenStream *orig_ts)
+static TokenStream *legacy_std_ts_clone_i(TokenStream *orig_ts)
 {
-    return ts_clone_size(orig_ts, sizeof(StandardTokenizer));
+    return ts_clone_size(orig_ts, sizeof(LegacyStandardTokenizer));
 }
 
-static TokenStream *std_ts_new()
+static TokenStream *legacy_std_ts_new()
 {
-    TokenStream *ts = ts_new(StandardTokenizer);
+    TokenStream *ts = ts_new(LegacyStandardTokenizer);
 
-    ts->clone_i     = &std_ts_clone_i;
-    ts->next        = &std_next;
+    ts->clone_i     = &legacy_std_ts_clone_i;
+    ts->next        = &legacy_std_next;
 
     return ts;
 }
 
-TokenStream *standard_tokenizer_new()
+TokenStream *legacy_standard_tokenizer_new()
 {
-    TokenStream *ts = std_ts_new();
+    TokenStream *ts = legacy_std_ts_new();
 
-    STDTS(ts)->advance_to_start = &std_advance_to_start;
-    STDTS(ts)->get_alpha        = &std_get_alpha;
-    STDTS(ts)->is_tok_char      = &std_is_tok_char;
-    STDTS(ts)->get_apostrophe   = &std_get_apostrophe;
+    LSTDTS(ts)->advance_to_start = &legacy_std_advance_to_start;
+    LSTDTS(ts)->get_alpha        = &legacy_std_get_alpha;
+    LSTDTS(ts)->is_tok_char      = &legacy_std_is_tok_char;
+    LSTDTS(ts)->get_apostrophe   = &legacy_std_get_apostrophe;
 
     return ts;
 }
 
-TokenStream *mb_standard_tokenizer_new()
+TokenStream *mb_legacy_standard_tokenizer_new()
 {
-    TokenStream *ts = std_ts_new();
+    TokenStream *ts = legacy_std_ts_new();
 
-    STDTS(ts)->advance_to_start = &mb_std_advance_to_start;
-    STDTS(ts)->get_alpha        = &mb_std_get_alpha;
-    STDTS(ts)->is_tok_char      = &mb_std_is_tok_char;
-    STDTS(ts)->get_apostrophe   = &mb_std_get_apostrophe;
+    LSTDTS(ts)->advance_to_start = &mb_legacy_std_advance_to_start;
+    LSTDTS(ts)->get_alpha        = &mb_legacy_std_get_alpha;
+    LSTDTS(ts)->is_tok_char      = &mb_legacy_std_is_tok_char;
+    LSTDTS(ts)->get_apostrophe   = &mb_legacy_std_get_apostrophe;
 
     return ts;
 }
@@ -1060,7 +1100,6 @@ static void filter_destroy_i(TokenStream *ts)
     free(ts);
 }
 
-#define tf_new(type, sub) tf_new_i(sizeof(type), sub)
 TokenStream *tf_new_i(size_t size, TokenStream *sub_ts)
 {
     TokenStream *ts     = (TokenStream *)ecalloc(size);
@@ -1097,7 +1136,7 @@ static TokenStream *sf_clone_i(TokenStream *orig_ts)
 static Token *sf_next(TokenStream *ts)
 {
     int pos_inc = 0;
-    HashTable *words = StopFilt(ts)->words;
+    Hash *words = StopFilt(ts)->words;
     TokenFilter *tf = TkFilt(ts);
     Token *tk = tf->sub_ts->next(tf->sub_ts);
 
@@ -1118,7 +1157,7 @@ TokenStream *stop_filter_new_with_words_len(TokenStream *sub_ts,
 {
     int i;
     char *word;
-    HashTable *word_table = h_new_str(&free, (free_ft) NULL);
+    Hash *word_table = h_new_str(&free, (free_ft) NULL);
     TokenStream *ts = tf_new(StopFilter, sub_ts);
 
     for (i = 0; i < len; i++) {
@@ -1136,7 +1175,7 @@ TokenStream *stop_filter_new_with_words(TokenStream *sub_ts,
                                         const char **words)
 {
     char *word;
-    HashTable *word_table = h_new_str(&free, (free_ft) NULL);
+    Hash *word_table = h_new_str(&free, (free_ft) NULL);
     TokenStream *ts = tf_new(StopFilter, sub_ts);
 
     while (*words) {
@@ -1234,7 +1273,7 @@ static Token *hf_next(TokenStream *ts)
     HyphenFilter *hf = HyphenFilt(ts);
     TokenFilter *tf = TkFilt(ts);
     Token *tk = hf->tk;
-    
+
     if (hf->pos < hf->len) {
         const int pos = hf->pos;
         const int text_len = strlen(hf->text + pos);
@@ -1301,7 +1340,7 @@ TokenStream *hyphen_filter_new(TokenStream *sub_ts)
  ****************************************************************************/
 
 
-Token *mb_lcf_next(TokenStream *ts)
+static Token *mb_lcf_next(TokenStream *ts)
 {
     wchar_t wbuf[MAX_WORD_SIZE + 1], *wchr;
     Token *tk = TkFilt(ts)->sub_ts->next(TkFilt(ts)->sub_ts);
@@ -1334,7 +1373,7 @@ TokenStream *mb_lowercase_filter_new(TokenStream *sub_ts)
     return ts;
 }
 
-Token *lcf_next(TokenStream *ts)
+static Token *lcf_next(TokenStream *ts)
 {
     int i = 0;
     Token *tk = TkFilt(ts)->sub_ts->next(TkFilt(ts)->sub_ts);
@@ -1361,7 +1400,7 @@ TokenStream *lowercase_filter_new(TokenStream *sub_ts)
 
 #define StemFilt(filter) ((StemFilter *)(filter))
 
-void stemf_destroy_i(TokenStream *ts)
+static void stemf_destroy_i(TokenStream *ts)
 {
     sb_stemmer_delete(StemFilt(ts)->stemmer);
     free(StemFilt(ts)->algorithm);
@@ -1369,7 +1408,7 @@ void stemf_destroy_i(TokenStream *ts)
     filter_destroy_i(ts);
 }
 
-Token *stemf_next(TokenStream *ts)
+static Token *stemf_next(TokenStream *ts)
 {
     int len;
     const sb_symbol *stemmed;
@@ -1391,7 +1430,7 @@ Token *stemf_next(TokenStream *ts)
     return tk;
 }
 
-TokenStream *stemf_clone_i(TokenStream *orig_ts)
+static TokenStream *stemf_clone_i(TokenStream *orig_ts)
 {
     TokenStream *new_ts      = filter_clone_size(orig_ts, sizeof(StemFilter));
     StemFilter *stemf        = StemFilt(new_ts);
@@ -1409,10 +1448,35 @@ TokenStream *stem_filter_new(TokenStream *ts, const char *algorithm,
                              const char *charenc)
 {
     TokenStream *tf = tf_new(StemFilter, ts);
+    char *my_algorithm = NULL;
+    char *my_charenc   = NULL;
+    char *s = NULL;
+
+    if (algorithm) {
+        my_algorithm = estrdup(algorithm);
+
+        /* algorithms are lowercase */
+        s = my_algorithm;
+        while (*s) {
+            *s = tolower(*s);
+            s++;
+        }
+        StemFilt(tf)->algorithm = my_algorithm;
+    }
+
+    if (charenc) {
+        my_charenc   = estrdup(charenc);
 
-    StemFilt(tf)->stemmer   = sb_stemmer_new(algorithm, charenc);
-    StemFilt(tf)->algorithm = algorithm ? estrdup(algorithm) : NULL;
-    StemFilt(tf)->charenc   = charenc ? estrdup(charenc) : NULL;
+        /* encodings are uppercase and use '_' instead of '-' */
+        s = my_charenc;
+        while (*s) {
+            *s = (*s == '-') ? '_' : toupper(*s);
+            s++;
+        }
+        StemFilt(tf)->charenc = my_charenc;
+    }
+
+    StemFilt(tf)->stemmer   = sb_stemmer_new(my_algorithm, my_charenc);
 
     tf->next = &stemf_next;
     tf->destroy_i = &stemf_destroy_i;
@@ -1474,6 +1538,28 @@ Analyzer *mb_standard_analyzer_new_with_words(const char **words,
     return analyzer_new(ts, NULL, NULL);
 }
 
+Analyzer *utf8_standard_analyzer_new_with_words_len(const char **words,
+                                                  int len, bool lowercase)
+{
+    TokenStream *ts = utf8_standard_tokenizer_new();
+    if (lowercase) {
+        ts = mb_lowercase_filter_new(ts);
+    }
+    ts = hyphen_filter_new(stop_filter_new_with_words_len(ts, words, len));
+    return analyzer_new(ts, NULL, NULL);
+}
+
+Analyzer *utf8_standard_analyzer_new_with_words(const char **words,
+                                              bool lowercase)
+{
+    TokenStream *ts = utf8_standard_tokenizer_new();
+    if (lowercase) {
+        ts = mb_lowercase_filter_new(ts);
+    }
+    ts = hyphen_filter_new(stop_filter_new_with_words(ts, words));
+    return analyzer_new(ts, NULL, NULL);
+}
+
 Analyzer *standard_analyzer_new(bool lowercase)
 {
     return standard_analyzer_new_with_words(FULL_ENGLISH_STOP_WORDS,
@@ -1486,14 +1572,79 @@ Analyzer *mb_standard_analyzer_new(bool lowercase)
                                                lowercase);
 }
 
+Analyzer *utf8_standard_analyzer_new(bool lowercase)
+{
+    return utf8_standard_analyzer_new_with_words(FULL_ENGLISH_STOP_WORDS,
+                                                 lowercase);
+}
+
+/****************************************************************************
+ * Legacy
+ ****************************************************************************/
+
+Analyzer *legacy_standard_analyzer_new_with_words_len(const char **words, int len,
+                                                      bool lowercase)
+{
+    TokenStream *ts = legacy_standard_tokenizer_new();
+    if (lowercase) {
+        ts = lowercase_filter_new(ts);
+    }
+    ts = hyphen_filter_new(stop_filter_new_with_words_len(ts, words, len));
+    return analyzer_new(ts, NULL, NULL);
+}
+
+Analyzer *legacy_standard_analyzer_new_with_words(const char **words,
+                                                  bool lowercase)
+{
+    TokenStream *ts = legacy_standard_tokenizer_new();
+    if (lowercase) {
+        ts = lowercase_filter_new(ts);
+    }
+    ts = hyphen_filter_new(stop_filter_new_with_words(ts, words));
+    return analyzer_new(ts, NULL, NULL);
+}
+
+Analyzer *mb_legacy_standard_analyzer_new_with_words_len(const char **words,
+                                                         int len, bool lowercase)
+{
+    TokenStream *ts = mb_legacy_standard_tokenizer_new();
+    if (lowercase) {
+        ts = mb_lowercase_filter_new(ts);
+    }
+    ts = hyphen_filter_new(stop_filter_new_with_words_len(ts, words, len));
+    return analyzer_new(ts, NULL, NULL);
+}
+
+Analyzer *mb_legacy_standard_analyzer_new_with_words(const char **words,
+                                                     bool lowercase)
+{
+    TokenStream *ts = mb_legacy_standard_tokenizer_new();
+    if (lowercase) {
+        ts = mb_lowercase_filter_new(ts);
+    }
+    ts = hyphen_filter_new(stop_filter_new_with_words(ts, words));
+    return analyzer_new(ts, NULL, NULL);
+}
+
+Analyzer *legacy_standard_analyzer_new(bool lowercase)
+{
+    return legacy_standard_analyzer_new_with_words(FULL_ENGLISH_STOP_WORDS,
+                                                   lowercase);
+}
+
+Analyzer *mb_legacy_standard_analyzer_new(bool lowercase)
+{
+    return mb_legacy_standard_analyzer_new_with_words(FULL_ENGLISH_STOP_WORDS,
+                                                      lowercase);
+}
+
 /****************************************************************************
  *
  * PerFieldAnalyzer
  *
  ****************************************************************************/
 
-#define PFA(analyzer) ((PerFieldAnalyzer *)(analyzer))
-void pfa_destroy_i(Analyzer *self)
+static void pfa_destroy_i(Analyzer *self)
 {
     h_destroy(PFA(self)->dict);
 
@@ -1501,24 +1652,27 @@ void pfa_destroy_i(Analyzer *self)
     free(self);
 }
 
-TokenStream *pfa_get_ts(Analyzer *self, char *field, char *text)
+static TokenStream *pfa_get_ts(Analyzer *self,
+                               Symbol field, char *text)
 {
-    Analyzer *a = h_get(PFA(self)->dict, field);
+    Analyzer *a = (Analyzer *)h_get(PFA(self)->dict, field);
     if (a == NULL) {
         a = PFA(self)->default_a;
     }
     return a_get_ts(a, field, text);
 }
 
-void pfa_sub_a_destroy_i(void *p)
+static void pfa_sub_a_destroy_i(void *p)
 {
     Analyzer *a = (Analyzer *) p;
     a_deref(a);
 }
 
-void pfa_add_field(Analyzer *self, char *field, Analyzer *analyzer)
+void pfa_add_field(Analyzer *self, 
+                   Symbol field,
+                   Analyzer *analyzer)
 {
-    h_set(PFA(self)->dict, estrdup(field), analyzer);
+    h_set(PFA(self)->dict, field, analyzer);
 }
 
 Analyzer *per_field_analyzer_new(Analyzer *default_a)
@@ -1526,22 +1680,23 @@ Analyzer *per_field_analyzer_new(Analyzer *default_a)
     Analyzer *a = (Analyzer *)ecalloc(sizeof(PerFieldAnalyzer));
 
     PFA(a)->default_a = default_a;
-    PFA(a)->dict = h_new_str(&free, &pfa_sub_a_destroy_i);
+    PFA(a)->dict = h_new_ptr(&pfa_sub_a_destroy_i);
 
     a->destroy_i = &pfa_destroy_i;
     a->get_ts    = pfa_get_ts;
     a->ref_cnt   = 1;
-    
+
     return a;
 }
 
-#ifdef ALONE
+#ifdef TOKENIZE
 int main(int argc, char **argv)
 {
     char buf[10000];
     Analyzer *a = standard_analyzer_new(true);
     TokenStream *ts;
     Token *tk;
+    (void)argc; (void)argv;
     while (fgets(buf, 9999, stdin) != NULL) {
         ts = a_get_ts(a, "hello", buf);
         while ((tk = ts->next(ts)) != NULL) {