ferret 0.11.6 → 0.11.8.4

data/ext/header.h

@@ -14,7 +14,7 @@
 
 struct among
 {   int s_size;     /* number of chars in string */
-    symbol * s;       /* search string */
+    const symbol * s;       /* search string */
     int substring_i;/* index to longest matching substring */
     int result;     /* result of the lookup */
     int (* function)(struct SN_env *);
@@ -25,31 +25,31 @@ extern void lose_s(symbol * p);
 
 extern int skip_utf8(const symbol * p, int c, int lb, int l, int n);
 
-extern int in_grouping_U(struct SN_env * z, unsigned char * s, int min, int max);
-extern int in_grouping_b_U(struct SN_env * z, unsigned char * s, int min, int max);
-extern int out_grouping_U(struct SN_env * z, unsigned char * s, int min, int max);
-extern int out_grouping_b_U(struct SN_env * z, unsigned char * s, int min, int max);
+extern int in_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+extern int in_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+extern int out_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+extern int out_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
 
-extern int in_grouping(struct SN_env * z, unsigned char * s, int min, int max);
-extern int in_grouping_b(struct SN_env * z, unsigned char * s, int min, int max);
-extern int out_grouping(struct SN_env * z, unsigned char * s, int min, int max);
-extern int out_grouping_b(struct SN_env * z, unsigned char * s, int min, int max);
+extern int in_grouping(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+extern int in_grouping_b(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+extern int out_grouping(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+extern int out_grouping_b(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
 
-extern int eq_s(struct SN_env * z, int s_size, symbol * s);
-extern int eq_s_b(struct SN_env * z, int s_size, symbol * s);
-extern int eq_v(struct SN_env * z, symbol * p);
-extern int eq_v_b(struct SN_env * z, symbol * p);
+extern int eq_s(struct SN_env * z, int s_size, const symbol * s);
+extern int eq_s_b(struct SN_env * z, int s_size, const symbol * s);
+extern int eq_v(struct SN_env * z, const symbol * p);
+extern int eq_v_b(struct SN_env * z, const symbol * p);
 
-extern int find_among(struct SN_env * z, struct among * v, int v_size);
-extern int find_among_b(struct SN_env * z, struct among * v, int v_size);
+extern int find_among(struct SN_env * z, const struct among * v, int v_size);
+extern int find_among_b(struct SN_env * z, const struct among * v, int v_size);
 
 extern int replace_s(struct SN_env * z, int c_bra, int c_ket, int s_size, const symbol * s, int * adjustment);
-extern int slice_from_s(struct SN_env * z, int s_size, symbol * s);
-extern int slice_from_v(struct SN_env * z, symbol * p);
+extern int slice_from_s(struct SN_env * z, int s_size, const symbol * s);
+extern int slice_from_v(struct SN_env * z, const symbol * p);
 extern int slice_del(struct SN_env * z);
 
-extern int insert_s(struct SN_env * z, int bra, int ket, int s_size, symbol * s);
-extern int insert_v(struct SN_env * z, int bra, int ket, symbol * p);
+extern int insert_s(struct SN_env * z, int bra, int ket, int s_size, const symbol * s);
+extern int insert_v(struct SN_env * z, int bra, int ket, const symbol * p);
 
 extern symbol * slice_to(struct SN_env * z, symbol * p);
 extern symbol * assign_to(struct SN_env * z, symbol * p);

data/ext/helper.c

@@ -1,4 +1,5 @@
 #include "helper.h"
+#include "internal.h"
 
 int hlp_string_diff(register const char *const s1,
                            register const char *const s2)
@@ -10,17 +11,17 @@ int hlp_string_diff(register const char *const s1,
     return i;
 }
 
-f_i32 float2int(float f)
+i32 float2int(float f)
 {
-    union { f_i32 i; float f; } tmp;
+    union { i32 i; float f; } tmp;
     tmp.f = f;
     return tmp.i;
 }
 
-float int2float(f_i32 i32)
+float int2float(i32 v)
 {
-    union { f_i32 i; float f; } tmp;
-    tmp.i = i32;
+    union { i32 i; float f; } tmp;
+    tmp.i = v;
     return tmp.f;
 }
 
@@ -30,8 +31,8 @@ float byte2float(unsigned char b)
         return 0.0;
     }
     else {
-        f_u32 mantissa = b & 0x07;
-        f_u32 exponent = (b >> 3) & 0x1f;
+        u32 mantissa = b & 0x07;
+        u32 exponent = (b >> 3) & 0x1f;
 
         return int2float((mantissa << 21) | ((exponent + 48) << 24));
     }
@@ -44,9 +45,9 @@ unsigned char float2byte(float f)
     }
     else {
         /* correctly order the bytes for encoding */
-        f_u32 i32 = float2int(f);
-        int mantissa = (i32 & 0xEf0000) >> 21;
-        int exponent = ((i32 >> 24) - 48);
+        u32 i = float2int(f);
+        int mantissa = (i & 0xEf0000) >> 21;
+        int exponent = ((i >> 24) - 48);
 
         if (exponent > 0x1f) {
             exponent = 0x1f;   /* 0x1f = 31 = 0b00011111 */

data/ext/helper.h

@@ -1,13 +1,21 @@
 #ifndef FRT_HELPER_H
 #define FRT_HELPER_H
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 #include "config.h"
 
-extern int hlp_string_diff(register const char *const s1,
-                                  register const char *const s2);
-extern f_i32 float2int(float f);
-extern float int2float(f_i32 i32);
-extern float byte2float(unsigned char b);
-extern unsigned char float2byte(float f);
+extern int frt_hlp_string_diff(register const char *const s1,
+                               register const char *const s2);
+extern frt_i32 frt_float2int(float f);
+extern float frt_int2float(frt_i32 i32);
+extern float frt_byte2float(unsigned char b);
+extern unsigned char frt_float2byte(float f);
+
+#ifdef __cplusplus
+} // extern "C"
+#endif
 
 #endif

data/ext/index.c

@@ -1,11 +1,17 @@
 #include "index.h"
+#include "symbol.h"
 #include "similarity.h"
 #include "helper.h"
 #include "array.h"
-#include "priorityqueue.h"
 #include <string.h>
 #include <limits.h>
 #include <ctype.h>
+#ifdef USE_ZLIB
+# include <zlib.h>
+#else
+# include "bzlib.h"
+#endif
+#include "internal.h"
 
 #define GET_LOCK(lock, name, store, err_msg) do {\
     lock = store->open_lock(store, name);\
@@ -26,7 +32,7 @@ const Config default_config = {
     SKIP_INTERVAL,  /* skip interval */
     10,             /* default merge factor */
     10000,          /* max_buffered_docs */
-    INT_MAX,        /* max_merged_docs */
+    INT_MAX,        /* max_merge_docs */
     10000,          /* maximum field length (number of terms) */
     true            /* use compound file by default */
 };
@@ -37,21 +43,22 @@ static char *ste_next(TermEnum *te);
 #define FORMAT 0
 #define SEGMENTS_GEN_FILE_NAME "segments"
 #define MAX_EXT_LEN 10
+#define ZIP_BUFFER_SIZE 16348
+#define ZIP_LEVEL 9
 
 /* *** Must be three characters *** */
-const char *INDEX_EXTENSIONS[] = {
+static const char *INDEX_EXTENSIONS[] = {
     "frq", "prx", "fdx", "fdt", "tfx", "tix", "tis", "del", "gen", "cfs"
 };
 
 /* *** Must be three characters *** */
-const char *COMPOUND_EXTENSIONS[] = {
+static const char *COMPOUND_EXTENSIONS[] = {
     "frq", "prx", "fdx", "fdt", "tfx", "tix", "tis"
 };
 
-
 static const char BASE36_DIGITMAP[] = "0123456789abcdefghijklmnopqrstuvwxyz";
 
-static char *u64_to_str36(char *buf, int buf_size, f_u64 u)
+static char *u64_to_str36(char *buf, int buf_size, u64 u)
 {
     int i = buf_size - 1;
     buf[i] = '\0';
@@ -69,9 +76,9 @@ static char *u64_to_str36(char *buf, int buf_size, f_u64 u)
     return buf + i;
 }
 
-static f_u64 str36_to_u64(char *p)
+static u64 str36_to_u64(char *p)
 {
-    f_u64 u = 0;
+    u64 u = 0;
     while (true) {
         if ('0' <= *p && '9' >= *p) {
             u = u * 36 + *p - '0';
@@ -98,14 +105,14 @@ static f_u64 str36_to_u64(char *p)
  * @param ext extension of the filename (including .)
  * @param gen generation
  */
-char *fn_for_generation(char *buf, char *base, char *ext, f_i64 gen)
+char *fn_for_generation(char *buf, char *base, char *ext, i64 gen)
 {
     if (-1 == gen) {
         return NULL;
     }
     else {
         char b[SEGMENT_NAME_MAX_LENGTH];
-        char *u = u64_to_str36(b, SEGMENT_NAME_MAX_LENGTH, (f_u64)gen);
+        char *u = u64_to_str36(b, SEGMENT_NAME_MAX_LENGTH, (u64)gen);
         if (ext == NULL) {
             sprintf(buf, "%s_%s", base, u);
         }
@@ -116,7 +123,7 @@ char *fn_for_generation(char *buf, char *base, char *ext, f_i64 gen)
     }
 }
 
-char *segfn_for_generation(char *buf, f_u64 generation)
+static char *segfn_for_generation(char *buf, u64 generation)
 {
     char b[SEGMENT_NAME_MAX_LENGTH];
     char *u = u64_to_str36(b, SEGMENT_NAME_MAX_LENGTH, generation);
@@ -137,9 +144,9 @@ char *segfn_for_generation(char *buf, f_u64 generation)
  * @param field_num field number
  */
 static char *fn_for_gen_field(char *buf,
-                              char *base,
-                              char *ext,
-                              f_i64 gen,
+                              const char *base,
+                              const char *ext,
+                              i64 gen,
                               int field_num)
 {
     if (-1 == gen) {
@@ -149,7 +156,7 @@ static char *fn_for_gen_field(char *buf,
         char b[SEGMENT_NAME_MAX_LENGTH];
         sprintf(buf, "%s_%s.%s%d",
                 base,
-                u64_to_str36(b, SEGMENT_NAME_MAX_LENGTH, (f_u64)gen),
+                u64_to_str36(b, SEGMENT_NAME_MAX_LENGTH, (u64)gen),
                 ext,
                 field_num);
         return buf;
@@ -172,7 +179,7 @@ static int co_eq(const void *key1, const void *key2)
     return (key1 == key2);
 }
 
-void co_destroy(CacheObject *self)
+static void co_destroy(CacheObject *self)
 {
     h_rem(self->ref_tab1, self->ref2, false);
     h_rem(self->ref_tab2, self->ref1, false);
@@ -180,7 +187,7 @@ void co_destroy(CacheObject *self)
     free(self);
 }
 
-CacheObject *co_create(HashTable *ref_tab1, HashTable *ref_tab2,
+CacheObject *co_create(Hash *ref_tab1, Hash *ref_tab2,
                        void *ref1, void *ref2, free_ft destroy, void *obj)
 {
     CacheObject *self = ALLOC(CacheObject);
@@ -195,7 +202,7 @@ CacheObject *co_create(HashTable *ref_tab1, HashTable *ref_tab2,
     return self;
 }
 
-HashTable *co_hash_create()
+Hash *co_hash_create()
 {
     return h_new(&co_hash, &co_eq, (free_ft)NULL, (free_ft)&co_destroy);
 }
@@ -206,7 +213,7 @@ HashTable *co_hash_create()
  *
  ****************************************************************************/
 
-INLINE void fi_set_store(FieldInfo *fi, int store)
+static INLINE void fi_set_store(FieldInfo *fi, int store)
 {
     switch (store) {
         case STORE_NO:
@@ -220,7 +227,7 @@ INLINE void fi_set_store(FieldInfo *fi, int store)
     }
 }
 
-INLINE void fi_set_index(FieldInfo *fi, int index)
+static INLINE void fi_set_index(FieldInfo *fi, int index)
 {
     switch (index) {
         case INDEX_NO:
@@ -241,7 +248,7 @@ INLINE void fi_set_index(FieldInfo *fi, int index)
     }
 }
 
-INLINE void fi_set_term_vector(FieldInfo *fi, int term_vector)
+static INLINE void fi_set_term_vector(FieldInfo *fi, int term_vector)
 {
     switch (term_vector) {
         case TERM_VECTOR_NO:
@@ -271,14 +278,15 @@ static void fi_check_params(int store, int index, int term_vector)
     }
 }
 
-FieldInfo *fi_new(const char *name,
-                  enum StoreValues store,
-                  enum IndexValues index,
-                  enum TermVectorValues term_vector)
+FieldInfo *fi_new(Symbol name,
+                  StoreValue store,
+                  IndexValue index,
+                  TermVectorValue term_vector)
 {
     FieldInfo *fi = ALLOC(FieldInfo);
+    assert(NULL != name);
     fi_check_params(store, index, term_vector);
-    fi->name = estrdup(name);
+    fi->name = name;
     fi->boost = 1.0;
     fi->bits = 0;
     fi_set_store(fi, store);
@@ -291,28 +299,28 @@ FieldInfo *fi_new(const char *name,
 void fi_deref(FieldInfo *fi)
 {
     if (0 == --(fi->ref_cnt)) {
-        free(fi->name);
         free(fi);
     }
 }
 
 char *fi_to_s(FieldInfo *fi)
 {
-    char *str = ALLOC_N(char, strlen(fi->name) + 200);
+    char *str = ALLOC_N(char, strlen((char *)fi->name) + 200);
     char *s = str;
-    sprintf(str, "[\"%s\":(%s%s%s%s%s%s%s%s", fi->name,
-            fi_is_stored(fi) ? "is_stored, " : "",
-            fi_is_compressed(fi) ? "is_compressed, " : "",
-            fi_is_indexed(fi) ? "is_indexed, " : "",
-            fi_is_tokenized(fi) ? "is_tokenized, " : "",
-            fi_omit_norms(fi) ? "omit_norms, " : "",
-            fi_store_term_vector(fi) ? "store_term_vector, " : "",
-            fi_store_positions(fi) ? "store_positions, " : "",
-            fi_store_offsets(fi) ? "store_offsets, " : "");
-    s += (int)strlen(str) - 2;
+    s += sprintf(str, "[\"%s\":(%s%s%s%s%s%s%s%s", (char *)fi->name,
+                 fi_is_stored(fi) ? "is_stored, " : "",
+                 fi_is_compressed(fi) ? "is_compressed, " : "",
+                 fi_is_indexed(fi) ? "is_indexed, " : "",
+                 fi_is_tokenized(fi) ? "is_tokenized, " : "",
+                 fi_omit_norms(fi) ? "omit_norms, " : "",
+                 fi_store_term_vector(fi) ? "store_term_vector, " : "",
+                 fi_store_positions(fi) ? "store_positions, " : "",
+                 fi_store_offsets(fi) ? "store_offsets, " : "");
+    s -= 2;
     if (*s != ',') {
         s += 2;
     }
+
     sprintf(s, ")]");
     return str;
 }
@@ -323,11 +331,12 @@ char *fi_to_s(FieldInfo *fi)
  *
  ****************************************************************************/
 
-FieldInfos *fis_new(int store, int index, int term_vector)
+FieldInfos *fis_new(StoreValue store, IndexValue index,
+                    TermVectorValue term_vector)
 {
     FieldInfos *fis = ALLOC(FieldInfos);
     fi_check_params(store, index, term_vector);
-    fis->field_dict = h_new_str((free_ft)NULL, (free_ft)&fi_deref);
+    fis->field_dict = h_new_ptr((free_ft)&fi_deref);
     fis->size = 0;
     fis->capa = FIELD_INFOS_INIT_CAPA;
     fis->fields = ALLOC_N(FieldInfo *, fis->capa);
@@ -346,7 +355,7 @@ FieldInfo *fis_add_field(FieldInfos *fis, FieldInfo *fi)
     }
     if (!h_set_safe(fis->field_dict, fi->name, fi)) {
         RAISE(ARG_ERROR,
-              "Field :%s already exists", fi->name);
+              "Field :%s already exists", (char *)fi->name);
     }
     fi->number = fis->size;
     fis->fields[fis->size] = fi;
@@ -354,14 +363,14 @@ FieldInfo *fis_add_field(FieldInfos *fis, FieldInfo *fi)
     return fi;
 }
 
-FieldInfo *fis_get_field(FieldInfos *fis, const char *name)
+FieldInfo *fis_get_field(FieldInfos *fis, Symbol name)
 {
-    return h_get(fis->field_dict, name);
+    return (FieldInfo *)h_get(fis->field_dict, name);
 }
 
-int fis_get_field_num(FieldInfos *fis, const char *name)
+int fis_get_field_num(FieldInfos *fis, Symbol name)
 {
-    FieldInfo *fi = h_get(fis->field_dict, name);
+    FieldInfo *fi = (FieldInfo *)h_get(fis->field_dict, name);
     if (fi) {
         return fi->number;
     }
@@ -370,11 +379,11 @@ int fis_get_field_num(FieldInfos *fis, const char *name)
     }
 }
 
-FieldInfo *fis_get_or_add_field(FieldInfos *fis, const char *name)
+FieldInfo *fis_get_or_add_field(FieldInfos *fis, Symbol name)
 {
-    FieldInfo *fi = h_get(fis->field_dict, name);
+    FieldInfo *fi = (FieldInfo *)h_get(fis->field_dict, name);
     if (!fi) {
-        fi = fi_new(name, fis->store, fis->index, fis->term_vector);
+        fi = (FieldInfo*)fi_new(name, fis->store, fis->index, fis->term_vector);
         fis_add_field(fis, fi);
     }
     return fi;
@@ -392,27 +401,28 @@ FieldInfo *fis_by_number(FieldInfos *fis, int num)
 
 FieldInfos *fis_read(InStream *is)
 {
-    FieldInfos *volatile fis;
+    FieldInfos *volatile fis = NULL;
     TRY
         do {
-            int store_val, index_val, term_vector_val;
-            int i;
-            union { f_u32 i; float f; } tmp;
+            StoreValue store_val;
+            IndexValue index_val;
+            TermVectorValue term_vector_val;
+            volatile int i;
+            union { u32 i; float f; } tmp;
             FieldInfo *volatile fi;
 
-            store_val = is_read_vint(is);
-            index_val = is_read_vint(is);
-            term_vector_val = is_read_vint(is);
+            store_val = (StoreValue)is_read_vint(is);
+            index_val = (IndexValue)is_read_vint(is);
+            term_vector_val = (TermVectorValue)is_read_vint(is);
             fis = fis_new(store_val, index_val, term_vector_val);
             for (i = is_read_vint(is); i > 0; i--) {
                 fi = ALLOC_AND_ZERO(FieldInfo);
                 TRY
-                    fi->name = is_read_string_safe(is);
+                    fi->name = intern_and_free(is_read_string_safe(is));
                     tmp.i = is_read_u32(is);
                     fi->boost = tmp.f;
                     fi->bits = is_read_vint(is);
                 XCATCHALL
-                    free(fi->name);
                     free(fi);
                 XENDTRY
                 fis_add_field(fis, fi);
@@ -423,13 +433,13 @@ FieldInfos *fis_read(InStream *is)
         fis_deref(fis);
     XENDTRY
 
-    return fis; 
+    return fis;
 }
 
 void fis_write(FieldInfos *fis, OutStream *os)
 {
     int i;
-    union { f_u32 i; float f; } tmp;
+    union { u32 i; float f; } tmp;
     FieldInfo *fi;
     const int fis_size = fis->size;
 
@@ -439,7 +449,7 @@ void fis_write(FieldInfos *fis, OutStream *os)
     os_write_vint(os, fis->size);
     for (i = 0; i < fis_size; i++) {
         fi = fis->fields[i];
-        os_write_string(os, fi->name);
+        os_write_string(os, S(fi->name));
         tmp.f = fi->boost;
         os_write_u32(os, tmp.i);
         os_write_vint(os, fi->bits);
@@ -497,27 +507,25 @@ char *fis_to_s(FieldInfos *fis)
     FieldInfo *fi;
     const int fis_size = fis->size;
 
-    sprintf(buf, 
-            "default:\n"
-            "  store: %s\n"
-            "  index: %s\n"
-            "  term_vector: %s\n"
-            "fields:\n",
-            store_str[fis->store], index_str[fis->index],
-            term_vector_str[fis->term_vector]);
-    pos = (int)strlen(buf);
+    pos = sprintf(buf,
+                  "default:\n"
+                  "  store: %s\n"
+                  "  index: %s\n"
+                  "  term_vector: %s\n"
+                  "fields:\n",
+                  store_str[fis->store],
+                  index_str[fis->index],
+                  term_vector_str[fis->term_vector]);
     for (i = 0; i < fis_size; i++) {
         fi = fis->fields[i];
-        sprintf(buf + pos, 
-                "  %s:\n"
-                "    boost: %f\n"
-                "    store: %s\n"
-                "    index: %s\n"
-                "    term_vector: %s\n", 
-                fi->name, fi->boost, fi_store_str(fi),
-                fi_index_str(fi), fi_term_vector_str(fi));
-
-        pos += strlen(buf + pos);
+        pos += sprintf(buf + pos,
+                       "  %s:\n"
+                       "    boost: %f\n"
+                       "    store: %s\n"
+                       "    index: %s\n"
+                       "    term_vector: %s\n",
+                       (char *)fi->name, fi->boost, fi_store_str(fi),
+                       fi_index_str(fi), fi_term_vector_str(fi));
     }
 
     return buf;
@@ -565,7 +573,7 @@ SegmentInfo *si_new(char *name, int doc_cnt, Store *store)
     return si;
 }
 
-SegmentInfo *si_read(Store *store, InStream *is)
+static SegmentInfo *si_read(Store *store, InStream *is)
 {
     SegmentInfo *volatile si = ALLOC_AND_ZERO(SegmentInfo);
     TRY
@@ -590,7 +598,7 @@ SegmentInfo *si_read(Store *store, InStream *is)
     return si;
 }
 
-void si_write(SegmentInfo *si, OutStream *os)
+static void si_write(SegmentInfo *si, OutStream *os)
 {
     os_write_string(os, si->name);
     os_write_vint(os, si->doc_cnt);
@@ -619,7 +627,9 @@ bool si_has_deletions(SegmentInfo *si)
     return si->del_gen >= 0;
 }
 
-char *si_del_file_name(SegmentInfo *si, char *buf)
+/*
+FIXME: not used
+static char *si_del_file_name(SegmentInfo *si, char *buf)
 {
     if (si->del_gen < 0) {
         return NULL;
@@ -628,6 +638,7 @@ char *si_del_file_name(SegmentInfo *si, char *buf)
         return fn_for_generation(buf, si->name, ".del", si->del_gen);
     }
 }
+*/
 
 bool si_has_separate_norms(SegmentInfo *si)
 {
@@ -653,7 +664,7 @@ void si_advance_norm_gen(SegmentInfo *si, int field_num)
     si->norm_gens[field_num]++;
 }
 
-char *si_norm_file_name(SegmentInfo *si, char *buf, int field_num)
+static char *si_norm_file_name(SegmentInfo *si, char *buf, int field_num)
 {
     int norm_gen;
     if (field_num >= si->norm_gens_size
@@ -661,12 +672,12 @@ char *si_norm_file_name(SegmentInfo *si, char *buf, int field_num)
         return NULL;
     }
     else {
-        char *ext = (si->use_compound_file && norm_gen > 0) ? "s" : "f";
+        const char *ext = (si->use_compound_file && norm_gen > 0) ? "s" : "f";
         return fn_for_gen_field(buf, si->name, ext, norm_gen, field_num);
     }
 }
 
-void deleter_queue_file(Deleter *dlr, char *file_name);
+static void deleter_queue_file(Deleter *dlr, const char *file_name);
 #define DEL(file_name) deleter_queue_file(dlr, file_name)
 
 static void si_delete_files(SegmentInfo *si, FieldInfos *fis, Deleter *dlr)
@@ -708,11 +719,11 @@ static void si_delete_files(SegmentInfo *si, FieldInfos *fis, Deleter *dlr)
  ****************************************************************************/
 
 #include <time.h>
-static char *new_segment(f_i64 generation) 
+static char *new_segment(i64 generation)
 {
     char buf[SEGMENT_NAME_MAX_LENGTH];
     char *fn_p = u64_to_str36(buf, SEGMENT_NAME_MAX_LENGTH - 1,
-                              (f_u64)generation);
+                              (u64)generation);
     *(--fn_p) = '_';
     return estrdup(fn_p);
 }
@@ -722,18 +733,21 @@ static char *new_segment(f_i64 generation)
  ****************************************************************************/
 
 typedef struct FindSegmentsFile {
-    f_i64  generation;
-    f_u64  u64_return;
-    void  *p_return;
+    i64  generation;
+    union {
+      SegmentInfos *sis;
+      IndexReader  *ir;
+      u64           uint64;
+    } ret;
 } FindSegmentsFile;
 
-static void which_gen_i(char *file_name, void *arg)
+static void which_gen_i(const char *file_name, void *arg)
 {
-    f_i64 *max_generation = (f_i64 *)arg;
+    i64 *max_generation = (i64 *)arg;
     if (0 == strncmp(SEGMENTS_FILE_NAME"_", file_name,
                      sizeof(SEGMENTS_FILE_NAME))) {
         char *p = strrchr(file_name, '_') + 1;
-        f_i64 generation = (f_i64)str36_to_u64(p);
+        i64 generation = (i64)str36_to_u64(p);
         if (generation > *max_generation) *max_generation = generation;
     }
 }
@@ -776,9 +790,9 @@ void sis_put(SegmentInfos *sis, FILE *stream)
  *
  * @param store - the Store to look in
  */
-f_i64 sis_current_segment_generation(Store *store)
+i64 sis_current_segment_generation(Store *store)
 {
-    f_i64 current_generation = -1;
+    i64 current_generation = -1;
     store->each(store, &which_gen_i, &current_generation);
     return current_generation;
 }
@@ -802,22 +816,25 @@ char *sis_curr_seg_file_name(char *buf, Store *store)
  * @param store - the Store to look in
  * @return segments_N where N is the +next+ generation
  */
-char *sis_next_seg_file_name(char *buf, Store *store)
+/*
+FIXME: not used
+static char *sis_next_seg_file_name(char *buf, Store *store)
 {
     return segfn_for_generation(buf, sis_current_segment_generation(store) + 1);
 }
+*/
 
 #define GEN_FILE_RETRY_COUNT 10
 #define GEN_LOOK_AHEAD_COUNT 10
-void sis_find_segments_file(Store *store, FindSegmentsFile *fsf,
+static void sis_find_segments_file(Store *store, FindSegmentsFile *fsf,
                             void (*run)(Store *store, FindSegmentsFile *fsf))
 {
-    int i;
-    int gen_look_ahead_count = 0;
-    bool retry = false;
-    int method = 0;
-    f_i64 last_gen = -1;
-    f_i64 gen = 0;
+    volatile int i;
+    volatile int gen_look_ahead_count = 0;
+    volatile bool retry = false;
+    volatile int method = 0;
+    volatile i64 last_gen = -1;
+    volatile i64 gen = 0;
 
     /* Loop until we succeed in calling doBody() without hitting an
      * IOException.  An IOException most likely means a commit was in process
@@ -855,7 +872,7 @@ void sis_find_segments_file(Store *store, FindSegmentsFile *fsf,
                 XENDTRY
 
                 if (NULL != gen_is) {
-                    f_i64 gen0 = -1, gen1 = -1;
+                    i64 gen0 = -1, gen1 = -1;
 
                     TRY
                         gen0 = is_read_u64(gen_is);
@@ -897,8 +914,14 @@ void sis_find_segments_file(Store *store, FindSegmentsFile *fsf,
                 /* OK, we've tried the same segments_N file twice in a row, so
                  * this must be a real error.  We throw the original exception
                  * we got. */
+                char *listing, listing_buffer[1024];
+                listing = store_to_s(store);
+                strncpy(listing_buffer, listing, 1023);
+                listing_buffer[1023] = '\0';
+                free(listing);
                 RAISE(IO_ERROR,
-                      "Error reading the segment infos. Store listing was\n");
+                      "Error reading the segment infos. Store:\n %s\n",
+                      listing_buffer);
             }
             else {
                 micro_sleep(50000);
@@ -985,7 +1008,7 @@ SegmentInfos *sis_new(FieldInfos *fis)
     REF(fis);
     sis->fis = fis;
     sis->format = FORMAT;
-    sis->version = (f_u64)time(NULL);
+    sis->version = (u64)time(NULL);
     sis->size = 0;
     sis->counter = 0;
     sis->generation = -1;
@@ -1053,7 +1076,7 @@ void sis_clear(SegmentInfos *sis)
     sis->size = 0;
 }
 
-void sis_read_i(Store *store, FindSegmentsFile *fsf)
+static void sis_read_i(Store *store, FindSegmentsFile *fsf)
 {
     int seg_cnt;
     int i;
@@ -1062,7 +1085,7 @@ void sis_read_i(Store *store, FindSegmentsFile *fsf)
     InStream *volatile is = NULL;
     SegmentInfos *volatile sis = ALLOC_AND_ZERO(SegmentInfos);
     segfn_for_generation(seg_file_name, fsf->generation);
-    fsf->p_return = NULL;
+    fsf->ret.sis = NULL;
     TRY
         is = store->open_input(store, seg_file_name);
         sis->store = store;
@@ -1090,20 +1113,20 @@ void sis_read_i(Store *store, FindSegmentsFile *fsf)
             sis_destroy(sis);
         }
     XENDTRY
-    fsf->p_return = sis;
+    fsf->ret.sis = sis;
 }
 
 SegmentInfos *sis_read(Store *store)
 {
     FindSegmentsFile fsf;
     sis_find_segments_file(store, &fsf, &sis_read_i);
-    return fsf.p_return;
+    return fsf.ret.sis;
 }
 
 void sis_write(SegmentInfos *sis, Store *store, Deleter *deleter)
 {
     int i;
-    OutStream *os = NULL;
+    OutStream *volatile os = NULL;
     const int sis_size = sis->size;
     char buf[SEGMENT_NAME_MAX_LENGTH];
     sis->generation++;
@@ -1114,7 +1137,7 @@ void sis_write(SegmentInfos *sis, Store *store, Deleter *deleter)
         os_write_u32(os, FORMAT);
         os_write_u64(os, ++(sis->version)); /* every write changes the index */
         os_write_u64(os, sis->counter);
-        os_write_vint(os, sis->size); 
+        os_write_vint(os, sis->size);
         for (i = 0; i < sis_size; i++) {
             si_write(sis->segs[i], os);
         }
@@ -1140,14 +1163,14 @@ void sis_write(SegmentInfos *sis, Store *store, Deleter *deleter)
     }
 }
 
-void sis_read_ver_i(Store *store, FindSegmentsFile *fsf)
+static void sis_read_ver_i(Store *store, FindSegmentsFile *fsf)
 {
     InStream *is;
-    f_u32 format = 0;
-    f_u64 version = 0;
+    u32 format = 0;
+    u64 version = 0;
     char seg_file_name[SEGMENT_NAME_MAX_LENGTH];
 
-    segfn_for_generation(seg_file_name, (f_u64)fsf->generation);
+    segfn_for_generation(seg_file_name, (u64)fsf->generation);
     is = store->open_input(store, seg_file_name);
 
     TRY
@@ -1157,14 +1180,14 @@ void sis_read_ver_i(Store *store, FindSegmentsFile *fsf)
         is_close(is);
     XENDTRY
 
-    fsf->u64_return = version;
+    fsf->ret.uint64 = version;
 }
 
-f_u64 sis_read_current_version(Store *store)
+u64 sis_read_current_version(Store *store)
 {
     FindSegmentsFile fsf;
     sis_find_segments_file(store, &fsf, &sis_read_ver_i);
-    return fsf.u64_return;
+    return fsf.ret.uint64;
 }
 
 /****************************************************************************
@@ -1173,12 +1196,14 @@ f_u64 sis_read_current_version(Store *store)
  *
  ****************************************************************************/
 
-static LazyDocField *lazy_df_new(const char *name, const int size)
+static LazyDocField *lazy_df_new(Symbol name, const int size,
+                                 bool is_compressed)
 {
     LazyDocField *self = ALLOC(LazyDocField);
-    self->name = estrdup(name);
+    self->name = name;
     self->size = size;
     self->data = ALLOC_AND_ZERO_N(LazyDocFieldData, size);
+    self->is_compressed = is_compressed;
     return self;
 }
 
@@ -1190,11 +1215,171 @@ static void lazy_df_destroy(LazyDocField *self)
             free(self->data[i].text);
          }
     }
-    free(self->name);
     free(self->data);
     free(self);
 }
 
+#ifdef USE_ZLIB
+/* good zlib example at http://www.zlib.net/zlib_how.html */
+
+/* report a zlib or i/o error */
+static void zraise(int ret)
+{
+    switch (ret) {
+    case Z_ERRNO:
+        if (ferror(stdin))
+            RAISE(IO_ERROR, "zlib: error reading stdin");
+        if (ferror(stdout))
+            RAISE(IO_ERROR, "zlib: error writing stdout");
+        break;
+    case Z_STREAM_ERROR:
+        RAISE(IO_ERROR, "zlib: invalid compression level");
+        break;
+    case Z_DATA_ERROR:
+        RAISE(IO_ERROR, "zlib: invalid or incomplete deflate data");
+        break;
+    case Z_MEM_ERROR:
+        RAISE(IO_ERROR, "zlib: out of memory");
+        break;
+    case Z_VERSION_ERROR:
+        RAISE(IO_ERROR, "zlib: version mismatch!");
+        break;
+    default:
+        RAISE(EXCEPTION, "zlib: unknown error");
+    }
+}
+
+static char *is_read_zipped_bytes(InStream *is, int zip_len, int *len)
+{
+    int buf_out_idx = 0, ret, read_len;
+    uchar *buf_out = NULL;
+    uchar buf_in[ZIP_BUFFER_SIZE];
+    z_stream zstrm;
+    zstrm.zalloc  = Z_NULL;
+    zstrm.zfree   = Z_NULL;
+    zstrm.opaque  = Z_NULL;
+    zstrm.next_in = Z_NULL;
+    zstrm.avail_in = 0;
+    if ((ret = inflateInit(&zstrm)) != Z_OK) zraise(ret);
+
+    do {
+        read_len = zip_len > ZIP_BUFFER_SIZE ? ZIP_BUFFER_SIZE : zip_len;
+        is_read_bytes(is, buf_in, zip_len);
+        zip_len -= read_len;
+        zstrm.avail_in = read_len;
+        zstrm.next_in = buf_in;
+        zstrm.avail_out = ZIP_BUFFER_SIZE;
+
+        do {
+            REALLOC_N(buf_out, uchar, buf_out_idx + ZIP_BUFFER_SIZE);
+            zstrm.next_out = buf_out + buf_out_idx;
+            ret = inflate(&zstrm, Z_NO_FLUSH);
+            assert(ret != Z_STREAM_ERROR);  /* state not clobbered */
+            switch(ret) {
+            case Z_NEED_DICT:
+                ret = Z_DATA_ERROR;     /* and fall through */
+            case Z_DATA_ERROR:
+            case Z_MEM_ERROR:
+                (void)inflateEnd(&zstrm);
+                zraise(ret);
+            }
+            buf_out_idx += ZIP_BUFFER_SIZE - zstrm.avail_out;
+        } while (zstrm.avail_out == 0);
+    } while (ret != Z_STREAM_END && zip_len != 0);
+
+    /* clean up */
+    (void)inflateEnd(&zstrm);
+
+    buf_out[buf_out_idx] = '\0';
+    REALLOC_N(buf_out, uchar, buf_out_idx + 1);
+    *len = buf_out_idx;
+    return (char *)buf_out;
+}
+#else /* use bzlib */
+static void zraise(int ret)
+{
+    switch (ret) {
+    case BZ_IO_ERROR:
+        if (ferror(stdin))
+            RAISE(IO_ERROR, "bzlib: error reading stdin");
+        if (ferror(stdout))
+            RAISE(IO_ERROR, "bzlib: error writing stdout");
+        break;
+    case BZ_CONFIG_ERROR:
+        RAISE(IO_ERROR, "bzlib: system configuration error");
+        break;
+    case BZ_SEQUENCE_ERROR: /* shouldn't occur if code is correct */
+        RAISE(IO_ERROR, "bzlib: !!BUG!! sequence error");
+        break;
+    case BZ_PARAM_ERROR:    /* shouldn't occur if code is correct */
+        RAISE(IO_ERROR, "bzlib: !!BUG!! parameter error");
+        break;
+    case BZ_MEM_ERROR:
+        RAISE(IO_ERROR, "bzlib: memory error");
+        break;
+    case BZ_DATA_ERROR:
+        RAISE(IO_ERROR, "bzlib: data integrity check error");
+        break;
+    case BZ_DATA_ERROR_MAGIC:
+        RAISE(IO_ERROR, "bzlib: data integrity check - non-matching magic");
+        break;
+    case BZ_UNEXPECTED_EOF:
+        RAISE(IO_ERROR, "bzlib: unexpected end-of-file");
+        break;
+    case BZ_OUTBUFF_FULL:
+        RAISE(IO_ERROR, "bzlib: output buffer full");
+        break;
+    default:
+        RAISE(EXCEPTION, "bzlib: unknown error");
+    }
+}
+
+static char *is_read_zipped_bytes(InStream *is, int zip_len, int *len)
+{
+    int buf_out_idx = 0, ret, read_len;
+    char *buf_out = NULL;
+    char buf_in[ZIP_BUFFER_SIZE];
+    bz_stream zstrm;
+    zstrm.bzalloc = NULL;
+    zstrm.bzfree  = NULL;
+    zstrm.opaque  = NULL;
+    zstrm.next_in = NULL;
+    zstrm.avail_in = 0;
+    if ((ret = BZ2_bzDecompressInit(&zstrm, 0, 0)) != BZ_OK) zraise(ret);
+
+    do {
+        read_len = zip_len > ZIP_BUFFER_SIZE ? ZIP_BUFFER_SIZE : zip_len;
+        is_read_bytes(is, (uchar *)buf_in, zip_len);
+        zip_len -= read_len;
+        zstrm.avail_in = read_len;
+        zstrm.next_in = buf_in;
+        zstrm.avail_out = ZIP_BUFFER_SIZE;
+
+        do {
+            REALLOC_N(buf_out, char, buf_out_idx + ZIP_BUFFER_SIZE);
+            zstrm.next_out = buf_out + buf_out_idx;
+            ret = BZ2_bzDecompress(&zstrm);
+            assert(ret != BZ_SEQUENCE_ERROR);  /* state not clobbered */
+            if (ret != BZ_OK && ret != BZ_STREAM_END) {
+                (void)BZ2_bzDecompressEnd(&zstrm);
+                zraise(ret);
+            }
+            buf_out_idx += ZIP_BUFFER_SIZE - zstrm.avail_out;
+        } while (zstrm.avail_out == 0);
+    } while (ret != BZ_STREAM_END && zip_len != 0);
+
+    /* clean up */
+    (void)BZ2_bzDecompressEnd(&zstrm);
+
+    buf_out[buf_out_idx] = '\0';
+    REALLOC_N(buf_out, char, buf_out_idx + 1);
+    *len = buf_out_idx;
+    return (char *)buf_out;
+}
+
+#endif
+
+
 char *lazy_df_get_data(LazyDocField *self, int i)
 {
     char *text = NULL;
@@ -1202,10 +1387,17 @@ char *lazy_df_get_data(LazyDocField *self, int i)
         text = self->data[i].text;
         if (NULL == text) {
             const int read_len = self->data[i].length + 1;
-            self->data[i].text = text = ALLOC_N(char, read_len);
             is_seek(self->doc->fields_in, self->data[i].start);
-            is_read_bytes(self->doc->fields_in, (uchar *)text, read_len);
-            text[read_len - 1] = '\0';
+            if (self->is_compressed) {
+                text = self->data[i].text =
+                    is_read_zipped_bytes(self->doc->fields_in, read_len,
+                                         &(self->data[i].length));
+            }
+            else {
+                self->data[i].text = text = ALLOC_N(char, read_len);
+                is_read_bytes(self->doc->fields_in, (uchar *)text, read_len);
+                text[read_len - 1] = '\0';
+            }
         }
     }
 
@@ -1214,6 +1406,16 @@ char *lazy_df_get_data(LazyDocField *self, int i)
 
 void lazy_df_get_bytes(LazyDocField *self, char *buf, int start, int len)
 {
+    if (self->is_compressed == 1) {
+        int i;
+        self->len = 0;
+        for (i = self->size-1; i >= 0; i--) {
+            (void)lazy_df_get_data(self, i);
+            self->len += self->data[i].length + 1;
+        }
+        self->len--; /* each field separated by ' ' but no need to add to end */
+        self->is_compressed = 2;
+    }
     if (start < 0 || start >= self->len) {
         RAISE(IO_ERROR, "start out of range in LazyDocField#get_bytes. %d "
               "is not between 0 and %d", start, self->len);
@@ -1225,8 +1427,37 @@ void lazy_df_get_bytes(LazyDocField *self, char *buf, int start, int len)
         RAISE(IO_ERROR, "Tried to read past end of field. Field is only %d "
               "bytes long but tried to read to %d", self->len, start + len);
     }
-    is_seek(self->doc->fields_in, self->data[0].start + start);
-    is_read_bytes(self->doc->fields_in, (uchar *)buf, len);
+    if (self->is_compressed) {
+        int cur_start = 0, buf_start = 0, cur_end, i, copy_start, copy_len;
+        for (i = 0; i < self->size; i++) {
+            cur_end = cur_start + self->data[i].length;
+            if (start < cur_end) {
+                copy_start = start > cur_start ? start - cur_start : 0;
+                copy_len = cur_end - cur_start - copy_start;
+                if (copy_len >= len) {
+                    copy_len = len;
+                    len = 0;
+                }
+                else {
+                    len -= copy_len;
+                }
+                memcpy(buf + buf_start,
+                       self->data[i].text + copy_start,
+                       copy_len);
+                buf_start += copy_len;
+                if (len > 0) {
+                    buf[buf_start++] = ' ';
+                    len--;
+                }
+                if (len == 0) break;
+            }
+            cur_start = cur_end + 1;
+        }
+    }
+    else {
+        is_seek(self->doc->fields_in, self->data[0].start + start);
+        is_read_bytes(self->doc->fields_in, (uchar *)buf, len);
+    }
 }
 
 /****************************************************************************
@@ -1238,7 +1469,7 @@ void lazy_df_get_bytes(LazyDocField *self, char *buf, int start, int len)
 static LazyDoc *lazy_doc_new(int size, InStream *fdt_in)
 {
     LazyDoc *self = ALLOC(LazyDoc);
-    self->field_dict = h_new_str(NULL, (free_ft)&lazy_df_destroy);
+    self->field_dictionary = h_new_ptr((free_ft)&lazy_df_destroy);
     self->size = size;
     self->fields = ALLOC_AND_ZERO_N(LazyDocField *, size);
     self->fields_in = is_clone(fdt_in);
@@ -1247,7 +1478,7 @@ static LazyDoc *lazy_doc_new(int size, InStream *fdt_in)
 
 void lazy_doc_close(LazyDoc *self)
 {
-    h_destroy(self->field_dict);
+    h_destroy(self->field_dictionary);
     is_close(self->fields_in);
     free(self->fields);
     free(self);
@@ -1256,10 +1487,15 @@ void lazy_doc_close(LazyDoc *self)
 static void lazy_doc_add_field(LazyDoc *self, LazyDocField *lazy_df, int i)
 {
     self->fields[i] = lazy_df;
-    h_set(self->field_dict, lazy_df->name, lazy_df);
+    h_set(self->field_dictionary, lazy_df->name, lazy_df);
     lazy_df->doc = self;
 }
 
+LazyDocField *frt_lazy_doc_get(LazyDoc *self, Symbol field)
+{
+    return (LazyDocField *)h_get(self->field_dictionary, field);
+}
+
 /****************************************************************************
  *
  * FieldsReader
@@ -1296,7 +1532,7 @@ FieldsReader *fr_clone(FieldsReader *orig)
     memcpy(fr, orig, sizeof(FieldsReader));
     fr->fdx_in = is_clone(orig->fdx_in);
     fr->fdt_in = is_clone(orig->fdt_in);
-    
+
     return fr;
 }
 
@@ -1307,25 +1543,36 @@ void fr_close(FieldsReader *fr)
     free(fr);
 }
 
-static DocField *fr_df_new(char *name, int size)
+static DocField *fr_df_new(Symbol name, int size, bool is_compressed)
 {
     DocField *df = ALLOC(DocField);
-    df->name = estrdup(name);
+    df->name = name;
     df->capa = df->size = size;
     df->data = ALLOC_N(char *, df->capa);
     df->lengths = ALLOC_N(int, df->capa);
     df->destroy_data = true;
     df->boost = 1.0;
+    df->is_compressed = is_compressed;
     return df;
 }
 
+static void fr_read_zipped_fields(FieldsReader *fr, DocField *df)
+{
+    int i;
+    const int df_size = df->size;
+    InStream *fdt_in = fr->fdt_in;
+
+    for (i = 0; i < df_size; i++) {
+        const int zip_len = df->lengths[i] + 1;
+        df->data[i] = is_read_zipped_bytes(fdt_in, zip_len, &(df->lengths[i]));
+    }
+}
+
 Document *fr_get_doc(FieldsReader *fr, int doc_num)
 {
     int i, j;
-    FieldInfo *fi;
     off_t pos;
-    int stored_cnt, field_num, df_size;
-    DocField *df;
+    int stored_cnt;
     Document *doc = doc_new();
     InStream *fdx_in = fr->fdx_in;
     InStream *fdt_in = fr->fdt_in;
@@ -1336,34 +1583,42 @@ Document *fr_get_doc(FieldsReader *fr, int doc_num)
     stored_cnt = is_read_vint(fdt_in);
 
     for (i = 0; i < stored_cnt; i++) {
-        field_num = is_read_vint(fdt_in);
-        fi = fr->fis->fields[field_num];
-        df_size = is_read_vint(fdt_in);
-        df = fr_df_new(fi->name, df_size);
+        const int field_num = is_read_vint(fdt_in);
+        FieldInfo *fi = fr->fis->fields[field_num];
+        const int df_size = is_read_vint(fdt_in);
+        DocField *df = fr_df_new(fi->name, df_size, fi_is_compressed(fi));
 
         for (j = 0; j < df_size; j++) {
             df->lengths[j] = is_read_vint(fdt_in);
         }
 
-        for (j = 0; j < df_size; j++) {
-            const int read_len = df->lengths[j] + 1;
-            df->data[j] = ALLOC_N(char, read_len);
-            is_read_bytes(fdt_in, (uchar *)df->data[j], read_len);
-            df->data[j][read_len - 1] = '\0';
-        }
         doc_add_field(doc, df);
     }
+    for (i = 0; i < stored_cnt; i++) {
+        DocField *df = doc->fields[i];
+        if (df->is_compressed) {
+            fr_read_zipped_fields(fr, df);
+        }
+        else {
+            const int df_size = df->size;
+            for (j = 0; j < df_size; j++) {
+                const int read_len = df->lengths[j] + 1;
+                df->data[j] = ALLOC_N(char, read_len);
+                is_read_bytes(fdt_in, (uchar *)df->data[j], read_len);
+                df->data[j][read_len - 1] = '\0';
+            }
+        }
+    }
 
     return doc;
 }
 
 LazyDoc *fr_get_lazy_doc(FieldsReader *fr, int doc_num)
 {
+    int start = 0;
     int i, j;
-    FieldInfo *fi;
     off_t pos;
-    int stored_cnt, field_num;
-    LazyDocField *lazy_df;
+    int stored_cnt;
     LazyDoc *lazy_doc;
     InStream *fdx_in = fr->fdx_in;
     InStream *fdt_in = fr->fdt_in;
@@ -1375,43 +1630,43 @@ LazyDoc *fr_get_lazy_doc(FieldsReader *fr, int doc_num)
     lazy_doc = lazy_doc_new(stored_cnt, fdt_in);
 
     for (i = 0; i < stored_cnt; i++) {
-        off_t start = 0, end;
-        int data_cnt;
-        field_num = is_read_vint(fdt_in);
-        fi = fr->fis->fields[field_num];
-        data_cnt = is_read_vint(fdt_in);
-        lazy_df = lazy_df_new(fi->name, data_cnt);
+        FieldInfo *fi = fr->fis->fields[is_read_vint(fdt_in)];
+        const int data_cnt = is_read_vint(fdt_in);
+        LazyDocField *lazy_df = lazy_df_new(fi->name, data_cnt,
+                                            fi_is_compressed(fi));
+        const int field_start = start;
 
         /* get the starts relative positions this time around */
         for (j = 0; j < data_cnt; j++) {
             lazy_df->data[j].start = start;
             start += 1 + (lazy_df->data[j].length = is_read_vint(fdt_in));
         }
-        end = is_pos(fdt_in) + start;
-        lazy_df->len = start - 1;
+        lazy_df->len = start - field_start - 1;
 
-        /* correct the starts to their correct absolute positions */
-        start = is_pos(fdt_in);
+        lazy_doc_add_field(lazy_doc, lazy_df, i);
+    }
+    /* correct the starts to their correct absolute positions */
+    for (i = 0; i < stored_cnt; i++) {
+        LazyDocField *lazy_df = lazy_doc->fields[i];
+        const int data_cnt = lazy_df->size;
+        const int start = is_pos(fdt_in);
         for (j = 0; j < data_cnt; j++) {
             lazy_df->data[j].start += start;
         }
-
-        lazy_doc_add_field(lazy_doc, lazy_df, i);
-        is_seek(fdt_in, end);
     }
 
     return lazy_doc;
 }
 
-TermVector *fr_read_term_vector(FieldsReader *fr, int field_num)
+static TermVector *fr_read_term_vector(FieldsReader *fr, int field_num)
 {
     TermVector *tv = ALLOC_AND_ZERO(TermVector);
     InStream *fdt_in = fr->fdt_in;
     FieldInfo *fi = fr->fis->fields[field_num];
     const int num_terms = is_read_vint(fdt_in);
-    
+
     tv->field_num = field_num;
-    tv->field = estrdup(fi->name);
+    tv->field = fi->name;
 
     if (num_terms > 0) {
         int i, j, delta_start, delta_len, total_len, freq;
@@ -1431,7 +1686,8 @@ TermVector *fr_read_term_vector(FieldsReader *fr, int field_num)
             total_len = delta_start + delta_len;
             is_read_bytes(fdt_in, buffer + delta_start, delta_len);
             buffer[total_len++] = '\0';
-            term->text = memcpy(ALLOC_N(char, total_len), buffer, total_len);
+            term->text = (char *)memcpy(ALLOC_N(char, total_len),
+                                        buffer, total_len);
 
             /* read freq */
             freq = term->freq = is_read_vint(fdt_in);
@@ -1450,21 +1706,21 @@ TermVector *fr_read_term_vector(FieldsReader *fr, int field_num)
         if (store_offsets) {
             int num_positions = tv->offset_cnt = is_read_vint(fdt_in);
             Offset *offsets = tv->offsets = ALLOC_N(Offset, num_positions);
-            long long offset = 0;
+            i64 offset = 0;
             for (i = 0; i < num_positions; i++) {
                 offsets[i].start =
-                    (off_t)(offset += (long long)is_read_vll(fdt_in));
+                    (off_t)(offset += (i64)is_read_vll(fdt_in));
                 offsets[i].end =
-                    (off_t)(offset += (long long)is_read_vll(fdt_in));
+                    (off_t)(offset += (i64)is_read_vll(fdt_in));
             }
         }
     }
     return tv;
 }
 
-HashTable *fr_get_tv(FieldsReader *fr, int doc_num)
+Hash *fr_get_tv(FieldsReader *fr, int doc_num)
 {
-    HashTable *term_vectors = h_new_str((free_ft)NULL, (free_ft)&tv_destroy);
+    Hash *term_vectors = h_new_ptr((free_ft)&tv_destroy);
     int i;
     InStream *fdx_in = fr->fdx_in;
     InStream *fdt_in = fr->fdt_in;
@@ -1556,6 +1812,8 @@ FieldsWriter *fw_open(Store *store, const char *segment, FieldInfos *fis)
     strcpy(file_name + segment_len, ".fdx");
     fw->fdx_out = store->new_output(store, file_name);
 
+    fw->buffer = ram_new_buffer();
+
     fw->fis = fis;
     fw->tv_fields = ary_new_type_capa(TVField, TV_FIELD_INIT_CAPA);
 
@@ -1566,15 +1824,68 @@ void fw_close(FieldsWriter *fw)
 {
     os_close(fw->fdt_out);
     os_close(fw->fdx_out);
+    ram_destroy_buffer(fw->buffer);
     ary_free(fw->tv_fields);
     free(fw);
 }
 
-static INLINE void save_data(OutStream *fdt_out, char *data, int dlen)
+#ifdef USE_ZLIB
+static int os_write_zipped_bytes(OutStream* out_stream, uchar *data, int length)
 {
-    os_write_vint(fdt_out, dlen);
-    os_write_bytes(fdt_out, (uchar *)data, dlen);
+    int ret, buf_size, zip_len = 0;
+    uchar out_buffer[ZIP_BUFFER_SIZE];
+    z_stream zstrm;
+    zstrm.zalloc = Z_NULL;
+    zstrm.zfree  = Z_NULL;
+    zstrm.opaque = Z_NULL;
+    if ((ret = deflateInit(&zstrm, ZIP_LEVEL)) != Z_OK) zraise(ret);
+
+    zstrm.avail_in = length;
+    zstrm.next_in = data;
+    zstrm.avail_out = ZIP_BUFFER_SIZE;
+    zstrm.next_out = out_buffer;
+
+    do {
+        ret = deflate(&zstrm, Z_FINISH); /* no bad return value */
+        assert(ret != Z_STREAM_ERROR) ;  /* state not clobbered */
+        zip_len += buf_size = ZIP_BUFFER_SIZE - zstrm.avail_out;
+        os_write_bytes(out_stream, out_buffer, buf_size);
+    } while (zstrm.avail_out == 0);
+    assert(zstrm.avail_in == 0);         /* all input will be used */
+
+    /* clean up */
+    (void)deflateEnd(&zstrm);
+    return zip_len;
+}
+#else
+static int os_write_zipped_bytes(OutStream* out_stream, uchar *data, int length)
+{
+    int ret, buf_size, zip_len = 0;
+    char out_buffer[ZIP_BUFFER_SIZE];
+    bz_stream zstrm;
+    zstrm.bzalloc = NULL;
+    zstrm.bzfree  = NULL;
+    zstrm.opaque = NULL;
+    if ((ret = BZ2_bzCompressInit(&zstrm, ZIP_LEVEL, 0, 0)) != BZ_OK) zraise(ret);
+
+    zstrm.avail_in = length;
+    zstrm.next_in = (char *)data;
+    zstrm.avail_out = ZIP_BUFFER_SIZE;
+    zstrm.next_out = out_buffer;
+
+    do {
+        ret = BZ2_bzCompress(&zstrm, BZ_FINISH); /* no bad return value */
+        assert(ret != BZ_SEQUENCE_ERROR);        /* state not clobbered */
+        zip_len += buf_size = ZIP_BUFFER_SIZE - zstrm.avail_out;
+        os_write_bytes(out_stream, (uchar *)out_buffer, buf_size);
+    } while (zstrm.avail_out == 0);
+    assert(zstrm.avail_in == 0);       /* all input will be used */
+
+    /* clean up */
+    (void)BZ2_bzCompressEnd(&zstrm);
+    return zip_len;
 }
+#endif
 
 void fw_add_doc(FieldsWriter *fw, Document *doc)
 {
@@ -1595,6 +1906,7 @@ void fw_add_doc(FieldsWriter *fw, Document *doc)
     ary_size(fw->tv_fields) = 0;
     os_write_u64(fdx_out, fw->start_ptr);
     os_write_vint(fdt_out, stored_cnt);
+    ramo_reset(fw->buffer);
 
     for (i = 0; i < doc_size; i++) {
         df = doc->fields[i];
@@ -1602,21 +1914,29 @@ void fw_add_doc(FieldsWriter *fw, Document *doc)
         if (fi_is_stored(fi)) {
             const int df_size = df->size;
             os_write_vint(fdt_out, fi->number);
-            os_write_vint(fdt_out, df->size);
-            /**
-             * TODO: add compression
-             */
-            for (j = 0; j < df_size; j++) {
-                os_write_vint(fdt_out, df->lengths[j]);
+            os_write_vint(fdt_out, df_size);
+            if (fi_is_compressed(fi)) {
+                for (j = 0; j < df_size; j++) {
+                    const int length = df->lengths[j];
+                    int zip_len = os_write_zipped_bytes(fw->buffer,
+                                                        (uchar*)df->data[j],
+                                                        length);
+                    os_write_vint(fdt_out, zip_len - 1);
+                }
             }
-            for (j = 0; j < df_size; j++) {
-                os_write_bytes(fdt_out, (uchar *)df->data[j], df->lengths[j]);
-                /* leave a space between fields as that is how they are
-                 * analyzed */
-                os_write_byte(fdt_out, ' ');
+            else {
+                for (j = 0; j < df_size; j++) {
+                    const int length = df->lengths[j];
+                    os_write_vint(fdt_out, length);
+                    os_write_bytes(fw->buffer, (uchar*)df->data[j], length);
+                    /* leave a space between fields as that is how they are
+                     * analyzed */
+                    os_write_byte(fw->buffer, ' ');
+                }
             }
         }
     }
+    ramo_write_to(fw->buffer, fdt_out);
 }
 
 void fw_write_tv_index(FieldsWriter *fw)
@@ -1624,7 +1944,7 @@ void fw_write_tv_index(FieldsWriter *fw)
     int i;
     const int tv_cnt = ary_size(fw->tv_fields);
     OutStream *fdt_out = fw->fdt_out;
-    os_write_u32(fw->fdx_out, (f_u32)(os_pos(fdt_out) - fw->start_ptr));
+    os_write_u32(fw->fdx_out, (u32)(os_pos(fdt_out) - fw->start_ptr));
     os_write_vint(fdt_out, tv_cnt);
     /* write in reverse order so we can count back from the start position to
      * the beginning of the TermVector's data */
@@ -1683,13 +2003,13 @@ void fw_add_postings(FieldsWriter *fw,
 
     if (fi_store_offsets(fi)) {
         /* use delta encoding for offsets */
-        long long last_end = 0;
+        i64 last_end = 0;
         os_write_vint(fdt_out, offset_count);  /* write shared prefix length */
         for (i = 0; i < offset_count; i++) {
-            long long start = (long long)offsets[i].start;
-            long long end = (long long)offsets[i].end;
-            os_write_vll(fdt_out, (unsigned long long)(start - last_end));
-            os_write_vll(fdt_out, (unsigned long long)(end - start));
+            i64 start = (i64)offsets[i].start;
+            i64 end = (i64)offsets[i].end;
+            os_write_vll(fdt_out, (u64)(start - last_end));
+            os_write_vll(fdt_out, (u64)(end - start));
             last_end = end;
         }
     }
@@ -1706,16 +2026,16 @@ void fw_add_postings(FieldsWriter *fw,
 
 char *te_get_term(TermEnum *te)
 {
-    return memcpy(ALLOC_N(char, te->curr_term_len + 1),
-                  te->curr_term, te->curr_term_len + 1);
+    return (char *)memcpy(ALLOC_N(char, te->curr_term_len + 1),
+                          te->curr_term, te->curr_term_len + 1);
 }
 
 TermInfo *te_get_ti(TermEnum *te)
 {
-    return memcpy(ALLOC(TermInfo), &(te->curr_ti), sizeof(TermInfo));
+    return (TermInfo*)memcpy(ALLOC(TermInfo), &(te->curr_ti), sizeof(TermInfo));
 }
 
-char *te_skip_to(TermEnum *te, const char *term)
+static char *te_skip_to(TermEnum *te, const char *term)
 {
     char *curr_term = te->curr_term;
     if (strcmp(curr_term, term) < 0) {
@@ -1742,8 +2062,8 @@ static void sti_destroy(SegmentTermIndex *sti)
 {
     if (sti->index_terms) {
         int i;
-        const int sti_index_size = sti->index_size;
-        for (i = 0; i < sti_index_size; i++) {
+        const int sti_index_cnt = sti->index_cnt;
+        for (i = 0; i < sti_index_cnt; i++) {
             free(sti->index_terms[i]);
         }
         free(sti->index_terms);
@@ -1759,20 +2079,20 @@ static void sti_ensure_index_is_read(SegmentTermIndex *sti,
 {
     if (NULL == sti->index_terms) {
         int i;
-        int index_size = sti->index_size;
+        int index_cnt = sti->index_cnt;
         off_t index_ptr = 0;
         ste_reset(index_te);
         is_seek(STE(index_te)->is, sti->index_ptr);
-        STE(index_te)->size = sti->index_size;
-        
-        sti->index_terms = ALLOC_N(char *, index_size);
-        sti->index_term_lens = ALLOC_N(int, index_size);
-        sti->index_term_infos = ALLOC_N(TermInfo, index_size);
-        sti->index_ptrs = ALLOC_N(off_t, index_size);
-        
+        STE(index_te)->size = sti->index_cnt;
+
+        sti->index_terms = ALLOC_N(char *, index_cnt);
+        sti->index_term_lens = ALLOC_N(int, index_cnt);
+        sti->index_term_infos = ALLOC_N(TermInfo, index_cnt);
+        sti->index_ptrs = ALLOC_N(off_t, index_cnt);
+
         for (i = 0; NULL != ste_next(index_te); i++) {
 #ifdef DEBUG
-            if (i >= index_size) {
+            if (i >= index_cnt) {
                 RAISE(FERRET_ERROR, "index term enum read too many terms");
             }
 #endif
@@ -1788,7 +2108,7 @@ static void sti_ensure_index_is_read(SegmentTermIndex *sti,
 static int sti_get_index_offset(SegmentTermIndex *sti, const char *term)
 {
     int lo = 0;
-    int hi = sti->index_size - 1;
+    int hi = sti->index_cnt - 1;
     int mid, delta;
     char **index_terms = sti->index_terms;
 
@@ -1842,7 +2162,7 @@ SegmentFieldIndex *sfi_open(Store *store, const char *segment)
         SegmentTermIndex *sti = ALLOC_AND_ZERO(SegmentTermIndex);
         sti->index_ptr = is_read_voff_t(is);
         sti->ptr = is_read_voff_t(is);
-        sti->index_size = is_read_vint(is);
+        sti->index_cnt = is_read_vint(is);
         sti->size = is_read_vint(is);
         h_set_int(sfi->field_dict, field_num, sti);
     }
@@ -1912,7 +2232,8 @@ static void ste_reset(TermEnum *te)
 
 static TermEnum *ste_set_field(TermEnum *te, int field_num)
 {
-    SegmentTermIndex *sti = h_get_int(STE(te)->sfi->field_dict, field_num);
+    SegmentTermIndex *sti
+        = (SegmentTermIndex *)h_get_int(STE(te)->sfi->field_dict, field_num);
     ste_reset(te);
     te->field_num = field_num;
     if (sti) {
@@ -1940,7 +2261,8 @@ static void ste_index_seek(TermEnum *te, SegmentTermIndex *sti, int idx_offset)
 static char *ste_scan_to(TermEnum *te, const char *term)
 {
     SegmentFieldIndex *sfi = STE(te)->sfi;
-    SegmentTermIndex *sti = h_get_int(sfi->field_dict, te->field_num);
+    SegmentTermIndex *sti
+        = (SegmentTermIndex *)h_get_int(sfi->field_dict, te->field_num);
     if (sti && sti->size > 0) {
         SFI_ENSURE_INDEX_IS_READ(sfi, sti);
         if (term[0] == '\0') {
@@ -1952,8 +2274,8 @@ static char *ste_scan_to(TermEnum *te, const char *term)
             int enum_offset = (int)(STE(te)->pos / sfi->index_interval) + 1;
             /* if we are at the end of the index or before the next index
              * ptr then a simple scan suffices */
-            if (sti->index_size == enum_offset ||
-                strcmp(term, sti->index_terms[enum_offset]) < 0) { 
+            if (sti->index_cnt == enum_offset ||
+                strcmp(term, sti->index_terms[enum_offset]) < 0) {
                 return te_skip_to(te, term);
             }
         }
@@ -2014,8 +2336,8 @@ static char *ste_get_term(TermEnum *te, int pos)
     else if (pos != ste->pos) {
         int idx_int = ste->sfi->index_interval;
         if ((pos < ste->pos) || pos > (1 + ste->pos / idx_int) * idx_int) {
-            SegmentTermIndex *sti = h_get_int(ste->sfi->field_dict,
-                                              te->field_num);
+            SegmentTermIndex *sti = (SegmentTermIndex *)h_get_int(
+                ste->sfi->field_dict, te->field_num);
             SFI_ENSURE_INDEX_IS_READ(ste->sfi, sti);
             ste_index_seek(te, sti, pos / idx_int);
         }
@@ -2119,8 +2441,8 @@ static void tew_destroy(TermEnumWrapper *tew)
     tew->te->close(tew->te);
 }
 
-TermEnumWrapper *tew_setup(TermEnumWrapper *tew, int index, TermEnum *te,
-                           IndexReader *ir)
+static TermEnumWrapper *tew_setup(TermEnumWrapper *tew, int index, TermEnum *te,
+                                  IndexReader *ir)
 {
     tew->index = index;
     tew->ir = ir;
@@ -2263,7 +2585,8 @@ TermEnum *mte_new(MultiReader *mr, int field_num, const char *term)
                 || (tew->term && (tew->term[0] != '\0'))) {
                 pq_push(mte->tew_queue, tew);          /* initialize queue */
             }
-        } else {
+        }
+        else {
             /* add the term_enum_wrapper just in case */
             sub_te = reader->terms(reader, 0);
             sub_te->field_num = -1;
@@ -2303,7 +2626,7 @@ TermInfosReader *tir_open(Store *store,
 static INLINE TermEnum *tir_enum(TermInfosReader *tir)
 {
     TermEnum *te;
-    if (NULL == (te = thread_getspecific(tir->thread_te))) {
+    if (NULL == (te = (TermEnum *)thread_getspecific(tir->thread_te))) {
         te = ste_clone(tir->orig_te);
         ste_set_field(te, tir->field_num);
         ary_push(tir->te_bucket, te);
@@ -2333,8 +2656,8 @@ TermInfo *tir_get_ti(TermInfosReader *tir, const char *term)
     return NULL;
 }
 
-TermInfo *tir_get_ti_field(TermInfosReader *tir, int field_num,
-                           const char *term)
+static TermInfo *tir_get_ti_field(TermInfosReader *tir, int field_num,
+                                  const char *term)
 {
     TermEnum *te = tir_enum(tir);
     char *match;
@@ -2352,7 +2675,7 @@ TermInfo *tir_get_ti_field(TermInfosReader *tir, int field_num,
 }
 
 char *tir_get_term(TermInfosReader *tir, int pos)
-{ 
+{
     if (pos < 0) {
         return NULL;
     }
@@ -2455,11 +2778,11 @@ static void tw_add(TermWriter *tw,
               tw->last_term, term, *tw->last_term, *term);
     }
     if (ti->frq_ptr < tw->last_term_info.frq_ptr) {
-        RAISE(STATE_ERROR, "%"F_OFF_T_PFX"d > %"F_OFF_T_PFX"d", ti->frq_ptr,
+        RAISE(STATE_ERROR, "%"OFF_T_PFX"d > %"OFF_T_PFX"d", ti->frq_ptr,
               tw->last_term_info.frq_ptr);
     }
     if (ti->prx_ptr < tw->last_term_info.prx_ptr) {
-        RAISE(STATE_ERROR, "%"F_OFF_T_PFX"d > %"F_OFF_T_PFX"d", ti->prx_ptr,
+        RAISE(STATE_ERROR, "%"OFF_T_PFX"d > %"OFF_T_PFX"d", ti->prx_ptr,
               tw->last_term_info.prx_ptr);
     }
 #endif
@@ -2611,7 +2934,7 @@ static bool stde_next(TermDocEnum *tde)
     int doc_code;
     SegmentTermDocEnum *stde = STDE(tde);
 
-    while (true) { 
+    while (true) {
         if (stde->count >= stde->doc_freq) {
             return false;
         }
@@ -2726,7 +3049,7 @@ static bool stde_skip_to(TermDocEnum *tde, int target_doc_num)
     }
 
     /* done skipping, now just scan */
-    do { 
+    do {
         if (!tde->next(tde)) {
             return false;
         }
@@ -2746,12 +3069,12 @@ static void stde_close(TermDocEnum *tde)
 }
 
 static void stde_skip_prox(SegmentTermDocEnum *stde)
-{ 
+{
     (void)stde;
 }
 
 static void stde_seek_prox(SegmentTermDocEnum *stde, off_t prx_ptr)
-{ 
+{
     (void)stde;
     (void)prx_ptr;
 }
@@ -2812,7 +3135,7 @@ static void stpe_seek(TermDocEnum *tde, int field_num, const char *term)
     stde->prx_cnt = 0;
 }
 
-bool stpe_next(TermDocEnum *tde)
+static bool stpe_next(TermDocEnum *tde)
 {
     SegmentTermDocEnum *stde = STDE(tde);
     is_skip_vints(stde->prx_in, stde->prx_cnt);
@@ -2829,7 +3152,7 @@ bool stpe_next(TermDocEnum *tde)
     }
 }
 
-int stpe_read(TermDocEnum *tde, int *docs, int *freqs, int req_num)
+static int stpe_read(TermDocEnum *tde, int *docs, int *freqs, int req_num)
 {
     (void)tde; (void)docs; (void)freqs; (void)req_num;
     RAISE(ARG_ERROR, "TermPosEnum does not handle processing multiple documents"
@@ -2944,9 +3267,11 @@ static void mtde_seek_te(TermDocEnum *tde, TermEnum *te)
         mtde->state[index] = 1;
         if (tde->close == stde_close) {
             stde_seek_ti(STDE(tde), MTE(te)->tis + i);
-        } else if (tde->close == stpe_close) {
+        }
+        else if (tde->close == stpe_close) {
             stpe_seek_ti(STDE(tde), MTE(te)->tis + i);
-        } else {
+        }
+        else {
             tde->seek(tde, MTE(te)->tews[index].te->field_num, te->curr_term);
         }
     }
@@ -2963,7 +3288,8 @@ static void mtde_seek(TermDocEnum *tde, int field_num, const char *term)
     te->set_field(te, field_num);
     if (NULL != (t = te->skip_to(te, term)) && 0 == strcmp(term, t)) {
         mtde_seek_te(tde, te);
-    } else {
+    }
+    else {
         memset(mtde->state, 0, mtde->ir_cnt);
     }
 }
@@ -3051,7 +3377,7 @@ static void mtde_close(TermDocEnum *tde)
     free(tde);
 }
 
-TermDocEnum *mtxe_new(MultiReader *mr)
+static TermDocEnum *mtxe_new(MultiReader *mr)
 {
     MultiTermDocEnum *mtde  = ALLOC_AND_ZERO(MultiTermDocEnum);
     TermDocEnum *tde        = TDE(mtde);
@@ -3074,7 +3400,7 @@ TermDocEnum *mtxe_new(MultiReader *mr)
     return tde;
 }
 
-TermDocEnum *mtde_new(MultiReader *mr)
+static TermDocEnum *mtde_new(MultiReader *mr)
 {
     int i;
     TermDocEnum *tde        = mtxe_new(mr);
@@ -3090,13 +3416,13 @@ TermDocEnum *mtde_new(MultiReader *mr)
  * MultiTermPosEnum
  ****************************************************************************/
 
-int mtpe_next_position(TermDocEnum *tde)
+static int mtpe_next_position(TermDocEnum *tde)
 {
     CHECK_CURR_TDE("next_position");
     return MTDE(tde)->curr_tde->next_position(MTDE(tde)->curr_tde);
 }
 
-TermDocEnum *mtpe_new(MultiReader *mr)
+static TermDocEnum *mtpe_new(MultiReader *mr)
 {
     int i;
     TermDocEnum *tde        = mtxe_new(mr);
@@ -3184,7 +3510,7 @@ static bool mtdpe_next(TermDocEnum *tde)
             pq_down(mtdpe->pq);
         }
         else {
-            sub_tde = pq_pop(mtdpe->pq);
+            sub_tde = (TermDocEnum *)pq_pop(mtdpe->pq);
             sub_tde->close(sub_tde);
         }
         sub_tde = (TermDocEnum *)pq_top(mtdpe->pq);
@@ -3199,12 +3525,12 @@ static bool mtdpe_next(TermDocEnum *tde)
     return true;
 }
 
-bool tdpe_less_than(TermDocEnum *p1, TermDocEnum *p2)
+static bool tdpe_less_than(TermDocEnum *p1, TermDocEnum *p2)
 {
     return p1->doc_num(p1) < p2->doc_num(p2);
 }
 
-bool mtdpe_skip_to(TermDocEnum *tde, int target_doc_num)
+static bool mtdpe_skip_to(TermDocEnum *tde, int target_doc_num)
 {
     TermDocEnum *sub_tde;
     PriorityQueue *mtdpe_pq = MTDPE(tde)->pq;
@@ -3215,7 +3541,7 @@ bool mtdpe_skip_to(TermDocEnum *tde, int target_doc_num)
             pq_down(mtdpe_pq);
         }
         else {
-            sub_tde = pq_pop(mtdpe_pq);
+            sub_tde = (TermDocEnum *)pq_pop(mtdpe_pq);
             sub_tde->close(sub_tde);
         }
     }
@@ -3284,22 +3610,21 @@ TermDocEnum *mtdpe_new(IndexReader *ir, int field_num, char **terms, int t_cnt)
  *
  ****************************************************************************/
 
-static HashTable *fn_extensions = NULL;
+static Hash *fn_extensions = NULL;
 static void file_name_filter_init()
 {
-    if (NULL == fn_extensions) {
-        int i;
-        fn_extensions = h_new_str((free_ft)NULL, (free_ft)NULL);
-        for (i = 0; i < NELEMS(INDEX_EXTENSIONS); i++) {
-          h_set(fn_extensions, INDEX_EXTENSIONS[i], (char *)INDEX_EXTENSIONS[i]);
-        }
-        register_for_cleanup(fn_extensions, (free_ft)&h_destroy);
+    int i;
+    fn_extensions = h_new_str((free_ft)NULL, (free_ft)NULL);
+    for (i = 0; i < NELEMS(INDEX_EXTENSIONS); i++) {
+      h_set(fn_extensions, INDEX_EXTENSIONS[i], (char *)INDEX_EXTENSIONS[i]);
     }
+    register_for_cleanup(fn_extensions, (free_ft)&h_destroy);
 }
 
-static bool file_name_filter_accept(char *file_name)
+bool file_name_filter_is_index_file(const char *file_name, bool include_locks)
 {
     char *p = strrchr(file_name, '.');
+    if (NULL == fn_extensions) file_name_filter_init();
     if (NULL != p) {
         char *extension = p + 1;
         if (NULL != h_get(fn_extensions, extension)) {
@@ -3310,6 +3635,10 @@ static bool file_name_filter_accept(char *file_name)
                  && *(extension + 1) <= '9') {
             return true;
         }
+        else if (include_locks && (strcmp(extension, "lck") == 0)
+                               && (strncmp(file_name, "ferret", 6) == 0)) {
+            return true;
+        }
     }
     else if (0 == strncmp(SEGMENTS_FILE_NAME, file_name,
                           sizeof(SEGMENTS_FILE_NAME) - 1)) {
@@ -3323,7 +3652,7 @@ static bool file_name_filter_accept(char *file_name)
  * function should only be called on files that pass the above "accept" (ie,
  * are already known to be a Lucene index file).
  */
-static bool file_name_filter_is_cfs_file(char *file_name) {
+static bool file_name_filter_is_cfs_file(const char *file_name) {
     char *p = strrchr(file_name, '.');
     if (NULL != p) {
         char *extension = p + 1;
@@ -3364,7 +3693,7 @@ void deleter_destroy(Deleter *dlr)
     free(dlr);
 }
 
-void deleter_queue_file(Deleter *dlr, char *file_name)
+static void deleter_queue_file(Deleter *dlr, const char *file_name)
 {
     hs_add(dlr->pending, estrdup(file_name));
 }
@@ -3382,12 +3711,12 @@ void deleter_delete_file(Deleter *dlr, char *file_name)
     XENDTRY
 }
 
-void deleter_commit_pending_deletions(Deleter *dlr)
+static void deleter_commit_pending_deletions(Deleter *dlr)
 {
-    int i;
-    char **pending = (char **)dlr->pending->elems;
-    for (i = dlr->pending->size - 1; i >= 0; i--) {
-        deleter_delete_file(dlr, pending[i]);
+    HashSetEntry *hse, *hse_next = dlr->pending->first;
+    while ((hse = hse_next) != NULL) {
+        hse_next = hse->next;
+        deleter_delete_file(dlr, (char *)hse->elem);
     }
 }
 
@@ -3403,15 +3732,15 @@ void deleter_delete_files(Deleter *dlr, char **files, int file_cnt)
 struct DelFilesArg {
     char  curr_seg_file_name[SEGMENT_NAME_MAX_LENGTH];
     Deleter *dlr;
-    HashTable *current;
+    Hash *current;
 };
 
-static void deleter_find_deletable_files_i(char *file_name, void *arg)
+static void deleter_find_deletable_files_i(const char *file_name, void *arg)
 {
     struct DelFilesArg *dfa = (struct DelFilesArg *)arg;
     Deleter *dlr = dfa->dlr;
 
-    if (file_name_filter_accept(file_name)
+    if (file_name_filter_is_index_file(file_name, false)
         && 0 != strcmp(file_name, dfa->curr_seg_file_name)
         && 0 != strcmp(file_name, SEGMENTS_GEN_FILE_NAME)) {
 
@@ -3427,7 +3756,8 @@ static void deleter_find_deletable_files_i(char *file_name, void *arg)
         if (NULL != p) {
             *p = '\0';
             extension = p + 1;
-        } else {
+        }
+        else {
             extension = NULL;
         }
 
@@ -3439,7 +3769,7 @@ static void deleter_find_deletable_files_i(char *file_name, void *arg)
 
         /* Delete this file if it's not a "current" segment, or, it is a
          * single index file but there is now a corresponding compound file: */
-        if (NULL == (si = h_get(dfa->current, segment_name))) {
+        if (NULL == (si = (SegmentInfo *)h_get(dfa->current, segment_name))) {
             /* Delete if segment is not referenced: */
             do_delete = true;
         }
@@ -3497,7 +3827,7 @@ void deleter_find_deletable_files(Deleter *dlr)
     SegmentInfos *sis = dlr->sis;
     Store *store = dlr->store;
     struct DelFilesArg dfa;
-    HashTable *current = dfa.current
+    Hash *current = dfa.current
                        = h_new_str((free_ft)NULL, (free_ft)si_deref);
     dfa.dlr = dlr;
 
@@ -3511,22 +3841,24 @@ void deleter_find_deletable_files(Deleter *dlr)
      * and add to deletable if they are not referenced by the current segments
      * info: */
     sis_curr_seg_file_name(dfa.curr_seg_file_name, store);
-    file_name_filter_init();
 
     store->each(store, &deleter_find_deletable_files_i, &dfa);
     h_destroy(dfa.current);
 }
 
-void deleter_delete_deletable_files(Deleter *dlr)
+static void deleter_delete_deletable_files(Deleter *dlr)
 {
     deleter_find_deletable_files(dlr);
     deleter_commit_pending_deletions(dlr);
 }
 
-void deleter_clear_pending_deletions(Deleter *dlr)
+/*
+TODO: currently not used. Why not?
+static void deleter_clear_pending_deletions(Deleter *dlr)
 {
     hs_clear(dlr->pending);
 }
+*/
 
 /****************************************************************************
  *
@@ -3534,13 +3866,13 @@ void deleter_clear_pending_deletions(Deleter *dlr)
  *
  ****************************************************************************/
 
-void ir_acquire_not_necessary(IndexReader *ir)
+static void ir_acquire_not_necessary(IndexReader *ir)
 {
     (void)ir;
 }
 
 #define I64_PFX POSH_I64_PRINTF_PREFIX
-void ir_acquire_write_lock(IndexReader *ir)
+static void ir_acquire_write_lock(IndexReader *ir)
 {
     if (ir->is_stale) {
         RAISE(STATE_ERROR, "IndexReader out of date and no longer valid for "
@@ -3579,10 +3911,11 @@ void ir_acquire_write_lock(IndexReader *ir)
     }
 }
 
-IndexReader *ir_setup(IndexReader *ir, Store *store, SegmentInfos *sis,
+static IndexReader *ir_setup(IndexReader *ir, Store *store, SegmentInfos *sis,
                       FieldInfos *fis, int is_owner)
 {
     mutex_init(&ir->mutex, NULL);
+    mutex_init(&ir->field_index_mutex, NULL);
 
     if (store) {
         ir->store = store;
@@ -3608,16 +3941,17 @@ bool ir_index_exists(Store *store)
     return sis_current_segment_generation(store) != 1;
 }
 
-int ir_get_field_num(IndexReader *ir, const char *field)
+int ir_get_field_num(IndexReader *ir, Symbol field)
 {
     int field_num = fis_get_field_num(ir->fis, field);
     if (field_num < 0) {
-        RAISE(ARG_ERROR, "Field :%s does not exist in this index", field);
+        RAISE(ARG_ERROR,
+              "Field :%s does not exist in this index", (char *)field);
     }
     return field_num;
 }
 
-int ir_doc_freq(IndexReader *ir, const char *field, const char *term)
+int ir_doc_freq(IndexReader *ir, Symbol field, const char *term)
 {
     int field_num = fis_get_field_num(ir->fis, field);
     if (field_num >= 0) {
@@ -3637,7 +3971,7 @@ static void ir_set_norm_i(IndexReader *ir, int doc_num, int field_num, uchar val
     mutex_unlock(&ir->mutex);
 }
 
-void ir_set_norm(IndexReader *ir, int doc_num, const char *field, uchar val)
+void ir_set_norm(IndexReader *ir, int doc_num, Symbol field, uchar val)
 {
     int field_num = fis_get_field_num(ir->fis, field);
     if (field_num >= 0) {
@@ -3653,20 +3987,20 @@ uchar *ir_get_norms_i(IndexReader *ir, int field_num)
     }
     if (!norms) {
         if (NULL == ir->fake_norms) {
-            ir->fake_norms = (uchar *)ecalloc(ir->max_doc(ir));
+            ir->fake_norms = ALLOC_AND_ZERO_N(uchar, ir->max_doc(ir));
         }
         norms = ir->fake_norms;
     }
     return norms;
 }
 
-uchar *ir_get_norms(IndexReader *ir, const char *field)
+uchar *ir_get_norms(IndexReader *ir, Symbol field)
 {
     int field_num = fis_get_field_num(ir->fis, field);
     return ir_get_norms_i(ir, field_num);
 }
 
-uchar *ir_get_norms_into(IndexReader *ir, const char *field, uchar *buf)
+uchar *ir_get_norms_into(IndexReader *ir, Symbol field, uchar *buf)
 {
     int field_num = fis_get_field_num(ir->fis, field);
     if (field_num >= 0) {
@@ -3698,7 +4032,7 @@ void ir_delete_doc(IndexReader *ir, int doc_num)
     }
 }
 
-Document *ir_get_doc_with_term(IndexReader *ir, const char *field,
+Document *ir_get_doc_with_term(IndexReader *ir, Symbol field,
                                const char *term)
 {
     TermDocEnum *tde = ir_term_docs_for(ir, field, term);
@@ -3713,7 +4047,7 @@ Document *ir_get_doc_with_term(IndexReader *ir, const char *field,
     return doc;
 }
 
-TermEnum *ir_terms(IndexReader *ir, const char *field)
+TermEnum *ir_terms(IndexReader *ir, Symbol field)
 {
     TermEnum *te = NULL;
     int field_num = fis_get_field_num(ir->fis, field);
@@ -3723,7 +4057,7 @@ TermEnum *ir_terms(IndexReader *ir, const char *field)
     return te;
 }
 
-TermEnum *ir_terms_from(IndexReader *ir, const char *field,
+TermEnum *ir_terms_from(IndexReader *ir, Symbol field,
                            const char *term)
 {
     TermEnum *te = NULL;
@@ -3734,7 +4068,7 @@ TermEnum *ir_terms_from(IndexReader *ir, const char *field,
     return te;
 }
 
-TermDocEnum *ir_term_docs_for(IndexReader *ir, const char *field,
+TermDocEnum *ir_term_docs_for(IndexReader *ir, Symbol field,
                               const char *term)
 {
     int field_num = fis_get_field_num(ir->fis, field);
@@ -3745,7 +4079,7 @@ TermDocEnum *ir_term_docs_for(IndexReader *ir, const char *field,
     return tde;
 }
 
-TermDocEnum *ir_term_positions_for(IndexReader *ir, const char *field,
+TermDocEnum *ir_term_positions_for(IndexReader *ir, Symbol field,
                                    const char *term)
 {
     int field_num = fis_get_field_num(ir->fis, field);
@@ -3756,7 +4090,7 @@ TermDocEnum *ir_term_positions_for(IndexReader *ir, const char *field,
     return tde;
 }
 
-void ir_commit_i(IndexReader *ir)
+static void ir_commit_i(IndexReader *ir)
 {
     if (ir->has_changes) {
         if (NULL == ir->deleter && NULL != ir->store) {
@@ -3769,7 +4103,7 @@ void ir_commit_i(IndexReader *ir)
             mutex_lock(&ir->store->mutex);
 
             sis_curr_seg_file_name(curr_seg_fn, ir->store);
-            
+
             ir->commit_i(ir);
             sis_write(ir->sis, ir->store, ir->deleter);
 
@@ -3813,8 +4147,8 @@ void ir_close(IndexReader *ir)
         if (ir->cache) {
             h_destroy(ir->cache);
         }
-        if (ir->sort_cache) {
-            h_destroy(ir->sort_cache);
+        if (ir->field_index_cache) {
+            h_destroy(ir->field_index_cache);
         }
         if (ir->deleter && ir->is_owner) {
             deleter_destroy(ir->deleter);
@@ -3822,8 +4156,10 @@ void ir_close(IndexReader *ir)
         free(ir->fake_norms);
 
         mutex_destroy(&ir->mutex);
+        mutex_destroy(&ir->field_index_mutex);
         free(ir);
-    } else {
+    }
+    else {
         mutex_unlock(&ir->mutex);
     }
 
@@ -3910,7 +4246,7 @@ typedef struct SegmentReader {
     TermInfosReader *tir;
     thread_key_t thread_fr;
     void **fr_bucket;
-    HashTable *norms;
+    Hash *norms;
     Store *cfs_store;
     bool deleted_docs_dirty : 1;
     bool undelete_all : 1;
@@ -3926,7 +4262,7 @@ static INLINE FieldsReader *sr_fr(SegmentReader *sr)
 {
     FieldsReader *fr;
 
-    if (NULL == (fr = thread_getspecific(sr->thread_fr))) {
+    if (NULL == (fr = (FieldsReader *)thread_getspecific(sr->thread_fr))) {
         fr = fr_clone(sr->fr);
         ary_push(sr->fr_bucket, fr);
         thread_setspecific(sr->thread_fr, fr);
@@ -3942,7 +4278,7 @@ static INLINE bool sr_is_deleted_i(SegmentReader *sr, int doc_num)
 static INLINE void sr_get_norms_into_i(SegmentReader *sr, int field_num,
                                        uchar *buf)
 {
-    Norm *norm = h_get_int(sr->norms, field_num);
+    Norm *norm = (Norm *)h_get_int(sr->norms, field_num);
     if (NULL == norm) {
         memset(buf, 0, SR_SIZE(sr));
     }
@@ -3960,7 +4296,7 @@ static INLINE void sr_get_norms_into_i(SegmentReader *sr, int field_num,
 
 static INLINE uchar *sr_get_norms_i(SegmentReader *sr, int field_num)
 {
-    Norm *norm = h_get_int(sr->norms, field_num);
+    Norm *norm = (Norm *)h_get_int(sr->norms, field_num);
     if (NULL == norm) {                           /* not an indexed field */
         return NULL;
     }
@@ -3975,7 +4311,7 @@ static INLINE uchar *sr_get_norms_i(SegmentReader *sr, int field_num)
 
 static void sr_set_norm_i(IndexReader *ir, int doc_num, int field_num, uchar b)
 {
-    Norm *norm = h_get_int(SR(ir)->norms, field_num);
+    Norm *norm = (Norm *)h_get_int(SR(ir)->norms, field_num);
     if (NULL != norm) { /* has_norms */
         ir->has_changes = true;
         norm->is_dirty = true; /* mark it dirty */
@@ -3984,7 +4320,7 @@ static void sr_set_norm_i(IndexReader *ir, int doc_num, int field_num, uchar b)
     }
 }
 
-static void sr_delete_doc_i(IndexReader *ir, int doc_num) 
+static void sr_delete_doc_i(IndexReader *ir, int doc_num)
 {
     if (NULL == SR(ir)->deleted_docs) {
         SR(ir)->deleted_docs = bv_new();
@@ -4017,7 +4353,7 @@ static void bv_write(BitVector *bv, Store *store, char *name)
     int i;
     OutStream *os = store->new_output(store, name);
     os_write_vint(os, bv->size);
-    for (i = (bv->size >> 5); i >= 0; i--) {
+    for (i = ((bv->size-1) >> 5); i >= 0; i--) {
         os_write_u32(os, bv->bits[i]);
     }
     os_close(os);
@@ -4031,10 +4367,10 @@ static BitVector *bv_read(Store *store, char *name)
     BitVector *volatile bv = ALLOC_AND_ZERO(BitVector);
     bv->size = (int)is_read_vint(is);
     bv->capa = (bv->size >> 5) + 1;
-    bv->bits = ALLOC_AND_ZERO_N(f_u32, bv->capa);
+    bv->bits = ALLOC_AND_ZERO_N(u32, bv->capa);
     bv->ref_cnt = 1;
     TRY
-        for (i = (bv->size >> 5); i >= 0; i--) {
+        for (i = ((bv->size-1) >> 5); i >= 0; i--) {
             bv->bits[i] = is_read_u32(is);
         }
         bv_recount(bv);
@@ -4065,7 +4401,8 @@ static void sr_commit_i(IndexReader *ir)
         if (SR(ir)->undelete_all) {
             si->del_gen = -1;
             SR(ir)->undelete_all = false;
-        } else {
+        }
+        else {
             /* (SR(ir)->deleted_docs_dirty) re-write deleted */
             si->del_gen++;
             fn_for_generation(tmp_file_name, segment, "del", si->del_gen);
@@ -4080,7 +4417,7 @@ static void sr_commit_i(IndexReader *ir)
         for (i = field_cnt - 1; i >= 0; i--) {
             fi = ir->fis->fields[i];
             if (fi_is_indexed(fi)) {
-                Norm *norm = h_get_int(SR(ir)->norms, fi->number);
+                Norm *norm = (Norm *)h_get_int(SR(ir)->norms, fi->number);
                 if (norm && norm->is_dirty) {
                     norm_rewrite(norm, ir->store, ir->deleter, SR(ir)->si,
                                  SR_SIZE(ir));
@@ -4208,9 +4545,9 @@ static TermDocEnum *sr_term_positions(IndexReader *ir)
 }
 
 static TermVector *sr_term_vector(IndexReader *ir, int doc_num,
-                                  const char *field)
+                                  Symbol field)
 {
-    FieldInfo *fi = h_get(ir->fis->field_dict, field);
+    FieldInfo *fi = (FieldInfo *)h_get(ir->fis->field_dict, field);
     FieldsReader *fr;
 
     if (!fi || !fi_store_term_vector(fi) || !SR(ir)->fr ||
@@ -4221,7 +4558,7 @@ static TermVector *sr_term_vector(IndexReader *ir, int doc_num,
     return fr_get_field_tv(fr, doc_num, fi->number);
 }
 
-static HashTable *sr_term_vectors(IndexReader *ir, int doc_num)
+static Hash *sr_term_vectors(IndexReader *ir, int doc_num)
 {
     FieldsReader *fr;
     if (!SR(ir)->fr || NULL == (fr = sr_fr(SR(ir)))) {
@@ -4266,7 +4603,7 @@ static void sr_open_norms(IndexReader *ir, Store *cfs_store)
 
 static IndexReader *sr_setup_i(SegmentReader *sr)
 {
-    Store *store = sr->si->store;
+    Store *volatile store = sr->si->store;
     IndexReader *ir = IR(sr);
     char file_name[SEGMENT_NAME_MAX_LENGTH];
     char *sr_segment = sr->si->name;
@@ -4375,7 +4712,7 @@ static int mr_reader_index_i(MultiReader *mr, int doc_num)
     return hi;
 }
 
-int mr_num_docs(IndexReader *ir)
+static int mr_num_docs(IndexReader *ir)
 {
     int i, num_docs;
     mutex_lock(&ir->mutex);
@@ -4429,7 +4766,7 @@ static uchar *mr_get_norms(IndexReader *ir, int field_num)
     uchar *bytes;
 
     mutex_lock(&ir->mutex);
-    bytes = h_get_int(MR(ir)->norms_cache, field_num);
+    bytes = (uchar *)h_get_int(MR(ir)->norms_cache, field_num);
     if (NULL == bytes) {
         int i;
         const int mr_reader_cnt = MR(ir)->r_cnt;
@@ -4455,7 +4792,7 @@ static uchar *mr_get_norms_into(IndexReader *ir, int field_num, uchar *buf)
     uchar *bytes;
 
     mutex_lock(&ir->mutex);
-    bytes = h_get_int(MR(ir)->norms_cache, field_num);
+    bytes = (uchar *)h_get_int(MR(ir)->norms_cache, field_num);
     if (NULL != bytes) {
         memcpy(buf, bytes, MR(ir)->max_doc);
     }
@@ -4509,13 +4846,13 @@ static TermDocEnum *mr_term_positions(IndexReader *ir)
 }
 
 static TermVector *mr_term_vector(IndexReader *ir, int doc_num,
-                                  const char *field)
+                                  Symbol field)
 {
     GET_READER();
     return reader->term_vector(reader, doc_num - MR(ir)->starts[i], field);
 }
 
-static HashTable *mr_term_vectors(IndexReader *ir, int doc_num)
+static Hash *mr_term_vectors(IndexReader *ir, int doc_num)
 {
     GET_READER();
     return reader->term_vectors(reader, doc_num - MR(ir)->starts[i]);
@@ -4667,7 +5004,7 @@ static IndexReader *mr_new(IndexReader **sub_readers, const int r_cnt)
     return ir;
 }
 
-IndexReader *mr_open_i(Store *store,
+static IndexReader *mr_open_i(Store *store,
                        SegmentInfos *sis,
                        FieldInfos *fis,
                        IndexReader **sub_readers,
@@ -4696,7 +5033,7 @@ IndexReader *mr_open(IndexReader **sub_readers, const int r_cnt)
     IndexReader *ir = mr_new(sub_readers, r_cnt);
     MultiReader *mr = MR(ir);
     /* defaults don't matter, this is just for reading fields, not adding */
-    FieldInfos *fis = fis_new(0, 0, 0);
+    FieldInfos *fis = fis_new(STORE_NO, INDEX_NO, TERM_VECTOR_NO);
     int i, j;
     bool need_field_map = false;
 
@@ -4731,7 +5068,7 @@ IndexReader *mr_open(IndexReader **sub_readers, const int r_cnt)
                 mr->field_num_map[i][j] = fi_sub ? fi_sub->number : -1;
             }
         }
-        /* print out the field map 
+        /* print out the field map
         for (i = 0; i < r_cnt; i++) {
             for (j = 0; j < fis->size; j++) {
                 printf("%d ", mr->field_num_map[i][j]);
@@ -4766,14 +5103,14 @@ static void ir_open_i(Store *store, FindSegmentsFile *fsf)
 
         mutex_lock(&store->mutex);
         sis_read_i(store, fsf);
-        sis = fsf->p_return;
+        sis = fsf->ret.sis;
         fis = sis->fis;
 
         if (sis->size == 1) {
             ir = sr_open(sis, fis, 0, true);
         }
         else {
-            int i;
+            volatile int i;
             IndexReader **readers = ALLOC_N(IndexReader *, sis->size);
             int num_segments = sis->size;
             for (i = num_segments - 1; i >= 0; i--) {
@@ -4788,7 +5125,7 @@ static void ir_open_i(Store *store, FindSegmentsFile *fsf)
             }
             ir = mr_open_i(store, sis, fis, readers, sis->size);
         }
-        fsf->p_return = ir;
+        fsf->ret.ir = ir;
         success = true;
     } while (0);
     XFINALLY
@@ -4812,7 +5149,7 @@ IndexReader *ir_open(Store *store)
 {
     FindSegmentsFile fsf;
     sis_find_segments_file(store, &fsf, &ir_open_i);
-    return (IndexReader *)fsf.p_return;
+    return fsf.ret.ir;
 }
 
 /****************************************************************************
@@ -4865,10 +5202,11 @@ Posting *p_new(MemoryPool *mp, int doc_num, int pos)
  *
  ****************************************************************************/
 
-PostingList *pl_new(MemoryPool *mp, const char *term, int term_len, Posting *p)
+PostingList *pl_new(MemoryPool *mp, const char *term,
+                           int term_len, Posting *p)
 {
     PostingList *pl = MP_ALLOC(mp, PostingList);
-    pl->term = mp_memdup(mp, term, term_len + 1);
+    pl->term = (char *)mp_memdup(mp, term, term_len + 1);
     pl->term_len = term_len;
     pl->first = pl->last = p;
     pl->last_occ = p->first_occ;
@@ -4881,7 +5219,7 @@ void pl_add_occ(MemoryPool *mp, PostingList *pl, int pos)
     pl->last->freq++;
 }
 
-void pl_add_posting(PostingList *pl, Posting *p)
+static void pl_add_posting(PostingList *pl, Posting *p)
 {
     pl->last = pl->last->next = p;
     pl->last_occ = p->first_occ;
@@ -4905,7 +5243,8 @@ static FieldInverter *fld_inv_new(DocWriter *dw, FieldInfo *fi)
     fld_inv->store_term_vector = fi_store_term_vector(fi);
     fld_inv->store_offsets = fi_store_offsets(fi);
     if ((fld_inv->has_norms = fi_has_norms(fi)) == true) {
-        fld_inv->norms = MP_ALLOC_AND_ZERO_N(dw->mp, uchar, dw->max_buffered_docs);
+        fld_inv->norms = MP_ALLOC_AND_ZERO_N(dw->mp, uchar,
+                                             dw->max_buffered_docs);
     }
     fld_inv->fi = fi;
 
@@ -4991,15 +5330,15 @@ static void dw_write_norms(DocWriter *dw, FieldInverter *fld_inv)
     char file_name[SEGMENT_NAME_MAX_LENGTH];
     OutStream *norms_out;
     si_advance_norm_gen(dw->si, fld_inv->fi->number);
-    si_norm_file_name(dw->si, file_name, fld_inv->fi->number); 
+    si_norm_file_name(dw->si, file_name, fld_inv->fi->number);
     norms_out = dw->store->new_output(dw->store, file_name);
     os_write_bytes(norms_out, fld_inv->norms, dw->doc_num);
     os_close(norms_out);
 }
 
-/* we'll use the postings HashTable's table area to sort the postings as it is
+/* we'll use the postings Hash's table area to sort the postings as it is
  * going to be zeroset soon anyway */
-static PostingList **dw_sort_postings(HashTable *plists_ht)
+static PostingList **dw_sort_postings(Hash *plists_ht)
 {
     int i, j;
     HashEntry *he;
@@ -5054,8 +5393,8 @@ static void dw_flush(DocWriter *dw)
 
     for (i = 0; i < fields_count; i++) {
         fi = fis->fields[i];
-        if (!fi_is_indexed(fi)
-            || NULL == (fld_inv = h_get_int(dw->fields, fi->number))) {
+        if (!fi_is_indexed(fi) || NULL ==
+            (fld_inv = (FieldInverter*)h_get_int(dw->fields, fi->number))) {
             continue;
         }
         if (!fi_omit_norms(fi)) {
@@ -5105,7 +5444,7 @@ static void dw_flush(DocWriter *dw)
     tiw_close(tiw);
     skip_buf_destroy(skip_buf);
     dw_flush_streams(dw);
-} 
+}
 
 DocWriter *dw_open(IndexWriter *iw, SegmentInfo *si)
 {
@@ -5130,7 +5469,7 @@ DocWriter *dw_open(IndexWriter *iw, SegmentInfo *si)
     dw->skip_interval       = iw->config.skip_interval;
     dw->max_field_length    = iw->config.max_field_length;
     dw->max_buffered_docs   = iw->config.max_buffered_docs;
-    
+
     dw->offsets             = ALLOC_AND_ZERO_N(Offset, DW_OFFSET_INIT_CAPA);
     dw->offsets_size        = 0;
     dw->offsets_capa        = DW_OFFSET_INIT_CAPA;
@@ -5147,7 +5486,7 @@ void dw_new_segment(DocWriter *dw, SegmentInfo *si)
 
 void dw_close(DocWriter *dw)
 {
-    if (dw->doc_num) { 
+    if (dw->doc_num) {
         dw_flush(dw);
     }
     if (dw->fw) {
@@ -5162,7 +5501,7 @@ void dw_close(DocWriter *dw)
 
 FieldInverter *dw_get_fld_inv(DocWriter *dw, FieldInfo *fi)
 {
-    FieldInverter *fld_inv = h_get_int(dw->fields, fi->number);
+    FieldInverter *fld_inv = (FieldInverter*)h_get_int(dw->fields, fi->number);
 
     if (!fld_inv) {
         fld_inv = fld_inv_new(dw, fi);
@@ -5172,31 +5511,33 @@ FieldInverter *dw_get_fld_inv(DocWriter *dw, FieldInfo *fi)
 }
 
 static void dw_add_posting(MemoryPool *mp,
-                           HashTable *curr_plists,
-                           HashTable *fld_plists,
+                           Hash *curr_plists,
+                           Hash *fld_plists,
                            int doc_num,
                            const char *text,
                            int len,
                            int pos)
 {
-    HashEntry *pl_he = h_set_ext(curr_plists, text);
-    if (pl_he->value) {
-        pl_add_occ(mp, pl_he->value, pos);
-    }
-    else {
-        HashEntry *fld_pl_he = h_set_ext(fld_plists, text);
-        PostingList *pl = fld_pl_he->value;
+    HashEntry *pl_he;
+    if (h_set_ext(curr_plists, text, &pl_he)) {
         Posting *p =  p_new(mp, doc_num, pos);
-        if (!pl) {
-            pl = fld_pl_he->value = pl_new(mp, text, len, p);
+        HashEntry *fld_pl_he;
+        PostingList *pl;
+
+        if (h_set_ext(fld_plists, text, &fld_pl_he)) {
+            fld_pl_he->value = pl = pl_new(mp, text, len, p);
             pl_he->key = fld_pl_he->key = (char *)pl->term;
         }
         else {
+            pl = (PostingList *)fld_pl_he->value;
             pl_add_posting(pl, p);
             pl_he->key = (char *)pl->term;
         }
         pl_he->value = pl;
     }
+    else {
+        pl_add_occ(mp, (PostingList *)pl_he->value, pos);
+    }
 }
 
 static INLINE void dw_add_offsets(DocWriter *dw, int pos, off_t start, off_t end)
@@ -5214,14 +5555,14 @@ static INLINE void dw_add_offsets(DocWriter *dw, int pos, off_t start, off_t end
     dw->offsets_size = pos + 1;
 }
 
-HashTable *dw_invert_field(DocWriter *dw,
+Hash *dw_invert_field(DocWriter *dw,
                            FieldInverter *fld_inv,
                            DocField *df)
 {
     MemoryPool *mp = dw->mp;
     Analyzer *a = dw->analyzer;
-    HashTable *curr_plists = dw->curr_plists;
-    HashTable *fld_plists = fld_inv->plists;
+    Hash *curr_plists = dw->curr_plists;
+    Hash *fld_plists = fld_inv->plists;
     const bool store_offsets = fld_inv->store_offsets;
     int doc_num = dw->doc_num;
     int i;
@@ -5238,6 +5579,11 @@ HashTable *dw_invert_field(DocWriter *dw,
             if (store_offsets) {
                 while (NULL != (tk = ts->next(ts))) {
                     pos += tk->pos_inc;
+                    /* if for some reason pos gets set to some number less
+                     * than 0 the we'll start pos at 0 */
+                    if (pos < 0) {
+                        pos = 0;
+                    }
                     dw_add_posting(mp, curr_plists, fld_plists, doc_num,
                                    tk->text, tk->len, pos);
                     dw_add_offsets(dw, pos,
@@ -5271,7 +5617,7 @@ HashTable *dw_invert_field(DocWriter *dw,
             char *data_ptr = df->data[i];
             if (len > MAX_WORD_SIZE) {
                 len = MAX_WORD_SIZE - 1;
-                data_ptr = memcpy(buf, df->data[i], len);
+                data_ptr = (char *)memcpy(buf, df->data[i], len);
             }
             dw_add_posting(mp, curr_plists, fld_plists, doc_num, data_ptr,
                            len, i);
@@ -5286,7 +5632,7 @@ HashTable *dw_invert_field(DocWriter *dw,
     return curr_plists;
 }
 
-void dw_reset_postings(HashTable *postings)
+void dw_reset_postings(Hash *postings)
 {
     ZEROSET_N(postings->table, HashEntry, postings->mask + 1);
     postings->fill = postings->size = 0;
@@ -5298,7 +5644,7 @@ void dw_add_doc(DocWriter *dw, Document *doc)
     float boost;
     DocField *df;
     FieldInverter *fld_inv;
-    HashTable *postings;
+    Hash *postings;
     FieldInfo *fi;
     const int doc_size = doc->size;
 
@@ -5543,7 +5889,7 @@ static void sm_merge_fields(SegmentMerger *sm)
             end = (off_t)is_read_u64(fdx_in);
         }
         for (j = 0; j < max_doc; j++) {
-            f_u32 tv_idx_offset = is_read_u32(fdx_in);
+            u32 tv_idx_offset = is_read_u32(fdx_in);
             start = end;
             if (j == max_doc - 1) {
                 end = is_length(fdt_in);
@@ -5594,14 +5940,9 @@ static int sm_append_postings(SegmentMerger *sm, SegmentMergeInfo **matches,
                 doc = doc_map[doc]; /* work around deletions */
             }
             doc += base;          /* convert to merged space */
+            assert(doc == 0 || doc > last_doc);
 
-#ifdef DEBUG
-            if (doc && doc <= last_doc) {
-                RAISE(STATE_ERROR, "Docs not ordered, %d < %d", doc, last_doc);
-            }
-#endif
             df++;
-
             if (0 == (df % skip_interval)) {
                 skip_buf_add(skip_buf, last_doc);
             }
@@ -5627,7 +5968,7 @@ static int sm_append_postings(SegmentMerger *sm, SegmentMergeInfo **matches,
 
 static char *sm_cache_term(SegmentMerger *sm, char *term, int term_len)
 {
-    term = memcpy(sm->term_buf + sm->term_buf_ptr, term, term_len + 1);
+    term = (char *)memcpy(sm->term_buf + sm->term_buf_ptr, term, term_len + 1);
     sm->term_buf_ptr += term_len + 1;
     if (sm->term_buf_ptr > sm->term_buf_size) {
         sm->term_buf_ptr = 0;
@@ -5688,14 +6029,14 @@ static void sm_merge_term_infos(SegmentMerger *sm)
                }printf("\n\n");
                */
             match_size = 0;     /* pop matching terms */
-            matches[0] = pq_pop(sm->queue);
+            matches[0] = (SegmentMergeInfo *)pq_pop(sm->queue);
             match_size++;
             term = matches[0]->term;
-            top = pq_top(sm->queue);
+            top = (SegmentMergeInfo *)pq_top(sm->queue);
             while ((NULL != top) && (0 == strcmp(term, top->term))) {
-                matches[match_size] = pq_pop(sm->queue);
+                matches[match_size] = (SegmentMergeInfo *)pq_pop(sm->queue);
                 match_size++;
-                top = pq_top(sm->queue);
+                top = (SegmentMergeInfo *)pq_top(sm->queue);
             }
 
             /* printf(">%s:%s<\n", matches[0]->tb->field, matches[0]->tb->text); */
@@ -5849,7 +6190,7 @@ int iw_doc_count(IndexWriter *iw)
 #define MOVE_TO_COMPOUND_DIR(file_name)\
     deleter_queue_file(dlr, file_name);\
     cw_add_file(cw, file_name)
-    
+
 static void iw_create_compound_file(Store *store, FieldInfos *fis,
                                     SegmentInfo *si, char *cfs_file_name,
                                     Deleter *dlr)
@@ -6015,7 +6356,7 @@ void iw_commit(IndexWriter *iw)
     mutex_unlock(&iw->mutex);
 }
 
-void iw_delete_term(IndexWriter *iw, const char *field, const char *term)
+void iw_delete_term(IndexWriter *iw, Symbol field, const char *term)
 {
     int field_num = fis_get_field_num(iw->fis, field);
     if (field_num >= 0) {
@@ -6049,6 +6390,45 @@ void iw_delete_term(IndexWriter *iw, const char *field, const char *term)
     }
 }
 
+void iw_delete_terms(IndexWriter *iw, Symbol field,
+                     char **terms, const int term_cnt)
+{
+    int field_num = fis_get_field_num(iw->fis, field);
+    if (field_num >= 0) {
+        int i;
+        mutex_lock(&iw->mutex);
+        iw_commit_i(iw);
+        do {
+            SegmentInfos *sis = iw->sis;
+            const int seg_cnt = sis->size;
+            bool did_delete = false;
+            for (i = 0; i < seg_cnt; i++) {
+                IndexReader *ir = sr_open(sis, iw->fis, i, false);
+                TermDocEnum *tde = ir->term_docs(ir);
+                int j;
+                for (j = 0 ; j < term_cnt; j++) {
+                    const char *term = terms[j];
+                    ir->deleter = iw->deleter;
+                    stde_seek(tde, field_num, term);
+                    while (tde->next(tde)) {
+                        did_delete = true;
+                        sr_delete_doc_i(ir, STDE(tde)->doc_num);
+                    }
+                }
+                tde_destroy(tde);
+                sr_commit_i(ir);
+                ir_close(ir);
+            }
+            if (did_delete) {
+                mutex_lock(&iw->store->mutex);
+                sis_write(iw->sis, iw->store, iw->deleter);
+                mutex_unlock(&iw->store->mutex);
+            }
+        } while (0);
+        mutex_unlock(&iw->mutex);
+    }
+}
+
 static void iw_optimize_i(IndexWriter *iw)
 {
     int min_segment;
@@ -6070,7 +6450,7 @@ void iw_optimize(IndexWriter *iw)
     mutex_lock(&iw->mutex);
     iw_optimize_i(iw);
     mutex_unlock(&iw->mutex);
-} 
+}
 
 void iw_close(IndexWriter *iw)
 {
@@ -6094,7 +6474,7 @@ void iw_close(IndexWriter *iw)
     free(iw);
 }
 
-IndexWriter *iw_open(Store *store, volatile Analyzer *analyzer,
+IndexWriter *iw_open(Store *store, Analyzer *volatile analyzer,
                      const Config *config)
 {
     IndexWriter *iw = ALLOC_AND_ZERO(IndexWriter);
@@ -6174,7 +6554,7 @@ static void iw_cp_fields(IndexWriter *iw, SegmentReader *sr,
         int i;
         const int max_doc = sr_max_doc(IR(sr));
         for (i = 0; i < max_doc; i++) {
-            int j;
+            int j, data_len = 0;
             const int field_cnt = is_read_vint(fdt_in);
             int tv_cnt;
             off_t doc_start_ptr = os_pos(fdt_out);
@@ -6186,7 +6566,6 @@ static void iw_cp_fields(IndexWriter *iw, SegmentReader *sr,
                 int k;
                 const int field_num = map[is_read_vint(fdt_in)];
                 const int df_size = is_read_vint(fdt_in);
-                int data_len = 0;
                 os_write_vint(fdt_out, field_num);
                 os_write_vint(fdt_out, df_size);
                 /* sum total lengths of DocField */
@@ -6196,18 +6575,18 @@ static void iw_cp_fields(IndexWriter *iw, SegmentReader *sr,
                     os_write_vint(fdt_out, flen);
                     data_len +=  flen + 1;
                 }
-                is2os_copy_bytes(fdt_in, fdt_out, data_len);
             }
+            is2os_copy_bytes(fdt_in, fdt_out, data_len);
 
             /* Write TermVectors */
             /* write TVs up to TV index */
             is2os_copy_bytes(fdt_in, fdt_out,
                              (int)(is_read_u64(fdx_in)
-                                   + (f_u64)is_read_u32(fdx_in)
-                                   - (f_u64)is_pos(fdt_in)));
+                                   + (u64)is_read_u32(fdx_in)
+                                   - (u64)is_pos(fdt_in)));
 
             /* Write TV index pos */
-            os_write_u32(fdx_out, (f_u32)(os_pos(fdt_out) - doc_start_ptr));
+            os_write_u32(fdx_out, (u32)(os_pos(fdt_out) - doc_start_ptr));
             tv_cnt = is_read_vint(fdt_in);
             os_write_vint(fdt_out, tv_cnt);
             for (j = 0; j < tv_cnt; j++) {
@@ -6242,7 +6621,7 @@ static void iw_cp_terms(IndexWriter *iw, SegmentReader *sr,
     tix_out = store_out->new_output(store_out, file_name);
     sprintf(file_name, "%s.tix", sr_segment);
     tix_in = store_in->open_input(store_in, file_name);
-    
+
     sprintf(file_name, "%s.tis", segment);
     tis_out = store_out->new_output(store_out, file_name);
     sprintf(file_name, "%s.tis", sr_segment);
@@ -6371,7 +6750,7 @@ static void iw_add_segment(IndexWriter *iw, SegmentReader *sr)
         FieldInfo *fi = sub_fis->fields[j];
         FieldInfo *new_fi = fis_get_field(fis, fi->name);
         if (NULL == new_fi) {
-            new_fi = fi_new(fi->name, 0, 0, 0);
+            new_fi = fi_new(fi->name, STORE_NO, INDEX_NO, TERM_VECTOR_NO);
             new_fi->bits = fi->bits;
             fis_add_field(fis, new_fi);
         }