isomorfeus-ferret 0.12.7 → 0.13.0

data/ext/isomorfeus_ferret_ext/frt_index.c

@@ -8,6 +8,11 @@
 #include <ctype.h>
 #include "brotli_decode.h"
 #include "brotli_encode.h"
+#include "bzlib.h"
+#include "lz4frame.h"
+
+#undef close
+#undef read
 
 extern void frt_micro_sleep(const int micro_seconds);
 
@@ -41,8 +46,9 @@ static char *ste_next(FrtTermEnum *te);
 #define FORMAT 0
 #define SEGMENTS_GEN_FILE_NAME "segments"
 #define MAX_EXT_LEN 10
-#define COMPRESSION_BUFFER_SIZE 16348
-#define COMPRESSION_LEVEL 9
+#define FRT_COMPRESSION_BUFFER_SIZE 16348
+#define FRT_BROTLI_COMPRESSION_LEVEL 4
+#define FRT_BZIP_COMPRESSION_LEVEL 9
 
 /* *** Must be three characters *** */
 static const char *INDEX_EXTENSIONS[] = {
@@ -103,29 +109,22 @@ static frt_u64 str36_to_u64(char *p)
  * @param ext extension of the filename (including .)
  * @param gen generation
  */
-char *frt_fn_for_generation(char *buf,
-                        const char *base,
-                        const char *ext,
-                        frt_i64 gen)
-{
+char *frt_fn_for_generation(char *buf, const char *base, const char *ext, frt_i64 gen) {
     if (-1 == gen) {
         return NULL;
-    }
-    else {
+    } else {
         char b[FRT_SEGMENT_NAME_MAX_LENGTH];
         char *u = u64_to_str36(b, FRT_SEGMENT_NAME_MAX_LENGTH, (frt_u64)gen);
         if (ext == NULL) {
             sprintf(buf, "%s_%s", base, u);
-        }
-        else {
+        } else {
             sprintf(buf, "%s_%s.%s", base, u, ext);
         }
         return buf;
     }
 }
 
-static char *segfn_for_generation(char *buf, frt_u64 generation)
-{
+static char *segfn_for_generation(char *buf, frt_u64 generation) {
     char b[FRT_SEGMENT_NAME_MAX_LENGTH];
     char *u = u64_to_str36(b, FRT_SEGMENT_NAME_MAX_LENGTH, generation);
     sprintf(buf, FRT_SEGMENTS_FILE_NAME"_%s", u);
@@ -203,8 +202,7 @@ FrtCacheObject *frt_co_create(FrtHash *ref_tab1, FrtHash *ref_tab2,
     return self;
 }
 
-FrtHash *frt_co_hash_create()
-{
+FrtHash *frt_co_hash_create(void) {
     return frt_h_new(&co_hash, &co_eq, (frt_free_ft)NULL, (frt_free_ft)&co_destroy);
 }
 
@@ -214,22 +212,33 @@ FrtHash *frt_co_hash_create()
  *
  ****************************************************************************/
 
-static void fi_set_store(FrtFieldInfo *fi, int store)
-{
+static void fi_set_store(FrtFieldInfo *fi, FrtStoreValue store) {
     switch (store) {
         case FRT_STORE_NO:
             break;
         case FRT_STORE_YES:
             fi->bits |= FRT_FI_IS_STORED_BM;
             break;
-        case FRT_STORE_COMPRESS:
-            fi->bits |= FRT_FI_IS_COMPRESSED_BM | FRT_FI_IS_STORED_BM;
+    }
+}
+
+static void fi_set_compression(FrtFieldInfo *fi, FrtCompressionType compression) {
+    switch (compression) {
+        case FRT_COMPRESSION_NONE:
+            break;
+        case FRT_COMPRESSION_BROTLI:
+            fi->bits |= FRT_FI_IS_COMPRESSED_BM | FRT_FI_COMPRESSION_BROTLI_BM;
+            break;
+        case FRT_COMPRESSION_BZ2:
+            fi->bits |= FRT_FI_IS_COMPRESSED_BM | FRT_FI_COMPRESSION_BZ2_BM;
+            break;
+        case FRT_COMPRESSION_LZ4:
+            fi->bits |= FRT_FI_IS_COMPRESSED_BM | FRT_FI_COMPRESSION_LZ4_BM;
             break;
     }
 }
 
-static void fi_set_index(FrtFieldInfo *fi, int index)
-{
+static void fi_set_index(FrtFieldInfo *fi, FrtIndexValue index) {
     switch (index) {
         case FRT_INDEX_NO:
             break;
@@ -249,8 +258,7 @@ static void fi_set_index(FrtFieldInfo *fi, int index)
     }
 }
 
-static void fi_set_term_vector(FrtFieldInfo *fi, int term_vector)
-{
+static void fi_set_term_vector(FrtFieldInfo *fi, FrtTermVectorValue term_vector) {
     switch (term_vector) {
         case FRT_TERM_VECTOR_NO:
             break;
@@ -270,33 +278,40 @@ static void fi_set_term_vector(FrtFieldInfo *fi, int term_vector)
     }
 }
 
-static void fi_check_params(int store, int index, int term_vector)
-{
+static void fi_check_params(FrtStoreValue store, FrtCompressionType compression, FrtIndexValue index, FrtTermVectorValue term_vector) {
     (void)store;
     if ((index == FRT_INDEX_NO) && (term_vector != FRT_TERM_VECTOR_NO)) {
-        FRT_RAISE(FRT_ARG_ERROR,
-              "You can't store the term vectors of an unindexed field");
+        FRT_RAISE(FRT_ARG_ERROR, "You can't store the term vectors of an unindexed field.");
+    }
+    if ((compression != FRT_COMPRESSION_NONE) && (store == FRT_STORE_NO)) {
+        FRT_RAISE(FRT_ARG_ERROR, "Field must be stored for compression to be useful.");
     }
 }
 
-FrtFieldInfo *frt_fi_new(FrtSymbol name,
-                  FrtStoreValue store,
-                  FrtIndexValue index,
-                  FrtTermVectorValue term_vector)
-{
-    FrtFieldInfo *fi = FRT_ALLOC(FrtFieldInfo);
+FrtFieldInfo *frt_fi_alloc(void) {
+    return FRT_ALLOC(FrtFieldInfo);
+}
+
+FrtFieldInfo *frt_fi_init(FrtFieldInfo *fi, ID name, FrtStoreValue store, FrtCompressionType compression, FrtIndexValue index, FrtTermVectorValue term_vector) {
     assert(NULL != name);
-    fi_check_params(store, index, term_vector);
+    fi_check_params(store, compression, index, term_vector);
     fi->name = name;
     fi->boost = 1.0f;
     fi->bits = 0;
     fi_set_store(fi, store);
+    fi_set_compression(fi, compression);
     fi_set_index(fi, index);
     fi_set_term_vector(fi, term_vector);
     fi->ref_cnt = 1;
+    fi->rfi = Qnil;
     return fi;
 }
 
+FrtFieldInfo *frt_fi_new(ID name, FrtStoreValue store, FrtCompressionType compression, FrtIndexValue index, FrtTermVectorValue term_vector) {
+    FrtFieldInfo *fi = frt_fi_alloc();
+    return frt_fi_init(fi, name, store, compression, index, term_vector);
+}
+
 void frt_fi_deref(FrtFieldInfo *fi)
 {
     if (0 == --(fi->ref_cnt)) {
@@ -304,6 +319,22 @@ void frt_fi_deref(FrtFieldInfo *fi)
     }
 }
 
+FrtCompressionType frt_fi_get_compression(FrtFieldInfo *fi) {
+    if (fi_is_compressed(fi)) {
+        if (fi_is_compressed_brotli(fi)) {
+            return FRT_COMPRESSION_BROTLI;
+        } else if (fi_is_compressed_bz2(fi)) {
+            return FRT_COMPRESSION_BZ2;
+        } else if (fi_is_compressed_lz4(fi)) {
+            return FRT_COMPRESSION_LZ4;
+        } else {
+            return FRT_COMPRESSION_BROTLI;
+        }
+    } else {
+        return FRT_COMPRESSION_NONE;
+    }
+}
+
 char *frt_fi_to_s(FrtFieldInfo *fi)
 {
     const char *fi_name = rb_id2name(fi->name);
@@ -333,24 +364,31 @@ char *frt_fi_to_s(FrtFieldInfo *fi)
  *
  ****************************************************************************/
 
-FrtFieldInfos *frt_fis_new(FrtStoreValue store, FrtIndexValue index,
-                    FrtTermVectorValue term_vector)
-{
-    FrtFieldInfos *fis = FRT_ALLOC(FrtFieldInfos);
-    fi_check_params(store, index, term_vector);
+FrtFieldInfos *frt_fis_alloc(void) {
+    return FRT_ALLOC(FrtFieldInfos);
+}
+
+FrtFieldInfos *frt_fis_init(FrtFieldInfos *fis, FrtStoreValue store, FrtCompressionType compression, FrtIndexValue index, FrtTermVectorValue term_vector) {
+    fi_check_params(store, compression, index, term_vector);
     fis->field_dict = frt_h_new_ptr((frt_free_ft)&frt_fi_deref);
     fis->size = 0;
     fis->capa = FIELD_INFOS_INIT_CAPA;
     fis->fields = FRT_ALLOC_N(FrtFieldInfo *, fis->capa);
     fis->store = store;
+    fis->compression = compression;
     fis->index = index;
     fis->term_vector = term_vector;
     fis->ref_cnt = 1;
+    fis->rfis = Qnil;
     return fis;
 }
 
-FrtFieldInfo *frt_fis_add_field(FrtFieldInfos *fis, FrtFieldInfo *fi)
-{
+FrtFieldInfos *frt_fis_new(FrtStoreValue store, FrtCompressionType compression, FrtIndexValue index, FrtTermVectorValue term_vector) {
+    FrtFieldInfos *fis = frt_fis_alloc();
+    return frt_fis_init(fis, store, compression, index, term_vector);
+}
+
+FrtFieldInfo *frt_fis_add_field(FrtFieldInfos *fis, FrtFieldInfo *fi) {
     if (fis->size == fis->capa) {
         fis->capa <<= 1;
         FRT_REALLOC_N(fis->fields, FrtFieldInfo *, fis->capa);
@@ -364,23 +402,20 @@ FrtFieldInfo *frt_fis_add_field(FrtFieldInfos *fis, FrtFieldInfo *fi)
     return fi;
 }
 
-FrtFieldInfo *frt_fis_get_field(FrtFieldInfos *fis, FrtSymbol name)
-{
+FrtFieldInfo *frt_fis_get_field(FrtFieldInfos *fis, ID name) {
     return (FrtFieldInfo *)frt_h_get(fis->field_dict, (void *)name);
 }
 
-int frt_fis_get_field_num(FrtFieldInfos *fis, FrtSymbol name)
-{
+int frt_fis_get_field_num(FrtFieldInfos *fis, ID name) {
     FrtFieldInfo *fi = (FrtFieldInfo *)frt_h_get(fis->field_dict, (void *)name);
     if (fi) { return fi->number; }
     else { return -1; }
 }
 
-FrtFieldInfo *frt_fis_get_or_add_field(FrtFieldInfos *fis, FrtSymbol name)
-{
+FrtFieldInfo *frt_fis_get_or_add_field(FrtFieldInfos *fis, ID name) {
     FrtFieldInfo *fi = (FrtFieldInfo *)frt_h_get(fis->field_dict, (void *)name);
     if (!fi) {
-        fi = (FrtFieldInfo*)frt_fi_new(name, fis->store, fis->index, fis->term_vector);
+        fi = (FrtFieldInfo*)frt_fi_new(name, fis->store, fis->compression, fis->index, fis->term_vector);
         frt_fis_add_field(fis, fi);
     }
     return fi;
@@ -392,16 +427,14 @@ FrtFieldInfos *frt_fis_read(FrtInStream *is)
     char *field_name;
     FRT_TRY
         do {
-            FrtStoreValue store_val;
-            FrtIndexValue index_val;
             FrtTermVectorValue term_vector_val;
             volatile int i;
             union { frt_u32 i; float f; } tmp;
             FrtFieldInfo *volatile fi;
-            store_val = (FrtStoreValue)frt_is_read_vint(is);
-            index_val = (FrtIndexValue)frt_is_read_vint(is);
+            FrtStoreValue store_val = (FrtStoreValue)frt_is_read_vint(is);
+            FrtIndexValue index_val = (FrtIndexValue)frt_is_read_vint(is);
             term_vector_val = (FrtTermVectorValue)frt_is_read_vint(is);
-            fis = frt_fis_new(store_val, index_val, term_vector_val);
+            fis = frt_fis_new(store_val, FRT_COMPRESSION_NONE, index_val, term_vector_val); // TODO compression, read from store?
             for (i = frt_is_read_vint(is); i > 0; i--) {
                 fi = FRT_ALLOC_AND_ZERO(FrtFieldInfo);
                 FRT_TRY
@@ -803,8 +836,7 @@ static char *sis_next_seg_file_name(char *buf, FrtStore *store)
 
 #define GEN_FILE_RETRY_COUNT 10
 #define GEN_LOOK_AHEAD_COUNT 10
-static void sis_find_segments_file(FrtStore *store, FindSegmentsFile *fsf,
-                            void (*run)(FrtStore *store, FindSegmentsFile *fsf))
+static void sis_find_segments_file(FrtStore *store, FindSegmentsFile *fsf, void (*run)(FrtStore *store, FindSegmentsFile *fsf, FrtIndexReader *ir), FrtIndexReader *ir)
 {
     volatile int i;
     volatile int gen_look_ahead_count = 0;
@@ -911,7 +943,7 @@ static void sis_find_segments_file(FrtStore *store, FindSegmentsFile *fsf,
         last_gen = gen;
         FRT_TRY
             fsf->generation = gen;
-            run(store, fsf);
+            run(store, fsf, ir);
             FRT_RETURN_EARLY();
             return;
         case FRT_IO_ERROR: case FRT_FILE_NOT_FOUND_ERROR: case FRT_EOF_ERROR:
@@ -957,7 +989,7 @@ static void sis_find_segments_file(FrtStore *store, FindSegmentsFile *fsf,
                      * prevSegmentFileName + "'" */
                     FRT_TRY
                         fsf->generation = gen - 1;
-                        run(store, fsf);
+                        run(store, fsf, ir);
                         /* TODO:LOG "success on fallback " +
                          * prev_seg_file_name */
 
@@ -1040,7 +1072,7 @@ void frt_sis_del_from_to(FrtSegmentInfos *sis, int from, int to)
     }
 }
 
-static void frt_sis_read_i(FrtStore *store, FindSegmentsFile *fsf)
+static void frt_sis_read_i(FrtStore *store, FindSegmentsFile *fsf, FrtIndexReader *ir_)
 {
     int seg_cnt;
     int i;
@@ -1079,7 +1111,7 @@ static void frt_sis_read_i(FrtStore *store, FindSegmentsFile *fsf)
 FrtSegmentInfos *frt_sis_read(FrtStore *store)
 {
     FindSegmentsFile fsf;
-    sis_find_segments_file(store, &fsf, &frt_sis_read_i);
+    sis_find_segments_file(store, &fsf, &frt_sis_read_i, NULL);
     return fsf.ret.sis;
 }
 
@@ -1119,7 +1151,7 @@ void frt_sis_write(FrtSegmentInfos *sis, FrtStore *store, FrtDeleter *deleter)
     }
 }
 
-static void frt_sis_read_ver_i(FrtStore *store, FindSegmentsFile *fsf)
+static void frt_sis_read_ver_i(FrtStore *store, FindSegmentsFile *fsf, FrtIndexReader *ir_)
 {
     FrtInStream *is;
     frt_u64 version;
@@ -1142,7 +1174,7 @@ static void frt_sis_read_ver_i(FrtStore *store, FindSegmentsFile *fsf)
 frt_u64 frt_sis_read_current_version(FrtStore *store)
 {
     FindSegmentsFile fsf;
-    sis_find_segments_file(store, &fsf, &frt_sis_read_ver_i);
+    sis_find_segments_file(store, &fsf, &frt_sis_read_ver_i, NULL);
     return fsf.ret.uint64;
 }
 
@@ -1152,18 +1184,17 @@ frt_u64 frt_sis_read_current_version(FrtStore *store)
  *
  ****************************************************************************/
 
-static FrtLazyDocField *lazy_df_new(FrtSymbol name, const int size, bool is_compressed)
-{
+static FrtLazyDocField *lazy_df_new(ID name, const int size, FrtCompressionType compression) {
     FrtLazyDocField *self = FRT_ALLOC(FrtLazyDocField);
     self->name = name;
     self->size = size;
     self->data = FRT_ALLOC_AND_ZERO_N(FrtLazyDocFieldData, size);
-    self->is_compressed = is_compressed;
+    self->compression = compression;
+    self->decompressed = false;
     return self;
 }
 
-static void lazy_df_destroy(FrtLazyDocField *self)
-{
+static void lazy_df_destroy(FrtLazyDocField *self) {
     int i;
     for (i = self->size - 1; i >= 0; i--) {
         if (self->data[i].text) {
@@ -1174,16 +1205,14 @@ static void lazy_df_destroy(FrtLazyDocField *self)
     free(self);
 }
 
-static void comp_raise()
-{
+static void comp_raise(void) {
     FRT_RAISE(EXCEPTION, "Compression error");
 }
 
-static char *is_read_compressed_bytes(FrtInStream *is, int compressed_len, int *len)
-{
+static char *is_read_brotli_compressed_bytes(FrtInStream *is, int compressed_len, int *len) {
     int buf_out_idx = 0;
     int read_len;
-    frt_uchar buf_in[COMPRESSION_BUFFER_SIZE];
+    frt_uchar buf_in[FRT_COMPRESSION_BUFFER_SIZE];
     const frt_uchar *next_in;
     size_t available_in;
     frt_uchar *buf_out = NULL;
@@ -1195,20 +1224,20 @@ static char *is_read_compressed_bytes(FrtInStream *is, int compressed_len, int *
     if (!b_state) { comp_raise(); return NULL; }
 
     do {
-        read_len = compressed_len > COMPRESSION_BUFFER_SIZE ? COMPRESSION_BUFFER_SIZE : compressed_len;
+        read_len = (compressed_len > FRT_COMPRESSION_BUFFER_SIZE) ? FRT_COMPRESSION_BUFFER_SIZE : compressed_len;
         frt_is_read_bytes(is, buf_in, read_len);
         compressed_len -= read_len;
         available_in = read_len;
         next_in = buf_in;
-        available_out = COMPRESSION_BUFFER_SIZE;
+        available_out = FRT_COMPRESSION_BUFFER_SIZE;
         do {
-            FRT_REALLOC_N(buf_out, frt_uchar, buf_out_idx + COMPRESSION_BUFFER_SIZE);
+            FRT_REALLOC_N(buf_out, frt_uchar, buf_out_idx + FRT_COMPRESSION_BUFFER_SIZE);
             next_out = buf_out + buf_out_idx;
             b_result = BrotliDecoderDecompressStream(b_state,
                 &available_in, &next_in,
                 &available_out, &next_out, NULL);
             if (b_result == BROTLI_DECODER_RESULT_ERROR) { comp_raise(); return NULL; }
-            buf_out_idx += COMPRESSION_BUFFER_SIZE - available_out;
+            buf_out_idx += FRT_COMPRESSION_BUFFER_SIZE - available_out;
         } while (b_result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT);
     } while (b_result != BROTLI_DECODER_RESULT_SUCCESS && compressed_len > 0);
 
@@ -1220,16 +1249,180 @@ static char *is_read_compressed_bytes(FrtInStream *is, int compressed_len, int *
     return (char *)buf_out;
 }
 
-char *frt_lazy_df_get_data(FrtLazyDocField *self, int i)
-{
+static void zraise(int ret) {
+    switch (ret) {
+    case BZ_IO_ERROR:
+        if (ferror(stdin))
+            FRT_RAISE(FRT_IO_ERROR, "bzlib: error reading stdin");
+        if (ferror(stdout))
+            FRT_RAISE(FRT_IO_ERROR, "bzlib: error writing stdout");
+        break;
+    case BZ_CONFIG_ERROR:
+        FRT_RAISE(FRT_IO_ERROR, "bzlib: system configuration error");
+        break;
+    case BZ_SEQUENCE_ERROR: /* shouldn't occur if code is correct */
+        FRT_RAISE(FRT_IO_ERROR, "bzlib: !!BUG!! sequence error");
+        break;
+    case BZ_PARAM_ERROR:    /* shouldn't occur if code is correct */
+        FRT_RAISE(FRT_IO_ERROR, "bzlib: !!BUG!! parameter error");
+        break;
+    case BZ_MEM_ERROR:
+        FRT_RAISE(FRT_IO_ERROR, "bzlib: memory error");
+        break;
+    case BZ_DATA_ERROR:
+        FRT_RAISE(FRT_IO_ERROR, "bzlib: data integrity check error");
+        break;
+    case BZ_DATA_ERROR_MAGIC:
+        FRT_RAISE(FRT_IO_ERROR, "bzlib: data integrity check - non-matching magic");
+        break;
+    case BZ_UNEXPECTED_EOF:
+        FRT_RAISE(FRT_IO_ERROR, "bzlib: unexpected end-of-file");
+        break;
+    case BZ_OUTBUFF_FULL:
+        FRT_RAISE(FRT_IO_ERROR, "bzlib: output buffer full");
+        break;
+    default:
+        FRT_RAISE(FRT_EXCEPTION, "bzlib: unknown error");
+    }
+}
+
+static char *is_read_bz2_compressed_bytes(FrtInStream *is, int compressed_len, int *len) {
+    int buf_out_idx = 0, ret, read_len;
+    char *buf_out = NULL;
+    char buf_in[FRT_COMPRESSION_BUFFER_SIZE];
+    bz_stream zstrm;
+    zstrm.bzalloc = NULL;
+    zstrm.bzfree  = NULL;
+    zstrm.opaque  = NULL;
+    zstrm.next_in = NULL;
+    zstrm.avail_in = 0;
+    if ((ret = BZ2_bzDecompressInit(&zstrm, 0, 0)) != BZ_OK) zraise(ret);
+
+    do {
+        read_len = (compressed_len > FRT_COMPRESSION_BUFFER_SIZE) ? FRT_COMPRESSION_BUFFER_SIZE : compressed_len;
+        frt_is_read_bytes(is, (frt_uchar *)buf_in, read_len);
+        compressed_len -= read_len;
+        zstrm.avail_in = read_len;
+        zstrm.next_in = buf_in;
+        zstrm.avail_out = FRT_COMPRESSION_BUFFER_SIZE;
+
+        do {
+            REALLOC_N(buf_out, char, buf_out_idx + FRT_COMPRESSION_BUFFER_SIZE);
+            zstrm.next_out = buf_out + buf_out_idx;
+            ret = BZ2_bzDecompress(&zstrm);
+            assert(ret != BZ_SEQUENCE_ERROR);  /* state not clobbered */
+            if (ret != BZ_OK && ret != BZ_STREAM_END) {
+                (void)BZ2_bzDecompressEnd(&zstrm);
+                zraise(ret);
+            }
+            buf_out_idx += FRT_COMPRESSION_BUFFER_SIZE - zstrm.avail_out;
+        } while (zstrm.avail_out == 0);
+    } while (ret != BZ_STREAM_END && compressed_len != 0);
+
+    (void)BZ2_bzDecompressEnd(&zstrm);
+
+    FRT_REALLOC_N(buf_out, char, buf_out_idx + 1);
+    buf_out[buf_out_idx] = '\0';
+
+    *len = buf_out_idx;
+    return (char *)buf_out;
+}
+
+static char *is_read_lz4_compressed_bytes(FrtInStream *is, int compressed_len, int *length) {
+    frt_uchar buf_in[FRT_COMPRESSION_BUFFER_SIZE];
+    char *buf_out = NULL;
+    int dc_length = 0;
+    LZ4F_dctx *dctx;
+    LZ4F_frameInfo_t frame_info;
+    LZ4F_errorCode_t dctx_status = LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION);
+    if (LZ4F_isError(dctx_status)) { *length = -1; return NULL; }
+
+    /* header and buffer */
+    int read_length = (compressed_len > FRT_COMPRESSION_BUFFER_SIZE) ? FRT_COMPRESSION_BUFFER_SIZE : compressed_len;
+    frt_is_read_bytes(is, buf_in, read_length);
+    compressed_len -= read_length;
+
+    size_t consumed_size = read_length;
+    size_t res = LZ4F_getFrameInfo(dctx, &frame_info, buf_in, &consumed_size);
+    if (LZ4F_isError(res)) { *length = -1; return NULL; }
+    size_t buf_out_length;
+    switch(frame_info.blockSizeID) {
+        case LZ4F_default:
+        case LZ4F_max64KB:
+            buf_out_length = 1 << 16;
+            break;
+        case LZ4F_max256KB:
+            buf_out_length = 1 << 18;
+            break;
+        case LZ4F_max1MB:
+            buf_out_length = 1 << 20;
+            break;
+        case LZ4F_max4MB:
+            buf_out_length = 1 << 22;
+            break;
+        default:
+            buf_out_length = 0;
+    }
+
+    res = 1;
+    int first_chunk = 1;
+
+    /* decompress data */
+    while (res != 0) {
+        if (!first_chunk) {
+            read_length = (compressed_len > FRT_COMPRESSION_BUFFER_SIZE) ? FRT_COMPRESSION_BUFFER_SIZE : compressed_len;
+            frt_is_read_bytes(is, buf_in, read_length);
+            compressed_len -= read_length;
+            consumed_size = 0;
+        }
+        first_chunk = 0;
+
+        char *src = (char *)(buf_in + consumed_size);
+        char *src_end = (char *)buf_in + read_length;
+
+        while (src < src_end && res != 0){
+            size_t dest_length = buf_out_length;
+            size_t consumed_size = read_length;
+            FRT_REALLOC_N(buf_out, char, dc_length + buf_out_length);
+            res = LZ4F_decompress(dctx, buf_out + dc_length, &dest_length, src, &consumed_size, NULL);
+            if (LZ4F_isError(res)) { *length = -1; return NULL; }
+            dc_length += dest_length;
+            src = src + consumed_size;
+        }
+    }
+
+    /* finish up */
+    LZ4F_freeDecompressionContext(dctx);
+
+    FRT_REALLOC_N(buf_out, char, dc_length + 1);
+    buf_out[dc_length] = '\0';
+
+    *length = dc_length;
+    return buf_out;
+}
+
+static char *is_read_compressed_bytes(FrtInStream *is, int compressed_len, int *len, FrtCompressionType compression) {
+    switch (compression) {
+        case FRT_COMPRESSION_BROTLI:
+            return is_read_brotli_compressed_bytes(is, compressed_len, len);
+        case FRT_COMPRESSION_BZ2:
+            return is_read_bz2_compressed_bytes(is, compressed_len, len);
+        case FRT_COMPRESSION_LZ4:
+            return is_read_lz4_compressed_bytes(is, compressed_len, len);
+        default:
+            return NULL;
+    }
+}
+
+char *frt_lazy_df_get_data(FrtLazyDocField *self, int i) {
     char *text = NULL;
     if (i < self->size && i >= 0) {
         text = self->data[i].text;
         if (NULL == text) {
             const int read_len = self->data[i].length + 1;
             frt_is_seek(self->doc->fields_in, self->data[i].start);
-            if (self->is_compressed) {
-                self->data[i].text = text = is_read_compressed_bytes(self->doc->fields_in, read_len, &(self->data[i].length));
+            if (self->data[i].compression != FRT_COMPRESSION_NONE) {
+                self->data[i].text = text = is_read_compressed_bytes(self->doc->fields_in, read_len, &(self->data[i].length), self->data[i].compression);
             } else {
                 self->data[i].text = text = FRT_ALLOC_N(char, read_len);
                 frt_is_read_bytes(self->doc->fields_in, (frt_uchar *)text, read_len);
@@ -1241,9 +1434,8 @@ char *frt_lazy_df_get_data(FrtLazyDocField *self, int i)
     return text;
 }
 
-void frt_lazy_df_get_bytes(FrtLazyDocField *self, char *buf, int start, int len)
-{
-    if (self->is_compressed == 1) {
+void frt_lazy_df_get_bytes(FrtLazyDocField *self, char *buf, int start, int len) {
+    if (self->compression != FRT_COMPRESSION_NONE && !self->decompressed) {
         int i;
         self->len = 0;
         for (i = self->size-1; i >= 0; i--) {
@@ -1251,7 +1443,7 @@ void frt_lazy_df_get_bytes(FrtLazyDocField *self, char *buf, int start, int len)
             self->len += self->data[i].length + 1;
         }
         self->len--; /* each field separated by ' ' but no need to add to end */
-        self->is_compressed = 2;
+        self->decompressed = true;
     }
     if (start < 0 || start >= self->len) {
         FRT_RAISE(FRT_IO_ERROR, "start out of range in LazyDocField#get_bytes. %d "
@@ -1264,7 +1456,7 @@ void frt_lazy_df_get_bytes(FrtLazyDocField *self, char *buf, int start, int len)
         FRT_RAISE(FRT_IO_ERROR, "Tried to read past end of field. Field is only %d "
               "bytes long but tried to read to %d", self->len, start + len);
     }
-    if (self->is_compressed) {
+    if (self->compression != FRT_COMPRESSION_NONE) {
         int cur_start = 0, buf_start = 0, cur_end, i, copy_start, copy_len;
         for (i = 0; i < self->size; i++) {
             cur_end = cur_start + self->data[i].length;
@@ -1328,21 +1520,17 @@ static void lazy_doc_add_field(FrtLazyDoc *self, FrtLazyDocField *lazy_df, int i
     lazy_df->doc = self;
 }
 
-FrtLazyDocField *frt_lazy_doc_get(FrtLazyDoc *self, FrtSymbol field)
-{
+FrtLazyDocField *frt_lazy_doc_get(FrtLazyDoc *self, ID field) {
     return (FrtLazyDocField *)frt_h_get(self->field_dictionary, (void *)field);
 }
 
 /****************************************************************************
- *
  * FrtFieldsReader
- *
  ****************************************************************************/
 
 #define FIELDS_IDX_PTR_SIZE 12
 
-FrtFieldsReader *frt_fr_open(FrtStore *store, const char *segment, FrtFieldInfos *fis)
-{
+FrtFieldsReader *frt_fr_open(FrtStore *store, const char *segment, FrtFieldInfos *fis) {
     FrtFieldsReader *fr = FRT_ALLOC(FrtFieldsReader);
     FrtInStream *fdx_in;
     char file_name[FRT_SEGMENT_NAME_MAX_LENGTH];
@@ -1362,8 +1550,7 @@ FrtFieldsReader *frt_fr_open(FrtStore *store, const char *segment, FrtFieldInfos
     return fr;
 }
 
-FrtFieldsReader *frt_fr_clone(FrtFieldsReader *orig)
-{
+FrtFieldsReader *frt_fr_clone(FrtFieldsReader *orig) {
     FrtFieldsReader *fr = FRT_ALLOC(FrtFieldsReader);
 
     memcpy(fr, orig, sizeof(FrtFieldsReader));
@@ -1373,35 +1560,33 @@ FrtFieldsReader *frt_fr_clone(FrtFieldsReader *orig)
     return fr;
 }
 
-void frt_fr_close(FrtFieldsReader *fr)
-{
+void frt_fr_close(FrtFieldsReader *fr) {
     frt_is_close(fr->fdt_in);
     frt_is_close(fr->fdx_in);
     free(fr);
 }
 
-static FrtDocField *frt_fr_df_new(FrtSymbol name, int size, bool is_compressed)
-{
+static FrtDocField *frt_fr_df_new(ID name, int size, FrtCompressionType compression) {
     FrtDocField *df = FRT_ALLOC(FrtDocField);
     df->name = name;
     df->capa = df->size = size;
     df->data = FRT_ALLOC_N(char *, df->capa);
     df->lengths = FRT_ALLOC_N(int, df->capa);
+    df->encodings = FRT_ALLOC_N(rb_encoding *, df->capa);
     df->destroy_data = true;
     df->boost = 1.0f;
-    df->is_compressed = is_compressed;
+    df->compression = compression;
     return df;
 }
 
-static void frt_fr_read_compressed_fields(FrtFieldsReader *fr, FrtDocField *df)
-{
+static void frt_fr_read_compressed_fields(FrtFieldsReader *fr, FrtDocField *df, FrtCompressionType compression) {
     int i;
     const int df_size = df->size;
     FrtInStream *fdt_in = fr->fdt_in;
 
     for (i = 0; i < df_size; i++) {
         const int compressed_len = df->lengths[i] + 1;
-        df->data[i] = is_read_compressed_bytes(fdt_in, compressed_len, &(df->lengths[i]));
+        df->data[i] = is_read_compressed_bytes(fdt_in, compressed_len, &(df->lengths[i]), compression);
     }
 }
 
@@ -1423,18 +1608,20 @@ FrtDocument *frt_fr_get_doc(FrtFieldsReader *fr, int doc_num)
         const int field_num = frt_is_read_vint(fdt_in);
         FrtFieldInfo *fi = fr->fis->fields[field_num];
         const int df_size = frt_is_read_vint(fdt_in);
-        FrtDocField *df = frt_fr_df_new(fi->name, df_size, fi_is_compressed(fi));
+        FrtDocField *df = frt_fr_df_new(fi->name, df_size, frt_fi_get_compression(fi));
 
         for (j = 0; j < df_size; j++) {
             df->lengths[j] = frt_is_read_vint(fdt_in);
+            df->encodings[j] = rb_enc_from_index(frt_is_read_vint(fdt_in));
+            df->compression = frt_is_read_vint(fdt_in);
         }
 
         frt_doc_add_field(doc, df);
     }
     for (i = 0; i < stored_cnt; i++) {
         FrtDocField *df = doc->fields[i];
-        if (df->is_compressed) {
-            frt_fr_read_compressed_fields(fr, df);
+        if (df->compression != FRT_COMPRESSION_NONE) {
+            frt_fr_read_compressed_fields(fr, df, df->compression);
         } else {
             const int df_size = df->size;
             for (j = 0; j < df_size; j++) {
@@ -1458,31 +1645,37 @@ FrtLazyDoc *frt_fr_get_lazy_doc(FrtFieldsReader *fr, int doc_num)
     FrtLazyDoc *lazy_doc;
     FrtInStream *fdx_in = fr->fdx_in;
     FrtInStream *fdt_in = fr->fdt_in;
+
     frt_is_seek(fdx_in, doc_num * FIELDS_IDX_PTR_SIZE);
     pos = (off_t)frt_is_read_u64(fdx_in);
     frt_is_seek(fdt_in, pos);
     stored_cnt = frt_is_read_vint(fdt_in);
+
     lazy_doc = lazy_doc_new(stored_cnt, fdt_in);
     for (i = 0; i < stored_cnt; i++) {
         FrtFieldInfo *fi = fr->fis->fields[frt_is_read_vint(fdt_in)];
-        const int data_cnt = frt_is_read_vint(fdt_in);
-        FrtLazyDocField *lazy_df = lazy_df_new(fi->name, data_cnt, fi_is_compressed(fi));
+        const int df_size = frt_is_read_vint(fdt_in);
+        FrtLazyDocField *lazy_df = lazy_df_new(fi->name, df_size, frt_fi_get_compression(fi));
         const int field_start = start;
         /* get the starts relative positions this time around */
-        for (j = 0; j < data_cnt; j++) {
+
+        for (j = 0; j < df_size; j++) {
             lazy_df->data[j].start = start;
             start += 1 + (lazy_df->data[j].length = frt_is_read_vint(fdt_in));
+            lazy_df->data[j].encoding = rb_enc_from_index(frt_is_read_vint(fdt_in));
+            lazy_df->data[j].compression = frt_is_read_vint(fdt_in);
         }
+
         lazy_df->len = start - field_start - 1;
         lazy_doc_add_field(lazy_doc, lazy_df, i);
     }
     /* correct the starts to their correct absolute positions */
+    const off_t abs_start = frt_is_pos(fdt_in);
     for (i = 0; i < stored_cnt; i++) {
         FrtLazyDocField *lazy_df = lazy_doc->fields[i];
-        const int data_cnt = lazy_df->size;
-        const off_t start = frt_is_pos(fdt_in);
-        for (j = 0; j < data_cnt; j++) {
-            lazy_df->data[j].start += start;
+        const int df_size = lazy_df->size;
+        for (j = 0; j < df_size; j++) {
+            lazy_df->data[j].start += abs_start;
         }
     }
 
@@ -1660,42 +1853,150 @@ void frt_fw_close(FrtFieldsWriter *fw)
     free(fw);
 }
 
-static int frt_os_write_compressed_bytes(FrtOutStream* out_stream, frt_uchar *data, int length)
-{
-    size_t compressed_len = 0;
+static int frt_os_write_brotli_compressed_bytes(FrtOutStream* out_stream, frt_uchar *data, int length) {
+    size_t compressed_length = 0;
     const frt_uchar *next_in = data;
     size_t available_in = length;
     size_t available_out;
-    frt_uchar compression_buffer[COMPRESSION_BUFFER_SIZE];
+    frt_uchar compression_buffer[FRT_COMPRESSION_BUFFER_SIZE];
     frt_uchar *next_out;
     BrotliEncoderState *b_state = BrotliEncoderCreateInstance(NULL, NULL, NULL);
     if (!b_state) { comp_raise(); return -1; }
 
-    BrotliEncoderSetParameter(b_state, BROTLI_PARAM_QUALITY, COMPRESSION_LEVEL);
+    BrotliEncoderSetParameter(b_state, BROTLI_PARAM_QUALITY, FRT_BROTLI_COMPRESSION_LEVEL);
 
     do {
-        available_out = COMPRESSION_BUFFER_SIZE;
+        available_out = FRT_COMPRESSION_BUFFER_SIZE;
         next_out = compression_buffer;
         if (!BrotliEncoderCompressStream(b_state, BROTLI_OPERATION_FINISH,
             &available_in, &next_in,
-            &available_out, &next_out, &compressed_len)) {
+            &available_out, &next_out, &compressed_length)) {
             BrotliEncoderDestroyInstance(b_state);
             comp_raise();
             return -1;
         }
-        frt_os_write_bytes(out_stream, compression_buffer, COMPRESSION_BUFFER_SIZE - available_out);
+        frt_os_write_bytes(out_stream, compression_buffer, FRT_COMPRESSION_BUFFER_SIZE - available_out);
     } while (!BrotliEncoderIsFinished(b_state));
 
     BrotliEncoderDestroyInstance(b_state);
-    // fprintf(stderr, "Compressed: %i -> %i\n", length, (int)compressed_len);
-    return (int)compressed_len;
+
+    return (int)compressed_length;
 }
 
-void frt_fw_add_doc(FrtFieldsWriter *fw, FrtDocument *doc)
-{
+static int frt_os_write_bz2_compressed_bytes(FrtOutStream* out_stream, frt_uchar *data, int length) {
+    int ret, buf_size, compressed_len = 0;
+    char out_buffer[FRT_COMPRESSION_BUFFER_SIZE];
+    bz_stream zstrm;
+    zstrm.bzalloc = NULL;
+    zstrm.bzfree  = NULL;
+    zstrm.opaque = NULL;
+    if ((ret = BZ2_bzCompressInit(&zstrm, FRT_BZIP_COMPRESSION_LEVEL, 0, 0)) != BZ_OK) zraise(ret);
+
+    zstrm.avail_in = length;
+    zstrm.next_in = (char *)data;
+    zstrm.avail_out = FRT_COMPRESSION_BUFFER_SIZE;
+    zstrm.next_out = out_buffer;
+
+    do {
+        ret = BZ2_bzCompress(&zstrm, BZ_FINISH); /* no bad return value */
+        assert(ret != BZ_SEQUENCE_ERROR);        /* state not clobbered */
+        compressed_len += buf_size = FRT_COMPRESSION_BUFFER_SIZE - zstrm.avail_out;
+        frt_os_write_bytes(out_stream, (frt_uchar *)out_buffer, buf_size);
+    } while (zstrm.avail_out == 0);
+    assert(zstrm.avail_in == 0);       /* all input will be used */
+
+    (void)BZ2_bzCompressEnd(&zstrm);
+    return compressed_len;
+}
+
+static const LZ4F_preferences_t lz4_prefs = {
+    {
+        LZ4F_default,
+        LZ4F_blockLinked,
+        LZ4F_noContentChecksum,
+        LZ4F_frame,
+        0, /* unknown content size */
+        0, /* no dictID */
+        LZ4F_noBlockChecksum
+    },
+    0,
+    1,
+    1,
+    {0,0,0}
+};
+
+static int frt_os_write_lz4_compressed_bytes(FrtOutStream* out_stream, frt_uchar *data, int length) {
+    int compressed_length = 0;
+    int remaining_length = length;
+    size_t ccmp_length = 0;
+    LZ4F_compressionContext_t ctx;
+    size_t out_buf_length = LZ4F_compressBound(FRT_COMPRESSION_BUFFER_SIZE, &lz4_prefs);
+    frt_uchar *out_buf = frt_ecalloc(out_buf_length);
+
+    size_t ctx_creation = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
+    if (LZ4F_isError(ctx_creation)) {
+        compressed_length = -1;
+        goto finish;
+    }
+
+    /* create header */
+    ccmp_length = LZ4F_compressBegin(ctx, out_buf, out_buf_length, &lz4_prefs);
+    if (LZ4F_isError(ccmp_length)) {
+        compressed_length = -1;
+        goto finish;
+    }
+    compressed_length = ccmp_length;
+    frt_os_write_bytes(out_stream, out_buf, ccmp_length);
+
+    /* compress data */
+    do {
+        int read_length = (FRT_COMPRESSION_BUFFER_SIZE > remaining_length) ? remaining_length : FRT_COMPRESSION_BUFFER_SIZE;
+        ccmp_length = LZ4F_compressUpdate(ctx, out_buf, out_buf_length, data + (length - remaining_length), read_length, NULL);
+        if (LZ4F_isError(ccmp_length)) {
+            compressed_length = -1;
+            goto finish;
+        }
+        frt_os_write_bytes(out_stream, out_buf, ccmp_length);
+        compressed_length += ccmp_length;
+        remaining_length -= read_length;
+    } while (remaining_length > 0);
+
+    /* finish up */
+    ccmp_length = LZ4F_compressEnd(ctx, out_buf, out_buf_length, NULL);
+    if (LZ4F_isError(ccmp_length)) {
+        compressed_length = -1;
+        goto finish;
+    }
+
+    frt_os_write_bytes(out_stream, out_buf, ccmp_length);
+    compressed_length += ccmp_length;
+
+finish:
+    LZ4F_freeCompressionContext(ctx);
+    free(out_buf);
+
+    return compressed_length;
+}
+
+static int frt_os_write_compressed_bytes(FrtOutStream* out_stream, frt_uchar *data, int length, FrtCompressionType compression) {
+    switch (compression) {
+        case FRT_COMPRESSION_BROTLI:
+            return frt_os_write_brotli_compressed_bytes(out_stream, data, length);
+        case FRT_COMPRESSION_BZ2:
+            return frt_os_write_bz2_compressed_bytes(out_stream, data, length);
+        case FRT_COMPRESSION_LZ4:
+            return frt_os_write_lz4_compressed_bytes(out_stream, data, length);
+        default:
+            return -1;
+    }
+
+}
+
+void frt_fw_add_doc(FrtFieldsWriter *fw, FrtDocument *doc) {
     int i, j, stored_cnt = 0;
     FrtDocField *df;
     FrtFieldInfo *fi;
+    FrtCompressionType compression;
     FrtOutStream *fdt_out = fw->fdt_out, *fdx_out = fw->fdx_out;
     const int doc_size = doc->size;
 
@@ -1719,16 +2020,22 @@ void frt_fw_add_doc(FrtFieldsWriter *fw, FrtDocument *doc)
             const int df_size = df->size;
             frt_os_write_vint(fdt_out, fi->number);
             frt_os_write_vint(fdt_out, df_size);
+
             if (fi_is_compressed(fi)) {
+                compression = frt_fi_get_compression(fi);
                 for (j = 0; j < df_size; j++) {
                     const int length = df->lengths[j];
-                    int compressed_len = frt_os_write_compressed_bytes(fw->buffer, (frt_uchar*)df->data[j], length);
+                    int compressed_len = frt_os_write_compressed_bytes(fw->buffer, (frt_uchar*)df->data[j], length, compression);
                     frt_os_write_vint(fdt_out, compressed_len - 1);
+                    frt_os_write_vint(fdt_out, rb_enc_to_index(df->encodings[j]));
+                    frt_os_write_vint(fdt_out, compression);
                 }
             } else {
                 for (j = 0; j < df_size; j++) {
                     const int length = df->lengths[j];
                     frt_os_write_vint(fdt_out, length);
+                    frt_os_write_vint(fdt_out, rb_enc_to_index(df->encodings[j]));
+                    frt_os_write_vint(fdt_out, FRT_COMPRESSION_NONE);
                     frt_os_write_bytes(fw->buffer, (frt_uchar*)df->data[j], length);
                     /* leave a space between fields as that is how they are analyzed */
                     frt_os_write_byte(fw->buffer, ' ');
@@ -2087,8 +2394,7 @@ static char *ste_scan_to(FrtTermEnum *te, const char *term)
     }
 }
 
-static FrtSegmentTermEnum *ste_allocate()
-{
+static FrtSegmentTermEnum *ste_allocate(void) {
     FrtSegmentTermEnum *ste = FRT_ALLOC_AND_ZERO(FrtSegmentTermEnum);
 
     TE(ste)->next = &ste_next;
@@ -2113,7 +2419,6 @@ void frt_ste_close(FrtTermEnum *te)
     free(te);
 }
 
-
 static char *frt_ste_get_term(FrtTermEnum *te, int pos)
 {
     FrtSegmentTermEnum *ste = STE(te);
@@ -2228,9 +2533,7 @@ static void tew_destroy(TermEnumWrapper *tew)
     tew->te->close(tew->te);
 }
 
-static TermEnumWrapper *tew_setup(TermEnumWrapper *tew, int index, FrtTermEnum *te,
-                                  FrtIndexReader *ir)
-{
+static TermEnumWrapper *tew_setup(TermEnumWrapper *tew, int index, FrtTermEnum *te, FrtIndexReader *ir) {
     tew->index = index;
     tew->ir = ir;
     tew->te = te;
@@ -2239,9 +2542,7 @@ static TermEnumWrapper *tew_setup(TermEnumWrapper *tew, int index, FrtTermEnum *
     return tew;
 }
 
-
-static char *mte_next(FrtTermEnum *te)
-{
+static char *mte_next(FrtTermEnum *te) {
     TermEnumWrapper *top =
         (TermEnumWrapper *)frt_pq_top(MTE(te)->tew_queue);
 
@@ -2271,8 +2572,7 @@ static char *mte_next(FrtTermEnum *te)
     return te->curr_term;
 }
 
-static FrtTermEnum *mte_set_field(FrtTermEnum *te, int field_num)
-{
+static FrtTermEnum *mte_set_field(FrtTermEnum *te, int field_num) {
     MultiTermEnum *mte = MTE(te);
     int i;
     const int size = mte->size;
@@ -2300,8 +2600,7 @@ static FrtTermEnum *mte_set_field(FrtTermEnum *te, int field_num)
     return te;
 }
 
-static char *mte_skip_to(FrtTermEnum *te, const char *term)
-{
+static char *mte_skip_to(FrtTermEnum *te, const char *term) {
     MultiTermEnum *mte = MTE(te);
     int i;
     const int size = mte->size;
@@ -2317,8 +2616,7 @@ static char *mte_skip_to(FrtTermEnum *te, const char *term)
     return mte_next(te);
 }
 
-static void mte_close(FrtTermEnum *te)
-{
+static void mte_close(FrtTermEnum *te) {
     int i;
     const int size = MTE(te)->size;
     for (i = 0; i < size; i++) {
@@ -2331,10 +2629,9 @@ static void mte_close(FrtTermEnum *te)
     free(te);
 }
 
-FrtTermEnum *frt_mte_new(FrtMultiReader *mr, int field_num, const char *term)
-{
-    FrtIndexReader **readers   = mr->sub_readers;
-    int r_cnt               = mr->r_cnt;
+FrtTermEnum *frt_mte_new(FrtMultiReader *mr, int field_num, const char *term) {
+    FrtIndexReader **readers = mr->sub_readers;
+    int r_cnt           = mr->r_cnt;
     int i;
     FrtIndexReader *reader;
     MultiTermEnum *mte  = FRT_ALLOC_AND_ZERO(MultiTermEnum);
@@ -2362,8 +2659,7 @@ FrtTermEnum *frt_mte_new(FrtMultiReader *mr, int field_num, const char *term)
 
             if (NULL != term) {
                 sub_te = reader->terms_from(reader, fnum, term);
-            }
-            else {
+            } else {
                 sub_te = reader->terms(reader, fnum);
             }
 
@@ -2372,8 +2668,7 @@ FrtTermEnum *frt_mte_new(FrtMultiReader *mr, int field_num, const char *term)
                 || (tew->term && (tew->term[0] != '\0'))) {
                 frt_pq_push(mte->tew_queue, tew);          /* initialize queue */
             }
-        }
-        else {
+        } else {
             /* add the term_enum_wrapper just in case */
             sub_te = reader->terms(reader, 0);
             sub_te->field_num = -1;
@@ -2395,9 +2690,7 @@ FrtTermEnum *frt_mte_new(FrtMultiReader *mr, int field_num, const char *term)
  *
  ****************************************************************************/
 
-FrtTermInfosReader *frt_tir_open(FrtStore *store,
-                          FrtSegmentFieldIndex *sfi, const char *segment)
-{
+FrtTermInfosReader *frt_tir_open(FrtStore *store, FrtSegmentFieldIndex *sfi, const char *segment) {
     FrtTermInfosReader *tir = FRT_ALLOC(FrtTermInfosReader);
     char file_name[FRT_SEGMENT_NAME_MAX_LENGTH];
 
@@ -2410,8 +2703,7 @@ FrtTermInfosReader *frt_tir_open(FrtStore *store,
     return tir;
 }
 
-static FrtTermEnum *tir_enum(FrtTermInfosReader *tir)
-{
+static FrtTermEnum *tir_enum(FrtTermInfosReader *tir) {
     FrtTermEnum *te;
     if (NULL == (te = (FrtTermEnum *)frt_thread_getspecific(tir->thread_te))) {
         te = frt_ste_clone(tir->orig_te);
@@ -2422,8 +2714,7 @@ static FrtTermEnum *tir_enum(FrtTermInfosReader *tir)
     return te;
 }
 
-FrtTermInfosReader *frt_tir_set_field(FrtTermInfosReader *tir, int field_num)
-{
+FrtTermInfosReader *frt_tir_set_field(FrtTermInfosReader *tir, int field_num) {
     if (field_num != tir->field_num) {
         ste_set_field(tir_enum(tir), field_num);
         tir->field_num = field_num;
@@ -2431,8 +2722,7 @@ FrtTermInfosReader *frt_tir_set_field(FrtTermInfosReader *tir, int field_num)
     return tir;
 }
 
-FrtTermInfo *frt_tir_get_ti(FrtTermInfosReader *tir, const char *term)
-{
+FrtTermInfo *frt_tir_get_ti(FrtTermInfosReader *tir, const char *term) {
     FrtTermEnum *te = tir_enum(tir);
     char *match;
 
@@ -2443,9 +2733,7 @@ FrtTermInfo *frt_tir_get_ti(FrtTermInfosReader *tir, const char *term)
     return NULL;
 }
 
-static FrtTermInfo *tir_get_ti_field(FrtTermInfosReader *tir, int field_num,
-                                  const char *term)
-{
+static FrtTermInfo *tir_get_ti_field(FrtTermInfosReader *tir, int field_num, const char *term) {
     FrtTermEnum *te = tir_enum(tir);
     char *match;
 
@@ -2461,19 +2749,16 @@ static FrtTermInfo *tir_get_ti_field(FrtTermInfosReader *tir, int field_num,
     return NULL;
 }
 
-char *frt_tir_get_term(FrtTermInfosReader *tir, int pos)
-{
+char *frt_tir_get_term(FrtTermInfosReader *tir, int pos) {
     if (pos < 0) {
         return NULL;
-    }
-    else {
+    } else {
         return frt_ste_get_term(tir_enum(tir), pos);
     }
 }
 
 
-void frt_tir_close(FrtTermInfosReader *tir)
-{
+void frt_tir_close(FrtTermInfosReader *tir) {
     frt_ary_destroy(tir->te_bucket, (frt_free_ft)&frt_ste_close);
     frt_ste_close(tir->orig_te);
 
@@ -2490,25 +2775,19 @@ void frt_tir_close(FrtTermInfosReader *tir)
  *
  ****************************************************************************/
 
-static FrtTermWriter *tw_new(FrtStore *store, char *file_name)
-{
+static FrtTermWriter *tw_new(FrtStore *store, char *file_name) {
     FrtTermWriter *tw = FRT_ALLOC_AND_ZERO(FrtTermWriter);
     tw->os = store->new_output(store, file_name);
     tw->last_term = FRT_EMPTY_STRING;
     return tw;
 }
 
-static void tw_close(FrtTermWriter *tw)
-{
+static void tw_close(FrtTermWriter *tw) {
     frt_os_close(tw->os);
     free(tw);
 }
 
-FrtTermInfosWriter *frt_tiw_open(FrtStore *store,
-                          const char *segment,
-                          int index_interval,
-                          int skip_interval)
-{
+FrtTermInfosWriter *frt_tiw_open(FrtStore *store, const char *segment, int index_interval, int skip_interval) {
     char file_name[FRT_SEGMENT_NAME_MAX_LENGTH];
     FrtTermInfosWriter *tiw = FRT_ALLOC(FrtTermInfosWriter);
     size_t segment_len = strlen(segment);
@@ -2537,11 +2816,7 @@ FrtTermInfosWriter *frt_tiw_open(FrtStore *store,
     return tiw;
 }
 
-static void tw_write_term(FrtTermWriter *tw,
-                                 FrtOutStream *os,
-                                 const char *term,
-                                 int term_len)
-{
+static void tw_write_term(FrtTermWriter *tw, FrtOutStream *os, const char *term, int term_len) {
     int start = frt_hlp_string_diff(tw->last_term, term);
     int length = term_len - start;
 
@@ -2552,12 +2827,7 @@ static void tw_write_term(FrtTermWriter *tw,
     tw->last_term = term;
 }
 
-static void tw_add(FrtTermWriter *tw,
-                   const char *term,
-                   int term_len,
-                   FrtTermInfo *ti,
-                   int skip_interval)
-{
+static void tw_add(FrtTermWriter *tw, const char *term, int term_len, FrtTermInfo *ti, int skip_interval) {
     FrtOutStream *os = tw->os;
 
 #ifdef DEBUG
@@ -2587,11 +2857,7 @@ static void tw_add(FrtTermWriter *tw,
     tw->counter++;
 }
 
-void frt_tiw_add(FrtTermInfosWriter *tiw,
-             const char *term,
-             int term_len,
-             FrtTermInfo *ti)
-{
+void frt_tiw_add(FrtTermInfosWriter *tiw, const char *term, int term_len, FrtTermInfo *ti) {
     off_t tis_pos;
 
     if (0 == (tiw->tis_writer->counter % tiw->index_interval)) {
@@ -2609,15 +2875,13 @@ void frt_tiw_add(FrtTermInfosWriter *tiw,
     tw_add(tiw->tis_writer, term, term_len, ti, tiw->skip_interval);
 }
 
-static void tw_reset(FrtTermWriter *tw)
-{
+static void tw_reset(FrtTermWriter *tw) {
     tw->counter = 0;
     tw->last_term = FRT_EMPTY_STRING;
     FRT_ZEROSET(&(tw->last_term_info), FrtTermInfo);
 }
 
-void frt_tiw_start_field(FrtTermInfosWriter *tiw, int field_num)
-{
+void frt_tiw_start_field(FrtTermInfosWriter *tiw, int field_num) {
     FrtOutStream *tfx_out = tiw->tfx_out;
     frt_os_write_vint(tfx_out, tiw->tix_writer->counter);    /* write tix size */
     frt_os_write_vint(tfx_out, tiw->tis_writer->counter);    /* write tis size */
@@ -2630,8 +2894,7 @@ void frt_tiw_start_field(FrtTermInfosWriter *tiw, int field_num)
     tiw->field_count++;
 }
 
-void frt_tiw_close(FrtTermInfosWriter *tiw)
-{
+void frt_tiw_close(FrtTermInfosWriter *tiw) {
     FrtOutStream *tfx_out = tiw->tfx_out;
     frt_os_write_vint(tfx_out, tiw->tix_writer->counter);
     frt_os_write_vint(tfx_out, tiw->tis_writer->counter);
@@ -2665,8 +2928,7 @@ void frt_tiw_close(FrtTermInfosWriter *tiw)
     }\
 } while (0)
 
-static void stde_seek_ti(FrtSegmentTermDocEnum *stde, FrtTermInfo *ti)
-{
+static void stde_seek_ti(FrtSegmentTermDocEnum *stde, FrtTermInfo *ti) {
     if (NULL == ti) {
         stde->doc_freq = 0;
     } else {
@@ -2684,14 +2946,12 @@ static void stde_seek_ti(FrtSegmentTermDocEnum *stde, FrtTermInfo *ti)
     }
 }
 
-static void stde_seek(FrtTermDocEnum *tde, int field_num, const char *term)
-{
+static void stde_seek(FrtTermDocEnum *tde, int field_num, const char *term) {
     FrtTermInfo *ti = tir_get_ti_field(STDE(tde)->tir, field_num, term);
     stde_seek_ti(STDE(tde), ti);
 }
 
-static void stde_seek_te(FrtTermDocEnum *tde, FrtTermEnum *te)
-{
+static void stde_seek_te(FrtTermDocEnum *tde, FrtTermEnum *te) {
 #ifdef DEBUG
     if (te->set_field != &ste_set_field) {
         FRT_RAISE(FRT_ARG_ERROR, "Passed an incorrect TermEnum type");
@@ -2700,20 +2960,17 @@ static void stde_seek_te(FrtTermDocEnum *tde, FrtTermEnum *te)
     stde_seek_ti(STDE(tde), &(te->curr_ti));
 }
 
-static int stde_doc_num(FrtTermDocEnum *tde)
-{
+static int stde_doc_num(FrtTermDocEnum *tde) {
     CHECK_STATE("doc_num");
     return STDE(tde)->doc_num;
 }
 
-static int stde_freq(FrtTermDocEnum *tde)
-{
+static int stde_freq(FrtTermDocEnum *tde) {
     CHECK_STATE("freq");
     return STDE(tde)->freq;
 }
 
-static bool stde_next(FrtTermDocEnum *tde)
-{
+static bool stde_next(FrtTermDocEnum *tde) {
     int doc_code;
     FrtSegmentTermDocEnum *stde = STDE(tde);
 
@@ -2741,8 +2998,7 @@ static bool stde_next(FrtTermDocEnum *tde)
     return true;
 }
 
-static int stde_read(FrtTermDocEnum *tde, int *docs, int *freqs, int req_num)
-{
+static int stde_read(FrtTermDocEnum *tde, int *docs, int *freqs, int req_num) {
     FrtSegmentTermDocEnum *stde = STDE(tde);
     int i = 0;
     int doc_code;
@@ -2769,8 +3025,7 @@ static int stde_read(FrtTermDocEnum *tde, int *docs, int *freqs, int req_num)
     return i;
 }
 
-static bool stde_skip_to(FrtTermDocEnum *tde, int target_doc_num)
-{
+static bool stde_skip_to(FrtTermDocEnum *tde, int target_doc_num) {
     FrtSegmentTermDocEnum *stde = STDE(tde);
 
     if (stde->doc_freq >= stde->skip_interval
@@ -2834,8 +3089,7 @@ static bool stde_skip_to(FrtTermDocEnum *tde, int target_doc_num)
     return true;
 }
 
-static void stde_close(FrtTermDocEnum *tde)
-{
+static void stde_close(FrtTermDocEnum *tde) {
     frt_is_close(STDE(tde)->frq_in);
 
     if (NULL != STDE(tde)->skip_in) {
@@ -2845,23 +3099,17 @@ static void stde_close(FrtTermDocEnum *tde)
     free(tde);
 }
 
-static void stde_skip_prox(FrtSegmentTermDocEnum *stde)
-{
+static void stde_skip_prox(FrtSegmentTermDocEnum *stde) {
     (void)stde;
 }
 
-static void stde_seek_prox(FrtSegmentTermDocEnum *stde, off_t prx_ptr)
-{
+static void stde_seek_prox(FrtSegmentTermDocEnum *stde, off_t prx_ptr) {
     (void)stde;
     (void)prx_ptr;
 }
 
 
-FrtTermDocEnum *frt_stde_new(FrtTermInfosReader *tir,
-                      FrtInStream *frq_in,
-                      FrtBitVector *deleted_docs,
-                      int skip_interval)
-{
+FrtTermDocEnum *frt_stde_new(FrtTermInfosReader *tir, FrtInStream *frq_in, FrtBitVector *deleted_docs, int skip_interval) {
     FrtSegmentTermDocEnum *stde = FRT_ALLOC_AND_ZERO(FrtSegmentTermDocEnum);
     FrtTermDocEnum *tde         = (FrtTermDocEnum *)stde;
 
@@ -2893,27 +3141,23 @@ FrtTermDocEnum *frt_stde_new(FrtTermInfosReader *tir,
  * SegmentTermPosEnum
  ****************************************************************************/
 
-static void stpe_seek_ti(FrtSegmentTermDocEnum *stde, FrtTermInfo *ti)
-{
+static void stpe_seek_ti(FrtSegmentTermDocEnum *stde, FrtTermInfo *ti) {
     if (NULL == ti) {
         stde->doc_freq = 0;
-    }
-    else {
+    } else {
         stde_seek_ti(stde, ti);
         frt_is_seek(stde->prx_in, ti->prx_ptr);
     }
 }
 
-static void stpe_seek(FrtTermDocEnum *tde, int field_num, const char *term)
-{
+static void stpe_seek(FrtTermDocEnum *tde, int field_num, const char *term) {
     FrtSegmentTermDocEnum *stde = STDE(tde);
     FrtTermInfo *ti = tir_get_ti_field(stde->tir, field_num, term);
     stpe_seek_ti(stde, ti);
     stde->prx_cnt = 0;
 }
 
-static bool stpe_next(FrtTermDocEnum *tde)
-{
+static bool stpe_next(FrtTermDocEnum *tde) {
     FrtSegmentTermDocEnum *stde = STDE(tde);
     frt_is_skip_vints(stde->prx_in, stde->prx_cnt);
 
@@ -3387,8 +3631,8 @@ FrtTermDocEnum *frt_mtdpe_new(FrtIndexReader *ir, int field_num, char **terms, i
  ****************************************************************************/
 
 static FrtHash *fn_extensions = NULL;
-static void file_name_filter_init()
-{
+
+static void file_name_filter_init(void) {
     int i;
     fn_extensions = frt_h_new_str((frt_free_ft)NULL, (frt_free_ft)NULL);
     for (i = 0; i < FRT_NELEMS(INDEX_EXTENSIONS); i++) {
@@ -3687,9 +3931,8 @@ static void ir_acquire_write_lock(FrtIndexReader *ir)
     }
 }
 
-static FrtIndexReader *ir_setup(FrtIndexReader *ir, FrtStore *store, FrtSegmentInfos *sis,
-                      FrtFieldInfos *fis, int is_owner)
-{
+static FrtIndexReader *ir_setup(FrtIndexReader *ir, FrtStore *store, FrtSegmentInfos *sis, FrtFieldInfos *fis, int is_owner) {
+    ir->type = FRT_INDEX_READER;
     frt_mutex_init(&ir->mutex, NULL);
     frt_mutex_init(&ir->field_index_mutex, NULL);
 
@@ -3712,8 +3955,7 @@ static FrtIndexReader *ir_setup(FrtIndexReader *ir, FrtStore *store, FrtSegmentI
     return ir;
 }
 
-int frt_ir_doc_freq(FrtIndexReader *ir, FrtSymbol field, const char *term)
-{
+int frt_ir_doc_freq(FrtIndexReader *ir, ID field, const char *term) {
     int field_num = frt_fis_get_field_num(ir->fis, field);
     if (field_num >= 0) {
         return ir->doc_freq(ir, field_num, term);
@@ -3723,8 +3965,7 @@ int frt_ir_doc_freq(FrtIndexReader *ir, FrtSymbol field, const char *term)
     }
 }
 
-static void ir_set_norm_i(FrtIndexReader *ir, int doc_num, int field_num, frt_uchar val)
-{
+static void ir_set_norm_i(FrtIndexReader *ir, int doc_num, int field_num, frt_uchar val) {
     frt_mutex_lock(&ir->mutex);
     ir->acquire_write_lock(ir);
     ir->set_norm_i(ir, doc_num, field_num, val);
@@ -3732,8 +3973,7 @@ static void ir_set_norm_i(FrtIndexReader *ir, int doc_num, int field_num, frt_uc
     frt_mutex_unlock(&ir->mutex);
 }
 
-void frt_ir_set_norm(FrtIndexReader *ir, int doc_num, FrtSymbol field, frt_uchar val)
-{
+void frt_ir_set_norm(FrtIndexReader *ir, int doc_num, ID field, frt_uchar val) {
     int field_num = frt_fis_get_field_num(ir->fis, field);
     if (field_num >= 0) {
         ir_set_norm_i(ir, doc_num, field_num, val);
@@ -3755,14 +3995,12 @@ frt_uchar *frt_ir_get_norms_i(FrtIndexReader *ir, int field_num)
     return norms;
 }
 
-frt_uchar *frt_ir_get_norms(FrtIndexReader *ir, FrtSymbol field)
-{
+frt_uchar *frt_ir_get_norms(FrtIndexReader *ir, ID field) {
     int field_num = frt_fis_get_field_num(ir->fis, field);
     return frt_ir_get_norms_i(ir, field_num);
 }
 
-frt_uchar *frt_ir_get_norms_into(FrtIndexReader *ir, FrtSymbol field, frt_uchar *buf)
-{
+frt_uchar *frt_ir_get_norms_into(FrtIndexReader *ir, ID field, frt_uchar *buf) {
     int field_num = frt_fis_get_field_num(ir->fis, field);
     if (field_num >= 0) {
         ir->get_norms_into(ir, field_num, buf);
@@ -3793,7 +4031,7 @@ void frt_ir_delete_doc(FrtIndexReader *ir, int doc_num)
     }
 }
 
-FrtDocument *frt_ir_get_doc_with_term(FrtIndexReader *ir, FrtSymbol field, const char *term) {
+FrtDocument *frt_ir_get_doc_with_term(FrtIndexReader *ir, ID field, const char *term) {
     FrtTermDocEnum *tde = ir_term_docs_for(ir, field, term);
     FrtDocument *doc = NULL;
 
@@ -3806,8 +4044,7 @@ FrtDocument *frt_ir_get_doc_with_term(FrtIndexReader *ir, FrtSymbol field, const
     return doc;
 }
 
-FrtTermEnum *frt_ir_terms(FrtIndexReader *ir, FrtSymbol field)
-{
+FrtTermEnum *frt_ir_terms(FrtIndexReader *ir, ID field) {
     FrtTermEnum *te = NULL;
     int field_num = frt_fis_get_field_num(ir->fis, field);
     if (field_num >= 0) {
@@ -3816,9 +4053,7 @@ FrtTermEnum *frt_ir_terms(FrtIndexReader *ir, FrtSymbol field)
     return te;
 }
 
-FrtTermEnum *frt_ir_terms_from(FrtIndexReader *ir, FrtSymbol field,
-                           const char *term)
-{
+FrtTermEnum *frt_ir_terms_from(FrtIndexReader *ir, ID field, const char *term) {
     FrtTermEnum *te = NULL;
     int field_num = frt_fis_get_field_num(ir->fis, field);
     if (field_num >= 0) {
@@ -3827,9 +4062,7 @@ FrtTermEnum *frt_ir_terms_from(FrtIndexReader *ir, FrtSymbol field,
     return te;
 }
 
-FrtTermDocEnum *ir_term_docs_for(FrtIndexReader *ir, FrtSymbol field,
-                              const char *term)
-{
+FrtTermDocEnum *ir_term_docs_for(FrtIndexReader *ir, ID field, const char *term) {
     int field_num = frt_fis_get_field_num(ir->fis, field);
     FrtTermDocEnum *tde = ir->term_docs(ir);
     if (field_num >= 0) {
@@ -3838,9 +4071,7 @@ FrtTermDocEnum *ir_term_docs_for(FrtIndexReader *ir, FrtSymbol field,
     return tde;
 }
 
-FrtTermDocEnum *frt_ir_term_positions_for(FrtIndexReader *ir, FrtSymbol field,
-                                   const char *term)
-{
+FrtTermDocEnum *frt_ir_term_positions_for(FrtIndexReader *ir, ID field, const char *term) {
     int field_num = frt_fis_get_field_num(ir->fis, field);
     FrtTermDocEnum *tde = ir->term_positions(ir);
     if (field_num >= 0) {
@@ -3854,7 +4085,7 @@ static void ir_commit_i(FrtIndexReader *ir)
     if (ir->has_changes) {
         if (NULL == ir->deleter && NULL != ir->store) {
             /* In the MultiReader case, we share this deleter across all
-             * SegmentReaders: */
+             * FrtSegmentReaders: */
             ir->set_deleter_i(ir, frt_deleter_new(ir->sis, ir->store));
         }
         if (ir->is_owner) {
@@ -3990,34 +4221,14 @@ static void norm_rewrite(Norm *norm, FrtStore *store, FrtDeleter *dlr,
 }
 
 /****************************************************************************
- * SegmentReader
+ * FrtSegmentReader
  ****************************************************************************/
 
-typedef struct SegmentReader {
-    FrtIndexReader ir;
-    FrtSegmentInfo *si;
-    char *segment;
-    FrtFieldsReader *fr;
-    FrtBitVector *deleted_docs;
-    FrtInStream *frq_in;
-    FrtInStream *prx_in;
-    FrtSegmentFieldIndex *sfi;
-    FrtTermInfosReader *tir;
-    frt_thread_key_t thread_fr;
-    void **fr_bucket;
-    FrtHash *norms;
-    FrtStore *cfs_store;
-    bool deleted_docs_dirty : 1;
-    bool undelete_all : 1;
-    bool norms_dirty : 1;
-} SegmentReader;
-
 #define IR(ir) ((FrtIndexReader *)(ir))
-
-#define SR(ir) ((SegmentReader *)(ir))
+#define SR(ir) ((FrtSegmentReader *)(ir))
 #define SR_SIZE(ir) (SR(ir)->fr->size)
 
-static FrtFieldsReader *sr_fr(SegmentReader *sr)
+static FrtFieldsReader *sr_fr(FrtSegmentReader *sr)
 {
     FrtFieldsReader *fr;
 
@@ -4029,12 +4240,12 @@ static FrtFieldsReader *sr_fr(SegmentReader *sr)
     return fr;
 }
 
-static bool sr_is_deleted_i(SegmentReader *sr, int doc_num)
+static bool sr_is_deleted_i(FrtSegmentReader *sr, int doc_num)
 {
     return (NULL != sr->deleted_docs && frt_bv_get(sr->deleted_docs, doc_num));
 }
 
-static void sr_get_norms_into_i(SegmentReader *sr, int field_num,
+static void sr_get_norms_into_i(FrtSegmentReader *sr, int field_num,
                                        frt_uchar *buf)
 {
     Norm *norm = (Norm *)frt_h_get_int(sr->norms, field_num);
@@ -4053,7 +4264,7 @@ static void sr_get_norms_into_i(SegmentReader *sr, int field_num,
     }
 }
 
-static frt_uchar *sr_get_norms_i(SegmentReader *sr, int field_num)
+static frt_uchar *sr_get_norms_i(FrtSegmentReader *sr, int field_num)
 {
     Norm *norm = (Norm *)frt_h_get_int(sr->norms, field_num);
     if (NULL == norm) {                           /* not an indexed field */
@@ -4189,7 +4400,7 @@ static void sr_commit_i(FrtIndexReader *ir)
 
 static void sr_close_i(FrtIndexReader *ir)
 {
-    SegmentReader *sr = SR(ir);
+    FrtSegmentReader *sr = SR(ir);
 
     if (sr->fr)           frt_fr_close(sr->fr);
     if (sr->tir)          frt_tir_close(sr->tir);
@@ -4298,14 +4509,12 @@ static FrtTermDocEnum *sr_term_docs(FrtIndexReader *ir)
 
 static FrtTermDocEnum *sr_term_positions(FrtIndexReader *ir)
 {
-    SegmentReader *sr = SR(ir);
+    FrtSegmentReader *sr = SR(ir);
     return frt_stpe_new(sr->tir, sr->frq_in, sr->prx_in, sr->deleted_docs,
                     STE(sr->tir->orig_te)->skip_interval);
 }
 
-static FrtTermVector *sr_term_vector(FrtIndexReader *ir, int doc_num,
-                                  FrtSymbol field)
-{
+static FrtTermVector *sr_term_vector(FrtIndexReader *ir, int doc_num, ID field) {
     FrtFieldInfo *fi = (FrtFieldInfo *)frt_h_get(ir->fis->field_dict, (void *)field);
     FrtFieldsReader *fr;
 
@@ -4360,7 +4569,7 @@ static void sr_open_norms(FrtIndexReader *ir, FrtStore *cfs_store)
     SR(ir)->norms_dirty = false;
 }
 
-static FrtIndexReader *sr_setup_i(SegmentReader *sr)
+static FrtIndexReader *sr_setup_i(FrtSegmentReader *sr)
 {
     FrtStore *volatile store = sr->si->store;
     FrtIndexReader *ir = IR(sr);
@@ -4391,6 +4600,8 @@ static FrtIndexReader *sr_setup_i(SegmentReader *sr)
     ir->commit_i            = &sr_commit_i;
     ir->close_i             = &sr_close_i;
 
+    ir->type                = FRT_SEGMENT_READER;
+
     sr->cfs_store   = NULL;
 
     FRT_TRY
@@ -4430,10 +4641,13 @@ static FrtIndexReader *sr_setup_i(SegmentReader *sr)
     return ir;
 }
 
-static FrtIndexReader *sr_open(FrtSegmentInfos *sis, FrtFieldInfos *fis, int si_num,
-                            bool is_owner)
-{
-    SegmentReader *sr = FRT_ALLOC_AND_ZERO(SegmentReader);
+FrtSegmentReader *frt_sr_alloc(void) {
+    return FRT_ALLOC_AND_ZERO(FrtSegmentReader);
+}
+
+static FrtIndexReader *sr_open(FrtSegmentInfos *sis, FrtFieldInfos *fis, int si_num, bool is_owner, FrtSegmentReader *sr) {
+    if (sr == NULL)
+        sr = frt_sr_alloc();
     sr->si = sis->segs[si_num];
     ir_setup(IR(sr), sr->si->store, sis, fis, is_owner);
     return sr_setup_i(sr);
@@ -4604,9 +4818,7 @@ static FrtTermDocEnum *mr_term_positions(FrtIndexReader *ir)
     return mtpe_new(MR(ir));
 }
 
-static FrtTermVector *mr_term_vector(FrtIndexReader *ir, int doc_num,
-                                  FrtSymbol field)
-{
+static FrtTermVector *mr_term_vector(FrtIndexReader *ir, int doc_num, ID field) {
     GET_READER();
     return reader->term_vector(reader, doc_num - MR(ir)->starts[i], field);
 }
@@ -4710,10 +4922,12 @@ static void mr_close_i(FrtIndexReader *ir)
     free(MR(ir)->starts);
 }
 
-static FrtIndexReader *mr_new(FrtIndexReader **sub_readers, const int r_cnt)
-{
+FrtMultiReader *frt_mr_alloc(void) {
+    return FRT_ALLOC_AND_ZERO(FrtMultiReader);
+}
+
+FrtMultiReader *frt_mr_init(FrtMultiReader *mr, FrtIndexReader **sub_readers, const int r_cnt) {
     int i;
-    FrtMultiReader *mr = FRT_ALLOC_AND_ZERO(FrtMultiReader);
     FrtIndexReader *ir = IR(mr);
 
     mr->sub_readers         = sub_readers;
@@ -4760,21 +4974,19 @@ static FrtIndexReader *mr_new(FrtIndexReader **sub_readers, const int r_cnt)
     ir->commit_i            = &mr_commit_i;
     ir->close_i             = &mr_close_i;
 
-    return ir;
+    ir->type                = FRT_MULTI_READER;
+
+    return mr;
 }
 
-static FrtIndexReader *frt_mr_open_i(FrtStore *store,
-                       FrtSegmentInfos *sis,
-                       FrtFieldInfos *fis,
-                       FrtIndexReader **sub_readers,
-                       const int r_cnt)
-{
-    FrtIndexReader *ir = mr_new(sub_readers, r_cnt);
+static FrtIndexReader *frt_mr_open_i(FrtStore *store, FrtSegmentInfos *sis, FrtFieldInfos *fis, FrtIndexReader **sub_readers, const int r_cnt, FrtIndexReader *ir) {
+    if (ir == NULL)
+        ir = (FrtIndexReader *)frt_mr_alloc();
+    ir = (FrtIndexReader *)frt_mr_init((FrtMultiReader *)ir, sub_readers, r_cnt);
     return ir_setup(ir, store, sis, fis, true);
 }
 
-static void mr_close_ext_i(FrtIndexReader *ir)
-{
+static void mr_close_ext_i(FrtIndexReader *ir) {
     int **field_num_map = MR(ir)->field_num_map;
     if (field_num_map) {
         int i;
@@ -4787,12 +4999,13 @@ static void mr_close_ext_i(FrtIndexReader *ir)
     mr_close_i(ir);
 }
 
-FrtIndexReader *frt_mr_open(FrtIndexReader **sub_readers, const int r_cnt)
-{
-    FrtIndexReader *ir = mr_new(sub_readers, r_cnt);
+FrtIndexReader *frt_mr_open(FrtIndexReader *ir, FrtIndexReader **sub_readers, const int r_cnt) {
+    if (ir == NULL)
+        ir = (FrtIndexReader *)frt_mr_alloc();
+    ir = (FrtIndexReader *)frt_mr_init((FrtMultiReader *)ir, sub_readers, r_cnt);
     FrtMultiReader *mr = MR(ir);
     /* defaults don't matter, this is just for reading fields, not adding */
-    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_NO, FRT_INDEX_NO, FRT_TERM_VECTOR_NO);
+    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_NO, FRT_TERM_VECTOR_NO);
     int i, j;
     bool need_field_map = false;
 
@@ -4827,12 +5040,10 @@ FrtIndexReader *frt_mr_open(FrtIndexReader **sub_readers, const int r_cnt)
                 mr->field_num_map[i][j] = fi_sub ? fi_sub->number : -1;
             }
         }
-    }
-    else {
+    } else {
         mr->field_num_map = NULL;
     }
 
-
     ir->close_i = &mr_close_ext_i;
 
     return ir_setup(ir, NULL, NULL, fis, false);
@@ -4842,21 +5053,19 @@ FrtIndexReader *frt_mr_open(FrtIndexReader **sub_readers, const int r_cnt)
  * IndexReader
  ****************************************************************************/
 
-
-static void ir_open_i(FrtStore *store, FindSegmentsFile *fsf)
-{
+static void ir_open_i(FrtStore *store, FindSegmentsFile *fsf, FrtIndexReader *ir) {
     volatile bool success = false;
-    FrtIndexReader *volatile ir = NULL;
+    // FrtIndexReader *volatile ir = NULL;
     FrtSegmentInfos *volatile sis = NULL;
     FRT_TRY
     do {
         FrtFieldInfos *fis;
         frt_mutex_lock(&store->mutex);
-        frt_sis_read_i(store, fsf);
+        frt_sis_read_i(store, fsf, NULL);
         sis = fsf->ret.sis;
         fis = sis->fis;
         if (sis->size == 1) {
-            ir = sr_open(sis, fis, 0, true);
+            ir = sr_open(sis, fis, 0, true, (FrtSegmentReader *)ir);
         }
         else {
             volatile int i;
@@ -4864,7 +5073,7 @@ static void ir_open_i(FrtStore *store, FindSegmentsFile *fsf)
             int num_segments = sis->size;
             for (i = num_segments - 1; i >= 0; i--) {
                 FRT_TRY
-                    readers[i] = sr_open(sis, fis, i, false);
+                    readers[i] = sr_open(sis, fis, i, false, NULL);
                 FRT_XCATCHALL
                     for (i++; i < num_segments; i++) {
                         frt_ir_close(readers[i]);
@@ -4872,7 +5081,7 @@ static void ir_open_i(FrtStore *store, FindSegmentsFile *fsf)
                     free(readers);
                 FRT_XENDTRY
             }
-            ir = frt_mr_open_i(store, sis, fis, readers, sis->size);
+            ir = frt_mr_open_i(store, sis, fis, readers, sis->size, ir);
         }
         fsf->ret.ir = ir;
         success = true;
@@ -4881,8 +5090,7 @@ static void ir_open_i(FrtStore *store, FindSegmentsFile *fsf)
         if (!success) {
             if (ir) {
                 frt_ir_close(ir);
-            }
-            else if (sis) {
+            } else if (sis) {
                 frt_sis_destroy(sis);
             }
         }
@@ -4894,15 +5102,12 @@ static void ir_open_i(FrtStore *store, FindSegmentsFile *fsf)
  * Will keep a reference to the store. To let this method delete the store
  * make sure you deref the store that you pass to it
  */
-FrtIndexReader *frt_ir_open(FrtStore *store)
-{
+FrtIndexReader *frt_ir_open(FrtIndexReader *ir, FrtStore *store) {
     FindSegmentsFile fsf;
-    sis_find_segments_file(store, &fsf, &ir_open_i);
+    sis_find_segments_file(store, &fsf, &ir_open_i, ir);
     return fsf.ret.ir;
 }
 
-
-
 /****************************************************************************
  *
  * Occurence
@@ -5292,10 +5497,7 @@ static void dw_add_offsets(FrtDocWriter *dw, int pos, off_t start, off_t end)
     dw->offsets_size = pos + 1;
 }
 
-FrtHash *frt_dw_invert_field(FrtDocWriter *dw,
-                           FrtFieldInverter *fld_inv,
-                           FrtDocField *df)
-{
+FrtHash *frt_dw_invert_field(FrtDocWriter *dw, FrtFieldInverter *fld_inv, FrtDocField *df) {
     FrtMemoryPool *mp = dw->mp;
     FrtAnalyzer *a = dw->analyzer;
     FrtHash *curr_plists = dw->curr_plists;
@@ -5311,7 +5513,7 @@ FrtHash *frt_dw_invert_field(FrtDocWriter *dw,
         int pos = -1, num_terms = 0;
 
         for (i = 0; i < df_size; i++) {
-            FrtTokenStream *ts = frt_a_get_ts(a, df->name, df->data[i]);
+            FrtTokenStream *ts = frt_a_get_ts(a, df->name, df->data[i], df->encodings[i]);
             /* ts->reset(ts, df->data[i]); no longer being called */
             if (store_offsets) {
                 while (NULL != (tk = ts->next(ts))) {
@@ -5321,21 +5523,16 @@ FrtHash *frt_dw_invert_field(FrtDocWriter *dw,
                     if (pos < 0) {
                         pos = 0;
                     }
-                    dw_add_posting(mp, curr_plists, fld_plists, doc_num,
-                                   tk->text, tk->len, pos);
-                    dw_add_offsets(dw, pos,
-                                   start_offset + tk->start,
-                                   start_offset + tk->end);
+                    dw_add_posting(mp, curr_plists, fld_plists, doc_num, tk->text, tk->len, pos);
+                    dw_add_offsets(dw, pos, start_offset + tk->start, start_offset + tk->end);
                     if (num_terms++ >= dw->max_field_length) {
                         break;
                     }
                 }
-            }
-            else {
+            } else {
                 while (NULL != (tk = ts->next(ts))) {
                     pos += tk->pos_inc;
-                    dw_add_posting(mp, curr_plists, fld_plists, doc_num,
-                                   tk->text, tk->len, pos);
+                    dw_add_posting(mp, curr_plists, fld_plists, doc_num, tk->text, tk->len, pos);
                     if (num_terms++ >= dw->max_field_length) {
                         break;
                     }
@@ -5345,8 +5542,7 @@ FrtHash *frt_dw_invert_field(FrtDocWriter *dw,
             start_offset += df->lengths[i] + 1;
         }
         fld_inv->length = num_terms;
-    }
-    else {
+    } else {
         char buf[FRT_MAX_WORD_SIZE];
         buf[FRT_MAX_WORD_SIZE - 1] = '\0';
         for (i = 0; i < df_size; i++) {
@@ -5356,11 +5552,9 @@ FrtHash *frt_dw_invert_field(FrtDocWriter *dw,
                 len = FRT_MAX_WORD_SIZE - 1;
                 data_ptr = (char *)memcpy(buf, df->data[i], len);
             }
-            dw_add_posting(mp, curr_plists, fld_plists, doc_num, data_ptr,
-                           len, i);
+            dw_add_posting(mp, curr_plists, fld_plists, doc_num, data_ptr, len, i);
             if (store_offsets) {
-                dw_add_offsets(dw, i, start_offset,
-                               start_offset + df->lengths[i]);
+                dw_add_offsets(dw, i, start_offset, start_offset + df->lengths[i]);
             }
             start_offset += df->lengths[i] + 1;
         }
@@ -5369,14 +5563,12 @@ FrtHash *frt_dw_invert_field(FrtDocWriter *dw,
     return curr_plists;
 }
 
-void frt_dw_reset_postings(FrtHash *postings)
-{
+void frt_dw_reset_postings(FrtHash *postings) {
     FRT_ZEROSET_N(postings->table, FrtHashEntry, postings->mask + 1);
     postings->fill = postings->size = 0;
 }
 
-void frt_dw_add_doc(FrtDocWriter *dw, FrtDocument *doc)
-{
+void frt_dw_add_doc(FrtDocWriter *dw, FrtDocument *doc) {
     int i;
     float boost;
     FrtDocField *df;
@@ -5398,16 +5590,12 @@ void frt_dw_add_doc(FrtDocWriter *dw, FrtDocument *doc)
 
         postings = frt_dw_invert_field(dw, fld_inv, df);
         if (fld_inv->store_term_vector) {
-            frt_fw_add_postings(dw->fw, fld_inv->fi->number,
-                            dw_sort_postings(postings), postings->size,
-                            dw->offsets, dw->offsets_size);
+            frt_fw_add_postings(dw->fw, fld_inv->fi->number, dw_sort_postings(postings), postings->size, dw->offsets, dw->offsets_size);
         }
 
         if (fld_inv->has_norms) {
-            boost = fld_inv->fi->boost * doc->boost * df->boost *
-                frt_sim_length_norm(dw->similarity, fi->name, fld_inv->length);
-            fld_inv->norms[dw->doc_num] =
-                frt_sim_encode_norm(dw->similarity, boost);
+            boost = fld_inv->fi->boost * doc->boost * df->boost * frt_sim_length_norm(dw->similarity, fi->name, fld_inv->length);
+            fld_inv->norms[dw->doc_num] = frt_sim_encode_norm(dw->similarity, boost);
         }
         frt_dw_reset_postings(postings);
         if (dw->offsets_size > 0) {
@@ -5960,15 +6148,12 @@ static void iw_commit_compound_file(FrtIndexWriter *iw, FrtSegmentInfo *si)
     iw_create_compound_file(iw->store, iw->fis, si, cfs_name, iw->deleter);
 }
 
-static void iw_merge_segments(FrtIndexWriter *iw, const int min_seg,
-                              const int max_seg)
-{
+static void iw_merge_segments(FrtIndexWriter *iw, const int min_seg, const int max_seg) {
     int i;
     FrtSegmentInfos *sis = iw->sis;
     FrtSegmentInfo *si = frt_sis_new_segment(sis, 0, iw->store);
 
-    SegmentMerger *merger = sm_create(iw, si, &sis->segs[min_seg],
-                                      max_seg - min_seg);
+    SegmentMerger *merger = sm_create(iw, si, &sis->segs[min_seg], max_seg - min_seg);
 
     /* This is where all the action happens. */
     si->doc_cnt = sm_merge(merger);
@@ -6080,8 +6265,7 @@ void frt_iw_commit(FrtIndexWriter *iw)
     frt_mutex_unlock(&iw->mutex);
 }
 
-void frt_iw_delete_term(FrtIndexWriter *iw, FrtSymbol field, const char *term)
-{
+void frt_iw_delete_term(FrtIndexWriter *iw, ID field, const char *term) {
     int field_num = frt_fis_get_field_num(iw->fis, field);
     if (field_num >= 0) {
         int i;
@@ -6092,7 +6276,7 @@ void frt_iw_delete_term(FrtIndexWriter *iw, FrtSymbol field, const char *term)
             const int seg_cnt = sis->size;
             bool did_delete = false;
             for (i = 0; i < seg_cnt; i++) {
-                FrtIndexReader *ir = sr_open(sis, iw->fis, i, false);
+                FrtIndexReader *ir = sr_open(sis, iw->fis, i, false, NULL);
                 FrtTermDocEnum *tde = ir->term_docs(ir);
                 ir->deleter = iw->deleter;
                 stde_seek(tde, field_num, term);
@@ -6114,9 +6298,7 @@ void frt_iw_delete_term(FrtIndexWriter *iw, FrtSymbol field, const char *term)
     }
 }
 
-void frt_iw_delete_terms(FrtIndexWriter *iw, FrtSymbol field,
-                     char **terms, const int term_cnt)
-{
+void frt_iw_delete_terms(FrtIndexWriter *iw, ID field, char **terms, const int term_cnt) {
     int field_num = frt_fis_get_field_num(iw->fis, field);
     if (field_num >= 0) {
         int i;
@@ -6127,7 +6309,7 @@ void frt_iw_delete_terms(FrtIndexWriter *iw, FrtSymbol field,
             const int seg_cnt = sis->size;
             bool did_delete = false;
             for (i = 0; i < seg_cnt; i++) {
-                FrtIndexReader *ir = sr_open(sis, iw->fis, i, false);
+                FrtIndexReader *ir = sr_open(sis, iw->fis, i, false, NULL);
                 FrtTermDocEnum *tde = ir->term_docs(ir);
                 int j;
                 for (j = 0 ; j < term_cnt; j++) {
@@ -6196,10 +6378,13 @@ void frt_iw_close(FrtIndexWriter *iw)
     free(iw);
 }
 
-FrtIndexWriter *frt_iw_open(FrtStore *store, FrtAnalyzer *volatile analyzer,
-                     const FrtConfig *config)
-{
-    FrtIndexWriter *iw = FRT_ALLOC_AND_ZERO(FrtIndexWriter);
+FrtIndexWriter *frt_iw_alloc(void) {
+    return FRT_ALLOC_AND_ZERO(FrtIndexWriter);
+}
+
+FrtIndexWriter *frt_iw_open(FrtIndexWriter *iw, FrtStore *store, FrtAnalyzer *volatile analyzer, const FrtConfig *config) {
+    if (iw == NULL)
+        iw = frt_iw_alloc();
     frt_mutex_init(&iw->mutex, NULL);
     iw->store = store;
     if (!config) {
@@ -6230,7 +6415,7 @@ FrtIndexWriter *frt_iw_open(FrtStore *store, FrtAnalyzer *volatile analyzer,
 
     iw->similarity = frt_sim_create_default();
     iw->analyzer = analyzer ? (FrtAnalyzer *)analyzer
-                            : frt_mb_standard_analyzer_new(true);
+                            : frt_standard_analyzer_new(true);
 
     iw->deleter = frt_deleter_new(iw->sis, store);
     deleter_delete_deletable_files(iw->deleter);
@@ -6242,9 +6427,7 @@ FrtIndexWriter *frt_iw_open(FrtStore *store, FrtAnalyzer *volatile analyzer,
 /*******************/
 /*** Add Indexes ***/
 /*******************/
-static void iw_cp_fields(FrtIndexWriter *iw, SegmentReader *sr,
-                         const char *segment, int *map)
-{
+static void iw_cp_fields(FrtIndexWriter *iw, FrtSegmentReader *sr, const char *segment, int *map) {
     char file_name[FRT_SEGMENT_NAME_MAX_LENGTH];
     FrtOutStream *fdt_out, *fdx_out;
     FrtInStream *fdt_in, *fdx_in;
@@ -6271,7 +6454,6 @@ static void iw_cp_fields(FrtIndexWriter *iw, SegmentReader *sr,
         frt_is2os_copy_bytes(del_in, del_out, frt_is_length(del_in));
     }
 
-
     if (map) {
         int i;
         const int max_doc = sr_max_doc(IR(sr));
@@ -6292,10 +6474,14 @@ static void iw_cp_fields(FrtIndexWriter *iw, SegmentReader *sr,
                 frt_os_write_vint(fdt_out, df_size);
                 /* sum total lengths of FrtDocField */
                 for (k = 0; k < df_size; k++) {
-                    /* Each field has one ' ' byte so add 1 */
-                    const int flen = frt_is_read_vint(fdt_in);
+                    const int flen = frt_is_read_vint(fdt_in); /* length */
+                    const int fenc = frt_is_read_vint(fdt_in); /* encoding */
+                    const int fcmp = frt_is_read_vint(fdt_in); /* compression */
                     frt_os_write_vint(fdt_out, flen);
-                    data_len +=  flen + 1;
+                    frt_os_write_vint(fdt_out, fenc);
+                    frt_os_write_vint(fdt_out, fcmp);
+                    /* Each field has one ' ' byte so add 1 */
+                    data_len += flen + 1;
                 }
             }
             frt_is2os_copy_bytes(fdt_in, fdt_out, data_len);
@@ -6318,8 +6504,7 @@ static void iw_cp_fields(FrtIndexWriter *iw, SegmentReader *sr,
                 frt_os_write_vint(fdt_out, tv_size);
             }
         }
-    }
-    else {
+    } else {
         frt_is2os_copy_bytes(fdt_in, fdt_out, frt_is_length(fdt_in));
         frt_is2os_copy_bytes(fdx_in, fdx_out, frt_is_length(fdx_in));
     }
@@ -6329,7 +6514,7 @@ static void iw_cp_fields(FrtIndexWriter *iw, SegmentReader *sr,
     frt_os_close(fdx_out);
 }
 
-static void iw_cp_terms(FrtIndexWriter *iw, SegmentReader *sr,
+static void iw_cp_terms(FrtIndexWriter *iw, FrtSegmentReader *sr,
                         const char *segment, int *map)
 {
     char file_name[FRT_SEGMENT_NAME_MAX_LENGTH];
@@ -6398,7 +6583,7 @@ static void iw_cp_terms(FrtIndexWriter *iw, SegmentReader *sr,
     frt_os_close(prx_out);
 }
 
-static void iw_cp_norms(FrtIndexWriter *iw, SegmentReader *sr,
+static void iw_cp_norms(FrtIndexWriter *iw, FrtSegmentReader *sr,
                         FrtSegmentInfo *si, int *map)
 {
     int i;
@@ -6429,9 +6614,7 @@ static void iw_cp_norms(FrtIndexWriter *iw, SegmentReader *sr,
     }
 }
 
-static void iw_cp_map_files(FrtIndexWriter *iw, SegmentReader *sr,
-                            FrtSegmentInfo *si)
-{
+static void iw_cp_map_files(FrtIndexWriter *iw, FrtSegmentReader *sr, FrtSegmentInfo *si) {
     int i;
     FrtFieldInfos *from_fis = IR(sr)->fis;
     FrtFieldInfos *to_fis = iw->fis;
@@ -6449,15 +6632,13 @@ static void iw_cp_map_files(FrtIndexWriter *iw, SegmentReader *sr,
     free(field_map);
 }
 
-static void iw_cp_files(FrtIndexWriter *iw, SegmentReader *sr,
-                        FrtSegmentInfo *si)
-{
+static void iw_cp_files(FrtIndexWriter *iw, FrtSegmentReader *sr, FrtSegmentInfo *si) {
     iw_cp_fields(iw, sr, si->name, NULL);
     iw_cp_terms( iw, sr, si->name, NULL);
     iw_cp_norms( iw, sr, si,       NULL);
 }
 
-static void iw_add_segment(FrtIndexWriter *iw, SegmentReader *sr)
+static void iw_add_segment(FrtIndexWriter *iw, FrtSegmentReader *sr)
 {
     FrtSegmentInfo *si = frt_sis_new_segment(iw->sis, 0, iw->store);
     FrtFieldInfos *fis = iw->fis;
@@ -6472,7 +6653,7 @@ static void iw_add_segment(FrtIndexWriter *iw, SegmentReader *sr)
         FrtFieldInfo *fi = sub_fis->fields[j];
         FrtFieldInfo *new_fi = frt_fis_get_field(fis, fi->name);
         if (NULL == new_fi) {
-            new_fi = frt_fi_new(fi->name, FRT_STORE_NO, FRT_INDEX_NO, FRT_TERM_VECTOR_NO);
+            new_fi = frt_fi_new(fi->name, FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_NO, FRT_TERM_VECTOR_NO);
             new_fi->bits = fi->bits;
             frt_fis_add_field(fis, new_fi);
         }