isomorfeus-ferret 0.12.7 → 0.13.0

data/ext/isomorfeus_ferret_ext/test_index.c

@@ -2,18 +2,17 @@
 #include "testhelper.h"
 #include "test.h"
 
-static FrtSymbol body, title, text, author, year, changing_field, compressed_field, tag;
+#undef close
+#undef read
 
-static FrtFieldInfos *prep_all_fis()
-{
-    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_NO, FRT_INDEX_YES, FRT_TERM_VECTOR_NO);
-    frt_fis_add_field(fis, frt_fi_new(rb_intern("tv"), FRT_STORE_NO, FRT_INDEX_YES, FRT_TERM_VECTOR_YES));
-    frt_fis_add_field(fis, frt_fi_new(rb_intern("tv un-t"), FRT_STORE_NO, FRT_INDEX_UNTOKENIZED,
-                              FRT_TERM_VECTOR_YES));
-    frt_fis_add_field(fis, frt_fi_new(rb_intern("tv+offsets"), FRT_STORE_NO, FRT_INDEX_YES,
-                              FRT_TERM_VECTOR_WITH_OFFSETS));
-    frt_fis_add_field(fis, frt_fi_new(rb_intern("tv+offsets un-t"), FRT_STORE_NO, FRT_INDEX_UNTOKENIZED,
-                              FRT_TERM_VECTOR_WITH_OFFSETS));
+static ID body, title, text, author, year, changing_field, compressed_field_brotli, compressed_field_bzip2, compressed_field_lz4, tag;
+
+static FrtFieldInfos *prep_all_fis(void) {
+    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_NO);
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("tv"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_YES));
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("tv un-t"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_YES));
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("tv+offsets"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_WITH_OFFSETS));
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("tv+offsets un-t"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_OFFSETS));
     return fis;
 
 }
@@ -27,290 +26,289 @@ static void destroy_docs(FrtDocument **docs, int len)
   free(docs);
 }
 
-static FrtFieldInfos *prep_book_fis()
-{
-    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_INDEX_YES,
-                              FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
-    frt_fis_add_field(fis, frt_fi_new(rb_intern("year"), FRT_STORE_YES, FRT_INDEX_NO, FRT_TERM_VECTOR_NO));
+static FrtFieldInfos *prep_book_fis(void) {
+    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("year"), FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_NO, FRT_TERM_VECTOR_NO));
     return fis;
 
 }
 
-FrtDocument *prep_book()
-{
+FrtDocument *prep_book(void) {
     FrtDocument *doc = frt_doc_new();
+    rb_encoding *enc = rb_enc_find("ASCII-8BIT");
+
     frt_doc_add_field(doc, frt_df_add_data(frt_df_new(author),
-            frt_estrdup("P.H. Newby")))->destroy_data = true;
+            frt_estrdup("P.H. Newby"), enc))->destroy_data = true;
     frt_doc_add_field(doc, frt_df_add_data(frt_df_new(title),
-            frt_estrdup("Something To Answer For")))->destroy_data = true;
+            frt_estrdup("Something To Answer For"), enc))->destroy_data = true;
     frt_doc_add_field(doc, frt_df_add_data(frt_df_new(year),
-            frt_estrdup("1969")))->destroy_data = true;
+            frt_estrdup("1969"), enc))->destroy_data = true;
     return doc;
 }
 
 #define BOOK_LIST_LENGTH 37
-FrtDocument **prep_book_list()
-{
+FrtDocument **prep_book_list(void) {
+    rb_encoding *enc = rb_enc_find("ASCII-8BIT");
     FrtDocument **docs = FRT_ALLOC_N(FrtDocument *, BOOK_LIST_LENGTH);
     docs[0] = frt_doc_new();
     frt_doc_add_field(docs[0], frt_df_add_data(frt_df_new(author),
-            frt_estrdup("P.H. Newby")))->destroy_data = true;
+            frt_estrdup("P.H. Newby"), enc))->destroy_data = true;
     frt_doc_add_field(docs[0], frt_df_add_data(frt_df_new(title),
-            frt_estrdup("Something To Answer For")))->destroy_data = true;
+            frt_estrdup("Something To Answer For"), enc))->destroy_data = true;
     frt_doc_add_field(docs[0], frt_df_add_data(frt_df_new(year),
-            frt_estrdup("1969")))->destroy_data = true;
+            frt_estrdup("1969"), enc))->destroy_data = true;
     docs[1] = frt_doc_new();
     frt_doc_add_field(docs[1], frt_df_add_data(frt_df_new(author),
-            frt_estrdup("Bernice Rubens")))->destroy_data = true;
+            frt_estrdup("Bernice Rubens"), enc))->destroy_data = true;
     frt_doc_add_field(docs[1], frt_df_add_data(frt_df_new(title),
-            frt_estrdup("The Elected Member")))->destroy_data = true;
+            frt_estrdup("The Elected Member"), enc))->destroy_data = true;
     frt_doc_add_field(docs[1], frt_df_add_data(frt_df_new(year),
-            frt_estrdup("1970")))->destroy_data = true;
+            frt_estrdup("1970"), enc))->destroy_data = true;
     docs[2] = frt_doc_new();
     frt_doc_add_field(docs[2], frt_df_add_data(frt_df_new(author),
-            frt_estrdup("V. S. Naipaul")))->destroy_data = true;
+            frt_estrdup("V. S. Naipaul"), enc))->destroy_data = true;
     frt_doc_add_field(docs[2], frt_df_add_data(frt_df_new(title),
-            frt_estrdup("In a Free State")))->destroy_data = true;
+            frt_estrdup("In a Free State"), enc))->destroy_data = true;
     frt_doc_add_field(docs[2], frt_df_add_data(frt_df_new(year),
-            frt_estrdup("1971")))->destroy_data = true;
+            frt_estrdup("1971"), enc))->destroy_data = true;
     docs[3] = frt_doc_new();
     frt_doc_add_field(docs[3], frt_df_add_data(frt_df_new(author),
-            frt_estrdup("John Berger")))->destroy_data = true;
+            frt_estrdup("John Berger"), enc))->destroy_data = true;
     frt_doc_add_field(docs[3], frt_df_add_data(frt_df_new(title),
-            frt_estrdup("G")))->destroy_data = true;
+            frt_estrdup("G"), enc))->destroy_data = true;
     frt_doc_add_field(docs[3], frt_df_add_data(frt_df_new(year),
-            frt_estrdup("1972")))->destroy_data = true;
+            frt_estrdup("1972"), enc))->destroy_data = true;
     docs[4] = frt_doc_new();
     frt_doc_add_field(docs[4], frt_df_add_data(frt_df_new(author),
-            frt_estrdup("J. G. Farrell")))->destroy_data = true;
+            frt_estrdup("J. G. Farrell"), enc))->destroy_data = true;
     frt_doc_add_field(docs[4], frt_df_add_data(frt_df_new(title),
-            frt_estrdup("The Siege of Krishnapur")))->destroy_data = true;
+            frt_estrdup("The Siege of Krishnapur"), enc))->destroy_data = true;
     frt_doc_add_field(docs[4], frt_df_add_data(frt_df_new(year),
-            frt_estrdup("1973")))->destroy_data = true;
+            frt_estrdup("1973"), enc))->destroy_data = true;
     docs[5] = frt_doc_new();
     frt_doc_add_field(docs[5], frt_df_add_data(frt_df_new(author),
-            frt_estrdup("Stanley Middleton")))->destroy_data = true;
+            frt_estrdup("Stanley Middleton"), enc))->destroy_data = true;
     frt_doc_add_field(docs[5], frt_df_add_data(frt_df_new(title),
-            frt_estrdup("Holiday")))->destroy_data = true;
+            frt_estrdup("Holiday"), enc))->destroy_data = true;
     frt_doc_add_field(docs[5], frt_df_add_data(frt_df_new(year),
-            frt_estrdup("1974")))->destroy_data = true;
+            frt_estrdup("1974"), enc))->destroy_data = true;
     docs[6] = frt_doc_new();
     frt_doc_add_field(docs[6], frt_df_add_data(frt_df_new(author),
-            frt_estrdup("Nadine Gordimer")))->destroy_data = true;
+            frt_estrdup("Nadine Gordimer"), enc))->destroy_data = true;
     frt_doc_add_field(docs[6], frt_df_add_data(frt_df_new(title),
-            frt_estrdup("The Conservationist")))->destroy_data = true;
+            frt_estrdup("The Conservationist"), enc))->destroy_data = true;
     frt_doc_add_field(docs[6], frt_df_add_data(frt_df_new(year),
-            frt_estrdup("1974")))->destroy_data = true;
+            frt_estrdup("1974"), enc))->destroy_data = true;
     docs[7] = frt_doc_new();
     frt_doc_add_field(docs[7], frt_df_add_data(frt_df_new(author),
-            frt_estrdup("Ruth Prawer Jhabvala")))->destroy_data = true;
+            frt_estrdup("Ruth Prawer Jhabvala"), enc))->destroy_data = true;
     frt_doc_add_field(docs[7], frt_df_add_data(frt_df_new(title),
-            frt_estrdup("Heat and Dust")))->destroy_data = true;
+            frt_estrdup("Heat and Dust"), enc))->destroy_data = true;
     frt_doc_add_field(docs[7], frt_df_add_data(frt_df_new(year),
-            frt_estrdup("1975")))->destroy_data = true;
+            frt_estrdup("1975"), enc))->destroy_data = true;
     docs[8] = frt_doc_new();
     frt_doc_add_field(docs[8], frt_df_add_data(frt_df_new(author),
-            frt_estrdup("David Storey")))->destroy_data = true;
+            frt_estrdup("David Storey"), enc))->destroy_data = true;
     frt_doc_add_field(docs[8], frt_df_add_data(frt_df_new(title),
-            frt_estrdup("Saville")))->destroy_data = true;
+            frt_estrdup("Saville"), enc))->destroy_data = true;
     frt_doc_add_field(docs[8], frt_df_add_data(frt_df_new(year),
-            frt_estrdup("1976")))->destroy_data = true;
+            frt_estrdup("1976"), enc))->destroy_data = true;
     docs[9] = frt_doc_new();
     frt_doc_add_field(docs[9], frt_df_add_data(frt_df_new(author),
-            frt_estrdup("Paul Scott")))->destroy_data = true;
+            frt_estrdup("Paul Scott"), enc))->destroy_data = true;
     frt_doc_add_field(docs[9], frt_df_add_data(frt_df_new(title),
-            frt_estrdup("Staying On")))->destroy_data = true;
+            frt_estrdup("Staying On"), enc))->destroy_data = true;
     frt_doc_add_field(docs[9], frt_df_add_data(frt_df_new(year),
-            frt_estrdup("1977")))->destroy_data = true;
+            frt_estrdup("1977"), enc))->destroy_data = true;
     docs[10] = frt_doc_new();
     frt_doc_add_field(docs[10], frt_df_add_data(frt_df_new(author),
-            frt_estrdup("Iris Murdoch")))->destroy_data = true;
+            frt_estrdup("Iris Murdoch"), enc))->destroy_data = true;
     frt_doc_add_field(docs[10], frt_df_add_data(frt_df_new(title),
-            frt_estrdup("The Sea")))->destroy_data = true;
+            frt_estrdup("The Sea"), enc))->destroy_data = true;
     frt_doc_add_field(docs[10], frt_df_add_data(frt_df_new(year),
-            frt_estrdup("1978")))->destroy_data = true;
+            frt_estrdup("1978"), enc))->destroy_data = true;
     docs[11] = frt_doc_new();
     frt_doc_add_field(docs[11], frt_df_add_data(frt_df_new(author),
-            frt_estrdup("Penelope Fitzgerald")))->destroy_data = true;
+            frt_estrdup("Penelope Fitzgerald"), enc))->destroy_data = true;
     frt_doc_add_field(docs[11], frt_df_add_data(frt_df_new(title),
-            frt_estrdup("Offshore")))->destroy_data = true;
+            frt_estrdup("Offshore"), enc))->destroy_data = true;
     frt_doc_add_field(docs[11], frt_df_add_data(frt_df_new(year),
-            frt_estrdup("1979")))->destroy_data = true;
+            frt_estrdup("1979"), enc))->destroy_data = true;
     docs[12] = frt_doc_new();
     frt_doc_add_field(docs[12], frt_df_add_data(frt_df_new(author),
-            frt_estrdup("William Golding")))->destroy_data = true;
+            frt_estrdup("William Golding"), enc))->destroy_data = true;
     frt_doc_add_field(docs[12], frt_df_add_data(frt_df_new(title),
-            frt_estrdup("Rites of Passage")))->destroy_data = true;
+            frt_estrdup("Rites of Passage"), enc))->destroy_data = true;
     frt_doc_add_field(docs[12], frt_df_add_data(frt_df_new(year),
-            frt_estrdup("1980")))->destroy_data = true;
+            frt_estrdup("1980"), enc))->destroy_data = true;
     docs[13] = frt_doc_new();
     frt_doc_add_field(docs[13], frt_df_add_data(frt_df_new(author),
-            frt_estrdup("Salman Rushdie")))->destroy_data = true;
+            frt_estrdup("Salman Rushdie"), enc))->destroy_data = true;
     frt_doc_add_field(docs[13], frt_df_add_data(frt_df_new(title),
-            frt_estrdup("Midnight's Children")))->destroy_data = true;
+            frt_estrdup("Midnight's Children"), enc))->destroy_data = true;
     frt_doc_add_field(docs[13], frt_df_add_data(frt_df_new(year),
-            frt_estrdup("1981")))->destroy_data = true;
+            frt_estrdup("1981"), enc))->destroy_data = true;
     docs[14] = frt_doc_new();
     frt_doc_add_field(docs[14], frt_df_add_data(frt_df_new(author),
-            frt_estrdup("Thomas Keneally")))->destroy_data = true;
+            frt_estrdup("Thomas Keneally"), enc))->destroy_data = true;
     frt_doc_add_field(docs[14], frt_df_add_data(frt_df_new(title),
-            frt_estrdup("Schindler's Ark")))->destroy_data = true;
+            frt_estrdup("Schindler's Ark"), enc))->destroy_data = true;
     frt_doc_add_field(docs[14], frt_df_add_data(frt_df_new(year),
-            frt_estrdup("1982")))->destroy_data = true;
+            frt_estrdup("1982"), enc))->destroy_data = true;
     docs[15] = frt_doc_new();
     frt_doc_add_field(docs[15], frt_df_add_data(frt_df_new(author),
-            frt_estrdup("J. M. Coetzee")))->destroy_data = true;
+            frt_estrdup("J. M. Coetzee"), enc))->destroy_data = true;
     frt_doc_add_field(docs[15], frt_df_add_data(frt_df_new(title),
-            frt_estrdup("Life and Times of Michael K")))->destroy_data = true;
+            frt_estrdup("Life and Times of Michael K"), enc))->destroy_data = true;
     frt_doc_add_field(docs[15], frt_df_add_data(frt_df_new(year),
-            frt_estrdup("1983")))->destroy_data = true;
+            frt_estrdup("1983"), enc))->destroy_data = true;
     docs[16] = frt_doc_new();
     frt_doc_add_field(docs[16], frt_df_add_data(frt_df_new(author),
-            frt_estrdup("Anita Brookner")))->destroy_data = true;
+            frt_estrdup("Anita Brookner"), enc))->destroy_data = true;
     frt_doc_add_field(docs[16], frt_df_add_data(frt_df_new(title),
-            frt_estrdup("Hotel du Lac")))->destroy_data = true;
+            frt_estrdup("Hotel du Lac"), enc))->destroy_data = true;
     frt_doc_add_field(docs[16], frt_df_add_data(frt_df_new(year),
-            frt_estrdup("1984")))->destroy_data = true;
+            frt_estrdup("1984"), enc))->destroy_data = true;
     docs[17] = frt_doc_new();
     frt_doc_add_field(docs[17], frt_df_add_data(frt_df_new(author),
-            frt_estrdup("Keri Hulme")))->destroy_data = true;
+            frt_estrdup("Keri Hulme"), enc))->destroy_data = true;
     frt_doc_add_field(docs[17], frt_df_add_data(frt_df_new(title),
-            frt_estrdup("The Bone People")))->destroy_data = true;
+            frt_estrdup("The Bone People"), enc))->destroy_data = true;
     frt_doc_add_field(docs[17], frt_df_add_data(frt_df_new(year),
-            frt_estrdup("1985")))->destroy_data = true;
+            frt_estrdup("1985"), enc))->destroy_data = true;
     docs[18] = frt_doc_new();
     frt_doc_add_field(docs[18], frt_df_add_data(frt_df_new(author),
-            frt_estrdup("Kingsley Amis")))->destroy_data = true;
+            frt_estrdup("Kingsley Amis"), enc))->destroy_data = true;
     frt_doc_add_field(docs[18], frt_df_add_data(frt_df_new(title),
-            frt_estrdup("The Old Devils")))->destroy_data = true;
+            frt_estrdup("The Old Devils"), enc))->destroy_data = true;
     frt_doc_add_field(docs[18], frt_df_add_data(frt_df_new(year),
-            frt_estrdup("1986")))->destroy_data = true;
+            frt_estrdup("1986"), enc))->destroy_data = true;
     docs[19] = frt_doc_new();
     frt_doc_add_field(docs[19], frt_df_add_data(frt_df_new(author),
-            frt_estrdup("Penelope Lively")))->destroy_data = true;
+            frt_estrdup("Penelope Lively"), enc))->destroy_data = true;
     frt_doc_add_field(docs[19], frt_df_add_data(frt_df_new(title),
-            frt_estrdup("Moon Tiger")))->destroy_data = true;
+            frt_estrdup("Moon Tiger"), enc))->destroy_data = true;
     frt_doc_add_field(docs[19], frt_df_add_data(frt_df_new(year),
-            frt_estrdup("1987")))->destroy_data = true;
+            frt_estrdup("1987"), enc))->destroy_data = true;
     docs[20] = frt_doc_new();
     frt_doc_add_field(docs[20], frt_df_add_data(frt_df_new(author),
-            frt_estrdup("Peter Carey")))->destroy_data = true;
+            frt_estrdup("Peter Carey"), enc))->destroy_data = true;
     frt_doc_add_field(docs[20], frt_df_add_data(frt_df_new(title),
-            frt_estrdup("Oscar and Lucinda")))->destroy_data = true;
+            frt_estrdup("Oscar and Lucinda"), enc))->destroy_data = true;
     frt_doc_add_field(docs[20], frt_df_add_data(frt_df_new(year),
-            frt_estrdup("1988")))->destroy_data = true;
+            frt_estrdup("1988"), enc))->destroy_data = true;
     docs[21] = frt_doc_new();
     frt_doc_add_field(docs[21], frt_df_add_data(frt_df_new(author),
-            frt_estrdup("Kazuo Ishiguro")))->destroy_data = true;
+            frt_estrdup("Kazuo Ishiguro"), enc))->destroy_data = true;
     frt_doc_add_field(docs[21], frt_df_add_data(frt_df_new(title),
-            frt_estrdup("The Remains of the Day")))->destroy_data = true;
+            frt_estrdup("The Remains of the Day"), enc))->destroy_data = true;
     frt_doc_add_field(docs[21], frt_df_add_data(frt_df_new(year),
-            frt_estrdup("1989")))->destroy_data = true;
+            frt_estrdup("1989"), enc))->destroy_data = true;
     docs[22] = frt_doc_new();
     frt_doc_add_field(docs[22], frt_df_add_data(frt_df_new(author),
-            frt_estrdup("A. S. Byatt")))->destroy_data = true;
+            frt_estrdup("A. S. Byatt"), enc))->destroy_data = true;
     frt_doc_add_field(docs[22], frt_df_add_data(frt_df_new(title),
-            frt_estrdup("Possession")))->destroy_data = true;
+            frt_estrdup("Possession"), enc))->destroy_data = true;
     frt_doc_add_field(docs[22], frt_df_add_data(frt_df_new(year),
-            frt_estrdup("1990")))->destroy_data = true;
+            frt_estrdup("1990"), enc))->destroy_data = true;
     docs[23] = frt_doc_new();
     frt_doc_add_field(docs[23], frt_df_add_data(frt_df_new(author),
-            frt_estrdup("Ben Okri")))->destroy_data = true;
+            frt_estrdup("Ben Okri"), enc))->destroy_data = true;
     frt_doc_add_field(docs[23], frt_df_add_data(frt_df_new(title),
-            frt_estrdup("The Famished Road")))->destroy_data = true;
+            frt_estrdup("The Famished Road"), enc))->destroy_data = true;
     frt_doc_add_field(docs[23], frt_df_add_data(frt_df_new(year),
-            frt_estrdup("1991")))->destroy_data = true;
+            frt_estrdup("1991"), enc))->destroy_data = true;
     docs[24] = frt_doc_new();
     frt_doc_add_field(docs[24], frt_df_add_data(frt_df_new(author),
-            frt_estrdup("Michael Ondaatje")))->destroy_data = true;
+            frt_estrdup("Michael Ondaatje"), enc))->destroy_data = true;
     frt_doc_add_field(docs[24], frt_df_add_data(frt_df_new(title),
-            frt_estrdup("The English Patient")))->destroy_data = true;
+            frt_estrdup("The English Patient"), enc))->destroy_data = true;
     frt_doc_add_field(docs[24], frt_df_add_data(frt_df_new(year),
-            frt_estrdup("1992")))->destroy_data = true;
+            frt_estrdup("1992"), enc))->destroy_data = true;
     docs[25] = frt_doc_new();
     frt_doc_add_field(docs[25], frt_df_add_data(frt_df_new(author),
-            frt_estrdup("Barry Unsworth")))->destroy_data = true;
+            frt_estrdup("Barry Unsworth"), enc))->destroy_data = true;
     frt_doc_add_field(docs[25], frt_df_add_data(frt_df_new(title),
-            frt_estrdup("Sacred Hunger")))->destroy_data = true;
+            frt_estrdup("Sacred Hunger"), enc))->destroy_data = true;
     frt_doc_add_field(docs[25], frt_df_add_data(frt_df_new(year),
-            frt_estrdup("1992")))->destroy_data = true;
+            frt_estrdup("1992"), enc))->destroy_data = true;
     docs[26] = frt_doc_new();
     frt_doc_add_field(docs[26], frt_df_add_data(frt_df_new(author),
-            frt_estrdup("Roddy Doyle")))->destroy_data = true;
+            frt_estrdup("Roddy Doyle"), enc))->destroy_data = true;
     frt_doc_add_field(docs[26], frt_df_add_data(frt_df_new(title),
-            frt_estrdup("Paddy Clarke Ha Ha Ha")))->destroy_data = true;
+            frt_estrdup("Paddy Clarke Ha Ha Ha"), enc))->destroy_data = true;
     frt_doc_add_field(docs[26], frt_df_add_data(frt_df_new(year),
-            frt_estrdup("1993")))->destroy_data = true;
+            frt_estrdup("1993"), enc))->destroy_data = true;
     docs[27] = frt_doc_new();
     frt_doc_add_field(docs[27], frt_df_add_data(frt_df_new(author),
-            frt_estrdup("James Kelman")))->destroy_data = true;
+            frt_estrdup("James Kelman"), enc))->destroy_data = true;
     frt_doc_add_field(docs[27], frt_df_add_data(frt_df_new(title),
-            frt_estrdup("How Late It Was, How Late")))->destroy_data = true;
+            frt_estrdup("How Late It Was, How Late"), enc))->destroy_data = true;
     frt_doc_add_field(docs[27], frt_df_add_data(frt_df_new(year),
-            frt_estrdup("1994")))->destroy_data = true;
+            frt_estrdup("1994"), enc))->destroy_data = true;
     docs[28] = frt_doc_new();
     frt_doc_add_field(docs[28], frt_df_add_data(frt_df_new(author),
-            frt_estrdup("Pat Barker")))->destroy_data = true;
+            frt_estrdup("Pat Barker"), enc))->destroy_data = true;
     frt_doc_add_field(docs[28], frt_df_add_data(frt_df_new(title),
-            frt_estrdup("The Ghost Road")))->destroy_data = true;
+            frt_estrdup("The Ghost Road"), enc))->destroy_data = true;
     frt_doc_add_field(docs[28], frt_df_add_data(frt_df_new(year),
-            frt_estrdup("1995")))->destroy_data = true;
+            frt_estrdup("1995"), enc))->destroy_data = true;
     docs[29] = frt_doc_new();
     frt_doc_add_field(docs[29], frt_df_add_data(frt_df_new(author),
-            frt_estrdup("Graham Swift")))->destroy_data = true;
+            frt_estrdup("Graham Swift"), enc))->destroy_data = true;
     frt_doc_add_field(docs[29], frt_df_add_data(frt_df_new(title),
-            frt_estrdup("Last Orders")))->destroy_data = true;
+            frt_estrdup("Last Orders"), enc))->destroy_data = true;
     frt_doc_add_field(docs[29], frt_df_add_data(frt_df_new(year),
-            frt_estrdup("1996")))->destroy_data = true;
+            frt_estrdup("1996"), enc))->destroy_data = true;
     docs[30] = frt_doc_new();
     frt_doc_add_field(docs[30], frt_df_add_data(frt_df_new(author),
-            frt_estrdup("Arundati Roy")))->destroy_data = true;
+            frt_estrdup("Arundati Roy"), enc))->destroy_data = true;
     frt_doc_add_field(docs[30], frt_df_add_data(frt_df_new(title),
-            frt_estrdup("The God of Small Things")))->destroy_data = true;
+            frt_estrdup("The God of Small Things"), enc))->destroy_data = true;
     frt_doc_add_field(docs[30], frt_df_add_data(frt_df_new(year),
-            frt_estrdup("1997")))->destroy_data = true;
+            frt_estrdup("1997"), enc))->destroy_data = true;
     docs[31] = frt_doc_new();
     frt_doc_add_field(docs[31], frt_df_add_data(frt_df_new(author),
-            frt_estrdup("Ian McEwan")))->destroy_data = true;
+            frt_estrdup("Ian McEwan"), enc))->destroy_data = true;
     frt_doc_add_field(docs[31], frt_df_add_data(frt_df_new(title),
-            frt_estrdup("Amsterdam")))->destroy_data = true;
+            frt_estrdup("Amsterdam"), enc))->destroy_data = true;
     frt_doc_add_field(docs[31], frt_df_add_data(frt_df_new(year),
-            frt_estrdup("1998")))->destroy_data = true;
+            frt_estrdup("1998"), enc))->destroy_data = true;
     docs[32] = frt_doc_new();
     frt_doc_add_field(docs[32], frt_df_add_data(frt_df_new(author),
-            frt_estrdup("J. M. Coetzee")))->destroy_data = true;
+            frt_estrdup("J. M. Coetzee"), enc))->destroy_data = true;
     frt_doc_add_field(docs[32], frt_df_add_data(frt_df_new(title),
-            frt_estrdup("Disgrace")))->destroy_data = true;
+            frt_estrdup("Disgrace"), enc))->destroy_data = true;
     frt_doc_add_field(docs[32], frt_df_add_data(frt_df_new(year),
-            frt_estrdup("1999")))->destroy_data = true;
+            frt_estrdup("1999"), enc))->destroy_data = true;
     docs[33] = frt_doc_new();
     frt_doc_add_field(docs[33], frt_df_add_data(frt_df_new(author),
-            frt_estrdup("Margaret Atwood")))->destroy_data = true;
+            frt_estrdup("Margaret Atwood"), enc))->destroy_data = true;
     frt_doc_add_field(docs[33], frt_df_add_data(frt_df_new(title),
-            frt_estrdup("The Blind Assassin")))->destroy_data = true;
+            frt_estrdup("The Blind Assassin"), enc))->destroy_data = true;
     frt_doc_add_field(docs[33], frt_df_add_data(frt_df_new(year),
-            frt_estrdup("2000")))->destroy_data = true;
+            frt_estrdup("2000"), enc))->destroy_data = true;
     docs[34] = frt_doc_new();
     frt_doc_add_field(docs[34], frt_df_add_data(frt_df_new(author),
-            frt_estrdup("Peter Carey")))->destroy_data = true;
+            frt_estrdup("Peter Carey"), enc))->destroy_data = true;
     frt_doc_add_field(docs[34], frt_df_add_data(frt_df_new(title),
-            frt_estrdup("True History of the Kelly Gang")))->destroy_data = true;
+            frt_estrdup("True History of the Kelly Gang"), enc))->destroy_data = true;
     frt_doc_add_field(docs[34], frt_df_add_data(frt_df_new(year),
-            frt_estrdup("2001")))->destroy_data = true;
+            frt_estrdup("2001"), enc))->destroy_data = true;
     docs[35] = frt_doc_new();
     frt_doc_add_field(docs[35], frt_df_add_data(frt_df_new(author),
-            frt_estrdup("Yann Martel")))->destroy_data = true;
+            frt_estrdup("Yann Martel"), enc))->destroy_data = true;
     frt_doc_add_field(docs[35], frt_df_add_data(frt_df_new(title),
-            frt_estrdup("The Life of Pi")))->destroy_data = true;
+            frt_estrdup("The Life of Pi"), enc))->destroy_data = true;
     frt_doc_add_field(docs[35], frt_df_add_data(frt_df_new(year),
-            frt_estrdup("2002")))->destroy_data = true;
+            frt_estrdup("2002"), enc))->destroy_data = true;
     docs[36] = frt_doc_new();
     frt_doc_add_field(docs[36], frt_df_add_data(frt_df_new(author),
-            frt_estrdup("DBC Pierre")))->destroy_data = true;
+            frt_estrdup("DBC Pierre"), enc))->destroy_data = true;
     frt_doc_add_field(docs[36], frt_df_add_data(frt_df_new(title),
-            frt_estrdup("Vernon God Little")))->destroy_data = true;
+            frt_estrdup("Vernon God Little"), enc))->destroy_data = true;
     frt_doc_add_field(docs[36], frt_df_add_data(frt_df_new(year),
-            frt_estrdup("2003")))->destroy_data = true;
+            frt_estrdup("2003"), enc))->destroy_data = true;
 
     return docs;
 }
@@ -327,7 +325,7 @@ static FrtIndexWriter *create_book_iw_conf(FrtStore *store, const FrtConfig *con
     FrtFieldInfos *fis = prep_book_fis();
     frt_index_create(store, fis);
     frt_fis_deref(fis);
-    return frt_iw_open(store, frt_whitespace_analyzer_new(false), config);
+    return frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), config);
 }
 
 static FrtIndexWriter *create_book_iw(FrtStore *store)
@@ -337,136 +335,154 @@ static FrtIndexWriter *create_book_iw(FrtStore *store)
 
 #define IR_TEST_DOC_CNT 256
 
-FrtDocument **prep_ir_test_docs()
-{
+FrtDocument **prep_ir_test_docs(void) {
     int i;
     char buf[2000] = "";
     FrtDocument **docs = FRT_ALLOC_N(FrtDocument *, IR_TEST_DOC_CNT);
     FrtDocField *df;
+    rb_encoding *enc = rb_enc_find("ASCII-8BIT");
 
     docs[0] = frt_doc_new();
     frt_doc_add_field(docs[0], frt_df_add_data(frt_df_new(changing_field),
             frt_estrdup("word3 word4 word1 word2 word1 word3 word4 word1 "
-                    "word3 word3")))->destroy_data = true;
-    frt_doc_add_field(docs[0], frt_df_add_data(frt_df_new(compressed_field),
+                    "word3 word3"), enc))->destroy_data = true;
+    frt_doc_add_field(docs[0], frt_df_add_data(frt_df_new(compressed_field_brotli),
+            frt_estrdup("word3 word4 word1 word2 word1 word3 word4 word1 "
+                    "word3 word3"), enc))->destroy_data = true;
+    frt_doc_add_field(docs[0], frt_df_add_data(frt_df_new(compressed_field_bzip2),
             frt_estrdup("word3 word4 word1 word2 word1 word3 word4 word1 "
-                    "word3 word3")))->destroy_data = true;
+                    "word3 word3"), enc))->destroy_data = true;
+    frt_doc_add_field(docs[0], frt_df_add_data(frt_df_new(compressed_field_lz4),
+           frt_estrdup("word3 word4 word1 word2 word1 word3 word4 word1 "
+                   "word3 word3"), enc))->destroy_data = true;
     frt_doc_add_field(docs[0], frt_df_add_data(frt_df_new(body),
-            frt_estrdup("Where is Wally")))->destroy_data = true;
+            frt_estrdup("Where is Wally"), enc))->destroy_data = true;
     docs[1] = frt_doc_new();
     frt_doc_add_field(docs[1], frt_df_add_data(frt_df_new(body),
-            frt_estrdup("Some Random Sentence read")))->destroy_data = true;
+            frt_estrdup("Some Random Sentence read"), enc))->destroy_data = true;
     frt_doc_add_field(docs[1], frt_df_add_data(frt_df_new(tag),
-            frt_estrdup("id_test")))->destroy_data = true;
+            frt_estrdup("id_test"), enc))->destroy_data = true;
     docs[2] = frt_doc_new();
     frt_doc_add_field(docs[2], frt_df_add_data(frt_df_new(body),
-            frt_estrdup("Some read Random Sentence read")))->destroy_data = true;
+            frt_estrdup("Some read Random Sentence read"), enc))->destroy_data = true;
     df = frt_df_new(tag);
-    frt_df_add_data(df, frt_estrdup("one"));
-    frt_df_add_data(df, frt_estrdup("two"));
-    frt_df_add_data(df, frt_estrdup("three"));
-    frt_df_add_data(df, frt_estrdup("four"));
+    frt_df_add_data(df, frt_estrdup("one"), enc);
+    frt_df_add_data(df, frt_estrdup("two"), enc);
+    frt_df_add_data(df, frt_estrdup("three"), enc);
+    frt_df_add_data(df, frt_estrdup("four"), enc);
+    frt_doc_add_field(docs[2], df)->destroy_data = true;
+    df = frt_df_new(compressed_field_brotli);
+    frt_df_add_data(df, frt_estrdup("one"), enc);
+    frt_df_add_data(df, frt_estrdup("two"), enc);
+    frt_df_add_data(df, frt_estrdup("three"), enc);
+    frt_df_add_data(df, frt_estrdup("four"), enc);
     frt_doc_add_field(docs[2], df)->destroy_data = true;
-    df = frt_df_new(compressed_field);
-    frt_df_add_data(df, frt_estrdup("one"));
-    frt_df_add_data(df, frt_estrdup("two"));
-    frt_df_add_data(df, frt_estrdup("three"));
-    frt_df_add_data(df, frt_estrdup("four"));
+    df = frt_df_new(compressed_field_bzip2);
+    frt_df_add_data(df, frt_estrdup("one"), enc);
+    frt_df_add_data(df, frt_estrdup("two"), enc);
+    frt_df_add_data(df, frt_estrdup("three"), enc);
+    frt_df_add_data(df, frt_estrdup("four"), enc);
+    frt_doc_add_field(docs[2], df)->destroy_data = true;
+    df = frt_df_new(compressed_field_lz4);
+    frt_df_add_data(df, frt_estrdup("one"), enc);
+    frt_df_add_data(df, frt_estrdup("two"), enc);
+    frt_df_add_data(df, frt_estrdup("three"), enc);
+    frt_df_add_data(df, frt_estrdup("four"), enc);
     frt_doc_add_field(docs[2], df)->destroy_data = true;
     docs[3] = frt_doc_new();
     frt_doc_add_field(docs[3], frt_df_add_data(frt_df_new(title),
-            frt_estrdup("War And Peace")))->destroy_data = true;
+            frt_estrdup("War And Peace"), enc))->destroy_data = true;
     frt_doc_add_field(docs[3], frt_df_add_data(frt_df_new(body),
             frt_estrdup("word3 word4 word1 word2 word1 "
-                    "word3 word4 word1 word3 word3")))->destroy_data = true;
+                    "word3 word4 word1 word3 word3"), enc))->destroy_data = true;
     frt_doc_add_field(docs[3], frt_df_add_data(frt_df_new(author),
-            frt_estrdup("Leo Tolstoy")))->destroy_data = true;
+            frt_estrdup("Leo Tolstoy"), enc))->destroy_data = true;
     frt_doc_add_field(docs[3], frt_df_add_data(frt_df_new(year),
-            frt_estrdup("1865")))->destroy_data = true;
+            frt_estrdup("1865"), enc))->destroy_data = true;
     frt_doc_add_field(docs[3], frt_df_add_data(frt_df_new(text),
-            frt_estrdup("more text which is not stored")))->destroy_data = true;
+            frt_estrdup("more text which is not stored"), enc))->destroy_data = true;
     docs[4] = frt_doc_new();
     frt_doc_add_field(docs[4], frt_df_add_data(frt_df_new(body),
-            frt_estrdup("Some Random Sentence")))->destroy_data = true;
+            frt_estrdup("Some Random Sentence"), enc))->destroy_data = true;
     docs[5] = frt_doc_new();
     frt_doc_add_field(docs[5], frt_df_add_data(frt_df_new(body),
-            frt_estrdup("Here's Wally")))->destroy_data = true;
+            frt_estrdup("Here's Wally"), enc))->destroy_data = true;
     frt_doc_add_field(docs[5], frt_df_add_data(frt_df_new(text),
-            frt_estrdup("so_that_norm_can_be_set")))->destroy_data = true;
+            frt_estrdup("so_that_norm_can_be_set"), enc))->destroy_data = true;
     docs[6] = frt_doc_new();
     frt_doc_add_field(docs[6], frt_df_add_data(frt_df_new(body),
             frt_estrdup("Some Random Sentence read read read read"
-                    )))->destroy_data = true;
+                    ), enc))->destroy_data = true;
     docs[7] = frt_doc_new();
     frt_doc_add_field(docs[7], frt_df_add_data(frt_df_new(body),
-            frt_estrdup("Some Random Sentence")))->destroy_data = true;
+            frt_estrdup("Some Random Sentence"), enc))->destroy_data = true;
     docs[8] = frt_doc_new();
     frt_doc_add_field(docs[8], frt_df_add_data(frt_df_new(body),
-            frt_estrdup("Some Random Sentence")))->destroy_data = true;
+            frt_estrdup("Some Random Sentence"), enc))->destroy_data = true;
     docs[9] = frt_doc_new();
     frt_doc_add_field(docs[9], frt_df_add_data(frt_df_new(body),
             frt_estrdup("read Some Random Sentence read this will be used after "
-                    "unfinished next position read")))->destroy_data = true;
+                    "unfinished next position read"), enc))->destroy_data = true;
     docs[10] = frt_doc_new();
     frt_doc_add_field(docs[10], frt_df_add_data(frt_df_new(body),
-            frt_estrdup("Some read Random Sentence")))->destroy_data = true;
+            frt_estrdup("Some read Random Sentence"), enc))->destroy_data = true;
     frt_doc_add_field(docs[10], frt_df_add_data(frt_df_new(changing_field),
             frt_estrdup("word3 word4 word1 word2 word1 word3 word4 word1 word3 "
-                    "word3")))->destroy_data = true;
+                    "word3"), enc))->destroy_data = true;
     docs[11] = frt_doc_new();
     frt_doc_add_field(docs[11], frt_df_add_data(frt_df_new(body),
-            frt_estrdup("And here too. Well, maybe Not")))->destroy_data = true;
+            frt_estrdup("And here too. Well, maybe Not"), enc))->destroy_data = true;
     docs[12] = frt_doc_new();
     frt_doc_add_field(docs[12], frt_df_add_data(frt_df_new(title),
-            frt_estrdup("Shawshank Redemption")))->destroy_data = true;
+            frt_estrdup("Shawshank Redemption"), enc))->destroy_data = true;
     frt_doc_add_field(docs[12], frt_df_add_data(frt_df_new(body),
-            frt_estrdup("Some Random Sentence")))->destroy_data = true;
+            frt_estrdup("Some Random Sentence"), enc))->destroy_data = true;
     docs[13] = frt_doc_new();
     frt_doc_add_field(docs[13], frt_df_add_data(frt_df_new(body),
-            frt_estrdup("Some Random Sentence")))->destroy_data = true;
+            frt_estrdup("Some Random Sentence"), enc))->destroy_data = true;
     docs[14] = frt_doc_new();
     frt_doc_add_field(docs[14], frt_df_add_data(frt_df_new(body),
-            frt_estrdup("Some Random Sentence")))->destroy_data = true;
+            frt_estrdup("Some Random Sentence"), enc))->destroy_data = true;
     docs[15] = frt_doc_new();
     frt_doc_add_field(docs[15], frt_df_add_data(frt_df_new(body),
-            frt_estrdup("Some read Random Sentence")))->destroy_data = true;
+            frt_estrdup("Some read Random Sentence"), enc))->destroy_data = true;
     docs[16] = frt_doc_new();
     frt_doc_add_field(docs[16], frt_df_add_data(frt_df_new(body),
-            frt_estrdup("Some Random read read Sentence")))->destroy_data = true;
+            frt_estrdup("Some Random read read Sentence"), enc))->destroy_data = true;
     docs[17] = frt_doc_new();
     frt_doc_add_field(docs[17], frt_df_add_data(frt_df_new(body),
-            frt_estrdup("Some Random read Sentence")))->destroy_data = true;
+            frt_estrdup("Some Random read Sentence"), enc))->destroy_data = true;
     frt_doc_add_field(docs[17], frt_df_add_data(frt_df_new(changing_field),
             frt_estrdup("word3 word4 word1 word2 word1 word3 word4 word1 word3 "
-                    "word3")))->destroy_data = true;
+                    "word3"), enc))->destroy_data = true;
     docs[18] = frt_doc_new();
     frt_doc_add_field(docs[18], frt_df_add_data(frt_df_new(body),
-            frt_estrdup("Wally Wally Wally")))->destroy_data = true;
+            frt_estrdup("Wally Wally Wally"), enc))->destroy_data = true;
     docs[19] = frt_doc_new();
     frt_doc_add_field(docs[19], frt_df_add_data(frt_df_new(body),
-            frt_estrdup("Some Random Sentence")))->destroy_data = true;
+            frt_estrdup("Some Random Sentence"), enc))->destroy_data = true;
     frt_doc_add_field(docs[19], frt_df_add_data(frt_df_new(changing_field),
             frt_estrdup("word3 word4 word1 word2 word1 word3 word4 word1 word3 "
-                    "word3")))->destroy_data = true;
+                    "word3"), enc))->destroy_data = true;
     docs[20] = frt_doc_new();
     frt_doc_add_field(docs[20], frt_df_add_data(frt_df_new(body),
             frt_estrdup("Wally is where Wally usually likes to go. Wally Mart! "
                     "Wally likes shopping there for Where's Wally books. "
-                    "Wally likes to read")))->destroy_data = true;
+                    "Wally likes to read"), enc))->destroy_data = true;
     frt_doc_add_field(docs[20], frt_df_add_data(frt_df_new(changing_field),
             frt_estrdup("word3 word4 word1 word2 word1 word3 word4 word1 word3 "
-                    "word3")))->destroy_data = true;
+                    "word3"), enc))->destroy_data = true;
     docs[21] = frt_doc_new();
     frt_doc_add_field(docs[21], frt_df_add_data(frt_df_new(body),
             frt_estrdup("Some Random Sentence read read read and more read read "
-                    "read")))->destroy_data = true;
+                    "read"), enc))->destroy_data = true;
     frt_doc_add_field(docs[21], frt_df_add_data(frt_df_new(changing_field),
             frt_estrdup("word3 word4 word1 word2 word1 word3 word4 word1 word3 "
-                    "word3")))->destroy_data = true;
+                    "word3"), enc))->destroy_data = true;
     frt_doc_add_field(docs[21], frt_df_add_data(frt_df_new(rb_intern("new field")),
-            frt_estrdup("zdata znot zto zbe zfound")))->destroy_data = true;
+            frt_estrdup("zdata znot zto zbe zfound"), enc))->destroy_data = true;
     frt_doc_add_field(docs[21], frt_df_add_data(frt_df_new(title),
-            frt_estrdup("title_too_long_for_max_word_lengthxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx")))->destroy_data = true;
+            frt_estrdup("title_too_long_for_max_word_lengthxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"), enc))->destroy_data = true;
 
     for (i = 1; i < 22; i++) {
         strcat(buf, "skip ");
@@ -475,7 +491,7 @@ FrtDocument **prep_ir_test_docs()
         strcat(buf, "skip ");
         docs[i] = frt_doc_new();
         frt_doc_add_field(docs[i], frt_df_add_data(frt_df_new(text),
-            frt_estrdup(buf)))->destroy_data = true;
+            frt_estrdup(buf), enc))->destroy_data = true;
     }
     return docs;
 }
@@ -488,12 +504,14 @@ static void prep_stde_test_docs(FrtDocument **docs, int doc_cnt, int num_words,
 {
     int i, j;
     char *buf = FRT_ALLOC_N(char, num_words * (TEST_WORD_LIST_MAX_LEN + 1));
+    rb_encoding *enc = rb_enc_find("ASCII-8BIT");
+
     for (i = 0; i < doc_cnt; i++) {
         docs[i] = frt_doc_new();
         for (j = 0; j < fis->size; j++) {
             if ((rand() % 2) == 0) {
                 FrtDocField *df = frt_df_new(fis->fields[j]->name);
-                frt_df_add_data(df, frt_estrdup(make_random_string(buf, num_words)));
+                frt_df_add_data(df, frt_estrdup(make_random_string(buf, num_words)), enc);
                 df->destroy_data = true;
                 frt_doc_add_field(docs[i], df);
             }
@@ -511,7 +529,7 @@ static void prep_test_1seg_index(FrtStore *store, FrtDocument **docs,
     FrtSegmentInfo *si = frt_si_new(frt_estrdup("_0"), doc_cnt, store);
 
     frt_index_create(store, fis);
-    iw = frt_iw_open(store, frt_whitespace_analyzer_new(false), NULL);
+    iw = frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), NULL);
 
     dw = frt_dw_open(iw, si);
 
@@ -658,6 +676,7 @@ static void test_segment_tde_deleted_docs(TestCase *tc, void *data)
     FrtBitVector *bv = frt_bv_new();
     FrtTermDocEnum *tde;
     FrtSegmentInfo *si = frt_si_new(frt_estrdup("_0"), NUM_STDE_TEST_DOCS, store);
+    rb_encoding *enc = rb_enc_find("ASCII-8BIT");
 
     dw = frt_dw_open(iw, si);
 
@@ -666,10 +685,10 @@ static void test_segment_tde_deleted_docs(TestCase *tc, void *data)
         if ((rand() % 2) == 0) {
             frt_bv_set(bv, i);
             Aiequal(1, frt_bv_get(bv, i));
-            frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("f")), (char *)double_word));
+            frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("f")), (char *)double_word, enc));
         }
         else {
-            frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("f")), (char *)triple_word));
+            frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("f")), (char *)triple_word, enc));
         }
         frt_dw_add_doc(dw, doc);
        frt_doc_destroy(doc);
@@ -718,15 +737,13 @@ static void test_segment_tde_deleted_docs(TestCase *tc, void *data)
 static void test_index_create(TestCase *tc, void *data)
 {
     FrtStore *store = (FrtStore *)data;
-    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_INDEX_YES, FRT_TERM_VECTOR_YES);
+    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_YES);
     (void)tc;
 
     store->clear_all(store);
-    Assert(!store->exists(store, "segments"),
-           "segments shouldn't exist yet");
+    Assert(!store->exists(store, "segments"), "segments shouldn't exist yet");
     frt_index_create(store, fis);
-    Assert(store->exists(store, "segments"),
-           "segments should now exist");
+    Assert(store->exists(store, "segments"), "segments should now exist");
     frt_fis_deref(fis);
 }
 
@@ -741,16 +758,16 @@ static void test_index_version(TestCase *tc, void *data)
     Atrue(frt_index_is_locked(store));  /* writer open, so dir is locked */
     frt_iw_close(iw);
     Atrue(!frt_index_is_locked(store));
-    ir = frt_ir_open(store);
+    ir = frt_ir_open(NULL, store);
     Atrue(!frt_index_is_locked(store)); /* reader only, no lock */
     version = frt_sis_read_current_version(store);
     frt_ir_close(ir);
 
     /* modify index and check version has been incremented: */
-    iw = frt_iw_open(store, frt_whitespace_analyzer_new(false), &frt_default_config);
+    iw = frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), &frt_default_config);
     add_document_with_fields(iw, 1);
     frt_iw_close(iw);
-    ir = frt_ir_open(store);
+    ir = frt_ir_open(NULL, store);
     Atrue(version < frt_sis_read_current_version(store));
     Atrue(frt_ir_is_latest(ir));
     frt_ir_close(ir);
@@ -764,15 +781,15 @@ static void test_index_undelete_all_after_close(TestCase *tc, void *data)
     add_document_with_fields(iw, 0);
     add_document_with_fields(iw, 1);
     frt_iw_close(iw);
-    ir = frt_ir_open(store);
+    ir = frt_ir_open(NULL, store);
     frt_ir_delete_doc(ir, 0);
     frt_ir_delete_doc(ir, 1);
     frt_ir_close(ir);
-    ir = frt_ir_open(store);
+    ir = frt_ir_open(NULL, store);
     frt_ir_undelete_all(ir);
     Aiequal(2, ir->num_docs(ir)); /* nothing has really been deleted */
     frt_ir_close(ir);
-    ir = frt_ir_open(store);
+    ir = frt_ir_open(NULL, store);
     Aiequal(2, ir->num_docs(ir)); /* nothing has really been deleted */
     Atrue(frt_ir_is_latest(ir));
     frt_ir_close(ir);
@@ -794,13 +811,14 @@ static void test_fld_inverter(TestCase *tc, void *data)
     FrtDocWriter *dw;
     FrtIndexWriter *iw = create_book_iw(store);
     FrtDocField *df;
+    rb_encoding *enc = rb_enc_find("ASCII-8BIT");
 
     dw = frt_dw_open(iw, frt_sis_new_segment(iw->sis, 0, iw->store));
 
     df = frt_df_new(rb_intern("no tv"));
-    frt_df_add_data(df, (char *)"one two three four five two three four five three four five four five");
-    frt_df_add_data(df, (char *)"ichi ni san yon go ni san yon go san yon go yon go go");
-    frt_df_add_data(df, (char *)"The quick brown fox jumped over five lazy dogs");
+    frt_df_add_data(df, (char *)"one two three four five two three four five three four five four five", enc);
+    frt_df_add_data(df, (char *)"ichi ni san yon go ni san yon go san yon go yon go go", enc);
+    frt_df_add_data(df, (char *)"The quick brown fox jumped over five lazy dogs", enc);
 
     curr_plists = frt_dw_invert_field(
         dw,
@@ -844,8 +862,8 @@ static void test_fld_inverter(TestCase *tc, void *data)
     frt_df_destroy(df);
 
     df = frt_df_new(rb_intern("no tv"));
-    frt_df_add_data(df, (char *)"seven new words and six old ones");
-    frt_df_add_data(df, (char *)"ichi ni one two quick dogs");
+    frt_df_add_data(df, (char *)"seven new words and six old ones", enc);
+    frt_df_add_data(df, (char *)"ichi ni one two quick dogs", enc);
 
     dw->doc_num++;
     frt_dw_reset_postings(dw->curr_plists);
@@ -916,7 +934,7 @@ static void test_iw_add_doc(TestCase *tc, void *data)
     frt_iw_close(iw);
     Assert(store->exists(store, "_0.cfs"), "data should still be there");
 
-    iw = frt_iw_open(store, frt_whitespace_analyzer_new(false), &frt_default_config);
+    iw = frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), &frt_default_config);
     frt_iw_add_doc(iw, docs[1]);
     Aiequal(2, frt_iw_doc_count(iw));
     Assert(!store->exists(store, "_1.cfs"),
@@ -951,7 +969,7 @@ static void test_create_with_reader(TestCase *tc, void *data)
     frt_iw_close(iw);
 
     /* now open reader: */
-    ir = frt_ir_open(store);
+    ir = frt_ir_open(NULL, store);
     Aiequal(1, ir->num_docs(ir));
 
     /* now open index for create: */
@@ -961,7 +979,7 @@ static void test_create_with_reader(TestCase *tc, void *data)
     frt_iw_close(iw);
 
     Aiequal(1, ir->num_docs(ir));
-    ir2 = frt_ir_open(store);
+    ir2 = frt_ir_open(NULL, store);
     Aiequal(1, ir2->num_docs(ir));
     frt_ir_close(ir);
     frt_ir_close(ir2);
@@ -1018,10 +1036,10 @@ static void test_simulated_crashed_writer(TestCase *tc, void *data)
     frt_is_close(is);
     frt_os_close(os);
 
-    ir = frt_ir_open(store);
+    ir = frt_ir_open(NULL, store);
     frt_ir_close(ir);
 
-    iw = frt_iw_open(store, frt_whitespace_analyzer_new(false), &config);
+    iw = frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), &config);
 
     /* add all books */
     for (i = 0; i < BOOK_LIST_LENGTH; i++) {
@@ -1081,7 +1099,7 @@ static void test_simulated_corrupt_index1(TestCase *tc, void *data)
     store->remove(store, file_name_in);
 
     FRT_TRY
-        ir = frt_ir_open(store);
+        ir = frt_ir_open(NULL, store);
         frt_ir_close(ir);
         Afail("reader should have failed to open on a crashed index");
         break;
@@ -1127,7 +1145,7 @@ static void test_simulated_corrupt_index2(TestCase *tc, void *data)
     store->remove(store, "_0.cfs");
 
     FRT_TRY
-        ir = frt_ir_open(store);
+        ir = frt_ir_open(NULL, store);
         frt_ir_close(ir);
         Afail("reader should have failed to open on a crashed index");
         break;
@@ -1171,16 +1189,18 @@ void test_iw_add_empty_tv(TestCase *tc, void *data)
     FrtStore *store = (FrtStore *)data;
     FrtIndexWriter *iw;
     FrtDocument *doc;
-    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_NO, FRT_INDEX_YES, FRT_TERM_VECTOR_YES);
-    frt_fis_add_field(fis, frt_fi_new(rb_intern("no_tv"), FRT_STORE_YES, FRT_INDEX_YES, FRT_TERM_VECTOR_NO));
+    rb_encoding *enc = rb_enc_find("ASCII-8BIT");
+
+    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_YES);
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("no_tv"), FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_NO));
     frt_index_create(store, fis);
     frt_fis_deref(fis);
 
-    iw = frt_iw_open(store, frt_whitespace_analyzer_new(false), &frt_default_config);
+    iw = frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), &frt_default_config);
     doc = frt_doc_new();
-    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("tv1")), (char *)""));
-    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("tv2")), (char *)""));
-    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("no_tv")), (char *)"one two three"));
+    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("tv1")), (char *)"", enc));
+    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("tv2")), (char *)"", enc));
+    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("no_tv")), (char *)"one two three", enc));
 
     frt_iw_add_doc(iw, doc);
     frt_iw_commit(iw);
@@ -1209,22 +1229,22 @@ static void test_iw_del_terms(TestCase *tc, void *data)
     frt_iw_close(iw);
     destroy_docs(docs, BOOK_LIST_LENGTH);
 
-    ir = frt_ir_open(store);
+    ir = frt_ir_open(NULL, store);
     Aiequal(BOOK_LIST_LENGTH, ir->num_docs(ir));
     Aiequal(BOOK_LIST_LENGTH, ir->max_doc(ir));
     frt_ir_close(ir);
 
-    iw = frt_iw_open(store, frt_whitespace_analyzer_new(false), &config);
+    iw = frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), &config);
     frt_iw_delete_term(iw, title, "State");
     frt_iw_close(iw);
 
-    ir = frt_ir_open(store);
+    ir = frt_ir_open(NULL, store);
     Aiequal(BOOK_LIST_LENGTH - 1, ir->num_docs(ir));
     Aiequal(BOOK_LIST_LENGTH, ir->max_doc(ir));
     frt_ir_close(ir);
 
     /* test deleting multiple Terms */
-    iw = frt_iw_open(store, frt_whitespace_analyzer_new(false), &config);
+    iw = frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), &config);
     frt_iw_delete_term(iw, title, "The");
     frt_iw_delete_term(iw, title, "Blind");
     terms[0] = "Berger";
@@ -1233,7 +1253,7 @@ static void test_iw_del_terms(TestCase *tc, void *data)
     frt_iw_delete_terms(iw, author, (char **)terms, 3);
     frt_iw_close(iw);
 
-    ir = frt_ir_open(store);
+    ir = frt_ir_open(NULL, store);
     Aiequal(BOOK_LIST_LENGTH - 17, ir->num_docs(ir));
     Aiequal(BOOK_LIST_LENGTH, ir->max_doc(ir));
     Atrue(!ir->is_deleted(ir, 0));
@@ -1260,13 +1280,13 @@ static void test_iw_del_terms(TestCase *tc, void *data)
     Atrue(ir->is_deleted(ir, 36));
     frt_ir_commit(ir);
 
-    iw = frt_iw_open(store, frt_whitespace_analyzer_new(false), &config);
+    iw = frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), &config);
     frt_iw_optimize(iw);
     frt_iw_close(iw);
 
     frt_ir_close(ir);
 
-    ir = frt_ir_open(store);
+    ir = frt_ir_open(NULL, store);
     Aiequal(BOOK_LIST_LENGTH - 17, ir->num_docs(ir));
     Aiequal(BOOK_LIST_LENGTH - 17, ir->max_doc(ir));
     frt_ir_close(ir);
@@ -1301,15 +1321,15 @@ static void reader_test_env_destroy(ReaderTestEnvironment *rte)
 static FrtIndexReader *reader_test_env_ir_open(ReaderTestEnvironment *rte)
 {
     if (rte->store_cnt == 1) {
-        return frt_ir_open(rte->stores[0]);
+        return frt_ir_open(NULL, rte->stores[0]);
     }
     else {
         FrtIndexReader **sub_readers = FRT_ALLOC_N(FrtIndexReader *, rte->store_cnt);
         int i;
         for (i = 0; i < rte->store_cnt; i++) {
-            sub_readers[i] = frt_ir_open(rte->stores[i]);
+            sub_readers[i] = frt_ir_open(NULL, rte->stores[i]);
         }
-        return (frt_mr_open(sub_readers, rte->store_cnt));
+        return (frt_mr_open(NULL, sub_readers, rte->store_cnt));
     }
 }
 
@@ -1326,9 +1346,8 @@ static ReaderTestEnvironment *reader_test_env_new(int type)
 
     rte->stores = FRT_ALLOC_N(FrtStore *, store_cnt);
     for (i = 0; i < store_cnt; i++) {
-        FrtStore *store = rte->stores[i] = frt_open_ram_store();
-        FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_INDEX_YES,
-                                  FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
+        FrtStore *store = rte->stores[i] = frt_open_ram_store(NULL);
+        FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
         int start_doc = i * doc_cnt;
         int end_doc = (i + 1) * doc_cnt;
         if (end_doc > IR_TEST_DOC_CNT) {
@@ -1338,7 +1357,7 @@ static ReaderTestEnvironment *reader_test_env_new(int type)
         frt_fis_deref(fis);
         config.max_buffered_docs = 3;
 
-        iw = frt_iw_open(store, frt_whitespace_analyzer_new(false), &config);
+        iw = frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), &config);
 
         for (j = start_doc; j < end_doc; j++) {
             int k;
@@ -1350,24 +1369,19 @@ static ReaderTestEnvironment *reader_test_env_new(int type)
                 fis = iw->fis;
                 if (NULL == frt_fis_get_field(fis, df->name)) {
                     if (author == df->name) {
-                        frt_fis_add_field(fis, frt_fi_new(author, FRT_STORE_YES, FRT_INDEX_YES,
-                                  FRT_TERM_VECTOR_WITH_POSITIONS));
+                        frt_fis_add_field(fis, frt_fi_new(author, FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_WITH_POSITIONS));
                     } else if (title == df->name) {
-                        frt_fis_add_field(fis, frt_fi_new(title, FRT_STORE_YES,
-                                                  FRT_INDEX_UNTOKENIZED,
-                                                  FRT_TERM_VECTOR_WITH_OFFSETS));
+                        frt_fis_add_field(fis, frt_fi_new(title, FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_OFFSETS));
                     } else if (year == df->name) {
-                        frt_fis_add_field(fis, frt_fi_new(year, FRT_STORE_YES,
-                                                  FRT_INDEX_UNTOKENIZED,
-                                                  FRT_TERM_VECTOR_NO));
+                        frt_fis_add_field(fis, frt_fi_new(year, FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_NO));
                     } else if (text == df->name) {
-                        frt_fis_add_field(fis, frt_fi_new(text, FRT_STORE_NO, FRT_INDEX_YES,
-                                                  FRT_TERM_VECTOR_NO));
-                    } else if (compressed_field == df->name) {
-                        frt_fis_add_field(fis, frt_fi_new(compressed_field,
-                                                  FRT_STORE_COMPRESS,
-                                                  FRT_INDEX_YES,
-                                                  FRT_TERM_VECTOR_NO));
+                        frt_fis_add_field(fis, frt_fi_new(text, FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_NO));
+                    } else if (compressed_field_brotli == df->name) {
+                        frt_fis_add_field(fis, frt_fi_new(compressed_field_brotli, FRT_STORE_YES, FRT_COMPRESSION_BROTLI, FRT_INDEX_YES, FRT_TERM_VECTOR_NO));
+                    } else if (compressed_field_bzip2 == df->name) {
+                        frt_fis_add_field(fis, frt_fi_new(compressed_field_bzip2, FRT_STORE_YES, FRT_COMPRESSION_BZ2, FRT_INDEX_YES, FRT_TERM_VECTOR_NO));
+                    } else if (compressed_field_lz4 == df->name) {
+                       frt_fis_add_field(fis, frt_fi_new(compressed_field_lz4, FRT_STORE_YES, FRT_COMPRESSION_LZ4, FRT_INDEX_YES, FRT_TERM_VECTOR_NO));
                     }
                 }
             }
@@ -1382,20 +1396,19 @@ static ReaderTestEnvironment *reader_test_env_new(int type)
 
     if (type == add_indexes_reader_type) {
         /* Prepare store for Add Indexes test */
-        FrtStore *store = frt_open_ram_store();
-        FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_INDEX_YES,
-                                  FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
+        FrtStore *store = frt_open_ram_store(NULL);
+        FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
         FrtIndexReader **readers = FRT_ALLOC_N(FrtIndexReader *, rte->store_cnt);
         int i;
         for (i = 0; i < rte->store_cnt; i++) {
-            readers[i] = frt_ir_open(rte->stores[i]);
+            readers[i] = frt_ir_open(NULL, rte->stores[i]);
         }
         frt_index_create(store, fis);
         frt_fis_deref(fis);
-        iw = frt_iw_open(store, frt_whitespace_analyzer_new(false), &config);
+        iw = frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), &config);
         frt_iw_add_readers(iw, readers, rte->store_cnt - 10);
         frt_iw_close(iw);
-        iw = frt_iw_open(store, frt_whitespace_analyzer_new(false), &config);
+        iw = frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), &config);
         frt_iw_add_readers(iw, readers + (rte->store_cnt - 10), 10);
         frt_iw_close(iw);
         for (i = 0; i < rte->store_cnt; i++) {
@@ -1418,23 +1431,19 @@ static void write_ir_test_docs(FrtStore *store)
     FrtIndexWriter *iw;
     FrtDocument **docs = prep_ir_test_docs();
 
-    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_INDEX_YES,
-                              FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
-    frt_fis_add_field(fis, frt_fi_new(author, FRT_STORE_YES, FRT_INDEX_YES,
-                              FRT_TERM_VECTOR_WITH_POSITIONS));
-    frt_fis_add_field(fis, frt_fi_new(title, FRT_STORE_YES, FRT_INDEX_UNTOKENIZED,
-                              FRT_TERM_VECTOR_WITH_OFFSETS));
-    frt_fis_add_field(fis, frt_fi_new(year, FRT_STORE_YES, FRT_INDEX_UNTOKENIZED,
-                              FRT_TERM_VECTOR_NO));
-    frt_fis_add_field(fis, frt_fi_new(text, FRT_STORE_NO, FRT_INDEX_YES,
-                              FRT_TERM_VECTOR_NO));
-    frt_fis_add_field(fis, frt_fi_new(compressed_field, FRT_STORE_COMPRESS, FRT_INDEX_YES,
-                              FRT_TERM_VECTOR_NO));
+    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
+    frt_fis_add_field(fis, frt_fi_new(author, FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_WITH_POSITIONS));
+    frt_fis_add_field(fis, frt_fi_new(title, FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_OFFSETS));
+    frt_fis_add_field(fis, frt_fi_new(year, FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_NO));
+    frt_fis_add_field(fis, frt_fi_new(text, FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_NO));
+    frt_fis_add_field(fis, frt_fi_new(compressed_field_brotli, FRT_STORE_YES, FRT_COMPRESSION_BROTLI, FRT_INDEX_YES, FRT_TERM_VECTOR_NO));
+    frt_fis_add_field(fis, frt_fi_new(compressed_field_bzip2, FRT_STORE_YES, FRT_COMPRESSION_BZ2, FRT_INDEX_YES, FRT_TERM_VECTOR_NO));
+    frt_fis_add_field(fis, frt_fi_new(compressed_field_lz4, FRT_STORE_YES, FRT_COMPRESSION_LZ4, FRT_INDEX_YES, FRT_TERM_VECTOR_NO));
     frt_index_create(store, fis);
     frt_fis_deref(fis);
     config.max_buffered_docs = 5;
 
-    iw = frt_iw_open(store, frt_whitespace_analyzer_new(false), &config);
+    iw = frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), &config);
 
     for (i = 0; i < IR_TEST_DOC_CNT; i++) {
         frt_iw_add_doc(iw, docs[i]);
@@ -1449,7 +1458,7 @@ static void test_ir_open_empty_index(TestCase *tc, void *data)
     FrtStore *store = (FrtStore *)data;
     store->clear_all(store);
     FRT_TRY
-        frt_ir_close(frt_ir_open(store));
+        frt_ir_close(frt_ir_open(NULL, store));
         Afail("IndexReader should have failed when opening empty index");
         break;
     case FRT_FILE_NOT_FOUND_ERROR:
@@ -1472,9 +1481,7 @@ static void test_ir_basic_ops(TestCase *tc, void *data)
     Atrue(frt_ir_is_latest(ir));
 }
 
-static void test_ir_term_docpos_enum_skip_to(TestCase *tc,
-                                             FrtTermDocEnum *tde,
-                                             int field_num)
+static void test_ir_term_docpos_enum_skip_to(TestCase *tc, FrtTermDocEnum *tde, int field_num)
 {
     /* test skip_to working skip interval */
     tde->seek(tde, field_num, "skip");
@@ -1708,8 +1715,8 @@ static void test_ir_term_vectors(TestCase *tc, void *data)
     Aiequal(5, tv->terms[2].positions[1]);
     Aiequal(8, tv->terms[2].positions[2]);
     Aiequal(9, tv->terms[2].positions[3]);
-    Aiequal(0,  tv->offsets[tv->terms[2].positions[0]].start);
-    Aiequal(5,  tv->offsets[tv->terms[2].positions[0]].end);
+    Aiequal(0, tv->offsets[tv->terms[2].positions[0]].start);
+    Aiequal(5, tv->offsets[tv->terms[2].positions[0]].end);
     Aiequal(30, tv->offsets[tv->terms[2].positions[1]].start);
     Aiequal(35, tv->offsets[tv->terms[2].positions[1]].end);
     Aiequal(48, tv->offsets[tv->terms[2].positions[2]].start);
@@ -1720,7 +1727,7 @@ static void test_ir_term_vectors(TestCase *tc, void *data)
     Aiequal(2, tv->terms[3].freq);
     Aiequal(1, tv->terms[3].positions[0]);
     Aiequal(6, tv->terms[3].positions[1]);
-    Aiequal(6,  tv->offsets[tv->terms[3].positions[0]].start);
+    Aiequal(6, tv->offsets[tv->terms[3].positions[0]].start);
     Aiequal(11, tv->offsets[tv->terms[3].positions[0]].end);
     Aiequal(36, tv->offsets[tv->terms[3].positions[1]].start);
     Aiequal(41, tv->offsets[tv->terms[3].positions[1]].end);
@@ -1792,48 +1799,76 @@ static void test_ir_compression(TestCase *tc, void *data)
     int i;
     FrtIndexReader *ir = (FrtIndexReader *)data;
     FrtLazyDoc *lz_doc;
-    FrtLazyDocField *lz_df1, *lz_df2;
+    FrtLazyDocField *lz_df1, *lz_df2, *lz_df3, *lz_df4;
     FrtDocument *doc = ir->get_doc(ir, 0);
-    FrtDocField *df1, *df2;
+    FrtDocField *df1, *df2, *df3, *df4;
     char buf1[20], buf2[20];
-    Aiequal(3, doc->size);
+    Aiequal(5, doc->size);
 
     df1 = frt_doc_get_field(doc, changing_field);
-    df2 = frt_doc_get_field(doc, compressed_field);
+    df2 = frt_doc_get_field(doc, compressed_field_brotli);
+    df3 = frt_doc_get_field(doc, compressed_field_bzip2);
+    df4 = frt_doc_get_field(doc, compressed_field_lz4);
     Asequal(df1->data[0], df2->data[0]);
+    Asequal(df1->data[0], df3->data[0]);
+    Asequal(df1->data[0], df4->data[0]);
     Assert(df1->lengths[0] == df2->lengths[0], "Field lengths should be equal");
-   frt_doc_destroy(doc);
+    Assert(df1->lengths[0] == df3->lengths[0], "Field lengths should be equal");
+    Assert(df1->lengths[0] == df4->lengths[0], "Field lengths should be equal");
+    frt_doc_destroy(doc);
 
     doc = ir->get_doc(ir, 2);
     df1 = frt_doc_get_field(doc, tag);
-    df2 = frt_doc_get_field(doc, compressed_field);
+    df2 = frt_doc_get_field(doc, compressed_field_brotli);
+    df3 = frt_doc_get_field(doc, compressed_field_bzip2);
+    df4 = frt_doc_get_field(doc, compressed_field_lz4);
     for (i = 0; i < 4; i++) {
         Asequal(df1->data[i], df2->data[i]);
+        Asequal(df1->data[i], df3->data[i]);
+        Asequal(df1->data[i], df4->data[i]);
         Assert(df1->lengths[i] == df2->lengths[i], "Field lengths not equal");
+        Assert(df1->lengths[i] == df3->lengths[i], "Field lengths not equal");
+        Assert(df1->lengths[i] == df4->lengths[i], "Field lengths not equal");
     }
-   frt_doc_destroy(doc);
+    frt_doc_destroy(doc);
 
     lz_doc = ir->get_lazy_doc(ir, 0);
     lz_df1 = frt_lazy_doc_get(lz_doc, changing_field);
-    lz_df2 = frt_lazy_doc_get(lz_doc, compressed_field);
+    lz_df2 = frt_lazy_doc_get(lz_doc, compressed_field_brotli);
+    lz_df3 = frt_lazy_doc_get(lz_doc, compressed_field_bzip2);
+    lz_df4 = frt_lazy_doc_get(lz_doc, compressed_field_lz4);
     Asequal(frt_lazy_df_get_data(lz_df1, 0), frt_lazy_df_get_data(lz_df2, 0));
+    Asequal(frt_lazy_df_get_data(lz_df1, 0), frt_lazy_df_get_data(lz_df3, 0));
+    Asequal(frt_lazy_df_get_data(lz_df1, 0), frt_lazy_df_get_data(lz_df4, 0));
     frt_lazy_doc_close(lz_doc);
 
     lz_doc = ir->get_lazy_doc(ir, 2);
     lz_df1 = frt_lazy_doc_get(lz_doc, tag);
-    lz_df2 = frt_lazy_doc_get(lz_doc, compressed_field);
+    lz_df2 = frt_lazy_doc_get(lz_doc, compressed_field_brotli);
+    lz_df3 = frt_lazy_doc_get(lz_doc, compressed_field_bzip2);
+    lz_df4 = frt_lazy_doc_get(lz_doc, compressed_field_lz4);
     for (i = 0; i < 4; i++) {
         Asequal(frt_lazy_df_get_data(lz_df1, i), frt_lazy_df_get_data(lz_df2, i));
+        Asequal(frt_lazy_df_get_data(lz_df1, i), frt_lazy_df_get_data(lz_df3, i));
+        Asequal(frt_lazy_df_get_data(lz_df1, i), frt_lazy_df_get_data(lz_df4, i));
     }
     frt_lazy_doc_close(lz_doc);
 
     lz_doc = ir->get_lazy_doc(ir, 2);
     lz_df1 = frt_lazy_doc_get(lz_doc, tag);
-    lz_df2 = frt_lazy_doc_get(lz_doc, compressed_field);
+    lz_df2 = frt_lazy_doc_get(lz_doc, compressed_field_brotli);
+    lz_df3 = frt_lazy_doc_get(lz_doc, compressed_field_bzip2);
+    lz_df4 = frt_lazy_doc_get(lz_doc, compressed_field_lz4);
     frt_lazy_df_get_bytes(lz_df1, buf1, 5, 11);
     frt_lazy_df_get_bytes(lz_df2, buf2, 5, 11);
     buf2[11] = buf1[11] = '\0';
     Asequal(buf1, buf2);
+    frt_lazy_df_get_bytes(lz_df3, buf2, 5, 11);
+    buf2[11] = buf1[11] = '\0';
+    Asequal(buf1, buf2);
+    frt_lazy_df_get_bytes(lz_df4, buf2, 5, 11);
+    buf2[11] = buf1[11] = '\0';
+    Asequal(buf1, buf2);
     frt_lazy_doc_close(lz_doc);
 }
 
@@ -1923,7 +1958,7 @@ static void test_ir_norms(TestCase *tc, void *data)
     frt_ir_commit(ir);
 
     for (i = 0; i < rte->store_cnt; i++) {
-        iw = frt_iw_open(rte->stores[i], frt_whitespace_analyzer_new(false),
+        iw = frt_iw_open(NULL, rte->stores[i], frt_whitespace_analyzer_new(false),
                      &frt_default_config);
         frt_iw_optimize(iw);
         frt_iw_close(iw);
@@ -1958,7 +1993,7 @@ static void test_ir_norms(TestCase *tc, void *data)
 static void test_ir_delete(TestCase *tc, void *data)
 {
     int i;
-    FrtStore *store = frt_open_ram_store();
+    FrtStore *store = frt_open_ram_store(NULL);
     FrtIndexReader *ir, *ir2;
     FrtIndexWriter *iw;
     int type = *((int *)data);
@@ -2050,8 +2085,7 @@ static void test_ir_delete(TestCase *tc, void *data)
     frt_ir_commit(ir);
 
     for (i = 0; i < rte->store_cnt; i++) {
-        iw = frt_iw_open(rte->stores[i], frt_whitespace_analyzer_new(false),
-                     &frt_default_config);
+        iw = frt_iw_open(NULL, rte->stores[i], frt_whitespace_analyzer_new(false), &frt_default_config);
         frt_iw_optimize(iw);
         frt_iw_close(iw);
     }
@@ -2080,11 +2114,11 @@ static void test_ir_read_while_optimizing(TestCase *tc, void *data)
 
     write_ir_test_docs(store);
 
-    ir = frt_ir_open(store);
+    ir = frt_ir_open(NULL, store);
 
     test_ir_term_doc_enum(tc, ir);
 
-    iw = frt_iw_open(store, frt_whitespace_analyzer_new(false), false);
+    iw = frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), false);
     frt_iw_optimize(iw);
     frt_iw_close(iw);
 
@@ -2101,23 +2135,23 @@ static void test_ir_multivalue_fields(TestCase *tc, void *data)
     FrtDocument *doc = frt_doc_new();
     FrtDocField *df;
     FrtIndexWriter *iw;
-    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_INDEX_YES,
-                              FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
+    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
     const char *body_text = "this is the body FrtDocument Field";
     const char *title_text = "this is the title FrtDocument Field";
     const char *author_text = "this is the author FrtDocument Field";
+    rb_encoding *enc = rb_enc_find("ASCII-8BIT");
 
     frt_index_create(store, fis);
     frt_fis_deref(fis);
-    iw = frt_iw_open(store, frt_whitespace_analyzer_new(false), NULL);
+    iw = frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), NULL);
 
-    df = frt_doc_add_field(doc, frt_df_add_data(frt_df_new(tag), (char *)"Ruby"));
-    frt_df_add_data(df, (char *)"C");
-    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(body), (char *)body_text));
-    frt_df_add_data(df, (char *)"Lucene");
-    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(title), (char *)title_text));
-    frt_df_add_data(df, (char *)"Ferret");
-    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(author), (char *)author_text));
+    df = frt_doc_add_field(doc, frt_df_add_data(frt_df_new(tag), (char *)"Ruby", enc));
+    frt_df_add_data(df, (char *)"C", enc);
+    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(body), (char *)body_text, enc));
+    frt_df_add_data(df, (char *)"Lucene", enc);
+    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(title), (char *)title_text, enc));
+    frt_df_add_data(df, (char *)"Ferret", enc);
+    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(author), (char *)author_text, enc));
 
     Aiequal(0, iw->fis->size);
 
@@ -2136,7 +2170,7 @@ static void test_ir_multivalue_fields(TestCase *tc, void *data)
    frt_doc_destroy(doc);
     frt_iw_close(iw);
 
-    ir = frt_ir_open(store);
+    ir = frt_ir_open(NULL, store);
 
     doc = ir->get_doc(ir, 0);
     Aiequal(4, doc->size);
@@ -2172,18 +2206,20 @@ static void test_ir_multivalue_fields(TestCase *tc, void *data)
 TestSuite *ts_index(TestSuite *suite)
 {
     FrtIndexReader *ir;
-    FrtStore *fs_store, *store = frt_open_ram_store();
+    FrtStore *fs_store, *store = frt_open_ram_store(NULL);
     ReaderTestEnvironment *rte = NULL;
     /* FrtStore *store = frt_open_fs_store(TEST_DIR); */
 
-    /* initialize FrtSymbols */
+    /* initialize IDs */
     body             = rb_intern("body");
     title            = rb_intern("title");
     text             = rb_intern("text");
     author           = rb_intern("author");
     year             = rb_intern("year");
     changing_field   = rb_intern("changing_field");
-    compressed_field = rb_intern("compressed_field");
+    compressed_field_brotli = rb_intern("compressed_field_brotli");
+    compressed_field_bzip2 = rb_intern("compressed_field_bzip2");
+    compressed_field_lz4 = rb_intern("compressed_field_lz4");
     tag              = rb_intern("tag");
 
     srand(5);
@@ -2233,25 +2269,16 @@ TestSuite *ts_index(TestSuite *suite)
     rte = reader_test_env_new(multi_reader_type);
     ir = reader_test_env_ir_open(rte);
 
-    tst_run_test_with_name(suite, test_ir_basic_ops, ir,
-                           "test_multi_reader_basic_ops");
-    tst_run_test_with_name(suite, test_ir_get_doc, ir,
-                           "test_multi_get_doc");
-    tst_run_test_with_name(suite, test_ir_compression, ir,
-                           "test_multi_compression");
-    tst_run_test_with_name(suite, test_ir_term_enum, ir,
-                           "test_multi_term_enum");
-    tst_run_test_with_name(suite, test_ir_term_doc_enum, ir,
-                           "test_multi_term_doc_enum");
-    tst_run_test_with_name(suite, test_ir_term_vectors, ir,
-                           "test_multi_term_vectors");
-    tst_run_test_with_name(suite, test_ir_mtdpe, ir,
-                           "test_multi_multiple_term_doc_pos_enum");
-
-    tst_run_test_with_name(suite, test_ir_norms, &multi_reader_type,
-                           "test_multi_norms");
-    tst_run_test_with_name(suite, test_ir_delete, &multi_reader_type,
-                           "test_multi_reader_delete");
+    tst_run_test_with_name(suite, test_ir_basic_ops, ir, "test_multi_reader_basic_ops");
+    tst_run_test_with_name(suite, test_ir_get_doc, ir, "test_multi_get_doc");
+    tst_run_test_with_name(suite, test_ir_compression, ir, "test_multi_compression");
+    tst_run_test_with_name(suite, test_ir_term_enum, ir, "test_multi_term_enum");
+    tst_run_test_with_name(suite, test_ir_term_doc_enum, ir, "test_multi_term_doc_enum");
+    tst_run_test_with_name(suite, test_ir_term_vectors, ir, "test_multi_term_vectors");
+    tst_run_test_with_name(suite, test_ir_mtdpe, ir, "test_multi_multiple_term_doc_pos_enum");
+
+    tst_run_test_with_name(suite, test_ir_norms, &multi_reader_type, "test_multi_norms");
+    tst_run_test_with_name(suite, test_ir_delete, &multi_reader_type, "test_multi_reader_delete");
     frt_ir_close(ir);
     reader_test_env_destroy(rte);
 
@@ -2259,25 +2286,16 @@ TestSuite *ts_index(TestSuite *suite)
     rte = reader_test_env_new(multi_external_reader_type);
     ir = reader_test_env_ir_open(rte);
 
-    tst_run_test_with_name(suite, test_ir_basic_ops, ir,
-                           "test_multi_ext_reader_basic_ops");
-    tst_run_test_with_name(suite, test_ir_get_doc, ir,
-                           "test_multi_ext_get_doc");
-    tst_run_test_with_name(suite, test_ir_compression, ir,
-                           "test_multi_ext_compression");
-    tst_run_test_with_name(suite, test_ir_term_enum, ir,
-                           "test_multi_ext_term_enum");
-    tst_run_test_with_name(suite, test_ir_term_doc_enum, ir,
-                           "test_multi_ext_term_doc_enum");
-    tst_run_test_with_name(suite, test_ir_term_vectors, ir,
-                           "test_multi_ext_term_vectors");
-    tst_run_test_with_name(suite, test_ir_mtdpe, ir,
-                           "test_multi_ext_multiple_term_doc_pos_enum");
-
-    tst_run_test_with_name(suite, test_ir_norms, &multi_external_reader_type,
-                           "test_multi_ext_norms");
-    tst_run_test_with_name(suite, test_ir_delete, &multi_external_reader_type,
-                           "test_multi_ext_reader_delete");
+    tst_run_test_with_name(suite, test_ir_basic_ops, ir, "test_multi_ext_reader_basic_ops");
+    tst_run_test_with_name(suite, test_ir_get_doc, ir, "test_multi_ext_get_doc");
+    tst_run_test_with_name(suite, test_ir_compression, ir, "test_multi_ext_compression");
+    tst_run_test_with_name(suite, test_ir_term_enum, ir, "test_multi_ext_term_enum");
+    tst_run_test_with_name(suite, test_ir_term_doc_enum, ir, "test_multi_ext_term_doc_enum");
+    tst_run_test_with_name(suite, test_ir_term_vectors, ir, "test_multi_ext_term_vectors");
+    tst_run_test_with_name(suite, test_ir_mtdpe, ir, "test_multi_ext_multiple_term_doc_pos_enum");
+
+    tst_run_test_with_name(suite, test_ir_norms, &multi_external_reader_type, "test_multi_ext_norms");
+    tst_run_test_with_name(suite, test_ir_delete, &multi_external_reader_type, "test_multi_ext_reader_delete");
 
     frt_ir_close(ir);
     reader_test_env_destroy(rte);
@@ -2286,36 +2304,25 @@ TestSuite *ts_index(TestSuite *suite)
     rte = reader_test_env_new(add_indexes_reader_type);
     ir = reader_test_env_ir_open(rte);
 
-    tst_run_test_with_name(suite, test_ir_basic_ops, ir,
-                           "test_add_indexes_reader_basic_ops");
-    tst_run_test_with_name(suite, test_ir_get_doc, ir,
-                           "test_add_indexes_get_doc");
-    tst_run_test_with_name(suite, test_ir_compression, ir,
-                           "test_add_indexes_compression");
-    tst_run_test_with_name(suite, test_ir_term_enum, ir,
-                           "test_add_indexes_term_enum");
-    tst_run_test_with_name(suite, test_ir_term_doc_enum, ir,
-                           "test_add_indexes_term_doc_enum");
-    tst_run_test_with_name(suite, test_ir_term_vectors, ir,
-                           "test_add_indexes_term_vectors");
-    tst_run_test_with_name(suite, test_ir_mtdpe, ir,
-                           "test_add_indexes_multiple_term_doc_pos_enum");
-
-    tst_run_test_with_name(suite, test_ir_norms, &add_indexes_reader_type,
-                           "test_add_indexes_norms");
-    tst_run_test_with_name(suite, test_ir_delete, &add_indexes_reader_type,
-                           "test_add_indexes_reader_delete");
+    tst_run_test_with_name(suite, test_ir_basic_ops, ir, "test_add_indexes_reader_basic_ops");
+    tst_run_test_with_name(suite, test_ir_get_doc, ir, "test_add_indexes_get_doc");
+    tst_run_test_with_name(suite, test_ir_compression, ir, "test_add_indexes_compression");
+    tst_run_test_with_name(suite, test_ir_term_enum, ir, "test_add_indexes_term_enum");
+    tst_run_test_with_name(suite, test_ir_term_doc_enum, ir, "test_add_indexes_term_doc_enum");
+    tst_run_test_with_name(suite, test_ir_term_vectors, ir, "test_add_indexes_term_vectors");
+    tst_run_test_with_name(suite, test_ir_mtdpe, ir, "test_add_indexes_multiple_term_doc_pos_enum");
+
+    tst_run_test_with_name(suite, test_ir_norms, &add_indexes_reader_type, "test_add_indexes_norms");
+    tst_run_test_with_name(suite, test_ir_delete, &add_indexes_reader_type, "test_add_indexes_reader_delete");
 
     frt_ir_close(ir);
     reader_test_env_destroy(rte);
 
     /* Other FrtIndexReader Tests */
-    tst_run_test_with_name(suite, test_ir_read_while_optimizing, store,
-                           "test_ir_read_while_optimizing_in_ram");
+    tst_run_test_with_name(suite, test_ir_read_while_optimizing, store, "test_ir_read_while_optimizing_in_ram");
 
     fs_store = frt_open_fs_store(TEST_DIR);
-    tst_run_test_with_name(suite, test_ir_read_while_optimizing, fs_store,
-                           "test_ir_read_while_optimizing_on_disk");
+    tst_run_test_with_name(suite, test_ir_read_while_optimizing, fs_store, "test_ir_read_while_optimizing_on_disk");
     fs_store->clear_all(fs_store);
     frt_store_deref(fs_store);