isomorfeus-ferret 0.12.7 → 0.13.0

data/ext/isomorfeus_ferret_ext/test_compound_io.c

@@ -155,7 +155,7 @@ void test_compound_io_many_files(TestCase *tc, void *data)
 
 TestSuite *ts_compound_io(TestSuite *suite)
 {
-    FrtStore *store = frt_open_ram_store();
+    FrtStore *store = frt_open_ram_store(NULL);
 
     suite = ADD_SUITE(suite);
 

data/ext/isomorfeus_ferret_ext/test_document.c

@@ -6,8 +6,9 @@ void test_df_standard(TestCase *tc, void *data)
     char *s;
     FrtDocField *df;
     (void)data;
+    rb_encoding *enc = rb_enc_find("ASCII-8BIT");
 
-    df = frt_df_add_data(frt_df_new(rb_intern("title")), frt_estrdup("Life of Pi"));
+    df = frt_df_add_data(frt_df_new(rb_intern("title")), frt_estrdup("Life of Pi"), enc);
     df->destroy_data = true;
     Aiequal(1, df->size);
     Asequal("title", rb_id2name(df->name));
@@ -18,7 +19,7 @@ void test_df_standard(TestCase *tc, void *data)
     free(s);
     frt_df_destroy(df);
 
-    df = frt_df_add_data_len(frt_df_new(rb_intern("title")), (char *)"new title", 9);
+    df = frt_df_add_data_len(frt_df_new(rb_intern("title")), (char *)"new title", 9, enc);
     Aiequal(1, df->size);
     Asequal("title", rb_id2name(df->name));
     Asequal("new title", df->data[0]);
@@ -32,20 +33,21 @@ void test_df_multi_fields(TestCase *tc, void *data)
     char *s;
     FrtDocField *df;
     (void)data;
+    rb_encoding *enc = rb_enc_find("ASCII-8BIT");
 
-    df = frt_df_add_data(frt_df_new(rb_intern("title")), frt_estrdup("Vernon God Little"));
+    df = frt_df_add_data(frt_df_new(rb_intern("title")), frt_estrdup("Vernon God Little"), enc);
     df->destroy_data = true;
     Aiequal(1, df->size);
     Asequal("title", rb_id2name(df->name));
     Asequal("Vernon God Little", df->data[0]);
     Aiequal(strlen("Vernon God Little"), df->lengths[0]);
 
-    frt_df_add_data(df, frt_estrdup("some more data"));
+    frt_df_add_data(df, frt_estrdup("some more data"), enc);
     Aiequal(2, df->size);
     Asequal("title: [\"Vernon God Little\", \"some more data\"]",
             s = frt_df_to_s(df));
     free(s);
-    frt_df_add_data_len(df, frt_estrdup("and more data"), 14);
+    frt_df_add_data_len(df, frt_estrdup("and more data"), 14, enc);
     Aiequal(3, df->size);
     Asequal("title", rb_id2name(df->name));
     Asequal("Vernon God Little", df->data[0]);
@@ -54,13 +56,13 @@ void test_df_multi_fields(TestCase *tc, void *data)
 
     frt_df_destroy(df);
 
-    df = frt_df_add_data(frt_df_new(rb_intern("data")), frt_estrdup("start"));
+    df = frt_df_add_data(frt_df_new(rb_intern("data")), frt_estrdup("start"), enc);
     df->destroy_data = true;
     Aiequal(1, df->size);
     for (i = 0; i < 1000; i++) {
         char buf[100];
         sprintf(buf, "<<%d>>", i);
-        frt_df_add_data(df, frt_estrdup(buf));
+        frt_df_add_data(df, frt_estrdup(buf), enc);
         Aiequal(i + 2, df->size);
     }
     frt_df_destroy(df);
@@ -72,14 +74,15 @@ void test_doc(TestCase *tc, void *data)
     FrtDocument *doc;
     FrtDocField *df;
     (void)data;
+    rb_encoding *enc = rb_enc_find("ASCII-8BIT");
 
     doc = frt_doc_new();
-    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("title")), (char *)"title"));
+    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("title")), (char *)"title", enc));
     Aiequal(1, doc->size);
-    df = frt_df_add_data(frt_df_new(rb_intern("data")), (char *)"data1");
-    frt_df_add_data(df, (char *)"data2");
-    frt_df_add_data(df, (char *)"data3");
-    frt_df_add_data(df, (char *)"data4");
+    df = frt_df_add_data(frt_df_new(rb_intern("data")), (char *)"data1", enc);
+    frt_df_add_data(df, (char *)"data2", enc);
+    frt_df_add_data(df, (char *)"data3", enc);
+    frt_df_add_data(df, (char *)"data4", enc);
     frt_doc_add_field(doc, df);
     Aiequal(2, doc->size);
     Asequal("title", rb_id2name(frt_doc_get_field(doc, rb_intern("title"))->name));
@@ -100,7 +103,7 @@ void test_doc(TestCase *tc, void *data)
         char *bufc;
         sprintf(buf, "<<%d>>", i);
         bufc = frt_estrdup(buf);
-        df = frt_df_add_data(frt_df_new(rb_intern(bufc)), bufc);
+        df = frt_df_add_data(frt_df_new(rb_intern(bufc)), bufc, enc);
         df->destroy_data = true;
         frt_doc_add_field(doc, df);
         Aiequal(i + 1, doc->size);
@@ -122,12 +125,13 @@ void test_double_field_exception(TestCase *tc, void *data)
     FrtDocument *doc;
     FrtDocField *volatile df = NULL;
     (void)data;
+    rb_encoding *enc = rb_enc_find("ASCII-8BIT");
 
     doc = frt_doc_new();
-    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("title")), (char *)"title"));
+    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("title")), (char *)"title", enc));
 
     FRT_TRY
-        df = frt_df_add_data_len(frt_df_new(rb_intern("title")), (char *)"title", 5);
+        df = frt_df_add_data_len(frt_df_new(rb_intern("title")), (char *)"title", 5, enc);
         frt_doc_add_field(doc, df);
     case FRT_EXCEPTION:
         exception_thrown = true;

data/ext/isomorfeus_ferret_ext/test_except.c

@@ -5,8 +5,7 @@
 static const char *msg1 = "Message One";
 static const char *msg2 = "Message Two";
 
-static void raise_exception()
-{
+static void raise_exception(void) {
     FRT_RAISE(FRT_EXCEPTION, "%s", msg1);
 }
 

data/ext/isomorfeus_ferret_ext/test_fields.c

@@ -13,7 +13,7 @@
 void field_prop_test(TestCase *tc,
                      int line_num,
                      FrtFieldInfo *fi,
-                     FrtSymbol name,
+                     ID name,
                      float boost,
                      bool is_stored,
                      bool is_compressed,
@@ -22,8 +22,7 @@ void field_prop_test(TestCase *tc,
                      bool omit_norms,
                      bool store_term_vector,
                      bool store_positions,
-                     bool store_offsets)
-{
+                     bool store_offsets) {
     tst_ptr_equal(line_num, tc, (void *)name, (void *)fi->name);
     tst_flt_equal(line_num, tc, boost, fi->boost);
     tst_int_equal(line_num, tc, is_stored,          fi_is_stored(fi));
@@ -46,19 +45,19 @@ static void test_fi_new(TestCase *tc, void *data)
 {
     FrtFieldInfo *fi;
     (void)data; /* suppress unused argument warning */
-    fi = frt_fi_new(rb_intern("name"), FRT_STORE_NO, FRT_INDEX_NO, FRT_TERM_VECTOR_NO);
+    fi = frt_fi_new(rb_intern("name"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_NO, FRT_TERM_VECTOR_NO);
     do_field_prop_test(tc, fi, rb_intern("name"), 1.0, F, F, F, F, F, F, F, F);
     frt_fi_deref(fi);
-    fi = frt_fi_new(rb_intern("name"), FRT_STORE_YES, FRT_INDEX_YES, FRT_TERM_VECTOR_YES);
+    fi = frt_fi_new(rb_intern("name"), FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_YES);
     do_field_prop_test(tc, fi, rb_intern("name"), 1.0, T, F, T, T, F, T, F, F);
     frt_fi_deref(fi);
-    fi = frt_fi_new(rb_intern("name"), FRT_STORE_COMPRESS, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_POSITIONS);
+    fi = frt_fi_new(rb_intern("name"), FRT_STORE_YES, FRT_COMPRESSION_BROTLI, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_POSITIONS);
     do_field_prop_test(tc, fi, rb_intern("name"), 1.0, T, T, T, F, F, T, T, F);
     frt_fi_deref(fi);
-    fi = frt_fi_new(rb_intern("name"), FRT_STORE_NO, FRT_INDEX_YES_OMIT_NORMS, FRT_TERM_VECTOR_WITH_OFFSETS);
+    fi = frt_fi_new(rb_intern("name"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_YES_OMIT_NORMS, FRT_TERM_VECTOR_WITH_OFFSETS);
     do_field_prop_test(tc, fi, rb_intern("name"), 1.0, F, F, T, T, T, T, F, T);
     frt_fi_deref(fi);
-    fi = frt_fi_new(rb_intern("name"), FRT_STORE_NO, FRT_INDEX_UNTOKENIZED_OMIT_NORMS, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
+    fi = frt_fi_new(rb_intern("name"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_UNTOKENIZED_OMIT_NORMS, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
     fi->boost = 1000.0;
     do_field_prop_test(tc, fi, rb_intern("name"), 1000.0, F, F, T, F, T, T, T, T);
     frt_fi_deref(fi);
@@ -77,14 +76,14 @@ static void test_fis_basic(TestCase *tc, void *data)
     volatile bool arg_error = false;
     (void)data; /* suppress unused argument warning */
 
-    fis = frt_fis_new(FRT_STORE_NO, FRT_INDEX_NO, FRT_TERM_VECTOR_NO);
-    frt_fis_add_field(fis, frt_fi_new(rb_intern("FFFFFFFF"), FRT_STORE_NO, FRT_INDEX_NO, FRT_TERM_VECTOR_NO));
-    frt_fis_add_field(fis, frt_fi_new(rb_intern("TFTTFTFF"), FRT_STORE_YES, FRT_INDEX_YES, FRT_TERM_VECTOR_YES));
-    frt_fis_add_field(fis, frt_fi_new(rb_intern("TTTFFTTF"), FRT_STORE_COMPRESS, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_POSITIONS));
-    frt_fis_add_field(fis, frt_fi_new(rb_intern("FFTTTTFT"), FRT_STORE_NO, FRT_INDEX_YES_OMIT_NORMS, FRT_TERM_VECTOR_WITH_OFFSETS));
-    frt_fis_add_field(fis, frt_fi_new(rb_intern("FFTFTTTT"), FRT_STORE_NO, FRT_INDEX_UNTOKENIZED_OMIT_NORMS, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS));
+    fis = frt_fis_new(FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_NO, FRT_TERM_VECTOR_NO);
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("FFFFFFFF"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_NO, FRT_TERM_VECTOR_NO));
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("TFTTFTFF"), FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_YES));
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("TTTFFTTF"), FRT_STORE_YES, FRT_COMPRESSION_BROTLI, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_POSITIONS));
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("FFTTTTFT"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_YES_OMIT_NORMS, FRT_TERM_VECTOR_WITH_OFFSETS));
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("FFTFTTTT"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_UNTOKENIZED_OMIT_NORMS, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS));
 
-    fi = frt_fi_new(rb_intern("FFTFTTTT"), FRT_STORE_NO, FRT_INDEX_UNTOKENIZED_OMIT_NORMS, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
+    fi = frt_fi_new(rb_intern("FFTFTTTT"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_UNTOKENIZED_OMIT_NORMS, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
     FRT_TRY
         Apnull(frt_fis_add_field(fis, fi));
     case FRT_ARG_ERROR:
@@ -132,7 +131,7 @@ static void test_fis_with_default(TestCase *tc, void *data)
     FrtFieldInfos *fis;
     (void)data; /* suppress unused argument warning */
 
-    fis = frt_fis_new(FRT_STORE_NO, FRT_INDEX_NO, FRT_TERM_VECTOR_NO);
+    fis = frt_fis_new(FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_NO, FRT_TERM_VECTOR_NO);
     do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("name")), rb_intern("name"), 1.0, F, F, F, F, F, F, F, F);
     do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("dave")), rb_intern("dave"), 1.0, F, F, F, F, F, F, F, F);
     do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("wert")), rb_intern("wert"), 1.0, F, F, F, F, F, F, F, F);
@@ -142,16 +141,16 @@ static void test_fis_with_default(TestCase *tc, void *data)
     Apnull(frt_fis_get_field(fis, rb_intern("random")));
     frt_fis_deref(fis);
 
-    fis = frt_fis_new(FRT_STORE_YES, FRT_INDEX_YES, FRT_TERM_VECTOR_YES);
+    fis = frt_fis_new(FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_YES);
     do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("name")), rb_intern("name"), 1.0, T, F, T, T, F, T, F, F);
     frt_fis_deref(fis);
-    fis = frt_fis_new(FRT_STORE_COMPRESS, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_POSITIONS);
+    fis = frt_fis_new(FRT_STORE_YES, FRT_COMPRESSION_BROTLI, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_POSITIONS);
     do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("name")), rb_intern("name"), 1.0, T, T, T, F, F, T, T, F);
     frt_fis_deref(fis);
-    fis = frt_fis_new(FRT_STORE_NO, FRT_INDEX_YES_OMIT_NORMS, FRT_TERM_VECTOR_WITH_OFFSETS);
+    fis = frt_fis_new(FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_YES_OMIT_NORMS, FRT_TERM_VECTOR_WITH_OFFSETS);
     do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("name")), rb_intern("name"), 1.0, F, F, T, T, T, T, F, T);
     frt_fis_deref(fis);
-    fis = frt_fis_new(FRT_STORE_NO, FRT_INDEX_UNTOKENIZED_OMIT_NORMS, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
+    fis = frt_fis_new(FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_UNTOKENIZED_OMIT_NORMS, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
     do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("name")), rb_intern("name"), 1.0, F, F, T, F, T, T, T, T);
     frt_fis_deref(fis);
 }
@@ -160,18 +159,17 @@ static void test_fis_rw(TestCase *tc, void *data)
 {
     char *str;
     FrtFieldInfos *fis;
-    FrtStore *store = frt_open_ram_store();
+    FrtStore *store = frt_open_ram_store(NULL);
     FrtInStream *is;
     FrtOutStream *os;
     (void)data; /* suppress unused argument warning */
 
-    fis = frt_fis_new(FRT_STORE_YES, FRT_INDEX_UNTOKENIZED_OMIT_NORMS,
-                  FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
-    frt_fis_add_field(fis, frt_fi_new(rb_intern("FFFFFFFF"), FRT_STORE_NO, FRT_INDEX_NO, FRT_TERM_VECTOR_NO));
-    frt_fis_add_field(fis, frt_fi_new(rb_intern("TFTTFTFF"), FRT_STORE_YES, FRT_INDEX_YES, FRT_TERM_VECTOR_YES));
-    frt_fis_add_field(fis, frt_fi_new(rb_intern("TTTFFTTF"), FRT_STORE_COMPRESS, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_POSITIONS));
-    frt_fis_add_field(fis, frt_fi_new(rb_intern("FFTTTTFT"), FRT_STORE_NO, FRT_INDEX_YES_OMIT_NORMS, FRT_TERM_VECTOR_WITH_OFFSETS));
-    frt_fis_add_field(fis, frt_fi_new(rb_intern("FFTFTTTT"), FRT_STORE_NO, FRT_INDEX_UNTOKENIZED_OMIT_NORMS, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS));
+    fis = frt_fis_new(FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_UNTOKENIZED_OMIT_NORMS, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("FFFFFFFF"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_NO, FRT_TERM_VECTOR_NO));
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("TFTTFTFF"), FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_YES));
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("TTTFFTTF"), FRT_STORE_YES, FRT_COMPRESSION_BROTLI, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_POSITIONS));
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("FFTTTTFT"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_YES_OMIT_NORMS, FRT_TERM_VECTOR_WITH_OFFSETS));
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("FFTFTTTT"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_UNTOKENIZED_OMIT_NORMS, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS));
     fis->fields[1]->boost = 2.0;
     fis->fields[2]->boost = 3.0;
     fis->fields[3]->boost = 4.0;
@@ -282,45 +280,44 @@ static char *prepare_bin_data(int len)
         Assert(memcmp(mdata, df->data[index], mlen) == 0, "Data should be equal");\
     } while (0)
 
-static FrtDocument *prepare_doc()
-{
+static FrtDocument *prepare_doc(void) {
     FrtDocument *doc = frt_doc_new();
     FrtDocField *df;
     char *bin_data = prepare_bin_data(BIN_DATA_LEN);
+    rb_encoding *enc = rb_enc_find("ASCII-8BIT");
 
-    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("ignored")), (char *)"this fld's ignored"));
-    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("unstored")), (char *)"unstored ignored"));
-    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("stored")), (char *)"Yay, a stored field"));
-    df = frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("stored_array")), (char *)"one"));
+    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("ignored")), (char *)"this fld's ignored", enc));
+    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("unstored")), (char *)"unstored ignored", enc));
+    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("stored")), (char *)"Yay, a stored field", enc));
+    df = frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("stored_array")), (char *)"one", enc));
     df->destroy_data = false;
-    frt_df_add_data(df, (char *)"two");
-    frt_df_add_data(df, (char *)"three");
-    frt_df_add_data(df, (char *)"four");
-    frt_df_add_data_len(df, bin_data, BIN_DATA_LEN);
+    frt_df_add_data(df, (char *)"two", enc);
+    frt_df_add_data(df, (char *)"three", enc);
+    frt_df_add_data(df, (char *)"four", enc);
+    frt_df_add_data_len(df, bin_data, BIN_DATA_LEN, enc);
     frt_doc_add_field(doc, frt_df_add_data_len(frt_df_new(rb_intern("binary")), bin_data,
-                                       BIN_DATA_LEN))->destroy_data = true;
-    df = frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("array")), (char *)"ichi"));
-    frt_df_add_data(df, (char *)"ni");
-    frt_df_add_data(df, (char *)"san");
-    frt_df_add_data(df, (char *)"yon");
-    frt_df_add_data(df, (char *)"go");
+                                       BIN_DATA_LEN, enc))->destroy_data = true;
+    df = frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("array")), (char *)"ichi", enc));
+    frt_df_add_data(df, (char *)"ni", enc);
+    frt_df_add_data(df, (char *)"san", enc);
+    frt_df_add_data(df, (char *)"yon", enc);
+    frt_df_add_data(df, (char *)"go", enc);
 
     return doc;
 }
 
-static FrtFieldInfos *prepare_fis()
-{
-    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_INDEX_YES, FRT_TERM_VECTOR_NO);
-    frt_fis_add_field(fis, frt_fi_new(rb_intern("ignored"), FRT_STORE_NO, FRT_INDEX_NO, FRT_TERM_VECTOR_NO));
-    frt_fis_add_field(fis, frt_fi_new(rb_intern("unstored"), FRT_STORE_NO, FRT_INDEX_YES, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS));
-    frt_fis_add_field(fis, frt_fi_new(rb_intern("stored"), FRT_STORE_YES, FRT_INDEX_YES, FRT_TERM_VECTOR_YES));
-    frt_fis_add_field(fis, frt_fi_new(rb_intern("stored_array"), FRT_STORE_COMPRESS, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_NO));
+static FrtFieldInfos *prepare_fis(void) {
+    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_NO);
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("ignored"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_NO, FRT_TERM_VECTOR_NO));
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("unstored"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS));
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("stored"), FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_YES));
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("stored_array"), FRT_STORE_YES, FRT_COMPRESSION_BROTLI, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_NO));
     return fis;
 }
 
 static void test_fields_rw_single(TestCase *tc, void *data)
 {
-    FrtStore *store = frt_open_ram_store();
+    FrtStore *store = frt_open_ram_store(NULL);
     char *bin_data = prepare_bin_data(BIN_DATA_LEN);
     FrtDocument *doc = prepare_doc();
     FrtFieldInfos *fis = prepare_fis();
@@ -384,7 +381,7 @@ static void test_fields_rw_single(TestCase *tc, void *data)
 static void test_fields_rw_multi(TestCase *tc, void *data)
 {
     int i;
-    FrtStore *store = frt_open_ram_store();
+    FrtStore *store = frt_open_ram_store(NULL);
     char *bin_data = prepare_bin_data(BIN_DATA_LEN);
     FrtDocument *doc;
     FrtFieldInfos *fis = prepare_fis();
@@ -392,6 +389,7 @@ static void test_fields_rw_multi(TestCase *tc, void *data)
     FrtFieldsReader *fr;
     FrtDocField *df;
     (void)data;
+    rb_encoding *enc = rb_enc_find("ASCII-8BIT");
 
     fw = frt_fw_open(store, "_as3", fis);
     for (i = 0; i < 100; i++) {
@@ -400,7 +398,7 @@ static void test_fields_rw_multi(TestCase *tc, void *data)
         sprintf(buf, "<<%d>>", i);
         bufc = frt_estrdup(buf);
         doc = frt_doc_new();
-        frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern(bufc)), bufc));
+        frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern(bufc)), bufc, enc));
         frt_fw_add_doc(fw, doc);
         frt_fw_write_tv_index(fw);
        frt_doc_destroy(doc);
@@ -462,7 +460,7 @@ static void test_fields_rw_multi(TestCase *tc, void *data)
 
 static void test_lazy_field_loading(TestCase *tc, void *data)
 {
-    FrtStore *store = frt_open_ram_store();
+    FrtStore *store = frt_open_ram_store(NULL);
     FrtDocument *doc;
     FrtFieldInfos *fis = prepare_fis();
     FrtFieldsWriter *fw;
@@ -472,15 +470,16 @@ static void test_lazy_field_loading(TestCase *tc, void *data)
     FrtLazyDocField *lazy_df;
     char *text, buf[1000];
     (void)data;
+    rb_encoding *enc = rb_enc_find("ASCII-8BIT");
 
     fw = frt_fw_open(store, "_as3", fis);
     doc = frt_doc_new();
     df = frt_df_new(rb_intern("stored"));
-    frt_df_add_data(df, (char *)"this is a stored field");
-    frt_df_add_data(df, (char *)"to be or not to be");
-    frt_df_add_data(df, (char *)"a stitch in time, saves nine");
-    frt_df_add_data(df, (char *)"the quick brown fox jumped over the lazy dog");
-    frt_df_add_data(df, (char *)"that's it folks");
+    frt_df_add_data(df, (char *)"this is a stored field", enc);
+    frt_df_add_data(df, (char *)"to be or not to be", enc);
+    frt_df_add_data(df, (char *)"a stitch in time, saves nine", enc);
+    frt_df_add_data(df, (char *)"the quick brown fox jumped over the lazy dog", enc);
+    frt_df_add_data(df, (char *)"that's it folks", enc);
     frt_doc_add_field(doc, df);
     frt_fw_add_doc(fw, doc);
     frt_fw_write_tv_index(fw);

data/ext/isomorfeus_ferret_ext/test_file_deleter.c

@@ -18,9 +18,8 @@ const FrtConfig lucene_config = {
 };
 
 
-static FrtFieldInfos *prep_fis()
-{
-    return frt_fis_new(FRT_STORE_NO, FRT_INDEX_YES, FRT_TERM_VECTOR_NO);
+static FrtFieldInfos *prep_fis(void) {
+    return frt_fis_new(FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_NO);
 }
 
 static void create_index(FrtStore *store)
@@ -30,37 +29,21 @@ static void create_index(FrtStore *store)
     frt_fis_deref(fis);
 }
 
-/*
-static FrtIndexWriter *create_iw(FrtStore *store)
-{
-    create_index(store);
-    return frt_iw_open(store, frt_whitespace_analyzer_new(false), &frt_default_config);
-}
-
-static FrtIndexWriter *create_iw_conf(FrtStore *store, int max_buffered_docs,
-                                   int merge_factor)
-{
-    FrtConfig config = frt_default_config;
-    config.max_buffered_docs = max_buffered_docs;
-    config.merge_factor = merge_factor;
-    create_index(store);
-    return frt_iw_open(store, frt_whitespace_analyzer_new(false), &config);
-}
-*/
-
 static FrtIndexWriter *create_iw_lucene(FrtStore *store)
 {
     create_index(store);
-    return frt_iw_open(store, frt_whitespace_analyzer_new(false), &lucene_config);
+    return frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), &lucene_config);
 }
 
 static void add_doc(FrtIndexWriter *iw, int id)
 {
     FrtDocument *doc = frt_doc_new();
+    rb_encoding *enc = rb_enc_find("ASCII-8BIT");
+
     frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern(content_f)),
-                                   frt_estrdup("aaa")))->destroy_data = true;
+                                   frt_estrdup("aaa"), enc))->destroy_data = true;
     frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern(id_f)),
-                                   frt_strfmt("%d", id)))->destroy_data = true;
+                                   frt_strfmt("%d", id), enc))->destroy_data = true;
     frt_iw_add_doc(iw, doc);
    frt_doc_destroy(doc);
 }
@@ -96,7 +79,7 @@ static void test_delete_leftover_files(TestCase *tc, void *data)
     frt_iw_close(iw);
 
     /* Delete one doc so we get a .del file: */
-    ir = frt_ir_open(store);
+    ir = frt_ir_open(NULL, store);
     frt_ir_delete_doc(ir, 7);
     Aiequal(1, ir->max_doc(ir) - ir->num_docs(ir));
 
@@ -158,7 +141,7 @@ static void test_delete_leftover_files(TestCase *tc, void *data)
 
 
     /* Open & close a writer: should delete the above files and nothing more: */
-    frt_iw_close(frt_iw_open(store, frt_whitespace_analyzer_new(false), &lucene_config));
+    frt_iw_close(frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), &lucene_config));
 
     store_after = frt_store_to_s(store);
 
@@ -175,7 +158,7 @@ static void test_delete_leftover_files(TestCase *tc, void *data)
 
 TestSuite *ts_file_deleter(TestSuite *suite)
 {
-    FrtStore *store = frt_open_ram_store();
+    FrtStore *store = frt_open_ram_store(NULL);
     suite = ADD_SUITE(suite);
 
     tst_run_test(suite, test_delete_leftover_files, store);

data/ext/isomorfeus_ferret_ext/test_filter.c

@@ -2,6 +2,8 @@
 #include "frt_search.h"
 #include "test.h"
 
+#undef close
+
 #define FILTER_DOCS_SIZE 10
 #define ARRAY_SIZE 20
 
@@ -11,13 +13,14 @@ struct FilterData {
     const char *flipflop;
 };
 
-static FrtSymbol num, date, flipflop;
+static ID num, date, flipflop;
 
 void prepare_filter_index(FrtStore *store)
 {
     int i;
     FrtIndexWriter *iw;
-    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_INDEX_YES, FRT_TERM_VECTOR_NO);
+    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_NO);
+    rb_encoding *enc = rb_enc_find("ASCII-8BIT");
 
     num      = rb_intern("num");
     date     = rb_intern("date");
@@ -39,13 +42,13 @@ void prepare_filter_index(FrtStore *store)
     frt_index_create(store, fis);
     frt_fis_deref(fis);
 
-    iw = frt_iw_open(store, frt_whitespace_analyzer_new(false), NULL);
+    iw = frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), NULL);
     for (i = 0; i < FILTER_DOCS_SIZE; i++) {
         FrtDocument *doc = frt_doc_new();
         doc->boost = (float)(i+1);
-        frt_doc_add_field(doc, frt_df_add_data(frt_df_new(num), (char *)data[i].num));
-        frt_doc_add_field(doc, frt_df_add_data(frt_df_new(date), (char *)data[i].date));
-        frt_doc_add_field(doc, frt_df_add_data(frt_df_new(flipflop), (char *)data[i].flipflop));
+        frt_doc_add_field(doc, frt_df_add_data(frt_df_new(num), (char *)data[i].num, enc));
+        frt_doc_add_field(doc, frt_df_add_data(frt_df_new(date), (char *)data[i].date, enc));
+        frt_doc_add_field(doc, frt_df_add_data(frt_df_new(flipflop), (char *)data[i].flipflop, enc));
         frt_iw_add_doc(iw, doc);
        frt_doc_destroy(doc);
     }
@@ -313,9 +316,9 @@ TestSuite *ts_filter(TestSuite *suite)
 
     suite = ADD_SUITE(suite);
 
-    store = frt_open_ram_store();
+    store = frt_open_ram_store(NULL);
     prepare_filter_index(store);
-    ir = frt_ir_open(store);
+    ir = frt_ir_open(NULL, store);
     searcher = frt_isea_new(ir);
 
     tst_run_test(suite, test_range_filter, (void *)searcher);

data/ext/isomorfeus_ferret_ext/test_hash.c

@@ -199,8 +199,8 @@ static void test_hash_int(TestCase *tc, void *data)
 static void test_hash_ptr(TestCase *tc, void *data)
 {
     FrtHash *h = frt_h_new_ptr(&free);
-    FrtSymbol word1 = rb_intern("one");
-    FrtSymbol word2 = rb_intern("two");
+    ID word1 = rb_intern("one");
+    ID word2 = rb_intern("two");
     char *word_one = frt_estrdup("one");
     int i;
     char buf[100];

data/ext/isomorfeus_ferret_ext/test_hashset.c

@@ -61,7 +61,7 @@ static void test_hs(TestCase *tc, void *data)
 static void test_hs_ptr(TestCase *tc, void *data)
 {
     FrtHashSet *hs = frt_hs_new_ptr(NULL);
-    FrtSymbol word1 = rb_intern("one"), word2 = rb_intern("two");
+    ID word1 = rb_intern("one"), word2 = rb_intern("two");
     char *word_one = frt_estrdup("one");
     (void)data; /* suppress unused argument warning */
 

data/ext/isomorfeus_ferret_ext/test_highlighter.c

@@ -3,6 +3,8 @@
 #include "testhelper.h"
 #include "test.h"
 
+#undef close
+
 #define ARRAY_SIZE 100
 
 static void test_match_vector(TestCase *tc, void *data)
@@ -79,7 +81,7 @@ static void test_match_vector(TestCase *tc, void *data)
 
 static void make_index(FrtStore *store)
 {
-    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_INDEX_YES,
+    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES,
                               FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
     frt_index_create(store, fis);
     frt_fis_deref(fis);
@@ -87,13 +89,14 @@ static void make_index(FrtStore *store)
 
 static void add_string_docs(FrtStore *store, const char *string[])
 {
-    FrtIndexWriter *iw = frt_iw_open(store, frt_whitespace_analyzer_new(true), NULL);
+    FrtIndexWriter *iw = frt_iw_open(NULL, store, frt_whitespace_analyzer_new(true), NULL);
+    rb_encoding *enc = rb_enc_find("ASCII-8BIT");
 
     while (*string) {
         FrtDocument *doc = frt_doc_new();
-        frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("field")), (char *)*string));
+        frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("field")), (char *)*string, enc));
         frt_iw_add_doc(iw, doc);
-       frt_doc_destroy(doc);
+        frt_doc_destroy(doc);
         string++;
     }
     frt_iw_close(iw);
@@ -102,7 +105,7 @@ static void add_string_docs(FrtStore *store, const char *string[])
 #define Chk_sea_mv(query, doc_num, expected) check_searcher_match_vector(tc, store, query, doc_num, expected)
 static void check_searcher_match_vector(TestCase *tc, FrtStore *store, FrtQuery *query, int doc_num, const char *expected)
 {
-    FrtIndexReader *ir = frt_ir_open(store);
+    FrtIndexReader *ir = frt_ir_open(NULL, store);
     FrtSearcher *sea = frt_isea_new(ir);
     FrtMatchVector *mv = frt_searcher_get_match_vector(sea, query, doc_num, rb_intern("field"));
     static int offset_array[ARRAY_SIZE];
@@ -122,7 +125,7 @@ static void check_searcher_match_vector(TestCase *tc, FrtStore *store, FrtQuery
 #define Chk_mv(query, doc_num, expected) check_match_vector(tc, store, query, doc_num, expected)
 static void check_match_vector(TestCase *tc, FrtStore *store, FrtQuery *query, int doc_num, const char *expected)
 {
-    FrtIndexReader *ir = frt_ir_open(store);
+    FrtIndexReader *ir = frt_ir_open(NULL, store);
     FrtMatchVector *mv = frt_matchv_new();
     FrtTermVector *term_vector = ir->term_vector(ir, doc_num, rb_intern("field"));
     static int offset_array[ARRAY_SIZE];
@@ -365,17 +368,18 @@ static void test_searcher_highlight(TestCase *tc, void *data)
         NULL
     };
     FrtDocument *doc = frt_doc_new();
+    rb_encoding *enc = rb_enc_find("ASCII-8BIT");
 
     make_index(store);
     add_string_docs(store, docs);
 
-    iw = frt_iw_open(store, frt_letter_analyzer_new(true), NULL);
-    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("field")), (char *)"That's how it goes now."));
+    iw = frt_iw_open(NULL, store, frt_letter_analyzer_new(true), NULL);
+    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("field")), (char *)"That's how it goes now.", enc));
     frt_iw_add_doc(iw, doc);
-   frt_doc_destroy(doc);
+    frt_doc_destroy(doc);
     frt_iw_close(iw);
 
-    ir = frt_ir_open(store);
+    ir = frt_ir_open(NULL, store);
     sea = frt_isea_new(ir);
 
     q = frt_tq_new(rb_intern("field"), "one");
@@ -529,7 +533,7 @@ static void test_searcher_highlight(TestCase *tc, void *data)
 
 TestSuite *ts_highlighter(TestSuite *suite)
 {
-    FrtStore *store = frt_open_ram_store();
+    FrtStore *store = frt_open_ram_store(NULL);
 
     suite = ADD_SUITE(suite);