isomorfeus-ferret 0.12.6 → 0.13.1

data/ext/isomorfeus_ferret_ext/test_compound_io.c

@@ -155,7 +155,7 @@ void test_compound_io_many_files(TestCase *tc, void *data)
 
 TestSuite *ts_compound_io(TestSuite *suite)
 {
-    FrtStore *store = frt_open_ram_store();
+    FrtStore *store = frt_open_ram_store(NULL);
 
     suite = ADD_SUITE(suite);
 

data/ext/isomorfeus_ferret_ext/test_document.c

@@ -6,8 +6,9 @@ void test_df_standard(TestCase *tc, void *data)
     char *s;
     FrtDocField *df;
     (void)data;
+    rb_encoding *enc = rb_enc_find("ASCII-8BIT");
 
-    df = frt_df_add_data(frt_df_new(rb_intern("title")), frt_estrdup("Life of Pi"));
+    df = frt_df_add_data(frt_df_new(rb_intern("title")), frt_estrdup("Life of Pi"), enc);
     df->destroy_data = true;
     Aiequal(1, df->size);
     Asequal("title", rb_id2name(df->name));
@@ -18,7 +19,7 @@ void test_df_standard(TestCase *tc, void *data)
     free(s);
     frt_df_destroy(df);
 
-    df = frt_df_add_data_len(frt_df_new(rb_intern("title")), (char *)"new title", 9);
+    df = frt_df_add_data_len(frt_df_new(rb_intern("title")), (char *)"new title", 9, enc);
     Aiequal(1, df->size);
     Asequal("title", rb_id2name(df->name));
     Asequal("new title", df->data[0]);
@@ -32,20 +33,21 @@ void test_df_multi_fields(TestCase *tc, void *data)
     char *s;
     FrtDocField *df;
     (void)data;
+    rb_encoding *enc = rb_enc_find("ASCII-8BIT");
 
-    df = frt_df_add_data(frt_df_new(rb_intern("title")), frt_estrdup("Vernon God Little"));
+    df = frt_df_add_data(frt_df_new(rb_intern("title")), frt_estrdup("Vernon God Little"), enc);
     df->destroy_data = true;
     Aiequal(1, df->size);
     Asequal("title", rb_id2name(df->name));
     Asequal("Vernon God Little", df->data[0]);
     Aiequal(strlen("Vernon God Little"), df->lengths[0]);
 
-    frt_df_add_data(df, frt_estrdup("some more data"));
+    frt_df_add_data(df, frt_estrdup("some more data"), enc);
     Aiequal(2, df->size);
     Asequal("title: [\"Vernon God Little\", \"some more data\"]",
             s = frt_df_to_s(df));
     free(s);
-    frt_df_add_data_len(df, frt_estrdup("and more data"), 14);
+    frt_df_add_data_len(df, frt_estrdup("and more data"), 14, enc);
     Aiequal(3, df->size);
     Asequal("title", rb_id2name(df->name));
     Asequal("Vernon God Little", df->data[0]);
@@ -54,13 +56,13 @@ void test_df_multi_fields(TestCase *tc, void *data)
 
     frt_df_destroy(df);
 
-    df = frt_df_add_data(frt_df_new(rb_intern("data")), frt_estrdup("start"));
+    df = frt_df_add_data(frt_df_new(rb_intern("data")), frt_estrdup("start"), enc);
     df->destroy_data = true;
     Aiequal(1, df->size);
     for (i = 0; i < 1000; i++) {
         char buf[100];
         sprintf(buf, "<<%d>>", i);
-        frt_df_add_data(df, frt_estrdup(buf));
+        frt_df_add_data(df, frt_estrdup(buf), enc);
         Aiequal(i + 2, df->size);
     }
     frt_df_destroy(df);
@@ -72,14 +74,15 @@ void test_doc(TestCase *tc, void *data)
     FrtDocument *doc;
     FrtDocField *df;
     (void)data;
+    rb_encoding *enc = rb_enc_find("ASCII-8BIT");
 
     doc = frt_doc_new();
-    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("title")), (char *)"title"));
+    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("title")), (char *)"title", enc));
     Aiequal(1, doc->size);
-    df = frt_df_add_data(frt_df_new(rb_intern("data")), (char *)"data1");
-    frt_df_add_data(df, (char *)"data2");
-    frt_df_add_data(df, (char *)"data3");
-    frt_df_add_data(df, (char *)"data4");
+    df = frt_df_add_data(frt_df_new(rb_intern("data")), (char *)"data1", enc);
+    frt_df_add_data(df, (char *)"data2", enc);
+    frt_df_add_data(df, (char *)"data3", enc);
+    frt_df_add_data(df, (char *)"data4", enc);
     frt_doc_add_field(doc, df);
     Aiequal(2, doc->size);
     Asequal("title", rb_id2name(frt_doc_get_field(doc, rb_intern("title"))->name));
@@ -100,7 +103,7 @@ void test_doc(TestCase *tc, void *data)
         char *bufc;
         sprintf(buf, "<<%d>>", i);
         bufc = frt_estrdup(buf);
-        df = frt_df_add_data(frt_df_new(rb_intern(bufc)), bufc);
+        df = frt_df_add_data(frt_df_new(rb_intern(bufc)), bufc, enc);
         df->destroy_data = true;
         frt_doc_add_field(doc, df);
         Aiequal(i + 1, doc->size);
@@ -122,12 +125,13 @@ void test_double_field_exception(TestCase *tc, void *data)
     FrtDocument *doc;
     FrtDocField *volatile df = NULL;
     (void)data;
+    rb_encoding *enc = rb_enc_find("ASCII-8BIT");
 
     doc = frt_doc_new();
-    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("title")), (char *)"title"));
+    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("title")), (char *)"title", enc));
 
     FRT_TRY
-        df = frt_df_add_data_len(frt_df_new(rb_intern("title")), (char *)"title", 5);
+        df = frt_df_add_data_len(frt_df_new(rb_intern("title")), (char *)"title", 5, enc);
         frt_doc_add_field(doc, df);
     case FRT_EXCEPTION:
         exception_thrown = true;

data/ext/isomorfeus_ferret_ext/test_except.c

@@ -5,8 +5,7 @@
 static const char *msg1 = "Message One";
 static const char *msg2 = "Message Two";
 
-static void raise_exception()
-{
+static void raise_exception(void) {
     FRT_RAISE(FRT_EXCEPTION, "%s", msg1);
 }
 

data/ext/isomorfeus_ferret_ext/test_fields.c

@@ -2,10 +2,10 @@
 #include "test.h"
 
 #define do_field_prop_test(tc, fi, name, boost, is_stored,\
-                           is_indexed, is_tokenized, omit_norms,\
+                           is_compressed, is_indexed, is_tokenized, omit_norms,\
                            store_term_vector, store_positions, store_offsets)\
         field_prop_test(tc, __LINE__, fi, name, boost, is_stored,\
-                        is_indexed, is_tokenized, omit_norms,\
+                        is_compressed, is_indexed, is_tokenized, omit_norms,\
                         store_term_vector, store_positions, store_offsets)
 #define T 1
 #define F 0
@@ -13,19 +13,20 @@
 void field_prop_test(TestCase *tc,
                      int line_num,
                      FrtFieldInfo *fi,
-                     FrtSymbol name,
+                     ID name,
                      float boost,
                      bool is_stored,
+                     bool is_compressed,
                      bool is_indexed,
                      bool is_tokenized,
                      bool omit_norms,
                      bool store_term_vector,
                      bool store_positions,
-                     bool store_offsets)
-{
+                     bool store_offsets) {
     tst_ptr_equal(line_num, tc, (void *)name, (void *)fi->name);
     tst_flt_equal(line_num, tc, boost, fi->boost);
     tst_int_equal(line_num, tc, is_stored,          fi_is_stored(fi));
+    tst_int_equal(line_num, tc, is_compressed,      fi_is_compressed(fi));
     tst_int_equal(line_num, tc, is_indexed,         fi_is_indexed(fi));
     tst_int_equal(line_num, tc, is_tokenized,       fi_is_tokenized(fi));
     tst_int_equal(line_num, tc, omit_norms,         fi_omit_norms(fi));
@@ -44,18 +45,21 @@ static void test_fi_new(TestCase *tc, void *data)
 {
     FrtFieldInfo *fi;
     (void)data; /* suppress unused argument warning */
-    fi = frt_fi_new(rb_intern("name"), FRT_STORE_NO, FRT_INDEX_NO, FRT_TERM_VECTOR_NO);
-    do_field_prop_test(tc, fi, rb_intern("name"), 1.0, F, F, F, F, F, F, F);
+    fi = frt_fi_new(rb_intern("name"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_NO, FRT_TERM_VECTOR_NO);
+    do_field_prop_test(tc, fi, rb_intern("name"), 1.0, F, F, F, F, F, F, F, F);
+    frt_fi_deref(fi);
+    fi = frt_fi_new(rb_intern("name"), FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_YES);
+    do_field_prop_test(tc, fi, rb_intern("name"), 1.0, T, F, T, T, F, T, F, F);
     frt_fi_deref(fi);
-    fi = frt_fi_new(rb_intern("name"), FRT_STORE_YES, FRT_INDEX_YES, FRT_TERM_VECTOR_YES);
-    do_field_prop_test(tc, fi, rb_intern("name"), 1.0, T, T, T, F, T, F, F);
+    fi = frt_fi_new(rb_intern("name"), FRT_STORE_YES, FRT_COMPRESSION_BROTLI, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_POSITIONS);
+    do_field_prop_test(tc, fi, rb_intern("name"), 1.0, T, T, T, F, F, T, T, F);
     frt_fi_deref(fi);
-    fi = frt_fi_new(rb_intern("name"), FRT_STORE_NO, FRT_INDEX_YES_OMIT_NORMS, FRT_TERM_VECTOR_WITH_OFFSETS);
-    do_field_prop_test(tc, fi, rb_intern("name"), 1.0, F, T, T, T, T, F, T);
+    fi = frt_fi_new(rb_intern("name"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_YES_OMIT_NORMS, FRT_TERM_VECTOR_WITH_OFFSETS);
+    do_field_prop_test(tc, fi, rb_intern("name"), 1.0, F, F, T, T, T, T, F, T);
     frt_fi_deref(fi);
-    fi = frt_fi_new(rb_intern("name"), FRT_STORE_NO, FRT_INDEX_UNTOKENIZED_OMIT_NORMS, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
+    fi = frt_fi_new(rb_intern("name"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_UNTOKENIZED_OMIT_NORMS, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
     fi->boost = 1000.0;
-    do_field_prop_test(tc, fi, rb_intern("name"), 1000.0, F, T, F, T, T, T, T);
+    do_field_prop_test(tc, fi, rb_intern("name"), 1000.0, F, F, T, F, T, T, T, T);
     frt_fi_deref(fi);
 }
 
@@ -72,14 +76,14 @@ static void test_fis_basic(TestCase *tc, void *data)
     volatile bool arg_error = false;
     (void)data; /* suppress unused argument warning */
 
-    fis = frt_fis_new(FRT_STORE_NO, FRT_INDEX_NO, FRT_TERM_VECTOR_NO);
-    frt_fis_add_field(fis, frt_fi_new(rb_intern("FFFFFFFF"), FRT_STORE_NO, FRT_INDEX_NO, FRT_TERM_VECTOR_NO));
-    frt_fis_add_field(fis, frt_fi_new(rb_intern("TFTTFTFF"), FRT_STORE_YES, FRT_INDEX_YES, FRT_TERM_VECTOR_YES));
-    frt_fis_add_field(fis, frt_fi_new(rb_intern("TTTFFTTF"), FRT_STORE_YES, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_POSITIONS));
-    frt_fis_add_field(fis, frt_fi_new(rb_intern("FFTTTTFT"), FRT_STORE_NO, FRT_INDEX_YES_OMIT_NORMS, FRT_TERM_VECTOR_WITH_OFFSETS));
-    frt_fis_add_field(fis, frt_fi_new(rb_intern("FFTFTTTT"), FRT_STORE_NO, FRT_INDEX_UNTOKENIZED_OMIT_NORMS, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS));
+    fis = frt_fis_new(FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_NO, FRT_TERM_VECTOR_NO);
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("FFFFFFFF"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_NO, FRT_TERM_VECTOR_NO));
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("TFTTFTFF"), FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_YES));
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("TTTFFTTF"), FRT_STORE_YES, FRT_COMPRESSION_BROTLI, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_POSITIONS));
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("FFTTTTFT"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_YES_OMIT_NORMS, FRT_TERM_VECTOR_WITH_OFFSETS));
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("FFTFTTTT"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_UNTOKENIZED_OMIT_NORMS, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS));
 
-    fi = frt_fi_new(rb_intern("FFTFTTTT"), FRT_STORE_NO, FRT_INDEX_UNTOKENIZED_OMIT_NORMS, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
+    fi = frt_fi_new(rb_intern("FFTFTTTT"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_UNTOKENIZED_OMIT_NORMS, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
     FRT_TRY
         Apnull(frt_fis_add_field(fis, fi));
     case FRT_ARG_ERROR:
@@ -113,11 +117,11 @@ static void test_fis_basic(TestCase *tc, void *data)
     fis->fields[3]->boost = 4.0;
     fis->fields[4]->boost = 5.0;
 
-    do_field_prop_test(tc, fis->fields[0], rb_intern("FFFFFFFF"), 1.0, F, F, F, F, F, F, F);
-    do_field_prop_test(tc, fis->fields[1], rb_intern("TFTTFTFF"), 2.0, T, T, T, F, T, F, F);
-    do_field_prop_test(tc, fis->fields[2], rb_intern("TTTFFTTF"), 3.0, T, T, F, F, T, T, F);
-    do_field_prop_test(tc, fis->fields[3], rb_intern("FFTTTTFT"), 4.0, F, T, T, T, T, F, T);
-    do_field_prop_test(tc, fis->fields[4], rb_intern("FFTFTTTT"), 5.0, F, T, F, T, T, T, T);
+    do_field_prop_test(tc, fis->fields[0], rb_intern("FFFFFFFF"), 1.0, F, F, F, F, F, F, F, F);
+    do_field_prop_test(tc, fis->fields[1], rb_intern("TFTTFTFF"), 2.0, T, F, T, T, F, T, F, F);
+    do_field_prop_test(tc, fis->fields[2], rb_intern("TTTFFTTF"), 3.0, T, T, T, F, F, T, T, F);
+    do_field_prop_test(tc, fis->fields[3], rb_intern("FFTTTTFT"), 4.0, F, F, T, T, T, T, F, T);
+    do_field_prop_test(tc, fis->fields[4], rb_intern("FFTFTTTT"), 5.0, F, F, T, F, T, T, T, T);
 
     frt_fis_deref(fis);
 }
@@ -127,27 +131,27 @@ static void test_fis_with_default(TestCase *tc, void *data)
     FrtFieldInfos *fis;
     (void)data; /* suppress unused argument warning */
 
-    fis = frt_fis_new(FRT_STORE_NO, FRT_INDEX_NO, FRT_TERM_VECTOR_NO);
-    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("name")), rb_intern("name"), 1.0, F, F, F, F, F, F, F);
-    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("dave")), rb_intern("dave"), 1.0, F, F, F, F, F, F, F);
-    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("wert")), rb_intern("wert"), 1.0, F, F, F, F, F, F, F);
-    do_field_prop_test(tc, fis->fields[0], rb_intern("name"), 1.0, F, F, F, F, F, F, F);
-    do_field_prop_test(tc, fis->fields[1], rb_intern("dave"), 1.0, F, F, F, F, F, F, F);
-    do_field_prop_test(tc, fis->fields[2], rb_intern("wert"), 1.0, F, F, F, F, F, F, F);
+    fis = frt_fis_new(FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_NO, FRT_TERM_VECTOR_NO);
+    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("name")), rb_intern("name"), 1.0, F, F, F, F, F, F, F, F);
+    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("dave")), rb_intern("dave"), 1.0, F, F, F, F, F, F, F, F);
+    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("wert")), rb_intern("wert"), 1.0, F, F, F, F, F, F, F, F);
+    do_field_prop_test(tc, fis->fields[0], rb_intern("name"), 1.0, F, F, F, F, F, F, F, F);
+    do_field_prop_test(tc, fis->fields[1], rb_intern("dave"), 1.0, F, F, F, F, F, F, F, F);
+    do_field_prop_test(tc, fis->fields[2], rb_intern("wert"), 1.0, F, F, F, F, F, F, F, F);
     Apnull(frt_fis_get_field(fis, rb_intern("random")));
     frt_fis_deref(fis);
 
-    fis = frt_fis_new(FRT_STORE_YES, FRT_INDEX_YES, FRT_TERM_VECTOR_YES);
-    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("name")), rb_intern("name"), 1.0, T, T, T, F, T, F, F);
+    fis = frt_fis_new(FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_YES);
+    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("name")), rb_intern("name"), 1.0, T, F, T, T, F, T, F, F);
     frt_fis_deref(fis);
-    fis = frt_fis_new(FRT_STORE_YES, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_POSITIONS);
-    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("name")), rb_intern("name"), 1.0, T, T, F, F, T, T, F);
+    fis = frt_fis_new(FRT_STORE_YES, FRT_COMPRESSION_BROTLI, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_POSITIONS);
+    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("name")), rb_intern("name"), 1.0, T, T, T, F, F, T, T, F);
     frt_fis_deref(fis);
-    fis = frt_fis_new(FRT_STORE_NO, FRT_INDEX_YES_OMIT_NORMS, FRT_TERM_VECTOR_WITH_OFFSETS);
-    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("name")), rb_intern("name"), 1.0, F, T, T, T, T, F, T);
+    fis = frt_fis_new(FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_YES_OMIT_NORMS, FRT_TERM_VECTOR_WITH_OFFSETS);
+    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("name")), rb_intern("name"), 1.0, F, F, T, T, T, T, F, T);
     frt_fis_deref(fis);
-    fis = frt_fis_new(FRT_STORE_NO, FRT_INDEX_UNTOKENIZED_OMIT_NORMS, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
-    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("name")), rb_intern("name"), 1.0, F, T, F, T, T, T, T);
+    fis = frt_fis_new(FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_UNTOKENIZED_OMIT_NORMS, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
+    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("name")), rb_intern("name"), 1.0, F, F, T, F, T, T, T, T);
     frt_fis_deref(fis);
 }
 
@@ -155,31 +159,31 @@ static void test_fis_rw(TestCase *tc, void *data)
 {
     char *str;
     FrtFieldInfos *fis;
-    FrtStore *store = frt_open_ram_store();
+    FrtStore *store = frt_open_ram_store(NULL);
     FrtInStream *is;
     FrtOutStream *os;
     (void)data; /* suppress unused argument warning */
 
-    fis = frt_fis_new(FRT_STORE_YES, FRT_INDEX_UNTOKENIZED_OMIT_NORMS,
-                  FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
-    frt_fis_add_field(fis, frt_fi_new(rb_intern("FFFFFFFF"), FRT_STORE_NO, FRT_INDEX_NO, FRT_TERM_VECTOR_NO));
-    frt_fis_add_field(fis, frt_fi_new(rb_intern("TFTTFTFF"), FRT_STORE_YES, FRT_INDEX_YES, FRT_TERM_VECTOR_YES));
-    frt_fis_add_field(fis, frt_fi_new(rb_intern("FFTTTTFT"), FRT_STORE_NO, FRT_INDEX_YES_OMIT_NORMS, FRT_TERM_VECTOR_WITH_OFFSETS));
-    frt_fis_add_field(fis, frt_fi_new(rb_intern("FFTFTTTT"), FRT_STORE_NO, FRT_INDEX_UNTOKENIZED_OMIT_NORMS, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS));
-    fis->fields[0]->boost = 2.0;
-    fis->fields[1]->boost = 3.0;
-    fis->fields[2]->boost = 4.0;
-    fis->fields[3]->boost = 5.0;
+    fis = frt_fis_new(FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_UNTOKENIZED_OMIT_NORMS, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("FFFFFFFF"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_NO, FRT_TERM_VECTOR_NO));
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("TFTTFTFF"), FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_YES));
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("TTTFFTTF"), FRT_STORE_YES, FRT_COMPRESSION_BROTLI, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_POSITIONS));
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("FFTTTTFT"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_YES_OMIT_NORMS, FRT_TERM_VECTOR_WITH_OFFSETS));
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("FFTFTTTT"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_UNTOKENIZED_OMIT_NORMS, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS));
+    fis->fields[1]->boost = 2.0;
+    fis->fields[2]->boost = 3.0;
+    fis->fields[3]->boost = 4.0;
+    fis->fields[4]->boost = 5.0;
     os = store->new_output(store, "fields");
     frt_fis_write(fis, os);
     frt_os_close(os);
 
     /* these fields won't be saved be will added again later */
-    Aiequal(4, fis->size);
-    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("new_field")), rb_intern("new_field"), 1.0, T, T, F, T, T, T, T);
     Aiequal(5, fis->size);
-    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("another")), rb_intern("another"), 1.0, T, T, F, T, T, T, T);
+    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("new_field")), rb_intern("new_field"), 1.0, T, F, T, F, T, T, T, T);
     Aiequal(6, fis->size);
+    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("another")), rb_intern("another"), 1.0, T, F, T, F, T, T, T, T);
+    Aiequal(7, fis->size);
 
     frt_fis_deref(fis);
 
@@ -190,15 +194,16 @@ static void test_fis_rw(TestCase *tc, void *data)
     Aiequal(FRT_INDEX_UNTOKENIZED_OMIT_NORMS, fis->index);
     Aiequal(FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS, fis->term_vector);
 
-    do_field_prop_test(tc, fis->fields[0], rb_intern("FFFFFFFF"), 2.0, F, F, F, F, F, F, F);
-    do_field_prop_test(tc, fis->fields[1], rb_intern("TFTTFTFF"), 3.0, T, T, T, F, T, F, F);
-    do_field_prop_test(tc, fis->fields[2], rb_intern("FFTTTTFT"), 4.0, F, T, T, T, T, F, T);
-    do_field_prop_test(tc, fis->fields[3], rb_intern("FFTFTTTT"), 5.0, F, T, F, T, T, T, T);
-    Aiequal(4, fis->size);
-    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("new_field")), rb_intern("new_field"), 1.0, T, T, F, T, T, T, T);
+    do_field_prop_test(tc, fis->fields[0], rb_intern("FFFFFFFF"), 1.0, F, F, F, F, F, F, F, F);
+    do_field_prop_test(tc, fis->fields[1], rb_intern("TFTTFTFF"), 2.0, T, F, T, T, F, T, F, F);
+    do_field_prop_test(tc, fis->fields[2], rb_intern("TTTFFTTF"), 3.0, T, T, T, F, F, T, T, F);
+    do_field_prop_test(tc, fis->fields[3], rb_intern("FFTTTTFT"), 4.0, F, F, T, T, T, T, F, T);
+    do_field_prop_test(tc, fis->fields[4], rb_intern("FFTFTTTT"), 5.0, F, F, T, F, T, T, T, T);
     Aiequal(5, fis->size);
-    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("another")), rb_intern("another"), 1.0, T, T, F, T, T, T, T);
+    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("new_field")), rb_intern("new_field"), 1.0, T, F, T, F, T, T, T, T);
     Aiequal(6, fis->size);
+    do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("another")), rb_intern("another"), 1.0, T, F, T, F, T, T, T, T);
+    Aiequal(7, fis->size);
     str = frt_fis_to_s(fis);
     Asequal("default:\n"
             "  store: :yes\n"
@@ -206,15 +211,20 @@ static void test_fis_rw(TestCase *tc, void *data)
             "  term_vector: :with_positions_offsets\n"
             "fields:\n"
             "  FFFFFFFF:\n"
-            "    boost: 2.000000\n"
+            "    boost: 1.000000\n"
             "    store: :no\n"
             "    index: :no\n"
             "    term_vector: :no\n"
             "  TFTTFTFF:\n"
-            "    boost: 3.000000\n"
+            "    boost: 2.000000\n"
             "    store: :yes\n"
             "    index: :yes\n"
             "    term_vector: :yes\n"
+            "  TTTFFTTF:\n"
+            "    boost: 3.000000\n"
+            "    store: :compressed\n"
+            "    index: :untokenized\n"
+            "    term_vector: :with_positions\n"
             "  FFTTTTFT:\n"
             "    boost: 4.000000\n"
             "    store: :no\n"
@@ -270,45 +280,44 @@ static char *prepare_bin_data(int len)
         Assert(memcmp(mdata, df->data[index], mlen) == 0, "Data should be equal");\
     } while (0)
 
-static FrtDocument *prepare_doc()
-{
+static FrtDocument *prepare_doc(void) {
     FrtDocument *doc = frt_doc_new();
     FrtDocField *df;
     char *bin_data = prepare_bin_data(BIN_DATA_LEN);
+    rb_encoding *enc = rb_enc_find("ASCII-8BIT");
 
-    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("ignored")), (char *)"this fld's ignored"));
-    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("unstored")), (char *)"unstored ignored"));
-    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("stored")), (char *)"Yay, a stored field"));
-    df = frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("stored_array")), (char *)"one"));
+    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("ignored")), (char *)"this fld's ignored", enc));
+    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("unstored")), (char *)"unstored ignored", enc));
+    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("stored")), (char *)"Yay, a stored field", enc));
+    df = frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("stored_array")), (char *)"one", enc));
     df->destroy_data = false;
-    frt_df_add_data(df, (char *)"two");
-    frt_df_add_data(df, (char *)"three");
-    frt_df_add_data(df, (char *)"four");
-    frt_df_add_data_len(df, bin_data, BIN_DATA_LEN);
+    frt_df_add_data(df, (char *)"two", enc);
+    frt_df_add_data(df, (char *)"three", enc);
+    frt_df_add_data(df, (char *)"four", enc);
+    frt_df_add_data_len(df, bin_data, BIN_DATA_LEN, enc);
     frt_doc_add_field(doc, frt_df_add_data_len(frt_df_new(rb_intern("binary")), bin_data,
-                                       BIN_DATA_LEN))->destroy_data = true;
-    df = frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("array")), (char *)"ichi"));
-    frt_df_add_data(df, (char *)"ni");
-    frt_df_add_data(df, (char *)"san");
-    frt_df_add_data(df, (char *)"yon");
-    frt_df_add_data(df, (char *)"go");
+                                       BIN_DATA_LEN, enc))->destroy_data = true;
+    df = frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("array")), (char *)"ichi", enc));
+    frt_df_add_data(df, (char *)"ni", enc);
+    frt_df_add_data(df, (char *)"san", enc);
+    frt_df_add_data(df, (char *)"yon", enc);
+    frt_df_add_data(df, (char *)"go", enc);
 
     return doc;
 }
 
-static FrtFieldInfos *prepare_fis()
-{
-    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_INDEX_YES, FRT_TERM_VECTOR_NO);
-    frt_fis_add_field(fis, frt_fi_new(rb_intern("ignored"), FRT_STORE_NO, FRT_INDEX_NO, FRT_TERM_VECTOR_NO));
-    frt_fis_add_field(fis, frt_fi_new(rb_intern("unstored"), FRT_STORE_NO, FRT_INDEX_YES, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS));
-    frt_fis_add_field(fis, frt_fi_new(rb_intern("stored"), FRT_STORE_YES, FRT_INDEX_YES, FRT_TERM_VECTOR_YES));
-    frt_fis_add_field(fis, frt_fi_new(rb_intern("stored_array"), FRT_STORE_YES, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_NO));
+static FrtFieldInfos *prepare_fis(void) {
+    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_NO);
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("ignored"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_NO, FRT_TERM_VECTOR_NO));
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("unstored"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS));
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("stored"), FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_YES));
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("stored_array"), FRT_STORE_YES, FRT_COMPRESSION_BROTLI, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_NO));
     return fis;
 }
 
 static void test_fields_rw_single(TestCase *tc, void *data)
 {
-    FrtStore *store = frt_open_ram_store();
+    FrtStore *store = frt_open_ram_store(NULL);
     char *bin_data = prepare_bin_data(BIN_DATA_LEN);
     FrtDocument *doc = prepare_doc();
     FrtFieldInfos *fis = prepare_fis();
@@ -327,8 +336,8 @@ static void test_fields_rw_single(TestCase *tc, void *data)
    frt_doc_destroy(doc);
 
     Aiequal(6, fis->size);
-    do_field_prop_test(tc, frt_fis_get_field(fis, rb_intern("binary")), rb_intern("binary"), 1.0, T, T, T, F, F, F, F);
-    do_field_prop_test(tc, frt_fis_get_field(fis, rb_intern("array")), rb_intern("array"), 1.0, T, T, T, F, F, F, F);
+    do_field_prop_test(tc, frt_fis_get_field(fis, rb_intern("binary")), rb_intern("binary"), 1.0, T, F, T, T, F, F, F, F);
+    do_field_prop_test(tc, frt_fis_get_field(fis, rb_intern("array")), rb_intern("array"), 1.0, T, F, T, T, F, F, F, F);
 
     fr = frt_fr_open(store, "_0", fis);
     doc = frt_fr_get_doc(fr, 0);
@@ -372,7 +381,7 @@ static void test_fields_rw_single(TestCase *tc, void *data)
 static void test_fields_rw_multi(TestCase *tc, void *data)
 {
     int i;
-    FrtStore *store = frt_open_ram_store();
+    FrtStore *store = frt_open_ram_store(NULL);
     char *bin_data = prepare_bin_data(BIN_DATA_LEN);
     FrtDocument *doc;
     FrtFieldInfos *fis = prepare_fis();
@@ -380,6 +389,7 @@ static void test_fields_rw_multi(TestCase *tc, void *data)
     FrtFieldsReader *fr;
     FrtDocField *df;
     (void)data;
+    rb_encoding *enc = rb_enc_find("ASCII-8BIT");
 
     fw = frt_fw_open(store, "_as3", fis);
     for (i = 0; i < 100; i++) {
@@ -388,7 +398,7 @@ static void test_fields_rw_multi(TestCase *tc, void *data)
         sprintf(buf, "<<%d>>", i);
         bufc = frt_estrdup(buf);
         doc = frt_doc_new();
-        frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern(bufc)), bufc));
+        frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern(bufc)), bufc, enc));
         frt_fw_add_doc(fw, doc);
         frt_fw_write_tv_index(fw);
        frt_doc_destroy(doc);
@@ -401,12 +411,12 @@ static void test_fields_rw_multi(TestCase *tc, void *data)
     frt_fw_close(fw);
 
     Aiequal(106, fis->size);
-    do_field_prop_test(tc, frt_fis_get_field(fis, rb_intern("binary")), rb_intern("binary"), 1.0, T, T, T, F, F, F, F);
-    do_field_prop_test(tc, frt_fis_get_field(fis, rb_intern("array")), rb_intern("array"), 1.0, T, T, T, F, F, F, F);
+    do_field_prop_test(tc, frt_fis_get_field(fis, rb_intern("binary")), rb_intern("binary"), 1.0, T, F, T, T, F, F, F, F);
+    do_field_prop_test(tc, frt_fis_get_field(fis, rb_intern("array")), rb_intern("array"), 1.0, T, F, T, T, F, F, F, F);
     for (i = 0; i < 100; i++) {
         char buf[100];
         sprintf(buf, "<<%d>>", i);
-        do_field_prop_test(tc, frt_fis_get_field(fis, rb_intern(buf)), rb_intern(buf), 1.0, T, T, T, F, F, F, F);
+        do_field_prop_test(tc, frt_fis_get_field(fis, rb_intern(buf)), rb_intern(buf), 1.0, T, F, T, T, F, F, F, F);
     }
 
     fr = frt_fr_open(store, "_as3", fis);
@@ -450,7 +460,7 @@ static void test_fields_rw_multi(TestCase *tc, void *data)
 
 static void test_lazy_field_loading(TestCase *tc, void *data)
 {
-    FrtStore *store = frt_open_ram_store();
+    FrtStore *store = frt_open_ram_store(NULL);
     FrtDocument *doc;
     FrtFieldInfos *fis = prepare_fis();
     FrtFieldsWriter *fw;
@@ -460,15 +470,16 @@ static void test_lazy_field_loading(TestCase *tc, void *data)
     FrtLazyDocField *lazy_df;
     char *text, buf[1000];
     (void)data;
+    rb_encoding *enc = rb_enc_find("ASCII-8BIT");
 
     fw = frt_fw_open(store, "_as3", fis);
     doc = frt_doc_new();
     df = frt_df_new(rb_intern("stored"));
-    frt_df_add_data(df, (char *)"this is a stored field");
-    frt_df_add_data(df, (char *)"to be or not to be");
-    frt_df_add_data(df, (char *)"a stitch in time, saves nine");
-    frt_df_add_data(df, (char *)"the quick brown fox jumped over the lazy dog");
-    frt_df_add_data(df, (char *)"that's it folks");
+    frt_df_add_data(df, (char *)"this is a stored field", enc);
+    frt_df_add_data(df, (char *)"to be or not to be", enc);
+    frt_df_add_data(df, (char *)"a stitch in time, saves nine", enc);
+    frt_df_add_data(df, (char *)"the quick brown fox jumped over the lazy dog", enc);
+    frt_df_add_data(df, (char *)"that's it folks", enc);
     frt_doc_add_field(doc, df);
     frt_fw_add_doc(fw, doc);
     frt_fw_write_tv_index(fw);

data/ext/isomorfeus_ferret_ext/test_file_deleter.c

@@ -18,9 +18,8 @@ const FrtConfig lucene_config = {
 };
 
 
-static FrtFieldInfos *prep_fis()
-{
-    return frt_fis_new(FRT_STORE_NO, FRT_INDEX_YES, FRT_TERM_VECTOR_NO);
+static FrtFieldInfos *prep_fis(void) {
+    return frt_fis_new(FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_NO);
 }
 
 static void create_index(FrtStore *store)
@@ -30,37 +29,21 @@ static void create_index(FrtStore *store)
     frt_fis_deref(fis);
 }
 
-/*
-static FrtIndexWriter *create_iw(FrtStore *store)
-{
-    create_index(store);
-    return frt_iw_open(store, frt_whitespace_analyzer_new(false), &frt_default_config);
-}
-
-static FrtIndexWriter *create_iw_conf(FrtStore *store, int max_buffered_docs,
-                                   int merge_factor)
-{
-    FrtConfig config = frt_default_config;
-    config.max_buffered_docs = max_buffered_docs;
-    config.merge_factor = merge_factor;
-    create_index(store);
-    return frt_iw_open(store, frt_whitespace_analyzer_new(false), &config);
-}
-*/
-
 static FrtIndexWriter *create_iw_lucene(FrtStore *store)
 {
     create_index(store);
-    return frt_iw_open(store, frt_whitespace_analyzer_new(false), &lucene_config);
+    return frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), &lucene_config);
 }
 
 static void add_doc(FrtIndexWriter *iw, int id)
 {
     FrtDocument *doc = frt_doc_new();
+    rb_encoding *enc = rb_enc_find("ASCII-8BIT");
+
     frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern(content_f)),
-                                   frt_estrdup("aaa")))->destroy_data = true;
+                                   frt_estrdup("aaa"), enc))->destroy_data = true;
     frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern(id_f)),
-                                   frt_strfmt("%d", id)))->destroy_data = true;
+                                   frt_strfmt("%d", id), enc))->destroy_data = true;
     frt_iw_add_doc(iw, doc);
    frt_doc_destroy(doc);
 }
@@ -96,7 +79,7 @@ static void test_delete_leftover_files(TestCase *tc, void *data)
     frt_iw_close(iw);
 
     /* Delete one doc so we get a .del file: */
-    ir = frt_ir_open(store);
+    ir = frt_ir_open(NULL, store);
     frt_ir_delete_doc(ir, 7);
     Aiequal(1, ir->max_doc(ir) - ir->num_docs(ir));
 
@@ -158,7 +141,7 @@ static void test_delete_leftover_files(TestCase *tc, void *data)
 
 
     /* Open & close a writer: should delete the above files and nothing more: */
-    frt_iw_close(frt_iw_open(store, frt_whitespace_analyzer_new(false), &lucene_config));
+    frt_iw_close(frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), &lucene_config));
 
     store_after = frt_store_to_s(store);
 
@@ -175,7 +158,7 @@ static void test_delete_leftover_files(TestCase *tc, void *data)
 
 TestSuite *ts_file_deleter(TestSuite *suite)
 {
-    FrtStore *store = frt_open_ram_store();
+    FrtStore *store = frt_open_ram_store(NULL);
     suite = ADD_SUITE(suite);
 
     tst_run_test(suite, test_delete_leftover_files, store);