isomorfeus-ferret 0.12.7 → 0.13.0

data/ext/isomorfeus_ferret_ext/test_search.c

@@ -5,9 +5,11 @@
 #include "testhelper.h"
 #include "test.h"
 
+#undef close
+
 #define ARRAY_SIZE 40
 
-static FrtSymbol date, field, cat, number;
+static ID date, field, cat, number;
 
 static void test_byte_float_conversion(TestCase *tc, void *data)
 {
@@ -19,9 +21,7 @@ static void test_byte_float_conversion(TestCase *tc, void *data)
     }
 }
 
-static int my_doc_freq(FrtSearcher *searcher, FrtSymbol field,
-                       const char *term)
-{
+static int my_doc_freq(FrtSearcher *searcher, ID field, const char *term) {
     (void)searcher; (void)field; (void)term;
     return 9;
 }
@@ -137,14 +137,13 @@ static FrtTokenStream *dbl_tf_clone_i(FrtTokenStream *ts)
 
 static FrtTokenStream *dbl_tf_new(FrtTokenStream *sub_ts)
 {
-    FrtTokenStream *ts = tf_new(DoubleFilter, sub_ts);
-    ts->next        = &dbl_tf_next;
-    ts->clone_i     = &dbl_tf_clone_i;
+    FrtTokenStream *ts = frt_tf_new_i(sizeof(DoubleFilter), sub_ts);
+    ts->next           = &dbl_tf_next;
+    ts->clone_i        = &dbl_tf_clone_i;
     return ts;
 }
 
-FrtAnalyzer *dbl_analyzer_new()
-{
+FrtAnalyzer *dbl_analyzer_new(void) {
     FrtTokenStream *ts;
     ts = dbl_tf_new(frt_whitespace_tokenizer_new());
     return frt_analyzer_new(ts, NULL, NULL);
@@ -206,20 +205,21 @@ static void prepare_search_index(FrtStore *store)
     int i;
     FrtIndexWriter *iw;
 
-    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_INDEX_YES, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
-    FrtFieldInfo *fi = frt_fi_new(rb_intern("empty-field"), FRT_STORE_NO, FRT_INDEX_NO, FRT_TERM_VECTOR_NO);
+    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
+    FrtFieldInfo *fi = frt_fi_new(rb_intern("empty-field"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_NO, FRT_TERM_VECTOR_NO);
     frt_fis_add_field(fis, fi);
     frt_index_create(store, fis);
     frt_fis_deref(fis);
+    rb_encoding *enc = rb_enc_find("ASCII-8BIT");
 
-    iw = frt_iw_open(store, dbl_analyzer_new(), NULL);
+    iw = frt_iw_open(NULL, store, dbl_analyzer_new(), NULL);
     for (i = 0; i < SEARCH_DOCS_SIZE; i++) {
         FrtDocument *doc = frt_doc_new();
         doc->boost = (float)(i+1);
-        frt_doc_add_field(doc, frt_df_add_data(frt_df_new(date), (char *)test_data[i].date));
-        frt_doc_add_field(doc, frt_df_add_data(frt_df_new(field), (char *)test_data[i].field));
-        frt_doc_add_field(doc, frt_df_add_data(frt_df_new(cat), (char *)test_data[i].cat));
-        frt_doc_add_field(doc, frt_df_add_data(frt_df_new(number), (char *)test_data[i].number));
+        frt_doc_add_field(doc, frt_df_add_data(frt_df_new(date), (char *)test_data[i].date, enc));
+        frt_doc_add_field(doc, frt_df_add_data(frt_df_new(field), (char *)test_data[i].field, enc));
+        frt_doc_add_field(doc, frt_df_add_data(frt_df_new(cat), (char *)test_data[i].cat, enc));
+        frt_doc_add_field(doc, frt_df_add_data(frt_df_new(number), (char *)test_data[i].number, enc));
         frt_iw_add_doc(iw, doc);
        frt_doc_destroy(doc);
     }
@@ -252,7 +252,7 @@ static void test_get_doc(TestCase *tc, void *data)
    frt_doc_destroy(doc);
 }
 
-void check_to_s(TestCase *tc, FrtQuery *query, FrtSymbol field, const char *q_str)
+void check_to_s(TestCase *tc, FrtQuery *query, ID field, const char *q_str)
 {
     char *q_res = query->to_s(query, field);
     Asequal(q_str, q_res);
@@ -318,7 +318,7 @@ void tst_check_hits(TestCase *tc, FrtSearcher *searcher, FrtQuery *query, const
 }
 
 void check_match_vector(TestCase *tc, FrtSearcher *searcher, FrtQuery *query,
-                        int doc, FrtSymbol field, const char *ranges)
+                        int doc, ID field, const char *ranges)
 {
     static int range_array[ARRAY_SIZE];
     FrtMatchVector *mv = frt_searcher_get_match_vector(searcher, query, doc, field);
@@ -342,11 +342,11 @@ static void test_term_query(TestCase *tc, void *data)
     char *t, e[100];
     FrtQuery *tq = frt_tq_new(field, "word2");
     check_to_s(tc, tq, field, "word2");
-    check_to_s(tc, tq, (FrtSymbol)NULL, "field:word2");
+    check_to_s(tc, tq, (ID)NULL, "field:word2");
     tq->boost = 100;
     tst_check_hits(tc, searcher, tq, "4, 8, 1", -1);
     check_to_s(tc, tq, field, "word2^100.0");
-    check_to_s(tc, tq, (FrtSymbol)NULL, "field:word2^100.0");
+    check_to_s(tc, tq, (ID)NULL, "field:word2^100.0");
 
     /* test TermWeight.to_s */
     w = searcher->create_weight(searcher, tq);
@@ -554,13 +554,13 @@ static void test_phrase_query(TestCase *tc, void *data)
     FrtWeight *w;
     char *t, e[100];
     check_to_s(tc, phq, field, "\"\"");
-    check_to_s(tc, phq, (FrtSymbol)NULL, "field:\"\"");
+    check_to_s(tc, phq, (ID)NULL, "field:\"\"");
 
     frt_phq_add_term(phq, "quick", 1);
     frt_phq_add_term(phq, "brown", 1);
     frt_phq_add_term(phq, "fox", 1);
     check_to_s(tc, phq, field, "\"quick brown fox\"");
-    check_to_s(tc, phq, (FrtSymbol)NULL, "field:\"quick brown fox\"");
+    check_to_s(tc, phq, (ID)NULL, "field:\"quick brown fox\"");
     tst_check_hits(tc, searcher, phq, "1", 1);
 
     frt_phq_set_slop(phq, 4);
@@ -581,7 +581,7 @@ static void test_phrase_query(TestCase *tc, void *data)
     frt_phq_add_term(phq, "quick", 1);
     frt_phq_add_term(phq, "fox", 2);
     check_to_s(tc, phq, field, "\"quick <> fox\"");
-    check_to_s(tc, phq, (FrtSymbol)NULL, "field:\"quick <> fox\"");
+    check_to_s(tc, phq, (ID)NULL, "field:\"quick <> fox\"");
     tst_check_hits(tc, searcher, phq, "1, 11, 14", 14);
 
     frt_phq_set_slop(phq, 1);
@@ -590,19 +590,19 @@ static void test_phrase_query(TestCase *tc, void *data)
     frt_phq_set_slop(phq, 4);
     tst_check_hits(tc, searcher, phq, "1, 11, 14, 16, 17", 14);
     frt_phq_add_term(phq, "red", -1);
-    check_to_s(tc, phq, (FrtSymbol)NULL, "field:\"quick red fox\"~4");
+    check_to_s(tc, phq, (ID)NULL, "field:\"quick red fox\"~4");
     tst_check_hits(tc, searcher, phq, "11", 11);
     frt_phq_add_term(phq, "RED", 0);
-    check_to_s(tc, phq, (FrtSymbol)NULL, "field:\"quick red RED&fox\"~4");
+    check_to_s(tc, phq, (ID)NULL, "field:\"quick red RED&fox\"~4");
     tst_check_hits(tc, searcher, phq, "11", 11);
     frt_phq_add_term(phq, "QUICK", -1);
     frt_phq_add_term(phq, "red", 0);
-    check_to_s(tc, phq, (FrtSymbol)NULL, "field:\"quick QUICK&red&red RED&fox\"~4");
+    check_to_s(tc, phq, (ID)NULL, "field:\"quick QUICK&red&red RED&fox\"~4");
     tst_check_hits(tc, searcher, phq, "11", 11);
     frt_phq_add_term(phq, "green", 0);
     frt_phq_add_term(phq, "yellow", 0);
     frt_phq_add_term(phq, "sentinel", 1);
-    check_to_s(tc, phq, (FrtSymbol)NULL,
+    check_to_s(tc, phq, (ID)NULL,
                "field:\"quick QUICK&red&red RED&fox&green&yellow sentinel\"~4");
     tst_check_hits(tc, searcher, phq, "", -1);
     frt_q_deref(phq);
@@ -615,7 +615,7 @@ static void test_phrase_query(TestCase *tc, void *data)
     frt_phq_add_term(phq, "quick", 0);
     frt_phq_add_term(phq, "QUICK", 1);
     tst_check_hits(tc, searcher, phq, "11, 14", 14);
-    check_to_s(tc, phq, (FrtSymbol)NULL, "field:\"WORD3&the THE&quick QUICK\"");
+    check_to_s(tc, phq, (ID)NULL, "field:\"WORD3&the THE&quick QUICK\"");
     frt_q_deref(phq);
 
     /* test repeating terms check */
@@ -740,30 +740,30 @@ static void test_multi_phrase_query(TestCase *tc, void *data)
     frt_phq_append_multi_term(phq, "fast");
     tst_check_hits(tc, searcher, phq, "1, 8, 11, 14, 16, 17", -1);
     check_to_s(tc, phq, field, "\"quick|fast\"");
-    check_to_s(tc, phq, (FrtSymbol)NULL, "field:\"quick|fast\"");
+    check_to_s(tc, phq, (ID)NULL, "field:\"quick|fast\"");
 
     frt_phq_add_term(phq, "brown", 1);
     frt_phq_append_multi_term(phq, "red");
     frt_phq_append_multi_term(phq, "hairy");
     frt_phq_add_term(phq, "fox", 1);
     check_to_s(tc, phq, field, "\"quick|fast brown|red|hairy fox\"");
-    check_to_s(tc, phq, (FrtSymbol)NULL, "field:\"quick|fast brown|red|hairy fox\"");
+    check_to_s(tc, phq, (ID)NULL, "field:\"quick|fast brown|red|hairy fox\"");
     tst_check_hits(tc, searcher, phq, "1, 8, 11, 14", -1);
 
     frt_phq_set_slop(phq, 4);
     tst_check_hits(tc, searcher, phq, "1, 8, 11, 14, 16, 17", -1);
-    check_to_s(tc, phq, (FrtSymbol)NULL, "field:\"quick|fast brown|red|hairy fox\"~4");
+    check_to_s(tc, phq, (ID)NULL, "field:\"quick|fast brown|red|hairy fox\"~4");
 
     frt_phq_add_term(phq, "QUICK", -1);
     frt_phq_append_multi_term(phq, "FAST");
     tst_check_hits(tc, searcher, phq, "1, 8, 11, 14, 16, 17", -1);
-    check_to_s(tc, phq, (FrtSymbol)NULL,
+    check_to_s(tc, phq, (ID)NULL,
                "field:\"quick|fast QUICK|FAST&brown|red|hairy fox\"~4");
 
     frt_phq_add_term(phq, "WORD3", -3);
     frt_phq_append_multi_term(phq, "WORD2");
     tst_check_hits(tc, searcher, phq, "1, 8, 11, 14", -1);
-    check_to_s(tc, phq, (FrtSymbol)NULL, "field:\"WORD3|WORD2 quick|fast "
+    check_to_s(tc, phq, (ID)NULL, "field:\"WORD3|WORD2 quick|fast "
                "QUICK|FAST&brown|red|hairy fox\"~4");
 
     frt_q_deref(phq);
@@ -776,11 +776,11 @@ static void test_multi_phrase_query(TestCase *tc, void *data)
     frt_phq_add_term(phq, "two", 1);
     frt_phq_add_term(phq, "one", 1);
     tst_check_hits(tc, searcher, phq, "2", -1);
-    check_to_s(tc, phq, (FrtSymbol)NULL, "field:\"WORD3|x&one two one\"");
+    check_to_s(tc, phq, (ID)NULL, "field:\"WORD3|x&one two one\"");
 
     frt_phq_set_slop(phq, 4);
     tst_check_hits(tc, searcher, phq, "2", -1);
-    check_to_s(tc, phq, (FrtSymbol)NULL, "field:\"WORD3|x&one two one\"~4");
+    check_to_s(tc, phq, (ID)NULL, "field:\"WORD3|x&one two one\"~4");
     frt_q_deref(phq);
 
     /* test phrase query on non-existing field doesn't break anything */
@@ -900,40 +900,40 @@ static void test_multi_term_query(TestCase *tc, void *data)
     mtq = frt_multi_tq_new_conf(field, 4, 0.5);
     tst_check_hits(tc, searcher, mtq, "", -1);
     check_to_s(tc, mtq, field, "\"\"");
-    check_to_s(tc, mtq, (FrtSymbol)NULL, "field:\"\"");
+    check_to_s(tc, mtq, (ID)NULL, "field:\"\"");
 
     frt_multi_tq_add_term(mtq, "brown");
     tst_check_hits(tc, searcher, mtq, "1, 8, 16, 17", -1);
     check_to_s(tc, mtq, field, "\"brown\"");
-    check_to_s(tc, mtq, (FrtSymbol)NULL, "field:\"brown\"");
+    check_to_s(tc, mtq, (ID)NULL, "field:\"brown\"");
 
 
     /* 0.4f boost is below the 0.5 threshold so term is ignored */
     frt_multi_tq_add_term_boost(mtq, "fox", 0.4f);
     tst_check_hits(tc, searcher, mtq, "1, 8, 16, 17", -1);
     check_to_s(tc, mtq, field, "\"brown\"");
-    check_to_s(tc, mtq, (FrtSymbol)NULL, "field:\"brown\"");
+    check_to_s(tc, mtq, (ID)NULL, "field:\"brown\"");
 
     /* 0.6f boost is above the 0.5 threshold so term is included */
     frt_multi_tq_add_term_boost(mtq, "fox", 0.6f);
     tst_check_hits(tc, searcher, mtq, "1, 8, 11, 14, 16, 17", -1);
     check_to_s(tc, mtq, field, "\"fox^0.6|brown\"");
-    check_to_s(tc, mtq, (FrtSymbol)NULL, "field:\"fox^0.6|brown\"");
+    check_to_s(tc, mtq, (ID)NULL, "field:\"fox^0.6|brown\"");
 
     frt_multi_tq_add_term_boost(mtq, "fast", 50.0f);
     tst_check_hits(tc, searcher, mtq, "1, 8, 11, 14, 16, 17", 8);
     check_to_s(tc, mtq, field, "\"fox^0.6|brown|fast^50.0\"");
-    check_to_s(tc, mtq, (FrtSymbol)NULL, "field:\"fox^0.6|brown|fast^50.0\"");
+    check_to_s(tc, mtq, (ID)NULL, "field:\"fox^0.6|brown|fast^50.0\"");
 
 
     mtq->boost = 80.1f;
-    check_to_s(tc, mtq, (FrtSymbol)NULL, "field:\"fox^0.6|brown|fast^50.0\"^80.1");
+    check_to_s(tc, mtq, (ID)NULL, "field:\"fox^0.6|brown|fast^50.0\"^80.1");
     frt_multi_tq_add_term(mtq, "word1");
-    check_to_s(tc, mtq, (FrtSymbol)NULL, "field:\"fox^0.6|brown|word1|fast^50.0\"^80.1");
+    check_to_s(tc, mtq, (ID)NULL, "field:\"fox^0.6|brown|word1|fast^50.0\"^80.1");
     frt_multi_tq_add_term(mtq, "word2");
-    check_to_s(tc, mtq, (FrtSymbol)NULL, "field:\"brown|word1|word2|fast^50.0\"^80.1");
+    check_to_s(tc, mtq, (ID)NULL, "field:\"brown|word1|word2|fast^50.0\"^80.1");
     frt_multi_tq_add_term(mtq, "word3");
-    check_to_s(tc, mtq, (FrtSymbol)NULL, "field:\"brown|word1|word2|fast^50.0\"^80.1");
+    check_to_s(tc, mtq, (ID)NULL, "field:\"brown|word1|word2|fast^50.0\"^80.1");
 
     /* test MultiTermWeight.to_s */
     w = searcher->create_weight(searcher, mtq);
@@ -956,7 +956,7 @@ static void test_multi_term_query(TestCase *tc, void *data)
     frt_bq_add_query(bq, mtq, FRT_BC_MUST);
     tst_check_hits(tc, searcher, bq, "1, 11, 14, 16, 17", -1);
     check_to_s(tc, bq, field, "+quick +\"fox^0.6|brown|word1\"");
-    check_to_s(tc, bq, (FrtSymbol)NULL, "+field:quick +field:\"fox^0.6|brown|word1\"");
+    check_to_s(tc, bq, (ID)NULL, "+field:quick +field:\"fox^0.6|brown|word1\"");
     frt_q_deref(bq);
     frt_q_deref(mtq);
 
@@ -978,7 +978,7 @@ static void test_multi_term_query_hash(TestCase *tc, void *data)
     (void)data;
 
 
-    check_to_s(tc, q1, (FrtSymbol)NULL, "field:\"\"");
+    check_to_s(tc, q1, (ID)NULL, "field:\"\"");
     Assert(frt_q_hash(q1) == frt_q_hash(q2), "Queries should be equal");
     Assert(frt_q_eq(q1, q1), "Same queries should be equal");
     Assert(frt_q_eq(q1, q2), "Queries should be equal");
@@ -1637,7 +1637,7 @@ static void test_search_unscored(TestCase *tc, void *data)
 
 TestSuite *ts_search(TestSuite *suite)
 {
-    FrtStore *store = frt_open_ram_store();
+    FrtStore *store = frt_open_ram_store(NULL);
     FrtIndexReader *ir;
     FrtSearcher *searcher;
 
@@ -1653,7 +1653,7 @@ TestSuite *ts_search(TestSuite *suite)
     tst_run_test(suite, test_default_similarity, NULL);
 
     prepare_search_index(store);
-    ir = frt_ir_open(store);
+    ir = frt_ir_open(NULL, store);
     searcher = frt_isea_new(ir);
 
     tst_run_test(suite, test_get_doc, (void *)searcher);
@@ -1702,22 +1702,21 @@ static void prepare_multi_search_index(FrtStore *store, struct Data data[],
 {
     int i;
     FrtIndexWriter *iw;
-    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES,
-                              FRT_INDEX_YES,
-                              FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
-    FrtFieldInfo *fi = frt_fi_new(rb_intern("empty-field"), FRT_STORE_NO, FRT_INDEX_NO, FRT_TERM_VECTOR_NO);
+    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
+    FrtFieldInfo *fi = frt_fi_new(rb_intern("empty-field"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_NO, FRT_TERM_VECTOR_NO);
     frt_fis_add_field(fis, fi);
     frt_index_create(store, fis);
     frt_fis_deref(fis);
+    rb_encoding *enc = rb_enc_find("ASCII-8BIT");
 
-    iw = frt_iw_open(store, dbl_analyzer_new(), NULL);
+    iw = frt_iw_open(NULL, store, dbl_analyzer_new(), NULL);
     for (i = 0; i < d_cnt; i++) {
         FrtDocument *doc = frt_doc_new();
         doc->boost = (float)(i+w);
-        frt_doc_add_field(doc, frt_df_add_data(frt_df_new(date), (char *)data[i].date));
-        frt_doc_add_field(doc, frt_df_add_data(frt_df_new(field), (char *)data[i].field));
-        frt_doc_add_field(doc, frt_df_add_data(frt_df_new(cat), (char *)data[i].cat));
-        frt_doc_add_field(doc, frt_df_add_data(frt_df_new(number), (char *)data[i].number));
+        frt_doc_add_field(doc, frt_df_add_data(frt_df_new(date), (char *)data[i].date, enc));
+        frt_doc_add_field(doc, frt_df_add_data(frt_df_new(field), (char *)data[i].field, enc));
+        frt_doc_add_field(doc, frt_df_add_data(frt_df_new(cat), (char *)data[i].cat, enc));
+        frt_doc_add_field(doc, frt_df_add_data(frt_df_new(number), (char *)data[i].number, enc));
         frt_iw_add_doc(iw, doc);
        frt_doc_destroy(doc);
     }
@@ -1826,8 +1825,8 @@ static void test_query_combine(TestCase *tc, void *data)
 
 TestSuite *ts_multi_search(TestSuite *suite)
 {
-    FrtStore *store0 = frt_open_ram_store();
-    FrtStore *store1 = frt_open_ram_store();
+    FrtStore *store0 = frt_open_ram_store(NULL);
+    FrtStore *store1 = frt_open_ram_store(NULL);
 
     FrtIndexReader *ir0, *ir1;
     FrtSearcher **searchers;
@@ -1843,8 +1842,8 @@ TestSuite *ts_multi_search(TestSuite *suite)
     prepare_multi_search_index(store0, test_data, 9, 1);
     prepare_multi_search_index(store1, test_data + 9, FRT_NELEMS(test_data) - 9, 10);
 
-    ir0 = frt_ir_open(store0);
-    ir1 = frt_ir_open(store1);
+    ir0 = frt_ir_open(NULL, store0);
+    ir1 = frt_ir_open(NULL, store1);
     searchers = FRT_ALLOC_N(FrtSearcher *, 2);
     searchers[0] = frt_isea_new(ir0);
     searchers[1] = frt_isea_new(ir1);
@@ -1870,4 +1869,3 @@ TestSuite *ts_multi_search(TestSuite *suite)
     frt_searcher_close(searcher);
     return suite;
 }
-

data/ext/isomorfeus_ferret_ext/test_segments.c

@@ -16,9 +16,10 @@
 #define Asi_has_vals(si, name, doc_cnt, store)\
     tst_si_eql(__LINE__, tc, si, name, doc_cnt, store)
 static FrtFieldInfos dummy_fis = {
-    FRT_STORE_NO,       /* store */
-    FRT_INDEX_NO,       /* index */
-    FRT_TERM_VECTOR_NO, /* term_vector */
+    FRT_STORE_NO,          /* store */
+    FRT_COMPRESSION_NONE,  /* compression */
+    FRT_INDEX_NO,          /* index */
+    FRT_TERM_VECTOR_NO,    /* term_vector */
     0,              /* size */
     0,              /* capa */
     NULL,           /* fields */
@@ -154,7 +155,7 @@ void test_sis_rw(TestCase *tc, void *data)
 
 TestSuite *ts_segments(TestSuite *suite)
 {
-    FrtStore *store = frt_open_ram_store();
+    FrtStore *store = frt_open_ram_store(NULL);
 
     suite = ADD_SUITE(suite);
 

data/ext/isomorfeus_ferret_ext/test_sort.c

@@ -3,9 +3,11 @@
 #include "frt_search.h"
 #include "test.h"
 
+#undef close
+
 #define ARRAY_SIZE 20
 
-static FrtSymbol search, string, integer, flt;
+static ID search, string, integer, flt;
 
 typedef struct SortTestData {
     const char *search;
@@ -17,10 +19,11 @@ typedef struct SortTestData {
 static void add_sort_test_data(SortTestData *std, FrtIndexWriter *iw)
 {
     FrtDocument *doc = frt_doc_new();
-    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(search), (char *)std->search));
-    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(string), (char *)std->string));
-    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(integer), (char *)std->integer));
-    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(flt), (char *)std->flt));
+    rb_encoding *enc = rb_enc_find("ASCII-8BIT");
+    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(search), (char *)std->search, enc));
+    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(string), (char *)std->string, enc));
+    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(integer), (char *)std->integer, enc));
+    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(flt), (char *)std->flt, enc));
 
     sscanf(std->flt, "%f", &doc->boost);
 
@@ -45,11 +48,11 @@ static void sort_test_setup(FrtStore *store)
 {
     int i;
     FrtIndexWriter *iw;
-    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_INDEX_YES, FRT_TERM_VECTOR_YES);
+    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_YES);
     frt_index_create(store, fis);
     frt_fis_deref(fis);
 
-    iw = frt_iw_open(store, frt_whitespace_analyzer_new(false), NULL);
+    iw = frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), NULL);
 
     for (i = 0; i < FRT_NELEMS(data); i++) {
         add_sort_test_data(&data[i], iw);
@@ -60,21 +63,21 @@ static void sort_test_setup(FrtStore *store)
 static void sort_multi_test_setup(FrtStore *store1, FrtStore *store2)
 {
     int i;
-    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_INDEX_YES, FRT_TERM_VECTOR_YES);
+    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_YES);
     FrtIndexWriter *iw;
 
     frt_index_create(store1, fis);
     frt_index_create(store2, fis);
     frt_fis_deref(fis);
 
-    iw = frt_iw_open(store1, frt_whitespace_analyzer_new(false), NULL);
+    iw = frt_iw_open(NULL, store1, frt_whitespace_analyzer_new(false), NULL);
 
     for (i = 0; i < FRT_NELEMS(data)/2; i++) {
         add_sort_test_data(&data[i], iw);
     }
     frt_iw_close(iw);
 
-    iw = frt_iw_open(store2, frt_whitespace_analyzer_new(false), NULL);
+    iw = frt_iw_open(NULL, store2, frt_whitespace_analyzer_new(false), NULL);
 
     for (i = FRT_NELEMS(data)/2; i < FRT_NELEMS(data); i++) {
         add_sort_test_data(&data[i], iw);
@@ -288,7 +291,7 @@ static void test_sorts(TestCase *tc, void *data)
 TestSuite *ts_sort(TestSuite *suite)
 {
     FrtSearcher *sea, **searchers;
-    FrtStore *store = frt_open_ram_store(), *fs_store;
+    FrtStore *store = frt_open_ram_store(NULL), *fs_store;
 
     search = rb_intern("search");
     string = rb_intern("string");
@@ -302,7 +305,7 @@ TestSuite *ts_sort(TestSuite *suite)
     tst_run_test(suite, test_sort_field_to_s, NULL);
     tst_run_test(suite, test_sort_to_s, NULL);
 
-    sea = frt_isea_new(frt_ir_open(store));
+    sea = frt_isea_new(frt_ir_open(NULL, store));
 
     tst_run_test(suite, test_sorts, (void *)sea);
 
@@ -319,8 +322,8 @@ TestSuite *ts_sort(TestSuite *suite)
 
     searchers = FRT_ALLOC_N(FrtSearcher *, 2);
 
-    searchers[0] = frt_isea_new(frt_ir_open(store));
-    searchers[1] = frt_isea_new(frt_ir_open(fs_store));
+    searchers[0] = frt_isea_new(frt_ir_open(NULL, store));
+    searchers[1] = frt_isea_new(frt_ir_open(NULL, fs_store));
 
     sea = frt_msea_new(searchers, 2, true);
     tst_run_test(suite, test_sorts, (void *)sea);

data/ext/isomorfeus_ferret_ext/test_store.c

@@ -3,6 +3,8 @@
 #include <limits.h>
 #include "test.h"
 
+#undef rename
+
 #define TEST_LOCK_NAME "test"
 
 typedef struct WithLockTestArg {

data/ext/isomorfeus_ferret_ext/test_term.c

@@ -1,6 +1,8 @@
 #include "frt_index.h"
 #include "test.h"
 
+#undef close
+
 /***************************************************************************
  *
  * FrtTermInfosWriter
@@ -336,7 +338,7 @@ static void test_term_infos_reader(TestCase *tc, void *data)
 
 TestSuite *ts_term(TestSuite *suite)
 {
-    FrtStore *store = frt_open_ram_store();
+    FrtStore *store = frt_open_ram_store(NULL);
 
     suite = ADD_SUITE(suite);
 

data/ext/isomorfeus_ferret_ext/test_term_vectors.c

@@ -33,14 +33,13 @@ static void test_posting(TestCase *tc, void *data)
     Apnull(pl->last_occ->next);
 }
 
-static FrtFieldInfos *create_tv_fis()
-{
-    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_NO, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_NO);
-    frt_fis_add_field(fis, frt_fi_new(rb_intern("tv"), FRT_STORE_NO, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_YES));
-    frt_fis_add_field(fis, frt_fi_new(rb_intern("tv2"), FRT_STORE_NO, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_YES));
-    frt_fis_add_field(fis, frt_fi_new(rb_intern("tv_with_positions"), FRT_STORE_NO, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_POSITIONS));
-    frt_fis_add_field(fis, frt_fi_new(rb_intern("tv_with_offsets"), FRT_STORE_NO, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_OFFSETS));
-    frt_fis_add_field(fis, frt_fi_new(rb_intern("tv_with_positions_offsets"), FRT_STORE_NO, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS));
+static FrtFieldInfos *create_tv_fis(void) {
+    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_NO);
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("tv"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_YES));
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("tv2"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_YES));
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("tv_with_positions"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_POSITIONS));
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("tv_with_offsets"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_OFFSETS));
+    frt_fis_add_field(fis, frt_fi_new(rb_intern("tv_with_positions_offsets"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS));
     return fis;
 }
 
@@ -85,7 +84,7 @@ static FrtOffset *create_tv_offsets(FrtMemoryPool *mp)
 static void test_tv_single_doc(TestCase *tc, void *data)
 {
     int i, j;
-    FrtStore *store = frt_open_ram_store();
+    FrtStore *store = frt_open_ram_store(NULL);
     FrtMemoryPool *mp = (FrtMemoryPool *)data;
     FrtFieldsReader *fr;
     FrtFieldsWriter *fw;
@@ -227,7 +226,7 @@ static void test_tv_single_doc(TestCase *tc, void *data)
 static void test_tv_multi_doc(TestCase *tc, void *data)
 {
     int i, j;
-    FrtStore *store = frt_open_ram_store();
+    FrtStore *store = frt_open_ram_store(NULL);
     FrtMemoryPool *mp = (FrtMemoryPool *)data;
     FrtFieldsReader *fr;
     FrtFieldsWriter *fw;

data/ext/isomorfeus_ferret_ext/test_test.c

@@ -1,8 +1,7 @@
 #include "frt_except.h"
 #include "test.h"
 
-static void raise_eof()
-{
+static void raise_eof(void) {
     FRT_RAISE(FRT_EOF_ERROR, "Error, end of file");
 }
 

data/ext/isomorfeus_ferret_ext/test_threading.c

@@ -71,12 +71,12 @@ static void do_add_doc(FrtIndex *index)
 {
     FrtDocument *doc = frt_doc_new();
     int n = rand();
-
-    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(id), frt_strfmt("%d", n)))->destroy_data = true;
-    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(contents), num_to_str(n)))->destroy_data = true;
+    rb_encoding *enc = rb_enc_find("ASCII-8BIT");
+    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(id), frt_strfmt("%d", n), enc))->destroy_data = true;
+    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(contents), num_to_str(n), enc))->destroy_data = true;
     tlog("Adding %d\n", n);
     frt_index_add_doc(index, doc);
-   frt_doc_destroy(doc);
+    frt_doc_destroy(doc);
 }
 
 static void do_search(FrtIndex *index)
@@ -84,17 +84,18 @@ static void do_search(FrtIndex *index)
     int n = rand(), i;
     char *query = num_to_str(n);
     FrtTopDocs *td;
+    rb_encoding *enc = rb_enc_find("ASCII-8BIT");
 
     tlog("Searching for %d\n", n);
 
     frt_mutex_lock(&index->mutex);
-    td = frt_index_search_str(index, query, 0, 3, NULL, NULL, NULL);
+    td = frt_index_search_str(index, query, 0, 3, NULL, NULL, NULL, enc);
     free(query);
     for (i = 0; i < td->size; i++) {
         FrtHit *hit = td->hits[i];
         FrtDocument *doc = frt_index_get_doc(index, hit->doc);
         tlog("Hit for %d: %s - %f\n", hit->doc, frt_doc_get_field(doc, id)->data[0], hit->score);
-       frt_doc_destroy(doc);
+        frt_doc_destroy(doc);
     }
     tlog("Searched for %d: total = %d\n", n, td->total_hits);
     frt_mutex_unlock(&index->mutex);
@@ -157,10 +158,8 @@ TestSuite *ts_threading(TestSuite *suite)
     FrtStore *store = frt_open_fs_store("./test/testdir/store");
     FrtIndex *index;
     FrtHashSet *def_fields = frt_hs_new_ptr(NULL);
-    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_INDEX_YES,
-                              FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
-    frt_fis_add_field(fis, frt_fi_new(id, FRT_STORE_YES, FRT_INDEX_UNTOKENIZED,
-                              FRT_TERM_VECTOR_YES));
+    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
+    frt_fis_add_field(fis, frt_fi_new(id, FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_YES));
     frt_index_create(store, fis);
     frt_fis_deref(fis);
 

data/ext/isomorfeus_ferret_ext/testhelper.c

@@ -555,7 +555,6 @@ char *num_to_str(int num)
     return res;
 }
 
-FILE *temp_open()
-{
+FILE *temp_open(void) {
     return fopen("test/testdir/tmp", "w+");
 }

data/lib/isomorfeus/ferret/version.rb

@@ -1,5 +1,5 @@
 module Isomorfeus
   module Ferret
-    VERSION = '0.12.7'
+    VERSION = '0.13.0'
   end
 end