isomorfeus-ferret 0.12.7 → 0.13.0

data/ext/isomorfeus_ferret_ext/test_q_const_score.c

@@ -1,7 +1,9 @@
 #include "frt_search.h"
 #include "test.h"
 
-static FrtSymbol num;
+#undef close
+
+static ID num;
 
 extern void prepare_filter_index(FrtStore *store);
 
@@ -63,7 +65,7 @@ static void test_const_score_query_hash(TestCase *tc, void *data)
 
 TestSuite *ts_q_const_score(TestSuite *suite)
 {
-    FrtStore *store = frt_open_ram_store();
+    FrtStore *store = frt_open_ram_store(NULL);
     FrtIndexReader *ir;
     FrtSearcher *searcher;
 
@@ -72,7 +74,7 @@ TestSuite *ts_q_const_score(TestSuite *suite)
     suite = ADD_SUITE(suite);
 
     prepare_filter_index(store);
-    ir = frt_ir_open(store);
+    ir = frt_ir_open(NULL, store);
     searcher = frt_isea_new(ir);
 
     tst_run_test(suite, test_const_score_query, (void *)searcher);

data/ext/isomorfeus_ferret_ext/test_q_filtered.c

@@ -1,7 +1,9 @@
 #include "frt_search.h"
 #include "test.h"
 
-static FrtSymbol num, flipflop;
+#undef close
+
+static ID num, flipflop;
 
 extern void prepare_filter_index(FrtStore *store);
 
@@ -40,7 +42,7 @@ static void test_filtered_query(TestCase *tc, void *data)
 
 TestSuite *ts_q_filtered(TestSuite *suite)
 {
-    FrtStore *store = frt_open_ram_store();
+    FrtStore *store = frt_open_ram_store(NULL);
     FrtIndexReader *ir;
     FrtSearcher *searcher;
 
@@ -50,7 +52,7 @@ TestSuite *ts_q_filtered(TestSuite *suite)
     suite = ADD_SUITE(suite);
 
     prepare_filter_index(store);
-    ir = frt_ir_open(store);
+    ir = frt_ir_open(NULL, store);
     searcher = frt_isea_new(ir);
 
     tst_run_test(suite, test_filtered_query, (void *)searcher);

data/ext/isomorfeus_ferret_ext/test_q_fuzzy.c

@@ -1,19 +1,22 @@
 #include "frt_search.h"
 #include "test.h"
 
+#undef close
+
 #define ARRAY_SIZE 20
 
-static FrtSymbol field;
+static ID field;
 
 static void add_doc(const char *text, FrtIndexWriter *iw)
 {
     FrtDocument *doc = frt_doc_new();
-    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(field), (char *)text));
+    rb_encoding *enc = rb_enc_find("ASCII-8BIT");
+    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(field), (char *)text, enc));
     frt_iw_add_doc(iw, doc);
     frt_doc_destroy(doc);
 }
 
-void check_to_s(TestCase *tc, FrtQuery *query, FrtSymbol field, const char *q_str);
+void check_to_s(TestCase *tc, FrtQuery *query, ID field, const char *q_str);
 
 static void do_prefix_test(TestCase *tc, FrtSearcher *searcher, const char *qstr, const char *expected_hits, int pre_len, float min_sim)
 {
@@ -30,11 +33,11 @@ static void test_fuzziness(TestCase *tc, void *data)
     FrtSearcher *sea;
     FrtTopDocs *top_docs;
     FrtQuery *q;
-    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_INDEX_YES, FRT_TERM_VECTOR_YES);
+    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_YES);
     frt_index_create(store, fis);
     frt_fis_deref(fis);
 
-    iw = frt_iw_open(store, frt_whitespace_analyzer_new(false), NULL);
+    iw = frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), NULL);
 
     add_doc("aaaaa", iw);
     add_doc("aaaab", iw);
@@ -47,7 +50,7 @@ static void test_fuzziness(TestCase *tc, void *data)
     add_doc("aaaaaaaaaaaaaaaaaaaaaaa", iw); /* test max_distances problem */
     frt_iw_close(iw);
 
-    ir = frt_ir_open(store);
+    ir = frt_ir_open(NULL, store);
     sea = frt_isea_new(ir);
 
     q = frt_fuzq_new_conf(field, "aaaaa", 0.0, 5, 10);
@@ -117,16 +120,16 @@ static void test_fuzziness_long(TestCase *tc, void *data)
     FrtIndexReader *ir;
     FrtTopDocs *top_docs;
     FrtQuery *q;
-    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_INDEX_YES, FRT_TERM_VECTOR_YES);
+    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_YES);
     frt_index_create(store, fis);
     frt_fis_deref(fis);
 
-    iw = frt_iw_open(store, frt_whitespace_analyzer_new(false), NULL);
+    iw = frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), NULL);
 
     add_doc("aaaaaaa", iw);
     add_doc("segment", iw);
     frt_iw_close(iw);
-    ir = frt_ir_open(store);
+    ir = frt_ir_open(NULL, store);
     sea = frt_isea_new(ir);
 
     /* not similar enough: */
@@ -225,7 +228,7 @@ static void test_fuzzy_query_hash(TestCase *tc, void *data)
 
 TestSuite *ts_q_fuzzy(TestSuite *suite)
 {
-    FrtStore *store = frt_open_ram_store();
+    FrtStore *store = frt_open_ram_store(NULL);
 
     field = rb_intern("field");
 

data/ext/isomorfeus_ferret_ext/test_q_parser.c

@@ -1,13 +1,15 @@
 #include "frt_search.h"
 #include "test.h"
 
+extern rb_encoding *utf8_encoding;
+
 typedef struct QPTestPair {
     const char *qstr;
     const char *qres;
 } QPTestPair;
 
 #define PARSER_TEST(str, res) do {\
-    FrtQuery *q = qp_parse(parser, (char *)str);\
+    FrtQuery *q = qp_parse(parser, (char *)str, enc);\
     char *qres = q->to_s(q, rb_intern("xx"));\
     Asequal(res, qres);\
     frt_q_deref(q);\
@@ -141,7 +143,7 @@ static void test_q_parser(TestCase *tc, void *data)
         {"f1:(aaa f2:bbb ccc)", "f1:aaa f2:bbb f1:ccc"}
     };
     (void)data;
-
+    rb_encoding *enc = rb_enc_find("ASCII-8BIT");
     FRT_REF(analyzer);
     parser = frt_qp_new(analyzer);
     frt_qp_add_field(parser, rb_intern("xx"),    true,  true);
@@ -186,7 +188,7 @@ static void test_q_parser(TestCase *tc, void *data)
 static void test_q_parser_standard_analyzer(TestCase *tc, void *data)
 {
     int i;
-    FrtAnalyzer *analyzer = frt_mb_standard_analyzer_new(true);
+    FrtAnalyzer *analyzer = frt_standard_analyzer_new(true);
     FrtQParser *parser;
     QPTestPair pairs[] = {
         {"", ""},
@@ -307,6 +309,7 @@ static void test_q_parser_standard_analyzer(TestCase *tc, void *data)
             */
     };
     (void)data;
+    rb_encoding *enc = rb_enc_find("ASCII-8BIT");
 
     FRT_REF(analyzer);
     parser = frt_qp_new(analyzer);
@@ -390,6 +393,38 @@ static void test_qp_bad_queries(TestCase *tc, void *data)
         {"::|)*&one)(*two(*&\"", "\"one two\"~1"}
     };
     (void)data;
+    rb_encoding *enc = rb_enc_find("ASCII-8BIT");
+
+    parser = frt_qp_new(frt_letter_analyzer_new(true));
+    frt_qp_add_field(parser, rb_intern("xx"),    true,  true);
+    frt_qp_add_field(parser, rb_intern("f1"),    false, true);
+    frt_qp_add_field(parser, rb_intern("f2"),    false, true);
+    frt_qp_add_field(parser, rb_intern("field"), false, true);
+
+    parser->handle_parse_errors = true;
+
+    for (i = 0; i < FRT_NELEMS(pairs); i++) {
+        PARSER_TEST(pairs[i].qstr, pairs[i].qres);
+    }
+    parser->clean_str = true;
+    for (i = 0; i < FRT_NELEMS(pairs); i++) {
+        PARSER_TEST(pairs[i].qstr, pairs[i].qres);
+    }
+    frt_qp_destroy(parser);
+}
+
+static void test_mb_qp_bad_queries(TestCase *tc, void *data)
+{
+    int i;
+    FrtQParser *parser;
+    QPTestPair pairs[] = {
+        {"[, ]", ""},
+        {"::*word", "word"},
+        {"::))*&)(*^&*(", ""},
+        {"::|)*&one)(*two(*&\"", "\"one two\"~1"}
+    };
+    (void)data;
+    rb_encoding *enc = utf8_encoding;
 
     parser = frt_qp_new(frt_letter_analyzer_new(true));
     frt_qp_add_field(parser, rb_intern("xx"),    true,  true);
@@ -414,20 +449,21 @@ static void test_qp_prefix_query(TestCase *tc, void *data)
     FrtQParser *parser;
     FrtQuery *q;
     (void)data;
+    rb_encoding *enc = rb_enc_find("ASCII-8BIT");
 
     parser = frt_qp_new(frt_letter_analyzer_new(true));
     frt_qp_add_field(parser, rb_intern("xx"), true,  true);
 
-    q = qp_parse(parser, (char *)"asdg*");
+    q = qp_parse(parser, (char *)"asdg*", enc);
     Aiequal(PREFIX_QUERY, q->type);
     frt_q_deref(q);
-    q = qp_parse(parser, (char *)"a?dg*");
+    q = qp_parse(parser, (char *)"a?dg*", enc);
     Aiequal(WILD_CARD_QUERY, q->type);
     frt_q_deref(q);
-    q = qp_parse(parser, (char *)"a*dg*");
+    q = qp_parse(parser, (char *)"a*dg*", enc);
     Aiequal(WILD_CARD_QUERY, q->type);
     frt_q_deref(q);
-    q = qp_parse(parser, (char *)"asdg*a");
+    q = qp_parse(parser, (char *)"asdg*a", enc);
     Aiequal(WILD_CARD_QUERY, q->type);
     frt_q_deref(q);
     frt_qp_destroy(parser);
@@ -437,6 +473,7 @@ static void test_qp_keyword_switch(TestCase *tc, void *data)
 {
     FrtQParser *parser;
     (void)data;
+    rb_encoding *enc = rb_enc_find("ASCII-8BIT");
 
     parser = frt_qp_new(frt_letter_analyzer_new(true));
     frt_qp_add_field(parser, rb_intern("xx"), true,  true);
@@ -457,6 +494,7 @@ TestSuite *ts_q_parser(TestSuite *suite)
     tst_run_test(suite, test_q_parser_standard_analyzer, NULL);
     tst_run_test(suite, test_qp_clean_str, NULL);
     tst_run_test(suite, test_qp_bad_queries, NULL);
+    tst_run_test(suite, test_mb_qp_bad_queries, NULL);
     tst_run_test(suite, test_qp_prefix_query, NULL);
     tst_run_test(suite, test_qp_keyword_switch, NULL);
 

data/ext/isomorfeus_ferret_ext/test_q_span.c

@@ -1,6 +1,8 @@
 #include "frt_search.h"
 #include "test.h"
 
+#undef close
+
 #define ARRAY_SIZE 20
 #define TEST_SE(query, ir, expected) do { \
     FrtSpanEnum *__se = ((FrtSpanQuery *)query)->get_spans(query, ir); \
@@ -10,12 +12,13 @@
     free(__tmp);                                                       \
 } while(0)
 
-static FrtSymbol field;
+static ID field;
 
 static void add_doc(const char *text, FrtIndexWriter *iw)
 {
     FrtDocument *doc = frt_doc_new();
-    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(field), (char *)text));
+    rb_encoding *enc = rb_enc_find("ASCII-8BIT");
+    frt_doc_add_field(doc, frt_df_add_data(frt_df_new(field), (char *)text, enc));
     frt_iw_add_doc(iw, doc);
    frt_doc_destroy(doc);
 }
@@ -25,7 +28,7 @@ static void span_test_setup(FrtStore *store)
 {
     const char **d;
     FrtIndexWriter *iw;
-    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_INDEX_YES, FRT_TERM_VECTOR_YES);
+    FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_YES);
     const char *data[] = {
         "start finish one two three four five six seven",
         "start one finish two three four five six seven",
@@ -63,7 +66,7 @@ static void span_test_setup(FrtStore *store)
     frt_index_create(store, fis);
     frt_fis_deref(fis);
 
-    iw = frt_iw_open(store, frt_whitespace_analyzer_new(false), NULL);
+    iw = frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), NULL);
 
     for (d = data; *d != NULL; d++) {
         add_doc(*d, iw);
@@ -78,7 +81,7 @@ static void test_span_term(TestCase *tc, void *data)
     FrtSearcher *sea;
     FrtQuery *tq;
 
-    ir = frt_ir_open(store);
+    ir = frt_ir_open(NULL, store);
     sea = frt_isea_new(ir);
 
     tq = frt_spantq_new(rb_intern("notafield"), "nine");
@@ -132,7 +135,7 @@ static void test_span_multi_term(TestCase *tc, void *data)
     FrtSearcher *sea;
     FrtQuery *mtq;
 
-    ir = frt_ir_open(store);
+    ir = frt_ir_open(NULL, store);
     sea = frt_isea_new(ir);
 
     mtq = frt_spanmtq_new(rb_intern("notafield"));
@@ -206,7 +209,7 @@ static void test_span_prefix(TestCase *tc, void *data)
     FrtQuery *prq;
     char *tmp;
 
-    ir = frt_ir_open(store);
+    ir = frt_ir_open(NULL, store);
     sea = frt_isea_new(ir);
 
     prq = frt_spanprq_new(rb_intern("notafield"), "fl");
@@ -261,7 +264,7 @@ static void test_span_first(TestCase *tc, void *data)
     FrtSearcher *sea;
     FrtQuery *q;
 
-    ir = frt_ir_open(store);
+    ir = frt_ir_open(NULL, store);
     sea = frt_isea_new(ir);
 
     q = frt_spanfq_new_nr(frt_spantq_new(field, "finish"), 1);
@@ -311,7 +314,7 @@ static void test_span_or(TestCase *tc, void *data)
     FrtSearcher *sea;
     FrtQuery *q;
 
-    ir = frt_ir_open(store);
+    ir = frt_ir_open(NULL, store);
     sea = frt_isea_new(ir);
     q = frt_spanoq_new();
     tst_check_hits(tc, sea, q, "", -1);
@@ -365,7 +368,7 @@ static void test_span_near(TestCase *tc, void *data)
     FrtSearcher *sea;
     FrtQuery *q;
 
-    ir = frt_ir_open(store);
+    ir = frt_ir_open(NULL, store);
     sea = frt_isea_new(ir);
 
     q = frt_spannq_new(0, true);
@@ -466,7 +469,7 @@ static void test_span_not(TestCase *tc, void *data)
     FrtSearcher *sea;
     FrtQuery *q, *nearq0, *nearq1;
 
-    ir = frt_ir_open(store);
+    ir = frt_ir_open(NULL, store);
     sea = frt_isea_new(ir);
 
     nearq0 = frt_spannq_new(4, true);
@@ -551,7 +554,7 @@ static void test_span_not_hash(TestCase *tc, void *data)
 
 TestSuite *ts_q_span(TestSuite *suite)
 {
-    FrtStore *store = frt_open_ram_store();
+    FrtStore *store = frt_open_ram_store(NULL);
     field = rb_intern("field");
     span_test_setup(store);
 

data/ext/isomorfeus_ferret_ext/test_ram_store.c

@@ -10,7 +10,7 @@ void test_write_to(TestCase *tc, void *data)
 {
     int i;
     char *tmp;
-    FrtStore *ram_store = frt_open_ram_store();
+    FrtStore *ram_store = frt_open_ram_store(NULL);
     FrtStore *fs_store = frt_open_fs_store("./test/testdir/store");
     char str[18] = "³³ øãíøäÄ";
     char buf[18000] = "";
@@ -43,7 +43,7 @@ void test_write_to(TestCase *tc, void *data)
 
     Aiequal(17021, fs_store->length(fs_store, "_rw_funny_string.cfs"));
     frt_store_deref(ram_store);
-    ram_store = frt_open_ram_store_and_copy(fs_store, false);
+    ram_store = frt_open_ram_store_and_copy(NULL, fs_store, false);
 
     istream = ram_store->open_input(ram_store, "_rw_funny_string.cfs");
     Asequal(str, tmp = frt_is_read_string(istream));
@@ -63,7 +63,7 @@ void test_write_to(TestCase *tc, void *data)
  */
 TestSuite *ts_ram_store(TestSuite *suite)
 {
-    FrtStore *store = frt_open_ram_store();
+    FrtStore *store = frt_open_ram_store(NULL);
 
     suite = ADD_SUITE(suite);