isomorfeus-ferret 0.12.7 → 0.13.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (166) hide show
  1. checksums.yaml +4 -4
  2. data/LICENSE +101 -19
  3. data/README.md +85 -13
  4. data/ext/isomorfeus_ferret_ext/bm_bitvector.c +22 -30
  5. data/ext/isomorfeus_ferret_ext/bm_hash.c +6 -12
  6. data/ext/isomorfeus_ferret_ext/bm_micro_string.c +3 -6
  7. data/ext/isomorfeus_ferret_ext/bm_store.c +11 -22
  8. data/ext/isomorfeus_ferret_ext/brotli_common_dictionary.c +1 -1
  9. data/ext/isomorfeus_ferret_ext/brotli_dec_decode.c +1 -1
  10. data/ext/isomorfeus_ferret_ext/bzlib.c +1572 -0
  11. data/ext/isomorfeus_ferret_ext/bzlib.h +282 -0
  12. data/ext/isomorfeus_ferret_ext/bzlib_blocksort.c +1094 -0
  13. data/ext/isomorfeus_ferret_ext/bzlib_compress.c +672 -0
  14. data/ext/isomorfeus_ferret_ext/bzlib_crctable.c +104 -0
  15. data/ext/isomorfeus_ferret_ext/bzlib_decompress.c +652 -0
  16. data/ext/isomorfeus_ferret_ext/bzlib_huffman.c +205 -0
  17. data/ext/isomorfeus_ferret_ext/bzlib_private.h +509 -0
  18. data/ext/isomorfeus_ferret_ext/bzlib_randtable.c +84 -0
  19. data/ext/isomorfeus_ferret_ext/fio_tmpfile.h +53 -53
  20. data/ext/isomorfeus_ferret_ext/frb_analysis.c +785 -1192
  21. data/ext/isomorfeus_ferret_ext/frb_index.c +497 -495
  22. data/ext/isomorfeus_ferret_ext/frb_qparser.c +48 -60
  23. data/ext/isomorfeus_ferret_ext/frb_search.c +1520 -1002
  24. data/ext/isomorfeus_ferret_ext/frb_store.c +96 -96
  25. data/ext/isomorfeus_ferret_ext/frb_threading.h +0 -1
  26. data/ext/isomorfeus_ferret_ext/frb_utils.c +147 -196
  27. data/ext/isomorfeus_ferret_ext/frt_analysis.c +695 -1090
  28. data/ext/isomorfeus_ferret_ext/frt_analysis.h +174 -170
  29. data/ext/isomorfeus_ferret_ext/frt_array.c +2 -4
  30. data/ext/isomorfeus_ferret_ext/frt_bitvector.c +9 -16
  31. data/ext/isomorfeus_ferret_ext/frt_bitvector.h +32 -81
  32. data/ext/isomorfeus_ferret_ext/frt_document.c +15 -20
  33. data/ext/isomorfeus_ferret_ext/frt_document.h +10 -10
  34. data/ext/isomorfeus_ferret_ext/frt_except.c +5 -12
  35. data/ext/isomorfeus_ferret_ext/frt_field_index.c +3 -3
  36. data/ext/isomorfeus_ferret_ext/frt_field_index.h +6 -7
  37. data/ext/isomorfeus_ferret_ext/frt_filter.c +35 -46
  38. data/ext/isomorfeus_ferret_ext/frt_fs_store.c +1 -0
  39. data/ext/isomorfeus_ferret_ext/frt_global.c +91 -200
  40. data/ext/isomorfeus_ferret_ext/frt_global.h +7 -18
  41. data/ext/isomorfeus_ferret_ext/frt_hash.c +1 -2
  42. data/ext/isomorfeus_ferret_ext/frt_ind.c +32 -35
  43. data/ext/isomorfeus_ferret_ext/frt_ind.h +9 -9
  44. data/ext/isomorfeus_ferret_ext/frt_index.c +603 -410
  45. data/ext/isomorfeus_ferret_ext/frt_index.h +272 -291
  46. data/ext/isomorfeus_ferret_ext/frt_lang.c +0 -2
  47. data/ext/isomorfeus_ferret_ext/frt_mempool.c +1 -2
  48. data/ext/isomorfeus_ferret_ext/frt_multimapper.c +4 -7
  49. data/ext/isomorfeus_ferret_ext/frt_q_boolean.c +68 -91
  50. data/ext/isomorfeus_ferret_ext/frt_q_const_score.c +35 -38
  51. data/ext/isomorfeus_ferret_ext/frt_q_filtered_query.c +53 -72
  52. data/ext/isomorfeus_ferret_ext/frt_q_fuzzy.c +25 -32
  53. data/ext/isomorfeus_ferret_ext/frt_q_match_all.c +21 -23
  54. data/ext/isomorfeus_ferret_ext/frt_q_multi_term.c +66 -103
  55. data/ext/isomorfeus_ferret_ext/frt_q_parser.c +207 -195
  56. data/ext/isomorfeus_ferret_ext/frt_q_phrase.c +20 -16
  57. data/ext/isomorfeus_ferret_ext/frt_q_prefix.c +17 -14
  58. data/ext/isomorfeus_ferret_ext/frt_q_range.c +102 -131
  59. data/ext/isomorfeus_ferret_ext/frt_q_span.c +179 -178
  60. data/ext/isomorfeus_ferret_ext/frt_q_term.c +47 -60
  61. data/ext/isomorfeus_ferret_ext/frt_q_wildcard.c +18 -16
  62. data/ext/isomorfeus_ferret_ext/frt_ram_store.c +45 -84
  63. data/ext/isomorfeus_ferret_ext/frt_search.c +105 -146
  64. data/ext/isomorfeus_ferret_ext/frt_search.h +331 -320
  65. data/ext/isomorfeus_ferret_ext/frt_similarity.c +5 -13
  66. data/ext/isomorfeus_ferret_ext/frt_similarity.h +7 -12
  67. data/ext/isomorfeus_ferret_ext/frt_sort.c +105 -149
  68. data/ext/isomorfeus_ferret_ext/frt_store.c +13 -7
  69. data/ext/isomorfeus_ferret_ext/frt_store.h +10 -2
  70. data/ext/isomorfeus_ferret_ext/frt_threading.h +0 -1
  71. data/ext/isomorfeus_ferret_ext/isomorfeus_ferret.c +22 -112
  72. data/ext/isomorfeus_ferret_ext/isomorfeus_ferret.h +2 -32
  73. data/ext/isomorfeus_ferret_ext/lz4.c +2495 -0
  74. data/ext/isomorfeus_ferret_ext/lz4.h +774 -0
  75. data/ext/isomorfeus_ferret_ext/lz4frame.c +1899 -0
  76. data/ext/isomorfeus_ferret_ext/lz4frame.h +623 -0
  77. data/ext/isomorfeus_ferret_ext/lz4hc.c +1615 -0
  78. data/ext/isomorfeus_ferret_ext/lz4hc.h +413 -0
  79. data/ext/isomorfeus_ferret_ext/lz4xxhash.c +1030 -0
  80. data/ext/isomorfeus_ferret_ext/lz4xxhash.h +328 -0
  81. data/ext/isomorfeus_ferret_ext/stem_modules.h +0 -86
  82. data/ext/isomorfeus_ferret_ext/test.c +0 -17
  83. data/ext/isomorfeus_ferret_ext/test_1710.c +11 -12
  84. data/ext/isomorfeus_ferret_ext/test_analysis.c +590 -583
  85. data/ext/isomorfeus_ferret_ext/test_compound_io.c +1 -1
  86. data/ext/isomorfeus_ferret_ext/test_document.c +19 -15
  87. data/ext/isomorfeus_ferret_ext/test_except.c +1 -2
  88. data/ext/isomorfeus_ferret_ext/test_fields.c +59 -60
  89. data/ext/isomorfeus_ferret_ext/test_file_deleter.c +10 -27
  90. data/ext/isomorfeus_ferret_ext/test_filter.c +11 -8
  91. data/ext/isomorfeus_ferret_ext/test_global.c +0 -46
  92. data/ext/isomorfeus_ferret_ext/test_hash.c +2 -2
  93. data/ext/isomorfeus_ferret_ext/test_hashset.c +1 -1
  94. data/ext/isomorfeus_ferret_ext/test_highlighter.c +15 -11
  95. data/ext/isomorfeus_ferret_ext/test_index.c +372 -365
  96. data/ext/isomorfeus_ferret_ext/test_q_const_score.c +5 -3
  97. data/ext/isomorfeus_ferret_ext/test_q_filtered.c +5 -3
  98. data/ext/isomorfeus_ferret_ext/test_q_fuzzy.c +13 -10
  99. data/ext/isomorfeus_ferret_ext/test_q_parser.c +45 -7
  100. data/ext/isomorfeus_ferret_ext/test_q_span.c +15 -12
  101. data/ext/isomorfeus_ferret_ext/test_ram_store.c +3 -3
  102. data/ext/isomorfeus_ferret_ext/test_search.c +60 -64
  103. data/ext/isomorfeus_ferret_ext/test_segments.c +5 -4
  104. data/ext/isomorfeus_ferret_ext/test_sort.c +17 -14
  105. data/ext/isomorfeus_ferret_ext/test_store.c +2 -0
  106. data/ext/isomorfeus_ferret_ext/test_term.c +3 -1
  107. data/ext/isomorfeus_ferret_ext/test_term_vectors.c +9 -10
  108. data/ext/isomorfeus_ferret_ext/test_test.c +1 -2
  109. data/ext/isomorfeus_ferret_ext/test_threading.c +9 -10
  110. data/ext/isomorfeus_ferret_ext/testhelper.c +1 -2
  111. data/lib/isomorfeus/ferret/version.rb +1 -1
  112. metadata +27 -57
  113. data/ext/isomorfeus_ferret_ext/email.rl +0 -21
  114. data/ext/isomorfeus_ferret_ext/frt_scanner.c +0 -900
  115. data/ext/isomorfeus_ferret_ext/frt_scanner.h +0 -28
  116. data/ext/isomorfeus_ferret_ext/frt_scanner_mb.c +0 -6706
  117. data/ext/isomorfeus_ferret_ext/frt_scanner_utf8.c +0 -4420
  118. data/ext/isomorfeus_ferret_ext/scanner.h +0 -28
  119. data/ext/isomorfeus_ferret_ext/scanner.in +0 -43
  120. data/ext/isomorfeus_ferret_ext/scanner.rl +0 -84
  121. data/ext/isomorfeus_ferret_ext/scanner_mb.rl +0 -200
  122. data/ext/isomorfeus_ferret_ext/scanner_utf8.rl +0 -85
  123. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_basque.c +0 -1167
  124. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_basque.h +0 -6
  125. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_catalan.c +0 -1433
  126. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_catalan.h +0 -6
  127. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_danish.c +0 -301
  128. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_danish.h +0 -6
  129. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_dutch.c +0 -590
  130. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_dutch.h +0 -6
  131. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_english.c +0 -1049
  132. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_english.h +0 -6
  133. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_finnish.c +0 -705
  134. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_finnish.h +0 -6
  135. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_french.c +0 -1239
  136. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_french.h +0 -6
  137. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_german.c +0 -477
  138. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_german.h +0 -6
  139. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_hungarian.c +0 -1217
  140. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_hungarian.h +0 -7
  141. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_indonesian.c +0 -394
  142. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_indonesian.h +0 -6
  143. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_irish.c +0 -457
  144. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_irish.h +0 -6
  145. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_italian.c +0 -1009
  146. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_italian.h +0 -6
  147. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_norwegian.c +0 -259
  148. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_norwegian.h +0 -6
  149. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_porter.c +0 -704
  150. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_porter.h +0 -6
  151. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_portuguese.c +0 -948
  152. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_portuguese.h +0 -6
  153. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_spanish.c +0 -1028
  154. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_spanish.h +0 -6
  155. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_swedish.c +0 -275
  156. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_swedish.h +0 -6
  157. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_2_hungarian.c +0 -849
  158. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_2_hungarian.h +0 -6
  159. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_2_romanian.c +0 -952
  160. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_2_romanian.h +0 -6
  161. data/ext/isomorfeus_ferret_ext/stem_KOI8_R_russian.c +0 -669
  162. data/ext/isomorfeus_ferret_ext/stem_KOI8_R_russian.h +0 -6
  163. data/ext/isomorfeus_ferret_ext/stem_modules.txt +0 -63
  164. data/ext/isomorfeus_ferret_ext/uchar-ucs4.rl +0 -1854
  165. data/ext/isomorfeus_ferret_ext/uchar-utf8.rl +0 -1999
  166. data/ext/isomorfeus_ferret_ext/url.rl +0 -27
@@ -155,7 +155,7 @@ void test_compound_io_many_files(TestCase *tc, void *data)
155
155
 
156
156
  TestSuite *ts_compound_io(TestSuite *suite)
157
157
  {
158
- FrtStore *store = frt_open_ram_store();
158
+ FrtStore *store = frt_open_ram_store(NULL);
159
159
 
160
160
  suite = ADD_SUITE(suite);
161
161
 
@@ -6,8 +6,9 @@ void test_df_standard(TestCase *tc, void *data)
6
6
  char *s;
7
7
  FrtDocField *df;
8
8
  (void)data;
9
+ rb_encoding *enc = rb_enc_find("ASCII-8BIT");
9
10
 
10
- df = frt_df_add_data(frt_df_new(rb_intern("title")), frt_estrdup("Life of Pi"));
11
+ df = frt_df_add_data(frt_df_new(rb_intern("title")), frt_estrdup("Life of Pi"), enc);
11
12
  df->destroy_data = true;
12
13
  Aiequal(1, df->size);
13
14
  Asequal("title", rb_id2name(df->name));
@@ -18,7 +19,7 @@ void test_df_standard(TestCase *tc, void *data)
18
19
  free(s);
19
20
  frt_df_destroy(df);
20
21
 
21
- df = frt_df_add_data_len(frt_df_new(rb_intern("title")), (char *)"new title", 9);
22
+ df = frt_df_add_data_len(frt_df_new(rb_intern("title")), (char *)"new title", 9, enc);
22
23
  Aiequal(1, df->size);
23
24
  Asequal("title", rb_id2name(df->name));
24
25
  Asequal("new title", df->data[0]);
@@ -32,20 +33,21 @@ void test_df_multi_fields(TestCase *tc, void *data)
32
33
  char *s;
33
34
  FrtDocField *df;
34
35
  (void)data;
36
+ rb_encoding *enc = rb_enc_find("ASCII-8BIT");
35
37
 
36
- df = frt_df_add_data(frt_df_new(rb_intern("title")), frt_estrdup("Vernon God Little"));
38
+ df = frt_df_add_data(frt_df_new(rb_intern("title")), frt_estrdup("Vernon God Little"), enc);
37
39
  df->destroy_data = true;
38
40
  Aiequal(1, df->size);
39
41
  Asequal("title", rb_id2name(df->name));
40
42
  Asequal("Vernon God Little", df->data[0]);
41
43
  Aiequal(strlen("Vernon God Little"), df->lengths[0]);
42
44
 
43
- frt_df_add_data(df, frt_estrdup("some more data"));
45
+ frt_df_add_data(df, frt_estrdup("some more data"), enc);
44
46
  Aiequal(2, df->size);
45
47
  Asequal("title: [\"Vernon God Little\", \"some more data\"]",
46
48
  s = frt_df_to_s(df));
47
49
  free(s);
48
- frt_df_add_data_len(df, frt_estrdup("and more data"), 14);
50
+ frt_df_add_data_len(df, frt_estrdup("and more data"), 14, enc);
49
51
  Aiequal(3, df->size);
50
52
  Asequal("title", rb_id2name(df->name));
51
53
  Asequal("Vernon God Little", df->data[0]);
@@ -54,13 +56,13 @@ void test_df_multi_fields(TestCase *tc, void *data)
54
56
 
55
57
  frt_df_destroy(df);
56
58
 
57
- df = frt_df_add_data(frt_df_new(rb_intern("data")), frt_estrdup("start"));
59
+ df = frt_df_add_data(frt_df_new(rb_intern("data")), frt_estrdup("start"), enc);
58
60
  df->destroy_data = true;
59
61
  Aiequal(1, df->size);
60
62
  for (i = 0; i < 1000; i++) {
61
63
  char buf[100];
62
64
  sprintf(buf, "<<%d>>", i);
63
- frt_df_add_data(df, frt_estrdup(buf));
65
+ frt_df_add_data(df, frt_estrdup(buf), enc);
64
66
  Aiequal(i + 2, df->size);
65
67
  }
66
68
  frt_df_destroy(df);
@@ -72,14 +74,15 @@ void test_doc(TestCase *tc, void *data)
72
74
  FrtDocument *doc;
73
75
  FrtDocField *df;
74
76
  (void)data;
77
+ rb_encoding *enc = rb_enc_find("ASCII-8BIT");
75
78
 
76
79
  doc = frt_doc_new();
77
- frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("title")), (char *)"title"));
80
+ frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("title")), (char *)"title", enc));
78
81
  Aiequal(1, doc->size);
79
- df = frt_df_add_data(frt_df_new(rb_intern("data")), (char *)"data1");
80
- frt_df_add_data(df, (char *)"data2");
81
- frt_df_add_data(df, (char *)"data3");
82
- frt_df_add_data(df, (char *)"data4");
82
+ df = frt_df_add_data(frt_df_new(rb_intern("data")), (char *)"data1", enc);
83
+ frt_df_add_data(df, (char *)"data2", enc);
84
+ frt_df_add_data(df, (char *)"data3", enc);
85
+ frt_df_add_data(df, (char *)"data4", enc);
83
86
  frt_doc_add_field(doc, df);
84
87
  Aiequal(2, doc->size);
85
88
  Asequal("title", rb_id2name(frt_doc_get_field(doc, rb_intern("title"))->name));
@@ -100,7 +103,7 @@ void test_doc(TestCase *tc, void *data)
100
103
  char *bufc;
101
104
  sprintf(buf, "<<%d>>", i);
102
105
  bufc = frt_estrdup(buf);
103
- df = frt_df_add_data(frt_df_new(rb_intern(bufc)), bufc);
106
+ df = frt_df_add_data(frt_df_new(rb_intern(bufc)), bufc, enc);
104
107
  df->destroy_data = true;
105
108
  frt_doc_add_field(doc, df);
106
109
  Aiequal(i + 1, doc->size);
@@ -122,12 +125,13 @@ void test_double_field_exception(TestCase *tc, void *data)
122
125
  FrtDocument *doc;
123
126
  FrtDocField *volatile df = NULL;
124
127
  (void)data;
128
+ rb_encoding *enc = rb_enc_find("ASCII-8BIT");
125
129
 
126
130
  doc = frt_doc_new();
127
- frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("title")), (char *)"title"));
131
+ frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("title")), (char *)"title", enc));
128
132
 
129
133
  FRT_TRY
130
- df = frt_df_add_data_len(frt_df_new(rb_intern("title")), (char *)"title", 5);
134
+ df = frt_df_add_data_len(frt_df_new(rb_intern("title")), (char *)"title", 5, enc);
131
135
  frt_doc_add_field(doc, df);
132
136
  case FRT_EXCEPTION:
133
137
  exception_thrown = true;
@@ -5,8 +5,7 @@
5
5
  static const char *msg1 = "Message One";
6
6
  static const char *msg2 = "Message Two";
7
7
 
8
- static void raise_exception()
9
- {
8
+ static void raise_exception(void) {
10
9
  FRT_RAISE(FRT_EXCEPTION, "%s", msg1);
11
10
  }
12
11
 
@@ -13,7 +13,7 @@
13
13
  void field_prop_test(TestCase *tc,
14
14
  int line_num,
15
15
  FrtFieldInfo *fi,
16
- FrtSymbol name,
16
+ ID name,
17
17
  float boost,
18
18
  bool is_stored,
19
19
  bool is_compressed,
@@ -22,8 +22,7 @@ void field_prop_test(TestCase *tc,
22
22
  bool omit_norms,
23
23
  bool store_term_vector,
24
24
  bool store_positions,
25
- bool store_offsets)
26
- {
25
+ bool store_offsets) {
27
26
  tst_ptr_equal(line_num, tc, (void *)name, (void *)fi->name);
28
27
  tst_flt_equal(line_num, tc, boost, fi->boost);
29
28
  tst_int_equal(line_num, tc, is_stored, fi_is_stored(fi));
@@ -46,19 +45,19 @@ static void test_fi_new(TestCase *tc, void *data)
46
45
  {
47
46
  FrtFieldInfo *fi;
48
47
  (void)data; /* suppress unused argument warning */
49
- fi = frt_fi_new(rb_intern("name"), FRT_STORE_NO, FRT_INDEX_NO, FRT_TERM_VECTOR_NO);
48
+ fi = frt_fi_new(rb_intern("name"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_NO, FRT_TERM_VECTOR_NO);
50
49
  do_field_prop_test(tc, fi, rb_intern("name"), 1.0, F, F, F, F, F, F, F, F);
51
50
  frt_fi_deref(fi);
52
- fi = frt_fi_new(rb_intern("name"), FRT_STORE_YES, FRT_INDEX_YES, FRT_TERM_VECTOR_YES);
51
+ fi = frt_fi_new(rb_intern("name"), FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_YES);
53
52
  do_field_prop_test(tc, fi, rb_intern("name"), 1.0, T, F, T, T, F, T, F, F);
54
53
  frt_fi_deref(fi);
55
- fi = frt_fi_new(rb_intern("name"), FRT_STORE_COMPRESS, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_POSITIONS);
54
+ fi = frt_fi_new(rb_intern("name"), FRT_STORE_YES, FRT_COMPRESSION_BROTLI, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_POSITIONS);
56
55
  do_field_prop_test(tc, fi, rb_intern("name"), 1.0, T, T, T, F, F, T, T, F);
57
56
  frt_fi_deref(fi);
58
- fi = frt_fi_new(rb_intern("name"), FRT_STORE_NO, FRT_INDEX_YES_OMIT_NORMS, FRT_TERM_VECTOR_WITH_OFFSETS);
57
+ fi = frt_fi_new(rb_intern("name"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_YES_OMIT_NORMS, FRT_TERM_VECTOR_WITH_OFFSETS);
59
58
  do_field_prop_test(tc, fi, rb_intern("name"), 1.0, F, F, T, T, T, T, F, T);
60
59
  frt_fi_deref(fi);
61
- fi = frt_fi_new(rb_intern("name"), FRT_STORE_NO, FRT_INDEX_UNTOKENIZED_OMIT_NORMS, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
60
+ fi = frt_fi_new(rb_intern("name"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_UNTOKENIZED_OMIT_NORMS, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
62
61
  fi->boost = 1000.0;
63
62
  do_field_prop_test(tc, fi, rb_intern("name"), 1000.0, F, F, T, F, T, T, T, T);
64
63
  frt_fi_deref(fi);
@@ -77,14 +76,14 @@ static void test_fis_basic(TestCase *tc, void *data)
77
76
  volatile bool arg_error = false;
78
77
  (void)data; /* suppress unused argument warning */
79
78
 
80
- fis = frt_fis_new(FRT_STORE_NO, FRT_INDEX_NO, FRT_TERM_VECTOR_NO);
81
- frt_fis_add_field(fis, frt_fi_new(rb_intern("FFFFFFFF"), FRT_STORE_NO, FRT_INDEX_NO, FRT_TERM_VECTOR_NO));
82
- frt_fis_add_field(fis, frt_fi_new(rb_intern("TFTTFTFF"), FRT_STORE_YES, FRT_INDEX_YES, FRT_TERM_VECTOR_YES));
83
- frt_fis_add_field(fis, frt_fi_new(rb_intern("TTTFFTTF"), FRT_STORE_COMPRESS, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_POSITIONS));
84
- frt_fis_add_field(fis, frt_fi_new(rb_intern("FFTTTTFT"), FRT_STORE_NO, FRT_INDEX_YES_OMIT_NORMS, FRT_TERM_VECTOR_WITH_OFFSETS));
85
- frt_fis_add_field(fis, frt_fi_new(rb_intern("FFTFTTTT"), FRT_STORE_NO, FRT_INDEX_UNTOKENIZED_OMIT_NORMS, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS));
79
+ fis = frt_fis_new(FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_NO, FRT_TERM_VECTOR_NO);
80
+ frt_fis_add_field(fis, frt_fi_new(rb_intern("FFFFFFFF"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_NO, FRT_TERM_VECTOR_NO));
81
+ frt_fis_add_field(fis, frt_fi_new(rb_intern("TFTTFTFF"), FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_YES));
82
+ frt_fis_add_field(fis, frt_fi_new(rb_intern("TTTFFTTF"), FRT_STORE_YES, FRT_COMPRESSION_BROTLI, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_POSITIONS));
83
+ frt_fis_add_field(fis, frt_fi_new(rb_intern("FFTTTTFT"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_YES_OMIT_NORMS, FRT_TERM_VECTOR_WITH_OFFSETS));
84
+ frt_fis_add_field(fis, frt_fi_new(rb_intern("FFTFTTTT"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_UNTOKENIZED_OMIT_NORMS, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS));
86
85
 
87
- fi = frt_fi_new(rb_intern("FFTFTTTT"), FRT_STORE_NO, FRT_INDEX_UNTOKENIZED_OMIT_NORMS, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
86
+ fi = frt_fi_new(rb_intern("FFTFTTTT"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_UNTOKENIZED_OMIT_NORMS, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
88
87
  FRT_TRY
89
88
  Apnull(frt_fis_add_field(fis, fi));
90
89
  case FRT_ARG_ERROR:
@@ -132,7 +131,7 @@ static void test_fis_with_default(TestCase *tc, void *data)
132
131
  FrtFieldInfos *fis;
133
132
  (void)data; /* suppress unused argument warning */
134
133
 
135
- fis = frt_fis_new(FRT_STORE_NO, FRT_INDEX_NO, FRT_TERM_VECTOR_NO);
134
+ fis = frt_fis_new(FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_NO, FRT_TERM_VECTOR_NO);
136
135
  do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("name")), rb_intern("name"), 1.0, F, F, F, F, F, F, F, F);
137
136
  do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("dave")), rb_intern("dave"), 1.0, F, F, F, F, F, F, F, F);
138
137
  do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("wert")), rb_intern("wert"), 1.0, F, F, F, F, F, F, F, F);
@@ -142,16 +141,16 @@ static void test_fis_with_default(TestCase *tc, void *data)
142
141
  Apnull(frt_fis_get_field(fis, rb_intern("random")));
143
142
  frt_fis_deref(fis);
144
143
 
145
- fis = frt_fis_new(FRT_STORE_YES, FRT_INDEX_YES, FRT_TERM_VECTOR_YES);
144
+ fis = frt_fis_new(FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_YES);
146
145
  do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("name")), rb_intern("name"), 1.0, T, F, T, T, F, T, F, F);
147
146
  frt_fis_deref(fis);
148
- fis = frt_fis_new(FRT_STORE_COMPRESS, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_POSITIONS);
147
+ fis = frt_fis_new(FRT_STORE_YES, FRT_COMPRESSION_BROTLI, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_POSITIONS);
149
148
  do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("name")), rb_intern("name"), 1.0, T, T, T, F, F, T, T, F);
150
149
  frt_fis_deref(fis);
151
- fis = frt_fis_new(FRT_STORE_NO, FRT_INDEX_YES_OMIT_NORMS, FRT_TERM_VECTOR_WITH_OFFSETS);
150
+ fis = frt_fis_new(FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_YES_OMIT_NORMS, FRT_TERM_VECTOR_WITH_OFFSETS);
152
151
  do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("name")), rb_intern("name"), 1.0, F, F, T, T, T, T, F, T);
153
152
  frt_fis_deref(fis);
154
- fis = frt_fis_new(FRT_STORE_NO, FRT_INDEX_UNTOKENIZED_OMIT_NORMS, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
153
+ fis = frt_fis_new(FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_UNTOKENIZED_OMIT_NORMS, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
155
154
  do_field_prop_test(tc, frt_fis_get_or_add_field(fis, rb_intern("name")), rb_intern("name"), 1.0, F, F, T, F, T, T, T, T);
156
155
  frt_fis_deref(fis);
157
156
  }
@@ -160,18 +159,17 @@ static void test_fis_rw(TestCase *tc, void *data)
160
159
  {
161
160
  char *str;
162
161
  FrtFieldInfos *fis;
163
- FrtStore *store = frt_open_ram_store();
162
+ FrtStore *store = frt_open_ram_store(NULL);
164
163
  FrtInStream *is;
165
164
  FrtOutStream *os;
166
165
  (void)data; /* suppress unused argument warning */
167
166
 
168
- fis = frt_fis_new(FRT_STORE_YES, FRT_INDEX_UNTOKENIZED_OMIT_NORMS,
169
- FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
170
- frt_fis_add_field(fis, frt_fi_new(rb_intern("FFFFFFFF"), FRT_STORE_NO, FRT_INDEX_NO, FRT_TERM_VECTOR_NO));
171
- frt_fis_add_field(fis, frt_fi_new(rb_intern("TFTTFTFF"), FRT_STORE_YES, FRT_INDEX_YES, FRT_TERM_VECTOR_YES));
172
- frt_fis_add_field(fis, frt_fi_new(rb_intern("TTTFFTTF"), FRT_STORE_COMPRESS, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_POSITIONS));
173
- frt_fis_add_field(fis, frt_fi_new(rb_intern("FFTTTTFT"), FRT_STORE_NO, FRT_INDEX_YES_OMIT_NORMS, FRT_TERM_VECTOR_WITH_OFFSETS));
174
- frt_fis_add_field(fis, frt_fi_new(rb_intern("FFTFTTTT"), FRT_STORE_NO, FRT_INDEX_UNTOKENIZED_OMIT_NORMS, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS));
167
+ fis = frt_fis_new(FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_UNTOKENIZED_OMIT_NORMS, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
168
+ frt_fis_add_field(fis, frt_fi_new(rb_intern("FFFFFFFF"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_NO, FRT_TERM_VECTOR_NO));
169
+ frt_fis_add_field(fis, frt_fi_new(rb_intern("TFTTFTFF"), FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_YES));
170
+ frt_fis_add_field(fis, frt_fi_new(rb_intern("TTTFFTTF"), FRT_STORE_YES, FRT_COMPRESSION_BROTLI, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_POSITIONS));
171
+ frt_fis_add_field(fis, frt_fi_new(rb_intern("FFTTTTFT"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_YES_OMIT_NORMS, FRT_TERM_VECTOR_WITH_OFFSETS));
172
+ frt_fis_add_field(fis, frt_fi_new(rb_intern("FFTFTTTT"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_UNTOKENIZED_OMIT_NORMS, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS));
175
173
  fis->fields[1]->boost = 2.0;
176
174
  fis->fields[2]->boost = 3.0;
177
175
  fis->fields[3]->boost = 4.0;
@@ -282,45 +280,44 @@ static char *prepare_bin_data(int len)
282
280
  Assert(memcmp(mdata, df->data[index], mlen) == 0, "Data should be equal");\
283
281
  } while (0)
284
282
 
285
- static FrtDocument *prepare_doc()
286
- {
283
+ static FrtDocument *prepare_doc(void) {
287
284
  FrtDocument *doc = frt_doc_new();
288
285
  FrtDocField *df;
289
286
  char *bin_data = prepare_bin_data(BIN_DATA_LEN);
287
+ rb_encoding *enc = rb_enc_find("ASCII-8BIT");
290
288
 
291
- frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("ignored")), (char *)"this fld's ignored"));
292
- frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("unstored")), (char *)"unstored ignored"));
293
- frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("stored")), (char *)"Yay, a stored field"));
294
- df = frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("stored_array")), (char *)"one"));
289
+ frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("ignored")), (char *)"this fld's ignored", enc));
290
+ frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("unstored")), (char *)"unstored ignored", enc));
291
+ frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("stored")), (char *)"Yay, a stored field", enc));
292
+ df = frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("stored_array")), (char *)"one", enc));
295
293
  df->destroy_data = false;
296
- frt_df_add_data(df, (char *)"two");
297
- frt_df_add_data(df, (char *)"three");
298
- frt_df_add_data(df, (char *)"four");
299
- frt_df_add_data_len(df, bin_data, BIN_DATA_LEN);
294
+ frt_df_add_data(df, (char *)"two", enc);
295
+ frt_df_add_data(df, (char *)"three", enc);
296
+ frt_df_add_data(df, (char *)"four", enc);
297
+ frt_df_add_data_len(df, bin_data, BIN_DATA_LEN, enc);
300
298
  frt_doc_add_field(doc, frt_df_add_data_len(frt_df_new(rb_intern("binary")), bin_data,
301
- BIN_DATA_LEN))->destroy_data = true;
302
- df = frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("array")), (char *)"ichi"));
303
- frt_df_add_data(df, (char *)"ni");
304
- frt_df_add_data(df, (char *)"san");
305
- frt_df_add_data(df, (char *)"yon");
306
- frt_df_add_data(df, (char *)"go");
299
+ BIN_DATA_LEN, enc))->destroy_data = true;
300
+ df = frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("array")), (char *)"ichi", enc));
301
+ frt_df_add_data(df, (char *)"ni", enc);
302
+ frt_df_add_data(df, (char *)"san", enc);
303
+ frt_df_add_data(df, (char *)"yon", enc);
304
+ frt_df_add_data(df, (char *)"go", enc);
307
305
 
308
306
  return doc;
309
307
  }
310
308
 
311
- static FrtFieldInfos *prepare_fis()
312
- {
313
- FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_INDEX_YES, FRT_TERM_VECTOR_NO);
314
- frt_fis_add_field(fis, frt_fi_new(rb_intern("ignored"), FRT_STORE_NO, FRT_INDEX_NO, FRT_TERM_VECTOR_NO));
315
- frt_fis_add_field(fis, frt_fi_new(rb_intern("unstored"), FRT_STORE_NO, FRT_INDEX_YES, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS));
316
- frt_fis_add_field(fis, frt_fi_new(rb_intern("stored"), FRT_STORE_YES, FRT_INDEX_YES, FRT_TERM_VECTOR_YES));
317
- frt_fis_add_field(fis, frt_fi_new(rb_intern("stored_array"), FRT_STORE_COMPRESS, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_NO));
309
+ static FrtFieldInfos *prepare_fis(void) {
310
+ FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_NO);
311
+ frt_fis_add_field(fis, frt_fi_new(rb_intern("ignored"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_NO, FRT_TERM_VECTOR_NO));
312
+ frt_fis_add_field(fis, frt_fi_new(rb_intern("unstored"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS));
313
+ frt_fis_add_field(fis, frt_fi_new(rb_intern("stored"), FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_YES));
314
+ frt_fis_add_field(fis, frt_fi_new(rb_intern("stored_array"), FRT_STORE_YES, FRT_COMPRESSION_BROTLI, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_NO));
318
315
  return fis;
319
316
  }
320
317
 
321
318
  static void test_fields_rw_single(TestCase *tc, void *data)
322
319
  {
323
- FrtStore *store = frt_open_ram_store();
320
+ FrtStore *store = frt_open_ram_store(NULL);
324
321
  char *bin_data = prepare_bin_data(BIN_DATA_LEN);
325
322
  FrtDocument *doc = prepare_doc();
326
323
  FrtFieldInfos *fis = prepare_fis();
@@ -384,7 +381,7 @@ static void test_fields_rw_single(TestCase *tc, void *data)
384
381
  static void test_fields_rw_multi(TestCase *tc, void *data)
385
382
  {
386
383
  int i;
387
- FrtStore *store = frt_open_ram_store();
384
+ FrtStore *store = frt_open_ram_store(NULL);
388
385
  char *bin_data = prepare_bin_data(BIN_DATA_LEN);
389
386
  FrtDocument *doc;
390
387
  FrtFieldInfos *fis = prepare_fis();
@@ -392,6 +389,7 @@ static void test_fields_rw_multi(TestCase *tc, void *data)
392
389
  FrtFieldsReader *fr;
393
390
  FrtDocField *df;
394
391
  (void)data;
392
+ rb_encoding *enc = rb_enc_find("ASCII-8BIT");
395
393
 
396
394
  fw = frt_fw_open(store, "_as3", fis);
397
395
  for (i = 0; i < 100; i++) {
@@ -400,7 +398,7 @@ static void test_fields_rw_multi(TestCase *tc, void *data)
400
398
  sprintf(buf, "<<%d>>", i);
401
399
  bufc = frt_estrdup(buf);
402
400
  doc = frt_doc_new();
403
- frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern(bufc)), bufc));
401
+ frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern(bufc)), bufc, enc));
404
402
  frt_fw_add_doc(fw, doc);
405
403
  frt_fw_write_tv_index(fw);
406
404
  frt_doc_destroy(doc);
@@ -462,7 +460,7 @@ static void test_fields_rw_multi(TestCase *tc, void *data)
462
460
 
463
461
  static void test_lazy_field_loading(TestCase *tc, void *data)
464
462
  {
465
- FrtStore *store = frt_open_ram_store();
463
+ FrtStore *store = frt_open_ram_store(NULL);
466
464
  FrtDocument *doc;
467
465
  FrtFieldInfos *fis = prepare_fis();
468
466
  FrtFieldsWriter *fw;
@@ -472,15 +470,16 @@ static void test_lazy_field_loading(TestCase *tc, void *data)
472
470
  FrtLazyDocField *lazy_df;
473
471
  char *text, buf[1000];
474
472
  (void)data;
473
+ rb_encoding *enc = rb_enc_find("ASCII-8BIT");
475
474
 
476
475
  fw = frt_fw_open(store, "_as3", fis);
477
476
  doc = frt_doc_new();
478
477
  df = frt_df_new(rb_intern("stored"));
479
- frt_df_add_data(df, (char *)"this is a stored field");
480
- frt_df_add_data(df, (char *)"to be or not to be");
481
- frt_df_add_data(df, (char *)"a stitch in time, saves nine");
482
- frt_df_add_data(df, (char *)"the quick brown fox jumped over the lazy dog");
483
- frt_df_add_data(df, (char *)"that's it folks");
478
+ frt_df_add_data(df, (char *)"this is a stored field", enc);
479
+ frt_df_add_data(df, (char *)"to be or not to be", enc);
480
+ frt_df_add_data(df, (char *)"a stitch in time, saves nine", enc);
481
+ frt_df_add_data(df, (char *)"the quick brown fox jumped over the lazy dog", enc);
482
+ frt_df_add_data(df, (char *)"that's it folks", enc);
484
483
  frt_doc_add_field(doc, df);
485
484
  frt_fw_add_doc(fw, doc);
486
485
  frt_fw_write_tv_index(fw);
@@ -18,9 +18,8 @@ const FrtConfig lucene_config = {
18
18
  };
19
19
 
20
20
 
21
- static FrtFieldInfos *prep_fis()
22
- {
23
- return frt_fis_new(FRT_STORE_NO, FRT_INDEX_YES, FRT_TERM_VECTOR_NO);
21
+ static FrtFieldInfos *prep_fis(void) {
22
+ return frt_fis_new(FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_NO);
24
23
  }
25
24
 
26
25
  static void create_index(FrtStore *store)
@@ -30,37 +29,21 @@ static void create_index(FrtStore *store)
30
29
  frt_fis_deref(fis);
31
30
  }
32
31
 
33
- /*
34
- static FrtIndexWriter *create_iw(FrtStore *store)
35
- {
36
- create_index(store);
37
- return frt_iw_open(store, frt_whitespace_analyzer_new(false), &frt_default_config);
38
- }
39
-
40
- static FrtIndexWriter *create_iw_conf(FrtStore *store, int max_buffered_docs,
41
- int merge_factor)
42
- {
43
- FrtConfig config = frt_default_config;
44
- config.max_buffered_docs = max_buffered_docs;
45
- config.merge_factor = merge_factor;
46
- create_index(store);
47
- return frt_iw_open(store, frt_whitespace_analyzer_new(false), &config);
48
- }
49
- */
50
-
51
32
  static FrtIndexWriter *create_iw_lucene(FrtStore *store)
52
33
  {
53
34
  create_index(store);
54
- return frt_iw_open(store, frt_whitespace_analyzer_new(false), &lucene_config);
35
+ return frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), &lucene_config);
55
36
  }
56
37
 
57
38
  static void add_doc(FrtIndexWriter *iw, int id)
58
39
  {
59
40
  FrtDocument *doc = frt_doc_new();
41
+ rb_encoding *enc = rb_enc_find("ASCII-8BIT");
42
+
60
43
  frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern(content_f)),
61
- frt_estrdup("aaa")))->destroy_data = true;
44
+ frt_estrdup("aaa"), enc))->destroy_data = true;
62
45
  frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern(id_f)),
63
- frt_strfmt("%d", id)))->destroy_data = true;
46
+ frt_strfmt("%d", id), enc))->destroy_data = true;
64
47
  frt_iw_add_doc(iw, doc);
65
48
  frt_doc_destroy(doc);
66
49
  }
@@ -96,7 +79,7 @@ static void test_delete_leftover_files(TestCase *tc, void *data)
96
79
  frt_iw_close(iw);
97
80
 
98
81
  /* Delete one doc so we get a .del file: */
99
- ir = frt_ir_open(store);
82
+ ir = frt_ir_open(NULL, store);
100
83
  frt_ir_delete_doc(ir, 7);
101
84
  Aiequal(1, ir->max_doc(ir) - ir->num_docs(ir));
102
85
 
@@ -158,7 +141,7 @@ static void test_delete_leftover_files(TestCase *tc, void *data)
158
141
 
159
142
 
160
143
  /* Open & close a writer: should delete the above files and nothing more: */
161
- frt_iw_close(frt_iw_open(store, frt_whitespace_analyzer_new(false), &lucene_config));
144
+ frt_iw_close(frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), &lucene_config));
162
145
 
163
146
  store_after = frt_store_to_s(store);
164
147
 
@@ -175,7 +158,7 @@ static void test_delete_leftover_files(TestCase *tc, void *data)
175
158
 
176
159
  TestSuite *ts_file_deleter(TestSuite *suite)
177
160
  {
178
- FrtStore *store = frt_open_ram_store();
161
+ FrtStore *store = frt_open_ram_store(NULL);
179
162
  suite = ADD_SUITE(suite);
180
163
 
181
164
  tst_run_test(suite, test_delete_leftover_files, store);
@@ -2,6 +2,8 @@
2
2
  #include "frt_search.h"
3
3
  #include "test.h"
4
4
 
5
+ #undef close
6
+
5
7
  #define FILTER_DOCS_SIZE 10
6
8
  #define ARRAY_SIZE 20
7
9
 
@@ -11,13 +13,14 @@ struct FilterData {
11
13
  const char *flipflop;
12
14
  };
13
15
 
14
- static FrtSymbol num, date, flipflop;
16
+ static ID num, date, flipflop;
15
17
 
16
18
  void prepare_filter_index(FrtStore *store)
17
19
  {
18
20
  int i;
19
21
  FrtIndexWriter *iw;
20
- FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_INDEX_YES, FRT_TERM_VECTOR_NO);
22
+ FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_NO);
23
+ rb_encoding *enc = rb_enc_find("ASCII-8BIT");
21
24
 
22
25
  num = rb_intern("num");
23
26
  date = rb_intern("date");
@@ -39,13 +42,13 @@ void prepare_filter_index(FrtStore *store)
39
42
  frt_index_create(store, fis);
40
43
  frt_fis_deref(fis);
41
44
 
42
- iw = frt_iw_open(store, frt_whitespace_analyzer_new(false), NULL);
45
+ iw = frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), NULL);
43
46
  for (i = 0; i < FILTER_DOCS_SIZE; i++) {
44
47
  FrtDocument *doc = frt_doc_new();
45
48
  doc->boost = (float)(i+1);
46
- frt_doc_add_field(doc, frt_df_add_data(frt_df_new(num), (char *)data[i].num));
47
- frt_doc_add_field(doc, frt_df_add_data(frt_df_new(date), (char *)data[i].date));
48
- frt_doc_add_field(doc, frt_df_add_data(frt_df_new(flipflop), (char *)data[i].flipflop));
49
+ frt_doc_add_field(doc, frt_df_add_data(frt_df_new(num), (char *)data[i].num, enc));
50
+ frt_doc_add_field(doc, frt_df_add_data(frt_df_new(date), (char *)data[i].date, enc));
51
+ frt_doc_add_field(doc, frt_df_add_data(frt_df_new(flipflop), (char *)data[i].flipflop, enc));
49
52
  frt_iw_add_doc(iw, doc);
50
53
  frt_doc_destroy(doc);
51
54
  }
@@ -313,9 +316,9 @@ TestSuite *ts_filter(TestSuite *suite)
313
316
 
314
317
  suite = ADD_SUITE(suite);
315
318
 
316
- store = frt_open_ram_store();
319
+ store = frt_open_ram_store(NULL);
317
320
  prepare_filter_index(store);
318
- ir = frt_ir_open(store);
321
+ ir = frt_ir_open(NULL, store);
319
322
  searcher = frt_isea_new(ir);
320
323
 
321
324
  tst_run_test(suite, test_range_filter, (void *)searcher);
@@ -145,50 +145,6 @@ static void test_dbl_to_s(TestCase *tc, void *data)
145
145
  Asequal("NaN", frt_dbl_to_s(buf, NAN));
146
146
  }
147
147
 
148
-
149
- /**
150
- * Generate a stacktrace, make sure it does something
151
- */
152
- static void test_stacktrace(TestCase *tc, void *data)
153
- {
154
- FILE *old_stream = frt_x_exception_stream;
155
- (void)data; /* suppress warning */
156
- int tfd = fio_tmpfile();
157
- frt_x_exception_stream = fdopen(tfd, "w+");
158
- Atrue(frt_x_exception_stream != NULL);
159
- if (frt_x_exception_stream) {
160
- frt_print_stacktrace();
161
- long int f = ftell(frt_x_exception_stream);
162
- Assert(f, "Stream position should not be 0");
163
- fclose(frt_x_exception_stream);
164
- }
165
- frt_x_exception_stream = old_stream;
166
- }
167
-
168
- /**
169
- * Generate a normally fatal signal, which gets caught
170
- */
171
- /*
172
- static void test_sighandler(TestCase *tc, void *data)
173
- {
174
- bool old_abort = frt_x_abort_on_exception;
175
- FILE *old_stream = frt_x_exception_stream;
176
- (void)data;
177
- (void)tc;
178
-
179
- frt_x_exception_stream = false;
180
- frt_x_exception_stream = tmpfile();
181
-
182
- raise(SIGSEGV);
183
-
184
- Assert(ftell(frt_x_exception_stream), "Stream position should not be 0");
185
- fclose(frt_x_exception_stream);
186
-
187
- frt_x_exception_stream = old_stream;
188
- frt_x_abort_on_exception = old_abort;
189
- }
190
- */
191
-
192
148
  static void test_count_leading_zeros(TestCase *tc, void *data)
193
149
  {
194
150
  (void)data;
@@ -284,8 +240,6 @@ TestSuite *ts_global(TestSuite *suite)
284
240
  tst_run_test(suite, test_alloc, NULL);
285
241
  tst_run_test(suite, test_strfmt, NULL);
286
242
  tst_run_test(suite, test_dbl_to_s, NULL);
287
- tst_run_test(suite, test_stacktrace, NULL);
288
- // tst_run_test(suite, test_sighandler, NULL);
289
243
  tst_run_test(suite, test_count_leading_zeros, NULL);
290
244
  tst_run_test(suite, test_count_leading_ones, NULL);
291
245
  tst_run_test(suite, test_count_trailing_zeros, NULL);
@@ -199,8 +199,8 @@ static void test_hash_int(TestCase *tc, void *data)
199
199
  static void test_hash_ptr(TestCase *tc, void *data)
200
200
  {
201
201
  FrtHash *h = frt_h_new_ptr(&free);
202
- FrtSymbol word1 = rb_intern("one");
203
- FrtSymbol word2 = rb_intern("two");
202
+ ID word1 = rb_intern("one");
203
+ ID word2 = rb_intern("two");
204
204
  char *word_one = frt_estrdup("one");
205
205
  int i;
206
206
  char buf[100];
@@ -61,7 +61,7 @@ static void test_hs(TestCase *tc, void *data)
61
61
  static void test_hs_ptr(TestCase *tc, void *data)
62
62
  {
63
63
  FrtHashSet *hs = frt_hs_new_ptr(NULL);
64
- FrtSymbol word1 = rb_intern("one"), word2 = rb_intern("two");
64
+ ID word1 = rb_intern("one"), word2 = rb_intern("two");
65
65
  char *word_one = frt_estrdup("one");
66
66
  (void)data; /* suppress unused argument warning */
67
67