isomorfeus-ferret 0.12.7 → 0.13.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (166) hide show
  1. checksums.yaml +4 -4
  2. data/LICENSE +101 -19
  3. data/README.md +85 -13
  4. data/ext/isomorfeus_ferret_ext/bm_bitvector.c +22 -30
  5. data/ext/isomorfeus_ferret_ext/bm_hash.c +6 -12
  6. data/ext/isomorfeus_ferret_ext/bm_micro_string.c +3 -6
  7. data/ext/isomorfeus_ferret_ext/bm_store.c +11 -22
  8. data/ext/isomorfeus_ferret_ext/brotli_common_dictionary.c +1 -1
  9. data/ext/isomorfeus_ferret_ext/brotli_dec_decode.c +1 -1
  10. data/ext/isomorfeus_ferret_ext/bzlib.c +1572 -0
  11. data/ext/isomorfeus_ferret_ext/bzlib.h +282 -0
  12. data/ext/isomorfeus_ferret_ext/bzlib_blocksort.c +1094 -0
  13. data/ext/isomorfeus_ferret_ext/bzlib_compress.c +672 -0
  14. data/ext/isomorfeus_ferret_ext/bzlib_crctable.c +104 -0
  15. data/ext/isomorfeus_ferret_ext/bzlib_decompress.c +652 -0
  16. data/ext/isomorfeus_ferret_ext/bzlib_huffman.c +205 -0
  17. data/ext/isomorfeus_ferret_ext/bzlib_private.h +509 -0
  18. data/ext/isomorfeus_ferret_ext/bzlib_randtable.c +84 -0
  19. data/ext/isomorfeus_ferret_ext/fio_tmpfile.h +53 -53
  20. data/ext/isomorfeus_ferret_ext/frb_analysis.c +785 -1192
  21. data/ext/isomorfeus_ferret_ext/frb_index.c +497 -495
  22. data/ext/isomorfeus_ferret_ext/frb_qparser.c +48 -60
  23. data/ext/isomorfeus_ferret_ext/frb_search.c +1520 -1002
  24. data/ext/isomorfeus_ferret_ext/frb_store.c +96 -96
  25. data/ext/isomorfeus_ferret_ext/frb_threading.h +0 -1
  26. data/ext/isomorfeus_ferret_ext/frb_utils.c +147 -196
  27. data/ext/isomorfeus_ferret_ext/frt_analysis.c +695 -1090
  28. data/ext/isomorfeus_ferret_ext/frt_analysis.h +174 -170
  29. data/ext/isomorfeus_ferret_ext/frt_array.c +2 -4
  30. data/ext/isomorfeus_ferret_ext/frt_bitvector.c +9 -16
  31. data/ext/isomorfeus_ferret_ext/frt_bitvector.h +32 -81
  32. data/ext/isomorfeus_ferret_ext/frt_document.c +15 -20
  33. data/ext/isomorfeus_ferret_ext/frt_document.h +10 -10
  34. data/ext/isomorfeus_ferret_ext/frt_except.c +5 -12
  35. data/ext/isomorfeus_ferret_ext/frt_field_index.c +3 -3
  36. data/ext/isomorfeus_ferret_ext/frt_field_index.h +6 -7
  37. data/ext/isomorfeus_ferret_ext/frt_filter.c +35 -46
  38. data/ext/isomorfeus_ferret_ext/frt_fs_store.c +1 -0
  39. data/ext/isomorfeus_ferret_ext/frt_global.c +91 -200
  40. data/ext/isomorfeus_ferret_ext/frt_global.h +7 -18
  41. data/ext/isomorfeus_ferret_ext/frt_hash.c +1 -2
  42. data/ext/isomorfeus_ferret_ext/frt_ind.c +32 -35
  43. data/ext/isomorfeus_ferret_ext/frt_ind.h +9 -9
  44. data/ext/isomorfeus_ferret_ext/frt_index.c +603 -410
  45. data/ext/isomorfeus_ferret_ext/frt_index.h +272 -291
  46. data/ext/isomorfeus_ferret_ext/frt_lang.c +0 -2
  47. data/ext/isomorfeus_ferret_ext/frt_mempool.c +1 -2
  48. data/ext/isomorfeus_ferret_ext/frt_multimapper.c +4 -7
  49. data/ext/isomorfeus_ferret_ext/frt_q_boolean.c +68 -91
  50. data/ext/isomorfeus_ferret_ext/frt_q_const_score.c +35 -38
  51. data/ext/isomorfeus_ferret_ext/frt_q_filtered_query.c +53 -72
  52. data/ext/isomorfeus_ferret_ext/frt_q_fuzzy.c +25 -32
  53. data/ext/isomorfeus_ferret_ext/frt_q_match_all.c +21 -23
  54. data/ext/isomorfeus_ferret_ext/frt_q_multi_term.c +66 -103
  55. data/ext/isomorfeus_ferret_ext/frt_q_parser.c +207 -195
  56. data/ext/isomorfeus_ferret_ext/frt_q_phrase.c +20 -16
  57. data/ext/isomorfeus_ferret_ext/frt_q_prefix.c +17 -14
  58. data/ext/isomorfeus_ferret_ext/frt_q_range.c +102 -131
  59. data/ext/isomorfeus_ferret_ext/frt_q_span.c +179 -178
  60. data/ext/isomorfeus_ferret_ext/frt_q_term.c +47 -60
  61. data/ext/isomorfeus_ferret_ext/frt_q_wildcard.c +18 -16
  62. data/ext/isomorfeus_ferret_ext/frt_ram_store.c +45 -84
  63. data/ext/isomorfeus_ferret_ext/frt_search.c +105 -146
  64. data/ext/isomorfeus_ferret_ext/frt_search.h +331 -320
  65. data/ext/isomorfeus_ferret_ext/frt_similarity.c +5 -13
  66. data/ext/isomorfeus_ferret_ext/frt_similarity.h +7 -12
  67. data/ext/isomorfeus_ferret_ext/frt_sort.c +105 -149
  68. data/ext/isomorfeus_ferret_ext/frt_store.c +13 -7
  69. data/ext/isomorfeus_ferret_ext/frt_store.h +10 -2
  70. data/ext/isomorfeus_ferret_ext/frt_threading.h +0 -1
  71. data/ext/isomorfeus_ferret_ext/isomorfeus_ferret.c +22 -112
  72. data/ext/isomorfeus_ferret_ext/isomorfeus_ferret.h +2 -32
  73. data/ext/isomorfeus_ferret_ext/lz4.c +2495 -0
  74. data/ext/isomorfeus_ferret_ext/lz4.h +774 -0
  75. data/ext/isomorfeus_ferret_ext/lz4frame.c +1899 -0
  76. data/ext/isomorfeus_ferret_ext/lz4frame.h +623 -0
  77. data/ext/isomorfeus_ferret_ext/lz4hc.c +1615 -0
  78. data/ext/isomorfeus_ferret_ext/lz4hc.h +413 -0
  79. data/ext/isomorfeus_ferret_ext/lz4xxhash.c +1030 -0
  80. data/ext/isomorfeus_ferret_ext/lz4xxhash.h +328 -0
  81. data/ext/isomorfeus_ferret_ext/stem_modules.h +0 -86
  82. data/ext/isomorfeus_ferret_ext/test.c +0 -17
  83. data/ext/isomorfeus_ferret_ext/test_1710.c +11 -12
  84. data/ext/isomorfeus_ferret_ext/test_analysis.c +590 -583
  85. data/ext/isomorfeus_ferret_ext/test_compound_io.c +1 -1
  86. data/ext/isomorfeus_ferret_ext/test_document.c +19 -15
  87. data/ext/isomorfeus_ferret_ext/test_except.c +1 -2
  88. data/ext/isomorfeus_ferret_ext/test_fields.c +59 -60
  89. data/ext/isomorfeus_ferret_ext/test_file_deleter.c +10 -27
  90. data/ext/isomorfeus_ferret_ext/test_filter.c +11 -8
  91. data/ext/isomorfeus_ferret_ext/test_global.c +0 -46
  92. data/ext/isomorfeus_ferret_ext/test_hash.c +2 -2
  93. data/ext/isomorfeus_ferret_ext/test_hashset.c +1 -1
  94. data/ext/isomorfeus_ferret_ext/test_highlighter.c +15 -11
  95. data/ext/isomorfeus_ferret_ext/test_index.c +372 -365
  96. data/ext/isomorfeus_ferret_ext/test_q_const_score.c +5 -3
  97. data/ext/isomorfeus_ferret_ext/test_q_filtered.c +5 -3
  98. data/ext/isomorfeus_ferret_ext/test_q_fuzzy.c +13 -10
  99. data/ext/isomorfeus_ferret_ext/test_q_parser.c +45 -7
  100. data/ext/isomorfeus_ferret_ext/test_q_span.c +15 -12
  101. data/ext/isomorfeus_ferret_ext/test_ram_store.c +3 -3
  102. data/ext/isomorfeus_ferret_ext/test_search.c +60 -64
  103. data/ext/isomorfeus_ferret_ext/test_segments.c +5 -4
  104. data/ext/isomorfeus_ferret_ext/test_sort.c +17 -14
  105. data/ext/isomorfeus_ferret_ext/test_store.c +2 -0
  106. data/ext/isomorfeus_ferret_ext/test_term.c +3 -1
  107. data/ext/isomorfeus_ferret_ext/test_term_vectors.c +9 -10
  108. data/ext/isomorfeus_ferret_ext/test_test.c +1 -2
  109. data/ext/isomorfeus_ferret_ext/test_threading.c +9 -10
  110. data/ext/isomorfeus_ferret_ext/testhelper.c +1 -2
  111. data/lib/isomorfeus/ferret/version.rb +1 -1
  112. metadata +27 -57
  113. data/ext/isomorfeus_ferret_ext/email.rl +0 -21
  114. data/ext/isomorfeus_ferret_ext/frt_scanner.c +0 -900
  115. data/ext/isomorfeus_ferret_ext/frt_scanner.h +0 -28
  116. data/ext/isomorfeus_ferret_ext/frt_scanner_mb.c +0 -6706
  117. data/ext/isomorfeus_ferret_ext/frt_scanner_utf8.c +0 -4420
  118. data/ext/isomorfeus_ferret_ext/scanner.h +0 -28
  119. data/ext/isomorfeus_ferret_ext/scanner.in +0 -43
  120. data/ext/isomorfeus_ferret_ext/scanner.rl +0 -84
  121. data/ext/isomorfeus_ferret_ext/scanner_mb.rl +0 -200
  122. data/ext/isomorfeus_ferret_ext/scanner_utf8.rl +0 -85
  123. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_basque.c +0 -1167
  124. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_basque.h +0 -6
  125. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_catalan.c +0 -1433
  126. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_catalan.h +0 -6
  127. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_danish.c +0 -301
  128. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_danish.h +0 -6
  129. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_dutch.c +0 -590
  130. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_dutch.h +0 -6
  131. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_english.c +0 -1049
  132. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_english.h +0 -6
  133. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_finnish.c +0 -705
  134. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_finnish.h +0 -6
  135. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_french.c +0 -1239
  136. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_french.h +0 -6
  137. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_german.c +0 -477
  138. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_german.h +0 -6
  139. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_hungarian.c +0 -1217
  140. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_hungarian.h +0 -7
  141. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_indonesian.c +0 -394
  142. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_indonesian.h +0 -6
  143. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_irish.c +0 -457
  144. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_irish.h +0 -6
  145. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_italian.c +0 -1009
  146. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_italian.h +0 -6
  147. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_norwegian.c +0 -259
  148. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_norwegian.h +0 -6
  149. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_porter.c +0 -704
  150. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_porter.h +0 -6
  151. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_portuguese.c +0 -948
  152. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_portuguese.h +0 -6
  153. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_spanish.c +0 -1028
  154. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_spanish.h +0 -6
  155. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_swedish.c +0 -275
  156. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_swedish.h +0 -6
  157. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_2_hungarian.c +0 -849
  158. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_2_hungarian.h +0 -6
  159. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_2_romanian.c +0 -952
  160. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_2_romanian.h +0 -6
  161. data/ext/isomorfeus_ferret_ext/stem_KOI8_R_russian.c +0 -669
  162. data/ext/isomorfeus_ferret_ext/stem_KOI8_R_russian.h +0 -6
  163. data/ext/isomorfeus_ferret_ext/stem_modules.txt +0 -63
  164. data/ext/isomorfeus_ferret_ext/uchar-ucs4.rl +0 -1854
  165. data/ext/isomorfeus_ferret_ext/uchar-utf8.rl +0 -1999
  166. data/ext/isomorfeus_ferret_ext/url.rl +0 -27
@@ -2,18 +2,17 @@
2
2
  #include "testhelper.h"
3
3
  #include "test.h"
4
4
 
5
- static FrtSymbol body, title, text, author, year, changing_field, compressed_field, tag;
5
+ #undef close
6
+ #undef read
6
7
 
7
- static FrtFieldInfos *prep_all_fis()
8
- {
9
- FrtFieldInfos *fis = frt_fis_new(FRT_STORE_NO, FRT_INDEX_YES, FRT_TERM_VECTOR_NO);
10
- frt_fis_add_field(fis, frt_fi_new(rb_intern("tv"), FRT_STORE_NO, FRT_INDEX_YES, FRT_TERM_VECTOR_YES));
11
- frt_fis_add_field(fis, frt_fi_new(rb_intern("tv un-t"), FRT_STORE_NO, FRT_INDEX_UNTOKENIZED,
12
- FRT_TERM_VECTOR_YES));
13
- frt_fis_add_field(fis, frt_fi_new(rb_intern("tv+offsets"), FRT_STORE_NO, FRT_INDEX_YES,
14
- FRT_TERM_VECTOR_WITH_OFFSETS));
15
- frt_fis_add_field(fis, frt_fi_new(rb_intern("tv+offsets un-t"), FRT_STORE_NO, FRT_INDEX_UNTOKENIZED,
16
- FRT_TERM_VECTOR_WITH_OFFSETS));
8
+ static ID body, title, text, author, year, changing_field, compressed_field_brotli, compressed_field_bzip2, compressed_field_lz4, tag;
9
+
10
+ static FrtFieldInfos *prep_all_fis(void) {
11
+ FrtFieldInfos *fis = frt_fis_new(FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_NO);
12
+ frt_fis_add_field(fis, frt_fi_new(rb_intern("tv"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_YES));
13
+ frt_fis_add_field(fis, frt_fi_new(rb_intern("tv un-t"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_YES));
14
+ frt_fis_add_field(fis, frt_fi_new(rb_intern("tv+offsets"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_WITH_OFFSETS));
15
+ frt_fis_add_field(fis, frt_fi_new(rb_intern("tv+offsets un-t"), FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_OFFSETS));
17
16
  return fis;
18
17
 
19
18
  }
@@ -27,290 +26,289 @@ static void destroy_docs(FrtDocument **docs, int len)
27
26
  free(docs);
28
27
  }
29
28
 
30
- static FrtFieldInfos *prep_book_fis()
31
- {
32
- FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_INDEX_YES,
33
- FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
34
- frt_fis_add_field(fis, frt_fi_new(rb_intern("year"), FRT_STORE_YES, FRT_INDEX_NO, FRT_TERM_VECTOR_NO));
29
+ static FrtFieldInfos *prep_book_fis(void) {
30
+ FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
31
+ frt_fis_add_field(fis, frt_fi_new(rb_intern("year"), FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_NO, FRT_TERM_VECTOR_NO));
35
32
  return fis;
36
33
 
37
34
  }
38
35
 
39
- FrtDocument *prep_book()
40
- {
36
+ FrtDocument *prep_book(void) {
41
37
  FrtDocument *doc = frt_doc_new();
38
+ rb_encoding *enc = rb_enc_find("ASCII-8BIT");
39
+
42
40
  frt_doc_add_field(doc, frt_df_add_data(frt_df_new(author),
43
- frt_estrdup("P.H. Newby")))->destroy_data = true;
41
+ frt_estrdup("P.H. Newby"), enc))->destroy_data = true;
44
42
  frt_doc_add_field(doc, frt_df_add_data(frt_df_new(title),
45
- frt_estrdup("Something To Answer For")))->destroy_data = true;
43
+ frt_estrdup("Something To Answer For"), enc))->destroy_data = true;
46
44
  frt_doc_add_field(doc, frt_df_add_data(frt_df_new(year),
47
- frt_estrdup("1969")))->destroy_data = true;
45
+ frt_estrdup("1969"), enc))->destroy_data = true;
48
46
  return doc;
49
47
  }
50
48
 
51
49
  #define BOOK_LIST_LENGTH 37
52
- FrtDocument **prep_book_list()
53
- {
50
+ FrtDocument **prep_book_list(void) {
51
+ rb_encoding *enc = rb_enc_find("ASCII-8BIT");
54
52
  FrtDocument **docs = FRT_ALLOC_N(FrtDocument *, BOOK_LIST_LENGTH);
55
53
  docs[0] = frt_doc_new();
56
54
  frt_doc_add_field(docs[0], frt_df_add_data(frt_df_new(author),
57
- frt_estrdup("P.H. Newby")))->destroy_data = true;
55
+ frt_estrdup("P.H. Newby"), enc))->destroy_data = true;
58
56
  frt_doc_add_field(docs[0], frt_df_add_data(frt_df_new(title),
59
- frt_estrdup("Something To Answer For")))->destroy_data = true;
57
+ frt_estrdup("Something To Answer For"), enc))->destroy_data = true;
60
58
  frt_doc_add_field(docs[0], frt_df_add_data(frt_df_new(year),
61
- frt_estrdup("1969")))->destroy_data = true;
59
+ frt_estrdup("1969"), enc))->destroy_data = true;
62
60
  docs[1] = frt_doc_new();
63
61
  frt_doc_add_field(docs[1], frt_df_add_data(frt_df_new(author),
64
- frt_estrdup("Bernice Rubens")))->destroy_data = true;
62
+ frt_estrdup("Bernice Rubens"), enc))->destroy_data = true;
65
63
  frt_doc_add_field(docs[1], frt_df_add_data(frt_df_new(title),
66
- frt_estrdup("The Elected Member")))->destroy_data = true;
64
+ frt_estrdup("The Elected Member"), enc))->destroy_data = true;
67
65
  frt_doc_add_field(docs[1], frt_df_add_data(frt_df_new(year),
68
- frt_estrdup("1970")))->destroy_data = true;
66
+ frt_estrdup("1970"), enc))->destroy_data = true;
69
67
  docs[2] = frt_doc_new();
70
68
  frt_doc_add_field(docs[2], frt_df_add_data(frt_df_new(author),
71
- frt_estrdup("V. S. Naipaul")))->destroy_data = true;
69
+ frt_estrdup("V. S. Naipaul"), enc))->destroy_data = true;
72
70
  frt_doc_add_field(docs[2], frt_df_add_data(frt_df_new(title),
73
- frt_estrdup("In a Free State")))->destroy_data = true;
71
+ frt_estrdup("In a Free State"), enc))->destroy_data = true;
74
72
  frt_doc_add_field(docs[2], frt_df_add_data(frt_df_new(year),
75
- frt_estrdup("1971")))->destroy_data = true;
73
+ frt_estrdup("1971"), enc))->destroy_data = true;
76
74
  docs[3] = frt_doc_new();
77
75
  frt_doc_add_field(docs[3], frt_df_add_data(frt_df_new(author),
78
- frt_estrdup("John Berger")))->destroy_data = true;
76
+ frt_estrdup("John Berger"), enc))->destroy_data = true;
79
77
  frt_doc_add_field(docs[3], frt_df_add_data(frt_df_new(title),
80
- frt_estrdup("G")))->destroy_data = true;
78
+ frt_estrdup("G"), enc))->destroy_data = true;
81
79
  frt_doc_add_field(docs[3], frt_df_add_data(frt_df_new(year),
82
- frt_estrdup("1972")))->destroy_data = true;
80
+ frt_estrdup("1972"), enc))->destroy_data = true;
83
81
  docs[4] = frt_doc_new();
84
82
  frt_doc_add_field(docs[4], frt_df_add_data(frt_df_new(author),
85
- frt_estrdup("J. G. Farrell")))->destroy_data = true;
83
+ frt_estrdup("J. G. Farrell"), enc))->destroy_data = true;
86
84
  frt_doc_add_field(docs[4], frt_df_add_data(frt_df_new(title),
87
- frt_estrdup("The Siege of Krishnapur")))->destroy_data = true;
85
+ frt_estrdup("The Siege of Krishnapur"), enc))->destroy_data = true;
88
86
  frt_doc_add_field(docs[4], frt_df_add_data(frt_df_new(year),
89
- frt_estrdup("1973")))->destroy_data = true;
87
+ frt_estrdup("1973"), enc))->destroy_data = true;
90
88
  docs[5] = frt_doc_new();
91
89
  frt_doc_add_field(docs[5], frt_df_add_data(frt_df_new(author),
92
- frt_estrdup("Stanley Middleton")))->destroy_data = true;
90
+ frt_estrdup("Stanley Middleton"), enc))->destroy_data = true;
93
91
  frt_doc_add_field(docs[5], frt_df_add_data(frt_df_new(title),
94
- frt_estrdup("Holiday")))->destroy_data = true;
92
+ frt_estrdup("Holiday"), enc))->destroy_data = true;
95
93
  frt_doc_add_field(docs[5], frt_df_add_data(frt_df_new(year),
96
- frt_estrdup("1974")))->destroy_data = true;
94
+ frt_estrdup("1974"), enc))->destroy_data = true;
97
95
  docs[6] = frt_doc_new();
98
96
  frt_doc_add_field(docs[6], frt_df_add_data(frt_df_new(author),
99
- frt_estrdup("Nadine Gordimer")))->destroy_data = true;
97
+ frt_estrdup("Nadine Gordimer"), enc))->destroy_data = true;
100
98
  frt_doc_add_field(docs[6], frt_df_add_data(frt_df_new(title),
101
- frt_estrdup("The Conservationist")))->destroy_data = true;
99
+ frt_estrdup("The Conservationist"), enc))->destroy_data = true;
102
100
  frt_doc_add_field(docs[6], frt_df_add_data(frt_df_new(year),
103
- frt_estrdup("1974")))->destroy_data = true;
101
+ frt_estrdup("1974"), enc))->destroy_data = true;
104
102
  docs[7] = frt_doc_new();
105
103
  frt_doc_add_field(docs[7], frt_df_add_data(frt_df_new(author),
106
- frt_estrdup("Ruth Prawer Jhabvala")))->destroy_data = true;
104
+ frt_estrdup("Ruth Prawer Jhabvala"), enc))->destroy_data = true;
107
105
  frt_doc_add_field(docs[7], frt_df_add_data(frt_df_new(title),
108
- frt_estrdup("Heat and Dust")))->destroy_data = true;
106
+ frt_estrdup("Heat and Dust"), enc))->destroy_data = true;
109
107
  frt_doc_add_field(docs[7], frt_df_add_data(frt_df_new(year),
110
- frt_estrdup("1975")))->destroy_data = true;
108
+ frt_estrdup("1975"), enc))->destroy_data = true;
111
109
  docs[8] = frt_doc_new();
112
110
  frt_doc_add_field(docs[8], frt_df_add_data(frt_df_new(author),
113
- frt_estrdup("David Storey")))->destroy_data = true;
111
+ frt_estrdup("David Storey"), enc))->destroy_data = true;
114
112
  frt_doc_add_field(docs[8], frt_df_add_data(frt_df_new(title),
115
- frt_estrdup("Saville")))->destroy_data = true;
113
+ frt_estrdup("Saville"), enc))->destroy_data = true;
116
114
  frt_doc_add_field(docs[8], frt_df_add_data(frt_df_new(year),
117
- frt_estrdup("1976")))->destroy_data = true;
115
+ frt_estrdup("1976"), enc))->destroy_data = true;
118
116
  docs[9] = frt_doc_new();
119
117
  frt_doc_add_field(docs[9], frt_df_add_data(frt_df_new(author),
120
- frt_estrdup("Paul Scott")))->destroy_data = true;
118
+ frt_estrdup("Paul Scott"), enc))->destroy_data = true;
121
119
  frt_doc_add_field(docs[9], frt_df_add_data(frt_df_new(title),
122
- frt_estrdup("Staying On")))->destroy_data = true;
120
+ frt_estrdup("Staying On"), enc))->destroy_data = true;
123
121
  frt_doc_add_field(docs[9], frt_df_add_data(frt_df_new(year),
124
- frt_estrdup("1977")))->destroy_data = true;
122
+ frt_estrdup("1977"), enc))->destroy_data = true;
125
123
  docs[10] = frt_doc_new();
126
124
  frt_doc_add_field(docs[10], frt_df_add_data(frt_df_new(author),
127
- frt_estrdup("Iris Murdoch")))->destroy_data = true;
125
+ frt_estrdup("Iris Murdoch"), enc))->destroy_data = true;
128
126
  frt_doc_add_field(docs[10], frt_df_add_data(frt_df_new(title),
129
- frt_estrdup("The Sea")))->destroy_data = true;
127
+ frt_estrdup("The Sea"), enc))->destroy_data = true;
130
128
  frt_doc_add_field(docs[10], frt_df_add_data(frt_df_new(year),
131
- frt_estrdup("1978")))->destroy_data = true;
129
+ frt_estrdup("1978"), enc))->destroy_data = true;
132
130
  docs[11] = frt_doc_new();
133
131
  frt_doc_add_field(docs[11], frt_df_add_data(frt_df_new(author),
134
- frt_estrdup("Penelope Fitzgerald")))->destroy_data = true;
132
+ frt_estrdup("Penelope Fitzgerald"), enc))->destroy_data = true;
135
133
  frt_doc_add_field(docs[11], frt_df_add_data(frt_df_new(title),
136
- frt_estrdup("Offshore")))->destroy_data = true;
134
+ frt_estrdup("Offshore"), enc))->destroy_data = true;
137
135
  frt_doc_add_field(docs[11], frt_df_add_data(frt_df_new(year),
138
- frt_estrdup("1979")))->destroy_data = true;
136
+ frt_estrdup("1979"), enc))->destroy_data = true;
139
137
  docs[12] = frt_doc_new();
140
138
  frt_doc_add_field(docs[12], frt_df_add_data(frt_df_new(author),
141
- frt_estrdup("William Golding")))->destroy_data = true;
139
+ frt_estrdup("William Golding"), enc))->destroy_data = true;
142
140
  frt_doc_add_field(docs[12], frt_df_add_data(frt_df_new(title),
143
- frt_estrdup("Rites of Passage")))->destroy_data = true;
141
+ frt_estrdup("Rites of Passage"), enc))->destroy_data = true;
144
142
  frt_doc_add_field(docs[12], frt_df_add_data(frt_df_new(year),
145
- frt_estrdup("1980")))->destroy_data = true;
143
+ frt_estrdup("1980"), enc))->destroy_data = true;
146
144
  docs[13] = frt_doc_new();
147
145
  frt_doc_add_field(docs[13], frt_df_add_data(frt_df_new(author),
148
- frt_estrdup("Salman Rushdie")))->destroy_data = true;
146
+ frt_estrdup("Salman Rushdie"), enc))->destroy_data = true;
149
147
  frt_doc_add_field(docs[13], frt_df_add_data(frt_df_new(title),
150
- frt_estrdup("Midnight's Children")))->destroy_data = true;
148
+ frt_estrdup("Midnight's Children"), enc))->destroy_data = true;
151
149
  frt_doc_add_field(docs[13], frt_df_add_data(frt_df_new(year),
152
- frt_estrdup("1981")))->destroy_data = true;
150
+ frt_estrdup("1981"), enc))->destroy_data = true;
153
151
  docs[14] = frt_doc_new();
154
152
  frt_doc_add_field(docs[14], frt_df_add_data(frt_df_new(author),
155
- frt_estrdup("Thomas Keneally")))->destroy_data = true;
153
+ frt_estrdup("Thomas Keneally"), enc))->destroy_data = true;
156
154
  frt_doc_add_field(docs[14], frt_df_add_data(frt_df_new(title),
157
- frt_estrdup("Schindler's Ark")))->destroy_data = true;
155
+ frt_estrdup("Schindler's Ark"), enc))->destroy_data = true;
158
156
  frt_doc_add_field(docs[14], frt_df_add_data(frt_df_new(year),
159
- frt_estrdup("1982")))->destroy_data = true;
157
+ frt_estrdup("1982"), enc))->destroy_data = true;
160
158
  docs[15] = frt_doc_new();
161
159
  frt_doc_add_field(docs[15], frt_df_add_data(frt_df_new(author),
162
- frt_estrdup("J. M. Coetzee")))->destroy_data = true;
160
+ frt_estrdup("J. M. Coetzee"), enc))->destroy_data = true;
163
161
  frt_doc_add_field(docs[15], frt_df_add_data(frt_df_new(title),
164
- frt_estrdup("Life and Times of Michael K")))->destroy_data = true;
162
+ frt_estrdup("Life and Times of Michael K"), enc))->destroy_data = true;
165
163
  frt_doc_add_field(docs[15], frt_df_add_data(frt_df_new(year),
166
- frt_estrdup("1983")))->destroy_data = true;
164
+ frt_estrdup("1983"), enc))->destroy_data = true;
167
165
  docs[16] = frt_doc_new();
168
166
  frt_doc_add_field(docs[16], frt_df_add_data(frt_df_new(author),
169
- frt_estrdup("Anita Brookner")))->destroy_data = true;
167
+ frt_estrdup("Anita Brookner"), enc))->destroy_data = true;
170
168
  frt_doc_add_field(docs[16], frt_df_add_data(frt_df_new(title),
171
- frt_estrdup("Hotel du Lac")))->destroy_data = true;
169
+ frt_estrdup("Hotel du Lac"), enc))->destroy_data = true;
172
170
  frt_doc_add_field(docs[16], frt_df_add_data(frt_df_new(year),
173
- frt_estrdup("1984")))->destroy_data = true;
171
+ frt_estrdup("1984"), enc))->destroy_data = true;
174
172
  docs[17] = frt_doc_new();
175
173
  frt_doc_add_field(docs[17], frt_df_add_data(frt_df_new(author),
176
- frt_estrdup("Keri Hulme")))->destroy_data = true;
174
+ frt_estrdup("Keri Hulme"), enc))->destroy_data = true;
177
175
  frt_doc_add_field(docs[17], frt_df_add_data(frt_df_new(title),
178
- frt_estrdup("The Bone People")))->destroy_data = true;
176
+ frt_estrdup("The Bone People"), enc))->destroy_data = true;
179
177
  frt_doc_add_field(docs[17], frt_df_add_data(frt_df_new(year),
180
- frt_estrdup("1985")))->destroy_data = true;
178
+ frt_estrdup("1985"), enc))->destroy_data = true;
181
179
  docs[18] = frt_doc_new();
182
180
  frt_doc_add_field(docs[18], frt_df_add_data(frt_df_new(author),
183
- frt_estrdup("Kingsley Amis")))->destroy_data = true;
181
+ frt_estrdup("Kingsley Amis"), enc))->destroy_data = true;
184
182
  frt_doc_add_field(docs[18], frt_df_add_data(frt_df_new(title),
185
- frt_estrdup("The Old Devils")))->destroy_data = true;
183
+ frt_estrdup("The Old Devils"), enc))->destroy_data = true;
186
184
  frt_doc_add_field(docs[18], frt_df_add_data(frt_df_new(year),
187
- frt_estrdup("1986")))->destroy_data = true;
185
+ frt_estrdup("1986"), enc))->destroy_data = true;
188
186
  docs[19] = frt_doc_new();
189
187
  frt_doc_add_field(docs[19], frt_df_add_data(frt_df_new(author),
190
- frt_estrdup("Penelope Lively")))->destroy_data = true;
188
+ frt_estrdup("Penelope Lively"), enc))->destroy_data = true;
191
189
  frt_doc_add_field(docs[19], frt_df_add_data(frt_df_new(title),
192
- frt_estrdup("Moon Tiger")))->destroy_data = true;
190
+ frt_estrdup("Moon Tiger"), enc))->destroy_data = true;
193
191
  frt_doc_add_field(docs[19], frt_df_add_data(frt_df_new(year),
194
- frt_estrdup("1987")))->destroy_data = true;
192
+ frt_estrdup("1987"), enc))->destroy_data = true;
195
193
  docs[20] = frt_doc_new();
196
194
  frt_doc_add_field(docs[20], frt_df_add_data(frt_df_new(author),
197
- frt_estrdup("Peter Carey")))->destroy_data = true;
195
+ frt_estrdup("Peter Carey"), enc))->destroy_data = true;
198
196
  frt_doc_add_field(docs[20], frt_df_add_data(frt_df_new(title),
199
- frt_estrdup("Oscar and Lucinda")))->destroy_data = true;
197
+ frt_estrdup("Oscar and Lucinda"), enc))->destroy_data = true;
200
198
  frt_doc_add_field(docs[20], frt_df_add_data(frt_df_new(year),
201
- frt_estrdup("1988")))->destroy_data = true;
199
+ frt_estrdup("1988"), enc))->destroy_data = true;
202
200
  docs[21] = frt_doc_new();
203
201
  frt_doc_add_field(docs[21], frt_df_add_data(frt_df_new(author),
204
- frt_estrdup("Kazuo Ishiguro")))->destroy_data = true;
202
+ frt_estrdup("Kazuo Ishiguro"), enc))->destroy_data = true;
205
203
  frt_doc_add_field(docs[21], frt_df_add_data(frt_df_new(title),
206
- frt_estrdup("The Remains of the Day")))->destroy_data = true;
204
+ frt_estrdup("The Remains of the Day"), enc))->destroy_data = true;
207
205
  frt_doc_add_field(docs[21], frt_df_add_data(frt_df_new(year),
208
- frt_estrdup("1989")))->destroy_data = true;
206
+ frt_estrdup("1989"), enc))->destroy_data = true;
209
207
  docs[22] = frt_doc_new();
210
208
  frt_doc_add_field(docs[22], frt_df_add_data(frt_df_new(author),
211
- frt_estrdup("A. S. Byatt")))->destroy_data = true;
209
+ frt_estrdup("A. S. Byatt"), enc))->destroy_data = true;
212
210
  frt_doc_add_field(docs[22], frt_df_add_data(frt_df_new(title),
213
- frt_estrdup("Possession")))->destroy_data = true;
211
+ frt_estrdup("Possession"), enc))->destroy_data = true;
214
212
  frt_doc_add_field(docs[22], frt_df_add_data(frt_df_new(year),
215
- frt_estrdup("1990")))->destroy_data = true;
213
+ frt_estrdup("1990"), enc))->destroy_data = true;
216
214
  docs[23] = frt_doc_new();
217
215
  frt_doc_add_field(docs[23], frt_df_add_data(frt_df_new(author),
218
- frt_estrdup("Ben Okri")))->destroy_data = true;
216
+ frt_estrdup("Ben Okri"), enc))->destroy_data = true;
219
217
  frt_doc_add_field(docs[23], frt_df_add_data(frt_df_new(title),
220
- frt_estrdup("The Famished Road")))->destroy_data = true;
218
+ frt_estrdup("The Famished Road"), enc))->destroy_data = true;
221
219
  frt_doc_add_field(docs[23], frt_df_add_data(frt_df_new(year),
222
- frt_estrdup("1991")))->destroy_data = true;
220
+ frt_estrdup("1991"), enc))->destroy_data = true;
223
221
  docs[24] = frt_doc_new();
224
222
  frt_doc_add_field(docs[24], frt_df_add_data(frt_df_new(author),
225
- frt_estrdup("Michael Ondaatje")))->destroy_data = true;
223
+ frt_estrdup("Michael Ondaatje"), enc))->destroy_data = true;
226
224
  frt_doc_add_field(docs[24], frt_df_add_data(frt_df_new(title),
227
- frt_estrdup("The English Patient")))->destroy_data = true;
225
+ frt_estrdup("The English Patient"), enc))->destroy_data = true;
228
226
  frt_doc_add_field(docs[24], frt_df_add_data(frt_df_new(year),
229
- frt_estrdup("1992")))->destroy_data = true;
227
+ frt_estrdup("1992"), enc))->destroy_data = true;
230
228
  docs[25] = frt_doc_new();
231
229
  frt_doc_add_field(docs[25], frt_df_add_data(frt_df_new(author),
232
- frt_estrdup("Barry Unsworth")))->destroy_data = true;
230
+ frt_estrdup("Barry Unsworth"), enc))->destroy_data = true;
233
231
  frt_doc_add_field(docs[25], frt_df_add_data(frt_df_new(title),
234
- frt_estrdup("Sacred Hunger")))->destroy_data = true;
232
+ frt_estrdup("Sacred Hunger"), enc))->destroy_data = true;
235
233
  frt_doc_add_field(docs[25], frt_df_add_data(frt_df_new(year),
236
- frt_estrdup("1992")))->destroy_data = true;
234
+ frt_estrdup("1992"), enc))->destroy_data = true;
237
235
  docs[26] = frt_doc_new();
238
236
  frt_doc_add_field(docs[26], frt_df_add_data(frt_df_new(author),
239
- frt_estrdup("Roddy Doyle")))->destroy_data = true;
237
+ frt_estrdup("Roddy Doyle"), enc))->destroy_data = true;
240
238
  frt_doc_add_field(docs[26], frt_df_add_data(frt_df_new(title),
241
- frt_estrdup("Paddy Clarke Ha Ha Ha")))->destroy_data = true;
239
+ frt_estrdup("Paddy Clarke Ha Ha Ha"), enc))->destroy_data = true;
242
240
  frt_doc_add_field(docs[26], frt_df_add_data(frt_df_new(year),
243
- frt_estrdup("1993")))->destroy_data = true;
241
+ frt_estrdup("1993"), enc))->destroy_data = true;
244
242
  docs[27] = frt_doc_new();
245
243
  frt_doc_add_field(docs[27], frt_df_add_data(frt_df_new(author),
246
- frt_estrdup("James Kelman")))->destroy_data = true;
244
+ frt_estrdup("James Kelman"), enc))->destroy_data = true;
247
245
  frt_doc_add_field(docs[27], frt_df_add_data(frt_df_new(title),
248
- frt_estrdup("How Late It Was, How Late")))->destroy_data = true;
246
+ frt_estrdup("How Late It Was, How Late"), enc))->destroy_data = true;
249
247
  frt_doc_add_field(docs[27], frt_df_add_data(frt_df_new(year),
250
- frt_estrdup("1994")))->destroy_data = true;
248
+ frt_estrdup("1994"), enc))->destroy_data = true;
251
249
  docs[28] = frt_doc_new();
252
250
  frt_doc_add_field(docs[28], frt_df_add_data(frt_df_new(author),
253
- frt_estrdup("Pat Barker")))->destroy_data = true;
251
+ frt_estrdup("Pat Barker"), enc))->destroy_data = true;
254
252
  frt_doc_add_field(docs[28], frt_df_add_data(frt_df_new(title),
255
- frt_estrdup("The Ghost Road")))->destroy_data = true;
253
+ frt_estrdup("The Ghost Road"), enc))->destroy_data = true;
256
254
  frt_doc_add_field(docs[28], frt_df_add_data(frt_df_new(year),
257
- frt_estrdup("1995")))->destroy_data = true;
255
+ frt_estrdup("1995"), enc))->destroy_data = true;
258
256
  docs[29] = frt_doc_new();
259
257
  frt_doc_add_field(docs[29], frt_df_add_data(frt_df_new(author),
260
- frt_estrdup("Graham Swift")))->destroy_data = true;
258
+ frt_estrdup("Graham Swift"), enc))->destroy_data = true;
261
259
  frt_doc_add_field(docs[29], frt_df_add_data(frt_df_new(title),
262
- frt_estrdup("Last Orders")))->destroy_data = true;
260
+ frt_estrdup("Last Orders"), enc))->destroy_data = true;
263
261
  frt_doc_add_field(docs[29], frt_df_add_data(frt_df_new(year),
264
- frt_estrdup("1996")))->destroy_data = true;
262
+ frt_estrdup("1996"), enc))->destroy_data = true;
265
263
  docs[30] = frt_doc_new();
266
264
  frt_doc_add_field(docs[30], frt_df_add_data(frt_df_new(author),
267
- frt_estrdup("Arundati Roy")))->destroy_data = true;
265
+ frt_estrdup("Arundati Roy"), enc))->destroy_data = true;
268
266
  frt_doc_add_field(docs[30], frt_df_add_data(frt_df_new(title),
269
- frt_estrdup("The God of Small Things")))->destroy_data = true;
267
+ frt_estrdup("The God of Small Things"), enc))->destroy_data = true;
270
268
  frt_doc_add_field(docs[30], frt_df_add_data(frt_df_new(year),
271
- frt_estrdup("1997")))->destroy_data = true;
269
+ frt_estrdup("1997"), enc))->destroy_data = true;
272
270
  docs[31] = frt_doc_new();
273
271
  frt_doc_add_field(docs[31], frt_df_add_data(frt_df_new(author),
274
- frt_estrdup("Ian McEwan")))->destroy_data = true;
272
+ frt_estrdup("Ian McEwan"), enc))->destroy_data = true;
275
273
  frt_doc_add_field(docs[31], frt_df_add_data(frt_df_new(title),
276
- frt_estrdup("Amsterdam")))->destroy_data = true;
274
+ frt_estrdup("Amsterdam"), enc))->destroy_data = true;
277
275
  frt_doc_add_field(docs[31], frt_df_add_data(frt_df_new(year),
278
- frt_estrdup("1998")))->destroy_data = true;
276
+ frt_estrdup("1998"), enc))->destroy_data = true;
279
277
  docs[32] = frt_doc_new();
280
278
  frt_doc_add_field(docs[32], frt_df_add_data(frt_df_new(author),
281
- frt_estrdup("J. M. Coetzee")))->destroy_data = true;
279
+ frt_estrdup("J. M. Coetzee"), enc))->destroy_data = true;
282
280
  frt_doc_add_field(docs[32], frt_df_add_data(frt_df_new(title),
283
- frt_estrdup("Disgrace")))->destroy_data = true;
281
+ frt_estrdup("Disgrace"), enc))->destroy_data = true;
284
282
  frt_doc_add_field(docs[32], frt_df_add_data(frt_df_new(year),
285
- frt_estrdup("1999")))->destroy_data = true;
283
+ frt_estrdup("1999"), enc))->destroy_data = true;
286
284
  docs[33] = frt_doc_new();
287
285
  frt_doc_add_field(docs[33], frt_df_add_data(frt_df_new(author),
288
- frt_estrdup("Margaret Atwood")))->destroy_data = true;
286
+ frt_estrdup("Margaret Atwood"), enc))->destroy_data = true;
289
287
  frt_doc_add_field(docs[33], frt_df_add_data(frt_df_new(title),
290
- frt_estrdup("The Blind Assassin")))->destroy_data = true;
288
+ frt_estrdup("The Blind Assassin"), enc))->destroy_data = true;
291
289
  frt_doc_add_field(docs[33], frt_df_add_data(frt_df_new(year),
292
- frt_estrdup("2000")))->destroy_data = true;
290
+ frt_estrdup("2000"), enc))->destroy_data = true;
293
291
  docs[34] = frt_doc_new();
294
292
  frt_doc_add_field(docs[34], frt_df_add_data(frt_df_new(author),
295
- frt_estrdup("Peter Carey")))->destroy_data = true;
293
+ frt_estrdup("Peter Carey"), enc))->destroy_data = true;
296
294
  frt_doc_add_field(docs[34], frt_df_add_data(frt_df_new(title),
297
- frt_estrdup("True History of the Kelly Gang")))->destroy_data = true;
295
+ frt_estrdup("True History of the Kelly Gang"), enc))->destroy_data = true;
298
296
  frt_doc_add_field(docs[34], frt_df_add_data(frt_df_new(year),
299
- frt_estrdup("2001")))->destroy_data = true;
297
+ frt_estrdup("2001"), enc))->destroy_data = true;
300
298
  docs[35] = frt_doc_new();
301
299
  frt_doc_add_field(docs[35], frt_df_add_data(frt_df_new(author),
302
- frt_estrdup("Yann Martel")))->destroy_data = true;
300
+ frt_estrdup("Yann Martel"), enc))->destroy_data = true;
303
301
  frt_doc_add_field(docs[35], frt_df_add_data(frt_df_new(title),
304
- frt_estrdup("The Life of Pi")))->destroy_data = true;
302
+ frt_estrdup("The Life of Pi"), enc))->destroy_data = true;
305
303
  frt_doc_add_field(docs[35], frt_df_add_data(frt_df_new(year),
306
- frt_estrdup("2002")))->destroy_data = true;
304
+ frt_estrdup("2002"), enc))->destroy_data = true;
307
305
  docs[36] = frt_doc_new();
308
306
  frt_doc_add_field(docs[36], frt_df_add_data(frt_df_new(author),
309
- frt_estrdup("DBC Pierre")))->destroy_data = true;
307
+ frt_estrdup("DBC Pierre"), enc))->destroy_data = true;
310
308
  frt_doc_add_field(docs[36], frt_df_add_data(frt_df_new(title),
311
- frt_estrdup("Vernon God Little")))->destroy_data = true;
309
+ frt_estrdup("Vernon God Little"), enc))->destroy_data = true;
312
310
  frt_doc_add_field(docs[36], frt_df_add_data(frt_df_new(year),
313
- frt_estrdup("2003")))->destroy_data = true;
311
+ frt_estrdup("2003"), enc))->destroy_data = true;
314
312
 
315
313
  return docs;
316
314
  }
@@ -327,7 +325,7 @@ static FrtIndexWriter *create_book_iw_conf(FrtStore *store, const FrtConfig *con
327
325
  FrtFieldInfos *fis = prep_book_fis();
328
326
  frt_index_create(store, fis);
329
327
  frt_fis_deref(fis);
330
- return frt_iw_open(store, frt_whitespace_analyzer_new(false), config);
328
+ return frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), config);
331
329
  }
332
330
 
333
331
  static FrtIndexWriter *create_book_iw(FrtStore *store)
@@ -337,136 +335,154 @@ static FrtIndexWriter *create_book_iw(FrtStore *store)
337
335
 
338
336
  #define IR_TEST_DOC_CNT 256
339
337
 
340
- FrtDocument **prep_ir_test_docs()
341
- {
338
+ FrtDocument **prep_ir_test_docs(void) {
342
339
  int i;
343
340
  char buf[2000] = "";
344
341
  FrtDocument **docs = FRT_ALLOC_N(FrtDocument *, IR_TEST_DOC_CNT);
345
342
  FrtDocField *df;
343
+ rb_encoding *enc = rb_enc_find("ASCII-8BIT");
346
344
 
347
345
  docs[0] = frt_doc_new();
348
346
  frt_doc_add_field(docs[0], frt_df_add_data(frt_df_new(changing_field),
349
347
  frt_estrdup("word3 word4 word1 word2 word1 word3 word4 word1 "
350
- "word3 word3")))->destroy_data = true;
351
- frt_doc_add_field(docs[0], frt_df_add_data(frt_df_new(compressed_field),
348
+ "word3 word3"), enc))->destroy_data = true;
349
+ frt_doc_add_field(docs[0], frt_df_add_data(frt_df_new(compressed_field_brotli),
350
+ frt_estrdup("word3 word4 word1 word2 word1 word3 word4 word1 "
351
+ "word3 word3"), enc))->destroy_data = true;
352
+ frt_doc_add_field(docs[0], frt_df_add_data(frt_df_new(compressed_field_bzip2),
352
353
  frt_estrdup("word3 word4 word1 word2 word1 word3 word4 word1 "
353
- "word3 word3")))->destroy_data = true;
354
+ "word3 word3"), enc))->destroy_data = true;
355
+ frt_doc_add_field(docs[0], frt_df_add_data(frt_df_new(compressed_field_lz4),
356
+ frt_estrdup("word3 word4 word1 word2 word1 word3 word4 word1 "
357
+ "word3 word3"), enc))->destroy_data = true;
354
358
  frt_doc_add_field(docs[0], frt_df_add_data(frt_df_new(body),
355
- frt_estrdup("Where is Wally")))->destroy_data = true;
359
+ frt_estrdup("Where is Wally"), enc))->destroy_data = true;
356
360
  docs[1] = frt_doc_new();
357
361
  frt_doc_add_field(docs[1], frt_df_add_data(frt_df_new(body),
358
- frt_estrdup("Some Random Sentence read")))->destroy_data = true;
362
+ frt_estrdup("Some Random Sentence read"), enc))->destroy_data = true;
359
363
  frt_doc_add_field(docs[1], frt_df_add_data(frt_df_new(tag),
360
- frt_estrdup("id_test")))->destroy_data = true;
364
+ frt_estrdup("id_test"), enc))->destroy_data = true;
361
365
  docs[2] = frt_doc_new();
362
366
  frt_doc_add_field(docs[2], frt_df_add_data(frt_df_new(body),
363
- frt_estrdup("Some read Random Sentence read")))->destroy_data = true;
367
+ frt_estrdup("Some read Random Sentence read"), enc))->destroy_data = true;
364
368
  df = frt_df_new(tag);
365
- frt_df_add_data(df, frt_estrdup("one"));
366
- frt_df_add_data(df, frt_estrdup("two"));
367
- frt_df_add_data(df, frt_estrdup("three"));
368
- frt_df_add_data(df, frt_estrdup("four"));
369
+ frt_df_add_data(df, frt_estrdup("one"), enc);
370
+ frt_df_add_data(df, frt_estrdup("two"), enc);
371
+ frt_df_add_data(df, frt_estrdup("three"), enc);
372
+ frt_df_add_data(df, frt_estrdup("four"), enc);
373
+ frt_doc_add_field(docs[2], df)->destroy_data = true;
374
+ df = frt_df_new(compressed_field_brotli);
375
+ frt_df_add_data(df, frt_estrdup("one"), enc);
376
+ frt_df_add_data(df, frt_estrdup("two"), enc);
377
+ frt_df_add_data(df, frt_estrdup("three"), enc);
378
+ frt_df_add_data(df, frt_estrdup("four"), enc);
369
379
  frt_doc_add_field(docs[2], df)->destroy_data = true;
370
- df = frt_df_new(compressed_field);
371
- frt_df_add_data(df, frt_estrdup("one"));
372
- frt_df_add_data(df, frt_estrdup("two"));
373
- frt_df_add_data(df, frt_estrdup("three"));
374
- frt_df_add_data(df, frt_estrdup("four"));
380
+ df = frt_df_new(compressed_field_bzip2);
381
+ frt_df_add_data(df, frt_estrdup("one"), enc);
382
+ frt_df_add_data(df, frt_estrdup("two"), enc);
383
+ frt_df_add_data(df, frt_estrdup("three"), enc);
384
+ frt_df_add_data(df, frt_estrdup("four"), enc);
385
+ frt_doc_add_field(docs[2], df)->destroy_data = true;
386
+ df = frt_df_new(compressed_field_lz4);
387
+ frt_df_add_data(df, frt_estrdup("one"), enc);
388
+ frt_df_add_data(df, frt_estrdup("two"), enc);
389
+ frt_df_add_data(df, frt_estrdup("three"), enc);
390
+ frt_df_add_data(df, frt_estrdup("four"), enc);
375
391
  frt_doc_add_field(docs[2], df)->destroy_data = true;
376
392
  docs[3] = frt_doc_new();
377
393
  frt_doc_add_field(docs[3], frt_df_add_data(frt_df_new(title),
378
- frt_estrdup("War And Peace")))->destroy_data = true;
394
+ frt_estrdup("War And Peace"), enc))->destroy_data = true;
379
395
  frt_doc_add_field(docs[3], frt_df_add_data(frt_df_new(body),
380
396
  frt_estrdup("word3 word4 word1 word2 word1 "
381
- "word3 word4 word1 word3 word3")))->destroy_data = true;
397
+ "word3 word4 word1 word3 word3"), enc))->destroy_data = true;
382
398
  frt_doc_add_field(docs[3], frt_df_add_data(frt_df_new(author),
383
- frt_estrdup("Leo Tolstoy")))->destroy_data = true;
399
+ frt_estrdup("Leo Tolstoy"), enc))->destroy_data = true;
384
400
  frt_doc_add_field(docs[3], frt_df_add_data(frt_df_new(year),
385
- frt_estrdup("1865")))->destroy_data = true;
401
+ frt_estrdup("1865"), enc))->destroy_data = true;
386
402
  frt_doc_add_field(docs[3], frt_df_add_data(frt_df_new(text),
387
- frt_estrdup("more text which is not stored")))->destroy_data = true;
403
+ frt_estrdup("more text which is not stored"), enc))->destroy_data = true;
388
404
  docs[4] = frt_doc_new();
389
405
  frt_doc_add_field(docs[4], frt_df_add_data(frt_df_new(body),
390
- frt_estrdup("Some Random Sentence")))->destroy_data = true;
406
+ frt_estrdup("Some Random Sentence"), enc))->destroy_data = true;
391
407
  docs[5] = frt_doc_new();
392
408
  frt_doc_add_field(docs[5], frt_df_add_data(frt_df_new(body),
393
- frt_estrdup("Here's Wally")))->destroy_data = true;
409
+ frt_estrdup("Here's Wally"), enc))->destroy_data = true;
394
410
  frt_doc_add_field(docs[5], frt_df_add_data(frt_df_new(text),
395
- frt_estrdup("so_that_norm_can_be_set")))->destroy_data = true;
411
+ frt_estrdup("so_that_norm_can_be_set"), enc))->destroy_data = true;
396
412
  docs[6] = frt_doc_new();
397
413
  frt_doc_add_field(docs[6], frt_df_add_data(frt_df_new(body),
398
414
  frt_estrdup("Some Random Sentence read read read read"
399
- )))->destroy_data = true;
415
+ ), enc))->destroy_data = true;
400
416
  docs[7] = frt_doc_new();
401
417
  frt_doc_add_field(docs[7], frt_df_add_data(frt_df_new(body),
402
- frt_estrdup("Some Random Sentence")))->destroy_data = true;
418
+ frt_estrdup("Some Random Sentence"), enc))->destroy_data = true;
403
419
  docs[8] = frt_doc_new();
404
420
  frt_doc_add_field(docs[8], frt_df_add_data(frt_df_new(body),
405
- frt_estrdup("Some Random Sentence")))->destroy_data = true;
421
+ frt_estrdup("Some Random Sentence"), enc))->destroy_data = true;
406
422
  docs[9] = frt_doc_new();
407
423
  frt_doc_add_field(docs[9], frt_df_add_data(frt_df_new(body),
408
424
  frt_estrdup("read Some Random Sentence read this will be used after "
409
- "unfinished next position read")))->destroy_data = true;
425
+ "unfinished next position read"), enc))->destroy_data = true;
410
426
  docs[10] = frt_doc_new();
411
427
  frt_doc_add_field(docs[10], frt_df_add_data(frt_df_new(body),
412
- frt_estrdup("Some read Random Sentence")))->destroy_data = true;
428
+ frt_estrdup("Some read Random Sentence"), enc))->destroy_data = true;
413
429
  frt_doc_add_field(docs[10], frt_df_add_data(frt_df_new(changing_field),
414
430
  frt_estrdup("word3 word4 word1 word2 word1 word3 word4 word1 word3 "
415
- "word3")))->destroy_data = true;
431
+ "word3"), enc))->destroy_data = true;
416
432
  docs[11] = frt_doc_new();
417
433
  frt_doc_add_field(docs[11], frt_df_add_data(frt_df_new(body),
418
- frt_estrdup("And here too. Well, maybe Not")))->destroy_data = true;
434
+ frt_estrdup("And here too. Well, maybe Not"), enc))->destroy_data = true;
419
435
  docs[12] = frt_doc_new();
420
436
  frt_doc_add_field(docs[12], frt_df_add_data(frt_df_new(title),
421
- frt_estrdup("Shawshank Redemption")))->destroy_data = true;
437
+ frt_estrdup("Shawshank Redemption"), enc))->destroy_data = true;
422
438
  frt_doc_add_field(docs[12], frt_df_add_data(frt_df_new(body),
423
- frt_estrdup("Some Random Sentence")))->destroy_data = true;
439
+ frt_estrdup("Some Random Sentence"), enc))->destroy_data = true;
424
440
  docs[13] = frt_doc_new();
425
441
  frt_doc_add_field(docs[13], frt_df_add_data(frt_df_new(body),
426
- frt_estrdup("Some Random Sentence")))->destroy_data = true;
442
+ frt_estrdup("Some Random Sentence"), enc))->destroy_data = true;
427
443
  docs[14] = frt_doc_new();
428
444
  frt_doc_add_field(docs[14], frt_df_add_data(frt_df_new(body),
429
- frt_estrdup("Some Random Sentence")))->destroy_data = true;
445
+ frt_estrdup("Some Random Sentence"), enc))->destroy_data = true;
430
446
  docs[15] = frt_doc_new();
431
447
  frt_doc_add_field(docs[15], frt_df_add_data(frt_df_new(body),
432
- frt_estrdup("Some read Random Sentence")))->destroy_data = true;
448
+ frt_estrdup("Some read Random Sentence"), enc))->destroy_data = true;
433
449
  docs[16] = frt_doc_new();
434
450
  frt_doc_add_field(docs[16], frt_df_add_data(frt_df_new(body),
435
- frt_estrdup("Some Random read read Sentence")))->destroy_data = true;
451
+ frt_estrdup("Some Random read read Sentence"), enc))->destroy_data = true;
436
452
  docs[17] = frt_doc_new();
437
453
  frt_doc_add_field(docs[17], frt_df_add_data(frt_df_new(body),
438
- frt_estrdup("Some Random read Sentence")))->destroy_data = true;
454
+ frt_estrdup("Some Random read Sentence"), enc))->destroy_data = true;
439
455
  frt_doc_add_field(docs[17], frt_df_add_data(frt_df_new(changing_field),
440
456
  frt_estrdup("word3 word4 word1 word2 word1 word3 word4 word1 word3 "
441
- "word3")))->destroy_data = true;
457
+ "word3"), enc))->destroy_data = true;
442
458
  docs[18] = frt_doc_new();
443
459
  frt_doc_add_field(docs[18], frt_df_add_data(frt_df_new(body),
444
- frt_estrdup("Wally Wally Wally")))->destroy_data = true;
460
+ frt_estrdup("Wally Wally Wally"), enc))->destroy_data = true;
445
461
  docs[19] = frt_doc_new();
446
462
  frt_doc_add_field(docs[19], frt_df_add_data(frt_df_new(body),
447
- frt_estrdup("Some Random Sentence")))->destroy_data = true;
463
+ frt_estrdup("Some Random Sentence"), enc))->destroy_data = true;
448
464
  frt_doc_add_field(docs[19], frt_df_add_data(frt_df_new(changing_field),
449
465
  frt_estrdup("word3 word4 word1 word2 word1 word3 word4 word1 word3 "
450
- "word3")))->destroy_data = true;
466
+ "word3"), enc))->destroy_data = true;
451
467
  docs[20] = frt_doc_new();
452
468
  frt_doc_add_field(docs[20], frt_df_add_data(frt_df_new(body),
453
469
  frt_estrdup("Wally is where Wally usually likes to go. Wally Mart! "
454
470
  "Wally likes shopping there for Where's Wally books. "
455
- "Wally likes to read")))->destroy_data = true;
471
+ "Wally likes to read"), enc))->destroy_data = true;
456
472
  frt_doc_add_field(docs[20], frt_df_add_data(frt_df_new(changing_field),
457
473
  frt_estrdup("word3 word4 word1 word2 word1 word3 word4 word1 word3 "
458
- "word3")))->destroy_data = true;
474
+ "word3"), enc))->destroy_data = true;
459
475
  docs[21] = frt_doc_new();
460
476
  frt_doc_add_field(docs[21], frt_df_add_data(frt_df_new(body),
461
477
  frt_estrdup("Some Random Sentence read read read and more read read "
462
- "read")))->destroy_data = true;
478
+ "read"), enc))->destroy_data = true;
463
479
  frt_doc_add_field(docs[21], frt_df_add_data(frt_df_new(changing_field),
464
480
  frt_estrdup("word3 word4 word1 word2 word1 word3 word4 word1 word3 "
465
- "word3")))->destroy_data = true;
481
+ "word3"), enc))->destroy_data = true;
466
482
  frt_doc_add_field(docs[21], frt_df_add_data(frt_df_new(rb_intern("new field")),
467
- frt_estrdup("zdata znot zto zbe zfound")))->destroy_data = true;
483
+ frt_estrdup("zdata znot zto zbe zfound"), enc))->destroy_data = true;
468
484
  frt_doc_add_field(docs[21], frt_df_add_data(frt_df_new(title),
469
- frt_estrdup("title_too_long_for_max_word_lengthxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx")))->destroy_data = true;
485
+ frt_estrdup("title_too_long_for_max_word_lengthxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"), enc))->destroy_data = true;
470
486
 
471
487
  for (i = 1; i < 22; i++) {
472
488
  strcat(buf, "skip ");
@@ -475,7 +491,7 @@ FrtDocument **prep_ir_test_docs()
475
491
  strcat(buf, "skip ");
476
492
  docs[i] = frt_doc_new();
477
493
  frt_doc_add_field(docs[i], frt_df_add_data(frt_df_new(text),
478
- frt_estrdup(buf)))->destroy_data = true;
494
+ frt_estrdup(buf), enc))->destroy_data = true;
479
495
  }
480
496
  return docs;
481
497
  }
@@ -488,12 +504,14 @@ static void prep_stde_test_docs(FrtDocument **docs, int doc_cnt, int num_words,
488
504
  {
489
505
  int i, j;
490
506
  char *buf = FRT_ALLOC_N(char, num_words * (TEST_WORD_LIST_MAX_LEN + 1));
507
+ rb_encoding *enc = rb_enc_find("ASCII-8BIT");
508
+
491
509
  for (i = 0; i < doc_cnt; i++) {
492
510
  docs[i] = frt_doc_new();
493
511
  for (j = 0; j < fis->size; j++) {
494
512
  if ((rand() % 2) == 0) {
495
513
  FrtDocField *df = frt_df_new(fis->fields[j]->name);
496
- frt_df_add_data(df, frt_estrdup(make_random_string(buf, num_words)));
514
+ frt_df_add_data(df, frt_estrdup(make_random_string(buf, num_words)), enc);
497
515
  df->destroy_data = true;
498
516
  frt_doc_add_field(docs[i], df);
499
517
  }
@@ -511,7 +529,7 @@ static void prep_test_1seg_index(FrtStore *store, FrtDocument **docs,
511
529
  FrtSegmentInfo *si = frt_si_new(frt_estrdup("_0"), doc_cnt, store);
512
530
 
513
531
  frt_index_create(store, fis);
514
- iw = frt_iw_open(store, frt_whitespace_analyzer_new(false), NULL);
532
+ iw = frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), NULL);
515
533
 
516
534
  dw = frt_dw_open(iw, si);
517
535
 
@@ -658,6 +676,7 @@ static void test_segment_tde_deleted_docs(TestCase *tc, void *data)
658
676
  FrtBitVector *bv = frt_bv_new();
659
677
  FrtTermDocEnum *tde;
660
678
  FrtSegmentInfo *si = frt_si_new(frt_estrdup("_0"), NUM_STDE_TEST_DOCS, store);
679
+ rb_encoding *enc = rb_enc_find("ASCII-8BIT");
661
680
 
662
681
  dw = frt_dw_open(iw, si);
663
682
 
@@ -666,10 +685,10 @@ static void test_segment_tde_deleted_docs(TestCase *tc, void *data)
666
685
  if ((rand() % 2) == 0) {
667
686
  frt_bv_set(bv, i);
668
687
  Aiequal(1, frt_bv_get(bv, i));
669
- frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("f")), (char *)double_word));
688
+ frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("f")), (char *)double_word, enc));
670
689
  }
671
690
  else {
672
- frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("f")), (char *)triple_word));
691
+ frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("f")), (char *)triple_word, enc));
673
692
  }
674
693
  frt_dw_add_doc(dw, doc);
675
694
  frt_doc_destroy(doc);
@@ -718,15 +737,13 @@ static void test_segment_tde_deleted_docs(TestCase *tc, void *data)
718
737
  static void test_index_create(TestCase *tc, void *data)
719
738
  {
720
739
  FrtStore *store = (FrtStore *)data;
721
- FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_INDEX_YES, FRT_TERM_VECTOR_YES);
740
+ FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_YES);
722
741
  (void)tc;
723
742
 
724
743
  store->clear_all(store);
725
- Assert(!store->exists(store, "segments"),
726
- "segments shouldn't exist yet");
744
+ Assert(!store->exists(store, "segments"), "segments shouldn't exist yet");
727
745
  frt_index_create(store, fis);
728
- Assert(store->exists(store, "segments"),
729
- "segments should now exist");
746
+ Assert(store->exists(store, "segments"), "segments should now exist");
730
747
  frt_fis_deref(fis);
731
748
  }
732
749
 
@@ -741,16 +758,16 @@ static void test_index_version(TestCase *tc, void *data)
741
758
  Atrue(frt_index_is_locked(store)); /* writer open, so dir is locked */
742
759
  frt_iw_close(iw);
743
760
  Atrue(!frt_index_is_locked(store));
744
- ir = frt_ir_open(store);
761
+ ir = frt_ir_open(NULL, store);
745
762
  Atrue(!frt_index_is_locked(store)); /* reader only, no lock */
746
763
  version = frt_sis_read_current_version(store);
747
764
  frt_ir_close(ir);
748
765
 
749
766
  /* modify index and check version has been incremented: */
750
- iw = frt_iw_open(store, frt_whitespace_analyzer_new(false), &frt_default_config);
767
+ iw = frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), &frt_default_config);
751
768
  add_document_with_fields(iw, 1);
752
769
  frt_iw_close(iw);
753
- ir = frt_ir_open(store);
770
+ ir = frt_ir_open(NULL, store);
754
771
  Atrue(version < frt_sis_read_current_version(store));
755
772
  Atrue(frt_ir_is_latest(ir));
756
773
  frt_ir_close(ir);
@@ -764,15 +781,15 @@ static void test_index_undelete_all_after_close(TestCase *tc, void *data)
764
781
  add_document_with_fields(iw, 0);
765
782
  add_document_with_fields(iw, 1);
766
783
  frt_iw_close(iw);
767
- ir = frt_ir_open(store);
784
+ ir = frt_ir_open(NULL, store);
768
785
  frt_ir_delete_doc(ir, 0);
769
786
  frt_ir_delete_doc(ir, 1);
770
787
  frt_ir_close(ir);
771
- ir = frt_ir_open(store);
788
+ ir = frt_ir_open(NULL, store);
772
789
  frt_ir_undelete_all(ir);
773
790
  Aiequal(2, ir->num_docs(ir)); /* nothing has really been deleted */
774
791
  frt_ir_close(ir);
775
- ir = frt_ir_open(store);
792
+ ir = frt_ir_open(NULL, store);
776
793
  Aiequal(2, ir->num_docs(ir)); /* nothing has really been deleted */
777
794
  Atrue(frt_ir_is_latest(ir));
778
795
  frt_ir_close(ir);
@@ -794,13 +811,14 @@ static void test_fld_inverter(TestCase *tc, void *data)
794
811
  FrtDocWriter *dw;
795
812
  FrtIndexWriter *iw = create_book_iw(store);
796
813
  FrtDocField *df;
814
+ rb_encoding *enc = rb_enc_find("ASCII-8BIT");
797
815
 
798
816
  dw = frt_dw_open(iw, frt_sis_new_segment(iw->sis, 0, iw->store));
799
817
 
800
818
  df = frt_df_new(rb_intern("no tv"));
801
- frt_df_add_data(df, (char *)"one two three four five two three four five three four five four five");
802
- frt_df_add_data(df, (char *)"ichi ni san yon go ni san yon go san yon go yon go go");
803
- frt_df_add_data(df, (char *)"The quick brown fox jumped over five lazy dogs");
819
+ frt_df_add_data(df, (char *)"one two three four five two three four five three four five four five", enc);
820
+ frt_df_add_data(df, (char *)"ichi ni san yon go ni san yon go san yon go yon go go", enc);
821
+ frt_df_add_data(df, (char *)"The quick brown fox jumped over five lazy dogs", enc);
804
822
 
805
823
  curr_plists = frt_dw_invert_field(
806
824
  dw,
@@ -844,8 +862,8 @@ static void test_fld_inverter(TestCase *tc, void *data)
844
862
  frt_df_destroy(df);
845
863
 
846
864
  df = frt_df_new(rb_intern("no tv"));
847
- frt_df_add_data(df, (char *)"seven new words and six old ones");
848
- frt_df_add_data(df, (char *)"ichi ni one two quick dogs");
865
+ frt_df_add_data(df, (char *)"seven new words and six old ones", enc);
866
+ frt_df_add_data(df, (char *)"ichi ni one two quick dogs", enc);
849
867
 
850
868
  dw->doc_num++;
851
869
  frt_dw_reset_postings(dw->curr_plists);
@@ -916,7 +934,7 @@ static void test_iw_add_doc(TestCase *tc, void *data)
916
934
  frt_iw_close(iw);
917
935
  Assert(store->exists(store, "_0.cfs"), "data should still be there");
918
936
 
919
- iw = frt_iw_open(store, frt_whitespace_analyzer_new(false), &frt_default_config);
937
+ iw = frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), &frt_default_config);
920
938
  frt_iw_add_doc(iw, docs[1]);
921
939
  Aiequal(2, frt_iw_doc_count(iw));
922
940
  Assert(!store->exists(store, "_1.cfs"),
@@ -951,7 +969,7 @@ static void test_create_with_reader(TestCase *tc, void *data)
951
969
  frt_iw_close(iw);
952
970
 
953
971
  /* now open reader: */
954
- ir = frt_ir_open(store);
972
+ ir = frt_ir_open(NULL, store);
955
973
  Aiequal(1, ir->num_docs(ir));
956
974
 
957
975
  /* now open index for create: */
@@ -961,7 +979,7 @@ static void test_create_with_reader(TestCase *tc, void *data)
961
979
  frt_iw_close(iw);
962
980
 
963
981
  Aiequal(1, ir->num_docs(ir));
964
- ir2 = frt_ir_open(store);
982
+ ir2 = frt_ir_open(NULL, store);
965
983
  Aiequal(1, ir2->num_docs(ir));
966
984
  frt_ir_close(ir);
967
985
  frt_ir_close(ir2);
@@ -1018,10 +1036,10 @@ static void test_simulated_crashed_writer(TestCase *tc, void *data)
1018
1036
  frt_is_close(is);
1019
1037
  frt_os_close(os);
1020
1038
 
1021
- ir = frt_ir_open(store);
1039
+ ir = frt_ir_open(NULL, store);
1022
1040
  frt_ir_close(ir);
1023
1041
 
1024
- iw = frt_iw_open(store, frt_whitespace_analyzer_new(false), &config);
1042
+ iw = frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), &config);
1025
1043
 
1026
1044
  /* add all books */
1027
1045
  for (i = 0; i < BOOK_LIST_LENGTH; i++) {
@@ -1081,7 +1099,7 @@ static void test_simulated_corrupt_index1(TestCase *tc, void *data)
1081
1099
  store->remove(store, file_name_in);
1082
1100
 
1083
1101
  FRT_TRY
1084
- ir = frt_ir_open(store);
1102
+ ir = frt_ir_open(NULL, store);
1085
1103
  frt_ir_close(ir);
1086
1104
  Afail("reader should have failed to open on a crashed index");
1087
1105
  break;
@@ -1127,7 +1145,7 @@ static void test_simulated_corrupt_index2(TestCase *tc, void *data)
1127
1145
  store->remove(store, "_0.cfs");
1128
1146
 
1129
1147
  FRT_TRY
1130
- ir = frt_ir_open(store);
1148
+ ir = frt_ir_open(NULL, store);
1131
1149
  frt_ir_close(ir);
1132
1150
  Afail("reader should have failed to open on a crashed index");
1133
1151
  break;
@@ -1171,16 +1189,18 @@ void test_iw_add_empty_tv(TestCase *tc, void *data)
1171
1189
  FrtStore *store = (FrtStore *)data;
1172
1190
  FrtIndexWriter *iw;
1173
1191
  FrtDocument *doc;
1174
- FrtFieldInfos *fis = frt_fis_new(FRT_STORE_NO, FRT_INDEX_YES, FRT_TERM_VECTOR_YES);
1175
- frt_fis_add_field(fis, frt_fi_new(rb_intern("no_tv"), FRT_STORE_YES, FRT_INDEX_YES, FRT_TERM_VECTOR_NO));
1192
+ rb_encoding *enc = rb_enc_find("ASCII-8BIT");
1193
+
1194
+ FrtFieldInfos *fis = frt_fis_new(FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_YES);
1195
+ frt_fis_add_field(fis, frt_fi_new(rb_intern("no_tv"), FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_NO));
1176
1196
  frt_index_create(store, fis);
1177
1197
  frt_fis_deref(fis);
1178
1198
 
1179
- iw = frt_iw_open(store, frt_whitespace_analyzer_new(false), &frt_default_config);
1199
+ iw = frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), &frt_default_config);
1180
1200
  doc = frt_doc_new();
1181
- frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("tv1")), (char *)""));
1182
- frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("tv2")), (char *)""));
1183
- frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("no_tv")), (char *)"one two three"));
1201
+ frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("tv1")), (char *)"", enc));
1202
+ frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("tv2")), (char *)"", enc));
1203
+ frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("no_tv")), (char *)"one two three", enc));
1184
1204
 
1185
1205
  frt_iw_add_doc(iw, doc);
1186
1206
  frt_iw_commit(iw);
@@ -1209,22 +1229,22 @@ static void test_iw_del_terms(TestCase *tc, void *data)
1209
1229
  frt_iw_close(iw);
1210
1230
  destroy_docs(docs, BOOK_LIST_LENGTH);
1211
1231
 
1212
- ir = frt_ir_open(store);
1232
+ ir = frt_ir_open(NULL, store);
1213
1233
  Aiequal(BOOK_LIST_LENGTH, ir->num_docs(ir));
1214
1234
  Aiequal(BOOK_LIST_LENGTH, ir->max_doc(ir));
1215
1235
  frt_ir_close(ir);
1216
1236
 
1217
- iw = frt_iw_open(store, frt_whitespace_analyzer_new(false), &config);
1237
+ iw = frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), &config);
1218
1238
  frt_iw_delete_term(iw, title, "State");
1219
1239
  frt_iw_close(iw);
1220
1240
 
1221
- ir = frt_ir_open(store);
1241
+ ir = frt_ir_open(NULL, store);
1222
1242
  Aiequal(BOOK_LIST_LENGTH - 1, ir->num_docs(ir));
1223
1243
  Aiequal(BOOK_LIST_LENGTH, ir->max_doc(ir));
1224
1244
  frt_ir_close(ir);
1225
1245
 
1226
1246
  /* test deleting multiple Terms */
1227
- iw = frt_iw_open(store, frt_whitespace_analyzer_new(false), &config);
1247
+ iw = frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), &config);
1228
1248
  frt_iw_delete_term(iw, title, "The");
1229
1249
  frt_iw_delete_term(iw, title, "Blind");
1230
1250
  terms[0] = "Berger";
@@ -1233,7 +1253,7 @@ static void test_iw_del_terms(TestCase *tc, void *data)
1233
1253
  frt_iw_delete_terms(iw, author, (char **)terms, 3);
1234
1254
  frt_iw_close(iw);
1235
1255
 
1236
- ir = frt_ir_open(store);
1256
+ ir = frt_ir_open(NULL, store);
1237
1257
  Aiequal(BOOK_LIST_LENGTH - 17, ir->num_docs(ir));
1238
1258
  Aiequal(BOOK_LIST_LENGTH, ir->max_doc(ir));
1239
1259
  Atrue(!ir->is_deleted(ir, 0));
@@ -1260,13 +1280,13 @@ static void test_iw_del_terms(TestCase *tc, void *data)
1260
1280
  Atrue(ir->is_deleted(ir, 36));
1261
1281
  frt_ir_commit(ir);
1262
1282
 
1263
- iw = frt_iw_open(store, frt_whitespace_analyzer_new(false), &config);
1283
+ iw = frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), &config);
1264
1284
  frt_iw_optimize(iw);
1265
1285
  frt_iw_close(iw);
1266
1286
 
1267
1287
  frt_ir_close(ir);
1268
1288
 
1269
- ir = frt_ir_open(store);
1289
+ ir = frt_ir_open(NULL, store);
1270
1290
  Aiequal(BOOK_LIST_LENGTH - 17, ir->num_docs(ir));
1271
1291
  Aiequal(BOOK_LIST_LENGTH - 17, ir->max_doc(ir));
1272
1292
  frt_ir_close(ir);
@@ -1301,15 +1321,15 @@ static void reader_test_env_destroy(ReaderTestEnvironment *rte)
1301
1321
  static FrtIndexReader *reader_test_env_ir_open(ReaderTestEnvironment *rte)
1302
1322
  {
1303
1323
  if (rte->store_cnt == 1) {
1304
- return frt_ir_open(rte->stores[0]);
1324
+ return frt_ir_open(NULL, rte->stores[0]);
1305
1325
  }
1306
1326
  else {
1307
1327
  FrtIndexReader **sub_readers = FRT_ALLOC_N(FrtIndexReader *, rte->store_cnt);
1308
1328
  int i;
1309
1329
  for (i = 0; i < rte->store_cnt; i++) {
1310
- sub_readers[i] = frt_ir_open(rte->stores[i]);
1330
+ sub_readers[i] = frt_ir_open(NULL, rte->stores[i]);
1311
1331
  }
1312
- return (frt_mr_open(sub_readers, rte->store_cnt));
1332
+ return (frt_mr_open(NULL, sub_readers, rte->store_cnt));
1313
1333
  }
1314
1334
  }
1315
1335
 
@@ -1326,9 +1346,8 @@ static ReaderTestEnvironment *reader_test_env_new(int type)
1326
1346
 
1327
1347
  rte->stores = FRT_ALLOC_N(FrtStore *, store_cnt);
1328
1348
  for (i = 0; i < store_cnt; i++) {
1329
- FrtStore *store = rte->stores[i] = frt_open_ram_store();
1330
- FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_INDEX_YES,
1331
- FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
1349
+ FrtStore *store = rte->stores[i] = frt_open_ram_store(NULL);
1350
+ FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
1332
1351
  int start_doc = i * doc_cnt;
1333
1352
  int end_doc = (i + 1) * doc_cnt;
1334
1353
  if (end_doc > IR_TEST_DOC_CNT) {
@@ -1338,7 +1357,7 @@ static ReaderTestEnvironment *reader_test_env_new(int type)
1338
1357
  frt_fis_deref(fis);
1339
1358
  config.max_buffered_docs = 3;
1340
1359
 
1341
- iw = frt_iw_open(store, frt_whitespace_analyzer_new(false), &config);
1360
+ iw = frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), &config);
1342
1361
 
1343
1362
  for (j = start_doc; j < end_doc; j++) {
1344
1363
  int k;
@@ -1350,24 +1369,19 @@ static ReaderTestEnvironment *reader_test_env_new(int type)
1350
1369
  fis = iw->fis;
1351
1370
  if (NULL == frt_fis_get_field(fis, df->name)) {
1352
1371
  if (author == df->name) {
1353
- frt_fis_add_field(fis, frt_fi_new(author, FRT_STORE_YES, FRT_INDEX_YES,
1354
- FRT_TERM_VECTOR_WITH_POSITIONS));
1372
+ frt_fis_add_field(fis, frt_fi_new(author, FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_WITH_POSITIONS));
1355
1373
  } else if (title == df->name) {
1356
- frt_fis_add_field(fis, frt_fi_new(title, FRT_STORE_YES,
1357
- FRT_INDEX_UNTOKENIZED,
1358
- FRT_TERM_VECTOR_WITH_OFFSETS));
1374
+ frt_fis_add_field(fis, frt_fi_new(title, FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_OFFSETS));
1359
1375
  } else if (year == df->name) {
1360
- frt_fis_add_field(fis, frt_fi_new(year, FRT_STORE_YES,
1361
- FRT_INDEX_UNTOKENIZED,
1362
- FRT_TERM_VECTOR_NO));
1376
+ frt_fis_add_field(fis, frt_fi_new(year, FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_NO));
1363
1377
  } else if (text == df->name) {
1364
- frt_fis_add_field(fis, frt_fi_new(text, FRT_STORE_NO, FRT_INDEX_YES,
1365
- FRT_TERM_VECTOR_NO));
1366
- } else if (compressed_field == df->name) {
1367
- frt_fis_add_field(fis, frt_fi_new(compressed_field,
1368
- FRT_STORE_COMPRESS,
1369
- FRT_INDEX_YES,
1370
- FRT_TERM_VECTOR_NO));
1378
+ frt_fis_add_field(fis, frt_fi_new(text, FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_NO));
1379
+ } else if (compressed_field_brotli == df->name) {
1380
+ frt_fis_add_field(fis, frt_fi_new(compressed_field_brotli, FRT_STORE_YES, FRT_COMPRESSION_BROTLI, FRT_INDEX_YES, FRT_TERM_VECTOR_NO));
1381
+ } else if (compressed_field_bzip2 == df->name) {
1382
+ frt_fis_add_field(fis, frt_fi_new(compressed_field_bzip2, FRT_STORE_YES, FRT_COMPRESSION_BZ2, FRT_INDEX_YES, FRT_TERM_VECTOR_NO));
1383
+ } else if (compressed_field_lz4 == df->name) {
1384
+ frt_fis_add_field(fis, frt_fi_new(compressed_field_lz4, FRT_STORE_YES, FRT_COMPRESSION_LZ4, FRT_INDEX_YES, FRT_TERM_VECTOR_NO));
1371
1385
  }
1372
1386
  }
1373
1387
  }
@@ -1382,20 +1396,19 @@ static ReaderTestEnvironment *reader_test_env_new(int type)
1382
1396
 
1383
1397
  if (type == add_indexes_reader_type) {
1384
1398
  /* Prepare store for Add Indexes test */
1385
- FrtStore *store = frt_open_ram_store();
1386
- FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_INDEX_YES,
1387
- FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
1399
+ FrtStore *store = frt_open_ram_store(NULL);
1400
+ FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
1388
1401
  FrtIndexReader **readers = FRT_ALLOC_N(FrtIndexReader *, rte->store_cnt);
1389
1402
  int i;
1390
1403
  for (i = 0; i < rte->store_cnt; i++) {
1391
- readers[i] = frt_ir_open(rte->stores[i]);
1404
+ readers[i] = frt_ir_open(NULL, rte->stores[i]);
1392
1405
  }
1393
1406
  frt_index_create(store, fis);
1394
1407
  frt_fis_deref(fis);
1395
- iw = frt_iw_open(store, frt_whitespace_analyzer_new(false), &config);
1408
+ iw = frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), &config);
1396
1409
  frt_iw_add_readers(iw, readers, rte->store_cnt - 10);
1397
1410
  frt_iw_close(iw);
1398
- iw = frt_iw_open(store, frt_whitespace_analyzer_new(false), &config);
1411
+ iw = frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), &config);
1399
1412
  frt_iw_add_readers(iw, readers + (rte->store_cnt - 10), 10);
1400
1413
  frt_iw_close(iw);
1401
1414
  for (i = 0; i < rte->store_cnt; i++) {
@@ -1418,23 +1431,19 @@ static void write_ir_test_docs(FrtStore *store)
1418
1431
  FrtIndexWriter *iw;
1419
1432
  FrtDocument **docs = prep_ir_test_docs();
1420
1433
 
1421
- FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_INDEX_YES,
1422
- FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
1423
- frt_fis_add_field(fis, frt_fi_new(author, FRT_STORE_YES, FRT_INDEX_YES,
1424
- FRT_TERM_VECTOR_WITH_POSITIONS));
1425
- frt_fis_add_field(fis, frt_fi_new(title, FRT_STORE_YES, FRT_INDEX_UNTOKENIZED,
1426
- FRT_TERM_VECTOR_WITH_OFFSETS));
1427
- frt_fis_add_field(fis, frt_fi_new(year, FRT_STORE_YES, FRT_INDEX_UNTOKENIZED,
1428
- FRT_TERM_VECTOR_NO));
1429
- frt_fis_add_field(fis, frt_fi_new(text, FRT_STORE_NO, FRT_INDEX_YES,
1430
- FRT_TERM_VECTOR_NO));
1431
- frt_fis_add_field(fis, frt_fi_new(compressed_field, FRT_STORE_COMPRESS, FRT_INDEX_YES,
1432
- FRT_TERM_VECTOR_NO));
1434
+ FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
1435
+ frt_fis_add_field(fis, frt_fi_new(author, FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_WITH_POSITIONS));
1436
+ frt_fis_add_field(fis, frt_fi_new(title, FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_WITH_OFFSETS));
1437
+ frt_fis_add_field(fis, frt_fi_new(year, FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_UNTOKENIZED, FRT_TERM_VECTOR_NO));
1438
+ frt_fis_add_field(fis, frt_fi_new(text, FRT_STORE_NO, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_NO));
1439
+ frt_fis_add_field(fis, frt_fi_new(compressed_field_brotli, FRT_STORE_YES, FRT_COMPRESSION_BROTLI, FRT_INDEX_YES, FRT_TERM_VECTOR_NO));
1440
+ frt_fis_add_field(fis, frt_fi_new(compressed_field_bzip2, FRT_STORE_YES, FRT_COMPRESSION_BZ2, FRT_INDEX_YES, FRT_TERM_VECTOR_NO));
1441
+ frt_fis_add_field(fis, frt_fi_new(compressed_field_lz4, FRT_STORE_YES, FRT_COMPRESSION_LZ4, FRT_INDEX_YES, FRT_TERM_VECTOR_NO));
1433
1442
  frt_index_create(store, fis);
1434
1443
  frt_fis_deref(fis);
1435
1444
  config.max_buffered_docs = 5;
1436
1445
 
1437
- iw = frt_iw_open(store, frt_whitespace_analyzer_new(false), &config);
1446
+ iw = frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), &config);
1438
1447
 
1439
1448
  for (i = 0; i < IR_TEST_DOC_CNT; i++) {
1440
1449
  frt_iw_add_doc(iw, docs[i]);
@@ -1449,7 +1458,7 @@ static void test_ir_open_empty_index(TestCase *tc, void *data)
1449
1458
  FrtStore *store = (FrtStore *)data;
1450
1459
  store->clear_all(store);
1451
1460
  FRT_TRY
1452
- frt_ir_close(frt_ir_open(store));
1461
+ frt_ir_close(frt_ir_open(NULL, store));
1453
1462
  Afail("IndexReader should have failed when opening empty index");
1454
1463
  break;
1455
1464
  case FRT_FILE_NOT_FOUND_ERROR:
@@ -1472,9 +1481,7 @@ static void test_ir_basic_ops(TestCase *tc, void *data)
1472
1481
  Atrue(frt_ir_is_latest(ir));
1473
1482
  }
1474
1483
 
1475
- static void test_ir_term_docpos_enum_skip_to(TestCase *tc,
1476
- FrtTermDocEnum *tde,
1477
- int field_num)
1484
+ static void test_ir_term_docpos_enum_skip_to(TestCase *tc, FrtTermDocEnum *tde, int field_num)
1478
1485
  {
1479
1486
  /* test skip_to working skip interval */
1480
1487
  tde->seek(tde, field_num, "skip");
@@ -1708,8 +1715,8 @@ static void test_ir_term_vectors(TestCase *tc, void *data)
1708
1715
  Aiequal(5, tv->terms[2].positions[1]);
1709
1716
  Aiequal(8, tv->terms[2].positions[2]);
1710
1717
  Aiequal(9, tv->terms[2].positions[3]);
1711
- Aiequal(0, tv->offsets[tv->terms[2].positions[0]].start);
1712
- Aiequal(5, tv->offsets[tv->terms[2].positions[0]].end);
1718
+ Aiequal(0, tv->offsets[tv->terms[2].positions[0]].start);
1719
+ Aiequal(5, tv->offsets[tv->terms[2].positions[0]].end);
1713
1720
  Aiequal(30, tv->offsets[tv->terms[2].positions[1]].start);
1714
1721
  Aiequal(35, tv->offsets[tv->terms[2].positions[1]].end);
1715
1722
  Aiequal(48, tv->offsets[tv->terms[2].positions[2]].start);
@@ -1720,7 +1727,7 @@ static void test_ir_term_vectors(TestCase *tc, void *data)
1720
1727
  Aiequal(2, tv->terms[3].freq);
1721
1728
  Aiequal(1, tv->terms[3].positions[0]);
1722
1729
  Aiequal(6, tv->terms[3].positions[1]);
1723
- Aiequal(6, tv->offsets[tv->terms[3].positions[0]].start);
1730
+ Aiequal(6, tv->offsets[tv->terms[3].positions[0]].start);
1724
1731
  Aiequal(11, tv->offsets[tv->terms[3].positions[0]].end);
1725
1732
  Aiequal(36, tv->offsets[tv->terms[3].positions[1]].start);
1726
1733
  Aiequal(41, tv->offsets[tv->terms[3].positions[1]].end);
@@ -1792,48 +1799,76 @@ static void test_ir_compression(TestCase *tc, void *data)
1792
1799
  int i;
1793
1800
  FrtIndexReader *ir = (FrtIndexReader *)data;
1794
1801
  FrtLazyDoc *lz_doc;
1795
- FrtLazyDocField *lz_df1, *lz_df2;
1802
+ FrtLazyDocField *lz_df1, *lz_df2, *lz_df3, *lz_df4;
1796
1803
  FrtDocument *doc = ir->get_doc(ir, 0);
1797
- FrtDocField *df1, *df2;
1804
+ FrtDocField *df1, *df2, *df3, *df4;
1798
1805
  char buf1[20], buf2[20];
1799
- Aiequal(3, doc->size);
1806
+ Aiequal(5, doc->size);
1800
1807
 
1801
1808
  df1 = frt_doc_get_field(doc, changing_field);
1802
- df2 = frt_doc_get_field(doc, compressed_field);
1809
+ df2 = frt_doc_get_field(doc, compressed_field_brotli);
1810
+ df3 = frt_doc_get_field(doc, compressed_field_bzip2);
1811
+ df4 = frt_doc_get_field(doc, compressed_field_lz4);
1803
1812
  Asequal(df1->data[0], df2->data[0]);
1813
+ Asequal(df1->data[0], df3->data[0]);
1814
+ Asequal(df1->data[0], df4->data[0]);
1804
1815
  Assert(df1->lengths[0] == df2->lengths[0], "Field lengths should be equal");
1805
- frt_doc_destroy(doc);
1816
+ Assert(df1->lengths[0] == df3->lengths[0], "Field lengths should be equal");
1817
+ Assert(df1->lengths[0] == df4->lengths[0], "Field lengths should be equal");
1818
+ frt_doc_destroy(doc);
1806
1819
 
1807
1820
  doc = ir->get_doc(ir, 2);
1808
1821
  df1 = frt_doc_get_field(doc, tag);
1809
- df2 = frt_doc_get_field(doc, compressed_field);
1822
+ df2 = frt_doc_get_field(doc, compressed_field_brotli);
1823
+ df3 = frt_doc_get_field(doc, compressed_field_bzip2);
1824
+ df4 = frt_doc_get_field(doc, compressed_field_lz4);
1810
1825
  for (i = 0; i < 4; i++) {
1811
1826
  Asequal(df1->data[i], df2->data[i]);
1827
+ Asequal(df1->data[i], df3->data[i]);
1828
+ Asequal(df1->data[i], df4->data[i]);
1812
1829
  Assert(df1->lengths[i] == df2->lengths[i], "Field lengths not equal");
1830
+ Assert(df1->lengths[i] == df3->lengths[i], "Field lengths not equal");
1831
+ Assert(df1->lengths[i] == df4->lengths[i], "Field lengths not equal");
1813
1832
  }
1814
- frt_doc_destroy(doc);
1833
+ frt_doc_destroy(doc);
1815
1834
 
1816
1835
  lz_doc = ir->get_lazy_doc(ir, 0);
1817
1836
  lz_df1 = frt_lazy_doc_get(lz_doc, changing_field);
1818
- lz_df2 = frt_lazy_doc_get(lz_doc, compressed_field);
1837
+ lz_df2 = frt_lazy_doc_get(lz_doc, compressed_field_brotli);
1838
+ lz_df3 = frt_lazy_doc_get(lz_doc, compressed_field_bzip2);
1839
+ lz_df4 = frt_lazy_doc_get(lz_doc, compressed_field_lz4);
1819
1840
  Asequal(frt_lazy_df_get_data(lz_df1, 0), frt_lazy_df_get_data(lz_df2, 0));
1841
+ Asequal(frt_lazy_df_get_data(lz_df1, 0), frt_lazy_df_get_data(lz_df3, 0));
1842
+ Asequal(frt_lazy_df_get_data(lz_df1, 0), frt_lazy_df_get_data(lz_df4, 0));
1820
1843
  frt_lazy_doc_close(lz_doc);
1821
1844
 
1822
1845
  lz_doc = ir->get_lazy_doc(ir, 2);
1823
1846
  lz_df1 = frt_lazy_doc_get(lz_doc, tag);
1824
- lz_df2 = frt_lazy_doc_get(lz_doc, compressed_field);
1847
+ lz_df2 = frt_lazy_doc_get(lz_doc, compressed_field_brotli);
1848
+ lz_df3 = frt_lazy_doc_get(lz_doc, compressed_field_bzip2);
1849
+ lz_df4 = frt_lazy_doc_get(lz_doc, compressed_field_lz4);
1825
1850
  for (i = 0; i < 4; i++) {
1826
1851
  Asequal(frt_lazy_df_get_data(lz_df1, i), frt_lazy_df_get_data(lz_df2, i));
1852
+ Asequal(frt_lazy_df_get_data(lz_df1, i), frt_lazy_df_get_data(lz_df3, i));
1853
+ Asequal(frt_lazy_df_get_data(lz_df1, i), frt_lazy_df_get_data(lz_df4, i));
1827
1854
  }
1828
1855
  frt_lazy_doc_close(lz_doc);
1829
1856
 
1830
1857
  lz_doc = ir->get_lazy_doc(ir, 2);
1831
1858
  lz_df1 = frt_lazy_doc_get(lz_doc, tag);
1832
- lz_df2 = frt_lazy_doc_get(lz_doc, compressed_field);
1859
+ lz_df2 = frt_lazy_doc_get(lz_doc, compressed_field_brotli);
1860
+ lz_df3 = frt_lazy_doc_get(lz_doc, compressed_field_bzip2);
1861
+ lz_df4 = frt_lazy_doc_get(lz_doc, compressed_field_lz4);
1833
1862
  frt_lazy_df_get_bytes(lz_df1, buf1, 5, 11);
1834
1863
  frt_lazy_df_get_bytes(lz_df2, buf2, 5, 11);
1835
1864
  buf2[11] = buf1[11] = '\0';
1836
1865
  Asequal(buf1, buf2);
1866
+ frt_lazy_df_get_bytes(lz_df3, buf2, 5, 11);
1867
+ buf2[11] = buf1[11] = '\0';
1868
+ Asequal(buf1, buf2);
1869
+ frt_lazy_df_get_bytes(lz_df4, buf2, 5, 11);
1870
+ buf2[11] = buf1[11] = '\0';
1871
+ Asequal(buf1, buf2);
1837
1872
  frt_lazy_doc_close(lz_doc);
1838
1873
  }
1839
1874
 
@@ -1923,7 +1958,7 @@ static void test_ir_norms(TestCase *tc, void *data)
1923
1958
  frt_ir_commit(ir);
1924
1959
 
1925
1960
  for (i = 0; i < rte->store_cnt; i++) {
1926
- iw = frt_iw_open(rte->stores[i], frt_whitespace_analyzer_new(false),
1961
+ iw = frt_iw_open(NULL, rte->stores[i], frt_whitespace_analyzer_new(false),
1927
1962
  &frt_default_config);
1928
1963
  frt_iw_optimize(iw);
1929
1964
  frt_iw_close(iw);
@@ -1958,7 +1993,7 @@ static void test_ir_norms(TestCase *tc, void *data)
1958
1993
  static void test_ir_delete(TestCase *tc, void *data)
1959
1994
  {
1960
1995
  int i;
1961
- FrtStore *store = frt_open_ram_store();
1996
+ FrtStore *store = frt_open_ram_store(NULL);
1962
1997
  FrtIndexReader *ir, *ir2;
1963
1998
  FrtIndexWriter *iw;
1964
1999
  int type = *((int *)data);
@@ -2050,8 +2085,7 @@ static void test_ir_delete(TestCase *tc, void *data)
2050
2085
  frt_ir_commit(ir);
2051
2086
 
2052
2087
  for (i = 0; i < rte->store_cnt; i++) {
2053
- iw = frt_iw_open(rte->stores[i], frt_whitespace_analyzer_new(false),
2054
- &frt_default_config);
2088
+ iw = frt_iw_open(NULL, rte->stores[i], frt_whitespace_analyzer_new(false), &frt_default_config);
2055
2089
  frt_iw_optimize(iw);
2056
2090
  frt_iw_close(iw);
2057
2091
  }
@@ -2080,11 +2114,11 @@ static void test_ir_read_while_optimizing(TestCase *tc, void *data)
2080
2114
 
2081
2115
  write_ir_test_docs(store);
2082
2116
 
2083
- ir = frt_ir_open(store);
2117
+ ir = frt_ir_open(NULL, store);
2084
2118
 
2085
2119
  test_ir_term_doc_enum(tc, ir);
2086
2120
 
2087
- iw = frt_iw_open(store, frt_whitespace_analyzer_new(false), false);
2121
+ iw = frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), false);
2088
2122
  frt_iw_optimize(iw);
2089
2123
  frt_iw_close(iw);
2090
2124
 
@@ -2101,23 +2135,23 @@ static void test_ir_multivalue_fields(TestCase *tc, void *data)
2101
2135
  FrtDocument *doc = frt_doc_new();
2102
2136
  FrtDocField *df;
2103
2137
  FrtIndexWriter *iw;
2104
- FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_INDEX_YES,
2105
- FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
2138
+ FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_COMPRESSION_NONE, FRT_INDEX_YES, FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
2106
2139
  const char *body_text = "this is the body FrtDocument Field";
2107
2140
  const char *title_text = "this is the title FrtDocument Field";
2108
2141
  const char *author_text = "this is the author FrtDocument Field";
2142
+ rb_encoding *enc = rb_enc_find("ASCII-8BIT");
2109
2143
 
2110
2144
  frt_index_create(store, fis);
2111
2145
  frt_fis_deref(fis);
2112
- iw = frt_iw_open(store, frt_whitespace_analyzer_new(false), NULL);
2146
+ iw = frt_iw_open(NULL, store, frt_whitespace_analyzer_new(false), NULL);
2113
2147
 
2114
- df = frt_doc_add_field(doc, frt_df_add_data(frt_df_new(tag), (char *)"Ruby"));
2115
- frt_df_add_data(df, (char *)"C");
2116
- frt_doc_add_field(doc, frt_df_add_data(frt_df_new(body), (char *)body_text));
2117
- frt_df_add_data(df, (char *)"Lucene");
2118
- frt_doc_add_field(doc, frt_df_add_data(frt_df_new(title), (char *)title_text));
2119
- frt_df_add_data(df, (char *)"Ferret");
2120
- frt_doc_add_field(doc, frt_df_add_data(frt_df_new(author), (char *)author_text));
2148
+ df = frt_doc_add_field(doc, frt_df_add_data(frt_df_new(tag), (char *)"Ruby", enc));
2149
+ frt_df_add_data(df, (char *)"C", enc);
2150
+ frt_doc_add_field(doc, frt_df_add_data(frt_df_new(body), (char *)body_text, enc));
2151
+ frt_df_add_data(df, (char *)"Lucene", enc);
2152
+ frt_doc_add_field(doc, frt_df_add_data(frt_df_new(title), (char *)title_text, enc));
2153
+ frt_df_add_data(df, (char *)"Ferret", enc);
2154
+ frt_doc_add_field(doc, frt_df_add_data(frt_df_new(author), (char *)author_text, enc));
2121
2155
 
2122
2156
  Aiequal(0, iw->fis->size);
2123
2157
 
@@ -2136,7 +2170,7 @@ static void test_ir_multivalue_fields(TestCase *tc, void *data)
2136
2170
  frt_doc_destroy(doc);
2137
2171
  frt_iw_close(iw);
2138
2172
 
2139
- ir = frt_ir_open(store);
2173
+ ir = frt_ir_open(NULL, store);
2140
2174
 
2141
2175
  doc = ir->get_doc(ir, 0);
2142
2176
  Aiequal(4, doc->size);
@@ -2172,18 +2206,20 @@ static void test_ir_multivalue_fields(TestCase *tc, void *data)
2172
2206
  TestSuite *ts_index(TestSuite *suite)
2173
2207
  {
2174
2208
  FrtIndexReader *ir;
2175
- FrtStore *fs_store, *store = frt_open_ram_store();
2209
+ FrtStore *fs_store, *store = frt_open_ram_store(NULL);
2176
2210
  ReaderTestEnvironment *rte = NULL;
2177
2211
  /* FrtStore *store = frt_open_fs_store(TEST_DIR); */
2178
2212
 
2179
- /* initialize FrtSymbols */
2213
+ /* initialize IDs */
2180
2214
  body = rb_intern("body");
2181
2215
  title = rb_intern("title");
2182
2216
  text = rb_intern("text");
2183
2217
  author = rb_intern("author");
2184
2218
  year = rb_intern("year");
2185
2219
  changing_field = rb_intern("changing_field");
2186
- compressed_field = rb_intern("compressed_field");
2220
+ compressed_field_brotli = rb_intern("compressed_field_brotli");
2221
+ compressed_field_bzip2 = rb_intern("compressed_field_bzip2");
2222
+ compressed_field_lz4 = rb_intern("compressed_field_lz4");
2187
2223
  tag = rb_intern("tag");
2188
2224
 
2189
2225
  srand(5);
@@ -2233,25 +2269,16 @@ TestSuite *ts_index(TestSuite *suite)
2233
2269
  rte = reader_test_env_new(multi_reader_type);
2234
2270
  ir = reader_test_env_ir_open(rte);
2235
2271
 
2236
- tst_run_test_with_name(suite, test_ir_basic_ops, ir,
2237
- "test_multi_reader_basic_ops");
2238
- tst_run_test_with_name(suite, test_ir_get_doc, ir,
2239
- "test_multi_get_doc");
2240
- tst_run_test_with_name(suite, test_ir_compression, ir,
2241
- "test_multi_compression");
2242
- tst_run_test_with_name(suite, test_ir_term_enum, ir,
2243
- "test_multi_term_enum");
2244
- tst_run_test_with_name(suite, test_ir_term_doc_enum, ir,
2245
- "test_multi_term_doc_enum");
2246
- tst_run_test_with_name(suite, test_ir_term_vectors, ir,
2247
- "test_multi_term_vectors");
2248
- tst_run_test_with_name(suite, test_ir_mtdpe, ir,
2249
- "test_multi_multiple_term_doc_pos_enum");
2250
-
2251
- tst_run_test_with_name(suite, test_ir_norms, &multi_reader_type,
2252
- "test_multi_norms");
2253
- tst_run_test_with_name(suite, test_ir_delete, &multi_reader_type,
2254
- "test_multi_reader_delete");
2272
+ tst_run_test_with_name(suite, test_ir_basic_ops, ir, "test_multi_reader_basic_ops");
2273
+ tst_run_test_with_name(suite, test_ir_get_doc, ir, "test_multi_get_doc");
2274
+ tst_run_test_with_name(suite, test_ir_compression, ir, "test_multi_compression");
2275
+ tst_run_test_with_name(suite, test_ir_term_enum, ir, "test_multi_term_enum");
2276
+ tst_run_test_with_name(suite, test_ir_term_doc_enum, ir, "test_multi_term_doc_enum");
2277
+ tst_run_test_with_name(suite, test_ir_term_vectors, ir, "test_multi_term_vectors");
2278
+ tst_run_test_with_name(suite, test_ir_mtdpe, ir, "test_multi_multiple_term_doc_pos_enum");
2279
+
2280
+ tst_run_test_with_name(suite, test_ir_norms, &multi_reader_type, "test_multi_norms");
2281
+ tst_run_test_with_name(suite, test_ir_delete, &multi_reader_type, "test_multi_reader_delete");
2255
2282
  frt_ir_close(ir);
2256
2283
  reader_test_env_destroy(rte);
2257
2284
 
@@ -2259,25 +2286,16 @@ TestSuite *ts_index(TestSuite *suite)
2259
2286
  rte = reader_test_env_new(multi_external_reader_type);
2260
2287
  ir = reader_test_env_ir_open(rte);
2261
2288
 
2262
- tst_run_test_with_name(suite, test_ir_basic_ops, ir,
2263
- "test_multi_ext_reader_basic_ops");
2264
- tst_run_test_with_name(suite, test_ir_get_doc, ir,
2265
- "test_multi_ext_get_doc");
2266
- tst_run_test_with_name(suite, test_ir_compression, ir,
2267
- "test_multi_ext_compression");
2268
- tst_run_test_with_name(suite, test_ir_term_enum, ir,
2269
- "test_multi_ext_term_enum");
2270
- tst_run_test_with_name(suite, test_ir_term_doc_enum, ir,
2271
- "test_multi_ext_term_doc_enum");
2272
- tst_run_test_with_name(suite, test_ir_term_vectors, ir,
2273
- "test_multi_ext_term_vectors");
2274
- tst_run_test_with_name(suite, test_ir_mtdpe, ir,
2275
- "test_multi_ext_multiple_term_doc_pos_enum");
2276
-
2277
- tst_run_test_with_name(suite, test_ir_norms, &multi_external_reader_type,
2278
- "test_multi_ext_norms");
2279
- tst_run_test_with_name(suite, test_ir_delete, &multi_external_reader_type,
2280
- "test_multi_ext_reader_delete");
2289
+ tst_run_test_with_name(suite, test_ir_basic_ops, ir, "test_multi_ext_reader_basic_ops");
2290
+ tst_run_test_with_name(suite, test_ir_get_doc, ir, "test_multi_ext_get_doc");
2291
+ tst_run_test_with_name(suite, test_ir_compression, ir, "test_multi_ext_compression");
2292
+ tst_run_test_with_name(suite, test_ir_term_enum, ir, "test_multi_ext_term_enum");
2293
+ tst_run_test_with_name(suite, test_ir_term_doc_enum, ir, "test_multi_ext_term_doc_enum");
2294
+ tst_run_test_with_name(suite, test_ir_term_vectors, ir, "test_multi_ext_term_vectors");
2295
+ tst_run_test_with_name(suite, test_ir_mtdpe, ir, "test_multi_ext_multiple_term_doc_pos_enum");
2296
+
2297
+ tst_run_test_with_name(suite, test_ir_norms, &multi_external_reader_type, "test_multi_ext_norms");
2298
+ tst_run_test_with_name(suite, test_ir_delete, &multi_external_reader_type, "test_multi_ext_reader_delete");
2281
2299
 
2282
2300
  frt_ir_close(ir);
2283
2301
  reader_test_env_destroy(rte);
@@ -2286,36 +2304,25 @@ TestSuite *ts_index(TestSuite *suite)
2286
2304
  rte = reader_test_env_new(add_indexes_reader_type);
2287
2305
  ir = reader_test_env_ir_open(rte);
2288
2306
 
2289
- tst_run_test_with_name(suite, test_ir_basic_ops, ir,
2290
- "test_add_indexes_reader_basic_ops");
2291
- tst_run_test_with_name(suite, test_ir_get_doc, ir,
2292
- "test_add_indexes_get_doc");
2293
- tst_run_test_with_name(suite, test_ir_compression, ir,
2294
- "test_add_indexes_compression");
2295
- tst_run_test_with_name(suite, test_ir_term_enum, ir,
2296
- "test_add_indexes_term_enum");
2297
- tst_run_test_with_name(suite, test_ir_term_doc_enum, ir,
2298
- "test_add_indexes_term_doc_enum");
2299
- tst_run_test_with_name(suite, test_ir_term_vectors, ir,
2300
- "test_add_indexes_term_vectors");
2301
- tst_run_test_with_name(suite, test_ir_mtdpe, ir,
2302
- "test_add_indexes_multiple_term_doc_pos_enum");
2303
-
2304
- tst_run_test_with_name(suite, test_ir_norms, &add_indexes_reader_type,
2305
- "test_add_indexes_norms");
2306
- tst_run_test_with_name(suite, test_ir_delete, &add_indexes_reader_type,
2307
- "test_add_indexes_reader_delete");
2307
+ tst_run_test_with_name(suite, test_ir_basic_ops, ir, "test_add_indexes_reader_basic_ops");
2308
+ tst_run_test_with_name(suite, test_ir_get_doc, ir, "test_add_indexes_get_doc");
2309
+ tst_run_test_with_name(suite, test_ir_compression, ir, "test_add_indexes_compression");
2310
+ tst_run_test_with_name(suite, test_ir_term_enum, ir, "test_add_indexes_term_enum");
2311
+ tst_run_test_with_name(suite, test_ir_term_doc_enum, ir, "test_add_indexes_term_doc_enum");
2312
+ tst_run_test_with_name(suite, test_ir_term_vectors, ir, "test_add_indexes_term_vectors");
2313
+ tst_run_test_with_name(suite, test_ir_mtdpe, ir, "test_add_indexes_multiple_term_doc_pos_enum");
2314
+
2315
+ tst_run_test_with_name(suite, test_ir_norms, &add_indexes_reader_type, "test_add_indexes_norms");
2316
+ tst_run_test_with_name(suite, test_ir_delete, &add_indexes_reader_type, "test_add_indexes_reader_delete");
2308
2317
 
2309
2318
  frt_ir_close(ir);
2310
2319
  reader_test_env_destroy(rte);
2311
2320
 
2312
2321
  /* Other FrtIndexReader Tests */
2313
- tst_run_test_with_name(suite, test_ir_read_while_optimizing, store,
2314
- "test_ir_read_while_optimizing_in_ram");
2322
+ tst_run_test_with_name(suite, test_ir_read_while_optimizing, store, "test_ir_read_while_optimizing_in_ram");
2315
2323
 
2316
2324
  fs_store = frt_open_fs_store(TEST_DIR);
2317
- tst_run_test_with_name(suite, test_ir_read_while_optimizing, fs_store,
2318
- "test_ir_read_while_optimizing_on_disk");
2325
+ tst_run_test_with_name(suite, test_ir_read_while_optimizing, fs_store, "test_ir_read_while_optimizing_on_disk");
2319
2326
  fs_store->clear_all(fs_store);
2320
2327
  frt_store_deref(fs_store);
2321
2328