isomorfeus-ferret 0.12.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (222) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +612 -0
  3. data/README.md +44 -0
  4. data/ext/isomorfeus_ferret_ext/benchmark.c +223 -0
  5. data/ext/isomorfeus_ferret_ext/benchmark.h +45 -0
  6. data/ext/isomorfeus_ferret_ext/benchmarks_all.h +25 -0
  7. data/ext/isomorfeus_ferret_ext/bm_bitvector.c +123 -0
  8. data/ext/isomorfeus_ferret_ext/bm_hash.c +118 -0
  9. data/ext/isomorfeus_ferret_ext/bm_micro_string.c +40 -0
  10. data/ext/isomorfeus_ferret_ext/bm_store.c +93 -0
  11. data/ext/isomorfeus_ferret_ext/email.rl +21 -0
  12. data/ext/isomorfeus_ferret_ext/extconf.rb +5 -0
  13. data/ext/isomorfeus_ferret_ext/fio_tmpfile.h +53 -0
  14. data/ext/isomorfeus_ferret_ext/frb_analysis.c +2577 -0
  15. data/ext/isomorfeus_ferret_ext/frb_index.c +3457 -0
  16. data/ext/isomorfeus_ferret_ext/frb_lang.c +9 -0
  17. data/ext/isomorfeus_ferret_ext/frb_lang.h +17 -0
  18. data/ext/isomorfeus_ferret_ext/frb_qparser.c +629 -0
  19. data/ext/isomorfeus_ferret_ext/frb_search.c +4460 -0
  20. data/ext/isomorfeus_ferret_ext/frb_store.c +515 -0
  21. data/ext/isomorfeus_ferret_ext/frb_threading.h +30 -0
  22. data/ext/isomorfeus_ferret_ext/frb_utils.c +1127 -0
  23. data/ext/isomorfeus_ferret_ext/frt_analysis.c +1644 -0
  24. data/ext/isomorfeus_ferret_ext/frt_analysis.h +247 -0
  25. data/ext/isomorfeus_ferret_ext/frt_array.c +124 -0
  26. data/ext/isomorfeus_ferret_ext/frt_array.h +54 -0
  27. data/ext/isomorfeus_ferret_ext/frt_bitvector.c +95 -0
  28. data/ext/isomorfeus_ferret_ext/frt_bitvector.h +586 -0
  29. data/ext/isomorfeus_ferret_ext/frt_compound_io.c +374 -0
  30. data/ext/isomorfeus_ferret_ext/frt_config.h +44 -0
  31. data/ext/isomorfeus_ferret_ext/frt_document.c +134 -0
  32. data/ext/isomorfeus_ferret_ext/frt_document.h +52 -0
  33. data/ext/isomorfeus_ferret_ext/frt_except.c +95 -0
  34. data/ext/isomorfeus_ferret_ext/frt_except.h +188 -0
  35. data/ext/isomorfeus_ferret_ext/frt_field_index.c +233 -0
  36. data/ext/isomorfeus_ferret_ext/frt_field_index.h +42 -0
  37. data/ext/isomorfeus_ferret_ext/frt_filter.c +157 -0
  38. data/ext/isomorfeus_ferret_ext/frt_fs_store.c +502 -0
  39. data/ext/isomorfeus_ferret_ext/frt_global.c +427 -0
  40. data/ext/isomorfeus_ferret_ext/frt_global.h +290 -0
  41. data/ext/isomorfeus_ferret_ext/frt_hash.c +518 -0
  42. data/ext/isomorfeus_ferret_ext/frt_hash.h +466 -0
  43. data/ext/isomorfeus_ferret_ext/frt_hashset.c +191 -0
  44. data/ext/isomorfeus_ferret_ext/frt_hashset.h +206 -0
  45. data/ext/isomorfeus_ferret_ext/frt_helper.c +62 -0
  46. data/ext/isomorfeus_ferret_ext/frt_helper.h +13 -0
  47. data/ext/isomorfeus_ferret_ext/frt_ind.c +353 -0
  48. data/ext/isomorfeus_ferret_ext/frt_ind.h +54 -0
  49. data/ext/isomorfeus_ferret_ext/frt_index.c +6377 -0
  50. data/ext/isomorfeus_ferret_ext/frt_index.h +880 -0
  51. data/ext/isomorfeus_ferret_ext/frt_lang.c +104 -0
  52. data/ext/isomorfeus_ferret_ext/frt_lang.h +44 -0
  53. data/ext/isomorfeus_ferret_ext/frt_mempool.c +87 -0
  54. data/ext/isomorfeus_ferret_ext/frt_mempool.h +33 -0
  55. data/ext/isomorfeus_ferret_ext/frt_multimapper.c +349 -0
  56. data/ext/isomorfeus_ferret_ext/frt_multimapper.h +52 -0
  57. data/ext/isomorfeus_ferret_ext/frt_posh.c +1006 -0
  58. data/ext/isomorfeus_ferret_ext/frt_posh.h +973 -0
  59. data/ext/isomorfeus_ferret_ext/frt_priorityqueue.c +147 -0
  60. data/ext/isomorfeus_ferret_ext/frt_priorityqueue.h +147 -0
  61. data/ext/isomorfeus_ferret_ext/frt_q_boolean.c +1612 -0
  62. data/ext/isomorfeus_ferret_ext/frt_q_const_score.c +157 -0
  63. data/ext/isomorfeus_ferret_ext/frt_q_filtered_query.c +209 -0
  64. data/ext/isomorfeus_ferret_ext/frt_q_fuzzy.c +281 -0
  65. data/ext/isomorfeus_ferret_ext/frt_q_match_all.c +147 -0
  66. data/ext/isomorfeus_ferret_ext/frt_q_multi_term.c +672 -0
  67. data/ext/isomorfeus_ferret_ext/frt_q_parser.c +3084 -0
  68. data/ext/isomorfeus_ferret_ext/frt_q_phrase.c +1182 -0
  69. data/ext/isomorfeus_ferret_ext/frt_q_prefix.c +98 -0
  70. data/ext/isomorfeus_ferret_ext/frt_q_range.c +665 -0
  71. data/ext/isomorfeus_ferret_ext/frt_q_span.c +2386 -0
  72. data/ext/isomorfeus_ferret_ext/frt_q_term.c +311 -0
  73. data/ext/isomorfeus_ferret_ext/frt_q_wildcard.c +166 -0
  74. data/ext/isomorfeus_ferret_ext/frt_ram_store.c +460 -0
  75. data/ext/isomorfeus_ferret_ext/frt_scanner.c +899 -0
  76. data/ext/isomorfeus_ferret_ext/frt_scanner.h +28 -0
  77. data/ext/isomorfeus_ferret_ext/frt_scanner_mb.c +6705 -0
  78. data/ext/isomorfeus_ferret_ext/frt_scanner_utf8.c +4419 -0
  79. data/ext/isomorfeus_ferret_ext/frt_search.c +1824 -0
  80. data/ext/isomorfeus_ferret_ext/frt_search.h +924 -0
  81. data/ext/isomorfeus_ferret_ext/frt_similarity.c +150 -0
  82. data/ext/isomorfeus_ferret_ext/frt_similarity.h +79 -0
  83. data/ext/isomorfeus_ferret_ext/frt_sort.c +796 -0
  84. data/ext/isomorfeus_ferret_ext/frt_stopwords.c +395 -0
  85. data/ext/isomorfeus_ferret_ext/frt_store.c +680 -0
  86. data/ext/isomorfeus_ferret_ext/frt_store.h +789 -0
  87. data/ext/isomorfeus_ferret_ext/frt_term_vectors.c +72 -0
  88. data/ext/isomorfeus_ferret_ext/frt_threading.h +23 -0
  89. data/ext/isomorfeus_ferret_ext/frt_win32.h +54 -0
  90. data/ext/isomorfeus_ferret_ext/isomorfeus_ferret.c +409 -0
  91. data/ext/isomorfeus_ferret_ext/isomorfeus_ferret.h +95 -0
  92. data/ext/isomorfeus_ferret_ext/libstemmer.c +93 -0
  93. data/ext/isomorfeus_ferret_ext/libstemmer.h +73 -0
  94. data/ext/isomorfeus_ferret_ext/q_parser.y +1366 -0
  95. data/ext/isomorfeus_ferret_ext/scanner.h +28 -0
  96. data/ext/isomorfeus_ferret_ext/scanner.in +43 -0
  97. data/ext/isomorfeus_ferret_ext/scanner.rl +84 -0
  98. data/ext/isomorfeus_ferret_ext/scanner_mb.rl +200 -0
  99. data/ext/isomorfeus_ferret_ext/scanner_utf8.rl +85 -0
  100. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_danish.c +324 -0
  101. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_danish.h +7 -0
  102. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_dutch.c +610 -0
  103. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_dutch.h +6 -0
  104. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_english.c +1104 -0
  105. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_english.h +6 -0
  106. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_finnish.c +749 -0
  107. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_finnish.h +7 -0
  108. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_french.c +1233 -0
  109. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_french.h +6 -0
  110. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_german.c +490 -0
  111. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_german.h +6 -0
  112. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_hungarian.c +1217 -0
  113. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_hungarian.h +7 -0
  114. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_italian.c +1052 -0
  115. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_italian.h +6 -0
  116. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_norwegian.c +283 -0
  117. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_norwegian.h +6 -0
  118. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_porter.c +735 -0
  119. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_porter.h +6 -0
  120. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_portuguese.c +1003 -0
  121. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_portuguese.h +7 -0
  122. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_spanish.c +1079 -0
  123. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_spanish.h +6 -0
  124. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_swedish.c +293 -0
  125. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_swedish.h +6 -0
  126. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_2_romanian.c +984 -0
  127. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_2_romanian.h +6 -0
  128. data/ext/isomorfeus_ferret_ext/stem_KOI8_R_russian.c +686 -0
  129. data/ext/isomorfeus_ferret_ext/stem_KOI8_R_russian.h +6 -0
  130. data/ext/isomorfeus_ferret_ext/stem_UTF_8_danish.c +325 -0
  131. data/ext/isomorfeus_ferret_ext/stem_UTF_8_danish.h +6 -0
  132. data/ext/isomorfeus_ferret_ext/stem_UTF_8_dutch.c +620 -0
  133. data/ext/isomorfeus_ferret_ext/stem_UTF_8_dutch.h +6 -0
  134. data/ext/isomorfeus_ferret_ext/stem_UTF_8_english.c +1111 -0
  135. data/ext/isomorfeus_ferret_ext/stem_UTF_8_english.h +6 -0
  136. data/ext/isomorfeus_ferret_ext/stem_UTF_8_finnish.c +754 -0
  137. data/ext/isomorfeus_ferret_ext/stem_UTF_8_finnish.h +6 -0
  138. data/ext/isomorfeus_ferret_ext/stem_UTF_8_french.c +1242 -0
  139. data/ext/isomorfeus_ferret_ext/stem_UTF_8_french.h +6 -0
  140. data/ext/isomorfeus_ferret_ext/stem_UTF_8_german.c +495 -0
  141. data/ext/isomorfeus_ferret_ext/stem_UTF_8_german.h +6 -0
  142. data/ext/isomorfeus_ferret_ext/stem_UTF_8_hungarian.c +1220 -0
  143. data/ext/isomorfeus_ferret_ext/stem_UTF_8_hungarian.h +6 -0
  144. data/ext/isomorfeus_ferret_ext/stem_UTF_8_italian.c +1059 -0
  145. data/ext/isomorfeus_ferret_ext/stem_UTF_8_italian.h +6 -0
  146. data/ext/isomorfeus_ferret_ext/stem_UTF_8_norwegian.c +285 -0
  147. data/ext/isomorfeus_ferret_ext/stem_UTF_8_norwegian.h +6 -0
  148. data/ext/isomorfeus_ferret_ext/stem_UTF_8_porter.c +741 -0
  149. data/ext/isomorfeus_ferret_ext/stem_UTF_8_porter.h +6 -0
  150. data/ext/isomorfeus_ferret_ext/stem_UTF_8_portuguese.c +1009 -0
  151. data/ext/isomorfeus_ferret_ext/stem_UTF_8_portuguese.h +6 -0
  152. data/ext/isomorfeus_ferret_ext/stem_UTF_8_romanian.c +990 -0
  153. data/ext/isomorfeus_ferret_ext/stem_UTF_8_romanian.h +6 -0
  154. data/ext/isomorfeus_ferret_ext/stem_UTF_8_russian.c +680 -0
  155. data/ext/isomorfeus_ferret_ext/stem_UTF_8_russian.h +6 -0
  156. data/ext/isomorfeus_ferret_ext/stem_UTF_8_spanish.c +1083 -0
  157. data/ext/isomorfeus_ferret_ext/stem_UTF_8_spanish.h +6 -0
  158. data/ext/isomorfeus_ferret_ext/stem_UTF_8_swedish.c +294 -0
  159. data/ext/isomorfeus_ferret_ext/stem_UTF_8_swedish.h +6 -0
  160. data/ext/isomorfeus_ferret_ext/stem_UTF_8_turkish.c +2191 -0
  161. data/ext/isomorfeus_ferret_ext/stem_UTF_8_turkish.h +6 -0
  162. data/ext/isomorfeus_ferret_ext/stem_api.c +66 -0
  163. data/ext/isomorfeus_ferret_ext/stem_api.h +26 -0
  164. data/ext/isomorfeus_ferret_ext/stem_header.h +57 -0
  165. data/ext/isomorfeus_ferret_ext/stem_modules.h +190 -0
  166. data/ext/isomorfeus_ferret_ext/stem_modules.txt +50 -0
  167. data/ext/isomorfeus_ferret_ext/stem_utilities.c +478 -0
  168. data/ext/isomorfeus_ferret_ext/test.c +850 -0
  169. data/ext/isomorfeus_ferret_ext/test.h +416 -0
  170. data/ext/isomorfeus_ferret_ext/test_1710.c +63 -0
  171. data/ext/isomorfeus_ferret_ext/test_analysis.c +1221 -0
  172. data/ext/isomorfeus_ferret_ext/test_array.c +272 -0
  173. data/ext/isomorfeus_ferret_ext/test_bitvector.c +600 -0
  174. data/ext/isomorfeus_ferret_ext/test_compound_io.c +170 -0
  175. data/ext/isomorfeus_ferret_ext/test_document.c +156 -0
  176. data/ext/isomorfeus_ferret_ext/test_except.c +244 -0
  177. data/ext/isomorfeus_ferret_ext/test_fields.c +522 -0
  178. data/ext/isomorfeus_ferret_ext/test_file_deleter.c +185 -0
  179. data/ext/isomorfeus_ferret_ext/test_filter.c +331 -0
  180. data/ext/isomorfeus_ferret_ext/test_fs_store.c +25 -0
  181. data/ext/isomorfeus_ferret_ext/test_global.c +299 -0
  182. data/ext/isomorfeus_ferret_ext/test_hash.c +485 -0
  183. data/ext/isomorfeus_ferret_ext/test_hashset.c +288 -0
  184. data/ext/isomorfeus_ferret_ext/test_helper.c +47 -0
  185. data/ext/isomorfeus_ferret_ext/test_highlighter.c +548 -0
  186. data/ext/isomorfeus_ferret_ext/test_index.c +2323 -0
  187. data/ext/isomorfeus_ferret_ext/test_lang.c +74 -0
  188. data/ext/isomorfeus_ferret_ext/test_mempool.c +102 -0
  189. data/ext/isomorfeus_ferret_ext/test_multimapper.c +64 -0
  190. data/ext/isomorfeus_ferret_ext/test_priorityqueue.c +213 -0
  191. data/ext/isomorfeus_ferret_ext/test_q_const_score.c +84 -0
  192. data/ext/isomorfeus_ferret_ext/test_q_filtered.c +61 -0
  193. data/ext/isomorfeus_ferret_ext/test_q_fuzzy.c +241 -0
  194. data/ext/isomorfeus_ferret_ext/test_q_parser.c +464 -0
  195. data/ext/isomorfeus_ferret_ext/test_q_span.c +575 -0
  196. data/ext/isomorfeus_ferret_ext/test_ram_store.c +77 -0
  197. data/ext/isomorfeus_ferret_ext/test_search.c +1874 -0
  198. data/ext/isomorfeus_ferret_ext/test_segments.c +167 -0
  199. data/ext/isomorfeus_ferret_ext/test_similarity.c +25 -0
  200. data/ext/isomorfeus_ferret_ext/test_sort.c +333 -0
  201. data/ext/isomorfeus_ferret_ext/test_store.c +591 -0
  202. data/ext/isomorfeus_ferret_ext/test_store.h +3 -0
  203. data/ext/isomorfeus_ferret_ext/test_term.c +351 -0
  204. data/ext/isomorfeus_ferret_ext/test_term_vectors.c +373 -0
  205. data/ext/isomorfeus_ferret_ext/test_test.c +83 -0
  206. data/ext/isomorfeus_ferret_ext/test_threading.c +188 -0
  207. data/ext/isomorfeus_ferret_ext/testhelper.c +561 -0
  208. data/ext/isomorfeus_ferret_ext/testhelper.h +25 -0
  209. data/ext/isomorfeus_ferret_ext/tests_all.h +87 -0
  210. data/ext/isomorfeus_ferret_ext/uchar-ucs4.rl +1854 -0
  211. data/ext/isomorfeus_ferret_ext/uchar-utf8.rl +1999 -0
  212. data/ext/isomorfeus_ferret_ext/url.rl +27 -0
  213. data/ext/isomorfeus_ferret_ext/word_list.h +15156 -0
  214. data/lib/isomorfeus/ferret/document.rb +132 -0
  215. data/lib/isomorfeus/ferret/field_symbol.rb +85 -0
  216. data/lib/isomorfeus/ferret/index/field_infos.rb +48 -0
  217. data/lib/isomorfeus/ferret/index/index.rb +970 -0
  218. data/lib/isomorfeus/ferret/monitor.rb +323 -0
  219. data/lib/isomorfeus/ferret/stdlib_patches.rb +151 -0
  220. data/lib/isomorfeus/ferret/version.rb +5 -0
  221. data/lib/isomorfeus-ferret.rb +8 -0
  222. metadata +307 -0
@@ -0,0 +1,167 @@
1
+ #include "frt_index.h"
2
+ #include "test.h"
3
+
4
+ #define T 1
5
+ #define F 0
6
+
7
+ /***************************************************************************
8
+ *
9
+ * SegmentInfo
10
+ *
11
+ ***************************************************************************/
12
+
13
+ #define Asi_equal(si1, si2)\
14
+ tst_si_eql(__LINE__, tc, si2, si1->name, si1->doc_cnt, si1->store)
15
+
16
+ #define Asi_has_vals(si, name, doc_cnt, store)\
17
+ tst_si_eql(__LINE__, tc, si, name, doc_cnt, store)
18
+ static FrtFieldInfos dummy_fis = {
19
+ FRT_STORE_NO, /* store */
20
+ FRT_INDEX_NO, /* index */
21
+ FRT_TERM_VECTOR_NO, /* term_vector */
22
+ 0, /* size */
23
+ 0, /* capa */
24
+ NULL, /* fields */
25
+ NULL, /* field_dict */
26
+ 1000000 /* ref_cnt so that it is never destroyed */
27
+ };
28
+
29
+ static int tst_si_eql(int line_num,
30
+ TestCase *tc,
31
+ FrtSegmentInfo *si,
32
+ const char *name,
33
+ int doc_cnt,
34
+ FrtStore *store)
35
+ {
36
+ if (tst_str_equal(line_num, tc, name, si->name) &&
37
+ tst_int_equal(line_num, tc, doc_cnt, si->doc_cnt) &&
38
+ tst_ptr_equal(line_num, tc, store, si->store)) {
39
+ return true;
40
+ }
41
+ else {
42
+ return false;
43
+ }
44
+
45
+ }
46
+
47
+ static void test_si(TestCase *tc, void *data)
48
+ {
49
+ FrtStore *store = (FrtStore *)data;
50
+ FrtSegmentInfo *si = frt_si_new(frt_estrdup("_1"), 10, store);
51
+ Asi_has_vals(si, "_1", 10, store);
52
+ store = frt_open_fs_store("./test/testdir/store");
53
+ si->name[1] = '2';
54
+ si->doc_cnt += 2;
55
+ si->store = store;
56
+ Asi_has_vals(si, "_2", 12, store);
57
+ Assert(!frt_si_has_separate_norms(si), "doesn't use compound file/have norms");
58
+ si->use_compound_file = true;
59
+ Assert(!frt_si_has_separate_norms(si), "doesn't have norms");
60
+ frt_si_advance_norm_gen(si, 3);
61
+ si->use_compound_file = false;
62
+ Assert(!frt_si_has_separate_norms(si), "doesn't use compound file");
63
+ si->use_compound_file = true;
64
+ Assert(!frt_si_has_separate_norms(si), "has norms in compound file");
65
+ frt_si_advance_norm_gen(si, 3);
66
+ Assert(frt_si_has_separate_norms(si), "has seperate norms");
67
+ frt_si_deref(si);
68
+ frt_store_deref(store);
69
+ }
70
+
71
+ /***************************************************************************
72
+ *
73
+ * SegmentInfos
74
+ *
75
+ ***************************************************************************/
76
+
77
+
78
+ void test_sis_add_del(TestCase *tc, void *data)
79
+ {
80
+ FrtStore *store = (FrtStore *)data;
81
+ FrtSegmentInfos *sis = frt_sis_new(&dummy_fis);
82
+ FrtSegmentInfo *seg0 = frt_sis_new_segment(sis, 123, store);
83
+ FrtSegmentInfo *seg1 = frt_sis_new_segment(sis, 98, store);
84
+ FrtSegmentInfo *seg2 = frt_sis_new_segment(sis, 3, store);
85
+ FrtSegmentInfo *seg3 = frt_sis_new_segment(sis, 87, store);
86
+ FrtSegmentInfo *seg4 = frt_sis_new_segment(sis, 12, store);
87
+
88
+ Asi_has_vals(seg0, "_0", 123, store);
89
+ Asi_has_vals(seg1, "_1", 98, store);
90
+ Asi_has_vals(seg2, "_2", 3, store);
91
+ Asi_has_vals(seg3, "_3", 87, store);
92
+ Asi_has_vals(seg4, "_4", 12, store);
93
+
94
+ Apequal(seg0, sis->segs[0]);
95
+ Apequal(seg1, sis->segs[1]);
96
+ Apequal(seg2, sis->segs[2]);
97
+ Apequal(seg3, sis->segs[3]);
98
+ Apequal(seg4, sis->segs[4]);
99
+
100
+ frt_sis_del_from_to(sis, 1, 4);
101
+ Apequal(seg0, sis->segs[0]);
102
+ Apequal(seg4, sis->segs[1]);
103
+ frt_sis_del_at(sis, 0);
104
+
105
+ Apequal(seg4, sis->segs[0]);
106
+ frt_sis_destroy(sis);
107
+ }
108
+
109
+ void test_sis_rw(TestCase *tc, void *data)
110
+ {
111
+ frt_u64 version;
112
+ FrtStore *store = (FrtStore *)data;
113
+ FrtSegmentInfos *sis = frt_sis_new(&dummy_fis);
114
+ FrtSegmentInfo *seg0 = frt_sis_new_segment(sis, 51, store);
115
+ FrtSegmentInfo *seg1 = frt_sis_new_segment(sis, 213, store);
116
+ FrtSegmentInfo *seg2 = frt_sis_new_segment(sis, 23, store);
117
+ FrtSegmentInfo *seg3 = frt_si_new(frt_estrdup("_3"), 9, store);
118
+ FrtSegmentInfos *sis2, *sis3;
119
+
120
+ FRT_TRY
121
+ frt_sis_read_current_version(store);
122
+ Afail("Should have failed trying to read index");
123
+ FRT_XCATCHALL
124
+ FRT_HANDLED();
125
+ FRT_XENDTRY
126
+
127
+ Aiequal(3, sis->size);
128
+ Apequal(seg0, sis->segs[0]);
129
+ Apequal(seg2, sis->segs[2]);
130
+ frt_sis_write(sis, store, NULL);
131
+ version = frt_sis_read_current_version(store);
132
+ Assert(store->exists(store, "segments_0"),
133
+ "segments file should have been created");
134
+ sis2 = frt_sis_read(store);
135
+ Aiequal(3, sis2->size);
136
+ Asi_equal(seg0, sis2->segs[0]);
137
+ Asi_equal(seg1, sis2->segs[1]);
138
+ Asi_equal(seg2, sis2->segs[2]);
139
+
140
+ frt_sis_add_si(sis2, seg3);
141
+ Aiequal(4, sis2->size);
142
+ frt_sis_write(sis2, store, NULL);
143
+ Aiequal(version + 1, frt_sis_read_current_version(store));
144
+ sis3 = frt_sis_read(store);
145
+ Aiequal(version + 1, sis3->version);
146
+ Aiequal(4, sis3->size);
147
+ Asi_equal(seg0, sis3->segs[0]);
148
+ Asi_equal(seg3, sis3->segs[3]);
149
+
150
+ frt_sis_destroy(sis);
151
+ frt_sis_destroy(sis2);
152
+ frt_sis_destroy(sis3);
153
+ }
154
+
155
+ TestSuite *ts_segments(TestSuite *suite)
156
+ {
157
+ FrtStore *store = frt_open_ram_store();
158
+
159
+ suite = ADD_SUITE(suite);
160
+
161
+ tst_run_test(suite, test_si, store);
162
+ tst_run_test(suite, test_sis_add_del, store);
163
+ tst_run_test(suite, test_sis_rw, store);
164
+
165
+ frt_store_deref(store);
166
+ return suite;
167
+ }
@@ -0,0 +1,25 @@
1
+ #include "frt_global.h"
2
+ #include "frt_similarity.h"
3
+ #include "test.h"
4
+
5
+ void test_default_similarity(TestCase *tc, void *data)
6
+ {
7
+ FrtSimilarity *dsim = frt_sim_create_default();
8
+ (void)data;
9
+
10
+ Afequal(1.0/4, frt_sim_length_norm(dsim, 0, 16));
11
+ Afequal(1.0/4, frt_sim_query_norm(dsim, 16));
12
+ Afequal(3.0, frt_sim_tf(dsim, 9));
13
+ Afequal(1.0/10, frt_sim_sloppy_freq(dsim, 9));
14
+ Afequal(1.0, frt_sim_idf(dsim, 9, 10));
15
+ Afequal(4.0, frt_sim_coord(dsim, 12, 3));
16
+ }
17
+
18
+ TestSuite *ts_similarity(TestSuite *suite)
19
+ {
20
+ suite = ADD_SUITE(suite);
21
+
22
+ tst_run_test(suite, test_default_similarity, NULL);
23
+
24
+ return suite;
25
+ }
@@ -0,0 +1,333 @@
1
+ #include "frt_global.h"
2
+ #include "testhelper.h"
3
+ #include "frt_search.h"
4
+ #include "test.h"
5
+
6
+ #define ARRAY_SIZE 20
7
+
8
+ static FrtSymbol search, string, integer, flt;
9
+
10
+ typedef struct SortTestData {
11
+ const char *search;
12
+ const char *string;
13
+ const char *integer;
14
+ const char *flt;
15
+ } SortTestData;
16
+
17
+ static void add_sort_test_data(SortTestData *std, FrtIndexWriter *iw)
18
+ {
19
+ FrtDocument *doc = frt_doc_new();
20
+ frt_doc_add_field(doc, frt_df_add_data(frt_df_new(search), (char *)std->search));
21
+ frt_doc_add_field(doc, frt_df_add_data(frt_df_new(string), (char *)std->string));
22
+ frt_doc_add_field(doc, frt_df_add_data(frt_df_new(integer), (char *)std->integer));
23
+ frt_doc_add_field(doc, frt_df_add_data(frt_df_new(flt), (char *)std->flt));
24
+
25
+ sscanf(std->flt, "%f", &doc->boost);
26
+
27
+ frt_iw_add_doc(iw, doc);
28
+ frt_doc_destroy(doc);
29
+ }
30
+
31
+ static SortTestData data[] = { /* len mod */
32
+ {"findall","a","6","0.01"}, /* 4 0 */
33
+ {"findall","c","5","0.1"}, /* 3 3 */
34
+ {"findall","e","2","0.001"}, /* 5 1 */
35
+ {"findall","g","1","1.0"}, /* 3 3 */
36
+ {"findall","i","3","0.0001"}, /* 6 2 */
37
+ {"findall","", "4","10.0"}, /* 4 0 */
38
+ {"findall","h","5","0.00001"}, /* 7 3 */
39
+ {"findall","f","2","100.0"}, /* 5 1 */
40
+ {"findall","d","3","1000.0"}, /* 6 2 */
41
+ {"findall","b","4","0.000001"} /* 8 0 */
42
+ };
43
+
44
+ static void sort_test_setup(FrtStore *store)
45
+ {
46
+ int i;
47
+ FrtIndexWriter *iw;
48
+ FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_INDEX_YES, FRT_TERM_VECTOR_YES);
49
+ frt_index_create(store, fis);
50
+ frt_fis_deref(fis);
51
+
52
+ iw = frt_iw_open(store, frt_whitespace_analyzer_new(false), NULL);
53
+
54
+ for (i = 0; i < FRT_NELEMS(data); i++) {
55
+ add_sort_test_data(&data[i], iw);
56
+ }
57
+ frt_iw_close(iw);
58
+ }
59
+
60
+ static void sort_multi_test_setup(FrtStore *store1, FrtStore *store2)
61
+ {
62
+ int i;
63
+ FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_INDEX_YES, FRT_TERM_VECTOR_YES);
64
+ FrtIndexWriter *iw;
65
+
66
+ frt_index_create(store1, fis);
67
+ frt_index_create(store2, fis);
68
+ frt_fis_deref(fis);
69
+
70
+ iw = frt_iw_open(store1, frt_whitespace_analyzer_new(false), NULL);
71
+
72
+ for (i = 0; i < FRT_NELEMS(data)/2; i++) {
73
+ add_sort_test_data(&data[i], iw);
74
+ }
75
+ frt_iw_close(iw);
76
+
77
+ iw = frt_iw_open(store2, frt_whitespace_analyzer_new(false), NULL);
78
+
79
+ for (i = FRT_NELEMS(data)/2; i < FRT_NELEMS(data); i++) {
80
+ add_sort_test_data(&data[i], iw);
81
+ }
82
+ frt_iw_close(iw);
83
+ }
84
+
85
+ #define R_START 3
86
+ #define R_END 6
87
+ static void do_test_top_docs(TestCase *tc, FrtSearcher *searcher, FrtQuery *query, const char *expected_hits, FrtSort *sort)
88
+ {
89
+ static int num_array[ARRAY_SIZE];
90
+ int i;
91
+ int total_hits = s2l(expected_hits, num_array);
92
+ FrtTopDocs *top_docs = frt_searcher_search(searcher, query, 0, total_hits, NULL, sort, NULL);
93
+ Aiequal(total_hits, top_docs->total_hits);
94
+ Aiequal(total_hits, top_docs->size);
95
+
96
+ for (i = 0; i < top_docs->size; i++) {
97
+ FrtHit *hit = top_docs->hits[i];
98
+ if (false && sort && searcher->doc_freq != frt_isea_doc_freq) {
99
+ FrtFieldDoc *fd = (FrtFieldDoc *)hit;
100
+ int j;
101
+ printf("%d == %d:%f ", num_array[i], hit->doc, hit->score);
102
+ for (j = 0; j < fd->size; j++) {
103
+ switch (fd->comparables[j].type) {
104
+ case FRT_SORT_TYPE_SCORE:
105
+ printf("sc:%f ", fd->comparables[j].val.f); break;
106
+ case FRT_SORT_TYPE_FLOAT:
107
+ printf("f:%f ", fd->comparables[j].val.f); break;
108
+ case FRT_SORT_TYPE_DOC:
109
+ printf("d:%ld ", fd->comparables[j].val.l); break;
110
+ case FRT_SORT_TYPE_INTEGER:
111
+ printf("i:%ld ", fd->comparables[j].val.l); break;
112
+ case FRT_SORT_TYPE_STRING:
113
+ printf("s:%s ", fd->comparables[j].val.s); break;
114
+ default:
115
+ printf("NA "); break;
116
+ }
117
+ }
118
+ printf("\n");
119
+ }
120
+ Aiequal(num_array[i], hit->doc);
121
+ }
122
+ frt_td_destroy(top_docs);
123
+
124
+ if (total_hits >= R_END) {
125
+ top_docs = frt_searcher_search(searcher, query, R_START, R_END - R_START, NULL, sort, NULL);
126
+ for (i = R_START; i < R_END; i++) {
127
+ FrtHit *hit = top_docs->hits[i - R_START];
128
+ Aiequal(num_array[i], hit->doc);
129
+ /*
130
+ printf("%d == %d\n", num_array[i], hit->doc);
131
+ */
132
+ }
133
+ frt_td_destroy(top_docs);
134
+ }
135
+ }
136
+
137
+ #define TEST_SF_TO_S(_str, _sf) \
138
+ do {\
139
+ FrtSortField *_sf_p = _sf;\
140
+ char *_field = frt_sort_field_to_s(_sf_p);\
141
+ Asequal(_str, _field);\
142
+ free(_field);\
143
+ frt_sort_field_destroy(_sf_p);\
144
+ } while (0)
145
+
146
+
147
+ static void test_sort_field_to_s(TestCase *tc, void *data)
148
+ {
149
+ (void)data;
150
+ TEST_SF_TO_S("<SCORE>", frt_sort_field_score_new(false));
151
+ TEST_SF_TO_S("<SCORE>!", frt_sort_field_score_new(true));
152
+ TEST_SF_TO_S("<DOC>", frt_sort_field_doc_new(false));
153
+ TEST_SF_TO_S("<DOC>!", frt_sort_field_doc_new(true));
154
+ TEST_SF_TO_S("date:<integer>", frt_sort_field_int_new(rb_intern("date"), false));
155
+ TEST_SF_TO_S("date:<integer>!", frt_sort_field_int_new(rb_intern("date"), true));
156
+ TEST_SF_TO_S("price:<float>", frt_sort_field_float_new(rb_intern("price"), false));
157
+ TEST_SF_TO_S("price:<float>!", frt_sort_field_float_new(rb_intern("price"), true));
158
+ TEST_SF_TO_S("content:<string>", frt_sort_field_string_new(rb_intern("content"), false));
159
+ TEST_SF_TO_S("content:<string>!", frt_sort_field_string_new(rb_intern("content"), true));
160
+ TEST_SF_TO_S("auto_field:<auto>", frt_sort_field_auto_new(rb_intern("auto_field"), false));
161
+ TEST_SF_TO_S("auto_field:<auto>!", frt_sort_field_auto_new(rb_intern("auto_field"), true));
162
+ }
163
+
164
+ #define TEST_SORT_TO_S(_expected_str, _sort) \
165
+ do {\
166
+ char *_str = frt_sort_to_s(_sort);\
167
+ Asequal(_expected_str, _str);\
168
+ free(_str);\
169
+ } while (0)
170
+
171
+ static void test_sort_to_s(TestCase *tc, void *data)
172
+ {
173
+ FrtSort *sort = frt_sort_new();
174
+ (void)data;
175
+
176
+ TEST_SORT_TO_S("Sort[]", sort);
177
+ frt_sort_add_sort_field(sort, frt_sort_field_score_new(false));
178
+ TEST_SORT_TO_S("Sort[<SCORE>]", sort);
179
+ frt_sort_add_sort_field(sort, frt_sort_field_doc_new(true));
180
+ TEST_SORT_TO_S("Sort[<SCORE>, <DOC>!]", sort);
181
+ frt_sort_add_sort_field(sort, frt_sort_field_int_new(rb_intern("date"), true));
182
+ TEST_SORT_TO_S("Sort[<SCORE>, <DOC>!, date:<integer>!]", sort);
183
+ frt_sort_add_sort_field(sort, frt_sort_field_float_new(rb_intern("price"), false));
184
+ TEST_SORT_TO_S("Sort[<SCORE>, <DOC>!, date:<integer>!, price:<float>]", sort);
185
+ frt_sort_add_sort_field(sort, frt_sort_field_string_new(rb_intern("content"), true));
186
+ TEST_SORT_TO_S("Sort[<SCORE>, <DOC>!, date:<integer>!, price:<float>, content:<string>!]", sort);
187
+ frt_sort_add_sort_field(sort, frt_sort_field_auto_new(rb_intern("auto_field"), false));
188
+ TEST_SORT_TO_S("Sort[<SCORE>, <DOC>!, date:<integer>!, price:<float>, content:<string>!, auto_field:<auto>]", sort);
189
+ frt_sort_clear(sort);
190
+ frt_sort_add_sort_field(sort, frt_sort_field_string_new(rb_intern("content"), true));
191
+ TEST_SORT_TO_S("Sort[content:<string>!]", sort);
192
+ frt_sort_add_sort_field(sort, frt_sort_field_auto_new(rb_intern("auto_field"), false));
193
+ TEST_SORT_TO_S("Sort[content:<string>!, auto_field:<auto>]", sort);
194
+ frt_sort_destroy(sort);
195
+ }
196
+
197
+ static bool do_byte_test = true;
198
+ static void test_sorts(TestCase *tc, void *data)
199
+ {
200
+ FrtSearcher *sea = (FrtSearcher *)data;
201
+ FrtQuery *q;
202
+ FrtSort *sort = NULL;
203
+
204
+ q = frt_tq_new(search, "findall");
205
+ do_test_top_docs(tc, sea, q, "8,7,5,3,1,0,2,4,6,9", NULL);
206
+
207
+ sort = frt_sort_new();
208
+
209
+ frt_sort_add_sort_field(sort, frt_sort_field_score_new(false));
210
+ do_test_top_docs(tc, sea, q, "8,7,5,3,1,0,2,4,6,9", sort);
211
+ sort->sort_fields[0]->reverse = true;
212
+ do_test_top_docs(tc, sea, q, "9,6,4,2,0,1,3,5,7,8", sort);
213
+ frt_sort_clear(sort);
214
+
215
+ frt_sort_add_sort_field(sort, frt_sort_field_doc_new(false));
216
+ do_test_top_docs(tc, sea, q, "0,1,2,3,4,5,6,7,8,9", sort);
217
+ sort->sort_fields[0]->reverse = true;
218
+ do_test_top_docs(tc, sea, q, "9,8,7,6,5,4,3,2,1,0", sort);
219
+ frt_sort_clear(sort);
220
+
221
+ frt_sort_add_sort_field(sort, frt_sort_field_int_new(integer, true));
222
+ do_test_top_docs(tc, sea, q, "0,1,6,5,9,4,8,2,7,3", sort);
223
+ frt_sort_add_sort_field(sort, frt_sort_field_score_new(false));
224
+ do_test_top_docs(tc, sea, q, "0,1,6,5,9,8,4,7,2,3", sort);
225
+ sort->size = 1; /* remove score sort_field */
226
+ sort->sort_fields[0]->reverse = false;
227
+ do_test_top_docs(tc, sea, q, "3,2,7,4,8,5,9,1,6,0", sort);
228
+ sort->size = 2; /* re-add score sort_field */
229
+ do_test_top_docs(tc, sea, q, "3,7,2,8,4,5,9,1,6,0", sort);
230
+ frt_sort_clear(sort);
231
+
232
+ if (do_byte_test) {
233
+ frt_sort_add_sort_field(sort, frt_sort_field_byte_new(integer, true));
234
+ do_test_top_docs(tc, sea, q, "0,1,6,5,9,4,8,2,7,3", sort);
235
+ frt_sort_add_sort_field(sort, frt_sort_field_score_new(false));
236
+ do_test_top_docs(tc, sea, q, "0,1,6,5,9,8,4,7,2,3", sort);
237
+ sort->size = 1; /* remove score sort_field */
238
+ sort->sort_fields[0]->reverse = false;
239
+ do_test_top_docs(tc, sea, q, "3,2,7,4,8,5,9,1,6,0", sort);
240
+ sort->size = 2; /* re-add score sort_field */
241
+ do_test_top_docs(tc, sea, q, "3,7,2,8,4,5,9,1,6,0", sort);
242
+ frt_sort_clear(sort);
243
+ }
244
+
245
+ frt_sort_add_sort_field(sort, frt_sort_field_float_new(flt, false));
246
+ do_test_top_docs(tc, sea, q, "9,6,4,2,0,1,3,5,7,8", sort);
247
+ sort->sort_fields[0]->reverse = true;
248
+ do_test_top_docs(tc, sea, q, "8,7,5,3,1,0,2,4,6,9", sort);
249
+ frt_sort_clear(sort);
250
+
251
+ frt_sort_add_sort_field(sort, frt_sort_field_string_new(string, false));
252
+ do_test_top_docs(tc, sea, q, "0,9,1,8,2,7,3,6,4,5", sort);
253
+ sort->sort_fields[0]->reverse = true;
254
+ do_test_top_docs(tc, sea, q, "5,4,6,3,7,2,8,1,9,0", sort);
255
+ frt_sort_clear(sort);
256
+
257
+ if (do_byte_test) {
258
+ frt_sort_add_sort_field(sort, frt_sort_field_byte_new(string, false));
259
+ do_test_top_docs(tc, sea, q, "5,0,9,1,8,2,7,3,6,4", sort);
260
+ sort->sort_fields[0]->reverse = true;
261
+ do_test_top_docs(tc, sea, q, "4,6,3,7,2,8,1,9,0,5", sort);
262
+ frt_sort_clear(sort);
263
+ }
264
+
265
+ /* Test Auto FrtSort */
266
+ frt_sort_add_sort_field(sort, frt_sort_field_auto_new(string, false));
267
+ do_test_top_docs(tc, sea, q, "0,9,1,8,2,7,3,6,4,5", sort);
268
+ frt_sort_clear(sort);
269
+
270
+ frt_sort_add_sort_field(sort, frt_sort_field_auto_new(integer, false));
271
+ do_test_top_docs(tc, sea, q, "3,2,7,4,8,5,9,1,6,0", sort);
272
+ frt_sort_clear(sort);
273
+
274
+ frt_sort_add_sort_field(sort, frt_sort_field_auto_new(flt, false));
275
+ do_test_top_docs(tc, sea, q, "9,6,4,2,0,1,3,5,7,8", sort);
276
+ sort->sort_fields[0]->reverse = true;
277
+ do_test_top_docs(tc, sea, q, "8,7,5,3,1,0,2,4,6,9", sort);
278
+ frt_sort_clear(sort);
279
+
280
+ frt_sort_add_sort_field(sort, frt_sort_field_auto_new(integer, false));
281
+ frt_sort_add_sort_field(sort, frt_sort_field_auto_new(string, false));
282
+ do_test_top_docs(tc, sea, q, "3,2,7,8,4,9,5,1,6,0", sort);
283
+
284
+ frt_sort_destroy(sort);
285
+ frt_q_deref(q);
286
+ }
287
+
288
+ TestSuite *ts_sort(TestSuite *suite)
289
+ {
290
+ FrtSearcher *sea, **searchers;
291
+ FrtStore *store = frt_open_ram_store(), *fs_store;
292
+
293
+ search = rb_intern("search");
294
+ string = rb_intern("string");
295
+ integer = rb_intern("integer");
296
+ flt = rb_intern("flt");
297
+
298
+ sort_test_setup(store);
299
+
300
+ suite = ADD_SUITE(suite);
301
+
302
+ tst_run_test(suite, test_sort_field_to_s, NULL);
303
+ tst_run_test(suite, test_sort_to_s, NULL);
304
+
305
+ sea = frt_isea_new(frt_ir_open(store));
306
+
307
+ tst_run_test(suite, test_sorts, (void *)sea);
308
+
309
+ frt_searcher_close(sea);
310
+
311
+ do_byte_test = false;
312
+
313
+ #if defined POSH_OS_WIN32 || defined POSH_OS_WIN64
314
+ fs_store = frt_open_fs_store(".\\test\\testdir\\store");
315
+ #else
316
+ fs_store = frt_open_fs_store("./test/testdir/store");
317
+ #endif
318
+ sort_multi_test_setup(store, fs_store);
319
+
320
+ searchers = FRT_ALLOC_N(FrtSearcher *, 2);
321
+
322
+ searchers[0] = frt_isea_new(frt_ir_open(store));
323
+ searchers[1] = frt_isea_new(frt_ir_open(fs_store));
324
+
325
+ sea = frt_msea_new(searchers, 2, true);
326
+ tst_run_test(suite, test_sorts, (void *)sea);
327
+ frt_searcher_close(sea);
328
+
329
+ frt_store_deref(store);
330
+ frt_store_deref(fs_store);
331
+
332
+ return suite;
333
+ }