isomorfeus-ferret 0.12.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (222) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +612 -0
  3. data/README.md +44 -0
  4. data/ext/isomorfeus_ferret_ext/benchmark.c +223 -0
  5. data/ext/isomorfeus_ferret_ext/benchmark.h +45 -0
  6. data/ext/isomorfeus_ferret_ext/benchmarks_all.h +25 -0
  7. data/ext/isomorfeus_ferret_ext/bm_bitvector.c +123 -0
  8. data/ext/isomorfeus_ferret_ext/bm_hash.c +118 -0
  9. data/ext/isomorfeus_ferret_ext/bm_micro_string.c +40 -0
  10. data/ext/isomorfeus_ferret_ext/bm_store.c +93 -0
  11. data/ext/isomorfeus_ferret_ext/email.rl +21 -0
  12. data/ext/isomorfeus_ferret_ext/extconf.rb +5 -0
  13. data/ext/isomorfeus_ferret_ext/fio_tmpfile.h +53 -0
  14. data/ext/isomorfeus_ferret_ext/frb_analysis.c +2577 -0
  15. data/ext/isomorfeus_ferret_ext/frb_index.c +3457 -0
  16. data/ext/isomorfeus_ferret_ext/frb_lang.c +9 -0
  17. data/ext/isomorfeus_ferret_ext/frb_lang.h +17 -0
  18. data/ext/isomorfeus_ferret_ext/frb_qparser.c +629 -0
  19. data/ext/isomorfeus_ferret_ext/frb_search.c +4460 -0
  20. data/ext/isomorfeus_ferret_ext/frb_store.c +515 -0
  21. data/ext/isomorfeus_ferret_ext/frb_threading.h +30 -0
  22. data/ext/isomorfeus_ferret_ext/frb_utils.c +1127 -0
  23. data/ext/isomorfeus_ferret_ext/frt_analysis.c +1644 -0
  24. data/ext/isomorfeus_ferret_ext/frt_analysis.h +247 -0
  25. data/ext/isomorfeus_ferret_ext/frt_array.c +124 -0
  26. data/ext/isomorfeus_ferret_ext/frt_array.h +54 -0
  27. data/ext/isomorfeus_ferret_ext/frt_bitvector.c +95 -0
  28. data/ext/isomorfeus_ferret_ext/frt_bitvector.h +586 -0
  29. data/ext/isomorfeus_ferret_ext/frt_compound_io.c +374 -0
  30. data/ext/isomorfeus_ferret_ext/frt_config.h +44 -0
  31. data/ext/isomorfeus_ferret_ext/frt_document.c +134 -0
  32. data/ext/isomorfeus_ferret_ext/frt_document.h +52 -0
  33. data/ext/isomorfeus_ferret_ext/frt_except.c +95 -0
  34. data/ext/isomorfeus_ferret_ext/frt_except.h +188 -0
  35. data/ext/isomorfeus_ferret_ext/frt_field_index.c +233 -0
  36. data/ext/isomorfeus_ferret_ext/frt_field_index.h +42 -0
  37. data/ext/isomorfeus_ferret_ext/frt_filter.c +157 -0
  38. data/ext/isomorfeus_ferret_ext/frt_fs_store.c +502 -0
  39. data/ext/isomorfeus_ferret_ext/frt_global.c +427 -0
  40. data/ext/isomorfeus_ferret_ext/frt_global.h +290 -0
  41. data/ext/isomorfeus_ferret_ext/frt_hash.c +518 -0
  42. data/ext/isomorfeus_ferret_ext/frt_hash.h +466 -0
  43. data/ext/isomorfeus_ferret_ext/frt_hashset.c +191 -0
  44. data/ext/isomorfeus_ferret_ext/frt_hashset.h +206 -0
  45. data/ext/isomorfeus_ferret_ext/frt_helper.c +62 -0
  46. data/ext/isomorfeus_ferret_ext/frt_helper.h +13 -0
  47. data/ext/isomorfeus_ferret_ext/frt_ind.c +353 -0
  48. data/ext/isomorfeus_ferret_ext/frt_ind.h +54 -0
  49. data/ext/isomorfeus_ferret_ext/frt_index.c +6377 -0
  50. data/ext/isomorfeus_ferret_ext/frt_index.h +880 -0
  51. data/ext/isomorfeus_ferret_ext/frt_lang.c +104 -0
  52. data/ext/isomorfeus_ferret_ext/frt_lang.h +44 -0
  53. data/ext/isomorfeus_ferret_ext/frt_mempool.c +87 -0
  54. data/ext/isomorfeus_ferret_ext/frt_mempool.h +33 -0
  55. data/ext/isomorfeus_ferret_ext/frt_multimapper.c +349 -0
  56. data/ext/isomorfeus_ferret_ext/frt_multimapper.h +52 -0
  57. data/ext/isomorfeus_ferret_ext/frt_posh.c +1006 -0
  58. data/ext/isomorfeus_ferret_ext/frt_posh.h +973 -0
  59. data/ext/isomorfeus_ferret_ext/frt_priorityqueue.c +147 -0
  60. data/ext/isomorfeus_ferret_ext/frt_priorityqueue.h +147 -0
  61. data/ext/isomorfeus_ferret_ext/frt_q_boolean.c +1612 -0
  62. data/ext/isomorfeus_ferret_ext/frt_q_const_score.c +157 -0
  63. data/ext/isomorfeus_ferret_ext/frt_q_filtered_query.c +209 -0
  64. data/ext/isomorfeus_ferret_ext/frt_q_fuzzy.c +281 -0
  65. data/ext/isomorfeus_ferret_ext/frt_q_match_all.c +147 -0
  66. data/ext/isomorfeus_ferret_ext/frt_q_multi_term.c +672 -0
  67. data/ext/isomorfeus_ferret_ext/frt_q_parser.c +3084 -0
  68. data/ext/isomorfeus_ferret_ext/frt_q_phrase.c +1182 -0
  69. data/ext/isomorfeus_ferret_ext/frt_q_prefix.c +98 -0
  70. data/ext/isomorfeus_ferret_ext/frt_q_range.c +665 -0
  71. data/ext/isomorfeus_ferret_ext/frt_q_span.c +2386 -0
  72. data/ext/isomorfeus_ferret_ext/frt_q_term.c +311 -0
  73. data/ext/isomorfeus_ferret_ext/frt_q_wildcard.c +166 -0
  74. data/ext/isomorfeus_ferret_ext/frt_ram_store.c +460 -0
  75. data/ext/isomorfeus_ferret_ext/frt_scanner.c +899 -0
  76. data/ext/isomorfeus_ferret_ext/frt_scanner.h +28 -0
  77. data/ext/isomorfeus_ferret_ext/frt_scanner_mb.c +6705 -0
  78. data/ext/isomorfeus_ferret_ext/frt_scanner_utf8.c +4419 -0
  79. data/ext/isomorfeus_ferret_ext/frt_search.c +1824 -0
  80. data/ext/isomorfeus_ferret_ext/frt_search.h +924 -0
  81. data/ext/isomorfeus_ferret_ext/frt_similarity.c +150 -0
  82. data/ext/isomorfeus_ferret_ext/frt_similarity.h +79 -0
  83. data/ext/isomorfeus_ferret_ext/frt_sort.c +796 -0
  84. data/ext/isomorfeus_ferret_ext/frt_stopwords.c +395 -0
  85. data/ext/isomorfeus_ferret_ext/frt_store.c +680 -0
  86. data/ext/isomorfeus_ferret_ext/frt_store.h +789 -0
  87. data/ext/isomorfeus_ferret_ext/frt_term_vectors.c +72 -0
  88. data/ext/isomorfeus_ferret_ext/frt_threading.h +23 -0
  89. data/ext/isomorfeus_ferret_ext/frt_win32.h +54 -0
  90. data/ext/isomorfeus_ferret_ext/isomorfeus_ferret.c +409 -0
  91. data/ext/isomorfeus_ferret_ext/isomorfeus_ferret.h +95 -0
  92. data/ext/isomorfeus_ferret_ext/libstemmer.c +93 -0
  93. data/ext/isomorfeus_ferret_ext/libstemmer.h +73 -0
  94. data/ext/isomorfeus_ferret_ext/q_parser.y +1366 -0
  95. data/ext/isomorfeus_ferret_ext/scanner.h +28 -0
  96. data/ext/isomorfeus_ferret_ext/scanner.in +43 -0
  97. data/ext/isomorfeus_ferret_ext/scanner.rl +84 -0
  98. data/ext/isomorfeus_ferret_ext/scanner_mb.rl +200 -0
  99. data/ext/isomorfeus_ferret_ext/scanner_utf8.rl +85 -0
  100. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_danish.c +324 -0
  101. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_danish.h +7 -0
  102. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_dutch.c +610 -0
  103. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_dutch.h +6 -0
  104. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_english.c +1104 -0
  105. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_english.h +6 -0
  106. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_finnish.c +749 -0
  107. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_finnish.h +7 -0
  108. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_french.c +1233 -0
  109. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_french.h +6 -0
  110. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_german.c +490 -0
  111. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_german.h +6 -0
  112. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_hungarian.c +1217 -0
  113. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_hungarian.h +7 -0
  114. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_italian.c +1052 -0
  115. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_italian.h +6 -0
  116. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_norwegian.c +283 -0
  117. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_norwegian.h +6 -0
  118. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_porter.c +735 -0
  119. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_porter.h +6 -0
  120. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_portuguese.c +1003 -0
  121. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_portuguese.h +7 -0
  122. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_spanish.c +1079 -0
  123. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_spanish.h +6 -0
  124. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_swedish.c +293 -0
  125. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_swedish.h +6 -0
  126. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_2_romanian.c +984 -0
  127. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_2_romanian.h +6 -0
  128. data/ext/isomorfeus_ferret_ext/stem_KOI8_R_russian.c +686 -0
  129. data/ext/isomorfeus_ferret_ext/stem_KOI8_R_russian.h +6 -0
  130. data/ext/isomorfeus_ferret_ext/stem_UTF_8_danish.c +325 -0
  131. data/ext/isomorfeus_ferret_ext/stem_UTF_8_danish.h +6 -0
  132. data/ext/isomorfeus_ferret_ext/stem_UTF_8_dutch.c +620 -0
  133. data/ext/isomorfeus_ferret_ext/stem_UTF_8_dutch.h +6 -0
  134. data/ext/isomorfeus_ferret_ext/stem_UTF_8_english.c +1111 -0
  135. data/ext/isomorfeus_ferret_ext/stem_UTF_8_english.h +6 -0
  136. data/ext/isomorfeus_ferret_ext/stem_UTF_8_finnish.c +754 -0
  137. data/ext/isomorfeus_ferret_ext/stem_UTF_8_finnish.h +6 -0
  138. data/ext/isomorfeus_ferret_ext/stem_UTF_8_french.c +1242 -0
  139. data/ext/isomorfeus_ferret_ext/stem_UTF_8_french.h +6 -0
  140. data/ext/isomorfeus_ferret_ext/stem_UTF_8_german.c +495 -0
  141. data/ext/isomorfeus_ferret_ext/stem_UTF_8_german.h +6 -0
  142. data/ext/isomorfeus_ferret_ext/stem_UTF_8_hungarian.c +1220 -0
  143. data/ext/isomorfeus_ferret_ext/stem_UTF_8_hungarian.h +6 -0
  144. data/ext/isomorfeus_ferret_ext/stem_UTF_8_italian.c +1059 -0
  145. data/ext/isomorfeus_ferret_ext/stem_UTF_8_italian.h +6 -0
  146. data/ext/isomorfeus_ferret_ext/stem_UTF_8_norwegian.c +285 -0
  147. data/ext/isomorfeus_ferret_ext/stem_UTF_8_norwegian.h +6 -0
  148. data/ext/isomorfeus_ferret_ext/stem_UTF_8_porter.c +741 -0
  149. data/ext/isomorfeus_ferret_ext/stem_UTF_8_porter.h +6 -0
  150. data/ext/isomorfeus_ferret_ext/stem_UTF_8_portuguese.c +1009 -0
  151. data/ext/isomorfeus_ferret_ext/stem_UTF_8_portuguese.h +6 -0
  152. data/ext/isomorfeus_ferret_ext/stem_UTF_8_romanian.c +990 -0
  153. data/ext/isomorfeus_ferret_ext/stem_UTF_8_romanian.h +6 -0
  154. data/ext/isomorfeus_ferret_ext/stem_UTF_8_russian.c +680 -0
  155. data/ext/isomorfeus_ferret_ext/stem_UTF_8_russian.h +6 -0
  156. data/ext/isomorfeus_ferret_ext/stem_UTF_8_spanish.c +1083 -0
  157. data/ext/isomorfeus_ferret_ext/stem_UTF_8_spanish.h +6 -0
  158. data/ext/isomorfeus_ferret_ext/stem_UTF_8_swedish.c +294 -0
  159. data/ext/isomorfeus_ferret_ext/stem_UTF_8_swedish.h +6 -0
  160. data/ext/isomorfeus_ferret_ext/stem_UTF_8_turkish.c +2191 -0
  161. data/ext/isomorfeus_ferret_ext/stem_UTF_8_turkish.h +6 -0
  162. data/ext/isomorfeus_ferret_ext/stem_api.c +66 -0
  163. data/ext/isomorfeus_ferret_ext/stem_api.h +26 -0
  164. data/ext/isomorfeus_ferret_ext/stem_header.h +57 -0
  165. data/ext/isomorfeus_ferret_ext/stem_modules.h +190 -0
  166. data/ext/isomorfeus_ferret_ext/stem_modules.txt +50 -0
  167. data/ext/isomorfeus_ferret_ext/stem_utilities.c +478 -0
  168. data/ext/isomorfeus_ferret_ext/test.c +850 -0
  169. data/ext/isomorfeus_ferret_ext/test.h +416 -0
  170. data/ext/isomorfeus_ferret_ext/test_1710.c +63 -0
  171. data/ext/isomorfeus_ferret_ext/test_analysis.c +1221 -0
  172. data/ext/isomorfeus_ferret_ext/test_array.c +272 -0
  173. data/ext/isomorfeus_ferret_ext/test_bitvector.c +600 -0
  174. data/ext/isomorfeus_ferret_ext/test_compound_io.c +170 -0
  175. data/ext/isomorfeus_ferret_ext/test_document.c +156 -0
  176. data/ext/isomorfeus_ferret_ext/test_except.c +244 -0
  177. data/ext/isomorfeus_ferret_ext/test_fields.c +522 -0
  178. data/ext/isomorfeus_ferret_ext/test_file_deleter.c +185 -0
  179. data/ext/isomorfeus_ferret_ext/test_filter.c +331 -0
  180. data/ext/isomorfeus_ferret_ext/test_fs_store.c +25 -0
  181. data/ext/isomorfeus_ferret_ext/test_global.c +299 -0
  182. data/ext/isomorfeus_ferret_ext/test_hash.c +485 -0
  183. data/ext/isomorfeus_ferret_ext/test_hashset.c +288 -0
  184. data/ext/isomorfeus_ferret_ext/test_helper.c +47 -0
  185. data/ext/isomorfeus_ferret_ext/test_highlighter.c +548 -0
  186. data/ext/isomorfeus_ferret_ext/test_index.c +2323 -0
  187. data/ext/isomorfeus_ferret_ext/test_lang.c +74 -0
  188. data/ext/isomorfeus_ferret_ext/test_mempool.c +102 -0
  189. data/ext/isomorfeus_ferret_ext/test_multimapper.c +64 -0
  190. data/ext/isomorfeus_ferret_ext/test_priorityqueue.c +213 -0
  191. data/ext/isomorfeus_ferret_ext/test_q_const_score.c +84 -0
  192. data/ext/isomorfeus_ferret_ext/test_q_filtered.c +61 -0
  193. data/ext/isomorfeus_ferret_ext/test_q_fuzzy.c +241 -0
  194. data/ext/isomorfeus_ferret_ext/test_q_parser.c +464 -0
  195. data/ext/isomorfeus_ferret_ext/test_q_span.c +575 -0
  196. data/ext/isomorfeus_ferret_ext/test_ram_store.c +77 -0
  197. data/ext/isomorfeus_ferret_ext/test_search.c +1874 -0
  198. data/ext/isomorfeus_ferret_ext/test_segments.c +167 -0
  199. data/ext/isomorfeus_ferret_ext/test_similarity.c +25 -0
  200. data/ext/isomorfeus_ferret_ext/test_sort.c +333 -0
  201. data/ext/isomorfeus_ferret_ext/test_store.c +591 -0
  202. data/ext/isomorfeus_ferret_ext/test_store.h +3 -0
  203. data/ext/isomorfeus_ferret_ext/test_term.c +351 -0
  204. data/ext/isomorfeus_ferret_ext/test_term_vectors.c +373 -0
  205. data/ext/isomorfeus_ferret_ext/test_test.c +83 -0
  206. data/ext/isomorfeus_ferret_ext/test_threading.c +188 -0
  207. data/ext/isomorfeus_ferret_ext/testhelper.c +561 -0
  208. data/ext/isomorfeus_ferret_ext/testhelper.h +25 -0
  209. data/ext/isomorfeus_ferret_ext/tests_all.h +87 -0
  210. data/ext/isomorfeus_ferret_ext/uchar-ucs4.rl +1854 -0
  211. data/ext/isomorfeus_ferret_ext/uchar-utf8.rl +1999 -0
  212. data/ext/isomorfeus_ferret_ext/url.rl +27 -0
  213. data/ext/isomorfeus_ferret_ext/word_list.h +15156 -0
  214. data/lib/isomorfeus/ferret/document.rb +132 -0
  215. data/lib/isomorfeus/ferret/field_symbol.rb +85 -0
  216. data/lib/isomorfeus/ferret/index/field_infos.rb +48 -0
  217. data/lib/isomorfeus/ferret/index/index.rb +970 -0
  218. data/lib/isomorfeus/ferret/monitor.rb +323 -0
  219. data/lib/isomorfeus/ferret/stdlib_patches.rb +151 -0
  220. data/lib/isomorfeus/ferret/version.rb +5 -0
  221. data/lib/isomorfeus-ferret.rb +8 -0
  222. metadata +307 -0
@@ -0,0 +1,548 @@
1
+ #include "frt_search.h"
2
+ #include "frt_array.h"
3
+ #include "testhelper.h"
4
+ #include "test.h"
5
+
6
+ #define ARRAY_SIZE 100
7
+
8
+ static void test_match_vector(TestCase *tc, void *data)
9
+ {
10
+ const int match_test_count = 100;
11
+ int i;
12
+ FrtMatchVector *mv = frt_matchv_new();
13
+ (void)data; /* suppress unused argument warning */
14
+ srand(5);
15
+
16
+ frt_matchv_add(mv, 0, 10);
17
+ frt_matchv_add(mv, 200, 220);
18
+ frt_matchv_add(mv, 50, 56);
19
+ frt_matchv_add(mv, 50, 55);
20
+ frt_matchv_add(mv, 57, 63);
21
+
22
+ Aiequal(0 , mv->matches[0].start);
23
+ Aiequal(10 , mv->matches[0].end);
24
+ Aiequal(200, mv->matches[1].start);
25
+ Aiequal(220, mv->matches[1].end);
26
+ Aiequal(50 , mv->matches[2].start);
27
+ Aiequal(56 , mv->matches[2].end);
28
+ Aiequal(50 , mv->matches[3].start);
29
+ Aiequal(55 , mv->matches[3].end);
30
+ Aiequal(57 , mv->matches[4].start);
31
+ Aiequal(63 , mv->matches[4].end);
32
+
33
+ frt_matchv_sort(mv);
34
+
35
+ Aiequal(0 , mv->matches[0].start);
36
+ Aiequal(10 , mv->matches[0].end);
37
+ Aiequal(50 , mv->matches[1].start);
38
+ Aiequal(56 , mv->matches[1].end);
39
+ Aiequal(50 , mv->matches[2].start);
40
+ Aiequal(55 , mv->matches[2].end);
41
+ Aiequal(57 , mv->matches[3].start);
42
+ Aiequal(63 , mv->matches[3].end);
43
+ Aiequal(200, mv->matches[4].start);
44
+ Aiequal(220, mv->matches[4].end);
45
+
46
+ frt_matchv_compact(mv);
47
+
48
+ Aiequal(3 , mv->size);
49
+ Aiequal(0 , mv->matches[0].start);
50
+ Aiequal(10 , mv->matches[0].end);
51
+ Aiequal(50 , mv->matches[1].start);
52
+ Aiequal(63 , mv->matches[1].end);
53
+ Aiequal(200, mv->matches[2].start);
54
+ Aiequal(220, mv->matches[2].end);
55
+
56
+ frt_matchv_destroy(mv);
57
+
58
+ mv = frt_matchv_new();
59
+
60
+ for (i = 0; i < match_test_count; i++)
61
+ {
62
+ int start = rand() % 10000000;
63
+ int end = start + rand() % 100;
64
+ frt_matchv_add(mv, start, end);
65
+ }
66
+
67
+ frt_matchv_sort(mv);
68
+
69
+ for (i = 1; i < match_test_count; i++)
70
+ {
71
+ Assert(mv->matches[i].start > mv->matches[i-1].start
72
+ || mv->matches[i].end > mv->matches[i-1].end,
73
+ "Offset(%d:%d) < FrtOffset(%d:%d)",
74
+ mv->matches[i].start, mv->matches[i].end,
75
+ mv->matches[i-1].start, mv->matches[i-1].end);
76
+ }
77
+ frt_matchv_destroy(mv);
78
+ }
79
+
80
+ static void make_index(FrtStore *store)
81
+ {
82
+ FrtFieldInfos *fis = frt_fis_new(FRT_STORE_YES, FRT_INDEX_YES,
83
+ FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS);
84
+ frt_index_create(store, fis);
85
+ frt_fis_deref(fis);
86
+ }
87
+
88
+ static void add_string_docs(FrtStore *store, const char *string[])
89
+ {
90
+ FrtIndexWriter *iw = frt_iw_open(store, frt_whitespace_analyzer_new(true), NULL);
91
+
92
+ while (*string) {
93
+ FrtDocument *doc = frt_doc_new();
94
+ frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("field")), (char *)*string));
95
+ frt_iw_add_doc(iw, doc);
96
+ frt_doc_destroy(doc);
97
+ string++;
98
+ }
99
+ frt_iw_close(iw);
100
+ }
101
+
102
+ #define Chk_sea_mv(query, doc_num, expected) check_searcher_match_vector(tc, store, query, doc_num, expected)
103
+ static void check_searcher_match_vector(TestCase *tc, FrtStore *store, FrtQuery *query, int doc_num, const char *expected)
104
+ {
105
+ FrtIndexReader *ir = frt_ir_open(store);
106
+ FrtSearcher *sea = frt_isea_new(ir);
107
+ FrtMatchVector *mv = frt_searcher_get_match_vector(sea, query, doc_num, rb_intern("field"));
108
+ static int offset_array[ARRAY_SIZE];
109
+ int matchv_size = s2l(expected, offset_array) / 2;
110
+ int i;
111
+
112
+ Aiequal(matchv_size, mv->size);
113
+ frt_matchv_sort(mv);
114
+ for (i = 0; i < matchv_size; i++) {
115
+ Aiequal(offset_array[i<<1], mv->matches[i].start);
116
+ Aiequal(offset_array[(i<<1)+1], mv->matches[i].end);
117
+ }
118
+ frt_matchv_destroy(mv);
119
+ frt_searcher_close(sea);
120
+ }
121
+
122
+ #define Chk_mv(query, doc_num, expected) check_match_vector(tc, store, query, doc_num, expected)
123
+ static void check_match_vector(TestCase *tc, FrtStore *store, FrtQuery *query, int doc_num, const char *expected)
124
+ {
125
+ FrtIndexReader *ir = frt_ir_open(store);
126
+ FrtMatchVector *mv = frt_matchv_new();
127
+ FrtTermVector *term_vector = ir->term_vector(ir, doc_num, rb_intern("field"));
128
+ static int offset_array[ARRAY_SIZE];
129
+ int matchv_size = s2l(expected, offset_array) / 2;
130
+ int i;
131
+
132
+ mv = query->get_matchv_i(query, mv, term_vector);
133
+ Aiequal(matchv_size, mv->size);
134
+ frt_matchv_sort(mv);
135
+ for (i = 0; i < matchv_size; i++) {
136
+ Aiequal(offset_array[i<<1], mv->matches[i].start);
137
+ Aiequal(offset_array[(i<<1)+1], mv->matches[i].end);
138
+ }
139
+ frt_matchv_destroy(mv);
140
+ frt_ir_close(ir);
141
+ frt_tv_destroy(term_vector);
142
+ check_searcher_match_vector(tc, store, query, doc_num, expected);
143
+ }
144
+
145
+ static void test_term_query(TestCase *tc, void *data)
146
+ {
147
+ FrtStore *store = (FrtStore *)data;
148
+ FrtQuery *q;
149
+ const char *docs[] = {
150
+ "the phrase has the word rabbit once",
151
+ "rabbit one rabbit two rabbit three rabbit four",
152
+ "Term doesn't appear in this sentence",
153
+ NULL
154
+ };
155
+ make_index(store);
156
+ add_string_docs(store, docs);
157
+ q = frt_tq_new(rb_intern("field"), "rabbit");
158
+ Chk_mv(q, 0, "5:5");
159
+ Chk_mv(q, 1, "0:0 2:2 4:4 6:6");
160
+ Chk_mv(q, 2, "");
161
+ frt_q_deref(q);
162
+ q = frt_tq_new(rb_intern("diff_field"), "rabbit");
163
+ Chk_mv(q, 0, "");
164
+ Chk_mv(q, 1, "");
165
+ frt_q_deref(q);
166
+ }
167
+
168
+ static void test_phrase_query(TestCase *tc, void *data)
169
+ {
170
+ FrtStore *store = (FrtStore *)data;
171
+ FrtQuery *q;
172
+ const char *docs[] = {
173
+ "the magic phrase of the day is one two three not three "
174
+ "two one one too three",
175
+ "one two three and again one two three and maybe one more for good "
176
+ "luck one two three",
177
+ "phrase doesn't appear in this sentence",
178
+ "multi phrase quick brown fox fast white wolf agile red fox quick "
179
+ "pink hound",
180
+ "multi phrase with slop brown quick fox the agile beautful and "
181
+ "cunning white wolf",
182
+ NULL
183
+ };
184
+ make_index(store);
185
+ add_string_docs(store, docs);
186
+ q = frt_phq_new(rb_intern("field"));
187
+ frt_phq_add_term(q, "one", 1);
188
+ frt_phq_add_term(q, "two", 1);
189
+ frt_phq_add_term(q, "three", 1);
190
+ Chk_mv(q, 0, "7:9");
191
+ Chk_mv(q, 1, "0:2 5:7 15:17");
192
+ Chk_mv(q, 2, "");
193
+ ((FrtPhraseQuery *)q)->slop = 3;
194
+ Chk_mv(q, 0, "7:9 12:16");
195
+ ((FrtPhraseQuery *)q)->slop = 4;
196
+ Chk_mv(q, 0, "7:9 11:13 12:16");
197
+ frt_q_deref(q);
198
+
199
+ /* test that it only works for the correct field */
200
+ q = frt_phq_new(rb_intern("wrong_field"));
201
+ frt_phq_add_term(q, "one", 1);
202
+ frt_phq_add_term(q, "two", 1);
203
+ frt_phq_add_term(q, "three", 1);
204
+ Chk_mv(q, 0, "");
205
+ Chk_mv(q, 1, "");
206
+ Chk_mv(q, 2, "");
207
+ ((FrtPhraseQuery *)q)->slop = 4;
208
+ Chk_mv(q, 0, "");
209
+ frt_q_deref(q);
210
+
211
+ q = frt_phq_new(rb_intern("field"));
212
+ frt_phq_add_term(q, "quick", 1);
213
+ frt_phq_append_multi_term(q, "fast");
214
+ frt_phq_append_multi_term(q, "agile");
215
+ frt_phq_add_term(q, "brown", 1);
216
+ frt_phq_append_multi_term(q, "pink");
217
+ frt_phq_append_multi_term(q, "red");
218
+ frt_phq_append_multi_term(q, "white");
219
+ frt_phq_add_term(q, "fox", 1);
220
+ frt_phq_append_multi_term(q, "wolf");
221
+ frt_phq_append_multi_term(q, "hound");
222
+ Chk_mv(q, 3, "2:4 5:7 8:10 11:13");
223
+ Chk_mv(q, 4, "");
224
+ ((FrtPhraseQuery *)q)->slop = 2;
225
+ Chk_mv(q, 4, "4:6");
226
+ ((FrtPhraseQuery *)q)->slop = 5;
227
+ Chk_mv(q, 4, "4:6 8:13");
228
+ frt_q_deref(q);
229
+ }
230
+
231
+ static void test_boolean_query(TestCase *tc, void *data)
232
+ {
233
+ FrtStore *store = (FrtStore *)data;
234
+ FrtQuery *q, *phq;
235
+ const char *docs[] = {
236
+ "one and some words and two and three and some more words one two",
237
+ NULL
238
+ };
239
+ make_index(store);
240
+ add_string_docs(store, docs);
241
+ q = frt_bq_new(false);
242
+ frt_bq_add_query_nr(q, frt_tq_new(rb_intern("field"), "one"), FRT_BC_SHOULD);
243
+ Chk_mv(q, 0, "0:0 12:12");
244
+ frt_bq_add_query_nr(q, frt_tq_new(rb_intern("field"), "two"), FRT_BC_MUST);
245
+ Chk_mv(q, 0, "0:0 5:5 12:12 13:13");
246
+ phq = frt_phq_new(rb_intern("field"));
247
+ frt_phq_add_term(phq, "one", 1);
248
+ frt_phq_add_term(phq, "two", 1);
249
+ Chk_mv(phq, 0, "12:13");
250
+ frt_bq_add_query_nr(q, phq, FRT_BC_SHOULD);
251
+ Chk_mv(q, 0, "0:0 5:5 12:13 12:12 13:13");
252
+ frt_q_deref(q);
253
+ }
254
+
255
+ static void test_multi_term_query(TestCase *tc, void *data)
256
+ {
257
+ FrtStore *store = (FrtStore *)data;
258
+ FrtQuery *q;
259
+ const char *docs[] = {
260
+ "one and some words and two and three and some more words one two",
261
+ NULL
262
+ };
263
+ make_index(store);
264
+ add_string_docs(store, docs);
265
+ q = frt_multi_tq_new(rb_intern("field"));
266
+ frt_multi_tq_add_term(q, "one");
267
+ Chk_mv(q, 0, "0:0 12:12");
268
+ frt_multi_tq_add_term(q, "two");
269
+ Chk_mv(q, 0, "0:0 5:5 12:12 13:13");
270
+ frt_multi_tq_add_term(q, "and");
271
+ Chk_mv(q, 0, "0:0 1:1 4:4 5:5 6:6 8:8 12:12 13:13");
272
+ frt_q_deref(q);
273
+ }
274
+
275
+ static void test_span_queries(TestCase *tc, void *data)
276
+ {
277
+ FrtStore *store = (FrtStore *)data;
278
+ FrtQuery *q, *oq;
279
+ const char *docs[] = {
280
+ "one and some words an two and three words and some more words one two",
281
+ "worda one wordb one worda one 2 wordb one 2 worda one two three wordb",
282
+ NULL
283
+ };
284
+ make_index(store);
285
+ add_string_docs(store, docs);
286
+ q = frt_spantq_new(rb_intern("wrong_field"), "words");
287
+ Chk_mv(q, 0, "");
288
+ frt_q_deref(q);
289
+
290
+ q = frt_spantq_new(rb_intern("field"), "words");
291
+ Chk_mv(q, 0, "3:3 8:8 12:12");
292
+ q = frt_spanfq_new_nr(q, 4);
293
+ Chk_mv(q, 0, "3:3");
294
+ ((FrtSpanFirstQuery *)q)->end = 8;
295
+ Chk_mv(q, 0, "3:3");
296
+ ((FrtSpanFirstQuery *)q)->end = 9;
297
+ Chk_mv(q, 0, "3:3 8:8");
298
+ ((FrtSpanFirstQuery *)q)->end = 12;
299
+ Chk_mv(q, 0, "3:3 8:8");
300
+ ((FrtSpanFirstQuery *)q)->end = 13;
301
+ Chk_mv(q, 0, "3:3 8:8 12:12");
302
+
303
+ oq = frt_spanoq_new();
304
+ frt_spanoq_add_clause_nr(oq, q);
305
+ Chk_mv(oq, 0, "3:3 8:8 12:12");
306
+ frt_spanoq_add_clause_nr(oq, frt_spantq_new(rb_intern("field"), "one"));
307
+ Chk_mv(oq, 0, "0:0 3:3 8:8 12:12 13:13");
308
+ frt_spanoq_add_clause_nr(oq, frt_spantq_new(rb_intern("field"), "two"));
309
+ Chk_mv(oq, 0, "0:0 3:3 5:5 8:8 12:12 13:13 14:14");
310
+ frt_q_deref(oq);
311
+
312
+ q = frt_spannq_new(1, true);
313
+ frt_spannq_add_clause_nr(q, frt_spantq_new(rb_intern("field"), "worda"));
314
+ Chk_mv(q, 0, "");
315
+ Chk_mv(q, 1, "0:0 4:4 10:10");
316
+ frt_spannq_add_clause_nr(q, frt_spantq_new(rb_intern("field"), "wordb"));
317
+ Chk_mv(q, 1, "0:0 2:2");
318
+ ((FrtSpanNearQuery *)q)->in_order = false;
319
+ Chk_mv(q, 1, "0:0 2:2 4:4");
320
+ ((FrtSpanNearQuery *)q)->slop = 2;
321
+ Chk_mv(q, 1, "0:0 2:2 4:4 7:7 10:10");
322
+ ((FrtSpanNearQuery *)q)->slop = 3;
323
+ Chk_mv(q, 1, "0:0 2:2 4:4 7:7 10:10 14:14");
324
+
325
+ q = frt_spanxq_new_nr(q, frt_spantq_new(rb_intern("field"), "2"));
326
+ Chk_mv(q, 1, "0:0 2:2 4:4 10:10 14:14");
327
+ frt_q_deref(q);
328
+ }
329
+
330
+ static void test_searcher_get_match_vector(TestCase *tc, void *data)
331
+ {
332
+ FrtStore *store = (FrtStore *)data;
333
+ FrtQuery *q;
334
+ const char *docs[] = {
335
+ "funnyword funniward funyword funywod funnywerd funniword finnywood",
336
+ NULL
337
+ };
338
+ make_index(store);
339
+ add_string_docs(store, docs);
340
+ q = frt_fuzq_new_conf(rb_intern("field"), "funnyword", 0.9f, 0, 512);
341
+ Chk_sea_mv(q, 0, "0:0");
342
+ frt_q_deref(q);
343
+
344
+ q = frt_fuzq_new_conf(rb_intern("field"), "funnyword", 0.8f, 0, 512);
345
+ Chk_sea_mv(q, 0, "0:0 2:2 4:4 5:5");
346
+ frt_q_deref(q);
347
+
348
+ q = frt_fuzq_new_conf(rb_intern("field"), "funnyword", 0.5f, 0, 512);
349
+ Chk_sea_mv(q, 0, "0:0 1:1 2:2 3:3 4:4 5:5 6:6");
350
+ frt_q_deref(q);
351
+ }
352
+
353
+ static void test_searcher_highlight(TestCase *tc, void *data)
354
+ {
355
+ FrtStore *store = (FrtStore *)data;
356
+ FrtQuery *q, *phq;
357
+ FrtIndexWriter *iw;
358
+ FrtIndexReader *ir;
359
+ FrtSearcher *sea;
360
+ char **highlights;
361
+ const char *docs[] = {
362
+ "the words we are searching for are one and two also sometimes "
363
+ "looking for them as a phrase like this; one two lets see "
364
+ "how it goes",
365
+ NULL
366
+ };
367
+ FrtDocument *doc = frt_doc_new();
368
+
369
+ make_index(store);
370
+ add_string_docs(store, docs);
371
+
372
+ iw = frt_iw_open(store, frt_letter_analyzer_new(true), NULL);
373
+ frt_doc_add_field(doc, frt_df_add_data(frt_df_new(rb_intern("field")), (char *)"That's how it goes now."));
374
+ frt_iw_add_doc(iw, doc);
375
+ frt_doc_destroy(doc);
376
+ frt_iw_close(iw);
377
+
378
+ ir = frt_ir_open(store);
379
+ sea = frt_isea_new(ir);
380
+
381
+ q = frt_tq_new(rb_intern("field"), "one");
382
+ highlights = frt_searcher_highlight(sea, q, 0, rb_intern("field"), 10, 1,
383
+ "<b>", "</b>", "...");
384
+ Aiequal(1, frt_ary_size(highlights));
385
+ Asequal("...are <b>one</b>...", highlights[0]);
386
+ frt_ary_destroy(highlights, &free);
387
+
388
+ highlights = frt_searcher_highlight(sea, q, 0, rb_intern("field"), 10, 2,
389
+ "<b>", "</b>", "...");
390
+ Aiequal(2, frt_ary_size(highlights));
391
+ Asequal("...are <b>one</b>...", highlights[0]);
392
+ Asequal("...this; <b>one</b>...", highlights[1]);
393
+ frt_ary_destroy(highlights, &free);
394
+
395
+ highlights = frt_searcher_highlight(sea, q, 0, rb_intern("field"), 10, 3,
396
+ "<b>", "</b>", "...");
397
+ Aiequal(3, frt_ary_size(highlights));
398
+ Asequal("the words...", highlights[0]);
399
+ Asequal("...are <b>one</b>...", highlights[1]);
400
+ Asequal("...this; <b>one</b>...", highlights[2]);
401
+ frt_ary_destroy(highlights, &free);
402
+
403
+ highlights = frt_searcher_highlight(sea, q, 0, rb_intern("field"), 10, 4,
404
+ "<b>", "</b>", "...");
405
+ Aiequal(3, frt_ary_size(highlights));
406
+ Asequal("the words we are...", highlights[0]);
407
+ Asequal("...are <b>one</b>...", highlights[1]);
408
+ Asequal("...this; <b>one</b>...", highlights[2]);
409
+ frt_ary_destroy(highlights, &free);
410
+
411
+ highlights = frt_searcher_highlight(sea, q, 0, rb_intern("field"), 10, 5,
412
+ "<b>", "</b>", "...");
413
+ Aiequal(2, frt_ary_size(highlights));
414
+ Asequal("the words we are searching for are <b>one</b>...", highlights[0]);
415
+ Asequal("...this; <b>one</b>...", highlights[1]);
416
+ frt_ary_destroy(highlights, &free);
417
+
418
+ highlights = frt_searcher_highlight(sea, q, 0, rb_intern("field"), 10, 20,
419
+ "<b>", "</b>", "...");
420
+ Aiequal(1, frt_ary_size(highlights));
421
+ Asequal("the words we are searching for are <b>one</b> and two also "
422
+ "sometimes looking for them as a phrase like this; <b>one</b> "
423
+ "two lets see how it goes", highlights[0]);
424
+ frt_ary_destroy(highlights, &free);
425
+
426
+ highlights = frt_searcher_highlight(sea, q, 0, rb_intern("field"), 1000, 1,
427
+ "<b>", "</b>", "...");
428
+ Aiequal(1, frt_ary_size(highlights));
429
+ Asequal("the words we are searching for are <b>one</b> and two also "
430
+ "sometimes looking for them as a phrase like this; <b>one</b> "
431
+ "two lets see how it goes", highlights[0]);
432
+ frt_ary_destroy(highlights, &free);
433
+
434
+ frt_q_deref(q);
435
+
436
+ q = frt_bq_new(false);
437
+ frt_bq_add_query_nr(q, frt_tq_new(rb_intern("field"), "one"), FRT_BC_SHOULD);
438
+ frt_bq_add_query_nr(q, frt_tq_new(rb_intern("field"), "two"), FRT_BC_SHOULD);
439
+
440
+ highlights = frt_searcher_highlight(sea, q, 0, rb_intern("field"), 15, 2,
441
+ "<b>", "</b>", "...");
442
+ Aiequal(2, frt_ary_size(highlights));
443
+ Asequal("...<b>one</b> and <b>two</b>...", highlights[0]);
444
+ Asequal("...this; <b>one</b> <b>two</b>...", highlights[1]);
445
+ frt_ary_destroy(highlights, &free);
446
+
447
+ phq = frt_phq_new(rb_intern("field"));
448
+ frt_phq_add_term(phq, "one", 1);
449
+ frt_phq_add_term(phq, "two", 1);
450
+
451
+ frt_bq_add_query_nr(q, phq, FRT_BC_SHOULD);
452
+
453
+ highlights = frt_searcher_highlight(sea, q, 0, rb_intern("field"), 15, 2,
454
+ "<b>", "</b>", "...");
455
+ Aiequal(2, frt_ary_size(highlights));
456
+ Asequal("...<b>one</b> and <b>two</b>...", highlights[0]);
457
+ Asequal("...this; <b>one two</b>...", highlights[1]);
458
+ frt_ary_destroy(highlights, &free);
459
+
460
+ highlights = frt_searcher_highlight(sea, q, 0, rb_intern("field"), 15, 1,
461
+ "<b>", "</b>", "...");
462
+ Aiequal(1, frt_ary_size(highlights));
463
+ /* should have a higher priority since it the merger of three matches */
464
+ Asequal("...this; <b>one two</b>...", highlights[0]);
465
+ frt_ary_destroy(highlights, &free);
466
+
467
+ highlights = frt_searcher_highlight(sea, q, 0, rb_intern("not_a_field"), 15, 1,
468
+ "<b>", "</b>", "...");
469
+ Apnull(highlights);
470
+
471
+ frt_q_deref(q);
472
+
473
+ q = frt_tq_new(rb_intern("wrong_field"), "one");
474
+ highlights = frt_searcher_highlight(sea, q, 0, rb_intern("not_a_field"), 15, 1,
475
+ "<b>", "</b>", "...");
476
+ Apnull(highlights);
477
+
478
+ frt_q_deref(q);
479
+
480
+ q = frt_bq_new(false);
481
+ phq = frt_phq_new(rb_intern("field"));
482
+ frt_phq_add_term(phq, "the", 1);
483
+ frt_phq_add_term(phq, "words", 1);
484
+ frt_bq_add_query_nr(q, phq, FRT_BC_SHOULD);
485
+ phq = frt_phq_new(rb_intern("field"));
486
+ frt_phq_add_term(phq, "for", 1);
487
+ frt_phq_add_term(phq, "are", 1);
488
+ frt_phq_add_term(phq, "one", 1);
489
+ frt_phq_add_term(phq, "and", 1);
490
+ frt_phq_add_term(phq, "two", 1);
491
+ frt_bq_add_query_nr(q, phq, FRT_BC_SHOULD);
492
+ frt_bq_add_query_nr(q, frt_tq_new(rb_intern("field"), "words"), FRT_BC_SHOULD);
493
+ frt_bq_add_query_nr(q, frt_tq_new(rb_intern("field"), "one"), FRT_BC_SHOULD);
494
+ frt_bq_add_query_nr(q, frt_tq_new(rb_intern("field"), "two"), FRT_BC_SHOULD);
495
+ frt_bq_add_query_nr(q, frt_tq_new(rb_intern("field"), "UnKnOwNfIeLd"), FRT_BC_SHOULD);
496
+
497
+ highlights = frt_searcher_highlight(sea, q, 0, rb_intern("field"), 10, 1,
498
+ "<b>", "</b>", "...");
499
+ Aiequal(1, frt_ary_size(highlights));
500
+ Asequal("<b>the words</b>...", highlights[0]);
501
+ frt_ary_destroy(highlights, &free);
502
+
503
+ highlights = frt_searcher_highlight(sea, q, 0, rb_intern("field"), 10, 2,
504
+ "<b>", "</b>", "...");
505
+ Aiequal(2, frt_ary_size(highlights));
506
+ Asequal("<b>the words</b>...", highlights[0]);
507
+ Asequal("...<b>one</b> <b>two</b>...", highlights[1]);
508
+ frt_ary_destroy(highlights, &free);
509
+
510
+ frt_q_deref(q);
511
+
512
+ q = frt_tq_new(rb_intern("field"), "goes");
513
+ highlights = frt_searcher_highlight(sea, q, 0, rb_intern("field"), 13, 2,
514
+ "<b>", "</b>", "...");
515
+ Aiequal(2, frt_ary_size(highlights));
516
+ Asequal("the words we...", highlights[0]);
517
+ Asequal("...how it <b>goes</b>", highlights[1]);
518
+ frt_ary_destroy(highlights, &free);
519
+
520
+ highlights = frt_searcher_highlight(sea, q, 1, rb_intern("field"), 16, 1,
521
+ "<b>", "</b>", "...");
522
+ Aiequal(1, frt_ary_size(highlights));
523
+ Asequal("...how it <b>goes</b> now.", highlights[0]);
524
+ frt_ary_destroy(highlights, &free);
525
+ frt_q_deref(q);
526
+
527
+ frt_searcher_close(sea);
528
+ }
529
+
530
+ TestSuite *ts_highlighter(TestSuite *suite)
531
+ {
532
+ FrtStore *store = frt_open_ram_store();
533
+
534
+ suite = ADD_SUITE(suite);
535
+
536
+ tst_run_test(suite, test_match_vector, NULL);
537
+ tst_run_test(suite, test_term_query, store);
538
+ tst_run_test(suite, test_phrase_query, store);
539
+ tst_run_test(suite, test_boolean_query, store);
540
+ tst_run_test(suite, test_multi_term_query, store);
541
+ tst_run_test(suite, test_span_queries, store);
542
+
543
+ tst_run_test(suite, test_searcher_get_match_vector, store);
544
+ tst_run_test(suite, test_searcher_highlight, store);
545
+
546
+ frt_store_deref(store);
547
+ return suite;
548
+ }