isomorfeus-ferret 0.12.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (222) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +612 -0
  3. data/README.md +44 -0
  4. data/ext/isomorfeus_ferret_ext/benchmark.c +223 -0
  5. data/ext/isomorfeus_ferret_ext/benchmark.h +45 -0
  6. data/ext/isomorfeus_ferret_ext/benchmarks_all.h +25 -0
  7. data/ext/isomorfeus_ferret_ext/bm_bitvector.c +123 -0
  8. data/ext/isomorfeus_ferret_ext/bm_hash.c +118 -0
  9. data/ext/isomorfeus_ferret_ext/bm_micro_string.c +40 -0
  10. data/ext/isomorfeus_ferret_ext/bm_store.c +93 -0
  11. data/ext/isomorfeus_ferret_ext/email.rl +21 -0
  12. data/ext/isomorfeus_ferret_ext/extconf.rb +5 -0
  13. data/ext/isomorfeus_ferret_ext/fio_tmpfile.h +53 -0
  14. data/ext/isomorfeus_ferret_ext/frb_analysis.c +2577 -0
  15. data/ext/isomorfeus_ferret_ext/frb_index.c +3457 -0
  16. data/ext/isomorfeus_ferret_ext/frb_lang.c +9 -0
  17. data/ext/isomorfeus_ferret_ext/frb_lang.h +17 -0
  18. data/ext/isomorfeus_ferret_ext/frb_qparser.c +629 -0
  19. data/ext/isomorfeus_ferret_ext/frb_search.c +4460 -0
  20. data/ext/isomorfeus_ferret_ext/frb_store.c +515 -0
  21. data/ext/isomorfeus_ferret_ext/frb_threading.h +30 -0
  22. data/ext/isomorfeus_ferret_ext/frb_utils.c +1127 -0
  23. data/ext/isomorfeus_ferret_ext/frt_analysis.c +1644 -0
  24. data/ext/isomorfeus_ferret_ext/frt_analysis.h +247 -0
  25. data/ext/isomorfeus_ferret_ext/frt_array.c +124 -0
  26. data/ext/isomorfeus_ferret_ext/frt_array.h +54 -0
  27. data/ext/isomorfeus_ferret_ext/frt_bitvector.c +95 -0
  28. data/ext/isomorfeus_ferret_ext/frt_bitvector.h +586 -0
  29. data/ext/isomorfeus_ferret_ext/frt_compound_io.c +374 -0
  30. data/ext/isomorfeus_ferret_ext/frt_config.h +44 -0
  31. data/ext/isomorfeus_ferret_ext/frt_document.c +134 -0
  32. data/ext/isomorfeus_ferret_ext/frt_document.h +52 -0
  33. data/ext/isomorfeus_ferret_ext/frt_except.c +95 -0
  34. data/ext/isomorfeus_ferret_ext/frt_except.h +188 -0
  35. data/ext/isomorfeus_ferret_ext/frt_field_index.c +233 -0
  36. data/ext/isomorfeus_ferret_ext/frt_field_index.h +42 -0
  37. data/ext/isomorfeus_ferret_ext/frt_filter.c +157 -0
  38. data/ext/isomorfeus_ferret_ext/frt_fs_store.c +502 -0
  39. data/ext/isomorfeus_ferret_ext/frt_global.c +427 -0
  40. data/ext/isomorfeus_ferret_ext/frt_global.h +290 -0
  41. data/ext/isomorfeus_ferret_ext/frt_hash.c +518 -0
  42. data/ext/isomorfeus_ferret_ext/frt_hash.h +466 -0
  43. data/ext/isomorfeus_ferret_ext/frt_hashset.c +191 -0
  44. data/ext/isomorfeus_ferret_ext/frt_hashset.h +206 -0
  45. data/ext/isomorfeus_ferret_ext/frt_helper.c +62 -0
  46. data/ext/isomorfeus_ferret_ext/frt_helper.h +13 -0
  47. data/ext/isomorfeus_ferret_ext/frt_ind.c +353 -0
  48. data/ext/isomorfeus_ferret_ext/frt_ind.h +54 -0
  49. data/ext/isomorfeus_ferret_ext/frt_index.c +6377 -0
  50. data/ext/isomorfeus_ferret_ext/frt_index.h +880 -0
  51. data/ext/isomorfeus_ferret_ext/frt_lang.c +104 -0
  52. data/ext/isomorfeus_ferret_ext/frt_lang.h +44 -0
  53. data/ext/isomorfeus_ferret_ext/frt_mempool.c +87 -0
  54. data/ext/isomorfeus_ferret_ext/frt_mempool.h +33 -0
  55. data/ext/isomorfeus_ferret_ext/frt_multimapper.c +349 -0
  56. data/ext/isomorfeus_ferret_ext/frt_multimapper.h +52 -0
  57. data/ext/isomorfeus_ferret_ext/frt_posh.c +1006 -0
  58. data/ext/isomorfeus_ferret_ext/frt_posh.h +973 -0
  59. data/ext/isomorfeus_ferret_ext/frt_priorityqueue.c +147 -0
  60. data/ext/isomorfeus_ferret_ext/frt_priorityqueue.h +147 -0
  61. data/ext/isomorfeus_ferret_ext/frt_q_boolean.c +1612 -0
  62. data/ext/isomorfeus_ferret_ext/frt_q_const_score.c +157 -0
  63. data/ext/isomorfeus_ferret_ext/frt_q_filtered_query.c +209 -0
  64. data/ext/isomorfeus_ferret_ext/frt_q_fuzzy.c +281 -0
  65. data/ext/isomorfeus_ferret_ext/frt_q_match_all.c +147 -0
  66. data/ext/isomorfeus_ferret_ext/frt_q_multi_term.c +672 -0
  67. data/ext/isomorfeus_ferret_ext/frt_q_parser.c +3084 -0
  68. data/ext/isomorfeus_ferret_ext/frt_q_phrase.c +1182 -0
  69. data/ext/isomorfeus_ferret_ext/frt_q_prefix.c +98 -0
  70. data/ext/isomorfeus_ferret_ext/frt_q_range.c +665 -0
  71. data/ext/isomorfeus_ferret_ext/frt_q_span.c +2386 -0
  72. data/ext/isomorfeus_ferret_ext/frt_q_term.c +311 -0
  73. data/ext/isomorfeus_ferret_ext/frt_q_wildcard.c +166 -0
  74. data/ext/isomorfeus_ferret_ext/frt_ram_store.c +460 -0
  75. data/ext/isomorfeus_ferret_ext/frt_scanner.c +899 -0
  76. data/ext/isomorfeus_ferret_ext/frt_scanner.h +28 -0
  77. data/ext/isomorfeus_ferret_ext/frt_scanner_mb.c +6705 -0
  78. data/ext/isomorfeus_ferret_ext/frt_scanner_utf8.c +4419 -0
  79. data/ext/isomorfeus_ferret_ext/frt_search.c +1824 -0
  80. data/ext/isomorfeus_ferret_ext/frt_search.h +924 -0
  81. data/ext/isomorfeus_ferret_ext/frt_similarity.c +150 -0
  82. data/ext/isomorfeus_ferret_ext/frt_similarity.h +79 -0
  83. data/ext/isomorfeus_ferret_ext/frt_sort.c +796 -0
  84. data/ext/isomorfeus_ferret_ext/frt_stopwords.c +395 -0
  85. data/ext/isomorfeus_ferret_ext/frt_store.c +680 -0
  86. data/ext/isomorfeus_ferret_ext/frt_store.h +789 -0
  87. data/ext/isomorfeus_ferret_ext/frt_term_vectors.c +72 -0
  88. data/ext/isomorfeus_ferret_ext/frt_threading.h +23 -0
  89. data/ext/isomorfeus_ferret_ext/frt_win32.h +54 -0
  90. data/ext/isomorfeus_ferret_ext/isomorfeus_ferret.c +409 -0
  91. data/ext/isomorfeus_ferret_ext/isomorfeus_ferret.h +95 -0
  92. data/ext/isomorfeus_ferret_ext/libstemmer.c +93 -0
  93. data/ext/isomorfeus_ferret_ext/libstemmer.h +73 -0
  94. data/ext/isomorfeus_ferret_ext/q_parser.y +1366 -0
  95. data/ext/isomorfeus_ferret_ext/scanner.h +28 -0
  96. data/ext/isomorfeus_ferret_ext/scanner.in +43 -0
  97. data/ext/isomorfeus_ferret_ext/scanner.rl +84 -0
  98. data/ext/isomorfeus_ferret_ext/scanner_mb.rl +200 -0
  99. data/ext/isomorfeus_ferret_ext/scanner_utf8.rl +85 -0
  100. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_danish.c +324 -0
  101. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_danish.h +7 -0
  102. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_dutch.c +610 -0
  103. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_dutch.h +6 -0
  104. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_english.c +1104 -0
  105. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_english.h +6 -0
  106. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_finnish.c +749 -0
  107. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_finnish.h +7 -0
  108. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_french.c +1233 -0
  109. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_french.h +6 -0
  110. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_german.c +490 -0
  111. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_german.h +6 -0
  112. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_hungarian.c +1217 -0
  113. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_hungarian.h +7 -0
  114. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_italian.c +1052 -0
  115. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_italian.h +6 -0
  116. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_norwegian.c +283 -0
  117. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_norwegian.h +6 -0
  118. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_porter.c +735 -0
  119. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_porter.h +6 -0
  120. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_portuguese.c +1003 -0
  121. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_portuguese.h +7 -0
  122. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_spanish.c +1079 -0
  123. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_spanish.h +6 -0
  124. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_swedish.c +293 -0
  125. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_swedish.h +6 -0
  126. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_2_romanian.c +984 -0
  127. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_2_romanian.h +6 -0
  128. data/ext/isomorfeus_ferret_ext/stem_KOI8_R_russian.c +686 -0
  129. data/ext/isomorfeus_ferret_ext/stem_KOI8_R_russian.h +6 -0
  130. data/ext/isomorfeus_ferret_ext/stem_UTF_8_danish.c +325 -0
  131. data/ext/isomorfeus_ferret_ext/stem_UTF_8_danish.h +6 -0
  132. data/ext/isomorfeus_ferret_ext/stem_UTF_8_dutch.c +620 -0
  133. data/ext/isomorfeus_ferret_ext/stem_UTF_8_dutch.h +6 -0
  134. data/ext/isomorfeus_ferret_ext/stem_UTF_8_english.c +1111 -0
  135. data/ext/isomorfeus_ferret_ext/stem_UTF_8_english.h +6 -0
  136. data/ext/isomorfeus_ferret_ext/stem_UTF_8_finnish.c +754 -0
  137. data/ext/isomorfeus_ferret_ext/stem_UTF_8_finnish.h +6 -0
  138. data/ext/isomorfeus_ferret_ext/stem_UTF_8_french.c +1242 -0
  139. data/ext/isomorfeus_ferret_ext/stem_UTF_8_french.h +6 -0
  140. data/ext/isomorfeus_ferret_ext/stem_UTF_8_german.c +495 -0
  141. data/ext/isomorfeus_ferret_ext/stem_UTF_8_german.h +6 -0
  142. data/ext/isomorfeus_ferret_ext/stem_UTF_8_hungarian.c +1220 -0
  143. data/ext/isomorfeus_ferret_ext/stem_UTF_8_hungarian.h +6 -0
  144. data/ext/isomorfeus_ferret_ext/stem_UTF_8_italian.c +1059 -0
  145. data/ext/isomorfeus_ferret_ext/stem_UTF_8_italian.h +6 -0
  146. data/ext/isomorfeus_ferret_ext/stem_UTF_8_norwegian.c +285 -0
  147. data/ext/isomorfeus_ferret_ext/stem_UTF_8_norwegian.h +6 -0
  148. data/ext/isomorfeus_ferret_ext/stem_UTF_8_porter.c +741 -0
  149. data/ext/isomorfeus_ferret_ext/stem_UTF_8_porter.h +6 -0
  150. data/ext/isomorfeus_ferret_ext/stem_UTF_8_portuguese.c +1009 -0
  151. data/ext/isomorfeus_ferret_ext/stem_UTF_8_portuguese.h +6 -0
  152. data/ext/isomorfeus_ferret_ext/stem_UTF_8_romanian.c +990 -0
  153. data/ext/isomorfeus_ferret_ext/stem_UTF_8_romanian.h +6 -0
  154. data/ext/isomorfeus_ferret_ext/stem_UTF_8_russian.c +680 -0
  155. data/ext/isomorfeus_ferret_ext/stem_UTF_8_russian.h +6 -0
  156. data/ext/isomorfeus_ferret_ext/stem_UTF_8_spanish.c +1083 -0
  157. data/ext/isomorfeus_ferret_ext/stem_UTF_8_spanish.h +6 -0
  158. data/ext/isomorfeus_ferret_ext/stem_UTF_8_swedish.c +294 -0
  159. data/ext/isomorfeus_ferret_ext/stem_UTF_8_swedish.h +6 -0
  160. data/ext/isomorfeus_ferret_ext/stem_UTF_8_turkish.c +2191 -0
  161. data/ext/isomorfeus_ferret_ext/stem_UTF_8_turkish.h +6 -0
  162. data/ext/isomorfeus_ferret_ext/stem_api.c +66 -0
  163. data/ext/isomorfeus_ferret_ext/stem_api.h +26 -0
  164. data/ext/isomorfeus_ferret_ext/stem_header.h +57 -0
  165. data/ext/isomorfeus_ferret_ext/stem_modules.h +190 -0
  166. data/ext/isomorfeus_ferret_ext/stem_modules.txt +50 -0
  167. data/ext/isomorfeus_ferret_ext/stem_utilities.c +478 -0
  168. data/ext/isomorfeus_ferret_ext/test.c +850 -0
  169. data/ext/isomorfeus_ferret_ext/test.h +416 -0
  170. data/ext/isomorfeus_ferret_ext/test_1710.c +63 -0
  171. data/ext/isomorfeus_ferret_ext/test_analysis.c +1221 -0
  172. data/ext/isomorfeus_ferret_ext/test_array.c +272 -0
  173. data/ext/isomorfeus_ferret_ext/test_bitvector.c +600 -0
  174. data/ext/isomorfeus_ferret_ext/test_compound_io.c +170 -0
  175. data/ext/isomorfeus_ferret_ext/test_document.c +156 -0
  176. data/ext/isomorfeus_ferret_ext/test_except.c +244 -0
  177. data/ext/isomorfeus_ferret_ext/test_fields.c +522 -0
  178. data/ext/isomorfeus_ferret_ext/test_file_deleter.c +185 -0
  179. data/ext/isomorfeus_ferret_ext/test_filter.c +331 -0
  180. data/ext/isomorfeus_ferret_ext/test_fs_store.c +25 -0
  181. data/ext/isomorfeus_ferret_ext/test_global.c +299 -0
  182. data/ext/isomorfeus_ferret_ext/test_hash.c +485 -0
  183. data/ext/isomorfeus_ferret_ext/test_hashset.c +288 -0
  184. data/ext/isomorfeus_ferret_ext/test_helper.c +47 -0
  185. data/ext/isomorfeus_ferret_ext/test_highlighter.c +548 -0
  186. data/ext/isomorfeus_ferret_ext/test_index.c +2323 -0
  187. data/ext/isomorfeus_ferret_ext/test_lang.c +74 -0
  188. data/ext/isomorfeus_ferret_ext/test_mempool.c +102 -0
  189. data/ext/isomorfeus_ferret_ext/test_multimapper.c +64 -0
  190. data/ext/isomorfeus_ferret_ext/test_priorityqueue.c +213 -0
  191. data/ext/isomorfeus_ferret_ext/test_q_const_score.c +84 -0
  192. data/ext/isomorfeus_ferret_ext/test_q_filtered.c +61 -0
  193. data/ext/isomorfeus_ferret_ext/test_q_fuzzy.c +241 -0
  194. data/ext/isomorfeus_ferret_ext/test_q_parser.c +464 -0
  195. data/ext/isomorfeus_ferret_ext/test_q_span.c +575 -0
  196. data/ext/isomorfeus_ferret_ext/test_ram_store.c +77 -0
  197. data/ext/isomorfeus_ferret_ext/test_search.c +1874 -0
  198. data/ext/isomorfeus_ferret_ext/test_segments.c +167 -0
  199. data/ext/isomorfeus_ferret_ext/test_similarity.c +25 -0
  200. data/ext/isomorfeus_ferret_ext/test_sort.c +333 -0
  201. data/ext/isomorfeus_ferret_ext/test_store.c +591 -0
  202. data/ext/isomorfeus_ferret_ext/test_store.h +3 -0
  203. data/ext/isomorfeus_ferret_ext/test_term.c +351 -0
  204. data/ext/isomorfeus_ferret_ext/test_term_vectors.c +373 -0
  205. data/ext/isomorfeus_ferret_ext/test_test.c +83 -0
  206. data/ext/isomorfeus_ferret_ext/test_threading.c +188 -0
  207. data/ext/isomorfeus_ferret_ext/testhelper.c +561 -0
  208. data/ext/isomorfeus_ferret_ext/testhelper.h +25 -0
  209. data/ext/isomorfeus_ferret_ext/tests_all.h +87 -0
  210. data/ext/isomorfeus_ferret_ext/uchar-ucs4.rl +1854 -0
  211. data/ext/isomorfeus_ferret_ext/uchar-utf8.rl +1999 -0
  212. data/ext/isomorfeus_ferret_ext/url.rl +27 -0
  213. data/ext/isomorfeus_ferret_ext/word_list.h +15156 -0
  214. data/lib/isomorfeus/ferret/document.rb +132 -0
  215. data/lib/isomorfeus/ferret/field_symbol.rb +85 -0
  216. data/lib/isomorfeus/ferret/index/field_infos.rb +48 -0
  217. data/lib/isomorfeus/ferret/index/index.rb +970 -0
  218. data/lib/isomorfeus/ferret/monitor.rb +323 -0
  219. data/lib/isomorfeus/ferret/stdlib_patches.rb +151 -0
  220. data/lib/isomorfeus/ferret/version.rb +5 -0
  221. data/lib/isomorfeus-ferret.rb +8 -0
  222. metadata +307 -0
@@ -0,0 +1,464 @@
1
+ #include "frt_search.h"
2
+ #include "test.h"
3
+
4
+ typedef struct QPTestPair {
5
+ const char *qstr;
6
+ const char *qres;
7
+ } QPTestPair;
8
+
9
+ #define PARSER_TEST(str, res) do {\
10
+ FrtQuery *q = qp_parse(parser, (char *)str);\
11
+ char *qres = q->to_s(q, rb_intern("xx"));\
12
+ Asequal(res, qres);\
13
+ frt_q_deref(q);\
14
+ free(qres);\
15
+ } while (0);
16
+
17
+ static void test_q_parser(TestCase *tc, void *data)
18
+ {
19
+ int i;
20
+ FrtAnalyzer *analyzer = frt_letter_analyzer_new(true);
21
+ FrtQParser *parser;
22
+ QPTestPair pairs[] = {
23
+ {"", ""},
24
+ {"word", "word"},
25
+ {"f1:word", "f1:word"},
26
+ {"f1|f2:word", "f1:word f2:word"},
27
+ {"field:word", "field:word"},
28
+ {"\"word1 word2 word3\"", "\"word word word\""},
29
+ {"\"word1 2342 word3\"", "\"word <> word\"~1"},
30
+ {"field:\"one TWO ThrEe\"", "field:\"one TWO ThrEe\""},
31
+ {"field:\"one 222 three\"", "field:\"one 222 three\""},
32
+ {"xx:\"one 222 three\"", "\"one <> three\"~1"},
33
+ {"field:\"one <> three\"", "field:\"one <> three\""},
34
+ {"field:\"<> two three\"", "field:\"two three\""},
35
+ {"field:\"one <> three <>\"", "field:\"one <> three\""},
36
+ {"field:\"oNe <> <> <> three <>\"", "field:\"oNe <> <> <> three\""},
37
+ {"field:\"one <> <> <> three|four|five <>\"",
38
+ "field:\"one <> <> <> three|four|five\""},
39
+ {"field:\"on1|tw2 three|four|five six|seven\"",
40
+ "field:\"on1|tw2 three|four|five six|seven\""},
41
+ {"field:\"testing|trucks\"", "field:\"testing|trucks\""},
42
+ {"[aaa bbb]", "[aaa bbb]"},
43
+ {"{aaa bbb]", "{aaa bbb]"},
44
+ {"field:[aaa bbb}", "field:[aaa bbb}"},
45
+ {"{aaa bbb}", "{aaa bbb}"},
46
+ {"{aaa>", "{aaa>"},
47
+ {"[aaa>", "[aaa>"},
48
+ {"field:<a\\ aa}", "field:<a aa}"},
49
+ {"<aaa]", "<aaa]"},
50
+ {">aaa", "{aaa>"},
51
+ {">=aaa", "[aaa>"},
52
+ {"<aaa", "<aaa}"},
53
+ {"field:<=aaa", "field:<aaa]"},
54
+ {"REQ one REQ two", "+one +two"},
55
+ {"REQ one two", "+one two"},
56
+ {"one REQ two", "one +two"},
57
+ {"+one +two", "+one +two"},
58
+ {"+one two", "+one two"},
59
+ {"one +two", "one +two"},
60
+ {"-one -two", "-one -two"},
61
+ {"-one two", "-one two"},
62
+ {"one -two", "one -two"},
63
+ {"!one !two", "-one -two"},
64
+ {"!one two", "-one two"},
65
+ {"one !two", "one -two"},
66
+ {"NOT one NOT two", "-one -two"},
67
+ {"NOT one two", "-one two"},
68
+ {"one NOT two", "one -two"},
69
+ {"one two", "one two"},
70
+ {"one OR two", "one two"},
71
+ {"one AND two", "+one +two"},
72
+ {"one two AND three", "one two +three"},
73
+ {"one two OR three", "one two three"},
74
+ {"Opus::City", "\"opus city\"~1"},
75
+ {"()", ""},
76
+ {"field:()", ""},
77
+ {"one AND (f1:two OR f2:three) AND four",
78
+ "+one +(f1:two f2:three) +four"},
79
+ {"xx:\"Hello Newman\" field:()", "\"hello newman\" ()"},
80
+ {"one (two AND three)", "one (+two +three)"},
81
+ {"one AND (two OR three)", "+one +(two three)"},
82
+ {"field:(one AND (two OR t\\=h\\=r\\=e\\=e))",
83
+ "+field:one +(field:two field:t=h=r=e=e)"},
84
+ {"one AND (two OR [aaa vvv})", "+one +(two [aaa vvv})"},
85
+ {"one AND (f1:two OR f2:three) AND four",
86
+ "+one +(f1:two f2:three) +four"},
87
+ {"one^1.2300", "one^1.23"},
88
+ {"(one AND two)^100.23", "(+one +two)^100.23"},
89
+ {"field:(one AND two)^100.23", "(+field:one +field:two)^100.23"},
90
+ {"field:(one AND [aaa bbb]^23.300)^100.23",
91
+ "(+field:one +field:[aaa bbb]^23.3)^100.23"},
92
+ {"(REQ field:\"one two three\")^23.000",
93
+ "field:\"one two three\"^23.0"},
94
+ {"asdf~0.2", "asdf~0.2"},
95
+ {"field:asdf~0.2", "field:asdf~0.2"},
96
+ {"asdf~0.2^100.00", "asdf~0.2^100.0"},
97
+ {"field:asdf~0.2^0.1", "field:asdf~0.2^0.1"},
98
+ {"field:\"asdf <> asdf|asdf\"~4", "field:\"asdf <> asdf|asdf\"~4"},
99
+ {"\"one two three four five\"~5", "\"one two three four five\"~5"},
100
+ {"ab?de", "ab?de"},
101
+ {"ab*de", "ab*de"},
102
+ {"asdf?*?asd*dsf?asfd*asdf?", "asdf?*?asd*dsf?asfd*asdf?"},
103
+ {"field:a* AND field:(b*)", "+field:a* +field:b*"},
104
+ {"field:abc~ AND field:(b*)", "+field:abc~ +field:b*"},
105
+ {"asdf?*?asd*dsf?asfd*asdf?^20.0", "asdf?*?asd*dsf?asfd*asdf?^20.0"},
106
+ {"field:ASDF?*?22d*dsf?ASFD*asdf?^20.0",
107
+ "field:ASDF?*?22d*dsf?ASFD*asdf?^20.0"},
108
+
109
+ {"*:xxx", "xxx f1:xxx f2:xxx field:xxx"},
110
+ {"f1|f2:xxx", "f1:xxx f2:xxx"},
111
+
112
+ {"*:asd~0.2", "asd~0.2 f1:asd~0.2 f2:asd~0.2 field:asd~0.2"},
113
+ {"f1|f2:asd~0.2", "f1:asd~0.2 f2:asd~0.2"},
114
+
115
+ {"*:a?d*^20.0", "(a?d* f1:a?d* f2:a?d* field:a?d*)^20.0"},
116
+ {"f1|f2:a?d*^20.0", "(f1:a?d* f2:a?d*)^20.0"},
117
+
118
+ {"*:\"asdf <> xxx|yyy\"",
119
+ "\"asdf <> xxx|yyy\" f1:\"asdf <> xxx|yyy\" f2:\"asdf <> xxx|yyy\" "
120
+ "field:\"asdf <> xxx|yyy\""},
121
+ {"f1|f2:\"asdf <> do|yyy\"",
122
+ "f1:\"asdf <> do|yyy\" f2:\"asdf <> do|yyy\""},
123
+
124
+ {"*:[bbb xxx]", "[bbb xxx] f1:[bbb xxx] f2:[bbb xxx] field:[bbb xxx]"},
125
+ {"f1|f2:[bbb xxx]", "f1:[bbb xxx] f2:[bbb xxx]"},
126
+
127
+ {"*:(xxx AND bbb)",
128
+ "+(xxx f1:xxx f2:xxx field:xxx) +(bbb f1:bbb f2:bbb field:bbb)"},
129
+ {"f1|f2:(xxx AND bbb)", "+(f1:xxx f2:xxx) +(f1:bbb f2:bbb)"},
130
+ {"ASDF?*?asd*dsf?ASFD*asdf?^20.0", "asdf?*?asd*dsf?asfd*asdf?^20.0"},
131
+ {"ASDFasdAasAasASD~", "asdfasdaasaasasd~"},
132
+ {"\"onewordphrase\"", "onewordphrase"},
133
+ {"one billion eight hundred and thirty three million four hundred "
134
+ "and eighty eight thousand two hundred and sixty three",
135
+ "one billion eight hundred and thirty three million four hundred "
136
+ "and eighty eight thousand two hundred and sixty three"},
137
+ {"f1:*", "*"},
138
+ {"f1:*^100.0", "*^100.0"},
139
+ {"f1:?*", "f1:?*"},
140
+ {"f1:?*^100.0", "f1:?*^100.0"},
141
+ {"f1:(aaa f2:bbb ccc)", "f1:aaa f2:bbb f1:ccc"}
142
+ };
143
+ (void)data;
144
+
145
+ FRT_REF(analyzer);
146
+ parser = frt_qp_new(analyzer);
147
+ frt_qp_add_field(parser, rb_intern("xx"), true, true);
148
+ frt_qp_add_field(parser, rb_intern("f1"), false, true);
149
+ frt_qp_add_field(parser, rb_intern("f2"), false, true);
150
+ frt_qp_add_field(parser, rb_intern("field"), false, false);
151
+
152
+ for (i = 0; i < FRT_NELEMS(pairs); i++) {
153
+ PARSER_TEST(pairs[i].qstr, pairs[i].qres);
154
+ }
155
+ parser->clean_str = true;
156
+ for (i = 0; i < FRT_NELEMS(pairs); i++) {
157
+ PARSER_TEST(pairs[i].qstr, pairs[i].qres);
158
+ }
159
+ PARSER_TEST("not_field:word", "");
160
+ frt_qp_destroy(parser);
161
+
162
+ /* This time let the query parser destroy the analyzer */
163
+ parser = frt_qp_new(analyzer);
164
+ frt_qp_add_field(parser, rb_intern("xx"), true, true);
165
+ frt_qp_add_field(parser, rb_intern("f1"), false, true);
166
+ frt_qp_add_field(parser, rb_intern("f2"), false, true);
167
+ frt_qp_add_field(parser, rb_intern("field"), false, false);
168
+
169
+ parser->clean_str = false;
170
+ parser->allow_any_fields = true;
171
+ for (i = 0; i < FRT_NELEMS(pairs); i++) {
172
+ PARSER_TEST(pairs[i].qstr, pairs[i].qres);
173
+ }
174
+ parser->clean_str = true;
175
+ for (i = 0; i < FRT_NELEMS(pairs); i++) {
176
+ PARSER_TEST(pairs[i].qstr, pairs[i].qres);
177
+ }
178
+ PARSER_TEST("not_field:word", "not_field:word");
179
+
180
+ parser->wild_lower = false;
181
+ PARSER_TEST("ASDF?*?asd*dsf?ASFD*asdf?^20.0", "ASDF?*?asd*dsf?ASFD*asdf?^20.0");
182
+ PARSER_TEST("ASDFasdAasAasASD~", "asdfasdaasaasasd~");
183
+ frt_qp_destroy(parser);
184
+ }
185
+
186
+ static void test_q_parser_standard_analyzer(TestCase *tc, void *data)
187
+ {
188
+ int i;
189
+ FrtAnalyzer *analyzer = frt_mb_standard_analyzer_new(true);
190
+ FrtQParser *parser;
191
+ QPTestPair pairs[] = {
192
+ {"", ""},
193
+ {"word", "word"},
194
+ {"f1:word", "f1:word"},
195
+ {"f1|f2:word", "f1:word f2:word"},
196
+ {"field:word", "field:word"},
197
+ {"\"word1 word2 word3\"", "\"word1 word2 word3\""},
198
+ {"\"word1 2342 word3\"", "\"word1 2342 word3\""},
199
+ {"field:\"one two three\"", "field:\"one two three\""},
200
+ {"field:\"one 222 three\"", "field:\"one 222 three\""},
201
+ {"field:\"one <> three\"", "field:\"one <> three\""},
202
+ {"field:\"<> two three\"", "field:\"two three\""},
203
+ {"field:\"one <> three <>\"", "field:\"one <> three\""},
204
+ {"field:\"one <> <> <> three <>\"", "field:\"one <> <> <> three\""},
205
+ {"field:\"one <> <> <> three|four|five <>\"",
206
+ "field:\"one <> <> <> three|four|five\""},
207
+ {"field:\"one|two three|four|five six|seven\"",
208
+ "field:\"one|two three|four|five six|seven\""},
209
+ {"field:\"testing|trucks\"", "field:\"testing|trucks\""},
210
+ {"[aaa bbb]", "[aaa bbb]"},
211
+ {"{aaa bbb]", "{aaa bbb]"},
212
+ {"field:[aaa bbb}", "field:[aaa bbb}"},
213
+ {"{aaa bbb}", "{aaa bbb}"},
214
+ {"{aaa>", "{aaa>"},
215
+ {"[aaa>", "[aaa>"},
216
+ {"field:<aaa}", "field:<aaa}"},
217
+ {"<aaa]", "<aaa]"},
218
+ {">aaa", "{aaa>"},
219
+ {">=aaa", "[aaa>"},
220
+ {"<aaa", "<aaa}"},
221
+ {"field:<=aaa", "field:<aaa]"},
222
+ {"REQ one REQ two", "+one +two"},
223
+ {"REQ one two", "+one two"},
224
+ {"one REQ two", "one +two"},
225
+ {"+one +two", "+one +two"},
226
+ {"+one two", "+one two"},
227
+ {"one +two", "one +two"},
228
+ {"-one -two", "-one -two"},
229
+ {"-one two", "-one two"},
230
+ {"one -two", "one -two"},
231
+ {"!one !two", "-one -two"},
232
+ {"!one two", "-one two"},
233
+ {"one !two", "one -two"},
234
+ {"NOT one NOT two", "-one -two"},
235
+ {"NOT one two", "-one two"},
236
+ {"one NOT two", "one -two"},
237
+ {"one two", "one two"},
238
+ {"one OR two", "one two"},
239
+ {"one AND two", "+one +two"},
240
+ {"one two AND three", "one two +three"},
241
+ {"one two OR three", "one two three"},
242
+ {"one (two AND three)", "one (+two +three)"},
243
+ {"one AND (two OR three)", "+one +(two three)"},
244
+ {"field:(one AND (two OR three))", "+field:one +(field:two field:three)"},
245
+ {"one AND (two OR [aaa vvv})", "+one +(two [aaa vvv})"},
246
+ {"one AND (f1:two OR f2:three) AND four", "+one +(f1:two f2:three) +four"},
247
+ {"one^1.2300", "one^1.23"},
248
+ {"(one AND two)^100.23", "(+one +two)^100.23"},
249
+ {"field:(one AND two)^100.23", "(+field:one +field:two)^100.23"},
250
+ {"field:(one AND [aaa bbb]^23.300)^100.23",
251
+ "(+field:one +field:[aaa bbb]^23.3)^100.23"},
252
+ {"(REQ field:\"one two three\")^23.000", "field:\"one two three\"^23.0"},
253
+ {"asdf~0.2", "asdf~0.2"},
254
+ {"field:asdf~0.2", "field:asdf~0.2"},
255
+ {"asdf~0.2^100.00", "asdf~0.2^100.0"},
256
+ {"field:asdf~0.2^0.1", "field:asdf~0.2^0.1"},
257
+ {"field:\"asdf <> asdf|asdf\"~4", "field:\"asdf <> asdf|asdf\"~4"},
258
+ {"\"one two three four five\"~5", "\"one two three four five\"~5"},
259
+ {"ab?de", "ab?de"},
260
+ {"ab*de", "ab*de"},
261
+ {"asdf?*?asd*dsf?asfd*asdf?", "asdf?*?asd*dsf?asfd*asdf?"},
262
+ {"field:a* AND field:(b*)", "+field:a* +field:b*"},
263
+ {"field:abc~ AND field:(b*)", "+field:abc~ +field:b*"},
264
+ {"asdf?*?asd*dsf?asfd*asdf?^20.0", "asdf?*?asd*dsf?asfd*asdf?^20.0"},
265
+
266
+ {"*:xxx", "xxx f1:xxx f2:xxx field:xxx"},
267
+ {"f1|f2:xxx", "f1:xxx f2:xxx"},
268
+
269
+ {"*:asd~0.2", "asd~0.2 f1:asd~0.2 f2:asd~0.2 field:asd~0.2"},
270
+ {"f1|f2:asd~0.2", "f1:asd~0.2 f2:asd~0.2"},
271
+
272
+ {"*:a?d*^20.0", "(a?d* f1:a?d* f2:a?d* field:a?d*)^20.0"},
273
+ {"f1|f2:a?d*^20.0", "(f1:a?d* f2:a?d*)^20.0"},
274
+
275
+ {"*:\"asdf <> xxx|yyy\"",
276
+ "\"asdf <> xxx|yyy\" f1:\"asdf <> xxx|yyy\" f2:\"asdf <> xxx|yyy\" "
277
+ "field:\"asdf <> xxx|yyy\""},
278
+ {"f1|f2:\"asdf <> xxx|yyy\"",
279
+ "f1:\"asdf <> xxx|yyy\" f2:\"asdf <> xxx|yyy\""},
280
+ {"f1|f2:\"do|yyy\"", "f1:yyy f2:yyy"},
281
+ {"f1|f2:\"asdf <> do|yyy\"",
282
+ "f1:\"asdf <> yyy\" f2:\"asdf <> yyy\""},
283
+
284
+ {"*:[bbb xxx]", "[bbb xxx] f1:[bbb xxx] f2:[bbb xxx] field:[bbb xxx]"},
285
+ {"f1|f2:[bbb xxx]", "f1:[bbb xxx] f2:[bbb xxx]"},
286
+
287
+ {"*:(xxx AND bbb)",
288
+ "+(xxx f1:xxx f2:xxx field:xxx) +(bbb f1:bbb f2:bbb field:bbb)"},
289
+ {"f1|f2:(xxx AND bbb)", "+(f1:xxx f2:xxx) +(f1:bbb f2:bbb)"},
290
+ {"ASDF?*?asd*dsf?ASFD*asdf?^20.0", "asdf?*?asd*dsf?asfd*asdf?^20.0"},
291
+ {"ASDFasdAasAasASD~", "asdfasdaasaasasd~"},
292
+ {"\"onewordphrase\"", "onewordphrase"},
293
+ {"one billion eight hundred and thirty three million four hundred and "
294
+ "eighty eight thousand two hundred and sixty three",
295
+ "one billion eight hundred thirty three million four hundred "
296
+ "eighty eight thousand two hundred sixty three"},
297
+ {"f1:*", "*"},
298
+ {"f1:*^100.0", "*^100.0"},
299
+ {"f1:?*", "f1:?*"},
300
+ {"*:this", ""},
301
+ {"this-is-a-hyphenated-word", "\"thisisahyphenatedword|this is a hyphenated word\"~4"},
302
+ {"\"the phrase and the phrase\"", "\"phrase <> <> phrase\"~3"},
303
+ {"\"the e-mail was in the inbox\"", "\"email|e mail <> <> <> inbox\"~5"},
304
+ {"f1:?*^100.0", "f1:?*^100.0"},
305
+ {"f1:(a1 f2:b2 c3)", "f1:a1 f2:b2 f1:c3"}
306
+ /*
307
+ */
308
+ };
309
+ (void)data;
310
+
311
+ FRT_REF(analyzer);
312
+ parser = frt_qp_new(analyzer);
313
+ frt_qp_add_field(parser, rb_intern("xx"), true, true);
314
+ frt_qp_add_field(parser, rb_intern("f1"), false, true);
315
+ frt_qp_add_field(parser, rb_intern("f2"), false, true);
316
+ frt_qp_add_field(parser, rb_intern("field"), false, true);
317
+
318
+ for (i = 0; i < FRT_NELEMS(pairs); i++) {
319
+ PARSER_TEST(pairs[i].qstr, pairs[i].qres);
320
+ }
321
+ parser->clean_str = true;
322
+ for (i = 0; i < FRT_NELEMS(pairs); i++) {
323
+ PARSER_TEST(pairs[i].qstr, pairs[i].qres);
324
+ }
325
+ PARSER_TEST("not_field:word", "");
326
+ frt_qp_destroy(parser);
327
+
328
+ /* This time let the query parser destroy the analyzer */
329
+ parser = frt_qp_new(analyzer);
330
+ frt_qp_add_field(parser, rb_intern("xx"), true, true);
331
+ frt_qp_add_field(parser, rb_intern("f1"), false, true);
332
+ frt_qp_add_field(parser, rb_intern("f2"), false, true);
333
+ frt_qp_add_field(parser, rb_intern("field"), false, true);
334
+
335
+ parser->clean_str = false;
336
+ parser->allow_any_fields = true;
337
+ for (i = 0; i < FRT_NELEMS(pairs); i++) {
338
+ PARSER_TEST(pairs[i].qstr, pairs[i].qres);
339
+ }
340
+ parser->clean_str = true;
341
+ for (i = 0; i < FRT_NELEMS(pairs); i++) {
342
+ PARSER_TEST(pairs[i].qstr, pairs[i].qres);
343
+ }
344
+ PARSER_TEST("not_field:word", "not_field:word");
345
+
346
+ parser->wild_lower = false;
347
+ PARSER_TEST("ASDF?*?asd*dsf?ASFD*asdf?^20.0", "ASDF?*?asd*dsf?ASFD*asdf?^20.0");
348
+ PARSER_TEST("ASDFasdAasAasASD~", "asdfasdaasaasasd~");
349
+ frt_qp_destroy(parser);
350
+ }
351
+
352
+ static void test_qp_clean_str(TestCase *tc, void *data)
353
+ {
354
+ int i;
355
+ QPTestPair pairs[] = {
356
+ {"", ""},
357
+ {"\"< <>\"", "\"\\< <>\""},
358
+ {"\"hello there", "\"hello there\""},
359
+ {"hello there\"", "hello there\"\""},
360
+ {"(hello there", "(hello there)"},
361
+ {"(hello (there", "(hello (there))"},
362
+ {"(hello\" (there", "(hello\" \\(there\")"},
363
+ {"(hello\" &:()[]{}!+~^-<|>=*? <>there",
364
+ "(hello\" \\&\\:\\(\\)\\[\\]\\{\\}\\!\\+\\~\\^\\-\\<|\\>\\=\\*\\? "
365
+ "<>there\")"},
366
+ {"hello) there)", "((hello) there)"},
367
+ {"hello) \"there)", "(hello) \"there\\)\""},
368
+ {"(hello \\&\\:\\(\\)\\[\\]\\{\\}\\!\\+\\~\\^\\-\\<|\\>\\=\\*\\?",
369
+ "(hello \\&\\:\\(\\)\\[\\]\\{\\}\\!\\+\\~\\^\\-\\<|\\>\\=\\*\\?)"},
370
+ {"hello \\\\&", "hello \\\\&"},
371
+ {"hello \\\\\\&", "hello \\\\\\&"},
372
+ {"\"hello \\\\\"", "\"hello \\\""},
373
+ };
374
+ (void)data;
375
+ for (i = 0; i < FRT_NELEMS(pairs); i++) {
376
+ char *qres = frt_qp_clean_str((char *)pairs[i].qstr);\
377
+ Asequal(pairs[i].qres, qres);\
378
+ free(qres);\
379
+ }
380
+ }
381
+
382
+ static void test_qp_bad_queries(TestCase *tc, void *data)
383
+ {
384
+ int i;
385
+ FrtQParser *parser;
386
+ QPTestPair pairs[] = {
387
+ {"[, ]", ""},
388
+ {"::*word", "word"},
389
+ {"::))*&)(*^&*(", ""},
390
+ {"::|)*&one)(*two(*&\"", "\"one two\"~1"}
391
+ };
392
+ (void)data;
393
+
394
+ parser = frt_qp_new(frt_letter_analyzer_new(true));
395
+ frt_qp_add_field(parser, rb_intern("xx"), true, true);
396
+ frt_qp_add_field(parser, rb_intern("f1"), false, true);
397
+ frt_qp_add_field(parser, rb_intern("f2"), false, true);
398
+ frt_qp_add_field(parser, rb_intern("field"), false, true);
399
+
400
+ parser->handle_parse_errors = true;
401
+
402
+ for (i = 0; i < FRT_NELEMS(pairs); i++) {
403
+ PARSER_TEST(pairs[i].qstr, pairs[i].qres);
404
+ }
405
+ parser->clean_str = true;
406
+ for (i = 0; i < FRT_NELEMS(pairs); i++) {
407
+ PARSER_TEST(pairs[i].qstr, pairs[i].qres);
408
+ }
409
+ frt_qp_destroy(parser);
410
+ }
411
+
412
+ static void test_qp_prefix_query(TestCase *tc, void *data)
413
+ {
414
+ FrtQParser *parser;
415
+ FrtQuery *q;
416
+ (void)data;
417
+
418
+ parser = frt_qp_new(frt_letter_analyzer_new(true));
419
+ frt_qp_add_field(parser, rb_intern("xx"), true, true);
420
+
421
+ q = qp_parse(parser, (char *)"asdg*");
422
+ Aiequal(PREFIX_QUERY, q->type);
423
+ frt_q_deref(q);
424
+ q = qp_parse(parser, (char *)"a?dg*");
425
+ Aiequal(WILD_CARD_QUERY, q->type);
426
+ frt_q_deref(q);
427
+ q = qp_parse(parser, (char *)"a*dg*");
428
+ Aiequal(WILD_CARD_QUERY, q->type);
429
+ frt_q_deref(q);
430
+ q = qp_parse(parser, (char *)"asdg*a");
431
+ Aiequal(WILD_CARD_QUERY, q->type);
432
+ frt_q_deref(q);
433
+ frt_qp_destroy(parser);
434
+ }
435
+
436
+ static void test_qp_keyword_switch(TestCase *tc, void *data)
437
+ {
438
+ FrtQParser *parser;
439
+ (void)data;
440
+
441
+ parser = frt_qp_new(frt_letter_analyzer_new(true));
442
+ frt_qp_add_field(parser, rb_intern("xx"), true, true);
443
+
444
+ PARSER_TEST("REQ www (xxx AND yyy) OR NOT zzz", "+www (+xxx +yyy) -zzz");
445
+
446
+ parser->use_keywords = false;
447
+ PARSER_TEST("REQ www (xxx AND yyy) OR NOT zzz", "req www (xxx and yyy) or not zzz");
448
+
449
+ frt_qp_destroy(parser);
450
+ }
451
+
452
+ TestSuite *ts_q_parser(TestSuite *suite)
453
+ {
454
+ suite = ADD_SUITE(suite);
455
+
456
+ tst_run_test(suite, test_q_parser, NULL);
457
+ tst_run_test(suite, test_q_parser_standard_analyzer, NULL);
458
+ tst_run_test(suite, test_qp_clean_str, NULL);
459
+ tst_run_test(suite, test_qp_bad_queries, NULL);
460
+ tst_run_test(suite, test_qp_prefix_query, NULL);
461
+ tst_run_test(suite, test_qp_keyword_switch, NULL);
462
+
463
+ return suite;
464
+ }