isomorfeus-ferret 0.12.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (222) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +612 -0
  3. data/README.md +44 -0
  4. data/ext/isomorfeus_ferret_ext/benchmark.c +223 -0
  5. data/ext/isomorfeus_ferret_ext/benchmark.h +45 -0
  6. data/ext/isomorfeus_ferret_ext/benchmarks_all.h +25 -0
  7. data/ext/isomorfeus_ferret_ext/bm_bitvector.c +123 -0
  8. data/ext/isomorfeus_ferret_ext/bm_hash.c +118 -0
  9. data/ext/isomorfeus_ferret_ext/bm_micro_string.c +40 -0
  10. data/ext/isomorfeus_ferret_ext/bm_store.c +93 -0
  11. data/ext/isomorfeus_ferret_ext/email.rl +21 -0
  12. data/ext/isomorfeus_ferret_ext/extconf.rb +5 -0
  13. data/ext/isomorfeus_ferret_ext/fio_tmpfile.h +53 -0
  14. data/ext/isomorfeus_ferret_ext/frb_analysis.c +2577 -0
  15. data/ext/isomorfeus_ferret_ext/frb_index.c +3457 -0
  16. data/ext/isomorfeus_ferret_ext/frb_lang.c +9 -0
  17. data/ext/isomorfeus_ferret_ext/frb_lang.h +17 -0
  18. data/ext/isomorfeus_ferret_ext/frb_qparser.c +629 -0
  19. data/ext/isomorfeus_ferret_ext/frb_search.c +4460 -0
  20. data/ext/isomorfeus_ferret_ext/frb_store.c +515 -0
  21. data/ext/isomorfeus_ferret_ext/frb_threading.h +30 -0
  22. data/ext/isomorfeus_ferret_ext/frb_utils.c +1127 -0
  23. data/ext/isomorfeus_ferret_ext/frt_analysis.c +1644 -0
  24. data/ext/isomorfeus_ferret_ext/frt_analysis.h +247 -0
  25. data/ext/isomorfeus_ferret_ext/frt_array.c +124 -0
  26. data/ext/isomorfeus_ferret_ext/frt_array.h +54 -0
  27. data/ext/isomorfeus_ferret_ext/frt_bitvector.c +95 -0
  28. data/ext/isomorfeus_ferret_ext/frt_bitvector.h +586 -0
  29. data/ext/isomorfeus_ferret_ext/frt_compound_io.c +374 -0
  30. data/ext/isomorfeus_ferret_ext/frt_config.h +44 -0
  31. data/ext/isomorfeus_ferret_ext/frt_document.c +134 -0
  32. data/ext/isomorfeus_ferret_ext/frt_document.h +52 -0
  33. data/ext/isomorfeus_ferret_ext/frt_except.c +95 -0
  34. data/ext/isomorfeus_ferret_ext/frt_except.h +188 -0
  35. data/ext/isomorfeus_ferret_ext/frt_field_index.c +233 -0
  36. data/ext/isomorfeus_ferret_ext/frt_field_index.h +42 -0
  37. data/ext/isomorfeus_ferret_ext/frt_filter.c +157 -0
  38. data/ext/isomorfeus_ferret_ext/frt_fs_store.c +502 -0
  39. data/ext/isomorfeus_ferret_ext/frt_global.c +427 -0
  40. data/ext/isomorfeus_ferret_ext/frt_global.h +290 -0
  41. data/ext/isomorfeus_ferret_ext/frt_hash.c +518 -0
  42. data/ext/isomorfeus_ferret_ext/frt_hash.h +466 -0
  43. data/ext/isomorfeus_ferret_ext/frt_hashset.c +191 -0
  44. data/ext/isomorfeus_ferret_ext/frt_hashset.h +206 -0
  45. data/ext/isomorfeus_ferret_ext/frt_helper.c +62 -0
  46. data/ext/isomorfeus_ferret_ext/frt_helper.h +13 -0
  47. data/ext/isomorfeus_ferret_ext/frt_ind.c +353 -0
  48. data/ext/isomorfeus_ferret_ext/frt_ind.h +54 -0
  49. data/ext/isomorfeus_ferret_ext/frt_index.c +6377 -0
  50. data/ext/isomorfeus_ferret_ext/frt_index.h +880 -0
  51. data/ext/isomorfeus_ferret_ext/frt_lang.c +104 -0
  52. data/ext/isomorfeus_ferret_ext/frt_lang.h +44 -0
  53. data/ext/isomorfeus_ferret_ext/frt_mempool.c +87 -0
  54. data/ext/isomorfeus_ferret_ext/frt_mempool.h +33 -0
  55. data/ext/isomorfeus_ferret_ext/frt_multimapper.c +349 -0
  56. data/ext/isomorfeus_ferret_ext/frt_multimapper.h +52 -0
  57. data/ext/isomorfeus_ferret_ext/frt_posh.c +1006 -0
  58. data/ext/isomorfeus_ferret_ext/frt_posh.h +973 -0
  59. data/ext/isomorfeus_ferret_ext/frt_priorityqueue.c +147 -0
  60. data/ext/isomorfeus_ferret_ext/frt_priorityqueue.h +147 -0
  61. data/ext/isomorfeus_ferret_ext/frt_q_boolean.c +1612 -0
  62. data/ext/isomorfeus_ferret_ext/frt_q_const_score.c +157 -0
  63. data/ext/isomorfeus_ferret_ext/frt_q_filtered_query.c +209 -0
  64. data/ext/isomorfeus_ferret_ext/frt_q_fuzzy.c +281 -0
  65. data/ext/isomorfeus_ferret_ext/frt_q_match_all.c +147 -0
  66. data/ext/isomorfeus_ferret_ext/frt_q_multi_term.c +672 -0
  67. data/ext/isomorfeus_ferret_ext/frt_q_parser.c +3084 -0
  68. data/ext/isomorfeus_ferret_ext/frt_q_phrase.c +1182 -0
  69. data/ext/isomorfeus_ferret_ext/frt_q_prefix.c +98 -0
  70. data/ext/isomorfeus_ferret_ext/frt_q_range.c +665 -0
  71. data/ext/isomorfeus_ferret_ext/frt_q_span.c +2386 -0
  72. data/ext/isomorfeus_ferret_ext/frt_q_term.c +311 -0
  73. data/ext/isomorfeus_ferret_ext/frt_q_wildcard.c +166 -0
  74. data/ext/isomorfeus_ferret_ext/frt_ram_store.c +460 -0
  75. data/ext/isomorfeus_ferret_ext/frt_scanner.c +899 -0
  76. data/ext/isomorfeus_ferret_ext/frt_scanner.h +28 -0
  77. data/ext/isomorfeus_ferret_ext/frt_scanner_mb.c +6705 -0
  78. data/ext/isomorfeus_ferret_ext/frt_scanner_utf8.c +4419 -0
  79. data/ext/isomorfeus_ferret_ext/frt_search.c +1824 -0
  80. data/ext/isomorfeus_ferret_ext/frt_search.h +924 -0
  81. data/ext/isomorfeus_ferret_ext/frt_similarity.c +150 -0
  82. data/ext/isomorfeus_ferret_ext/frt_similarity.h +79 -0
  83. data/ext/isomorfeus_ferret_ext/frt_sort.c +796 -0
  84. data/ext/isomorfeus_ferret_ext/frt_stopwords.c +395 -0
  85. data/ext/isomorfeus_ferret_ext/frt_store.c +680 -0
  86. data/ext/isomorfeus_ferret_ext/frt_store.h +789 -0
  87. data/ext/isomorfeus_ferret_ext/frt_term_vectors.c +72 -0
  88. data/ext/isomorfeus_ferret_ext/frt_threading.h +23 -0
  89. data/ext/isomorfeus_ferret_ext/frt_win32.h +54 -0
  90. data/ext/isomorfeus_ferret_ext/isomorfeus_ferret.c +409 -0
  91. data/ext/isomorfeus_ferret_ext/isomorfeus_ferret.h +95 -0
  92. data/ext/isomorfeus_ferret_ext/libstemmer.c +93 -0
  93. data/ext/isomorfeus_ferret_ext/libstemmer.h +73 -0
  94. data/ext/isomorfeus_ferret_ext/q_parser.y +1366 -0
  95. data/ext/isomorfeus_ferret_ext/scanner.h +28 -0
  96. data/ext/isomorfeus_ferret_ext/scanner.in +43 -0
  97. data/ext/isomorfeus_ferret_ext/scanner.rl +84 -0
  98. data/ext/isomorfeus_ferret_ext/scanner_mb.rl +200 -0
  99. data/ext/isomorfeus_ferret_ext/scanner_utf8.rl +85 -0
  100. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_danish.c +324 -0
  101. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_danish.h +7 -0
  102. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_dutch.c +610 -0
  103. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_dutch.h +6 -0
  104. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_english.c +1104 -0
  105. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_english.h +6 -0
  106. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_finnish.c +749 -0
  107. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_finnish.h +7 -0
  108. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_french.c +1233 -0
  109. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_french.h +6 -0
  110. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_german.c +490 -0
  111. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_german.h +6 -0
  112. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_hungarian.c +1217 -0
  113. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_hungarian.h +7 -0
  114. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_italian.c +1052 -0
  115. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_italian.h +6 -0
  116. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_norwegian.c +283 -0
  117. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_norwegian.h +6 -0
  118. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_porter.c +735 -0
  119. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_porter.h +6 -0
  120. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_portuguese.c +1003 -0
  121. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_portuguese.h +7 -0
  122. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_spanish.c +1079 -0
  123. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_spanish.h +6 -0
  124. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_swedish.c +293 -0
  125. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_swedish.h +6 -0
  126. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_2_romanian.c +984 -0
  127. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_2_romanian.h +6 -0
  128. data/ext/isomorfeus_ferret_ext/stem_KOI8_R_russian.c +686 -0
  129. data/ext/isomorfeus_ferret_ext/stem_KOI8_R_russian.h +6 -0
  130. data/ext/isomorfeus_ferret_ext/stem_UTF_8_danish.c +325 -0
  131. data/ext/isomorfeus_ferret_ext/stem_UTF_8_danish.h +6 -0
  132. data/ext/isomorfeus_ferret_ext/stem_UTF_8_dutch.c +620 -0
  133. data/ext/isomorfeus_ferret_ext/stem_UTF_8_dutch.h +6 -0
  134. data/ext/isomorfeus_ferret_ext/stem_UTF_8_english.c +1111 -0
  135. data/ext/isomorfeus_ferret_ext/stem_UTF_8_english.h +6 -0
  136. data/ext/isomorfeus_ferret_ext/stem_UTF_8_finnish.c +754 -0
  137. data/ext/isomorfeus_ferret_ext/stem_UTF_8_finnish.h +6 -0
  138. data/ext/isomorfeus_ferret_ext/stem_UTF_8_french.c +1242 -0
  139. data/ext/isomorfeus_ferret_ext/stem_UTF_8_french.h +6 -0
  140. data/ext/isomorfeus_ferret_ext/stem_UTF_8_german.c +495 -0
  141. data/ext/isomorfeus_ferret_ext/stem_UTF_8_german.h +6 -0
  142. data/ext/isomorfeus_ferret_ext/stem_UTF_8_hungarian.c +1220 -0
  143. data/ext/isomorfeus_ferret_ext/stem_UTF_8_hungarian.h +6 -0
  144. data/ext/isomorfeus_ferret_ext/stem_UTF_8_italian.c +1059 -0
  145. data/ext/isomorfeus_ferret_ext/stem_UTF_8_italian.h +6 -0
  146. data/ext/isomorfeus_ferret_ext/stem_UTF_8_norwegian.c +285 -0
  147. data/ext/isomorfeus_ferret_ext/stem_UTF_8_norwegian.h +6 -0
  148. data/ext/isomorfeus_ferret_ext/stem_UTF_8_porter.c +741 -0
  149. data/ext/isomorfeus_ferret_ext/stem_UTF_8_porter.h +6 -0
  150. data/ext/isomorfeus_ferret_ext/stem_UTF_8_portuguese.c +1009 -0
  151. data/ext/isomorfeus_ferret_ext/stem_UTF_8_portuguese.h +6 -0
  152. data/ext/isomorfeus_ferret_ext/stem_UTF_8_romanian.c +990 -0
  153. data/ext/isomorfeus_ferret_ext/stem_UTF_8_romanian.h +6 -0
  154. data/ext/isomorfeus_ferret_ext/stem_UTF_8_russian.c +680 -0
  155. data/ext/isomorfeus_ferret_ext/stem_UTF_8_russian.h +6 -0
  156. data/ext/isomorfeus_ferret_ext/stem_UTF_8_spanish.c +1083 -0
  157. data/ext/isomorfeus_ferret_ext/stem_UTF_8_spanish.h +6 -0
  158. data/ext/isomorfeus_ferret_ext/stem_UTF_8_swedish.c +294 -0
  159. data/ext/isomorfeus_ferret_ext/stem_UTF_8_swedish.h +6 -0
  160. data/ext/isomorfeus_ferret_ext/stem_UTF_8_turkish.c +2191 -0
  161. data/ext/isomorfeus_ferret_ext/stem_UTF_8_turkish.h +6 -0
  162. data/ext/isomorfeus_ferret_ext/stem_api.c +66 -0
  163. data/ext/isomorfeus_ferret_ext/stem_api.h +26 -0
  164. data/ext/isomorfeus_ferret_ext/stem_header.h +57 -0
  165. data/ext/isomorfeus_ferret_ext/stem_modules.h +190 -0
  166. data/ext/isomorfeus_ferret_ext/stem_modules.txt +50 -0
  167. data/ext/isomorfeus_ferret_ext/stem_utilities.c +478 -0
  168. data/ext/isomorfeus_ferret_ext/test.c +850 -0
  169. data/ext/isomorfeus_ferret_ext/test.h +416 -0
  170. data/ext/isomorfeus_ferret_ext/test_1710.c +63 -0
  171. data/ext/isomorfeus_ferret_ext/test_analysis.c +1221 -0
  172. data/ext/isomorfeus_ferret_ext/test_array.c +272 -0
  173. data/ext/isomorfeus_ferret_ext/test_bitvector.c +600 -0
  174. data/ext/isomorfeus_ferret_ext/test_compound_io.c +170 -0
  175. data/ext/isomorfeus_ferret_ext/test_document.c +156 -0
  176. data/ext/isomorfeus_ferret_ext/test_except.c +244 -0
  177. data/ext/isomorfeus_ferret_ext/test_fields.c +522 -0
  178. data/ext/isomorfeus_ferret_ext/test_file_deleter.c +185 -0
  179. data/ext/isomorfeus_ferret_ext/test_filter.c +331 -0
  180. data/ext/isomorfeus_ferret_ext/test_fs_store.c +25 -0
  181. data/ext/isomorfeus_ferret_ext/test_global.c +299 -0
  182. data/ext/isomorfeus_ferret_ext/test_hash.c +485 -0
  183. data/ext/isomorfeus_ferret_ext/test_hashset.c +288 -0
  184. data/ext/isomorfeus_ferret_ext/test_helper.c +47 -0
  185. data/ext/isomorfeus_ferret_ext/test_highlighter.c +548 -0
  186. data/ext/isomorfeus_ferret_ext/test_index.c +2323 -0
  187. data/ext/isomorfeus_ferret_ext/test_lang.c +74 -0
  188. data/ext/isomorfeus_ferret_ext/test_mempool.c +102 -0
  189. data/ext/isomorfeus_ferret_ext/test_multimapper.c +64 -0
  190. data/ext/isomorfeus_ferret_ext/test_priorityqueue.c +213 -0
  191. data/ext/isomorfeus_ferret_ext/test_q_const_score.c +84 -0
  192. data/ext/isomorfeus_ferret_ext/test_q_filtered.c +61 -0
  193. data/ext/isomorfeus_ferret_ext/test_q_fuzzy.c +241 -0
  194. data/ext/isomorfeus_ferret_ext/test_q_parser.c +464 -0
  195. data/ext/isomorfeus_ferret_ext/test_q_span.c +575 -0
  196. data/ext/isomorfeus_ferret_ext/test_ram_store.c +77 -0
  197. data/ext/isomorfeus_ferret_ext/test_search.c +1874 -0
  198. data/ext/isomorfeus_ferret_ext/test_segments.c +167 -0
  199. data/ext/isomorfeus_ferret_ext/test_similarity.c +25 -0
  200. data/ext/isomorfeus_ferret_ext/test_sort.c +333 -0
  201. data/ext/isomorfeus_ferret_ext/test_store.c +591 -0
  202. data/ext/isomorfeus_ferret_ext/test_store.h +3 -0
  203. data/ext/isomorfeus_ferret_ext/test_term.c +351 -0
  204. data/ext/isomorfeus_ferret_ext/test_term_vectors.c +373 -0
  205. data/ext/isomorfeus_ferret_ext/test_test.c +83 -0
  206. data/ext/isomorfeus_ferret_ext/test_threading.c +188 -0
  207. data/ext/isomorfeus_ferret_ext/testhelper.c +561 -0
  208. data/ext/isomorfeus_ferret_ext/testhelper.h +25 -0
  209. data/ext/isomorfeus_ferret_ext/tests_all.h +87 -0
  210. data/ext/isomorfeus_ferret_ext/uchar-ucs4.rl +1854 -0
  211. data/ext/isomorfeus_ferret_ext/uchar-utf8.rl +1999 -0
  212. data/ext/isomorfeus_ferret_ext/url.rl +27 -0
  213. data/ext/isomorfeus_ferret_ext/word_list.h +15156 -0
  214. data/lib/isomorfeus/ferret/document.rb +132 -0
  215. data/lib/isomorfeus/ferret/field_symbol.rb +85 -0
  216. data/lib/isomorfeus/ferret/index/field_infos.rb +48 -0
  217. data/lib/isomorfeus/ferret/index/index.rb +970 -0
  218. data/lib/isomorfeus/ferret/monitor.rb +323 -0
  219. data/lib/isomorfeus/ferret/stdlib_patches.rb +151 -0
  220. data/lib/isomorfeus/ferret/version.rb +5 -0
  221. data/lib/isomorfeus-ferret.rb +8 -0
  222. metadata +307 -0
@@ -0,0 +1,311 @@
1
+ #include "frt_global.h"
2
+ #include <string.h>
3
+ #include "frt_search.h"
4
+
5
+ #define TQ(query) ((FrtTermQuery *)(query))
6
+ #define TSc(scorer) ((TermScorer *)(scorer))
7
+
8
+ /***************************************************************************
9
+ *
10
+ * TermScorer
11
+ *
12
+ ***************************************************************************/
13
+
14
+ #define SCORE_CACHE_SIZE 32
15
+ #define TDE_READ_SIZE 32
16
+
17
+ typedef struct TermScorer
18
+ {
19
+ FrtScorer super;
20
+ int docs[TDE_READ_SIZE];
21
+ int freqs[TDE_READ_SIZE];
22
+ int pointer;
23
+ int pointer_max;
24
+ float score_cache[SCORE_CACHE_SIZE];
25
+ FrtWeight *weight;
26
+ FrtTermDocEnum *tde;
27
+ frt_uchar *norms;
28
+ float weight_value;
29
+ } TermScorer;
30
+
31
+ static float tsc_score(FrtScorer *self)
32
+ {
33
+ TermScorer *ts = TSc(self);
34
+ int freq = ts->freqs[ts->pointer];
35
+ float score;
36
+ /* compute tf(f)*weight */
37
+ if (freq < SCORE_CACHE_SIZE) { /* check cache */
38
+ score = ts->score_cache[freq]; /* cache hit */
39
+ }
40
+ else {
41
+ /* cache miss */
42
+ score = frt_sim_tf(self->similarity, (float)freq) * ts->weight_value;
43
+ }
44
+ /* normalize for field */
45
+ score *= frt_sim_decode_norm(self->similarity, ts->norms[self->doc]);
46
+ return score;
47
+ }
48
+
49
+ static bool tsc_next(FrtScorer *self)
50
+ {
51
+ TermScorer *ts = TSc(self);
52
+
53
+ ts->pointer++;
54
+ if (ts->pointer >= ts->pointer_max) {
55
+ /* refill buffer */
56
+ ts->pointer_max = ts->tde->read(ts->tde, ts->docs, ts->freqs,
57
+ TDE_READ_SIZE);
58
+ if (ts->pointer_max != 0) {
59
+ ts->pointer = 0;
60
+ }
61
+ else {
62
+ return false;
63
+ }
64
+ }
65
+ self->doc = ts->docs[ts->pointer];
66
+ return true;
67
+ }
68
+
69
+ static bool tsc_skip_to(FrtScorer *self, int doc_num)
70
+ {
71
+ TermScorer *ts = TSc(self);
72
+ FrtTermDocEnum *tde = ts->tde;
73
+
74
+ /* first scan in cache */
75
+ while (++(ts->pointer) < ts->pointer_max) {
76
+ if (ts->docs[ts->pointer] >= doc_num) {
77
+ self->doc = ts->docs[ts->pointer];
78
+ return true;
79
+ }
80
+ }
81
+
82
+ /* not found in cache, seek underlying stream */
83
+ if (tde->skip_to(tde, doc_num)) {
84
+ ts->pointer_max = 1;
85
+ ts->pointer = 0;
86
+ ts->docs[0] = self->doc = tde->doc_num(tde);
87
+ ts->freqs[0] = tde->freq(tde);
88
+ return true;
89
+ }
90
+ else {
91
+ return false;
92
+ }
93
+ }
94
+
95
+ static FrtExplanation *tsc_explain(FrtScorer *self, int doc_num)
96
+ {
97
+ TermScorer *ts = TSc(self);
98
+ FrtQuery *query = ts->weight->get_query(ts->weight);
99
+ int tf = 0;
100
+
101
+ tsc_skip_to(self, doc_num);
102
+ if (self->doc == doc_num) {
103
+ tf = ts->freqs[ts->pointer];
104
+ }
105
+ return frt_expl_new(frt_sim_tf(self->similarity, (float)tf),
106
+ "tf(term_freq(%s:%s)=%d)",
107
+ rb_id2name(TQ(query)->field), TQ(query)->term, tf);
108
+ }
109
+
110
+ static void tsc_destroy(FrtScorer *self)
111
+ {
112
+ TSc(self)->tde->close(TSc(self)->tde);
113
+ frt_scorer_destroy_i(self);
114
+ }
115
+
116
+ static FrtScorer *tsc_new(FrtWeight *weight, FrtTermDocEnum *tde, frt_uchar *norms)
117
+ {
118
+ int i;
119
+ FrtScorer *self = frt_scorer_new(TermScorer, weight->similarity);
120
+ TSc(self)->weight = weight;
121
+ TSc(self)->tde = tde;
122
+ TSc(self)->norms = norms;
123
+ TSc(self)->weight_value = weight->value;
124
+
125
+ for (i = 0; i < SCORE_CACHE_SIZE; i++) {
126
+ TSc(self)->score_cache[i]
127
+ = frt_sim_tf(self->similarity, (float)i) * TSc(self)->weight_value;
128
+ }
129
+
130
+ self->score = &tsc_score;
131
+ self->next = &tsc_next;
132
+ self->skip_to = &tsc_skip_to;
133
+ self->explain = &tsc_explain;
134
+ self->destroy = &tsc_destroy;
135
+ return self;
136
+ }
137
+
138
+ /***************************************************************************
139
+ *
140
+ * TermWeight
141
+ *
142
+ ***************************************************************************/
143
+
144
+ static FrtScorer *tw_scorer(FrtWeight *self, FrtIndexReader *ir)
145
+ {
146
+ FrtTermQuery *tq = TQ(self->query);
147
+ FrtTermDocEnum *tde = ir_term_docs_for(ir, tq->field, tq->term);
148
+ /* ir_term_docs_for should always return a TermDocEnum */
149
+ assert(NULL != tde);
150
+
151
+ return tsc_new(self, tde, frt_ir_get_norms(ir, tq->field));
152
+ }
153
+
154
+ static FrtExplanation *tw_explain(FrtWeight *self, FrtIndexReader *ir, int doc_num)
155
+ {
156
+ FrtExplanation *qnorm_expl;
157
+ FrtExplanation *field_expl;
158
+ FrtScorer *scorer;
159
+ FrtExplanation *tf_expl;
160
+ frt_uchar *field_norms;
161
+ float field_norm;
162
+ FrtExplanation *field_norm_expl;
163
+ char *query_str = self->query->to_s(self->query, (FrtSymbol)NULL);
164
+ FrtTermQuery *tq = TQ(self->query);
165
+ char *term = tq->term;
166
+ FrtExplanation *expl = frt_expl_new(0.0, "weight(%s in %d), product of:", query_str, doc_num);
167
+ /* We need two of these as it's included in both the query explanation
168
+ * and the field explanation */
169
+ FrtExplanation *idf_expl1 = frt_expl_new(self->idf, "idf(doc_freq=%d)", frt_ir_doc_freq(ir, tq->field, term));
170
+ FrtExplanation *idf_expl2 = frt_expl_new(self->idf, "idf(doc_freq=%d)", frt_ir_doc_freq(ir, tq->field, term));
171
+ /* explain query weight */
172
+ FrtExplanation *query_expl = frt_expl_new(0.0, "query_weight(%s), product of:", query_str);
173
+ free(query_str);
174
+ if (self->query->boost != 1.0) {
175
+ frt_expl_add_detail(query_expl, frt_expl_new(self->query->boost, "boost"));
176
+ }
177
+ frt_expl_add_detail(query_expl, idf_expl1);
178
+ qnorm_expl = frt_expl_new(self->qnorm, "query_norm");
179
+ frt_expl_add_detail(query_expl, qnorm_expl);
180
+ query_expl->value = self->query->boost
181
+ * idf_expl1->value * qnorm_expl->value;
182
+ frt_expl_add_detail(expl, query_expl);
183
+ /* explain field weight */
184
+ field_expl = frt_expl_new(0.0, "field_weight(%s:%s in %d), product of:", rb_id2name(tq->field), term, doc_num);
185
+ scorer = self->scorer(self, ir);
186
+ tf_expl = scorer->explain(scorer, doc_num);
187
+ scorer->destroy(scorer);
188
+ frt_expl_add_detail(field_expl, tf_expl);
189
+ frt_expl_add_detail(field_expl, idf_expl2);
190
+ field_norms = frt_ir_get_norms(ir, tq->field);
191
+ field_norm = (field_norms ? frt_sim_decode_norm(self->similarity, field_norms[doc_num]) : (float)0.0);
192
+ field_norm_expl = frt_expl_new(field_norm, "field_norm(field=%s, doc=%d)", rb_id2name(tq->field), doc_num);
193
+ frt_expl_add_detail(field_expl, field_norm_expl);
194
+ field_expl->value = tf_expl->value * idf_expl2->value * field_norm_expl->value;
195
+ /* combine them */
196
+ if (query_expl->value == 1.0) {
197
+ frt_expl_destroy(expl);
198
+ return field_expl;
199
+ } else {
200
+ expl->value = (query_expl->value * field_expl->value);
201
+ frt_expl_add_detail(expl, field_expl);
202
+ return expl;
203
+ }
204
+ }
205
+
206
+ static char *tw_to_s(FrtWeight *self)
207
+ {
208
+ return frt_strfmt("TermWeight(%f)", self->value);
209
+ }
210
+
211
+ static FrtWeight *tw_new(FrtQuery *query, FrtSearcher *searcher)
212
+ {
213
+ FrtWeight *self = w_new(FrtWeight, query);
214
+ self->scorer = &tw_scorer;
215
+ self->explain = &tw_explain;
216
+ self->to_s = &tw_to_s;
217
+
218
+ self->similarity = query->get_similarity(query, searcher);
219
+ self->idf = frt_sim_idf(self->similarity,
220
+ searcher->doc_freq(searcher,
221
+ TQ(query)->field,
222
+ TQ(query)->term),
223
+ searcher->max_doc(searcher)); /* compute idf */
224
+
225
+ return self;
226
+ }
227
+
228
+ /***************************************************************************
229
+ *
230
+ * TermQuery
231
+ *
232
+ ***************************************************************************/
233
+
234
+ static void tq_destroy(FrtQuery *self)
235
+ {
236
+ free(TQ(self)->term);
237
+ frt_q_destroy_i(self);
238
+ }
239
+
240
+ static char *tq_to_s(FrtQuery *self, FrtSymbol default_field)
241
+ {
242
+ const char *field_name = rb_id2name(TQ(self)->field);
243
+ size_t flen = strlen(field_name);
244
+ const char *term = TQ(self)->term;
245
+ size_t tlen = strlen(term);
246
+ char *buffer = FRT_ALLOC_N(char, 34 + flen + tlen);
247
+ char *b = buffer;
248
+ if (default_field != TQ(self)->field) {
249
+ memcpy(b, field_name, sizeof(char) * flen);
250
+ b[flen] = ':';
251
+ b += flen + 1;
252
+ }
253
+ memcpy(b, term, tlen);
254
+ b += tlen;
255
+ *b = 0;
256
+ if (self->boost != 1.0) {
257
+ *b = '^';
258
+ frt_dbl_to_s(b+1, self->boost);
259
+ }
260
+ return buffer;
261
+ }
262
+
263
+ static void tq_extract_terms(FrtQuery *self, FrtHashSet *terms)
264
+ {
265
+ frt_hs_add(terms, frt_term_new(TQ(self)->field, TQ(self)->term));
266
+ }
267
+
268
+ static unsigned long long tq_hash(FrtQuery *self)
269
+ {
270
+ return frt_str_hash(TQ(self)->term) ^ frt_str_hash(rb_id2name(TQ(self)->field));
271
+ }
272
+
273
+ static int tq_eq(FrtQuery *self, FrtQuery *o)
274
+ {
275
+ return (strcmp(TQ(self)->term, TQ(o)->term) == 0)
276
+ && (TQ(self)->field == TQ(o)->field);
277
+ }
278
+
279
+ static FrtMatchVector *tq_get_matchv_i(FrtQuery *self, FrtMatchVector *mv,
280
+ FrtTermVector *tv)
281
+ {
282
+ if (tv->field == TQ(self)->field) {
283
+ int i;
284
+ FrtTVTerm *tv_term = frt_tv_get_tv_term(tv, TQ(self)->term);
285
+ if (tv_term) {
286
+ for (i = 0; i < tv_term->freq; i++) {
287
+ int pos = tv_term->positions[i];
288
+ frt_matchv_add(mv, pos, pos);
289
+ }
290
+ }
291
+ }
292
+ return mv;
293
+ }
294
+
295
+ FrtQuery *frt_tq_new(FrtSymbol field, const char *term)
296
+ {
297
+ FrtQuery *self = frt_q_new(FrtTermQuery);
298
+ TQ(self)->field = field;
299
+ TQ(self)->term = frt_estrdup(term);
300
+ self->type = TERM_QUERY;
301
+ self->extract_terms = &tq_extract_terms;
302
+ self->to_s = &tq_to_s;
303
+ self->hash = &tq_hash;
304
+ self->eq = &tq_eq;
305
+
306
+ self->destroy_i = &tq_destroy;
307
+ self->create_weight_i = &tw_new;
308
+ self->get_matchv_i = &tq_get_matchv_i;
309
+
310
+ return self;
311
+ }
@@ -0,0 +1,166 @@
1
+ #include <string.h>
2
+ #include "frt_global.h"
3
+ #include "frt_search.h"
4
+
5
+ /****************************************************************************
6
+ *
7
+ * FrtWildCardQuery
8
+ *
9
+ ****************************************************************************/
10
+
11
+ #define WCQ(query) ((FrtWildCardQuery *)(query))
12
+
13
+ static char *wcq_to_s(FrtQuery *self, FrtSymbol default_field)
14
+ {
15
+ char *buffer, *bptr;
16
+ const char *field_name = rb_id2name(WCQ(self)->field);
17
+ const char *pattern = WCQ(self)->pattern;
18
+ bptr = buffer = FRT_ALLOC_N(char, strlen(pattern) + strlen(field_name) + 35);
19
+
20
+ if (WCQ(self)->field != default_field) {
21
+ bptr += sprintf(bptr, "%s:", field_name);
22
+ }
23
+ bptr += sprintf(bptr, "%s", pattern);
24
+
25
+ if (self->boost != 1.0) {
26
+ *bptr = '^';
27
+ frt_dbl_to_s(++bptr, self->boost);
28
+ }
29
+
30
+ return buffer;
31
+ }
32
+
33
+ bool frt_wc_match(const char *pattern, const char *text)
34
+ {
35
+ const char *p = pattern, *t = text, *xt;
36
+
37
+ /* include '\0' as we need to match empty string */
38
+ const char *text_last = t + strlen(t);
39
+
40
+ for (;; p++, t++) {
41
+
42
+ /* end of text so make sure end of pattern doesn't matter */
43
+ if (*t == '\0') {
44
+ while (*p) {
45
+ if (*p != FRT_WILD_STRING) {
46
+ return false;
47
+ }
48
+ p++;
49
+ }
50
+ return true;
51
+ }
52
+
53
+ /* If we've gone past the end of the pattern, return false. */
54
+ if (*p == '\0') {
55
+ return false;
56
+ }
57
+
58
+ /* Match a single character, so continue. */
59
+ if (*p == FRT_WILD_CHAR) {
60
+ continue;
61
+ }
62
+
63
+ if (*p == FRT_WILD_STRING) {
64
+ /* Look at the character beyond the '*'. */
65
+ p++;
66
+ /* Examine the string, starting at the last character. */
67
+ for (xt = text_last; xt >= t; xt--) {
68
+ if (frt_wc_match(p, xt)) return true;
69
+ }
70
+ return false;
71
+ }
72
+ if (*p != *t) {
73
+ return false;
74
+ }
75
+ }
76
+
77
+ return false;
78
+ }
79
+
80
+ static FrtQuery *wcq_rewrite(FrtQuery *self, FrtIndexReader *ir)
81
+ {
82
+ FrtQuery *q;
83
+ const char *pattern = WCQ(self)->pattern;
84
+ const char *first_star = strchr(pattern, FRT_WILD_STRING);
85
+ const char *first_ques = strchr(pattern, FRT_WILD_CHAR);
86
+
87
+ if (NULL == first_star && NULL == first_ques) {
88
+ q = frt_tq_new(WCQ(self)->field, pattern);
89
+ q->boost = self->boost;
90
+ }
91
+ else {
92
+ const int field_num = frt_fis_get_field_num(ir->fis, WCQ(self)->field);
93
+ q = frt_multi_tq_new_conf(WCQ(self)->field, FrtMTQMaxTerms(self), 0.0);
94
+
95
+ if (field_num >= 0) {
96
+ FrtTermEnum *te;
97
+ char prefix[FRT_MAX_WORD_SIZE] = "";
98
+ int prefix_len;
99
+
100
+ pattern = (first_ques && (!first_star || first_star > first_ques))
101
+ ? first_ques : first_star;
102
+
103
+ prefix_len = (int)(pattern - WCQ(self)->pattern);
104
+
105
+ if (prefix_len > 0) {
106
+ memcpy(prefix, WCQ(self)->pattern, prefix_len);
107
+ prefix[prefix_len] = '\0';
108
+ }
109
+
110
+ te = ir->terms_from(ir, field_num, prefix);
111
+
112
+ if (te != NULL) {
113
+ const char *term = te->curr_term;
114
+ const char *pat_term = term + prefix_len;
115
+ do {
116
+ if (prefix[0] && strncmp(term, prefix, prefix_len) != 0) {
117
+ break;
118
+ }
119
+
120
+ if (frt_wc_match(pattern, pat_term)) {
121
+ frt_multi_tq_add_term(q, term);
122
+ }
123
+ } while (te->next(te) != NULL);
124
+ te->close(te);
125
+ }
126
+ }
127
+ }
128
+
129
+ return q;
130
+ }
131
+
132
+ static void wcq_destroy(FrtQuery *self)
133
+ {
134
+ free(WCQ(self)->pattern);
135
+ frt_q_destroy_i(self);
136
+ }
137
+
138
+ static unsigned long long wcq_hash(FrtQuery *self)
139
+ {
140
+ return frt_str_hash(rb_id2name(WCQ(self)->field)) ^ frt_str_hash(WCQ(self)->pattern);
141
+ }
142
+
143
+ static int wcq_eq(FrtQuery *self, FrtQuery *o)
144
+ {
145
+ return (strcmp(WCQ(self)->pattern, WCQ(o)->pattern) == 0)
146
+ && (WCQ(self)->field == WCQ(o)->field);
147
+ }
148
+
149
+ FrtQuery *frt_wcq_new(FrtSymbol field, const char *pattern)
150
+ {
151
+ FrtQuery *self = frt_q_new(FrtWildCardQuery);
152
+
153
+ WCQ(self)->field = field;
154
+ WCQ(self)->pattern = frt_estrdup(pattern);
155
+ FrtMTQMaxTerms(self) = FRT_WILD_CARD_QUERY_MAX_TERMS;
156
+
157
+ self->type = WILD_CARD_QUERY;
158
+ self->rewrite = &wcq_rewrite;
159
+ self->to_s = &wcq_to_s;
160
+ self->hash = &wcq_hash;
161
+ self->eq = &wcq_eq;
162
+ self->destroy_i = &wcq_destroy;
163
+ self->create_weight_i = &frt_q_create_weight_unsup;
164
+
165
+ return self;
166
+ }