isomorfeus-ferret 0.12.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (222) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +612 -0
  3. data/README.md +44 -0
  4. data/ext/isomorfeus_ferret_ext/benchmark.c +223 -0
  5. data/ext/isomorfeus_ferret_ext/benchmark.h +45 -0
  6. data/ext/isomorfeus_ferret_ext/benchmarks_all.h +25 -0
  7. data/ext/isomorfeus_ferret_ext/bm_bitvector.c +123 -0
  8. data/ext/isomorfeus_ferret_ext/bm_hash.c +118 -0
  9. data/ext/isomorfeus_ferret_ext/bm_micro_string.c +40 -0
  10. data/ext/isomorfeus_ferret_ext/bm_store.c +93 -0
  11. data/ext/isomorfeus_ferret_ext/email.rl +21 -0
  12. data/ext/isomorfeus_ferret_ext/extconf.rb +5 -0
  13. data/ext/isomorfeus_ferret_ext/fio_tmpfile.h +53 -0
  14. data/ext/isomorfeus_ferret_ext/frb_analysis.c +2577 -0
  15. data/ext/isomorfeus_ferret_ext/frb_index.c +3457 -0
  16. data/ext/isomorfeus_ferret_ext/frb_lang.c +9 -0
  17. data/ext/isomorfeus_ferret_ext/frb_lang.h +17 -0
  18. data/ext/isomorfeus_ferret_ext/frb_qparser.c +629 -0
  19. data/ext/isomorfeus_ferret_ext/frb_search.c +4460 -0
  20. data/ext/isomorfeus_ferret_ext/frb_store.c +515 -0
  21. data/ext/isomorfeus_ferret_ext/frb_threading.h +30 -0
  22. data/ext/isomorfeus_ferret_ext/frb_utils.c +1127 -0
  23. data/ext/isomorfeus_ferret_ext/frt_analysis.c +1644 -0
  24. data/ext/isomorfeus_ferret_ext/frt_analysis.h +247 -0
  25. data/ext/isomorfeus_ferret_ext/frt_array.c +124 -0
  26. data/ext/isomorfeus_ferret_ext/frt_array.h +54 -0
  27. data/ext/isomorfeus_ferret_ext/frt_bitvector.c +95 -0
  28. data/ext/isomorfeus_ferret_ext/frt_bitvector.h +586 -0
  29. data/ext/isomorfeus_ferret_ext/frt_compound_io.c +374 -0
  30. data/ext/isomorfeus_ferret_ext/frt_config.h +44 -0
  31. data/ext/isomorfeus_ferret_ext/frt_document.c +134 -0
  32. data/ext/isomorfeus_ferret_ext/frt_document.h +52 -0
  33. data/ext/isomorfeus_ferret_ext/frt_except.c +95 -0
  34. data/ext/isomorfeus_ferret_ext/frt_except.h +188 -0
  35. data/ext/isomorfeus_ferret_ext/frt_field_index.c +233 -0
  36. data/ext/isomorfeus_ferret_ext/frt_field_index.h +42 -0
  37. data/ext/isomorfeus_ferret_ext/frt_filter.c +157 -0
  38. data/ext/isomorfeus_ferret_ext/frt_fs_store.c +502 -0
  39. data/ext/isomorfeus_ferret_ext/frt_global.c +427 -0
  40. data/ext/isomorfeus_ferret_ext/frt_global.h +290 -0
  41. data/ext/isomorfeus_ferret_ext/frt_hash.c +518 -0
  42. data/ext/isomorfeus_ferret_ext/frt_hash.h +466 -0
  43. data/ext/isomorfeus_ferret_ext/frt_hashset.c +191 -0
  44. data/ext/isomorfeus_ferret_ext/frt_hashset.h +206 -0
  45. data/ext/isomorfeus_ferret_ext/frt_helper.c +62 -0
  46. data/ext/isomorfeus_ferret_ext/frt_helper.h +13 -0
  47. data/ext/isomorfeus_ferret_ext/frt_ind.c +353 -0
  48. data/ext/isomorfeus_ferret_ext/frt_ind.h +54 -0
  49. data/ext/isomorfeus_ferret_ext/frt_index.c +6377 -0
  50. data/ext/isomorfeus_ferret_ext/frt_index.h +880 -0
  51. data/ext/isomorfeus_ferret_ext/frt_lang.c +104 -0
  52. data/ext/isomorfeus_ferret_ext/frt_lang.h +44 -0
  53. data/ext/isomorfeus_ferret_ext/frt_mempool.c +87 -0
  54. data/ext/isomorfeus_ferret_ext/frt_mempool.h +33 -0
  55. data/ext/isomorfeus_ferret_ext/frt_multimapper.c +349 -0
  56. data/ext/isomorfeus_ferret_ext/frt_multimapper.h +52 -0
  57. data/ext/isomorfeus_ferret_ext/frt_posh.c +1006 -0
  58. data/ext/isomorfeus_ferret_ext/frt_posh.h +973 -0
  59. data/ext/isomorfeus_ferret_ext/frt_priorityqueue.c +147 -0
  60. data/ext/isomorfeus_ferret_ext/frt_priorityqueue.h +147 -0
  61. data/ext/isomorfeus_ferret_ext/frt_q_boolean.c +1612 -0
  62. data/ext/isomorfeus_ferret_ext/frt_q_const_score.c +157 -0
  63. data/ext/isomorfeus_ferret_ext/frt_q_filtered_query.c +209 -0
  64. data/ext/isomorfeus_ferret_ext/frt_q_fuzzy.c +281 -0
  65. data/ext/isomorfeus_ferret_ext/frt_q_match_all.c +147 -0
  66. data/ext/isomorfeus_ferret_ext/frt_q_multi_term.c +672 -0
  67. data/ext/isomorfeus_ferret_ext/frt_q_parser.c +3084 -0
  68. data/ext/isomorfeus_ferret_ext/frt_q_phrase.c +1182 -0
  69. data/ext/isomorfeus_ferret_ext/frt_q_prefix.c +98 -0
  70. data/ext/isomorfeus_ferret_ext/frt_q_range.c +665 -0
  71. data/ext/isomorfeus_ferret_ext/frt_q_span.c +2386 -0
  72. data/ext/isomorfeus_ferret_ext/frt_q_term.c +311 -0
  73. data/ext/isomorfeus_ferret_ext/frt_q_wildcard.c +166 -0
  74. data/ext/isomorfeus_ferret_ext/frt_ram_store.c +460 -0
  75. data/ext/isomorfeus_ferret_ext/frt_scanner.c +899 -0
  76. data/ext/isomorfeus_ferret_ext/frt_scanner.h +28 -0
  77. data/ext/isomorfeus_ferret_ext/frt_scanner_mb.c +6705 -0
  78. data/ext/isomorfeus_ferret_ext/frt_scanner_utf8.c +4419 -0
  79. data/ext/isomorfeus_ferret_ext/frt_search.c +1824 -0
  80. data/ext/isomorfeus_ferret_ext/frt_search.h +924 -0
  81. data/ext/isomorfeus_ferret_ext/frt_similarity.c +150 -0
  82. data/ext/isomorfeus_ferret_ext/frt_similarity.h +79 -0
  83. data/ext/isomorfeus_ferret_ext/frt_sort.c +796 -0
  84. data/ext/isomorfeus_ferret_ext/frt_stopwords.c +395 -0
  85. data/ext/isomorfeus_ferret_ext/frt_store.c +680 -0
  86. data/ext/isomorfeus_ferret_ext/frt_store.h +789 -0
  87. data/ext/isomorfeus_ferret_ext/frt_term_vectors.c +72 -0
  88. data/ext/isomorfeus_ferret_ext/frt_threading.h +23 -0
  89. data/ext/isomorfeus_ferret_ext/frt_win32.h +54 -0
  90. data/ext/isomorfeus_ferret_ext/isomorfeus_ferret.c +409 -0
  91. data/ext/isomorfeus_ferret_ext/isomorfeus_ferret.h +95 -0
  92. data/ext/isomorfeus_ferret_ext/libstemmer.c +93 -0
  93. data/ext/isomorfeus_ferret_ext/libstemmer.h +73 -0
  94. data/ext/isomorfeus_ferret_ext/q_parser.y +1366 -0
  95. data/ext/isomorfeus_ferret_ext/scanner.h +28 -0
  96. data/ext/isomorfeus_ferret_ext/scanner.in +43 -0
  97. data/ext/isomorfeus_ferret_ext/scanner.rl +84 -0
  98. data/ext/isomorfeus_ferret_ext/scanner_mb.rl +200 -0
  99. data/ext/isomorfeus_ferret_ext/scanner_utf8.rl +85 -0
  100. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_danish.c +324 -0
  101. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_danish.h +7 -0
  102. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_dutch.c +610 -0
  103. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_dutch.h +6 -0
  104. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_english.c +1104 -0
  105. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_english.h +6 -0
  106. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_finnish.c +749 -0
  107. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_finnish.h +7 -0
  108. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_french.c +1233 -0
  109. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_french.h +6 -0
  110. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_german.c +490 -0
  111. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_german.h +6 -0
  112. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_hungarian.c +1217 -0
  113. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_hungarian.h +7 -0
  114. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_italian.c +1052 -0
  115. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_italian.h +6 -0
  116. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_norwegian.c +283 -0
  117. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_norwegian.h +6 -0
  118. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_porter.c +735 -0
  119. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_porter.h +6 -0
  120. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_portuguese.c +1003 -0
  121. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_portuguese.h +7 -0
  122. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_spanish.c +1079 -0
  123. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_spanish.h +6 -0
  124. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_swedish.c +293 -0
  125. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_swedish.h +6 -0
  126. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_2_romanian.c +984 -0
  127. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_2_romanian.h +6 -0
  128. data/ext/isomorfeus_ferret_ext/stem_KOI8_R_russian.c +686 -0
  129. data/ext/isomorfeus_ferret_ext/stem_KOI8_R_russian.h +6 -0
  130. data/ext/isomorfeus_ferret_ext/stem_UTF_8_danish.c +325 -0
  131. data/ext/isomorfeus_ferret_ext/stem_UTF_8_danish.h +6 -0
  132. data/ext/isomorfeus_ferret_ext/stem_UTF_8_dutch.c +620 -0
  133. data/ext/isomorfeus_ferret_ext/stem_UTF_8_dutch.h +6 -0
  134. data/ext/isomorfeus_ferret_ext/stem_UTF_8_english.c +1111 -0
  135. data/ext/isomorfeus_ferret_ext/stem_UTF_8_english.h +6 -0
  136. data/ext/isomorfeus_ferret_ext/stem_UTF_8_finnish.c +754 -0
  137. data/ext/isomorfeus_ferret_ext/stem_UTF_8_finnish.h +6 -0
  138. data/ext/isomorfeus_ferret_ext/stem_UTF_8_french.c +1242 -0
  139. data/ext/isomorfeus_ferret_ext/stem_UTF_8_french.h +6 -0
  140. data/ext/isomorfeus_ferret_ext/stem_UTF_8_german.c +495 -0
  141. data/ext/isomorfeus_ferret_ext/stem_UTF_8_german.h +6 -0
  142. data/ext/isomorfeus_ferret_ext/stem_UTF_8_hungarian.c +1220 -0
  143. data/ext/isomorfeus_ferret_ext/stem_UTF_8_hungarian.h +6 -0
  144. data/ext/isomorfeus_ferret_ext/stem_UTF_8_italian.c +1059 -0
  145. data/ext/isomorfeus_ferret_ext/stem_UTF_8_italian.h +6 -0
  146. data/ext/isomorfeus_ferret_ext/stem_UTF_8_norwegian.c +285 -0
  147. data/ext/isomorfeus_ferret_ext/stem_UTF_8_norwegian.h +6 -0
  148. data/ext/isomorfeus_ferret_ext/stem_UTF_8_porter.c +741 -0
  149. data/ext/isomorfeus_ferret_ext/stem_UTF_8_porter.h +6 -0
  150. data/ext/isomorfeus_ferret_ext/stem_UTF_8_portuguese.c +1009 -0
  151. data/ext/isomorfeus_ferret_ext/stem_UTF_8_portuguese.h +6 -0
  152. data/ext/isomorfeus_ferret_ext/stem_UTF_8_romanian.c +990 -0
  153. data/ext/isomorfeus_ferret_ext/stem_UTF_8_romanian.h +6 -0
  154. data/ext/isomorfeus_ferret_ext/stem_UTF_8_russian.c +680 -0
  155. data/ext/isomorfeus_ferret_ext/stem_UTF_8_russian.h +6 -0
  156. data/ext/isomorfeus_ferret_ext/stem_UTF_8_spanish.c +1083 -0
  157. data/ext/isomorfeus_ferret_ext/stem_UTF_8_spanish.h +6 -0
  158. data/ext/isomorfeus_ferret_ext/stem_UTF_8_swedish.c +294 -0
  159. data/ext/isomorfeus_ferret_ext/stem_UTF_8_swedish.h +6 -0
  160. data/ext/isomorfeus_ferret_ext/stem_UTF_8_turkish.c +2191 -0
  161. data/ext/isomorfeus_ferret_ext/stem_UTF_8_turkish.h +6 -0
  162. data/ext/isomorfeus_ferret_ext/stem_api.c +66 -0
  163. data/ext/isomorfeus_ferret_ext/stem_api.h +26 -0
  164. data/ext/isomorfeus_ferret_ext/stem_header.h +57 -0
  165. data/ext/isomorfeus_ferret_ext/stem_modules.h +190 -0
  166. data/ext/isomorfeus_ferret_ext/stem_modules.txt +50 -0
  167. data/ext/isomorfeus_ferret_ext/stem_utilities.c +478 -0
  168. data/ext/isomorfeus_ferret_ext/test.c +850 -0
  169. data/ext/isomorfeus_ferret_ext/test.h +416 -0
  170. data/ext/isomorfeus_ferret_ext/test_1710.c +63 -0
  171. data/ext/isomorfeus_ferret_ext/test_analysis.c +1221 -0
  172. data/ext/isomorfeus_ferret_ext/test_array.c +272 -0
  173. data/ext/isomorfeus_ferret_ext/test_bitvector.c +600 -0
  174. data/ext/isomorfeus_ferret_ext/test_compound_io.c +170 -0
  175. data/ext/isomorfeus_ferret_ext/test_document.c +156 -0
  176. data/ext/isomorfeus_ferret_ext/test_except.c +244 -0
  177. data/ext/isomorfeus_ferret_ext/test_fields.c +522 -0
  178. data/ext/isomorfeus_ferret_ext/test_file_deleter.c +185 -0
  179. data/ext/isomorfeus_ferret_ext/test_filter.c +331 -0
  180. data/ext/isomorfeus_ferret_ext/test_fs_store.c +25 -0
  181. data/ext/isomorfeus_ferret_ext/test_global.c +299 -0
  182. data/ext/isomorfeus_ferret_ext/test_hash.c +485 -0
  183. data/ext/isomorfeus_ferret_ext/test_hashset.c +288 -0
  184. data/ext/isomorfeus_ferret_ext/test_helper.c +47 -0
  185. data/ext/isomorfeus_ferret_ext/test_highlighter.c +548 -0
  186. data/ext/isomorfeus_ferret_ext/test_index.c +2323 -0
  187. data/ext/isomorfeus_ferret_ext/test_lang.c +74 -0
  188. data/ext/isomorfeus_ferret_ext/test_mempool.c +102 -0
  189. data/ext/isomorfeus_ferret_ext/test_multimapper.c +64 -0
  190. data/ext/isomorfeus_ferret_ext/test_priorityqueue.c +213 -0
  191. data/ext/isomorfeus_ferret_ext/test_q_const_score.c +84 -0
  192. data/ext/isomorfeus_ferret_ext/test_q_filtered.c +61 -0
  193. data/ext/isomorfeus_ferret_ext/test_q_fuzzy.c +241 -0
  194. data/ext/isomorfeus_ferret_ext/test_q_parser.c +464 -0
  195. data/ext/isomorfeus_ferret_ext/test_q_span.c +575 -0
  196. data/ext/isomorfeus_ferret_ext/test_ram_store.c +77 -0
  197. data/ext/isomorfeus_ferret_ext/test_search.c +1874 -0
  198. data/ext/isomorfeus_ferret_ext/test_segments.c +167 -0
  199. data/ext/isomorfeus_ferret_ext/test_similarity.c +25 -0
  200. data/ext/isomorfeus_ferret_ext/test_sort.c +333 -0
  201. data/ext/isomorfeus_ferret_ext/test_store.c +591 -0
  202. data/ext/isomorfeus_ferret_ext/test_store.h +3 -0
  203. data/ext/isomorfeus_ferret_ext/test_term.c +351 -0
  204. data/ext/isomorfeus_ferret_ext/test_term_vectors.c +373 -0
  205. data/ext/isomorfeus_ferret_ext/test_test.c +83 -0
  206. data/ext/isomorfeus_ferret_ext/test_threading.c +188 -0
  207. data/ext/isomorfeus_ferret_ext/testhelper.c +561 -0
  208. data/ext/isomorfeus_ferret_ext/testhelper.h +25 -0
  209. data/ext/isomorfeus_ferret_ext/tests_all.h +87 -0
  210. data/ext/isomorfeus_ferret_ext/uchar-ucs4.rl +1854 -0
  211. data/ext/isomorfeus_ferret_ext/uchar-utf8.rl +1999 -0
  212. data/ext/isomorfeus_ferret_ext/url.rl +27 -0
  213. data/ext/isomorfeus_ferret_ext/word_list.h +15156 -0
  214. data/lib/isomorfeus/ferret/document.rb +132 -0
  215. data/lib/isomorfeus/ferret/field_symbol.rb +85 -0
  216. data/lib/isomorfeus/ferret/index/field_infos.rb +48 -0
  217. data/lib/isomorfeus/ferret/index/index.rb +970 -0
  218. data/lib/isomorfeus/ferret/monitor.rb +323 -0
  219. data/lib/isomorfeus/ferret/stdlib_patches.rb +151 -0
  220. data/lib/isomorfeus/ferret/version.rb +5 -0
  221. data/lib/isomorfeus-ferret.rb +8 -0
  222. metadata +307 -0
@@ -0,0 +1,880 @@
1
+ #ifndef FRT_INDEX_H
2
+ #define FRT_INDEX_H
3
+
4
+ #include "frt_global.h"
5
+ #include "frt_document.h"
6
+ #include "frt_analysis.h"
7
+ #include "frt_hash.h"
8
+ #include "frt_hashset.h"
9
+ #include "frt_store.h"
10
+ #include "frt_mempool.h"
11
+ #include "frt_similarity.h"
12
+ #include "frt_bitvector.h"
13
+ #include "frt_priorityqueue.h"
14
+
15
+ typedef struct FrtIndexReader FrtIndexReader;
16
+ typedef struct FrtMultiReader FrtMultiReader;
17
+ typedef struct FrtDeleter FrtDeleter;
18
+
19
+ /****************************************************************************
20
+ *
21
+ * FrtConfig
22
+ *
23
+ ****************************************************************************/
24
+
25
+ typedef struct FrtConfig
26
+ {
27
+ int chunk_size;
28
+ int max_buffer_memory;
29
+ int index_interval;
30
+ int skip_interval;
31
+ int merge_factor;
32
+ int max_buffered_docs;
33
+ int max_merge_docs;
34
+ int max_field_length;
35
+ bool use_compound_file;
36
+ } FrtConfig;
37
+
38
+ extern const FrtConfig frt_default_config;
39
+
40
+ /***************************************************************************
41
+ *
42
+ * FrtCacheObject
43
+ *
44
+ ***************************************************************************/
45
+
46
+ typedef struct FrtCacheObject {
47
+ FrtHash *ref_tab1;
48
+ FrtHash *ref_tab2;
49
+ void *ref1;
50
+ void *ref2;
51
+ void *obj;
52
+ void (*destroy)(void *p);
53
+ } FrtCacheObject;
54
+
55
+ extern FrtCacheObject *frt_co_create(FrtHash *ref_tab1,
56
+ FrtHash *ref_tab2,
57
+ void *ref1, void *ref2, frt_free_ft destroy, void *obj);
58
+ extern FrtHash *frt_co_hash_create();
59
+
60
+ /****************************************************************************
61
+ *
62
+ * FrtFieldInfo
63
+ *
64
+ ****************************************************************************/
65
+
66
+ typedef enum
67
+ {
68
+ FRT_STORE_NO = 0,
69
+ FRT_STORE_YES = 1
70
+ } FrtStoreValue;
71
+
72
+ typedef enum
73
+ {
74
+ FRT_INDEX_NO = 0,
75
+ FRT_INDEX_UNTOKENIZED = 1,
76
+ FRT_INDEX_YES = 3,
77
+ FRT_INDEX_UNTOKENIZED_OMIT_NORMS = 5,
78
+ FRT_INDEX_YES_OMIT_NORMS = 7
79
+ } FrtIndexValue;
80
+
81
+ typedef enum
82
+ {
83
+ FRT_TERM_VECTOR_NO = 0,
84
+ FRT_TERM_VECTOR_YES = 1,
85
+ FRT_TERM_VECTOR_WITH_POSITIONS = 3,
86
+ FRT_TERM_VECTOR_WITH_OFFSETS = 5,
87
+ FRT_TERM_VECTOR_WITH_POSITIONS_OFFSETS = 7
88
+ } FrtTermVectorValue;
89
+
90
+ #define FRT_FI_IS_STORED_BM 0x001
91
+ #define FRT_FI_IS_INDEXED_BM 0x004
92
+ #define FRT_FI_IS_TOKENIZED_BM 0x008
93
+ #define FRT_FI_OMIT_NORMS_BM 0x010
94
+ #define FRT_FI_STORE_TERM_VECTOR_BM 0x020
95
+ #define FRT_FI_STORE_POSITIONS_BM 0x040
96
+ #define FRT_FI_STORE_OFFSETS_BM 0x080
97
+
98
+ typedef struct FrtFieldInfo
99
+ {
100
+ FrtSymbol name;
101
+ float boost;
102
+ unsigned int bits;
103
+ int number;
104
+ int ref_cnt;
105
+ } FrtFieldInfo;
106
+
107
+ extern FrtFieldInfo *frt_fi_new(FrtSymbol name,
108
+ FrtStoreValue store,
109
+ FrtIndexValue index,
110
+ FrtTermVectorValue term_vector);
111
+ extern char *frt_fi_to_s(FrtFieldInfo *fi);
112
+ extern void frt_fi_deref(FrtFieldInfo *fi);
113
+
114
+ #define fi_is_stored(fi) (((fi)->bits & FRT_FI_IS_STORED_BM) != 0)
115
+ #define fi_is_indexed(fi) (((fi)->bits & FRT_FI_IS_INDEXED_BM) != 0)
116
+ #define fi_is_tokenized(fi) (((fi)->bits & FRT_FI_IS_TOKENIZED_BM) != 0)
117
+ #define fi_omit_norms(fi) (((fi)->bits & FRT_FI_OMIT_NORMS_BM) != 0)
118
+ #define fi_store_term_vector(fi) (((fi)->bits & FRT_FI_STORE_TERM_VECTOR_BM) != 0)
119
+ #define fi_store_positions(fi) (((fi)->bits & FRT_FI_STORE_POSITIONS_BM) != 0)
120
+ #define fi_store_offsets(fi) (((fi)->bits & FRT_FI_STORE_OFFSETS_BM) != 0)
121
+ #define fi_has_norms(fi)\
122
+ (((fi)->bits & (FRT_FI_OMIT_NORMS_BM|FRT_FI_IS_INDEXED_BM)) == FRT_FI_IS_INDEXED_BM)
123
+
124
+ /****************************************************************************
125
+ *
126
+ * FrtFieldInfos
127
+ *
128
+ ****************************************************************************/
129
+
130
+ #define FIELD_INFOS_INIT_CAPA 4
131
+ /* carry changes over to dummy_fis in test/test_segments.c */
132
+ typedef struct FrtFieldInfos
133
+ {
134
+ FrtStoreValue store;
135
+ FrtIndexValue index;
136
+ FrtTermVectorValue term_vector;
137
+ int size;
138
+ int capa;
139
+ FrtFieldInfo **fields;
140
+ FrtHash *field_dict;
141
+ int ref_cnt;
142
+ } FrtFieldInfos;
143
+
144
+ FrtFieldInfos *frt_fis_new(FrtStoreValue store, FrtIndexValue index,
145
+ FrtTermVectorValue term_vector);
146
+ extern FrtFieldInfo *frt_fis_add_field(FrtFieldInfos *fis, FrtFieldInfo *fi);
147
+ extern FrtFieldInfo *frt_fis_get_field(FrtFieldInfos *fis, FrtSymbol name);
148
+ extern int frt_fis_get_field_num(FrtFieldInfos *fis, FrtSymbol name);
149
+ extern FrtFieldInfo *frt_fis_get_or_add_field(FrtFieldInfos *fis,
150
+ FrtSymbol name);
151
+ extern void frt_fis_write(FrtFieldInfos *fis, FrtOutStream *os);
152
+ extern FrtFieldInfos *frt_fis_read(FrtInStream *is);
153
+ extern char *frt_fis_to_s(FrtFieldInfos *fis);
154
+ extern void frt_fis_deref(FrtFieldInfos *fis);
155
+
156
+ /****************************************************************************
157
+ *
158
+ * FrtSegmentInfo
159
+ *
160
+ ****************************************************************************/
161
+
162
+ #define FRT_SEGMENT_NAME_MAX_LENGTH 100
163
+ #define FRT_SEGMENTS_FILE_NAME "segments"
164
+
165
+ typedef struct FrtSegmentInfo
166
+ {
167
+ int ref_cnt;
168
+ char *name;
169
+ FrtStore *store;
170
+ int doc_cnt;
171
+ int del_gen;
172
+ int *norm_gens;
173
+ int norm_gens_size;
174
+ bool use_compound_file;
175
+ } FrtSegmentInfo;
176
+
177
+ extern FrtSegmentInfo *frt_si_new(char *name, int doc_cnt, FrtStore *store);
178
+ extern void frt_si_deref(FrtSegmentInfo *si);
179
+ extern bool frt_si_has_deletions(FrtSegmentInfo *si);
180
+ extern bool frt_si_has_separate_norms(FrtSegmentInfo *si);
181
+ extern void frt_si_advance_norm_gen(FrtSegmentInfo *si, int field_num);
182
+
183
+ /****************************************************************************
184
+ *
185
+ * FrtSegmentInfos
186
+ *
187
+ ****************************************************************************/
188
+
189
+ typedef struct FrtSegmentInfos
190
+ {
191
+ FrtFieldInfos *fis;
192
+ frt_u64 counter;
193
+ frt_u64 version;
194
+ frt_i64 generation;
195
+ frt_i32 format;
196
+ FrtStore *store;
197
+ FrtSegmentInfo **segs;
198
+ int size;
199
+ int capa;
200
+ } FrtSegmentInfos;
201
+
202
+ extern char *frt_fn_for_generation(char *buf,
203
+ const char *base,
204
+ const char *ext,
205
+ frt_i64 gen);
206
+
207
+ extern FrtSegmentInfos *frt_sis_new(FrtFieldInfos *fis);
208
+ extern FrtSegmentInfo *frt_sis_new_segment(FrtSegmentInfos *sis, int dcnt, FrtStore *store);
209
+ extern FrtSegmentInfo *frt_sis_add_si(FrtSegmentInfos *sis, FrtSegmentInfo *si);
210
+ extern void frt_sis_del_at(FrtSegmentInfos *sis, int at);
211
+ extern void frt_sis_del_from_to(FrtSegmentInfos *sis, int from, int to);
212
+ extern FrtSegmentInfos *frt_sis_read(FrtStore *store);
213
+ extern void frt_sis_write(FrtSegmentInfos *sis, FrtStore *store, FrtDeleter *deleter);
214
+ extern frt_u64 frt_sis_read_current_version(FrtStore *store);
215
+ extern void frt_sis_destroy(FrtSegmentInfos *sis);
216
+ extern frt_i64 frt_sis_current_segment_generation(FrtStore *store);
217
+ extern char *frt_sis_curr_seg_file_name(char *buf, FrtStore *store);
218
+ extern void frt_sis_put(FrtSegmentInfos *sis, FILE *stream);
219
+
220
+ /****************************************************************************
221
+ *
222
+ * FrtTermInfo
223
+ *
224
+ ****************************************************************************/
225
+
226
+ typedef struct FrtTermInfo
227
+ {
228
+ int doc_freq;
229
+ off_t frq_ptr;
230
+ off_t prx_ptr;
231
+ off_t skip_offset;
232
+ } FrtTermInfo;
233
+
234
+ #define frt_ti_set(ti, mdf, mfp, mpp, mso) do {\
235
+ (ti).doc_freq = mdf;\
236
+ (ti).frq_ptr = mfp;\
237
+ (ti).prx_ptr = mpp;\
238
+ (ti).skip_offset = mso;\
239
+ } while (0)
240
+
241
+ /****************************************************************************
242
+ *
243
+ * FrtTermEnum
244
+ *
245
+ ****************************************************************************/
246
+
247
+ typedef struct FrtTermEnum FrtTermEnum;
248
+
249
+ struct FrtTermEnum
250
+ {
251
+ char curr_term[FRT_MAX_WORD_SIZE];
252
+ char prev_term[FRT_MAX_WORD_SIZE];
253
+ FrtTermInfo curr_ti;
254
+ int curr_term_len;
255
+ int field_num;
256
+ FrtTermEnum *(*set_field)(FrtTermEnum *te, int field_num);
257
+ char *(*next)(FrtTermEnum *te);
258
+ char *(*skip_to)(FrtTermEnum *te, const char *term);
259
+ void (*close)(FrtTermEnum *te);
260
+ FrtTermEnum *(*clone)(FrtTermEnum *te);
261
+ };
262
+
263
+ char *frt_te_get_term(struct FrtTermEnum *te);
264
+ FrtTermInfo *frt_te_get_ti(struct FrtTermEnum *te);
265
+
266
+ /****************************************************************************
267
+ *
268
+ * FrtSegmentTermEnum
269
+ *
270
+ ****************************************************************************/
271
+
272
+ /* * FrtSegmentTermIndex * */
273
+
274
+ typedef struct FrtSegmentTermIndex
275
+ {
276
+ off_t index_ptr;
277
+ off_t ptr;
278
+ int index_cnt;
279
+ int size;
280
+ char **index_terms;
281
+ int *index_term_lens;
282
+ FrtTermInfo *index_term_infos;
283
+ off_t *index_ptrs;
284
+ } FrtSegmentTermIndex;
285
+
286
+ /* * FrtSegmentFieldIndex * */
287
+
288
+ typedef struct FrtSegmentTermEnum FrtSegmentTermEnum;
289
+
290
+ typedef struct FrtSegmentFieldIndex
291
+ {
292
+ frt_mutex_t mutex;
293
+ int skip_interval;
294
+ int index_interval;
295
+ off_t index_ptr;
296
+ FrtTermEnum *index_te;
297
+ FrtHash *field_dict;
298
+ } FrtSegmentFieldIndex;
299
+
300
+ extern FrtSegmentFieldIndex *frt_sfi_open(FrtStore *store, const char *segment);
301
+ extern void frt_sfi_close(FrtSegmentFieldIndex *sfi);
302
+
303
+
304
+ /* * FrtSegmentTermEnum * */
305
+ struct FrtSegmentTermEnum
306
+ {
307
+ FrtTermEnum te;
308
+ FrtInStream *is;
309
+ int size;
310
+ int pos;
311
+ int skip_interval;
312
+ FrtSegmentFieldIndex *sfi;
313
+ };
314
+
315
+ extern void frt_ste_close(FrtTermEnum *te);
316
+ extern FrtTermEnum *frt_ste_clone(FrtTermEnum *te);
317
+ extern FrtTermEnum *frt_ste_new(FrtInStream *is, FrtSegmentFieldIndex *sfi);
318
+
319
+ /* * MultiTermEnum * */
320
+
321
+ extern FrtTermEnum *frt_mte_new(FrtMultiReader *mr, int field_num, const char *term);
322
+
323
+ /****************************************************************************
324
+ *
325
+ * FrtTermInfosReader
326
+ *
327
+ ****************************************************************************/
328
+
329
+ typedef struct FrtTermInfosReader
330
+ {
331
+ frt_thread_key_t thread_te;
332
+ void **te_bucket;
333
+ FrtTermEnum *orig_te;
334
+ int field_num;
335
+ } FrtTermInfosReader;
336
+
337
+ extern FrtTermInfosReader *frt_tir_open(FrtStore *store,
338
+ FrtSegmentFieldIndex *sfi,
339
+ const char *segment);
340
+ extern FrtTermInfosReader *frt_tir_set_field(FrtTermInfosReader *tir, int field_num);
341
+ extern FrtTermInfo *frt_tir_get_ti(FrtTermInfosReader *tir, const char *term);
342
+ extern char *frt_tir_get_term(FrtTermInfosReader *tir, int pos);
343
+ extern void frt_tir_close(FrtTermInfosReader *tir);
344
+
345
+ /****************************************************************************
346
+ *
347
+ * FrtTermInfosWriter
348
+ *
349
+ ****************************************************************************/
350
+
351
+ #define FRT_INDEX_INTERVAL 128
352
+ #define FRT_SKIP_INTERVAL 16
353
+
354
+ typedef struct FrtTermWriter
355
+ {
356
+ int counter;
357
+ const char *last_term;
358
+ FrtTermInfo last_term_info;
359
+ FrtOutStream *os;
360
+ } FrtTermWriter;
361
+
362
+ typedef struct FrtTermInfosWriter
363
+ {
364
+ int field_count;
365
+ int index_interval;
366
+ int skip_interval;
367
+ off_t last_index_ptr;
368
+ FrtOutStream *tfx_out;
369
+ FrtTermWriter *tix_writer;
370
+ FrtTermWriter *tis_writer;
371
+ } FrtTermInfosWriter;
372
+
373
+ extern FrtTermInfosWriter *frt_tiw_open(FrtStore *store,
374
+ const char *segment,
375
+ int index_interval,
376
+ int skip_interval);
377
+ extern void frt_tiw_start_field(FrtTermInfosWriter *tiw, int field_num);
378
+ extern void frt_tiw_add(FrtTermInfosWriter *tiw,
379
+ const char *term,
380
+ int t_len,
381
+ FrtTermInfo *ti);
382
+ extern void frt_tiw_close(FrtTermInfosWriter *tiw);
383
+
384
+ /****************************************************************************
385
+ *
386
+ * FrtTermDocEnum
387
+ *
388
+ ****************************************************************************/
389
+
390
+ typedef struct FrtTermDocEnum FrtTermDocEnum;
391
+ struct FrtTermDocEnum
392
+ {
393
+ void (*seek)(FrtTermDocEnum *tde, int field_num, const char *term);
394
+ void (*seek_te)(FrtTermDocEnum *tde, FrtTermEnum *te);
395
+ void (*seek_ti)(FrtTermDocEnum *tde, FrtTermInfo *ti);
396
+ int (*doc_num)(FrtTermDocEnum *tde);
397
+ int (*freq)(FrtTermDocEnum *tde);
398
+ bool (*next)(FrtTermDocEnum *tde);
399
+ int (*read)(FrtTermDocEnum *tde, int *docs, int *freqs, int req_num);
400
+ bool (*skip_to)(FrtTermDocEnum *tde, int target);
401
+ int (*next_position)(FrtTermDocEnum *tde);
402
+ void (*close)(FrtTermDocEnum *tde);
403
+ };
404
+
405
+ /* * FrtSegmentTermDocEnum * */
406
+
407
+ typedef struct FrtSegmentTermDocEnum FrtSegmentTermDocEnum;
408
+ struct FrtSegmentTermDocEnum
409
+ {
410
+ FrtTermDocEnum tde;
411
+ void (*seek_prox)(FrtSegmentTermDocEnum *stde, off_t prx_ptr);
412
+ void (*skip_prox)(FrtSegmentTermDocEnum *stde);
413
+ FrtTermInfosReader *tir;
414
+ FrtInStream *frq_in;
415
+ FrtInStream *prx_in;
416
+ FrtInStream *skip_in;
417
+ FrtBitVector *deleted_docs;
418
+ int count; /* number of docs for this term skipped */
419
+ int doc_freq; /* number of doc this term appears in */
420
+ int doc_num;
421
+ int freq;
422
+ int num_skips;
423
+ int skip_interval;
424
+ int skip_count;
425
+ int skip_doc;
426
+ int prx_cnt;
427
+ int position;
428
+ off_t frq_ptr;
429
+ off_t prx_ptr;
430
+ off_t skip_ptr;
431
+ bool have_skipped : 1;
432
+ };
433
+
434
+ extern FrtTermDocEnum *frt_stde_new(FrtTermInfosReader *tir, FrtInStream *frq_in,
435
+ FrtBitVector *deleted_docs, int skip_interval);
436
+
437
+ /* * FrtSegmentTermDocEnum * */
438
+ extern FrtTermDocEnum *frt_stpe_new(FrtTermInfosReader *tir, FrtInStream *frq_in,
439
+ FrtInStream *prx_in, FrtBitVector *deleted_docs,
440
+ int skip_interval);
441
+
442
+ /****************************************************************************
443
+ * MultipleTermDocPosEnum
444
+ ****************************************************************************/
445
+
446
+ extern FrtTermDocEnum *frt_mtdpe_new(FrtIndexReader *ir, int field_num, char **terms,
447
+ int t_cnt);
448
+
449
+ /****************************************************************************
450
+ *
451
+ * FrtOffset
452
+ *
453
+ ****************************************************************************/
454
+
455
+ typedef struct FrtOffset
456
+ {
457
+ off_t start;
458
+ off_t end;
459
+ } FrtOffset;
460
+
461
+ /****************************************************************************
462
+ *
463
+ * FrtOccurence
464
+ *
465
+ ****************************************************************************/
466
+
467
+ typedef struct FrtOccurence
468
+ {
469
+ struct FrtOccurence *next;
470
+ int pos;
471
+ } FrtOccurence;
472
+
473
+ /****************************************************************************
474
+ *
475
+ * FrtPosting
476
+ *
477
+ ****************************************************************************/
478
+
479
+ typedef struct FrtPosting
480
+ {
481
+ int freq;
482
+ int doc_num;
483
+ FrtOccurence *first_occ;
484
+ struct FrtPosting *next;
485
+ } FrtPosting;
486
+
487
+ extern FrtPosting *frt_p_new(FrtMemoryPool *mp, int doc_num, int pos);
488
+
489
+ /****************************************************************************
490
+ *
491
+ * FrtPostingList
492
+ *
493
+ ****************************************************************************/
494
+
495
+ typedef struct FrtPostingList
496
+ {
497
+ const char *term;
498
+ int term_len;
499
+ FrtPosting *first;
500
+ FrtPosting *last;
501
+ FrtOccurence *last_occ;
502
+ } FrtPostingList;
503
+
504
+ extern FrtPostingList *frt_pl_new(FrtMemoryPool *mp, const char *term,
505
+ int term_len, FrtPosting *p);
506
+ extern void frt_pl_add_occ(FrtMemoryPool *mp, FrtPostingList *pl, int pos);
507
+ extern int frt_pl_cmp(const FrtPostingList **pl1, const FrtPostingList **pl2);
508
+
509
+ /****************************************************************************
510
+ *
511
+ * FrtTVField
512
+ *
513
+ ****************************************************************************/
514
+
515
+ typedef struct FrtTVField
516
+ {
517
+ int field_num;
518
+ int size;
519
+ } FrtTVField;
520
+
521
+ /****************************************************************************
522
+ *
523
+ * FrtTVTerm
524
+ *
525
+ ****************************************************************************/
526
+
527
+ typedef struct FrtTVTerm
528
+ {
529
+ char *text;
530
+ int freq;
531
+ int *positions;
532
+ } FrtTVTerm;
533
+
534
+ /****************************************************************************
535
+ *
536
+ * FrtTermVector
537
+ *
538
+ ****************************************************************************/
539
+
540
+ #define FRT_TV_FIELD_INIT_CAPA 8
541
+ typedef struct FrtTermVector
542
+ {
543
+ int field_num;
544
+ FrtSymbol field;
545
+ int term_cnt;
546
+ FrtTVTerm *terms;
547
+ int offset_cnt;
548
+ FrtOffset *offsets;
549
+ } FrtTermVector;
550
+
551
+ extern void frt_tv_destroy(FrtTermVector *tv);
552
+ extern int frt_tv_get_term_index(FrtTermVector *tv, const char *term);
553
+ extern int frt_tv_scan_to_term_index(FrtTermVector *tv, const char *term);
554
+ extern FrtTVTerm *frt_tv_get_tv_term(FrtTermVector *tv, const char *term);
555
+
556
+ /****************************************************************************
557
+ *
558
+ * FrtLazyDoc
559
+ *
560
+ ****************************************************************************/
561
+
562
+ /* * * FrtLazyDocField * * */
563
+ typedef struct FrtLazyDocFieldData
564
+ {
565
+ off_t start;
566
+ int length;
567
+ char *text;
568
+ } FrtLazyDocFieldData;
569
+
570
+ typedef struct FrtLazyDoc FrtLazyDoc;
571
+ typedef struct FrtLazyDocField
572
+ {
573
+ FrtSymbol name;
574
+ FrtLazyDocFieldData *data;
575
+ FrtLazyDoc *doc;
576
+ int size; /* number of data elements */
577
+ int len; /* length of data elements concatenated */
578
+ } FrtLazyDocField;
579
+
580
+ extern char *frt_lazy_df_get_data(FrtLazyDocField *self, int i);
581
+ extern void frt_lazy_df_get_bytes(FrtLazyDocField *self, char *buf,
582
+ int start, int len);
583
+
584
+ /* * * FrtLazyDoc * * */
585
+ struct FrtLazyDoc
586
+ {
587
+ FrtHash *field_dictionary;
588
+ int size;
589
+ FrtLazyDocField **fields;
590
+ FrtInStream *fields_in;
591
+ };
592
+
593
+ extern void frt_lazy_doc_close(FrtLazyDoc *self);
594
+ extern FrtLazyDocField *frt_lazy_doc_get(FrtLazyDoc *self, FrtSymbol field);
595
+
596
+ /****************************************************************************
597
+ *
598
+ * FrtFieldsReader
599
+ *
600
+ ****************************************************************************/
601
+
602
+ typedef struct FrtFieldsReader
603
+ {
604
+ int size;
605
+ FrtFieldInfos *fis;
606
+ FrtStore *store;
607
+ FrtInStream *fdx_in;
608
+ FrtInStream *fdt_in;
609
+ } FrtFieldsReader;
610
+
611
+ extern FrtFieldsReader *frt_fr_open(FrtStore *store,
612
+ const char *segment, FrtFieldInfos *fis);
613
+ extern FrtFieldsReader *frt_fr_clone(FrtFieldsReader *orig);
614
+ extern void frt_fr_close(FrtFieldsReader *fr);
615
+ extern FrtDocument *frt_fr_get_doc(FrtFieldsReader *fr, int doc_num);
616
+ extern FrtLazyDoc *frt_fr_get_lazy_doc(FrtFieldsReader *fr, int doc_num);
617
+ extern FrtHash *frt_fr_get_tv(FrtFieldsReader *fr, int doc_num);
618
+ extern FrtTermVector *frt_fr_get_field_tv(FrtFieldsReader *fr, int doc_num,
619
+ int field_num);
620
+
621
+ /****************************************************************************
622
+ *
623
+ * FrtFieldsWriter
624
+ *
625
+ ****************************************************************************/
626
+
627
+ typedef struct FrtFieldsWriter
628
+ {
629
+ FrtFieldInfos *fis;
630
+ FrtOutStream *fdt_out;
631
+ FrtOutStream *fdx_out;
632
+ FrtOutStream *buffer;
633
+ FrtTVField *tv_fields;
634
+ off_t start_ptr;
635
+ } FrtFieldsWriter;
636
+
637
+ extern FrtFieldsWriter *frt_fw_open(FrtStore *store,
638
+ const char *segment, FrtFieldInfos *fis);
639
+ extern void frt_fw_close(FrtFieldsWriter *fw);
640
+ extern void frt_fw_add_doc(FrtFieldsWriter *fw, FrtDocument *doc);
641
+ extern void frt_fw_add_postings(FrtFieldsWriter *fw,
642
+ int field_num,
643
+ FrtPostingList **plists,
644
+ int posting_count,
645
+ FrtOffset *offsets,
646
+ int offset_count);
647
+ extern void frt_fw_write_tv_index(FrtFieldsWriter *fw);
648
+
649
+ /****************************************************************************
650
+ *
651
+ * FrtDeleter
652
+ *
653
+ * A utility class (used by both FrtIndexReader and FrtIndexWriter) to keep track of
654
+ * files that need to be deleted because they are no longer referenced by the
655
+ * index.
656
+ *
657
+ ****************************************************************************/
658
+
659
+ struct FrtDeleter
660
+ {
661
+ FrtStore *store;
662
+ FrtSegmentInfos *sis;
663
+ FrtHashSet *pending;
664
+ };
665
+
666
+ extern FrtDeleter *frt_deleter_new(FrtSegmentInfos *sis, FrtStore *store);
667
+ extern void frt_deleter_destroy(FrtDeleter *dlr);
668
+ extern void frt_deleter_delete_file(FrtDeleter *dlr, char *file_name);
669
+ extern void frt_deleter_find_deletable_files(FrtDeleter *dlr);
670
+ extern void frt_deleter_delete_files(FrtDeleter *dlr, char **files, int file_cnt);
671
+
672
+ /****************************************************************************
673
+ *
674
+ * FrtIndexReader
675
+ *
676
+ ****************************************************************************/
677
+
678
+ #define FRT_WRITE_LOCK_NAME "write"
679
+ #define FRT_COMMIT_LOCK_NAME "commit"
680
+
681
+ struct FrtIndexReader
682
+ {
683
+ int (*num_docs)(FrtIndexReader *ir);
684
+ int (*max_doc)(FrtIndexReader *ir);
685
+ FrtDocument *(*get_doc)(FrtIndexReader *ir, int doc_num);
686
+ FrtLazyDoc *(*get_lazy_doc)(FrtIndexReader *ir, int doc_num);
687
+ frt_uchar *(*get_norms)(FrtIndexReader *ir, int field_num);
688
+ frt_uchar *(*get_norms_into)(FrtIndexReader *ir, int field_num,
689
+ frt_uchar *buf);
690
+ FrtTermEnum *(*terms)(FrtIndexReader *ir, int field_num);
691
+ FrtTermEnum *(*terms_from)(FrtIndexReader *ir, int field_num,
692
+ const char *term);
693
+ int (*doc_freq)(FrtIndexReader *ir, int field_num,
694
+ const char *term);
695
+ FrtTermDocEnum *(*term_docs)(FrtIndexReader *ir);
696
+ FrtTermDocEnum *(*term_positions)(FrtIndexReader *ir);
697
+ FrtTermVector *(*term_vector)(FrtIndexReader *ir, int doc_num,
698
+ FrtSymbol field);
699
+ FrtHash *(*term_vectors)(FrtIndexReader *ir, int doc_num);
700
+ bool (*is_deleted)(FrtIndexReader *ir, int doc_num);
701
+ bool (*has_deletions)(FrtIndexReader *ir);
702
+ void (*acquire_write_lock)(FrtIndexReader *ir);
703
+ void (*set_norm_i)(FrtIndexReader *ir, int doc_num,
704
+ int field_num, frt_uchar val);
705
+ void (*delete_doc_i)(FrtIndexReader *ir, int doc_num);
706
+ void (*undelete_all_i)(FrtIndexReader *ir);
707
+ void (*set_deleter_i)(FrtIndexReader *ir, FrtDeleter *dlr);
708
+ bool (*is_latest_i)(FrtIndexReader *ir);
709
+ void (*commit_i)(FrtIndexReader *ir);
710
+ void (*close_i)(FrtIndexReader *ir);
711
+ int ref_cnt;
712
+ FrtDeleter *deleter;
713
+ FrtStore *store;
714
+ FrtLock *write_lock;
715
+ FrtSegmentInfos *sis;
716
+ FrtFieldInfos *fis;
717
+ FrtHash *cache;
718
+ FrtHash *field_index_cache;
719
+ frt_mutex_t field_index_mutex;
720
+ frt_uchar *fake_norms;
721
+ frt_mutex_t mutex;
722
+ bool has_changes : 1;
723
+ bool is_stale : 1;
724
+ bool is_owner : 1;
725
+ };
726
+
727
+ extern FrtIndexReader *frt_ir_open(FrtStore *store);
728
+ extern void frt_ir_close(FrtIndexReader *ir);
729
+ extern void frt_ir_commit(FrtIndexReader *ir);
730
+ extern void frt_ir_delete_doc(FrtIndexReader *ir, int doc_num);
731
+ extern void frt_ir_undelete_all(FrtIndexReader *ir);
732
+ extern int frt_ir_doc_freq(FrtIndexReader *ir, FrtSymbol field, const char *term);
733
+ extern void frt_ir_set_norm(FrtIndexReader *ir, int doc_num, FrtSymbol field, frt_uchar val);
734
+ extern frt_uchar *frt_ir_get_norms_i(FrtIndexReader *ir, int field_num);
735
+ extern frt_uchar *frt_ir_get_norms(FrtIndexReader *ir, FrtSymbol field);
736
+ extern frt_uchar *frt_ir_get_norms_into(FrtIndexReader *ir, FrtSymbol field, frt_uchar *buf);
737
+ extern FrtDocument *frt_ir_get_doc_with_term(FrtIndexReader *ir, FrtSymbol field, const char *term);
738
+ extern FrtTermEnum *frt_ir_terms(FrtIndexReader *ir, FrtSymbol field);
739
+ extern FrtTermEnum *frt_ir_terms_from(FrtIndexReader *ir, FrtSymbol field, const char *t);
740
+ extern FrtTermDocEnum *ir_term_docs_for(FrtIndexReader *ir, FrtSymbol field, const char *term);
741
+ extern FrtTermDocEnum *frt_ir_term_positions_for(FrtIndexReader *ir, FrtSymbol field, const char *t);
742
+ extern void frt_ir_add_cache(FrtIndexReader *ir);
743
+ extern bool frt_ir_is_latest(FrtIndexReader *ir);
744
+
745
+ /****************************************************************************
746
+ * FrtMultiReader
747
+ ****************************************************************************/
748
+
749
+ struct FrtMultiReader {
750
+ FrtIndexReader ir;
751
+ int max_doc;
752
+ int num_docs_cache;
753
+ int r_cnt;
754
+ int *starts;
755
+ FrtIndexReader **sub_readers;
756
+ FrtHash *norms_cache;
757
+ bool has_deletions : 1;
758
+ int **field_num_map;
759
+ };
760
+
761
+ extern int frt_mr_get_field_num(FrtMultiReader *mr, int ir_num, int f_num);
762
+ extern FrtIndexReader *frt_mr_open(FrtIndexReader **sub_readers, const int r_cnt);
763
+
764
+ /****************************************************************************
765
+ *
766
+ * FrtFieldInverter
767
+ *
768
+ ****************************************************************************/
769
+
770
+ typedef struct FrtFieldInverter
771
+ {
772
+ FrtHash *plists;
773
+ frt_uchar *norms;
774
+ FrtFieldInfo *fi;
775
+ int length;
776
+ bool is_tokenized : 1;
777
+ bool store_term_vector : 1;
778
+ bool store_offsets : 1;
779
+ bool has_norms : 1;
780
+ } FrtFieldInverter;
781
+
782
+ /****************************************************************************
783
+ *
784
+ * FrtDocWriter
785
+ *
786
+ ****************************************************************************/
787
+
788
+ #define DW_OFFSET_INIT_CAPA 512
789
+ typedef struct FrtIndexWriter FrtIndexWriter;
790
+
791
+ typedef struct FrtDocWriter
792
+ {
793
+ FrtStore *store;
794
+ FrtSegmentInfo *si;
795
+ FrtFieldInfos *fis;
796
+ FrtFieldsWriter *fw;
797
+ FrtMemoryPool *mp;
798
+ FrtAnalyzer *analyzer;
799
+ FrtHash *curr_plists;
800
+ FrtHash *fields;
801
+ FrtSimilarity *similarity;
802
+ FrtOffset *offsets;
803
+ int offsets_size;
804
+ int offsets_capa;
805
+ int doc_num;
806
+ int index_interval;
807
+ int skip_interval;
808
+ int max_field_length;
809
+ int max_buffered_docs;
810
+ } FrtDocWriter;
811
+
812
+ extern FrtDocWriter *frt_dw_open(FrtIndexWriter *is, FrtSegmentInfo *si);
813
+ extern void frt_dw_close(FrtDocWriter *dw);
814
+ extern void frt_dw_add_doc(FrtDocWriter *dw, FrtDocument *doc);
815
+ extern void frt_dw_new_segment(FrtDocWriter *dw, FrtSegmentInfo *si);
816
+ /* For testing. need to remove somehow. FIXME */
817
+ extern FrtHash *frt_dw_invert_field(FrtDocWriter *dw,
818
+ FrtFieldInverter *fld_inv,
819
+ FrtDocField *df);
820
+ extern FrtFieldInverter *frt_dw_get_fld_inv(FrtDocWriter *dw, FrtFieldInfo *fi);
821
+ extern void frt_dw_reset_postings(FrtHash *postings);
822
+
823
+ /****************************************************************************
824
+ *
825
+ * FrtIndexWriter
826
+ *
827
+ ****************************************************************************/
828
+
829
+ struct FrtIndexWriter
830
+ {
831
+ FrtConfig config;
832
+ frt_mutex_t mutex;
833
+ FrtStore *store;
834
+ FrtAnalyzer *analyzer;
835
+ FrtSegmentInfos *sis;
836
+ FrtFieldInfos *fis;
837
+ FrtDocWriter *dw;
838
+ FrtSimilarity *similarity;
839
+ FrtLock *write_lock;
840
+ FrtDeleter *deleter;
841
+ };
842
+
843
+ extern void frt_index_create(FrtStore *store, FrtFieldInfos *fis);
844
+ extern bool frt_index_is_locked(FrtStore *store);
845
+ extern FrtIndexWriter *frt_iw_open(FrtStore *store, FrtAnalyzer *analyzer, const FrtConfig *config);
846
+ extern void frt_iw_delete_term(FrtIndexWriter *iw, FrtSymbol field, const char *term);
847
+ extern void frt_iw_delete_terms(FrtIndexWriter *iw, FrtSymbol field, char **terms, const int term_cnt);
848
+ extern void frt_iw_close(FrtIndexWriter *iw);
849
+ extern void frt_iw_add_doc(FrtIndexWriter *iw, FrtDocument *doc);
850
+ extern int frt_iw_doc_count(FrtIndexWriter *iw);
851
+ extern void frt_iw_commit(FrtIndexWriter *iw);
852
+ extern void frt_iw_optimize(FrtIndexWriter *iw);
853
+ extern void frt_iw_add_readers(FrtIndexWriter *iw, FrtIndexReader **readers, const int r_cnt);
854
+
855
+ /****************************************************************************
856
+ *
857
+ * FrtCompoundWriter
858
+ *
859
+ ****************************************************************************/
860
+
861
+ #define FRT_CW_INIT_CAPA 16
862
+ typedef struct FrtCWFileEntry
863
+ {
864
+ char *name;
865
+ off_t dir_offset;
866
+ off_t data_offset;
867
+ } FrtCWFileEntry;
868
+
869
+ typedef struct FrtCompoundWriter {
870
+ FrtStore *store;
871
+ const char *name;
872
+ FrtHashSet *ids;
873
+ FrtCWFileEntry *file_entries;
874
+ } FrtCompoundWriter;
875
+
876
+ extern FrtCompoundWriter *frt_open_cw(FrtStore *store, char *name);
877
+ extern void frt_cw_add_file(FrtCompoundWriter *cw, char *id);
878
+ extern void frt_cw_close(FrtCompoundWriter *cw);
879
+
880
+ #endif