isomorfeus-ferret 0.12.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (222) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +612 -0
  3. data/README.md +44 -0
  4. data/ext/isomorfeus_ferret_ext/benchmark.c +223 -0
  5. data/ext/isomorfeus_ferret_ext/benchmark.h +45 -0
  6. data/ext/isomorfeus_ferret_ext/benchmarks_all.h +25 -0
  7. data/ext/isomorfeus_ferret_ext/bm_bitvector.c +123 -0
  8. data/ext/isomorfeus_ferret_ext/bm_hash.c +118 -0
  9. data/ext/isomorfeus_ferret_ext/bm_micro_string.c +40 -0
  10. data/ext/isomorfeus_ferret_ext/bm_store.c +93 -0
  11. data/ext/isomorfeus_ferret_ext/email.rl +21 -0
  12. data/ext/isomorfeus_ferret_ext/extconf.rb +5 -0
  13. data/ext/isomorfeus_ferret_ext/fio_tmpfile.h +53 -0
  14. data/ext/isomorfeus_ferret_ext/frb_analysis.c +2577 -0
  15. data/ext/isomorfeus_ferret_ext/frb_index.c +3457 -0
  16. data/ext/isomorfeus_ferret_ext/frb_lang.c +9 -0
  17. data/ext/isomorfeus_ferret_ext/frb_lang.h +17 -0
  18. data/ext/isomorfeus_ferret_ext/frb_qparser.c +629 -0
  19. data/ext/isomorfeus_ferret_ext/frb_search.c +4460 -0
  20. data/ext/isomorfeus_ferret_ext/frb_store.c +515 -0
  21. data/ext/isomorfeus_ferret_ext/frb_threading.h +30 -0
  22. data/ext/isomorfeus_ferret_ext/frb_utils.c +1127 -0
  23. data/ext/isomorfeus_ferret_ext/frt_analysis.c +1644 -0
  24. data/ext/isomorfeus_ferret_ext/frt_analysis.h +247 -0
  25. data/ext/isomorfeus_ferret_ext/frt_array.c +124 -0
  26. data/ext/isomorfeus_ferret_ext/frt_array.h +54 -0
  27. data/ext/isomorfeus_ferret_ext/frt_bitvector.c +95 -0
  28. data/ext/isomorfeus_ferret_ext/frt_bitvector.h +586 -0
  29. data/ext/isomorfeus_ferret_ext/frt_compound_io.c +374 -0
  30. data/ext/isomorfeus_ferret_ext/frt_config.h +44 -0
  31. data/ext/isomorfeus_ferret_ext/frt_document.c +134 -0
  32. data/ext/isomorfeus_ferret_ext/frt_document.h +52 -0
  33. data/ext/isomorfeus_ferret_ext/frt_except.c +95 -0
  34. data/ext/isomorfeus_ferret_ext/frt_except.h +188 -0
  35. data/ext/isomorfeus_ferret_ext/frt_field_index.c +233 -0
  36. data/ext/isomorfeus_ferret_ext/frt_field_index.h +42 -0
  37. data/ext/isomorfeus_ferret_ext/frt_filter.c +157 -0
  38. data/ext/isomorfeus_ferret_ext/frt_fs_store.c +502 -0
  39. data/ext/isomorfeus_ferret_ext/frt_global.c +427 -0
  40. data/ext/isomorfeus_ferret_ext/frt_global.h +290 -0
  41. data/ext/isomorfeus_ferret_ext/frt_hash.c +518 -0
  42. data/ext/isomorfeus_ferret_ext/frt_hash.h +466 -0
  43. data/ext/isomorfeus_ferret_ext/frt_hashset.c +191 -0
  44. data/ext/isomorfeus_ferret_ext/frt_hashset.h +206 -0
  45. data/ext/isomorfeus_ferret_ext/frt_helper.c +62 -0
  46. data/ext/isomorfeus_ferret_ext/frt_helper.h +13 -0
  47. data/ext/isomorfeus_ferret_ext/frt_ind.c +353 -0
  48. data/ext/isomorfeus_ferret_ext/frt_ind.h +54 -0
  49. data/ext/isomorfeus_ferret_ext/frt_index.c +6377 -0
  50. data/ext/isomorfeus_ferret_ext/frt_index.h +880 -0
  51. data/ext/isomorfeus_ferret_ext/frt_lang.c +104 -0
  52. data/ext/isomorfeus_ferret_ext/frt_lang.h +44 -0
  53. data/ext/isomorfeus_ferret_ext/frt_mempool.c +87 -0
  54. data/ext/isomorfeus_ferret_ext/frt_mempool.h +33 -0
  55. data/ext/isomorfeus_ferret_ext/frt_multimapper.c +349 -0
  56. data/ext/isomorfeus_ferret_ext/frt_multimapper.h +52 -0
  57. data/ext/isomorfeus_ferret_ext/frt_posh.c +1006 -0
  58. data/ext/isomorfeus_ferret_ext/frt_posh.h +973 -0
  59. data/ext/isomorfeus_ferret_ext/frt_priorityqueue.c +147 -0
  60. data/ext/isomorfeus_ferret_ext/frt_priorityqueue.h +147 -0
  61. data/ext/isomorfeus_ferret_ext/frt_q_boolean.c +1612 -0
  62. data/ext/isomorfeus_ferret_ext/frt_q_const_score.c +157 -0
  63. data/ext/isomorfeus_ferret_ext/frt_q_filtered_query.c +209 -0
  64. data/ext/isomorfeus_ferret_ext/frt_q_fuzzy.c +281 -0
  65. data/ext/isomorfeus_ferret_ext/frt_q_match_all.c +147 -0
  66. data/ext/isomorfeus_ferret_ext/frt_q_multi_term.c +672 -0
  67. data/ext/isomorfeus_ferret_ext/frt_q_parser.c +3084 -0
  68. data/ext/isomorfeus_ferret_ext/frt_q_phrase.c +1182 -0
  69. data/ext/isomorfeus_ferret_ext/frt_q_prefix.c +98 -0
  70. data/ext/isomorfeus_ferret_ext/frt_q_range.c +665 -0
  71. data/ext/isomorfeus_ferret_ext/frt_q_span.c +2386 -0
  72. data/ext/isomorfeus_ferret_ext/frt_q_term.c +311 -0
  73. data/ext/isomorfeus_ferret_ext/frt_q_wildcard.c +166 -0
  74. data/ext/isomorfeus_ferret_ext/frt_ram_store.c +460 -0
  75. data/ext/isomorfeus_ferret_ext/frt_scanner.c +899 -0
  76. data/ext/isomorfeus_ferret_ext/frt_scanner.h +28 -0
  77. data/ext/isomorfeus_ferret_ext/frt_scanner_mb.c +6705 -0
  78. data/ext/isomorfeus_ferret_ext/frt_scanner_utf8.c +4419 -0
  79. data/ext/isomorfeus_ferret_ext/frt_search.c +1824 -0
  80. data/ext/isomorfeus_ferret_ext/frt_search.h +924 -0
  81. data/ext/isomorfeus_ferret_ext/frt_similarity.c +150 -0
  82. data/ext/isomorfeus_ferret_ext/frt_similarity.h +79 -0
  83. data/ext/isomorfeus_ferret_ext/frt_sort.c +796 -0
  84. data/ext/isomorfeus_ferret_ext/frt_stopwords.c +395 -0
  85. data/ext/isomorfeus_ferret_ext/frt_store.c +680 -0
  86. data/ext/isomorfeus_ferret_ext/frt_store.h +789 -0
  87. data/ext/isomorfeus_ferret_ext/frt_term_vectors.c +72 -0
  88. data/ext/isomorfeus_ferret_ext/frt_threading.h +23 -0
  89. data/ext/isomorfeus_ferret_ext/frt_win32.h +54 -0
  90. data/ext/isomorfeus_ferret_ext/isomorfeus_ferret.c +409 -0
  91. data/ext/isomorfeus_ferret_ext/isomorfeus_ferret.h +95 -0
  92. data/ext/isomorfeus_ferret_ext/libstemmer.c +93 -0
  93. data/ext/isomorfeus_ferret_ext/libstemmer.h +73 -0
  94. data/ext/isomorfeus_ferret_ext/q_parser.y +1366 -0
  95. data/ext/isomorfeus_ferret_ext/scanner.h +28 -0
  96. data/ext/isomorfeus_ferret_ext/scanner.in +43 -0
  97. data/ext/isomorfeus_ferret_ext/scanner.rl +84 -0
  98. data/ext/isomorfeus_ferret_ext/scanner_mb.rl +200 -0
  99. data/ext/isomorfeus_ferret_ext/scanner_utf8.rl +85 -0
  100. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_danish.c +324 -0
  101. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_danish.h +7 -0
  102. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_dutch.c +610 -0
  103. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_dutch.h +6 -0
  104. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_english.c +1104 -0
  105. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_english.h +6 -0
  106. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_finnish.c +749 -0
  107. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_finnish.h +7 -0
  108. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_french.c +1233 -0
  109. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_french.h +6 -0
  110. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_german.c +490 -0
  111. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_german.h +6 -0
  112. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_hungarian.c +1217 -0
  113. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_hungarian.h +7 -0
  114. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_italian.c +1052 -0
  115. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_italian.h +6 -0
  116. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_norwegian.c +283 -0
  117. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_norwegian.h +6 -0
  118. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_porter.c +735 -0
  119. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_porter.h +6 -0
  120. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_portuguese.c +1003 -0
  121. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_portuguese.h +7 -0
  122. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_spanish.c +1079 -0
  123. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_spanish.h +6 -0
  124. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_swedish.c +293 -0
  125. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_swedish.h +6 -0
  126. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_2_romanian.c +984 -0
  127. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_2_romanian.h +6 -0
  128. data/ext/isomorfeus_ferret_ext/stem_KOI8_R_russian.c +686 -0
  129. data/ext/isomorfeus_ferret_ext/stem_KOI8_R_russian.h +6 -0
  130. data/ext/isomorfeus_ferret_ext/stem_UTF_8_danish.c +325 -0
  131. data/ext/isomorfeus_ferret_ext/stem_UTF_8_danish.h +6 -0
  132. data/ext/isomorfeus_ferret_ext/stem_UTF_8_dutch.c +620 -0
  133. data/ext/isomorfeus_ferret_ext/stem_UTF_8_dutch.h +6 -0
  134. data/ext/isomorfeus_ferret_ext/stem_UTF_8_english.c +1111 -0
  135. data/ext/isomorfeus_ferret_ext/stem_UTF_8_english.h +6 -0
  136. data/ext/isomorfeus_ferret_ext/stem_UTF_8_finnish.c +754 -0
  137. data/ext/isomorfeus_ferret_ext/stem_UTF_8_finnish.h +6 -0
  138. data/ext/isomorfeus_ferret_ext/stem_UTF_8_french.c +1242 -0
  139. data/ext/isomorfeus_ferret_ext/stem_UTF_8_french.h +6 -0
  140. data/ext/isomorfeus_ferret_ext/stem_UTF_8_german.c +495 -0
  141. data/ext/isomorfeus_ferret_ext/stem_UTF_8_german.h +6 -0
  142. data/ext/isomorfeus_ferret_ext/stem_UTF_8_hungarian.c +1220 -0
  143. data/ext/isomorfeus_ferret_ext/stem_UTF_8_hungarian.h +6 -0
  144. data/ext/isomorfeus_ferret_ext/stem_UTF_8_italian.c +1059 -0
  145. data/ext/isomorfeus_ferret_ext/stem_UTF_8_italian.h +6 -0
  146. data/ext/isomorfeus_ferret_ext/stem_UTF_8_norwegian.c +285 -0
  147. data/ext/isomorfeus_ferret_ext/stem_UTF_8_norwegian.h +6 -0
  148. data/ext/isomorfeus_ferret_ext/stem_UTF_8_porter.c +741 -0
  149. data/ext/isomorfeus_ferret_ext/stem_UTF_8_porter.h +6 -0
  150. data/ext/isomorfeus_ferret_ext/stem_UTF_8_portuguese.c +1009 -0
  151. data/ext/isomorfeus_ferret_ext/stem_UTF_8_portuguese.h +6 -0
  152. data/ext/isomorfeus_ferret_ext/stem_UTF_8_romanian.c +990 -0
  153. data/ext/isomorfeus_ferret_ext/stem_UTF_8_romanian.h +6 -0
  154. data/ext/isomorfeus_ferret_ext/stem_UTF_8_russian.c +680 -0
  155. data/ext/isomorfeus_ferret_ext/stem_UTF_8_russian.h +6 -0
  156. data/ext/isomorfeus_ferret_ext/stem_UTF_8_spanish.c +1083 -0
  157. data/ext/isomorfeus_ferret_ext/stem_UTF_8_spanish.h +6 -0
  158. data/ext/isomorfeus_ferret_ext/stem_UTF_8_swedish.c +294 -0
  159. data/ext/isomorfeus_ferret_ext/stem_UTF_8_swedish.h +6 -0
  160. data/ext/isomorfeus_ferret_ext/stem_UTF_8_turkish.c +2191 -0
  161. data/ext/isomorfeus_ferret_ext/stem_UTF_8_turkish.h +6 -0
  162. data/ext/isomorfeus_ferret_ext/stem_api.c +66 -0
  163. data/ext/isomorfeus_ferret_ext/stem_api.h +26 -0
  164. data/ext/isomorfeus_ferret_ext/stem_header.h +57 -0
  165. data/ext/isomorfeus_ferret_ext/stem_modules.h +190 -0
  166. data/ext/isomorfeus_ferret_ext/stem_modules.txt +50 -0
  167. data/ext/isomorfeus_ferret_ext/stem_utilities.c +478 -0
  168. data/ext/isomorfeus_ferret_ext/test.c +850 -0
  169. data/ext/isomorfeus_ferret_ext/test.h +416 -0
  170. data/ext/isomorfeus_ferret_ext/test_1710.c +63 -0
  171. data/ext/isomorfeus_ferret_ext/test_analysis.c +1221 -0
  172. data/ext/isomorfeus_ferret_ext/test_array.c +272 -0
  173. data/ext/isomorfeus_ferret_ext/test_bitvector.c +600 -0
  174. data/ext/isomorfeus_ferret_ext/test_compound_io.c +170 -0
  175. data/ext/isomorfeus_ferret_ext/test_document.c +156 -0
  176. data/ext/isomorfeus_ferret_ext/test_except.c +244 -0
  177. data/ext/isomorfeus_ferret_ext/test_fields.c +522 -0
  178. data/ext/isomorfeus_ferret_ext/test_file_deleter.c +185 -0
  179. data/ext/isomorfeus_ferret_ext/test_filter.c +331 -0
  180. data/ext/isomorfeus_ferret_ext/test_fs_store.c +25 -0
  181. data/ext/isomorfeus_ferret_ext/test_global.c +299 -0
  182. data/ext/isomorfeus_ferret_ext/test_hash.c +485 -0
  183. data/ext/isomorfeus_ferret_ext/test_hashset.c +288 -0
  184. data/ext/isomorfeus_ferret_ext/test_helper.c +47 -0
  185. data/ext/isomorfeus_ferret_ext/test_highlighter.c +548 -0
  186. data/ext/isomorfeus_ferret_ext/test_index.c +2323 -0
  187. data/ext/isomorfeus_ferret_ext/test_lang.c +74 -0
  188. data/ext/isomorfeus_ferret_ext/test_mempool.c +102 -0
  189. data/ext/isomorfeus_ferret_ext/test_multimapper.c +64 -0
  190. data/ext/isomorfeus_ferret_ext/test_priorityqueue.c +213 -0
  191. data/ext/isomorfeus_ferret_ext/test_q_const_score.c +84 -0
  192. data/ext/isomorfeus_ferret_ext/test_q_filtered.c +61 -0
  193. data/ext/isomorfeus_ferret_ext/test_q_fuzzy.c +241 -0
  194. data/ext/isomorfeus_ferret_ext/test_q_parser.c +464 -0
  195. data/ext/isomorfeus_ferret_ext/test_q_span.c +575 -0
  196. data/ext/isomorfeus_ferret_ext/test_ram_store.c +77 -0
  197. data/ext/isomorfeus_ferret_ext/test_search.c +1874 -0
  198. data/ext/isomorfeus_ferret_ext/test_segments.c +167 -0
  199. data/ext/isomorfeus_ferret_ext/test_similarity.c +25 -0
  200. data/ext/isomorfeus_ferret_ext/test_sort.c +333 -0
  201. data/ext/isomorfeus_ferret_ext/test_store.c +591 -0
  202. data/ext/isomorfeus_ferret_ext/test_store.h +3 -0
  203. data/ext/isomorfeus_ferret_ext/test_term.c +351 -0
  204. data/ext/isomorfeus_ferret_ext/test_term_vectors.c +373 -0
  205. data/ext/isomorfeus_ferret_ext/test_test.c +83 -0
  206. data/ext/isomorfeus_ferret_ext/test_threading.c +188 -0
  207. data/ext/isomorfeus_ferret_ext/testhelper.c +561 -0
  208. data/ext/isomorfeus_ferret_ext/testhelper.h +25 -0
  209. data/ext/isomorfeus_ferret_ext/tests_all.h +87 -0
  210. data/ext/isomorfeus_ferret_ext/uchar-ucs4.rl +1854 -0
  211. data/ext/isomorfeus_ferret_ext/uchar-utf8.rl +1999 -0
  212. data/ext/isomorfeus_ferret_ext/url.rl +27 -0
  213. data/ext/isomorfeus_ferret_ext/word_list.h +15156 -0
  214. data/lib/isomorfeus/ferret/document.rb +132 -0
  215. data/lib/isomorfeus/ferret/field_symbol.rb +85 -0
  216. data/lib/isomorfeus/ferret/index/field_infos.rb +48 -0
  217. data/lib/isomorfeus/ferret/index/index.rb +970 -0
  218. data/lib/isomorfeus/ferret/monitor.rb +323 -0
  219. data/lib/isomorfeus/ferret/stdlib_patches.rb +151 -0
  220. data/lib/isomorfeus/ferret/version.rb +5 -0
  221. data/lib/isomorfeus-ferret.rb +8 -0
  222. metadata +307 -0
@@ -0,0 +1,680 @@
1
+ #include "frt_store.h"
2
+ #include <string.h>
3
+
4
+ #define VINT_MAX_LEN 10
5
+ #define VINT_END FRT_BUFFER_SIZE - VINT_MAX_LEN
6
+
7
+ /*
8
+ * TODO: add try finally
9
+ */
10
+ void frt_with_lock(FrtLock *lock, void (*func)(void *arg), void *arg) {
11
+ if (!lock->obtain(lock)) {
12
+ FRT_RAISE(FRT_LOCK_ERROR, "couldn't obtain lock \"%s\"", lock->name);
13
+ }
14
+ func(arg);
15
+ lock->release(lock);
16
+ }
17
+
18
+ /*
19
+ * TODO: add try finally
20
+ */
21
+ void frt_with_lock_name(FrtStore *store, const char *lock_name, void (*func)(void *arg), void *arg) {
22
+ FrtLock *lock = store->open_lock_i(store, lock_name);
23
+ if (!lock->obtain(lock)) {
24
+ FRT_RAISE(FRT_LOCK_ERROR, "couldn't obtain lock \"%s\"", lock->name);
25
+ }
26
+ func(arg);
27
+ lock->release(lock);
28
+ store->close_lock_i(lock);
29
+ }
30
+
31
+ void frt_store_deref(FrtStore *store)
32
+ {
33
+ frt_mutex_lock(&store->mutex_i);
34
+ if (--store->ref_cnt <= 0) {
35
+ store->close_i(store);
36
+ }
37
+ else {
38
+ frt_mutex_unlock(&store->mutex_i);
39
+ }
40
+ }
41
+
42
+ FrtLock *frt_open_lock(FrtStore *store, const char *lockname)
43
+ {
44
+ FrtLock *lock = store->open_lock_i(store, lockname);
45
+ frt_hs_add(store->locks, lock);
46
+ return lock;
47
+ }
48
+
49
+ void frt_close_lock(FrtLock *lock)
50
+ {
51
+ frt_hs_del(lock->store->locks, lock);
52
+ }
53
+
54
+ static void frt_close_lock_i(FrtLock *lock)
55
+ {
56
+ lock->store->close_lock_i(lock);
57
+ }
58
+
59
+ /**
60
+ * Create a store struct initializing the mutex.
61
+ */
62
+ FrtStore *frt_store_new()
63
+ {
64
+ FrtStore *store = FRT_ALLOC(FrtStore);
65
+ store->ref_cnt = 1;
66
+ frt_mutex_init(&store->mutex_i, NULL);
67
+ frt_mutex_init(&store->mutex, NULL);
68
+ store->locks = frt_hs_new_ptr((frt_free_ft)&frt_close_lock_i);
69
+ return store;
70
+ }
71
+
72
+ /**
73
+ * Destroy the store freeing allocated resources
74
+ *
75
+ * @param store the store struct to free
76
+ */
77
+ void frt_store_destroy(FrtStore *store)
78
+ {
79
+ frt_mutex_destroy(&store->mutex_i);
80
+ frt_mutex_destroy(&store->mutex);
81
+ frt_hs_destroy(store->locks);
82
+ free(store);
83
+ }
84
+
85
+ /**
86
+ * Create a newly allocated and initialized OutStream object
87
+ *
88
+ * @return a newly allocated and initialized OutStream object
89
+ */
90
+ FrtOutStream *frt_os_new()
91
+ {
92
+ FrtOutStream *os = FRT_ALLOC(FrtOutStream);
93
+ os->buf.start = 0;
94
+ os->buf.pos = 0;
95
+ os->buf.len = 0;
96
+ return os;
97
+ }
98
+
99
+ /**
100
+ * Flush the countents of the FrtOutStream's buffers
101
+ *
102
+ * @param the OutStream to flush
103
+ */
104
+ void frt_os_flush(FrtOutStream *os)
105
+ {
106
+ os->m->flush_i(os, os->buf.buf, os->buf.pos);
107
+ os->buf.start += os->buf.pos;
108
+ os->buf.pos = 0;
109
+ }
110
+
111
+ void frt_os_close(FrtOutStream *os)
112
+ {
113
+ frt_os_flush(os);
114
+ os->m->close_i(os);
115
+ free(os);
116
+ }
117
+
118
+ off_t frt_os_pos(FrtOutStream *os)
119
+ {
120
+ return os->buf.start + os->buf.pos;
121
+ }
122
+
123
+ void frt_os_seek(FrtOutStream *os, off_t new_pos)
124
+ {
125
+ frt_os_flush(os);
126
+ os->buf.start = new_pos;
127
+ os->m->seek_i(os, new_pos);
128
+ }
129
+
130
+ /**
131
+ * Unsafe alternative to os_write_byte. Only use this method if you know there
132
+ * is no chance of buffer overflow.
133
+ */
134
+ #define write_byte(os, b) os->buf.buf[os->buf.pos++] = (frt_uchar)b
135
+
136
+ /**
137
+ * Write a single byte +b+ to the OutStream +os+
138
+ *
139
+ * @param os the OutStream to write to
140
+ * @param b the byte to write
141
+ * @raise FRT_IO_ERROR if there is an IO error writing to the filesystem
142
+ */
143
+ void frt_os_write_byte(FrtOutStream *os, frt_uchar b)
144
+ {
145
+ if (os->buf.pos >= FRT_BUFFER_SIZE) {
146
+ frt_os_flush(os);
147
+ }
148
+ write_byte(os, b);
149
+ }
150
+
151
+ void frt_os_write_bytes(FrtOutStream *os, const frt_uchar *buf, int len)
152
+ {
153
+ if (os->buf.pos > 0) { /* flush buffer */
154
+ frt_os_flush(os);
155
+ }
156
+
157
+ if (len < FRT_BUFFER_SIZE) {
158
+ os->m->flush_i(os, buf, len);
159
+ os->buf.start += len;
160
+ }
161
+ else {
162
+ int pos = 0;
163
+ int size;
164
+ while (pos < len) {
165
+ if (len - pos < FRT_BUFFER_SIZE) {
166
+ size = len - pos;
167
+ }
168
+ else {
169
+ size = FRT_BUFFER_SIZE;
170
+ }
171
+ os->m->flush_i(os, buf + pos, size);
172
+ pos += size;
173
+ os->buf.start += size;
174
+ }
175
+ }
176
+ }
177
+
178
+ /**
179
+ * Create a newly allocated and initialized InStream
180
+ *
181
+ * @return a newly allocated and initialized InStream
182
+ */
183
+ FrtInStream *frt_is_new()
184
+ {
185
+ FrtInStream *is = FRT_ALLOC(FrtInStream);
186
+ is->buf.start = 0;
187
+ is->buf.pos = 0;
188
+ is->buf.len = 0;
189
+ is->ref_cnt_ptr = FRT_ALLOC_AND_ZERO(int);
190
+ return is;
191
+ }
192
+
193
+ /**
194
+ * Refill the InStream's buffer from the store source (filesystem or memory).
195
+ *
196
+ * @param is the InStream to refill
197
+ * @raise FRT_IO_ERROR if there is a error reading from the filesystem
198
+ * @raise FRT_EOF_ERROR if there is an attempt to read past the end of the file
199
+ */
200
+ static void is_refill(FrtInStream *is)
201
+ {
202
+ off_t start = is->buf.start + is->buf.pos;
203
+ off_t last = start + FRT_BUFFER_SIZE;
204
+ off_t flen = is->m->length_i(is);
205
+
206
+ if (last > flen) { /* don't read past EOF */
207
+ last = flen;
208
+ }
209
+
210
+ is->buf.len = last - start;
211
+ if (is->buf.len <= 0) {
212
+ FRT_RAISE(FRT_EOF_ERROR, "current pos = %"FRT_OFF_T_PFX"d, "
213
+ "file length = %"FRT_OFF_T_PFX"d", start, flen);
214
+ }
215
+
216
+ is->m->read_i(is, is->buf.buf, is->buf.len);
217
+
218
+ is->buf.start = start;
219
+ is->buf.pos = 0;
220
+ }
221
+
222
+ /**
223
+ * Unsafe alternative to frt_is_read_byte. Only use this method when you know
224
+ * there is no chance that you will read past the end of the InStream's
225
+ * buffer.
226
+ */
227
+ #define read_byte(is) is->buf.buf[is->buf.pos++]
228
+
229
+ /**
230
+ * Read a singly byte (unsigned char) from the InStream +is+.
231
+ *
232
+ * @param is the Instream to read from
233
+ * @return a single unsigned char read from the InStream +is+
234
+ * @raise FRT_IO_ERROR if there is a error reading from the filesystem
235
+ * @raise FRT_EOF_ERROR if there is an attempt to read past the end of the file
236
+ */
237
+ frt_uchar frt_is_read_byte(FrtInStream *is)
238
+ {
239
+ if (is->buf.pos >= is->buf.len) {
240
+ is_refill(is);
241
+ }
242
+
243
+ return read_byte(is);
244
+ }
245
+
246
+ off_t frt_is_pos(FrtInStream *is)
247
+ {
248
+ return is->buf.start + is->buf.pos;
249
+ }
250
+
251
+ frt_uchar *frt_is_read_bytes(FrtInStream *is, frt_uchar *buf, int len)
252
+ {
253
+ int i;
254
+ off_t start;
255
+
256
+ if ((is->buf.pos + len) < is->buf.len) {
257
+ for (i = 0; i < len; i++) {
258
+ buf[i] = read_byte(is);
259
+ }
260
+ }
261
+ else { /* read all-at-once */
262
+ start = frt_is_pos(is);
263
+ is->m->seek_i(is, start);
264
+ is->m->read_i(is, buf, len);
265
+
266
+ is->buf.start = start + len; /* adjust stream variables */
267
+ is->buf.pos = 0;
268
+ is->buf.len = 0; /* trigger refill on read */
269
+ }
270
+ return buf;
271
+ }
272
+
273
+ void frt_is_seek(FrtInStream *is, off_t pos)
274
+ {
275
+ if (pos >= is->buf.start && pos < (is->buf.start + is->buf.len)) {
276
+ is->buf.pos = pos - is->buf.start; /* seek within buffer */
277
+ }
278
+ else {
279
+ is->buf.start = pos;
280
+ is->buf.pos = 0;
281
+ is->buf.len = 0; /* trigger refill() on read() */
282
+ is->m->seek_i(is, pos);
283
+ }
284
+ }
285
+
286
+ void frt_is_close(FrtInStream *is)
287
+ {
288
+ if (--(*(is->ref_cnt_ptr)) < 0) {
289
+ is->m->close_i(is);
290
+ free(is->ref_cnt_ptr);
291
+ }
292
+ free(is);
293
+ }
294
+
295
+ FrtInStream *frt_is_clone(FrtInStream *is)
296
+ {
297
+ FrtInStream *new_index_i = FRT_ALLOC(FrtInStream);
298
+ memcpy(new_index_i, is, sizeof(FrtInStream));
299
+ (*(new_index_i->ref_cnt_ptr))++;
300
+ return new_index_i;
301
+ }
302
+
303
+ frt_i32 frt_is_read_i32(FrtInStream *is)
304
+ {
305
+ return ((frt_i32)frt_is_read_byte(is) << 24) |
306
+ ((frt_i32)frt_is_read_byte(is) << 16) |
307
+ ((frt_i32)frt_is_read_byte(is) << 8) |
308
+ ((frt_i32)frt_is_read_byte(is));
309
+ }
310
+
311
+
312
+ frt_i64 frt_is_read_i64(FrtInStream *is)
313
+ {
314
+ return ((frt_i64)frt_is_read_byte(is) << 56) |
315
+ ((frt_i64)frt_is_read_byte(is) << 48) |
316
+ ((frt_i64)frt_is_read_byte(is) << 40) |
317
+ ((frt_i64)frt_is_read_byte(is) << 32) |
318
+ ((frt_i64)frt_is_read_byte(is) << 24) |
319
+ ((frt_i64)frt_is_read_byte(is) << 16) |
320
+ ((frt_i64)frt_is_read_byte(is) << 8) |
321
+ ((frt_i64)frt_is_read_byte(is));
322
+ }
323
+
324
+ frt_u32 frt_is_read_u32(FrtInStream *is)
325
+ {
326
+ return ((frt_u32)frt_is_read_byte(is) << 24) |
327
+ ((frt_u32)frt_is_read_byte(is) << 16) |
328
+ ((frt_u32)frt_is_read_byte(is) << 8) |
329
+ ((frt_u32)frt_is_read_byte(is));
330
+ }
331
+
332
+ frt_u64 frt_is_read_u64(FrtInStream *is)
333
+ {
334
+ return ((frt_u64)frt_is_read_byte(is) << 56) |
335
+ ((frt_u64)frt_is_read_byte(is) << 48) |
336
+ ((frt_u64)frt_is_read_byte(is) << 40) |
337
+ ((frt_u64)frt_is_read_byte(is) << 32) |
338
+ ((frt_u64)frt_is_read_byte(is) << 24) |
339
+ ((frt_u64)frt_is_read_byte(is) << 16) |
340
+ ((frt_u64)frt_is_read_byte(is) << 8) |
341
+ ((frt_u64)frt_is_read_byte(is));
342
+ }
343
+
344
+ /* optimized to use unchecked read_byte if there is definitely space */
345
+ unsigned int frt_is_read_vint(FrtInStream *is)
346
+ {
347
+ register unsigned int res, b;
348
+ register int shift = 7;
349
+
350
+ if (is->buf.pos > (is->buf.len - VINT_MAX_LEN)) {
351
+ b = frt_is_read_byte(is);
352
+ res = b & 0x7F; /* 0x7F = 0b01111111 */
353
+
354
+ while ((b & 0x80) != 0) { /* 0x80 = 0b10000000 */
355
+ b = frt_is_read_byte(is);
356
+ res |= (b & 0x7F) << shift;
357
+ shift += 7;
358
+ }
359
+ }
360
+ else { /* unchecked optimization */
361
+ b = read_byte(is);
362
+ res = b & 0x7F; /* 0x7F = 0b01111111 */
363
+
364
+ while ((b & 0x80) != 0) { /* 0x80 = 0b10000000 */
365
+ b = read_byte(is);
366
+ res |= (b & 0x7F) << shift;
367
+ shift += 7;
368
+ }
369
+ }
370
+
371
+ return res;
372
+ }
373
+
374
+ /* optimized to use unchecked read_byte if there is definitely space */
375
+ off_t frt_is_read_voff_t(FrtInStream *is)
376
+ {
377
+ register off_t res, b;
378
+ register int shift = 7;
379
+
380
+ if (is->buf.pos > (is->buf.len - VINT_MAX_LEN)) {
381
+ b = frt_is_read_byte(is);
382
+ res = b & 0x7F; /* 0x7F = 0b01111111 */
383
+
384
+ while ((b & 0x80) != 0) { /* 0x80 = 0b10000000 */
385
+ b = frt_is_read_byte(is);
386
+ res |= (b & 0x7F) << shift;
387
+ shift += 7;
388
+ }
389
+ }
390
+ else { /* unchecked optimization */
391
+ b = read_byte(is);
392
+ res = b & 0x7F; /* 0x7F = 0b01111111 */
393
+
394
+ while ((b & 0x80) != 0) { /* 0x80 = 0b10000000 */
395
+ b = read_byte(is);
396
+ res |= (b & 0x7F) << shift;
397
+ shift += 7;
398
+ }
399
+ }
400
+
401
+ return res;
402
+ }
403
+
404
+ /* optimized to use unchecked read_byte if there is definitely space */
405
+ frt_u64 frt_is_read_vll(FrtInStream *is)
406
+ {
407
+ register frt_u64 res, b;
408
+ register int shift = 7;
409
+
410
+ if (is->buf.pos > (is->buf.len - VINT_MAX_LEN)) {
411
+ b = frt_is_read_byte(is);
412
+ res = b & 0x7F; /* 0x7F = 0b01111111 */
413
+
414
+ while ((b & 0x80) != 0) { /* 0x80 = 0b10000000 */
415
+ b = frt_is_read_byte(is);
416
+ res |= (b & 0x7F) << shift;
417
+ shift += 7;
418
+ }
419
+ }
420
+ else { /* unchecked optimization */
421
+ b = read_byte(is);
422
+ res = b & 0x7F; /* 0x7F = 0b01111111 */
423
+
424
+ while ((b & 0x80) != 0) { /* 0x80 = 0b10000000 */
425
+ b = read_byte(is);
426
+ res |= (b & 0x7F) << shift;
427
+ shift += 7;
428
+ }
429
+ }
430
+
431
+ return res;
432
+ }
433
+
434
+ void frt_is_skip_vints(FrtInStream *is, register int cnt)
435
+ {
436
+ for (; cnt > 0; cnt--) {
437
+ while ((frt_is_read_byte(is) & 0x80) != 0) {
438
+ }
439
+ }
440
+ }
441
+
442
+ char *frt_is_read_string(FrtInStream *is)
443
+ {
444
+ register int length = (int) frt_is_read_vint(is);
445
+ char *str = FRT_ALLOC_N(char, length + 1);
446
+ str[length] = '\0';
447
+
448
+ if (is->buf.pos > (is->buf.len - length)) {
449
+ register int i;
450
+ for (i = 0; i < length; i++) {
451
+ str[i] = frt_is_read_byte(is);
452
+ }
453
+ }
454
+ else { /* unchecked optimization */
455
+ memcpy(str, is->buf.buf + is->buf.pos, length);
456
+ is->buf.pos += length;
457
+ }
458
+
459
+ return str;
460
+ }
461
+
462
+ char *frt_is_read_string_safe(FrtInStream *is)
463
+ {
464
+ register int length = (int) frt_is_read_vint(is);
465
+ char *str = FRT_ALLOC_N(char, length + 1);
466
+ str[length] = '\0';
467
+
468
+ FRT_TRY
469
+ if (is->buf.pos > (is->buf.len - length)) {
470
+ register int i;
471
+ for (i = 0; i < length; i++) {
472
+ str[i] = frt_is_read_byte(is);
473
+ }
474
+ }
475
+ else { /* unchecked optimization */
476
+ memcpy(str, is->buf.buf + is->buf.pos, length);
477
+ is->buf.pos += length;
478
+ }
479
+ FRT_XCATCHALL
480
+ free(str);
481
+ FRT_XENDTRY
482
+
483
+ return str;
484
+ }
485
+
486
+ void frt_os_write_i32(FrtOutStream *os, frt_i32 num)
487
+ {
488
+ frt_os_write_byte(os, (frt_uchar)((num >> 24) & 0xFF));
489
+ frt_os_write_byte(os, (frt_uchar)((num >> 16) & 0xFF));
490
+ frt_os_write_byte(os, (frt_uchar)((num >> 8) & 0xFF));
491
+ frt_os_write_byte(os, (frt_uchar)(num & 0xFF));
492
+ }
493
+
494
+ void frt_os_write_i64(FrtOutStream *os, frt_i64 num)
495
+ {
496
+ frt_os_write_byte(os, (frt_uchar)((num >> 56) & 0xFF));
497
+ frt_os_write_byte(os, (frt_uchar)((num >> 48) & 0xFF));
498
+ frt_os_write_byte(os, (frt_uchar)((num >> 40) & 0xFF));
499
+ frt_os_write_byte(os, (frt_uchar)((num >> 32) & 0xFF));
500
+ frt_os_write_byte(os, (frt_uchar)((num >> 24) & 0xFF));
501
+ frt_os_write_byte(os, (frt_uchar)((num >> 16) & 0xFF));
502
+ frt_os_write_byte(os, (frt_uchar)((num >> 8) & 0xFF));
503
+ frt_os_write_byte(os, (frt_uchar)(num & 0xFF));
504
+ }
505
+
506
+ void frt_os_write_u32(FrtOutStream *os, frt_u32 num)
507
+ {
508
+ frt_os_write_byte(os, (frt_uchar)((num >> 24) & 0xFF));
509
+ frt_os_write_byte(os, (frt_uchar)((num >> 16) & 0xFF));
510
+ frt_os_write_byte(os, (frt_uchar)((num >> 8) & 0xFF));
511
+ frt_os_write_byte(os, (frt_uchar)(num & 0xFF));
512
+ }
513
+
514
+ void frt_os_write_u64(FrtOutStream *os, frt_u64 num)
515
+ {
516
+ frt_os_write_byte(os, (frt_uchar)((num >> 56) & 0xFF));
517
+ frt_os_write_byte(os, (frt_uchar)((num >> 48) & 0xFF));
518
+ frt_os_write_byte(os, (frt_uchar)((num >> 40) & 0xFF));
519
+ frt_os_write_byte(os, (frt_uchar)((num >> 32) & 0xFF));
520
+ frt_os_write_byte(os, (frt_uchar)((num >> 24) & 0xFF));
521
+ frt_os_write_byte(os, (frt_uchar)((num >> 16) & 0xFF));
522
+ frt_os_write_byte(os, (frt_uchar)((num >> 8) & 0xFF));
523
+ frt_os_write_byte(os, (frt_uchar)(num & 0xFF));
524
+ }
525
+
526
+ /* optimized to use an unchecked write if there is space */
527
+ void frt_os_write_vint(FrtOutStream *os, register unsigned int num)
528
+ {
529
+ if (os->buf.pos > VINT_END) {
530
+ while (num > 127) {
531
+ frt_os_write_byte(os, (frt_uchar)((num & 0x7f) | 0x80));
532
+ num >>= 7;
533
+ }
534
+ frt_os_write_byte(os, (frt_uchar)(num));
535
+ }
536
+ else {
537
+ while (num > 127) {
538
+ write_byte(os, (frt_uchar)((num & 0x7f) | 0x80));
539
+ num >>= 7;
540
+ }
541
+ write_byte(os, (frt_uchar)(num));
542
+ }
543
+ }
544
+
545
+ /* optimized to use an unchecked write if there is space */
546
+ void frt_os_write_voff_t(FrtOutStream *os, register off_t num)
547
+ {
548
+ if (os->buf.pos > VINT_END) {
549
+ while (num > 127) {
550
+ frt_os_write_byte(os, (frt_uchar)((num & 0x7f) | 0x80));
551
+ num >>= 7;
552
+ }
553
+ frt_os_write_byte(os, (frt_uchar)num);
554
+ }
555
+ else {
556
+ while (num > 127) {
557
+ write_byte(os, (frt_uchar)((num & 0x7f) | 0x80));
558
+ num >>= 7;
559
+ }
560
+ write_byte(os, (frt_uchar)num);
561
+ }
562
+ }
563
+
564
+ /* optimized to use an unchecked write if there is space */
565
+ void frt_os_write_vll(FrtOutStream *os, register frt_u64 num)
566
+ {
567
+ if (os->buf.pos > VINT_END) {
568
+ while (num > 127) {
569
+ frt_os_write_byte(os, (frt_uchar)((num & 0x7f) | 0x80));
570
+ num >>= 7;
571
+ }
572
+ frt_os_write_byte(os, (frt_uchar)num);
573
+ }
574
+ else {
575
+ while (num > 127) {
576
+ write_byte(os, (frt_uchar)((num & 0x7f) | 0x80));
577
+ num >>= 7;
578
+ }
579
+ write_byte(os, (frt_uchar)num);
580
+ }
581
+ }
582
+
583
+ void frt_os_write_string_len(FrtOutStream *os, const char *str, int len)
584
+ {
585
+ frt_os_write_vint(os, len);
586
+ frt_os_write_bytes(os, (frt_uchar *)str, len);
587
+ }
588
+ void frt_os_write_string(FrtOutStream *os, const char *str)
589
+ {
590
+ frt_os_write_string_len(os, str, (int)strlen(str));
591
+ }
592
+
593
+ /**
594
+ * Determine if the filename is the name of a lock file. Return 1 if it is, 0
595
+ * otherwise.
596
+ *
597
+ * @param filename the name of the file to check
598
+ * @return 1 (true) if the file is a lock file, 0 (false) otherwise
599
+ */
600
+ int frt_file_is_lock(const char *filename)
601
+ {
602
+ int start = (int) strlen(filename) - 4;
603
+ return ((start > 0) && (strcmp(FRT_LOCK_EXT, &filename[start]) == 0));
604
+ }
605
+
606
+ void frt_is2os_copy_bytes(FrtInStream *is, FrtOutStream *os, int cnt)
607
+ {
608
+ int len;
609
+ frt_uchar buf[FRT_BUFFER_SIZE];
610
+
611
+ for (; cnt > 0; cnt -= FRT_BUFFER_SIZE) {
612
+ len = ((cnt > FRT_BUFFER_SIZE) ? FRT_BUFFER_SIZE : cnt);
613
+ frt_is_read_bytes(is, buf, len);
614
+ frt_os_write_bytes(os, buf, len);
615
+ }
616
+ }
617
+
618
+ void frt_is2os_copy_vints(FrtInStream *is, FrtOutStream *os, int cnt)
619
+ {
620
+ frt_uchar b;
621
+ for (; cnt > 0; cnt--) {
622
+ while (((b = frt_is_read_byte(is)) & 0x80) != 0) {
623
+ frt_os_write_byte(os, b);
624
+ }
625
+ frt_os_write_byte(os, b);
626
+ }
627
+ }
628
+
629
+ /**
630
+ * Test argument used to test the store->each function
631
+ */
632
+ struct FileNameListArg
633
+ {
634
+ int count;
635
+ int size;
636
+ int total_len;
637
+ char **files;
638
+ };
639
+
640
+ /**
641
+ * Test function used to test store->each function
642
+ */
643
+ static void add_file_name(const char *fname, void *arg)
644
+ {
645
+ struct FileNameListArg *fnl = (struct FileNameListArg *)arg;
646
+ if (fnl->count >= fnl->size) {
647
+ fnl->size *= 2;
648
+ FRT_REALLOC_N(fnl->files, char *, fnl->size);
649
+ }
650
+ fnl->files[fnl->count++] = frt_estrdup(fname);
651
+ fnl->total_len += strlen(fname) + 2;
652
+ }
653
+
654
+ char *frt_store_to_s(FrtStore *store)
655
+ {
656
+ struct FileNameListArg fnl;
657
+ char *buf, *b;
658
+ int i;
659
+ fnl.count = 0;
660
+ fnl.size = 16;
661
+ fnl.total_len = 10;
662
+ fnl.files = FRT_ALLOC_N(char *, 16);
663
+
664
+ store->each(store, &add_file_name, &fnl);
665
+ qsort(fnl.files, fnl.count, sizeof(char *), &frt_scmp);
666
+ b = buf = FRT_ALLOC_N(char, fnl.total_len);
667
+
668
+ for (i = 0; i < fnl.count; i++) {
669
+ char *fn = fnl.files[i];
670
+ int len = strlen(fn);
671
+ memcpy(b, fn, len);
672
+ b += len;
673
+ *b++ = '\n';
674
+ free(fn);
675
+ }
676
+ *b = '\0';
677
+ free(fnl.files);
678
+
679
+ return buf;
680
+ }