isomorfeus-ferret 0.12.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (222) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +612 -0
  3. data/README.md +44 -0
  4. data/ext/isomorfeus_ferret_ext/benchmark.c +223 -0
  5. data/ext/isomorfeus_ferret_ext/benchmark.h +45 -0
  6. data/ext/isomorfeus_ferret_ext/benchmarks_all.h +25 -0
  7. data/ext/isomorfeus_ferret_ext/bm_bitvector.c +123 -0
  8. data/ext/isomorfeus_ferret_ext/bm_hash.c +118 -0
  9. data/ext/isomorfeus_ferret_ext/bm_micro_string.c +40 -0
  10. data/ext/isomorfeus_ferret_ext/bm_store.c +93 -0
  11. data/ext/isomorfeus_ferret_ext/email.rl +21 -0
  12. data/ext/isomorfeus_ferret_ext/extconf.rb +5 -0
  13. data/ext/isomorfeus_ferret_ext/fio_tmpfile.h +53 -0
  14. data/ext/isomorfeus_ferret_ext/frb_analysis.c +2577 -0
  15. data/ext/isomorfeus_ferret_ext/frb_index.c +3457 -0
  16. data/ext/isomorfeus_ferret_ext/frb_lang.c +9 -0
  17. data/ext/isomorfeus_ferret_ext/frb_lang.h +17 -0
  18. data/ext/isomorfeus_ferret_ext/frb_qparser.c +629 -0
  19. data/ext/isomorfeus_ferret_ext/frb_search.c +4460 -0
  20. data/ext/isomorfeus_ferret_ext/frb_store.c +515 -0
  21. data/ext/isomorfeus_ferret_ext/frb_threading.h +30 -0
  22. data/ext/isomorfeus_ferret_ext/frb_utils.c +1127 -0
  23. data/ext/isomorfeus_ferret_ext/frt_analysis.c +1644 -0
  24. data/ext/isomorfeus_ferret_ext/frt_analysis.h +247 -0
  25. data/ext/isomorfeus_ferret_ext/frt_array.c +124 -0
  26. data/ext/isomorfeus_ferret_ext/frt_array.h +54 -0
  27. data/ext/isomorfeus_ferret_ext/frt_bitvector.c +95 -0
  28. data/ext/isomorfeus_ferret_ext/frt_bitvector.h +586 -0
  29. data/ext/isomorfeus_ferret_ext/frt_compound_io.c +374 -0
  30. data/ext/isomorfeus_ferret_ext/frt_config.h +44 -0
  31. data/ext/isomorfeus_ferret_ext/frt_document.c +134 -0
  32. data/ext/isomorfeus_ferret_ext/frt_document.h +52 -0
  33. data/ext/isomorfeus_ferret_ext/frt_except.c +95 -0
  34. data/ext/isomorfeus_ferret_ext/frt_except.h +188 -0
  35. data/ext/isomorfeus_ferret_ext/frt_field_index.c +233 -0
  36. data/ext/isomorfeus_ferret_ext/frt_field_index.h +42 -0
  37. data/ext/isomorfeus_ferret_ext/frt_filter.c +157 -0
  38. data/ext/isomorfeus_ferret_ext/frt_fs_store.c +502 -0
  39. data/ext/isomorfeus_ferret_ext/frt_global.c +427 -0
  40. data/ext/isomorfeus_ferret_ext/frt_global.h +290 -0
  41. data/ext/isomorfeus_ferret_ext/frt_hash.c +518 -0
  42. data/ext/isomorfeus_ferret_ext/frt_hash.h +466 -0
  43. data/ext/isomorfeus_ferret_ext/frt_hashset.c +191 -0
  44. data/ext/isomorfeus_ferret_ext/frt_hashset.h +206 -0
  45. data/ext/isomorfeus_ferret_ext/frt_helper.c +62 -0
  46. data/ext/isomorfeus_ferret_ext/frt_helper.h +13 -0
  47. data/ext/isomorfeus_ferret_ext/frt_ind.c +353 -0
  48. data/ext/isomorfeus_ferret_ext/frt_ind.h +54 -0
  49. data/ext/isomorfeus_ferret_ext/frt_index.c +6377 -0
  50. data/ext/isomorfeus_ferret_ext/frt_index.h +880 -0
  51. data/ext/isomorfeus_ferret_ext/frt_lang.c +104 -0
  52. data/ext/isomorfeus_ferret_ext/frt_lang.h +44 -0
  53. data/ext/isomorfeus_ferret_ext/frt_mempool.c +87 -0
  54. data/ext/isomorfeus_ferret_ext/frt_mempool.h +33 -0
  55. data/ext/isomorfeus_ferret_ext/frt_multimapper.c +349 -0
  56. data/ext/isomorfeus_ferret_ext/frt_multimapper.h +52 -0
  57. data/ext/isomorfeus_ferret_ext/frt_posh.c +1006 -0
  58. data/ext/isomorfeus_ferret_ext/frt_posh.h +973 -0
  59. data/ext/isomorfeus_ferret_ext/frt_priorityqueue.c +147 -0
  60. data/ext/isomorfeus_ferret_ext/frt_priorityqueue.h +147 -0
  61. data/ext/isomorfeus_ferret_ext/frt_q_boolean.c +1612 -0
  62. data/ext/isomorfeus_ferret_ext/frt_q_const_score.c +157 -0
  63. data/ext/isomorfeus_ferret_ext/frt_q_filtered_query.c +209 -0
  64. data/ext/isomorfeus_ferret_ext/frt_q_fuzzy.c +281 -0
  65. data/ext/isomorfeus_ferret_ext/frt_q_match_all.c +147 -0
  66. data/ext/isomorfeus_ferret_ext/frt_q_multi_term.c +672 -0
  67. data/ext/isomorfeus_ferret_ext/frt_q_parser.c +3084 -0
  68. data/ext/isomorfeus_ferret_ext/frt_q_phrase.c +1182 -0
  69. data/ext/isomorfeus_ferret_ext/frt_q_prefix.c +98 -0
  70. data/ext/isomorfeus_ferret_ext/frt_q_range.c +665 -0
  71. data/ext/isomorfeus_ferret_ext/frt_q_span.c +2386 -0
  72. data/ext/isomorfeus_ferret_ext/frt_q_term.c +311 -0
  73. data/ext/isomorfeus_ferret_ext/frt_q_wildcard.c +166 -0
  74. data/ext/isomorfeus_ferret_ext/frt_ram_store.c +460 -0
  75. data/ext/isomorfeus_ferret_ext/frt_scanner.c +899 -0
  76. data/ext/isomorfeus_ferret_ext/frt_scanner.h +28 -0
  77. data/ext/isomorfeus_ferret_ext/frt_scanner_mb.c +6705 -0
  78. data/ext/isomorfeus_ferret_ext/frt_scanner_utf8.c +4419 -0
  79. data/ext/isomorfeus_ferret_ext/frt_search.c +1824 -0
  80. data/ext/isomorfeus_ferret_ext/frt_search.h +924 -0
  81. data/ext/isomorfeus_ferret_ext/frt_similarity.c +150 -0
  82. data/ext/isomorfeus_ferret_ext/frt_similarity.h +79 -0
  83. data/ext/isomorfeus_ferret_ext/frt_sort.c +796 -0
  84. data/ext/isomorfeus_ferret_ext/frt_stopwords.c +395 -0
  85. data/ext/isomorfeus_ferret_ext/frt_store.c +680 -0
  86. data/ext/isomorfeus_ferret_ext/frt_store.h +789 -0
  87. data/ext/isomorfeus_ferret_ext/frt_term_vectors.c +72 -0
  88. data/ext/isomorfeus_ferret_ext/frt_threading.h +23 -0
  89. data/ext/isomorfeus_ferret_ext/frt_win32.h +54 -0
  90. data/ext/isomorfeus_ferret_ext/isomorfeus_ferret.c +409 -0
  91. data/ext/isomorfeus_ferret_ext/isomorfeus_ferret.h +95 -0
  92. data/ext/isomorfeus_ferret_ext/libstemmer.c +93 -0
  93. data/ext/isomorfeus_ferret_ext/libstemmer.h +73 -0
  94. data/ext/isomorfeus_ferret_ext/q_parser.y +1366 -0
  95. data/ext/isomorfeus_ferret_ext/scanner.h +28 -0
  96. data/ext/isomorfeus_ferret_ext/scanner.in +43 -0
  97. data/ext/isomorfeus_ferret_ext/scanner.rl +84 -0
  98. data/ext/isomorfeus_ferret_ext/scanner_mb.rl +200 -0
  99. data/ext/isomorfeus_ferret_ext/scanner_utf8.rl +85 -0
  100. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_danish.c +324 -0
  101. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_danish.h +7 -0
  102. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_dutch.c +610 -0
  103. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_dutch.h +6 -0
  104. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_english.c +1104 -0
  105. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_english.h +6 -0
  106. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_finnish.c +749 -0
  107. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_finnish.h +7 -0
  108. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_french.c +1233 -0
  109. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_french.h +6 -0
  110. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_german.c +490 -0
  111. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_german.h +6 -0
  112. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_hungarian.c +1217 -0
  113. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_hungarian.h +7 -0
  114. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_italian.c +1052 -0
  115. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_italian.h +6 -0
  116. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_norwegian.c +283 -0
  117. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_norwegian.h +6 -0
  118. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_porter.c +735 -0
  119. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_porter.h +6 -0
  120. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_portuguese.c +1003 -0
  121. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_portuguese.h +7 -0
  122. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_spanish.c +1079 -0
  123. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_spanish.h +6 -0
  124. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_swedish.c +293 -0
  125. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_swedish.h +6 -0
  126. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_2_romanian.c +984 -0
  127. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_2_romanian.h +6 -0
  128. data/ext/isomorfeus_ferret_ext/stem_KOI8_R_russian.c +686 -0
  129. data/ext/isomorfeus_ferret_ext/stem_KOI8_R_russian.h +6 -0
  130. data/ext/isomorfeus_ferret_ext/stem_UTF_8_danish.c +325 -0
  131. data/ext/isomorfeus_ferret_ext/stem_UTF_8_danish.h +6 -0
  132. data/ext/isomorfeus_ferret_ext/stem_UTF_8_dutch.c +620 -0
  133. data/ext/isomorfeus_ferret_ext/stem_UTF_8_dutch.h +6 -0
  134. data/ext/isomorfeus_ferret_ext/stem_UTF_8_english.c +1111 -0
  135. data/ext/isomorfeus_ferret_ext/stem_UTF_8_english.h +6 -0
  136. data/ext/isomorfeus_ferret_ext/stem_UTF_8_finnish.c +754 -0
  137. data/ext/isomorfeus_ferret_ext/stem_UTF_8_finnish.h +6 -0
  138. data/ext/isomorfeus_ferret_ext/stem_UTF_8_french.c +1242 -0
  139. data/ext/isomorfeus_ferret_ext/stem_UTF_8_french.h +6 -0
  140. data/ext/isomorfeus_ferret_ext/stem_UTF_8_german.c +495 -0
  141. data/ext/isomorfeus_ferret_ext/stem_UTF_8_german.h +6 -0
  142. data/ext/isomorfeus_ferret_ext/stem_UTF_8_hungarian.c +1220 -0
  143. data/ext/isomorfeus_ferret_ext/stem_UTF_8_hungarian.h +6 -0
  144. data/ext/isomorfeus_ferret_ext/stem_UTF_8_italian.c +1059 -0
  145. data/ext/isomorfeus_ferret_ext/stem_UTF_8_italian.h +6 -0
  146. data/ext/isomorfeus_ferret_ext/stem_UTF_8_norwegian.c +285 -0
  147. data/ext/isomorfeus_ferret_ext/stem_UTF_8_norwegian.h +6 -0
  148. data/ext/isomorfeus_ferret_ext/stem_UTF_8_porter.c +741 -0
  149. data/ext/isomorfeus_ferret_ext/stem_UTF_8_porter.h +6 -0
  150. data/ext/isomorfeus_ferret_ext/stem_UTF_8_portuguese.c +1009 -0
  151. data/ext/isomorfeus_ferret_ext/stem_UTF_8_portuguese.h +6 -0
  152. data/ext/isomorfeus_ferret_ext/stem_UTF_8_romanian.c +990 -0
  153. data/ext/isomorfeus_ferret_ext/stem_UTF_8_romanian.h +6 -0
  154. data/ext/isomorfeus_ferret_ext/stem_UTF_8_russian.c +680 -0
  155. data/ext/isomorfeus_ferret_ext/stem_UTF_8_russian.h +6 -0
  156. data/ext/isomorfeus_ferret_ext/stem_UTF_8_spanish.c +1083 -0
  157. data/ext/isomorfeus_ferret_ext/stem_UTF_8_spanish.h +6 -0
  158. data/ext/isomorfeus_ferret_ext/stem_UTF_8_swedish.c +294 -0
  159. data/ext/isomorfeus_ferret_ext/stem_UTF_8_swedish.h +6 -0
  160. data/ext/isomorfeus_ferret_ext/stem_UTF_8_turkish.c +2191 -0
  161. data/ext/isomorfeus_ferret_ext/stem_UTF_8_turkish.h +6 -0
  162. data/ext/isomorfeus_ferret_ext/stem_api.c +66 -0
  163. data/ext/isomorfeus_ferret_ext/stem_api.h +26 -0
  164. data/ext/isomorfeus_ferret_ext/stem_header.h +57 -0
  165. data/ext/isomorfeus_ferret_ext/stem_modules.h +190 -0
  166. data/ext/isomorfeus_ferret_ext/stem_modules.txt +50 -0
  167. data/ext/isomorfeus_ferret_ext/stem_utilities.c +478 -0
  168. data/ext/isomorfeus_ferret_ext/test.c +850 -0
  169. data/ext/isomorfeus_ferret_ext/test.h +416 -0
  170. data/ext/isomorfeus_ferret_ext/test_1710.c +63 -0
  171. data/ext/isomorfeus_ferret_ext/test_analysis.c +1221 -0
  172. data/ext/isomorfeus_ferret_ext/test_array.c +272 -0
  173. data/ext/isomorfeus_ferret_ext/test_bitvector.c +600 -0
  174. data/ext/isomorfeus_ferret_ext/test_compound_io.c +170 -0
  175. data/ext/isomorfeus_ferret_ext/test_document.c +156 -0
  176. data/ext/isomorfeus_ferret_ext/test_except.c +244 -0
  177. data/ext/isomorfeus_ferret_ext/test_fields.c +522 -0
  178. data/ext/isomorfeus_ferret_ext/test_file_deleter.c +185 -0
  179. data/ext/isomorfeus_ferret_ext/test_filter.c +331 -0
  180. data/ext/isomorfeus_ferret_ext/test_fs_store.c +25 -0
  181. data/ext/isomorfeus_ferret_ext/test_global.c +299 -0
  182. data/ext/isomorfeus_ferret_ext/test_hash.c +485 -0
  183. data/ext/isomorfeus_ferret_ext/test_hashset.c +288 -0
  184. data/ext/isomorfeus_ferret_ext/test_helper.c +47 -0
  185. data/ext/isomorfeus_ferret_ext/test_highlighter.c +548 -0
  186. data/ext/isomorfeus_ferret_ext/test_index.c +2323 -0
  187. data/ext/isomorfeus_ferret_ext/test_lang.c +74 -0
  188. data/ext/isomorfeus_ferret_ext/test_mempool.c +102 -0
  189. data/ext/isomorfeus_ferret_ext/test_multimapper.c +64 -0
  190. data/ext/isomorfeus_ferret_ext/test_priorityqueue.c +213 -0
  191. data/ext/isomorfeus_ferret_ext/test_q_const_score.c +84 -0
  192. data/ext/isomorfeus_ferret_ext/test_q_filtered.c +61 -0
  193. data/ext/isomorfeus_ferret_ext/test_q_fuzzy.c +241 -0
  194. data/ext/isomorfeus_ferret_ext/test_q_parser.c +464 -0
  195. data/ext/isomorfeus_ferret_ext/test_q_span.c +575 -0
  196. data/ext/isomorfeus_ferret_ext/test_ram_store.c +77 -0
  197. data/ext/isomorfeus_ferret_ext/test_search.c +1874 -0
  198. data/ext/isomorfeus_ferret_ext/test_segments.c +167 -0
  199. data/ext/isomorfeus_ferret_ext/test_similarity.c +25 -0
  200. data/ext/isomorfeus_ferret_ext/test_sort.c +333 -0
  201. data/ext/isomorfeus_ferret_ext/test_store.c +591 -0
  202. data/ext/isomorfeus_ferret_ext/test_store.h +3 -0
  203. data/ext/isomorfeus_ferret_ext/test_term.c +351 -0
  204. data/ext/isomorfeus_ferret_ext/test_term_vectors.c +373 -0
  205. data/ext/isomorfeus_ferret_ext/test_test.c +83 -0
  206. data/ext/isomorfeus_ferret_ext/test_threading.c +188 -0
  207. data/ext/isomorfeus_ferret_ext/testhelper.c +561 -0
  208. data/ext/isomorfeus_ferret_ext/testhelper.h +25 -0
  209. data/ext/isomorfeus_ferret_ext/tests_all.h +87 -0
  210. data/ext/isomorfeus_ferret_ext/uchar-ucs4.rl +1854 -0
  211. data/ext/isomorfeus_ferret_ext/uchar-utf8.rl +1999 -0
  212. data/ext/isomorfeus_ferret_ext/url.rl +27 -0
  213. data/ext/isomorfeus_ferret_ext/word_list.h +15156 -0
  214. data/lib/isomorfeus/ferret/document.rb +132 -0
  215. data/lib/isomorfeus/ferret/field_symbol.rb +85 -0
  216. data/lib/isomorfeus/ferret/index/field_infos.rb +48 -0
  217. data/lib/isomorfeus/ferret/index/index.rb +970 -0
  218. data/lib/isomorfeus/ferret/monitor.rb +323 -0
  219. data/lib/isomorfeus/ferret/stdlib_patches.rb +151 -0
  220. data/lib/isomorfeus/ferret/version.rb +5 -0
  221. data/lib/isomorfeus-ferret.rb +8 -0
  222. metadata +307 -0
@@ -0,0 +1,502 @@
1
+ #include "frt_store.h"
2
+ #include "frt_lang.h"
3
+ #include <time.h>
4
+ #include <sys/types.h>
5
+ #include <fcntl.h>
6
+ #include <sys/stat.h>
7
+ #include <errno.h>
8
+ #include <string.h>
9
+ #include <stdio.h>
10
+ #if (defined POSH_OS_WIN32 || defined POSH_OS_WIN64) && !defined __MINGW32__
11
+ # include <io.h>
12
+ # include "frt_win32.h"
13
+ # ifndef sleep
14
+ # define sleep _sleep
15
+ # endif
16
+ # ifndef DIR_SEPARATOR
17
+ # define DIR_SEPARATOR "\\"
18
+ # define DIR_SEPARATOR_CHAR '\\'
19
+ # endif
20
+ # ifndef S_IRUSR
21
+ # define S_IRUSR _S_IREAD
22
+ # endif
23
+ # ifndef S_IWUSR
24
+ # define S_IWUSR _S_IWRITE
25
+ # endif
26
+ #else
27
+ # define DIR_SEPARATOR "/"
28
+ # define DIR_SEPARATOR_CHAR '/'
29
+ # include <unistd.h>
30
+ # include <dirent.h>
31
+ #endif
32
+ #ifndef O_BINARY
33
+ # define O_BINARY 0
34
+ #endif
35
+
36
+ extern void frt_micro_sleep(const int micro_seconds);
37
+
38
+ /**
39
+ * Create a filepath for a file in the store using the operating systems
40
+ * default file seperator.
41
+ */
42
+ static char *join_path(char *buf, const char *base, const char *filename)
43
+ {
44
+ snprintf(buf, FRT_MAX_FILE_PATH, "%s"DIR_SEPARATOR"%s", base, filename);
45
+ return buf;
46
+ }
47
+
48
+ static void fs_touch(FrtStore *store, const char *filename)
49
+ {
50
+ int f;
51
+ char path[FRT_MAX_FILE_PATH];
52
+ join_path(path, store->dir.path, filename);
53
+ if ((f = creat(path, store->file_mode)) == 0) {
54
+ FRT_RAISE(FRT_IO_ERROR, "couldn't create file %s: <%s>", path,
55
+ strerror(errno));
56
+ }
57
+ close(f);
58
+ }
59
+
60
+ static int fs_exists(FrtStore *store, const char *filename)
61
+ {
62
+ int fd;
63
+ char path[FRT_MAX_FILE_PATH];
64
+ join_path(path, store->dir.path, filename);
65
+ fd = open(path, 0);
66
+ if (fd < 0) {
67
+ if (errno != ENOENT) {
68
+ FRT_RAISE(FRT_IO_ERROR, "checking existance of %s: <%s>", path,
69
+ strerror(errno));
70
+ }
71
+ return false;
72
+ }
73
+ close(fd);
74
+ return true;
75
+ }
76
+
77
+ static int fs_remove(FrtStore *store, const char *filename)
78
+ {
79
+ char path[FRT_MAX_FILE_PATH];
80
+ return remove(join_path(path, store->dir.path, filename));
81
+ }
82
+
83
+ static void fs_rename(FrtStore *store, const char *from, const char *to)
84
+ {
85
+ char path1[FRT_MAX_FILE_PATH], path2[FRT_MAX_FILE_PATH];
86
+
87
+ #if defined POSH_OS_WIN32 || defined POSH_OS_WIN64
88
+ remove(join_path(path1, store->dir.path, to));
89
+ #endif
90
+
91
+ if (rename(join_path(path1, store->dir.path, from),
92
+ join_path(path2, store->dir.path, to)) < 0) {
93
+ FRT_RAISE(FRT_IO_ERROR, "couldn't rename file \"%s\" to \"%s\": <%s>",
94
+ path1, path2, strerror(errno));
95
+ }
96
+ }
97
+
98
+ static int fs_count(FrtStore *store)
99
+ {
100
+ int cnt = 0;
101
+ struct dirent *de;
102
+ DIR *d = opendir(store->dir.path);
103
+
104
+ if (!d) {
105
+ FRT_RAISE(FRT_IO_ERROR, "counting files in %s: <%s>",
106
+ store->dir.path, strerror(errno));
107
+ }
108
+
109
+ while ((de = readdir(d)) != NULL) {
110
+ if (de->d_name[0] > '/') { /* skip ., .., / and '\0'*/
111
+ cnt++;
112
+ }
113
+ }
114
+ closedir(d);
115
+
116
+ return cnt;
117
+ }
118
+
119
+ static void fs_each(FrtStore *store, void (*func)(const char *fname, void *arg), void *arg)
120
+ {
121
+ struct dirent *de;
122
+ DIR *d = opendir(store->dir.path);
123
+
124
+ if (!d) {
125
+ FRT_RAISE(FRT_IO_ERROR, "doing 'each' in %s: <%s>",
126
+ store->dir.path, strerror(errno));
127
+ }
128
+
129
+ while ((de = readdir(d)) != NULL) {
130
+ if (de->d_name[0] > '/' /* skip ., .., / and '\0'*/
131
+ && !frt_file_is_lock(de->d_name)) {
132
+ func(de->d_name, arg);
133
+ }
134
+ }
135
+ closedir(d);
136
+ }
137
+
138
+ static void fs_clear_locks(FrtStore *store)
139
+ {
140
+ struct dirent *de;
141
+ DIR *d = opendir(store->dir.path);
142
+
143
+ if (!d) {
144
+ FRT_RAISE(FRT_IO_ERROR, "clearing locks in %s: <%s>",
145
+ store->dir.path, strerror(errno));
146
+ }
147
+
148
+ while ((de = readdir(d)) != NULL) {
149
+ if (frt_file_is_lock(de->d_name)) {
150
+ char path[FRT_MAX_FILE_PATH];
151
+ remove(join_path(path, store->dir.path, de->d_name));
152
+ }
153
+ }
154
+ closedir(d);
155
+ }
156
+
157
+ static void remove_if_index_file(const char *base_path, const char *file_name)
158
+ {
159
+ char path[FRT_MAX_FILE_PATH];
160
+ char *basename;
161
+ join_path(path, base_path, file_name);
162
+ /* get basename of path */
163
+ basename = strrchr(path, DIR_SEPARATOR_CHAR);
164
+ basename = (basename ? basename + 1 : path);
165
+ /* we don't want to delete non-index files here */
166
+ if (frt_file_name_filter_is_index_file(basename, true)) {
167
+ remove(path);
168
+ }
169
+ }
170
+
171
+ static void fs_clear(FrtStore *store)
172
+ {
173
+ struct dirent *de;
174
+ DIR *d = opendir(store->dir.path);
175
+
176
+ if (!d) {
177
+ FRT_RAISE(FRT_IO_ERROR, "clearing files in %s: <%s>",
178
+ store->dir.path, strerror(errno));
179
+ }
180
+
181
+ while ((de = readdir(d)) != NULL) {
182
+ if (de->d_name[0] > '/' /* skip ., .., / and '\0'*/
183
+ && !frt_file_is_lock(de->d_name)) {
184
+ remove_if_index_file(store->dir.path, de->d_name);
185
+ }
186
+ }
187
+ closedir(d);
188
+ }
189
+
190
+ /**
191
+ * Clear all files which belong to the index. Use fs_clear to clear the
192
+ * directory regardless of the files origin.
193
+ */
194
+ static void fs_clear_all(FrtStore *store)
195
+ {
196
+ struct dirent *de;
197
+ DIR *d = opendir(store->dir.path);
198
+
199
+ if (!d) {
200
+ FRT_RAISE(FRT_IO_ERROR, "clearing all files in %s: <%s>",
201
+ store->dir.path, strerror(errno));
202
+ }
203
+
204
+ while ((de = readdir(d)) != NULL) {
205
+ if (de->d_name[0] > '/') { /* skip ., .., / and '\0'*/
206
+ remove_if_index_file(store->dir.path, de->d_name);
207
+ }
208
+ }
209
+ closedir(d);
210
+ }
211
+
212
+ /**
213
+ * Destroy the store.
214
+ *
215
+ * @param p the store to destroy
216
+ * @rb_raise FRT_IO_ERROR if there is an error deleting the locks
217
+ */
218
+ static void fs_destroy(FrtStore *store)
219
+ {
220
+ FRT_TRY
221
+ fs_clear_locks(store);
222
+ FRT_XCATCHALL
223
+ FRT_HANDLED();
224
+ FRT_XENDTRY
225
+ free(store->dir.path);
226
+ frt_store_destroy(store);
227
+ }
228
+
229
+ static off_t fs_length(FrtStore *store, const char *filename)
230
+ {
231
+ char path[FRT_MAX_FILE_PATH];
232
+ struct stat stt;
233
+
234
+ if (stat(join_path(path, store->dir.path, filename), &stt)) {
235
+ FRT_RAISE(FRT_IO_ERROR, "getting lenth of %s: <%s>", path,
236
+ strerror(errno));
237
+ }
238
+
239
+ return stt.st_size;
240
+ }
241
+
242
+ static void fso_flush_i(FrtOutStream *os, const frt_uchar *src, int len)
243
+ {
244
+ if (len != write(os->file.fd, src, len)) {
245
+ FRT_RAISE(FRT_IO_ERROR, "flushing src of length %d, <%s>", len,
246
+ strerror(errno));
247
+ }
248
+ }
249
+
250
+ static void fso_seek_i(FrtOutStream *os, off_t pos)
251
+ {
252
+ if (lseek(os->file.fd, pos, SEEK_SET) < 0) {
253
+ FRT_RAISE(FRT_IO_ERROR, "seeking position %"FRT_OFF_T_PFX"d: <%s>",
254
+ pos, strerror(errno));
255
+ }
256
+ }
257
+
258
+ static void fso_close_i(FrtOutStream *os)
259
+ {
260
+ if (close(os->file.fd)) {
261
+ FRT_RAISE(FRT_IO_ERROR, "closing file: <%s>", strerror(errno));
262
+ }
263
+ }
264
+
265
+ static const struct FrtOutStreamMethods FS_OUT_STREAM_METHODS = {
266
+ fso_flush_i,
267
+ fso_seek_i,
268
+ fso_close_i
269
+ };
270
+
271
+ static FrtOutStream *fs_new_output(FrtStore *store, const char *filename)
272
+ {
273
+ char path[FRT_MAX_FILE_PATH];
274
+ int fd = open(join_path(path, store->dir.path, filename),
275
+ O_WRONLY | O_CREAT | O_BINARY, store->file_mode);
276
+ FrtOutStream *os;
277
+ if (fd < 0) {
278
+ FRT_RAISE(FRT_IO_ERROR, "couldn't create OutStream %s: <%s>",
279
+ path, strerror(errno));
280
+ }
281
+
282
+ os = frt_os_new();
283
+ os->file.fd = fd;
284
+ os->m = &FS_OUT_STREAM_METHODS;
285
+ return os;
286
+ }
287
+
288
+ static void fsi_read_i(FrtInStream *is, frt_uchar *path, int len)
289
+ {
290
+ int fd = is->file.fd;
291
+ off_t pos = frt_is_pos(is);
292
+ if (pos != lseek(fd, 0, SEEK_CUR)) {
293
+ lseek(fd, pos, SEEK_SET);
294
+ }
295
+ if (read(fd, path, len) != len) {
296
+ /* win: the wrong value can be returned for some reason so double check */
297
+ if (lseek(fd, 0, SEEK_CUR) != (pos + len)) {
298
+ FRT_RAISE(FRT_IO_ERROR, "couldn't read %d chars from %s: <%s>",
299
+ len, path, strerror(errno));
300
+ }
301
+ }
302
+ }
303
+
304
+ static void fsi_seek_i(FrtInStream *is, off_t pos)
305
+ {
306
+ if (lseek(is->file.fd, pos, SEEK_SET) < 0) {
307
+ FRT_RAISE(FRT_IO_ERROR, "seeking pos %"FRT_OFF_T_PFX"d: <%s>",
308
+ pos, strerror(errno));
309
+ }
310
+ }
311
+
312
+ static void fsi_close_i(FrtInStream *is)
313
+ {
314
+ if (close(is->file.fd)) {
315
+ FRT_RAISE(FRT_IO_ERROR, "%s", strerror(errno));
316
+ }
317
+ free(is->d.path);
318
+ }
319
+
320
+ static off_t fsi_length_i(FrtInStream *is)
321
+ {
322
+ struct stat stt;
323
+ if (fstat(is->file.fd, &stt)) {
324
+ FRT_RAISE(FRT_IO_ERROR, "fstat failed: <%s>", strerror(errno));
325
+ }
326
+ return stt.st_size;
327
+ }
328
+
329
+ static const struct FrtInStreamMethods FS_IN_STREAM_METHODS = {
330
+ fsi_read_i,
331
+ fsi_seek_i,
332
+ fsi_length_i,
333
+ fsi_close_i
334
+ };
335
+
336
+ static FrtInStream *fs_open_input(FrtStore *store, const char *filename)
337
+ {
338
+ FrtInStream *is;
339
+ char path[FRT_MAX_FILE_PATH];
340
+ int fd = open(join_path(path, store->dir.path, filename), O_RDONLY | O_BINARY);
341
+ if (fd < 0) {
342
+ FRT_RAISE(FRT_FILE_NOT_FOUND_ERROR,
343
+ "tried to open \"%s\" but it doesn't exist: <%s>",
344
+ path, strerror(errno));
345
+ }
346
+ is = frt_is_new();
347
+ is->file.fd = fd;
348
+ is->d.path = frt_estrdup(path);
349
+ is->m = &FS_IN_STREAM_METHODS;
350
+ return is;
351
+ }
352
+
353
+ #define LOCK_OBTAIN_TIMEOUT 10
354
+
355
+ static int fs_lock_obtain(FrtLock *lock)
356
+ {
357
+ int f;
358
+ int trys = LOCK_OBTAIN_TIMEOUT;
359
+ while (((f =
360
+ open(lock->name, O_CREAT | O_EXCL | O_RDWR,
361
+ S_IRUSR | S_IWUSR)) < 0) && (trys > 0)) {
362
+
363
+ /* sleep for 10 milliseconds */
364
+ frt_micro_sleep(10000);
365
+ trys--;
366
+ }
367
+ if (f >= 0) {
368
+ close(f);
369
+ return true;
370
+ }
371
+ else {
372
+ return false;
373
+ }
374
+ }
375
+
376
+ static int fs_lock_is_locked(FrtLock *lock)
377
+ {
378
+ int f = open(lock->name, O_CREAT | O_EXCL | O_WRONLY, S_IRUSR | S_IWUSR);
379
+ if (f >= 0) {
380
+ if (close(f) || remove(lock->name)) {
381
+ FRT_RAISE(FRT_IO_ERROR, "couldn't close lock \"%s\": <%s>", lock->name,
382
+ strerror(errno));
383
+ }
384
+ return false;
385
+ }
386
+ else {
387
+ return true;
388
+ }
389
+ }
390
+
391
+ static void fs_lock_release(FrtLock *lock)
392
+ {
393
+ remove(lock->name);
394
+ }
395
+
396
+ static FrtLock *fs_open_lock_i(FrtStore *store, const char *lockname)
397
+ {
398
+ FrtLock *lock = FRT_ALLOC(FrtLock);
399
+ char lname[100];
400
+ char path[FRT_MAX_FILE_PATH];
401
+ snprintf(lname, 100, "%s%s.lck", FRT_LOCK_PREFIX, lockname);
402
+ lock->name = frt_estrdup(join_path(path, store->dir.path, lname));
403
+ lock->store = store;
404
+ lock->obtain = &fs_lock_obtain;
405
+ lock->release = &fs_lock_release;
406
+ lock->is_locked = &fs_lock_is_locked;
407
+ return lock;
408
+ }
409
+
410
+ static void fs_close_lock_i(FrtLock *lock)
411
+ {
412
+ remove(lock->name);
413
+ free(lock->name);
414
+ free(lock);
415
+ }
416
+
417
+ static FrtHash *stores = NULL;
418
+
419
+ #ifndef UNTHREADED
420
+ static frt_mutex_t stores_mutex = FRT_MUTEX_INITIALIZER;
421
+ #endif
422
+
423
+ static void fs_close_i(FrtStore *store)
424
+ {
425
+ frt_mutex_lock(&stores_mutex);
426
+ frt_h_del(stores, store->dir.path);
427
+ frt_mutex_unlock(&stores_mutex);
428
+ }
429
+
430
+ static FrtStore *fs_store_new(const char *pathname)
431
+ {
432
+ FrtStore *new_store = frt_store_new();
433
+
434
+ new_store->file_mode = S_IRUSR | S_IWUSR;
435
+ #if !defined POSH_OS_WIN32 && !defined POSH_OS_WIN64
436
+ struct stat stt;
437
+ if (!stat(pathname, &stt)) {
438
+ gid_t st_gid = stt.st_gid;
439
+ bool is_grp = (st_gid == getgid());
440
+
441
+ if (!is_grp) {
442
+ int size = getgroups(0, NULL);
443
+ gid_t list[size];
444
+
445
+ if (getgroups(size, list) != -1) {
446
+ int i = 0;
447
+ while (i < size && !(is_grp = (st_gid == list[i]))) i++;
448
+ }
449
+ }
450
+
451
+ if (is_grp) {
452
+ if (stt.st_mode & S_IWGRP) {
453
+ umask(S_IWOTH);
454
+ }
455
+ new_store->file_mode |= stt.st_mode & (S_IRGRP | S_IWGRP);
456
+ }
457
+ }
458
+ #endif
459
+
460
+ new_store->dir.path = frt_estrdup(pathname);
461
+ new_store->touch = &fs_touch;
462
+ new_store->exists = &fs_exists;
463
+ new_store->remove = &fs_remove;
464
+ new_store->rename = &fs_rename;
465
+ new_store->count = &fs_count;
466
+ new_store->close_i = &fs_close_i;
467
+ new_store->clear = &fs_clear;
468
+ new_store->clear_all = &fs_clear_all;
469
+ new_store->clear_locks = &fs_clear_locks;
470
+ new_store->length = &fs_length;
471
+ new_store->each = &fs_each;
472
+ new_store->new_output = &fs_new_output;
473
+ new_store->open_input = &fs_open_input;
474
+ new_store->open_lock_i = &fs_open_lock_i;
475
+ new_store->close_lock_i = &fs_close_lock_i;
476
+ return new_store;
477
+ }
478
+
479
+ FrtStore *frt_open_fs_store(const char *pathname)
480
+ {
481
+ FrtStore *store = NULL;
482
+
483
+ if (!stores) {
484
+ stores = frt_h_new_str(NULL, (frt_free_ft)fs_destroy);
485
+ frt_register_for_cleanup(stores, (frt_free_ft)frt_h_destroy);
486
+ }
487
+
488
+ frt_mutex_lock(&stores_mutex);
489
+ store = (FrtStore *)frt_h_get(stores, pathname);
490
+ if (store) {
491
+ frt_mutex_lock(&store->mutex);
492
+ store->ref_cnt++;
493
+ frt_mutex_unlock(&store->mutex);
494
+ }
495
+ else {
496
+ store = fs_store_new(pathname);
497
+ frt_h_set(stores, store->dir.path, store);
498
+ }
499
+ frt_mutex_unlock(&stores_mutex);
500
+
501
+ return store;
502
+ }