isomorfeus-ferret 0.12.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (222) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +612 -0
  3. data/README.md +44 -0
  4. data/ext/isomorfeus_ferret_ext/benchmark.c +223 -0
  5. data/ext/isomorfeus_ferret_ext/benchmark.h +45 -0
  6. data/ext/isomorfeus_ferret_ext/benchmarks_all.h +25 -0
  7. data/ext/isomorfeus_ferret_ext/bm_bitvector.c +123 -0
  8. data/ext/isomorfeus_ferret_ext/bm_hash.c +118 -0
  9. data/ext/isomorfeus_ferret_ext/bm_micro_string.c +40 -0
  10. data/ext/isomorfeus_ferret_ext/bm_store.c +93 -0
  11. data/ext/isomorfeus_ferret_ext/email.rl +21 -0
  12. data/ext/isomorfeus_ferret_ext/extconf.rb +5 -0
  13. data/ext/isomorfeus_ferret_ext/fio_tmpfile.h +53 -0
  14. data/ext/isomorfeus_ferret_ext/frb_analysis.c +2577 -0
  15. data/ext/isomorfeus_ferret_ext/frb_index.c +3457 -0
  16. data/ext/isomorfeus_ferret_ext/frb_lang.c +9 -0
  17. data/ext/isomorfeus_ferret_ext/frb_lang.h +17 -0
  18. data/ext/isomorfeus_ferret_ext/frb_qparser.c +629 -0
  19. data/ext/isomorfeus_ferret_ext/frb_search.c +4460 -0
  20. data/ext/isomorfeus_ferret_ext/frb_store.c +515 -0
  21. data/ext/isomorfeus_ferret_ext/frb_threading.h +30 -0
  22. data/ext/isomorfeus_ferret_ext/frb_utils.c +1127 -0
  23. data/ext/isomorfeus_ferret_ext/frt_analysis.c +1644 -0
  24. data/ext/isomorfeus_ferret_ext/frt_analysis.h +247 -0
  25. data/ext/isomorfeus_ferret_ext/frt_array.c +124 -0
  26. data/ext/isomorfeus_ferret_ext/frt_array.h +54 -0
  27. data/ext/isomorfeus_ferret_ext/frt_bitvector.c +95 -0
  28. data/ext/isomorfeus_ferret_ext/frt_bitvector.h +586 -0
  29. data/ext/isomorfeus_ferret_ext/frt_compound_io.c +374 -0
  30. data/ext/isomorfeus_ferret_ext/frt_config.h +44 -0
  31. data/ext/isomorfeus_ferret_ext/frt_document.c +134 -0
  32. data/ext/isomorfeus_ferret_ext/frt_document.h +52 -0
  33. data/ext/isomorfeus_ferret_ext/frt_except.c +95 -0
  34. data/ext/isomorfeus_ferret_ext/frt_except.h +188 -0
  35. data/ext/isomorfeus_ferret_ext/frt_field_index.c +233 -0
  36. data/ext/isomorfeus_ferret_ext/frt_field_index.h +42 -0
  37. data/ext/isomorfeus_ferret_ext/frt_filter.c +157 -0
  38. data/ext/isomorfeus_ferret_ext/frt_fs_store.c +502 -0
  39. data/ext/isomorfeus_ferret_ext/frt_global.c +427 -0
  40. data/ext/isomorfeus_ferret_ext/frt_global.h +290 -0
  41. data/ext/isomorfeus_ferret_ext/frt_hash.c +518 -0
  42. data/ext/isomorfeus_ferret_ext/frt_hash.h +466 -0
  43. data/ext/isomorfeus_ferret_ext/frt_hashset.c +191 -0
  44. data/ext/isomorfeus_ferret_ext/frt_hashset.h +206 -0
  45. data/ext/isomorfeus_ferret_ext/frt_helper.c +62 -0
  46. data/ext/isomorfeus_ferret_ext/frt_helper.h +13 -0
  47. data/ext/isomorfeus_ferret_ext/frt_ind.c +353 -0
  48. data/ext/isomorfeus_ferret_ext/frt_ind.h +54 -0
  49. data/ext/isomorfeus_ferret_ext/frt_index.c +6377 -0
  50. data/ext/isomorfeus_ferret_ext/frt_index.h +880 -0
  51. data/ext/isomorfeus_ferret_ext/frt_lang.c +104 -0
  52. data/ext/isomorfeus_ferret_ext/frt_lang.h +44 -0
  53. data/ext/isomorfeus_ferret_ext/frt_mempool.c +87 -0
  54. data/ext/isomorfeus_ferret_ext/frt_mempool.h +33 -0
  55. data/ext/isomorfeus_ferret_ext/frt_multimapper.c +349 -0
  56. data/ext/isomorfeus_ferret_ext/frt_multimapper.h +52 -0
  57. data/ext/isomorfeus_ferret_ext/frt_posh.c +1006 -0
  58. data/ext/isomorfeus_ferret_ext/frt_posh.h +973 -0
  59. data/ext/isomorfeus_ferret_ext/frt_priorityqueue.c +147 -0
  60. data/ext/isomorfeus_ferret_ext/frt_priorityqueue.h +147 -0
  61. data/ext/isomorfeus_ferret_ext/frt_q_boolean.c +1612 -0
  62. data/ext/isomorfeus_ferret_ext/frt_q_const_score.c +157 -0
  63. data/ext/isomorfeus_ferret_ext/frt_q_filtered_query.c +209 -0
  64. data/ext/isomorfeus_ferret_ext/frt_q_fuzzy.c +281 -0
  65. data/ext/isomorfeus_ferret_ext/frt_q_match_all.c +147 -0
  66. data/ext/isomorfeus_ferret_ext/frt_q_multi_term.c +672 -0
  67. data/ext/isomorfeus_ferret_ext/frt_q_parser.c +3084 -0
  68. data/ext/isomorfeus_ferret_ext/frt_q_phrase.c +1182 -0
  69. data/ext/isomorfeus_ferret_ext/frt_q_prefix.c +98 -0
  70. data/ext/isomorfeus_ferret_ext/frt_q_range.c +665 -0
  71. data/ext/isomorfeus_ferret_ext/frt_q_span.c +2386 -0
  72. data/ext/isomorfeus_ferret_ext/frt_q_term.c +311 -0
  73. data/ext/isomorfeus_ferret_ext/frt_q_wildcard.c +166 -0
  74. data/ext/isomorfeus_ferret_ext/frt_ram_store.c +460 -0
  75. data/ext/isomorfeus_ferret_ext/frt_scanner.c +899 -0
  76. data/ext/isomorfeus_ferret_ext/frt_scanner.h +28 -0
  77. data/ext/isomorfeus_ferret_ext/frt_scanner_mb.c +6705 -0
  78. data/ext/isomorfeus_ferret_ext/frt_scanner_utf8.c +4419 -0
  79. data/ext/isomorfeus_ferret_ext/frt_search.c +1824 -0
  80. data/ext/isomorfeus_ferret_ext/frt_search.h +924 -0
  81. data/ext/isomorfeus_ferret_ext/frt_similarity.c +150 -0
  82. data/ext/isomorfeus_ferret_ext/frt_similarity.h +79 -0
  83. data/ext/isomorfeus_ferret_ext/frt_sort.c +796 -0
  84. data/ext/isomorfeus_ferret_ext/frt_stopwords.c +395 -0
  85. data/ext/isomorfeus_ferret_ext/frt_store.c +680 -0
  86. data/ext/isomorfeus_ferret_ext/frt_store.h +789 -0
  87. data/ext/isomorfeus_ferret_ext/frt_term_vectors.c +72 -0
  88. data/ext/isomorfeus_ferret_ext/frt_threading.h +23 -0
  89. data/ext/isomorfeus_ferret_ext/frt_win32.h +54 -0
  90. data/ext/isomorfeus_ferret_ext/isomorfeus_ferret.c +409 -0
  91. data/ext/isomorfeus_ferret_ext/isomorfeus_ferret.h +95 -0
  92. data/ext/isomorfeus_ferret_ext/libstemmer.c +93 -0
  93. data/ext/isomorfeus_ferret_ext/libstemmer.h +73 -0
  94. data/ext/isomorfeus_ferret_ext/q_parser.y +1366 -0
  95. data/ext/isomorfeus_ferret_ext/scanner.h +28 -0
  96. data/ext/isomorfeus_ferret_ext/scanner.in +43 -0
  97. data/ext/isomorfeus_ferret_ext/scanner.rl +84 -0
  98. data/ext/isomorfeus_ferret_ext/scanner_mb.rl +200 -0
  99. data/ext/isomorfeus_ferret_ext/scanner_utf8.rl +85 -0
  100. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_danish.c +324 -0
  101. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_danish.h +7 -0
  102. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_dutch.c +610 -0
  103. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_dutch.h +6 -0
  104. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_english.c +1104 -0
  105. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_english.h +6 -0
  106. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_finnish.c +749 -0
  107. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_finnish.h +7 -0
  108. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_french.c +1233 -0
  109. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_french.h +6 -0
  110. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_german.c +490 -0
  111. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_german.h +6 -0
  112. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_hungarian.c +1217 -0
  113. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_hungarian.h +7 -0
  114. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_italian.c +1052 -0
  115. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_italian.h +6 -0
  116. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_norwegian.c +283 -0
  117. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_norwegian.h +6 -0
  118. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_porter.c +735 -0
  119. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_porter.h +6 -0
  120. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_portuguese.c +1003 -0
  121. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_portuguese.h +7 -0
  122. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_spanish.c +1079 -0
  123. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_spanish.h +6 -0
  124. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_swedish.c +293 -0
  125. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_1_swedish.h +6 -0
  126. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_2_romanian.c +984 -0
  127. data/ext/isomorfeus_ferret_ext/stem_ISO_8859_2_romanian.h +6 -0
  128. data/ext/isomorfeus_ferret_ext/stem_KOI8_R_russian.c +686 -0
  129. data/ext/isomorfeus_ferret_ext/stem_KOI8_R_russian.h +6 -0
  130. data/ext/isomorfeus_ferret_ext/stem_UTF_8_danish.c +325 -0
  131. data/ext/isomorfeus_ferret_ext/stem_UTF_8_danish.h +6 -0
  132. data/ext/isomorfeus_ferret_ext/stem_UTF_8_dutch.c +620 -0
  133. data/ext/isomorfeus_ferret_ext/stem_UTF_8_dutch.h +6 -0
  134. data/ext/isomorfeus_ferret_ext/stem_UTF_8_english.c +1111 -0
  135. data/ext/isomorfeus_ferret_ext/stem_UTF_8_english.h +6 -0
  136. data/ext/isomorfeus_ferret_ext/stem_UTF_8_finnish.c +754 -0
  137. data/ext/isomorfeus_ferret_ext/stem_UTF_8_finnish.h +6 -0
  138. data/ext/isomorfeus_ferret_ext/stem_UTF_8_french.c +1242 -0
  139. data/ext/isomorfeus_ferret_ext/stem_UTF_8_french.h +6 -0
  140. data/ext/isomorfeus_ferret_ext/stem_UTF_8_german.c +495 -0
  141. data/ext/isomorfeus_ferret_ext/stem_UTF_8_german.h +6 -0
  142. data/ext/isomorfeus_ferret_ext/stem_UTF_8_hungarian.c +1220 -0
  143. data/ext/isomorfeus_ferret_ext/stem_UTF_8_hungarian.h +6 -0
  144. data/ext/isomorfeus_ferret_ext/stem_UTF_8_italian.c +1059 -0
  145. data/ext/isomorfeus_ferret_ext/stem_UTF_8_italian.h +6 -0
  146. data/ext/isomorfeus_ferret_ext/stem_UTF_8_norwegian.c +285 -0
  147. data/ext/isomorfeus_ferret_ext/stem_UTF_8_norwegian.h +6 -0
  148. data/ext/isomorfeus_ferret_ext/stem_UTF_8_porter.c +741 -0
  149. data/ext/isomorfeus_ferret_ext/stem_UTF_8_porter.h +6 -0
  150. data/ext/isomorfeus_ferret_ext/stem_UTF_8_portuguese.c +1009 -0
  151. data/ext/isomorfeus_ferret_ext/stem_UTF_8_portuguese.h +6 -0
  152. data/ext/isomorfeus_ferret_ext/stem_UTF_8_romanian.c +990 -0
  153. data/ext/isomorfeus_ferret_ext/stem_UTF_8_romanian.h +6 -0
  154. data/ext/isomorfeus_ferret_ext/stem_UTF_8_russian.c +680 -0
  155. data/ext/isomorfeus_ferret_ext/stem_UTF_8_russian.h +6 -0
  156. data/ext/isomorfeus_ferret_ext/stem_UTF_8_spanish.c +1083 -0
  157. data/ext/isomorfeus_ferret_ext/stem_UTF_8_spanish.h +6 -0
  158. data/ext/isomorfeus_ferret_ext/stem_UTF_8_swedish.c +294 -0
  159. data/ext/isomorfeus_ferret_ext/stem_UTF_8_swedish.h +6 -0
  160. data/ext/isomorfeus_ferret_ext/stem_UTF_8_turkish.c +2191 -0
  161. data/ext/isomorfeus_ferret_ext/stem_UTF_8_turkish.h +6 -0
  162. data/ext/isomorfeus_ferret_ext/stem_api.c +66 -0
  163. data/ext/isomorfeus_ferret_ext/stem_api.h +26 -0
  164. data/ext/isomorfeus_ferret_ext/stem_header.h +57 -0
  165. data/ext/isomorfeus_ferret_ext/stem_modules.h +190 -0
  166. data/ext/isomorfeus_ferret_ext/stem_modules.txt +50 -0
  167. data/ext/isomorfeus_ferret_ext/stem_utilities.c +478 -0
  168. data/ext/isomorfeus_ferret_ext/test.c +850 -0
  169. data/ext/isomorfeus_ferret_ext/test.h +416 -0
  170. data/ext/isomorfeus_ferret_ext/test_1710.c +63 -0
  171. data/ext/isomorfeus_ferret_ext/test_analysis.c +1221 -0
  172. data/ext/isomorfeus_ferret_ext/test_array.c +272 -0
  173. data/ext/isomorfeus_ferret_ext/test_bitvector.c +600 -0
  174. data/ext/isomorfeus_ferret_ext/test_compound_io.c +170 -0
  175. data/ext/isomorfeus_ferret_ext/test_document.c +156 -0
  176. data/ext/isomorfeus_ferret_ext/test_except.c +244 -0
  177. data/ext/isomorfeus_ferret_ext/test_fields.c +522 -0
  178. data/ext/isomorfeus_ferret_ext/test_file_deleter.c +185 -0
  179. data/ext/isomorfeus_ferret_ext/test_filter.c +331 -0
  180. data/ext/isomorfeus_ferret_ext/test_fs_store.c +25 -0
  181. data/ext/isomorfeus_ferret_ext/test_global.c +299 -0
  182. data/ext/isomorfeus_ferret_ext/test_hash.c +485 -0
  183. data/ext/isomorfeus_ferret_ext/test_hashset.c +288 -0
  184. data/ext/isomorfeus_ferret_ext/test_helper.c +47 -0
  185. data/ext/isomorfeus_ferret_ext/test_highlighter.c +548 -0
  186. data/ext/isomorfeus_ferret_ext/test_index.c +2323 -0
  187. data/ext/isomorfeus_ferret_ext/test_lang.c +74 -0
  188. data/ext/isomorfeus_ferret_ext/test_mempool.c +102 -0
  189. data/ext/isomorfeus_ferret_ext/test_multimapper.c +64 -0
  190. data/ext/isomorfeus_ferret_ext/test_priorityqueue.c +213 -0
  191. data/ext/isomorfeus_ferret_ext/test_q_const_score.c +84 -0
  192. data/ext/isomorfeus_ferret_ext/test_q_filtered.c +61 -0
  193. data/ext/isomorfeus_ferret_ext/test_q_fuzzy.c +241 -0
  194. data/ext/isomorfeus_ferret_ext/test_q_parser.c +464 -0
  195. data/ext/isomorfeus_ferret_ext/test_q_span.c +575 -0
  196. data/ext/isomorfeus_ferret_ext/test_ram_store.c +77 -0
  197. data/ext/isomorfeus_ferret_ext/test_search.c +1874 -0
  198. data/ext/isomorfeus_ferret_ext/test_segments.c +167 -0
  199. data/ext/isomorfeus_ferret_ext/test_similarity.c +25 -0
  200. data/ext/isomorfeus_ferret_ext/test_sort.c +333 -0
  201. data/ext/isomorfeus_ferret_ext/test_store.c +591 -0
  202. data/ext/isomorfeus_ferret_ext/test_store.h +3 -0
  203. data/ext/isomorfeus_ferret_ext/test_term.c +351 -0
  204. data/ext/isomorfeus_ferret_ext/test_term_vectors.c +373 -0
  205. data/ext/isomorfeus_ferret_ext/test_test.c +83 -0
  206. data/ext/isomorfeus_ferret_ext/test_threading.c +188 -0
  207. data/ext/isomorfeus_ferret_ext/testhelper.c +561 -0
  208. data/ext/isomorfeus_ferret_ext/testhelper.h +25 -0
  209. data/ext/isomorfeus_ferret_ext/tests_all.h +87 -0
  210. data/ext/isomorfeus_ferret_ext/uchar-ucs4.rl +1854 -0
  211. data/ext/isomorfeus_ferret_ext/uchar-utf8.rl +1999 -0
  212. data/ext/isomorfeus_ferret_ext/url.rl +27 -0
  213. data/ext/isomorfeus_ferret_ext/word_list.h +15156 -0
  214. data/lib/isomorfeus/ferret/document.rb +132 -0
  215. data/lib/isomorfeus/ferret/field_symbol.rb +85 -0
  216. data/lib/isomorfeus/ferret/index/field_infos.rb +48 -0
  217. data/lib/isomorfeus/ferret/index/index.rb +970 -0
  218. data/lib/isomorfeus/ferret/monitor.rb +323 -0
  219. data/lib/isomorfeus/ferret/stdlib_patches.rb +151 -0
  220. data/lib/isomorfeus/ferret/version.rb +5 -0
  221. data/lib/isomorfeus-ferret.rb +8 -0
  222. metadata +307 -0
@@ -0,0 +1,104 @@
1
+ #include <stdlib.h>
2
+ #include <stdio.h>
3
+ #include <string.h>
4
+ #include <errno.h>
5
+ #include <unistd.h>
6
+ #include "frt_lang.h"
7
+ #include "frt_except.h"
8
+ #include "frt_global.h"
9
+
10
+ /* emalloc: malloc and report if error */
11
+ void *frt_emalloc(size_t size)
12
+ {
13
+ void *p = malloc(size);
14
+
15
+ if (p == NULL) {
16
+ FRT_RAISE(FRT_MEM_ERROR, "failed to allocate %d bytes", (int)size);
17
+ }
18
+
19
+ return p;
20
+ }
21
+
22
+ /* frt_ecalloc: malloc, zeroset and report if error */
23
+ void *frt_ecalloc(size_t size)
24
+ {
25
+ void *p = calloc(1, size);
26
+
27
+ if (p == NULL) {
28
+ FRT_RAISE(FRT_MEM_ERROR, "failed to allocate %d bytes", (int)size);
29
+ }
30
+
31
+ return p;
32
+ }
33
+
34
+ /* frt_erealloc: realloc and report if error */
35
+ void *frt_erealloc(void *ptr, size_t size)
36
+ {
37
+ void *p = realloc(ptr, size);
38
+
39
+ if (p == NULL) {
40
+ FRT_RAISE(FRT_MEM_ERROR, "failed to reallocate %d bytes", (int)size);
41
+ }
42
+
43
+ return p;
44
+ }
45
+
46
+ /* void micro_sleep(const int micro_seconds)
47
+ {
48
+ #if (defined POSH_OS_WIN32 || defined POSH_OS_WIN64) && !defined __MINGW32__
49
+ Sleep(micro_seconds / 1000);
50
+ #else
51
+ usleep(micro_seconds);
52
+ #endif
53
+ } */
54
+
55
+ /* xexit: print error message and exit */
56
+ # ifdef FRT_HAS_VARARGS
57
+ void frt_vexit(const char *file, int line_num, const char *func,
58
+ const char *err_type, const char *fmt, va_list args)
59
+ # else
60
+ void FRT_VEXIT(const char *err_type, const char *fmt, va_list args)
61
+ # endif
62
+ {
63
+ fflush(stdout);
64
+ fprintf(EXCEPTION_STREAM, "\n%s: ", frt_progname());
65
+
66
+ # ifdef FRT_HAS_VARARGS
67
+ fprintf(EXCEPTION_STREAM, "%s occurred at <%s>:%d in %s\n",
68
+ err_type, file, line_num, func);
69
+ # else
70
+ fprintf(EXCEPTION_STREAM, "%s occurred:\n", err_type);
71
+ # endif
72
+ vfprintf(EXCEPTION_STREAM, fmt, args);
73
+
74
+ if (fmt[0] != '\0' && fmt[strlen(fmt) - 1] == ':') {
75
+ fprintf(EXCEPTION_STREAM, " %s", strerror(errno));
76
+ }
77
+
78
+ fprintf(EXCEPTION_STREAM, "\n");
79
+ frt_print_stacktrace();
80
+ if (frt_x_abort_on_exception) {
81
+ exit(2); /* conventional value for failed execution */
82
+ }
83
+ else {
84
+ frt_x_has_aborted = true;
85
+ }
86
+ }
87
+
88
+
89
+ # ifdef FRT_HAS_VARARGS
90
+ void frt_xexit(const char *file, int line_num, const char *func,
91
+ const char *err_type, const char *fmt, ...)
92
+ # else
93
+ void FRT_XEXIT(const char *err_type, const char *fmt, ...)
94
+ # endif
95
+ {
96
+ va_list args;
97
+ va_start(args, fmt);
98
+ # ifdef FRT_HAS_VARARGS
99
+ frt_vexit(file, line_num, func, err_type, fmt, args);
100
+ # else
101
+ FRT_VEXIT(err_type, fmt, args);
102
+ # endif
103
+ va_end(args);
104
+ }
@@ -0,0 +1,44 @@
1
+ #ifndef FRT_LANG_H
2
+ #define FRT_LANG_H
3
+
4
+ #include <stdarg.h>
5
+ #include <stddef.h>
6
+
7
+ extern void *frt_emalloc(size_t n);
8
+ extern void *frt_ecalloc(size_t n);
9
+ extern void *frt_erealloc(void *ptr, size_t n);
10
+
11
+ #define FRT_ALLOC(type) (type *)frt_emalloc(sizeof(type))
12
+ #define FRT_ALLOC_N(type,n) (type *)frt_emalloc(sizeof(type)*(n))
13
+ #define FRT_REALLOC_N(ptr, type, n)\
14
+ (ptr)=(type *)frt_erealloc((ptr),sizeof(type)*(n))
15
+
16
+ #ifdef FRT_HAS_ISO_VARARGS
17
+ /* C99-compliant compiler */
18
+
19
+ # define FRT_XEXIT(...) frt_xexit(__FILE__, __LINE__, __func__, __VA_ARGS__)
20
+ extern void frt_xexit(const char *file, int line_num, const char *func,
21
+ const char *err_type, const char *fmt, ...);
22
+
23
+ # define FRT_VEXIT(err_type, fmt, args) \
24
+ frt_vexit(__FILE__, __LINE__, __func__, err_type, fmt, args)
25
+ extern void frt_vexit(const char *file, int line_num, const char *func,
26
+ const char *err_type, const char *fmt, va_list args);
27
+
28
+ #elif defined(FRT_HAS_GNUC_VARARGS)
29
+ /* gcc has an extension */
30
+ # define FRT_XEXIT(args...) frt_xexit(__FILE__, __LINE__, __func__, ##args)
31
+ extern void frt_xexit(const char *file, int line_num, const char *func,
32
+ const char *err_type, const char *fmt, ...);
33
+
34
+ # define FRT_VEXIT(err_type, fmt, args) \
35
+ frt_vexit(__FILE__, __LINE__, __func__, err_type, fmt, args)
36
+ extern void frt_vexit(const char *file, int line_num, const char *func,
37
+ const char *err_type, const char *fmt, va_list args);
38
+ #else
39
+ /* Can't do VARARGS */
40
+ extern void FRT_XEXIT(const char *err_type, const char *fmt, ...);
41
+ extern void FRT_VEXIT(const char *err_type, const char *fmt, va_list args);
42
+ #endif
43
+
44
+ #endif
@@ -0,0 +1,87 @@
1
+ #include "frt_global.h"
2
+ #include "frt_mempool.h"
3
+ #include <string.h>
4
+
5
+ FrtMemoryPool *frt_mp_new_capa(int chuck_size, int init_buf_capa)
6
+ {
7
+ FrtMemoryPool *mp = FRT_ALLOC(FrtMemoryPool);
8
+ mp->chunk_size = chuck_size;
9
+ mp->buf_capa = init_buf_capa;
10
+ mp->buffers = FRT_ALLOC_N(char *, init_buf_capa);
11
+
12
+ mp->buffers[0] = mp->curr_buffer = (char *)frt_emalloc(mp->chunk_size);
13
+ mp->buf_alloc = 1;
14
+ mp->buf_pointer = 0;
15
+ mp->pointer = 0;
16
+ return mp;
17
+ }
18
+
19
+ FrtMemoryPool *frt_mp_new()
20
+ {
21
+ return frt_mp_new_capa(FRT_MP_BUF_SIZE, FRT_MP_INIT_CAPA);
22
+ }
23
+
24
+ void *frt_mp_alloc(FrtMemoryPool *mp, int size)
25
+ {
26
+ char *p;
27
+ p = mp->curr_buffer + mp->pointer;
28
+ #if defined POSH_OS_SOLARIS || defined POSH_OS_SUNOS
29
+ size = (((size - 1) >> 3) + 1) << 3;
30
+ #endif
31
+ mp->pointer += size;
32
+
33
+ if (mp->pointer > mp->chunk_size) {
34
+ mp->buf_pointer++;
35
+ if (mp->buf_pointer >= mp->buf_alloc) {
36
+ mp->buf_alloc++;
37
+ if (mp->buf_alloc >= mp->buf_capa) {
38
+ mp->buf_capa <<= 1;
39
+ FRT_REALLOC_N(mp->buffers, char *, mp->buf_capa);
40
+ }
41
+ mp->buffers[mp->buf_pointer] = (char *)frt_emalloc(mp->chunk_size);
42
+ }
43
+ p = mp->curr_buffer = mp->buffers[mp->buf_pointer];
44
+ mp->pointer = size;
45
+ }
46
+ return p;
47
+ }
48
+
49
+ char *frt_mp_strdup(FrtMemoryPool *mp, const char *str)
50
+ {
51
+ int len = strlen(str) + 1;
52
+ return (char *)memcpy(frt_mp_alloc(mp, len), str, len);
53
+ }
54
+
55
+ char *frt_mp_strndup(FrtMemoryPool *mp, const char *str, int len)
56
+ {
57
+ char *s = (char *)memcpy(frt_mp_alloc(mp, len + 1), str, len);
58
+ s[len] = '\0';
59
+ return s;
60
+ }
61
+
62
+ void *frt_mp_memdup(FrtMemoryPool *mp, const void *p, int len)
63
+ {
64
+ return memcpy(frt_mp_alloc(mp, len), p, len);
65
+ }
66
+
67
+ int frt_mp_used(FrtMemoryPool *mp)
68
+ {
69
+ return mp->buf_pointer * mp->chunk_size + mp->pointer;
70
+ }
71
+
72
+ void frt_mp_reset(FrtMemoryPool *mp)
73
+ {
74
+ mp->buf_pointer = 0;
75
+ mp->pointer = 0;
76
+ mp->curr_buffer = mp->buffers[0];
77
+ }
78
+
79
+ void frt_mp_destroy(FrtMemoryPool *mp)
80
+ {
81
+ int i;
82
+ for (i = 0; i < mp->buf_alloc; i++) {
83
+ free(mp->buffers[i]);
84
+ }
85
+ free(mp->buffers);
86
+ free(mp);
87
+ }
@@ -0,0 +1,33 @@
1
+ #ifndef FRT_MEM_POOL_H
2
+ #define FRT_MEM_POOL_H
3
+
4
+ #define FRT_MP_BUF_SIZE 65536
5
+ #define FRT_MP_INIT_CAPA 4
6
+
7
+ typedef struct FrtMemoryPool {
8
+ int buf_alloc;
9
+ int buf_capa;
10
+ int buf_pointer;
11
+ int pointer;
12
+ int chunk_size;
13
+ char *curr_buffer;
14
+ char **buffers;
15
+ } FrtMemoryPool;
16
+
17
+ extern FrtMemoryPool *frt_mp_new();
18
+ extern FrtMemoryPool *frt_mp_new_capa(int chunk_size, int init_capa);
19
+ extern void *frt_mp_alloc(FrtMemoryPool *mp, int size);
20
+ extern void frt_mp_reset(FrtMemoryPool *mp);
21
+ extern void frt_mp_destroy(FrtMemoryPool *mp);
22
+ extern char *frt_mp_strdup(FrtMemoryPool *mp, const char *str);
23
+ extern char *frt_mp_strndup(FrtMemoryPool *mp, const char *str, int len);
24
+ extern void *frt_mp_memdup(FrtMemoryPool *mp, const void *p, int len);
25
+ extern int frt_mp_used(FrtMemoryPool *mp);
26
+
27
+ #define FRT_MP_ALLOC_N(mp,type,n) (type *)frt_mp_alloc(mp, sizeof(type)*(n))
28
+ #define FRT_MP_ALLOC(mp,type) (type *)frt_mp_alloc(mp, sizeof(type))
29
+
30
+ #define FRT_MP_ALLOC_AND_ZERO_N(mp,type,n)\
31
+ (type*)FRT_ZEROSET_N(frt_mp_alloc(mp, sizeof(type)*(n)), type, n)
32
+
33
+ #endif
@@ -0,0 +1,349 @@
1
+ #include "frt_multimapper.h"
2
+ #include "frt_array.h"
3
+ #include "frt_bitvector.h"
4
+ #include <string.h>
5
+
6
+ #define St(state) ((FrtState *)(state))
7
+ #define UCtoI(val) ((int)(unsigned char)(val))
8
+
9
+ static void state_destroy(FrtState *state)
10
+ {
11
+ state->destroy_i(state);
12
+ }
13
+
14
+ typedef struct LetterState
15
+ {
16
+ FrtState super;
17
+ int c;
18
+ int val;
19
+ char *mapping;
20
+ } LetterState;
21
+ #define LSt(state) ((LetterState *)(state))
22
+
23
+
24
+ static int lstate_next(LetterState *self, int c, int *states)
25
+ {
26
+ if (c == self->c) {
27
+ states[0] = self->val;
28
+ return 1;
29
+ }
30
+ else {
31
+ return 0;
32
+ }
33
+ }
34
+
35
+ static int lstate_is_match(LetterState *self, char **mapping)
36
+ {
37
+ if (self->val < 0) {
38
+ *mapping = self->mapping;
39
+ return self->val;
40
+ }
41
+ else {
42
+ return 0;
43
+ }
44
+ }
45
+
46
+ static LetterState *lstate_new(int c, int val)
47
+ {
48
+ LetterState *self = FRT_ALLOC(LetterState);
49
+ self->c = c;
50
+ self->val = val;
51
+ self->mapping = NULL;
52
+ St(self)->next = (int (*)(FrtState *, int, int *))&lstate_next;
53
+ St(self)->destroy_i = (void (*)(FrtState *))&free;
54
+ St(self)->is_match = (int (*)(FrtState *, char **))&lstate_is_match;
55
+ return self;
56
+ }
57
+
58
+ typedef struct NonDeterministicState
59
+ {
60
+ FrtState super;
61
+ int *states[256];
62
+ int size[256];
63
+ int capa[256];
64
+ } NonDeterministicState;
65
+
66
+ static int ndstate_next(NonDeterministicState *self, int c, int *states)
67
+ {
68
+ int size = self->size[c];
69
+ memcpy(states, self->states[c], size * sizeof(int));
70
+ return size;
71
+ }
72
+
73
+ static void ndstate_add(NonDeterministicState *self, int c, int state)
74
+ {
75
+ if (self->capa[c] <= self->size[c]) {
76
+ if (self->capa[c] == 0) {
77
+ self->capa[c] = 4;
78
+ }
79
+ else {
80
+ self->capa[c] <<= 1;
81
+ }
82
+ FRT_REALLOC_N(self->states[c], int, self->capa[c]);
83
+ }
84
+ self->states[c][self->size[c]++] = state;
85
+ }
86
+
87
+ static void ndstate_destroy_i(NonDeterministicState *self)
88
+ {
89
+ int i;
90
+ for (i = 0; i < 256; i++) {
91
+ free(self->states[i]);
92
+ }
93
+ free(self);
94
+ }
95
+
96
+ static int ndstate_is_match(FrtState *self, char **mapping)
97
+ {
98
+ (void)self; (void)mapping;
99
+ return 0;
100
+ }
101
+
102
+ static NonDeterministicState *ndstate_new()
103
+ {
104
+ NonDeterministicState *self = FRT_ALLOC_AND_ZERO(NonDeterministicState);
105
+ St(self)->next = (int (*)(FrtState *, int, int *))&ndstate_next;
106
+ St(self)->destroy_i = (void (*)(FrtState *))&ndstate_destroy_i;
107
+ St(self)->is_match = &ndstate_is_match;
108
+ return self;
109
+ }
110
+
111
+ FrtMultiMapper *frt_mulmap_new()
112
+ {
113
+ FrtMultiMapper *self = FRT_ALLOC_AND_ZERO(FrtMultiMapper);
114
+ self->capa = 128;
115
+ self->mappings = FRT_ALLOC_N(FrtMapping *, 128);
116
+ self->d_capa = 128;
117
+ self->dstates = FRT_ALLOC_N(FrtDeterministicState *, 128);
118
+ self->dstates_map = NULL;
119
+ self->nstates = NULL;
120
+ self->ref_cnt = 1;
121
+ return self;
122
+ }
123
+
124
+ static void mulmap_free_dstates(FrtMultiMapper *self)
125
+ {
126
+ if (self->d_size > 0) {
127
+ int i;
128
+ for (i = self->d_size - 1; i >= 0; i--) {
129
+ free(self->dstates[i]);
130
+ }
131
+ self->d_size = 0;
132
+ }
133
+ }
134
+
135
+ void frt_mulmap_add_mapping(FrtMultiMapper *self, const char *pattern, const char *rep)
136
+ {
137
+ if (pattern == NULL || pattern[0] == '\0') {
138
+ FRT_RAISE(FRT_ARG_ERROR, "Tried to add empty pattern to multi_mapper");
139
+ } else {
140
+ FrtMapping *mapping = FRT_ALLOC(FrtMapping);
141
+ if (self->size >= self->capa) {
142
+ self->capa <<= 1;
143
+ FRT_REALLOC_N(self->mappings, FrtMapping *, self->capa);
144
+ }
145
+ mapping->pattern = frt_estrdup(pattern);
146
+ mapping->replacement = frt_estrdup(rep);
147
+ self->mappings[self->size++] = mapping;
148
+ mulmap_free_dstates(self);
149
+ }
150
+ }
151
+
152
+
153
+ static void mulmap_bv_set_states(FrtBitVector *bv, int *states, int cnt)
154
+ {
155
+ int i;
156
+ for (i = cnt - 1; i >= 0; i--) {
157
+ frt_bv_set(bv, states[i]);
158
+ }
159
+ }
160
+
161
+ static FrtDeterministicState *mulmap_process_state(FrtMultiMapper *self, FrtBitVector *bv)
162
+ {
163
+ FrtDeterministicState *current_state
164
+ = (FrtDeterministicState *)frt_h_get(self->dstates_map, bv);
165
+ if (current_state == NULL) {
166
+ int bit, i;
167
+ int match_len = 0, max_match_len = 0;
168
+ FrtState *start = self->nstates[0];
169
+ FrtDeterministicState *start_ds;
170
+ current_state = FRT_ALLOC_AND_ZERO(FrtDeterministicState);
171
+ frt_h_set(self->dstates_map, bv, current_state);
172
+ if (self->d_size >= self->d_capa) {
173
+ self->d_capa <<= 1;
174
+ FRT_REALLOC_N(self->dstates, FrtDeterministicState *, self->d_capa);
175
+ }
176
+ self->dstates[self->d_size++] = current_state;
177
+ start_ds = self->dstates[0];
178
+ for (i = 0; i < 256; i++) {
179
+ current_state->next[i] = start_ds;
180
+ }
181
+ while ((bit = frt_bv_scan_next(bv)) >= 0) {
182
+ char *mapping;
183
+ FrtState *st = self->nstates[bit];
184
+ if ((match_len = -st->is_match(st, &mapping)) > max_match_len) {
185
+ current_state->longest_match = max_match_len = match_len;
186
+ current_state->mapping = mapping;
187
+ current_state->mapping_len = strlen(mapping);
188
+ }
189
+ }
190
+ for (i = self->a_size - 1; i >= 0; i--) {
191
+ unsigned char c = self->alphabet[i];
192
+ FrtBitVector *nxt_bv = frt_bv_new_capa(self->nsize);
193
+ mulmap_bv_set_states(nxt_bv, self->next_states,
194
+ start->next(start, (int)c, self->next_states));
195
+ frt_bv_scan_reset(bv);
196
+ while ((bit = frt_bv_scan_next(bv)) >= 0) {
197
+ FrtState *state = self->nstates[bit];
198
+ mulmap_bv_set_states(nxt_bv, self->next_states,
199
+ state->next(state, (int)c, self->next_states));
200
+ }
201
+ current_state->next[(int)c] = mulmap_process_state(self, nxt_bv);
202
+ }
203
+ }
204
+ else {
205
+ frt_bv_destroy(bv);
206
+ }
207
+ return current_state;
208
+ }
209
+
210
+ void frt_mulmap_compile(FrtMultiMapper *self)
211
+ {
212
+ NonDeterministicState *start = ndstate_new();
213
+ int i, j;
214
+ int size = 1;
215
+ int capa = 128;
216
+ LetterState *ls;
217
+ FrtState **nstates = FRT_ALLOC_N(FrtState *, capa);
218
+ FrtMapping **mappings = self->mappings;
219
+ unsigned char alphabet[256];
220
+ nstates[0] = (FrtState *)start;
221
+ memset(alphabet, 0, 256);
222
+
223
+ for (i = self->size - 1; i >= 0; i--) {
224
+ const char *pattern = mappings[i]->pattern;
225
+ const int plen = (int)strlen(pattern);
226
+ ndstate_add(start, UCtoI(pattern[0]), size);
227
+ if (size + plen + 1 >= capa) {
228
+ capa <<= 2;
229
+ FRT_REALLOC_N(nstates, FrtState *, capa);
230
+ }
231
+ for (j = 0; j < plen; j++) {
232
+ alphabet[UCtoI(pattern[j])] = 1;
233
+ size += 1;
234
+ nstates[size-1] = (FrtState *)lstate_new(UCtoI(pattern[j+1]), size);
235
+ }
236
+ ls = LSt(nstates[size-1]);
237
+ ls->mapping = mappings[i]->replacement;
238
+ ls->val = -plen;
239
+ ls->c = -1;
240
+ }
241
+ for (i = j = 0; i < 256; i++) {
242
+ if (alphabet[i]) self->alphabet[j++] = i;
243
+ }
244
+ self->a_size = j;
245
+ mulmap_free_dstates(self);
246
+ self->nstates = nstates;
247
+ self->nsize = size;
248
+ self->next_states = FRT_ALLOC_N(int, size);
249
+ self->dstates_map = frt_h_new((frt_hash_ft)&frt_bv_hash, (frt_eq_ft)&frt_bv_eq,
250
+ (frt_free_ft)&frt_bv_destroy, (frt_free_ft)NULL);
251
+ mulmap_process_state(self, frt_bv_new_capa(0));
252
+ frt_h_destroy(self->dstates_map);
253
+ for (i = size - 1; i >= 0; i--) {
254
+ state_destroy(nstates[i]);
255
+ }
256
+ free(self->next_states);
257
+ free(nstates);
258
+ }
259
+
260
+ int frt_mulmap_map_len(FrtMultiMapper *self, char *to, char *from, int capa)
261
+ {
262
+ FrtDeterministicState *start = self->dstates[0];
263
+ FrtDeterministicState *state = start;
264
+ char *s = from, *d = to, *end = to + capa - 1;
265
+ if (self->d_size == 0) {
266
+ frt_mulmap_compile(self);
267
+ }
268
+ while (*s && d < end) {
269
+ state = state->next[UCtoI(*s)];
270
+ if (state->mapping) {
271
+ int len = state->mapping_len;
272
+ d -= (state->longest_match - 1);
273
+ if ((d + len) > end) {
274
+ len = end - d;
275
+ }
276
+ memcpy(d, state->mapping, len);
277
+ d += len;
278
+ state = start;
279
+ }
280
+ else {
281
+ *(d++) = *s;
282
+ }
283
+ s++;
284
+ }
285
+ *d = '\0';
286
+ return d - to;
287
+ }
288
+
289
+ char *frt_mulmap_map(FrtMultiMapper *self, char *to, char *from, int capa)
290
+ {
291
+ frt_mulmap_map_len(self, to, from, capa);
292
+ return to;
293
+ }
294
+
295
+ /* Maps a string to a dynamically allocated string */
296
+ char *frt_mulmap_dynamic_map(FrtMultiMapper *self, char *from)
297
+ {
298
+ FrtDeterministicState *start = self->dstates[0];
299
+ FrtDeterministicState *state = start;
300
+ int capa = strlen(from);
301
+ char *to = (char *)frt_ecalloc(capa);
302
+ char *s = from, *d = to, *end = to + capa - 1;
303
+ if (self->d_size == 0) {
304
+ frt_mulmap_compile(self);
305
+ }
306
+ do {
307
+ while (*s && d < end) {
308
+ state = state->next[UCtoI(*s)];
309
+ if (state->mapping) {
310
+ int len = state->mapping_len;
311
+ d -= (state->longest_match - 1);
312
+ if ((d + len) > end) {
313
+ len = end - d;
314
+ }
315
+ memcpy(d, state->mapping, len);
316
+ d += len;
317
+ state = start;
318
+ }
319
+ else {
320
+ *(d++) = *s;
321
+ }
322
+ s++;
323
+ }
324
+ if (*s) {
325
+ capa += 1024;
326
+ frt_erealloc(to, capa);
327
+ end = to + capa - 1;
328
+ }
329
+ } while(*s);
330
+ *d = '\0';
331
+ return to;
332
+ }
333
+
334
+ void frt_mulmap_destroy(FrtMultiMapper *self)
335
+ {
336
+ if (--(self->ref_cnt) <= 0) {
337
+ int i;
338
+ mulmap_free_dstates(self);
339
+ for (i = self->size - 1; i >= 0; i--) {
340
+ FrtMapping *mapping = self->mappings[i];
341
+ free(mapping->pattern);
342
+ free(mapping->replacement);
343
+ free(mapping);
344
+ }
345
+ free(self->mappings);
346
+ free(self->dstates);
347
+ free(self);
348
+ }
349
+ }
@@ -0,0 +1,52 @@
1
+ #ifndef FRT_MAPPER_H
2
+ #define FRT_MAPPER_H
3
+
4
+ #include "frt_hash.h"
5
+
6
+ typedef struct FrtState
7
+ {
8
+ int (*next)(struct FrtState *self, int c, int *states);
9
+ void (*destroy_i)(struct FrtState *self);
10
+ int (*is_match)(struct FrtState *self, char **mapping);
11
+ } FrtState;
12
+
13
+ typedef struct FrtDeterministicState
14
+ {
15
+ struct FrtDeterministicState *next[256];
16
+ int longest_match;
17
+ char *mapping;
18
+ int mapping_len;
19
+ } FrtDeterministicState;
20
+
21
+ typedef struct FrtMapping
22
+ {
23
+ char *pattern;
24
+ char *replacement;
25
+ } FrtMapping;
26
+
27
+ typedef struct FrtMultiMapper
28
+ {
29
+ FrtMapping **mappings;
30
+ int size;
31
+ int capa;
32
+ FrtDeterministicState **dstates;
33
+ int d_size;
34
+ int d_capa;
35
+ unsigned char alphabet[256];
36
+ int a_size;
37
+ FrtHash *dstates_map;
38
+ FrtState **nstates;
39
+ int nsize;
40
+ int *next_states;
41
+ int ref_cnt;
42
+ } FrtMultiMapper;
43
+
44
+ extern FrtMultiMapper *frt_mulmap_new();
45
+ extern void frt_mulmap_add_mapping(FrtMultiMapper *self, const char *p, const char *r);
46
+ extern void frt_mulmap_compile(FrtMultiMapper *self);
47
+ extern char *frt_mulmap_map(FrtMultiMapper *self, char *to, char *from, int capa);
48
+ extern char *frt_mulmap_dynamic_map(FrtMultiMapper *self, char *from);
49
+ extern int frt_mulmap_map_len(FrtMultiMapper *self, char *to, char *from, int capa);
50
+ extern void frt_mulmap_destroy(FrtMultiMapper *self);
51
+
52
+ #endif