@delali/narsil-react-native 0.3.1-next.154 → 0.3.1-next.156

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (253) hide show
  1. package/NOTICE +144 -0
  2. package/NarsilReactNative.podspec +2 -2
  3. package/android/CMakeLists.txt +1 -1
  4. package/cpp/HybridNarsilCoreAnalysis.cpp +277 -0
  5. package/cpp/HybridNarsilCoreImpl.hpp +43 -0
  6. package/cpp/HybridNarsilCoreKeyword.cpp +253 -0
  7. package/cpp/HybridNarsilCoreMath.cpp +47 -0
  8. package/cpp/core/include/narsil_analysis.h +86 -0
  9. package/cpp/core/include/narsil_core.h +1 -1
  10. package/cpp/core/include/narsil_keyword.h +110 -0
  11. package/cpp/core/include/narsil_math.h +36 -0
  12. package/cpp/core/src/analysis/analyse.c +281 -0
  13. package/cpp/core/src/analysis/analyser.h +49 -0
  14. package/cpp/core/src/analysis/languages.c +79 -0
  15. package/cpp/core/src/analysis/languages.h +42 -0
  16. package/cpp/core/src/analysis/normalise.c +242 -0
  17. package/cpp/core/src/analysis/normalise.h +28 -0
  18. package/cpp/core/src/analysis/split.c +183 -0
  19. package/cpp/core/src/analysis/split.h +27 -0
  20. package/cpp/core/src/analysis/stop_words.c +82 -0
  21. package/cpp/core/src/analysis/stop_words.h +19 -0
  22. package/cpp/core/src/analysis/term_cache.c +88 -0
  23. package/cpp/core/src/analysis/term_cache.h +36 -0
  24. package/cpp/core/src/analysis/text.c +161 -0
  25. package/cpp/core/src/analysis/text.h +45 -0
  26. package/cpp/core/src/analysis/transform.c +103 -0
  27. package/cpp/core/src/analysis/unicode.c +252 -0
  28. package/cpp/core/src/analysis/unicode.h +54 -0
  29. package/cpp/core/src/internal.h +4 -0
  30. package/cpp/core/src/keyword/pass.h +117 -0
  31. package/cpp/core/src/keyword/prune.c +324 -0
  32. package/cpp/core/src/keyword/request.c +179 -0
  33. package/cpp/core/src/keyword/rows.c +83 -0
  34. package/cpp/core/src/keyword/rows.h +18 -0
  35. package/cpp/core/src/keyword/score.c +257 -0
  36. package/cpp/core/src/keyword/score.h +17 -0
  37. package/cpp/core/src/keyword/scratch.c +226 -0
  38. package/cpp/core/src/keyword/scratch.h +55 -0
  39. package/cpp/core/src/keyword/select.c +155 -0
  40. package/cpp/core/src/keyword/select.h +17 -0
  41. package/cpp/core/src/languages/bulgarian_stemmer.c +114 -0
  42. package/cpp/core/src/languages/data/albanian.c +54 -0
  43. package/cpp/core/src/languages/data/amharic.c +93 -0
  44. package/cpp/core/src/languages/data/arabic.c +146 -0
  45. package/cpp/core/src/languages/data/armenian.c +46 -0
  46. package/cpp/core/src/languages/data/azerbaijani.c +68 -0
  47. package/cpp/core/src/languages/data/bambara.c +48 -0
  48. package/cpp/core/src/languages/data/basque.c +62 -0
  49. package/cpp/core/src/languages/data/belarusian.c +56 -0
  50. package/cpp/core/src/languages/data/bengali.c +63 -0
  51. package/cpp/core/src/languages/data/bosnian.c +73 -0
  52. package/cpp/core/src/languages/data/breton.c +52 -0
  53. package/cpp/core/src/languages/data/bulgarian.c +144 -0
  54. package/cpp/core/src/languages/data/burmese.c +48 -0
  55. package/cpp/core/src/languages/data/catalan.c +84 -0
  56. package/cpp/core/src/languages/data/chinese.c +70 -0
  57. package/cpp/core/src/languages/data/croatian.c +72 -0
  58. package/cpp/core/src/languages/data/czech.c +72 -0
  59. package/cpp/core/src/languages/data/dagbani.c +88 -0
  60. package/cpp/core/src/languages/data/danish.c +56 -0
  61. package/cpp/core/src/languages/data/declarations.h +114 -0
  62. package/cpp/core/src/languages/data/dutch.c +58 -0
  63. package/cpp/core/src/languages/data/english.c +172 -0
  64. package/cpp/core/src/languages/data/esperanto.c +71 -0
  65. package/cpp/core/src/languages/data/estonian.c +417 -0
  66. package/cpp/core/src/languages/data/ewe.c +89 -0
  67. package/cpp/core/src/languages/data/faroese.c +54 -0
  68. package/cpp/core/src/languages/data/fijian.c +44 -0
  69. package/cpp/core/src/languages/data/finnish.c +100 -0
  70. package/cpp/core/src/languages/data/french.c +74 -0
  71. package/cpp/core/src/languages/data/ga.c +83 -0
  72. package/cpp/core/src/languages/data/galician.c +70 -0
  73. package/cpp/core/src/languages/data/georgian.c +57 -0
  74. package/cpp/core/src/languages/data/german.c +96 -0
  75. package/cpp/core/src/languages/data/greek.c +202 -0
  76. package/cpp/core/src/languages/data/guarani.c +110 -0
  77. package/cpp/core/src/languages/data/gujarati.c +90 -0
  78. package/cpp/core/src/languages/data/haitian-creole.c +54 -0
  79. package/cpp/core/src/languages/data/hausa.c +85 -0
  80. package/cpp/core/src/languages/data/hawaiian.c +50 -0
  81. package/cpp/core/src/languages/data/hebrew.c +85 -0
  82. package/cpp/core/src/languages/data/hindi.c +91 -0
  83. package/cpp/core/src/languages/data/hungarian.c +98 -0
  84. package/cpp/core/src/languages/data/icelandic.c +73 -0
  85. package/cpp/core/src/languages/data/igbo.c +90 -0
  86. package/cpp/core/src/languages/data/indonesian.c +296 -0
  87. package/cpp/core/src/languages/data/irish.c +60 -0
  88. package/cpp/core/src/languages/data/italian.c +107 -0
  89. package/cpp/core/src/languages/data/japanese.c +52 -0
  90. package/cpp/core/src/languages/data/kannada.c +60 -0
  91. package/cpp/core/src/languages/data/kazakh.c +57 -0
  92. package/cpp/core/src/languages/data/khmer.c +71 -0
  93. package/cpp/core/src/languages/data/kinyarwanda.c +54 -0
  94. package/cpp/core/src/languages/data/kirundi.c +53 -0
  95. package/cpp/core/src/languages/data/korean.c +74 -0
  96. package/cpp/core/src/languages/data/kurmanji.c +43 -0
  97. package/cpp/core/src/languages/data/kyrgyz.c +65 -0
  98. package/cpp/core/src/languages/data/lao.c +46 -0
  99. package/cpp/core/src/languages/data/latin.c +46 -0
  100. package/cpp/core/src/languages/data/latvian.c +75 -0
  101. package/cpp/core/src/languages/data/lingala.c +88 -0
  102. package/cpp/core/src/languages/data/lithuanian.c +65 -0
  103. package/cpp/core/src/languages/data/luxembourgish.c +80 -0
  104. package/cpp/core/src/languages/data/macedonian.c +286 -0
  105. package/cpp/core/src/languages/data/malagasy.c +51 -0
  106. package/cpp/core/src/languages/data/malay.c +184 -0
  107. package/cpp/core/src/languages/data/malayalam.c +64 -0
  108. package/cpp/core/src/languages/data/maltese.c +102 -0
  109. package/cpp/core/src/languages/data/maori.c +52 -0
  110. package/cpp/core/src/languages/data/marathi.c +62 -0
  111. package/cpp/core/src/languages/data/nepali.c +72 -0
  112. package/cpp/core/src/languages/data/norwegian.c +74 -0
  113. package/cpp/core/src/languages/data/oromo.c +51 -0
  114. package/cpp/core/src/languages/data/persian.c +150 -0
  115. package/cpp/core/src/languages/data/polish.c +111 -0
  116. package/cpp/core/src/languages/data/portuguese.c +91 -0
  117. package/cpp/core/src/languages/data/punjabi.c +80 -0
  118. package/cpp/core/src/languages/data/registry.c +116 -0
  119. package/cpp/core/src/languages/data/romanian.c +93 -0
  120. package/cpp/core/src/languages/data/russian.c +76 -0
  121. package/cpp/core/src/languages/data/samoan.c +47 -0
  122. package/cpp/core/src/languages/data/sanskrit.c +53 -0
  123. package/cpp/core/src/languages/data/scottish-gaelic.c +119 -0
  124. package/cpp/core/src/languages/data/serbian.c +84 -0
  125. package/cpp/core/src/languages/data/shona.c +55 -0
  126. package/cpp/core/src/languages/data/sinhala.c +78 -0
  127. package/cpp/core/src/languages/data/slovak.c +135 -0
  128. package/cpp/core/src/languages/data/slovenian.c +139 -0
  129. package/cpp/core/src/languages/data/sorani.c +55 -0
  130. package/cpp/core/src/languages/data/spanish.c +84 -0
  131. package/cpp/core/src/languages/data/swahili.c +59 -0
  132. package/cpp/core/src/languages/data/swedish.c +60 -0
  133. package/cpp/core/src/languages/data/tagalog.c +75 -0
  134. package/cpp/core/src/languages/data/tamil.c +69 -0
  135. package/cpp/core/src/languages/data/tatar.c +548 -0
  136. package/cpp/core/src/languages/data/telugu.c +54 -0
  137. package/cpp/core/src/languages/data/thai.c +46 -0
  138. package/cpp/core/src/languages/data/tibetan.c +55 -0
  139. package/cpp/core/src/languages/data/tigrinya.c +73 -0
  140. package/cpp/core/src/languages/data/tongan.c +47 -0
  141. package/cpp/core/src/languages/data/turkish.c +100 -0
  142. package/cpp/core/src/languages/data/twi.c +88 -0
  143. package/cpp/core/src/languages/data/ukrainian.c +99 -0
  144. package/cpp/core/src/languages/data/urdu.c +174 -0
  145. package/cpp/core/src/languages/data/vietnamese.c +88 -0
  146. package/cpp/core/src/languages/data/welsh.c +143 -0
  147. package/cpp/core/src/languages/data/wolof.c +51 -0
  148. package/cpp/core/src/languages/data/xhosa.c +56 -0
  149. package/cpp/core/src/languages/data/yoruba.c +93 -0
  150. package/cpp/core/src/languages/data/zulu.c +58 -0
  151. package/cpp/core/src/languages/marks.c +64 -0
  152. package/cpp/core/src/languages/rules.c +63 -0
  153. package/cpp/core/src/languages/rules.h +68 -0
  154. package/cpp/core/src/languages/sanskrit_stemmer.c +50 -0
  155. package/cpp/core/src/languages/scripts.c +143 -0
  156. package/cpp/core/src/languages/slovenian_stemmer.c +46 -0
  157. package/cpp/core/src/languages/snowball/api.c +52 -0
  158. package/cpp/core/src/languages/snowball/api.h +43 -0
  159. package/cpp/core/src/languages/snowball/arabic_stemmer.c +1506 -0
  160. package/cpp/core/src/languages/snowball/arabic_stemmer.h +24 -0
  161. package/cpp/core/src/languages/snowball/armenian_stemmer.c +523 -0
  162. package/cpp/core/src/languages/snowball/armenian_stemmer.h +24 -0
  163. package/cpp/core/src/languages/snowball/basque_stemmer.c +1089 -0
  164. package/cpp/core/src/languages/snowball/basque_stemmer.h +24 -0
  165. package/cpp/core/src/languages/snowball/catalan_stemmer.c +1380 -0
  166. package/cpp/core/src/languages/snowball/catalan_stemmer.h +24 -0
  167. package/cpp/core/src/languages/snowball/czech_stemmer.c +582 -0
  168. package/cpp/core/src/languages/snowball/czech_stemmer.h +24 -0
  169. package/cpp/core/src/languages/snowball/danish_stemmer.c +352 -0
  170. package/cpp/core/src/languages/snowball/danish_stemmer.h +24 -0
  171. package/cpp/core/src/languages/snowball/dutch_stemmer.c +1657 -0
  172. package/cpp/core/src/languages/snowball/dutch_stemmer.h +24 -0
  173. package/cpp/core/src/languages/snowball/english_stemmer.c +1019 -0
  174. package/cpp/core/src/languages/snowball/english_stemmer.h +24 -0
  175. package/cpp/core/src/languages/snowball/esperanto_stemmer.c +716 -0
  176. package/cpp/core/src/languages/snowball/esperanto_stemmer.h +24 -0
  177. package/cpp/core/src/languages/snowball/estonian_stemmer.c +1361 -0
  178. package/cpp/core/src/languages/snowball/estonian_stemmer.h +24 -0
  179. package/cpp/core/src/languages/snowball/finnish_stemmer.c +800 -0
  180. package/cpp/core/src/languages/snowball/finnish_stemmer.h +24 -0
  181. package/cpp/core/src/languages/snowball/french_stemmer.c +1217 -0
  182. package/cpp/core/src/languages/snowball/french_stemmer.h +24 -0
  183. package/cpp/core/src/languages/snowball/german_stemmer.c +567 -0
  184. package/cpp/core/src/languages/snowball/german_stemmer.h +24 -0
  185. package/cpp/core/src/languages/snowball/greek_stemmer.c +3450 -0
  186. package/cpp/core/src/languages/snowball/greek_stemmer.h +24 -0
  187. package/cpp/core/src/languages/snowball/hindi_stemmer.c +325 -0
  188. package/cpp/core/src/languages/snowball/hindi_stemmer.h +24 -0
  189. package/cpp/core/src/languages/snowball/hungarian_stemmer.c +742 -0
  190. package/cpp/core/src/languages/snowball/hungarian_stemmer.h +24 -0
  191. package/cpp/core/src/languages/snowball/indonesian_stemmer.c +443 -0
  192. package/cpp/core/src/languages/snowball/indonesian_stemmer.h +24 -0
  193. package/cpp/core/src/languages/snowball/irish_stemmer.c +428 -0
  194. package/cpp/core/src/languages/snowball/irish_stemmer.h +24 -0
  195. package/cpp/core/src/languages/snowball/italian_stemmer.c +983 -0
  196. package/cpp/core/src/languages/snowball/italian_stemmer.h +24 -0
  197. package/cpp/core/src/languages/snowball/lithuanian_stemmer.c +794 -0
  198. package/cpp/core/src/languages/snowball/lithuanian_stemmer.h +24 -0
  199. package/cpp/core/src/languages/snowball/nepali_stemmer.c +361 -0
  200. package/cpp/core/src/languages/snowball/nepali_stemmer.h +24 -0
  201. package/cpp/core/src/languages/snowball/norwegian_stemmer.c +332 -0
  202. package/cpp/core/src/languages/snowball/norwegian_stemmer.h +24 -0
  203. package/cpp/core/src/languages/snowball/persian_stemmer.c +538 -0
  204. package/cpp/core/src/languages/snowball/persian_stemmer.h +24 -0
  205. package/cpp/core/src/languages/snowball/polish_stemmer.c +501 -0
  206. package/cpp/core/src/languages/snowball/polish_stemmer.h +24 -0
  207. package/cpp/core/src/languages/snowball/portuguese_stemmer.c +866 -0
  208. package/cpp/core/src/languages/snowball/portuguese_stemmer.h +24 -0
  209. package/cpp/core/src/languages/snowball/romanian_stemmer.c +940 -0
  210. package/cpp/core/src/languages/snowball/romanian_stemmer.h +24 -0
  211. package/cpp/core/src/languages/snowball/russian_stemmer.c +598 -0
  212. package/cpp/core/src/languages/snowball/russian_stemmer.h +24 -0
  213. package/cpp/core/src/languages/snowball/serbian_stemmer.c +6765 -0
  214. package/cpp/core/src/languages/snowball/serbian_stemmer.h +24 -0
  215. package/cpp/core/src/languages/snowball/snowball_runtime.h +122 -0
  216. package/cpp/core/src/languages/snowball/spanish_stemmer.c +955 -0
  217. package/cpp/core/src/languages/snowball/spanish_stemmer.h +24 -0
  218. package/cpp/core/src/languages/snowball/swedish_stemmer.c +364 -0
  219. package/cpp/core/src/languages/snowball/swedish_stemmer.h +24 -0
  220. package/cpp/core/src/languages/snowball/tamil_stemmer.c +1305 -0
  221. package/cpp/core/src/languages/snowball/tamil_stemmer.h +24 -0
  222. package/cpp/core/src/languages/snowball/turkish_stemmer.c +1700 -0
  223. package/cpp/core/src/languages/snowball/turkish_stemmer.h +24 -0
  224. package/cpp/core/src/languages/snowball/utilities.c +716 -0
  225. package/cpp/core/src/languages/substitutions.c +131 -0
  226. package/cpp/core/src/languages/swahili_stemmer.c +85 -0
  227. package/cpp/core/src/languages/ukrainian_stemmer.c +168 -0
  228. package/cpp/core/src/locks.c +2 -15
  229. package/cpp/core/src/math/apply.c +44 -0
  230. package/cpp/core/src/math/fdlibm.h +16 -0
  231. package/cpp/core/src/math/inverse.c +242 -0
  232. package/cpp/core/src/math/log.c +90 -0
  233. package/cpp/core/src/math/reduce.c +325 -0
  234. package/cpp/core/src/math/trig.c +188 -0
  235. package/cpp/core/src/math/words.h +45 -0
  236. package/cpp/core/src/thread_yield.h +18 -0
  237. package/cpp/core/src/unicode/decompositions.c +310 -0
  238. package/cpp/core/src/unicode/lowercase.c +172 -0
  239. package/cpp/core/src/unicode/properties.c +214 -0
  240. package/cpp/core/src/workspace.c +5 -1
  241. package/lib/engine/analysis.d.ts +3 -0
  242. package/lib/engine/analysis.js +76 -0
  243. package/lib/engine/core.js +5 -0
  244. package/lib/engine/keyword.d.ts +3 -0
  245. package/lib/engine/keyword.js +25 -0
  246. package/lib/specs/core.nitro.d.ts +1 -0
  247. package/nitrogen/generated/shared/c++/HybridNarsilCoreSpec.cpp +1 -0
  248. package/nitrogen/generated/shared/c++/HybridNarsilCoreSpec.hpp +1 -0
  249. package/package.json +4 -3
  250. package/src/engine/analysis.ts +87 -0
  251. package/src/engine/core.ts +6 -0
  252. package/src/engine/keyword.ts +26 -0
  253. package/src/specs/core.nitro.ts +1 -0
@@ -0,0 +1,716 @@
1
+ /*
2
+ * Copied from runtime/utilities.c by the Snowball compiler 3.1.1, with each symbol widened to a
3
+ * UTF-16 code unit so that the stemmer reads the same units as the TypeScript stemmer.
4
+ *
5
+ * Snowball is BSD-3-Clause, copyright (c) 2001, Dr Martin Porter and contributors. The NOTICE file at the root
6
+ * of the repository carries the licence. Source from https://github.com/snowballstem/snowball.git at
7
+ * 6772636350acfd63797e8cd24ff86c70fd2df6fc, verified against https://github.com/snowballstem/snowball-data.git at
8
+ * a0ec0d0a2839ec885878868de20fcb63209d92b0.
9
+ *
10
+ * packages/ts/snowball/build.ts writes this file, so regenerate it with that script.
11
+ */
12
+
13
+ #if defined(__GNUC__)
14
+ #pragma GCC diagnostic ignored "-Wcast-align"
15
+ #pragma GCC diagnostic ignored "-Wsign-conversion"
16
+ #endif
17
+
18
+
19
+ #include <stdio.h>
20
+ #include <stdlib.h>
21
+ #include <string.h>
22
+
23
+ #include "snowball_runtime.h"
24
+
25
+ #ifdef SNOWBALL_RUNTIME_THROW_EXCEPTIONS
26
+ # include <new>
27
+ # include <stdexcept>
28
+ # define SNOWBALL_RETURN_OK return
29
+ # define SNOWBALL_RETURN_OR_THROW(R, E) throw E
30
+ # define SNOWBALL_PROPAGATE_ERR(F) F
31
+ #else
32
+ # define SNOWBALL_RETURN_OK return 0
33
+ # define SNOWBALL_RETURN_OR_THROW(R, E) return R
34
+ # define SNOWBALL_PROPAGATE_ERR(F) do { \
35
+ int snowball_err = F; \
36
+ if (snowball_err < 0) return snowball_err; \
37
+ } while (0)
38
+ #endif
39
+
40
+ #define HEAD (2 * sizeof(int))
41
+
42
+ /* Note that sizeof(symbol) should divide HEAD without remainder, otherwise
43
+ * there is an alignment problem.
44
+ *
45
+ * We support C90 here so use a typedef trick instead of static_assert.
46
+ */
47
+ typedef int sizeof_symbol_divides_head[(HEAD % sizeof(symbol) == 0) ? 1 : -1];
48
+
49
+ #define CREATE_SIZE 31
50
+
51
+ extern symbol * create_s(void) {
52
+ symbol * p;
53
+ void * mem = malloc(HEAD + (CREATE_SIZE + 1) * sizeof(symbol));
54
+ if (mem == NULL)
55
+ SNOWBALL_RETURN_OR_THROW(NULL, std::bad_alloc());
56
+ p = (symbol *) (HEAD + (char *) mem);
57
+ CAPACITY(p) = CREATE_SIZE;
58
+ SET_SIZE(p, 0);
59
+ return p;
60
+ }
61
+
62
+ extern void lose_s(symbol * p) {
63
+ if (p == NULL) return;
64
+ free((char *) p - HEAD);
65
+ }
66
+
67
+ /*
68
+ new_p = skip_utf8(p, c, l, n); skips n characters forwards from p + c.
69
+ new_p is the new position, or -1 on failure (if c would be > l).
70
+
71
+ Caller ensures n >= 0.
72
+
73
+ -- used to implement hop and next in the utf8 case.
74
+ */
75
+
76
+ extern int skip_utf8(const symbol * p, int c, int limit, int n) {
77
+ int b;
78
+ for (; n > 0; n--) {
79
+ if (c >= limit) return -1;
80
+ b = p[c++];
81
+ if (b >= 0xC0) { /* 1100 0000 */
82
+ while (c < limit) {
83
+ b = p[c];
84
+ if (b >= 0xC0 || b < 0x80) break;
85
+ /* break unless b is 10------ */
86
+ c++;
87
+ }
88
+ }
89
+ }
90
+ return c;
91
+ }
92
+
93
+ /*
94
+ new_p = skip_b_utf8(p, c, lb, n); skips n characters backwards from p + c - 1
95
+ new_p is the new position, or -1 on failure (if c would be < lb).
96
+
97
+ Caller ensures n >= 0.
98
+
99
+ -- used to implement hop and next in the utf8 case.
100
+ */
101
+
102
+ extern int skip_b_utf8(const symbol * p, int c, int limit, int n) {
103
+ int b;
104
+ for (; n > 0; n--) {
105
+ if (c <= limit) return -1;
106
+ b = p[--c];
107
+ if (b >= 0x80) { /* 1000 0000 */
108
+ while (c > limit) {
109
+ b = p[c];
110
+ if (b >= 0xC0) break; /* 1100 0000 */
111
+ c--;
112
+ }
113
+ }
114
+ }
115
+ return c;
116
+ }
117
+
118
+ /* Code for character groupings: utf8 cases */
119
+
120
+ static int get_utf8(const symbol * p, int c, int l, int * slot) {
121
+ int b0, b1, b2;
122
+ if (c >= l) return 0;
123
+ b0 = p[c++];
124
+ if (b0 < 0xC0 || c == l) { /* 1100 0000 */
125
+ *slot = b0;
126
+ return 1;
127
+ }
128
+ b1 = p[c++] & 0x3F;
129
+ if (b0 < 0xE0 || c == l) { /* 1110 0000 */
130
+ *slot = (b0 & 0x1F) << 6 | b1;
131
+ return 2;
132
+ }
133
+ b2 = p[c++] & 0x3F;
134
+ if (b0 < 0xF0 || c == l) { /* 1111 0000 */
135
+ *slot = (b0 & 0xF) << 12 | b1 << 6 | b2;
136
+ return 3;
137
+ }
138
+ *slot = (b0 & 0x7) << 18 | b1 << 12 | b2 << 6 | (p[c] & 0x3F);
139
+ return 4;
140
+ }
141
+
142
+ static int get_b_utf8(const symbol * p, int c, int lb, int * slot) {
143
+ int a, b;
144
+ if (c <= lb) return 0;
145
+ b = p[--c];
146
+ if (b < 0x80 || c == lb) { /* 1000 0000 */
147
+ *slot = b;
148
+ return 1;
149
+ }
150
+ a = b & 0x3F;
151
+ b = p[--c];
152
+ if (b >= 0xC0 || c == lb) { /* 1100 0000 */
153
+ *slot = (b & 0x1F) << 6 | a;
154
+ return 2;
155
+ }
156
+ a |= (b & 0x3F) << 6;
157
+ b = p[--c];
158
+ if (b >= 0xE0 || c == lb) { /* 1110 0000 */
159
+ *slot = (b & 0xF) << 12 | a;
160
+ return 3;
161
+ }
162
+ *slot = (p[--c] & 0x7) << 18 | (b & 0x3F) << 12 | a;
163
+ return 4;
164
+ }
165
+
166
+ #ifdef SNOWBALL_COVERAGE
167
+ /* The grouping number gets stored in a byte, clamped to 255. */
168
+ static char grouping_seen[255];
169
+
170
+ static void report_coverage(const unsigned char * s, int min, int max, int ch, const unsigned char * p, int w) {
171
+ int i = 0;
172
+ int j;
173
+ int outof = 0;
174
+ const unsigned char * loc = s + (max - min + 8) / 8;
175
+ int grouping_number = *loc++;
176
+ /* Adjust ch be an offset from min if it's past the end of the range. If
177
+ * we already subtracted min then this will condition will be false. Only
178
+ * needed for the "out" case but the condition can never be true for the
179
+ * "in" case.
180
+ */
181
+ if (ch > max) ch -= min;
182
+ /* Find the index of this character in the grouping. */
183
+ for (j = 0; j != max - min; ++j) {
184
+ if (s[j >> 3] & (0X1 << (j & 0X7))) {
185
+ ++outof;
186
+ if (j < ch) ++i;
187
+ }
188
+ }
189
+ if (grouping_number < (int)sizeof(grouping_seen) &&
190
+ grouping_seen[grouping_number] == 0) {
191
+ /* Report every entry once, then unused cases will appear (and we can
192
+ * decrement each count when generating the coverage report).
193
+ */
194
+ int k = 0;
195
+ for (j = 0; j != max - min; ++j) {
196
+ if (s[j >> 3] & (0X1 << (j & 0X7))) {
197
+ fprintf(stderr, "%s index %d of %d '", loc, k, outof + 1);
198
+ int codepoint = j + min;
199
+ if (codepoint < 0x80) {
200
+ putc(codepoint, stderr);
201
+ } else if (codepoint < 0x800) {
202
+ putc((codepoint >> 6) | 0xC0, stderr);
203
+ putc((codepoint & 0x3F) | 0x80, stderr);
204
+ } else {
205
+ putc((codepoint >> 12) | 0xE0, stderr);
206
+ putc(((codepoint >> 6) & 0x3F) | 0x80, stderr);
207
+ putc((codepoint & 0x3F) | 0x80, stderr);
208
+ }
209
+ fprintf(stderr, "'\n");
210
+ ++k;
211
+ }
212
+ }
213
+ grouping_seen[grouping_number] = 1;
214
+ }
215
+ fprintf(stderr, "%s index %d of %d '%.*s'\n", loc, i, outof + 1, w, p);
216
+ }
217
+
218
+ static void report_coverage_nomatch(const unsigned char * s, int min, int max) {
219
+ const unsigned char * loc = s + (max - min + 8) / 8;
220
+ ++loc;
221
+ fprintf(stderr, "%s no match\n", loc);
222
+ }
223
+ #endif
224
+
225
+ extern int in_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
226
+ do {
227
+ int ch;
228
+ int w = get_utf8(z->p, z->c, z->l, & ch);
229
+ if (!w) return -1;
230
+ if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0) {
231
+ #ifdef SNOWBALL_COVERAGE
232
+ report_coverage_nomatch(s, min, max);
233
+ #endif
234
+ return w;
235
+ }
236
+ #ifdef SNOWBALL_COVERAGE
237
+ report_coverage(s, min, max, ch, z->p + z->c, w);
238
+ #endif
239
+ z->c += w;
240
+ } while (repeat);
241
+ return 0;
242
+ }
243
+
244
+ extern int in_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
245
+ do {
246
+ int ch;
247
+ int w = get_b_utf8(z->p, z->c, z->lb, & ch);
248
+ if (!w) return -1;
249
+ if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0) {
250
+ #ifdef SNOWBALL_COVERAGE
251
+ report_coverage_nomatch(s, min, max);
252
+ #endif
253
+ return w;
254
+ }
255
+ #ifdef SNOWBALL_COVERAGE
256
+ report_coverage(s, min, max, ch, z->p + z->c - w, w);
257
+ #endif
258
+ z->c -= w;
259
+ } while (repeat);
260
+ return 0;
261
+ }
262
+
263
+ extern int out_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
264
+ do {
265
+ int ch;
266
+ int w = get_utf8(z->p, z->c, z->l, & ch);
267
+ if (!w) return -1;
268
+ if (!(ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)) {
269
+ #ifdef SNOWBALL_COVERAGE
270
+ report_coverage(s, min, max, ch, z->p + z->c, w);
271
+ #endif
272
+ return w;
273
+ }
274
+ #ifdef SNOWBALL_COVERAGE
275
+ report_coverage_nomatch(s, min, max);
276
+ #endif
277
+ z->c += w;
278
+ } while (repeat);
279
+ return 0;
280
+ }
281
+
282
+ extern int out_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
283
+ do {
284
+ int ch;
285
+ int w = get_b_utf8(z->p, z->c, z->lb, & ch);
286
+ if (!w) return -1;
287
+ if (!(ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)) {
288
+ #ifdef SNOWBALL_COVERAGE
289
+ report_coverage(s, min, max, ch, z->p + z->c - w, w);
290
+ #endif
291
+ return w;
292
+ }
293
+ #ifdef SNOWBALL_COVERAGE
294
+ report_coverage_nomatch(s, min, max);
295
+ #endif
296
+ z->c -= w;
297
+ } while (repeat);
298
+ return 0;
299
+ }
300
+
301
+ /* Code for character groupings: non-utf8 cases */
302
+
303
+ extern int in_grouping(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
304
+ do {
305
+ int ch;
306
+ if (z->c >= z->l) return -1;
307
+ ch = z->p[z->c];
308
+ if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
309
+ return 1;
310
+ z->c++;
311
+ } while (repeat);
312
+ return 0;
313
+ }
314
+
315
+ extern int in_grouping_b(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
316
+ do {
317
+ int ch;
318
+ if (z->c <= z->lb) return -1;
319
+ ch = z->p[z->c - 1];
320
+ if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
321
+ return 1;
322
+ z->c--;
323
+ } while (repeat);
324
+ return 0;
325
+ }
326
+
327
+ extern int out_grouping(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
328
+ do {
329
+ int ch;
330
+ if (z->c >= z->l) return -1;
331
+ ch = z->p[z->c];
332
+ if (!(ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0))
333
+ return 1;
334
+ z->c++;
335
+ } while (repeat);
336
+ return 0;
337
+ }
338
+
339
+ extern int out_grouping_b(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
340
+ do {
341
+ int ch;
342
+ if (z->c <= z->lb) return -1;
343
+ ch = z->p[z->c - 1];
344
+ if (!(ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0))
345
+ return 1;
346
+ z->c--;
347
+ } while (repeat);
348
+ return 0;
349
+ }
350
+
351
+ extern int eq_s(struct SN_env * z, int s_size, const symbol * s) {
352
+ if (z->l - z->c < s_size || memcmp(z->p + z->c, s, s_size * sizeof(symbol)) != 0) return 0;
353
+ z->c += s_size; return 1;
354
+ }
355
+
356
+ extern int eq_s_b(struct SN_env * z, int s_size, const symbol * s) {
357
+ if (z->c - z->lb < s_size || memcmp(z->p + z->c - s_size, s, s_size * sizeof(symbol)) != 0) return 0;
358
+ z->c -= s_size; return 1;
359
+ }
360
+
361
+ extern int eq_v(struct SN_env * z, const symbol * p) {
362
+ return eq_s(z, SIZE(p), p);
363
+ }
364
+
365
+ extern int eq_v_b(struct SN_env * z, const symbol * p) {
366
+ return eq_s_b(z, SIZE(p), p);
367
+ }
368
+
369
+ #ifdef SNOWBALL_COVERAGE
370
+ /* Declare more entries than any real Snowball program will have. */
371
+ static char among_seen[4096];
372
+ #endif
373
+
374
+ extern int find_among(struct SN_env * z, const struct among * v, int v_size,
375
+ int (*call_among_func)(struct SN_env*)) {
376
+
377
+ int i = 0;
378
+ int j = v_size;
379
+
380
+ int c = z->c; int l = z->l;
381
+ const symbol * q = z->p + c;
382
+
383
+ const struct among * w;
384
+
385
+ int common_i = 0;
386
+ int common_j = 0;
387
+
388
+ int first_key_inspected = 0;
389
+
390
+ #ifdef SNOWBALL_COVERAGE
391
+ int among_number = v[v_size].s_size;
392
+ if (among_number < (int)sizeof(among_seen) &&
393
+ among_seen[among_number] == 0) {
394
+ /* Report every entry once, then unused cases will appear (and we can
395
+ * decrement each count when generating the coverage report).
396
+ */
397
+ int k;
398
+ for (k = 0; k < v_size; ++k) {
399
+ w = v + k;
400
+ fprintf(stderr, "%s: among %d : %d of %d string '%.*s'\n", w[v_size].s, among_number, w[v_size].result, v_size, w->s_size, w->s);
401
+ if (w->function) {
402
+ fprintf(stderr, "%s: among %d : %d of %d func-f '%.*s'\n", w[v_size].s, among_number, w[v_size].result, v_size, w->s_size, w->s);
403
+ }
404
+ }
405
+ /* If the among matches the empty string without a gating function then
406
+ * the "no match" case is impossible and so not useful to include in a
407
+ * coverage report.
408
+ */
409
+ if (v[v_size * 2].s_size != -1) {
410
+ fprintf(stderr, "%s: among %d no match\n", v[v_size * 2].s, among_number);
411
+ }
412
+ among_seen[among_number] = 1;
413
+ }
414
+ #endif
415
+ while (1) {
416
+ int k = i + ((j - i) >> 1);
417
+ int diff = 0;
418
+ int common = common_i < common_j ? common_i : common_j; /* smaller */
419
+ w = v + k;
420
+ {
421
+ int i2; for (i2 = common; i2 < w->s_size; i2++) {
422
+ if (c + common == l) { diff = -1; break; }
423
+ diff = q[common] - w->s[i2];
424
+ if (diff != 0) break;
425
+ common++;
426
+ }
427
+ }
428
+ if (diff < 0) {
429
+ j = k;
430
+ common_j = common;
431
+ } else {
432
+ i = k;
433
+ common_i = common;
434
+ }
435
+ if (j - i <= 1) {
436
+ if (i > 0) break; /* v->s has been inspected */
437
+ if (j == i) break; /* only one item in v */
438
+
439
+ /* - but now we need to go round once more to get
440
+ v->s inspected. This looks messy, but is actually
441
+ the optimal approach. */
442
+
443
+ if (first_key_inspected) break;
444
+ first_key_inspected = 1;
445
+ }
446
+ }
447
+ w = v + i;
448
+ while (1) {
449
+ if (common_i >= w->s_size) {
450
+ z->c = c + w->s_size;
451
+ if (!w->function) {
452
+ #ifdef SNOWBALL_COVERAGE
453
+ fprintf(stderr, "%s: among %d : %d of %d string '%.*s'\n", w[v_size].s, among_number, w[v_size].result, v_size, w->s_size, w->s);
454
+ #endif
455
+ return w->result;
456
+ }
457
+ z->af = w->function;
458
+ if (call_among_func(z)) {
459
+ z->c = c + w->s_size;
460
+ #ifdef SNOWBALL_COVERAGE
461
+ fprintf(stderr, "%s: among %d : %d of %d string '%.*s'\n", w[v_size].s, among_number, w[v_size].result, v_size, w->s_size, w->s);
462
+ #endif
463
+ return w->result;
464
+ }
465
+ #ifdef SNOWBALL_COVERAGE
466
+ fprintf(stderr, "%s: among %d : %d of %d func-f '%.*s'\n", w[v_size].s, among_number, w[v_size].result, v_size, w->s_size, w->s);
467
+ #endif
468
+ }
469
+ if (!w->substring_i) {
470
+ #ifdef SNOWBALL_COVERAGE
471
+ fprintf(stderr, "%s: among %d no match\n", v[v_size * 2].s, among_number);
472
+ #endif
473
+ return 0;
474
+ }
475
+ w += w->substring_i;
476
+ }
477
+ }
478
+
479
+ /* find_among_b is for backwards processing. Same comments apply */
480
+
481
+ extern int find_among_b(struct SN_env * z, const struct among * v, int v_size,
482
+ int (*call_among_func)(struct SN_env*)) {
483
+
484
+ int i = 0;
485
+ int j = v_size;
486
+
487
+ int c = z->c; int lb = z->lb;
488
+ const symbol * q = z->p + c - 1;
489
+
490
+ const struct among * w;
491
+
492
+ int common_i = 0;
493
+ int common_j = 0;
494
+
495
+ int first_key_inspected = 0;
496
+
497
+ #ifdef SNOWBALL_COVERAGE
498
+ int among_number = v[v_size].s_size;
499
+ if (among_number < (int)sizeof(among_seen) &&
500
+ among_seen[among_number] == 0) {
501
+ /* Report every entry once, then unused cases will appear (and we can
502
+ * decrement each count when generating the coverage report).
503
+ */
504
+ int k;
505
+ for (k = 0; k < v_size; ++k) {
506
+ w = v + k;
507
+ fprintf(stderr, "%s: among %d : %d of %d string '%.*s'\n", w[v_size].s, among_number, w[v_size].result, v_size, w->s_size, w->s);
508
+ if (w->function) {
509
+ fprintf(stderr, "%s: among %d : %d of %d func-f '%.*s'\n", w[v_size].s, among_number, w[v_size].result, v_size, w->s_size, w->s);
510
+ }
511
+ }
512
+ /* If the among matches the empty string without a gating function then
513
+ * the "no match" case is impossible and so not useful to include in a
514
+ * coverage report.
515
+ */
516
+ if (v[v_size * 2].s_size != -1) {
517
+ fprintf(stderr, "%s: among %d no match\n", v[v_size * 2].s, among_number);
518
+ }
519
+ among_seen[among_number] = 1;
520
+ }
521
+ #endif
522
+ while (1) {
523
+ int k = i + ((j - i) >> 1);
524
+ int diff = 0;
525
+ int common = common_i < common_j ? common_i : common_j;
526
+ w = v + k;
527
+ {
528
+ int i2; for (i2 = w->s_size - 1 - common; i2 >= 0; i2--) {
529
+ if (c - common == lb) { diff = -1; break; }
530
+ diff = q[- common] - w->s[i2];
531
+ if (diff != 0) break;
532
+ common++;
533
+ }
534
+ }
535
+ if (diff < 0) { j = k; common_j = common; }
536
+ else { i = k; common_i = common; }
537
+ if (j - i <= 1) {
538
+ if (i > 0) break;
539
+ if (j == i) break;
540
+ if (first_key_inspected) break;
541
+ first_key_inspected = 1;
542
+ }
543
+ }
544
+ w = v + i;
545
+ while (1) {
546
+ if (common_i >= w->s_size) {
547
+ z->c = c - w->s_size;
548
+ if (!w->function) {
549
+ #ifdef SNOWBALL_COVERAGE
550
+ fprintf(stderr, "%s: among %d : %d of %d string '%.*s'\n", w[v_size].s, among_number, w[v_size].result, v_size, w->s_size, w->s);
551
+ #endif
552
+ return w->result;
553
+ }
554
+ z->af = w->function;
555
+ if (call_among_func(z)) {
556
+ #ifdef SNOWBALL_COVERAGE
557
+ fprintf(stderr, "%s: among %d : %d of %d string '%.*s'\n", w[v_size].s, among_number, w[v_size].result, v_size, w->s_size, w->s);
558
+ #endif
559
+ z->c = c - w->s_size;
560
+ return w->result;
561
+ }
562
+ #ifdef SNOWBALL_COVERAGE
563
+ fprintf(stderr, "%s: among %d : %d of %d func-f '%.*s'\n", w[v_size].s, among_number, w[v_size].result, v_size, w->s_size, w->s);
564
+ #endif
565
+ }
566
+ if (!w->substring_i) {
567
+ #ifdef SNOWBALL_COVERAGE
568
+ fprintf(stderr, "%s: among %d no match\n", v[v_size * 2].s, among_number);
569
+ #endif
570
+ return 0;
571
+ }
572
+ w += w->substring_i;
573
+ }
574
+ }
575
+
576
+
577
+ /* Increase the size of the buffer pointed to by p to at least n symbols.
578
+ * On success, returns 0. If insufficient memory, returns -1.
579
+ */
580
+ static int increase_size(symbol ** p, int n) {
581
+ int new_size = n + 20;
582
+ void * mem = realloc((char *) *p - HEAD,
583
+ HEAD + (new_size + 1) * sizeof(symbol));
584
+ symbol * q;
585
+ if (mem == NULL) return -1;
586
+ q = (symbol *) (HEAD + (char *)mem);
587
+ CAPACITY(q) = new_size;
588
+ *p = q;
589
+ return 0;
590
+ }
591
+
592
+ /* to replace symbols between c_bra and c_ket in z->p by the
593
+ s_size symbols at s.
594
+ Returns 0 on success, -1 on error.
595
+ */
596
+ extern SNOWBALL_ERR replace_s(struct SN_env * z, int c_bra, int c_ket, int s_size, const symbol * s)
597
+ {
598
+ int adjustment = s_size - (c_ket - c_bra);
599
+ if (adjustment != 0) {
600
+ int len = SIZE(z->p);
601
+ if (adjustment + len > CAPACITY(z->p)) {
602
+ SNOWBALL_PROPAGATE_ERR(increase_size(&z->p, adjustment + len));
603
+ }
604
+ memmove(z->p + c_ket + adjustment,
605
+ z->p + c_ket,
606
+ (len - c_ket) * sizeof(symbol));
607
+ SET_SIZE(z->p, adjustment + len);
608
+ z->l += adjustment;
609
+ if (z->c >= c_ket)
610
+ z->c += adjustment;
611
+ else if (z->c > c_bra)
612
+ z->c = c_bra;
613
+ }
614
+ if (s_size) memmove(z->p + c_bra, s, s_size * sizeof(symbol));
615
+ SNOWBALL_RETURN_OK;
616
+ }
617
+
618
+ # define REPLACE_S(Z, B, K, SIZE, S) \
619
+ SNOWBALL_PROPAGATE_ERR(replace_s(Z, B, K, SIZE, S))
620
+
621
+ static SNOWBALL_ERR slice_check(struct SN_env * z) {
622
+
623
+ if (z->bra < 0 ||
624
+ z->bra > z->ket ||
625
+ z->ket > z->l ||
626
+ z->l > SIZE(z->p)) /* this line could be removed */
627
+ {
628
+ #if 0
629
+ fprintf(stderr, "faulty slice operation:\n");
630
+ debug(z, -1, 0);
631
+ #endif
632
+ SNOWBALL_RETURN_OR_THROW(-1, std::logic_error("Snowball slice invalid"));
633
+ }
634
+ SNOWBALL_RETURN_OK;
635
+ }
636
+
637
+ # define SLICE_CHECK(Z) SNOWBALL_PROPAGATE_ERR(slice_check(Z))
638
+
639
+ extern SNOWBALL_ERR slice_from_s(struct SN_env * z, int s_size, const symbol * s) {
640
+ SLICE_CHECK(z);
641
+ REPLACE_S(z, z->bra, z->ket, s_size, s);
642
+ z->ket = z->bra + s_size;
643
+ SNOWBALL_RETURN_OK;
644
+ }
645
+
646
+ extern SNOWBALL_ERR slice_from_v(struct SN_env * z, const symbol * p) {
647
+ return slice_from_s(z, SIZE(p), p);
648
+ }
649
+
650
+ extern SNOWBALL_ERR slice_del(struct SN_env * z) {
651
+ SLICE_CHECK(z);
652
+ {
653
+ int slice_size = z->ket - z->bra;
654
+ if (slice_size != 0) {
655
+ int len = SIZE(z->p);
656
+ memmove(z->p + z->bra,
657
+ z->p + z->ket,
658
+ (len - z->ket) * sizeof(symbol));
659
+ SET_SIZE(z->p, len - slice_size);
660
+ z->l -= slice_size;
661
+ if (z->c >= z->ket)
662
+ z->c -= slice_size;
663
+ else if (z->c > z->bra)
664
+ z->c = z->bra;
665
+ z->ket = z->bra;
666
+ }
667
+ }
668
+ SNOWBALL_RETURN_OK;
669
+ }
670
+
671
+ extern SNOWBALL_ERR insert_s(struct SN_env * z, int bra, int ket, int s_size, const symbol * s) {
672
+ REPLACE_S(z, bra, ket, s_size, s);
673
+ if (bra <= z->ket) {
674
+ int adjustment = s_size - (ket - bra);
675
+ z->ket += adjustment;
676
+ if (bra <= z->bra) z->bra += adjustment;
677
+ }
678
+ SNOWBALL_RETURN_OK;
679
+ }
680
+
681
+ extern SNOWBALL_ERR insert_v(struct SN_env * z, int bra, int ket, const symbol * p) {
682
+ return insert_s(z, bra, ket, SIZE(p), p);
683
+ }
684
+
685
+ extern SNOWBALL_ERR slice_to(struct SN_env * z, symbol ** p) {
686
+ SLICE_CHECK(z);
687
+ {
688
+ int len = z->ket - z->bra;
689
+ if (CAPACITY(*p) < len) {
690
+ SNOWBALL_PROPAGATE_ERR(increase_size(p, len));
691
+ }
692
+ memmove(*p, z->p + z->bra, len * sizeof(symbol));
693
+ SET_SIZE(*p, len);
694
+ }
695
+ SNOWBALL_RETURN_OK;
696
+ }
697
+
698
+ extern SNOWBALL_ERR assign_to(struct SN_env * z, symbol ** p) {
699
+ int len = z->l;
700
+ if (CAPACITY(*p) < len) {
701
+ SNOWBALL_PROPAGATE_ERR(increase_size(p, len));
702
+ }
703
+ memmove(*p, z->p, len * sizeof(symbol));
704
+ SET_SIZE(*p, len);
705
+ SNOWBALL_RETURN_OK;
706
+ }
707
+
708
+ extern int len_utf8(const symbol * p) {
709
+ int size = SIZE(p);
710
+ int len = 0;
711
+ while (size--) {
712
+ symbol b = *p++;
713
+ if (b >= 0xC0 || b < 0x80) ++len;
714
+ }
715
+ return len;
716
+ }