iv-phonic 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. data/.autotest +24 -0
  2. data/Manifest.txt +49 -0
  3. data/README.rdoc +32 -0
  4. data/Rakefile +54 -0
  5. data/ext/include/iv/algorithm.h +23 -0
  6. data/ext/include/iv/alloc.h +200 -0
  7. data/ext/include/iv/any.h +71 -0
  8. data/ext/include/iv/ast-factory.h +277 -0
  9. data/ext/include/iv/ast-fwd.h +92 -0
  10. data/ext/include/iv/ast-serializer.h +579 -0
  11. data/ext/include/iv/ast-visitor.h +121 -0
  12. data/ext/include/iv/ast.h +1127 -0
  13. data/ext/include/iv/chars.h +83 -0
  14. data/ext/include/iv/cmdline.h +830 -0
  15. data/ext/include/iv/conversions.h +308 -0
  16. data/ext/include/iv/dtoa.h +20 -0
  17. data/ext/include/iv/enable_if.h +18 -0
  18. data/ext/include/iv/errors.h +15 -0
  19. data/ext/include/iv/fixedcontainer.h +42 -0
  20. data/ext/include/iv/functor.h +29 -0
  21. data/ext/include/iv/lexer.h +1281 -0
  22. data/ext/include/iv/location.h +23 -0
  23. data/ext/include/iv/mt19937.h +175 -0
  24. data/ext/include/iv/noncopyable.h +30 -0
  25. data/ext/include/iv/none.h +10 -0
  26. data/ext/include/iv/parser.h +2150 -0
  27. data/ext/include/iv/source.h +27 -0
  28. data/ext/include/iv/space.h +178 -0
  29. data/ext/include/iv/static_assert.h +30 -0
  30. data/ext/include/iv/stringpiece.h +385 -0
  31. data/ext/include/iv/token.h +311 -0
  32. data/ext/include/iv/ucdata.h +58 -0
  33. data/ext/include/iv/uchar.h +8 -0
  34. data/ext/include/iv/ustring.h +28 -0
  35. data/ext/include/iv/ustringpiece.h +9 -0
  36. data/ext/include/iv/utils.h +83 -0
  37. data/ext/include/iv/xorshift.h +74 -0
  38. data/ext/iv/phonic/ast-fwd.h +21 -0
  39. data/ext/iv/phonic/ast.h +10 -0
  40. data/ext/iv/phonic/creator.h +530 -0
  41. data/ext/iv/phonic/encoding.h +110 -0
  42. data/ext/iv/phonic/extconf.rb +5 -0
  43. data/ext/iv/phonic/factory.h +247 -0
  44. data/ext/iv/phonic/parser.h +12 -0
  45. data/ext/iv/phonic/phonic.cc +69 -0
  46. data/ext/iv/phonic/rnode.h +15 -0
  47. data/ext/iv/phonic/rparser.h +48 -0
  48. data/ext/iv/phonic/source.h +146 -0
  49. data/test/test_iv_phonic.rb +32 -0
  50. metadata +159 -0
@@ -0,0 +1,308 @@
1
+ #ifndef IV_CONVERSIONS_H_
2
+ #define IV_CONVERSIONS_H_
3
+ #include <cstdio>
4
+ #include <cmath>
5
+ #include <string>
6
+ #include <limits>
7
+ #include <tr1/array>
8
+ #include <tr1/cstdint>
9
+ #include "chars.h"
10
+ #include "dtoa.h"
11
+ #include "ustringpiece.h"
12
+ #include "none.h"
13
+ namespace iv {
14
+ namespace core {
15
+ namespace detail {
16
+ template<typename T>
17
+ class Conversions {
18
+ public:
19
+ static const double kNaN;
20
+ static const int kMaxSignificantDigits = 772;
21
+ static const std::string kInfinity;
22
+ static const double DoubleToInt32_Two32;
23
+ static const double DoubleToInt32_Two31;
24
+ };
25
+ template<typename T>
26
+ const double Conversions<T>::kNaN = std::numeric_limits<double>::quiet_NaN();
27
+
28
+ template<typename T>
29
+ const std::string Conversions<T>::kInfinity = "Infinity";
30
+
31
+ template<typename T>
32
+ const double Conversions<T>::DoubleToInt32_Two32 = 4294967296.0;
33
+
34
+ template<typename T>
35
+ const double Conversions<T>::DoubleToInt32_Two31 = 2147483648.0;
36
+ } // namespace iv::core::detail
37
+
38
+ typedef detail::Conversions<None> Conversions;
39
+
40
+ template<typename Iter>
41
+ inline double StringToDouble(Iter it, Iter last) {
42
+ bool is_decimal = true;
43
+ bool is_signed = false;
44
+ std::size_t pos = 0;
45
+ int significant_digits = 0;
46
+ int insignificant_digits = 0;
47
+ std::tr1::array<char, Conversions::kMaxSignificantDigits+10> buffer;
48
+
49
+ // empty string ""
50
+ if (it == last) {
51
+ return 0;
52
+ }
53
+
54
+ while (it != last &&
55
+ (Chars::IsWhiteSpace(*it) || Chars::IsLineTerminator(*it))) {
56
+ ++it;
57
+ }
58
+
59
+ // white space only " "
60
+ if (it == last) {
61
+ return 0;
62
+ }
63
+
64
+ if (*it == '-') {
65
+ buffer[pos++] = '-';
66
+ ++it;
67
+ is_signed = true;
68
+ } else if (*it == '+') {
69
+ ++it;
70
+ }
71
+
72
+ if (it == last) {
73
+ return Conversions::kNaN;
74
+ }
75
+
76
+ if (Chars::IsDecimalDigit(*it)) {
77
+ if (*it == '0') {
78
+ ++it;
79
+ if (it == last) {
80
+ return 0;
81
+ }
82
+ if (*it == 'x' || *it == 'X') {
83
+ is_decimal = false;
84
+ buffer[pos++] = '0';
85
+ buffer[pos++] = *it;
86
+ ++it;
87
+ ++significant_digits;
88
+ if (it == last || !Chars::IsHexDigit(*it)) {
89
+ return Conversions::kNaN;
90
+ }
91
+ // waste leading zero
92
+ while (it != last && *it == '0') {
93
+ ++it;
94
+ }
95
+ while (it != last && Chars::IsHexDigit(*it)) {
96
+ if (significant_digits < Conversions::kMaxSignificantDigits) {
97
+ buffer[pos++] = *it;
98
+ ++it;
99
+ ++significant_digits;
100
+ } else {
101
+ ++it;
102
+ }
103
+ }
104
+ } else {
105
+ // waste leading zero
106
+ while (it != last && *it == '0') {
107
+ ++it;
108
+ }
109
+ }
110
+ }
111
+ if (is_decimal) {
112
+ while (it != last &&
113
+ Chars::IsDecimalDigit(*it)) {
114
+ if (significant_digits < Conversions::kMaxSignificantDigits) {
115
+ buffer[pos++] = *it;
116
+ ++significant_digits;
117
+ } else {
118
+ ++insignificant_digits;
119
+ }
120
+ ++it;
121
+ }
122
+ if (it != last && *it == '.') {
123
+ buffer[pos++] = '.';
124
+ ++it;
125
+ while (it != last &&
126
+ Chars::IsDecimalDigit(*it)) {
127
+ if (significant_digits < Conversions::kMaxSignificantDigits) {
128
+ buffer[pos++] = *it;
129
+ ++significant_digits;
130
+ }
131
+ ++it;
132
+ }
133
+ }
134
+ }
135
+ } else {
136
+ if (*it == '.') {
137
+ buffer[pos++] = '.';
138
+ ++it;
139
+ while (it != last &&
140
+ Chars::IsDecimalDigit(*it)) {
141
+ if (significant_digits < Conversions::kMaxSignificantDigits) {
142
+ buffer[pos++] = *it;
143
+ ++significant_digits;
144
+ }
145
+ ++it;
146
+ }
147
+ } else {
148
+ for (std::string::const_iterator inf_it = Conversions::kInfinity.begin(),
149
+ inf_last = Conversions::kInfinity.end();
150
+ inf_it != inf_last; ++inf_it, ++it) {
151
+ if (it == last ||
152
+ (*inf_it) != (*it)) {
153
+ return Conversions::kNaN;
154
+ }
155
+ }
156
+ // infinity
157
+ while (it != last &&
158
+ (Chars::IsWhiteSpace(*it) || Chars::IsLineTerminator(*it))) {
159
+ ++it;
160
+ }
161
+ if (it == last) {
162
+ if (is_signed) {
163
+ return -std::numeric_limits<double>::infinity();
164
+ } else {
165
+ return std::numeric_limits<double>::infinity();
166
+ }
167
+ } else {
168
+ return Conversions::kNaN;
169
+ }
170
+ }
171
+ }
172
+
173
+ // exponent part
174
+ if (it != last && (*it == 'e' || *it == 'E')) {
175
+ if (!is_decimal) {
176
+ return Conversions::kNaN;
177
+ }
178
+ buffer[pos++] = *it;
179
+ ++it;
180
+ if (it == last) {
181
+ return Conversions::kNaN;
182
+ }
183
+ if (*it == '+' || *it == '-') {
184
+ buffer[pos++] = *it;
185
+ ++it;
186
+ }
187
+ if (it == last) {
188
+ return Conversions::kNaN;
189
+ }
190
+ // more than 1 decimal digit required
191
+ if (!Chars::IsDecimalDigit(*it)) {
192
+ return Conversions::kNaN;
193
+ }
194
+ int exponent = 0;
195
+ do {
196
+ if (exponent > 9999) {
197
+ exponent = 9999;
198
+ } else {
199
+ exponent = exponent * 10 + (*it - '0');
200
+ }
201
+ ++it;
202
+ } while (it != last && Chars::IsDecimalDigit(*it));
203
+ exponent+=insignificant_digits;
204
+ if (exponent > 9999) {
205
+ exponent = 9999;
206
+ }
207
+ std::snprintf(buffer.data()+pos, 5, "%d", exponent); // NOLINT
208
+ pos+=4;
209
+ }
210
+
211
+ while (it != last &&
212
+ (Chars::IsWhiteSpace(*it) || Chars::IsLineTerminator(*it))) {
213
+ ++it;
214
+ }
215
+
216
+ if (it == last) {
217
+ if (pos == 0) {
218
+ // empty
219
+ return 0;
220
+ } else {
221
+ buffer[pos++] = '\0';
222
+ return std::strtod(buffer.data(), NULL);
223
+ }
224
+ } else {
225
+ return Conversions::kNaN;
226
+ }
227
+ }
228
+
229
+ inline double StringToDouble(const StringPiece& str) {
230
+ return StringToDouble(str.begin(), str.end());
231
+ }
232
+
233
+ inline double StringToDouble(const UStringPiece& str) {
234
+ return StringToDouble(str.begin(), str.end());
235
+ }
236
+
237
+ inline std::size_t StringToHash(const UStringPiece& x) {
238
+ std::size_t len = x.size();
239
+ std::size_t step = (len >> 5) + 1;
240
+ std::size_t h = 0;
241
+ for (std::size_t l1 = len; l1 >= step; l1 -= step) {
242
+ h = h ^ ((h << 5) + (h >> 2) + x[l1-1]);
243
+ }
244
+ return h;
245
+ }
246
+
247
+ inline std::size_t StringToHash(const StringPiece& x) {
248
+ std::size_t len = x.size();
249
+ std::size_t step = (len >> 5) + 1;
250
+ std::size_t h = 0;
251
+ for (std::size_t l1 = len; l1 >= step; l1 -= step) {
252
+ h = h ^ ((h << 5) + (h >> 2) + x[l1-1]);
253
+ }
254
+ return h;
255
+ }
256
+
257
+
258
+ inline int32_t DoubleToInt32(double d) {
259
+ int32_t i = static_cast<int32_t>(d);
260
+ if (static_cast<double>(i) == d) {
261
+ return i;
262
+ }
263
+ if (!std::isfinite(d) || d == 0) {
264
+ return 0;
265
+ }
266
+ if (d < 0 || d >= Conversions::DoubleToInt32_Two32) {
267
+ d = std::fmod(d, Conversions::DoubleToInt32_Two32);
268
+ }
269
+ d = (d >= 0) ?
270
+ std::floor(d) : std::ceil(d) + Conversions::DoubleToInt32_Two32;
271
+ return static_cast<int32_t>(d >= Conversions::DoubleToInt32_Two31 ?
272
+ d - Conversions::DoubleToInt32_Two32 : d);
273
+ }
274
+
275
+ inline int32_t DoubleToUInt32(double d) {
276
+ return static_cast<uint32_t>(d);
277
+ }
278
+
279
+ inline bool ConvertToUInt32(const UStringPiece& str, uint32_t* value) {
280
+ uint16_t ch;
281
+ *value = 0;
282
+ UStringPiece::const_iterator it = str.begin();
283
+ const UStringPiece::const_iterator last = str.end();
284
+ if (it != last && *it != '0' && Chars::IsDecimalDigit(*it)) {
285
+ ch = *it - '0';
286
+ *value = ch;
287
+ } else {
288
+ return false;
289
+ }
290
+ ++it;
291
+ uint32_t prev = *value;
292
+ for (;it != last; ++it) {
293
+ prev = *value;
294
+ if (Chars::IsDecimalDigit(*it)) {
295
+ ch = *it - '0';
296
+ *value = ch + (prev * 10);
297
+ } else {
298
+ return false;
299
+ }
300
+ }
301
+ return (prev < (std::numeric_limits<uint32_t>::max() / 10) ||
302
+ ((prev == (std::numeric_limits<uint32_t>::max() / 10)) &&
303
+ (ch < (std::numeric_limits<uint32_t>::max() % 10))));
304
+ }
305
+
306
+
307
+ } } // namespace iv::core
308
+ #endif // IV_CONVERSIONS_H_
@@ -0,0 +1,20 @@
1
+ #ifndef IV_LV5_DTOA_H_
2
+ #define IV_LV5_DTOA_H_
3
+ // Double to String
4
+
5
+ // David M Gay's algorithm and V8 fast-dtoa
6
+
7
+ namespace v8 {
8
+ namespace internal {
9
+ // Printing floating-point numbers quickly and accurately with integers.
10
+ // Florian Loitsch, PLDI 2010.
11
+ extern char* DoubleToCString(double v, char* buffer, int buflen);
12
+ } } // namespace v8::internal
13
+
14
+ namespace iv {
15
+ namespace core {
16
+
17
+ using v8::internal::DoubleToCString;
18
+
19
+ } } // namespace iv::lv5
20
+ #endif // IV_LV5_DTOA_H_
@@ -0,0 +1,18 @@
1
+ #ifndef _IV_ENABLE_IF_H_
2
+ #define _IV_ENABLE_IF_H_
3
+ namespace iv {
4
+
5
+ template<bool B, class T = void>
6
+ struct enable_if_c {
7
+ typedef T type;
8
+ };
9
+ template<class T>
10
+ struct enable_if_c<false, T> {
11
+ // NO TYPEDEF!
12
+ };
13
+ template<class Cond, class T = void>
14
+ struct enable_if : public enable_if_c<Cond::value, T> {
15
+ };
16
+
17
+ } // namespace iv
18
+ #endif // _IV_ENABLE_IF_H_
@@ -0,0 +1,15 @@
1
+ #ifndef _IV_ERRORS_H_
2
+ #define _IV_ERRORS_H_
3
+ namespace iv {
4
+ namespace core {
5
+
6
+ class Errors {
7
+ public:
8
+ enum Type {
9
+ kILLEGAL = 0,
10
+ kLEFTHANDSIDE = 0,
11
+ };
12
+ };
13
+
14
+ } } // namespace iv::core
15
+ #endif // _IV_ERRORS_H_
@@ -0,0 +1,42 @@
1
+ #ifndef IV_FIXEDCONTAINE_H_
2
+ #define IV_FIXEDCONTAINE_H_
3
+ #include <cstddef>
4
+ #include <iterator>
5
+ #include <tr1/type_traits>
6
+ namespace iv {
7
+ namespace core {
8
+
9
+ template<typename T>
10
+ class FixedContainer {
11
+ public:
12
+ typedef T value_type;
13
+ typedef typename std::tr1::add_pointer<T>::type pointer;
14
+ typedef pointer iterator;
15
+ typedef typename std::tr1::add_const<iterator>::type const_iterator;
16
+ typedef typename std::tr1::add_reference<T>::type reference;
17
+ typedef typename std::tr1::add_const<reference>::type const_reference;
18
+ typedef std::reverse_iterator<iterator> reverse_iterator;
19
+ typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
20
+ typedef std::ptrdiff_t difference_type;
21
+ typedef std::size_t size_type;
22
+ FixedContainer(pointer buffer, size_type size)
23
+ : buf_(buffer),
24
+ size_(size) { }
25
+ // iterator begin() { return buf_; }
26
+ // iterator end() { return buf_+size_; }
27
+ const_iterator begin() const { return buf_; }
28
+ const_iterator end() const { return buf_+size_; }
29
+ size_type size() const { return size_; }
30
+ reference operator[](std::size_t n) {
31
+ return *(buf_+n);
32
+ }
33
+ const_reference operator[](std::size_t n) const {
34
+ return *(buf_+n);
35
+ }
36
+ protected:
37
+ pointer buf_;
38
+ size_type size_;
39
+ };
40
+
41
+ } } // namespace iv::core
42
+ #endif // IV_FIXEDCONTAINER_H_
@@ -0,0 +1,29 @@
1
+ #ifndef _IV_FUNCTOR_H_
2
+ #define _IV_FUNCTOR_H_
3
+ #include <functional>
4
+ namespace iv {
5
+ namespace core {
6
+
7
+ template<typename T>
8
+ struct Deleter : public std::unary_function<T, void> {
9
+ void operator()(T* p) const {
10
+ delete p;
11
+ }
12
+ };
13
+
14
+ template<typename T>
15
+ struct Deleter<T[]> : public std::unary_function<T[], void> {
16
+ void operator()(T* p) const {
17
+ delete [] p;
18
+ }
19
+ };
20
+
21
+ template<typename T>
22
+ struct Destructor : public std::unary_function<T, void> {
23
+ void operator()(T* p) const {
24
+ p->~T();
25
+ }
26
+ };
27
+
28
+ } } // namespace iv::core
29
+ #endif // _IV_FUNCTOR_H_