iv-phonic 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (50) hide show
  1. data/.autotest +24 -0
  2. data/Manifest.txt +49 -0
  3. data/README.rdoc +32 -0
  4. data/Rakefile +54 -0
  5. data/ext/include/iv/algorithm.h +23 -0
  6. data/ext/include/iv/alloc.h +200 -0
  7. data/ext/include/iv/any.h +71 -0
  8. data/ext/include/iv/ast-factory.h +277 -0
  9. data/ext/include/iv/ast-fwd.h +92 -0
  10. data/ext/include/iv/ast-serializer.h +579 -0
  11. data/ext/include/iv/ast-visitor.h +121 -0
  12. data/ext/include/iv/ast.h +1127 -0
  13. data/ext/include/iv/chars.h +83 -0
  14. data/ext/include/iv/cmdline.h +830 -0
  15. data/ext/include/iv/conversions.h +308 -0
  16. data/ext/include/iv/dtoa.h +20 -0
  17. data/ext/include/iv/enable_if.h +18 -0
  18. data/ext/include/iv/errors.h +15 -0
  19. data/ext/include/iv/fixedcontainer.h +42 -0
  20. data/ext/include/iv/functor.h +29 -0
  21. data/ext/include/iv/lexer.h +1281 -0
  22. data/ext/include/iv/location.h +23 -0
  23. data/ext/include/iv/mt19937.h +175 -0
  24. data/ext/include/iv/noncopyable.h +30 -0
  25. data/ext/include/iv/none.h +10 -0
  26. data/ext/include/iv/parser.h +2150 -0
  27. data/ext/include/iv/source.h +27 -0
  28. data/ext/include/iv/space.h +178 -0
  29. data/ext/include/iv/static_assert.h +30 -0
  30. data/ext/include/iv/stringpiece.h +385 -0
  31. data/ext/include/iv/token.h +311 -0
  32. data/ext/include/iv/ucdata.h +58 -0
  33. data/ext/include/iv/uchar.h +8 -0
  34. data/ext/include/iv/ustring.h +28 -0
  35. data/ext/include/iv/ustringpiece.h +9 -0
  36. data/ext/include/iv/utils.h +83 -0
  37. data/ext/include/iv/xorshift.h +74 -0
  38. data/ext/iv/phonic/ast-fwd.h +21 -0
  39. data/ext/iv/phonic/ast.h +10 -0
  40. data/ext/iv/phonic/creator.h +530 -0
  41. data/ext/iv/phonic/encoding.h +110 -0
  42. data/ext/iv/phonic/extconf.rb +5 -0
  43. data/ext/iv/phonic/factory.h +247 -0
  44. data/ext/iv/phonic/parser.h +12 -0
  45. data/ext/iv/phonic/phonic.cc +69 -0
  46. data/ext/iv/phonic/rnode.h +15 -0
  47. data/ext/iv/phonic/rparser.h +48 -0
  48. data/ext/iv/phonic/source.h +146 -0
  49. data/test/test_iv_phonic.rb +32 -0
  50. metadata +159 -0
@@ -0,0 +1,308 @@
1
+ #ifndef IV_CONVERSIONS_H_
2
+ #define IV_CONVERSIONS_H_
3
+ #include <cstdio>
4
+ #include <cmath>
5
+ #include <string>
6
+ #include <limits>
7
+ #include <tr1/array>
8
+ #include <tr1/cstdint>
9
+ #include "chars.h"
10
+ #include "dtoa.h"
11
+ #include "ustringpiece.h"
12
+ #include "none.h"
13
+ namespace iv {
14
+ namespace core {
15
+ namespace detail {
16
+ template<typename T>
17
+ class Conversions {
18
+ public:
19
+ static const double kNaN;
20
+ static const int kMaxSignificantDigits = 772;
21
+ static const std::string kInfinity;
22
+ static const double DoubleToInt32_Two32;
23
+ static const double DoubleToInt32_Two31;
24
+ };
25
+ template<typename T>
26
+ const double Conversions<T>::kNaN = std::numeric_limits<double>::quiet_NaN();
27
+
28
+ template<typename T>
29
+ const std::string Conversions<T>::kInfinity = "Infinity";
30
+
31
+ template<typename T>
32
+ const double Conversions<T>::DoubleToInt32_Two32 = 4294967296.0;
33
+
34
+ template<typename T>
35
+ const double Conversions<T>::DoubleToInt32_Two31 = 2147483648.0;
36
+ } // namespace iv::core::detail
37
+
38
+ typedef detail::Conversions<None> Conversions;
39
+
40
+ template<typename Iter>
41
+ inline double StringToDouble(Iter it, Iter last) {
42
+ bool is_decimal = true;
43
+ bool is_signed = false;
44
+ std::size_t pos = 0;
45
+ int significant_digits = 0;
46
+ int insignificant_digits = 0;
47
+ std::tr1::array<char, Conversions::kMaxSignificantDigits+10> buffer;
48
+
49
+ // empty string ""
50
+ if (it == last) {
51
+ return 0;
52
+ }
53
+
54
+ while (it != last &&
55
+ (Chars::IsWhiteSpace(*it) || Chars::IsLineTerminator(*it))) {
56
+ ++it;
57
+ }
58
+
59
+ // white space only " "
60
+ if (it == last) {
61
+ return 0;
62
+ }
63
+
64
+ if (*it == '-') {
65
+ buffer[pos++] = '-';
66
+ ++it;
67
+ is_signed = true;
68
+ } else if (*it == '+') {
69
+ ++it;
70
+ }
71
+
72
+ if (it == last) {
73
+ return Conversions::kNaN;
74
+ }
75
+
76
+ if (Chars::IsDecimalDigit(*it)) {
77
+ if (*it == '0') {
78
+ ++it;
79
+ if (it == last) {
80
+ return 0;
81
+ }
82
+ if (*it == 'x' || *it == 'X') {
83
+ is_decimal = false;
84
+ buffer[pos++] = '0';
85
+ buffer[pos++] = *it;
86
+ ++it;
87
+ ++significant_digits;
88
+ if (it == last || !Chars::IsHexDigit(*it)) {
89
+ return Conversions::kNaN;
90
+ }
91
+ // waste leading zero
92
+ while (it != last && *it == '0') {
93
+ ++it;
94
+ }
95
+ while (it != last && Chars::IsHexDigit(*it)) {
96
+ if (significant_digits < Conversions::kMaxSignificantDigits) {
97
+ buffer[pos++] = *it;
98
+ ++it;
99
+ ++significant_digits;
100
+ } else {
101
+ ++it;
102
+ }
103
+ }
104
+ } else {
105
+ // waste leading zero
106
+ while (it != last && *it == '0') {
107
+ ++it;
108
+ }
109
+ }
110
+ }
111
+ if (is_decimal) {
112
+ while (it != last &&
113
+ Chars::IsDecimalDigit(*it)) {
114
+ if (significant_digits < Conversions::kMaxSignificantDigits) {
115
+ buffer[pos++] = *it;
116
+ ++significant_digits;
117
+ } else {
118
+ ++insignificant_digits;
119
+ }
120
+ ++it;
121
+ }
122
+ if (it != last && *it == '.') {
123
+ buffer[pos++] = '.';
124
+ ++it;
125
+ while (it != last &&
126
+ Chars::IsDecimalDigit(*it)) {
127
+ if (significant_digits < Conversions::kMaxSignificantDigits) {
128
+ buffer[pos++] = *it;
129
+ ++significant_digits;
130
+ }
131
+ ++it;
132
+ }
133
+ }
134
+ }
135
+ } else {
136
+ if (*it == '.') {
137
+ buffer[pos++] = '.';
138
+ ++it;
139
+ while (it != last &&
140
+ Chars::IsDecimalDigit(*it)) {
141
+ if (significant_digits < Conversions::kMaxSignificantDigits) {
142
+ buffer[pos++] = *it;
143
+ ++significant_digits;
144
+ }
145
+ ++it;
146
+ }
147
+ } else {
148
+ for (std::string::const_iterator inf_it = Conversions::kInfinity.begin(),
149
+ inf_last = Conversions::kInfinity.end();
150
+ inf_it != inf_last; ++inf_it, ++it) {
151
+ if (it == last ||
152
+ (*inf_it) != (*it)) {
153
+ return Conversions::kNaN;
154
+ }
155
+ }
156
+ // infinity
157
+ while (it != last &&
158
+ (Chars::IsWhiteSpace(*it) || Chars::IsLineTerminator(*it))) {
159
+ ++it;
160
+ }
161
+ if (it == last) {
162
+ if (is_signed) {
163
+ return -std::numeric_limits<double>::infinity();
164
+ } else {
165
+ return std::numeric_limits<double>::infinity();
166
+ }
167
+ } else {
168
+ return Conversions::kNaN;
169
+ }
170
+ }
171
+ }
172
+
173
+ // exponent part
174
+ if (it != last && (*it == 'e' || *it == 'E')) {
175
+ if (!is_decimal) {
176
+ return Conversions::kNaN;
177
+ }
178
+ buffer[pos++] = *it;
179
+ ++it;
180
+ if (it == last) {
181
+ return Conversions::kNaN;
182
+ }
183
+ if (*it == '+' || *it == '-') {
184
+ buffer[pos++] = *it;
185
+ ++it;
186
+ }
187
+ if (it == last) {
188
+ return Conversions::kNaN;
189
+ }
190
+ // more than 1 decimal digit required
191
+ if (!Chars::IsDecimalDigit(*it)) {
192
+ return Conversions::kNaN;
193
+ }
194
+ int exponent = 0;
195
+ do {
196
+ if (exponent > 9999) {
197
+ exponent = 9999;
198
+ } else {
199
+ exponent = exponent * 10 + (*it - '0');
200
+ }
201
+ ++it;
202
+ } while (it != last && Chars::IsDecimalDigit(*it));
203
+ exponent+=insignificant_digits;
204
+ if (exponent > 9999) {
205
+ exponent = 9999;
206
+ }
207
+ std::snprintf(buffer.data()+pos, 5, "%d", exponent); // NOLINT
208
+ pos+=4;
209
+ }
210
+
211
+ while (it != last &&
212
+ (Chars::IsWhiteSpace(*it) || Chars::IsLineTerminator(*it))) {
213
+ ++it;
214
+ }
215
+
216
+ if (it == last) {
217
+ if (pos == 0) {
218
+ // empty
219
+ return 0;
220
+ } else {
221
+ buffer[pos++] = '\0';
222
+ return std::strtod(buffer.data(), NULL);
223
+ }
224
+ } else {
225
+ return Conversions::kNaN;
226
+ }
227
+ }
228
+
229
+ inline double StringToDouble(const StringPiece& str) {
230
+ return StringToDouble(str.begin(), str.end());
231
+ }
232
+
233
+ inline double StringToDouble(const UStringPiece& str) {
234
+ return StringToDouble(str.begin(), str.end());
235
+ }
236
+
237
+ inline std::size_t StringToHash(const UStringPiece& x) {
238
+ std::size_t len = x.size();
239
+ std::size_t step = (len >> 5) + 1;
240
+ std::size_t h = 0;
241
+ for (std::size_t l1 = len; l1 >= step; l1 -= step) {
242
+ h = h ^ ((h << 5) + (h >> 2) + x[l1-1]);
243
+ }
244
+ return h;
245
+ }
246
+
247
+ inline std::size_t StringToHash(const StringPiece& x) {
248
+ std::size_t len = x.size();
249
+ std::size_t step = (len >> 5) + 1;
250
+ std::size_t h = 0;
251
+ for (std::size_t l1 = len; l1 >= step; l1 -= step) {
252
+ h = h ^ ((h << 5) + (h >> 2) + x[l1-1]);
253
+ }
254
+ return h;
255
+ }
256
+
257
+
258
+ inline int32_t DoubleToInt32(double d) {
259
+ int32_t i = static_cast<int32_t>(d);
260
+ if (static_cast<double>(i) == d) {
261
+ return i;
262
+ }
263
+ if (!std::isfinite(d) || d == 0) {
264
+ return 0;
265
+ }
266
+ if (d < 0 || d >= Conversions::DoubleToInt32_Two32) {
267
+ d = std::fmod(d, Conversions::DoubleToInt32_Two32);
268
+ }
269
+ d = (d >= 0) ?
270
+ std::floor(d) : std::ceil(d) + Conversions::DoubleToInt32_Two32;
271
+ return static_cast<int32_t>(d >= Conversions::DoubleToInt32_Two31 ?
272
+ d - Conversions::DoubleToInt32_Two32 : d);
273
+ }
274
+
275
+ inline int32_t DoubleToUInt32(double d) {
276
+ return static_cast<uint32_t>(d);
277
+ }
278
+
279
+ inline bool ConvertToUInt32(const UStringPiece& str, uint32_t* value) {
280
+ uint16_t ch;
281
+ *value = 0;
282
+ UStringPiece::const_iterator it = str.begin();
283
+ const UStringPiece::const_iterator last = str.end();
284
+ if (it != last && *it != '0' && Chars::IsDecimalDigit(*it)) {
285
+ ch = *it - '0';
286
+ *value = ch;
287
+ } else {
288
+ return false;
289
+ }
290
+ ++it;
291
+ uint32_t prev = *value;
292
+ for (;it != last; ++it) {
293
+ prev = *value;
294
+ if (Chars::IsDecimalDigit(*it)) {
295
+ ch = *it - '0';
296
+ *value = ch + (prev * 10);
297
+ } else {
298
+ return false;
299
+ }
300
+ }
301
+ return (prev < (std::numeric_limits<uint32_t>::max() / 10) ||
302
+ ((prev == (std::numeric_limits<uint32_t>::max() / 10)) &&
303
+ (ch < (std::numeric_limits<uint32_t>::max() % 10))));
304
+ }
305
+
306
+
307
+ } } // namespace iv::core
308
+ #endif // IV_CONVERSIONS_H_
@@ -0,0 +1,20 @@
1
+ #ifndef IV_LV5_DTOA_H_
2
+ #define IV_LV5_DTOA_H_
3
+ // Double to String
4
+
5
+ // David M Gay's algorithm and V8 fast-dtoa
6
+
7
+ namespace v8 {
8
+ namespace internal {
9
+ // Printing floating-point numbers quickly and accurately with integers.
10
+ // Florian Loitsch, PLDI 2010.
11
+ extern char* DoubleToCString(double v, char* buffer, int buflen);
12
+ } } // namespace v8::internal
13
+
14
+ namespace iv {
15
+ namespace core {
16
+
17
+ using v8::internal::DoubleToCString;
18
+
19
+ } } // namespace iv::lv5
20
+ #endif // IV_LV5_DTOA_H_
@@ -0,0 +1,18 @@
1
+ #ifndef _IV_ENABLE_IF_H_
2
+ #define _IV_ENABLE_IF_H_
3
+ namespace iv {
4
+
5
+ template<bool B, class T = void>
6
+ struct enable_if_c {
7
+ typedef T type;
8
+ };
9
+ template<class T>
10
+ struct enable_if_c<false, T> {
11
+ // NO TYPEDEF!
12
+ };
13
+ template<class Cond, class T = void>
14
+ struct enable_if : public enable_if_c<Cond::value, T> {
15
+ };
16
+
17
+ } // namespace iv
18
+ #endif // _IV_ENABLE_IF_H_
@@ -0,0 +1,15 @@
1
+ #ifndef _IV_ERRORS_H_
2
+ #define _IV_ERRORS_H_
3
+ namespace iv {
4
+ namespace core {
5
+
6
+ class Errors {
7
+ public:
8
+ enum Type {
9
+ kILLEGAL = 0,
10
+ kLEFTHANDSIDE = 0,
11
+ };
12
+ };
13
+
14
+ } } // namespace iv::core
15
+ #endif // _IV_ERRORS_H_
@@ -0,0 +1,42 @@
1
+ #ifndef IV_FIXEDCONTAINE_H_
2
+ #define IV_FIXEDCONTAINE_H_
3
+ #include <cstddef>
4
+ #include <iterator>
5
+ #include <tr1/type_traits>
6
+ namespace iv {
7
+ namespace core {
8
+
9
+ template<typename T>
10
+ class FixedContainer {
11
+ public:
12
+ typedef T value_type;
13
+ typedef typename std::tr1::add_pointer<T>::type pointer;
14
+ typedef pointer iterator;
15
+ typedef typename std::tr1::add_const<iterator>::type const_iterator;
16
+ typedef typename std::tr1::add_reference<T>::type reference;
17
+ typedef typename std::tr1::add_const<reference>::type const_reference;
18
+ typedef std::reverse_iterator<iterator> reverse_iterator;
19
+ typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
20
+ typedef std::ptrdiff_t difference_type;
21
+ typedef std::size_t size_type;
22
+ FixedContainer(pointer buffer, size_type size)
23
+ : buf_(buffer),
24
+ size_(size) { }
25
+ // iterator begin() { return buf_; }
26
+ // iterator end() { return buf_+size_; }
27
+ const_iterator begin() const { return buf_; }
28
+ const_iterator end() const { return buf_+size_; }
29
+ size_type size() const { return size_; }
30
+ reference operator[](std::size_t n) {
31
+ return *(buf_+n);
32
+ }
33
+ const_reference operator[](std::size_t n) const {
34
+ return *(buf_+n);
35
+ }
36
+ protected:
37
+ pointer buf_;
38
+ size_type size_;
39
+ };
40
+
41
+ } } // namespace iv::core
42
+ #endif // IV_FIXEDCONTAINER_H_
@@ -0,0 +1,29 @@
1
+ #ifndef _IV_FUNCTOR_H_
2
+ #define _IV_FUNCTOR_H_
3
+ #include <functional>
4
+ namespace iv {
5
+ namespace core {
6
+
7
+ template<typename T>
8
+ struct Deleter : public std::unary_function<T, void> {
9
+ void operator()(T* p) const {
10
+ delete p;
11
+ }
12
+ };
13
+
14
+ template<typename T>
15
+ struct Deleter<T[]> : public std::unary_function<T[], void> {
16
+ void operator()(T* p) const {
17
+ delete [] p;
18
+ }
19
+ };
20
+
21
+ template<typename T>
22
+ struct Destructor : public std::unary_function<T, void> {
23
+ void operator()(T* p) const {
24
+ p->~T();
25
+ }
26
+ };
27
+
28
+ } } // namespace iv::core
29
+ #endif // _IV_FUNCTOR_H_