iv-phonic 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.autotest +24 -0
- data/Manifest.txt +49 -0
- data/README.rdoc +32 -0
- data/Rakefile +54 -0
- data/ext/include/iv/algorithm.h +23 -0
- data/ext/include/iv/alloc.h +200 -0
- data/ext/include/iv/any.h +71 -0
- data/ext/include/iv/ast-factory.h +277 -0
- data/ext/include/iv/ast-fwd.h +92 -0
- data/ext/include/iv/ast-serializer.h +579 -0
- data/ext/include/iv/ast-visitor.h +121 -0
- data/ext/include/iv/ast.h +1127 -0
- data/ext/include/iv/chars.h +83 -0
- data/ext/include/iv/cmdline.h +830 -0
- data/ext/include/iv/conversions.h +308 -0
- data/ext/include/iv/dtoa.h +20 -0
- data/ext/include/iv/enable_if.h +18 -0
- data/ext/include/iv/errors.h +15 -0
- data/ext/include/iv/fixedcontainer.h +42 -0
- data/ext/include/iv/functor.h +29 -0
- data/ext/include/iv/lexer.h +1281 -0
- data/ext/include/iv/location.h +23 -0
- data/ext/include/iv/mt19937.h +175 -0
- data/ext/include/iv/noncopyable.h +30 -0
- data/ext/include/iv/none.h +10 -0
- data/ext/include/iv/parser.h +2150 -0
- data/ext/include/iv/source.h +27 -0
- data/ext/include/iv/space.h +178 -0
- data/ext/include/iv/static_assert.h +30 -0
- data/ext/include/iv/stringpiece.h +385 -0
- data/ext/include/iv/token.h +311 -0
- data/ext/include/iv/ucdata.h +58 -0
- data/ext/include/iv/uchar.h +8 -0
- data/ext/include/iv/ustring.h +28 -0
- data/ext/include/iv/ustringpiece.h +9 -0
- data/ext/include/iv/utils.h +83 -0
- data/ext/include/iv/xorshift.h +74 -0
- data/ext/iv/phonic/ast-fwd.h +21 -0
- data/ext/iv/phonic/ast.h +10 -0
- data/ext/iv/phonic/creator.h +530 -0
- data/ext/iv/phonic/encoding.h +110 -0
- data/ext/iv/phonic/extconf.rb +5 -0
- data/ext/iv/phonic/factory.h +247 -0
- data/ext/iv/phonic/parser.h +12 -0
- data/ext/iv/phonic/phonic.cc +69 -0
- data/ext/iv/phonic/rnode.h +15 -0
- data/ext/iv/phonic/rparser.h +48 -0
- data/ext/iv/phonic/source.h +146 -0
- data/test/test_iv_phonic.rb +32 -0
- metadata +159 -0
@@ -0,0 +1,308 @@
|
|
1
|
+
#ifndef IV_CONVERSIONS_H_
|
2
|
+
#define IV_CONVERSIONS_H_
|
3
|
+
#include <cstdio>
|
4
|
+
#include <cmath>
|
5
|
+
#include <string>
|
6
|
+
#include <limits>
|
7
|
+
#include <tr1/array>
|
8
|
+
#include <tr1/cstdint>
|
9
|
+
#include "chars.h"
|
10
|
+
#include "dtoa.h"
|
11
|
+
#include "ustringpiece.h"
|
12
|
+
#include "none.h"
|
13
|
+
namespace iv {
|
14
|
+
namespace core {
|
15
|
+
namespace detail {
|
16
|
+
template<typename T>
|
17
|
+
class Conversions {
|
18
|
+
public:
|
19
|
+
static const double kNaN;
|
20
|
+
static const int kMaxSignificantDigits = 772;
|
21
|
+
static const std::string kInfinity;
|
22
|
+
static const double DoubleToInt32_Two32;
|
23
|
+
static const double DoubleToInt32_Two31;
|
24
|
+
};
|
25
|
+
template<typename T>
|
26
|
+
const double Conversions<T>::kNaN = std::numeric_limits<double>::quiet_NaN();
|
27
|
+
|
28
|
+
template<typename T>
|
29
|
+
const std::string Conversions<T>::kInfinity = "Infinity";
|
30
|
+
|
31
|
+
template<typename T>
|
32
|
+
const double Conversions<T>::DoubleToInt32_Two32 = 4294967296.0;
|
33
|
+
|
34
|
+
template<typename T>
|
35
|
+
const double Conversions<T>::DoubleToInt32_Two31 = 2147483648.0;
|
36
|
+
} // namespace iv::core::detail
|
37
|
+
|
38
|
+
typedef detail::Conversions<None> Conversions;
|
39
|
+
|
40
|
+
template<typename Iter>
|
41
|
+
inline double StringToDouble(Iter it, Iter last) {
|
42
|
+
bool is_decimal = true;
|
43
|
+
bool is_signed = false;
|
44
|
+
std::size_t pos = 0;
|
45
|
+
int significant_digits = 0;
|
46
|
+
int insignificant_digits = 0;
|
47
|
+
std::tr1::array<char, Conversions::kMaxSignificantDigits+10> buffer;
|
48
|
+
|
49
|
+
// empty string ""
|
50
|
+
if (it == last) {
|
51
|
+
return 0;
|
52
|
+
}
|
53
|
+
|
54
|
+
while (it != last &&
|
55
|
+
(Chars::IsWhiteSpace(*it) || Chars::IsLineTerminator(*it))) {
|
56
|
+
++it;
|
57
|
+
}
|
58
|
+
|
59
|
+
// white space only " "
|
60
|
+
if (it == last) {
|
61
|
+
return 0;
|
62
|
+
}
|
63
|
+
|
64
|
+
if (*it == '-') {
|
65
|
+
buffer[pos++] = '-';
|
66
|
+
++it;
|
67
|
+
is_signed = true;
|
68
|
+
} else if (*it == '+') {
|
69
|
+
++it;
|
70
|
+
}
|
71
|
+
|
72
|
+
if (it == last) {
|
73
|
+
return Conversions::kNaN;
|
74
|
+
}
|
75
|
+
|
76
|
+
if (Chars::IsDecimalDigit(*it)) {
|
77
|
+
if (*it == '0') {
|
78
|
+
++it;
|
79
|
+
if (it == last) {
|
80
|
+
return 0;
|
81
|
+
}
|
82
|
+
if (*it == 'x' || *it == 'X') {
|
83
|
+
is_decimal = false;
|
84
|
+
buffer[pos++] = '0';
|
85
|
+
buffer[pos++] = *it;
|
86
|
+
++it;
|
87
|
+
++significant_digits;
|
88
|
+
if (it == last || !Chars::IsHexDigit(*it)) {
|
89
|
+
return Conversions::kNaN;
|
90
|
+
}
|
91
|
+
// waste leading zero
|
92
|
+
while (it != last && *it == '0') {
|
93
|
+
++it;
|
94
|
+
}
|
95
|
+
while (it != last && Chars::IsHexDigit(*it)) {
|
96
|
+
if (significant_digits < Conversions::kMaxSignificantDigits) {
|
97
|
+
buffer[pos++] = *it;
|
98
|
+
++it;
|
99
|
+
++significant_digits;
|
100
|
+
} else {
|
101
|
+
++it;
|
102
|
+
}
|
103
|
+
}
|
104
|
+
} else {
|
105
|
+
// waste leading zero
|
106
|
+
while (it != last && *it == '0') {
|
107
|
+
++it;
|
108
|
+
}
|
109
|
+
}
|
110
|
+
}
|
111
|
+
if (is_decimal) {
|
112
|
+
while (it != last &&
|
113
|
+
Chars::IsDecimalDigit(*it)) {
|
114
|
+
if (significant_digits < Conversions::kMaxSignificantDigits) {
|
115
|
+
buffer[pos++] = *it;
|
116
|
+
++significant_digits;
|
117
|
+
} else {
|
118
|
+
++insignificant_digits;
|
119
|
+
}
|
120
|
+
++it;
|
121
|
+
}
|
122
|
+
if (it != last && *it == '.') {
|
123
|
+
buffer[pos++] = '.';
|
124
|
+
++it;
|
125
|
+
while (it != last &&
|
126
|
+
Chars::IsDecimalDigit(*it)) {
|
127
|
+
if (significant_digits < Conversions::kMaxSignificantDigits) {
|
128
|
+
buffer[pos++] = *it;
|
129
|
+
++significant_digits;
|
130
|
+
}
|
131
|
+
++it;
|
132
|
+
}
|
133
|
+
}
|
134
|
+
}
|
135
|
+
} else {
|
136
|
+
if (*it == '.') {
|
137
|
+
buffer[pos++] = '.';
|
138
|
+
++it;
|
139
|
+
while (it != last &&
|
140
|
+
Chars::IsDecimalDigit(*it)) {
|
141
|
+
if (significant_digits < Conversions::kMaxSignificantDigits) {
|
142
|
+
buffer[pos++] = *it;
|
143
|
+
++significant_digits;
|
144
|
+
}
|
145
|
+
++it;
|
146
|
+
}
|
147
|
+
} else {
|
148
|
+
for (std::string::const_iterator inf_it = Conversions::kInfinity.begin(),
|
149
|
+
inf_last = Conversions::kInfinity.end();
|
150
|
+
inf_it != inf_last; ++inf_it, ++it) {
|
151
|
+
if (it == last ||
|
152
|
+
(*inf_it) != (*it)) {
|
153
|
+
return Conversions::kNaN;
|
154
|
+
}
|
155
|
+
}
|
156
|
+
// infinity
|
157
|
+
while (it != last &&
|
158
|
+
(Chars::IsWhiteSpace(*it) || Chars::IsLineTerminator(*it))) {
|
159
|
+
++it;
|
160
|
+
}
|
161
|
+
if (it == last) {
|
162
|
+
if (is_signed) {
|
163
|
+
return -std::numeric_limits<double>::infinity();
|
164
|
+
} else {
|
165
|
+
return std::numeric_limits<double>::infinity();
|
166
|
+
}
|
167
|
+
} else {
|
168
|
+
return Conversions::kNaN;
|
169
|
+
}
|
170
|
+
}
|
171
|
+
}
|
172
|
+
|
173
|
+
// exponent part
|
174
|
+
if (it != last && (*it == 'e' || *it == 'E')) {
|
175
|
+
if (!is_decimal) {
|
176
|
+
return Conversions::kNaN;
|
177
|
+
}
|
178
|
+
buffer[pos++] = *it;
|
179
|
+
++it;
|
180
|
+
if (it == last) {
|
181
|
+
return Conversions::kNaN;
|
182
|
+
}
|
183
|
+
if (*it == '+' || *it == '-') {
|
184
|
+
buffer[pos++] = *it;
|
185
|
+
++it;
|
186
|
+
}
|
187
|
+
if (it == last) {
|
188
|
+
return Conversions::kNaN;
|
189
|
+
}
|
190
|
+
// more than 1 decimal digit required
|
191
|
+
if (!Chars::IsDecimalDigit(*it)) {
|
192
|
+
return Conversions::kNaN;
|
193
|
+
}
|
194
|
+
int exponent = 0;
|
195
|
+
do {
|
196
|
+
if (exponent > 9999) {
|
197
|
+
exponent = 9999;
|
198
|
+
} else {
|
199
|
+
exponent = exponent * 10 + (*it - '0');
|
200
|
+
}
|
201
|
+
++it;
|
202
|
+
} while (it != last && Chars::IsDecimalDigit(*it));
|
203
|
+
exponent+=insignificant_digits;
|
204
|
+
if (exponent > 9999) {
|
205
|
+
exponent = 9999;
|
206
|
+
}
|
207
|
+
std::snprintf(buffer.data()+pos, 5, "%d", exponent); // NOLINT
|
208
|
+
pos+=4;
|
209
|
+
}
|
210
|
+
|
211
|
+
while (it != last &&
|
212
|
+
(Chars::IsWhiteSpace(*it) || Chars::IsLineTerminator(*it))) {
|
213
|
+
++it;
|
214
|
+
}
|
215
|
+
|
216
|
+
if (it == last) {
|
217
|
+
if (pos == 0) {
|
218
|
+
// empty
|
219
|
+
return 0;
|
220
|
+
} else {
|
221
|
+
buffer[pos++] = '\0';
|
222
|
+
return std::strtod(buffer.data(), NULL);
|
223
|
+
}
|
224
|
+
} else {
|
225
|
+
return Conversions::kNaN;
|
226
|
+
}
|
227
|
+
}
|
228
|
+
|
229
|
+
inline double StringToDouble(const StringPiece& str) {
|
230
|
+
return StringToDouble(str.begin(), str.end());
|
231
|
+
}
|
232
|
+
|
233
|
+
inline double StringToDouble(const UStringPiece& str) {
|
234
|
+
return StringToDouble(str.begin(), str.end());
|
235
|
+
}
|
236
|
+
|
237
|
+
inline std::size_t StringToHash(const UStringPiece& x) {
|
238
|
+
std::size_t len = x.size();
|
239
|
+
std::size_t step = (len >> 5) + 1;
|
240
|
+
std::size_t h = 0;
|
241
|
+
for (std::size_t l1 = len; l1 >= step; l1 -= step) {
|
242
|
+
h = h ^ ((h << 5) + (h >> 2) + x[l1-1]);
|
243
|
+
}
|
244
|
+
return h;
|
245
|
+
}
|
246
|
+
|
247
|
+
inline std::size_t StringToHash(const StringPiece& x) {
|
248
|
+
std::size_t len = x.size();
|
249
|
+
std::size_t step = (len >> 5) + 1;
|
250
|
+
std::size_t h = 0;
|
251
|
+
for (std::size_t l1 = len; l1 >= step; l1 -= step) {
|
252
|
+
h = h ^ ((h << 5) + (h >> 2) + x[l1-1]);
|
253
|
+
}
|
254
|
+
return h;
|
255
|
+
}
|
256
|
+
|
257
|
+
|
258
|
+
inline int32_t DoubleToInt32(double d) {
|
259
|
+
int32_t i = static_cast<int32_t>(d);
|
260
|
+
if (static_cast<double>(i) == d) {
|
261
|
+
return i;
|
262
|
+
}
|
263
|
+
if (!std::isfinite(d) || d == 0) {
|
264
|
+
return 0;
|
265
|
+
}
|
266
|
+
if (d < 0 || d >= Conversions::DoubleToInt32_Two32) {
|
267
|
+
d = std::fmod(d, Conversions::DoubleToInt32_Two32);
|
268
|
+
}
|
269
|
+
d = (d >= 0) ?
|
270
|
+
std::floor(d) : std::ceil(d) + Conversions::DoubleToInt32_Two32;
|
271
|
+
return static_cast<int32_t>(d >= Conversions::DoubleToInt32_Two31 ?
|
272
|
+
d - Conversions::DoubleToInt32_Two32 : d);
|
273
|
+
}
|
274
|
+
|
275
|
+
inline int32_t DoubleToUInt32(double d) {
|
276
|
+
return static_cast<uint32_t>(d);
|
277
|
+
}
|
278
|
+
|
279
|
+
inline bool ConvertToUInt32(const UStringPiece& str, uint32_t* value) {
|
280
|
+
uint16_t ch;
|
281
|
+
*value = 0;
|
282
|
+
UStringPiece::const_iterator it = str.begin();
|
283
|
+
const UStringPiece::const_iterator last = str.end();
|
284
|
+
if (it != last && *it != '0' && Chars::IsDecimalDigit(*it)) {
|
285
|
+
ch = *it - '0';
|
286
|
+
*value = ch;
|
287
|
+
} else {
|
288
|
+
return false;
|
289
|
+
}
|
290
|
+
++it;
|
291
|
+
uint32_t prev = *value;
|
292
|
+
for (;it != last; ++it) {
|
293
|
+
prev = *value;
|
294
|
+
if (Chars::IsDecimalDigit(*it)) {
|
295
|
+
ch = *it - '0';
|
296
|
+
*value = ch + (prev * 10);
|
297
|
+
} else {
|
298
|
+
return false;
|
299
|
+
}
|
300
|
+
}
|
301
|
+
return (prev < (std::numeric_limits<uint32_t>::max() / 10) ||
|
302
|
+
((prev == (std::numeric_limits<uint32_t>::max() / 10)) &&
|
303
|
+
(ch < (std::numeric_limits<uint32_t>::max() % 10))));
|
304
|
+
}
|
305
|
+
|
306
|
+
|
307
|
+
} } // namespace iv::core
|
308
|
+
#endif // IV_CONVERSIONS_H_
|
@@ -0,0 +1,20 @@
|
|
1
|
+
#ifndef IV_LV5_DTOA_H_
|
2
|
+
#define IV_LV5_DTOA_H_
|
3
|
+
// Double to String
|
4
|
+
|
5
|
+
// David M Gay's algorithm and V8 fast-dtoa
|
6
|
+
|
7
|
+
namespace v8 {
|
8
|
+
namespace internal {
|
9
|
+
// Printing floating-point numbers quickly and accurately with integers.
|
10
|
+
// Florian Loitsch, PLDI 2010.
|
11
|
+
extern char* DoubleToCString(double v, char* buffer, int buflen);
|
12
|
+
} } // namespace v8::internal
|
13
|
+
|
14
|
+
namespace iv {
|
15
|
+
namespace core {
|
16
|
+
|
17
|
+
using v8::internal::DoubleToCString;
|
18
|
+
|
19
|
+
} } // namespace iv::lv5
|
20
|
+
#endif // IV_LV5_DTOA_H_
|
@@ -0,0 +1,18 @@
|
|
1
|
+
#ifndef _IV_ENABLE_IF_H_
|
2
|
+
#define _IV_ENABLE_IF_H_
|
3
|
+
namespace iv {
|
4
|
+
|
5
|
+
template<bool B, class T = void>
|
6
|
+
struct enable_if_c {
|
7
|
+
typedef T type;
|
8
|
+
};
|
9
|
+
template<class T>
|
10
|
+
struct enable_if_c<false, T> {
|
11
|
+
// NO TYPEDEF!
|
12
|
+
};
|
13
|
+
template<class Cond, class T = void>
|
14
|
+
struct enable_if : public enable_if_c<Cond::value, T> {
|
15
|
+
};
|
16
|
+
|
17
|
+
} // namespace iv
|
18
|
+
#endif // _IV_ENABLE_IF_H_
|
@@ -0,0 +1,42 @@
|
|
1
|
+
#ifndef IV_FIXEDCONTAINE_H_
|
2
|
+
#define IV_FIXEDCONTAINE_H_
|
3
|
+
#include <cstddef>
|
4
|
+
#include <iterator>
|
5
|
+
#include <tr1/type_traits>
|
6
|
+
namespace iv {
|
7
|
+
namespace core {
|
8
|
+
|
9
|
+
template<typename T>
|
10
|
+
class FixedContainer {
|
11
|
+
public:
|
12
|
+
typedef T value_type;
|
13
|
+
typedef typename std::tr1::add_pointer<T>::type pointer;
|
14
|
+
typedef pointer iterator;
|
15
|
+
typedef typename std::tr1::add_const<iterator>::type const_iterator;
|
16
|
+
typedef typename std::tr1::add_reference<T>::type reference;
|
17
|
+
typedef typename std::tr1::add_const<reference>::type const_reference;
|
18
|
+
typedef std::reverse_iterator<iterator> reverse_iterator;
|
19
|
+
typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
|
20
|
+
typedef std::ptrdiff_t difference_type;
|
21
|
+
typedef std::size_t size_type;
|
22
|
+
FixedContainer(pointer buffer, size_type size)
|
23
|
+
: buf_(buffer),
|
24
|
+
size_(size) { }
|
25
|
+
// iterator begin() { return buf_; }
|
26
|
+
// iterator end() { return buf_+size_; }
|
27
|
+
const_iterator begin() const { return buf_; }
|
28
|
+
const_iterator end() const { return buf_+size_; }
|
29
|
+
size_type size() const { return size_; }
|
30
|
+
reference operator[](std::size_t n) {
|
31
|
+
return *(buf_+n);
|
32
|
+
}
|
33
|
+
const_reference operator[](std::size_t n) const {
|
34
|
+
return *(buf_+n);
|
35
|
+
}
|
36
|
+
protected:
|
37
|
+
pointer buf_;
|
38
|
+
size_type size_;
|
39
|
+
};
|
40
|
+
|
41
|
+
} } // namespace iv::core
|
42
|
+
#endif // IV_FIXEDCONTAINER_H_
|
@@ -0,0 +1,29 @@
|
|
1
|
+
#ifndef _IV_FUNCTOR_H_
|
2
|
+
#define _IV_FUNCTOR_H_
|
3
|
+
#include <functional>
|
4
|
+
namespace iv {
|
5
|
+
namespace core {
|
6
|
+
|
7
|
+
template<typename T>
|
8
|
+
struct Deleter : public std::unary_function<T, void> {
|
9
|
+
void operator()(T* p) const {
|
10
|
+
delete p;
|
11
|
+
}
|
12
|
+
};
|
13
|
+
|
14
|
+
template<typename T>
|
15
|
+
struct Deleter<T[]> : public std::unary_function<T[], void> {
|
16
|
+
void operator()(T* p) const {
|
17
|
+
delete [] p;
|
18
|
+
}
|
19
|
+
};
|
20
|
+
|
21
|
+
template<typename T>
|
22
|
+
struct Destructor : public std::unary_function<T, void> {
|
23
|
+
void operator()(T* p) const {
|
24
|
+
p->~T();
|
25
|
+
}
|
26
|
+
};
|
27
|
+
|
28
|
+
} } // namespace iv::core
|
29
|
+
#endif // _IV_FUNCTOR_H_
|