iv-phonic 0.1.7 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gemtest +0 -0
- data/Manifest.txt +47 -47
- data/Rakefile +1 -1
- data/ext/include/iv/ast.h +0 -17
- data/ext/include/iv/ast_factory.h +0 -6
- data/ext/include/iv/ast_fwd.h +0 -1
- data/ext/include/iv/ast_serializer.h +4 -4
- data/ext/include/iv/conversions.h +1 -1
- data/ext/include/iv/lexer.h +20 -7
- data/ext/include/iv/parser.h +58 -31
- data/ext/include/iv/platform.h +18 -0
- data/ext/include/iv/unicode.h +56 -0
- data/ext/iv/phonic/factory.h +0 -8
- data/ext/iv/phonic/third_party/netlib_dtoa/netlib_dtoa.c +4 -0
- data/test/test_iv_phonic.rb +7 -12
- metadata +87 -90
- data/ext/include/iv/ucdata.h +0 -58
- data/ext/include/iv/xorshift.h +0 -74
data/.gemtest
ADDED
File without changes
|
data/Manifest.txt
CHANGED
@@ -4,73 +4,73 @@ README.rdoc
|
|
4
4
|
Rakefile
|
5
5
|
lib/iv/phonic/ast.rb
|
6
6
|
test/test_iv_phonic.rb
|
7
|
-
ext/include/iv/
|
8
|
-
ext/include/iv/
|
9
|
-
ext/include/iv/
|
10
|
-
ext/include/iv/
|
11
|
-
ext/include/iv/
|
12
|
-
ext/include/iv/
|
7
|
+
ext/include/iv/chars.h
|
8
|
+
ext/include/iv/fixedcontainer.h
|
9
|
+
ext/include/iv/ast_visitor.h
|
10
|
+
ext/include/iv/location.h
|
11
|
+
ext/include/iv/ast_serializer.h
|
12
|
+
ext/include/iv/ustringpiece.h
|
13
13
|
ext/include/iv/mt19937.h
|
14
|
-
ext/include/iv/byteorder.h
|
15
14
|
ext/include/iv/about.h
|
16
|
-
ext/include/iv/
|
17
|
-
ext/include/iv/noncopyable.h
|
15
|
+
ext/include/iv/ast_info.h
|
18
16
|
ext/include/iv/stringpiece.h
|
19
|
-
ext/include/iv/
|
17
|
+
ext/include/iv/cmdline.h
|
18
|
+
ext/include/iv/ustring.h
|
19
|
+
ext/include/iv/token.h
|
20
20
|
ext/include/iv/functor.h
|
21
|
-
ext/include/iv/
|
21
|
+
ext/include/iv/keyword.h
|
22
|
+
ext/include/iv/lexer.h
|
22
23
|
ext/include/iv/parser.h
|
23
|
-
ext/include/iv/errors.h
|
24
|
-
ext/include/iv/token.h
|
25
|
-
ext/include/iv/enable_if.h
|
26
24
|
ext/include/iv/any.h
|
27
|
-
ext/include/iv/keyword.h
|
28
|
-
ext/include/iv/cmdline.h
|
29
|
-
ext/include/iv/xorshift.h
|
30
|
-
ext/include/iv/location.h
|
31
|
-
ext/include/iv/ast_info.h
|
32
|
-
ext/include/iv/none.h
|
33
|
-
ext/include/iv/ustring.h
|
34
|
-
ext/include/iv/ast_serializer.h
|
35
|
-
ext/include/iv/utils.h
|
36
|
-
ext/include/iv/ast_visitor.h
|
37
25
|
ext/include/iv/alloc.h
|
38
|
-
ext/include/iv/
|
39
|
-
ext/include/iv/
|
26
|
+
ext/include/iv/space.h
|
27
|
+
ext/include/iv/dtoa.h
|
28
|
+
ext/include/iv/static_assert.h
|
29
|
+
ext/include/iv/conversions.h
|
30
|
+
ext/include/iv/ast_fwd.h
|
31
|
+
ext/include/iv/ast_factory.h
|
32
|
+
ext/include/iv/enable_if.h
|
33
|
+
ext/include/iv/unicode.h
|
34
|
+
ext/include/iv/noncopyable.h
|
35
|
+
ext/include/iv/uchar.h
|
40
36
|
ext/include/iv/maybe.h
|
41
|
-
ext/include/iv/ustringpiece.h
|
42
37
|
ext/include/iv/character.h
|
43
|
-
ext/include/iv/
|
38
|
+
ext/include/iv/ast.h
|
39
|
+
ext/include/iv/byteorder.h
|
40
|
+
ext/include/iv/none.h
|
41
|
+
ext/include/iv/platform.h
|
42
|
+
ext/include/iv/utils.h
|
43
|
+
ext/include/iv/errors.h
|
44
44
|
ext/iv/phonic/third_party/netlib_dtoa/SConscript
|
45
45
|
ext/iv/phonic/third_party/netlib_dtoa/netlib_dtoa.c
|
46
46
|
ext/iv/phonic/third_party/netlib_dtoa/netlib_dtoa_config.h
|
47
|
-
ext/iv/phonic/third_party/v8_dtoa/
|
48
|
-
ext/iv/phonic/third_party/v8_dtoa/conversions.h
|
49
|
-
ext/iv/phonic/third_party/v8_dtoa/dtoa.h
|
50
|
-
ext/iv/phonic/third_party/v8_dtoa/globals.h
|
51
|
-
ext/iv/phonic/third_party/v8_dtoa/checks.cc
|
52
|
-
ext/iv/phonic/third_party/v8_dtoa/platform.h
|
47
|
+
ext/iv/phonic/third_party/v8_dtoa/diy-fp.cc
|
53
48
|
ext/iv/phonic/third_party/v8_dtoa/README
|
54
|
-
ext/iv/phonic/third_party/v8_dtoa/v8-dtoa.cc
|
55
49
|
ext/iv/phonic/third_party/v8_dtoa/SConscript
|
56
|
-
ext/iv/phonic/third_party/v8_dtoa/double.h
|
57
50
|
ext/iv/phonic/third_party/v8_dtoa/include-v8.h
|
58
|
-
ext/iv/phonic/third_party/v8_dtoa/
|
51
|
+
ext/iv/phonic/third_party/v8_dtoa/diy-fp.h
|
59
52
|
ext/iv/phonic/third_party/v8_dtoa/checks.h
|
53
|
+
ext/iv/phonic/third_party/v8_dtoa/fast-dtoa.h
|
54
|
+
ext/iv/phonic/third_party/v8_dtoa/powers-ten.h
|
60
55
|
ext/iv/phonic/third_party/v8_dtoa/conversions.cc
|
61
56
|
ext/iv/phonic/third_party/v8_dtoa/utils.cc
|
62
|
-
ext/iv/phonic/third_party/v8_dtoa/
|
63
|
-
ext/iv/phonic/third_party/v8_dtoa/utils.h
|
64
|
-
ext/iv/phonic/third_party/v8_dtoa/powers-ten.h
|
57
|
+
ext/iv/phonic/third_party/v8_dtoa/v8-dtoa.cc
|
65
58
|
ext/iv/phonic/third_party/v8_dtoa/platform.cc
|
59
|
+
ext/iv/phonic/third_party/v8_dtoa/globals.h
|
60
|
+
ext/iv/phonic/third_party/v8_dtoa/checks.cc
|
61
|
+
ext/iv/phonic/third_party/v8_dtoa/dtoa.h
|
62
|
+
ext/iv/phonic/third_party/v8_dtoa/conversions.h
|
66
63
|
ext/iv/phonic/third_party/v8_dtoa/v8.h
|
67
|
-
ext/iv/phonic/third_party/v8_dtoa/
|
68
|
-
ext/iv/phonic/third_party/v8_dtoa/
|
69
|
-
ext/iv/phonic/
|
70
|
-
ext/iv/phonic/
|
71
|
-
ext/iv/phonic/
|
64
|
+
ext/iv/phonic/third_party/v8_dtoa/cached-powers.h
|
65
|
+
ext/iv/phonic/third_party/v8_dtoa/double.h
|
66
|
+
ext/iv/phonic/third_party/v8_dtoa/fast-dtoa.cc
|
67
|
+
ext/iv/phonic/third_party/v8_dtoa/platform.h
|
68
|
+
ext/iv/phonic/third_party/v8_dtoa/utils.h
|
72
69
|
ext/iv/phonic/encoding.h
|
73
|
-
ext/iv/phonic/source.h
|
74
70
|
ext/iv/phonic/phonic.cc
|
71
|
+
ext/iv/phonic/source.h
|
75
72
|
ext/iv/phonic/creator.h
|
76
|
-
ext/iv/phonic/
|
73
|
+
ext/iv/phonic/parser.h
|
74
|
+
ext/iv/phonic/factory.h
|
75
|
+
ext/iv/phonic/extconf.rb
|
76
|
+
ext/iv/phonic/ast_fwd.h
|
data/Rakefile
CHANGED
data/ext/include/iv/ast.h
CHANGED
@@ -929,23 +929,6 @@ class StringLiteral : public StringLiteralBase<Factory> {
|
|
929
929
|
value_type value_;
|
930
930
|
};
|
931
931
|
|
932
|
-
// Directivable
|
933
|
-
template<typename Factory>
|
934
|
-
class Inherit<Factory, kDirectivable>
|
935
|
-
: public StringLiteral<Factory> {
|
936
|
-
};
|
937
|
-
INHERIT(Directivable);
|
938
|
-
|
939
|
-
template<typename Factory>
|
940
|
-
class Directivable : public DirectivableBase<Factory> {
|
941
|
-
public:
|
942
|
-
explicit Directivable(const std::vector<uc16>& buffer,
|
943
|
-
Factory* factory) {
|
944
|
-
InitializeStringLiteral(buffer, factory);
|
945
|
-
}
|
946
|
-
DECLARE_NODE_TYPE(Directivable)
|
947
|
-
};
|
948
|
-
|
949
932
|
// NumberLiteral
|
950
933
|
template<typename Factory>
|
951
934
|
class Inherit<Factory, kNumberLiteral>
|
@@ -71,12 +71,6 @@ class BasicAstFactory {
|
|
71
71
|
StringLiteral(buffer, static_cast<Factory*>(this));
|
72
72
|
}
|
73
73
|
|
74
|
-
Directivable* NewDirectivable(const std::vector<uc16>& buffer,
|
75
|
-
std::size_t begin, std::size_t end) {
|
76
|
-
return new (static_cast<Factory*>(this))
|
77
|
-
Directivable(buffer, static_cast<Factory*>(this));
|
78
|
-
}
|
79
|
-
|
80
74
|
RegExpLiteral* NewRegExpLiteral(const std::vector<uc16>& content,
|
81
75
|
const std::vector<uc16>& flags,
|
82
76
|
std::size_t begin,
|
data/ext/include/iv/ast_fwd.h
CHANGED
@@ -556,21 +556,21 @@ class AstSerializer: public AstVisitor<Factory>::const_type {
|
|
556
556
|
if (val < 0x20) {
|
557
557
|
if (val < 0x10) {
|
558
558
|
Append("\\u000");
|
559
|
-
|
559
|
+
snprintf(buf, sizeof(buf), "%x", val);
|
560
560
|
Append(buf);
|
561
561
|
} else if (0x10 <= val && val < 0x20) {
|
562
562
|
Append("\\u00");
|
563
|
-
|
563
|
+
snprintf(buf, sizeof(buf), "%x", val);
|
564
564
|
Append(buf);
|
565
565
|
}
|
566
566
|
} else if (0x80 <= val) {
|
567
567
|
if (0x80 <= val && val < 0x1000) {
|
568
568
|
Append("\\u0");
|
569
|
-
|
569
|
+
snprintf(buf, sizeof(buf), "%x", val);
|
570
570
|
Append(buf);
|
571
571
|
} else if (0x1000 <= val) {
|
572
572
|
Append("\\u");
|
573
|
-
|
573
|
+
snprintf(buf, sizeof(buf), "%x", val);
|
574
574
|
Append(buf);
|
575
575
|
}
|
576
576
|
} else {
|
@@ -239,7 +239,7 @@ inline double StringToDouble(Iter it, Iter last, bool parse_float) {
|
|
239
239
|
if (exponent > 9999) {
|
240
240
|
exponent = 9999;
|
241
241
|
}
|
242
|
-
|
242
|
+
snprintf(buffer.data()+pos, 5, "%d", exponent); // NOLINT
|
243
243
|
pos+=4;
|
244
244
|
}
|
245
245
|
|
data/ext/include/iv/lexer.h
CHANGED
@@ -467,7 +467,6 @@ class Lexer: private Noncopyable<Lexer<Source> >::type {
|
|
467
467
|
|
468
468
|
bool ScanRegExpFlags() {
|
469
469
|
buffer16_.clear();
|
470
|
-
uc16 uc;
|
471
470
|
while (Chars::IsIdentifierPart(c_)) {
|
472
471
|
if (c_ == '\\') {
|
473
472
|
Advance();
|
@@ -476,7 +475,7 @@ class Lexer: private Noncopyable<Lexer<Source> >::type {
|
|
476
475
|
}
|
477
476
|
Advance();
|
478
477
|
bool ng = false;
|
479
|
-
uc = ScanHexEscape('u', 4, &ng);
|
478
|
+
const uc16 uc = ScanHexEscape('u', 4, &ng);
|
480
479
|
if (ng || uc == '\\') {
|
481
480
|
return false;
|
482
481
|
}
|
@@ -614,8 +613,6 @@ class Lexer: private Noncopyable<Lexer<Source> >::type {
|
|
614
613
|
|
615
614
|
template<typename LexType>
|
616
615
|
Token::Type ScanIdentifier(bool strict) {
|
617
|
-
uc16 uc;
|
618
|
-
|
619
616
|
buffer16_.clear();
|
620
617
|
|
621
618
|
if (c_ == '\\') {
|
@@ -625,7 +622,7 @@ class Lexer: private Noncopyable<Lexer<Source> >::type {
|
|
625
622
|
}
|
626
623
|
Advance();
|
627
624
|
bool ng = false;
|
628
|
-
uc = ScanHexEscape('u', 4, &ng);
|
625
|
+
const uc16 uc = ScanHexEscape('u', 4, &ng);
|
629
626
|
if (ng || uc == '\\' || !Chars::IsIdentifierStart(uc)) {
|
630
627
|
return Token::ILLEGAL;
|
631
628
|
}
|
@@ -642,7 +639,7 @@ class Lexer: private Noncopyable<Lexer<Source> >::type {
|
|
642
639
|
}
|
643
640
|
Advance();
|
644
641
|
bool ng = false;
|
645
|
-
uc = ScanHexEscape('u', 4, &ng);
|
642
|
+
const uc16 uc = ScanHexEscape('u', 4, &ng);
|
646
643
|
if (ng || uc == '\\' || !Chars::IsIdentifierPart(uc)) {
|
647
644
|
return Token::ILLEGAL;
|
648
645
|
}
|
@@ -695,26 +692,32 @@ class Lexer: private Noncopyable<Lexer<Source> >::type {
|
|
695
692
|
case '\\':
|
696
693
|
Record16Advance();
|
697
694
|
break;
|
695
|
+
|
698
696
|
case 'b' :
|
699
697
|
Record16('\b');
|
700
698
|
Advance();
|
701
699
|
break;
|
700
|
+
|
702
701
|
case 'f' :
|
703
702
|
Record16('\f');
|
704
703
|
Advance();
|
705
704
|
break;
|
705
|
+
|
706
706
|
case 'n' :
|
707
707
|
Record16('\n');
|
708
708
|
Advance();
|
709
709
|
break;
|
710
|
+
|
710
711
|
case 'r' :
|
711
712
|
Record16('\r');
|
712
713
|
Advance();
|
713
714
|
break;
|
715
|
+
|
714
716
|
case 't' :
|
715
717
|
Record16('\t');
|
716
718
|
Advance();
|
717
719
|
break;
|
720
|
+
|
718
721
|
case 'u' : {
|
719
722
|
Advance();
|
720
723
|
bool ng = false;
|
@@ -725,10 +728,12 @@ class Lexer: private Noncopyable<Lexer<Source> >::type {
|
|
725
728
|
Record16(uc);
|
726
729
|
break;
|
727
730
|
}
|
731
|
+
|
728
732
|
case 'v' :
|
729
733
|
Record16('\v');
|
730
734
|
Advance();
|
731
735
|
break;
|
736
|
+
|
732
737
|
case 'x' : {
|
733
738
|
Advance();
|
734
739
|
bool ng = false;
|
@@ -739,7 +744,15 @@ class Lexer: private Noncopyable<Lexer<Source> >::type {
|
|
739
744
|
Record16(uc);
|
740
745
|
break;
|
741
746
|
}
|
742
|
-
|
747
|
+
|
748
|
+
case '0' : {
|
749
|
+
if (type_ != OCTAL) {
|
750
|
+
type_ = OCTAL;
|
751
|
+
}
|
752
|
+
Record16(ScanOctalEscape());
|
753
|
+
break;
|
754
|
+
}
|
755
|
+
|
743
756
|
case '1' :
|
744
757
|
case '2' :
|
745
758
|
case '3' :
|
data/ext/include/iv/parser.h
CHANGED
@@ -293,33 +293,66 @@ class Parser
|
|
293
293
|
Statements* body,
|
294
294
|
bool *res) {
|
295
295
|
Statement* stmt;
|
296
|
-
bool recognize_directive = true;
|
297
296
|
const StrictSwitcher strict_switcher(this);
|
297
|
+
|
298
|
+
// directive prologue
|
299
|
+
{
|
300
|
+
bool octal_escaped_directive_found = false;
|
301
|
+
std::size_t line;
|
302
|
+
while (token_ != end) {
|
303
|
+
if (token_ != Token::STRING) {
|
304
|
+
// this is not directive
|
305
|
+
break;
|
306
|
+
}
|
307
|
+
const typename lexer_type::State state = lexer_.StringEscapeType();
|
308
|
+
if (!octal_escaped_directive_found &&
|
309
|
+
state == lexer_type::OCTAL) {
|
310
|
+
// octal escaped string literal
|
311
|
+
octal_escaped_directive_found = true;
|
312
|
+
line = lexer_.line_number();
|
313
|
+
}
|
314
|
+
stmt = ParseStatement(CHECK);
|
315
|
+
body->push_back(stmt);
|
316
|
+
if (stmt->AsExpressionStatement() &&
|
317
|
+
stmt->AsExpressionStatement()->expr()->AsStringLiteral()) {
|
318
|
+
Expression* const expr = stmt->AsExpressionStatement()->expr();
|
319
|
+
// expression is directive
|
320
|
+
if (!strict_switcher.IsStrict() &&
|
321
|
+
state == lexer_type::NONE &&
|
322
|
+
expr->AsStringLiteral()->value().compare(
|
323
|
+
ParserData::kUseStrict.data()) == 0) {
|
324
|
+
strict_switcher.SwitchStrictMode();
|
325
|
+
if (octal_escaped_directive_found) {
|
326
|
+
RAISE_WITH_NUMBER(
|
327
|
+
"octal excape sequence not allowed in strict code",
|
328
|
+
line);
|
329
|
+
}
|
330
|
+
// and one token lexed is not in strict
|
331
|
+
// so rescan
|
332
|
+
if (token_ == Token::IDENTIFIER) {
|
333
|
+
typedef detail::Keyword<IdentifyReservedWords> KeywordChecker;
|
334
|
+
token_ = KeywordChecker::Detect(lexer_.Buffer(), true);
|
335
|
+
break;
|
336
|
+
}
|
337
|
+
} else {
|
338
|
+
// other directive
|
339
|
+
}
|
340
|
+
} else {
|
341
|
+
// not directive, like
|
342
|
+
// "String", "Comma"
|
343
|
+
break;
|
344
|
+
}
|
345
|
+
}
|
346
|
+
}
|
347
|
+
|
348
|
+
// statements
|
298
349
|
while (token_ != end) {
|
299
350
|
if (token_ == Token::FUNCTION) {
|
300
351
|
// FunctionDeclaration
|
301
352
|
stmt = ParseFunctionDeclaration(CHECK);
|
302
353
|
body->push_back(stmt);
|
303
|
-
recognize_directive = false;
|
304
354
|
} else {
|
305
355
|
stmt = ParseStatement(CHECK);
|
306
|
-
// directive prologue
|
307
|
-
if (recognize_directive) {
|
308
|
-
if (stmt->AsExpressionStatement()) {
|
309
|
-
Expression* const expr = stmt->AsExpressionStatement()->expr();
|
310
|
-
if (expr->AsDirectivable()) {
|
311
|
-
// expression is directive
|
312
|
-
if (expr->AsStringLiteral()->value().compare(
|
313
|
-
ParserData::kUseStrict.data()) == 0) {
|
314
|
-
strict_switcher.SwitchStrictMode();
|
315
|
-
}
|
316
|
-
} else {
|
317
|
-
recognize_directive = false;
|
318
|
-
}
|
319
|
-
} else {
|
320
|
-
recognize_directive = false;
|
321
|
-
}
|
322
|
-
}
|
323
356
|
body->push_back(stmt);
|
324
357
|
}
|
325
358
|
}
|
@@ -1753,15 +1786,9 @@ class Parser
|
|
1753
1786
|
if (strict_ && state == lexer_type::OCTAL) {
|
1754
1787
|
RAISE("octal excape sequence not allowed in strict code");
|
1755
1788
|
}
|
1756
|
-
|
1757
|
-
|
1758
|
-
|
1759
|
-
lexer_.end_position());
|
1760
|
-
} else {
|
1761
|
-
result = factory_->NewStringLiteral(lexer_.Buffer(),
|
1762
|
-
lexer_.begin_position(),
|
1763
|
-
lexer_.end_position());
|
1764
|
-
}
|
1789
|
+
result = factory_->NewStringLiteral(lexer_.Buffer(),
|
1790
|
+
lexer_.begin_position(),
|
1791
|
+
lexer_.end_position());
|
1765
1792
|
Next();
|
1766
1793
|
break;
|
1767
1794
|
}
|
@@ -2336,9 +2363,9 @@ class Parser
|
|
2336
2363
|
void SetErrorHeader(std::size_t line) {
|
2337
2364
|
std::tr1::array<char, 40> buf;
|
2338
2365
|
error_.append(lexer_.filename());
|
2339
|
-
const int num =
|
2340
|
-
|
2341
|
-
|
2366
|
+
const int num = snprintf(buf.data(), buf.size(),
|
2367
|
+
":%lu: SyntaxError: ",
|
2368
|
+
static_cast<unsigned long>(line)); // NOLINT
|
2342
2369
|
error_.append(buf.data(), num);
|
2343
2370
|
}
|
2344
2371
|
|
@@ -0,0 +1,18 @@
|
|
1
|
+
#ifndef _IV_PLATFORM_H_
|
2
|
+
#define _IV_PLATFORM_H_
|
3
|
+
|
4
|
+
#if defined(WIN32) || defined(WIN64)
|
5
|
+
#ifndef OS_WIN
|
6
|
+
#define OS_WIN 1
|
7
|
+
#endif // OS_WIN
|
8
|
+
#elif defined(__APPLE__) || defined(__darwin__)
|
9
|
+
#ifndef OS_MACOSX
|
10
|
+
#define OS_MACOSX 1
|
11
|
+
#endif // OS_MACOSX
|
12
|
+
#elif defined(__linux__)
|
13
|
+
#ifndef OS_LINUX
|
14
|
+
#define OS_LINUX 1
|
15
|
+
#endif // OS_LINUX
|
16
|
+
#endif
|
17
|
+
|
18
|
+
#endif // _IV_PLATFORM_H_
|