iv-phonic 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. data/.autotest +24 -0
  2. data/Manifest.txt +49 -0
  3. data/README.rdoc +32 -0
  4. data/Rakefile +54 -0
  5. data/ext/include/iv/algorithm.h +23 -0
  6. data/ext/include/iv/alloc.h +200 -0
  7. data/ext/include/iv/any.h +71 -0
  8. data/ext/include/iv/ast-factory.h +277 -0
  9. data/ext/include/iv/ast-fwd.h +92 -0
  10. data/ext/include/iv/ast-serializer.h +579 -0
  11. data/ext/include/iv/ast-visitor.h +121 -0
  12. data/ext/include/iv/ast.h +1127 -0
  13. data/ext/include/iv/chars.h +83 -0
  14. data/ext/include/iv/cmdline.h +830 -0
  15. data/ext/include/iv/conversions.h +308 -0
  16. data/ext/include/iv/dtoa.h +20 -0
  17. data/ext/include/iv/enable_if.h +18 -0
  18. data/ext/include/iv/errors.h +15 -0
  19. data/ext/include/iv/fixedcontainer.h +42 -0
  20. data/ext/include/iv/functor.h +29 -0
  21. data/ext/include/iv/lexer.h +1281 -0
  22. data/ext/include/iv/location.h +23 -0
  23. data/ext/include/iv/mt19937.h +175 -0
  24. data/ext/include/iv/noncopyable.h +30 -0
  25. data/ext/include/iv/none.h +10 -0
  26. data/ext/include/iv/parser.h +2150 -0
  27. data/ext/include/iv/source.h +27 -0
  28. data/ext/include/iv/space.h +178 -0
  29. data/ext/include/iv/static_assert.h +30 -0
  30. data/ext/include/iv/stringpiece.h +385 -0
  31. data/ext/include/iv/token.h +311 -0
  32. data/ext/include/iv/ucdata.h +58 -0
  33. data/ext/include/iv/uchar.h +8 -0
  34. data/ext/include/iv/ustring.h +28 -0
  35. data/ext/include/iv/ustringpiece.h +9 -0
  36. data/ext/include/iv/utils.h +83 -0
  37. data/ext/include/iv/xorshift.h +74 -0
  38. data/ext/iv/phonic/ast-fwd.h +21 -0
  39. data/ext/iv/phonic/ast.h +10 -0
  40. data/ext/iv/phonic/creator.h +530 -0
  41. data/ext/iv/phonic/encoding.h +110 -0
  42. data/ext/iv/phonic/extconf.rb +5 -0
  43. data/ext/iv/phonic/factory.h +247 -0
  44. data/ext/iv/phonic/parser.h +12 -0
  45. data/ext/iv/phonic/phonic.cc +69 -0
  46. data/ext/iv/phonic/rnode.h +15 -0
  47. data/ext/iv/phonic/rparser.h +48 -0
  48. data/ext/iv/phonic/source.h +146 -0
  49. data/test/test_iv_phonic.rb +32 -0
  50. metadata +159 -0
@@ -0,0 +1,311 @@
1
+ #ifndef _IV_TOKEN_H_
2
+ #define _IV_TOKEN_H_
3
+ #include <cstddef>
4
+ #include <cassert>
5
+ #include "none.h"
6
+ namespace iv {
7
+ namespace core {
8
+ namespace detail {
9
+ template<typename T>
10
+ struct TokenContents {
11
+ static const char* kContents[];
12
+ };
13
+ } // namespace iv::core::detail
14
+ typedef detail::TokenContents<None> TokenContents;
15
+
16
+ class Token {
17
+ public:
18
+ enum Type {
19
+ EOS, // EOS
20
+ ILLEGAL, // ILLEGAL
21
+
22
+ PERIOD, // .
23
+ COLON, // :
24
+ SEMICOLON, // ;
25
+ COMMA, // ,
26
+
27
+ LPAREN, // (
28
+ RPAREN, // )
29
+ LBRACK, // [
30
+ RBRACK, // ]
31
+ LBRACE, // {
32
+ RBRACE, // }
33
+
34
+ CONDITIONAL, // ?
35
+
36
+ EQ, // ==
37
+ EQ_STRICT, // ===
38
+
39
+ NOT, // !
40
+ NE, // !=
41
+ NE_STRICT, // !==
42
+
43
+ INC, // ++
44
+ DEC, // --
45
+
46
+ ADD, // +
47
+ SUB, // -
48
+ MUL, // *
49
+ DIV, // /
50
+ MOD, // %
51
+
52
+ REL_FIRST, // RELATIONAL FIRST
53
+ LT, // <
54
+ GT, // >
55
+ LTE, // <=
56
+ GTE, // >=
57
+ INSTANCEOF, // instanceof
58
+ REL_LAST, // RELATIONAL LAST
59
+
60
+ SAR, // >>
61
+ SHR, // >>>
62
+ SHL, // <<
63
+
64
+ BIT_AND, // &
65
+ BIT_OR, // |
66
+ BIT_XOR, // ^
67
+ BIT_NOT, // ~
68
+
69
+ LOGICAL_AND, // &&
70
+ LOGICAL_OR, // ||
71
+
72
+ HTML_COMMENT, // <!--
73
+
74
+ ASSIGN_FIRST, // ASSIGN OP FIRST
75
+ ASSIGN, // =
76
+ ASSIGN_ADD, // +=
77
+ ASSIGN_SUB, // -=
78
+ ASSIGN_MUL, // *=
79
+ ASSIGN_MOD, // %=
80
+ ASSIGN_DIV, // /=
81
+ ASSIGN_SAR, // >>=
82
+ ASSIGN_SHR, // >>>=
83
+ ASSIGN_SHL, // <<=
84
+ ASSIGN_BIT_AND, // &=
85
+ ASSIGN_BIT_OR, // |=
86
+ ASSIGN_LAST, // ASSIGN OP LAST
87
+
88
+ DELETE, // delete
89
+ TYPEOF, // typeof
90
+ VOID, // void
91
+ BREAK, // break
92
+ CASE, // case
93
+ CATCH, // catch
94
+ CONTINUE, // continue
95
+ DEBUGGER, // debugger
96
+ DEFAULT, // default
97
+ DO, // do
98
+ ELSE, // else
99
+ FINALLY, // finaly
100
+ FOR, // for
101
+ FUNCTION, // function
102
+ IF, // if
103
+ IN, // in
104
+ NEW, // new
105
+ RETURN, // return
106
+ SWITCH, // switch
107
+ THIS, // this
108
+ THROW, // throw
109
+ TRY, // try
110
+ VAR, // var
111
+ WHILE, // while
112
+ WITH, // with
113
+
114
+ ABSTRACT, // abstract
115
+ BOOLEAN, // boolean
116
+ BYTE, // byte
117
+ CHAR, // char
118
+ CLASS, // class
119
+ CONST, // const
120
+ DOUBLE, // double
121
+ ENUM, // enum
122
+ EXPORT, // export
123
+ EXTENDS, // extends
124
+ FINAL, // final
125
+ FLOAT, // float
126
+ GOTO, // goto
127
+ IMPLEMENTS, // implements
128
+ IMPORT, // import
129
+ INT, // int
130
+ INTERFACE, // interface
131
+ LONG, // long
132
+ NATIVE, // native
133
+ PACKAGE, // package
134
+ PRIVATE, // private
135
+ PROTECTED, // protected
136
+ PUBLIC, // public
137
+ SHORT, // short
138
+ STATIC, // static
139
+ SUPER, // super
140
+ SYNCHRONIZED, // synchronized
141
+ THROWS, // throws
142
+ TRANSIENT, // transient
143
+ VOLATILE, // volatile
144
+ LET, // let
145
+ YIELD, // yield
146
+
147
+ GET, // get
148
+ SET, // set
149
+
150
+ NULL_LITERAL, // NULL LITERAL
151
+ FALSE_LITERAL, // FALSE LITERAL
152
+ TRUE_LITERAL, // TRUE LITERAL
153
+ NUMBER, // NUMBER LITERAL
154
+ STRING, // STRING LITERAL
155
+
156
+ IDENTIFIER, // IDENTIFIER
157
+
158
+ NOT_FOUND, // NOT_FOUND
159
+
160
+ NUM_TOKENS // number of tokens
161
+ };
162
+ static const std::size_t kMaxSize = 12; // "synchronized"
163
+
164
+ static inline bool IsAssignOp(Token::Type type) {
165
+ return Token::ASSIGN_FIRST < type && type < Token::ASSIGN_LAST;
166
+ }
167
+ static inline const char* ToString(Token::Type type) {
168
+ assert(0 <= type && type < NUM_TOKENS);
169
+ assert(type != Token::ASSIGN_FIRST &&
170
+ type != Token::ASSIGN_LAST &&
171
+ type != Token::REL_FIRST &&
172
+ type != Token::REL_LAST &&
173
+ type != Token::STRING &&
174
+ type != Token::NUMBER &&
175
+ type != Token::IDENTIFIER &&
176
+ type != Token::EOS &&
177
+ type != Token::ILLEGAL &&
178
+ type != Token::NOT_FOUND);
179
+ return TokenContents::kContents[type];
180
+ }
181
+ };
182
+
183
+ namespace detail {
184
+ template<typename T>
185
+ const char* TokenContents<T>::kContents[Token::NUM_TOKENS] = {
186
+ NULL, // EOS
187
+ NULL, // ILLEGAL
188
+ ".",
189
+ ":",
190
+ ";",
191
+ ",",
192
+ "(",
193
+ ")",
194
+ "[",
195
+ "]",
196
+ "{",
197
+ "}",
198
+ "?",
199
+ "==",
200
+ "===",
201
+ "!",
202
+ "!=",
203
+ "!==",
204
+ "++",
205
+ "--",
206
+ "+",
207
+ "-",
208
+ "*",
209
+ "/",
210
+ "%",
211
+ NULL, // RELATIONAL FIRST
212
+ "<",
213
+ ">",
214
+ "<=",
215
+ ">=",
216
+ "instanceof",
217
+ NULL, // RELATIONAL LAST
218
+ ">>",
219
+ ">>>",
220
+ "<<",
221
+ "&",
222
+ "|",
223
+ "^",
224
+ "~",
225
+ "&&",
226
+ "||",
227
+ "<!--",
228
+ NULL, // ASSIGN OP FIRST
229
+ "=",
230
+ "+=",
231
+ "-=",
232
+ "*=",
233
+ "%=",
234
+ "/=",
235
+ ">>=",
236
+ ">>>=",
237
+ "<<=",
238
+ "&=",
239
+ "|=",
240
+ NULL, // ASSIGN OP LAST
241
+ "delete",
242
+ "typeof",
243
+ "void",
244
+ "break",
245
+ "case",
246
+ "catch",
247
+ "continue",
248
+ "debugger",
249
+ "default",
250
+ "do",
251
+ "else",
252
+ "finaly",
253
+ "for",
254
+ "function",
255
+ "if",
256
+ "in",
257
+ "new",
258
+ "return",
259
+ "switch",
260
+ "this",
261
+ "throw",
262
+ "try",
263
+ "var",
264
+ "while",
265
+ "with",
266
+ "abstract",
267
+ "boolean",
268
+ "byte",
269
+ "char",
270
+ "class",
271
+ "const",
272
+ "double",
273
+ "enum",
274
+ "export",
275
+ "extends",
276
+ "final",
277
+ "float",
278
+ "goto",
279
+ "implements",
280
+ "import",
281
+ "int",
282
+ "interface",
283
+ "long",
284
+ "native",
285
+ "package",
286
+ "private",
287
+ "protected",
288
+ "public",
289
+ "short",
290
+ "static",
291
+ "super",
292
+ "synchronized",
293
+ "throws",
294
+ "transient",
295
+ "volatile",
296
+ "let",
297
+ "yield",
298
+ "get",
299
+ "set",
300
+ "null",
301
+ "false",
302
+ "true",
303
+ NULL, // NUMBER LITERAL
304
+ NULL, // STRING LITERAL
305
+ NULL, // IDENTIFIER
306
+ NULL // NOT FOUND
307
+ };
308
+
309
+ } // namespace iv::core::detail
310
+ } } // namespace iv::core
311
+ #endif // _IV_TOKEN_H_