rubycc 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +56 -0
  3. data/LICENSE.txt +21 -0
  4. data/NOTICE +52 -0
  5. data/README.md +208 -0
  6. data/data/README.md +117 -0
  7. data/data/r10_corpus_scan.json +4082 -0
  8. data/data/r10_manual_classification.json +5342 -0
  9. data/data/r10_verification_m4a.json +1531 -0
  10. data/data/r10_verification_m4b.json +1818 -0
  11. data/data/r10_verification_m4c.json +1489 -0
  12. data/data/r10_verification_m4d.json +318 -0
  13. data/data/r10_verification_rbs.json +192 -0
  14. data/data/verified_gems.json +397 -0
  15. data/exe/rmake +16 -0
  16. data/exe/rubycc +9 -0
  17. data/exe/rubycc-ar +114 -0
  18. data/exe/rubycc-doctor +14 -0
  19. data/exe/rubycc-pkgconf +8 -0
  20. data/include/float.h +87 -0
  21. data/include/iso646.h +18 -0
  22. data/include/libc/alloca.h +18 -0
  23. data/include/libc/arpa/inet.h +61 -0
  24. data/include/libc/assert.h +43 -0
  25. data/include/libc/dirent.h +72 -0
  26. data/include/libc/dlfcn.h +58 -0
  27. data/include/libc/features.h +205 -0
  28. data/include/libc/glibc/aarch64/ctype.h +108 -0
  29. data/include/libc/glibc/aarch64/endian.h +54 -0
  30. data/include/libc/glibc/aarch64/errno.h +154 -0
  31. data/include/libc/glibc/aarch64/fcntl.h +144 -0
  32. data/include/libc/glibc/aarch64/inttypes.h +182 -0
  33. data/include/libc/glibc/aarch64/limits.h +63 -0
  34. data/include/libc/glibc/aarch64/pthread.h +131 -0
  35. data/include/libc/glibc/aarch64/setjmp.h +84 -0
  36. data/include/libc/glibc/aarch64/stdint.h +174 -0
  37. data/include/libc/glibc/aarch64/sys/epoll.h +88 -0
  38. data/include/libc/glibc/aarch64/sys/fcntl.h +19 -0
  39. data/include/libc/glibc/aarch64/sys/select.h +75 -0
  40. data/include/libc/glibc/aarch64/sys/stat.h +136 -0
  41. data/include/libc/glibc/aarch64/sys/syscall.h +167 -0
  42. data/include/libc/glibc/aarch64/sys/time.h +74 -0
  43. data/include/libc/glibc/aarch64/sys/types.h +127 -0
  44. data/include/libc/glibc/aarch64/time.h +113 -0
  45. data/include/libc/glibc/x86_64/ctype.h +108 -0
  46. data/include/libc/glibc/x86_64/endian.h +54 -0
  47. data/include/libc/glibc/x86_64/errno.h +154 -0
  48. data/include/libc/glibc/x86_64/fcntl.h +134 -0
  49. data/include/libc/glibc/x86_64/inttypes.h +182 -0
  50. data/include/libc/glibc/x86_64/limits.h +61 -0
  51. data/include/libc/glibc/x86_64/pthread.h +111 -0
  52. data/include/libc/glibc/x86_64/setjmp.h +82 -0
  53. data/include/libc/glibc/x86_64/stdint.h +163 -0
  54. data/include/libc/glibc/x86_64/sys/epoll.h +91 -0
  55. data/include/libc/glibc/x86_64/sys/fcntl.h +19 -0
  56. data/include/libc/glibc/x86_64/sys/select.h +75 -0
  57. data/include/libc/glibc/x86_64/sys/stat.h +130 -0
  58. data/include/libc/glibc/x86_64/sys/syscall.h +188 -0
  59. data/include/libc/glibc/x86_64/sys/time.h +74 -0
  60. data/include/libc/glibc/x86_64/sys/types.h +123 -0
  61. data/include/libc/glibc/x86_64/time.h +113 -0
  62. data/include/libc/grp.h +50 -0
  63. data/include/libc/langinfo.h +126 -0
  64. data/include/libc/link.h +18 -0
  65. data/include/libc/locale.h +83 -0
  66. data/include/libc/math.h +191 -0
  67. data/include/libc/netinet/in.h +126 -0
  68. data/include/libc/netinet/tcp.h +44 -0
  69. data/include/libc/poll.h +46 -0
  70. data/include/libc/pwd.h +58 -0
  71. data/include/libc/regex.h +53 -0
  72. data/include/libc/sched.h +35 -0
  73. data/include/libc/signal.h +204 -0
  74. data/include/libc/stdio.h +157 -0
  75. data/include/libc/stdlib.h +92 -0
  76. data/include/libc/string.h +74 -0
  77. data/include/libc/strings.h +25 -0
  78. data/include/libc/sys/cdefs.h +126 -0
  79. data/include/libc/sys/inotify.h +111 -0
  80. data/include/libc/sys/ioctl.h +36 -0
  81. data/include/libc/sys/mman.h +65 -0
  82. data/include/libc/sys/param.h +41 -0
  83. data/include/libc/sys/resource.h +109 -0
  84. data/include/libc/sys/socket.h +167 -0
  85. data/include/libc/sys/statfs.h +82 -0
  86. data/include/libc/sys/timerfd.h +55 -0
  87. data/include/libc/sys/uio.h +40 -0
  88. data/include/libc/sys/un.h +25 -0
  89. data/include/libc/sys/utsname.h +35 -0
  90. data/include/libc/sys/wait.h +135 -0
  91. data/include/libc/termios.h +179 -0
  92. data/include/libc/unistd.h +194 -0
  93. data/include/stdalign.h +16 -0
  94. data/include/stdarg.h +31 -0
  95. data/include/stdatomic.h +158 -0
  96. data/include/stdbool.h +15 -0
  97. data/include/stdckdint.h +28 -0
  98. data/include/stddef.h +60 -0
  99. data/include/stdnoreturn.h +18 -0
  100. data/include/x86intrin.h +16 -0
  101. data/lib/rubycc/backend/aarch64.rb +1724 -0
  102. data/lib/rubycc/backend/x86_64.rb +1369 -0
  103. data/lib/rubycc/compile_error.rb +36 -0
  104. data/lib/rubycc/compiler.rb +305 -0
  105. data/lib/rubycc/doctor/builder.rb +151 -0
  106. data/lib/rubycc/doctor/cli.rb +190 -0
  107. data/lib/rubycc/doctor/fetcher.rb +84 -0
  108. data/lib/rubycc/doctor/gemfile.rb +115 -0
  109. data/lib/rubycc/doctor/verified_gems.rb +114 -0
  110. data/lib/rubycc/doctor.rb +16 -0
  111. data/lib/rubycc/driver.rb +463 -0
  112. data/lib/rubycc/front/ast.rb +528 -0
  113. data/lib/rubycc/front/constant_evaluator.rb +631 -0
  114. data/lib/rubycc/front/initializer_resolver.rb +592 -0
  115. data/lib/rubycc/front/lexeme_reader.rb +460 -0
  116. data/lib/rubycc/front/lexer.rb +232 -0
  117. data/lib/rubycc/front/parser.rb +4122 -0
  118. data/lib/rubycc/front/token.rb +49 -0
  119. data/lib/rubycc/ir/call_convention.rb +486 -0
  120. data/lib/rubycc/ir/generator.rb +6036 -0
  121. data/lib/rubycc/ir/ir.rb +417 -0
  122. data/lib/rubycc/link/compat_runtime.rb +89 -0
  123. data/lib/rubycc/link/errors.rb +13 -0
  124. data/lib/rubycc/link/executable_linker.rb +405 -0
  125. data/lib/rubycc/link/library_resolver.rb +437 -0
  126. data/lib/rubycc/link/partial_linker.rb +546 -0
  127. data/lib/rubycc/link/shared_linker.rb +1732 -0
  128. data/lib/rubycc/mkmf_shim.rb +83 -0
  129. data/lib/rubycc/objfile/ar_archive.rb +437 -0
  130. data/lib/rubycc/objfile/elf_reader.rb +646 -0
  131. data/lib/rubycc/objfile/elf_writer.rb +891 -0
  132. data/lib/rubycc/objfile/relocatable_writer.rb +376 -0
  133. data/lib/rubycc/pkgconf/cli.rb +139 -0
  134. data/lib/rubycc/pkgconf/errors.rb +39 -0
  135. data/lib/rubycc/pkgconf/model.rb +28 -0
  136. data/lib/rubycc/pkgconf/parser.rb +109 -0
  137. data/lib/rubycc/pkgconf/pkgconf.rb +15 -0
  138. data/lib/rubycc/pkgconf/resolver.rb +70 -0
  139. data/lib/rubycc/pkgconf/search_path.rb +38 -0
  140. data/lib/rubycc/pkgconf/system_path_filter.rb +131 -0
  141. data/lib/rubycc/preprocess/constant_expression.rb +219 -0
  142. data/lib/rubycc/preprocess/glibc_version.rb +124 -0
  143. data/lib/rubycc/preprocess/pp_token.rb +62 -0
  144. data/lib/rubycc/preprocess/preprocessor.rb +2020 -0
  145. data/lib/rubycc/preprocess/scanner.rb +290 -0
  146. data/lib/rubycc/preprocess/token_converter.rb +157 -0
  147. data/lib/rubycc/rmake/cli.rb +143 -0
  148. data/lib/rubycc/rmake/errors.rb +71 -0
  149. data/lib/rubycc/rmake/executor.rb +818 -0
  150. data/lib/rubycc/rmake/expander.rb +251 -0
  151. data/lib/rubycc/rmake/makefile.rb +352 -0
  152. data/lib/rubycc/rmake/model.rb +137 -0
  153. data/lib/rubycc/rmake/parser.rb +226 -0
  154. data/lib/rubycc/rmake/rmake.rb +15 -0
  155. data/lib/rubycc/type.rb +1236 -0
  156. data/lib/rubycc/version.rb +5 -0
  157. data/lib/rubycc.rb +36 -0
  158. data/lib/rubygems_plugin.rb +102 -0
  159. metadata +219 -0
@@ -0,0 +1,460 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rubycc
4
+ module Front
5
+ # Raised while decoding the interior of a single lexeme (a numeric constant,
6
+ # a string/character literal). It is deliberately position-agnostic: it only
7
+ # names the problem and the character offset (into the text handed to the
8
+ # reader) where it occurred, leaving the caller to translate that offset into
9
+ # a source line/column for a user-facing CompileError. This lets the same
10
+ # decoder serve both the streaming Lexer (offsets into the whole source) and
11
+ # the preprocessor's converter (offsets into an isolated lexeme).
12
+ class LexError < StandardError
13
+ attr_reader :offset
14
+
15
+ def initialize(message, offset)
16
+ @offset = offset
17
+ super(message)
18
+ end
19
+ end
20
+
21
+ # The shared spelling-to-value decoder for the C subset's lexemes. Both the
22
+ # streaming Front::Lexer and the preprocessor's token converter delegate here
23
+ # so the two paths fold numeric constants, resolve escapes and classify
24
+ # identifiers with byte-for-byte identical rules. The reader walks a plain
25
+ # string with its own cursor (no line/column bookkeeping); positions are the
26
+ # caller's concern. Each `read_*` leaves `pos` just past the lexeme.
27
+ class LexemeReader
28
+ # A decoded lexeme. `type` is :num, :float, :string, :ident or :keyword.
29
+ # `value` is the folded Integer/Float, the escape-resolved ASCII-8BIT bytes
30
+ # of a string, or the identifier text. `base`/`suffix` accompany an integer
31
+ # :num and `suffix` a :float, matching Front::Token's numeric fields.
32
+ Result = Struct.new(:type, :value, :base, :suffix)
33
+
34
+ KEYWORDS = %w[int char void short long signed unsigned _Bool float double struct union
35
+ enum typedef static extern const volatile inline _Noreturn register auto
36
+ return if else while do for break continue
37
+ switch case default goto sizeof
38
+ _Static_assert _Alignof _Alignas _Atomic __int128
39
+ __builtin_va_start __builtin_va_arg __builtin_va_end __builtin_va_copy
40
+ __builtin_expect __builtin_alloca __builtin_offsetof
41
+ __builtin_constant_p __builtin_choose_expr
42
+ __builtin_ctz __builtin_ctzll __builtin_clz __builtin_clzll
43
+ __builtin_unreachable __builtin_memcpy
44
+ __builtin_add_overflow __builtin_sub_overflow __builtin_mul_overflow
45
+ __atomic_load_n __atomic_store_n __atomic_exchange_n
46
+ __atomic_compare_exchange_n
47
+ __atomic_fetch_add __atomic_fetch_sub
48
+ __atomic_add_fetch __atomic_sub_fetch __atomic_or_fetch
49
+ __atomic_thread_fence
50
+ __sync_fetch_and_add __sync_fetch_and_sub
51
+ __sync_add_and_fetch __sync_sub_and_fetch __sync_or_and_fetch
52
+ __sync_lock_test_and_set __sync_lock_release __sync_synchronize
53
+ __sync_bool_compare_and_swap __sync_val_compare_and_swap
54
+ __asm__
55
+ __attribute__ __extension__].freeze
56
+
57
+ # Membership lookup for every identifier the lexer produces, so it must
58
+ # be O(1); KEYWORDS.include? showed up as a linear scan in profiling.
59
+ KEYWORD_SET = KEYWORDS.to_h { |word| [word, true] }.freeze
60
+
61
+ # gcc's reserved "__x"/"__x__" alternate spellings for a handful of
62
+ # keywords (6.10.8.4's rationale: a header built with strict-ISO options
63
+ # such as -ansi still needs the keyword's meaning without colliding with
64
+ # a user identifier of the plain name). glibc's uapi-derived headers lean
65
+ # on these unconditionally, e.g. asm-generic/int-ll64.h's
66
+ # "typedef __signed__ char __s8". Each maps straight to the plain
67
+ # keyword's own spelling, so every downstream check keyed on that
68
+ # spelling (DECL_SPECIFIER_KEYWORDS, "const"/"volatile"/"inline" in
69
+ # Front::Parser, ...) sees an ordinary keyword token and needs no
70
+ # separate case for the alias.
71
+ KEYWORD_ALIASES = {
72
+ "__signed" => "signed", "__signed__" => "signed",
73
+ "__const" => "const", "__const__" => "const",
74
+ "__volatile" => "volatile", "__volatile__" => "volatile",
75
+ "__inline" => "inline", "__inline__" => "inline"
76
+ }.freeze
77
+
78
+ # The eleven simple escape sequences of 6.4.4.4p1, shared by character
79
+ # constants and string literals, mapping the character after the backslash
80
+ # to the byte value it denotes. "\?" is the one whose escaped and plain
81
+ # spellings mean the same byte ('?'): it exists only so a source line can
82
+ # avoid accidentally spelling a trigraph, and gcc accepts it in both
83
+ # constructs. "\x" (hexadecimal, any number of digits) and octal ("\ooo",
84
+ # 1-3 digits, "0" included) are handled separately in #read_escaped_byte
85
+ # since their value comes from digits rather than a fixed table lookup.
86
+ ESCAPES = {
87
+ "n" => 10, "t" => 9, "r" => 13, "\\" => 92,
88
+ "'" => 39, "\"" => 34, "?" => 63,
89
+ "a" => 7, "b" => 8, "f" => 12, "v" => 11
90
+ }.freeze
91
+
92
+ # Three-character punctuators, matched before the shorter lists so the
93
+ # longest one always wins: "<<=" must beat "<<" (and "<="/"<"), ">>=" must
94
+ # beat ">>" (and ">="/">"), and "..." (the variadic-parameter ellipsis)
95
+ # must beat a lone "." (which is not a two-character punctuator, so ".."
96
+ # never forms; three consecutive dots are the only way "..." arises).
97
+ PUNCTUATORS_3 = %w[<<= >>= ...].freeze
98
+
99
+ # Two-character punctuators, matched before the single-character list so
100
+ # the lexer always prefers the longest punctuator ("==" over two "=",
101
+ # "&&" over two "&", "++" or "+=" over a lone "+", "->" over a lone "-",
102
+ # "<<" over two "<", "&=" over "&"/"&&").
103
+ PUNCTUATORS_2 = %w[== != <= >= && || += -= *= /= %= ++ -- -> << >> &= |= ^=].freeze
104
+
105
+ # Single-character punctuators used by this slice. "&" is both the
106
+ # address-of operator and the bitwise-and operator, "*" doubles as
107
+ # dereference and pointer-declarator marker, "|" "^" "~" are the remaining
108
+ # bitwise operators, "[" "]" bracket array declarators and subscripts,
109
+ # "?" ":" form the conditional operator, and "." selects a struct member.
110
+ PUNCTUATORS_1 = %w[+ - * / % ( ) { } ; = , < > ! & | ^ ~ [ ] ? : .].freeze
111
+
112
+ # Whether `name` (an identifier's spelling) is a reserved keyword.
113
+ def self.keyword?(name)
114
+ KEYWORD_SET.key?(name)
115
+ end
116
+
117
+ # The keyword token `name` denotes: itself when it already is one of
118
+ # KEYWORDS, or the plain spelling it aliases (see KEYWORD_ALIASES) when it
119
+ # is one of gcc's reserved "__x"/"__x__" spellings. nil when `name` names
120
+ # neither, so the caller lexes it as an ordinary identifier.
121
+ def self.keyword_spelling(name)
122
+ return name if KEYWORD_SET.key?(name)
123
+
124
+ KEYWORD_ALIASES[name]
125
+ end
126
+
127
+ attr_reader :pos
128
+
129
+ def initialize(text, start = 0)
130
+ @text = text
131
+ @pos = start
132
+ end
133
+
134
+ # An identifier or keyword run "[A-Za-z_][A-Za-z0-9_]*". The caller must
135
+ # position the cursor on the leading identifier character.
136
+ def read_identifier
137
+ name = +""
138
+ name << advance while identifier_char?(current)
139
+ spelling = LexemeReader.keyword_spelling(name)
140
+ Result.new(spelling ? :keyword : :ident, spelling || name, nil, nil)
141
+ end
142
+
143
+ # A numeric constant. A "0x"/"0X" prefix is hexadecimal (an integer, or a
144
+ # hexadecimal floating constant this subset does not lower yet); otherwise
145
+ # a "." or an exponent anywhere in the run marks a decimal floating
146
+ # constant, and its absence a decimal or octal integer. The floating check
147
+ # precedes the octal one so "08.5" reads as the float 8.5 rather than
148
+ # tripping the octal-digit rule on its '8'.
149
+ def read_number
150
+ if current == "0" && (peek == "x" || peek == "X")
151
+ read_hexadecimal_constant
152
+ elsif current == "0" && (peek == "b" || peek == "B")
153
+ read_binary_constant
154
+ elsif floating_constant_ahead?
155
+ read_floating_constant
156
+ else
157
+ read_integer_constant
158
+ end
159
+ end
160
+
161
+ # A string literal "abc" including its surrounding quotes: returns the
162
+ # escape-resolved bytes (ASCII-8BIT), without the NUL terminator the
163
+ # generator later appends. An unterminated literal (newline or end before
164
+ # the closing quote) and an unknown escape are rejected.
165
+ def read_string
166
+ start = @pos
167
+ advance # opening quote
168
+ bytes = +"".b
169
+ loop do
170
+ if at_end? || current == "\n"
171
+ raise LexError.new("unterminated string literal", start)
172
+ end
173
+ break if current == "\""
174
+
175
+ bytes << read_escaped_byte(start, "string literal")
176
+ end
177
+ advance # closing quote
178
+ Result.new(:string, bytes, nil, nil)
179
+ end
180
+
181
+ # A character constant 'c' or '\n' including its quotes: returns a :num
182
+ # result whose value is the character's byte code, since an ISO C character
183
+ # constant has type int (6.4.4.4). The constant must hold exactly one
184
+ # character; an empty '', a multi-character 'ab', an unterminated ' and an
185
+ # unknown escape are all rejected.
186
+ def read_char
187
+ start = @pos
188
+ advance # opening quote
189
+ if at_end? || current == "\n"
190
+ raise LexError.new("unterminated character constant", start)
191
+ elsif current == "'"
192
+ raise LexError.new("empty character constant", start)
193
+ end
194
+ value = read_escaped_byte(start, "character constant")
195
+ if at_end? || current == "\n"
196
+ raise LexError.new("unterminated character constant", start)
197
+ elsif current != "'"
198
+ raise LexError.new("multi-character character constant", start)
199
+ end
200
+ advance # closing quote
201
+ Result.new(:num, value, 10, "")
202
+ end
203
+
204
+ private
205
+
206
+ def at_end?
207
+ @pos >= @text.length
208
+ end
209
+
210
+ def current
211
+ @text[@pos]
212
+ end
213
+
214
+ def peek(offset = 1)
215
+ @text[@pos + offset]
216
+ end
217
+
218
+ def advance
219
+ ch = @text[@pos]
220
+ @pos += 1
221
+ ch
222
+ end
223
+
224
+ def identifier_char?(ch)
225
+ !ch.nil? && ch.match?(/[A-Za-z0-9_]/)
226
+ end
227
+
228
+ # Reads one logical character and returns its byte value, decoding a
229
+ # backslash escape via ESCAPES, a hexadecimal escape ("\x" then one or
230
+ # more hex digits) or an octal escape ("\ooo", 1-3 octal digits, 6.4.4.4).
231
+ # Shared by character constants and string literals; `context` names the
232
+ # construct for error messages and `start` locates the opening quote for
233
+ # a positioned error.
234
+ def read_escaped_byte(start, context)
235
+ ch = current
236
+ if ch.nil? || ch == "\n"
237
+ raise LexError.new("unterminated #{context}", start)
238
+ elsif ch == "\\"
239
+ advance # backslash
240
+ esc = current
241
+ if esc.nil? || esc == "\n"
242
+ raise LexError.new("unterminated #{context}", start)
243
+ elsif esc == "x"
244
+ advance
245
+ read_hex_escape(start)
246
+ elsif octal_digit?(esc)
247
+ read_octal_escape(start)
248
+ else
249
+ code = ESCAPES[esc]
250
+ raise LexError.new("unknown escape sequence in #{context}", start) unless code
251
+
252
+ advance
253
+ code
254
+ end
255
+ else
256
+ advance
257
+ ch.ord
258
+ end
259
+ end
260
+
261
+ # A "\x" escape's value: one or more hex digits (unbounded in the grammar,
262
+ # 6.4.4.4), rejected here when it exceeds a byte's range since a char or a
263
+ # string element is one byte in this implementation. At least one digit is
264
+ # required ("\x" alone is ill-formed).
265
+ def read_hex_escape(start)
266
+ digits = +""
267
+ digits << advance while hex_digit?(current)
268
+ raise LexError.new("\\x used with no following hex digits", start) if digits.empty?
269
+
270
+ value = digits.to_i(16)
271
+ raise LexError.new("hex escape sequence out of range", start) if value > 255
272
+
273
+ value
274
+ end
275
+
276
+ # An octal escape's value: 1-3 octal digits (the leading one already
277
+ # confirmed present by the caller), rejected when it exceeds a byte's
278
+ # range ("\777" is 511, past 255).
279
+ def read_octal_escape(start)
280
+ digits = +""
281
+ 3.times do
282
+ break unless octal_digit?(current)
283
+
284
+ digits << advance
285
+ end
286
+ value = digits.to_i(8)
287
+ raise LexError.new("octal escape sequence out of range", start) if value > 255
288
+
289
+ value
290
+ end
291
+
292
+ # Whether the digit run at the cursor is a decimal floating constant: it is
293
+ # exactly when a "." or an exponent marker ('e'/'E') follows the leading
294
+ # digits. Only consulted for a decimal constant (hexadecimal is split off
295
+ # first), so an 'e'/'E' here is always an exponent, never a hex digit.
296
+ def floating_constant_ahead?
297
+ i = @pos
298
+ i += 1 while @text[i]&.match?(/[0-9]/)
299
+ ch = @text[i]
300
+ ch == "." || ch == "e" || ch == "E"
301
+ end
302
+
303
+ # A hexadecimal integer constant "0x...". A '.' or a binary-exponent marker
304
+ # ('p'/'P') after the hex digits is a hexadecimal *floating* constant
305
+ # (0x1.8p3), a distinct grammar this subset does not lower yet, so it is
306
+ # diagnosed rather than silently misread as an integer.
307
+ def read_hexadecimal_constant
308
+ start = @pos
309
+ advance # 0
310
+ advance # x
311
+ digits = +""
312
+ digits << advance while hex_digit?(current)
313
+ raise LexError.new("invalid hexadecimal constant", start) if digits.empty?
314
+ if current == "." || current == "p" || current == "P"
315
+ raise LexError.new("hexadecimal floating constants are not supported yet", start)
316
+ end
317
+ suffix = read_integer_suffix(start)
318
+ if identifier_char?(current)
319
+ raise LexError.new("invalid suffix on integer constant", @pos)
320
+ end
321
+ Result.new(:num, digits.to_i(16), 16, suffix)
322
+ end
323
+
324
+ # A binary integer constant "0b...."/"0B...." (a GNU extension, made
325
+ # standard in C23): the "0b" prefix, one or more binary digits, and an
326
+ # optional integer suffix. It shares the octal/hexadecimal type rules (a
327
+ # non-decimal constant may take an unsigned type without a "u" suffix), so
328
+ # the result carries base 2 for #integer_literal_type to treat it as such.
329
+ # At least one digit is required; a trailing identifier character after the
330
+ # suffix is rejected like any other invalid suffix.
331
+ def read_binary_constant
332
+ start = @pos
333
+ advance # 0
334
+ advance # b / B
335
+ digits = +""
336
+ digits << advance while current == "0" || current == "1"
337
+ raise LexError.new("invalid binary constant", start) if digits.empty?
338
+
339
+ suffix = read_integer_suffix(start)
340
+ if identifier_char?(current)
341
+ raise LexError.new("invalid suffix on integer constant", @pos)
342
+ end
343
+ Result.new(:num, digits.to_i(2), 2, suffix)
344
+ end
345
+
346
+ # A decimal or octal integer constant, with an optional u/U and l/L/ll/LL
347
+ # suffix run. The result carries the folded value with its base and
348
+ # normalized suffix. A trailing identifier character (e.g. 12abc, once the
349
+ # real suffix letters are consumed) is rejected.
350
+ def read_integer_constant
351
+ start = @pos
352
+ if current == "0" && digit?(peek)
353
+ advance # leading 0
354
+ digits = +"0"
355
+ while digit?(current)
356
+ unless octal_digit?(current)
357
+ raise LexError.new("invalid digit in octal constant", @pos)
358
+ end
359
+ digits << advance
360
+ end
361
+ base = 8
362
+ else
363
+ digits = +""
364
+ digits << advance while digit?(current)
365
+ base = 10
366
+ end
367
+
368
+ suffix = read_integer_suffix(start)
369
+ if identifier_char?(current)
370
+ raise LexError.new("invalid suffix on integer constant", @pos)
371
+ end
372
+ Result.new(:num, digits.to_i(base), base, suffix)
373
+ end
374
+
375
+ # A decimal floating constant (6.4.4.2): an integer part, an optional
376
+ # fraction after a ".", and an optional exponent ("e"/"E" with an optional
377
+ # sign and required digits). A trailing f/F makes it `float`, l/L `long
378
+ # double` (treated as `double`), and no suffix `double`. The decimal is
379
+ # converted through Rational so a C header constant outside Ruby's Float
380
+ # range (for example `1e10000`) produces Infinity or 0.0 without Ruby's
381
+ # `-w` range warning. An exponent with no digits, and a trailing identifier
382
+ # or "." character, are rejected.
383
+ def read_floating_constant
384
+ start = @pos
385
+ text = +""
386
+ text << advance while digit?(current)
387
+ saw_fraction_digit = true
388
+ if current == "."
389
+ text << advance
390
+ saw_fraction_digit = digit?(current)
391
+ text << advance while digit?(current)
392
+ end
393
+ if current == "e" || current == "E"
394
+ text << advance
395
+ text << advance if current == "+" || current == "-"
396
+ unless digit?(current)
397
+ raise LexError.new("exponent has no digits", start)
398
+ end
399
+ text << advance while digit?(current)
400
+ end
401
+ suffix = read_floating_suffix
402
+ if !current.nil? && current.match?(/[A-Za-z0-9_.]/)
403
+ raise LexError.new("invalid suffix on floating constant", @pos)
404
+ end
405
+ # C allows a floating constant with a "." followed by no fraction
406
+ # digits before the exponent (6.4.4.2), e.g. "1.e5". Since "N." and
407
+ # "N.0" are the same number, pad the fraction with a "0" before
408
+ # Rational parses it. This only affects the conversion buffer, not the
409
+ # token's spelling (Result carries no spelling field).
410
+ conversion_text = saw_fraction_digit ? text : text.sub(".", ".0")
411
+ Result.new(:float, Rational(conversion_text).to_f, nil, suffix)
412
+ end
413
+
414
+ # Consumes a floating constant's f/F or l/L suffix, returning it normalized
415
+ # ("f" for float, "l" for long double, "" for a plain double). At most one
416
+ # letter is valid; a longer run is caught by the trailing-character check.
417
+ def read_floating_suffix
418
+ ch = current
419
+ if ch == "f" || ch == "F"
420
+ advance
421
+ "f"
422
+ elsif ch == "l" || ch == "L"
423
+ advance
424
+ "l"
425
+ else
426
+ ""
427
+ end
428
+ end
429
+
430
+ # Consumes an integer constant's u/U and l/L suffix run and returns it in a
431
+ # normalized (lower-case) form ("", "u", "l", "ul", "ll", "ull", ...). A
432
+ # valid suffix is at most one "u" together with at most one l-part
433
+ # ("l"/"L" or "ll"/"LL", never mixed case), in either order; anything else
434
+ # is rejected. `start` locates the constant for a positioned error.
435
+ def read_integer_suffix(start)
436
+ raw = +""
437
+ raw << advance while current&.match?(/[uUlL]/)
438
+ return "" if raw.empty?
439
+
440
+ valid = raw.match?(/\A(?:[uU])?(?:ll|LL|[lL])?\z/) ||
441
+ raw.match?(/\A(?:ll|LL|[lL])?(?:[uU])?\z/)
442
+ raise LexError.new("invalid suffix #{raw.inspect} on integer constant", start) unless valid
443
+
444
+ raw.downcase
445
+ end
446
+
447
+ def digit?(ch)
448
+ !ch.nil? && ch.match?(/[0-9]/)
449
+ end
450
+
451
+ def octal_digit?(ch)
452
+ !ch.nil? && ch.match?(/[0-7]/)
453
+ end
454
+
455
+ def hex_digit?(ch)
456
+ !ch.nil? && ch.match?(/[0-9A-Fa-f]/)
457
+ end
458
+ end
459
+ end
460
+ end
@@ -0,0 +1,232 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "token"
4
+ require_relative "lexeme_reader"
5
+ require_relative "../compile_error"
6
+
7
+ module Rubycc
8
+ module Front
9
+ # Hand-written lexer for the C subset. Tracks 1-based line/column positions
10
+ # and keeps each source line around so tokens (and errors) can be reported
11
+ # with source excerpts. Handles // and /* */ comments and whitespace. The
12
+ # spelling-to-value decoding of numbers, string/character literals and
13
+ # identifiers is delegated to the shared LexemeReader, which the preprocessor
14
+ # reuses so both token sources agree exactly.
15
+ class Lexer
16
+ def initialize(source, filename:)
17
+ @src = source
18
+ @filename = filename
19
+ @pos = 0
20
+ @line = 1
21
+ @column = 1
22
+ # -1 keeps a trailing empty field, so line numbers map 1:1 to entries.
23
+ @lines = source.split("\n", -1)
24
+ end
25
+
26
+ def tokenize
27
+ tokens = []
28
+ loop do
29
+ skip_whitespace_and_comments
30
+ break if at_end?
31
+
32
+ tokens << next_token
33
+ end
34
+ tokens << make_token(:eof, nil, @line, @column)
35
+ tokens
36
+ end
37
+
38
+ private
39
+
40
+ def at_end?
41
+ @pos >= @src.length
42
+ end
43
+
44
+ def current_char
45
+ @src[@pos]
46
+ end
47
+
48
+ def peek(offset = 0)
49
+ @src[@pos + offset]
50
+ end
51
+
52
+ # Consumes one character, maintaining line/column bookkeeping.
53
+ def advance
54
+ ch = @src[@pos]
55
+ @pos += 1
56
+ if ch == "\n"
57
+ @line += 1
58
+ @column = 1
59
+ else
60
+ @column += 1
61
+ end
62
+ ch
63
+ end
64
+
65
+ def skip_whitespace_and_comments
66
+ loop do
67
+ ch = current_char
68
+ if ch.nil?
69
+ return
70
+ elsif ch =~ /\s/
71
+ advance
72
+ elsif ch == "/" && peek(1) == "/"
73
+ advance until at_end? || current_char == "\n"
74
+ elsif ch == "/" && peek(1) == "*"
75
+ skip_block_comment
76
+ else
77
+ return
78
+ end
79
+ end
80
+ end
81
+
82
+ def skip_block_comment
83
+ start_line = @line
84
+ start_col = @column
85
+ advance # /
86
+ advance # *
87
+ loop do
88
+ if at_end?
89
+ raise_error("unterminated block comment", start_line, start_col)
90
+ elsif current_char == "*" && peek(1) == "/"
91
+ advance # *
92
+ advance # /
93
+ return
94
+ else
95
+ advance
96
+ end
97
+ end
98
+ end
99
+
100
+ def next_token
101
+ line = @line
102
+ column = @column
103
+ ch = current_char
104
+
105
+ if ch =~ /[0-9]/
106
+ read_lexeme(:read_number, line, column)
107
+ elsif ch == "." && peek(1) =~ /[0-9]/
108
+ # A leading-dot floating constant (".5"): a "." immediately followed
109
+ # by a digit is the fractional-only form. A "." in any other position
110
+ # is the member-access punctuator (or part of "..."), handled by
111
+ # #lex_punctuator.
112
+ read_lexeme(:read_number, line, column)
113
+ elsif ch =~ /[A-Za-z_]/
114
+ read_lexeme(:read_identifier, line, column)
115
+ elsif ch == "'"
116
+ read_lexeme(:read_char, line, column)
117
+ elsif ch == "\""
118
+ read_lexeme(:read_string, line, column)
119
+ else
120
+ lex_punctuator(line, column)
121
+ end
122
+ end
123
+
124
+ # Decodes one lexeme with the shared LexemeReader, then walks the streaming
125
+ # cursor over the exact characters it consumed so line/column bookkeeping
126
+ # stays correct. A LexError from the reader carries an absolute offset into
127
+ # the source, which maps to a column on this (single) line since none of
128
+ # these lexemes span a newline.
129
+ def read_lexeme(method, line, column)
130
+ start = @pos
131
+ reader = LexemeReader.new(@src, start)
132
+ begin
133
+ result = reader.public_send(method)
134
+ rescue LexError => e
135
+ raise_error(e.message, line, column + (e.offset - start))
136
+ end
137
+ advance while @pos < reader.pos
138
+ build_token(result, line, column)
139
+ end
140
+
141
+ def build_token(result, line, column)
142
+ case result.type
143
+ when :num
144
+ make_num_token(result.value, result.base, result.suffix, line, column)
145
+ when :float
146
+ make_float_token(result.value, result.suffix, line, column)
147
+ else
148
+ make_token(result.type, result.value, line, column)
149
+ end
150
+ end
151
+
152
+ # Longest-match punctuator scan: try the three-character punctuators
153
+ # first, then the two-character ones, then fall back to the
154
+ # single-character ones. peek(1)/peek(2) yield nil past the end, which
155
+ # interpolate to "", so the assembled candidates simply fail to match near
156
+ # EOF and the scan falls through to a shorter length.
157
+ def lex_punctuator(line, column)
158
+ three = "#{current_char}#{peek(1)}#{peek(2)}"
159
+ two = "#{current_char}#{peek(1)}"
160
+ if LexemeReader::PUNCTUATORS_3.include?(three)
161
+ advance
162
+ advance
163
+ advance
164
+ make_token(:punct, three, line, column)
165
+ elsif LexemeReader::PUNCTUATORS_2.include?(two)
166
+ advance
167
+ advance
168
+ make_token(:punct, two, line, column)
169
+ elsif LexemeReader::PUNCTUATORS_1.include?(current_char)
170
+ ch = advance
171
+ make_token(:punct, ch, line, column)
172
+ else
173
+ raise_error("unexpected character #{current_char.inspect}", line, column)
174
+ end
175
+ end
176
+
177
+ def make_token(type, value, line, column)
178
+ Token.new(
179
+ type: type,
180
+ value: value,
181
+ filename: @filename,
182
+ line: line,
183
+ column: column,
184
+ source_line: source_line_for(line)
185
+ )
186
+ end
187
+
188
+ # A :num token carrying the extra base/suffix an integer constant needs
189
+ # for the parser to fix its type.
190
+ def make_num_token(value, base, suffix, line, column)
191
+ Token.new(
192
+ type: :num,
193
+ value: value,
194
+ filename: @filename,
195
+ line: line,
196
+ column: column,
197
+ source_line: source_line_for(line),
198
+ base: base,
199
+ suffix: suffix
200
+ )
201
+ end
202
+
203
+ # A :float token carrying the folded Ruby Float value and the normalized
204
+ # floating suffix the parser fixes the constant's type from.
205
+ def make_float_token(value, suffix, line, column)
206
+ Token.new(
207
+ type: :float,
208
+ value: value,
209
+ filename: @filename,
210
+ line: line,
211
+ column: column,
212
+ source_line: source_line_for(line),
213
+ suffix: suffix
214
+ )
215
+ end
216
+
217
+ def source_line_for(line)
218
+ @lines[line - 1] || ""
219
+ end
220
+
221
+ def raise_error(description, line, column)
222
+ raise CompileError.new(
223
+ description,
224
+ filename: @filename,
225
+ line: line,
226
+ column: column,
227
+ source_line: source_line_for(line)
228
+ )
229
+ end
230
+ end
231
+ end
232
+ end