rubycc 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +56 -0
- data/LICENSE.txt +21 -0
- data/NOTICE +52 -0
- data/README.md +208 -0
- data/data/README.md +117 -0
- data/data/r10_corpus_scan.json +4082 -0
- data/data/r10_manual_classification.json +5342 -0
- data/data/r10_verification_m4a.json +1531 -0
- data/data/r10_verification_m4b.json +1818 -0
- data/data/r10_verification_m4c.json +1489 -0
- data/data/r10_verification_m4d.json +318 -0
- data/data/r10_verification_rbs.json +192 -0
- data/data/verified_gems.json +397 -0
- data/exe/rmake +16 -0
- data/exe/rubycc +9 -0
- data/exe/rubycc-ar +114 -0
- data/exe/rubycc-doctor +14 -0
- data/exe/rubycc-pkgconf +8 -0
- data/include/float.h +87 -0
- data/include/iso646.h +18 -0
- data/include/libc/alloca.h +18 -0
- data/include/libc/arpa/inet.h +61 -0
- data/include/libc/assert.h +43 -0
- data/include/libc/dirent.h +72 -0
- data/include/libc/dlfcn.h +58 -0
- data/include/libc/features.h +205 -0
- data/include/libc/glibc/aarch64/ctype.h +108 -0
- data/include/libc/glibc/aarch64/endian.h +54 -0
- data/include/libc/glibc/aarch64/errno.h +154 -0
- data/include/libc/glibc/aarch64/fcntl.h +144 -0
- data/include/libc/glibc/aarch64/inttypes.h +182 -0
- data/include/libc/glibc/aarch64/limits.h +63 -0
- data/include/libc/glibc/aarch64/pthread.h +131 -0
- data/include/libc/glibc/aarch64/setjmp.h +84 -0
- data/include/libc/glibc/aarch64/stdint.h +174 -0
- data/include/libc/glibc/aarch64/sys/epoll.h +88 -0
- data/include/libc/glibc/aarch64/sys/fcntl.h +19 -0
- data/include/libc/glibc/aarch64/sys/select.h +75 -0
- data/include/libc/glibc/aarch64/sys/stat.h +136 -0
- data/include/libc/glibc/aarch64/sys/syscall.h +167 -0
- data/include/libc/glibc/aarch64/sys/time.h +74 -0
- data/include/libc/glibc/aarch64/sys/types.h +127 -0
- data/include/libc/glibc/aarch64/time.h +113 -0
- data/include/libc/glibc/x86_64/ctype.h +108 -0
- data/include/libc/glibc/x86_64/endian.h +54 -0
- data/include/libc/glibc/x86_64/errno.h +154 -0
- data/include/libc/glibc/x86_64/fcntl.h +134 -0
- data/include/libc/glibc/x86_64/inttypes.h +182 -0
- data/include/libc/glibc/x86_64/limits.h +61 -0
- data/include/libc/glibc/x86_64/pthread.h +111 -0
- data/include/libc/glibc/x86_64/setjmp.h +82 -0
- data/include/libc/glibc/x86_64/stdint.h +163 -0
- data/include/libc/glibc/x86_64/sys/epoll.h +91 -0
- data/include/libc/glibc/x86_64/sys/fcntl.h +19 -0
- data/include/libc/glibc/x86_64/sys/select.h +75 -0
- data/include/libc/glibc/x86_64/sys/stat.h +130 -0
- data/include/libc/glibc/x86_64/sys/syscall.h +188 -0
- data/include/libc/glibc/x86_64/sys/time.h +74 -0
- data/include/libc/glibc/x86_64/sys/types.h +123 -0
- data/include/libc/glibc/x86_64/time.h +113 -0
- data/include/libc/grp.h +50 -0
- data/include/libc/langinfo.h +126 -0
- data/include/libc/link.h +18 -0
- data/include/libc/locale.h +83 -0
- data/include/libc/math.h +191 -0
- data/include/libc/netinet/in.h +126 -0
- data/include/libc/netinet/tcp.h +44 -0
- data/include/libc/poll.h +46 -0
- data/include/libc/pwd.h +58 -0
- data/include/libc/regex.h +53 -0
- data/include/libc/sched.h +35 -0
- data/include/libc/signal.h +204 -0
- data/include/libc/stdio.h +157 -0
- data/include/libc/stdlib.h +92 -0
- data/include/libc/string.h +74 -0
- data/include/libc/strings.h +25 -0
- data/include/libc/sys/cdefs.h +126 -0
- data/include/libc/sys/inotify.h +111 -0
- data/include/libc/sys/ioctl.h +36 -0
- data/include/libc/sys/mman.h +65 -0
- data/include/libc/sys/param.h +41 -0
- data/include/libc/sys/resource.h +109 -0
- data/include/libc/sys/socket.h +167 -0
- data/include/libc/sys/statfs.h +82 -0
- data/include/libc/sys/timerfd.h +55 -0
- data/include/libc/sys/uio.h +40 -0
- data/include/libc/sys/un.h +25 -0
- data/include/libc/sys/utsname.h +35 -0
- data/include/libc/sys/wait.h +135 -0
- data/include/libc/termios.h +179 -0
- data/include/libc/unistd.h +194 -0
- data/include/stdalign.h +16 -0
- data/include/stdarg.h +31 -0
- data/include/stdatomic.h +158 -0
- data/include/stdbool.h +15 -0
- data/include/stdckdint.h +28 -0
- data/include/stddef.h +60 -0
- data/include/stdnoreturn.h +18 -0
- data/include/x86intrin.h +16 -0
- data/lib/rubycc/backend/aarch64.rb +1724 -0
- data/lib/rubycc/backend/x86_64.rb +1369 -0
- data/lib/rubycc/compile_error.rb +36 -0
- data/lib/rubycc/compiler.rb +305 -0
- data/lib/rubycc/doctor/builder.rb +151 -0
- data/lib/rubycc/doctor/cli.rb +190 -0
- data/lib/rubycc/doctor/fetcher.rb +84 -0
- data/lib/rubycc/doctor/gemfile.rb +115 -0
- data/lib/rubycc/doctor/verified_gems.rb +114 -0
- data/lib/rubycc/doctor.rb +16 -0
- data/lib/rubycc/driver.rb +463 -0
- data/lib/rubycc/front/ast.rb +528 -0
- data/lib/rubycc/front/constant_evaluator.rb +631 -0
- data/lib/rubycc/front/initializer_resolver.rb +592 -0
- data/lib/rubycc/front/lexeme_reader.rb +460 -0
- data/lib/rubycc/front/lexer.rb +232 -0
- data/lib/rubycc/front/parser.rb +4122 -0
- data/lib/rubycc/front/token.rb +49 -0
- data/lib/rubycc/ir/call_convention.rb +486 -0
- data/lib/rubycc/ir/generator.rb +6036 -0
- data/lib/rubycc/ir/ir.rb +417 -0
- data/lib/rubycc/link/compat_runtime.rb +89 -0
- data/lib/rubycc/link/errors.rb +13 -0
- data/lib/rubycc/link/executable_linker.rb +405 -0
- data/lib/rubycc/link/library_resolver.rb +437 -0
- data/lib/rubycc/link/partial_linker.rb +546 -0
- data/lib/rubycc/link/shared_linker.rb +1732 -0
- data/lib/rubycc/mkmf_shim.rb +83 -0
- data/lib/rubycc/objfile/ar_archive.rb +437 -0
- data/lib/rubycc/objfile/elf_reader.rb +646 -0
- data/lib/rubycc/objfile/elf_writer.rb +891 -0
- data/lib/rubycc/objfile/relocatable_writer.rb +376 -0
- data/lib/rubycc/pkgconf/cli.rb +139 -0
- data/lib/rubycc/pkgconf/errors.rb +39 -0
- data/lib/rubycc/pkgconf/model.rb +28 -0
- data/lib/rubycc/pkgconf/parser.rb +109 -0
- data/lib/rubycc/pkgconf/pkgconf.rb +15 -0
- data/lib/rubycc/pkgconf/resolver.rb +70 -0
- data/lib/rubycc/pkgconf/search_path.rb +38 -0
- data/lib/rubycc/pkgconf/system_path_filter.rb +131 -0
- data/lib/rubycc/preprocess/constant_expression.rb +219 -0
- data/lib/rubycc/preprocess/glibc_version.rb +124 -0
- data/lib/rubycc/preprocess/pp_token.rb +62 -0
- data/lib/rubycc/preprocess/preprocessor.rb +2020 -0
- data/lib/rubycc/preprocess/scanner.rb +290 -0
- data/lib/rubycc/preprocess/token_converter.rb +157 -0
- data/lib/rubycc/rmake/cli.rb +143 -0
- data/lib/rubycc/rmake/errors.rb +71 -0
- data/lib/rubycc/rmake/executor.rb +818 -0
- data/lib/rubycc/rmake/expander.rb +251 -0
- data/lib/rubycc/rmake/makefile.rb +352 -0
- data/lib/rubycc/rmake/model.rb +137 -0
- data/lib/rubycc/rmake/parser.rb +226 -0
- data/lib/rubycc/rmake/rmake.rb +15 -0
- data/lib/rubycc/type.rb +1236 -0
- data/lib/rubycc/version.rb +5 -0
- data/lib/rubycc.rb +36 -0
- data/lib/rubygems_plugin.rb +102 -0
- metadata +219 -0
|
@@ -0,0 +1,460 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Rubycc
|
|
4
|
+
module Front
|
|
5
|
+
# Raised while decoding the interior of a single lexeme (a numeric constant,
|
|
6
|
+
# a string/character literal). It is deliberately position-agnostic: it only
|
|
7
|
+
# names the problem and the character offset (into the text handed to the
|
|
8
|
+
# reader) where it occurred, leaving the caller to translate that offset into
|
|
9
|
+
# a source line/column for a user-facing CompileError. This lets the same
|
|
10
|
+
# decoder serve both the streaming Lexer (offsets into the whole source) and
|
|
11
|
+
# the preprocessor's converter (offsets into an isolated lexeme).
|
|
12
|
+
class LexError < StandardError
|
|
13
|
+
attr_reader :offset
|
|
14
|
+
|
|
15
|
+
def initialize(message, offset)
|
|
16
|
+
@offset = offset
|
|
17
|
+
super(message)
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# The shared spelling-to-value decoder for the C subset's lexemes. Both the
|
|
22
|
+
# streaming Front::Lexer and the preprocessor's token converter delegate here
|
|
23
|
+
# so the two paths fold numeric constants, resolve escapes and classify
|
|
24
|
+
# identifiers with byte-for-byte identical rules. The reader walks a plain
|
|
25
|
+
# string with its own cursor (no line/column bookkeeping); positions are the
|
|
26
|
+
# caller's concern. Each `read_*` leaves `pos` just past the lexeme.
|
|
27
|
+
class LexemeReader
|
|
28
|
+
# A decoded lexeme. `type` is :num, :float, :string, :ident or :keyword.
|
|
29
|
+
# `value` is the folded Integer/Float, the escape-resolved ASCII-8BIT bytes
|
|
30
|
+
# of a string, or the identifier text. `base`/`suffix` accompany an integer
|
|
31
|
+
# :num and `suffix` a :float, matching Front::Token's numeric fields.
|
|
32
|
+
Result = Struct.new(:type, :value, :base, :suffix)
|
|
33
|
+
|
|
34
|
+
KEYWORDS = %w[int char void short long signed unsigned _Bool float double struct union
|
|
35
|
+
enum typedef static extern const volatile inline _Noreturn register auto
|
|
36
|
+
return if else while do for break continue
|
|
37
|
+
switch case default goto sizeof
|
|
38
|
+
_Static_assert _Alignof _Alignas _Atomic __int128
|
|
39
|
+
__builtin_va_start __builtin_va_arg __builtin_va_end __builtin_va_copy
|
|
40
|
+
__builtin_expect __builtin_alloca __builtin_offsetof
|
|
41
|
+
__builtin_constant_p __builtin_choose_expr
|
|
42
|
+
__builtin_ctz __builtin_ctzll __builtin_clz __builtin_clzll
|
|
43
|
+
__builtin_unreachable __builtin_memcpy
|
|
44
|
+
__builtin_add_overflow __builtin_sub_overflow __builtin_mul_overflow
|
|
45
|
+
__atomic_load_n __atomic_store_n __atomic_exchange_n
|
|
46
|
+
__atomic_compare_exchange_n
|
|
47
|
+
__atomic_fetch_add __atomic_fetch_sub
|
|
48
|
+
__atomic_add_fetch __atomic_sub_fetch __atomic_or_fetch
|
|
49
|
+
__atomic_thread_fence
|
|
50
|
+
__sync_fetch_and_add __sync_fetch_and_sub
|
|
51
|
+
__sync_add_and_fetch __sync_sub_and_fetch __sync_or_and_fetch
|
|
52
|
+
__sync_lock_test_and_set __sync_lock_release __sync_synchronize
|
|
53
|
+
__sync_bool_compare_and_swap __sync_val_compare_and_swap
|
|
54
|
+
__asm__
|
|
55
|
+
__attribute__ __extension__].freeze
|
|
56
|
+
|
|
57
|
+
# Membership lookup for every identifier the lexer produces, so it must
|
|
58
|
+
# be O(1); KEYWORDS.include? showed up as a linear scan in profiling.
|
|
59
|
+
KEYWORD_SET = KEYWORDS.to_h { |word| [word, true] }.freeze
|
|
60
|
+
|
|
61
|
+
# gcc's reserved "__x"/"__x__" alternate spellings for a handful of
|
|
62
|
+
# keywords (6.10.8.4's rationale: a header built with strict-ISO options
|
|
63
|
+
# such as -ansi still needs the keyword's meaning without colliding with
|
|
64
|
+
# a user identifier of the plain name). glibc's uapi-derived headers lean
|
|
65
|
+
# on these unconditionally, e.g. asm-generic/int-ll64.h's
|
|
66
|
+
# "typedef __signed__ char __s8". Each maps straight to the plain
|
|
67
|
+
# keyword's own spelling, so every downstream check keyed on that
|
|
68
|
+
# spelling (DECL_SPECIFIER_KEYWORDS, "const"/"volatile"/"inline" in
|
|
69
|
+
# Front::Parser, ...) sees an ordinary keyword token and needs no
|
|
70
|
+
# separate case for the alias.
|
|
71
|
+
KEYWORD_ALIASES = {
|
|
72
|
+
"__signed" => "signed", "__signed__" => "signed",
|
|
73
|
+
"__const" => "const", "__const__" => "const",
|
|
74
|
+
"__volatile" => "volatile", "__volatile__" => "volatile",
|
|
75
|
+
"__inline" => "inline", "__inline__" => "inline"
|
|
76
|
+
}.freeze
|
|
77
|
+
|
|
78
|
+
# The eleven simple escape sequences of 6.4.4.4p1, shared by character
|
|
79
|
+
# constants and string literals, mapping the character after the backslash
|
|
80
|
+
# to the byte value it denotes. "\?" is the one whose escaped and plain
|
|
81
|
+
# spellings mean the same byte ('?'): it exists only so a source line can
|
|
82
|
+
# avoid accidentally spelling a trigraph, and gcc accepts it in both
|
|
83
|
+
# constructs. "\x" (hexadecimal, any number of digits) and octal ("\ooo",
|
|
84
|
+
# 1-3 digits, "0" included) are handled separately in #read_escaped_byte
|
|
85
|
+
# since their value comes from digits rather than a fixed table lookup.
|
|
86
|
+
ESCAPES = {
|
|
87
|
+
"n" => 10, "t" => 9, "r" => 13, "\\" => 92,
|
|
88
|
+
"'" => 39, "\"" => 34, "?" => 63,
|
|
89
|
+
"a" => 7, "b" => 8, "f" => 12, "v" => 11
|
|
90
|
+
}.freeze
|
|
91
|
+
|
|
92
|
+
# Three-character punctuators, matched before the shorter lists so the
|
|
93
|
+
# longest one always wins: "<<=" must beat "<<" (and "<="/"<"), ">>=" must
|
|
94
|
+
# beat ">>" (and ">="/">"), and "..." (the variadic-parameter ellipsis)
|
|
95
|
+
# must beat a lone "." (which is not a two-character punctuator, so ".."
|
|
96
|
+
# never forms; three consecutive dots are the only way "..." arises).
|
|
97
|
+
PUNCTUATORS_3 = %w[<<= >>= ...].freeze
|
|
98
|
+
|
|
99
|
+
# Two-character punctuators, matched before the single-character list so
|
|
100
|
+
# the lexer always prefers the longest punctuator ("==" over two "=",
|
|
101
|
+
# "&&" over two "&", "++" or "+=" over a lone "+", "->" over a lone "-",
|
|
102
|
+
# "<<" over two "<", "&=" over "&"/"&&").
|
|
103
|
+
PUNCTUATORS_2 = %w[== != <= >= && || += -= *= /= %= ++ -- -> << >> &= |= ^=].freeze
|
|
104
|
+
|
|
105
|
+
# Single-character punctuators used by this slice. "&" is both the
|
|
106
|
+
# address-of operator and the bitwise-and operator, "*" doubles as
|
|
107
|
+
# dereference and pointer-declarator marker, "|" "^" "~" are the remaining
|
|
108
|
+
# bitwise operators, "[" "]" bracket array declarators and subscripts,
|
|
109
|
+
# "?" ":" form the conditional operator, and "." selects a struct member.
|
|
110
|
+
PUNCTUATORS_1 = %w[+ - * / % ( ) { } ; = , < > ! & | ^ ~ [ ] ? : .].freeze
|
|
111
|
+
|
|
112
|
+
# Whether `name` (an identifier's spelling) is a reserved keyword.
|
|
113
|
+
def self.keyword?(name)
|
|
114
|
+
KEYWORD_SET.key?(name)
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
# The keyword token `name` denotes: itself when it already is one of
|
|
118
|
+
# KEYWORDS, or the plain spelling it aliases (see KEYWORD_ALIASES) when it
|
|
119
|
+
# is one of gcc's reserved "__x"/"__x__" spellings. nil when `name` names
|
|
120
|
+
# neither, so the caller lexes it as an ordinary identifier.
|
|
121
|
+
def self.keyword_spelling(name)
|
|
122
|
+
return name if KEYWORD_SET.key?(name)
|
|
123
|
+
|
|
124
|
+
KEYWORD_ALIASES[name]
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
attr_reader :pos
|
|
128
|
+
|
|
129
|
+
def initialize(text, start = 0)
|
|
130
|
+
@text = text
|
|
131
|
+
@pos = start
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
# An identifier or keyword run "[A-Za-z_][A-Za-z0-9_]*". The caller must
|
|
135
|
+
# position the cursor on the leading identifier character.
|
|
136
|
+
def read_identifier
|
|
137
|
+
name = +""
|
|
138
|
+
name << advance while identifier_char?(current)
|
|
139
|
+
spelling = LexemeReader.keyword_spelling(name)
|
|
140
|
+
Result.new(spelling ? :keyword : :ident, spelling || name, nil, nil)
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
# A numeric constant. A "0x"/"0X" prefix is hexadecimal (an integer, or a
|
|
144
|
+
# hexadecimal floating constant this subset does not lower yet); otherwise
|
|
145
|
+
# a "." or an exponent anywhere in the run marks a decimal floating
|
|
146
|
+
# constant, and its absence a decimal or octal integer. The floating check
|
|
147
|
+
# precedes the octal one so "08.5" reads as the float 8.5 rather than
|
|
148
|
+
# tripping the octal-digit rule on its '8'.
|
|
149
|
+
def read_number
|
|
150
|
+
if current == "0" && (peek == "x" || peek == "X")
|
|
151
|
+
read_hexadecimal_constant
|
|
152
|
+
elsif current == "0" && (peek == "b" || peek == "B")
|
|
153
|
+
read_binary_constant
|
|
154
|
+
elsif floating_constant_ahead?
|
|
155
|
+
read_floating_constant
|
|
156
|
+
else
|
|
157
|
+
read_integer_constant
|
|
158
|
+
end
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
# A string literal "abc" including its surrounding quotes: returns the
|
|
162
|
+
# escape-resolved bytes (ASCII-8BIT), without the NUL terminator the
|
|
163
|
+
# generator later appends. An unterminated literal (newline or end before
|
|
164
|
+
# the closing quote) and an unknown escape are rejected.
|
|
165
|
+
def read_string
|
|
166
|
+
start = @pos
|
|
167
|
+
advance # opening quote
|
|
168
|
+
bytes = +"".b
|
|
169
|
+
loop do
|
|
170
|
+
if at_end? || current == "\n"
|
|
171
|
+
raise LexError.new("unterminated string literal", start)
|
|
172
|
+
end
|
|
173
|
+
break if current == "\""
|
|
174
|
+
|
|
175
|
+
bytes << read_escaped_byte(start, "string literal")
|
|
176
|
+
end
|
|
177
|
+
advance # closing quote
|
|
178
|
+
Result.new(:string, bytes, nil, nil)
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
# A character constant 'c' or '\n' including its quotes: returns a :num
|
|
182
|
+
# result whose value is the character's byte code, since an ISO C character
|
|
183
|
+
# constant has type int (6.4.4.4). The constant must hold exactly one
|
|
184
|
+
# character; an empty '', a multi-character 'ab', an unterminated ' and an
|
|
185
|
+
# unknown escape are all rejected.
|
|
186
|
+
def read_char
|
|
187
|
+
start = @pos
|
|
188
|
+
advance # opening quote
|
|
189
|
+
if at_end? || current == "\n"
|
|
190
|
+
raise LexError.new("unterminated character constant", start)
|
|
191
|
+
elsif current == "'"
|
|
192
|
+
raise LexError.new("empty character constant", start)
|
|
193
|
+
end
|
|
194
|
+
value = read_escaped_byte(start, "character constant")
|
|
195
|
+
if at_end? || current == "\n"
|
|
196
|
+
raise LexError.new("unterminated character constant", start)
|
|
197
|
+
elsif current != "'"
|
|
198
|
+
raise LexError.new("multi-character character constant", start)
|
|
199
|
+
end
|
|
200
|
+
advance # closing quote
|
|
201
|
+
Result.new(:num, value, 10, "")
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
private
|
|
205
|
+
|
|
206
|
+
def at_end?
|
|
207
|
+
@pos >= @text.length
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
def current
|
|
211
|
+
@text[@pos]
|
|
212
|
+
end
|
|
213
|
+
|
|
214
|
+
def peek(offset = 1)
|
|
215
|
+
@text[@pos + offset]
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
def advance
|
|
219
|
+
ch = @text[@pos]
|
|
220
|
+
@pos += 1
|
|
221
|
+
ch
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
def identifier_char?(ch)
|
|
225
|
+
!ch.nil? && ch.match?(/[A-Za-z0-9_]/)
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
# Reads one logical character and returns its byte value, decoding a
|
|
229
|
+
# backslash escape via ESCAPES, a hexadecimal escape ("\x" then one or
|
|
230
|
+
# more hex digits) or an octal escape ("\ooo", 1-3 octal digits, 6.4.4.4).
|
|
231
|
+
# Shared by character constants and string literals; `context` names the
|
|
232
|
+
# construct for error messages and `start` locates the opening quote for
|
|
233
|
+
# a positioned error.
|
|
234
|
+
def read_escaped_byte(start, context)
|
|
235
|
+
ch = current
|
|
236
|
+
if ch.nil? || ch == "\n"
|
|
237
|
+
raise LexError.new("unterminated #{context}", start)
|
|
238
|
+
elsif ch == "\\"
|
|
239
|
+
advance # backslash
|
|
240
|
+
esc = current
|
|
241
|
+
if esc.nil? || esc == "\n"
|
|
242
|
+
raise LexError.new("unterminated #{context}", start)
|
|
243
|
+
elsif esc == "x"
|
|
244
|
+
advance
|
|
245
|
+
read_hex_escape(start)
|
|
246
|
+
elsif octal_digit?(esc)
|
|
247
|
+
read_octal_escape(start)
|
|
248
|
+
else
|
|
249
|
+
code = ESCAPES[esc]
|
|
250
|
+
raise LexError.new("unknown escape sequence in #{context}", start) unless code
|
|
251
|
+
|
|
252
|
+
advance
|
|
253
|
+
code
|
|
254
|
+
end
|
|
255
|
+
else
|
|
256
|
+
advance
|
|
257
|
+
ch.ord
|
|
258
|
+
end
|
|
259
|
+
end
|
|
260
|
+
|
|
261
|
+
# A "\x" escape's value: one or more hex digits (unbounded in the grammar,
|
|
262
|
+
# 6.4.4.4), rejected here when it exceeds a byte's range since a char or a
|
|
263
|
+
# string element is one byte in this implementation. At least one digit is
|
|
264
|
+
# required ("\x" alone is ill-formed).
|
|
265
|
+
def read_hex_escape(start)
|
|
266
|
+
digits = +""
|
|
267
|
+
digits << advance while hex_digit?(current)
|
|
268
|
+
raise LexError.new("\\x used with no following hex digits", start) if digits.empty?
|
|
269
|
+
|
|
270
|
+
value = digits.to_i(16)
|
|
271
|
+
raise LexError.new("hex escape sequence out of range", start) if value > 255
|
|
272
|
+
|
|
273
|
+
value
|
|
274
|
+
end
|
|
275
|
+
|
|
276
|
+
# An octal escape's value: 1-3 octal digits (the leading one already
|
|
277
|
+
# confirmed present by the caller), rejected when it exceeds a byte's
|
|
278
|
+
# range ("\777" is 511, past 255).
|
|
279
|
+
def read_octal_escape(start)
|
|
280
|
+
digits = +""
|
|
281
|
+
3.times do
|
|
282
|
+
break unless octal_digit?(current)
|
|
283
|
+
|
|
284
|
+
digits << advance
|
|
285
|
+
end
|
|
286
|
+
value = digits.to_i(8)
|
|
287
|
+
raise LexError.new("octal escape sequence out of range", start) if value > 255
|
|
288
|
+
|
|
289
|
+
value
|
|
290
|
+
end
|
|
291
|
+
|
|
292
|
+
# Whether the digit run at the cursor is a decimal floating constant: it is
|
|
293
|
+
# exactly when a "." or an exponent marker ('e'/'E') follows the leading
|
|
294
|
+
# digits. Only consulted for a decimal constant (hexadecimal is split off
|
|
295
|
+
# first), so an 'e'/'E' here is always an exponent, never a hex digit.
|
|
296
|
+
def floating_constant_ahead?
|
|
297
|
+
i = @pos
|
|
298
|
+
i += 1 while @text[i]&.match?(/[0-9]/)
|
|
299
|
+
ch = @text[i]
|
|
300
|
+
ch == "." || ch == "e" || ch == "E"
|
|
301
|
+
end
|
|
302
|
+
|
|
303
|
+
# A hexadecimal integer constant "0x...". A '.' or a binary-exponent marker
|
|
304
|
+
# ('p'/'P') after the hex digits is a hexadecimal *floating* constant
|
|
305
|
+
# (0x1.8p3), a distinct grammar this subset does not lower yet, so it is
|
|
306
|
+
# diagnosed rather than silently misread as an integer.
|
|
307
|
+
def read_hexadecimal_constant
|
|
308
|
+
start = @pos
|
|
309
|
+
advance # 0
|
|
310
|
+
advance # x
|
|
311
|
+
digits = +""
|
|
312
|
+
digits << advance while hex_digit?(current)
|
|
313
|
+
raise LexError.new("invalid hexadecimal constant", start) if digits.empty?
|
|
314
|
+
if current == "." || current == "p" || current == "P"
|
|
315
|
+
raise LexError.new("hexadecimal floating constants are not supported yet", start)
|
|
316
|
+
end
|
|
317
|
+
suffix = read_integer_suffix(start)
|
|
318
|
+
if identifier_char?(current)
|
|
319
|
+
raise LexError.new("invalid suffix on integer constant", @pos)
|
|
320
|
+
end
|
|
321
|
+
Result.new(:num, digits.to_i(16), 16, suffix)
|
|
322
|
+
end
|
|
323
|
+
|
|
324
|
+
# A binary integer constant "0b...."/"0B...." (a GNU extension, made
|
|
325
|
+
# standard in C23): the "0b" prefix, one or more binary digits, and an
|
|
326
|
+
# optional integer suffix. It shares the octal/hexadecimal type rules (a
|
|
327
|
+
# non-decimal constant may take an unsigned type without a "u" suffix), so
|
|
328
|
+
# the result carries base 2 for #integer_literal_type to treat it as such.
|
|
329
|
+
# At least one digit is required; a trailing identifier character after the
|
|
330
|
+
# suffix is rejected like any other invalid suffix.
|
|
331
|
+
def read_binary_constant
|
|
332
|
+
start = @pos
|
|
333
|
+
advance # 0
|
|
334
|
+
advance # b / B
|
|
335
|
+
digits = +""
|
|
336
|
+
digits << advance while current == "0" || current == "1"
|
|
337
|
+
raise LexError.new("invalid binary constant", start) if digits.empty?
|
|
338
|
+
|
|
339
|
+
suffix = read_integer_suffix(start)
|
|
340
|
+
if identifier_char?(current)
|
|
341
|
+
raise LexError.new("invalid suffix on integer constant", @pos)
|
|
342
|
+
end
|
|
343
|
+
Result.new(:num, digits.to_i(2), 2, suffix)
|
|
344
|
+
end
|
|
345
|
+
|
|
346
|
+
# A decimal or octal integer constant, with an optional u/U and l/L/ll/LL
|
|
347
|
+
# suffix run. The result carries the folded value with its base and
|
|
348
|
+
# normalized suffix. A trailing identifier character (e.g. 12abc, once the
|
|
349
|
+
# real suffix letters are consumed) is rejected.
|
|
350
|
+
def read_integer_constant
|
|
351
|
+
start = @pos
|
|
352
|
+
if current == "0" && digit?(peek)
|
|
353
|
+
advance # leading 0
|
|
354
|
+
digits = +"0"
|
|
355
|
+
while digit?(current)
|
|
356
|
+
unless octal_digit?(current)
|
|
357
|
+
raise LexError.new("invalid digit in octal constant", @pos)
|
|
358
|
+
end
|
|
359
|
+
digits << advance
|
|
360
|
+
end
|
|
361
|
+
base = 8
|
|
362
|
+
else
|
|
363
|
+
digits = +""
|
|
364
|
+
digits << advance while digit?(current)
|
|
365
|
+
base = 10
|
|
366
|
+
end
|
|
367
|
+
|
|
368
|
+
suffix = read_integer_suffix(start)
|
|
369
|
+
if identifier_char?(current)
|
|
370
|
+
raise LexError.new("invalid suffix on integer constant", @pos)
|
|
371
|
+
end
|
|
372
|
+
Result.new(:num, digits.to_i(base), base, suffix)
|
|
373
|
+
end
|
|
374
|
+
|
|
375
|
+
# A decimal floating constant (6.4.4.2): an integer part, an optional
|
|
376
|
+
# fraction after a ".", and an optional exponent ("e"/"E" with an optional
|
|
377
|
+
# sign and required digits). A trailing f/F makes it `float`, l/L `long
|
|
378
|
+
# double` (treated as `double`), and no suffix `double`. The decimal is
|
|
379
|
+
# converted through Rational so a C header constant outside Ruby's Float
|
|
380
|
+
# range (for example `1e10000`) produces Infinity or 0.0 without Ruby's
|
|
381
|
+
# `-w` range warning. An exponent with no digits, and a trailing identifier
|
|
382
|
+
# or "." character, are rejected.
|
|
383
|
+
def read_floating_constant
|
|
384
|
+
start = @pos
|
|
385
|
+
text = +""
|
|
386
|
+
text << advance while digit?(current)
|
|
387
|
+
saw_fraction_digit = true
|
|
388
|
+
if current == "."
|
|
389
|
+
text << advance
|
|
390
|
+
saw_fraction_digit = digit?(current)
|
|
391
|
+
text << advance while digit?(current)
|
|
392
|
+
end
|
|
393
|
+
if current == "e" || current == "E"
|
|
394
|
+
text << advance
|
|
395
|
+
text << advance if current == "+" || current == "-"
|
|
396
|
+
unless digit?(current)
|
|
397
|
+
raise LexError.new("exponent has no digits", start)
|
|
398
|
+
end
|
|
399
|
+
text << advance while digit?(current)
|
|
400
|
+
end
|
|
401
|
+
suffix = read_floating_suffix
|
|
402
|
+
if !current.nil? && current.match?(/[A-Za-z0-9_.]/)
|
|
403
|
+
raise LexError.new("invalid suffix on floating constant", @pos)
|
|
404
|
+
end
|
|
405
|
+
# C allows a floating constant with a "." followed by no fraction
|
|
406
|
+
# digits before the exponent (6.4.4.2), e.g. "1.e5". Since "N." and
|
|
407
|
+
# "N.0" are the same number, pad the fraction with a "0" before
|
|
408
|
+
# Rational parses it. This only affects the conversion buffer, not the
|
|
409
|
+
# token's spelling (Result carries no spelling field).
|
|
410
|
+
conversion_text = saw_fraction_digit ? text : text.sub(".", ".0")
|
|
411
|
+
Result.new(:float, Rational(conversion_text).to_f, nil, suffix)
|
|
412
|
+
end
|
|
413
|
+
|
|
414
|
+
# Consumes a floating constant's f/F or l/L suffix, returning it normalized
|
|
415
|
+
# ("f" for float, "l" for long double, "" for a plain double). At most one
|
|
416
|
+
# letter is valid; a longer run is caught by the trailing-character check.
|
|
417
|
+
def read_floating_suffix
|
|
418
|
+
ch = current
|
|
419
|
+
if ch == "f" || ch == "F"
|
|
420
|
+
advance
|
|
421
|
+
"f"
|
|
422
|
+
elsif ch == "l" || ch == "L"
|
|
423
|
+
advance
|
|
424
|
+
"l"
|
|
425
|
+
else
|
|
426
|
+
""
|
|
427
|
+
end
|
|
428
|
+
end
|
|
429
|
+
|
|
430
|
+
# Consumes an integer constant's u/U and l/L suffix run and returns it in a
|
|
431
|
+
# normalized (lower-case) form ("", "u", "l", "ul", "ll", "ull", ...). A
|
|
432
|
+
# valid suffix is at most one "u" together with at most one l-part
|
|
433
|
+
# ("l"/"L" or "ll"/"LL", never mixed case), in either order; anything else
|
|
434
|
+
# is rejected. `start` locates the constant for a positioned error.
|
|
435
|
+
def read_integer_suffix(start)
|
|
436
|
+
raw = +""
|
|
437
|
+
raw << advance while current&.match?(/[uUlL]/)
|
|
438
|
+
return "" if raw.empty?
|
|
439
|
+
|
|
440
|
+
valid = raw.match?(/\A(?:[uU])?(?:ll|LL|[lL])?\z/) ||
|
|
441
|
+
raw.match?(/\A(?:ll|LL|[lL])?(?:[uU])?\z/)
|
|
442
|
+
raise LexError.new("invalid suffix #{raw.inspect} on integer constant", start) unless valid
|
|
443
|
+
|
|
444
|
+
raw.downcase
|
|
445
|
+
end
|
|
446
|
+
|
|
447
|
+
def digit?(ch)
|
|
448
|
+
!ch.nil? && ch.match?(/[0-9]/)
|
|
449
|
+
end
|
|
450
|
+
|
|
451
|
+
def octal_digit?(ch)
|
|
452
|
+
!ch.nil? && ch.match?(/[0-7]/)
|
|
453
|
+
end
|
|
454
|
+
|
|
455
|
+
def hex_digit?(ch)
|
|
456
|
+
!ch.nil? && ch.match?(/[0-9A-Fa-f]/)
|
|
457
|
+
end
|
|
458
|
+
end
|
|
459
|
+
end
|
|
460
|
+
end
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "token"
|
|
4
|
+
require_relative "lexeme_reader"
|
|
5
|
+
require_relative "../compile_error"
|
|
6
|
+
|
|
7
|
+
module Rubycc
|
|
8
|
+
module Front
|
|
9
|
+
# Hand-written lexer for the C subset. Tracks 1-based line/column positions
|
|
10
|
+
# and keeps each source line around so tokens (and errors) can be reported
|
|
11
|
+
# with source excerpts. Handles // and /* */ comments and whitespace. The
|
|
12
|
+
# spelling-to-value decoding of numbers, string/character literals and
|
|
13
|
+
# identifiers is delegated to the shared LexemeReader, which the preprocessor
|
|
14
|
+
# reuses so both token sources agree exactly.
|
|
15
|
+
class Lexer
|
|
16
|
+
def initialize(source, filename:)
|
|
17
|
+
@src = source
|
|
18
|
+
@filename = filename
|
|
19
|
+
@pos = 0
|
|
20
|
+
@line = 1
|
|
21
|
+
@column = 1
|
|
22
|
+
# -1 keeps a trailing empty field, so line numbers map 1:1 to entries.
|
|
23
|
+
@lines = source.split("\n", -1)
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def tokenize
|
|
27
|
+
tokens = []
|
|
28
|
+
loop do
|
|
29
|
+
skip_whitespace_and_comments
|
|
30
|
+
break if at_end?
|
|
31
|
+
|
|
32
|
+
tokens << next_token
|
|
33
|
+
end
|
|
34
|
+
tokens << make_token(:eof, nil, @line, @column)
|
|
35
|
+
tokens
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
private
|
|
39
|
+
|
|
40
|
+
def at_end?
|
|
41
|
+
@pos >= @src.length
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def current_char
|
|
45
|
+
@src[@pos]
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def peek(offset = 0)
|
|
49
|
+
@src[@pos + offset]
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# Consumes one character, maintaining line/column bookkeeping.
|
|
53
|
+
def advance
|
|
54
|
+
ch = @src[@pos]
|
|
55
|
+
@pos += 1
|
|
56
|
+
if ch == "\n"
|
|
57
|
+
@line += 1
|
|
58
|
+
@column = 1
|
|
59
|
+
else
|
|
60
|
+
@column += 1
|
|
61
|
+
end
|
|
62
|
+
ch
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def skip_whitespace_and_comments
|
|
66
|
+
loop do
|
|
67
|
+
ch = current_char
|
|
68
|
+
if ch.nil?
|
|
69
|
+
return
|
|
70
|
+
elsif ch =~ /\s/
|
|
71
|
+
advance
|
|
72
|
+
elsif ch == "/" && peek(1) == "/"
|
|
73
|
+
advance until at_end? || current_char == "\n"
|
|
74
|
+
elsif ch == "/" && peek(1) == "*"
|
|
75
|
+
skip_block_comment
|
|
76
|
+
else
|
|
77
|
+
return
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def skip_block_comment
|
|
83
|
+
start_line = @line
|
|
84
|
+
start_col = @column
|
|
85
|
+
advance # /
|
|
86
|
+
advance # *
|
|
87
|
+
loop do
|
|
88
|
+
if at_end?
|
|
89
|
+
raise_error("unterminated block comment", start_line, start_col)
|
|
90
|
+
elsif current_char == "*" && peek(1) == "/"
|
|
91
|
+
advance # *
|
|
92
|
+
advance # /
|
|
93
|
+
return
|
|
94
|
+
else
|
|
95
|
+
advance
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
def next_token
|
|
101
|
+
line = @line
|
|
102
|
+
column = @column
|
|
103
|
+
ch = current_char
|
|
104
|
+
|
|
105
|
+
if ch =~ /[0-9]/
|
|
106
|
+
read_lexeme(:read_number, line, column)
|
|
107
|
+
elsif ch == "." && peek(1) =~ /[0-9]/
|
|
108
|
+
# A leading-dot floating constant (".5"): a "." immediately followed
|
|
109
|
+
# by a digit is the fractional-only form. A "." in any other position
|
|
110
|
+
# is the member-access punctuator (or part of "..."), handled by
|
|
111
|
+
# #lex_punctuator.
|
|
112
|
+
read_lexeme(:read_number, line, column)
|
|
113
|
+
elsif ch =~ /[A-Za-z_]/
|
|
114
|
+
read_lexeme(:read_identifier, line, column)
|
|
115
|
+
elsif ch == "'"
|
|
116
|
+
read_lexeme(:read_char, line, column)
|
|
117
|
+
elsif ch == "\""
|
|
118
|
+
read_lexeme(:read_string, line, column)
|
|
119
|
+
else
|
|
120
|
+
lex_punctuator(line, column)
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
# Decodes one lexeme with the shared LexemeReader, then walks the streaming
|
|
125
|
+
# cursor over the exact characters it consumed so line/column bookkeeping
|
|
126
|
+
# stays correct. A LexError from the reader carries an absolute offset into
|
|
127
|
+
# the source, which maps to a column on this (single) line since none of
|
|
128
|
+
# these lexemes span a newline.
|
|
129
|
+
def read_lexeme(method, line, column)
|
|
130
|
+
start = @pos
|
|
131
|
+
reader = LexemeReader.new(@src, start)
|
|
132
|
+
begin
|
|
133
|
+
result = reader.public_send(method)
|
|
134
|
+
rescue LexError => e
|
|
135
|
+
raise_error(e.message, line, column + (e.offset - start))
|
|
136
|
+
end
|
|
137
|
+
advance while @pos < reader.pos
|
|
138
|
+
build_token(result, line, column)
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
def build_token(result, line, column)
|
|
142
|
+
case result.type
|
|
143
|
+
when :num
|
|
144
|
+
make_num_token(result.value, result.base, result.suffix, line, column)
|
|
145
|
+
when :float
|
|
146
|
+
make_float_token(result.value, result.suffix, line, column)
|
|
147
|
+
else
|
|
148
|
+
make_token(result.type, result.value, line, column)
|
|
149
|
+
end
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
# Longest-match punctuator scan: try the three-character punctuators
|
|
153
|
+
# first, then the two-character ones, then fall back to the
|
|
154
|
+
# single-character ones. peek(1)/peek(2) yield nil past the end, which
|
|
155
|
+
# interpolate to "", so the assembled candidates simply fail to match near
|
|
156
|
+
# EOF and the scan falls through to a shorter length.
|
|
157
|
+
def lex_punctuator(line, column)
|
|
158
|
+
three = "#{current_char}#{peek(1)}#{peek(2)}"
|
|
159
|
+
two = "#{current_char}#{peek(1)}"
|
|
160
|
+
if LexemeReader::PUNCTUATORS_3.include?(three)
|
|
161
|
+
advance
|
|
162
|
+
advance
|
|
163
|
+
advance
|
|
164
|
+
make_token(:punct, three, line, column)
|
|
165
|
+
elsif LexemeReader::PUNCTUATORS_2.include?(two)
|
|
166
|
+
advance
|
|
167
|
+
advance
|
|
168
|
+
make_token(:punct, two, line, column)
|
|
169
|
+
elsif LexemeReader::PUNCTUATORS_1.include?(current_char)
|
|
170
|
+
ch = advance
|
|
171
|
+
make_token(:punct, ch, line, column)
|
|
172
|
+
else
|
|
173
|
+
raise_error("unexpected character #{current_char.inspect}", line, column)
|
|
174
|
+
end
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
def make_token(type, value, line, column)
|
|
178
|
+
Token.new(
|
|
179
|
+
type: type,
|
|
180
|
+
value: value,
|
|
181
|
+
filename: @filename,
|
|
182
|
+
line: line,
|
|
183
|
+
column: column,
|
|
184
|
+
source_line: source_line_for(line)
|
|
185
|
+
)
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
# A :num token carrying the extra base/suffix an integer constant needs
|
|
189
|
+
# for the parser to fix its type.
|
|
190
|
+
def make_num_token(value, base, suffix, line, column)
|
|
191
|
+
Token.new(
|
|
192
|
+
type: :num,
|
|
193
|
+
value: value,
|
|
194
|
+
filename: @filename,
|
|
195
|
+
line: line,
|
|
196
|
+
column: column,
|
|
197
|
+
source_line: source_line_for(line),
|
|
198
|
+
base: base,
|
|
199
|
+
suffix: suffix
|
|
200
|
+
)
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
# A :float token carrying the folded Ruby Float value and the normalized
|
|
204
|
+
# floating suffix the parser fixes the constant's type from.
|
|
205
|
+
def make_float_token(value, suffix, line, column)
|
|
206
|
+
Token.new(
|
|
207
|
+
type: :float,
|
|
208
|
+
value: value,
|
|
209
|
+
filename: @filename,
|
|
210
|
+
line: line,
|
|
211
|
+
column: column,
|
|
212
|
+
source_line: source_line_for(line),
|
|
213
|
+
suffix: suffix
|
|
214
|
+
)
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
def source_line_for(line)
|
|
218
|
+
@lines[line - 1] || ""
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
def raise_error(description, line, column)
|
|
222
|
+
raise CompileError.new(
|
|
223
|
+
description,
|
|
224
|
+
filename: @filename,
|
|
225
|
+
line: line,
|
|
226
|
+
column: column,
|
|
227
|
+
source_line: source_line_for(line)
|
|
228
|
+
)
|
|
229
|
+
end
|
|
230
|
+
end
|
|
231
|
+
end
|
|
232
|
+
end
|