rubycc 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +56 -0
- data/LICENSE.txt +21 -0
- data/NOTICE +52 -0
- data/README.md +208 -0
- data/data/README.md +117 -0
- data/data/r10_corpus_scan.json +4082 -0
- data/data/r10_manual_classification.json +5342 -0
- data/data/r10_verification_m4a.json +1531 -0
- data/data/r10_verification_m4b.json +1818 -0
- data/data/r10_verification_m4c.json +1489 -0
- data/data/r10_verification_m4d.json +318 -0
- data/data/r10_verification_rbs.json +192 -0
- data/data/verified_gems.json +397 -0
- data/exe/rmake +16 -0
- data/exe/rubycc +9 -0
- data/exe/rubycc-ar +114 -0
- data/exe/rubycc-doctor +14 -0
- data/exe/rubycc-pkgconf +8 -0
- data/include/float.h +87 -0
- data/include/iso646.h +18 -0
- data/include/libc/alloca.h +18 -0
- data/include/libc/arpa/inet.h +61 -0
- data/include/libc/assert.h +43 -0
- data/include/libc/dirent.h +72 -0
- data/include/libc/dlfcn.h +58 -0
- data/include/libc/features.h +205 -0
- data/include/libc/glibc/aarch64/ctype.h +108 -0
- data/include/libc/glibc/aarch64/endian.h +54 -0
- data/include/libc/glibc/aarch64/errno.h +154 -0
- data/include/libc/glibc/aarch64/fcntl.h +144 -0
- data/include/libc/glibc/aarch64/inttypes.h +182 -0
- data/include/libc/glibc/aarch64/limits.h +63 -0
- data/include/libc/glibc/aarch64/pthread.h +131 -0
- data/include/libc/glibc/aarch64/setjmp.h +84 -0
- data/include/libc/glibc/aarch64/stdint.h +174 -0
- data/include/libc/glibc/aarch64/sys/epoll.h +88 -0
- data/include/libc/glibc/aarch64/sys/fcntl.h +19 -0
- data/include/libc/glibc/aarch64/sys/select.h +75 -0
- data/include/libc/glibc/aarch64/sys/stat.h +136 -0
- data/include/libc/glibc/aarch64/sys/syscall.h +167 -0
- data/include/libc/glibc/aarch64/sys/time.h +74 -0
- data/include/libc/glibc/aarch64/sys/types.h +127 -0
- data/include/libc/glibc/aarch64/time.h +113 -0
- data/include/libc/glibc/x86_64/ctype.h +108 -0
- data/include/libc/glibc/x86_64/endian.h +54 -0
- data/include/libc/glibc/x86_64/errno.h +154 -0
- data/include/libc/glibc/x86_64/fcntl.h +134 -0
- data/include/libc/glibc/x86_64/inttypes.h +182 -0
- data/include/libc/glibc/x86_64/limits.h +61 -0
- data/include/libc/glibc/x86_64/pthread.h +111 -0
- data/include/libc/glibc/x86_64/setjmp.h +82 -0
- data/include/libc/glibc/x86_64/stdint.h +163 -0
- data/include/libc/glibc/x86_64/sys/epoll.h +91 -0
- data/include/libc/glibc/x86_64/sys/fcntl.h +19 -0
- data/include/libc/glibc/x86_64/sys/select.h +75 -0
- data/include/libc/glibc/x86_64/sys/stat.h +130 -0
- data/include/libc/glibc/x86_64/sys/syscall.h +188 -0
- data/include/libc/glibc/x86_64/sys/time.h +74 -0
- data/include/libc/glibc/x86_64/sys/types.h +123 -0
- data/include/libc/glibc/x86_64/time.h +113 -0
- data/include/libc/grp.h +50 -0
- data/include/libc/langinfo.h +126 -0
- data/include/libc/link.h +18 -0
- data/include/libc/locale.h +83 -0
- data/include/libc/math.h +191 -0
- data/include/libc/netinet/in.h +126 -0
- data/include/libc/netinet/tcp.h +44 -0
- data/include/libc/poll.h +46 -0
- data/include/libc/pwd.h +58 -0
- data/include/libc/regex.h +53 -0
- data/include/libc/sched.h +35 -0
- data/include/libc/signal.h +204 -0
- data/include/libc/stdio.h +157 -0
- data/include/libc/stdlib.h +92 -0
- data/include/libc/string.h +74 -0
- data/include/libc/strings.h +25 -0
- data/include/libc/sys/cdefs.h +126 -0
- data/include/libc/sys/inotify.h +111 -0
- data/include/libc/sys/ioctl.h +36 -0
- data/include/libc/sys/mman.h +65 -0
- data/include/libc/sys/param.h +41 -0
- data/include/libc/sys/resource.h +109 -0
- data/include/libc/sys/socket.h +167 -0
- data/include/libc/sys/statfs.h +82 -0
- data/include/libc/sys/timerfd.h +55 -0
- data/include/libc/sys/uio.h +40 -0
- data/include/libc/sys/un.h +25 -0
- data/include/libc/sys/utsname.h +35 -0
- data/include/libc/sys/wait.h +135 -0
- data/include/libc/termios.h +179 -0
- data/include/libc/unistd.h +194 -0
- data/include/stdalign.h +16 -0
- data/include/stdarg.h +31 -0
- data/include/stdatomic.h +158 -0
- data/include/stdbool.h +15 -0
- data/include/stdckdint.h +28 -0
- data/include/stddef.h +60 -0
- data/include/stdnoreturn.h +18 -0
- data/include/x86intrin.h +16 -0
- data/lib/rubycc/backend/aarch64.rb +1724 -0
- data/lib/rubycc/backend/x86_64.rb +1369 -0
- data/lib/rubycc/compile_error.rb +36 -0
- data/lib/rubycc/compiler.rb +305 -0
- data/lib/rubycc/doctor/builder.rb +151 -0
- data/lib/rubycc/doctor/cli.rb +190 -0
- data/lib/rubycc/doctor/fetcher.rb +84 -0
- data/lib/rubycc/doctor/gemfile.rb +115 -0
- data/lib/rubycc/doctor/verified_gems.rb +114 -0
- data/lib/rubycc/doctor.rb +16 -0
- data/lib/rubycc/driver.rb +463 -0
- data/lib/rubycc/front/ast.rb +528 -0
- data/lib/rubycc/front/constant_evaluator.rb +631 -0
- data/lib/rubycc/front/initializer_resolver.rb +592 -0
- data/lib/rubycc/front/lexeme_reader.rb +460 -0
- data/lib/rubycc/front/lexer.rb +232 -0
- data/lib/rubycc/front/parser.rb +4122 -0
- data/lib/rubycc/front/token.rb +49 -0
- data/lib/rubycc/ir/call_convention.rb +486 -0
- data/lib/rubycc/ir/generator.rb +6036 -0
- data/lib/rubycc/ir/ir.rb +417 -0
- data/lib/rubycc/link/compat_runtime.rb +89 -0
- data/lib/rubycc/link/errors.rb +13 -0
- data/lib/rubycc/link/executable_linker.rb +405 -0
- data/lib/rubycc/link/library_resolver.rb +437 -0
- data/lib/rubycc/link/partial_linker.rb +546 -0
- data/lib/rubycc/link/shared_linker.rb +1732 -0
- data/lib/rubycc/mkmf_shim.rb +83 -0
- data/lib/rubycc/objfile/ar_archive.rb +437 -0
- data/lib/rubycc/objfile/elf_reader.rb +646 -0
- data/lib/rubycc/objfile/elf_writer.rb +891 -0
- data/lib/rubycc/objfile/relocatable_writer.rb +376 -0
- data/lib/rubycc/pkgconf/cli.rb +139 -0
- data/lib/rubycc/pkgconf/errors.rb +39 -0
- data/lib/rubycc/pkgconf/model.rb +28 -0
- data/lib/rubycc/pkgconf/parser.rb +109 -0
- data/lib/rubycc/pkgconf/pkgconf.rb +15 -0
- data/lib/rubycc/pkgconf/resolver.rb +70 -0
- data/lib/rubycc/pkgconf/search_path.rb +38 -0
- data/lib/rubycc/pkgconf/system_path_filter.rb +131 -0
- data/lib/rubycc/preprocess/constant_expression.rb +219 -0
- data/lib/rubycc/preprocess/glibc_version.rb +124 -0
- data/lib/rubycc/preprocess/pp_token.rb +62 -0
- data/lib/rubycc/preprocess/preprocessor.rb +2020 -0
- data/lib/rubycc/preprocess/scanner.rb +290 -0
- data/lib/rubycc/preprocess/token_converter.rb +157 -0
- data/lib/rubycc/rmake/cli.rb +143 -0
- data/lib/rubycc/rmake/errors.rb +71 -0
- data/lib/rubycc/rmake/executor.rb +818 -0
- data/lib/rubycc/rmake/expander.rb +251 -0
- data/lib/rubycc/rmake/makefile.rb +352 -0
- data/lib/rubycc/rmake/model.rb +137 -0
- data/lib/rubycc/rmake/parser.rb +226 -0
- data/lib/rubycc/rmake/rmake.rb +15 -0
- data/lib/rubycc/type.rb +1236 -0
- data/lib/rubycc/version.rb +5 -0
- data/lib/rubycc.rb +36 -0
- data/lib/rubygems_plugin.rb +102 -0
- metadata +219 -0
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "strscan"
|
|
4
|
+
|
|
5
|
+
require_relative "pp_token"
|
|
6
|
+
require_relative "../front/lexeme_reader"
|
|
7
|
+
require_relative "../compile_error"
|
|
8
|
+
|
|
9
|
+
module Rubycc
|
|
10
|
+
module Preprocess
|
|
11
|
+
# Turns raw source into a stream of preprocessing tokens, carrying out
|
|
12
|
+
# translation phases 2 and 3: backslash-newline line splicing and the
|
|
13
|
+
# replacement of comments by whitespace. Newlines survive as explicit tokens
|
|
14
|
+
# so the directive layer (added later) stays line-oriented, and every token
|
|
15
|
+
# keeps the physical location of its first character.
|
|
16
|
+
#
|
|
17
|
+
# Splicing works by deleting every backslash-newline pair up front and
|
|
18
|
+
# remembering where each deletion happened (@splice_points): tokens are then
|
|
19
|
+
# matched over the spliced text with StringScanner-driven regexps — one
|
|
20
|
+
# match per token instead of a method call per character, which is what
|
|
21
|
+
# makes the preprocessor's dominant cost scale. A token's physical
|
|
22
|
+
# line/column stays truthful because the recorded splice points replay the
|
|
23
|
+
# deleted line breaks: everything at or past a splice point is located on
|
|
24
|
+
# the following physical line (see #sync). Positions are byte offsets
|
|
25
|
+
# (StringScanner's native unit); columns are converted back to character
|
|
26
|
+
# counts only when a token is actually made, and only for non-ASCII source.
|
|
27
|
+
class Scanner
|
|
28
|
+
# Two- and one-character punctuators gain the preprocessor-only "##" and
|
|
29
|
+
# "#" over the shared punctuator tables; the three-character set is unchanged.
|
|
30
|
+
PP_PUNCTUATORS_3 = Front::LexemeReader::PUNCTUATORS_3
|
|
31
|
+
PP_PUNCTUATORS_2 = (Front::LexemeReader::PUNCTUATORS_2 + ["##"]).freeze
|
|
32
|
+
PP_PUNCTUATORS_1 = (Front::LexemeReader::PUNCTUATORS_1 + ["#"]).freeze
|
|
33
|
+
|
|
34
|
+
# Longest alternative first, so one anchored attempt is a longest match.
|
|
35
|
+
PUNCTUATOR_RE = Regexp.union(PP_PUNCTUATORS_3 + PP_PUNCTUATORS_2 + PP_PUNCTUATORS_1)
|
|
36
|
+
|
|
37
|
+
HORIZONTAL_WS_RE = /[ \t\r\v\f]+/
|
|
38
|
+
# A // comment runs to (but not over) the end of the physical line; a
|
|
39
|
+
# trailing backslash-newline was already spliced away, so the comment
|
|
40
|
+
# continues onto the next line just as it does under gcc.
|
|
41
|
+
LINE_COMMENT_RE = %r{//[^\n]*}
|
|
42
|
+
BLOCK_OPEN_RE = %r{/\*}
|
|
43
|
+
BLOCK_CLOSE_RE = %r{\*/}
|
|
44
|
+
NEWLINE_RE = /\n/
|
|
45
|
+
|
|
46
|
+
IDENTIFIER_RE = /[A-Za-z_][A-Za-z0-9_]*/
|
|
47
|
+
# A preprocessing number (6.4.8): a digit, or a "." then a digit, and then
|
|
48
|
+
# digits, identifier characters, ".", and a sign immediately after an
|
|
49
|
+
# 'e'/'E'/'p'/'P'. Whether the run is a valid C constant is decided later,
|
|
50
|
+
# at conversion time.
|
|
51
|
+
PP_NUMBER_RE = /(?:[0-9]|\.[0-9])(?:[eEpP][+-]|[0-9A-Za-z_.])*/
|
|
52
|
+
|
|
53
|
+
# A string literal or character constant, kept verbatim (quotes and
|
|
54
|
+
# escapes included). Only the boundary is recognized here, matching the
|
|
55
|
+
# streaming lexer: a backslash shields the following character so an
|
|
56
|
+
# escaped quote does not close the literal. An unterminated literal keeps
|
|
57
|
+
# whatever was read (the trailing `("|\)?` picks up a lone backslash cut
|
|
58
|
+
# off by end-of-line or end-of-file); the converter re-scans it and raises
|
|
59
|
+
# the positioned error.
|
|
60
|
+
STRING_RE = /"(?:[^"\\\n]|\\[^\n])*(?:"|\\)?/
|
|
61
|
+
CHAR_RE = /'(?:[^'\\\n]|\\[^\n])*(?:'|\\)?/
|
|
62
|
+
# A wide character constant L'c' or wide string literal L"..." (6.4.4.4,
|
|
63
|
+
# 6.4.5): recognized only when the "L" abuts the quote, so an identifier
|
|
64
|
+
# named "L" is unaffected. The "L" is kept in the spelling so the converter
|
|
65
|
+
# can tell a wide literal from a plain one; it decides a wide character's
|
|
66
|
+
# value (int here, as for a plain constant) and rejects a wide string. The
|
|
67
|
+
# u/U/u8 prefixes are out of scope, so they still scan as an identifier.
|
|
68
|
+
WIDE_CHAR_RE = /L'(?:[^'\\\n]|\\[^\n])*(?:'|\\)?/
|
|
69
|
+
WIDE_STRING_RE = /L"(?:[^"\\\n]|\\[^\n])*(?:"|\\)?/
|
|
70
|
+
|
|
71
|
+
def initialize(source, filename:)
|
|
72
|
+
@filename = filename
|
|
73
|
+
# -1 keeps a trailing empty field so line numbers map 1:1 to entries.
|
|
74
|
+
@lines = source.split("\n", -1)
|
|
75
|
+
splice(source)
|
|
76
|
+
@scanner = StringScanner.new(@spliced)
|
|
77
|
+
@ascii_only = @spliced.ascii_only?
|
|
78
|
+
@line = 1
|
|
79
|
+
# Byte offset in @spliced where the current physical line starts; a
|
|
80
|
+
# column is the distance from here (plus one). @column_pos/@column_chars
|
|
81
|
+
# are the incremental cursor #column_at advances (see there); they are
|
|
82
|
+
# only meaningful for non-ASCII source, and reset_line_start resets all
|
|
83
|
+
# three together so they can never drift apart.
|
|
84
|
+
reset_line_start(0)
|
|
85
|
+
# Index into @splice_points of the first point not yet replayed.
|
|
86
|
+
@next_splice = 0
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def scan
|
|
90
|
+
tokens = []
|
|
91
|
+
ss = @scanner
|
|
92
|
+
loop do
|
|
93
|
+
before = ss.pos
|
|
94
|
+
skip_whitespace_and_comments(ss, tokens)
|
|
95
|
+
# Whether any whitespace or comment separated this token from the
|
|
96
|
+
# previous one on the same logical line. The directive layer needs it
|
|
97
|
+
# to tell "#define F(x)" (function-like) from "#define F (x)" (an
|
|
98
|
+
# object macro whose replacement begins with a parenthesis).
|
|
99
|
+
space_before = ss.pos != before
|
|
100
|
+
break if ss.eos?
|
|
101
|
+
|
|
102
|
+
start = ss.pos
|
|
103
|
+
sync(start)
|
|
104
|
+
if ss.skip(NEWLINE_RE)
|
|
105
|
+
tokens << make_token(:newline, "\n", @line, column_at(start), space_before)
|
|
106
|
+
@line += 1
|
|
107
|
+
reset_line_start(ss.pos)
|
|
108
|
+
else
|
|
109
|
+
tokens << scan_token(ss, @line, column_at(start), space_before)
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
sync(@spliced.bytesize)
|
|
113
|
+
tokens << make_token(:eof, nil, @line, column_at(@spliced.bytesize))
|
|
114
|
+
tokens
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
private
|
|
118
|
+
|
|
119
|
+
# Translation phase 2: delete every backslash-newline pair, recording for
|
|
120
|
+
# each deletion the byte offset (in the spliced text) where the next
|
|
121
|
+
# physical line begins. Two adjacent continuations record the same offset
|
|
122
|
+
# twice — each still advances the line counter once when replayed. A
|
|
123
|
+
# source with no continuations (the common case) is shared, not copied.
|
|
124
|
+
def splice(source)
|
|
125
|
+
@splice_points = []
|
|
126
|
+
unless source.include?("\\\n")
|
|
127
|
+
@spliced = source
|
|
128
|
+
return
|
|
129
|
+
end
|
|
130
|
+
spliced = +""
|
|
131
|
+
pos = 0
|
|
132
|
+
while (idx = source.index("\\\n", pos))
|
|
133
|
+
spliced << source[pos...idx]
|
|
134
|
+
@splice_points << spliced.bytesize
|
|
135
|
+
pos = idx + 2
|
|
136
|
+
end
|
|
137
|
+
spliced << source[pos..]
|
|
138
|
+
@spliced = spliced
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
# Replays every splice point at or before byte offset `pos`: each one
|
|
142
|
+
# marks a physical line break the deletion hid, so the line counter
|
|
143
|
+
# advances and the line start moves to the point itself (its first
|
|
144
|
+
# character is column 1 of the continued-onto line). Called with a token's
|
|
145
|
+
# start offset before its location is read, and with interior offsets by
|
|
146
|
+
# the block-comment walk; a point strictly inside a token is replayed when
|
|
147
|
+
# the next token is located, which yields the same stream (a token is
|
|
148
|
+
# located only by its first character).
|
|
149
|
+
def sync(pos)
|
|
150
|
+
points = @splice_points
|
|
151
|
+
while @next_splice < points.length && points[@next_splice] <= pos
|
|
152
|
+
@line += 1
|
|
153
|
+
reset_line_start(points[@next_splice])
|
|
154
|
+
@next_splice += 1
|
|
155
|
+
end
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
# Moves the current physical line's start to byte offset `pos`. Every
|
|
159
|
+
# assignment to @line_start goes through here so the column cursor is
|
|
160
|
+
# rebased at the same time — a missed reset would not crash, it would
|
|
161
|
+
# silently report wrong columns (N3).
|
|
162
|
+
def reset_line_start(pos)
|
|
163
|
+
@line_start = pos
|
|
164
|
+
@column_pos = pos
|
|
165
|
+
@column_chars = 0
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
# The 1-based column of byte offset `pos`, counted in characters from the
|
|
169
|
+
# current physical line's start. Byte arithmetic serves ASCII source
|
|
170
|
+
# directly; non-ASCII source has to count characters, and must count them
|
|
171
|
+
# *incrementally*: measuring the whole @line_start..pos span per token
|
|
172
|
+
# costs O(line length) each time, so a single long line of L tokens costs
|
|
173
|
+
# O(L^2) — and one non-ASCII byte anywhere in the file (a comment in
|
|
174
|
+
# Japanese, a UTF-8 BOM) is enough to switch @ascii_only off for all of
|
|
175
|
+
# it. Tokens are located left to right, so keeping the last (byte offset,
|
|
176
|
+
# column) pair as a cursor and counting only the span since then makes a
|
|
177
|
+
# line cost O(line length) in total. Should a caller ever ask for an
|
|
178
|
+
# offset behind the cursor, the count restarts from the line start, so the
|
|
179
|
+
# answer stays right even if that monotonicity is lost.
|
|
180
|
+
def column_at(pos)
|
|
181
|
+
return pos - @line_start + 1 if @ascii_only
|
|
182
|
+
|
|
183
|
+
if pos < @column_pos
|
|
184
|
+
@column_pos = @line_start
|
|
185
|
+
@column_chars = 0
|
|
186
|
+
end
|
|
187
|
+
if pos > @column_pos
|
|
188
|
+
@column_chars += @spliced.byteslice(@column_pos, pos - @column_pos).length
|
|
189
|
+
@column_pos = pos
|
|
190
|
+
end
|
|
191
|
+
@column_chars + 1
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
# Skips horizontal whitespace and comments (translation phase 3). Newlines
|
|
195
|
+
# are left for the caller so they become tokens; a newline retained inside
|
|
196
|
+
# a block comment is emitted here directly, keeping the logical line count
|
|
197
|
+
# exact for later directive processing.
|
|
198
|
+
def skip_whitespace_and_comments(ss, tokens)
|
|
199
|
+
loop do
|
|
200
|
+
next if ss.skip(HORIZONTAL_WS_RE)
|
|
201
|
+
next if ss.skip(LINE_COMMENT_RE)
|
|
202
|
+
return unless ss.match?(BLOCK_OPEN_RE)
|
|
203
|
+
|
|
204
|
+
skip_block_comment(ss, tokens)
|
|
205
|
+
end
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
# A /* */ comment, which may span physical lines. Each retained newline it
|
|
209
|
+
# crosses is emitted as a token so the line count stays consistent; a
|
|
210
|
+
# spliced-away continuation inside it adds no token, only a line. An
|
|
211
|
+
# unterminated comment is a hard error, positioned at the opening "/*".
|
|
212
|
+
def skip_block_comment(ss, tokens)
|
|
213
|
+
sync(ss.pos)
|
|
214
|
+
start_line = @line
|
|
215
|
+
start_column = column_at(ss.pos)
|
|
216
|
+
ss.skip(BLOCK_OPEN_RE)
|
|
217
|
+
body = ss.scan_until(BLOCK_CLOSE_RE)
|
|
218
|
+
raise_error("unterminated block comment", start_line, start_column) if body.nil?
|
|
219
|
+
|
|
220
|
+
base = ss.pos - body.bytesize
|
|
221
|
+
from = 0
|
|
222
|
+
while (idx = body.byteindex("\n", from))
|
|
223
|
+
at = base + idx
|
|
224
|
+
sync(at)
|
|
225
|
+
tokens << make_token(:newline, "\n", @line, column_at(at))
|
|
226
|
+
@line += 1
|
|
227
|
+
reset_line_start(at + 1)
|
|
228
|
+
from = idx + 1
|
|
229
|
+
end
|
|
230
|
+
end
|
|
231
|
+
|
|
232
|
+
# Scans a single preprocessing token (never a newline or whitespace),
|
|
233
|
+
# tagged with the physical location of its first character. One anchored
|
|
234
|
+
# regexp attempt per candidate class, ordered so a prefix can never steal
|
|
235
|
+
# a longer token: a pp-number before "." the punctuator, a wide literal
|
|
236
|
+
# before the identifier "L".
|
|
237
|
+
def scan_token(ss, line, column, space_before)
|
|
238
|
+
if (text = ss.scan(PP_NUMBER_RE))
|
|
239
|
+
type = :pp_number
|
|
240
|
+
elsif (text = ss.scan(IDENTIFIER_RE))
|
|
241
|
+
type = :identifier
|
|
242
|
+
# An "L" abutting a quote is a wide literal, not an identifier.
|
|
243
|
+
if text == "L" && (quoted = ss.scan(CHAR_RE) || ss.scan(STRING_RE))
|
|
244
|
+
type = quoted.start_with?("'") ? :char : :string
|
|
245
|
+
text = "L" + quoted
|
|
246
|
+
end
|
|
247
|
+
elsif (text = ss.scan(CHAR_RE))
|
|
248
|
+
type = :char
|
|
249
|
+
elsif (text = ss.scan(STRING_RE))
|
|
250
|
+
type = :string
|
|
251
|
+
elsif (text = ss.scan(PUNCTUATOR_RE))
|
|
252
|
+
type = :punct
|
|
253
|
+
else
|
|
254
|
+
# Anything that matches nothing above becomes a single-character
|
|
255
|
+
# :other token (6.4p1), deferring the decision of whether it is an
|
|
256
|
+
# error to a later phase.
|
|
257
|
+
type = :other
|
|
258
|
+
text = ss.getch
|
|
259
|
+
end
|
|
260
|
+
make_token(type, text, line, column, space_before)
|
|
261
|
+
end
|
|
262
|
+
|
|
263
|
+
def make_token(type, text, line, column, space_before = false)
|
|
264
|
+
PPToken.new(
|
|
265
|
+
type: type,
|
|
266
|
+
text: text,
|
|
267
|
+
filename: @filename,
|
|
268
|
+
line: line,
|
|
269
|
+
column: column,
|
|
270
|
+
source_line: source_line_for(line),
|
|
271
|
+
space_before: space_before
|
|
272
|
+
)
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
def source_line_for(line)
|
|
276
|
+
@lines[line - 1] || ""
|
|
277
|
+
end
|
|
278
|
+
|
|
279
|
+
def raise_error(description, line, column)
|
|
280
|
+
raise CompileError.new(
|
|
281
|
+
description,
|
|
282
|
+
filename: @filename,
|
|
283
|
+
line: line,
|
|
284
|
+
column: column,
|
|
285
|
+
source_line: source_line_for(line)
|
|
286
|
+
)
|
|
287
|
+
end
|
|
288
|
+
end
|
|
289
|
+
end
|
|
290
|
+
end
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "pp_token"
|
|
4
|
+
require_relative "../front/token"
|
|
5
|
+
require_relative "../front/lexeme_reader"
|
|
6
|
+
require_relative "../compile_error"
|
|
7
|
+
|
|
8
|
+
module Rubycc
|
|
9
|
+
module Preprocess
|
|
10
|
+
# Ends translation phase 4 by turning preprocessing tokens into the
|
|
11
|
+
# Front::Token stream the parser consumes. Numbers are folded, string and
|
|
12
|
+
# character literals decoded, and identifiers classified as keywords, all via
|
|
13
|
+
# the shared LexemeReader so the result matches the streaming lexer exactly.
|
|
14
|
+
# Newlines are dropped here; a "#"/"##" that survives to this point is a
|
|
15
|
+
# stray operator with no macro meaning (directive lines are diagnosed
|
|
16
|
+
# earlier), so it is rejected.
|
|
17
|
+
#
|
|
18
|
+
# This is where translation phases 5-7 sit in this pipeline, so adjacent
|
|
19
|
+
# string-literal concatenation (phase 6, ISO C 6.4.5p5) happens here: a run
|
|
20
|
+
# of neighbouring string tokens folds into one, its bytes the per-literal
|
|
21
|
+
# decodings laid end to end. The streaming Front::Lexer takes the direct path
|
|
22
|
+
# and does not concatenate — nothing but this converter ever feeds the
|
|
23
|
+
# parser in the real compile pipeline (which always runs through the
|
|
24
|
+
# preprocessor), so the lexer's simpler single-literal path serves its
|
|
25
|
+
# unit-test-only role without the phase-6 fold.
|
|
26
|
+
class TokenConverter
|
|
27
|
+
def convert(pp_tokens)
|
|
28
|
+
tokens = []
|
|
29
|
+
index = 0
|
|
30
|
+
while index < pp_tokens.length
|
|
31
|
+
pp = pp_tokens[index]
|
|
32
|
+
case pp.type
|
|
33
|
+
when :newline
|
|
34
|
+
index += 1
|
|
35
|
+
when :eof
|
|
36
|
+
tokens << front_token(pp, :eof, nil)
|
|
37
|
+
index += 1
|
|
38
|
+
when :identifier
|
|
39
|
+
spelling = Front::LexemeReader.keyword_spelling(pp.text)
|
|
40
|
+
tokens << front_token(pp, spelling ? :keyword : :ident, spelling || pp.text)
|
|
41
|
+
index += 1
|
|
42
|
+
when :pp_number
|
|
43
|
+
tokens << convert_number(pp)
|
|
44
|
+
index += 1
|
|
45
|
+
when :string
|
|
46
|
+
value, index = concatenate_strings(pp_tokens, index)
|
|
47
|
+
tokens << front_token(pp, :string, value)
|
|
48
|
+
when :char
|
|
49
|
+
tokens << num_token(pp, decode_char(pp), 10, "")
|
|
50
|
+
index += 1
|
|
51
|
+
when :punct
|
|
52
|
+
if pp.text == "#" || pp.text == "##"
|
|
53
|
+
raise_at(pp, "stray '#' in program")
|
|
54
|
+
end
|
|
55
|
+
tokens << front_token(pp, :punct, pp.text)
|
|
56
|
+
index += 1
|
|
57
|
+
when :other
|
|
58
|
+
raise_at(pp, "unexpected character #{pp.text.inspect}")
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
tokens
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
private
|
|
65
|
+
|
|
66
|
+
# Folds the maximal run of adjacent string-literal tokens starting at
|
|
67
|
+
# `index` into a single value (translation phase 6). Each literal is decoded
|
|
68
|
+
# on its own, so an escape resolves within the literal that spells it —
|
|
69
|
+
# "\x41" "1" is the two bytes 'A' and '1', never one "\x411" digit run
|
|
70
|
+
# (6.4.5p4) — and the decoded bytes are concatenated. Intervening newline
|
|
71
|
+
# tokens are inter-token whitespace and do not break adjacency; in the
|
|
72
|
+
# compile pipeline the expander has already dropped them, so consecutive
|
|
73
|
+
# string tokens simply abut. Returns [concatenated-bytes, index-past-run].
|
|
74
|
+
def concatenate_strings(pp_tokens, index)
|
|
75
|
+
bytes = decode_string(pp_tokens[index])
|
|
76
|
+
index += 1
|
|
77
|
+
loop do
|
|
78
|
+
nxt = index
|
|
79
|
+
nxt += 1 while pp_tokens[nxt]&.type == :newline
|
|
80
|
+
break unless pp_tokens[nxt]&.type == :string
|
|
81
|
+
|
|
82
|
+
bytes += decode_string(pp_tokens[nxt])
|
|
83
|
+
index = nxt + 1
|
|
84
|
+
end
|
|
85
|
+
[bytes, index]
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
# The escape-resolved bytes of one string literal. A wide string literal
|
|
89
|
+
# (an "L" prefix) has element type wchar_t, which this subset cannot
|
|
90
|
+
# represent, so it is rejected rather than silently narrowed.
|
|
91
|
+
def decode_string(pp)
|
|
92
|
+
raise_at(pp, "wide string literals are not supported") if pp.text.start_with?("L")
|
|
93
|
+
|
|
94
|
+
decode(pp, pp.text) { |reader| reader.read_string }.value
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# A character constant's integer value. A wide character constant L'c' has
|
|
98
|
+
# type wchar_t (int on this target); for the single-byte characters this
|
|
99
|
+
# subset lexes its value equals the plain constant's, so the "L" prefix is
|
|
100
|
+
# dropped and the remainder decoded like an ordinary 'c'.
|
|
101
|
+
def decode_char(pp)
|
|
102
|
+
spelling = pp.text.start_with?("L") ? pp.text[1..] : pp.text
|
|
103
|
+
decode(pp, spelling) { |reader| reader.read_char }.value
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# Folds a preprocessing number into a :num or :float token. A pp-number is
|
|
107
|
+
# broader than a C constant, so an ill-formed one (e.g. "12abc", "1.2.3")
|
|
108
|
+
# is rejected here rather than at scan time.
|
|
109
|
+
def convert_number(pp)
|
|
110
|
+
result = decode(pp, pp.text) { |reader| reader.read_number }
|
|
111
|
+
if result.type == :float
|
|
112
|
+
Front::Token.new(
|
|
113
|
+
type: :float, value: result.value,
|
|
114
|
+
filename: pp.filename, line: pp.line, column: pp.column,
|
|
115
|
+
source_line: pp.source_line, suffix: result.suffix
|
|
116
|
+
)
|
|
117
|
+
else
|
|
118
|
+
num_token(pp, result.value, result.base, result.suffix)
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
# Runs a LexemeReader over `spelling` (the token's own text, or that text
|
|
123
|
+
# with a wide-literal "L" prefix already stripped), mapping any LexError
|
|
124
|
+
# onto the token's start position. A lexeme spliced across a line
|
|
125
|
+
# continuation is reported at its start, matching where it begins in source.
|
|
126
|
+
def decode(pp, spelling)
|
|
127
|
+
yield Front::LexemeReader.new(spelling)
|
|
128
|
+
rescue Front::LexError => e
|
|
129
|
+
raise_at(pp, e.message)
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def front_token(pp, type, value)
|
|
133
|
+
Front::Token.new(
|
|
134
|
+
type: type, value: value,
|
|
135
|
+
filename: pp.filename, line: pp.line, column: pp.column,
|
|
136
|
+
source_line: pp.source_line
|
|
137
|
+
)
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def num_token(pp, value, base, suffix)
|
|
141
|
+
Front::Token.new(
|
|
142
|
+
type: :num, value: value,
|
|
143
|
+
filename: pp.filename, line: pp.line, column: pp.column,
|
|
144
|
+
source_line: pp.source_line, base: base, suffix: suffix
|
|
145
|
+
)
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def raise_at(pp, description)
|
|
149
|
+
raise CompileError.new(
|
|
150
|
+
description,
|
|
151
|
+
filename: pp.filename, line: pp.line, column: pp.column,
|
|
152
|
+
source_line: pp.source_line
|
|
153
|
+
)
|
|
154
|
+
end
|
|
155
|
+
end
|
|
156
|
+
end
|
|
157
|
+
end
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "errors"
|
|
4
|
+
require_relative "makefile"
|
|
5
|
+
|
|
6
|
+
module Rubycc
|
|
7
|
+
module Rmake
|
|
8
|
+
# The command-line front end to rmake (M3 / ROADMAP §6 B6). It exists so
|
|
9
|
+
# RubyGems can drive rubycc's builds by setting `ENV["MAKE"]` to this program:
|
|
10
|
+
# Gem::Ext::Builder invokes `$(MAKE)` as three separate processes shaped like
|
|
11
|
+
#
|
|
12
|
+
# rmake DESTDIR= sitearchdir=<tmp> sitelibdir=<tmp> clean
|
|
13
|
+
# rmake DESTDIR= sitearchdir=<tmp> sitelibdir=<tmp> (default goal)
|
|
14
|
+
# rmake DESTDIR= sitearchdir=<tmp> sitelibdir=<tmp> install
|
|
15
|
+
#
|
|
16
|
+
# so the argument grammar it must accept is: command-line variable definitions
|
|
17
|
+
# (`VAR=value`, which override the Makefile's own assignments), zero or more
|
|
18
|
+
# target names (an absent target builds the default goal) and, though RubyGems
|
|
19
|
+
# does not pass them, the `-j`/`-f` options make users expect. Because rmake is
|
|
20
|
+
# rubycc's build-only make, tool substitution (`$(CC)`/`$(LDSHARED)` routed to
|
|
21
|
+
# rubycc's Driver) is always on — a `CC = gcc` left in the Makefile is still
|
|
22
|
+
# built by rubycc.
|
|
23
|
+
#
|
|
24
|
+
# It is a plain method object rather than an OptionParser front end: the mkmf
|
|
25
|
+
# invocation shape is fixed and narrow, so hand-parsing keeps the accepted
|
|
26
|
+
# grammar explicit (and the "everything routes to rubycc" contract obvious).
|
|
27
|
+
class CLI
|
|
28
|
+
DEFAULT_MAKEFILE = "Makefile"
|
|
29
|
+
|
|
30
|
+
# Run rmake with +argv+ in +dir+, returning the process exit status
|
|
31
|
+
# (0 success, 2 for any make-level failure — the code GNU make uses). +out+
|
|
32
|
+
# and +err+ are injectable so the CLI can be exercised without spawning.
|
|
33
|
+
def self.run(argv, dir: Dir.pwd, out: $stdout, err: $stderr)
|
|
34
|
+
new(dir: dir, out: out, err: err).run(argv)
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def initialize(dir:, out:, err:)
|
|
38
|
+
@dir = dir
|
|
39
|
+
@out = out
|
|
40
|
+
@err = err
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def run(argv)
|
|
44
|
+
options = parse_argv(argv)
|
|
45
|
+
|
|
46
|
+
makefile_path = File.expand_path(options[:file] || DEFAULT_MAKEFILE, @dir)
|
|
47
|
+
unless File.file?(makefile_path)
|
|
48
|
+
@err.puts("rmake: #{options[:file] || DEFAULT_MAKEFILE}: No such file")
|
|
49
|
+
return 2
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
mk = Makefile.parse(File.read(makefile_path), dir: @dir, overrides: options[:overrides],
|
|
53
|
+
defaults: { "MAKE" => make_default })
|
|
54
|
+
goals = options[:targets].empty? ? [nil] : options[:targets]
|
|
55
|
+
# Tool substitution is always on: rmake is rubycc's build CLI, so the
|
|
56
|
+
# compiler/linker words in every recipe are handed to rubycc's Driver.
|
|
57
|
+
goals.each { |goal| mk.run(goal, out: @out, err: @err, tools: :rubycc, jobs: options[:jobs]) }
|
|
58
|
+
0
|
|
59
|
+
rescue RmakeError => e
|
|
60
|
+
@err.puts("rmake: #{e.message}")
|
|
61
|
+
2
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
private
|
|
65
|
+
|
|
66
|
+
# Split +argv+ into { overrides:, targets:, jobs:, file: }. Anything that is
|
|
67
|
+
# not a recognised option or a `VAR=value` definition is a target name.
|
|
68
|
+
def parse_argv(argv)
|
|
69
|
+
overrides = {}
|
|
70
|
+
targets = []
|
|
71
|
+
# Default to all cores: rmake is invoked by `gem install` as a plain
|
|
72
|
+
# `make`, so a serial default would leave multi-file extensions
|
|
73
|
+
# building one recipe at a time (H5). The scheduler's dependency DAG
|
|
74
|
+
# is unaffected, and each step's output is buffered and flushed whole
|
|
75
|
+
# (like `-O`), so parallel output never interleaves and the built
|
|
76
|
+
# artifacts match a serial run; an explicit `-j1` still gets the old
|
|
77
|
+
# serial behaviour.
|
|
78
|
+
jobs = processor_count
|
|
79
|
+
file = nil
|
|
80
|
+
|
|
81
|
+
i = 0
|
|
82
|
+
while i < argv.length
|
|
83
|
+
arg = argv[i]
|
|
84
|
+
case arg
|
|
85
|
+
when "-f", "--file", "--makefile"
|
|
86
|
+
file = argv[i + 1]
|
|
87
|
+
i += 1
|
|
88
|
+
when /\A-f(.+)\z/, /\A--file=(.+)\z/, /\A--makefile=(.+)\z/
|
|
89
|
+
file = Regexp.last_match(1)
|
|
90
|
+
when "-j", "--jobs"
|
|
91
|
+
# A bare -j (its argument omitted, as make allows) means "as many as
|
|
92
|
+
# possible"; approximate that with the processor count.
|
|
93
|
+
if (nxt = argv[i + 1]) && nxt =~ /\A\d+\z/
|
|
94
|
+
jobs = nxt.to_i
|
|
95
|
+
i += 1
|
|
96
|
+
else
|
|
97
|
+
jobs = processor_count
|
|
98
|
+
end
|
|
99
|
+
when /\A-j(\d+)\z/, /\A--jobs=(\d+)\z/
|
|
100
|
+
jobs = Regexp.last_match(1).to_i
|
|
101
|
+
when /\A([A-Za-z_][A-Za-z0-9_]*)=(.*)\z/m
|
|
102
|
+
overrides[Regexp.last_match(1)] = Regexp.last_match(2)
|
|
103
|
+
when /\A-/
|
|
104
|
+
# An unrecognised option: rmake is a narrow make, so ignore flags it
|
|
105
|
+
# does not model rather than abort a build over an option that does
|
|
106
|
+
# not change what gets built.
|
|
107
|
+
else
|
|
108
|
+
targets << arg
|
|
109
|
+
end
|
|
110
|
+
i += 1
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
{ overrides: overrides, targets: targets, jobs: [jobs, 1].max, file: file }
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def processor_count
|
|
117
|
+
require "etc"
|
|
118
|
+
Etc.nprocessors
|
|
119
|
+
rescue StandardError
|
|
120
|
+
1
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
# POSIX requires make to define `MAKE` built in, so `$(MAKE)` in a recipe
|
|
124
|
+
# (a recursive-make invocation, e.g. `cd sub && $(MAKE)`) expands to
|
|
125
|
+
# something runnable rather than the empty string a silently-undefined
|
|
126
|
+
# variable would give — which would collapse the recipe line to `cd sub &&`
|
|
127
|
+
# and turn a recursive build into a no-op. The value is an absolute path to
|
|
128
|
+
# this very program, so the recursive invocation is rmake again, never a
|
|
129
|
+
# bare "make" that would hand the recursive build to a host GNU make (or
|
|
130
|
+
# fail outright) instead of rubycc.
|
|
131
|
+
#
|
|
132
|
+
# `ENV["MAKE"]` is honoured first because RubyGems' rubygems_plugin sets it
|
|
133
|
+
# to rmake before invoking `$(MAKE)` at the top level, and that value must
|
|
134
|
+
# propagate unchanged into any recipe this run itself expands.
|
|
135
|
+
def make_default
|
|
136
|
+
env_make = ENV["MAKE"]
|
|
137
|
+
return env_make if env_make && !env_make.empty?
|
|
138
|
+
|
|
139
|
+
File.expand_path($PROGRAM_NAME)
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
end
|
|
143
|
+
end
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Rubycc
|
|
4
|
+
# The base error is normally provided by lib/rubycc.rb. rmake is loadable on
|
|
5
|
+
# its own (B1 ships no CLI wiring), so define a stand-in only when the full
|
|
6
|
+
# library has not been required yet.
|
|
7
|
+
Error = Class.new(StandardError) unless defined?(Error)
|
|
8
|
+
|
|
9
|
+
module Rmake
|
|
10
|
+
# Raised for a defect in the Makefile the user handed us: a syntax rmake's
|
|
11
|
+
# mkmf-subset does not accept, or a runaway variable expansion. rmake never
|
|
12
|
+
# parses C, so this is deliberately distinct from CompileError; it names the
|
|
13
|
+
# make-level construct at fault so the failure points back at the Makefile.
|
|
14
|
+
class RmakeError < Rubycc::Error; end
|
|
15
|
+
|
|
16
|
+
# A line the parser cannot classify as an assignment, a rule, a recipe, or a
|
|
17
|
+
# directive that mkmf is known to emit. Carries the 1-based source line so
|
|
18
|
+
# the offending text can be located in the generated Makefile.
|
|
19
|
+
class ParseError < RmakeError
|
|
20
|
+
attr_reader :line_number
|
|
21
|
+
|
|
22
|
+
def initialize(message, line_number:)
|
|
23
|
+
@line_number = line_number
|
|
24
|
+
super("Makefile:#{line_number}: #{message}")
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# Raised when variable expansion exceeds its depth budget, which in practice
|
|
29
|
+
# means a reference cycle (`A = $(B)` / `B = $(A)`). It is a DoS fail-safe,
|
|
30
|
+
# not a diagnosis of the exact cycle — the budget stops unbounded recursion
|
|
31
|
+
# before it can exhaust the Ruby stack.
|
|
32
|
+
class ExpansionError < RmakeError; end
|
|
33
|
+
|
|
34
|
+
# Base for a failure that happens while *running* a plan (as opposed to
|
|
35
|
+
# parsing or planning it). It always names the target whose recipe was
|
|
36
|
+
# executing and the exact expanded recipe line at fault, so a build failure
|
|
37
|
+
# points back at "which command of which target" the way make's own
|
|
38
|
+
# `*** [target] Error` line does (N3).
|
|
39
|
+
class ExecutionError < RmakeError
|
|
40
|
+
attr_reader :target, :command
|
|
41
|
+
|
|
42
|
+
def initialize(message, target:, command:)
|
|
43
|
+
@target = target
|
|
44
|
+
@command = command
|
|
45
|
+
super("#{target}: #{message}: #{command.inspect}")
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# A recipe command exited non-zero (or an internal utility reported failure)
|
|
50
|
+
# and the line was not marked to ignore errors (`-`). Carries the exit reason
|
|
51
|
+
# when one is known (a missing external tool, a utility's own message).
|
|
52
|
+
class CommandFailedError < ExecutionError
|
|
53
|
+
def initialize(target:, command:, reason: nil)
|
|
54
|
+
super(reason ? "recipe command failed (#{reason})" : "recipe command failed",
|
|
55
|
+
target: target, command: command)
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# The shell-less runner met a construct it does not interpret (a pipe,
|
|
60
|
+
# background `&`, command substitution, an unterminated quote, ...). Since
|
|
61
|
+
# rubycc runs recipes without /bin/sh, an unhandled construct must fail
|
|
62
|
+
# loudly with the offending target and line rather than be silently dropped —
|
|
63
|
+
# this is the signal that a gem's recipe needs to be added to the runner's
|
|
64
|
+
# scope or the gem listed as unsupported (ROADMAP §6 B2).
|
|
65
|
+
class UnsupportedRecipeError < ExecutionError
|
|
66
|
+
def initialize(construct, target:, command:)
|
|
67
|
+
super("unsupported shell construct (#{construct})", target: target, command: command)
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|