rubycc 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +56 -0
  3. data/LICENSE.txt +21 -0
  4. data/NOTICE +52 -0
  5. data/README.md +208 -0
  6. data/data/README.md +117 -0
  7. data/data/r10_corpus_scan.json +4082 -0
  8. data/data/r10_manual_classification.json +5342 -0
  9. data/data/r10_verification_m4a.json +1531 -0
  10. data/data/r10_verification_m4b.json +1818 -0
  11. data/data/r10_verification_m4c.json +1489 -0
  12. data/data/r10_verification_m4d.json +318 -0
  13. data/data/r10_verification_rbs.json +192 -0
  14. data/data/verified_gems.json +397 -0
  15. data/exe/rmake +16 -0
  16. data/exe/rubycc +9 -0
  17. data/exe/rubycc-ar +114 -0
  18. data/exe/rubycc-doctor +14 -0
  19. data/exe/rubycc-pkgconf +8 -0
  20. data/include/float.h +87 -0
  21. data/include/iso646.h +18 -0
  22. data/include/libc/alloca.h +18 -0
  23. data/include/libc/arpa/inet.h +61 -0
  24. data/include/libc/assert.h +43 -0
  25. data/include/libc/dirent.h +72 -0
  26. data/include/libc/dlfcn.h +58 -0
  27. data/include/libc/features.h +205 -0
  28. data/include/libc/glibc/aarch64/ctype.h +108 -0
  29. data/include/libc/glibc/aarch64/endian.h +54 -0
  30. data/include/libc/glibc/aarch64/errno.h +154 -0
  31. data/include/libc/glibc/aarch64/fcntl.h +144 -0
  32. data/include/libc/glibc/aarch64/inttypes.h +182 -0
  33. data/include/libc/glibc/aarch64/limits.h +63 -0
  34. data/include/libc/glibc/aarch64/pthread.h +131 -0
  35. data/include/libc/glibc/aarch64/setjmp.h +84 -0
  36. data/include/libc/glibc/aarch64/stdint.h +174 -0
  37. data/include/libc/glibc/aarch64/sys/epoll.h +88 -0
  38. data/include/libc/glibc/aarch64/sys/fcntl.h +19 -0
  39. data/include/libc/glibc/aarch64/sys/select.h +75 -0
  40. data/include/libc/glibc/aarch64/sys/stat.h +136 -0
  41. data/include/libc/glibc/aarch64/sys/syscall.h +167 -0
  42. data/include/libc/glibc/aarch64/sys/time.h +74 -0
  43. data/include/libc/glibc/aarch64/sys/types.h +127 -0
  44. data/include/libc/glibc/aarch64/time.h +113 -0
  45. data/include/libc/glibc/x86_64/ctype.h +108 -0
  46. data/include/libc/glibc/x86_64/endian.h +54 -0
  47. data/include/libc/glibc/x86_64/errno.h +154 -0
  48. data/include/libc/glibc/x86_64/fcntl.h +134 -0
  49. data/include/libc/glibc/x86_64/inttypes.h +182 -0
  50. data/include/libc/glibc/x86_64/limits.h +61 -0
  51. data/include/libc/glibc/x86_64/pthread.h +111 -0
  52. data/include/libc/glibc/x86_64/setjmp.h +82 -0
  53. data/include/libc/glibc/x86_64/stdint.h +163 -0
  54. data/include/libc/glibc/x86_64/sys/epoll.h +91 -0
  55. data/include/libc/glibc/x86_64/sys/fcntl.h +19 -0
  56. data/include/libc/glibc/x86_64/sys/select.h +75 -0
  57. data/include/libc/glibc/x86_64/sys/stat.h +130 -0
  58. data/include/libc/glibc/x86_64/sys/syscall.h +188 -0
  59. data/include/libc/glibc/x86_64/sys/time.h +74 -0
  60. data/include/libc/glibc/x86_64/sys/types.h +123 -0
  61. data/include/libc/glibc/x86_64/time.h +113 -0
  62. data/include/libc/grp.h +50 -0
  63. data/include/libc/langinfo.h +126 -0
  64. data/include/libc/link.h +18 -0
  65. data/include/libc/locale.h +83 -0
  66. data/include/libc/math.h +191 -0
  67. data/include/libc/netinet/in.h +126 -0
  68. data/include/libc/netinet/tcp.h +44 -0
  69. data/include/libc/poll.h +46 -0
  70. data/include/libc/pwd.h +58 -0
  71. data/include/libc/regex.h +53 -0
  72. data/include/libc/sched.h +35 -0
  73. data/include/libc/signal.h +204 -0
  74. data/include/libc/stdio.h +157 -0
  75. data/include/libc/stdlib.h +92 -0
  76. data/include/libc/string.h +74 -0
  77. data/include/libc/strings.h +25 -0
  78. data/include/libc/sys/cdefs.h +126 -0
  79. data/include/libc/sys/inotify.h +111 -0
  80. data/include/libc/sys/ioctl.h +36 -0
  81. data/include/libc/sys/mman.h +65 -0
  82. data/include/libc/sys/param.h +41 -0
  83. data/include/libc/sys/resource.h +109 -0
  84. data/include/libc/sys/socket.h +167 -0
  85. data/include/libc/sys/statfs.h +82 -0
  86. data/include/libc/sys/timerfd.h +55 -0
  87. data/include/libc/sys/uio.h +40 -0
  88. data/include/libc/sys/un.h +25 -0
  89. data/include/libc/sys/utsname.h +35 -0
  90. data/include/libc/sys/wait.h +135 -0
  91. data/include/libc/termios.h +179 -0
  92. data/include/libc/unistd.h +194 -0
  93. data/include/stdalign.h +16 -0
  94. data/include/stdarg.h +31 -0
  95. data/include/stdatomic.h +158 -0
  96. data/include/stdbool.h +15 -0
  97. data/include/stdckdint.h +28 -0
  98. data/include/stddef.h +60 -0
  99. data/include/stdnoreturn.h +18 -0
  100. data/include/x86intrin.h +16 -0
  101. data/lib/rubycc/backend/aarch64.rb +1724 -0
  102. data/lib/rubycc/backend/x86_64.rb +1369 -0
  103. data/lib/rubycc/compile_error.rb +36 -0
  104. data/lib/rubycc/compiler.rb +305 -0
  105. data/lib/rubycc/doctor/builder.rb +151 -0
  106. data/lib/rubycc/doctor/cli.rb +190 -0
  107. data/lib/rubycc/doctor/fetcher.rb +84 -0
  108. data/lib/rubycc/doctor/gemfile.rb +115 -0
  109. data/lib/rubycc/doctor/verified_gems.rb +114 -0
  110. data/lib/rubycc/doctor.rb +16 -0
  111. data/lib/rubycc/driver.rb +463 -0
  112. data/lib/rubycc/front/ast.rb +528 -0
  113. data/lib/rubycc/front/constant_evaluator.rb +631 -0
  114. data/lib/rubycc/front/initializer_resolver.rb +592 -0
  115. data/lib/rubycc/front/lexeme_reader.rb +460 -0
  116. data/lib/rubycc/front/lexer.rb +232 -0
  117. data/lib/rubycc/front/parser.rb +4122 -0
  118. data/lib/rubycc/front/token.rb +49 -0
  119. data/lib/rubycc/ir/call_convention.rb +486 -0
  120. data/lib/rubycc/ir/generator.rb +6036 -0
  121. data/lib/rubycc/ir/ir.rb +417 -0
  122. data/lib/rubycc/link/compat_runtime.rb +89 -0
  123. data/lib/rubycc/link/errors.rb +13 -0
  124. data/lib/rubycc/link/executable_linker.rb +405 -0
  125. data/lib/rubycc/link/library_resolver.rb +437 -0
  126. data/lib/rubycc/link/partial_linker.rb +546 -0
  127. data/lib/rubycc/link/shared_linker.rb +1732 -0
  128. data/lib/rubycc/mkmf_shim.rb +83 -0
  129. data/lib/rubycc/objfile/ar_archive.rb +437 -0
  130. data/lib/rubycc/objfile/elf_reader.rb +646 -0
  131. data/lib/rubycc/objfile/elf_writer.rb +891 -0
  132. data/lib/rubycc/objfile/relocatable_writer.rb +376 -0
  133. data/lib/rubycc/pkgconf/cli.rb +139 -0
  134. data/lib/rubycc/pkgconf/errors.rb +39 -0
  135. data/lib/rubycc/pkgconf/model.rb +28 -0
  136. data/lib/rubycc/pkgconf/parser.rb +109 -0
  137. data/lib/rubycc/pkgconf/pkgconf.rb +15 -0
  138. data/lib/rubycc/pkgconf/resolver.rb +70 -0
  139. data/lib/rubycc/pkgconf/search_path.rb +38 -0
  140. data/lib/rubycc/pkgconf/system_path_filter.rb +131 -0
  141. data/lib/rubycc/preprocess/constant_expression.rb +219 -0
  142. data/lib/rubycc/preprocess/glibc_version.rb +124 -0
  143. data/lib/rubycc/preprocess/pp_token.rb +62 -0
  144. data/lib/rubycc/preprocess/preprocessor.rb +2020 -0
  145. data/lib/rubycc/preprocess/scanner.rb +290 -0
  146. data/lib/rubycc/preprocess/token_converter.rb +157 -0
  147. data/lib/rubycc/rmake/cli.rb +143 -0
  148. data/lib/rubycc/rmake/errors.rb +71 -0
  149. data/lib/rubycc/rmake/executor.rb +818 -0
  150. data/lib/rubycc/rmake/expander.rb +251 -0
  151. data/lib/rubycc/rmake/makefile.rb +352 -0
  152. data/lib/rubycc/rmake/model.rb +137 -0
  153. data/lib/rubycc/rmake/parser.rb +226 -0
  154. data/lib/rubycc/rmake/rmake.rb +15 -0
  155. data/lib/rubycc/type.rb +1236 -0
  156. data/lib/rubycc/version.rb +5 -0
  157. data/lib/rubycc.rb +36 -0
  158. data/lib/rubygems_plugin.rb +102 -0
  159. metadata +219 -0
@@ -0,0 +1,290 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "strscan"
4
+
5
+ require_relative "pp_token"
6
+ require_relative "../front/lexeme_reader"
7
+ require_relative "../compile_error"
8
+
9
+ module Rubycc
10
+ module Preprocess
11
+ # Turns raw source into a stream of preprocessing tokens, carrying out
12
+ # translation phases 2 and 3: backslash-newline line splicing and the
13
+ # replacement of comments by whitespace. Newlines survive as explicit tokens
14
+ # so the directive layer (added later) stays line-oriented, and every token
15
+ # keeps the physical location of its first character.
16
+ #
17
+ # Splicing works by deleting every backslash-newline pair up front and
18
+ # remembering where each deletion happened (@splice_points): tokens are then
19
+ # matched over the spliced text with StringScanner-driven regexps — one
20
+ # match per token instead of a method call per character, which is what
21
+ # makes the preprocessor's dominant cost scale. A token's physical
22
+ # line/column stays truthful because the recorded splice points replay the
23
+ # deleted line breaks: everything at or past a splice point is located on
24
+ # the following physical line (see #sync). Positions are byte offsets
25
+ # (StringScanner's native unit); columns are converted back to character
26
+ # counts only when a token is actually made, and only for non-ASCII source.
27
+ class Scanner
28
+ # Two- and one-character punctuators gain the preprocessor-only "##" and
29
+ # "#" over the shared punctuator tables; the three-character set is unchanged.
30
+ PP_PUNCTUATORS_3 = Front::LexemeReader::PUNCTUATORS_3
31
+ PP_PUNCTUATORS_2 = (Front::LexemeReader::PUNCTUATORS_2 + ["##"]).freeze
32
+ PP_PUNCTUATORS_1 = (Front::LexemeReader::PUNCTUATORS_1 + ["#"]).freeze
33
+
34
+ # Longest alternative first, so one anchored attempt is a longest match.
35
+ PUNCTUATOR_RE = Regexp.union(PP_PUNCTUATORS_3 + PP_PUNCTUATORS_2 + PP_PUNCTUATORS_1)
36
+
37
+ HORIZONTAL_WS_RE = /[ \t\r\v\f]+/
38
+ # A // comment runs to (but not over) the end of the physical line; a
39
+ # trailing backslash-newline was already spliced away, so the comment
40
+ # continues onto the next line just as it does under gcc.
41
+ LINE_COMMENT_RE = %r{//[^\n]*}
42
+ BLOCK_OPEN_RE = %r{/\*}
43
+ BLOCK_CLOSE_RE = %r{\*/}
44
+ NEWLINE_RE = /\n/
45
+
46
+ IDENTIFIER_RE = /[A-Za-z_][A-Za-z0-9_]*/
47
+ # A preprocessing number (6.4.8): a digit, or a "." then a digit, and then
48
+ # digits, identifier characters, ".", and a sign immediately after an
49
+ # 'e'/'E'/'p'/'P'. Whether the run is a valid C constant is decided later,
50
+ # at conversion time.
51
+ PP_NUMBER_RE = /(?:[0-9]|\.[0-9])(?:[eEpP][+-]|[0-9A-Za-z_.])*/
52
+
53
+ # A string literal or character constant, kept verbatim (quotes and
54
+ # escapes included). Only the boundary is recognized here, matching the
55
+ # streaming lexer: a backslash shields the following character so an
56
+ # escaped quote does not close the literal. An unterminated literal keeps
57
+ # whatever was read (the trailing `("|\)?` picks up a lone backslash cut
58
+ # off by end-of-line or end-of-file); the converter re-scans it and raises
59
+ # the positioned error.
60
+ STRING_RE = /"(?:[^"\\\n]|\\[^\n])*(?:"|\\)?/
61
+ CHAR_RE = /'(?:[^'\\\n]|\\[^\n])*(?:'|\\)?/
62
+ # A wide character constant L'c' or wide string literal L"..." (6.4.4.4,
63
+ # 6.4.5): recognized only when the "L" abuts the quote, so an identifier
64
+ # named "L" is unaffected. The "L" is kept in the spelling so the converter
65
+ # can tell a wide literal from a plain one; it decides a wide character's
66
+ # value (int here, as for a plain constant) and rejects a wide string. The
67
+ # u/U/u8 prefixes are out of scope, so they still scan as an identifier.
68
+ WIDE_CHAR_RE = /L'(?:[^'\\\n]|\\[^\n])*(?:'|\\)?/
69
+ WIDE_STRING_RE = /L"(?:[^"\\\n]|\\[^\n])*(?:"|\\)?/
70
+
71
+ def initialize(source, filename:)
72
+ @filename = filename
73
+ # -1 keeps a trailing empty field so line numbers map 1:1 to entries.
74
+ @lines = source.split("\n", -1)
75
+ splice(source)
76
+ @scanner = StringScanner.new(@spliced)
77
+ @ascii_only = @spliced.ascii_only?
78
+ @line = 1
79
+ # Byte offset in @spliced where the current physical line starts; a
80
+ # column is the distance from here (plus one). @column_pos/@column_chars
81
+ # are the incremental cursor #column_at advances (see there); they are
82
+ # only meaningful for non-ASCII source, and reset_line_start resets all
83
+ # three together so they can never drift apart.
84
+ reset_line_start(0)
85
+ # Index into @splice_points of the first point not yet replayed.
86
+ @next_splice = 0
87
+ end
88
+
89
+ def scan
90
+ tokens = []
91
+ ss = @scanner
92
+ loop do
93
+ before = ss.pos
94
+ skip_whitespace_and_comments(ss, tokens)
95
+ # Whether any whitespace or comment separated this token from the
96
+ # previous one on the same logical line. The directive layer needs it
97
+ # to tell "#define F(x)" (function-like) from "#define F (x)" (an
98
+ # object macro whose replacement begins with a parenthesis).
99
+ space_before = ss.pos != before
100
+ break if ss.eos?
101
+
102
+ start = ss.pos
103
+ sync(start)
104
+ if ss.skip(NEWLINE_RE)
105
+ tokens << make_token(:newline, "\n", @line, column_at(start), space_before)
106
+ @line += 1
107
+ reset_line_start(ss.pos)
108
+ else
109
+ tokens << scan_token(ss, @line, column_at(start), space_before)
110
+ end
111
+ end
112
+ sync(@spliced.bytesize)
113
+ tokens << make_token(:eof, nil, @line, column_at(@spliced.bytesize))
114
+ tokens
115
+ end
116
+
117
+ private
118
+
119
+ # Translation phase 2: delete every backslash-newline pair, recording for
120
+ # each deletion the byte offset (in the spliced text) where the next
121
+ # physical line begins. Two adjacent continuations record the same offset
122
+ # twice — each still advances the line counter once when replayed. A
123
+ # source with no continuations (the common case) is shared, not copied.
124
+ def splice(source)
125
+ @splice_points = []
126
+ unless source.include?("\\\n")
127
+ @spliced = source
128
+ return
129
+ end
130
+ spliced = +""
131
+ pos = 0
132
+ while (idx = source.index("\\\n", pos))
133
+ spliced << source[pos...idx]
134
+ @splice_points << spliced.bytesize
135
+ pos = idx + 2
136
+ end
137
+ spliced << source[pos..]
138
+ @spliced = spliced
139
+ end
140
+
141
+ # Replays every splice point at or before byte offset `pos`: each one
142
+ # marks a physical line break the deletion hid, so the line counter
143
+ # advances and the line start moves to the point itself (its first
144
+ # character is column 1 of the continued-onto line). Called with a token's
145
+ # start offset before its location is read, and with interior offsets by
146
+ # the block-comment walk; a point strictly inside a token is replayed when
147
+ # the next token is located, which yields the same stream (a token is
148
+ # located only by its first character).
149
+ def sync(pos)
150
+ points = @splice_points
151
+ while @next_splice < points.length && points[@next_splice] <= pos
152
+ @line += 1
153
+ reset_line_start(points[@next_splice])
154
+ @next_splice += 1
155
+ end
156
+ end
157
+
158
+ # Moves the current physical line's start to byte offset `pos`. Every
159
+ # assignment to @line_start goes through here so the column cursor is
160
+ # rebased at the same time — a missed reset would not crash, it would
161
+ # silently report wrong columns (N3).
162
+ def reset_line_start(pos)
163
+ @line_start = pos
164
+ @column_pos = pos
165
+ @column_chars = 0
166
+ end
167
+
168
+ # The 1-based column of byte offset `pos`, counted in characters from the
169
+ # current physical line's start. Byte arithmetic serves ASCII source
170
+ # directly; non-ASCII source has to count characters, and must count them
171
+ # *incrementally*: measuring the whole @line_start..pos span per token
172
+ # costs O(line length) each time, so a single long line of L tokens costs
173
+ # O(L^2) — and one non-ASCII byte anywhere in the file (a comment in
174
+ # Japanese, a UTF-8 BOM) is enough to switch @ascii_only off for all of
175
+ # it. Tokens are located left to right, so keeping the last (byte offset,
176
+ # column) pair as a cursor and counting only the span since then makes a
177
+ # line cost O(line length) in total. Should a caller ever ask for an
178
+ # offset behind the cursor, the count restarts from the line start, so the
179
+ # answer stays right even if that monotonicity is lost.
180
+ def column_at(pos)
181
+ return pos - @line_start + 1 if @ascii_only
182
+
183
+ if pos < @column_pos
184
+ @column_pos = @line_start
185
+ @column_chars = 0
186
+ end
187
+ if pos > @column_pos
188
+ @column_chars += @spliced.byteslice(@column_pos, pos - @column_pos).length
189
+ @column_pos = pos
190
+ end
191
+ @column_chars + 1
192
+ end
193
+
194
+ # Skips horizontal whitespace and comments (translation phase 3). Newlines
195
+ # are left for the caller so they become tokens; a newline retained inside
196
+ # a block comment is emitted here directly, keeping the logical line count
197
+ # exact for later directive processing.
198
+ def skip_whitespace_and_comments(ss, tokens)
199
+ loop do
200
+ next if ss.skip(HORIZONTAL_WS_RE)
201
+ next if ss.skip(LINE_COMMENT_RE)
202
+ return unless ss.match?(BLOCK_OPEN_RE)
203
+
204
+ skip_block_comment(ss, tokens)
205
+ end
206
+ end
207
+
208
+ # A /* */ comment, which may span physical lines. Each retained newline it
209
+ # crosses is emitted as a token so the line count stays consistent; a
210
+ # spliced-away continuation inside it adds no token, only a line. An
211
+ # unterminated comment is a hard error, positioned at the opening "/*".
212
+ def skip_block_comment(ss, tokens)
213
+ sync(ss.pos)
214
+ start_line = @line
215
+ start_column = column_at(ss.pos)
216
+ ss.skip(BLOCK_OPEN_RE)
217
+ body = ss.scan_until(BLOCK_CLOSE_RE)
218
+ raise_error("unterminated block comment", start_line, start_column) if body.nil?
219
+
220
+ base = ss.pos - body.bytesize
221
+ from = 0
222
+ while (idx = body.byteindex("\n", from))
223
+ at = base + idx
224
+ sync(at)
225
+ tokens << make_token(:newline, "\n", @line, column_at(at))
226
+ @line += 1
227
+ reset_line_start(at + 1)
228
+ from = idx + 1
229
+ end
230
+ end
231
+
232
+ # Scans a single preprocessing token (never a newline or whitespace),
233
+ # tagged with the physical location of its first character. One anchored
234
+ # regexp attempt per candidate class, ordered so a prefix can never steal
235
+ # a longer token: a pp-number before "." the punctuator, a wide literal
236
+ # before the identifier "L".
237
+ def scan_token(ss, line, column, space_before)
238
+ if (text = ss.scan(PP_NUMBER_RE))
239
+ type = :pp_number
240
+ elsif (text = ss.scan(IDENTIFIER_RE))
241
+ type = :identifier
242
+ # An "L" abutting a quote is a wide literal, not an identifier.
243
+ if text == "L" && (quoted = ss.scan(CHAR_RE) || ss.scan(STRING_RE))
244
+ type = quoted.start_with?("'") ? :char : :string
245
+ text = "L" + quoted
246
+ end
247
+ elsif (text = ss.scan(CHAR_RE))
248
+ type = :char
249
+ elsif (text = ss.scan(STRING_RE))
250
+ type = :string
251
+ elsif (text = ss.scan(PUNCTUATOR_RE))
252
+ type = :punct
253
+ else
254
+ # Anything that matches nothing above becomes a single-character
255
+ # :other token (6.4p1), deferring the decision of whether it is an
256
+ # error to a later phase.
257
+ type = :other
258
+ text = ss.getch
259
+ end
260
+ make_token(type, text, line, column, space_before)
261
+ end
262
+
263
+ def make_token(type, text, line, column, space_before = false)
264
+ PPToken.new(
265
+ type: type,
266
+ text: text,
267
+ filename: @filename,
268
+ line: line,
269
+ column: column,
270
+ source_line: source_line_for(line),
271
+ space_before: space_before
272
+ )
273
+ end
274
+
275
+ def source_line_for(line)
276
+ @lines[line - 1] || ""
277
+ end
278
+
279
+ def raise_error(description, line, column)
280
+ raise CompileError.new(
281
+ description,
282
+ filename: @filename,
283
+ line: line,
284
+ column: column,
285
+ source_line: source_line_for(line)
286
+ )
287
+ end
288
+ end
289
+ end
290
+ end
@@ -0,0 +1,157 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "pp_token"
4
+ require_relative "../front/token"
5
+ require_relative "../front/lexeme_reader"
6
+ require_relative "../compile_error"
7
+
8
+ module Rubycc
9
+ module Preprocess
10
+ # Ends translation phase 4 by turning preprocessing tokens into the
11
+ # Front::Token stream the parser consumes. Numbers are folded, string and
12
+ # character literals decoded, and identifiers classified as keywords, all via
13
+ # the shared LexemeReader so the result matches the streaming lexer exactly.
14
+ # Newlines are dropped here; a "#"/"##" that survives to this point is a
15
+ # stray operator with no macro meaning (directive lines are diagnosed
16
+ # earlier), so it is rejected.
17
+ #
18
+ # This is where translation phases 5-7 sit in this pipeline, so adjacent
19
+ # string-literal concatenation (phase 6, ISO C 6.4.5p5) happens here: a run
20
+ # of neighbouring string tokens folds into one, its bytes the per-literal
21
+ # decodings laid end to end. The streaming Front::Lexer takes the direct path
22
+ # and does not concatenate — nothing but this converter ever feeds the
23
+ # parser in the real compile pipeline (which always runs through the
24
+ # preprocessor), so the lexer's simpler single-literal path serves its
25
+ # unit-test-only role without the phase-6 fold.
26
+ class TokenConverter
27
+ def convert(pp_tokens)
28
+ tokens = []
29
+ index = 0
30
+ while index < pp_tokens.length
31
+ pp = pp_tokens[index]
32
+ case pp.type
33
+ when :newline
34
+ index += 1
35
+ when :eof
36
+ tokens << front_token(pp, :eof, nil)
37
+ index += 1
38
+ when :identifier
39
+ spelling = Front::LexemeReader.keyword_spelling(pp.text)
40
+ tokens << front_token(pp, spelling ? :keyword : :ident, spelling || pp.text)
41
+ index += 1
42
+ when :pp_number
43
+ tokens << convert_number(pp)
44
+ index += 1
45
+ when :string
46
+ value, index = concatenate_strings(pp_tokens, index)
47
+ tokens << front_token(pp, :string, value)
48
+ when :char
49
+ tokens << num_token(pp, decode_char(pp), 10, "")
50
+ index += 1
51
+ when :punct
52
+ if pp.text == "#" || pp.text == "##"
53
+ raise_at(pp, "stray '#' in program")
54
+ end
55
+ tokens << front_token(pp, :punct, pp.text)
56
+ index += 1
57
+ when :other
58
+ raise_at(pp, "unexpected character #{pp.text.inspect}")
59
+ end
60
+ end
61
+ tokens
62
+ end
63
+
64
+ private
65
+
66
+ # Folds the maximal run of adjacent string-literal tokens starting at
67
+ # `index` into a single value (translation phase 6). Each literal is decoded
68
+ # on its own, so an escape resolves within the literal that spells it —
69
+ # "\x41" "1" is the two bytes 'A' and '1', never one "\x411" digit run
70
+ # (6.4.5p4) — and the decoded bytes are concatenated. Intervening newline
71
+ # tokens are inter-token whitespace and do not break adjacency; in the
72
+ # compile pipeline the expander has already dropped them, so consecutive
73
+ # string tokens simply abut. Returns [concatenated-bytes, index-past-run].
74
+ def concatenate_strings(pp_tokens, index)
75
+ bytes = decode_string(pp_tokens[index])
76
+ index += 1
77
+ loop do
78
+ nxt = index
79
+ nxt += 1 while pp_tokens[nxt]&.type == :newline
80
+ break unless pp_tokens[nxt]&.type == :string
81
+
82
+ bytes += decode_string(pp_tokens[nxt])
83
+ index = nxt + 1
84
+ end
85
+ [bytes, index]
86
+ end
87
+
88
+ # The escape-resolved bytes of one string literal. A wide string literal
89
+ # (an "L" prefix) has element type wchar_t, which this subset cannot
90
+ # represent, so it is rejected rather than silently narrowed.
91
+ def decode_string(pp)
92
+ raise_at(pp, "wide string literals are not supported") if pp.text.start_with?("L")
93
+
94
+ decode(pp, pp.text) { |reader| reader.read_string }.value
95
+ end
96
+
97
+ # A character constant's integer value. A wide character constant L'c' has
98
+ # type wchar_t (int on this target); for the single-byte characters this
99
+ # subset lexes its value equals the plain constant's, so the "L" prefix is
100
+ # dropped and the remainder decoded like an ordinary 'c'.
101
+ def decode_char(pp)
102
+ spelling = pp.text.start_with?("L") ? pp.text[1..] : pp.text
103
+ decode(pp, spelling) { |reader| reader.read_char }.value
104
+ end
105
+
106
+ # Folds a preprocessing number into a :num or :float token. A pp-number is
107
+ # broader than a C constant, so an ill-formed one (e.g. "12abc", "1.2.3")
108
+ # is rejected here rather than at scan time.
109
+ def convert_number(pp)
110
+ result = decode(pp, pp.text) { |reader| reader.read_number }
111
+ if result.type == :float
112
+ Front::Token.new(
113
+ type: :float, value: result.value,
114
+ filename: pp.filename, line: pp.line, column: pp.column,
115
+ source_line: pp.source_line, suffix: result.suffix
116
+ )
117
+ else
118
+ num_token(pp, result.value, result.base, result.suffix)
119
+ end
120
+ end
121
+
122
+ # Runs a LexemeReader over `spelling` (the token's own text, or that text
123
+ # with a wide-literal "L" prefix already stripped), mapping any LexError
124
+ # onto the token's start position. A lexeme spliced across a line
125
+ # continuation is reported at its start, matching where it begins in source.
126
+ def decode(pp, spelling)
127
+ yield Front::LexemeReader.new(spelling)
128
+ rescue Front::LexError => e
129
+ raise_at(pp, e.message)
130
+ end
131
+
132
+ def front_token(pp, type, value)
133
+ Front::Token.new(
134
+ type: type, value: value,
135
+ filename: pp.filename, line: pp.line, column: pp.column,
136
+ source_line: pp.source_line
137
+ )
138
+ end
139
+
140
+ def num_token(pp, value, base, suffix)
141
+ Front::Token.new(
142
+ type: :num, value: value,
143
+ filename: pp.filename, line: pp.line, column: pp.column,
144
+ source_line: pp.source_line, base: base, suffix: suffix
145
+ )
146
+ end
147
+
148
+ def raise_at(pp, description)
149
+ raise CompileError.new(
150
+ description,
151
+ filename: pp.filename, line: pp.line, column: pp.column,
152
+ source_line: pp.source_line
153
+ )
154
+ end
155
+ end
156
+ end
157
+ end
@@ -0,0 +1,143 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "errors"
4
+ require_relative "makefile"
5
+
6
+ module Rubycc
7
+ module Rmake
8
+ # The command-line front end to rmake (M3 / ROADMAP §6 B6). It exists so
9
+ # RubyGems can drive rubycc's builds by setting `ENV["MAKE"]` to this program:
10
+ # Gem::Ext::Builder invokes `$(MAKE)` as three separate processes shaped like
11
+ #
12
+ # rmake DESTDIR= sitearchdir=<tmp> sitelibdir=<tmp> clean
13
+ # rmake DESTDIR= sitearchdir=<tmp> sitelibdir=<tmp> (default goal)
14
+ # rmake DESTDIR= sitearchdir=<tmp> sitelibdir=<tmp> install
15
+ #
16
+ # so the argument grammar it must accept is: command-line variable definitions
17
+ # (`VAR=value`, which override the Makefile's own assignments), zero or more
18
+ # target names (an absent target builds the default goal) and, though RubyGems
19
+ # does not pass them, the `-j`/`-f` options make users expect. Because rmake is
20
+ # rubycc's build-only make, tool substitution (`$(CC)`/`$(LDSHARED)` routed to
21
+ # rubycc's Driver) is always on — a `CC = gcc` left in the Makefile is still
22
+ # built by rubycc.
23
+ #
24
+ # It is a plain method object rather than an OptionParser front end: the mkmf
25
+ # invocation shape is fixed and narrow, so hand-parsing keeps the accepted
26
+ # grammar explicit (and the "everything routes to rubycc" contract obvious).
27
+ class CLI
28
+ DEFAULT_MAKEFILE = "Makefile"
29
+
30
+ # Run rmake with +argv+ in +dir+, returning the process exit status
31
+ # (0 success, 2 for any make-level failure — the code GNU make uses). +out+
32
+ # and +err+ are injectable so the CLI can be exercised without spawning.
33
+ def self.run(argv, dir: Dir.pwd, out: $stdout, err: $stderr)
34
+ new(dir: dir, out: out, err: err).run(argv)
35
+ end
36
+
37
+ def initialize(dir:, out:, err:)
38
+ @dir = dir
39
+ @out = out
40
+ @err = err
41
+ end
42
+
43
+ def run(argv)
44
+ options = parse_argv(argv)
45
+
46
+ makefile_path = File.expand_path(options[:file] || DEFAULT_MAKEFILE, @dir)
47
+ unless File.file?(makefile_path)
48
+ @err.puts("rmake: #{options[:file] || DEFAULT_MAKEFILE}: No such file")
49
+ return 2
50
+ end
51
+
52
+ mk = Makefile.parse(File.read(makefile_path), dir: @dir, overrides: options[:overrides],
53
+ defaults: { "MAKE" => make_default })
54
+ goals = options[:targets].empty? ? [nil] : options[:targets]
55
+ # Tool substitution is always on: rmake is rubycc's build CLI, so the
56
+ # compiler/linker words in every recipe are handed to rubycc's Driver.
57
+ goals.each { |goal| mk.run(goal, out: @out, err: @err, tools: :rubycc, jobs: options[:jobs]) }
58
+ 0
59
+ rescue RmakeError => e
60
+ @err.puts("rmake: #{e.message}")
61
+ 2
62
+ end
63
+
64
+ private
65
+
66
+ # Split +argv+ into { overrides:, targets:, jobs:, file: }. Anything that is
67
+ # not a recognised option or a `VAR=value` definition is a target name.
68
+ def parse_argv(argv)
69
+ overrides = {}
70
+ targets = []
71
+ # Default to all cores: rmake is invoked by `gem install` as a plain
72
+ # `make`, so a serial default would leave multi-file extensions
73
+ # building one recipe at a time (H5). The scheduler's dependency DAG
74
+ # is unaffected, and each step's output is buffered and flushed whole
75
+ # (like `-O`), so parallel output never interleaves and the built
76
+ # artifacts match a serial run; an explicit `-j1` still gets the old
77
+ # serial behaviour.
78
+ jobs = processor_count
79
+ file = nil
80
+
81
+ i = 0
82
+ while i < argv.length
83
+ arg = argv[i]
84
+ case arg
85
+ when "-f", "--file", "--makefile"
86
+ file = argv[i + 1]
87
+ i += 1
88
+ when /\A-f(.+)\z/, /\A--file=(.+)\z/, /\A--makefile=(.+)\z/
89
+ file = Regexp.last_match(1)
90
+ when "-j", "--jobs"
91
+ # A bare -j (its argument omitted, as make allows) means "as many as
92
+ # possible"; approximate that with the processor count.
93
+ if (nxt = argv[i + 1]) && nxt =~ /\A\d+\z/
94
+ jobs = nxt.to_i
95
+ i += 1
96
+ else
97
+ jobs = processor_count
98
+ end
99
+ when /\A-j(\d+)\z/, /\A--jobs=(\d+)\z/
100
+ jobs = Regexp.last_match(1).to_i
101
+ when /\A([A-Za-z_][A-Za-z0-9_]*)=(.*)\z/m
102
+ overrides[Regexp.last_match(1)] = Regexp.last_match(2)
103
+ when /\A-/
104
+ # An unrecognised option: rmake is a narrow make, so ignore flags it
105
+ # does not model rather than abort a build over an option that does
106
+ # not change what gets built.
107
+ else
108
+ targets << arg
109
+ end
110
+ i += 1
111
+ end
112
+
113
+ { overrides: overrides, targets: targets, jobs: [jobs, 1].max, file: file }
114
+ end
115
+
116
+ def processor_count
117
+ require "etc"
118
+ Etc.nprocessors
119
+ rescue StandardError
120
+ 1
121
+ end
122
+
123
+ # POSIX requires make to define `MAKE` built in, so `$(MAKE)` in a recipe
124
+ # (a recursive-make invocation, e.g. `cd sub && $(MAKE)`) expands to
125
+ # something runnable rather than the empty string a silently-undefined
126
+ # variable would give — which would collapse the recipe line to `cd sub &&`
127
+ # and turn a recursive build into a no-op. The value is an absolute path to
128
+ # this very program, so the recursive invocation is rmake again, never a
129
+ # bare "make" that would hand the recursive build to a host GNU make (or
130
+ # fail outright) instead of rubycc.
131
+ #
132
+ # `ENV["MAKE"]` is honoured first because RubyGems' rubygems_plugin sets it
133
+ # to rmake before invoking `$(MAKE)` at the top level, and that value must
134
+ # propagate unchanged into any recipe this run itself expands.
135
+ def make_default
136
+ env_make = ENV["MAKE"]
137
+ return env_make if env_make && !env_make.empty?
138
+
139
+ File.expand_path($PROGRAM_NAME)
140
+ end
141
+ end
142
+ end
143
+ end
@@ -0,0 +1,71 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rubycc
4
+ # The base error is normally provided by lib/rubycc.rb. rmake is loadable on
5
+ # its own (B1 ships no CLI wiring), so define a stand-in only when the full
6
+ # library has not been required yet.
7
+ Error = Class.new(StandardError) unless defined?(Error)
8
+
9
+ module Rmake
10
+ # Raised for a defect in the Makefile the user handed us: a syntax rmake's
11
+ # mkmf-subset does not accept, or a runaway variable expansion. rmake never
12
+ # parses C, so this is deliberately distinct from CompileError; it names the
13
+ # make-level construct at fault so the failure points back at the Makefile.
14
+ class RmakeError < Rubycc::Error; end
15
+
16
+ # A line the parser cannot classify as an assignment, a rule, a recipe, or a
17
+ # directive that mkmf is known to emit. Carries the 1-based source line so
18
+ # the offending text can be located in the generated Makefile.
19
+ class ParseError < RmakeError
20
+ attr_reader :line_number
21
+
22
+ def initialize(message, line_number:)
23
+ @line_number = line_number
24
+ super("Makefile:#{line_number}: #{message}")
25
+ end
26
+ end
27
+
28
+ # Raised when variable expansion exceeds its depth budget, which in practice
29
+ # means a reference cycle (`A = $(B)` / `B = $(A)`). It is a DoS fail-safe,
30
+ # not a diagnosis of the exact cycle — the budget stops unbounded recursion
31
+ # before it can exhaust the Ruby stack.
32
+ class ExpansionError < RmakeError; end
33
+
34
+ # Base for a failure that happens while *running* a plan (as opposed to
35
+ # parsing or planning it). It always names the target whose recipe was
36
+ # executing and the exact expanded recipe line at fault, so a build failure
37
+ # points back at "which command of which target" the way make's own
38
+ # `*** [target] Error` line does (N3).
39
+ class ExecutionError < RmakeError
40
+ attr_reader :target, :command
41
+
42
+ def initialize(message, target:, command:)
43
+ @target = target
44
+ @command = command
45
+ super("#{target}: #{message}: #{command.inspect}")
46
+ end
47
+ end
48
+
49
+ # A recipe command exited non-zero (or an internal utility reported failure)
50
+ # and the line was not marked to ignore errors (`-`). Carries the exit reason
51
+ # when one is known (a missing external tool, a utility's own message).
52
+ class CommandFailedError < ExecutionError
53
+ def initialize(target:, command:, reason: nil)
54
+ super(reason ? "recipe command failed (#{reason})" : "recipe command failed",
55
+ target: target, command: command)
56
+ end
57
+ end
58
+
59
+ # The shell-less runner met a construct it does not interpret (a pipe,
60
+ # background `&`, command substitution, an unterminated quote, ...). Since
61
+ # rubycc runs recipes without /bin/sh, an unhandled construct must fail
62
+ # loudly with the offending target and line rather than be silently dropped —
63
+ # this is the signal that a gem's recipe needs to be added to the runner's
64
+ # scope or the gem listed as unsupported (ROADMAP §6 B2).
65
+ class UnsupportedRecipeError < ExecutionError
66
+ def initialize(construct, target:, command:)
67
+ super("unsupported shell construct (#{construct})", target: target, command: command)
68
+ end
69
+ end
70
+ end
71
+ end