rubycc 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +61 -0
- data/README.md +26 -14
- data/data/verified_gems.json +85 -63
- data/exe/rubycc-ar +11 -3
- data/include/libc/sys/cdefs.h +12 -0
- data/lib/rubycc/backend/aarch64.rb +705 -117
- data/lib/rubycc/backend/slot_residency.rb +169 -0
- data/lib/rubycc/backend/x86_64.rb +924 -137
- data/lib/rubycc/command_line.rb +339 -0
- data/lib/rubycc/compile_error.rb +6 -3
- data/lib/rubycc/compiler.rb +17 -2
- data/lib/rubycc/diagnostics.rb +105 -0
- data/lib/rubycc/doctor/gemfile.rb +12 -3
- data/lib/rubycc/doctor/verified_gems.rb +5 -1
- data/lib/rubycc/driver.rb +66 -10
- data/lib/rubycc/front/ast.rb +18 -7
- data/lib/rubycc/front/constant_evaluator.rb +12 -0
- data/lib/rubycc/front/lexeme_reader.rb +3 -1
- data/lib/rubycc/front/parser.rb +51 -18
- data/lib/rubycc/ir/analysis.rb +82 -0
- data/lib/rubycc/ir/call_convention.rb +74 -7
- data/lib/rubycc/ir/generator.rb +319 -9
- data/lib/rubycc/ir/ir.rb +39 -1
- data/lib/rubycc/ir/promotion.rb +255 -0
- data/lib/rubycc/ir/simplify.rb +570 -0
- data/lib/rubycc/link/library_resolver.rb +17 -5
- data/lib/rubycc/link/partial_linker.rb +8 -1
- data/lib/rubycc/link/shared_linker.rb +2 -2
- data/lib/rubycc/mkmf_shim.rb +178 -12
- data/lib/rubycc/objfile/ar_archive.rb +13 -2
- data/lib/rubycc/objfile/elf_reader.rb +13 -2
- data/lib/rubycc/pkgconf/parser.rb +4 -0
- data/lib/rubycc/pkgconf/resolver.rb +3 -1
- data/lib/rubycc/pkgconf/system_path_filter.rb +8 -2
- data/lib/rubycc/preprocess/preprocessor.rb +161 -37
- data/lib/rubycc/preprocess/scanner.rb +69 -14
- data/lib/rubycc/preprocess/token_converter.rb +11 -1
- data/lib/rubycc/rmake/cli.rb +32 -4
- data/lib/rubycc/rmake/executor.rb +157 -226
- data/lib/rubycc/rmake/makefile.rb +35 -13
- data/lib/rubycc/rmake/parser.rb +8 -2
- data/lib/rubycc/rmake/rmake.rb +1 -0
- data/lib/rubycc/rmake/tool_command.rb +69 -0
- data/lib/rubycc/shell.rb +510 -0
- data/lib/rubycc/type.rb +23 -7
- data/lib/rubycc/version.rb +1 -1
- data/lib/rubycc.rb +12 -0
- data/lib/rubygems_plugin.rb +31 -3
- metadata +14 -3
|
@@ -245,7 +245,8 @@ module Rubycc
|
|
|
245
245
|
# The identifiers __has_builtin (6.10.1) answers true for: exactly the
|
|
246
246
|
# builtins rubycc's front end actually recognizes — the varargs intrinsics,
|
|
247
247
|
# the branch-prediction hint, the stack allocator, offsetof, the
|
|
248
|
-
# constant/choose folds, the count-leading/trailing-zero scans
|
|
248
|
+
# constant/choose folds, the count-leading/trailing-zero scans and the
|
|
249
|
+
# set-bit count (each in its plain, "l" and "ll" spelling), the
|
|
249
250
|
# unreachable hint, memcpy, the three overflow-checked arithmetic forms,
|
|
250
251
|
# the nine __atomic_* forms and the ten legacy __sync_* forms. Every other
|
|
251
252
|
# builtin query is false, so a header that guards a fallback behind
|
|
@@ -255,7 +256,10 @@ module Rubycc
|
|
|
255
256
|
KNOWN_BUILTINS = %w[__builtin_va_start __builtin_va_arg __builtin_va_end __builtin_va_copy
|
|
256
257
|
__builtin_expect __builtin_alloca __builtin_offsetof
|
|
257
258
|
__builtin_constant_p __builtin_choose_expr
|
|
258
|
-
__builtin_ctz __builtin_ctzll
|
|
259
|
+
__builtin_ctz __builtin_ctzl __builtin_ctzll
|
|
260
|
+
__builtin_clz __builtin_clzl __builtin_clzll
|
|
261
|
+
__builtin_popcount __builtin_popcountl
|
|
262
|
+
__builtin_popcountll
|
|
259
263
|
__builtin_unreachable __builtin_memcpy
|
|
260
264
|
__builtin_add_overflow __builtin_sub_overflow
|
|
261
265
|
__builtin_mul_overflow
|
|
@@ -353,7 +357,19 @@ module Rubycc
|
|
|
353
357
|
"__ATOMIC_ACQUIRE" => "2",
|
|
354
358
|
"__ATOMIC_RELEASE" => "3",
|
|
355
359
|
"__ATOMIC_ACQ_REL" => "4",
|
|
356
|
-
"__ATOMIC_SEQ_CST" => "5"
|
|
360
|
+
"__ATOMIC_SEQ_CST" => "5",
|
|
361
|
+
# Byte order: the __ORDER_*__ enumerators are gcc's fixed values for
|
|
362
|
+
# the classic 4321/1234/3412 encodings (independent of target), while
|
|
363
|
+
# __BYTE_ORDER__ and __FLOAT_WORD_ORDER__ carry gcc's verbatim
|
|
364
|
+
# replacement text — a reference to __ORDER_LITTLE_ENDIAN__, not the
|
|
365
|
+
# number itself — so a #if re-expands it just like gcc does. Both
|
|
366
|
+
# x86-64 and aarch64 are little-endian, so no per-target branch here;
|
|
367
|
+
# values are `gcc -dM -E </dev/null` on both, 2026-08-25.
|
|
368
|
+
"__ORDER_LITTLE_ENDIAN__" => "1234",
|
|
369
|
+
"__ORDER_BIG_ENDIAN__" => "4321",
|
|
370
|
+
"__ORDER_PDP_ENDIAN__" => "3412",
|
|
371
|
+
"__BYTE_ORDER__" => "__ORDER_LITTLE_ENDIAN__",
|
|
372
|
+
"__FLOAT_WORD_ORDER__" => "__ORDER_LITTLE_ENDIAN__"
|
|
357
373
|
}.freeze
|
|
358
374
|
|
|
359
375
|
# The libc this host's C library is: "musl" or "glibc" (see LIBCS). Read
|
|
@@ -508,9 +524,19 @@ module Rubycc
|
|
|
508
524
|
# <stdarg.h> or a libc header resolves with no explicit -I; the driver's
|
|
509
525
|
# -nostdinc passes it false to search only the caller's directories.
|
|
510
526
|
def preprocess(source, filename:, include_paths: [], defines: [], system_includes: true)
|
|
527
|
+
# The directory a relative path is completed against (see
|
|
528
|
+
# #absolute_path). It comes from the process, so it is held as bytes,
|
|
529
|
+
# and it is read per run rather than once, since a caller driving
|
|
530
|
+
# several translation units may chdir between them.
|
|
531
|
+
@working_directory = Dir.pwd.b
|
|
511
532
|
system_paths = system_includes ? default_system_include_paths : []
|
|
512
|
-
@system_include_paths = system_paths.map { |path|
|
|
513
|
-
|
|
533
|
+
@system_include_paths = system_paths.map { |path| absolute_path(path) }
|
|
534
|
+
# Bytes (lib/rubycc.rb): the only spot where a caller's -I (already
|
|
535
|
+
# bytes) and the bundled/libc directories (process-derived, so not)
|
|
536
|
+
# merge before joining with a header name, itself bytes.
|
|
537
|
+
@include_paths = (include_paths + system_paths).map do |path|
|
|
538
|
+
path.encoding == Encoding::BINARY ? path : path.b
|
|
539
|
+
end
|
|
514
540
|
# #resolve_include's cache keys a resolved path off @include_paths (and,
|
|
515
541
|
# for quote includes, the includer's directory), so it must start empty
|
|
516
542
|
# every run rather than survive across calls with a different search path.
|
|
@@ -670,6 +696,7 @@ module Rubycc
|
|
|
670
696
|
when "undef" then handle_undef(name, body)
|
|
671
697
|
when "line" then handle_line(name, body)
|
|
672
698
|
when "error" then handle_error(hash, body)
|
|
699
|
+
when "warning" then handle_warning(hash, body)
|
|
673
700
|
when "pragma" then handle_pragma(body, filename)
|
|
674
701
|
else
|
|
675
702
|
raise_at(name, "invalid preprocessing directive '##{name.text}'")
|
|
@@ -1033,7 +1060,7 @@ module Rubycc
|
|
|
1033
1060
|
def process_include(hash, name, path, output)
|
|
1034
1061
|
# A header that asked for "#pragma once" is read at most once per unit; a
|
|
1035
1062
|
# later #include resolving to the same file is silently skipped (6.10.6).
|
|
1036
|
-
return if @pragma_once.key?(
|
|
1063
|
+
return if @pragma_once.key?(absolute_path(path))
|
|
1037
1064
|
|
|
1038
1065
|
# A guarded header whose guard macro is currently defined would emit
|
|
1039
1066
|
# nothing and change nothing — skip its walk entirely (see @guard_cache).
|
|
@@ -1237,7 +1264,7 @@ module Rubycc
|
|
|
1237
1264
|
# resolution beside its includer) has no "here" to resume past, so gcc
|
|
1238
1265
|
# falls back to plain #include semantics for it, which this does too.
|
|
1239
1266
|
def resolve_include_next(kind, name, includer, hash)
|
|
1240
|
-
origin = @include_origin[
|
|
1267
|
+
origin = @include_origin[absolute_path(includer)]
|
|
1241
1268
|
return resolve_include(kind, name, includer, hash) unless origin
|
|
1242
1269
|
|
|
1243
1270
|
index, path = search_include_paths(name, origin + 1)
|
|
@@ -1260,11 +1287,27 @@ module Rubycc
|
|
|
1260
1287
|
end
|
|
1261
1288
|
|
|
1262
1289
|
def record_include_origin(path, index)
|
|
1263
|
-
@include_origin[
|
|
1290
|
+
@include_origin[absolute_path(path)] = index
|
|
1291
|
+
end
|
|
1292
|
+
|
|
1293
|
+
# A path in the absolute form that serves as a file's identity here: the
|
|
1294
|
+
# key of the "#pragma once" set and of the #include_next origin map, and
|
|
1295
|
+
# the spelling the system-header test compares against. Both halves are
|
|
1296
|
+
# bytes (lib/rubycc.rb) — the path itself, which reaches this class from a
|
|
1297
|
+
# `-I` operand or a header name, and the base a relative one is completed
|
|
1298
|
+
# against, which File.expand_path would otherwise take from the process
|
|
1299
|
+
# itself and then refuse to splice a byte path onto.
|
|
1300
|
+
def absolute_path(path)
|
|
1301
|
+
path = path.b unless path.encoding == Encoding::BINARY
|
|
1302
|
+
File.expand_path(path, @working_directory)
|
|
1264
1303
|
end
|
|
1265
1304
|
|
|
1305
|
+
# A header is read as bytes, never as text in the process's locale:
|
|
1306
|
+
# File.read would tag the result with Encoding.default_external, and under
|
|
1307
|
+
# a locale of "C" that is US-ASCII, which makes the first byte past 0x7F
|
|
1308
|
+
# in the file an ArgumentError instead of a comment (see Scanner).
|
|
1266
1309
|
def read_source(path, name, hash)
|
|
1267
|
-
File.
|
|
1310
|
+
File.binread(path)
|
|
1268
1311
|
rescue SystemCallError
|
|
1269
1312
|
raise_at(hash, "#{name}: No such file or directory")
|
|
1270
1313
|
end
|
|
@@ -1324,8 +1367,8 @@ module Rubycc
|
|
|
1324
1367
|
end
|
|
1325
1368
|
|
|
1326
1369
|
# GCC diagnoses a conflicting redefinition from a system header as a
|
|
1327
|
-
# warning, not a preprocessing error. Rubycc
|
|
1328
|
-
#
|
|
1370
|
+
# warning, not a preprocessing error. Rubycc reports nothing for it and
|
|
1371
|
+
# keeps the header's later definition, continuing; this is required by
|
|
1329
1372
|
# libffi, whose target header changes FFI_GO_CLOSURES after fiddle's local
|
|
1330
1373
|
# compatibility header intentionally set it to zero. User-source
|
|
1331
1374
|
# conflicts retain the strict diagnostic above.
|
|
@@ -1333,7 +1376,7 @@ module Rubycc
|
|
|
1333
1376
|
filename = name.filename.to_s
|
|
1334
1377
|
return false if filename.empty? || filename.start_with?("<")
|
|
1335
1378
|
|
|
1336
|
-
path =
|
|
1379
|
+
path = absolute_path(filename)
|
|
1337
1380
|
@system_include_paths.any? { |root| path == root || path.start_with?("#{root}/") }
|
|
1338
1381
|
end
|
|
1339
1382
|
|
|
@@ -1460,13 +1503,29 @@ module Rubycc
|
|
|
1460
1503
|
one.zip(other).all? { |a, b| a.type == b.type && a.text == b.text }
|
|
1461
1504
|
end
|
|
1462
1505
|
|
|
1463
|
-
# --- #error
|
|
1506
|
+
# --- #error / #warning -----------------------------------------------------
|
|
1464
1507
|
|
|
1465
1508
|
def handle_error(hash, body)
|
|
1466
|
-
|
|
1467
|
-
|
|
1509
|
+
raise_at(hash, directive_message(body, "error"))
|
|
1510
|
+
end
|
|
1511
|
+
|
|
1512
|
+
# "#warning" (C23 6.10.2p2, long a GNU extension): the same message, on the
|
|
1513
|
+
# channel that reports and returns. The translation unit keeps going and
|
|
1514
|
+
# the compile still succeeds, which is the whole point — a portability
|
|
1515
|
+
# header announcing "unrecognized compiler" must not be the thing that
|
|
1516
|
+
# kills the build.
|
|
1517
|
+
def handle_warning(hash, body)
|
|
1518
|
+
warn_at(hash, directive_message(body, "warning"))
|
|
1519
|
+
end
|
|
1520
|
+
|
|
1521
|
+
# The text a #error / #warning reports: the directive's tokens spelled out
|
|
1522
|
+
# and single-spaced, never macro-expanded (matching gcc), with the
|
|
1523
|
+
# directive's own name standing in when it carried no message. Quoting is
|
|
1524
|
+
# not required of the message and nothing is stripped from it, so
|
|
1525
|
+
# `#warning foo bar` and `#warning "foo bar"` each report what they spell.
|
|
1526
|
+
def directive_message(body, name)
|
|
1468
1527
|
message = body.map(&:text).join(" ")
|
|
1469
|
-
|
|
1528
|
+
message.empty? ? "##{name}" : message
|
|
1470
1529
|
end
|
|
1471
1530
|
|
|
1472
1531
|
# --- #line -----------------------------------------------------------------
|
|
@@ -1527,7 +1586,7 @@ module Rubycc
|
|
|
1527
1586
|
first = body[0]
|
|
1528
1587
|
return unless first&.type == :identifier && first.text == "once"
|
|
1529
1588
|
|
|
1530
|
-
@pragma_once[
|
|
1589
|
+
@pragma_once[absolute_path(filename)] = true
|
|
1531
1590
|
end
|
|
1532
1591
|
|
|
1533
1592
|
# --- macro expansion -------------------------------------------------------
|
|
@@ -1667,10 +1726,10 @@ module Rubycc
|
|
|
1667
1726
|
return
|
|
1668
1727
|
end
|
|
1669
1728
|
|
|
1670
|
-
raw, commas = collect_arguments(tok, queue)
|
|
1729
|
+
raw, commas, close = collect_arguments(tok, queue)
|
|
1671
1730
|
raw = match_arity(tok, macro, raw)
|
|
1672
1731
|
invocation = Invocation.new(raw, commas, Array.new(raw.length))
|
|
1673
|
-
queue.unshift(*substitute(tok, macro, invocation))
|
|
1732
|
+
queue.unshift(*substitute(tok, macro, invocation, close))
|
|
1674
1733
|
end
|
|
1675
1734
|
|
|
1676
1735
|
# Whether the next non-newline token waiting in `queue` opens an argument
|
|
@@ -1685,9 +1744,12 @@ module Rubycc
|
|
|
1685
1744
|
# confirmed by #call_follows?, then tokens up to the matching ")", split on
|
|
1686
1745
|
# top-level commas. Nested parentheses are balanced so a comma or ")" inside
|
|
1687
1746
|
# them belongs to an argument, and a newline is inter-token space that is
|
|
1688
|
-
# dropped. Returns [arguments, commas]: the argument token lists
|
|
1747
|
+
# dropped. Returns [arguments, commas, close]: the argument token lists, the
|
|
1689
1748
|
# separating comma tokens themselves (kept so #__VA_ARGS__ can reproduce the
|
|
1690
|
-
# exact spelling)
|
|
1749
|
+
# exact spelling), and the closing ")" token itself -- #expand_function_macro
|
|
1750
|
+
# passes it on to #paint, whose hide-set intersection (6.10.3.4) needs the
|
|
1751
|
+
# ")"'s own painting, not just its text. Running out of tokens is an
|
|
1752
|
+
# unterminated invocation.
|
|
1691
1753
|
def collect_arguments(tok, queue)
|
|
1692
1754
|
queue.shift while queue.first&.newline?
|
|
1693
1755
|
queue.shift # the "("
|
|
@@ -1703,7 +1765,7 @@ module Rubycc
|
|
|
1703
1765
|
next
|
|
1704
1766
|
elsif depth.zero? && token.punct?(")")
|
|
1705
1767
|
arguments << current
|
|
1706
|
-
return [arguments, commas]
|
|
1768
|
+
return [arguments, commas, token]
|
|
1707
1769
|
elsif depth.zero? && token.punct?(",")
|
|
1708
1770
|
arguments << current
|
|
1709
1771
|
commas << token
|
|
@@ -1755,15 +1817,18 @@ module Rubycc
|
|
|
1755
1817
|
|
|
1756
1818
|
# Builds a macro's substitution by walking its replacement list once
|
|
1757
1819
|
# (6.10.3). `invocation` carries a function-like call's arguments, or is nil
|
|
1758
|
-
# for an object-like macro. "
|
|
1759
|
-
#
|
|
1760
|
-
#
|
|
1761
|
-
#
|
|
1762
|
-
#
|
|
1763
|
-
#
|
|
1764
|
-
#
|
|
1765
|
-
|
|
1766
|
-
|
|
1820
|
+
# for an object-like macro. `close` is the ")" token that closed a
|
|
1821
|
+
# function-like call's argument list (nil for an object-like macro, which
|
|
1822
|
+
# has none); #paint needs it for the hide-set intersection rule. "#"
|
|
1823
|
+
# stringizes the following parameter's raw argument; "##" pastes the token
|
|
1824
|
+
# to its left onto the operand to its right; a plain parameter becomes its
|
|
1825
|
+
# pre-expanded argument, unless it abuts a "##", where the raw argument is
|
|
1826
|
+
# used instead (6.10.3.1p1); every other token is a literal relocated to the
|
|
1827
|
+
# use site and painted with the macro's name. `span` tracks how many tokens
|
|
1828
|
+
# the token just placed contributed, so a following "##" knows its left
|
|
1829
|
+
# operand (0 marks a placemarker).
|
|
1830
|
+
def substitute(tok, macro, invocation, close = nil)
|
|
1831
|
+
painted = paint(tok, close)
|
|
1767
1832
|
rep = macro.replacement
|
|
1768
1833
|
result = []
|
|
1769
1834
|
span = 0
|
|
@@ -1791,7 +1856,40 @@ module Rubycc
|
|
|
1791
1856
|
index += 1
|
|
1792
1857
|
end
|
|
1793
1858
|
end
|
|
1794
|
-
|
|
1859
|
+
# hsadd(HS, OS), the last step of Prosser's subst (6.10.3.4p2): every
|
|
1860
|
+
# token this call produces, argument-derived ones included, gains this
|
|
1861
|
+
# call's paint. An argument keeps whatever painting it arrived with, but
|
|
1862
|
+
# a macro name passed as an argument is unpainted against *this* call --
|
|
1863
|
+
# it never passed through this replacement list -- and without this it
|
|
1864
|
+
# would sit at the end of the substitution able to meet a "(" from the
|
|
1865
|
+
# surrounding tokens and open a call the standard does not allow
|
|
1866
|
+
# ("#define f(x) x" / "f(f)(1)" reads as "f(1)", not "1").
|
|
1867
|
+
#
|
|
1868
|
+
# Non-identifiers are painted too, because a ")" among them is read by
|
|
1869
|
+
# #paint as one end of a later call: a ")" that lost this paint would
|
|
1870
|
+
# empty the hide-set intersection and stop halting a self-referential or
|
|
1871
|
+
# mutually recursive expansion. relocate and its kin already stamp
|
|
1872
|
+
# exactly `painted`, so only argument-sourced tokens really gain here.
|
|
1873
|
+
result.map! { |t| add_paint(t, painted) }
|
|
1874
|
+
end
|
|
1875
|
+
|
|
1876
|
+
# A copy of `token` with `painted` added to its suppress set, or `token`
|
|
1877
|
+
# itself when that would add nothing. The identity test comes first because
|
|
1878
|
+
# it settles the common case in one comparison: relocate and kin stamp the
|
|
1879
|
+
# very array `painted`, and a substitution is mostly their output, so the
|
|
1880
|
+
# per-name scan is only paid for tokens that came from an argument. Union,
|
|
1881
|
+
# not replacement: the token may carry a history of its own that still has
|
|
1882
|
+
# to hold.
|
|
1883
|
+
def add_paint(token, painted)
|
|
1884
|
+
return token if token.suppress.equal?(painted)
|
|
1885
|
+
return token if painted.all? { |name| token.suppress.include?(name) }
|
|
1886
|
+
|
|
1887
|
+
PPToken.new(
|
|
1888
|
+
type: token.type, text: token.text,
|
|
1889
|
+
filename: token.filename, line: token.line, column: token.column,
|
|
1890
|
+
source_line: token.source_line, space_before: token.space_before,
|
|
1891
|
+
suppress: (token.suppress | painted).freeze
|
|
1892
|
+
)
|
|
1795
1893
|
end
|
|
1796
1894
|
|
|
1797
1895
|
# The tokens a plain (non-operator) replacement element expands to: a
|
|
@@ -1972,11 +2070,27 @@ module Rubycc
|
|
|
1972
2070
|
)
|
|
1973
2071
|
end
|
|
1974
2072
|
|
|
1975
|
-
# The painting a token produced by expanding `tok` carries:
|
|
1976
|
-
#
|
|
1977
|
-
# shared list is never mutated by a later token's painting.
|
|
1978
|
-
|
|
1979
|
-
|
|
2073
|
+
# The painting a token produced by expanding `tok` carries: the names it
|
|
2074
|
+
# must not expand into again, plus the macro name now being expanded,
|
|
2075
|
+
# frozen so the shared list is never mutated by a later token's painting.
|
|
2076
|
+
# `close` is the ")" that closed a call, or nil for an object-like macro,
|
|
2077
|
+
# which has no call to close and so keeps `tok`'s own set entire.
|
|
2078
|
+
#
|
|
2079
|
+
# A call carries over only the names suppressed for *both* its name and
|
|
2080
|
+
# its ")" -- the hide-set intersection of 6.10.3.4. The two sets differ
|
|
2081
|
+
# only when the call was stitched together out of tokens with different
|
|
2082
|
+
# histories: a name produced by an expansion (painted against it) reaching
|
|
2083
|
+
# across to a ")" spelled in the source (painted against nothing). Taking
|
|
2084
|
+
# the union there would leak an enclosing expansion's paint into a call it
|
|
2085
|
+
# never wrote, which is what stalled c-testsuite 00201, where a "##" paste
|
|
2086
|
+
# forms the macro name and the source supplies its argument list. Where
|
|
2087
|
+
# name and ")" descend from the same replacement -- the ordinary case, and
|
|
2088
|
+
# what a self-referential or mutually recursive macro produces -- the two
|
|
2089
|
+
# sets are equal, so the intersection changes nothing and the recursion
|
|
2090
|
+
# guard stands.
|
|
2091
|
+
def paint(tok, close)
|
|
2092
|
+
hidden = close ? (tok.suppress & close.suppress) : tok.suppress
|
|
2093
|
+
(hidden + [tok.text]).freeze
|
|
1980
2094
|
end
|
|
1981
2095
|
|
|
1982
2096
|
def comma_token(site, suppress)
|
|
@@ -2006,6 +2120,16 @@ module Rubycc
|
|
|
2006
2120
|
)
|
|
2007
2121
|
end
|
|
2008
2122
|
|
|
2123
|
+
# As #raise_at, but on the non-fatal channel: the message is written to the
|
|
2124
|
+
# diagnostic stream in the same format and preprocessing continues.
|
|
2125
|
+
def warn_at(pp, description)
|
|
2126
|
+
Diagnostics.warn(
|
|
2127
|
+
description,
|
|
2128
|
+
filename: pp.filename, line: pp.line, column: pp.column,
|
|
2129
|
+
source_line: pp.source_line
|
|
2130
|
+
)
|
|
2131
|
+
end
|
|
2132
|
+
|
|
2009
2133
|
# As #raise_at, but for a Front::Token (produced during #if evaluation)
|
|
2010
2134
|
# rather than a preprocessing token; both expose the same location fields.
|
|
2011
2135
|
def raise_at_front(token, description)
|
|
@@ -24,6 +24,20 @@ module Rubycc
|
|
|
24
24
|
# the following physical line (see #sync). Positions are byte offsets
|
|
25
25
|
# (StringScanner's native unit); columns are converted back to character
|
|
26
26
|
# counts only when a token is actually made, and only for non-ASCII source.
|
|
27
|
+
#
|
|
28
|
+
# The unit of the whole scan is the *byte*: a C source file is a sequence of
|
|
29
|
+
# bytes (5.1.1.2), not text in whatever encoding the process's locale
|
|
30
|
+
# happens to name, and every construct this scanner recognizes is spelled in
|
|
31
|
+
# the basic character set. Anything else — a UTF-8 comment, a string literal
|
|
32
|
+
# holding non-ASCII bytes, a file that is not valid UTF-8 at all — only has
|
|
33
|
+
# to be carried through, and bytes carry it. #initialize therefore re-tags
|
|
34
|
+
# its input as ASCII-8BIT. That is what makes the compiler independent of
|
|
35
|
+
# Encoding.default_external: reading a header with File.read under a locale
|
|
36
|
+
# of "C" yields a US-ASCII string, and the first regexp or #split over it
|
|
37
|
+
# raises ArgumentError on the first byte past 0x7F (the bundled stddef.h has
|
|
38
|
+
# three, in a comment). Callers that read files hand bytes over already
|
|
39
|
+
# (File.binread); an embedder passing a String gets it re-tagged here, so no
|
|
40
|
+
# entry point can smuggle a locale-dependent encoding past this point.
|
|
27
41
|
class Scanner
|
|
28
42
|
# Two- and one-character punctuators gain the preprocessor-only "##" and
|
|
29
43
|
# "#" over the shared punctuator tables; the three-character set is unchanged.
|
|
@@ -68,8 +82,27 @@ module Rubycc
|
|
|
68
82
|
WIDE_CHAR_RE = /L'(?:[^'\\\n]|\\[^\n])*(?:'|\\)?/
|
|
69
83
|
WIDE_STRING_RE = /L"(?:[^"\\\n]|\\[^\n])*(?:"|\\)?/
|
|
70
84
|
|
|
85
|
+
# One character that matched no token class above (6.4p1). "Character",
|
|
86
|
+
# not "byte": a lead byte takes the continuation bytes that follow it, so
|
|
87
|
+
# a stray multibyte character stays one token spanning one column. Byte
|
|
88
|
+
# tokens would break the rule that a token's spelling starts at its
|
|
89
|
+
# reported column — a caret would point at the character and the message
|
|
90
|
+
# would name a third of it — and would report one error per byte. A byte
|
|
91
|
+
# that is not UTF-8 at all (a lone continuation byte, or a lead byte with
|
|
92
|
+
# nothing after it) is a token of its own, which keeps the scan total for
|
|
93
|
+
# input that is not UTF-8. Tagged /n because the scanned text is bytes.
|
|
94
|
+
OTHER_RE = /[\xC0-\xFF][\x80-\xBF]*|./n
|
|
95
|
+
|
|
96
|
+
# UTF-8 continuation bytes (0b10xxxxxx): the bytes #column_at does not
|
|
97
|
+
# count, so that a multibyte character spans one column. Spelled as a byte
|
|
98
|
+
# range over an ASCII-8BIT string, which is what the scanned text is.
|
|
99
|
+
CONTINUATION_BYTES = "\x80-\xBF".b.freeze
|
|
100
|
+
|
|
71
101
|
def initialize(source, filename:)
|
|
72
102
|
@filename = filename
|
|
103
|
+
# The scan is over bytes, so anything handed over as text is re-tagged
|
|
104
|
+
# here — never transcoded (see the class comment).
|
|
105
|
+
source = source.b unless source.encoding == Encoding::BINARY
|
|
73
106
|
# -1 keeps a trailing empty field so line numbers map 1:1 to entries.
|
|
74
107
|
@lines = source.split("\n", -1)
|
|
75
108
|
splice(source)
|
|
@@ -127,7 +160,7 @@ module Rubycc
|
|
|
127
160
|
@spliced = source
|
|
128
161
|
return
|
|
129
162
|
end
|
|
130
|
-
spliced = +""
|
|
163
|
+
spliced = +"".b
|
|
131
164
|
pos = 0
|
|
132
165
|
while (idx = source.index("\\\n", pos))
|
|
133
166
|
spliced << source[pos...idx]
|
|
@@ -166,17 +199,38 @@ module Rubycc
|
|
|
166
199
|
end
|
|
167
200
|
|
|
168
201
|
# The 1-based column of byte offset `pos`, counted in characters from the
|
|
169
|
-
# current physical line's start.
|
|
170
|
-
#
|
|
171
|
-
#
|
|
172
|
-
#
|
|
173
|
-
#
|
|
174
|
-
#
|
|
175
|
-
#
|
|
176
|
-
#
|
|
177
|
-
#
|
|
178
|
-
#
|
|
179
|
-
#
|
|
202
|
+
# current physical line's start. A "character" here is a byte that is not
|
|
203
|
+
# a UTF-8 continuation byte: the scanned text is bytes, so the count
|
|
204
|
+
# cannot be delegated to String#length, and this rule gives the character
|
|
205
|
+
# count exactly for well-formed UTF-8 (the only multibyte encoding a
|
|
206
|
+
# caret-and-column diagnostic can hope to line up on a terminal) while
|
|
207
|
+
# staying defined — one column per byte that can begin a character — for
|
|
208
|
+
# input that is not UTF-8 at all. It is also decomposable, which the
|
|
209
|
+
# incremental cursor below needs: the count over a span is the sum over
|
|
210
|
+
# its parts however the span is cut, even mid-character.
|
|
211
|
+
#
|
|
212
|
+
# A rule that looked at the byte before — "count a
|
|
213
|
+
# continuation byte unless a lead byte precedes it" — would be more
|
|
214
|
+
# accurate on malformed input and would lose exactly that: a span cut
|
|
215
|
+
# between a lead byte and its continuation would be counted one way by
|
|
216
|
+
# the first half and another by the second, and the running total below
|
|
217
|
+
# would drift. The price is paid where the input is not UTF-8 anyway: a
|
|
218
|
+
# lone continuation byte counts as nothing, so two of them report the same
|
|
219
|
+
# column and the rest of that line is reported as many columns short (the
|
|
220
|
+
# answer is pinned in test/test_source_encoding.rb rather than left to be
|
|
221
|
+
# discovered).
|
|
222
|
+
#
|
|
223
|
+
# Byte arithmetic serves ASCII source directly; non-ASCII source has to
|
|
224
|
+
# count characters, and must count them *incrementally*: measuring the
|
|
225
|
+
# whole @line_start..pos span per token costs O(line length) each time, so
|
|
226
|
+
# a single long line of L tokens costs O(L^2) — and one non-ASCII byte
|
|
227
|
+
# anywhere in the file (a comment in Japanese, a UTF-8 BOM) is enough to
|
|
228
|
+
# switch @ascii_only off for all of it. Tokens are located left to right,
|
|
229
|
+
# so keeping the last (byte offset, column) pair as a cursor and counting
|
|
230
|
+
# only the span since then makes a line cost O(line length) in total.
|
|
231
|
+
# Should a caller ever ask for an offset behind the cursor, the count
|
|
232
|
+
# restarts from the line start, so the answer stays right even if that
|
|
233
|
+
# monotonicity is lost.
|
|
180
234
|
def column_at(pos)
|
|
181
235
|
return pos - @line_start + 1 if @ascii_only
|
|
182
236
|
|
|
@@ -185,7 +239,8 @@ module Rubycc
|
|
|
185
239
|
@column_chars = 0
|
|
186
240
|
end
|
|
187
241
|
if pos > @column_pos
|
|
188
|
-
|
|
242
|
+
span = @spliced.byteslice(@column_pos, pos - @column_pos)
|
|
243
|
+
@column_chars += span.bytesize - span.count(CONTINUATION_BYTES)
|
|
189
244
|
@column_pos = pos
|
|
190
245
|
end
|
|
191
246
|
@column_chars + 1
|
|
@@ -255,7 +310,7 @@ module Rubycc
|
|
|
255
310
|
# :other token (6.4p1), deferring the decision of whether it is an
|
|
256
311
|
# error to a later phase.
|
|
257
312
|
type = :other
|
|
258
|
-
text = ss.
|
|
313
|
+
text = ss.scan(OTHER_RE)
|
|
259
314
|
end
|
|
260
315
|
make_token(type, text, line, column, space_before)
|
|
261
316
|
end
|
|
@@ -55,7 +55,7 @@ module Rubycc
|
|
|
55
55
|
tokens << front_token(pp, :punct, pp.text)
|
|
56
56
|
index += 1
|
|
57
57
|
when :other
|
|
58
|
-
raise_at(pp, "unexpected character #{pp.text
|
|
58
|
+
raise_at(pp, "unexpected character #{spelling_of(pp.text)}")
|
|
59
59
|
end
|
|
60
60
|
end
|
|
61
61
|
tokens
|
|
@@ -145,6 +145,16 @@ module Rubycc
|
|
|
145
145
|
)
|
|
146
146
|
end
|
|
147
147
|
|
|
148
|
+
# How an :other token is named in its diagnostic. The scanner works on
|
|
149
|
+
# bytes, so the token's text is bytes; a well-formed UTF-8 character is
|
|
150
|
+
# re-tagged so the message shows the character the author typed rather
|
|
151
|
+
# than the escapes of its encoding, and anything else stays bytes and is
|
|
152
|
+
# shown escaped (there is no reading of it to offer).
|
|
153
|
+
def spelling_of(text)
|
|
154
|
+
utf8 = text.dup.force_encoding(Encoding::UTF_8)
|
|
155
|
+
(utf8.valid_encoding? ? utf8 : text).inspect
|
|
156
|
+
end
|
|
157
|
+
|
|
148
158
|
def raise_at(pp, description)
|
|
149
159
|
raise CompileError.new(
|
|
150
160
|
description,
|
data/lib/rubycc/rmake/cli.rb
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "rbconfig"
|
|
4
|
+
|
|
3
5
|
require_relative "errors"
|
|
6
|
+
require_relative "../command_line"
|
|
4
7
|
require_relative "makefile"
|
|
5
8
|
|
|
6
9
|
module Rubycc
|
|
@@ -41,15 +44,22 @@ module Rubycc
|
|
|
41
44
|
end
|
|
42
45
|
|
|
43
46
|
def run(argv)
|
|
44
|
-
|
|
47
|
+
# The command line crosses into rmake here, so it is re-tagged as bytes
|
|
48
|
+
# (lib/rubycc.rb): every goal and `VAR=value` operand is matched against
|
|
49
|
+
# words of the Makefile, which the Parser hands over as bytes. The array
|
|
50
|
+
# is rebuilt rather than mutated, since an in-process caller owns the one
|
|
51
|
+
# it passes.
|
|
52
|
+
options = parse_argv(argv.map { |arg| arg.encoding == Encoding::BINARY ? arg : arg.b })
|
|
45
53
|
|
|
46
|
-
makefile_path = File.expand_path(options[:file] || DEFAULT_MAKEFILE,
|
|
54
|
+
makefile_path = File.expand_path(options[:file] || DEFAULT_MAKEFILE, dir_bytes)
|
|
47
55
|
unless File.file?(makefile_path)
|
|
48
56
|
@err.puts("rmake: #{options[:file] || DEFAULT_MAKEFILE}: No such file")
|
|
49
57
|
return 2
|
|
50
58
|
end
|
|
51
59
|
|
|
52
|
-
|
|
60
|
+
# Bytes: a Makefile carries comments (and occasionally a path) in
|
|
61
|
+
# whatever encoding its author used.
|
|
62
|
+
mk = Makefile.parse(File.binread(makefile_path), dir: @dir, overrides: options[:overrides],
|
|
53
63
|
defaults: { "MAKE" => make_default })
|
|
54
64
|
goals = options[:targets].empty? ? [nil] : options[:targets]
|
|
55
65
|
# Tool substitution is always on: rmake is rubycc's build CLI, so the
|
|
@@ -113,6 +123,15 @@ module Rubycc
|
|
|
113
123
|
{ overrides: overrides, targets: targets, jobs: [jobs, 1].max, file: file }
|
|
114
124
|
end
|
|
115
125
|
|
|
126
|
+
# A relative -f operand is completed against #dir, which came from the
|
|
127
|
+
# process (Dir.pwd, or the caller's own string) rather than from bytes, so
|
|
128
|
+
# the base is re-tagged too: File.expand_path joins the two, and a byte
|
|
129
|
+
# operand under a non-ASCII working directory is exactly the join Ruby
|
|
130
|
+
# refuses when their tags differ.
|
|
131
|
+
def dir_bytes
|
|
132
|
+
@dir_bytes ||= @dir.encoding == Encoding::BINARY ? @dir : @dir.b
|
|
133
|
+
end
|
|
134
|
+
|
|
116
135
|
def processor_count
|
|
117
136
|
require "etc"
|
|
118
137
|
Etc.nprocessors
|
|
@@ -129,6 +148,14 @@ module Rubycc
|
|
|
129
148
|
# bare "make" that would hand the recursive build to a host GNU make (or
|
|
130
149
|
# fail outright) instead of rubycc.
|
|
131
150
|
#
|
|
151
|
+
# The path is prefixed with the running interpreter and both words are
|
|
152
|
+
# quoted (a path may contain a space), the same shape the rubygems_plugin
|
|
153
|
+
# and the mkmf shim use: rmake's own `#!/usr/bin/env ruby`
|
|
154
|
+
# line cannot be resolved on a host with no /usr/bin/env (DESIGN R5), so a
|
|
155
|
+
# recursive invocation has to name the interpreter itself — and naming
|
|
156
|
+
# *this* interpreter also keeps the recursive build on the Ruby the
|
|
157
|
+
# top-level one is running under.
|
|
158
|
+
#
|
|
132
159
|
# `ENV["MAKE"]` is honoured first because RubyGems' rubygems_plugin sets it
|
|
133
160
|
# to rmake before invoking `$(MAKE)` at the top level, and that value must
|
|
134
161
|
# propagate unchanged into any recipe this run itself expands.
|
|
@@ -136,7 +163,8 @@ module Rubycc
|
|
|
136
163
|
env_make = ENV["MAKE"]
|
|
137
164
|
return env_make if env_make && !env_make.empty?
|
|
138
165
|
|
|
139
|
-
File.expand_path($PROGRAM_NAME)
|
|
166
|
+
[RbConfig.ruby, File.expand_path($PROGRAM_NAME)]
|
|
167
|
+
.map { |word| CommandLine.quote(word) }.join(" ")
|
|
140
168
|
end
|
|
141
169
|
end
|
|
142
170
|
end
|