rubycc 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +61 -0
  3. data/README.md +26 -14
  4. data/data/verified_gems.json +85 -63
  5. data/exe/rubycc-ar +11 -3
  6. data/include/libc/sys/cdefs.h +12 -0
  7. data/lib/rubycc/backend/aarch64.rb +705 -117
  8. data/lib/rubycc/backend/slot_residency.rb +169 -0
  9. data/lib/rubycc/backend/x86_64.rb +924 -137
  10. data/lib/rubycc/command_line.rb +339 -0
  11. data/lib/rubycc/compile_error.rb +6 -3
  12. data/lib/rubycc/compiler.rb +17 -2
  13. data/lib/rubycc/diagnostics.rb +105 -0
  14. data/lib/rubycc/doctor/gemfile.rb +12 -3
  15. data/lib/rubycc/doctor/verified_gems.rb +5 -1
  16. data/lib/rubycc/driver.rb +66 -10
  17. data/lib/rubycc/front/ast.rb +18 -7
  18. data/lib/rubycc/front/constant_evaluator.rb +12 -0
  19. data/lib/rubycc/front/lexeme_reader.rb +3 -1
  20. data/lib/rubycc/front/parser.rb +51 -18
  21. data/lib/rubycc/ir/analysis.rb +82 -0
  22. data/lib/rubycc/ir/call_convention.rb +74 -7
  23. data/lib/rubycc/ir/generator.rb +319 -9
  24. data/lib/rubycc/ir/ir.rb +39 -1
  25. data/lib/rubycc/ir/promotion.rb +255 -0
  26. data/lib/rubycc/ir/simplify.rb +570 -0
  27. data/lib/rubycc/link/library_resolver.rb +17 -5
  28. data/lib/rubycc/link/partial_linker.rb +8 -1
  29. data/lib/rubycc/link/shared_linker.rb +2 -2
  30. data/lib/rubycc/mkmf_shim.rb +178 -12
  31. data/lib/rubycc/objfile/ar_archive.rb +13 -2
  32. data/lib/rubycc/objfile/elf_reader.rb +13 -2
  33. data/lib/rubycc/pkgconf/parser.rb +4 -0
  34. data/lib/rubycc/pkgconf/resolver.rb +3 -1
  35. data/lib/rubycc/pkgconf/system_path_filter.rb +8 -2
  36. data/lib/rubycc/preprocess/preprocessor.rb +161 -37
  37. data/lib/rubycc/preprocess/scanner.rb +69 -14
  38. data/lib/rubycc/preprocess/token_converter.rb +11 -1
  39. data/lib/rubycc/rmake/cli.rb +32 -4
  40. data/lib/rubycc/rmake/executor.rb +157 -226
  41. data/lib/rubycc/rmake/makefile.rb +35 -13
  42. data/lib/rubycc/rmake/parser.rb +8 -2
  43. data/lib/rubycc/rmake/rmake.rb +1 -0
  44. data/lib/rubycc/rmake/tool_command.rb +69 -0
  45. data/lib/rubycc/shell.rb +510 -0
  46. data/lib/rubycc/type.rb +23 -7
  47. data/lib/rubycc/version.rb +1 -1
  48. data/lib/rubycc.rb +12 -0
  49. data/lib/rubygems_plugin.rb +31 -3
  50. metadata +14 -3
@@ -245,7 +245,8 @@ module Rubycc
245
245
  # The identifiers __has_builtin (6.10.1) answers true for: exactly the
246
246
  # builtins rubycc's front end actually recognizes — the varargs intrinsics,
247
247
  # the branch-prediction hint, the stack allocator, offsetof, the
248
- # constant/choose folds, the count-leading/trailing-zero scans, the
248
+ # constant/choose folds, the count-leading/trailing-zero scans and the
249
+ # set-bit count (each in its plain, "l" and "ll" spelling), the
249
250
  # unreachable hint, memcpy, the three overflow-checked arithmetic forms,
250
251
  # the nine __atomic_* forms and the ten legacy __sync_* forms. Every other
251
252
  # builtin query is false, so a header that guards a fallback behind
@@ -255,7 +256,10 @@ module Rubycc
255
256
  KNOWN_BUILTINS = %w[__builtin_va_start __builtin_va_arg __builtin_va_end __builtin_va_copy
256
257
  __builtin_expect __builtin_alloca __builtin_offsetof
257
258
  __builtin_constant_p __builtin_choose_expr
258
- __builtin_ctz __builtin_ctzll __builtin_clz __builtin_clzll
259
+ __builtin_ctz __builtin_ctzl __builtin_ctzll
260
+ __builtin_clz __builtin_clzl __builtin_clzll
261
+ __builtin_popcount __builtin_popcountl
262
+ __builtin_popcountll
259
263
  __builtin_unreachable __builtin_memcpy
260
264
  __builtin_add_overflow __builtin_sub_overflow
261
265
  __builtin_mul_overflow
@@ -353,7 +357,19 @@ module Rubycc
353
357
  "__ATOMIC_ACQUIRE" => "2",
354
358
  "__ATOMIC_RELEASE" => "3",
355
359
  "__ATOMIC_ACQ_REL" => "4",
356
- "__ATOMIC_SEQ_CST" => "5"
360
+ "__ATOMIC_SEQ_CST" => "5",
361
+ # Byte order: the __ORDER_*__ enumerators are gcc's fixed values for
362
+ # the classic 4321/1234/3412 encodings (independent of target), while
363
+ # __BYTE_ORDER__ and __FLOAT_WORD_ORDER__ carry gcc's verbatim
364
+ # replacement text — a reference to __ORDER_LITTLE_ENDIAN__, not the
365
+ # number itself — so a #if re-expands it just like gcc does. Both
366
+ # x86-64 and aarch64 are little-endian, so no per-target branch here;
367
+ # values are `gcc -dM -E </dev/null` on both, 2026-08-25.
368
+ "__ORDER_LITTLE_ENDIAN__" => "1234",
369
+ "__ORDER_BIG_ENDIAN__" => "4321",
370
+ "__ORDER_PDP_ENDIAN__" => "3412",
371
+ "__BYTE_ORDER__" => "__ORDER_LITTLE_ENDIAN__",
372
+ "__FLOAT_WORD_ORDER__" => "__ORDER_LITTLE_ENDIAN__"
357
373
  }.freeze
358
374
 
359
375
  # The libc this host's C library is: "musl" or "glibc" (see LIBCS). Read
@@ -508,9 +524,19 @@ module Rubycc
508
524
  # <stdarg.h> or a libc header resolves with no explicit -I; the driver's
509
525
  # -nostdinc passes it false to search only the caller's directories.
510
526
  def preprocess(source, filename:, include_paths: [], defines: [], system_includes: true)
527
+ # The directory a relative path is completed against (see
528
+ # #absolute_path). It comes from the process, so it is held as bytes,
529
+ # and it is read per run rather than once, since a caller driving
530
+ # several translation units may chdir between them.
531
+ @working_directory = Dir.pwd.b
511
532
  system_paths = system_includes ? default_system_include_paths : []
512
- @system_include_paths = system_paths.map { |path| File.expand_path(path) }
513
- @include_paths = include_paths + system_paths
533
+ @system_include_paths = system_paths.map { |path| absolute_path(path) }
534
+ # Bytes (lib/rubycc.rb): the only spot where a caller's -I (already
535
+ # bytes) and the bundled/libc directories (process-derived, so not)
536
+ # merge before joining with a header name, itself bytes.
537
+ @include_paths = (include_paths + system_paths).map do |path|
538
+ path.encoding == Encoding::BINARY ? path : path.b
539
+ end
514
540
  # #resolve_include's cache keys a resolved path off @include_paths (and,
515
541
  # for quote includes, the includer's directory), so it must start empty
516
542
  # every run rather than survive across calls with a different search path.
@@ -670,6 +696,7 @@ module Rubycc
670
696
  when "undef" then handle_undef(name, body)
671
697
  when "line" then handle_line(name, body)
672
698
  when "error" then handle_error(hash, body)
699
+ when "warning" then handle_warning(hash, body)
673
700
  when "pragma" then handle_pragma(body, filename)
674
701
  else
675
702
  raise_at(name, "invalid preprocessing directive '##{name.text}'")
@@ -1033,7 +1060,7 @@ module Rubycc
1033
1060
  def process_include(hash, name, path, output)
1034
1061
  # A header that asked for "#pragma once" is read at most once per unit; a
1035
1062
  # later #include resolving to the same file is silently skipped (6.10.6).
1036
- return if @pragma_once.key?(File.expand_path(path))
1063
+ return if @pragma_once.key?(absolute_path(path))
1037
1064
 
1038
1065
  # A guarded header whose guard macro is currently defined would emit
1039
1066
  # nothing and change nothing — skip its walk entirely (see @guard_cache).
@@ -1237,7 +1264,7 @@ module Rubycc
1237
1264
  # resolution beside its includer) has no "here" to resume past, so gcc
1238
1265
  # falls back to plain #include semantics for it, which this does too.
1239
1266
  def resolve_include_next(kind, name, includer, hash)
1240
- origin = @include_origin[File.expand_path(includer)]
1267
+ origin = @include_origin[absolute_path(includer)]
1241
1268
  return resolve_include(kind, name, includer, hash) unless origin
1242
1269
 
1243
1270
  index, path = search_include_paths(name, origin + 1)
@@ -1260,11 +1287,27 @@ module Rubycc
1260
1287
  end
1261
1288
 
1262
1289
  def record_include_origin(path, index)
1263
- @include_origin[File.expand_path(path)] = index
1290
+ @include_origin[absolute_path(path)] = index
1291
+ end
1292
+
1293
+ # A path in the absolute form that serves as a file's identity here: the
1294
+ # key of the "#pragma once" set and of the #include_next origin map, and
1295
+ # the spelling the system-header test compares against. Both halves are
1296
+ # bytes (lib/rubycc.rb) — the path itself, which reaches this class from a
1297
+ # `-I` operand or a header name, and the base a relative one is completed
1298
+ # against, which File.expand_path would otherwise take from the process
1299
+ # itself and then refuse to splice a byte path onto.
1300
+ def absolute_path(path)
1301
+ path = path.b unless path.encoding == Encoding::BINARY
1302
+ File.expand_path(path, @working_directory)
1264
1303
  end
1265
1304
 
1305
+ # A header is read as bytes, never as text in the process's locale:
1306
+ # File.read would tag the result with Encoding.default_external, and under
1307
+ # a locale of "C" that is US-ASCII, which makes the first byte past 0x7F
1308
+ # in the file an ArgumentError instead of a comment (see Scanner).
1266
1309
  def read_source(path, name, hash)
1267
- File.read(path)
1310
+ File.binread(path)
1268
1311
  rescue SystemCallError
1269
1312
  raise_at(hash, "#{name}: No such file or directory")
1270
1313
  end
@@ -1324,8 +1367,8 @@ module Rubycc
1324
1367
  end
1325
1368
 
1326
1369
  # GCC diagnoses a conflicting redefinition from a system header as a
1327
- # warning, not a preprocessing error. Rubycc has no warning stream, so
1328
- # keep the header's later definition and continue; this is required by
1370
+ # warning, not a preprocessing error. Rubycc reports nothing for it and
1371
+ # keeps the header's later definition, continuing; this is required by
1329
1372
  # libffi, whose target header changes FFI_GO_CLOSURES after fiddle's local
1330
1373
  # compatibility header intentionally set it to zero. User-source
1331
1374
  # conflicts retain the strict diagnostic above.
@@ -1333,7 +1376,7 @@ module Rubycc
1333
1376
  filename = name.filename.to_s
1334
1377
  return false if filename.empty? || filename.start_with?("<")
1335
1378
 
1336
- path = File.expand_path(filename)
1379
+ path = absolute_path(filename)
1337
1380
  @system_include_paths.any? { |root| path == root || path.start_with?("#{root}/") }
1338
1381
  end
1339
1382
 
@@ -1460,13 +1503,29 @@ module Rubycc
1460
1503
  one.zip(other).all? { |a, b| a.type == b.type && a.text == b.text }
1461
1504
  end
1462
1505
 
1463
- # --- #error ----------------------------------------------------------------
1506
+ # --- #error / #warning -----------------------------------------------------
1464
1507
 
1465
1508
  def handle_error(hash, body)
1466
- # The message is the directive's tokens spelled out and single-spaced; it
1467
- # is never macro-expanded (matching gcc).
1509
+ raise_at(hash, directive_message(body, "error"))
1510
+ end
1511
+
1512
+ # "#warning" (C23 6.10.2p2, long a GNU extension): the same message, on the
1513
+ # channel that reports and returns. The translation unit keeps going and
1514
+ # the compile still succeeds, which is the whole point — a portability
1515
+ # header announcing "unrecognized compiler" must not be the thing that
1516
+ # kills the build.
1517
+ def handle_warning(hash, body)
1518
+ warn_at(hash, directive_message(body, "warning"))
1519
+ end
1520
+
1521
+ # The text a #error / #warning reports: the directive's tokens spelled out
1522
+ # and single-spaced, never macro-expanded (matching gcc), with the
1523
+ # directive's own name standing in when it carried no message. Quoting is
1524
+ # not required of the message and nothing is stripped from it, so
1525
+ # `#warning foo bar` and `#warning "foo bar"` each report what they spell.
1526
+ def directive_message(body, name)
1468
1527
  message = body.map(&:text).join(" ")
1469
- raise_at(hash, message.empty? ? "#error" : message)
1528
+ message.empty? ? "##{name}" : message
1470
1529
  end
1471
1530
 
1472
1531
  # --- #line -----------------------------------------------------------------
@@ -1527,7 +1586,7 @@ module Rubycc
1527
1586
  first = body[0]
1528
1587
  return unless first&.type == :identifier && first.text == "once"
1529
1588
 
1530
- @pragma_once[File.expand_path(filename)] = true
1589
+ @pragma_once[absolute_path(filename)] = true
1531
1590
  end
1532
1591
 
1533
1592
  # --- macro expansion -------------------------------------------------------
@@ -1667,10 +1726,10 @@ module Rubycc
1667
1726
  return
1668
1727
  end
1669
1728
 
1670
- raw, commas = collect_arguments(tok, queue)
1729
+ raw, commas, close = collect_arguments(tok, queue)
1671
1730
  raw = match_arity(tok, macro, raw)
1672
1731
  invocation = Invocation.new(raw, commas, Array.new(raw.length))
1673
- queue.unshift(*substitute(tok, macro, invocation))
1732
+ queue.unshift(*substitute(tok, macro, invocation, close))
1674
1733
  end
1675
1734
 
1676
1735
  # Whether the next non-newline token waiting in `queue` opens an argument
@@ -1685,9 +1744,12 @@ module Rubycc
1685
1744
  # confirmed by #call_follows?, then tokens up to the matching ")", split on
1686
1745
  # top-level commas. Nested parentheses are balanced so a comma or ")" inside
1687
1746
  # them belongs to an argument, and a newline is inter-token space that is
1688
- # dropped. Returns [arguments, commas]: the argument token lists and the
1747
+ # dropped. Returns [arguments, commas, close]: the argument token lists, the
1689
1748
  # separating comma tokens themselves (kept so #__VA_ARGS__ can reproduce the
1690
- # exact spelling). Running out of tokens is an unterminated invocation.
1749
+ # exact spelling), and the closing ")" token itself -- #expand_function_macro
1750
+ # passes it on to #paint, whose hide-set intersection (6.10.3.4) needs the
1751
+ # ")"'s own painting, not just its text. Running out of tokens is an
1752
+ # unterminated invocation.
1691
1753
  def collect_arguments(tok, queue)
1692
1754
  queue.shift while queue.first&.newline?
1693
1755
  queue.shift # the "("
@@ -1703,7 +1765,7 @@ module Rubycc
1703
1765
  next
1704
1766
  elsif depth.zero? && token.punct?(")")
1705
1767
  arguments << current
1706
- return [arguments, commas]
1768
+ return [arguments, commas, token]
1707
1769
  elsif depth.zero? && token.punct?(",")
1708
1770
  arguments << current
1709
1771
  commas << token
@@ -1755,15 +1817,18 @@ module Rubycc
1755
1817
 
1756
1818
  # Builds a macro's substitution by walking its replacement list once
1757
1819
  # (6.10.3). `invocation` carries a function-like call's arguments, or is nil
1758
- # for an object-like macro. "#" stringizes the following parameter's raw
1759
- # argument; "##" pastes the token to its left onto the operand to its right;
1760
- # a plain parameter becomes its pre-expanded argument, unless it abuts a
1761
- # "##", where the raw argument is used instead (6.10.3.1p1); every other
1762
- # token is a literal relocated to the use site and painted with the macro's
1763
- # name. `span` tracks how many tokens the token just placed contributed, so
1764
- # a following "##" knows its left operand (0 marks a placemarker).
1765
- def substitute(tok, macro, invocation)
1766
- painted = paint(tok)
1820
+ # for an object-like macro. `close` is the ")" token that closed a
1821
+ # function-like call's argument list (nil for an object-like macro, which
1822
+ # has none); #paint needs it for the hide-set intersection rule. "#"
1823
+ # stringizes the following parameter's raw argument; "##" pastes the token
1824
+ # to its left onto the operand to its right; a plain parameter becomes its
1825
+ # pre-expanded argument, unless it abuts a "##", where the raw argument is
1826
+ # used instead (6.10.3.1p1); every other token is a literal relocated to the
1827
+ # use site and painted with the macro's name. `span` tracks how many tokens
1828
+ # the token just placed contributed, so a following "##" knows its left
1829
+ # operand (0 marks a placemarker).
1830
+ def substitute(tok, macro, invocation, close = nil)
1831
+ painted = paint(tok, close)
1767
1832
  rep = macro.replacement
1768
1833
  result = []
1769
1834
  span = 0
@@ -1791,7 +1856,40 @@ module Rubycc
1791
1856
  index += 1
1792
1857
  end
1793
1858
  end
1794
- result
1859
+ # hsadd(HS, OS), the last step of Prosser's subst (6.10.3.4p2): every
1860
+ # token this call produces, argument-derived ones included, gains this
1861
+ # call's paint. An argument keeps whatever painting it arrived with, but
1862
+ # a macro name passed as an argument is unpainted against *this* call --
1863
+ # it never passed through this replacement list -- and without this it
1864
+ # would sit at the end of the substitution able to meet a "(" from the
1865
+ # surrounding tokens and open a call the standard does not allow
1866
+ # ("#define f(x) x" / "f(f)(1)" reads as "f(1)", not "1").
1867
+ #
1868
+ # Non-identifiers are painted too, because a ")" among them is read by
1869
+ # #paint as one end of a later call: a ")" that lost this paint would
1870
+ # empty the hide-set intersection and stop halting a self-referential or
1871
+ # mutually recursive expansion. relocate and its kin already stamp
1872
+ # exactly `painted`, so only argument-sourced tokens really gain here.
1873
+ result.map! { |t| add_paint(t, painted) }
1874
+ end
1875
+
1876
+ # A copy of `token` with `painted` added to its suppress set, or `token`
1877
+ # itself when that would add nothing. The identity test comes first because
1878
+ # it settles the common case in one comparison: relocate and kin stamp the
1879
+ # very array `painted`, and a substitution is mostly their output, so the
1880
+ # per-name scan is only paid for tokens that came from an argument. Union,
1881
+ # not replacement: the token may carry a history of its own that still has
1882
+ # to hold.
1883
+ def add_paint(token, painted)
1884
+ return token if token.suppress.equal?(painted)
1885
+ return token if painted.all? { |name| token.suppress.include?(name) }
1886
+
1887
+ PPToken.new(
1888
+ type: token.type, text: token.text,
1889
+ filename: token.filename, line: token.line, column: token.column,
1890
+ source_line: token.source_line, space_before: token.space_before,
1891
+ suppress: (token.suppress | painted).freeze
1892
+ )
1795
1893
  end
1796
1894
 
1797
1895
  # The tokens a plain (non-operator) replacement element expands to: a
@@ -1972,11 +2070,27 @@ module Rubycc
1972
2070
  )
1973
2071
  end
1974
2072
 
1975
- # The painting a token produced by expanding `tok` carries: `tok`'s own
1976
- # suppression set plus the macro name now being expanded, frozen so the
1977
- # shared list is never mutated by a later token's painting.
1978
- def paint(tok)
1979
- (tok.suppress + [tok.text]).freeze
2073
+ # The painting a token produced by expanding `tok` carries: the names it
2074
+ # must not expand into again, plus the macro name now being expanded,
2075
+ # frozen so the shared list is never mutated by a later token's painting.
2076
+ # `close` is the ")" that closed a call, or nil for an object-like macro,
2077
+ # which has no call to close and so keeps `tok`'s own set entire.
2078
+ #
2079
+ # A call carries over only the names suppressed for *both* its name and
2080
+ # its ")" -- the hide-set intersection of 6.10.3.4. The two sets differ
2081
+ # only when the call was stitched together out of tokens with different
2082
+ # histories: a name produced by an expansion (painted against it) reaching
2083
+ # across to a ")" spelled in the source (painted against nothing). Taking
2084
+ # the union there would leak an enclosing expansion's paint into a call it
2085
+ # never wrote, which is what stalled c-testsuite 00201, where a "##" paste
2086
+ # forms the macro name and the source supplies its argument list. Where
2087
+ # name and ")" descend from the same replacement -- the ordinary case, and
2088
+ # what a self-referential or mutually recursive macro produces -- the two
2089
+ # sets are equal, so the intersection changes nothing and the recursion
2090
+ # guard stands.
2091
+ def paint(tok, close)
2092
+ hidden = close ? (tok.suppress & close.suppress) : tok.suppress
2093
+ (hidden + [tok.text]).freeze
1980
2094
  end
1981
2095
 
1982
2096
  def comma_token(site, suppress)
@@ -2006,6 +2120,16 @@ module Rubycc
2006
2120
  )
2007
2121
  end
2008
2122
 
2123
+ # As #raise_at, but on the non-fatal channel: the message is written to the
2124
+ # diagnostic stream in the same format and preprocessing continues.
2125
+ def warn_at(pp, description)
2126
+ Diagnostics.warn(
2127
+ description,
2128
+ filename: pp.filename, line: pp.line, column: pp.column,
2129
+ source_line: pp.source_line
2130
+ )
2131
+ end
2132
+
2009
2133
  # As #raise_at, but for a Front::Token (produced during #if evaluation)
2010
2134
  # rather than a preprocessing token; both expose the same location fields.
2011
2135
  def raise_at_front(token, description)
@@ -24,6 +24,20 @@ module Rubycc
24
24
  # the following physical line (see #sync). Positions are byte offsets
25
25
  # (StringScanner's native unit); columns are converted back to character
26
26
  # counts only when a token is actually made, and only for non-ASCII source.
27
+ #
28
+ # The unit of the whole scan is the *byte*: a C source file is a sequence of
29
+ # bytes (5.1.1.2), not text in whatever encoding the process's locale
30
+ # happens to name, and every construct this scanner recognizes is spelled in
31
+ # the basic character set. Anything else — a UTF-8 comment, a string literal
32
+ # holding non-ASCII bytes, a file that is not valid UTF-8 at all — only has
33
+ # to be carried through, and bytes carry it. #initialize therefore re-tags
34
+ # its input as ASCII-8BIT. That is what makes the compiler independent of
35
+ # Encoding.default_external: reading a header with File.read under a locale
36
+ # of "C" yields a US-ASCII string, and the first regexp or #split over it
37
+ # raises ArgumentError on the first byte past 0x7F (the bundled stddef.h has
38
+ # three, in a comment). Callers that read files hand bytes over already
39
+ # (File.binread); an embedder passing a String gets it re-tagged here, so no
40
+ # entry point can smuggle a locale-dependent encoding past this point.
27
41
  class Scanner
28
42
  # Two- and one-character punctuators gain the preprocessor-only "##" and
29
43
  # "#" over the shared punctuator tables; the three-character set is unchanged.
@@ -68,8 +82,27 @@ module Rubycc
68
82
  WIDE_CHAR_RE = /L'(?:[^'\\\n]|\\[^\n])*(?:'|\\)?/
69
83
  WIDE_STRING_RE = /L"(?:[^"\\\n]|\\[^\n])*(?:"|\\)?/
70
84
 
85
+ # One character that matched no token class above (6.4p1). "Character",
86
+ # not "byte": a lead byte takes the continuation bytes that follow it, so
87
+ # a stray multibyte character stays one token spanning one column. Byte
88
+ # tokens would break the rule that a token's spelling starts at its
89
+ # reported column — a caret would point at the character and the message
90
+ # would name a third of it — and would report one error per byte. A byte
91
+ # that is not UTF-8 at all (a lone continuation byte, or a lead byte with
92
+ # nothing after it) is a token of its own, which keeps the scan total for
93
+ # input that is not UTF-8. Tagged /n because the scanned text is bytes.
94
+ OTHER_RE = /[\xC0-\xFF][\x80-\xBF]*|./n
95
+
96
+ # UTF-8 continuation bytes (0b10xxxxxx): the bytes #column_at does not
97
+ # count, so that a multibyte character spans one column. Spelled as a byte
98
+ # range over an ASCII-8BIT string, which is what the scanned text is.
99
+ CONTINUATION_BYTES = "\x80-\xBF".b.freeze
100
+
71
101
  def initialize(source, filename:)
72
102
  @filename = filename
103
+ # The scan is over bytes, so anything handed over as text is re-tagged
104
+ # here — never transcoded (see the class comment).
105
+ source = source.b unless source.encoding == Encoding::BINARY
73
106
  # -1 keeps a trailing empty field so line numbers map 1:1 to entries.
74
107
  @lines = source.split("\n", -1)
75
108
  splice(source)
@@ -127,7 +160,7 @@ module Rubycc
127
160
  @spliced = source
128
161
  return
129
162
  end
130
- spliced = +""
163
+ spliced = +"".b
131
164
  pos = 0
132
165
  while (idx = source.index("\\\n", pos))
133
166
  spliced << source[pos...idx]
@@ -166,17 +199,38 @@ module Rubycc
166
199
  end
167
200
 
168
201
  # The 1-based column of byte offset `pos`, counted in characters from the
169
- # current physical line's start. Byte arithmetic serves ASCII source
170
- # directly; non-ASCII source has to count characters, and must count them
171
- # *incrementally*: measuring the whole @line_start..pos span per token
172
- # costs O(line length) each time, so a single long line of L tokens costs
173
- # O(L^2) — and one non-ASCII byte anywhere in the file (a comment in
174
- # Japanese, a UTF-8 BOM) is enough to switch @ascii_only off for all of
175
- # it. Tokens are located left to right, so keeping the last (byte offset,
176
- # column) pair as a cursor and counting only the span since then makes a
177
- # line cost O(line length) in total. Should a caller ever ask for an
178
- # offset behind the cursor, the count restarts from the line start, so the
179
- # answer stays right even if that monotonicity is lost.
202
+ # current physical line's start. A "character" here is a byte that is not
203
+ # a UTF-8 continuation byte: the scanned text is bytes, so the count
204
+ # cannot be delegated to String#length, and this rule gives the character
205
+ # count exactly for well-formed UTF-8 (the only multibyte encoding a
206
+ # caret-and-column diagnostic can hope to line up on a terminal) while
207
+ # staying defined one column per byte that can begin a character — for
208
+ # input that is not UTF-8 at all. It is also decomposable, which the
209
+ # incremental cursor below needs: the count over a span is the sum over
210
+ # its parts however the span is cut, even mid-character.
211
+ #
212
+ # A rule that looked at the byte before — "count a
213
+ # continuation byte unless a lead byte precedes it" — would be more
214
+ # accurate on malformed input and would lose exactly that: a span cut
215
+ # between a lead byte and its continuation would be counted one way by
216
+ # the first half and another by the second, and the running total below
217
+ # would drift. The price is paid where the input is not UTF-8 anyway: a
218
+ # lone continuation byte counts as nothing, so two of them report the same
219
+ # column and the rest of that line is reported as many columns short (the
220
+ # answer is pinned in test/test_source_encoding.rb rather than left to be
221
+ # discovered).
222
+ #
223
+ # Byte arithmetic serves ASCII source directly; non-ASCII source has to
224
+ # count characters, and must count them *incrementally*: measuring the
225
+ # whole @line_start..pos span per token costs O(line length) each time, so
226
+ # a single long line of L tokens costs O(L^2) — and one non-ASCII byte
227
+ # anywhere in the file (a comment in Japanese, a UTF-8 BOM) is enough to
228
+ # switch @ascii_only off for all of it. Tokens are located left to right,
229
+ # so keeping the last (byte offset, column) pair as a cursor and counting
230
+ # only the span since then makes a line cost O(line length) in total.
231
+ # Should a caller ever ask for an offset behind the cursor, the count
232
+ # restarts from the line start, so the answer stays right even if that
233
+ # monotonicity is lost.
180
234
  def column_at(pos)
181
235
  return pos - @line_start + 1 if @ascii_only
182
236
 
@@ -185,7 +239,8 @@ module Rubycc
185
239
  @column_chars = 0
186
240
  end
187
241
  if pos > @column_pos
188
- @column_chars += @spliced.byteslice(@column_pos, pos - @column_pos).length
242
+ span = @spliced.byteslice(@column_pos, pos - @column_pos)
243
+ @column_chars += span.bytesize - span.count(CONTINUATION_BYTES)
189
244
  @column_pos = pos
190
245
  end
191
246
  @column_chars + 1
@@ -255,7 +310,7 @@ module Rubycc
255
310
  # :other token (6.4p1), deferring the decision of whether it is an
256
311
  # error to a later phase.
257
312
  type = :other
258
- text = ss.getch
313
+ text = ss.scan(OTHER_RE)
259
314
  end
260
315
  make_token(type, text, line, column, space_before)
261
316
  end
@@ -55,7 +55,7 @@ module Rubycc
55
55
  tokens << front_token(pp, :punct, pp.text)
56
56
  index += 1
57
57
  when :other
58
- raise_at(pp, "unexpected character #{pp.text.inspect}")
58
+ raise_at(pp, "unexpected character #{spelling_of(pp.text)}")
59
59
  end
60
60
  end
61
61
  tokens
@@ -145,6 +145,16 @@ module Rubycc
145
145
  )
146
146
  end
147
147
 
148
+ # How an :other token is named in its diagnostic. The scanner works on
149
+ # bytes, so the token's text is bytes; a well-formed UTF-8 character is
150
+ # re-tagged so the message shows the character the author typed rather
151
+ # than the escapes of its encoding, and anything else stays bytes and is
152
+ # shown escaped (there is no reading of it to offer).
153
+ def spelling_of(text)
154
+ utf8 = text.dup.force_encoding(Encoding::UTF_8)
155
+ (utf8.valid_encoding? ? utf8 : text).inspect
156
+ end
157
+
148
158
  def raise_at(pp, description)
149
159
  raise CompileError.new(
150
160
  description,
@@ -1,6 +1,9 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require "rbconfig"
4
+
3
5
  require_relative "errors"
6
+ require_relative "../command_line"
4
7
  require_relative "makefile"
5
8
 
6
9
  module Rubycc
@@ -41,15 +44,22 @@ module Rubycc
41
44
  end
42
45
 
43
46
  def run(argv)
44
- options = parse_argv(argv)
47
+ # The command line crosses into rmake here, so it is re-tagged as bytes
48
+ # (lib/rubycc.rb): every goal and `VAR=value` operand is matched against
49
+ # words of the Makefile, which the Parser hands over as bytes. The array
50
+ # is rebuilt rather than mutated, since an in-process caller owns the one
51
+ # it passes.
52
+ options = parse_argv(argv.map { |arg| arg.encoding == Encoding::BINARY ? arg : arg.b })
45
53
 
46
- makefile_path = File.expand_path(options[:file] || DEFAULT_MAKEFILE, @dir)
54
+ makefile_path = File.expand_path(options[:file] || DEFAULT_MAKEFILE, dir_bytes)
47
55
  unless File.file?(makefile_path)
48
56
  @err.puts("rmake: #{options[:file] || DEFAULT_MAKEFILE}: No such file")
49
57
  return 2
50
58
  end
51
59
 
52
- mk = Makefile.parse(File.read(makefile_path), dir: @dir, overrides: options[:overrides],
60
+ # Bytes: a Makefile carries comments (and occasionally a path) in
61
+ # whatever encoding its author used.
62
+ mk = Makefile.parse(File.binread(makefile_path), dir: @dir, overrides: options[:overrides],
53
63
  defaults: { "MAKE" => make_default })
54
64
  goals = options[:targets].empty? ? [nil] : options[:targets]
55
65
  # Tool substitution is always on: rmake is rubycc's build CLI, so the
@@ -113,6 +123,15 @@ module Rubycc
113
123
  { overrides: overrides, targets: targets, jobs: [jobs, 1].max, file: file }
114
124
  end
115
125
 
126
+ # A relative -f operand is completed against #dir, which came from the
127
+ # process (Dir.pwd, or the caller's own string) rather than from bytes, so
128
+ # the base is re-tagged too: File.expand_path joins the two, and a byte
129
+ # operand under a non-ASCII working directory is exactly the join Ruby
130
+ # refuses when their tags differ.
131
+ def dir_bytes
132
+ @dir_bytes ||= @dir.encoding == Encoding::BINARY ? @dir : @dir.b
133
+ end
134
+
116
135
  def processor_count
117
136
  require "etc"
118
137
  Etc.nprocessors
@@ -129,6 +148,14 @@ module Rubycc
129
148
  # bare "make" that would hand the recursive build to a host GNU make (or
130
149
  # fail outright) instead of rubycc.
131
150
  #
151
+ # The path is prefixed with the running interpreter and both words are
152
+ # quoted (a path may contain a space), the same shape the rubygems_plugin
153
+ # and the mkmf shim use: rmake's own `#!/usr/bin/env ruby`
154
+ # line cannot be resolved on a host with no /usr/bin/env (DESIGN R5), so a
155
+ # recursive invocation has to name the interpreter itself — and naming
156
+ # *this* interpreter also keeps the recursive build on the Ruby the
157
+ # top-level one is running under.
158
+ #
132
159
  # `ENV["MAKE"]` is honoured first because RubyGems' rubygems_plugin sets it
133
160
  # to rmake before invoking `$(MAKE)` at the top level, and that value must
134
161
  # propagate unchanged into any recipe this run itself expands.
@@ -136,7 +163,8 @@ module Rubycc
136
163
  env_make = ENV["MAKE"]
137
164
  return env_make if env_make && !env_make.empty?
138
165
 
139
- File.expand_path($PROGRAM_NAME)
166
+ [RbConfig.ruby, File.expand_path($PROGRAM_NAME)]
167
+ .map { |word| CommandLine.quote(word) }.join(" ")
140
168
  end
141
169
  end
142
170
  end