rubycc 1.0.0 → 1.1.0

data/lib/rubycc/mkmf_shim.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "rbconfig"
+require_relative "command_line"
 
 # The mkmf integration shim: requiring this file (before `require "mkmf"`)
 # reroutes every conftest command mkmf issues — compile, link, preprocess — to
@@ -20,31 +21,195 @@ require "rbconfig"
 # in-process only (no file on disk is modified) and idempotent: requiring the
 # shim twice is a no-op.
 #
-# The rubycc executables shebang-launch `require "rubycc"`, which needs the
-# repository's `lib/` on the load path when rubycc is run from a checkout rather
-# than an installed gem. The shim prepends that directory to `RUBYLIB` so the
-# value propagates to the child processes mkmf spawns.
+# The rubycc executables are launched through the running interpreter — the
+# rewritten `CC` is `<RbConfig.ruby> <path>/exe/rubycc`, not the bare path. Their
+# `#!/usr/bin/env ruby` shebang is resolved by the kernel, and a minimal image
+# (DESIGN R5) need not have /usr/bin/env: `dhi.io/ruby:4` has no coreutils at
+# all, and there every exec of the executable fails with status 127 before a line
+# of rubycc runs. Naming the interpreter also settles *which* Ruby compiles —
+# `env ruby` takes the first one on PATH, which on a multi-Ruby host is not
+# necessarily the one the gem is being installed into.
+#
+# They also `require "rubycc"`, which needs the repository's `lib/` on the load
+# path when rubycc is run from a checkout rather than an installed gem. The shim
+# prepends that directory to `RUBYLIB` so the value propagates to the child
+# processes mkmf spawns.
+#
+# Rewriting the RbConfig keys settles *what* mkmf runs; ShellFreeCommands
+# settles *how*. mkmf builds each conftest command as one string and hands it to
+# `system`/`IO.popen`, which lets Ruby decide between exec'ing it and handing it
+# to /bin/sh — and the environment DESIGN R5 targets has no /bin/sh. The shim
+# prepends the two methods that spawn (`xsystem`, `xpopen`) and converts a string
+# command into an argv array first, so nothing about the run depends on a shell
+# being there. A command that genuinely needs shell interpretation is refused
+# with its reason written to mkmf.log rather than being run through a shell that
+# may or may not exist.
 module Rubycc
   module MkmfShim
     # The repository's lib/ directory (this file lives at lib/rubycc/mkmf_shim.rb)
     # and the gcc-compatible executables that stand in for the external toolchain.
     LIB_DIR     = File.expand_path("..", __dir__)
-    RUBYCC_EXE  = File.expand_path("../../exe/rubycc", __dir__)
-    PKGCONF_EXE = File.expand_path("../../exe/rubycc-pkgconf", __dir__)
+    EXE_DIR     = File.expand_path("../../exe", __dir__)
+    RUBYCC_EXE  = File.join(EXE_DIR, "rubycc")
+    PKGCONF_EXE = File.join(EXE_DIR, "rubycc-pkgconf")
+
+    # The interpreter every rubycc executable is launched through: this very
+    # process's Ruby, by absolute path. RbConfig.ruby is what mkmf, RubyGems and
+    # rubycc's own doctor already use to re-enter Ruby.
+    RUBY = RbConfig.ruby
+
+    # A command mkmf can embed verbatim, built from +words+: the interpreter
+    # first, then the executable (see the file comment — the shebang is not
+    # resolvable everywhere). Each word is quoted, because a command that has to
+    # travel as a single string comes apart at the first space in an
+    # installation path, and both splitters that read it back — rubycc's own and
+    # the Shellwords RubyGems uses — honour the same quoting.
+    def self.launcher(*words)
+      words.map { |word| CommandLine.quote(word) }.join(" ")
+    end
+
+    # The rubycc compiler as mkmf must spell it.
+    RUBYCC_COMMAND = launcher(RUBY, RUBYCC_EXE)
 
     # The RbConfig keys rewritten to point at rubycc. Each maps to the command
-    # string mkmf embeds verbatim when it expands `$(CC)` and friends: a single
-    # shell-metacharacter-free executable path (plus, for the compound tools, the
-    # driver's own mode flag) so mkmf's `system(env, *command)` execs it directly.
+    # string mkmf embeds verbatim when it expands `$(CC)` and friends: shell-
+    # metacharacter-free words (the interpreter, the executable and, for the
+    # compound tools, the driver's own mode flag) so mkmf's `system(env,
+    # *command)` execs them directly.
+    #
+    # PKG_CONFIG is the exception that stays a bare path: mkmf resolves it with
+    # `find_executable0` (mkmf.rb's pkg_config) before running it, and that stats
+    # the value as one file name — a two-word command would simply not be found
+    # and pkg-config support would silently switch itself off. The interpreter is
+    # put in front of it at spawn time instead, by ShellFreeCommands.
     def self.replacements
       {
-        "CC"        => RUBYCC_EXE,
-        "LDSHARED"  => "#{RUBYCC_EXE} -shared",
-        "CPP"       => "#{RUBYCC_EXE} -E",
+        "CC"        => RUBYCC_COMMAND,
+        "LDSHARED"  => "#{RUBYCC_COMMAND} -shared",
+        "CPP"       => "#{RUBYCC_COMMAND} -E",
         "PKG_CONFIG" => PKGCONF_EXE
       }
     end
 
+    # Raised when a conftest command cannot be run without a shell. It is a
+    # refusal, not a build failure: mkmf reads a false return as "the compiler
+    # cannot build this", which would misreport an unrunnable command as a
+    # missing header or a broken toolchain, so the shim stops the extconf
+    # instead.
+    class ShellRequiredError < Rubycc::Error; end
+
+    # Prepended to MakeMakefile so the two methods that spawn a conftest —
+    # `xsystem` (compile/link/run) and `xpopen` (the same, reading the output) —
+    # receive an argv array where mkmf built a string. mkmf's own
+    # `expand_command` keeps whatever shape it is given (an Array stays an Array
+    # and is spawned directly; a String stays a String and reaches the shell),
+    # so converting the command on the way in is the whole change: the logging,
+    # the werror handling and the env hash are still mkmf's.
+    #
+    # The conversion runs mkmf's `$(VAR)` expansion *before* splitting, which is
+    # the order a shell saw: mkmf expanded the variables and sh did the word
+    # splitting. `super` expands again, on words that no longer hold `$(...)`.
+    module ShellFreeCommands
+      def xsystem(command, werror: false)
+        super(rubycc_spawn_form(command), werror: werror)
+      end
+
+      def xpopen(command, *mode, &block)
+        super(rubycc_spawn_form(command), *mode, &block)
+      end
+
+      private
+
+      # The command as something spawnable without a shell. Arrays are already
+      # that and only have the interpreter supplied (mkmf builds the pkg-config
+      # calls that way); a string is expanded, split and then treated the same.
+      # The method name is prefixed because prepending to MakeMakefile puts it on
+      # every extconf's Object.
+      def rubycc_spawn_form(command)
+        return MkmfShim.interpreter_form(command) unless command.is_a?(String)
+
+        _env, expanded = expand_command(command)
+        MkmfShim.spawn_form(expanded)
+      end
+    end
+
+    # Split +command+ into the argv array mkmf should spawn, or refuse it.
+    # A one-word command comes back as `[[program, program]]`: `system(env, str)`
+    # with a single string would put Ruby back in charge of choosing between exec
+    # and the shell (it scans for metacharacters and splits on spaces), and the
+    # explicit [program, argv0] form takes that choice away — `./conftest` and a
+    # path with a space in it are then spawned the same way.
+    def self.spawn_form(command)
+      argv = interpreter_form(CommandLine.argv(command))
+      argv.length == 1 ? [[argv[0], argv[0]]] : argv
+    rescue CommandLine::UnsupportedSyntaxError => e
+      refuse!(e, command)
+    end
+
+    # +argv+ with the interpreter put in front when the command runs one of
+    # rubycc's own executables directly. The rewritten `CC`/`LDSHARED`/`CPP`
+    # already carry it, but `PKG_CONFIG` cannot (see .replacements), and mkmf
+    # spawns pkg-config as an array — which reaches the kernel verbatim, shebang
+    # line and all. Supplying the interpreter here covers both spellings with one
+    # rule. A leading env hash (mkmf passes PKG_CONFIG_PATH that way) is skipped
+    # over rather than counted as the program word.
+    def self.interpreter_form(argv)
+      return argv unless argv.is_a?(Array)
+
+      index = argv.index { |word| !word.is_a?(Hash) }
+      return argv if index.nil? || !own_executable?(argv[index])
+
+      argv.dup.insert(index, RUBY)
+    end
+
+    # Whether +word+ names one of the executables this gem ships — the files
+    # whose `#!/usr/bin/env ruby` line an exec would have to resolve.
+    def self.own_executable?(word)
+      word.is_a?(String) && File.dirname(word) == EXE_DIR
+    end
+
+    # Report a command the shim will not run. The reason goes to mkmf.log, where
+    # anyone debugging a failed extension build already looks, and then the
+    # exception stops the extconf: a silent false here is exactly the
+    # misdiagnosis ("You have to install development tools first") that hid this
+    # whole problem before.
+    def self.refuse!(error, command)
+      message = <<~MSG
+        rubycc: refusing to run a conftest command that needs a shell (#{error.construct}):
+          #{command}
+        The mkmf shim spawns conftest commands as an argv array because the target
+        environment has no /bin/sh (DESIGN R5), and it does not fall back to one:
+        a fallback would make the result depend on whether a shell happens to be
+        installed. Nothing was run.
+      MSG
+      log_to_mkmf(message)
+      raise ShellRequiredError, message
+    end
+
+    # Append +message+ to mkmf.log through mkmf's own Logging module (which
+    # opens the file if it is not open yet). Logging is best-effort: if the log
+    # cannot be written the refusal itself must still be raised.
+    def self.log_to_mkmf(message)
+      return unless defined?(::MakeMakefile::Logging)
+
+      ::MakeMakefile::Logging.message("%s", message)
+    rescue SystemCallError, IOError
+      nil
+    end
+
+    # Put ShellFreeCommands in front of MakeMakefile's own methods.
+    #
+    # mkmf is normally *not* loaded yet at this point: the shim is injected with
+    # `RUBYOPT=-rrubycc/mkmf_shim`, which runs before extconf.rb requires mkmf.
+    # Defining the module first and prepending to it works in that order because
+    # mkmf reopens the module rather than replacing it, and its closing
+    # `include MakeMakefile` carries the prepended module into Object with it.
+    # Prepending an already-prepended module is a no-op, so this is idempotent.
+    def self.prepend_shell_free_commands!
+      Object.const_set(:MakeMakefile, Module.new) unless defined?(::MakeMakefile)
+      ::MakeMakefile.prepend(ShellFreeCommands)
+    end
+
     # Rewrites the toolchain keys in both RbConfig hashes and prepends the
     # repository lib/ to RUBYLIB. Idempotent: the RUBYLIB entry is added only
     # once, and re-assigning the same command strings is harmless.
@@ -65,6 +230,7 @@ module Rubycc
       end
 
       prepend_rubylib!
+      prepend_shell_free_commands!
     end
 
     # Puts the repository lib/ at the front of RUBYLIB so a rubycc executable

data/lib/rubycc/objfile/ar_archive.rb

@@ -193,6 +193,14 @@ module Rubycc
 
       # Resolves a `/N` reference: read from byte `offset` in the `//` table up to
       # the terminating `\n` and drop the trailing `/` GNU appends to each entry.
+      #
+      # The slice is left as bytes, like every other name this reader hands back
+      # (lib/rubycc.rb). An archive names a member in whatever bytes the
+      # filesystem gave it, and two strings holding the same non-ASCII bytes
+      # under different encodings are neither == nor eql? and hash apart — so
+      # tagging a name here would make it differ from a SHORT name spelled with
+      # the same bytes, and whether a caller's comparison hit would then depend
+      # on how long the name happened to be.
       def resolve_long_name(offset, pos)
         table = @name_table or
           raise ArFormatError, "member at offset #{pos} references a name table that is absent"
@@ -201,7 +209,7 @@ module Rubycc
         end
 
         stop = table.index("\n".b, offset) || table.bytesize
-        table.byteslice(offset...stop).chomp("/").force_encoding(Encoding::UTF_8)
+        table.byteslice(offset...stop).chomp("/")
       end
 
       # Turns the raw `/` symbol table into name -> member entries. Deferred until
@@ -223,7 +231,10 @@ module Rubycc
         cursor = 0
         count.times do |i|
           stop = names_blob.index("\0".b, cursor) or break
-          name = names_blob.byteslice(cursor...stop).force_encoding(Encoding::UTF_8)
+          # Bytes, like the member names: a symbol name is whatever bytes the
+          # member's ELF string table held, and one spelling for every name a
+          # reader hands back is the whole point (see resolve_long_name).
+          name = names_blob.byteslice(cursor...stop)
           cursor = stop + 1
           member = by_offset[offsets[i]]
           @symbols << { name: name, member: member } if member

data/lib/rubycc/objfile/elf_reader.rb

@@ -627,11 +627,22 @@ module Rubycc
       # A NUL-terminated string starting at `offset` within a string table's
       # bytes; an out-of-range offset yields "" rather than raising, since a
       # zero sh_name legitimately points at the leading NUL (the empty name).
+      #
+      # The slice is left as bytes, like everything rubycc reads (lib/rubycc.rb),
+      # and this one method is where every name the reader hands back comes from:
+      # section names, .symtab / .dynsym symbol names, DT_SONAME / DT_NEEDED, and
+      # version names. A string table holds whatever bytes produced the object —
+      # a C identifier cannot carry non-ASCII, an assembler label can — and two
+      # strings holding the same non-ASCII bytes under different encodings are
+      # neither == nor eql? and hash apart, so tagging here would leave a name
+      # unfindable through the bytes a caller already holds (#symbol, #section)
+      # and spelled differently from the same name read by ArReader, which takes
+      # it out of this very string table when it builds an archive's index.
       def read_string(strtab, offset)
-        return "" if offset >= strtab.bytesize
+        return "".b if offset >= strtab.bytesize
 
         stop = strtab.index("\0".b, offset) || strtab.bytesize
-        strtab.byteslice(offset...stop).force_encoding(Encoding::UTF_8)
+        strtab.byteslice(offset...stop)
       end
 
       # Guards every structural read: the whole [offset, length) span must lie

data/lib/rubycc/pkgconf/parser.rb

@@ -27,7 +27,11 @@ module Rubycc
         new(text, path: path).parse
       end
 
+      # Bytes in (lib/rubycc.rb): the .pc grammar is spelled in ASCII, and the
+      # field values are carried through (a US-ASCII-tagged Description makes
+      # String#strip raise).
       def initialize(text, path: nil)
+        text = text.b unless text.encoding == Encoding::BINARY
         @text = text
         @path = path
         @variables = {}

data/lib/rubycc/pkgconf/resolver.rb

@@ -34,7 +34,9 @@ module Rubycc
         path = SearchPath.find(name, directories: @directories)
         raise NotFoundError, "package '#{name}' not found (searched #{@directories.join(File::PATH_SEPARATOR)})" unless path
 
-        @cache[name] = Parser.parse(File.read(path), path: path)
+        # Bytes: a .pc file's Name/Description routinely carry non-ASCII (a
+        # copyright sign, an accented maintainer name).
+        @cache[name] = Parser.parse(File.binread(path), path: path)
       end
 
       # Own Cflags, then each public Requires recursively, then each

data/lib/rubycc/pkgconf/system_path_filter.rb

@@ -108,9 +108,15 @@ module Rubycc
       # An explicit value replaces the default outright, even when it is empty
       # (`PKG_CONFIG_SYSTEM_INCLUDE_PATH=` then means "no directory is a
       # system directory"); only an unset variable falls back to +defaults+.
+      #
+      # The list is held as bytes (lib/rubycc.rb): +value+ comes from the
+      # environment, while the directories it is compared against in
+      # #reject_system come out of a .pc file, and for a non-ASCII directory a
+      # differing encoding never compares equal — the `-I`/`-L` the environment
+      # asked to suppress would be emitted instead.
       def directory_list(value, defaults)
-        list = value.nil? ? defaults : value.split(File::PATH_SEPARATOR).reject(&:empty?)
-        list.map { |dir| normalize(dir) }
+        list = value.nil? ? defaults : value.b.split(File::PATH_SEPARATOR).reject(&:empty?)
+        list.map { |dir| normalize(dir.b) }
       end
 
       def set?(value)