rubycc 1.0.0 → 1.1.0

data/lib/rubycc/command_line.rb

@@ -0,0 +1,339 @@
+# frozen_string_literal: true
+
+module Rubycc
+  # The base error is normally provided by lib/rubycc.rb. This file is loadable
+  # on its own (the mkmf shim pulls it in without the compiler behind it), so
+  # define a stand-in only when the full library has not been required yet.
+  Error = Class.new(StandardError) unless defined?(Error)
+
+  # Splitting a command line the way a POSIX shell would — and *only* the part a
+  # shell does before it execs: word splitting and quote removal. The point is to
+  # run a command that was written as one string without /bin/sh, which the
+  # minimal target environment does not have (DESIGN R5).
+  #
+  # Two callers share this: rmake, which interprets Makefile recipe lines
+  # (connectors, redirections, `VAR=value` prefixes and all), and the mkmf shim,
+  # which turns the single conftest command string mkmf builds into an argv array
+  # before mkmf spawns it. Both need exactly the same word-splitting rules, so
+  # the splitter lives here rather than in either of them.
+  #
+  # Nothing here expands anything: no variables, no globs, no command
+  # substitution, and no grammar above one command's words — `for` and `if`
+  # belong to the layer above (Rubycc::Shell), which calls this one on the
+  # simple commands it has already cut out of a line. A construct that would
+  # need a shell to interpret raises UnsupportedSyntaxError — the caller reports
+  # it. Falling back to a shell is not an option we keep in reserve: a fallback
+  # would make the result depend on whether a shell happens to exist, which is
+  # the very thing this code exists to remove.
+  module CommandLine
+    # A construct the splitter does not interpret (a pipe, background `&`,
+    # command substitution, an unterminated quote, ...). It names the construct
+    # and the line it was found in; callers wrap it in their own error type with
+    # whatever context they have (rmake adds the target, the mkmf shim writes the
+    # reason to mkmf.log).
+    class UnsupportedSyntaxError < Rubycc::Error
+      attr_reader :construct, :command
+
+      def initialize(construct, command)
+        @construct = construct
+        @command = command
+        super("unsupported shell construct (#{construct}): #{command.inspect}")
+      end
+    end
+
+    # One redirection parsed off a command: which stream (:stdout/:stderr),
+    # whether it truncates or appends, and the target path (relative to the
+    # command's cwd).
+    Redirection = Struct.new(:stream, :mode, :path)
+
+    # One simple command: leading `VAR=value` assignments, the argv words and
+    # the redirections that apply to it.
+    SimpleCommand = Struct.new(:assignments, :argv, :redirections)
+
+    # Characters after which a backslash inside double quotes keeps its
+    # special meaning (POSIX quote removal, XCU 2.2 / 2.2.3): only these five
+    # make the backslash consume — and remove itself along with — the next
+    # character; before a newline both vanish (line continuation). Before any
+    # other character the backslash inside double quotes is left in the word
+    # verbatim: `"a\b"` stays `a\b`, while `"a\"b"` becomes `a"b`. Verified
+    # against /bin/sh (dash).
+    DQUOTE_BACKSLASH_SPECIAL = ["$", "`", '"', "\\", "\n"].freeze
+
+    # The word that names each connector in a diagnostic.
+    CONNECTOR_NAMES = { and: "&&", or: "||", semi: ";" }.freeze
+
+    # Shell reserved words that introduce a compound command Rubycc::Shell has a
+    # grammar for (the `for` loop, `if`, the brace group). They are still not
+    # something *this* module can make sense of — it splits one command's words,
+    # and these words are the seams between commands — so the single-command
+    # entry point (#argv) keeps refusing them. #parse does not: it is what Shell
+    # calls once it has cut a simple command out of the line, by which point the
+    # keywords are gone.
+    COMPOUND_WORDS = %w[for do done if then elif else fi { }].freeze
+
+    # Reserved words nothing here interprets: no layer of rubycc has a grammar
+    # for `case`, `while`, `until` or `!`, so they are refused wherever they
+    # appear in the command-name position. Only the unambiguous ones are listed:
+    # `in`, `time` and `select` are left out because they double as ordinary
+    # words often enough that flagging them would misfire. Checked only in the
+    # command-name position (see #parse) — `echo while` is a plain word.
+    RESERVED_WORDS = %w[case esac while until !].freeze
+
+    module_function
+
+    # Split +text+ into tokens: :word (quote-stripped), the connectors
+    # :and/:or/:semi and :redirect markers. Single quotes protect everything
+    # verbatim — backslash has no special meaning inside them. Double quotes
+    # strip a backslash only before `$`/`` ` ``/`"`/`\`/newline
+    # (DQUOTE_BACKSLASH_SPECIAL); elsewhere the backslash stays in the word.
+    # Outside any quote, a backslash preserves the literal value of the
+    # following character and disappears itself, except before a newline
+    # where both vanish (line continuation) — this is POSIX quote removal,
+    # verified against /bin/sh. mkmf relies on the outside-quotes rule: it
+    # writes `-DSYSCONFDIR=\"...\"` expecting the shell to unescape the
+    # backslash-quotes into a literal `"` in the word. Genuinely unhandled
+    # shell syntax (pipe, background, substitution, subshell) stops the split.
+    def tokenize(text)
+      tokenize_spans(text).map(&:first)
+    end
+
+    # #tokenize's split, with each token paired with the `start...stop` range it
+    # occupied in +text+. Rubycc::Shell needs to get the *source* of a word back
+    # after the split: a POSIX shell recognises words first and only then expands
+    # parameters in them (XCU 2.1), and whether `$x` is inside quotes decides
+    # whether its value is field-split. Quote removal has already thrown that
+    # away by the time #tokenize returns a word, so Shell parses the structure
+    # from these tokens, cuts each simple command out of +text+ by its span,
+    # expands it, and calls #parse on the result.
+    def tokenize_spans(text)
+      tokens = []
+      word = nil
+      word_start = nil
+      i = 0
+      n = text.length
+      # Close off the word being accumulated, if any, at +stop+.
+      flush = lambda do |stop|
+        next if word.nil?
+
+        tokens << [[:word, word], word_start...stop]
+        word = nil
+        word_start = nil
+      end
+      # Start (or continue) a word at the current position.
+      begin_word = lambda { word_start ||= i }
+
+      while i < n
+        c = text[i]
+        case c
+        when "'"
+          close = text.index(c, i + 1)
+          unsupported!("unterminated quote", text) if close.nil?
+          begin_word.call
+          word = (word || +"") + text[(i + 1)...close]
+          i = close + 1
+        when '"'
+          begin_word.call
+          segment, i = scan_double_quoted(text, i)
+          word = (word || +"") + segment
+        when "\\"
+          nxt = text[i + 1]
+          begin_word.call
+          if nxt.nil?
+            # A lone trailing backslash with nothing to escape is kept
+            # literally (verified against /bin/sh).
+            word = (word || +"") + c
+            i += 1
+          elsif nxt == "\n"
+            i += 2 # line continuation: backslash and newline both vanish
+            # Nothing was added to the word; if it is still empty the span must
+            # not claim to have started here.
+            word_start = nil if word.nil?
+          else
+            word = (word || +"") + nxt
+            i += 2
+          end
+        when " ", "\t"
+          flush.call(i)
+          i += 1
+        when "&"
+          flush.call(i)
+          unsupported!("background '&'", text) unless text[i + 1] == "&"
+          tokens << [[:and], i...(i + 2)]
+          i += 2
+        when "|"
+          flush.call(i)
+          unsupported!("pipe '|'", text) unless text[i + 1] == "|"
+          tokens << [[:or], i...(i + 2)]
+          i += 2
+        when ";"
+          flush.call(i)
+          tokens << [[:semi], i...(i + 1)]
+          i += 1
+        when ">"
+          flush.call(i)
+          if text[i + 1] == ">"
+            tokens << [[:redirect, :stdout, :append], i...(i + 2)]
+            i += 2
+          else
+            tokens << [[:redirect, :stdout, :truncate], i...(i + 1)]
+            i += 1
+          end
+        when "<", "`", "(", ")"
+          unsupported!("shell metacharacter '#{c}'", text)
+        else
+          if word.nil? && (c == "1" || c == "2") && text[i + 1] == ">"
+            stream = c == "2" ? :stderr : :stdout
+            if text[i + 2] == ">"
+              tokens << [[:redirect, stream, :append], i...(i + 3)]
+              i += 3
+            else
+              tokens << [[:redirect, stream, :truncate], i...(i + 2)]
+              i += 2
+            end
+          else
+            begin_word.call
+            word = (word || +"") + c
+            i += 1
+          end
+        end
+      end
+      flush.call(n)
+      tokens
+    end
+
+    # Scan a double-quoted segment starting at +i+ (text[i] == '"'). Applies
+    # the backslash-removal rule that is special to double quotes (see
+    # DQUOTE_BACKSLASH_SPECIAL) and returns [content, index_after_closing_quote].
+    def scan_double_quoted(text, i)
+      n = text.length
+      j = i + 1
+      buf = +""
+      loop do
+        unsupported!("unterminated quote", text) if j >= n
+
+        c = text[j]
+        if c == '"'
+          j += 1
+          break
+        elsif c == "\\" && j + 1 < n && DQUOTE_BACKSLASH_SPECIAL.include?(text[j + 1])
+          nxt = text[j + 1]
+          buf << nxt unless nxt == "\n" # backslash-newline vanishes entirely
+          j += 2
+        else
+          buf << c
+          j += 1
+        end
+      end
+      [buf, j]
+    end
+
+    # Turn +text+ into [[connector, SimpleCommand], ...]: the simple commands it
+    # holds, each tagged with the connector that precedes it (:first for the
+    # leading one, then :and/:or/:semi). A leading run of `VAR=value` words
+    # becomes that command's environment; a redirect marker consumes the
+    # following word as its target path.
+    def parse(text)
+      tokens = tokenize(text)
+      commands = []
+      connector = :first
+      assignments = []
+      argv = []
+      redirections = []
+      i = 0
+
+      flush = lambda do
+        unless assignments.empty? && argv.empty? && redirections.empty?
+          commands << [connector, SimpleCommand.new(assignments, argv, redirections)]
+        end
+        assignments = []
+        argv = []
+        redirections = []
+      end
+
+      while i < tokens.length
+        tok = tokens[i]
+        case tok[0]
+        when :word
+          w = tok[1]
+          if argv.empty? && w =~ /\A[A-Za-z_][A-Za-z0-9_]*=/
+            assignments << w
+          else
+            # A reserved word in the command-name position needs a shell
+            # grammar to make sense of. The ones a grammar now exists for
+            # (COMPOUND_WORDS) never reach here — Rubycc::Shell consumes them
+            # while cutting the line into simple commands, and calls this on
+            # what is left. The rest have no interpreter anywhere in rubycc, so
+            # they are refused: running one as a plain command would exec
+            # `while` itself and fail like a missing tool rather than like
+            # unsupported syntax, and approximating the construct is the
+            # shell-fallback risk this module exists to remove (see the file
+            # banner and mkmf-shell-free-conftest-1).
+            if argv.empty? && RESERVED_WORDS.include?(w)
+              unsupported!("shell reserved word '#{w}'", text)
+            end
+            argv << w
+          end
+        when :and, :or, :semi
+          flush.call
+          connector = tok[0]
+        when :redirect
+          nxt = tokens[i + 1]
+          unsupported!("redirection without a target", text) if nxt.nil? || nxt[0] != :word
+          redirections << Redirection.new(tok[1], tok[2], nxt[1])
+          i += 1
+        end
+        i += 1
+      end
+      flush.call
+      commands
+    end
+
+    # The argv array of +text+ read as one plain command — a program word and
+    # its arguments, nothing else. Anything a shell would have to interpret
+    # beyond word splitting and quote removal (a connector, a redirection, an
+    # environment assignment, an empty command) raises UnsupportedSyntaxError
+    # rather than being approximated. This is the mkmf shim's entry point: the
+    # conftest commands mkmf builds are plain commands, and one that is not has
+    # to be reported, not guessed at.
+    def argv(text)
+      commands = parse(text)
+      unsupported!("empty command", text) if commands.empty?
+      if commands.length > 1
+        unsupported!("command connector '#{CONNECTOR_NAMES[commands[1][0]]}'", text)
+      end
+
+      command = commands.first[1]
+      # A compound keyword is refused here even though #parse now lets it
+      # through: this entry point promises *one plain command*, and a caller
+      # that has no shell grammar (the mkmf shim) must not exec a `for`.
+      unsupported!("shell reserved word '#{command.argv.first}'", text) \
+        if COMPOUND_WORDS.include?(command.argv.first)
+      unsupported!("redirection", text) unless command.redirections.empty?
+      unsupported!("environment assignment '#{command.assignments.first}'", text) \
+        unless command.assignments.empty?
+
+      command.argv
+    end
+
+    # The inverse of the splitter for one word: +word+ spelled so that #argv
+    # (and a POSIX shell, and Shellwords, which is what RubyGems splits
+    # `ENV["MAKE"]` with) reads it back as this exact single word. Needed because
+    # a tool command has to be carried as *one string* — `CC = <ruby> <exe>` in a
+    # Makefile, `ENV["MAKE"]` for RubyGems — and an installation path is allowed
+    # to contain a space.
+    #
+    # A word of ordinary path characters is left alone, so the common case stays
+    # readable; anything else is single-quoted (which protects every character
+    # but a single quote), and a word containing a single quote is spelled with
+    # the `'\''` idiom every one of those splitters understands.
+    def quote(word)
+      return word if word.match?(/\A[A-Za-z0-9_@%+=:,.\/-]+\z/)
+
+      "'" + word.gsub("'", %q('"'"')) + "'"
+    end
+
+    def unsupported!(construct, text)
+      raise UnsupportedSyntaxError.new(construct, text)
+    end
+  end
+end

data/lib/rubycc/compile_error.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative "diagnostics"
+
 module Rubycc
   # Base error class (also defined in rubycc.rb; reopened here so this file can
   # be required standalone).
@@ -27,10 +29,11 @@ module Rubycc
 
     private
 
+    # Rendered by the shared diagnostic formatter, so an error and a warning
+    # (Diagnostics.warn) differ only in the severity word.
     def build_message
-      header = "#{filename}:#{line}:#{column}: error: #{description}"
-      caret = "#{" " * (column - 1)}^"
-      "#{header}\n#{source_line}\n#{caret}"
+      Diagnostics.render("error", description, filename: filename, line: line,
+                                               column: column, source_line: source_line)
     end
   end
 end

data/lib/rubycc/compiler.rb

@@ -5,6 +5,8 @@ require_relative "compile_error"
 require_relative "preprocess/preprocessor"
 require_relative "front/parser"
 require_relative "ir/generator"
+require_relative "ir/analysis"
+require_relative "ir/simplify"
 require_relative "backend/x86_64"
 require_relative "backend/aarch64"
 require_relative "objfile/elf_writer"
@@ -148,7 +150,19 @@ module Rubycc
       defined_names = Set.new
       relocations = []
       ir_program.functions.each do |ir_func|
-        result = backend.compile(ir_func)
+        # The local rewrites (IR::Simplify) sit here, between the generator and
+        # the backend, rather than inside either: the generator's output stays
+        # the plain lowering of the source, which is what makes it readable and
+        # testable, and a backend keeps receiving one well-defined IR rather
+        # than having to run the pass itself. Both backends lower everything the
+        # pass can produce, so the seam needs no per-target condition.
+        #
+        # The census the pass took on its way through travels with the function
+        # (IR::Analysis), because the backend needs the same counts — the
+        # transient set, and through IR::Promotion the occurrence weights — and
+        # counting the list again is the one thing this seam can save it.
+        analysis = IR::Analysis.simplified(ir_func)
+        result = backend.compile(analysis.function, analysis)
         # Align each function to 16 bytes with the target's NOP filler, keeping
         # the output deterministic and every entry point aligned.
         pad_to_alignment(text, 16, entry[:machine].text_padding)
@@ -293,7 +307,8 @@ module Rubycc
                           system_includes: true, target: "x86_64",
                           libc: Preprocess::Preprocessor.host_libc,
                           default_visibility: :default)
-      source = File.read(input_path)
+      # Bytes, not locale-encoded text: see Scanner's class comment.
+      source = File.binread(input_path)
       binary = new.compile(source, filename: input_path, include_paths: include_paths,
                                    pic: pic, defines: defines, system_includes: system_includes,
                                    target: target, libc: libc,

data/lib/rubycc/diagnostics.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+module Rubycc
+  # The compiler's diagnostic channel: one renderer for every severity, and the
+  # stream the non-fatal ones are written to.
+  #
+  # Until `#warning` there was only one severity here. A diagnostic was a raised
+  # CompileError and nothing else, so "report it" and "give up on this
+  # translation unit" were the same act, and several places in the front end say
+  # so ("a warning is not a channel this compiler has"). `#warning` needs the
+  # other half: a located message that is reported and then *returned from*, so
+  # the compile continues and exits 0 (C23 6.10.2p2, and gcc's long-standing
+  # extension of the same shape).
+  #
+  # The two halves share #render, so a warning is spelled exactly like an error
+  # apart from the severity word — one place to change the format, not two.
+  # CompileError renders itself through it (see compile_error.rb); a warning is
+  # written straight to the stream, because there is nothing to raise.
+  #
+  # The stream is process-wide state rather than a parameter threaded through
+  # every component, for the reason the first caller shows: the warning is
+  # raised deep inside the preprocessor's directive walk, whose callers
+  # (Compiler, and the parser and IR generator that will want this channel next)
+  # take no stream and have no business growing one. Concurrency does not argue
+  # against it here — rmake's `-j` isolates each step in a forked child (see
+  # Rmake::Executor), so no two translation units share this state.
+  module Diagnostics
+    # The stream state when nobody has chosen one: warnings go to the process's
+    # $stderr, looked up at emit time so a test (or a caller) that swaps $stderr
+    # is honored. It is a distinct object from nil, which means "discard".
+    INHERIT = Object.new
+    private_constant :INHERIT
+
+    # The stream lives in thread-local storage rather than in a class variable.
+    # Nothing in this compiler runs two Drivers in one process today (rmake's
+    # `-j` forks each step, and a forked worker runs its Driver in-process but
+    # alone), so a shared slot would be correct as things stand. Thread-local
+    # costs one lookup and removes the whole class of bug the moment somebody
+    # does thread a second compile through here.
+    KEY = :rubycc_diagnostics_stream
+    private_constant :KEY
+
+    # Thread-local storage cannot tell "set to nil" from "never set", and the
+    # difference matters here: nil is a caller asking to discard, an absent slot
+    # means inherit $stderr. DISCARD carries the request instead.
+    DISCARD = Object.new
+    private_constant :DISCARD
+
+    class << self
+      # A gcc-style diagnostic: the "file:line:column: severity: text" header,
+      # the offending source line, and a caret under the offending column.
+      # `severity` is the bare word ("error", "warning").
+      #
+      # The message is assembled as *bytes*. Two of its pieces come from outside
+      # and carry whatever encoding their origin gave them: the source line is
+      # ASCII-8BIT (the scanner reads bytes, see Preprocess::Scanner), while the
+      # file name — and, through a header name or a macro spelling, sometimes the
+      # description — comes from the command line and is tagged with the locale's
+      # encoding. Splicing two such strings with plain interpolation raises
+      # Encoding::CompatibilityError as soon as both hold a byte past 0x7F
+      # (compiling "日本.c" under a UTF-8 locale, with a comment in Japanese on
+      # the offending line), which would replace the diagnostic with a Ruby
+      # backtrace — the very failure this renderer exists to avoid. Re-tagging
+      # each piece as bytes removes the question: nothing is transcoded, the
+      # bytes reach the terminal as they were spelled in the source, and the
+      # caret still counts in characters (see Scanner#column_at).
+      def render(severity, description, filename:, line:, column:, source_line:)
+        message = +"".b
+        message << filename.to_s.b << ":#{line}:#{column}: #{severity}: " << description.to_s.b
+        message << "\n" << source_line.to_s.b << "\n" << (" " * (column - 1)) << "^"
+      end
+
+      # Reports a warning at a source position and returns; the compile goes on.
+      # Nothing is written when warnings are being discarded (see .to).
+      def warn(description, filename:, line:, column:, source_line:)
+        target = stream
+        return if target.nil?
+
+        target.puts render("warning", description, filename: filename, line: line,
+                                                   column: column, source_line: source_line)
+      end
+
+      # Runs the block with warnings written to `stream` — an IO, or nil to
+      # discard them (what the driver's `-w` selects). The previous target is
+      # restored afterwards, so a nested run (rmake invoking the Driver
+      # in-process) leaves nothing behind.
+      def to(stream)
+        previous = Thread.current.thread_variable_get(KEY) || INHERIT
+        Thread.current.thread_variable_set(KEY, stream.nil? ? DISCARD : stream)
+        yield
+      ensure
+        Thread.current.thread_variable_set(KEY, previous)
+      end
+
+      private
+
+      def stream
+        current = Thread.current.thread_variable_get(KEY) || INHERIT
+        return $stderr if current.equal?(INHERIT)
+
+        current.equal?(DISCARD) ? nil : current
+      end
+    end
+  end
+end

data/lib/rubycc/doctor/gemfile.rb

@@ -32,12 +32,15 @@ module Rubycc
       # Load gem information from +dir+ (or an explicit --gemfile path). A
       # Gemfile.lock next to the Gemfile is preferred; otherwise the Gemfile
       # itself is parsed. Returns a Result, or nil when neither file exists.
+      #
+      # Read as bytes (lib/rubycc.rb): a Gemfile that comments in its author's
+      # own language is ordinary, and the doctor is the first command anyone runs.
       def load(gemfile_path)
         lock = "#{gemfile_path}.lock"
         if File.file?(lock)
-          parse_lock(File.read(lock), path: lock)
+          parse_lock(File.binread(lock), path: lock)
         elsif File.file?(gemfile_path)
-          parse_gemfile(File.read(gemfile_path), path: gemfile_path)
+          parse_gemfile(File.binread(gemfile_path), path: gemfile_path)
         end
       end
 
@@ -47,7 +50,11 @@ module Rubycc
       # 4-space indent (`name (version)`), each optionally trailed by its own
       # dependencies at 6-space indent (`dep (constraint)`). DEPENDENCIES lists
       # the directly-declared gems at 2-space indent.
+      #
+      # Bytes in (lib/rubycc.rb): every construct named above is spelled in
+      # ASCII, and the rest is carried through.
       def parse_lock(text, path: nil)
+        text = text.b unless text.encoding == Encoding::BINARY
         entries = []
         direct = []
         section = nil       # current top-level section header
@@ -96,8 +103,10 @@ module Rubycc
       # declaration and an optional literal version argument. This never evaluates
       # the file, so conditionals, variables and computed names are invisible —
       # the caller reports the reduced fidelity. Group/platform blocks are not
-      # tracked; every declared gem is returned.
+      # tracked; every declared gem is returned. Bytes in, like #parse_lock: the
+      # comment this strips is exactly where a Gemfile's non-ASCII lives.
       def parse_gemfile(text, path: nil)
+        text = text.b unless text.encoding == Encoding::BINARY
         entries = []
         text.each_line do |raw|
           line = raw.sub(/#.*/, "")

data/lib/rubycc/doctor/verified_gems.rb

@@ -46,8 +46,12 @@ module Rubycc
       end
 
       # Load the database from +path+ (defaults to the shipped file).
+      #
+      # The encoding is named rather than inherited from the locale: JSON *is*
+      # UTF-8 (RFC 8259 §8.1), so there is nothing to guess. The shipped file's
+      # notes carry non-ASCII punctuation, which a locale-tagged read rejects.
       def self.load(path = DEFAULT_PATH)
-        new(JSON.parse(File.read(path)))
+        new(JSON.parse(File.read(path, encoding: Encoding::UTF_8)))
       end
 
       def initialize(raw)