ocran 1.4.6 → 1.5.0

data/lib/ocran/aot_toolchain.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+module Ocran
+  # Locates the ahead-of-time compilers behind --spinel and --roundhouse,
+  # and words the instructions for installing whichever of them is missing.
+  #
+  # None of them is a gem: Spinel (https://github.com/matz/spinel) is built
+  # from source and installs two commands, the compiler `spinel` and its
+  # project tool `spin`; Roundhouse (https://github.com/rubys/roundhouse) is
+  # a single prebuilt binary. Each is looked up in the environment variable
+  # named after it (a path to the command or to the directory holding it),
+  # then in PATH, then where its own install instructions put it.
+  module AotToolchain
+    Tool = Struct.new(:command, :env, :locations, keyword_init: true)
+
+    TOOLS = {
+      spinel: Tool.new(
+        command: "spinel",
+        env: "SPINEL",
+        # `make install` (default PREFIX /usr/local, or PREFIX=$HOME/.local),
+        # and the convenience symlink `make` leaves at the root of a checkout.
+        locations: %w[~/.local/bin/spinel /usr/local/bin/spinel /opt/homebrew/bin/spinel
+                      ~/spinel/spinel ~/src/spinel/spinel]
+      ),
+      spin: Tool.new(
+        command: "spin",
+        env: "SPIN",
+        locations: %w[~/.local/bin/spin /usr/local/bin/spin /opt/homebrew/bin/spin
+                      ~/spinel/bin/spin ~/src/spinel/bin/spin]
+      ),
+      roundhouse: Tool.new(
+        command: "roundhouse",
+        env: "ROUNDHOUSE",
+        # The installer script puts it in ~/.local/bin, `cargo install` in
+        # ~/.cargo/bin.
+        locations: %w[~/.local/bin/roundhouse ~/.cargo/bin/roundhouse /usr/local/bin/roundhouse
+                      /opt/homebrew/bin/roundhouse]
+      ),
+    }.freeze
+
+    SPINEL_URL = "https://github.com/matz/spinel"
+    ROUNDHOUSE_URL = "https://github.com/rubys/roundhouse"
+
+    module_function
+
+    # Absolute path of the given tool (:spinel, :spin or :roundhouse), or
+    # nil when it is not installed anywhere OCRAN knows to look. Raises when
+    # the tool's environment variable names something that is not it, since
+    # silently falling back to another copy would hide the typo.
+    def find(name, env = ENV)
+      tool = TOOLS.fetch(name)
+
+      if (specified = env[tool.env]) && !specified.empty?
+        found = executable_in(File.expand_path(specified), tool.command)
+        return found if found
+
+        raise "#{tool.env}=#{specified} does not name the #{tool.command} command " \
+              "(expected the executable itself or the directory containing it)"
+      end
+
+      search_path(tool.command, env) ||
+        tool.locations.lazy.flat_map { |pattern| Dir.glob(File.expand_path(pattern)) }
+                          .find { |path| executable?(path) }
+    end
+
+    # Hash of name => path for the given tools, with nil for missing ones.
+    def find_all(names, env = ENV)
+      names.to_h { |name| [name, find(name, env)] }
+    end
+
+    # The first line `<tool> --version` prints, or nil if it cannot be run.
+    def version(path)
+      out = IO.popen([path, "--version"], err: [:child, :out], &:read)
+      line = out.to_s.lines.first&.strip
+      $?.success? && line && !line.empty? ? line : nil
+    rescue SystemCallError
+      nil
+    end
+
+    # Instructions for installing the given missing tools, as one string.
+    def install_instructions(missing)
+      lines = []
+
+      if Gem.win_platform?
+        lines << "Spinel does not build natively on Windows yet (Roundhouse's Windows build is untested)."
+        lines << "Until it does, run OCRAN inside WSL (https://learn.microsoft.com/windows/wsl/install),"
+        lines << "where both work as on Linux; the result is then a Linux binary. Once they support"
+        lines << "Windows, installing them there is all this option needs."
+        lines << ""
+      end
+
+      if missing.include?(:spinel) || missing.include?(:spin)
+        lines << "Spinel (provides `spinel` and `spin`) is built from source, it is not a gem:"
+        lines << ""
+        lines << "    git clone #{SPINEL_URL}"
+        lines << "    cd spinel"
+        lines << "    make deps                       # fetch libprism (one-time)"
+        lines << "    make"
+        lines << "    make install PREFIX=$HOME/.local  # or: sudo make install"
+        lines << ""
+        lines << "It needs a C compiler (gcc or clang) and make. Release archives that build"
+        lines << "offline are on #{SPINEL_URL}/releases."
+        lines << ""
+      end
+
+      if missing.include?(:roundhouse)
+        lines << "Roundhouse ships as a single binary:"
+        lines << ""
+        lines << "    curl --proto '=https' --tlsv1.2 -LsSf \\"
+        lines << "      #{ROUNDHOUSE_URL}/releases/latest/download/roundhouse-installer.sh | sh"
+        lines << ""
+        lines << "or, with a Rust toolchain:"
+        lines << ""
+        lines << "    cargo install --git #{ROUNDHOUSE_URL} --bin roundhouse"
+        lines << ""
+        lines << "The Rails app compiled through it links SQLite and jemalloc, so their"
+        lines << "development headers are needed as well (Debian/Ubuntu:"
+        lines << "`sudo apt install libsqlite3-dev libjemalloc-dev`, macOS: `brew install sqlite jemalloc`)."
+        lines << ""
+      end
+
+      env_vars = missing.map { |name| TOOLS.fetch(name).env }
+      lines << "OCRAN looks in PATH and the usual install locations; set #{env_vars.join(" / ")} " \
+               "to point it at a copy installed elsewhere."
+      lines.join("\n")
+    end
+
+    def search_path(command, env = ENV)
+      exts = Gem.win_platform? ? env.fetch("PATHEXT", ".EXE;.BAT;.CMD").split(";") : [""]
+      env.fetch("PATH", "").split(File::PATH_SEPARATOR).each do |dir|
+        next if dir.empty?
+
+        exts.each do |ext|
+          path = File.join(File.expand_path(dir), "#{command}#{ext}")
+          return path if executable?(path)
+        end
+      end
+      nil
+    end
+
+    def executable_in(path, command)
+      return path if executable?(path)
+      return nil unless File.directory?(path)
+
+      names = Gem.win_platform? ? ["#{command}.exe", command] : [command]
+      [path, File.join(path, "bin")].product(names).map { |dir, name| File.join(dir, name) }
+                                    .find { |p| executable?(p) }
+    end
+
+    def executable?(path)
+      File.file?(path) && File.executable?(path)
+    end
+  end
+end

data/lib/ocran/command_output.rb

@@ -11,11 +11,18 @@ module Ocran
       puts s if Ocran.option&.verbose?
     end
 
+    # Progress goes to stdout, which is buffered when it is not a terminal:
+    # flushed first, so that a warning or an error shows up after what led
+    # to it when both streams go to the same log.
     def warning(s)
-      STDERR.puts "WARNING: #{s}" if Ocran.option&.warning?
+      return unless Ocran.option&.warning?
+
+      $stdout.flush
+      STDERR.puts "WARNING: #{s}"
     end
 
     def error(s)
+      $stdout.flush
       STDERR.puts "ERROR: #{s}"
     end
   end

data/lib/ocran/compatibility_scan.rb

@@ -0,0 +1,414 @@
+# frozen_string_literal: true
+require "pathname"
+require "rbconfig"
+
+module Ocran
+  # A static look at whether a program is likely to run on a Ruby other
+  # than the one OCRAN normally packages: Spinel (--spinel), ruby.wasm and
+  # PicoRuby (--wasm). Each of them runs a subset of what CRuby runs, and
+  # their own errors name the first thing they trip over at best. This walks
+  # the whole program instead - the script, the files it requires, and the
+  # source of every gem it requires - and reports everything known to stand
+  # in the way, sorted by where it comes from, so that "which of my gems
+  # won't work" has an answer before the first build is attempted.
+  #
+  # It is advice, not a verdict: nothing here prevents a build. Subclasses
+  # say what their target provides and what it rejects; the walk, the
+  # require resolution and the report are shared.
+  class CompatibilityScan
+    # A construct or require that will (:error) or may (:warning) fail.
+    Finding = Struct.new(:severity, :origin, :path, :line, :message, keyword_init: true)
+
+    # A gem the program requires.
+    GemUse = Struct.new(:spec, :native, :stdlib, keyword_init: true) do
+      def name = spec.name
+      def label = "#{spec.name} #{spec.version}"
+    end
+
+    # A `require` or `require_relative` of one of the program's own files,
+    # by byte range in the requiring file.
+    LocalRequire = Struct.new(:from, :start_offset, :end_offset, :target, keyword_init: true)
+
+    # Requires CRuby itself satisfies that mean nothing outside it.
+    IGNORED_FEATURES = %w[rubygems bundler/setup].freeze
+
+    MAX_SHOWN_PER_ORIGIN = 8
+
+    PROJECT = "your code"
+
+    attr_reader :findings, :gems, :include_dirs, :files, :local_requires
+
+    # The scanned files that are the program's own rather than a gem's.
+    attr_reader :project_files
+
+    # script:       the program's entry file
+    # project_dirs: directories whose files count as the program's own
+    def initialize(script, project_dirs: nil)
+      @script = Pathname(script).expand_path
+      @root = @script.dirname
+      @project_dirs = (project_dirs || [@root]).map { |d| Pathname(d).expand_path }
+      @findings = []
+      @gems = {}
+      @include_dirs = []
+      @files = []
+      @project_files = []
+      @local_requires = []
+    end
+
+    # The name the report uses for the target, e.g. "Spinel".
+    def target = raise(NotImplementedError)
+
+    # Walks the program. Returns self.
+    def analyze
+      unless prism_available?
+        @prism_missing = true
+        return self
+      end
+
+      queue = [[@script, PROJECT]]
+      seen = {}
+      until queue.empty?
+        path, origin = queue.shift
+        next if seen[path.to_s]
+
+        seen[path.to_s] = true
+        @files << path
+        @project_files << path if origin == PROJECT
+        scan_file(path, origin) { |dep, dep_origin| queue << [dep, dep_origin] }
+      end
+      self
+    end
+
+    def errors = @findings.select { |f| f.severity == :error }
+
+    def problem_gems
+      @gems.values.select { |g| gem_problem(g) || @findings.any? { |f| f.severity == :error && f.origin == g.name } }
+    end
+
+    def clean?
+      !@prism_missing && errors.empty?
+    end
+
+    def scanned? = !@prism_missing
+
+    # The report as text: a summary of the gems involved, then what was
+    # found, grouped by the code it was found in.
+    def report
+      out = []
+      if @prism_missing
+        out << "The source could not be scanned: the prism gem is not available (gem install prism)."
+        return out.join("\n")
+      end
+
+      unless @gems.empty?
+        out << "Gems required by the program:"
+        @gems.values.sort_by(&:name).each do |gem|
+          out << "  #{gem.label}: #{gem_status(gem)}"
+        end
+        out << ""
+      end
+
+      by_origin = @findings.group_by(&:origin)
+      origins = by_origin.keys.sort_by { |o| o == PROJECT ? "" : o }
+      origins.each do |origin|
+        list = by_origin[origin].sort_by { |f| [f.severity == :error ? 0 : 1, f.path.to_s, f.line.to_i] }
+        heading = origin == PROJECT ? "In your code:" : "In gem #{@gems[origin]&.label || origin}:"
+        out << heading
+        list.first(MAX_SHOWN_PER_ORIGIN).each do |f|
+          out << "  #{f.severity == :error ? "error" : "warning"}: #{location(f)}#{f.message}"
+        end
+        if list.size > MAX_SHOWN_PER_ORIGIN
+          out << "  ... and #{list.size - MAX_SHOWN_PER_ORIGIN} more"
+        end
+        out << ""
+      end
+
+      if @findings.empty?
+        out << "Nothing #{target} is known to reject was found in #{@files.size} file(s)."
+      else
+        errors_count = errors.size
+        warnings_count = @findings.size - errors_count
+        out << "#{errors_count} error(s), #{warnings_count} warning(s) in #{@files.size} scanned file(s)."
+      end
+      out.join("\n")
+    end
+
+    private
+
+    # --- Hooks for subclasses -------------------------------------------
+
+    # True when the target provides the feature itself, so `require` of it
+    # needs nothing from the host.
+    def provided_feature?(_feature) = false
+
+    # Why the target cannot use the given library (by its require name),
+    # as [severity, reason], or nil to resolve it as usual.
+    def feature_problem(_feature) = nil
+
+    # Why the target cannot use the given gem, as [severity, reason], or nil
+    # when it can. A gem without a problem has its source scanned too.
+    def gem_problem(_gem) = nil
+
+    # The status line of a gem in the report.
+    def gem_status(gem)
+      if (problem = gem_problem(gem))
+        "#{problem[0] == :error ? "incompatible" : "may not work"} - #{problem[1]}"
+      elsif (n = @findings.count { |f| f.severity == :error && f.origin == gem.name }).positive?
+        "likely incompatible - #{n} unsupported construct(s), see below"
+      else
+        "pure Ruby, nothing unsupported found"
+      end
+    end
+
+    # Called with a gem the program requires that has no problem, before
+    # its source is scanned. Spinel compiles gems from source, so it makes
+    # their require paths -I roots.
+    def gem_used(gem); end
+
+    # What to say about `require` of a standard library the target lacks,
+    # or nil when the target has Ruby's whole standard library.
+    def missing_stdlib_message(name) = "require #{name.inspect}: part of Ruby's standard library, but not one #{target} provides"
+
+    def unresolved_message(name) = "require #{name.inspect}: not a #{target} library, a project file or an installed gem"
+
+    def dynamic_require_message(name) = "#{name} with a computed name: the file cannot be found before the program runs"
+
+    # Per-node checks: called with the node, and flag findings with #flag.
+    def check_call(_node, _name, _args, _receiver, _kernel); end
+    def check_def(_node); end
+    def check_global(_node); end
+    def check_constant(_node); end
+    def check_xstring(_node); end
+
+    # --- Shared machinery -----------------------------------------------
+
+    def flag(severity, node, message)
+      add(severity, @current_origin, @current_path, node&.location&.start_line, message)
+    end
+
+    def literal?(node)
+      node.is_a?(Prism::SymbolNode) || node.is_a?(Prism::StringNode)
+    end
+
+    def kernel_receiver?(receiver)
+      receiver.nil? || (receiver.is_a?(Prism::ConstantReadNode) && receiver.name == :Kernel)
+    end
+
+    def constant?(node, name)
+      node.is_a?(Prism::ConstantReadNode) && node.name == name
+    end
+
+    def location(f)
+      return "" unless f.path
+
+      path = display_path(f.path, @gems[f.origin])
+      f.line ? "#{path}:#{f.line}: " : "#{path}: "
+    end
+
+    # Gem files relative to the gem, the program's own relative to the
+    # script's directory.
+    def display_path(path, gem)
+      base = gem ? Pathname(gem.spec.full_gem_path) : @root
+      rel = Pathname(path).relative_path_from(base).to_s
+      rel.start_with?("../..") ? path.to_s : rel
+    rescue ArgumentError
+      path.to_s
+    end
+
+    def prism_available?
+      require "prism"
+      true
+    rescue LoadError
+      false
+    end
+
+    def add(severity, origin, path, line, message)
+      @findings << Finding.new(severity: severity, origin: origin, path: path, line: line, message: message)
+    end
+
+    def scan_file(path, origin, &enqueue)
+      result = Prism.parse_file(path.to_s)
+      @current_path = path
+      @current_origin = origin
+      @enqueue = enqueue
+      result.errors.first(3).each do |e|
+        add(:error, origin, path, e.location.start_line, "syntax error: #{e.message}")
+      end
+      Visitor.new(self).visit(result.value)
+    rescue SystemCallError => e
+      add(:error, origin, path, nil, "cannot be read: #{e.message}")
+    end
+
+    # Every call: requires are resolved here, the rest goes to the target's
+    # check_call.
+    def visit_call(node)
+      name = node.name
+      args = node.arguments&.arguments || []
+      receiver = node.receiver
+      kernel = kernel_receiver?(receiver)
+
+      if %i[require require_relative].include?(name) && kernel
+        arg = args.first
+        if arg.is_a?(Prism::StringNode)
+          if name == :require
+            require_feature(node, arg.unescaped)
+          else
+            require_relative(node, arg.unescaped)
+          end
+        elsif arg
+          flag(:error, node, dynamic_require_message(name))
+        end
+        return
+      end
+
+      check_call(node, name, args, receiver, kernel)
+    end
+
+    def require_relative(node, name)
+      target = Pathname(File.expand_path(name, File.dirname(@current_path)))
+      target = Pathname("#{target}.rb") unless target.extname == ".rb"
+      if target.file?
+        record_local(node, target)
+        @enqueue.call(target, @current_origin)
+      else
+        flag(:error, node, "require_relative #{name.inspect}: #{target} does not exist")
+      end
+    end
+
+    def require_feature(node, name)
+      feature = name.delete_suffix(".rb")
+      return if IGNORED_FEATURES.include?(feature)
+
+      if (problem = feature_problem(feature))
+        flag(problem[0], node, "require #{name.inspect}: #{problem[1]}")
+        return
+      end
+      return if provided_feature?(feature)
+
+      # The program's own files, on a load path it would set up with -I or
+      # $LOAD_PATH: the script's directory and its lib/, and the require
+      # paths of the gems already found (a gem requiring its own files).
+      local_roots = [@root, @root + "lib"] + @include_dirs.map { |d| Pathname(d) }
+      local_roots.each do |root|
+        candidate = root + "#{feature}.rb"
+        next unless candidate.file?
+
+        add_include_dir(root) unless root == @root
+        owner = owner_of(candidate) || (project_file?(candidate) ? PROJECT : @current_origin)
+        record_local(node, candidate) if owner == PROJECT
+        @enqueue.call(candidate, owner)
+        return
+      end
+
+      spec = find_gem(feature)
+      unless spec
+        message = stdlib_feature?(feature) ? missing_stdlib_message(name) : unresolved_message(name)
+        flag(:error, node, message) if message
+        return
+      end
+
+      gem = (@gems[spec.name] ||= classify_gem(spec))
+      if (problem = gem_problem(gem))
+        # Reported once per requiring site outside the gem itself.
+        flag(problem[0], node, "require #{name.inspect}: gem #{gem.label} - #{problem[1]}") if @current_origin != gem.name
+        return
+      end
+
+      gem_used(gem)
+      spec.full_require_paths.each { |dir| add_include_dir(dir) }
+      file = spec.full_require_paths.lazy.map { |dir| Pathname(dir) + "#{feature}.rb" }.find(&:file?)
+      @enqueue.call(file, spec.name) if file
+    end
+
+    def record_local(node, target)
+      return unless @current_origin == PROJECT
+
+      @local_requires << LocalRequire.new(from: @current_path, start_offset: node.location.start_offset,
+                                          end_offset: node.location.end_offset, target: target)
+    end
+
+    def add_include_dir(dir)
+      dir = dir.to_s
+      @include_dirs << dir if File.directory?(dir) && !@include_dirs.include?(dir)
+    end
+
+    # The gem a file belongs to, when it is in one already found.
+    def owner_of(path)
+      @gems.each_value do |gem|
+        return gem.name if path.to_s.start_with?(gem.spec.full_gem_path + "/")
+      end
+      nil
+    end
+
+    def project_file?(path)
+      @project_dirs.any? { |dir| path.to_s.start_with?("#{dir}/") }
+    end
+
+    def find_gem(feature)
+      Gem::Specification.find_by_path(feature)
+    rescue StandardError
+      nil
+    end
+
+    def classify_gem(spec)
+      # A default gem is how Ruby ships its standard library.
+      stdlib = spec.default_gem?
+      native = !stdlib && (!spec.extensions.empty? ||
+                           Dir.glob(File.join(spec.full_gem_path, "**", "*.{so,bundle,dll}")).any?)
+      GemUse.new(spec: spec, native: native, stdlib: stdlib)
+    end
+
+    def stdlib_feature?(feature)
+      %w[rubylibdir rubyarchdir].any? do |key|
+        dir = RbConfig::CONFIG[key] or next false
+        Dir.glob(File.join(dir, "#{feature}.{rb,so,bundle,dll}")).any?
+      end
+    end
+
+    # Walks one file's syntax tree, handing every node of interest to the
+    # scan. Defined on first use, so that Prism is only loaded when a scan
+    # actually runs.
+    def self.const_missing(name)
+      return super unless name == :Visitor
+
+      require "prism"
+      visitor = Class.new(Prism::Visitor) do
+        def initialize(scan)
+          super()
+          @scan = scan
+        end
+
+        def visit_call_node(node)
+          @scan.send(:visit_call, node)
+          super
+        end
+
+        def visit_def_node(node)
+          @scan.send(:check_def, node)
+          super
+        end
+
+        def visit_global_variable_read_node(node)
+          @scan.send(:check_global, node)
+          super
+        end
+
+        def visit_constant_read_node(node)
+          @scan.send(:check_constant, node)
+          super
+        end
+
+        def visit_x_string_node(node)
+          @scan.send(:check_xstring, node)
+          super
+        end
+
+        def visit_interpolated_x_string_node(node)
+          @scan.send(:check_xstring, node)
+          super
+        end
+      end
+      CompatibilityScan.const_set(:Visitor, visitor)
+    end
+  end
+end

data/lib/ocran/cosmo_toolchain.rb

@@ -38,6 +38,11 @@ module Ocran
       "BUNDLE_LOCKFILE" => nil
     }.freeze
 
+    # Environment for running the payload on the build host: out of any
+    # bundle (see above), and seeing only its own embedded stdlib and gems,
+    # not the build host's.
+    PAYLOAD_ENV = BUNDLER_FREE_ENV.merge("GEM_HOME" => nil, "GEM_PATH" => nil, "RUBYLIB" => nil).freeze
+
     # Name of the environment variable a CosmoRuby build honors to switch
     # OFF running an embedded /zip/main.rb, i.e. to behave as an ordinary
     # interpreter (useful for inspecting a packaged application). Its
@@ -150,8 +155,7 @@ module Ocran
     def query_ruby(ruby)
       script = 'print RUBY_VERSION; print "\t"; print Gem.default_dir; ' \
                'print "\t"; print Gem::Specification.map(&:name).uniq.sort.join(",")'
-      out = IO.popen([BUNDLER_FREE_ENV.merge("GEM_HOME" => nil, "GEM_PATH" => nil, "RUBYLIB" => nil),
-                      "/bin/sh", ruby, "-e", script],
+      out = IO.popen([PAYLOAD_ENV, "/bin/sh", ruby, "-e", script],
                      err: IO::NULL, &:read)
       ok = $?.success?
       version, default_gem_dir, gem_names = out.to_s.split("\t", 3)
@@ -193,8 +197,10 @@ module Ocran
           end
         end
       RUBY
-      out = IO.popen([{ "GEM_HOME" => nil, "GEM_PATH" => nil, "RUBYOPT" => nil, "RUBYLIB" => nil },
-                      "/bin/sh", ruby, "-e", script, *features],
+      # The same environment as query_ruby: under `bundle exec` the payload
+      # would otherwise die setting up the build host's bundle, and every
+      # feature would come back as not provided.
+      out = IO.popen([PAYLOAD_ENV, "/bin/sh", ruby, "-e", script, *features],
                      err: IO::NULL, &:read)
       return [] unless $?.success?
 
@@ -347,13 +353,29 @@ module Ocran
           output = File.exist?(log) ? File.read(log) : "(no build output captured)"
           raise "Failed to build the stub with cosmocc (make -C src stub CC=#{cc}):\n#{output}"
         end
-        FileUtils.mkdir_p(File.dirname(cached))
-        FileUtils.cp(File.join(build_dir, "stub"), cached)
-        File.chmod(0755, cached)
+        install_cached(File.join(build_dir, "stub"), cached)
       end
       cached
     end
 
+    # Puts a compiled stub into the cache. The cache entry is used as soon
+    # as it exists, so it must never be seen half-written - by a build that
+    # runs at the same time, or after one that was interrupted: the copy is
+    # made under a temporary name and renamed into place, which is atomic.
+    def install_cached(stub, cached)
+      require "fileutils"
+
+      FileUtils.mkdir_p(File.dirname(cached))
+      tmp = "#{cached}.#{Process.pid}.tmp"
+      begin
+        FileUtils.cp(stub, tmp)
+        File.chmod(0755, tmp)
+        File.rename(tmp, cached)
+      ensure
+        FileUtils.rm_f(tmp)
+      end
+    end
+
     # Cache key covering the toolchain (path, mtime, size — so an updated
     # toolchain at the same path recompiles) and every stub source file.
     def cache_key(cc)