ocran 1.4.5 → 1.5.0

data/lib/ocran/roundhouse_builder.rb

@@ -0,0 +1,316 @@
+# frozen_string_literal: true
+require "fileutils"
+require "open3"
+require "pathname"
+require "tmpdir"
+require_relative "command_output" unless defined?(Ocran::CommandOutput)
+require_relative "aot_toolchain"
+
+module Ocran
+  # Builds --roundhouse output: a Rails application compiled to a native
+  # server binary by Roundhouse (https://github.com/rubys/roundhouse) and
+  # Spinel (https://github.com/matz/spinel).
+  #
+  # Roundhouse lowers the app to the subset of Ruby Spinel compiles and
+  # writes it out as a spin project; its `make assets` builds static/ and
+  # `spin build` compiles the rest. The binary
+  # reads its static assets, configuration and SQLite database from beside
+  # it, so the result is a directory: the binary plus those files.
+  #
+  # Whether an app gets through depends on how much of it Roundhouse covers
+  # yet. When it does not - or when the tools are missing - the user gets
+  # Roundhouse's own analysis of the app (`roundhouse check`), whose gem
+  # census says which of the app's gems it does not model.
+  class RoundhouseBuilder
+    include CommandOutput
+
+    # What the binary reads at run time, copied from the generated project.
+    RUNTIME_DIRS = %w[static public db config].freeze
+
+    MAX_OUTPUT_LINES = 60
+    MAX_CHECK_ERRORS = 15
+
+    # Keeps the child tools out of whatever bundle OCRAN was started under:
+    # Roundhouse's spin project may shell out to Ruby (asset steps, tests).
+    BUNDLER_FREE_ENV = {
+      "RUBYOPT" => nil,
+      "BUNDLER_SETUP" => nil,
+      "BUNDLE_GEMFILE" => nil,
+      "BUNDLE_LOCKFILE" => nil
+    }.freeze
+
+    def initialize(option)
+      @option = option
+      @app = option.roundhouse_app
+      @output = option.roundhouse_output
+    end
+
+    def build
+      tools = AotToolchain.find_all(%i[roundhouse spinel spin])
+      missing = tools.select { |_, path| path.nil? }.keys
+      report_missing_tools(tools, missing) unless missing.empty?
+
+      tools.each do |name, path|
+        verbose "Using #{name} at #{path} (#{AotToolchain.version(path) || "version unknown"})"
+      end
+
+      work = Dir.mktmpdir("ocran-roundhouse")
+      project = File.join(work, "spinel")
+      @env = child_env(tools)
+
+      say "Transpiling #{display(@app)} with Roundhouse"
+      command = [tools[:roundhouse], "--target", "spinel", *@option.roundhouse_options, "-o", project, @app.to_s]
+      output, status = run(command)
+      transpile_failed(tools, output, status, work) unless status.success?
+
+      build_assets(project, work)
+
+      say "Compiling with Spinel (spin build); the first build of a large app can take a few minutes"
+      output, status = run([tools[:spin], "build"], chdir: project, env: library_path_env(project))
+      compile_failed(tools, output, status, work) unless status.success?
+
+      install(project)
+      FileUtils.remove_entry(work)
+    end
+
+    private
+
+    def run(command, chdir: Dir.pwd, env: {})
+      verbose command.join(" ")
+      Open3.capture2e(@env.merge(env), *command, chdir: chdir)
+    rescue SystemCallError => e
+      [e.message, nil]
+    end
+
+    # The generated project builds its static assets with `make assets`:
+    # Turbo and Stimulus copied out of their gems, the app's own JavaScript
+    # and stylesheets, and Tailwind when the app uses it (through npx).
+    # Bundler installs only the Gemfile's assets group for it, so no
+    # C-extension gem is compiled for a step that only copies files.
+    # Without static/ the binary still serves pages, with every stylesheet
+    # and script a 404, so a failure here stops the build.
+    def build_assets(project, work)
+      makefile = File.join(project, "Makefile")
+      return unless File.file?(makefile) && File.foreach(makefile).any? { |l| l.start_with?("assets:") }
+
+      retry_command = "cd #{File.join(work, "spinel")} && BUNDLE_ONLY=assets bundle install && BUNDLE_ONLY=assets make assets"
+      make = AotToolchain.search_path("make", @env)
+      bundle = AotToolchain.search_path("bundle", @env)
+      unless make && bundle
+        error "Building the static assets needs `make` and `bundle`, which #{make ? "`bundle` was" : bundle ? "`make` was" : "were"} not found."
+        STDERR.puts "  The generated project is kept; with both installed: #{retry_command}"
+        raise "the static assets could not be built"
+      end
+
+      say "Building static assets (make assets)"
+      env = { "BUNDLE_ONLY" => "assets" }
+      [[bundle, "install"], [make, "assets"]].each do |command|
+        output, status = run(command, chdir: project, env: env)
+        next if status&.success?
+
+        print_failure("Building the static assets failed (#{command.map { |c| File.basename(c) }.join(" ")})", output, status)
+        STDERR.puts "Things to check:"
+        STDERR.puts "  - an app that builds Tailwind needs Node.js and npm for `npx @tailwindcss/cli`"
+        STDERR.puts "  - the generated project is kept; to retry: #{retry_command}"
+        raise "the static assets could not be built"
+      end
+    end
+
+    # spin links the allocator spin.toml names by bare name (-ljemalloc), and
+    # the linker searches only its default directories, which on Apple
+    # Silicon do not include Homebrew's /opt/homebrew/lib. pkg-config knows
+    # where the library is, so its -L directories go on LIBRARY_PATH (which
+    # gcc and clang both read) for the build.
+    def library_path_env(project)
+      manifest = File.join(project, "spin.toml")
+      allocator = File.file?(manifest) && File.read(manifest)[/^\s*allocator\s*=\s*"([^"]+)"/, 1]
+      return {} unless allocator && allocator != "system"
+
+      pkg_config = AotToolchain.search_path("pkg-config", @env)
+      return {} unless pkg_config
+
+      output, status = Open3.capture2(pkg_config, "--libs-only-L", allocator, err: File::NULL)
+      return {} unless status.success?
+
+      dirs = output.split.select { |flag| flag.start_with?("-L") }.map { |flag| flag.delete_prefix("-L") }
+      return {} if dirs.empty?
+
+      verbose "Adding #{dirs.join(", ")} (pkg-config #{allocator}) to LIBRARY_PATH"
+      { "LIBRARY_PATH" => (dirs + [ENV.fetch("LIBRARY_PATH", "")]).reject(&:empty?).join(File::PATH_SEPARATOR) }
+    rescue SystemCallError
+      {}
+    end
+
+    # spin calls spinel by name, so a spinel found outside PATH (through
+    # SPINEL or a conventional location) is put on the child's PATH.
+    def child_env(tools)
+      dirs = tools.values.compact.map { |path| File.dirname(path) }.uniq
+      BUNDLER_FREE_ENV.merge("PATH" => (dirs + [ENV.fetch("PATH", "")]).join(File::PATH_SEPARATOR))
+    end
+
+    def report_missing_tools(tools, missing)
+      commands = missing.map { |name| "`#{AotToolchain::TOOLS[name].command}`" }
+      list = commands.size > 1 ? "#{commands[0..-2].join(", ")} and #{commands[-1]}" : commands.first
+      error "--roundhouse needs #{list}, which #{missing.size == 1 ? "was" : "were"} not found."
+      STDERR.puts
+      STDERR.puts AotToolchain.install_instructions(missing)
+      STDERR.puts
+
+      if tools[:roundhouse]
+        @env = child_env(tools)
+        print_check(tools[:roundhouse], "Meanwhile, Roundhouse's analysis of #{display(@app)}:")
+      else
+        STDERR.puts "Once Roundhouse is installed, `roundhouse check --continue #{display(@app)}` lists the gems"
+        STDERR.puts "and constructs of the app it does not cover yet; OCRAN shows it when a build fails."
+      end
+      raise "the Roundhouse toolchain is not installed"
+    end
+
+    def transpile_failed(tools, output, status, work)
+      print_failure("Roundhouse could not transpile #{display(@app)}", output, status)
+      print_check(tools[:roundhouse], "Roundhouse's analysis of the app:")
+      STDERR.puts
+      STDERR.puts "Things to try:"
+      STDERR.puts "  - what Roundhouse covers: #{AotToolchain::ROUNDHOUSE_URL}/blob/main/docs/guide/rails-coverage.md"
+      STDERR.puts "  - --roundhouse-opt --survey --roundhouse-opt --allow-unsupported transpiles an app that is"
+      STDERR.puts "    not fully covered yet, with a stub at each unsupported site"
+      STDERR.puts "  - the generated project so far is in #{work}"
+      raise "Roundhouse could not transpile #{display(@app)}"
+    end
+
+    def compile_failed(tools, output, status, work)
+      print_failure("Spinel could not compile the transpiled app (spin build)", output, status)
+
+      STDERR.puts "Things to check:"
+      versions = tools.map { |name, path| "#{name}: #{AotToolchain.version(path) || "version unknown"}" }
+      STDERR.puts "  - versions in use (#{versions.join("; ")}); each Roundhouse snapshot names the Spinel"
+      STDERR.puts "    release it was tested against in #{AotToolchain::ROUNDHOUSE_URL}/blob/main/RELEASES.md,"
+      STDERR.puts "    and a mismatch is the most common cause of a failing build"
+      missing_libs.each do |lib, package|
+        STDERR.puts "  - the #{lib} development headers seem to be missing (#{package})"
+      end
+      unlinked_libs(output).each do |lib|
+        STDERR.puts "  - the linker could not find lib#{lib}: install it (Debian/Ubuntu: lib#{lib}-dev; macOS: brew install #{lib});"
+        STDERR.puts "    if it is installed outside the linker's default directories, set LIBRARY_PATH to its lib directory"
+      end
+      STDERR.puts "  - the generated project is kept in #{File.join(work, "spinel")}; run `spin build` there to retry"
+      STDERR.puts "  - a compiler error in the generated code is a bug in Roundhouse or Spinel: please report it"
+      raise "Spinel could not compile the transpiled app"
+    end
+
+    def print_failure(heading, output, status)
+      lines = output.to_s.lines
+      exit_status = status ? "exit status #{status.exitstatus.inspect}" : "could not be started"
+      STDERR.puts "#{heading} (#{exit_status}):"
+      STDERR.puts "  ... (#{lines.size - MAX_OUTPUT_LINES} earlier lines omitted)" if lines.size > MAX_OUTPUT_LINES
+      STDERR.puts lines.last(MAX_OUTPUT_LINES).map { |l| "  #{l}" }.join
+      STDERR.puts
+    end
+
+    # Prints the useful part of `roundhouse check --continue`: the gem
+    # census (which gems Roundhouse does not model), the summary, the first
+    # errors and the punch list of unsupported constructs. The full report
+    # of a large app runs to thousands of lines.
+    def print_check(roundhouse, heading)
+      say "Analyzing #{display(@app)} with roundhouse check"
+      output, status = run([roundhouse, "check", "--continue", @app.to_s])
+      return STDERR.puts("(roundhouse check could not be run: #{output.to_s.strip})") unless status
+
+      lines = output.lines.map(&:chomp)
+      summary = lines.grep(/\Aroundhouse-check:/)
+      errors = lines.grep(/: error\[/)
+      survey_start = lines.index { |l| l.include?("Survey:") }
+      survey = survey_start ? lines[survey_start..].take_while { |l| !l.start_with?("roundhouse-check:") }.reject(&:empty?) : []
+
+      STDERR.puts heading
+      STDERR.puts
+      summary.each { |l| STDERR.puts "  #{l}" }
+      if summary.any? { |l| l.match?(/\d+ unknown/) && !l.match?(/\b0 unknown/) }
+        STDERR.puts "  (\"unknown\" gems are ones Roundhouse does not model: code using them cannot be compiled yet)"
+      end
+      unless errors.empty?
+        STDERR.puts
+        errors.first(MAX_CHECK_ERRORS).each { |l| STDERR.puts "  #{l}" }
+        STDERR.puts "  ... and #{errors.size - MAX_CHECK_ERRORS} more error(s)" if errors.size > MAX_CHECK_ERRORS
+      end
+      unless survey.empty?
+        STDERR.puts
+        survey.first(30).each { |l| STDERR.puts "  #{l}" }
+      end
+      STDERR.puts
+      STDERR.puts "  Full report: roundhouse check --continue #{display(@app)}"
+    end
+
+    # Libraries the generated project links that pkg-config cannot find,
+    # as [library, package hint] pairs. Empty when pkg-config is not there
+    # to ask.
+    def missing_libs
+      return [] unless AotToolchain.search_path("pkg-config")
+
+      { "sqlite3" => "libsqlite3-dev / brew install sqlite",
+        "jemalloc" => "libjemalloc-dev / brew install jemalloc" }.reject do |lib, _|
+        Kernel.system("pkg-config", "--exists", lib)
+      end.to_a
+    end
+
+    # Libraries the link step could not find, from GNU ld's
+    # "cannot find -lfoo" and Apple ld's "library 'foo' not found".
+    def unlinked_libs(output)
+      output.to_s.scan(/cannot find -l([\w.+-]+)|library '([\w.+-]+)' not found/).flatten.compact.uniq
+    end
+
+    def install(project)
+      bin_dir = File.join(project, "build", "bin")
+      binaries = Dir.glob(File.join(bin_dir, "*")).select { |f| File.file?(f) && File.executable?(f) }
+      binary = binaries.find { |f| File.basename(f, ".*") == @app.basename.to_s } || binaries.first
+      raise "spin build succeeded but produced no executable in #{bin_dir}" unless binary
+
+      @output.mkpath
+      target = @output + File.basename(binary)
+      FileUtils.cp(binary, target)
+      File.chmod(0o755, target)
+
+      RUNTIME_DIRS.each do |dir|
+        source = File.join(project, dir)
+        next unless File.directory?(source)
+
+        FileUtils.rm_rf(@output + dir)
+        FileUtils.cp_r(source, @output + dir)
+      end
+      # storage/ holds the database and uploads, so a rebuild keeps it.
+      (@output + "storage").mkpath
+
+      seed_database
+      say "Finished building #{display(@output)}"
+      say "Run it with: cd #{display(@output)} && ./#{File.basename(binary)}  (serves on port 3000; PORT or -p to change)"
+      say "It needs libsqlite3 and libjemalloc at run time (Debian/Ubuntu: libsqlite3-0 libjemalloc2)."
+    end
+
+    # Creates the database from the generated seed on the first build, when
+    # the sqlite3 command is there to do it.
+    def seed_database
+      seed = @output + "db" + "seed.sql"
+      database = @output + "storage" + "development.sqlite3"
+      return unless seed.file? && !database.exist?
+
+      sqlite3 = AotToolchain.search_path("sqlite3")
+      unless sqlite3
+        say "Create the database before the first start: sqlite3 storage/development.sqlite3 < db/seed.sql"
+        return
+      end
+
+      verbose "Seeding #{database} from #{seed}"
+      unless Kernel.system(sqlite3, database.to_s, in: seed.to_s)
+        warning "Could not create the database from #{seed}; run: sqlite3 storage/development.sqlite3 < db/seed.sql"
+      end
+    end
+
+    # Relative to the working directory when inside it, else absolute.
+    def display(path)
+      rel = Pathname(path).relative_path_from(Pathname.pwd).to_s
+      rel.start_with?("..") ? path.to_s : rel
+    rescue ArgumentError
+      path.to_s
+    end
+  end
+end

data/lib/ocran/runner.rb

@@ -6,8 +6,19 @@ module Ocran
     load File.expand_path("command_output.rb", __dir__)
     include CommandOutput
 
-    def fatal_error(statement)
-      error statement
+    # Errors that describe a problem with the input or the build host - an
+    # invalid option, a missing file, a path OCRAN cannot write, a value too
+    # large for the executable format - rather than a bug in OCRAN. They
+    # are reported by their message; a backtrace would only bury it. Any
+    # other exception is a bug and keeps Ruby's full report.
+    USER_ERRORS = [RuntimeError, ArgumentError, SystemCallError, IOError].freeze
+
+    # Reports a USER_ERRORS exception and exits. With --verbose the
+    # backtrace is shown too, for when the message is not enough.
+    def user_error(e)
+      message = e.instance_of?(RuntimeError) ? e.message : "#{e.message} (#{e.class})"
+      error message
+      STDERR.puts e.backtrace.map { |line| "\tfrom #{line}" } if @option&.verbose? && e.backtrace
       exit false
     end
 
@@ -18,11 +29,9 @@ module Ocran
       load File.expand_path("option.rb", __dir__)
       @option = Option.new.tap do |opt|
         opt.parse(ARGV)
-      rescue RuntimeError => e
-        # Capture RuntimeError during parsing and display an appropriate
-        # error message to the user. This error usually occurs from invalid
-        # option arguments.
-        fatal_error e.message
+      rescue *USER_ERRORS => e
+        # Invalid option arguments, usually; report them as such.
+        user_error e
       else
         # Update ARGV with the parsed command line arguments to pass to
         # the user's script. This ensures the script executes based on
@@ -31,6 +40,7 @@ module Ocran
       end
 
       Ocran.option = @option
+      @option.warnings.each { |message| warning message }
 
       @ignore_modules = ObjectSpace.each_object(Module).to_a
     end
@@ -49,6 +59,7 @@ module Ocran
       else
         warn_about_foreign_bundle
       end
+      expose_bundle_install_path
       say "Loading script to check dependencies"
       $PROGRAM_NAME = @option.script.to_s
     end
@@ -110,6 +121,58 @@ module Ocran
               (app_gemfile ? "; pass --gemfile #{app_gemfile} to package against that bundle" : "")
     end
 
+    # `bundle install` puts the gems where Bundler is configured to put them,
+    # and a configured path - `bundle config set path vendor/bundle`, which is
+    # what ruby/setup-ruby's `bundler-cache: true` does - is a place only
+    # Bundler looks. The dependency run loads the script as plain Ruby, so
+    # its first `require` of a gem from such a bundle raised LoadError
+    # although the bundle was installed (github issue #61).
+    #
+    # Add the bundle's install directory to the gem path. Its gems become
+    # visible exactly as if they had been installed into Ruby's own gem
+    # directory, and nothing more: setting the bundle up would also hide
+    # every gem the Gemfile does not list, breaking scripts that work today.
+    # A bundle installed into the system gems, the ordinary case outside CI,
+    # changes nothing, and under `bundle exec` Bundler has already put its
+    # gems on the gem path.
+    def expose_bundle_install_path
+      return if @pre_env.bundler_setup_loaded?
+
+      gemfile = @option.application_gemfile or return
+      path = bundle_install_path(gemfile) or return
+      return if Gem.path.any? { |gem_path| same_file?(gem_path, path) }
+
+      verbose "Adding #{path}, where the bundle of #{gemfile} is installed, to the gem path"
+      Gem.paths = { "GEM_HOME" => Gem.dir, "GEM_PATH" => (Gem.path + [path]).join(File::PATH_SEPARATOR) }
+    end
+
+    # Ruby run in a child process to print where Bundler installs the bundle
+    # of BUNDLE_GEMFILE, or nothing when that is the system gems.
+    BUNDLE_PATH_QUERY = <<~RUBY
+      require "bundler"
+      print Bundler.bundle_path unless Bundler.configured_bundle_path.use_system_gems?
+    RUBY
+    private_constant :BUNDLE_PATH_QUERY
+
+    # The directory the bundle of the given Gemfile is installed into when
+    # that is not the system gems, or nil. Bundler decides it from its
+    # settings - .bundle/config beside the Gemfile, the user's configuration
+    # and BUNDLE_* variables - without evaluating the Gemfile.
+    #
+    # Asked in a child process: loading Bundler here, before the dependency
+    # run, would put Bundler's files among the script's loaded features and
+    # pack them into every application with a Gemfile.
+    def bundle_install_path(gemfile)
+      env = { "BUNDLE_GEMFILE" => gemfile.to_s, "RUBYOPT" => nil, "BUNDLER_SETUP" => nil }
+      path = IO.popen(env, [RbConfig.ruby, "-e", BUNDLE_PATH_QUERY], err: File::NULL, &:read)
+      return nil unless $?.success? && !path.empty? && File.directory?(path)
+
+      path
+    rescue SystemCallError => e
+      verbose "Could not ask Bundler where the bundle of #{gemfile} is installed: #{e.message}"
+      nil
+    end
+
     # The Gemfile this process was started against, if any.
     def active_bundler_gemfile
       return ENV["BUNDLE_GEMFILE"] unless ENV["BUNDLE_GEMFILE"].to_s.empty?
@@ -153,6 +216,24 @@ module Ocran
     end
 
     def build
+      # Native compilation hands the source to an ahead-of-time compiler, and
+      # --wasm to a WebAssembly runtime; nothing was loaded, and none of the
+      # packaging below applies.
+      if @option.spinel? || @option.roundhouse? || @option.wasm_runtime
+        Dir.chdir(@pre_env.pwd)
+        if @option.wasm_runtime
+          require_relative "wasm_builder"
+          WasmBuilder.new(@option).build
+        elsif @option.roundhouse?
+          require_relative "roundhouse_builder"
+          RoundhouseBuilder.new(@option).build
+        else
+          require_relative "spinel_builder"
+          SpinelBuilder.new(@option).build
+        end
+        return
+      end
+
       # If the script was run and autoload is enabled, attempt to autoload libraries.
       if @option.force_autoload?
         attempt_load_autoload(@ignore_modules)
@@ -173,8 +254,10 @@ module Ocran
 
       if @option.use_inno_setup?
         # Native on Windows; on POSIX allow it when an ISCC command is
-        # available (e.g. via Wine wrapper scripts or in tests).
-        if Gem.win_platform? || system("command -v ISCC > /dev/null 2>&1")
+        # available (e.g. via Wine wrapper scripts or in tests). Checked
+        # here, before the files are collected, so a missing ISCC fails fast.
+        require_relative "inno_setup_script_builder"
+        if InnoSetupScriptBuilder.iscc_available?
           direction.build_inno_setup_installer
         else
           raise "Inno Setup is only supported on Windows (no ISCC command found in PATH)"
@@ -185,13 +268,15 @@ module Ocran
         direction.build_output_dir(@option.output_dir)
       elsif @option.output_zip
         direction.build_zip(@option.output_zip)
+      elsif @option.static_runtime?
+        direction.build_static_runtime_exe
       elsif @option.cosmo_zip?
         direction.build_cosmo_zip_exe
       else
         direction.build_stab_exe
       end
-    rescue RuntimeError => e
-      fatal_error e.message
+    rescue *USER_ERRORS => e
+      user_error e
     end
   end
 end

data/lib/ocran/runtime_environment.rb

@@ -15,18 +15,26 @@ module Ocran
     # under.
     BUNDLER_SETUP_FEATURE = %r{[\\/]bundler[\\/]setup\.rb\z}
 
-    attr_reader :env, :load_path, :loaded_features, :pwd, :activated_gems
+    attr_reader :env, :load_path, :loaded_features, :pwd, :loaded_specs
 
     def initialize
       @env = ENV.to_hash.freeze
       @load_path = $LOAD_PATH.dup.freeze
       @loaded_features = $LOADED_FEATURES.dup.freeze
       @pwd = Dir.pwd.freeze
-      # The gems RubyGems had activated at this point. Under `bundle exec`
-      # that is the entire bundle, activated before OCRAN's first line runs,
-      # so a snapshot taken before the dependency run is what tells the build
-      # environment's gems apart from the application's.
-      @activated_gems = (defined?(Gem) ? Gem.loaded_specs.keys : []).freeze
+      # The specs of the gems RubyGems had activated at this point. Taken
+      # with the loaded features for the same reason: OCRAN goes on to
+      # activate gems of its own while it builds (fiddle for DLL detection,
+      # tempfile for the builder, ...), and those are not the application's.
+      # Under `bundle exec` the snapshot taken before the dependency run is
+      # the entire bundle, activated before OCRAN's first line runs, which is
+      # what tells the build environment's gems apart from the application's.
+      @loaded_specs = (defined?(Gem) ? Gem.loaded_specs.values : []).freeze
+    end
+
+    # The names of the gems in #loaded_specs.
+    def activated_gems
+      @loaded_specs.map(&:name)
     end
 
     # Whether Bundler had already set this process up when the snapshot was