reflex-packager 0.2.0 → 0.3.0

data/lib/reflex/packager/data_loader.rb

@@ -0,0 +1,127 @@
+require_relative 'data_file'
+
+
+module Reflex
+
+
+  module Packager
+
+
+    # Lets require, require_relative and load read the Ruby scripts of an app
+    # from its data file, as the instruction sequences the packager compiled
+    # them into, named as the scripts but with .rbc, in place of the scripts,
+    # which the app directory has none of. The other files of the app are in
+    # it as they are.
+    #
+    # The scripts are compiled with their paths relative to the app directory,
+    # which a package runs in, so that __FILE__ and __dir__ have them as they
+    # are, and the others, as the libraries, are required as they are.
+    #
+    # A package carries this file in the reflex library of it, with DataFile
+    # beside, which is all it requires.
+    #
+    module DataLoader
+
+      DATA_FILE = 'data.bin'
+
+      class << self
+
+        # Reads the scripts of the app in +dir+ from the data file in it.
+        #
+        # @param [String] dir the app directory, which has data.bin
+        #
+        def setup(dir)
+          # as the current directory has it, for the paths to compare
+          @dir  = File.realpath dir
+          @data = DataFile.new File.join(@dir, DATA_FILE)
+
+          version = @data.read('.ruby-version') || '(unknown)'
+          raise "the data file is for Ruby #{version}, not #{RUBY_VERSION}" if
+            version != RUBY_VERSION
+
+          Kernel.prepend Require
+        end
+
+        attr_reader :dir, :data
+
+        # The name of the script in the data file for +feature+ to require, or
+        # nil.
+        #
+        def find(feature)
+          feature = feature.to_s
+          return script(File.expand_path feature) if
+            File.absolute_path?(feature) || feature.start_with?('./', '../')
+
+          $LOAD_PATH.each do |load_path|
+            name = script File.expand_path(feature, load_path.to_s)
+            return name if name
+          end
+          nil
+        end
+
+        # The name of the script in the data file at +path+, or nil.
+        #
+        def script(path)
+          path = File.expand_path path, @dir
+          return nil unless path.start_with? "#{@dir}/"
+          name = path[(@dir.size + 1)..].sub(/\.rb\z/, '') + '.rbc'
+          @data.include?(name) ? name : nil
+        end
+
+        # Requires the script +name+ in the data file unless it is already.
+        #
+        def require_script(name)
+          path = File.join @dir, name.sub(/\.rbc\z/, '.rb')
+          return false if $LOADED_FEATURES.include? path
+
+          # before running it, as require does, for a script it requires to
+          # require it back
+          $LOADED_FEATURES << path
+          begin
+            load_script name
+          rescue Exception
+            $LOADED_FEATURES.delete path
+            raise
+          end
+          true
+        end
+
+        def load_script(name)
+          RubyVM::InstructionSequence.load_from_binary(@data.read name).eval
+        end
+
+      end# self
+
+      module Require
+
+        private
+
+        def require(feature)
+          name = DataLoader.find feature
+          name ? DataLoader.require_script(name) : super
+        end
+
+        def require_relative(feature)
+          location = caller_locations(1, 1).first
+          base     = File.expand_path(location.absolute_path || location.path, DataLoader.dir)
+          path     = File.expand_path feature, File.dirname(base)
+          name     = DataLoader.script path
+          name ? DataLoader.require_script(name) : require(path)
+        end
+
+        def load(file, wrap = false)
+          name = DataLoader.script File.expand_path(file)
+          return super unless name
+          DataLoader.load_script name
+          true
+        end
+
+      end# Require
+
+    end# DataLoader
+
+
+  end# Packager
+
+
+end# Reflex

data/lib/reflex/packager/gems.rb

@@ -0,0 +1,282 @@
+require 'fileutils'
+require 'json'
+require 'open3'
+require 'rbconfig'
+require 'reflex/packager/library'
+
+
+module Reflex
+
+
+  module Packager
+
+
+    # Gems shipped with an app as they are: the ones of the Gemfile of the
+    # app, and the standard gems, which were default gems once. Included by a
+    # platform, which puts them where its load path finds them, and has the
+    # profile and the config they are looked for with.
+    #
+    module Gems
+
+      # Default gems that became bundled gems, as the NEWS of each Ruby says.
+      # Gem::BUNDLED_GEMS::SINCE, which Ruby 3.3 and later have, misses some
+      # of them; it adds the ones of Ruby versions newer than this list.
+      #
+      FORMER_DEFAULT_GEMS = [
+        *%w[rexml rss],                                              # 3.0
+        *%w[net-ftp net-imap net-pop net-smtp matrix prime debug],   # 3.1
+        *%w[racc],                                                   # 3.3
+        *%w[mutex_m getoptlong base64 bigdecimal observer abbrev
+            resolv-replace rinda drb nkf syslog csv],                # 3.4
+        *%w[ostruct pstore benchmark logger rdoc win32ole irb reline
+            readline fiddle win32-registry]                          # 4.0
+      ]
+
+      # Former default gems for development rather than for an app to run
+      # with, left out of the standard gems; with what they depend on, as
+      # prism and rbs for irb and rdoc, they would add over 10 MB. An app
+      # that runs with them lists them in its Gemfile.
+      #
+      DEVELOPMENT_GEMS = %w[irb rdoc debug]
+
+      # Extensions of the binaries a gem with a native extension has.
+      #
+      NATIVE_EXTS = %w[.so .bundle .dll]
+
+      # Names of the libraries and the gems, put on the load path in this
+      # order: a library before the ones it depends on, as bundler puts them,
+      # and the libraries before the gems, which the runtime is not to be
+      # hidden by.
+      #
+      def lib_names()
+        [*library_names.reverse, *gem_names]
+      end
+
+      # Names of the gems copy_gems copies, put on the load path in this order.
+      #
+      def gem_names()
+        [*('bundler' if gemfile), *gem_dirs.keys]
+      end
+
+      # Require paths of the gems by name: the ones of the Gemfile of the app,
+      # then the standard gems.
+      #
+      def gem_dirs()
+        @gem_dirs ||= standard_gem_dirs
+          .reject {|name, _| app_gem_dirs.key? name}
+          .then {app_gem_dirs.merge _1}
+      end
+
+      # Require paths of the gems in the default group of the Gemfile of the
+      # app, by name. A gem with a native extension, where it is not
+      # supported, is left out if the Ruby a platform embeds has it.
+      #
+      def app_gem_dirs()
+        @app_gem_dirs ||= gemfile_specs.each.with_object({}) do |spec, dirs|
+          name, paths = spec.values_at 'name', 'require_paths'
+          next unless shipped? spec
+          if !native_gems? && native_gem?(paths)
+            next if embedded_gem? name
+            raise Error, "gem '#{name}' has a native extension, which is not supported"
+          end
+          dirs[name] = paths
+        end
+      end
+
+      # Require paths of the standard gems installed, by name, leaving out the
+      # ones with a native extension where it is not supported.
+      #
+      def standard_gem_dirs()
+        return {} unless standard_gems?
+        @standard_gem_dirs ||= standard_specs.each.with_object({}) do |spec, dirs|
+          name, paths = spec.values_at 'name', 'require_paths'
+          next unless shipped? spec
+          next if !native_gems? && native_gem?(paths)
+          dirs[name] = paths
+        end
+      end
+
+      # The libraries the app runs on: the ones of the profile, and the ones in
+      # the Gemfile of the app, with what they depend on.
+      #
+      def libraries()
+        @libraries ||= Library.collect(*gemfile_library_roots, known: profile.libraries)
+          .then {profile.libraries + _1}
+      end
+
+      # Native extensions to register (Init_<name>).
+      #
+      def extensions()
+        libraries.filter_map(&:extension)
+      end
+
+      def library_names()
+        libraries.map(&:name)
+      end
+
+      private
+
+      # Whether to ship the standard gems, which the Ruby a platform embeds may
+      # have in its standard library already.
+      #
+      def standard_gems?()
+        true
+      end
+
+      # Whether the Ruby a platform embeds has the gem +name+ in its standard
+      # library, which the app runs with in place of the one of its Gemfile
+      # that cannot be shipped.
+      #
+      def embedded_gem?(name)
+        false
+      end
+
+      # Whether a gem with a native extension can be shipped.
+      #
+      def native_gems?()
+        false
+      end
+
+      def native_gem?(paths)
+        paths.any? do |path|
+          Dir.glob('**/*', base: path).any? {NATIVE_EXTS.include? File.extname(_1)}
+        end
+      end
+
+      # Leaves out bundler, default gems, which are in the standard library
+      # already, and the libraries, known by their extension.rb.
+      #
+      def shipped?(spec)
+        name, default_gem, paths = spec.values_at 'name', 'default_gem', 'require_paths'
+        return false if name == 'bundler' || default_gem
+        library_names.none? do |lib|
+          paths.any? {File.file? File.join(_1, lib, 'extension.rb')}
+        end
+      end
+
+      # Copies the gems as they are into the directories the block returns for
+      # their names, which the platform puts on the load path.
+      #
+      def copy_gems(&lib_dir)
+        gem_dirs.each do |name, paths|
+          dest = lib_dir.call name
+          FileUtils.mkdir_p dest
+          paths.each {FileUtils.cp_r File.join(_1, '.'), dest}
+        end
+        write_bundler_setup lib_dir.call('bundler') if gemfile
+      end
+
+      # An app with a Gemfile may require bundler/setup, which would find the
+      # bundler of the standard library, and no Gemfile to set up with. The
+      # gems are on the load path already, so the one found first does
+      # nothing.
+      #
+      def write_bundler_setup(dir)
+        path = File.join dir, 'bundler', 'setup.rb'
+        FileUtils.mkdir_p File.dirname(path)
+        File.write path, "# the gems of the Gemfile are on the load path already\n"
+      end
+
+      # Directories of the gems in the Gemfile which are libraries, known by
+      # their extension.rb, and not the ones of the profile; the ones RUBYLIB
+      # points to come first, as for the libraries of the profile.
+      #
+      def gemfile_library_roots()
+        names = profile.libraries.map(&:name)
+        gemfile_specs.filter_map do |spec|
+          name = Library.library_name spec['root']
+          next if !name || names.include?(name)
+          Library.source_root(spec['name']) || spec['root']
+        end
+      end
+
+      def gemfile()
+        path = File.join config.dir, 'Gemfile'
+        File.file?(path) ? path : nil
+      end
+
+      SPECS_TO_JSON = <<~RUBY
+        def specs_to_json(specs)
+          $stdout.write JSON.generate(specs.map {|s|
+            {
+              name:          s.name,
+              root:          s.full_gem_path,
+              default_gem:   s.default_gem?,
+              require_paths: s.full_require_paths
+            }
+          })
+        end
+      RUBY
+
+      GEMFILE_SPECS = <<~RUBY
+        require 'bundler'
+        require 'json'
+        #{SPECS_TO_JSON}
+        Bundler.ui = Bundler::UI::Silent.new
+        begin
+          specs_to_json Bundler.definition.specs_for([:default])
+        rescue Bundler::BundlerError => e
+          abort e.message
+        end
+      RUBY
+
+      STANDARD_SPECS = <<~RUBY
+        require 'json'
+        #{SPECS_TO_JSON}
+        names, excluded = JSON.parse ARGV[0]
+        begin
+          require 'bundled_gems'
+          names |= Gem::BUNDLED_GEMS::SINCE.keys
+        rescue LoadError
+        end
+        names -= excluded
+
+        # with what they depend on, which may be neither a default gem nor
+        # one of these
+        specs = {}
+        deps  = names.map {Gem::Dependency.new _1}
+        until deps.empty?
+          dep = deps.shift
+          next if specs.key? dep.name
+          specs[dep.name] = begin
+            dep.to_spec
+          rescue Gem::LoadError
+          end
+          deps.concat specs[dep.name].runtime_dependencies if specs[dep.name]
+        end
+        specs_to_json specs.values.compact
+      RUBY
+
+      # The gems of the default group of the Gemfile of the app.
+      #
+      def gemfile_specs()
+        return [] unless gemfile
+        run_ruby GEMFILE_SPECS, env: {'BUNDLE_GEMFILE' => gemfile},
+          error: 'failed to resolve the Gemfile (bundle install?)'
+      end
+
+      # The standard gems installed, and the gems they depend on.
+      #
+      def standard_specs()
+        run_ruby STANDARD_SPECS, JSON.generate([FORMER_DEFAULT_GEMS, DEVELOPMENT_GEMS]),
+          error: 'failed to look for the standard gems'
+      end
+
+      # Runs the script in a ruby of its own, out of the reach of any bundler
+      # this process runs under, which would hide the gems out of its bundle.
+      #
+      def run_ruby(script, *args, env: {}, error:)
+        env = ENV.keys.grep(/\ABUNDLER?_/).to_h {[_1, nil]}
+          .merge 'RUBYOPT' => nil, **env
+        out, err, status = Open3.capture3 env, RbConfig.ruby, '-e', script, *args
+        raise Error, "#{error}: #{err.strip}" unless status.success?
+        JSON.parse out
+      end
+
+    end# Gems
+
+
+  end# Packager
+
+
+end# Reflex

data/lib/reflex/packager/library.rb

@@ -0,0 +1,120 @@
+require 'rubygems'
+
+
+module Reflex
+
+
+  module Packager
+
+
+    # A library a packaged app runs on: a gem with lib/<name>/extension.rb, as
+    # xot, rays and reflex are, whose Ruby code a package bundles and whose
+    # extension, if it has lib/<name>/ext.rb, a package links.
+    #
+    class Library
+
+      # @param [String] name the directory of the library under lib/
+      # @param [String] root the directory of the gem
+      #
+      def initialize(name, root)
+        @name, @root = name, root
+      end
+
+      attr_reader :name, :root
+
+      # The extension to register (Init_<extension>), or nil.
+      #
+      def extension()
+        return nil unless File.file? File.join(root, 'lib', name, 'ext.rb')
+        "#{name.tr '-', '_'}_ext"
+      end
+
+      def to_s()
+        name
+      end
+
+      # Returns the libraries the gems at +roots+ depend on, as their gemspecs
+      # have them, and the gems themselves, each after the ones it depends on,
+      # leaving out the +known+ ones.
+      #
+      # A gem is looked for in the directories on the load path first, as
+      # RUBYLIB points to the ones of a repository being worked on, and then in
+      # the installed gems.
+      #
+      # @param [Array<String>]  roots the directories of the gems
+      # @param [Array<Library>] known the libraries collected already
+      #
+      # @return [Array<Library>] libraries
+      #
+      def self.collect(*roots, known: [])
+        libraries, visited, sources = [], known.to_h {[_1.root, true]}, source_roots
+        visit = -> (root, spec) do
+          name = library_name root
+          next if !name || visited[root]
+          visited[root] = true
+          spec.runtime_dependencies.each {|dep| find_gem(dep, sources)&.then {visit.call(*_1)}}
+          libraries << new(name, root)
+        end
+        roots.map {File.expand_path _1}.each {visit.call _1, spec_of(_1)}
+        libraries
+      end
+
+      # The directory of the gem +name+ on the load path, which is not an
+      # installed one, as RUBYLIB points to, or nil.
+      #
+      def self.source_root(name)
+        source_roots[name]
+      end
+
+      # The directory under lib/ with extension.rb, or nil.
+      #
+      def self.library_name(root)
+        path = Dir.glob(File.join root, 'lib', '*', 'extension.rb').first
+        path && File.basename(File.dirname path)
+      end
+
+      # Returns [root, spec] of the gem +dep+ asks for, or nil.
+      #
+      def self.find_gem(dep, sources)
+        root = sources[dep.name]
+        return [root, spec_of(root)] if root
+        spec = dep.to_spec
+        [spec.full_gem_path, spec]
+      rescue Gem::LoadError
+        nil
+      end
+
+      # The gems on the load path which are not installed ones, by name.
+      #
+      def self.source_roots()
+        $LOAD_PATH
+          .map {File.dirname File.expand_path(_1.to_s)}
+          .uniq
+          .reject {installed? _1}
+          .select {Dir.glob(File.join _1, '*.gemspec').any?}
+          .to_h {[spec_of(_1).name, _1]}
+      end
+
+      def self.spec_of(root)
+        installed = Gem::Specification.find {_1.full_gem_path == root}
+        return installed if installed
+        path = Dir.glob(File.join root, '*.gemspec').first or
+          raise Error, "no gemspec in '#{root}'"
+        # a gemspec may run git ls-files, which lists the files of the repository it is run in
+        Dir.chdir(root) {Gem::Specification.load path} or
+          raise Error, "failed to load '#{path}'"
+      end
+
+      def self.installed?(root)
+        Gem.path.any? {root.start_with? File.join(_1, 'gems', '')}
+      end
+
+      private_class_method :find_gem, :source_roots, :spec_of, :installed?
+
+    end# Library
+
+
+  end# Packager
+
+
+end# Reflex