reflex-packager 0.1.5 → 0.3.0

data/lib/reflex/packager/data_file.rb

@@ -0,0 +1,183 @@
+module Reflex
+
+
+  module Packager
+
+
+    # A file the files of an app are put together in, as app/data.bin, which
+    # a package reads them from as they are, not extracting them.
+    #
+    # It has a header, the bytes of the files one after another, and the
+    # index of them, each byte of which but the header's is substituted with
+    # another by a table from the seed in the header, which keeps them from
+    # being read as they are, though it is no encryption.
+    #
+    # A package carries this file to read its data file, so it requires
+    # nothing.
+    #
+    class DataFile
+
+      MAGIC   = 'RXDF'
+
+      VERSION = 1
+
+      # The magic, the version, the seed, and the offset and the size of the
+      # index.
+      #
+      HEADER = 'a4 L< L< Q< Q<'
+
+      HEADER_SIZE = 28
+
+      class << self
+
+        # Writes the files to +path+.
+        #
+        # @param [String]               path  data file path
+        # @param [Hash<String, String>] files {name => bytes}, a name as 'dir/file.png'
+        # @param [Integer]              seed  of the substitution of the bytes, a random one by default
+        #
+        def write(path, files, seed: rand(1..0xffffffff))
+          tr_substitutes seed # raises on an invalid one before writing
+
+          index, offset = [], HEADER_SIZE
+          files.each do |name, bytes|
+            raise ArgumentError, "invalid name: #{name.inspect}" if
+              name =~ /[\t\n]/ || name.start_with?('/') || name.split('/').include?('..')
+            index << "#{name}\t#{offset}\t#{bytes.bytesize}\n"
+            offset += bytes.bytesize
+          end
+          index = index.join.b
+
+          File.open path, 'wb' do |f|
+            f.write [MAGIC, VERSION, seed, offset, index.bytesize].pack HEADER
+            files.each_value {f.write encode(_1, seed)}
+            f.write encode(index, seed)
+          end
+        end
+
+        def open(path, &block)
+          file = new path
+          return file unless block
+          begin
+            block.call file
+          ensure
+            file.close
+          end
+        end
+
+        # +bytes+ with each of them substituted with another by the table from
+        # +seed+.
+        #
+        def encode(bytes, seed)
+          bytes.b.tap {_1.tr! TR_BYTES, tr_substitutes(seed)}
+        end
+
+        # +bytes+ encoded, with the substitution undone.
+        #
+        def decode(bytes, seed)
+          bytes.b.tap {_1.tr! tr_substitutes(seed), TR_BYTES}
+        end
+
+        private
+
+        def tr_substitutes(seed)
+          (@tr_substitutes ||= {})[seed] ||= tr_bytes substitutes(seed)
+        end
+
+        # The bytes in an order shuffled by xorshift from +seed+, which is the
+        # same on any Ruby, as Random may not be.
+        #
+        def substitutes(seed)
+          raise ArgumentError, "invalid seed: #{seed}" unless (1..0xffffffff).include? seed
+          x = seed
+          255.downto(1).with_object((0..255).to_a) {|i, bytes|
+            x ^= (x << 13) & 0xffffffff
+            x ^=  x >> 17
+            x ^= (x <<  5) & 0xffffffff
+            j  = x % (i + 1)
+            bytes[i], bytes[j] = bytes[j], bytes[i]
+          }.pack('C*')
+        end
+
+        # String#tr takes '\', '-' and '^' as escape, range and negation.
+        #
+        def tr_bytes(bytes)
+          bytes.gsub(/[\\\-^]/n) {"\\#{_1}"}
+        end
+
+      end# self
+
+      TR_BYTES = tr_bytes (0..255).map(&:chr).join.b
+
+      def initialize(path)
+        @path  = path
+        @file  = File.open path, 'rb'
+        @mutex = Mutex.new
+        @seed, @index = read_index
+      rescue
+        @file&.close
+        raise
+      end
+
+      attr_reader :path
+
+      def names()
+        @index.keys
+      end
+
+      def include?(name)
+        @index.key? name
+      end
+
+      # The size of the file in bytes, or nil.
+      #
+      def size(name)
+        @index[name]&.last
+      end
+
+      # The bytes of the file, or nil.
+      #
+      def read(name)
+        offset, size = @index[name]
+        return nil unless offset
+        DataFile.decode read_at(offset, size), @seed
+      end
+
+      def close()
+        @file.close
+      end
+
+      private
+
+      def read_at(offset, size)
+        @mutex.synchronize do
+          @file.seek offset
+          @file.read(size) || '' # nil at the end
+        end
+      end
+
+      def read_index()
+        header = read_at 0, HEADER_SIZE
+        raise "not a data file: #{@path}" unless header.bytesize == HEADER_SIZE
+
+        magic, version, seed, offset, size = header.unpack HEADER
+        raise "not a data file: #{@path}"              unless magic == MAGIC
+        raise "unknown data file version: #{version}" unless version == VERSION
+
+        index = DataFile.decode(read_at(offset, size), seed)
+          .force_encoding(Encoding::UTF_8)
+          .lines(chomp: true)
+          .to_h {
+            name, offset, size = _1.split "\t"
+            [name, [offset.to_i, size.to_i]]
+          }
+        return seed, index
+      end
+
+    end# DataFile
+
+
+  end# Packager
+
+
+end# Reflex

data/lib/reflex/packager/data_loader.rb

@@ -0,0 +1,127 @@
+require_relative 'data_file'
+
+
+module Reflex
+
+
+  module Packager
+
+
+    # Lets require, require_relative and load read the Ruby scripts of an app
+    # from its data file, as the instruction sequences the packager compiled
+    # them into, named as the scripts but with .rbc, in place of the scripts,
+    # which the app directory has none of. The other files of the app are in
+    # it as they are.
+    #
+    # The scripts are compiled with their paths relative to the app directory,
+    # which a package runs in, so that __FILE__ and __dir__ have them as they
+    # are, and the others, as the libraries, are required as they are.
+    #
+    # A package carries this file in the reflex library of it, with DataFile
+    # beside, which is all it requires.
+    #
+    module DataLoader
+
+      DATA_FILE = 'data.bin'
+
+      class << self
+
+        # Reads the scripts of the app in +dir+ from the data file in it.
+        #
+        # @param [String] dir the app directory, which has data.bin
+        #
+        def setup(dir)
+          # as the current directory has it, for the paths to compare
+          @dir  = File.realpath dir
+          @data = DataFile.new File.join(@dir, DATA_FILE)
+
+          version = @data.read('.ruby-version') || '(unknown)'
+          raise "the data file is for Ruby #{version}, not #{RUBY_VERSION}" if
+            version != RUBY_VERSION
+
+          Kernel.prepend Require
+        end
+
+        attr_reader :dir, :data
+
+        # The name of the script in the data file for +feature+ to require, or
+        # nil.
+        #
+        def find(feature)
+          feature = feature.to_s
+          return script(File.expand_path feature) if
+            File.absolute_path?(feature) || feature.start_with?('./', '../')
+
+          $LOAD_PATH.each do |load_path|
+            name = script File.expand_path(feature, load_path.to_s)
+            return name if name
+          end
+          nil
+        end
+
+        # The name of the script in the data file at +path+, or nil.
+        #
+        def script(path)
+          path = File.expand_path path, @dir
+          return nil unless path.start_with? "#{@dir}/"
+          name = path[(@dir.size + 1)..].sub(/\.rb\z/, '') + '.rbc'
+          @data.include?(name) ? name : nil
+        end
+
+        # Requires the script +name+ in the data file unless it is already.
+        #
+        def require_script(name)
+          path = File.join @dir, name.sub(/\.rbc\z/, '.rb')
+          return false if $LOADED_FEATURES.include? path
+
+          # before running it, as require does, for a script it requires to
+          # require it back
+          $LOADED_FEATURES << path
+          begin
+            load_script name
+          rescue Exception
+            $LOADED_FEATURES.delete path
+            raise
+          end
+          true
+        end
+
+        def load_script(name)
+          RubyVM::InstructionSequence.load_from_binary(@data.read name).eval
+        end
+
+      end# self
+
+      module Require
+
+        private
+
+        def require(feature)
+          name = DataLoader.find feature
+          name ? DataLoader.require_script(name) : super
+        end
+
+        def require_relative(feature)
+          location = caller_locations(1, 1).first
+          base     = File.expand_path(location.absolute_path || location.path, DataLoader.dir)
+          path     = File.expand_path feature, File.dirname(base)
+          name     = DataLoader.script path
+          name ? DataLoader.require_script(name) : require(path)
+        end
+
+        def load(file, wrap = false)
+          name = DataLoader.script File.expand_path(file)
+          return super unless name
+          DataLoader.load_script name
+          true
+        end
+
+      end# Require
+
+    end# DataLoader
+
+
+  end# Packager
+
+
+end# Reflex

data/lib/reflex/packager/gems.rb

@@ -0,0 +1,282 @@
+require 'fileutils'
+require 'json'
+require 'open3'
+require 'rbconfig'
+require 'reflex/packager/library'
+
+
+module Reflex
+
+
+  module Packager
+
+
+    # Gems shipped with an app as they are: the ones of the Gemfile of the
+    # app, and the standard gems, which were default gems once. Included by a
+    # platform, which puts them where its load path finds them, and has the
+    # profile and the config they are looked for with.
+    #
+    module Gems
+
+      # Default gems that became bundled gems, as the NEWS of each Ruby says.
+      # Gem::BUNDLED_GEMS::SINCE, which Ruby 3.3 and later have, misses some
+      # of them; it adds the ones of Ruby versions newer than this list.
+      #
+      FORMER_DEFAULT_GEMS = [
+        *%w[rexml rss],                                              # 3.0
+        *%w[net-ftp net-imap net-pop net-smtp matrix prime debug],   # 3.1
+        *%w[racc],                                                   # 3.3
+        *%w[mutex_m getoptlong base64 bigdecimal observer abbrev
+            resolv-replace rinda drb nkf syslog csv],                # 3.4
+        *%w[ostruct pstore benchmark logger rdoc win32ole irb reline
+            readline fiddle win32-registry]                          # 4.0
+      ]
+
+      # Former default gems for development rather than for an app to run
+      # with, left out of the standard gems; with what they depend on, as
+      # prism and rbs for irb and rdoc, they would add over 10 MB. An app
+      # that runs with them lists them in its Gemfile.
+      #
+      DEVELOPMENT_GEMS = %w[irb rdoc debug]
+
+      # Extensions of the binaries a gem with a native extension has.
+      #
+      NATIVE_EXTS = %w[.so .bundle .dll]
+
+      # Names of the libraries and the gems, put on the load path in this
+      # order: a library before the ones it depends on, as bundler puts them,
+      # and the libraries before the gems, which the runtime is not to be
+      # hidden by.
+      #
+      def lib_names()
+        [*library_names.reverse, *gem_names]
+      end
+
+      # Names of the gems copy_gems copies, put on the load path in this order.
+      #
+      def gem_names()
+        [*('bundler' if gemfile), *gem_dirs.keys]
+      end
+
+      # Require paths of the gems by name: the ones of the Gemfile of the app,
+      # then the standard gems.
+      #
+      def gem_dirs()
+        @gem_dirs ||= standard_gem_dirs
+          .reject {|name, _| app_gem_dirs.key? name}
+          .then {app_gem_dirs.merge _1}
+      end
+
+      # Require paths of the gems in the default group of the Gemfile of the
+      # app, by name. A gem with a native extension, where it is not
+      # supported, is left out if the Ruby a platform embeds has it.
+      #
+      def app_gem_dirs()
+        @app_gem_dirs ||= gemfile_specs.each.with_object({}) do |spec, dirs|
+          name, paths = spec.values_at 'name', 'require_paths'
+          next unless shipped? spec
+          if !native_gems? && native_gem?(paths)
+            next if embedded_gem? name
+            raise Error, "gem '#{name}' has a native extension, which is not supported"
+          end
+          dirs[name] = paths
+        end
+      end
+
+      # Require paths of the standard gems installed, by name, leaving out the
+      # ones with a native extension where it is not supported.
+      #
+      def standard_gem_dirs()
+        return {} unless standard_gems?
+        @standard_gem_dirs ||= standard_specs.each.with_object({}) do |spec, dirs|
+          name, paths = spec.values_at 'name', 'require_paths'
+          next unless shipped? spec
+          next if !native_gems? && native_gem?(paths)
+          dirs[name] = paths
+        end
+      end
+
+      # The libraries the app runs on: the ones of the profile, and the ones in
+      # the Gemfile of the app, with what they depend on.
+      #
+      def libraries()
+        @libraries ||= Library.collect(*gemfile_library_roots, known: profile.libraries)
+          .then {profile.libraries + _1}
+      end
+
+      # Native extensions to register (Init_<name>).
+      #
+      def extensions()
+        libraries.filter_map(&:extension)
+      end
+
+      def library_names()
+        libraries.map(&:name)
+      end
+
+      private
+
+      # Whether to ship the standard gems, which the Ruby a platform embeds may
+      # have in its standard library already.
+      #
+      def standard_gems?()
+        true
+      end
+
+      # Whether the Ruby a platform embeds has the gem +name+ in its standard
+      # library, which the app runs with in place of the one of its Gemfile
+      # that cannot be shipped.
+      #
+      def embedded_gem?(name)
+        false
+      end
+
+      # Whether a gem with a native extension can be shipped.
+      #
+      def native_gems?()
+        false
+      end
+
+      def native_gem?(paths)
+        paths.any? do |path|
+          Dir.glob('**/*', base: path).any? {NATIVE_EXTS.include? File.extname(_1)}
+        end
+      end
+
+      # Leaves out bundler, default gems, which are in the standard library
+      # already, and the libraries, known by their extension.rb.
+      #
+      def shipped?(spec)
+        name, default_gem, paths = spec.values_at 'name', 'default_gem', 'require_paths'
+        return false if name == 'bundler' || default_gem
+        library_names.none? do |lib|
+          paths.any? {File.file? File.join(_1, lib, 'extension.rb')}
+        end
+      end
+
+      # Copies the gems as they are into the directories the block returns for
+      # their names, which the platform puts on the load path.
+      #
+      def copy_gems(&lib_dir)
+        gem_dirs.each do |name, paths|
+          dest = lib_dir.call name
+          FileUtils.mkdir_p dest
+          paths.each {FileUtils.cp_r File.join(_1, '.'), dest}
+        end
+        write_bundler_setup lib_dir.call('bundler') if gemfile
+      end
+
+      # An app with a Gemfile may require bundler/setup, which would find the
+      # bundler of the standard library, and no Gemfile to set up with. The
+      # gems are on the load path already, so the one found first does
+      # nothing.
+      #
+      def write_bundler_setup(dir)
+        path = File.join dir, 'bundler', 'setup.rb'
+        FileUtils.mkdir_p File.dirname(path)
+        File.write path, "# the gems of the Gemfile are on the load path already\n"
+      end
+
+      # Directories of the gems in the Gemfile which are libraries, known by
+      # their extension.rb, and not the ones of the profile; the ones RUBYLIB
+      # points to come first, as for the libraries of the profile.
+      #
+      def gemfile_library_roots()
+        names = profile.libraries.map(&:name)
+        gemfile_specs.filter_map do |spec|
+          name = Library.library_name spec['root']
+          next if !name || names.include?(name)
+          Library.source_root(spec['name']) || spec['root']
+        end
+      end
+
+      def gemfile()
+        path = File.join config.dir, 'Gemfile'
+        File.file?(path) ? path : nil
+      end
+
+      SPECS_TO_JSON = <<~RUBY
+        def specs_to_json(specs)
+          $stdout.write JSON.generate(specs.map {|s|
+            {
+              name:          s.name,
+              root:          s.full_gem_path,
+              default_gem:   s.default_gem?,
+              require_paths: s.full_require_paths
+            }
+          })
+        end
+      RUBY
+
+      GEMFILE_SPECS = <<~RUBY
+        require 'bundler'
+        require 'json'
+        #{SPECS_TO_JSON}
+        Bundler.ui = Bundler::UI::Silent.new
+        begin
+          specs_to_json Bundler.definition.specs_for([:default])
+        rescue Bundler::BundlerError => e
+          abort e.message
+        end
+      RUBY
+
+      STANDARD_SPECS = <<~RUBY
+        require 'json'
+        #{SPECS_TO_JSON}
+        names, excluded = JSON.parse ARGV[0]
+        begin
+          require 'bundled_gems'
+          names |= Gem::BUNDLED_GEMS::SINCE.keys
+        rescue LoadError
+        end
+        names -= excluded
+
+        # with what they depend on, which may be neither a default gem nor
+        # one of these
+        specs = {}
+        deps  = names.map {Gem::Dependency.new _1}
+        until deps.empty?
+          dep = deps.shift
+          next if specs.key? dep.name
+          specs[dep.name] = begin
+            dep.to_spec
+          rescue Gem::LoadError
+          end
+          deps.concat specs[dep.name].runtime_dependencies if specs[dep.name]
+        end
+        specs_to_json specs.values.compact
+      RUBY
+
+      # The gems of the default group of the Gemfile of the app.
+      #
+      def gemfile_specs()
+        return [] unless gemfile
+        run_ruby GEMFILE_SPECS, env: {'BUNDLE_GEMFILE' => gemfile},
+          error: 'failed to resolve the Gemfile (bundle install?)'
+      end
+
+      # The standard gems installed, and the gems they depend on.
+      #
+      def standard_specs()
+        run_ruby STANDARD_SPECS, JSON.generate([FORMER_DEFAULT_GEMS, DEVELOPMENT_GEMS]),
+          error: 'failed to look for the standard gems'
+      end
+
+      # Runs the script in a ruby of its own, out of the reach of any bundler
+      # this process runs under, which would hide the gems out of its bundle.
+      #
+      def run_ruby(script, *args, env: {}, error:)
+        env = ENV.keys.grep(/\ABUNDLER?_/).to_h {[_1, nil]}
+          .merge 'RUBYOPT' => nil, **env
+        out, err, status = Open3.capture3 env, RbConfig.ruby, '-e', script, *args
+        raise Error, "#{error}: #{err.strip}" unless status.success?
+        JSON.parse out
+      end
+
+    end# Gems
+
+
+  end# Packager
+
+
+end# Reflex

data/lib/reflex/packager/library.rb

@@ -0,0 +1,120 @@
+require 'rubygems'
+
+
+module Reflex
+
+
+  module Packager
+
+
+    # A library a packaged app runs on: a gem with lib/<name>/extension.rb, as
+    # xot, rays and reflex are, whose Ruby code a package bundles and whose
+    # extension, if it has lib/<name>/ext.rb, a package links.
+    #
+    class Library
+
+      # @param [String] name the directory of the library under lib/
+      # @param [String] root the directory of the gem
+      #
+      def initialize(name, root)
+        @name, @root = name, root
+      end
+
+      attr_reader :name, :root
+
+      # The extension to register (Init_<extension>), or nil.
+      #
+      def extension()
+        return nil unless File.file? File.join(root, 'lib', name, 'ext.rb')
+        "#{name.tr '-', '_'}_ext"
+      end
+
+      def to_s()
+        name
+      end
+
+      # Returns the libraries the gems at +roots+ depend on, as their gemspecs
+      # have them, and the gems themselves, each after the ones it depends on,
+      # leaving out the +known+ ones.
+      #
+      # A gem is looked for in the directories on the load path first, as
+      # RUBYLIB points to the ones of a repository being worked on, and then in
+      # the installed gems.
+      #
+      # @param [Array<String>]  roots the directories of the gems
+      # @param [Array<Library>] known the libraries collected already
+      #
+      # @return [Array<Library>] libraries
+      #
+      def self.collect(*roots, known: [])
+        libraries, visited, sources = [], known.to_h {[_1.root, true]}, source_roots
+        visit = -> (root, spec) do
+          name = library_name root
+          next if !name || visited[root]
+          visited[root] = true
+          spec.runtime_dependencies.each {|dep| find_gem(dep, sources)&.then {visit.call(*_1)}}
+          libraries << new(name, root)
+        end
+        roots.map {File.expand_path _1}.each {visit.call _1, spec_of(_1)}
+        libraries
+      end
+
+      # The directory of the gem +name+ on the load path, which is not an
+      # installed one, as RUBYLIB points to, or nil.
+      #
+      def self.source_root(name)
+        source_roots[name]
+      end
+
+      # The directory under lib/ with extension.rb, or nil.
+      #
+      def self.library_name(root)
+        path = Dir.glob(File.join root, 'lib', '*', 'extension.rb').first
+        path && File.basename(File.dirname path)
+      end
+
+      # Returns [root, spec] of the gem +dep+ asks for, or nil.
+      #
+      def self.find_gem(dep, sources)
+        root = sources[dep.name]
+        return [root, spec_of(root)] if root
+        spec = dep.to_spec
+        [spec.full_gem_path, spec]
+      rescue Gem::LoadError
+        nil
+      end
+
+      # The gems on the load path which are not installed ones, by name.
+      #
+      def self.source_roots()
+        $LOAD_PATH
+          .map {File.dirname File.expand_path(_1.to_s)}
+          .uniq
+          .reject {installed? _1}
+          .select {Dir.glob(File.join _1, '*.gemspec').any?}
+          .to_h {[spec_of(_1).name, _1]}
+      end
+
+      def self.spec_of(root)
+        installed = Gem::Specification.find {_1.full_gem_path == root}
+        return installed if installed
+        path = Dir.glob(File.join root, '*.gemspec').first or
+          raise Error, "no gemspec in '#{root}'"
+        # a gemspec may run git ls-files, which lists the files of the repository it is run in
+        Dir.chdir(root) {Gem::Specification.load path} or
+          raise Error, "failed to load '#{path}'"
+      end
+
+      def self.installed?(root)
+        Gem.path.any? {root.start_with? File.join(_1, 'gems', '')}
+      end
+
+      private_class_method :find_gem, :source_roots, :spec_of, :installed?
+
+    end# Library
+
+
+  end# Packager
+
+
+end# Reflex