reflex-packager 0.2.0 → 0.3.0

data/lib/reflex/packager/cli.rb

@@ -14,6 +14,16 @@ module Reflex
     #
     class CLI
 
+      # The options to notarize the app by, and the steps of the notarization
+      # they take.
+      #
+      NOTARIZE_OPTIONS = {
+        notarize:          %i[submit export],
+        'notarize-submit': %i[submit],
+        'notarize-wait':   %i[wait],
+        'notarize-export': %i[export]
+      }
+
       def initialize(profile)
         @profile = profile
       end
@@ -61,20 +71,43 @@ module Reflex
       end
 
       def package(argv)
-        profile      = @profile
-        argv, params = parse argv, "Usage: #{profile.command} package [options] [DIR]" do
-          on '--platform PLATFORM', 'target platform (default: macos)'
-          on '--config PATH',       "config file path (default: DIR/#{profile.config_files.first})"
-          on '--generate-only',     'generate project files but do not build'
-          on '--verbose',           'verbose output'
+        profile, platform = @profile, Packager.default_platform
+        argv, params      = parse argv, "Usage: #{profile.command} package [options] [DIR]" do
+          on '--platform PLATFORM',
+            "target platform: #{PLATFORMS.keys.join ' or '} (default: #{platform})"
+          on '--config PATH',      "config file path (default: DIR/#{profile.config_files.first})"
+          on '--generate-only',    'generate project files but do not build'
+          on '--pack',             'put the files of the app together in a data file, for a release'
+          on '--generic',          'package the app signed with Developer ID, also with --appstore'
+          on '--appstore',         'package the app for the Mac App Store, only it unless --generic'
+          on '--notarize',         'have Apple check the app with Developer ID, on macos'
+          on '--notarize-submit',  'only submit the app to be notarized, not waiting'
+          on '--notarize-wait ID', 'only wait for the notarization ID, by the API key'
+          on '--notarize-export',  'only wait for the notarization submitted and export the app'
+          on '--verbose',          'verbose output'
         end
 
+        notarize = NOTARIZE_OPTIONS.filter_map {|key, value| value if params[key]}
+        raise Error, "only one of #{NOTARIZE_OPTIONS.keys.map {"--#{_1}"}.join ', '}" if
+          notarize.size > 1
+
+        id = params[:'notarize-wait']
+        raise Error, '--notarize-wait packages nothing, not with --appstore' if
+          id && params[:appstore]
+        return AppStoreConnect.new.wait_until_notarized id if id
+
         dir      = argv.shift || '.'
-        platform = (params[:platform] || 'macos').to_sym
+        platform = (params[:platform] || platform).to_sym
         klass    = PLATFORMS[platform] || raise(Error, "unknown platform: '#{platform}'")
         config   = Config.load profile, dir, params[:config]
-        klass.new(config, verbose: params[:verbose])
-          .package generate_only: params[:'generate-only']
+        klass.new(
+          config,
+          verbose:  params[:verbose],
+          pack:     params[:pack],
+          notarize: notarize.first,
+          generic:  params[:generic],
+          appstore: params[:appstore]
+        ).package generate_only: params[:'generate-only']
       end
 
       private

data/lib/reflex/packager/config.rb

@@ -1,3 +1,4 @@
+require 'pathname'
 require 'yaml'
 
 
@@ -20,15 +21,15 @@ module Reflex
         {
           name:      name,
           bundle_id: hash[:bundle_id] || default_bundle_id(profile, name),
-          version:   '0.1.0',
+          version:   (hash[:version].nil? && version_file(dir)) || '0.1.0',
+          copyright: nil,
           main:      profile.main,
           icon:      nil,
           files:     nil,
-          pods: {
-            :cruby          => {tag: nil, branch: nil, git: nil, path: nil},
-            profile.pod_key => {tag: nil, branch: nil, git: nil, path: nil}
-          },
-          macos: MacOSConfig.defaults
+          excludes:  nil,
+          macos:   MacOSConfig.defaults,
+          windows: WindowsConfig.defaults,
+          upload:  UploadConfig.defaults
         }
       end
 
@@ -51,40 +52,68 @@ module Reflex
       def initialize(profile, dir, hash = nil, stderr: $stderr)
         raise Error, "no such directory: '#{dir}'" unless File.directory? dir
 
-        @profile = profile
-        @dir     = File.expand_path dir
-        hash = symbolize_keys hash || {}
-        hash = validate_input hash, Config.defaults(profile, @dir, hash), stderr: stderr
-
-        @name      = hash[:name]     .to_s
-        @bundle_id = hash[:bundle_id].to_s
-        @version   = hash[:version]  .to_s
-        @main      = hash[:main]     .to_s
-        @icon      = hash[:icon]    &.to_s
+        @profile      = profile
+        @dir          = File.expand_path dir
+        hash          = symbolize_keys hash || {}
+        localizations = hash.delete :localizations
+        hash          = validate_input hash, Config.defaults(profile, @dir, hash), stderr: stderr
+
+        # YAML reads 1.10 as a number, the same as 1.1
+        raise Error, "version #{hash[:version]} is read as a number, quote it" if
+          hash[:version].is_a? Float
+
+        @name      = hash[:name]      .to_s
+        @bundle_id = hash[:bundle_id] .to_s
+        @version   = hash[:version]   .to_s
+        @copyright = hash[:copyright]&.to_s
+        @main      = hash[:main]      .to_s
+        @icon      = hash[:icon]     &.to_s
         @files     = hash[:files]&.then {Array(_1).map(&:to_s)}
-        @pods      = hash[:pods].transform_values &:compact
-        @macos     = MacOSConfig.new hash[:macos]
+        @excludes  = glob_patterns hash[:excludes], :excludes
+        @macos     = MacOSConfig.new   hash[:macos]
+        @windows   = WindowsConfig.new hash[:windows]
+        @upload    = UploadConfig.new  hash[:upload]
+
+        @localizations = localize localizations, stderr: stderr
         validate
       end
 
-      attr_reader :profile, :dir, :name, :bundle_id, :version, :main, :icon,
-        :files, :macos, :pods
+      attr_reader :profile, :dir, :name, :bundle_id, :version, :copyright,
+        :main, :icon, :files, :excludes, :macos, :windows, :upload
+
+      # The values that differ by language, in English, the language of the
+      # ones out of localizations, and in each language of the localizations
+      # with the English ones for what they leave out.
+      #
+      # @return [Hash<String, Hash>] {language => {name:, copyright:}}
+      #
+      attr_reader :localizations
+
+      # The version to show: the numbers of the version, up to three.
+      #
+      def display_version()
+        @version.split('.').first(3).join('.')
+      end
+
+      # The version to tell builds apart with: the version, a fourth number
+      # of which tells a build apart from another of the same version.
+      #
+      def build_version()
+        @version
+      end
 
       # Returns paths to be bundled into the application, relative to the
-      # project directory.
+      # project directory: the files only, with the ones in the directories
+      # matched, for excludes to take out a file in them.
       #
       # @return [Array<String>] relative paths
       #
       def app_files()
-        excludes = EXCLUDES + @profile.config_files
-        [@main, *@files]
-          .flat_map {Dir.glob _1, base: @dir}
-          .reject {_1.start_with?('.') || excludes.include?(_1)}
-          .uniq
-          .sort
+        excluded = glob_files [*EXCLUDES, *@profile.config_files, *@excludes]
+        (glob_files([@main, *@files]) - excluded).reject {_1.start_with? '.'}.sort
       end
 
-      EXCLUDES = %w[build dist]
+      EXCLUDES = %w[dist]
 
       private
 
@@ -101,6 +130,19 @@ module Reflex
         "#{profile.bundle_id_prefix}.#{id}"
       end
 
+      # The version in the VERSION file of the app, which the config may leave
+      # it to, as the gems have theirs in one.
+      #
+      def self.version_file(dir)
+        path = File.join dir, 'VERSION'
+        return nil unless File.file? path
+        # without the BOM an editor on Windows may write
+        version = File.read(path, encoding: 'bom|utf-8').strip
+        raise Error, "invalid version in the VERSION file: '#{version}'" if
+          version !~ VERSION_FORMAT
+        version
+      end
+
       def symbolize_keys(hash)
         hash.map {|k, v|
           [
@@ -110,6 +152,26 @@ module Reflex
         }.to_h
       end
 
+      # A language tag, as ja or zh-Hans.
+      #
+      LANGUAGE_FORMAT = /\A[a-z]{2,3}(-[A-Za-z0-9]{2,8})*\z/
+
+      def localize(localizations, stderr: nil)
+        english = {name: @name, copyright: @copyright}
+        raise Error, "not a Hash: '/localizations'" unless
+          localizations.nil? || localizations.is_a?(Hash)
+
+        (localizations || {}).each.with_object({'en' => english}) do |(lang, hash), result|
+          lang = lang.to_s
+          raise Error, "invalid language: '#{lang}'" if lang !~ LANGUAGE_FORMAT
+          raise Error, "'en' is the language of the values out of localizations" if
+            lang == 'en'
+          result[lang] = validate_input(
+            hash, english, parent: "/localizations/#{lang}", stderr: stderr
+          ).transform_values {_1&.to_s}
+        end
+      end
+
       def validate_input(hash, defaults, parent: '', stderr: nil)
         return defaults.dup if hash.nil?
         raise Error, "not a Hash: '#{parent}'" unless hash.is_a? Hash
@@ -127,16 +189,50 @@ module Reflex
         end
       end
 
+      def glob_patterns(patterns, key)
+        return nil if patterns.nil?
+        Array(patterns).map do |pattern|
+          raise Error, "not a glob pattern in '/#{key}': #{pattern.inspect}" if
+            pattern.is_a?(Hash) || pattern.is_a?(Array) || pattern.to_s.strip.empty?
+          pattern.to_s
+        end
+      end
+
+      # The files matched by +patterns+ in the project directory, and the
+      # ones in the directories matched but the dotfiles, as lib/a.rb for
+      # ./lib/a.rb too, for the excludes to be compared with.
+      #
+      def glob_files(patterns)
+        patterns
+          .flat_map {Dir.glob _1, base: @dir}
+          .map {Pathname(_1).cleanpath.to_s}
+          .flat_map {|path|
+            # by base:, not to take the name of the directory as a pattern
+            [path, *Dir.glob('**/*', base: File.join(@dir, path)).map {File.join path, _1}]
+          }
+          .reject {File.directory? File.join(@dir, _1)}
+          .uniq
+      end
+
+      # Numbers only, which macOS and Windows compare versions by.
+      #
+      VERSION_FORMAT = /\A\d+(\.\d+)*\z/
+
       def validate()
+        # the name of the .app on macos, which can not have a path separator
+        raise Error, "invalid name: '#{@name}'" if @name.strip.empty? || @name =~ %r{[/:]}
+
         raise Error, "invalid bundle_id: '#{@bundle_id}'" if
           @bundle_id !~ /\A[A-Za-z0-9\-]+(\.[A-Za-z0-9\-]+)+\z/
 
-        raise Error, "invalid version: '#{@version}'" if
-          @version !~ /\A\d+(\.\d+)*\z/
+        raise Error, "invalid version: '#{@version}'" if @version !~ VERSION_FORMAT
 
         raise Error, "main script not found: '#{@main}'" if
           !File.file?(File.join @dir, @main)
 
+        raise Error, "main script excluded: '#{@main}'" if
+          @excludes && (glob_files([@main]) - glob_files(@excludes)).empty?
+
         raise Error, "icon not found: '#{@icon}'" if
           @icon && !File.file?(File.join @dir, @icon)
       end
@@ -148,10 +244,20 @@ module Reflex
     #
     class MacOSConfig
 
+      # The keys of codesign in the config, and the environment variables
+      # that override them.
+      #
+      CODESIGN_ENV_KEYS = {
+        identity: 'APPLE_CODESIGN_IDENTITY',
+        team_id:  'APPLE_TEAM_ID'
+      }
+
       def self.defaults()
         {
           deployment_target: '11.0',
-          archs:             'arm64',
+          archs:             %w[arm64 x86_64],
+          cruby:             nil,
+          category:          nil,
           codesign: {
             identity: '-',
             team_id:  nil
@@ -160,22 +266,113 @@ module Reflex
       end
 
       def initialize(hash)
-        @deployment_target = hash[:deployment_target]  .to_s
-        @archs             = Array(hash[:archs]).map  &:to_s
-        @codesign_identity = hash[:codesign][:identity].to_s
-        @codesign_team_id  = hash[:codesign][:team_id]&.to_s
+        codesign = hash[:codesign].merge CODESIGN_ENV_KEYS.transform_values {env _1}.compact
+
+        @deployment_target = hash[:deployment_target] .to_s
+        @archs             = Array(hash[:archs]).map &:to_s
+        @cruby             = hash[:cruby]            &.to_s
+        @category          = hash[:category]         &.to_s
+        @codesign_identity = codesign[:identity]      .to_s
+        @codesign_team_id  = codesign[:team_id]      &.to_s
         validate
       end
 
-      attr_reader :deployment_target, :archs, :codesign_identity, :codesign_team_id
+      # The version of CRuby, or the path of a checkout of the cruby
+      # repository, to embed.
+      #
+      attr_reader :cruby
+
+      attr_reader :deployment_target, :archs
+
+      # The category of the app in its Info.plist, as public.app-category.games,
+      # which Finder groups apps by and the App Store requires.
+      #
+      attr_reader :category
+
+      # The identity and the team to sign the app with, which
+      # APPLE_CODESIGN_IDENTITY and APPLE_TEAM_ID override, for the name of
+      # the developer in the identity not to be in the config.
+      #
+      attr_reader :codesign_identity, :codesign_team_id
 
       def validate()
         raise Error, "invalid archs: '#{@archs}'" if @archs.empty?
+        raise Error, "invalid category: '#{@category}'" if
+          @category && @category !~ /\Apublic\.app-category\.[a-z\-]+\z/
+      end
+
+      private
+
+      # The environment variable +name+, or nil if empty, as a secret of
+      # GitHub Actions not set is.
+      #
+      def env(name)
+        ENV[name].then {_1 unless _1.to_s.empty?}
       end
 
     end# MacOSConfig
 
 
+    # Windows specific configuration.
+    #
+    class WindowsConfig
+
+      def self.defaults()
+        {
+          console: false
+        }
+      end
+
+      def initialize(hash)
+        @console = hash[:console]
+        validate
+      end
+
+      # Whether the executable keeps a console window, which shows what an app
+      # prints and the errors it dies of while it is being worked on.
+      #
+      def console?()
+        @console
+      end
+
+      def validate()
+        raise Error, "invalid console: '#{@console}'" unless
+          [true, false].include? @console
+      end
+
+    end# WindowsConfig
+
+
+    # Configuration of the stores to upload the app to.
+    #
+    class UploadConfig
+
+      def self.defaults()
+        {
+          itchio: {
+            project: nil
+          }
+        }
+      end
+
+      def initialize(hash)
+        @itchio_project = hash[:itchio][:project]&.to_s
+        validate
+      end
+
+      # The project of itch.io to upload the app to, as user/game of
+      # https://user.itch.io/game, which butler takes.
+      #
+      attr_reader :itchio_project
+
+      def validate()
+        raise Error, "invalid upload.itchio.project: '#{@itchio_project}'" if
+          @itchio_project && @itchio_project !~ %r{\A[a-z0-9_\-]+/[a-z0-9_\-]+\z}
+      end
+
+    end# UploadConfig
+
+
     # Raised on invalid configuration or packaging failure.
     #
     class Error < StandardError; end

data/lib/reflex/packager/data_compiler.rb

@@ -0,0 +1,59 @@
+require_relative 'data_file'
+
+
+module Reflex
+
+
+  module Packager
+
+
+    # Compiles the Ruby scripts of an app into instruction sequences for its
+    # data file, which DataLoader reads them from, named as the scripts but
+    # with .rbc.
+    #
+    # They are compiled by the Ruby running this, which has to be the one a
+    # package runs them on, with their paths relative to the app directory,
+    # as the one the package runs in has them.
+    #
+    # A package carries none of this, which requires nothing but DataFile,
+    # for an app to compile its scripts with the Ruby in it.
+    #
+    module DataCompiler
+
+      class << self
+
+        # The names of the Ruby scripts of the app in +dir+.
+        #
+        def scripts(dir)
+          Dir.glob('**/*.rb', base: dir).select {File.file? File.join(dir, _1)}.sort
+        end
+
+        # Compiles the Ruby scripts of the app in +dir+ into the files of its
+        # data file, with the version of the Ruby.
+        #
+        def compile(dir)
+          data = scripts(dir).to_h do |name|
+            source = File.read File.join(dir, name), encoding: Encoding::UTF_8
+            iseq   = RubyVM::InstructionSequence.compile source, name, name
+            [name.sub(/\.rb\z/, '.rbc'), iseq.to_binary]
+          end
+          data['.ruby-version'] = RUBY_VERSION
+          data
+        end
+
+        # Writes the data file at +path+ with the Ruby scripts of the app in
+        # +dir+ compiled, all of them before writing it.
+        #
+        def write(dir, path)
+          DataFile.write path, compile(dir)
+        end
+
+      end# self
+
+    end# DataCompiler
+
+
+  end# Packager
+
+
+end# Reflex

data/lib/reflex/packager/data_file.rb

@@ -0,0 +1,183 @@
+module Reflex
+
+
+  module Packager
+
+
+    # A file the files of an app are put together in, as app/data.bin, which
+    # a package reads them from as they are, not extracting them.
+    #
+    # It has a header, the bytes of the files one after another, and the
+    # index of them, each byte of which but the header's is substituted with
+    # another by a table from the seed in the header, which keeps them from
+    # being read as they are, though it is no encryption.
+    #
+    # A package carries this file to read its data file, so it requires
+    # nothing.
+    #
+    class DataFile
+
+      MAGIC   = 'RXDF'
+
+      VERSION = 1
+
+      # The magic, the version, the seed, and the offset and the size of the
+      # index.
+      #
+      HEADER = 'a4 L< L< Q< Q<'
+
+      HEADER_SIZE = 28
+
+      class << self
+
+        # Writes the files to +path+.
+        #
+        # @param [String]               path  data file path
+        # @param [Hash<String, String>] files {name => bytes}, a name as 'dir/file.png'
+        # @param [Integer]              seed  of the substitution of the bytes, a random one by default
+        #
+        def write(path, files, seed: rand(1..0xffffffff))
+          tr_substitutes seed # raises on an invalid one before writing
+
+          index, offset = [], HEADER_SIZE
+          files.each do |name, bytes|
+            raise ArgumentError, "invalid name: #{name.inspect}" if
+              name =~ /[\t\n]/ || name.start_with?('/') || name.split('/').include?('..')
+            index << "#{name}\t#{offset}\t#{bytes.bytesize}\n"
+            offset += bytes.bytesize
+          end
+          index = index.join.b
+
+          File.open path, 'wb' do |f|
+            f.write [MAGIC, VERSION, seed, offset, index.bytesize].pack HEADER
+            files.each_value {f.write encode(_1, seed)}
+            f.write encode(index, seed)
+          end
+        end
+
+        def open(path, &block)
+          file = new path
+          return file unless block
+          begin
+            block.call file
+          ensure
+            file.close
+          end
+        end
+
+        # +bytes+ with each of them substituted with another by the table from
+        # +seed+.
+        #
+        def encode(bytes, seed)
+          bytes.b.tap {_1.tr! TR_BYTES, tr_substitutes(seed)}
+        end
+
+        # +bytes+ encoded, with the substitution undone.
+        #
+        def decode(bytes, seed)
+          bytes.b.tap {_1.tr! tr_substitutes(seed), TR_BYTES}
+        end
+
+        private
+
+        def tr_substitutes(seed)
+          (@tr_substitutes ||= {})[seed] ||= tr_bytes substitutes(seed)
+        end
+
+        # The bytes in an order shuffled by xorshift from +seed+, which is the
+        # same on any Ruby, as Random may not be.
+        #
+        def substitutes(seed)
+          raise ArgumentError, "invalid seed: #{seed}" unless (1..0xffffffff).include? seed
+          x = seed
+          255.downto(1).with_object((0..255).to_a) {|i, bytes|
+            x ^= (x << 13) & 0xffffffff
+            x ^=  x >> 17
+            x ^= (x <<  5) & 0xffffffff
+            j  = x % (i + 1)
+            bytes[i], bytes[j] = bytes[j], bytes[i]
+          }.pack('C*')
+        end
+
+        # String#tr takes '\', '-' and '^' as escape, range and negation.
+        #
+        def tr_bytes(bytes)
+          bytes.gsub(/[\\\-^]/n) {"\\#{_1}"}
+        end
+
+      end# self
+
+      TR_BYTES = tr_bytes (0..255).map(&:chr).join.b
+
+      def initialize(path)
+        @path  = path
+        @file  = File.open path, 'rb'
+        @mutex = Mutex.new
+        @seed, @index = read_index
+      rescue
+        @file&.close
+        raise
+      end
+
+      attr_reader :path
+
+      def names()
+        @index.keys
+      end
+
+      def include?(name)
+        @index.key? name
+      end
+
+      # The size of the file in bytes, or nil.
+      #
+      def size(name)
+        @index[name]&.last
+      end
+
+      # The bytes of the file, or nil.
+      #
+      def read(name)
+        offset, size = @index[name]
+        return nil unless offset
+        DataFile.decode read_at(offset, size), @seed
+      end
+
+      def close()
+        @file.close
+      end
+
+      private
+
+      def read_at(offset, size)
+        @mutex.synchronize do
+          @file.seek offset
+          @file.read(size) || '' # nil at the end
+        end
+      end
+
+      def read_index()
+        header = read_at 0, HEADER_SIZE
+        raise "not a data file: #{@path}" unless header.bytesize == HEADER_SIZE
+
+        magic, version, seed, offset, size = header.unpack HEADER
+        raise "not a data file: #{@path}"              unless magic == MAGIC
+        raise "unknown data file version: #{version}" unless version == VERSION
+
+        index = DataFile.decode(read_at(offset, size), seed)
+          .force_encoding(Encoding::UTF_8)
+          .lines(chomp: true)
+          .to_h {
+            name, offset, size = _1.split "\t"
+            [name, [offset.to_i, size.to_i]]
+          }
+        return seed, index
+      end
+
+    end# DataFile
+
+
+  end# Packager
+
+
+end# Reflex