fastlane 2.196.0 → 2.212.2

data/match/lib/match/storage/gitlab_secure_files.rb

@@ -0,0 +1,182 @@
+require 'fastlane_core/command_executor'
+require 'fastlane_core/configuration/configuration'
+require 'net/http/post/multipart'
+
+require_relative './gitlab/client'
+require_relative './gitlab/secure_file'
+
+require_relative '../options'
+require_relative '../module'
+require_relative '../spaceship_ensure'
+require_relative './interface'
+
+module Match
+  module Storage
+    # Store the code signing identities in GitLab Secure Files
+    class GitLabSecureFiles < Interface
+      attr_reader :gitlab_client
+      attr_reader :project_id
+      attr_reader :readonly
+      attr_reader :username
+      attr_reader :team_id
+      attr_reader :team_name
+      attr_reader :api_key_path
+      attr_reader :api_key
+
+      def self.configure(params)
+        api_v4_url     = params[:api_v4_url] || ENV['CI_API_V4_URL'] || 'https://gitlab.com/api/v4'
+        project_id     = params[:gitlab_project] || ENV['GITLAB_PROJECT'] || ENV['CI_PROJECT_ID']
+        job_token      = params[:job_token] || ENV['CI_JOB_TOKEN']
+        private_token  = params[:private_token] || ENV['PRIVATE_TOKEN']
+
+        if params[:git_url].to_s.length > 0
+          UI.important("Looks like you still define a `git_url` somewhere, even though")
+          UI.important("you use GitLab Secure Files. You can remove the `git_url`")
+          UI.important("from your Matchfile and Fastfile")
+          UI.message("The above is just a warning, fastlane will continue as usual now...")
+        end
+
+        return self.new(
+          api_v4_url: api_v4_url,
+          project_id: project_id,
+          job_token: job_token,
+          private_token: private_token,
+          readonly: params[:readonly],
+          username: params[:username],
+          team_id: params[:team_id],
+          team_name: params[:team_name],
+          api_key_path: params[:api_key_path],
+          api_key: params[:api_key]
+        )
+      end
+
+      def initialize(api_v4_url: nil,
+                     project_id: nil,
+                     job_token: nil,
+                     private_token: nil,
+                     readonly: nil,
+                     username: nil,
+                     team_id: nil,
+                     team_name: nil,
+                     api_key_path: nil,
+                     api_key: nil)
+
+        @readonly = readonly
+        @username = username
+        @team_id = team_id
+        @team_name = team_name
+        @api_key_path = api_key_path
+        @api_key = api_key
+
+        @job_token = job_token
+        @private_token = private_token
+        @api_v4_url = api_v4_url
+        @project_id = project_id
+        @gitlab_client = GitLab::Client.new(job_token: job_token, private_token: private_token, project_id: project_id, api_v4_url: api_v4_url)
+
+        UI.message("Initializing match for GitLab project #{@project_id}")
+      end
+
+      # To make debugging easier, we have a custom exception here
+      def prefixed_working_directory
+        # We fall back to "*", which means certificates and profiles
+        # from all teams that use this bucket would be installed. This is not ideal, but
+        # unless the user provides a `team_id`, we can't know which one to use
+        # This only happens if `readonly` is activated, and no `team_id` was provided
+        @_folder_prefix ||= currently_used_team_id
+        if @_folder_prefix.nil?
+          # We use a `@_folder_prefix` variable, to keep state between multiple calls of this
+          # method, as the value won't change. This way the warning is only printed once
+          UI.important("Looks like you run `match` in `readonly` mode, and didn't provide a `team_id`. This will still work, however it is recommended to provide a `team_id` in your Appfile or Matchfile")
+          @_folder_prefix = "*"
+        end
+        return File.join(working_directory, @_folder_prefix)
+      end
+
+      def download
+        # Check if we already have a functional working_directory
+        return if @working_directory
+
+        # No existing working directory, creating a new one now
+        self.working_directory = Dir.mktmpdir
+
+        @gitlab_client.files.each do |secure_file|
+          secure_file.download(self.working_directory)
+        end
+
+        UI.verbose("Successfully downloaded all Secure Files from GitLab to #{self.working_directory}")
+      end
+
+      def currently_used_team_id
+        if self.readonly
+          # In readonly mode, we still want to see if the user provided a team_id
+          # see `prefixed_working_directory` comments for more details
+          return self.team_id
+        else
+          UI.user_error!("The `team_id` option is required. fastlane cannot automatically determine portal team id via the App Store Connect API (yet)") if self.team_id.to_s.empty?
+
+          spaceship = SpaceshipEnsure.new(self.username, self.team_id, self.team_name, api_token)
+          return spaceship.team_id
+        end
+      end
+
+      def api_token
+        api_token = Spaceship::ConnectAPI::Token.from(hash: self.api_key, filepath: self.api_key_path)
+        api_token ||= Spaceship::ConnectAPI.token
+        return api_token
+      end
+
+      # Returns a short string describing + identifing the current
+      # storage backend. This will be printed when nuking a storage
+      def human_readable_description
+        "GitLab Secure Files Storage [#{self.project_id}]"
+      end
+
+      def upload_files(files_to_upload: [], custom_message: nil)
+        # `files_to_upload` is an array of files that need to be uploaded to GitLab Secure Files
+        # Those doesn't mean they're new, it might just be they're changed
+        # Either way, we'll upload them using the same technique
+
+        files_to_upload.each do |current_file|
+          # Go from
+          #   "/var/folders/px/bz2kts9n69g8crgv4jpjh6b40000gn/T/d20181026-96528-1av4gge/profiles/development/Development_me.mobileprovision"
+          # to
+          #   "profiles/development/Development_me.mobileprovision"
+          #
+
+          # We also remove the trailing `/`
+          target_file = current_file.gsub(self.working_directory + "/", "")
+          UI.verbose("Uploading '#{target_file}' to GitLab Secure Files...")
+          @gitlab_client.upload_file(current_file, target_file)
+        end
+      end
+
+      def delete_files(files_to_delete: [], custom_message: nil)
+        files_to_delete.each do |current_file|
+          target_path = current_file.gsub(self.working_directory + "/", "")
+
+          secure_file = @gitlab_client.find_file_by_name(target_path)
+          UI.message("Deleting '#{target_path}' from GitLab Secure Files...")
+          secure_file.delete
+        end
+      end
+
+      def skip_docs
+        true
+      end
+
+      def list_files(file_name: "", file_ext: "")
+        Dir[File.join(working_directory, self.team_id, "**", file_name, "*.#{file_ext}")]
+      end
+
+      # Implement this for the `fastlane match init` command
+      # This method must return the content of the Matchfile
+      # that should be generated
+      def generate_matchfile_content(template: nil)
+        project = UI.input("What is your GitLab Project (i.e. gitlab-org/gitlab): ")
+
+        return "gitlab_project(\"#{project}\")"
+      end
+    end
+  end
+end

data/match/lib/match/storage/google_cloud_storage.rb

@@ -48,7 +48,8 @@ module Match
           team_id: params[:team_id],
           team_name: params[:team_name],
           api_key_path: params[:api_key_path],
-          api_key: params[:api_key]
+          api_key: params[:api_key],
+          skip_google_cloud_account_confirmation: params[:skip_google_cloud_account_confirmation]
         )
       end
 
@@ -62,7 +63,8 @@ module Match
                      team_id: nil,
                      team_name: nil,
                      api_key_path: nil,
-                     api_key: nil)
+                     api_key: nil,
+                     skip_google_cloud_account_confirmation: nil)
         @type = type if type
         @platform = platform if platform
         @google_cloud_project_id = google_cloud_project_id if google_cloud_project_id
@@ -76,8 +78,7 @@ module Match
         @api_key_path = api_key_path
         @api_key = api_key
 
-        @google_cloud_keys_file = ensure_keys_file_exists(google_cloud_keys_file, google_cloud_project_id)
-
+        @google_cloud_keys_file = ensure_keys_file_exists(google_cloud_keys_file, google_cloud_project_id, skip_google_cloud_account_confirmation)
         if self.google_cloud_keys_file.to_s.length > 0
           # Extract the Project ID from the `JSON` file
           # so the user doesn't have to provide it manually
@@ -223,7 +224,7 @@ module Match
 
       # This method will make sure the keys file exists
       # If it's missing, it will help the user set things up
-      def ensure_keys_file_exists(google_cloud_keys_file, google_cloud_project_id)
+      def ensure_keys_file_exists(google_cloud_keys_file, google_cloud_project_id, skip_google_cloud_account_confirmation)
         if google_cloud_keys_file && File.exist?(google_cloud_keys_file)
           return google_cloud_keys_file
         end
@@ -251,7 +252,7 @@ module Match
             # we can continue and ask the user if they want to use a keys file.
           end
 
-          if application_default_keys && UI.confirm("Do you want to use this system's Google Cloud application default keys?")
+          if application_default_keys && (skip_google_cloud_account_confirmation || UI.confirm("Do you want to use this system's Google Cloud application default keys?"))
             return nil
           end
         end

data/match/lib/match/storage/s3_storage.rb

@@ -168,10 +168,10 @@ module Match
       private
 
       def s3_object_path(file_name)
-        santized = sanitize_file_name(file_name)
-        return santized if santized.start_with?(s3_object_prefix)
+        sanitized = sanitize_file_name(file_name)
+        return sanitized if sanitized.start_with?(s3_object_prefix)
 
-        s3_object_prefix + santized
+        s3_object_prefix + sanitized
       end
 
       def strip_s3_object_prefix(object_path)

data/match/lib/match/storage.rb

@@ -2,6 +2,7 @@ require_relative 'storage/interface'
 require_relative 'storage/git_storage'
 require_relative 'storage/google_cloud_storage'
 require_relative 'storage/s3_storage'
+require_relative 'storage/gitlab_secure_files'
 
 module Match
   module Storage
@@ -16,6 +17,9 @@ module Match
           },
           "s3" => lambda { |params|
             return Storage::S3Storage.configure(params)
+          },
+          "gitlab_secure_files" => lambda { |params|
+            return Storage::GitLabSecureFiles.configure(params)
           }
         }
       end

data/match/lib/match/table_printer.rb

@@ -33,7 +33,8 @@ module Match
         Utils.environment_variable_name(app_identifier: app_identifier, type: type, platform: platform) => "Profile UUID",
         Utils.environment_variable_name_profile_name(app_identifier: app_identifier, type: type, platform: platform) => "Profile Name",
         Utils.environment_variable_name_profile_path(app_identifier: app_identifier, type: type, platform: platform) => "Profile Path",
-        Utils.environment_variable_name_team_id(app_identifier: app_identifier, type: type, platform: platform) => "Development Team ID"
+        Utils.environment_variable_name_team_id(app_identifier: app_identifier, type: type, platform: platform) => "Development Team ID",
+        Utils.environment_variable_name_certificate_name(app_identifier: app_identifier, type: type, platform: platform) => "Certificate Name"
       }.each do |env_key, name|
         rows << [name, env_key, ENV[env_key]]
       end

data/match/lib/match/utils.rb

@@ -31,8 +31,21 @@ module Match
       (base_environment_variable_name(app_identifier: app_identifier, type: type, platform: platform) + ["profile-path"]).join("_")
     end
 
-    def self.get_cert_info(cer_certificate_path)
-      cert = OpenSSL::X509::Certificate.new(File.binread(cer_certificate_path))
+    def self.environment_variable_name_certificate_name(app_identifier: nil, type: nil, platform: :ios)
+      (base_environment_variable_name(app_identifier: app_identifier, type: type, platform: platform) + ["certificate-name"]).join("_")
+    end
+
+    def self.get_cert_info(cer_certificate)
+      # can receive a certificate path or the file data
+      begin
+        if File.exist?(cer_certificate)
+          cer_certificate = File.binread(cer_certificate)
+        end
+      rescue ArgumentError
+        # cert strings have null bytes; suppressing output
+      end
+
+      cert = OpenSSL::X509::Certificate.new(cer_certificate)
 
       # openssl output:
       # subject= /UID={User ID}/CN={Certificate Name}/OU={Certificate User}/O={Organisation}/C={Country}

data/pem/lib/pem/manager.rb

@@ -18,7 +18,13 @@ module PEM
 
         if existing_certificate
           remaining_days = (existing_certificate.expires - Time.now) / 60 / 60 / 24
-          UI.message("Existing push notification profile for '#{existing_certificate.owner_name}' is valid for #{remaining_days.round} more days.")
+
+          display_platform = ''
+          unless PEM.config[:website_push]
+            display_platform = "#{PEM.config[:platform]} "
+          end
+
+          UI.message("Existing #{display_platform}push notification profile for '#{existing_certificate.owner_name}' is valid for #{remaining_days.round} more days.")
           if remaining_days > PEM.config[:active_days_limit]
             if PEM.config[:force]
               UI.success("You already have an existing push certificate, but a new one will be created since the --force option has been set.")
@@ -59,7 +65,7 @@ module PEM
 
         x509_certificate = cert.download
 
-        filename_base = PEM.config[:pem_name] || "#{certificate_type}_#{PEM.config[:app_identifier]}"
+        filename_base = PEM.config[:pem_name] || "#{certificate_type}_#{PEM.config[:app_identifier]}_#{PEM.config[:platform]}"
         filename_base = File.basename(filename_base, ".pem") # strip off the .pem if it was provided.
 
         output_path = File.expand_path(PEM.config[:output_path])
@@ -75,7 +81,7 @@ module PEM
           p12_cert_path = File.join(output_path, "#{filename_base}.p12")
           p12_password = PEM.config[:p12_password] == "" ? nil : PEM.config[:p12_password]
           p12 = OpenSSL::PKCS12.create(p12_password, certificate_type, pkey, x509_certificate)
-          File.write(p12_cert_path, p12.to_der)
+          File.write(p12_cert_path, p12.to_der.force_encoding("UTF-8"))
           UI.message("p12 certificate: ".green + Pathname.new(p12_cert_path).realpath.to_s)
         end
 
@@ -86,12 +92,29 @@ module PEM
       end
 
       def certificate
-        if PEM.config[:development]
-          Spaceship.certificate.development_push
-        elsif PEM.config[:website_push]
+        if PEM.config[:website_push]
           Spaceship.certificate.website_push
         else
-          Spaceship.certificate.production_push
+          platform = PEM.config[:platform]
+          UI.user_error!('platform parameter is unspecified.') unless platform
+
+          case platform
+          when 'ios'
+            if PEM.config[:development]
+              Spaceship.certificate.development_push
+            else
+              Spaceship.certificate.production_push
+            end
+          when 'macos'
+            if PEM.config[:development]
+              Spaceship.certificate.mac_development_push
+            else
+              Spaceship.certificate.mac_production_push
+            end
+          else
+            UI.user_error!("Unsupported platform '#{platform}'. Supported platforms for development and production certificates are 'ios' & 'macos'")
+          end
+
         end
       end
 

data/pem/lib/pem/options.rb

@@ -1,5 +1,6 @@
 require 'fastlane_core/configuration/config_item'
 require 'credentials_manager/appfile_config'
+require 'fastlane/helper/lane_helper'
 
 require_relative 'module'
 
@@ -10,6 +11,14 @@ module PEM
       user ||= CredentialsManager::AppfileConfig.try_fetch_value(:apple_id)
 
       [
+        FastlaneCore::ConfigItem.new(key: :platform,
+                                     description: "Set certificate's platform. Used for creation of production & development certificates. Supported platforms: ios, macos",
+                                     short_option: "-m",
+                                     env_name: "PEM_PLATFORM",
+                                     default_value: "ios",
+                                     verify_block: proc do |value|
+                                       UI.user_error!("The platform can only be ios or macos") unless ['ios', 'macos'].include?(value)
+                                     end),
         FastlaneCore::ConfigItem.new(key: :development,
                                      env_name: "PEM_DEVELOPMENT",
                                      description: "Renew the development push certificate instead of the production one",

data/pilot/lib/pilot/build_manager.rb

@@ -18,6 +18,11 @@ module Pilot
 
       UI.user_error!("No ipa or pkg file given") if config[:ipa].nil? && config[:pkg].nil?
 
+      if config[:ipa] && config[:pkg]
+        UI.important("WARNING: Both `ipa` and `pkg` options are defined either explicitly or with default_value (build found in directory)")
+        UI.important("Uploading `ipa` is preferred by default. Set `app_platform` to `osx` to force uploading `pkg`")
+      end
+
       check_for_changelog_or_whats_new!(options)
 
       UI.success("Ready to upload new build to TestFlight (App: #{fetch_app_id})...")
@@ -25,24 +30,29 @@ module Pilot
       dir = Dir.mktmpdir
 
       platform = fetch_app_platform
-      if options[:ipa]
+      ipa_path = options[:ipa]
+      if ipa_path && platform != 'osx'
+        asset_path = ipa_path
         package_path = FastlaneCore::IpaUploadPackageBuilder.new.generate(app_id: fetch_app_id,
-                                                                      ipa_path: options[:ipa],
+                                                                      ipa_path: ipa_path,
                                                                   package_path: dir,
                                                                       platform: platform)
       else
+        pkg_path = options[:pkg]
+        asset_path = pkg_path
         package_path = FastlaneCore::PkgUploadPackageBuilder.new.generate(app_id: fetch_app_id,
-                                                                        pkg_path: options[:pkg],
+                                                                        pkg_path: pkg_path,
                                                                     package_path: dir,
                                                                         platform: platform)
       end
 
       transporter = transporter_for_selected_team(options)
-      result = transporter.upload(package_path: package_path)
+      result = transporter.upload(package_path: package_path, asset_path: asset_path, platform: platform)
 
       unless result
         transporter_errors = transporter.displayable_errors
-        UI.user_error!("Error uploading ipa file: \n #{transporter_errors}")
+        file_type = platform == "osx" ? "pkg" : "ipa"
+        UI.user_error!("Error uploading #{file_type} file: \n #{transporter_errors}")
       end
 
       UI.success("Successfully uploaded the new binary to App Store Connect")
@@ -98,10 +108,10 @@ module Pilot
 
     def wait_for_build_processing_to_be_complete(return_when_build_appears = false)
       platform = fetch_app_platform
-      if config[:ipa]
+      if config[:ipa] && platform != "osx" && !config[:distribute_only]
         app_version = FastlaneCore::IpaFileAnalyser.fetch_app_version(config[:ipa])
         app_build = FastlaneCore::IpaFileAnalyser.fetch_app_build(config[:ipa])
-      elsif config[:pkg]
+      elsif config[:pkg] && !config[:distribute_only]
         app_version = FastlaneCore::PkgFileAnalyser.fetch_app_version(config[:pkg])
         app_build = FastlaneCore::PkgFileAnalyser.fetch_app_build(config[:pkg])
       else
@@ -147,7 +157,7 @@ module Pilot
           end
         end
         platform = Spaceship::ConnectAPI::Platform.map(fetch_app_platform)
-        build ||= Spaceship::ConnectAPI::Build.all(app_id: app.id, version: app_version, build_number: build_number, sort: "-uploadedDate", platform: platform, limit: 1).first
+        build ||= Spaceship::ConnectAPI::Build.all(app_id: app.id, version: app_version, build_number: build_number, sort: "-uploadedDate", platform: platform, limit: Spaceship::ConnectAPI::Platform.size).first
       end
 
       # Verify the build has all the includes that we need
@@ -379,24 +389,34 @@ module Pilot
     def transporter_for_selected_team(options)
       # Use JWT auth
       api_token = Spaceship::ConnectAPI.token
+      api_key = if options[:api_key].nil? && !api_token.nil?
+                  # Load api key info if user set api_key_path, not api_key
+                  { key_id: api_token.key_id, issuer_id: api_token.issuer_id, key: api_token.key_raw }
+                elsif !options[:api_key].nil?
+                  api_key = options[:api_key].transform_keys(&:to_sym).dup
+                  # key is still base 64 style if api_key is loaded from option
+                  api_key[:key] = Base64.decode64(api_key[:key]) if api_key[:is_key_content_base64]
+                  api_key
+                end
+
       unless api_token.nil?
         api_token.refresh! if api_token.expired?
-        return FastlaneCore::ItunesTransporter.new(nil, nil, false, nil, api_token.text)
+        return FastlaneCore::ItunesTransporter.new(nil, nil, false, nil, api_token.text, altool_compatible_command: true, api_key: api_key)
       end
 
       # Otherwise use username and password
       tunes_client = Spaceship::ConnectAPI.client ? Spaceship::ConnectAPI.client.tunes_client : nil
 
-      generic_transporter = FastlaneCore::ItunesTransporter.new(options[:username], nil, false, options[:itc_provider])
+      generic_transporter = FastlaneCore::ItunesTransporter.new(options[:username], nil, false, options[:itc_provider], altool_compatible_command: true, api_key: api_key)
       return generic_transporter if options[:itc_provider] || tunes_client.nil?
       return generic_transporter unless tunes_client.teams.count > 1
 
       begin
-        team = tunes_client.teams.find { |t| t['contentProvider']['contentProviderId'].to_s == tunes_client.team_id }
-        name = team['contentProvider']['name']
+        team = tunes_client.teams.find { |t| t['providerId'].to_s == tunes_client.team_id }
+        name = team['name']
         provider_id = generic_transporter.provider_ids[name]
         UI.verbose("Inferred provider id #{provider_id} for team #{name}.")
-        return FastlaneCore::ItunesTransporter.new(options[:username], nil, false, provider_id)
+        return FastlaneCore::ItunesTransporter.new(options[:username], nil, false, provider_id, altool_compatible_command: true, api_key: api_key)
       rescue => ex
         STDERR.puts(ex.to_s)
         UI.verbose("Couldn't infer a provider short name for team with id #{tunes_client.team_id} automatically: #{ex}. Proceeding without provider short name.")
@@ -410,7 +430,7 @@ module Pilot
       # This is where we could add a check to see if encryption is required and has been updated
       uploaded_build = set_export_compliance_if_needed(uploaded_build, options)
 
-      if options[:groups] || options[:distribute_external]
+      if options[:submit_beta_review] && (options[:groups] || options[:distribute_external])
         if uploaded_build.ready_for_beta_submission?
           uploaded_build.post_beta_app_review_submission
         else

data/pilot/lib/pilot/options.rb

@@ -324,7 +324,12 @@ module Pilot
                                      env_name: "PILOT_REJECT_PREVIOUS_BUILD",
                                      description: "Expire previous if it's 'waiting for review'",
                                      is_string: false,
-                                     default_value: false)
+                                     default_value: false),
+        FastlaneCore::ConfigItem.new(key: :submit_beta_review,
+                                     env_name: "PILOT_DISTRIBUTE_EXTERNAL",
+                                     description: "Send the build for a beta review",
+                                     type: Boolean,
+                                     default_value: true)
       ]
     end
   end

data/scan/lib/scan/detect_values.rb

@@ -209,6 +209,12 @@ module Scan
 
     def self.detect_destination
       if Scan.config[:destination]
+        # No need to show below warnings message(s) for xcode13+, because
+        # Apple recommended to have destination in all xcodebuild commands
+        # otherwise, Apple will generate warnings in console logs
+        # see: https://github.com/fastlane/fastlane/issues/19579
+        return if Helper.xcode_at_least?("13.0")
+
         UI.important("It's not recommended to set the `destination` value directly")
         UI.important("Instead use the other options available in `fastlane scan --help`")
         UI.important("Using your value '#{Scan.config[:destination]}' for now")

data/scan/lib/scan/error_handler.rb

@@ -27,6 +27,15 @@ module Scan
           print("If you are using zshell or another shell, make sure to edit the correct bash file.")
           print("For more information visit this stackoverflow answer:")
           print("https://stackoverflow.com/a/17031697/445598")
+        when /Testing failed on/
+          # This is important because xcbeautify and raw output will print:
+          # Testing failed on 'iPhone 13 Pro Max'
+          # when multiple devices are use.
+          # xcpretty hides this output so its not an issue with xcpretty.
+          # We need to catch "Testing failed on" before "Test failed"
+          # so that an error isn't raised.
+          # Raising an error prevents trainer from processing the xcresult
+          return
         when /Testing failed/
           UI.build_failure!("Error building the application. #{details}")
         when /Executed/, /Failing tests:/