fastlane 2.155.3 → 2.157.2

data/deliver/lib/deliver/upload_screenshots.rb

@@ -4,10 +4,17 @@ require 'digest/md5'
 require_relative 'app_screenshot'
 require_relative 'module'
 require_relative 'loader'
+require_relative 'queue_worker'
+require_relative 'app_screenshot_iterator'
 
 module Deliver
   # upload screenshots to App Store Connect
   class UploadScreenshots
+    DeleteScreenshotJob = Struct.new(:app_screenshot, :localization, :app_screenshot_set)
+    UploadScreenshotJob = Struct.new(:app_screenshot_set, :path)
+
+    NUMBER_OF_THREADS = Helper.test? ? 1 : [ENV.fetch("DELIVER_NUMBER_OF_THREADS", 10).to_i, 10].min
+
     def upload(options, screenshots)
       return if options[:skip_screenshots]
       return if options[:edit_live]
@@ -50,57 +57,47 @@ module Deliver
         localizations = version.get_app_store_version_localizations
       end
 
-      upload_screenshots(screenshots_per_language, localizations, options)
+      upload_screenshots(localizations, screenshots_per_language)
+
+      Helper.show_loading_indicator("Sorting screenshots uploaded...")
+      sort_screenshots(localizations)
+      Helper.hide_loading_indicator
+
+      UI.success("Successfully uploaded screenshots to App Store Connect")
     end
 
     def delete_screenshots(localizations, screenshots_per_language, tries: 5)
       tries -= 1
 
-      # Get localizations on version
-      localizations.each do |localization|
-        # Only delete screenshots if trying to upload
-        next unless screenshots_per_language.keys.include?(localization.locale)
-
-        # Iterate over all screenshots for each set and delete
-        screenshot_sets = localization.get_app_screenshot_sets
-
-        # Multi threading delete on single localization
-        threads = []
-        errors = []
-
-        screenshot_sets.each do |screenshot_set|
-          UI.message("Removing all previously uploaded screenshots for '#{localization.locale}' '#{screenshot_set.screenshot_display_type}'...")
-          screenshot_set.app_screenshots.each do |screenshot|
-            UI.verbose("Deleting screenshot - #{localization.locale} #{screenshot_set.screenshot_display_type} #{screenshot.id}")
-            threads << Thread.new do
-              begin
-                screenshot.delete!
-                UI.verbose("Deleted screenshot - #{localization.locale} #{screenshot_set.screenshot_display_type} #{screenshot.id}")
-              rescue => error
-                UI.verbose("Failed to delete screenshot - #{localization.locale} #{screenshot_set.screenshot_display_type} #{screenshot.id}")
-                errors << error
-              end
-            end
-          end
+      worker = QueueWorker.new(NUMBER_OF_THREADS) do |job|
+        start_time = Time.now
+        target = "#{job.localization.locale} #{job.app_screenshot_set.screenshot_display_type} #{job.app_screenshot.id}"
+        begin
+          UI.verbose("Deleting '#{target}'")
+          job.app_screenshot.delete!
+          UI.message("Deleted '#{target}' -  (#{Time.now - start_time} secs)")
+        rescue => error
+          UI.error("Failed to delete screenshot #{target} - (#{Time.now - start_time} secs)")
+          UI.error(error.message)
         end
+      end
 
-        sleep(1) # Feels bad but sleeping a bit to let the threads catchup
-
-        unless threads.empty?
-          Helper.show_loading_indicator("Waiting for screenshots to be deleted for '#{localization.locale}'... (might be slow)") unless FastlaneCore::Globals.verbose?
-          threads.each(&:join)
-          Helper.hide_loading_indicator unless FastlaneCore::Globals.verbose?
-        end
+      iterator = AppScreenshotIterator.new(localizations)
+      iterator.each_app_screenshot do |localization, app_screenshot_set, app_screenshot|
+        # Only delete screenshots if trying to upload
+        next unless screenshots_per_language.keys.include?(localization.locale)
 
-        # Crash if any errors happen while deleting
-        errors.each do |error|
-          UI.error(error.message)
-        end
+        UI.verbose("Queued delete sceeenshot job for #{localization.locale} #{app_screenshot_set.screenshot_display_type} #{app_screenshot.id}")
+        worker.enqueue(DeleteScreenshotJob.new(app_screenshot, localization, app_screenshot_set))
       end
 
+      worker.start
+
       # Verify all screenshots have been deleted
       # Sometimes API requests will fail but screenshots will still be deleted
-      count = count_screenshots(localizations)
+      count = iterator.each_app_screenshot_set.map { |_, app_screenshot_set| app_screenshot_set }
+                      .reduce(0) { |sum, app_screenshot_set| sum + app_screenshot_set.app_screenshots.size }
+
       UI.important("Number of screenshots not deleted: #{count}")
       if count > 0
         if tries.zero?
@@ -114,113 +111,127 @@ module Deliver
       end
     end
 
-    def count_screenshots(localizations)
-      count = 0
-      localizations.each do |localization|
-        screenshot_sets = localization.get_app_screenshot_sets
-        screenshot_sets.each do |screenshot_set|
-          count += screenshot_set.app_screenshots.size
+    def upload_screenshots(localizations, screenshots_per_language, tries: 5)
+      tries -= 1
+
+      # Upload screenshots
+      worker = QueueWorker.new(NUMBER_OF_THREADS) do |job|
+        begin
+          UI.verbose("Uploading '#{job.path}'...")
+          start_time = Time.now
+          job.app_screenshot_set.upload_screenshot(path: job.path, wait_for_processing: false)
+          UI.message("Uploaded '#{job.path}'... (#{Time.now - start_time} secs)")
+        rescue => error
+          UI.error(error)
         end
       end
 
-      return count
-    end
+      number_of_screenshots = 0
+      iterator = AppScreenshotIterator.new(localizations)
+      iterator.each_local_screenshot(screenshots_per_language) do |localization, app_screenshot_set, screenshot, index|
+        if index >= 10
+          UI.error("Too many screenshots found for device '#{screenshot.device_type}' in '#{screenshot.language}', skipping this one (#{screenshot.path})")
+          next
+        end
 
-    def upload_screenshots(screenshots_per_language, localizations, options)
-      # Check if should wait for processing
-      # Default to waiting if submitting for review (since needed for submission)
-      # Otherwise use enviroment variable
-      if ENV["DELIVER_SKIP_WAIT_FOR_SCREENSHOT_PROCESSING"].nil?
-        wait_for_processing = options[:submit_for_review]
-        UI.verbose("Setting wait_for_processing from ':submit_for_review' option")
-      else
-        UI.verbose("Setting wait_for_processing from 'DELIVER_SKIP_WAIT_FOR_SCREENSHOT_PROCESSING' environment variable")
-        wait_for_processing = !FastlaneCore::Env.truthy?("DELIVER_SKIP_WAIT_FOR_SCREENSHOT_PROCESSING")
-      end
+        checksum = UploadScreenshots.calculate_checksum(screenshot.path)
+        duplicate = (app_screenshot_set.app_screenshots || []).any? { |s| s.source_file_checksum == checksum }
 
-      if wait_for_processing
-        UI.important("Will wait for screenshot image processing")
-        UI.important("Set env DELIVER_SKIP_WAIT_FOR_SCREENSHOT_PROCESSING=true to skip waiting for screenshots to process")
-      else
-        UI.important("Skipping the wait for screenshot image processing (which may affect submission)")
-        UI.important("Set env DELIVER_SKIP_WAIT_FOR_SCREENSHOT_PROCESSING=false to wait for screenshots to process")
+        # Enqueue uploading job if it's not duplicated otherwise screenshot will be skipped
+        if duplicate
+          UI.message("Previous uploaded. Skipping '#{screenshot.path}'...")
+        else
+          worker.enqueue(UploadScreenshotJob.new(app_screenshot_set, screenshot.path))
+        end
+
+        number_of_screenshots += 1
       end
 
-      # Upload screenshots
-      indized = {} # per language and device type
+      worker.start
 
-      screenshots_per_language.each do |language, screenshots_for_language|
-        # Find localization to upload screenshots to
-        localization = localizations.find do |l|
-          l.locale == language
-        end
+      UI.verbose('Uploading jobs are completed')
 
-        unless localization
-          UI.error("Couldn't find localization on version for #{language}")
-          next
-        end
+      Helper.show_loading_indicator("Waiting for all the screenshots processed...")
+      states = wait_for_complete(iterator)
+      Helper.hide_loading_indicator
+      retry_upload_screenshots_if_needed(iterator, states, number_of_screenshots, tries, localizations, screenshots_per_language)
 
-        indized[localization.locale] ||= {}
+      UI.message("Successfully uploaded all screenshots")
+    end
 
-        # Create map to find screenshot set to add screenshot to
-        app_screenshot_sets_map = {}
-        app_screenshot_sets = localization.get_app_screenshot_sets
-        app_screenshot_sets.each do |app_screenshot_set|
-          app_screenshot_sets_map[app_screenshot_set.screenshot_display_type] = app_screenshot_set
+    # Verify all screenshots have been processed
+    def wait_for_complete(iterator)
+      loop do
+        states = iterator.each_app_screenshot.map { |_, _, app_screenshot| app_screenshot }.each_with_object({}) do |app_screenshot, hash|
+          state = app_screenshot.asset_delivery_state['state']
+          hash[state] ||= 0
+          hash[state] += 1
+        end
 
-          # Set initial screnshot count
-          indized[localization.locale][app_screenshot_set.screenshot_display_type] ||= {
-            count: app_screenshot_set.app_screenshots.size,
-            checksums: []
-          }
+        is_processing = states.fetch('UPLOAD_COMPLETE', 0) > 0
+        return states unless is_processing
 
-          checksums = app_screenshot_set.app_screenshots.map(&:source_file_checksum).uniq
-          indized[localization.locale][app_screenshot_set.screenshot_display_type][:checksums] = checksums
-        end
+        UI.verbose("There are still incomplete screenshots - #{states}")
+        sleep(5)
+      end
+    end
 
-        UI.message("Uploading #{screenshots_for_language.length} screenshots for language #{language}")
-        screenshots_for_language.each do |screenshot|
-          display_type = screenshot.device_type
-          set = app_screenshot_sets_map[display_type]
+    # Verify all screenshots states on App Store Connect are okay
+    def retry_upload_screenshots_if_needed(iterator, states, number_of_screenshots, tries, localizations, screenshots_per_language)
+      is_failure = states.fetch("FAILED", 0) > 0
+      is_missing_screenshot = !screenshots_per_language.empty? && !verify_local_screenshots_are_uploaded(iterator, screenshots_per_language)
+      return unless is_failure || is_missing_screenshot
 
-          if display_type.nil?
-            UI.error("Error... Screenshot size #{screenshot.screen_size} not valid for App Store Connect")
-            next
-          end
+      if tries.zero?
+        iterator.each_app_screenshot.select { |_, _, app_screenshot| app_screenshot.error? }.each do |localization, _, app_screenshot|
+          UI.error("#{app_screenshot.file_name} for #{localization.locale} has error(s) - #{app_screenshot.error_messages.join(', ')}")
+        end
+        incomplete_screenshot_count = states.reject { |k, v| k == 'COMPLETE' }.reduce(0) { |sum, (k, v)| sum + v }
+        UI.user_error!("Failed verification of all screenshots uploaded... #{incomplete_screenshot_count} incomplete screenshot(s) still exist")
+      else
+        UI.error("Failed to upload all screenshots... Tries remaining: #{tries}")
+        # Delete bad entries before retry
+        iterator.each_app_screenshot do |_, _, app_screenshot|
+          app_screenshot.delete! unless app_screenshot.complete?
+        end
+        upload_screenshots(localizations, screenshots_per_language, tries: tries)
+      end
+    end
 
-          unless set
-            set = localization.create_app_screenshot_set(attributes: {
-              screenshotDisplayType: display_type
-            })
-            app_screenshot_sets_map[display_type] = set
+    # Return `true` if all the local screenshots are uploaded to App Store Connect
+    def verify_local_screenshots_are_uploaded(iterator, screenshots_per_language)
+      # Check if local screenshots' checksum exist on App Store Connect
+      checksum_to_app_screenshot = iterator.each_app_screenshot.map { |_, _, app_screenshot| [app_screenshot.source_file_checksum, app_screenshot] }.to_h
 
-            indized[localization.locale][set.screenshot_display_type] = {
-              count: 0,
-              checksums: []
-            }
-          end
+      missing_local_screenshots = iterator.each_local_screenshot(screenshots_per_language).select do |_, _, local_screenshot, index|
+        checksum = UploadScreenshots.calculate_checksum(local_screenshot.path)
+        checksum_to_app_screenshot[checksum].nil? && index < 10 # if index is more than 10, it's skipped
+      end
 
-          index = indized[localization.locale][set.screenshot_display_type][:count]
+      missing_local_screenshots.each do |_, _, screenshot, _|
+        UI.error("#{screenshot.path} is missing on App Store Connect.")
+      end
 
-          if index >= 10
-            UI.error("Too many screenshots found for device '#{screenshot.device_type}' in '#{screenshot.language}', skipping this one (#{screenshot.path})")
-            next
-          end
+      missing_local_screenshots.empty?
+    end
 
-          bytes = File.binread(screenshot.path)
-          checksum = Digest::MD5.hexdigest(bytes)
-          duplicate = indized[localization.locale][set.screenshot_display_type][:checksums].include?(checksum)
+    def sort_screenshots(localizations)
+      iterator = AppScreenshotIterator.new(localizations)
 
-          if duplicate
-            UI.message("Previous uploaded. Skipping '#{screenshot.path}'...")
-          else
-            indized[localization.locale][set.screenshot_display_type][:count] += 1
-            UI.message("Uploading '#{screenshot.path}'...")
-            set.upload_screenshot(path: screenshot.path, wait_for_processing: wait_for_processing)
-          end
+      # Re-order screenshots within app_screenshot_set
+      worker = QueueWorker.new(NUMBER_OF_THREADS) do |app_screenshot_set|
+        original_ids = app_screenshot_set.app_screenshots.map(&:id)
+        sorted_ids = app_screenshot_set.app_screenshots.sort_by(&:file_name).map(&:id)
+        if original_ids != sorted_ids
+          app_screenshot_set.reorder_screenshots(app_screenshot_ids: sorted_ids)
         end
       end
-      UI.success("Successfully uploaded screenshots to App Store Connect")
+
+      iterator.each_app_screenshot_set do |_, app_screenshot_set|
+        worker.enqueue(app_screenshot_set)
+      end
+
+      worker.start
     end
 
     def collect_screenshots(options)
@@ -293,11 +304,15 @@ module Deliver
 
     # helper method so Spaceship::Tunes.client.available_languages is easier to test
     def self.available_languages
-      if Helper.test?
-        FastlaneCore::Languages::ALL_LANGUAGES
-      else
-        Spaceship::Tunes.client.available_languages
-      end
+      # 2020-08-24 - Available locales are not available as an endpoint in App Store Connect
+      # Update with Spaceship::Tunes.client.available_languages.sort (as long as endpoint is avilable)
+      Deliver::Languages::ALL_LANGUAGES
+    end
+
+    # helper method to mock this step in tests
+    def self.calculate_checksum(path)
+      bytes = File.binread(path)
+      Digest::MD5.hexdigest(bytes)
     end
   end
 end

data/fastlane/lib/fastlane/actions/app_store_connect_api_key.rb

@@ -0,0 +1,120 @@
+module Fastlane
+  module Actions
+    module SharedValues
+      APP_STORE_CONNECT_API_KEY = :APP_STORE_CONNECT_API_KEY
+    end
+
+    class AppStoreConnectApiKeyAction < Action
+      def self.run(options)
+        key_id = options[:key_id]
+        issuer_id = options[:issuer_id]
+        key_content = options[:key_content]
+        key_filepath = options[:key_filepath]
+        duration = options[:duration]
+        in_house = options[:in_house]
+
+        if key_content.nil? && key_filepath.nil?
+          UI.user_error!(":key_content or :key_filepath is required")
+        end
+
+        # This hash matches the named arguments on
+        # the Spaceship::ConnectAPI::Token.create method
+        key = {
+          key_id: key_id,
+          issuer_id: issuer_id,
+          key: key_content || File.binread(key_filepath),
+          duration: duration,
+          in_house: in_house
+        }
+
+        Actions.lane_context[SharedValues::APP_STORE_CONNECT_API_KEY] = key
+
+        return key
+      end
+
+      def self.description
+        "Load the App Store Connect API token to use in other fastlane tools and actions"
+      end
+
+      def self.available_options
+        [
+          FastlaneCore::ConfigItem.new(key: :key_id,
+                                       env_name: "APP_STORE_CONNECT_API_KEY_KEY_ID",
+                                       description: "The key ID"),
+          FastlaneCore::ConfigItem.new(key: :issuer_id,
+                                       env_name: "APP_STORE_CONNECT_API_KEY_ISSUER_ID",
+                                       description: "The issuer ID"),
+          FastlaneCore::ConfigItem.new(key: :key_filepath,
+                                       env_name: "APP_STORE_CONNECT_API_KEY_KEY_FILEPATH",
+                                       description: "The path to the key p8 file",
+                                       optional: true,
+                                       conflicting_options: [:key_content],
+                                       verify_block: proc do |value|
+                                         UI.user_error!("Couldn't find key p8 file at path '#{value}'") unless File.exist?(value)
+                                       end),
+          FastlaneCore::ConfigItem.new(key: :key_content,
+                                       env_name: "APP_STORE_CONNECT_API_KEY_KEY",
+                                       description: "The content of the key p8 file",
+                                       optional: true,
+                                       conflicting_options: [:filepath]),
+          FastlaneCore::ConfigItem.new(key: :duration,
+                                       env_name: "APP_STORE_CONNECT_API_KEY_DURATION",
+                                       description: "The token session duration",
+                                       optional: true,
+                                       type: Integer),
+          FastlaneCore::ConfigItem.new(key: :in_house,
+                                       env_name: "APP_STORE_CONNECT_API_KEY_IN_HOUSE",
+                                       description: "Is App Store or Enterprise (in house) team? App Store Connect API cannot not determine this on its own (yet)",
+                                       optional: true,
+                                       type: Boolean)
+        ]
+      end
+
+      def self.output
+        [
+          ['APP_STORE_CONNECT_API_KEY', 'The App Store Connect API key information used for authorization requests. This hash can be passed directly into the :api_key options on other tools or into Spaceship::ConnectAPI::Token.create method']
+        ]
+      end
+
+      def self.author
+        ["joshdholtz"]
+      end
+
+      def self.is_supported?(platform)
+        true
+      end
+
+      def self.details
+        [
+          "Load the App Store Connect API token to use in other fastlane tools and actions"
+        ].join("\n")
+      end
+
+      def self.example_code
+        [
+          'app_store_connect_api_key(
+            key_id: "D83848D23",
+            issuer_id: "227b0bbf-ada8-458c-9d62-3d8022b7d07f",
+            key_filepath: "D83848D23.p8"
+          )',
+          'app_store_connect_api_key(
+            key_id: "D83848D23",
+            issuer_id: "227b0bbf-ada8-458c-9d62-3d8022b7d07f",
+            key_filepath: "D83848D23.p8",
+            duration: 200,
+            in_house: true
+          )',
+          'app_store_connect_api_key(
+            key_id: "D83848D23",
+            issuer_id: "227b0bbf-ada8-458c-9d62-3d8022b7d07f",
+            key_content: "-----BEGIN EC PRIVATE KEY-----\nfewfawefawfe\n-----END EC PRIVATE KEY-----"
+          )'
+        ]
+      end
+
+      def self.category
+        :app_store_connect
+      end
+    end
+  end
+end