fastlane 2.155.0 → 2.156.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d05003197423c1e94c2728d49100fbacb7d5f5b657491b94e90f34432d2774ba
-  data.tar.gz: 99b28cc3d83ec3500d9a90554db8464ee479a7e533141e9246164d26fef8edba
+  metadata.gz: 8ddb9f8d09ffcb09f5ddd4d6365f67d41c538e8804630e40911e058a8174aaf2
+  data.tar.gz: 8bf0f6513c658bb9891058e5bee6fb22689bc496376e4aff6a2d2e5f1a2b9b74
 SHA512:
-  metadata.gz: 9a7b6ce2df7348ad80f366ad54a619ad1d966348e3e2a0f616f43d160681c1ba15625f7c16923d4b0bbfa3c873b3936891001f53c5349abba1e0a3c0a6bb4704
-  data.tar.gz: c75739db5b1e8dcea81230f67effe0d1b46de274cab20d85183aa3dafb64088ed85a7260f2aa47e14e802c8a81879b541d7ba4fbca55f56a2c84c1cb09fbeae0
+  metadata.gz: 9627a6de633cc011c1cd083468869543125e4eb021f025152c3aaf0e4e6067c80bcc514e71c1d425485138df0ff7ab43fa70771d846e0270c49ad50b60dc47c1
+  data.tar.gz: 177ba0af8b688d96e5374f7a3175663cebe4e3bae3966733880e75e0634cc4e1cdedc3f12c1c8ac0cd415b024eb3aa9385489d67f98b2e9d4e5175631d2874bf

data/README.md

@@ -34,43 +34,17 @@ If the above doesn't help, please [submit an issue](https://github.com/fastlane/
 <!-- This table is regenerated and resorted on each release -->
 <table id='team'>
 <tr>
-<td id='danielle-tomlinson'>
-<a href='https://github.com/endocrimes'>
-<img src='https://github.com/endocrimes.png?size=140'>
-</a>
-<h4 align='center'><a href='https://twitter.com/endocrimes'>Danielle Tomlinson</a></h4>
-</td>
-<td id='olivier-halligon'>
-<a href='https://github.com/AliSoftware'>
-<img src='https://github.com/AliSoftware.png?size=140'>
-</a>
-<h4 align='center'><a href='https://twitter.com/aligatr'>Olivier Halligon</a></h4>
-</td>
-<td id='max-ott'>
-<a href='https://github.com/max-ott'>
-<img src='https://github.com/max-ott.png?size=140'>
-</a>
-<h4 align='center'><a href='https://twitter.com/ott_max'>Max Ott</a></h4>
-</td>
 <td id='jorge-revuelta-h'>
 <a href='https://github.com/minuscorp'>
 <img src='https://github.com/minuscorp.png?size=140'>
 </a>
 <h4 align='center'><a href='https://twitter.com/minuscorp'>Jorge Revuelta H</a></h4>
 </td>
-<td id='andrew-mcburney'>
-<a href='https://github.com/armcburney'>
-<img src='https://github.com/armcburney.png?size=140'>
-</a>
-<h4 align='center'><a href='https://twitter.com/armcburney'>Andrew McBurney</a></h4>
-</td>
-</tr>
-<tr>
-<td id='jérôme-lacoste'>
-<a href='https://github.com/lacostej'>
-<img src='https://github.com/lacostej.png?size=140'>
+<td id='luka-mirosevic'>
+<a href='https://github.com/lmirosevic'>
+<img src='https://github.com/lmirosevic.png?size=140'>
 </a>
-<h4 align='center'><a href='https://twitter.com/lacostej'>Jérôme Lacoste</a></h4>
+<h4 align='center'><a href='https://twitter.com/lmirosevic'>Luka Mirosevic</a></h4>
 </td>
 <td id='josh-holtz'>
 <a href='https://github.com/joshdholtz'>
@@ -78,11 +52,11 @@ If the above doesn't help, please [submit an issue](https://github.com/fastlane/
 </a>
 <h4 align='center'><a href='https://twitter.com/joshdholtz'>Josh Holtz</a></h4>
 </td>
-<td id='manu-wallner'>
-<a href='https://github.com/milch'>
-<img src='https://github.com/milch.png?size=140'>
+<td id='felix-krause'>
+<a href='https://github.com/KrauseFx'>
+<img src='https://github.com/KrauseFx.png?size=140'>
 </a>
-<h4 align='center'><a href='https://twitter.com/acrooow'>Manu Wallner</a></h4>
+<h4 align='center'><a href='https://twitter.com/KrauseFx'>Felix Krause</a></h4>
 </td>
 <td id='fumiya-nakamura'>
 <a href='https://github.com/nafu'>
@@ -90,57 +64,51 @@ If the above doesn't help, please [submit an issue](https://github.com/fastlane/
 </a>
 <h4 align='center'><a href='https://twitter.com/nafu003'>Fumiya Nakamura</a></h4>
 </td>
-<td id='felix-krause'>
-<a href='https://github.com/KrauseFx'>
-<img src='https://github.com/KrauseFx.png?size=140'>
-</a>
-<h4 align='center'><a href='https://twitter.com/KrauseFx'>Felix Krause</a></h4>
-</td>
 </tr>
 <tr>
+<td id='max-ott'>
+<a href='https://github.com/max-ott'>
+<img src='https://github.com/max-ott.png?size=140'>
+</a>
+<h4 align='center'><a href='https://twitter.com/ott_max'>Max Ott</a></h4>
+</td>
 <td id='stefan-natchev'>
 <a href='https://github.com/snatchev'>
 <img src='https://github.com/snatchev.png?size=140'>
 </a>
 <h4 align='center'><a href='https://twitter.com/snatchev'>Stefan Natchev</a></h4>
 </td>
-<td id='matthew-ellis'>
-<a href='https://github.com/matthewellis'>
-<img src='https://github.com/matthewellis.png?size=140'>
-</a>
-<h4 align='center'><a href='https://twitter.com/mellis1995'>Matthew Ellis</a></h4>
-</td>
-<td id='daniel-jankowski'>
-<a href='https://github.com/mollyIV'>
-<img src='https://github.com/mollyIV.png?size=140'>
+<td id='jérôme-lacoste'>
+<a href='https://github.com/lacostej'>
+<img src='https://github.com/lacostej.png?size=140'>
 </a>
-<h4 align='center'><a href='https://twitter.com/mollyIV'>Daniel Jankowski</a></h4>
+<h4 align='center'><a href='https://twitter.com/lacostej'>Jérôme Lacoste</a></h4>
 </td>
-<td id='jimmy-dee'>
-<a href='https://github.com/jdee'>
-<img src='https://github.com/jdee.png?size=140'>
+<td id='joshua-liebowitz'>
+<a href='https://github.com/taquitos'>
+<img src='https://github.com/taquitos.png?size=140'>
 </a>
-<h4 align='center'>Jimmy Dee</h4>
+<h4 align='center'><a href='https://twitter.com/taquitos'>Joshua Liebowitz</a></h4>
 </td>
-<td id='jan-piotrowski'>
-<a href='https://github.com/janpio'>
-<img src='https://github.com/janpio.png?size=140'>
+<td id='iulian-onofrei'>
+<a href='https://github.com/revolter'>
+<img src='https://github.com/revolter.png?size=140'>
 </a>
-<h4 align='center'><a href='https://twitter.com/Sujan'>Jan Piotrowski</a></h4>
+<h4 align='center'><a href='https://twitter.com/Revolt666'>Iulian Onofrei</a></h4>
 </td>
 </tr>
 <tr>
-<td id='luka-mirosevic'>
-<a href='https://github.com/lmirosevic'>
-<img src='https://github.com/lmirosevic.png?size=140'>
+<td id='jimmy-dee'>
+<a href='https://github.com/jdee'>
+<img src='https://github.com/jdee.png?size=140'>
 </a>
-<h4 align='center'><a href='https://twitter.com/lmirosevic'>Luka Mirosevic</a></h4>
+<h4 align='center'>Jimmy Dee</h4>
 </td>
-<td id='maksym-grebenets'>
-<a href='https://github.com/mgrebenets'>
-<img src='https://github.com/mgrebenets.png?size=140'>
+<td id='andrew-mcburney'>
+<a href='https://github.com/armcburney'>
+<img src='https://github.com/armcburney.png?size=140'>
 </a>
-<h4 align='center'><a href='https://twitter.com/mgrebenets'>Maksym Grebenets</a></h4>
+<h4 align='center'><a href='https://twitter.com/armcburney'>Andrew McBurney</a></h4>
 </td>
 <td id='kohki-miki'>
 <a href='https://github.com/giginet'>
@@ -148,31 +116,63 @@ If the above doesn't help, please [submit an issue](https://github.com/fastlane/
 </a>
 <h4 align='center'><a href='https://twitter.com/giginet'>Kohki Miki</a></h4>
 </td>
+<td id='jan-piotrowski'>
+<a href='https://github.com/janpio'>
+<img src='https://github.com/janpio.png?size=140'>
+</a>
+<h4 align='center'><a href='https://twitter.com/Sujan'>Jan Piotrowski</a></h4>
+</td>
+<td id='olivier-halligon'>
+<a href='https://github.com/AliSoftware'>
+<img src='https://github.com/AliSoftware.png?size=140'>
+</a>
+<h4 align='center'><a href='https://twitter.com/aligatr'>Olivier Halligon</a></h4>
+</td>
+</tr>
+<tr>
 <td id='helmut-januschka'>
 <a href='https://github.com/hjanuschka'>
 <img src='https://github.com/hjanuschka.png?size=140'>
 </a>
 <h4 align='center'><a href='https://twitter.com/hjanuschka'>Helmut Januschka</a></h4>
 </td>
+<td id='daniel-jankowski'>
+<a href='https://github.com/mollyIV'>
+<img src='https://github.com/mollyIV.png?size=140'>
+</a>
+<h4 align='center'><a href='https://twitter.com/mollyIV'>Daniel Jankowski</a></h4>
+</td>
+<td id='manu-wallner'>
+<a href='https://github.com/milch'>
+<img src='https://github.com/milch.png?size=140'>
+</a>
+<h4 align='center'><a href='https://twitter.com/acrooow'>Manu Wallner</a></h4>
+</td>
 <td id='aaron-brager'>
 <a href='https://github.com/getaaron'>
 <img src='https://github.com/getaaron.png?size=140'>
 </a>
 <h4 align='center'><a href='https://twitter.com/getaaron'>Aaron Brager</a></h4>
 </td>
+<td id='matthew-ellis'>
+<a href='https://github.com/matthewellis'>
+<img src='https://github.com/matthewellis.png?size=140'>
+</a>
+<h4 align='center'><a href='https://twitter.com/mellis1995'>Matthew Ellis</a></h4>
+</td>
 </tr>
 <tr>
-<td id='iulian-onofrei'>
-<a href='https://github.com/revolter'>
-<img src='https://github.com/revolter.png?size=140'>
+<td id='maksym-grebenets'>
+<a href='https://github.com/mgrebenets'>
+<img src='https://github.com/mgrebenets.png?size=140'>
 </a>
-<h4 align='center'><a href='https://twitter.com/Revolt666'>Iulian Onofrei</a></h4>
+<h4 align='center'><a href='https://twitter.com/mgrebenets'>Maksym Grebenets</a></h4>
 </td>
-<td id='joshua-liebowitz'>
-<a href='https://github.com/taquitos'>
-<img src='https://github.com/taquitos.png?size=140'>
+<td id='danielle-tomlinson'>
+<a href='https://github.com/endocrimes'>
+<img src='https://github.com/endocrimes.png?size=140'>
 </a>
-<h4 align='center'><a href='https://twitter.com/taquitos'>Joshua Liebowitz</a></h4>
+<h4 align='center'><a href='https://twitter.com/endocrimes'>Danielle Tomlinson</a></h4>
 </td>
 </table>
 

data/deliver/lib/deliver/app_screenshot_iterator.rb

@@ -0,0 +1,98 @@
+module Deliver
+  # This is a convinient class that enumerates app store connect's screenshots in various degrees.
+  class AppScreenshotIterator
+    NUMBER_OF_THREADS = Helper.test? ? 1 : [ENV.fetch("DELIVER_NUMBER_OF_THREADS", 10).to_i, 10].min
+
+    # @param localizations [Array<Spaceship::ConnectAPI::AppStoreVersionLocalization>]
+    def initialize(localizations)
+      @localizations = localizations
+    end
+
+    # Iterate app_screenshot_set over localizations
+    #
+    # @yield [localization, app_screenshot_set]
+    # @yieldparam [optional, Spaceship::ConnectAPI::AppStoreVersionLocalization] localization
+    # @yieldparam [optional, Spaceship::ConnectAPI::AppStoreScreenshotSet] app_screenshot_set
+    def each_app_screenshot_set(&block)
+      return enum_for(__method__) unless block_given?
+
+      # Collect app_screenshot_sets from localizations in parallel but
+      # limit the number of threads working at a time with using `lazy` and `force` controls
+      # to not attack App Store Connect
+      results = @localizations.each_slice(NUMBER_OF_THREADS).lazy.map do |localizations|
+        localizations.map do |localization|
+          Thread.new do
+            [localization, localization.get_app_screenshot_sets]
+          end
+        end
+      end.flat_map do |threads|
+        threads.map { |t| t.join.value }
+      end.force
+
+      results.each do |localization, app_screenshot_sets|
+        app_screenshot_sets.each do |app_screenshot_set|
+          yield(localization, app_screenshot_set)
+        end
+      end
+    end
+
+    # Iterate app_screenshot over localizations and app_screenshot_sets
+    #
+    # @yield [localization, app_screenshot_set, app_screenshot]
+    # @yieldparam [optional, Spaceship::ConnectAPI::AppStoreVersionLocalization] localization
+    # @yieldparam [optional, Spaceship::ConnectAPI::AppStoreScreenshotSet] app_screenshot_set
+    # @yieldparam [optional, Spaceship::ConnectAPI::AppStoreScreenshot] app_screenshot
+    def each_app_screenshot(&block)
+      return enum_for(__method__) unless block_given?
+
+      each_app_screenshot_set do |localization, app_screenshot_set|
+        app_screenshot_set.app_screenshots.each do |app_screenshot|
+          yield(localization, app_screenshot_set, app_screenshot)
+        end
+      end
+    end
+
+    # Iterate given local app_screenshot over localizations and app_screenshot_sets with index within each app_screenshot_set
+    #
+    # @param screenshots_per_language [Hash<String, Array<Deliver::AppScreenshot>]
+    # @yield [localization, app_screenshot_set, app_screenshot, index]
+    # @yieldparam [optional, Spaceship::ConnectAPI::AppStoreVersionLocalization] localization
+    # @yieldparam [optional, Spaceship::ConnectAPI::AppStoreScreenshotSet] app_screenshot_set
+    # @yieldparam [optional, Deliver::AppScreenshot] screenshot
+    # @yieldparam [optional, Integer] index a number reperesents which position the screenshot will be
+    def each_local_screenshot(screenshots_per_language, &block)
+      return enum_for(__method__, screenshots_per_language) unless block_given?
+
+      # Iterate over all the screenshots per language and display_type
+      # and then enqueue them to worker one by one if it's not duplciated on App Store Connect
+      screenshots_per_language.map do |language, screenshots_for_language|
+        localization = @localizations.find { |l| l.locale == language }
+        [localization, screenshots_for_language]
+      end.reject do |localization, _|
+        localization.nil?
+      end.each do |localization, screenshots_for_language|
+        iterate_over_screenshots_per_language(localization, screenshots_for_language, &block)
+      end
+    end
+
+    private
+
+    def iterate_over_screenshots_per_language(localization, screenshots_for_language, &block)
+      app_screenshot_sets_per_display_type = localization.get_app_screenshot_sets.map { |set| [set.screenshot_display_type, set] }.to_h
+      screenshots_per_display_type = screenshots_for_language.reject { |screenshot| screenshot.device_type.nil? }.group_by(&:device_type)
+
+      screenshots_per_display_type.each do |display_type, screenshots|
+        # Create AppScreenshotSet for given display_type if it doesn't exsit
+        app_screenshot_set = app_screenshot_sets_per_display_type[display_type]
+        app_screenshot_set ||= localization.create_app_screenshot_set(attributes: { screenshotDisplayType: display_type })
+        iterate_over_screenshots_per_display_type(localization, app_screenshot_set, screenshots, &block)
+      end
+    end
+
+    def iterate_over_screenshots_per_display_type(localization, app_screenshot_set, screenshots, &block)
+      screenshots.each.with_index do |screenshot, index|
+        yield(localization, app_screenshot_set, screenshot, index)
+      end
+    end
+  end
+end

data/deliver/lib/deliver/queue_worker.rb

@@ -0,0 +1,64 @@
+require 'thread'
+
+module Deliver
+  # This dispatches jobs to worker threads and make it work in parallel.
+  # It's suitable for I/O bounds works and not for CPU bounds works.
+  # Use this when you have all the items that you'll process in advance.
+  # Simply enqueue them to this and call `QueueWorker#start`.
+  class QueueWorker
+    # @param concurrency (Numeric) - A number of threads to be created
+    # @param block (Proc) - A task you want to execute with enqueued items
+    def initialize(concurrency, &block)
+      @concurrency = concurrency
+      @block = block
+      @queue = Queue.new
+    end
+
+    # @param job (Object) - An arbitary object that keeps parameters
+    def enqueue(job)
+      @queue.push(job)
+    end
+
+    # Call this after you enqueuned all the jobs you want to process
+    # This method blocks current thread until all the enqueued jobs are processed
+    def start
+      threads = []
+      @concurrency.times do
+        threads << Thread.new do
+          while running? && !empty?
+            job = @queue.pop
+            @block.call(job) if job
+          end
+        end
+      end
+
+      wait_for_complete
+      threads.each(&:join)
+    end
+
+    private
+
+    def running?
+      !@queue.closed?
+    end
+
+    def empty?
+      @queue.empty?
+    end
+
+    def wait_for_complete
+      wait_thread = Thread.new do
+        loop do
+          if @queue.empty?
+            @queue.close
+            break
+          end
+
+          sleep(1)
+        end
+      end
+
+      wait_thread.join
+    end
+  end
+end

data/deliver/lib/deliver/upload_screenshots.rb

@@ -4,10 +4,17 @@ require 'digest/md5'
 require_relative 'app_screenshot'
 require_relative 'module'
 require_relative 'loader'
+require_relative 'queue_worker'
+require_relative 'app_screenshot_iterator'
 
 module Deliver
   # upload screenshots to App Store Connect
   class UploadScreenshots
+    DeleteScreenshotJob = Struct.new(:app_screenshot, :localization, :app_screenshot_set)
+    UploadScreenshotJob = Struct.new(:app_screenshot_set, :path)
+
+    NUMBER_OF_THREADS = Helper.test? ? 1 : [ENV.fetch("DELIVER_NUMBER_OF_THREADS", 10).to_i, 10].min
+
     def upload(options, screenshots)
       return if options[:skip_screenshots]
       return if options[:edit_live]
@@ -50,57 +57,47 @@ module Deliver
         localizations = version.get_app_store_version_localizations
       end
 
-      upload_screenshots(screenshots_per_language, localizations, options)
+      upload_screenshots(localizations, screenshots_per_language)
+
+      Helper.show_loading_indicator("Sorting screenshots uploaded...")
+      sort_screenshots(localizations)
+      Helper.hide_loading_indicator
+
+      UI.success("Successfully uploaded screenshots to App Store Connect")
     end
 
     def delete_screenshots(localizations, screenshots_per_language, tries: 5)
       tries -= 1
 
-      # Get localizations on version
-      localizations.each do |localization|
-        # Only delete screenshots if trying to upload
-        next unless screenshots_per_language.keys.include?(localization.locale)
-
-        # Iterate over all screenshots for each set and delete
-        screenshot_sets = localization.get_app_screenshot_sets
-
-        # Multi threading delete on single localization
-        threads = []
-        errors = []
-
-        screenshot_sets.each do |screenshot_set|
-          UI.message("Removing all previously uploaded screenshots for '#{localization.locale}' '#{screenshot_set.screenshot_display_type}'...")
-          screenshot_set.app_screenshots.each do |screenshot|
-            UI.verbose("Deleting screenshot - #{localization.locale} #{screenshot_set.screenshot_display_type} #{screenshot.id}")
-            threads << Thread.new do
-              begin
-                screenshot.delete!
-                UI.verbose("Deleted screenshot - #{localization.locale} #{screenshot_set.screenshot_display_type} #{screenshot.id}")
-              rescue => error
-                UI.verbose("Failed to delete screenshot - #{localization.locale} #{screenshot_set.screenshot_display_type} #{screenshot.id}")
-                errors << error
-              end
-            end
-          end
+      worker = QueueWorker.new(NUMBER_OF_THREADS) do |job|
+        start_time = Time.now
+        target = "#{job.localization.locale} #{job.app_screenshot_set.screenshot_display_type} #{job.app_screenshot.id}"
+        begin
+          UI.verbose("Deleting '#{target}'")
+          job.app_screenshot.delete!
+          UI.message("Deleted '#{target}' -  (#{Time.now - start_time} secs)")
+        rescue => error
+          UI.error("Failed to delete screenshot #{target} - (#{Time.now - start_time} secs)")
+          UI.error(error.message)
         end
+      end
 
-        sleep(1) # Feels bad but sleeping a bit to let the threads catchup
-
-        unless threads.empty?
-          Helper.show_loading_indicator("Waiting for screenshots to be deleted for '#{localization.locale}'... (might be slow)") unless FastlaneCore::Globals.verbose?
-          threads.each(&:join)
-          Helper.hide_loading_indicator unless FastlaneCore::Globals.verbose?
-        end
+      iterator = AppScreenshotIterator.new(localizations)
+      iterator.each_app_screenshot do |localization, app_screenshot_set, app_screenshot|
+        # Only delete screenshots if trying to upload
+        next unless screenshots_per_language.keys.include?(localization.locale)
 
-        # Crash if any errors happen while deleting
-        errors.each do |error|
-          UI.error(error.message)
-        end
+        UI.verbose("Queued delete sceeenshot job for #{localization.locale} #{app_screenshot_set.screenshot_display_type} #{app_screenshot.id}")
+        worker.enqueue(DeleteScreenshotJob.new(app_screenshot, localization, app_screenshot_set))
       end
 
+      worker.start
+
       # Verify all screenshots have been deleted
       # Sometimes API requests will fail but screenshots will still be deleted
-      count = count_screenshots(localizations)
+      count = iterator.each_app_screenshot_set.map { |_, app_screenshot_set| app_screenshot_set }
+                      .reduce(0) { |sum, app_screenshot_set| sum + app_screenshot_set.app_screenshots.size }
+
       UI.important("Number of screenshots not deleted: #{count}")
       if count > 0
         if tries.zero?
@@ -114,113 +111,108 @@ module Deliver
       end
     end
 
-    def count_screenshots(localizations)
-      count = 0
-      localizations.each do |localization|
-        screenshot_sets = localization.get_app_screenshot_sets
-        screenshot_sets.each do |screenshot_set|
-          count += screenshot_set.app_screenshots.size
-        end
-      end
-
-      return count
-    end
-
-    def upload_screenshots(screenshots_per_language, localizations, options)
-      # Check if should wait for processing
-      # Default to waiting if submitting for review (since needed for submission)
-      # Otherwise use enviroment variable
-      if ENV["DELIVER_SKIP_WAIT_FOR_SCREENSHOT_PROCESSING"].nil?
-        wait_for_processing = options[:submit_for_review]
-        UI.verbose("Setting wait_for_processing from ':submit_for_review' option")
-      else
-        UI.verbose("Setting wait_for_processing from 'DELIVER_SKIP_WAIT_FOR_SCREENSHOT_PROCESSING' environment variable")
-        wait_for_processing = !FastlaneCore::Env.truthy?("DELIVER_SKIP_WAIT_FOR_SCREENSHOT_PROCESSING")
-      end
-
-      if wait_for_processing
-        UI.important("Will wait for screenshot image processing")
-        UI.important("Set env DELIVER_SKIP_WAIT_FOR_SCREENSHOT_PROCESSING=true to skip waiting for screenshots to process")
-      else
-        UI.important("Skipping the wait for screenshot image processing (which may affect submission)")
-        UI.important("Set env DELIVER_SKIP_WAIT_FOR_SCREENSHOT_PROCESSING=false to wait for screenshots to process")
-      end
+    def upload_screenshots(localizations, screenshots_per_language, tries: 5)
+      tries -= 1
 
       # Upload screenshots
-      indized = {} # per language and device type
-
-      screenshots_per_language.each do |language, screenshots_for_language|
-        # Find localization to upload screenshots to
-        localization = localizations.find do |l|
-          l.locale == language
+      worker = QueueWorker.new(NUMBER_OF_THREADS) do |job|
+        begin
+          UI.verbose("Uploading '#{job.path}'...")
+          start_time = Time.now
+          job.app_screenshot_set.upload_screenshot(path: job.path, wait_for_processing: false)
+          UI.message("Uploaded '#{job.path}'... (#{Time.now - start_time} secs)")
+        rescue => error
+          UI.error(error)
         end
+      end
 
-        unless localization
-          UI.error("Couldn't find localization on version for #{language}")
+      number_of_screenshots = 0
+      iterator = AppScreenshotIterator.new(localizations)
+      iterator.each_local_screenshot(screenshots_per_language) do |localization, app_screenshot_set, screenshot, index|
+        if index >= 10
+          UI.error("Too many screenshots found for device '#{screenshot.device_type}' in '#{screenshot.language}', skipping this one (#{screenshot.path})")
           next
         end
 
-        indized[localization.locale] ||= {}
+        checksum = UploadScreenshots.calculate_checksum(screenshot.path)
+        duplicate = (app_screenshot_set.app_screenshots || []).any? { |s| s.source_file_checksum == checksum }
 
-        # Create map to find screenshot set to add screenshot to
-        app_screenshot_sets_map = {}
-        app_screenshot_sets = localization.get_app_screenshot_sets
-        app_screenshot_sets.each do |app_screenshot_set|
-          app_screenshot_sets_map[app_screenshot_set.screenshot_display_type] = app_screenshot_set
+        # Enqueue uploading job if it's not duplicated otherwise screenshot will be skipped
+        if duplicate
+          UI.message("Previous uploaded. Skipping '#{screenshot.path}'...")
+        else
+          worker.enqueue(UploadScreenshotJob.new(app_screenshot_set, screenshot.path))
+        end
 
-          # Set initial screnshot count
-          indized[localization.locale][app_screenshot_set.screenshot_display_type] ||= {
-            count: app_screenshot_set.app_screenshots.size,
-            checksums: []
-          }
+        number_of_screenshots += 1
+      end
 
-          checksums = app_screenshot_set.app_screenshots.map(&:source_file_checksum).uniq
-          indized[localization.locale][app_screenshot_set.screenshot_display_type][:checksums] = checksums
-        end
+      worker.start
 
-        UI.message("Uploading #{screenshots_for_language.length} screenshots for language #{language}")
-        screenshots_for_language.each do |screenshot|
-          display_type = screenshot.device_type
-          set = app_screenshot_sets_map[display_type]
+      UI.verbose('Uploading jobs are completed')
 
-          if display_type.nil?
-            UI.error("Error... Screenshot size #{screenshot.screen_size} not valid for App Store Connect")
-            next
-          end
+      Helper.show_loading_indicator("Waiting for all the screenshots processed...")
+      states = wait_for_complete(iterator)
+      Helper.hide_loading_indicator
+      retry_upload_screenshots_if_needed(iterator, states, number_of_screenshots, tries, localizations, screenshots_per_language)
 
-          unless set
-            set = localization.create_app_screenshot_set(attributes: {
-              screenshotDisplayType: display_type
-            })
-            app_screenshot_sets_map[display_type] = set
+      UI.message("Successfully uploaded all screenshots")
+    end
 
-            indized[localization.locale][set.screenshot_display_type] = {
-              count: 0,
-              checksums: []
-            }
-          end
+    # Verify all screenshots have been processed
+    def wait_for_complete(iterator)
+      loop do
+        states = iterator.each_app_screenshot.map { |_, _, app_screenshot| app_screenshot }.each_with_object({}) do |app_screenshot, hash|
+          state = app_screenshot.asset_delivery_state['state']
+          hash[state] ||= 0
+          hash[state] += 1
+        end
 
-          index = indized[localization.locale][set.screenshot_display_type][:count]
+        is_processing = states.fetch('UPLOAD_COMPLETE', 0) > 0
+        return states unless is_processing
 
-          if index >= 10
-            UI.error("Too many screenshots found for device '#{screenshot.device_type}' in '#{screenshot.language}', skipping this one (#{screenshot.path})")
-            next
-          end
+        UI.verbose("There are still incomplete screenshots - #{states}")
+        sleep(5)
+      end
+    end
 
-          bytes = File.binread(screenshot.path)
-          checksum = Digest::MD5.hexdigest(bytes)
-          duplicate = indized[localization.locale][set.screenshot_display_type][:checksums].include?(checksum)
+    # Verify all screenshots states on App Store Connect are okay
+    def retry_upload_screenshots_if_needed(iterator, states, number_of_screenshots, tries, localizations, screenshots_per_language)
+      is_failure = states.fetch("FAILED", 0) > 0
+      is_missing_screenshot = states.reduce(0) { |sum, (k, v)| sum + v } != number_of_screenshots && !screenshots_per_language.empty?
 
-          if duplicate
-            UI.message("Previous uploaded. Skipping '#{screenshot.path}'...")
-          else
-            indized[localization.locale][set.screenshot_display_type][:count] += 1
-            UI.message("Uploading '#{screenshot.path}'...")
-            set.upload_screenshot(path: screenshot.path, wait_for_processing: wait_for_processing)
+      if is_failure || is_missing_screenshot
+        if tries.zero?
+          incomplete_screenshot_count = states.reject { |k, v| k == 'COMPLETE' }.reduce(0) { |sum, (k, v)| sum + v }
+          UI.user_error!("Failed verification of all screenshots uploaded... #{incomplete_screenshot_count} incomplete screenshot(s) still exist")
+        else
+          UI.error("Failed to upload all screenshots... Tries remaining: #{tries}")
+          # Delete bad entries before retry
+          iterator.each_app_screenshot do |_, _, app_screenshot|
+            app_screenshot.delete! unless app_screenshot.complete?
           end
+          upload_screenshots(localizations, screenshots_per_language, tries: tries)
         end
       end
-      UI.success("Successfully uploaded screenshots to App Store Connect")
+    end
+
+    def sort_screenshots(localizations)
+      iterator = AppScreenshotIterator.new(localizations)
+
+      # Re-order screenshots within app_screenshot_set
+      worker = QueueWorker.new(NUMBER_OF_THREADS) do |app_screenshot_set|
+        original_ids = app_screenshot_set.app_screenshots.map(&:id)
+        sorted_ids = app_screenshot_set.app_screenshots.sort_by(&:file_name).map(&:id)
+        if original_ids != sorted_ids
+          app_screenshot_set.reorder_screenshots(app_screenshot_ids: sorted_ids)
+        end
+      end
+
+      iterator.each_app_screenshot_set do |_, app_screenshot_set|
+        worker.enqueue(app_screenshot_set)
+      end
+
+      worker.start
     end
 
     def collect_screenshots(options)
@@ -299,5 +291,11 @@ module Deliver
         Spaceship::Tunes.client.available_languages
       end
     end
+
+    # helper method to mock this step in tests
+    def self.calculate_checksum(path)
+      bytes = File.binread(path)
+      Digest::MD5.hexdigest(bytes)
+    end
   end
 end

data/fastlane/lib/fastlane/version.rb

@@ -1,5 +1,5 @@
 module Fastlane
-  VERSION = '2.155.0'.freeze
+  VERSION = '2.156.1'.freeze
   DESCRIPTION = "The easiest way to automate beta deployments and releases for your iOS and Android apps".freeze
   MINIMUM_XCODE_RELEASE = "7.0".freeze
   RUBOCOP_REQUIREMENT = '0.49.1'.freeze

data/fastlane/swift/Deliverfile.swift

@@ -14,4 +14,4 @@ class Deliverfile: DeliverfileProtocol {
     // during the `init` process, and you won't see this message
 }
 
-// Generated with fastlane 2.155.0
+// Generated with fastlane 2.156.1

data/fastlane/swift/DeliverfileProtocol.swift

@@ -245,4 +245,4 @@ extension DeliverfileProtocol {
 
 // Please don't remove the lines below
 // They are used to detect outdated files
-// FastlaneRunnerAPIVersion [0.9.29]
+// FastlaneRunnerAPIVersion [0.9.34]

data/fastlane/swift/Fastlane.swift

@@ -9204,4 +9204,4 @@ let snapshotfile: Snapshotfile = Snapshotfile()
 
 // Please don't remove the lines below
 // They are used to detect outdated files
-// FastlaneRunnerAPIVersion [0.9.82]
+// FastlaneRunnerAPIVersion [0.9.87]

data/fastlane/swift/Gymfile.swift

@@ -14,4 +14,4 @@ class Gymfile: GymfileProtocol {
     // during the `init` process, and you won't see this message
 }
 
-// Generated with fastlane 2.155.0
+// Generated with fastlane 2.156.1

data/fastlane/swift/GymfileProtocol.swift

@@ -181,4 +181,4 @@ extension GymfileProtocol {
 
 // Please don't remove the lines below
 // They are used to detect outdated files
-// FastlaneRunnerAPIVersion [0.9.32]
+// FastlaneRunnerAPIVersion [0.9.37]

data/fastlane/swift/Matchfile.swift

@@ -14,4 +14,4 @@ class Matchfile: MatchfileProtocol {
     // during the `init` process, and you won't see this message
 }
 
-// Generated with fastlane 2.155.0
+// Generated with fastlane 2.156.1

data/fastlane/swift/MatchfileProtocol.swift

@@ -165,4 +165,4 @@ extension MatchfileProtocol {
 
 // Please don't remove the lines below
 // They are used to detect outdated files
-// FastlaneRunnerAPIVersion [0.9.26]
+// FastlaneRunnerAPIVersion [0.9.31]

data/fastlane/swift/Precheckfile.swift

@@ -14,4 +14,4 @@ class Precheckfile: PrecheckfileProtocol {
     // during the `init` process, and you won't see this message
 }
 
-// Generated with fastlane 2.155.0
+// Generated with fastlane 2.156.1

data/fastlane/swift/PrecheckfileProtocol.swift

@@ -33,4 +33,4 @@ extension PrecheckfileProtocol {
 
 // Please don't remove the lines below
 // They are used to detect outdated files
-// FastlaneRunnerAPIVersion [0.9.25]
+// FastlaneRunnerAPIVersion [0.9.30]

data/fastlane/swift/Scanfile.swift

@@ -14,4 +14,4 @@ class Scanfile: ScanfileProtocol {
     // during the `init` process, and you won't see this message
 }
 
-// Generated with fastlane 2.155.0
+// Generated with fastlane 2.156.1

data/fastlane/swift/ScanfileProtocol.swift

@@ -257,4 +257,4 @@ extension ScanfileProtocol {
 
 // Please don't remove the lines below
 // They are used to detect outdated files
-// FastlaneRunnerAPIVersion [0.9.37]
+// FastlaneRunnerAPIVersion [0.9.42]

data/fastlane/swift/Screengrabfile.swift

@@ -14,4 +14,4 @@ class Screengrabfile: ScreengrabfileProtocol {
     // during the `init` process, and you won't see this message
 }
 
-// Generated with fastlane 2.155.0
+// Generated with fastlane 2.156.1

data/fastlane/swift/ScreengrabfileProtocol.swift

@@ -93,4 +93,4 @@ extension ScreengrabfileProtocol {
 
 // Please don't remove the lines below
 // They are used to detect outdated files
-// FastlaneRunnerAPIVersion [0.9.27]
+// FastlaneRunnerAPIVersion [0.9.32]

data/fastlane/swift/Snapshotfile.swift

@@ -14,4 +14,4 @@ class Snapshotfile: SnapshotfileProtocol {
     // during the `init` process, and you won't see this message
 }
 
-// Generated with fastlane 2.155.0
+// Generated with fastlane 2.156.1

data/fastlane/swift/SnapshotfileProtocol.swift

@@ -181,4 +181,4 @@ extension SnapshotfileProtocol {
 
 // Please don't remove the lines below
 // They are used to detect outdated files
-// FastlaneRunnerAPIVersion [0.9.21]
+// FastlaneRunnerAPIVersion [0.9.26]

data/fastlane_core/lib/fastlane_core/project.rb

@@ -320,6 +320,7 @@ module FastlaneCore
       proj << "-scheme #{options[:scheme].shellescape}" if options[:scheme]
       proj << "-project #{options[:project].shellescape}" if options[:project]
       proj << "-configuration #{options[:configuration].shellescape}" if options[:configuration]
+      proj << "-derivedDataPath #{options[:derived_data_path].shellescape}" if options[:derived_data_path]
       proj << "-xcconfig #{options[:xcconfig].shellescape}" if options[:xcconfig]
 
       if FastlaneCore::Helper.xcode_at_least?('11.0') && options[:cloned_source_packages_path]

data/gym/lib/gym/generators/build_command_generator.rb

@@ -38,7 +38,6 @@ module Gym
         options << "-toolchain '#{config[:toolchain]}'" if config[:toolchain]
         options << "-destination '#{config[:destination]}'" if config[:destination]
         options << "-archivePath #{archive_path.shellescape}" unless config[:skip_archive]
-        options << "-derivedDataPath '#{config[:derived_data_path]}'" if config[:derived_data_path]
         options << "-resultBundlePath '#{result_bundle_path}'" if config[:result_bundle]
         options << config[:xcargs] if config[:xcargs]
         options << "OTHER_SWIFT_FLAGS=\"-Xfrontend -debug-time-function-bodies\"" if config[:analyze_build_time]

data/scan/lib/scan/test_command_generator.rb

@@ -35,7 +35,9 @@ module Scan
       options << "-sdk '#{config[:sdk]}'" if config[:sdk]
       options << destination # generated in `detect_values`
       options << "-toolchain '#{config[:toolchain]}'" if config[:toolchain]
-      options << "-derivedDataPath '#{config[:derived_data_path]}'" if config[:derived_data_path]
+      if config[:derived_data_path] && !options.include?("-derivedDataPath #{config[:derived_data_path].shellescape}")
+        options << "-derivedDataPath #{config[:derived_data_path].shellescape}"
+      end
       options << "-resultBundlePath '#{result_bundle_path}'" if config[:result_bundle]
       if FastlaneCore::Helper.xcode_at_least?(10)
         options << "-parallel-testing-worker-count #{config[:concurrent_workers]}" if config[:concurrent_workers]

data/screengrab/lib/screengrab/runner.rb

@@ -66,7 +66,7 @@ module Screengrab
         run_adb_command("-s #{device_serial} root", print_all: false, print_command: true)
       end
 
-      clear_device_previous_screenshots(device_serial, device_screenshots_paths)
+      clear_device_previous_screenshots(@config[:app_package_name], device_serial, device_screenshots_paths)
 
       app_apk_path ||= select_app_apk(discovered_apk_paths)
       tests_apk_path ||= select_tests_apk(discovered_apk_paths)
@@ -157,12 +157,12 @@ module Screengrab
       end.flatten
     end
 
-    def clear_device_previous_screenshots(device_serial, device_screenshots_paths)
+    def clear_device_previous_screenshots(app_package_name, device_serial, device_screenshots_paths)
       UI.message('Cleaning screenshots on device')
 
       device_screenshots_paths.each do |device_path|
-        if_device_path_exists(device_serial, device_path) do |path|
-          run_adb_command("-s #{device_serial} shell rm -rf #{path}",
+        if_device_path_exists(app_package_name, device_serial, device_path) do |path|
+          run_adb_command("-s #{device_serial} shell run-as #{app_package_name} rm -rf #{path}",
                           print_all: true,
                           print_command: true)
         end
@@ -296,7 +296,7 @@ module Screengrab
 
       Dir.mktmpdir do |tempdir|
         device_screenshots_paths.each do |device_path|
-          if_device_path_exists(device_serial, device_path) do |path|
+          if_device_path_exists(@config[:app_package_name], device_serial, device_path) do |path|
             next unless path.include?(locale)
             run_adb_command("-s #{device_serial} pull #{path} #{tempdir}",
                             print_all: false,
@@ -361,8 +361,8 @@ module Screengrab
 
     # Some device commands fail if executed against a device path that does not exist, so this helper method
     # provides a way to conditionally execute a block only if the provided path exists on the device.
-    def if_device_path_exists(device_serial, device_path)
-      return if run_adb_command("-s #{device_serial} shell ls #{device_path}",
+    def if_device_path_exists(app_package_name, device_serial, device_path)
+      return if run_adb_command("-s #{device_serial} shell run-as #{app_package_name} ls #{device_path}",
                                 print_all: false,
                                 print_command: false).include?('No such file')
 

data/sigh/lib/sigh/runner.rb

@@ -171,7 +171,9 @@ module Sigh
         name: name,
         profile_type: profile_type,
         bundle_id_id: bundle_id.id,
-        certificate_ids: [certificate_to_use.id]
+        certificate_ids: certificates_to_use.map(&:id),
+        device_ids: devices_to_use.map(&:id),
+        template_name: Sigh.config[:template_name]
       )
 
       profile
@@ -226,8 +228,30 @@ module Sigh
       certificates
     end
 
+    def devices_to_use
+      # Only use devices if development or adhoc
+      return [] if !Sigh.config[:development] && !Sigh.config[:adhoc]
+
+      device_class = case Sigh.config[:platform].to_s
+                     when 'ios'
+                       [
+                         Spaceship::ConnectAPI::Device::DeviceClass::APPLE_WATCH,
+                         Spaceship::ConnectAPI::Device::DeviceClass::IPAD,
+                         Spaceship::ConnectAPI::Device::DeviceClass::IPHONE,
+                         Spaceship::ConnectAPI::Device::DeviceClass::IPOD
+                       ].join(",")
+                     when 'tvos'
+                       Spaceship::ConnectAPI::Device::DeviceClass::APPLE_TV
+                     when 'macos', 'catalyst'
+                       Spaceship::ConnectAPI::Device::DeviceClass::MAC
+                     end
+
+      filter = { deviceClass: device_class }
+      return Spaceship::ConnectAPI::Device.all(filter: filter)
+    end
+
     # Certificate to use based on the current distribution mode
-    def certificate_to_use
+    def certificates_to_use
       certificates = certificates_for_profile_and_platform
 
       # Filter them
@@ -277,7 +301,7 @@ module Sigh
       end
 
       return certificates if Sigh.config[:development] # development profiles support multiple certificates
-      return certificates.first
+      return [certificates.first]
     end
 
     # Downloads and stores the provisioning profile

data/snapshot/lib/snapshot/test_command_generator_base.rb

@@ -24,7 +24,9 @@ module Snapshot
         options = []
         options += project_path_array
         options << "-sdk '#{config[:sdk]}'" if config[:sdk]
-        options << "-derivedDataPath '#{derived_data_path}'"
+        if derived_data_path && !options.include?("-derivedDataPath #{derived_data_path.shellescape}")
+          options << "-derivedDataPath #{derived_data_path.shellescape}"
+        end
         options << "-resultBundlePath '#{result_bundle_path}'" if result_bundle_path
         if FastlaneCore::Helper.xcode_at_least?(11)
           options << "-testPlan '#{config[:testplan]}'" if config[:testplan]

data/spaceship/lib/spaceship/connect_api/models/profile.rb

@@ -69,14 +69,15 @@ module Spaceship
         return resps.flat_map(&:to_models)
       end
 
-      def self.create(name: nil, profile_type: nil, bundle_id_id: nil, certificate_ids: nil, device_ids: nil)
+      def self.create(name: nil, profile_type: nil, bundle_id_id: nil, certificate_ids: nil, device_ids: nil, template_name: nil)
         resp = Spaceship::ConnectAPI.post_profiles(
           bundle_id_id: bundle_id_id,
           certificates: certificate_ids,
           devices: device_ids,
           attributes: {
             name: name,
-            profileType: profile_type
+            profileType: profile_type,
+            templateName: template_name
           }
         )
         return resp.to_models.first

data/spaceship/lib/spaceship/connect_api/provisioning/provisioning.rb

@@ -65,10 +65,10 @@ module Spaceship
                 end
               },
               devices: {
-                data: (devices || []).map do |certificate|
+                data: (devices || []).map do |device|
                   {
                     type: "devices",
-                    id: devices
+                    id: device
                   }
                 end
               }

metadata

@@ -1,35 +1,35 @@
 --- !ruby/object:Gem::Specification
 name: fastlane
 version: !ruby/object:Gem::Version
-  version: 2.155.0
+  version: 2.156.1
 platform: ruby
 authors:
+- Aaron Brager
+- Luka Mirosevic
+- Maksym Grebenets
+- Jorge Revuelta H
 - Manu Wallner
-- Helmut Januschka
-- Jérôme Lacoste
-- Fumiya Nakamura
-- Felix Krause
-- Jimmy Dee
+- Stefan Natchev
+- Andrew McBurney
+- Matthew Ellis
+- Josh Holtz
+- Max Ott
 - Iulian Onofrei
-- Danielle Tomlinson
-- Maksym Grebenets
-- Jan Piotrowski
-- Luka Mirosevic
 - Joshua Liebowitz
+- Jimmy Dee
 - Daniel Jankowski
-- Max Ott
-- Olivier Halligon
-- Josh Holtz
-- Matthew Ellis
-- Stefan Natchev
-- Jorge Revuelta H
+- Jan Piotrowski
 - Kohki Miki
-- Aaron Brager
-- Andrew McBurney
+- Jérôme Lacoste
+- Fumiya Nakamura
+- Olivier Halligon
+- Felix Krause
+- Danielle Tomlinson
+- Helmut Januschka
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-08-06 00:00:00.000000000 Z
+date: 2020-08-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: slack-notifier
@@ -944,6 +944,7 @@ files:
 - deliver/lib/assets/summary.html.erb
 - deliver/lib/deliver.rb
 - deliver/lib/deliver/app_screenshot.rb
+- deliver/lib/deliver/app_screenshot_iterator.rb
 - deliver/lib/deliver/commands_generator.rb
 - deliver/lib/deliver/detect_values.rb
 - deliver/lib/deliver/download_screenshots.rb
@@ -952,6 +953,7 @@ files:
 - deliver/lib/deliver/loader.rb
 - deliver/lib/deliver/module.rb
 - deliver/lib/deliver/options.rb
+- deliver/lib/deliver/queue_worker.rb
 - deliver/lib/deliver/runner.rb
 - deliver/lib/deliver/setup.rb
 - deliver/lib/deliver/submit_for_review.rb
@@ -1608,6 +1610,7 @@ files:
 - spaceship/lib/spaceship/client.rb
 - spaceship/lib/spaceship/commands_generator.rb
 - spaceship/lib/spaceship/connect_api.rb
+- spaceship/lib/spaceship/connect_api/.client.rb.swp
 - spaceship/lib/spaceship/connect_api/client.rb
 - spaceship/lib/spaceship/connect_api/file_uploader.rb
 - spaceship/lib/spaceship/connect_api/model.rb