shopify-cli 2.29.0 → 2.30.0

data/lib/shopify_cli/theme/syncer.rb

@@ -5,17 +5,18 @@ require "json"
 require "base64"
 require "forwardable"
 
+require_relative "backoff_helper"
+require_relative "ignore_helper"
+require_relative "theme_admin_api"
+
 require_relative "syncer/checksums"
+require_relative "syncer/downloader"
 require_relative "syncer/error_reporter"
-require_relative "syncer/json_delete_handler"
-require_relative "syncer/json_update_handler"
 require_relative "syncer/merger"
 require_relative "syncer/operation"
 require_relative "syncer/standard_reporter"
 require_relative "syncer/unsupported_script_warning"
-require_relative "theme_admin_api"
-require_relative "ignore_helper"
-require_relative "theme_admin_api_throttler"
+require_relative "syncer/uploader"
 
 module ShopifyCLI
   module Theme
@@ -23,8 +24,7 @@ module ShopifyCLI
       extend Forwardable
 
       include ShopifyCLI::Theme::IgnoreHelper
-      include JsonDeleteHandler
-      include JsonUpdateHandler
+      include ShopifyCLI::Theme::BackoffHelper
 
       QUEUEABLE_METHODS = [
         :get,         # - Updates the local file with the remote file content
@@ -33,7 +33,7 @@ module ShopifyCLI
         :union_merge, # - Union merges the local file content with the remote file content
       ]
 
-      attr_reader :theme, :checksums, :error_checksums, :api_client
+      attr_reader :ctx, :theme, :checksums, :error_checksums, :api_client, :pending
       attr_accessor :include_filter, :ignore_filter
 
       def_delegators :@error_reporter, :has_any_error?
@@ -57,12 +57,6 @@ module ShopifyCLI
         # Thread making the API requests.
         @threads = []
 
-        # Mutex used to pause all threads when backing-off when hitting API rate limits
-        @backoff_mutex = Mutex.new
-
-        # Mutex used to coordinate changes in the `pending` list
-        @pending_mutex = Mutex.new
-
         # Latest theme assets checksums. Updated on each upload.
         @checksums = Checksums.new(theme)
 
@@ -71,14 +65,14 @@ module ShopifyCLI
 
         # Do not use the throttler when --stable is passed or a Theme Access password is set
         # (Theme Access API is not compatible yet with bulks)
-        throttler_enabled = !stable && !Environment.theme_access_password?
+        @bulk_updates_activated = !stable && !Environment.theme_access_password?
 
         # Initialize `api_client` on main thread
-        @api_client = ThemeAdminAPIThrottler.new(
-          @ctx,
-          ThemeAdminAPI.new(@ctx, @theme.shop),
-          throttler_enabled
-        )
+        @api_client = ThemeAdminAPI.new(ctx, theme.shop)
+
+        # Initialize backoff helper on main thread to pause all threads when the
+        # requests are reaching API rate limits.
+        initialize_backoff_helper!
       end
 
       def lock_io!
@@ -146,7 +140,6 @@ module ShopifyCLI
       end
 
       def shutdown
-        api_client.shutdown
         @queue.close unless @queue.closed?
       ensure
         @threads.each { |thread| thread.join if thread.alive? }
@@ -169,64 +162,70 @@ module ShopifyCLI
       end
 
       def upload_theme!(delay_low_priority_files: false, delete: true, &block)
-        fetch_checksums!
-
-        if delete
-          removed_json_files, removed_files = checksums
-            .keys
-            .-(@theme.theme_files.map(&:relative_path))
-            .map { |file| @theme[file] }
-            .partition(&:json?)
+        uploader = Uploader.new(self, delete, delay_low_priority_files, &block)
+        uploader.upload!
+      end
 
-          enqueue_deletes(removed_files)
-          enqueue_json_deletes(removed_json_files)
-        end
+      def download_theme!(delete: true, &block)
+        downloader = Downloader.new(self, delete, &block)
+        downloader.download!
+      end
 
-        enqueue_updates(@theme.liquid_files)
-        enqueue_json_updates(@theme.json_files)
+      def bulk_updates_activated?
+        @bulk_updates_activated
+      end
 
-        if delay_low_priority_files
-          # Wait for liquid & JSON files to upload, because those are rendered remotely
-          wait!(&block)
-        end
+      def enqueueable?(operation)
+        file = operation.file
+        method = operation.method
 
-        # Process lower-priority files in the background
+        # Already enqueued or ignored
+        return false if @pending.include?(operation) || ignore_operation?(operation)
 
-        # Assets are served locally, so can be uploaded in the background
-        enqueue_updates(@theme.static_asset_files)
+        if [:update, :get].include?(method) && file.exist?
+          # File is fixed (and it has been never updated)
+          if !!@error_checksums.delete(file.checksum)
+            @standard_reporter.report(operation.as_fix_message)
+          end
 
-        unless delay_low_priority_files
-          wait!(&block)
+          return checksums.file_has_changed?(file)
         end
 
-        api_client.deactivate_throttler!
-        enqueue_delayed_files_updates
+        true
       end
 
-      def download_theme!(delete: true, &block)
-        fetch_checksums!
-
-        if delete
-          # Delete local files not present remotely
-          missing_files = @theme.theme_files
-            .reject { |file| checksums.has?(file) }.uniq
-            .reject { |file| ignore_file?(file) }
-          missing_files.each do |file|
-            @ctx.debug("rm #{file.relative_path}")
-            file.delete
-          end
+      def handle_operation_error(operation, error)
+        error_suffix = ":\n  " + parse_api_errors(operation, error).join("\n  ")
+        report_error(operation, error_suffix)
+      end
+
+      def overwrite_json?
+        theme_created_at_runtime? || @overwrite_json
+      end
+
+      def update_checksums(api_response)
+        api_response.values.flatten.each do |asset|
+          next unless asset["key"]
+          checksums[asset["key"]] = asset["checksum"]
         end
 
-        enqueue_get(checksums.keys)
+        checksums.reject_duplicated_checksums!
+      end
+
+      def report_file_error(file, error_message = "")
+        path = file.relative_path
 
-        wait!(&block)
+        @error_checksums << checksums[path]
+        @error_reporter.report(error_message)
       end
 
       private
 
       def report_error(operation, error_suffix = "")
-        @error_checksums << checksums[operation.file_path]
-        @error_reporter.report("#{operation.as_error_message}#{error_suffix}")
+        file = operation.file
+        error_message = "#{operation.as_error_message}#{error_suffix}"
+
+        report_file_error(file, error_message)
       end
 
       def enqueue(method, file)
@@ -235,22 +234,24 @@ module ShopifyCLI
 
         operation = Operation.new(@ctx, method, @theme[file])
 
-        # Already enqueued
-        return if @pending.include?(operation)
+        return unless enqueueable?(operation)
 
-        if ignore_operation?(operation)
-          @ctx.debug("ignore #{operation.file_path}")
-          return
-        end
+        @pending << operation
+        @queue << operation unless @queue.closed?
+      end
 
-        if [:update, :get].include?(method) && operation.file.exist?
-          is_fixed = !!@error_checksums.delete(operation.file.checksum)
-          @standard_reporter.report(operation.as_fix_message) if is_fixed
-          return unless checksums.file_has_changed?(operation.file)
+      def report_performed_operation(operation)
+        file = operation.file
+
+        if file.warnings.any?
+          warning_message =
+            operation.as_synced_message(color: :yellow) +
+            UnsupportedScriptWarning.new(@ctx, file).to_s
+
+          return @error_reporter.report(warning_message)
         end
 
-        @pending << operation
-        @queue << operation unless @queue.closed?
+        @standard_reporter.report(operation.as_synced_message)
       end
 
       def perform(operation)
@@ -258,36 +259,15 @@ module ShopifyCLI
         wait_for_backoff!
         @ctx.debug(operation.to_s)
 
-        send(operation.method, operation.file) do |response|
-          raise response if response.is_a?(StandardError)
-
-          file = operation.file
+        response = send(operation.method, operation.file)
 
-          if file.warnings.any?
-            warning_message =
-              operation.as_synced_message(color: :yellow) +
-              UnsupportedScriptWarning.new(@ctx, file).to_s
-            @error_reporter.report(warning_message)
-          else
-            @standard_reporter.report(operation.as_synced_message)
-          end
+        report_performed_operation(operation)
+        backoff_if_near_limit!(response)
 
-          # Check if the API told us we're near the rate limit
-          if !backingoff? && (limit = response["x-shopify-shop-api-call-limit"])
-            used, total = limit.split("/").map(&:to_i)
-            backoff_if_near_limit!(used, total)
-          end
-        rescue StandardError => error
-          handle_operation_error(operation, error)
-        ensure
-          @pending_mutex.synchronize do
-            # Avoid abrupt jumps in the progress bar
-            wait(0.05)
-            @pending.delete(operation)
-          end
-        end
       rescue StandardError => error
         handle_operation_error(operation, error)
+      ensure
+        @pending.delete(operation)
       end
 
       def update(file)
@@ -300,15 +280,16 @@ module ShopifyCLI
         end
 
         path = "themes/#{@theme.id}/assets.json"
-        req_body = JSON.generate(asset: asset)
 
-        api_client.put(path: path, body: req_body) do |_status, resp_body, response|
-          update_checksums(resp_body)
+        _status, body, response = api_client.put(
+          path: path,
+          body: JSON.generate(asset: asset)
+        )
+        file.warnings = body.dig("asset", "warnings")
 
-          file.warnings = resp_body.dig("asset", "warnings")
+        update_checksums(body)
 
-          yield(response) if block_given?
-        end
+        response
       end
 
       def get(file)
@@ -326,7 +307,7 @@ module ShopifyCLI
           file.write(body.dig("asset", "value"))
         end
 
-        yield(response)
+        response
       end
 
       def delete(file)
@@ -337,7 +318,7 @@ module ShopifyCLI
           })
         )
 
-        yield(response)
+        response
       end
 
       def union_merge(file)
@@ -346,11 +327,11 @@ module ShopifyCLI
           query: URI.encode_www_form("asset[key]" => file.relative_path),
         )
 
-        return yield(response) unless file.text?
+        return response unless file.text?
 
         remote_content = body.dig("asset", "value")
 
-        return yield(response) if remote_content.nil?
+        return response if remote_content.nil?
 
         content = Merger.union_merge(file, remote_content)
 
@@ -358,16 +339,7 @@ module ShopifyCLI
 
         enqueue(:update, file)
 
-        yield(response)
-      end
-
-      def update_checksums(api_response)
-        api_response.values.flatten.each do |asset|
-          next unless asset["key"]
-          checksums[asset["key"]] = asset["checksum"]
-        end
-
-        checksums.reject_duplicated_checksums!
+        response
       end
 
       def parse_api_errors(operation, exception)
@@ -397,35 +369,10 @@ module ShopifyCLI
         ["The asset #{operation.file} could not be synced #{cause} #{backtrace}"]
       end
 
-      def backoff_if_near_limit!(used, limit)
-        if used > limit - @threads.size
-          @ctx.debug("Near API call limit, waiting 2 sec…")
-          @backoff_mutex.synchronize { wait(2) }
-        end
-      end
-
-      def overwrite_json?
-        theme_created_at_runtime? || @overwrite_json
-      end
-
       def theme_created_at_runtime?
         @theme.created_at_runtime?
       end
 
-      def backingoff?
-        @backoff_mutex.locked?
-      end
-
-      def wait_for_backoff!
-        # Sleeping in the mutex in another thread. Wait for unlock
-        @backoff_mutex.synchronize {} if backingoff?
-      end
-
-      def handle_operation_error(operation, error)
-        error_suffix = ":\n  " + parse_api_errors(operation, error).join("\n  ")
-        report_error(operation, error_suffix)
-      end
-
       def wait(duration)
         sleep(duration)
       end

data/lib/shopify_cli/version.rb

@@ -1,3 +1,3 @@
 module ShopifyCLI
-  VERSION = "2.29.0"
+  VERSION = "2.30.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: shopify-cli
 version: !ruby/object:Gem::Version
-  version: 2.29.0
+  version: 2.30.0
 platform: ruby
 authors:
 - Shopify
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-10-19 00:00:00.000000000 Z
+date: 2022-11-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -456,6 +456,7 @@ files:
 - lib/shopify_cli/tasks/ensure_project_type.rb
 - lib/shopify_cli/tasks/select_org_and_shop.rb
 - lib/shopify_cli/tasks/update_dashboard_urls.rb
+- lib/shopify_cli/theme/backoff_helper.rb
 - lib/shopify_cli/theme/dev_server.rb
 - lib/shopify_cli/theme/dev_server/cdn_fonts.rb
 - lib/shopify_cli/theme/dev_server/certificate_manager.rb
@@ -504,28 +505,27 @@ files:
 - lib/shopify_cli/theme/root.rb
 - lib/shopify_cli/theme/syncer.rb
 - lib/shopify_cli/theme/syncer/checksums.rb
+- lib/shopify_cli/theme/syncer/downloader.rb
 - lib/shopify_cli/theme/syncer/error_reporter.rb
-- lib/shopify_cli/theme/syncer/forms/apply_to_all.rb
-- lib/shopify_cli/theme/syncer/forms/apply_to_all_form.rb
-- lib/shopify_cli/theme/syncer/forms/base_strategy_form.rb
-- lib/shopify_cli/theme/syncer/forms/select_delete_strategy.rb
-- lib/shopify_cli/theme/syncer/forms/select_update_strategy.rb
-- lib/shopify_cli/theme/syncer/json_delete_handler.rb
-- lib/shopify_cli/theme/syncer/json_update_handler.rb
 - lib/shopify_cli/theme/syncer/merger.rb
 - lib/shopify_cli/theme/syncer/operation.rb
 - lib/shopify_cli/theme/syncer/standard_reporter.rb
 - lib/shopify_cli/theme/syncer/unsupported_script_warning.rb
+- lib/shopify_cli/theme/syncer/uploader.rb
+- lib/shopify_cli/theme/syncer/uploader/bulk.rb
+- lib/shopify_cli/theme/syncer/uploader/bulk_item.rb
+- lib/shopify_cli/theme/syncer/uploader/bulk_job.rb
+- lib/shopify_cli/theme/syncer/uploader/bulk_request.rb
+- lib/shopify_cli/theme/syncer/uploader/forms/apply_to_all.rb
+- lib/shopify_cli/theme/syncer/uploader/forms/apply_to_all_form.rb
+- lib/shopify_cli/theme/syncer/uploader/forms/base_strategy_form.rb
+- lib/shopify_cli/theme/syncer/uploader/forms/select_delete_strategy.rb
+- lib/shopify_cli/theme/syncer/uploader/forms/select_update_strategy.rb
+- lib/shopify_cli/theme/syncer/uploader/json_delete_handler.rb
+- lib/shopify_cli/theme/syncer/uploader/json_update_handler.rb
 - lib/shopify_cli/theme/theme.rb
 - lib/shopify_cli/theme/theme_access_api.rb
 - lib/shopify_cli/theme/theme_admin_api.rb
-- lib/shopify_cli/theme/theme_admin_api_throttler.rb
-- lib/shopify_cli/theme/theme_admin_api_throttler/bulk.rb
-- lib/shopify_cli/theme/theme_admin_api_throttler/bulk_job.rb
-- lib/shopify_cli/theme/theme_admin_api_throttler/errors.rb
-- lib/shopify_cli/theme/theme_admin_api_throttler/put_request.rb
-- lib/shopify_cli/theme/theme_admin_api_throttler/request_parser.rb
-- lib/shopify_cli/theme/theme_admin_api_throttler/response_parser.rb
 - lib/shopify_cli/thread_pool.rb
 - lib/shopify_cli/thread_pool/job.rb
 - lib/shopify_cli/transform_data_structure.rb

data/lib/shopify_cli/theme/syncer/forms/apply_to_all.rb

@@ -1,39 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "apply_to_all_form"
-
-module ShopifyCLI
-  module Theme
-    class Syncer
-      module Forms
-        class ApplyToAll
-          attr_reader :value
-
-          def initialize(ctx, number_of_files)
-            @ctx = ctx
-            @number_of_files = number_of_files
-            @value = nil
-            @apply = nil
-          end
-
-          def apply?(value)
-            return unless @number_of_files > 1
-
-            if @apply.nil?
-              @apply = ask.apply?
-              @value = value if @apply
-            end
-
-            @apply
-          end
-
-          private
-
-          def ask
-            ApplyToAllForm.ask(@ctx, [], number_of_files: @number_of_files)
-          end
-        end
-      end
-    end
-  end
-end

data/lib/shopify_cli/theme/syncer/forms/apply_to_all_form.rb

@@ -1,35 +0,0 @@
-# frozen_string_literal: true
-
-module ShopifyCLI
-  module Theme
-    class Syncer
-      module Forms
-        class ApplyToAllForm < ShopifyCLI::Form
-          attr_accessor :apply
-          flag_arguments :number_of_files
-
-          def ask
-            title = message("title", number_of_files - 1)
-
-            self.apply = CLI::UI::Prompt.ask(title, allow_empty: false) do |handler|
-              handler.option(message("yes")) { true }
-              handler.option(message("no")) { false }
-            end
-
-            self
-          end
-
-          def apply?
-            apply
-          end
-
-          private
-
-          def message(key, *params)
-            ctx.message("theme.serve.syncer.forms.apply_to_all.#{key}", *params)
-          end
-        end
-      end
-    end
-  end
-end

data/lib/shopify_cli/theme/syncer/forms/base_strategy_form.rb

@@ -1,62 +0,0 @@
-# frozen_string_literal: true
-
-module ShopifyCLI
-  module Theme
-    class Syncer
-      module Forms
-        class BaseStrategyForm < ShopifyCLI::Form
-          attr_accessor :strategy
-
-          def ask
-            ctx.puts(title_context(file))
-
-            self.strategy = CLI::UI::Prompt.ask(title_question, allow_empty: false) do |handler|
-              strategies.each do |strategy|
-                handler.option(as_text(strategy)) { strategy }
-              end
-            end
-
-            exit_cli if self.strategy == :exit
-
-            self
-          end
-
-          protected
-
-          ##
-          # List of strategies that populate the form options
-          #
-          def strategies
-            raise "`#{self.class.name}#strategies' must be defined"
-          end
-
-          ##
-          # Message prefix for the form title and options (strategies).
-          # See the methods `title` and `as_text`
-          #
-          def prefix
-            raise "`#{self.class.name}#prefix' must be defined"
-          end
-
-          private
-
-          def exit_cli
-            exit(0)
-          end
-
-          def title_context(file)
-            ctx.message("#{prefix}.title_context", file.relative_path)
-          end
-
-          def title_question
-            ctx.message("#{prefix}.title_question")
-          end
-
-          def as_text(strategy)
-            ctx.message("#{prefix}.#{strategy}")
-          end
-        end
-      end
-    end
-  end
-end

data/lib/shopify_cli/theme/syncer/forms/select_delete_strategy.rb

@@ -1,27 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "base_strategy_form"
-
-module ShopifyCLI
-  module Theme
-    class Syncer
-      module Forms
-        class SelectDeleteStrategy < BaseStrategyForm
-          flag_arguments :file
-
-          def strategies
-            %i[
-              delete
-              restore
-              exit
-            ]
-          end
-
-          def prefix
-            "theme.serve.syncer.forms.delete_strategy"
-          end
-        end
-      end
-    end
-  end
-end

data/lib/shopify_cli/theme/syncer/forms/select_update_strategy.rb

@@ -1,28 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "base_strategy_form"
-
-module ShopifyCLI
-  module Theme
-    class Syncer
-      module Forms
-        class SelectUpdateStrategy < BaseStrategyForm
-          flag_arguments :file, :exists_remotely
-
-          def strategies
-            %i[
-              keep_remote
-              keep_local
-              union_merge
-              exit
-            ]
-          end
-
-          def prefix
-            "theme.serve.syncer.forms.#{exists_remotely ? "update_strategy" : "update_remote_deleted_strategy"}"
-          end
-        end
-      end
-    end
-  end
-end