shopify-cli 2.19.0 → 2.20.0

data/lib/shopify_cli/theme/syncer/unsupported_script_warning.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module ShopifyCLI
+  module Theme
+    class Syncer
+      class UnsupportedScriptWarning
+        attr_reader :ctx
+
+        def initialize(ctx, file)
+          @ctx = ctx
+          @file = file
+        end
+
+        def to_s
+          "\n\n#{occurrences} #{long_text}"
+        end
+
+        private
+
+        def occurrences
+          warnings.map { |w| occurrence(w) }.join("\n")
+        end
+
+        def occurrence(warning)
+          line_number = "{{blue: #{warning.line} |}}"
+          pointer = pointer_message(warning)
+
+          <<~OCCURRENCE
+            #{line_number} #{warning.line_content}
+            #{pointer}
+          OCCURRENCE
+        end
+
+        def long_text
+          lines_and_columns = warnings.map do |warning|
+            message("line_and_column", warning.line, warning.column)
+          end
+
+          message("unsupported_script_text", lines_and_columns.join)
+            .split("\n")
+            .reduce("") do |text, line|
+              # Add indentation in the long text to improve readability
+              line = " #{line}"
+
+              # Inline yellow (otherwise `CLI::UI::Frame` breaks multiline formatting)
+              line = "{{yellow:#{line}}}"
+
+              "#{text}#{line}\n"
+            end
+        end
+
+        def pointer_message(warning)
+          padding = warning.column + warning.line.to_s.size + 2
+          text = message("unsupported_script")
+
+          "{{yellow:#{" " * padding} ^ {{bold:#{text}}}}}"
+        end
+
+        def message(*args)
+          key = args.shift
+          @ctx.message("theme.serve.syncer.warnings.#{key}", *args)
+        end
+
+        def warnings
+          @warnings ||= @file.warnings.map { |w| Warning.new(@file, w) }
+        end
+
+        class Warning
+          attr_reader :line, :column
+
+          def initialize(file, warning_hash)
+            @file = file
+            @line = warning_hash["line"].to_i
+            @column = warning_hash["column"].to_i
+          end
+
+          def line_content
+            file_lines[line - 1]
+          end
+
+          private
+
+          def file_lines
+            @file_lines ||= @file.read.split("\n")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/shopify_cli/theme/syncer.rb

@@ -13,7 +13,9 @@ require_relative "syncer/json_update_handler"
 require_relative "syncer/merger"
 require_relative "syncer/operation"
 require_relative "syncer/standard_reporter"
+require_relative "syncer/unsupported_script_warning"
 require_relative "theme_admin_api"
+require_relative "theme_admin_api_throttler"
 
 module ShopifyCLI
   module Theme
@@ -31,12 +33,12 @@ module ShopifyCLI
         :union_merge, # - Union merges the local file content with the remote file content
       ]
 
-      attr_reader :theme, :checksums, :error_checksums
+      attr_reader :theme, :checksums, :error_checksums, :api_client
       attr_accessor :include_filter, :ignore_filter
 
       def_delegators :@error_reporter, :has_any_error?
 
-      def initialize(ctx, theme:, include_filter: nil, ignore_filter: nil, overwrite_json: true)
+      def initialize(ctx, theme:, include_filter: nil, ignore_filter: nil, overwrite_json: true, stable: false)
         @ctx = ctx
         @theme = theme
         @include_filter = include_filter
@@ -58,15 +60,21 @@ module ShopifyCLI
         # Mutex used to pause all threads when backing-off when hitting API rate limits
         @backoff_mutex = Mutex.new
 
+        # Mutex used to coordinate changes in the `pending` list
+        @pending_mutex = Mutex.new
+
         # Latest theme assets checksums. Updated on each upload.
         @checksums = Checksums.new(theme)
 
         # Checksums of assets with errors.
         @error_checksums = []
-      end
 
-      def api_client
-        @api_client ||= ThemeAdminAPI.new(@ctx, @theme.shop)
+        # Initialize `api_client` on main thread
+        @api_client = ThemeAdminAPIThrottler.new(
+          @ctx,
+          ThemeAdminAPI.new(@ctx, @theme.shop),
+          !stable
+        )
       end
 
       def lock_io!
@@ -134,6 +142,7 @@ module ShopifyCLI
       end
 
       def shutdown
+        api_client.shutdown
         @queue.close unless @queue.closed?
       ensure
         @threads.each { |thread| thread.join if thread.alive? }
@@ -185,6 +194,9 @@ module ShopifyCLI
         unless delay_low_priority_files
           wait!(&block)
         end
+
+        api_client.deactivate_throttler!
+        enqueue_delayed_files_updates
       end
 
       def download_theme!(delete: true, &block)
@@ -242,38 +254,57 @@ module ShopifyCLI
         wait_for_backoff!
         @ctx.debug(operation.to_s)
 
-        response = send(operation.method, operation.file)
+        send(operation.method, operation.file) do |response|
+          raise response if response.is_a?(StandardError)
+
+          file = operation.file
 
-        @standard_reporter.report(operation.as_synced_message)
+          if file.warnings.any?
+            warning_message =
+              operation.as_synced_message(color: :yellow) +
+              UnsupportedScriptWarning.new(@ctx, file).to_s
+            @error_reporter.report(warning_message)
+          else
+            @standard_reporter.report(operation.as_synced_message)
+          end
 
-        # Check if the API told us we're near the rate limit
-        if !backingoff? && (limit = response["x-shopify-shop-api-call-limit"])
-          used, total = limit.split("/").map(&:to_i)
-          backoff_if_near_limit!(used, total)
+          # Check if the API told us we're near the rate limit
+          if !backingoff? && (limit = response["x-shopify-shop-api-call-limit"])
+            used, total = limit.split("/").map(&:to_i)
+            backoff_if_near_limit!(used, total)
+          end
+        rescue StandardError => error
+          handle_operation_error(operation, error)
+        ensure
+          @pending_mutex.synchronize do
+            # Avoid abrupt jumps in the progress bar
+            wait(0.05)
+            @pending.delete(operation)
+          end
         end
-      rescue ShopifyCLI::API::APIRequestError => e
-        error_suffix = ":\n  " + parse_api_errors(e).join("\n  ")
-        report_error(operation, error_suffix)
-      ensure
-        @pending.delete(operation)
+      rescue StandardError => error
+        handle_operation_error(operation, error)
       end
 
       def update(file)
         asset = { key: file.relative_path }
+
         if file.text?
           asset[:value] = file.read
         else
           asset[:attachment] = Base64.encode64(file.read)
         end
 
-        _status, body, response = api_client.put(
-          path: "themes/#{@theme.id}/assets.json",
-          body: JSON.generate(asset: asset)
-        )
+        path = "themes/#{@theme.id}/assets.json"
+        req_body = JSON.generate(asset: asset)
 
-        update_checksums(body)
+        api_client.put(path: path, body: req_body) do |_status, resp_body, response|
+          update_checksums(resp_body)
+
+          file.warnings = resp_body.dig("asset", "warnings")
 
-        response
+          yield(response) if block_given?
+        end
       end
 
       def get(file)
@@ -291,7 +322,7 @@ module ShopifyCLI
           file.write(body.dig("asset", "value"))
         end
 
-        response
+        yield(response)
       end
 
       def delete(file)
@@ -302,7 +333,7 @@ module ShopifyCLI
           })
         )
 
-        response
+        yield(response)
       end
 
       def union_merge(file)
@@ -311,11 +342,11 @@ module ShopifyCLI
           query: URI.encode_www_form("asset[key]" => file.relative_path),
         )
 
-        return response unless file.text?
+        return yield(response) unless file.text?
 
         remote_content = body.dig("asset", "value")
 
-        return response if remote_content.nil?
+        return yield(response) if remote_content.nil?
 
         content = Merger.union_merge(file, remote_content)
 
@@ -323,7 +354,7 @@ module ShopifyCLI
 
         enqueue(:update, file)
 
-        response
+        yield(response)
       end
 
       def update_checksums(api_response)
@@ -336,7 +367,11 @@ module ShopifyCLI
       end
 
       def parse_api_errors(exception)
-        parsed_body = JSON.parse(exception&.response&.body)
+        parsed_body = if exception&.response&.is_a?(Hash)
+          exception&.response&.[](:body)
+        else
+          JSON.parse(exception&.response&.body)
+        end
         message = parsed_body.dig("errors", "asset") || parsed_body["message"] || exception.message
         # Truncate to first lines
         [message].flatten.map { |m| m.split("\n", 2).first }
@@ -347,7 +382,7 @@ module ShopifyCLI
       def backoff_if_near_limit!(used, limit)
         if used > limit - @threads.size
           @ctx.debug("Near API call limit, waiting 2 sec…")
-          @backoff_mutex.synchronize { sleep(2) }
+          @backoff_mutex.synchronize { wait(2) }
         end
       end
 
@@ -363,6 +398,15 @@ module ShopifyCLI
         # Sleeping in the mutex in another thread. Wait for unlock
         @backoff_mutex.synchronize {} if backingoff?
       end
+
+      def handle_operation_error(operation, error)
+        error_suffix = ":\n  " + parse_api_errors(error).join("\n  ")
+        report_error(operation, error_suffix)
+      end
+
+      def wait(duration)
+        sleep(duration)
+      end
     end
   end
 end

data/lib/shopify_cli/theme/theme_admin_api.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module ShopifyCLI
   module Theme
     class ThemeAdminAPI
@@ -10,35 +12,36 @@ module ShopifyCLI
         @shop = shop || get_shop_or_abort
       end
 
-      def get(path:, **args)
-        rest_request(method: "GET", path: path, **args)
+      def get(path:, **args, &block)
+        rest_request(method: "GET", path: path, **args, &block)
       end
 
-      def put(path:, **args)
-        rest_request(method: "PUT", path: path, **args)
+      def put(path:, **args, &block)
+        rest_request(method: "PUT", path: path, **args, &block)
       end
 
-      def post(path:, **args)
-        rest_request(method: "POST", path: path, **args)
+      def post(path:, **args, &block)
+        rest_request(method: "POST", path: path, **args, &block)
       end
 
-      def delete(path:, **args)
-        rest_request(method: "DELETE", path: path, **args)
+      def delete(path:, **args, &block)
+        rest_request(method: "DELETE", path: path, **args, &block)
       end
 
       def get_shop_or_abort # rubocop:disable Naming/AccessorMethodName
         ShopifyCLI::AdminAPI.get_shop_or_abort(@ctx)
       end
 
-      private
-
       def rest_request(**args)
-        ShopifyCLI::AdminAPI.rest_request(
+        status, body, response = ShopifyCLI::AdminAPI.rest_request(
           @ctx,
           shop: @shop,
           api_version: API_VERSION,
           **args.compact
         )
+        return yield(status, body, response) if block_given?
+
+        [status, body, response]
       rescue ShopifyCLI::API::APIRequestForbiddenError,
              ShopifyCLI::API::APIRequestUnauthorizedError => error
 
@@ -62,6 +65,8 @@ module ShopifyCLI
         raise error
       end
 
+      private
+
       def permission_error
         ensure_user_error = @ctx.message("theme.ensure_user_error", shop)
         ensure_user_try_this = @ctx.message("theme.ensure_user_try_this")

data/lib/shopify_cli/theme/theme_admin_api_throttler/bulk.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require_relative "bulk_job"
+require "shopify_cli/thread_pool"
+
+module ShopifyCLI
+  module Theme
+    class ThemeAdminAPIThrottler
+      class Bulk
+        MAX_BULK_BYTESIZE = 10_485_760 # 10MB
+        MAX_BULK_FILES = 20 # files
+        QUEUE_TIMEOUT = 0.2 # 200ms
+
+        attr_accessor :admin_api
+
+        def initialize(ctx, admin_api, pool_size: 20)
+          @ctx = ctx
+          @admin_api = admin_api
+          @latest_enqueued_at = now
+
+          @thread_pool = ShopifyCLI::ThreadPool.new(pool_size: pool_size)
+
+          pool_size.times do
+            @thread_pool.schedule(
+              BulkJob.new(ctx, self)
+            )
+          end
+
+          @put_requests = []
+          @mut = Mutex.new
+        end
+
+        def enqueue(put_request)
+          @mut.synchronize do
+            @latest_enqueued_at = now
+            @put_requests << put_request
+          end
+        end
+
+        def shutdown
+          wait_put_requests
+          @thread_pool.shutdown
+        end
+
+        def consume_put_requests
+          to_batch = []
+          to_batch_size_bytes = 0
+          @mut.synchronize do
+            # sort requests to perform less retries at the `bulk_job` level
+            @put_requests.sort_by! { |r| r.liquid? ? 0 : 1 }
+
+            is_ready = false
+            until is_ready || @put_requests.empty?
+              request = @put_requests.first
+              if to_batch.empty? && request.size > MAX_BULK_BYTESIZE
+                is_ready = true
+                to_batch << request
+                to_batch_size_bytes += request.size
+                @put_requests.shift
+              elsif to_batch.size + 1 > MAX_BULK_FILES || to_batch_size_bytes + request.size > MAX_BULK_BYTESIZE
+                is_ready = true
+              else
+                to_batch << request
+                to_batch_size_bytes += request.size
+                @put_requests.shift
+              end
+            end
+          end
+          [to_batch, to_batch_size_bytes]
+        end
+
+        def ready?
+          queue_timeout? || bulk_size >= MAX_BULK_FILES || bulk_bytesize >= MAX_BULK_BYTESIZE
+        end
+
+        def bulk_bytesize
+          @put_requests.map(&:size).reduce(:+).to_i
+        end
+
+        private
+
+        def bulk_size
+          @put_requests.size
+        end
+
+        def queue_timeout?
+          return false if bulk_size.zero?
+          elapsed_time = now - @latest_enqueued_at
+          elapsed_time > QUEUE_TIMEOUT
+        end
+
+        def wait_put_requests
+          sleep(0.2) until @put_requests.empty?
+        end
+
+        def now
+          Time.now.to_f
+        end
+      end
+    end
+  end
+end

data/lib/shopify_cli/theme/theme_admin_api_throttler/bulk_job.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require "shopify_cli/thread_pool/job"
+require_relative "request_parser"
+require_relative "response_parser"
+require_relative "errors"
+
+module ShopifyCLI
+  module Theme
+    class ThemeAdminAPIThrottler
+      class BulkJob < ShopifyCLI::ThreadPool::Job
+        JOB_TIMEOUT = 0.2 # 200ms
+        MAX_RETRIES = 5
+
+        attr_reader :bulk
+
+        def initialize(ctx, bulk)
+          super(JOB_TIMEOUT)
+          @ctx = ctx
+          @bulk = bulk
+
+          # Mutex used to coordinate changes performed by the bulk item block
+          @block_mutex = Mutex.new
+        end
+
+        def perform!
+          return unless bulk.ready?
+          put_requests, bulk_size = bulk.consume_put_requests
+          return if put_requests.empty?
+
+          @ctx.debug("[BulkJob] size: #{put_requests.size}, bytesize: #{bulk_size}")
+          bulk_status, bulk_body, response = rest_request(put_requests)
+
+          if bulk_status == 207
+            responses(bulk_body).each_with_index do |tuple, index|
+              status, body = tuple
+              put_request = put_requests[index]
+              if status == 200 || put_request.retries >= MAX_RETRIES
+                @block_mutex.synchronize do
+                  if status == 200
+                    @ctx.debug("[BulkJob] asset saved: #{put_request.key}")
+                    put_request.block.call(status, body, response)
+                  else
+                    @ctx.debug("[BulkJob] asset continuing with error: #{put_request.key}")
+                    err = AssetUploadError.new(body, response: { body: body })
+                    put_request.block.call(status, {}, err)
+                  end
+                end
+              else
+                @ctx.debug("[BulkJob] asset error: #{put_request.key}")
+                @block_mutex.synchronize do
+                  put_request.retries += 1
+                  bulk.enqueue(put_request)
+                end
+              end
+            end
+          else
+            @ctx.puts(@ctx.message("theme.stable_flag_suggestion"))
+          end
+        end
+
+        private
+
+        def rest_request(put_requests)
+          request = RequestParser.new(put_requests).parse
+          bulk.admin_api.rest_request(**request)
+        end
+
+        def responses(response_body)
+          ResponseParser.new(response_body).parse
+        end
+      end
+    end
+  end
+end

data/lib/shopify_cli/theme/theme_admin_api_throttler/errors.rb

@@ -0,0 +1,7 @@
+module ShopifyCLI
+  module Theme
+    class ThemeAdminAPIThrottler
+      class AssetUploadError < ShopifyCLI::API::APIRequestError; end
+    end
+  end
+end

data/lib/shopify_cli/theme/theme_admin_api_throttler/put_request.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require "shopify_cli/thread_pool/job"
+require_relative "request_parser"
+require_relative "response_parser"
+
+module ShopifyCLI
+  module Theme
+    class ThemeAdminAPIThrottler
+      class PutRequest
+        attr_reader :method, :body, :path, :block
+        attr_accessor :retries
+
+        def initialize(path, body, &block)
+          @method = "PUT"
+          @path = path
+          @body = body
+          @block = block
+          @retries = 0
+        end
+
+        def to_h
+          {
+            method: method,
+            path: path,
+            body: body,
+          }
+        end
+
+        def to_s
+          "#{key}, retries: #{retries}"
+        end
+
+        def liquid?
+          key.end_with?(".liquid")
+        end
+
+        def key
+          @key ||= JSON.parse(body)["asset"]["key"]
+        end
+
+        def bulk_path
+          path.gsub(/.json$/, "/bulk.json")
+        end
+
+        def size
+          @size ||= body.bytesize
+        end
+      end
+    end
+  end
+end

data/lib/shopify_cli/theme/theme_admin_api_throttler/request_parser.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module ShopifyCLI
+  module Theme
+    class ThemeAdminAPIThrottler
+      class RequestParser
+        def initialize(requests)
+          @requests = requests
+        end
+
+        def parse
+          {
+            path: path,
+            method: method,
+            body: JSON.generate({ assets: assets }),
+          }
+        end
+
+        private
+
+        def method
+          @requests.sample.method
+        end
+
+        def path
+          @requests.sample.bulk_path
+        end
+
+        def assets
+          @requests.map do |request|
+            body = JSON.parse(request.body)
+            body = body.is_a?(Hash) ? body : JSON.parse(body)
+            body["asset"]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/shopify_cli/theme/theme_admin_api_throttler/response_parser.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module ShopifyCLI
+  module Theme
+    class ThemeAdminAPIThrottler
+      class ResponseParser
+        def initialize(response_body)
+          @response_body = response_body
+        end
+
+        def parse
+          result = []
+          @response_body["results"]&.each do |resp|
+            result << [resp["code"], resp["body"]]
+          end
+          result
+        end
+      end
+    end
+  end
+end