shopify-cli 2.19.0 → 2.21.0

data/lib/shopify_cli/theme/theme_admin_api_throttler/bulk.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require_relative "bulk_job"
+require "shopify_cli/thread_pool"
+
+module ShopifyCLI
+  module Theme
+    class ThemeAdminAPIThrottler
+      class Bulk
+        MAX_BULK_BYTESIZE = 10_485_760 # 10MB
+        MAX_BULK_FILES = 20 # files
+        QUEUE_TIMEOUT = 0.2 # 200ms
+
+        attr_accessor :admin_api
+
+        def initialize(ctx, admin_api, pool_size: 20)
+          @ctx = ctx
+          @admin_api = admin_api
+          @latest_enqueued_at = now
+
+          @thread_pool = ShopifyCLI::ThreadPool.new(pool_size: pool_size)
+
+          pool_size.times do
+            @thread_pool.schedule(
+              BulkJob.new(ctx, self)
+            )
+          end
+
+          @put_requests = []
+          @mut = Mutex.new
+        end
+
+        def enqueue(put_request)
+          @mut.synchronize do
+            @latest_enqueued_at = now
+            @put_requests << put_request
+          end
+        end
+
+        def shutdown
+          wait_put_requests
+          @thread_pool.shutdown
+        end
+
+        def consume_put_requests
+          to_batch = []
+          to_batch_size_bytes = 0
+          @mut.synchronize do
+            # sort requests to perform less retries at the `bulk_job` level
+            @put_requests.sort_by! { |r| r.liquid? ? 0 : 1 }
+
+            is_ready = false
+            until is_ready || @put_requests.empty?
+              request = @put_requests.first
+              if to_batch.empty? && request.size > MAX_BULK_BYTESIZE
+                is_ready = true
+                to_batch << request
+                to_batch_size_bytes += request.size
+                @put_requests.shift
+              elsif to_batch.size + 1 > MAX_BULK_FILES || to_batch_size_bytes + request.size > MAX_BULK_BYTESIZE
+                is_ready = true
+              else
+                to_batch << request
+                to_batch_size_bytes += request.size
+                @put_requests.shift
+              end
+            end
+          end
+          [to_batch, to_batch_size_bytes]
+        end
+
+        def ready?
+          queue_timeout? || bulk_size >= MAX_BULK_FILES || bulk_bytesize >= MAX_BULK_BYTESIZE
+        end
+
+        def bulk_bytesize
+          @put_requests.map(&:size).reduce(:+).to_i
+        end
+
+        private
+
+        def bulk_size
+          @put_requests.size
+        end
+
+        def queue_timeout?
+          return false if bulk_size.zero?
+          elapsed_time = now - @latest_enqueued_at
+          elapsed_time > QUEUE_TIMEOUT
+        end
+
+        def wait_put_requests
+          sleep(0.2) until @put_requests.empty?
+        end
+
+        def now
+          Time.now.to_f
+        end
+      end
+    end
+  end
+end

data/lib/shopify_cli/theme/theme_admin_api_throttler/bulk_job.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require "shopify_cli/thread_pool/job"
+require_relative "request_parser"
+require_relative "response_parser"
+require_relative "errors"
+
+module ShopifyCLI
+  module Theme
+    class ThemeAdminAPIThrottler
+      class BulkJob < ShopifyCLI::ThreadPool::Job
+        JOB_TIMEOUT = 0.2 # 200ms
+        MAX_RETRIES = 10
+
+        attr_reader :bulk
+
+        def initialize(ctx, bulk)
+          super(JOB_TIMEOUT)
+          @ctx = ctx
+          @bulk = bulk
+
+          # Mutex used to coordinate changes performed by the bulk item block
+          @block_mutex = Mutex.new
+        end
+
+        def perform!
+          return unless bulk.ready?
+          put_requests, bulk_size = bulk.consume_put_requests
+          return if put_requests.empty?
+
+          @ctx.debug("[BulkJob] size: #{put_requests.size}, bytesize: #{bulk_size}")
+          bulk_status, bulk_body, response = rest_request(put_requests)
+
+          if bulk_status == 207
+            responses(bulk_body).each_with_index do |tuple, index|
+              status, body = tuple
+              put_request = put_requests[index]
+              if status == 200 || put_request.retries >= MAX_RETRIES
+                @block_mutex.synchronize do
+                  if status == 200
+                    @ctx.debug("[BulkJob] asset saved: #{put_request.key}")
+                    put_request.block.call(status, body, response)
+                  else
+                    @ctx.debug("[BulkJob] asset continuing with error: #{put_request.key}")
+                    err = AssetUploadError.new(body, response: { body: body })
+                    put_request.block.call(status, {}, err)
+                  end
+                end
+              else
+                @ctx.debug("[BulkJob] asset error: #{put_request.key}")
+                @block_mutex.synchronize do
+                  put_request.retries += 1
+                  bulk.enqueue(put_request)
+                end
+              end
+            end
+          else
+            @ctx.puts(@ctx.message("theme.stable_flag_suggestion"))
+          end
+        end
+
+        private
+
+        def rest_request(put_requests)
+          request = RequestParser.new(put_requests).parse
+          bulk.admin_api.rest_request(**request)
+        end
+
+        def responses(response_body)
+          ResponseParser.new(response_body).parse
+        end
+      end
+    end
+  end
+end

data/lib/shopify_cli/theme/theme_admin_api_throttler/errors.rb

@@ -0,0 +1,7 @@
+module ShopifyCLI
+  module Theme
+    class ThemeAdminAPIThrottler
+      class AssetUploadError < ShopifyCLI::API::APIRequestError; end
+    end
+  end
+end

data/lib/shopify_cli/theme/theme_admin_api_throttler/put_request.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require "shopify_cli/thread_pool/job"
+require_relative "request_parser"
+require_relative "response_parser"
+
+module ShopifyCLI
+  module Theme
+    class ThemeAdminAPIThrottler
+      class PutRequest
+        attr_reader :method, :body, :path, :block
+        attr_accessor :retries
+
+        def initialize(path, body, &block)
+          @method = "PUT"
+          @path = path
+          @body = body
+          @block = block
+          @retries = 0
+        end
+
+        def to_h
+          {
+            method: method,
+            path: path,
+            body: body,
+          }
+        end
+
+        def to_s
+          "#{key}, retries: #{retries}"
+        end
+
+        def liquid?
+          key.end_with?(".liquid")
+        end
+
+        def key
+          @key ||= JSON.parse(body)["asset"]["key"]
+        end
+
+        def bulk_path
+          path.gsub(/.json$/, "/bulk.json")
+        end
+
+        def size
+          @size ||= body.bytesize
+        end
+      end
+    end
+  end
+end

data/lib/shopify_cli/theme/theme_admin_api_throttler/request_parser.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module ShopifyCLI
+  module Theme
+    class ThemeAdminAPIThrottler
+      class RequestParser
+        def initialize(requests)
+          @requests = requests
+        end
+
+        def parse
+          {
+            path: path,
+            method: method,
+            body: JSON.generate({ assets: assets }),
+          }
+        end
+
+        private
+
+        def method
+          @requests.sample.method
+        end
+
+        def path
+          @requests.sample.bulk_path
+        end
+
+        def assets
+          @requests.map do |request|
+            body = JSON.parse(request.body)
+            body = body.is_a?(Hash) ? body : JSON.parse(body)
+            body["asset"]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/shopify_cli/theme/theme_admin_api_throttler/response_parser.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module ShopifyCLI
+  module Theme
+    class ThemeAdminAPIThrottler
+      class ResponseParser
+        def initialize(response_body)
+          @response_body = response_body
+        end
+
+        def parse
+          result = []
+          @response_body["results"]&.each do |resp|
+            result << [resp["code"], resp["body"]]
+          end
+          result
+        end
+      end
+    end
+  end
+end

data/lib/shopify_cli/theme/theme_admin_api_throttler.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require "forwardable"
+
+require_relative "theme_admin_api_throttler/bulk"
+require_relative "theme_admin_api_throttler/put_request"
+
+module ShopifyCLI
+  module Theme
+    class ThemeAdminAPIThrottler
+      extend Forwardable
+
+      attr_reader :bulk, :admin_api
+
+      def_delegators :@admin_api, :get, :post, :delete
+
+      def initialize(ctx, admin_api, active = true)
+        @ctx = ctx
+        @admin_api = admin_api
+        @active = active
+        @bulk = Bulk.new(ctx, admin_api)
+      end
+
+      def put(path:, **args, &block)
+        request = PutRequest.new(path, args[:body], &block)
+        if active?
+          bulk_request(request)
+        else
+          rest_request(request)
+        end
+      end
+
+      def activate_throttler!
+        @active = true
+      end
+
+      def deactivate_throttler!
+        @active = false
+      end
+
+      def active?
+        @active
+      end
+
+      def shutdown
+        bulk.shutdown
+      end
+
+      private
+
+      def rest_request(request)
+        request.block.call(admin_api.rest_request(**request.to_h))
+      rescue StandardError => error
+        request.block.call(500, {}, error)
+      end
+
+      def bulk_request(request)
+        bulk.enqueue(request)
+      end
+    end
+  end
+end

data/lib/shopify_cli/version.rb

@@ -1,3 +1,3 @@
 module ShopifyCLI
-  VERSION = "2.19.0"
+  VERSION = "2.21.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: shopify-cli
 version: !ruby/object:Gem::Version
-  version: 2.19.0
+  version: 2.21.0
 platform: ruby
 authors:
 - Shopify
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-06-17 00:00:00.000000000 Z
+date: 2022-08-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -120,10 +120,11 @@ files:
 - ".github/DESIGN.md"
 - ".github/ISSUE_TEMPLATE.md"
 - ".github/ISSUE_TEMPLATE/bug_report.yaml"
+- ".github/ISSUE_TEMPLATE/config.yml"
 - ".github/ISSUE_TEMPLATE/enhancement.yaml"
 - ".github/ISSUE_TEMPLATE/feature.yaml"
 - ".github/PULL_REQUEST_TEMPLATE.md"
-- ".github/probots.yml"
+- ".github/workflows/cla.yml"
 - ".github/workflows/shopify.yml"
 - ".github/workflows/stale.yml"
 - ".github/workflows/triage.yml"
@@ -383,6 +384,8 @@ files:
 - lib/shopify_cli/admin_api/schema.rb
 - lib/shopify_cli/api.rb
 - lib/shopify_cli/app_type_detector.rb
+- lib/shopify_cli/assets/post_auth_page/index.html.erb
+- lib/shopify_cli/assets/post_auth_page/style.css
 - lib/shopify_cli/changelog.rb
 - lib/shopify_cli/command.rb
 - lib/shopify_cli/command/app_sub_command.rb
@@ -501,8 +504,10 @@ files:
 - lib/shopify_cli/theme/dev_server/cdn_fonts.rb
 - lib/shopify_cli/theme/dev_server/certificate_manager.rb
 - lib/shopify_cli/theme/dev_server/header_hash.rb
+- lib/shopify_cli/theme/dev_server/hot-reload-no-script.html
 - lib/shopify_cli/theme/dev_server/hot-reload.js
 - lib/shopify_cli/theme/dev_server/hot_reload.rb
+- lib/shopify_cli/theme/dev_server/hot_reload/remote_file_deleter.rb
 - lib/shopify_cli/theme/dev_server/hot_reload/remote_file_reloader.rb
 - lib/shopify_cli/theme/dev_server/hot_reload/sections_index.rb
 - lib/shopify_cli/theme/dev_server/local_assets.rb
@@ -534,8 +539,16 @@ files:
 - lib/shopify_cli/theme/syncer/merger.rb
 - lib/shopify_cli/theme/syncer/operation.rb
 - lib/shopify_cli/theme/syncer/standard_reporter.rb
+- lib/shopify_cli/theme/syncer/unsupported_script_warning.rb
 - lib/shopify_cli/theme/theme.rb
 - lib/shopify_cli/theme/theme_admin_api.rb
+- lib/shopify_cli/theme/theme_admin_api_throttler.rb
+- lib/shopify_cli/theme/theme_admin_api_throttler/bulk.rb
+- lib/shopify_cli/theme/theme_admin_api_throttler/bulk_job.rb
+- lib/shopify_cli/theme/theme_admin_api_throttler/errors.rb
+- lib/shopify_cli/theme/theme_admin_api_throttler/put_request.rb
+- lib/shopify_cli/theme/theme_admin_api_throttler/request_parser.rb
+- lib/shopify_cli/theme/theme_admin_api_throttler/response_parser.rb
 - lib/shopify_cli/thread_pool.rb
 - lib/shopify_cli/thread_pool/job.rb
 - lib/shopify_cli/transform_data_structure.rb

data/.github/probots.yml

@@ -1,3 +0,0 @@
-# .github/probots.yml
-enabled:
-  - cla