shopify-cli 2.18.1 → 2.20.1

data/lib/shopify_cli/theme/dev_server/hot-reload-no-script.html

@@ -0,0 +1,27 @@
+<noscript>
+  <style type="text/css">
+    .shopify-cli-no-script-message {
+      font-family: -apple-system, BlinkMacSystemFont, San Francisco, Segoe UI, Roboto, Helvetica Neue, sans-serif;
+      text-size-adjust: 100%;
+      text-rendering: optimizeLegibility;
+      -webkit-font-smoothing: antialiased;
+      -moz-osx-font-smoothing: grayscale;
+      position: fixed;
+      z-index: 999;
+      font-weight: 500;
+      left: 0;
+      top: 0;
+      width: 100vw;
+      height: 100vh;
+      padding: 10rem 0;
+      color: #202223;
+      text-align: center;
+      background-color: #F6F6F7;
+    }
+  </style>
+  <div class="shopify-cli-no-script-message">
+    Shopify CLI requires JavaScript to work.
+    <br />
+    Activate JavaScript support or try a different browser.
+  </div>
+</noscript>

data/lib/shopify_cli/theme/dev_server/hot-reload.js

@@ -1,3 +1,15 @@
+(() => {
+  function verifySSE() {
+    if (typeof (EventSource) === "undefined") {
+      console.error("[HotReload] Error: SSE features are not supported. Try a different browser.");
+    }
+  }
+
+  console.log("[HotReload] Initializing...");
+
+  verifySSE();
+})();
+
 (() => {
   function connect() {
     const eventSource = new EventSource('/hot-reload');
@@ -32,7 +44,7 @@
 
   function fetchDOMSections(name) {
     const domSections = sectionNamesByType(name).flatMap((n) => querySelectDOMSections(n));
-    
+
     if (domSections.length > 0) {
       return domSections;
     }
@@ -40,11 +52,11 @@
     return querySelectDOMSections(name);
   }
 
-  function isFullPageReloadMode(){
+  function isFullPageReloadMode() {
     return reloadMode() === 'full-page';
   }
 
-  function isReloadModeActive(){
+  function isReloadModeActive() {
     return reloadMode() !== 'off';
   }
 
@@ -72,7 +84,7 @@
 
     // Hot reload cookie expires in 3 seconds
     date.setSeconds(date.getSeconds() + 3);
-    
+
     var sections = files.join(',');
     var expires = date.toUTCString();
 

data/lib/shopify_cli/theme/dev_server/hot_reload.rb

@@ -67,7 +67,9 @@ module ShopifyCLI
 
         def inject_hot_reload_javascript(body)
           hot_reload_js = ::File.read("#{__dir__}/hot-reload.js")
+          hot_reload_no_script = ::File.read("#{__dir__}/hot-reload-no-script.html")
           hot_reload_script = [
+            hot_reload_no_script,
             "<script>",
             params_js,
             hot_reload_js,

data/lib/shopify_cli/theme/dev_server.rb

@@ -28,11 +28,12 @@ module ShopifyCLI
         attr_accessor :ctx
 
         def start(ctx, root, host: "127.0.0.1", theme: nil, port: 9292, poll: false, editor_sync: false,
-          mode: ReloadMode.default)
+          mode: ReloadMode.default, stable: false)
           @ctx = ctx
           theme = find_theme(root, theme)
           ignore_filter = IgnoreFilter.from_path(root)
-          @syncer = Syncer.new(ctx, theme: theme, ignore_filter: ignore_filter, overwrite_json: !editor_sync)
+          @syncer = Syncer.new(ctx, theme: theme, ignore_filter: ignore_filter, overwrite_json: !editor_sync,
+            stable: stable)
           watcher = Watcher.new(ctx, theme: theme, ignore_filter: ignore_filter, syncer: @syncer, poll: poll)
           remote_watcher = RemoteWatcher.to(theme: theme, syncer: @syncer)
 

data/lib/shopify_cli/theme/file.rb

@@ -5,6 +5,7 @@ module ShopifyCLI
   module Theme
     class File < Struct.new(:path)
       attr_accessor :remote_checksum
+      attr_writer :warnings
 
       def initialize(path, root)
         super(Pathname.new(path))
@@ -104,6 +105,10 @@ module ShopifyCLI
         @relative_path.to_s
       end
 
+      def warnings
+        @warnings || []
+      end
+
       private
 
       def normalize_json(content)

data/lib/shopify_cli/theme/syncer/json_update_handler.rb

@@ -8,16 +8,11 @@ module ShopifyCLI
     class Syncer
       module JsonUpdateHandler
         def enqueue_json_updates(files)
-          # Some files must be uploaded after the other ones
-          delayed_files = [
-            theme["config/settings_schema.json"],
-            theme["config/settings_data.json"],
-          ]
-
           # Update remote JSON files and delays `delayed_files` update
           files = files
-            .select { |file| !ignore_file?(file) && file.exist? && checksums.file_has_changed?(file) }
+            .select { |file| ready_to_update?(file) }
             .sort_by { |file| delayed_files.include?(file) ? 1 : 0 }
+            .reject { |file| overwrite_json? && delayed_files.include?(file) }
 
           if overwrite_json?
             enqueue_updates(files)
@@ -27,6 +22,21 @@ module ShopifyCLI
           end
         end
 
+        def enqueue_delayed_files_updates
+          return unless overwrite_json?
+          # Update delayed files synchronously
+          delayed_files.each do |file|
+            update(file) if ready_to_update?(file)
+          end
+        end
+
+        def delayed_files
+          [
+            theme["config/settings_schema.json"],
+            theme["config/settings_data.json"],
+          ]
+        end
+
         private
 
         def handle_update_conflicts(files)
@@ -76,6 +86,10 @@ module ShopifyCLI
         def ask_update_strategy(file)
           Forms::SelectUpdateStrategy.ask(@ctx, [], file: file, exists_remotely: file_exist_remotely?(file)).strategy
         end
+
+        def ready_to_update?(file)
+          !ignore_file?(file) && file.exist? && checksums.file_has_changed?(file)
+        end
       end
     end
   end

data/lib/shopify_cli/theme/syncer/operation.rb

@@ -9,6 +9,7 @@ module ShopifyCLI
         COLOR_BY_STATUS = {
           error: :red,
           synced: :green,
+          warning: :yellow,
           fixed: :cyan,
         }
 
@@ -26,8 +27,8 @@ module ShopifyCLI
           as_message_with(status: :error)
         end
 
-        def as_synced_message
-          as_message_with(status: :synced)
+        def as_synced_message(color: :green)
+          as_message_with(status: :synced, color: color)
         end
 
         def as_fix_message
@@ -40,11 +41,11 @@ module ShopifyCLI
 
         private
 
-        def as_message_with(status:)
-          status_color = COLOR_BY_STATUS[status]
-          status_text = @ctx.message("theme.serve.operation.status.#{status}").ljust(6)
+        def as_message_with(status:, color: nil)
+          color ||= COLOR_BY_STATUS[status]
+          text = @ctx.message("theme.serve.operation.status.#{status}").ljust(6)
 
-          "#{timestamp} {{#{status_color}:#{status_text}}} {{>}} {{blue:#{self}}}"
+          "#{timestamp} {{#{color}:#{text}}} {{>}} {{blue:#{self}}}"
         end
 
         def timestamp

data/lib/shopify_cli/theme/syncer/unsupported_script_warning.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module ShopifyCLI
+  module Theme
+    class Syncer
+      class UnsupportedScriptWarning
+        attr_reader :ctx
+
+        def initialize(ctx, file)
+          @ctx = ctx
+          @file = file
+        end
+
+        def to_s
+          "\n\n#{occurrences} #{long_text}"
+        end
+
+        private
+
+        def occurrences
+          warnings.map { |w| occurrence(w) }.join("\n")
+        end
+
+        def occurrence(warning)
+          line_number = "{{blue: #{warning.line} |}}"
+          pointer = pointer_message(warning)
+
+          <<~OCCURRENCE
+            #{line_number} #{warning.line_content}
+            #{pointer}
+          OCCURRENCE
+        end
+
+        def long_text
+          lines_and_columns = warnings.map do |warning|
+            message("line_and_column", warning.line, warning.column)
+          end
+
+          message("unsupported_script_text", lines_and_columns.join)
+            .split("\n")
+            .reduce("") do |text, line|
+              # Add indentation in the long text to improve readability
+              line = " #{line}"
+
+              # Inline yellow (otherwise `CLI::UI::Frame` breaks multiline formatting)
+              line = "{{yellow:#{line}}}"
+
+              "#{text}#{line}\n"
+            end
+        end
+
+        def pointer_message(warning)
+          padding = warning.column + warning.line.to_s.size + 2
+          text = message("unsupported_script")
+
+          "{{yellow:#{" " * padding} ^ {{bold:#{text}}}}}"
+        end
+
+        def message(*args)
+          key = args.shift
+          @ctx.message("theme.serve.syncer.warnings.#{key}", *args)
+        end
+
+        def warnings
+          @warnings ||= @file.warnings.map { |w| Warning.new(@file, w) }
+        end
+
+        class Warning
+          attr_reader :line, :column
+
+          def initialize(file, warning_hash)
+            @file = file
+            @line = warning_hash["line"].to_i
+            @column = warning_hash["column"].to_i
+          end
+
+          def line_content
+            file_lines[line - 1]
+          end
+
+          private
+
+          def file_lines
+            @file_lines ||= @file.read.split("\n")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/shopify_cli/theme/syncer.rb

@@ -13,7 +13,9 @@ require_relative "syncer/json_update_handler"
 require_relative "syncer/merger"
 require_relative "syncer/operation"
 require_relative "syncer/standard_reporter"
+require_relative "syncer/unsupported_script_warning"
 require_relative "theme_admin_api"
+require_relative "theme_admin_api_throttler"
 
 module ShopifyCLI
   module Theme
@@ -31,12 +33,12 @@ module ShopifyCLI
         :union_merge, # - Union merges the local file content with the remote file content
       ]
 
-      attr_reader :theme, :checksums, :error_checksums
+      attr_reader :theme, :checksums, :error_checksums, :api_client
       attr_accessor :include_filter, :ignore_filter
 
       def_delegators :@error_reporter, :has_any_error?
 
-      def initialize(ctx, theme:, include_filter: nil, ignore_filter: nil, overwrite_json: true)
+      def initialize(ctx, theme:, include_filter: nil, ignore_filter: nil, overwrite_json: true, stable: false)
         @ctx = ctx
         @theme = theme
         @include_filter = include_filter
@@ -58,15 +60,21 @@ module ShopifyCLI
         # Mutex used to pause all threads when backing-off when hitting API rate limits
         @backoff_mutex = Mutex.new
 
+        # Mutex used to coordinate changes in the `pending` list
+        @pending_mutex = Mutex.new
+
         # Latest theme assets checksums. Updated on each upload.
         @checksums = Checksums.new(theme)
 
         # Checksums of assets with errors.
         @error_checksums = []
-      end
 
-      def api_client
-        @api_client ||= ThemeAdminAPI.new(@ctx, @theme.shop)
+        # Initialize `api_client` on main thread
+        @api_client = ThemeAdminAPIThrottler.new(
+          @ctx,
+          ThemeAdminAPI.new(@ctx, @theme.shop),
+          !stable
+        )
       end
 
       def lock_io!
@@ -134,6 +142,7 @@ module ShopifyCLI
       end
 
       def shutdown
+        api_client.shutdown
         @queue.close unless @queue.closed?
       ensure
         @threads.each { |thread| thread.join if thread.alive? }
@@ -185,6 +194,9 @@ module ShopifyCLI
         unless delay_low_priority_files
           wait!(&block)
         end
+
+        api_client.deactivate_throttler!
+        enqueue_delayed_files_updates
       end
 
       def download_theme!(delete: true, &block)
@@ -242,38 +254,57 @@ module ShopifyCLI
         wait_for_backoff!
         @ctx.debug(operation.to_s)
 
-        response = send(operation.method, operation.file)
+        send(operation.method, operation.file) do |response|
+          raise response if response.is_a?(StandardError)
 
-        @standard_reporter.report(operation.as_synced_message)
+          file = operation.file
 
-        # Check if the API told us we're near the rate limit
-        if !backingoff? && (limit = response["x-shopify-shop-api-call-limit"])
-          used, total = limit.split("/").map(&:to_i)
-          backoff_if_near_limit!(used, total)
+          if file.warnings.any?
+            warning_message =
+              operation.as_synced_message(color: :yellow) +
+              UnsupportedScriptWarning.new(@ctx, file).to_s
+            @error_reporter.report(warning_message)
+          else
+            @standard_reporter.report(operation.as_synced_message)
+          end
+
+          # Check if the API told us we're near the rate limit
+          if !backingoff? && (limit = response["x-shopify-shop-api-call-limit"])
+            used, total = limit.split("/").map(&:to_i)
+            backoff_if_near_limit!(used, total)
+          end
+        rescue StandardError => error
+          handle_operation_error(operation, error)
+        ensure
+          @pending_mutex.synchronize do
+            # Avoid abrupt jumps in the progress bar
+            wait(0.05)
+            @pending.delete(operation)
+          end
         end
-      rescue ShopifyCLI::API::APIRequestError => e
-        error_suffix = ":\n  " + parse_api_errors(e).join("\n  ")
-        report_error(operation, error_suffix)
-      ensure
-        @pending.delete(operation)
+      rescue StandardError => error
+        handle_operation_error(operation, error)
       end
 
       def update(file)
         asset = { key: file.relative_path }
+
         if file.text?
           asset[:value] = file.read
         else
           asset[:attachment] = Base64.encode64(file.read)
         end
 
-        _status, body, response = api_client.put(
-          path: "themes/#{@theme.id}/assets.json",
-          body: JSON.generate(asset: asset)
-        )
+        path = "themes/#{@theme.id}/assets.json"
+        req_body = JSON.generate(asset: asset)
 
-        update_checksums(body)
+        api_client.put(path: path, body: req_body) do |_status, resp_body, response|
+          update_checksums(resp_body)
+
+          file.warnings = resp_body.dig("asset", "warnings")
 
-        response
+          yield(response) if block_given?
+        end
       end
 
       def get(file)
@@ -291,7 +322,7 @@ module ShopifyCLI
           file.write(body.dig("asset", "value"))
         end
 
-        response
+        yield(response)
       end
 
       def delete(file)
@@ -302,7 +333,7 @@ module ShopifyCLI
           })
         )
 
-        response
+        yield(response)
       end
 
       def union_merge(file)
@@ -311,11 +342,11 @@ module ShopifyCLI
           query: URI.encode_www_form("asset[key]" => file.relative_path),
         )
 
-        return response unless file.text?
+        return yield(response) unless file.text?
 
         remote_content = body.dig("asset", "value")
 
-        return response if remote_content.nil?
+        return yield(response) if remote_content.nil?
 
         content = Merger.union_merge(file, remote_content)
 
@@ -323,7 +354,7 @@ module ShopifyCLI
 
         enqueue(:update, file)
 
-        response
+        yield(response)
       end
 
       def update_checksums(api_response)
@@ -335,19 +366,29 @@ module ShopifyCLI
         checksums.reject_duplicated_checksums!
       end
 
-      def parse_api_errors(exception)
-        parsed_body = JSON.parse(exception&.response&.body)
-        message = parsed_body.dig("errors", "asset") || parsed_body["message"] || exception.message
+      def parse_api_errors(operation, exception)
+        parsed_body = if exception&.response&.is_a?(Hash)
+          exception&.response&.[](:body)
+        else
+          JSON.parse(exception&.response&.body)
+        end
+
+        errors = parsed_body.dig("errors") # either nil or another type
+        errors = errors.dig("asset") if errors&.is_a?(Hash)
+
+        message = errors || parsed_body["message"] || exception.message
         # Truncate to first lines
         [message].flatten.map { |m| m.split("\n", 2).first }
       rescue JSON::ParserError
         [exception.message]
+      rescue StandardError => e
+        ["The asset #{operation.file} is could not be synced (cause: #{e.message})."]
       end
 
       def backoff_if_near_limit!(used, limit)
         if used > limit - @threads.size
           @ctx.debug("Near API call limit, waiting 2 sec…")
-          @backoff_mutex.synchronize { sleep(2) }
+          @backoff_mutex.synchronize { wait(2) }
         end
       end
 
@@ -363,6 +404,15 @@ module ShopifyCLI
         # Sleeping in the mutex in another thread. Wait for unlock
         @backoff_mutex.synchronize {} if backingoff?
       end
+
+      def handle_operation_error(operation, error)
+        error_suffix = ":\n  " + parse_api_errors(operation, error).join("\n  ")
+        report_error(operation, error_suffix)
+      end
+
+      def wait(duration)
+        sleep(duration)
+      end
     end
   end
 end

data/lib/shopify_cli/theme/theme_admin_api.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module ShopifyCLI
   module Theme
     class ThemeAdminAPI
@@ -10,35 +12,36 @@ module ShopifyCLI
         @shop = shop || get_shop_or_abort
       end
 
-      def get(path:, **args)
-        rest_request(method: "GET", path: path, **args)
+      def get(path:, **args, &block)
+        rest_request(method: "GET", path: path, **args, &block)
       end
 
-      def put(path:, **args)
-        rest_request(method: "PUT", path: path, **args)
+      def put(path:, **args, &block)
+        rest_request(method: "PUT", path: path, **args, &block)
       end
 
-      def post(path:, **args)
-        rest_request(method: "POST", path: path, **args)
+      def post(path:, **args, &block)
+        rest_request(method: "POST", path: path, **args, &block)
       end
 
-      def delete(path:, **args)
-        rest_request(method: "DELETE", path: path, **args)
+      def delete(path:, **args, &block)
+        rest_request(method: "DELETE", path: path, **args, &block)
       end
 
       def get_shop_or_abort # rubocop:disable Naming/AccessorMethodName
         ShopifyCLI::AdminAPI.get_shop_or_abort(@ctx)
       end
 
-      private
-
       def rest_request(**args)
-        ShopifyCLI::AdminAPI.rest_request(
+        status, body, response = ShopifyCLI::AdminAPI.rest_request(
           @ctx,
           shop: @shop,
           api_version: API_VERSION,
           **args.compact
         )
+        return yield(status, body, response) if block_given?
+
+        [status, body, response]
       rescue ShopifyCLI::API::APIRequestForbiddenError,
              ShopifyCLI::API::APIRequestUnauthorizedError => error
 
@@ -62,6 +65,8 @@ module ShopifyCLI
         raise error
       end
 
+      private
+
       def permission_error
         ensure_user_error = @ctx.message("theme.ensure_user_error", shop)
         ensure_user_try_this = @ctx.message("theme.ensure_user_try_this")

data/lib/shopify_cli/theme/theme_admin_api_throttler/bulk.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require_relative "bulk_job"
+require "shopify_cli/thread_pool"
+
+module ShopifyCLI
+  module Theme
+    class ThemeAdminAPIThrottler
+      class Bulk
+        MAX_BULK_BYTESIZE = 10_485_760 # 10MB
+        MAX_BULK_FILES = 20 # files
+        QUEUE_TIMEOUT = 0.2 # 200ms
+
+        attr_accessor :admin_api
+
+        def initialize(ctx, admin_api, pool_size: 20)
+          @ctx = ctx
+          @admin_api = admin_api
+          @latest_enqueued_at = now
+
+          @thread_pool = ShopifyCLI::ThreadPool.new(pool_size: pool_size)
+
+          pool_size.times do
+            @thread_pool.schedule(
+              BulkJob.new(ctx, self)
+            )
+          end
+
+          @put_requests = []
+          @mut = Mutex.new
+        end
+
+        def enqueue(put_request)
+          @mut.synchronize do
+            @latest_enqueued_at = now
+            @put_requests << put_request
+          end
+        end
+
+        def shutdown
+          wait_put_requests
+          @thread_pool.shutdown
+        end
+
+        def consume_put_requests
+          to_batch = []
+          to_batch_size_bytes = 0
+          @mut.synchronize do
+            # sort requests to perform less retries at the `bulk_job` level
+            @put_requests.sort_by! { |r| r.liquid? ? 0 : 1 }
+
+            is_ready = false
+            until is_ready || @put_requests.empty?
+              request = @put_requests.first
+              if to_batch.empty? && request.size > MAX_BULK_BYTESIZE
+                is_ready = true
+                to_batch << request
+                to_batch_size_bytes += request.size
+                @put_requests.shift
+              elsif to_batch.size + 1 > MAX_BULK_FILES || to_batch_size_bytes + request.size > MAX_BULK_BYTESIZE
+                is_ready = true
+              else
+                to_batch << request
+                to_batch_size_bytes += request.size
+                @put_requests.shift
+              end
+            end
+          end
+          [to_batch, to_batch_size_bytes]
+        end
+
+        def ready?
+          queue_timeout? || bulk_size >= MAX_BULK_FILES || bulk_bytesize >= MAX_BULK_BYTESIZE
+        end
+
+        def bulk_bytesize
+          @put_requests.map(&:size).reduce(:+).to_i
+        end
+
+        private
+
+        def bulk_size
+          @put_requests.size
+        end
+
+        def queue_timeout?
+          return false if bulk_size.zero?
+          elapsed_time = now - @latest_enqueued_at
+          elapsed_time > QUEUE_TIMEOUT
+        end
+
+        def wait_put_requests
+          sleep(0.2) until @put_requests.empty?
+        end
+
+        def now
+          Time.now.to_f
+        end
+      end
+    end
+  end
+end