@shopify/cli-kit 3.38.0 → 3.40.0

package/assets/cli-ruby/lib/shopify_cli/theme/syncer/downloader.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require "forwardable"
+
+module ShopifyCLI
+  module Theme
+    class Syncer
+      class Downloader
+        extend Forwardable
+
+        attr_reader :syncer, :ctx
+
+        def_delegators :syncer,
+          :ctx,
+          :checksums,
+          :enqueue_get,
+          :ignore_file?,
+          :fetch_checksums!,
+          :wait!
+
+        def initialize(syncer, delete, &update_progress_bar_block)
+          @syncer = syncer
+          @delete = delete
+          @update_progress_bar_block = update_progress_bar_block
+        end
+
+        def download!
+          fetch_checksums!
+
+          if delete_local_files?
+            to_be_deleted.each { |file| delete(file) }
+          end
+
+          enqueue_get(checksums.keys)
+          wait!(&@update_progress_bar_block)
+        end
+
+        private
+
+        def delete(file)
+          ctx.debug("[#{self.class}] rm #{file.relative_path}")
+          file.delete
+        end
+
+        def delete_local_files?
+          @delete
+        end
+
+        def to_be_deleted
+          @to_be_deleted ||= syncer
+            .theme
+            .theme_files
+            .reject { |file| present_remotely?(file) }.uniq
+            .reject { |file| ignore_file?(file) }
+        end
+
+        def present_remotely?(file)
+          checksums.has?(file)
+        end
+      end
+    end
+  end
+end

package/assets/cli-ruby/lib/shopify_cli/theme/syncer/error_reporter.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module ShopifyCLI
+  module Theme
+    class Syncer
+      ##
+      # ShopifyCLI::Theme::Syncer::ErrorReporter allows delaying log of errors,
+      # mainly to not break the progress bar.
+      #
+      class ErrorReporter
+        attr_reader :ctx, :delayed_errors
+
+        def initialize(ctx)
+          @ctx = ctx
+          @has_any_error = false
+          @delay_errors = false
+          @delayed_errors = []
+        end
+
+        def disable!
+          @delay_errors = true
+        end
+
+        def enable!
+          @delay_errors = false
+          @delayed_errors.each { |error| report(error) }
+          @delayed_errors.clear
+        end
+
+        def report(error_message)
+          if @delay_errors
+            @delayed_errors << error_message
+          else
+            @has_any_error = true
+            @ctx.error(error_message)
+          end
+        end
+
+        def has_any_error?
+          @has_any_error
+        end
+      end
+    end
+  end
+end

package/assets/cli-ruby/lib/shopify_cli/theme/syncer/merger.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require "tempfile"
+
+module ShopifyCLI
+  module Theme
+    class Syncer
+      class Merger
+        class << self
+          ##
+          # Merge `theme_file` with the `new_content` by relying on the union merge
+          #
+          def union_merge(theme_file, new_content)
+            git_merge(theme_file, new_content, ["--union", "-p"])
+          end
+
+          private
+
+          ##
+          # Merge theme file (`ShopifyCLI::Theme::File`) with a new content (String),
+          # by creating a temporary file based on the `new_content`.
+          #
+          def git_merge(theme_file, new_content, opts)
+            remote_file = create_tmp_file(tmp_file_name(theme_file), new_content)
+            empty_file = create_tmp_file("empty")
+
+            ShopifyCLI::Git.merge_file(
+              theme_file.absolute_path,
+              empty_file.path,
+              remote_file.path,
+              opts
+            )
+          ensure
+            # Remove temporary files on Windows as well
+            remote_file.close!
+            empty_file.close!
+          end
+
+          def create_tmp_file(basename, content = "")
+            tmp_file = Tempfile.new(basename)
+            tmp_file.write(content)
+            tmp_file.close # Make it ready to merge
+            tmp_file
+          end
+
+          def tmp_file_name(ref_file)
+            "shopify-cli-merge-#{ref_file.name(".*")}"
+          end
+        end
+      end
+    end
+  end
+end

package/assets/cli-ruby/lib/shopify_cli/theme/syncer/operation.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module ShopifyCLI
+  module Theme
+    class Syncer
+      class Operation
+        attr_accessor :method, :file
+
+        COLOR_BY_STATUS = {
+          error: :red,
+          synced: :green,
+          warning: :yellow,
+          fixed: :cyan,
+        }
+
+        def initialize(ctx, method, file)
+          @ctx = ctx
+          @method = method
+          @file = file
+        end
+
+        def to_s
+          "#{method} #{file_path}"
+        end
+
+        def as_error_message
+          as_message_with(status: :error)
+        end
+
+        def as_synced_message(color: :green)
+          as_message_with(status: :synced, color: color)
+        end
+
+        def as_fix_message
+          as_message_with(status: :fixed)
+        end
+
+        def file_path
+          file&.relative_path
+        end
+
+        private
+
+        def as_message_with(status:, color: nil)
+          color ||= COLOR_BY_STATUS[status]
+          text = @ctx.message("theme.serve.operation.status.#{status}").ljust(6)
+
+          "#{timestamp} {{#{color}:#{text}}} {{>}} {{blue:#{self}}}"
+        end
+
+        def timestamp
+          Time.now.strftime("%T")
+        end
+      end
+    end
+  end
+end

package/assets/cli-ruby/lib/shopify_cli/theme/syncer/standard_reporter.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module ShopifyCLI
+  module Theme
+    class Syncer
+      ##
+      # ShopifyCLI::Theme::Syncer::StdReporter allows disabling/enabling
+      # messages reported in the standard output (ShopifyCLI::Context#puts).
+      #
+      class StandardReporter
+        attr_reader :ctx
+
+        def initialize(ctx)
+          @enabled = true
+          @ctx = ctx
+        end
+
+        def disable!
+          @enabled = false
+        end
+
+        def enable!
+          @enabled = true
+        end
+
+        def report(message)
+          ctx.puts(message) if @enabled
+        end
+      end
+    end
+  end
+end

package/assets/cli-ruby/lib/shopify_cli/theme/syncer/unsupported_script_warning.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module ShopifyCLI
+  module Theme
+    class Syncer
+      class UnsupportedScriptWarning
+        attr_reader :ctx
+
+        def initialize(ctx, file)
+          @ctx = ctx
+          @file = file
+        end
+
+        def to_s
+          "\n\n#{occurrences} #{long_text}"
+        end
+
+        private
+
+        def occurrences
+          warnings.map { |w| occurrence(w) }.join("\n")
+        end
+
+        def occurrence(warning)
+          line_number = "{{blue: #{warning.line} |}}"
+          pointer = pointer_message(warning)
+
+          <<~OCCURRENCE
+            #{line_number} #{warning.line_content}
+            #{pointer}
+          OCCURRENCE
+        end
+
+        def long_text
+          lines_and_columns = warnings.map do |warning|
+            message("line_and_column", warning.line, warning.column)
+          end
+
+          message("unsupported_script_text", lines_and_columns.join)
+            .split("\n")
+            .reduce("") do |text, line|
+              # Add indentation in the long text to improve readability
+              line = " #{line}"
+
+              # Inline yellow (otherwise `CLI::UI::Frame` breaks multiline formatting)
+              line = "{{yellow:#{line}}}"
+
+              "#{text}#{line}\n"
+            end
+        end
+
+        def pointer_message(warning)
+          padding = warning.column + warning.line.to_s.size + 2
+          text = message("unsupported_script")
+
+          "{{yellow:#{" " * padding} ^ {{bold:#{text}}}}}"
+        end
+
+        def message(*args)
+          key = args.shift
+          @ctx.message("theme.serve.syncer.warnings.#{key}", *args)
+        end
+
+        def warnings
+          @warnings ||= @file.warnings.map { |w| Warning.new(@file, w) }
+        end
+
+        class Warning
+          attr_reader :line, :column
+
+          def initialize(file, warning_hash)
+            @file = file
+            @line = warning_hash["line"].to_i
+            @column = warning_hash["column"].to_i
+          end
+
+          def line_content
+            file_lines[line - 1]
+          end
+
+          private
+
+          def file_lines
+            @file_lines ||= @file.read.split("\n")
+          end
+        end
+      end
+    end
+  end
+end

package/assets/cli-ruby/lib/shopify_cli/theme/syncer/uploader/bulk.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+require "shopify_cli/thread_pool"
+require "shopify_cli/theme/backoff_helper"
+
+require_relative "bulk_job"
+
+module ShopifyCLI
+  module Theme
+    class Syncer
+      class Uploader
+        class Bulk
+          include ShopifyCLI::Theme::BackoffHelper
+
+          MAX_BULK_BYTESIZE = 102_400   # 100KB
+          MAX_BULK_FILES = 10           # 10 files
+          SHUTDOWN_TIMEOUT = 20         # 20 seconds
+          SHUTDOWN_RETRY_INTERVAL = 0.2 # 200 milliseconds
+
+          attr_reader :ctx, :theme, :admin_api
+
+          def initialize(ctx, theme, admin_api, pool_size: 15)
+            @ctx = ctx
+            @theme = theme
+            @admin_api = admin_api
+            @pending_items = []
+            @in_progress_items = []
+
+            # Mutex used to coordinate changes in the `@pending_items` shared
+            # accross the `@thread_pool` threads
+            @pending_items_mutex = Mutex.new
+
+            # Mutex used to coordinate changes in the `@in_progress_items`
+            # shared accross the `@thread_pool` threads
+            @in_progress_items_mutex = Mutex.new
+
+            # Initialize thread pool and the jobs
+            @thread_pool = initialize_thread_pool!(pool_size)
+
+            # Initialize backoff helper on main thread to pause all jobs when
+            # requests are reaching API rate limits
+            initialize_backoff_helper!(margin: pool_size, backoff_delay: 5)
+          end
+
+          def enqueue(bulk_item)
+            @pending_items_mutex.synchronize { @pending_items << bulk_item }
+          end
+
+          def shutdown
+            wait_bulk_items
+            @thread_pool.shutdown
+          end
+
+          def consume_bulk_items
+            items = []
+            items_bytesize = 0
+
+            @pending_items_mutex.synchronize do
+              has_enough_items = false
+              has_enough_bytesize = false
+
+              until has_enough_items || has_enough_bytesize || @pending_items.empty?
+                bulk_item = @pending_items.first
+
+                has_enough_items = items.size + 1 > MAX_BULK_FILES
+                has_enough_bytesize = items_bytesize + bulk_item.size > MAX_BULK_BYTESIZE
+
+                break if items.any? && (has_enough_items || has_enough_bytesize)
+
+                items << bulk_item
+                items_bytesize += bulk_item.size
+
+                @pending_items.shift
+              end
+            end
+
+            @in_progress_items_mutex.synchronize do
+              @in_progress_items += items
+            end
+
+            [items, items_bytesize]
+          end
+
+          def clean_in_progress_items(items)
+            @in_progress_items_mutex.synchronize do
+              @in_progress_items -= items
+            end
+          end
+
+          def remaining_items
+            @pending_items + @in_progress_items
+          end
+
+          private
+
+          def wait_bulk_items
+            start_time = Time.now
+
+            wait(SHUTDOWN_RETRY_INTERVAL) while remaining_items? && start_time - Time.now < SHUTDOWN_TIMEOUT
+
+            files = remaining_items.map { |item| "- #{item.key}" }.join("\n")
+
+            log("shutdown, remaining_items=#{remaining_items.size}\n#{files}")
+          end
+
+          def remaining_items?
+            !@pending_items.empty? || !@in_progress_items.empty?
+          end
+
+          def initialize_thread_pool!(pool_size)
+            ShopifyCLI::ThreadPool
+              .new(pool_size: pool_size)
+              .tap do |thread_pool|
+                pool_size.times { thread_pool.schedule(spawn_job) }
+              end
+          end
+
+          def spawn_job
+            BulkJob.new(ctx, self)
+          end
+
+          def log(message)
+            ctx.debug("[Bulk] #{message}")
+          end
+
+          def wait(seconds)
+            sleep(seconds)
+          end
+        end
+      end
+    end
+  end
+end

package/assets/cli-ruby/lib/shopify_cli/theme/syncer/uploader/bulk_item.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require "shopify_cli/thread_pool/job"
+
+module ShopifyCLI
+  module Theme
+    class Syncer
+      class Uploader
+        class BulkItem
+          attr_reader :file, :block
+          attr_accessor :retries
+
+          def initialize(file, &block)
+            @file = file
+            @block = block
+            @retries = 0
+          end
+
+          def to_h
+            { body: body }
+          end
+
+          def to_s
+            "#<#{self.class.name} key=#{key}, retries=#{retries}>"
+          end
+
+          def liquid?
+            file.liquid?
+          end
+
+          def key
+            file.relative_path
+          end
+
+          def size
+            @size ||= body.bytesize
+          end
+
+          def body
+            @body ||= JSON.generate(asset: asset)
+          end
+
+          def asset
+            @asset ||= asset_hash
+          end
+
+          private
+
+          def asset_hash
+            asset = { key: file.relative_path }
+
+            if file.text?
+              asset[:value] = file.read
+            else
+              asset[:attachment] = Base64.encode64(file.read)
+            end
+
+            asset
+          end
+        end
+      end
+    end
+  end
+end

package/assets/cli-ruby/lib/shopify_cli/theme/syncer/uploader/bulk_job.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+require "shopify_cli/thread_pool/job"
+
+require_relative "bulk_request"
+
+module ShopifyCLI
+  module Theme
+    class Syncer
+      class Uploader
+        class BulkJob < ShopifyCLI::ThreadPool::Job
+          INTERVAL = 0.2 # 200ms
+          MAX_RETRIES = 10
+
+          attr_reader :ctx, :bulk, :admin_api
+
+          def initialize(ctx, bulk)
+            super(INTERVAL)
+
+            @ctx = ctx
+            @bulk = bulk
+            @admin_api = bulk.admin_api
+
+            # Mutex used to coordinate changes in the bulk items
+            @bulk_item_mutex = Mutex.new
+
+            # Mutex used to coordinate changes in the bulk items
+            @bulk_item_mutex = Mutex.new
+          end
+
+          def perform!
+            bulk.wait_for_backoff!
+
+            # Fetch bulk items
+            bulk_items, bulk_size = bulk.consume_bulk_items
+            return if bulk_items.empty?
+
+            # Perform bulk request
+            log("job request: size=#{bulk_items.size}, bytesize=#{bulk_size}")
+            bulk_status, bulk_body, response = rest_request(bulk_items)
+            bulk.backoff_if_near_limit!(response)
+            log("job response: http_status=#{bulk_status}")
+
+            # Abort execution when a fatal error happens
+            return stable_flag_suggestion! if bulk_status != 207
+
+            # Handle item reponses
+            responses = parse_responses(bulk_body)
+            responses
+              .each_with_index do |tuple, index|
+                status, body = tuple
+                bulk_item = bulk_items[index]
+                handle_item_response(bulk_item, status, body, response)
+              end
+          ensure
+            bulk.clean_in_progress_items(bulk_items)
+          end
+
+          private
+
+          def handle_item_response(bulk_item, status, body, response)
+            if status == 200
+              return handle_success(bulk_item, status, body, response)
+            end
+
+            if bulk_item.retries < MAX_RETRIES
+              return handle_retry(bulk_item, status, body, response)
+            end
+
+            handle_error(bulk_item, status, body)
+          end
+
+          def handle_success(bulk_item, status, body, response)
+            log("bulk item success (item=#{bulk_item.key})")
+
+            @bulk_item_mutex.synchronize do
+              bulk_item.block.call(status, body, response)
+            end
+          end
+
+          def handle_retry(bulk_item, status, _body, _response)
+            key = bulk_item.key
+            retries = bulk_item.retries
+
+            log("bulk item error (item=#{key}, status=#{status}, retries=#{retries})")
+
+            @bulk_item_mutex.synchronize do
+              bulk_item.retries += 1
+              bulk.enqueue(bulk_item)
+            end
+          end
+
+          def handle_error(bulk_item, status, body)
+            log("bulk item fatal error (item=#{bulk_item.key}, status=#{status})")
+
+            @bulk_item_mutex.synchronize do
+              bulk_item.block.call(status, body, error_response(body))
+            end
+          end
+
+          def stable_flag_suggestion!
+            ctx.abort(ctx.message("theme.stable_flag_suggestion"))
+          end
+
+          def rest_request(bulk_items)
+            theme = bulk.theme
+            args = BulkRequest.new(theme, bulk_items).to_h
+
+            measure("bulk rest_request") { admin_api.rest_request(**args) }
+          end
+
+          def measure(subject)
+            return yield unless ctx.debug?
+
+            start_time = Time.now
+            result = yield
+            time_elapsed = (Time.now - start_time) * 1000
+
+            log("#{subject} time: #{time_elapsed}ms")
+
+            result
+          end
+
+          def parse_responses(body)
+            body["results"]&.map { |r| [r["code"], r["body"]] } || []
+          end
+
+          def error_response(body)
+            ShopifyCLI::API::APIRequestError.new(body, response: { body: body })
+          end
+
+          def log(message)
+            ctx.debug("[BulkJob ##{object_id}] #{message}")
+          end
+        end
+      end
+    end
+  end
+end

package/assets/cli-ruby/lib/shopify_cli/theme/syncer/uploader/bulk_request.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module ShopifyCLI
+  module Theme
+    class Syncer
+      class Uploader
+        class BulkRequest
+          def initialize(theme, bulk_items)
+            @theme = theme
+            @bulk_items = bulk_items
+          end
+
+          def to_h
+            {
+              path: "themes/#{@theme.id}/assets/bulk.json",
+              method: "PUT",
+              body: JSON.generate({ assets: assets }),
+            }
+          end
+
+          private
+
+          def assets
+            @bulk_items.map(&:asset)
+          end
+        end
+      end
+    end
+  end
+end