shopify-cli 2.29.0 → 2.30.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5f07c45a6fc9d6b93d9f2826a7ad9eb139842d8b7f35717ca5fb17291285c330
-  data.tar.gz: 774e222e76f9a8d30b68e8b0756ed4cecbc8281d302d94c131d072c123b19f8c
+  metadata.gz: 3050c5be253ff04fbd3fa04ddd60389c0e6c0c2c0a3ae3fdcc6b059dc428c193
+  data.tar.gz: 2d373f27b01e11cd4df467114ba00849c978ae7cca1d8529749be5ba5d5d663d
 SHA512:
-  metadata.gz: eab29689be4de3a23b0b9dd14401a130d6f37cc1064c02f16bbce0656d378d487666bd964dfa673b8fcc48344b01acf8d10899e4fb3755c7f5f46c5dd8c46385
-  data.tar.gz: 01d9830606768a70de1de312527875d997dce26f1913c44fd6ffd82407afc7962931fe38dd7568e49c899438e33bf3f1b465b6b5d73bb0585d25643c0d26d5a2
+  metadata.gz: 8427e448b97accd935d7b6b16452070139a70139d53cfd3e4d7f4efff495873f4d1372e9e08bf4b75fae7150c95ddb431363b7d970dbcfffc1fbeb1b22f7c0af
+  data.tar.gz: be22490e0dea78184b7bcd58c4bca6a1a52c2963c45063e9a7ad6703ecd295c6e31d37a02716f741a3a7682d0fc12f86a167b0d78d1ef9bd973894bc5b7b4ba1

data/CHANGELOG.md

@@ -2,6 +2,12 @@ From version 2.6.0, the sections in this file adhere to the [keep a changelog](h
 
 ## [Unreleased]
 
+## Version 2.30.0 - 2022-11-01
+
+### Fixed
+* [#2668](https://github.com/Shopify/shopify-cli/pull/2668): Introduce `--only/--ignore` in the `shopify theme serve` help message
+* [#2667](https://github.com/Shopify/shopify-cli/pull/2667): Fix for "X zip is required for packaging a theme" on Windows
+
 ## Version 2.29.0 - 2022-10-19
 
 ### Added
@@ -11,6 +17,7 @@ From version 2.6.0, the sections in this file adhere to the [keep a changelog](h
 
 ### Fixed
 * [#2646](https://github.com/Shopify/shopify-cli/pull/2646): Demo themes shouldn't appear in the `shopify theme pull/push/list/open` commands
+* [#2650](https://github.com/Shopify/shopify-cli/pull/2650): The `shopify theme push`/`shopify theme serve` commands no longer freeze in some scenarios
 
 ### Changed
 * [#2648](https://github.com/Shopify/shopify-cli/pull/2648): Do not warn users when the CLI 2.x is running as a subprocess

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    shopify-cli (2.29.0)
+    shopify-cli (2.30.0)
       bugsnag (~> 6.22)
       listen (~> 3.7.0)
       theme-check (~> 1.11.0)

data/lib/project_types/theme/commands/package.rb

@@ -18,11 +18,15 @@ module Theme
         release-notes.md
       ]
 
+      ZIP = "zip"
+      SEVEN_ZIP = "7z"
+
       def call(args, _name)
         path = args.first || "."
 
-        check_prereq_command("zip")
+        check_prereq_command
         zip_name = theme_name(path) + ".zip"
+
         zip(zip_name, path, THEME_DIRECTORIES)
         @ctx.done(@ctx.message("theme.package.done", zip_name))
       end
@@ -33,13 +37,12 @@ module Theme
 
       private
 
-      def check_prereq_command(command)
-        cmd_path = @ctx.which(command)
-        @ctx.abort(@ctx.message("theme.package.error.prereq_command_required", command)) if cmd_path.nil?
+      def check_prereq_command
+        @ctx.abort(@ctx.message("theme.package.error.prereq_command_required")) if command.nil?
       end
 
       def zip(zip_name, path, files)
-        @ctx.system("zip", "-r", zip_name, *files, chdir: path)
+        @ctx.system(command, command_flags, zip_name, *files, chdir: path)
       end
 
       def theme_name(path)
@@ -53,6 +56,18 @@ module Theme
 
         [theme_name, theme_info["theme_version"]].compact.join("-")
       end
+
+      def command
+        @command ||= if @ctx.which(ZIP)
+          ZIP
+        elsif @ctx.which(SEVEN_ZIP)
+          SEVEN_ZIP
+        end
+      end
+
+      def command_flags
+        @command_flags ||= command == ZIP ? "-r" : "a"
+      end
     end
   end
 end

data/lib/project_types/theme/messages/messages.rb

@@ -124,6 +124,8 @@ module Theme
 
             Options:
               {{command:-t, --theme=NAME_OR_ID}} Theme ID or name of the remote theme.
+              {{command:-o, --only}}             Hot reload only files that match the specified pattern.
+              {{command:-x, --ignore}}           Skip hot reloading any files that match the specified pattern.
               {{command:--port=PORT}}            Local port to serve theme preview from.
               {{command:--poll}}                 Force polling to detect file changes.
               {{command:--host=HOST}}            Set which network interface the web server listens on. The default value is 127.0.0.1.
@@ -300,8 +302,8 @@ module Theme
             Usage: {{command:%s theme package [ ROOT ]}}
           HELP
           error: {
-            prereq_command_required: "%1$s is required for packaging a theme. Please install %1$s "\
-              "using the appropriate package manager for your system.",
+            prereq_command_required: "zip or 7zip is required for packaging a theme. Please install "\
+              "zip or 7zip using the appropriate package manager for your system.",
             missing_config: "Provide a config/settings_schema.json to package your theme",
             missing_theme_name: "Provide a theme_info.theme_name configuration in config/settings_schema.json",
           },

data/lib/shopify_cli/packager.rb

@@ -69,14 +69,13 @@ module ShopifyCLI
     def build_homebrew
       root_dir = File.join(PACKAGING_DIR, "homebrew")
 
-      build_path = File.join(BUILDS_DIR, "shopify-cli.rb")
-      build_path_2 = File.join(BUILDS_DIR, "shopify-cli@2.rb")
-      puts "\nBuilding Homebrew packages"
+      build_path = File.join(BUILDS_DIR, "shopify-cli@2.rb")
+      puts "\nBuilding Homebrew package"
 
-      puts "Generating formulae…"
+      puts "Generating formula…"
       File.delete(build_path) if File.exist?(build_path)
 
-      spec_contents = File.read(File.join(root_dir, "shopify-cli.base.rb"))
+      spec_contents = File.read(File.join(root_dir, "shopify-cli@2.base.rb"))
       spec_contents = spec_contents.gsub("SHOPIFY_CLI_VERSION", ShopifyCLI::VERSION)
 
       puts "Grabbing sha256 checksum from Rubygems.org"
@@ -90,15 +89,7 @@ module ShopifyCLI
       spec_contents = spec_contents.gsub("SHOPIFY_CLI_GEM_CHECKSUM", gem_checksum)
 
       puts "Writing generated formula\n  To: #{build_path}\n\n"
-      File.write(build_path, spec_contents.gsub("SHOPIFY_CLI_BINSTUB_SUFFIX", ""))
-
-      puts "Writing generated formula\n  To: #{build_path_2}\n\n"
-      File.write(
-        build_path_2,
-        spec_contents
-          .sub("class ShopifyCli < Formula", "class ShopifyCliAT2 < Formula")
-          .sub("SHOPIFY_CLI_BINSTUB_SUFFIX", "2")
-      )
+      File.write(build_path, spec_contents)
     end
 
     private

data/lib/shopify_cli/theme/backoff_helper.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module ShopifyCLI
+  module Theme
+    module BackoffHelper
+      def initialize_backoff_helper!(margin: 2, backoff_delay: 2)
+        @margin = margin
+        @backoff_delay = backoff_delay
+        @backoff_mutex = Mutex.new
+      end
+
+      def backoff_if_near_limit!(response)
+        # Check if the API told us we're near the rate limit
+        return if backingoff? || !response
+
+        limit = response.fetch("x-shopify-shop-api-call-limit", "0/999")
+        used, total = limit.split("/").map(&:to_i)
+
+        backoff! if used > total - @margin
+      end
+
+      def wait_for_backoff!
+        # Sleeping in the mutex in another thread. Wait for unlock
+        backoff_mutex.synchronize {} if backingoff?
+      end
+
+      def backoff!
+        ctx.debug("Near API call limit, waiting #{@backoff_delay} seconds")
+        backoff_mutex.synchronize { wait(@backoff_delay) }
+      end
+
+      def backingoff?
+        backoff_mutex.locked?
+      end
+
+      def backoff_mutex
+        @backoff_mutex || raise("Backoff helper must be initialized")
+      end
+
+      private
+
+      def wait(seconds)
+        sleep(seconds)
+      end
+    end
+  end
+end

data/lib/shopify_cli/theme/ignore_helper.rb

@@ -14,7 +14,13 @@ module ShopifyCLI
       end
 
       def ignore_path?(path)
-        ignored_by_ignore_filter?(path) || ignored_by_include_filter?(path)
+        is_ignored = ignored_by_ignore_filter?(path) || ignored_by_include_filter?(path)
+
+        if is_ignored && @ctx
+          @ctx.debug("ignore #{path}")
+        end
+
+        is_ignored
       end
 
       private

data/lib/shopify_cli/theme/syncer/downloader.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require "forwardable"
+
+module ShopifyCLI
+  module Theme
+    class Syncer
+      class Downloader
+        extend Forwardable
+
+        attr_reader :syncer, :ctx
+
+        def_delegators :syncer,
+          :ctx,
+          :checksums,
+          :enqueue_get,
+          :ignore_file?,
+          :fetch_checksums!,
+          :wait!
+
+        def initialize(syncer, delete, &update_progress_bar_block)
+          @syncer = syncer
+          @delete = delete
+          @update_progress_bar_block = update_progress_bar_block
+        end
+
+        def download!
+          fetch_checksums!
+
+          if delete_local_files?
+            to_be_deleted.each { |file| delete(file) }
+          end
+
+          enqueue_get(checksums.keys)
+          wait!(&@update_progress_bar_block)
+        end
+
+        private
+
+        def delete(file)
+          ctx.debug("[#{self.class}] rm #{file.relative_path}")
+          file.delete
+        end
+
+        def delete_local_files?
+          @delete
+        end
+
+        def to_be_deleted
+          @to_be_deleted ||= syncer
+            .theme
+            .theme_files
+            .reject { |file| present_remotely?(file) }.uniq
+            .reject { |file| ignore_file?(file) }
+        end
+
+        def present_remotely?(file)
+          checksums.has?(file)
+        end
+      end
+    end
+  end
+end

data/lib/shopify_cli/theme/syncer/uploader/bulk.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+require "shopify_cli/thread_pool"
+require "shopify_cli/theme/backoff_helper"
+
+require_relative "bulk_job"
+
+module ShopifyCLI
+  module Theme
+    class Syncer
+      class Uploader
+        class Bulk
+          include ShopifyCLI::Theme::BackoffHelper
+
+          MAX_BULK_BYTESIZE = 102_400   # 100KB
+          MAX_BULK_FILES = 10           # 10 files
+          SHUTDOWN_TIMEOUT = 20         # 20 seconds
+          SHUTDOWN_RETRY_INTERVAL = 0.2 # 200 milliseconds
+
+          attr_reader :ctx, :theme, :admin_api
+
+          def initialize(ctx, theme, admin_api, pool_size: 15)
+            @ctx = ctx
+            @theme = theme
+            @admin_api = admin_api
+            @pending_items = []
+            @in_progress_items = []
+
+            # Mutex used to coordinate changes in the `@pending_items` shared
+            # accross the `@thread_pool` threads
+            @pending_items_mutex = Mutex.new
+
+            # Mutex used to coordinate changes in the `@in_progress_items`
+            # shared accross the `@thread_pool` threads
+            @in_progress_items_mutex = Mutex.new
+
+            # Initialize thread pool and the jobs
+            @thread_pool = initialize_thread_pool!(pool_size)
+
+            # Initialize backoff helper on main thread to pause all jobs when
+            # requests are reaching API rate limits
+            initialize_backoff_helper!(margin: pool_size, backoff_delay: 5)
+          end
+
+          def enqueue(bulk_item)
+            @pending_items_mutex.synchronize { @pending_items << bulk_item }
+          end
+
+          def shutdown
+            wait_bulk_items
+            @thread_pool.shutdown
+          end
+
+          def consume_bulk_items
+            items = []
+            items_bytesize = 0
+
+            @pending_items_mutex.synchronize do
+              has_enough_items = false
+              has_enough_bytesize = false
+
+              until has_enough_items || has_enough_bytesize || @pending_items.empty?
+                bulk_item = @pending_items.first
+
+                has_enough_items = items.size + 1 > MAX_BULK_FILES
+                has_enough_bytesize = items_bytesize + bulk_item.size > MAX_BULK_BYTESIZE
+
+                break if items.any? && (has_enough_items || has_enough_bytesize)
+
+                items << bulk_item
+                items_bytesize += bulk_item.size
+
+                @pending_items.shift
+              end
+            end
+
+            @in_progress_items_mutex.synchronize do
+              @in_progress_items += items
+            end
+
+            [items, items_bytesize]
+          end
+
+          def clean_in_progress_items(items)
+            @in_progress_items_mutex.synchronize do
+              @in_progress_items -= items
+            end
+          end
+
+          def remaining_items
+            @pending_items + @in_progress_items
+          end
+
+          private
+
+          def wait_bulk_items
+            start_time = Time.now
+
+            wait(SHUTDOWN_RETRY_INTERVAL) while remaining_items? && start_time - Time.now < SHUTDOWN_TIMEOUT
+
+            files = remaining_items.map { |item| "- #{item.key}" }.join("\n")
+
+            log("shutdown, remaining_items=#{remaining_items.size}\n#{files}")
+          end
+
+          def remaining_items?
+            !@pending_items.empty? || !@in_progress_items.empty?
+          end
+
+          def initialize_thread_pool!(pool_size)
+            ShopifyCLI::ThreadPool
+              .new(pool_size: pool_size)
+              .tap do |thread_pool|
+                pool_size.times { thread_pool.schedule(spawn_job) }
+              end
+          end
+
+          def spawn_job
+            BulkJob.new(ctx, self)
+          end
+
+          def log(message)
+            ctx.debug("[Bulk] #{message}")
+          end
+
+          def wait(seconds)
+            sleep(seconds)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/shopify_cli/theme/syncer/uploader/bulk_item.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require "shopify_cli/thread_pool/job"
+
+module ShopifyCLI
+  module Theme
+    class Syncer
+      class Uploader
+        class BulkItem
+          attr_reader :file, :block
+          attr_accessor :retries
+
+          def initialize(file, &block)
+            @file = file
+            @block = block
+            @retries = 0
+          end
+
+          def to_h
+            { body: body }
+          end
+
+          def to_s
+            "#<#{self.class.name} key=#{key}, retries=#{retries}>"
+          end
+
+          def liquid?
+            file.liquid?
+          end
+
+          def key
+            file.relative_path
+          end
+
+          def size
+            @size ||= body.bytesize
+          end
+
+          def body
+            @body ||= JSON.generate(asset: asset)
+          end
+
+          def asset
+            @asset ||= asset_hash
+          end
+
+          private
+
+          def asset_hash
+            asset = { key: file.relative_path }
+
+            if file.text?
+              asset[:value] = file.read
+            else
+              asset[:attachment] = Base64.encode64(file.read)
+            end
+
+            asset
+          end
+        end
+      end
+    end
+  end
+end

data/lib/shopify_cli/theme/syncer/uploader/bulk_job.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+require "shopify_cli/thread_pool/job"
+
+require_relative "bulk_request"
+
+module ShopifyCLI
+  module Theme
+    class Syncer
+      class Uploader
+        class BulkJob < ShopifyCLI::ThreadPool::Job
+          INTERVAL = 0.2 # 200ms
+          MAX_RETRIES = 10
+
+          attr_reader :ctx, :bulk, :admin_api
+
+          def initialize(ctx, bulk)
+            super(INTERVAL)
+
+            @ctx = ctx
+            @bulk = bulk
+            @admin_api = bulk.admin_api
+
+            # Mutex used to coordinate changes in the bulk items
+            @bulk_item_mutex = Mutex.new
+
+            # Mutex used to coordinate changes in the bulk items
+            @bulk_item_mutex = Mutex.new
+          end
+
+          def perform!
+            bulk.wait_for_backoff!
+
+            # Fetch bulk items
+            bulk_items, bulk_size = bulk.consume_bulk_items
+            return if bulk_items.empty?
+
+            # Perform bulk request
+            log("job request: size=#{bulk_items.size}, bytesize=#{bulk_size}")
+            bulk_status, bulk_body, response = rest_request(bulk_items)
+            bulk.backoff_if_near_limit!(response)
+            log("job response: http_status=#{bulk_status}")
+
+            # Abort execution when a fatal error happens
+            return stable_flag_suggestion! if bulk_status != 207
+
+            # Handle item reponses
+            responses = parse_responses(bulk_body)
+            responses
+              .each_with_index do |tuple, index|
+                status, body = tuple
+                bulk_item = bulk_items[index]
+                handle_item_response(bulk_item, status, body, response)
+              end
+          ensure
+            bulk.clean_in_progress_items(bulk_items)
+          end
+
+          private
+
+          def handle_item_response(bulk_item, status, body, response)
+            if status == 200
+              return handle_success(bulk_item, status, body, response)
+            end
+
+            if bulk_item.retries < MAX_RETRIES
+              return handle_retry(bulk_item, status, body, response)
+            end
+
+            handle_error(bulk_item, status, body)
+          end
+
+          def handle_success(bulk_item, status, body, response)
+            log("bulk item success (item=#{bulk_item.key})")
+
+            @bulk_item_mutex.synchronize do
+              bulk_item.block.call(status, body, response)
+            end
+          end
+
+          def handle_retry(bulk_item, status, _body, _response)
+            key = bulk_item.key
+            retries = bulk_item.retries
+
+            log("bulk item error (item=#{key}, status=#{status}, retries=#{retries})")
+
+            @bulk_item_mutex.synchronize do
+              bulk_item.retries += 1
+              bulk.enqueue(bulk_item)
+            end
+          end
+
+          def handle_error(bulk_item, status, body)
+            log("bulk item fatal error (item=#{bulk_item.key}, status=#{status})")
+
+            @bulk_item_mutex.synchronize do
+              bulk_item.block.call(status, body, error_response(body))
+            end
+          end
+
+          def stable_flag_suggestion!
+            ctx.abort(ctx.message("theme.stable_flag_suggestion"))
+          end
+
+          def rest_request(bulk_items)
+            theme = bulk.theme
+            args = BulkRequest.new(theme, bulk_items).to_h
+
+            measure("bulk rest_request") { admin_api.rest_request(**args) }
+          end
+
+          def measure(subject)
+            return yield unless ctx.debug?
+
+            start_time = Time.now
+            result = yield
+            time_elapsed = (Time.now - start_time) * 1000
+
+            log("#{subject} time: #{time_elapsed}ms")
+
+            result
+          end
+
+          def parse_responses(body)
+            body["results"]&.map { |r| [r["code"], r["body"]] } || []
+          end
+
+          def error_response(body)
+            ShopifyCLI::API::APIRequestError.new(body, response: { body: body })
+          end
+
+          def log(message)
+            ctx.debug("[BulkJob ##{object_id}] #{message}")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/shopify_cli/theme/syncer/uploader/bulk_request.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module ShopifyCLI
+  module Theme
+    class Syncer
+      class Uploader
+        class BulkRequest
+          def initialize(theme, bulk_items)
+            @theme = theme
+            @bulk_items = bulk_items
+          end
+
+          def to_h
+            {
+              path: "themes/#{@theme.id}/assets/bulk.json",
+              method: "PUT",
+              body: JSON.generate({ assets: assets }),
+            }
+          end
+
+          private
+
+          def assets
+            @bulk_items.map(&:asset)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/shopify_cli/theme/syncer/uploader/forms/apply_to_all.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require_relative "apply_to_all_form"
+
+module ShopifyCLI
+  module Theme
+    class Syncer
+      class Uploader
+        module Forms
+          class ApplyToAll
+            attr_reader :value
+
+            def initialize(ctx, number_of_files)
+              @ctx = ctx
+              @number_of_files = number_of_files
+              @value = nil
+              @apply = nil
+            end
+
+            def apply?(value)
+              return unless @number_of_files > 1
+
+              if @apply.nil?
+                @apply = ask.apply?
+                @value = value if @apply
+              end
+
+              @apply
+            end
+
+            private
+
+            def ask
+              ApplyToAllForm.ask(@ctx, [], number_of_files: @number_of_files)
+            end
+          end
+        end
+      end
+    end
+  end
+end