shopify-cli 1.13.1 → 2.1.0

data/lib/shopify-cli/theme/ignore_filter.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module ShopifyCli
+  module Theme
+    class IgnoreFilter
+      FILE = ".shopifyignore"
+
+      DEFAULT_REGEXES = [
+        /\.git/,
+        /\.hg/,
+        /\.bzr/,
+        /\.svn/,
+        /_darcs/,
+        /CVS/,
+        /\.sublime-(project|workspace)/,
+        /\.DS_Store/,
+        /\.sass-cache/,
+        /Thumbs\.db/,
+        /desktop\.ini/,
+        /config.yml/,
+        /node_modules/,
+      ].freeze
+
+      DEFAULT_GLOBS = [].freeze
+
+      attr_reader :root, :globs, :regexes
+
+      def self.from_path(root)
+        root = Pathname.new(root)
+        ignore_file = root.join(FILE)
+        patterns = if ignore_file.file?
+          parse_ignore_file(ignore_file)
+        else
+          []
+        end
+        new(root, patterns: patterns)
+      end
+
+      def self.parse_ignore_file(file)
+        patterns = []
+
+        file.each_line do |line|
+          line.strip!
+
+          next if line.empty? || line.start_with?("#")
+
+          patterns << line
+        end
+
+        patterns
+      end
+
+      def initialize(root, patterns: [])
+        @root = root
+
+        regexes, globs = patterns_to_regexes_and_globs(patterns)
+
+        @regexes = regexes
+        @globs = globs
+      end
+
+      def match?(path)
+        path = path.to_s
+
+        return true if path.empty?
+
+        regexes.each do |regex|
+          return true if regex.match(path)
+        end
+
+        globs.each do |glob|
+          return true if ::File.fnmatch?(glob, path)
+        end
+
+        false
+      end
+      alias_method :ignore?, :match?
+
+      private
+
+      # Take in string patterns and convert them to either
+      # regex patterns or glob patterns so that they are handled in an expected manner.
+      def patterns_to_regexes_and_globs(patterns)
+        new_regexes = DEFAULT_REGEXES.dup
+        new_globs = DEFAULT_GLOBS.dup
+
+        patterns.each do |pattern|
+          pattern = pattern.strip
+
+          if pattern.start_with?("/") && pattern.end_with?("/")
+            new_regexes << Regexp.new(pattern.gsub(%r{^\/|\/$}, ""))
+            next
+          end
+
+          # if specifying a directory, match everything below it
+          pattern += "*" if pattern.end_with?("/")
+
+          # The pattern will be scoped to root directory, so it should match anything
+          # within that space
+          pattern.prepend("*") unless pattern.start_with?("*")
+
+          new_globs << pattern
+        end
+
+        [new_regexes, new_globs]
+      end
+    end
+  end
+end

data/lib/shopify-cli/theme/mime_type.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+require "webrick"
+
+module ShopifyCli
+  module Theme
+    class MimeType < Struct.new(:name)
+      MIME_TYPES = WEBrick::HTTPUtils::DefaultMimeTypes.merge(
+        "liquid" => "text/x-liquid",
+      )
+
+      class << self
+        def by_filename(filename)
+          new(WEBrick::HTTPUtils.mime_type(filename.to_s, MIME_TYPES))
+        end
+      end
+
+      def text?
+        /text/.match?(name) || json? || javascript?
+      end
+
+      def json?
+        name == "application/json"
+      end
+
+      def javascript?
+        name == "application/javascript"
+      end
+
+      def to_s
+        name
+      end
+    end
+  end
+end

data/lib/shopify-cli/theme/syncer.rb

@@ -0,0 +1,332 @@
+# frozen_string_literal: true
+require "thread"
+require "json"
+require "base64"
+
+module ShopifyCli
+  module Theme
+    class Syncer
+      class Operation < Struct.new(:method, :file)
+        def to_s
+          "#{method} #{file&.relative_path}"
+        end
+      end
+      API_VERSION = "unstable"
+
+      attr_reader :checksums
+      attr_accessor :ignore_filter
+
+      def initialize(ctx, theme:, ignore_filter: nil)
+        @ctx = ctx
+        @theme = theme
+        @ignore_filter = ignore_filter
+
+        # Queue of `Operation`s waiting to be picked up from a thread for processing.
+        @queue = Queue.new
+        # `Operation`s will be removed from this Array completed.
+        @pending = []
+        # Thread making the API requests.
+        @threads = []
+        # Mutex used to pause all threads when backing-off when hitting API rate limits
+        @backoff_mutex = Mutex.new
+
+        # Allows delaying log of errors, mainly to not break the progress bar.
+        @delay_errors = false
+        @delayed_errors = []
+
+        # Latest theme assets checksums. Updated on each upload.
+        @checksums = {}
+      end
+
+      def enqueue_updates(files)
+        files.each { |file| enqueue(:update, file) }
+      end
+
+      def enqueue_get(files)
+        files.each { |file| enqueue(:get, file) }
+      end
+
+      def enqueue_deletes(files)
+        files.each { |file| enqueue(:delete, file) }
+      end
+
+      def size
+        @pending.size
+      end
+
+      def empty?
+        @pending.empty?
+      end
+
+      def pending_updates
+        @pending.select { |op| op.method == :update }.map(&:file)
+      end
+
+      def remote_file?(file)
+        checksums.key?(@theme[file].relative_path.to_s)
+      end
+
+      def wait!
+        raise ThreadError, "No syncer threads" if @threads.empty?
+        total = size
+        last_size = size
+        until empty? || @queue.closed?
+          if block_given? && last_size != size
+            yield size, total
+            last_size = size
+          end
+          Thread.pass
+        end
+      end
+
+      def fetch_checksums!
+        _status, response = ShopifyCli::AdminAPI.rest_request(
+          @ctx,
+          shop: @theme.shop,
+          path: "themes/#{@theme.id}/assets.json",
+          api_version: API_VERSION,
+        )
+        update_checksums(response)
+      end
+
+      def shutdown
+        @queue.close unless @queue.closed?
+      ensure
+        @threads.each { |thread| thread.join if thread.alive? }
+      end
+
+      def start_threads(count = 2)
+        count.times do
+          @threads << Thread.new do
+            loop do
+              operation = @queue.pop
+              break if operation.nil? # shutdown was called
+              perform(operation)
+            rescue Exception => e
+              report_error(
+                "{{red:ERROR}} {{blue:#{operation}}}: #{e}" +
+                (@ctx.debug? ? "\n\t#{e.backtrace.join("\n\t")}" : "")
+              )
+            end
+          end
+        end
+      end
+
+      def delay_errors!
+        @delay_errors = true
+      end
+
+      def report_errors!
+        @delay_errors = false
+        @delayed_errors.each { |error| report_error(error) }
+        @delayed_errors.clear
+      end
+
+      def upload_theme!(delay_low_priority_files: false, delete: true, &block)
+        fetch_checksums!
+
+        if delete
+          # Delete remote files not present locally
+          removed_files = checksums.keys - @theme.theme_files.map { |file| file.relative_path.to_s }
+          enqueue_deletes(removed_files)
+        end
+
+        # Some files must be uploaded after the other ones
+        delayed_config_files = [
+          @theme["config/settings_schema.json"],
+          @theme["config/settings_data.json"],
+        ]
+
+        enqueue_updates(@theme.liquid_files)
+        enqueue_updates(@theme.json_files - delayed_config_files)
+        enqueue_updates(delayed_config_files)
+
+        if delay_low_priority_files
+          # Wait for liquid & JSON files to upload, because those are rendered remotely
+          wait!(&block)
+        end
+
+        # Process lower-priority files in the background
+
+        # Assets are served locally, so can be uploaded in the background
+        enqueue_updates(@theme.static_asset_files)
+
+        unless delay_low_priority_files
+          wait!(&block)
+        end
+      end
+
+      def download_theme!(delete: true, &block)
+        fetch_checksums!
+
+        if delete
+          # Delete local files not present remotely
+          missing_files = @theme.theme_files
+            .reject { |file| checksums.key?(file.relative_path.to_s) }.uniq
+            .reject { |file| @ignore_filter&.ignore?(file) }
+          missing_files.each do |file|
+            @ctx.debug("rm #{file.relative_path}")
+            file.delete
+          end
+        end
+
+        enqueue_get(checksums.keys)
+
+        wait!(&block)
+      end
+
+      private
+
+      def enqueue(method, file)
+        raise ArgumentError, "file required" unless file
+
+        operation = Operation.new(method, @theme[file])
+
+        # Already enqueued
+        return if @pending.include?(operation)
+
+        if @ignore_filter&.ignore?(operation.file.relative_path)
+          @ctx.debug("ignore #{operation.file.relative_path}")
+          return
+        end
+
+        if [:update, :get].include?(method) && operation.file.exist? && !file_has_changed?(operation.file)
+          @ctx.debug("skip #{operation}")
+          return
+        end
+
+        @pending << operation
+        @queue << operation unless @queue.closed?
+      end
+
+      def perform(operation)
+        return if @queue.closed?
+        wait_for_backoff!
+        @ctx.debug(operation.to_s)
+
+        response = send(operation.method, operation.file)
+
+        # Check if the API told us we're near the rate limit
+        if !backingoff? && (limit = response["x-shopify-shop-api-call-limit"])
+          used, total = limit.split("/").map(&:to_i)
+          backoff_if_near_limit!(used, total)
+        end
+      rescue ShopifyCli::API::APIRequestError => e
+        report_error(
+          "{{red:ERROR}} {{blue:#{operation}}}:\n  " +
+          parse_api_errors(e).join("\n  ")
+        )
+      ensure
+        @pending.delete(operation)
+      end
+
+      def update(file)
+        asset = { key: file.relative_path.to_s }
+        if file.text?
+          asset[:value] = file.read
+        else
+          asset[:attachment] = Base64.encode64(file.read)
+        end
+
+        _status, body, response = ShopifyCli::AdminAPI.rest_request(
+          @ctx,
+          shop: @theme.shop,
+          path: "themes/#{@theme.id}/assets.json",
+          method: "PUT",
+          api_version: API_VERSION,
+          body: JSON.generate(asset: asset)
+        )
+
+        update_checksums(body)
+
+        response
+      end
+
+      def get(file)
+        _status, body, response = ShopifyCli::AdminAPI.rest_request(
+          @ctx,
+          shop: @theme.shop,
+          path: "themes/#{@theme.id}/assets.json",
+          method: "GET",
+          api_version: API_VERSION,
+          query: URI.encode_www_form("asset[key]" => file.relative_path.to_s),
+        )
+
+        update_checksums(body)
+
+        attachment = body.dig("asset", "attachment")
+        value = if attachment
+          file.write(Base64.decode64(attachment), 0, mode: "wb")
+        else
+          file.write(body.dig("asset", "value"))
+        end
+
+        response
+      end
+
+      def delete(file)
+        _status, _body, response = ShopifyCli::AdminAPI.rest_request(
+          @ctx,
+          shop: @theme.shop,
+          path: "themes/#{@theme.id}/assets.json",
+          method: "DELETE",
+          api_version: API_VERSION,
+          body: JSON.generate(asset: {
+            key: file.relative_path.to_s
+          })
+        )
+
+        response
+      end
+
+      def update_checksums(api_response)
+        api_response.values.flatten.each do |asset|
+          if asset["key"]
+            @checksums[asset["key"]] = asset["checksum"]
+          end
+        end
+        # Generate .liquid asset files are reported twice in checksum:
+        # once of generated, once for .liquid. We only keep the .liquid, that's the one we have
+        # on disk.
+        @checksums.reject! { |key, _| @checksums.key?("#{key}.liquid") }
+      end
+
+      def file_has_changed?(file)
+        file.checksum != @checksums[file.relative_path.to_s]
+      end
+
+      def report_error(error)
+        if @delay_errors
+          @delayed_errors << error
+        else
+          @ctx.puts(error)
+        end
+      end
+
+      def parse_api_errors(exception)
+        parsed_body = JSON.parse(exception&.response&.body)
+        message = parsed_body.dig("errors", "asset") || parsed_body["message"] || exception.message
+        # Truncate to first lines
+        [message].flatten.map { |message| message.split("\n", 2).first }
+      rescue JSON::ParserError
+        [exception.message]
+      end
+
+      def backoff_if_near_limit!(used, limit)
+        if used > limit - @threads.size
+          @ctx.debug("Near API call limit, waiting 2 sec…")
+          @backoff_mutex.synchronize { sleep 2 }
+        end
+      end
+
+      def backingoff?
+        @backoff_mutex.locked?
+      end
+
+      def wait_for_backoff!
+        # Sleeping in the mutex in another thread. Wait for unlock
+        @backoff_mutex.synchronize {} if backingoff?
+      end
+    end
+  end
+end