shrine 1.3.0 → 1.4.0

data/lib/shrine/plugins/migration_helpers.rb

@@ -1,46 +1,65 @@
 class Shrine
   module Plugins
-    # The migration_helpers plugin gives the model additional helper methods
-    # which are convenient when doing attachment migrations.
+    # The migration_helpers plugin gives the attacher additional helper methods
+    # which are convenient when doing file migrations.
     #
-    #     plugin :migration_helpers
+    # By default additional methods are also added to the model which delegate
+    # to the underlying attacher. If you want to disable that, you can load the
+    # plugin with `delegate: false`:
     #
-    # ## `<attachment>_cache` and `<attachment>_store`
+    #     plugin :migration_helpers, delegate: false
     #
-    # If your attachment's name is "avatar", the model will get `#avatar_cache`
-    # and `#avatar_store` methods.
+    # ## `attachment_cache` and `attachment_store`
+    #
+    # These methods return cache and store uploaders used by the underlying
+    # attacher:
     #
-    #     user = User.new
     #     user.avatar_cache #=> #<Shrine @storage_key=:cache @storage=#<Shrine::Storage::FileSystem @directory=public/uploads>>
     #     user.avatar_store #=> #<Shrine @storage_key=:store @storage=#<Shrine::Storage::S3:0x007fb8343397c8 @bucket=#<Aws::S3::Bucket name="foo">>>
     #
-    # ## `<attachment>_cached?` and `<attachment>_stored?`
+    #     # attacher equivalents
+    #     user.avatar_attacher.cache
+    #     user.avatar_attacher.store
+    #
+    # ## `attachment_cached?` and `attachment_stored?`
     #
-    # You can use these methods to check whether attachment exists and is
-    # cached/stored:
+    # These methods return true if attachment exists and is cached/stored:
     #
     #     user.avatar_cached? # user.avatar && user.avatar_cache.uploaded?(user.avatar)
     #     user.avatar_stored? # user.avatar && user.avatar_store.uploaded?(user.avatar)
     #
-    # ## `update_<attachment>`
+    #     # attacher equivalents
+    #     user.avatar_attacher.cached?
+    #     user.avatar_attacher.stored?
     #
-    # The model will also get `#update_avatar` method, which can be used when
-    # doing attachment migrations. It will update the record's attachment with
-    # the result of the passed in block.
+    # ## `update_attachment`
+    #
+    # This method updates the record's attachment with the result of the given
+    # block.
     #
     #     user.update_avatar do |avatar|
     #       user.avatar_store.upload(avatar) # saved to the record
     #     end
     #
-    # This will get triggered _only_ if the attachment is not nil and is
-    # stored, and will get saved only if the current attachment hasn't changed
-    # while executing the block. The result can be anything that responds to
-    # `#to_json` and evaluates to uploaded files' data.
+    #     # attacher equivalent
+    #     user.avatar_attacher.update_stored { |avatar| }
+    #
+    # The block will get triggered _only_ if the attachment is present and not
+    # cached, *and* will save the record only if the record's attachment
+    # hasn't changed in the time it took to execute the block. This method is
+    # most useful for adding/removing versions and changing locations of files.
     module MigrationHelpers
+      def self.configure(uploader, options = {})
+        warn "The :delegate option in migration_helpers Shrine plugin will default to false in Shrine 2. To remove this warning, set :delegate explicitly." if !options.key?(:delegate)
+        uploader.opts[:migration_helpers_delegate] = options.fetch(:delegate, true)
+      end
+
       module AttachmentMethods
         def initialize(name)
           super
 
+          return if shrine_class.opts[:migration_helpers_delegate] == false
+
           module_eval <<-RUBY, __FILE__, __LINE__ + 1
             def update_#{name}(&block)
               #{name}_attacher.update_stored(&block)

data/lib/shrine/plugins/moving.rb

@@ -25,7 +25,7 @@ class Shrine
     # deleted after upload.
     module Moving
       def self.configure(uploader, storages:)
-        uploader.opts[:move_files_to_storages] = storages
+        uploader.opts[:moving_storages] = storages
       end
 
       module InstanceMethods
@@ -38,6 +38,7 @@ class Shrine
             if movable?(io, context)
               move(io, context)
             else
+              warn "The #{storage_key.inspect} Shrine storage doesn't support moving a #{io.inspect}. It is currently still deleted, but it won't be in Shrine 2."
               super
               io.delete if io.respond_to?(:delete)
             end
@@ -55,7 +56,7 @@ class Shrine
         end
 
         def move?(io, context)
-          opts[:move_files_to_storages].include?(storage_key)
+          opts[:moving_storages].include?(storage_key)
         end
       end
     end

data/lib/shrine/plugins/parallelize.rb

@@ -1,24 +1,15 @@
-require "thread/pool"
-
-Thread::Pool.abort_on_exception = true
+require "thread"
 
 class Shrine
   module Plugins
-    # The parallelize plugin parallelizes your uploads and deletes using the
-    # [thread] gem.
+    # The parallelize plugin parallelizes uploads and deletes when handling
+    # versions, using threads.
     #
     #     plugin :parallelize
     #
-    # This plugin is generally only useful as an addition to the versions
-    # plugin, where multiple files are being uploaded and deleted at once. Note
-    # that it's not possible for this plugin to parallelize processing, but it
-    # should be easy to do that manually.
-    #
     # By default a pool of 3 threads will be used, but you can change that:
     #
     #     plugin :parallelize, threads: 5
-    #
-    # [thread]: https://github.com/meh/ruby-thread
     module Parallelize
       def self.configure(uploader, threads: 3)
         uploader.opts[:parallelize_threads] = threads
@@ -36,20 +27,48 @@ class Shrine
         private
 
         def put(io, context)
-          context[:thread_pool].process { super }
+          context[:thread_pool].enqueue { super }
         end
 
         def remove(uploaded_file, context)
-          context[:thread_pool].process { super }
+          context[:thread_pool].enqueue { super }
         end
 
         # We initialize a thread pool with configured number of threads.
-        def with_pool
-          pool = Thread.pool(opts[:parallelize_threads])
+        def with_pool(&block)
+          pool = ThreadPool.new(opts[:parallelize_threads])
           result = yield pool
-          pool.shutdown
+          pool.perform
           result
         end
+
+        class ThreadPool
+          def initialize(thread_count)
+            @thread_count = thread_count
+            @tasks = Queue.new
+          end
+
+          def enqueue(&task)
+            @tasks.enq(task)
+          end
+
+          def perform
+            threads = @thread_count.times.map { spawn_thread }
+            threads.each(&:join)
+          end
+
+          private
+
+          def spawn_thread
+            Thread.new do
+              Thread.current.abort_on_exception = true
+              loop do
+                task = @tasks.deq(true) rescue break
+                task.call
+              end
+            end
+          end
+        end
       end
     end
 

data/lib/shrine/plugins/restore_cached.rb

@@ -1,35 +1,3 @@
-require "json"
-
-class Shrine
-  module Plugins
-    # The restore_cached plugin ensures the cached file data hasn't been
-    # tampered with, by restoring its metadata after assignment. The user can
-    # tamper with the cached file data by modifying the hidden field before
-    # submitting the form.
-    #
-    # Firstly the assignment is terminated if the cached file doesn't exist,
-    # which can happen if the user changes the "id" or "storage" data. If the
-    # cached file exists, the metadata is reextracted from the original file
-    # and replaced with the potentially tampered with ones.
-    #
-    #     plugin :restore_cached
-    module RestoreCached
-      module AttacherMethods
-        private
-
-        def assign_cached(value)
-          uploaded_file = uploaded_file(value) do |file|
-            next unless cache.uploaded?(file)
-            return unless file.exists?
-            real_metadata = cache.extract_metadata(file.to_io, context)
-            file.metadata.update(real_metadata)
-          end
-
-          super(uploaded_file)
-        end
-      end
-    end
-
-    register_plugin(:restore_cached, RestoreCached)
-  end
-end
+warn "The restore_cached Shrine plugin has been renamed to \"restore_cached_data\". Loading the plugin through \"restore_cached\" will not work in Shrine 2."
+require "shrine/plugins/restore_cached_data"
+Shrine::Plugins.register_plugin(:restore_cached, Shrine::Plugins::RestoreCachedData)

data/lib/shrine/plugins/restore_cached_data.rb

@@ -0,0 +1,34 @@
+class Shrine
+  module Plugins
+    # The restore_cached_data plugin ensures the cached file data hasn't been
+    # tampered with, by restoring the metadata after assignment. The user can
+    # tamper with the cached file data by modifying the hidden field before
+    # submitting the form.
+    #
+    # Firstly the assignment is terminated if the cached file doesn't exist,
+    # which can happen if the user changes the "id" or "storage" data. If the
+    # cached file exists, the metadata is reextracted from the original file
+    # and replaced with the potentially tampered with ones.
+    #
+    #     plugin :restore_cached_data
+    module RestoreCachedData
+      module AttacherMethods
+        private
+
+        def assign_cached(value)
+          cached_file = uploaded_file(value) do |cached_file|
+            next unless cache.uploaded?(cached_file)
+            return unless cached_file.exists?
+            real_metadata = cache.extract_metadata(cached_file.to_io, context)
+            cached_file.metadata.update(real_metadata)
+            cached_file.close
+          end
+
+          super(cached_file)
+        end
+      end
+    end
+
+    register_plugin(:restore_cached_data, RestoreCachedData)
+  end
+end

data/lib/shrine/plugins/sequel.rb

@@ -73,18 +73,12 @@ class Shrine
 
         # Updates the current attachment with the new one, unless the current
         # attachment has changed.
-        def swap(uploaded_file)
-          record.db.transaction do
-            break if record.send("#{name}_data") != record.reload.send("#{name}_data")
-            super
-          end
-        rescue ::Sequel::Error
-        end
-
-        # We save the record after updating, raising any validation errors.
         def update(uploaded_file)
-          super
-          record.save(raise_on_failure: true)
+          record.this
+            .where(:"#{name}_data" => record.send(:"#{name}_data"))
+            .update(:"#{name}_data" => uploaded_file.to_json)
+          record.reload
+        rescue ::Sequel::Error
         end
 
         # Support for Postgres JSON columns.

data/lib/shrine/plugins/store_dimensions.rb

@@ -50,21 +50,23 @@ class Shrine
         def extract_dimensions(io)
           analyzer = opts[:dimensions_analyzer]
 
-          if io.respond_to?(:width) && io.respond_to?(:height)
+          dimensions = if io.respond_to?(:width) && io.respond_to?(:height)
             [io.width, io.height]
           elsif analyzer.is_a?(Symbol)
             send(:"_extract_dimensions_with_#{analyzer}", io)
           else
             analyzer.call(io)
           end
+
+          io.rewind
+
+          dimensions
         end
 
         private
 
         def _extract_dimensions_with_fastimage(io)
-          result = FastImage.size(io)
-          io.rewind # https://github.com/sdsykes/fastimage/pull/66
-          result
+          FastImage.size(io)
         end
       end
 

data/lib/shrine/plugins/validation_helpers.rb

@@ -75,7 +75,7 @@ class Shrine
         # store_dimensions plugin.
         def validate_max_width(max, message: nil)
           raise Error, ":store_dimensions plugin is required" if !get.respond_to?(:width)
-          if get.width > max
+          if get.width && get.width > max
             errors << error_message(:max_width, message, max)
           end
         end
@@ -84,7 +84,7 @@ class Shrine
         # store_dimensions plugin.
         def validate_min_width(min, message: nil)
           raise Error, ":store_dimensions plugin is required" if !get.respond_to?(:width)
-          if get.width < min
+          if get.width && get.width < min
             errors << error_message(:min_width, message, min)
           end
         end
@@ -93,7 +93,7 @@ class Shrine
         # store_dimensions plugin.
         def validate_max_height(max, message: nil)
           raise Error, ":store_dimensions plugin is required" if !get.respond_to?(:height)
-          if get.height > max
+          if get.height && get.height > max
             errors << error_message(:max_height, message, max)
           end
         end
@@ -102,7 +102,7 @@ class Shrine
         # store_dimensions plugin.
         def validate_min_height(min, message: nil)
           raise Error, ":store_dimensions plugin is required" if !get.respond_to?(:height)
-          if get.height < min
+          if get.height && get.height < min
             errors << error_message(:min_height, message, min)
           end
         end

data/lib/shrine/plugins/versions.rb

@@ -62,6 +62,18 @@ class Shrine
     #
     #     user.avatar_url(:medium) #=> "http://example.com/medium.jpg"
     #
+    # If you already have some versions processed in the foreground when a
+    # background job is kicked off, you can setup explicit URL fallbacks:
+    #
+    #     plugin :versions,
+    #       names: [:thumb, :thumb_2x, :large, :large_2x],
+    #       fallbacks: {:thumb_2x => :thumb, :large_2x => :large}
+    #
+    #     # ... (background job is kicked off)
+    #
+    #     user.avatar_url(:thumb_2x) # returns :thumb URL until :thumb_2x becomes available
+    #     user.avatar_url(:large_2x) # returns :large URL until :large_2x becomes available
+    #
     # Any additional options will be properly forwarded to the underlying
     # storage:
     #
@@ -75,7 +87,7 @@ class Shrine
     #     end
     #
     # When deleting versions, any multi delete capabilities will be leveraged,
-    # so when usingStorage::S3, deleting versions will issue only a single HTTP
+    # so when using Storage::S3, deleting versions will issue only a single HTTP
     # request. If you want to delete versions manually, you can use
     # `Shrine#delete`:
     #
@@ -86,8 +98,9 @@ class Shrine
         uploader.plugin :multi_delete
       end
 
-      def self.configure(uploader, names:)
+      def self.configure(uploader, names:, fallbacks: {})
         uploader.opts[:version_names] = names
+        uploader.opts[:version_fallbacks] = fallbacks
       end
 
       module ClassMethods
@@ -95,25 +108,20 @@ class Shrine
           opts[:version_names]
         end
 
+        def version_fallbacks
+          opts[:version_fallbacks]
+        end
+
         # Checks that the identifier is a registered version.
         def version?(name)
           version_names.map(&:to_s).include?(name.to_s)
         end
 
-        # Asserts that the hash doesn't contain any unknown versions.
-        def versions!(hash)
-          hash.select { |name, _| version?(name) or raise Error, "unknown version: #{name.inspect}" }
-        end
-
-        # Filters the hash to contain only the registered versions.
-        def versions(hash)
-          hash.select { |name, _| version?(name) }
-        end
-
         # Converts a hash of data into a hash of versions.
         def uploaded_file(object, &block)
-          if object.is_a?(Hash) && !object.key?("storage")
-            versions(object).inject({}) do |result, (name, data)|
+          if (hash = object).is_a?(Hash) && !hash.key?("storage")
+            hash.inject({}) do |result, (name, data)|
+              next result if !version?(name)
               result.update(name.to_sym => uploaded_file(data, &block))
             end
           else
@@ -141,7 +149,8 @@ class Shrine
         def _store(io, context)
           if (hash = io).is_a?(Hash)
             raise Error, ":location is not applicable to versions" if context.key?(:location)
-            self.class.versions!(hash).inject({}) do |result, (name, version)|
+            hash.inject({}) do |result, (name, version)|
+              raise Error, "unknown version: #{name.inspect}" if !self.class.version?(name)
               result.update(name => _store(version, version: name, **context))
             end
           else
@@ -166,9 +175,11 @@ class Shrine
         def url(version = nil, **options)
           if get.is_a?(Hash)
             if version
-              raise Error, "unknown version: #{version.inspect}" if !shrine_class.version_names.include?(version)
+              raise Error, "unknown version: #{version.inspect}" if !shrine_class.version?(version)
               if file = get[version]
                 file.url(**options)
+              elsif fallback = shrine_class.version_fallbacks[version]
+                url(fallback, **options)
               else
                 default_url(**options, version: version)
               end

data/lib/shrine/storage/linter.rb

@@ -7,18 +7,21 @@ require "tempfile"
 class Shrine
   # Error which is thrown when Storage::Linter fails.
   class LintError < Error
-    attr_reader :errors
-
-    def initialize(errors)
-      @errors = errors
-      super(errors.to_s)
-    end
   end
 
   module Storage
-    # Checks if the storage conforms to Shrine's specification. If the check
-    # fails, by default it raises a LintError, but you can also specify
-    # `action: :warn`.
+    # Checks if the storage conforms to Shrine's specification.
+    #
+    #   Shrine::Storage::Linter.new(storage).call
+    #
+    # If the check fails, by default it raises a `Shrine::LintError`, but you
+    # can also specify `action: :warn`:
+    #
+    #   Shrine::Storage::Linter.new(storage, action: :warn).call
+    #
+    # You can also specify an IO factory which the storage will use:
+    #
+    #   Shrine::Storage::Linter.new(storage).call(->{File.open("test/fixtures/image.jpg")})
     class Linter
       def self.call(*args)
         new(*args).call
@@ -27,68 +30,107 @@ class Shrine
       def initialize(storage, action: :error)
         @storage = storage
         @action  = action
-        @errors  = []
       end
 
-      def call(io_factory = ->{FakeIO.new("image")})
-        storage.upload(io_factory.call, id = "foo.jpg", {"mime_type" => "image/jpeg"})
+      def call(io_factory = default_io_factory)
+        storage.upload(io_factory.call, id = "foo", {})
 
-        file = storage.download(id)
-        error! "#download doesn't return a Tempfile" if !file.is_a?(Tempfile)
-        error! "#download returns an empty file" if file.read.empty?
+        lint_download(id)
+        lint_open(id)
+        lint_read(id)
+        lint_exists(id)
+        lint_url(id)
+        lint_stream(id) if storage.respond_to?(:stream)
+        lint_delete(id)
 
-        error! "#open doesn't return a valid IO object" if !io?(storage.open(id))
-        error! "#read returns an empty string" if storage.read(id).empty?
-        error! "#exists? returns false for a file that was uploaded" if !storage.exists?(id)
-        error! "#url doesn't return a string" if !storage.url(id, {}).is_a?(String)
+        if storage.respond_to?(:move)
+          uploaded_file = uploader.upload(io_factory.call, location: "bar")
+          lint_move(uploaded_file, "quux")
+        end
 
-        if storage.respond_to?(:stream)
-          content = ""
-          storage.stream(id) { |chunk| content << chunk }
-          error! "#stream does not yield any chunks" if content.empty?
+        if storage.respond_to?(:multi_delete)
+          storage.upload(io_factory.call, id = "baz")
+          lint_multi_delete(id)
         end
 
-        storage.delete(id)
-        error! "#exists? returns true for a file that was deleted" if storage.exists?(id)
+        storage.upload(io_factory.call, id = "quux")
+        lint_clear(id)
+      end
 
-        if storage.respond_to?(:move)
-          if storage.respond_to?(:movable?)
-            error! "#movable? doesn't accept 2 arguments" if !(storage.method(:movable?).arity == 2)
-            error! "#move doesn't accept 3 arguments" if !(storage.method(:move).arity == -3)
-
-            uploaded_file = uploader.upload(io_factory.call, location: "bar.jpg")
-
-            if storage.movable?(uploaded_file, "quux.jpg")
-              storage.move(uploaded_file, id = "quux.jpg")
-              error! "#exists? returns false for destination after #move" if !storage.exists?(id)
-              error! "#exists? returns true for source after #move" if storage.exists?(uploaded_file.id)
-            end
-          else
-            error! "responds to #move but doesn't respond to #movable?" if !storage.respond_to?(:movable?)
-          end
+      def lint_download(id)
+        downloaded = storage.download(id)
+        error :download, "doesn't return a Tempfile" if !downloaded.is_a?(Tempfile)
+        error :download, "returns an empty IO object" if downloaded.read.empty?
+      end
+
+      def lint_open(id)
+        opened = storage.open(id)
+        error :open, "doesn't return a valid IO object" if !io?(opened)
+        error :open, "returns an empty IO object" if opened.read.empty?
+      end
+
+      def lint_read(id)
+        read = storage.read(id)
+        error :read, "doesn't return a string" if !read.is_a?(String)
+        error :read, "returns an empty string" if read.empty?
+      end
+
+      def lint_exists(id)
+        error :exists?, "returns false for a file that was uploaded" if !storage.exists?(id)
+      end
+
+      def lint_url(id)
+        # just assert #url exists, it isn't required to return anything
+        url = storage.url(id)
+        error :url, "should return either nil or a string" if !(url.nil? || url.is_a?(String))
+      end
+
+      def lint_stream(id)
+        streamed = storage.enum_for(:stream, id).to_a
+        chunks = streamed.map { |(chunk, _)| chunk }
+        content_length = Array(streamed[0])[1]
+
+        error :stream, "doesn't yield any chunks" if chunks.empty?
+        error :stream, "yielded chunks sum up to empty content" if chunks.inject("", :+).empty?
+
+        if Array(streamed.first).size == 2
+          error :stream, "yielded content length isn't a number" if !content_length.is_a?(Integer)
         end
+      end
 
-        if storage.respond_to?(:multi_delete)
-          storage.upload(io_factory.call, id = "foo.jpg")
-          storage.multi_delete([id])
-          error! "#exists? returns true for a file that was multi-deleted" if storage.exists?(id)
+      def lint_delete(id)
+        storage.delete(id)
+        error :delete, "file still #exists? after deleting" if storage.exists?(id)
+      end
+
+      def lint_move(uploaded_file, id)
+        if storage.movable?(uploaded_file, id)
+          storage.move(uploaded_file, id, {})
+          error :exists?, "returns false for destination after #move" if !storage.exists?(id)
+          error :exists?, "returns true for source after #move" if storage.exists?(uploaded_file.id)
         end
+      end
+
+      def lint_multi_delete(id)
+        storage.multi_delete([id])
+        error :exists?, "returns true for a file that was multi-deleted" if storage.exists?(id)
+      end
+
+      def lint_clear(id)
+        storage.clear!(:confirm)
+        error :clear!, "file still #exists? after clearing" if storage.exists?(id)
 
         begin
           storage.clear!
-          error! "#clear! should raise Shrine::Confirm unless :confirm is passed in"
+          error :clear!, "should raise Shrine::Confirm if :confirm is not passed in"
         rescue Shrine::Confirm
         end
-
-        storage.upload(io_factory.call, id = "foo.jpg")
-        storage.clear!(:confirm)
-        error! "file still #exists? after #clear! was called" if storage.exists?(id)
-
-        raise LintError.new(@errors) if @errors.any? && @action == :error
       end
 
       private
 
+      attr_reader :storage
+
       def uploader
         shrine = Class.new(Shrine)
         shrine.storages[:storage] = storage
@@ -96,18 +138,27 @@ class Shrine
       end
 
       def io?(object)
-        missing_methods = IO_METHODS.reject do |m, a|
-          object.respond_to?(m) && [a.count, -1].include?(object.method(m).arity)
+        uploader.send(:_enforce_io, object)
+        true
+      rescue Shrine::InvalidFile
+        false
+      end
+
+      def error(method_name, message)
+        if @action == :error
+          raise LintError, full_message(method_name, message)
+        else
+          warn full_message(method_name, message)
         end
-        missing_methods.empty?
       end
 
-      def error!(message)
-        @errors << message
-        warn(message) if @action.to_s.start_with?("warn")
+      def full_message(method_name, message)
+        "#{@storage.class}##{method_name} - #{message}"
       end
 
-      attr_reader :storage
+      def default_io_factory
+        -> { FakeIO.new("file") }
+      end
 
       class FakeIO
         def initialize(content)