shrine 1.0.0 → 1.1.0

data/lib/shrine/plugins/versions.rb

@@ -22,8 +22,8 @@ class Shrine
     #       end
     #     end
     #
-    # Now when you access the attachment through the model, a hash of uploaded
-    # files will be returned:
+    # Now when you access the stored attachment through the model, a hash of
+    # uploaded files will be returned:
     #
     #     JSON.parse(user.avatar_data) #=>
     #     # {
@@ -44,6 +44,9 @@ class Shrine
     #     user.avatar[:medium].width #=> 500
     #     user.avatar[:small].width  #=> 300
     #
+    # You probably want to load the `delete_uploaded` plugin to automatically
+    # delete processed files after they have been uploaded.
+    #
     # The plugin also extends the `avatar_url` method to accept versions:
     #
     #     user.avatar_url(:medium)
@@ -55,7 +58,7 @@ class Shrine
     #     user.avatar #=> #<Shrine::UploadedFile>
     #     user.avatar_url(:medium) #=> "http://example.com/original.jpg"
     #
-    #     # the versions have finished generating
+    #     # the background job has finished generating versions
     #
     #     user.avatar_url(:medium) #=> "http://example.com/medium.jpg"
     #
@@ -67,10 +70,8 @@ class Shrine
     # You can also easily generate default URLs for specific versions, since
     # the `context` will include the version name:
     #
-    #     class ImageUploader
-    #       def default_url(io, context)
-    #         "/images/defaults/#{context[:version]}.jpg"
-    #       end
+    #     plugin :default_url do |context|
+    #       "/images/defaults/#{context[:version]}.jpg"
     #     end
     #
     # When deleting versions, any multi delete capabilities will be leveraged,
@@ -139,6 +140,7 @@ class Shrine
         # version.
         def _store(io, context)
           if (hash = io).is_a?(Hash)
+            raise Error, ":location is not applicable to versions" if context.key?(:location)
             self.class.versions!(hash).inject({}) do |result, (name, version)|
               result.update(name => _store(version, version: name, **context))
             end

data/lib/shrine/storage/file_system.rb

@@ -6,14 +6,14 @@ require "pathname"
 class Shrine
   module Storage
     # The FileSystem storage handles uploads to the filesystem, and it is
-    # most commonly initialized with a "base" folder and a "subdirectory":
+    # most commonly initialized with a "base" folder and a "prefix":
     #
-    #     storage = Shrine::Storage::FileSystem.new("public", subdirectory: "uploads")
+    #     storage = Shrine::Storage::FileSystem.new("public", prefix: "uploads")
     #     storage.url("image.jpg") #=> "/uploads/image.jpg"
     #
     # This storage will upload all files to "public/uploads", and the URLs
     # of the uploaded files will start with "/uploads/*". This way you can
-    # use FileSystem for both cache and store, one having subdirectory
+    # use FileSystem for both cache and store, one having the prefix
     # "uploads/cache" and other "uploads/store".
     #
     # You can also initialize the storage just with the "base" directory, and
@@ -22,20 +22,23 @@ class Shrine
     #     storage = Shrine::Storage::FileSystem.new(Dir.tmpdir)
     #     storage.url("image.jpg") #=> "/var/folders/k7/6zx6dx6x7ys3rv3srh0nyfj00000gn/T/image.jpg"
     #
-    # ## CDN
+    # ## Host
     #
     # It's generally a good idea to serve your files via a CDN, so an
     # additional `:host` option can be provided:
     #
     #     storage = Shrine::Storage::FileSystem.new("public",
-    #       subdirectory: "uploads", host: "//abc123.cloudfront.net")
-    #     storage.url("image.jpg") #=> "//abc123.cloudfront.net/uploads/image.jpg"
+    #       prefix: "uploads", host: "http://abc123.cloudfront.net")
+    #     storage.url("image.jpg") #=> "http://abc123.cloudfront.net/uploads/image.jpg"
     #
-    # The `:host` option can also be used wihout `:subdirectory`, and is
+    # If you're not using a CDN, it's recommended that you still set `:host` to
+    # your application's domain (at least in production).
+    #
+    # The `:host` option can also be used wihout `:prefix`, and is
     # useful if you for example have files located on another server:
     #
-    #     storage = Shrine::Storage::FileSystem.new("files", host: "943.23.43.1")
-    #     storage.url("image.jpg") #=> "943.23.43.1/files/image.jpg"
+    #     storage = Shrine::Storage::FileSystem.new("files", host: "http://943.23.43.1")
+    #     storage.url("image.jpg") #=> "http://943.23.43.1/files/image.jpg"
     #
     # ## Clearing cache
     #
@@ -52,17 +55,32 @@ class Shrine
     # pass the `:permissions` option:
     #
     #     Shrine::Storage::FileSystem.new("directory", permissions: 0755)
+    #
+    # ## Heroku
+    #
+    # Note that Heroku has a read-only filesystem, and doesn't allow you to
+    # upload your files to the "public" directory, you can however upload to
+    # "tmp" directory:
+    #
+    #     Shrine::Storage::FileSystem.new("tmp/uploads")
+    #
+    # Note that this approach has a couple of downsides. For example, you can
+    # only use it for cache, since Heroku wipes this directory between app
+    # restarts. This also means that deploying the app can cancel someone's
+    # uploading if you're using backgrounding. Also, by default you cannot
+    # generate URLs to files in the "tmp" directory, but you can with the
+    # download_endpoint plugin.
     class FileSystem
-      attr_reader :directory, :subdirectory, :host, :permissions
+      attr_reader :directory, :prefix, :host, :permissions
 
       # Initializes a storage for uploading to the filesystem.
       #
-      # :subdirectory
+      # :prefix
       # :  The directory relative to `directory` to which files will be stored,
       #    and it is included in the URL.
       #
       # :host
-      # :  URLs will by default be relative if `:subdirectory` is set, and you
+      # :  URLs will by default be relative if `:prefix` is set, and you
       #    can use this option to set a CDN host (e.g. `//abc123.cloudfront.net`).
       #
       # :permissions
@@ -73,10 +91,11 @@ class Shrine
       # :  By default empty folders inside the directory are automatically
       #    deleted, but if it happens that it causes too much load on the
       #    filesystem, you can set this option to `false`.
-      def initialize(directory, subdirectory: nil, host: nil, clean: true, permissions: nil)
-        if subdirectory
-          @subdirectory = Pathname(relative(subdirectory))
-          @directory = Pathname(directory).join(@subdirectory)
+      def initialize(directory, prefix: nil, host: nil, clean: true, permissions: nil, subdirectory: nil)
+        if prefix || subdirectory
+          warn "The :subdirectory option is deprecated and will be removed in Shrine 2. You should use :prefix instead." if subdirectory
+          @prefix = Pathname(relative(prefix || subdirectory))
+          @directory = Pathname(directory).join(@prefix)
         else
           @directory = Pathname(directory)
         end
@@ -91,13 +110,13 @@ class Shrine
 
       # Copies the file into the given location.
       def upload(io, id, metadata = {})
-        IO.copy_stream(io, path!(id)); io.rewind
+        IO.copy_stream(io, path!(id))
         path(id).chmod(permissions) if permissions
       end
 
       # Downloads the file from the given location, and returns a `Tempfile`.
       def download(id)
-        Down.copy_to_tempfile(id, open(id))
+        open(id) { |file| Down.copy_to_tempfile(id, file) }
       end
 
       # Moves the file to the given location. This gets called by the "moving"
@@ -120,8 +139,8 @@ class Shrine
       end
 
       # Opens the file on the given location in read mode.
-      def open(id)
-        path(id).open("rb")
+      def open(id, &block)
+        path(id).open("rb", &block)
       end
 
       # Returns the contents of the file as a String.
@@ -141,19 +160,12 @@ class Shrine
         clean(id) if clean?
       end
 
-      # If #subdirectory is present, returns the path relative to #directory,
+      # If #prefix is present, returns the path relative to #directory,
       # with an optional #host in front. Otherwise returns the full path to the
       # file (also with an optional #host).
       def url(id, **options)
-        if subdirectory
-          File.join(host || "", subdirectory, id)
-        else
-          if host
-            File.join(host, path(id))
-          else
-            path(id).to_s
-          end
-        end
+        path = (prefix ? relative_path(id) : path(id)).to_s
+        host ? host + path : path
       end
 
       # Without any options it deletes all files from the #directory (and this
@@ -202,6 +214,10 @@ class Shrine
         path(id)
       end
 
+      def relative_path(id)
+        "/" + prefix.join(id.gsub("/", File::SEPARATOR)).to_s
+      end
+
       def relative(path)
         path.sub(%r{^/}, "")
       end

data/lib/shrine/storage/linter.rb

@@ -17,62 +17,84 @@ class Shrine
 
   module Storage
     # Checks if the storage conforms to Shrine's specification. If the check
-    # fails a LintError is raised.
+    # fails, by default it raises a LintError, but you can also specify
+    # `action: :warn`.
     class Linter
-      def self.call(storage)
-        new(storage).call
+      def self.call(*args)
+        new(*args).call
       end
 
-      def initialize(storage)
+      def initialize(storage, action: :error)
         @storage = storage
-        @errors = []
+        @action  = action
+        @errors  = []
       end
 
-      def call
-        fakeio = FakeIO.new("image")
+      def call(io_factory = ->{FakeIO.new("image")})
+        storage.upload(io_factory.call, id = "foo.jpg", {"mime_type" => "image/jpeg"})
 
-        storage.upload(fakeio, "foo.jpg", {"mime_type" => "image/jpeg"})
-        error! "#upload doesn't rewind the file" if !(fakeio.read == "image")
-
-        file = storage.download("foo.jpg")
+        file = storage.download(id)
         error! "#download doesn't return a Tempfile" if !file.is_a?(Tempfile)
-        error! "#download doesn't return the uploaded file" if !(file.read == "image")
+        error! "#download returns an empty file" if file.read.empty?
+
+        error! "#open doesn't return a valid IO object" if !io?(storage.open(id))
+        error! "#read returns an empty string" if storage.read(id).empty?
+        error! "#exists? returns false for a file that was uploaded" if !storage.exists?(id)
+        error! "#url doesn't return a string" if !storage.url(id, {}).is_a?(String)
+
+        if storage.respond_to?(:stream)
+          content = ""
+          storage.stream(id) { |chunk| content << chunk }
+          error! "#stream does not yield any chunks" if content.empty?
+        end
+
+        storage.delete(id)
+        error! "#exists? returns true for a file that was deleted" if storage.exists?(id)
 
         if storage.respond_to?(:move)
           if storage.respond_to?(:movable?)
             error! "#movable? doesn't accept 2 arguments" if !(storage.method(:movable?).arity == 2)
             error! "#move doesn't accept 3 arguments" if !(storage.method(:move).arity == -3)
+
+            uploaded_file = uploader.upload(io_factory.call, location: "bar.jpg")
+
+            if storage.movable?(uploaded_file, "quux.jpg")
+              storage.move(uploaded_file, id = "quux.jpg")
+              error! "#exists? returns false for destination after #move" if !storage.exists?(id)
+              error! "#exists? returns true for source after #move" if storage.exists?(uploaded_file.id)
+            end
           else
-            error! "doesn't respond to #movable?" if !storage.respond_to?(:movable?)
+            error! "responds to #move but doesn't respond to #movable?" if !storage.respond_to?(:movable?)
           end
         end
 
-        if !io?(storage.open("foo.jpg"))
-          error! "#open doesn't return a valid IO object"
+        if storage.respond_to?(:multi_delete)
+          storage.upload(io_factory.call, id = "foo.jpg")
+          storage.multi_delete([id])
+          error! "#exists? returns true for a file that was multi-deleted" if storage.exists?(id)
         end
 
-        error! "#read doesn't return content of the uploaded file" if !(storage.read("foo.jpg") == "image")
-        error! "#exists? returns false for a file that was uploaded" if !storage.exists?("foo.jpg")
-        error! "#url doesn't return a string" if !storage.url("foo.jpg", {}).is_a?(String)
-
-        storage.delete("foo.jpg")
-        error! "#exists? returns true for a file that was deleted" if storage.exists?("foo.jpg")
-
         begin
           storage.clear!
           error! "#clear! should raise Shrine::Confirm unless :confirm is passed in"
         rescue Shrine::Confirm
         end
 
-        storage.upload(FakeIO.new("image"), "foo.jpg", {"mime_type" => "image/jpeg"})
+        storage.upload(io_factory.call, id = "foo.jpg")
         storage.clear!(:confirm)
-        error! "a file still #exists? after #clear! was called" if storage.exists?("foo.jpg")
+        error! "file still #exists? after #clear! was called" if storage.exists?(id)
 
-        raise LintError.new(@errors) if @errors.any?
+        raise LintError.new(@errors) if @errors.any? && @action == :error
       end
 
       private
 
+      def uploader
+        shrine = Class.new(Shrine)
+        shrine.storages[:storage] = storage
+        shrine.new(:storage)
+      end
+
       def io?(object)
         missing_methods = IO_METHODS.reject do |m, a|
           object.respond_to?(m) && [a.count, -1].include?(object.method(m).arity)
@@ -82,6 +104,7 @@ class Shrine
 
       def error!(message)
         @errors << message
+        warn(message) if @action.to_s.start_with?("warn")
       end
 
       attr_reader :storage

data/lib/shrine/storage/s3.rb

@@ -4,8 +4,12 @@ require "uri"
 
 class Shrine
   module Storage
-    # The S3 storage handles uploads to Amazon S3 service, and it is
-    # initialized with the following 4 required options:
+    # The S3 storage handles uploads to Amazon S3 service, it depends on the
+    # [aws-sdk] gem:
+    #
+    #     gem "aws-sdk", "~> 2.1"
+    #
+    # It is initialized with the following 4 required options:
     #
     #     Shrine::Storage::S3.new(
     #       access_key_id: "xyz",
@@ -23,12 +27,38 @@ class Shrine
     #     Shrine::Storage::S3.new(prefix: "cache", **s3_options)
     #     Shrine::Storage::S3.new(prefix: "store", **s3_options)
     #
+    # ## Upload options
+    #
+    # Sometimes you'll want to add additional upload options to all S3 uploads.
+    # You can do that by passing the `:upload` option:
+    #
+    #     Shrine::Storage::S3.new(
+    #       upload_options: {acl: "public-read", cache_control: "public, max-age=3600"},
+    #       **s3_options
+    #     )
+    #
+    # These options will be passed to aws-sdk's methods for [uploading],
+    # [copying] and [presigning].
+    #
+    # You can also forward additional upload options per upload with the
+    # `upload_options` plugin:
+    #
+    #     class MyUploader < Shrine
+    #       plugin :upload_options, store: ->(io, context) do
+    #         if context[:version] == :thumb
+    #           {acl: "public-read"}
+    #         else
+    #           {acl: "private"}
+    #         end
+    #       end
+    #     end
+    #
     # ## CDN
     #
     # If you're using a CDN with S3 like Amazon CloudFront, you can specify
     # the `:host` option to have all your URLs use the CDN host:
     #
-    #     Shrine::Storage::S3.new(host: "//abc123.cloudfront.net", **s3_options)
+    #     Shrine::Storage::S3.new(host: "http://abc123.cloudfront.net", **s3_options)
     #
     # ## Clearing cache
     #
@@ -36,8 +66,13 @@ class Shrine
     # delete old files which aren't used anymore. S3 has a built-in way to do
     # this, read [this article](http://docs.aws.amazon.com/AmazonS3/latest/UG/lifecycle-configuration-bucket-no-versioning.html)
     # for instructions.
+    #
+    # [uploading]: http://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Object.html#put-instance_method
+    # [copying]: http://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Object.html#copy_from-instance_method
+    # [presigning]: http://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Object.html#presigned_post-instance_method
+    # [aws-sdk]: https://github.com/aws/aws-sdk-ruby
     class S3
-      attr_reader :prefix, :bucket, :s3, :host
+      attr_reader :prefix, :bucket, :s3, :host, :upload_options
 
       # Initializes a storage for uploading to S3.
       #
@@ -51,14 +86,26 @@ class Shrine
       # :   "Folder" name inside the bucket to store files into.
       #
       # :host
-      # :   This option is used for setting CDNs, e.g. it can be set to `//abc123.cloudfront.net`.
+      # :   This option is used for setting CDNs, e.g. it can be set to
+      #      `//abc123.cloudfront.net`.
       #
-      # All other options are forwarded to [`Aws::S3::Client#initialize`](http://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html#initialize-instance_method).
-      def initialize(bucket:, prefix: nil, host: nil, **s3_options)
+      # :upload_options
+      # :   Additional options that will be used for uploading files, they will
+      #     be passed to [`Aws::S3::Object#put`], [`Aws::S3::Object#copy_from`]
+      #     and [`Aws::S3::Bucket#presigned_post`].
+      #
+      # All other options are forwarded to [`Aws::S3::Client#initialize`].
+      #
+      # [`Aws::S3::Object#put`]: http://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Object.html#put-instance_method
+      # [`Aws::S3::Object#copy_from`]: http://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Object.html#copy_from-instance_method
+      # [`Aws::S3::Bucket#presigned_post`]: http://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Object.html#presigned_post-instance_method
+      # [`Aws::S3::Client#initialize`]: http://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html#initialize-instance_method
+      def initialize(bucket:, prefix: nil, host: nil, upload_options: {}, **s3_options)
         @prefix = prefix
         @s3 = Aws::S3::Resource.new(**s3_options)
         @bucket = @s3.bucket(bucket)
         @host = host
+        @upload_options = upload_options
       end
 
       # If the file is an UploadedFile from S3, issues a COPY command, otherwise
@@ -68,12 +115,13 @@ class Shrine
       # by default S3 sets everything to "application/octet-stream".
       def upload(io, id, metadata = {})
         options = {content_type: metadata["mime_type"]}
+        options.update(upload_options)
+        options.update(metadata.delete("s3") || {})
 
         if copyable?(io)
-          object(id).copy_from(io.storage.object(io.id), **options)
+          copy(io, id, **options)
         else
-          object(id).put(body: io, **options)
-          io.rewind
+          put(io, id, **options)
         end
       end
 
@@ -82,6 +130,11 @@ class Shrine
         Down.download(url(id))
       end
 
+      # Streams the object from S3, yielding downloaded chunks.
+      def stream(id)
+        object(id).get { |chunk| yield chunk }
+      end
+
       # Alias for #download.
       def open(id)
         download(id)
@@ -89,7 +142,7 @@ class Shrine
 
       # Returns the contents of the file as a String.
       def read(id)
-        object(id).get.body.read
+        object(id).get.body.string
       end
 
       # Returns true file exists on S3.
@@ -102,11 +155,14 @@ class Shrine
         object(id).delete
       end
 
-      # This is called when multiple files are being deleted at once. Issues
-      # a single MULTI DELETE command.
+      # This is called when multiple files are being deleted at once. Issues a
+      # single MULTI DELETE command for each 1000 objects (S3 delete limit).
       def multi_delete(ids)
-        objects = ids.map { |id| {key: object(id).key} }
+        objects = ids.take(1000).map { |id| {key: object(id).key} }
         bucket.delete_objects(delete: {objects: objects})
+
+        rest = Array(ids[1000..-1])
+        multi_delete(rest) unless rest.empty?
       end
 
       # Returns the presigned URL to the file.
@@ -120,13 +176,7 @@ class Shrine
       # :  Creates an unsigned version of the URL (requires setting appropriate
       #    permissions on the S3 bucket).
       #
-      # options
-      # :  All other optione are forwarded to 
-      # If `download: true` is passed,
-      # returns a forced download link. If `public: true` is passed, it returns
-      # an unsigned S3 URL. All other options are forwarded to
-      # [`Aws::S3::Object#presigned_url`], so take a look there for the
-      # complete list of additional options.
+      # All other options are forwarded to [`Aws::S3::Object#presigned_url`].
       #
       # [`Aws::S3::Object#presigned_url`]: http://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Object.html#presigned_url-instance_method
       def url(id, download: nil, public: nil, **options)
@@ -154,7 +204,8 @@ class Shrine
       #
       # [`Aws::S3::Bucket#presigned_post`]: http://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Bucket.html#presigned_post-instance_method
       def presign(id, **options)
-        @bucket.presigned_post(key: object(id).key, **options)
+        options = upload_options.merge(options)
+        object(id).presigned_post(options)
       end
 
       protected
@@ -171,12 +222,28 @@ class Shrine
 
       private
 
+      # Copies an existing S3 object to a new location.
+      def copy(io, id, **options)
+        options.update(multipart_copy: true) if large?(io)
+        object(id).copy_from(io.storage.object(io.id), **options)
+      end
+
+      # Uploads the file to S3.
+      def put(io, id, **options)
+        object(id).put(body: io, **options)
+      end
+
       # The file is copyable if it's on S3 and on the same Amazon account.
       def copyable?(io)
         io.respond_to?(:storage) &&
         io.storage.is_a?(Storage::S3) &&
         io.storage.access_key_id == access_key_id
       end
+
+      # Amazon requires multipart copy from S3 objects larger than 5 GB.
+      def large?(io)
+        io.size >= 5*1024*1024*1024 # 5GB
+      end
     end
   end
 end