uppy-s3_multipart 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: c747885d4a3c5e57af416694daa5ad809831f42a73e03ea0f72c6be2106bca4b
+  data.tar.gz: 3dfdb337a8d1ff8f9096018233435d4e3a8b1cc71092ca10fae0582b660e1d1a
+SHA512:
+  metadata.gz: 8366713f060f860ecaaf98177751471a3cbab32bcb4e29414fea742a13bd90ffa6b46845371a10c4deb90273bdfe0a5c5c6db68f579ad6d502f479dcd2fbc4bc
+  data.tar.gz: e2aa4485b564ba814ddea95b0d77d8872b9afde9baedbd2c0c298cd5434054c2bc56ce6eac48722cddd9719a7bc3f44ccecba696a8f34a1bd86297c72fbec47d

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 Janko Marohnić
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,309 @@
+# Uppy::S3Multipart
+
+Provides a Rack application that implements endpoints for the [AwsS3Multipart]
+Uppy plugin. This enables multipart uploads directly to S3, which is
+recommended when dealing with large files, as it allows resuming interrupted
+uploads.
+
+## Installation
+
+Add the gem to your Gemfile:
+
+```rb
+gem "uppy-s3_multipart"
+```
+
+## Setup
+
+Once you've created your S3 bucket, you need to set up CORS for it. The
+following script sets up minimal CORS configuration needed for multipart
+uploads on your bucket using the `aws-sdk-s3` gem:
+
+```rb
+require "aws-sdk-s3"
+
+client = Aws::S3::Client.new(
+  access_key_id:     "<YOUR KEY>",
+  secret_access_key: "<YOUR SECRET>",
+  region:            "<REGION>",
+)
+
+client.put_bucket_cors(
+  bucket: "<YOUR BUCKET>",
+  cors_configuration: {
+    cors_rules: [{
+      allowed_headers: ["Authorization", "Content-Type", "Origin", "ETag"],
+      allowed_methods: ["GET", "POST", "PUT", "DELETE"],
+      allowed_origins: ["*"],
+      max_age_seconds: 3000,
+    }]
+  }
+)
+```
+
+## Usage
+
+This gem provides a Rack application that you can mount inside your main
+application. If you're using [Shrine], you can initialize the Rack application
+via the `uppy_s3_multipart` Shrine plugin, otherwise you can initialize it
+directly.
+
+### Shrine
+
+In the initializer load the `uppy_s3_multipart` plugin:
+
+```rb
+require "shrine"
+require "shrine/storage/s3"
+
+Shrine.storages = {
+  cache: Shrine::Storage::S3.new(...),
+  store: Shrine::Storage::S3.new(...),
+}
+
+# ...
+Shrine.plugin :uppy_s3_multipart # load the plugin
+```
+
+The plugin will provide a `Shrine.uppy_s3_multipart` method, which returns an
+instance of `Uppy::S3Multipart::App`, which is a Rack app that you can mount
+inside your main application:
+
+```rb
+# Rails (config/routes.rb)
+Rails.application.routes.draw do
+  mount Shrine.uppy_s3_multipart(:cache) => "/s3"
+end
+
+# Rack (config.ru)
+map "/s3" do
+  run Shrine.uppy_s3_multipart(:cache)
+end
+```
+
+This will add the routes that the `AwsS3Multipart` Uppy plugin expects:
+
+```
+POST   /s3/multipart
+GET    /s3/multipart/:uploadId
+GET    /s3/multipart/:uploadId/:partNumber
+POST   /s3/multipart/:uploadId/complete
+DELETE /s3/multipart/:uploadId
+```
+
+Finally, in your Uppy configuration pass your app's URL as the `serverUrl`:
+
+```js
+// ...
+uppy.use(Uppy.AwsS3Multipart, {
+  serverUrl: "https://your-app.com/",
+})
+```
+
+Both the plugin and method accepts `:options` for specifying additional options
+to the aws-sdk calls (read further for more details on these options):
+
+```rb
+Shrine.plugin :uppy_s3_multipart, options: {
+  create_multipart_upload: { acl: "public-read" } # static
+}
+
+# OR
+
+Shrine.uppy_s3_multipart(:cache, options: {
+  create_multipart_upload: -> (request) { { acl: "public-read" } } # dynamic
+})
+```
+
+### Standalone
+
+You can also initialize `Uppy::S3Multipart::App` directly:
+
+```rb
+require "uppy/s3_multipart"
+
+resource = Aws::S3::Resource.new(
+  access_key_id:     "...",
+  secret_access_key: "...",
+  region:            "...",
+)
+
+bucket = resource.bucket("my-bucket")
+
+UPPY_S3_MULTIPART_APP = Uppy::S3Multipart::App.new(bucket: bucket)
+```
+
+and mount it in your app in the same way:
+
+```rb
+# Rails (config/routes.rb)
+Rails.application.routes.draw do
+  mount UPPY_S3_MULTIPART_APP => "/s3"
+end
+
+# Rack (config.ru)
+map "/s3" do
+  run UPPY_S3_MULTIPART_APP
+end
+```
+
+In your Uppy configuration point the `serverUrl` to your application:
+
+```js
+// ...
+uppy.use(Uppy.AwsS3Multipart, {
+  serverUrl: "https://your-app.com/",
+})
+```
+
+The `Uppy::S3Mutipart::App` initializer accepts `:options` for specifying
+additional options to the aws-sdk calls (read further for more details on these
+options):
+
+```rb
+Uppy::S3Multipart::App.new(bucket: bucket, options: {
+  create_multipart_upload: { acl: "public-read" }
+})
+
+# OR
+
+Uppy::S3Multipart::App.new(bucket: bucket, options: {
+  create_multipart_upload: -> (request) { { acl: "public-read" } }
+})
+```
+
+### Custom implementation
+
+If you would rather implement the endpoints yourself, you can utilize
+`Uppy::S3Multipart::Client` to make S3 requests.
+
+```rb
+require "uppy/s3_multipart/client"
+
+client = Uppy::S3Multipart::Client.new(bucket: bucket)
+```
+
+#### `create_multipart_upload`
+
+Initiates a new multipart upload.
+
+```rb
+client.create_multipart_upload(key: "foo", **options)
+#=> { upload_id: "MultipartUploadId", key: "foo" }
+```
+
+Accepts:
+
+* `:key` -- object key
+* additional options for [`Aws::S3::Client#create_multipart_upload`]
+
+Returns:
+
+* `:upload_id` -- id of the created multipart upload
+* `:key` -- object key
+
+#### `#list_parts`
+
+Retrieves currently uploaded parts of a multipart upload.
+
+```rb
+client.list_parts(upload_id: "MultipartUploadId", key: "foo", **options)
+#=> [ { part_number: 1, size: 5402383, etag: "etag1" },
+#     { part_number: 2, size: 5982742, etag: "etag2" },
+#     ... ]
+```
+
+Accepts:
+
+* `:upload_id` -- multipart upload id
+* `:key` -- object key
+* additional options for [`Aws::S3::Client#list_parts`]
+
+Returns:
+
+* array of parts
+
+  - `:part_number` -- position of the part
+  - `:size` -- filesize of the part
+  - `:etag` -- etag of the part
+
+#### `#prepare_upload_part`
+
+Returns the endpoint that should be used for uploading a new multipart part.
+
+```rb
+client.prepare_upload_part(upload_id: "MultipartUploadId", key: "foo", part_number: 1, **options)
+#=> { url: "https://my-bucket.s3.amazonaws.com/foo?partNumber=1&uploadId=MultipartUploadId&..." }
+```
+
+Accepts:
+
+* `:upload_id` -- multipart upload id
+* `:key` -- object key
+* `:part_number` -- number of the next part
+* additional options for [`Aws::S3::Client#upload_part`] and [`Aws::S3::Presigner#presigned_url`]
+
+Returns:
+
+* `:url` -- endpoint that should be used for uploading a new multipart part via a `PUT` request
+
+#### `#complete_multipart_upload`
+
+Finalizes the multipart upload and returns URL to the object.
+
+```rb
+client.complete_multipart_upload(upload_id: upload_id, key: key, parts: [{ part_number: 1, etag: "etag1" }], **options)
+#=> { location: "https://my-bucket.s3.amazonaws.com/foo?..." }
+```
+
+Accepts:
+
+* `:upload_id` -- multipart upload id
+* `:key` -- object key
+* `:parts` -- list of all uploaded parts, consisting of `:part_number` and `:etag`
+* additional options for [`Aws::S3::Client#complete_multipart_upload`]
+
+Returns:
+
+* `:location` -- URL to the uploaded object
+
+#### `#abort_multipart_upload`
+
+Aborts the multipart upload, removing all parts uploaded so far.
+
+```rb
+client.abort_multipart_upload(upload_id: upload_id, key: key, **options)
+#=> {}
+```
+
+Accepts:
+
+* `:upload_id` -- multipart upload id
+* `:key` -- object key
+* additional options for [`Aws::S3::Client#abort_multipart_upload`]
+
+## Contributing
+
+You can run the test suite with
+
+```
+$ bundle exec rake test
+```
+
+This project is intended to be a safe, welcoming space for collaboration, and
+contributors are expected to adhere to the [Contributor
+Covenant](http://contributor-covenant.org) code of conduct.
+
+## License
+
+The gem is available as open source under the terms of the [MIT
+License](https://opensource.org/licenses/MIT).
+
+[AwsS3Multipart]: https://uppy.io/docs/aws-s3-multipart/
+[Shrine]: https://shrinerb.com
+[`Aws::S3::Client#create_multipart_upload`]: https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html#create_multipart_upload-instance_method
+[`Aws::S3::Client#list_parts`]: https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html#list_parts-instance_method
+[`Aws::S3::Client#upload_part`]: https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html#upload_part-instance_method
+[`Aws::S3::Presigner#presigned_url`]: https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Presigner.html#presigned_url-instance_method
+[`Aws::S3::Client#complete_multipart_upload`]: https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html#complete_multipart_upload-instance_method
+[`Aws::S3::Client#abort_multipart_upload`]: https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html#abort_multipart_upload-instance_method

data/lib/shrine/plugins/uppy_s3_multipart.rb

@@ -0,0 +1,28 @@
+require "uppy/s3_multipart"
+
+require "securerandom"
+
+class Shrine
+  module Plugins
+    module UppyS3Multipart
+      def self.configure(uploader, options = {})
+        uploader.opts[:uppy_s3_multipart_options] = (uploader.opts[:uppy_s3_multipart_options] || {}).merge(options[:options] || {})
+      end
+
+      module ClassMethods
+        def uppy_s3_multipart(storage_key, **options)
+          s3 = find_storage(storage_key)
+
+          ::Uppy::S3Multipart::App.new(
+            bucket:  s3.bucket,
+            prefix:  s3.prefix,
+            options: opts[:uppy_s3_multipart_options],
+            **options
+          )
+        end
+      end
+    end
+
+    register_plugin(:uppy_s3_multipart, UppyS3Multipart)
+  end
+end

data/lib/uppy-s3_multipart.rb

@@ -0,0 +1 @@
+require "uppy/s3_multipart"

data/lib/uppy/s3_multipart.rb

@@ -0,0 +1,2 @@
+require "uppy/s3_multipart/app"
+require "uppy/s3_multipart/client"

data/lib/uppy/s3_multipart/app.rb

@@ -0,0 +1,115 @@
+require "uppy/s3_multipart/client"
+
+require "roda"
+
+require "securerandom"
+require "cgi"
+
+module Uppy
+  module S3Multipart
+    class App
+      def initialize(bucket:, prefix: nil, options: {})
+        @router = Class.new(Router)
+        @router.opts[:client]  = Client.new(bucket: bucket)
+        @router.opts[:prefix]  = prefix
+        @router.opts[:options] = options
+      end
+
+      def call(env)
+        @router.call(env)
+      end
+
+      class Router < Roda
+        plugin :all_verbs
+        plugin :json
+        plugin :json_parser
+        plugin :halt
+
+        route do |r|
+          # POST /multipart
+          r.post "multipart" do
+            content_type = r.params["type"]
+            filename     = r.params["filename"]
+
+            extension = File.extname(filename.to_s)
+            key       = SecureRandom.hex + extension
+            key       = "#{opts[:prefix]}/#{key}" if opts[:prefix]
+
+            # CGI-escape the filename because aws-sdk's signature calculator trips on special characters
+            content_disposition = "inline; filename=\"#{CGI.escape(filename)}\"" if filename
+
+            result = client_call(:create_multipart_upload, key: key, content_type: content_type, content_disposition: content_disposition)
+
+            { uploadId: result.fetch(:upload_id), key: result.fetch(:key) }
+          end
+
+          # GET /multipart/:uploadId
+          r.get "multipart", String do |upload_id|
+            key = param!("key")
+
+            result = client_call(:list_parts, upload_id: upload_id, key: key)
+
+            result.map do |part|
+              { PartNumber: part.fetch(:part_number), Size: part.fetch(:size), ETag: part.fetch(:etag) }
+            end
+          end
+
+          # GET /multipart/:uploadId/:partNumber
+          r.get "multipart", String, String do |upload_id, part_number|
+            key = param!("key")
+
+            result = client_call(:prepare_upload_part, upload_id: upload_id, key: key, part_number: part_number)
+
+            { url: result.fetch(:url) }
+          end
+
+          # POST /multipart/:uploadId/complete
+          r.post "multipart", String, "complete" do |upload_id|
+            key   = param!("key")
+            parts = param!("parts")
+
+            parts = parts.map do |part|
+              begin
+                { part_number: part.fetch("PartNumber"), etag: part.fetch("ETag") }
+              rescue KeyError
+                r.halt 400, { error: "At least one part is missing \"PartNumber\" or \"ETag\" field" }
+              end
+            end
+
+            result = client_call(:complete_multipart_upload, upload_id: upload_id, key: key, parts: parts)
+
+            { location: result.fetch(:location) }
+          end
+
+          # DELETE /multipart/:uploadId
+          r.delete "multipart", String do |upload_id|
+            key = param!("key")
+
+            client_call(:abort_multipart_upload, upload_id: upload_id, key: key)
+
+            {}
+          end
+        end
+
+        def client_call(operation, **options)
+          client = opts[:client]
+
+          overrides = opts[:options][operation] || {}
+          overrides = overrides.call(request) if overrides.respond_to?(:call)
+
+          options = options.merge(overrides)
+
+          client.send(operation, **options)
+        end
+
+        def param!(name)
+          value = request.params[name]
+
+          request.halt 400, { error: "Missing \"#{name}\" parameter" } if value.nil?
+
+          value
+        end
+      end
+    end
+  end
+end

data/lib/uppy/s3_multipart/client.rb

@@ -0,0 +1,81 @@
+require "aws-sdk-s3"
+
+module Uppy
+  module S3Multipart
+    class Client
+      attr_reader :bucket
+
+      def initialize(bucket:)
+        @bucket = bucket
+      end
+
+      def create_multipart_upload(key:, **options)
+        multipart_upload = object(key).initiate_multipart_upload(**options)
+
+        { upload_id: multipart_upload.id, key: multipart_upload.object_key }
+      end
+
+      def list_parts(upload_id:, key:, **options)
+        multipart_upload = multipart_upload(upload_id, key)
+        multipart_parts  = multipart_upload.parts(**options).to_a
+
+        multipart_parts.map do |part|
+          { part_number: part.part_number, size: part.size, etag: part.etag }
+        end
+      end
+
+      def prepare_upload_part(upload_id:, key:, part_number:, **options)
+        presigned_url = presigner.presigned_url "upload_part",
+          bucket: bucket.name,
+          key: object(key).key,
+          upload_id: upload_id,
+          part_number: part_number,
+          body: "",
+          **options
+
+        { url: presigned_url }
+      end
+
+      def complete_multipart_upload(upload_id:, key:, parts:, **options)
+        multipart_upload = multipart_upload(upload_id, key)
+        multipart_upload.complete(
+          multipart_upload: { parts: parts },
+          **options
+        )
+
+        { location: object(key).presigned_url(:get) }
+      end
+
+      def abort_multipart_upload(upload_id:, key:, **options)
+        multipart_upload = multipart_upload(upload_id, key)
+
+        # aws-sdk-s3 docs recommend retrying the abort in case the multipart
+        # upload still has parts
+        loop do
+          multipart_upload.abort(**options)
+          break unless multipart_upload.parts.any?
+        end
+
+        {}
+      end
+
+      private
+
+      def multipart_upload(upload_id, key)
+        object(key).multipart_upload(upload_id)
+      end
+
+      def object(key)
+        bucket.object(key)
+      end
+
+      def presigner
+        Aws::S3::Presigner.new(client: client)
+      end
+
+      def client
+        bucket.client
+      end
+    end
+  end
+end

data/uppy-s3_multipart.gemspec

@@ -0,0 +1,24 @@
+Gem::Specification.new do |gem|
+  gem.name          = "uppy-s3_multipart"
+  gem.version       = "0.1.0"
+
+  gem.required_ruby_version = ">= 2.2"
+
+  gem.summary      = "Provides a Rack application that implements endpoints for the AwsS3Multipart Uppy plugin."
+  gem.homepage     = "https://github.com/janko-m/uppy-s3_multipart"
+  gem.authors      = ["Janko Marohnić"]
+  gem.email        = ["janko.marohnic@gmail.com"]
+  gem.license      = "MIT"
+
+  gem.files        = Dir["README.md", "LICENSE.txt", "lib/**/*.rb", "*.gemspec"]
+  gem.require_path = "lib"
+
+  gem.add_dependency "roda", ">= 2.27", "< 4"
+  gem.add_dependency "aws-sdk-s3", "~> 1.0"
+
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "minitest"
+  gem.add_development_dependency "rack-test_app"
+  gem.add_development_dependency "shrine", "~> 2.0"
+  gem.add_development_dependency "aws-sdk-core", "~> 3.23"
+end

metadata

@@ -0,0 +1,157 @@
+--- !ruby/object:Gem::Specification
+name: uppy-s3_multipart
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Janko Marohnić
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-08-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: roda
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.27'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.27'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rack-test_app
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: shrine
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.23'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.23'
+description: 
+email:
+- janko.marohnic@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE.txt
+- README.md
+- lib/shrine/plugins/uppy_s3_multipart.rb
+- lib/uppy-s3_multipart.rb
+- lib/uppy/s3_multipart.rb
+- lib/uppy/s3_multipart/app.rb
+- lib/uppy/s3_multipart/client.rb
+- uppy-s3_multipart.gemspec
+homepage: https://github.com/janko-m/uppy-s3_multipart
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.2'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.6
+signing_key: 
+specification_version: 4
+summary: Provides a Rack application that implements endpoints for the AwsS3Multipart
+  Uppy plugin.
+test_files: []