tus-server 2.2.1 → 2.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 316f557990315ae072a266cb92bd5acdf4a5e930f734fc7886cb8c2cc2ec2eb3
-  data.tar.gz: a98df5216aa495f2f18bad0f61b2ed7797f714139991f700fdb403b94abd0271
+  metadata.gz: c07ca8df0a0b881580058e119e27a4cf63efbd0854018d00d8f3cd3f7a093b04
+  data.tar.gz: ea0da9293bee4cee32f77ce5383adc389d34d745a025a01d570ff971b21f2a0d
 SHA512:
-  metadata.gz: '02662191fa1b2e8df42bbba1a5f32d1b70ae3623cc89736ad4bbaa4c46cbc97364c0c4edd3c5fdfc348611bbca76132e7c71d6c785e953aa6f556f986755680b'
-  data.tar.gz: 6ac96ebc6710e231dabaa66faf8fced5fbf80faceb12f4992f3afa2235d4908f40649f00e794e0923a2311f0ee91ed2f4df0e7adf8c19321f1554f451010d2f8
+  metadata.gz: 1284b1b2af5e29ccbd5ae20658da75f0ad8de5d11c2198ed98704f120aa8ebcbe6f9a9815d728d518dd12ea16046725733055a3868b8a2e0685985fedca4fa4f
+  data.tar.gz: 0a9779bc5f55e685329a7cc76ed071ecf4e73b574609da3db298303697da898cd32d4a7da3e41f4e7ef59b273abfd69c79f03ec9aecf94d754ef6ccbbcdb08cf

data/CHANGELOG.md

@@ -1,120 +1,152 @@
+## 2.3.0 (2019-05-14)
+
+* Allow uploading files larger than 50 GB to S3 storage by scaling the part size according to upload length (@janko)
+
+* Allow S3 limits to be overriden via the `:limits` option (@janko)
+
+* Reject uploads larger than 5 TB in `S3#create_file` (@janko)
+
+* Remove superfluous `#head_object` operation from `S3#concatenate` (@janko)
+
+* Validate that partial uploads are finished on the concatenation request (@janko)
+
+* Unparallelize validating partial uploads for concatentation (@janko)
+
+* Move total length calculation for concatenation into `Tus::Server` (@janko)
+
+* Allow `Upload-Concat` and `Upload-Defer-Length` headers in CORS headers (@janko)
+
+* Always override `:content_disposition` option with the value from `:upload_options` in `S3#create_file` (@janko)
+
+* Apply `name`/`type` metadata to `:content_type`/`:content_disposition` options in `S3#create_file` (@janko)
+
+* Use `content_disposition` gem for `:content_disposition` option in `S3#create_file` (@janko)
+
+* Remove checks whether multipart upload was successfully aborted in S3 storage (@janko)
+
+* Don't return any `Content-Type` when type is not specified in metadata (@janko)
+
+* Add `ETag` header to download endpoint to prevent `Rack::ETag` buffering file content (@janko)
+
+* Take `:prefix` into account in `Tus::Storage::S3#expire_files` (@janko)
+
 ## 2.2.1 (2018-12-19)
 
-* Use `content_disposition` gem to generate `Content-Disposition` in download endpoint (@janko-m)
+* Use `content_disposition` gem to generate `Content-Disposition` in download endpoint (@janko)
 
 ## 2.2.0 (2018-12-02)
 
-* Add `before_create`, `after_create`, `after_finish`, and `after_terminate` hooks (@janko-m)
+* Add `before_create`, `after_create`, `after_finish`, and `after_terminate` hooks (@janko)
 
-* Rename `Tus::Info#concatenation?` to `Tus::Info#final?` (@janko-m)
+* Rename `Tus::Info#concatenation?` to `Tus::Info#final?` (@janko)
 
-* Use `Storage#concurrency` for parallelized retrieval of partial uploads in `Upload-Concat` validation (@janko-m)
+* Use `Storage#concurrency` for parallelized retrieval of partial uploads in `Upload-Concat` validation (@janko)
 
-* Replace `:thread_count` with `:concurrency` in S3 storage (@janko-m)
+* Replace `:thread_count` with `:concurrency` in S3 storage (@janko)
 
-* Validate that sum of partial uploads doesn't exceed `Tus-Max-Size` on concatenation (@janko-m)
+* Validate that sum of partial uploads doesn't exceed `Tus-Max-Size` on concatenation (@janko)
 
-* Drop MRI 2.2 support (@janko-m)
+* Drop MRI 2.2 support (@janko)
 
-* Accept absolute URLs of partial uploads when creating a final upload (@janko-m)
+* Accept absolute URLs of partial uploads when creating a final upload (@janko)
 
 ## 2.1.2 (2018-10-21)
 
-* Make tus-ruby-server fully work with non-rewindable Rack input (@janko-m)
+* Make tus-ruby-server fully work with non-rewindable Rack input (@janko)
 
 ## 2.1.1 (2018-05-26)
 
-* Rename `:download_url` option to `:redirect_download` (@janko-m)
+* Rename `:download_url` option to `:redirect_download` (@janko)
 
 ## 2.1.0 (2018-05-15)
 
-* Add `:download_url` server option for redirecting to a download URL (@janko-m)
+* Add `:download_url` server option for redirecting to a download URL (@janko)
 
-* Allow application servers to serve files stored on disk via the `Rack::Sendfile` middleware (@janko-m)
+* Allow application servers to serve files stored on disk via the `Rack::Sendfile` middleware (@janko)
 
-* Reject `Upload-Metadata` which contains key-value pairs separated by spaces (@janko-m)
+* Reject `Upload-Metadata` which contains key-value pairs separated by spaces (@janko)
 
-* Don't overwite info file if it already exists in `Tus::Storage::FileSystem` (@janko-m)
+* Don't overwite info file if it already exists in `Tus::Storage::FileSystem` (@janko)
 
 ## 2.0.2 (2017-12-24)
 
-* Handle `name` and `type` metadata for Uppy compatibility (@janko-m)
+* Handle `name` and `type` metadata for Uppy compatibility (@janko)
 
 ## 2.0.1 (2017-11-13)
 
-* Add back support for Roda 2.x (@janko-m)
+* Add back support for Roda 2.x (@janko)
 
 ## 2.0.0 (2017-11-13)
 
-* Upgrade to Roda 3 (@janko-m)
+* Upgrade to Roda 3 (@janko)
 
-* Remove deprecated support for aws-sdk 2.x in `Tus::Storage::S3` (@janko-m)
+* Remove deprecated support for aws-sdk 2.x in `Tus::Storage::S3` (@janko)
 
-* Drop official support for MRI 2.1 (@janko-m)
+* Drop official support for MRI 2.1 (@janko)
 
-* Add generic `Tus::Response` class that storages can use (@janko-m)
+* Add generic `Tus::Response` class that storages can use (@janko)
 
-* Remove `Tus::Response#length` (@janko-m)
+* Remove `Tus::Response#length` (@janko)
 
-* Remove deprecated Goliath integration (@janko-m)
+* Remove deprecated Goliath integration (@janko)
 
-* Return `400 Bad Request` instead of `404 Not Found` when some partial uploads are missing in a concatenation request (@janko-m)
+* Return `400 Bad Request` instead of `404 Not Found` when some partial uploads are missing in a concatenation request (@janko)
 
-* Use Rack directly instead of Roda's `streaming` plugin for downloding (@janko-m)
+* Use Rack directly instead of Roda's `streaming` plugin for downloding (@janko)
 
 ## 1.2.1 (2017-11-05)
 
-* Improve communication when handling `aws-sdk 2.x` fallback in `Tus::Storage::S3` (@janko-m)
+* Improve communication when handling `aws-sdk 2.x` fallback in `Tus::Storage::S3` (@janko)
 
 ## 1.2.0 (2017-09-18)
 
-* Deprecate `aws-sdk` 2.x in favour of the new `aws-sdk-s3` gem (@janko-m)
+* Deprecate `aws-sdk` 2.x in favour of the new `aws-sdk-s3` gem (@janko)
 
 ## 1.1.3 (2017-09-17)
 
-* Return `Accept-Ranges: bytes` response header in download endpoint (@janko-m)
+* Return `Accept-Ranges: bytes` response header in download endpoint (@janko)
 
 ## 1.1.2 (2017-09-12)
 
-* Add support for the new `aws-sdk-s3` gem (@janko-m)
+* Add support for the new `aws-sdk-s3` gem (@janko)
 
 ## 1.1.1 (2017-07-23)
 
-* Restore backwards compatibility with MRI 2.1 and MRI 2.2 that was broken in previous release (@janko-m)
+* Restore backwards compatibility with MRI 2.1 and MRI 2.2 that was broken in previous release (@janko)
 
 ## 1.1.0 (2017-07-23)
 
-* Ignore retryable networking errors in `Tus::Storage::S3#patch_file` for resiliency (@janko-m)
+* Ignore retryable networking errors in `Tus::Storage::S3#patch_file` for resiliency (@janko)
 
-* Deprecate `Tus::Server::Goliath` in favour of [goliath-rack_proxy](https://github.com/janko-m/goliath-rack_proxy) (@janko-m)
+* Deprecate `Tus::Server::Goliath` in favour of [goliath-rack_proxy](https://github.com/janko/goliath-rack_proxy) (@janko)
 
-* Reduce string allocations in MRI 2.3+ with `frozen-string-literal: true` magic comments (@janko-m)
+* Reduce string allocations in MRI 2.3+ with `frozen-string-literal: true` magic comments (@janko)
 
 ## 1.0.0 (2017-07-17)
 
-* Add Goliath integration (@janko-m)
+* Add Goliath integration (@janko)
 
-* [BREAKING] Save data in `"#{uid}"` instead of `"#{uid}.file"` in `Tus::Storage::Filesystem` (@janko-m)
+* [BREAKING] Save data in `"#{uid}"` instead of `"#{uid}.file"` in `Tus::Storage::Filesystem` (@janko)
 
-* Modify S3 storage to cache chunks into memory instead of disk, which reduces disk IO (@janko-m)
+* Modify S3 storage to cache chunks into memory instead of disk, which reduces disk IO (@janko)
 
-* [BREAKING] Require each storage to return the number of bytes uploaded in `#patch_file` (@janko-m)
+* [BREAKING] Require each storage to return the number of bytes uploaded in `#patch_file` (@janko)
 
-* Make S3 storage upload all received data from `tus-js-client` that doesn't have max chunk size configured (@janko-m)
+* Make S3 storage upload all received data from `tus-js-client` that doesn't have max chunk size configured (@janko)
 
-* Verify that all partial uploads have `Upload-Concat: partial` before concatenation (@janko-m)
+* Verify that all partial uploads have `Upload-Concat: partial` before concatenation (@janko)
 
-* Include CORS and tus response headers in 404 responses (@janko-m)
+* Include CORS and tus response headers in 404 responses (@janko)
 
-* Improve streaming on dynamic Rack inputs such as `Unicorn::TeeInput` for S3 and Gridfs storage (@janko-m)
+* Improve streaming on dynamic Rack inputs such as `Unicorn::TeeInput` for S3 and Gridfs storage (@janko)
 
-* Terminate HTTP connection to S3 when response is closed (@janko-m)
+* Terminate HTTP connection to S3 when response is closed (@janko)
 
-* Allow `Transfer-Encoding: chunked` to be used, meaning `Content-Length` can be blank (@janko-m)
+* Allow `Transfer-Encoding: chunked` to be used, meaning `Content-Length` can be blank (@janko)
 
-* Remove newlines from the base64-encoded CRC32 signature (@janko-m)
+* Remove newlines from the base64-encoded CRC32 signature (@janko)
 
-* Lazily require `digest`, `zlib`, and `base64` standard libraries (@janko-m)
+* Lazily require `digest`, `zlib`, and `base64` standard libraries (@janko)
 
 ## 0.10.2 (2017-04-19)
 
@@ -122,94 +154,94 @@
 
 ## 0.10.1 (2017-04-13)
 
-* Fix download endpoint returning incorrect response body in some cases in development (@janko-m)
+* Fix download endpoint returning incorrect response body in some cases in development (@janko)
 
-* Remove `concatenation-unfinished` from list of supported extensions (@janko-m)
+* Remove `concatenation-unfinished` from list of supported extensions (@janko)
 
 ## 0.10.0 (2017-03-27)
 
-* Fix invalid `Content-Disposition` header in GET requests to due mutation of `Tus::Server.opts[:disposition]` (@janko-m)
+* Fix invalid `Content-Disposition` header in GET requests to due mutation of `Tus::Server.opts[:disposition]` (@janko)
 
-* Make `Response` object from `Tus::Server::S3` also respond to `#close` (@janko-m)
+* Make `Response` object from `Tus::Server::S3` also respond to `#close` (@janko)
 
-* Don't return `Content-Type` header when there is no content returned (@janko-m)
+* Don't return `Content-Type` header when there is no content returned (@janko)
 
-* Return `Content-Type: text/plain` when returning errors (@janko-m)
+* Return `Content-Type: text/plain` when returning errors (@janko)
 
-* Return `Content-Type: application/octet-stream` by default in the GET endpoint (@janko-m)
+* Return `Content-Type: application/octet-stream` by default in the GET endpoint (@janko)
 
-* Make UNIX permissions configurable via `:permissions` and `:directory_permissions` in `Tus::Storage::Filesystem` (@janko-m)
+* Make UNIX permissions configurable via `:permissions` and `:directory_permissions` in `Tus::Storage::Filesystem` (@janko)
 
-* Apply UNIX permissions `0644` for files and `0777` for directories in `Tus::Storage::Filesystem` (@janko-m)
+* Apply UNIX permissions `0644` for files and `0777` for directories in `Tus::Storage::Filesystem` (@janko)
 
-* Fix `creation-defer-length` feature not working with unlimited upload size (@janko-m)
+* Fix `creation-defer-length` feature not working with unlimited upload size (@janko)
 
-* Make the filesize of accepted uploads unlimited by default (@janko-m)
+* Make the filesize of accepted uploads unlimited by default (@janko)
 
-* Modify tus server to call `Storage#finalize_file` when the last chunk was uploaded (@janko-m)
+* Modify tus server to call `Storage#finalize_file` when the last chunk was uploaded (@janko)
 
-* Don't require length of uploaded chunks to be a multiple of `:chunkSize` in `Tus::Storage::Gridfs` (@janko-m)
+* Don't require length of uploaded chunks to be a multiple of `:chunkSize` in `Tus::Storage::Gridfs` (@janko)
 
-* Don't infer `:chunkSize` from first uploaded chunk in `Tus::Storage::Gridfs` (@janko-m)
+* Don't infer `:chunkSize` from first uploaded chunk in `Tus::Storage::Gridfs` (@janko)
 
-* Add `#length` to `Response` objects returned from `Storage#get_file` (@janko-m)
+* Add `#length` to `Response` objects returned from `Storage#get_file` (@janko)
 
 ## 0.9.1 (2017-03-24)
 
-* Fix `Tus::Storage::S3` not properly supporting the concatenation feature (@janko-m)
+* Fix `Tus::Storage::S3` not properly supporting the concatenation feature (@janko)
 
 ## 0.9.0 (2017-03-24)
 
-* Add Amazon S3 storage under `Tus::Storage::S3` (@janko-m)
+* Add Amazon S3 storage under `Tus::Storage::S3` (@janko)
 
-* Make the checksum feature actually work by generating the checksum correctly (@janko-m)
+* Make the checksum feature actually work by generating the checksum correctly (@janko)
 
-* Make `Content-Disposition` header on the GET endpoint configurable (@janko-m)
+* Make `Content-Disposition` header on the GET endpoint configurable (@janko)
 
-* Change `Content-Disposition` header on the GET endpoint from "attachment" to "inline" (@janko-m)
+* Change `Content-Disposition` header on the GET endpoint from "attachment" to "inline" (@janko)
 
-* Delegate concatenation logic to individual storages, allowing the storages to implement it much more efficiently (@janko-m)
+* Delegate concatenation logic to individual storages, allowing the storages to implement it much more efficiently (@janko)
 
-* Allow storages to save additional information in the info hash (@janko-m)
+* Allow storages to save additional information in the info hash (@janko)
 
-* Don't automatically delete expired files, instead require the developer to call `Storage#expire_files` in a recurring task (@janko-m)
+* Don't automatically delete expired files, instead require the developer to call `Storage#expire_files` in a recurring task (@janko)
 
-* Delegate expiration logic to the individual storages, allowing the storages to implement it much more efficiently (@janko-m)
+* Delegate expiration logic to the individual storages, allowing the storages to implement it much more efficiently (@janko)
 
-* Modify storages to raise `Tus::NotFound` when file wasn't found (@janko-m)
+* Modify storages to raise `Tus::NotFound` when file wasn't found (@janko)
 
-* Add `Tus::Error` which storages can use (@janko-m)
+* Add `Tus::Error` which storages can use (@janko)
 
-* In `Tus::Storage::Gridfs` require that each uploaded chunk except the last one can be distributed into even Mongo chunks (@janko-m)
+* In `Tus::Storage::Gridfs` require that each uploaded chunk except the last one can be distributed into even Mongo chunks (@janko)
 
-* Return `403 Forbidden` in the GET endpoint when attempting to download an unfinished upload (@janko-m)
+* Return `403 Forbidden` in the GET endpoint when attempting to download an unfinished upload (@janko)
 
-* Allow client to send `Upload-Length` on any PATCH request when `Upload-Defer-Length` is used (@janko-m)
+* Allow client to send `Upload-Length` on any PATCH request when `Upload-Defer-Length` is used (@janko)
 
-* Support `Range` requests in the GET endpoint (@janko-m)
+* Support `Range` requests in the GET endpoint (@janko)
 
-* Stream file content in the GET endpoint directly from the storage (@janko-m)
+* Stream file content in the GET endpoint directly from the storage (@janko)
 
-* Update `:length`, `:uploadDate` and `:contentType` Mongo fields on each PATCH request (@janko-m)
+* Update `:length`, `:uploadDate` and `:contentType` Mongo fields on each PATCH request (@janko)
 
-* Insert all sub-chunks in a single Mongo operation in `Tus::Storage::Gridfs` (@janko-m)
+* Insert all sub-chunks in a single Mongo operation in `Tus::Storage::Gridfs` (@janko)
 
-* Infer Mongo chunk size from the size of the first uploaded chunk (@janko-m)
+* Infer Mongo chunk size from the size of the first uploaded chunk (@janko)
 
-* Add `:chunk_size` option to `Tus::Storage::Gridfs` (@janko-m)
+* Add `:chunk_size` option to `Tus::Storage::Gridfs` (@janko)
 
-* Avoid reading the whole request body into memory by doing streaming uploads (@janko-m)
+* Avoid reading the whole request body into memory by doing streaming uploads (@janko)
 
 ## 0.2.0 (2016-11-23)
 
-* Refresh `Upload-Expires` for the file after each PATCH request (@janko-m)
+* Refresh `Upload-Expires` for the file after each PATCH request (@janko)
 
 ## 0.1.1 (2016-11-21)
 
-* Support Rack 1.x in addition to Rack 2.x (@janko-m)
+* Support Rack 1.x in addition to Rack 2.x (@janko)
 
-* Don't return 404 when deleting a non-existing file (@janko-m)
+* Don't return 404 when deleting a non-existing file (@janko)
 
-* Return 204 for OPTIONS requests even when the file is missing (@janko-m)
+* Return 204 for OPTIONS requests even when the file is missing (@janko)
 
-* Make sure that none of the "empty status codes" return content (@janko-m)
+* Make sure that none of the "empty status codes" return content (@janko)

data/README.md

@@ -206,57 +206,90 @@ Tus::Server.opts[:storage] = Tus::Storage::S3.new(
 )
 ```
 
-One thing to note is that S3's multipart API requires each chunk except the
-last to be **5MB or larger**, so that is the minimum chunk size that you can
-specify on your tus client if you want to use the S3 storage.
+If you want to files to be stored in a certain subdirectory, you can specify
+a `:prefix` in the storage configuration.
 
-If you'll be retrieving uploaded files through the tus server app, it's
-recommended to set `Tus::Server.opts[:redirect_download]` to `true`. This will
-avoid tus server downloading and serving the file from S3, and instead have the
-download endpoint redirect to the direct S3 object URL.
+```rb
+Tus::Storage::S3.new(prefix: "tus", **options)
+```
+
+You can also specify additional options that will be fowarded to
+[`Aws::S3::Client#create_multipart_upload`] using `:upload_options`.
 
 ```rb
-Tus::Server.opts[:redirect_download] = true
+Tus::Storage::S3.new(upload_options: { acl: "public-read" }, **options)
 ```
 
-You can customize how the S3 object URL is being generated by passing a block
-to `:redirect_download`, which will then be evaluated in the context of the
-`Tus::Server` instance (which allows accessing the `request` object). See
-[`Aws::S3::Object#get`] for the list of options that
-`Tus::Storage::S3#file_url` accepts.
+All other options will be forwarded to [`Aws::S3::Client#initialize`]:
 
 ```rb
-Tus::Server.opts[:redirect_download] = -> (uid, info, **options) do
-  storage.file_url(uid, info, expires_in: 10, **options) # link expires after 10 seconds
-end
+Tus::Storage::S3.new(
+  use_accelerate_endpoint: true,
+  logger: Logger.new(STDOUT),
+  retry_limit: 5,
+  http_open_timeout: 10,
+  # ...
+)
 ```
 
-If you want to files to be stored in a certain subdirectory, you can specify
-a `:prefix` in the storage configuration.
+If you're using [concatenation], you can specify the concurrency in which S3
+storage will copy partial uploads to the final upload (defaults to `10`):
 
 ```rb
-Tus::Storage::S3.new(prefix: "tus", **options)
+Tus::Storage::S3.new(concurrency: { concatenation: 20 }, **options)
 ```
 
-You can also specify additional options that will be fowarded to
-[`Aws::S3::Client#create_multipart_upload`] using `:upload_options`.
+#### Limits
+
+Be aware that the AWS S3 Multipart Upload API has the following limits:
+
+| Item                               | Specification                         |
+| ----                               | -------------                         |
+| Part size                          | 5 MB to 5 GB, last part can be < 5 MB |
+| Maximum number of parts per upload | 10,000                                |
+| Maximum object size                | 5 TB                                  |
+
+This means that if you're chunking uploads in your tus client, the chunk size
+needs to be **5 MB or larger**. Furthermore, if you're allowing your users to
+upload files larger than 50 GB, you will the minimum chunk size needs to be
+higher (`ceil(max_length, max_multipart_parts)`). Note that chunking is
+optional if you're running on Falcon, but it's mandatory if you're using Puma
+or another web server.
+
+`Tus::Storage::S3` is relying on the above limits for determining the multipart
+part size. If you're using a different S3-compatible service which has different
+limits, you should pass them in when initializing the storage:
 
 ```rb
-Tus::Storage::S3.new(upload_options: { content_disposition: "attachment" }, **options)
+Tus::Storage::S3.new(limits: {
+  min_part_size:       5 * 1024 * 1024,
+  max_part_size:       5 * 1024 * 1024 * 1024,
+  max_multipart_parts: 10_000,
+  max_object_size:     5 * 1024 * 1024 * 1024,
+}, **options)
 ```
 
-All other options will be forwarded to [`Aws::S3::Client#initialize`], so you
-can for example change the `:endpoint` to use S3's accelerate host:
+#### Serving files
+
+If you'll be retrieving uploaded files through the tus server app, it's
+recommended to set `Tus::Server.opts[:redirect_download]` to `true`. This will
+avoid tus server downloading and serving the file from S3, and instead have the
+download endpoint redirect to the direct S3 object URL.
 
 ```rb
-Tus::Storage::S3.new(endpoint: "https://s3-accelerate.amazonaws.com", **options)
+Tus::Server.opts[:redirect_download] = true
 ```
 
-If you're using [concatenation], you can specify the concurrency in which S3
-storage will copy partial uploads to the final upload (defaults to `10`):
+You can customize how the S3 object URL is being generated by passing a block
+to `:redirect_download`, which will then be evaluated in the context of the
+`Tus::Server` instance (which allows accessing the `request` object). See
+[`Aws::S3::Object#get`] for the list of options that
+`Tus::Storage::S3#file_url` accepts.
 
 ```rb
-Tus::Storage::S3.new(concurrency: { concatenation: 20 }, **options)
+Tus::Server.opts[:redirect_download] = -> (uid, info, **options) do
+  storage.file_url(uid, info, expires_in: 10, **options) # link expires after 10 seconds
+end
 ```
 
 ### Google Cloud Storage, Microsoft Azure Blob Storage

data/lib/tus/info.rb

@@ -40,13 +40,21 @@ module Tus
     end
 
     def offset
-      Integer(@hash["Upload-Offset"])
+      Integer(@hash["Upload-Offset"]) if @hash["Upload-Offset"]
     end
 
     def metadata
       parse_metadata(@hash["Upload-Metadata"])
     end
 
+    def name
+      metadata["name"] || metadata["filename"]
+    end
+
+    def type
+      metadata["type"] || metadata["content_type"]
+    end
+
     def expires
       Time.parse(@hash["Upload-Expires"])
     end

data/lib/tus/server.rb

@@ -94,9 +94,9 @@ module Tus
           before_create(uid, info)
 
           if info.final?
-            validate_partial_uploads!(info.partial_uploads)
+            length = validate_partial_uploads!(info.partial_uploads)
 
-            length = storage.concatenate(uid, info.partial_uploads, info.to_h)
+            storage.concatenate(uid, info.partial_uploads, info.to_h)
             info["Upload-Length"] = length.to_s
             info["Upload-Offset"] = length.to_s
           else
@@ -185,26 +185,19 @@ module Tus
         r.get do
           validate_upload_finished!(info)
 
-          metadata = info.metadata
-          name     = metadata["name"] || metadata["filename"]
-          type     = metadata["type"] || metadata["content_type"]
-
-          content_disposition = ContentDisposition.(disposition: opts[:disposition], filename: name)
-          content_type        = type || "application/octet-stream"
-
           if redirect_download
             redirect_url = instance_exec(uid, info.to_h,
-              content_type:        content_type,
-              content_disposition: content_disposition,
+              content_type:        info.type,
+              content_disposition: ContentDisposition.(disposition: opts[:disposition], filename: info.name),
               &redirect_download)
 
             r.redirect redirect_url
           else
             range = handle_range_request!(info.length)
 
-            response.headers["Content-Length"]      = range.size.to_s
-            response.headers["Content-Disposition"] = content_disposition
-            response.headers["Content-Type"]        = content_type
+            response.headers["Content-Disposition"] = ContentDisposition.(disposition: opts[:disposition], filename: info.name)
+            response.headers["Content-Type"]        = info.type if info.type
+            response.headers["ETag"]                = %(W/"#{uid}")
 
             body = storage.get_file(uid, info.to_h, range: range)
 
@@ -310,50 +303,34 @@ module Tus
       end
     end
 
-    # Validates that each partial upload exists and is marked as one.
+    # Validates that each partial upload exists and is marked as one, and at the
+    # same time calculates the sum of part lengths.
     def validate_partial_uploads!(part_uids)
-      input = Queue.new
-      part_uids.each { |part_uid| input << part_uid }
-      input.close
-
-      results = Queue.new
+      length = 0
 
-      thread_count   = storage.concurrency[:concatenation] if storage.respond_to?(:concurrency)
-      thread_count ||= 10
-
-      threads = thread_count.times.map do
-        Thread.new do
-          begin
-            loop do
-              part_uid = input.pop or break
-              part_info = storage.read_info(part_uid)
-              results << Tus::Info.new(part_info)
-            end
-            nil
-          rescue => error
-            input.clear
-            error
-          end
+      part_uids.each do |part_uid|
+        begin
+          part_info = storage.read_info(part_uid)
+        rescue Tus::NotFound
+          error!(400, "Partial upload not found")
         end
-      end
 
-      errors = threads.map(&:value).compact
+        part_info = Tus::Info.new(part_info)
 
-      if errors.any? { |error| error.is_a?(Tus::NotFound) }
-        error!(400, "One or more partial uploads were not found")
-      elsif errors.any?
-        fail errors.first
-      end
+        error!(400, "Upload is not partial") unless part_info.partial?
 
-      part_infos = Array.new(results.size) { results.pop } # convert Queue into an Array
+        unless part_info.length == part_info.offset
+          error!(400, "Partial upload is not finished")
+        end
 
-      unless part_infos.all?(&:partial?)
-        error!(400, "One or more uploads were not partial")
+        length += part_info.length
       end
 
-      if max_size && part_infos.map(&:length).inject(0, :+) > max_size
-        error!(400, "The sum of partial upload lengths exceed Tus-Max-Size")
+      if max_size && length > max_size
+        error!(400, "The sum of partial upload lengths exceeds Tus-Max-Size")
       end
+
+      length
     end
 
     def validate_upload_checksum!(input)
@@ -392,9 +369,24 @@ module Tus
         response.headers["Content-Range"] = "bytes #{range.begin}-#{range.end}/#{length}"
       end
 
+      response.headers["Content-Length"] = range.size.to_s
+
       range
     end
 
+    def redirect_download
+      value = opts[:redirect_download]
+
+      if opts[:download_url]
+        value ||= opts[:download_url]
+        warn "[TUS-RUBY-SERVER DEPRECATION] The :download_url option has been renamed to :redirect_download."
+      end
+
+      value = storage.method(:file_url) if value == true
+
+      value
+    end
+
     def handle_cors!
       origin = request.headers["Origin"]
 
@@ -404,10 +396,10 @@ module Tus
 
       if request.options?
         response.headers["Access-Control-Allow-Methods"] = "POST, GET, HEAD, PATCH, DELETE, OPTIONS"
-        response.headers["Access-Control-Allow-Headers"] = "Origin, X-Requested-With, Content-Type, Upload-Length, Upload-Offset, Tus-Resumable, Upload-Metadata"
+        response.headers["Access-Control-Allow-Headers"] = "Origin, X-Requested-With, Content-Type, Upload-Length, Upload-Offset, Tus-Resumable, Upload-Metadata, Upload-Defer-Length, Upload-Concat"
         response.headers["Access-Control-Max-Age"]       = "86400"
       else
-        response.headers["Access-Control-Expose-Headers"] = "Upload-Offset, Location, Upload-Length, Tus-Version, Tus-Resumable, Tus-Max-Size, Tus-Extension, Upload-Metadata"
+        response.headers["Access-Control-Expose-Headers"] = "Upload-Offset, Location, Upload-Length, Tus-Version, Tus-Resumable, Tus-Max-Size, Tus-Extension, Upload-Metadata, Upload-Defer-Length, Upload-Concat"
       end
     end
 
@@ -429,19 +421,6 @@ module Tus
       request.halt
     end
 
-    def redirect_download
-      value = opts[:redirect_download]
-
-      if opts[:download_url]
-        value ||= opts[:download_url]
-        warn "[TUS-RUBY-SERVER DEPRECATION] The :download_url option has been renamed to :redirect_download."
-      end
-
-      value = storage.method(:file_url) if value == true
-
-      value
-    end
-
     def storage
       opts[:storage] || Tus::Storage::Filesystem.new("data")
     end

data/lib/tus/storage/filesystem.rb

@@ -54,9 +54,6 @@ module Tus
 
         # Delete parts after concatenation.
         delete(part_uids)
-
-        # Tus server requires us to return the size of the concatenated file.
-        file_path(uid).size
       end
 
       # Appends data to the specified upload in a streaming fashion, and

data/lib/tus/storage/gridfs.rb

@@ -53,7 +53,7 @@ module Tus
         validate_parts!(grid_infos, part_uids)
 
         length       = grid_infos.map { |doc| doc[:length] }.reduce(0, :+)
-        content_type = Tus::Info.new(info).metadata["content_type"]
+        content_type = Tus::Info.new(info).type
 
         grid_file = create_grid_file(
           filename:     uid,
@@ -75,9 +75,6 @@ module Tus
 
         # Delete the parts after concatenation.
         files_collection.delete_many(filename: {"$in" => part_uids})
-
-        # Tus server requires us to return the size of the concatenated file.
-        length
       end
 
       # Appends data to the specified upload in a streaming fashion, and

data/lib/tus/storage/s3.rb

@@ -3,6 +3,7 @@
 gem "aws-sdk-s3", "~> 1.2"
 
 require "aws-sdk-s3"
+require "content_disposition"
 
 require "tus/info"
 require "tus/response"
@@ -14,12 +15,16 @@ require "cgi"
 module Tus
   module Storage
     class S3
-      MIN_PART_SIZE = 5 * 1024 * 1024 # 5MB is the minimum part size for S3 multipart uploads
+      # AWS S3 multipart upload limits
+      MIN_PART_SIZE       = 5 * 1024 * 1024
+      MAX_PART_SIZE       = 5 * 1024 * 1024 * 1024
+      MAX_MULTIPART_PARTS = 10_000
+      MAX_OBJECT_SIZE     = 5 * 1024 * 1024 * 1024 * 1024
 
-      attr_reader :client, :bucket, :prefix, :upload_options, :concurrency
+      attr_reader :client, :bucket, :prefix, :upload_options, :limits, :concurrency
 
       # Initializes an aws-sdk-s3 client with the given credentials.
-      def initialize(bucket:, prefix: nil, upload_options: {}, concurrency: {}, thread_count: nil, **client_options)
+      def initialize(bucket:, prefix: nil, upload_options: {}, limits: {}, concurrency: {}, thread_count: nil, **client_options)
         fail ArgumentError, "the :bucket option was nil" unless bucket
 
         if thread_count
@@ -33,6 +38,7 @@ module Tus
         @bucket         = resource.bucket(bucket)
         @prefix         = prefix
         @upload_options = upload_options
+        @limits         = limits
         @concurrency    = concurrency
       end
 
@@ -41,18 +47,15 @@ module Tus
       def create_file(uid, info = {})
         tus_info = Tus::Info.new(info)
 
-        options = upload_options.dup
-        options[:content_type] = tus_info.metadata["content_type"]
-
-        if filename = tus_info.metadata["filename"]
-          # Aws-sdk-s3 doesn't sign non-ASCII characters correctly, and browsers
-          # will automatically URI-decode filenames.
-          filename = CGI.escape(filename).gsub("+", " ")
-
-          options[:content_disposition] ||= "inline"
-          options[:content_disposition]  += "; filename=\"#{filename}\""
+        if tus_info.length && tus_info.length > max_object_size
+          fail Tus::Error, "upload length exceeds maximum S3 object size"
         end
 
+        options = {}
+        options[:content_type] = tus_info.type if tus_info.type
+        options[:content_disposition] = ContentDisposition.inline(tus_info.name) if tus_info.name
+        options.merge!(upload_options)
+
         multipart_upload = object(uid).initiate_multipart_upload(options)
 
         info["multipart_id"]    = multipart_upload.id
@@ -81,12 +84,8 @@ module Tus
         finalize_file(uid, info)
 
         delete(part_uids.flat_map { |part_uid| [object(part_uid), object("#{part_uid}.info")] })
-
-        # Tus server requires us to return the size of the concatenated file.
-        object = client.head_object(bucket: bucket.name, key: object(uid).key)
-        object.content_length
       rescue => error
-        abort_multipart_upload(multipart_upload) if multipart_upload
+        multipart_upload.abort if multipart_upload
         raise error
       end
 
@@ -109,21 +108,22 @@ module Tus
         part_offset    = info["multipart_parts"].count
         bytes_uploaded = 0
 
-        jobs = []
-        chunk = input.read(MIN_PART_SIZE)
+        part_size = calculate_part_size(tus_info.length)
+
+        chunk = input.read(part_size)
 
         while chunk
-          next_chunk = input.read(MIN_PART_SIZE)
+          next_chunk = input.read(part_size)
 
           # merge next chunk into previous if it's smaller than minimum chunk size
-          if next_chunk && next_chunk.bytesize < MIN_PART_SIZE
+          if next_chunk && next_chunk.bytesize < part_size
             chunk << next_chunk
             next_chunk.clear
             next_chunk = nil
           end
 
-          # abort if chunk is smaller than 5MB and is not the last chunk
-          if chunk.bytesize < MIN_PART_SIZE
+          # abort if chunk is smaller than part size and is not the last chunk
+          if chunk.bytesize < part_size
             break if (tus_info.length && tus_info.offset) &&
                      chunk.bytesize + tus_info.offset < tus_info.length
           end
@@ -197,7 +197,7 @@ module Tus
       def delete_file(uid, info = {})
         if info["multipart_id"]
           multipart_upload = object(uid).multipart_upload(info["multipart_id"])
-          abort_multipart_upload(multipart_upload)
+          multipart_upload.abort
 
           delete [object("#{uid}.info")]
         else
@@ -209,21 +209,17 @@ module Tus
       # multipart uploads still in progress, it checks the upload date of the
       # last multipart part.
       def expire_files(expiration_date)
-        old_objects = bucket.objects.select do |object|
-          object.last_modified <= expiration_date
-        end
-
-        delete(old_objects)
-
-        bucket.multipart_uploads.each do |multipart_upload|
-          # no need to check multipart uploads initiated before expiration date
-          next if multipart_upload.initiated > expiration_date
-
-          most_recent_part = multipart_upload.parts.sort_by(&:last_modified).last
-          if most_recent_part.nil? || most_recent_part.last_modified <= expiration_date
-            abort_multipart_upload(multipart_upload)
-          end
-        end
+        delete bucket.objects(prefix: @prefix)
+          .select { |object| object.last_modified <= expiration_date }
+
+        bucket.multipart_uploads
+          .select { |multipart_upload| multipart_upload.key.start_with?(prefix.to_s) }
+          .select { |multipart_upload| multipart_upload.initiated <= expiration_date }
+          .select { |multipart_upload|
+            last_modified = multipart_upload.parts.map(&:last_modified).max
+            last_modified.nil? || last_modified <= expiration_date
+          }
+          .each(&:abort)
       end
 
       private
@@ -240,6 +236,23 @@ module Tus
         { "part_number" => part_number, "etag" => response.etag }
       end
 
+      # Calculates minimum multipart part size required to upload the whole
+      # file, taking into account AWS S3 multipart limits on part size and
+      # number of parts.
+      def calculate_part_size(length)
+        return min_part_size if length.nil?
+        return length        if length <= min_part_size
+        return min_part_size if length <= min_part_size * max_multipart_parts
+
+        part_size = Rational(length, max_multipart_parts).ceil
+
+        if part_size > max_part_size
+          fail Tus::Error, "chunk size for upload exceeds maximum part size"
+        end
+
+        part_size
+      end
+
       def delete(objects)
         # S3 can delete maximum of 1000 objects in a single request
         objects.each_slice(1000) do |objects_batch|
@@ -248,18 +261,6 @@ module Tus
         end
       end
 
-      # In order to ensure the multipart upload was successfully aborted,
-      # we need to check whether all parts have been deleted, and retry
-      # the abort if the list is nonempty.
-      def abort_multipart_upload(multipart_upload)
-        loop do
-          multipart_upload.abort
-          break unless multipart_upload.parts.any?
-        end
-      rescue Aws::S3::Errors::NoSuchUpload
-        # multipart upload was successfully aborted or doesn't exist
-      end
-
       # Creates multipart parts for the specified multipart upload by copying
       # given objects into them. It uses a queue and a fixed-size thread pool
       # which consumes that queue.
@@ -325,6 +326,11 @@ module Tus
       def object(key)
         bucket.object([*prefix, key].join("/"))
       end
+
+      def min_part_size;       limits.fetch(:min_part_size,       MIN_PART_SIZE);       end
+      def max_part_size;       limits.fetch(:max_part_size,       MAX_PART_SIZE);       end
+      def max_multipart_parts; limits.fetch(:max_multipart_parts, MAX_MULTIPART_PARTS); end
+      def max_object_size;     limits.fetch(:max_object_size,     MAX_OBJECT_SIZE);     end
     end
   end
 end

data/tus-server.gemspec

@@ -1,12 +1,12 @@
 Gem::Specification.new do |gem|
   gem.name         = "tus-server"
-  gem.version      = "2.2.1"
+  gem.version      = "2.3.0"
 
   gem.required_ruby_version = ">= 2.3"
 
   gem.summary      = "Ruby server implementation of tus.io, the open protocol for resumable file uploads."
 
-  gem.homepage     = "https://github.com/janko-m/tus-ruby-server"
+  gem.homepage     = "https://github.com/janko/tus-ruby-server"
   gem.authors      = ["Janko Marohnić"]
   gem.email        = ["janko.marohnic@gmail.com"]
   gem.license      = "MIT"
@@ -21,6 +21,6 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency "minitest", "~> 5.8"
   gem.add_development_dependency "rack-test_app"
   gem.add_development_dependency "cucumber", "~> 3.1"
-  gem.add_development_dependency "mongo"
   gem.add_development_dependency "aws-sdk-s3", "~> 1.2"
+  gem.add_development_dependency "aws-sdk-core", "~> 3.23"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tus-server
 version: !ruby/object:Gem::Version
-  version: 2.2.1
+  version: 2.3.0
 platform: ruby
 authors:
 - Janko Marohnić
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-12-19 00:00:00.000000000 Z
+date: 2019-05-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: roda
@@ -101,33 +101,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '3.1'
 - !ruby/object:Gem::Dependency
-  name: mongo
+  name: aws-sdk-s3
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.2'
 - !ruby/object:Gem::Dependency
-  name: aws-sdk-s3
+  name: aws-sdk-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '3.23'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '3.23'
 description: 
 email:
 - janko.marohnic@gmail.com
@@ -150,7 +150,7 @@ files:
 - lib/tus/storage/gridfs.rb
 - lib/tus/storage/s3.rb
 - tus-server.gemspec
-homepage: https://github.com/janko-m/tus-ruby-server
+homepage: https://github.com/janko/tus-ruby-server
 licenses:
 - MIT
 metadata: {}
@@ -169,8 +169,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Ruby server implementation of tus.io, the open protocol for resumable file