tus-server 0.2.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +205 -52
- data/lib/tus/checksum.rb +30 -17
- data/lib/tus/errors.rb +4 -0
- data/lib/tus/info.rb +16 -3
- data/lib/tus/input.rb +31 -0
- data/lib/tus/server.rb +96 -77
- data/lib/tus/storage/filesystem.rb +82 -28
- data/lib/tus/storage/gridfs.rb +161 -35
- data/lib/tus/storage/s3.rb +242 -0
- data/tus-server.gemspec +4 -2
- metadata +35 -4
- data/lib/tus/expirator.rb +0 -58
data/lib/tus/input.rb
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
module Tus
|
2
|
+
class Input
|
3
|
+
def initialize(input)
|
4
|
+
@input = input
|
5
|
+
@bytes_read = 0
|
6
|
+
end
|
7
|
+
|
8
|
+
def read(*args)
|
9
|
+
result = @input.read(*args)
|
10
|
+
@bytes_read += result.bytesize if result.is_a?(String)
|
11
|
+
result
|
12
|
+
end
|
13
|
+
|
14
|
+
def eof?
|
15
|
+
@bytes_read == size
|
16
|
+
end
|
17
|
+
|
18
|
+
def rewind
|
19
|
+
@input.rewind
|
20
|
+
@bytes_read = 0
|
21
|
+
end
|
22
|
+
|
23
|
+
def size
|
24
|
+
@input.size
|
25
|
+
end
|
26
|
+
|
27
|
+
def close
|
28
|
+
# Rack input shouldn't be closed, we just support the interface
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
data/lib/tus/server.rb
CHANGED
@@ -2,10 +2,12 @@ require "roda"
|
|
2
2
|
|
3
3
|
require "tus/storage/filesystem"
|
4
4
|
require "tus/info"
|
5
|
-
require "tus/
|
5
|
+
require "tus/input"
|
6
6
|
require "tus/checksum"
|
7
|
+
require "tus/errors"
|
7
8
|
|
8
9
|
require "securerandom"
|
10
|
+
require "time"
|
9
11
|
|
10
12
|
module Tus
|
11
13
|
class Server < Roda
|
@@ -20,18 +22,18 @@ module Tus
|
|
20
22
|
SUPPORTED_CHECKSUM_ALGORITHMS = %w[sha1 sha256 sha384 sha512 md5 crc32]
|
21
23
|
RESUMABLE_CONTENT_TYPE = "application/offset+octet-stream"
|
22
24
|
|
23
|
-
opts[:max_size]
|
24
|
-
opts[:expiration_time]
|
25
|
-
opts[:
|
25
|
+
opts[:max_size] = 1024*1024*1024
|
26
|
+
opts[:expiration_time] = 7*24*60*60
|
27
|
+
opts[:disposition] = "inline"
|
26
28
|
|
27
29
|
plugin :all_verbs
|
28
30
|
plugin :delete_empty_headers
|
29
31
|
plugin :request_headers
|
30
32
|
plugin :not_allowed
|
33
|
+
plugin :streaming
|
34
|
+
plugin :error_handler
|
31
35
|
|
32
36
|
route do |r|
|
33
|
-
expire_files
|
34
|
-
|
35
37
|
if request.headers["X-HTTP-Method-Override"]
|
36
38
|
request.env["REQUEST_METHOD"] = request.headers["X-HTTP-Method-Override"]
|
37
39
|
end
|
@@ -56,12 +58,12 @@ module Tus
|
|
56
58
|
end
|
57
59
|
|
58
60
|
r.post do
|
59
|
-
validate_upload_concat! if request.headers["Upload-Concat"]
|
60
61
|
validate_upload_length! unless request.headers["Upload-Concat"].to_s.start_with?("final") || request.headers["Upload-Defer-Length"] == "1"
|
61
62
|
validate_upload_metadata! if request.headers["Upload-Metadata"]
|
63
|
+
validate_upload_concat! if request.headers["Upload-Concat"]
|
62
64
|
|
63
|
-
uid
|
64
|
-
info = Info.new(
|
65
|
+
uid = SecureRandom.hex
|
66
|
+
info = Tus::Info.new(
|
65
67
|
"Upload-Length" => request.headers["Upload-Length"],
|
66
68
|
"Upload-Offset" => "0",
|
67
69
|
"Upload-Defer-Length" => request.headers["Upload-Defer-Length"],
|
@@ -70,23 +72,17 @@ module Tus
|
|
70
72
|
"Upload-Expires" => (Time.now + expiration_time).httpdate,
|
71
73
|
)
|
72
74
|
|
73
|
-
|
74
|
-
|
75
|
-
if info.final_upload?
|
76
|
-
length = info.partial_uploads.inject(0) do |length, partial_uid|
|
77
|
-
content = storage.read_file(partial_uid)
|
78
|
-
storage.patch_file(uid, content)
|
79
|
-
storage.delete_file(partial_uid)
|
80
|
-
length += content.length
|
81
|
-
end
|
82
|
-
|
75
|
+
if info.concatenation?
|
76
|
+
length = storage.concatenate(uid, info.partial_uploads, info.to_h)
|
83
77
|
info["Upload-Length"] = length.to_s
|
84
78
|
info["Upload-Offset"] = length.to_s
|
85
|
-
|
86
|
-
storage.
|
79
|
+
else
|
80
|
+
storage.create_file(uid, info.to_h)
|
87
81
|
end
|
88
82
|
|
89
|
-
|
83
|
+
storage.update_info(uid, info.to_h)
|
84
|
+
|
85
|
+
response.headers.update(info.headers)
|
90
86
|
|
91
87
|
file_url = "#{request.url.chomp("/")}/#{uid}"
|
92
88
|
created!(file_url)
|
@@ -105,84 +101,74 @@ module Tus
|
|
105
101
|
no_content!
|
106
102
|
end
|
107
103
|
|
108
|
-
|
109
|
-
storage.delete_file(uid)
|
110
|
-
|
111
|
-
no_content!
|
112
|
-
end
|
113
|
-
|
114
|
-
not_found! unless storage.file_exists?(uid)
|
115
|
-
|
116
|
-
r.get do
|
117
|
-
path = storage.download_file(uid)
|
118
|
-
info = Info.new(storage.read_info(uid))
|
119
|
-
|
120
|
-
server = Rack::File.new(File.dirname(path))
|
121
|
-
|
122
|
-
result = if ::Rack.release > "2"
|
123
|
-
server.serving(request, path)
|
124
|
-
else
|
125
|
-
server = server.dup
|
126
|
-
server.path = path
|
127
|
-
server.serving(env)
|
128
|
-
end
|
129
|
-
|
130
|
-
response.status = result[0]
|
131
|
-
response.headers.update(result[1])
|
132
|
-
|
133
|
-
metadata = info.metadata
|
134
|
-
response.headers["Content-Disposition"] = "attachment; filename=\"#{metadata["filename"]}\"" if metadata["filename"]
|
135
|
-
response.headers["Content-Type"] = metadata["content_type"] if metadata["content_type"]
|
136
|
-
|
137
|
-
request.halt response.finish_with_body(result[2])
|
138
|
-
end
|
104
|
+
info = Tus::Info.new(storage.read_info(uid))
|
139
105
|
|
140
106
|
r.head do
|
141
|
-
info
|
142
|
-
|
143
|
-
response.headers.update(info.to_h)
|
107
|
+
response.headers.update(info.headers)
|
144
108
|
response.headers["Cache-Control"] = "no-store"
|
145
109
|
|
146
110
|
no_content!
|
147
111
|
end
|
148
112
|
|
149
113
|
r.patch do
|
150
|
-
|
151
|
-
|
152
|
-
content = request.body.read
|
153
|
-
info = Info.new(storage.read_info(uid))
|
114
|
+
input = Tus::Input.new(request.body)
|
154
115
|
|
155
|
-
if info.defer_length?
|
116
|
+
if info.defer_length? && request.headers["Upload-Length"]
|
156
117
|
validate_upload_length!
|
118
|
+
|
157
119
|
info["Upload-Length"] = request.headers["Upload-Length"]
|
158
120
|
info["Upload-Defer-Length"] = nil
|
159
121
|
end
|
160
122
|
|
161
|
-
|
123
|
+
validate_content_type!
|
124
|
+
validate_content_length!(info.offset, info.length)
|
162
125
|
validate_upload_offset!(info.offset)
|
163
|
-
|
126
|
+
validate_upload_checksum!(input) if request.headers["Upload-Checksum"]
|
164
127
|
|
165
|
-
storage.patch_file(uid,
|
128
|
+
storage.patch_file(uid, input, info.to_h)
|
166
129
|
|
167
|
-
info["Upload-Offset"] = (info.offset +
|
130
|
+
info["Upload-Offset"] = (info.offset + input.size).to_s
|
168
131
|
info["Upload-Expires"] = (Time.now + expiration_time).httpdate
|
169
132
|
|
170
133
|
storage.update_info(uid, info.to_h)
|
171
|
-
response.headers.update(info.
|
134
|
+
response.headers.update(info.headers)
|
135
|
+
|
136
|
+
no_content!
|
137
|
+
end
|
138
|
+
|
139
|
+
r.get do
|
140
|
+
validate_upload_finished!(info.length, info.offset)
|
141
|
+
range = handle_range_request!(info.length)
|
142
|
+
|
143
|
+
response.headers["Content-Length"] = (range.end - range.begin + 1).to_s
|
144
|
+
|
145
|
+
metadata = info.metadata
|
146
|
+
response.headers["Content-Disposition"] = opts[:disposition]
|
147
|
+
response.headers["Content-Disposition"] << "; filename=\"#{metadata["filename"]}\"" if metadata["filename"]
|
148
|
+
response.headers["Content-Type"] = metadata["content_type"] if metadata["content_type"]
|
149
|
+
|
150
|
+
response = storage.get_file(uid, info.to_h, range: range)
|
151
|
+
|
152
|
+
stream(callback: ->{response.close}) do |out|
|
153
|
+
response.each { |chunk| out << chunk }
|
154
|
+
end
|
155
|
+
end
|
156
|
+
|
157
|
+
r.delete do
|
158
|
+
storage.delete_file(uid, info.to_h)
|
172
159
|
|
173
160
|
no_content!
|
174
161
|
end
|
175
162
|
end
|
176
163
|
end
|
177
164
|
|
178
|
-
|
179
|
-
|
180
|
-
|
165
|
+
error do |exception|
|
166
|
+
not_found! if exception.is_a?(Tus::NotFound)
|
167
|
+
raise
|
181
168
|
end
|
182
169
|
|
183
170
|
def validate_content_type!
|
184
|
-
|
185
|
-
error!(415, "Invalid Content-Type header") if content_type != RESUMABLE_CONTENT_TYPE
|
171
|
+
error!(415, "Invalid Content-Type header") if request.content_type != RESUMABLE_CONTENT_TYPE
|
186
172
|
end
|
187
173
|
|
188
174
|
def validate_tus_resumable!
|
@@ -218,9 +204,17 @@ module Tus
|
|
218
204
|
end
|
219
205
|
end
|
220
206
|
|
221
|
-
def validate_content_length!(
|
222
|
-
|
223
|
-
|
207
|
+
def validate_content_length!(current_offset, length)
|
208
|
+
if length
|
209
|
+
error!(403, "Cannot modify completed upload") if current_offset == length
|
210
|
+
error!(413, "Size of this chunk surpasses Upload-Length") if Integer(request.content_length) + current_offset > length
|
211
|
+
else
|
212
|
+
error!(413, "Size of this chunk surpasses Tus-Max-Size") if Integer(request.content_length) + current_offset > max_size
|
213
|
+
end
|
214
|
+
end
|
215
|
+
|
216
|
+
def validate_upload_finished!(length, current_offset)
|
217
|
+
error!(403, "Cannot download unfinished upload") unless length && current_offset && length == current_offset
|
224
218
|
end
|
225
219
|
|
226
220
|
def validate_upload_metadata!
|
@@ -250,15 +244,40 @@ module Tus
|
|
250
244
|
end
|
251
245
|
end
|
252
246
|
|
253
|
-
def validate_upload_checksum!(
|
247
|
+
def validate_upload_checksum!(input)
|
254
248
|
algorithm, checksum = request.headers["Upload-Checksum"].split(" ")
|
255
249
|
|
256
250
|
error!(400, "Invalid Upload-Checksum header") if algorithm.nil? || checksum.nil?
|
257
251
|
error!(400, "Invalid Upload-Checksum header") unless SUPPORTED_CHECKSUM_ALGORITHMS.include?(algorithm)
|
258
252
|
|
259
|
-
|
260
|
-
|
253
|
+
generated_checksum = Tus::Checksum.generate(algorithm, input)
|
254
|
+
error!(460, "Checksum from Upload-Checksum header doesn't match generated") if generated_checksum != checksum
|
255
|
+
end
|
256
|
+
|
257
|
+
# "Range" header handling logic copied from Rack::File
|
258
|
+
def handle_range_request!(length)
|
259
|
+
if Rack.release >= "2.0"
|
260
|
+
ranges = Rack::Utils.get_byte_ranges(request.headers["Range"], length)
|
261
|
+
else
|
262
|
+
ranges = Rack::Utils.byte_ranges(request.env, length)
|
263
|
+
end
|
264
|
+
|
265
|
+
if ranges.nil? || ranges.length > 1
|
266
|
+
# No ranges, or multiple ranges (which we don't support):
|
267
|
+
response.status = 200
|
268
|
+
range = 0..length-1
|
269
|
+
elsif ranges.empty?
|
270
|
+
# Unsatisfiable. Return error, and file size:
|
271
|
+
response.headers["Content-Range"] = "bytes */#{length}"
|
272
|
+
error!(416, "Byte range unsatisfiable")
|
273
|
+
else
|
274
|
+
# Partial content:
|
275
|
+
range = ranges[0]
|
276
|
+
response.status = 206
|
277
|
+
response.headers["Content-Range"] = "bytes #{range.begin}-#{range.end}/#{length}"
|
261
278
|
end
|
279
|
+
|
280
|
+
range
|
262
281
|
end
|
263
282
|
|
264
283
|
def handle_cors!
|
@@ -1,5 +1,8 @@
|
|
1
|
+
require "tus/errors"
|
2
|
+
|
1
3
|
require "pathname"
|
2
4
|
require "json"
|
5
|
+
require "fileutils"
|
3
6
|
|
4
7
|
module Tus
|
5
8
|
module Storage
|
@@ -13,54 +16,90 @@ module Tus
|
|
13
16
|
end
|
14
17
|
|
15
18
|
def create_file(uid, info = {})
|
16
|
-
|
17
|
-
write(info_path(uid), info.to_json)
|
19
|
+
file_path(uid).open("wb") { |file| file.write("") }
|
18
20
|
end
|
19
21
|
|
20
|
-
def
|
21
|
-
file_path(uid).
|
22
|
-
|
22
|
+
def concatenate(uid, part_uids, info = {})
|
23
|
+
file_path(uid).open("wb") do |file|
|
24
|
+
begin
|
25
|
+
part_uids.each do |part_uid|
|
26
|
+
IO.copy_stream(file_path(part_uid), file)
|
27
|
+
end
|
28
|
+
rescue Errno::ENOENT
|
29
|
+
raise Tus::Error, "some parts for concatenation are missing"
|
30
|
+
end
|
31
|
+
end
|
23
32
|
|
24
|
-
|
25
|
-
file_path(uid).binread
|
26
|
-
end
|
33
|
+
delete(part_uids)
|
27
34
|
|
28
|
-
|
29
|
-
|
35
|
+
# server requires us to return the size of the concatenated file
|
36
|
+
file_path(uid).size
|
30
37
|
end
|
31
38
|
|
32
|
-
def
|
33
|
-
file_path(uid).
|
34
|
-
end
|
39
|
+
def patch_file(uid, io, info = {})
|
40
|
+
raise Tus::NotFound if !file_path(uid).exist?
|
35
41
|
|
36
|
-
|
37
|
-
if file_exists?(uid)
|
38
|
-
file_path(uid).delete
|
39
|
-
info_path(uid).delete
|
40
|
-
end
|
42
|
+
file_path(uid).open("ab") { |file| IO.copy_stream(io, file) }
|
41
43
|
end
|
42
44
|
|
43
45
|
def read_info(uid)
|
44
|
-
|
46
|
+
raise Tus::NotFound if !file_path(uid).exist?
|
47
|
+
|
48
|
+
begin
|
49
|
+
data = info_path(uid).binread
|
50
|
+
rescue Errno::ENOENT
|
51
|
+
data = "{}"
|
52
|
+
end
|
53
|
+
|
45
54
|
JSON.parse(data)
|
46
55
|
end
|
47
56
|
|
48
57
|
def update_info(uid, info)
|
49
|
-
|
58
|
+
info_path(uid).open("wb") { |file| file.write(info.to_json) }
|
59
|
+
end
|
60
|
+
|
61
|
+
def get_file(uid, info = {}, range: nil)
|
62
|
+
raise Tus::NotFound if !file_path(uid).exist?
|
63
|
+
|
64
|
+
file = file_path(uid).open("rb")
|
65
|
+
range ||= 0..file.size-1
|
66
|
+
|
67
|
+
chunks = Enumerator.new do |yielder|
|
68
|
+
file.seek(range.begin)
|
69
|
+
remaining_length = range.end - range.begin + 1
|
70
|
+
buffer = ""
|
71
|
+
|
72
|
+
while remaining_length > 0
|
73
|
+
chunk = file.read([16*1024, remaining_length].min, buffer)
|
74
|
+
break unless chunk
|
75
|
+
remaining_length -= chunk.length
|
76
|
+
|
77
|
+
yielder << chunk
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
Response.new(chunks: chunks, close: ->{file.close})
|
50
82
|
end
|
51
83
|
|
52
|
-
def
|
53
|
-
|
54
|
-
paths.map { |path| File.basename(path, ".file") }
|
84
|
+
def delete_file(uid, info = {})
|
85
|
+
delete([uid])
|
55
86
|
end
|
56
87
|
|
57
|
-
|
88
|
+
def expire_files(expiration_date)
|
89
|
+
uids = []
|
58
90
|
|
59
|
-
|
60
|
-
|
61
|
-
file.sync = true
|
62
|
-
file.write(content)
|
91
|
+
Pathname.glob(directory.join("*.file")).each do |pathname|
|
92
|
+
uids << pathname.basename(".*") if pathname.mtime <= expiration_date
|
63
93
|
end
|
94
|
+
|
95
|
+
delete(uids)
|
96
|
+
end
|
97
|
+
|
98
|
+
private
|
99
|
+
|
100
|
+
def delete(uids)
|
101
|
+
paths = uids.flat_map { |uid| [file_path(uid), info_path(uid)] }
|
102
|
+
FileUtils.rm_f paths
|
64
103
|
end
|
65
104
|
|
66
105
|
def file_path(uid)
|
@@ -75,6 +114,21 @@ module Tus
|
|
75
114
|
directory.mkpath
|
76
115
|
directory.chmod(0755)
|
77
116
|
end
|
117
|
+
|
118
|
+
class Response
|
119
|
+
def initialize(chunks:, close:)
|
120
|
+
@chunks = chunks
|
121
|
+
@close = close
|
122
|
+
end
|
123
|
+
|
124
|
+
def each(&block)
|
125
|
+
@chunks.each(&block)
|
126
|
+
end
|
127
|
+
|
128
|
+
def close
|
129
|
+
@close.call
|
130
|
+
end
|
131
|
+
end
|
78
132
|
end
|
79
133
|
end
|
80
134
|
end
|