canistor 0.3.2 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 2621d73e017b7bbfb296384b7bceb87699d7f71942fa9b85c346d824103f8fc0
4
- data.tar.gz: ff98d15d36834943e177a66c4db1c7fee64f96b4cb7398ad4f4c4a179bbfc2c9
3
+ metadata.gz: af06960c106c4d6a66f2e157cf5392df11bd27182c9014f934658b1981f095c8
4
+ data.tar.gz: 9c8d3c968c879d31bf6b471d599cc132b1843ed7cf7c8282cc0b21a27b04b351
5
5
  SHA512:
6
- metadata.gz: 1b92e5cbae6bb7d863b3e437df73cd0a23fda1ce3b6533a00499f26ad4eef07c2bd305d263ff68930efc5ab77aeb763c91ba561c0a8dc8aadd608488779e7bf7
7
- data.tar.gz: 3460dd2aad6248bddd71cd1cd8595da3b80a871a5568edd47b8455307e552e9e2b3228e5cb4cd45f8e4d28b41b07c68519f155704dd7748463d670101dca4d24
6
+ metadata.gz: 2275495c50f747fe474ff02c745976dcf1f05a87a8073002ef53fae3b71a3256aff2ebb8d2fc2f3ed18167c064ed6557271839c83d9dd68600b0e1dff306a6c3
7
+ data.tar.gz: 4fd8d042c1acb1bbcfbda6be5a8a865d24bb0d1160d971a09cb6742b4c4175139e9b4ca442a7c6206d1e7249749f1b2b1ed3c1b7f5b135a92889351ae9e3c87e
data/LICENSE.txt CHANGED
@@ -1,4 +1,4 @@
1
- Copyright (c) Manfred Stienstra <manfred@fngtps.com>
1
+ Copyright (c) Procore <info@procore.com>
2
2
 
3
3
  MIT License
4
4
 
@@ -68,6 +68,20 @@ module Canistor
68
68
  end.to_xml)
69
69
  end
70
70
 
71
+ def serve_no_such_upload(subject)
72
+ serve_error(404, Nokogiri::XML::Builder.new do |xml|
73
+ xml.Error do
74
+ xml.Code 'NoSuchUpload'
75
+ xml.Message 'The specified upload does not exist. The upload ID ' \
76
+ 'may be invalid, or the upload may have been aborted ' \
77
+ 'or completed.'
78
+ xml.Key subject.key
79
+ xml.RequestId request_id
80
+ xml.HostId host_id
81
+ end
82
+ end.to_xml)
83
+ end
84
+
71
85
  def serve_no_such_key(subject)
72
86
  serve_error(404, Nokogiri::XML::Builder.new do |xml|
73
87
  xml.Error do
@@ -80,6 +94,35 @@ module Canistor
80
94
  end.to_xml)
81
95
  end
82
96
 
97
+ def serve_invalid_part(upload_id, part_number, part_etag)
98
+ serve_error(400, Nokogiri::XML::Builder.new do |xml|
99
+ xml.Error do
100
+ xml.Code 'InvalidPart'
101
+ xml.Message 'One or more of the specified parts could not be found. '\
102
+ 'The part may not have been uploaded, or the specified ' \
103
+ ' entity tag may not match the part\'s entity tag.'
104
+ xml.UploadId upload_id
105
+ xml.PartNumber part_number
106
+ xml.ETag part_etag
107
+ xml.RequestId request_id
108
+ xml.HostId host_id
109
+ end
110
+ end.to_xml)
111
+ end
112
+
113
+ def serve_invalid_part_order(upload_id)
114
+ serve_error(400, Nokogiri::XML::Builder.new do |xml|
115
+ xml.Error do
116
+ xml.Code 'InvalidPartOrder'
117
+ xml.Message 'The list of parts was not in ascending order. Parts ' \
118
+ 'must be ordered by part number.'
119
+ xml.UploadId upload_id
120
+ xml.RequestId request_id
121
+ xml.HostId host_id
122
+ end
123
+ end.to_xml)
124
+ end
125
+
83
126
  def serve_access_denied(subject)
84
127
  serve_error(403, Nokogiri::XML::Builder.new do |xml|
85
128
  xml.Error do
@@ -139,10 +182,22 @@ module Canistor
139
182
  new(context).serve_no_such_bucket(subject)
140
183
  end
141
184
 
185
+ def self.serve_no_such_upload(context, subject)
186
+ new(context).serve_no_such_upload(subject)
187
+ end
188
+
142
189
  def self.serve_no_such_key(context, subject)
143
190
  new(context).serve_no_such_key(subject)
144
191
  end
145
192
 
193
+ def self.serve_invalid_part(context, upload_id, part_number, part_etag)
194
+ new(context).serve_invalid_part(upload_id, part_number, part_etag)
195
+ end
196
+
197
+ def self.serve_invalid_part_order(context, upload_id)
198
+ new(context).serve_invalid_part_order(upload_id)
199
+ end
200
+
146
201
  def self.access_denied(context, subject)
147
202
  new(context).access_denied(subject)
148
203
  end
@@ -7,4 +7,5 @@ require 'securerandom'
7
7
 
8
8
  require_relative "storage/bucket"
9
9
  require_relative "storage/object"
10
+ require_relative "storage/part"
10
11
  require_relative "storage/upload"
@@ -8,6 +8,7 @@ require 'securerandom'
8
8
 
9
9
  module Canistor
10
10
  module Storage
11
+ # Holds information about a bucket and implements interaction with it.
11
12
  class Bucket
12
13
  attr_accessor :region
13
14
  attr_accessor :name
@@ -47,23 +48,37 @@ module Canistor
47
48
  end
48
49
 
49
50
  def get(context, access_key_id, subject)
50
- if !access_keys.include?(access_key_id)
51
- Canistor::ErrorHandler.serve_access_denied(context, subject)
52
- elsif subject.key.nil? || subject.key == ''
53
- list_bucket(context)
54
- elsif object = objects[subject.key]
55
- object.get(context, subject)
56
- else
57
- Canistor::ErrorHandler.serve_no_such_key(context, subject)
51
+ params = CGI::parse(context.http_request.endpoint.query.to_s)
52
+ catch(:rendered_error) do
53
+ if !access_keys.include?(access_key_id)
54
+ Canistor::ErrorHandler.serve_access_denied(context, subject)
55
+ elsif params.has_key?('uploads')
56
+ list_bucket_uploads(context)
57
+ elsif params.has_key?('uploadId')
58
+ list_bucket_upload_parts(context, subject, params)
59
+ elsif subject.key.nil? || subject.key == ''
60
+ list_bucket(context)
61
+ elsif object = objects[subject.key]
62
+ object.get(context, subject)
63
+ else
64
+ Canistor::ErrorHandler.serve_no_such_key(context, subject)
65
+ end
58
66
  end
59
67
  end
60
68
 
61
69
  def put(context, access_key_id, subject)
62
70
  if access_keys.include?(access_key_id)
63
71
  Canistor.take_fail(:store) { return }
64
- object = find_or_build_object(subject, context)
65
- self[subject.key] = object
66
- object.put(context, subject)
72
+ params = CGI::parse(context.http_request.endpoint.query.to_s)
73
+ catch(:rendered_error) do
74
+ if params.has_key?('uploadId')
75
+ # Client wants to create a new upload part when uploadId is
76
+ # present in the query.
77
+ put_upload_part(context, subject, params)
78
+ else
79
+ put_object(context, subject)
80
+ end
81
+ end
67
82
  else
68
83
  Canistor::ErrorHandler.serve_access_denied(context, subject)
69
84
  end
@@ -72,9 +87,18 @@ module Canistor
72
87
  def post(context, access_key_id, subject)
73
88
  if access_keys.include?(access_key_id)
74
89
  Canistor.take_fail(:store) { return }
75
- upload = build_upload(subject, context)
76
- self[upload.id] = upload
77
- upload.get(context, subject)
90
+ params = CGI::parse(context.http_request.endpoint.query.to_s)
91
+ catch(:rendered_error) do
92
+ if params.has_key?('uploads')
93
+ # Client wants to create a new upload when uploads is present in
94
+ # the query.
95
+ post_upload(context, subject)
96
+ elsif params.has_key?('uploadId')
97
+ # Client wants to complete the upload when uploadId is present in
98
+ # the query.
99
+ complete_upload(context, subject, params)
100
+ end
101
+ end
78
102
  else
79
103
  Canistor::ErrorHandler.serve_access_denied(context, subject)
80
104
  end
@@ -115,7 +139,7 @@ module Canistor
115
139
 
116
140
  private
117
141
 
118
- def build_upload(subject, context)
142
+ def build_upload(context, subject)
119
143
  Canistor::Storage::Upload.new(
120
144
  region: subject.region,
121
145
  bucket: subject.bucket,
@@ -123,7 +147,7 @@ module Canistor
123
147
  )
124
148
  end
125
149
 
126
- def find_or_build_object(subject, context)
150
+ def find_or_build_object(context, subject)
127
151
  objects[subject.key] || Canistor::Storage::Object.new(
128
152
  region: subject.region,
129
153
  bucket: subject.bucket,
@@ -131,6 +155,37 @@ module Canistor
131
155
  )
132
156
  end
133
157
 
158
+ def post_upload(context, subject)
159
+ upload = build_upload(context, subject)
160
+ @uploads[upload.id] = upload
161
+ upload.put(context, subject)
162
+ end
163
+
164
+ def put_upload_part(context, subject, params)
165
+ if upload = uploads.dig(params['uploadId'][0])
166
+ upload.put(context, subject)
167
+ else
168
+ Canistor::ErrorHandler.serve_no_such_upload(context, subject)
169
+ throw :rendered_error
170
+ end
171
+ end
172
+
173
+ def complete_upload(context, subject, params)
174
+ if upload = uploads.dig(params['uploadId'][0])
175
+ object = upload.post(context, subject)
176
+ self[subject.key] = object
177
+ else
178
+ Canistor::ErrorHandler.serve_no_such_upload(context, subject)
179
+ throw :rendered_error
180
+ end
181
+ end
182
+
183
+ def put_object(context, subject)
184
+ object = find_or_build_object(context, subject)
185
+ self[subject.key] = object
186
+ object.put(context, subject)
187
+ end
188
+
134
189
  # Iterate over all objects in the bucket using the filter and pagination
135
190
  # options which exist in S3.
136
191
  def each(prefix:, marker:, max_keys:, &block)
@@ -146,6 +201,28 @@ module Canistor
146
201
  end
147
202
  end
148
203
 
204
+ def upload_matches?(upload, upload_id_marker:, key_marker:)
205
+ if upload_id_marker && !upload.id.start_with?(upload_id_marker)
206
+ return false
207
+ end
208
+ if key_marker && !upload.key.start_with?(key_marker)
209
+ return false
210
+ end
211
+ true
212
+ end
213
+
214
+ def each_upload(upload_id_marker:, key_marker:, &block)
215
+ uploads.each do |upload_id, upload|
216
+ if upload_matches?(
217
+ upload,
218
+ upload_id_marker: upload_id_marker,
219
+ key_marker: key_marker
220
+ )
221
+ yield upload
222
+ end
223
+ end
224
+ end
225
+
149
226
  def list_bucket(context)
150
227
  context.http_response.signal_headers(
151
228
  200,
@@ -157,8 +234,27 @@ module Canistor
157
234
  end
158
235
  end
159
236
 
237
+ def list_bucket_uploads(context)
238
+ context.http_response.signal_headers(
239
+ 200,
240
+ 'date' => Time.now.httpdate,
241
+ 'x-amz-request-id' => SecureRandom.hex(8).upcase
242
+ )
243
+ context.http_response.signal_data(to_uploads_xml(context))
244
+ end
245
+
246
+ def list_bucket_upload_parts(context, subject, params)
247
+ upload = uploads.dig(params['uploadId'][0])
248
+ if upload && upload.key == subject.key
249
+ upload.get(context)
250
+ else
251
+ Canistor::ErrorHandler.serve_no_such_upload(context, subject)
252
+ throw :rendered_error
253
+ end
254
+ end
255
+
160
256
  def to_xml(context)
161
- # Only returns objects with keys that start with the prefix.
257
+ # Only return objects with keys that start with the prefix.
162
258
  prefix = context.params[:prefix]
163
259
  # Return objects until we find a key that matches the marker. Can
164
260
  # be used to group objects.
@@ -182,6 +278,31 @@ module Canistor
182
278
  end
183
279
  end.to_xml
184
280
  end
281
+
282
+ def to_uploads_xml(context)
283
+ # Only return uploads with ID's that start with id marker
284
+ upload_id_marker = context.params[:upload_id_marker]
285
+ # Only return uploads with keys that start with key marker
286
+ key_marker = context.params[:key_marker]
287
+ Nokogiri::XML::Builder.new do |xml|
288
+ xml.ListMultipartUploadsResult(
289
+ xmlns: 'http://s3.amazonaws.com/doc/2006-03-01/'
290
+ ) do
291
+ xml.Bucket name
292
+ xml.KeyMarker key_marker
293
+ xml.UploadIdMarker upload_id_marker
294
+ each_upload(
295
+ upload_id_marker: upload_id_marker,
296
+ key_marker: key_marker
297
+ ) do |upload|
298
+ xml.Upload do
299
+ xml.Key upload.key
300
+ xml.UploadId upload.id
301
+ end
302
+ end
303
+ end
304
+ end.to_xml
305
+ end
185
306
  end
186
307
  end
187
308
  end
@@ -42,6 +42,18 @@ module Canistor
42
42
  '"' + digest + '"'
43
43
  end
44
44
 
45
+ def endpoint
46
+ Aws::Partitions::EndpointProvider.resolve(region, 's3')
47
+ end
48
+
49
+ def location
50
+ [
51
+ endpoint,
52
+ bucket,
53
+ key,
54
+ ].join('/')
55
+ end
56
+
45
57
  def headers
46
58
  @headers.merge(identity_headers).merge(
47
59
  'date' => Time.now.httpdate,
@@ -72,8 +84,10 @@ module Canistor
72
84
  def put(context, subject)
73
85
  catch(:rendered_error) do
74
86
  source_object = source_object(context, subject)
75
- self.data = object_data(context, source_object)
76
- self.headers = object_headers(context, source_object)
87
+ write(
88
+ object_headers(context, source_object),
89
+ object_data(context, source_object)
90
+ )
77
91
  context.http_response.signal_headers(200, identity_headers)
78
92
  end
79
93
  end
@@ -0,0 +1,68 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Canistor
4
+ module Storage
5
+ class Part
6
+ attr_accessor :region
7
+ attr_accessor :bucket
8
+ attr_accessor :key
9
+ attr_accessor :upload_id
10
+ attr_accessor :number
11
+
12
+ attr_reader :data
13
+ attr_reader :etag
14
+
15
+ def initialize(**attributes)
16
+ attributes.each do |name, value|
17
+ send("#{name}=", value)
18
+ end
19
+ end
20
+
21
+ def put(context, subject)
22
+ write(context.http_request.body.read)
23
+ context.http_response.signal_headers(200, identity_headers)
24
+ end
25
+
26
+ def etag
27
+ '"' + digest + '"'
28
+ end
29
+
30
+ def size
31
+ data&.size
32
+ end
33
+
34
+ def attributes
35
+ {
36
+ region: region,
37
+ bucket: bucket,
38
+ key: key,
39
+ upload_id: upload_id,
40
+ number: number,
41
+ digest: digest,
42
+ etag: etag
43
+ }
44
+ end
45
+
46
+ private
47
+
48
+ attr_writer :data
49
+
50
+ def write(data)
51
+ self.data = data
52
+ end
53
+
54
+ def digest
55
+ @digest ||= Digest::SHA1.hexdigest(data)
56
+ end
57
+
58
+ def identity_headers
59
+ {
60
+ 'x-amz-request-id' => Base64.strict_encode64(SecureRandom.hex(16)),
61
+ 'x-amz-id' => digest[0, 16],
62
+ 'x-amz-id-2' => Base64.strict_encode64(digest),
63
+ 'etag' => etag
64
+ }
65
+ end
66
+ end
67
+ end
68
+ end
@@ -7,32 +7,157 @@ module Canistor
7
7
  attr_accessor :region
8
8
  attr_accessor :bucket
9
9
  attr_accessor :key
10
+ attr_accessor :parts
11
+
12
+ attr_reader :headers
10
13
 
11
14
  def initialize(**attributes)
12
15
  @id = SecureRandom.hex(64)
16
+ @parts = {}
17
+ @headers = {}
13
18
  attributes.each do |name, value|
14
19
  send("#{name}=", value)
15
20
  end
16
21
  end
17
22
 
18
- def get(context, subject)
19
- show_upload(context)
23
+ def write(headers)
24
+ self.headers = headers
25
+ end
26
+
27
+ def get(context)
28
+ list_parts(context)
29
+ end
30
+
31
+ def put(context, subject)
32
+ params = CGI::parse(context.http_request.endpoint.query.to_s)
33
+ if params.has_key?('uploads')
34
+ write(context.http_request.headers)
35
+ show_initiate_upload(context)
36
+ elsif params.has_key?('partNumber')
37
+ part_number = Integer(params['partNumber'][0])
38
+ part = find_or_build_part(subject, context, part_number)
39
+ parts[part_number] = part
40
+ part.put(context, subject)
41
+ else
42
+ raise(
43
+ RuntimeError,
44
+ "Implementation should never attempt to PUT an upload when there " \
45
+ "is no uploads or partNumber parameter present in the request."
46
+ )
47
+ end
48
+ end
49
+
50
+ def post(context, subject)
51
+ catch(:rendered_error) do
52
+ object = Canistor::Storage::Object.new(
53
+ region: region,
54
+ bucket: bucket,
55
+ key: key
56
+ )
57
+ object.write(headers, String.new)
58
+ each_part_in_body(context) do |part|
59
+ object.data << part.data
60
+ end
61
+ show_complete_upload(context, object)
62
+ object
63
+ end
20
64
  end
21
65
 
22
66
  private
23
67
 
24
- def show_upload(context)
68
+ META_HEADERS = %w(
69
+ content-disposition
70
+ content-type
71
+ )
72
+
73
+ def headers=(headers)
74
+ return if headers.nil?
75
+ headers.each do |name, value|
76
+ if META_HEADERS.include?(name)
77
+ @headers[name] = value
78
+ end
79
+ end
80
+ end
81
+
82
+ def find_or_build_part(subject, context, part_number)
83
+ parts[part_number] || Canistor::Storage::Part.new(
84
+ region: subject.region,
85
+ bucket: subject.bucket,
86
+ key: subject.key,
87
+ upload_id: id,
88
+ number: part_number
89
+ )
90
+ end
91
+
92
+ def show_initiate_upload(context)
25
93
  context.http_response.signal_headers(
26
94
  200,
27
95
  'date' => Time.now.httpdate,
28
96
  'x-amz-request-id' => SecureRandom.hex(8).upcase
29
97
  )
30
98
  unless context.http_request.http_method == 'HEAD'
31
- context.http_response.signal_data(to_xml(context))
99
+ context.http_response.signal_data(to_initiate_xml(context))
100
+ end
101
+ end
102
+
103
+ def show_complete_upload(context, etag)
104
+ context.http_response.signal_headers(
105
+ 200,
106
+ 'date' => Time.now.httpdate,
107
+ 'x-amz-request-id' => SecureRandom.hex(8).upcase
108
+ )
109
+ context.http_response.signal_data(to_complete_xml(context, etag))
110
+ end
111
+
112
+ def each_part_in_body(context)
113
+ document = Nokogiri::XML.parse(context.http_request.body.read)
114
+ number = 0
115
+ document.css('Part').each do |element|
116
+ part_number = Integer(element.css('PartNumber').text)
117
+ if part_number > number
118
+ part_etag = element.css('ETag').text
119
+ found = find_part(part_number, part_etag)
120
+ if found
121
+ yield found
122
+ else
123
+ Canistor::ErrorHandler.serve_invalid_part(
124
+ context, id, part_number, part_etag
125
+ )
126
+ throw :rendered_error
127
+ end
128
+ number = part_number
129
+ else
130
+ Canistor::ErrorHandler.serve_invalid_part_order(context, id)
131
+ throw :rendered_error
132
+ end
133
+ end
134
+ end
135
+
136
+ def find_part(part_number, part_etag)
137
+ if part = parts[part_number]
138
+ if part.etag == part_etag
139
+ return part
140
+ end
141
+ end
142
+ nil
143
+ end
144
+
145
+ def each_part(&block)
146
+ parts.keys.sort.each do |number|
147
+ yield parts[number]
32
148
  end
33
149
  end
34
150
 
35
- def to_xml(context)
151
+ def list_parts(context)
152
+ context.http_response.signal_headers(
153
+ 200,
154
+ 'date' => Time.now.httpdate,
155
+ 'x-amz-request-id' => SecureRandom.hex(8).upcase
156
+ )
157
+ context.http_response.signal_data(to_upload_parts_xml(context))
158
+ end
159
+
160
+ def to_initiate_xml(context)
36
161
  Nokogiri::XML::Builder.new do |xml|
37
162
  xml.InitiateMultipartUploadResult(
38
163
  xmlns: 'http://s3.amazonaws.com/doc/2006-03-01/'
@@ -43,6 +168,38 @@ module Canistor
43
168
  end
44
169
  end.to_xml
45
170
  end
171
+
172
+ def to_complete_xml(context, object)
173
+ Nokogiri::XML::Builder.new do |xml|
174
+ xml.CompleteMultipartUploadResult(
175
+ xmlns: 'http://s3.amazonaws.com/doc/2006-03-01/'
176
+ ) do
177
+ xml.Location object.location
178
+ xml.Bucket object.bucket
179
+ xml.Key object.key
180
+ xml.ETag object.etag
181
+ end
182
+ end.to_xml
183
+ end
184
+
185
+ def to_upload_parts_xml(context)
186
+ Nokogiri::XML::Builder.new do |xml|
187
+ xml.ListPartsResult(
188
+ xmlns: 'http://s3.amazonaws.com/doc/2006-03-01/'
189
+ ) do
190
+ xml.Bucket bucket
191
+ xml.Key key
192
+ xml.UploadId id
193
+ each_part do |part|
194
+ xml.Part do
195
+ xml.PartNumber part.number
196
+ xml.ETag part.etag
197
+ xml.Size part.size
198
+ end
199
+ end
200
+ end
201
+ end.to_xml
202
+ end
46
203
  end
47
204
  end
48
205
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Canistor
4
- VERSION = '0.3.2'
4
+ VERSION = '0.4.0'
5
5
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: canistor
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.2
4
+ version: 0.4.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Manfred Stienstra
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-07-26 00:00:00.000000000 Z
11
+ date: 2018-08-24 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -99,10 +99,11 @@ files:
99
99
  - lib/canistor/storage.rb
100
100
  - lib/canistor/storage/bucket.rb
101
101
  - lib/canistor/storage/object.rb
102
+ - lib/canistor/storage/part.rb
102
103
  - lib/canistor/storage/upload.rb
103
104
  - lib/canistor/subject.rb
104
105
  - lib/canistor/version.rb
105
- homepage: https://erm.im/canistor
106
+ homepage: https://github.com/procore/canistor
106
107
  licenses:
107
108
  - MIT
108
109
  metadata: {}