fakes3-ruby18 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,21 @@
1
+ module FakeS3
2
+ class RateLimitableFile < File
3
+ @@rate_limit = nil
4
+ # Specify a rate limit in bytes per second
5
+ def self.rate_limit
6
+ @@rate_limit
7
+ end
8
+
9
+ def self.rate_limit=(rate_limit)
10
+ @@rate_limit = rate_limit
11
+ end
12
+
13
+ def read(args)
14
+ if @@rate_limit
15
+ time_to_sleep = args / @@rate_limit
16
+ sleep(time_to_sleep)
17
+ end
18
+ return super(args)
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,19 @@
1
+ module FakeS3
2
+ class S3Object
3
+ include Comparable
4
+ attr_accessor :name,:size,:creation_date,:modified_date,:md5,:io,:content_type,:custom_metadata
5
+
6
+ def hash
7
+ @name.hash
8
+ end
9
+
10
+ def eql?(object)
11
+ @name == object.name
12
+ end
13
+
14
+ # Sort by the object's name
15
+ def <=>(object)
16
+ @name <=> object.name
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,541 @@
1
+ require 'time'
2
+ require 'webrick'
3
+ require 'webrick/https'
4
+ require 'openssl'
5
+ require 'securerandom'
6
+ require 'cgi'
7
+ require 'fakes3/file_store'
8
+ require 'fakes3/xml_adapter'
9
+ require 'fakes3/bucket_query'
10
+ require 'fakes3/unsupported_operation'
11
+ require 'fakes3/errors'
12
+
13
+ module FakeS3
14
+ class Request
15
+ CREATE_BUCKET = "CREATE_BUCKET"
16
+ LIST_BUCKETS = "LIST_BUCKETS"
17
+ LS_BUCKET = "LS_BUCKET"
18
+ HEAD = "HEAD"
19
+ STORE = "STORE"
20
+ COPY = "COPY"
21
+ GET = "GET"
22
+ GET_ACL = "GET_ACL"
23
+ SET_ACL = "SET_ACL"
24
+ MOVE = "MOVE"
25
+ DELETE_OBJECT = "DELETE_OBJECT"
26
+ DELETE_BUCKET = "DELETE_BUCKET"
27
+
28
+ attr_accessor :bucket,:object,:type,:src_bucket,
29
+ :src_object,:method,:webrick_request,
30
+ :path,:is_path_style,:query,:http_verb
31
+
32
+ def inspect
33
+ puts "-----Inspect FakeS3 Request"
34
+ puts "Type: #{@type}"
35
+ puts "Is Path Style: #{@is_path_style}"
36
+ puts "Request Method: #{@method}"
37
+ puts "Bucket: #{@bucket}"
38
+ puts "Object: #{@object}"
39
+ puts "Src Bucket: #{@src_bucket}"
40
+ puts "Src Object: #{@src_object}"
41
+ puts "Query: #{@query}"
42
+ puts "-----Done"
43
+ end
44
+ end
45
+
46
+ class Servlet < WEBrick::HTTPServlet::AbstractServlet
47
+ def initialize(server,store,hostname)
48
+ super(server)
49
+ @store = store
50
+ @hostname = hostname
51
+ @port = server.config[:Port]
52
+ @root_hostnames = [hostname,'localhost','s3.amazonaws.com','s3.localhost']
53
+ end
54
+
55
+ def validate_request(request)
56
+ req = request.webrick_request
57
+ return if req.nil?
58
+ return if not req.header.has_key?('expect')
59
+ req.continue if req.header['expect'].first=='100-continue'
60
+ end
61
+
62
+ def do_GET(request, response)
63
+ s_req = normalize_request(request)
64
+
65
+ case s_req.type
66
+ when 'LIST_BUCKETS'
67
+ response.status = 200
68
+ response['Content-Type'] = 'application/xml'
69
+ buckets = @store.buckets
70
+ response.body = XmlAdapter.buckets(buckets)
71
+ when 'LS_BUCKET'
72
+ bucket_obj = @store.get_bucket(s_req.bucket)
73
+ if bucket_obj
74
+ response.status = 200
75
+ response['Content-Type'] = "application/xml"
76
+ query = {
77
+ :marker => s_req.query["marker"] ? s_req.query["marker"].to_s : nil,
78
+ :prefix => s_req.query["prefix"] ? s_req.query["prefix"].to_s : nil,
79
+ :max_keys => s_req.query["max_keys"] ? s_req.query["max_keys"].to_s : nil,
80
+ :delimiter => s_req.query["delimiter"] ? s_req.query["delimiter"].to_s : nil
81
+ }
82
+ bq = bucket_obj.query_for_range(query)
83
+ response.body = XmlAdapter.bucket_query(bq)
84
+ else
85
+ response.status = 404
86
+ response.body = XmlAdapter.error_no_such_bucket(s_req.bucket)
87
+ response['Content-Type'] = "application/xml"
88
+ end
89
+ when 'GET_ACL'
90
+ response.status = 200
91
+ response.body = XmlAdapter.acl()
92
+ response['Content-Type'] = 'application/xml'
93
+ when 'GET'
94
+ real_obj = @store.get_object(s_req.bucket,s_req.object,request)
95
+ if !real_obj
96
+ response.status = 404
97
+ response.body = XmlAdapter.error_no_such_key(s_req.object)
98
+ response['Content-Type'] = "application/xml"
99
+ return
100
+ end
101
+
102
+ if_none_match = request["If-None-Match"]
103
+ if if_none_match == "\"#{real_obj.md5}\"" or if_none_match == "*"
104
+ response.status = 304
105
+ return
106
+ end
107
+
108
+ if_modified_since = request["If-Modified-Since"]
109
+ if if_modified_since
110
+ time = Time.httpdate(if_modified_since)
111
+ if time >= Time.iso8601(real_obj.modified_date)
112
+ response.status = 304
113
+ return
114
+ end
115
+ end
116
+
117
+ response.status = 200
118
+ response['Content-Type'] = real_obj.content_type
119
+ stat = File::Stat.new(real_obj.io.path)
120
+
121
+ response['Last-Modified'] = Time.iso8601(real_obj.modified_date).httpdate()
122
+ response.header['ETag'] = "\"#{real_obj.md5}\""
123
+ response['Accept-Ranges'] = "bytes"
124
+ response['Last-Ranges'] = "bytes"
125
+ response['Access-Control-Allow-Origin'] = '*'
126
+
127
+ real_obj.custom_metadata.each do |header, value|
128
+ response.header['x-amz-meta-' + header] = value
129
+ end
130
+
131
+ content_length = stat.size
132
+
133
+ # Added Range Query support
134
+ if range = request.header["range"].first
135
+ response.status = 206
136
+ if range =~ /bytes=(\d*)-(\d*)/
137
+ start = $1.to_i
138
+ finish = $2.to_i
139
+ finish_str = ""
140
+ if finish == 0
141
+ finish = content_length - 1
142
+ finish_str = "#{finish}"
143
+ else
144
+ finish_str = finish.to_s
145
+ end
146
+
147
+ bytes_to_read = finish - start + 1
148
+ response['Content-Range'] = "bytes #{start}-#{finish_str}/#{content_length}"
149
+ real_obj.io.pos = start
150
+ response.body = real_obj.io.read(bytes_to_read)
151
+ return
152
+ end
153
+ end
154
+ response['Content-Length'] = File::Stat.new(real_obj.io.path).size
155
+ if s_req.http_verb == 'HEAD'
156
+ response.body = ""
157
+ else
158
+ response.body = real_obj.io
159
+ end
160
+ end
161
+ end
162
+
163
+ def do_PUT(request,response)
164
+ s_req = normalize_request(request)
165
+ query = CGI::parse(request.request_uri.query || "")
166
+
167
+ return do_multipartPUT(request, response) if query['uploadId'].first
168
+
169
+ response.status = 200
170
+ response.body = ""
171
+ response['Content-Type'] = "text/xml"
172
+ response['Access-Control-Allow-Origin'] = '*'
173
+
174
+ case s_req.type
175
+ when Request::COPY
176
+ object = @store.copy_object(s_req.src_bucket,s_req.src_object,s_req.bucket,s_req.object,request)
177
+ response.body = XmlAdapter.copy_object_result(object)
178
+ when Request::STORE
179
+ bucket_obj = @store.get_bucket(s_req.bucket)
180
+ if !bucket_obj
181
+ # Lazily create a bucket. TODO fix this to return the proper error
182
+ bucket_obj = @store.create_bucket(s_req.bucket)
183
+ end
184
+
185
+ real_obj = @store.store_object(bucket_obj,s_req.object,s_req.webrick_request)
186
+ response.header['ETag'] = "\"#{real_obj.md5}\""
187
+ when Request::CREATE_BUCKET
188
+ @store.create_bucket(s_req.bucket)
189
+ end
190
+ end
191
+
192
+ def do_multipartPUT(request, response)
193
+ s_req = normalize_request(request)
194
+ query = CGI::parse(request.request_uri.query)
195
+
196
+ part_number = query['partNumber'].first
197
+ upload_id = query['uploadId'].first
198
+ part_name = "#{upload_id}_#{s_req.object}_part#{part_number}"
199
+
200
+ # store the part
201
+ if s_req.type == Request::COPY
202
+ real_obj = @store.copy_object(
203
+ s_req.src_bucket, s_req.src_object,
204
+ s_req.bucket , part_name,
205
+ request
206
+ )
207
+
208
+ response['Content-Type'] = "text/xml"
209
+ response.body = XmlAdapter.copy_object_result real_obj
210
+ else
211
+ bucket_obj = @store.get_bucket(s_req.bucket)
212
+ if !bucket_obj
213
+ bucket_obj = @store.create_bucket(s_req.bucket)
214
+ end
215
+ real_obj = @store.store_object(
216
+ bucket_obj, part_name,
217
+ request
218
+ )
219
+
220
+ response.body = ""
221
+ response.header['ETag'] = "\"#{real_obj.md5}\""
222
+ end
223
+
224
+ response['Access-Control-Allow-Origin'] = '*'
225
+ response['Access-Control-Allow-Headers'] = 'Authorization, Content-Length'
226
+ response['Access-Control-Expose-Headers'] = 'ETag'
227
+
228
+ response.status = 200
229
+ end
230
+
231
+ def do_POST(request,response)
232
+ s_req = normalize_request(request)
233
+ key = request.query['key']
234
+ query = CGI::parse(request.request_uri.query || "")
235
+
236
+ if query.has_key?('uploads')
237
+ upload_id = SecureRandom.hex
238
+
239
+ response.body = <<-eos.strip
240
+ <?xml version="1.0" encoding="UTF-8"?>
241
+ <InitiateMultipartUploadResult>
242
+ <Bucket>#{ s_req.bucket }</Bucket>
243
+ <Key>#{ key }</Key>
244
+ <UploadId>#{ upload_id }</UploadId>
245
+ </InitiateMultipartUploadResult>
246
+ eos
247
+ elsif query.has_key?('uploadId')
248
+ upload_id = query['uploadId'].first
249
+ bucket_obj = @store.get_bucket(s_req.bucket)
250
+ real_obj = @store.combine_object_parts(
251
+ bucket_obj,
252
+ upload_id,
253
+ s_req.object,
254
+ parse_complete_multipart_upload(request),
255
+ request
256
+ )
257
+
258
+ response.body = XmlAdapter.complete_multipart_result real_obj
259
+ elsif request.content_type =~ /^multipart\/form-data; boundary=(.+)/
260
+ key=request.query['key']
261
+
262
+ success_action_redirect = request.query['success_action_redirect']
263
+ success_action_status = request.query['success_action_status']
264
+
265
+ filename = 'default'
266
+ filename = $1 if request.body =~ /filename="(.*)"/
267
+ key = key.gsub('${filename}', filename)
268
+
269
+ bucket_obj = @store.get_bucket(s_req.bucket) || @store.create_bucket(s_req.bucket)
270
+ real_obj = @store.store_object(bucket_obj, key, s_req.webrick_request)
271
+
272
+ response['Etag'] = "\"#{real_obj.md5}\""
273
+
274
+ if success_action_redirect
275
+ response.status = 307
276
+ response.body = ""
277
+ response['Location'] = success_action_redirect
278
+ else
279
+ response.status = success_action_status || 204
280
+ if response.status == "201"
281
+ response.body = <<-eos.strip
282
+ <?xml version="1.0" encoding="UTF-8"?>
283
+ <PostResponse>
284
+ <Location>http://#{s_req.bucket}.localhost:#{@port}/#{key}</Location>
285
+ <Bucket>#{s_req.bucket}</Bucket>
286
+ <Key>#{key}</Key>
287
+ <ETag>#{response['Etag']}</ETag>
288
+ </PostResponse>
289
+ eos
290
+ end
291
+ end
292
+ else
293
+ raise WEBrick::HTTPStatus::BadRequest
294
+ end
295
+
296
+ response['Content-Type'] = 'text/xml'
297
+ response['Access-Control-Allow-Origin'] = '*'
298
+ response['Access-Control-Allow-Headers'] = 'Authorization, Content-Length'
299
+ response['Access-Control-Expose-Headers'] = 'ETag'
300
+ end
301
+
302
+ def do_DELETE(request,response)
303
+ s_req = normalize_request(request)
304
+
305
+ case s_req.type
306
+ when Request::DELETE_OBJECT
307
+ bucket_obj = @store.get_bucket(s_req.bucket)
308
+ @store.delete_object(bucket_obj,s_req.object,s_req.webrick_request)
309
+ when Request::DELETE_BUCKET
310
+ @store.delete_bucket(s_req.bucket)
311
+ end
312
+
313
+ response.status = 204
314
+ response.body = ""
315
+ end
316
+
317
+ def do_OPTIONS(request, response)
318
+ super
319
+
320
+ response['Access-Control-Allow-Origin'] = '*'
321
+ response['Access-Control-Allow-Methods'] = 'PUT, POST, HEAD, GET, OPTIONS'
322
+ response['Access-Control-Allow-Headers'] = 'Accept, Content-Type, Authorization, Content-Length, ETag'
323
+ response['Access-Control-Expose-Headers'] = 'ETag'
324
+ end
325
+
326
+ private
327
+
328
+ def normalize_delete(webrick_req,s_req)
329
+ path = webrick_req.path
330
+ path_len = path.size
331
+ query = webrick_req.query
332
+ if path == "/" and s_req.is_path_style
333
+ # Probably do a 404 here
334
+ else
335
+ if s_req.is_path_style
336
+ elems = path[1,path_len].split("/")
337
+ s_req.bucket = elems[0]
338
+ else
339
+ elems = path.split("/")
340
+ end
341
+
342
+ if elems.size == 0
343
+ raise UnsupportedOperation
344
+ elsif elems.size == 1
345
+ s_req.type = Request::DELETE_BUCKET
346
+ s_req.query = query
347
+ else
348
+ s_req.type = Request::DELETE_OBJECT
349
+ object = elems[1,elems.size].join('/')
350
+ s_req.object = object
351
+ end
352
+ end
353
+ end
354
+
355
+ def normalize_get(webrick_req,s_req)
356
+ path = webrick_req.path
357
+ path_len = path.size
358
+ query = webrick_req.query
359
+ if path == "/" and s_req.is_path_style
360
+ s_req.type = Request::LIST_BUCKETS
361
+ else
362
+ if s_req.is_path_style
363
+ elems = path[1,path_len].split("/")
364
+ s_req.bucket = elems[0]
365
+ else
366
+ elems = path.split("/")
367
+ end
368
+
369
+ if elems.size < 2
370
+ s_req.type = Request::LS_BUCKET
371
+ s_req.query = query
372
+ else
373
+ if query["acl"] == ""
374
+ s_req.type = Request::GET_ACL
375
+ else
376
+ s_req.type = Request::GET
377
+ end
378
+ object = elems[1,elems.size].join('/')
379
+ s_req.object = object
380
+ end
381
+ end
382
+ end
383
+
384
+ def normalize_put(webrick_req,s_req)
385
+ path = webrick_req.path
386
+ path_len = path.size
387
+ if path == "/"
388
+ if s_req.bucket
389
+ s_req.type = Request::CREATE_BUCKET
390
+ end
391
+ else
392
+ if s_req.is_path_style
393
+ elems = path[1,path_len].split("/")
394
+ s_req.bucket = elems[0]
395
+ if elems.size == 1
396
+ s_req.type = Request::CREATE_BUCKET
397
+ else
398
+ if webrick_req.request_line =~ /\?acl/
399
+ s_req.type = Request::SET_ACL
400
+ else
401
+ s_req.type = Request::STORE
402
+ end
403
+ s_req.object = elems[1,elems.size].join('/')
404
+ end
405
+ else
406
+ if webrick_req.request_line =~ /\?acl/
407
+ s_req.type = Request::SET_ACL
408
+ else
409
+ s_req.type = Request::STORE
410
+ end
411
+ s_req.object = webrick_req.path[1..-1]
412
+ end
413
+ end
414
+
415
+ # TODO: also parse the x-amz-copy-source-range:bytes=first-last header
416
+ # for multipart copy
417
+ copy_source = webrick_req.header["x-amz-copy-source"]
418
+ if copy_source and copy_source.size == 1
419
+ src_elems = copy_source.first.split("/")
420
+ root_offset = src_elems[0] == "" ? 1 : 0
421
+ s_req.src_bucket = src_elems[root_offset]
422
+ s_req.src_object = src_elems[1 + root_offset,src_elems.size].join("/")
423
+ s_req.type = Request::COPY
424
+ end
425
+
426
+ s_req.webrick_request = webrick_req
427
+ end
428
+
429
+ def normalize_post(webrick_req,s_req)
430
+ path = webrick_req.path
431
+ path_len = path.size
432
+
433
+ s_req.path = webrick_req.query['key']
434
+
435
+ s_req.webrick_request = webrick_req
436
+
437
+ if s_req.is_path_style
438
+ elems = path[1,path_len].split("/")
439
+ s_req.bucket = elems[0]
440
+ s_req.object = elems[1..-1].join('/') if elems.size >= 2
441
+ else
442
+ s_req.object = path[1..-1]
443
+ end
444
+ end
445
+
446
+ # This method takes a webrick request and generates a normalized FakeS3 request
447
+ def normalize_request(webrick_req)
448
+ host_header= webrick_req["Host"]
449
+ host = host_header.split(':')[0]
450
+
451
+ s_req = Request.new
452
+ s_req.path = webrick_req.path
453
+ s_req.is_path_style = true
454
+
455
+ if !@root_hostnames.include?(host)
456
+ s_req.bucket = host.split(".")[0]
457
+ s_req.is_path_style = false
458
+ end
459
+
460
+ s_req.http_verb = webrick_req.request_method
461
+
462
+ case webrick_req.request_method
463
+ when 'PUT'
464
+ normalize_put(webrick_req,s_req)
465
+ when 'GET','HEAD'
466
+ normalize_get(webrick_req,s_req)
467
+ when 'DELETE'
468
+ normalize_delete(webrick_req,s_req)
469
+ when 'POST'
470
+ normalize_post(webrick_req,s_req)
471
+ else
472
+ raise "Unknown Request"
473
+ end
474
+
475
+ validate_request(s_req)
476
+
477
+ return s_req
478
+ end
479
+
480
+ def parse_complete_multipart_upload request
481
+ parts_xml = ""
482
+ request.body { |chunk| parts_xml << chunk }
483
+
484
+ # TODO: I suck at parsing xml
485
+ parts_xml = parts_xml.scan /\<Part\>.*?<\/Part\>/m
486
+
487
+ parts_xml.collect do |xml|
488
+ {
489
+ :number => xml[/\<PartNumber\>(\d+)\<\/PartNumber\>/, 1].to_i,
490
+ :etag => xml[/\<ETag\>\"(.+)\"\<\/ETag\>/, 1]
491
+ }
492
+ end
493
+ end
494
+
495
+ def dump_request(request)
496
+ puts "----------Dump Request-------------"
497
+ puts request.request_method
498
+ puts request.path
499
+ request.each do |k,v|
500
+ puts "#{k}:#{v}"
501
+ end
502
+ puts "----------End Dump -------------"
503
+ end
504
+ end
505
+
506
+
507
+ class Server
508
+ def initialize(address,port,store,hostname,ssl_cert_path,ssl_key_path)
509
+ @address = address
510
+ @port = port
511
+ @store = store
512
+ @hostname = hostname
513
+ @ssl_cert_path = ssl_cert_path
514
+ @ssl_key_path = ssl_key_path
515
+ webrick_config = {
516
+ :BindAddress => @address,
517
+ :Port => @port
518
+ }
519
+ if !@ssl_cert_path.to_s.empty?
520
+ webrick_config.merge!(
521
+ {
522
+ :SSLEnable => true,
523
+ :SSLCertificate => OpenSSL::X509::Certificate.new(File.read(@ssl_cert_path)),
524
+ :SSLPrivateKey => OpenSSL::PKey::RSA.new(File.read(@ssl_key_path))
525
+ }
526
+ )
527
+ end
528
+ @server = WEBrick::HTTPServer.new(webrick_config)
529
+ end
530
+
531
+ def serve
532
+ @server.mount "/", Servlet, @store,@hostname
533
+ trap "INT" do @server.shutdown end
534
+ @server.start
535
+ end
536
+
537
+ def shutdown
538
+ @server.shutdown
539
+ end
540
+ end
541
+ end