rack_fake_s3 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,442 @@
1
+ require 'rack'
2
+ require 'rack/request'
3
+ require 'rack/response'
4
+ require 'rack/server'
5
+ require 'rack/lint'
6
+
7
+ require 'rack_fake_s3/file_store'
8
+ require 'rack_fake_s3/xml_adapter'
9
+ require 'rack_fake_s3/bucket_query'
10
+ require 'rack_fake_s3/errors'
11
+
12
+ module RackFakeS3
13
+ require 'webrick'
14
+ class Request
15
+ CREATE_BUCKET = "CREATE_BUCKET"
16
+ LIST_BUCKETS = "LIST_BUCKETS"
17
+ LS_BUCKET = "LS_BUCKET"
18
+ HEAD = "HEAD"
19
+ STORE = "STORE"
20
+ OPTIONS = "OPTIONS"
21
+ COPY = "COPY"
22
+ GET = "GET"
23
+ GET_ACL = "GET_ACL"
24
+ SET_ACL = "SET_ACL"
25
+ MOVE = "MOVE"
26
+ DELETE_OBJECT = "DELETE_OBJECT"
27
+ DELETE_BUCKET = "DELETE_BUCKET"
28
+
29
+ attr_accessor :bucket,:object,:type,:src_bucket,
30
+ :src_object,:method,:rack_request,
31
+ :path,:is_path_style,:query,:http_verb
32
+
33
+ def inspect
34
+ puts "-----Inspect RackFakeS3 Request"
35
+ puts "Type: #{@type}"
36
+ puts "Is Path Style: #{@is_path_style}"
37
+ puts "Request Method: #{@method}"
38
+ puts "Bucket: #{@bucket}"
39
+ puts "Object: #{@object}"
40
+ puts "Src Bucket: #{@src_bucket}"
41
+ puts "Src Object: #{@src_object}"
42
+ puts "Query: #{@query}"
43
+ puts "-----Done"
44
+ end
45
+ end
46
+
47
+ class Servlet
48
+ def initialize(app,store,hostname)
49
+ raise "store can't be nil" if store.nil?
50
+ raise "hostname can't be nil" if hostname.nil?
51
+
52
+ @app = app
53
+ @store = store
54
+ @hostname = hostname
55
+ @root_hostnames = [hostname,'localhost','s3.amazonaws.com','s3.localhost']
56
+ end
57
+
58
+ def call(env)
59
+ request = Rack::Request.new(env)
60
+ s_req = normalize_request(request)
61
+
62
+ if s_req
63
+ dup.perform(env, s_req)
64
+ elsif @app
65
+ @app.call(env)
66
+ else
67
+ halt 404
68
+ end
69
+ end
70
+
71
+ def perform(env, s_req)
72
+ response = Rack::Response.new
73
+
74
+ send(:"do_#{s_req.http_verb}", s_req, response)
75
+
76
+ response.finish
77
+ end
78
+
79
+ def do_GET(s_req, response)
80
+ request = s_req.rack_request
81
+
82
+ case s_req.type
83
+ when 'LIST_BUCKETS'
84
+ response.status = 200
85
+ response['Content-Type'] = 'application/xml'
86
+ buckets = @store.buckets
87
+ response.write XmlAdapter.buckets(buckets)
88
+ when 'LS_BUCKET'
89
+ bucket_obj = @store.get_bucket(s_req.bucket)
90
+ if bucket_obj
91
+ response.status = 200
92
+ response['Content-Type'] = "application/xml"
93
+ query = {
94
+ :marker => s_req.query["marker"] ? s_req.query["marker"].to_s : nil,
95
+ :prefix => s_req.query["prefix"] ? s_req.query["prefix"].to_s : nil,
96
+ :max_keys => s_req.query["max-keys"] ? s_req.query["max-keys"].to_i : nil,
97
+ :delimiter => s_req.query["delimiter"] ? s_req.query["delimiter"].to_s : nil
98
+ }
99
+
100
+ bq = bucket_obj.query_for_range(query)
101
+ response.write XmlAdapter.bucket_query(bq)
102
+ else
103
+ response.status = 404
104
+ response.write XmlAdapter.error_no_such_bucket(s_req.bucket)
105
+ response['Content-Type'] = "application/xml"
106
+ end
107
+ when 'GET_ACL'
108
+ response.status = 200
109
+ response.write XmlAdapter.acl()
110
+ response['Content-Type'] = 'application/xml'
111
+ when 'GET'
112
+ real_obj = @store.get_object(s_req.bucket,s_req.object,request)
113
+ if !real_obj
114
+ response.status = 404
115
+ response.write ""
116
+ return
117
+ end
118
+
119
+ response.status = 200
120
+ response['Content-Type'] = real_obj.content_type
121
+ content_length = File::Stat.new(real_obj.io.path).size
122
+ response['Etag'] = real_obj.md5
123
+ response['Accept-Ranges'] = "bytes"
124
+
125
+ # Added Range Query support
126
+ if range = request.env["HTTP_RANGE"]
127
+ response.status = 206
128
+ if range =~ /bytes=(\d*)-(\d*)/
129
+ start = $1.to_i
130
+ finish = $2.to_i
131
+ finish_str = ""
132
+ if finish == 0
133
+ finish = content_length - 1
134
+ finish_str = "#{finish}"
135
+ else
136
+ finish_str = finish.to_s
137
+ end
138
+
139
+ bytes_to_read = finish - start + 1
140
+ response['Content-Range'] = "bytes #{start}-#{finish_str}/#{content_length}"
141
+ real_obj.io.pos = start
142
+ response.write real_obj.io.read(bytes_to_read)
143
+ return
144
+ end
145
+ end
146
+ response['Content-Length'] = File::Stat.new(real_obj.io.path).size.to_s
147
+ if s_req.http_verb == 'HEAD'
148
+ response.write ""
149
+ else
150
+ response.length = real_obj.io.size.to_s
151
+ response.body = real_obj.io
152
+ end
153
+ end
154
+ end
155
+ alias :do_HEAD :do_GET
156
+
157
+ def do_PUT(s_req,response)
158
+ request = s_req.rack_request
159
+
160
+ case s_req.type
161
+ when Request::COPY
162
+ @store.copy_object(s_req.src_bucket,s_req.src_object,s_req.bucket,s_req.object)
163
+ when Request::STORE
164
+ bucket_obj = @store.get_bucket(s_req.bucket)
165
+ if !bucket_obj
166
+ # Lazily create a bucket. TODO fix this to return the proper error
167
+ bucket_obj = @store.create_bucket(s_req.bucket)
168
+ end
169
+
170
+ real_obj = @store.store_object(bucket_obj,s_req.object,s_req.rack_request)
171
+
172
+ response['Etag'] = real_obj.md5
173
+ when Request::CREATE_BUCKET
174
+ @store.create_bucket(s_req.bucket)
175
+ end
176
+
177
+ response.status = 200
178
+ response.write ""
179
+ response['Content-Type'] = "text/xml"
180
+ end
181
+
182
+ def do_POST(request,response)
183
+ # check that we've received file data
184
+ unless request.rack_request.content_type =~ /^multipart\/form-data; boundary=(.+)/
185
+ raise 'bad request'
186
+ end
187
+
188
+ key=request.path
189
+ success_action_redirect=request.rack_request.params['success_action_redirect']
190
+ success_action_status=request.rack_request.params['success_action_status']
191
+
192
+ filename = 'default'
193
+ filename = $1 if request.rack_request.body =~ /filename="(.*)"/
194
+ key=key.gsub('${filename}', filename)
195
+
196
+ bucket_obj = @store.get_bucket(request.bucket) || @store.create_bucket(request.bucket)
197
+ real_obj=@store.store_object(bucket_obj, key, request.rack_request)
198
+
199
+ response['Etag'] = "\"#{real_obj.md5}\""
200
+ response.body = []
201
+ if success_action_redirect
202
+ response.status = 307
203
+ response['Location']=success_action_redirect
204
+ else
205
+ response.status = success_action_status || 204
206
+ if response.status=="201"
207
+ response.body << <<-eos.strip
208
+ <?xml version="1.0" encoding="UTF-8"?>
209
+ <PostResponse>
210
+ <Location>#{request.rack_request.scheme}://#{request.rack_request.host_with_port}/#{key}</Location>
211
+ <Bucket>#{request.bucket}</Bucket>
212
+ <Key>#{key}</Key>
213
+ <ETag>#{response['Etag']}</ETag>
214
+ </PostResponse>
215
+ eos
216
+ end
217
+ end
218
+ response['Content-Type'] = 'text/xml'
219
+ response['Access-Control-Allow-Origin']='*'
220
+ end
221
+
222
+ def do_DELETE(s_req,response)
223
+ request = s_req.rack_request
224
+ case s_req.type
225
+ when Request::DELETE_OBJECT
226
+ bucket_obj = @store.get_bucket(s_req.bucket)
227
+ @store.delete_object(bucket_obj,s_req.object,s_req.rack_request)
228
+ when Request::DELETE_BUCKET
229
+ @store.delete_bucket(s_req.bucket)
230
+ end
231
+
232
+ response.status = 204
233
+ response.write ""
234
+ end
235
+
236
+ def do_OPTIONS(request, response)
237
+ response["Access-Control-Allow-Origin"]="*"
238
+ end
239
+
240
+ private
241
+
242
+ def normalize_delete(rack_req,s_req)
243
+ path = rack_req.path
244
+ path_len = path.size
245
+ query = rack_req.params
246
+ if path == "/" and s_req.is_path_style
247
+ # Probably do a 404 here
248
+ else
249
+ if s_req.is_path_style
250
+ elems = path[1,path_len].split("/")
251
+ s_req.bucket = elems[0]
252
+ else
253
+ elems = path.split("/")
254
+ end
255
+
256
+ if elems.size == 0
257
+ raise UnsupportedOperation
258
+ elsif elems.size == 1
259
+ s_req.type = Request::DELETE_BUCKET
260
+ s_req.query = query
261
+ else
262
+ s_req.type = Request::DELETE_OBJECT
263
+ object = elems[1,elems.size].join('/')
264
+ s_req.object = object
265
+ end
266
+ end
267
+ end
268
+
269
+ def normalize_get(rack_req,s_req)
270
+ path = rack_req.path
271
+ path_len = path.size
272
+ query = rack_req.params
273
+ if path == "/" and s_req.is_path_style
274
+ s_req.type = Request::LIST_BUCKETS
275
+ else
276
+ if s_req.is_path_style
277
+ elems = path[1,path_len].split("/")
278
+ s_req.bucket = elems[0]
279
+ else
280
+ elems = path.split("/")
281
+ end
282
+
283
+ if elems.size == 0
284
+ # List buckets
285
+ s_req.type = Request::LIST_BUCKETS
286
+ elsif elems.size == 1
287
+ s_req.type = Request::LS_BUCKET
288
+ s_req.query = query
289
+ else
290
+ if query.has_key?("acl")
291
+ s_req.type = Request::GET_ACL
292
+ else
293
+ s_req.type = Request::GET
294
+ end
295
+ object = elems[1,elems.size].join('/')
296
+ s_req.object = object
297
+ end
298
+ end
299
+ end
300
+
301
+ def normalize_put(rack_req,s_req)
302
+ path = rack_req.path
303
+ path_len = path.size
304
+ if path == "/"
305
+ if s_req.bucket
306
+ s_req.type = Request::CREATE_BUCKET
307
+ end
308
+ else
309
+ if s_req.is_path_style
310
+ elems = path[1,path_len].split("/")
311
+ s_req.bucket = elems[0]
312
+ if elems.size == 1
313
+ s_req.type = Request::CREATE_BUCKET
314
+ else
315
+ if rack_req.fullpath =~ /\?acl/
316
+ s_req.type = Request::SET_ACL
317
+ else
318
+ s_req.type = Request::STORE
319
+ end
320
+ s_req.object = elems[1,elems.size].join('/')
321
+ end
322
+ else
323
+ if rack_req.fullpath =~ /\?acl/
324
+ s_req.type = Request::SET_ACL
325
+ else
326
+ s_req.type = Request::STORE
327
+ end
328
+ s_req.object = rack_req.path
329
+ end
330
+ end
331
+
332
+ copy_source = rack_req.env["HTTP_X_AMZ_COPY_SOURCE"]
333
+ if copy_source
334
+ src_elems = copy_source.split("/")
335
+ root_offset = src_elems[0] == "" ? 1 : 0
336
+ s_req.src_bucket = src_elems[root_offset]
337
+ s_req.src_object = src_elems[1 + root_offset,src_elems.size].join("/")
338
+ s_req.type = Request::COPY
339
+ end
340
+ end
341
+
342
+ def normalize_post(rack_req,s_req)
343
+ path = rack_req.path
344
+ path_len = path.size
345
+ s_req.path = rack_req.params['key']
346
+ s_req.type = Request::STORE
347
+
348
+ s_req.rack_request = rack_req
349
+ end
350
+
351
+ def nomalize_options(rack_req, s_req)
352
+ s_req.type = Request::OPTIONS
353
+ end
354
+
355
+ # This method takes a rack request and generates a normalized RackFakeS3 request
356
+ def normalize_request(rack_req)
357
+ host = rack_req.host
358
+
359
+ s_req = Request.new
360
+ s_req.path = rack_req.path
361
+ s_req.is_path_style = true
362
+ s_req.rack_request = rack_req
363
+
364
+ if !@root_hostnames.include?(host)
365
+ s_req.bucket = host.split(".")[0]
366
+ s_req.is_path_style = false
367
+ end
368
+
369
+ s_req.http_verb = rack_req.request_method
370
+
371
+ case rack_req.request_method
372
+ when 'PUT'
373
+ normalize_put(rack_req,s_req)
374
+ when 'GET','HEAD'
375
+ normalize_get(rack_req,s_req)
376
+ when 'DELETE'
377
+ normalize_delete(rack_req,s_req)
378
+ when 'POST'
379
+ normalize_post(rack_req,s_req)
380
+ when 'OPTIONS'
381
+ nomalize_options(rack_req,s_req)
382
+ else
383
+ return false
384
+ end
385
+
386
+ if s_req.type.nil?
387
+ return false
388
+ end
389
+
390
+ return s_req
391
+ end
392
+
393
+ def dump_request(request)
394
+ puts "----------Dump Request-------------"
395
+ puts request.request_method
396
+ puts request.path
397
+ request.each do |k,v|
398
+ puts "#{k}:#{v}"
399
+ end
400
+ puts "----------End Dump -------------"
401
+ end
402
+ end
403
+
404
+ class App
405
+ def initialize(path, hostname)
406
+ @servlet = Servlet.new(nil, RackFakeS3::FileStore.new(path), hostname)
407
+ end
408
+
409
+ def call(env)
410
+ @servlet.call(env)
411
+ end
412
+
413
+ def inspect
414
+ self.class.inspect
415
+ end
416
+ end
417
+
418
+ class Server
419
+ def initialize(port,root,hostname)
420
+ @port = port
421
+ @root = root
422
+ @hostname = hostname
423
+ end
424
+
425
+ def serve
426
+ ENV['FAKE_S3_ROOT'] = @root
427
+ ENV['FAKE_S3_HOSTNAME'] = @hostname
428
+
429
+ @server = Rack::Server.new(:Port => @port, :config => config_ru)
430
+
431
+ @server.start
432
+ end
433
+
434
+ def config_ru
435
+ File.expand_path("../../../config.ru", __FILE__)
436
+ end
437
+
438
+ def shutdown
439
+ @server.shutdown
440
+ end
441
+ end
442
+ end
@@ -0,0 +1,100 @@
1
+ require 'set'
2
+ module RackFakeS3
3
+ class S3MatchSet
4
+ attr_accessor :matches,:is_truncated
5
+ def initialize
6
+ @matches = []
7
+ @is_truncated = false
8
+ end
9
+ end
10
+
11
+ # This class has some of the semantics necessary for how buckets can return
12
+ # their items
13
+ #
14
+ # It is currently implemented naively as a sorted set + hash If you are going
15
+ # to try to put massive lists inside buckets and ls them, you will be sorely
16
+ # disappointed about this performance.
17
+ class SortedObjectList
18
+
19
+ def initialize
20
+ @sorted_set = SortedSet.new
21
+ @object_map = {}
22
+ @mutex = Mutex.new
23
+ end
24
+
25
+ def count
26
+ @sorted_set.count
27
+ end
28
+
29
+ def find(object_name)
30
+ @object_map[object_name]
31
+ end
32
+
33
+ # Add an S3 object into the sorted list
34
+ def add(s3_object)
35
+ return if !s3_object
36
+
37
+ @object_map[s3_object.name] = s3_object
38
+ @sorted_set << s3_object
39
+ end
40
+
41
+ def remove(s3_object)
42
+ return if !s3_object
43
+
44
+ @object_map.delete(s3_object.name)
45
+ @sorted_set.delete(s3_object)
46
+ end
47
+
48
+ # Return back a set of matches based on the passed in options
49
+ #
50
+ # options:
51
+ #
52
+ # :marker : a string to start the lexographical search (it is not included
53
+ # in the result)
54
+ # :max_keys : a maximum number of results
55
+ # :prefix : a string to filter the results by
56
+ # :delimiter : not supported yet
57
+ def list(options)
58
+ marker = options[:marker]
59
+ prefix = options[:prefix]
60
+ max_keys = options[:max_keys] || 1000
61
+ delimiter = options[:delimiter]
62
+
63
+ ms = S3MatchSet.new
64
+
65
+ marker_found = true
66
+ pseudo = nil
67
+ if marker
68
+ marker_found = false
69
+ if !@object_map[marker]
70
+ pseudo = S3Object.new
71
+ pseudo.name = marker
72
+ @sorted_set << pseudo
73
+ end
74
+ end
75
+
76
+ count = 0
77
+ @sorted_set.each do |s3_object|
78
+ if marker_found && (!prefix or s3_object.name.index(prefix) == 0)
79
+ count += 1
80
+ if count <= max_keys
81
+ ms.matches << s3_object
82
+ else
83
+ is_truncated = true
84
+ break
85
+ end
86
+ end
87
+
88
+ if marker and marker == s3_object.name
89
+ marker_found = true
90
+ end
91
+ end
92
+
93
+ if pseudo
94
+ @sorted_set.delete(pseudo)
95
+ end
96
+
97
+ return ms
98
+ end
99
+ end
100
+ end
@@ -0,0 +1,3 @@
1
+ module RackFakeS3
2
+ VERSION = '0.2.0'
3
+ end