fakes3 0.1.5.2 → 0.1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0fd5e9498cd058b485b059f03ca06d9030d8d7c1
-  data.tar.gz: 31b5e10a57f2a723a8a04bbdec62f7c726c8cb5c
+  metadata.gz: afc9d8c94ab0b848399aae28b8031acf3a27cadb
+  data.tar.gz: d46b07e6f772a54bcece517ae7a7c6517a4243c8
 SHA512:
-  metadata.gz: 56353c5a81798343a51989019c484a5e203b91188e813a3bddbb33ff120d451b5d99c7c6c2d9cbf13b2c929f9c651eb9d0f846ddae3d1529fbfc3718b96d07e0
-  data.tar.gz: b24c2db7d546dd0772dd2c897f90d2ae331b9b286f326be62e8d5794eaa606b331e48a0216c3fddb0ccffa42719c7802b1b8952f3d9c0b796d8a27943470e6cb
+  metadata.gz: 9db326bbb641bfedc2be1553b1846dcaab6a886d133a50be6912de8bce365a4bf509e73399650508148d125bee1cd11039a2988d2a644d081f7bdc2ef79f79d3
+  data.tar.gz: 3dafd2e1c59227812fd5b004c99e8a9c41287402d154cc3eea8d70fba121b94b6d88822e37f173e5362de976f8d7a46c39ec1dedbcb63fb0ee8a854a33d3861b

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fakes3 (0.1.5.2)
+    fakes3 (0.1.6.0)
       builder
       thor
 
@@ -15,6 +15,8 @@ GEM
     builder (3.2.2)
     mime-types (1.25)
     rake (10.1.0)
+    rest-client (1.6.7)
+      mime-types (>= 1.16)
     right_aws (3.1.0)
       right_http_connection (>= 1.2.5)
     right_http_connection (1.4.0)
@@ -29,4 +31,5 @@ DEPENDENCIES
   bundler (>= 1.0.0)
   fakes3!
   rake
+  rest-client
   right_aws

data/README.md

@@ -1,7 +1,7 @@
 ## Introduction
-FakeS3 is a lightweight server that responds to the same calls Amazon S3 responds to.  
+FakeS3 is a lightweight server that responds to the same calls Amazon S3 responds to.
 It is extremely useful for testing of S3 in a sandbox environment without actually
-making calls to Amazon, which not only require network, but also cost you precious dollars.  
+making calls to Amazon, which not only require network, but also cost you precious dollars.
 
 The goal of Fake S3 is to minimize runtime dependencies and be more of a
 development tool to test S3 calls in your code rather than a production server
@@ -31,6 +31,20 @@ Here is a running list of [supported clients](https://github.com/jubos/fake-s3/w
 
 ## Running Tests
 
+There are some pre-requesites to actually being able to run the unit/integration tests
+
+### On OSX
+
+Edit your /etc/hosts and add the following line:
+
+    127.0.0.1 posttest.localhost
+
+Then ensure that the following packages are installed (boto, s3cmd)
+
+    > pip install boto
+    > brew install s3cmd
+
+
 Start the test server using
 
     rake test_server
@@ -39,7 +53,7 @@ Then in another terminal window run
 
     rake test
 
-It is a TODO to get this to be just one command
+It is a still a TODO to get this to be just one command
 
 ## More Information
 

data/fakes3.gemspec

@@ -16,10 +16,11 @@ Gem::Specification.new do |s|
   s.add_development_dependency "bundler", ">= 1.0.0"
   s.add_development_dependency "aws-s3"
   s.add_development_dependency "right_aws"
+  s.add_development_dependency "rest-client"
   s.add_development_dependency "rake"
   #s.add_development_dependency "aws-sdk"
   #s.add_development_dependency "ruby-debug"
-  #s.add_development_dependency "ruby-debug19"
+  #s.add_development_dependency "debugger"
   s.add_dependency "thor"
   s.add_dependency "builder"
 

data/lib/fakes3/file_store.rb

@@ -82,8 +82,9 @@ module FakeS3
         #real_obj.io = File.open(File.join(obj_root,"content"),'rb')
         real_obj.io = RateLimitableFile.open(File.join(obj_root,"content"),'rb')
         real_obj.size = metadata.fetch(:size) { 0 }
-        real_obj.creation_date = File.ctime(obj_root).iso8601()
-        real_obj.modified_date = metadata.fetch(:modified_date) { File.mtime(File.join(obj_root,"content")).iso8601() }
+        real_obj.creation_date = File.ctime(obj_root).utc.iso8601()
+        real_obj.modified_date = metadata.fetch(:modified_date) { File.mtime(File.join(obj_root,"content")).utc.iso8601() }
+        real_obj.custom_metadata = metadata.fetch(:custom_metadata) { {} }
         return real_obj
       rescue
         puts $!
@@ -95,7 +96,7 @@ module FakeS3
     def object_metadata(bucket,object)
     end
 
-    def copy_object(src_bucket_name,src_name,dst_bucket_name,dst_name)
+    def copy_object(src_bucket_name,src_name,dst_bucket_name,dst_name,request)
       src_root = File.join(@root,src_bucket_name,src_name,SHUCK_METADATA_DIR)
       src_metadata_filename = File.join(src_root,"metadata")
       src_metadata = YAML.load(File.open(src_metadata_filename,'rb').read)
@@ -110,15 +111,25 @@ module FakeS3
       content = File.join(metadata_dir,"content")
       metadata = File.join(metadata_dir,"metadata")
 
-      File.open(content,'wb') do |f|
-        File.open(src_content_filename,'rb') do |input|
-          f << input.read
+      if src_bucket_name != dst_bucket_name || src_name != dst_name
+        File.open(content,'wb') do |f|
+          File.open(src_content_filename,'rb') do |input|
+            f << input.read
+          end
+        end
+
+        File.open(metadata,'w') do |f|
+          File.open(src_metadata_filename,'r') do |input|
+            f << input.read
+          end
         end
       end
 
-      File.open(metadata,'w') do |f|
-        File.open(src_metadata_filename,'r') do |input|
-          f << input.read
+      metadata_directive = request.header["x-amz-metadata-directive"].first
+      if metadata_directive == "REPLACE"
+        metadata_struct = create_metadata(content,request)
+        File.open(metadata,'w') do |f|
+          f << YAML::dump(metadata_struct)
         end
       end
 
@@ -149,21 +160,25 @@ module FakeS3
         content = File.join(filename,SHUCK_METADATA_DIR,"content")
         metadata = File.join(filename,SHUCK_METADATA_DIR,"metadata")
 
-        md5 = Digest::MD5.new
         # TODO put a tmpfile here first and mv it over at the end
 
-        File.open(content,'wb') do |f|
-          request.body do |chunk|
-            f << chunk
-            md5 << chunk
+        match=request.content_type.match(/^multipart\/form-data; boundary=(.+)/)
+      	boundary = match[1] if match
+        if boundary
+          boundary = WEBrick::HTTPUtils::dequote(boundary)
+          filedata = WEBrick::HTTPUtils::parse_form_data(request.body, boundary)
+          raise HTTPStatus::BadRequest if filedata['file'].empty?
+          File.open(content, 'wb') do |f|
+            f << filedata['file']
+          end
+        else
+          File.open(content,'wb') do |f|
+            request.body do |chunk|
+              f << chunk
+            end
           end
         end
-
-        metadata_struct = {}
-        metadata_struct[:md5] = md5.hexdigest
-        metadata_struct[:content_type] = request.header["content-type"].first
-        metadata_struct[:size] = File.size(content)
-        metadata_struct[:modified_date] = File.mtime(content).iso8601()
+        metadata_struct = create_metadata(content,request)
 
         File.open(metadata,'w') do |f|
           f << YAML::dump(metadata_struct)
@@ -197,5 +212,22 @@ module FakeS3
         return nil
       end
     end
+
+    def create_metadata(content,request)
+      metadata = {}
+      metadata[:md5] = Digest::MD5.file(content).hexdigest
+      metadata[:content_type] = request.header["content-type"].first
+      metadata[:size] = File.size(content)
+      metadata[:modified_date] = File.mtime(content).utc.iso8601()
+
+      # Add custom metadata from the request header
+      request.header.each do |key, value|
+        match = /^x-amz-meta-(.*)$/.match(key)
+        if match
+          metadata_struct[:custom_metadata][match[1]] = value.join(', ')
+        end
+      end
+      return metadata
+    end
   end
 end

data/lib/fakes3/s3_object.rb

@@ -1,7 +1,7 @@
 module FakeS3
   class S3Object
     include Comparable
-    attr_accessor :name,:size,:creation_date,:modified_date,:md5,:io,:content_type
+    attr_accessor :name,:size,:creation_date,:modified_date,:md5,:io,:content_type,:custom_metadata
 
     def hash
       @name.hash

data/lib/fakes3/server.rb

@@ -1,3 +1,4 @@
+require 'time'
 require 'webrick'
 require 'fakes3/file_store'
 require 'fakes3/xml_adapter'
@@ -43,6 +44,7 @@ module FakeS3
       super(server)
       @store = store
       @hostname = hostname
+      @port = server.config[:Port]
       @root_hostnames = [hostname,'localhost','s3.amazonaws.com','s3.localhost']
     end
 
@@ -81,7 +83,8 @@ module FakeS3
         real_obj = @store.get_object(s_req.bucket,s_req.object,request)
         if !real_obj
           response.status = 404
-          response.body = ""
+          response.body = XmlAdapter.error_no_such_key(s_req.object)
+          response['Content-Type'] = "application/xml"
           return
         end
 
@@ -89,11 +92,15 @@ module FakeS3
         response['Content-Type'] = real_obj.content_type
         stat = File::Stat.new(real_obj.io.path)
 
-        response['Last-Modified'] = stat.mtime.iso8601()
+        response['Last-Modified'] = Time.iso8601(real_obj.modified_date).httpdate()
         response.header['ETag'] = "\"#{real_obj.md5}\""
         response['Accept-Ranges'] = "bytes"
         response['Last-Ranges'] = "bytes"
 
+        real_obj.custom_metadata.each do |header, value|
+          response.header['x-amz-meta-' + header] = value
+        end
+
         content_length = stat.size
 
         # Added Range Query support
@@ -118,7 +125,6 @@ module FakeS3
           end
         end
         response['Content-Length'] = File::Stat.new(real_obj.io.path).size
-        response['Last-Modified'] = real_obj.modified_date
         if s_req.http_verb == 'HEAD'
           response.body = ""
         else
@@ -130,9 +136,14 @@ module FakeS3
     def do_PUT(request,response)
       s_req = normalize_request(request)
 
+      response.status = 200
+      response.body = ""
+      response['Content-Type'] = "text/xml"
+
       case s_req.type
       when Request::COPY
-        @store.copy_object(s_req.src_bucket,s_req.src_object,s_req.bucket,s_req.object)
+        object = @store.copy_object(s_req.src_bucket,s_req.src_object,s_req.bucket,s_req.object,request)
+        response.body = XmlAdapter.copy_object_result(object)
       when Request::STORE
         bucket_obj = @store.get_bucket(s_req.bucket)
         if !bucket_obj
@@ -145,14 +156,46 @@ module FakeS3
       when Request::CREATE_BUCKET
         @store.create_bucket(s_req.bucket)
       end
-
-      response.status = 200
-      response.body = ""
-      response['Content-Type'] = "text/xml"
     end
 
-    # Posts aren't supported yet
     def do_POST(request,response)
+      # check that we've received file data
+      unless request.content_type =~ /^multipart\/form-data; boundary=(.+)/
+        raise WEBrick::HTTPStatus::BadRequest
+      end
+      s_req = normalize_request(request)
+      key=request.query['key']
+      success_action_redirect=request.query['success_action_redirect']
+      success_action_status=request.query['success_action_status']
+
+      filename = 'default'
+      filename = $1 if request.body =~ /filename="(.*)"/
+      key=key.gsub('${filename}', filename)
+      
+      bucket_obj = @store.get_bucket(s_req.bucket) || @store.create_bucket(s_req.bucket)
+      real_obj=@store.store_object(bucket_obj, key, s_req.webrick_request)
+      
+      response['Etag'] = "\"#{real_obj.md5}\""
+      response.body = ""
+      if success_action_redirect
+        response.status = 307
+        response['Location']=success_action_redirect
+      else
+        response.status = success_action_status || 204
+        if response.status=="201"
+          response.body= <<-eos.strip
+            <?xml version="1.0" encoding="UTF-8"?>
+            <PostResponse>
+              <Location>http://#{s_req.bucket}.localhost:#{@port}/#{key}</Location>
+              <Bucket>#{s_req.bucket}</Bucket>
+              <Key>#{key}</Key>
+              <ETag>#{response['Etag']}</ETag>
+            </PostResponse>
+          eos
+        end
+      end
+      response['Content-Type'] = 'text/xml'
+      response['Access-Control-Allow-Origin']='*'
     end
 
     def do_DELETE(request,response)
@@ -169,6 +212,11 @@ module FakeS3
       response.status = 204
       response.body = ""
     end
+    
+    def do_OPTIONS(request, response)
+      super
+      response["Access-Control-Allow-Origin"]="*"
+    end  
 
     private
 
@@ -255,7 +303,7 @@ module FakeS3
           else
             s_req.type = Request::STORE
           end
-          s_req.object = webrick_req.path
+          s_req.object = webrick_req.path[1..-1]
         end
       end
 
@@ -271,6 +319,15 @@ module FakeS3
       s_req.webrick_request = webrick_req
     end
 
+    def normalize_post(webrick_req,s_req)
+      path = webrick_req.path
+      path_len = path.size
+      
+      s_req.path = webrick_req.query['key']
+
+      s_req.webrick_request = webrick_req
+    end
+
     # This method takes a webrick request and generates a normalized FakeS3 request
     def normalize_request(webrick_req)
       host_header= webrick_req["Host"]
@@ -294,6 +351,8 @@ module FakeS3
         normalize_get(webrick_req,s_req)
       when 'DELETE'
         normalize_delete(webrick_req,s_req)
+      when 'POST'
+        normalize_post(webrick_req,s_req)
       else
         raise "Unknown Request"
       end

data/lib/fakes3/version.rb

@@ -1,3 +1,3 @@
 module FakeS3
-  VERSION = "0.1.5.2"
+  VERSION = "0.1.6.0"
 end

data/lib/fakes3/xml_adapter.rb

@@ -175,5 +175,20 @@ module FakeS3
       }
       output
     end
+
+    # <CopyObjectResult>
+    #   <LastModified>2009-10-28T22:32:00</LastModified>
+    #   <ETag>"9b2cf535f27731c974343645a3985328"</ETag>
+    # </CopyObjectResult>
+    def self.copy_object_result(object)
+      output = ""
+      xml = Builder::XmlMarkup.new(:target => output)
+      xml.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+      xml.CopyObjectResult { |result|
+        result.LastModified(object.modified_date)
+        result.ETag("\"#{object.md5}\"")
+      }
+      output
+    end    
   end
 end

data/test/post_test.rb

@@ -0,0 +1,54 @@
+require 'test/test_helper'
+require 'rest-client'
+
+class PostTest < Test::Unit::TestCase
+  # Make sure you have a posttest.localhost in your /etc/hosts/
+  def setup
+    @url='http://posttest.localhost:10453/'
+  end
+
+  def teardown
+  end
+
+  def test_options
+    res= RestClient.options(@url) { |response|
+      assert_equal(response.headers[:access_control_allow_origin],"*")
+    }
+  end
+
+  def test_redirect
+    res = RestClient.post(
+      @url,
+      'key'=>'uploads/12345/${filename}',
+      'success_action_redirect'=>'http://somewhere.else.com/',
+      'file'=>File.new(__FILE__,"rb")
+    ) { |response|
+      assert_equal(response.code, 307)
+      assert_equal(response.headers[:location], 'http://somewhere.else.com/')
+    }
+  end
+
+  def test_status_200
+    res = RestClient.post(
+      @url,
+      'key'=>'uploads/12345/${filename}',
+      'success_action_status'=>'200',
+      'file'=>File.new(__FILE__,"rb")
+    ) { |response|
+      assert_equal(response.code, 200)
+    }
+  end
+
+  def test_status_201
+    res = RestClient.post(
+      @url,
+      'key'=>'uploads/12345/${filename}',
+      'success_action_status'=>'201',
+      'file'=>File.new(__FILE__,"rb")
+    ) { |response|
+      assert_equal(response.code, 201)
+      assert_match(%r{^\<\?xml.*uploads/12345/post_test\.rb}m, response.body)
+    }
+  end
+
+end

data/test/right_aws_commands_test.rb

@@ -27,6 +27,16 @@ class RightAWSCommandsTest < Test::Unit::TestCase
     obj = @s3.get("s3media","helloworld")
   end
 
+  def test_store_not_found
+    begin
+      obj = @s3.get("s3media","helloworldnotexist")
+    rescue RightAws::AwsError
+      assert $!.message.include?('NoSuchKey')
+    rescue
+      fail 'Should have caught NoSuchKey Exception'
+    end
+  end
+
   def test_large_store
     @s3.put("s3media","helloworld","Hello World Man!")
     buffer = ""
@@ -60,6 +70,24 @@ class RightAWSCommandsTest < Test::Unit::TestCase
     assert_equal "Hello World",obj[:object]
   end
 
+  def test_copy_in_place
+    @s3.put("s3media","foo","Hello World")
+    @s3.copy("s3media","foo","s3media","foo")
+    obj = @s3.get("s3media","foo")
+    assert_equal "Hello World",obj[:object]
+  end
+
+  def test_copy_replace_metadata
+    @s3.put("s3media","foo","Hello World",{"content-type"=>"application/octet-stream"})
+    obj = @s3.get("s3media","foo")
+    assert_equal "Hello World",obj[:object]
+    assert_equal "application/octet-stream",obj[:headers]["content-type"]
+    @s3.copy("s3media","foo","s3media","foo",:replace,{"content-type"=>"text/plain"})
+    obj = @s3.get("s3media","foo")
+    assert_equal "Hello World",obj[:object]
+    assert_equal "text/plain",obj[:headers]["content-type"]
+  end
+
   def test_larger_lists
     @s3.create_bucket('right_aws_many')
     (0..50).each do |i|

data/test/s3_commands_test.rb

@@ -67,6 +67,49 @@ class S3CommandsTest < Test::Unit::TestCase
     assert_equal buf_len,output.size
   end
 
+  def test_metadata_store
+    assert_equal true, Bucket.create("ruby_aws_s3")
+    bucket = Bucket.find("ruby_aws_s3")
+
+    # Note well: we can't seem to access obj.metadata until we've stored
+    # the object and found it again. Thus the store, find, store
+    # runaround below.
+    obj = bucket.new_object(:value => "foo")
+    obj.key = "key_with_metadata"
+    obj.store
+    obj = S3Object.find("key_with_metadata", "ruby_aws_s3")
+    obj.metadata[:param1] = "one"
+    obj.metadata[:param2] = "two, three"
+    obj.store
+    obj = S3Object.find("key_with_metadata", "ruby_aws_s3")
+
+    assert_equal "one", obj.metadata[:param1]
+    assert_equal "two, three", obj.metadata[:param2]
+  end
+
+  def test_metadata_copy
+    assert_equal true, Bucket.create("ruby_aws_s3")
+    bucket = Bucket.find("ruby_aws_s3")
+
+    # Note well: we can't seem to access obj.metadata until we've stored
+    # the object and found it again. Thus the store, find, store
+    # runaround below.
+    obj = bucket.new_object(:value => "foo")
+    obj.key = "key_with_metadata"
+    obj.store
+    obj = S3Object.find("key_with_metadata", "ruby_aws_s3")
+    obj.metadata[:param1] = "one"
+    obj.metadata[:param2] = "two, three"
+    obj.store
+
+    S3Object.copy("key_with_metadata", "key_with_metadata2", "ruby_aws_s3")
+    obj = S3Object.find("key_with_metadata2", "ruby_aws_s3")
+
+    assert_equal "one", obj.metadata[:param1]
+    assert_equal "two, three", obj.metadata[:param2]
+  end
+
+
   def test_multi_directory
     bucket = Bucket.create("ruby_aws_s3")
     S3Object.store("dir/myfile/123.txt","recursive","ruby_aws_s3")

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fakes3
 version: !ruby/object:Gem::Version
-  version: 0.1.5.2
+  version: 0.1.6.0
 platform: ruby
 authors:
 - Curtis Spencer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-21 00:00:00.000000000 Z
+date: 2014-11-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -52,6 +52,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -127,6 +141,7 @@ files:
 - test/boto_test.rb
 - test/botocmd.py
 - test/local_s3_cfg
+- test/post_test.rb
 - test/right_aws_commands_test.rb
 - test/s3_commands_test.rb
 - test/s3cmd_test.rb
@@ -150,7 +165,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: fakes3
-rubygems_version: 2.0.2
+rubygems_version: 2.0.14
 signing_key: 
 specification_version: 4
 summary: FakeS3 is a server that simulates S3 commands so you can test your S3 functionality
@@ -159,6 +174,7 @@ test_files:
 - test/boto_test.rb
 - test/botocmd.py
 - test/local_s3_cfg
+- test/post_test.rb
 - test/right_aws_commands_test.rb
 - test/s3_commands_test.rb
 - test/s3cmd_test.rb