fakes3 1.2.0 → 1.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d257019492e12513dd0de611fd983d31277420ee
-  data.tar.gz: fb8546e21de97909554805fc6082b07e7f43db48
+  metadata.gz: db9520921cf6e3133db80395102de95622fde8c1
+  data.tar.gz: 3627b3c4a37f71be5f9a3b6af00443efd5c4cdc2
 SHA512:
-  metadata.gz: 2d2cad4af718747f5b93ab6f6831c6230a209221c0d5950b8ba6b3976f78982edf7ddda23a33b144cd671bb661b3baec887c7a30eaa9af422473883d46e7397d
-  data.tar.gz: 125638e4c8b6d312cf34a26ebaa724dab1a87f70d2d61007f2e86f76e838a406731deba8f8fb50c2eac7ea54112b2cb8c01759a265ec9be5e21ac19e26976b45
+  metadata.gz: 34d0d8a995420525d0faf38ebae7d7bcba6b96cd38ec2297ed35a5a1702ab3912184cfee15b61caa9db6ed5041f8bec1ea7e425c485d0a52b88f1d40b49b138a
+  data.tar.gz: 43de1b4446acb27e2e99c0624a496e3c8b4efc7c65a3eda0342e8102351a6e1fe3876b966cc2d32c643bd804e62712de2a6562b05accc4e9c5dfafe40f1d6b37

data/.gitignore

@@ -14,3 +14,5 @@ test_root
 Gemfile.lock
 
 .DS_Store
+.rvmrc
+root/*

data/Gemfile

@@ -1,3 +1,3 @@
 source 'https://rubygems.org'
 # Specify your gem's dependencies in fakes3.gemspec
-gemspec
+gemspec

data/Rakefile

@@ -8,6 +8,10 @@ Rake::TestTask.new(:test) do |t|
   t.libs << "."
   t.test_files =
     FileList['test/*_test.rb'].exclude('test/s3_commands_test.rb')
+
+  # A lot of the gems like right aws and amazon sdk have a bunch of warnings, so
+  # this suppresses them for the test runs
+  t.warning = false
 end
 
 desc "Run the test_server"

data/fakes3.gemspec

@@ -26,6 +26,7 @@ Gem::Specification.new do |s|
   #s.add_development_dependency "debugger"
   s.add_dependency "thor"
   s.add_dependency "builder"
+  s.add_dependency "xml-simple"
 
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")

data/lib/fakes3/file_store.rb

@@ -96,6 +96,7 @@ module FakeS3
         real_obj.modified_date = metadata.fetch(:modified_date) do
           File.mtime(File.join(obj_root, "content")).utc.iso8601(SUBSECOND_PRECISION)
         end
+        real_obj.cache_control = metadata[:cache_control]
         real_obj.custom_metadata = metadata.fetch(:custom_metadata) { {} }
         return real_obj
       rescue
@@ -158,6 +159,7 @@ module FakeS3
       obj.content_encoding = src_metadata[:content_encoding] # if src_metadata[:content_encoding]
       obj.size = src_metadata[:size]
       obj.modified_date = src_metadata[:modified_date]
+      obj.cache_control = src_metadata[:cache_control]
 
       src_bucket.find(src_name)
       dst_bucket.add(obj)
@@ -214,6 +216,7 @@ module FakeS3
         obj.content_encoding = metadata_struct[:content_encoding] # if metadata_struct[:content_encoding]
         obj.size = metadata_struct[:size]
         obj.modified_date = metadata_struct[:modified_date]
+        obj.cache_control = metadata_struct[:cache_control]
 
         bucket.add(obj)
         return obj
@@ -269,6 +272,23 @@ module FakeS3
       end
     end
 
+    def delete_objects(bucket, objects, request)
+      begin
+        filenames = []
+        objects.each do |object_name|
+          filenames << File.join(@root,bucket.name,object_name)
+          object = bucket.find(object_name)
+          bucket.remove(object)
+        end
+
+        FileUtils.rm_rf(filenames)
+      rescue
+        puts $!
+        $!.backtrace.each { |line| puts line }
+        return nil
+      end
+    end
+
     # TODO: abstract getting meta data from request.
     def create_metadata(content, request)
       metadata = {}
@@ -277,6 +297,11 @@ module FakeS3
       if request.header['content-disposition']
         metadata[:content_disposition] = request.header['content-disposition'].first
       end
+
+      if request.header['cache-control']
+        metadata[:cache_control] = request.header['cache-control'].first
+      end
+
       content_encoding = request.header["content-encoding"].first
       metadata[:content_encoding] = content_encoding
       #if content_encoding

data/lib/fakes3/s3_object.rb

@@ -1,7 +1,7 @@
 module FakeS3
   class S3Object
     include Comparable
-    attr_accessor :name,:size,:creation_date,:modified_date,:md5,:io,:content_type,:content_disposition,:content_encoding,:custom_metadata
+    attr_accessor :name,:size,:creation_date,:modified_date,:md5,:io,:content_type,:content_disposition,:content_encoding,:custom_metadata,:cache_control
 
     def hash
       @name.hash

data/lib/fakes3/server.rb

@@ -4,9 +4,11 @@ require 'webrick/https'
 require 'openssl'
 require 'securerandom'
 require 'cgi'
+require 'uri'
 require 'fakes3/util'
 require 'fakes3/file_store'
 require 'fakes3/xml_adapter'
+require 'fakes3/xml_parser'
 require 'fakes3/bucket_query'
 require 'fakes3/unsupported_operation'
 require 'fakes3/errors'
@@ -26,6 +28,7 @@ module FakeS3
     MOVE = "MOVE"
     DELETE_OBJECT = "DELETE_OBJECT"
     DELETE_BUCKET = "DELETE_BUCKET"
+    DELETE_OBJECTS = "DELETE_OBJECTS"
 
     attr_accessor :bucket, :object, :type, :src_bucket,
                   :src_object, :method, :webrick_request,
@@ -98,6 +101,7 @@ module FakeS3
           response.status = 404
           response.body = XmlAdapter.error_no_such_key(s_req.object)
           response['Content-Type'] = "application/xml"
+          response['Access-Control-Allow-Origin'] = '*'
           return
         end
 
@@ -124,8 +128,7 @@ module FakeS3
           response.header['Content-Encoding'] = real_obj.content_encoding
         end
 
-        response['Content-Disposition'] = real_obj.content_disposition if real_obj.content_disposition
-        stat = File::Stat.new(real_obj.io.path)
+        response['Content-Disposition'] = real_obj.content_disposition ? real_obj.content_disposition : 'attachment'
 
         response['Last-Modified'] = Time.iso8601(real_obj.modified_date).httpdate
         response.header['ETag'] = "\"#{real_obj.md5}\""
@@ -137,6 +140,7 @@ module FakeS3
           response.header['x-amz-meta-' + header] = value
         end
 
+        stat = File::Stat.new(real_obj.io.path)
         content_length = stat.size
 
         # Added Range Query support
@@ -168,6 +172,10 @@ module FakeS3
         else
           response.body = real_obj.io
         end
+
+        if real_obj.cache_control
+          response['Cache-Control'] = real_obj.cache_control
+        end
       end
     end
 
@@ -240,6 +248,10 @@ module FakeS3
     end
 
     def do_POST(request,response)
+      if request.query_string === 'delete'
+        return do_DELETE(request, response)
+      end
+
       s_req = normalize_request(request)
       key   = request.query['key']
       query = CGI::parse(request.request_uri.query || "")
@@ -319,6 +331,10 @@ module FakeS3
       s_req = normalize_request(request)
 
       case s_req.type
+      when Request::DELETE_OBJECTS
+        bucket_obj = @store.get_bucket(s_req.bucket)
+        keys = XmlParser.delete_objects(s_req.webrick_request)
+        @store.delete_objects(bucket_obj,keys,s_req.webrick_request)
       when Request::DELETE_OBJECT
         bucket_obj = @store.get_bucket(s_req.bucket)
         @store.delete_object(bucket_obj,s_req.object,s_req.webrick_request)
@@ -332,7 +348,6 @@ module FakeS3
 
     def do_OPTIONS(request, response)
       super
-
       response['Access-Control-Allow-Origin']   = '*'
       response['Access-Control-Allow-Methods']  = 'PUT, POST, HEAD, GET, OPTIONS'
       response['Access-Control-Allow-Headers']  = 'Accept, Content-Type, Authorization, Content-Length, ETag, X-CSRF-Token, Content-Disposition'
@@ -356,10 +371,13 @@ module FakeS3
         end
 
         if elems.size == 0
-          raise UnsupportedOperation
+          s_req.type = Request::DELETE_OBJECTS
+          s_req.query = query
+          s_req.webrick_request = webrick_req
         elsif elems.size == 1
-          s_req.type = Request::DELETE_BUCKET
+          s_req.type = webrick_req.query_string == 'delete' ? Request::DELETE_OBJECTS : Request::DELETE_BUCKET
           s_req.query = query
+          s_req.webrick_request = webrick_req
         else
           s_req.type = Request::DELETE_OBJECT
           object = elems[1,elems.size].join('/')
@@ -432,7 +450,8 @@ module FakeS3
       # for multipart copy
       copy_source = webrick_req.header["x-amz-copy-source"]
       if copy_source and copy_source.size == 1
-        src_elems   = copy_source.first.split("/")
+        copy_source = URI.unescape copy_source.first
+        src_elems   = copy_source.split("/")
         root_offset = src_elems[0] == "" ? 1 : 0
         s_req.src_bucket = src_elems[root_offset]
         s_req.src_object = src_elems[1 + root_offset,src_elems.size].join("/")
@@ -482,7 +501,11 @@ module FakeS3
       when 'DELETE'
         normalize_delete(webrick_req,s_req)
       when 'POST'
-        normalize_post(webrick_req,s_req)
+        if webrick_req.query_string != 'delete'
+          normalize_post(webrick_req,s_req)
+        else
+          normalize_delete(webrick_req,s_req)
+        end
       else
         raise "Unknown Request"
       end

data/lib/fakes3/version.rb

@@ -1,3 +1,3 @@
 module FakeS3
-  VERSION = "1.2.0"
+  VERSION = "1.2.1"
 end

data/lib/fakes3/xml_parser.rb

@@ -0,0 +1,16 @@
+require 'xmlsimple'
+
+module FakeS3
+  class XmlParser
+    def self.delete_objects(request)
+      keys = []
+
+      objects = XmlSimple.xml_in(request.body, {'NoAttr' => true})['Object']
+      objects.each do |key|
+        keys << key['Key'][0]
+      end
+
+      keys
+    end
+  end
+end

data/test/botocmd.py

@@ -1,6 +1,8 @@
 #!/usr/bin/python
+
 # -*- coding: utf-8 -*-
 # fakes3cmd.py -- an s3cmd-like script that accepts a custom host and portname
+from __future__ import print_function
 import re
 import os
 from optparse import OptionParser
@@ -21,12 +23,12 @@ class FakeS3Cmd(object):
         self._connect()
 
     def _connect(self):
-        print 'Connecting: %s:%s' % (self.host, self.port)
+        print('Connecting: %s:%s' % (self.host, self.port))
         self.conn = S3Connection(is_secure=False,
-                             calling_format=OrdinaryCallingFormat(),
-                             aws_access_key_id='',
-                             aws_secret_access_key='',
-                             port=self.port, host=self.host)
+                                 calling_format=OrdinaryCallingFormat(),
+                                 aws_access_key_id='',
+                                 aws_secret_access_key='',
+                                 port=self.port, host=self.host)
 
 
     @staticmethod
@@ -41,7 +43,7 @@ class FakeS3Cmd(object):
 
         bucket, _ = self._parse_uri(path)
         self.conn.create_bucket(bucket)
-        print 'made bucket: [%s]' % bucket
+        print('made bucket: [%s]' % bucket)
 
     def rb(self, path, *args):
         if not self.conn:
@@ -49,7 +51,7 @@ class FakeS3Cmd(object):
 
         bucket, _ = self._parse_uri(path)
         self.conn.delete_bucket(bucket)
-        print 'removed bucket: [%s]' % bucket
+        print('removed bucket: [%s]' % bucket)
 
     def put(self, *args):
         if not self.conn:
@@ -63,7 +65,7 @@ class FakeS3Cmd(object):
             key = Key(bucket)
             key.key = os.path.join(prefix, os.path.basename(src_file))
             key.set_contents_from_filename(src_file)
-            print 'stored: [%s]' % key.key
+            print('stored: [%s]' % key.key)
 
 
 if __name__ == "__main__":

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fakes3
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.2.1
 platform: ruby
 authors:
 - Curtis Spencer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-13 00:00:00.000000000 Z
+date: 2017-12-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -164,6 +164,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: xml-simple
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Use Fake S3 to test basic Amazon S3 functionality without actually connecting
   to AWS
 email:
@@ -200,6 +214,7 @@ files:
 - lib/fakes3/util.rb
 - lib/fakes3/version.rb
 - lib/fakes3/xml_adapter.rb
+- lib/fakes3/xml_parser.rb
 - static/button.svg
 - static/logo.png
 - test/aws_sdk_commands_test.rb
@@ -210,7 +225,6 @@ files:
 - test/local_s3_cfg
 - test/minitest_helper.rb
 - test/post_test.rb
-- test/right_aws_commands_test.rb
 - test/s3_commands_test.rb
 - test/s3cmd_test.rb
 - test/test_helper.rb
@@ -235,7 +249,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: Fake S3 is a server that simulates Amazon S3 commands so you can test your
@@ -249,7 +263,6 @@ test_files:
 - test/local_s3_cfg
 - test/minitest_helper.rb
 - test/post_test.rb
-- test/right_aws_commands_test.rb
 - test/s3_commands_test.rb
 - test/s3cmd_test.rb
 - test/test_helper.rb

data/test/right_aws_commands_test.rb

@@ -1,219 +0,0 @@
-require 'test/test_helper'
-require 'fileutils'
-require 'right_aws'
-require 'time'
-
-class RightAWSCommandsTest < Test::Unit::TestCase
-
-  def setup
-    @s3 = RightAws::S3Interface.new('1E3GDYEOGFJPIT7XXXXXX','hgTHt68JY07JKUY08ftHYtERkjgtfERn57XXXXXX',
-                                    {:multi_thread => false, :server => 'localhost',
-                                      :port => 10453, :protocol => 'http', :logger => Logger.new("/dev/null"),
-                                      :no_subdomains => true })
-  end
-
-  def teardown
-  end
-
-  def test_create_bucket
-    bucket = @s3.create_bucket("s3media")
-    assert_not_nil bucket
-  end
-
-  def test_store
-    @s3.put("s3media", "helloworld", "Hello World Man!")
-    obj = @s3.get("s3media", "helloworld")
-    assert_equal "Hello World Man!", obj[:object]
-  end
-
-  # TODO - get Chinese to work
-  #def test_store_chinese
-  #  ni_hao = "你好"
-  #  great_wall = "中国的长城"
-#
-  #  @s3.put("s3media", ni_hao, great_wall)
-  #  obj = @s3.get("s3media", ni_hao)
-  #  assert_equal(great_wall, obj[:object])
-  #end
-
-  def test_store_not_found
-    begin
-      obj = @s3.get("s3media", "helloworldnotexist")
-    rescue RightAws::AwsError
-      assert $!.message.include?('NoSuchKey')
-    rescue
-      fail 'Should have caught NoSuchKey Exception'
-    end
-  end
-
-  def test_large_store
-    @s3.put("s3media", "helloworld", "Hello World Man!")
-    buffer = ""
-    500000.times do
-      buffer << "#{(rand * 100).to_i}"
-    end
-
-    buf_len = buffer.length
-    time_before = Time.now
-    @s3.put("s3media", "big", buffer)
-
-    output = ""
-    @s3.get("s3media","big") do |chunk|
-      output << chunk
-    end
-    time_after = Time.now
-
-    assert(time_after - time_before < 2) # Should run in under 2 seconds on normal machines
-    assert_equal(buf_len, output.size)
-  end
-
-  # Test that GET requests with a delimiter return a list of
-  def test_list_by_delimiter
-    @s3.create_bucket("s3media")
-
-    @s3.put("s3media", "delimited/item", "item")
-
-    expected_prefixes = []
-    (1..50).each do |i|
-      key_prefix = "delimited/%02d/" % i
-      @s3.put("s3media", key_prefix + "foo", "foo")
-      @s3.put("s3media", key_prefix + "fie", "fie")
-      expected_prefixes << key_prefix
-    end
-
-    key_names = []
-    common_prefixes = []
-    @s3.incrementally_list_bucket("s3media", {:prefix => "delimited", :delimiter => '/'}) do |currentResponse|
-      common_prefixes += currentResponse[:common_prefixes]
-    end
-    assert_equal ["delimited/"], common_prefixes
-
-    common_prefixes = []
-    @s3.incrementally_list_bucket("s3media", {:prefix => "delimited/", :delimiter => '/', "max-keys" => 100}) do |currentResponse|
-      key_names += currentResponse[:contents].map do |key|
-        key[:key]
-      end
-      common_prefixes += currentResponse[:common_prefixes]
-    end
-    assert_equal expected_prefixes, common_prefixes
-    assert_equal ["delimited/item"], key_names
-  end
-
-  def test_multi_directory
-    @s3.put("s3media", "dir/right/123.txt", "recursive")
-    output = ""
-    obj = @s3.get("s3media", "dir/right/123.txt") do |chunk|
-      output << chunk
-    end
-    assert_equal "recursive", output
-  end
-
-  def test_intra_bucket_copy
-    @s3.put("s3media", "original.txt", "Hello World")
-    @s3.copy("s3media", "original.txt", "s3media", "copy.txt")
-    obj = @s3.get("s3media", "copy.txt")
-    assert_equal "Hello World", obj[:object]
-  end
-
-  def test_copy_in_place
-    @s3.put("s3media", "copy-in-place", "Hello World")
-    @s3.copy("s3media", "copy-in-place", "s3media","copy-in-place")
-    obj = @s3.get("s3media", "copy-in-place")
-    assert_equal "Hello World", obj[:object]
-  end
-
-  def test_content_encoding
-    foo_compressed = Zlib::Deflate.deflate("foo")
-    @s3.put("s3media", "foo", foo_compressed, {"content-encoding" => "gzip"})
-    obj = @s3.get("s3media", "foo")
-    # assert_equal "gzip", obj[:headers]["content-encoding"] # TODO why doesn't checking content-encoding work?
-    assert_equal "gzip", obj[:headers]["x-content-encoding"] # TODO why doesn't checking content-encoding work?
-  end
-
-  # def test_content_encoding_data
-  #   foo_compressed = Zlib::Deflate.deflate("foo-two")
-  #   @s3.put("s3media", "foo-two", foo_compressed, {"content-encoding" => "gzip"})
-  #   obj = @s3.get("s3media", "foo-two")
-  #   puts "*** GOT HERE 1 #{ obj[:object] }"
-  #   assert_equal "foo-two", Zlib::Inflate::inflate(obj[:object])
-  # end
-
-  def test_copy_replace_metadata
-    @s3.put("s3media", "copy_replace", "Hello World", {"content-type" => "application/octet-stream"})
-    obj = @s3.get("s3media", "copy_replace")
-    assert_equal "Hello World", obj[:object]
-    assert_equal "application/octet-stream", obj[:headers]["content-type"]
-    @s3.copy("s3media", "copy_replace", "s3media", "copy_replace", :replace, {"content-type"=>"text/plain"})
-    obj = @s3.get("s3media", "copy_replace")
-    assert_equal "Hello World", obj[:object]
-    assert_equal "text/plain", obj[:headers]["content-type"]
-  end
-
-  def test_larger_lists
-    @s3.create_bucket('right_aws_many')
-    (0..50).each do |i|
-      ('a'..'z').each do |letter|
-        name = "#{letter}#{i}"
-        @s3.put('right_aws_many', name, 'asdf')
-      end
-    end
-
-    keys = @s3.list_bucket('right_aws_many')
-    assert_equal(1000, keys.size)
-    assert_equal('a0', keys.first[:key])
-  end
-
-  def test_destroy_bucket
-    @s3.create_bucket('deletebucket')
-    @s3.delete_bucket('deletebucket')
-
-    begin
-      bucket = @s3.list_bucket('deletebucket')
-      fail("Shouldn't succeed here")
-    rescue RightAws::AwsError
-      assert $!.message.include?('NoSuchBucket')
-    rescue
-      fail 'Should have caught NoSuchBucket Exception'
-    end
-  end
-
-  def test_if_none_match
-    @s3.put("s3media", "if_none_match_test", "Hello World 1!")
-    obj = @s3.get("s3media", "if_none_match_test")
-    tag = obj[:headers]["etag"]
-    begin
-      @s3.get("s3media", "if_none_match_test", {"If-None-Match" => tag})
-    rescue URI::InvalidURIError
-      # expected error for 304
-    else
-      fail 'Should have encountered an error due to the server not returning a response due to caching'
-    end
-    @s3.put("s3media", "if_none_match_test", "Hello World 2!")
-    obj = @s3.get("s3media", "if_none_match_test", {"If-None-Match" => tag})
-    assert_equal "Hello World 2!", obj[:object]
-  end
-
-  def test_if_modified_since
-    @s3.put("s3media", "if_modified_since_test", "Hello World 1!")
-    obj = @s3.get("s3media", "if_modified_since_test")
-    modified = obj[:headers]["last-modified"]
-    begin
-      @s3.get("s3media", "if_modified_since_test", {"If-Modified-Since" => modified})
-    rescue URI::InvalidURIError
-      # expected error for 304
-    else
-      fail 'Should have encountered an error due to the server not returning a response due to caching'
-    end
-    # Granularity of an HTTP Date is 1 second which isn't enough for the test
-    # so manually rewind the clock by a second
-    time_in_the_past = Time.httpdate(modified) - 1
-    begin
-      obj = @s3.get("s3media", "if_modified_since_test", {"If-Modified-Since" => time_in_the_past.httpdate})
-    rescue
-      fail 'Should have been downloaded since the date is in the past now'
-    else
-      #expected scenario
-    end
-  end
-
-end