fakes3as 1.2.0

data/test/aws_sdk_v2_commands_test.rb

@@ -0,0 +1,65 @@
+require 'test/test_helper'
+require 'aws-sdk'
+
+class AwsSdkV2CommandsTest < Test::Unit::TestCase
+  def setup
+    @creds = Aws::Credentials.new('123', 'abc')
+    @s3    = Aws::S3::Client.new(credentials: @creds, region: 'us-east-1', endpoint: 'http://localhost:10453/')
+    @resource = Aws::S3::Resource.new(client: @s3)
+    @bucket = @resource.create_bucket(bucket: 'v2_bucket')
+
+    # Delete all objects to avoid sharing state between tests
+    @bucket.objects.each(&:delete)
+  end
+
+  def test_create_bucket
+    bucket = @resource.create_bucket(bucket: 'v2_create_bucket')
+    assert_not_nil bucket
+
+    bucket_names = @resource.buckets.map(&:name)
+    assert(bucket_names.index("v2_create_bucket") >= 0)
+  end
+
+  def test_destroy_bucket
+    @bucket.delete
+
+    begin
+      @s3.head_bucket(bucket: 'v2_bucket')
+      assert_fail("Shouldn't succeed here")
+    rescue
+    end
+  end
+
+  def test_create_object
+    object = @bucket.object('key')
+    object.put(body: 'test')
+
+    assert_equal 'test', object.get.body.string
+  end
+
+  def test_delete_object
+    object = @bucket.object('exists')
+    object.put(body: 'test')
+
+    assert_equal 'test', object.get.body.string
+
+    object.delete
+
+    assert_raise Aws::S3::Errors::NoSuchKey do
+      object.get
+    end
+  end
+
+  # TODO - get this test working
+  #
+  #def test_copy_object
+  #  object = @bucket.object("key_one")
+  #  object.put(body: 'asdf')
+
+  #  # TODO: explore why 'key1' won't work but 'key_one' will
+  #  object2 = @bucket.object('key_two')
+  #  object2.copy_from(copy_source: 'testing_copy/key_one')
+
+  #  assert_equal 2, @bucket.objects.count
+  #end
+end

data/test/boto_test.rb

@@ -0,0 +1,25 @@
+require 'test/test_helper'
+require 'fileutils'
+
+class BotoTest < Test::Unit::TestCase
+  def setup
+    cmdpath = File.expand_path(File.join(File.dirname(__FILE__),'botocmd.py'))
+    @botocmd = "python #{cmdpath} -t localhost -p 10453"
+  end
+
+  def teardown
+  end
+
+  def test_store
+    File.open(__FILE__,'rb') do |input|
+      File.open("/tmp/fakes3_upload",'wb') do |output|
+        output << input.read
+      end
+    end
+    output = `#{@botocmd} put /tmp/fakes3_upload s3://s3cmd_bucket/upload`
+    assert_match(/stored/,output)
+
+    FileUtils.rm("/tmp/fakes3_upload")
+  end
+
+end

data/test/botocmd.py

@@ -0,0 +1,87 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# fakes3cmd.py -- an s3cmd-like script that accepts a custom host and portname
+import re
+import os
+from optparse import OptionParser
+
+try:
+    from boto.s3.connection import S3Connection, OrdinaryCallingFormat
+    from boto.s3.key import Key
+except ImportError:
+    raise Exception('You must install the boto package for python')
+
+
+class FakeS3Cmd(object):
+    COMMANDS = ['mb', 'rb', 'put', ]
+    def __init__(self, host, port):
+        self.host = host
+        self.port = port
+        self.conn = None
+        self._connect()
+
+    def _connect(self):
+        print 'Connecting: %s:%s' % (self.host, self.port)
+        self.conn = S3Connection(is_secure=False,
+                             calling_format=OrdinaryCallingFormat(),
+                             aws_access_key_id='',
+                             aws_secret_access_key='',
+                             port=self.port, host=self.host)
+
+
+    @staticmethod
+    def _parse_uri(path):
+        match = re.match(r's3://([^/]+)(?:/(.*))?', path, re.I)
+        ## (bucket, key)
+        return match.groups()
+
+    def mb(self, path, *args):
+        if not self.conn:
+            self._connect()
+
+        bucket, _ = self._parse_uri(path)
+        self.conn.create_bucket(bucket)
+        print 'made bucket: [%s]' % bucket
+
+    def rb(self, path, *args):
+        if not self.conn:
+            self._connect()
+
+        bucket, _ = self._parse_uri(path)
+        self.conn.delete_bucket(bucket)
+        print 'removed bucket: [%s]' % bucket
+
+    def put(self, *args):
+        if not self.conn:
+            self._connect()
+
+        args = list(args)
+        path = args.pop()
+        bucket_name, prefix = self._parse_uri(path)
+        bucket = self.conn.create_bucket(bucket_name)
+        for src_file in args:
+            key = Key(bucket)
+            key.key = os.path.join(prefix, os.path.basename(src_file))
+            key.set_contents_from_filename(src_file)
+            print 'stored: [%s]' % key.key
+
+
+if __name__ == "__main__":
+    # check for options. TODO: This requires a more verbose help message
+    # to explain how the positional arguments work.
+    parser = OptionParser()
+    parser.add_option("-t", "--host", type="string", default='localhost')
+    parser.add_option("-p", "--port", type='int', default=80)
+    o, args = parser.parse_args()
+
+    if len(args) < 2:
+        raise ValueError('you must minimally supply a desired command and s3 uri')
+
+    cmd = args.pop(0)
+
+    if cmd not in FakeS3Cmd.COMMANDS:
+        raise ValueError('%s is not a valid command' % cmd)
+
+    fs3 = FakeS3Cmd(o.host, o.port)
+    handler = getattr(fs3, cmd)
+    handler(*args)

data/test/cli_test.rb

@@ -0,0 +1,18 @@
+require 'test/test_helper'
+require 'test/minitest_helper'
+require 'fakes3/cli'
+
+
+class CLITest < Test::Unit::TestCase
+  def setup
+    super
+    FakeS3::Server.any_instance.stubs(:serve)
+  end
+
+  def test_quiet_mode
+    script = FakeS3::CLI.new([], :root => '.', :port => 4567, :quiet => true)
+    assert_output('') do
+      script.invoke(:server)
+    end
+  end
+end

data/test/local_s3_cfg

@@ -0,0 +1,34 @@
+[default]
+access_key = abc
+acl_public = False
+bucket_location = US
+cloudfront_host = cloudfront.amazonaws.com
+cloudfront_resource = /2008-06-30/distribution
+default_mime_type = binary/octet-stream
+delete_removed = False
+dry_run = False
+encoding = UTF-8
+encrypt = False
+force = False
+get_continue = False
+gpg_command = None
+gpg_decrypt = %(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
+gpg_encrypt = %(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
+gpg_passphrase = 
+guess_mime_type = True
+host_base = localhost:10453
+host_bucket = %(bucket)s.localhost:10453
+human_readable_sizes = False
+list_md5 = False
+preserve_attrs = True
+progress_meter = True
+proxy_host = 
+proxy_port = 0
+recursive = False
+recv_chunk = 4096
+secret_key = def
+send_chunk = 4096
+simpledb_host = sdb.amazonaws.com
+skip_existing = False
+use_https = False
+verbosity = WARNING

data/test/minitest_helper.rb

@@ -0,0 +1,46 @@
+# LICENSE:
+#
+# (The MIT License)
+#
+# Copyright © Ryan Davis, seattle.rb
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the 'Software'), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+# The following is from minitest:
+# TODO - decide whether to switch to minitest or what to do about these:
+
+def capture_io
+  require 'stringio'
+
+  captured_stdout, captured_stderr = StringIO.new, StringIO.new
+
+  orig_stdout, orig_stderr = $stdout, $stderr
+  $stdout, $stderr         = captured_stdout, captured_stderr
+
+  begin
+    yield
+  ensure
+    $stdout = orig_stdout
+    $stderr = orig_stderr
+  end
+
+  return captured_stdout.string, captured_stderr.string
+end
+
+def assert_output stdout = nil, stderr = nil
+  out, err = capture_io do
+    yield
+  end
+
+  err_msg = Regexp === stderr ? :assert_match : :assert_equal if stderr
+  out_msg = Regexp === stdout ? :assert_match : :assert_equal if stdout
+
+  y = send err_msg, stderr, err, "In stderr" if err_msg
+  x = send out_msg, stdout, out, "In stdout" if out_msg
+
+  (!stdout || x) && (!stderr || y)
+end

data/test/post_test.rb

@@ -0,0 +1,58 @@
+require 'test/test_helper'
+require 'rest-client'
+
+class PostTest < Test::Unit::TestCase
+  # Make sure you have a posttest.localhost in your /etc/hosts/
+  def setup
+    @url='http://posttest.localhost:10453/'
+  end
+
+  def teardown
+  end
+
+  def test_options
+    RestClient.options(@url) do |response|
+      assert_equal(response.code, 200)
+      assert_equal(response.headers[:access_control_allow_origin],"*")
+      assert_equal(response.headers[:access_control_allow_methods], "PUT, POST, HEAD, GET, OPTIONS")
+      assert_equal(response.headers[:access_control_allow_headers], "Accept, Content-Type, Authorization, Content-Length, ETag, X-CSRF-Token, Content-Disposition")
+      assert_equal(response.headers[:access_control_expose_headers], "ETag")
+    end
+  end
+
+  def test_redirect
+    res = RestClient.post(
+      @url,
+      'key'=>'uploads/12345/${filename}',
+      'success_action_redirect'=>'http://somewhere.else.com/?foo=bar',
+      'file'=>File.new(__FILE__,"rb")
+    ) { |response|
+      assert_equal(response.code, 303)
+      assert_equal(response.headers[:location], 'http://somewhere.else.com/?foo=bar&bucket=posttest&key=uploads%2F12345%2Fpost_test.rb')
+    }
+  end
+
+  def test_status_200
+    res = RestClient.post(
+      @url,
+      'key'=>'uploads/12345/${filename}',
+      'success_action_status'=>'200',
+      'file'=>File.new(__FILE__,"rb")
+    ) { |response|
+      assert_equal(response.code, 200)
+    }
+  end
+
+  def test_status_201
+    res = RestClient.post(
+      @url,
+      'key'=>'uploads/12345/${filename}',
+      'success_action_status'=>'201',
+      'file'=>File.new(__FILE__,"rb")
+    ) { |response|
+      assert_equal(response.code, 201)
+      assert_match(%r{^\<\?xml.*uploads/12345/post_test\.rb}m, response.body)
+    }
+  end
+
+end

data/test/right_aws_commands_test.rb

@@ -0,0 +1,219 @@
+require 'test/test_helper'
+require 'fileutils'
+require 'right_aws'
+require 'time'
+
+class RightAWSCommandsTest < Test::Unit::TestCase
+
+  def setup
+    @s3 = RightAws::S3Interface.new('1E3GDYEOGFJPIT7XXXXXX','hgTHt68JY07JKUY08ftHYtERkjgtfERn57XXXXXX',
+                                    {:multi_thread => false, :server => 'localhost',
+                                      :port => 10453, :protocol => 'http', :logger => Logger.new("/dev/null"),
+                                      :no_subdomains => true })
+  end
+
+  def teardown
+  end
+
+  def test_create_bucket
+    bucket = @s3.create_bucket("s3media")
+    assert_not_nil bucket
+  end
+
+  def test_store
+    @s3.put("s3media", "helloworld", "Hello World Man!")
+    obj = @s3.get("s3media", "helloworld")
+    assert_equal "Hello World Man!", obj[:object]
+  end
+
+  # TODO - get Chinese to work
+  #def test_store_chinese
+  #  ni_hao = "你好"
+  #  great_wall = "中国的长城"
+#
+  #  @s3.put("s3media", ni_hao, great_wall)
+  #  obj = @s3.get("s3media", ni_hao)
+  #  assert_equal(great_wall, obj[:object])
+  #end
+
+  def test_store_not_found
+    begin
+      obj = @s3.get("s3media", "helloworldnotexist")
+    rescue RightAws::AwsError
+      assert $!.message.include?('NoSuchKey')
+    rescue
+      fail 'Should have caught NoSuchKey Exception'
+    end
+  end
+
+  def test_large_store
+    @s3.put("s3media", "helloworld", "Hello World Man!")
+    buffer = ""
+    500000.times do
+      buffer << "#{(rand * 100).to_i}"
+    end
+
+    buf_len = buffer.length
+    time_before = Time.now
+    @s3.put("s3media", "big", buffer)
+
+    output = ""
+    @s3.get("s3media","big") do |chunk|
+      output << chunk
+    end
+    time_after = Time.now
+
+    assert(time_after - time_before < 2) # Should run in under 2 seconds on normal machines
+    assert_equal(buf_len, output.size)
+  end
+
+  # Test that GET requests with a delimiter return a list of
+  def test_list_by_delimiter
+    @s3.create_bucket("s3media")
+
+    @s3.put("s3media", "delimited/item", "item")
+
+    expected_prefixes = []
+    (1..50).each do |i|
+      key_prefix = "delimited/%02d/" % i
+      @s3.put("s3media", key_prefix + "foo", "foo")
+      @s3.put("s3media", key_prefix + "fie", "fie")
+      expected_prefixes << key_prefix
+    end
+
+    key_names = []
+    common_prefixes = []
+    @s3.incrementally_list_bucket("s3media", {:prefix => "delimited", :delimiter => '/'}) do |currentResponse|
+      common_prefixes += currentResponse[:common_prefixes]
+    end
+    assert_equal ["delimited/"], common_prefixes
+
+    common_prefixes = []
+    @s3.incrementally_list_bucket("s3media", {:prefix => "delimited/", :delimiter => '/', "max-keys" => 100}) do |currentResponse|
+      key_names += currentResponse[:contents].map do |key|
+        key[:key]
+      end
+      common_prefixes += currentResponse[:common_prefixes]
+    end
+    assert_equal expected_prefixes, common_prefixes
+    assert_equal ["delimited/item"], key_names
+  end
+
+  def test_multi_directory
+    @s3.put("s3media", "dir/right/123.txt", "recursive")
+    output = ""
+    obj = @s3.get("s3media", "dir/right/123.txt") do |chunk|
+      output << chunk
+    end
+    assert_equal "recursive", output
+  end
+
+  def test_intra_bucket_copy
+    @s3.put("s3media", "original.txt", "Hello World")
+    @s3.copy("s3media", "original.txt", "s3media", "copy.txt")
+    obj = @s3.get("s3media", "copy.txt")
+    assert_equal "Hello World", obj[:object]
+  end
+
+  def test_copy_in_place
+    @s3.put("s3media", "copy-in-place", "Hello World")
+    @s3.copy("s3media", "copy-in-place", "s3media","copy-in-place")
+    obj = @s3.get("s3media", "copy-in-place")
+    assert_equal "Hello World", obj[:object]
+  end
+
+  def test_content_encoding
+    foo_compressed = Zlib::Deflate.deflate("foo")
+    @s3.put("s3media", "foo", foo_compressed, {"content-encoding" => "gzip"})
+    obj = @s3.get("s3media", "foo")
+    # assert_equal "gzip", obj[:headers]["content-encoding"] # TODO why doesn't checking content-encoding work?
+    assert_equal "gzip", obj[:headers]["x-content-encoding"] # TODO why doesn't checking content-encoding work?
+  end
+
+  # def test_content_encoding_data
+  #   foo_compressed = Zlib::Deflate.deflate("foo-two")
+  #   @s3.put("s3media", "foo-two", foo_compressed, {"content-encoding" => "gzip"})
+  #   obj = @s3.get("s3media", "foo-two")
+  #   puts "*** GOT HERE 1 #{ obj[:object] }"
+  #   assert_equal "foo-two", Zlib::Inflate::inflate(obj[:object])
+  # end
+
+  def test_copy_replace_metadata
+    @s3.put("s3media", "copy_replace", "Hello World", {"content-type" => "application/octet-stream"})
+    obj = @s3.get("s3media", "copy_replace")
+    assert_equal "Hello World", obj[:object]
+    assert_equal "application/octet-stream", obj[:headers]["content-type"]
+    @s3.copy("s3media", "copy_replace", "s3media", "copy_replace", :replace, {"content-type"=>"text/plain"})
+    obj = @s3.get("s3media", "copy_replace")
+    assert_equal "Hello World", obj[:object]
+    assert_equal "text/plain", obj[:headers]["content-type"]
+  end
+
+  def test_larger_lists
+    @s3.create_bucket('right_aws_many')
+    (0..50).each do |i|
+      ('a'..'z').each do |letter|
+        name = "#{letter}#{i}"
+        @s3.put('right_aws_many', name, 'asdf')
+      end
+    end
+
+    keys = @s3.list_bucket('right_aws_many')
+    assert_equal(1000, keys.size)
+    assert_equal('a0', keys.first[:key])
+  end
+
+  def test_destroy_bucket
+    @s3.create_bucket('deletebucket')
+    @s3.delete_bucket('deletebucket')
+
+    begin
+      bucket = @s3.list_bucket('deletebucket')
+      fail("Shouldn't succeed here")
+    rescue RightAws::AwsError
+      assert $!.message.include?('NoSuchBucket')
+    rescue
+      fail 'Should have caught NoSuchBucket Exception'
+    end
+  end
+
+  def test_if_none_match
+    @s3.put("s3media", "if_none_match_test", "Hello World 1!")
+    obj = @s3.get("s3media", "if_none_match_test")
+    tag = obj[:headers]["etag"]
+    begin
+      @s3.get("s3media", "if_none_match_test", {"If-None-Match" => tag})
+    rescue URI::InvalidURIError
+      # expected error for 304
+    else
+      fail 'Should have encountered an error due to the server not returning a response due to caching'
+    end
+    @s3.put("s3media", "if_none_match_test", "Hello World 2!")
+    obj = @s3.get("s3media", "if_none_match_test", {"If-None-Match" => tag})
+    assert_equal "Hello World 2!", obj[:object]
+  end
+
+  def test_if_modified_since
+    @s3.put("s3media", "if_modified_since_test", "Hello World 1!")
+    obj = @s3.get("s3media", "if_modified_since_test")
+    modified = obj[:headers]["last-modified"]
+    begin
+      @s3.get("s3media", "if_modified_since_test", {"If-Modified-Since" => modified})
+    rescue URI::InvalidURIError
+      # expected error for 304
+    else
+      fail 'Should have encountered an error due to the server not returning a response due to caching'
+    end
+    # Granularity of an HTTP Date is 1 second which isn't enough for the test
+    # so manually rewind the clock by a second
+    time_in_the_past = Time.httpdate(modified) - 1
+    begin
+      obj = @s3.get("s3media", "if_modified_since_test", {"If-Modified-Since" => time_in_the_past.httpdate})
+    rescue
+      fail 'Should have been downloaded since the date is in the past now'
+    else
+      #expected scenario
+    end
+  end
+
+end