fakes3-docker 0.2.4

data/test/botocmd.py

@@ -0,0 +1,87 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# fakes3cmd.py -- an s3cmd-like script that accepts a custom host and portname
+import re
+import os
+from optparse import OptionParser
+
+try:
+    from boto.s3.connection import S3Connection, OrdinaryCallingFormat
+    from boto.s3.key import Key
+except ImportError:
+    raise Exception('You must install the boto package for python')
+
+
+class FakeS3Cmd(object):
+    COMMANDS = ['mb', 'rb', 'put', ]
+    def __init__(self, host, port):
+        self.host = host
+        self.port = port
+        self.conn = None
+        self._connect()
+
+    def _connect(self):
+        print 'Connecting: %s:%s' % (self.host, self.port)
+        self.conn = S3Connection(is_secure=False,
+                             calling_format=OrdinaryCallingFormat(),
+                             aws_access_key_id='',
+                             aws_secret_access_key='',
+                             port=self.port, host=self.host)
+
+
+    @staticmethod
+    def _parse_uri(path):
+        match = re.match(r's3://([^/]+)(?:/(.*))?', path, re.I)
+        ## (bucket, key)
+        return match.groups()
+
+    def mb(self, path, *args):
+        if not self.conn:
+            self._connect()
+
+        bucket, _ = self._parse_uri(path)
+        self.conn.create_bucket(bucket)
+        print 'made bucket: [%s]' % bucket
+
+    def rb(self, path, *args):
+        if not self.conn:
+            self._connect()
+
+        bucket, _ = self._parse_uri(path)
+        self.conn.delete_bucket(bucket)
+        print 'removed bucket: [%s]' % bucket
+
+    def put(self, *args):
+        if not self.conn:
+            self._connect()
+
+        args = list(args)
+        path = args.pop()
+        bucket_name, prefix = self._parse_uri(path)
+        bucket = self.conn.create_bucket(bucket_name)
+        for src_file in args:
+            key = Key(bucket)
+            key.key = os.path.join(prefix, os.path.basename(src_file))
+            key.set_contents_from_filename(src_file)
+            print 'stored: [%s]' % key.key
+
+
+if __name__ == "__main__":
+    # check for options. TODO: This requires a more verbose help message
+    # to explain how the positional arguments work.
+    parser = OptionParser()
+    parser.add_option("-t", "--host", type="string", default='localhost')
+    parser.add_option("-p", "--port", type='int', default=80)
+    o, args = parser.parse_args()
+
+    if len(args) < 2:
+        raise ValueError('you must minimally supply a desired command and s3 uri')
+
+    cmd = args.pop(0)
+
+    if cmd not in FakeS3Cmd.COMMANDS:
+        raise ValueError('%s is not a valid command' % cmd)
+
+    fs3 = FakeS3Cmd(o.host, o.port)
+    handler = getattr(fs3, cmd)
+    handler(*args)

data/test/local_s3_cfg

@@ -0,0 +1,34 @@
+[default]
+access_key = abc
+acl_public = False
+bucket_location = US
+cloudfront_host = cloudfront.amazonaws.com
+cloudfront_resource = /2008-06-30/distribution
+default_mime_type = binary/octet-stream
+delete_removed = False
+dry_run = False
+encoding = UTF-8
+encrypt = False
+force = False
+get_continue = False
+gpg_command = None
+gpg_decrypt = %(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
+gpg_encrypt = %(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
+gpg_passphrase = 
+guess_mime_type = True
+host_base = localhost:10453
+host_bucket = %(bucket)s.localhost:10453
+human_readable_sizes = False
+list_md5 = False
+preserve_attrs = True
+progress_meter = True
+proxy_host = 
+proxy_port = 0
+recursive = False
+recv_chunk = 4096
+secret_key = def
+send_chunk = 4096
+simpledb_host = sdb.amazonaws.com
+skip_existing = False
+use_https = False
+verbosity = WARNING

data/test/post_test.rb

@@ -0,0 +1,54 @@
+require 'test/test_helper'
+require 'rest-client'
+
+class PostTest < Test::Unit::TestCase
+  # Make sure you have a posttest.localhost in your /etc/hosts/
+  def setup
+    @url='http://posttest.localhost:10453/'
+  end
+
+  def teardown
+  end
+
+  def test_options
+    res= RestClient.options(@url) { |response|
+      assert_equal(response.headers[:access_control_allow_origin],"*")
+    }
+  end
+
+  def test_redirect
+    res = RestClient.post(
+      @url,
+      'key'=>'uploads/12345/${filename}',
+      'success_action_redirect'=>'http://somewhere.else.com/',
+      'file'=>File.new(__FILE__,"rb")
+    ) { |response|
+      assert_equal(response.code, 307)
+      assert_equal(response.headers[:location], 'http://somewhere.else.com/')
+    }
+  end
+
+  def test_status_200
+    res = RestClient.post(
+      @url,
+      'key'=>'uploads/12345/${filename}',
+      'success_action_status'=>'200',
+      'file'=>File.new(__FILE__,"rb")
+    ) { |response|
+      assert_equal(response.code, 200)
+    }
+  end
+
+  def test_status_201
+    res = RestClient.post(
+      @url,
+      'key'=>'uploads/12345/${filename}',
+      'success_action_status'=>'201',
+      'file'=>File.new(__FILE__,"rb")
+    ) { |response|
+      assert_equal(response.code, 201)
+      assert_match(%r{^\<\?xml.*uploads/12345/post_test\.rb}m, response.body)
+    }
+  end
+
+end

data/test/right_aws_commands_test.rb

@@ -0,0 +1,192 @@
+require 'test/test_helper'
+require 'fileutils'
+#require 'fakes3/server'
+require 'right_aws'
+require 'time'
+
+class RightAWSCommandsTest < Test::Unit::TestCase
+
+  def setup
+    @s3 = RightAws::S3Interface.new('1E3GDYEOGFJPIT7XXXXXX','hgTHt68JY07JKUY08ftHYtERkjgtfERn57XXXXXX',
+                                    {:multi_thread => false, :server => 'localhost',
+                                      :port => 10453, :protocol => 'http',:logger => Logger.new("/dev/null"),:no_subdomains => true })
+  end
+
+  def teardown
+  end
+
+  def test_create_bucket
+    bucket = @s3.create_bucket("s3media")
+    assert_not_nil bucket
+  end
+
+  def test_store
+    @s3.put("s3media","helloworld","Hello World Man!")
+    obj = @s3.get("s3media","helloworld")
+    assert_equal "Hello World Man!",obj[:object]
+
+    obj = @s3.get("s3media","helloworld")
+  end
+
+  def test_store_not_found
+    begin
+      obj = @s3.get("s3media","helloworldnotexist")
+    rescue RightAws::AwsError
+      assert $!.message.include?('NoSuchKey')
+    rescue
+      fail 'Should have caught NoSuchKey Exception'
+    end
+  end
+
+  def test_large_store
+    @s3.put("s3media","helloworld","Hello World Man!")
+    buffer = ""
+    500000.times do
+      buffer << "#{(rand * 100).to_i}"
+    end
+
+    buf_len = buffer.length
+    @s3.put("s3media","big",buffer)
+
+    output = ""
+    @s3.get("s3media","big") do |chunk|
+      output << chunk
+    end
+    assert_equal buf_len,output.size
+  end
+
+  # Test that GET requests with a delimiter return a list of
+  def test_list_by_delimiter
+    @s3.create_bucket("s3media")
+
+    @s3.put("s3media", "delimited/item", "item")
+
+    expected_prefixes = []
+    (1..50).each do |i|
+      key_prefix = "delimited/%02d/" % i
+      @s3.put("s3media", key_prefix + "foo", "foo")
+      @s3.put("s3media", key_prefix + "fie", "fie")
+      expected_prefixes << key_prefix
+    end
+
+    key_names = []
+    common_prefixes = []
+    @s3.incrementally_list_bucket("s3media", {:prefix => "delimited", :delimiter => '/'}) do |currentResponse|
+      common_prefixes += currentResponse[:common_prefixes]
+    end
+    assert_equal ["delimited/"], common_prefixes
+
+    common_prefixes = []
+    @s3.incrementally_list_bucket("s3media", {:prefix => "delimited/", :delimiter => '/', "max-keys" => 5}) do |currentResponse|
+      key_names += currentResponse[:contents].map do |key|
+        key[:key]
+      end
+      common_prefixes += currentResponse[:common_prefixes]
+    end
+    assert_equal expected_prefixes, common_prefixes
+    assert_equal ["delimited/item"], key_names
+  end
+
+  def test_multi_directory
+    @s3.put("s3media","dir/right/123.txt","recursive")
+    output = ""
+    obj = @s3.get("s3media","dir/right/123.txt") do |chunk|
+      output << chunk
+    end
+    assert_equal "recursive", output
+  end
+
+  def test_intra_bucket_copy
+    @s3.put("s3media","original.txt","Hello World")
+    @s3.copy("s3media","original.txt","s3media","copy.txt")
+    obj = @s3.get("s3media","copy.txt")
+    assert_equal "Hello World",obj[:object]
+  end
+
+  def test_copy_in_place
+    @s3.put("s3media","foo","Hello World")
+    @s3.copy("s3media","foo","s3media","foo")
+    obj = @s3.get("s3media","foo")
+    assert_equal "Hello World",obj[:object]
+  end
+
+  def test_copy_replace_metadata
+    @s3.put("s3media","foo","Hello World",{"content-type"=>"application/octet-stream"})
+    obj = @s3.get("s3media","foo")
+    assert_equal "Hello World",obj[:object]
+    assert_equal "application/octet-stream",obj[:headers]["content-type"]
+    @s3.copy("s3media","foo","s3media","foo",:replace,{"content-type"=>"text/plain"})
+    obj = @s3.get("s3media","foo")
+    assert_equal "Hello World",obj[:object]
+    assert_equal "text/plain",obj[:headers]["content-type"]
+  end
+
+  def test_larger_lists
+    @s3.create_bucket('right_aws_many')
+    (0..50).each do |i|
+      ('a'..'z').each do |letter|
+        name = "#{letter}#{i}"
+        @s3.put('right_aws_many', name, 'asdf')
+      end
+    end
+
+    keys = @s3.list_bucket('right_aws_many')
+    assert_equal(1000, keys.size)
+    assert_equal('a0', keys.first[:key])
+  end
+
+  def test_destroy_bucket
+    @s3.create_bucket('deletebucket')
+    @s3.delete_bucket('deletebucket')
+
+    begin
+      bucket = @s3.list_bucket('deletebucket')
+      fail("Shouldn't succeed here")
+    rescue RightAws::AwsError
+      assert $!.message.include?('NoSuchBucket')
+    rescue
+      fail 'Should have caught NoSuchBucket Exception'
+    end
+
+  end
+
+  def test_if_none_match
+    @s3.put("s3media","if_none_match_test","Hello World 1!")
+    obj = @s3.get("s3media","if_none_match_test")
+    tag = obj[:headers]["etag"]
+    begin
+      @s3.get("s3media", "if_none_match_test", {"If-None-Match"=>tag})
+    rescue URI::InvalidURIError
+      # expected error for 304
+    else
+      fail 'Should have encountered an error due to the server not returning a response due to caching'
+    end
+    @s3.put("s3media","if_none_match_test","Hello World 2!")
+    obj = @s3.get("s3media", "if_none_match_test", {"If-None-Match"=>tag})
+    assert_equal "Hello World 2!",obj[:object]
+  end
+
+  def test_if_modified_since
+    @s3.put("s3media","if_modified_since_test","Hello World 1!")
+    obj = @s3.get("s3media","if_modified_since_test")
+    modified = obj[:headers]["last-modified"]
+    begin
+      @s3.get("s3media", "if_modified_since_test", {"If-Modified-Since"=>modified})
+    rescue URI::InvalidURIError
+      # expected error for 304
+    else
+      fail 'Should have encountered an error due to the server not returning a response due to caching'
+    end
+    # Granularity of an HTTP Date is 1 second which isn't enough for the test
+    # so manually rewind the clock by a second
+    timeInThePast = Time.httpdate(modified) - 1
+    begin
+      obj = @s3.get("s3media", "if_modified_since_test", {"If-Modified-Since"=>timeInThePast.httpdate()})
+    rescue
+      fail 'Should have been downloaded since the date is in the past now'
+    else
+      #expected scenario
+    end
+  end
+
+end

data/test/s3_commands_test.rb

@@ -0,0 +1,209 @@
+require 'test/test_helper'
+require 'fileutils'
+#require 'fakes3/server'
+require 'aws/s3'
+
+class S3CommandsTest < Test::Unit::TestCase
+  include AWS::S3
+
+  def setup
+    AWS::S3::Base.establish_connection!(:access_key_id => "123",
+                                        :secret_access_key => "abc",
+                                        :server => "localhost",
+                                        :port => "10453" )
+  end
+
+  def teardown
+    AWS::S3::Base.disconnect!
+  end
+
+  def test_create_bucket
+    bucket = Bucket.create("ruby_aws_s3")
+    assert_not_nil bucket
+
+    bucket_names = []
+    Service.buckets.each do |bucket|
+      bucket_names << bucket.name
+    end
+    assert(bucket_names.index("ruby_aws_s3") >= 0)
+  end
+
+  def test_destroy_bucket
+    Bucket.create("deletebucket")
+    Bucket.delete("deletebucket")
+
+    begin
+      bucket = Bucket.find("deletebucket")
+      assert_fail("Shouldn't succeed here")
+    rescue
+    end
+  end
+
+  def test_store
+    bucket = Bucket.create("ruby_aws_s3")
+    S3Object.store("hello","world","ruby_aws_s3")
+
+    output = ""
+    obj = S3Object.stream("hello","ruby_aws_s3") do |chunk|
+      output << chunk
+    end
+    assert_equal "world", output
+  end
+
+  def test_large_store
+    bucket = Bucket.create("ruby_aws_s3")
+    buffer = ""
+    500000.times do
+      buffer << "#{(rand * 100).to_i}"
+    end
+
+    buf_len = buffer.length
+    S3Object.store("big",buffer,"ruby_aws_s3")
+
+    output = ""
+    S3Object.stream("big","ruby_aws_s3") do |chunk|
+      output << chunk
+    end
+    assert_equal buf_len,output.size
+  end
+
+  def test_metadata_store
+    assert_equal true, Bucket.create("ruby_aws_s3")
+    bucket = Bucket.find("ruby_aws_s3")
+
+    # Note well: we can't seem to access obj.metadata until we've stored
+    # the object and found it again. Thus the store, find, store
+    # runaround below.
+    obj = bucket.new_object(:value => "foo")
+    obj.key = "key_with_metadata"
+    obj.store
+    obj = S3Object.find("key_with_metadata", "ruby_aws_s3")
+    obj.metadata[:param1] = "one"
+    obj.metadata[:param2] = "two, three"
+    obj.store
+    obj = S3Object.find("key_with_metadata", "ruby_aws_s3")
+
+    assert_equal "one", obj.metadata[:param1]
+    assert_equal "two, three", obj.metadata[:param2]
+  end
+
+  def test_metadata_copy
+    assert_equal true, Bucket.create("ruby_aws_s3")
+    bucket = Bucket.find("ruby_aws_s3")
+
+    # Note well: we can't seem to access obj.metadata until we've stored
+    # the object and found it again. Thus the store, find, store
+    # runaround below.
+    obj = bucket.new_object(:value => "foo")
+    obj.key = "key_with_metadata"
+    obj.store
+    obj = S3Object.find("key_with_metadata", "ruby_aws_s3")
+    obj.metadata[:param1] = "one"
+    obj.metadata[:param2] = "two, three"
+    obj.store
+
+    S3Object.copy("key_with_metadata", "key_with_metadata2", "ruby_aws_s3")
+    obj = S3Object.find("key_with_metadata2", "ruby_aws_s3")
+
+    assert_equal "one", obj.metadata[:param1]
+    assert_equal "two, three", obj.metadata[:param2]
+  end
+
+
+  def test_multi_directory
+    bucket = Bucket.create("ruby_aws_s3")
+    S3Object.store("dir/myfile/123.txt","recursive","ruby_aws_s3")
+
+    output = ""
+    obj = S3Object.stream("dir/myfile/123.txt","ruby_aws_s3") do |chunk|
+      output << chunk
+    end
+    assert_equal "recursive", output
+  end
+
+  def test_find_nil_bucket
+    begin
+      bucket = Bucket.find("unknown")
+      assert_fail "Bucket.find didn't throw an exception"
+    rescue
+      assert_equal AWS::S3::NoSuchBucket,$!.class
+    end
+  end
+
+  def test_find_object
+    bucket = Bucket.create('find_bucket')
+    obj_name = 'short'
+    S3Object.store(obj_name,'short_text','find_bucket')
+    short = S3Object.find(obj_name,"find_bucket")
+    assert_not_nil(short)
+    assert_equal(short.value,'short_text')
+  end
+
+  def test_find_non_existent_object
+    bucket = Bucket.create('find_bucket')
+    obj_name = 'doesnotexist'
+    assert_raise AWS::S3::NoSuchKey do
+      should_throw = S3Object.find(obj_name,"find_bucket")
+    end
+
+    # Try something higher in the alphabet
+    assert_raise AWS::S3::NoSuchKey do
+      should_throw = S3Object.find("zzz","find_bucket")
+    end
+  end
+
+  def test_exists?
+    bucket = Bucket.create('ruby_aws_s3')
+    obj_name = 'dir/myfile/exists.txt'
+    S3Object.store(obj_name,'exists','ruby_aws_s3')
+    assert S3Object.exists?(obj_name, 'ruby_aws_s3')
+    assert !S3Object.exists?('dir/myfile/doesnotexist.txt','ruby_aws_s3')
+  end
+
+  def test_delete
+    bucket = Bucket.create("ruby_aws_s3")
+    S3Object.store("something_to_delete","asdf","ruby_aws_s3")
+    something = S3Object.find("something_to_delete","ruby_aws_s3")
+    S3Object.delete("something_to_delete","ruby_aws_s3")
+
+    assert_raise AWS::S3::NoSuchKey do
+      should_throw = S3Object.find("something_to_delete","ruby_aws_s3")
+    end
+  end
+
+  def test_rename
+    bucket = Bucket.create("ruby_aws_s3")
+    S3Object.store("something_to_rename","asdf","ruby_aws_s3")
+    S3Object.rename("something_to_rename","renamed","ruby_aws_s3")
+
+    renamed = S3Object.find("renamed","ruby_aws_s3")
+    assert_not_nil(renamed)
+    assert_equal(renamed.value,'asdf')
+
+    assert_raise AWS::S3::NoSuchKey do
+      should_throw = S3Object.find("something_to_rename","ruby_aws_s3")
+    end
+  end
+
+  def test_larger_lists
+    Bucket.create("ruby_aws_s3_many")
+    (0..50).each do |i|
+      ('a'..'z').each do |letter|
+        name = "#{letter}#{i}"
+        S3Object.store(name,"asdf","ruby_aws_s3_many")
+      end
+    end
+
+    bucket = Bucket.find("ruby_aws_s3_many")
+    assert_equal(bucket.size,1000)
+    assert_equal(bucket.objects.first.key,"a0")
+  end
+
+
+  # Copying an object
+  #S3Object.copy 'headshot.jpg', 'headshot2.jpg', 'photos'
+
+  # Renaming an object
+  #S3Object.rename 'headshot.jpg', 'portrait.jpg', 'photos'
+
+end