rack_fake_s3 0.2.0

data/lib/rack_fake_s3/xml_adapter.rb

@@ -0,0 +1,179 @@
+require 'builder'
+require 'time'
+
+module RackFakeS3
+  class XmlAdapter
+    def self.buckets(bucket_objects)
+      output = ""
+      xml = Builder::XmlMarkup.new(:target => output)
+      xml.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+      xml.ListAllMyBucketsResult(:xmlns => "http://s3.amazonaws.com/doc/2006-03-01/") { |lam|
+        lam.Owner { |owner|
+          owner.ID("123")
+          owner.DisplayName("FakeS3")
+        }
+        lam.Buckets { |buckets|
+          bucket_objects.each do |bucket|
+            buckets.Bucket do |b|
+              b.Name(bucket.name)
+              b.CreationDate(bucket.creation_date.strftime("%Y-%m-%dT%H:%M:%S.000Z"))
+            end
+          end
+        }
+      }
+      output
+    end
+
+    def self.error(error)
+      output = ""
+      xml = Builder::XmlMarkup.new(:target => output)
+      xml.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+      xml.Error { |err|
+        err.Code(error.code)
+        err.Message(error.message)
+        err.Resource(error.resource)
+        err.RequestId(1)
+      }
+      output
+    end
+
+    # <?xml version="1.0" encoding="UTF-8"?>
+    #<Error>
+    #  <Code>NoSuchKey</Code>
+    #  <Message>The resource you requested does not exist</Message>
+    #  <Resource>/mybucket/myfoto.jpg</Resource>
+    #  <RequestId>4442587FB7D0A2F9</RequestId>
+    #</Error>
+    #
+    def self.error_no_such_bucket(name)
+      output = ""
+      xml = Builder::XmlMarkup.new(:target => output)
+      xml.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+      xml.Error { |err|
+        err.Code("NoSuchBucket")
+        err.Message("The resource you requested does not exist")
+        err.Resource(name)
+        err.RequestId(1)
+      }
+      output
+    end
+
+    def self.error_bucket_not_empty(name)
+      output = ""
+      xml = Builder::XmlMarkup.new(:target => output)
+      xml.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+      xml.Error { |err|
+        err.Code("BucketNotEmpty")
+        err.Message("The bucket you tried to delete is not empty.")
+        err.Resource(name)
+        err.RequestId(1)
+      }
+      output
+    end
+
+    def self.error_no_such_key(name)
+      output = ""
+      xml = Builder::XmlMarkup.new(:target => output)
+      xml.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+      xml.Error { |err|
+        err.Code("NoSuchKey")
+        err.Message("The specified key does not exist")
+        err.Key(name)
+        err.RequestId(1)
+        err.HostId(2)
+      }
+      output
+    end
+
+    def self.bucket(bucket)
+      output = ""
+      xml = Builder::XmlMarkup.new(:target => output)
+      xml.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+      xml.ListBucketResult(:xmlns => "http://s3.amazonaws.com/doc/2006-03-01/") { |lbr|
+        lbr.Name(bucket.name)
+        lbr.Prefix
+        lbr.Marker
+        lbr.MaxKeys("1000")
+        lbr.IsTruncated("false")
+      }
+      output
+    end
+
+    # A bucket query gives back the bucket along with contents
+    #  <Contents>
+    #<Key>Nelson</Key>
+#    <LastModified>2006-01-01T12:00:00.000Z</LastModified>
+#    <ETag>&quot;828ef3fdfa96f00ad9f27c383fc9ac7f&quot;</ETag>
+#    <Size>5</Size>
+#    <StorageClass>STANDARD</StorageClass>
+#    <Owner>
+#      <ID>bcaf161ca5fb16fd081034f</ID>
+#      <DisplayName>webfile</DisplayName>
+#     </Owner>
+#    </Contents>
+
+    def self.append_objects_to_list_bucket_result(lbr,objects)
+      return if objects.nil? or objects.size == 0
+
+      if objects.index(nil)
+        require 'ruby-debug'
+        Debugger.start
+        debugger
+      end
+
+      objects.each do |s3_object|
+        lbr.Contents { |contents|
+          contents.Key(s3_object.name)
+          contents.LastModifed(s3_object.creation_date)
+          contents.ETag("\"#{s3_object.md5}\"")
+          contents.Size(s3_object.size)
+          contents.StorageClass("STANDARD")
+
+          contents.Owner { |owner|
+            owner.ID("abc")
+            owner.DisplayName("You")
+          }
+        }
+      end
+    end
+
+    def self.bucket_query(bucket_query)
+      output = ""
+      bucket = bucket_query.bucket
+      xml = Builder::XmlMarkup.new(:target => output)
+      xml.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+      xml.ListBucketResult(:xmlns => "http://s3.amazonaws.com/doc/2006-03-01/") { |lbr|
+        lbr.Name(bucket.name)
+        lbr.Prefix(bucket_query.prefix)
+        lbr.Marker(bucket_query.marker)
+        lbr.MaxKeys(bucket_query.max_keys)
+        lbr.IsTruncated(bucket_query.is_truncated?)
+        append_objects_to_list_bucket_result(lbr,bucket_query.matches)
+      }
+      output
+    end
+
+    # ACL xml
+    def self.acl(object = nil)
+      output = ""
+      xml = Builder::XmlMarkup.new(:target => output)
+      xml.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+      xml.AccessControlPolicy(:xmlns => "http://s3.amazonaws.com/doc/2006-03-01/") { |acp|
+        acp.Owner do |owner|
+          owner.ID("abc")
+          owner.DisplayName("You")
+        end
+        acp.AccessControlList do |acl|
+          acl.Grant do |grant|
+            grant.Grantee("xmlns:xsi" => "http://www.w3.org/2001/XMLSchema-instance", "xsi:type" => "CanonicalUser") do |grantee|
+              grantee.ID("abc")
+              grantee.DisplayName("You")
+            end
+            grant.Permission("FULL_CONTROL")
+          end
+        end
+      }
+      output
+    end
+  end
+end

data/lib/rack_fake_s3.rb

@@ -0,0 +1,7 @@
+require 'rack_fake_s3/version'
+require 'rack_fake_s3/file_store'
+require 'rack_fake_s3/server'
+
+module RackFakeS3
+  class UnsupportedOperation < RuntimeError; end
+end

data/rack_fake_s3.gemspec

@@ -0,0 +1,28 @@
+# -*- encoding: utf-8 -*-
+require File.join(File.dirname(__FILE__), 'lib', 'rack_fake_s3', 'version')
+
+Gem::Specification.new do |s|
+  s.name        = "rack_fake_s3"
+  s.version     = RackFakeS3::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ['Mario Visic']
+  s.email       = ['mario.visic@envato.com']
+  s.homepage    = "https://github.com/envato/rack_fake_s3"
+  s.summary     = %q{Rack Fake S3 is a dummy rack app that simulates S3 for local integration testing. Originally created by Curtis Spencer, modified by Mario Visic at Envato.}
+  s.description = %q{Test S3 integration locally. Originally created by Curtis Spencer, modified by Mario Visic at Envato.}
+
+  s.add_development_dependency "bundler", ">= 1.0.0"
+  s.add_development_dependency "aws-s3"
+  s.add_development_dependency "right_aws"
+  s.add_development_dependency "rake"
+  s.add_development_dependency "rest-client"
+
+  s.add_dependency "thor"
+  s.add_dependency "builder"
+  s.add_dependency "rack"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

data/test/local_s3_cfg

@@ -0,0 +1,34 @@
+[default]
+access_key = abc
+acl_public = False
+bucket_location = US
+cloudfront_host = cloudfront.amazonaws.com
+cloudfront_resource = /2008-06-30/distribution
+default_mime_type = binary/octet-stream
+delete_removed = False
+dry_run = False
+encoding = UTF-8
+encrypt = False
+force = False
+get_continue = False
+gpg_command = None
+gpg_decrypt = %(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
+gpg_encrypt = %(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
+gpg_passphrase = 
+guess_mime_type = True
+host_base = localhost:10453
+host_bucket = %(bucket)s.localhost:10453
+human_readable_sizes = False
+list_md5 = False
+preserve_attrs = True
+progress_meter = True
+proxy_host = 
+proxy_port = 0
+recursive = False
+recv_chunk = 4096
+secret_key = def
+send_chunk = 4096
+simpledb_host = sdb.amazonaws.com
+skip_existing = False
+use_https = False
+verbosity = WARNING

data/test/right_aws_commands_test.rb

@@ -0,0 +1,65 @@
+require 'test/test_helper'
+require 'fileutils'
+#require 'fakes3/server'
+require 'right_aws'
+
+class RightAWSCommandsTest < Test::Unit::TestCase
+
+  def setup
+    @s3 = RightAws::S3Interface.new('1E3GDYEOGFJPIT7XXXXXX','hgTHt68JY07JKUY08ftHYtERkjgtfERn57XXXXXX',
+                                    {:multi_thread => false, :server => 'localhost',
+                                      :port => 10453, :protocol => 'http',:logger => Logger.new("/dev/null"),:no_subdomains => true })
+  end
+
+  def teardown
+  end
+
+  def test_create_bucket
+    bucket = @s3.create_bucket("s3media")
+    assert_not_nil bucket
+  end
+
+  def test_store
+    @s3.put("s3media","helloworld","Hello World Man!")
+    obj = @s3.get("s3media","helloworld")
+    assert_equal "Hello World Man!",obj[:object]
+
+    obj = @s3.get("s3media","helloworld")
+  end
+
+  def test_large_store
+    @s3.put("s3media","helloworld","Hello World Man!")
+    buffer = ""
+    500000.times do
+      buffer << "#{(rand * 100).to_i}"
+    end
+
+    buf_len = buffer.length
+    @s3.put("s3media","big",buffer)
+
+    output = ""
+    @s3.get("s3media","big") do |chunk|
+      output << chunk
+    end
+    assert_equal buf_len,output.size
+  end
+
+  def test_multi_directory
+    @s3.put("s3media","dir/right/123.txt","recursive")
+    output = ""
+    obj = @s3.get("s3media","dir/right/123.txt") do |chunk|
+      output << chunk
+    end
+    
+    assert_equal "recursive", output
+  end
+
+  def test_intra_bucket_copy
+    @s3.put("s3media","original.txt","Hello World")
+    @s3.copy("s3media","original.txt","s3media","copy.txt")
+    obj = @s3.get("s3media","copy.txt")
+
+    assert_equal "Hello World",obj[:object]
+  end
+
+end

data/test/s3_commands_test.rb

@@ -0,0 +1,166 @@
+require 'test/test_helper'
+require 'fileutils'
+#require 'fakes3/server'
+require 'aws/s3'
+
+class S3CommandsTest < Test::Unit::TestCase
+  include AWS::S3
+
+  def setup
+    AWS::S3::Base.establish_connection!(:access_key_id => "123",
+                                        :secret_access_key => "abc",
+                                        :server => "localhost",
+                                        :port => "10453" )
+  end
+
+  def teardown
+    AWS::S3::Base.disconnect!
+  end
+
+  def test_create_bucket
+    bucket = Bucket.create("ruby_aws_s3")
+    assert_not_nil bucket
+
+    bucket_names = []
+    Service.buckets.each do |bucket|
+      bucket_names << bucket.name
+    end
+    assert(bucket_names.index("ruby_aws_s3") >= 0)
+  end
+
+  def test_destroy_bucket
+    Bucket.create("deletebucket")
+    Bucket.delete("deletebucket")
+
+    begin
+      bucket = Bucket.find("deletebucket")
+      assert_fail("Shouldn't succeed here")
+    rescue
+    end
+  end
+
+  def test_store
+    bucket = Bucket.create("ruby_aws_s3")
+    S3Object.store("hello","world","ruby_aws_s3")
+
+    output = ""
+    obj = S3Object.stream("hello","ruby_aws_s3") do |chunk|
+      output << chunk
+    end
+    assert_equal "world", output
+  end
+
+  def test_large_store
+    bucket = Bucket.create("ruby_aws_s3")
+    buffer = ""
+    500000.times do
+      buffer << "#{(rand * 100).to_i}"
+    end
+
+    buf_len = buffer.length
+    S3Object.store("big",buffer,"ruby_aws_s3")
+
+    output = ""
+    S3Object.stream("big","ruby_aws_s3") do |chunk|
+      output << chunk
+    end
+    assert_equal buf_len,output.size
+  end
+
+  def test_multi_directory
+    bucket = Bucket.create("ruby_aws_s3")
+    S3Object.store("dir/myfile/123.txt","recursive","ruby_aws_s3")
+
+    output = ""
+    obj = S3Object.stream("dir/myfile/123.txt","ruby_aws_s3") do |chunk|
+      output << chunk
+    end
+    assert_equal "recursive", output
+  end
+
+  def test_find_nil_bucket
+    begin
+      bucket = Bucket.find("unknown")
+      assert_fail "Bucket.find didn't throw an exception"
+    rescue
+      assert_equal AWS::S3::NoSuchBucket,$!.class
+    end
+  end
+
+  def test_find_object
+    bucket = Bucket.create('find_bucket')
+    obj_name = 'short'
+    S3Object.store(obj_name,'short_text','find_bucket')
+    short = S3Object.find(obj_name,"find_bucket")
+    assert_not_nil(short)
+    assert_equal(short.value,'short_text')
+  end
+
+  def test_find_non_existent_object
+    bucket = Bucket.create('find_bucket')
+    obj_name = 'doesnotexist'
+    assert_raise AWS::S3::NoSuchKey do
+      should_throw = S3Object.find(obj_name,"find_bucket")
+    end
+
+    # Try something higher in the alphabet
+    assert_raise AWS::S3::NoSuchKey do
+      should_throw = S3Object.find("zzz","find_bucket")
+    end
+  end
+
+  def test_exists?
+    bucket = Bucket.create('ruby_aws_s3')
+    obj_name = 'dir/myfile/exists.txt'
+    S3Object.store(obj_name,'exists','ruby_aws_s3')
+    assert S3Object.exists?(obj_name, 'ruby_aws_s3')
+    assert !S3Object.exists?('dir/myfile/doesnotexist.txt','ruby_aws_s3')
+  end
+
+  def test_delete
+    bucket = Bucket.create("ruby_aws_s3")
+    S3Object.store("something_to_delete","asdf","ruby_aws_s3")
+    something = S3Object.find("something_to_delete","ruby_aws_s3")
+    S3Object.delete("something_to_delete","ruby_aws_s3")
+
+    assert_raise AWS::S3::NoSuchKey do
+      should_throw = S3Object.find("something_to_delete","ruby_aws_s3")
+    end
+  end
+
+  def test_rename
+    bucket = Bucket.create("ruby_aws_s3")
+    S3Object.store("something_to_rename","asdf","ruby_aws_s3")
+    S3Object.rename("something_to_rename","renamed","ruby_aws_s3")
+
+    renamed = S3Object.find("renamed","ruby_aws_s3")
+    assert_not_nil(renamed)
+    assert_equal(renamed.value,'asdf')
+
+    assert_raise AWS::S3::NoSuchKey do
+      should_throw = S3Object.find("something_to_rename","ruby_aws_s3")
+    end
+  end
+
+  def test_larger_lists
+    Bucket.create("ruby_aws_s3_many")
+    (0..50).each do |i|
+      ('a'..'z').each do |letter|
+        name = "#{letter}#{i}"
+        S3Object.store(name,"asdf","ruby_aws_s3_many")
+      end
+    end
+
+    bucket = Bucket.find("ruby_aws_s3_many")
+    assert_equal(bucket.size,1000)
+    assert_equal(bucket.objects.first.key,"a0")
+  end
+
+
+  # Copying an object
+  #S3Object.copy 'headshot.jpg', 'headshot2.jpg', 'photos'
+
+  # Renaming an object
+  #S3Object.rename 'headshot.jpg', 'portrait.jpg', 'photos'
+
+end

data/test/s3cmd_test.rb

@@ -0,0 +1,52 @@
+require 'test/test_helper'
+require 'fileutils'
+
+class S3CmdTest < Test::Unit::TestCase
+  def setup
+    config = File.expand_path(File.join(File.dirname(__FILE__),'local_s3_cfg'))
+    raise "Please install s3cmd" if `which s3cmd`.empty?
+    @s3cmd = "s3cmd --config #{config}"
+  end
+
+  def teardown
+  end
+
+  def test_create_bucket
+    `#{@s3cmd} mb s3://s3cmd_bucket`
+    output = `#{@s3cmd} ls`
+    assert_match(/s3cmd_bucket/,output)
+  end
+
+  def test_store
+    File.open(__FILE__,'rb') do |input|
+      File.open("/tmp/fakes3_upload",'wb') do |output|
+        output << input.read
+      end
+    end
+    output = `#{@s3cmd} put /tmp/fakes3_upload s3://s3cmd_bucket/upload`
+    assert_match(/stored/,output)
+
+    FileUtils.rm("/tmp/fakes3_upload")
+  end
+
+  def test_acl
+    File.open(__FILE__,'rb') do |input|
+      File.open("/tmp/fakes3_acl_upload",'wb') do |output|
+        output << input.read
+      end
+    end
+    output = `#{@s3cmd} put /tmp/fakes3_acl_upload s3://s3cmd_bucket/acl_upload`
+    assert_match(/stored/,output)
+
+    output = `#{@s3cmd} --force setacl -P s3://s3cmd_bucket/acl_upload`
+  end
+
+  def test_large_store
+  end
+
+  def test_multi_directory
+  end
+
+  def test_intra_bucket_copy
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,4 @@
+require 'test/unit'
+require 'rubygems'
+require 'bundler/setup'
+require 'rack_fake_s3'