fakes3test10 1.2.0

data/lib/fakes3/sorted_object_list.rb

@@ -0,0 +1,137 @@
+require 'set'
+module FakeS3
+  class S3MatchSet
+    attr_accessor :matches,:is_truncated,:common_prefixes
+    def initialize
+      @matches = []
+      @is_truncated = false
+      @common_prefixes = []
+    end
+  end
+
+  # This class has some of the semantics necessary for how buckets can return
+  # their items
+  #
+  # It is currently implemented naively as a sorted set + hash If you are going
+  # to try to put massive lists inside buckets and ls them, you will be sorely
+  # disappointed about this performance.
+  class SortedObjectList
+
+    def initialize
+      @sorted_set = SortedSet.new
+      @object_map = {}
+      @mutex = Mutex.new
+    end
+
+    def count
+      @sorted_set.count
+    end
+
+    def find(object_name)
+      @object_map[object_name]
+    end
+
+    # Add an S3 object into the sorted list
+    def add(s3_object)
+      return if !s3_object
+
+      @object_map[s3_object.name] = s3_object
+      @sorted_set << s3_object
+    end
+
+    def remove(s3_object)
+      return if !s3_object
+
+      @object_map.delete(s3_object.name)
+      @sorted_set.delete(s3_object)
+    end
+
+    # Return back a set of matches based on the passed in options
+    #
+    # options:
+    #
+    # :marker : a string to start the lexographical search (it is not included
+    #           in the result)
+    # :max_keys  : a maximum number of results
+    # :prefix    : a string to filter the results by
+    # :delimiter : not supported yet
+    def list(options)
+      marker = options[:marker]
+      prefix = options[:prefix]
+      max_keys = options[:max_keys] || 1000
+      delimiter = options[:delimiter]
+
+      ms = S3MatchSet.new
+
+      marker_found = true
+      pseudo = nil
+      if marker
+        marker_found = false
+        if !@object_map[marker]
+          pseudo = S3Object.new
+          pseudo.name = marker
+          @sorted_set << pseudo
+        end
+      end
+
+      if delimiter
+        if prefix
+          base_prefix = prefix
+        else
+          base_prefix = ""
+        end
+        prefix_offset = base_prefix.length
+      end
+
+      count = 0
+      last_chunk = nil
+      @sorted_set.each do |s3_object|
+        if marker_found && (!prefix or s3_object.name.index(prefix) == 0)
+          if delimiter
+            name = s3_object.name
+            remainder = name.slice(prefix_offset, name.length)
+            chunks = remainder.split(delimiter, 2)
+            if chunks.length > 1
+              if (last_chunk != chunks[0])
+                # "All of the keys rolled up in a common prefix count as
+                # a single return when calculating the number of
+                # returns. See MaxKeys."
+                # (http://awsdocs.s3.amazonaws.com/S3/latest/s3-api.pdf)
+                count += 1
+                if count <= max_keys
+                  ms.common_prefixes << base_prefix + chunks[0] + delimiter
+                  last_chunk = chunks[0]
+                else
+                  is_truncated = true
+                  break
+                end
+              end
+
+              # Continue to the next key, since this one has a
+              # delimiter.
+              next
+            end
+          end
+
+          count += 1
+          if count <= max_keys
+            ms.matches << s3_object
+          else
+            is_truncated = true
+            break
+          end
+        end
+
+        if marker and marker == s3_object.name
+          marker_found = true
+        end
+      end
+
+      if pseudo
+        @sorted_set.delete(pseudo)
+      end
+
+      return ms
+    end
+  end
+end

data/lib/fakes3/unsupported_operation.rb

@@ -0,0 +1,4 @@
+module FakeS3
+  class UnsupportedOperation < RuntimeError
+  end
+end

data/lib/fakes3/util.rb

@@ -0,0 +1,8 @@
+module FakeS3
+  module Util
+    def Util.strip_before_and_after(string, strip_this)
+      regex_friendly_strip_this = Regexp.escape(strip_this)
+      string.gsub(/\A[#{regex_friendly_strip_this}]+|[#{regex_friendly_strip_this}]+\z/, '')
+    end
+  end
+end

data/lib/fakes3/version.rb

@@ -0,0 +1,3 @@
+module FakeS3
+  VERSION = "1.2.0"
+end

data/lib/fakes3/xml_adapter.rb

@@ -0,0 +1,222 @@
+require 'builder'
+require 'time'
+
+module FakeS3
+  class XmlAdapter
+    def self.buckets(bucket_objects)
+      output = ""
+      xml = Builder::XmlMarkup.new(:target => output)
+      xml.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+      xml.ListAllMyBucketsResult(:xmlns => "http://s3.amazonaws.com/doc/2006-03-01/") { |lam|
+        lam.Owner { |owner|
+          owner.ID("123")
+          owner.DisplayName("FakeS3")
+        }
+        lam.Buckets { |buckets|
+          bucket_objects.each do |bucket|
+            buckets.Bucket do |b|
+              b.Name(bucket.name)
+              b.CreationDate(bucket.creation_date.strftime("%Y-%m-%dT%H:%M:%S.000Z"))
+            end
+          end
+        }
+      }
+      output
+    end
+
+    def self.error(error)
+      output = ""
+      xml = Builder::XmlMarkup.new(:target => output)
+      xml.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+      xml.Error { |err|
+        err.Code(error.code)
+        err.Message(error.message)
+        err.Resource(error.resource)
+        err.RequestId(1)
+      }
+      output
+    end
+
+    # <?xml version="1.0" encoding="UTF-8"?>
+    #<Error>
+    #  <Code>NoSuchKey</Code>
+    #  <Message>The resource you requested does not exist</Message>
+    #  <Resource>/mybucket/myfoto.jpg</Resource>
+    #  <RequestId>4442587FB7D0A2F9</RequestId>
+    #</Error>
+    #
+    def self.error_no_such_bucket(name)
+      output = ""
+      xml = Builder::XmlMarkup.new(:target => output)
+      xml.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+      xml.Error { |err|
+        err.Code("NoSuchBucket")
+        err.Message("The resource you requested does not exist")
+        err.Resource(name)
+        err.RequestId(1)
+      }
+      output
+    end
+
+    def self.error_bucket_not_empty(name)
+      output = ""
+      xml = Builder::XmlMarkup.new(:target => output)
+      xml.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+      xml.Error { |err|
+        err.Code("BucketNotEmpty")
+        err.Message("The bucket you tried to delete is not empty.")
+        err.Resource(name)
+        err.RequestId(1)
+      }
+      output
+    end
+
+    def self.error_no_such_key(name)
+      output = ""
+      xml = Builder::XmlMarkup.new(:target => output)
+      xml.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+      xml.Error { |err|
+        err.Code("NoSuchKey")
+        err.Message("The specified key does not exist")
+        err.Key(name)
+        err.RequestId(1)
+        err.HostId(2)
+      }
+      output
+    end
+
+    def self.bucket(bucket)
+      output = ""
+      xml = Builder::XmlMarkup.new(:target => output)
+      xml.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+      xml.ListBucketResult(:xmlns => "http://s3.amazonaws.com/doc/2006-03-01/") { |lbr|
+        lbr.Name(bucket.name)
+        lbr.Prefix
+        lbr.Marker
+        lbr.MaxKeys("1000")
+        lbr.IsTruncated("false")
+      }
+      output
+    end
+
+    # A bucket query gives back the bucket along with contents
+    #  <Contents>
+    #<Key>Nelson</Key>
+#    <LastModified>2006-01-01T12:00:00.000Z</LastModified>
+#    <ETag>&quot;828ef3fdfa96f00ad9f27c383fc9ac7f&quot;</ETag>
+#    <Size>5</Size>
+#    <StorageClass>STANDARD</StorageClass>
+#    <Owner>
+#      <ID>bcaf161ca5fb16fd081034f</ID>
+#      <DisplayName>webfile</DisplayName>
+#     </Owner>
+#    </Contents>
+
+    def self.append_objects_to_list_bucket_result(lbr,objects)
+      return if objects.nil? or objects.size == 0
+
+      if objects.index(nil)
+        require 'ruby-debug'
+        Debugger.start
+        debugger
+      end
+
+      objects.each do |s3_object|
+        lbr.Contents { |contents|
+          contents.Key(s3_object.name)
+          contents.LastModified(s3_object.modified_date)
+          contents.ETag("\"#{s3_object.md5}\"")
+          contents.Size(s3_object.size)
+          contents.StorageClass("STANDARD")
+
+          contents.Owner { |owner|
+            owner.ID("abc")
+            owner.DisplayName("You")
+          }
+        }
+      end
+    end
+
+    def self.append_common_prefixes_to_list_bucket_result(lbr, prefixes)
+      return if prefixes.nil? or prefixes.size == 0
+
+      prefixes.each do |common_prefix|
+        lbr.CommonPrefixes { |contents| contents.Prefix(common_prefix) }
+      end
+    end
+
+    def self.bucket_query(bucket_query)
+      output = ""
+      bucket = bucket_query.bucket
+      xml = Builder::XmlMarkup.new(:target => output)
+      xml.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+      xml.ListBucketResult(:xmlns => "http://s3.amazonaws.com/doc/2006-03-01/") { |lbr|
+        lbr.Name(bucket.name)
+        lbr.Prefix(bucket_query.prefix)
+        lbr.Marker(bucket_query.marker)
+        lbr.MaxKeys(bucket_query.max_keys)
+        lbr.IsTruncated(bucket_query.is_truncated?)
+        append_objects_to_list_bucket_result(lbr,bucket_query.matches)
+        append_common_prefixes_to_list_bucket_result(lbr, bucket_query.common_prefixes)
+      }
+      output
+    end
+
+    # ACL xml
+    def self.acl(object = nil)
+      output = ""
+      xml = Builder::XmlMarkup.new(:target => output)
+      xml.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+      xml.AccessControlPolicy(:xmlns => "http://s3.amazonaws.com/doc/2006-03-01/") { |acp|
+        acp.Owner do |owner|
+          owner.ID("abc")
+          owner.DisplayName("You")
+        end
+        acp.AccessControlList do |acl|
+          acl.Grant do |grant|
+            grant.Grantee("xmlns:xsi" => "http://www.w3.org/2001/XMLSchema-instance", "xsi:type" => "CanonicalUser") do |grantee|
+              grantee.ID("abc")
+              grantee.DisplayName("You")
+            end
+            grant.Permission("FULL_CONTROL")
+          end
+        end
+      }
+      output
+    end
+
+    # <CopyObjectResult>
+    #   <LastModified>2009-10-28T22:32:00</LastModified>
+    #   <ETag>"9b2cf535f27731c974343645a3985328"</ETag>
+    # </CopyObjectResult>
+    def self.copy_object_result(object)
+      output = ""
+      xml = Builder::XmlMarkup.new(:target => output)
+      xml.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+      xml.CopyObjectResult { |result|
+        result.LastModified(object.modified_date)
+        result.ETag("\"#{object.md5}\"")
+      }
+      output
+    end
+
+    # <CompleteMultipartUploadResult>
+    #   <Location>http://Example-Bucket.s3.amazonaws.com/Example-Object</Location>
+    #   <Bucket>Example-Bucket</Bucket>
+    #   <Key>Example-Object</Key>
+    #   <ETag>"3858f62230ac3c915f300c664312c11f-9"</ETag>
+    # </CompleteMultipartUploadResult>
+    def self.complete_multipart_result(object)
+      output = ""
+      xml = Builder::XmlMarkup.new(:target => output)
+      xml.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+      xml.CompleteMultipartUploadResult { |result|
+        result.Location("TODO: implement")
+        result.Bucket("TODO: implement")
+        result.Key(object.name)
+        result.ETag("\"#{object.md5}\"")
+      }
+      output
+    end
+  end
+end

data/lib/fakes3/xml_parser.rb

@@ -0,0 +1,16 @@
+require 'xmlsimple'
+
+module FakeS3
+  class XmlParser
+    def self.delete_objects(request)
+      keys = []
+
+      objects = XmlSimple.xml_in(request.body, {'NoAttr' => true})['Object']
+      objects.each do |key|
+        keys << key['Key'][0]
+      end
+
+      keys
+    end
+  end
+end

data/static/button.svg

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" width="110px" height="20px" viewBox="0 0 110 20" enable-background="new 0 0 110 20" xml:space="preserve">  <image id="image0" width="110" height="20" x="0" y="0" xlink:href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAG4AAAAUCAMAAABbPPhuAAAABGdBTUEAALGPC/xhBQAAACBjSFJN AAB6JQAAgIMAAPn/AACA6QAAdTAAAOpgAAA6mAAAF2+SX8VGAAABL1BMVEUAUJX///8AUJUAUJUA UJUAUJUAUJUAUJUAUJUAUJUAUJUAUJUAUJUAUJUAUJUAUJUAUJUAUJUAUJUAUJUAUJUAUJUAUJUA UJUAUJUAUJUAUJUAUJUAUJUAUJU5e7Rmns59r9uAs917sNx6r9yDtN5lnc0vdK9EhLtzq9o/i8wp fsYhecQdd8Mke8U9iss6fLU4e7RyqtougccfeMNQjcx0otRjmNAhesQzhMliodVKib8+i8zT4PDI 2OyTtdyvx+ShvuD///8qf8dOlNCAstwlfMU7fbZ/st0gecQ5g8e7z+iErNibw+QyhMnq8Pj19/vf 6PTG3fAwg8jm8PhNlND5+/12rNs0hcluqNh3rdtNisDn8fl5rtvH3fBnns4xdbCcw+VhoNU5fLVo oM99sNsds+t8AAAAHXRSTlMAAAZux/NbG8+ZCNAobZoK+dNcn/VgLF+dB2/L9Kp1EagAAAABYktH RAH/Ai3eAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAB3RJTUUH4AsECgU1BHGi1AAAAmZJREFUSMe9 lgtT1DAQxytwgoLC+X4BeiegGEUkCXqXtgkomlabqvh+P77/Z3A3SXst5Wa4cY6duUmbbPaX/HeT XhAEE5NTrcXx2ckTFQuC6Zml5Zu3Ot3x2O2V1VOnK7iJmbU7d9fvkTHZ/Qedjdm5Ae7Mw81HW0ef Thkfjbf9+EmvVQoaTC33HU2ghdGQWZGIbSuVGHF/Wzu7K6tn5z1u4em661bO4sMnFRgmo6NRKvas 39lrO0GDxecvChzDqMpFjCTjUiJH0gZOy1BIDS+JbcBXigTfQxFTHBZJ6oK+tEWx/QoEPedwHVLB pSpzkYWSVClKuDK13WGrc5RBkAgbo8HXwENOKL5nfli6oK/flIKen0dct8TlAibyEkdC+MUqaeAS lVMdc21UCj4J+GYMZOESMsEYumiOS7VB3+57QbsoaBXnc1fiUlhxrngDlykrFQwzwGTWF8VhsGWQ OleSsVxFPui7965CP3zsXbhYxYGYFEKVOIjLIFoDZz1tnq2VOCJB1IQY1y0L3KcC97mBw6XhgoEk MWDmc1DD2fCY1gzKQ5MBjpLUKI45wG4X9MuuF/PrXvtSrVQgdxmkIIWn3K4Oc08LnFsy4sAhiQ3m TEgByS5w0sSwP8qUiXHYBv3mS2X/+w9XKoODgHUmIC2h5eG2hApJE0ci0MtERAvoCXWJSzOnoR12 J+GnPwi/eq3L9WNeMc7cGYt9wskwD830gW4nBq/ddL4q65dY02CXIflvs2fuSnGJTS4Nv6L5iNfx YXbgir46u7bZH+cHqLvRnqt9Xn//+Xtsn1f483Dt+o3j+vPwD8d6fpTeP3ACAAAAJXRFWHRkYXRl OmNyZWF0ZQAyMDE2LTExLTA0VDEwOjA1OjUzLTA3OjAwJ5wXuQAAACV0RVh0ZGF0ZTptb2RpZnkA MjAxNi0xMS0wNFQxMDowNTo1My0wNzowMFbBrwUAAAAASUVORK5CYII="/>
+</svg>

data/test/aws_sdk_commands_test.rb

@@ -0,0 +1,98 @@
+require 'test/test_helper'
+require 'aws-sdk-v1'
+require 'rest-client'
+
+class AwsSdkCommandsTest < Test::Unit::TestCase
+  def setup
+    @s3 = AWS::S3.new(:access_key_id => '123',
+                      :secret_access_key => 'abc',
+                      :s3_endpoint => 'localhost',
+                      :s3_port => 10453,
+                      :use_ssl => false)
+  end
+
+  def test_copy_to
+    bucket = @s3.buckets["test_copy_to"]
+    object = bucket.objects["key1"]
+    object.write("asdf")
+
+    assert object.exists?
+    object.copy_to("key2")
+
+    assert_equal 2, bucket.objects.count
+  end
+
+  def test_multipart_upload
+    bucket = @s3.buckets["test_multipart_upload"]
+    object = bucket.objects["key1"]
+    object.write("thisisaverybigfile", :multipart_threshold => 5)
+    assert object.exists?
+    assert_equal "thisisaverybigfile", object.read
+  end
+
+  def test_metadata
+    file_path = './test_root/test_metadata/metaobject'
+    FileUtils.rm_rf file_path
+
+    bucket = @s3.buckets["test_metadata"]
+    object = bucket.objects["metaobject"]
+    object.write(
+      'data',
+      # this is sent as header x-amz-storage-class
+      :storage_class => 'REDUCED_REDUNDANCY',
+      # this is sent as header x-amz-meta-custom1
+      :metadata => {
+        "custom1" => "foobar"
+      }
+    )
+    assert object.exists?
+    metadata_file = YAML.load(IO.read("#{file_path}/.fakes3_metadataFFF/metadata"))
+
+    assert metadata_file.has_key?(:custom_metadata), 'Metadata file does not contain a :custom_metadata key'
+    assert metadata_file[:custom_metadata].has_key?('custom1'), ':custom_metadata does not contain field "custom1"'
+    assert_equal 'foobar', metadata_file[:custom_metadata]['custom1'], '"custom1" does not equal expected value "foobar"'
+
+    assert metadata_file.has_key?(:amazon_metadata), 'Metadata file does not contain an :amazon_metadata key'
+    assert metadata_file[:amazon_metadata].has_key?('storage-class'), ':amazon_metadata does not contain field "storage-class"'
+    assert_equal 'REDUCED_REDUNDANCY', metadata_file[:amazon_metadata]['storage-class'], '"storage-class" does not equal expected value "REDUCED_REDUNDANCY"'
+  end
+
+  def test_content_disposition
+    bucket = @s3.buckets["test_bucket"]
+    bucket.objects.create("test_object", "asdf", :content_disposition => "application/test")
+    assert_equal "application/test", content_disposition("test_bucket", "test_object")
+  end
+
+  def test_content_disposition_copy
+    bucket = @s3.buckets["test_bucket"]
+    object = bucket.objects.create("test_object", "asdf", :content_disposition => "application/test")
+    object.copy_to("test_copy_object")
+    assert_equal "application/test", content_disposition("test_bucket", "test_copy_object")
+  end
+
+  def test_content_disposition_request_parameter
+    bucket = @s3.buckets["test_bucket"]
+    object = bucket.objects.create("test_object", "asdf")
+    url = object.url_for(:read, :response_content_disposition => "application/test", :signature_version => :v4)
+    assert_equal "application/test", response_header(url, :content_disposition)
+  end
+
+  def test_content_type_request_parameter
+    bucket = @s3.buckets["test_bucket"]
+    object = bucket.objects.create("test_object", "asdf")
+    url = object.url_for(:read, :response_content_type => "application/test", :signature_version => :v4)
+    assert_equal "application/test", response_header(url, :content_type)
+  end
+
+  # Unfortunately v1 of the AWS SDK doesn't support reading the content_disposition of an object
+  def content_disposition(bucket_name, key)
+    url = "http://localhost:#{@s3.client.port}/#{bucket_name}/#{key}"
+    response_header(url, :content_disposition)
+  end
+
+  def response_header(url, header_name)
+    RestClient.head(url.to_s) do |response|
+      response.headers[header_name]
+    end
+  end
+end