fakes3-ruby18 0.2.1

data/lib/fakes3/sorted_object_list.rb

@@ -0,0 +1,137 @@
+require 'set'
+module FakeS3
+  class S3MatchSet
+    attr_accessor :matches,:is_truncated,:common_prefixes
+    def initialize
+      @matches = []
+      @is_truncated = false
+      @common_prefixes = []
+    end
+  end
+
+  # This class has some of the semantics necessary for how buckets can return
+  # their items
+  #
+  # It is currently implemented naively as a sorted set + hash If you are going
+  # to try to put massive lists inside buckets and ls them, you will be sorely
+  # disappointed about this performance.
+  class SortedObjectList
+
+    def initialize
+      @sorted_set = SortedSet.new
+      @object_map = {}
+      @mutex = Mutex.new
+    end
+
+    def count
+      @sorted_set.count
+    end
+
+    def find(object_name)
+      @object_map[object_name]
+    end
+
+    # Add an S3 object into the sorted list
+    def add(s3_object)
+      return if !s3_object
+
+      @object_map[s3_object.name] = s3_object
+      @sorted_set << s3_object
+    end
+
+    def remove(s3_object)
+      return if !s3_object
+
+      @object_map.delete(s3_object.name)
+      @sorted_set.delete(s3_object)
+    end
+
+    # Return back a set of matches based on the passed in options
+    #
+    # options:
+    #
+    # :marker : a string to start the lexographical search (it is not included
+    #           in the result)
+    # :max_keys  : a maximum number of results
+    # :prefix    : a string to filter the results by
+    # :delimiter : not supported yet
+    def list(options)
+      marker = options[:marker]
+      prefix = options[:prefix]
+      max_keys = options[:max_keys] || 1000
+      delimiter = options[:delimiter]
+
+      ms = S3MatchSet.new
+
+      marker_found = true
+      pseudo = nil
+      if marker
+        marker_found = false
+        if !@object_map[marker]
+          pseudo = S3Object.new
+          pseudo.name = marker
+          @sorted_set << pseudo
+        end
+      end
+
+      if delimiter
+        if prefix
+          base_prefix = prefix
+        else
+          base_prefix = ""
+        end
+        prefix_offset = base_prefix.length
+      end
+
+      count = 0
+      last_chunk = nil
+      @sorted_set.each do |s3_object|
+        if marker_found && (!prefix or s3_object.name.index(prefix) == 0)
+          if delimiter
+            name = s3_object.name
+            remainder = name.slice(prefix_offset, name.length)
+            chunks = remainder.split(delimiter, 2)
+            if chunks.length > 1
+              if (last_chunk != chunks[0])
+                # "All of the keys rolled up in a common prefix count as
+                # a single return when calculating the number of
+                # returns. See MaxKeys."
+                # (http://awsdocs.s3.amazonaws.com/S3/latest/s3-api.pdf)
+                count += 1
+                if count <= max_keys
+                  ms.common_prefixes << base_prefix + chunks[0] + delimiter
+                  last_chunk = chunks[0]
+                else
+                  is_truncated = true
+                  break
+                end
+              end
+
+              # Continue to the next key, since this one has a
+              # delimiter.
+              next
+            end
+          end
+
+          count += 1
+          if count <= max_keys
+            ms.matches << s3_object
+          else
+            is_truncated = true
+            break
+          end
+        end
+
+        if marker and marker == s3_object.name
+          marker_found = true
+        end
+      end
+
+      if pseudo
+        @sorted_set.delete(pseudo)
+      end
+
+      return ms
+    end
+  end
+end

data/lib/fakes3/unsupported_operation.rb

@@ -0,0 +1,4 @@
+module FakeS3
+  class UnsupportedOperation < RuntimeError
+  end
+end

data/lib/fakes3/version.rb

@@ -0,0 +1,3 @@
+module FakeS3
+  VERSION = "0.2.1"
+end

data/lib/fakes3/xml_adapter.rb

@@ -0,0 +1,222 @@
+require 'builder'
+require 'time'
+
+module FakeS3
+  class XmlAdapter
+    def self.buckets(bucket_objects)
+      output = ""
+      xml = Builder::XmlMarkup.new(:target => output)
+      xml.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+      xml.ListAllMyBucketsResult(:xmlns => "http://s3.amazonaws.com/doc/2006-03-01/") { |lam|
+        lam.Owner { |owner|
+          owner.ID("123")
+          owner.DisplayName("FakeS3")
+        }
+        lam.Buckets { |buckets|
+          bucket_objects.each do |bucket|
+            buckets.Bucket do |b|
+              b.Name(bucket.name)
+              b.CreationDate(bucket.creation_date.strftime("%Y-%m-%dT%H:%M:%S.000Z"))
+            end
+          end
+        }
+      }
+      output
+    end
+
+    def self.error(error)
+      output = ""
+      xml = Builder::XmlMarkup.new(:target => output)
+      xml.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+      xml.Error { |err|
+        err.Code(error.code)
+        err.Message(error.message)
+        err.Resource(error.resource)
+        err.RequestId(1)
+      }
+      output
+    end
+
+    # <?xml version="1.0" encoding="UTF-8"?>
+    #<Error>
+    #  <Code>NoSuchKey</Code>
+    #  <Message>The resource you requested does not exist</Message>
+    #  <Resource>/mybucket/myfoto.jpg</Resource>
+    #  <RequestId>4442587FB7D0A2F9</RequestId>
+    #</Error>
+    #
+    def self.error_no_such_bucket(name)
+      output = ""
+      xml = Builder::XmlMarkup.new(:target => output)
+      xml.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+      xml.Error { |err|
+        err.Code("NoSuchBucket")
+        err.Message("The resource you requested does not exist")
+        err.Resource(name)
+        err.RequestId(1)
+      }
+      output
+    end
+
+    def self.error_bucket_not_empty(name)
+      output = ""
+      xml = Builder::XmlMarkup.new(:target => output)
+      xml.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+      xml.Error { |err|
+        err.Code("BucketNotEmpty")
+        err.Message("The bucket you tried to delete is not empty.")
+        err.Resource(name)
+        err.RequestId(1)
+      }
+      output
+    end
+
+    def self.error_no_such_key(name)
+      output = ""
+      xml = Builder::XmlMarkup.new(:target => output)
+      xml.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+      xml.Error { |err|
+        err.Code("NoSuchKey")
+        err.Message("The specified key does not exist")
+        err.Key(name)
+        err.RequestId(1)
+        err.HostId(2)
+      }
+      output
+    end
+
+    def self.bucket(bucket)
+      output = ""
+      xml = Builder::XmlMarkup.new(:target => output)
+      xml.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+      xml.ListBucketResult(:xmlns => "http://s3.amazonaws.com/doc/2006-03-01/") { |lbr|
+        lbr.Name(bucket.name)
+        lbr.Prefix
+        lbr.Marker
+        lbr.MaxKeys("1000")
+        lbr.IsTruncated("false")
+      }
+      output
+    end
+
+    # A bucket query gives back the bucket along with contents
+    #  <Contents>
+    #<Key>Nelson</Key>
+#    <LastModified>2006-01-01T12:00:00.000Z</LastModified>
+#    <ETag>&quot;828ef3fdfa96f00ad9f27c383fc9ac7f&quot;</ETag>
+#    <Size>5</Size>
+#    <StorageClass>STANDARD</StorageClass>
+#    <Owner>
+#      <ID>bcaf161ca5fb16fd081034f</ID>
+#      <DisplayName>webfile</DisplayName>
+#     </Owner>
+#    </Contents>
+
+    def self.append_objects_to_list_bucket_result(lbr,objects)
+      return if objects.nil? or objects.size == 0
+
+      if objects.index(nil)
+        require 'ruby-debug'
+        Debugger.start
+        debugger
+      end
+
+      objects.each do |s3_object|
+        lbr.Contents { |contents|
+          contents.Key(s3_object.name)
+          contents.LastModified(s3_object.modified_date)
+          contents.ETag("\"#{s3_object.md5}\"")
+          contents.Size(s3_object.size)
+          contents.StorageClass("STANDARD")
+
+          contents.Owner { |owner|
+            owner.ID("abc")
+            owner.DisplayName("You")
+          }
+        }
+      end
+    end
+
+    def self.append_common_prefixes_to_list_bucket_result(lbr, prefixes)
+      return if prefixes.nil? or prefixes.size == 0
+
+      prefixes.each do |common_prefix|
+        lbr.CommonPrefixes { |contents| contents.Prefix(common_prefix) }
+      end
+    end
+
+    def self.bucket_query(bucket_query)
+      output = ""
+      bucket = bucket_query.bucket
+      xml = Builder::XmlMarkup.new(:target => output)
+      xml.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+      xml.ListBucketResult(:xmlns => "http://s3.amazonaws.com/doc/2006-03-01/") { |lbr|
+        lbr.Name(bucket.name)
+        lbr.Prefix(bucket_query.prefix)
+        lbr.Marker(bucket_query.marker)
+        lbr.MaxKeys(bucket_query.max_keys)
+        lbr.IsTruncated(bucket_query.is_truncated?)
+        append_objects_to_list_bucket_result(lbr,bucket_query.matches)
+        append_common_prefixes_to_list_bucket_result(lbr, bucket_query.common_prefixes)
+      }
+      output
+    end
+
+    # ACL xml
+    def self.acl(object = nil)
+      output = ""
+      xml = Builder::XmlMarkup.new(:target => output)
+      xml.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+      xml.AccessControlPolicy(:xmlns => "http://s3.amazonaws.com/doc/2006-03-01/") { |acp|
+        acp.Owner do |owner|
+          owner.ID("abc")
+          owner.DisplayName("You")
+        end
+        acp.AccessControlList do |acl|
+          acl.Grant do |grant|
+            grant.Grantee("xmlns:xsi" => "http://www.w3.org/2001/XMLSchema-instance", "xsi:type" => "CanonicalUser") do |grantee|
+              grantee.ID("abc")
+              grantee.DisplayName("You")
+            end
+            grant.Permission("FULL_CONTROL")
+          end
+        end
+      }
+      output
+    end
+
+    # <CopyObjectResult>
+    #   <LastModified>2009-10-28T22:32:00</LastModified>
+    #   <ETag>"9b2cf535f27731c974343645a3985328"</ETag>
+    # </CopyObjectResult>
+    def self.copy_object_result(object)
+      output = ""
+      xml = Builder::XmlMarkup.new(:target => output)
+      xml.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+      xml.CopyObjectResult { |result|
+        result.LastModified(object.modified_date)
+        result.ETag("\"#{object.md5}\"")
+      }
+      output
+    end
+
+    # <CompleteMultipartUploadResult>
+    #   <Location>http://Example-Bucket.s3.amazonaws.com/Example-Object</Location>
+    #   <Bucket>Example-Bucket</Bucket>
+    #   <Key>Example-Object</Key>
+    #   <ETag>"3858f62230ac3c915f300c664312c11f-9"</ETag>
+    # </CompleteMultipartUploadResult>
+    def self.complete_multipart_result(object)
+      output = ""
+      xml = Builder::XmlMarkup.new(:target => output)
+      xml.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+      xml.CompleteMultipartUploadResult { |result|
+        result.Location("TODO: implement")
+        result.Bucket("TODO: implement")
+        result.Key(object.name)
+        result.ETag("\"#{object.md5}\"")
+      }
+      output
+    end
+  end
+end

data/test/aws_sdk_commands_test.rb

@@ -0,0 +1,31 @@
+require 'test/test_helper'
+require 'aws-sdk-v1'
+
+class AwsSdkCommandsTest < Test::Unit::TestCase
+  def setup
+    @s3 = AWS::S3.new(:access_key_id => '123',
+                      :secret_access_key => 'abc',
+                      :s3_endpoint => 'localhost',
+                      :s3_port => 10453,
+                      :use_ssl => false)
+  end
+
+  def test_copy_to
+    bucket = @s3.buckets["test_copy_to"]
+    object = bucket.objects["key1"]
+    object.write("asdf")
+
+    assert object.exists?
+    object.copy_to("key2")
+
+    assert_equal 2, bucket.objects.count
+  end
+
+  def test_multipart_upload
+    bucket = @s3.buckets["test_multipart_upload"]
+    object = bucket.objects["key1"]
+    object.write("thisisaverybigfile", :multipart_threshold => 5)
+    assert object.exists?
+    assert_equal "thisisaverybigfile", object.read
+  end
+end

data/test/boto_test.rb

@@ -0,0 +1,25 @@
+require 'test/test_helper'
+require 'fileutils'
+
+class BotoTest < Test::Unit::TestCase
+  def setup
+    cmdpath = File.expand_path(File.join(File.dirname(__FILE__),'botocmd.py'))
+    @botocmd = "python #{cmdpath} -t localhost -p 10453"
+  end
+
+  def teardown
+  end
+
+  def test_store
+    File.open(__FILE__,'rb') do |input|
+      File.open("/tmp/fakes3_upload",'wb') do |output|
+        output << input.read
+      end
+    end
+    output = `#{@botocmd} put /tmp/fakes3_upload s3://s3cmd_bucket/upload`
+    assert_match(/stored/,output)
+
+    FileUtils.rm("/tmp/fakes3_upload")
+  end
+
+end

data/test/botocmd.py

@@ -0,0 +1,87 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# fakes3cmd.py -- an s3cmd-like script that accepts a custom host and portname
+import re
+import os
+from optparse import OptionParser
+
+try:
+    from boto.s3.connection import S3Connection, OrdinaryCallingFormat
+    from boto.s3.key import Key
+except ImportError:
+    raise Exception('You must install the boto package for python')
+
+
+class FakeS3Cmd(object):
+    COMMANDS = ['mb', 'rb', 'put', ]
+    def __init__(self, host, port):
+        self.host = host
+        self.port = port
+        self.conn = None
+        self._connect()
+
+    def _connect(self):
+        print 'Connecting: %s:%s' % (self.host, self.port)
+        self.conn = S3Connection(is_secure=False,
+                             calling_format=OrdinaryCallingFormat(),
+                             aws_access_key_id='',
+                             aws_secret_access_key='',
+                             port=self.port, host=self.host)
+
+
+    @staticmethod
+    def _parse_uri(path):
+        match = re.match(r's3://([^/]+)(?:/(.*))?', path, re.I)
+        ## (bucket, key)
+        return match.groups()
+
+    def mb(self, path, *args):
+        if not self.conn:
+            self._connect()
+
+        bucket, _ = self._parse_uri(path)
+        self.conn.create_bucket(bucket)
+        print 'made bucket: [%s]' % bucket
+
+    def rb(self, path, *args):
+        if not self.conn:
+            self._connect()
+
+        bucket, _ = self._parse_uri(path)
+        self.conn.delete_bucket(bucket)
+        print 'removed bucket: [%s]' % bucket
+
+    def put(self, *args):
+        if not self.conn:
+            self._connect()
+
+        args = list(args)
+        path = args.pop()
+        bucket_name, prefix = self._parse_uri(path)
+        bucket = self.conn.create_bucket(bucket_name)
+        for src_file in args:
+            key = Key(bucket)
+            key.key = os.path.join(prefix, os.path.basename(src_file))
+            key.set_contents_from_filename(src_file)
+            print 'stored: [%s]' % key.key
+
+
+if __name__ == "__main__":
+    # check for options. TODO: This requires a more verbose help message
+    # to explain how the positional arguments work.
+    parser = OptionParser()
+    parser.add_option("-t", "--host", type="string", default='localhost')
+    parser.add_option("-p", "--port", type='int', default=80)
+    o, args = parser.parse_args()
+
+    if len(args) < 2:
+        raise ValueError('you must minimally supply a desired command and s3 uri')
+
+    cmd = args.pop(0)
+
+    if cmd not in FakeS3Cmd.COMMANDS:
+        raise ValueError('%s is not a valid command' % cmd)
+
+    fs3 = FakeS3Cmd(o.host, o.port)
+    handler = getattr(fs3, cmd)
+    handler(*args)