fakes3-docker 0.2.4

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c3de236d260469450faeff406eb2f80085732bc6
+  data.tar.gz: 8ca413e3c812847298b2c85cab432b8ae1fbaac3
+SHA512:
+  metadata.gz: fb245968df5e0f5970b08e87acc8aa7ef81b825144d4564bc6197bb0e807c9d019eff55202868344a759b484f70b2089dc02f0621e1fe044c274924deac88fc0
+  data.tar.gz: 20add51c0e285b595774aa88dd6b8ed5c410d23d26112dcef599e3ad49c48cfaa1889bde6690e91fb9aede6e252a1e158d43d9705496e74d8f43694df9e29e5a

data/.gitignore

@@ -0,0 +1,14 @@
+pkg/*
+*.gem
+.bundle
+tmp
+test_root
+
+# Don't check in RVM/rbenv files
+.ruby-version
+
+# RubyMine
+.idea
+
+# Don't check in the Gemfile.lock
+Gemfile.lock

data/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+gem 'fakes3-docker', :require => 'fakes3'
+gem 'fakes3-docker', :path => '.' # for dev and test, use local fakes3-docker
+# Specify your gem's dependencies in fakes3.gemspec
+gemspec

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2011,2012 Curtis W Spencer (@jubos) and Spool
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,62 @@
+## Introduction
+FakeS3 is a lightweight server that responds to the same calls Amazon S3 responds to.
+It is extremely useful for testing of S3 in a sandbox environment without actually
+making calls to Amazon, which not only require network, but also cost you precious dollars.
+
+The goal of Fake S3 is to minimize runtime dependencies and be more of a
+development tool to test S3 calls in your code rather than a production server
+looking to duplicate S3 functionality.  Trying RiakCS, ParkPlace/Boardwalk, or
+Ceph might be a place to start if that is your goal.
+
+FakeS3 doesn't support all of the S3 command set, but the basic ones like put, get,
+list, copy, and make bucket are supported.  More coming soon.
+
+## Installation
+
+    gem install fakes3
+
+## Running
+
+To run a fakes3 server, you just specify a root and a port.
+
+    fakes3 -r /mnt/fakes3_root -p 4567
+
+## Connecting to FakeS3
+
+Take a look at the test cases to see client example usage.  For now, FakeS3 is
+mainly tested with s3cmd, aws-s3 gem, and right_aws.  There are plenty more
+libraries out there, and please do mention if other clients work or not.
+
+Here is a running list of [supported clients](https://github.com/jubos/fake-s3/wiki/Supported-Clients "Supported Clients")
+
+## Running Tests
+
+There are some pre-requesites to actually being able to run the unit/integration tests
+
+### On OSX
+
+Edit your /etc/hosts and add the following line:
+
+    127.0.0.1 posttest.localhost
+
+Then ensure that the following packages are installed (boto, s3cmd)
+
+    > pip install boto
+    > brew install s3cmd
+
+
+Start the test server using
+
+    rake test_server
+
+Then in another terminal window run
+
+    rake test
+
+It is a still a TODO to get this to be just one command
+
+## More Information
+
+Check out the [wiki](https://github.com/jubos/fake-s3/wiki)
+
+[![Join the chat at https://gitter.im/jubos/fake-s3](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/jubos/fake-s3?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)

data/Rakefile

@@ -0,0 +1,18 @@
+require 'rubygems'
+require 'bundler'
+require 'rake/testtask'
+include Rake::DSL
+Bundler::GemHelper.install_tasks
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "."
+  t.test_files =
+    FileList['test/*_test.rb'].exclude('test/s3_commands_test.rb')
+end
+
+desc "Run the test_server"
+task :test_server do |t|
+  system("bundle exec bin/fakes3 --port 10453 --root test_root")
+end
+
+task :default => :test

data/bin/fakes3

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+$: << './lib'
+
+require 'fakes3/cli'
+FakeS3::CLI.start

data/fakes3.gemspec

@@ -0,0 +1,33 @@
+# -*- encoding: utf-8 -*-
+require File.join(File.dirname(__FILE__), 'lib', 'fakes3', 'version')
+
+Gem::Specification.new do |s|
+  s.name        = "fakes3-docker"
+  s.version     = FakeS3::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Curtis Spencer"]
+  s.email       = ["thorin@gmail.com"]
+  s.homepage    = "https://github.com/jubos/fake-s3"
+  s.summary     = %q{FakeS3 is a server that simulates S3 commands so you can test your S3 functionality in your projects}
+  s.description = %q{Use FakeS3 to test basic S3 functionality without actually connecting to S3}
+  s.license     = "MIT"
+
+  s.rubyforge_project = "fakes3-docker"
+
+  s.add_development_dependency "bundler", ">= 1.0.0"
+  s.add_development_dependency "aws-s3"
+  s.add_development_dependency "right_aws"
+  s.add_development_dependency "rest-client"
+  s.add_development_dependency "rake"
+  s.add_development_dependency "aws-sdk-v1"
+  s.add_development_dependency "test-unit"
+  #s.add_development_dependency "ruby-debug"
+  #s.add_development_dependency "debugger"
+  s.add_dependency "thor"
+  s.add_dependency "builder"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

data/lib/fakes3.rb

@@ -0,0 +1,3 @@
+require 'fakes3/version'
+require 'fakes3/file_store'
+require 'fakes3/server'

data/lib/fakes3/bucket.rb

@@ -0,0 +1,65 @@
+require 'builder'
+require 'thread'
+require 'fakes3/s3_object'
+require 'fakes3/sorted_object_list'
+
+module FakeS3
+  class Bucket
+    attr_accessor :name,:creation_date,:objects
+
+    def initialize(name,creation_date,objects)
+      @name = name
+      @creation_date = creation_date
+      @objects = SortedObjectList.new
+      objects.each do |obj|
+        @objects.add(obj)
+      end
+      @mutex = Mutex.new
+    end
+
+    def find(object_name)
+      @mutex.synchronize do
+        @objects.find(object_name)
+      end
+    end
+
+    def add(object)
+      # Unfortunately have to synchronize here since the our SortedObjectList
+      # not thread safe. Probably can get finer granularity if performance is
+      # important
+      @mutex.synchronize do
+        @objects.add(object)
+      end
+    end
+
+    def remove(object)
+      @mutex.synchronize do
+        @objects.remove(object)
+      end
+    end
+
+    def query_for_range(options)
+      marker = options[:marker]
+      prefix = options[:prefix]
+      max_keys = options[:max_keys] || 1000
+      delimiter = options[:delimiter]
+
+      match_set = nil
+      @mutex.synchronize do
+        match_set = @objects.list(options)
+      end
+
+      bq = BucketQuery.new
+      bq.bucket = self
+      bq.marker = marker
+      bq.prefix = prefix
+      bq.max_keys = max_keys
+      bq.delimiter = delimiter
+      bq.matches = match_set.matches
+      bq.is_truncated = match_set.is_truncated
+      bq.common_prefixes = match_set.common_prefixes
+      return bq
+    end
+
+  end
+end

data/lib/fakes3/bucket_query.rb

@@ -0,0 +1,11 @@
+module FakeS3
+  class BucketQuery
+    attr_accessor :prefix,:matches,:marker,:max_keys,
+                  :delimiter,:bucket,:is_truncated,:common_prefixes
+
+    # Syntactic sugar
+    def is_truncated?
+      @is_truncated
+    end
+  end
+end

data/lib/fakes3/cli.rb

@@ -0,0 +1,70 @@
+require 'thor'
+require 'fakes3/server'
+require 'fakes3/version'
+
+module FakeS3
+  class CLI < Thor
+    default_task("server")
+
+    desc "server", "Run a server on a particular hostname"
+    method_option :root, :type => :string, :aliases => '-r', :required => true
+    method_option :port, :type => :numeric, :aliases => '-p', :required => true
+    method_option :address, :type => :string, :aliases => '-a', :required => false, :desc => "Bind to this address. Defaults to 0.0.0.0"
+    method_option :hostname, :type => :string, :aliases => '-H', :desc => "The root name of the host.  Defaults to s3.amazonaws.com."
+    method_option :limit, :aliases => '-l', :type => :string, :desc => 'Rate limit for serving (ie. 50K, 1.0M)'
+    method_option :sslcert, :type => :string, :desc => 'Path to SSL certificate'
+    method_option :sslkey, :type => :string, :desc => 'Path to SSL certificate key'
+
+    def server
+      store = nil
+      if options[:root]
+        root = File.expand_path(options[:root])
+        # TODO Do some sanity checking here
+        store = FileStore.new(root)
+      end
+
+      if store.nil?
+        abort "You must specify a root to use a file store (the current default)"
+      end
+
+      hostname = 's3.amazonaws.com'
+      if options[:hostname]
+        hostname = options[:hostname]
+        # In case the user has put a port on the hostname
+        if hostname =~ /:(\d+)/
+          hostname = hostname.split(":")[0]
+        end
+      end
+
+      if options[:limit]
+        begin
+          store.rate_limit = options[:limit]
+        rescue
+          abort $!.message
+        end
+      end
+
+      address = options[:address] || '0.0.0.0'
+      ssl_cert_path = options[:sslcert]
+      ssl_key_path = options[:sslkey]
+
+      if (ssl_cert_path.nil? && !ssl_key_path.nil?) || (!ssl_cert_path.nil? && ssl_key_path.nil?)
+        abort "If you specify an SSL certificate you must also specify an SSL certificate key"
+      end
+
+      puts "Loading FakeS3 with #{root} on port #{options[:port]} with hostname #{hostname}"
+      server = FakeS3::Server.new(address,options[:port],store,hostname,ssl_cert_path,ssl_key_path)
+      server.serve
+    end
+
+    desc "version", "Report the current fakes3 version"
+    def version
+      puts <<"EOF"
+======================
+FakeS3 #{FakeS3::VERSION}
+
+Copyright 2012, Curtis Spencer (@jubos)
+EOF
+    end
+  end
+end

data/lib/fakes3/errors.rb

@@ -0,0 +1,46 @@
+module FakeS3
+  class FakeS3Exception < RuntimeError
+    attr_accessor :resource,:request_id
+
+    def self.metaclass; class << self; self; end; end
+
+    def self.traits(*arr)
+      return @traits if arr.empty?
+      attr_accessor *arr
+
+      arr.each do |a|
+        metaclass.instance_eval do
+          define_method( a ) do |val|
+            @traits ||= {}
+            @traits[a] = val
+          end
+        end
+      end
+
+      class_eval do
+        define_method( :initialize ) do
+          self.class.traits.each do |k,v|
+            instance_variable_set("@#{k}", v)
+          end
+        end
+      end
+    end
+
+    traits :message,:http_status
+
+    def code
+      self.class.to_s
+    end
+  end
+
+  class NoSuchBucket < FakeS3Exception
+    message "The bucket you tried to delete is not empty."
+    http_status "404"
+  end
+
+  class BucketNotEmpty < FakeS3Exception
+    message "The bucket you tried to delete is not empty."
+    http_status "409"
+  end
+
+end

data/lib/fakes3/file_store.rb

@@ -0,0 +1,282 @@
+require 'fileutils'
+require 'time'
+require 'fakes3/s3_object'
+require 'fakes3/bucket'
+require 'fakes3/rate_limitable_file'
+require 'digest/md5'
+require 'yaml'
+
+module FakeS3
+  class FileStore
+    SHUCK_METADATA_DIR = ".fakes3_metadataFFF"
+    # S3 clients with overly strict date parsing fails to parse ISO 8601 dates
+    # without any sub second precision (e.g. jets3t v0.7.2), and the examples
+    # given in the official AWS S3 documentation specify three (3) decimals for
+    # sub second precision.
+    SUBSECOND_PRECISION = 3
+
+    def initialize(root)
+      @root = root
+      @buckets = []
+      @bucket_hash = {}
+      Dir[File.join(root,"*")].each do |bucket|
+        bucket_name = File.basename(bucket)
+        bucket_obj = Bucket.new(bucket_name,Time.now,[])
+        @buckets << bucket_obj
+        @bucket_hash[bucket_name] = bucket_obj
+      end
+    end
+
+    # Pass a rate limit in bytes per second
+    def rate_limit=(rate_limit)
+      if rate_limit.is_a?(String)
+        if rate_limit =~ /^(\d+)$/
+          RateLimitableFile.rate_limit = rate_limit.to_i
+        elsif rate_limit =~ /^(.*)K$/
+          RateLimitableFile.rate_limit = $1.to_f * 1000
+        elsif rate_limit =~ /^(.*)M$/
+          RateLimitableFile.rate_limit = $1.to_f * 1000000
+        elsif rate_limit =~ /^(.*)G$/
+          RateLimitableFile.rate_limit = $1.to_f * 1000000000
+        else
+          raise "Invalid Rate Limit Format: Valid values include (1000,10K,1.1M)"
+        end
+      else
+        RateLimitableFile.rate_limit = nil
+      end
+    end
+
+    def buckets
+      @buckets
+    end
+
+    def get_bucket_folder(bucket)
+      File.join(@root,bucket.name)
+    end
+
+    def get_bucket(bucket)
+      @bucket_hash[bucket]
+    end
+
+    def create_bucket(bucket)
+      FileUtils.mkdir_p(File.join(@root,bucket))
+      bucket_obj = Bucket.new(bucket,Time.now,[])
+      if !@bucket_hash[bucket]
+        @buckets << bucket_obj
+        @bucket_hash[bucket] = bucket_obj
+      end
+      bucket_obj
+    end
+
+    def delete_bucket(bucket_name)
+      bucket = get_bucket(bucket_name)
+      raise NoSuchBucket if !bucket
+      raise BucketNotEmpty if bucket.objects.count > 0
+      FileUtils.rm_r(get_bucket_folder(bucket))
+      @bucket_hash.delete(bucket_name)
+    end
+
+    def get_object(bucket,object_name, request)
+      begin
+        real_obj = S3Object.new
+        obj_root = File.join(@root,bucket,object_name,SHUCK_METADATA_DIR)
+        metadata = YAML.load(File.open(File.join(obj_root,"metadata"),'rb'))
+        real_obj.name = object_name
+        real_obj.md5 = metadata[:md5]
+        real_obj.content_type = metadata.fetch(:content_type) { "application/octet-stream" }
+        #real_obj.io = File.open(File.join(obj_root,"content"),'rb')
+        real_obj.io = RateLimitableFile.open(File.join(obj_root,"content"),'rb')
+        real_obj.size = metadata.fetch(:size) { 0 }
+        real_obj.creation_date = File.ctime(obj_root).utc.iso8601(SUBSECOND_PRECISION)
+        real_obj.modified_date = metadata.fetch(:modified_date) do
+          File.mtime(File.join(obj_root,"content")).utc.iso8601(SUBSECOND_PRECISION)
+        end
+        real_obj.custom_metadata = metadata.fetch(:custom_metadata) { {} }
+        return real_obj
+      rescue
+        puts $!
+        $!.backtrace.each { |line| puts line }
+        return nil
+      end
+    end
+
+    def object_metadata(bucket,object)
+    end
+
+    def copy_object(src_bucket_name, src_name, dst_bucket_name, dst_name, request)
+      src_root = File.join(@root,src_bucket_name,src_name,SHUCK_METADATA_DIR)
+      src_metadata_filename = File.join(src_root,"metadata")
+      src_metadata = YAML.load(File.open(src_metadata_filename,'rb').read)
+      src_content_filename = File.join(src_root,"content")
+
+      dst_filename= File.join(@root,dst_bucket_name,dst_name)
+      FileUtils.mkdir_p(dst_filename)
+
+      metadata_dir = File.join(dst_filename,SHUCK_METADATA_DIR)
+      FileUtils.mkdir_p(metadata_dir)
+
+      content = File.join(metadata_dir,"content")
+      metadata = File.join(metadata_dir,"metadata")
+
+      if src_bucket_name != dst_bucket_name || src_name != dst_name
+        File.open(content,'wb') do |f|
+          File.open(src_content_filename,'rb') do |input|
+            f << input.read
+          end
+        end
+
+        File.open(metadata,'w') do |f|
+          File.open(src_metadata_filename,'r') do |input|
+            f << input.read
+          end
+        end
+      end
+
+      metadata_directive = request.header["x-amz-metadata-directive"].first
+      if metadata_directive == "REPLACE"
+        metadata_struct = create_metadata(content,request)
+        File.open(metadata,'w') do |f|
+          f << YAML::dump(metadata_struct)
+        end
+      end
+
+      src_bucket = get_bucket(src_bucket_name) || create_bucket(src_bucket_name)
+      dst_bucket = get_bucket(dst_bucket_name) || create_bucket(dst_bucket_name)
+
+      obj = S3Object.new
+      obj.name = dst_name
+      obj.md5 = src_metadata[:md5]
+      obj.content_type = src_metadata[:content_type]
+      obj.size = src_metadata[:size]
+      obj.modified_date = src_metadata[:modified_date]
+
+      src_obj = src_bucket.find(src_name)
+      dst_bucket.add(obj)
+      return obj
+    end
+
+    def store_object(bucket, object_name, request)
+      filedata = ""
+
+      # TODO put a tmpfile here first and mv it over at the end
+      content_type = request.content_type || ""
+
+      match = content_type.match(/^multipart\/form-data; boundary=(.+)/)
+      boundary = match[1] if match
+      if boundary
+        boundary  = WEBrick::HTTPUtils::dequote(boundary)
+        form_data = WEBrick::HTTPUtils::parse_form_data(request.body, boundary)
+
+        if form_data['file'] == nil or form_data['file'] == ""
+          raise WEBrick::HTTPStatus::BadRequest
+        end
+
+        filedata = form_data['file']
+      else
+        request.body { |chunk| filedata << chunk }
+      end
+
+      do_store_object(bucket, object_name, filedata, request)
+    end
+
+    def do_store_object(bucket, object_name, filedata, request)
+      begin
+        filename = File.join(@root,bucket.name,object_name)
+        FileUtils.mkdir_p(filename)
+
+        metadata_dir = File.join(filename,SHUCK_METADATA_DIR)
+        FileUtils.mkdir_p(metadata_dir)
+
+        content  = File.join(filename,SHUCK_METADATA_DIR,"content")
+        metadata = File.join(filename,SHUCK_METADATA_DIR,"metadata")
+
+        File.open(content,'wb') { |f| f << filedata }
+
+        metadata_struct = create_metadata(content,request)
+        File.open(metadata,'w') do |f|
+          f << YAML::dump(metadata_struct)
+        end
+
+        obj = S3Object.new
+        obj.name = object_name
+        obj.md5 = metadata_struct[:md5]
+        obj.content_type = metadata_struct[:content_type]
+        obj.size = metadata_struct[:size]
+        obj.modified_date = metadata_struct[:modified_date]
+
+        bucket.add(obj)
+        return obj
+      rescue
+        puts $!
+        $!.backtrace.each { |line| puts line }
+        return nil
+      end
+    end
+
+    def combine_object_parts(bucket, upload_id, object_name, parts, request)
+      upload_path   = File.join(@root, bucket.name)
+      base_path     = File.join(upload_path, "#{upload_id}_#{object_name}")
+
+      complete_file = ""
+      chunk         = ""
+      part_paths    = []
+
+      parts.sort_by { |part| part[:number] }.each do |part|
+        part_path    = "#{base_path}_part#{part[:number]}"
+        content_path = File.join(part_path, SHUCK_METADATA_DIR, 'content')
+
+        File.open(content_path, 'rb') { |f| chunk = f.read }
+        etag = Digest::MD5.hexdigest(chunk)
+
+        raise new Error "invalid file chunk" unless part[:etag] == etag
+        complete_file << chunk
+        part_paths    << part_path
+      end
+
+      object = do_store_object(bucket, object_name, complete_file, request)
+
+      # clean up parts
+      part_paths.each do |path|
+        FileUtils.remove_dir(path)
+      end
+
+      object
+    end
+
+    def delete_object(bucket,object_name,request)
+      begin
+        filename = File.join(@root,bucket.name,object_name)
+        FileUtils.rm_rf(filename)
+        object = bucket.find(object_name)
+        bucket.remove(object)
+      rescue
+        puts $!
+        $!.backtrace.each { |line| puts line }
+        return nil
+      end
+    end
+
+    # TODO: abstract getting meta data from request.
+    def create_metadata(content,request)
+      metadata = {}
+      metadata[:md5] = Digest::MD5.file(content).hexdigest
+      metadata[:content_type] = request.header["content-type"].first
+      metadata[:size] = File.size(content)
+      metadata[:modified_date] = File.mtime(content).utc.iso8601(SUBSECOND_PRECISION)
+      metadata[:amazon_metadata] = {}
+      metadata[:custom_metadata] = {}
+
+      # Add custom metadata from the request header
+      request.header.each do |key, value|
+        match = /^x-amz-([^-]+)-(.*)$/.match(key)
+        next unless match
+        if match[1].eql?('meta') && (match_key = match[2])
+          metadata[:custom_metadata][match_key] = value.join(', ')
+          next
+        end
+        metadata[:amazon_metadata][key.gsub(/^x-amz-/, '')] = value.join(', ')
+      end
+      return metadata
+    end
+  end
+end