amione-fakes3 0.1.5.1

data/.gitignore

@@ -0,0 +1,5 @@
+pkg/*
+*.gem
+.bundle
+tmp
+test_root

data/Gemfile

@@ -0,0 +1,4 @@
+source :rubygems
+gem 'fakes3', :path => '.' # for dev and test, use local fakes3
+# Specify your gem's dependencies in fakes3.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,32 @@
+PATH
+  remote: .
+  specs:
+    fakes3 (0.1.5)
+      builder
+      thor
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    aws-s3 (0.6.2)
+      builder
+      mime-types
+      xml-simple
+    builder (3.0.0)
+    mime-types (1.18)
+    rake (0.9.2.2)
+    right_aws (3.0.4)
+      right_http_connection (>= 1.2.5)
+    right_http_connection (1.3.0)
+    thor (0.14.6)
+    xml-simple (1.1.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  aws-s3
+  bundler (>= 1.0.0)
+  fakes3!
+  rake
+  right_aws

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2011,2012 Curtis W Spencer (@jubos) and Spool
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,46 @@
+## Introduction
+FakeS3 is a lightweight server that responds to the same calls Amazon S3 responds to.  
+It is extremely useful for testing of S3 in a sandbox environment without actually
+making calls to Amazon, which not only require network, but also cost you precious dollars.  
+
+The goal of Fake S3 is to minimize runtime dependencies and be more of a
+development tool to test S3 calls in your code rather than a production server
+looking to duplicate S3 functionality.  Trying RiakCS, ParkPlace/Boardwalk, or
+Ceph might be a place to start if that is your goal.
+
+FakeS3 doesn't support all of the S3 command set, but the basic ones like put, get,
+list, copy, and make bucket are supported.  More coming soon.
+
+## Installation
+
+    gem install fakes3
+
+## Running
+
+To run a fakes3 server, you just specify a root and a port.
+
+    fakes3 -r /mnt/fakes3_root -p 4567
+
+## Connecting to FakeS3
+
+Take a look at the test cases to see client example usage.  For now, FakeS3 is
+mainly tested with s3cmd, aws-s3 gem, and right_aws.  There are plenty more
+libraries out there, and please do mention if other clients work or not.
+
+Here is a running list of [supported clients](https://github.com/jubos/fake-s3/wiki/Supported-Clients "Supported Clients")
+
+## Running Tests
+
+Start the test server using
+
+    rake test_server
+
+Then in another terminal window run
+
+    rake test
+
+It is a TODO to get this to be just one command
+
+## More Information
+
+Check out the [wiki](https://github.com/jubos/fake-s3/wiki)

data/Rakefile

@@ -0,0 +1,17 @@
+require 'rubygems'
+require 'bundler'
+require 'rake/testtask'
+include Rake::DSL
+Bundler::GemHelper.install_tasks
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "."
+  t.test_files = FileList['test/*_test.rb']
+end
+
+desc "Run the test_server"
+task :test_server do |t|
+  system("bundle exec bin/fakes3 --port 10453 --root test_root")
+end
+
+task :default => :test

data/bin/fakes3

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require 'fakes3/cli'
+FakeS3::CLI.start

data/fakes3.gemspec

@@ -0,0 +1,30 @@
+# -*- encoding: utf-8 -*-
+require File.join(File.dirname(__FILE__), 'lib', 'fakes3', 'version')
+
+Gem::Specification.new do |s|
+  s.name        = "amione-fakes3"
+  s.version     = FakeS3::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Curtis Spencer"]
+  s.email       = ["thorin@gmail.com"]
+  s.homepage    = ""
+  s.summary     = %q{FakeS3 is a server that simulates S3 commands so you can test your S3 functionality in your projects}
+  s.description = %q{Use FakeS3 to test basic S3 functionality without actually connecting to S3}
+
+  s.rubyforge_project = "fakes3"
+
+  s.add_development_dependency "bundler", ">= 1.0.0"
+  s.add_development_dependency "aws-s3"
+  s.add_development_dependency "right_aws"
+  s.add_development_dependency "rake"
+  #s.add_development_dependency "aws-sdk"
+  #s.add_development_dependency "ruby-debug"
+  #s.add_development_dependency "ruby-debug19"
+  s.add_dependency "thor"
+  s.add_dependency "builder"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

data/lib/fakes3.rb

@@ -0,0 +1,3 @@
+require 'fakes3/version'
+require 'fakes3/file_store'
+require 'fakes3/server'

data/lib/fakes3/bucket.rb

@@ -0,0 +1,64 @@
+require 'builder'
+require 'thread'
+require 'fakes3/s3_object'
+require 'fakes3/sorted_object_list'
+
+module FakeS3
+  class Bucket
+    attr_accessor :name,:creation_date,:objects
+
+    def initialize(name,creation_date,objects)
+      @name = name
+      @creation_date = creation_date
+      @objects = SortedObjectList.new
+      objects.each do |obj|
+        @objects.add(obj)
+      end
+      @mutex = Mutex.new
+    end
+
+    def find(object_name)
+      @mutex.synchronize do
+        @objects.find(object_name)
+      end
+    end
+
+    def add(object)
+      # Unfortunately have to synchronize here since the our SortedObjectList
+      # not thread safe. Probably can get finer granularity if performance is
+      # important
+      @mutex.synchronize do
+        @objects.add(object)
+      end
+    end
+
+    def remove(object)
+      @mutex.synchronize do
+        @objects.remove(object)
+      end
+    end
+
+    def query_for_range(options)
+      marker = options[:marker]
+      prefix = options[:prefix]
+      max_keys = options[:max_keys] || 1000
+      delimiter = options[:delimiter]
+
+      match_set = nil
+      @mutex.synchronize do
+        match_set = @objects.list(options)
+      end
+
+      bq = BucketQuery.new
+      bq.bucket = self
+      bq.marker = marker
+      bq.prefix = prefix
+      bq.max_keys = max_keys
+      bq.delimiter = delimiter
+      bq.matches = match_set.matches
+      bq.is_truncated = match_set.is_truncated
+      return bq
+    end
+
+  end
+end

data/lib/fakes3/bucket_query.rb

@@ -0,0 +1,11 @@
+module FakeS3
+  class BucketQuery
+    attr_accessor :prefix,:matches,:marker,:max_keys,
+                  :delimiter,:bucket,:is_truncated
+
+    # Syntactic sugar
+    def is_truncated?
+      @is_truncated
+    end
+  end
+end

data/lib/fakes3/cli.rb

@@ -0,0 +1,63 @@
+require 'thor'
+require 'fakes3/server'
+require 'fakes3/version'
+
+module FakeS3
+  class CLI < Thor
+    default_task("server")
+
+    desc "server", "Run a server on a particular hostname"
+    method_option :root, :type => :string, :aliases => '-r', :required => true
+    method_option :port, :type => :numeric, :aliases => '-p', :required => true
+    method_option :address, :type => :string, :aliases => '-a', :required => false, :desc => "Bind to this address. Defaults to 0.0.0.0"
+    method_option :hostname, :type => :string, :aliases => '-h', :desc => "The root name of the host.  Defaults to s3.amazonaws.com."
+    method_option :limit, :aliases => '-l', :type => :string, :desc => 'Rate limit for serving (ie. 50K, 1.0M)'
+    def server
+      store = nil
+      if options[:root]
+        root = File.expand_path(options[:root])
+        # TODO Do some sanity checking here
+        store = FileStore.new(root)
+      end
+
+      if store.nil?
+        puts "You must specify a root to use a file store (the current default)"
+        exit(-1)
+      end
+
+      hostname = 's3.amazonaws.com'
+      if options[:hostname]
+        hostname = options[:hostname]
+        # In case the user has put a port on the hostname
+        if hostname =~ /:(\d+)/
+          hostname = hostname.split(":")[0]
+        end
+      end
+
+      if options[:limit]
+        begin
+          store.rate_limit = options[:limit]
+        rescue
+          puts $!.message
+          exit(-1)
+        end
+      end
+
+      address = options[:address] || '0.0.0.0'
+
+      puts "Loading FakeS3 with #{root} on port #{options[:port]} with hostname #{hostname}"
+      server = FakeS3::Server.new(address,options[:port],store,hostname)
+      server.serve
+    end
+
+    desc "version", "Report the current fakes3 version"
+    def version
+      puts <<"EOF"
+======================
+FakeS3 #{FakeS3::VERSION}
+
+Copyright 2012, Curtis Spencer (@jubos)
+EOF
+    end
+  end
+end

data/lib/fakes3/errors.rb

@@ -0,0 +1,46 @@
+module FakeS3
+  class FakeS3Exception < RuntimeError
+    attr_accessor :resource,:request_id
+
+    def self.metaclass; class << self; self; end; end
+
+    def self.traits(*arr)
+      return @traits if arr.empty?
+      attr_accessor *arr
+
+      arr.each do |a|
+        metaclass.instance_eval do
+          define_method( a ) do |val|
+            @traits ||= {}
+            @traits[a] = val
+          end
+        end
+      end
+
+      class_eval do
+        define_method( :initialize ) do
+          self.class.traits.each do |k,v|
+            instance_variable_set("@#{k}", v)
+          end
+        end
+      end
+    end
+
+    traits :message,:http_status
+
+    def code
+      self.class.to_s
+    end
+  end
+
+  class NoSuchBucket < FakeS3Exception
+    message "The bucket you tried to delete is not empty."
+    http_status "404"
+  end
+
+  class BucketNotEmpty < FakeS3Exception
+    message "The bucket you tried to delete is not empty."
+    http_status "409"
+  end
+
+end

data/lib/fakes3/file_store.rb

@@ -0,0 +1,201 @@
+require 'fileutils'
+require 'time'
+require 'fakes3/s3_object'
+require 'fakes3/bucket'
+require 'fakes3/rate_limitable_file'
+require 'digest/md5'
+require 'yaml'
+
+module FakeS3
+  class FileStore
+    SHUCK_METADATA_DIR = ".fakes3_metadataFFF"
+
+    def initialize(root)
+      @root = root
+      @buckets = []
+      @bucket_hash = {}
+      Dir[File.join(root,"*")].each do |bucket|
+        bucket_name = File.basename(bucket)
+        bucket_obj = Bucket.new(bucket_name,Time.now,[])
+        @buckets << bucket_obj
+        @bucket_hash[bucket_name] = bucket_obj
+      end
+    end
+
+    # Pass a rate limit in bytes per second
+    def rate_limit=(rate_limit)
+      if rate_limit.is_a?(String)
+        if rate_limit =~ /^(\d+)$/
+          RateLimitableFile.rate_limit = rate_limit.to_i
+        elsif rate_limit =~ /^(.*)K$/
+          RateLimitableFile.rate_limit = $1.to_f * 1000
+        elsif rate_limit =~ /^(.*)M$/
+          RateLimitableFile.rate_limit = $1.to_f * 1000000
+        elsif rate_limit =~ /^(.*)G$/
+          RateLimitableFile.rate_limit = $1.to_f * 1000000000
+        else
+          raise "Invalid Rate Limit Format: Valid values include (1000,10K,1.1M)"
+        end
+      else
+        RateLimitableFile.rate_limit = nil
+      end
+    end
+
+    def buckets
+      @buckets
+    end
+
+    def get_bucket_folder(bucket)
+      File.join(@root,bucket.name)
+    end
+
+    def get_bucket(bucket)
+      @bucket_hash[bucket]
+    end
+
+    def create_bucket(bucket)
+      FileUtils.mkdir_p(File.join(@root,bucket))
+      bucket_obj = Bucket.new(bucket,Time.now,[])
+      if !@bucket_hash[bucket]
+        @buckets << bucket_obj
+        @bucket_hash[bucket] = bucket_obj
+      end
+      bucket_obj
+    end
+
+    def delete_bucket(bucket_name)
+      bucket = get_bucket(bucket_name)
+      raise NoSuchBucket if !bucket
+      raise BucketNotEmpty if bucket.objects.count > 0
+      FileUtils.rm_r(get_bucket_folder(bucket))
+      @bucket_hash.delete(bucket_name)
+    end
+
+    def get_object(bucket,object_name, request)
+      begin
+        real_obj = S3Object.new
+        obj_root = File.join(@root,bucket,object_name,SHUCK_METADATA_DIR)
+        metadata = YAML.load(File.open(File.join(obj_root,"metadata"),'rb'))
+        real_obj.name = object_name
+        real_obj.md5 = metadata[:md5]
+        real_obj.content_type = metadata.fetch(:content_type) { "application/octet-stream" }
+        #real_obj.io = File.open(File.join(obj_root,"content"),'rb')
+        real_obj.io = RateLimitableFile.open(File.join(obj_root,"content"),'rb')
+        real_obj.size = metadata.fetch(:size) { 0 }
+        real_obj.creation_date = File.ctime(obj_root).iso8601()
+        real_obj.modified_date = metadata.fetch(:modified_date) { File.mtime(File.join(obj_root,"content")).iso8601() }
+        return real_obj
+      rescue
+        puts $!
+        $!.backtrace.each { |line| puts line }
+        return nil
+      end
+    end
+
+    def object_metadata(bucket,object)
+    end
+
+    def copy_object(src_bucket_name,src_name,dst_bucket_name,dst_name)
+      src_root = File.join(@root,src_bucket_name,src_name,SHUCK_METADATA_DIR)
+      src_metadata_filename = File.join(src_root,"metadata")
+      src_metadata = YAML.load(File.open(src_metadata_filename,'rb').read)
+      src_content_filename = File.join(src_root,"content")
+
+      dst_filename= File.join(@root,dst_bucket_name,dst_name)
+      FileUtils.mkdir_p(dst_filename)
+
+      metadata_dir = File.join(dst_filename,SHUCK_METADATA_DIR)
+      FileUtils.mkdir_p(metadata_dir)
+
+      content = File.join(metadata_dir,"content")
+      metadata = File.join(metadata_dir,"metadata")
+
+      File.open(content,'wb') do |f|
+        File.open(src_content_filename,'rb') do |input|
+          f << input.read
+        end
+      end
+
+      File.open(metadata,'w') do |f|
+        File.open(src_metadata_filename,'r') do |input|
+          f << input.read
+        end
+      end
+
+      src_bucket = self.get_bucket(src_bucket_name)
+      dst_bucket = self.get_bucket(dst_bucket_name)
+
+      obj = S3Object.new
+      obj.name = dst_name
+      obj.md5 = src_metadata[:md5]
+      obj.content_type = src_metadata[:content_type]
+      obj.size = src_metadata[:size]
+      obj.modified_date = src_metadata[:modified_date]
+
+      src_obj = src_bucket.find(src_name)
+      dst_bucket.add(obj)
+      src_bucket.remove(src_obj)
+      return obj
+    end
+
+    def store_object(bucket,object_name,request)
+      begin
+        filename = File.join(@root,bucket.name,object_name)
+        FileUtils.mkdir_p(filename)
+
+        metadata_dir = File.join(filename,SHUCK_METADATA_DIR)
+        FileUtils.mkdir_p(metadata_dir)
+
+        content = File.join(filename,SHUCK_METADATA_DIR,"content")
+        metadata = File.join(filename,SHUCK_METADATA_DIR,"metadata")
+
+        md5 = Digest::MD5.new
+        # TODO put a tmpfile here first and mv it over at the end
+
+        File.open(content,'wb') do |f|
+          request.body do |chunk|
+            f << chunk
+            md5 << chunk
+          end
+        end
+
+        metadata_struct = {}
+        metadata_struct[:md5] = md5.hexdigest
+        metadata_struct[:content_type] = request.header["content-type"].first
+        metadata_struct[:size] = File.size(content)
+        metadata_struct[:modified_date] = File.mtime(content).iso8601()
+
+        File.open(metadata,'w') do |f|
+          f << YAML::dump(metadata_struct)
+        end
+
+        obj = S3Object.new
+        obj.name = object_name
+        obj.md5 = metadata_struct[:md5]
+        obj.content_type = metadata_struct[:content_type]
+        obj.size = metadata_struct[:size]
+        obj.modified_date = metadata_struct[:modified_date]
+
+        bucket.add(obj)
+        return obj
+      rescue
+        puts $!
+        $!.backtrace.each { |line| puts line }
+        return nil
+      end
+    end
+
+    def delete_object(bucket,object_name,request)
+      begin
+        filename = File.join(@root,bucket.name,object_name)
+        FileUtils.rm_rf(filename)
+        object = bucket.find(object_name)
+        bucket.remove(object)
+      rescue
+        puts $!
+        $!.backtrace.each { |line| puts line }
+        return nil
+      end
+    end
+  end
+end