rack_fake_s3 0.2.0

data/.gitignore

@@ -0,0 +1,5 @@
+pkg/*
+*.gem
+.bundle
+tmp
+test_root

data/Gemfile

@@ -0,0 +1,2 @@
+source :rubygems
+gemspec

data/Gemfile.lock

@@ -0,0 +1,37 @@
+PATH
+  remote: .
+  specs:
+    rack_fake_s3 (0.2.0)
+      builder
+      rack
+      thor
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    aws-s3 (0.6.2)
+      builder
+      mime-types
+      xml-simple
+    builder (3.0.0)
+    mime-types (1.18)
+    rack (1.4.1)
+    rake (0.9.2.2)
+    rest-client (1.6.7)
+      mime-types (>= 1.16)
+    right_aws (3.0.4)
+      right_http_connection (>= 1.2.5)
+    right_http_connection (1.3.0)
+    thor (0.16.0)
+    xml-simple (1.1.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  aws-s3
+  bundler (>= 1.0.0)
+  rack_fake_s3!
+  rake
+  rest-client
+  right_aws

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2011,2012 Curtis W Spencer (@jubos) and Spool
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,39 @@
+## Introduction
+
+This library is a modified version of the FakeS3 gem.
+You can find the original implementation here: [https://github.com/jubos/fake-s3](https://github.com/jubos/fake-s3)
+
+This version is modified to run on rack.
+
+## What it does
+
+Rack Fake S3 is a lightweight server that responds to the same calls Amazon S3 responds to.
+It is extremely useful for testing of S3 in a sandbox environment without actually
+making calls to Amazon, which not only require network, but also cost you precious dollars.
+
+The goal of Rack Fake S3 is to minimize runtime dependencies and be more of a
+development tool to test S3 calls in your code rather than a production server
+looking to duplicate S3 functionality.  Trying RiakCS, ParkPlace/Boardwalk, or
+Ceph might be a place to start if that is your goal.
+
+Rack Fake S3 doesn't support all of the S3 command set, but the basic ones like put, get,
+list, copy, and make bucket are supported. More coming soon.
+
+## Installation
+
+### Rails (as a mounted rack app)
+
+  Add rack_fake_s3 to your Gemfile and bundle
+
+      # Gemfile
+      gem 'rack_fake_s3'
+
+  Mount the app to your routes
+
+    # config/routes.rb
+
+    constraints :host => domain do
+      mount RackFakeS3::App.new(root_path, domain)
+    end
+
+  Setup your app to point to the specified domain instead of the real AWS S3.

data/Rakefile

@@ -0,0 +1,12 @@
+require 'rubygems'
+require 'bundler'
+require 'rake/testtask'
+include Rake::DSL
+Bundler::GemHelper.install_tasks
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "."
+  t.test_files = FileList['test/*_test.rb']
+end
+
+task :default => :test

data/config.ru

@@ -0,0 +1,4 @@
+$:.unshift File.expand_path("../lib", __FILE__)
+require 'rack_fake_s3'
+
+run RackFakeS3::App.new(RackFakeS3::FileStore.new(ENV['FAKE_S3_ROOT']), ENV['FAKE_S3_HOSTNAME'])

data/lib/rack_fake_s3/bucket.rb

@@ -0,0 +1,64 @@
+require 'builder'
+require 'thread'
+require 'rack_fake_s3/s3_object'
+require 'rack_fake_s3/sorted_object_list'
+
+module RackFakeS3
+  class Bucket
+    attr_accessor :name,:creation_date,:objects
+
+    def initialize(name,creation_date,objects)
+      @name = name
+      @creation_date = creation_date
+      @objects = SortedObjectList.new
+      objects.each do |obj|
+        @objects.add(obj)
+      end
+      @mutex = Mutex.new
+    end
+
+    def find(object_name)
+      @mutex.synchronize do
+        @objects.find(object_name)
+      end
+    end
+
+    def add(object)
+      # Unfortunately have to synchronize here since the our SortedObjectList
+      # not thread safe. Probably can get finer granularity if performance is
+      # important
+      @mutex.synchronize do
+        @objects.add(object)
+      end
+    end
+
+    def remove(object)
+      @mutex.synchronize do
+        @objects.remove(object)
+      end
+    end
+
+    def query_for_range(options)
+      marker = options[:marker]
+      prefix = options[:prefix]
+      max_keys = options[:max_keys] || 1000
+      delimiter = options[:delimiter]
+
+      match_set = nil
+      @mutex.synchronize do
+        match_set = @objects.list(options)
+      end
+
+      bq = BucketQuery.new
+      bq.bucket = self
+      bq.marker = marker
+      bq.prefix = prefix
+      bq.max_keys = max_keys
+      bq.delimiter = delimiter
+      bq.matches = match_set.matches
+      bq.is_truncated = match_set.is_truncated
+      return bq
+    end
+
+  end
+end

data/lib/rack_fake_s3/bucket_query.rb

@@ -0,0 +1,11 @@
+module RackFakeS3
+  class BucketQuery
+    attr_accessor :prefix,:matches,:marker,:max_keys,
+                  :delimiter,:bucket,:is_truncated
+
+    # Syntactic sugar
+    def is_truncated?
+      @is_truncated
+    end
+  end
+end

data/lib/rack_fake_s3/errors.rb

@@ -0,0 +1,46 @@
+module RackFakeS3
+  class RackFakeS3Exception < RuntimeError
+    attr_accessor :resource,:request_id
+
+    def self.metaclass; class << self; self; end; end
+
+    def self.traits(*arr)
+      return @traits if arr.empty?
+      attr_accessor *arr
+
+      arr.each do |a|
+        metaclass.instance_eval do
+          define_method( a ) do |val|
+            @traits ||= {}
+            @traits[a] = val
+          end
+        end
+      end
+
+      class_eval do
+        define_method( :initialize ) do
+          self.class.traits.each do |k,v|
+            instance_variable_set("@#{k}", v)
+          end
+        end
+      end
+    end
+
+    traits :message,:http_status
+
+    def code
+      self.class.to_s
+    end
+  end
+
+  class NoSuchBucket < RackFakeS3Exception
+    message "The bucket you tried to delete is not empty."
+    http_status "404"
+  end
+
+  class BucketNotEmpty < RackFakeS3Exception
+    message "The bucket you tried to delete is not empty."
+    http_status "409"
+  end
+
+end

data/lib/rack_fake_s3/file_store.rb

@@ -0,0 +1,211 @@
+require 'fileutils'
+require 'time'
+require 'rack_fake_s3/s3_object'
+require 'rack_fake_s3/bucket'
+require 'rack_fake_s3/rate_limitable_file'
+require 'digest/md5'
+require 'yaml'
+
+module RackFakeS3
+  class FileStore
+    SHUCK_METADATA_DIR = ".rack_fake_s3_metadataFFF"
+
+    def initialize(root)
+      @root = root
+      @buckets = []
+      @bucket_hash = {}
+      Dir[File.join(root,"*")].each do |bucket|
+        bucket_name = File.basename(bucket)
+        bucket_obj = Bucket.new(bucket_name,Time.now,[])
+        @buckets << bucket_obj
+        @bucket_hash[bucket_name] = bucket_obj
+
+        # FIXME: this is not the best place to do this
+        # Dir[File.join(bucket, "*")].each do |s3object|
+        #   obj = get_object(bucket_name, File.basename(s3object), nil)
+        #   bucket_obj.add obj
+        # end
+      end
+    end
+
+    # Pass a rate limit in bytes per second
+    def rate_limit=(rate_limit)
+      if rate_limit.is_a?(String)
+        if rate_limit =~ /^(\d+)$/
+          RateLimitableFile.rate_limit = rate_limit.to_i
+        elsif rate_limit =~ /^(.*)K$/
+          RateLimitableFile.rate_limit = $1.to_f * 1000
+        elsif rate_limit =~ /^(.*)M$/
+          RateLimitableFile.rate_limit = $1.to_f * 1000000
+        elsif rate_limit =~ /^(.*)G$/
+          RateLimitableFile.rate_limit = $1.to_f * 1000000000
+        else
+          raise "Invalid Rate Limit Format: Valid values include (1000,10K,1.1M)"
+        end
+      else
+        RateLimitableFile.rate_limit = nil
+      end
+    end
+
+    def buckets
+      @buckets
+    end
+
+    def get_bucket_folder(bucket)
+      File.join(@root,bucket.name)
+    end
+
+    def get_bucket(bucket)
+      @bucket_hash[bucket]
+    end
+
+    def create_bucket(bucket)
+      FileUtils.mkdir_p(File.join(@root,bucket))
+      bucket_obj = Bucket.new(bucket,Time.now,[])
+      if !@bucket_hash[bucket]
+        @buckets << bucket_obj
+        @bucket_hash[bucket] = bucket_obj
+      end
+      bucket_obj
+    end
+
+    def delete_bucket(bucket_name)
+      bucket = get_bucket(bucket_name)
+      raise NoSuchBucket if !bucket
+      raise BucketNotEmpty if bucket.objects.count > 0
+      FileUtils.rm_r(get_bucket_folder(bucket))
+      @bucket_hash.delete(bucket_name)
+    end
+
+    def get_object(bucket,object_name, request)
+      begin
+        real_obj = S3Object.new
+        obj_root = File.join(@root,bucket,object_name,SHUCK_METADATA_DIR)
+        metadata = YAML.load(File.open(File.join(obj_root,"metadata"),'rb'))
+        real_obj.name = object_name
+        real_obj.md5 = metadata[:md5]
+        real_obj.content_type = metadata.fetch(:content_type) { "application/octet-stream" }
+        #real_obj.io = File.open(File.join(obj_root,"content"),'rb')
+        real_obj.io = RateLimitableFile.open(File.join(obj_root,"content"),'rb')
+        return real_obj
+      rescue
+        puts $!
+        $!.backtrace.each { |line| puts line }
+        return nil
+      end
+    end
+
+    def object_metadata(bucket,object)
+    end
+
+    def copy_object(src_bucket_name,src_name,dst_bucket_name,dst_name)
+      src_root = File.join(@root,src_bucket_name,src_name,SHUCK_METADATA_DIR)
+      src_metadata_filename = File.join(src_root,"metadata")
+      src_metadata = YAML.load(File.open(src_metadata_filename,'rb').read)
+      src_content_filename = File.join(src_root,"content")
+
+      dst_filename= File.join(@root,dst_bucket_name,dst_name)
+      FileUtils.mkdir_p(dst_filename)
+
+      metadata_dir = File.join(dst_filename,SHUCK_METADATA_DIR)
+      FileUtils.mkdir_p(metadata_dir)
+
+      content = File.join(metadata_dir,"content")
+      metadata = File.join(metadata_dir,"metadata")
+
+      File.open(content,'wb') do |f|
+        File.open(src_content_filename,'rb') do |input|
+          f << input.read
+        end
+      end
+
+      File.open(metadata,'w') do |f|
+        File.open(src_metadata_filename,'r') do |input|
+          f << input.read
+        end
+      end
+
+      src_bucket = self.get_bucket(src_bucket_name)
+      dst_bucket = self.get_bucket(dst_bucket_name)
+
+      obj = S3Object.new
+      obj.name = dst_name
+      obj.md5 = src_metadata[:md5]
+      obj.content_type = src_metadata[:content_type]
+
+      src_obj = src_bucket.find(src_name)
+      dst_bucket.add(obj)
+      src_bucket.remove(src_obj)
+      return obj
+    end
+
+    def store_object(bucket,object_name,request)
+      begin
+        filename = File.join(@root,bucket.name,object_name)
+        FileUtils.mkdir_p(filename)
+
+        metadata_dir = File.join(filename,SHUCK_METADATA_DIR)
+        FileUtils.mkdir_p(metadata_dir)
+
+        content = File.join(filename,SHUCK_METADATA_DIR,"content")
+        metadata = File.join(filename,SHUCK_METADATA_DIR,"metadata")
+
+        md5 = Digest::MD5.new
+        # TODO put a tmpfile here first and mv it over at the end
+
+        match=request.content_type.match(/^multipart\/form-data; boundary=(.+)/)
+        boundary = match[1] if match
+        if boundary
+          boundary = WEBrick::HTTPUtils::dequote(boundary)
+          filedata = WEBrick::HTTPUtils::parse_form_data(request.body, boundary)
+          raise HTTPStatus::BadRequest if filedata['file'].empty?
+          File.open(content, 'wb') do |f|
+            f<<filedata['file']
+            md5<<filedata['file']
+          end
+        else
+          File.open(content,'wb') do |f|
+            request.body do |chunk|
+              f << chunk
+              md5 << chunk
+            end
+          end
+        end
+
+        metadata_struct = {}
+        metadata_struct[:md5] = md5.hexdigest
+        metadata_struct[:content_type] = request.media_type || ""
+
+        yaml = YAML::dump(metadata_struct)
+        File.open(metadata,'w') do |f|
+          f << yaml
+        end
+
+        obj = S3Object.new
+        obj.name = object_name
+        obj.md5 = metadata_struct[:md5]
+        obj.content_type = metadata_struct[:content_type]
+
+        bucket.add(obj)
+        return obj
+      rescue
+        puts $!
+        $!.backtrace.each { |line| puts line }
+        return nil
+      end
+    end
+
+    def delete_object(bucket,object_name,request)
+      begin
+        filename = File.join(@root,bucket.name,object_name)
+        FileUtils.rm_rf(filename)
+        object = bucket.find(object_name)
+        bucket.remove(object)
+      rescue
+        puts $!
+        $!.backtrace.each { |line| puts line }
+        return nil
+      end
+    end
+  end
+end

data/lib/rack_fake_s3/rate_limitable_file.rb

@@ -0,0 +1,21 @@
+module RackFakeS3
+  class RateLimitableFile < File
+    @@rate_limit = nil
+    # Specify a rate limit in bytes per second
+    def self.rate_limit
+      @@rate_limit
+    end
+
+    def self.rate_limit=(rate_limit)
+      @@rate_limit = rate_limit
+    end
+
+    def read(args)
+      if @@rate_limit
+        time_to_sleep = args / @@rate_limit
+        sleep(time_to_sleep)
+      end
+      return super(args)
+    end
+  end
+end

data/lib/rack_fake_s3/s3_object.rb

@@ -0,0 +1,19 @@
+module RackFakeS3
+  class S3Object
+    include Comparable
+    attr_accessor :name,:size,:creation_date,:md5,:io,:content_type
+
+    def hash
+      @name.hash
+    end
+
+    def eql?(object)
+      @name == object.name
+    end
+
+    # Sort by the object's name
+    def <=>(object)
+      @name <=> object.name
+    end
+  end
+end