shuck 0.0.7 → 0.0.8

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    shuck (0.0.6)
+    shuck (0.0.8)
       builder
       thor
 
@@ -13,17 +13,10 @@ GEM
       mime-types
       xml-simple
     builder (2.1.2)
-    columnize (0.3.2)
-    linecache (0.43)
     mime-types (1.16)
     right_aws (2.0.0)
       right_http_connection (>= 1.2.1)
     right_http_connection (1.2.4)
-    ruby-debug (0.10.4)
-      columnize (>= 0.1)
-      ruby-debug-base (~> 0.10.4.0)
-    ruby-debug-base (0.10.4)
-      linecache (>= 0.3)
     thor (0.14.4)
     xml-simple (1.0.12)
 
@@ -35,6 +28,5 @@ DEPENDENCIES
   builder
   bundler (>= 1.0.0)
   right_aws
-  ruby-debug
   shuck!
   thor

data/Rakefile

@@ -1,5 +1,6 @@
 require 'bundler'
 require 'rake/testtask'
+include Rake::DSL
 Bundler::GemHelper.install_tasks
 
 Rake::TestTask.new(:test) do |t|

data/lib/shuck/cli.rb

@@ -9,12 +9,14 @@ module Shuck
     method_option :root, :type => :string, :aliases => '-r', :required => true
     method_option :port, :type => :numeric, :aliases => '-p', :required => true
     method_option :hostname, :type => :string, :aliases => '-h', :desc => "The root name of the host.  Defaults to localhost."
+    method_option :limit, :aliases => '-l', :type => :string, :desc => 'Rate limit for serving (ie. 50K, 1.0M)'
     def server
       store = nil
       if options[:root]
         root = File.expand_path(options[:root])
         store = FileStore.new(root)
       end
+
       if store.nil?
         puts "You must specify a root to use a file store (the current default)"
         exit(-1)
@@ -25,6 +27,15 @@ module Shuck
         hostname = options[:hostname]
       end
 
+      if options[:limit]
+        begin
+          store.rate_limit = options[:limit]
+        rescue
+          puts $!.message
+          exit(-1)
+        end
+      end
+
       puts "Loading Shuck with #{root} on port #{options[:port]} with hostname #{hostname}"
       server = Shuck::Server.new(options[:port],store,hostname)
       server.serve

data/lib/shuck/file_store.rb

@@ -2,6 +2,7 @@ require 'fileutils'
 require 'time'
 require 'shuck/s3_object'
 require 'shuck/bucket'
+require 'shuck/rate_limitable_file'
 require 'digest/md5'
 require 'yaml'
 
@@ -21,6 +22,25 @@ module Shuck
       end
     end
 
+    # Pass a rate limit in bytes per second
+    def rate_limit=(rate_limit)
+      if rate_limit.is_a?(String)
+        if rate_limit =~ /^(\d+)$/
+          RateLimitableFile.rate_limit = rate_limit.to_i
+        elsif rate_limit =~ /^(.*)K$/
+          RateLimitableFile.rate_limit = $1.to_f * 1000
+        elsif rate_limit =~ /^(.*)M$/
+          RateLimitableFile.rate_limit = $1.to_f * 1000000
+        elsif rate_limit =~ /^(.*)G$/
+          RateLimitableFile.rate_limit = $1.to_f * 1000000000
+        else
+          raise "Invalid Rate Limit Format: Valid values include (1000,10K,1.1M)"
+        end
+      else
+        RateLimitableFile.rate_limit = nil
+      end
+    end
+
     def buckets
       @buckets
     end
@@ -37,14 +57,15 @@ module Shuck
     end
 
     def get_object(bucket,object, request)
-      begin 
+      begin
         real_obj = S3Object.new
         obj_root = File.join(@root,bucket,object,SHUCK_METADATA_DIR)
         metadata = YAML.parse(File.open(File.join(obj_root,"metadata"),'rb').read)
         real_obj.name = object
         real_obj.md5 = metadata[:md5].value
         real_obj.content_type = metadata[:content_type] ? metadata[:content_type].value : "application/octet-stream"
-        real_obj.io = File.open(File.join(obj_root,"content"),'rb')
+        #real_obj.io = File.open(File.join(obj_root,"content"),'rb')
+        real_obj.io = RateLimitableFile.open(File.join(obj_root,"content"),'rb')
         return real_obj
       rescue
         puts $!
@@ -55,6 +76,40 @@ module Shuck
     def object_metadata(bucket,object)
     end
 
+    def copy_object(src_bucket,src_object,dst_bucket,dst_object)
+      src_root = File.join(@root,src_bucket,src_object,SHUCK_METADATA_DIR)
+      src_obj = S3Object.new
+      src_metadata_filename = File.join(src_root,"metadata")
+      src_metadata = YAML.parse(File.open(src_metadata_filename,'rb').read)
+      src_content_filename = File.join(src_root,"content")
+
+      dst_filename= File.join(@root,dst_bucket,dst_object)
+      FileUtils.mkdir_p(dst_filename)
+
+      metadata_dir = File.join(dst_filename,SHUCK_METADATA_DIR)
+      FileUtils.mkdir_p(metadata_dir)
+
+      content = File.join(metadata_dir,"content")
+      metadata = File.join(metadata_dir,"metadata")
+
+      File.open(content,'wb') do |f|
+        File.open(src_content_filename,'rb') do |input|
+          f << input.read
+        end
+      end
+
+      File.open(metadata,'w') do |f|
+        File.open(src_metadata_filename,'r') do |input|
+          f << input.read
+        end
+      end
+
+      obj = S3Object.new
+      obj.md5 = src_metadata[:md5]
+      obj.content_type = src_metadata[:content_type]
+      return obj
+    end
+
     def store_object(bucket,object,request)
       begin
         filename = File.join(@root,bucket,object)
@@ -67,7 +122,8 @@ module Shuck
         metadata = File.join(filename,SHUCK_METADATA_DIR,"metadata")
 
         md5 = Digest::MD5.new
-        File.open(content,'w') do |f|
+
+        File.open(content,'wb') do |f|
           request.body do |chunk|
             f << chunk
             md5 << chunk

data/lib/shuck/rate_limitable_file.rb

@@ -0,0 +1,21 @@
+module Shuck
+  class RateLimitableFile < File
+    @@rate_limit = nil
+    # Specify a rate limit in bytes per second
+    def self.rate_limit
+      @@rate_limit
+    end
+
+    def self.rate_limit=(rate_limit)
+      @@rate_limit = rate_limit
+    end
+
+    def read(args)
+      if @@rate_limit
+        time_to_sleep = args / @@rate_limit
+        sleep(time_to_sleep)
+      end
+      return super(args)
+    end
+  end
+end

data/lib/shuck/server.rb

@@ -3,6 +3,28 @@ require 'shuck/file_store'
 require 'shuck/xml_adapter'
 
 module Shuck
+  class Request
+    CREATE_BUCKET = "CREATE_BUCKET"
+    STORE = "STORE"
+    COPY = "COPY"
+    GET = "GET"
+    MOVE = "MOVE"
+    DELETE = "DELETE"
+
+    attr_accessor :bucket,:object,:type,:src_bucket,:src_object,:method,:webrick_request,:path
+
+    def inspect
+      puts "-----Inspect Shuck Request"
+      puts "Type: #{@type}"
+      puts "Request Method: #{@method}"
+      puts "Bucket: #{@bucket}"
+      puts "Object: #{@object}"
+      puts "Src Bucket: #{@src_bucket}"
+      puts "Src Object: #{@src_object}"
+      puts "-----Done"
+    end
+  end
+
   class Servlet < WEBrick::HTTPServlet::AbstractServlet
     def initialize(server,store,hostname)
       super(server)
@@ -34,16 +56,22 @@ module Shuck
           elems = path.split("/")
         end
 
-        if elems.size == 1 
+        if elems.size == 0
+          # List buckets
+          buckets = @store.buckets
+          response.status = 200
+          response.body = XmlAdapter.buckets(buckets)
+          response['Content-Type'] = "application/xml"
+        elsif elems.size == 1
           bucket_obj = @store.get_bucket(bucket)
           if bucket_obj
             response.status = 200
             response.body = XmlAdapter.bucket(bucket_obj)
-            response['Content-Type'] = "application/xml" 
+            response['Content-Type'] = "application/xml"
           else
             response.status = 404
             response.body = XmlAdapter.error_no_such_bucket(bucket)
-            response['Content-Type'] = "application/xml" 
+            response['Content-Type'] = "application/xml"
           end
         else
           object = elems[1,elems.size].join('/')
@@ -88,61 +116,97 @@ module Shuck
     end
 
     def do_PUT(request,response)
-      path = request.path
+      s_req = normalize_request(request)
+
+      case s_req.type
+      when Request::COPY
+        @store.copy_object(s_req.src_bucket,s_req.src_object,s_req.bucket,s_req.object)
+      when Request::STORE
+        real_obj = @store.store_object(s_req.bucket,s_req.object,s_req.webrick_request)
+        response['Etag'] = real_obj.md5
+      when Request::CREATE_BUCKET
+        @store.create_bucket(s_req.bucket)
+      end
+
+      response.status = 200
+      response.body = ""
+      response['Content-Type'] = "text/xml"
+    end
+
+    def do_POST(request,response)
+      p request
+    end
+
+    def do_DELETE(request,response)
+      p request
+    end
+
+    private
+    # This method takes a webrick request and generates a normalized Shuck request
+    def normalize_request(webrick_r)
+      path = webrick_r.path
       path_len = path.size
       path_style_request = true
-      host = request["Host"]
-      puts host
+      host = webrick_r["Host"]
       bucket = nil
+      object = nil
       path_style_request = true
+
       if host != @hostname
         bucket = host.split(".")[0]
         path_style_request = false
       end
 
+      req = Request.new
+      req.path = webrick_r.path
+
       if path == "/"
         if bucket
-          @store.create_bucket(bucket)
-        else
-          puts "Unsure what to do"
+          req.type = Request::CREATE_BUCKET
         end
-      else 
+      else
         if path_style_request
           elems = path[1,path_len].split("/")
           bucket = elems[0]
-        else
-          elems = path.split("/")
-        end
-
-        if path_style_request
-          if elems.size == 1 
-            @store.create_bucket(bucket)
-          else 
+          if elems.size == 1
+            req.type = Request::CREATE_BUCKET
+          else
+            req.type = Request::STORE
             object = elems[1,elems.size].join('/')
-            real_obj = @store.store_object(bucket,object,request)
-            response['Etag'] = real_obj.md5
           end
         else
-          real_obj = @store.store_object(bucket,path,request)
-          response['Etag'] = real_obj.md5
+          req.type = Request::STORE
+          object = webrick_r.path
         end
       end
-      response.status = 200
-      response.body = ""
-      response['Content-Type'] = "text/xml" 
-    end
 
-    def do_POST(request,response)
-      puts "Put"
-      p request
+      copy_source = webrick_r.header["x-amz-copy-source"]
+      if copy_source and copy_source.size == 1
+        src_elems = copy_source.first.split("/")
+        root_offset = src_elems[0] == "" ? 1 : 0
+        req.src_bucket = src_elems[root_offset]
+        req.src_object = src_elems[1 + root_offset,src_elems.size].join("/")
+        req.type = Request::COPY
+      end
+
+      req.bucket = bucket
+      req.object = object
+      req.webrick_request = webrick_r
+      return req
     end
 
-    def do_DELETE(request,response)
-      puts "Delete"
-      p request
+    def dump_request(request)
+      puts "----------Dump Request-------------"
+      puts request.request_method
+      puts request.path
+      request.each do |k,v|
+        puts "#{k}:#{v}"
+      end
+      puts "----------End Dump -------------"
     end
   end
 
+
   class Server
     def initialize(port,store,hostname = "localhost")
       @port = port

data/lib/shuck/version.rb

@@ -1,3 +1,3 @@
 module Shuck
-  VERSION = "0.0.7"
+  VERSION = "0.0.8"
 end

data/lib/shuck/xml_adapter.rb

@@ -3,7 +3,7 @@ require 'time'
 
 module Shuck
   class XmlAdapter
-    def self.buckets(buckets)
+    def self.buckets(bucket_objects)
       output = ""
       xml = Builder::XmlMarkup.new(:target => output)
       xml.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
@@ -13,10 +13,10 @@ module Shuck
           owner.DisplayName("Shuck")
         }
         lam.Buckets { |buckets|
-          @buckets.each do |bucket|
+          bucket_objects.each do |bucket|
             buckets.Bucket do |b|
               b.Name(bucket.name)
-              b.CreationDate(bucket.creation_date.xmlschema)
+              b.CreationDate(bucket.creation_date.strftime("%Y-%m-%dT%H:%M:%S.000Z"))
             end
           end
         }

data/shuck.gemspec

@@ -17,7 +17,8 @@ Gem::Specification.new do |s|
   s.add_development_dependency "bundler", ">= 1.0.0"
   s.add_development_dependency "aws-s3"
   s.add_development_dependency "right_aws"
-  s.add_development_dependency "ruby-debug"
+  #s.add_development_dependency "aws-sdk"
+  #s.add_development_dependency "ruby-debug19"
   s.add_dependency "thor"
   s.add_dependency "builder"
 

data/test/local_s3_cfg

@@ -0,0 +1,34 @@
+[default]
+access_key = abc
+acl_public = False
+bucket_location = US
+cloudfront_host = cloudfront.amazonaws.com
+cloudfront_resource = /2008-06-30/distribution
+default_mime_type = binary/octet-stream
+delete_removed = False
+dry_run = False
+encoding = UTF-8
+encrypt = False
+force = False
+get_continue = False
+gpg_command = None
+gpg_decrypt = %(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
+gpg_encrypt = %(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
+gpg_passphrase = 
+guess_mime_type = True
+host_base = s3.localhost:10453
+host_bucket = %(bucket)s.s3.localhost:10453
+human_readable_sizes = False
+list_md5 = False
+preserve_attrs = True
+progress_meter = True
+proxy_host = 
+proxy_port = 0
+recursive = False
+recv_chunk = 4096
+secret_key = def
+send_chunk = 4096
+simpledb_host = sdb.amazonaws.com
+skip_existing = False
+use_https = False
+verbosity = WARNING

data/test/right_aws_commands_test.rb

@@ -6,7 +6,9 @@ require 'right_aws'
 class RightAWSCommandsTest < Test::Unit::TestCase
 
   def setup
-    @s3 = RightAws::S3Interface.new('1E3GDYEOGFJPIT7XXXXXX','hgTHt68JY07JKUY08ftHYtERkjgtfERn57XXXXXX', {:multi_thread => false, :server => 'localhost', :port => 10453, :protocol => 'http' })
+    @s3 = RightAws::S3Interface.new('1E3GDYEOGFJPIT7XXXXXX','hgTHt68JY07JKUY08ftHYtERkjgtfERn57XXXXXX',
+                                    {:multi_thread => false, :server => 'localhost',
+                                      :port => 10453, :protocol => 'http',:logger => Logger.new("/dev/null") })
   end
 
   def teardown
@@ -51,4 +53,11 @@ class RightAWSCommandsTest < Test::Unit::TestCase
     assert_equal "recursive", output
   end
 
+  def test_intra_bucket_copy
+    @s3.put("s3media","original.txt","Hello World")
+    @s3.copy("s3media","original.txt","s3media","copy.txt")
+    obj = @s3.get("s3media","copy.txt")
+    assert_equal "Hello World",obj[:object]
+  end
+
 end

data/test/s3cmd_test.rb

@@ -0,0 +1,30 @@
+require 'test/test_helper'
+require 'fileutils'
+require 'shuck/server'
+
+# You need to have s3cmd installed to use this
+class S3CmdTest < Test::Unit::TestCase
+
+  def setup
+    @s3cmd = "s3cmd --config"
+  end
+
+  def teardown
+  end
+
+  def test_create_bucket
+  end
+
+  def test_store
+  end
+
+  def test_large_store
+  end
+
+  def test_multi_directory
+  end
+
+  def test_intra_bucket_copy
+  end
+
+end

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 0
-  - 7
-  version: 0.0.7
+  - 8
+  version: 0.0.8
 platform: ruby
 authors: 
 - Curtis Spencer
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-06-06 00:00:00 -07:00
+date: 2012-02-22 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -59,7 +59,7 @@ dependencies:
   type: :development
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
-  name: ruby-debug
+  name: thor
   prerelease: false
   requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
@@ -69,10 +69,10 @@ dependencies:
         segments: 
         - 0
         version: "0"
-  type: :development
+  type: :runtime
   version_requirements: *id004
 - !ruby/object:Gem::Dependency 
-  name: thor
+  name: builder
   prerelease: false
   requirement: &id005 !ruby/object:Gem::Requirement 
     none: false
@@ -84,19 +84,6 @@ dependencies:
         version: "0"
   type: :runtime
   version_requirements: *id005
-- !ruby/object:Gem::Dependency 
-  name: builder
-  prerelease: false
-  requirement: &id006 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 0
-        version: "0"
-  type: :runtime
-  version_requirements: *id006
 description: Use Shuck to test basic S3 functionality without actually connecting to S3
 email: 
 - curtis@sevenforge.com
@@ -117,13 +104,16 @@ files:
 - lib/shuck/bucket.rb
 - lib/shuck/cli.rb
 - lib/shuck/file_store.rb
+- lib/shuck/rate_limitable_file.rb
 - lib/shuck/s3_object.rb
 - lib/shuck/server.rb
 - lib/shuck/version.rb
 - lib/shuck/xml_adapter.rb
 - shuck.gemspec
+- test/local_s3_cfg
 - test/right_aws_commands_test.rb
 - test/s3_commands_test.rb
+- test/s3cmd_test.rb
 - test/test_helper.rb
 has_rdoc: true
 homepage: ""
@@ -158,6 +148,8 @@ signing_key:
 specification_version: 3
 summary: Shuck is a phony S3 engine with minimal dependencies
 test_files: 
+- test/local_s3_cfg
 - test/right_aws_commands_test.rb
 - test/s3_commands_test.rb
+- test/s3cmd_test.rb
 - test/test_helper.rb