shuck 0.0.8 → 0.0.9

data/lib/shuck/cli.rb

@@ -8,7 +8,7 @@ module Shuck
     desc "server", "Run a server on a particular hostname"
     method_option :root, :type => :string, :aliases => '-r', :required => true
     method_option :port, :type => :numeric, :aliases => '-p', :required => true
-    method_option :hostname, :type => :string, :aliases => '-h', :desc => "The root name of the host.  Defaults to localhost."
+    method_option :hostname, :type => :string, :aliases => '-h', :desc => "The root name of the host.  Defaults to s3.amazonaws.com."
     method_option :limit, :aliases => '-l', :type => :string, :desc => 'Rate limit for serving (ie. 50K, 1.0M)'
     def server
       store = nil
@@ -25,6 +25,10 @@ module Shuck
       hostname = 's3.amazonaws.com'
       if options[:hostname]
         hostname = options[:hostname]
+        # In case the user has put a port on the hostname
+        if hostname =~ /:(\d+)/
+          hostname = hostname.split(":")[0]
+        end
       end
 
       if options[:limit]

data/lib/shuck/file_store.rb

@@ -52,8 +52,10 @@ module Shuck
     def create_bucket(bucket)
       FileUtils.mkdir_p(File.join(@root,bucket))
       bucket_obj = Bucket.new(bucket,Time.now,[])
-      @buckets << bucket_obj
-      @bucket_hash[bucket] = bucket_obj
+      if !@bucket_hash[bucket]
+        @buckets << bucket_obj
+        @bucket_hash[bucket] = bucket_obj
+      end
     end
 
     def get_object(bucket,object, request)

data/lib/shuck/server.rb

@@ -5,17 +5,22 @@ require 'shuck/xml_adapter'
 module Shuck
   class Request
     CREATE_BUCKET = "CREATE_BUCKET"
+    LIST_BUCKETS = "LIST_BUCKETS"
+    LS_BUCKET = "LS_BUCKET"
     STORE = "STORE"
     COPY = "COPY"
     GET = "GET"
+    GET_ACL = "GET_ACL"
+    SET_ACL = "SET_ACL"
     MOVE = "MOVE"
     DELETE = "DELETE"
 
-    attr_accessor :bucket,:object,:type,:src_bucket,:src_object,:method,:webrick_request,:path
+    attr_accessor :bucket,:object,:type,:src_bucket,:src_object,:method,:webrick_request,:path,:is_path_style
 
     def inspect
       puts "-----Inspect Shuck Request"
       puts "Type: #{@type}"
+      puts "Is Path Style: #{@is_path_style}"
       puts "Request Method: #{@method}"
       puts "Bucket: #{@bucket}"
       puts "Object: #{@object}"
@@ -30,94 +35,77 @@ module Shuck
       super(server)
       @store = store
       @hostname = hostname
+      @root_hostnames = [hostname,'localhost','s3.amazonaws.com','s3.localhost']
     end
 
     def do_GET(request, response)
-      path = request.path
-      path_len = path.size
-      host = request["HOST"]
-      path_style_request = true
-      bucket = nil
-      if host != @hostname
-        bucket = host.split(".")[0]
-        path_style_request = false
-      end
+      s_req = normalize_request(request)
 
-      if path == "/" and path_style_request
+      case s_req.type
+      when 'LIST_BUCKETS'
         response.status = 200
-        response['Content-Type'] = 'text/xml'
+        response['Content-Type'] = 'application/xml'
         buckets = @store.buckets
         response.body = XmlAdapter.buckets(buckets)
-      else
-        if path_style_request
-          elems = path[1,path_len].split("/")
-          bucket = elems[0]
-        else
-          elems = path.split("/")
-        end
-
-        if elems.size == 0
-          # List buckets
-          buckets = @store.buckets
+      when 'LS_BUCKET'
+        bucket_obj = @store.get_bucket(s_req.bucket)
+        if bucket_obj
           response.status = 200
-          response.body = XmlAdapter.buckets(buckets)
+          response.body = XmlAdapter.bucket(bucket_obj)
           response['Content-Type'] = "application/xml"
-        elsif elems.size == 1
-          bucket_obj = @store.get_bucket(bucket)
-          if bucket_obj
-            response.status = 200
-            response.body = XmlAdapter.bucket(bucket_obj)
-            response['Content-Type'] = "application/xml"
-          else
-            response.status = 404
-            response.body = XmlAdapter.error_no_such_bucket(bucket)
-            response['Content-Type'] = "application/xml"
-          end
         else
-          object = elems[1,elems.size].join('/')
-          real_obj = @store.get_object(bucket,object, request)
-
-          if real_obj
-            response.status = 200
-            response['Content-Type'] = real_obj.content_type
-            content_length = File::Stat.new(real_obj.io.path).size
-            response['Etag'] = real_obj.md5
-            response['Accept-Ranges'] = "bytes"
-
-            # Added Range Query support
-            if range = request.header["range"].first
-              response.status = 206
-              if range =~ /bytes=(\d*)-(\d*)/
-                start = $1.to_i
-                finish = $2.to_i
-                finish_str = ""
-                if finish == 0
-                  finish = content_length - 1
-                  finish_str = "#{finish}"
-                else
-                  finish_str = finish.to_s
-                end
-
-                bytes_to_read = finish - start + 1
-                response['Content-Range'] = "bytes #{start}-#{finish_str}/#{content_length}"
-                real_obj.io.pos = start
-                response.body = real_obj.io.read(bytes_to_read)
-                return
-              end
+          response.status = 404
+          response.body = XmlAdapter.error_no_such_bucket(s_req.bucket)
+          response['Content-Type'] = "application/xml"
+        end
+      when 'GET_ACL'
+        response.status = 200
+        response.body = XmlAdapter.acl()
+        response['Content-Type'] = 'application/xml'
+      when 'GET'
+        real_obj = @store.get_object(s_req.bucket,s_req.object,request)
+        if !real_obj
+          response.status = 404
+          response.body = ""
+          return
+        end
+
+        response.status = 200
+        response['Content-Type'] = real_obj.content_type
+        content_length = File::Stat.new(real_obj.io.path).size
+        response['Etag'] = real_obj.md5
+        response['Accept-Ranges'] = "bytes"
+
+        # Added Range Query support
+        if range = request.header["range"].first
+          response.status = 206
+          if range =~ /bytes=(\d*)-(\d*)/
+            start = $1.to_i
+            finish = $2.to_i
+            finish_str = ""
+            if finish == 0
+              finish = content_length - 1
+              finish_str = "#{finish}"
+            else
+              finish_str = finish.to_s
             end
-            response['Content-Length'] = File::Stat.new(real_obj.io.path).size
-            response.body = real_obj.io
-          else
-            response.status = 404
-            response.body = ""
+
+            bytes_to_read = finish - start + 1
+            response['Content-Range'] = "bytes #{start}-#{finish_str}/#{content_length}"
+            real_obj.io.pos = start
+            response.body = real_obj.io.read(bytes_to_read)
+            return
           end
         end
+        response['Content-Length'] = File::Stat.new(real_obj.io.path).size
+        response.body = real_obj.io
       end
     end
 
     def do_PUT(request,response)
       s_req = normalize_request(request)
 
+
       case s_req.type
       when Request::COPY
         @store.copy_object(s_req.src_bucket,s_req.src_object,s_req.bucket,s_req.object)
@@ -142,57 +130,105 @@ module Shuck
     end
 
     private
-    # This method takes a webrick request and generates a normalized Shuck request
-    def normalize_request(webrick_r)
-      path = webrick_r.path
+
+    def normalize_get(webrick_req,s_req)
+      path = webrick_req.path
       path_len = path.size
-      path_style_request = true
-      host = webrick_r["Host"]
-      bucket = nil
-      object = nil
-      path_style_request = true
-
-      if host != @hostname
-        bucket = host.split(".")[0]
-        path_style_request = false
-      end
+      query = webrick_req.query
+      if path == "/" and s_req.is_path_style
+        s_req.type = Request::LIST_BUCKETS
+      else
+        if s_req.is_path_style
+          elems = path[1,path_len].split("/")
+          s_req.bucket = elems[0]
+        else
+          elems = path.split("/")
+        end
 
-      req = Request.new
-      req.path = webrick_r.path
+        if elems.size == 0
+          # List buckets
+          s_req.type = Request::LIST_BUCKETS
+        elsif elems.size == 1
+          s_req.type = Request::LS_BUCKET
+        else
+          if query["acl"] == ""
+            s_req.type = Request::GET_ACL
+          else
+            s_req.type = Request::GET
+          end
+          object = elems[1,elems.size].join('/')
+          s_req.object = object
+        end
+      end
+    end
 
+    def normalize_put(webrick_req,s_req)
+      path = webrick_req.path
+      path_len = path.size
       if path == "/"
-        if bucket
-          req.type = Request::CREATE_BUCKET
+        if s_req.bucket
+          s_req.type = Request::CREATE_BUCKET
         end
       else
-        if path_style_request
+        if s_req.is_path_style
           elems = path[1,path_len].split("/")
-          bucket = elems[0]
+          s_req.bucket = elems[0]
           if elems.size == 1
-            req.type = Request::CREATE_BUCKET
+            s_req.type = Request::CREATE_BUCKET
           else
-            req.type = Request::STORE
-            object = elems[1,elems.size].join('/')
+            if webrick_req.request_line =~ /\?acl/
+              s_req.type = Request::SET_ACL
+            else
+              s_req.type = Request::STORE
+            end
+            s_req.object = elems[1,elems.size].join('/')
           end
         else
-          req.type = Request::STORE
-          object = webrick_r.path
+          if webrick_req.request_line =~ /\?acl/
+            s_req.type = Request::SET_ACL
+          else
+            s_req.type = Request::STORE
+          end
+          s_req.object = webrick_req.path
         end
       end
 
-      copy_source = webrick_r.header["x-amz-copy-source"]
+      copy_source = webrick_req.header["x-amz-copy-source"]
       if copy_source and copy_source.size == 1
         src_elems = copy_source.first.split("/")
         root_offset = src_elems[0] == "" ? 1 : 0
-        req.src_bucket = src_elems[root_offset]
-        req.src_object = src_elems[1 + root_offset,src_elems.size].join("/")
-        req.type = Request::COPY
+        s_req.src_bucket = src_elems[root_offset]
+        s_req.src_object = src_elems[1 + root_offset,src_elems.size].join("/")
+        s_req.type = Request::COPY
+      end
+
+      s_req.webrick_request = webrick_req
+    end
+
+    # This method takes a webrick request and generates a normalized Shuck request
+    def normalize_request(webrick_req)
+      host_header= webrick_req["Host"]
+      host = host_header.split(':')[0]
+
+      s_req = Request.new
+      s_req.path = webrick_req.path
+      s_req.is_path_style = true
+
+      if !@root_hostnames.include?(host)
+        s_req.bucket = host.split(".")[0]
+        s_req.is_path_style = false
+      end
+
+      case webrick_req.request_method
+      when 'PUT'
+        normalize_put(webrick_req,s_req)
+      when 'GET'
+        normalize_get(webrick_req,s_req)
+      else
+        raise "Unknown Request"
       end
 
-      req.bucket = bucket
-      req.object = object
-      req.webrick_request = webrick_r
-      return req
+      return s_req
     end
 
     def dump_request(request)
@@ -208,7 +244,7 @@ module Shuck
 
 
   class Server
-    def initialize(port,store,hostname = "localhost")
+    def initialize(port,store,hostname)
       @port = port
       @store = store
       @hostname = hostname

data/lib/shuck/version.rb

@@ -1,3 +1,3 @@
 module Shuck
-  VERSION = "0.0.8"
+  VERSION = "0.0.9"
 end

data/lib/shuck/xml_adapter.rb

@@ -44,6 +44,20 @@ module Shuck
       output
     end
 
+    def self.error_no_such_key(name)
+      output = ""
+      xml = Builder::XmlMarkup.new(:target => output)
+      xml.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+      xml.Error { |err|
+        err.Code("NoSuchKey")
+        err.Message("The specified key does not exist")
+        err.Key(name)
+        err.RequestId(1)
+        err.HostId(2)
+      }
+      output
+    end
+
     def self.bucket(bucket)
       output = ""
       xml = Builder::XmlMarkup.new(:target => output)
@@ -58,5 +72,27 @@ module Shuck
       output
     end
 
+    # ACL xml
+    def self.acl(object = nil)
+      output = ""
+      xml = Builder::XmlMarkup.new(:target => output)
+      xml.instruct! :xml, :version=>"1.0", :encoding=>"UTF-8"
+      xml.AccessControlPolicy(:xmlns => "http://s3.amazonaws.com/doc/2006-03-01/") { |acp|
+        acp.Owner do |owner|
+          owner.ID("abc")
+          owner.DisplayName("You")
+        end
+        acp.AccessControlList do |acl|
+          acl.Grant do |grant|
+            grant.Grantee("xmlns:xsi" => "http://www.w3.org/2001/XMLSchema-instance", "xsi:type" => "CanonicalUser") do |grantee|
+              grantee.ID("abc")
+              grantee.DisplayName("You")
+            end
+            grant.Permission("FULL_CONTROL")
+          end
+        end
+      }
+      output
+    end
   end
 end

data/test/local_s3_cfg

@@ -16,8 +16,8 @@ gpg_decrypt = %(gpg_command)s -d --verbose --no-use-agent --batch --yes --passph
 gpg_encrypt = %(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
 gpg_passphrase = 
 guess_mime_type = True
-host_base = s3.localhost:10453
-host_bucket = %(bucket)s.s3.localhost:10453
+host_base = localhost:10453
+host_bucket = %(bucket)s.localhost:10453
 human_readable_sizes = False
 list_md5 = False
 preserve_attrs = True

data/test/s3cmd_test.rb

@@ -6,16 +6,41 @@ require 'shuck/server'
 class S3CmdTest < Test::Unit::TestCase
 
   def setup
-    @s3cmd = "s3cmd --config"
+    config = File.expand_path(File.join(File.dirname(__FILE__),'local_s3_cfg'))
+    @s3cmd = "s3cmd --config #{config}"
   end
 
   def teardown
   end
 
   def test_create_bucket
+    `#{@s3cmd} mb s3://s3cmd_bucket`
+    output = `#{@s3cmd} ls`
+    assert_match(/s3cmd_bucket/,output)
   end
 
   def test_store
+    File.open(__FILE__,'rb') do |input|
+      File.open("/tmp/shuck_upload",'wb') do |output|
+        output << input.read
+      end
+    end
+    output = `#{@s3cmd} put /tmp/shuck_upload s3://s3cmd_bucket/upload`
+    assert_match(/stored/,output)
+
+    FileUtils.rm("/tmp/shuck_upload")
+  end
+
+  def test_acl
+    File.open(__FILE__,'rb') do |input|
+      File.open("/tmp/shuck_acl_upload",'wb') do |output|
+        output << input.read
+      end
+    end
+    output = `#{@s3cmd} put /tmp/shuck_acl_upload s3://s3cmd_bucket/acl_upload`
+    assert_match(/stored/,output)
+
+    output = `#{@s3cmd} --force setacl -P s3://s3cmd_bucket/acl_upload`
   end
 
   def test_large_store

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 0
-  - 8
-  version: 0.0.8
+  - 9
+  version: 0.0.9
 platform: ruby
 authors: 
 - Curtis Spencer
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-02-22 00:00:00 -08:00
+date: 2012-02-23 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency