blobsterix 0.0.14 → 0.0.19

data/lib/blobsterix/s3/s3_api.rb

@@ -1,6 +1,7 @@
 module Blobsterix
   class S3Api < AppRouterBase
     include S3UrlHelper
+    include UrlHelper
 
     get "/", :list_buckets
 
@@ -26,12 +27,20 @@ module Blobsterix
     post "*any", :next_api
 
     private
+      def check_auth
+        return true unless Blobsterix.secret_key_store
+        Blobsterix::S3Auth.authenticate(env).check(Blobsterix.secret_key_store)
+      end
+
       def list_buckets
-         Blobsterix.event("s3_api.list_bucket",:bucket => bucket)
-        Http.OK storage.list(bucket).to_xml, "xml"
+        return Http.NotAuthorized unless check_auth
+        Blobsterix.event("s3_api.list_bucket",:bucket => bucket)
+        start_path = env["params"]["marker"] if env["params"]
+        Http.OK storage.list(bucket, :start_path => start_path).to_xml, "xml"
       end
 
       def get_file(send_with_data=true)
+        return Http.NotAuthorized unless check_auth
         return Http.NotFound if favicon
 
         if bucket?
@@ -41,25 +50,28 @@ module Blobsterix
             Http.NotFound
           end
         else
-          Http.OK storage.list(bucket).to_xml, "xml"
+          list_buckets
         end
       end
 
       def get_file_head
-                           #TODO: add event?
+        return Http.NotAuthorized unless check_auth
+        #TODO: add event?
         get_file(false)
       end
 
       def create_bucket
-              Blobsterix.event("s3_api.upload",:bucket => bucket)
+        return Http.NotAuthorized unless check_auth
+        Blobsterix.event("s3_api.upload",:bucket => bucket)
         Http.OK storage.create(bucket), "xml"
       end
 
       def upload_data
+        return Http.NotAuthorized unless check_auth
         source = cached_upload
         accept = AcceptType.new("*/*")#source.accept_type()
 
-        trafo_current = trafo(trafo_string)
+        trafo_current = trafo(transformation_string)
         file_current = file
         bucket_current = bucket
         Blobsterix.event("s3_api.upload", :bucket => bucket_current, 
@@ -71,6 +83,7 @@ module Blobsterix
       end
 
       def delete_bucket
+        return Http.NotAuthorized unless check_auth
         Blobsterix.event("s3_api.delete_bucket", :bucket => bucket)
 
         if bucket?
@@ -81,6 +94,7 @@ module Blobsterix
       end
 
       def delete_file
+        return Http.NotAuthorized unless check_auth
          Blobsterix.event("s3_api.delete_file", :bucket => bucket,:file => file)
         if bucket?
           Http.OK_no_data storage.delete_key(bucket, file), "xml"

data/lib/blobsterix/s3/s3_auth.rb

@@ -0,0 +1,32 @@
+module Blobsterix
+  module S3Auth
+
+    VERSIONS = [
+      V2,
+      V2Query,
+      V4
+    ]
+
+    def self.authenticate(env)
+      VERSIONS.each do |version|
+        v = version.create(env)
+        return v if v
+      end
+      NoAuth.new
+    end
+
+    def self.current_time
+      (@current_time||=lambda{Time.now}).call
+    end
+
+    def self.current_time=(obj)
+      @current_time=obj
+    end
+
+    class NoAuth
+      def check(secret)
+        false
+      end
+    end
+  end
+end

data/lib/blobsterix/s3/s3_auth_key_store.rb

@@ -0,0 +1,14 @@
+module Blobsterix
+  module S3Auth
+    class KeyStore
+      def initialize(keys={})
+        @keys = keys
+        @keys.default=""
+      end
+      # if no key is found it should return an empty string
+      def get_key(id)
+        @keys[id]
+      end
+    end
+  end
+end

data/lib/blobsterix/s3/s3_auth_v2.rb

@@ -0,0 +1,62 @@
+module Blobsterix
+  module S3Auth
+    class V2
+      include ::Blobsterix::S3UrlHelper
+      include ::Blobsterix::UrlHelper
+      include ::Blobsterix::Logable
+      include V2Helper
+
+      # these should actually be used when calculating
+      # the signature when suplied as query parameters, 
+      # but they are not needed in this usecase
+      SUBRESOURCES = [
+        "acl",
+        "lifecycle",
+        "location",
+        "logging",
+        "notification",
+        "partNumber",
+        "policy",
+        "requestPayment",
+        "torrent",
+        "uploadId",
+        "uploads",
+        "versionId",
+        "versioning",
+        "versions",
+        "website"
+      ]
+
+      V2_REGEX = /AWS (\w+):(.+)/
+
+      def self.create(env)
+        auth_string = env["HTTP_AUTHORIZATION"]
+        matcher = V2_REGEX.match(auth_string)
+        matcher ? V2.new(env, matcher[1], matcher[2]) : nil
+      end
+
+      attr_reader :env, :access_key, :signature
+      def initialize(env, access_key, signature)
+        @env = env
+        @access_key = access_key
+        @signature = signature
+      end
+
+      def time_to_check
+        env["HTTP_X_AMZ_DATE"]||env["HTTP_DATE"]
+      end
+
+      def is_expired?
+        ::Blobsterix::S3Auth.current_time-Time.parse(time_to_check) > 15*60
+      end
+
+      def time_of_request
+        env["HTTP_DATE"] unless env["HTTP_X_AMZ_DATE"]
+      end
+
+      def server_signature(secret_key, str)
+        gen_signature(secret_key, str)
+      end
+    end
+  end
+end

data/lib/blobsterix/s3/s3_auth_v2_helper.rb

@@ -0,0 +1,42 @@
+module Blobsterix
+  module S3Auth
+    module V2Helper
+
+      def canonicalized_amz_headers
+        amz_headers = env.select{|key,value|
+          /HTTP_X_AMZ/i.match(key) if key.is_a?(String)
+        }.sort.map{|key,value|
+          "#{key.gsub("HTTP_","").downcase.gsub("_","-")}:#{value}"
+        }
+        return amz_headers.join("\n")+"\n" if amz_headers.length > 0
+        ""
+      end
+
+      def canonicalized_resource(escape=false)
+        return "/" if bucket == "root"
+        escape ? "/#{bucket}/#{CGI.escape(file)}" : "/#{bucket}/#{file}"
+      end
+
+      def string_to_sign(escape=false)
+        "#{env["REQUEST_METHOD"]}\n"+
+        "#{env["HTTP_CONTENT_MD5"]||""}\n"+
+        "#{env["CONTENT_TYPE"]||""}\n"+
+        "#{time_of_request}\n"+
+        "#{canonicalized_amz_headers}#{canonicalized_resource(escape)}"
+      end
+
+      def check(secret_key_store)
+        return false if is_expired?
+        # logger.info string_to_sign
+        own_key_0 = server_signature(secret_key_store.get_key(access_key), string_to_sign)
+        own_key_1 = server_signature(secret_key_store.get_key(access_key), string_to_sign(true))
+        # logger.info "[#{@signature}] == [#{own_key_0}] OR [#{@signature}] == [#{own_key_1}]"
+        @signature == own_key_0 || @signature == own_key_1
+      end
+
+      def gen_signature(secret_key, str)
+        Base64.encode64( OpenSSL::HMAC.digest("sha1", secret_key, str)).chop
+      end
+    end
+  end
+end

data/lib/blobsterix/s3/s3_auth_v2_query.rb

@@ -0,0 +1,37 @@
+module Blobsterix
+  module S3Auth
+    class V2Query
+      include ::Blobsterix::S3UrlHelper
+      include ::Blobsterix::UrlHelper
+      include ::Blobsterix::Logable
+      include V2Helper
+
+      def self.create(env)
+        return nil unless env["params"] && env["params"]["AWSAccessKeyId"] && env["params"]["Signature"]
+        V2Query.new(env, env["params"]["AWSAccessKeyId"], env["params"]["Signature"], env["params"]["Expires"])
+      end
+
+      attr_reader :env, :access_key, :signature, :expires
+      def initialize(env, access_key, signature, expires)
+        @env = env
+        @access_key = access_key
+        @signature = signature
+        @expires = expires
+      end
+
+      def time_of_request
+        expires
+      end
+
+      def is_expired?
+        return false unless expires
+        ::Blobsterix::S3Auth.current_time>Time.at(expires.to_i)
+      end
+
+      def server_signature(secret_key, str)
+        # URI::encode(gen_signature(secret_key, str))
+        gen_signature(secret_key, str)
+      end
+    end
+  end
+end

data/lib/blobsterix/s3/s3_auth_v4.rb

@@ -0,0 +1,33 @@
+module Blobsterix
+  module S3Auth
+    class V4
+      include ::Blobsterix::S3UrlHelper
+      include ::Blobsterix::UrlHelper
+      
+
+      V4_REGEX = /AWS4-HMAC-SHA256 Credential=(\w+\/\d+\/.+\/\w+\/aws4_request),.*SignedHeaders=(.+),.*Signature=(\w+)/
+
+      def self.create(env)
+        auth_string = env["HTTP_AUTHORIZATION"]
+        matcher = V4_REGEX.match(auth_string)
+        matcher ? V4.new(env, matcher[1], matcher[2], matcher[3]) : nil
+      end
+
+      attr_reader :env, :credential, :signed_headers, :signature
+      def initialize(env, credential, signed_headers, signature)
+        @env = env
+        @credential = credential
+        @signed_headers = signed_headers
+        @signature = signature
+      end
+      def getSignatureKey(key, dateStamp, regionName, serviceName)
+        kDate    = OpenSSL::HMAC.digest('sha256', "AWS4" + key, dateStamp)
+        kRegion  = OpenSSL::HMAC.digest('sha256', kDate, regionName)
+        kService = OpenSSL::HMAC.digest('sha256', kRegion, serviceName)
+        kSigning = OpenSSL::HMAC.digest('sha256', kService, "aws4_request")
+
+        kSigning
+      end
+    end
+  end
+end

data/lib/blobsterix/s3/s3_url_helper.rb

@@ -9,10 +9,6 @@ module Blobsterix
       end
     end
 
-    def favicon
-      @favicon ||= file.match /favicon/
-    end
-
     def cache_upload
       cache.put_stream(cache_upload_key, env['rack.input'])
     end
@@ -30,20 +26,9 @@ module Blobsterix
       @cache_upload_key ||= Blobsterix::BlobAccess.new(:bucket => bucket, :id => "upload_#{file.gsub("/", "_")}")
     end
 
-    def trafo_string
+    def transformation_string
       @trafo ||= env["HTTP_X_AMZ_META_TRAFO"] || ""
     end
-
-    #TransformationCommand
-    def trafo(trafo_s='')
-      trafo_a = []
-      trafo_s.split(",").each{|command|
-        parts = command.split("_")
-        key = parts.delete_at(0)
-        trafo_a << [key, parts.join("_")]
-      }
-      trafo_a
-    end
     
     def bucket
       host = bucket_matcher(env['HTTP_HOST'])
@@ -66,27 +51,5 @@ module Blobsterix
       host = bucket_matcher(env['HTTP_HOST'])
       host || env[nil][:bucket] || included_bucket
     end
-
-    def format
-      @format ||= env[nil][:format]
-    end
-
-    def included_bucket
-      if env[nil][:bucket_or_file] && env[nil][:bucket_or_file].include?("/")
-        env[nil][:bucket] = env[nil][:bucket_or_file].split("/")[0]
-        env[nil][:bucket_or_file] = env[nil][:bucket_or_file].gsub("#{env[nil][:bucket]}/", "")
-        true
-      else
-        false
-      end
-    end
-
-    def file
-      if format
-        [env[nil][:file] || env[nil][:bucket_or_file] || "", format].join(".")
-      else
-        env[nil][:file] || env[nil][:bucket_or_file] || ""
-      end
-    end
   end
 end

data/lib/blobsterix/service.rb

@@ -1,6 +1,7 @@
 module Blobsterix
   class Service < Goliath::API
     use Goliath::Rack::Params
+    # use SudiMiddleWare
     include Logable
 =begin
     def on_headers(env, headers)
@@ -17,16 +18,9 @@ module Blobsterix
       env.logger.info 'closing connection'
     end
 =end
-    def get_request_id
-      @request_id||=0
-      @request_id+=1
-    end
     def response(env)
-      env["BLOBSTERIX_REQUEST_ID"] = get_request_id
-      logger.info "RAM USAGE Before[#{Process.pid}]: " + `pmap #{Process.pid} | tail -1`[10,40].strip
-      a = call_stack(env, BlobApi, StatusApi, S3Api)
-      logger.info "RAM USAGE After[#{Process.pid}]: " + `pmap #{Process.pid} | tail -1`[10,40].strip
-      a
+      # env["params"] = params
+      call_stack(env, BlobApi, StatusApi, S3Api)
     end
 
     def call_stack(env, *apis)

data/lib/blobsterix/storage/bucket.rb

@@ -1,11 +1,12 @@
 module Blobsterix
   module Storage
     class Bucket
-      attr_accessor :name, :creation_date, :contents
+      attr_accessor :name, :creation_date, :contents, :truncated, :next_marker, :marker
       def initialize(name, date)
         @name = name
         @creation_date = date
         @contents = []
+        @truncated = false
         yield self if block_given?
       end
       def to_xml()
@@ -14,9 +15,11 @@ module Blobsterix
             xml.ListBucketResult(:xmlns => "http://doc.s3.amazonaws.com/#{date.year}-#{date.month}-#{date.day}") {
               xml.Name name
               xml.Prefix
-              xml.Marker
+              xml.Marker marker
+              xml.NextMarker next_marker
               xml.MaxKeys 1000
-              xml.IsTruncated false
+              xml.KeyCount contents.length
+              xml.IsTruncated truncated
                 contents.each{|entry|
                   entry.insert_xml(xml)
                 }

data/lib/blobsterix/storage/bucket_entry.rb

@@ -2,6 +2,16 @@ module Blobsterix
   module Storage
     class BucketEntry
       attr_accessor :key, :last_modified, :etag, :size, :storage_class, :mimetype, :fullpath
+
+      def self.create(file, meta)
+        BucketEntry.new(file.to_s.gsub("\\", "/")) do |entry|
+          entry.last_modified =  meta.last_modified.strftime("%Y-%m-%dT%H:%M:%S.000Z")
+          entry.etag =  meta.etag
+          entry.size =  meta.size
+          entry.mimetype = meta.mimetype
+        end
+      end
+      
       def initialize(key)
         @key = key
         @last_modified = "2009-10-12T17:50:30.000Z"
@@ -21,7 +31,7 @@ module Blobsterix
           xml.Size size
           xml.StorageClass storage_class
           xml.MimeType mimetype
-          xml.FullPath fullpath
+          # xml.FullPath fullpath
         }
       end
     end

data/lib/blobsterix/storage/cache.rb

@@ -4,15 +4,16 @@ module Blobsterix
       include Blobsterix::Logable
 
       def invalidation
-        Blobsterix.wait_for(Proc.new {
-          each_meta_file do |meta_file|
-            blob_access=meta_to_blob_access(meta_file)
-            if Blobsterix.cache_checker.call(blob_access,meta_file.last_accessed,meta_file.last_modified)
-              invalidate(blob_access, true)
-            end
+        each_meta_file do |meta_file|
+          blob_access = Blobsterix::SimpleProxy.new(Proc.new {
+            meta_to_blob_access(FileSystemMetaData.new(meta_file))
+          })
+          if Blobsterix.cache_checker.call(blob_access,File.atime(meta_file),File.ctime(meta_file))
+            invalidate(blob_access, true)
           end
-        })
+        end
       end
+
       def initialize(path)
         @path = Pathname.new(path)
         FileUtils.mkdir_p(@path) if !Dir.exist?(@path)
@@ -68,10 +69,10 @@ module Blobsterix
       private
 
       def each_meta_file
-        Dir.glob(@path.join("**/*")).each {|file|
-          cache_file = Pathname.new file
-          if block_given? && !cache_file.to_s.match(/\.meta$/) && !cache_file.directory?
-            yield FileSystemMetaData.new(cache_file)
+        Blobsterix::DirectoryList.each(@path) {|file_path, file|
+          cache_file = file_path.join file
+          if block_given? && !cache_file.to_s.match(/\.meta$/)
+            yield cache_file
             cache_file
           end
         }
@@ -98,16 +99,7 @@ module Blobsterix
       end
 
       def hash_filename(filename)
-        hash = Murmur.Hash64B(filename)
-        bits =  hash.to_s(2)
-        parts = []
-        6.times { |index|
-          len = 11
-          len = bits.length if len >= bits.length
-          value = bits.slice!(0, len).to_i(2).to_s(16).rjust(3,"0")
-          parts.push(value)
-        }
-        parts.join("/")
+        Murmur.map_filename(filename)
       end
     end
   end

data/lib/blobsterix/storage/file_system.rb

@@ -1,4 +1,5 @@
 require 'fileutils'
+require 'benchmark'
 
 module Blobsterix
   module Storage
@@ -15,29 +16,15 @@ module Blobsterix
         Dir.entries(contents).include?(bucket) and File.directory?(File.join(contents,bucket))
       end
 
-      def list(bucket="root")
+      def list(bucket="root", opts={})
         if bucket =~ /root/
-          BucketList.new do |l|
-
-            Dir.entries("#{contents}").each{|dir|
-              l.buckets << Bucket.new(dir, time_string_of(dir)) if File.directory? File.join("#{contents}",dir) and !(dir =='.' || dir == '..') 
-            }
-          end
+          list_buckets
         else
           if bucket_exist(bucket)
             b = Bucket.new(bucket, time_string_of(bucket))
-            files = bucket_files(bucket)
+            b.marker = opts[:start_path] if opts[:start_path]
             Blobsterix.wait_for(Proc.new {
-              files.each do |file|
-                b.contents << BucketEntry.new(file) do |entry|
-                  meta = metaData(bucket, file)
-                  entry.last_modified =  meta.last_modified.strftime("%Y-%m-%dT%H:%M:%S.000Z")
-                  entry.etag =  meta.etag
-                  entry.size =  meta.size
-                  entry.mimetype = meta.mimetype
-                  entry.fullpath = contents(bucket, file).gsub("#{contents}/", "")
-                end
-              end
+              collect_bucket_entries(bucket, b, opts)
             })
             b
           else
@@ -80,7 +67,7 @@ module Blobsterix
 
       def delete(bucket)
         logger.info "Storage: delete bucket #{contents(bucket)}"
-        FileUtils.rm_rf(contents(bucket)) if bucket_exist(bucket) && bucket_files(bucket).empty?
+        FileUtils.rm_rf(contents(bucket)) if bucket_exist(bucket) && bucket_empty?(bucket)
         #Dir.rmdir(contents(bucket)) if bucket_exist(bucket) && bucket_files(bucket).empty?
       end
 
@@ -92,6 +79,29 @@ module Blobsterix
       end
 
       private
+        def list_buckets
+          BucketList.new do |l|
+            Dir.entries("#{contents}").each{|dir|
+              l.buckets << Bucket.new(dir, time_string_of(dir)) if File.directory? File.join("#{contents}",dir) and !(dir =='.' || dir == '..') 
+            }
+          end
+        end
+        def collect_bucket_entries(bucket, b, opts)
+          start_path = map_filename(opts[:start_path].to_s.gsub("/", "\\")) if opts[:start_path]
+          current_obj = Blobsterix::DirectoryList.each_limit(contents(bucket), :limit => 20, :start_path => start_path) do |path, file|
+            if file.to_s.end_with?(".meta")
+              false
+            else
+              b.contents << BucketEntry.create(file, metaData(bucket, file.to_s))
+              true
+            end
+          end
+          next_marker = current_obj.current_file.to_s.gsub("\\", "/")
+          if current_obj.next
+            b.next_marker=next_marker
+            b.truncated=true
+          end
+        end
         def contents(bucket=nil, key=nil)
           if bucket
             key ? File.join(@contents, bucket, map_filename(key.gsub("/", "\\"))) : File.join(@contents, bucket)
@@ -101,29 +111,16 @@ module Blobsterix
         end
 
         def map_filename(filename)
-          hash = Murmur.Hash64B(filename)
-          bits =  hash.to_s(2)
-          parts = []
-          6.times { |index|
-            len = 11
-            len = bits.length if len >= bits.length
-            value = bits.slice!(0, len).to_i(2).to_s(16).rjust(3,"0")
-            parts.push(value)
-          }
-          parts.push(filename)
-          parts.join("/")
+          Murmur.map_filename(filename, filename)
         end
 
-        def bucket_files(bucket)
-          Blobsterix.wait_for(Proc.new {
-            if (bucket_exist(bucket))
-              Dir.glob("#{contents}/#{bucket}/**/*").select{|e| !File.directory?(e) and not e.end_with?(".meta")}.map{ |e|
-                e.gsub("#{contents}/#{bucket}/","").gsub(/\w+\/\w+\/\w+\/\w+\/\w+\/\w+\//, "").gsub("\\", "/")
-              }
-            else
-              []
-            end
-          })
+        def bucket_empty?(bucket)
+          empty = true
+          Blobsterix::DirectoryList.each(contents(bucket)) do
+            empty = false
+            break
+          end
+          empty
         end
 
         def metaData(bucket, key)

data/lib/blobsterix/storage/storage.rb

@@ -1,7 +1,7 @@
 module Blobsterix
   module Storage
     class Storage
-      def list(bucket="root")
+      def list(bucket="root", opts={})
         Nokogiri::XML::Builder.new do |xml|
           xml.Error "no such bucket"
         end
@@ -12,7 +12,7 @@ module Blobsterix
       def get(bucket, key)
         Blobsterix::Storage::BlobMetaData.new
       end
-      def put(bucket, key, value)
+      def put(bucket, key, value, opts={})
         Blobsterix::Storage::BlobMetaData.new
       end
       def create(bucket)