blobsterix 0.0.14 → 0.0.19

data/spec/lib/helper/directory_list_spec.rb

@@ -0,0 +1,51 @@
+require "spec_helper"
+
+describe Blobsterix::DirectoryList do
+  include Blobsterix::SpecHelper
+
+  describe "list all" do
+    before :each do
+      10.times do|id|
+        Blobsterix.cache.put_raw(Blobsterix::BlobAccess.new(:bucket => "test", :id => "#{id}"), "data")
+      end
+    end
+    after :each do
+      clear_cache
+    end
+    it "should give the same entries as glob" do
+      count = Dir.glob("#{Blobsterix.cache_dir}/**/*").select { |f| File.file?(f) && !f.to_s.match(/\.meta$/) }.length
+      Blobsterix::DirectoryList.each(Blobsterix.cache_dir) do |path, filename|
+        count-=1 if !filename.to_s.match(/\.meta$/)
+      end
+      count.should eql 0
+    end
+    it "should list all files in the directories" do
+      count = 0
+      Blobsterix.cache.delete(Blobsterix::BlobAccess.new(:bucket => "test", :id => "5"))
+      Blobsterix::DirectoryList.each(Blobsterix.cache_dir) do |path, filename|
+        count+=1 if !filename.to_s.match(/\.meta$/)
+      end
+      count.should eql(9)
+    end
+    it "should get the next entry" do
+      a = Blobsterix::DirectoryWalker.new(Blobsterix.cache_dir, :start_path => Blobsterix::DirectoryWalker.new(Blobsterix.cache_dir).next)
+      b = Blobsterix::DirectoryWalker.new(Blobsterix.cache_dir)
+      first = a.next
+      real_first = b.next
+      second = b.next
+      first.should eql(second)
+      first.should_not eql(real_first)
+    end
+    it "should just run shit" do
+      counter = 0
+      a = Blobsterix::DirectoryWalker.new(Blobsterix.cache_dir)
+      while a.next
+        counter += 1
+      end
+      Blobsterix::DirectoryList.each(Blobsterix.cache_dir) do |dir, file|
+        counter -= 1
+      end
+      counter.should eql(0)
+    end
+  end
+end

data/spec/lib/s3/s3_api_spec.rb

@@ -109,6 +109,33 @@ describe Blobsterix::S3Api do
     end
   end
 
+  context "with two data objects" do
+    before :each do
+      Blobsterix.storage.put(bucket, key, StringIO.new(data, "r"))
+      Blobsterix.storage.put(bucket, "u#{key}", StringIO.new(data, "r"))
+    end
+
+    after :each do
+      clear_storage
+    end
+
+    describe 'list bucket truncated' do
+      it 'should return all files for the bucket starting at u#{key}' do
+        get "/#{bucket}", "", "params" => {"marker" => "u#{key}"}
+        expect(last_response.status).to eql(200)
+        response = Hash.from_xml last_response.body
+        expect(response).to have_key(:ListBucketResult)
+        expect(response[:ListBucketResult]).to have_key(:Name)
+        expect(response[:ListBucketResult][:Name]).to eql(bucket)
+        expect(response[:ListBucketResult][:Contents]).to_not be_empty
+        expect(response[:ListBucketResult][:Contents][:Key]).to eql(key)
+        expect(response[:ListBucketResult][:Contents][:MimeType]).to eql("text/plain")
+        expect(response[:ListBucketResult][:Contents][:Size]).to eql(data.length)
+        expect(response[:ListBucketResult][:Contents][:ETag]).to eql(Digest::MD5.hexdigest(data))
+      end
+    end
+  end
+
   context "with data" do
 
     before :each do

data/spec/lib/s3/s3_auth_spec.rb

@@ -0,0 +1,181 @@
+require "spec_helper"
+
+describe Blobsterix::S3Auth do
+  include Blobsterix::SpecHelper
+  include Goliath::TestHelper
+
+  let(:v2_delete_file) {
+    {
+      :path=>"/profile_photo%2F1023066_1407492523",
+      :head => {
+        "host"=>"career.blob.localhost.local",
+        "date" => "Fri, 08 Aug 2014 10:09:03 +0000",
+        "authorization" => "AWS somethingIdid:LxTRXgW+E0SHU2xSkMI5Q62wKhU="
+      }
+    }
+  }
+
+  let(:v2_list_bucket) {
+    {
+      :path=>"/",
+      :head => {
+        "host"=>"johnsmith.s3.amazonaws.com",
+        "date" => "Tue, 27 Mar 2007 19:42:41 +0000",
+        "authorization" => "AWS AKIAIOSFODNN7EXAMPLE:htDYFYduRNen8P9ZfE/s9SuKy0U="
+      }
+    }
+  }
+
+  let(:v2_upload_photo) {
+    {
+      :path=>"/photos/puppy.jpg",
+      :head => {
+        "host"=>"johnsmith.s3.amazonaws.com",
+        "content-type" => "image/jpeg",
+        "date" => "Tue, 27 Mar 2007 21:15:45 +0000",
+        "authorization" => "AWS AKIAIOSFODNN7EXAMPLE:MyyxeRY7whkBe+bq8fHCL/2kKUg="
+      }
+    }
+  }
+
+  let(:v2_list_root_date) {
+    {
+      :path=>"/",
+      :head => {
+        "date" => "Wed, 28 Mar 2007 01:29:59 +0000",
+        "authorization" => "AWS AKIAIOSFODNN7EXAMPLE:qGdzdERIC03wnaRNKh6OqZehG9s="
+      }
+    }
+  }
+
+  let(:v2_list_root_amz_date) {
+    {
+      :path=>"/",
+      :head => {
+        "x-amz-date" => "Fri, 08 Aug 2014 10:28:22 +0000",
+        "authorization" => "AWS somethingIdid:CEyyoVY9bnq4Ujjgwwo5ozYXEfI="
+      }
+    }
+  }
+
+  let(:v2_query_download_file) {
+    {
+      :path=>"/photos/puppy.jpg",
+      :query=>"AWSAccessKeyId=AKIAIOSFODNN7EXAMPLE&Signature=NpgCjnDzrM%2BWFzoENXmpNDUsSn8%3D&Expires=1175139620",
+      :head => {
+        "host"=>"johnsmith.s3.amazonaws.com"              
+      }
+    }
+  }
+
+  let(:v2_req_env) {
+    {
+      "HTTP_USER_AGENT"=>"fog/1.22.1",
+      "HTTP_PROXY_CONNECTION"=>"Keep-Alive",
+      "HTTP_DATE"=>"Fri, 08 Aug 2014 10:09:03 +0000",
+      "HTTP_AUTHORIZATION"=>"AWS somethingIdid:LxTRXgW+E0SHU2xSkMI5Q62wKhU=",
+      "HTTP_HOST"=>"career.blob.localhost.local:80",
+      "HTTP_TE"=>"trailers, deflate, gzip",
+      "HTTP_CONNECTION"=>"TE, close",
+      "CONTENT_LENGTH"=>"0",
+      "REQUEST_METHOD"=>"DELETE",
+      "REQUEST_URI"=>"http://career.blob.localhost.local:80/profile_photo%2F1023066_1407492523",
+      "QUERY_STRING"=>nil,
+      "HTTP_VERSION"=>"1.1",
+      "SCRIPT_NAME"=>"",
+      "REQUEST_PATH"=>"/profile_photo%2F1023066_1407492523",
+      "PATH_INFO"=>"/profile_photo%2F1023066_1407492523",
+      nil => {
+        :file => "profile_photo%2F1023066_1407492523"
+      }
+    }
+  }
+
+  let(:v4_req_env) {
+    {
+      "HTTP_USER_AGENT"=>"fog/1.22.1",
+      "HTTP_PROXY_CONNECTION"=>"Keep-Alive",
+      "HTTP_DATE"=>"Fri, 24 May 2013 00:00:00 GMT",
+      "HTTP_AUTHORIZATION"=>"AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=host;range;x-amz-content-sha256;x-amz-date,Signature=f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41",
+      "HTTP_HOST"=>"examplebucket.s3.amazonaws.com:80",
+      "HTTP_TE"=>"trailers, deflate, gzip",
+      "HTTP_CONNECTION"=>"TE, close",
+      "CONTENT_LENGTH"=>"0",
+      "REQUEST_METHOD"=>"GET",
+      "REQUEST_URI"=>"http://examplebucket.s3.amazonaws.com:80/test.txt",
+      "QUERY_STRING"=>nil,
+      "HTTP_VERSION"=>"1.1",
+      "SCRIPT_NAME"=>"",
+      "REQUEST_PATH"=>"/test.txt",
+      "PATH_INFO"=>"/test.txt",
+      "HTTP_RANGE" => "bytes=0-9",
+      "HTTP_X_AMZ_CONTENT_SHA256" => "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+      "HTTP_X_AMZ_DATE" => "20130524T000000Z",
+      nil => {
+        :file => "test.txt"
+      }
+    }
+  }
+
+  before :all do
+    Blobsterix.secret_key_store = Blobsterix::S3Auth::KeyStore.new(
+      "AKIAIOSFODNN7EXAMPLE" => "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+      "somethingIdid" => "somethingIdidInSecret"
+    )
+  end
+
+  after :all do
+    Blobsterix.secret_key_store = nil
+  end
+
+  def run_request(method, params, test, key)
+    with_api( Blobsterix::Service, :log_stdout => false) do |a|
+      Blobsterix.logger = a.logger
+      send(method, params) do |resp|
+        resp.response_header.status.should eql test
+      end
+    end
+  end
+
+  it "should work with aws v2 query" do
+    Blobsterix::S3Auth.current_time=lambda{Time.parse("Tue, 27 Mar 2007 21:15:45 +0000")}
+    run_request("put_request", v2_upload_photo, 200, "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
+    Blobsterix::S3Auth.current_time=lambda{Time.at(1175139610)}
+    run_request("get_request", v2_query_download_file, 200, "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
+  end
+
+  it "should work with aws v2" do
+    Blobsterix::S3Auth.current_time=lambda{Time.parse("Fri, 08 Aug 2014 10:28:22 +0000")}
+    run_request("get_request", v2_list_root_amz_date, 200, "somethingIdidInSecret")
+    Blobsterix::S3Auth.current_time=lambda{Time.parse("Wed, 28 Mar 2007 01:29:59 +0000")}
+    run_request("get_request", v2_list_root_date, 200, "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
+    Blobsterix::S3Auth.current_time=lambda{Time.parse("Tue, 27 Mar 2007 21:15:45 +0000")}
+    run_request("put_request", v2_upload_photo, 200, "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
+    Blobsterix::S3Auth.current_time=lambda{Time.parse("Tue, 27 Mar 2007 19:42:41 +0000")}
+    run_request("get_request", v2_list_bucket, 200, "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
+    Blobsterix::S3Auth.current_time=lambda{Time.parse("Fri, 08 Aug 2014 10:09:03 +0000")}
+    run_request("delete_request", v2_delete_file, 204, "somethingIdidInSecret")
+  end
+
+  it "should expire after 15 minutes" do
+    Blobsterix::S3Auth.current_time=lambda{Time.parse("Fri, 08 Aug 2014 10:28:22 +0000")+60*16}
+    run_request("get_request", v2_list_root_amz_date, 401, "somethingIdidInSecret")
+  end
+
+  it "should expire query when expire time is passed" do
+    Blobsterix::S3Auth.current_time=lambda{Time.parse("Tue, 27 Mar 2007 21:15:45 +0000")}
+    run_request("put_request", v2_upload_photo, 200, "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
+    Blobsterix::S3Auth.current_time=lambda{Time.at(1175139621)}
+    run_request("get_request", v2_query_download_file, 401, "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
+  end
+
+  it "should at least recognize aws v4" do
+    auth = Blobsterix::S3Auth.authenticate(v4_req_env)
+    auth.class.should eql Blobsterix::S3Auth::V4
+  end
+
+  it "should at least recognize aws v2" do
+    auth = Blobsterix::S3Auth.authenticate(v2_req_env)
+    auth.class.should eql Blobsterix::S3Auth::V2
+  end
+end

data/spec/lib/storage/file_system_spec.rb

@@ -43,6 +43,20 @@ describe Blobsterix::Storage::FileSystem do
       expect(response[:Error]).to eql("no such bucket")
     end
 
+    it "should not destroy bucket when files still exist" do
+      Blobsterix.storage.create(bucket)
+      Blobsterix.storage.put(bucket, key, StringIO.new(data, "r"))
+      response = Hash.from_xml Blobsterix.storage.list(bucket).to_xml
+      expect(response).to have_key(:ListBucketResult)
+      expect(response[:ListBucketResult]).to have_key(:Name)
+      expect(response[:ListBucketResult][:Name]).to eql(bucket)
+
+      Blobsterix.storage.delete(bucket)
+      response = Hash.from_xml Blobsterix.storage.list(bucket).to_xml
+      expect(response).to_not have_key(:Error)
+      expect(response[:Error]).to_not eql("no such bucket")
+    end
+
     it "should return the key info if it exists when listing bucket" do
       Blobsterix.storage.put(bucket, key, StringIO.new(data, "r"))
       response = Hash.from_xml Blobsterix.storage.list(bucket).to_xml

data/spec/spec_helper.rb

@@ -9,6 +9,7 @@ Bundler.setup
 Bundler.require
 
 require 'rack/test'
+require 'log4r'
 require 'goliath/test_helper'
 
 Blobsterix.storage_dir=Blobsterix.root.join("tmp/contents")

data/templates/storage_template.rb

@@ -1,7 +1,7 @@
 module Blobsterix
   module Storage
     class $ClassNameStorage < Storage
-      def list(bucket="root")
+      def list(bucket="root", opts={})
         Nokogiri::XML::Builder.new do |xml|
           xml.Error "no such bucket"
         end
@@ -12,7 +12,7 @@ module Blobsterix
       def get(bucket, key)
         Blobsterix::Storage::BlobMetaData.new
       end
-      def put(bucket, key, value)
+      def put(bucket, key, value, opts={})
         Blobsterix::Storage::BlobMetaData.new
       end
       def create(bucket)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: blobsterix
 version: !ruby/object:Gem::Version
-  version: 0.0.14
+  version: 0.0.19
 platform: ruby
 authors:
 - Daniel Sudmann
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-01 00:00:00.000000000 Z
+date: 2014-08-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
@@ -94,6 +94,20 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 3.5.0
+- !ruby/object:Gem::Dependency
+  name: log4r
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -221,16 +235,26 @@ files:
 - lib/blobsterix/helper/blob_access.rb
 - lib/blobsterix/helper/config_loader.rb
 - lib/blobsterix/helper/data_response.rb
+- lib/blobsterix/helper/directory_list.rb
 - lib/blobsterix/helper/http.rb
+- lib/blobsterix/helper/jsonizer.rb
 - lib/blobsterix/helper/logable.rb
 - lib/blobsterix/helper/murmur.rb
+- lib/blobsterix/helper/simple_proxy.rb
 - lib/blobsterix/helper/status_info.rb
 - lib/blobsterix/helper/template_renderer.rb
+- lib/blobsterix/helper/url_helper.rb
 - lib/blobsterix/mimemagic/magic.rb
 - lib/blobsterix/mimemagic/tables.rb
 - lib/blobsterix/mimemagic/version.rb
 - lib/blobsterix/router/app_router.rb
 - lib/blobsterix/s3/s3_api.rb
+- lib/blobsterix/s3/s3_auth.rb
+- lib/blobsterix/s3/s3_auth_key_store.rb
+- lib/blobsterix/s3/s3_auth_v2.rb
+- lib/blobsterix/s3/s3_auth_v2_helper.rb
+- lib/blobsterix/s3/s3_auth_v2_query.rb
+- lib/blobsterix/s3/s3_auth_v4.rb
 - lib/blobsterix/s3/s3_url_helper.rb
 - lib/blobsterix/service.rb
 - lib/blobsterix/status/status_api.rb
@@ -253,7 +277,9 @@ files:
 - spec/lib/blob/blob_api_spec.rb
 - spec/lib/blobsterix_spec.rb
 - spec/lib/helper/blob_access_spec.rb
+- spec/lib/helper/directory_list_spec.rb
 - spec/lib/s3/s3_api_spec.rb
+- spec/lib/s3/s3_auth_spec.rb
 - spec/lib/service_spec.rb
 - spec/lib/status/status_api_spec.rb
 - spec/lib/storage/cache_spec.rb
@@ -299,7 +325,9 @@ test_files:
 - spec/lib/blob/blob_api_spec.rb
 - spec/lib/blobsterix_spec.rb
 - spec/lib/helper/blob_access_spec.rb
+- spec/lib/helper/directory_list_spec.rb
 - spec/lib/s3/s3_api_spec.rb
+- spec/lib/s3/s3_auth_spec.rb
 - spec/lib/service_spec.rb
 - spec/lib/status/status_api_spec.rb
 - spec/lib/storage/cache_spec.rb