blobsterix 0.0.13 → 0.0.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 61c63e268a35487d1180f80f6f9865fd0c0cb0e2
-  data.tar.gz: 648c1525917b0577eb25bcfe5dbc1b7e24df3366
+  metadata.gz: f481ca0770aed1210888cb8be44b046d78d42e8d
+  data.tar.gz: 2f68e7ca5f0a995c31bc3770a769a843144de8b0
 SHA512:
-  metadata.gz: 023f0e53440327eaa1fc5cbc63136b9684d09ff7d799dc32dd456b3e96f9f0de34309ce92c13f9b392b51d1f31a1138e640ba9aae984c62d806d40fff85b0c50
-  data.tar.gz: d2fd9b85f3d99303f8cdb40961d061c40916234a04f611cacd6ccc95abd0b8c3e9b46a8191f5221ee5165fe4fad0e8714e72cfe53030f31870fd78b092721968
+  metadata.gz: 712bc1e5ca1c13f41703d630cc9bfbe9f0d9641bc9bb13a8d83bf41e3e6fd1d33103f0f35641d092e4bfdf81ffe85ca0bb1160b4c90861fdc144e84e72b8c556
+  data.tar.gz: 402a885f0c1f8676eea6d5c5dcb754e0e202e754b462cf63057259b99765c83fb49bc2073ed6b592c51a023043ffb4ed4238049bbbcfb8d9b2ab5b70bc1858c9

data/CHANGELOG.txt

@@ -1,3 +1,7 @@
+= 0.0.14
+  * now files are streamed instead of read internally
+  * storages now require IO streams on put
+  * storage is made async
 = 0.0.13
   * fixing getters for 0.0.12
 = 0.0.12

data/lib/blobsterix/helper/blob_access.rb

@@ -43,7 +43,7 @@ module Blobsterix
           if raw_trafo? || raw_accept_type?(metaData.accept_type)
             # puts "Load from storage"
             return metaData unless Blobsterix.cache_original?
-            Blobsterix.cache.put(BlobAccess.new(:bucket => bucket, :id => id), metaData.data) if metaData.valid?
+            Blobsterix.cache.put_raw(BlobAccess.new(:bucket => bucket, :id => id), metaData.data) if metaData.valid?
             return Blobsterix.cache.get(BlobAccess.new(:bucket => bucket, :id => id)) if metaData.valid?
           else
             # puts "accept type doesn't work"

data/lib/blobsterix/helper/data_response.rb

@@ -19,7 +19,7 @@ module Blobsterix
           if env != nil && meta.size > 30000 && Blobsterix.allow_chunked_stream
             chunkresponse
           else
-            [200, meta.header, (with_data ? meta.data : "")]
+            [200, meta.header, (with_data ? File.open(meta.path, "rb") : "")]
           end
         else
           [304, meta.header, ""]

data/lib/blobsterix/s3/s3_api.rb

@@ -67,7 +67,7 @@ module Blobsterix
         blob_access=BlobAccess.new(:source => source, :bucket => bucket_current, :id => file_current, :accept_type => accept, :trafo => trafo_current)
         data = transformation.run(blob_access)
         cached_upload_clear
-        storage.put(bucket_current, file_current, data).response(false)
+        storage.put(bucket_current, file_current, data.open, :close_after_write => true).response(false)
       end
 
       def delete_bucket

data/lib/blobsterix/s3/s3_url_helper.rb

@@ -14,7 +14,7 @@ module Blobsterix
     end
 
     def cache_upload
-      cache.put(cache_upload_key, env['rack.input'].read)
+      cache.put_stream(cache_upload_key, env['rack.input'])
     end
 
     def cached_upload
@@ -27,7 +27,6 @@ module Blobsterix
     end
 
     def cache_upload_key
-      #@cache_upload_key ||= "upload/"+bucket.gsub("/", "_")+"_"+file.gsub("/", "_")
       @cache_upload_key ||= Blobsterix::BlobAccess.new(:bucket => bucket, :id => "upload_#{file.gsub("/", "_")}")
     end
 

data/lib/blobsterix/service.rb

@@ -1,6 +1,7 @@
 module Blobsterix
   class Service < Goliath::API
     use Goliath::Rack::Params
+    include Logable
 =begin
     def on_headers(env, headers)
       env.logger.info 'received headers: ' + headers.inspect
@@ -16,8 +17,16 @@ module Blobsterix
       env.logger.info 'closing connection'
     end
 =end
+    def get_request_id
+      @request_id||=0
+      @request_id+=1
+    end
     def response(env)
-      call_stack(env, BlobApi, StatusApi, S3Api)
+      env["BLOBSTERIX_REQUEST_ID"] = get_request_id
+      logger.info "RAM USAGE Before[#{Process.pid}]: " + `pmap #{Process.pid} | tail -1`[10,40].strip
+      a = call_stack(env, BlobApi, StatusApi, S3Api)
+      logger.info "RAM USAGE After[#{Process.pid}]: " + `pmap #{Process.pid} | tail -1`[10,40].strip
+      a
     end
 
     def call_stack(env, *apis)

data/lib/blobsterix/storage/blob_meta_data.rb

@@ -50,6 +50,12 @@ module Blobsterix
         end
         self
       end
+      def open
+        if block_given?
+          #should yield file
+        end
+        self
+      end
       def delete
       end
       def response(with_data=true, _etag=nil, env = nil)

data/lib/blobsterix/storage/cache.rb

@@ -4,12 +4,14 @@ module Blobsterix
       include Blobsterix::Logable
 
       def invalidation
-        each_meta_file do |meta_file|
-          blob_access=meta_to_blob_access(meta_file)
-          if Blobsterix.cache_checker.call(blob_access,meta_file.last_accessed,meta_file.last_modified)
-            invalidate(blob_access, true)
+        Blobsterix.wait_for(Proc.new {
+          each_meta_file do |meta_file|
+            blob_access=meta_to_blob_access(meta_file)
+            if Blobsterix.cache_checker.call(blob_access,meta_file.last_accessed,meta_file.last_modified)
+              invalidate(blob_access, true)
+            end
           end
-        end
+        })
       end
       def initialize(path)
         @path = Pathname.new(path)
@@ -20,12 +22,25 @@ module Blobsterix
         FileSystemMetaData.new(cache_file_path(blob_access))
       end
 
-      def put(blob_access, data)
+      def put_raw(blob_access, data)
         FileSystemMetaData.new(cache_file_path(blob_access),:bucket => blob_access.bucket, :id => blob_access.id, :trafo => blob_access.trafo, :accept_type => "#{blob_access.accept_type}").write() {|f|
           f.write(data)
         }
       end
 
+      def put_stream(blob_access, stream)
+        FileSystemMetaData.new(cache_file_path(blob_access),:bucket => blob_access.bucket, :id => blob_access.id, :trafo => blob_access.trafo, :accept_type => "#{blob_access.accept_type}").write do |f|
+          FileUtils.copy_stream(stream, f)
+        end
+      end
+
+      def put(blob_access, path)
+        target_path = cache_file_path(blob_access)
+        FileUtils.mkdir_p(File.dirname(target_path))
+        FileUtils.cp(path, target_path, :preserve => false)
+        FileSystemMetaData.new(target_path,:bucket => blob_access.bucket, :id => blob_access.id, :trafo => blob_access.trafo, :accept_type => "#{blob_access.accept_type}")
+      end
+
       def delete(blob_access)
         FileSystemMetaData.new(cache_file_path(blob_access)).delete if exists?(blob_access)
       end

data/lib/blobsterix/storage/file_system.rb

@@ -26,16 +26,19 @@ module Blobsterix
         else
           if bucket_exist(bucket)
             b = Bucket.new(bucket, time_string_of(bucket))
-            bucket_files(bucket).each do |file|
-              b.contents << BucketEntry.new(file) do |entry|
-                meta = metaData(bucket, file)
-                entry.last_modified =  meta.last_modified.strftime("%Y-%m-%dT%H:%M:%S.000Z")
-                entry.etag =  meta.etag
-                entry.size =  meta.size
-                entry.mimetype = meta.mimetype
-                entry.fullpath = contents(bucket, file).gsub("#{contents}/", "")
+            files = bucket_files(bucket)
+            Blobsterix.wait_for(Proc.new {
+              files.each do |file|
+                b.contents << BucketEntry.new(file) do |entry|
+                  meta = metaData(bucket, file)
+                  entry.last_modified =  meta.last_modified.strftime("%Y-%m-%dT%H:%M:%S.000Z")
+                  entry.etag =  meta.etag
+                  entry.size =  meta.size
+                  entry.mimetype = meta.mimetype
+                  entry.fullpath = contents(bucket, file).gsub("#{contents}/", "")
+                end
               end
-            end
+            })
             b
           else
             Nokogiri::XML::Builder.new do |xml|
@@ -46,7 +49,7 @@ module Blobsterix
       end
 
       def get(bucket, key)
-        if (not File.directory?(contents(bucket, key))) and bucket_files(bucket).include?(key)
+        if (not File.directory?(contents(bucket, key))) # and bucket_files(bucket).include?(key)
           Blobsterix.storage_read(BlobAccess.new(:bucket => bucket, :id => key))
           metaData(bucket, key)
         else
@@ -55,12 +58,15 @@ module Blobsterix
         end
       end
 
-      def put(bucket, key, value)
+      def put(bucket, key, value, opts={})
         Blobsterix.storage_write(BlobAccess.new(:bucket => bucket, :id => key))
 
-        meta = metaData(bucket, key).write() {|f| f.write(value.read) }
+        meta = Blobsterix.wait_for(Proc.new {metaData(bucket, key).write() {|f| FileUtils.copy_stream(value, f) }})
+
+        value.close if opts[:close_after_write]
+
+        Blobsterix.wait_for(Proc.new {Blobsterix.cache.invalidate(Blobsterix::BlobAccess.new(:bucket => bucket, :id => key))})
 
-        Blobsterix.cache.invalidate(Blobsterix::BlobAccess.new(:bucket => bucket, :id => key))
         meta
       end
 
@@ -74,17 +80,15 @@ module Blobsterix
 
       def delete(bucket)
         logger.info "Storage: delete bucket #{contents(bucket)}"
-
         FileUtils.rm_rf(contents(bucket)) if bucket_exist(bucket) && bucket_files(bucket).empty?
-
         #Dir.rmdir(contents(bucket)) if bucket_exist(bucket) && bucket_files(bucket).empty?
       end
 
       def delete_key(bucket, key)
         Blobsterix.storage_delete(BlobAccess.new(:bucket => bucket, :id => key))
-        Blobsterix.cache.invalidate(Blobsterix::BlobAccess.new(:bucket => bucket, :id => key))
+        Blobsterix.wait_for(Proc.new {Blobsterix.cache.invalidate(Blobsterix::BlobAccess.new(:bucket => bucket, :id => key))})
 
-        metaData(bucket, key).delete if bucket_files(bucket).include? key
+        metaData(bucket, key).delete # if bucket_files(bucket).include? key
       end
 
       private
@@ -111,13 +115,15 @@ module Blobsterix
         end
 
         def bucket_files(bucket)
-          if (bucket_exist(bucket))
-            Dir.glob("#{contents}/#{bucket}/**/*").select{|e| !File.directory?(e) and not e.end_with?(".meta")}.map{ |e|
-              e.gsub("#{contents}/#{bucket}/","").gsub(/\w+\/\w+\/\w+\/\w+\/\w+\/\w+\//, "").gsub("\\", "/")
-            }
-          else
-            []
-          end
+          Blobsterix.wait_for(Proc.new {
+            if (bucket_exist(bucket))
+              Dir.glob("#{contents}/#{bucket}/**/*").select{|e| !File.directory?(e) and not e.end_with?(".meta")}.map{ |e|
+                e.gsub("#{contents}/#{bucket}/","").gsub(/\w+\/\w+\/\w+\/\w+\/\w+\/\w+\//, "").gsub("\\", "/")
+              }
+            else
+              []
+            end
+          })
         end
 
         def metaData(bucket, key)

data/lib/blobsterix/storage/file_system_meta_data.rb

@@ -20,10 +20,10 @@ module Blobsterix
 
       def etag
         if @last_modified === last_modified
-          @etag ||= Digest::MD5.hexdigest(data)
+          @etag ||= Digest::MD5.file(path).hexdigest
         else
           @last_modified = last_modified
-          @etag = Digest::MD5.hexdigest(data)
+          @etag = Digest::MD5.file(path).hexdigest
         end
       end
 
@@ -90,6 +90,16 @@ module Blobsterix
         self
       end
 
+      def open
+        if block_given?
+          f = File.open(path, "rb")
+          yield f
+          f.close
+        else
+          File.open(path, "rb")
+        end
+      end
+
       def delete
         File.delete(meta_path) if File.exists?(meta_path)
         File.delete(path) if valid

data/lib/blobsterix/transformation/transformation_chain.rb

@@ -44,7 +44,7 @@ module Blobsterix::Transformations
           break
         end
 
-        cache.put(@target_blob_access,Blobsterix::Storage::FileSystemMetaData.new(last_key).read)
+        cache.put(@target_blob_access, last_key)
         @target_blob_access.reset!
       end unless @target_blob_access.get.valid
 

data/lib/blobsterix/version.rb

@@ -1,3 +1,3 @@
 module Blobsterix
-  VERSION = "0.0.13"
+  VERSION = "0.0.14"
 end

data/lib/blobsterix.rb

@@ -227,4 +227,18 @@ module Blobsterix
   def self.storage_delete(blob_access)
     event("storage.delete",:blob_access => blob_access)
   end
+
+  def self.wait_for(op = nil)
+    fiber = Fiber.current
+    EM.defer(op, Proc.new {|result|
+            fiber.resume result
+          })
+    Fiber.yield
+  end
+
+  def self.wait_for_next(op = nil)
+    EM.next_tick do 
+      wait_for(op)
+    end
+  end
 end

data/spec/lib/blob/blob_api_spec.rb

@@ -1,10 +1,16 @@
 require "spec_helper"
 
 describe Blobsterix::BlobApi do
+  include Blobsterix::SpecHelper
   include Rack::Test::Methods
   def app
     Blobsterix::BlobApi
   end
+
+  around(:each) do |example|
+    run_em(&example)
+  end
+
   describe 'GET /blob/v1/' do
     it 'get several categories of repositories by name' do
       get "/blob/v1/"
@@ -13,7 +19,6 @@ describe Blobsterix::BlobApi do
   end
 
   describe 'Transformed get' do
-    include Blobsterix::SpecHelper
 
     context "with data" do
       let(:data) {"Hi my name is Test"}
@@ -39,18 +44,14 @@ describe Blobsterix::BlobApi do
 
       it "should return the file" do
         expect(Blobsterix.transformation).to receive(:cue_transformation).once.and_call_original
-        run_em do 
-          get "/blob/v1/dummy_#{data_transformed}.test/test.txt"
-        end
+        get "/blob/v1/dummy_#{data_transformed}.test/test.txt"
         expect(last_response.status).to eql(200)
         expect(last_response.body).to eql(data_transformed)
       end
 
       it "should return the file head" do
         expect(Blobsterix.transformation).to receive(:cue_transformation).once.and_call_original
-        run_em do 
-          head "/blob/v1/dummy.test/test.txt"
-        end
+        head "/blob/v1/dummy.test/test.txt"
         expect(last_response.status).to eql(200)
         expect(last_response.body).to eql("")
       end
@@ -58,10 +59,8 @@ describe Blobsterix::BlobApi do
       it "should return the file and wait for previous trafos to finish" do
         expect(Blobsterix.transformation).to receive(:cue_transformation).once.and_call_original
         expect(Blobsterix.transformation).to receive(:wait_for_transformation).once.and_call_original
-        run_em do 
-          get "/blob/v1/dummy_#{data_transformed}.test/test.txt"
-          get "/blob/v1/dummy_#{data_transformed}.test/test.txt"
-        end
+        get "/blob/v1/dummy_#{data_transformed}.test/test.txt"
+        get "/blob/v1/dummy_#{data_transformed}.test/test.txt"
         expect(last_response.status).to eql(200)
         expect(last_response.body).to eql(data_transformed)
       end
@@ -69,10 +68,8 @@ describe Blobsterix::BlobApi do
       it "should return the file and not wait for different trafos to finish" do
         expect(Blobsterix.transformation).to receive(:cue_transformation).twice.and_call_original
         expect(Blobsterix.transformation).to receive(:wait_for_transformation).never.and_call_original
-        run_em do 
-          get "/blob/v1/dummy_#{data_transformed},dummy_#{data_transformed}.test/test.txt"
-          get "/blob/v1/dummy_#{data_transformed}.test/test.txt"
-        end
+        get "/blob/v1/dummy_#{data_transformed},dummy_#{data_transformed}.test/test.txt"
+        get "/blob/v1/dummy_#{data_transformed}.test/test.txt"
         expect(last_response.status).to eql(200)
         expect(last_response.body).to eql(data_transformed)
       end

data/spec/lib/helper/blob_access_spec.rb

@@ -23,6 +23,10 @@ describe Blobsterix::BlobAccess do
     Blobsterix::BlobAccess.new(:bucket => bucket, :id => key, :trafo => [["raw", ""]], :accept_type => Blobsterix::AcceptType.new("text/plain"))
   end
 
+  around(:each) do |example|
+    run_em(&example)
+  end
+
   describe "blob_access" do
     after :each do
       clear_data

data/spec/lib/s3/s3_api_spec.rb

@@ -11,6 +11,10 @@ describe Blobsterix::S3Api do
   let(:key) {"test.txt"}
   let(:bucket) {"test"}
 
+  around(:each) do |example|
+    run_em(&example)
+  end
+
   after :each do
       clear_data
   end
@@ -18,9 +22,7 @@ describe Blobsterix::S3Api do
   describe "create a bucket" do
     it "should have bucket after creation" do
 
-      run_em do
-        put "/", "", "HTTP_HOST" => "#{bucket}.s3.blah.de"
-      end
+      put "/", "", "HTTP_HOST" => "#{bucket}.s3.blah.de"
 
       get "/#{bucket}"
       expect(last_response.status).to eql(200)
@@ -46,9 +48,7 @@ describe Blobsterix::S3Api do
     it "should have file in bucket after upload" do
       #expect(Blobsterix.transformation).to receive(:cue_transformation).never.and_call_original
 
-      run_em do
-        put "/#{key}", data, {"HTTP_HOST" => "#{bucket}.s3.blah.de"}
-      end
+      put "/#{key}", data, {"HTTP_HOST" => "#{bucket}.s3.blah.de"}
 
       expect(last_response.status).to eql(200)
       expect(Blobsterix.storage.get(bucket, key).read).to eql(data)
@@ -57,9 +57,7 @@ describe Blobsterix::S3Api do
     it "should have file in bucket after upload with trafo" do
       #expect(Blobsterix.transformation).to receive(:cue_transformation).once.and_call_original
 
-      run_em do
-        put "/#{key}", data, {"HTTP_HOST" => "#{bucket}.s3.blah.de", "HTTP_X_AMZ_META_TRAFO" => "dummy_Yeah"}
-      end
+      put "/#{key}", data, {"HTTP_HOST" => "#{bucket}.s3.blah.de", "HTTP_X_AMZ_META_TRAFO" => "dummy_Yeah"}
 
       expect(last_response.status).to eql(200)
       expect(Blobsterix.storage.get(bucket, key).read).to eql("Yeah")

data/spec/lib/storage/cache_spec.rb

@@ -12,12 +12,16 @@ describe Blobsterix::Storage::Cache do
   let(:blob_access_2) {Blobsterix::BlobAccess.new(:bucket => bucket, :id => key, :trafo => [["dummy", ""]])}
   let(:blob_access_3) {Blobsterix::BlobAccess.new(:bucket => bucket, :id => key, :trafo => [["test", ""],["dummy", ""]])}
 
+  around(:each) do |example|
+    run_em(&example)
+  end
+
   describe "invalidation" do
     before :each do
-      Blobsterix.cache.put(blob_access, data)
-      Blobsterix.cache.put(blob_access_1, data)
-      Blobsterix.cache.put(blob_access_2, data)
-      Blobsterix.cache.put(blob_access_3, data)
+      Blobsterix.cache.put_raw(blob_access, data)
+      Blobsterix.cache.put_raw(blob_access_1, data)
+      Blobsterix.cache.put_raw(blob_access_2, data)
+      Blobsterix.cache.put_raw(blob_access_3, data)
     end
 
     after :each do
@@ -68,14 +72,32 @@ describe Blobsterix::Storage::Cache do
     end
 
     it "should return valid blob when key exists" do
-      Blobsterix.cache.put(blob_access, data)
+      Blobsterix.cache.put_raw(blob_access, data)
+      metaData = Blobsterix.cache.get(blob_access)
+      expect(metaData.valid).to be(true)
+      expect(metaData.read).to eql(data)
+    end
+
+    it "should return valid blob when key exists and was copied via Stream" do
+      Blobsterix.cache.put_stream(blob_access, StringIO.new(data, "r"))
+      metaData = Blobsterix.cache.get(blob_access)
+      expect(metaData.valid).to be(true)
+      expect(metaData.read).to eql(data)
+    end
+
+    it "should return valid blob when key exists and was copied via path" do
+      tmp = Tempfile.new("sldhgs")
+      tmp.write(data)
+      tmp.close
+      Blobsterix.cache.put(blob_access, tmp.path)
       metaData = Blobsterix.cache.get(blob_access)
       expect(metaData.valid).to be(true)
       expect(metaData.read).to eql(data)
+      tmp.unlink
     end
 
     it "should return invalid blob when key is invalidated" do
-      Blobsterix.cache.put(blob_access, data)
+      Blobsterix.cache.put_raw(blob_access, data)
       metaData = Blobsterix.cache.get(blob_access)
       expect(metaData.valid).to be(true)
       expect(metaData.read).to eql(data)
@@ -86,8 +108,8 @@ describe Blobsterix::Storage::Cache do
     end
 
     it "should return invalid blob when key is invalidated for all trafos" do
-      Blobsterix.cache.put(blob_access, data)
-      Blobsterix.cache.put(blob_access_1, data)
+      Blobsterix.cache.put_raw(blob_access, data)
+      Blobsterix.cache.put_raw(blob_access_1, data)
 
       metaData = Blobsterix.cache.get(blob_access)
       expect(metaData.valid).to be(true)
@@ -104,8 +126,8 @@ describe Blobsterix::Storage::Cache do
     end
 
     it "should return invalid blob when key is invalidated for one trafos" do
-      Blobsterix.cache.put(blob_access, data)
-      Blobsterix.cache.put(blob_access_1, data)
+      Blobsterix.cache.put_raw(blob_access, data)
+      Blobsterix.cache.put_raw(blob_access_1, data)
 
       metaData = Blobsterix.cache.get(blob_access)
       expect(metaData.valid).to be(true)
@@ -122,7 +144,7 @@ describe Blobsterix::Storage::Cache do
     end
 
     it "should return invalid blob when key is deleted" do
-      Blobsterix.cache.put(blob_access, data)
+      Blobsterix.cache.put_raw(blob_access, data)
       metaData = Blobsterix.cache.get(blob_access)
       expect(metaData.valid).to be(true)
       expect(metaData.read).to eql(data)

data/spec/lib/storage/file_system_spec.rb

@@ -6,9 +6,11 @@ describe Blobsterix::Storage::FileSystem do
   let(:data) {"Hi my name is Test"}
   let(:key) {"test.txt"}
   let(:bucket) {"test"}
-
-
-
+  
+  around(:each) do |example|
+    run_em(&example)
+  end
+  
   describe "bucket" do
     after :each do
       clear_storage

data/spec/spec_helper.rb

@@ -30,9 +30,12 @@ module Blobsterix
     end
     def run_em(&block)
       EM.run {
-        f = Fiber.new(&block)
+        f = Fiber.new{
+          yield block
+          while not EM.defers_finished?; end
+          EM.stop
+        }
         f.resume
-        EM.stop
       }
     end
     class DummyTrafo < Blobsterix::Transformations::Transformation

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: blobsterix
 version: !ruby/object:Gem::Version
-  version: 0.0.13
+  version: 0.0.14
 platform: ruby
 authors:
 - Daniel Sudmann
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-29 00:00:00.000000000 Z
+date: 2014-08-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json