blobsterix 0.0.19 → 0.0.23

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 552f0bd886f26d55598a1d70c64c3d67ccf0e283
-  data.tar.gz: c2442bc068866a0bcc44b2f3eb78c4b3be012b27
+  metadata.gz: 8a2f7162cc83974f24c94ba4a2fbd6fc7c69c981
+  data.tar.gz: e0c24542aac5d21c336f1ddd04af31a60959f4fc
 SHA512:
-  metadata.gz: 6edb548afa51c035dadbeb90b756e246b6662f6c1f16fad9730979423408cac1b176c610498acefc596919fd02ce763e8ba1360762429e33b9b8d7ce455b447d
-  data.tar.gz: 5b9b46c8f65293fb56057d244247dbc2d5c28f47358326e9587fa277549ada89051b761512a400ef90f3d14828649a0d91ab5a14fb69b26b56bc357ba2f8fbbb
+  metadata.gz: 0c9e2aeca38c24f9ec9a810462c787512ce5260b2b9101894d5ccab554a3a2c669cf4832bd9e6ded058896b3b806207f29cd11908e32fb9ba26444849f488875
+  data.tar.gz: 69d6830a6efe0f6280ca108a0cc28579063cdb0e6ed5f2b794b9d83261eebcc8736e7a78b75970c77a365a6fddc2d650542fa2d9d041a355418be748c4c57cd5

data/CHANGELOG.txt

@@ -1,3 +1,11 @@
+= 0.0.23
+  * fixed output type of transformation output
+= 0.0.22
+  * added at exit callback
+= 0.0.21
+  * removed log that slowed down the whole system
+= 0.0.20
+  * s3 upload does not crash the server anymore if the transformation fails
 = 0.0.19
   * s3 requests expire as expected
 = 0.0.18

data/bin/blobsterix

@@ -141,12 +141,17 @@ if ARGV.length == 0
   puts "\t\t storage (StorageName) - Creates a new blobsterix storage with the given name or just new as name."
   exit
 end
-if ARGV[0] == "server"
-  BlobsterixRunner.new(ARGV).run_blobsterix
-elsif ARGV[0] == "console"
-  BlobsterixRunner.new(ARGV).run_console
-elsif ARGV[0] == "run"
-  BlobsterixRunner.new(ARGV).run_command
-elsif ARGV[0] == "generate"
+if ARGV[0] == "generate"
   Generator.new(ARGV).run
+else
+  at_exit do
+    Blobsterix.run_at_exit
+  end
+  if ARGV[0] == "server"
+    BlobsterixRunner.new(ARGV).run_blobsterix
+  elsif ARGV[0] == "console"
+    BlobsterixRunner.new(ARGV).run_console
+  elsif ARGV[0] == "run"
+    BlobsterixRunner.new(ARGV).run_command
+  end
 end

data/lib/blobsterix/helper/directory_list.rb

@@ -48,7 +48,7 @@ module Blobsterix
       begin
         return nil if @current_id+1 > entries.size
         @current_id+=1
-      end while (current_entry == "." || current_entry == "..")
+      end while (current_entry == "." || current_entry == ".." || current_entry == ".keep")
       @current_id
     end
 

data/lib/blobsterix/helper/murmur.rb

@@ -8,6 +8,14 @@ class Murmur
     i % 2**16   # or equivalently: i & 0xffffffff
   end
 
+  def self.force_overflow_unsigned_18(i)
+    i % 2**18   # or equivalently: i & 0xffffffff
+  end
+
+  def self.force_overflow_unsigned_20(i)
+    i % 2**20   # or equivalently: i & 0xffffffff
+  end
+
   def self.force_overflow_unsigned(i)
     i % 2**32   # or equivalently: i & 0xffffffff
   end
@@ -136,11 +144,11 @@ class Murmur
   end
 
   def self.map_filename(filename, *additional)
-    hash = Murmur.Hash64B(filename)
+    hash = Murmur.force_overflow_unsigned_20(Murmur.Hash64B(filename))
     bits =  hash.to_s(2)
     parts = []
-    6.times { |index|
-      len = 11
+    2.times { |index|
+      len = 10
       len = bits.length if len >= bits.length
       value = bits.slice!(0, len).to_i(2).to_s(16).rjust(3,"0")
       parts.push(value)

data/lib/blobsterix/router/app_router.rb

@@ -57,13 +57,14 @@ module Blobsterix
 
     def self.call(env)
       Blobsterix::StatusInfo.connections+=1
-      print_ram_usage("RAM USAGE Before")
+      # print_ram_usage("RAM USAGE Before")
       result=router.call(env)
-      print_ram_usage("RAM USAGE After")
+      # print_ram_usage("RAM USAGE After")
       Blobsterix::StatusInfo.connections-=1
       result
     end
 
+    # Only use this function if you really have to it slows down the whole system
     def self.print_ram_usage(text)
       Blobsterix.logger.info "#{text}[#{Process.pid}]: " + `pmap #{Process.pid} | tail -1`[10,40].strip
     end

data/lib/blobsterix/s3/s3_api.rb

@@ -79,7 +79,11 @@ module Blobsterix
         blob_access=BlobAccess.new(:source => source, :bucket => bucket_current, :id => file_current, :accept_type => accept, :trafo => trafo_current)
         data = transformation.run(blob_access)
         cached_upload_clear
-        storage.put(bucket_current, file_current, data.open, :close_after_write => true).response(false)
+        if data.valid?
+          storage.put(bucket_current, file_current, data.open, :close_after_write => true).response(false)
+        else
+          Http.ServerError "Upload failed"
+        end
       end
 
       def delete_bucket

data/lib/blobsterix/service.rb

@@ -1,23 +1,9 @@
 module Blobsterix
   class Service < Goliath::API
     use Goliath::Rack::Params
-    # use SudiMiddleWare
-    include Logable
-=begin
-    def on_headers(env, headers)
-      env.logger.info 'received headers: ' + headers.inspect
-        env['async-headers'] = headers
-    end
 
-    def on_body(env, data)
-      env.logger.info 'received data: ' + data
-      (env['async-body'] ||= '') << data
-    end
+    include Logable
 
-    def on_close(env)
-      env.logger.info 'closing connection'
-    end
-=end
     def response(env)
       # env["params"] = params
       call_stack(env, BlobApi, StatusApi, S3Api)

data/lib/blobsterix/storage/cache.rb

@@ -16,7 +16,8 @@ module Blobsterix
 
       def initialize(path)
         @path = Pathname.new(path)
-        FileUtils.mkdir_p(@path) if !Dir.exist?(@path)
+        FileUtils.mkdir_p(@path) unless Dir.exist?(@path)
+        FileUtils.touch File.join(@path,".keep")
       end
 
       def get(blob_access)
@@ -59,7 +60,7 @@ module Blobsterix
         else
           cache_path(blob_access).entries.each {|cache_file|
             unless cache_file.to_s.match(/\.meta$/) || cache_file.directory?
-              base_name = cache_file.to_s.split("_")[0..1].join("_")
+              base_name = cache_file.to_s
               FileSystemMetaData.new(cache_path(blob_access).join(cache_file)).delete if base_name.match(cache_file_base(blob_access))
             end
           } if Dir.exist?(cache_path(blob_access))

data/lib/blobsterix/storage/file_system.rb

@@ -9,7 +9,8 @@ module Blobsterix
       def initialize(path)
         logger.info "Create FileSystem at #{path}"
         @contents = path
-        FileUtils.mkdir_p(@contents) if !Dir.exist?(@contents)
+        FileUtils.mkdir_p(@contents) unless Dir.exist?(@contents)
+        FileUtils.touch File.join(@contents,".keep")
       end
 
       def bucket_exist(bucket="root")
@@ -48,7 +49,13 @@ module Blobsterix
       def put(bucket, key, value, opts={})
         Blobsterix.storage_write(BlobAccess.new(:bucket => bucket, :id => key))
 
-        meta = Blobsterix.wait_for(Proc.new {metaData(bucket, key).write() {|f| FileUtils.copy_stream(value, f) }})
+        meta = Blobsterix.wait_for(Proc.new do
+          result = metaData(bucket, key).write() do |f|
+            FileUtils.copy_stream(value, f)
+          end
+          FileUtils.touch File.join(contents(bucket), ".keep")
+          result
+        end)
 
         value.close if opts[:close_after_write]
 
@@ -59,7 +66,8 @@ module Blobsterix
 
       def create(bucket)
         logger.info "Storage: create bucket #{contents(bucket)}"
-        FileUtils.mkdir_p(contents(bucket)) if not File.exist?(contents(bucket))
+        FileUtils.mkdir_p(contents(bucket)) unless File.exist?(contents(bucket))
+        FileUtils.touch File.join(contents(bucket), ".keep")
 
         Nokogiri::XML::Builder.new do |xml|
         end.to_s

data/lib/blobsterix/storage/file_system_meta_data.rb

@@ -80,8 +80,8 @@ module Blobsterix
 
       def write
         if block_given?
-          FileUtils.mkdir_p(File.dirname(path))
           delete
+          FileUtils.mkdir_p(File.dirname(path))
           f = File.open(path, "wb")
           yield f
           f.close
@@ -102,7 +102,16 @@ module Blobsterix
 
       def delete
         File.delete(meta_path) if File.exists?(meta_path)
-        File.delete(path) if valid
+        # File.delete(path) if valid
+        Pathname.new(path).ascend do |p|
+          begin
+            p.delete
+          rescue Errno::ENOTEMPTY
+            # stop deleting path when non empty dir is reached
+            break
+          end
+        end if valid
+        true
       end
 
       def to_json

data/lib/blobsterix/transformation/image_transformation.rb

@@ -28,7 +28,7 @@ module Blobsterix::Transformations::Impl
       end
       
       def self.output_type_
-        @input
+        @output
       end
 
       def self.body=(obj)

data/lib/blobsterix/transformation/transformation_manager.rb

@@ -83,12 +83,18 @@ module Blobsterix::Transformations
         else
           logger.debug "Transformation: run #{blob_access}"
           EM.defer(Proc.new {
-            chain.do()
+            begin
+              chain.do()
+            rescue Exception => e
+              e
+            end
           }, Proc.new {|result|
             finish_connection(result, blob_access)
           })
 
-          Fiber.yield
+          result = Fiber.yield
+          raise result if result.is_a? Exception
+          result
         end
       end
 

data/lib/blobsterix/version.rb

@@ -1,3 +1,3 @@
 module Blobsterix
-  VERSION = "0.0.19"
+  VERSION = "0.0.23"
 end

data/lib/blobsterix.rb

@@ -251,10 +251,19 @@ module Blobsterix
 
   def self.wait_for(op = nil)
     fiber = Fiber.current
-    EM.defer(op, Proc.new {|result|
+    EM.defer(Proc.new {
+        begin
+          op.call
+        rescue Exception => e
+          e
+        end
+      },
+      Proc.new {|result|
             fiber.resume result
           })
-    Fiber.yield
+     result = Fiber.yield
+     raise result if result.is_a? Exception
+     result
   end
 
   def self.wait_for_next(op = nil)
@@ -262,4 +271,14 @@ module Blobsterix
       wait_for(op)
     end
   end
+
+  def self.at_exit(&proc)
+    (@at_exit_callback||=[]).push(proc)
+  end
+
+  def self.run_at_exit
+    (@at_exit_callback||=[]).each do |proc|
+      proc.call
+    end
+  end
 end

metadata

@@ -1,195 +1,195 @@
 --- !ruby/object:Gem::Specification
 name: blobsterix
 version: !ruby/object:Gem::Version
-  version: 0.0.19
+  version: 0.0.23
 platform: ruby
 authors:
 - Daniel Sudmann
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-14 00:00:00.000000000 Z
+date: 2014-09-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.8.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.8.1
 - !ruby/object:Gem::Dependency
   name: goliath
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.0.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.0.3
 - !ruby/object:Gem::Dependency
   name: journey
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.0.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.0.4
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.6.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.6.1
 - !ruby/object:Gem::Dependency
   name: ruby-webp
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.1.0
 - !ruby/object:Gem::Dependency
   name: mini_magick
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 3.5.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 3.5.0
 - !ruby/object:Gem::Dependency
   name: log4r
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.9.6
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.9.6
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 2.14.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 2.14.0
 - !ruby/object:Gem::Dependency
   name: rspec-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec-expectations
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec-mocks
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: Blobsterix is a transcoding, caching, storage server that can transform
@@ -201,8 +201,8 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .travis.yml
+- ".gitignore"
+- ".travis.yml"
 - CHANGELOG.txt
 - Gemfile
 - LICENSE
@@ -307,17 +307,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: 2.0.0.451
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Blobsterix is a transcoding, caching, storage server.
@@ -333,4 +333,3 @@ test_files:
 - spec/lib/storage/cache_spec.rb
 - spec/lib/storage/file_system_spec.rb
 - spec/spec_helper.rb
-has_rdoc: