blobsterix 0.0.14 → 0.0.19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f481ca0770aed1210888cb8be44b046d78d42e8d
-  data.tar.gz: 2f68e7ca5f0a995c31bc3770a769a843144de8b0
+  metadata.gz: 552f0bd886f26d55598a1d70c64c3d67ccf0e283
+  data.tar.gz: c2442bc068866a0bcc44b2f3eb78c4b3be012b27
 SHA512:
-  metadata.gz: 712bc1e5ca1c13f41703d630cc9bfbe9f0d9641bc9bb13a8d83bf41e3e6fd1d33103f0f35641d092e4bfdf81ffe85ca0bb1160b4c90861fdc144e84e72b8c556
-  data.tar.gz: 402a885f0c1f8676eea6d5c5dcb754e0e202e754b462cf63057259b99765c83fb49bc2073ed6b592c51a023043ffb4ed4238049bbbcfb8d9b2ab5b70bc1858c9
+  metadata.gz: 6edb548afa51c035dadbeb90b756e246b6662f6c1f16fad9730979423408cac1b176c610498acefc596919fd02ce763e8ba1360762429e33b9b8d7ce455b447d
+  data.tar.gz: 5b9b46c8f65293fb56057d244247dbc2d5c28f47358326e9587fa277549ada89051b761512a400ef90f3d14828649a0d91ab5a14fb69b26b56bc357ba2f8fbbb

data/CHANGELOG.txt

@@ -1,3 +1,12 @@
+= 0.0.19
+  * s3 requests expire as expected
+= 0.0.18
+  * added s3 authentification
+= 0.0.17
+  * updated directory listing, truncated listing of 100 files each
+  * made the invalidation object a proxy that is only called when accessed
+= 0.0.15
+  * changed the way directories are listed to allow direct traversal.
 = 0.0.14
   * now files are streamed instead of read internally
   * storages now require IO streams on put

data/README.md

@@ -78,7 +78,11 @@ Check the amazon s3 specs for more information. It only supports the REST api wi
   * put file into bucket
   * delete file from bucket
 
-The s3 interface itself is unsecured and should not be visible to the outside.
+The s3 interface can be secured with a key value pair just like the AmazonS3 store:
+
+  * Blobsterix.secret_key_store = Blobsterix::S3Auth::KeyStore.new("myID" => "myVerySecretKey")
+
+You can also create your own key store. The only method it needs to implement is the get_key(ID) method. Simply check the example implementation.
 
 ## Blob interface
 

data/blobsterix.gemspec

@@ -30,6 +30,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency "nokogiri",    "~> 1.6.1"
   gem.add_dependency "ruby-webp",   "~> 0.1.0"
   gem.add_dependency "mini_magick", "~> 3.5.0"
+  gem.add_dependency "log4r"
 
   gem.add_development_dependency "rake","~> 0.9.6"
   gem.add_development_dependency "rspec", "~> 2.14.0"

data/lib/blobsterix/blob/blob_api.rb

@@ -1,6 +1,7 @@
 module Blobsterix
   class BlobApi < AppRouterBase
     include BlobUrlHelper
+    include UrlHelper
 
     get "/blob/v1", :not_allowed
     get "/blob", :not_allowed

data/lib/blobsterix/blob/blob_url_helper.rb

@@ -1,10 +1,6 @@
 module Blobsterix
   module BlobUrlHelper
-
-    def favicon
-      @favicon ||= file.match /favicon/
-    end
-
+    
     def bucket
       if  (env[nil] && env[nil][:bucket])
         env[nil][:bucket]
@@ -15,40 +11,7 @@ module Blobsterix
       end
     end
 
-    def format
-      @format ||= env[nil][:format]
-    end
-
-    def included_bucket
-      if env[nil][:bucket_or_file] && env[nil][:bucket_or_file].include?("/")
-        env[nil][:bucket] = env[nil][:bucket_or_file].split("/")[0]
-        env[nil][:bucket_or_file] = env[nil][:bucket_or_file].gsub("#{env[nil][:bucket]}/", "")
-        true
-      else
-        false
-      end
-    end
-
-    def file
-      if format
-        [env[nil][:file] || env[nil][:bucket_or_file] || "", format].join(".")
-      else
-        env[nil][:file] || env[nil][:bucket_or_file] || ""
-      end
-    end
-
-    #TransformationCommand
-    def trafo(trafo_s='')
-      trafo_a = []
-      trafo_s.split(",").each{|command|
-        parts = command.split("_")
-        key = parts.delete_at(0)
-        trafo_a << [key, parts.join("_")]
-      }
-      trafo_a
-    end
-
-    def transformation_string()
+    def transformation_string
       (env["params"] && env["params"]["trafo"]) || env[nil][:trafo] || ""
     end
   end

data/lib/blobsterix/helper/blob_access.rb

@@ -41,18 +41,10 @@ module Blobsterix
         if trafo.empty? || raw_trafo?
           metaData = Blobsterix.storage.get(self.bucket, self.id)
           if raw_trafo? || raw_accept_type?(metaData.accept_type)
-            # puts "Load from storage"
-            return metaData unless Blobsterix.cache_original?
-            Blobsterix.cache.put_raw(BlobAccess.new(:bucket => bucket, :id => id), metaData.data) if metaData.valid?
-            return Blobsterix.cache.get(BlobAccess.new(:bucket => bucket, :id => id)) if metaData.valid?
-          else
-            # puts "accept type doesn't work"
+            load_from_storage(metaData)
           end
-        else
-          # puts "trafo is not raw"
         end
-      end
-      Blobsterix.cache.get(self)
+      end || Blobsterix.cache.get(self)
     end
 
     def raw_trafo?
@@ -63,6 +55,13 @@ module Blobsterix
       @raw_accept_type||= (!accept_type || accept_type.equal?(other))
     end
 
+    def load_from_storage(metaData)
+      return metaData unless Blobsterix.cache_original?
+      Blobsterix.cache.put_raw(BlobAccess.new(:bucket => bucket, :id => id), metaData.data) if metaData.valid?
+      return Blobsterix.cache.get(BlobAccess.new(:bucket => bucket, :id => id)) if metaData.valid?
+      nil
+    end
+
 
     def subtype
       accept_type ? accept_type.subtype : ""

data/lib/blobsterix/helper/config_loader.rb

@@ -10,24 +10,16 @@ module Blobsterix
       require Blobsterix.root.join("config.rb") if (File.exist?(Blobsterix.root.join("config.rb")))
     end
 
-    def require_transformators()
-      trafo_dir = Blobsterix.root.join("transformators")
-      return if not File.exist?(trafo_dir)
-      Dir.entries(trafo_dir).each{|dir|
-        if !File.directory? File.join(trafo_dir,dir) and !(dir =='.' || dir == '..')
-          require "#{File.join(trafo_dir,dir)}"
-        end
-      }
-    end
-
-    def require_storages()
-      storages_dir = Blobsterix.root.join("storages")
-      return if not File.exist?(storages_dir)
-      Dir.entries(storages_dir).each{|dir|
-        if !File.directory? File.join(storages_dir,dir) and !(dir =='.' || dir == '..')
-          require "#{File.join(storages_dir,dir)}"
-        end
-      }
+    ["transformators", "storages"].each do |name|
+      define_method "require_#{name}".to_sym do
+        load_dir = Blobsterix.root.join(name)
+        return if not File.exist?(load_dir)
+        Dir.entries(load_dir).each{|dir|
+          if !File.directory? File.join(load_dir,dir) and !(dir =='.' || dir == '..')
+            require "#{File.join(load_dir,dir)}"
+          end
+        }
+      end
     end
   end
 end

data/lib/blobsterix/helper/directory_list.rb

@@ -0,0 +1,131 @@
+module Blobsterix
+  class DirectoryWalker
+    attr_accessor :path, :child_walker, :child_index
+
+    def initialize(base_path, opts = {})
+      @current_id = 0
+      @child_walker = nil
+      @path = Pathname.new base_path
+      @child_index = opts[:child_index]
+      init_path(opts[:start_path]) if opts[:start_path]
+    end
+
+    def init_path(start_path)
+      @start = Pathname.new(start_path)
+      myentry = path_root(@start)
+
+      entries.each_with_index do |entry,index|
+        @current_id=index+1 if myentry.to_s == entry.to_s
+      end
+
+      set_childwalker(path.join(myentry), current_id-1, @start.relative_path_from(myentry)) if path.join(myentry).directory?
+    end
+
+    def next
+      out = nil
+      begin
+        return current if @child_walker && @child_walker.next
+        return nil unless increment_id
+        out = current
+      end while out == nil
+      out
+    end
+
+    def entries
+      @entries ||= Dir.entries(path).sort
+    end
+
+    def current_entry
+      entries[@current_id-1]
+    end
+
+    def current_id
+      return @current_id if @current_id > 0
+      increment_id
+    end
+
+    def increment_id
+      begin
+        return nil if @current_id+1 > entries.size
+        @current_id+=1
+      end while (current_entry == "." || current_entry == "..")
+      @current_id
+    end
+
+    def current_path
+      current_(
+        lambda{|walker|walker.current_path},
+        lambda{|walker|walker.current_path},
+        lambda{|new_path|path}
+      )
+    end
+
+    def current_file
+      current_(
+        lambda{|walker|walker.current_file},
+        lambda{|walker|walker.current_file},
+        lambda{|new_path|entries[current_id-1]}
+      )
+    end
+
+    def current
+      current_(
+        lambda{|walker|walker.current},
+        lambda{|walker|walker.next},
+        lambda{|new_path|new_path}
+      )
+    end
+
+    private
+      def set_childwalker(path_, index_=nil, start_path_=nil)
+        options = {}
+        options[:child_index] = index_      if index_
+        options[:start_path]  = start_path_ if start_path_
+        @child_walker = DirectoryWalker.new(path_, options)
+      end
+
+      def path_root(path_)
+        myentry = nil
+        path_.descend do |entry|
+          myentry = entry
+          break
+        end
+        myentry
+      end
+
+      def current_(on_valid, on_new, on_file)
+        return nil unless current_id
+        return on_valid.call(@child_walker) if valid_childwalker?
+
+        new_path = path.join(entries[current_id-1])
+        if new_path.directory?
+          @child_walker = DirectoryWalker.new(new_path, :child_index => current_id-1)
+          on_new.call(@child_walker)
+        else
+          on_file.call(new_path)
+        end
+      end
+
+      def valid_childwalker?
+        @child_walker && @child_walker.child_index == current_id-1
+      end
+  end
+  class DirectoryList
+    def self.each_limit(path, opts={})
+      used = 0
+      limit = opts[:limit]||0
+      start_path = opts[:start_path]||nil
+      a = DirectoryWalker.new(path, :start_path => start_path)
+      while (!limit || used < limit) && a.next
+        used +=1 if yield a.current_path, a.current_file
+      end
+      a
+    end
+    def self.each(path)
+      a = DirectoryWalker.new(path)
+      while a.next
+        yield a.current_path, a.current_file
+      end
+    end
+  end
+end

data/lib/blobsterix/helper/http.rb

@@ -1,7 +1,7 @@
 module Blobsterix
   module Http
     def self.renderer
-      @@renderer||=(Blobsterix.respond_to?(:env) && Blobsterix.env == :production) ? TemplateRenderer.new(binding) : ReloadTemplateRenderer.new(binding)
+      @@renderer||=TemplateRenderer.create(binding)
     end
     def self.error_object_binding(obj)
       obj||={}
@@ -19,23 +19,22 @@ module Blobsterix
       end
       obj.get_binding
     end
-    def self.NextApi(data="Not Found", content_type="txt")
-      [600, {"Content-Type" => MimeMagic.by_extension(content_type).type}, data]
-    end
-    def self.NotFound(data="Not Found", content_type="html")
-      [404, {"Content-Type" => MimeMagic.by_extension(content_type).type}, renderer.render("error_page", error_object_binding(:title=>"Not Found", :content=>data, :error_code => 404))]
-    end
-    def self.ServerError(data="Server Error", content_type="html")
-      [500, {"Content-Type" => MimeMagic.by_extension(content_type).type}, renderer.render("error_page", error_object_binding(:title=>"Server Error", :content=>data, :error_code => 500))]
+
+    def self.error_massages
+      @error_massages||={"Not Found" => 404, "Server Error" => 500, "Not Allowed" => 403, "Not Authorized" => 401}
     end
-    def self.NotAllowed(data="Not Allowed", content_type="html")
-      [403, {"Content-Type" => MimeMagic.by_extension(content_type).type}, renderer.render("error_page", error_object_binding(:title=>"Not Allowed", :content=>data, :error_code => 403))]
+
+    error_massages.each do |error_name, error_code|
+      define_singleton_method error_name.gsub(" ", "").to_sym do |data=error_name, content_type="html"|
+        Response(error_code, renderer.render("error_page", error_object_binding(:title=>error_name, :content=>data, :error_code => 404)), content_type)
+      end
     end
-    def self.NotAuthorized(data="Not Authorized", content_type="html")
-      [401, {"Content-Type" => MimeMagic.by_extension(content_type).type}, renderer.render("error_page", error_object_binding(:title=>"Not Authorized", :content=>data, :error_code => 401))]
+
+    def self.NextApi(data="Not Found", content_type="txt")
+      Response(600, data, content_type)
     end
     def self.OK(data="", content_type="txt")
-      [200, {"Content-Type" => MimeMagic.by_extension(content_type).type}, data]
+      Response(200, data, content_type)
     end
     def self.Response(status_code=200, data="", content_type="txt")
       [status_code, {"Content-Type" => MimeMagic.by_extension(content_type).type}, data]

data/lib/blobsterix/helper/jsonizer.rb

@@ -0,0 +1,45 @@
+module Blobsterix
+  module Jsonizer
+    def json_var(*var_names)
+      @json_vars = (@json_vars||[])+var_names.flatten
+    end
+
+    def json_vars
+      @json_vars||= []
+    end
+
+    module Methods
+      def render_json(obj=nil)
+        Http.OK (obj||self).to_json, "json"
+      end
+
+      def render_xml(obj=nil)
+        obj = Nokogiri::XML::Builder.new do |xml|
+          yield xml
+        end if block_given?
+        Http.OK (obj||self).to_xml, "xml"
+      end
+
+      def to_json
+        stuff = Hash.new
+        self.class.json_vars.each{|var_name|
+          stuff[var_name.to_sym]=send(var_name) if respond_to?(var_name)
+        }
+        stuff.to_json
+      end
+      
+      def to_xml()
+        xml = Nokogiri::XML::Builder.new do |xml|
+        xml.BlobsterixStatus() {
+          self.class.json_vars.each{|var_name|
+            var = send(var_name)
+            var = var.to_xml if var.respond_to?(:to_xml)
+            xml.send(var_name, var) if respond_to?(var_name)
+          }
+        }
+        end
+        xml.to_xml
+      end
+    end
+  end
+end

data/lib/blobsterix/helper/logable.rb

@@ -1,4 +1,24 @@
 module Blobsterix
+
+  class BlobsterixLogger
+    def initialize logger, req_id
+      @logger=logger
+      @req_id=req_id
+    end
+    def info msg
+      @logger.info "[#{@req_id}] -> #{msg}"
+    end
+    def warn msg
+      @logger.warn "[#{@req_id}] -> #{msg}"
+    end
+    def error msg
+      @logger.error "[#{@req_id}] -> #{msg}"
+    end
+    def debug msg
+      @logger.debug "[#{@req_id}] -> #{msg}"
+    end
+  end
+
   module Logable
     def logger
       @logger ||= Blobsterix.logger
@@ -7,5 +27,10 @@ module Blobsterix
     def logger=(_logger)
       @logger=_logger
     end
+
+    def self.next_id
+      @req_id ||= 0
+      @req_id+=1
+    end
   end
 end

data/lib/blobsterix/helper/murmur.rb

@@ -134,4 +134,18 @@ class Murmur
 
     h
   end
+
+  def self.map_filename(filename, *additional)
+    hash = Murmur.Hash64B(filename)
+    bits =  hash.to_s(2)
+    parts = []
+    6.times { |index|
+      len = 11
+      len = bits.length if len >= bits.length
+      value = bits.slice!(0, len).to_i(2).to_s(16).rjust(3,"0")
+      parts.push(value)
+    }
+    parts = parts+additional
+    parts.join("/")
+  end
 end

data/lib/blobsterix/helper/simple_proxy.rb

@@ -0,0 +1,11 @@
+module Blobsterix
+  class SimpleProxy
+    def initialize(init_proc)
+      @init_proc = init_proc
+    end
+    def method_missing(meth, *args, &block)
+      @proc||=@init_proc.call
+      @proc.send(meth, *args, &block)
+    end
+  end
+end

data/lib/blobsterix/helper/template_renderer.rb

@@ -10,6 +10,10 @@ module Blobsterix
   end
 
   class TemplateRenderer
+    def self.create(binding_)
+      (Blobsterix.respond_to?(:env) && Blobsterix.env == :production) ? TemplateRenderer.new(binding_) : ReloadTemplateRenderer.new(binding_)
+    end
+
     def initialize(controller_binding_)
       @controller_binding=controller_binding_
     end

data/lib/blobsterix/helper/url_helper.rb

@@ -0,0 +1,41 @@
+module Blobsterix
+  module UrlHelper
+    
+    def favicon
+      @favicon ||= file.match /favicon/
+    end
+
+    #TransformationCommand
+    def trafo(trafo_s='')
+      trafo_a = []
+      trafo_s.split(",").each{|command|
+        parts = command.split("_")
+        key = parts.delete_at(0)
+        trafo_a << [key, parts.join("_")]
+      }
+      trafo_a
+    end
+
+    def file
+      if format
+        [env[nil][:file] || env[nil][:bucket_or_file] || "", format].join(".")
+      else
+        env[nil][:file] || env[nil][:bucket_or_file] || ""
+      end
+    end
+
+    def included_bucket
+      if env[nil][:bucket_or_file] && env[nil][:bucket_or_file].include?("/")
+        env[nil][:bucket] = env[nil][:bucket_or_file].split("/")[0]
+        env[nil][:bucket_or_file] = env[nil][:bucket_or_file].gsub("#{env[nil][:bucket]}/", "")
+        true
+      else
+        false
+      end
+    end
+
+    def format
+      @format ||= env[nil][:format]
+    end
+  end
+end

data/lib/blobsterix/router/app_router.rb

@@ -1,25 +1,14 @@
 module Blobsterix
-  module Jsonizer
-    def json_var(*var_names)
-      @json_vars = (@json_vars||[])+var_names.flatten
-    end
-
-    def json_vars
-      @json_vars||= []
-    end
-  end
-
   class AppRouterBase
 
     extend Jsonizer
+    include Jsonizer::Methods
     include Logable
 
-    #attr_reader :logger
     attr_accessor :env
 
     def initialize(env)
       @env = env
-      #@logger = env["rack.logger"]
     end
 
     def storage
@@ -39,7 +28,7 @@ module Blobsterix
     end
 
     def renderer
-      @@renderer||=(Blobsterix.respond_to?(:env) && Blobsterix.env == :production) ? TemplateRenderer.new(binding) : ReloadTemplateRenderer.new(binding)
+      @@renderer||=TemplateRenderer.create(binding)
     end
 
     def render(template_name, status_code=200, bind=nil)
@@ -50,74 +39,35 @@ module Blobsterix
       end
     end
 
-    def render_json(obj=nil)
-      Http.OK (obj||self).to_json, "json"
-    end
-
-    def render_xml(obj=nil)
-      obj = Nokogiri::XML::Builder.new do |xml|
-        yield xml
-      end if block_given?
-      Http.OK (obj||self).to_xml, "xml"
-    end
-
-    def to_json
-      stuff = Hash.new
-      self.class.json_vars.each{|var_name|
-        stuff[var_name.to_sym]=send(var_name) if respond_to?(var_name)
-      }
-      stuff.to_json
-    end
-    def to_xml()
-      xml = Nokogiri::XML::Builder.new do |xml|
-      xml.BlobsterixStatus() {
-        self.class.json_vars.each{|var_name|
-          var = send(var_name)
-          var = var.to_xml if var.respond_to?(:to_xml)
-          xml.send(var_name, var) if respond_to?(var_name)
-        }
-      }
-      end
-      xml.to_xml
-    end
-
     def self.options(opt)
       opt = {:function => opt.to_sym} if opt.class != Hash
       {:controller => self.name, :function => :call}.merge(opt)
     end
 
-    def self.get(path, opt = {})
-      path  = Journey::Path::Pattern.new path
-      router.routes.add_route(lambda{|env| call_controller(options(opt), env)}, path, {:request_method => "GET"}, {})
-    end
-
-    def self.post(path, opt = {})
-      path  = Journey::Path::Pattern.new path
-      router.routes.add_route(lambda{|env| call_controller(options(opt), env)}, path, {:request_method => "POST"}, {})
-    end
-
-    def self.put(path, opt = {})
-      path  = Journey::Path::Pattern.new path
-      router.routes.add_route(lambda{|env| call_controller(options(opt), env)}, path, {:request_method => "PUT"}, {})
+    def self.http_verbs
+      @http_verbs||=["GET", "POST", "PUT", "DELETE", "HEAD"]
     end
 
-    def self.delete(path, opt = {})
-      path  = Journey::Path::Pattern.new path
-      router.routes.add_route(lambda{|env| call_controller(options(opt), env)}, path, {:request_method => "DELETE"}, {})
-    end
-
-    def self.head(path, opt = {})
-      path  = Journey::Path::Pattern.new path
-      router.routes.add_route(lambda{|env| call_controller(options(opt), env)}, path, {:request_method => "HEAD"}, {})
+    http_verbs.each do |method|
+      define_singleton_method method.downcase.to_sym do |path, opt = {}|
+        path  = Journey::Path::Pattern.new path
+        router.routes.add_route(lambda{|env| call_controller(options(opt), env)}, path, {:request_method => method}, {})
+      end
     end
 
     def self.call(env)
       Blobsterix::StatusInfo.connections+=1
+      print_ram_usage("RAM USAGE Before")
       result=router.call(env)
+      print_ram_usage("RAM USAGE After")
       Blobsterix::StatusInfo.connections-=1
       result
     end
 
+    def self.print_ram_usage(text)
+      Blobsterix.logger.info "#{text}[#{Process.pid}]: " + `pmap #{Process.pid} | tail -1`[10,40].strip
+    end
+
     def self.call_controller(options, env)
       options[:controller].respond_to?(options[:function]) ? options[:controller].send(options[:function], env) : Blobsterix.const_get(options[:controller]).new(env).send(options[:function])
     end