blobsterix 0.0.9

data/lib/blobsterix/mimemagic/version.rb

@@ -0,0 +1,5 @@
+class MimeMagic
+  # MimeMagic version string
+  # @api public
+  VERSION = '0.2.1'
+end

data/lib/blobsterix/router/app_router.rb

@@ -0,0 +1,134 @@
+module Blobsterix
+  module Jsonizer
+    def json_var(*var_names)
+      @json_vars = (@json_vars||[])+var_names.flatten
+    end
+
+    def json_vars
+      @json_vars||= []
+    end
+  end
+
+  class AppRouterBase
+
+    extend Jsonizer
+    include Logable
+
+    #attr_reader :logger
+    attr_accessor :env
+
+    def initialize(env)
+      @env = env
+      #@logger = env["rack.logger"]
+    end
+
+    def storage
+      @storage ||= Blobsterix.storage
+    end
+
+    def cache
+      @cache ||= Blobsterix.cache
+    end
+
+    def transformation
+      @transformation ||= Blobsterix.transformation
+    end
+
+    def next_api
+      Http.NextApi
+    end
+
+    def renderer
+      @@renderer||=(Blobsterix.respond_to?(:env) && Blobsterix.env == :production) ? TemplateRenderer.new(binding) : ReloadTemplateRenderer.new(binding)
+    end
+
+    def render(template_name, status_code=200, bind=nil)
+      begin
+        Http.Response(status_code, renderer.render(template_name, bind||binding), "html")
+      rescue Errno::ENOENT => e
+        Http.NotFound
+      end
+    end
+
+    def render_json(obj=nil)
+      Http.OK (obj||self).to_json, "json"
+    end
+
+    def render_xml(obj=nil)
+      obj = Nokogiri::XML::Builder.new do |xml|
+        yield xml
+      end if block_given?
+      Http.OK (obj||self).to_xml, "xml"
+    end
+
+    def to_json
+      stuff = Hash.new
+      self.class.json_vars.each{|var_name|
+        stuff[var_name.to_sym]=send(var_name) if respond_to?(var_name)
+      }
+      stuff.to_json
+    end
+    def to_xml()
+      xml = Nokogiri::XML::Builder.new do |xml|
+      xml.BlobsterixStatus() {
+        self.class.json_vars.each{|var_name|
+          var = send(var_name)
+          var = var.to_xml if var.respond_to?(:to_xml)
+          xml.send(var_name, var) if respond_to?(var_name)
+        }
+      }
+      end
+      xml.to_xml
+    end
+
+    def self.options(opt)
+      opt = {:function => opt.to_sym} if opt.class != Hash
+      {:controller => self.name, :function => :call}.merge(opt)
+    end
+
+    def self.get(path, opt = {})
+      path  = Journey::Path::Pattern.new path
+      router.routes.add_route(lambda{|env| call_controller(options(opt), env)}, path, {:request_method => "GET"}, {})
+    end
+
+    def self.post(path, opt = {})
+      path  = Journey::Path::Pattern.new path
+      router.routes.add_route(lambda{|env| call_controller(options(opt), env)}, path, {:request_method => "POST"}, {})
+    end
+
+    def self.put(path, opt = {})
+      path  = Journey::Path::Pattern.new path
+      router.routes.add_route(lambda{|env| call_controller(options(opt), env)}, path, {:request_method => "PUT"}, {})
+    end
+
+    def self.delete(path, opt = {})
+      path  = Journey::Path::Pattern.new path
+      router.routes.add_route(lambda{|env| call_controller(options(opt), env)}, path, {:request_method => "DELETE"}, {})
+    end
+
+    def self.head(path, opt = {})
+      path  = Journey::Path::Pattern.new path
+      router.routes.add_route(lambda{|env| call_controller(options(opt), env)}, path, {:request_method => "HEAD"}, {})
+    end
+
+    def self.call(env)
+      Blobsterix::StatusInfo.connections+=1
+      result=router.call(env)
+      Blobsterix::StatusInfo.connections-=1
+      result
+    end
+
+    def self.call_controller(options, env)
+      options[:controller].respond_to?(options[:function]) ? options[:controller].send(options[:function], env) : Blobsterix.const_get(options[:controller]).new(env).send(options[:function])
+    end
+
+    private
+      def self.routes()
+        (@@routes ||= {})[self.name] ||= Journey::Routes.new
+      end
+
+      def self.router()
+        (@@router ||= {})[self.name] ||= Journey::Router.new routes, {}
+      end
+  end
+end

data/lib/blobsterix/s3/s3_api.rb

@@ -0,0 +1,92 @@
+module Blobsterix
+  class S3Api < AppRouterBase
+    include S3UrlHelper
+
+    get "/", :list_buckets
+
+    get "/*bucket_or_file.:format", :get_file
+    get "/*bucket_or_file", :get_file
+
+    head "/*bucket_or_file.:format", :get_file_head
+    head "/*bucket_or_file", :get_file_head
+
+    put "/", :create_bucket
+
+    put "/*file.:format", :upload_data
+    put "/*file", :upload_data
+
+    delete "/", :delete_bucket
+    delete "/*file.:format", :delete_file
+    delete "/*file", :delete_file
+
+    get "*any", :next_api
+    put "*any", :next_api
+    delete "*any", :next_api
+    head "*any", :next_api
+    post "*any", :next_api
+
+    private
+      def list_buckets
+         Blobsterix.event("s3_api.list_bucket",:bucket => bucket)
+        Http.OK storage.list(bucket).to_xml, "xml"
+      end
+
+      def get_file(send_with_data=true)
+        return Http.NotFound if favicon
+
+        if bucket?
+          if meta = storage.get(bucket, file)
+            send_with_data ? meta.response(true, env["HTTP_IF_NONE_MATCH"], env, env["HTTP_X_FILE"] === "yes", false) : meta.response(false)
+          else
+            Http.NotFound
+          end
+        else
+          Http.OK storage.list(bucket).to_xml, "xml"
+        end
+      end
+
+      def get_file_head
+                           #TODO: add event?
+        get_file(false)
+      end
+
+      def create_bucket
+              Blobsterix.event("s3_api.upload",:bucket => bucket)
+        Http.OK storage.create(bucket), "xml"
+      end
+
+      def upload_data
+        source = cached_upload
+        accept = AcceptType.new("*/*")#source.accept_type()
+
+        trafo_current = trafo(trafo_string)
+        file_current = file
+        bucket_current = bucket
+        Blobsterix.event("s3_api.upload", :bucket => bucket_current, 
+                                              :file => file_current, :accept_type => accept.type, :trafo => trafo_current)
+        blob_access=BlobAccess.new(:source => source, :bucket => bucket_current, :id => file_current, :accept_type => accept, :trafo => trafo_current)
+        data = transformation.run(blob_access)
+        cached_upload_clear
+        storage.put(bucket_current, file_current, data).response(false)
+      end
+
+      def delete_bucket
+        Blobsterix.event("s3_api.delete_bucket", :bucket => bucket)
+
+        if bucket?
+          Http.OK_no_data storage.delete(bucket), "xml"
+        else
+          Http.NotFound "no such bucket"
+        end
+      end
+
+      def delete_file
+         Blobsterix.event("s3_api.delete_file", :bucket => bucket,:file => file)
+        if bucket?
+          Http.OK_no_data storage.delete_key(bucket, file), "xml"
+        else
+          Http.NotFound "no such bucket"
+        end
+      end
+  end
+end

data/lib/blobsterix/s3/s3_url_helper.rb

@@ -0,0 +1,93 @@
+module Blobsterix
+  module S3UrlHelper
+    HOST_PATH = /(\w+)(\.s3)?\.\w+\.\w+/
+    def bucket_matcher(str)
+      if str.include?("s3")
+        str.match(/(\w+)\.s3\.\w+\.\w+/)
+      else
+        str.match(/(\w+)\.\w+\.\w+/)
+      end
+    end
+
+    def favicon
+      @favicon ||= file.match /favicon/
+    end
+
+    def cache_upload
+      cache.put(cache_upload_key, env['rack.input'].read)
+    end
+
+    def cached_upload
+      cache_upload if not cache.exists?(cache_upload_key)
+      cache.get(cache_upload_key)
+    end
+
+    def cached_upload_clear
+      cache.delete(cache_upload_key)
+    end
+
+    def cache_upload_key
+      #@cache_upload_key ||= "upload/"+bucket.gsub("/", "_")+"_"+file.gsub("/", "_")
+      @cache_upload_key ||= Blobsterix::BlobAccess.new(:bucket => bucket, :id => "upload_#{file.gsub("/", "_")}")
+    end
+
+    def trafo_string
+      @trafo ||= env["HTTP_X_AMZ_META_TRAFO"] || ""
+    end
+
+    #TransformationCommand
+    def trafo(trafo_s='')
+      trafo_a = []
+      trafo_s.split(",").each{|command|
+        parts = command.split("_")
+        key = parts.delete_at(0)
+        trafo_a << [key, parts.join("_")]
+      }
+      trafo_a
+    end
+    
+    def bucket
+      host = bucket_matcher(env['HTTP_HOST'])
+      if host
+        host[1]
+      elsif  (env[nil] && env[nil][:bucket])
+        env[nil][:bucket]
+      elsif  (env[nil] && env[nil][:bucket_or_file])
+        if env[nil][:bucket_or_file].include?("/")
+          env[nil][:bucket_or_file].split("/")[0]
+        else
+          env[nil][:bucket_or_file]
+        end
+      else
+        "root"
+      end
+    end
+
+    def bucket?
+      host = bucket_matcher(env['HTTP_HOST'])
+      host || env[nil][:bucket] || included_bucket
+    end
+
+    def format
+      @format ||= env[nil][:format]
+    end
+
+    def included_bucket
+      if env[nil][:bucket_or_file] && env[nil][:bucket_or_file].include?("/")
+        env[nil][:bucket] = env[nil][:bucket_or_file].split("/")[0]
+        env[nil][:bucket_or_file] = env[nil][:bucket_or_file].gsub("#{env[nil][:bucket]}/", "")
+        true
+      else
+        false
+      end
+    end
+
+    def file
+      if format
+        [env[nil][:file] || env[nil][:bucket_or_file] || "", format].join(".")
+      else
+        env[nil][:file] || env[nil][:bucket_or_file] || ""
+      end
+    end
+  end
+end

data/lib/blobsterix/service.rb

@@ -0,0 +1,34 @@
+module Blobsterix
+  class Service < Goliath::API
+    use Goliath::Rack::Params
+=begin
+    def on_headers(env, headers)
+      env.logger.info 'received headers: ' + headers.inspect
+        env['async-headers'] = headers
+    end
+
+    def on_body(env, data)
+      env.logger.info 'received data: ' + data
+      (env['async-body'] ||= '') << data
+    end
+
+    def on_close(env)
+      env.logger.info 'closing connection'
+    end
+=end
+    def response(env)
+      call_stack(env, BlobApi, StatusApi, S3Api)
+    end
+
+    def call_stack(env, *apis)
+      last_answer = [404,{}, ""]
+      apis.each do |api|
+        last_answer = api.call(env)
+        if last_answer[0] != 600
+          return last_answer
+        end
+      end
+      last_answer[0] != 600 ? last_answer : [404,{}, ""]
+    end
+  end
+end

data/lib/blobsterix/status/status_api.rb

@@ -0,0 +1,62 @@
+module Blobsterix
+  class StatusApi < AppRouterBase
+    include StatusUrlHelper
+
+    get "/status(.:format)", :status
+
+    get "*any", :next_api
+    put "*any", :next_api
+    delete "*any", :next_api
+    head "*any", :next_api
+    post "*any", :next_api
+
+    json_var :cache_hits, :cache_misses, :cache_errors, :cache_accesses, :connections, :cache_hit_rate, :ram_usage, :uptime
+
+    def status
+      case format
+      when :json
+        render_json
+      when :xml
+        render_xml
+      else
+        render "status_page"
+      end
+    end
+
+    def ram_usage
+      `pmap #{Process.pid} | tail -1`[10,40].strip
+    end
+
+    def uptime
+      @uptime||=StatusInfo.uptime
+    end
+
+    def cache_hits
+      @cache_hits||=StatusInfo.cache_hit
+    end
+
+    def cache_hit_rate
+      if cache_hits > 0 && cache_accesses > 0
+        cache_hits.to_f/cache_accesses.to_f
+      else
+        1.to_f
+      end
+    end
+
+    def cache_misses
+      @cache_misses||=StatusInfo.cache_miss
+    end
+
+    def cache_errors
+      @cache_errors||=StatusInfo.cache_error
+    end
+
+    def cache_accesses
+      @cache_accesses||=StatusInfo.cache_access
+    end
+
+    def connections
+      @connections||=StatusInfo.connections
+    end
+  end
+end

data/lib/blobsterix/status/status_url_helper.rb

@@ -0,0 +1,11 @@
+module Blobsterix
+  module StatusUrlHelper
+    def format
+      if env[nil] && env[nil][:format]
+        env[nil][:format].to_sym
+      else
+        :html
+      end
+    end
+  end
+end

data/lib/blobsterix/storage/blob_meta_data.rb

@@ -0,0 +1,60 @@
+module Blobsterix
+  module Storage
+    class BlobMetaData
+      def check(key)
+        false
+      end
+      def etag
+        ""
+      end
+      def read
+        data()
+      end
+      def data
+        ""
+      end
+      def path
+        ""
+      end
+      def last_modified
+      end
+      def last_accessed
+      end
+      def payload
+        {}
+      end
+      def header
+        {}
+      end
+      def mimetype
+        "*/*"
+      end
+      def mediatype
+        "*"
+      end
+      def size
+        0
+      end
+      def accept_type
+        AcceptType.new
+      end
+      def valid
+        false
+      end
+      def valid?
+        valid
+      end
+      def write
+        if block_given?
+          #should yield file
+        end
+        self
+      end
+      def delete
+      end
+      def response(with_data=true, _etag=nil, env = nil, xfile = false, allow_chunks=true)
+        Blobsterix::Http::DataResponse.new(self, with_data, _etag, env).call(xfile, allow_chunks)
+      end
+    end
+  end
+end