blobsterix 0.0.9

data/lib/blobsterix.rb

@@ -0,0 +1,213 @@
+require "blobsterix/version"
+
+#libs
+require 'tempfile'
+require 'scanf'
+require 'nokogiri'
+require 'journey'
+#require 'vips'
+#require 'pry'
+require 'eventmachine'
+require 'em-synchrony'
+require 'mini_magick'
+#require 'mimemagic'
+#require 'ruby-webp'
+#require 'grape'
+#require 'evma_httpserver'
+require 'json'
+require 'logger'
+require 'erb'
+require 'goliath/api'
+
+#utility
+require 'blobsterix/mimemagic/tables'
+require 'blobsterix/mimemagic/version'
+require 'blobsterix/mimemagic/magic'
+
+#helper
+require 'blobsterix/helper/http'
+require 'blobsterix/helper/accept_type'
+require 'blobsterix/helper/data_response'
+require 'blobsterix/helper/murmur'
+require 'blobsterix/helper/logable'
+require 'blobsterix/helper/blob_access'
+require 'blobsterix/helper/status_info'
+require 'blobsterix/helper/template_renderer'
+require 'blobsterix/helper/config_loader'
+
+#router base
+require 'blobsterix/router/app_router'
+
+#helper
+require 'blobsterix/s3/s3_url_helper'
+require 'blobsterix/blob/blob_url_helper'
+require 'blobsterix/status/status_url_helper'
+
+#apis
+require 'blobsterix/s3/s3_api'
+require 'blobsterix/blob/blob_api'
+require 'blobsterix/status/status_api'
+
+#interfaces
+require 'blobsterix/storage/blob_meta_data'
+require 'blobsterix/storage/storage'
+require 'blobsterix/storage/cache'
+require 'blobsterix/storage/bucket_entry'
+require 'blobsterix/storage/bucket'
+require 'blobsterix/storage/bucket_list'
+require 'blobsterix/storage/storage'
+require 'blobsterix/transformation/transformation_manager'
+require 'blobsterix/transformation/transformation_chain'
+require 'blobsterix/transformation/transformation'
+
+#implementation
+require 'blobsterix/storage/file_system_meta_data'
+require 'blobsterix/storage/file_system'
+
+require 'blobsterix/transformation/image_transformation'
+
+#service base
+require 'blobsterix/service'
+
+BLOBSTERIX_ROOT=Dir.pwd
+BLOBSTERIX_GEM_DIR = File.join(File.dirname(__FILE__), "../")
+
+module Blobsterix
+  def self.root
+    @root ||= Pathname.new(BLOBSTERIX_ROOT)
+  end
+
+  def self.root_gem
+    @root_gem ||= Pathname.new(BLOBSTERIX_GEM_DIR)
+  end
+
+  def self.logger=(obj)
+    @logger=obj
+  end
+
+  def self.logger
+    @logger ||= Logger.new(STDOUT)
+  end
+
+  def self.storage_dir
+    @storage_dir||=root.join("contents")
+  end
+
+  def self.storage_dir=(obj)
+    @storage_dir=obj
+  end
+
+  def self.storage
+    @storage ||= Storage::FileSystem.new(storage_dir)
+  end
+
+  def self.storage=(obj)
+    @storage = obj
+  end
+
+  def self.cache_dir
+    @cache_dir||=root.join("cache")
+  end
+
+  def self.cache_dir=(obj)
+    @cache_dir=obj
+  end
+
+  def self.cache_original?
+    @cache_original||=false
+  end
+
+  def self.cache_original=(obj)
+    @cache_original=obj
+  end
+
+  def self.cache
+    @cache ||= Storage::Cache.new(cache_dir)
+  end
+
+  def self.cache=(obj)
+    @cache = obj
+  end
+
+  def self.decrypt_trafo=(obj)
+    @decrypt_trafo=obj
+  end
+
+  def self.decrypt_trafo(blob_access,trafo_string,logger)
+    @decrypt_trafo||=lambda{|b,t,l|t}
+    if !trafo_string
+      return @decrypt_trafo
+    end
+    @decrypt_trafo.call(blob_access, trafo_string, logger)
+  end
+
+  def self.transformation
+    @transformation ||= Blobsterix::Transformations::TransformationManager.new
+  end
+
+  def self.transformation=(obj)
+    @transformation=obj
+  end
+
+  def self.cache_checker=(obj)
+     @@cache_checker=obj
+  end
+
+  def self.cache_checker
+     @@cache_checker||=lambda{|blob_access, last_accessed_at, created_at|
+        false
+     }
+  end
+
+  def self.storage_event_listener=(obj)
+    @storage_event_listener=obj
+  end
+
+  def self.storage_event_listener
+    @storage_event_listener||=lambda{|target, hash|
+      logger.info("#{target}: #{hash.inspect}")
+    }
+  end
+
+  def self.event(name,hash)
+    storage_event_listener.call(name,hash)
+  end
+  
+  def self.encryption_error(blob_access)
+    event("encryption.error",:blob_access => blob_access)
+  end
+
+  def self.cache_miss(blob_access)
+    StatusInfo.cache_miss+=1
+    StatusInfo.cache_access+=1
+    event("cache.miss", :blob_access => blob_access)
+  end
+
+  def self.cache_fatal_error(blob_access)
+    StatusInfo.cache_error+=1
+    StatusInfo.cache_access+=1
+    event("cache.fatal_error", :blob_access => blob_access)
+  end
+
+  def self.cache_hit(blob_access)
+    StatusInfo.cache_hit+=1
+    StatusInfo.cache_access+=1
+    event("cache.hit",:blob_access => blob_access)
+  end
+
+  def self.storage_read(blob_access)
+    event("storage.read",:blob_access => blob_access)
+  end
+
+  def self.storage_read_fail(blob_access)
+    event("storage.read_fail",:blob_access => blob_access)
+  end
+
+  def self.storage_write(blob_access)
+    event("storage.write",:blob_access => blob_access)
+  end
+
+  def self.storage_delete(blob_access)
+    event("storage.delete",:blob_access => blob_access)
+  end
+end

data/lib/blobsterix/blob/blob_api.rb

@@ -0,0 +1,55 @@
+module Blobsterix
+  class BlobApi < AppRouterBase
+    include BlobUrlHelper
+
+    get "/blob/v1", :not_allowed
+    get "/blob", :not_allowed
+    put "/blob/v1", :not_allowed
+    put "/blob", :not_allowed
+
+    get "/blob/v1/(:trafo.)*bucket_or_file.:format", :get_file
+    get "/blob/v1/(:trafo.)*bucket_or_file", :get_file
+
+    head "/blob/v1/(:trafo.)*bucket_or_file.:format", :get_file_head
+    head "/blob/v1/(:trafo.)*bucket_or_file", :get_file_head
+
+    get "*any", :next_api
+    put "*any", :next_api
+    delete "*any", :next_api
+    head "*any", :next_api
+    post "*any", :next_api
+
+    private
+      def not_allowed
+        Http.NotAllowed "listing blob server not allowed"
+      end
+
+      def get_file(send_with_data=true)
+        accept = AcceptType.get(env, format)[0]
+
+        # check trafo encryption
+        trafo_string = Blobsterix.decrypt_trafo(BlobAccess.new(:bucket => bucket, :id => file), transformation_string, logger)
+        if !trafo_string
+          Blobsterix.encryption_error(BlobAccess.new(:bucket => bucket, :id => file))
+          return Http.NotAuthorized
+        end
+
+        
+        blob_access=BlobAccess.new(:bucket => bucket, :id => file, :accept_type => accept, :trafo => trafo(trafo_string))
+        Blobsterix.storage_event_listener.call("blob_api.get", :trafo_string => trafo_string, :blob_access => blob_access)
+
+        begin
+          data = transformation.run(blob_access)
+          send_with_data ? data.response(true, env["HTTP_IF_NONE_MATCH"], env, env["HTTP_X_FILE"] === "yes") : data.response(false)
+        rescue Errno::ENOENT => e
+          logger.error "Cache deleted: #{blob_access}"
+          Blobsterix.cache_fatal_error(blob_access)
+          Http.ServerError
+        end
+      end
+
+      def get_file_head
+        get_file(false)
+      end
+  end
+end

data/lib/blobsterix/blob/blob_url_helper.rb

@@ -0,0 +1,55 @@
+module Blobsterix
+  module BlobUrlHelper
+
+    def favicon
+      @favicon ||= file.match /favicon/
+    end
+
+    def bucket
+      if  (env[nil] && env[nil][:bucket])
+        env[nil][:bucket]
+      elsif included_bucket
+        env[nil][:bucket]
+      else
+        "root"
+      end
+    end
+
+    def format
+      @format ||= env[nil][:format]
+    end
+
+    def included_bucket
+      if env[nil][:bucket_or_file] && env[nil][:bucket_or_file].include?("/")
+        env[nil][:bucket] = env[nil][:bucket_or_file].split("/")[0]
+        env[nil][:bucket_or_file] = env[nil][:bucket_or_file].gsub("#{env[nil][:bucket]}/", "")
+        true
+      else
+        false
+      end
+    end
+
+    def file
+      if format
+        [env[nil][:file] || env[nil][:bucket_or_file] || "", format].join(".")
+      else
+        env[nil][:file] || env[nil][:bucket_or_file] || ""
+      end
+    end
+
+    #TransformationCommand
+    def trafo(trafo_s='')
+      trafo_a = []
+      trafo_s.split(",").each{|command|
+        parts = command.split("_")
+        key = parts.delete_at(0)
+        trafo_a << [key, parts.join("_")]
+      }
+      trafo_a
+    end
+
+    def transformation_string()
+      (env["params"] && env["params"]["trafo"]) || env[nil][:trafo] || ""
+    end
+  end
+end

data/lib/blobsterix/helper/accept_type.rb

@@ -0,0 +1,62 @@
+module Blobsterix
+  class AcceptType
+
+    def self.parse(header, format)
+      fields = (header||"").split(",")
+      fields << (MimeMagic.by_extension(format) || MimeMagic.new("*/*")).type if format
+      fields.map{|entry| AcceptType.new(entry.split(";"))}.sort {|a,b| b.score <=> a.score}
+    end
+
+    def self.get(env, format)
+      parse(env["HTTP_ACCEPT"], format)
+    end
+
+    def initialize(*data)
+      data = ["*/*"] if data.empty?
+      @mimetype = data.flatten[0]
+      set_q_factor_string(data.flatten[1] || "q=0.0")
+      mediatype
+      subtype
+      score
+    end
+
+    def to_s()
+      @mimetype.to_s
+    end
+
+    def type()
+      @mimetype
+    end
+
+    def mediatype()
+      @mediatype ||= @mimetype.split("/")[0]
+    end
+
+    def subtype()
+      @subtype ||= @mimetype.split("/")[1]
+    end
+
+    def score
+      @score ||= factor+(mediatype != "*" ? 1.0: 0.0)+(subtype != "*" ? 1.0: 0.0)
+    end
+
+    def factor
+      @q_factor
+    end
+
+    def is? other_type
+      mediatype === other_type.mediatype || mediatype === "*" || other_type.mediatype === "*"
+    end
+
+    def equal? other_type
+      mediatype === other_type.mediatype and subtype === other_type.subtype# and factor == other_type.factor
+    end
+
+    private
+    def set_q_factor_string(str)
+      str.scanf("%c=%f"){|char, num|
+        @q_factor = num if char === 'q'
+      }
+    end
+  end
+end

data/lib/blobsterix/helper/blob_access.rb

@@ -0,0 +1,73 @@
+module Blobsterix
+  class BlobAccess
+    attr_reader :bucket, :id,:trafo,:accept_type
+    attr_reader :source,:target
+
+    def initialize(atts={})
+      @trafo = []
+      atts.each do |key,value| send("#{key}=",value) end
+      identifier
+    end
+
+    def identifier
+       @identifier||= "#{bucket}_#{id.gsub("/","_")}_#{trafo.map {|trafo_pair|"#{trafo_pair[0]}_#{trafo_pair[1]}"}.join(",")}.#{subtype}"
+    end
+
+    def to_s
+      "BlobAccess: bucket(#{bucket}), id(#{id}), trafo(#{trafo}), accept_type(#{accept_type})"
+    end
+
+    def get
+      @meta||=find_blob
+    end
+
+    def equals?(blob_access)
+      identifier == blob_access.identifier
+    end
+
+    def copy
+      BlobAccess.new(:bucket => bucket, :id => id, :trafo => trafo, :accept_type => accept_type, :source => source, :target => target)
+    end
+
+    def reset!
+      @meta = nil
+      self
+    end
+
+    private
+
+    def find_blob
+      unless Blobsterix.cache.exists?(self)
+        if trafo.empty? || raw_trafo?
+          metaData = Blobsterix.storage.get(self.bucket, self.id)
+          if raw_trafo? || raw_accept_type?(metaData.accept_type)
+            # puts "Load from storage"
+            return metaData unless Blobsterix.cache_original?
+            Blobsterix.cache.put(BlobAccess.new(:bucket => bucket, :id => id), metaData.data) if metaData.valid?
+            return Blobsterix.cache.get(BlobAccess.new(:bucket => bucket, :id => id)) if metaData.valid?
+          else
+            # puts "accept type doesn't work"
+          end
+        else
+          # puts "trafo is not raw"
+        end
+      end
+      Blobsterix.cache.get(self)
+    end
+
+    def raw_trafo?
+      @raw_trafo||=(trafo.length == 1 && trafo[0][0]=="raw")
+    end
+
+    def raw_accept_type?(other)
+      @raw_accept_type||= (!accept_type || accept_type.equal?(other))
+    end
+
+
+    def subtype
+      accept_type ? accept_type.subtype : ""
+    end
+    attr_writer :bucket, :id,:trafo,:accept_type
+    attr_writer :source, :target
+  end
+end