blobsterix 0.0.9

data/lib/blobsterix/storage/bucket.rb

@@ -0,0 +1,36 @@
+module Blobsterix
+  module Storage
+    class Bucket
+      attr_accessor :name, :creation_date, :contents
+      def initialize(name, date)
+        @name = name
+        @creation_date = date
+        @contents = []
+        yield self if block_given?
+      end
+      def to_xml()
+        date = Date.today
+        xml = Nokogiri::XML::Builder.new do |xml|
+            xml.ListBucketResult(:xmlns => "http://doc.s3.amazonaws.com/#{date.year}-#{date.month}-#{date.day}") {
+              xml.Name name
+              xml.Prefix
+              xml.Marker
+              xml.MaxKeys 1000
+              xml.IsTruncated false
+                contents.each{|entry|
+                  entry.insert_xml(xml)
+                }
+            }
+
+        end
+        xml.to_xml
+      end
+      def insert_xml(xml)
+        xml.Bucket{
+          xml.Name name
+          xml.CreationDate creation_date
+        }
+      end
+    end
+  end
+end

data/lib/blobsterix/storage/bucket_entry.rb

@@ -0,0 +1,29 @@
+module Blobsterix
+  module Storage
+    class BucketEntry
+      attr_accessor :key, :last_modified, :etag, :size, :storage_class, :mimetype, :fullpath
+      def initialize(key)
+        @key = key
+        @last_modified = "2009-10-12T17:50:30.000Z"
+        @etag = ""fba9dede5f27731c9771645a39863328""
+        @size = "0"
+        @storage_class = "STANDARD"
+        @mimetype = "none"
+        @fullpath = ""
+        yield self if block_given?
+      end
+
+      def insert_xml(xml)
+        xml.Contents{
+          xml.Key key
+          xml.LastModified last_modified
+          xml.ETag etag
+          xml.Size size
+          xml.StorageClass storage_class
+          xml.MimeType mimetype
+          xml.FullPath fullpath
+        }
+      end
+    end
+  end
+end

data/lib/blobsterix/storage/bucket_list.rb

@@ -0,0 +1,26 @@
+module Blobsterix
+  module Storage
+    class BucketList
+      attr_accessor :buckets
+
+      def initialize()
+        @buckets = []
+        yield self if block_given?
+      end
+
+      def to_xml()
+        date = Date.today
+        xml = Nokogiri::XML::Builder.new do |xml|
+          xml.ListAllMyBucketsResult(:xmlns => "http://doc.s3.amazonaws.com/#{date.year}-#{date.month}-#{date.day}") {
+            xml.Buckets {
+              buckets.each{|entry|
+                entry.insert_xml(xml)
+              }
+            }
+          }
+        end
+        xml.to_xml
+      end
+    end
+  end
+end

data/lib/blobsterix/storage/cache.rb

@@ -0,0 +1,90 @@
+module Blobsterix
+  module Storage
+    class Cache
+      include Blobsterix::Logable
+
+      def invalidation
+        each_meta_file do |meta_file|
+          blob_access=meta_to_blob_access(meta_file)
+          if Blobsterix.cache_checker.call(blob_access,meta_file.last_accessed,meta_file.last_modified)
+            invalidate(blob_access, true)
+          end
+        end
+      end
+      def initialize(path)
+        @path = Pathname.new(path)
+        FileUtils.mkdir_p(@path) if !Dir.exist?(@path)
+      end
+
+      def get(blob_access)
+        FileSystemMetaData.new(cache_file_path(blob_access))
+      end
+
+      def put(blob_access, data)
+        FileSystemMetaData.new(cache_file_path(blob_access),:bucket => blob_access.bucket, :id => blob_access.id, :trafo => blob_access.trafo, :accept_type => "#{blob_access.accept_type}").write() {|f|
+          f.write(data)
+        }
+      end
+
+      def delete(blob_access)
+        FileSystemMetaData.new(cache_file_path(blob_access)).delete if exists?(blob_access)
+      end
+
+      def exists?(blob_access)
+        valid = File.exist?(cache_file_path(blob_access))
+        valid ? Blobsterix.cache_hit(blob_access) : Blobsterix.cache_miss(blob_access)
+        valid
+      end
+
+      #invalidates all!!! formats of a blob_access
+      def invalidate(blob_access, delete_single=false)
+        if delete_single
+          FileSystemMetaData.new(cache_file_path(blob_access)).delete
+        else
+          cache_path(blob_access).entries.each {|cache_file|
+            unless cache_file.to_s.match(/\.meta$/) || cache_file.directory?
+              FileSystemMetaData.new(cache_path(blob_access).join(cache_file)).delete if cache_file.to_s.match(blob_access.identifier)
+            end
+          } if Dir.exist?(cache_path(blob_access))
+        end
+      end
+
+      private
+
+      def each_meta_file
+        Dir.glob(@path.join("**/*")).each {|file|
+          cache_file = Pathname.new file
+          if block_given? && !cache_file.to_s.match(/\.meta$/) && !cache_file.directory?
+            yield FileSystemMetaData.new(cache_file)
+            cache_file
+          end
+        }
+      end
+
+      def meta_to_blob_access(meta_file)
+        BlobAccess.new(:bucket => meta_file.payload["bucket"], :id => meta_file.payload["id"], :trafo => meta_file.payload["trafo"], :accept_type => AcceptType.new(meta_file.payload["accept_type"]||""))
+      end
+
+      def cache_file_path(blob_access)
+        cache_path(blob_access).join(blob_access.identifier)
+      end
+
+      def cache_path(blob_access)
+        @path.join(hash_filename("#{blob_access.bucket}_#{blob_access.id.gsub("/","_")}"))
+      end
+
+      def hash_filename(filename)
+        hash = Murmur.Hash64B(filename)
+        bits =  hash.to_s(2)
+        parts = []
+        6.times { |index|
+          len = 11
+          len = bits.length if len >= bits.length
+          value = bits.slice!(0, len).to_i(2).to_s(16).rjust(3,"0")
+          parts.push(value)
+        }
+        parts.join("/")
+      end
+    end
+  end
+end

data/lib/blobsterix/storage/file_system.rb

@@ -0,0 +1,132 @@
+require 'fileutils'
+
+module Blobsterix
+  module Storage
+    class FileSystem < Storage
+      include Blobsterix::Logable
+
+      def initialize(path)
+        logger.info "Create FileSystem at #{path}"
+        @contents = path
+        FileUtils.mkdir_p(@contents) if !Dir.exist?(@contents)
+      end
+
+      def bucket_exist(bucket="root")
+        Dir.entries(contents).include?(bucket) and File.directory?(File.join(contents,bucket))
+      end
+
+      def list(bucket="root")
+        if bucket =~ /root/
+          BucketList.new do |l|
+
+            Dir.entries("#{contents}").each{|dir|
+              l.buckets << Bucket.new(dir, time_string_of(dir)) if File.directory? File.join("#{contents}",dir) and !(dir =='.' || dir == '..') 
+            }
+          end
+        else
+          if bucket_exist(bucket)
+            b = Bucket.new(bucket, time_string_of(bucket))
+            bucket_files(bucket).each do |file|
+              b.contents << BucketEntry.new(file) do |entry|
+                meta = metaData(bucket, file)
+                entry.last_modified =  meta.last_modified.strftime("%Y-%m-%dT%H:%M:%S.000Z")
+                entry.etag =  meta.etag
+                entry.size =  meta.size
+                entry.mimetype = meta.mimetype
+                entry.fullpath = contents(bucket, file).gsub("#{contents}/", "")
+              end
+            end
+            b
+          else
+            Nokogiri::XML::Builder.new do |xml|
+              xml.Error "no such bucket"
+            end
+          end
+        end
+      end
+
+      def get(bucket, key)
+        if (not File.directory?(contents(bucket, key))) and bucket_files(bucket).include?(key)
+          Blobsterix.storage_read(BlobAccess.new(:bucket => bucket, :id => key))
+          metaData(bucket, key)
+        else
+          Blobsterix.storage_read_fail(BlobAccess.new(:bucket => bucket, :id => key))
+          Blobsterix::Storage::BlobMetaData.new
+        end
+      end
+
+      def put(bucket, key, value)
+        Blobsterix.storage_write(BlobAccess.new(:bucket => bucket, :id => key))
+
+        meta = metaData(bucket, key).write() {|f| f.write(value.read) }
+
+        Blobsterix.cache.invalidate(Blobsterix::BlobAccess.new(:bucket => bucket, :id => key))
+        meta
+      end
+
+      def create(bucket)
+        logger.info "Storage: create bucket #{contents(bucket)}"
+        FileUtils.mkdir_p(contents(bucket)) if not File.exist?(contents(bucket))
+
+        Nokogiri::XML::Builder.new do |xml|
+        end.to_s
+      end
+
+      def delete(bucket)
+        logger.info "Storage: delete bucket #{contents(bucket)}"
+
+        FileUtils.rm_rf(contents(bucket)) if bucket_exist(bucket) && bucket_files(bucket).empty?
+
+        #Dir.rmdir(contents(bucket)) if bucket_exist(bucket) && bucket_files(bucket).empty?
+      end
+
+      def delete_key(bucket, key)
+        Blobsterix.storage_delete(BlobAccess.new(:bucket => bucket, :id => key))
+        Blobsterix.cache.invalidate(Blobsterix::BlobAccess.new(:bucket => bucket, :id => key))
+
+        metaData(bucket, key).delete if bucket_files(bucket).include? key
+      end
+
+      private
+        def contents(bucket=nil, key=nil)
+          if bucket
+            key ? File.join(@contents, bucket, map_filename(key.gsub("/", "\\"))) : File.join(@contents, bucket)
+          else
+            @contents
+          end
+        end
+
+        def map_filename(filename)
+          hash = Murmur.Hash64B(filename)
+          bits =  hash.to_s(2)
+          parts = []
+          6.times { |index|
+            len = 11
+            len = bits.length if len >= bits.length
+            value = bits.slice!(0, len).to_i(2).to_s(16).rjust(3,"0")
+            parts.push(value)
+          }
+          parts.push(filename)
+          parts.join("/")
+        end
+
+        def bucket_files(bucket)
+          if (bucket_exist(bucket))
+            Dir.glob("#{contents}/#{bucket}/**/*").select{|e| !File.directory?(e) and not e.end_with?(".meta")}.map{ |e|
+              e.gsub("#{contents}/#{bucket}/","").gsub(/\w+\/\w+\/\w+\/\w+\/\w+\/\w+\//, "").gsub("\\", "/")
+            }
+          else
+            []
+          end
+        end
+
+        def metaData(bucket, key)
+          Blobsterix::Storage::FileSystemMetaData.new(contents(bucket, key))
+        end
+
+        def time_string_of(*file_name)
+          File.ctime("#{contents}/#{file_name.flatten.join("/")}").strftime("%Y-%m-%dT%H:%M:%S.000Z")
+        end
+    end
+  end
+end

data/lib/blobsterix/storage/file_system_meta_data.rb

@@ -0,0 +1,136 @@
+module Blobsterix
+  module Storage
+    class FileSystemMetaData < BlobMetaData
+    include Blobsterix::Logable
+
+      def initialize(path_, payload={})
+        @payload=payload
+        @path = path_
+        @last_modified = ""
+        load_meta_file
+      end
+
+      def to_s
+        @path
+      end
+
+      def check(key)
+        @key === key
+      end
+
+      def etag
+        if @last_modified === last_modified
+          @etag ||= Digest::MD5.hexdigest(data)
+        else
+          @last_modified = last_modified
+          @etag = Digest::MD5.hexdigest(data)
+        end
+      end
+
+      def mimetype
+        (@mimetype ||= get_mime.type)
+      end
+
+      def mediatype
+        (@mediatype ||= get_mime.mediatype)
+      end
+
+      def data
+        # begin
+        #   raise StandardError.new
+        # rescue StandardError => e
+        #   puts "VERY EXPENSIVE FILE READ"
+        #   puts e.backtrace
+        # end
+        # logger.info "VERY EXPENSIVE FILE READ"
+        File.exists?(path) ? File.read(path) : ""
+      end
+
+      def path
+        @path
+      end
+
+      def size
+        @size ||= File.exists?(path) ? File.size(path) : 0
+      end
+
+      def last_modified
+        File.ctime(path)#.strftime("%Y-%m-%dT%H:%M:%S.000Z")
+      end
+
+      def last_accessed
+        File.atime(path)#.strftime("%Y-%m-%dT%H:%M:%S.000Z")
+      end
+
+      def accept_type
+        @accept_type ||= AcceptType.new(get_mime().to_s)
+      end
+
+      def header()
+        {"Etag" => etag, "Content-Type" => mimetype, "Last-Modified" => last_modified.strftime("%Y-%m-%dT%H:%M:%S.000Z"), "Cache-Control" => "max-age=#{60*60*24}", "Expires" => (Time.new+(60*60*24)).strftime("%Y-%m-%dT%H:%M:%S.000Z")}
+      end
+
+      def valid
+        File.exists?(path)
+      end
+
+      def payload
+        @payload
+      end
+
+      def write
+        if block_given?
+          FileUtils.mkdir_p(File.dirname(path))
+          delete
+          f = File.open(path, "wb")
+          yield f
+          f.close
+        end
+        save_meta_file
+        self
+      end
+
+      def delete
+        File.delete(meta_path) if File.exists?(meta_path)
+        File.delete(path) if valid
+      end
+
+      def to_json
+        as_json.to_json
+      end
+
+      def as_json
+        {'mimetype' => mimetype, 'mediatype' => mediatype, 'etag' => etag, 'size' => size,'payload' => @payload.to_json}
+      end
+
+      private
+        def meta_path
+          @meta_path ||= "#{path}.meta"
+        end
+
+        def get_mime
+          @mimeclass ||= (MimeMagic.by_magic(File.open(path)) if File.exists?(path) )|| MimeMagic.new("text/plain")
+        end
+        def save_meta_file
+          return if not valid
+
+          File.write(meta_path, to_json)
+        end
+        def load_meta_file
+          return if not valid
+
+          if not File.exists?(meta_path)
+            save_meta_file
+          else
+            data = JSON.load File.read(meta_path)
+            @mimetype = data["mimetype"]
+            @mediatype = data["mediatype"]
+            @etag = data["etag"]
+            @size = data["size"]
+            @payload = JSON.load(data["payload"]) || {}
+            @mimeclass = MimeMagic.new(@mimetype)
+          end
+        end
+    end
+  end
+end