blobsterix 0.0.9

data/spec/lib/service_spec.rb

@@ -0,0 +1,12 @@
+require "spec_helper"
+
+describe  Blobsterix::Service do
+  include Goliath::TestHelper
+  it "it should route to the status page" do
+    with_api( Blobsterix::Service) do |a|
+      resp = get_request(:path=>"/status.json") do
+      end
+      expect(JSON.parse(resp.response)).to include("cache_hits")
+    end
+  end
+end

data/spec/lib/status/status_api_spec.rb

@@ -0,0 +1,42 @@
+require "spec_helper"
+
+describe Blobsterix::StatusApi do
+  include Rack::Test::Methods
+  def app
+    Blobsterix::StatusApi
+  end
+  describe 'GET /status' do
+    it 'should get status as json' do
+      get "/status.json"
+      expect(last_response.status).to eql(200)
+      body = JSON.parse(last_response.body)
+      expect(body).to have_key("cache_accesses")
+      expect(body).to have_key("cache_errors")
+      expect(body).to have_key("cache_hit_rate")
+      expect(body).to have_key("cache_hits")
+      expect(body).to have_key("cache_misses")
+      expect(body).to have_key("connections")
+      expect(body).to have_key("ram_usage")
+      expect(body).to have_key("uptime")
+    end
+    it 'should get status as xml' do
+      get "/status.xml"
+      expect(last_response.status).to eql(200)
+      body = Hash.from_xml(last_response.body)
+      expect(body).to have_key(:BlobsterixStatus)
+      expect(body[:BlobsterixStatus]).to have_key(:cache_accesses)
+      expect(body[:BlobsterixStatus]).to have_key(:cache_errors)
+      expect(body[:BlobsterixStatus]).to have_key(:cache_hit_rate)
+      expect(body[:BlobsterixStatus]).to have_key(:cache_hits)
+      expect(body[:BlobsterixStatus]).to have_key(:cache_misses)
+      expect(body[:BlobsterixStatus]).to have_key(:connections)
+      expect(body[:BlobsterixStatus]).to have_key(:ram_usage)
+      expect(body[:BlobsterixStatus]).to have_key(:uptime)
+    end
+
+    it 'should get status as html' do
+      get "/status"
+      expect(last_response.status).to eql(200)
+    end
+  end
+end

data/spec/lib/storage/cache_spec.rb

@@ -0,0 +1,135 @@
+require "spec_helper"
+
+describe Blobsterix::Storage::Cache do
+  include Blobsterix::SpecHelper
+
+  let(:data) {"Hi my name is Test"}
+  let(:key) {"test.txt"}
+  let(:bucket) {"test"}
+
+  let(:blob_access) {Blobsterix::BlobAccess.new(:bucket => bucket, :id => key)}
+  let(:blob_access_1) {Blobsterix::BlobAccess.new(:bucket => bucket, :id => key, :trafo => [["test", ""]])}
+  let(:blob_access_2) {Blobsterix::BlobAccess.new(:bucket => bucket, :id => key, :trafo => [["dummy", ""]])}
+  let(:blob_access_3) {Blobsterix::BlobAccess.new(:bucket => bucket, :id => key, :trafo => [["test", ""],["dummy", ""]])}
+
+  describe "invalidation" do
+    before :each do
+      Blobsterix.cache.put(blob_access, data)
+      Blobsterix.cache.put(blob_access_1, data)
+      Blobsterix.cache.put(blob_access_2, data)
+      Blobsterix.cache.put(blob_access_3, data)
+    end
+
+    after :each do
+      clear_cache
+      Blobsterix.cache_checker=lambda{|blob_access, last_accessed_at, created_at|
+        false
+      }
+    end
+
+    it "should follow invalidation structure" do
+      Blobsterix.cache_checker=lambda{|blob_access_, last_accessed_at, created_at|
+        blob_access_1.equals?(blob_access_) || blob_access_3.equals?(blob_access_)
+      }
+      Blobsterix.cache.invalidation
+      expect(blob_access.get().valid).to eql(true)
+      expect(blob_access_1.get().valid).to eql(false)
+      expect(blob_access_2.get().valid).to eql(true)
+      expect(blob_access_3.get().valid).to eql(false)
+    end
+
+    it "should invalidate all" do
+      Blobsterix.cache_checker=lambda{|blob_access_, last_accessed_at, created_at|
+        true
+      }
+      Blobsterix.cache.invalidation
+      expect(blob_access.get().valid).to eql(false)
+      expect(blob_access_1.get().valid).to eql(false)
+      expect(blob_access_2.get().valid).to eql(false)
+      expect(blob_access_3.get().valid).to eql(false)
+    end
+
+    it "should invalidate none" do
+      Blobsterix.cache.invalidation
+      expect(blob_access.get().valid).to eql(true)
+      expect(blob_access_1.get().valid).to eql(true)
+      expect(blob_access_2.get().valid).to eql(true)
+      expect(blob_access_3.get().valid).to eql(true)
+    end
+  end
+
+  describe "cache" do
+    after :each do
+      clear_cache
+    end
+
+    it "should return invalid blob when key doesn't exist" do
+      expect(Blobsterix.cache.get(blob_access).valid).to be(false)
+    end
+
+    it "should return valid blob when key exists" do
+      Blobsterix.cache.put(blob_access, data)
+      metaData = Blobsterix.cache.get(blob_access)
+      expect(metaData.valid).to be(true)
+      expect(metaData.read).to eql(data)
+    end
+
+    it "should return invalid blob when key is invalidated" do
+      Blobsterix.cache.put(blob_access, data)
+      metaData = Blobsterix.cache.get(blob_access)
+      expect(metaData.valid).to be(true)
+      expect(metaData.read).to eql(data)
+
+      Blobsterix.cache.invalidate(blob_access)
+
+      expect(Blobsterix.cache.get(blob_access).valid).to be(false)
+    end
+
+    it "should return invalid blob when key is invalidated for all trafos" do
+      Blobsterix.cache.put(blob_access, data)
+      Blobsterix.cache.put(blob_access_1, data)
+
+      metaData = Blobsterix.cache.get(blob_access)
+      expect(metaData.valid).to be(true)
+      expect(metaData.read).to eql(data)
+
+      metaData = Blobsterix.cache.get(blob_access_1)
+      expect(metaData.valid).to be(true)
+      expect(metaData.read).to eql(data)
+
+      Blobsterix.cache.invalidate(blob_access)
+
+      expect(Blobsterix.cache.get(blob_access).valid).to be(false)
+      expect(Blobsterix.cache.get(blob_access_1).valid).to be(false)
+    end
+
+    it "should return invalid blob when key is invalidated for one trafos" do
+      Blobsterix.cache.put(blob_access, data)
+      Blobsterix.cache.put(blob_access_1, data)
+
+      metaData = Blobsterix.cache.get(blob_access)
+      expect(metaData.valid).to be(true)
+      expect(metaData.read).to eql(data)
+
+      metaData = Blobsterix.cache.get(blob_access_1)
+      expect(metaData.valid).to be(true)
+      expect(metaData.read).to eql(data)
+
+      Blobsterix.cache.invalidate(blob_access, true)
+
+      expect(Blobsterix.cache.get(blob_access).valid).to be(false)
+      expect(Blobsterix.cache.get(blob_access_1).valid).to be(true)
+    end
+
+    it "should return invalid blob when key is deleted" do
+      Blobsterix.cache.put(blob_access, data)
+      metaData = Blobsterix.cache.get(blob_access)
+      expect(metaData.valid).to be(true)
+      expect(metaData.read).to eql(data)
+
+      Blobsterix.cache.delete(blob_access)
+
+      expect(Blobsterix.cache.get(blob_access).valid).to be(false)
+    end
+  end
+end

data/spec/lib/storage/file_system_spec.rb

@@ -0,0 +1,84 @@
+require "spec_helper"
+
+describe Blobsterix::Storage::FileSystem do
+  include Blobsterix::SpecHelper
+
+  let(:data) {"Hi my name is Test"}
+  let(:key) {"test.txt"}
+  let(:bucket) {"test"}
+
+
+
+  describe "bucket" do
+    after :each do
+      clear_storage
+    end
+
+    it "should return error if bucket doesn't exist" do
+      response = Hash.from_xml Blobsterix.storage.list(bucket).to_xml
+      expect(response).to have_key(:Error)
+      expect(response[:Error]).to eql("no such bucket")
+    end
+
+    it "should return the bucket info if it exists" do
+      Blobsterix.storage.create(bucket)
+      response = Hash.from_xml Blobsterix.storage.list(bucket).to_xml
+      expect(response).to have_key(:ListBucketResult)
+      expect(response[:ListBucketResult]).to have_key(:Name)
+      expect(response[:ListBucketResult][:Name]).to eql(bucket)
+    end
+
+    it "should return error after bucket is destroyed" do
+      Blobsterix.storage.create(bucket)
+      response = Hash.from_xml Blobsterix.storage.list(bucket).to_xml
+      expect(response).to have_key(:ListBucketResult)
+      expect(response[:ListBucketResult]).to have_key(:Name)
+      expect(response[:ListBucketResult][:Name]).to eql(bucket)
+
+      Blobsterix.storage.delete(bucket)
+      response = Hash.from_xml Blobsterix.storage.list(bucket).to_xml
+      expect(response).to have_key(:Error)
+      expect(response[:Error]).to eql("no such bucket")
+    end
+
+    it "should return the key info if it exists when listing bucket" do
+      Blobsterix.storage.put(bucket, key, StringIO.new(data, "r"))
+      response = Hash.from_xml Blobsterix.storage.list(bucket).to_xml
+      expect(response).to have_key(:ListBucketResult)
+      expect(response[:ListBucketResult]).to have_key(:Name)
+      expect(response[:ListBucketResult][:Name]).to eql(bucket)
+      expect(response[:ListBucketResult][:Contents]).to_not be_empty
+      expect(response[:ListBucketResult][:Contents][:Key]).to eql(key)
+      expect(response[:ListBucketResult][:Contents][:MimeType]).to eql("text/plain")
+      expect(response[:ListBucketResult][:Contents][:Size]).to eql(data.length)
+      expect(response[:ListBucketResult][:Contents][:ETag]).to eql(Digest::MD5.hexdigest(data))
+    end
+  end
+  
+  describe "keys" do
+    after :each do
+      clear_storage
+    end
+
+    it "should return invalid meta blob when key doesn't exist" do
+      expect(Blobsterix.storage.get(bucket, key).valid).to be(false)
+    end
+
+    it "should return valid meta blob when key exist" do
+      Blobsterix.storage.put(bucket, key, StringIO.new(data, "r"))
+      metaData = Blobsterix.storage.get(bucket, key)
+      expect(metaData.valid).to be(true)
+      expect(metaData.read).to eql(data)
+    end
+
+    it "should return invalid meta blob when key is deleted" do
+      Blobsterix.storage.put(bucket, key, StringIO.new(data, "r"))
+      metaData = Blobsterix.storage.get(bucket, key)
+      expect(metaData.valid).to be(true)
+      expect(metaData.read).to eql(data)
+
+      Blobsterix.storage.delete_key(bucket, key)
+      expect(Blobsterix.storage.get(bucket, key).valid).to be(false)
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,139 @@
+require 'simplecov'
+SimpleCov.start do 
+  command_name "Tests"
+end
+
+require 'bundler'
+
+Bundler.setup
+Bundler.require
+
+require 'rack/test'
+require 'goliath/test_helper'
+
+Blobsterix.storage_dir=Blobsterix.root.join("tmp/contents")
+Blobsterix.cache_dir=Blobsterix.root.join("tmp/cache")
+Blobsterix.storage_event_listener=lambda{|a,b|}
+Blobsterix.logger=Logger.new(Blobsterix.root.join("spec.log"))
+
+module Blobsterix
+  module SpecHelper
+    def clear_storage
+      FileUtils.rm_rf Dir.glob(Blobsterix.storage_dir.join("**/*"))
+    end
+    def clear_cache
+      FileUtils.rm_rf Dir.glob(Blobsterix.cache_dir.join("**/*"))
+    end
+    def clear_data
+      clear_cache
+      clear_storage
+    end
+    def run_em(&block)
+      EM.run {
+        f = Fiber.new(&block)
+        f.resume
+        EM.stop
+      }
+    end
+    class DummyTrafo < Blobsterix::Transformations::Transformation
+      def name()
+        "dummy"
+      end
+      def input_type()
+        @input_type ||= Blobsterix::AcceptType.new "text/plain"
+      end
+
+      def output_type()
+        @output_type ||= Blobsterix::AcceptType.new "text/plain"
+      end
+
+      def transform(input_path, target_path, value)
+        File.open(target_path, "w+") {|file|
+          file.write("#{value||"dummy"}")
+        }
+      end
+    end
+  end
+end
+
+class Nokogiri::XML::Node
+  TYPENAMES = {1=>'element',2=>'attribute',3=>'text',4=>'cdata',8=>'comment'}
+  def to_hash
+    {kind:TYPENAMES[node_type],name:name}.tap do |h|
+      h.merge! nshref:namespace.href, nsprefix:namespace.prefix if namespace
+      h.merge! text:text
+      h.merge! attr:attribute_nodes.map(&:to_hash) if element?
+      h.merge! kids:children.map(&:to_hash) if element?
+    end
+  end
+end
+class Nokogiri::XML::Document
+  def to_hash; root.to_hash; end
+end
+
+class Hash
+  class << self
+    def from_xml(xml_io) 
+      begin
+        result = Nokogiri::XML(xml_io)
+        return { result.root.name.to_sym => xml_node_to_hash(result.root)} 
+      rescue Exception => e
+        # raise your custom exception here
+      end
+    end 
+ 
+    def xml_node_to_hash(node) 
+      # If we are at the root of the document, start the hash 
+      if node.element?
+        result_hash = {}
+        if node.attributes != {}
+          result_hash[:attributes] = {}
+          node.attributes.keys.each do |key|
+            result_hash[:attributes][node.attributes[key].name.to_sym] = prepare(node.attributes[key].value)
+          end
+        end
+        if node.children.size > 0
+          node.children.each do |child| 
+            result = xml_node_to_hash(child) 
+ 
+            if child.name == "text"
+              unless child.next_sibling || child.previous_sibling
+                return prepare(result)
+              end
+            elsif result_hash[child.name.to_sym]
+              if result_hash[child.name.to_sym].is_a?(Object::Array)
+                result_hash[child.name.to_sym] << prepare(result)
+              else
+                result_hash[child.name.to_sym] = [result_hash[child.name.to_sym]] << prepare(result)
+              end
+            else 
+              result_hash[child.name.to_sym] = prepare(result)
+            end
+          end
+ 
+          return result_hash 
+        else 
+          return result_hash
+        end 
+      else 
+        return prepare(node.content.to_s) 
+      end 
+    end          
+ 
+    def prepare(data)
+      (data.class == String && data.to_i.to_s == data) ? data.to_i : data
+    end
+  end
+  
+  def to_struct(struct_name)
+      Struct.new(struct_name,*keys).new(*values)
+  end
+end
+
+
+
+RSpec.configure do |c|
+  c.include Goliath::TestHelper, :example_group => {
+    :file_path => /spec\/integration/
+  }
+end

data/templates/app/Gemfile

@@ -0,0 +1,12 @@
+source 'https://rubygems.org'
+
+gem "blobsterix"
+
+group :development do
+  gem "pry"
+  gem "capistrano", "2.15.4"
+  gem "capistrano-ext", "1.2.1"
+  gem "rvm-capistrano", "~> 1.5.1"
+  gem 'whenever', '=0.8.2', :require => false
+  gem "railsless-deploy", :require => false
+end

data/templates/app/Rakefile

@@ -0,0 +1,7 @@
+desc "Invalidate the cache"
+task :invalidate_cache do
+  require "blobsterix"
+  include Blobsterix::ConfigLoader
+  load_blobsterix_config
+  Blobsterix.cache.invalidation
+end

data/templates/app/config.rb

@@ -0,0 +1,61 @@
+module Blobsterix
+
+  # # If this function is uncommented it should return an environment
+  # # this env will override the one supplied from the command line
+  # def self.env()
+  #   :development or :production or :test
+  # end
+
+  # # Use this method to set the port. (standard is 9000)
+  # def self.port()
+  #   9000
+  # end
+
+  # # Use this to set the address. (standard is all)
+  # def self.address()
+  #   "127.0.0.1"
+  # end
+
+  # # Override those methods if you want to change the storage and cache dirs from their default location
+  # # For a different way check the environments files
+  # def self.storage_dir
+  #   Blobsterix.root.join("contents")
+  # end
+  # def self.cache_dir
+  #   Blobsterix.root.join("cache")
+  # end
+
+  # # Override those if you want to change the default storage and cache system.
+  # # For a different way check the environments files
+  # def self.storage
+  #   @@storage ||= Storage::FileSystem.new(Blobsterix.storage_dir)
+  # end
+  # def self.cache
+  #   @@cache ||= Storage::Cache.new(Blobsterix.cache_dir)
+  # end
+
+
+  # # Override this method if you want to use a different transformation manager.
+  # # For a different way check the environments files
+  # # normally not needed
+  # def self.transformation
+  #   @@transformation ||= Blobsterix::Transformations::TransformationManager.new
+  # end
+
+  # # Override this method incase you expect the trafo string in a special format.
+  # # For a different way check the environments files
+  # # The return should be in format of: trafoName_value,trafoName_value,....
+  # # Example: scale_2,rotate_20
+  # def self.decrypt_trafo(trafo_string,logger)
+  #   trafo_string
+  # end
+
+  # # Use a specific storage and cache event listener. Check the environment files for a better description
+  # def self.storage_event_listener
+  #   @storage_event_listener||=lambda{|target, blob_access|
+  #     logger.info("#{target}: #{blob_access}")
+  #   }
+  # end
+end
+
+require Blobsterix.root.join('config','environments',"#{Blobsterix.env}.rb")