blobsterix 0.0.9

data/lib/blobsterix/transformation/transformation.rb

@@ -0,0 +1,30 @@
+module Blobsterix::Transformations
+  class Transformation
+    include Blobsterix::Logable
+
+    def name
+      ""
+    end
+
+    def to_s
+      name
+    end
+
+    def is_format?
+      false
+    end
+
+    def input_type
+      Blobsterix::AcceptType.new
+    end
+
+    def output_type
+      Blobsterix::AcceptType.new
+    end
+
+    def transform(input_path, target_path, value)
+      logger.debug "run transformation!!!!!"
+      system("cp #{input_path} #{target_path}")
+    end
+  end
+end

data/lib/blobsterix/transformation/transformation_chain.rb

@@ -0,0 +1,78 @@
+module Blobsterix::Transformations
+  class TransformationChain
+    attr_reader :logger, :target_blob_access
+
+    def initialize(blob_access, input_data, logger)
+      @blob_access=blob_access
+      @target_blob_access=nil
+      @input_data = input_data
+      @transformations = []
+      @logger = logger
+    end
+
+    def cache
+      @cache||=Blobsterix.cache
+    end
+
+    def last_type()
+      return Blobsterix::AcceptType.new(@input_data.mimetype) if @transformations.empty?
+      @transformations.last[0].output_type
+    end
+
+    def add(transfo, value)
+      return if transfo == nil
+      @transformations << [transfo, value]
+    end
+
+    def do()
+
+      with_tempfiles do |keys|
+        last_key = "#{@input_data.path}"
+
+        begin
+          current_transformation = nil
+          @transformations.each{|trafo|
+
+            current_transformation = trafo
+
+            new_key = keys.delete_at(0)
+            trafo[0].transform(last_key, new_key, trafo[1])
+            last_key = new_key
+          }
+        rescue StandardError => e
+          logger.error "Transformation: #{current_transformation} failed with #{e.message}"
+          break
+        end
+
+        cache.put(@target_blob_access,Blobsterix::Storage::FileSystemMetaData.new(last_key).read)
+        @target_blob_access.reset!
+      end unless @target_blob_access.get.valid
+
+      @target_blob_access
+    end
+
+    def finish(accept_type, trafo)
+      if @transformations.empty? or (not @transformations.last[0].output_type.equal?(accept_type) and not @transformations.last[0].is_format?)
+        @transformations << [trafo, nil] if trafo != nil && trafo.is_format?
+      end
+      accept_type =  @transformations.empty? || !@transformations.last[0].is_format? ? nil : @transformations.last[0].output_type
+      @target_blob_access = Blobsterix::BlobAccess.new(:bucket => @blob_access.bucket, :id => @blob_access.id, :trafo => @transformations.map{|trafo,value| [trafo.name, value]}, :accept_type => accept_type)
+    end
+
+    private
+
+    def with_tempfiles
+      tmpFiles = @transformations.size.times.map{|index|
+        Tempfile.new("#{@blob_access.identifier}_#{index}")
+      }
+      keys = tmpFiles.map{|f| f.path }
+
+      yield keys
+
+      tmpFiles.each { |f|
+        f.close
+        f.unlink
+      }
+    end
+  end
+end

data/lib/blobsterix/transformation/transformation_manager.rb

@@ -0,0 +1,115 @@
+module Blobsterix::Transformations
+  #a TransormationManager cares about:
+  class TransformationManager
+    include Blobsterix::Logable
+
+    def initialize()
+      auto_load
+    end
+
+    def add(trafo)
+      transformation = (trafo.is_a?(String) ? ::Blobsterix::Transformations::Impl::const_get(trafo).new : trafo)
+      transformations << transformation if transformations.select{|trafo|trafo.name === transformation.name}.empty?
+      self
+    end
+
+    def run(blob_access)
+
+      blob_access = wait_for_transformation(blob_access) if transformation_in_progress?(blob_access)
+
+      return blob_access.get if blob_access.get.valid
+
+      cue_transformation(blob_access)
+
+      blob_access = run_transformation(blob_access)
+
+      blob_access.get.valid ? blob_access.get : Blobsterix::Storage::BlobMetaData.new
+    end
+
+    private
+      def running_transformations
+        @running_transformations ||= {}
+      end
+
+      def transformations
+        @transformations ||= []
+      end
+
+      def wait_for_transformation(blob_access)
+        running_transformations[blob_access.identifier] << Fiber.current
+        logger.debug "Transformation: wait for it to finish #{blob_access}"
+        Fiber.yield
+      end
+
+      def cue_transformation(blob_access)
+        running_transformations[blob_access.identifier] = [Fiber.current]
+        blob_access
+      end
+
+      def uncue_transformation(blob_access)
+        running_transformations.delete(blob_access.identifier)
+        blob_access
+      end
+
+      def transformation_in_progress?(blob_access)
+        running = running_transformations.has_key?(blob_access.identifier)
+        running
+      end
+
+      def auto_load()
+        Blobsterix::Transformations::Impl.constants.each{|c|
+          add(c.to_s)
+        }
+      end
+
+      def run_transformation(blob_access)
+        logger.debug "Transformation: build #{blob_access}"
+
+        metaData = blob_access.source || Blobsterix::BlobAccess.new(:bucket => blob_access.bucket,:id => blob_access.id).get
+
+        return uncue_transformation(blob_access) unless metaData.valid
+
+        chain = TransformationChain.new(blob_access, metaData, logger)
+
+        blob_access.trafo.each {|trafo_pair|
+          chain.add(findTransformation(trafo_pair[0], chain.last_type), trafo_pair[1])
+        }
+
+        chain.finish(blob_access.accept_type, findTransformation_out(chain.last_type, blob_access.accept_type))
+
+        if chain.target_blob_access.get.valid
+          uncue_transformation(blob_access)
+          chain.target_blob_access
+        else
+          logger.debug "Transformation: run #{blob_access}"
+          EM.defer(Proc.new {
+            chain.do()
+          }, Proc.new {|result|
+            finish_connection(result, blob_access)
+          })
+
+          Fiber.yield
+        end
+      end
+
+      def finish_connection(result, blob_access)
+        logger.debug "Transformation: done #{blob_access} finish connections"
+        running_transformations[blob_access.identifier].each{|fiber|
+          fiber.resume(result)
+        }
+        uncue_transformation(blob_access)
+      end
+
+      def findTransformation(name, input_type)
+        trafos = transformations.select{|trafo| trafo.name === name and trafo.input_type.is?(input_type)}
+        trafos.empty? ? nil : trafos[0]
+      end
+
+      def findTransformation_out(input_type, output_type)
+        trafos = transformations.select{|trafo|
+          trafo.input_type.is?(input_type) and trafo.output_type.equal?(output_type)
+        }
+        trafos.empty? ? nil : trafos[0]
+      end
+  end
+end

data/lib/blobsterix/version.rb

@@ -0,0 +1,3 @@
+module Blobsterix
+  VERSION = "0.0.9"
+end

data/scripts/download.rb

@@ -0,0 +1,30 @@
+#!/usr/bin/env ruby
+
+require 'net/http'
+
+uri = URI.parse("http://localhost:9000/blob/v1/rotate_25,ascii_250.images/expired.pngd")
+
+# Net::HTTP.start("localhost", 9000) do |http|
+#     resp = http.get("/images/syncview21force.png")
+#     open("/home/dsudmann/desktop/syncview21force.png", "wb") do |file|
+#         file.write(resp.body)
+#     end
+# end
+
+# Net::HTTP.start(uri.host,uri.port) do |http|
+#     resp = http.get(uri.path)
+#     open("/home/dsudmann/desktop/syncview21force.png", "wb") do |file|
+#         file.write(resp.body)
+#     end
+# end
+
+Net::HTTP.start(uri.host,uri.port) do |http|
+  open("/home/dsudmann/desktop/syncview21force", "wb") do |file|
+    http.request_get(uri.path){ |resp|
+      puts resp.code
+      resp.read_body{ |seg|
+        file << seg
+      }
+    }
+  end
+end

data/scripts/test

@@ -0,0 +1,6 @@
+ab -n 100 -c 50 http://localhost:3000/blob/v1/rotate_10.images/expired.png > cache/log_00 &
+ab -n 100 -c 50 http://localhost:3000/blob/v1/rotate_20.images/expired.png > cache/log_01 &
+ab -n 100 -c 50 http://localhost:3000/blob/v1/rotate_30.images/expired.png > cache/log_02 &
+ab -n 100 -c 50 http://localhost:3000/blob/v1/rotate_40.images/expired.png > cache/log_03 &
+ab -n 100 -c 50 http://localhost:3000/blob/v1/rotate_50.images/expired.png > cache/log_04 &
+ab -n 100 -c 50 http://localhost:3000/blob/v1/rotate_60.images/expired.png > cache/log_05 &

data/spec/lib/blob/blob_api_spec.rb

@@ -0,0 +1,81 @@
+require "spec_helper"
+
+describe Blobsterix::BlobApi do
+  include Rack::Test::Methods
+  def app
+    Blobsterix::BlobApi
+  end
+  describe 'GET /blob/v1/' do
+    it 'get several categories of repositories by name' do
+      get "/blob/v1/"
+      expect(last_response.status).to eql(403)
+    end
+  end
+
+  describe 'Transformed get' do
+    include Blobsterix::SpecHelper
+
+    context "with data" do
+      let(:data) {"Hi my name is Test"}
+      let(:data_transformed) {"Hi_my_name_is_Test_Transformed"}
+      let(:key) {"test.txt"}
+      let(:bucket) {"test"}
+
+      before :all do
+        Blobsterix.transformation.add Blobsterix::SpecHelper::DummyTrafo.new
+      end
+
+      after :all do
+        Blobsterix.transformation=Blobsterix::Transformations::TransformationManager.new
+      end
+
+      before :each do
+        Blobsterix.storage.put(bucket, key, StringIO.new(data, "r"))
+      end
+
+      after :each do
+        clear_data
+      end
+
+      it "should return the file" do
+        expect(Blobsterix.transformation).to receive(:cue_transformation).once.and_call_original
+        run_em do 
+          get "/blob/v1/dummy_#{data_transformed}.test/test.txt"
+        end
+        expect(last_response.status).to eql(200)
+        expect(last_response.body).to eql(data_transformed)
+      end
+
+      it "should return the file head" do
+        expect(Blobsterix.transformation).to receive(:cue_transformation).once.and_call_original
+        run_em do 
+          head "/blob/v1/dummy.test/test.txt"
+        end
+        expect(last_response.status).to eql(200)
+        expect(last_response.body).to eql("")
+      end
+
+      it "should return the file and wait for previous trafos to finish" do
+        expect(Blobsterix.transformation).to receive(:cue_transformation).once.and_call_original
+        expect(Blobsterix.transformation).to receive(:wait_for_transformation).once.and_call_original
+        run_em do 
+          get "/blob/v1/dummy_#{data_transformed}.test/test.txt"
+          get "/blob/v1/dummy_#{data_transformed}.test/test.txt"
+        end
+        expect(last_response.status).to eql(200)
+        expect(last_response.body).to eql(data_transformed)
+      end
+
+      it "should return the file and not wait for different trafos to finish" do
+        expect(Blobsterix.transformation).to receive(:cue_transformation).twice.and_call_original
+        expect(Blobsterix.transformation).to receive(:wait_for_transformation).never.and_call_original
+        run_em do 
+          get "/blob/v1/dummy_#{data_transformed},dummy_#{data_transformed}.test/test.txt"
+          get "/blob/v1/dummy_#{data_transformed}.test/test.txt"
+        end
+        expect(last_response.status).to eql(200)
+        expect(last_response.body).to eql(data_transformed)
+      end
+    end
+  end
+end

data/spec/lib/helper/blob_access_spec.rb

@@ -0,0 +1,72 @@
+require "spec_helper"
+
+describe Blobsterix::BlobAccess do
+  include Blobsterix::SpecHelper
+
+  let(:data) {"Hi my name is Test"}
+  let(:key) {"test.txt"}
+  let(:bucket) {"test"}
+
+  def blob_access
+    Blobsterix::BlobAccess.new(:bucket => bucket, :id => key)
+  end
+  def blob_access_1
+    Blobsterix::BlobAccess.new(:bucket => bucket, :id => key, :trafo => [["test", ""]])
+  end
+  def blob_access_raw
+    Blobsterix::BlobAccess.new(:bucket => bucket, :id => key, :trafo => [["raw", ""]])
+  end
+  def blob_access_same_accept_type
+    Blobsterix::BlobAccess.new(:bucket => bucket, :id => key, :accept_type => Blobsterix::AcceptType.new("text/plain"))
+  end
+  def blob_access_raw_same_accept_type
+    Blobsterix::BlobAccess.new(:bucket => bucket, :id => key, :trafo => [["raw", ""]], :accept_type => Blobsterix::AcceptType.new("text/plain"))
+  end
+
+  describe "blob_access" do
+    after :each do
+      clear_data
+    end
+
+    it "should return valid blob data when it exists" do
+      Blobsterix.storage.put(bucket, key, StringIO.new(data, "r"))
+      expect(blob_access.get().valid).to be(true)
+    end
+
+    it "should return invalid blob data when it does not exist" do
+      expect(blob_access.get().valid).to be(false)
+    end
+
+    it "should automaticly guess raw trafo" do
+      Blobsterix.storage.put(bucket, key, StringIO.new(data, "r"))
+      expect(blob_access.get().valid).to be(true)
+      expect(blob_access_raw.get().valid).to be(true)
+      expect(blob_access_same_accept_type.get().valid).to be(true)
+      expect(blob_access_raw_same_accept_type.get().valid).to be(true)
+    end
+
+    context "dont cache raw" do 
+      before :each do
+        Blobsterix.cache_original= false
+      end
+
+      it "should not create a cache entry for raw transforms" do
+        Blobsterix.storage.put(bucket, key, StringIO.new(data, "r"))
+        expect(blob_access.get().valid).to be(true)
+        expect(Blobsterix.cache.get(blob_access).valid).to be(false)
+      end
+    end
+
+    context "cache raw" do 
+      before :each do
+        Blobsterix.cache_original= true
+      end
+
+      it "should create a cache entry for raw transforms" do
+        Blobsterix.storage.put(bucket, key, StringIO.new(data, "r"))
+        expect(blob_access.get().valid).to be(true)
+        expect(Blobsterix.cache.get(blob_access).valid).to be(true)
+      end
+    end
+  end
+end

data/spec/lib/s3/s3_api_spec.rb

@@ -0,0 +1,183 @@
+require "spec_helper"
+
+describe Blobsterix::S3Api do
+  include Rack::Test::Methods
+  include Blobsterix::SpecHelper
+  def app
+    Blobsterix::S3Api
+  end
+
+  let(:data) {"Hi my name is Test"}
+  let(:key) {"test.txt"}
+  let(:bucket) {"test"}
+
+  after :each do
+      clear_data
+  end
+
+  describe "create a bucket" do
+    it "should have bucket after creation" do
+
+      run_em do
+        put "/", "", "HTTP_HOST" => "#{bucket}.s3.blah.de"
+      end
+
+      get "/#{bucket}"
+      expect(last_response.status).to eql(200)
+      response = Hash.from_xml last_response.body
+      expect(response).to_not have_key(:Error)
+    end
+  end
+
+  describe "upload" do
+
+    before :all do
+      Blobsterix.transformation.add Blobsterix::SpecHelper::DummyTrafo.new
+    end
+
+    after :all do
+      Blobsterix.transformation=Blobsterix::Transformations::TransformationManager.new
+    end
+
+    after :each do
+      clear_data
+    end
+
+    it "should have file in bucket after upload" do
+      #expect(Blobsterix.transformation).to receive(:cue_transformation).never.and_call_original
+
+      run_em do
+        put "/#{key}", data, {"HTTP_HOST" => "#{bucket}.s3.blah.de"}
+      end
+
+      expect(last_response.status).to eql(200)
+      expect(Blobsterix.storage.get(bucket, key).read).to eql(data)
+    end
+
+    it "should have file in bucket after upload with trafo" do
+      #expect(Blobsterix.transformation).to receive(:cue_transformation).once.and_call_original
+
+      run_em do
+        put "/#{key}", data, {"HTTP_HOST" => "#{bucket}.s3.blah.de", "HTTP_X_AMZ_META_TRAFO" => "dummy_Yeah"}
+      end
+
+      expect(last_response.status).to eql(200)
+      expect(Blobsterix.storage.get(bucket, key).read).to eql("Yeah")
+    end
+  end
+
+  context "with no data" do
+
+    describe "bucket" do
+      it 'should return an empty set' do
+        get "/"
+        expect(last_response.status).to eql(200)
+        response = Hash.from_xml last_response.body
+        expect(response).to have_key(:ListAllMyBucketsResult)
+        expect(response[:ListAllMyBucketsResult]).to have_key(:Buckets)
+        expect(response[:ListAllMyBucketsResult][:Buckets]).to be_empty
+      end
+
+      it 'should say no such bucket' do
+        get "/#{bucket}"
+        expect(last_response.status).to eql(200)
+        response = Hash.from_xml last_response.body
+        expect(response).to have_key(:Error)
+        expect(response[:Error]).to eql("no such bucket")
+      end
+
+      it "should return 404 on delete bucket" do
+        expect(Blobsterix.storage.bucket_exist(bucket)).to eql(false)
+
+        delete "/", "", "HTTP_HOST" => "s3.blah.de"
+
+        expect(last_response.status).to eql(404)
+        expect(Blobsterix.storage.bucket_exist(bucket)).to eql(false)
+      end
+    end
+
+    describe 'file' do
+      it "should return a 404 when bucket doesn't exist" do
+        get "/#{key}", "", "HTTP_HOST" => "#{bucket}.s3.blah.de"
+        expect(last_response.status).to eql(404)
+      end
+
+      it 'should return a 404 when bucket exists' do
+        Blobsterix.storage.create(bucket)
+        get "/#{key}", "", "HTTP_HOST" => "#{bucket}.s3.blah.de"
+        expect(last_response.status).to eql(404)
+        Blobsterix.storage.delete(bucket)
+      end
+    end
+  end
+
+  context "with data" do
+
+    before :each do
+      Blobsterix.storage.put(bucket, key, StringIO.new(data, "r"))
+    end
+
+    after :each do
+      clear_storage
+    end
+
+    describe "bucket" do
+
+      it 'should return one bucket in the list' do
+        get "/"
+        expect(last_response.status).to eql(200)
+        response = Hash.from_xml last_response.body
+        expect(response).to have_key(:ListAllMyBucketsResult)
+        expect(response[:ListAllMyBucketsResult]).to have_key(:Buckets)
+        expect(response[:ListAllMyBucketsResult][:Buckets]).to_not be_empty
+      end
+
+      it 'should return all files for the bucket' do
+        get "/#{bucket}"
+        expect(last_response.status).to eql(200)
+        response = Hash.from_xml last_response.body
+        expect(response).to have_key(:ListBucketResult)
+        expect(response[:ListBucketResult]).to have_key(:Name)
+        expect(response[:ListBucketResult][:Name]).to eql(bucket)
+        expect(response[:ListBucketResult][:Contents]).to_not be_empty
+        expect(response[:ListBucketResult][:Contents][:Key]).to eql(key)
+        expect(response[:ListBucketResult][:Contents][:MimeType]).to eql("text/plain")
+        expect(response[:ListBucketResult][:Contents][:Size]).to eql(data.length)
+        expect(response[:ListBucketResult][:Contents][:ETag]).to eql(Digest::MD5.hexdigest(data))
+      end
+
+      it "should delete bucket" do
+        Blobsterix.storage.delete_key(bucket, key)
+        expect(Blobsterix.storage.bucket_exist(bucket)).to eql(true)
+
+        delete "/", "", "HTTP_HOST" => "#{bucket}.s3.blah.de"
+
+        expect(last_response.status).to eql(204)
+        expect(Blobsterix.storage.bucket_exist(bucket)).to eql(false)
+      end
+
+    end
+
+    describe "file" do
+
+      it "should return file" do
+        get "/#{key}", "", "HTTP_HOST" => "#{bucket}.s3.blah.de"
+        expect(last_response.status).to eql(200)
+        expect(last_response.body).to eql(data)
+      end
+
+      it "should return file head" do
+        head "/#{key}", "", "HTTP_HOST" => "#{bucket}.s3.blah.de"
+        expect(last_response.status).to eql(200)
+        expect(last_response.body).to eql("")
+      end
+
+      it "should delete key" do
+        delete "/#{key}", "", "HTTP_HOST" => "#{bucket}.s3.blah.de"
+
+        expect(last_response.status).to eql(204)
+        expect(Blobsterix.storage.get(bucket, key).valid).to eql(false)
+      end
+    end
+  end
+end