azure_client 0.1.1 → 0.1.6

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    azure_client (0.0.9)
+    azure_client (0.1.6)
       rest-client
 
 GEM

data/lib/azure_client.rb

@@ -1,6 +1,7 @@
 # standard
 require 'json'
 require 'base64'
+require "zlib"
 
 # azure
 require 'azure'
@@ -10,6 +11,7 @@ require 'azure_client/buffered_queue'
 require 'azure_client/buffered_queue_factory'
 require 'azure_client/buffered_queue_message'
 require 'azure_client/client'
+require 'azure_client/compression'
 require 'azure_client/container'
 require 'azure_client/exponential_retry_policy'
 require 'azure_client/linear_retry_policy'

data/lib/azure_client/blob.rb

@@ -11,6 +11,10 @@ module AzureClient
     end
 
     def get_content
+      content_encoding = @blob_service.get_blob_properties(@container_name, @name).properties[:content_encoding]
+      if (content_encoding && content_encoding.downcase == "gzip")
+        return Compression.decompress(@content)
+      end
       @content
     end
 

data/lib/azure_client/buffered_queue.rb

@@ -9,7 +9,7 @@ module AzureClient
       @retry_policy = retry_policy
     end
     
-    def add_message(content, metadata = "", retry_policy = @retry_policy)
+    def add_message(content, metadata = "", options = {}, retry_policy = @retry_policy)
       payload = {"content" => content, "metadata" => metadata}.to_json
       queue = select_queue
       if is_allowed_payload_size(payload)
@@ -18,7 +18,7 @@ module AzureClient
         #pick random blob name
         blob_name = name + (0...9).map{ ('a'..'z').to_a[rand(26)] }.join
         reference = {"type" => "azure_blob_reference", "name" => blob_name, "metadata" => metadata}
-        @container.store_blob(blob_name, content, {}, retry_policy)
+        @container.store_blob(blob_name, content, options, retry_policy)
         queue.add_message(reference.to_json, retry_policy)
       end
     end

data/lib/azure_client/client.rb

@@ -14,16 +14,32 @@ module AzureClient
     end
 
     def get_container(name, retry_policy = ExponentialRetryPolicy.new(5,1,2))
-      if !(blob_service.list_containers.map {|container| container.name}.include?(name))
-        blob_service.create_container(name)
-      end
+      retry_policy.retry {
+        begin
+          blob_service.get_container_properties(name)
+        rescue Exception =>  e
+          if e.status_code == 404
+            blob_service.create_container(name)
+          else
+            raise e
+          end
+        end
+      }
       return Container.new(name, blob_service, retry_policy)
     end
 
     def get_table(name, retry_policy = ExponentialRetryPolicy.new(2,1,2))
-      if !(table_service.query_tables.map {|hash| hash[:properties]["TableName"]}.include?(name))
-        table_service.create_table(name)
-      end
+      retry_policy.retry {
+        begin
+          table_service.get_table(name)
+        rescue StandardError =>  e
+          if e.status_code == 404
+            table_service.create_table(name)
+          else
+            raise e
+          end
+        end
+      }
       return Table.new(name, table_service, retry_policy)
     end
 

data/lib/azure_client/compression.rb

@@ -0,0 +1,18 @@
+module AzureClient
+  class Compression
+
+    def self.compress(content)
+      stream = StringIO.new("w")
+      gz = Zlib::GzipWriter.new(stream)
+      gz.write(content)
+      gz.close
+      compressed = stream.string 
+    end
+
+    def self.decompress(compressed)
+      gz = Zlib::GzipReader.new(StringIO.new(compressed))
+      content = gz.read
+    end
+
+  end
+end

data/lib/azure_client/container.rb

@@ -1,5 +1,8 @@
 module AzureClient
   class Container
+    BLOB_SIZE_LIMIT = 1024 * 1024 * 60 # actual limit is 64 MB
+    BLOCK_SIZE_LIMIT = 1024 * 1024 * 3 # actual limit is 4 MB
+
     attr_reader :name
 
     def initialize(name, blob_service, retry_policy)
@@ -7,12 +10,22 @@ module AzureClient
       @blob_service = blob_service
       @retry_policy = retry_policy
     end
+    
+    def list_blobs(options = {}, retry_policy = @retry_policy)
+      retry_policy.retry {
+        @blob_service.list_blobs(name, options)
+      }
+    end
 
     #will overwrite content if blob with same name already exists
     def store_blob(blob_name, content, options = {}, retry_policy = @retry_policy)
-      retry_policy.retry {
-        @blob_service.create_block_blob(name, blob_name, content, options)
-      }
+      if content.kind_of?(String)
+        store_string_to_blob(blob_name, content, options, retry_policy)
+      elsif content.kind_of?(IO)
+        store_io_to_blob(blob_name, content, options, retry_policy)
+      else
+        raise "Trying to upload unknown type #{content.class} into Azure, only String/IO allowed!"
+      end
     end
 
     #throws exception if no blob found
@@ -57,25 +70,73 @@ module AzureClient
       }
     end
 
-    def poll_blob(blob_name, key, val, timeout = 10800)
-      time = 0
+    def poll_blob(blob_name, key, val, polling_policy = ExponentialRetryPolicy.new(75,5,1.1))
+      start_time = Time.now
       #create blob if it does not exist
       begin
         blob = get_blob(blob_name)
       rescue
         store_blob(blob_name,"")
       end
-      loop do 
-        sleep(5)
-        time += 5;
-        puts "*** polling Azure blob #{blob_name} for #{time} seconds"
+
+      polling_policy.retry {
+        puts "*** polling Azure blob #{blob_name} for #{Time.now - start_time} seconds"
         metadata = get_blob_metadata(blob_name)
-        return val if metadata && metadata[key] == val
-        if time > timeout
-          raise "Polling blob #{blob_name} took more than #{timeout} seconds to run. Time out."
+        if metadata && metadata[key] == val then
+          return val
+        else
+          raise "Polled the blob #{blob_name} for #{Time.now - start_time} seconds. Time out!"
+        end
+      }
+    end
+
+    private
+
+    def store_string_to_blob(blob_name, content, options = {}, retry_policy = @retry_policy)
+      if options[:compressed]
+        content = Compression.compress(content) 
+        options.merge!(:content_encoding => "gzip")
+      end
+
+      #use block list if content is too big
+      if content.length > BLOB_SIZE_LIMIT
+        block_number = content.length / BLOCK_SIZE_LIMIT 
+        block_list = (0..block_number).map{|n| [pad_zero(n), :latest]}
+        block_list.each do |block|
+          retry_policy.retry {
+            @blob_service.create_blob_block(name, blob_name, block[0], content[n*BLOCK_SIZE_LIMIT..(n+1)*BLOCK_SIZE_LIMIT-1], options)
+          }
         end
+        retry_policy.retry {
+          @blob_service.commit_blob_blocks(name, blob_name, block_list, options)  
+        }
+      else
+        retry_policy.retry {
+          @blob_service.create_block_blob(name, blob_name, content, options)
+        }
       end
     end
 
+    def store_io_to_blob(blob_name, io, options = {}, retry_policy = @retry_policy)
+      #always use block list if content is too big
+      block_id = 0
+      block_list = []
+      while !io.eof?
+        retry_policy.retry {
+          str = io.read(BLOCK_SIZE_LIMIT)
+          @blob_service.create_blob_block(name, blob_name, pad_zero(block_id), str, options)
+        }
+        block_list << [pad_zero(block_id), :latest]
+        block_id += 1
+      end
+
+      retry_policy.retry {
+        @blob_service.commit_blob_blocks(name, blob_name, block_list, options)  
+      }
+    end
+
+    def pad_zero(number)
+      "%06d" % number
+    end
   end
 end

data/lib/azure_client/version.rb

@@ -1,3 +1,3 @@
 module AzureClient
-  VERSION = "0.1.1"
+  VERSION = "0.1.6"
 end

data/spec/azure_client/container_spec.rb

@@ -0,0 +1,75 @@
+require File.join(File.dirname(__FILE__), '..', 'spec_helper')
+
+describe 'Container' do
+  before(:each) do
+    @storage = get_client
+    @container_name = (0...9).map{ ('a'..'z').to_a[rand(26)] }.join
+    @container = @storage.get_container(@container_name)
+  end
+
+  after(:each) do
+    @container.delete
+  end
+
+  it 'should add have a name' do
+    @container.name.should == @container_name
+  end
+  
+  it 'stores and gets blobs' do
+    @container.store_blob("blob1", "blob1_content")
+    blob = @container.get_blob("blob1")
+    blob.get_content.should == "blob1_content"
+  end
+
+  it 'stores and gets compressed blobs' do
+    container = @storage.get_container("test")
+    container.store_blob("blob1", "blob_content", {:compressed => true})
+    blob = container.get_blob("blob1")
+    blob.get_content.should == "blob_content"
+  end
+
+#  it 'deletes blobs' do
+#    @container.store_blob("blob1", "blob1_content")
+#    @container.delete_blob("blob1")
+#    expect { @container.get_blob("blob1") }.to raise_error
+#  end
+#
+#  it 'acquires leases' do
+#    @container.store_blob("blob1", "blob1_content")
+#    lease = @container.get_blob_lease('blob1')
+#    lease.should_not be_nil
+#    properties =  @container.get_blob_properties('blob1')
+#    properties[:lease_status].should == 'locked'
+#    properties[:lease_state].should == 'leased'
+#    expect { @container.get_blob_lease('blob1') }.to raise_error
+#  end
+#
+#  it 'retrieves blob properties' do
+#    @container.store_blob('blob1', 'blob1_content')
+#    properties = nil
+#    expect { properties = @container.get_blob_properties('blob1') }.to_not raise_error
+#    properties.should respond_to(:[])
+#  end
+#
+#  it 'sets blob metadata' do
+#    @container.store_blob('blob1', 'blob1_content')
+#    metadata = { meta: 'data' }
+#    expect { @container.set_blob_metadata('blob1', metadata) }.to_not raise_error
+#  end
+#
+#  it 'retrieves set metadata' do
+#    @container.store_blob('blob1', 'blob1_content')
+#    metadata = { meta: 'data' }
+#    @container.set_blob_metadata('blob1', metadata)
+#    @container.get_blob_metadata('blob1')['meta'].should == 'data'
+#  end
+#
+#  it 'polls blobs' do
+#    res = ""
+#    Thread.new { res = @container.poll_blob("blob2","state","Ready") }
+#    sleep(6.0)
+#    @container.set_blob_metadata("blob2", {"state" => "Ready"})
+#    sleep (6.0)
+#    res.should == "Ready"
+#  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: azure_client
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.6
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-31 00:00:00.000000000 Z
+date: 2013-12-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -94,6 +94,7 @@ files:
 - lib/azure_client/buffered_queue_factory.rb
 - lib/azure_client/buffered_queue_message.rb
 - lib/azure_client/client.rb
+- lib/azure_client/compression.rb
 - lib/azure_client/container.rb
 - lib/azure_client/exponential_retry_policy.rb
 - lib/azure_client/linear_retry_policy.rb
@@ -102,6 +103,7 @@ files:
 - lib/azure_client/table.rb
 - lib/azure_client/table_entity.rb
 - lib/azure_client/version.rb
+- spec/azure_client/container_spec.rb
 homepage: ''
 licenses: []
 post_install_message: 
@@ -126,4 +128,5 @@ rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: ''
-test_files: []
+test_files:
+- spec/azure_client/container_spec.rb