azure-contrib 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 36da49c6b4377b239c91448549113c8758086380
-  data.tar.gz: aeaddf13a7d8ba562620c0a706fcf9dc5bcee053
+  metadata.gz: 59524c3c741127e9245d85ccec4ec4cd9f03fce4
+  data.tar.gz: 6dbcae31f84dea2d866683b45ddd7f2c36293d55
 SHA512:
-  metadata.gz: a4c366aa16cbec9af691d5f0871de427c0dc0fc591d69e90d80b6f14b0207bd35271748ef00f725c8a95091368d6c55c931971705876aa8b2ab32a6a01b2e944
-  data.tar.gz: f8639319d3bc0ce2893585c01b72704423a854ffe47eef3cd22bec9ed5f793d27fb1c92d2c86bfbb820901c682be4c7d98cee88515a88337554b8d9414423433
+  metadata.gz: c078b3c1a21432ab1c5a005d71d4e670993f04584da898f107271c5c7f7e9f43d4fa8d053aa41c59cb9666be45bd078b2ffbae796dcf3d2bc121741acbf21a24
+  data.tar.gz: 6eff77bffe8cb298521f03003db4003fd65b6b82bbc52e3b36a2e5ebee1fac2cf5338a262b85c1ff8bd0f6912b28f19cef1aeea0fbc2e5cfecec42ac90ecc8aa

data/.gitignore

@@ -15,3 +15,5 @@ spec/reports
 test/tmp
 test/version_tmp
 tmp
+*.iml
+*.idea

data/azure-contrib.gemspec

@@ -23,4 +23,5 @@ Gem::Specification.new do |spec|
   spec.add_dependency "azure"
   spec.add_dependency "hashie"
   spec.add_dependency "addressable"
+  spec.add_dependency "celluloid"
 end

data/azure-contrib.iml

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="RUBY_MODULE" version="4">
+  <component name="CompassSettings">
+    <option name="compassSupportEnabled" value="true" />
+  </component>
+  <component name="NewModuleRootManager" inherit-compiler-output="true">
+    <exclude-output />
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="jdk" jdkName="RVM: ruby-2.1.1" jdkType="RUBY_SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+    <orderEntry type="library" scope="PROVIDED" name="addressable (v2.3.6, RVM: ruby-2.1.1) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="azure (v0.6.4, RVM: ruby-2.1.1) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="hashie (v3.1.0, RVM: ruby-2.1.1) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="macaddr (v1.7.1, RVM: ruby-2.1.1) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="mime-types (v1.25.1, RVM: ruby-2.1.1) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="mini_portile (v0.6.0, RVM: ruby-2.1.1) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="nokogiri (v1.6.2.1, RVM: ruby-2.1.1) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="rake (v10.3.2, RVM: ruby-2.1.1) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="systemu (v2.6.4, RVM: ruby-2.1.1) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="uuid (v2.3.7, RVM: ruby-2.1.1) [gem]" level="application" />
+  </component>
+</module>
+

data/lib/azure/blob_service.rb

@@ -0,0 +1,93 @@
+# This code requires Ruby 2.0+ ... it's 2014, people
+
+# Make sure the original is included
+require 'azure/blob/blob_service'
+require 'celluloid'
+require 'timeout'
+
+class ::File
+  def each_chunk(chunk_size=2**20)
+    yield read(chunk_size) until eof?
+  end
+end
+
+# The maximum size for a block blob is 200 GB, and a block blob can include no more than 50,000 blocks.
+  # http://msdn.microsoft.com/en-us/library/azure/ee691964.aspx
+
+class BlockActor
+  include Celluloid
+
+  def initialize(service, container, blob, options = {})
+    @service, @container, @blob, @options = service, container, blob, options
+  end
+
+  def upload(block_id, chunk, retries = 0)
+    Timeout::timeout(@options[:timeout] || 60){
+      log "Uploading block #{block_id}"
+      options = @options.dup
+      options[:content_md5] = Base64.strict_encode64(Digest::MD5.digest(chunk))
+      content_md5 = @service.create_blob_block(@container, @blob, block_id, chunk, options)
+      log "Done uploading block #{block_id} #{content_md5}"
+      [block_id, :uncommitted]
+    }
+  rescue Timeout::Error, Azure::Core::Error => e
+    log "Failed to upload #{block_id}: #{e.class} #{e.message}"
+    if retries < 5
+      log "Retrying upload (#{retries})"
+      upload(block_id, chunk, retries += 1)
+    else
+      log "Complete failure to upload #{retries} retries"
+    end
+  end
+
+  def log(message)
+    puts message
+  end
+end
+
+module Azure
+  module BlobServiceExtensions
+    def create_block_blob(container, blob, content_or_filepath, options={})
+      chunking = options.delete(:chunking)
+      if chunking
+        filepath = content_or_filepath
+        block_list = upload_chunks(container, blob, filepath, options)
+
+        return false unless block_list
+
+        puts "Done uploading #{block_list.size} blocks, committing ..."
+        options[:blob_content_type] = options[:content_type]
+        commit_blob_blocks(container, blob, block_list, options)
+        puts "done."
+      else
+        content = content_or_filepath
+        super(container, blob, content, options)
+      end
+    end
+
+    # The maximum size for a block blob is 200 GB, and a block blob can include no more than 50,000 blocks.
+    # http://msdn.microsoft.com/en-us/library/azure/ee691964.aspx
+    def upload_chunks(container, blob, filepath, options = {})
+      counter = 1
+      futures = []
+      pool    = BlockActor.pool(size: 20, args: [self, container, blob, options])
+
+      open(filepath, 'rb') do |f|
+        f.each_chunk() {|chunk|
+          block_id = counter.to_s.rjust(5, '0')
+          futures << pool.future.upload(block_id, chunk)
+          counter += 1
+        }
+      end
+
+      block_list = futures.map(&:value)
+      pool.terminate
+      return block_list
+    end
+  end
+
+  # Why alias_method chain when Ruby gives you a more reasonable way to do this
+  class BlobService
+    prepend BlobServiceExtensions
+  end
+end

data/lib/azure/contrib/version.rb

@@ -1,5 +1,5 @@
 module Azure
   module Contrib
-    VERSION = "0.0.2"
+    VERSION = "0.0.3"
   end
 end

data/lib/azure/contrib.rb

@@ -1,3 +1,4 @@
+require 'azure'
 require "azure/contrib/version"
 
 
@@ -7,4 +8,5 @@ module Azure
   end
 end
 
-require "azure/contrib/auth/shared_access_signature"
+require "azure/contrib/auth/shared_access_signature"
+require File.join(File.dirname(__FILE__), 'blob_service')

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: azure-contrib
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - David Michael
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-01 00:00:00.000000000 Z
+date: 2014-07-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -80,6 +80,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: celluloid
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Extensions to the Azure Ruby SDK - specifically SAS
 email:
 - david.michael@giantmachines.com
@@ -93,6 +107,8 @@ files:
 - README.md
 - Rakefile
 - azure-contrib.gemspec
+- azure-contrib.iml
+- lib/azure/blob_service.rb
 - lib/azure/contrib.rb
 - lib/azure/contrib/auth/shared_access_signature.rb
 - lib/azure/contrib/version.rb