azure_client 0.1.0

data/.gitignore

@@ -0,0 +1,6 @@
+tmp/
+*.swp
+*.bundle
+*.idea
+*.jar
+*.swo

data/.rvmrc

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+# This is an RVM Project .rvmrc file, used to automatically load the ruby
+# development environment upon cd'ing into the directory
+
+# First we specify our desired <ruby>[@<gemset>], the @gemset name is optional,
+# Only full ruby name is supported here, for short names use:
+#     echo "rvm use 1.9.3" > .rvmrc
+environment_id="ruby-1.9.3-p327@azure_client"
+
+# Uncomment the following lines if you want to verify rvm version per project
+# rvmrc_rvm_version="1.17.3 (stable)" # 1.10.1 seams as a safe start
+# eval "$(echo ${rvm_version}.${rvmrc_rvm_version} | awk -F. '{print "[[ "$1*65536+$2*256+$3" -ge "$4*65536+$5*256+$6" ]]"}' )" || {
+#   echo "This .rvmrc file requires at least RVM ${rvmrc_rvm_version}, aborting loading."
+#   return 1
+# }
+
+# First we attempt to load the desired environment directly from the environment
+# file. This is very fast and efficient compared to running through the entire
+# CLI and selector. If you want feedback on which environment was used then
+# insert the word 'use' after --create as this triggers verbose mode.
+if [[ -d "${rvm_path:-$HOME/.rvm}/environments"
+  && -s "${rvm_path:-$HOME/.rvm}/environments/$environment_id" ]]
+then
+  \. "${rvm_path:-$HOME/.rvm}/environments/$environment_id"
+  [[ -s "${rvm_path:-$HOME/.rvm}/hooks/after_use" ]] &&
+    \. "${rvm_path:-$HOME/.rvm}/hooks/after_use" || true
+else
+  # If the environment file has not yet been created, use the RVM CLI to select.
+  rvm --create  "$environment_id" || {
+    echo "Failed to create RVM environment '${environment_id}'."
+    return 1
+  }
+fi
+
+# If you use bundler, this might be useful to you:
+# if [[ -s Gemfile ]] && {
+#   ! builtin command -v bundle >/dev/null ||
+#   builtin command -v bundle | GREP_OPTIONS= \grep $rvm_path/bin/bundle >/dev/null
+# }
+# then
+#   printf "%b" "The rubygem 'bundler' is not installed. Installing it now.\n"
+#   gem install bundler
+# fi
+# if [[ -s Gemfile ]] && builtin command -v bundle >/dev/null
+# then
+#   bundle install | GREP_OPTIONS= \grep -vE '^Using|Your bundle is complete'
+# fi

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,43 @@
+PATH
+  remote: .
+  specs:
+    azure_client (0.0.9)
+      rest-client
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    azure (0.5.0)
+      json
+      mime-types (~> 1.0)
+      nokogiri (~> 1.5)
+      uuid
+    diff-lcs (1.2.4)
+    json (1.8.0)
+    macaddr (1.6.1)
+      systemu (~> 2.5.0)
+    mime-types (1.23)
+    nokogiri (1.5.9)
+    rake (10.0.4)
+    rest-client (1.6.7)
+      mime-types (>= 1.16)
+    rspec (2.13.0)
+      rspec-core (~> 2.13.0)
+      rspec-expectations (~> 2.13.0)
+      rspec-mocks (~> 2.13.0)
+    rspec-core (2.13.1)
+    rspec-expectations (2.13.0)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.13.1)
+    systemu (2.5.2)
+    uuid (2.3.7)
+      macaddr (~> 1.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  azure
+  azure_client!
+  rake
+  rspec

data/azure_client.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'azure_client/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "azure_client"
+  gem.version       = AzureClient::VERSION
+  gem.authors       = ["Karl Skucha"]
+  gem.email         = ["kskucha@yammer-inc.com"]
+  gem.description   = %q{Encapsulate the API's of Azure SDK}
+  gem.summary       = %q{}
+  gem.homepage      = ""
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_dependency 'rest-client'
+  gem.add_development_dependency 'azure'
+  gem.add_development_dependency 'rspec'
+  gem.add_development_dependency 'rake'
+end

data/lib/azure_client/blob.rb

@@ -0,0 +1,24 @@
+module AzureClient
+  class Blob
+
+    def initialize(name, azure_blob, content, container_name, blob_service, retry_policy) 
+      @name = name
+      @azure_blob = azure_blob
+      @content = content
+      @container_name = container_name
+      @blob_service = blob_service
+      @retry_policy = retry_policy
+    end
+
+    def get_content
+      @content
+    end
+
+    def delete(retry_policy = @retry_policy)
+      retry_policy.retry {
+        @blob_service.delete_blob(@container_name, @name)
+      }
+    end
+  
+  end
+end

data/lib/azure_client/blob_lease.rb

@@ -0,0 +1,58 @@
+module AzureClient
+  class BlobLease
+    RENEW_MARGIN = 10
+
+    attr_reader :lease_id, :duration, :start_time, :container, :blob_name
+    alias_method :name, :lease_id
+    alias_method :to_s, :name
+
+    def initialize(container, blob_name, blob_service, retry_policy, options = {})
+      @container = container
+      @blob_name = blob_name
+      @blob_service = blob_service
+      @retry_policy = retry_policy
+
+      options[:duration] ||= 60 # NO default to infinity
+      retry_policy.retry {
+        tries = 0
+        begin
+          @lease_id = @blob_service.acquire_lease(container, blob_name, options)
+        rescue Azure::Core::Http::HTTPError => e
+          if tries < 1 and e.status_code == 404 then
+            @blob_service.create_block_blob(container, blob_name, '')
+            tries += 1
+            retry
+          else
+            raise
+          end
+        end
+      }
+      @start_time = Time.now
+      @duration = options[:duration]
+    end
+
+    def change(new_lease_id, retry_policy = @retry_policy)
+      retry_policy.retry {
+        @blob_service.change_lease(container, blob_name, "#{new_lease_id}", lease_id)
+        @lease_id = new_lease_id
+      }
+    end
+
+    def renew(retry_policy = @retry_policy)
+      retry_policy.retry {
+        @blob_service.renew_lease(container, blob_name, lease_id)
+      }
+    end
+
+    def renew_if_necessary(retry_policy = nil)
+      age = Time.now - start_time
+      renew(retry_policy) if duration - age <= RENEW_MARGIN
+    end
+
+    def release(retry_policy = @retry_policy)
+      retry_policy.retry {
+        @blob_service.release_lease(container, blob_name, lease_id)
+      }
+    end
+  end
+end

data/lib/azure_client/buffered_queue.rb

@@ -0,0 +1,52 @@
+module AzureClient
+  class BufferedQueue
+    attr_accessor :name
+    
+    def initialize(queues, container, retry_policy = ExponentialRetryPolicy.new(5,1,2))
+      @queues = queues.kind_of?(Array) ? queues : [queues]
+      @name = @queues[0].name
+      @container = container
+      @retry_policy = retry_policy
+    end
+    
+    def add_message(content, metadata = "", retry_policy = @retry_policy)
+      payload = {"content" => content, "metadata" => metadata}.to_json
+      queue = select_queue
+      if is_allowed_payload_size(payload)
+        queue.add_message(payload, retry_policy)
+      else
+        #pick random blob name
+        blob_name = name + (0...9).map{ ('a'..'z').to_a[rand(26)] }.join
+        reference = {"type" => "azure_blob_reference", "name" => blob_name, "metadata" => metadata}
+        @container.store_blob(blob_name, content, {}, retry_policy)
+        queue.add_message(reference.to_json, retry_policy)
+      end
+    end
+
+    #default to one try
+    def get_message(retry_policy = LinearRetryPolicy.new(1,1))
+      queue = select_queue 
+      message = queue.get_message(retry_policy)
+      message ? BufferedQueueMessage.new(message, @container, @retry_policy) : nil
+    end
+
+    def delete(retry_policy = @retry_policy)
+      @queues.each do |queue|
+        queue.delete(retry_policy)
+      end
+      @container.delete(retry_policy)
+    end
+
+    private
+
+    def is_allowed_payload_size(payload)
+      #leave a 1kb margin
+      return payload.bytesize <= 63 * 1024 / 4 * 3
+    end
+
+    def select_queue
+      @queues.sample
+    end
+
+  end
+end

data/lib/azure_client/buffered_queue_factory.rb

@@ -0,0 +1,19 @@
+module AzureClient
+  class BufferedQueueFactory
+
+    def self.get_buffered_queue(container, queues, storage, retryPolicy = ExponentialRetryPolicy.new(5,1,2))
+      buffered_queue = AzureClient::BufferedQueue.new(get_queues(queues, storage), storage.get_container(container), retryPolicy)
+    end
+
+    private
+
+    def self.get_queues(queues, storage)
+      result = []
+      queues.each do |queue|
+        result << storage.get_queue(queue)
+      end
+      result
+    end
+
+  end
+end

data/lib/azure_client/buffered_queue_message.rb

@@ -0,0 +1,51 @@
+module AzureClient
+  class BufferedQueueMessage
+
+    def initialize(message, container, retry_policy = ExponentialRetryPolicy.new(5,1,2))
+      @message = message
+      @parsed_message = parse
+      @container = container
+      @retry_policy = retry_policy
+      @blob = get_blob
+    end
+
+    def get_content
+      if @blob
+        @blob.get_content
+      else
+        @parsed_message["content"]
+      end
+    end
+
+    def get_metadata
+      @parsed_message["metadata"]
+    end
+
+    def delete(retry_policy = @retry_policy)
+      if @blob
+        @blob.delete(@retry_policy)
+      else
+        @message.delete(@retry_policy)
+      end
+    end
+
+    private
+
+    def get_blob
+      if @parsed_message && @parsed_message["type"] == "azure_blob_reference"
+        blob_name = @parsed_message["name"]
+        return @container.get_blob(blob_name)
+      end
+      return nil
+    end
+
+    def parse
+      begin
+        JSON.parse(@message.get_content)
+      rescue 
+        return nil
+      end
+    end
+
+  end
+end

data/lib/azure_client/client.rb

@@ -0,0 +1,45 @@
+module AzureClient
+  class Client
+
+    def initialize(account_name, access_key)
+      Azure.configure do |config|
+        config.storage_account_name = account_name
+        config.storage_access_key   = access_key
+      end
+    end
+
+    def get_queue(name, retry_policy = ExponentialRetryPolicy.new(5,1,2))
+      queue_service.create_queue(name)
+      return Queue.new(name, queue_service, retry_policy) 
+    end
+
+    def get_container(name, retry_policy = ExponentialRetryPolicy.new(5,1,2))
+      if !(blob_service.list_containers.map {|container| container.name}.include?(name))
+        blob_service.create_container(name)
+      end
+      return Container.new(name, blob_service, retry_policy)
+    end
+
+    def get_table(name, retry_policy = ExponentialRetryPolicy.new(2,1,2))
+      if !(table_service.query_tables.map {|hash| hash[:properties]["TableName"]}.include?(name))
+        table_service.create_table(name)
+      end
+      return Table.new(name, table_service, retry_policy)
+    end
+
+    private
+
+    def blob_service
+      @blob_service || Azure::BlobService.new
+    end
+
+    def queue_service
+      @queue_service || Azure::QueueService.new
+    end
+
+    def table_service
+      @table_service || Azure::TableService.new
+    end
+
+  end
+end

data/lib/azure_client/container.rb

@@ -0,0 +1,81 @@
+module AzureClient
+  class Container
+    attr_reader :name
+
+    def initialize(name, blob_service, retry_policy)
+      @name = name
+      @blob_service = blob_service
+      @retry_policy = retry_policy
+    end
+
+    #will overwrite content if blob with same name already exists
+    def store_blob(blob_name, content, options = {}, retry_policy = @retry_policy)
+      retry_policy.retry {
+        @blob_service.create_block_blob(name, blob_name, content, options)
+      }
+    end
+
+    #throws exception if no blob found
+    def get_blob(blob_name, options = {}, retry_policy = @retry_policy) 
+      retry_policy.retry {
+        azure_blob, content = @blob_service.get_blob(name, blob_name, options)
+        Blob.new(blob_name, azure_blob, content, name, @blob_service, @retry_policy)
+      }
+    end
+
+    def get_blob_lease(blob_name, retry_policy = @retry_policy, options = {})
+      BlobLease.new(@name, blob_name, @blob_service, retry_policy, options)
+    end
+
+    def get_blob_properties(blob_name, retry_policy = @retry_policy)
+      retry_policy.retry {
+        @blob_service.get_blob_properties(@name, blob_name)
+      }.properties
+    end
+
+    def get_blob_metadata(blob_name, retry_policy = @retry_policy)
+      retry_policy.retry {
+        @blob_service.get_blob_metadata(@name, blob_name).metadata
+      }
+    end
+
+    def set_blob_metadata(blob_name, metadata, retry_policy = @retry_policy)
+      retry_policy.retry {
+        @blob_service.set_blob_metadata(@name, blob_name, metadata)
+      }
+    end
+
+    def delete(retry_policy = @retry_policy)
+      retry_policy.retry {
+        @blob_service.delete_container(@name) 
+      }
+    end
+
+    def delete_blob(blob_name, retry_policy = @retry_policy)
+      retry_policy.retry {
+        @blob_service.delete_blob(name, blob_name)
+      }
+    end
+
+    def poll_blob(blob_name, key, val, timeout = 10800)
+      time = 0
+      #create blob if it does not exist
+      begin
+        blob = get_blob(blob_name)
+      rescue
+        store_blob(blob_name,"")
+      end
+      loop do 
+        sleep(5)
+        time += 5;
+        puts "*** polling Azure blob #{blob_name} for #{time} seconds"
+        metadata = get_blob_metadata(blob_name)
+        return val if metadata && metadata[key] == val
+        if time > timeout
+          raise "Polling blob #{blob_name} took more than #{timeout} seconds to run. Time out."
+        end
+      end
+    end
+
+  end
+end

data/lib/azure_client/exponential_retry_policy.rb

@@ -0,0 +1,26 @@
+module AzureClient
+  class ExponentialRetryPolicy
+
+    def initialize(num_retries, initial_timeout, multiplier)
+      @num_retries = num_retries
+      @initial_timeout = initial_timeout
+      @multiplier = multiplier
+    end
+
+    def retry
+      @num_retries.times do |count|
+        begin
+          return yield
+        rescue StandardError => e
+          if count == @num_retries - 1
+            raise e
+          else
+            sleep((@multiplier**count) * @initial_timeout)
+          end
+        end
+      end
+    end
+
+  end
+end
+

data/lib/azure_client/linear_retry_policy.rb

@@ -0,0 +1,30 @@
+module AzureClient
+  class LinearRetryPolicy
+
+    def initialize(num_retries, timeout)
+      @num_retries = num_retries
+      @timeout = timeout
+    end
+
+    def retry
+      @num_retries.times do |count|
+        begin
+          return yield
+        rescue StandardError => e
+          if count == @num_retries - 1
+            raise e
+          else
+            sleep(@timeout)
+          end
+        end
+      end
+    end
+
+  end
+end
+
+
+
+
+
+

data/lib/azure_client/message.rb

@@ -0,0 +1,37 @@
+module AzureClient
+  class Message
+
+    def initialize(azure_message, queue_name, queue_service, retry_policy)
+      @azure_message = azure_message
+      @queue_name = queue_name
+      @queue_service = queue_service
+      @retry_policy = retry_policy
+    end
+
+    def get_content
+      content = @azure_message.message_text
+      begin
+        Base64.decode64(content)
+      rescue
+        raise $!, "Failed to decode content of azure queue message, check to make sure message is encoded, exception: #{$!}"
+      end
+    end
+
+    def id
+      @azure_message.id
+    end
+
+    def delete(retry_policy = @retry_policy)
+      retry_policy.retry {
+        @queue_service.delete_message(@queue_name, id, pop_receipt)
+      }
+    end
+
+    private
+
+    def pop_receipt
+      @azure_message.pop_receipt
+    end
+
+  end
+end

data/lib/azure_client/queue.rb

@@ -0,0 +1,33 @@
+module AzureClient
+  class Queue
+    attr_accessor :name
+    
+    def initialize(name, queue_service, retry_policy = ExponentialRetryPolicy.new(5,1,2))
+      @name = name
+      @queue_service = queue_service
+      @retry_policy = retry_policy
+    end
+    
+    def add_message(content, retry_policy = @retry_policy)
+      retry_policy.retry {
+        encoded = Base64.encode64(content)
+        @queue_service.create_message(name, encoded)
+      }
+    end
+
+    #default to only one try
+    def get_message(retry_policy = LinearRetryPolicy.new(1,1))
+      retry_policy.retry {
+        azure_message = @queue_service.list_messages(name, 30).first
+        azure_message ? Message.new(azure_message, name, @queue_service, @retry_policy) : nil
+      }
+    end
+
+    def delete(retry_policy = @retry_policy)
+      retry_policy.retry {
+        @queue_service.delete_queue(name)
+      }
+    end
+
+  end
+end

data/lib/azure_client/table.rb

@@ -0,0 +1,45 @@
+module AzureClient
+  class Table
+    attr_reader :name
+
+    def initialize(name, table_service, retry_policy)
+      @name = name
+      @table_service = table_service
+      @retry_policy = retry_policy
+    end
+
+    def store_entity(entity, retry_policy = @retry_policy)
+      retry_policy.retry {
+        entity_hash = {"PartitionKey" => entity.partition_key, "RowKey" => entity.row_key}.merge(entity.content_hash)
+        @table_service.insert_or_replace_entity(name, entity_hash)
+      }
+    end
+
+
+    def get_entity(partition_key, row_key, retry_policy = @retry_policy) 
+      retry_policy.retry {
+        entity_hash = @table_service.get_entity(name, partition_key, row_key).properties
+        TableEntity.new(entity_hash["PartitionKey"], entity_hash["RowKey"], entity_hash["TimeStamp"], entity_hash.tap { |h| h.delete(:PartitionKey); h.delete(:RowKey)})
+      }
+    end
+
+    def delete(retry_policy = @retry_policy)
+      retry_policy.retry {
+        @table_service.delete_table(name) 
+      }
+    end
+
+    def delete_entity(partition_key, row_key, retry_policy = @retry_policy)
+      retry_policy.retry {
+        @table_service.delete_entity(name, partition_key, row_key)
+      }
+    end
+
+    def query(query_hash, retry_policy = @retry_policy)
+      retry_policy.retry {
+        @table_service.query_entities(name, query_hash)
+      }
+    end
+
+  end
+end

data/lib/azure_client/table_entity.rb

@@ -0,0 +1,15 @@
+module AzureClient
+  class TableEntity
+    attr_accessor :partition_key, :row_key, :content_hash
+
+    def initialize(partition_key, row_key, timestamp = nil, content_hash)
+      @partition_key = partition_key
+      @row_key = row_key
+      @timestamp = timestamp
+      @content_hash = content_hash
+    end
+
+  end
+end
+
+

data/lib/azure_client/version.rb

@@ -0,0 +1,3 @@
+module AzureClient
+  VERSION = "0.1.0"
+end

data/lib/azure_client.rb

@@ -0,0 +1,8 @@
+# standard
+require 'json'
+require 'base64'
+
+# azure
+require 'azure'
+Dir.glob(File.dirname(File.absolute_path(__FILE__)) + '/azure_client/*.rb') {|file| require file}
+

metadata

@@ -0,0 +1,129 @@
+--- !ruby/object:Gem::Specification
+name: azure_client
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Karl Skucha
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-07-31 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: azure
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Encapsulate the API's of Azure SDK
+email:
+- kskucha@yammer-inc.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rvmrc
+- Gemfile
+- Gemfile.lock
+- azure_client.gemspec
+- lib/azure_client.rb
+- lib/azure_client/blob.rb
+- lib/azure_client/blob_lease.rb
+- lib/azure_client/buffered_queue.rb
+- lib/azure_client/buffered_queue_factory.rb
+- lib/azure_client/buffered_queue_message.rb
+- lib/azure_client/client.rb
+- lib/azure_client/container.rb
+- lib/azure_client/exponential_retry_policy.rb
+- lib/azure_client/linear_retry_policy.rb
+- lib/azure_client/message.rb
+- lib/azure_client/queue.rb
+- lib/azure_client/table.rb
+- lib/azure_client/table_entity.rb
+- lib/azure_client/version.rb
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: ''
+test_files: []