sparqcode-waz-storage 1.1.1

data/lib/waz-queues.rb

@@ -0,0 +1,6 @@
+$:.unshift(File.dirname(__FILE__))
+require 'waz-storage'
+# Application Files (massive include)
+app_files = File.expand_path(File.join(File.dirname(__FILE__), 'waz','queues', '*.rb'))
+Dir[app_files].each(&method(:load))
+

data/lib/waz-storage.rb

@@ -0,0 +1,39 @@
+require 'time'
+require 'cgi' 
+require 'base64'
+require 'rexml/document'
+require 'rexml/xpath'
+require 'restclient' 
+require 'hmac-sha2'
+require 'net/http'
+
+app_files = File.expand_path(File.join(File.dirname(__FILE__), 'waz', 'storage', '*.rb'))
+Dir[app_files].each(&method(:load))
+
+# It will depende on which version of Ruby (or if you have Rails) 
+# but this method is required so we will add it the String class.
+unless String.method_defined? :start_with?
+  class String
+    def start_with?(prefix)
+      prefix = prefix.to_s
+      self[0, prefix.length] == prefix
+    end
+  end
+end
+
+# The Merge method is not defined in the RFC 2616 
+# and it's required to Merge entities in Windows Azure
+module Net
+  class HTTP < Protocol
+    class Merge < HTTPRequest
+      METHOD = 'MERGE'
+      REQUEST_HAS_BODY  = true
+      RESPONSE_HAS_BODY = false
+    end
+  end
+end
+
+# extendes the Symbol class to assign a type to an entity field
+class Symbol
+  attr_accessor :edm_type
+end

data/lib/waz-tables.rb

@@ -0,0 +1,5 @@
+$:.unshift(File.dirname(__FILE__))
+require 'waz-storage'
+# Application Files (massive include)
+app_files = File.expand_path(File.join(File.dirname(__FILE__), 'waz', 'tables', '*.rb'))
+Dir[app_files].each(&method(:load))

data/lib/waz/blobs/blob_object.rb

@@ -0,0 +1,121 @@
+module WAZ
+  module Blobs
+    # This class is used to model the Blob inside Windows Azure Blobs the usage
+    # it's pretty simple, here you can see a couple of samples. These are the implemented methods of the blob API up to now. 
+    # The basics are implemented although blocks management is not yet completed, I consider that the API is pretty usable since 
+    # it covers the basics
+    #
+    #   # retrieve blob name, uri and content-type
+    #   blob.name
+    #   blob.url
+    #   blob.content_type
+    #
+    # 	# retrieve blob value 
+    #   blob.value #=> lazy loaded payload of the blob
+    #
+    #   # retrieve blob metadata (+ properties)
+    #   blob.metadata #=> hash containing beautified metadata (:x_ms_meta_name)
+    #   
+    #   # put blob metadata
+    #   blob.put_properties(:x_ms_meta_MyProperty => "my value") 
+    #
+    # 	# update value
+    #   blob.value = "my new value" #=> this will update the blob content on WAZ
+    #
+    # *REMARKS*: This class is not meant to be manually instanciated it's basicaly an internal 
+    # representation returned by the WAZ::Blobs::Container.
+    class BlobObject      
+      class << self
+        # This method is internally used by this class. It's the way we keep a single instance of the 
+        # service that wraps the calls the Windows Azure Blobs API. It's initialized with the values
+        # from the default_connection on WAZ::Storage::Base initialized thru establish_connection!
+        def service_instance
+          options = WAZ::Storage::Base.default_connection.merge(:type_of_service => "blob")
+          (@service_instances ||= {})[options[:account_name]] ||= Service.new(options)
+        end
+      end
+      
+      attr_accessor :name, :url, :content_type, :snapshot_date
+      
+      # Creates a new instance of the Blob object. This constructor is internally used by the Container
+      # it's initialized thru a hash that's received as parameter. It has the following requirements:
+      # <em>:name</em> which is the blob name (usually the file name), <em>:url</em> that is the url of the blob (used to download or access it via browser) 
+      # and <em>:content_type</em> which is the content type of the blob and is a required parameter by the Azure API
+      def initialize(options = {})
+        raise WAZ::Storage::InvalidOption, :name unless options.keys.include?(:name) and !options[:name].empty?
+        raise WAZ::Storage::InvalidOption, :url unless options.keys.include?(:url) and !options[:url].empty?
+        raise WAZ::Storage::InvalidOption, :content_type unless options.keys.include?(:content_type) and !options[:content_type].empty?
+        self.name = options[:name]
+        self.url = options[:url]
+        self.content_type = options[:content_type]
+        self.snapshot_date = options[:snapshot_date]
+      end
+      
+      # Returns the blob properties from Windows Azure. This properties always come as HTTP Headers and they include standard headers like
+      # <em>Content-Type</em>, <em>Content-Length</em>, etc. combined with custom properties like with <em>x-ms-meta-Name</em>.
+      def metadata
+        self.class.service_instance.get_blob_properties(path)
+      end
+      
+      # Returns the actual blob content, this method is specially used to avoid retrieving the whole blob
+      # while iterating and retrieving the blob collection from the Container.
+      def value
+        @value ||= self.class.service_instance.get_blob(path)
+      end
+      
+      # Assigns the given value to the blob content. It also stores a local copy of it in order to avoid round trips 
+      # on scenarios when you do Save and Display on the same context.
+      def value=(new_value)
+        raise WAZ::Blobs::InvalidOperation if self.snapshot_date
+        self.class.service_instance.put_blob(path, new_value, content_type, metadata)
+        @value = new_value
+      end
+
+      # Stores the blob properties. Those properties are sent as HTTP Headers it's really important that you name your custom 
+      # properties with the <em>x-ms-meta</em> prefix, if not the won't be persisted by the Windows Azure Blob Storage API.
+      def put_properties!(properties = {})
+        raise WAZ::Blobs::InvalidOperation if self.snapshot_date
+        self.class.service_instance.set_blob_properties(path, properties)
+      end
+
+      # Stores blob metadata. User metadata must be prefixed with 'x-ms-meta-'. The advantage of this over put_properties
+      # is that it only affect user_metadata and doesn't overwrite any system values, like 'content_type'.
+      def put_metadata!(metadata = {})  
+        self.class.service_instance.set_blob_metadata(path, metadata)
+      end 
+      
+      # Removes the blob from the container.
+      def destroy!
+        self.class.service_instance.delete_blob(path)
+      end
+      
+      # Copies the blob to the destination and returns
+      # the copied blob instance. 
+      #
+      # destination should be formed as "container/blob"
+      def copy(destination)
+        self.class.service_instance.copy_blob(self.path, destination)
+        properties = self.class.service_instance.get_blob_properties(destination)
+        return BlobObject.new(:name => destination, 
+                              :url => self.class.service_instance.generate_request_uri(destination),
+                              :content_type => properties[:content_type])
+      end
+      
+      # Creates and returns a read-only snapshot of a blob as it looked like in time.
+      def snapshot
+        date = self.class.service_instance.snapshot_blob(self.path)
+        properties = self.class.service_instance.get_blob_properties(self.path)
+        return BlobObject.new(:name => self.name, 
+                              :url => self.class.service_instance.generate_request_uri(self.path) + "?snapshot=#{date}",
+                              :content_type => properties[:content_type],
+                              :snapshot_date => date)
+      end
+      
+      # Returns the blob path. This is specially important when simulating containers inside containers
+      # by enabling the API to point to the appropiated resource.
+      def path
+        url.gsub(/https?:\/\/[^\/]+\//i, '').scan(/([^&]+)/i).first().first()
+      end      
+    end
+  end
+end

data/lib/waz/blobs/container.rb

@@ -0,0 +1,160 @@
+module WAZ
+  module Blobs
+    # This class is used to model the Container inside Windows Azure Blobs the usage
+    # it's pretty simple, here you can see a couple of samples. Here you can find Microsoft's REST API description available on MSDN 
+    # at http://msdn.microsoft.com/en-us/library/dd179361.aspx
+    #
+    #  # list available containers
+  	#  WAZ::Blobs::Container.list
+    #
+  	#  # create a container
+  	#  WAZ::Blobs::Container.create('my-container')
+    #
+  	#  # get a specific container 
+  	#  my_container = WAZ::Blobs::Container.find('my-container')
+    #
+  	#  # get container properties (including default headers)
+  	#  my_container.metadata #=> hash containing beautified metadata (:x_ms_meta_name)
+    #
+  	#  # set container properties (should follow x-ms-meta to be persisted)
+  	#  my_container.put_properties(:x_ms_meta_MyProperty => "my value")
+    #
+  	#  # get a the value indicating whether the container is public or not
+  	#  my_container.public_access? #=> true or false based on x-ms-prop-publicaccess
+  	#
+  	#  # set a value indicating whether the container is public or not
+  	#  my_container.public_access = false
+    #
+  	#  # delete container
+  	#  my_container.destroy!
+  	#
+  	#  # store a blob on the given container
+  	#  my_container.store('my-blob', blob_content, 'application/xml')
+  	#
+  	#  # retrieve a particular blob from a container
+    #  my_container['my-blob']
+    #
+  	#  # retrieve a blob list from a container
+  	#  my_container.blobs #=> WAZ::Blobs::BlobObject collection
+  	#
+    class Container
+      class << self 
+        # Creates a new container with the given name.
+        def create(name)
+          raise WAZ::Storage::InvalidParameterValue, {:name => "name", :values => ["lower letters, numbers or - (hypen), and must not start or end with - (hyphen)"]} unless WAZ::Storage::ValidationRules.valid_name?(name)
+          service_instance.create_container(name)
+          return Container.new(:name => name)
+        end
+        
+        # Finds a container by name. It will return nil if no container was found.
+        def find(name)
+          begin 
+            properties = service_instance.get_container_properties(name)
+            return Container.new(properties.merge(:name => name))
+          rescue RestClient::ResourceNotFound
+            return nil
+          end
+        end
+        
+        # Returns all the containers on the given account.
+        def list(options = {})
+          service_instance.list_containers(options).map { |container| Container.new(container) }
+        end
+        
+        # This method is internally used by this class. It's the way we keep a single instance of the 
+        # service that wraps the calls the Windows Azure Blobs API. It's initialized with the values
+        # from the default_connection on WAZ::Storage::Base initialized thru establish_connection!
+        def service_instance
+          options = WAZ::Storage::Base.default_connection.merge(:type_of_service => "blob")
+          (@service_instances ||= {})[options[:account_name]] ||= Service.new(options)
+        end        
+      end
+      
+      attr_accessor :name
+      
+      # Creates a new instance of the WAZ::Blobs::Container. This class isn't intended for external use
+      # to access or create a container you should use the class methods provided like list, create, or find.
+      def initialize(options = {})
+        raise WAZ::Storage::InvalidOption, :name unless options.keys.include?(:name) and !options[:name].empty?
+        self.name = options[:name]
+      end
+      
+      # Returns the container metadata.
+      def metadata
+        self.class.service_instance.get_container_properties(self.name)
+      end
+      
+      # Adds metadata for the container.Those properties are sent as HTTP Headers it's really important that you name your custom 
+      # properties with the <em>x-ms-meta</em> prefix, if not the won't be persisted by the Windows Azure Blob Storage API.
+      def put_properties!(properties = {})
+        self.class.service_instance.set_container_properties(self.name, properties)
+      end
+      
+      # Removes the container from the current account.
+      def destroy!
+        self.class.service_instance.delete_container(self.name)
+      end
+      
+      # Retuns a value indicating whether the container is public accessible (i.e. from a Web Browser) or not.
+      def public_access?
+        self.class.service_instance.get_container_acl(self.name)
+      end
+      
+      # Sets a value indicating whether the container is public accessible (i.e. from a Web Browser) or not.  
+      def public_access=(value)
+        self.class.service_instance.set_container_acl(self.name, value)
+      end
+      
+      # Returns a list of blobs (WAZ::Blobs::BlobObject) contained on the current container.
+      def blobs
+        self.class.service_instance.list_blobs(name).map { |blob| WAZ::Blobs::BlobObject.new(blob) }
+      end
+      
+      # Stores a blob on the container with under the given name, with the given content and 
+      # the required <em>content_type</em>. <strong>The <em>options</em> parameters if provided
+      # will set the default metadata properties for the blob</strong>.
+      def store(blob_name, payload, content_type, options = {})
+        blob_name.gsub!(%r{^/}, '')
+        self.class.service_instance.put_blob("#{self.name}/#{blob_name}", payload, content_type, options)
+        return BlobObject.new(:name => blob_name, 
+                              :url => self.class.service_instance.generate_request_uri("#{self.name}/#{blob_name}"),
+                              :content_type => content_type)
+      end
+
+      # Uploads the contents of a stream to the specified blob within this container, using
+      # the required <em>content_type</em>. The stream will be uploaded in blocks of size
+      # <em>block_size</em> bytes, which defaults to four megabytes. <strong>The <em>options</em>
+      # parameter, if provided, will set the default metadata properties for the blob</strong>.
+      def upload(blob_name, stream, content_type, options = {}, block_size = 4 * 2**20)
+        blob_name.gsub!(%r{^/}, '')
+        path = "#{self.name}/#{blob_name}"
+        n = 0
+        until stream.eof?
+          self.class.service_instance.put_block path, Base64.encode64('%064d' % n), stream.read(block_size)
+          n += 1
+        end
+        self.class.service_instance.put_block_list path, (0...n).map{|id| Base64.encode64('%064d' % id)}, content_type, options
+        return BlobObject.new(:name => blob_name, :url => self.class.service_instance.generate_request_uri("#{self.name}/#{blob_name}"), :content_type => content_type)
+      end
+      
+      # Retrieves the blob by the given name. If the blob isn't found on the current container
+      # it will return nil instead of throwing an exception.
+      def [](blob_name)
+        begin
+          blob_name.gsub!(%r{^/}, '')
+          properties = self.class.service_instance.get_blob_properties("#{self.name}/#{blob_name}")
+          return BlobObject.new(:name => blob_name, 
+                                :url => self.class.service_instance.generate_request_uri("#{self.name}/#{blob_name}"),
+                                :content_type => properties[:content_type])
+        rescue RestClient::ResourceNotFound
+          return nil
+        end
+      end
+    end
+    class BlobSecurity
+      Container = 'container'
+      Blob = 'blob'
+      Private = ''
+    end
+  end
+end

data/lib/waz/blobs/exceptions.rb

@@ -0,0 +1,11 @@
+module WAZ
+  module Blobs    
+    # This exception is raised when the user tries to perform an operation over a snapshoted blob. Since
+    # Snapshots are read-only copies of the original blob they cannot be modified
+    class InvalidOperation < WAZ::Storage::StorageException
+      def initialize()
+        super("A snapshoted blob cannot be modified.")
+      end
+    end
+  end
+end

data/lib/waz/blobs/service.rb

@@ -0,0 +1,156 @@
+module WAZ
+  module Blobs
+    # This is internally used by the waz-blobs part of the gem and it exposes the Windows Azure Blob API REST methods 
+    # implementation. You can use this class to perform an specific operation that isn't provided by the current API.
+    class Service
+      include WAZ::Storage::SharedKeyCoreService
+      
+      # Creates a container on the current Windows Azure Storage account.
+      def create_container(container_name)
+        execute :put, container_name, {:restype => 'container'}, {:x_ms_version => '2009-09-19'}
+      end
+      
+      # Retrieves all the properties existing on the container.
+      def get_container_properties(container_name)
+        execute(:get, container_name, {:restype => 'container'}, {:x_ms_version => '2009-09-19'}).headers
+      end
+      
+      # Set the container properties (metadata). 
+      #
+      # Remember that custom properties should be named as :x_ms_meta_{propertyName} in order
+      # to have Windows Azure to persist them.
+      def set_container_properties(container_name, properties = {})
+        execute :put, container_name, { :restype => 'container', :comp => 'metadata' }, properties.merge!({:x_ms_version => '2009-09-19'})
+      end
+      
+      # Retrieves the value of the :x_ms_prop_publicaccess header from the
+      # container properties indicating whether the container is publicly 
+      # accessible or not.
+      def get_container_acl(container_name)
+        headers = execute(:get, container_name, { :restype => 'container', :comp => 'acl' }, {:x_ms_version => '2009-09-19'}).headers
+        headers[:x_ms_blob_public_access]
+      end
+
+      # Sets the value of the :x_ms_prop_publicaccess header from the
+      # container properties indicating whether the container is publicly 
+      # accessible or not.
+      #
+      # Default is _false_
+      def set_container_acl(container_name, public_available = WAZ::Blobs::BlobSecurity::Private)
+        publicity = {:x_ms_version => '2009-09-19' }
+        publicity[:x_ms_blob_public_access] = public_available unless public_available == WAZ::Blobs::BlobSecurity::Private
+        execute :put, container_name, { :restype => 'container', :comp => 'acl' }, publicity
+      end
+
+      # Lists all the containers existing on the current storage account.
+      def list_containers(options = {})
+        content = execute(:get, nil, options.merge(:comp => 'list'))
+        doc = REXML::Document.new(content)
+        containers = []
+        REXML::XPath.each(doc, '//Container/') do |item|
+          containers << { :name => REXML::XPath.first(item, "Name").text,
+                          :url => REXML::XPath.first(item, "Url").text,
+                          :last_modified => REXML::XPath.first(item, "LastModified").text}
+        end
+        return containers
+      end
+
+      # Deletes the given container from the Windows Azure Storage account.
+      def delete_container(container_name)
+        execute :delete, container_name, {:restype => 'container'}, {:x_ms_version => '2009-09-19'}
+      end
+
+      # Lists all the blobs inside the given container.
+      def list_blobs(container_name)
+        content = execute(:get, container_name, { :restype => 'container', :comp => 'list'}, {:x_ms_version => '2009-09-19'})
+        doc = REXML::Document.new(content)
+        containers = []
+        REXML::XPath.each(doc, '//Blob/') do |item|
+          containers << { :name => REXML::XPath.first(item, "Name").text,
+                          :url => REXML::XPath.first(item, "Url").text,
+                          :content_type =>  REXML::XPath.first(item.elements["Properties"], "Content-Type").text }
+
+        end
+        return containers
+      end
+
+      # Stores a blob on the given container.
+      # 
+      # Remarks path and payload are just text.
+      # 
+      # content_type is required by the blobs api, but on this method is defaulted to "application/octect-stream"
+      #
+      # metadata is a hash that stores all the properties that you want to add to the blob when creating it.
+      def put_blob(path, payload, content_type = "application/octet-stream", metadata = {})
+        default_headers = {"Content-Type" => content_type, :x_ms_version => "2009-09-19", :x_ms_blob_type => "BlockBlob"}
+        execute :put, path, nil, metadata.merge(default_headers), payload
+      end
+
+      # Commits a list of blocks to the given blob.
+      #
+      # blockids is a list of valid, already-uploaded block IDs (base64-encoded)
+      #
+      # content_type is required by the blobs api, but on this method is defaulted to "application/octect-stream"
+      #
+      # metadata is a hash that stores all the properties that you want to add to the blob when creating it.
+      def put_block_list(path, blockids, content_type = "application/octet-stream", metadata = {})
+        default_headers = {"Content-Type" => content_type, :x_ms_version => "2009-09-19"}
+        execute :put, path, { :comp => 'blocklist' }, metadata.merge(default_headers), '<?xml version="1.0" encoding="utf-8"?><BlockList>' + blockids.map {|id| "<Latest>#{id.rstrip}</Latest>"}.join + '</BlockList>'
+      end
+
+      # Retrieves a blob (content + headers) from the current path.
+      def get_blob(path, options = {})
+        execute :get, path, options, {:x_ms_version => "2009-09-19"}
+      end
+
+      # Deletes the blob existing on the current path.
+      def delete_blob(path)
+        execute :delete, path, nil, {:x_ms_version => "2009-09-19"}
+      end
+            
+      # Retrieves the properties associated with the blob at the given path.
+      def get_blob_properties(path, options = {})
+        execute(:head, path, options, {:x_ms_version => "2009-09-19"}).headers
+      end
+
+      # Sets the properties (metadata) associated to the blob at given path.
+      def set_blob_properties(path, properties ={})
+        execute :put, path, { :comp => 'properties' }, properties.merge({:x_ms_version => "2009-09-19"})
+      end
+      
+      # Set user defined metadata - overwrites any previous metadata key:value pairs
+      def set_blob_metadata(path, metadata = {}) 
+        execute :put, path, { :comp => 'metadata' }, metadata.merge({:x_ms_version => "2009-09-19"})
+      end 
+
+      # Copies a blob within the same account (not necessarily to the same container)
+      def copy_blob(source_path, dest_path)
+        execute :put, dest_path, nil, { :x_ms_version => "2009-09-19", :x_ms_copy_source => canonicalize_message(source_path) }
+      end
+      
+      # Adds a block to the block list of the given blob
+      def put_block(path, identifier, payload)
+        execute :put, path, { :comp => 'block', :blockid => identifier }, {'Content-Type' => "application/octet-stream"}, payload
+      end
+      
+      # Retrieves the list of blocks associated with a single blob. The list is filtered (or not) by type of blob
+      def list_blocks(path, block_list_type = 'all')
+        raise WAZ::Storage::InvalidParameterValue , {:name => :blocklisttype, :values => ['all', 'uncommitted', 'committed']} unless (block_list_type or "") =~ /all|committed|uncommitted/i
+        content = execute(:get, path, {:comp => 'blocklist'}.merge(:blocklisttype => block_list_type.downcase), { :x_ms_version => "2009-04-14" })
+        doc = REXML::Document.new(content)
+        blocks = []
+        REXML::XPath.each(doc, '//Block/') do |item|
+          blocks << { :name => REXML::XPath.first(item, "Name").text,
+                      :size => REXML::XPath.first(item, "Size").text,
+                      :committed => item.parent.name == "CommittedBlocks" }
+        end
+        return blocks
+      end
+      
+      # Creates a read-only snapshot of a blob as it looked like in time.
+      def snapshot_blob(path)
+        execute(:put, path, { :comp => 'snapshot' }, {:x_ms_version => "2009-09-19"}).headers[:x_ms_snapshot]
+      end
+    end
+  end
+end