fog-azure-rm 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fcef6aa93014d6ab06b64750d6ab060e209f8d68
-  data.tar.gz: 35bc5f202cb58139facb70944270721c96436ab1
+  metadata.gz: 15dd6d696881e8c4cee014dda7af43a3d8cc764f
+  data.tar.gz: 4b5d62e0d1e723cc7eb3a972caf1b291d59ad5d9
 SHA512:
-  metadata.gz: 360d9b598b28e2669f160425a05fd6996d764e52b943cc626ec6df995b2c3754cecc895d404bbade9af4cfa63bb674e6e430cf2b6e284309c4b5c89a407d511b
-  data.tar.gz: 67912e5342afef2657d176406f39a0239377f40d3240e04567641e3c88f5fefa2f513006ce05d9c82780692937ebf47f2c89e5f29b75d75b937b479083045f89
+  metadata.gz: afc8b7054bd8b01ff6d47caae60f4482aede6bdb4c2e01cb70cf273b615e0867c013fdf5260d79a1f7fdff077096d61680f218c7dac41c7785418e54157d8fa0
+  data.tar.gz: d9c876d9da40fd246df634beb2d6eb35e696e7f90f0f7c1aa8715cfd816c1adeae0ca490490b625224f93d7afc7e2ad8abbd4f3945607a2555a7f6dae9548929

data/CHANGELOG.md

@@ -1,3 +1,23 @@
+## 0.1.1
+
+**Added:**
+- Storage Service - Multi-thread support for uploading blobs
+- Added support for all Azure locations(Global, China, Gov and Germany) 
+
+**Changed:**
+* Updated fog-azure-rm runtime gem dependencies to latest:
+  * azure_mgmt_compute 0.7.0
+  * azure_mgmt_network 0.7.0
+  * azure_mgmt_resources 0.7.0
+  * azure_mgmt_storage 0.7.0
+  * azure_mgmt_dns 0.7.0
+  * azure_mgmt_traffic_manager 0.7.0
+  * azure-storage 0.11.3.preview'
+* Provided same storage interfaces as other providers (PR: 204)
+* Updated Data Disk operations (PR: 220)
+* Updated namespaces
+* Updated integration scripts
+
 ## 0.1.0
 
 **Added:**

data/README.md

@@ -33,7 +33,8 @@ Or install it yourself as:
 ## Usage
 
 You can use this gem against the Microsoft Azure Resource Manager Services in the cloud. Of course, to use the Microsoft Azure Resource Manager Services in the cloud, you need to first [create a Microsoft Azure account](http://www.azure.com/en-us/pricing/free-trial/).
-
+* Set 'CLOUD' constant from 'Azure', 'AzureChina', 'AzureGermanCloud' or 'AzureUSGovernment' in [config.rb] (https://github.com/fog/fog-azure-rm/blob/master/lib/fog/azurerm/config.rb) file. By default it will be 'Azure' (Azure Global).
+* Set 'LOCATION' constant based on the Azure cloud you are using in [config.rb] (https://github.com/fog/fog-azure-rm/blob/master/lib/fog/azurerm/config.rb) file. By default it will be 'eastus'.
 
 ### Authentication
 

data/fog-azure-rm.gemspec

@@ -20,15 +20,17 @@ Gem::Specification.new do |spec|
   spec.post_install_message = 'Thanks for installing!'
   spec.add_development_dependency 'rake', '~> 10.0'
   spec.add_development_dependency 'minitest', '~> 5.8.4'
-  spec.add_development_dependency 'simplecov', '~> 0.11.2'
-  spec.add_development_dependency 'codeclimate-test-reporter'
+  spec.add_development_dependency 'simplecov'
+  spec.add_development_dependency 'codeclimate-test-reporter' , '~> 1.0.0'
   spec.add_dependency 'fog-core', '~> 1.43.0'
+  spec.add_dependency 'fog-json', '~> 1.0'
   spec.add_dependency 'rest-client', '~> 2.0.0'
-  spec.add_dependency 'azure_mgmt_compute', '~> 0.6.0'
-  spec.add_dependency 'azure_mgmt_resources', '~> 0.6.0'
-  spec.add_dependency 'azure_mgmt_storage', '~> 0.6.0'
-  spec.add_dependency 'azure_mgmt_network', '~> 0.6.0'
-  spec.add_dependency 'azure_mgmt_dns', '~> 0.6.0'
-  spec.add_dependency 'azure_mgmt_traffic_manager', '~> 0.6.0'
-  spec.add_dependency 'azure-storage', '~> 0.11.1.preview'
+  spec.add_dependency 'azure_mgmt_compute', '~> 0.7.0'
+  spec.add_dependency 'azure_mgmt_resources', '~> 0.7.0'
+  spec.add_dependency 'azure_mgmt_storage', '~> 0.7.0'
+  spec.add_dependency 'azure_mgmt_network', '~> 0.7.0'
+  spec.add_dependency 'azure_mgmt_dns', '~> 0.7.0'
+  spec.add_dependency 'azure_mgmt_traffic_manager', '~> 0.7.0'
+  spec.add_dependency 'azure-storage', '~> 0.11.3.preview'
+  spec.add_dependency 'vhd', '0.0.4'
 end

data/lib/fog/azurerm.rb

@@ -5,6 +5,7 @@ require 'fog/azurerm/config'
 require 'fog/azurerm/utilities/general'
 require 'fog/azurerm/version'
 require 'fog/core'
+require 'fog/json'
 
 module Fog
   # Autoload Module for Credentials

data/lib/fog/azurerm/compute.rb

@@ -49,7 +49,7 @@ module Fog
           end
         end
       end
-      # This class provides the actual implemention for service calls.
+      # This class provides the actual implementation for service calls.
       class Real
         def initialize(options)
           begin
@@ -62,9 +62,9 @@ module Fog
           end
 
           credentials = Fog::Credentials::AzureRM.get_credentials(options[:tenant_id], options[:client_id], options[:client_secret])
-          @compute_mgmt_client = ::Azure::ARM::Compute::ComputeManagementClient.new(credentials)
+          @compute_mgmt_client = ::Azure::ARM::Compute::ComputeManagementClient.new(credentials, resource_manager_endpoint_url)
           @compute_mgmt_client.subscription_id = options[:subscription_id]
-          @storage_mgmt_client = ::Azure::ARM::Storage::StorageManagementClient.new(credentials)
+          @storage_mgmt_client = ::Azure::ARM::Storage::StorageManagementClient.new(credentials, resource_manager_endpoint_url)
           @storage_mgmt_client.subscription_id = options[:subscription_id]
           @storage_service = Fog::Storage::AzureRM.new(tenant_id: options[:tenant_id], client_id: options[:client_id], client_secret: options[:client_secret], subscription_id: options[:subscription_id])
         end

data/lib/fog/azurerm/config.rb

@@ -20,3 +20,36 @@ AZURE_ENDPOINTS = 'azureEndpoints'.freeze
 EXTERNAL_ENDPOINTS = 'externalEndpoints'.freeze
 NESTED_ENDPOINTS = 'nestedEndpoints'.freeze
 GLOBAL = 'global'.freeze
+UPLOAD_BLOB_WORKER_THREAD_COUNT = 8
+VM_NAME_POSITION = 8
+
+# Set a CLOUD value from 'Azure', 'AzureChina', 'AzureGermanCloud' and 'AzureUSGovernment'
+CLOUD = 'Azure'.freeze
+
+AZURE_GLOBAL_RM_ENDPOINT_URL = 'https://management.azure.com/'.freeze
+AZURE_CHINA_RM_ENDPOINT_URL = 'https://management.chinacloudapi.cn'.freeze
+GERMAN_CLOUD_RM_ENDPOINT_URL = 'https://management.microsoftazure.de'.freeze
+US_GOVERNMENT_RM_ENDPOINT_URL = 'https://management.usgovcloudapi.net'.freeze
+
+LOCATION = 'eastus'.freeze
+
+# State of the copy operation
+COPY_STATUS = {
+  # The copy completed successfully.
+  SUCCESS: 'success',
+  # The copy is in progress
+  PENDING: 'pending'
+}.freeze
+
+# https://msdn.microsoft.com/en-us/library/azure/dd179451.aspx
+# The maximum size for a block blob created via Put Blob is 64 MB. But for better performance, this size should be 32 MB.
+# If your blob is larger than 32 MB, you must upload it as a set of blocks.
+SINGLE_BLOB_PUT_THRESHOLD = 32 * 1024 * 1024
+
+# Block blob: https://msdn.microsoft.com/en-us/library/azure/dd135726.aspx
+# Page blob: https://msdn.microsoft.com/en-us/library/azure/ee691975.aspx
+# Each block/page can be a different size, up to a maximum of 4 MB
+MAXIMUM_CHUNK_SIZE = 4 * 1024 * 1024
+
+# The hash value of 4MB empty content
+HASH_OF_4MB_EMPTY_CONTENT = 'b5cfa9d6c8febd618f91ac2843d50a1c'.freeze

data/lib/fog/azurerm/credentials.rb

@@ -20,7 +20,7 @@ module Fog
         @client_id = client_id
         @client_secret = client_secret
         return if @tenant_id.nil? || @client_id.nil? || @client_secret.nil?
-        @token_provider = MsRestAzure::ApplicationTokenProvider.new(@tenant_id, @client_id, @client_secret)
+        @token_provider = MsRestAzure::ApplicationTokenProvider.new(@tenant_id, @client_id, @client_secret, active_directory_service_settings)
         @credentials = MsRest::TokenCredentials.new(@token_provider)
         @credentials
       end

data/lib/fog/azurerm/dns.rb

@@ -31,7 +31,7 @@ module Fog
         end
       end
 
-      # This class provides the actual implemention for service calls.
+      # This class provides the actual implementation for service calls.
       class Real
         def initialize(options)
           begin
@@ -42,7 +42,7 @@ module Fog
           end
 
           credentials = Fog::Credentials::AzureRM.get_credentials(options[:tenant_id], options[:client_id], options[:client_secret])
-          @dns_client = ::Azure::ARM::Dns::DnsManagementClient.new(credentials)
+          @dns_client = ::Azure::ARM::Dns::DnsManagementClient.new(credentials, resource_manager_endpoint_url)
           @dns_client.subscription_id = options[:subscription_id]
           @tenant_id = options[:tenant_id]
           @client_id = options[:client_id]

data/lib/fog/azurerm/docs/storage.md

@@ -163,10 +163,11 @@ azure_storage_service.delete_disk('<Data Disk Name>')
 Create a storage container in the current storage account.
 
 ```ruby
-container = azure_storage_service.create_container(
-  name: '<container name>'
+directory = azure_storage_service.directories.create(
+  key: '<container name>',
+  public: true
 )
-puts "#{container.name}"
+puts directory.key
 ```
 
 ## List storage containers
@@ -175,18 +176,39 @@ List all the storage containers in the current storage accounts.
 
 ```ruby
 azure_storage_service.directories.each do |directory|
-  puts "#{directory.name}"
+  puts directory.key
 end
 ```
 
-## Get the access control list of the storage container
+## Get the access control level of the storage container
 
 Get the permissions for the specified container. The permissions indicate whether container data may be accessed publicly.
 
 ```ruby
-directory = azure_storage_service.directories.get('<container name>')
-access_control_list = directory.get_access_control_list('<container name>')
-puts "#{access_control_list.inspect}"
+directory = azure_storage_service.directories.get('<Container Name>', max_results: 1)
+puts directory.acl
+```
+
+## Set the access control level of the storage container
+
+Set the permissions for the specified container. The permissions indicate whether container data may be accessed publicly. The container permissions provide the following options for managing container access:
+
+  - container
+
+    Full public read access. Container and blob data can be read via anonymous request. Clients can enumerate blobs within the container via anonymous request, but cannot enumerate containers within the storage account.
+
+  - blob
+
+    Public read access for blobs only. Blob data within this container can be read via anonymous request, but container data is not available. Clients cannot enumerate blobs within the container via anonymous request.
+
+  - nil
+
+    No public read access. Container and blob data can be read by the account owner only.
+
+```ruby
+directory = azure_storage_service.directories.get('<Container Name>', max_results: 1)
+directory.acl = 'container'
+directory.save(is_create: false)
 ```
 
 ## Delete the storage container
@@ -194,91 +216,117 @@ puts "#{access_control_list.inspect}"
 Mark the specified container for deletion. The container and any blobs contained within it are later deleted during garbage collection.
 
 ```ruby
-directory = azure_storage_service.directories.get('<container name>')
-result = directory.destroy
-puts "#{result}"
+directory = azure_storage_service.directories.get('<Container Name>', max_results: 1)
+puts directory.destroy
 ```
 
-## Upload a local file as a blob
+## Upload data as a block blob
 ```ruby
-new_cloud_file = azure_storage_service.files.get('<Container name>', '<Blob name>').create(file_path: '<file path>')
-puts "#{new_cloud_file.inspect}"
+directory = azure_storage_service.directories.get('<Container Name>', max_results: 1)
+options = {
+  key: '<Blob Name>',
+  body: '<Blob Content>'
+}
+new_block_blob = directory.files.create(options)
+puts new_block_blob.inspect
 ```
 
-## Copy Blob from one container to another
+## Upload a local file as a block blob
 ```ruby
-puts storage_data.copy_blob('<destination_container_name>', '<destination_blob_name>', '<source_container_name>', '<source_blob_name>')
+directory = azure_storage_service.directories.get('<Container Name>', max_results: 1)
+File.open('<File Path>') do |file|
+  options = {
+    key: '<Blob Name>',
+    body: file
+  }
+  new_block_blob = directory.files.create(options)
+  puts new_block_blob.inspect
+end
 ```
 
-## Download a blob to a local file
+## Upload VHD data as a page blob
 ```ruby
-blob = azure_storage_service.files.get('<Container name>', '<Blob name>').save_to_file('<file path>')
-puts "#{blob.inspect}"
-puts "File Size: #{::File.size <file_path>}"
+directory = azure_storage_service.directories.get('<Container Name>', max_results: 1)
+options = {
+  key: '<Blob Name>',
+  body: '<Blob Content>',
+  blob_type: 'PageBlob'
+}
+new_page_blob = directory.files.create(options)
+puts new_page_blob.inspect
 ```
 
-## Delete the storage blob
-
-Mark the specified blob for deletion. The blob is later deleted during garbage collection.
-
+## Upload a local VHD as a page blob
 ```ruby
-cloud_file = azure_storage_service.files.get('<container name>', '<blob name>')
-result = cloud_file.destroy
-puts "#{result}"
+directory = azure_storage_service.directories.get('<Container Name>', max_results: 1)
+File.open('<File Path>') do |file|
+  options = {
+    key: '<Blob Name>',
+    body: file,
+    blob_type: 'PageBlob'
+  }
+  new_page_blob = directory.files.create(options)
+  puts new_page_blob.inspect
+end
 ```
 
-Note that in order to delete a blob, you must delete all of its snapshots.
-
+## Copy Blob from one container to another
 ```ruby
-cloud_file = azure_storage_service.files.get('<container name>', '<blob name>')
-result = cloud_file.destroy(delete_snapshots: 'only')
-puts "#{result}"
-
-result = cloud_file.destroy
-puts "#{result}"
+directory = azure_storage_service.directories.get('<Source Container Name>', max_results: 1)
+copied_blob = directory.files.head('<Source Blob Name>').copy('<Destination Container Name>', '<Destination Blob Name>')
+puts copied_blob.inspect
 ```
 
-You can delete both at the same time by specifying the option.
-
+## Copy Blob from one uri to self
 ```ruby
-cloud_file = azure_storage_service.files.get('<container name>', '<blob name>')
-result = cloud_file.destroy(delete_snapshots: 'inlcude')
-puts "#{result}"
+directory = azure_storage_service.directories.get('<Destination Container Name>', max_results: 1)
+copied_blob = directory.files.new(key: '<Destination Blob Name>')
+copied_blob.copy_from_uri('<Source Blob Uri>')
+puts copied_blob.inspect
 ```
 
-## Properties
-
-### Get storage container properties
-
-Get the storage container properties. The properties will not fetch the access control list. Call `get_container_access_control_list` to fetch it.
+## Download a small blob to a local file
+```ruby
+directory = azure_storage_service.directories.get('<Container Name>', max_results: 1)
+blob = directory.files.get('<Blob Name>')
+File.open('<File Path>', 'wb') do |file|
+  file.write(blob.body)
+end
+puts "File Size: #{::File.size <File Path>}"
+```
 
+## Download a large blob to a local file
 ```ruby
-directory = azure_storage_service.directories.get('<container name>')
-properties = directory.get_properties
-puts "#{properties.inspect}"
+directory = azure_storage_service.directories.get('<Container Name>', max_results: 1)
+File.open('<File Path>', 'wb') do |file|
+  directory.files.get('<Blob Name>') do |chunk, remaining_bytes, total_bytes|
+    puts "remaining_bytes: #{remaining_bytes}, total_bytes: #{total_bytes}"
+    file.write(chunk)
+  end
+end
+puts "File Size: #{::File.size <File Path>}"
 ```
 
-### Get storage blob properties
+## Delete the storage blob
 
-Get the storage blob properties.
+Mark the specified blob for deletion. The blob is later deleted during garbage collection.
 
 ```ruby
-cloud_file = azure_storage_service.files.get('<container name>', '<blob name>')
-properties = cloud_file.get_properties
-puts "#{properties.inspect}"
+directory = azure_storage_service.directories.get('<Container Name>', max_results: 1)
+blob = directory.files.head('<Blob Name>')
+puts blob.destroy
 ```
 
 ### Set storage blob properties
 
-Set the storage blob properties. The properties are passed in name/value pairs.
+Set the storage blob properties.
 
 ```ruby
-cloud_file = azure_storage_service.files.get('<container name>', '<blob name>')
-properties = {
-  "content_language" => "English",
-  "content_disposition" => "attachment"
-}
-cloud_file.set_properties(properties)
+directory = azure_storage_service.directories.get('<Container Name>', max_results: 1)
+blob = directory.files.head('<Blob Name>')
+blob.content_language = "English"
+blob.content_disposition = "attachment"
+blob.save(update_body: false)
 ```
 
 ## Metadata
@@ -288,35 +336,41 @@ Metadata allows us to provide descriptive information about specific containers
 ### Get Blob Metadata
 
 ```ruby
-azure_storage_service.files.get('<Container name>', '<Blob name>').get_metadata
+directory = azure_storage_service.directories.get('<Container Name>', max_results: 1)
+blob = directory.files.head('<Blob Name>')
+puts blob.metadata
 ```
 
 ### Set Blob Metadata
 
 ```ruby
-metadata = {
+directory = azure_storage_service.directories.get('<Container Name>', max_results: 1)
+blob = directory.files.head('<Blob Name>')
+blob.metadata = {
   "Category" => "Images",
   "Resolution" => "High"
 }
-azure_storage_service.files.get('<Container name>', '<Blob name>').set_metadata(metadata)
+blob.save(update_body: false)
 ```
 
 ### Get Container Metadata
 
 ```ruby
-azure_storage_service.directories.get_metadata('<Container name>')
+directory = azure_storage_service.directories.get('<Container Name>', max_results: 1)
+puts directory.metadata
 ```
 
 ### Set Container Metadata
 
 ```ruby
-metadata = {
+directory = azure_storage_service.directories.get('<Container Name>', max_results: 1)
+directory.metadata = {
   "CreatedBy" => "User",
   "SourceMachine" => "Mymachine",
   "category" => "guidance",
   "docType" => "textDocuments"
   }
-azure_storage_service.directories.set_metadata('<Container name>', metadata)
+directory.save(is_create: false)
 ```
 
 ### Create Recovery Vault