terraspace_plugin_azurerm 0.1.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3845e196c53c10cdeb018eb26c60e29668f4cfd2de55296b8ccdd60373aa1582
-  data.tar.gz: 49ba0fa4c80a3be68dfa90814e8dbad2ad4516591b447fd24c4bbe0a70c0e604
+  metadata.gz: 8704700ce76610490c5f29da3091ea5b9b14489911095418deb51561920f2062
+  data.tar.gz: e60b1a62a544c6cbc21838c13c41c93541806786600c85620a650e45a0aa7ae2
 SHA512:
-  metadata.gz: c4299d6ce1c0e8d0f96187803d2d34fdeb1dc7a0b17c14ccf11777b2b26e54d031a9a166754932c6f1575e09eac722cf5d6c0073bd3f2e219b2c9365605df646
-  data.tar.gz: f44e6b81bc3bf2a6b7e3932d5221d71e9d45582c1ad7c36cedbf49ed270e52bdc8ffab496284df5aecf6bbf01a6c3f8f8b405213b1f3c0cae98562862dc1c2f9
+  metadata.gz: f800a5c07d7ce75482b99b7ee459d8a87c9607e99086d149b83cc8cb81618a569906480df8c9b2457a81a4a4374eef0d2bcab7224998c894161f6e041e9444b3
+  data.tar.gz: 5ffcda4941b3b43d275d907861606b5c20f6cf76a570169c6369c2093e1a7dc4c0022180f7df6de33d1c135fdfcb73de61896f78ab186705f231fe1e73ac5a66

data/CHANGELOG.md

@@ -3,6 +3,22 @@
 All notable changes to this project will be documented in this file.
 This project *loosely tries* to adhere to [Semantic Versioning](http://semver.org/).
 
+## [0.3.0] - 2020-11-15
+- [#5](https://github.com/boltops-tools/terraspace_plugin_azurerm/pull/5) helper and secrets support
+- azure_secret helper
+
+## [0.2.3]
+- #4 validate env vars and use older storage client
+
+## [0.2.2]
+- #3 fix test template
+
+## [0.2.1]
+- set prefix to @folder for performance improvement
+
+## [0.2.0]
+- #1 include layer interface, update template to use expansion method, add region method
+
 ## [0.1.1]
 - update generator init terraform state path
 

data/README.md

@@ -27,7 +27,7 @@ end
 
 By default, this plugin will automatically create the:
 
-* [resource group](Pluginazurerm)
+* [resource group](https://docs.microsoft.com/en-us/azure/azure-resource-manager/management/manage-resource-groups-portal)
 * [storage account](https://docs.microsoft.com/en-us/azure/storage/common/storage-account-create?tabs=azure-portal)
 * [storage container](https://docs.microsoft.com/en-us/cli/azure/storage/container?view=azure-cli-latest#az-storage-container-create)
 
@@ -35,15 +35,17 @@ The settings generally only apply if the resource does not yet exist yet and is
 
 ## Environment Variables
 
-To create the Azure resources like [resource group](Pluginazurerm), [storage account](https://docs.microsoft.com/en-us/azure/storage/common/storage-account-create?tabs=azure-portal), and [storage container](https://docs.microsoft.com/en-us/cli/azure/storage/container?view=azure-cli-latest#az-storage-container-create) these environment variables are required:
+To create the Azure resources like [resource group](https://docs.microsoft.com/en-us/azure/azure-resource-manager/management/manage-resource-groups-portal), [storage account](https://docs.microsoft.com/en-us/azure/storage/common/storage-account-create?tabs=azure-portal), and [storage container](https://docs.microsoft.com/en-us/cli/azure/storage/container?view=azure-cli-latest#az-storage-container-create) these environment variables are required:
 
-    AZURE_CLIENT_ID
-    AZURE_CLIENT_SECRET
+    ARM_CLIENT_ID
+    ARM_CLIENT_SECRET
 
-There's other env variables can also be set, but are generally inferred.
+Other env variables can be optionally set:
 
-    AZURE_TENANT_ID
-    AZURE_SUBSCRIPTION_ID
+    ARM_TENANT_ID
+    ARM_SUBSCRIPTION_ID
+
+When not set, their values are inferred from the [az cli](https://docs.microsoft.com/en-us/cli/azure/) settings. For those interested, this is done with the [boltops-tools/azure_info](https://github.com/boltops-tools/azure_info) library.
 
 ## Contributing
 

data/lib/templates/hcl/project/config/terraform/backend.tf

@@ -1,10 +1,10 @@
 # SUBSCRIPTION_HASH is a short 4-char consistent hash of the longer subscription id.
-# This is useful because azure storage accounts not allowed special characters and can only be 24 chars long.
+# This is useful because azure storage account names are not allowed special characters and are limited to 24 chars.
 terraform {
   backend "azurerm" {
-    resource_group_name  = "<%= backend_expand('azurerm', 'terraform-resources-:LOCATION') %>"
-    storage_account_name = "<%= backend_expand('azurerm', 'ts:SUBSCRIPTION_HASH:LOCATION:ENV') %>"
+    resource_group_name  = "<%= expansion('terraform-resources-:LOCATION') %>"
+    storage_account_name = "<%= expansion('ts:SUBSCRIPTION_HASH:LOCATION:ENV') %>"
     container_name       = "terraform-state"
-    key                  = "<%= backend_expand('azurerm', ':LOCATION/:ENV/:BUILD_DIR/terraform.tfstate') %>"
+    key                  = "<%= expansion(':LOCATION/:ENV/:BUILD_DIR/terraform.tfstate') %>"
   }
 }

data/lib/templates/ruby/project/config/terraform/backend.rb

@@ -1,5 +1,5 @@
 # SUBSCRIPTION_HASH is a short 4-char consistent hash of the longer subscription id.
-# This is useful because azure storage accounts not allowed special characters and can only be 24 chars long.
+# This is useful because azure storage account names are not allowed special characters and are limited to 24 chars.
 backend("azurerm",
   resource_group_name:  "terraform-resources-:LOCATION",
   storage_account_name: "ts:SUBSCRIPTION_HASH:LOCATION:ENV",

data/lib/templates/test/rspec/project/spec/fixtures/config/app.rb

@@ -0,0 +1,4 @@
+Terraspace.configure do |config|
+  config.logger.level = :info
+  config.test_framework = "rspec"
+end

data/lib/templates/test/rspec/project/spec/stacks/demo/main_spec.rb

@@ -8,6 +8,7 @@ describe "main" do
       stacks:  "app/stacks",           # include all stacks in this folder
       # override demo stack tfvars for testing
       # copied over to test harness' app/stacks/demo/tfvars/test.tfvars
+      # need for azure
       tfvars:  {demo: "spec/fixtures/tfvars/demo.tfvars"},
       config: "spec/fixtures/config",
     )
@@ -18,9 +19,6 @@ describe "main" do
   end
 
   it "successful deploy" do
-    # Replace with your actual test
-    expect(true).to be true
-    # Example
     storage_account_id = terraspace.output("demo", "storage_account_id")
     expect(storage_account_id).to include("sa") # starts with sa
   end

data/lib/terraspace_plugin_azurerm.rb

@@ -23,12 +23,22 @@ module TerraspacePluginAzurerm
     Interfaces::Config.instance.config
   end
 
+  @@logger = nil
+  def logger
+    @@logger ||= Terraspace.logger
+  end
+
+  def logger=(v)
+    @@logger = v
+  end
+
   extend self
 end
 
 Terraspace::Plugin.register("azurerm",
   backend: "azurerm",
   config_class: TerraspacePluginAzurerm::Interfaces::Config,
+  helper_class: TerraspacePluginAzurerm::Interfaces::Helper,
   layer_class: TerraspacePluginAzurerm::Interfaces::Layer,
   root: File.dirname(__dir__),
 )

data/lib/terraspace_plugin_azurerm/clients/options.rb

@@ -3,21 +3,49 @@ module TerraspacePluginAzurerm::Clients
     extend Memoist
 
     def client_options
-      client_id       = ENV['AZURE_CLIENT_ID']
-      client_secret   = ENV['AZURE_CLIENT_SECRET']
-      subscription_id = ENV['AZURE_SUBSCRIPTION_ID'] || AzureInfo.subscription_id
-      tenant_id       = ENV['AZURE_TENANT_ID'] || AzureInfo.tenant_id
+      o = base_client_options
+      o[:credentials] = credentials
+      o
+    end
+
+    def credentials
+      o = base_client_options
+      provider = MsRestAzure::ApplicationTokenProvider.new(o[:tenant_id], o[:client_id], o[:client_secret])
+      MsRest::TokenCredentials.new(provider)
+    end
 
-      provider = MsRestAzure::ApplicationTokenProvider.new(tenant_id, client_id, client_secret)
-      credentials = MsRest::TokenCredentials.new(provider)
+    def base_client_options
+      # AZURE_* is used by ruby generally.
+      # ARM_* is used by Terraform azurerm provider: https://www.terraform.io/docs/providers/azurerm/index.html
+      # Favor ARM_ because this plugin is designed for Terraspace.
+      client_id       = ENV['ARM_CLIENT_ID'] || ENV['AZURE_CLIENT_ID']
+      client_secret   = ENV['ARM_CLIENT_SECRET'] || ENV['AZURE_CLIENT_SECRET']
+      subscription_id = ENV['ARM_SUBSCRIPTION_ID'] || ENV['AZURE_SUBSCRIPTION_ID'] || AzureInfo.subscription_id
+      tenant_id       = ENV['ARM_TENANT_ID'] || ENV['AZURE_TENANT_ID'] || AzureInfo.tenant_id
 
-      {
+      o = {
         tenant_id: tenant_id,
         client_id: client_id,
         client_secret: client_secret,
         subscription_id: subscription_id,
-        credentials: credentials
       }
+      validate_base_options!(o)
+      o
+    end
+    memoize :base_client_options
+
+    def validate_base_options!(options)
+      vars = []
+      options.each do |k,v|
+        vars << "ARM_#{k}".upcase if v.nil?
+      end
+      return if vars.empty?
+
+      logger.error "ERROR: Required Azure env variables missing. Please set these env variables:".color(:red)
+      vars.each do |var|
+        logger.error "    #{var}"
+      end
+      exit 1
     end
   end
 end

data/lib/terraspace_plugin_azurerm/clients/storage.rb

@@ -6,8 +6,8 @@ module TerraspacePluginAzurerm::Clients
     extend Memoist
 
     # Include SDK modules to ease access to Storage classes.
-    include Azure::Storage::Profiles::Latest::Mgmt
-    include Azure::Storage::Profiles::Latest::Mgmt::Models
+    include Azure::Storage::Mgmt::V2019_06_01
+    include Azure::Storage::Mgmt::V2019_06_01::Models
 
     def storage_accounts
       mgmt.storage_accounts
@@ -19,7 +19,9 @@ module TerraspacePluginAzurerm::Clients
     memoize :blob_containers
 
     def mgmt
-      Client.new(client_options)
+      client = StorageManagementClient.new(credentials)
+      client.subscription_id = client_options[:subscription_id]
+      client
     end
     memoize :mgmt
   end

data/lib/terraspace_plugin_azurerm/interfaces/backend/storage_container.rb

@@ -1,7 +1,6 @@
 class TerraspacePluginAzurerm::Interfaces::Backend
   class StorageContainer < Base
     include TerraspacePluginAzurerm::Clients::Storage
-    extend Memoist
 
     def create
       if exist?

data/lib/terraspace_plugin_azurerm/interfaces/config.rb

@@ -15,12 +15,12 @@ module TerraspacePluginAzurerm::Interfaces
       c = ActiveSupport::OrderedOptions.new
       c.auto_create = true
       c.location = nil # AzureInfo.location not assigned here so it can be lazily inferred
-
+      c.secrets = ActiveSupport::OrderedOptions.new
+      c.secrets.vault = nil
       c.storage_account = ActiveSupport::OrderedOptions.new
       c.storage_account.sku = ActiveSupport::OrderedOptions.new
       c.storage_account.sku.name = "Standard_LRS"
       c.storage_account.sku.tier = "Standard"
-
       c
     end
   end

data/lib/terraspace_plugin_azurerm/interfaces/expander.rb

@@ -16,13 +16,14 @@ module TerraspacePluginAzurerm::Interfaces
 
     delegate :subscription_id, :subscription, :tenant_id, :tenant_id, :group, :location, to: :azure_info
     alias_method :namespace, :subscription
+    alias_method :region, :location
 
     def azure_info
       AzureInfo
     end
 
     # subscription_hash is a short 4-char consistent hash of the longer subscription id.
-    # This is useful because azure storage accounts not allowed special characters and can only be 24 chars long.
+    # This is useful because azure storage account names are not allowed special characters and are limited to 24 chars.
     # NOTE: be careful to not change this! or else state path will change
     def subscription_hash
       Digest::SHA1.hexdigest(subscription)[0..3]

data/lib/terraspace_plugin_azurerm/interfaces/helper.rb

@@ -0,0 +1,10 @@
+module TerraspacePluginAzurerm::Interfaces
+  module Helper
+    include Terraspace::Plugin::Helper::Interface
+
+    def azure_secret(name, options={})
+      Secret.new(options).fetch(name, options)
+    end
+    cache_helper :azure_secret
+  end
+end

data/lib/terraspace_plugin_azurerm/interfaces/helper/secret.rb

@@ -0,0 +1,26 @@
+require "base64"
+
+module TerraspacePluginAzurerm::Interfaces::Helper
+  class Secret
+    extend Memoist
+    include TerraspacePluginAzurerm::Logging
+    include TerraspacePluginAzurerm::Clients::Options
+
+    def initialize(options={})
+      @options = options
+      @base64 = options[:base64]
+    end
+
+    # opts: version, vault
+    def fetch(name, opts={})
+      value = fetcher.fetch(name, opts)
+      value = Base64.strict_encode64(value).strip if @base64
+      value
+    end
+
+    def fetcher
+      Fetcher.new
+    end
+    memoize :fetcher
+  end
+end

data/lib/terraspace_plugin_azurerm/interfaces/helper/secret/fetcher.rb

@@ -0,0 +1,112 @@
+require "net/http"
+
+class TerraspacePluginAzurerm::Interfaces::Helper::Secret
+  class Fetcher
+    class Error < StandardError; end
+    class VaultNotFoundError < Error; end
+
+    include TerraspacePluginAzurerm::Logging
+    include TerraspacePluginAzurerm::Clients::Options
+
+    def initialize
+      o = base_client_options
+      @client_id     = o[:client_id]
+      @client_secret = o[:client_secret]
+      @tenant_id     = o[:tenant_id]
+    end
+
+    def fetch(name, opts={})
+      opts[:vault] ||= TerraspacePluginAzurerm.config.secrets.vault
+      get_secret(name, opts)
+    end
+
+    def get_secret(name, vault: nil, version: nil)
+      unless token
+        return "ERROR: Unable to authorize and get the temporary token. Double check your ARM_ env variables."
+      end
+
+      version = "/#{version}" if version
+      vault_subdomain = vault.downcase
+      # Using Azure REST API since the old gem doesnt support secrets https://github.com/Azure/azure-sdk-for-ruby
+      # https://docs.microsoft.com/en-us/rest/api/keyvault/getsecret/getsecret
+      url = "https://#{vault_subdomain}.vault.azure.net/secrets/#{name}#{version}?api-version=7.1"
+      uri = URI(url)
+      req = Net::HTTP::Get.new(uri)
+      req["Authorization"] = token
+      req["Content-Type"] = "application/json"
+
+      resp = nil
+      begin
+        resp = send_request(uri, req)
+      rescue VaultNotFoundError
+        message = "WARN: Vault not found #{vault}"
+        logger.info message.color(:yellow)
+        return message
+      end
+
+      case resp.code.to_s
+      when /^2/
+        data = JSON.load(resp.body)
+        data['value']
+      else
+        message = standard_error_message(resp)
+        logger.info "WARN: #{message}".color(:yellow)
+        message
+      end
+    end
+
+    def send_request(uri, req)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.open_timeout = http.read_timeout = 30
+      http.use_ssl = true if uri.scheme == 'https'
+
+      begin
+        http.request(req) # response
+      rescue SocketError => e
+        # SocketError: Failed to open TCP connection to MISSING-VAULT.vault.azure.net:443 (getaddrinfo: Name or service not known)
+        if e.message.include?("vault.azure.net")
+          raise VaultNotFoundError.new(e)
+        else
+          raise
+        end
+      end
+    end
+
+    # Secret error handling: 1. network 2. json parse 3. missing secret
+    #
+    # Azure API responses with decent error message when
+    #   403 Forbidden - KeyVault Access Policy needs to be set up
+    #   404 Not Found - Secret name is incorrect
+    #
+    def standard_error_message(resp)
+      data = JSON.load(resp.body)
+      data['error']['message']
+    rescue JSON::ParserError
+      resp.body
+    end
+
+    @@token = nil
+    def token
+      return @@token unless @@token.nil?
+      url = "https://login.microsoftonline.com/#{@tenant_id}/oauth2/token"
+      uri = URI(url)
+      req = Net::HTTP::Get.new(uri)
+      req.set_form_data(
+        grant_type: "client_credentials",
+        client_id: @client_id,
+        client_secret: @client_secret,
+        resource: "https://vault.azure.net",
+      )
+      resp = send_request(uri, req)
+      data = JSON.load(resp.body)
+      if resp.code =~ /^2/
+        @@token = "Bearer #{data['access_token']}" if data
+      else
+        logger.info "WARN: #{data['error_description']}".color(:yellow)
+        # return false otherwise error message is used as the bearer toke and get this error:
+        # ArgumentError: header field value cannot include CR/LF
+        @@token = false
+      end
+    end
+  end
+end

data/lib/terraspace_plugin_azurerm/interfaces/layer.rb

@@ -1,6 +1,7 @@
 module TerraspacePluginAzurerm::Interfaces
   class Layer
     extend Memoist
+    include Terraspace::Plugin::Layer::Interface
 
     # interface method
     def namespace
@@ -11,10 +12,5 @@ module TerraspacePluginAzurerm::Interfaces
     def region
       AzureInfo.location
     end
-
-    # interface method
-    def provider
-      "azurerm"
-    end
   end
 end

data/lib/terraspace_plugin_azurerm/interfaces/summary.rb

@@ -53,7 +53,7 @@ module TerraspacePluginAzurerm::Interfaces
 
     # Friendly error handling for user
     def list_blobs(container_name, marker:)
-      blob_client.list_blobs(container_name, marker: marker)
+      blob_client.list_blobs(container_name, marker: marker, prefix: @folder)
     rescue Azure::Core::Http::HTTPError => e
       if e.message.include?("AuthenticationFailed")
         logger.error "e.class #{e.class}: #{e.message}"

data/lib/terraspace_plugin_azurerm/logging.rb

@@ -0,0 +1,7 @@
+module TerraspacePluginAzurerm
+  module Logging
+    def logger
+      Terraspace.logger
+    end
+  end
+end

data/lib/terraspace_plugin_azurerm/version.rb

@@ -1,3 +1,3 @@
 module TerraspacePluginAzurerm
-  VERSION = "0.1.1"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: terraspace_plugin_azurerm
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Tung Nguyen
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-07-24 00:00:00.000000000 Z
+date: 2020-11-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: azure-storage-blob
@@ -134,6 +134,7 @@ files:
 - lib/templates/test/rspec/module/test/spec/fixtures/stack/variables.tf
 - lib/templates/test/rspec/module/test/spec/main_spec.rb.tt
 - lib/templates/test/rspec/module/test/spec/spec_helper.rb
+- lib/templates/test/rspec/project/spec/fixtures/config/app.rb
 - lib/templates/test/rspec/project/spec/fixtures/config/terraform/provider.tf
 - lib/templates/test/rspec/project/spec/fixtures/tfvars/demo.tfvars
 - lib/templates/test/rspec/project/spec/spec_helper.rb
@@ -150,8 +151,12 @@ files:
 - lib/terraspace_plugin_azurerm/interfaces/backend/storage_container.rb
 - lib/terraspace_plugin_azurerm/interfaces/config.rb
 - lib/terraspace_plugin_azurerm/interfaces/expander.rb
+- lib/terraspace_plugin_azurerm/interfaces/helper.rb
+- lib/terraspace_plugin_azurerm/interfaces/helper/secret.rb
+- lib/terraspace_plugin_azurerm/interfaces/helper/secret/fetcher.rb
 - lib/terraspace_plugin_azurerm/interfaces/layer.rb
 - lib/terraspace_plugin_azurerm/interfaces/summary.rb
+- lib/terraspace_plugin_azurerm/logging.rb
 - lib/terraspace_plugin_azurerm/version.rb
 - terraspace_plugin_azurerm.gemspec
 homepage: https://github.com/boltops-tools/terraspace_plugin_azurerm
@@ -174,7 +179,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Terraspace Azurerm Cloud Plugin