terraspace_plugin_aws 0.3.1 → 0.3.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 855a5c7166b24f8881c2d735edd0c252cefd6d0175a9a0f1156090ca08a11e5d
-  data.tar.gz: be95c5a77b116c62dac152ec6f8ec9cfb5b68ad372475373823508472bca3566
+  metadata.gz: c20f17d758fbb6924ceece1c1bbc16ec17e3eba0fa8a0a6fa4d644b9875579b6
+  data.tar.gz: 73b7f625a5f84f7252206717523807c7607ac4a4fe6f6572cfd01e7110641b38
 SHA512:
-  metadata.gz: 12ff9c7c0fcc8b10d82385855b5a5724392729ef79254cad41fce51b0ddab0d3ec17a927b08da1e071adde23e1c8e1289e48d32f7948753675b6d0fd680596c7
-  data.tar.gz: 717dffbad397bb5fd3953739060e5e2ad7bb6d90f842c13ab6f0fb00f5f3cf1b82357ed401b118a2ddde584c9dd6ccdda0712cce493fd2ec0be48433b35f52ab
+  metadata.gz: 39978c0db0055dba8dc59246b9da3dd20c469daf36805526f4b2c0c051425dd6dbf0be054812de9e7ae0c32b31657222225ae039915ddc3ef11aab657d9adbb9
+  data.tar.gz: ff11fee1dec2508d941dacd4f727ad362802285aca20ac2b0aa9cc87a034e0d287f9f10db3816e330958025e608fdeff45720b543047f1d3fe5fe52f5f305fb1

data/CHANGELOG.md

@@ -3,6 +3,20 @@
 All notable changes to this project will be documented in this file.
 This project *loosely tries* to adhere to [Semantic Versioning](http://semver.org/).
 
+## [0.3.5] - 2021-12-30
+- [#15](https://github.com/boltops-tools/terraspace_plugin_aws/pull/15) block public access support
+- [#16](https://github.com/boltops-tools/terraspace_plugin_aws/pull/16) tagging support for s3 bucket and dynamodb table
+
+## [0.3.4] - 2021-12-30
+- [#13](https://github.com/boltops-tools/terraspace_plugin_aws/pull/13) check aws setup and provide friendly message
+- [#14](https://github.com/boltops-tools/terraspace_plugin_aws/pull/14) fix aws_secret helper
+
+## [0.3.3] - 2021-12-14
+- [#10](https://github.com/boltops-tools/terraspace_plugin_aws/pull/10) implement expand_string? to not expand aws arn values
+
+## [0.3.2] - 2021-12-14
+- [#9](https://github.com/boltops-tools/terraspace_plugin_aws/pull/9) support separate aws account for s3 backend bucket
+
 ## [0.3.1] - 2021-12-14
 - [#8](https://github.com/boltops-tools/terraspace_plugin_aws/pull/8) use region configured in the backend.tf for the s3 client
 

data/lib/terraspace_plugin_aws/clients/options.rb

@@ -0,0 +1,98 @@
+module TerraspacePluginAws::Clients
+  module Options
+  private
+    def client_options
+      return {} unless @info # aws_secret helper wont have @info
+      if @info['role_arn']
+        client_assume_role_options
+      else
+        client_default_options
+      end
+    end
+
+    # Typically, aws sdk client options are inferred from the user environment unless set in the backend.tf
+    #
+    # terraform s3 backend assume role configuration: https://www.terraform.io/docs/language/settings/backends/s3.html
+    #
+    #     assume_role_duration_seconds - (Optional) Number of seconds to restrict the assume role session duration.
+    #     assume_role_policy - (Optional) IAM Policy JSON describing further restricting permissions for the IAM Role being assumed.
+    #     assume_role_policy_arns - (Optional) Set of Amazon Resource Names (ARNs) of IAM Policies describing further restricting permissions for the IAM Role being assumed.
+    #     assume_role_tags - (Optional) Map of assume role session tags.
+    #     assume_role_transitive_tag_keys - (Optional) Set of assume role session tag keys to pass to any subsequent sessions.
+    #     external_id - (Optional) External identifier to use when assuming the role.
+    #     role_arn - (Optional) Amazon Resource Name (ARN) of the IAM Role to assume.
+    #     session_name - (Optional) Session name to use when assuming the role.
+    #
+    # ruby sdk: https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/AssumeRoleCredentials.html
+    #
+    #     :role_arn (required, String)
+    #     :role_session_name (required, String)
+    #     :policy (String)
+    #     :duration_seconds (Integer)
+    #     :external_id (String)
+    #     :client (STS::Client)
+    #
+    def client_assume_role_options
+      whitelist = %w[
+        assume_role_duration_seconds
+        assume_role_policy
+        session_name
+        external_id
+        role_arn
+      ]
+      assume_role_config = @info.slice(*whitelist)
+      # not supported?
+      #   assume_role_policy_arns
+      #   assume_role_tags
+      #   assume_role_transitive_tag_keys
+      # already matches
+      #   external_id
+      #   role_arn
+      # rest needs to be mapped
+      map = {
+        'assume_role_duration_seconds' => 'duration_seconds',
+        'assume_role_policy' => 'policy',
+        'session_name' => 'role_session_name',
+      }
+      map.each do |terraform_key, ruby_sdk_key|
+        v = assume_role_config.delete(terraform_key)
+        assume_role_config[ruby_sdk_key] = v if v
+      end
+      assume_role_config.symbolize_keys! # ruby sdk expects symbols for keys
+      assume_role_config[:role_session_name] ||= [ENV['C9_USER'] || ENV['USER'], 'session'].compact.join('-') # session name is required for the ruby sdk
+      role_credentials = Aws::AssumeRoleCredentials.new(assume_role_config)
+      {credentials: role_credentials}
+    end
+
+    # terraform s3 backend configuration: https://www.terraform.io/docs/language/settings/backends/s3.html
+    #
+    #     access_key - (Optional) AWS access key. If configured, must also configure secret_key. This can also be sourced from the AWS_ACCESS_KEY_ID environment variable, AWS shared credentials file (e.g. ~/.aws/credentials), or AWS shared configuration file (e.g. ~/.aws/config).
+    #     secret_key - (Optional) AWS access key. If configured, must also configure access_key. This can also be sourced from the AWS_SECRET_ACCESS_KEY environment variable, AWS shared credentials file (e.g. ~/.aws/credentials), or AWS shared configuration file (e.g. ~/.aws/config).
+    #
+    # ruby sdk: https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/Credentials.html
+    #
+    #     access_key_id (String)
+    #     secret_access_key (String)
+    #     session_token (String) (defaults to: nil) — (nil)
+    #
+    def client_default_options
+      whitelist = %w[
+        access_key_id
+        secret_access_key
+        session_token
+        profile
+      ]
+      options = @info.slice(*whitelist)
+      options.symbolize_keys! # ruby sdk expects symbols for keys
+      client_region_option.merge(options)
+    end
+
+    def client_region_option
+      if @info['region']
+        {region: @info['region']}
+      else
+        {}
+      end
+    end
+  end
+end

data/lib/terraspace_plugin_aws/clients.rb

@@ -6,6 +6,7 @@ require "aws-sdk-ssm"
 module TerraspacePluginAws
   module Clients
     extend Memoist
+    include Options
 
     def s3
       Aws::S3::Client.new(client_options)
@@ -22,18 +23,14 @@ module TerraspacePluginAws
     end
     memoize :ssm
 
+    def sts
+      Aws::STS::Client.new(client_options)
+    end
+    memoize :sts
+
     def dynamodb
       Aws::DynamoDB::Client.new(client_options)
     end
     memoize :dynamodb
-
-    # Typically inferred from AWS_REGION unless set in the backend.tf
-    def client_options
-      if @info['region']
-        {region: @info['region']}
-      else
-        {}
-      end
-    end
   end
 end

data/lib/terraspace_plugin_aws/interfaces/backend/bucket/secure.rb

@@ -2,6 +2,27 @@ require "s3-secure"
 
 class TerraspacePluginAws::Interfaces::Backend::Bucket
   module Secure
+    # Why the retry logic?
+    #
+    # When using profile or role_arn in the terraform backend it the ruby aws sdk
+    # assumes the profile or role.
+    # In doing so, it errors when the s3-secure library calls s3_client.get_bucket_location
+    #
+    #   https://github.com/boltops-tools/s3-secure/blob/d2c8e9eba745a75d094a3c566bd5fe47476d3638/lib/s3_secure/aws_services/s3.rb#L43
+    #
+    # Here's an example stack trace of the error:
+    #
+    #   https://gist.github.com/tongueroo/dd74b67c17433c6f8dd890225104aef9
+    #
+    # Unsure if this is a terraform backend interfering with the ruby sdk thing (unlikely)
+    # Or if it's a general AWS sdk thing.
+    # Or if it's how I'm calling the sdk and initializing the client. Maybe an initializing the client early on and it caches it.
+    # Unsure. But using this hack instead because life's short.
+    #
+    # Throwing the retry logic in here fixes the issue. This only happens the when the bucket is brand new.
+    # Limiting the retry to only a single attempt.
+    #
+    @@retries = 0
     def secure(bucket)
       c = TerraspacePluginAws::Interfaces::Config.instance.config.s3
       options = {bucket: bucket, quiet: true}
@@ -10,6 +31,10 @@ class TerraspacePluginAws::Interfaces::Backend::Bucket
       S3Secure::Versioning::Enable.new(options).run if c.versioning
       S3Secure::Lifecycle::Add.new(options).run if c.lifecycle
       S3Secure::AccessLogs::Enable.new(options).run if c.access_logging
+      S3Secure::PublicAccess::Block.new(options).run if c.block_public_access
+    rescue Aws::S3::Errors::AccessDenied => e
+      @@retries += 1
+      retry unless @@retries > 1
     end
   end
 end

data/lib/terraspace_plugin_aws/interfaces/backend/bucket/tagging.rb

@@ -0,0 +1,44 @@
+class TerraspacePluginAws::Interfaces::Backend::Bucket
+  class Tagging
+    include TerraspacePluginAws::Clients
+    include TerraspacePluginAws::Logging
+
+    def initialize(bucket)
+      @bucket = bucket
+    end
+
+    def tag
+      return if tagging.nil? || tagging[:tag_set].empty? # safeguard: dont overwrite current tags
+      s3.put_bucket_tagging(bucket: @bucket, tagging: tagging)
+    end
+
+    # Merges existing tag_set structure so always appends tags, wont remove tags.
+    # This behavior is consistent with the dynamodb tagging.
+    #
+    # Example return:
+    #
+    #     {
+    #       tag_set: [
+    #         { key: "Key1", value: "Value1" },
+    #         { key: "Key2", value: "Value2" },
+    #       ],
+    #     }
+    #
+    def tagging
+      c = TerraspacePluginAws::Interfaces::Config.instance.config
+      tags = !c.s3.tags.empty? ? c.s3.tags : c.tags
+      tag_set = tags.map do |k,v|
+        {key: k.to_s, value: v}
+      end
+      return if tag_set == existing_tagging[:tag_set] # return nil so we can avoid the put_bucket_tagging call
+      tag_set += existing_tagging[:tag_set]
+      { tag_set: tag_set }
+    end
+
+    def existing_tagging
+      s3.get_bucket_tagging(bucket: @bucket).to_h
+    rescue Aws::S3::Errors::NoSuchTagSet
+      {tag_set: []} # normalize return structure
+    end
+  end
+end

data/lib/terraspace_plugin_aws/interfaces/backend/bucket.rb

@@ -10,12 +10,14 @@ class TerraspacePluginAws::Interfaces::Backend
       end
       if exist?(bucket)
         logger.debug "Bucket already exist: #{bucket}"
-        c = TerraspacePluginAws::Interfaces::Config.instance.config.s3
-        secure(bucket) if c.secure_existing
+        c = TerraspacePluginAws::Interfaces::Config.instance.config
+        secure(bucket) if c.s3.secure_existing
+        tag(bucket) if c.tag_existing
       else
         logger.info "Creating bucket: #{bucket}"
         s3.create_bucket(bucket: bucket)
         secure(bucket)
+        tag(bucket)
       end
     end
 
@@ -30,5 +32,9 @@ class TerraspacePluginAws::Interfaces::Backend
       logger.error "Bucket might be owned by someone else or is on another one of your AWS accounts."
       exit 1
     end
+
+    def tag(bucket)
+      Tagging.new(@info["bucket"]).tag
+    end
   end
 end

data/lib/terraspace_plugin_aws/interfaces/backend/setup.rb

@@ -0,0 +1,15 @@
+class TerraspacePluginAws::Interfaces::Backend
+  class Setup < Base
+    def check!
+      sts.get_caller_identity
+    rescue Aws::Errors::MissingCredentialsError => e
+      logger.info "ERROR: #{e.class}: #{e.message}".color(:red)
+      logger.info <<~EOL
+        It doesnt look like AWS credentials and access has been setup.
+        Please double check the AWS credentials setup.
+        IE: ~/.aws/config and the AWS_PROFILE env variable.
+      EOL
+      exit 1
+    end
+  end
+end

data/lib/terraspace_plugin_aws/interfaces/backend/table.rb

@@ -6,6 +6,8 @@ class TerraspacePluginAws::Interfaces::Backend
 
       if exist?(table)
         logger.debug "Table already exist: #{table}"
+        c = TerraspacePluginAws::Interfaces::Config.instance.config
+        tag_existing(table) if c.tag_existing
       else
         logger.info "Creating dynamodb table: #{table}"
         create_table(table)
@@ -36,6 +38,7 @@ class TerraspacePluginAws::Interfaces::Backend
         table_name: name,
       }
       secure(definition)
+      tag(definition)
       definition
     end
 
@@ -64,6 +67,29 @@ class TerraspacePluginAws::Interfaces::Backend
       definition
     end
 
+    def tag(definition)
+      definition[:tags] = tags unless tags.empty?
+    end
+
+    def tag_existing(table_name)
+      return if tags.empty?
+      resp = dynamodb.describe_table(table_name: table_name)
+      # Always appends tags, wont remove tags.
+      dynamodb.tag_resource(
+        resource_arn: resp.table.table_arn,
+        tags: tags
+      )
+    end
+
+    def tags
+      c = TerraspacePluginAws::Interfaces::Config.instance.config
+      tags = !c.dynamodb.tags.empty? ? c.dynamodb.tags : c.tags
+      # Note there is no map! method for Hash
+      tags = tags.map do |k,v|
+        {key: k.to_s, value: v}
+      end
+    end
+
     def exist?(name)
       dynamodb.describe_table(table_name: name)
       true  # table exist

data/lib/terraspace_plugin_aws/interfaces/backend.rb

@@ -6,6 +6,7 @@ module TerraspacePluginAws::Interfaces
     def call
       return unless TerraspacePluginAws.config.auto_create
 
+      Setup.new(@info).check!
       Bucket.new(@info).create
       Table.new(@info).create
     end

data/lib/terraspace_plugin_aws/interfaces/config.rb

@@ -14,19 +14,24 @@ module TerraspacePluginAws::Interfaces
       c = ActiveSupport::OrderedOptions.new
 
       c.auto_create = true
+      c.tags = {} # can set tags for both s3 bucket and dynamodb table with this config
+      c.tag_existing = true
 
       c.s3 = ActiveSupport::OrderedOptions.new
+      c.s3.access_logging = false
+      c.s3.block_public_access = true
       c.s3.encryption = true
       c.s3.enforce_ssl = true
-      c.s3.versioning = true
       c.s3.lifecycle = true
-      c.s3.access_logging = false
+      c.s3.versioning = true
       c.s3.secure_existing = false # run the security controls on existing buckets. by default, only run on newly created bucket the first time
+      c.s3.tags = {} # cannot assign to c.tags here because it's a copy
 
       c.dynamodb = ActiveSupport::OrderedOptions.new
       c.dynamodb.encryption = true
       c.dynamodb.kms_master_key_id = nil
       c.dynamodb.sse_type = "KMS"
+      c.dynamodb.tags = {} # cannot assign to c.tags here because it's a copy
 
       c
     end

data/lib/terraspace_plugin_aws/interfaces/expander.rb

@@ -10,5 +10,9 @@ module TerraspacePluginAws::Interfaces
     def aws_data
       $__aws_data ||= AwsData.new
     end
+
+    def expand_string?(string)
+      !string.include?("arn:")
+    end
   end
 end

data/lib/terraspace_plugin_aws/version.rb

@@ -1,3 +1,3 @@
 module TerraspacePluginAws
-  VERSION = "0.3.1"
+  VERSION = "0.3.5"
 end

data/terraspace_plugin_aws.gemspec

@@ -28,6 +28,6 @@ Gem::Specification.new do |spec|
   spec.add_dependency "aws-sdk-ssm"
   spec.add_dependency "aws_data"
   spec.add_dependency "memoist"
-  spec.add_dependency "s3-secure"
+  spec.add_dependency "s3-secure", "~> 0.6.1"
   spec.add_dependency "zeitwerk"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: terraspace_plugin_aws
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.5
 platform: ruby
 authors:
 - Tung Nguyen
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-12-14 00:00:00.000000000 Z
+date: 2021-12-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-dynamodb
@@ -98,16 +98,16 @@ dependencies:
   name: s3-secure
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.6.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.6.1
 - !ruby/object:Gem::Dependency
   name: zeitwerk
   requirement: !ruby/object:Gem::Requirement
@@ -167,10 +167,13 @@ files:
 - lib/terraspace_plugin_aws.rb
 - lib/terraspace_plugin_aws/autoloader.rb
 - lib/terraspace_plugin_aws/clients.rb
+- lib/terraspace_plugin_aws/clients/options.rb
 - lib/terraspace_plugin_aws/interfaces/backend.rb
 - lib/terraspace_plugin_aws/interfaces/backend/base.rb
 - lib/terraspace_plugin_aws/interfaces/backend/bucket.rb
 - lib/terraspace_plugin_aws/interfaces/backend/bucket/secure.rb
+- lib/terraspace_plugin_aws/interfaces/backend/bucket/tagging.rb
+- lib/terraspace_plugin_aws/interfaces/backend/setup.rb
 - lib/terraspace_plugin_aws/interfaces/backend/table.rb
 - lib/terraspace_plugin_aws/interfaces/config.rb
 - lib/terraspace_plugin_aws/interfaces/decorator.rb