openstax_aws 1.1.0 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5a062d5bb4d0c5c73fd23b6ed4c1d1b36e18d24143e6a74c7a51e4e3c552048d
-  data.tar.gz: a7fcc57860108b12434d66f5e5ccc90e27494bf892a67a1f961aba942398ab8e
+  metadata.gz: 75f65f32cd2e572d1e83f624977186c7aee220c6d51924333fa08af2255543ad
+  data.tar.gz: bf3969a29e8fb4ff94b54bbd9685581cdff28f9c160f3ebe0fa26b92f4875575
 SHA512:
-  metadata.gz: 3be0e6e5c49c30f40e98575d0957fad14e025c6297f7276bcc71b8f42f427b0b349ff47b1c60ca3089a1901be4f9643afe68275aa468bb04e2a1412f426d5b37
-  data.tar.gz: a369f3fb8b8a8f39165358cc24352b0a1b21c6af2fa1baf4bd4fc816877cf3ae8d2fa3e4bff67e49e7930ceb7e42e61942f31ee78c3e00bc92605fa3cb30dee8
+  metadata.gz: 76287d75f76a51cbeb635e6013f5e896174537fcba6352f5febba64c562c3cb857f82aa2a03f71af9583c5db3e8527bf0298e4e2759db9ac658b7a13c0fa6c14
+  data.tar.gz: f86924695c16ce1b3729e9a405e36603761477aa4c7af798ea50bd5faeffcaf18d3caca056db5f8d72127c8bf9c7edfd464d46a7e109f8597d1437ecffb2788f

data/.github/workflows/tests.yml

@@ -0,0 +1,31 @@
+name: Tests
+
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+
+jobs:
+  tests:
+    timeout-minutes: 10
+    runs-on: ubuntu-18.04
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-ruby@v1
+        with:
+          ruby-version: 2.6
+      - uses: actions/cache@v2
+        with:
+          path: vendor/bundle
+          key: ${{ runner.os }}-gems-pr-${{ hashFiles('**/Gemfile.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-gems-pr-
+      - name: Test
+        run: |
+          gem install bundler --force --no-document --version 2.1.4
+          bundle config path vendor/bundle
+          bundle config jobs 2
+          bundle install
+          bundle exec rspec

data/CHANGELOG.md

@@ -6,16 +6,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
-## [1.1.0] - 2020-10-20
-
-Fixed Packer debug mode by reading and printing each character from Packer instead of each line.
+## [1.2.0] - 2021-01-12
 
-Restricted AMI search to images owned by the same account to prevent a potential security flaw.
+- Added additional method to MskCluster resource, to return sorted list of bootstrap brokers
+- Replaced Travis with GH Actions
+- All secret types now default to SecureString, not just the RSA keys
+- Added CloudFormation ERB templates (cfn files that end in .yml.erb)
+- Gracefully handle interrupts while running Packer and return Packer's exit status from the call
 
-Gitignored Gemfile.lock.
+## [1.1.0] - 2020-10-20
 
-Removed development dependency on bundler 1.
+- Fixed Packer debug mode by reading and printing each character from Packer instead of each line.
+- Restricted AMI search to images owned by the same account to prevent a potential security flaw.
+- Gitignored Gemfile.lock.
+- Removed development dependency on bundler 1.
 
 ## [1.0.0] - 2020-10-03
 
-First official version.
+- First official version.

data/README.md

@@ -1,6 +1,6 @@
 # aws-ruby
 
-[![Build Status](https://travis-ci.org/openstax/aws-ruby.svg?branch=master)](https://travis-ci.org/openstax/aws-ruby)
+[![Tests](https://github.com/openstax/aws-ruby/workflows/Tests/badge.svg)](https://github.com/openstax/aws-ruby/actions?query=workflow:Tests)
 
 The `openstax_aws` gem helps you deploy your applications to AWS using CloudFormation.  It provides a layer on top of
 the AWS SDKs to help coordinate common deployment steps and configurations.

data/lib/openstax/aws/deployment_base.rb

@@ -260,6 +260,32 @@ module OpenStax::Aws
       nil # can be overridden by the DSL
     end
 
+    def failed_statuses_table
+      rows = []
+
+      stacks.each do |stack|
+        stack.status(reload: false).failed_events_since_last_user_event.each do |event|
+          rows.push([stack.name, event.status_text, event.status_reason])
+        end if stack.status.failed?
+      end
+
+      column_widths = [
+        2 + rows.reduce(0) { |result, rowdata| [result, rowdata[0].length].max },
+        2 + rows.reduce(0) { |result, rowdata| [result, rowdata[1].length].max },
+        0
+      ]
+
+      output = []
+
+      output.push(["Stack", "Status", "Reason"].each_with_index.map { |header, index| header.ljust(column_widths[index]) }.join(''))
+
+      rows.each { |rowdata|
+        output.push(rowdata.each_with_index.map { |value, index| value.ljust(column_widths[index]) }.join(''))
+      }
+
+      output.join("\n")
+    end
+
     protected
 
     def parameter_default(parameter_name)
@@ -360,13 +386,24 @@ module OpenStax::Aws
       get_image_tag(image_id: image_id, key: "sha")
     end
 
-    protected
-
     def secrets_namespace(id: 'default')
       raise "Override this method in your deployment class and provide a namespace " \
             "for secrets data in the AWS Parameter Store.  The key there will be this namespace " \
             "prefixed by the environment name and suffixed with the secret name."
     end
 
+    def log_and_exit_if_failed_status
+      begin
+        yield
+      rescue
+        if status.failed?
+          logger.fatal("The following errors have occurred: \n#{failed_statuses_table}")
+          exit(1)
+        else
+          raise
+        end
+      end
+    end
+
   end
 end

data/lib/openstax/aws/distribution.rb

@@ -34,7 +34,8 @@ module OpenStax::Aws
       begin
         Aws::CloudFront::Waiters::InvalidationCompleted.new(
           client: client,
-          before_attempt: ->(*) { wait_message.say_it }
+          before_attempt: ->(*) { wait_message.say_it },
+          max_attempts: 60
         ).wait(
           distribution_id: id,
           id: invalidation_id

data/lib/openstax/aws/git_helper.rb

@@ -8,11 +8,20 @@ module OpenStax::Aws
       ::Git.ls_remote("https://github.com/#{org_slash_repo}")["branches"][branch][:sha]
     end
 
-    def self.file_content_at_sha(org_slash_repo:, sha:, path:)
-      location = "https://raw.githubusercontent.com/#{org_slash_repo}/#{sha}/#{path}"
-      file = open(location)
-      file.read
+    def self.file_content_at_sha(org_slash_repo:, sha:, path:, github_token: nil )
+      if github_token.blank?
+        location = "https://raw.githubusercontent.com/#{org_slash_repo}/#{sha}/#{path}"
+        file = URI.open(location)
+        file.read
+      else
+        uri = URI("https://raw.githubusercontent.com/#{org_slash_repo}/#{sha}/#{path}")
+        Net::HTTP.start(uri.host, uri.port, :use_ssl => uri.scheme == 'https') do |http|
+          request = Net::HTTP::Get.new uri.request_uri
+           request.basic_auth 'token', github_token
+           response = http.request request
+           response.body
+        end
+      end
     end
-
   end
 end

data/lib/openstax/aws/msk_cluster.rb

@@ -14,6 +14,10 @@ module OpenStax::Aws
       client.get_bootstrap_brokers(cluster_arn: cluster_arn).bootstrap_broker_string
     end
 
+    def sorted_bootstrap_broker_string
+      bootstrap_broker_string.split(',').sort.join(',')
+    end
+
   end
 end
 

data/lib/openstax/aws/packer_1_2_5.rb

@@ -55,7 +55,7 @@ module OpenStax::Aws
           stdout_err.sync = true
 
           while char = stdout_err.getc do
-            print char
+            STDERR.print char
           end
         end
       end

data/lib/openstax/aws/packer_1_4_1.rb

@@ -41,36 +41,81 @@ module OpenStax::Aws
       cmd = "PACKER_LOG=1 #{cmd}" if @verbose
       cmd = "#{cmd} --debug" if @debug
 
-      cmd = "#{cmd} #{@absolute_file_path}"
+      cmd
     end
 
     def run
       @logger.info("**** DRY RUN ****") if @dry_run
-      @logger.info("Running: #{command}")
+      @logger.info("Running: #{command} #{@absolute_file_path}")
 
       if !@dry_run
         @logger.info("Printing stderr for desired verbosity")
-        ami = ""
 
-        Open3.popen2e(command) do |stdin, stdout_err, wait_thr|
-          stdout_err.sync = true
+        tmpdir = nil
 
-          line = ''
+        begin
+          config_path = @absolute_file_path
 
-          while char = stdout_err.getc do
-            line << char
-            print char
+          # Can't handle modifying HCL2 templates yet
+          if config_path.ends_with?('.json')
+            config = JSON.parse(File.read(config_path))
+            config['post-processors'] ||= []
+            manifest_config = (config['post-processors']).find do |processor|
+              processor['type'] == 'manifest'
+            end
 
-            next unless char == "\n"
+            # Configure a manifest post-processor if not already configured
+            if manifest_config.nil?
+              tmpdir = Dir.mktmpdir
 
-            matchami = line.match(/AMI: (ami-[0-9\-a-z]*)/i)
-            ami = matchami.captures[0] if matchami
+              manifest_config = { 'type' => 'manifest', 'output' => "#{tmpdir}/manifest.json" }
 
-            line = ''
+              config['post-processors'] << manifest_config
+
+              config_path = "#{tmpdir}/packer.json"
+
+              File.write(config_path, JSON.dump(config))
+            end
           end
-        end
 
-        puts ami
+          Open3.popen2e("#{command} #{config_path}") do |stdin, stdout_err, wait_thr|
+            begin
+              previous_interrupt_handler = Signal.trap 'INT' do
+                # Interrupt Packer
+                Process.kill 'INT', wait_thr.pid
+
+                # Restore previous interrupt handler so we don't interrupt Packer again
+                Signal.trap 'INT', previous_interrupt_handler
+
+                # Disable other code that restores previous interrupt
+                previous_interrupt_handler = nil
+              end
+
+              stdout_err.sync = true
+
+              # Send all packer output to STDERR
+              while char = stdout_err.getc do
+                STDERR.print char
+              end
+            ensure
+              # Restore previous interrupt unless we did so already
+              Signal.trap 'INT', previous_interrupt_handler unless previous_interrupt_handler.nil?
+            end
+
+            # Read the AMI ID from the manifest file and output it to STDOUT
+            unless manifest_config.nil?
+              manifest = File.read(manifest_config['output']) rescue nil
+
+              puts JSON.parse(manifest)['builds'].last['artifact_id'].split(':', 2).last \
+                unless manifest.nil?
+            end
+
+            # Return Packer's exit status wrapped in a Process::Status object
+            wait_thr.value
+          end
+        ensure
+          FileUtils.remove_entry(tmpdir) unless tmpdir.nil?
+        end
       end
     end
 

data/lib/openstax/aws/sam_stack.rb

@@ -32,9 +32,12 @@ module OpenStax::Aws
             dry_run: dry_run)
     end
 
-    def build
+    def build(debug: false, env_vars: {})
       # SAM doesn't have an API or SDK - we have to make calls to its CLI
+      env_vars_prefix = env_vars.map { |variable, value| "#{variable}=#{value}" }.join(' ')
       command = "sam build -t #{absolute_template_path} -b #{build_directory}"
+      command += ' --debug' if debug
+      command = "#{env_vars_prefix} #{command}" if env_vars_prefix.present?
       System.call(command, logger: logger, dry_run: dry_run)
     end
 
@@ -50,7 +53,8 @@ module OpenStax::Aws
                 " --capabilities CAPABILITY_IAM" \
                 " --s3-bucket #{bucket_name}" \
                 " --s3-prefix #{name}" \
-                " --stack-name #{name}"
+                " --stack-name #{name}" \
+                " --region #{region}"
 
       if params.any?
         command += " --parameter-overrides #{self.class.format_hash_as_cli_stack_parameters(params)}"

data/lib/openstax/aws/secrets.rb

@@ -146,7 +146,7 @@ module OpenStax::Aws
 
     def process_individual_spec_value(spec_value, substitutions)
       generated = false
-      type = "String"
+      type = "SecureString"
 
       value = case spec_value
       when /^random\(hex,(\d+)\)$/
@@ -158,7 +158,6 @@ module OpenStax::Aws
         num_characters = $1.to_i
         SecureRandom.urlsafe_base64(num_characters)[0..num_characters-1]
       when /^rsa\((\d+)\)$/
-        type = "SecureString"
         generated = true
         key_length = $1.to_i
         OpenSSL::PKey::RSA.new(key_length).to_s

data/lib/openstax/aws/secrets_factory.rb

@@ -39,6 +39,7 @@ module OpenStax::Aws
             org_slash_repo: attributes[:org_slash_repo],
             sha: attributes[:sha],
             path: attributes[:path],
+            github_token: attributes[:github_token],
             format: attributes[:format].to_sym,
             top_key: attributes[:top_key].to_sym,
             preparser: attributes[:preparser]

data/lib/openstax/aws/secrets_specification.rb

@@ -13,11 +13,12 @@ module OpenStax::Aws
       new(content: content, format: format, top_key: top_key, preparser: preparser)
     end
 
-    def self.from_git(org_slash_repo:, sha:, path:, format:, top_key: nil, preparser: nil)
+    def self.from_git(org_slash_repo:, sha:, path:, github_token: nil, format:, top_key: nil, preparser: nil)
       content = OpenStax::Aws::GitHelper.file_content_at_sha(
                   org_slash_repo: org_slash_repo,
                   sha: sha,
-                  path: path
+                  path: path,
+                  github_token: github_token
                 )
       new(content: content, format: format, top_key: top_key, preparser: preparser)
     end

data/lib/openstax/aws/stack_factory.rb

@@ -52,9 +52,13 @@ module OpenStax::Aws
         path = File.join(base_directory, "#{@id}.yml")
 
         if !File.file?(path)
-          path = File.join(base_directory, "#{@id}.json")
+          path = File.join(base_directory, "#{@id}.yml.erb")
+
           if !File.file?(path)
-            raise "Couldn't infer an existing template file for stack #{@id}"
+            path = File.join(base_directory, "#{@id}.json")
+            if !File.file?(path)
+              raise "Couldn't infer an existing template file for stack #{@id}"
+            end
           end
         end
       end

data/lib/openstax/aws/template.rb

@@ -26,8 +26,20 @@ module OpenStax::Aws
       File.basename(absolute_file_path)
     end
 
+    def extname
+      File.extname(absolute_file_path)
+    end
+
+    def erb?
+      extname == '.erb'
+    end
+
     def body
-      @body ||= File.read(absolute_file_path)
+      return @body unless @body.nil?
+
+      @body = File.read(absolute_file_path)
+      @body = ERB.new(@body).tap { |erb| erb.filename = absolute_file_path }.result if erb?
+      @body
     end
 
     def hash
@@ -88,7 +100,7 @@ module OpenStax::Aws
     end
 
     def is_sam?
-      body.match(/Transform: AWS::Serverless/).present?
+      body.match(/Transform: .?AWS::Serverless/).present?
     end
 
   protected

data/lib/openstax/aws/version.rb

@@ -1,5 +1,5 @@
 module OpenStax
   module Aws
-    VERSION = "1.1.0"
+    VERSION = "1.4.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: openstax_aws
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.4.0
 platform: ruby
 authors:
 - JP Slavinsky
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-10-20 00:00:00.000000000 Z
+date: 2022-05-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-autoscaling
@@ -273,8 +273,8 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/tests.yml"
 - ".gitignore"
-- ".travis.yml"
 - CHANGELOG.md
 - Gemfile
 - LICENSE.txt
@@ -327,7 +327,7 @@ licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -343,7 +343,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.1.4
-signing_key: 
+signing_key:
 specification_version: 4
 summary: openstax IaC
 test_files: []

data/.travis.yml

@@ -1,12 +0,0 @@
-sudo: false
-language: ruby
-rvm:
-  - 2.6
-cache: bundler
-bundler_args: --retry=6
-script:
-  - bundle exec rake
-notifications:
-  email: false
-before_install:
-  - gem install bundler