iapi-idlc-sdk-deploy 1.0.2 → 1.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 51c3af5405162472763a357065de1cda57296675681129a2bcd683cdca894459
-  data.tar.gz: fbd1e05b52899cd422058078384c33fdc16ec5e523a8b9a0d19b41fadb6fc32c
+  metadata.gz: 8534c9fbb72035cb5311fc8774816a260cffe31dd1efa6ca1ba757a687fb76db
+  data.tar.gz: 3589081a82af030b2f89bed835275b4643be577be976800abad822ac44349111
 SHA512:
-  metadata.gz: e913a47ae3da654c91ebda072ebedc034c13e1c4dd0a9e101a86c420998a8e4fb0492d1c84667fc05fc57cac97ec8b2032b215947df21a00225b02b6ab753a6d
-  data.tar.gz: 56e6dbbf13116fcd7e1d0013a7f1e31a9ac21cb69e2641c6041e07b77d2ec279d03d41d33044b507695cdfd58e9973fff77ac4aa07667fbaa7fc478570049c36
+  metadata.gz: 9aaf22738ec4d0ac9e4acaa6a3a07cc804bfeca9f4b2a9d336c9a34e0010171aefe33c39c85813e6b1d56a37a46bb88a623c1a811b15b28b1fafa9a9134a49b0
+  data.tar.gz: 24cf040333351675685f7f2d9a38df556768fc7e49bbc7d1abf7dee9aed358d0c82c90bbb07436b8f43fb82091ed1f38e4cb08328b7b7ede1ab93cca1e18e143

data/.gitignore

@@ -0,0 +1,13 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/vendor
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.4.1
+before_install: gem install bundler -v 1.15.4

data/.yardopts

@@ -0,0 +1 @@
+-m markdown

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in idlc-sdk-deploy.gemspec
+gemspec

data/Jenkinsfile

@@ -0,0 +1,21 @@
+pipeline {
+  agent any
+  options {
+    buildDiscarder logRotator(daysToKeepStr: '5', numToKeepStr: '10')
+  }
+  stages {
+    stage('Ruby Gem Push') {
+      environment {
+        GEM_HOST_API_KEY = credentials('rubygems-apikey')
+      }
+      steps {
+        sh '''
+          DOCKER_CMD="docker run --rm -v $PWD:/opt -w /opt -e GEM_HOST_API_KEY -e BUILD_NUMBER -u 999 ruby:3.0"
+          $DOCKER_CMD gem build iapi-idlc-sdk-deploy.gemspec -o iapi-idlc-sdk-deploy.gem
+          $DOCKER_CMD gem push iapi-idlc-sdk-deploy.gem
+        '''
+      }
+    }
+  }
+}
+

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Nathan Cazell
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,35 @@
+# Idlc::Sdk::Deploy
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/idlc/sdk/deploy`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'iapi-idlc-sdk-deploy'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install iapi-idlc-sdk-deploy
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "idlc-sdk-deploy"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/iapi-idlc-sdk-deploy.gemspec

@@ -0,0 +1,43 @@
+# coding: utf-8
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'iapi-idlc-sdk-deploy/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'iapi-idlc-sdk-deploy'
+  spec.version       = Idlc::Deploy::VERSION
+  spec.authors       = ["Nathan Cazell", "Michael McHolm"]
+  spec.email         = ["techsupport@imageapi.com"]
+
+  spec.summary       = 'IDLC SDK for AWS resources - Deploy'
+  spec.description   = 'Provides deploy libraries for idlc-sdk. This gem is part of the IDLC SDK'
+  spec.homepage      = 'https://bitbucket.org/imageapi/iapi-idlc-sdk-deploy'
+  spec.license       = 'MIT'
+
+  spec.metadata = {
+    'source_code_uri' => 'https://bitbucket.org/imageapi/iapi-idlc-sdk-deploy'
+  }
+
+  spec.files = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 2'
+  spec.add_development_dependency 'rake', '>= 12.3.3'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'rubocop', '0.48.1'
+  spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'yard'
+
+  spec.add_runtime_dependency 'iapi-idlc-sdk-core'
+  spec.add_runtime_dependency 'aws-sdk-elasticsearchservice'
+  spec.add_runtime_dependency 'aws-sdk-ec2'
+  spec.add_runtime_dependency 'net-telnet'
+  spec.add_runtime_dependency 'sshkey', '2.0.0'
+  spec.add_runtime_dependency 'terraform-binary'
+  spec.add_runtime_dependency 'commander', '4.4.6'
+  spec.add_runtime_dependency 'terraform_landscape', '0.2.1'
+end

data/lib/iapi-idlc-sdk-deploy/config.rb

@@ -0,0 +1,218 @@
+module Idlc
+  module Deploy
+    class Config
+      include Idlc::Helpers
+
+      class << self
+        def load_tasks
+          Dir.glob("#{__dir__}/tasks/*.rake").each do |task_file|
+            load task_file
+          end
+        end
+
+        def add_deployment_var(key, value)
+          ENV["TF_VAR_#{key}"] = value
+        end
+
+        def get_deployment_var(key)
+          ENV["TF_VAR_#{key}"]
+        end
+
+        def get_deployment_output(key)
+          `#{Terraform::Binary::Command.binary} output #{key}`.strip!
+        end
+
+        def whoami
+          # This method is meant to be run on an instance inside of a chef run to
+          # provision instance and environment metadata.
+
+          ENV['AWS_REGION'] = get_region
+
+          # Get the current instance id from the instance metadata.
+          instance = get_instance
+
+          # return environment metadata
+          metadata = get_env_metadata(instance['tags']['environment_key'])
+          metadata['hostname'] = set_hostname(instance)
+
+          metadata
+        end
+
+        def get_env_metadata(env_key)
+          client = Idlc::AWSLambdaProxy.new()
+
+          request = {
+            service: 'deploy',
+            method: 'GET',
+            lambda: 'metadata',
+            pathParameters: {
+              jobName: env_key
+            }
+          }
+          metadata = client.fetch(request)['deployments'].first
+
+          request = {
+            service: 'config',
+            method: 'GET',
+            lambda: "accounts",
+            pathParameters: {
+              accountName: metadata['environment']['account_alias']
+            }
+          }
+          account = client.fetch(request)
+
+          metadata['account'] = account['accounts'].first
+
+          request = {
+            service: 'config',
+            method: 'GET',
+            lambda: "applications",
+            pathParameters: {
+              appName: metadata['environment']['application_name'].downcase
+            }
+          }
+          application = client.fetch(request)
+
+          metadata['application'] = application['applications'].first
+
+          # find db and fs instance
+          metadata['instances'].each do |instance|
+            if (instance['hostname'].start_with?('db') || instance['hostname'].start_with?('rds'))
+              metadata['db_instance'] = instance
+            end
+
+            if (instance['hostname'].start_with?('fs'))
+              metadata['fs_instance'] = instance
+            end
+          end
+
+          metadata
+        end
+
+        def get_region
+          # Get the current az from the instance metadata.
+          metadata_endpoint = 'http://169.254.169.254/latest/meta-data/'
+          az = Net::HTTP.get( URI.parse( metadata_endpoint + 'placement/availability-zone' ) )
+
+          # return
+          az[0..-2]
+        end
+
+        def get_instance
+          # Get the current instance id from the instance metadata.
+          metadata_endpoint = 'http://169.254.169.254/latest/meta-data/'
+          instance_id = Net::HTTP.get( URI.parse( metadata_endpoint + 'instance-id' ) )
+
+          # Create instance object with instance id.
+          instance = Aws::EC2::Instance.new( id: instance_id, region: ENV['AWS_REGION'] )
+
+          # save some instance info
+          i = {}
+          i['instance_id'] = instance_id
+
+          # save tags
+          i['tags'] = {}
+          instance.tags.each do |tag|
+            # Grab all of the tags as node attributes
+            i['tags'][tag.key] = tag.value
+          end
+
+          i
+        end
+
+        def set_hostname (instance)
+          hostname = instance['tags']['Name']
+
+          unless (instance['tags']['Name'].start_with?('db') || instance['tags']['Name'].start_with?('fs'))
+            # Use instance id for unique hostname
+            hostname = instance['tags']['Name'][0..4] + '-' + instance['instance_id'][2..10]
+          end
+
+          ec2_instance = Aws::EC2::Instance.new( id: instance['instance_id'], region: ENV['AWS_REGION'] )
+          ec2_instance.create_tags(
+            dry_run: false,
+            tags: [ # required
+              {
+                key: 'hostname',
+                value: hostname
+              }
+            ]
+          )
+
+          #return
+          hostname
+        end
+      end
+
+      def initialize(region)
+        @region = region
+
+        Idlc::Utility.check_for_creds
+
+      rescue Idlc::Utility::MissingCredentials => e
+        msg("WARN: #{e.message}\nFalling back to implicit authentication.")
+      end
+
+      def configure_state(bucket, sub_bucket, working_directory)
+        validate_environment
+
+        tf_version = Terraform::Binary::config.version.split('.')
+
+        configure_tfstatev8(bucket, sub_bucket, working_directory) if tf_version[0].to_i == 0 && tf_version[1].to_i <= 8
+        configure_tfstatev9(bucket, sub_bucket, working_directory) if tf_version[0].to_i >= 0 && tf_version[1].to_i > 8
+      end
+
+      def parse(config_file)
+        raise ArgumentError, "#{config_file} does not exist" unless File.exist? config_file
+        Config.add_deployment_var('inf_config_file', config_file)
+
+        # Parse the config file
+        YAML.load_file(config_file)['configuration'].each do |section, body|
+          next if section == 'dynamics' # skip the dynamics sections
+          next unless (section =~ /overrides/).nil? # skip the app overrides sections
+          next if body.nil?
+          body.each do |key, value|
+            debug("#{section}: #{key} = #{value}")
+            Config.add_deployment_var(key, value)
+          end
+        end
+      end
+
+      private
+
+      def configure_tfstatev8(bucket, sub_bucket, working_directory)
+        args = []
+        args << '-backend=s3'
+        args << '-backend-config="acl=private"'
+        args << "-backend-config=\"bucket=#{bucket}\""
+        args << '-backend-config="encrypt=true"'
+        args << "-backend-config=\"key=#{sub_bucket}/terraform.tfstate\""
+        args << "-backend-config=\"region=#{@region}\""
+
+        Terraform::Binary.remote("config #{args.join(' ')}")
+        Terraform::Binary.get("-update #{working_directory}")
+      end
+
+      def configure_tfstatev9(bucket, sub_bucket, working_directory)
+        args = []
+        args << "-backend-config=\"bucket=#{bucket}\""
+        args << "-backend-config=\"key=#{sub_bucket}/terraform.tfstate\""
+        args << "-backend-config=\"region=#{@region}\""
+        args << "-force-copy"
+
+        Terraform::Binary.init("#{args.join(' ')} #{working_directory}")
+      end
+
+      def validate_environment
+        %w[
+          TF_VAR_tfstate_bucket
+          TF_VAR_job_code
+          TF_VAR_env
+          TF_VAR_domain
+        ].each do |var|
+          raise "missing #{var} in environment" unless ENV.include? var
+        end
+      end
+    end
+  end
+end

data/lib/iapi-idlc-sdk-deploy/keypair.rb

@@ -0,0 +1,25 @@
+module Idlc
+  module Deploy
+    module Keypair
+      class << self
+        def generate(outdir)
+          raise ArgumentError, 'Must specify output directory' if outdir.nil?
+
+          FileUtils.mkdir_p outdir unless File.directory? outdir
+
+          private_key_file = "#{outdir}/private_key.pem"
+          public_key_file = "#{outdir}/public_key.pem"
+
+          return if File.exist? private_key_file
+
+          rsa_key = SSHKey.generate
+          private_key = rsa_key.private_key
+          public_key = rsa_key.ssh_public_key
+
+          File.open(private_key_file, 'w') { |file| file.write(private_key) }
+          File.open(public_key_file, 'w') { |file| file.write(public_key) }
+        end
+      end
+    end
+  end
+end

data/lib/iapi-idlc-sdk-deploy/power.rb

@@ -0,0 +1,198 @@
+module Idlc
+  module Deploy
+    module Power
+      class ConnectionError < StandardError; end
+      class InstanceKeepAlive < StandardError; end
+
+      class << self
+        include Idlc::Helpers
+
+        def start_instance(instance, async=false)
+          msg("Starting Instance (#{instance.id})...")
+          instance.start(
+            dry_run: false
+          )
+          unless async
+            obj = instance.wait_until_running
+            msg('Started Instance: ' + get_name(obj.tags))
+          end
+        end
+
+        def stop_instance(instance, async=false)
+          raise InstanceKeepAlive if keep_alive?(instance.tags)
+
+          msg("Stopping Instance (#{instance.id})...")
+          instance.stop(
+            dry_run: false
+          )
+          unless async
+            obj = instance.wait_until_stopped
+            msg('Stopped Instance: ' + get_name(obj.tags))
+          end
+        end
+
+        def enable_keep_alive(instance)
+          instance.create_tags(
+            dry_run: false,
+            tags: [ # required
+              {
+                key: 'keep_alive',
+                value: "true-#{Time.now.to_i}"
+              }
+            ]
+          )
+        end
+
+        def disable_keep_alive(instance)
+          instance.create_tags(
+            dry_run: false,
+            tags: [ # required
+              {
+                key: 'keep_alive',
+                value: 'false'
+              }
+            ]
+          )
+        end
+
+        def update_instance_type(instance, type)
+          unless instance.instance_type == type
+            name = get_name(instance.tags)
+            msg "Changing #{name}: #{instance.instance_type} => #{type}"
+
+            instance.modify_attribute(
+              dry_run: false,
+              attribute: 'instanceType',
+              value: type
+            )
+          end
+        end
+
+        def wait_for_tcp_connection(host, port, connection_timeout = 5, wait_timeout = 1500, sleep_time = 10)
+          connected = false
+          start_time = Time.now
+
+          until connected
+            begin
+              Net::Telnet.new(
+                'Host' => host,
+                'Port' => port,
+                'Telnetmode' => false,
+                'Timeout' => connection_timeout
+              )
+
+              connected = true
+            rescue ConnectionError, Net::OpenTimeout, Errno::ECONNREFUSED
+              check_timeout(start_time, wait_timeout)
+              debug("waiting for #{host}:#{port} ... (#{(Time.now - start_time)}s elapsed)")
+              sleep sleep_time
+            end
+          end
+
+          msg "recieved response from #{host}:#{port} !"
+          true
+        end
+
+        def wait_for_response(endpoint, success_text = nil, wait_timeout = 1500, sleep_time = 10)
+          connected = false
+          start_time = Time.now
+
+          until connected
+            begin
+              if success_text.nil? || success_text == ''
+                response = http_request("#{endpoint}/diagnostics/ping")
+                connected = success(response.body)
+              else
+                response = http_request(endpoint)
+                connected = simple_success(response.body, success_text)
+              end
+            rescue ConnectionError, Net::OpenTimeout, JSON::ParserError, Errno::ECONNREFUSED
+              check_timeout(start_time, wait_timeout)
+              sleep sleep_time
+            end
+          end
+
+          msg "recieved response from #{endpoint} !"
+          true
+        end
+
+        def get_name(tags)
+          name = ''
+
+          tags.each do |t|
+            name = t.value if t.key == 'Name'
+          end
+
+          # Return
+          name
+        end
+
+        private
+
+        def keep_alive?(tags)
+          k = 'false'
+          k = get_keep_alive(tags)
+
+        return !keep_alive_expired?(k) if true?(k.split('-')[0])
+        false
+        end
+
+        def keep_alive_expired?(tag)
+          tag_parts = tag.split('-')
+
+          msg("keep_alive expired..") if one_week_old?(tag_parts[1].to_i)
+          msg("keep_alive expires in #{((Time.now.to_i - (tag_parts[1].to_i + (7*24*3600)))/24/3600).abs} days..") unless one_week_old?(tag_parts[1].to_i)
+
+          one_week_old?(tag_parts[1].to_i)
+        end
+
+        def true?(string)
+          string.to_s == 'true'
+        end
+
+        def one_week_old?(ts)
+          (Time.now.to_i - ts) > (7*(24*3600))
+        end
+
+        def get_keep_alive(tags)
+          v = ''
+
+          tags.each do |t|
+            v = t.value if t.key == 'keep_alive'
+          end
+
+          # Return
+          v
+        end
+
+        def http_request(endpoint)
+          uri = URI.parse endpoint
+          http = Net::HTTP.new(uri.host, uri.port)
+          http.use_ssl = true
+          http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+
+          request = Net::HTTP::Get.new(uri.path)
+          http.request(request)
+        end
+
+        def success(body)
+          pong = JSON.parse(body)['data']['pong']
+          raise ConnectionError unless pong == true
+
+          true
+        end
+
+        def simple_success(body, search_text)
+          raise ConnectionError if (body =~ /#{search_text}/).nil?
+
+          true
+        end
+
+        def check_timeout(start_time, timeout)
+          elapsed_time = Time.now - start_time
+          raise ConnectionError, 'Exceeded Timeout for Completion..' if elapsed_time.to_i >= timeout
+        end
+      end
+    end
+  end
+end

data/lib/iapi-idlc-sdk-deploy/tasks/setupv3.rake

@@ -0,0 +1,7 @@
+require 'json'
+
+# desc 'setup environment'
+task :setupv3 do
+  raise 'missing metadata file' unless File.exist? 'metadata.json'
+  ENV_METADATA = JSON.parse(open('metadata.json').read)['deployments'].first if File.exist? 'metadata.json'
+end

data/lib/iapi-idlc-sdk-deploy/version.rb

@@ -0,0 +1,6 @@
+module Idlc
+  module Deploy
+    build_number = ENV.fetch('BUILD_NUMBER', '0')
+    VERSION = "1.0.#{build_number}"
+  end
+end

data/lib/iapi-idlc-sdk-deploy.rb

@@ -0,0 +1,24 @@
+require 'aws-sdk-elasticsearchservice'
+require 'aws-sdk-ec2'
+require 'aws-sigv4'
+require 'net/https'
+require 'net/http'
+require 'net-telnet'
+require 'rspec/core/rake_task'
+require 'sshkey'
+require 'terraform/binary'
+require 'tmpdir'
+
+# Use the packer-binary gem to provide the executable
+Terraform::Binary.configure do |config|
+  config.version = '0.8.7'
+end
+
+# Load the core gem, this also has the service deinitions defined in the $services global variable
+require 'iapi-idlc-sdk-core'
+
+require 'iapi-idlc-sdk-deploy/config'
+require 'iapi-idlc-sdk-deploy/power'
+require 'iapi-idlc-sdk-deploy/keypair'
+
+Idlc::Deploy::Config.load_tasks

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: iapi-idlc-sdk-deploy
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.0.3
 platform: ruby
 authors:
 - Nathan Cazell
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-11-07 00:00:00.000000000 Z
+date: 2023-11-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -214,7 +214,25 @@ email:
 executables: []
 extensions: []
 extra_rdoc_files: []
-files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- ".yardopts"
+- Gemfile
+- Jenkinsfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- iapi-idlc-sdk-deploy.gemspec
+- lib/iapi-idlc-sdk-deploy.rb
+- lib/iapi-idlc-sdk-deploy/config.rb
+- lib/iapi-idlc-sdk-deploy/keypair.rb
+- lib/iapi-idlc-sdk-deploy/power.rb
+- lib/iapi-idlc-sdk-deploy/tasks/setupv3.rake
+- lib/iapi-idlc-sdk-deploy/version.rb
 homepage: https://bitbucket.org/imageapi/iapi-idlc-sdk-deploy
 licenses:
 - MIT