idlc-sdk-pfm 1.0.0.pre.2

data/lib/idlc-sdk-pfm/command/base.rb

@@ -0,0 +1,195 @@
+require 'mixlib/cli'
+require 'idlc-sdk-pfm/helpers'
+
+module Pfm
+  module Command
+    class Base
+      include Mixlib::CLI
+      include Pfm::Helpers
+
+      class InvalidRepository < StandardError; end
+      class BuildFailure < StandardError; end
+      class DeploymentFailure < StandardError; end
+
+      option :help,
+             short: '-h',
+             long: '--help',
+             description: 'Show this message',
+             boolean: true
+
+      option :version,
+             short: '-v',
+             long: '--version',
+             description: 'Show pfm version',
+             boolean: true
+
+      option :verbose,
+             short:        '-V',
+             long:         '--verbose',
+             description:  'Show detailed output',
+             boolean:      true,
+             default:      false
+
+      def initialize
+        super
+
+        @workspace = Idlc::Workspace.new
+      end
+
+      #
+      # optparser overwrites -h / --help options with its own.
+      # In order to control this behavior, make sure the default options are
+      # handled here.
+      #
+      def run_with_default_options(params = [])
+        if needs_help?(params)
+          msg(opt_parser.to_s)
+          0
+        elsif needs_version?(params)
+          msg("Pfm Version: #{Pfm::VERSION}")
+          0
+        else
+          ENV['DEBUG'] = 'true' if verbose?(params)
+          run(params)
+        end
+      rescue NameError => e
+        err("ERROR: #{e.message}\n")
+        1
+      rescue OptionParser::InvalidOption, OptionParser::MissingArgument => e
+        err("ERROR: #{e.message}\n")
+        msg(opt_parser)
+        1
+      rescue Idlc::Build::MissingMetadataFile, Idlc::Build::MissingRequiredMetadataAttribute, InvalidRepository => e
+        err("ERROR: #{e.message}\n")
+        1
+      ensure
+        @workspace.cleanup unless @workspace.empty?
+      end
+
+      def build_setup
+        Packer::Binary.configure do |config|
+          config.version = SETTINGS['PACKER_VERSION']
+        end
+
+        @build_config = Idlc::Build::Config.new(SETTINGS['AWS_REGION'])
+        @workspace.add(build_base_dir)
+
+        build_dir = "#{@workspace.tmp_dir}/#{build_base_dir}".freeze
+
+        msg("Using build template: Build::#{@params.first}::#{@config[:build_template]}::#{@config[:build_metadata]}")
+
+        # Include build metadata
+        @build_metadata = Idlc::Build::Metadata.new(@params.first, @config[:build_metadata])
+
+        # load the rest of the metadata
+        @build_metadata.load
+
+        # load version from command line if specified
+        @build_metadata.attributes['version'] = Idlc::Build::Metadata::MetadataAttribute.new(@config[:build_version], true) unless @config[:build_version].nil?
+
+        # check metadata requirements
+        @build_metadata.requirements_satisfied?
+
+        msg("Template Version: #{@build_metadata.attributes['version'].value}")
+
+        @build_metadata.attributes.each do |key, att|
+          # load metadata file as packer user vars
+          @build_config.add_build_var_v2(key, att.value)
+        end
+
+        # Copy over the base template and auxillary files for Packer
+        tpl = Idlc::Build::Template.new(
+          @build_metadata.attributes,
+          "#{build_dir}/build.json"
+        )
+        tpl.write
+
+        # copy auxiliary files
+        system("cp -a #{templates_dir}/files #{build_dir}")
+
+        Dir.chdir(build_dir)
+      end
+
+      def deploy_setup
+        Terraform::Binary.configure do |config|
+          config.version = SETTINGS['TERRAFORM_VERSION']
+        end
+
+        raise InvalidRepository, 'This doesn\'t look like a valid infrastructure repository' unless File.directory? "#{inf_base_dir}/tf"
+        config = Idlc::Deploy::Config.new(SETTINGS['AWS_REGION'])
+
+        config.parse("#{inf_base_dir}/env/config/default.yml") if File.exist? "#{inf_base_dir}/env/config/default.yml"
+
+        if ENV['PROD'] == 'true' || ENV['PROD'] == '1'
+          config.parse("#{inf_base_dir}/env/config/prod.yml")
+        else
+          config.parse("#{inf_base_dir}/env/config/devtest.yml")
+        end
+
+        config.parse("#{inf_base_dir}/env/size/#{ENV['SIZE']}.yml") if File.exist? "#{inf_base_dir}/env/size/#{ENV['SIZE']}.yml"
+
+        inf_conf_file = 'inf.config.yml'
+
+        # For unit tests
+        inf_conf_file = 'inf.config.example.yml' unless File.exist? inf_conf_file
+        config.parse(inf_conf_file)
+
+        bucket_name = Idlc::Deploy::Config.get_deployment_var('tfstate_bucket')
+        sub_bucket = "#{Idlc::Deploy::Config.get_deployment_var('job_code')}"\
+          "#{Idlc::Deploy::Config.get_deployment_var('job')}"\
+          "-#{Idlc::Deploy::Config.get_deployment_var('env')}".freeze
+
+        # Pass some ENV vars for Terraform
+        Idlc::Deploy::Config.add_deployment_var('environment_key', sub_bucket)
+        Idlc::Deploy::Config.add_deployment_var('version', REPO_VERSION)
+        Idlc::Deploy::Config.add_deployment_var('major_minor', Idlc::Utility.major_minor(REPO_VERSION))
+        Idlc::Deploy::Config.add_deployment_var('major_minor_patch', Idlc::Utility.major_minor_patch(REPO_VERSION))
+        Idlc::Deploy::Config.add_deployment_var('build', @config[:server_build])
+        Idlc::Deploy::Config.add_deployment_var('app_release', @config[:app_release])
+
+        config.configure_state(bucket_name, sub_bucket)
+
+        Idlc::Deploy::Keypair.generate("#{inf_base_dir}/env/kp")
+        @workspace.flatten("#{inf_base_dir}/tf", 'tf')
+        @workspace.add("#{inf_base_dir}/env/kp")
+        @workspace.add('lib/tf/modules')
+      end
+
+      def templates_dir
+        "#{__dir__}/templates/#{@build_metadata.attributes['build_stage'].value}"
+      end
+
+      def build_base_dir
+        "#{SETTINGS['BUILD_BASE_DIR']}/#{@params.first}"
+      end
+
+      def inf_base_dir
+        SETTINGS['INF_BASE_DIR']
+      end
+
+      def build_dir
+        "#{@workspace.tmp_dir}/#{build_base_dir}".freeze
+      end
+
+      def build_exists?
+        return true if Dir.exist? build_base_dir
+
+        # doesn't exist
+        @errors.push("Build::#{@params.first} doesnt exist")
+        false
+      end
+
+      def needs_help?(params)
+        params.include?('-h') || params.include?('--help')
+      end
+
+      def needs_version?(params)
+        params.include?('-v') || params.include?('--version')
+      end
+
+      def verbose?(params)
+        params.include?('-V') || params.include?('--verbose')
+      end
+    end
+  end
+end

data/lib/idlc-sdk-pfm/command/build.rb

@@ -0,0 +1,120 @@
+require 'idlc-sdk-pfm/command/base'
+require 'mixlib/shellout'
+
+module Pfm
+  module Command
+    class Build < Base
+      banner 'Usage: pfm build BUILD_NAME [options]'
+
+      option :app_release,
+             short:        '-a VERSION',
+             long:         '--app-release VERSION',
+             description:  'Application Version Number to build',
+             default:      ''
+
+      option :build_version,
+             short:        '-b VERSION',
+             long:         '--build-version VERSION',
+             description:  'override version number of build',
+             default:      nil
+
+      option :build_template,
+             short:        '-t TEMPLATE',
+             long:         '--build-template TEMPLATE',
+             description:  'The Build Template to use. Defaults to build.json',
+             default:      'build.json'
+
+      option :build_metadata,
+             short:        '-m METADATA_FILE',
+             long:         '--build-metadata METADATA_FILE',
+             description:  'The build metadata file to use. Defaults to \'metadata\'',
+             default:      'metadata'
+
+      option :build_number,
+             short:        '-n NUMBER',
+             long:         '--build-number NUMBER',
+             description:  'Override the build number. Default is ENV::BUILD_NUMBER',
+             default:      ENV['BUILD_NUMBER']
+
+      def initialize
+        super
+        @params_valid = true
+        @errors = []
+      end
+
+      def run(params)
+        @params = params
+        read_and_validate_params
+
+        if params_valid?
+          build_setup
+          build
+          # @workspace.cleanup causing bundler issues
+          0
+        else
+          @errors.each { |error| err("Error: #{error}") }
+          parse_options(params)
+          msg(opt_parser)
+          1
+        end
+      rescue BuildFailure => e
+        err("ERROR: #{e.message}\n")
+        1
+      end
+
+      def build
+        # Zip the cookbooks for transfer
+        Idlc::Workspace.zip_folder('./chef', "#{@workspace.tmp_dir}/cookbooks.zip")
+
+        # Start the HTTP server to facilitate file transfers. This is to replace
+        # WINRM for file transfers in Packer due to slowness. This function returns
+        # the process id of the server instance.
+        pid = Idlc::Build::Httpd.start(@workspace.tmp_dir)
+
+        # #start will return twice when forking the HTTP server off, once for the
+        # parent and once for the child. When the child return, the pid is nil. We want
+        # to skip that run.
+        unless pid.nil?
+          # Pass some ENV vars for Packer
+          @build_config.add_build_var_v2('aws_region', SETTINGS['AWS_REGION'])
+          @build_config.add_build_var_v2('app_release', @config[:app_release])
+          @build_config.add_build_var_v2('build_uuid', SecureRandom.uuid.to_s)
+          @build_config.add_build_var_v2('build_number', @config[:build_number])
+          @build_config.add_build_var_v2('httpd_server', Idlc::Build::Httpd.private_ip.to_s)
+          @build_config.add_build_var_v2('httpd_port', ENV['HTTPD_PORT'])
+
+          begin
+            Packer::Binary.build("#{@build_config.dump_build_vars} #{@config[:build_template]}")
+            Idlc::Build::Httpd.stop(pid)
+            Dir.chdir(build_base_dir)
+          rescue
+            Idlc::Build::Httpd.stop(pid)
+            Dir.chdir(build_base_dir)
+            raise BuildFailure, 'The build finished with errors'
+          end
+
+        end
+
+        Dir.chdir(build_base_dir)
+      end
+
+      def read_and_validate_params
+        arguments = parse_options(@params)
+
+        case arguments.size
+        when 1
+          @params_valid = build_exists?
+
+        when 2
+
+        else
+          @params_valid = false
+        end
+      end
+
+      def params_valid?
+        @params_valid
+      end
+    end
+  end
+end

data/lib/idlc-sdk-pfm/command/configure.rb

@@ -0,0 +1,41 @@
+require 'idlc-sdk-pfm/command/base'
+require 'mixlib/shellout'
+
+module Pfm
+  module Command
+    class Configure < Pfm::Command::Base
+      banner 'Usage: pfm configure SYSTEM_COMMAND'
+
+      def run(params)
+        current_set = Pfm::Settings.new
+        new_settings = {}
+
+        current_set.settings.each do |key, setting|
+          print("#{key} [#{setting.value}]: ")
+          stdin = STDIN.gets.chomp.strip
+
+          required = setting.required?
+          new_value = setting.value
+          new_value = stdin unless null?(stdin)
+
+          new_settings[key] = Pfm::Settings::Setting.new(new_value, required)
+        end
+
+        current_set.save_config(new_settings)
+      end
+
+      def needs_version?(_params)
+        # Force version to get passed down to command
+        false
+      end
+
+      def null?(value)
+        value.nil? || value == ''
+      end
+
+      def needs_help?(params)
+        ['-h', '--help'].include? params[0]
+      end
+    end
+  end
+end

data/lib/idlc-sdk-pfm/command/destroy.rb

@@ -0,0 +1,61 @@
+require 'idlc-sdk-pfm/command/base'
+require 'mixlib/shellout'
+
+module Pfm
+  module Command
+    class Destroy < Base
+      banner 'Usage: pfm destroy [options]'
+
+      def initialize
+        super
+        @params_valid = true
+        @errors = []
+      end
+
+      def run(params)
+        @params = params
+        read_and_validate_params
+
+        if params_valid?
+          deploy_setup
+          destroy
+          # @workspace.cleanup causing bundler issues
+          0
+        else
+          @errors.each { |error| err("Error: #{error}") }
+          parse_options(params)
+          msg(opt_parser)
+          1
+        end
+      rescue DeploymentFailure => e
+        err("ERROR: #{e.message}\n")
+        1
+      end
+
+      def destroy
+        Idlc::Deploy::Config.add_deployment_var('build', 'null')
+        Idlc::Deploy::Config.add_deployment_var('app_release', 'null')
+
+        Terraform::Binary.get("-update #{@workspace.tmp_dir}")
+        Terraform::Binary.destroy("-refresh=false #{@workspace.tmp_dir}")
+      rescue
+        raise DeploymentFailure, 'Finished with errors'
+      end
+
+      def read_and_validate_params
+        arguments = parse_options(@params)
+
+        @params_valid = case arguments.size
+                        when 0
+                          true
+                        else
+                          false
+                        end
+      end
+
+      def params_valid?
+        @params_valid
+      end
+    end
+  end
+end

data/lib/idlc-sdk-pfm/command/exec.rb

@@ -0,0 +1,24 @@
+require 'idlc-sdk-pfm/command/base'
+require 'mixlib/shellout'
+
+module Pfm
+  module Command
+    class Exec < Pfm::Command::Base
+      banner 'Usage: pfm exec SYSTEM_COMMAND'
+
+      def run(params)
+        exec(*params)
+        raise 'Exec failed without an exception, your ruby is buggy' # should never get here
+      end
+
+      def needs_version?(_params)
+        # Force version to get passed down to command
+        false
+      end
+
+      def needs_help?(params)
+        ['-h', '--help'].include? params[0]
+      end
+    end
+  end
+end

data/lib/idlc-sdk-pfm/command/format.rb

@@ -0,0 +1,64 @@
+require 'idlc-sdk-pfm/command/base'
+require 'mixlib/shellout'
+
+module Pfm
+  module Command
+    class Format < Base
+      banner 'Usage: pfm format [options]'
+
+      def initialize
+        super
+        @params_valid = true
+        @errors = []
+      end
+
+      def run(params)
+        @params = params
+        read_and_validate_params
+
+        if params_valid?
+          fmt
+          # @workspace.cleanup causing bundler issues
+          0
+        else
+          @errors.each { |error| err("Error: #{error}") }
+          parse_options(params)
+          msg(opt_parser)
+          1
+        end
+      rescue DeploymentFailure => e
+        err("ERROR: #{e.message}\n")
+        1
+      end
+
+      def fmt
+        raise InvalidRepository, 'This doesn\'t look like a valid infrastructure repository' unless File.directory? "#{inf_base_dir}/tf"
+        tf_paths = %W[#{inf_base_dir}/tf lib/tf/modules]
+
+        begin
+          tf_paths.each do |path|
+            # Format the file to a canonical syntax
+            Terraform::Binary.fmt(path)
+          end
+        rescue
+          raise DeploymentFailure, 'Finished with errors'
+        end
+      end
+
+      def read_and_validate_params
+        arguments = parse_options(@params)
+
+        case arguments.size
+        when 0
+          @params_valid = true
+        else
+          @params_valid = false
+        end
+      end
+
+      def params_valid?
+        @params_valid
+      end
+    end
+  end
+end

data/lib/idlc-sdk-pfm/command/generate.rb

@@ -0,0 +1,83 @@
+require 'idlc-sdk-pfm/command/base'
+require 'idlc-sdk-pfm/command/generator_commands'
+require 'idlc-sdk-pfm/command/generator_commands/base'
+require 'idlc-sdk-pfm/command/generator_commands/server_build'
+
+module Pfm
+  module Command
+    class Generate < Base
+
+      GeneratorCommand = Struct.new(:name, :class_name, :description)
+
+      def self.generators
+        @generators ||= []
+      end
+
+      def self.generator(name, class_name, description)
+        generators << GeneratorCommand.new(name, class_name, description)
+      end
+
+      generator('server-build', :ServerBuild, 'Generate a server build repo')
+
+      def self.banner_headline
+        <<-E
+Usage: pfm generate GENERATOR [options]
+Available generators:
+E
+      end
+
+      def self.generator_list
+        justify_size = generators.map { |g| g.name.size }.max + 2
+        generators.map { |g| "  #{g.name.to_s.ljust(justify_size)}#{g.description}" }.join("\n")
+      end
+
+      def self.banner
+        banner_headline + generator_list + "\n"
+      end
+
+      # pfm generate app path/to/basename --skel=path/to/skeleton --example
+      # pfm generate file name [path/to/cookbook_root] (inferred from cwd) --from=source_file
+
+      def initialize(*args)
+        super
+      end
+
+      def run(params)
+        if generator_spec = generator_for(params[0])
+          params.shift
+          generator = GeneratorCommands.build(generator_spec.class_name, params)
+          generator.run
+        else
+          msg(banner)
+          1
+        end
+      rescue OptionParser::InvalidOption, OptionParser::MissingArgument => e
+        # Pfm::Command::Base also handles this error in the same way, but it
+        # does not have access to the correct option parser, so it cannot print
+        # the usage correctly. Therefore, invalid CLI usage needs to be handled
+        # here.
+        err("ERROR: #{e.message}\n")
+        msg(generator.opt_parser)
+        1
+      end
+
+      def generator_for(arg)
+        self.class.generators.find { |g| g.name.to_s == arg }
+      end
+
+      # In the Base class, this is defined to be true if any args match "-h" or
+      # "--help". Here we override that behavior such that if the first
+      # argument is a valid generator name, like `pfm generate cookbook -h`,
+      # we delegate the request to the specified generator.
+      def needs_help?(params)
+        return false if have_generator?(params[0])
+        super
+      end
+
+      def have_generator?(name)
+        self.class.generators.map { |g| g.name.to_s }.include?(name)
+      end
+
+    end
+  end
+end

data/lib/idlc-sdk-pfm/command/generator_commands/base.rb

@@ -0,0 +1,50 @@
+require "idlc-sdk-pfm/command/generator_commands"
+
+module Pfm
+  module Command
+    module GeneratorCommands
+      # ## Base
+      #
+      # Base class for `chef generate` subcommands. Contains basic behaviors
+      # for setting up the generator context, detecting git, and launching a
+      # chef converge.
+      #
+      # The behavior of the generators is largely delegated to a chef cookbook.
+      # The default implementation is the `code_generator` cookbook in
+      # pfm/skeletons/code_generator.
+      class Base < Command::Base
+
+        attr_reader :params
+        attr_reader :errors
+
+        options.merge!(SharedGeneratorOptions.options)
+
+        def initialize(params)
+          super()
+          @params_valid = true
+          @errors = []
+          @params = params
+        end
+
+        def setup_context
+        end
+
+        def read_and_validate_params
+          arguments = parse_options(params)
+          case arguments.size
+          when 1
+
+          when 2
+
+          else
+            @params_valid = false
+          end
+        end
+
+        def params_valid?
+          @params_valid
+        end
+      end
+    end
+  end
+end

data/lib/idlc-sdk-pfm/command/generator_commands/server_build.rb

@@ -0,0 +1,58 @@
+module Pfm
+  module Command
+    module GeneratorCommands
+      class ServerBuild < Base
+        banner 'Usage: pfm generate server-build NAME [options]'
+
+        options.merge!(SharedGeneratorOptions.options)
+
+        def run
+          read_and_validate_params
+          if params_valid?
+            setup_context
+
+            mk_build_dirs
+            default_build_files
+            default_cookbooks
+          else
+            errors.each { |error| err("Error: #{error}") }
+            parse_options(params)
+            msg(opt_parser)
+            1
+          end
+        end
+
+        def mk_build_dirs
+          %w(
+            chef/bake/cookbooks
+            chef/fry/cookbooks
+            chef/vendor/cookbooks
+          ).each do |path|
+            FileUtils.mkdir_p("#{base_dir}/#{path}", verbose: verbose?(params))
+          end
+        end
+
+        def default_build_files
+          %w(
+            metadata
+          ).each do |file|
+            FileUtils.touch("#{base_dir}/#{file}", verbose: verbose?(params))
+          end
+        end
+
+        def default_cookbooks
+          %w(
+            chef/bake/cookbooks/bake
+            chef/fry/cookbooks/fry
+          ).each do |cookbook|
+            system("chef generate cookbook #{base_dir}/#{cookbook} -g #{__dir__}/skeletons/code_generator")
+          end
+        end
+
+        def base_dir
+          "builds/#{@params.first}"
+        end
+      end
+    end
+  end
+end

data/lib/idlc-sdk-pfm/command/generator_commands/skeletons/code_generator/files/default/Berksfile

@@ -0,0 +1,3 @@
+source 'https://supermarket.chef.io'
+
+metadata