jets 5.0.17 → 6.0.0.beta1

data/lib/jets/cfn/builder/parent/genesis.rb

@@ -0,0 +1,84 @@
+module Jets::Cfn::Builder::Parent
+  class Genesis
+    extend Memoist
+    include Jets::AwsServices
+    include Jets::Cfn::Builder::Interface
+    include Jets::Util::Logging
+
+    def initialize(options)
+      @options = options
+      @template = ActiveSupport::HashWithIndifferentAccess.new(Resources: {})
+    end
+
+    # interface method
+    def compose
+      clean
+      add_description("Jets: #{Jets::VERSION}")
+      add_resource(Jets::Cfn::Resource::S3::JetsBucket.new)
+
+      # codebuild resources
+      config = Jets.bootstrap.config # want bootstrap not deploy config here
+      add_resource(Jets::Cfn::Resource::Codebuild::IamRole.new)
+      unless config.infra
+        add_resource(Jets::Cfn::Resource::Codebuild::Project::Ec2.new)
+      end
+      if config.infra || config.codebuild.lambda.enable
+        add_resource(Jets::Cfn::Resource::Codebuild::Project::Lambda.new)
+      end
+      if config.codebuild.fleet.enable
+        add_resource(Jets::Cfn::Resource::Codebuild::Fleet.new)
+      end
+
+      merge_existing_template! if @options[:bootstrap]
+    end
+
+    # interface method: Finale overrides
+    def clean?
+      @options[:bootstrap]
+    end
+
+    def clean
+      return unless clean?
+      templates_path = "#{Jets.build_root}/templates"
+      logger.debug "Parent Genesis clean #{templates_path}"
+      FileUtils.rm_rf(templates_path)
+    end
+
+    # interface method
+    def template_path
+      Jets::Names.parent_template_path
+    end
+
+    # Note: Tried concept of marking resources as a genesis resource as part of
+    # add_resource but that approach cannot handle deletion of resources.
+    # So we need a pre-defined list of genesis resources.
+    class_attribute :genesis_resources
+    self.genesis_resources = %w[S3Bucket Codebuild CodebuildRole CodebuildFleet]
+    delegate :genesis_resources, to: :class
+
+    # This is how cfn delta updates are achieved.
+    def merge_existing_template!
+      existing = existing_template
+      return unless existing
+
+      # Delete resources managed by the bootstrap genesis stack.
+      existing["Resources"].delete_if { |k, v| genesis_resources.include?(k) }
+      # Note: In case Outputs are all deleted by something else in the future
+      # Finale stack does not delete outputs, it filters them.
+      # Genesis resource outputs names should match the genesis resource logic id.
+      outputs = existing["Outputs"]
+      outputs&.delete_if { |k, v| genesis_resources.include?(k) }
+
+      @template.deep_merge!(existing)
+    end
+
+    def existing_template
+      stack_name = Jets::Names.parent_stack_name
+      template_body = cfn.get_template(stack_name: stack_name).template_body
+      # TODO: vs Jets::Util::Yamler
+      Jets::Cfn::Stack::Yamler.load(template_body)
+    rescue Aws::CloudFormation::Errors::ValidationError => e
+      raise unless /does not exist/.match?(e.message)
+    end
+  end
+end

data/lib/jets/cfn/builder/post_process.rb

@@ -1,7 +1,7 @@
 # post process the text so that
 # "!Ref IamRole" => !Ref IamRole
 # We strip the surrounding quotes
-class Jets::Cfn::Builder
+module Jets::Cfn::Builder
   class PostProcess
     def initialize(text)
       @text = text
@@ -10,10 +10,10 @@ class Jets::Cfn::Builder
     def process
       results = @text.split("\n").map do |line|
         if line.include?(': "!') # IE: IamRole: "!Ref IamRole",
-           # IamRole: "!Ref IamRole" => IamRole: !Ref IamRole
+          # IamRole: "!Ref IamRole" => IamRole: !Ref IamRole
           line.sub(/: "(.*)"/, ': \1')
         elsif line.include?('- "!') # IE: - "!GetAtt Foo.Arn"
-           # IamRole: - "!GetAtt Foo.Arn" => - !GetAtt Foo.Arn
+          # IamRole: - "!GetAtt Foo.Arn" => - !GetAtt Foo.Arn
           line.sub(/- "(.*)"/, '- \1')
         else
           line

data/lib/jets/cfn/delete.rb

@@ -0,0 +1,17 @@
+module Jets::Cfn
+  class Delete < Stack
+    def run
+      bootstrap_if_needed
+      Jets::Remote::Runner.new(@options.merge(dummy: true, command: "delete")).run
+      Teardown.new(@options).run
+    end
+
+    # In case user has deleted stack already and needs to delete the Jets API deployment record.
+    def bootstrap_if_needed
+      stack_name = Jets.project.namespace
+      return if stack_exists?(stack_name)
+      log.info "Creating dummy stack for deletion: #{stack_name}"
+      Jets::Cfn::Bootstrap.new(@options).run
+    end
+  end
+end

data/lib/jets/cfn/deploy.rb

@@ -0,0 +1,151 @@
+module Jets::Cfn
+  class Deploy < Stack
+    def sync
+      delete_rollback_complete!
+      continue_update_rollback!
+      check_deployable!
+
+      log.debug "Parent template changed: #{changed?.inspect}"
+      # return true when no changes so deploy will continue and start remote runner
+      return true unless changed?
+
+      # bootstrap can be "delete" or true
+      # Set quiet before stack exists check
+      quiet_sync = @options[:bootstrap] == true && stack_exists?(stack_name)
+      deploy_message
+      set_resource_tags
+      begin
+        sync_stack
+      rescue Aws::CloudFormation::Errors::InsufficientCapabilitiesException => e
+        capabilities = e.message.match(/\[(.*)\]/)[1]
+        confirm = prompt_for_iam(capabilities)
+        if /^y/.match?(confirm)
+          @options.merge!(capabilities: [capabilities])
+          log.info "Re-running: #{command_with_iam(capabilities).color(:green)}"
+          retry
+        else
+          log.error "ERROR: Unable to deploy #{e.message}"
+          exit 1
+        end
+      end
+
+      # waits for /(_COMPLETE|_FAILED)$/ status to see if successfully deployed
+      success = cfn_status.wait(quiet: quiet_sync)
+      if success
+        log.info "Bootstrap synced" if @options[:bootstrap] == true # dont use quiet_sync
+      else
+        if quiet_sync # show full cfn stack status if quiet_sync
+          cfn_status.run
+        end
+        cfn_status.failure_message
+      end
+      success
+    end
+
+    # note: required method: rollback.rb module uses
+    def cfn_status
+      Jets::Cfn::Status.new
+    end
+    memoize :cfn_status
+
+    # CloudFormation always performs an update there are nested templates, even when the
+    # parent template has not changed at all.
+    # Note: Jets ressurects the template body with a slight difference
+    #    !Ref S3Bucket vs {Ref: S3Bucket}
+    # However, tested with the exact template body without this difference
+    # and cloudformation still performs an update.
+    # So we have to check if the template has changed ourselves.
+    # This is useful for the bootstrap genesis update.
+    def changed?
+      return true if @options[:force_changed]
+      return true unless stack_exists?(stack_name)
+
+      template_body = cfn.get_template(stack_name: stack_name).template_body
+      existing = Yamler.load(template_body)
+      fresh = Yamler.load(template.body)
+
+      FileUtils.mkdir_p("#{Jets.build_root}/cfn/diff")
+      IO.write("#{Jets.build_root}/cfn/diff/existing.yml", YAML.dump(existing))
+      IO.write("#{Jets.build_root}/cfn/diff/fresh.yml", YAML.dump(fresh))
+
+      existing != fresh
+    end
+    memoize :changed?
+
+    def deploy_message
+      if @options[:bootstrap_message]
+        log.info "#{@options[:bootstrap_message]}: #{stack_name}"
+      elsif @options[:bootstrap]
+        log.info "Syncing bootstrap: #{stack_name}"
+      else
+        log.info "Deploying app: #{stack_name}"
+      end
+    end
+
+    def set_resource_tags
+      @tags = Jets.bootstrap.config.cfn.resource_tags.map { |k, v| {key: k, value: v} }
+    end
+
+    def sync_stack
+      if stack_exists?(stack_name)
+        update_stack
+      else
+        create_stack
+      end
+    end
+
+    def create_stack
+      # initial create stack template is on filesystem
+      cfn.create_stack(stack_options)
+    end
+
+    def update_stack
+      cfn.update_stack(stack_options)
+    rescue Aws::CloudFormation::Errors::ValidationError => e
+      log.debug "DEBUG bootstrap/deploy.rb update_stack #{e.message}" # ERROR: No updates are to be performed.
+      true
+    end
+
+    # options common to both create_stack and update_stack
+    def stack_options
+      {
+        stack_name: stack_name,
+        capabilities: capabilities,
+        tags: @tags
+      }.merge(template.template_option)
+    end
+
+    def template
+      Template.new(@options)
+    end
+    memoize :template
+
+    # All CloudFormation states listed here:
+    # http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-describing-stacks.html
+    def stack_status
+      resp = cfn.describe_stacks(stack_name: stack_name)
+      status = resp.stacks[0].stack_status
+      [resp, status]
+    end
+
+    def prompt_for_iam(capabilities)
+      log.info "This stack will create IAM resources.  Please approve to run the command again with #{capabilities} capabilities."
+      log.info "  #{command_with_iam(capabilities)}"
+
+      log.info "Please confirm (y/n)"
+      $stdin.gets # confirm
+    end
+
+    def command_with_iam(capabilities)
+      "#{File.basename($0)} #{ARGV.join(" ")} --capabilities #{capabilities}"
+    end
+
+    def capabilities
+      ["CAPABILITY_IAM", "CAPABILITY_NAMED_IAM"]
+    end
+
+    def stack_name
+      Jets::Names.parent_stack_name
+    end
+  end
+end

data/lib/jets/cfn/iam/managed_policy.rb

@@ -0,0 +1,21 @@
+module Jets::Cfn::Iam
+  # Examples:
+  # config.codebuild.iam.managed_policies = [AmazonSSMReadOnlyAccess]
+  class ManagedPolicy
+    def initialize(policies)
+      @policies = policies.compact.flatten.uniq
+    end
+
+    def standardize
+      return if @policies.nil? || @policies.empty?
+
+      @policies.map do |policy|
+        if policy.include?("arn:")
+          policy
+        else
+          "arn:aws:iam::aws:policy/#{policy}"
+        end
+      end
+    end
+  end
+end

data/lib/jets/cfn/iam/policy.rb

@@ -0,0 +1,121 @@
+module Jets::Cfn::Iam
+  # Examples:
+  # config.codebuild.iam.policies = ["s3", "ec2"]
+  # config.codebuild.iam.policies = [
+  #   "s3",
+  #   {
+  #     PolicyName: "hello",
+  #     PolicyDocument: {
+  #       Version: "2012-10-17",
+  #       Statement: [{Action: ["s3:*"], Effect: "Allow", Resource: "*"}]
+  #     }
+  #   }
+  # ]
+  class Policy
+    def initialize(policy_name, definitions)
+      @policy_name, @definitions = policy_name, definitions.compact.flatten.uniq
+    end
+
+    # Returns a standardize policy document. Example:
+    #   {
+    #     PolicyName: "hello",
+    #     PolicyDocument: {
+    #       Version: "2012-10-17",
+    #       Statement: [{Action: ["s3:*"], Effect: "Allow", Resource: "*"}]
+    #     }
+    #   }
+    #
+    # A definition is a String or a Hash. It's very close to a Statement
+    #   String: "s3"
+    #   Hash: {Action: ["s3:*"], Effect: "Allow", Resource: "*"} # Action item
+    def standardize
+      return if @definitions.nil? || @definitions.empty?
+
+      if @definitions.is_a?(Hash)
+        standardize_hash(@definitions) # final policy
+      else # Array of definitions
+        statement = statement_from_array(@definitions) # statement is Array
+        # final policy
+        # Note since we always extract the statement we ignore the PolicyDocument Version
+        # and always use 2012-10-17
+        {
+          PolicyName: @policy_name,
+          PolicyDocument: {
+            Version: "2012-10-17",
+            Statement: statement
+          }
+        }
+      end
+    end
+
+    def statement_from_array(definitions)
+      if definitions.all? { |definition| definition.is_a?(String) }
+        statement_from_all_strings(definitions)
+      else
+        definitions.map do |definition|
+          if definition.is_a?(String)
+            statement_from_string(definition)
+          else # assume hash
+            statement_from_hash(definition) # possible Array or Hash
+          end
+        end.flatten # due to statement_from_hash
+      end
+    end
+
+    def all_actions_colon_star(action)
+      action.include?(":") ? action : "#{action}:*"
+    end
+
+    def statement_from_all_strings(definitions)
+      action = definitions.map do |definition|
+        all_actions_colon_star(definition)
+      end
+      [Action: action, Effect: "Allow", Resource: "*"]
+    end
+
+    def statement_from_string(definition)
+      action = [all_actions_colon_star(definition)]
+      [Action: action, Effect: "Allow", Resource: "*"]
+    end
+
+    def statement_from_hash(definition)
+      if definition.key?(:Statement) # full PolicyDocument. Has Version and Statement
+        # Will have an Array of Statements that needs to be flattened later
+        definition[:Statement] # This is an Array
+      elsif definition.key?(:Action)
+        definition
+      else
+        definition.merge(Action: [all_actions_colon_star(definition[:Action])])
+      end
+    end
+
+    # Example return value:
+    #   - Effect: Allow
+    #     Action: '*'
+    #     Resource: '*'
+    def standardize_hash(hash)
+      if hash.key?(:Action)
+        {
+          PolicyName: @policy_name,
+          PolicyDocument: {
+            Version: "2012-10-17",
+            Statement: [hash]
+          }
+        }
+      elsif hash.key?(:Statement)
+        {
+          PolicyName: @policy_name,
+          PolicyDocument: hash
+        }
+      elsif hash.key?(:PolicyDocument)
+        if hash.key?(:PolicyName)
+          hash # full hash with both PolicyName and PolicyDocument
+        else
+          hash.merge(PolicyName: @policy_name) # almost full hash with PolicyDocument
+        end
+      else
+        raise "Invalid hash format: #{hash.inspect}"
+      end
+    end
+  end
+end

data/lib/jets/cfn/resource/associated.rb

@@ -34,4 +34,4 @@ class Jets::Cfn::Resource
     end
     memoize :standardized
   end
-end
+end

data/lib/jets/cfn/resource/codebuild/fleet.rb

@@ -0,0 +1,48 @@
+module Jets::Cfn::Resource::Codebuild
+  class Fleet < Jets::Cfn::Base
+    def definition
+      {
+        CodebuildFleet: {
+          Type: "AWS::CodeBuild::Fleet",
+          Properties: props
+        }
+      }
+    end
+
+    def props
+      {
+        BaseCapacity: base_capacity,        # Integer
+        ComputeType: compute_type,          # String
+        EnvironmentType: environment_type   # String
+        # Name: "",                         # String
+        # Tags: "",                         # [ Tag, ... ]
+      }
+    end
+
+    def base_capacity
+      Jets.bootstrap.config.codebuild.fleet.base_capacity
+    end
+
+    def compute_type
+      codebuild_properties[:Environment][:ComputeType]
+    end
+
+    def environment_type
+      codebuild_properties[:Environment][:Type]
+    end
+
+    def outputs
+      {
+        "CodebuildFleet" => "!Ref CodebuildFleet"
+      }
+    end
+
+    private
+
+    def codebuild_properties
+      project = Jets::Cfn::Resource::Codebuild::Project::Ec2.new
+      project.properties
+    end
+    memoize :codebuild_properties
+  end
+end

data/lib/jets/cfn/resource/codebuild/iam_role.rb

@@ -0,0 +1,63 @@
+module Jets::Cfn::Resource::Codebuild
+  class IamRole < Jets::Cfn::Base
+    def definition
+      {
+        "CodebuildRole" => {
+          Type: "AWS::IAM::Role",
+          Properties: props
+        }
+      }
+    end
+
+    # Do not name this method properties as that's a computed method
+    def props
+      text = <<~EOL
+        AssumeRolePolicyDocument:
+          Statement:
+          - Action:
+            - sts:AssumeRole
+            Effect: Allow
+            Principal:
+              Service:
+              - codebuild.amazonaws.com
+          Version: '2012-10-17'
+        Path: "/"
+      EOL
+      props = YAML.load(text).deep_symbolize_keys
+      props[:Policies] = policies
+      props[:ManagedPolicyArns] = managed_policy_arns
+      props
+    end
+
+    def policies
+      [default_policy, custom_policy, vpc_policy].flatten.compact
+    end
+
+    def default_policy
+      Jets::Cfn::Iam::Policy.new("DefaultPolicy", config.codebuild.iam.default_policy).standardize
+    end
+
+    def vpc_policy
+      policy = config.codebuild.iam.default_vpc_policy
+      if !policy.nil? && !policy.empty?
+        Jets::Cfn::Iam::Policy.new("VpcPolicy", policy).standardize
+      end
+    end
+
+    def custom_policy
+      Jets::Cfn::Iam::Policy.new("CustomPolicy", config.codebuild.iam.policy).standardize
+    end
+
+    def managed_policy_arns
+      [default_managed_policy, custom_managed_policy].flatten.compact
+    end
+
+    def default_managed_policy
+      Jets::Cfn::Iam::ManagedPolicy.new(config.codebuild.iam.default_managed_policy).standardize
+    end
+
+    def custom_managed_policy
+      Jets::Cfn::Iam::ManagedPolicy.new(config.codebuild.iam.managed_policy).standardize
+    end
+  end
+end

data/lib/jets/cfn/resource/codebuild/project/base.rb

@@ -0,0 +1,4 @@
+module Jets::Cfn::Resource::Codebuild::Project
+  class Base < Jets::Cfn::Base
+  end
+end

data/lib/jets/cfn/resource/codebuild/project/ec2.rb

@@ -0,0 +1,125 @@
+module Jets::Cfn::Resource::Codebuild::Project
+  class Ec2 < Base
+    def definition
+      {
+        codebuild_logical_id => {
+          Type: "AWS::CodeBuild::Project",
+          Properties: finalize_properties(props)
+        }
+      }
+    end
+
+    # Do not name this method properties as that's a computed method
+    def props
+      {
+        Name: project_name,
+        Artifacts: {
+          Type: "NO_ARTIFACTS"
+        },
+        Environment: environment,
+        Source: {
+          Type: "NO_SOURCE",
+          BuildSpec: build_spec
+        },
+        ServiceRole: "!Ref CodebuildRole",
+        LogsConfig: {
+          CloudWatchLogs: {
+            Status: "ENABLED"
+          }
+        }
+      }
+    end
+
+    # Only supported for EC2 compute type. Lambda compute type does not support.
+    def timeout_in_minutes
+      config.codebuild.project.timeout || config.codebuild.project.timeout_in_minutes
+    end
+
+    # Certain properties are not supported for certain Compute Types
+    # We set this at the end based on the final Environment Type
+    #
+    # PrivilegedMode: Only supported for non-LAMBDA environment types
+    # Cache: non-LAMBDA environment types
+    #
+    #   CREATE_FAILED AWS::CodeBuild::Project Codebuild Cannot specify timeoutInMinutes for lambda compute
+    #   TimeoutInMinutes: 20, # TODO: make this configurable
+    #   CREATE_FAILED AWS::CodeBuild::Project
+    #   Codebuild Cannot specify cache for lambda compute. Error Code: InvalidInputException
+    #
+    # PrivilegedMode: true, # Note: does not seem to be actually needed for docker support
+    # When using PrivilegedMode for ARM_LAMBDA_CONTAINER it'll error though.
+    # Not officially supporting ARM_LAMBDA_CONTAINER when packaging code anyway.
+    # Just noting for posterity.
+    def finalize_properties(props)
+      if environment[:Type].include?("LAMBDA")
+        props[:Cache] = {
+          Type: "NO_CACHE"
+        }
+        props[:Environment][:PrivilegedMode] = false
+      else
+        props[:Cache] = {
+          Type: "LOCAL",
+          Modes: ["LOCAL_DOCKER_LAYER_CACHE", "LOCAL_CUSTOM_CACHE"]
+        }
+        props[:TimeoutInMinutes] = timeout_in_minutes # not supported for lambda compute
+        props[:Environment][:PrivilegedMode] = true
+      end
+
+      if config.codebuild.fleet.enable
+        props[:Environment][:Fleet] = {
+          FleetArn: "!Ref CodebuildFleet"
+        }
+      end
+
+      props
+    end
+
+    def build_spec
+      <<~EOL
+        version: 0.2
+
+        phases:
+          build:
+            commands:
+               - uname -a
+      EOL
+    end
+
+    # https://docs.aws.amazon.com/codebuild/latest/userguide/build-env-ref-available.html
+    def environment
+      env = compute_type
+      env.deep_merge!(config.codebuild.project.environment)
+      env.deep_merge!(EnvironmentVariables: environment_variables) if environment_variables
+      # sort for correct diffing. Deploy#changed? checks the template body diff
+      env[:EnvironmentVariables].sort_by! { |h| h[:Name] }
+      env
+    end
+    memoize :environment
+
+    def environment_variables
+      Env.new.vars
+    end
+
+    def outputs
+      {
+        codebuild_logical_id => "!Ref #{codebuild_logical_id}"
+      }
+    end
+
+    # interface method
+    def codebuild_logical_id
+      "Codebuild"
+    end
+
+    # interface method
+    def project_name
+      "#{Jets.project.namespace}-remote"
+    end
+
+    # interface method
+    def compute_type
+      # Note: PrivilegedMode is set at the end in finalize_properties based on the final Type
+      config.codebuild.project.compute_type
+    end
+  end
+end