dpl 2.0.0.alpha.2 → 2.0.0.alpha.5

data/lib/dpl/providers/chef_supermarket.rb

@@ -22,10 +22,12 @@ module Dpl
       gem 'net-telnet', '~> 0.1.0' if ruby_pre?('2.3')
       gem 'rack'
 
+      env :chef
+
       opt '--user_id ID',         'Chef Supermarket user name', required: true
-      opt '--client_key KEY',     'Client API key file name', required: true
       opt '--name NAME',          'Cookbook name', note: 'defaults to the name given in metadata.json or metadata.rb', alias: :cookbook_name, deprecated: :cookbook_name
       opt '--category CAT',       'Cookbook category in Supermarket', required: true, see: 'https://docs.getchef.com/knife_cookbook_site.html#id12', alias: :cookbook_category, deprecated: :cookbook_category
+      opt '--client_key KEY',     'Client API key file name', default: 'client.pem'
       opt '--dir DIR',            'Directory containing the cookbook', default: '.'
 
       URL = "https://supermarket.chef.io/api/v1/cookbooks"

data/lib/dpl/providers/cloud66.rb

@@ -7,6 +7,8 @@ module Dpl
         tbd
       str
 
+      env :cloud66
+
       opt '--redeployment_hook URL', 'The redeployment hook URL', required: true, secret: true
 
       msgs failed: 'Redeployment failed (%s)'

data/lib/dpl/providers/cloudfiles.rb

@@ -13,6 +13,8 @@ module Dpl
       gem 'fog-core', '= 2.1.0', require: 'fog/core'
       gem 'fog-rackspace', '~> 0.1.6', require: 'fog/rackspace'
 
+      env :cloudfiles
+
       opt '--username USER',  'Rackspace username', required: true
       opt '--api_key KEY',    'Rackspace API key', required: true, secret: true
       opt '--region REGION',  'Cloudfiles region', required: true, enum: %w(ord dfw syd iad hkg)

data/lib/dpl/providers/cloudformation.rb

@@ -0,0 +1,278 @@
+module Dpl
+  module Providers
+    class Cloudformation < Provider
+      status :dev
+
+      full_name 'AWS CloudFormation'
+
+      description sq(<<-str)
+        tbd
+      str
+
+      gem 'aws-sdk-cloudformation', '~> 1.0'
+
+      env :aws, :cloudformation
+      config '~/.aws/credentials', prefix: 'aws'
+
+      opt '--access_key_id ID', 'AWS Access Key ID', required: true, secret: true
+      opt '--secret_access_key KEY', 'AWS Secret Key', required: true, secret: true
+      opt '--region REGION', 'AWS Region to deploy to', default: 'us-east-1'
+      opt '--template STR', 'CloudFormation template file', required: true, note: 'can be either a local path or an S3 URL'
+      opt '--stack_name NAME', 'CloudFormation Stack Name.', required: true
+      opt '--stack_name_prefix STR', 'CloudFormation Stack Name Prefix.'
+      opt '--promote', 'Deploy changes', default: true, note: 'otherwise a change set is created'
+      opt '--role_arn ARN', 'AWS Role ARN'
+      opt '--sts_assume_role ARN', 'AWS Role ARN for cross account deployments (assumed by travis using given AWS credentials).'
+      opt '--capabilities STR', 'CloudFormation allowed capabilities', type: :array, enum: %w(CAPABILITY_IAM CAPABILITY_NAMED_IAM CAPABILITY_AUTO_EXPAND), sep: ',', see: 'https://docs.aws.amazon.com/AWSCloudFormation/latest/APIReference/API_CreateStack.html'
+      opt '--wait', 'Wait for CloutFormation to finish the stack creation and update', default: true
+      opt '--wait_timeout SEC', 'How many seconds to wait for stack creation and update.', type: :integer, default: 3600
+      opt '--create_timeout SEC', 'How many seconds to wait before the stack status becomes CREATE_FAILED', type: :integer, default: 3600, note: 'valid only when creating a stack'
+      # if passing a session_token is not recommended in CI/CD why do we add it to dpl?
+      opt '--session_token STR', 'AWS Session Access Token if using STS assume role', note: 'Not recommended on CI/CD'
+      opt '--parameters STR', 'key=value pairs or ENV var names', type: :array, sep: ',', eg: 'one=1 or ENV_VAR_TWO'
+      opt '--output_file PATH', 'Path to output file to store CloudFormation outputs to'
+
+      msgs login:             'Using Access Key: %{access_key_id}',
+           create_stack:      'Creating stack ...',
+           promote_stack:     'Promoting stack ...',
+           create_change_set: 'Creating change set ...',
+           stack_up_to_date:  'Stack already up to date.',
+           delete_change_set: 'No changes in stack. Removing changeset.',
+           done:              'Done.',
+           missing_template:  'File does not exist: %{template}',
+           invalid_creds:     'Invalid credentials'
+
+      strs change_set_name:   'travis-ci-build-%{build_number}-%{now}',
+           change_set_desc:   'Changeset created by Travis CI job for build #%{build_number} (%{git_sha})'
+
+      def login
+        info :login
+      end
+
+      def deploy
+        stack_exists? ? update : create
+        store_events if output_file?
+      rescue Aws::CloudFormation::Errors::InvalidAccessKeyId
+        error :invalid_creds
+      end
+
+      private
+
+        def update
+          promote? ? promote : create_change_set(:update)
+        rescue Aws::CloudFormation::Errors::ValidationError => e
+          raise e unless e.message.start_with?('No updates are to be performed')
+          info :stack_up_to_date
+        end
+
+        def promote
+          info :promote_stack
+          client.update_stack(common_params)
+          stream_events(stack_name, :stack_update_complete) if wait?
+          info :done
+        end
+
+        def create
+          promote? ? create_stack : create_change_set(:create)
+        end
+
+        def create_stack
+          info :create_stack
+          params = { timeout_in_minutes: create_timeout, on_failure: 'ROLLBACK' }
+          client.create_stack(common_params.merge(params))
+          stream_events(stack_name, :stack_create_complete) if wait?
+          info :done
+        end
+
+        def create_change_set(type)
+          info :create_change_set
+          set = client.create_change_set(common_params.merge(change_set_params(type)))
+          wait_for(:change_set_create_complete, change_set_name: set.id) if wait? && !test?
+          info :done
+        rescue Aws::Waiters::Errors::FailureStateError => e
+          raise e unless change_set_contains_changes?(set)
+          info :delete_change_set
+          client.delete_change_set(change_set_name: set.id)
+        end
+
+        def change_set_params(type)
+          {
+            change_set_type: type.to_s.upcase,
+            change_set_name: interpolate(str(:change_set_name)),
+            description: interpolate(str(:change_set_desc))
+          }
+        end
+
+        def change_set_contains_changes?(change_set)
+          data = client.describe_change_set(change_set_name: change_set.id)
+          data.status_reason.start_with?(%(The submitted information didn't contain changes))
+        end
+
+        def stack_exists?
+          stack = last_stack
+          stack && stack.stack_status != 'REVIEW_IN_PROGRESS'
+        rescue Aws::CloudFormation::Errors::ValidationError => e
+          raise e unless e.message.include?('does not exist')
+          false
+        end
+
+        def stream_events(stack_name, condition)
+          stream = EventStream.new(client, stack_name, method(:info))
+          wait_for(condition, stack_name: stack_name) unless test? # hmm.
+        ensure
+          stream.stop unless stream.nil?
+        end
+
+        def wait_for(cond, params)
+          started_at = Time.now
+          timeout = lambda { |*| throw :failure if Time.now - started_at > wait_timeout }
+          # params = params.merge(max_attempts: nil, delay: 5, before_wait: timeout)
+          client.wait_until(cond, params) { |w| w.before_wait(&timeout) }
+        end
+
+        def store_events
+          logs = last_stack.outputs || {}
+          logs = logs.map { |log| "#{log[:output_key]}=#{log[:output_value]}" }
+          File.write(output_file, logs.join("\n"))
+        end
+
+        def last_stack
+          client.describe_stacks(stack_name: stack_name)[:stacks].first
+        end
+
+        def common_params
+          params = {
+            stack_name: stack_name,
+            role_arn: role_arn,
+            capabilities: capabilities,
+            parameters: parameters
+          }
+          params.merge!(template_param)
+          @common_params ||= compact(params)
+        end
+
+        def parameters
+          @parameters ||= Array(super).map do |str|
+            key, value = str.split('=', 2)
+            { parameter_key: key, parameter_value: value || ENV[key] }
+          end
+        end
+
+        def create_timeout
+          super / 60
+        end
+
+        def stack_name
+          @stack_name ||= "#{stack_name_prefix}#{super}"
+        end
+
+        def template_param
+          str = template
+          return { template_url: str } if url?(str)
+          return { template_body: read(str) } if file?(str)
+          error(:missing_template)
+        end
+
+        def client
+          @client ||= Aws::CloudFormation::Client.new(client_options)
+        end
+
+        def client_options
+          params = { region: region, credentials: credentials }
+          params = params.merge(credentials: assume_role(params)) if sts_assume_role?
+          params
+        end
+
+        def credentials
+          Aws::Credentials.new(access_key_id, secret_access_key, session_token)
+        end
+
+        def assume_role(params)
+          assumed_role = Aws::STS::Client.new(params).assume_role(
+            role_arn: sts_assume_role,
+            role_session_name: "travis-build-#{build_number}"
+          )
+          Aws::Credentials.new(
+            assumed_role.credentials.access_key_id,
+            assumed_role.credentials.secret_access_key,
+            assumed_role.credentials.session_token
+          )
+        end
+
+        def now
+          Time.now.strftime('%Y-%m-%dT%H:%M:%S')
+        end
+
+        def url?(str)
+          str =~ %r(^https?://)
+        end
+
+        class EventStream < Struct.new(:client, :stack_name, :handler)
+          attr_reader :thread
+
+          def initialize(*)
+            super
+            @event = describe_stack_events.stack_events.first
+            @thread = Thread.new(&method(:process))
+          end
+
+          def stop
+            mutex.synchronize { @stop = true }
+            thread.join
+          end
+
+          private
+
+            def process
+              until mutex.synchronize { @stop }
+                @event, events = events_since(@event)
+                events.each { |e| handler.call(format_event(e)) }
+                sleep 5 unless ENV['ENV'] == 'test'
+              end
+            end
+
+            # source: https://github.com/rvedotrc/cfn-events/blob/master/lib/cfn-events/runner.rb
+            def events_since(event)
+              described_stack = describe_stack_events
+              stack_events = described_stack.stack_events
+              return [event, []] if stack_events.first.event_id == event.event_id
+
+              events = []
+              described_stack.each_page do |page|
+
+
+                if (oldest_new = page.stack_events.index { |e| e.event_id == event.event_id })
+                  events.concat(page.stack_events[0..oldest_new - 1])
+                  return [events.first, events.reverse]
+                end
+                events.concat(page.stack_events)
+              end
+
+              warn %(Last-seen stack event is no longer returned by AWS. Please raise this as a provider's bug.)
+              [events.first, events.reverse]
+            end
+
+            def describe_stack_events
+              client.describe_stack_events(stack_name: stack_name)
+            end
+
+            def mutex
+              @mutex ||= Mutex.new
+            end
+
+            EVENT_KEYS = %i(timestamp resource_type resource_status logical_resource_id
+              physical_resource_id resource_status_reason)
+
+            def format_event(event)
+              parts = EVENT_KEYS.map { |key| event.send(key) }
+              parts[0] = format_timestamp(parts[0])
+              parts.join(' ')
+            end
+
+            def format_timestamp(timestamp)
+              timestamp.utc.strftime('%Y-%m-%dT%H:%M:%SZ')
+            end
+        end
+    end
+  end
+end

data/lib/dpl/providers/cloudfoundry.rb

@@ -9,13 +9,15 @@ module Dpl
         tbd
       str
 
+      env :cloudfoundry
+
       opt '--username USER',       'Cloud Foundry username', required: true
       opt '--password PASS',       'Cloud Foundry password', required: true, secret: true
-      opt '--organization ORG',    'Cloud Foundry target organization', required: true
-      opt '--space SPACE',         'Cloud Foundry target space', required: true
-      opt '--api URL',             'Cloud Foundry api URL', required: true
+      opt '--organization ORG',    'Cloud Foundry organization', required: true
+      opt '--space SPACE',         'Cloud Foundry space', required: true
+      opt '--api URL',             'Cloud Foundry api URL', default: 'https://api.run.pivotal.io'
       opt '--app_name APP',        'Application name'
-      opt '--buildpack PACK',      'Custom buildpack name or Git URL'
+      opt '--buildpack PACK',      'Buildpack name or Git URL'
       opt '--manifest FILE',       'Path to the manifest'
       opt '--skip_ssl_validation', 'Skip SSL validation'
       opt '--v3',                  'Use the v3 API version to push the application'

data/lib/dpl/providers/codedeploy.rb

@@ -14,7 +14,7 @@ module Dpl
       gem 'aws-sdk-codedeploy', '~> 1.0'
       gem 'aws-sdk-s3', '~> 1.0'
 
-      env :aws
+      env :aws, :codedeploy
       config '~/.aws/credentials', '~/.aws/config', prefix: 'aws'
 
       opt '--access_key_id ID', 'AWS access key', required: true, secret: true
@@ -28,10 +28,10 @@ module Dpl
       opt '--region REGION', 'AWS availability zone', default: 'us-east-1'
       opt '--file_exists_behavior STR', 'How to handle files that already exist in a deployment target location', enum: %w(disallow overwrite retain), default: 'disallow'
       opt '--wait_until_deployed', 'Wait until the deployment has finished'
-      opt '--bundle_type TYPE'
-      opt '--endpoint ENDPOINT'
-      opt '--key KEY'
-      opt '--description DESCR'
+      opt '--bundle_type TYPE', 'Bundle type of the revision'
+      opt '--key KEY', 'S3 bucket key of the revision'
+      opt '--description DESCR', 'Description of the revision'
+      opt '--endpoint ENDPOINT', 'S3 endpoint url'
 
       msgs login:                 'Using Access Key: %{access_key_id}',
            deploy_triggered:      'Deployment triggered: %s',

data/lib/dpl/providers/convox.rb

@@ -0,0 +1,121 @@
+module Dpl
+  module Providers
+    class Convox < Provider
+      status :dev
+
+      description sq(<<-str)
+        tbd
+      str
+
+      gem 'json'
+
+      env :convox
+
+      # needs descriptions
+      opt '--host HOST', default: 'console.convox.com'
+      opt '--app APP', required: true
+      opt '--rack RACK', required: true
+      opt '--password PASS', required: true
+      opt '--install_url URL', default: 'https://convox.com/cli/linux/convox'
+      opt '--update_cli'
+      opt '--create'
+      opt '--promote', default: true
+      opt '--env VARS', type: :array, sep: ','
+      opt '--env_file FILE'
+      opt '--description STR'
+      opt '--generation NUM', type: :int, default: '2'
+
+      # if app and rack are exported to the env, do they need to be passed to these commands?
+      cmds login:    'convox version --rack %{rack}',
+           validate: 'convox apps info --rack %{rack} --app %{app}',
+           create:   'convox apps create %{app} --generation %{generation} --rack %{rack} --wait',
+           update:   'convox update',
+           set_env:  'convox env set %{env} --rack %{rack} --app %{app} --replace',
+           build:    'convox build --rack %{rack} --app %{app} --id --description %{escaped_description}',
+           deploy:   'convox deploy --rack %{rack} --app %{app} --wait --id --description %{escaped_description}'
+
+      msgs create:   'Application %{app} does not exist on rack %{rack}. Creating it ...',
+           missing:  'Application %{app} does not exist on rack %{rack}.',
+           env_file: 'The given env_file does not exist.',
+           deploy:   'Building and promoting application ...',
+           build:    'Building application ...'
+
+      errs login:    'Login failed.'
+
+      def install
+        script :install
+        shell :update if update_cli?
+        export
+      end
+
+      def login
+        shell :login
+      end
+
+      def validate
+        shell :validate, assert: false and return
+        error :missing unless create?
+        shell :create
+      end
+
+      def deploy
+        shell :set_env, echo: false unless env.empty?
+        shell promote ? :deploy : :build, echo: false
+      end
+
+      # not sure about this api. i like that there is an api for people to include
+      # env vars from the current build env, but maybe it would be better to expose
+      # FOO=$FOO? is mapping a bare env key to a key/value pair a concept in convox?
+      #
+      # def env
+      #   env = env_file.concat(super || []) # TODO Cl should return an empty array, shouldn't it?
+      #   env = env.map { |str| str.include?('=') ? str : "#{str}=#{ENV[str]}" }
+      #   env.map { |str| escape(str) }.join(' ')
+      # end
+
+      # here's an alternative implementation that would expose FOO=$FOO:
+      gem 'sh_vars', '~> 1.0.2'
+
+      def env
+        env = env_file.concat(super || [])
+        env = env.map { |str| ShVars.parse(str).to_h }.inject(&:merge) || {}
+        env.map { |key, value| "#{key}=#{value.inspect}" }.join(' ')
+      end
+
+      def env_file
+        return [] unless env_file?
+        error :env_file unless file?(super)
+        lines = read(super).split("\n").map(&:strip)
+        lines.reject(&:empty?)
+      end
+
+      def description
+        description? ? super : JSON.dump(
+          repo_slug: repo_slug,
+          git_commit_sha: git_sha,
+          git_commit_message: git_commit_msg,
+          git_commit_author: git_author_name,
+          git_tag: git_tag,
+          branch: git_branch,
+          travis_build_id: ENV['TRAVIS_BUILD_ID'],
+          travis_build_number: ENV['TRAVIS_BUILD_NUMBER'],
+          pull_request: ENV['TRAVIS_PULL_REQUEST']
+        )
+      end
+
+      def export
+        env_vars.each { |key, value| ENV[key.to_s] = value.to_s }
+      end
+
+      def env_vars
+        {
+          CONVOX_HOST: host,
+          CONVOX_PASSWORD: password,
+          CONVOX_APP: app,
+          CONVOX_RACK: rack,
+          CONVOX_CLI: 'convox'
+        }
+      end
+    end
+  end
+end