dpl 2.0.0.alpha.12 → 2.0.2.beta.1

data/lib/dpl/assets/dpl/README.erb.md

@@ -1,7 +1,7 @@
 # Dpl [![Build Status](https://travis-ci.com/travis-ci/dpl.svg?branch=master)](https://travis-ci.com/travis-ci/dpl) [![Code Climate](https://codeclimate.com/github/travis-ci/dpl.svg)](https://codeclimate.com/github/travis-ci/dpl) [![Coverage Status](https://coveralls.io/repos/travis-ci/dpl/badge.svg?branch=master&service=github&cache=2019-08-09_17:00)](https://coveralls.io/github/travis-ci/dpl?branch=master) [![Gem Version](https://img.shields.io/gem/v/dpl)](http://rubygems.org/gems/dpl) [![Yard Docs](http://img.shields.io/badge/yard-docs-blue.svg)](http://rubydoc.info/github/travis-ci/dpl)
 
 This version of the README documents dpl v2, the next major version of dpl.
-The REAMDE for dpl v1, the version that is currently used in production on
+The README for dpl v1, the version that is currently used in production on
 Travis CI can be found [here](https://github.com/travis-ci/dpl/blob/v1/README.md).
 
 Dpl is command line tool for deploying code, html, packages, or build artifacts

data/lib/dpl/ctx/test.rb

@@ -36,10 +36,13 @@ module Dpl
       end
 
       def gems_require(gems)
-        gems.each { |gem| gem_require(gem) }
+        gems.each { |gem| gem_require(*gem) }
       end
 
       def gem_require(name, version = nil, opts = {})
+        # not sure why this is needed. bundler should take care of this, but
+        # it does not for octokit for whatever reason
+        require opts[:require] || name rescue nil
         cmds << "[gem:require] #{name} (#{version}, #{opts})"
       end
 

data/lib/dpl/helper/interpolate.rb

@@ -52,6 +52,14 @@ module Dpl
       Interpolator.new(str, self, args || {}, opts).apply
     end
 
+    # Interpolation variables as declared by the provider.
+    #
+    # By default this contains string option names, but additional
+    # methods can be added using Provider::Dsl#vars.
+    def vars
+      self.class.vars
+    end
+
     # Obfuscates the given string.
     #
     # Replaces all but the first N characters with asterisks, and paddes
@@ -71,6 +79,7 @@ module Dpl
       PATTERN  = /%\{(\$?[\w]+)\}/
       ENV_VAR  = /^\$[A-Z_]+$/
       UPCASE   = /^[A-Z_]+$/
+      UNKNOWN  = '[unknown variable: %s]'
 
       def apply
         str = interpolate(self.str.to_s)
@@ -102,7 +111,9 @@ module Dpl
       end
 
       def lookup(key)
-        if mod = modifier(key)
+        if vars? && !var?(key)
+          UNKNOWN % key
+        elsif mod = modifier(key)
           key = key.to_s.sub("#{mod}d_", '')
           obj.send(mod, lookup(key))
         elsif key.to_s =~ ENV_VAR
@@ -121,6 +132,18 @@ module Dpl
       def modifier(key)
         MODIFIER.detect { |mod| key.to_s.start_with?("#{mod}d_") }
       end
+
+      def var?(key)
+        vars.include?(key)
+      end
+
+      def vars
+        opts[:vars]
+      end
+
+      def vars?
+        !!vars
+      end
     end
   end
 end

data/lib/dpl/provider.rb

@@ -145,8 +145,6 @@ module Dpl
 
     abstract
 
-    arg :provider, 'The provider name', required: true
-
     opt '--cleanup',      'Clean up build artifacts from the Git working directory before the deployment', negate: %w(skip)
     opt '--run CMD',      'Commands to execute after the deployment finished successfully', type: :array
     opt '--stage NAME',   'Execute the given stage(s) only', type: :array, internal: true, default: STAGES
@@ -154,6 +152,16 @@ module Dpl
     opt '--fold',         'Wrap log output in folds', internal: true
     opt '--edge',         internal: true
 
+    vars *%i(
+      git_author_email
+      git_author_name
+      git_branch
+      git_commit_author
+      git_commit_msg
+      git_sha
+      git_tag
+    )
+
     msgs before_install:  'Installing deployment dependencies',
          before_setup:    'Setting the build environment up for the deployment',
          setup_git_ssh:   'Setting up git-ssh',
@@ -545,7 +553,7 @@ module Dpl
 
     # Double quotes the given string.
     def quote(str)
-      %("#{str.gsub('"', '\"')}")
+      %("#{str.to_s.gsub('"', '\"')}")
     end
 
     # Outdents the given string.

data/lib/dpl/provider/dsl.rb

@@ -2,6 +2,15 @@ require 'dpl/helper/squiggle'
 require 'dpl/helper/wrap'
 require 'dpl/provider/status'
 
+# TODO figure out how to allow adding domain specific behavior like this to Cl
+class Cl::Opt
+  OPTS << :interpolate
+
+  def interpolate?
+    opts[:interpolate]
+  end
+end
+
 module Dpl
   class Provider < Cl::Cmd
     # DSL available on the provider's class body.
@@ -36,6 +45,22 @@ module Dpl
         status ? @status = Status.new(self, status, msg) : @status
       end
 
+      # Declare additional variables available for interpolation.
+      #
+      # Interpolating strings, when these exposed to the user, should safelist
+      # which variables are available. Options declared on a provider are
+      # always available, except if they are flags, arrays, internal, or
+      # secrets. This method can be used to allow additional variables, e.g.
+      # from the git context.
+      def vars(*vars)
+        return self.vars.concat(vars) if vars.any?
+        return @vars if instance_variable_defined?(:@vars)
+        vars = superclass.respond_to?(:vars) ? superclass.vars : []
+        reject = %i(flag array internal interpolate secret)
+        opts = reject.inject(self.opts) { |opts, attr| opts.reject(&:"#{attr}?") }
+        @vars = vars.dup.concat(opts.map(&:name)).uniq.sort - [:strategy]
+      end
+
       # @!method env
       # Declare an environment variable prefix to accept env vars as options
       #

data/lib/dpl/providers.rb

@@ -11,12 +11,15 @@ require 'dpl/providers/cloudfiles'
 require 'dpl/providers/cloudformation'
 require 'dpl/providers/cloudfoundry'
 require 'dpl/providers/codedeploy'
+require 'dpl/providers/ecr'
 require 'dpl/providers/convox'
 require 'dpl/providers/elasticbeanstalk'
 require 'dpl/providers/engineyard'
 require 'dpl/providers/firebase'
+require 'dpl/providers/flynn'
 require 'dpl/providers/gae'
 require 'dpl/providers/gcs'
+require 'dpl/providers/git_push'
 require 'dpl/providers/gleis'
 require 'dpl/providers/hackage'
 require 'dpl/providers/hephy'
@@ -25,6 +28,7 @@ require 'dpl/providers/lambda'
 require 'dpl/providers/launchpad'
 require 'dpl/providers/netlify'
 require 'dpl/providers/npm'
+require 'dpl/providers/nuget'
 require 'dpl/providers/openshift'
 require 'dpl/providers/opsworks'
 require 'dpl/providers/packagecloud'

data/lib/dpl/providers/anynines.rb

@@ -1,6 +1,8 @@
 module Dpl
   module Providers
     class Anynines < Provider
+      register :anynines
+
       status :alpha
 
       description sq(<<-str)

data/lib/dpl/providers/azure_web_apps.rb

@@ -1,6 +1,8 @@
 module Dpl
   module Providers
     class AzureWebApps < Provider
+      register :azure_web_apps
+
       status :alpha
 
       full_name 'Azure Web Apps'

data/lib/dpl/providers/bintray.rb

@@ -5,6 +5,8 @@ require 'find'
 module Dpl
   module Providers
     class Bintray < Provider
+      register :bintray
+
       status :stable
 
       description sq(<<-str)

data/lib/dpl/providers/bluemixcloudfoundry.rb

@@ -1,7 +1,9 @@
 module Dpl
   module Providers
     class Bluemixcloudfoundry < Provider
-      status :alpha
+      register :bluemixcloudfoundry
+
+      status :stable
 
       full_name 'Bluemix Cloud Foundry'
 

data/lib/dpl/providers/boxfuse.rb

@@ -1,6 +1,8 @@
 module Dpl
   module Providers
     class Boxfuse < Provider
+      register :boxfuse
+
       status :alpha
 
       description sq(<<-str)

data/lib/dpl/providers/cargo.rb

@@ -1,7 +1,9 @@
 module Dpl
   module Providers
     class Cargo < Provider
-      status :beta
+      register :cargo
+
+      status :stable
 
       description sq(<<-str)
         tbd

data/lib/dpl/providers/cloud66.rb

@@ -1,6 +1,8 @@
 module Dpl
   module Providers
     class Cloud66 < Provider
+      register :cloud66
+
       status :alpha
 
       description sq(<<-str)

data/lib/dpl/providers/cloudfiles.rb

@@ -1,6 +1,8 @@
 module Dpl
   module Providers
     class Cloudfiles < Provider
+      register :cloudfiles
+
       status :alpha
 
       full_name 'Cloud Files'

data/lib/dpl/providers/cloudformation.rb

@@ -1,6 +1,7 @@
 module Dpl
   module Providers
     class Cloudformation < Provider
+      register :cloudformation
       status :stable
 
       full_name 'AWS CloudFormation'

data/lib/dpl/providers/cloudfoundry.rb

@@ -1,6 +1,8 @@
 module Dpl
   module Providers
     class Cloudfoundry < Provider
+      register :cloudfoundry
+
       status :stable
 
       full_name 'Cloud Foundry'
@@ -20,10 +22,11 @@ module Dpl
       opt '--buildpack PACK',      'Buildpack name or Git URL'
       opt '--manifest FILE',       'Path to the manifest'
       opt '--skip_ssl_validation', 'Skip SSL validation'
+      opt '--deployment_strategy STRATEGY', 'Deployment strategy, either rolling or null'
       opt '--v3',                  'Use the v3 API version to push the application'
       opt '--logout', default: true, internal: true
 
-      cmds install: 'test $(uname) = "Linux" && rel="linux64-binary" || rel="macosx64"; wget "https://cli.run.pivotal.io/stable?release=${rel}&source=github" -qO cf.tgz && tar -zxvf cf.tgz && rm cf.tgz',
+      cmds install: 'test $(uname) = "Linux" && rel="linux64-binary" || rel="macosx64"; wget "https://cli.run.pivotal.io/stable?release=${rel}&version=v7&source=github" -qO cf.tgz && tar -zxvf cf.tgz && rm cf.tgz',
            api:     './cf api %{api} %{skip_ssl_validation_opt}',
            login:   './cf login -u %{username} -p %{password} -o %{organization} -s %{space}',
            push:    './cf %{push_cmd} %{push_args}',
@@ -68,6 +71,7 @@ module Dpl
           args = []
           args << quote(app_name)  if app_name?
           args << "-f #{manifest}" if manifest?
+          args << "--strategy #{deployment_strategy}" if deployment_strategy?
           args.join(' ')
         end
 

data/lib/dpl/providers/codedeploy.rb

@@ -3,7 +3,9 @@ module Dpl
     # split this up to CodeDeploy::Github and CodeDeploy::S3 using the
     # revision_type, in order to make opts more strict
     class Codedeploy < Provider
-      status :alpha
+      register :codedeploy
+
+      status :stable
 
       full_name 'AWS Code Deploy'
 
@@ -30,7 +32,7 @@ module Dpl
       opt '--wait_until_deployed', 'Wait until the deployment has finished'
       opt '--bundle_type TYPE', 'Bundle type of the revision'
       opt '--key KEY', 'S3 bucket key of the revision'
-      opt '--description DESCR', 'Description of the revision'
+      opt '--description DESCR', 'Description of the revision', interpolate: true
       opt '--endpoint ENDPOINT', 'S3 endpoint url'
 
       msgs login:                 'Using Access Key: %{access_key_id}',
@@ -44,6 +46,8 @@ module Dpl
            unknown_revision_type: 'Unknown revision type %p',
            unknown_bundle_type:   'Unknown bundle type'
 
+      vars :build_number
+
       def login
         info :login
       end
@@ -80,7 +84,12 @@ module Dpl
       def wait_until_deployed(id)
         print :waiting_for_deploy
         status = poll(id) until %w(Succeeded Failed Stopped).include?(status)
-        info :finished_deploy, status
+        case status
+        when 'Succeeded'
+          info :finished_deploy, status
+        else
+          error :finished_deploy, status
+        end
       end
 
       def poll(id)
@@ -152,7 +161,7 @@ module Dpl
       end
 
       def description
-        super || interpolate(msg(:description))
+        interpolate(super || msg(:description), vars: vars)
       end
 
       def build_number

data/lib/dpl/providers/convox.rb

@@ -1,6 +1,8 @@
 module Dpl
   module Providers
     class Convox < Provider
+      register :convox
+
       status :dev
 
       description sq(<<-str)

data/lib/dpl/providers/datica.rb

@@ -1,6 +1,8 @@
 module Dpl
   module Providers
     class Datica < Provider
+      register :datica
+
       status :dev
 
       register :datica, :catalyze

data/lib/dpl/providers/ecr.rb

@@ -0,0 +1,127 @@
+module Dpl
+  module Providers
+    class Ecr < Provider
+      status :alpha
+
+      full_name 'AWS ECR'
+
+      description sq(<<-str)
+        tbd
+      str
+
+      gem 'aws-sdk-ecr', '~> 1.0'
+      # gem 'docker-api', '~> 1.34'
+      gem 'json'
+
+      env :aws
+
+      opt '--access_key_id ID', 'AWS access key', required: true, secret: true
+      opt '--secret_access_key KEY', 'AWS secret access key', required: true, secret: true
+      opt '--account_id ID', 'AWS Account ID', note: 'Required if the repository is owned by a different account than the IAM user'
+      opt '--source SOURCE', 'Image to push', note: 'can be the id or the name and optional tag (e.g. mysql:5.6)', required: true
+      opt '--target TARGET', 'Comma separated list of partial repository names to push to', eg: 'image-one:tag,image-two', required: true
+      opt '--region REGION', 'Comma separated list of regions to push to', default: 'us-east-1'
+
+      msgs login:        'Using Access Key: %{access_key_id}',
+           auth_region:  'Authenticated with %{url}',
+           deploy:       'Pushing image %{source} to regions %{regions} as %{targets}',
+           image_pushed: 'Pushed image %{source} to region %{region} as %{target}'
+
+      cmds login: 'docker login -u %{user} -p %{pass} %{url}',
+           tag:   'docker tag %{source} %{url}/%{repo}:%{tag}',
+           push:  'docker push %{url}/%{repo}'
+
+      errs unknown_image: 'Image %{source} not found in the local Docker repository'
+
+      attr_reader :endpoints
+
+      def login
+        info :login
+        auth_regions
+      end
+
+      def validate
+        # TODO validate the image exists locally
+      end
+
+      def deploy
+        info :deploy, regions: regions.join(', '), targets: targets.join(', ')
+        regions.product(targets).each do |region, target|
+          push(region, target)
+        end
+      end
+
+      private
+
+        def push(region, target)
+          url, repo, tag = endpoints[region], *target.split(':')
+          shell :tag, url: url, repo: repo, tag: tag || 'latest'
+          shell :push, url: url, repo: repo
+          info :image_pushed, region: region, target: target
+        end
+
+        def auth_regions
+          @endpoints = regions.map { |region| [region, auth_region(region)] }.to_h
+        end
+
+        def auth_region(region)
+          token = auth_token(region)
+          user, pass = parse_auth(token.authorization_token)
+          url = token.proxy_endpoint
+          shell :login, user: user, pass: pass, url: url, echo: false, silent: true
+          info :auth_region, url: url
+          strip_protocol(url)
+        end
+
+        def auth_token(region)
+          ecr(region).get_authorization_token(registry_ids).authorization_data[0]
+        end
+
+        def registry_ids
+          account_id? ? { registry_ids: [account_id] } : {}
+        end
+
+        def regions
+          # not sure how this was meant to be normalized when being a YAML list
+          region.split(',')
+        end
+
+        def targets
+          # not sure how this was meant to be normalized when being a YAML list
+          target.split(',')
+        end
+
+        def creds
+          @creds ||= only(opts, :access_key_id, :secret_access_key)
+        end
+
+        def ecr(region)
+          Aws::ECR::Client.new(region: region, **creds)
+        end
+
+        def parse_auth(str)
+          user, pass = Base64.decode64(str).split(':')
+          [user, pass.chomp]
+        end
+
+        def strip_protocol(url)
+          url.sub(/^https?:\/\//, '')
+        end
+
+        def progress(events)
+          events.split("\r\n").each do |event|
+            event = JSON.parse(event)
+            if e = event['error']
+              error e
+            elsif %w(Preparing Pushing).include?(event['status'])
+              nil
+            elsif event['id']
+              info "#{event['status']} [#{event['id']}]"
+            elsif event['status']
+              info event['status']
+            end
+          end
+        end
+    end
+  end
+end