travis_dpl_test 2.0.3.beta.4.ror

data/lib/dpl/providers/opsworks.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+module Dpl
+  module Providers
+    class Opsworks < Provider
+      register :opsworks
+
+      status :stable
+
+      full_name 'AWS OpsWorks'
+
+      description sq(<<-STR)
+        tbd
+      STR
+
+      gem 'aws-sdk-opsworks', '~> 1.0'
+
+      env :aws, :opsworks
+      config '~/.aws/credentials', '~/.aws/config', prefix: 'aws'
+
+      opt '--access_key_id ID', 'AWS access key id', required: true, secret: true
+      opt '--secret_access_key KEY', 'AWS secret key', required: true, secret: true
+      opt '--app_id APP', 'The app id', required: true
+      opt '--region REGION', 'AWS region', default: 'us-east-1'
+      opt '--instance_ids ID', 'An instance id', type: :array
+      opt '--layer_ids ID', 'A layer id', type: :array
+      opt '--migrate', 'Migrate the database.'
+      opt '--wait_until_deployed', 'Wait until the app is deployed and return the deployment status.'
+      opt '--update_on_success', 'When wait-until-deployed and updated-on-success are both not given, application source is updated to the current SHA. Ignored when wait-until-deployed is not given.', alias: :update_app_on_success
+      opt '--custom_json JSON', 'Custom json options override (overwrites default configuration)'
+
+      msgs login: 'Using Access Key: %{access_key_id}',
+           create_deploy: 'Creating deployment ... ',
+           done: 'Done: %s',
+           waiting: 'Deploying ',
+           failed: 'Failed.',
+           success: 'Success.',
+           update_app: 'Updating application source branch/revision setting.',
+           app_not_found: 'App %s not found.',
+           timeout: 'Timeout: failed to finish deployment within 10 minutes.',
+           service_error: 'Deployment failed. OpsWorks service error: %s',
+           comment: 'Deploy build %{build_number} via Travis CI'
+
+      def login
+        info :login
+      end
+
+      def deploy
+        timeout(600) { create_deployment }
+      rescue Aws::Errors::ServiceError => e
+        error :service_error, e.message
+      end
+
+      def create_deployment
+        print :create_deploy
+        id = opsworks.create_deployment(deploy_config)[:deployment_id]
+        info :done, id
+        wait_until_deployed(id) if wait_until_deployed?
+      end
+
+      def deploy_config
+        compact(
+          stack_id:,
+          app_id:,
+          command: { name: 'deploy' },
+          comment:,
+          custom_json:,
+          instance_ids:,
+          layer_ids:
+        )
+      end
+
+      def wait_until_deployed(id)
+        print :waiting
+        depl = poll_deployment(id) while depl.nil? || depl[:status] == 'running'
+        error :failed if depl[:status] != 'successful'
+        info :success
+        update_app if update_on_success?
+      end
+
+      def poll_deployment(id)
+        print '.'
+        sleep 5
+        describe_deployments(id)[:deployments].first
+      end
+
+      def update_app
+        info :update_app
+        opsworks.update_app(update_config)
+      end
+
+      def update_config
+        {
+          app_id:,
+          app_source: {
+            revision: git_sha
+          }
+        }
+      end
+
+      def custom_json
+        super || { deploy: { app[:shortname] => { migrate: migrate?, scm: { revision: git_sha } } } }.to_json
+      end
+
+      def stack_id
+        app[:stack_id]
+      end
+
+      def app
+        @app ||= describe_app
+      end
+
+      def comment
+        interpolate(msg(:comment))
+      end
+
+      def build_number
+        super || sha
+      end
+
+      def describe_app
+        data = opsworks.describe_apps(app_ids: [app_id])
+        error :app_not_found, app_id unless data[:apps]&.any?
+        data[:apps].first
+      end
+
+      def describe_deployments(id)
+        opsworks.describe_deployments(deployment_ids: [id])
+      end
+
+      def opsworks
+        @opsworks ||= Aws::OpsWorks::Client.new(region:, credentials:)
+      end
+
+      def credentials
+        Aws::Credentials.new(access_key_id, secret_access_key)
+      end
+
+      def timeout(sec, &block)
+        Timeout.timeout(sec, &block)
+      rescue Timeout::Error
+        error :timeout
+      end
+    end
+  end
+end

data/lib/dpl/providers/packagecloud.rb_

@@ -0,0 +1,194 @@
+# talk to previous contributors about the logic in source_files
+#
+# the docs recommend to just `packagecloud push src target` https://packagecloud.io/docs#travis
+
+module Dpl
+  module Providers
+    class Packagecloud < Provider
+      register :packagecloud
+
+      status :alpha
+
+      description sq(<<-str)
+        tbd
+      str
+
+      gem 'packagecloud-ruby', '~> 1.0.8', require: 'packagecloud'
+
+      env :packagecloud
+
+      opt '--username USER', 'The packagecloud.io username.', required: true
+      opt '--token TOKEN', 'The packagecloud.io api token.', required: true, secret: true
+      opt '--repository REPO', 'The repository to push to.', required: true
+      opt '--local_dir DIR', 'The sub-directory of the built assets for deployment.', default: '.'
+      opt '--dist DIST', 'Required for debian, rpm, and node.js packages (use "node" for node.js packages). The complete list of supported strings can be found on the packagecloud.io docs.'
+      opt '--force', 'Whether package has to be (re)uploaded / deleted before upload'
+      opt '--connect_timeout SEC', type: :integer, default: 60
+      opt '--read_timeout SEC', type: :integer, default: 60
+      opt '--write_timeout SEC', type: :integer, default: 180
+      opt '--package_glob GLOB', type: :array, default: ['**/*']
+
+      # previous implementation checks against all of these in dist_required? but
+      # then the error message only lists rmp, deb, python, and dsc. i think the
+      # error message might be out of date? so is the option description in the
+      # readme (and thus above).
+      NEED_DIST = %w(rpm deb dsc whl egg egg-info gz zip tar bz2 z tgz)
+
+      msgs authenticate:       'Logging in to https://packagecloud.io with %{username}:%{token}',
+           timeouts:           'Timeouts: %{timeout_info}',
+           unauthenticated:    'Could not authenticate to https://packagecloud.io, please check the credentials',
+           supported_packages: 'Supported packages: %s',
+           source_packages:    'Source packages: %s',
+           delete_package:     'Deleting package: %s on %s',
+           push_package:       'Pushing package: %s to %s/%s',
+           source_fragment:    'Found source fragment: %s for %s',
+           missing_packages:   'No supported packages found',
+           missing_dist:       'Distribution needed for rpm, deb, python, and dsc packages (e.g. dist: ubuntu/breezy)',
+           unknown_dist:       'Failed to find distribution %{dist}',
+           error:              'Error: %s'
+
+      def install
+        @cwd = Dir.pwd
+        Dir.chdir(local_dir)
+      end
+
+      def login
+        info :authenticate
+        info :timeouts
+        client
+      rescue ::Packagecloud::UnauthenticatedException
+        error :unauthenticated
+      end
+
+      def timeout_info
+        to_pairs(timeouts)
+      end
+
+      def validate
+        error :missing_packages if paths.empty?
+        error :missing_dist if missing_dist?
+        info :supported_packages, paths.join(', ')
+        info :source_packages, source_paths.join(', ') if source_paths.any?
+      end
+
+      def deploy
+        packages.each do |package|
+          delete(package) if force?
+          push(package)
+        end
+      end
+
+      def finish
+        Dir.chdir(@cwd)
+      end
+
+      private
+
+        def delete(package)
+          info :delete_package, package.filename, dist
+          result = client.delete_package(repository, *dist.split('/'), package.filename)
+          error "Error #{result.response}" unless result.succeeded
+        end
+
+        def push(package)
+          info :push_package, package.filename, username, repository
+          args = [repository, package]
+          args << distro if dist_required?(package.filename)
+          result = client.put_package(*args)
+          error :error, result.response unless result.succeeded
+        end
+
+        def packages
+          paths.map { |path| package(path) }
+        end
+
+        def package(path)
+          opts = { file: path }
+          opts[:source_files] = source_files(path) if source?(path)
+          ::Packagecloud::Package.new(opts)
+        end
+
+        def paths
+          @paths ||= glob(package_glob).select { |path| supported?(path) }
+        end
+
+        def source_paths
+          paths.select { |path| source?(path) }
+        end
+
+        # I believe this resembles the logic in the previous version, but it seems
+        # very odd to me. Files are considered source files only if they are already
+        # part of the package (it looks up the package from the client, and extracts
+        # the file names), and if they are not located in sub directories (it uses
+        # File.basename to filter). Not sure that's how it's supposed to work?
+        def source_files(path)
+          files = contents(path)
+          paths = glob('**/*').select { |path| files.include?(path) }
+          paths = paths.map { |path| [File.basename(path), path] }
+          paths.each { |name, _| info(:source_fragement, name, path) }
+          paths.map { |name, path| [name, open(path)] }.to_h
+        end
+
+        def contents(path)
+          package = ::Packagecloud::Package.new(file: path)
+          result = client.package_contents(repository, package, distro)
+          error :error, result.response unless result.succeeded
+          result.response['files'].map { |file| file['filename'] }
+        end
+
+        def distro
+          @distro ||= client.find_distribution_id(dist) || error(:unknown_dist)
+        rescue ArgumentError => e
+          error :error, e.message
+        end
+
+        def supported?(path)
+          ::Packagecloud::SUPPORTED_EXTENSIONS.include?(ext(path))
+        end
+
+        def source?(path)
+          ext(path) == 'dsc'
+        end
+
+        def dist_required?(path)
+          NEED_DIST.include?(ext(path))
+        end
+
+        def package_glob
+          "{#{super.join(',')}}"
+        end
+
+        def timeouts
+          only(opts, :connect_timeout, :read_timeout, :write_timeout)
+        end
+
+        def ext(path)
+          File.extname(path).to_s.gsub('.','').downcase
+        end
+
+        def glob(glob)
+          Dir.glob(glob).reject { |path| File.directory?(path) }
+        end
+
+        def missing_dist?
+          !dist? && paths.any? { |path| dist_required?(path) }
+        end
+
+        def client
+          @client ||= ::Packagecloud::Client.new(credentials, "travis-ci dpl #{Dpl::VERSION}", connection)
+        end
+
+        def credentials
+          ::Packagecloud::Credentials.new(username, token)
+        end
+
+        def connection
+          ::Packagecloud::Connection.new('https', 'packagecloud.io', '443', timeouts)
+        end
+
+        def to_pairs(hash)
+          hash.map { |pair| pair.join('=') }.join(' ')
+        end
+    end
+  end
+end

data/lib/dpl/providers/pages/api.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+require 'timeout'
+
+module Dpl
+  module Providers
+    class Pages
+      class Api < Pages
+        register :'pages:api'
+
+        status :dev
+
+        PAGES_PREVIEW_MEDIA_TYPE = 'application/vnd.github.mister-fantastic-preview+json'
+
+        # suppress warnings about preview API
+        ENV['OCTOKIT_SILENT'] = 'true'
+
+        TIMEOUTS = {
+          timeout: 180,
+          open_timeout: 180
+        }.freeze
+
+        gem 'octokit', '~> 7'
+
+        full_name 'GitHub Pages (API)'
+
+        description sq(<<-STR)
+          This provider requests GitHub Pages build for the repository given by
+          the `--repo` flag, or the current one, if the flag is not given.
+          Note that `dpl` does not perform any check about the fitness of the request;
+          it is assumed that the target repository (and the branch that GitHub Pages is
+          configured to use) is ready for building.
+          For example, if your GitHub Pages is configured to use `gh-pages` but the
+          deployment is run on the `master` branch, you would have to ensure that the
+          `gh-pages` would be updated accordingly during the build.
+        STR
+
+        opt '--repo SLUG', 'GitHub repo slug', default: :repo_slug
+        opt '--token TOKEN', 'GitHub oauth token with repo permission', required: true, secret: true, alias: :github_token
+
+        msgs not_found: sq(<<-MSG),
+               GitHub Pages not found for %{slug}.
+               Either the given token has insufficient scope (repo or public_repo), or
+               GitHub Pages is not enabled for this repo (see https://github.com/%{slug}/settings)'
+        MSG
+             timeout: 'GitHub Pages build request timed out',
+             deploy: 'Requesting GitHub Pages build using API'
+
+        def validate
+          error :not_found unless pages_enabled?
+        end
+
+        def deploy
+          info :deploy
+
+          api.request_page_build slug
+
+          response = api.pages slug
+          logger.debug response
+
+          Timeout.timeout(30) do
+            until response.status == 'built'
+              response = api.pages slug
+              logger.debug response
+              sleep 1
+            end
+          end
+
+          latest_pages_build = api.latest_pages_build slug
+          if msg = latest_pages_build.error.message
+            error "Build failed: #{msg}"
+          end
+
+          info "Pages deployed to #{response.html_url}, using commit #{latest_pages_build.commit}"
+          logger.debug latest_pages_build
+        rescue Octokit::Forbidden => e
+          error e.message
+        rescue Timeout::Error
+          error :timeout
+        end
+
+        private
+
+        def slug
+          repo || repo_slug
+        end
+
+        def api
+          ::Octokit.default_media_type = PAGES_PREVIEW_MEDIA_TYPE unless @api
+
+          @api ||= Octokit::Client.new(**creds, auto_paginate: true, connection_options: { request: TIMEOUTS })
+        end
+
+        def creds
+          { access_token: token }
+        end
+
+        def pages_enabled?
+          api.pages slug
+        rescue Octokit::NotFound
+          false
+        end
+      end
+    end
+  end
+end

data/lib/dpl/providers/pages/git.rb

@@ -0,0 +1,262 @@
+# frozen_string_literal: true
+
+module Dpl
+  module Providers
+    class Pages
+      class Git < Pages
+        register :'pages:git'
+
+        status :stable
+
+        full_name 'GitHub Pages'
+
+        description sq(<<-STR)
+          tbd
+        STR
+
+        gem 'octokit', '~> 7'
+        gem 'public_suffix', '~> 5'
+
+        required :token, :deploy_key
+
+        opt '--repo SLUG', 'Repo slug', default: :repo_slug
+        opt '--token TOKEN', 'GitHub token with repo permission', secret: true, alias: :github_token
+        opt '--deploy_key PATH', 'Path to a file containing a private deploy key with write access to the repository', see: 'https://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys'
+        opt '--target_branch BRANCH', 'Branch to push force to', default: 'gh-pages'
+        opt '--keep_history', 'Create incremental commit instead of doing push force', default: true
+        opt '--commit_message MSG', default: 'Deploy %{project_name} to %{url}:%{target_branch}', interpolate: true
+        opt '--allow_empty_commit', 'Allow an empty commit to be created', requires: :keep_history
+        opt '--verbose', 'Be verbose about the deploy process'
+        opt '--local_dir DIR', 'Directory to push to GitHub Pages', default: '.'
+        opt '--fqdn FQDN', 'Write the given domain name to the CNAME file'
+        opt '--project_name NAME', 'Used in the commit message only (defaults to fqdn or the current repo slug)'
+        opt '--name NAME', 'Committer name', note: 'defaults to the current git commit author name'
+        opt '--email EMAIL', 'Committer email', note: 'defaults to the current git commit author email'
+        opt '--committer_from_gh', 'Use the token\'s owner name and email for the commit', requires: :token
+        opt '--deployment_file', 'Enable creation of a deployment-info file'
+        opt '--url URL', default: 'github.com', alias: :github_url
+
+        needs :git
+
+        msgs login: 'Authenticated as %s',
+             invalid_token: 'The provided GitHub token is invalid (error: %s)',
+             insufficient_scopes: 'Dpl does not have permission to access %{url} using the provided GitHub token. Please make sure the token have the repo or public_repo scope.',
+             setup_deploy_key: 'Moving deploy key %{deploy_key} to %{path}',
+             check_deploy_key: 'Checking deploy key',
+             deploy: 'Deploying branch %{target_branch} to %{url}',
+             keep_history: 'The deployment is configured to preserve the target branch if it exists on remote.',
+             work_dir: 'Using temporary work directory %{work_dir}',
+             committer_from_gh: 'The repo is configured to use committer user and email.',
+             setup_dir: 'The source dir for deployment is %s',
+             git_clone: 'Cloning the branch %{target_branch} from the remote repo',
+             git_init: 'Initializing local git repo',
+             git_checkout: 'Checking out orphan branch %{target_branch}',
+             copy_files: 'Copying %{src_dir} contents to %{work_dir}',
+             git_config: 'Configuring git committer to be %{name} <%{email}>',
+             prepare: 'Preparing to deploy %{target_branch} branch to gh-pages',
+             git_push: 'Pushing to %{url}',
+             stop: 'There are no changes to commit, stopping.'
+
+        cmds git_clone: 'git clone --quiet --branch="%{target_branch}" --depth=1 "%{remote_url}" . > /dev/null 2>&1',
+             git_init: 'git init .',
+             git_checkout: 'git checkout --orphan "%{target_branch}"',
+             check_deploy_key: 'ssh -i %{key} -T git@%{url} 2>&1 | grep successful > /dev/null',
+             copy_files: 'rsync -rl --exclude .git --delete "%{src_dir}/" .',
+             git_config_email: 'git config user.email %{quoted_email}',
+             git_config_name: 'git config user.name %{quoted_name}',
+             deployment_file: 'touch "deployed at %{now} by %{name}"',
+             cname: 'echo "%{fqdn}" > CNAME',
+             git_add: 'git add -A .',
+             git_commit_hook: 'cp %{path} .git/hooks/pre-commit',
+             git_commit: 'git commit %{git_commit_opts} -q %{git_commit_msg_opts}',
+             git_show: 'git show --stat-count=10 HEAD',
+             git_push: 'git push%{git_push_opts} --quiet "%{remote_url}" "%{target_branch}":"%{target_branch}" > /dev/null 2>&1'
+
+        errs copy_files: 'Failed to copy %{src_dir}.',
+             check_deploy_key: 'Failed to authenticate using the deploy key',
+             git_init: 'Failed to create new git repo',
+             git_checkout: 'Failed to create the orphan branch',
+             git_push: 'Failed to push the build to %{url}:%{target_branch}'
+
+        def login
+          token? ? login_token : setup_deploy_key
+        end
+
+        def setup
+          info :setup_dir, src_dir
+          info :committer_from_gh if committer_from_gh?
+          info :git_config
+        end
+
+        def prepare
+          info :deploy
+          info :keep_history if keep_history?
+          debug :work_dir
+          Dir.chdir(work_dir)
+        end
+
+        def deploy
+          git_clone? ? git_clone : git_init
+          copy_files
+          return info :stop if git_clone? && !git_dirty?
+
+          git_config
+          git_commit
+          git_push
+          git_status if verbose?
+        end
+
+        private
+
+        def login_token
+          user.login
+          info :login, user.login
+          error :insufficient_scopes unless sufficient_scopes?
+        rescue Octokit::Unauthorized => e
+          error :invalid_token, e.message
+        end
+
+        def setup_deploy_key
+          path = '~/.dpl/deploy_key'
+          info(:setup_deploy_key, path:)
+          mv deploy_key, path
+          chmod 0600, path
+          setup_git_ssh path
+          shell :check_deploy_key, key: path, url: opts[:url]
+        end
+
+        def git_clone?
+          keep_history? && git_branch_exists?
+        end
+
+        def git_clone
+          shell :git_clone, echo: false
+        end
+
+        def git_init
+          shell :git_init
+          shell :git_checkout
+        end
+
+        def copy_files
+          shell :copy_files
+        end
+
+        def git_config
+          shell :git_config_name, echo: false
+          shell :git_config_email, echo: false
+        end
+
+        def git_commit
+          info :prepare
+          shell :git_commit_hook, path: asset(:git, :detect_private_key).path, echo: false if deploy_key?
+          shell :deployment_file if deployment_file?
+          shell :cname if fqdn?
+          shell :git_add
+          shell :git_commit
+          shell :git_show
+        end
+
+        def git_push
+          shell :git_push, echo: false
+        end
+
+        def git_status
+          shell 'git status'
+        end
+
+        def git_branch_exists?
+          git_ls_remote?(remote_url, target_branch)
+        end
+
+        def git_commit_opts
+          ' --allow-empty' if allow_empty_commit?
+        end
+
+        def git_commit_msg_opts
+          msg = interpolate(commit_message, vars:)
+          msg.split("\n").reject(&:empty?).map { |message| %(-m #{quote(message)}) }
+        end
+
+        def git_push_opts
+          ' --force' unless keep_history?
+        end
+
+        def name
+          str = super if name?
+          str ||= user.name if committer_from_gh?
+          str ||= git_author_name
+          str = "#{str} (via Travis CI)" if ENV['TRAVIS'] && !name?
+          str
+        end
+        memoize :name
+
+        def email
+          str = super if email?
+          str ||= user.email if committer_from_gh?
+          str || git_author_email
+        end
+        memoize :email
+
+        def project_name
+          super || fqdn || repo_slug
+        end
+
+        def sufficient_scopes?
+          api.scopes.include?('public_repo') || api.scopes.include?('repo')
+        end
+
+        def remote_url
+          token? ? https_url_with_token : git_url
+        end
+
+        def https_url_with_token
+          "https://#{token}@#{url}"
+        end
+
+        def git_url
+          "git@#{opts[:url]}:#{slug}.git"
+        end
+
+        def url
+          "#{opts[:url]}/#{slug}.git"
+        end
+
+        def slug
+          repo || repo_slug
+        end
+
+        def user
+          @user ||= api.user
+        rescue StandardError => e
+          puts "ERR: #{e.inspect}"
+          puts e.backtrace
+        end
+
+        def src_dir
+          @src_dir ||= expand(local_dir)
+        end
+
+        def work_dir
+          @work_dir ||= tmp_dir
+        end
+
+        def api
+          @api ||= Octokit::Client.new(access_token: token, api_endpoint:)
+        end
+
+        def api_endpoint
+          opts[:url] == 'github.com' ? 'https://api.github.com/' : "https://#{opts[:url]}/api/v3/"
+        end
+
+        def now
+          Time.now
+        end
+
+        def debug(*args)
+          info(*args) if verbose?
+        end
+      end
+    end
+  end
+end