dpl 1.10.17.travis.6637.6 → 2.0.0.alpha.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/CHANGELOG.md +74 -0
- data/CONTRIBUTING.md +392 -0
- data/Gemfile +17 -3
- data/Gemfile.lock +373 -0
- data/LICENSE +16 -19
- data/NOTES.md +275 -0
- data/README.md +1977 -707
- data/Rakefile +2 -2
- data/bin/dpl +7 -3
- data/lib/dpl.rb +20 -0
- data/lib/dpl/assets/atlas/install +19 -0
- data/lib/dpl/assets/dpl/README.erb.md +133 -0
- data/lib/dpl/assets/dpl/git_ssh +2 -0
- data/lib/dpl/assets/git/detect_private_key +8 -0
- data/lib/dpl/assets/hephy/filter_log +3 -0
- data/lib/dpl/assets/pypi/install +4 -0
- data/lib/dpl/assets/scalingo/install +6 -0
- data/lib/dpl/cli.rb +36 -48
- data/lib/dpl/ctx.rb +2 -0
- data/lib/dpl/ctx/bash.rb +543 -0
- data/lib/dpl/ctx/test.rb +242 -0
- data/lib/dpl/helper/assets.rb +36 -0
- data/lib/dpl/helper/cmd.rb +167 -0
- data/lib/dpl/helper/config_file.rb +47 -0
- data/lib/dpl/helper/env.rb +39 -0
- data/lib/dpl/helper/interpolate.rb +126 -0
- data/lib/dpl/helper/memoize.rb +20 -0
- data/lib/dpl/helper/squiggle.rb +22 -0
- data/lib/dpl/helper/zip.rb +69 -0
- data/lib/dpl/provider.rb +562 -234
- data/lib/dpl/provider/dsl.rb +369 -0
- data/lib/dpl/provider/examples.rb +128 -0
- data/lib/dpl/provider/status.rb +59 -0
- data/lib/dpl/providers.rb +40 -0
- data/lib/dpl/providers/anynines.rb +65 -0
- data/lib/dpl/providers/atlas.rb +49 -0
- data/lib/dpl/providers/azure_web_apps.rb +59 -0
- data/lib/dpl/providers/bintray.rb +313 -0
- data/lib/dpl/providers/bluemixcloudfoundry.rb +92 -0
- data/lib/dpl/providers/boxfuse.rb +48 -0
- data/lib/dpl/providers/cargo.rb +19 -0
- data/lib/dpl/providers/chef_supermarket.rb +128 -0
- data/lib/dpl/providers/cloud66.rb +40 -0
- data/lib/dpl/providers/cloudfiles.rb +56 -0
- data/lib/dpl/providers/cloudfoundry.rb +81 -0
- data/lib/dpl/providers/codedeploy.rb +179 -0
- data/lib/dpl/providers/datica.rb +60 -0
- data/lib/dpl/providers/elasticbeanstalk.rb +195 -0
- data/lib/dpl/providers/engineyard.rb +107 -0
- data/lib/dpl/providers/firebase.rb +41 -0
- data/lib/dpl/providers/gae.rb +74 -0
- data/lib/dpl/providers/gcs.rb +105 -0
- data/lib/dpl/providers/hackage.rb +47 -0
- data/lib/dpl/providers/hephy.rb +101 -0
- data/lib/dpl/providers/heroku.rb +111 -0
- data/lib/dpl/providers/heroku/api.rb +119 -0
- data/lib/dpl/providers/heroku/git.rb +50 -0
- data/lib/dpl/providers/lambda.rb +202 -0
- data/lib/dpl/providers/launchpad.rb +74 -0
- data/lib/dpl/providers/netlify.rb +30 -0
- data/lib/dpl/providers/npm.rb +88 -0
- data/lib/dpl/providers/openshift.rb +46 -0
- data/lib/dpl/providers/opsworks.rb +142 -0
- data/lib/dpl/providers/packagecloud.rb +190 -0
- data/lib/dpl/providers/pages.rb +17 -0
- data/lib/dpl/providers/pages/api.rb +102 -0
- data/lib/dpl/providers/pages/git.rb +251 -0
- data/lib/dpl/providers/puppetforge.rb +44 -0
- data/lib/dpl/providers/pypi.rb +120 -0
- data/lib/dpl/providers/releases.rb +214 -0
- data/lib/dpl/providers/rubygems.rb +89 -0
- data/lib/dpl/providers/s3.rb +243 -0
- data/lib/dpl/providers/scalingo.rb +63 -0
- data/lib/dpl/providers/script.rb +28 -0
- data/lib/dpl/providers/snap.rb +59 -0
- data/lib/dpl/providers/surge.rb +55 -0
- data/lib/dpl/providers/testfairy.rb +93 -0
- data/lib/dpl/providers/transifex.rb +66 -0
- data/lib/dpl/support/aws_sdk_patch.rb +23 -0
- data/lib/dpl/support/gems.rb +69 -0
- data/lib/dpl/support/gstore_patch.rb +6 -0
- data/lib/dpl/support/version.rb +83 -0
- data/lib/dpl/version.rb +2 -2
- metadata +98 -169
- data/.coveralls.yml +0 -1
- data/.github/CONTRIBUTING.md +0 -173
- data/.github/stale.yml +0 -53
- data/.gitignore +0 -13
- data/.rspec +0 -2
- data/.travis.yml +0 -56
- data/dpl-anynines.gemspec +0 -3
- data/dpl-atlas.gemspec +0 -3
- data/dpl-azure_webapps.gemspec +0 -3
- data/dpl-bintray.gemspec +0 -3
- data/dpl-bitballoon.gemspec +0 -3
- data/dpl-bluemix_cloud_foundry.gemspec +0 -3
- data/dpl-boxfuse.gemspec +0 -3
- data/dpl-cargo.gemspec +0 -3
- data/dpl-catalyze.gemspec +0 -3
- data/dpl-chef_supermarket.gemspec +0 -20
- data/dpl-cloud66.gemspec +0 -3
- data/dpl-cloud_files.gemspec +0 -3
- data/dpl-cloud_foundry.gemspec +0 -3
- data/dpl-code_deploy.gemspec +0 -3
- data/dpl-deis.gemspec +0 -3
- data/dpl-elastic_beanstalk.gemspec +0 -3
- data/dpl-engine_yard.gemspec +0 -3
- data/dpl-firebase.gemspec +0 -3
- data/dpl-gae.gemspec +0 -3
- data/dpl-gcs.gemspec +0 -3
- data/dpl-hackage.gemspec +0 -3
- data/dpl-hephy.gemspec +0 -3
- data/dpl-heroku.gemspec +0 -3
- data/dpl-lambda.gemspec +0 -3
- data/dpl-launchpad.gemspec +0 -3
- data/dpl-npm.gemspec +0 -3
- data/dpl-openshift.gemspec +0 -3
- data/dpl-ops_works.gemspec +0 -3
- data/dpl-packagecloud.gemspec +0 -3
- data/dpl-pages.gemspec +0 -3
- data/dpl-puppet_forge.gemspec +0 -3
- data/dpl-pypi.gemspec +0 -3
- data/dpl-releases.gemspec +0 -8
- data/dpl-rubygems.gemspec +0 -3
- data/dpl-s3.gemspec +0 -3
- data/dpl-scalingo.gemspec +0 -3
- data/dpl-script.gemspec +0 -3
- data/dpl-snap.gemspec +0 -3
- data/dpl-surge.gemspec +0 -3
- data/dpl-testfairy.gemspec +0 -3
- data/dpl-transifex.gemspec +0 -3
- data/dpl.gemspec +0 -3
- data/gemspec_helper.rb +0 -51
- data/lib/dpl/error.rb +0 -3
- data/notes/engine_yard.md +0 -1
- data/notes/heroku.md +0 -3
- data/spec/cli_spec.rb +0 -36
- data/spec/provider_spec.rb +0 -191
- data/spec/spec_helper.rb +0 -20
data/Rakefile
CHANGED
@@ -43,7 +43,7 @@ end
|
|
43
43
|
|
44
44
|
gemspecs = FileList[File.join(top, "dpl-*.gemspec")]
|
45
45
|
|
46
|
-
providers = gemspecs.map { |f|
|
46
|
+
providers = gemspecs.map { |f| /dpl-(?<provider>.*)\.gemspec/ =~ f && provider }
|
47
47
|
|
48
48
|
desc "Build dpl gem"
|
49
49
|
file "dpl-#{gem_version}.gem" do
|
@@ -184,7 +184,7 @@ providers.each do |provider|
|
|
184
184
|
logger.info green("Installing dpl-#{provider} gem")
|
185
185
|
sh "gem install --no-post-install-message dpl-#{provider}-#{gem_version}.gem"
|
186
186
|
logger.info green("Testing dpl-#{provider} loads correctly")
|
187
|
-
ruby "-S dpl --provider=#{provider} --
|
187
|
+
ruby "-S dpl --provider=#{provider} --stage=init"
|
188
188
|
end
|
189
189
|
|
190
190
|
desc "Uninstall dpl-#{provider}"
|
data/bin/dpl
CHANGED
@@ -1,5 +1,9 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
|
-
|
2
|
+
$: << File.expand_path('../../lib', __FILE__)
|
3
3
|
|
4
|
-
|
5
|
-
|
4
|
+
$stdout.sync = true
|
5
|
+
$stderr.sync = true
|
6
|
+
|
7
|
+
require 'dpl'
|
8
|
+
|
9
|
+
Dpl::Cli.new.run(ARGV)
|
data/lib/dpl.rb
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'dpl/cli'
|
2
|
+
require 'dpl/ctx'
|
3
|
+
require 'dpl/provider'
|
4
|
+
require 'dpl/version'
|
5
|
+
|
6
|
+
module Dpl
|
7
|
+
class Error < StandardError
|
8
|
+
attr_reader :opts
|
9
|
+
|
10
|
+
def initialize(msg, opts = {})
|
11
|
+
super(msg)
|
12
|
+
@opts = opts
|
13
|
+
set_backtrace(opts[:backtrace]) if backtrace?
|
14
|
+
end
|
15
|
+
|
16
|
+
def backtrace?
|
17
|
+
!!opts[:backtrace]
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
if ! command -v atlas-upload &>/dev/null ; then
|
2
|
+
mkdir -p $HOME/bin $HOME/gopath/src
|
3
|
+
export PATH="$HOME/bin:$PATH"
|
4
|
+
|
5
|
+
if ! command -v gimme &>/dev/null ; then
|
6
|
+
curl -sL -o $HOME/bin/gimme https://raw.githubusercontent.com/meatballhat/gimme/master/gimme
|
7
|
+
chmod +x $HOME/bin/gimme
|
8
|
+
fi
|
9
|
+
|
10
|
+
if [ -z $GOPATH ]; then
|
11
|
+
export GOPATH="$HOME/gopath"
|
12
|
+
else
|
13
|
+
export GOPATH="$HOME/gopath:$GOPATH"
|
14
|
+
fi
|
15
|
+
eval "$(gimme 1.6)" &> /dev/null
|
16
|
+
|
17
|
+
go get github.com/hashicorp/atlas-upload-cli
|
18
|
+
cp $HOME/gopath/bin/atlas-upload-cli $HOME/bin/atlas-upload
|
19
|
+
fi
|
@@ -0,0 +1,133 @@
|
|
1
|
+
# Dpl [![Build Status](https://travis-ci.com/travis-ci/dpl.svg?branch=master)](https://travis-ci.com/travis-ci/dpl) [![Code Climate](https://codeclimate.com/github/travis-ci/dpl.png)](https://codeclimate.com/github/travis-ci/dpl) [![Coverage Status](https://coveralls.io/repos/travis-ci/dpl/badge.svg?branch=master&service=github&cache=2019-08-09_17:00)](https://coveralls.io/github/travis-ci/dpl?branch=master) [![Gem Version](https://img.shields.io/gem/v/dpl)](http://rubygems.org/gems/dpl) [![Yard Docs](http://img.shields.io/badge/yard-docs-blue.svg)](http://rubydoc.info/github/travis-ci/dpl)
|
2
|
+
|
3
|
+
This version of the README documents dpl v2, the next major version of dpl.
|
4
|
+
The REAMDE for dpl v1, the version that is currently used in production on
|
5
|
+
Travis CI can be found [here](https://github.com/travis-ci/dpl/blob/v1/README.md).
|
6
|
+
|
7
|
+
Dpl is command line tool for deploying code, html, packages, or build artifacts
|
8
|
+
to various service providers.
|
9
|
+
|
10
|
+
It is tightly integrated into Travis CI's [deployment integration](https://docs.travis-ci.com/user/deployment),
|
11
|
+
but also used, and recommended by others, such as [GitLab](https://docs.gitlab.com/ee/ci/examples/deployment/).
|
12
|
+
|
13
|
+
It is maintained by Travis CI, largely community driven, and it has existed
|
14
|
+
since 2013. If you find support your preferred deployment target missing,
|
15
|
+
please do not hesitate to get in touch, and we'll help you [add it](#contributing-to-dpl).
|
16
|
+
|
17
|
+
## Table of Contents
|
18
|
+
|
19
|
+
* [Requirements](#requirements)
|
20
|
+
* [Installation](#installation)
|
21
|
+
* [Usage](#usage)
|
22
|
+
* [Maturity Levels](#maturity-levels)
|
23
|
+
* [Supported Providers](#supported-providers)
|
24
|
+
* [Contributing to Dpl](#contributing-to-dpl)
|
25
|
+
* [Old Issues](#old-issues)
|
26
|
+
* [Code of Conduct](#code-of-conduct)
|
27
|
+
* [License](#license)
|
28
|
+
* [Credits](#credits)
|
29
|
+
|
30
|
+
## Requirements
|
31
|
+
|
32
|
+
Dpl requires Ruby 2.2 or later.
|
33
|
+
|
34
|
+
Depending on the deployment target dpl might require additional runtimes (e.g.
|
35
|
+
Go, Node.js, or Python) to be installed. It also might require sudo access in
|
36
|
+
order to install a Debian package.
|
37
|
+
|
38
|
+
Dpl is generally optimized for usage on Linux systems.
|
39
|
+
|
40
|
+
## Installation
|
41
|
+
|
42
|
+
Installation:
|
43
|
+
|
44
|
+
```
|
45
|
+
gem install dpl
|
46
|
+
```
|
47
|
+
|
48
|
+
## Usage
|
49
|
+
|
50
|
+
Dpl is meant and optimized for usage in ephemeral build environments, such
|
51
|
+
as Travis CI, or any other CI/CD pipeline.
|
52
|
+
|
53
|
+
Dpl is integrated to Travis CI's build configuration and build script compilation
|
54
|
+
tooling, so all you need to do is add the proper configuration to your `.travis.yml`
|
55
|
+
file. Please refer to [the documentation](https://docs.travis-ci.com/user/deployment)
|
56
|
+
for details.
|
57
|
+
|
58
|
+
For usage outside of Travis CI dpl can be executed as follows. Please refer to
|
59
|
+
the respective [providers](#supported-providers) for details.
|
60
|
+
|
61
|
+
```
|
62
|
+
dpl [provider] [options]
|
63
|
+
```
|
64
|
+
|
65
|
+
Dpl can be used locally, e.g. on your development machine, but it might leave
|
66
|
+
artifacts that may alter the behaviour of your system. If you encounter this
|
67
|
+
behaviour and it presents a serious issue to you then please open an
|
68
|
+
[issue](https://github.com/travis-ci/dpl/issues/new).
|
69
|
+
|
70
|
+
### Cleaning up the Git working directory
|
71
|
+
|
72
|
+
Dpl v1 has cleaned up the Git working directory by default, using `git stash
|
73
|
+
--all`. The default for this option has been changed in dpl v2, and users now
|
74
|
+
need to opt in to cleaning up any left over artifacts from the build process
|
75
|
+
by passing the option `--cleanup`.
|
76
|
+
|
77
|
+
The status of the working directory is relevant only to providers that package
|
78
|
+
and push it to the respective remote service (e.g. `heroku` when using the
|
79
|
+
`api` strategy, package registry providers, etc.). Most providers will either
|
80
|
+
push the latest Git commit, or pull code from a remote repository.
|
81
|
+
|
82
|
+
## Maturity Levels
|
83
|
+
|
84
|
+
In order to communicate the current development status and maturity of dpl's
|
85
|
+
support for a particular service the respective provider is marked with one of
|
86
|
+
the following maturity levels, according to the given criteria:
|
87
|
+
|
88
|
+
* `dev` - the provider is in development (initial level)
|
89
|
+
* `alpha` - the provider is fully tested
|
90
|
+
* `beta` - the provider has been in alpha for at least a month, and successful real-world production deployments have been observed
|
91
|
+
* `stable` - the provider has been in beta for at least two months, and there are no open issues that qualify as critical (such as deployments failing, documented functionality broken, etc)
|
92
|
+
|
93
|
+
## Supported Providers
|
94
|
+
|
95
|
+
Dpl supports the following providers:
|
96
|
+
|
97
|
+
<% providers.each do |key, name| -%>
|
98
|
+
* <%= "[#{name}](##{name.gsub(/\W+/, '-').downcase})" %>
|
99
|
+
<% end -%>
|
100
|
+
|
101
|
+
<% providers.each do |key, name|%>
|
102
|
+
### <%= name %>
|
103
|
+
|
104
|
+
```
|
105
|
+
<%= help(key) %>
|
106
|
+
```
|
107
|
+
<% end -%>
|
108
|
+
|
109
|
+
<%= File.read('./CONTRIBUTING.md').gsub(/^#/, '##') %>
|
110
|
+
|
111
|
+
## Old Issues
|
112
|
+
|
113
|
+
If an issue has been left open and untouched for 90 days or more, we
|
114
|
+
automatically close them. We do this to ensure that new issues are more easily
|
115
|
+
noticeable, and that old issues that have been resolved or are no longer
|
116
|
+
relevant are closed. You can read more about this [here](https://blog.travis-ci.com/2018-03-09-closing-old-issues).
|
117
|
+
|
118
|
+
## Code of Conduct
|
119
|
+
|
120
|
+
Please see [our code of conduct](CODE_OF_CONDUCT.md) for how to interact with
|
121
|
+
this project and its community.
|
122
|
+
|
123
|
+
## License
|
124
|
+
|
125
|
+
Dpl is licensed under the [MIT License](https://github.com/travis-ci/dpl/blob/master/LICENSE).
|
126
|
+
|
127
|
+
## Credits
|
128
|
+
|
129
|
+
This tool would not exist without your help.
|
130
|
+
|
131
|
+
A huge thank you goes out to all of our current and past [contributors](https://github.com/travis-ci/dpl/graphs/contributors):
|
132
|
+
|
133
|
+
<%= contributors %>
|
@@ -0,0 +1,6 @@
|
|
1
|
+
#!/bin/bash
|
2
|
+
curl --remote-name --location https://cli-dl.scalingo.io/release/scalingo_latest_linux_amd64.tar.gz && \
|
3
|
+
tar -zxvf scalingo_latest_linux_amd64.tar.gz && \
|
4
|
+
mv scalingo_*_linux_amd64/scalingo . && \
|
5
|
+
rm scalingo_latest_linux_amd64.tar.gz && \
|
6
|
+
rm -r scalingo_*_linux_amd64
|
data/lib/dpl/cli.rb
CHANGED
@@ -1,66 +1,54 @@
|
|
1
|
-
require '
|
2
|
-
require 'dpl/provider'
|
1
|
+
require 'cl'
|
3
2
|
|
4
|
-
module
|
5
|
-
class
|
6
|
-
def self.
|
7
|
-
new
|
3
|
+
module Dpl
|
4
|
+
class Cli < Cl
|
5
|
+
def self.new(ctx = nil, name = 'dpl')
|
6
|
+
ctx ||= Dpl::Ctx::Bash.new
|
7
|
+
super
|
8
8
|
end
|
9
9
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
args.
|
16
|
-
|
17
|
-
|
18
|
-
key = match[1].tr('-', '_').to_sym
|
19
|
-
if options.include? key
|
20
|
-
options[key] = Array(options[key]) << match[2]
|
21
|
-
else
|
22
|
-
options[key] = match[2] || true
|
23
|
-
end
|
24
|
-
end
|
25
|
-
|
26
|
-
self.fold_count = 0
|
27
|
-
self.options = default_options.merge(options)
|
10
|
+
def run(args)
|
11
|
+
args = untaint(args)
|
12
|
+
args = with_provider_opt(args)
|
13
|
+
super
|
14
|
+
rescue UnknownCmd
|
15
|
+
unknown_provider(args.first)
|
16
|
+
rescue Error => e
|
17
|
+
error(e)
|
28
18
|
end
|
29
19
|
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
options[:debug] ? raise(error) : die(error.message)
|
20
|
+
# Tainting is being used for automatically obfuscating values for secure
|
21
|
+
# options, so we want to untaint all incoming args here.
|
22
|
+
def untaint(args)
|
23
|
+
args.map(&:dup).each(&:untaint)
|
35
24
|
end
|
36
25
|
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
ensure
|
43
|
-
print "\ntravis_fold:end:dpl.#{fold_count}\r" if options[:fold]
|
26
|
+
# backwards compatibility for travis-build dpl v1 integration
|
27
|
+
def with_provider_opt(args)
|
28
|
+
return args unless arg = args.detect { |arg| arg.include?('--provider') }
|
29
|
+
args.delete(arg)
|
30
|
+
[arg.split('=').last, *args]
|
44
31
|
end
|
45
32
|
|
46
|
-
def
|
47
|
-
{
|
48
|
-
|
49
|
-
|
50
|
-
}
|
33
|
+
def error(e)
|
34
|
+
msg = "\e[31m#{e.message}\e[0m"
|
35
|
+
msg = [msg, *e.backtrace].join("\n") if e.backtrace?
|
36
|
+
abort msg
|
51
37
|
end
|
52
38
|
|
53
|
-
def
|
54
|
-
|
39
|
+
def unknown_provider(name)
|
40
|
+
msg = "\e[31mUnknown provider: #{name}\e[0m"
|
41
|
+
msg << "\nDid you mean: #{suggestions(name).join(', ')}?" if suggestions(name).any?
|
42
|
+
abort msg
|
55
43
|
end
|
56
44
|
|
57
|
-
def
|
58
|
-
|
59
|
-
|
45
|
+
def suggestions(name)
|
46
|
+
return [] unless defined?(DidYouMean)
|
47
|
+
DidYouMean::SpellChecker.new(dictionary: providers).correct(name)
|
60
48
|
end
|
61
49
|
|
62
|
-
def
|
63
|
-
|
50
|
+
def providers
|
51
|
+
Cl::Cmd.registry.keys.map(&:to_s)
|
64
52
|
end
|
65
53
|
end
|
66
54
|
end
|
data/lib/dpl/ctx.rb
ADDED
data/lib/dpl/ctx/bash.rb
ADDED
@@ -0,0 +1,543 @@
|
|
1
|
+
require 'cl'
|
2
|
+
require 'fileutils'
|
3
|
+
require 'logger'
|
4
|
+
require 'open3'
|
5
|
+
require 'tmpdir'
|
6
|
+
require 'securerandom'
|
7
|
+
require 'dpl/support/version'
|
8
|
+
|
9
|
+
module Dpl
|
10
|
+
module Ctx
|
11
|
+
class Bash < Cl::Ctx
|
12
|
+
include FileUtils
|
13
|
+
|
14
|
+
attr_accessor :folds, :stdout, :stderr, :last_out, :last_err
|
15
|
+
|
16
|
+
def initialize(stdout = $stdout, stderr = $stderr)
|
17
|
+
@stdout, @stderr = stdout, stderr
|
18
|
+
@folds = 0
|
19
|
+
super('dpl', abort: false)
|
20
|
+
end
|
21
|
+
|
22
|
+
# Folds any log output from the given block
|
23
|
+
#
|
24
|
+
# Starts a log fold with the given fold message, calls the block, and
|
25
|
+
# closes the fold.
|
26
|
+
#
|
27
|
+
# @param msg [String] the message that will appear on the log fold
|
28
|
+
def fold(msg, &block)
|
29
|
+
self.folds += 1
|
30
|
+
print "travis_fold:start:dpl.#{folds}\r\e[K"
|
31
|
+
time do
|
32
|
+
info "\e[33m#{msg}\e[0m"
|
33
|
+
yield
|
34
|
+
end
|
35
|
+
ensure
|
36
|
+
print "\ntravis_fold:end:dpl.#{folds}\r\e[K"
|
37
|
+
end
|
38
|
+
|
39
|
+
# Times the given block
|
40
|
+
#
|
41
|
+
# Starts a travis time log tag, calls the block, and closes the tag,
|
42
|
+
# including timing information. This makes a timing badge appear on
|
43
|
+
# the surrounding log fold.
|
44
|
+
def time(&block)
|
45
|
+
id = SecureRandom.hex[0, 8]
|
46
|
+
start = Time.now.to_i * (10 ** 9)
|
47
|
+
print "travis_time:start:#{id}\r\e[K"
|
48
|
+
yield
|
49
|
+
ensure
|
50
|
+
finish = Time.now.to_i * (10 ** 9)
|
51
|
+
duration = finish - start
|
52
|
+
print "\ntravis_time:end:#{id}:start=#{start},finish=#{finish},duration=#{duration}\r\e[K"
|
53
|
+
end
|
54
|
+
|
55
|
+
# Outputs a deprecation warning for a given deprecated option key to stderr.
|
56
|
+
#
|
57
|
+
# @param key [Symbol] the deprecated option key
|
58
|
+
# @param msg [String or Symbol] the deprecation message. if given a Symbol this will be wrapped into the string "Please use #{symbol}".
|
59
|
+
def deprecate_opt(key, msg)
|
60
|
+
msg = "please use #{msg}" if msg.is_a?(Symbol)
|
61
|
+
warn "Deprecated option #{key} used (#{msg})."
|
62
|
+
end
|
63
|
+
|
64
|
+
# Outputs an info level message to stdout.
|
65
|
+
def info(*msgs)
|
66
|
+
stdout.puts(*msgs)
|
67
|
+
end
|
68
|
+
|
69
|
+
# Prints an info level message to stdout.
|
70
|
+
#
|
71
|
+
# This method does not append a newline character to the given message,
|
72
|
+
# which usually is not the desired behaviour. The method is intended to
|
73
|
+
# be used if an initial, partial message is supposed to be printed, which
|
74
|
+
# will be completed later (using the method `info`).
|
75
|
+
#
|
76
|
+
# For example:
|
77
|
+
#
|
78
|
+
# print 'Starting a long running task ...'
|
79
|
+
# run_long_running_task
|
80
|
+
# info 'done.'
|
81
|
+
def print(chars)
|
82
|
+
stdout.print(chars)
|
83
|
+
end
|
84
|
+
|
85
|
+
# Outputs an warning message to stderr
|
86
|
+
#
|
87
|
+
# This method is intended to be used for warning messages that are
|
88
|
+
# supposed to show up in the build log, but do not qualify as errors that
|
89
|
+
# would abort the deployment process. The warning will be highlighted as
|
90
|
+
# yellow text. Use sparingly.
|
91
|
+
def warn(*msgs)
|
92
|
+
msgs = msgs.join("\n").lines
|
93
|
+
msgs.each { |msg| stderr.puts("\e[33;1m#{msg}\e[0m") }
|
94
|
+
end
|
95
|
+
|
96
|
+
# Raises an exception, halting the deployment process.
|
97
|
+
#
|
98
|
+
# The calling executable `bin/dpl` will catch the exception, and abort
|
99
|
+
# the ruby process with the given error message.
|
100
|
+
#
|
101
|
+
# This method is intended to be used for all error conditions that
|
102
|
+
# require the deployment process to be aborted.
|
103
|
+
def error(message)
|
104
|
+
raise Error, message
|
105
|
+
end
|
106
|
+
|
107
|
+
# Returns a logger
|
108
|
+
#
|
109
|
+
# Returns a logger instance, with the given log level set. This can be
|
110
|
+
# used to pass to clients that accept a Ruby logger, such as Faraday,
|
111
|
+
# for debugging purposes.
|
112
|
+
#
|
113
|
+
# Use with care.
|
114
|
+
#
|
115
|
+
# @param level [Symbol] the Ruby logger log level
|
116
|
+
def logger(level = :info)
|
117
|
+
logger = Logger.new(stderr)
|
118
|
+
logger.level = Logger.const_get(level.to_s.upcase)
|
119
|
+
logger
|
120
|
+
end
|
121
|
+
|
122
|
+
def validate_runtimes(runtimes)
|
123
|
+
failed = runtimes.reject(&method(:validate_runtime))
|
124
|
+
failed = failed.map { |name, versions| "#{name} (#{versions.join(', ')})" }
|
125
|
+
error "Failed validating runtimes: #{failed.join(', ')}" if failed.any?
|
126
|
+
end
|
127
|
+
|
128
|
+
def validate_runtime(args)
|
129
|
+
name, required = *args
|
130
|
+
info "Validating required runtime version: #{name} (#{required.join(', ')})"
|
131
|
+
version = name == :node_js ? node_version : python_version
|
132
|
+
required.all? { |required| Version.new(version).satisfies?(required) }
|
133
|
+
end
|
134
|
+
|
135
|
+
def apts_get(packages)
|
136
|
+
packages = packages.reject { |name, cmd = name| which(cmd || name) }
|
137
|
+
return unless packages.any?
|
138
|
+
apt_update
|
139
|
+
packages.each { |package, cmd| apt_get(package, cmd || package, update: false) }
|
140
|
+
end
|
141
|
+
|
142
|
+
# Installs an APT package
|
143
|
+
#
|
144
|
+
# Installs the APT package with the given name, unless the command is already
|
145
|
+
# available (as determined by `which [cmd]`.
|
146
|
+
#
|
147
|
+
# @param package [String] the package name
|
148
|
+
# @param cmd [String] an executable installed by the package, defaults to the package name
|
149
|
+
def apt_get(package, cmd = package, opts = {})
|
150
|
+
return if which(cmd)
|
151
|
+
apt_update unless opts[:update].is_a?(FalseClass)
|
152
|
+
shell "sudo apt-get -qq install #{package}", retry: true
|
153
|
+
end
|
154
|
+
|
155
|
+
def apt_update
|
156
|
+
shell 'sudo apt-get update', retry: true
|
157
|
+
end
|
158
|
+
|
159
|
+
# Requires source files from Ruby gems, installing them on demand if required
|
160
|
+
#
|
161
|
+
# Installs the Ruby gems with the given version, if not already installed, and
|
162
|
+
# requires the specified source files from that gem.
|
163
|
+
#
|
164
|
+
# This happens using the bundler/inline API.
|
165
|
+
#
|
166
|
+
# @param gems [Array<String, String, Hash>] Array of gem requirements: gem name, version, and options (`require`: A single path or a list of paths to source files to require from this Ruby gem)
|
167
|
+
#
|
168
|
+
# @see https://bundler.io/v2.0/guides/bundler_in_a_single_file_ruby_script.html
|
169
|
+
def gems_require(gems)
|
170
|
+
# A local Gemfile.lock might interfer with bundler/inline, even though
|
171
|
+
# it should not. Switching to a temporary dir fixes this.
|
172
|
+
Dir.chdir(tmp_dir) do
|
173
|
+
require 'bundler/inline'
|
174
|
+
info "Installing gem dependencies: #{gems.map { |name, version, _| "#{name} #{"(#{version})" if version}".strip }.join(', ')}"
|
175
|
+
env = ENV.to_h
|
176
|
+
# Bundler.reset!
|
177
|
+
# Gem.loaded_specs.clear
|
178
|
+
gemfile do
|
179
|
+
source 'https://rubygems.org'
|
180
|
+
gems.each { |g| gem *g }
|
181
|
+
end
|
182
|
+
# https://github.com/bundler/bundler/issues/7181
|
183
|
+
ENV.replace(env)
|
184
|
+
end
|
185
|
+
end
|
186
|
+
|
187
|
+
# Installs an NPM package
|
188
|
+
#
|
189
|
+
# Installs the NPM package with the given name, unless the command is already
|
190
|
+
# available (as determined by `which [cmd]`.
|
191
|
+
#
|
192
|
+
# @param package [String] the package name
|
193
|
+
# @param cmd [String] an executable installed by the package, defaults to the package name
|
194
|
+
def npm_install(package, cmd = package)
|
195
|
+
shell "npm install -g #{package}", retry: true unless which(cmd)
|
196
|
+
end
|
197
|
+
|
198
|
+
# Installs a Python package
|
199
|
+
#
|
200
|
+
# Installs the Python package with the given name. A previously installed
|
201
|
+
# package is uninstalled before that, but only if `version` was given.
|
202
|
+
#
|
203
|
+
# @param package [String] Package name (required).
|
204
|
+
# @param cmd [String] Executable command installed by that package (optional, defaults to the package name).
|
205
|
+
# @param version [String] Package version (optional).
|
206
|
+
def pip_install(package, cmd = package, version = nil)
|
207
|
+
ENV['VIRTUAL_ENV'] = File.expand_path('~/dpl_venv')
|
208
|
+
ENV['PATH'] = File.expand_path("~/dpl_venv/bin:#{ENV['PATH']}")
|
209
|
+
shell 'virtualenv --no-site-packages ~/dpl_venv', echo: true
|
210
|
+
shell 'pip install urllib3[secure]'
|
211
|
+
cmd = "pip install #{package}"
|
212
|
+
cmd << pip_version(version) if version
|
213
|
+
shell cmd, retry: true
|
214
|
+
end
|
215
|
+
|
216
|
+
def pip_version(version)
|
217
|
+
version =~ /^\d+/ ? "==#{version}" : version
|
218
|
+
end
|
219
|
+
|
220
|
+
# Generates an SSH key
|
221
|
+
#
|
222
|
+
# @param name [String] the key name
|
223
|
+
# @param file [String] path to the key file
|
224
|
+
def ssh_keygen(name, file)
|
225
|
+
shell %(ssh-keygen -t rsa -N "" -C #{name} -f #{file})
|
226
|
+
end
|
227
|
+
|
228
|
+
# Runs a single shell command
|
229
|
+
#
|
230
|
+
# This the is the central point of executing any shell commands. It allows two
|
231
|
+
# strategies for running commands in subprocesses:
|
232
|
+
#
|
233
|
+
# * Using [Kernel#system](https://ruby-doc.org/core-2.6.3/Kernel.html#method-i-system)
|
234
|
+
# which is the default strategy, and should be used when possible. The stdout
|
235
|
+
# and stderr streams will not be captured, but streamed directly to the parent
|
236
|
+
# process (so any output on these streams appears in the build log as soon as
|
237
|
+
# possible).
|
238
|
+
#
|
239
|
+
# * Using [Open3.capture3](https://ruby-doc.org/stdlib-2.6.3/libdoc/open3/rdoc/Open3.html#method-c-capture3)
|
240
|
+
# which captures both stdout and stderr, and does not automatically output it
|
241
|
+
# to the build log. Implementors can choose to display it after the shell command
|
242
|
+
# has completed, using the `%{out}` and `%{err}` interpolation variables. Use
|
243
|
+
# sparingly.
|
244
|
+
#
|
245
|
+
# The method accepts the following options:
|
246
|
+
#
|
247
|
+
# @param cmd [String] the shell command to execute
|
248
|
+
# @param opts [Hash] options
|
249
|
+
#
|
250
|
+
# @option opts [Boolean] :echo output the command to stdout before running it
|
251
|
+
# @option opts [Boolean] :silence silence all log output by redirecting stdout and stderr to `/dev/null`
|
252
|
+
# @option opts [Boolean] :capture use `Open3.capture3` to capture stdout and stderr
|
253
|
+
# @option opts [String] :python wrap the command into Bash code that enforces the given Python version to be used
|
254
|
+
# @option opts [String] :retry retries the command 2 more times if it fails
|
255
|
+
# @option opts [String] :info message to output to stdout if the command has exited with the exit code 0 (supports the interpolation variable `${out}` for stdout in case it was captured.
|
256
|
+
# @option opts [String] :assert error message to be raised if the command has exited with a non-zero exit code (supports the interpolation variable `${out}` for stdout in case it was captured.
|
257
|
+
#
|
258
|
+
# @return [Boolean] whether or not the command was successful (has exited with the exit code 0)
|
259
|
+
def shell(cmd, opts = {})
|
260
|
+
cmd = Cmd.new(nil, cmd, opts) if cmd.is_a?(String)
|
261
|
+
info cmd.msg if cmd.msg?
|
262
|
+
info cmd.echo if cmd.echo?
|
263
|
+
|
264
|
+
@last_out, @last_err, @last_status = retrying(cmd.retry ? 2 : 0) do
|
265
|
+
send(cmd.capture? ? :open3 : :system, cmd.cmd, cmd.opts)
|
266
|
+
end
|
267
|
+
|
268
|
+
info cmd.success % { out: last_out } if success? && cmd.success?
|
269
|
+
error cmd.error % { err: last_err } if failed? && cmd.assert?
|
270
|
+
|
271
|
+
success? && cmd.capture? ? last_out.chomp : @last_status
|
272
|
+
end
|
273
|
+
|
274
|
+
def retrying(max, tries = 0, status = false)
|
275
|
+
loop do
|
276
|
+
tries += 1
|
277
|
+
out, err, status = yield
|
278
|
+
return [out, err, status] if status || tries > max
|
279
|
+
end
|
280
|
+
end
|
281
|
+
|
282
|
+
# Runs a shell command and captures stdout, stderr, and the exit status
|
283
|
+
#
|
284
|
+
# Runs the given command using `Open3.capture3`, which will capture the
|
285
|
+
# stdout and stderr streams, as well as the exit status. I.e. this will
|
286
|
+
# *not* stream log output in real time, but capture the output, and allow
|
287
|
+
# implementors to display it later (using the `%{out}` and `%{err}`
|
288
|
+
# interpolation variables.
|
289
|
+
#
|
290
|
+
# Use sparingly.
|
291
|
+
#
|
292
|
+
# @option chdir [String] directory temporarily to change to before running the command
|
293
|
+
def open3(cmd, opts)
|
294
|
+
opts = [opts[:chdir] ? only(opts, :chdir) : nil].compact
|
295
|
+
out, err, status = Open3.capture3(cmd, *opts)
|
296
|
+
[out, err, status.success?]
|
297
|
+
end
|
298
|
+
|
299
|
+
# Runs a shell command, streaming any stdout or stderr output, and
|
300
|
+
# returning the exit status
|
301
|
+
#
|
302
|
+
# This is the default method for executing shell commands. The stdout and
|
303
|
+
# stderr will not be captured, but streamed directly to the parent process.
|
304
|
+
#
|
305
|
+
# @option chdir [String] directory temporarily to change to before running the command
|
306
|
+
def system(cmd, opts = {})
|
307
|
+
opts = [opts[:chdir] ? only(opts, :chdir) : nil].compact
|
308
|
+
Kernel.system(cmd, *opts)
|
309
|
+
['', '', last_process_status]
|
310
|
+
end
|
311
|
+
|
312
|
+
# Whether or not the last executed shell command was successful.
|
313
|
+
def success?
|
314
|
+
!!@last_status
|
315
|
+
end
|
316
|
+
|
317
|
+
# Whether or not the last executed shell command has failed.
|
318
|
+
def failed?
|
319
|
+
!success?
|
320
|
+
end
|
321
|
+
|
322
|
+
# Returns the last child process' exit status
|
323
|
+
#
|
324
|
+
# Internal, and not to be used by implementors. $? is a read-only
|
325
|
+
# variable, so we use a method that we can stub during tests.
|
326
|
+
def last_process_status
|
327
|
+
$?.success?
|
328
|
+
end
|
329
|
+
|
330
|
+
# Whether or not the current Ruby process runs with superuser priviledges.
|
331
|
+
def sudo?
|
332
|
+
Process::UID.eid == 0
|
333
|
+
end
|
334
|
+
|
335
|
+
# Returns current repository name
|
336
|
+
#
|
337
|
+
# Uses the environment variable `TRAVIS_REPO_SLUG` if present, or the
|
338
|
+
# current directory's base name.
|
339
|
+
#
|
340
|
+
# Note that this might return an unexpected string outside of the context
|
341
|
+
# of Travis CI build environments if the method is called at a time when
|
342
|
+
# the current working directory has changed.
|
343
|
+
def repo_name
|
344
|
+
ENV['TRAVIS_REPO_SLUG'] ? ENV['TRAVIS_REPO_SLUG'].split('/').last : File.basename(Dir.pwd)
|
345
|
+
end
|
346
|
+
|
347
|
+
# Returns current repository slug
|
348
|
+
#
|
349
|
+
# Uses the environment variable `TRAVIS_REPO_SLUG` if present, or the
|
350
|
+
# last two segmens of the current working directory's path.
|
351
|
+
#
|
352
|
+
# Note that this might return an unexpected string outside of the context
|
353
|
+
# of Travis CI build environments if the method is called at a time when
|
354
|
+
# the current working directory has changed.
|
355
|
+
def repo_slug
|
356
|
+
ENV['TRAVIS_REPO_SLUG'] || Dir.pwd.split('/')[-2, 2].join('/')
|
357
|
+
end
|
358
|
+
|
359
|
+
# Returns the current build directory
|
360
|
+
#
|
361
|
+
# Uses the environment variable `TRAVIS_REPO_SLUG` if present, and
|
362
|
+
# defaults to `.` otherwise.
|
363
|
+
#
|
364
|
+
# Note that this might return an unexpected string outside of the context
|
365
|
+
# of Travis CI build environments if the method is called at a time when
|
366
|
+
# the current working directory has changed.
|
367
|
+
def build_dir
|
368
|
+
ENV['TRAVIS_BUILD_DIR'] || '.'
|
369
|
+
end
|
370
|
+
|
371
|
+
# Returns the current build number
|
372
|
+
#
|
373
|
+
# Returns the value of the environment variable `TRAVIS_BUILD_NUMBER` if
|
374
|
+
# present.
|
375
|
+
def build_number
|
376
|
+
ENV['TRAVIS_BUILD_NUMBER'] || raise('TRAVIS_BUILD_NUMBER not set')
|
377
|
+
end
|
378
|
+
|
379
|
+
# Returns the encoding of the given file, as determined by `file`.
|
380
|
+
def encoding(path)
|
381
|
+
case `file '#{path}'`
|
382
|
+
when /gzip compressed/
|
383
|
+
'gzip'
|
384
|
+
when /compress'd/
|
385
|
+
'compress'
|
386
|
+
when /text/
|
387
|
+
'text'
|
388
|
+
when /data/
|
389
|
+
# shrugs?
|
390
|
+
end
|
391
|
+
end
|
392
|
+
|
393
|
+
# Returns the current branch name
|
394
|
+
def git_branch
|
395
|
+
ENV['TRAVIS_BRANCH'] || git_rev_parse('HEAD')
|
396
|
+
end
|
397
|
+
|
398
|
+
# Returns the message of the commit `git_sha`.
|
399
|
+
def git_commit_msg
|
400
|
+
`git log #{git_sha} -n 1 --pretty=%B`.chomp
|
401
|
+
end
|
402
|
+
|
403
|
+
# Returns the committer name of the commit `git_sha`.
|
404
|
+
def git_author_name
|
405
|
+
`git log #{git_sha} -n 1 --pretty=%an`.chomp
|
406
|
+
end
|
407
|
+
|
408
|
+
# Returns the comitter email of the commit `git_sha`.
|
409
|
+
def git_author_email
|
410
|
+
`git log #{git_sha} -n 1 --pretty=%ae`.chomp
|
411
|
+
end
|
412
|
+
|
413
|
+
# Whether or not the git working directory is dirty
|
414
|
+
def git_dirty?
|
415
|
+
!Kernel.system('git diff --quiet')
|
416
|
+
end
|
417
|
+
|
418
|
+
# Returns the output of `git log`, using the given args.
|
419
|
+
def git_log(args)
|
420
|
+
`git log #{args}`.chomp
|
421
|
+
end
|
422
|
+
|
423
|
+
# Returns the Git log, separated by NULs
|
424
|
+
#
|
425
|
+
# Returns the output of `git ls-files -z`, which separates log entries by
|
426
|
+
# NULs, rather than newline characters.
|
427
|
+
def git_ls_files
|
428
|
+
`git ls-files -z`.split("\x0")
|
429
|
+
end
|
430
|
+
|
431
|
+
# Returns true if the given ref exists remotely
|
432
|
+
def git_ls_remote?(url, ref)
|
433
|
+
Kernel.system("git ls-remote --exit-code #{url} #{ref} > /dev/null 2>&1")
|
434
|
+
end
|
435
|
+
|
436
|
+
# Returns known Git remote URLs
|
437
|
+
def git_remote_urls
|
438
|
+
`git remote -v`.scan(/\t[^\s]+\s/).map(&:strip).uniq
|
439
|
+
end
|
440
|
+
|
441
|
+
# Returns the sha for the given Git ref
|
442
|
+
def git_rev_parse(ref)
|
443
|
+
`git rev-parse #{ref}`.strip
|
444
|
+
end
|
445
|
+
|
446
|
+
# Returns the latest tag name, if any
|
447
|
+
def git_tag
|
448
|
+
`git describe --tags --exact-match 2>/dev/null`.chomp
|
449
|
+
end
|
450
|
+
|
451
|
+
# Returns the current commit sha
|
452
|
+
def git_sha
|
453
|
+
ENV['TRAVIS_COMMIT'] || `git rev-parse HEAD`.chomp
|
454
|
+
end
|
455
|
+
|
456
|
+
# Returns the local machine's hostname
|
457
|
+
def machine_name
|
458
|
+
`hostname`.strip
|
459
|
+
end
|
460
|
+
|
461
|
+
# Returns the current Node.js version
|
462
|
+
def node_version
|
463
|
+
`node -v`.sub(/^v/, '').chomp
|
464
|
+
end
|
465
|
+
|
466
|
+
# Returns the current NPM version
|
467
|
+
def npm_version
|
468
|
+
`npm --version`
|
469
|
+
end
|
470
|
+
|
471
|
+
# Returns the current Node.js version
|
472
|
+
def python_version
|
473
|
+
`python --version 2>&1`.sub(/^Python /, '').chomp
|
474
|
+
end
|
475
|
+
|
476
|
+
# Returns true or false depending if the given command can be found
|
477
|
+
def which(cmd)
|
478
|
+
!`which #{cmd}`.chomp.empty? if cmd
|
479
|
+
end
|
480
|
+
|
481
|
+
# Returns a unique temporary directory name
|
482
|
+
def tmp_dir
|
483
|
+
@tmp_dir ||= Dir.mktmpdir
|
484
|
+
end
|
485
|
+
|
486
|
+
# Returns the size of the given file path
|
487
|
+
def file_size(path)
|
488
|
+
File.size(path)
|
489
|
+
end
|
490
|
+
|
491
|
+
def move_files(paths)
|
492
|
+
paths.each do |path|
|
493
|
+
target = "#{tmp_dir}/#{File.basename(path)}"
|
494
|
+
mv(path, target) if File.exists?(path)
|
495
|
+
end
|
496
|
+
end
|
497
|
+
|
498
|
+
def unmove_files(paths)
|
499
|
+
paths.each do |path|
|
500
|
+
source = "#{tmp_dir}/#{File.basename(path)}"
|
501
|
+
mv(source, path) if File.exists?(source)
|
502
|
+
end
|
503
|
+
end
|
504
|
+
|
505
|
+
def mv(src, dest)
|
506
|
+
Kernel.system("sudo mv #{src} #{dest} 2> /dev/null")
|
507
|
+
end
|
508
|
+
|
509
|
+
# Writes the given content to the given file path
|
510
|
+
def write_file(path, content, chmod = nil)
|
511
|
+
path = File.expand_path(path)
|
512
|
+
FileUtils.mkdir_p(File.dirname(path))
|
513
|
+
File.open(path, 'w+') { |f| f.write(content) }
|
514
|
+
FileUtils.chmod(chmod, path) if chmod
|
515
|
+
end
|
516
|
+
|
517
|
+
# Writes the given machine, login, and password to ~/.netrc
|
518
|
+
def write_netrc(machine, login, password)
|
519
|
+
require 'netrc'
|
520
|
+
netrc = Netrc.read
|
521
|
+
netrc[machine] = [login, password]
|
522
|
+
netrc.save
|
523
|
+
end
|
524
|
+
|
525
|
+
def sleep(sec)
|
526
|
+
Kernel.sleep(sec)
|
527
|
+
end
|
528
|
+
|
529
|
+
def tty?
|
530
|
+
$stdout.isatty
|
531
|
+
end
|
532
|
+
|
533
|
+
# Returns a copy of the given hash, reduced to the given keys
|
534
|
+
def only(hash, *keys)
|
535
|
+
hash.select { |key, _| keys.include?(key) }.to_h
|
536
|
+
end
|
537
|
+
|
538
|
+
def test?
|
539
|
+
false
|
540
|
+
end
|
541
|
+
end
|
542
|
+
end
|
543
|
+
end
|