dpl 1.10.17.travis.6637.6 → 2.0.0.alpha.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (140) hide show
  1. checksums.yaml +5 -5
  2. data/CHANGELOG.md +74 -0
  3. data/CONTRIBUTING.md +392 -0
  4. data/Gemfile +17 -3
  5. data/Gemfile.lock +373 -0
  6. data/LICENSE +16 -19
  7. data/NOTES.md +275 -0
  8. data/README.md +1977 -707
  9. data/Rakefile +2 -2
  10. data/bin/dpl +7 -3
  11. data/lib/dpl.rb +20 -0
  12. data/lib/dpl/assets/atlas/install +19 -0
  13. data/lib/dpl/assets/dpl/README.erb.md +133 -0
  14. data/lib/dpl/assets/dpl/git_ssh +2 -0
  15. data/lib/dpl/assets/git/detect_private_key +8 -0
  16. data/lib/dpl/assets/hephy/filter_log +3 -0
  17. data/lib/dpl/assets/pypi/install +4 -0
  18. data/lib/dpl/assets/scalingo/install +6 -0
  19. data/lib/dpl/cli.rb +36 -48
  20. data/lib/dpl/ctx.rb +2 -0
  21. data/lib/dpl/ctx/bash.rb +543 -0
  22. data/lib/dpl/ctx/test.rb +242 -0
  23. data/lib/dpl/helper/assets.rb +36 -0
  24. data/lib/dpl/helper/cmd.rb +167 -0
  25. data/lib/dpl/helper/config_file.rb +47 -0
  26. data/lib/dpl/helper/env.rb +39 -0
  27. data/lib/dpl/helper/interpolate.rb +126 -0
  28. data/lib/dpl/helper/memoize.rb +20 -0
  29. data/lib/dpl/helper/squiggle.rb +22 -0
  30. data/lib/dpl/helper/zip.rb +69 -0
  31. data/lib/dpl/provider.rb +562 -234
  32. data/lib/dpl/provider/dsl.rb +369 -0
  33. data/lib/dpl/provider/examples.rb +128 -0
  34. data/lib/dpl/provider/status.rb +59 -0
  35. data/lib/dpl/providers.rb +40 -0
  36. data/lib/dpl/providers/anynines.rb +65 -0
  37. data/lib/dpl/providers/atlas.rb +49 -0
  38. data/lib/dpl/providers/azure_web_apps.rb +59 -0
  39. data/lib/dpl/providers/bintray.rb +313 -0
  40. data/lib/dpl/providers/bluemixcloudfoundry.rb +92 -0
  41. data/lib/dpl/providers/boxfuse.rb +48 -0
  42. data/lib/dpl/providers/cargo.rb +19 -0
  43. data/lib/dpl/providers/chef_supermarket.rb +128 -0
  44. data/lib/dpl/providers/cloud66.rb +40 -0
  45. data/lib/dpl/providers/cloudfiles.rb +56 -0
  46. data/lib/dpl/providers/cloudfoundry.rb +81 -0
  47. data/lib/dpl/providers/codedeploy.rb +179 -0
  48. data/lib/dpl/providers/datica.rb +60 -0
  49. data/lib/dpl/providers/elasticbeanstalk.rb +195 -0
  50. data/lib/dpl/providers/engineyard.rb +107 -0
  51. data/lib/dpl/providers/firebase.rb +41 -0
  52. data/lib/dpl/providers/gae.rb +74 -0
  53. data/lib/dpl/providers/gcs.rb +105 -0
  54. data/lib/dpl/providers/hackage.rb +47 -0
  55. data/lib/dpl/providers/hephy.rb +101 -0
  56. data/lib/dpl/providers/heroku.rb +111 -0
  57. data/lib/dpl/providers/heroku/api.rb +119 -0
  58. data/lib/dpl/providers/heroku/git.rb +50 -0
  59. data/lib/dpl/providers/lambda.rb +202 -0
  60. data/lib/dpl/providers/launchpad.rb +74 -0
  61. data/lib/dpl/providers/netlify.rb +30 -0
  62. data/lib/dpl/providers/npm.rb +88 -0
  63. data/lib/dpl/providers/openshift.rb +46 -0
  64. data/lib/dpl/providers/opsworks.rb +142 -0
  65. data/lib/dpl/providers/packagecloud.rb +190 -0
  66. data/lib/dpl/providers/pages.rb +17 -0
  67. data/lib/dpl/providers/pages/api.rb +102 -0
  68. data/lib/dpl/providers/pages/git.rb +251 -0
  69. data/lib/dpl/providers/puppetforge.rb +44 -0
  70. data/lib/dpl/providers/pypi.rb +120 -0
  71. data/lib/dpl/providers/releases.rb +214 -0
  72. data/lib/dpl/providers/rubygems.rb +89 -0
  73. data/lib/dpl/providers/s3.rb +243 -0
  74. data/lib/dpl/providers/scalingo.rb +63 -0
  75. data/lib/dpl/providers/script.rb +28 -0
  76. data/lib/dpl/providers/snap.rb +59 -0
  77. data/lib/dpl/providers/surge.rb +55 -0
  78. data/lib/dpl/providers/testfairy.rb +93 -0
  79. data/lib/dpl/providers/transifex.rb +66 -0
  80. data/lib/dpl/support/aws_sdk_patch.rb +23 -0
  81. data/lib/dpl/support/gems.rb +69 -0
  82. data/lib/dpl/support/gstore_patch.rb +6 -0
  83. data/lib/dpl/support/version.rb +83 -0
  84. data/lib/dpl/version.rb +2 -2
  85. metadata +98 -169
  86. data/.coveralls.yml +0 -1
  87. data/.github/CONTRIBUTING.md +0 -173
  88. data/.github/stale.yml +0 -53
  89. data/.gitignore +0 -13
  90. data/.rspec +0 -2
  91. data/.travis.yml +0 -56
  92. data/dpl-anynines.gemspec +0 -3
  93. data/dpl-atlas.gemspec +0 -3
  94. data/dpl-azure_webapps.gemspec +0 -3
  95. data/dpl-bintray.gemspec +0 -3
  96. data/dpl-bitballoon.gemspec +0 -3
  97. data/dpl-bluemix_cloud_foundry.gemspec +0 -3
  98. data/dpl-boxfuse.gemspec +0 -3
  99. data/dpl-cargo.gemspec +0 -3
  100. data/dpl-catalyze.gemspec +0 -3
  101. data/dpl-chef_supermarket.gemspec +0 -20
  102. data/dpl-cloud66.gemspec +0 -3
  103. data/dpl-cloud_files.gemspec +0 -3
  104. data/dpl-cloud_foundry.gemspec +0 -3
  105. data/dpl-code_deploy.gemspec +0 -3
  106. data/dpl-deis.gemspec +0 -3
  107. data/dpl-elastic_beanstalk.gemspec +0 -3
  108. data/dpl-engine_yard.gemspec +0 -3
  109. data/dpl-firebase.gemspec +0 -3
  110. data/dpl-gae.gemspec +0 -3
  111. data/dpl-gcs.gemspec +0 -3
  112. data/dpl-hackage.gemspec +0 -3
  113. data/dpl-hephy.gemspec +0 -3
  114. data/dpl-heroku.gemspec +0 -3
  115. data/dpl-lambda.gemspec +0 -3
  116. data/dpl-launchpad.gemspec +0 -3
  117. data/dpl-npm.gemspec +0 -3
  118. data/dpl-openshift.gemspec +0 -3
  119. data/dpl-ops_works.gemspec +0 -3
  120. data/dpl-packagecloud.gemspec +0 -3
  121. data/dpl-pages.gemspec +0 -3
  122. data/dpl-puppet_forge.gemspec +0 -3
  123. data/dpl-pypi.gemspec +0 -3
  124. data/dpl-releases.gemspec +0 -8
  125. data/dpl-rubygems.gemspec +0 -3
  126. data/dpl-s3.gemspec +0 -3
  127. data/dpl-scalingo.gemspec +0 -3
  128. data/dpl-script.gemspec +0 -3
  129. data/dpl-snap.gemspec +0 -3
  130. data/dpl-surge.gemspec +0 -3
  131. data/dpl-testfairy.gemspec +0 -3
  132. data/dpl-transifex.gemspec +0 -3
  133. data/dpl.gemspec +0 -3
  134. data/gemspec_helper.rb +0 -51
  135. data/lib/dpl/error.rb +0 -3
  136. data/notes/engine_yard.md +0 -1
  137. data/notes/heroku.md +0 -3
  138. data/spec/cli_spec.rb +0 -36
  139. data/spec/provider_spec.rb +0 -191
  140. data/spec/spec_helper.rb +0 -20
data/Rakefile CHANGED
@@ -43,7 +43,7 @@ end
43
43
 
44
44
  gemspecs = FileList[File.join(top, "dpl-*.gemspec")]
45
45
 
46
- providers = gemspecs.map { |f| /^dpl-(?<provider>.*)\.gemspec$/ =~ File.basename(f) && provider }
46
+ providers = gemspecs.map { |f| /dpl-(?<provider>.*)\.gemspec/ =~ f && provider }
47
47
 
48
48
  desc "Build dpl gem"
49
49
  file "dpl-#{gem_version}.gem" do
@@ -184,7 +184,7 @@ providers.each do |provider|
184
184
  logger.info green("Installing dpl-#{provider} gem")
185
185
  sh "gem install --no-post-install-message dpl-#{provider}-#{gem_version}.gem"
186
186
  logger.info green("Testing dpl-#{provider} loads correctly")
187
- ruby "-S dpl --provider=#{provider} --skip-cleanup=true --no-deploy"
187
+ ruby "-S dpl --provider=#{provider} --stage=init"
188
188
  end
189
189
 
190
190
  desc "Uninstall dpl-#{provider}"
data/bin/dpl CHANGED
@@ -1,5 +1,9 @@
1
1
  #!/usr/bin/env ruby
2
- $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
2
+ $: << File.expand_path('../../lib', __FILE__)
3
3
 
4
- require 'dpl/cli'
5
- DPL::CLI.run(ARGV)
4
+ $stdout.sync = true
5
+ $stderr.sync = true
6
+
7
+ require 'dpl'
8
+
9
+ Dpl::Cli.new.run(ARGV)
data/lib/dpl.rb ADDED
@@ -0,0 +1,20 @@
1
+ require 'dpl/cli'
2
+ require 'dpl/ctx'
3
+ require 'dpl/provider'
4
+ require 'dpl/version'
5
+
6
+ module Dpl
7
+ class Error < StandardError
8
+ attr_reader :opts
9
+
10
+ def initialize(msg, opts = {})
11
+ super(msg)
12
+ @opts = opts
13
+ set_backtrace(opts[:backtrace]) if backtrace?
14
+ end
15
+
16
+ def backtrace?
17
+ !!opts[:backtrace]
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,19 @@
1
+ if ! command -v atlas-upload &>/dev/null ; then
2
+ mkdir -p $HOME/bin $HOME/gopath/src
3
+ export PATH="$HOME/bin:$PATH"
4
+
5
+ if ! command -v gimme &>/dev/null ; then
6
+ curl -sL -o $HOME/bin/gimme https://raw.githubusercontent.com/meatballhat/gimme/master/gimme
7
+ chmod +x $HOME/bin/gimme
8
+ fi
9
+
10
+ if [ -z $GOPATH ]; then
11
+ export GOPATH="$HOME/gopath"
12
+ else
13
+ export GOPATH="$HOME/gopath:$GOPATH"
14
+ fi
15
+ eval "$(gimme 1.6)" &> /dev/null
16
+
17
+ go get github.com/hashicorp/atlas-upload-cli
18
+ cp $HOME/gopath/bin/atlas-upload-cli $HOME/bin/atlas-upload
19
+ fi
@@ -0,0 +1,133 @@
1
+ # Dpl [![Build Status](https://travis-ci.com/travis-ci/dpl.svg?branch=master)](https://travis-ci.com/travis-ci/dpl) [![Code Climate](https://codeclimate.com/github/travis-ci/dpl.png)](https://codeclimate.com/github/travis-ci/dpl) [![Coverage Status](https://coveralls.io/repos/travis-ci/dpl/badge.svg?branch=master&service=github&cache=2019-08-09_17:00)](https://coveralls.io/github/travis-ci/dpl?branch=master) [![Gem Version](https://img.shields.io/gem/v/dpl)](http://rubygems.org/gems/dpl) [![Yard Docs](http://img.shields.io/badge/yard-docs-blue.svg)](http://rubydoc.info/github/travis-ci/dpl)
2
+
3
+ This version of the README documents dpl v2, the next major version of dpl.
4
+ The REAMDE for dpl v1, the version that is currently used in production on
5
+ Travis CI can be found [here](https://github.com/travis-ci/dpl/blob/v1/README.md).
6
+
7
+ Dpl is command line tool for deploying code, html, packages, or build artifacts
8
+ to various service providers.
9
+
10
+ It is tightly integrated into Travis CI's [deployment integration](https://docs.travis-ci.com/user/deployment),
11
+ but also used, and recommended by others, such as [GitLab](https://docs.gitlab.com/ee/ci/examples/deployment/).
12
+
13
+ It is maintained by Travis CI, largely community driven, and it has existed
14
+ since 2013. If you find support your preferred deployment target missing,
15
+ please do not hesitate to get in touch, and we'll help you [add it](#contributing-to-dpl).
16
+
17
+ ## Table of Contents
18
+
19
+ * [Requirements](#requirements)
20
+ * [Installation](#installation)
21
+ * [Usage](#usage)
22
+ * [Maturity Levels](#maturity-levels)
23
+ * [Supported Providers](#supported-providers)
24
+ * [Contributing to Dpl](#contributing-to-dpl)
25
+ * [Old Issues](#old-issues)
26
+ * [Code of Conduct](#code-of-conduct)
27
+ * [License](#license)
28
+ * [Credits](#credits)
29
+
30
+ ## Requirements
31
+
32
+ Dpl requires Ruby 2.2 or later.
33
+
34
+ Depending on the deployment target dpl might require additional runtimes (e.g.
35
+ Go, Node.js, or Python) to be installed. It also might require sudo access in
36
+ order to install a Debian package.
37
+
38
+ Dpl is generally optimized for usage on Linux systems.
39
+
40
+ ## Installation
41
+
42
+ Installation:
43
+
44
+ ```
45
+ gem install dpl
46
+ ```
47
+
48
+ ## Usage
49
+
50
+ Dpl is meant and optimized for usage in ephemeral build environments, such
51
+ as Travis CI, or any other CI/CD pipeline.
52
+
53
+ Dpl is integrated to Travis CI's build configuration and build script compilation
54
+ tooling, so all you need to do is add the proper configuration to your `.travis.yml`
55
+ file. Please refer to [the documentation](https://docs.travis-ci.com/user/deployment)
56
+ for details.
57
+
58
+ For usage outside of Travis CI dpl can be executed as follows. Please refer to
59
+ the respective [providers](#supported-providers) for details.
60
+
61
+ ```
62
+ dpl [provider] [options]
63
+ ```
64
+
65
+ Dpl can be used locally, e.g. on your development machine, but it might leave
66
+ artifacts that may alter the behaviour of your system. If you encounter this
67
+ behaviour and it presents a serious issue to you then please open an
68
+ [issue](https://github.com/travis-ci/dpl/issues/new).
69
+
70
+ ### Cleaning up the Git working directory
71
+
72
+ Dpl v1 has cleaned up the Git working directory by default, using `git stash
73
+ --all`. The default for this option has been changed in dpl v2, and users now
74
+ need to opt in to cleaning up any left over artifacts from the build process
75
+ by passing the option `--cleanup`.
76
+
77
+ The status of the working directory is relevant only to providers that package
78
+ and push it to the respective remote service (e.g. `heroku` when using the
79
+ `api` strategy, package registry providers, etc.). Most providers will either
80
+ push the latest Git commit, or pull code from a remote repository.
81
+
82
+ ## Maturity Levels
83
+
84
+ In order to communicate the current development status and maturity of dpl's
85
+ support for a particular service the respective provider is marked with one of
86
+ the following maturity levels, according to the given criteria:
87
+
88
+ * `dev` - the provider is in development (initial level)
89
+ * `alpha` - the provider is fully tested
90
+ * `beta` - the provider has been in alpha for at least a month, and successful real-world production deployments have been observed
91
+ * `stable` - the provider has been in beta for at least two months, and there are no open issues that qualify as critical (such as deployments failing, documented functionality broken, etc)
92
+
93
+ ## Supported Providers
94
+
95
+ Dpl supports the following providers:
96
+
97
+ <% providers.each do |key, name| -%>
98
+ * <%= "[#{name}](##{name.gsub(/\W+/, '-').downcase})" %>
99
+ <% end -%>
100
+
101
+ <% providers.each do |key, name|%>
102
+ ### <%= name %>
103
+
104
+ ```
105
+ <%= help(key) %>
106
+ ```
107
+ <% end -%>
108
+
109
+ <%= File.read('./CONTRIBUTING.md').gsub(/^#/, '##') %>
110
+
111
+ ## Old Issues
112
+
113
+ If an issue has been left open and untouched for 90 days or more, we
114
+ automatically close them. We do this to ensure that new issues are more easily
115
+ noticeable, and that old issues that have been resolved or are no longer
116
+ relevant are closed. You can read more about this [here](https://blog.travis-ci.com/2018-03-09-closing-old-issues).
117
+
118
+ ## Code of Conduct
119
+
120
+ Please see [our code of conduct](CODE_OF_CONDUCT.md) for how to interact with
121
+ this project and its community.
122
+
123
+ ## License
124
+
125
+ Dpl is licensed under the [MIT License](https://github.com/travis-ci/dpl/blob/master/LICENSE).
126
+
127
+ ## Credits
128
+
129
+ This tool would not exist without your help.
130
+
131
+ A huge thank you goes out to all of our current and past [contributors](https://github.com/travis-ci/dpl/graphs/contributors):
132
+
133
+ <%= contributors %>
@@ -0,0 +1,2 @@
1
+ #!/bin/sh
2
+ exec ssh -o StrictHostKeychecking=no -o CheckHostIP=no -o UserKnownHostsFile=/dev/null -i %s $@
@@ -0,0 +1,8 @@
1
+ #!/bin/sh
2
+
3
+ for file in $(git ls-files --cached); do
4
+ if cat $file | head -n 1 | grep 'BEGIN' | grep 'PRIVATE KEY' > /dev/null; then
5
+ echo "Commit rejected: private key detected in $file."
6
+ exit 1
7
+ fi
8
+ done
@@ -0,0 +1,3 @@
1
+ #!/bin/bash
2
+ $@ 2>&1 | sed -E 's/\[[^ ]+ //'
3
+ exit ${PIPESTATUS[0]}
@@ -0,0 +1,4 @@
1
+ #!/bin/bash
2
+ if [ -z ${VIRTUAL_ENV+x} ]; then export PIP_USER=yes; fi &&
3
+ wget -nv -O - https://bootstrap.pypa.io/get-pip.py | python - --no-setuptools --no-wheel &&
4
+ pip install --upgrade --ignore-installed %{setuptools_arg} %{twine_arg} %{wheel_arg}
@@ -0,0 +1,6 @@
1
+ #!/bin/bash
2
+ curl --remote-name --location https://cli-dl.scalingo.io/release/scalingo_latest_linux_amd64.tar.gz && \
3
+ tar -zxvf scalingo_latest_linux_amd64.tar.gz && \
4
+ mv scalingo_*_linux_amd64/scalingo . && \
5
+ rm scalingo_latest_linux_amd64.tar.gz && \
6
+ rm -r scalingo_*_linux_amd64
data/lib/dpl/cli.rb CHANGED
@@ -1,66 +1,54 @@
1
- require 'dpl/error'
2
- require 'dpl/provider'
1
+ require 'cl'
3
2
 
4
- module DPL
5
- class CLI
6
- def self.run(*args)
7
- new(args).run
3
+ module Dpl
4
+ class Cli < Cl
5
+ def self.new(ctx = nil, name = 'dpl')
6
+ ctx ||= Dpl::Ctx::Bash.new
7
+ super
8
8
  end
9
9
 
10
- OPTION_PATTERN = /\A--([a-z][a-z_\-]*)(?:=(.+))?\z/
11
- attr_accessor :options, :fold_count
12
-
13
- def initialize(*args)
14
- options = {}
15
- args.flatten.each do |arg|
16
- next options.update(arg) if arg.is_a? Hash
17
- die("invalid option %p" % arg) unless match = OPTION_PATTERN.match(arg)
18
- key = match[1].tr('-', '_').to_sym
19
- if options.include? key
20
- options[key] = Array(options[key]) << match[2]
21
- else
22
- options[key] = match[2] || true
23
- end
24
- end
25
-
26
- self.fold_count = 0
27
- self.options = default_options.merge(options)
10
+ def run(args)
11
+ args = untaint(args)
12
+ args = with_provider_opt(args)
13
+ super
14
+ rescue UnknownCmd
15
+ unknown_provider(args.first)
16
+ rescue Error => e
17
+ error(e)
28
18
  end
29
19
 
30
- def run
31
- provider = Provider.new(self, options)
32
- provider.deploy
33
- rescue Error => error
34
- options[:debug] ? raise(error) : die(error.message)
20
+ # Tainting is being used for automatically obfuscating values for secure
21
+ # options, so we want to untaint all incoming args here.
22
+ def untaint(args)
23
+ args.map(&:dup).each(&:untaint)
35
24
  end
36
25
 
37
- def fold(message)
38
- self.fold_count += 1
39
- print "travis_fold:start:dpl.#{fold_count}\r" if options[:fold]
40
- puts "\e[33m#{message}\e[0m"
41
- yield
42
- ensure
43
- print "\ntravis_fold:end:dpl.#{fold_count}\r" if options[:fold]
26
+ # backwards compatibility for travis-build dpl v1 integration
27
+ def with_provider_opt(args)
28
+ return args unless arg = args.detect { |arg| arg.include?('--provider') }
29
+ args.delete(arg)
30
+ [arg.split('=').last, *args]
44
31
  end
45
32
 
46
- def default_options
47
- {
48
- :app => File.basename(Dir.pwd),
49
- :key_name => %x[hostname].strip
50
- }
33
+ def error(e)
34
+ msg = "\e[31m#{e.message}\e[0m"
35
+ msg = [msg, *e.backtrace].join("\n") if e.backtrace?
36
+ abort msg
51
37
  end
52
38
 
53
- def shell(command)
54
- system(command)
39
+ def unknown_provider(name)
40
+ msg = "\e[31mUnknown provider: #{name}\e[0m"
41
+ msg << "\nDid you mean: #{suggestions(name).join(', ')}?" if suggestions(name).any?
42
+ abort msg
55
43
  end
56
44
 
57
- def die(message)
58
- $stderr.puts(message)
59
- exit 1
45
+ def suggestions(name)
46
+ return [] unless defined?(DidYouMean)
47
+ DidYouMean::SpellChecker.new(dictionary: providers).correct(name)
60
48
  end
61
49
 
62
- def env
63
- ENV
50
+ def providers
51
+ Cl::Cmd.registry.keys.map(&:to_s)
64
52
  end
65
53
  end
66
54
  end
data/lib/dpl/ctx.rb ADDED
@@ -0,0 +1,2 @@
1
+ require 'dpl/ctx/bash'
2
+ require 'dpl/ctx/test'
@@ -0,0 +1,543 @@
1
+ require 'cl'
2
+ require 'fileutils'
3
+ require 'logger'
4
+ require 'open3'
5
+ require 'tmpdir'
6
+ require 'securerandom'
7
+ require 'dpl/support/version'
8
+
9
+ module Dpl
10
+ module Ctx
11
+ class Bash < Cl::Ctx
12
+ include FileUtils
13
+
14
+ attr_accessor :folds, :stdout, :stderr, :last_out, :last_err
15
+
16
+ def initialize(stdout = $stdout, stderr = $stderr)
17
+ @stdout, @stderr = stdout, stderr
18
+ @folds = 0
19
+ super('dpl', abort: false)
20
+ end
21
+
22
+ # Folds any log output from the given block
23
+ #
24
+ # Starts a log fold with the given fold message, calls the block, and
25
+ # closes the fold.
26
+ #
27
+ # @param msg [String] the message that will appear on the log fold
28
+ def fold(msg, &block)
29
+ self.folds += 1
30
+ print "travis_fold:start:dpl.#{folds}\r\e[K"
31
+ time do
32
+ info "\e[33m#{msg}\e[0m"
33
+ yield
34
+ end
35
+ ensure
36
+ print "\ntravis_fold:end:dpl.#{folds}\r\e[K"
37
+ end
38
+
39
+ # Times the given block
40
+ #
41
+ # Starts a travis time log tag, calls the block, and closes the tag,
42
+ # including timing information. This makes a timing badge appear on
43
+ # the surrounding log fold.
44
+ def time(&block)
45
+ id = SecureRandom.hex[0, 8]
46
+ start = Time.now.to_i * (10 ** 9)
47
+ print "travis_time:start:#{id}\r\e[K"
48
+ yield
49
+ ensure
50
+ finish = Time.now.to_i * (10 ** 9)
51
+ duration = finish - start
52
+ print "\ntravis_time:end:#{id}:start=#{start},finish=#{finish},duration=#{duration}\r\e[K"
53
+ end
54
+
55
+ # Outputs a deprecation warning for a given deprecated option key to stderr.
56
+ #
57
+ # @param key [Symbol] the deprecated option key
58
+ # @param msg [String or Symbol] the deprecation message. if given a Symbol this will be wrapped into the string "Please use #{symbol}".
59
+ def deprecate_opt(key, msg)
60
+ msg = "please use #{msg}" if msg.is_a?(Symbol)
61
+ warn "Deprecated option #{key} used (#{msg})."
62
+ end
63
+
64
+ # Outputs an info level message to stdout.
65
+ def info(*msgs)
66
+ stdout.puts(*msgs)
67
+ end
68
+
69
+ # Prints an info level message to stdout.
70
+ #
71
+ # This method does not append a newline character to the given message,
72
+ # which usually is not the desired behaviour. The method is intended to
73
+ # be used if an initial, partial message is supposed to be printed, which
74
+ # will be completed later (using the method `info`).
75
+ #
76
+ # For example:
77
+ #
78
+ # print 'Starting a long running task ...'
79
+ # run_long_running_task
80
+ # info 'done.'
81
+ def print(chars)
82
+ stdout.print(chars)
83
+ end
84
+
85
+ # Outputs an warning message to stderr
86
+ #
87
+ # This method is intended to be used for warning messages that are
88
+ # supposed to show up in the build log, but do not qualify as errors that
89
+ # would abort the deployment process. The warning will be highlighted as
90
+ # yellow text. Use sparingly.
91
+ def warn(*msgs)
92
+ msgs = msgs.join("\n").lines
93
+ msgs.each { |msg| stderr.puts("\e[33;1m#{msg}\e[0m") }
94
+ end
95
+
96
+ # Raises an exception, halting the deployment process.
97
+ #
98
+ # The calling executable `bin/dpl` will catch the exception, and abort
99
+ # the ruby process with the given error message.
100
+ #
101
+ # This method is intended to be used for all error conditions that
102
+ # require the deployment process to be aborted.
103
+ def error(message)
104
+ raise Error, message
105
+ end
106
+
107
+ # Returns a logger
108
+ #
109
+ # Returns a logger instance, with the given log level set. This can be
110
+ # used to pass to clients that accept a Ruby logger, such as Faraday,
111
+ # for debugging purposes.
112
+ #
113
+ # Use with care.
114
+ #
115
+ # @param level [Symbol] the Ruby logger log level
116
+ def logger(level = :info)
117
+ logger = Logger.new(stderr)
118
+ logger.level = Logger.const_get(level.to_s.upcase)
119
+ logger
120
+ end
121
+
122
+ def validate_runtimes(runtimes)
123
+ failed = runtimes.reject(&method(:validate_runtime))
124
+ failed = failed.map { |name, versions| "#{name} (#{versions.join(', ')})" }
125
+ error "Failed validating runtimes: #{failed.join(', ')}" if failed.any?
126
+ end
127
+
128
+ def validate_runtime(args)
129
+ name, required = *args
130
+ info "Validating required runtime version: #{name} (#{required.join(', ')})"
131
+ version = name == :node_js ? node_version : python_version
132
+ required.all? { |required| Version.new(version).satisfies?(required) }
133
+ end
134
+
135
+ def apts_get(packages)
136
+ packages = packages.reject { |name, cmd = name| which(cmd || name) }
137
+ return unless packages.any?
138
+ apt_update
139
+ packages.each { |package, cmd| apt_get(package, cmd || package, update: false) }
140
+ end
141
+
142
+ # Installs an APT package
143
+ #
144
+ # Installs the APT package with the given name, unless the command is already
145
+ # available (as determined by `which [cmd]`.
146
+ #
147
+ # @param package [String] the package name
148
+ # @param cmd [String] an executable installed by the package, defaults to the package name
149
+ def apt_get(package, cmd = package, opts = {})
150
+ return if which(cmd)
151
+ apt_update unless opts[:update].is_a?(FalseClass)
152
+ shell "sudo apt-get -qq install #{package}", retry: true
153
+ end
154
+
155
+ def apt_update
156
+ shell 'sudo apt-get update', retry: true
157
+ end
158
+
159
+ # Requires source files from Ruby gems, installing them on demand if required
160
+ #
161
+ # Installs the Ruby gems with the given version, if not already installed, and
162
+ # requires the specified source files from that gem.
163
+ #
164
+ # This happens using the bundler/inline API.
165
+ #
166
+ # @param gems [Array<String, String, Hash>] Array of gem requirements: gem name, version, and options (`require`: A single path or a list of paths to source files to require from this Ruby gem)
167
+ #
168
+ # @see https://bundler.io/v2.0/guides/bundler_in_a_single_file_ruby_script.html
169
+ def gems_require(gems)
170
+ # A local Gemfile.lock might interfer with bundler/inline, even though
171
+ # it should not. Switching to a temporary dir fixes this.
172
+ Dir.chdir(tmp_dir) do
173
+ require 'bundler/inline'
174
+ info "Installing gem dependencies: #{gems.map { |name, version, _| "#{name} #{"(#{version})" if version}".strip }.join(', ')}"
175
+ env = ENV.to_h
176
+ # Bundler.reset!
177
+ # Gem.loaded_specs.clear
178
+ gemfile do
179
+ source 'https://rubygems.org'
180
+ gems.each { |g| gem *g }
181
+ end
182
+ # https://github.com/bundler/bundler/issues/7181
183
+ ENV.replace(env)
184
+ end
185
+ end
186
+
187
+ # Installs an NPM package
188
+ #
189
+ # Installs the NPM package with the given name, unless the command is already
190
+ # available (as determined by `which [cmd]`.
191
+ #
192
+ # @param package [String] the package name
193
+ # @param cmd [String] an executable installed by the package, defaults to the package name
194
+ def npm_install(package, cmd = package)
195
+ shell "npm install -g #{package}", retry: true unless which(cmd)
196
+ end
197
+
198
+ # Installs a Python package
199
+ #
200
+ # Installs the Python package with the given name. A previously installed
201
+ # package is uninstalled before that, but only if `version` was given.
202
+ #
203
+ # @param package [String] Package name (required).
204
+ # @param cmd [String] Executable command installed by that package (optional, defaults to the package name).
205
+ # @param version [String] Package version (optional).
206
+ def pip_install(package, cmd = package, version = nil)
207
+ ENV['VIRTUAL_ENV'] = File.expand_path('~/dpl_venv')
208
+ ENV['PATH'] = File.expand_path("~/dpl_venv/bin:#{ENV['PATH']}")
209
+ shell 'virtualenv --no-site-packages ~/dpl_venv', echo: true
210
+ shell 'pip install urllib3[secure]'
211
+ cmd = "pip install #{package}"
212
+ cmd << pip_version(version) if version
213
+ shell cmd, retry: true
214
+ end
215
+
216
+ def pip_version(version)
217
+ version =~ /^\d+/ ? "==#{version}" : version
218
+ end
219
+
220
+ # Generates an SSH key
221
+ #
222
+ # @param name [String] the key name
223
+ # @param file [String] path to the key file
224
+ def ssh_keygen(name, file)
225
+ shell %(ssh-keygen -t rsa -N "" -C #{name} -f #{file})
226
+ end
227
+
228
+ # Runs a single shell command
229
+ #
230
+ # This the is the central point of executing any shell commands. It allows two
231
+ # strategies for running commands in subprocesses:
232
+ #
233
+ # * Using [Kernel#system](https://ruby-doc.org/core-2.6.3/Kernel.html#method-i-system)
234
+ # which is the default strategy, and should be used when possible. The stdout
235
+ # and stderr streams will not be captured, but streamed directly to the parent
236
+ # process (so any output on these streams appears in the build log as soon as
237
+ # possible).
238
+ #
239
+ # * Using [Open3.capture3](https://ruby-doc.org/stdlib-2.6.3/libdoc/open3/rdoc/Open3.html#method-c-capture3)
240
+ # which captures both stdout and stderr, and does not automatically output it
241
+ # to the build log. Implementors can choose to display it after the shell command
242
+ # has completed, using the `%{out}` and `%{err}` interpolation variables. Use
243
+ # sparingly.
244
+ #
245
+ # The method accepts the following options:
246
+ #
247
+ # @param cmd [String] the shell command to execute
248
+ # @param opts [Hash] options
249
+ #
250
+ # @option opts [Boolean] :echo output the command to stdout before running it
251
+ # @option opts [Boolean] :silence silence all log output by redirecting stdout and stderr to `/dev/null`
252
+ # @option opts [Boolean] :capture use `Open3.capture3` to capture stdout and stderr
253
+ # @option opts [String] :python wrap the command into Bash code that enforces the given Python version to be used
254
+ # @option opts [String] :retry retries the command 2 more times if it fails
255
+ # @option opts [String] :info message to output to stdout if the command has exited with the exit code 0 (supports the interpolation variable `${out}` for stdout in case it was captured.
256
+ # @option opts [String] :assert error message to be raised if the command has exited with a non-zero exit code (supports the interpolation variable `${out}` for stdout in case it was captured.
257
+ #
258
+ # @return [Boolean] whether or not the command was successful (has exited with the exit code 0)
259
+ def shell(cmd, opts = {})
260
+ cmd = Cmd.new(nil, cmd, opts) if cmd.is_a?(String)
261
+ info cmd.msg if cmd.msg?
262
+ info cmd.echo if cmd.echo?
263
+
264
+ @last_out, @last_err, @last_status = retrying(cmd.retry ? 2 : 0) do
265
+ send(cmd.capture? ? :open3 : :system, cmd.cmd, cmd.opts)
266
+ end
267
+
268
+ info cmd.success % { out: last_out } if success? && cmd.success?
269
+ error cmd.error % { err: last_err } if failed? && cmd.assert?
270
+
271
+ success? && cmd.capture? ? last_out.chomp : @last_status
272
+ end
273
+
274
+ def retrying(max, tries = 0, status = false)
275
+ loop do
276
+ tries += 1
277
+ out, err, status = yield
278
+ return [out, err, status] if status || tries > max
279
+ end
280
+ end
281
+
282
+ # Runs a shell command and captures stdout, stderr, and the exit status
283
+ #
284
+ # Runs the given command using `Open3.capture3`, which will capture the
285
+ # stdout and stderr streams, as well as the exit status. I.e. this will
286
+ # *not* stream log output in real time, but capture the output, and allow
287
+ # implementors to display it later (using the `%{out}` and `%{err}`
288
+ # interpolation variables.
289
+ #
290
+ # Use sparingly.
291
+ #
292
+ # @option chdir [String] directory temporarily to change to before running the command
293
+ def open3(cmd, opts)
294
+ opts = [opts[:chdir] ? only(opts, :chdir) : nil].compact
295
+ out, err, status = Open3.capture3(cmd, *opts)
296
+ [out, err, status.success?]
297
+ end
298
+
299
+ # Runs a shell command, streaming any stdout or stderr output, and
300
+ # returning the exit status
301
+ #
302
+ # This is the default method for executing shell commands. The stdout and
303
+ # stderr will not be captured, but streamed directly to the parent process.
304
+ #
305
+ # @option chdir [String] directory temporarily to change to before running the command
306
+ def system(cmd, opts = {})
307
+ opts = [opts[:chdir] ? only(opts, :chdir) : nil].compact
308
+ Kernel.system(cmd, *opts)
309
+ ['', '', last_process_status]
310
+ end
311
+
312
+ # Whether or not the last executed shell command was successful.
313
+ def success?
314
+ !!@last_status
315
+ end
316
+
317
+ # Whether or not the last executed shell command has failed.
318
+ def failed?
319
+ !success?
320
+ end
321
+
322
+ # Returns the last child process' exit status
323
+ #
324
+ # Internal, and not to be used by implementors. $? is a read-only
325
+ # variable, so we use a method that we can stub during tests.
326
+ def last_process_status
327
+ $?.success?
328
+ end
329
+
330
+ # Whether or not the current Ruby process runs with superuser priviledges.
331
+ def sudo?
332
+ Process::UID.eid == 0
333
+ end
334
+
335
+ # Returns current repository name
336
+ #
337
+ # Uses the environment variable `TRAVIS_REPO_SLUG` if present, or the
338
+ # current directory's base name.
339
+ #
340
+ # Note that this might return an unexpected string outside of the context
341
+ # of Travis CI build environments if the method is called at a time when
342
+ # the current working directory has changed.
343
+ def repo_name
344
+ ENV['TRAVIS_REPO_SLUG'] ? ENV['TRAVIS_REPO_SLUG'].split('/').last : File.basename(Dir.pwd)
345
+ end
346
+
347
+ # Returns current repository slug
348
+ #
349
+ # Uses the environment variable `TRAVIS_REPO_SLUG` if present, or the
350
+ # last two segmens of the current working directory's path.
351
+ #
352
+ # Note that this might return an unexpected string outside of the context
353
+ # of Travis CI build environments if the method is called at a time when
354
+ # the current working directory has changed.
355
+ def repo_slug
356
+ ENV['TRAVIS_REPO_SLUG'] || Dir.pwd.split('/')[-2, 2].join('/')
357
+ end
358
+
359
+ # Returns the current build directory
360
+ #
361
+ # Uses the environment variable `TRAVIS_REPO_SLUG` if present, and
362
+ # defaults to `.` otherwise.
363
+ #
364
+ # Note that this might return an unexpected string outside of the context
365
+ # of Travis CI build environments if the method is called at a time when
366
+ # the current working directory has changed.
367
+ def build_dir
368
+ ENV['TRAVIS_BUILD_DIR'] || '.'
369
+ end
370
+
371
+ # Returns the current build number
372
+ #
373
+ # Returns the value of the environment variable `TRAVIS_BUILD_NUMBER` if
374
+ # present.
375
+ def build_number
376
+ ENV['TRAVIS_BUILD_NUMBER'] || raise('TRAVIS_BUILD_NUMBER not set')
377
+ end
378
+
379
+ # Returns the encoding of the given file, as determined by `file`.
380
+ def encoding(path)
381
+ case `file '#{path}'`
382
+ when /gzip compressed/
383
+ 'gzip'
384
+ when /compress'd/
385
+ 'compress'
386
+ when /text/
387
+ 'text'
388
+ when /data/
389
+ # shrugs?
390
+ end
391
+ end
392
+
393
+ # Returns the current branch name
394
+ def git_branch
395
+ ENV['TRAVIS_BRANCH'] || git_rev_parse('HEAD')
396
+ end
397
+
398
+ # Returns the message of the commit `git_sha`.
399
+ def git_commit_msg
400
+ `git log #{git_sha} -n 1 --pretty=%B`.chomp
401
+ end
402
+
403
+ # Returns the committer name of the commit `git_sha`.
404
+ def git_author_name
405
+ `git log #{git_sha} -n 1 --pretty=%an`.chomp
406
+ end
407
+
408
+ # Returns the comitter email of the commit `git_sha`.
409
+ def git_author_email
410
+ `git log #{git_sha} -n 1 --pretty=%ae`.chomp
411
+ end
412
+
413
+ # Whether or not the git working directory is dirty
414
+ def git_dirty?
415
+ !Kernel.system('git diff --quiet')
416
+ end
417
+
418
+ # Returns the output of `git log`, using the given args.
419
+ def git_log(args)
420
+ `git log #{args}`.chomp
421
+ end
422
+
423
+ # Returns the Git log, separated by NULs
424
+ #
425
+ # Returns the output of `git ls-files -z`, which separates log entries by
426
+ # NULs, rather than newline characters.
427
+ def git_ls_files
428
+ `git ls-files -z`.split("\x0")
429
+ end
430
+
431
+ # Returns true if the given ref exists remotely
432
+ def git_ls_remote?(url, ref)
433
+ Kernel.system("git ls-remote --exit-code #{url} #{ref} > /dev/null 2>&1")
434
+ end
435
+
436
+ # Returns known Git remote URLs
437
+ def git_remote_urls
438
+ `git remote -v`.scan(/\t[^\s]+\s/).map(&:strip).uniq
439
+ end
440
+
441
+ # Returns the sha for the given Git ref
442
+ def git_rev_parse(ref)
443
+ `git rev-parse #{ref}`.strip
444
+ end
445
+
446
+ # Returns the latest tag name, if any
447
+ def git_tag
448
+ `git describe --tags --exact-match 2>/dev/null`.chomp
449
+ end
450
+
451
+ # Returns the current commit sha
452
+ def git_sha
453
+ ENV['TRAVIS_COMMIT'] || `git rev-parse HEAD`.chomp
454
+ end
455
+
456
+ # Returns the local machine's hostname
457
+ def machine_name
458
+ `hostname`.strip
459
+ end
460
+
461
+ # Returns the current Node.js version
462
+ def node_version
463
+ `node -v`.sub(/^v/, '').chomp
464
+ end
465
+
466
+ # Returns the current NPM version
467
+ def npm_version
468
+ `npm --version`
469
+ end
470
+
471
+ # Returns the current Node.js version
472
+ def python_version
473
+ `python --version 2>&1`.sub(/^Python /, '').chomp
474
+ end
475
+
476
+ # Returns true or false depending if the given command can be found
477
+ def which(cmd)
478
+ !`which #{cmd}`.chomp.empty? if cmd
479
+ end
480
+
481
+ # Returns a unique temporary directory name
482
+ def tmp_dir
483
+ @tmp_dir ||= Dir.mktmpdir
484
+ end
485
+
486
+ # Returns the size of the given file path
487
+ def file_size(path)
488
+ File.size(path)
489
+ end
490
+
491
+ def move_files(paths)
492
+ paths.each do |path|
493
+ target = "#{tmp_dir}/#{File.basename(path)}"
494
+ mv(path, target) if File.exists?(path)
495
+ end
496
+ end
497
+
498
+ def unmove_files(paths)
499
+ paths.each do |path|
500
+ source = "#{tmp_dir}/#{File.basename(path)}"
501
+ mv(source, path) if File.exists?(source)
502
+ end
503
+ end
504
+
505
+ def mv(src, dest)
506
+ Kernel.system("sudo mv #{src} #{dest} 2> /dev/null")
507
+ end
508
+
509
+ # Writes the given content to the given file path
510
+ def write_file(path, content, chmod = nil)
511
+ path = File.expand_path(path)
512
+ FileUtils.mkdir_p(File.dirname(path))
513
+ File.open(path, 'w+') { |f| f.write(content) }
514
+ FileUtils.chmod(chmod, path) if chmod
515
+ end
516
+
517
+ # Writes the given machine, login, and password to ~/.netrc
518
+ def write_netrc(machine, login, password)
519
+ require 'netrc'
520
+ netrc = Netrc.read
521
+ netrc[machine] = [login, password]
522
+ netrc.save
523
+ end
524
+
525
+ def sleep(sec)
526
+ Kernel.sleep(sec)
527
+ end
528
+
529
+ def tty?
530
+ $stdout.isatty
531
+ end
532
+
533
+ # Returns a copy of the given hash, reduced to the given keys
534
+ def only(hash, *keys)
535
+ hash.select { |key, _| keys.include?(key) }.to_h
536
+ end
537
+
538
+ def test?
539
+ false
540
+ end
541
+ end
542
+ end
543
+ end