packaging 0.99.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +17 -0
- data/README-Solaris.md +117 -0
- data/README.md +1031 -0
- data/lib/packaging.rb +32 -0
- data/lib/packaging/artifactory.rb +278 -0
- data/lib/packaging/config.rb +392 -0
- data/lib/packaging/config/params.rb +366 -0
- data/lib/packaging/deb.rb +28 -0
- data/lib/packaging/deb/repo.rb +263 -0
- data/lib/packaging/gem.rb +112 -0
- data/lib/packaging/ips.rb +57 -0
- data/lib/packaging/msi.rb +89 -0
- data/lib/packaging/nuget.rb +39 -0
- data/lib/packaging/osx.rb +36 -0
- data/lib/packaging/paths.rb +238 -0
- data/lib/packaging/platforms.rb +480 -0
- data/lib/packaging/repo.rb +55 -0
- data/lib/packaging/retrieve.rb +46 -0
- data/lib/packaging/rpm.rb +5 -0
- data/lib/packaging/rpm/repo.rb +257 -0
- data/lib/packaging/tar.rb +154 -0
- data/lib/packaging/util.rb +146 -0
- data/lib/packaging/util/date.rb +15 -0
- data/lib/packaging/util/execution.rb +85 -0
- data/lib/packaging/util/file.rb +125 -0
- data/lib/packaging/util/git.rb +174 -0
- data/lib/packaging/util/git_tags.rb +73 -0
- data/lib/packaging/util/gpg.rb +62 -0
- data/lib/packaging/util/jenkins.rb +95 -0
- data/lib/packaging/util/misc.rb +69 -0
- data/lib/packaging/util/net.rb +368 -0
- data/lib/packaging/util/os.rb +17 -0
- data/lib/packaging/util/platform.rb +40 -0
- data/lib/packaging/util/rake_utils.rb +111 -0
- data/lib/packaging/util/serialization.rb +19 -0
- data/lib/packaging/util/ship.rb +171 -0
- data/lib/packaging/util/tool.rb +41 -0
- data/lib/packaging/util/version.rb +326 -0
- data/spec/fixtures/config/ext/build_defaults.yaml +2 -0
- data/spec/fixtures/config/ext/project_data.yaml +2 -0
- data/spec/fixtures/config/params.yaml +2 -0
- data/spec/fixtures/configs/components/test_file.json +1 -0
- data/spec/fixtures/configs/components/test_file_2.json +0 -0
- data/spec/fixtures/configs/components/test_file_not_tagged.json +1 -0
- data/spec/fixtures/configs/components/test_file_wrong_ext.txt +0 -0
- data/spec/fixtures/configs/components/test_file_wrong_ext.wrong +0 -0
- data/spec/fixtures/util/pre_tasks.yaml +4 -0
- data/spec/lib/packaging/artifactory_spec.rb +171 -0
- data/spec/lib/packaging/config_spec.rb +556 -0
- data/spec/lib/packaging/deb/repo_spec.rb +148 -0
- data/spec/lib/packaging/deb_spec.rb +52 -0
- data/spec/lib/packaging/paths_spec.rb +153 -0
- data/spec/lib/packaging/platforms_spec.rb +153 -0
- data/spec/lib/packaging/repo_spec.rb +97 -0
- data/spec/lib/packaging/retrieve_spec.rb +61 -0
- data/spec/lib/packaging/rpm/repo_spec.rb +133 -0
- data/spec/lib/packaging/tar_spec.rb +122 -0
- data/spec/lib/packaging/util/execution_spec.rb +56 -0
- data/spec/lib/packaging/util/file_spec.rb +139 -0
- data/spec/lib/packaging/util/git_spec.rb +160 -0
- data/spec/lib/packaging/util/git_tag_spec.rb +36 -0
- data/spec/lib/packaging/util/gpg_spec.rb +64 -0
- data/spec/lib/packaging/util/jenkins_spec.rb +112 -0
- data/spec/lib/packaging/util/misc_spec.rb +31 -0
- data/spec/lib/packaging/util/net_spec.rb +239 -0
- data/spec/lib/packaging/util/os_spec.rb +31 -0
- data/spec/lib/packaging/util/rake_utils_spec.rb +70 -0
- data/spec/lib/packaging/util/ship_spec.rb +117 -0
- data/spec/lib/packaging/util/version_spec.rb +123 -0
- data/spec/lib/packaging_spec.rb +19 -0
- data/spec/spec_helper.rb +36 -0
- data/static_artifacts/PackageInfo.plist +3 -0
- data/tasks/00_utils.rake +216 -0
- data/tasks/30_metrics.rake +33 -0
- data/tasks/apple.rake +266 -0
- data/tasks/build.rake +12 -0
- data/tasks/clean.rake +5 -0
- data/tasks/config.rake +30 -0
- data/tasks/deb.rake +129 -0
- data/tasks/deb_repos.rake +28 -0
- data/tasks/deprecated.rake +130 -0
- data/tasks/doc.rake +20 -0
- data/tasks/education.rake +57 -0
- data/tasks/fetch.rake +57 -0
- data/tasks/gem.rake +146 -0
- data/tasks/jenkins.rake +494 -0
- data/tasks/jenkins_dynamic.rake +202 -0
- data/tasks/load_extras.rake +21 -0
- data/tasks/mock.rake +348 -0
- data/tasks/nightly_repos.rake +335 -0
- data/tasks/pe_deb.rake +12 -0
- data/tasks/pe_rpm.rake +13 -0
- data/tasks/pe_ship.rake +221 -0
- data/tasks/pe_sign.rake +13 -0
- data/tasks/pe_tar.rake +5 -0
- data/tasks/retrieve.rake +45 -0
- data/tasks/rpm.rake +66 -0
- data/tasks/rpm_repos.rake +29 -0
- data/tasks/ship.rake +752 -0
- data/tasks/sign.rake +226 -0
- data/tasks/tag.rake +8 -0
- data/tasks/tar.rake +34 -0
- data/tasks/update.rake +16 -0
- data/tasks/vanagon.rake +35 -0
- data/tasks/vendor_gems.rake +117 -0
- data/tasks/version.rake +33 -0
- data/tasks/z_data_dump.rake +65 -0
- data/templates/README +1 -0
- data/templates/downstream.xml.erb +47 -0
- data/templates/msi.xml.erb +197 -0
- data/templates/packaging.xml.erb +344 -0
- data/templates/repo.xml.erb +114 -0
- metadata +234 -0
@@ -0,0 +1,73 @@
|
|
1
|
+
module Pkg::Util
|
2
|
+
class Git_tag
|
3
|
+
attr_reader :address, :ref, :ref_name, :ref_type, :branch_name
|
4
|
+
|
5
|
+
GIT = Pkg::Util::Tool::GIT
|
6
|
+
DEVNULL = Pkg::Util::OS::DEVNULL
|
7
|
+
|
8
|
+
# A SHA1 sum is 20 characters long, but Git will match on
|
9
|
+
# the first ~8 or so. And 8 is long enough for fun test sums
|
10
|
+
# like 'cafebeef' or 'deadfeed`.
|
11
|
+
SHA1 = /[0-9A-F]{8,20}/i
|
12
|
+
|
13
|
+
def initialize(address, reference)
|
14
|
+
@address = address
|
15
|
+
@ref = reference
|
16
|
+
parse_ref!
|
17
|
+
end
|
18
|
+
|
19
|
+
# Parse ref in one of three ways: if the ref is already in a good format just grab
|
20
|
+
# the ref type from the string. if it's not, check if it's a sha, if that is true
|
21
|
+
# then list it as a sha. finally if it's neither of those fetch the full ref and
|
22
|
+
# parse that.
|
23
|
+
def parse_ref!
|
24
|
+
if ref?
|
25
|
+
split_ref(ref)
|
26
|
+
elsif sha?
|
27
|
+
@ref_type = "sha"
|
28
|
+
else
|
29
|
+
split_ref(fetch_full_ref)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
# Split the ref based on slashes, set ref_name and ref_type based on the last two
|
34
|
+
# items from the split. i.e. refs/tags/1.1.1 would return:
|
35
|
+
# @ref_name => 1.1.1 @ref_type => tags
|
36
|
+
def split_ref(ref)
|
37
|
+
ref_parts = ref.split('/', 3)
|
38
|
+
@ref_name = ref_parts.pop
|
39
|
+
@ref_type = ref_parts.pop
|
40
|
+
[@ref_type, @ref_name]
|
41
|
+
end
|
42
|
+
|
43
|
+
# Fetch the full ref using ls-remote, this should raise an error if it returns non-zero
|
44
|
+
# because that means this ref doesn't exist in the repo
|
45
|
+
def fetch_full_ref
|
46
|
+
stdout, _, _ = Pkg::Util::Execution.capture3("#{GIT} ls-remote --tags --heads --exit-code #{address} #{ref}")
|
47
|
+
stdout.split.last
|
48
|
+
rescue RuntimeError => e
|
49
|
+
raise "ERROR : Not a ref or sha!\n#{e}"
|
50
|
+
end
|
51
|
+
|
52
|
+
def branch_name
|
53
|
+
branch? ? ref_name : nil
|
54
|
+
end
|
55
|
+
|
56
|
+
def ref?
|
57
|
+
`#{GIT} check-ref-format #{ref} >#{DEVNULL} 2>&1`
|
58
|
+
$?.success?
|
59
|
+
end
|
60
|
+
|
61
|
+
def branch?
|
62
|
+
ref_type.downcase == "heads"
|
63
|
+
end
|
64
|
+
|
65
|
+
def tag?
|
66
|
+
ref_type.downcase == "tags"
|
67
|
+
end
|
68
|
+
|
69
|
+
def sha?
|
70
|
+
!!(ref =~ SHA1)
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
@@ -0,0 +1,62 @@
|
|
1
|
+
module Pkg::Util::Gpg
|
2
|
+
class << self
|
3
|
+
|
4
|
+
# Please note that this method is not used in determining what key is used
|
5
|
+
# to sign the debian repos. That is defined in the freight config that
|
6
|
+
# lives on our internal repo staging host. The debian conf/distribution
|
7
|
+
# files that are generated with this repo use the default gpg key to
|
8
|
+
# reflect that.
|
9
|
+
def key
|
10
|
+
fail "You need to set `gpg_key` in your build defaults." unless Pkg::Config.gpg_key && !Pkg::Config.gpg_key.empty?
|
11
|
+
Pkg::Config.gpg_key
|
12
|
+
end
|
13
|
+
|
14
|
+
def keychain
|
15
|
+
if @keychain.nil?
|
16
|
+
@keychain = Pkg::Util::Tool.find_tool('keychain')
|
17
|
+
else
|
18
|
+
@keychain
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def load_keychain
|
23
|
+
unless @keychain_loaded
|
24
|
+
unless ENV['RPM_GPG_AGENT']
|
25
|
+
kill_keychain
|
26
|
+
start_keychain
|
27
|
+
end
|
28
|
+
@keychain_loaded = TRUE
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def kill_keychain
|
33
|
+
if keychain
|
34
|
+
stdout, _, _ = Pkg::Util::Execution.capture3("#{keychain} -k mine")
|
35
|
+
stdout
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def start_keychain
|
40
|
+
if keychain
|
41
|
+
keychain_output, _, _ = Pkg::Util::Execution.capture3("#{keychain} -q --agents gpg --eval #{key}")
|
42
|
+
keychain_output.chomp!
|
43
|
+
new_env = keychain_output.match(/GPG_AGENT_INFO=([^;]*)/)
|
44
|
+
ENV["GPG_AGENT_INFO"] = new_env[1]
|
45
|
+
else
|
46
|
+
fail "Keychain is not installed, it is required to autosign using gpg."
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def sign_file(file)
|
51
|
+
gpg ||= Pkg::Util::Tool.find_tool('gpg')
|
52
|
+
|
53
|
+
if gpg
|
54
|
+
use_tty = "--no-tty --use-agent" if ENV['RPM_GPG_AGENT']
|
55
|
+
stdout, _, _ = Pkg::Util::Execution.capture3("#{gpg} #{use_tty} --armor --detach-sign -u #{key} #{file}")
|
56
|
+
stdout
|
57
|
+
else
|
58
|
+
fail "No gpg available. Cannot sign #{file}."
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
@@ -0,0 +1,95 @@
|
|
1
|
+
# Utility methods for handling Jenkins
|
2
|
+
require 'net/http'
|
3
|
+
require 'json'
|
4
|
+
|
5
|
+
module Pkg::Util::Jenkins
|
6
|
+
|
7
|
+
class << self
|
8
|
+
|
9
|
+
# Use the curl to create a jenkins job from a valid XML
|
10
|
+
# configuration file.
|
11
|
+
# Returns the URL to the job
|
12
|
+
def create_jenkins_job(name, xml_file)
|
13
|
+
create_url = "http://#{Pkg::Config.jenkins_build_host}/createItem?name=#{name}"
|
14
|
+
form_args = ["-H", '"Content-Type: application/xml"', "--data-binary", "@#{xml_file}"]
|
15
|
+
Pkg::Util::Net.curl_form_data(create_url, form_args)
|
16
|
+
"http://#{Pkg::Config.jenkins_build_host}/job/#{name}"
|
17
|
+
end
|
18
|
+
|
19
|
+
# Use the curl to check of a named job is defined on the jenkins server. We
|
20
|
+
# curl the config file rather than just checking if the job exists by curling
|
21
|
+
# the job url and passing --head because jenkins will mistakenly return 200 OK
|
22
|
+
# if you issue multiple very fast requests just requesting the header.
|
23
|
+
def jenkins_job_exists?(name)
|
24
|
+
job_url = "http://#{Pkg::Config.jenkins_build_host}/job/#{name}/config.xml"
|
25
|
+
form_args = ["--silent", "--fail"]
|
26
|
+
output, retval = Pkg::Util::Net.curl_form_data(job_url, form_args, :quiet => true)
|
27
|
+
return output if retval.nil?
|
28
|
+
return Pkg::Util::Execution.success?(retval)
|
29
|
+
end
|
30
|
+
|
31
|
+
# Wait for last build of job to finish.
|
32
|
+
#
|
33
|
+
# @param build_url [String] Valid build uri of a Jenkins job.
|
34
|
+
# @param polling_interval [Int] Timeout in seconds between HTTP GET on given
|
35
|
+
# build_uri.
|
36
|
+
# @param log_frequency [Int] Frequency in seconds of polling log
|
37
|
+
#
|
38
|
+
def wait_for_build(build_url, polling_interval = 2, log_frequency = 60)
|
39
|
+
$stdout.sync = true
|
40
|
+
build_hash = get_jenkins_info(build_url)
|
41
|
+
total_time = 0
|
42
|
+
while build_hash['building']
|
43
|
+
build_hash = get_jenkins_info(build_url)
|
44
|
+
sleep polling_interval
|
45
|
+
total_time += polling_interval
|
46
|
+
if total_time >= log_frequency
|
47
|
+
$stdout.puts "Polling #{build_url}..."
|
48
|
+
total_time = 0
|
49
|
+
end
|
50
|
+
end
|
51
|
+
$stdout.sync = false
|
52
|
+
return build_hash
|
53
|
+
end
|
54
|
+
|
55
|
+
# Query jenkins api and return a hash parsed from the JSON response if
|
56
|
+
# response is usable. Raise Runtime Error if response code is other than
|
57
|
+
# HTTP 200.
|
58
|
+
#
|
59
|
+
# @param url [String] Valid url of a Jenkins job.
|
60
|
+
#
|
61
|
+
def get_jenkins_info(url)
|
62
|
+
uri = URI("#{url}/api/json")
|
63
|
+
response = Net::HTTP.get_response(uri)
|
64
|
+
unless response.code == '200'
|
65
|
+
raise "Unable to query #{uri}, please check that it is valid."
|
66
|
+
end
|
67
|
+
return JSON.parse(response.body)
|
68
|
+
end
|
69
|
+
|
70
|
+
# Poll the job at the given url until it is finished, then return the final
|
71
|
+
# map of build information for calling context to do with as it pleases.
|
72
|
+
#
|
73
|
+
# Note that this method uses the build specified by the job api's lastBuild
|
74
|
+
# parameter.
|
75
|
+
#
|
76
|
+
# @param job_url [String] Valid url of a Jenkins job.
|
77
|
+
#
|
78
|
+
def poll_jenkins_job(job_url)
|
79
|
+
job_hash = get_jenkins_info(job_url)
|
80
|
+
|
81
|
+
##
|
82
|
+
# Sometimes we get a nil because we get here too soon after the jenkins
|
83
|
+
# job's build was triggered. This is kind of an ugly workaround but
|
84
|
+
# whatever.
|
85
|
+
#
|
86
|
+
while job_hash['lastBuild'].nil?
|
87
|
+
job_hash = get_jenkins_info(job_url)
|
88
|
+
sleep 1
|
89
|
+
end
|
90
|
+
|
91
|
+
wait_for_build job_hash['lastBuild']['url']
|
92
|
+
end
|
93
|
+
|
94
|
+
end
|
95
|
+
end
|
@@ -0,0 +1,69 @@
|
|
1
|
+
# A collection of utility methods that don't belong in any of our other top level Pkg::Util modules
|
2
|
+
# and probably won't anytime soon.
|
3
|
+
|
4
|
+
module Pkg::Util::Misc
|
5
|
+
class << self
|
6
|
+
# This method takes a string and a list of tokens and variables and it replaces
|
7
|
+
# the listed tokens with the matched variable if it exists. All values will
|
8
|
+
# be explicitly coerced to strings.
|
9
|
+
def search_and_replace(search_string, replacements)
|
10
|
+
raise ArgumentError "replacements must respond to #each_pair" unless
|
11
|
+
replacements.respond_to? :each_pair
|
12
|
+
|
13
|
+
replacements.each_pair do |token, value|
|
14
|
+
unless value
|
15
|
+
warn "replacement value for '#{token}' probably shouldn't be nil"
|
16
|
+
next
|
17
|
+
end
|
18
|
+
|
19
|
+
search_string.gsub!(token.to_s, value.to_s)
|
20
|
+
end
|
21
|
+
|
22
|
+
search_string
|
23
|
+
end
|
24
|
+
|
25
|
+
# Loads and parses json from a file. Will treat the keys in the
|
26
|
+
# json as methods to invoke on the component in question
|
27
|
+
#
|
28
|
+
# @param file [String] Path to the json file
|
29
|
+
# @raise [RuntimeError] exceptions are raised if there is no file, if it refers to methods that don't exist, or if it does not contain a Hash
|
30
|
+
def load_from_json(file)
|
31
|
+
data = JSON.parse(File.read(file))
|
32
|
+
unless data.is_a?(Hash)
|
33
|
+
raise "Hash required. Got '#{data.class}' when parsing '#{file}'"
|
34
|
+
end
|
35
|
+
# We explicity return data here b/c the unless clause above will cause the
|
36
|
+
# Function to return nil.
|
37
|
+
# -Sean P. M. 05/11/2016
|
38
|
+
data
|
39
|
+
end
|
40
|
+
|
41
|
+
# @param gem the name of the gem to see if it is installed
|
42
|
+
# @return boolean for whether or not the gem is installed
|
43
|
+
def check_gem(gem)
|
44
|
+
if %x(gem list -q #{gem}).chomp.empty?
|
45
|
+
return false
|
46
|
+
else
|
47
|
+
return true
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
# This loads your ~/.gem/credentials file and uses your api key
|
52
|
+
# to query rubygems.org for which gems you own. There may be better ways
|
53
|
+
# to query this but having to pull this information using curl is sort of
|
54
|
+
# gross. It works though, so ¯\_(ツ)_/¯
|
55
|
+
# @param gem_name the gem to check if you are an owner
|
56
|
+
# @return boolean whether or not you own the gem
|
57
|
+
def check_rubygems_ownership(gem_name)
|
58
|
+
require 'yaml'
|
59
|
+
credentials = YAML.load_file("#{ENV['HOME']}/.gem/credentials")
|
60
|
+
gems = YAML.load(%x(curl -H 'Authorization:#{credentials[:rubygems_api_key]}' https://rubygems.org/api/v1/gems.yaml))
|
61
|
+
gems.each do |gem|
|
62
|
+
if gem['name'] == gem_name
|
63
|
+
return true
|
64
|
+
end
|
65
|
+
end
|
66
|
+
return false
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
@@ -0,0 +1,368 @@
|
|
1
|
+
# Utility methods for handling network calls and interactions
|
2
|
+
|
3
|
+
module Pkg::Util::Net
|
4
|
+
|
5
|
+
class << self
|
6
|
+
|
7
|
+
# This simple method does an HTTP get of a URI and writes it to a file
|
8
|
+
# in a slightly more platform agnostic way than curl/wget
|
9
|
+
def fetch_uri(uri, target)
|
10
|
+
require 'open-uri'
|
11
|
+
if Pkg::Util::File.file_writable?(File.dirname(target))
|
12
|
+
File.open(target, 'w') { |f| f.puts(open(uri).read) }
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
# Get the hostname of the current host
|
17
|
+
def hostname
|
18
|
+
require 'socket'
|
19
|
+
Socket.gethostname
|
20
|
+
end
|
21
|
+
|
22
|
+
# Check that the current host matches the one we think it should
|
23
|
+
def check_host(host, args = { :required => true })
|
24
|
+
if hostname == host
|
25
|
+
return true
|
26
|
+
else
|
27
|
+
fail "#{hostname} does not match #{host}" if args[:required]
|
28
|
+
return nil
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
# @param hosts - An array of hosts to try ssh-ing into
|
33
|
+
# If the host needs a special username it should be passed
|
34
|
+
# in as user@host
|
35
|
+
# @return an array of hosts where ssh access failed. Empty array if
|
36
|
+
# successful
|
37
|
+
def check_host_ssh(hosts)
|
38
|
+
errs = []
|
39
|
+
Array(hosts).flatten.each do |host|
|
40
|
+
begin
|
41
|
+
remote_ssh_cmd(host, 'exit', false, '-oBatchMode=yes')
|
42
|
+
rescue
|
43
|
+
errs << host
|
44
|
+
end
|
45
|
+
end
|
46
|
+
return errs
|
47
|
+
end
|
48
|
+
|
49
|
+
# @param hosts - An array of hosts to check for gpg keys
|
50
|
+
# If the host needs a special username it should be passed
|
51
|
+
# in as user@host
|
52
|
+
# @param gpg - The gpg secret key to look for
|
53
|
+
# @return an array of hosts where ssh access failed. Empty array if
|
54
|
+
# successful
|
55
|
+
def check_host_gpg(hosts, gpg)
|
56
|
+
errs = []
|
57
|
+
Array(hosts).flatten.each do |host|
|
58
|
+
begin
|
59
|
+
remote_ssh_cmd(host, "gpg --list-secret-keys #{gpg} > /dev/null 2&>1", false, '-oBatchMode=yes')
|
60
|
+
rescue
|
61
|
+
errs << host
|
62
|
+
end
|
63
|
+
end
|
64
|
+
return errs
|
65
|
+
end
|
66
|
+
|
67
|
+
def remote_ssh_cmd(target, command, capture_output = false, extra_options = '', fail_fast = true)
|
68
|
+
ssh = Pkg::Util::Tool.check_tool('ssh')
|
69
|
+
|
70
|
+
# we pass some pretty complicated commands in via ssh. We need this to fail
|
71
|
+
# if any part of the remote ssh command fails.
|
72
|
+
command = "set -e; #{command}" if fail_fast
|
73
|
+
cmd = "#{ssh} #{extra_options} -t #{target} '#{command.gsub("'", "'\\\\''")}'"
|
74
|
+
|
75
|
+
# This is NOT a good way to support this functionality.
|
76
|
+
# It needs to be refactored into a set of methods that
|
77
|
+
# other methods can use to safely and deterministically
|
78
|
+
# support dry-run operations.
|
79
|
+
# But I need to ship packages RIGHT NOW.
|
80
|
+
# - Ryan McKern, 13/01/2016
|
81
|
+
if ENV['DRYRUN']
|
82
|
+
puts "[DRY-RUN] Executing '#{command}' on #{target}"
|
83
|
+
puts "[DRY-RUN] #{cmd}"
|
84
|
+
return
|
85
|
+
end
|
86
|
+
|
87
|
+
puts "Executing '#{command}' on #{target}"
|
88
|
+
if capture_output
|
89
|
+
stdout, stderr, exitstatus = Pkg::Util::Execution.capture3(cmd)
|
90
|
+
Pkg::Util::Execution.success?(exitstatus) or raise "Remote ssh command failed."
|
91
|
+
return stdout, stderr
|
92
|
+
else
|
93
|
+
Kernel.system(cmd)
|
94
|
+
Pkg::Util::Execution.success? or raise "Remote ssh command failed."
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
# Construct a valid rsync command
|
99
|
+
# @return [String] a rsync command that can be used in shell or ssh methods
|
100
|
+
# @param [String, Pathname] origin_path the path to sync from; if opts[:target_path]
|
101
|
+
# is not passed, then the parent directory of `origin_path` will be used to
|
102
|
+
# construct a target path to sync to.
|
103
|
+
# @param [Hash] opts additional options that can be used to construct
|
104
|
+
# the rsync command.
|
105
|
+
# @option opts [String] :bin ('rsync') the path to rsync
|
106
|
+
# (can be relative or fully qualified).
|
107
|
+
# @option opts [String] :origin_host the remote host to sync data from; cannot
|
108
|
+
# be specified alongside :target_host
|
109
|
+
# @option opts [String] :target_host the remote host to sync data to; cannot
|
110
|
+
# be specified alongside :origin_host.
|
111
|
+
# @option opts [String] :extra_flags (["--ignore-existing"]) extra flags to
|
112
|
+
# use when constructing an rsync command
|
113
|
+
# @option opts [String] :dryrun (false) tell rsync to perform a trial run
|
114
|
+
# with no changes made.
|
115
|
+
# @raise [ArgumentError] if opts[:origin_host] and opts[:target_host] names
|
116
|
+
# are both defined.
|
117
|
+
# @raise [ArgumentError] if :origin_path exists without opts[:target_path],
|
118
|
+
# opts[:origin_host], remote target is defined.
|
119
|
+
def rsync_cmd(origin_path, opts = {})
|
120
|
+
options = {
|
121
|
+
bin: 'rsync',
|
122
|
+
origin_host: nil,
|
123
|
+
target_path: nil,
|
124
|
+
target_host: nil,
|
125
|
+
extra_flags: nil,
|
126
|
+
dryrun: false }.merge(opts)
|
127
|
+
origin = Pathname.new(origin_path)
|
128
|
+
target = options[:target_path] || origin.parent
|
129
|
+
|
130
|
+
raise(ArgumentError, "Cannot sync between two remote hosts") if
|
131
|
+
options[:origin_host] && options[:target_host]
|
132
|
+
|
133
|
+
raise(ArgumentError, "Cannot sync path '#{origin}' to itself") unless
|
134
|
+
options[:origin_host] || options[:target_host]
|
135
|
+
|
136
|
+
cmd = %W(
|
137
|
+
#{options[:bin]}
|
138
|
+
--recursive
|
139
|
+
--hard-links
|
140
|
+
--links
|
141
|
+
--verbose
|
142
|
+
--omit-dir-times
|
143
|
+
--no-perms
|
144
|
+
--no-owner
|
145
|
+
--no-group
|
146
|
+
) + [*options[:extra_flags]]
|
147
|
+
|
148
|
+
cmd << '--dry-run' if options[:dryrun]
|
149
|
+
cmd << Pkg::Util.pseudo_uri(path: origin, host: options[:origin_host])
|
150
|
+
cmd << Pkg::Util.pseudo_uri(path: target, host: options[:target_host])
|
151
|
+
|
152
|
+
cmd.uniq.compact.join("\s")
|
153
|
+
end
|
154
|
+
|
155
|
+
# A generic rsync execution method that wraps rsync_cmd in a
|
156
|
+
# call to Pkg::Util::Execution#capture3()
|
157
|
+
def rsync_exec(source, opts = {})
|
158
|
+
options = {
|
159
|
+
bin: Pkg::Util::Tool.check_tool('rsync'),
|
160
|
+
origin_host: nil,
|
161
|
+
target_path: nil,
|
162
|
+
target_host: nil,
|
163
|
+
extra_flags: nil,
|
164
|
+
dryrun: ENV['DRYRUN'] }.merge(opts.delete_if { |_, value| value.nil? })
|
165
|
+
|
166
|
+
stdout, _, _ = Pkg::Util::Execution.capture3(rsync_cmd(source, options), true)
|
167
|
+
stdout
|
168
|
+
end
|
169
|
+
|
170
|
+
# A wrapper method to maintain the existing interface for executing
|
171
|
+
# outbound rsync commands with minimal changes to existing code.
|
172
|
+
def rsync_to(source, target_host, dest, opts = { extra_flags: ["--ignore-existing"] })
|
173
|
+
rsync_exec(
|
174
|
+
source,
|
175
|
+
target_host: target_host,
|
176
|
+
target_path: dest,
|
177
|
+
extra_flags: opts[:extra_flags],
|
178
|
+
dryrun: opts[:dryrun],
|
179
|
+
bin: opts[:bin],
|
180
|
+
)
|
181
|
+
end
|
182
|
+
|
183
|
+
# A wrapper method to maintain the existing interface for executing
|
184
|
+
# incoming rsync commands with minimal changes to existing code.
|
185
|
+
def rsync_from(source, origin_host, dest, opts = {})
|
186
|
+
rsync_exec(
|
187
|
+
source,
|
188
|
+
origin_host: origin_host,
|
189
|
+
target_path: dest,
|
190
|
+
extra_flags: opts[:extra_flags],
|
191
|
+
dryrun: opts[:dryrun],
|
192
|
+
bin: opts[:bin],
|
193
|
+
)
|
194
|
+
end
|
195
|
+
|
196
|
+
def s3sync_to(source, target_bucket, target_directory = "", flags = [])
|
197
|
+
s3cmd = Pkg::Util::Tool.check_tool('s3cmd')
|
198
|
+
|
199
|
+
if Pkg::Util::File.file_exists?(File.join(ENV['HOME'], '.s3cfg'))
|
200
|
+
stdout, _, _ = Pkg::Util::Execution.capture3("#{s3cmd} sync #{flags.join(' ')} '#{source}' s3://#{target_bucket}/#{target_directory}/")
|
201
|
+
stdout
|
202
|
+
else
|
203
|
+
fail "#{File.join(ENV['HOME'], '.s3cfg')} does not exist. It is required to ship files using s3cmd."
|
204
|
+
end
|
205
|
+
end
|
206
|
+
|
207
|
+
# This is fairly absurd. We're implementing curl by shelling out. What do I
|
208
|
+
# wish we were doing? Using a sweet ruby wrapper around curl, such as Curb or
|
209
|
+
# Curb-fu. However, because we're using clean build systems and trying to
|
210
|
+
# make this portable with minimal system requirements, we can't very well
|
211
|
+
# depend on libraries that aren't in the ruby standard libaries. We could
|
212
|
+
# also do this using Net::HTTP but that set of libraries is a rabbit hole to
|
213
|
+
# go down when what we're trying to accomplish is posting multi-part form
|
214
|
+
# data that includes file uploads to jenkins. It gets hairy fairly quickly,
|
215
|
+
# but, as they say, pull requests accepted.
|
216
|
+
#
|
217
|
+
# This method takes three arguments
|
218
|
+
# 1) String - the URL to post to
|
219
|
+
# 2) Array - Ordered array of name=VALUE curl form parameters
|
220
|
+
# 3) Hash - Options to be set
|
221
|
+
def curl_form_data(uri, form_data = [], options = {})
|
222
|
+
curl = Pkg::Util::Tool.check_tool("curl")
|
223
|
+
#
|
224
|
+
# Begin constructing the post string.
|
225
|
+
# First, assemble the form_data arguments
|
226
|
+
#
|
227
|
+
post_string = "-i "
|
228
|
+
form_data.each do |param|
|
229
|
+
post_string << "#{param} "
|
230
|
+
end
|
231
|
+
|
232
|
+
# Add the uri
|
233
|
+
post_string << "'#{uri}'"
|
234
|
+
|
235
|
+
# If this is quiet, we're going to silence all output
|
236
|
+
begin
|
237
|
+
stdout, _, retval = Pkg::Util::Execution.capture3("#{curl} #{post_string}")
|
238
|
+
if options[:quiet]
|
239
|
+
stdout = ''
|
240
|
+
end
|
241
|
+
return stdout, retval
|
242
|
+
rescue RuntimeError => e
|
243
|
+
puts e
|
244
|
+
return false
|
245
|
+
end
|
246
|
+
end
|
247
|
+
|
248
|
+
def uri_status_code(uri)
|
249
|
+
data = [
|
250
|
+
'--request GET',
|
251
|
+
'--silent',
|
252
|
+
'--location',
|
253
|
+
'--write-out "%{http_code}"',
|
254
|
+
'--output /dev/null'
|
255
|
+
]
|
256
|
+
stdout, _ = Pkg::Util::Net.curl_form_data(uri, data)
|
257
|
+
stdout
|
258
|
+
end
|
259
|
+
|
260
|
+
# Use the provided URL string to print important information with
|
261
|
+
# ASCII emphasis
|
262
|
+
def print_url_info(url_string)
|
263
|
+
puts "\n////////////////////////////////////////////////////////////////////////////////\n\n
|
264
|
+
Build submitted. To view your build progress, go to\n#{url_string}\n\n
|
265
|
+
////////////////////////////////////////////////////////////////////////////////\n\n"
|
266
|
+
end
|
267
|
+
|
268
|
+
def remote_set_ownership(host, owner, group, files)
|
269
|
+
remote_cmd = "for file in #{files.join(" ")}; do if [[ -d $file ]] || ! `lsattr $file | grep -q '\\-i\\-'`; then sudo chown #{owner}:#{group} $file; else echo \"$file is immutable\"; fi; done"
|
270
|
+
Pkg::Util::Net.remote_ssh_cmd(host, remote_cmd)
|
271
|
+
end
|
272
|
+
|
273
|
+
def remote_set_permissions(host, permissions, files)
|
274
|
+
remote_cmd = "for file in #{files.join(" ")}; do if [[ -d $file ]] || ! `lsattr $file | grep -q '\\-i\\-'`; then sudo chmod #{permissions} $file; else echo \"$file is immutable\"; fi; done"
|
275
|
+
Pkg::Util::Net.remote_ssh_cmd(host, remote_cmd)
|
276
|
+
end
|
277
|
+
|
278
|
+
# Remotely set the immutable bit on a list of files
|
279
|
+
def remote_set_immutable(host, files)
|
280
|
+
Pkg::Util::Net.remote_ssh_cmd(host, "sudo chattr +i #{files.join(" ")}")
|
281
|
+
end
|
282
|
+
|
283
|
+
# Create a symlink indicating the latest version of a package
|
284
|
+
#
|
285
|
+
# @param package_name [String] The name of the package you want to symlink
|
286
|
+
# to, e.g. 'puppet-agent', 'facter', etc.
|
287
|
+
# @param dir [String] The directory you want to find the latest package and
|
288
|
+
# create the symlink in.
|
289
|
+
# @param platform_ext [String] The type of files you want to consider, e.g.
|
290
|
+
# 'dmg', 'msi', etc.
|
291
|
+
# @param options [Hash] Additional optional params:
|
292
|
+
# @option :arch [String] Architecture you want to narrow your search by.
|
293
|
+
# @option :excludes [Array] Strings you want to exclude from your search,
|
294
|
+
# e.g. 'agent' if only searching for 'puppet'.
|
295
|
+
def remote_create_latest_symlink(package_name, dir, platform_ext, options = {})
|
296
|
+
ls_cmd = "ls -1 *.#{platform_ext} | grep -v latest | grep -v rc | grep #{package_name} "
|
297
|
+
if options[:arch]
|
298
|
+
ls_cmd << "| grep #{options[:arch]}"
|
299
|
+
package_name << "-#{options[:arch]}"
|
300
|
+
end
|
301
|
+
if options[:excludes]
|
302
|
+
options[:excludes].each do |excl|
|
303
|
+
ls_cmd << "| grep -v #{excl} "
|
304
|
+
end
|
305
|
+
end
|
306
|
+
ls_cmd << '| sort --version-sort | tail -1'
|
307
|
+
cmd = <<-CMD
|
308
|
+
if [ ! -d '#{dir}' ] ; then
|
309
|
+
echo "directory '#{dir}' does not exist, not creating latest package link"
|
310
|
+
exit 0
|
311
|
+
fi
|
312
|
+
pushd '#{dir}'
|
313
|
+
link_target=$(#{ls_cmd})
|
314
|
+
if [ -z "$link_target" ] ; then
|
315
|
+
echo "Unable to find a link target for '#{package_name}' in '#{dir}'; skipping link creation"
|
316
|
+
exit 0
|
317
|
+
fi
|
318
|
+
echo "creating link to '$link_target'"
|
319
|
+
ln -sf "$link_target" #{package_name}-latest.#{platform_ext}
|
320
|
+
CMD
|
321
|
+
|
322
|
+
_, err = Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.staging_server, cmd, true)
|
323
|
+
$stderr.puts err
|
324
|
+
end
|
325
|
+
|
326
|
+
def escape_html(uri)
|
327
|
+
require 'cgi'
|
328
|
+
CGI.escapeHTML(uri)
|
329
|
+
end
|
330
|
+
|
331
|
+
# Add a parameter to a given uri. If we were sane we'd use
|
332
|
+
# encode_www_form(params) of URI, but because we're not, because that will http
|
333
|
+
# encode it, which isn't what we want since we're require the encoding provided
|
334
|
+
# by escapeHTML of CGI, since this is being transfered in the xml of a jenkins
|
335
|
+
# job via curl and DEAR JEEBUS WHAT HAVE WE DONE.
|
336
|
+
def add_param_to_uri(uri, param)
|
337
|
+
require 'uri'
|
338
|
+
uri = URI.parse(uri)
|
339
|
+
uri.query = [uri.query, param].compact.join('&')
|
340
|
+
uri.to_s
|
341
|
+
end
|
342
|
+
|
343
|
+
# We take a tar argument for cases where `tar` isn't best, e.g. Solaris. We
|
344
|
+
# also take an optional argument of the tarball containing the git bundle to
|
345
|
+
# use.
|
346
|
+
def remote_bootstrap(host, treeish, tar_cmd = nil, tarball = nil)
|
347
|
+
unless tar = tar_cmd
|
348
|
+
tar = 'tar'
|
349
|
+
end
|
350
|
+
tarball ||= Pkg::Util::Git.bundle(treeish)
|
351
|
+
tarball_name = File.basename(tarball).gsub('.tar.gz', '')
|
352
|
+
Pkg::Util::Net.rsync_to(tarball, host, '/tmp')
|
353
|
+
appendix = Pkg::Util.rand_string
|
354
|
+
Pkg::Util::Net.remote_ssh_cmd(host, "#{tar} -zxvf /tmp/#{tarball_name}.tar.gz -C /tmp/ ; git clone --recursive /tmp/#{tarball_name} /tmp/#{Pkg::Config.project}-#{appendix} ; cd /tmp/#{Pkg::Config.project}-#{appendix} ; rake package:bootstrap")
|
355
|
+
"/tmp/#{Pkg::Config.project}-#{appendix}"
|
356
|
+
end
|
357
|
+
|
358
|
+
# Given a BuildInstance object and a host, send its params to the host. Return
|
359
|
+
# the remote path to the params.
|
360
|
+
def remote_buildparams(host, build)
|
361
|
+
params_file = build.config_to_yaml
|
362
|
+
params_file_name = File.basename(params_file)
|
363
|
+
params_dir = Pkg::Util.rand_string
|
364
|
+
Pkg::Util::Net.rsync_to(params_file, host, "/tmp/#{params_dir}/")
|
365
|
+
"/tmp/#{params_dir}/#{params_file_name}"
|
366
|
+
end
|
367
|
+
end
|
368
|
+
end
|