packaging 0.99.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +17 -0
  3. data/README-Solaris.md +117 -0
  4. data/README.md +1031 -0
  5. data/lib/packaging.rb +32 -0
  6. data/lib/packaging/artifactory.rb +278 -0
  7. data/lib/packaging/config.rb +392 -0
  8. data/lib/packaging/config/params.rb +366 -0
  9. data/lib/packaging/deb.rb +28 -0
  10. data/lib/packaging/deb/repo.rb +263 -0
  11. data/lib/packaging/gem.rb +112 -0
  12. data/lib/packaging/ips.rb +57 -0
  13. data/lib/packaging/msi.rb +89 -0
  14. data/lib/packaging/nuget.rb +39 -0
  15. data/lib/packaging/osx.rb +36 -0
  16. data/lib/packaging/paths.rb +238 -0
  17. data/lib/packaging/platforms.rb +480 -0
  18. data/lib/packaging/repo.rb +55 -0
  19. data/lib/packaging/retrieve.rb +46 -0
  20. data/lib/packaging/rpm.rb +5 -0
  21. data/lib/packaging/rpm/repo.rb +257 -0
  22. data/lib/packaging/tar.rb +154 -0
  23. data/lib/packaging/util.rb +146 -0
  24. data/lib/packaging/util/date.rb +15 -0
  25. data/lib/packaging/util/execution.rb +85 -0
  26. data/lib/packaging/util/file.rb +125 -0
  27. data/lib/packaging/util/git.rb +174 -0
  28. data/lib/packaging/util/git_tags.rb +73 -0
  29. data/lib/packaging/util/gpg.rb +62 -0
  30. data/lib/packaging/util/jenkins.rb +95 -0
  31. data/lib/packaging/util/misc.rb +69 -0
  32. data/lib/packaging/util/net.rb +368 -0
  33. data/lib/packaging/util/os.rb +17 -0
  34. data/lib/packaging/util/platform.rb +40 -0
  35. data/lib/packaging/util/rake_utils.rb +111 -0
  36. data/lib/packaging/util/serialization.rb +19 -0
  37. data/lib/packaging/util/ship.rb +171 -0
  38. data/lib/packaging/util/tool.rb +41 -0
  39. data/lib/packaging/util/version.rb +326 -0
  40. data/spec/fixtures/config/ext/build_defaults.yaml +2 -0
  41. data/spec/fixtures/config/ext/project_data.yaml +2 -0
  42. data/spec/fixtures/config/params.yaml +2 -0
  43. data/spec/fixtures/configs/components/test_file.json +1 -0
  44. data/spec/fixtures/configs/components/test_file_2.json +0 -0
  45. data/spec/fixtures/configs/components/test_file_not_tagged.json +1 -0
  46. data/spec/fixtures/configs/components/test_file_wrong_ext.txt +0 -0
  47. data/spec/fixtures/configs/components/test_file_wrong_ext.wrong +0 -0
  48. data/spec/fixtures/util/pre_tasks.yaml +4 -0
  49. data/spec/lib/packaging/artifactory_spec.rb +171 -0
  50. data/spec/lib/packaging/config_spec.rb +556 -0
  51. data/spec/lib/packaging/deb/repo_spec.rb +148 -0
  52. data/spec/lib/packaging/deb_spec.rb +52 -0
  53. data/spec/lib/packaging/paths_spec.rb +153 -0
  54. data/spec/lib/packaging/platforms_spec.rb +153 -0
  55. data/spec/lib/packaging/repo_spec.rb +97 -0
  56. data/spec/lib/packaging/retrieve_spec.rb +61 -0
  57. data/spec/lib/packaging/rpm/repo_spec.rb +133 -0
  58. data/spec/lib/packaging/tar_spec.rb +122 -0
  59. data/spec/lib/packaging/util/execution_spec.rb +56 -0
  60. data/spec/lib/packaging/util/file_spec.rb +139 -0
  61. data/spec/lib/packaging/util/git_spec.rb +160 -0
  62. data/spec/lib/packaging/util/git_tag_spec.rb +36 -0
  63. data/spec/lib/packaging/util/gpg_spec.rb +64 -0
  64. data/spec/lib/packaging/util/jenkins_spec.rb +112 -0
  65. data/spec/lib/packaging/util/misc_spec.rb +31 -0
  66. data/spec/lib/packaging/util/net_spec.rb +239 -0
  67. data/spec/lib/packaging/util/os_spec.rb +31 -0
  68. data/spec/lib/packaging/util/rake_utils_spec.rb +70 -0
  69. data/spec/lib/packaging/util/ship_spec.rb +117 -0
  70. data/spec/lib/packaging/util/version_spec.rb +123 -0
  71. data/spec/lib/packaging_spec.rb +19 -0
  72. data/spec/spec_helper.rb +36 -0
  73. data/static_artifacts/PackageInfo.plist +3 -0
  74. data/tasks/00_utils.rake +216 -0
  75. data/tasks/30_metrics.rake +33 -0
  76. data/tasks/apple.rake +266 -0
  77. data/tasks/build.rake +12 -0
  78. data/tasks/clean.rake +5 -0
  79. data/tasks/config.rake +30 -0
  80. data/tasks/deb.rake +129 -0
  81. data/tasks/deb_repos.rake +28 -0
  82. data/tasks/deprecated.rake +130 -0
  83. data/tasks/doc.rake +20 -0
  84. data/tasks/education.rake +57 -0
  85. data/tasks/fetch.rake +57 -0
  86. data/tasks/gem.rake +146 -0
  87. data/tasks/jenkins.rake +494 -0
  88. data/tasks/jenkins_dynamic.rake +202 -0
  89. data/tasks/load_extras.rake +21 -0
  90. data/tasks/mock.rake +348 -0
  91. data/tasks/nightly_repos.rake +335 -0
  92. data/tasks/pe_deb.rake +12 -0
  93. data/tasks/pe_rpm.rake +13 -0
  94. data/tasks/pe_ship.rake +221 -0
  95. data/tasks/pe_sign.rake +13 -0
  96. data/tasks/pe_tar.rake +5 -0
  97. data/tasks/retrieve.rake +45 -0
  98. data/tasks/rpm.rake +66 -0
  99. data/tasks/rpm_repos.rake +29 -0
  100. data/tasks/ship.rake +752 -0
  101. data/tasks/sign.rake +226 -0
  102. data/tasks/tag.rake +8 -0
  103. data/tasks/tar.rake +34 -0
  104. data/tasks/update.rake +16 -0
  105. data/tasks/vanagon.rake +35 -0
  106. data/tasks/vendor_gems.rake +117 -0
  107. data/tasks/version.rake +33 -0
  108. data/tasks/z_data_dump.rake +65 -0
  109. data/templates/README +1 -0
  110. data/templates/downstream.xml.erb +47 -0
  111. data/templates/msi.xml.erb +197 -0
  112. data/templates/packaging.xml.erb +344 -0
  113. data/templates/repo.xml.erb +114 -0
  114. metadata +234 -0
@@ -0,0 +1,73 @@
1
+ module Pkg::Util
2
+ class Git_tag
3
+ attr_reader :address, :ref, :ref_name, :ref_type, :branch_name
4
+
5
+ GIT = Pkg::Util::Tool::GIT
6
+ DEVNULL = Pkg::Util::OS::DEVNULL
7
+
8
+ # A SHA1 sum is 20 characters long, but Git will match on
9
+ # the first ~8 or so. And 8 is long enough for fun test sums
10
+ # like 'cafebeef' or 'deadfeed`.
11
+ SHA1 = /[0-9A-F]{8,20}/i
12
+
13
+ def initialize(address, reference)
14
+ @address = address
15
+ @ref = reference
16
+ parse_ref!
17
+ end
18
+
19
+ # Parse ref in one of three ways: if the ref is already in a good format just grab
20
+ # the ref type from the string. if it's not, check if it's a sha, if that is true
21
+ # then list it as a sha. finally if it's neither of those fetch the full ref and
22
+ # parse that.
23
+ def parse_ref!
24
+ if ref?
25
+ split_ref(ref)
26
+ elsif sha?
27
+ @ref_type = "sha"
28
+ else
29
+ split_ref(fetch_full_ref)
30
+ end
31
+ end
32
+
33
+ # Split the ref based on slashes, set ref_name and ref_type based on the last two
34
+ # items from the split. i.e. refs/tags/1.1.1 would return:
35
+ # @ref_name => 1.1.1 @ref_type => tags
36
+ def split_ref(ref)
37
+ ref_parts = ref.split('/', 3)
38
+ @ref_name = ref_parts.pop
39
+ @ref_type = ref_parts.pop
40
+ [@ref_type, @ref_name]
41
+ end
42
+
43
+ # Fetch the full ref using ls-remote, this should raise an error if it returns non-zero
44
+ # because that means this ref doesn't exist in the repo
45
+ def fetch_full_ref
46
+ stdout, _, _ = Pkg::Util::Execution.capture3("#{GIT} ls-remote --tags --heads --exit-code #{address} #{ref}")
47
+ stdout.split.last
48
+ rescue RuntimeError => e
49
+ raise "ERROR : Not a ref or sha!\n#{e}"
50
+ end
51
+
52
+ def branch_name
53
+ branch? ? ref_name : nil
54
+ end
55
+
56
+ def ref?
57
+ `#{GIT} check-ref-format #{ref} >#{DEVNULL} 2>&1`
58
+ $?.success?
59
+ end
60
+
61
+ def branch?
62
+ ref_type.downcase == "heads"
63
+ end
64
+
65
+ def tag?
66
+ ref_type.downcase == "tags"
67
+ end
68
+
69
+ def sha?
70
+ !!(ref =~ SHA1)
71
+ end
72
+ end
73
+ end
@@ -0,0 +1,62 @@
1
+ module Pkg::Util::Gpg
2
+ class << self
3
+
4
+ # Please note that this method is not used in determining what key is used
5
+ # to sign the debian repos. That is defined in the freight config that
6
+ # lives on our internal repo staging host. The debian conf/distribution
7
+ # files that are generated with this repo use the default gpg key to
8
+ # reflect that.
9
+ def key
10
+ fail "You need to set `gpg_key` in your build defaults." unless Pkg::Config.gpg_key && !Pkg::Config.gpg_key.empty?
11
+ Pkg::Config.gpg_key
12
+ end
13
+
14
+ def keychain
15
+ if @keychain.nil?
16
+ @keychain = Pkg::Util::Tool.find_tool('keychain')
17
+ else
18
+ @keychain
19
+ end
20
+ end
21
+
22
+ def load_keychain
23
+ unless @keychain_loaded
24
+ unless ENV['RPM_GPG_AGENT']
25
+ kill_keychain
26
+ start_keychain
27
+ end
28
+ @keychain_loaded = TRUE
29
+ end
30
+ end
31
+
32
+ def kill_keychain
33
+ if keychain
34
+ stdout, _, _ = Pkg::Util::Execution.capture3("#{keychain} -k mine")
35
+ stdout
36
+ end
37
+ end
38
+
39
+ def start_keychain
40
+ if keychain
41
+ keychain_output, _, _ = Pkg::Util::Execution.capture3("#{keychain} -q --agents gpg --eval #{key}")
42
+ keychain_output.chomp!
43
+ new_env = keychain_output.match(/GPG_AGENT_INFO=([^;]*)/)
44
+ ENV["GPG_AGENT_INFO"] = new_env[1]
45
+ else
46
+ fail "Keychain is not installed, it is required to autosign using gpg."
47
+ end
48
+ end
49
+
50
+ def sign_file(file)
51
+ gpg ||= Pkg::Util::Tool.find_tool('gpg')
52
+
53
+ if gpg
54
+ use_tty = "--no-tty --use-agent" if ENV['RPM_GPG_AGENT']
55
+ stdout, _, _ = Pkg::Util::Execution.capture3("#{gpg} #{use_tty} --armor --detach-sign -u #{key} #{file}")
56
+ stdout
57
+ else
58
+ fail "No gpg available. Cannot sign #{file}."
59
+ end
60
+ end
61
+ end
62
+ end
@@ -0,0 +1,95 @@
1
+ # Utility methods for handling Jenkins
2
+ require 'net/http'
3
+ require 'json'
4
+
5
+ module Pkg::Util::Jenkins
6
+
7
+ class << self
8
+
9
+ # Use the curl to create a jenkins job from a valid XML
10
+ # configuration file.
11
+ # Returns the URL to the job
12
+ def create_jenkins_job(name, xml_file)
13
+ create_url = "http://#{Pkg::Config.jenkins_build_host}/createItem?name=#{name}"
14
+ form_args = ["-H", '"Content-Type: application/xml"', "--data-binary", "@#{xml_file}"]
15
+ Pkg::Util::Net.curl_form_data(create_url, form_args)
16
+ "http://#{Pkg::Config.jenkins_build_host}/job/#{name}"
17
+ end
18
+
19
+ # Use the curl to check of a named job is defined on the jenkins server. We
20
+ # curl the config file rather than just checking if the job exists by curling
21
+ # the job url and passing --head because jenkins will mistakenly return 200 OK
22
+ # if you issue multiple very fast requests just requesting the header.
23
+ def jenkins_job_exists?(name)
24
+ job_url = "http://#{Pkg::Config.jenkins_build_host}/job/#{name}/config.xml"
25
+ form_args = ["--silent", "--fail"]
26
+ output, retval = Pkg::Util::Net.curl_form_data(job_url, form_args, :quiet => true)
27
+ return output if retval.nil?
28
+ return Pkg::Util::Execution.success?(retval)
29
+ end
30
+
31
+ # Wait for last build of job to finish.
32
+ #
33
+ # @param build_url [String] Valid build uri of a Jenkins job.
34
+ # @param polling_interval [Int] Timeout in seconds between HTTP GET on given
35
+ # build_uri.
36
+ # @param log_frequency [Int] Frequency in seconds of polling log
37
+ #
38
+ def wait_for_build(build_url, polling_interval = 2, log_frequency = 60)
39
+ $stdout.sync = true
40
+ build_hash = get_jenkins_info(build_url)
41
+ total_time = 0
42
+ while build_hash['building']
43
+ build_hash = get_jenkins_info(build_url)
44
+ sleep polling_interval
45
+ total_time += polling_interval
46
+ if total_time >= log_frequency
47
+ $stdout.puts "Polling #{build_url}..."
48
+ total_time = 0
49
+ end
50
+ end
51
+ $stdout.sync = false
52
+ return build_hash
53
+ end
54
+
55
+ # Query jenkins api and return a hash parsed from the JSON response if
56
+ # response is usable. Raise Runtime Error if response code is other than
57
+ # HTTP 200.
58
+ #
59
+ # @param url [String] Valid url of a Jenkins job.
60
+ #
61
+ def get_jenkins_info(url)
62
+ uri = URI("#{url}/api/json")
63
+ response = Net::HTTP.get_response(uri)
64
+ unless response.code == '200'
65
+ raise "Unable to query #{uri}, please check that it is valid."
66
+ end
67
+ return JSON.parse(response.body)
68
+ end
69
+
70
+ # Poll the job at the given url until it is finished, then return the final
71
+ # map of build information for calling context to do with as it pleases.
72
+ #
73
+ # Note that this method uses the build specified by the job api's lastBuild
74
+ # parameter.
75
+ #
76
+ # @param job_url [String] Valid url of a Jenkins job.
77
+ #
78
+ def poll_jenkins_job(job_url)
79
+ job_hash = get_jenkins_info(job_url)
80
+
81
+ ##
82
+ # Sometimes we get a nil because we get here too soon after the jenkins
83
+ # job's build was triggered. This is kind of an ugly workaround but
84
+ # whatever.
85
+ #
86
+ while job_hash['lastBuild'].nil?
87
+ job_hash = get_jenkins_info(job_url)
88
+ sleep 1
89
+ end
90
+
91
+ wait_for_build job_hash['lastBuild']['url']
92
+ end
93
+
94
+ end
95
+ end
@@ -0,0 +1,69 @@
1
+ # A collection of utility methods that don't belong in any of our other top level Pkg::Util modules
2
+ # and probably won't anytime soon.
3
+
4
+ module Pkg::Util::Misc
5
+ class << self
6
+ # This method takes a string and a list of tokens and variables and it replaces
7
+ # the listed tokens with the matched variable if it exists. All values will
8
+ # be explicitly coerced to strings.
9
+ def search_and_replace(search_string, replacements)
10
+ raise ArgumentError "replacements must respond to #each_pair" unless
11
+ replacements.respond_to? :each_pair
12
+
13
+ replacements.each_pair do |token, value|
14
+ unless value
15
+ warn "replacement value for '#{token}' probably shouldn't be nil"
16
+ next
17
+ end
18
+
19
+ search_string.gsub!(token.to_s, value.to_s)
20
+ end
21
+
22
+ search_string
23
+ end
24
+
25
+ # Loads and parses json from a file. Will treat the keys in the
26
+ # json as methods to invoke on the component in question
27
+ #
28
+ # @param file [String] Path to the json file
29
+ # @raise [RuntimeError] exceptions are raised if there is no file, if it refers to methods that don't exist, or if it does not contain a Hash
30
+ def load_from_json(file)
31
+ data = JSON.parse(File.read(file))
32
+ unless data.is_a?(Hash)
33
+ raise "Hash required. Got '#{data.class}' when parsing '#{file}'"
34
+ end
35
+ # We explicity return data here b/c the unless clause above will cause the
36
+ # Function to return nil.
37
+ # -Sean P. M. 05/11/2016
38
+ data
39
+ end
40
+
41
+ # @param gem the name of the gem to see if it is installed
42
+ # @return boolean for whether or not the gem is installed
43
+ def check_gem(gem)
44
+ if %x(gem list -q #{gem}).chomp.empty?
45
+ return false
46
+ else
47
+ return true
48
+ end
49
+ end
50
+
51
+ # This loads your ~/.gem/credentials file and uses your api key
52
+ # to query rubygems.org for which gems you own. There may be better ways
53
+ # to query this but having to pull this information using curl is sort of
54
+ # gross. It works though, so ¯\_(ツ)_/¯
55
+ # @param gem_name the gem to check if you are an owner
56
+ # @return boolean whether or not you own the gem
57
+ def check_rubygems_ownership(gem_name)
58
+ require 'yaml'
59
+ credentials = YAML.load_file("#{ENV['HOME']}/.gem/credentials")
60
+ gems = YAML.load(%x(curl -H 'Authorization:#{credentials[:rubygems_api_key]}' https://rubygems.org/api/v1/gems.yaml))
61
+ gems.each do |gem|
62
+ if gem['name'] == gem_name
63
+ return true
64
+ end
65
+ end
66
+ return false
67
+ end
68
+ end
69
+ end
@@ -0,0 +1,368 @@
1
+ # Utility methods for handling network calls and interactions
2
+
3
+ module Pkg::Util::Net
4
+
5
+ class << self
6
+
7
+ # This simple method does an HTTP get of a URI and writes it to a file
8
+ # in a slightly more platform agnostic way than curl/wget
9
+ def fetch_uri(uri, target)
10
+ require 'open-uri'
11
+ if Pkg::Util::File.file_writable?(File.dirname(target))
12
+ File.open(target, 'w') { |f| f.puts(open(uri).read) }
13
+ end
14
+ end
15
+
16
+ # Get the hostname of the current host
17
+ def hostname
18
+ require 'socket'
19
+ Socket.gethostname
20
+ end
21
+
22
+ # Check that the current host matches the one we think it should
23
+ def check_host(host, args = { :required => true })
24
+ if hostname == host
25
+ return true
26
+ else
27
+ fail "#{hostname} does not match #{host}" if args[:required]
28
+ return nil
29
+ end
30
+ end
31
+
32
+ # @param hosts - An array of hosts to try ssh-ing into
33
+ # If the host needs a special username it should be passed
34
+ # in as user@host
35
+ # @return an array of hosts where ssh access failed. Empty array if
36
+ # successful
37
+ def check_host_ssh(hosts)
38
+ errs = []
39
+ Array(hosts).flatten.each do |host|
40
+ begin
41
+ remote_ssh_cmd(host, 'exit', false, '-oBatchMode=yes')
42
+ rescue
43
+ errs << host
44
+ end
45
+ end
46
+ return errs
47
+ end
48
+
49
+ # @param hosts - An array of hosts to check for gpg keys
50
+ # If the host needs a special username it should be passed
51
+ # in as user@host
52
+ # @param gpg - The gpg secret key to look for
53
+ # @return an array of hosts where ssh access failed. Empty array if
54
+ # successful
55
+ def check_host_gpg(hosts, gpg)
56
+ errs = []
57
+ Array(hosts).flatten.each do |host|
58
+ begin
59
+ remote_ssh_cmd(host, "gpg --list-secret-keys #{gpg} > /dev/null 2&>1", false, '-oBatchMode=yes')
60
+ rescue
61
+ errs << host
62
+ end
63
+ end
64
+ return errs
65
+ end
66
+
67
+ def remote_ssh_cmd(target, command, capture_output = false, extra_options = '', fail_fast = true)
68
+ ssh = Pkg::Util::Tool.check_tool('ssh')
69
+
70
+ # we pass some pretty complicated commands in via ssh. We need this to fail
71
+ # if any part of the remote ssh command fails.
72
+ command = "set -e; #{command}" if fail_fast
73
+ cmd = "#{ssh} #{extra_options} -t #{target} '#{command.gsub("'", "'\\\\''")}'"
74
+
75
+ # This is NOT a good way to support this functionality.
76
+ # It needs to be refactored into a set of methods that
77
+ # other methods can use to safely and deterministically
78
+ # support dry-run operations.
79
+ # But I need to ship packages RIGHT NOW.
80
+ # - Ryan McKern, 13/01/2016
81
+ if ENV['DRYRUN']
82
+ puts "[DRY-RUN] Executing '#{command}' on #{target}"
83
+ puts "[DRY-RUN] #{cmd}"
84
+ return
85
+ end
86
+
87
+ puts "Executing '#{command}' on #{target}"
88
+ if capture_output
89
+ stdout, stderr, exitstatus = Pkg::Util::Execution.capture3(cmd)
90
+ Pkg::Util::Execution.success?(exitstatus) or raise "Remote ssh command failed."
91
+ return stdout, stderr
92
+ else
93
+ Kernel.system(cmd)
94
+ Pkg::Util::Execution.success? or raise "Remote ssh command failed."
95
+ end
96
+ end
97
+
98
+ # Construct a valid rsync command
99
+ # @return [String] a rsync command that can be used in shell or ssh methods
100
+ # @param [String, Pathname] origin_path the path to sync from; if opts[:target_path]
101
+ # is not passed, then the parent directory of `origin_path` will be used to
102
+ # construct a target path to sync to.
103
+ # @param [Hash] opts additional options that can be used to construct
104
+ # the rsync command.
105
+ # @option opts [String] :bin ('rsync') the path to rsync
106
+ # (can be relative or fully qualified).
107
+ # @option opts [String] :origin_host the remote host to sync data from; cannot
108
+ # be specified alongside :target_host
109
+ # @option opts [String] :target_host the remote host to sync data to; cannot
110
+ # be specified alongside :origin_host.
111
+ # @option opts [String] :extra_flags (["--ignore-existing"]) extra flags to
112
+ # use when constructing an rsync command
113
+ # @option opts [String] :dryrun (false) tell rsync to perform a trial run
114
+ # with no changes made.
115
+ # @raise [ArgumentError] if opts[:origin_host] and opts[:target_host] names
116
+ # are both defined.
117
+ # @raise [ArgumentError] if :origin_path exists without opts[:target_path],
118
+ # opts[:origin_host], remote target is defined.
119
+ def rsync_cmd(origin_path, opts = {})
120
+ options = {
121
+ bin: 'rsync',
122
+ origin_host: nil,
123
+ target_path: nil,
124
+ target_host: nil,
125
+ extra_flags: nil,
126
+ dryrun: false }.merge(opts)
127
+ origin = Pathname.new(origin_path)
128
+ target = options[:target_path] || origin.parent
129
+
130
+ raise(ArgumentError, "Cannot sync between two remote hosts") if
131
+ options[:origin_host] && options[:target_host]
132
+
133
+ raise(ArgumentError, "Cannot sync path '#{origin}' to itself") unless
134
+ options[:origin_host] || options[:target_host]
135
+
136
+ cmd = %W(
137
+ #{options[:bin]}
138
+ --recursive
139
+ --hard-links
140
+ --links
141
+ --verbose
142
+ --omit-dir-times
143
+ --no-perms
144
+ --no-owner
145
+ --no-group
146
+ ) + [*options[:extra_flags]]
147
+
148
+ cmd << '--dry-run' if options[:dryrun]
149
+ cmd << Pkg::Util.pseudo_uri(path: origin, host: options[:origin_host])
150
+ cmd << Pkg::Util.pseudo_uri(path: target, host: options[:target_host])
151
+
152
+ cmd.uniq.compact.join("\s")
153
+ end
154
+
155
+ # A generic rsync execution method that wraps rsync_cmd in a
156
+ # call to Pkg::Util::Execution#capture3()
157
+ def rsync_exec(source, opts = {})
158
+ options = {
159
+ bin: Pkg::Util::Tool.check_tool('rsync'),
160
+ origin_host: nil,
161
+ target_path: nil,
162
+ target_host: nil,
163
+ extra_flags: nil,
164
+ dryrun: ENV['DRYRUN'] }.merge(opts.delete_if { |_, value| value.nil? })
165
+
166
+ stdout, _, _ = Pkg::Util::Execution.capture3(rsync_cmd(source, options), true)
167
+ stdout
168
+ end
169
+
170
+ # A wrapper method to maintain the existing interface for executing
171
+ # outbound rsync commands with minimal changes to existing code.
172
+ def rsync_to(source, target_host, dest, opts = { extra_flags: ["--ignore-existing"] })
173
+ rsync_exec(
174
+ source,
175
+ target_host: target_host,
176
+ target_path: dest,
177
+ extra_flags: opts[:extra_flags],
178
+ dryrun: opts[:dryrun],
179
+ bin: opts[:bin],
180
+ )
181
+ end
182
+
183
+ # A wrapper method to maintain the existing interface for executing
184
+ # incoming rsync commands with minimal changes to existing code.
185
+ def rsync_from(source, origin_host, dest, opts = {})
186
+ rsync_exec(
187
+ source,
188
+ origin_host: origin_host,
189
+ target_path: dest,
190
+ extra_flags: opts[:extra_flags],
191
+ dryrun: opts[:dryrun],
192
+ bin: opts[:bin],
193
+ )
194
+ end
195
+
196
+ def s3sync_to(source, target_bucket, target_directory = "", flags = [])
197
+ s3cmd = Pkg::Util::Tool.check_tool('s3cmd')
198
+
199
+ if Pkg::Util::File.file_exists?(File.join(ENV['HOME'], '.s3cfg'))
200
+ stdout, _, _ = Pkg::Util::Execution.capture3("#{s3cmd} sync #{flags.join(' ')} '#{source}' s3://#{target_bucket}/#{target_directory}/")
201
+ stdout
202
+ else
203
+ fail "#{File.join(ENV['HOME'], '.s3cfg')} does not exist. It is required to ship files using s3cmd."
204
+ end
205
+ end
206
+
207
+ # This is fairly absurd. We're implementing curl by shelling out. What do I
208
+ # wish we were doing? Using a sweet ruby wrapper around curl, such as Curb or
209
+ # Curb-fu. However, because we're using clean build systems and trying to
210
+ # make this portable with minimal system requirements, we can't very well
211
+ # depend on libraries that aren't in the ruby standard libaries. We could
212
+ # also do this using Net::HTTP but that set of libraries is a rabbit hole to
213
+ # go down when what we're trying to accomplish is posting multi-part form
214
+ # data that includes file uploads to jenkins. It gets hairy fairly quickly,
215
+ # but, as they say, pull requests accepted.
216
+ #
217
+ # This method takes three arguments
218
+ # 1) String - the URL to post to
219
+ # 2) Array - Ordered array of name=VALUE curl form parameters
220
+ # 3) Hash - Options to be set
221
+ def curl_form_data(uri, form_data = [], options = {})
222
+ curl = Pkg::Util::Tool.check_tool("curl")
223
+ #
224
+ # Begin constructing the post string.
225
+ # First, assemble the form_data arguments
226
+ #
227
+ post_string = "-i "
228
+ form_data.each do |param|
229
+ post_string << "#{param} "
230
+ end
231
+
232
+ # Add the uri
233
+ post_string << "'#{uri}'"
234
+
235
+ # If this is quiet, we're going to silence all output
236
+ begin
237
+ stdout, _, retval = Pkg::Util::Execution.capture3("#{curl} #{post_string}")
238
+ if options[:quiet]
239
+ stdout = ''
240
+ end
241
+ return stdout, retval
242
+ rescue RuntimeError => e
243
+ puts e
244
+ return false
245
+ end
246
+ end
247
+
248
+ def uri_status_code(uri)
249
+ data = [
250
+ '--request GET',
251
+ '--silent',
252
+ '--location',
253
+ '--write-out "%{http_code}"',
254
+ '--output /dev/null'
255
+ ]
256
+ stdout, _ = Pkg::Util::Net.curl_form_data(uri, data)
257
+ stdout
258
+ end
259
+
260
+ # Use the provided URL string to print important information with
261
+ # ASCII emphasis
262
+ def print_url_info(url_string)
263
+ puts "\n////////////////////////////////////////////////////////////////////////////////\n\n
264
+ Build submitted. To view your build progress, go to\n#{url_string}\n\n
265
+ ////////////////////////////////////////////////////////////////////////////////\n\n"
266
+ end
267
+
268
+ def remote_set_ownership(host, owner, group, files)
269
+ remote_cmd = "for file in #{files.join(" ")}; do if [[ -d $file ]] || ! `lsattr $file | grep -q '\\-i\\-'`; then sudo chown #{owner}:#{group} $file; else echo \"$file is immutable\"; fi; done"
270
+ Pkg::Util::Net.remote_ssh_cmd(host, remote_cmd)
271
+ end
272
+
273
+ def remote_set_permissions(host, permissions, files)
274
+ remote_cmd = "for file in #{files.join(" ")}; do if [[ -d $file ]] || ! `lsattr $file | grep -q '\\-i\\-'`; then sudo chmod #{permissions} $file; else echo \"$file is immutable\"; fi; done"
275
+ Pkg::Util::Net.remote_ssh_cmd(host, remote_cmd)
276
+ end
277
+
278
+ # Remotely set the immutable bit on a list of files
279
+ def remote_set_immutable(host, files)
280
+ Pkg::Util::Net.remote_ssh_cmd(host, "sudo chattr +i #{files.join(" ")}")
281
+ end
282
+
283
+ # Create a symlink indicating the latest version of a package
284
+ #
285
+ # @param package_name [String] The name of the package you want to symlink
286
+ # to, e.g. 'puppet-agent', 'facter', etc.
287
+ # @param dir [String] The directory you want to find the latest package and
288
+ # create the symlink in.
289
+ # @param platform_ext [String] The type of files you want to consider, e.g.
290
+ # 'dmg', 'msi', etc.
291
+ # @param options [Hash] Additional optional params:
292
+ # @option :arch [String] Architecture you want to narrow your search by.
293
+ # @option :excludes [Array] Strings you want to exclude from your search,
294
+ # e.g. 'agent' if only searching for 'puppet'.
295
+ def remote_create_latest_symlink(package_name, dir, platform_ext, options = {})
296
+ ls_cmd = "ls -1 *.#{platform_ext} | grep -v latest | grep -v rc | grep #{package_name} "
297
+ if options[:arch]
298
+ ls_cmd << "| grep #{options[:arch]}"
299
+ package_name << "-#{options[:arch]}"
300
+ end
301
+ if options[:excludes]
302
+ options[:excludes].each do |excl|
303
+ ls_cmd << "| grep -v #{excl} "
304
+ end
305
+ end
306
+ ls_cmd << '| sort --version-sort | tail -1'
307
+ cmd = <<-CMD
308
+ if [ ! -d '#{dir}' ] ; then
309
+ echo "directory '#{dir}' does not exist, not creating latest package link"
310
+ exit 0
311
+ fi
312
+ pushd '#{dir}'
313
+ link_target=$(#{ls_cmd})
314
+ if [ -z "$link_target" ] ; then
315
+ echo "Unable to find a link target for '#{package_name}' in '#{dir}'; skipping link creation"
316
+ exit 0
317
+ fi
318
+ echo "creating link to '$link_target'"
319
+ ln -sf "$link_target" #{package_name}-latest.#{platform_ext}
320
+ CMD
321
+
322
+ _, err = Pkg::Util::Net.remote_ssh_cmd(Pkg::Config.staging_server, cmd, true)
323
+ $stderr.puts err
324
+ end
325
+
326
+ def escape_html(uri)
327
+ require 'cgi'
328
+ CGI.escapeHTML(uri)
329
+ end
330
+
331
+ # Add a parameter to a given uri. If we were sane we'd use
332
+ # encode_www_form(params) of URI, but because we're not, because that will http
333
+ # encode it, which isn't what we want since we're require the encoding provided
334
+ # by escapeHTML of CGI, since this is being transfered in the xml of a jenkins
335
+ # job via curl and DEAR JEEBUS WHAT HAVE WE DONE.
336
+ def add_param_to_uri(uri, param)
337
+ require 'uri'
338
+ uri = URI.parse(uri)
339
+ uri.query = [uri.query, param].compact.join('&')
340
+ uri.to_s
341
+ end
342
+
343
+ # We take a tar argument for cases where `tar` isn't best, e.g. Solaris. We
344
+ # also take an optional argument of the tarball containing the git bundle to
345
+ # use.
346
+ def remote_bootstrap(host, treeish, tar_cmd = nil, tarball = nil)
347
+ unless tar = tar_cmd
348
+ tar = 'tar'
349
+ end
350
+ tarball ||= Pkg::Util::Git.bundle(treeish)
351
+ tarball_name = File.basename(tarball).gsub('.tar.gz', '')
352
+ Pkg::Util::Net.rsync_to(tarball, host, '/tmp')
353
+ appendix = Pkg::Util.rand_string
354
+ Pkg::Util::Net.remote_ssh_cmd(host, "#{tar} -zxvf /tmp/#{tarball_name}.tar.gz -C /tmp/ ; git clone --recursive /tmp/#{tarball_name} /tmp/#{Pkg::Config.project}-#{appendix} ; cd /tmp/#{Pkg::Config.project}-#{appendix} ; rake package:bootstrap")
355
+ "/tmp/#{Pkg::Config.project}-#{appendix}"
356
+ end
357
+
358
+ # Given a BuildInstance object and a host, send its params to the host. Return
359
+ # the remote path to the params.
360
+ def remote_buildparams(host, build)
361
+ params_file = build.config_to_yaml
362
+ params_file_name = File.basename(params_file)
363
+ params_dir = Pkg::Util.rand_string
364
+ Pkg::Util::Net.rsync_to(params_file, host, "/tmp/#{params_dir}/")
365
+ "/tmp/#{params_dir}/#{params_file_name}"
366
+ end
367
+ end
368
+ end