puppet 2.7.20 → 2.7.21

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of puppet might be problematic. Click here for more details.

Files changed (81) hide show
  1. data/CHANGELOG +226 -0
  2. data/conf/auth.conf +3 -3
  3. data/ext/packaging/README.md +191 -57
  4. data/ext/packaging/spec/spec_helper.rb +2 -2
  5. data/ext/packaging/spec/tasks/00_utils_spec.rb +63 -18
  6. data/ext/packaging/spec/tasks/build_object_spec.rb +171 -0
  7. data/ext/packaging/tasks/00_utils.rake +186 -33
  8. data/ext/packaging/tasks/10_setupvars.rake +94 -65
  9. data/ext/packaging/tasks/20_setupextravars.rake +45 -26
  10. data/ext/packaging/tasks/30_metrics.rake +41 -0
  11. data/ext/packaging/tasks/apple.rake +92 -36
  12. data/ext/packaging/tasks/build.rake +183 -0
  13. data/ext/packaging/tasks/deb.rake +45 -40
  14. data/ext/packaging/tasks/deb_repos.rake +103 -0
  15. data/ext/packaging/tasks/doc.rake +5 -5
  16. data/ext/packaging/tasks/fetch.rake +35 -10
  17. data/ext/packaging/tasks/gem.rake +38 -27
  18. data/ext/packaging/tasks/ips.rake +14 -14
  19. data/ext/packaging/tasks/jenkins.rake +337 -0
  20. data/ext/packaging/tasks/mock.rake +153 -72
  21. data/ext/packaging/tasks/pe_deb.rake +2 -2
  22. data/ext/packaging/tasks/pe_remote.rake +22 -19
  23. data/ext/packaging/tasks/pe_rpm.rake +5 -5
  24. data/ext/packaging/tasks/pe_ship.rake +31 -21
  25. data/ext/packaging/tasks/pe_sign.rake +20 -19
  26. data/ext/packaging/tasks/pe_sles.rake +40 -36
  27. data/ext/packaging/tasks/pe_tar.rake +5 -0
  28. data/ext/packaging/tasks/release.rake +32 -12
  29. data/ext/packaging/tasks/remote_build.rake +141 -83
  30. data/ext/packaging/tasks/retrieve.rake +23 -0
  31. data/ext/packaging/tasks/rpm.rake +11 -19
  32. data/ext/packaging/tasks/rpm_repos.rake +127 -0
  33. data/ext/packaging/tasks/ship.rake +68 -55
  34. data/ext/packaging/tasks/sign.rake +38 -10
  35. data/ext/packaging/tasks/tar.rake +25 -9
  36. data/ext/packaging/tasks/update.rake +2 -2
  37. data/ext/packaging/tasks/version.rake +34 -14
  38. data/ext/packaging/tasks/z_data_dump.rake +33 -0
  39. data/lib/puppet/indirector/catalog/compiler.rb +13 -2
  40. data/lib/puppet/indirector/certificate_status/file.rb +5 -0
  41. data/lib/puppet/indirector/errors.rb +5 -0
  42. data/lib/puppet/indirector/file_bucket_file/file.rb +4 -0
  43. data/lib/puppet/indirector/file_bucket_file/selector.rb +4 -0
  44. data/lib/puppet/indirector/indirection.rb +1 -0
  45. data/lib/puppet/indirector/resource/active_record.rb +3 -0
  46. data/lib/puppet/indirector/resource/ral.rb +4 -0
  47. data/lib/puppet/indirector/resource/store_configs.rb +3 -0
  48. data/lib/puppet/indirector/resource/validator.rb +8 -0
  49. data/lib/puppet/indirector/rest.rb +8 -0
  50. data/lib/puppet/indirector/run/local.rb +4 -0
  51. data/lib/puppet/indirector/terminus.rb +20 -0
  52. data/lib/puppet/network/formats.rb +3 -3
  53. data/lib/puppet/network/handler/master.rb +1 -1
  54. data/lib/puppet/network/handler/report.rb +1 -1
  55. data/lib/puppet/network/http/handler.rb +7 -1
  56. data/lib/puppet/network/http/rack/rest.rb +7 -2
  57. data/lib/puppet/network/http/webrick.rb +1 -0
  58. data/lib/puppet/network/rest_authconfig.rb +1 -1
  59. data/lib/puppet/parser/templatewrapper.rb +17 -17
  60. data/lib/puppet/util/monkey_patches.rb +58 -0
  61. data/lib/puppet/version.rb +1 -1
  62. data/spec/integration/indirector/catalog/compiler_spec.rb +1 -0
  63. data/spec/integration/indirector/catalog/queue_spec.rb +1 -1
  64. data/spec/integration/resource/catalog_spec.rb +1 -0
  65. data/spec/unit/indirector/catalog/compiler_spec.rb +29 -2
  66. data/spec/unit/indirector/indirection_spec.rb +18 -1
  67. data/spec/unit/indirector/terminus_spec.rb +191 -177
  68. data/spec/unit/network/formats_spec.rb +6 -6
  69. data/spec/unit/network/http/handler_spec.rb +25 -0
  70. data/spec/unit/network/http/rack/rest_spec.rb +17 -0
  71. data/spec/unit/network/http/webrick_spec.rb +4 -0
  72. data/spec/unit/network/http_pool_spec.rb +0 -1
  73. data/spec/unit/network/rest_authconfig_spec.rb +16 -1
  74. data/spec/unit/parser/functions/inline_template_spec.rb +13 -0
  75. data/spec/unit/parser/functions/template_spec.rb +15 -0
  76. data/spec/unit/parser/templatewrapper_spec.rb +19 -4
  77. data/spec/unit/ssl/certificate_request_spec.rb +2 -0
  78. data/spec/unit/ssl/host_spec.rb +1 -0
  79. data/spec/unit/util/monkey_patches_spec.rb +12 -0
  80. data/test/language/snippets.rb +1 -1
  81. metadata +13 -2
@@ -1,21 +1,46 @@
1
- if @build_pe
2
- pl_packaging_url = "https://raw.github.com/puppetlabs/build-data/#{@team}"
3
- else
4
- pl_packaging_url = "https://raw.github.com/puppetlabs/build-data/#{@name}"
1
+ # Each team has a build-defaults file that specifies local infrastructure targets
2
+ # for things like builders, target locations for build artifacts, etc Since much
3
+ # of these don't change, one file can be maintained for the team. Each project
4
+ # also has a data file for information specific to it. If the project builds
5
+ # both PE and not PE, it has two files, one for PE, and the other for FOSS
6
+ #
7
+ data_repo = 'https://raw.github.com/puppetlabs/build-data'
8
+ project_data_branch = @build.project
9
+ team_data_branch = @build.team
10
+
11
+ if @build.build_pe
12
+ project_data_branch = 'pe-' + project_data_branch unless project_data_branch =~ /^pe-/
13
+ team_data_branch = 'pe-' + team_data_branch unless team_data_branch =~ /^pe-/
5
14
  end
6
15
 
7
- # The pl:fetch task pulls down a file from the build-data repo that contains additional
16
+ project_data_url = data_repo + '/' + project_data_branch
17
+ team_data_url = data_repo + '/' + team_data_branch
18
+
19
+
20
+ # The pl:fetch task pulls down two files from the build-data repo that contain additional
8
21
  # data specific to Puppet Labs release infrastructure intended to augment/override any
9
22
  # defaults specified in the source project repo, e.g. in ext/build_defaults.yaml
10
23
  #
11
- # It uses curl to download the file, and places it in a hidden directory in the home
12
- # directory, e.g. ~/.packaging/@builder_data_file
24
+ # It uses curl to download the files, and places them in a temporary
25
+ # directory, e.g. /tmp/somedirectory/{project,team}/@build.builder_data_file
13
26
  namespace :pl do
14
27
  task :fetch do
15
- rm_rf "#{ENV['HOME']}/.packaging"
16
- mkdir_p "#{ENV['HOME']}/.packaging"
28
+ # Remove .packaging directory from old-style extras loading
29
+ rm_rf "#{ENV['HOME']}/.packaging" if File.directory?("#{ENV['HOME']}/.packaging")
30
+ # Touch the .packaging file which is allows packaging to present remote tasks
31
+ touch "#{ENV['HOME']}/.packaging"
32
+ tempdir = get_temp
33
+ mkdir_pr("#{tempdir}/team", "#{tempdir}/project")
17
34
  begin
18
- sh "curl #{pl_packaging_url}/#{@builder_data_file} > #{ENV['HOME']}/.packaging/#{@builder_data_file}"
35
+ if dist = el_version
36
+ if dist.to_i < 6
37
+ flag = "-k"
38
+ end
39
+ end
40
+ sh "curl #{flag} #{project_data_url}/#{@build.builder_data_file} > #{tempdir}/project/#{@build.builder_data_file}"
41
+ sh "curl #{flag} #{team_data_url}/#{@build.builder_data_file} > #{tempdir}/team/#{@build.builder_data_file}"
42
+ invoke_task("pl:load_extras", tempdir)
43
+ rm_rf(tempdir)
19
44
  rescue
20
45
  STDERR.puts "There was an error fetching the builder extras data."
21
46
  exit 1
@@ -1,10 +1,10 @@
1
- if @build_gem
1
+ if @build.build_gem
2
2
  require 'rubygems/package_task'
3
3
 
4
4
  def glob_gem_files
5
5
  gem_files = []
6
6
  gem_excludes_file_list = []
7
- gem_excludes_raw = @gem_excludes.nil? ? [] : @gem_excludes.split(' ')
7
+ gem_excludes_raw = @build.gem_excludes.nil? ? [] : @build.gem_excludes.split(' ')
8
8
  gem_excludes_raw << 'ext/packaging'
9
9
  gem_excludes_raw.each do |exclude|
10
10
  if File.directory?(exclude)
@@ -13,7 +13,7 @@ if @build_gem
13
13
  gem_excludes_file_list << exclude
14
14
  end
15
15
  end
16
- files = FileList[@gem_files.split(' ')]
16
+ files = FileList[@build.gem_files.split(' ')]
17
17
  files.each do |file|
18
18
  if File.directory?(file)
19
19
  gem_files += FileList["#{file}/**/*"]
@@ -25,44 +25,55 @@ if @build_gem
25
25
  end
26
26
 
27
27
  spec = Gem::Specification.new do |s|
28
- s.name = @name unless @name.nil?
29
- s.name = @gem_name unless @gem_name.nil?
30
- s.version = @gemversion unless @gemversion.nil?
31
- s.author = @author unless @author.nil?
32
- s.email = @email unless @email.nil?
33
- s.homepage = @homepage unless @homepage.nil?
34
- s.summary = @summary unless @summary.nil?
35
- s.summary = @gem_summary unless @gem_summary.nil?
36
- s.description = @description unless @description.nil?
37
- s.description = @gem_description unless @gem_description.nil?
38
- s.files = glob_gem_files unless glob_gem_files.nil?
39
- s.executables = @gem_executables unless @gem_executables.nil?
40
- s.require_path = @gem_require_path unless @gem_require_path.nil?
41
- s.test_files = FileList[@gem_test_files.split(' ')] unless @gem_test_files.nil?
42
- s.rubyforge_project = @gem_forge_project unless @gem_forge_project.nil?
28
+ s.name = @build.project unless @build.project.nil?
29
+ s.name = @build.gem_name unless @build.gem_name.nil?
30
+ s.version = @build.gemversion unless @build.gemversion.nil?
31
+ s.author = @build.author unless @build.author.nil?
32
+ s.email = @build.email unless @build.email.nil?
33
+ s.homepage = @build.homepage unless @build.homepage.nil?
34
+ s.summary = @build.summary unless @build.summary.nil?
35
+ s.summary = @build.gem_summary unless @build.gem_summary.nil?
36
+ s.description = @build.description unless @build.description.nil?
37
+ s.description = @build.gem_description unless @build.gem_description.nil?
38
+ s.files = glob_gem_files unless glob_gem_files.nil?
39
+ s.executables = @build.gem_executables unless @build.gem_executables.nil?
40
+ s.require_path = @build.gem_require_path unless @build.gem_require_path.nil?
41
+ s.test_files = FileList[@build.gem_test_files.split(' ')] unless @build.gem_test_files.nil?
42
+ s.rubyforge_project = @build.gem_forge_project unless @build.gem_forge_project.nil?
43
43
 
44
- @gem_runtime_dependencies.each do |gem, version|
44
+ @build.gem_runtime_dependencies.each do |gem, version|
45
45
  s.add_runtime_dependency("#{gem}", "#{version}") unless (version.nil? or version.empty?)
46
46
  s.add_runtime_dependency("#{gem}") if (version.nil? or version.empty?)
47
- end unless @gem_runtime_dependencies.nil?
47
+ end unless @build.gem_runtime_dependencies.nil?
48
48
 
49
- @gem_devel_dependencies.each do |gem, version|
49
+ @build.gem_devel_dependencies.each do |gem, version|
50
50
  s.add_devel_dependency("#{gem}", "#{version}") unless (version.nil? or version.empty?)
51
51
  s.add_devel_dependency("#{gem}") if (version.nil? or version.empty?)
52
- end unless @gem_devel_dependencies.nil?
52
+ end unless @build.gem_devel_dependencies.nil?
53
53
 
54
- @gem_rdoc_options.each do |option|
54
+ @build.gem_rdoc_options.each do |option|
55
55
  s.rdoc_options << option
56
- end unless @gem_rdoc_options.nil?
56
+ end unless @build.gem_rdoc_options.nil?
57
57
  end
58
58
 
59
59
  namespace :package do
60
60
  gem_task = Gem::PackageTask.new(spec)
61
61
  desc "Build a gem"
62
62
  task :gem => [ "clean" ] do
63
- gem_task.define
64
- Rake::Task[:gem].invoke
65
- rm_rf "pkg/#{@name}-#{@gemversion}"
63
+ bench = Benchmark.realtime do
64
+ gem_task.define
65
+ Rake::Task[:gem].invoke
66
+ rm_rf "pkg/#{@build.project}-#{@build.gemversion}"
67
+ end
68
+ if @build.benchmark
69
+ add_metrics({ :dist => 'gem', :bench => bench })
70
+ post_metrics
71
+ end
66
72
  end
67
73
  end
74
+
75
+ # An alias task to simplify our remote logic in jenkins.rake
76
+ namespace :pl do
77
+ task :gem => "package:gem"
78
+ end
68
79
  end
@@ -7,7 +7,7 @@ if @build_ips
7
7
  repo = workdir + '/repo'
8
8
  pkgs = 'pkg/ips/pkgs'
9
9
  repouri = 'file://' + Dir.pwd + '/' + repo
10
- artifact = pkgs + "/#{@name}@#{@ipsversion}.p5p"
10
+ artifact = pkgs + "/#{@build.project}@#{@build.ipsversion}.p5p"
11
11
 
12
12
  # Create a source repo
13
13
  # We dont clean the base pkg directory only ips work dir.
@@ -27,34 +27,34 @@ if @build_ips
27
27
 
28
28
  # Process templates and write the initial manifest
29
29
  task :prototmpl do
30
- erb("ext/ips/#{@name}.p5m.erb", workdir + '/' + @name + '.p5m.x')
30
+ erb("ext/ips/#{@build.project}.p5m.erb", workdir + '/' + @build.project + '.p5m.x')
31
31
  end
32
32
 
33
33
  # Update manifest to include the installation image information.
34
34
  task :protogen => :prototmpl do
35
- sh "pkgsend generate #{proto} >> #{workdir}/#{@name}.p5m.x"
35
+ sh "pkgsend generate #{proto} >> #{workdir}/#{@build.project}.p5m.x"
36
36
  end
37
37
 
38
38
  # Generate and resolve dependency list
39
39
  task :protodeps => :protogen do
40
- sh "pkgdepend generate -d #{proto} #{workdir}/#{@name}.p5m.x > #{workdir}/#{@name}.depends"
41
- sh "pkgdepend resolve -m #{workdir}/#{@name}.depends"
42
- sh "cat #{workdir}/#{@name}.depends.res >> #{workdir}/#{@name}.p5m.x"
40
+ sh "pkgdepend generate -d #{proto} #{workdir}/#{@build.project}.p5m.x > #{workdir}/#{@build.project}.depends"
41
+ sh "pkgdepend resolve -m #{workdir}/#{@build.project}.depends"
42
+ sh "cat #{workdir}/#{@build.project}.depends.res >> #{workdir}/#{@build.project}.p5m.x"
43
43
  end
44
44
 
45
45
  # Mogrify manifest to remove unncecessary info, and other kinds of transforms.
46
46
  task :protomogrify => :protodeps do
47
- sh "pkgmogrify ./ext/ips/transforms ./#{workdir}/#{@name}.p5m.x| pkgfmt >> #{workdir}/#{@name}.p5m"
47
+ sh "pkgmogrify ./ext/ips/transforms ./#{workdir}/#{@build.project}.p5m.x| pkgfmt >> #{workdir}/#{@build.project}.p5m"
48
48
  end
49
49
 
50
50
  # Generate and resolve dependency list
51
51
  task :license => :protomogrify do
52
- cp 'LICENSE', "#{proto}/#{@name}.license"
52
+ cp 'LICENSE', "#{proto}/#{@build.project}.license"
53
53
  end
54
54
 
55
55
  # Ensure that our manifest is sane.
56
56
  task :lint => :license do
57
- print %x{pkglint #{workdir}/#{@name}.p5m}
57
+ print %x{pkglint #{workdir}/#{@build.project}.p5m}
58
58
  end
59
59
 
60
60
  task :package => [:clean_pkgs, :clean, :prepare, :lint] do
@@ -71,18 +71,18 @@ if @build_ips
71
71
  # Send a created package to the local IPS repository
72
72
  task :send do
73
73
  check_tool('pkgsend')
74
- sh "pkgsend -s #{repouri} publish -d #{proto} --fmri-in-manifest #{workdir}/#{@name}.p5m"
74
+ sh "pkgsend -s #{repouri} publish -d #{proto} --fmri-in-manifest #{workdir}/#{@build.project}.p5m"
75
75
  end
76
76
 
77
77
  # Retrieve the package from the remote repository in .p5p archive format
78
78
  task :receive do
79
79
  check_tool('pkgrecv')
80
- sh "pkgrecv -s #{repouri} -a -d #{artifact} #{@name}@#{@ipsversion}"
80
+ sh "pkgrecv -s #{repouri} -a -d #{artifact} #{@build.project}@#{@build.ipsversion}"
81
81
  end
82
82
 
83
83
 
84
84
  task :dry_install do
85
- sh "pkg install -nv -g #{artifact} #{@name}@#{@ipsversion}"
85
+ sh "pkg install -nv -g #{artifact} #{@build.project}@#{@build.ipsversion}"
86
86
  end
87
87
 
88
88
  task :p5p, :sign_ips do |t, args|
@@ -100,7 +100,7 @@ if @build_ips
100
100
  # publish the package to the repository
101
101
  Rake::Task['package:ips:send'].invoke
102
102
  # signing the package occurs remotely in the repository
103
- Rake::Task['pl:sign_ips'].invoke(repouri,"#{@name}@#{@ipsversion}") if sign_ips
103
+ Rake::Task['pl:sign_ips'].invoke(repouri,"#{@build.project}@#{@build.ipsversion}") if sign_ips
104
104
  # retrieve the signed package in a .p5p archive file format
105
105
  Rake::Task['package:ips:receive'].invoke
106
106
  # clean up the workdir area
@@ -115,7 +115,7 @@ if @build_ips
115
115
 
116
116
  namespace :pl do
117
117
  desc "Create and sign a p5p archive package from this repository"
118
- task :ips => ['pl:fetch', 'pl:load_extras'] do
118
+ task :ips => 'pl:fetch' do
119
119
  Rake::Task['package:ips:p5p'].reenable
120
120
  Rake::Task['package:ips:p5p'].invoke(TRUE)
121
121
  end
@@ -0,0 +1,337 @@
1
+ ##
2
+ # The jenkins tasks enable the packaging repo to kick off packaging builds on a
3
+ # remote jenkins slave. They work in a similar way to the :remote tasks, but
4
+ # with a few key differences. The jenkins tasks transmit information to a
5
+ # jenkins coordinator, which handles the rest. The data passed are the
6
+ # following:
7
+ #
8
+ # 1) $PROJECT_BUNDLE - a tar.gz of a git-bundle from HEAD of the current
9
+ # project, which is cloned on the builder to set up a duplicate of this
10
+ # environment
11
+ #
12
+ # 2) $BUILD_PROPERTIES - a build parameters file, containing all information about the build
13
+ #
14
+ # 3) $BUILD_TYPE - the "type" of build, e.g. rpm, deb, gem, etc The jenkins url and job name
15
+ # are obtained via the team build-data file at
16
+ # git@github.com/puppetlabs/build-data
17
+ #
18
+ # 4) $PROJECT - the project we're building, e.g. facter, puppet. This is used later in
19
+ # determining the target for the build artifacts on the distribution server
20
+ #
21
+ # On the Jenkins end, the job is a parameterized job that accepts four
22
+ # parameters. Jenkins has the Parameterized Trigger Plugin, Workspace Cleanup
23
+ # Plugin, and Node and Label Parameter Plugin in use for this job. The
24
+ # workspace cleanup plugin cleans the workspace before each build. Two are file
25
+ # parameters, a string parameter, and a Label parameter provided by the Node
26
+ # and Label Parameter Plugin, as described above. When th pl:jenkins:post task
27
+ # triggers a build, it passes values for all of these parameters. The Label
28
+ # parameter is associated with the build type. This way we can queue the job on
29
+ # a builder with the appropriate capabilities just by assigning a builder the
30
+ # label "deb" or "rpm," etc. The actual build itself is accomplished via a
31
+ # shell build task. The contents of the task are:
32
+ #
33
+ #################
34
+ #
35
+ # #!/bin/bash
36
+ #
37
+ # SHA=$(echo $BUILD_PROPERTIES | cut -d '.' -f1)
38
+ #
39
+ # echo "Build type: $BUILD_TYPE"
40
+ #
41
+ # ### Create a local clone of the git-bundle that was passed
42
+ # # The bundle is a tarball, and since this is a project-agnostic
43
+ # # job, we don't actually know what's in it, just that it's a
44
+ # # git bundle.
45
+ # #
46
+ #
47
+ # [ -f "PROJECT_BUNDLE" ] || exit 1
48
+ # mkdir project && tar -xzf PROJECT_BUNDLE -C project/
49
+ #
50
+ # pushd project
51
+ # git clone --recursive $(ls) git_repo
52
+ #
53
+ # pushd git_repo
54
+ #
55
+ # ### Clone the packaging repo
56
+ # rake package:bootstrap
57
+ #
58
+ # ### Perform the build
59
+ # rake pl:build_from_params PARAMS_FILE=$WORKSPACE/BUILD_PROPERTIES
60
+ #
61
+ # ### Send the results
62
+ # rake pl:jenkins:ship["artifacts"] PARAMS_FILE=$WORKSPACE/BUILD_PROPERTIES
63
+ #
64
+ # popd
65
+ # popd
66
+ #
67
+ # ### Create the repositories from our project by trigger a downstream job
68
+ # ### Because we can't trigger downstream with a File Parameter, we use curl
69
+ # if [ "$BUILD_TYPE" = "rpm" ] || [ "$BUILD_TYPE" = "deb" ] ; then
70
+ # curl -i -Fname=PROJECT_BUNDLE -Ffile0=@PROJECT_BUNDLE -FSubmit=Build -Fjson="{\"parameter\":[{\"name\":\"PROJECT_BUNDLE\",\"file\":\"file0\"}]}" \
71
+ # http://jenkins-release.delivery.puppetlabs.net/job/puppetlabs-packaging-repo-creation/build
72
+ # fi
73
+ #
74
+ # ### If a downstream job was passed, trigger it now
75
+ # if [ -n "$DOWNSTREAM_JOB" ] ; then
76
+ # pushd project
77
+ # pushd git_repo
78
+ # rake pl:jenkins:post["$DOWNSTREAM_JOB"] PARAMS_FILE=$WORKSPACE/BUILD_PROPERTIES
79
+ # popd
80
+ # popd
81
+ # fi
82
+ #
83
+ #################
84
+
85
+ namespace :pl do
86
+ namespace :jenkins do
87
+ ##
88
+ # Do the heavy lifting. This task generates the URL for the jenkins job and posts it.
89
+ # It expects a the following arguments
90
+ # 1. :build_task => The lower-level pl: or pe: task we're executing, e.g. pl:deb_all
91
+ #
92
+ task :post_build, :build_task do |t, args|
93
+ # Check for a dirty tree before allowing a remote build that is doomed to unexpected results
94
+ if source_dirty?
95
+ warn "The source tree is dirty, e.g. there are uncommited changes. Please commit/discard changes and try again."
96
+ exit 1
97
+ end
98
+
99
+ # We use JSON for parsing the json part of the submission to JSON
100
+ begin
101
+ require 'json'
102
+ rescue LoadError
103
+ warn "Couldn't require 'json'. JSON is required for sanely generating the string we curl to Jenkins."
104
+ exit 1
105
+ end
106
+
107
+ build_task = args.build_task
108
+ ##
109
+ # We set @:task of @build manually with our task data so the remote
110
+ # build knows what to do. Puppetdb needs early knowledge of if this is
111
+ # a PE build, so we always this along as an environment variable task
112
+ # argument if its the case.
113
+ #
114
+ @build.task = { :task => "#{build_task}", :args => nil }
115
+ @build.task[:args] = ["PE_BUILD=true"] if @build_pe
116
+ #
117
+ # Determine the type of build we're doing to inform jenkins
118
+ build_type = case build_task
119
+ when /deb/ then "deb"
120
+ when /mock/ then "rpm"
121
+ when /dmg|apple/ then "dmg"
122
+ when /gem/ then "gem"
123
+ when /tar/ then "tar"
124
+ when /sles/ then "sles"
125
+ else raise "Could not determine build type for #{build_task}"
126
+ end
127
+ #
128
+ # Create the data files to send to jenkins
129
+ properties = @build.params_to_yaml
130
+ bundle = git_bundle('HEAD')
131
+
132
+ # Construct the parameters, which is an array of hashes we turn into JSON
133
+ parameters = [{ "name" => "BUILD_PROPERTIES", "file" => "file0" },
134
+ { "name" => "PROJECT_BUNDLE", "file" => "file1" },
135
+ { "name" => "PROJECT", "value" => "#{@build.project}" },
136
+ { "name" => "BUILD_TYPE", "label" => "#{build_type}" }]
137
+
138
+ # Initialize the args array that will hold all of the arguments we pass
139
+ # to the curl utility method.
140
+ args = []
141
+
142
+ # If the environment variable "DOWNSTREAM_JOB" was passed, we want to
143
+ # send this value to the build job as well, so it knows to trigger a
144
+ # downstream job, and with what URI.
145
+ if ENV['DOWNSTREAM_JOB']
146
+ parameters << { "name" => "DOWNSTREAM_JOB", "value" => ENV['DOWNSTREAM_JOB'] }
147
+ args << [ "-Fname=DOWNSTREAM_JOB", "-Fvalue=#{ENV['DOWNSTREAM_JOB']}" ]
148
+ end
149
+
150
+ # Contruct the json string
151
+ json = JSON.generate("parameter" => parameters)
152
+
153
+ # Construct the remaining form arguments. For visual clarity, params that are tied
154
+ # together are on the same line.
155
+ #
156
+ args << [
157
+ "-Fname=BUILD_PROPERTIES", "-Ffile0=@#{properties}",
158
+ "-Fname=PROJECT_BUNDLE" , "-Ffile1=@#{bundle}",
159
+ "-Fname=PROJECT" , "-Fvalue=#{@build.project}",
160
+ "-Fname=BUILD_TYPE" , "-Fvalue=#{build_type}",
161
+ "-FSubmit=Build",
162
+ "-Fjson=#{json.to_json}",
163
+ ]
164
+
165
+ # We have several arrays inside args by now, flatten it up.
166
+ args.flatten!
167
+
168
+ # Construct the job url
169
+ #
170
+ job_url = "#{@build.jenkins_build_host}/job/#{@build.jenkins_packaging_job}"
171
+ trigger_url = "#{job_url}/build"
172
+
173
+ # Call out to the curl_form_data utility method in 00_utils.rake
174
+ #
175
+ if curl_form_data(trigger_url, args)
176
+ puts "Build submitted. To view your build results, go to #{job_url}"
177
+ puts "Your packages will be available at #{@build.distribution_server}:#{@build.jenkins_repo_path}/#{@build.project}/#{@build.ref}"
178
+ else
179
+ warn "An error occurred submitting the job to jenkins. Take a look at the preceding http response for more info."
180
+ end
181
+ end
182
+ end
183
+ end
184
+
185
+ ##
186
+ # A task listing for creating jenkins tasks for our various pl: and pe: build
187
+ # tasks. We can assume deb, mock, but not gem/dmg.
188
+ #
189
+ tasks = ["deb", "mock", "tar"]
190
+ tasks << "gem" if @build.build_gem and ! @build.build_pe
191
+ tasks << "dmg" if @build.build_dmg and ! @build.build_pe
192
+
193
+ namespace :pl do
194
+ namespace :jenkins do
195
+ tasks.each do |build_task|
196
+ desc "Queue pl:#{build_task} build on jenkins builder"
197
+ task build_task => "pl:fetch" do
198
+ invoke_task("pl:jenkins:post_build", "pl:#{build_task}")
199
+ end
200
+ end
201
+
202
+ # While pl:remote:deb_all does all cows in serially, with jenkins we
203
+ # parallelize them. This breaks the cows up and posts a build for all of
204
+ # them. We have to sleep 5 because jenkins drops the builds when we're
205
+ # DOSing it with our packaging.
206
+ desc "Queue pl:deb_all on jenkins builder"
207
+ task :deb_all => "pl:fetch" do
208
+ @build.cows.split(' ').each do |cow|
209
+ @build.default_cow = cow
210
+ invoke_task("pl:jenkins:post_build", "pl:deb")
211
+ sleep 5
212
+ end
213
+ end
214
+
215
+ # This does the mocks in parallel
216
+ desc "Queue pl:mock-all on jenkins builder"
217
+ task :mock_all => "pl:fetch" do
218
+ @build.final_mocks.split(' ').each do |mock|
219
+ @build.default_mock = mock
220
+ invoke_task("pl:jenkins:post_build", "pl:mock")
221
+ sleep 5
222
+ end
223
+ end
224
+
225
+ desc "Jenkins UBER build: build all the things with jenkins"
226
+ task :uber_build do
227
+ uber_tasks = ["jenkins:deb_all", "jenkins:mock_all", "jenkins:tar"]
228
+ uber_tasks << "jenkins:dmg" if @build.build_dmg
229
+ uber_tasks << "jenkins:gem" if @build.build_gem
230
+ uber_tasks.map { |t| "pl:#{t}" }.each do |t|
231
+ invoke_task(t)
232
+ sleep 5
233
+ end
234
+ end
235
+
236
+ desc "Retrieve packages built by jenkins, sign, and ship all!"
237
+ task :uber_ship => "pl:fetch" do
238
+ uber_tasks = ["jenkins:retrieve", "jenkins:sign_all", "uber_ship", "remote:freight", "remote:update_yum_repo" ]
239
+ uber_tasks.map { |t| "pl:#{t}" }.each { |t| Rake::Task[t].invoke }
240
+ Rake::Task["pl:jenkins:ship"].invoke("shipped")
241
+ end
242
+ end
243
+ end
244
+
245
+ ##
246
+ # If this is a PE project, we want PE tasks as well. However, because the
247
+ # PE tasks use :remote as their default (e.g., not namespaced under remote)
248
+ # we have to explicily use the "local" tasks, since these will be local
249
+ # builds on jenkins agents. Also, we support building on SLES for PE, so we
250
+ # add a sles task.
251
+ #
252
+ if @build.build_pe
253
+ namespace :pe do
254
+ namespace :jenkins do
255
+ tasks << "sles"
256
+ tasks.each do |build_task|
257
+ desc "Queue pe:#{build_task} build on jenkins builder"
258
+ task build_task => "pl:fetch" do
259
+ check_var("PE_VER", @build.pe_version)
260
+ invoke_task("pl:jenkins:post_build", "pe:local_#{build_task}")
261
+ end
262
+ end
263
+
264
+ desc "Queue builds of all PE packages for this project in Jenkins"
265
+ task :uber_build => "pl:fetch" do
266
+ check_var("PE_VER", @build.pe_version)
267
+ ["deb_all", "mock_all", "sles"].each do |task|
268
+ invoke_task("pe:jenkins:#{task}")
269
+ sleep 5
270
+ end
271
+ end
272
+
273
+ desc "Retrieve PE packages built by jenkins, sign, and ship all!"
274
+ task :uber_ship => "pl:fetch" do
275
+ check_var("PE_VER", @build.pe_version)
276
+ ["pl:jenkins:retrieve", "pe:ship_rpms", "pe:ship_debs"].each do |task|
277
+ Rake::Task[task].invoke
278
+ end
279
+ Rake::Task["pl:jenkins:ship"].invoke("shipped")
280
+ end
281
+ end
282
+ end
283
+ end
284
+
285
+ ##
286
+ # This task allows the packaging repo to post to an arbitrary jenkins job but
287
+ # it is very limited in that it does not model well the key => value format
288
+ # used when submitting form data on websites. This is primarily because rake
289
+ # does not allow us to elegantly pass arbitrary key => value pairs on the
290
+ # command line and have any idea how to reference them inside rake. We can pass
291
+ # KEY=VALUE along with our invokation, but unless KEY is statically coded into
292
+ # our task, we won't know how to reference it. Thus, this task will only take
293
+ # one argument, the uri of the jenkins job to post to. This can be passed
294
+ # either as an argument directly or as an environment variable "JOB" with the
295
+ # uri as the value. The argument is required. The second requirement is that
296
+ # the job to be called accept a string parameter with the name SHA. This will
297
+ # be the SHA of the commit of the project source code HEAD, and should be used
298
+ # by the job to check out this specific ref. To maintain the abstraction of the
299
+ # jenkins jobs, this specific task passes on no information about the build
300
+ # itself. The assumption is that the upstream jobs know about their project,
301
+ # and so do the downstream jobs, but packaging itself has no business knowing
302
+ # about it.
303
+ #
304
+ namespace :pl do
305
+ namespace :jenkins do
306
+ desc "Trigger a jenkins uri with SHA of HEAD as a string param, requires \"URI\""
307
+ task :post, :uri do |t, args|
308
+ uri = args.uri || ENV['URI']
309
+ raise "pl:jenkins:post requires a URI, either via URI= or pl:jenkin:post[URI]" if uri.nil?
310
+
311
+ # We use JSON for parsing the json part of the submission.
312
+ begin
313
+ require 'json'
314
+ rescue LoadError
315
+ warn "Couldn't require 'json'. JSON is required for sanely generating the string we curl to Jenkins."
316
+ exit 1
317
+ end
318
+
319
+ # Assemble the JSON string for the JSON parameter
320
+ json = JSON.generate("parameter" => [{ "name" => "SHA", "value" => "#{@build.ref}" }])
321
+
322
+ # Assemble our arguments to the post
323
+ args = [
324
+ "-Fname=SHA", "-Fvalue=#{@build.ref}",
325
+ "-Fjson=#{json.to_json}",
326
+ "-FSubmit=Build"
327
+ ]
328
+
329
+ if curl_form_data(uri, args)
330
+ puts "Job triggered at #{uri}."
331
+ else
332
+ puts "An error occurred attempting to trigger the job at #{uri}. Please see the preceding http response for more info."
333
+ end
334
+ end
335
+ end
336
+ end
337
+