packaging 0.99.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +17 -0
  3. data/README-Solaris.md +117 -0
  4. data/README.md +1031 -0
  5. data/lib/packaging.rb +32 -0
  6. data/lib/packaging/artifactory.rb +278 -0
  7. data/lib/packaging/config.rb +392 -0
  8. data/lib/packaging/config/params.rb +366 -0
  9. data/lib/packaging/deb.rb +28 -0
  10. data/lib/packaging/deb/repo.rb +263 -0
  11. data/lib/packaging/gem.rb +112 -0
  12. data/lib/packaging/ips.rb +57 -0
  13. data/lib/packaging/msi.rb +89 -0
  14. data/lib/packaging/nuget.rb +39 -0
  15. data/lib/packaging/osx.rb +36 -0
  16. data/lib/packaging/paths.rb +238 -0
  17. data/lib/packaging/platforms.rb +480 -0
  18. data/lib/packaging/repo.rb +55 -0
  19. data/lib/packaging/retrieve.rb +46 -0
  20. data/lib/packaging/rpm.rb +5 -0
  21. data/lib/packaging/rpm/repo.rb +257 -0
  22. data/lib/packaging/tar.rb +154 -0
  23. data/lib/packaging/util.rb +146 -0
  24. data/lib/packaging/util/date.rb +15 -0
  25. data/lib/packaging/util/execution.rb +85 -0
  26. data/lib/packaging/util/file.rb +125 -0
  27. data/lib/packaging/util/git.rb +174 -0
  28. data/lib/packaging/util/git_tags.rb +73 -0
  29. data/lib/packaging/util/gpg.rb +62 -0
  30. data/lib/packaging/util/jenkins.rb +95 -0
  31. data/lib/packaging/util/misc.rb +69 -0
  32. data/lib/packaging/util/net.rb +368 -0
  33. data/lib/packaging/util/os.rb +17 -0
  34. data/lib/packaging/util/platform.rb +40 -0
  35. data/lib/packaging/util/rake_utils.rb +111 -0
  36. data/lib/packaging/util/serialization.rb +19 -0
  37. data/lib/packaging/util/ship.rb +171 -0
  38. data/lib/packaging/util/tool.rb +41 -0
  39. data/lib/packaging/util/version.rb +326 -0
  40. data/spec/fixtures/config/ext/build_defaults.yaml +2 -0
  41. data/spec/fixtures/config/ext/project_data.yaml +2 -0
  42. data/spec/fixtures/config/params.yaml +2 -0
  43. data/spec/fixtures/configs/components/test_file.json +1 -0
  44. data/spec/fixtures/configs/components/test_file_2.json +0 -0
  45. data/spec/fixtures/configs/components/test_file_not_tagged.json +1 -0
  46. data/spec/fixtures/configs/components/test_file_wrong_ext.txt +0 -0
  47. data/spec/fixtures/configs/components/test_file_wrong_ext.wrong +0 -0
  48. data/spec/fixtures/util/pre_tasks.yaml +4 -0
  49. data/spec/lib/packaging/artifactory_spec.rb +171 -0
  50. data/spec/lib/packaging/config_spec.rb +556 -0
  51. data/spec/lib/packaging/deb/repo_spec.rb +148 -0
  52. data/spec/lib/packaging/deb_spec.rb +52 -0
  53. data/spec/lib/packaging/paths_spec.rb +153 -0
  54. data/spec/lib/packaging/platforms_spec.rb +153 -0
  55. data/spec/lib/packaging/repo_spec.rb +97 -0
  56. data/spec/lib/packaging/retrieve_spec.rb +61 -0
  57. data/spec/lib/packaging/rpm/repo_spec.rb +133 -0
  58. data/spec/lib/packaging/tar_spec.rb +122 -0
  59. data/spec/lib/packaging/util/execution_spec.rb +56 -0
  60. data/spec/lib/packaging/util/file_spec.rb +139 -0
  61. data/spec/lib/packaging/util/git_spec.rb +160 -0
  62. data/spec/lib/packaging/util/git_tag_spec.rb +36 -0
  63. data/spec/lib/packaging/util/gpg_spec.rb +64 -0
  64. data/spec/lib/packaging/util/jenkins_spec.rb +112 -0
  65. data/spec/lib/packaging/util/misc_spec.rb +31 -0
  66. data/spec/lib/packaging/util/net_spec.rb +239 -0
  67. data/spec/lib/packaging/util/os_spec.rb +31 -0
  68. data/spec/lib/packaging/util/rake_utils_spec.rb +70 -0
  69. data/spec/lib/packaging/util/ship_spec.rb +117 -0
  70. data/spec/lib/packaging/util/version_spec.rb +123 -0
  71. data/spec/lib/packaging_spec.rb +19 -0
  72. data/spec/spec_helper.rb +36 -0
  73. data/static_artifacts/PackageInfo.plist +3 -0
  74. data/tasks/00_utils.rake +216 -0
  75. data/tasks/30_metrics.rake +33 -0
  76. data/tasks/apple.rake +266 -0
  77. data/tasks/build.rake +12 -0
  78. data/tasks/clean.rake +5 -0
  79. data/tasks/config.rake +30 -0
  80. data/tasks/deb.rake +129 -0
  81. data/tasks/deb_repos.rake +28 -0
  82. data/tasks/deprecated.rake +130 -0
  83. data/tasks/doc.rake +20 -0
  84. data/tasks/education.rake +57 -0
  85. data/tasks/fetch.rake +57 -0
  86. data/tasks/gem.rake +146 -0
  87. data/tasks/jenkins.rake +494 -0
  88. data/tasks/jenkins_dynamic.rake +202 -0
  89. data/tasks/load_extras.rake +21 -0
  90. data/tasks/mock.rake +348 -0
  91. data/tasks/nightly_repos.rake +335 -0
  92. data/tasks/pe_deb.rake +12 -0
  93. data/tasks/pe_rpm.rake +13 -0
  94. data/tasks/pe_ship.rake +221 -0
  95. data/tasks/pe_sign.rake +13 -0
  96. data/tasks/pe_tar.rake +5 -0
  97. data/tasks/retrieve.rake +45 -0
  98. data/tasks/rpm.rake +66 -0
  99. data/tasks/rpm_repos.rake +29 -0
  100. data/tasks/ship.rake +752 -0
  101. data/tasks/sign.rake +226 -0
  102. data/tasks/tag.rake +8 -0
  103. data/tasks/tar.rake +34 -0
  104. data/tasks/update.rake +16 -0
  105. data/tasks/vanagon.rake +35 -0
  106. data/tasks/vendor_gems.rake +117 -0
  107. data/tasks/version.rake +33 -0
  108. data/tasks/z_data_dump.rake +65 -0
  109. data/templates/README +1 -0
  110. data/templates/downstream.xml.erb +47 -0
  111. data/templates/msi.xml.erb +197 -0
  112. data/templates/packaging.xml.erb +344 -0
  113. data/templates/repo.xml.erb +114 -0
  114. metadata +234 -0
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 60597eac7cb43a07b9c9c5f2e0446eeb97741697
4
+ data.tar.gz: 4b2493e3dac7fc620c221f29207c1aac0c4628d3
5
+ SHA512:
6
+ metadata.gz: 945d03dbfe203b4ce43210494fba3da11c365064e54d014d5d2a04f1c42704295a355b0a1bd6a8b1f7b63388b4210db71d8b463b9d388a9ddc1b0d52a6f4bcce
7
+ data.tar.gz: 0f6e50d73f6d91d9666b49ff6e7a5a8019045c9cfb190a5c0ead9c04d341a5375e95d180bf2560511c404f860a6cedb10a4be4470a9cc717c5a331de61026492
data/LICENSE ADDED
@@ -0,0 +1,17 @@
1
+ packaging - Puppet Labs packaging framework
2
+
3
+ Copyright (C) 2011-2013 Puppet Labs Inc
4
+
5
+ Puppet Labs can be contacted at: info@puppetlabs.com
6
+
7
+ Licensed under the Apache License, Version 2.0 (the "License");
8
+ you may not use this file except in compliance with the License.
9
+ You may obtain a copy of the License at
10
+
11
+ http://www.apache.org/licenses/LICENSE-2.0
12
+
13
+ Unless required by applicable law or agreed to in writing, software
14
+ distributed under the License is distributed on an "AS IS" BASIS,
15
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+ See the License for the specific language governing permissions and
17
+ limitations under the License.
@@ -0,0 +1,117 @@
1
+ ## Create self signed certificate
2
+
3
+ - Generate the key pair
4
+
5
+ | keytool -genkey -keyalg RSA -alias selfsigned -keystore keystore.jks -storepass password -validity 360 -keysize 2048
6
+
7
+
8
+ - Extract Certificate
9
+
10
+ | keytool -export -rfc -keystore keystore.jks -storepass password -alias selfsigned -file mycert.pem
11
+
12
+ - Extract Cert Key to PEM format
13
+
14
+ | keytool -importkeystore -srcstoretype JKS -srckeystore keystore.jks -deststoretype PKCS12 -destkeystore mykey.der
15
+ | openssl pkcs12 -in mykey.der -nodes -out mykey.pem
16
+
17
+ ## For installation.
18
+
19
+ - Check the publishers
20
+
21
+ | pkg publisher
22
+ solaris origin online http://pkg.oracle.com/solaris/release/
23
+
24
+ - Add puppetlabs.com as a publisher
25
+
26
+ | pkg set-publisher -p http://solaris-11-ips-repo.acctest.dc1.puppetlabs.net puppetlabs.com
27
+
28
+ - Check that we have it correct
29
+
30
+ | pkg publisher
31
+ solaris origin online http://pkg.oracle.com/solaris/release/
32
+ puppetlabs.com origin online http://solaris-11-ips-repo.acctest.dc1.puppetlabs.net/
33
+
34
+ - Verify that puppetlabs.com was set correctly,
35
+
36
+ | pkg publisher puppetlabs.com
37
+ Publisher: puppetlabs.com
38
+ Alias:
39
+ Origin URI: http://solaris-11-ips-repo.acctest.dc1.puppetlabs.net/
40
+ SSL Key: None
41
+ SSL Cert: None
42
+ Client UUID: 37084c4a-fdc6-11e1-832b-8800273bd610
43
+ Catalog Updated: September 13, 2012 08:31:25 PM
44
+ Enabled: Yes
45
+ Signature Policy: verify
46
+
47
+ - Try installing puppet
48
+
49
+ | pkg install puppet
50
+ Creating Plan |
51
+ pkg install: Chain was rooted in an untrusted self-signed certificate.
52
+ The package involved is:pkg://puppetlabs.com/system/management/puppet@3.0.0,5.11-6.192:20120913T212942Z
53
+
54
+ - We have two choices here, first, to ignore the signature policy for both publisher and image
55
+
56
+ | pkg set-publisher --set-property=signature-policy=ignore puppetlabs.com
57
+ | pkg publisher puppetlabs.com
58
+ Publisher: puppetlabs.com
59
+ Alias:
60
+ Origin URI: http://solaris-11-ips-repo.acctest.dc1.puppetlabs.net/
61
+ SSL Key: None
62
+ SSL Cert: None
63
+ Client UUID: 140bb5c8-fe62-11e1-af70-8800273bd610
64
+ Catalog Updated: September 14, 2012 05:11:45 PM
65
+ Enabled: Yes
66
+ Signature Policy: ignore
67
+ | pkg set-property signature-policy ignore
68
+ | pkg install puppet
69
+
70
+ - Or trust the self signed certificate from puppetlabs.com (this certificate needs to be published in our web site)
71
+
72
+ | pkg set-publisher --approve-ca-cert /root/mycert.pem puppetlabs.com
73
+
74
+ - Verify that approve cert went well.
75
+
76
+ | pkg publisher puppetlabs.com
77
+
78
+ Publisher: puppetlabs.com
79
+ Alias:
80
+ Origin URI: http://solaris-11-ips-repo.acctest.dc1.puppetlabs.net/
81
+ SSL Key: None
82
+ SSL Cert: None
83
+ Client UUID: 37084c4a-fdc6-11e1-832b-8800273bd610
84
+ Catalog Updated: September 13, 2012 08:31:25 PM
85
+ Approved CAs: 791b5791a81e9c2eb3fb9f84f4f86f8ea6fcd672
86
+ Enabled: Yes
87
+ Signature Policy: verify
88
+
89
+ | pkg install puppet
90
+
91
+ - Checking about information on the package (See the human readable version string)
92
+
93
+ | pkg info -r puppet
94
+ Name: system/management/puppet
95
+ Summary: Puppet, an automated configuration management tool
96
+ Description: Puppet, an automated configuration management tool
97
+ Category: System/Administration and Configuration
98
+ State: Not installed
99
+ Publisher: puppetlabs.com
100
+ Version: 3.0.0 (3.0.0-rc6)
101
+ Build Release: 5.11
102
+ Branch: 6.170
103
+ Packaging Date: September 14, 2012 06:11:05 PM
104
+ Size: 2.43 MB
105
+ FMRI: pkg://puppetlabs.com/system/management/puppet@3.0.0,5.11-6.170:20120914T181105Z
106
+
107
+ - Check license
108
+
109
+ | pkg info -r --license puppet
110
+ Puppet - Automating Configuration Management.
111
+
112
+ Copyright (C) 2005-2012 Puppet Labs Inc
113
+
114
+
115
+ - Reference
116
+
117
+ http://docs.oracle.com/cd/E19963-01/html/820-6572/managepkgs.html
@@ -0,0 +1,1031 @@
1
+ # Packaging
2
+
3
+ [![Build Status](https://travis-ci.org/puppetlabs/packaging.png?branch=master)](https://travis-ci.org/puppetlabs/packaging)
4
+
5
+ This is a repository for packaging automation for Puppet software.
6
+ The goal is to abstract and automate packaging processes beyond individual
7
+ software projects to a level where this repo can be cloned inside any project
8
+ and used to build Debian and Redhat packages, as well as gems, apple packages
9
+ and tarballs. This repo is currently under heavy development and in a state
10
+ flux, and it should not be considered to have a formal API. However, every
11
+ effort is being made to ensure existing tasks/behavior are not broken as we
12
+ continue to iterate and improve upon it.
13
+
14
+ ## Getting Started
15
+
16
+ As a developer, you can get started working on this by typing:
17
+
18
+ ```
19
+ bundle install
20
+ bundle exec rspec
21
+ ```
22
+
23
+ ## Using the Packaging Repo
24
+
25
+ Several Puppet projects are using the packaging repo. They are:
26
+
27
+ * puppet
28
+ * facter
29
+ * puppet-dashboard
30
+ * hiera
31
+ * puppetdb
32
+ * razor
33
+
34
+ as well as several closed-source projects, including
35
+ * live-management
36
+ * console-auth
37
+ * console
38
+
39
+ Generally speaking, the packaging repo should be compatible with ruby 1.8.7,
40
+ ruby 1.9.3 and rake 0.9.x. To pull the packaging tasks into your source repo,
41
+ do a `rake package:bootstrap`. This will clone this repo into the ext directory
42
+ of the project and make many packaging tasks available. The tasks are
43
+ generally grouped into two categories, `package:` namespaced tasks and `pl:`
44
+ namespaced tasks.
45
+
46
+ ## `package:` tasks
47
+ `package:` namespaced tasks are general purpose tasks that are set up to use
48
+ the most minimal tool chain possible for creating packages. These tasks will
49
+ create rpms and debs, but any build dependencies will need to be satisifed by
50
+ the building host, and any dynamically generated dependencies may result in
51
+ packages that are only suitable for the OS/version of the build host. However,
52
+ for rolling one's own debs and rpms or for use in environments without many
53
+ OSes/versions, this may work just fine. To build an rpm using the packaging
54
+ repo, do a `rake package:rpm`. To build a deb, use `rake package:deb`.
55
+
56
+ ## `pl:` tasks
57
+ `pl:` namespaced tasks rely on a slighly more complex toolchain for packaging
58
+ inside clean chroot environments for the various operating systems and versions
59
+ that Puppet supports. On the rpm side, this is done with
60
+ [mock](http://fedoraproject.org/wiki/Projects/Mock) and for debs, we use
61
+ pdebuild and [cowbuilder](http://wiki.debian.org/cowbuilder). For the most
62
+ part, these tasks are keyed to puppet infrastructure, and are used by the
63
+ Release Engineering team to create release packages. However, they can
64
+ certainly be modified to suit other environments, and much effort went into
65
+ making tasks as modular and reusable as possible. Several Puppet-specific
66
+ tasks are only available if the file '~/.packaging' is present. This file is
67
+ created by the `pl:fetch` task, which curls two yaml files into 'team' and
68
+ 'project' subdirectories from a [separate build data
69
+ repository](https://github.com/puppetlabs/build-data), which contains
70
+ additional settings/data specific to Puppet release infrastructure. By
71
+ default, the team data file is pulled from the 'dev' branch of the repo, and
72
+ the project data file is pulled from a branch named after the project (e.g. for
73
+ puppet, there is a branch named puppet with a build data file). The goal in
74
+ separating these data and tasks out is to refrain from presenting by
75
+ default yet more Puppet-specific tasks that aren't generally consumable by
76
+ everyone. To build a deb from a local repository using a `pl` task, ssh into a
77
+ builder (e.g., one stood up using the modules detailed below) and clone the
78
+ source repo, e.g. puppet. Then, run `rake package:bootstrap` and `rake pl:deb`
79
+ to create a deb, and `rake pl:mock` to make an rpm (on a debian or redhat host,
80
+ respectively).
81
+
82
+ ## `pe:` tasks
83
+ There is also a `pe:` namespace, for the building of Puppet
84
+ Labs' Puppet Enterprise packages that have been converted to using this repo.
85
+ The `pe:` tasks rely heavily on PL internal infrastructure, and are not
86
+ generally useful outside of this environment. To create packages, in the source
87
+ repository run `rake package:bootstrap`, followed by `rake pl:fetch`. These two
88
+ commands bootstrap the packaging environment and pull in the additional data
89
+ needed for PE building (see `pl:fetch` notes above). Then, to make a debian
90
+ package, run `rake pe:deb`, and to make an rpm, run `rake pe:mock`. There are
91
+ also `pe:deb_all` and `pe:mock_all` tasks, which build packages against all
92
+ shipped debian/redhat targets. The `pe:deb_all` task is not generally necessary
93
+ for developer use for building test packages; the `pe:deb` task creates a
94
+ package that will work against virtually all supported PE debian versions.
95
+ The same is generally true for PE internal rpms, but because of variances in
96
+ build macros for rpm, rpms should generally be built with `pe:mock_all`, and
97
+ then the desired version installed, or by building only for a specific
98
+ target. This is accomplished by passing MOCK=<mock> to the rake call, e.g.
99
+ `rake pe:mock MOCK=<mock>`. The available mocks are listed in
100
+ `ext/build_defaults.yaml` after `final_mocks:`. For PE, the mocks are
101
+ formatted as `pupent-<peversion>-<distversion>-<arch>`, e.g.
102
+ `pupent-2.7-el5-i386`. To build for a specific target, set `MOCK=<mock>` to
103
+ the mock that matches the target.
104
+
105
+ ## `:remote:` tasks
106
+ There are also sub-namespaces of `:pl` and `:pe` that are
107
+ worth noting. First, the `:remote` namespace. Tasks under `:remote` perform
108
+ builds remotely on internal builders from your local workstation. How they
109
+ work:
110
+
111
+ 1) Run `pl:fetch` to obtain extra data from the build-data repo. The data
112
+ includes the hostnames of builders to use for packaging.
113
+
114
+ 2) Create a git bundle of the local workspace and tar it up.
115
+
116
+ 3) Create a build parameters file. The params file includes all the information
117
+ about the build, including any values overridden with env vars, and the actual
118
+ task to run, e.g. `rake pl:deb`.
119
+
120
+ 4) scp the git bundle and build parameters file to a temporary directory on the
121
+ builder hostname assigned to that particular package build type.
122
+
123
+ 5) ssh into the builder, untar the git bundle, clone it, and run `rake
124
+ package:bootstrap`.
125
+
126
+ 6) ssh into the builder, cd into the cloned repo, and run `rake
127
+ pl:build_from_params PARAMS_FILE=/path/to/previously/sent/file`.
128
+
129
+ 7) Maintain the ssh connection until the build finishes, and rsync the packages
130
+ from the builder to the local workstation.
131
+
132
+ Note that these tasks require ssh access to the builder hosts that are
133
+ specified in the build-data file, and appropriate membership in the build
134
+ groups, e.g. to use mock on the builder, membership in the mock group. This is
135
+ a major hurdle, and is resolved with the `jenkins` tasks below.
136
+
137
+ ## legacy `:jenkins:` workflow tasks
138
+ (Deprecated - see "dyamic jenkins task workflow" below)
139
+ Jenkins tasks are similar to the `:remote:` tasks, but they do not require ssh
140
+ access to the builders. They do require being on the local network - the
141
+ jenkins instance that performs package builds is an internal server only,
142
+ accessible when connected via VPN or on-site. The jenkins tasks enable the
143
+ packaging repo to kick off packaging builds on a remote jenkins slave.
144
+ There are two workflows of jenkins tasks in the packaging repo. The first
145
+ workflow, which is used for creating individual platform packages on jenkins
146
+ (e.g. creating a deb with pl:jenkins:deb) relies on a job that exists on a
147
+ remote jenkins server. The tasks transmit information to the jenkins job, which
148
+ handles the rest. The data passed are the following:
149
+
150
+ 1) $PROJECT\_BUNDLE - a tar.gz of a git-bundle from HEAD of the current
151
+ project, which is cloned on the builder to set up a duplicate of this
152
+ environment
153
+
154
+ 2) $BUILD\_PROPERTIES - a build parameters file, containing all information
155
+ about the build
156
+
157
+ 3) $BUILD\_TYPE - the "type" of build, e.g. rpm, deb, gem, etc The jenkins url
158
+ and job name are obtained via the team build-data file from [the build data
159
+ repository](https://github.com/puppetlabs/build-data)
160
+
161
+ 4) $PROJECT - the project we're building, e.g. facter, puppet. This is used
162
+ later in determining the target for the build artifacts on the distribution
163
+ server
164
+
165
+ 4) $METRICS - a string of data points related to the build which are used for
166
+ data analysis. Contents of this string are items which cannot be obtained from
167
+ within the Jenkins job itself.
168
+
169
+ 5) $DOWNSTREAM\_JOB - The URL of a downstream job that jenkins should post to
170
+ upon success. This is obtained via the DOWNSTREAM\_JOB environment variable.
171
+
172
+
173
+ On the Jenkins end, the job is a parameterized job that accepts five
174
+ parameters. Jenkins has the Parameterized Trigger Plugin, Workspace Cleanup
175
+ Plugin, and Node and Label Parameter Plugin in use for this job. The workspace
176
+ cleanup plugin cleans the workspace before each build. Two are file parameters,
177
+ two string parameters, and a Label parameter provided by the Node and Label
178
+ Parameter Plugin, as described above. When the pl:jenkins:\* task triggers a
179
+ build, it passes values for all of these parameters. The Label parameter is
180
+ associated with the build type. This way we can queue the job on a builder with
181
+ the appropriate capabilities just by assigning a builder the label "deb" or
182
+ "rpm," etc. The job allows parallel execution of jobs - in this way, we can
183
+ queue many package jobs on the jenkins instance, which will farm them out to
184
+ builders that are slaves of that jenkins instance. This also allows us to scale
185
+ building capacity simply by adding builders as slaves to the jenkins instance.
186
+ The actual build itself is accomplished via a shell build task. The contents of
187
+ the task are:
188
+
189
+ ```bash
190
+ #################
191
+
192
+ SHA=$(echo $BUILD_PROPERTIES | cut -d '.' -f1)
193
+
194
+ echo "Build type: $BUILD_TYPE"
195
+
196
+ ### Create a local clone of the git-bundle that was passed
197
+ # The bundle is a tarball, and since this is a project-agnostic
198
+ # job, we don't actually know what's in it, just that it's a
199
+ # git bundle.
200
+
201
+ [ -f "PROJECT_BUNDLE" ] || exit 1
202
+ mkdir project && tar -xzf PROJECT_BUNDLE -C project/
203
+
204
+ cd project
205
+ git clone --recursive $(ls) git_repo
206
+
207
+ cd git_repo
208
+
209
+ ### Clone the packaging repo
210
+ rake package:bootstrap && rake pl:fetch
211
+
212
+ ### Perform the build
213
+ rake pl:load_extras pl:build_from_params PARAMS_FILE=$WORKSPACE/BUILD_PROPERTIES
214
+
215
+ ### Send the results
216
+ rake pl:jenkins:ship["artifacts"]
217
+
218
+ ### If a downstream job was passed, trigger it now
219
+ if [ -n "$DOWNSTREAM_JOB" ] ; then
220
+ rake pl:jenkins:post["$DOWNSTREAM_JOB"]
221
+ fi
222
+
223
+ #################
224
+ ```
225
+
226
+ To gather metrics related to a Jenkins build, the Groovy Postbuild plugin is used.
227
+ For tasks carried out on the static Jenkins job, the script must be manually added to the job's
228
+ configuration. The script in its entirety can be seen [here.](https://github.com/Whopper92/buildboard#groovyScript)
229
+
230
+ ## dynamic `:jenkins:` task workflow
231
+
232
+ The recommended and far simpler jenkins-based workflow is for initiating the
233
+ "uber_build", or a package build for all of our target platforms.
234
+
235
+ The uber_build is invoked as "pl:jenkins:uber_build" or "pe:jenkins:uber_build"
236
+ depending on if this is a FOSS or PE package.
237
+
238
+ This workflow doesn't actually use a static job on the jenkins-server. Instead
239
+ it _creates_ the jenkins jobs for you, on-demand. Specifically, it creates two
240
+ jenkins-jobs, and can create an optional third.
241
+
242
+ The first job is a matrix job, the cells of which are individual package tasks
243
+ for all of the build targets. This job takes four parameters:
244
+
245
+ 1) $PROJECT\_BUNDLE - a tar.gz of a git-bundle from HEAD of the current
246
+ project, which is cloned on the builder to set up a duplicate of this
247
+ environment
248
+
249
+ 2) $BUILD\_PROPERTIES - a build parameters file, containing all information
250
+ about the build
251
+
252
+ 3) $PROJECT - the project we're building, e.g. facter, puppet. This is used
253
+ later in determining the target for the build artifacts on the distribution
254
+ server
255
+
256
+ 4) $METRICS - a string of data points related to the build which are used for data
257
+ analysis. Contents of this string are items which cannot be obtained from within
258
+ the Jenkins job itself. Note that the Groovy postbuild script needed for metrics
259
+ gathering is dynamically passed to each job.
260
+
261
+ This first job clones the git bundle passed in as a parameter, then clones the
262
+ packaging repo (rake package:bootstrap) and for every cell in its matrix
263
+ performs a package build for a specific target (e.g. rake pl:deb
264
+ COW=base-lucid-i386.cow). Once all cells in the matrix complete (either succeed
265
+ or fail), this job automatically triggers the second of the new jobs
266
+ as a downstream job.
267
+
268
+ To receive an email notification from jenkins about the status of the packaging
269
+ job, pass NOTIFY=<recipient> as an environment variable to the uber_build
270
+ invocation, e.g.:
271
+
272
+ rake pl:jenkins:uber_build NOTIFY="foo@puppet.com bar@puppet.com"
273
+
274
+ The second job is an automatic repository creation task for this git repo.
275
+ Specifically, the job copies the git bundle from the packaging job and clones
276
+ it, and uses the git information in the git bundle to clone the packaging repo
277
+ and invoke the repository creation jobs `pl:jenkins:rpm_repos` and
278
+ `pl:jenkins:deb_repos`. The job will always be invoked, but will only actually
279
+ create repos if the upstream packaging job actually succeeded.
280
+
281
+ The third job is only created _if_ the environment variable
282
+ `DOWNSTREAM_JOB=<job_url>` was passed to the initial "pl:jenkins:uber_build"
283
+ invocation. This third job takes the value assigned to `DOWNSTREAM_JOB` and
284
+ creates a proxy jenkins job with a single build step, a curl call to this
285
+ value, presumably a url to a jenkins job to trigger programmatically.
286
+
287
+ An important note about `DOWNSTREAM_JOB`: `DOWNSTREAM_JOB` in the dynamic jenkins
288
+ workflow is _always_ invoked if it is passed in as an environment variable.
289
+ However, it is appended with an additional parameter, `PACKAGE_BUILD_STATUS`,
290
+ which will be the string "success" if package and repo builds succeeded, or
291
+ "failure" if package or repo builds failed. By modifying the actual downstream
292
+ jenkins job to accept a string parameter of `PACKAGE_BUILD_STATUS`, one can
293
+ switch on the success or failure of the packaging job, responding
294
+ appropriately. A second parameter, `PACKAGE_BUILD_URL` is also appended to
295
+ `DOWNSTREAM_JOB`, the value of which is the url of the packaging job itself.
296
+ This is to assist with tracing failures in a multi-jenkins environment. By
297
+ modifying the downstream jenkins job to accept a string parameter of
298
+ `PACKAGE_BUILD_URL`, one can use the value to display the url prominently in
299
+ case of failure, for example.
300
+
301
+ E.g., a job url:
302
+ http://jenkins.example.net/job/downstream/buildWithParameters?FOO=bar
303
+
304
+ in the success case will be transformed into
305
+
306
+ http://jenkins.example.net/job/downstream/buildWithParameters?FOO=bar&PACKAGE_BUILD_STATUS=success&PACKAGE_BUILD_URL=http://jenkins.example.net/job/packaging_job
307
+
308
+ and in the failure case transformed into
309
+
310
+ http://jenkins.example.net/job/downstream/buildWithParameters?FOO=bar&PACKAGE_BUILD_STATUS=failure&PACKAGE_BUILD_URL=http://jenkins.example.net/job/packaging_job
311
+
312
+ Since jenkins will just drop parameters that are not configured in the job,
313
+ accepting PACKAGE_BUILD_STATUS and PACKAGE_BUILD_URL in the downstream job
314
+ isn't mandatory.
315
+
316
+ All 3 jobs are configured by default for removal by jenkins after 3 days, to
317
+ avoid clutter.
318
+
319
+ The goal is to move toward migrating all of the jenkins tasks from the first
320
+ workflow, using a static job that is called many times per package, to this
321
+ second workflow of creating the jobs on demand.
322
+
323
+ ### Polling behavior with `pl:jenkins:uber_build:`
324
+
325
+ It is possible to pass an integer argument to the `uber_build` task which will be
326
+ used by the job as polling interval. If this value is passed in, the task will
327
+ query the dynamic job using the Jenkins API periodically until the build is
328
+ finished. Then, it will query the build to determine the SUCCESS/FAILURE status.
329
+ The job output will look something like:
330
+
331
+ ```bash
332
+ Packaging SUCCESS
333
+ Repo SUCCESS
334
+ ```
335
+
336
+ If for example the Packaging job had failed the output would look like this:
337
+
338
+ ```bash
339
+ Packaging FAILURE
340
+ ```
341
+
342
+ If a build fails then the rake task will terminate with a nonzero exit status
343
+ which can be used during CI or other automated contexts to detect and act on the
344
+ failure. This eliminates the need to pass a `DOWNSTREAM_JOB` variable to the
345
+ uber_build job although it is still possible to do so.
346
+
347
+ ## Task Explanations
348
+ For a listing of all available tasks and their functions, see the [Task
349
+ Dictionary](https://github.com/puppetlabs/packaging#task-dictionary)
350
+ at the end of this README.
351
+
352
+ ## Modules
353
+
354
+ A puppet module,
355
+ [puppetlabs-debbuilder](https://github.com/puppetlabs/puppetlabs-debbuilder),
356
+ has been created to stand up a debian build host compatible with the debian
357
+ side of this packaging repo. The rpm-side module,
358
+ [puppetlabs-rpmbuilder](https://github.com/puppetlabs/puppetlabs-rpmbuilder),
359
+ will set up an rpm builder.
360
+
361
+ ## Clean up
362
+ To remove the packaging repo, remove the ext/packaging directory or run `rake
363
+ package:implode`.
364
+
365
+ ##Setting up projects for the Packaging Repo
366
+
367
+ The packaging repo requires many project-side artifacts inside the ext
368
+ directory at the top level. [facter](https://github.com:puppetlabs/facter) and
369
+ [hiera](https://github.com:puppetlabs/hiera) are good examples.
370
+ It expects the following directory structure in the project
371
+
372
+ * ext/{debian,redhat,osx}
373
+
374
+ each of which contains templated erb files using the instance variables
375
+ specified in the setupvars task. These include a debian changelog, a redhat
376
+ spec file, and an osx preflight and plist.
377
+
378
+ The top level Rakefile or a separate task file in the project should have the following added:
379
+
380
+ (this assumes RAKE\_ROOT is defined in the top-level Rakefile using something like the following)
381
+ ```ruby
382
+ RAKE_ROOT = File.expand_path(File.dirname(__FILE__))
383
+ ```
384
+
385
+ ```ruby
386
+ build_defs_file = File.join(RAKE_ROOT, 'ext', 'build_defaults.yaml')
387
+ if File.exist?(build_defs_file)
388
+ begin
389
+ require 'yaml'
390
+ @build_defaults ||= YAML.load_file(build_defs_file)
391
+ rescue Exception => e
392
+ $stderr.puts "Unable to load yaml from #{build_defs_file}:"
393
+ raise e
394
+ end
395
+ @packaging_url = @build_defaults['packaging_url']
396
+ @packaging_repo = @build_defaults['packaging_repo']
397
+ raise "Could not find packaging url in #{build_defs_file}" if @packaging_url.nil?
398
+ raise "Could not find packaging repo in #{build_defs_file}" if @packaging_repo.nil?
399
+
400
+ namespace :package do
401
+ desc "Bootstrap packaging automation, e.g. clone into packaging repo"
402
+ task :bootstrap do
403
+ if File.exist?(File.join(RAKE_ROOT, "ext", @packaging_repo))
404
+ puts "It looks like you already have ext/#{@packaging_repo}. If you don't like it, blow it away with package:implode."
405
+ else
406
+ cd File.join(RAKE_ROOT, 'ext') do
407
+ %x{git clone #{@packaging_url}}
408
+ end
409
+ end
410
+ end
411
+ desc "Remove all cloned packaging automation"
412
+ task :implode do
413
+ rm_rf File.join(RAKE_ROOT, "ext", @packaging_repo)
414
+ end
415
+ end
416
+ end
417
+
418
+ begin
419
+ load File.join(RAKE_ROOT, 'ext', 'packaging', 'packaging.rake')
420
+ rescue LoadError
421
+ end
422
+ ```
423
+
424
+ Also in ext should be two files, build_defaults.yaml and project_data.yaml (optional).
425
+
426
+ This is the sample build_defaults.yaml file from Hiera:
427
+ ```yaml
428
+ ---
429
+ packaging_url: 'git@github.com:puppetlabs/packaging --branch=master'
430
+ packaging_repo: 'packaging'
431
+ default_cow: 'base-squeeze-i386.cow'
432
+ # Which debian distributions to build for. Noarch packages only need one arch of each cow.
433
+ cows: 'base-lucid-amd64.cow base-lucid-i386.cow base-natty-amd64.cow base-natty-i386.cow base-oneiric-amd64.cow base-oneiric-i386.cow base-precise-amd64.cow base-precise-i386.cow base-sid-amd64.cow base-sid-i386.cow base-squeeze-amd64.cow base-squeeze-i386.cow base-testing-amd64.cow base-testing-i386.cow base-wheezy-i386.cow'
434
+ # The pbuilder configuration file to use
435
+ pbuild_conf: '/etc/pbuilderrc'
436
+ # Alternate debian mirrors to build against (must be an array)
437
+ # The __DIST__ string is automatically replaced with codename of the cow being built, so when the squeeze cow is being built the deb_build_mirrors will be:
438
+ # deb http://apt.puppetlabs.com squeeze main dependencies
439
+ # deb http://somethingelse.com/debian squeeze
440
+ # This will happen for each cow during the build.
441
+ deb_build_mirrors:
442
+ - deb http://apt.puppetlabs.com __DIST__ main dependencies
443
+ - deb http://somethingelse.com/debian __DIST__
444
+ # Who is packaging. Turns up in various packaging artifacts
445
+ packager: 'puppetlabs'
446
+ # GPG key ID of the signer
447
+ gpg_key: '7F438280EF8D349F'
448
+ # Whether to require tarball signing as a prerequisite of other package building
449
+ sign_tar: FALSE
450
+ # a space separated list of mock configs. These are the rpm distributions to package for. If a noarch package, only one arch of each is needed.
451
+ final_mocks: 'pl-el-5-i386 pl-el-5-x86_64 pl-el-6-i386 pl-el-6-x86_64 pl-fedora-16-i386 pl-fedora-16-x86_64 pl-fedora-17-i386 pl-fedora-17-x86_64'
452
+ # The host that contains the yum repository to ship to
453
+ yum_host: 'yum.puppetlabs.com'
454
+ # The remote path the repository on the yum\_host
455
+ yum_repo_path: '/some/repo/'
456
+ # The host that contains the apt repository to ship to
457
+ apt_host: 'apt.puppetlabs.com'
458
+ # The URL to use for the apt dependencies in cow building
459
+ apt_repo_url: 'http://apt.puppetlabs.com'
460
+ # The path on the remote apt host that debs should ship to
461
+ apt_repo_path: '/opt/repository/incoming'
462
+ # The host that stores the tarballs for downloading
463
+ tar_host: 'downloads.puppetlabs.com'
464
+ # Whether to present the gem and apple tasks
465
+ build_gem: TRUE
466
+ build_dmg: TRUE
467
+ # Whether to execute the rdoc rake tasks prior to composing the tarball
468
+ build_doc: FALSE
469
+ # Whether to kick of a dynamic msi build job along side the uber_build
470
+ # If present, a dynamically generated jenkins job will be kicked off.
471
+ # The automation in puppet_for_the_win is used to build the msi with the
472
+ # following components.
473
+ build_msi:
474
+ puppet_for_the_win:
475
+ ref: 'origin/master'
476
+ repo: 'git://github.com/puppetlabs/puppet_for_the_win.git'
477
+ facter:
478
+ ref: 'refs/tags/2.1.0'
479
+ repo: 'git://github.com/puppetlabs/facter.git'
480
+ hiera:
481
+ ref: 'refs/tags/1.3.4'
482
+ repo: 'git://github.com/puppetlabs/hiera.git'
483
+ sys:
484
+ ref:
485
+ x86: 'origin/1.9.3-x86'
486
+ x64: 'origin/2.0.0-x64'
487
+ repo: 'git://github.com/puppetlabs/puppet-win32-ruby.git'
488
+ # Whether to present the Solaris 11 IPS packaging tasks
489
+ # This requires suitable IPS packaging artifacts in the project in ext/ips
490
+ build_ips: FALSE
491
+ # Whether this project is a PE project or not
492
+ build_pe: FALSE
493
+ # An optional task to execute pre-tarball composition. See the tasks in
494
+ # the 'pretasks' directory
495
+ pre_tar_task: 'package:vendor_gems'
496
+ ```
497
+ This is the sample project_data.yaml file:
498
+ ```yaml
499
+ ---
500
+ project: 'hiera'
501
+ author: 'Puppet'
502
+ email: 'info@puppet.com'
503
+ homepage: 'https://github.com/puppetlabs/hiera'
504
+ summary: 'Light weight hierarchical data store'
505
+ description: 'A pluggable data store for hierarcical data'
506
+ # file containing hard coded version information, if present
507
+ version_file: '/lib/hiera.rb'
508
+ # A string indicating the version strategy for the project (one of 'odd_even' or 'rc_final'), defaults to rc_final
509
+ # odd_even is a final release when the minor version is even, and a development release when it is odd
510
+ # rc_final is a final release when there is no rc at the end of the version string, and a development release when there is
511
+ version_strategy: 'rc_final'
512
+ # Boolean value of whether or not to automatically update the version file before packaging (defaults to false)
513
+ update_version_file: true
514
+ # files and gem\_files are space separated lists
515
+ # files to be packaged into a tarball and released with deb/rpm
516
+ files: '[A-Z]* ext lib bin spec acceptance_tests'
517
+ # space separated list of files to *exclude* from the tarball
518
+ # note that each listing in files, above, is recursively copied into the tarball, so
519
+ # 'tar\_excludes' only needs to include any undesired subdirectories/files of the 'files'
520
+ # list to exclude
521
+ tar_excludes: 'ext/packaging lib/some_excluded_file'
522
+ # Array of templates or globs of templates to evaluate. Note that without this key, the packaging will
523
+ # default to searching for any files in `ext` with the extension '.erb' and evaluate them. When this
524
+ # key is supplied, its values override the defaults, and all desired erb files must be specified with a path or glob.
525
+ templates:
526
+ - ext/redhat/project.spec.erb
527
+ - ext/templates/**/*.erb
528
+ # files to be packaged into a gem
529
+ gem_files: '{bin,lib}/**/* CHANGELOG COPYING README.md LICENSE'
530
+ # To exclude specific files from inclusion in a gem:
531
+ gem_excludes: 'lib/hiera/file_to_exclude.rb lib/hiera/directory_to_exclude'
532
+ # If gem name differs in some way from project name, e.g. only build gem for a client end
533
+ gem_name: hiera_the_gem
534
+ # If gem summary and/or description differs from general summary
535
+ gem_summary: 'A sub-set of the Hiera pluggable data store'
536
+ gem_description: 'A gem of the pluggable data store for hierarchical data'
537
+ gem_require_path: 'lib'
538
+ gem_test_files: 'spec/**/*'
539
+ gem_executables: 'hiera'
540
+ gem_default_executables: 'hiera'
541
+ # To add gem dependencies, indent.
542
+ # This is an example only, hiera doesn't really depend on hiera-puppet/json/facter
543
+ # For no specific version, leave version empty
544
+ gem_runtime_dependencies:
545
+ hiera-puppet: '1.0.0rc'
546
+ hiera-json:
547
+ gem_development_dependencies:
548
+ facter: '>= 1.6.11'
549
+ # To add gem dependencies which only apply to a specific platform, add the key "gem_platform_dependencies".
550
+ # The first key under the gem_platform_dependencies has to be a value that
551
+ # corresponds to a value of RUBY_PLATFORM. The subkeys are the same as the
552
+ # top-level gem dependency keys:
553
+ gem_platform_dependencies:
554
+ x86-mingw32:
555
+ gem_runtime_dependencies:
556
+ win32process: '~> 0.6.5'
557
+ gem_development_dependencies:
558
+ rake: '~> 0.9.0'
559
+ x86_64-darwin:
560
+ gem_runtime_dependencies:
561
+ CFPropertyList: '~> 2.2.4'
562
+ # rdoc options as an array
563
+ gem_rdoc_options:
564
+ - --line-numbers
565
+ - --main
566
+ - Hiera.README
567
+ ```
568
+ For basic mac packaging, add an osx directory in ext containing the following files:
569
+ 1. a preflight.erb template for any pre-flight actions, perhaps removing the old package if present.
570
+ 2. a prototype.plist.erb that is templated into the pkginfo.plist file for the package. This is the one from puppet. Note that these variable names aren't mutable here, but there's no need to worry about their value assignment, it's done in the apple task:
571
+ ```
572
+ <?xml version="1.0" encoding="UTF-8"?>
573
+ <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
574
+ <plist version="1.0">
575
+ <dict>
576
+ <key>CFBundleIdentifier</key>
577
+ <string><%= @title %></string>
578
+ <key>CFBundleShortVersionString</key>
579
+ <string><%= @version %></string>
580
+ <key>IFMajorVersion</key>
581
+ <integer><%= @package_major_version %></integer>
582
+ <key>IFMinorVersion</key>
583
+ <integer><%= @package_minor_version %></integer>
584
+ <key>IFPkgBuildDate</key>
585
+ <date><%= @build_date %></date>
586
+ <key>IFPkgFlagAllowBackRev</key>
587
+ <false/>
588
+ <key>IFPkgFlagAuthorizationAction</key>
589
+ <string>RootAuthorization</string>
590
+ <key>IFPkgFlagDefaultLocation</key>
591
+ <string>/</string>
592
+ <key>IFPkgFlagFollowLinks</key>
593
+ <true/>
594
+ <key>IFPkgFlagInstallFat</key>
595
+ <false/>
596
+ <key>IFPkgFlagIsRequired</key>
597
+ <false/>
598
+ <key>IFPkgFlagOverwritePermissions</key>
599
+ <false/>
600
+ <key>IFPkgFlagRelocatable</key>
601
+ <false/>
602
+ <key>IFPkgFlagRestartAction</key>
603
+ <string><%= @pm_restart %></string>
604
+ <key>IFPkgFlagRootVolumeOnly</key>
605
+ <true/>
606
+ <key>IFPkgFlagUpdateInstalledLanguages</key>
607
+ <false/>
608
+ </dict>
609
+ </plist>
610
+ ```
611
+ A file_mapping.yaml file that specifies a set of files and a set of directories
612
+ from the source to install, with destinations, ownership, and permissions. The
613
+ directories are top level directories in the source to install. The files are
614
+ files somewhere in the source to install. This is the one from puppet 3.x:
615
+ ```yaml
616
+ ---
617
+ directories:
618
+ # this will take the contents of lib, e.g. puppet/lib/\* and place them in
619
+ # /usr/lib/ruby/site\_ruby/1.8
620
+ lib:
621
+ path: 'usr/lib/ruby/site_ruby/1.8'
622
+ owner: 'root'
623
+ group: 'wheel'
624
+ perms: '0644'
625
+ bin:
626
+ path: 'usr/bin'
627
+ owner: 'root'
628
+ group: 'wheel'
629
+ perms: '0755'
630
+ 'man/man8':
631
+ path: 'usr/share/man/man8'
632
+ owner: 'root'
633
+ group: 'wheel'
634
+ perms: '0755'
635
+ files:
636
+ # this will take the file puppet/conf/auth.conf and place it in
637
+ # /private/etc/puppet/, creating the directory if not present
638
+ 'conf/auth.conf':
639
+ path: 'private/etc/puppet'
640
+ owner: 'root'
641
+ group: 'wheel'
642
+ perms: '0644'
643
+ 'man/man5/puppet.conf.5':
644
+ path: 'usr/share/man/man5'
645
+ owner: 'root'
646
+ group: 'wheel'
647
+ perms: '0644'
648
+ '[A-Z]*':
649
+ path: 'usr/share/doc/puppet'
650
+ owner: 'root'
651
+ group: 'wheel'
652
+ perms: '0644'
653
+ ```
654
+
655
+ ## Task Dictionary
656
+
657
+ * **package:apple**
658
+
659
+ Use `PackageMaker` to create a pkg package inside a dmg. Requires 'sudo'
660
+ privileges. `build_dmg: TRUE` must be set in `ext/build_defaults.yaml`.
661
+ Packages are staged in ./pkg/apple. See the Mac packaging section of
662
+ [Setting up projects for the Packaging
663
+ Repo](https://github.com/MosesMendoza/packaging/tree/more_documentation#setting-up-projects-for-the-packaging-repo).
664
+
665
+ * **package:bootstrap**
666
+
667
+ Clone the packaging repo into this project. This task isn't actually in the
668
+ packaging repo itself, but resides in the project. See [Setting up projects
669
+ for the Packaging
670
+ Repo](https://github.com/puppetlabs/packaging#setting-up-projects-for-the-packaging-repo).
671
+
672
+ * **package:deb**
673
+
674
+ Use `debbuild` to create a deb package and associated debian package
675
+ artifacts from the repository. Requires all build dependencies be satisfied
676
+ locally. Packages are staged in ./pkg/deb.
677
+
678
+ * **package:gem**
679
+ Use the `rubygems/package_task` library to create a rubygem from the
680
+ repository. Requires `build_gem: TRUE` and gem-related parameters be set in
681
+ `ext/build_defaults.yaml` and `ext/project\_data.yaml`. The gem is staged
682
+ in `./pkg`.
683
+
684
+ * **package:implode**
685
+
686
+ Remove the packaging repo entirely from the project. This task isn't
687
+ actually in the packaging repo itself, but resides in the project. See
688
+ [Setting up projects for the Packaging
689
+ Repo](https://github.com/puppetlabs/packaging#setting-up-projects-for-the-packaging-repo).
690
+
691
+ * **package:ips**
692
+
693
+ Use Solaris 11 pkg* tools to create a IPS package from the project.
694
+ Packages are staged in `./pkg/ips/pkgs`. Requires all `pkg`, `pkgdepend`,
695
+ `pkgsend`, `pkglint`, and `pkgmogrify`. Currently only puppet, facter, and
696
+ hiera have this capability.
697
+
698
+ * **package:rpm**
699
+
700
+ Use `rpmbuild` to create an rpm of the project. This will also make a
701
+ source rpm. Requires all build dependencies by satisfied locally. Packages
702
+ are staged in `./pkg/rpm`.
703
+
704
+ * **package:srpm**
705
+
706
+ Use `rpmbuild` to create a source rpm of the project. Source rpm is staged
707
+ in `./pkg/srpm`.
708
+
709
+ * **package:tar**
710
+
711
+ Create a source tarball of the project. The tarball is staged in `./pkg`.
712
+
713
+ * **package:update**
714
+
715
+ Update the clone of the packaging repo by pulling from origin.
716
+
717
+ * **pl:build_from_params**
718
+
719
+ Invoke a build from a build parameters yaml file. The parameters file
720
+ should be created with `rake pl:write_build_params`. The settings in the
721
+ build parameters file will override all values contained in
722
+ `./ext/build_defaults.yaml` and `./ext/project_data.yaml`.
723
+
724
+ * **pl:deb**
725
+
726
+ Use pdebuild with cowbuilder to create a debian package and associated
727
+ source artifacts from the default "cow", currently Debian Squeeze i386.
728
+ Requires that pbuilder/cowbuilder be installed and set up with a Debian
729
+ Squeeze cow. See the
730
+ [puppetlabs-debbuilder](https://github.com/puppetlabs/puppetlabs-debbuilder)
731
+ module for an easy way to set up a host for building with cows. The deb and
732
+ source artifacts are staged in `./pkg/deb/squeeze`.
733
+
734
+ * **pl:deb_all***
735
+
736
+ The same as `rake pl:deb`, but a package is built for every cow listed in
737
+ `ext/build_defaults.yaml` on the line `cows:<cows>`. The packages and
738
+ associated source artifacts are staged in `./pkg/deb/$distribution`, where
739
+ $distribution is the Debian/Ubuntu codename of the cow that was used to
740
+ build the package, e.g. "wheezy" or "precise."
741
+
742
+ * **pl:ips**
743
+
744
+ Invoke package:ips, but do so after pl:fetch and pl:load_extras, which load
745
+ signing/certificate information. The resulting packages are signed. IPS
746
+ packages are staged in `./pkg/ips/pkgs`
747
+
748
+ * **pl:jenkins:deb**
749
+
750
+ Perform `pl:deb` by posting a jenkins build, as described above. Once the
751
+ build is complete, run `pl:jenkins:retrieve` to retrieve the built
752
+ packages.
753
+
754
+ * **pl:jenkins:deb_all**
755
+
756
+ Perform what is a effectively a `pl:deb_all` but in a different fashion.
757
+ `pl:deb_all` performs debian cow builds in serial with every cow listed in
758
+ ext/build_defaults.yaml. `pl:jenkins:deb_all` splits the cows listed, and
759
+ posts a separate `pl:jenkins:deb` job for every cow listed to the jenkins
760
+ server, allowing jenkins to parallelize the building of packages for every
761
+ cow. Execute `pl:jenkins:retrieve` to retrieve all packages.
762
+
763
+ * **pl:jenkins:deb_repo_configs**
764
+
765
+ On the distribution server generate a listing of all debian repositories
766
+ that exist for the current SHA/ref of HEAD of the project repository. Then
767
+ generate debian apt client configuration files for every existing
768
+ repository. The distribution server is a web server, so the client
769
+ configurations can be placed on a debian client in /etc/apt/sources.list.d/
770
+ and the client will be able to install the built packages via apt. Requires
771
+ SSH access to the distribution server.
772
+
773
+ * **pl:jenkins:deb_repos**
774
+
775
+ On the distribution server, generate debian apt repositories for every
776
+ distribution containing any packages that are stored that match the current
777
+ SHA/ref of HEAD of the project repository. Requires SSH access to the
778
+ distribution server.
779
+
780
+ * **pl:jenkins:dmg**
781
+
782
+ Perform `package:apple` by posting a jenkins build. Run
783
+ `pl:jenkins:retrieve` to retrieve the built packages.
784
+
785
+ * **pl:jenkins:gem**
786
+
787
+ Perform `package:gem` by posting a jenkins build. Run `pl:jenkins:retrieve`
788
+ to retrieve the built packages.
789
+
790
+ * **pl:jenkins:mock**
791
+
792
+ Perform `pl:mock` by posting a jenkins build. Run `pl:jenkins:retrieve` to
793
+ retrieve the built packages.
794
+
795
+ * **pl:jenkins:mock_all**
796
+
797
+ Perform what is effectively a `pl:mock_all` but in a different fashion.
798
+ `pl:mock_all` performs mock builds in serial with every mock listed in
799
+ ext/build_defaults.yaml. `pl:jenkins:mock_all` splits the mocks listed, and
800
+ posts a separate `pl:jenkins:mock` job for every mock to the jenkins
801
+ server, allowing jenkins to parallelize the building of packages for every
802
+ mock configuration. The mock build root is randomized by the packaging
803
+ repo, avoiding conflicts with existing builds of the same mock
804
+ configuration. To retrieve built packages, call `pl:jenkins:retrieve`.
805
+
806
+ * **pl:jenkins:post[uri]**
807
+
808
+ Post to the jenkins server as specified in the team build_extras.yaml file,
809
+ with the job uri specified.
810
+
811
+ * **pl:jenkins:retrieve[target]**
812
+
813
+ Retrieve packages stored on the distribution server that have been built
814
+ from the current SHA/ref of HEAD of the project repository. Optionally pass
815
+ [target] to override the default, which is to retrieve the contents of the
816
+ "artifacts" subdirectory. Other targets are "repos" and "shipped".
817
+
818
+ * **pl:jenkins:rpm_repo_configs**
819
+
820
+ On the distribution server generate a listing of all yum rpm package
821
+ repositories that exist for the current SHA/ref of HEAD of the project
822
+ repository. Then generate yum client configuration files for every existing
823
+ repository. The distribution server is a web server, so the client
824
+ configurations can be placed on a redhat client in /etc/yum.repos.d/ and
825
+ the client will be able to install the packages via yum install. Requires
826
+ SSH access to the distribution server.
827
+
828
+ * **pl:jenkins:rpm_repos**
829
+
830
+ On the distribution server, generate yum rpm repositories for every
831
+ distribution containing any packages that are stored that match the current
832
+ SHA/ref of HEAD of the project repository. Requires SSH access to the
833
+ distribution server. The yum repos are created in a "repos" subdirectory of
834
+ the standard builds location, e.g.
835
+ /opt/jenkins-builds/$project/${SHA|ref}/repos, using everything currently
836
+ in the "artifacts" subdirectory of the same location.
837
+
838
+ * **pl:jenkins:ship[target]**
839
+
840
+ Take the packages staged in pkg/ and ship them to locations partially
841
+ specified by data in the project build_extras.yaml file. The current
842
+ paradigm is to ship the files to a subdirectory of /opt/jenkins-builds on
843
+ the distribution server. The subdirectory is constructed with the project
844
+ and SHA or ref of HEAD of the project repository. That is, if project HEAD
845
+ is currently at the tag "1.2.3", then the directory that packages will be
846
+ shipped to is /opt/jenkins-builds/$project/1.2.3/. If HEAD is a git SHA,
847
+ then "1.2.3" will instead be that SHA. By default, all artifacts in pkg/
848
+ will be shipped to a "artifacts" subdirectory of the standard target. E.g.
849
+ /opt/jenkins-builds/$project/1.2.3/artifacts. When a final shipping occurs,
850
+ e.g. when shipping signed artifacts into production, a second subdirectory
851
+ is created - "shipped" - and all artifacts that are shipped to production
852
+ are also shipped here. This allows a historical archive of all shipped
853
+ artifacts.
854
+
855
+ * **pl:jenkins:sign_all**
856
+
857
+ Take all packages staged in pkg/ and sign them via the various signing
858
+ tasks. All signing occurs on the distribution server:
859
+ * create a git-bundle of the project and rsync it to the distribution
860
+ * server ssh to the distribution server and clone the git-bundle, and clone
861
+ * the packaging repository rsync the contents of the local pkg/ directory
862
+ * into the pkg/ directory of the remote git project ssh to the distribution
863
+ * server and execute the following rake tasks:
864
+ - pl:sign_tar
865
+ - pl:sign_rpms
866
+ - pl:sign_deb_changes
867
+ * rsync the remote pkg/ directory contents to the local pkg/ directory
868
+
869
+ * **pl:jenkins:tar**
870
+
871
+ Perform `package:tar` by posting a jenkins build. Run `pl:jenkins:retrieve` to
872
+ retrieve the built packages.
873
+
874
+ * **pl:jenkins:uber_build**
875
+
876
+ Create a jenkins job on the fly that performs an aggregate of build tasks.
877
+ These include all the debian builds using `pl:deb COW=<cow>`, rpm builds
878
+ with `pl:mock MOCKS=<mock>`, `package:tar`, `package:apple`, and
879
+ `package:gem` if applicable. See
880
+ [jenkins-tasks](https://github.com/puppetlabs/packaging#jenkins-tasks)
881
+ above for more detail.
882
+
883
+ * **pl:jenkins:uber_ship**
884
+
885
+ An aggregate of retrieval, signing, and shipping tasks. Execute
886
+ `pl:jenkins:retrieve` to retrieve any packages on the distribution server
887
+ that were built from the SHA/ref of HEAD. Then `pl:jenkins:sign_all` to
888
+ sign all packages. Finally, `pl:uber_ship`, `pl:remote:freight`, and
889
+ `pl:remote:update_yum_repo`. `pl:jenkins:uber_build` combined with
890
+ `pl:jenkins:uber_ship` performs the entire build and release process for a
891
+ project.
892
+
893
+ * **pl:mock**
894
+
895
+ Use `mock` to build an rpm package using the default mock distribution,
896
+ Redhat Linux 5, i386. Requires that the `mock` package be installed. See
897
+ the
898
+ [puppetlabs-rpmbuilder](https://github.com/puppetlabs/puppetlabs-rpmbuilder)
899
+ module for an easy way to set up a host for building with mock. Resulting
900
+ rpm is staged in `./pkg/<repo_name>/el/rpm/5/(i386 | SRPMS)`. The
901
+ <repo_name> is determined by whether the package is a final version or not.
902
+ Whether the version is final or not is determined by parsing the
903
+ `git describe` string - anything containing 'rc', 'SNAPSHOT', '-dirty', a
904
+ git sha, or a dash after the version (e.g. 1.0.0-22) is considered non-final.
905
+
906
+ * **pl:mock_all**
907
+
908
+ The same as `rake pl:mock`, but a package is built for every mock listed in
909
+ `ext/build_defaults.yaml` on the line `mocks:<mocks>`. Packages are staged
910
+ in `./pkg/<repo_name>/(el | fedora)/$version/(i386 | x86_64 |
911
+ SRPMS)`. The subdirectories are dependent on the mock that is used. The
912
+ task assumes that the mock configurations are the standard Puppet mock
913
+ configurations that are generated by the
914
+ [puppetlabs-rpmbuilder](https://github.com/puppetlabs/puppetlabs-rpmbuilder)
915
+ module.
916
+
917
+ * **pl:print_build_params**
918
+
919
+ Print all build parameters to $stdout as they would be used in a package
920
+ build. This prints data that is loaded from `ext/build_defaults.yaml` and
921
+ `ext/project_data.yaml`, as well as whatever is overridden with environment
922
+ variables. Useful for debugging problems with parameter values.
923
+
924
+ * **pl:print_build_param[param]**
925
+
926
+ Print a specific build parameter to $stdout as it would be used in a package
927
+ build. This prints data that is loaded from `ext/build_defaults.yaml` and
928
+ `ext/project_data.yaml`, as well as whatever is overridden with environment
929
+ variables. Useful for debugging problems with parameter values. param
930
+ should be the name of the parameter as a symbol, e.g. :ref or :version
931
+
932
+ * **pl:remote:freight**
933
+
934
+ Performs an ssh call to the package server that calls a server-side rake
935
+ task. The rake task takes the debian packages that have (presumably) been
936
+ shipped via `pl:ship_debs` and invokes
937
+ [freight](https://github.com/rcrowley/freight) with them, which places them
938
+ in the apt repository.
939
+
940
+ * **pl:remote:update_yum_repo**
941
+
942
+ As with `pl:remote:freight`, this task performs an ssh call to the yum RPM
943
+ package server, and invokes an existing server-side rake task. The task
944
+ iterates through the `el` and `fedora` directories of the yum repository
945
+ and re-creates package server metadata for rpms in the subdirectories.
946
+
947
+ * **pl:ship_debs**
948
+
949
+ Rsync pkg/deb/* to the "incoming" directory on the debian apt package
950
+ repository server. Note: this task does not place the packages into
951
+ production - it is more accurate to consider the packages "staged" on the
952
+ repository server rather than actually shipped. The `pl:remote:freight`
953
+ task takes the packages in the "incoming" directory and actually places
954
+ them in the apt server.
955
+
956
+ * **pl:ship_gem**
957
+
958
+ Takes the built gem in pkg/ and pushes it to rubygems.com. This task assumes
959
+ you have the appropriate rubygems.com access and config to push the gem.
960
+
961
+ * **pl:ship_ips**
962
+
963
+ Takes the IPS packages in pkg/ips/pkgs/ and rsyncs them to a holding
964
+ directory on a package download server. This is not a true IPS server yet,
965
+ but just a basic file server. Eventually the goal is to have a true IPS
966
+ package repository running.
967
+
968
+ * **pl:ship_rpms**
969
+
970
+ Rsyncs the contents of pkg/el and pkg/fedora into the yum repository
971
+ server. While these packages are available immediately for download by
972
+ browsing the yum server directories directly, the yum repodata metadata has
973
+ not been updated, and thus the packages are not available to yum clients.
974
+ The `pl:remote:update_yum_repo` task updates the metadata, after which the
975
+ packages will be available to yum clients.
976
+
977
+ * **pl:sign_rpms**
978
+
979
+ Sign the rpms staged locally under pkg/ with the gpg key user ID (e.g.
980
+ email) specified in ext/build_defaults.yaml as `gpg_name`. This value can
981
+ be overridden by passing GPG_NAME as an environment variable to the rake
982
+ task.
983
+
984
+ * **pl:sign_tar**
985
+
986
+ Use gpg to create a detached signature of the tarball. By default this uses
987
+ the gpg_key value specified in ext/build_defaults.yaml in the project. This
988
+ can be overridden by passing GPG_KEY as an environment variable to the rake
989
+ task.
990
+
991
+ * **pl:tag**
992
+
993
+ Create a signed, annotated git tag of the current repository. Requires TAG
994
+ be passed as an environment variable to the rake task, which is the value
995
+ that will be used as both the tag message and the name of the tag. The gpg
996
+ key that is used for signing is assumed from gpg_key in
997
+ ext/build_defaults.yaml. This can be overridden by passing GPG_KEY as an
998
+ environment variable to the rake task.
999
+
1000
+ * **pl:uber_ship**
1001
+
1002
+ A composite task that performs the following tasks:
1003
+ `pl:ship_gem`
1004
+ `pl:ship_rpms`
1005
+ `pl:ship_debs`
1006
+ `pl:ship_dmgs`
1007
+ `pl:ship_tar`
1008
+ `pl:jenkins:ship`
1009
+ This is essentially a "ship all the things" task, but it is important to
1010
+ note that it does not update either yum or apt repo metadata on these
1011
+ respective servers - this has to be done via `pl:remote:update_yum_repo`
1012
+ and `pl:remote:freight`.
1013
+
1014
+ * **pl:update_ips_repo**
1015
+
1016
+ Take the packages in pkg/ips/pkg and rsync them to the IPS repository
1017
+ server specified in the build_extras.yaml file. Via ssh commands, call
1018
+ pkgrecv, pkgrepo, svcadm on the IPS repository server. Via ssh, restart the
1019
+ IPS repository service.
1020
+
1021
+ * **pl:write_build_params**
1022
+
1023
+ Generate a yaml file with all the build properties that have been loaded
1024
+ from build_defaults.yaml, project_data.yaml, (optionally)
1025
+ build_extras.yaml(s) via `pl:fetch`, and any environment variables. This
1026
+ file can be used by the packaging repo as a single source of truth for
1027
+ build data via `pl:build_from_params`. By default it is written to a
1028
+ temporary location and its location is printed to $stdout. To override the
1029
+ destination, pass OUTPUT_DIR as a environment variable to the task. By
1030
+ default, the name of the file will be either the git tag, if HEAD of the
1031
+ project repository is a tag, or the git sha of HEAD.