puppet 2.7.20 → 2.7.21

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of puppet might be problematic. Click here for more details.

Files changed (81) hide show
  1. data/CHANGELOG +226 -0
  2. data/conf/auth.conf +3 -3
  3. data/ext/packaging/README.md +191 -57
  4. data/ext/packaging/spec/spec_helper.rb +2 -2
  5. data/ext/packaging/spec/tasks/00_utils_spec.rb +63 -18
  6. data/ext/packaging/spec/tasks/build_object_spec.rb +171 -0
  7. data/ext/packaging/tasks/00_utils.rake +186 -33
  8. data/ext/packaging/tasks/10_setupvars.rake +94 -65
  9. data/ext/packaging/tasks/20_setupextravars.rake +45 -26
  10. data/ext/packaging/tasks/30_metrics.rake +41 -0
  11. data/ext/packaging/tasks/apple.rake +92 -36
  12. data/ext/packaging/tasks/build.rake +183 -0
  13. data/ext/packaging/tasks/deb.rake +45 -40
  14. data/ext/packaging/tasks/deb_repos.rake +103 -0
  15. data/ext/packaging/tasks/doc.rake +5 -5
  16. data/ext/packaging/tasks/fetch.rake +35 -10
  17. data/ext/packaging/tasks/gem.rake +38 -27
  18. data/ext/packaging/tasks/ips.rake +14 -14
  19. data/ext/packaging/tasks/jenkins.rake +337 -0
  20. data/ext/packaging/tasks/mock.rake +153 -72
  21. data/ext/packaging/tasks/pe_deb.rake +2 -2
  22. data/ext/packaging/tasks/pe_remote.rake +22 -19
  23. data/ext/packaging/tasks/pe_rpm.rake +5 -5
  24. data/ext/packaging/tasks/pe_ship.rake +31 -21
  25. data/ext/packaging/tasks/pe_sign.rake +20 -19
  26. data/ext/packaging/tasks/pe_sles.rake +40 -36
  27. data/ext/packaging/tasks/pe_tar.rake +5 -0
  28. data/ext/packaging/tasks/release.rake +32 -12
  29. data/ext/packaging/tasks/remote_build.rake +141 -83
  30. data/ext/packaging/tasks/retrieve.rake +23 -0
  31. data/ext/packaging/tasks/rpm.rake +11 -19
  32. data/ext/packaging/tasks/rpm_repos.rake +127 -0
  33. data/ext/packaging/tasks/ship.rake +68 -55
  34. data/ext/packaging/tasks/sign.rake +38 -10
  35. data/ext/packaging/tasks/tar.rake +25 -9
  36. data/ext/packaging/tasks/update.rake +2 -2
  37. data/ext/packaging/tasks/version.rake +34 -14
  38. data/ext/packaging/tasks/z_data_dump.rake +33 -0
  39. data/lib/puppet/indirector/catalog/compiler.rb +13 -2
  40. data/lib/puppet/indirector/certificate_status/file.rb +5 -0
  41. data/lib/puppet/indirector/errors.rb +5 -0
  42. data/lib/puppet/indirector/file_bucket_file/file.rb +4 -0
  43. data/lib/puppet/indirector/file_bucket_file/selector.rb +4 -0
  44. data/lib/puppet/indirector/indirection.rb +1 -0
  45. data/lib/puppet/indirector/resource/active_record.rb +3 -0
  46. data/lib/puppet/indirector/resource/ral.rb +4 -0
  47. data/lib/puppet/indirector/resource/store_configs.rb +3 -0
  48. data/lib/puppet/indirector/resource/validator.rb +8 -0
  49. data/lib/puppet/indirector/rest.rb +8 -0
  50. data/lib/puppet/indirector/run/local.rb +4 -0
  51. data/lib/puppet/indirector/terminus.rb +20 -0
  52. data/lib/puppet/network/formats.rb +3 -3
  53. data/lib/puppet/network/handler/master.rb +1 -1
  54. data/lib/puppet/network/handler/report.rb +1 -1
  55. data/lib/puppet/network/http/handler.rb +7 -1
  56. data/lib/puppet/network/http/rack/rest.rb +7 -2
  57. data/lib/puppet/network/http/webrick.rb +1 -0
  58. data/lib/puppet/network/rest_authconfig.rb +1 -1
  59. data/lib/puppet/parser/templatewrapper.rb +17 -17
  60. data/lib/puppet/util/monkey_patches.rb +58 -0
  61. data/lib/puppet/version.rb +1 -1
  62. data/spec/integration/indirector/catalog/compiler_spec.rb +1 -0
  63. data/spec/integration/indirector/catalog/queue_spec.rb +1 -1
  64. data/spec/integration/resource/catalog_spec.rb +1 -0
  65. data/spec/unit/indirector/catalog/compiler_spec.rb +29 -2
  66. data/spec/unit/indirector/indirection_spec.rb +18 -1
  67. data/spec/unit/indirector/terminus_spec.rb +191 -177
  68. data/spec/unit/network/formats_spec.rb +6 -6
  69. data/spec/unit/network/http/handler_spec.rb +25 -0
  70. data/spec/unit/network/http/rack/rest_spec.rb +17 -0
  71. data/spec/unit/network/http/webrick_spec.rb +4 -0
  72. data/spec/unit/network/http_pool_spec.rb +0 -1
  73. data/spec/unit/network/rest_authconfig_spec.rb +16 -1
  74. data/spec/unit/parser/functions/inline_template_spec.rb +13 -0
  75. data/spec/unit/parser/functions/template_spec.rb +15 -0
  76. data/spec/unit/parser/templatewrapper_spec.rb +19 -4
  77. data/spec/unit/ssl/certificate_request_spec.rb +2 -0
  78. data/spec/unit/ssl/host_spec.rb +1 -0
  79. data/spec/unit/util/monkey_patches_spec.rb +12 -0
  80. data/test/language/snippets.rb +1 -1
  81. metadata +13 -2
data/CHANGELOG CHANGED
@@ -1,3 +1,229 @@
1
+ 2.7.21
2
+ ===
3
+ 570e4d2 Update CHANGELOG, PUPPETVERSION for 2.7.21
4
+ 4b0a7e2 Add missing 2.7.20 CHANGELOG entries
5
+ 3ecd376 (#19391) Find the catalog for the specified node name
6
+ c240299 (#19392) Don't validate key for certificate_status
7
+ a1c4abd Don't assume master supports SSLv2
8
+ 60eebed Don't require openssl client to return 0 on failure
9
+ 12728c0 Display SSL messages so we can match our regex
10
+ 70cdc63 Don't assume puppetbindir is defined
11
+ 9cbfb9d Remove unnecessary rubygems require
12
+ 0f4ac20 Run openssl from windows when trying to downgrade master
13
+ 7d62aa0 Separate tests for same CVEs into separate files
14
+ 61109fa Fix order-dependent test failure in rest_authconfig_spec
15
+ a3d3c95 Always read request body when using Rack
16
+ 79b875e Acceptance tests for CVEs 2013 (1640, 1652, 1653, 1654, 2274, 2275)
17
+ 632e12d (#19531) (CVE-2013-2275) Only allow report save from the node matching the certname
18
+ 7df884b Fix module tool acceptance test
19
+ 4a272ea Updating module tool acceptance tests with new expectations.
20
+ 0a7d61f (#19392) (CVE-2013-1653) Validate instances passed to indirector
21
+ be920ac (#19151) Reject SSLv2 SSL handshakes and ciphers
22
+ 516142e (#19391) (CVE-2013-1652) Disallow use_node compiler parameter for remote requests
23
+ bd942ec (#14093) Restore access to the filename in the template
24
+ cf6cf81 (#14093) Remove unsafe attributes from TemplateWrapper
25
+ f2a3d5c (#19393) Safely load YAML from the network
26
+
27
+ 2.7.20
28
+ ===
29
+ cb9696d Update lib/puppet/version.rb for 2.7.20
30
+ 1d01f2a Update lib/puppet/version.rb for 2.7.20-rc1
31
+ d16d357 (#17458) Add acceptance test for executing external subcommands
32
+ fd92eb1 (#17458) Partial revert of "don't rebuild regexp every time"
33
+ 28229d7 Revert "(#17458) Load 'puppet' at the right time"
34
+ b5ed15b (#17458) Load 'puppet' at the right time
35
+ 0524a0a (#7422) Support arrow syntax with metaparams
36
+ fc6571e rm packaging artifacts in rpm spec, debian rules
37
+ b1adcd2 Fix build targets for 2.7.x
38
+ 0cc80ad Preserve timestamps when installing files
39
+ 6da5aa8 Replace dashes with dots for gem version
40
+ 5c61f32 (#17260) Include link to information about deprecation
41
+ 858e4b4 Update Rakefile to make rspec optional
42
+ 0c1991f Remove the asc file from the source of spec file
43
+ bd0caf5 (#17260) Warn when variables contain hyphens
44
+ 35b9ff2 Edit description of hyphenated variables compatibility setting
45
+ 5ee2558 (#10146) `-` in variable names should be deprecated!
46
+ d892859 (#16791) Remove tests for unused behavior
47
+ aba3c67 (#16791) Re-instate deprecated symbolize and symbolizehash!
48
+ 65b5c5c Eliminate `symbolize` everywhere, and remove it.
49
+ 3cda4f4 (#16791) Revert "Merge branch 'feature/2.7.x/json-terminus-backport' of https://github.com/daniel-pittman/puppet into 2.7.x"
50
+ 1e5baee (#16791) Revert "Maint: Fix intermittently failing spec test"
51
+ 13ab956 (#16791) Revert "Merge branch 'perf/2.7.x/backport-activesupport-hook-disabling' of https://github.com/daniel-pittman/puppet into 2.7.x"
52
+ 5c96254 (#16581) Fix mis-stubbed test
53
+ c305a32 (#11042) indirect via the property to convert group names to numbers
54
+ f08fc39 (#16581) Refactor code for sid validation
55
+ 92f0688 (#16798) Add more tests to cover revoke by serial number
56
+ b55d885 (#16798) Update revocation by serial number documentation
57
+ 5e8236f (#16798) Fix certificate revocation by serial number
58
+ c60a82d (#16581) Deprecate sid_for_account
59
+ c86e685 (#16581) Use native Win32 APIs to resolve SIDs in file provider
60
+ 9291ae2 (#16581) Use native Win32 APIs to resolve SIDs in providers
61
+ 3d360e7 (#16581) Documentation changes
62
+ be848e6 (#16581) Use win32-security gem to resolve SIDs
63
+ da3aea4 (#16581) Refactor code for converting string and binary sids
64
+ e6865c4 (maint) Update mailmap for git shortlog
65
+ dc50ec7 (#16922) Quote strings that contain ":"
66
+ b2e31b5 fail better in package repo rake tasks
67
+ 5e7cce5 Disable ActiveSupport::Dependencies hooks.
68
+ 68dea3d zaml: handle multi-line map keys and array values correctly.
69
+ 67e5957 zaml: don't use backreferences on strings
70
+ fbca3b0 zaml: avoid calling emit when we don't have anything to output
71
+ 4f6289b zaml: add whitespace for readability.
72
+ 83defc0 zaml: rework strings for correctness and speed
73
+ e45961f Maint: Fix intermittently failing spec test
74
+ 69315c2 Implement `Fixnum#ord` for Ruby 1.8.5
75
+ f60e402 tagging: use a static regexp rather than rebuilding it
76
+ 314c3da file_serving: avoid rewriting paths if possible.
77
+ b54f8cd log_paths: memoize the entire value, not just part of it.
78
+ 33c127b zaml: speed micro-optimization for Time output
79
+ e70273b whit: cache the `to_s` result.
80
+ 9099a42 zaml: don't sort object attributes for output
81
+ 20efe94 util: don't rebuild regexp every time we invoke absolute_file?
82
+ 58f6ca1 parser/files: use a cheaper test for "is this path absolute?"
83
+ 275c94d Eliminate remaining uses of FileCollection.
84
+ 912ed34 FileCollection is a memory loss, not a memory win.
85
+ 89e0f14 type: support implicit "identity" transformation
86
+ 397dfa8 monkey_patch: cache Symbol to_proc result.
87
+ 54c4f0a lexer: remove some unused stats collection
88
+ 9176f5c tagging: document why fixing this horror is hard.
89
+ aca7959 type: memoize key_attributes for types.
90
+ 8173a6e Avoid object creation/destruction when possible.
91
+ 5e3fbca (Maint) Remove fragile test
92
+ fd8343a zaml: unify on a single definition of `to_zaml` for Symbol
93
+ 4d4a75a zaml: statically determine the to_ascii8bit implementation.
94
+ fbd5105 Use `intern` rather than `symbolize` it Puppet::Type
95
+ c0e5f4d Implement Symbol#intern when it is missing.
96
+ 141c83b Add catalog JSON terminus to improve performance.
97
+ a0287bf Implement a JSON file terminus to mirror YAML.
98
+ 61a34d3 (#2888) Fix race condition with puppetdlockfile
99
+ 315ebad (#2888) Add settings catalog info to README_DEVELOPER
100
+ fe1f4a2 (#4680) Reject CA network operations when master CA is disabled
101
+ 312b467 Git ignore an .rvmrc file in the root of the project
102
+ f0ef301 (#16330) Update ZAML to latest upstream.
103
+ f8a4ddc (#16376) Fix rails compatibility layer for activerecord < 3.0
104
+ a1c4467 Fixup apple package plist for use with packaging repo
105
+ e84f448 (#16347) Qualify references to windows-pr gem
106
+ 68c61fa (#16347) Replace \x92 character with apostrophe
107
+ 0e7fc8d Update copyright years in LICENSE
108
+ 81c8afa Skip symbolic link demanding test on Windows
109
+ d260b90 RSpec 3 is deprecating `expect {}.should` - respect that.
110
+ dff6e53 (maint) Confine pty-based test to non-Windows hosts.
111
+ b48a172 (Maint) Fix syntax and semantic erros in test
112
+ da24d54 (#15959) Fix groupadd/useradd spec when run on non-linux systems
113
+ bd875f5 (#16208): Correct spec test to match new failed parameter message
114
+ a753b86 Maint: Fix inconsistent expect-should usage
115
+ 1a9d122 Maint: Fix leaked tempfile handle
116
+ d9ab06f (#16208) fix tests
117
+ f30af22 (#16208) use #{ref} instead of #{type} #{title} as suggested by dpittman
118
+ cb3c912 (#16208) Better error messages for failed parameter validation
119
+ 294fae2 Port the SemVer test from Test::Unit to RSpec.
120
+ 14d1fdc Don't make executables in /tmp in posix_spec
121
+ ca36893 Backport fix for commandline spec
122
+ 6f556ef Don't pollute Functions namespace in shellquote function
123
+ 13cb623 (#8714) Don't follow symlinks in SELinux FS detection
124
+ 8fa6575 Make SELinux internal helper functions private.
125
+ 0a72c5c Fix yum repo path in yaml file
126
+ 0620497 (#15797) Update chkconfig test to reflect reality
127
+ 775d0dc Revert "(#15797) Change the argument to chkconfig from 'on' to 'reset'"
128
+ f9463ea (#15731) Add clarifying comments to host.rb
129
+ 82ebe28 Remove version check regex from puppet
130
+ 67f8b0b Remove broken dual build-requires of facter in redhat spec
131
+ 2ea1e98 Add in manpage listings that should have been there
132
+ b1648b2 Fixup Rakefile to use packaging repo
133
+ a670a4f Move and template all apple pkging artifacts to ext/osx
134
+ 302248b Add debian packaging artifacts and template
135
+ 5dc896b Move redhat spec file to erb template in ext/redhat
136
+ b67e4dc (#16019) Don't add Unix paths to Windows search path
137
+ 489352b (#16119) Recognize carriage return escape sequence
138
+ db74432 Switch off building gem for 2.7.x
139
+ 77afeba Add yaml files
140
+ 2b06b7e (#15193) Fix windows failing on unix style paths
141
+ 5e84c97 (#15560) Delete user's profile after deleting account
142
+ 94aaf15 (#15193) Fix StaticCompiler#store_content
143
+ 58f9679 (#15193) Add spec test coverage for static compiler
144
+ 2bbc4e4 Document the different forms of catalogs to help test writers
145
+ 1a432fd (#15193) Document how to use the static compiler
146
+ 1820927 (Maint) Stop watchr from reading non-existent spec.opts
147
+ 576fd56 Whitespace only cleanup
148
+ 67b9227 (#15560) Create the user's profile if managehome is true
149
+ 1075b74 (#15560) Refactor LogonUser
150
+ 7efbfc6 Maint: Large or complex HEREDOC strings w/ no interpolation should be single-quote type
151
+ 13e0109 (#15591) Skip test that causes segfault on Windows
152
+ f10d364 maint: remove .rspec entirely
153
+ 7b4ac10 (#15731) Move explicit fingerprints to hash
154
+ 4ffc02d (Maint) Enable colorized rspec output on Windows
155
+ 700caba (#15739) Bump report_version since #10064 change its format
156
+ 1a2e8fc (#14822) Use feature confine for feature tests during run
157
+ c6e1aa3 (#14822) Re-evaluate features if they previously were false
158
+ 65dbdea (#15797) Disable RedHat services in all run levels
159
+ 5ef0df0 (Maint) Whitespace only cleanup
160
+ 7ca6b70 (#15797) Add test coverage for redhat service provider
161
+ d2012ae (#15797) Change the argument to chkconfig from 'on' to 'reset'
162
+ b2c3675 Fix typo in create_resources function's description string
163
+ 48a9cc2 (#15464) Exclude gem command from bundler
164
+ 54ea4fe Revert "Revert "Merge branch 'ticket/2.7.x/15464_a_gemfile_would_improve_contributor_on-boarding' into 2.7.x""
165
+ 76ef99b Revert "Merge branch 'ticket/2.7.x/15464_a_gemfile_would_improve_contributor_on-boarding' into 2.7.x"
166
+ 6350e6c (#15959) Do not support system group on Solaris and HP-UX
167
+ c404d99 maint: Reduce stubbing in groupadd spec
168
+ c50f552 maint: Rearrange tests in groupadd unittests
169
+ 5e64c1f (#15959) Do not support system user on Solaris and HP-UX
170
+ 03c7ddf1 maint: Reduce stubbing in useradd spec
171
+ e111f56 maint: Rearrange tests in useradd unittests
172
+ d7cb3c7 (#15464) Make Puppet.version settable via Puppet.version=
173
+ f9cfec7 (#15912) Make function access consistent
174
+ e067c2a (#15693) Allow restart command to be specified
175
+ 07f0b0e (#15464) Make Puppet usable as a Gem from source
176
+ b33d517 Move Puppet version into lib/puppet/version.rb
177
+ 1c1a383 (#15464) Make contributing easy via bundle Gemfile
178
+ 83cfe9d (Maint) Fix rspec options file
179
+ 86dac63 (Maint) Skip crontab filetype tests on Windows
180
+ dd25c9e (#15346) Add spec test for --source when checking latest
181
+ e9d10ca (#15346) add --source to the gem list command
182
+ 1d80007 (#14283) Call Tempfile#close and #unlink separately
183
+ 7a72361 (#14283) Raise an error when crontab returns non-zero on AIX
184
+ d9c7698 (maint) Refactor filetype specs
185
+ bbdbb06 (maint) Consolidate flat file tests
186
+ 5ceb55c (Maint) Add logging for when Puppet is enabled/disabled.
187
+ 07aeef3 (#15940) Add test coverage for rdoc exclude option
188
+ f08dd53 (#15940) Update puppet doc to ignore files and templates in modules
189
+ 08c62da (#7442) solaris: return "-1" for password_max_age when password aging is disabled
190
+ e0b1e40 (#7442) solaris: add tests for password_max_age attribute
191
+ 0a41053 Always set log functions on root environment
192
+ b47e910 Use TestHelper in old unit tests
193
+ 9815de4 Convert expect..should to expect..to in configurer spec
194
+ f2a4cca (maint) Remove spurious `puts` line
195
+ c95bda7 (maint) Whitespace cleanup
196
+ ed22b06 (maint) Include backtrace in re-raised exception
197
+ c8152d4 (#15920) Make forward-sexp ignore comments in puppet-mode.el
198
+ 4cd6686 (#14283) Fix suntab filetype when run as normal user
199
+ d2b4f08 maint: Refactor suntab and aixtab filetype
200
+ a7d6c3d Add Puppet::Parser::Functions::clear for specs
201
+ 279c323 Clear Environment.current when calling Environment.clear
202
+ 536b68c Check that function loading worked correctly.
203
+ abc7885 Avoid stubbing Thread::new due to issues in newer mocha
204
+ fdb8ca3 Rename specs to end in _spec.rb
205
+ 9c361f1 Don't require mocha in spec_helper
206
+ 6d3d591 Fix stubbing on nil in some specs
207
+ 60a6a2a (#14515) Tests fail with mocha 0.11.4
208
+ d2e5268 (#10915) Add test for provider throwing NoMethodError
209
+ 9e26fe0 (#10915) Fix tests to work with new parameter checking
210
+ da7cd8e Don't mask NoMethodErrors
211
+ fcd4957 (#15521) Convert to using host stub wrapper
212
+ fbede51 Fixed #11686 - Updated documentation for exec timeout
213
+ 6f7303a Mark git diff --check as inline code
214
+ cf04e5e Make the example in CONTRIBUTING imperative and concrete
215
+ 9862940 Add HOWTO hint on topic branches in CONTRIBUTING
216
+ 9c60ed3 (Maint) Whitespace cleanup of CONTRIBUTING.md
217
+ 65b35a8 (Maint) Simplify CONTRIBUTING.md
218
+ b61d041 (#15731) explicit fingerprint digest for pson
219
+ 6dcf122 (#15731) Full certificate metadata for pson
220
+ 31fac81 (#15665) Fix issue with eix-update on newer eix (>=0.25.2).
221
+ adfc27e Maint: Fix borked help/man output for node clean
222
+ 367926a (#15264) Custom title patterns can fail cryptically.
223
+ b8ca748 (#11727) Support STDIN for `puppet parser validate`
224
+ 04ba8e2 Add stdin parsing to puppet parser validate
225
+ 368a1cb Fix for #3984 -- SELinux warns about changes it doesn't actually make
226
+
1
227
  2.7.19
2
228
  ===
3
229
  85f5543 Ruby 1.9.3 has a different error when `require` fails.
@@ -63,10 +63,10 @@ path /certificate_revocation_list/ca
63
63
  method find
64
64
  allow *
65
65
 
66
- # allow all nodes to store their reports
67
- path /report
66
+ # allow all nodes to store their own reports
67
+ path ~ ^/report/([^/]+)$
68
68
  method save
69
- allow *
69
+ allow $1
70
70
 
71
71
  # inconditionnally allow access to all files services
72
72
  # which means in practice that fileserver.conf will
@@ -11,14 +11,28 @@ continue to iterate and improve upon it.
11
11
 
12
12
  ##Using the Packaging Repo
13
13
 
14
- Several Puppet Labs projects have been migrated to the packaging repo,
15
- including puppet, facter, puppet-dashboard, and hiera. Generally speaking,
16
- ruby 1.9.3 and rake 0.9.x seem to work best. To pull the packaging tasks into
17
- your source repo, do a `rake package:bootstrap`. This will clone this repo
18
- into the ext directory of the project and make many packaging tasks
19
- available. The tasks are generally grouped into two categories, `package:`
20
- namespaced tasks and `pl:` namespaced tasks.
14
+ Several Puppet Labs projects are using the packaging repo. They are:
21
15
 
16
+ * puppet
17
+ * facter
18
+ * puppet-dashboard
19
+ * hiera
20
+ * puppetdb
21
+ * razor
22
+
23
+ as well as several closed-source projects, including
24
+ * live-management
25
+ * console-auth
26
+ * console
27
+
28
+ Generally speaking, the packaging repo should be compatible with ruby 1.8.7,
29
+ ruby 1.9.3 and rake 0.9.x. To pull the packaging tasks into your source repo,
30
+ do a `rake package:bootstrap`. This will clone this repo into the ext directory
31
+ of the project and make many packaging tasks available. The tasks are
32
+ generally grouped into two categories, `package:` namespaced tasks and `pl:`
33
+ namespaced tasks.
34
+
35
+ ## `package:` tasks
22
36
  `package:` namespaced tasks are general purpose tasks that are set up to use
23
37
  the most minimal tool chain possible for creating packages. These tasks will
24
38
  create rpms and debs, but any build dependencies will need to be satisifed by
@@ -28,59 +42,170 @@ for rolling one's own debs and rpms or for use in environments without many
28
42
  OSes/versions, this may work just fine. To build an rpm using the packaging
29
43
  repo, do a `rake package:rpm`. To build a deb, use `rake package:deb`.
30
44
 
45
+ ## `pl:` tasks
31
46
  `pl:` namespaced tasks rely on a slighly more complex toolchain for packaging
32
- inside clean chroot environments for the various operating systems and
33
- versions that Puppet Labs supports. On the rpm side, this is done with
47
+ inside clean chroot environments for the various operating systems and versions
48
+ that Puppet Labs supports. On the rpm side, this is done with
34
49
  [mock](http://fedoraproject.org/wiki/Projects/Mock) and for debs, we use
35
50
  pdebuild and [cowbuilder](http://wiki.debian.org/cowbuilder). For the most
36
51
  part, these tasks are keyed to puppetlabs infrastructure, and are used by the
37
52
  Release Engineering team to create release packages. However, they can
38
53
  certainly be modified to suit other environments, and much effort went into
39
54
  making tasks as modular and reusable as possible. Several Puppet Labs-specific
40
- tasks are only available if the file '~/.packaging/builder\_data.yaml' is present.
41
- This file is created by the `pl:fetch` task, which pulls down another yaml file
42
- from a [separate build data repository](https://github.com/puppetlabs/build-data),
43
- which contains additional settings/data specific to Puppet Labs release
44
- infrastructure. The goal in separating these data and tasks out is to refrain
45
- from presenting by default yet more Puppet Labs-specific tasks that aren't
46
- generally consumable by everyone. To build a deb from a local repository using
47
- a `pl` task, ssh into a builder (e.g., one stood up using the modules detailed
48
- below) and clone the source repo, e.g. puppet. Then, run `rake package:bootstrap`
49
- and `rake pl:deb` to create a deb, and `rake pl:mock` to make an rpm (on a debian
50
- or redhat host, respectively).
51
-
52
- There is also a `pe:` namespace, for the building of Puppet Labs' Puppet
53
- Enterprise packages that have been converted to using this repo. The `pe:`
54
- tasks rely heavily on PL internal infrastructure, and are not generally useful
55
- outside of this environment. To create packages, in the source repository run
56
- `rake package:bootstrap`, followed by `rake pl:fetch`. These two commands
57
- bootstrap the packaging environment and pull in the additional data needed for
58
- PE building (see `pl:fetch` notes above).
59
- Then, to make a debian package, run `rake pe:deb`, and to make an rpm, run
60
- `rake pe:mock`. There are also `pe:deb_all` and `pe:mock_all` tasks, which build
61
- packages against all shipped debian/redhat targets. The `pe:deb_all` task is not
62
- generally necessary for developer use for building test packages; the `pe:deb`
63
- task creates a package that will work against virtually all supported PE debian
64
- versions. The same is generally true for PE internal rpms, but because of variances
65
- in build macros for rpm, rpms should generally be built with `pe:mock_all`, and
66
- then the desired version installed, or by building only for a specific target.
67
- This is accomplished by passing MOCK=<mock> to the rake call, e.g. `rake pe:mock MOCK=<mock>`.
68
- The available mocks are listed in `ext/build_defaults.yaml` after `final_mocks:`.
69
- For PE, the mocks are formatted as `pupent-<peversion>-<distversion>-<arch>`, e.g.
70
- `pupent-2.7-el5-i386`. To build for a specific target, set `MOCK=<mock>` to the mock
71
- that matches the target. The `pe:deb` and `pe:mock` tasks work by building on a
72
- remote builder using the current committed state of the source repository. To forego
73
- remote building and build on the local station (e.g., by ssh-ing into a remote
74
- builder first), the tasks `pe:local_mock` and `pe:local_deb` build using the
75
- local host.
55
+ tasks are only available if the file '~/.packaging' is present. This file is
56
+ created by the `pl:fetch` task, which curls two yaml files into 'team' and
57
+ 'project' subdirectories. from a [separate build data
58
+ repository](https://github.com/puppetlabs/build-data), which contains
59
+ additional settings/data specific to Puppet Labs release infrastructure. The
60
+ goal in separating these data and tasks out is to refrain from presenting by
61
+ default yet more Puppet Labs-specific tasks that aren't generally consumable by
62
+ everyone. To build a deb from a local repository using a `pl` task, ssh into a
63
+ builder (e.g., one stood up using the modules detailed below) and clone the
64
+ source repo, e.g. puppet. Then, run `rake package:bootstrap` and `rake pl:deb`
65
+ to create a deb, and `rake pl:mock` to make an rpm (on a debian or redhat host,
66
+ respectively).
67
+
68
+ ## `pe:` tasks
69
+ There is also a `pe:` namespace, for the building of Puppet
70
+ Labs' Puppet Enterprise packages that have been converted to using this repo.
71
+ The `pe:` tasks rely heavily on PL internal infrastructure, and are not
72
+ generally useful outside of this environment. To create packages, in the source
73
+ repository run `rake package:bootstrap`, followed by `rake pl:fetch`. These two
74
+ commands bootstrap the packaging environment and pull in the additional data
75
+ needed for PE building (see `pl:fetch` notes above). Then, to make a debian
76
+ package, run `rake pe:deb`, and to make an rpm, run `rake pe:mock`. There are
77
+ also `pe:deb_all` and `pe:mock_all` tasks, which build packages against all
78
+ shipped debian/redhat targets. The `pe:deb_all` task is not generally necessary
79
+ for developer use for building test packages; the `pe:deb` task creates a
80
+ package that will work against virtually all supported PE debian versions.
81
+ The same is generally true for PE internal rpms, but because of variances in
82
+ build macros for rpm, rpms should generally be built with `pe:mock_all`, and
83
+ then the desired version installed, or by building only for a specific
84
+ target. This is accomplished by passing MOCK=<mock> to the rake call, e.g.
85
+ `rake pe:mock MOCK=<mock>`. The available mocks are listed in
86
+ `ext/build_defaults.yaml` after `final_mocks:`. For PE, the mocks are
87
+ formatted as `pupent-<peversion>-<distversion>-<arch>`, e.g.
88
+ `pupent-2.7-el5-i386`. To build for a specific target, set `MOCK=<mock>` to
89
+ the mock that matches the target. The `pe:deb` and `pe:mock` tasks work by
90
+ using the `:remote` tasks for building on a remote builder using the current
91
+ committed state of the source repository. To forego remote building and build
92
+ on the local station (e.g., by ssh-ing into a remote builder first), the
93
+ tasks `pe:local_mock` and `pe:local_deb` build using the local host.
94
+
95
+ ## `:remote:` tasks
96
+ There are also sub-namespaces of `:pl` and `:pe` that are
97
+ worth noting. First, the `:remote` namespace. Tasks under `:remote` perform
98
+ builds remotely on internal builders from your local workstation. How they
99
+ work:
100
+
101
+ 1) Run `pl:fetch` to obtain extra data from the build-data repo. The data
102
+ includes the hostnames of builders to use for packaging.
103
+
104
+ 2) Create a git bundle of the local workspace and tar it up.
105
+
106
+ 3) Create a build parameters file. The params file includes all the information
107
+ about the build, including any values overridden with env vars, and the actual
108
+ task to run, e.g. `rake pl:deb`.
109
+
110
+ 4) scp the git bundle and build parameters file to a temporary directory on the
111
+ builder hostname assigned to that particular package build type.
112
+
113
+ 5) ssh into the builder, untar the git bundle, clone it, and run `rake
114
+ package:bootstrap`.
115
+
116
+ 6) ssh into the builder, cd into the cloned repo, and run `rake
117
+ pl:build_from_params PARAMS_FILE=/path/to/previously/sent/file`.
118
+
119
+ 7) Maintain the ssh connection until the build finishes, and rsync the packages
120
+ from the builder to the local workstation.
121
+
122
+ ## `:jenkins:` tasks
123
+ Jenkins tasks are similar to the `:remote:` tasks, but
124
+ they do not require ssh access to the builders. The jenkins tasks enable the
125
+ packaging repo to kick off packaging builds on a remote jenkins slave. They
126
+ work in a similar way to the :remote tasks, but with a few key differences. The
127
+ jenkins tasks transmit information to a jenkins coordinator, which handles the
128
+ rest. The data passed are the following:
129
+
130
+ 1) $PROJECT\_BUNDLE - a tar.gz of a git-bundle from HEAD of the current
131
+ project, which is cloned on the builder to set up a duplicate of this
132
+ environment
133
+
134
+ 2) $BUILD\_PROPERTIES - a build parameters file, containing all information
135
+ about the build
136
+
137
+ 3) $BUILD\_TYPE - the "type" of build, e.g. rpm, deb, gem, etc The jenkins url
138
+ and job name are obtained via the team build-data file from [the build data
139
+ repository](https://github.com/puppetlabs/build-data)
140
+
141
+ 4) $PROJECT - the project we're building, e.g. facter, puppet. This is used
142
+ later in determining the target for the build artifacts on the distribution
143
+ server
144
+
145
+ 5) $DOWNSTREAM\_JOB - The URL of a downstream job that jenkins should post to
146
+ upon success. This is obtained via the DOWNSTREAM\_JOB environment variable.
147
+
148
+
149
+ On the Jenkins end, the job is a parameterized job that accepts five
150
+ parameters. Jenkins has the Parameterized Trigger Plugin, Workspace Cleanup
151
+ Plugin, and Node and Label Parameter Plugin in use for this job. The workspace
152
+ cleanup plugin cleans the workspace before each build. Two are file parameters,
153
+ two string parameters, and a Label parameter provided by the Node and Label
154
+ Parameter Plugin, as described above. When the pl:jenkins:\* task triggers a
155
+ build, it passes values for all of these parameters. The Label parameter is
156
+ associated with the build type. This way we can queue the job on a builder with
157
+ the appropriate capabilities just by assigning a builder the label "deb" or
158
+ "rpm," etc. The actual build itself is accomplished via a shell build task. The
159
+ contents of the task are:
160
+
161
+ ```bash
162
+ #################
163
+
164
+ SHA=$(echo $BUILD_PROPERTIES | cut -d '.' -f1)
165
+
166
+ echo "Build type: $BUILD_TYPE"
167
+
168
+ ### Create a local clone of the git-bundle that was passed
169
+ # The bundle is a tarball, and since this is a project-agnostic
170
+ # job, we don't actually know what's in it, just that it's a
171
+ # git bundle.
172
+
173
+
174
+ [ -f "PROJECT_BUNDLE" ] || exit 1
175
+ mkdir project && tar -xzf PROJECT_BUNDLE -C project/
176
+
177
+ cd project
178
+ git clone --recursive $(ls) git_repo
179
+
180
+ cd git_repo
181
+
182
+ ### Clone the packaging repo
183
+ rake package:bootstrap && rake pl:fetch
184
+
185
+ ### Perform the build
186
+ rake pl:load_extras pl:build_from_params PARAMS_FILE=$WORKSPACE/BUILD_PROPERTIES
187
+
188
+ ### Send the results
189
+ rake pl:jenkins:ship["artifacts"]
190
+
191
+ ### If a downstream job was passed, trigger it now
192
+ if [ -n "$DOWNSTREAM_JOB" ] ; then
193
+ rake pl:jenkins:post["$DOWNSTREAM_JOB"]
194
+ fi
195
+
196
+ #################
197
+ ```
198
+
199
+ ## Modules
76
200
 
77
201
  A puppet module,
78
202
  [puppetlabs-debbuilder](https://github.com/puppetlabs/puppetlabs-debbuilder),
79
203
  has been created to stand up a debian build host compatible with the debian
80
204
  side of this packaging repo. The rpm-side module,
81
205
  [puppetlabs-rpmbuilder](https://github.com/puppetlabs/puppetlabs-rpmbuilder),
82
- is currently a work in progress and.
206
+ will set up an rpm builder.
83
207
 
208
+ ## Clean up
84
209
  To remove the packaging repo, remove the ext/packaging directory or run `rake
85
210
  package:implode`.
86
211
 
@@ -97,7 +222,7 @@ each of which contains templated erb files using the instance variables
97
222
  specified in the setupvars task. These include a debian changelog, a redhat
98
223
  spec file, and an osx preflight and plist.
99
224
 
100
- The top level Rakefile in the project should have the following added:
225
+ The top level Rakefile or a separate task file in the project should have the following added:
101
226
 
102
227
  ```ruby
103
228
  Dir['ext/packaging/tasks/**/*.rake'].sort.each { |t| load t }
@@ -143,20 +268,24 @@ This is the sample build_defaults.yaml file from Hiera:
143
268
  packaging_url: 'git@github.com:puppetlabs/packaging --branch=master'
144
269
  packaging_repo: 'packaging'
145
270
  default_cow: 'base-squeeze-i386.cow'
271
+ # Which debian distributions to build for. Noarch packages only need one arch of each cow.
146
272
  cows: 'base-lucid-amd64.cow base-lucid-i386.cow base-natty-amd64.cow base-natty-i386.cow base-oneiric-amd64.cow base-oneiric-i386.cow base-precise-amd64.cow base-precise-i386.cow base-sid-amd64.cow base-sid-i386.cow base-squeeze-amd64.cow base-squeeze-i386.cow base-testing-amd64.cow base-testing-i386.cow base-wheezy-i386.cow'
273
+ # The pbuilder configuration file to use
147
274
  pbuild_conf: '/etc/pbuilderrc'
275
+ # Who is packaging. Turns up in various packaging artifacts
148
276
  packager: 'puppetlabs'
277
+ # Who is signing packages
149
278
  gpg_name: 'info@puppetlabs.com'
279
+ # GPG key ID of the signer
150
280
  gpg_key: '4BD6EC30'
151
281
  # Whether to require tarball signing as a prerequisite of other package building
152
282
  sign_tar: FALSE
153
- # a space separated list of mock configs, one set for final releases, another for devel
154
- final_mocks: 'pl-5-i386 pl-5-x86_64 pl-6-i386 pl-6-x86_64 fedora-15-i386 fedora-15-x86_64 fedora-16-i386 fedora-16-x86_64 fedora-17-i386 fedora-17-x86_64'
155
- rc_mocks: 'pl-5-i386-dev pl-5-x86_64-dev pl-6-i386-dev pl-6-x86_64-dev fedora-15-i386-dev fedora-15-x86_64-dev fedora-16-i386-dev fedora-16-x86_64-dev fedora-17-i386-dev fedora-17-x86_64-dev'
283
+ # a space separated list of mock configs. These are the rpm distributions to package for. If a noarch package, only one arch of each is needed.
284
+ final_mocks: 'pl-el-5-i386 pl-el-5-x86_64 pl-el-6-i386 pl-el-6-x86_64 pl-fedora-16-i386 pl-fedora-16-x86_64 pl-fedora-17-i386 pl-fedora-17-x86_64'
156
285
  # The host that contains the yum repository to ship to
157
286
  yum_host: 'burji.puppetlabs.com'
158
- # The remote path the repository on the yum_host
159
- yum_repo_path: '~/repo/'
287
+ # The remote path the repository on the yum\_host
288
+ yum_repo_path: '/some/repo/'
160
289
  # The host that contains the apt repository to ship to
161
290
  apt_host: 'burji.puppetlabs.com'
162
291
  # The URL to use for the apt dependencies in cow building
@@ -178,12 +307,12 @@ summary: 'Light weight hierarchical data store'
178
307
  description: 'A pluggable data store for hierarcical data'
179
308
  # file containing hard coded version information, if present
180
309
  version_file: '/lib/hiera.rb'
181
- # files and gem_files are space separated lists
310
+ # files and gem\_files are space separated lists
182
311
  # files to be packaged into a tarball and released with deb/rpm
183
312
  files: '[A-Z]* ext lib bin spec acceptance_tests'
184
313
  # space separated list of files to *exclude* from the tarball
185
314
  # note that each listing in files, above, is recursively copied into the tarball, so
186
- # 'tar_excludes' only needs to include any undesired subdirectories/files of the 'files'
315
+ # 'tar\_excludes' only needs to include any undesired subdirectories/files of the 'files'
187
316
  # list to exclude
188
317
  tar_excludes: 'ext/packaging lib/some_excluded_file'
189
318
  # files to be packaged into a gem
@@ -256,11 +385,15 @@ For basic mac packaging, add an osx directory in ext containing the following fi
256
385
  </dict>
257
386
  </plist>
258
387
  ```
259
- A file_mapping.yaml file that specifies a set of files and a set of directories from the source to install, with destinations, ownership, and permissions. The directories are top level directories in the source to install. The files are files somewhere in the source to install. This is the one from puppet 3.x:
388
+ A file_mapping.yaml file that specifies a set of files and a set of directories
389
+ from the source to install, with destinations, ownership, and permissions. The
390
+ directories are top level directories in the source to install. The files are
391
+ files somewhere in the source to install. This is the one from puppet 3.x:
260
392
  ```yaml
261
393
  ---
262
394
  directories:
263
- # this will take the contents of lib, e.g. puppet/lib/* and place them in /usr/lib/ruby/site_ruby/1.8
395
+ # this will take the contents of lib, e.g. puppet/lib/\* and place them in
396
+ # /usr/lib/ruby/site\_ruby/1.8
264
397
  lib:
265
398
  path: 'usr/lib/ruby/site_ruby/1.8'
266
399
  owner: 'root'
@@ -277,7 +410,8 @@ directories:
277
410
  group: 'wheel'
278
411
  perms: '0755'
279
412
  files:
280
- # this will take the file puppet/conf/auth.conf and place it in /private/etc/puppet/, creating the directory if not present
413
+ # this will take the file puppet/conf/auth.conf and place it in
414
+ # /private/etc/puppet/, creating the directory if not present
281
415
  'conf/auth.conf':
282
416
  path: 'private/etc/puppet'
283
417
  owner: 'root'