image_builder 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +22 -0
  3. data/.rubocop.yml +16 -0
  4. data/Gemfile +4 -0
  5. data/LICENSE.txt +22 -0
  6. data/README.md +32 -0
  7. data/Rakefile +29 -0
  8. data/bin/image_builder +234 -0
  9. data/image_builder.gemspec +30 -0
  10. data/lib/image_builder.rb +38 -0
  11. data/lib/image_builder/backends/backend_base.rb +10 -0
  12. data/lib/image_builder/backends/packer.rb +109 -0
  13. data/lib/image_builder/builders/aws_base.rb +101 -0
  14. data/lib/image_builder/builders/aws_chroot.rb +33 -0
  15. data/lib/image_builder/builders/aws_ebs.rb +15 -0
  16. data/lib/image_builder/builders/aws_instance.rb +45 -0
  17. data/lib/image_builder/builders/builder_base.rb +12 -0
  18. data/lib/image_builder/builders/null.rb +39 -0
  19. data/lib/image_builder/post_processors/compress.rb +24 -0
  20. data/lib/image_builder/post_processors/post_processors_base.rb +10 -0
  21. data/lib/image_builder/post_processors/vagrant.rb +41 -0
  22. data/lib/image_builder/provisioners/chef_base.rb +96 -0
  23. data/lib/image_builder/provisioners/chef_client.rb +115 -0
  24. data/lib/image_builder/provisioners/chef_solo.rb +33 -0
  25. data/lib/image_builder/provisioners/file.rb +26 -0
  26. data/lib/image_builder/provisioners/provisioner_base.rb +10 -0
  27. data/lib/image_builder/provisioners/shell.rb +47 -0
  28. data/lib/image_builder/version.rb +4 -0
  29. data/spec/aws_chroot_builder_spec.rb +30 -0
  30. data/spec/aws_ebs_builder_spec.rb +30 -0
  31. data/spec/aws_instance_builder_spec.rb +30 -0
  32. data/spec/chef_client_provisioner_spec.rb +32 -0
  33. data/spec/chef_solo_provisioner_spec.rb +28 -0
  34. data/spec/compress_postprocessor_spec.rb +15 -0
  35. data/spec/fixtures/test-knife.rb +40 -0
  36. data/spec/null_builder_spec.rb +21 -0
  37. data/spec/packer_backend_spec.rb +145 -0
  38. data/spec/vagrant_postprocessor_spec.rb +20 -0
  39. metadata +236 -0
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 96fdbf490de1409c12b7a35690ec161016076e46
4
+ data.tar.gz: 4cf8e63dd9fc03672579739c72ee8e6a95571718
5
+ SHA512:
6
+ metadata.gz: 4a69a23cb0a72a8cb509511c4848f79b8534222e67252384d5d43ec5a4eac99cab99001d99ddf5d80655b5ffa06a4501aefd47a58c44a0b8ee3d76713e6b0766
7
+ data.tar.gz: 8bc8a87b2541b9e78c51bed13ddad2f0227e267ebefd53ba41aca9cd4157cdb492ee1acacc1143c8022cd0be5537c24da79c1038ebe684bbbea18ae359f199f5
data/.gitignore ADDED
@@ -0,0 +1,22 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ Gemfile.lock
7
+ InstalledFiles
8
+ _yardoc
9
+ coverage
10
+ doc/
11
+ lib/bundler/man
12
+ pkg
13
+ rdoc
14
+ spec/reports
15
+ test/tmp
16
+ test/version_tmp
17
+ tmp
18
+ *.bundle
19
+ *.so
20
+ *.o
21
+ *.a
22
+ mkmf.log
data/.rubocop.yml ADDED
@@ -0,0 +1,16 @@
1
+ AllCops:
2
+ Exclude:
3
+ - 'image_builder.gemspec'
4
+ - spec/fixtures/**
5
+
6
+ Metrics/LineLength:
7
+ Max: 132
8
+
9
+ Metrics/MethodLength:
10
+ Max: 25
11
+
12
+ Metrics/CyclomaticComplexity:
13
+ Max: 10
14
+
15
+ Metrics/PerceivedComplexity:
16
+ Max: 10
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in image_builder.gemspec
4
+ gemspec
data/LICENSE.txt ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2014 Mike Morris
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,32 @@
1
+ # ImageBuilder
2
+
3
+ A gem to build operating system images for various platforms. At initial release, this gem supports
4
+ building images using packer to build images for the AWS platform
5
+
6
+ ## Installation
7
+
8
+ Add this line to your application's Gemfile:
9
+
10
+ gem 'image_builder'
11
+
12
+ And then execute:
13
+
14
+ $ bundle
15
+
16
+ Or install it yourself as:
17
+
18
+ $ gem install image_builder
19
+
20
+ ## Usage
21
+
22
+ This is how you use the gem, should probably write something useful here.
23
+ But since it's just a library gem that basically just wraps the packer utility,
24
+ read this code, and the packer documentation to figure out what to do
25
+
26
+ ## Contributing
27
+
28
+ 1. Fork it ( https://github.com/[my-github-username]/image_builder/fork )
29
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
30
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
31
+ 4. Push to the branch (`git push origin my-new-feature`)
32
+ 5. Create a new Pull Request
data/Rakefile ADDED
@@ -0,0 +1,29 @@
1
+ require 'bundler/gem_tasks'
2
+ require 'rspec/core/rake_task'
3
+
4
+ gem_dir = ::File.dirname(__FILE__)
5
+
6
+ desc 'Rubocop'
7
+ task rc: [:rubocop]
8
+ task :rubocop do
9
+ sh "bundle exec rubocop -D #{gem_dir}"
10
+ end
11
+
12
+ task default: [:rc] do
13
+ Rake::Task[:spec].invoke('progress')
14
+ end
15
+
16
+ desc 'Run rspec'
17
+ RSpec::Core::RakeTask.new(:spec, [:format, :tags]) do |t, args|
18
+ format = args[:format] || 'documentation'
19
+ tags = args[:tags] || []
20
+ t.verbose = false
21
+ t.fail_on_error = true
22
+ t.rspec_opts = "--format=#{format}"
23
+
24
+ tags.flatten.compact.each do |tag|
25
+ t.rspec_opts += " --tag #{tag}"
26
+ end
27
+
28
+ t.ruby_opts = '-W0'
29
+ end
data/bin/image_builder ADDED
@@ -0,0 +1,234 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'commander'
4
+ require 'cloudutil/aws/ec2'
5
+ require 'chef/node'
6
+ require 'json'
7
+
8
+ # All sorts of funkiness happens if I try to put these in the
9
+ # top-level image_builder module, and try to require just that
10
+ require 'image_builder/version'
11
+ require 'image_builder/backends/packer'
12
+ require 'image_builder/builders/aws_ebs'
13
+ require 'image_builder/provisioners/chef_client'
14
+ require 'image_builder/provisioners/chef_solo'
15
+ require 'image_builder/provisioners/shell'
16
+ require 'image_builder/post_processors/vagrant'
17
+
18
+ def prog_name
19
+ ::File.basename(__FILE__)
20
+ end
21
+
22
+ def attr_missing?(attr_value)
23
+ attr_value.nil? || attr_value.strip.empty?
24
+ end
25
+
26
+ def read_attrs(attr_file)
27
+ # I didn't want to have to dip into the chef gem to do this,
28
+ # but it appears to be the best way to get attributes
29
+ node = Chef::Node.build('image-builder')
30
+ node.from_file(::File.expand_path(attr_file))
31
+ node.attributes.merged_attributes
32
+ end
33
+
34
+ def create_chef_client_provisioner(knife_file, chef_attrs, options)
35
+ chef_client_prov = ImageBuilder::Provisioners::Chef::Client.from_file(knife_file)
36
+ chef_client_prov.chef_environment = chef_attrs['amibakery']['build']['chef_env']
37
+ chef_client_prov.node_name = "#{prog_name}_{{uuid}}"
38
+ chef_client_prov.run_list = options.r_run_list
39
+ chef_client_prov.json = JSON.parse(options.j_json) # Placeholder for custom json used to pass in build-specific data
40
+ chef_client_prov
41
+ end
42
+
43
+ def resolve_sec_grps(grp_ary, util_obj)
44
+ grps = []
45
+
46
+ grp_ary.each do |g|
47
+ grps << util_obj.resolve_security_group_id(g)
48
+ end
49
+
50
+ grps.flatten.compact.uniq
51
+ end
52
+
53
+ def format_string(str, subs = {})
54
+ str % subs unless str.nil?
55
+ end
56
+
57
+ def format_tags(hash, subs = {})
58
+ new_hash = hash
59
+
60
+ unless hash.nil?
61
+ hash.each_pair do |k, v|
62
+ new_hash[k] = format_string(v, subs)
63
+ end
64
+ end
65
+
66
+ new_hash
67
+ end
68
+
69
+ # rubocop:disable Metrics/MethodLength
70
+ def create_builders(attrs, options)
71
+ # Create builder objects, resolve AWS names/tags to ids along the way
72
+ builders = []
73
+ cu = Cloudutil::AWS::EC2.new
74
+
75
+ build_attrs = attrs['amibakery']['build']
76
+ %w(centos ubuntu).each do |p|
77
+ build_attrs[p].keys.each do |v|
78
+ plat_attrs = build_attrs[p][v]
79
+ root_type = plat_attrs['root_dev']
80
+ virt_type = plat_attrs['virt_type']
81
+ sub_hash = { p: p, v: v, root_type: root_type, virt_type: virt_type }
82
+
83
+ b = ImageBuilder::Builders::AWS::EBS.new
84
+ b.name = "#{p}#{v}-#{root_type}"
85
+ b.ami_name = format_string(options.n_ami_name, sub_hash)
86
+ b.instance_type = build_attrs['inst_type']
87
+ b.source_ami = cu.resolve_ami_id(plat_attrs['ami_id'])
88
+ b.ssh_username = plat_attrs['build_user']
89
+ b.ami_description = format_string(options.ami_description, sub_hash)
90
+ b.ami_regions = options.c_copy_regions
91
+ b.ami_users = options.u_users
92
+ b.ami_virtualization_type = virt_type
93
+ b.security_group_ids = resolve_sec_grps([build_attrs['sec_grp_id']].flatten, cu)
94
+ b.ssh_private_ip = true
95
+ b.subnet_id = cu.resolve_subnet_id(build_attrs['subnet_id'])
96
+ b.tags = { Name: b.ami_name }.merge(format_tags(options.tags.clone, sub_hash))
97
+ b.launch_block_device_mappings = b.class.default_launch_block_device_mappings
98
+ b.ami_block_device_mappings = b.class.default_block_device_mappings
99
+
100
+ builders << b
101
+ end
102
+ end
103
+
104
+ builders
105
+ end
106
+ # rubocop:enable Metrics/MethodLength
107
+
108
+ def run_packer(builders, provisioners, postprocessors, path = nil)
109
+ ImageBuilder::Backends::Packer.packer_path(path) unless path.nil? || path.strip.empty?
110
+ packer = ImageBuilder::Backends::Packer.new
111
+
112
+ builders.each do |b|
113
+ packer.add_builder(b)
114
+ end
115
+
116
+ provisioners.each do |p|
117
+ packer.add_provisioner(p)
118
+ end
119
+
120
+ postprocessors.each do |p|
121
+ packer.add_post_processor(p)
122
+ end
123
+
124
+ packer.build(['-machine-readable'])
125
+ end
126
+
127
+ def convert_tags(tag_opt)
128
+ # Should only ever be an empty Hash (option default),
129
+ # or an Array from the command-line
130
+ hash = {}
131
+
132
+ if tag_opt.is_a? Array
133
+ tag_opt.each do |t|
134
+ (tag, value) = t.split('=', 2)
135
+ hash[tag] = value
136
+ end
137
+ end
138
+
139
+ hash
140
+ end
141
+
142
+ # NOTE: Array options are in format '--opt arg1,arg2' (comma-seperated, no space!)
143
+ Commander.configure do
144
+ program :name, prog_name
145
+ program :version, ImageBuilder::VERSION
146
+ program :description, 'A program to generate AWS AMIs with Chef and Packer.'
147
+ .concat(' It is very opinionated about how it works, so use it as a reference, and write your own')
148
+
149
+ default_command :build
150
+ global_option('-d', '--debug', 'Provide debugging output')
151
+
152
+ command :build do |c|
153
+ c.syntax = "#{prog_name} [build] [OPTIONS]"
154
+ c.description = 'A program to build AWS AMIs using Chef and Packer. This is the default action to run, if not supplied'
155
+
156
+ c.option '-r, --run-list RUN_LIST', Array, 'The run-list for Chef to execute to provision the node'
157
+ c.option '-c, --copy-regions REGIONS', Array, 'A list of regions to copy the generated AMI to'
158
+ c.option '-u, --users USERS', Array, 'A list of users to grant access to the generated AMI'
159
+ c.option '-s, --serverspec-path SS_PATH', String, 'The path to the serverspec directory to use to validate the build'
160
+ c.option '-k, --knife-file KNIFE_FILE', String, 'The path to a knife config file to use to configure a Chef provisioner'
161
+ c.option '-a, --attrib-file ATTRIB_FILE', String, 'The path to a Chef recipes attribute file to use to configure an AWS builder'
162
+ c.option '-p, --packer-path PACKER_PATH', String, 'The path to the packer executable, if it\'s in a non-standard location'
163
+ c.option '-j, --json JSON', String, 'A string of JSON text to pass along to the chef-client provisioner'
164
+ c.option '-n, --ami-name NAME', String, 'The name of the ami to create'
165
+ c.option '--tags TAGS', Array, 'Tags to apply to the AMI (Name tag is auto-generated as --ami-name value)'
166
+ c.option '--ami-description DESC', String, 'A description of the ami'
167
+ c.option '--cleanup-run-list RUN_LIST', Array, 'A run-list used to clean up the provisioned node before AMI creation'
168
+ c.option '--[no-]vagrant-box', 'Create a Vagrant box defintion after the build completes'
169
+
170
+ c.action do |_args, options|
171
+ options.default vagrant_box: false
172
+ options.default tags: {}
173
+
174
+ knife_file = options.k_knife_file
175
+ attrib_file = options.a_attrib_file
176
+
177
+ # Do a fixup for --tags arg to convert from Array to Hash
178
+ options.tags = convert_tags(options.tags)
179
+
180
+ # Check to make sure -k, -a, & -n args are given, since it's missing from doing in Commander
181
+ err_msgs = []
182
+
183
+ err_msgs << 'Missing required -k option' if attr_missing? knife_file
184
+ err_msgs << 'Missing required -a option' if attr_missing? attrib_file
185
+ err_msgs << 'Missing required -n option' if attr_missing? options.n_ami_name
186
+
187
+ fail ArgumentError, err_msgs.join("\n") unless err_msgs.empty?
188
+
189
+ # create AWS builder info via data from -a option
190
+ chef_attrs = read_attrs(attrib_file)
191
+ builders = create_builders(chef_attrs, options)
192
+
193
+ # create chef-client provisioner from knife.rb and chef attributes
194
+ chef_client_prov = create_chef_client_provisioner(knife_file, chef_attrs, options)
195
+
196
+ dir_prov = ImageBuilder::Provisioners::Shell.new
197
+ dir_prov.inline = ['sudo mkdir -p -m 777 /etc/chef /tmp/packer-chef-client']
198
+
199
+ # Do 'require' here, to avoid collisions with potential unqualified references to the File class
200
+ require 'image_builder/provisioners/file'
201
+
202
+ unless attr_missing? chef_client_prov.encrypted_data_bag_secret_path
203
+ sec_prov = ImageBuilder::Provisioners::File.new
204
+ sec_prov.source = chef_client_prov.encrypted_data_bag_secret_path
205
+ sec_prov.destination = '/etc/chef/encrypted_data_bag_secret'
206
+ end
207
+
208
+ unless attr_missing? options.s_serverspec_path
209
+ ss_file_prov = ImageBuilder::Provisioners::File.new
210
+ ss_file_prov.source = options.s_serverspec_path
211
+ ss_file_prov.destination = '/var/tmp'
212
+
213
+ ss_shell_prov = ImageBuilder::Provisioners::Shell.new
214
+ ss_shell_prov.inline = ['/var/tmp/serverspec/serverspec.sh']
215
+ end
216
+
217
+ unless options.cleanup_run_list.nil?
218
+ cleanup_prov = ImageBuilder::Provisioners::Chef::Solo.new
219
+ cleanup_prov.run_list = [options.cleanup_run_list].flatten
220
+ cleanup_prov.remote_cookbook_paths = ['/var/chef/cache/cookbooks']
221
+ cleanup_prov.skip_install = true
222
+ end
223
+
224
+ # add vagrant post-processor, if requested
225
+ vagrant = ImageBuilder::PostProcessors::Vagrant.new if options.vagrant_box
226
+
227
+ # order matters!
228
+ provisioners = [dir_prov, sec_prov, chef_client_prov, ss_file_prov, ss_shell_prov, cleanup_prov].compact
229
+
230
+ # run packer
231
+ run_packer(builders, provisioners, [vagrant].compact, options.p_packer_path)
232
+ end
233
+ end
234
+ end
@@ -0,0 +1,30 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'image_builder/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = 'image_builder'
8
+ spec.version = ImageBuilder::VERSION
9
+ spec.authors = ['Mike Morris']
10
+ spec.email = ['michael.m.morris@pearson.com']
11
+ spec.summary = 'A gem to create operating system images using various methods'
12
+ spec.description = IO.read(File.join(File.dirname(__FILE__), 'README.md'))
13
+ spec.homepage = ''
14
+ spec.license = 'MIT'
15
+
16
+ spec.files = `git ls-files -z`.split("\x0")
17
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
18
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
19
+ spec.require_paths = ['lib']
20
+
21
+ spec.add_development_dependency 'bundler', '~> 1.6'
22
+ spec.add_development_dependency 'rake'
23
+ spec.add_development_dependency 'rspec'
24
+ spec.add_development_dependency 'rubocop'
25
+
26
+ spec.add_dependency 'mixlib-shellout'
27
+ spec.add_dependency 'commander'
28
+ spec.add_dependency 'cloudutil'
29
+ spec.add_dependency 'chef', '~> 11.12'
30
+ end
@@ -0,0 +1,38 @@
1
+ require 'image_builder/version'
2
+
3
+ # Generic top-level module comment
4
+ module ImageBuilder
5
+ protected
6
+
7
+ def attr_to_hash(src_hash, attr_sym, required = false)
8
+ val = send(attr_sym)
9
+
10
+ if required
11
+ src_hash[attr_sym] = val
12
+ else
13
+ unless val.nil?
14
+ if val.respond_to? :empty?
15
+ v = check_empty(val)
16
+ src_hash[attr_sym] = v unless v.nil? # rubocop:disable Metrics/BlockNesting
17
+ else
18
+ # Not nil, and doesn't support empty?, so assign
19
+ src_hash[attr_sym] = val
20
+ end
21
+ end
22
+ end
23
+ end
24
+
25
+ private
26
+
27
+ def check_empty(val)
28
+ if val.is_a? String
29
+ # Strip whitespace surrounding strings before testing for empty
30
+ return val unless val.strip.empty?
31
+ else
32
+ # Array, Hash, etc...
33
+ return val unless val.empty?
34
+ end
35
+
36
+ nil
37
+ end
38
+ end