builderator 0.3.15 → 1.0.0.pre.rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile +9 -0
  3. data/Gemfile.lock +440 -0
  4. data/README.md +72 -18
  5. data/Rakefile +1 -2
  6. data/VERSION +1 -1
  7. data/bin/build-clean +102 -0
  8. data/bin/build-data +45 -0
  9. data/builderator.gemspec +7 -4
  10. data/docs/configuration.md +154 -0
  11. data/docs/configuration/cookbook.md +19 -0
  12. data/docs/configuration/profile.md +71 -0
  13. data/docs/versioning.md +65 -0
  14. data/lib/builderator.rb +3 -0
  15. data/lib/builderator/config.rb +93 -0
  16. data/lib/builderator/config/attributes.rb +287 -0
  17. data/lib/builderator/config/defaults.rb +163 -0
  18. data/lib/builderator/config/file.rb +336 -0
  19. data/lib/builderator/config/rash.rb +80 -0
  20. data/lib/builderator/control/cleaner.rb +138 -0
  21. data/lib/builderator/control/cookbook.rb +16 -0
  22. data/lib/builderator/control/data.rb +16 -0
  23. data/lib/builderator/control/data/image.rb +98 -0
  24. data/lib/builderator/control/version.rb +128 -0
  25. data/lib/builderator/control/version/auto.rb +48 -0
  26. data/lib/builderator/control/version/bump.rb +82 -0
  27. data/lib/builderator/control/version/comparable.rb +77 -0
  28. data/lib/builderator/control/version/git.rb +45 -0
  29. data/lib/builderator/control/version/scm.rb +92 -0
  30. data/lib/builderator/interface.rb +67 -0
  31. data/lib/builderator/interface/berkshelf.rb +38 -0
  32. data/lib/builderator/interface/packer.rb +75 -0
  33. data/lib/builderator/interface/vagrant.rb +31 -0
  34. data/lib/builderator/metadata.rb +5 -3
  35. data/lib/builderator/model/cleaner.rb +49 -0
  36. data/lib/builderator/model/cleaner/images.rb +93 -0
  37. data/lib/builderator/model/cleaner/instances.rb +58 -0
  38. data/lib/builderator/model/cleaner/launch_configs.rb +47 -0
  39. data/lib/builderator/model/cleaner/scaling_groups.rb +45 -0
  40. data/lib/builderator/model/cleaner/snapshots.rb +50 -0
  41. data/lib/builderator/model/cleaner/volumes.rb +48 -0
  42. data/lib/builderator/patch/berkshelf.rb +18 -0
  43. data/lib/builderator/patch/thor-actions.rb +47 -0
  44. data/lib/builderator/tasks.rb +127 -17
  45. data/lib/builderator/tasks/berkshelf.rb +63 -0
  46. data/lib/builderator/tasks/packer.rb +17 -56
  47. data/lib/builderator/tasks/vagrant.rb +111 -42
  48. data/lib/builderator/tasks/vendor.rb +94 -0
  49. data/lib/builderator/tasks/version.rb +58 -0
  50. data/lib/builderator/util.rb +37 -11
  51. data/lib/builderator/util/aws_exception.rb +1 -1
  52. data/lib/builderator/util/limit_exception.rb +12 -11
  53. data/lib/builderator/util/task_exception.rb +0 -2
  54. data/mkmf.log +4 -0
  55. data/spec/config_spec.rb +30 -0
  56. data/spec/data/Berksfile +6 -0
  57. data/spec/data/Buildfile +0 -0
  58. data/spec/data/Vagrantfile +0 -0
  59. data/spec/data/history.json +483 -0
  60. data/spec/data/packer.json +0 -0
  61. data/spec/interface_spec.rb +36 -0
  62. data/spec/resource/Buildfile +27 -0
  63. data/spec/spec_helper.rb +90 -0
  64. data/spec/version_spec.rb +282 -0
  65. data/template/Berksfile.erb +10 -0
  66. data/template/Buildfile.erb +28 -0
  67. data/template/Gemfile.erb +16 -0
  68. data/template/README.md.erb +61 -0
  69. data/template/Vagrantfile.erb +75 -0
  70. data/template/gitignore.erb +104 -0
  71. data/{.rubocop.yml → template/rubocop.erb} +0 -0
  72. metadata +203 -56
  73. data/.gitignore +0 -14
  74. data/lib/builderator/control/ami.rb +0 -65
  75. data/lib/builderator/control/clean.rb +0 -130
  76. data/lib/builderator/model.rb +0 -46
  77. data/lib/builderator/model/images.rb +0 -89
  78. data/lib/builderator/model/instances.rb +0 -55
  79. data/lib/builderator/model/launch_configs.rb +0 -46
  80. data/lib/builderator/model/scaling_groups.rb +0 -43
  81. data/lib/builderator/model/snapshots.rb +0 -49
  82. data/lib/builderator/model/volumes.rb +0 -48
  83. data/lib/builderator/tasks/ami.rb +0 -47
  84. data/lib/builderator/tasks/berks.rb +0 -68
  85. data/lib/builderator/tasks/clean.rb +0 -97
  86. data/lib/builderator/util/berkshim.rb +0 -34
  87. data/lib/builderator/util/cookbook.rb +0 -87
  88. data/lib/builderator/util/packer.rb +0 -39
  89. data/lib/builderator/util/shell.rb +0 -44
@@ -1,43 +0,0 @@
1
- require 'aws-sdk'
2
- require_relative '../util'
3
-
4
- module Builderator
5
- module Model
6
-
7
- def self.scaling_groups
8
- @scaling_groups ||= ScalingGroups.new
9
- end
10
-
11
- ##
12
- # AutoScaling Group Resoruces
13
- ##
14
- class ScalingGroups < Model::Base
15
- attr_reader :resources
16
- PROPERTIES = %w(auto_scaling_group_arn min_size max_size desired_capacity
17
- default_cooldown availability_zones load_balancer_names
18
- vpc_zone_identifier status termination_policies)
19
-
20
- def fetch
21
- @resources = {}.tap do |i|
22
- Util.asg.describe_auto_scaling_groups.each do |page|
23
- page.auto_scaling_groups.each do |a|
24
- properties = Util.from_tags(a.tags)
25
- properties['creation_date'] = a.created_time.to_datetime
26
- PROPERTIES.each { |pp| properties[pp] = a[pp.to_sym] }
27
-
28
- i[a.launch_configuration_name] = {
29
- :id => a.auto_scaling_group_name,
30
- :properties => properties,
31
- :config => a.launch_configuration_name
32
- }
33
- end
34
- end
35
- end
36
- end
37
-
38
- def launch_configs
39
- resources.values.map { |g| g[:config] }
40
- end
41
- end
42
- end
43
- end
@@ -1,49 +0,0 @@
1
- require 'aws-sdk'
2
- require_relative '../util'
3
-
4
- module Builderator
5
- module Model
6
-
7
- def self.snapshots
8
- @snapshots ||= Snapshots.new
9
- end
10
-
11
- ##
12
- # EC2 Snapshot Resources
13
- ##
14
- class Snapshots < Model::Base
15
- LIMIT = 24
16
- PROPERTIES = %w(state owner_id description volume_size)
17
-
18
- def fetch
19
- @resources = {}.tap do |s|
20
- Util.ec2.describe_snapshots(:filters => [
21
- {
22
- :name => 'status',
23
- :values => %w(completed)
24
- }
25
- ], :owner_ids => ['self']).each do |page|
26
- page.snapshots.each do |snap|
27
- properties = Util.from_tags(snap.tags)
28
- properties['creation_date'] = snap.start_time.to_datetime
29
- PROPERTIES.each { |pp| properties[pp] = snap[pp.to_sym] }
30
-
31
- s[snap.snapshot_id] = {
32
- :id => snap.snapshot_id,
33
- :properties => properties,
34
- :volume => snap.volume_id
35
- }
36
- end
37
- end
38
- end
39
- end
40
-
41
- def in_use(_)
42
- {}.tap do |used|
43
- used.merge!(select(Model.volumes.snapshots))
44
- used.merge!(select(Model.images.snapshots))
45
- end
46
- end
47
- end
48
- end
49
- end
@@ -1,48 +0,0 @@
1
- require 'aws-sdk'
2
- require_relative '../util'
3
-
4
- module Builderator
5
- module Model
6
-
7
- def self.volumes
8
- @volumes ||= Volumes.new
9
- end
10
-
11
- ##
12
- # EC2 Volume Resources
13
- ##
14
- class Volumes < Model::Base
15
- LIMIT = 8
16
- PROPERTIES = %w(size availability_zone state volume_type iops)
17
-
18
- def fetch
19
- @resources = {}.tap do |v|
20
- Util.ec2.describe_volumes.each do |page|
21
- page.volumes.each do |vol|
22
- properties = Util.from_tags(vol.tags)
23
- properties['creation_date'] = vol.create_time.to_datetime
24
- PROPERTIES.each { |pp| properties[pp] = vol[pp.to_sym] }
25
-
26
-
27
- v[vol.volume_id] = {
28
- :id => vol.volume_id,
29
- :properties => properties,
30
- :snapshot => vol.snapshot_id
31
- }
32
- end
33
- end
34
- end
35
- end
36
-
37
- def snapshots
38
- resources.values.map { |v| v[:snapshot] }
39
- end
40
-
41
- def in_use(_)
42
- {}.tap do |used|
43
- used.merge!(select(Model.instances.volumes))
44
- end
45
- end
46
- end
47
- end
48
- end
@@ -1,47 +0,0 @@
1
- require 'thor'
2
- require_relative '../control/ami'
3
-
4
- module Builderator
5
- module Tasks
6
- class AMI < Thor
7
- namespace :ami
8
-
9
- class_option :root_device_type,
10
- :default => 'ebs',
11
- :desc => 'The type of the root device volume (ebs | instance-store)'
12
- class_option :virtualization_type,
13
- :default => 'hvm',
14
- :desc => 'The virtualization type (paravirtual | hvm)'
15
- class_option :architecture,
16
- :default => 'x86_64',
17
- :desc => 'The image architecture (i386 | x86_64)'
18
-
19
- desc 'ubuntu SEARCH', 'Print the latest AMI ID for an Ubuntu image matching the SEARCH string'
20
- def ubuntu(search = '*/hvm-ssd/ubuntu-trusty-daily-amd64-server-20*')
21
- puts Control::AMI.latest(:owner => Builderator::Control::AMI::Owners::UBUNTU,
22
- 'root-device-type' => options['root_device_type'],
23
- 'virtualization-type' => options['virtualization_type'],
24
- 'architecture' => options['architecture'],
25
- 'name' => search).image_id
26
- end
27
-
28
- desc 'private [KEY VALUE ...]', 'Find the latest AMI ID with tags KEY=VALUE'
29
- def private(*args)
30
- puts Control::AMI.latest({ :owner => Builderator::Control::AMI::Owners::SELF,
31
- 'root-device-type' => options['root_device_type'],
32
- 'virtualization-type' => options['virtualization_type'],
33
- 'architecture' => options['architecture'] }.merge(Hash[*args])).image_id
34
- end
35
-
36
- desc 'windows SEARCH', 'Print the latest AMI ID for a Windows image matching the SEARCH string'
37
- def windows(search = 'Windows_Server-2012-R2_RTM-English-64Bit-Base*')
38
- puts Control::AMI.latest(:owner => Builderator::Control::AMI::Owners::AMAZON,
39
- 'root-device-type' => options['root_device_type'],
40
- 'virtualization-type' => options['virtualization_type'],
41
- 'architecture' => options['architecture'],
42
- 'name' => search).image_id
43
- end
44
-
45
- end
46
- end
47
- end
@@ -1,68 +0,0 @@
1
- require 'thor'
2
- require 'thor/actions'
3
- require_relative './cookbook'
4
- require_relative '../util/cookbook'
5
-
6
- module Builderator
7
- module Tasks
8
- class Berks < Thor
9
- include Thor::Actions
10
- class_option :config, :aliases => :c, :desc => "Path to Berkshelf's config.json"
11
- class_option :cookbook, :aliases => :b, :desc => 'Path to the cookbook to use'
12
- class_option :version, :type => :boolean,
13
- :default => true,
14
- :desc => 'Write current verison to file'
15
-
16
- desc "local [PATH = #{ Util::Cookbook::DEFAULT_VENDOR }]", 'Vendor the local cookbook source and its dependencies'
17
- def local(path = Util::Cookbook::DEFAULT_VENDOR.to_s)
18
- Util::Cookbook.path(options['cookbook'])
19
-
20
- command = 'BERKS_INSTALL_FROM=source'
21
- command << " berks vendor #{ path }"
22
- command << " -c #{ options['config'] }" if options.include?('config')
23
- command << " -b #{ Util::Cookbook.berksfile }"
24
-
25
- remove_file File.expand_path('Berksfile.lock', Util::Cookbook.path)
26
- invoke Tasks::Cookbook, 'metadata', [], options
27
- run command
28
- end
29
-
30
- desc "vendor [PATH = #{ Util::Cookbook::DEFAULT_VENDOR }]", 'Vendor a cookbook release and its dependencies'
31
- def vendor(path = Util::Cookbook::DEFAULT_VENDOR.to_s)
32
- Util::Cookbook.path(options['cookbook'])
33
-
34
- command = 'BERKS_INSTALL_FROM=release'
35
- command << " berks vendor #{ path }"
36
- command << " -c #{ options['config'] }" if options.include?('config')
37
- command << " -b #{ Util::Cookbook.berksfile }"
38
-
39
- remove_file File.expand_path('Berksfile.lock', Util::Cookbook.path)
40
- run command
41
- end
42
-
43
- desc 'upload', 'Upload the local cookbook source and its dependencies to the Chef server'
44
- option 'dry-run', :type => :boolean, :default => false
45
- def upload(path = Util::Cookbook::DEFAULT_VENDOR.to_s)
46
- command = 'BERKS_INSTALL_FROM=source'
47
- command << " berks upload"
48
- command << " -c #{ options['config'] }" if options.include?('config')
49
- command << " -b #{ Util::Cookbook.berksfile }"
50
-
51
- invoke Tasks::Berks, :local, [path], options
52
-
53
- return say_status :dryrun, command if options['dry-run']
54
- run command
55
- end
56
-
57
- desc 'uncache', 'Delete the Berkshelf cache'
58
- def uncache
59
- remove_dir File.join(ENV['HOME'], '.berkshelf/cookbooks')
60
- end
61
-
62
- desc "clean [PATH = #{ Util::Cookbook::DEFAULT_VENDOR }]", 'Remove a local vendor directory'
63
- def clean(path = Util::Cookbook::DEFAULT_VENDOR.to_s)
64
- remove_dir path
65
- end
66
- end
67
- end
68
- end
@@ -1,97 +0,0 @@
1
- require 'thor'
2
- require_relative '../control/clean'
3
-
4
- module Builderator
5
- module Tasks
6
- class Clean < Thor
7
- class_option :region,
8
- :type => :string,
9
- :default => 'us-east-1',
10
- :aliases => :r,
11
- :desc => 'AWS Region in which to perform tasks'
12
- class_option :commit,
13
- :type => :boolean,
14
- :default => false,
15
- :desc => 'Perform mutating API calls to cleanup resources'
16
- class_option :filter,
17
- :type => :array,
18
- :aliases => :f,
19
- :desc => 'Key/value pairs to filter resources (--filter name foo owner_id 123456789)'
20
- class_option :limit,
21
- :type => :boolean,
22
- :default => true,
23
- :desc => 'By default, limit the number of resources to remove'
24
-
25
- def initialize(*_)
26
- super
27
-
28
- ## Convert array of filter key-values to a hash
29
- options['filters'] = Hash[*options['filter']] if options['filter'].is_a?(Array)
30
-
31
- Control::Clean.options(options)
32
- end
33
-
34
- desc 'configs', 'Delete unused launch configurations'
35
- def configs
36
- Control::Clean.configs!(&method(:say_status))
37
- end
38
-
39
- desc 'images', 'Deregister unused images'
40
- option 'group-by',
41
- :type => :array,
42
- :desc => 'Tags/properties to group images by for pruning'
43
- option 'sort-by',
44
- :type => :string,
45
- :default => 'creation_date',
46
- :desc => 'Tag/property to sort grouped images on'
47
- option :keep,
48
- :type => :numeric,
49
- :default => 5,
50
- :desc => 'Number of images in each group to keep'
51
- def images
52
- Control::Clean.images!(&method(:say_status))
53
- end
54
-
55
- desc 'snapshots', 'Delete unused snapshots'
56
- def snapshots
57
- Control::Clean.snapshots!(&method(:say_status))
58
- end
59
-
60
- desc 'volumes', 'Delete unused volumes'
61
- def volumes
62
- Control::Clean.volumes!(&method(:say_status))
63
- end
64
-
65
- desc 'all', 'Clean volumes, launch configs, images, and snapshots in order'
66
- option 'group-by',
67
- :type => :array,
68
- :desc => 'Tags/properties to group images by for pruning'
69
- option 'sort-by',
70
- :type => :string,
71
- :default => 'creation_date',
72
- :desc => 'Tag/property to sort grouped images on'
73
- option :keep,
74
- :type => :numeric,
75
- :default => 5,
76
- :desc => 'Number of images in each group to keep'
77
- def all
78
- invoke :volumes, [], options
79
- invoke :configs, [], options
80
- invoke :images, [], options
81
- invoke :snapshots, [], options
82
-
83
- ## TODO Print resource counts here.
84
- return if Control::Clean.exceptions.empty?
85
-
86
- say_status :fail, 'Not all tasks completed successfully. The following '\
87
- 'exceptions occured:', :red
88
- Control::Clean.exceptions.each do |e|
89
- say_status(*e.status)
90
- end
91
-
92
- ## Mark the Jenkins job as fail if there were errors.
93
- exit(1)
94
- end
95
- end
96
- end
97
- end
@@ -1,34 +0,0 @@
1
- ##
2
- # Roll up some shared logic for Berksfiles
3
- ##
4
- require_relative './cookbook'
5
-
6
- module Builderator
7
- module Util
8
- module Berkshim
9
- def shims
10
-
11
- ## Root cookbook sources
12
- metadata if ENV['BERKS_INSTALL_FROM'] == 'source'
13
-
14
- if ENV['BERKS_INSTALL_FROM'] == 'release'
15
- cookbook_spec = [].tap do |arguments|
16
- arguments << Util::Cookbook.metadata.name
17
- arguments << ENV['VERSION'] if ENV.include?('VERSION')
18
-
19
- arguments << {}.tap do |options|
20
- options[:path] = ENV['COOKBOOK_PATH'] if ENV.include?('COOKBOOK_PATH')
21
- options[:git] = ENV['COOKBOOK_REPO'] if ENV.include?('COOKBOOK_REPO')
22
- options[:github] = ENV['COOKBOOK_GITHUB'] if ENV.include?('COOKBOOK_GITHUB')
23
- options[:branch] = ENV['COOKBOOK_BRANCH'] if ENV.include?('COOKBOOK_BRANCH')
24
- options[:ref] = ENV['COOKBOOK_REF'] if ENV.include?('COOKBOOK_REF')
25
- options[:tag] = ENV['COOKBOOK_TAG'] if ENV.include?('COOKBOOK_TAG')
26
- end
27
- end
28
-
29
- cookbook(*cookbook_spec)
30
- end
31
- end
32
- end
33
- end
34
- end
@@ -1,87 +0,0 @@
1
- require 'chef/cookbook/metadata'
2
- require 'ignorefile'
3
-
4
- require_relative '../util'
5
-
6
- module Builderator
7
- module Util
8
- module Cookbook
9
- DEFAULT_VENDOR = Util.working_dir('vendor/chef/cookbooks')
10
-
11
- ## Don't vendor VCS files.
12
- ## Reference GNU tar --exclude-vcs: https://www.gnu.org/software/tar/manual/html_section/tar_49.html
13
- ## Boosted from https://github.com/berkshelf/berkshelf/blob/master/lib/berkshelf/berksfile.rb
14
- EXCLUDED_VCS_FILES = [
15
- '.arch-ids', '{arch}', '.bzr', '.bzrignore', '.bzrtags',
16
- 'CVS', '.cvsignore', '_darcs', '.git', '.hg', '.hgignore',
17
- '.hgrags', 'RCS', 'SCCS', '.svn', '**/.git', '.temp'].freeze
18
-
19
- class Metadata < Chef::Cookbook::Metadata
20
- def files
21
- return @files unless @files.nil?
22
-
23
- @files ||= Pathname.glob(Util::Cookbook.path.join('**/{*,.*}'))
24
- ignorefile.apply!(@files)
25
-
26
- @files
27
- end
28
-
29
- def archive
30
- Util.working_dir("#{ name }-#{ version }.tgz")
31
- end
32
-
33
- def chefignore
34
- Util::Cookbook.path.join('chefignore')
35
- end
36
-
37
- def gitignore
38
- Util.working_dir('.gitignore')
39
- end
40
-
41
- def ignorefile
42
- return @ignorefile unless @ignorefile.nil?
43
-
44
- ## Construct an ignorefile
45
- @ignorefile = IgnoreFile.new(Util::Cookbook::EXCLUDED_VCS_FILES)
46
- @ignorefile.load_file(chefignore)
47
- @ignorefile.load_file(gitignore)
48
- end
49
- end
50
-
51
- class << self
52
- def path(arg = nil)
53
- ## Set an explicit path to a cookbook
54
- return @path = Pathname.new(arg) unless arg.nil?
55
- return @path unless @path.nil?
56
-
57
- ## Check for an embedded cookbook? ('./cookbook')
58
- return @path = Util.working_dir('cookbook') if Util.working_dir('cookbook').exist?
59
-
60
- @path = Util.working_dir
61
- end
62
-
63
- def archive_path(metadata, file)
64
- Pathname.new(metadata.name).join(Pathname.new(file).relative_path_from(path))
65
- end
66
-
67
- def berksfile
68
- path.join('Berksfile')
69
- end
70
-
71
- def metadata
72
- Metadata.new.tap do |c|
73
- if path.join('metadata.rb').exist?
74
- c.from_file(path.join('metadata.rb').to_s)
75
-
76
- elsif path.join('metadata.json').exist?
77
- c.from_json(path.join('metadata.json').read)
78
-
79
- else
80
- fail IOError, 'Unable to read metadata.rb or metadata.json!'
81
- end
82
- end
83
- end
84
- end
85
- end
86
- end
87
- end