vanagon 0.8.2 → 0.9.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +1 -0
  3. data/bin/build +1 -1
  4. data/lib/vanagon/component/dsl.rb +14 -24
  5. data/lib/vanagon/component/source/git.rb +1 -6
  6. data/lib/vanagon/component/source/http.rb +7 -0
  7. data/lib/vanagon/component/source/local.rb +8 -2
  8. data/lib/vanagon/driver.rb +31 -11
  9. data/lib/vanagon/engine/always_be_scheduling.rb +92 -0
  10. data/lib/vanagon/engine/base.rb +1 -2
  11. data/lib/vanagon/engine/hardware.rb +1 -1
  12. data/lib/vanagon/engine/pooler.rb +42 -10
  13. data/lib/vanagon/optparse.rb +1 -0
  14. data/lib/vanagon/platform.rb +10 -7
  15. data/lib/vanagon/platform/dsl.rb +13 -2
  16. data/lib/vanagon/project.rb +41 -3
  17. data/lib/vanagon/project/dsl.rb +9 -4
  18. data/lib/vanagon/utilities.rb +10 -13
  19. data/resources/Makefile.erb +1 -1
  20. data/resources/rpm/project.spec.erb +9 -0
  21. data/spec/fixtures/files/fake_dir/fake_file.txt +0 -0
  22. data/spec/fixtures/files/fake_nested_dir/fake_dir/fake_file.txt +0 -0
  23. data/spec/lib/vanagon/component/dsl_spec.rb +51 -2
  24. data/spec/lib/vanagon/component/source/git_spec.rb +8 -0
  25. data/spec/lib/vanagon/component/source/http_spec.rb +15 -15
  26. data/spec/lib/vanagon/component/source/local_spec.rb +17 -1
  27. data/spec/lib/vanagon/component/source_spec.rb +1 -1
  28. data/spec/lib/vanagon/driver_spec.rb +33 -1
  29. data/spec/lib/vanagon/engine/always_be_scheduling_spec.rb +84 -0
  30. data/spec/lib/vanagon/engine/pooler_spec.rb +80 -16
  31. data/spec/lib/vanagon/platform/dsl_spec.rb +14 -0
  32. data/spec/lib/vanagon/platform/windows_spec.rb +2 -2
  33. data/spec/lib/vanagon/project_spec.rb +79 -5
  34. data/spec/lib/vanagon/utilities_spec.rb +5 -0
  35. data/spec/spec_helper.rb +22 -2
  36. metadata +29 -24
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 94843aa8a37502b7e0548dc934267087f1ea5ed6
4
- data.tar.gz: f8daf4b6a0e88855775c58f2a73fa8cdb354e322
3
+ metadata.gz: 66a9286e3d7f89a5d145e4a5745cbb4d08ac9276
4
+ data.tar.gz: 6e1cf377bd79799c726e583bd8401d81fc8e1475
5
5
  SHA512:
6
- metadata.gz: 6212f217ffe247fecead7f8a105a22470e523d16dc87acc07e6df7ec013a806e9fad76f63eac876ce8b1b668a88c8ef3b3e7fef39a5cce7bb0d7fd21c2d0c9be
7
- data.tar.gz: fea048b71df645483b7a9c988fc74784eb06897a520044d0e9b5c5e5ac3643c56ca94f968a4f489b5078b27aaf280afd485cc28011306a34a854f276484e4fdd
6
+ metadata.gz: 594d72e7506ae70075e34a690981f144b4d332204ceeee433462aaa82ec9611baf12f3e516d9283598532d4bc150b097b1d2fd2491f3d566feddb024cfb1e0bc
7
+ data.tar.gz: f2387a452a90af391f7cff1a5e498bb2322cc4ce8729b6fa26c6c43166173a0dd44285814bb1a2ab0009ec8d5b3b720e1b7dc7167a9f8707c2725689d3669fc1
data/README.md CHANGED
@@ -68,6 +68,7 @@ the comma). If less targets are specified than platforms, the default engine
68
68
  (`pooler`) will be used for platforms without a target. If more targets are specified
69
69
  than platforms, the extra platforms will be ignored.
70
70
 
71
+ Build machines should be cleaned between builds.
71
72
 
72
73
  #### Flagged arguments (can be anywhere in the command)
73
74
 
data/bin/build CHANGED
@@ -2,7 +2,7 @@
2
2
  load File.expand_path(File.join(File.dirname(__FILE__), "..", "lib", "vanagon.rb"))
3
3
 
4
4
  optparse = Vanagon::OptParse.new("#{File.basename(__FILE__)} <project-name> <platform-name> [<target>] [options]",
5
- [:workdir, :configdir, :engine, :preserve, :verbose, :skipcheck])
5
+ [:workdir, :configdir, :engine, :preserve, :verbose, :skipcheck, :only_build])
6
6
  options = optparse.parse! ARGV
7
7
 
8
8
  project = ARGV[0]
@@ -39,8 +39,8 @@ class Vanagon
39
39
  # We only magically handle get_ methods, any other methods just get the
40
40
  # standard method_missing treatment.
41
41
  #
42
- def method_missing(method, *args)
43
- attribute_match = method.to_s.match(/get_(.*)/)
42
+ def method_missing(method_name, *args)
43
+ attribute_match = method_name.to_s.match(/get_(.*)/)
44
44
  if attribute_match
45
45
  attribute = attribute_match.captures.first
46
46
  else
@@ -50,6 +50,10 @@ class Vanagon
50
50
  @component.send(attribute)
51
51
  end
52
52
 
53
+ def respond_to_missing?(method_name, include_private = false)
54
+ method_name.to_s.start_with?('get_') || super
55
+ end
56
+
53
57
  # Set or add to the configure call for the component. The commands required to configure the component before building it.
54
58
  #
55
59
  # @param block [Proc] the command(s) required to configure the component
@@ -273,29 +277,15 @@ class Vanagon
273
277
  @component.url = the_url
274
278
  end
275
279
 
276
- # Sets the md5 sum to verify the sum of the source
277
- #
278
- # @param md5 [String] md5 sum of the source for verification
279
- def md5sum(md5)
280
- @component.options[:sum] = md5
281
- @component.options[:sum_type] = "md5"
282
- end
283
-
284
- # Sets the sha256 sum to verify the sum of the source
285
- #
286
- # @param sha256 [String] sha256 sum of the source for verification
287
- def sha256sum(sha256)
288
- @component.options[:sum] = sha256
289
- @component.options[:sum_type] = "sha256"
290
- end
291
-
292
- # Sets the sha512 sum to verify the sum of the source
293
- #
294
- # @param sha512 [String] sha512 sum of the source for verification
295
- def sha512sum(sha512)
296
- @component.options[:sum] = sha512
297
- @component.options[:sum_type] = "sha512"
280
+ def sum(value)
281
+ type = __callee__.to_s.gsub(/sum$/, '')
282
+ @component.options[:sum] = value
283
+ @component.options[:sum_type] = type
298
284
  end
285
+ alias_method :md5sum, :sum
286
+ alias_method :sha1sum, :sum
287
+ alias_method :sha256sum, :sum
288
+ alias_method :sha512sum, :sum
299
289
 
300
290
  # Sets the ref of the source for use in a git source
301
291
  #
@@ -30,7 +30,7 @@ class Vanagon
30
30
  # Default options used when cloning; this may expand
31
31
  # or change over time.
32
32
  def default_options
33
- @default_options ||= { ref: "refs/heads/master" }
33
+ @default_options ||= { ref: "HEAD" }
34
34
  end
35
35
  private :default_options
36
36
 
@@ -46,7 +46,6 @@ class Vanagon
46
46
  @url = URI.parse(url.to_s)
47
47
  @ref = opts[:ref]
48
48
  @workdir = workdir
49
- @ref_name, @ref_type, = @ref.split('/', 3).reverse
50
49
 
51
50
  # We can test for Repo existence without cloning
52
51
  raise Vanagon::InvalidRepo, "#{url} not a valid Git repo" unless valid_remote?
@@ -62,10 +61,6 @@ class Vanagon
62
61
  update_submodules
63
62
  end
64
63
 
65
- def ref
66
- @ref_name || @ref
67
- end
68
-
69
64
  # Return the correct incantation to cleanup the source directory for a given source
70
65
  #
71
66
  # @return [String] command to cleanup the source
@@ -12,6 +12,9 @@ class Vanagon
12
12
  # Accessors :url, :file, :extension, :workdir, :cleanup are inherited from Local
13
13
  attr_accessor :sum, :sum_type
14
14
 
15
+ # Allowed checksum algorithms to use when validating files
16
+ CHECKSUM_TYPES = %w(md5 sha1 sha256 sha512).freeze
17
+
15
18
  class << self
16
19
  def valid_url?(target_url) # rubocop:disable Metrics/AbcSize
17
20
  uri = URI.parse(target_url.to_s)
@@ -51,6 +54,10 @@ class Vanagon
51
54
  unless sum_type
52
55
  fail "sum_type is required to validate the http source"
53
56
  end
57
+ unless CHECKSUM_TYPES.include? sum_type
58
+ fail %(checksum type "#{sum_type}" is invalid; please use #{CHECKSUM_TYPES.join(', ')})
59
+ end
60
+
54
61
  @url = url
55
62
  @sum = sum
56
63
  @workdir = workdir
@@ -60,7 +60,7 @@ class Vanagon
60
60
  def copy
61
61
  puts "Copying file '#{url.basename}' to workdir"
62
62
 
63
- FileUtils.cp(url, file)
63
+ FileUtils.cp_r(url, file)
64
64
  end
65
65
  alias_method :fetch, :copy
66
66
 
@@ -148,7 +148,13 @@ class Vanagon
148
148
  # @return [String] the directory that should be traversed into to build this source
149
149
  # @raise [RuntimeError] if the @extension for the @file isn't currently handled by the method
150
150
  def dirname
151
- archive? ? File.basename(file, extension) : './'
151
+ # We are not treating file as a Pathname since other sources can inherit from this class
152
+ # which could cause file to be a URI instead of a string.
153
+ if archive? || File.directory?(file)
154
+ File.basename(file, extension)
155
+ else
156
+ './'
157
+ end
152
158
  end
153
159
 
154
160
  # Wrapper around the class method '.mangle'
@@ -13,12 +13,13 @@ class Vanagon
13
13
  attr_accessor :platform, :project, :target, :workdir, :verbose, :preserve
14
14
  attr_accessor :timeout, :retry_count
15
15
 
16
- def initialize(platform, project, options = { :configdir => nil, :target => nil, :engine => nil, :components => nil, :skipcheck => false, :verbose => false, :preserve => false }) # rubocop:disable Metrics/AbcSize
16
+ def initialize(platform, project, options = { :configdir => nil, :target => nil, :engine => nil, :components => nil, :skipcheck => false, :verbose => false, :preserve => false, :only_build => nil }) # rubocop:disable Metrics/AbcSize
17
17
  @verbose = options[:verbose]
18
18
  @preserve = options[:preserve]
19
19
 
20
20
  @@configdir = options[:configdir] || File.join(Dir.pwd, "configs")
21
21
  components = options[:components] || []
22
+ only_build = options[:only_build]
22
23
  target = options[:target]
23
24
  engine = options[:engine] || 'pooler'
24
25
 
@@ -26,6 +27,7 @@ class Vanagon
26
27
  @project = Vanagon::Project.load_project(project, File.join(@@configdir, "projects"), @platform, components)
27
28
  @project.settings[:verbose] = options[:verbose]
28
29
  @project.settings[:skipcheck] = options[:skipcheck]
30
+ filter_out_components(only_build) if only_build
29
31
  loginit('vanagon_hosts.log')
30
32
 
31
33
  load_engine(engine, @platform, target)
@@ -33,24 +35,42 @@ class Vanagon
33
35
  raise Vanagon::Error.wrap(e, "Could not load the desired engine '#{engine}'")
34
36
  end
35
37
 
38
+ def filter_out_components(only_build)
39
+ # map each element in the only_build array to it's set of filtered components, then
40
+ # flatten all the results in to one array and set project.components to that.
41
+ @project.components = only_build.flat_map { |comp| @project.filter_component(comp) }.uniq
42
+ if @verbose
43
+ puts "Only building:"
44
+ @project.components.each { |comp| puts comp.name }
45
+ end
46
+ end
47
+
36
48
  def load_engine(engine_type, platform, target)
37
- if platform.build_hosts
38
- engine_type = 'hardware'
39
- elsif platform.aws_ami
40
- engine_type = 'ec2'
41
- elsif platform.docker_image
42
- engine_type = 'docker'
43
- elsif target
44
- engine_type = 'base'
49
+ if engine_type != 'always_be_scheduling'
50
+ if platform.build_hosts
51
+ engine_type = 'hardware'
52
+ elsif platform.aws_ami
53
+ engine_type = 'ec2'
54
+ elsif platform.docker_image
55
+ engine_type = 'docker'
56
+ elsif target
57
+ engine_type = 'base'
58
+ end
45
59
  end
46
60
  load_engine_object(engine_type, platform, target)
47
61
  end
48
62
 
49
63
  def load_engine_object(engine_type, platform, target)
50
64
  require "vanagon/engine/#{engine_type}"
51
- @engine = Object::const_get("Vanagon::Engine::#{engine_type.capitalize}").new(platform, target)
65
+ @engine = Object::const_get("Vanagon::Engine::#{camelize(engine_type)}").new(platform, target)
52
66
  rescue
53
- fail "No such engine '#{engine_type.capitalize}'"
67
+ fail "No such engine '#{camelize(engine_type)}'"
68
+ end
69
+
70
+ def camelize(string)
71
+ string.gsub(/(?:^|_)([a-z])?/) do |match|
72
+ (Regexp.last_match[1] || '').capitalize
73
+ end
54
74
  end
55
75
 
56
76
  def cleanup_workdir
@@ -0,0 +1,92 @@
1
+ require 'vanagon/engine/base'
2
+ require 'json'
3
+
4
+ class Vanagon
5
+ class Engine
6
+ # This engine allows build resources to be managed by the "Always be
7
+ # Scheduling" (ABS) scheduler (https://github.com/puppetlabs/always-be-scheduling)
8
+ #
9
+ # ABS expects to ask `build_host_info` for the needed resources for a build,
10
+ # and to have that return a platform name. ABS will then acquire the
11
+ # desired build host resources and will later run a vanagon build, passing
12
+ # those resource hostnames in specifically.
13
+ #
14
+ # `build_host_info` will normally use the `hardware` engine when a hardware
15
+ # platform is queried. The `always_be_scheduling` engine's behavior will
16
+ # be invoked instead when:
17
+ #
18
+ # `build_host_info ... --engine always_be_scheduling` is specified on the
19
+ # command-line.
20
+ #
21
+ # Configuration:
22
+ #
23
+ # Project platform configurations can specify the platform name to be returned
24
+ # via the `abs_resource_name` attribute. If this is not set but `vmpooler_template`
25
+ # is set, then the `vmpooler_template` value will be used. Otherwise, the
26
+ # platform name will be returned unchanged.
27
+ #
28
+ # Example 1:
29
+ #
30
+ # platform 'ubuntu-10.04-amd64' do |plat|
31
+ # plat.vmpooler_template 'ubuntu-1004-amd64'
32
+ # end
33
+ #
34
+ # $ build_host_info puppet-agent ubuntu-10.04-amd64
35
+ # {"name":"ubuntu-10.04-amd64","engine":"pooler"}
36
+ #
37
+ # $ build_host_info puppet-agent ubuntu-10.04-amd64 --engine always_be_scheduling
38
+ # {"name":"ubuntu-10.04-amd64","engine":"always_be_scheduling"}
39
+ #
40
+ #
41
+ # Example 2:
42
+ #
43
+ # platform 'aix-5.3-ppc' do |plat|
44
+ # plat.build_host ['aix53-builder-1.example.com']
45
+ # plat.abs_resource_name 'aix-53-ppc'
46
+ # end
47
+ #
48
+ # $ build_host_info puppet-agent aix-5.3-ppc
49
+ # {"name":"aix53-builder-1.example.com","engine":"hardware"}
50
+ #
51
+ # $ build_host_info puppet-agent aix-5.3-ppc --engine always_be_scheduling
52
+ # {"name":"aix-53-ppc","engine":"always_be_scheduling"}
53
+ #
54
+ #
55
+ # Example 3:
56
+ #
57
+ # platform 'aix-5.3-ppc' do |plat|
58
+ # plat.build_host ['aix53-builder-1.example.com']
59
+ # plat.vmpooler_template
60
+ # plat.abs_resource_name 'aix-53-ppc'
61
+ # end
62
+ #
63
+ # $ build_host_info puppet-agent aix-5.3-ppc
64
+ # {"name":"aix53-builder-1.example.com","engine":"hardware"}
65
+ #
66
+ # $ build_host_info puppet-agent aix-5.3-ppc --engine always_be_scheduling
67
+ # {"name":"aix-53-ppc","engine":"always_be_scheduling"}
68
+ class AlwaysBeScheduling < Base
69
+ def initialize(platform, target)
70
+ super
71
+
72
+ Vanagon::Driver.logger.debug "AlwaysBeScheduling engine invoked."
73
+ end
74
+
75
+ # Get the engine name
76
+ def name
77
+ 'always_be_scheduling'
78
+ end
79
+
80
+ # return the platform name as the "host" name
81
+ def build_host_name
82
+ if @platform.abs_resource_name
83
+ @platform.abs_resource_name
84
+ elsif @platform.vmpooler_template
85
+ @platform.vmpooler_template
86
+ else
87
+ @platform.name
88
+ end
89
+ end
90
+ end
91
+ end
92
+ end
@@ -36,8 +36,7 @@ class Vanagon
36
36
 
37
37
  # Steps needed to tear down or clean up the system after the build is
38
38
  # complete
39
- def teardown
40
- end
39
+ def teardown; end
41
40
 
42
41
  # Applies the steps needed to extend the system to build packages against
43
42
  # the target system
@@ -27,7 +27,7 @@ class Vanagon
27
27
  host
28
28
  end
29
29
 
30
- # Iterarte over the options and find a node open to lock.
30
+ # Iterate over the options and find a node open to lock.
31
31
  def node_lock(hosts)
32
32
  hosts.each do |h|
33
33
  Vanagon::Driver.logger.info "Attempting to lock #{h}."
@@ -1,4 +1,5 @@
1
1
  require 'vanagon/engine/base'
2
+ require 'yaml'
2
3
 
3
4
  class Vanagon
4
5
  class Engine
@@ -29,19 +30,50 @@ class Vanagon
29
30
  @build_host_template_name
30
31
  end
31
32
 
32
- # This method loads the pooler token from one of two locations
33
+ # Retrieve the pooler token from an environment variable
34
+ # ("VMPOOLER_TOKEN") or from a number of potential configuration
35
+ # files (~/.vanagon-token or ~/.vmfloaty.yml).
33
36
  # @return [String, nil] token for use with the vmpooler
34
37
  def load_token
35
- if ENV['VMPOOLER_TOKEN']
36
- token = ENV['VMPOOLER_TOKEN']
37
- else
38
- token_file = File.expand_path("~/.vanagon-token")
39
- if File.exist?(token_file)
40
- token = File.open(token_file).read.chomp
41
- end
42
- end
43
- token
38
+ ENV['VMPOOLER_TOKEN'] || token_from_file
39
+ end
40
+
41
+ # a wrapper method around retrieving a vmpooler token,
42
+ # with an explicitly ordered preference for a Vanagon-specific
43
+ # token file or a preexisting vmfoaty yaml file.
44
+ #
45
+ # @return [String, nil] token for use with the vmpooler
46
+ def token_from_file
47
+ read_vanagon_token || read_vmfloaty_token
48
+ end
49
+ private :token_from_file
50
+
51
+ # Read a vmpooler token from the plaintext vanagon-token file,
52
+ # as outlined in the project README.
53
+ #
54
+ # @return [String, nil] the vanagon vmpooler token value
55
+ def read_vanagon_token(path = "~/.vanagon-token")
56
+ absolute_path = File.expand_path(path)
57
+ return nil unless File.exist?(absolute_path)
58
+
59
+ puts "Reading vmpooler token from: #{path}"
60
+ File.read(absolute_path).chomp
61
+ end
62
+ private :read_vanagon_token
63
+
64
+ # Read a vmpooler token from the yaml formatted vmfloaty config,
65
+ # as outlined by the vmfloaty project:
66
+ # https://github.com/briancain/vmfloaty
67
+ #
68
+ # @return [String, nil] the vmfloaty vmpooler token value
69
+ def read_vmfloaty_token(path = "~/.vmfloaty.yml")
70
+ absolute_path = File.expand_path(path)
71
+ return nil unless File.exist?(absolute_path)
72
+
73
+ puts "Reading vmpooler token from: #{path}"
74
+ YAML.load_file(absolute_path)['token']
44
75
  end
76
+ private :read_vmfloaty_token
45
77
 
46
78
  # This method is used to obtain a vm to build upon using the Puppet Labs'
47
79
  # vmpooler (https://github.com/puppetlabs/vmpooler)
@@ -10,6 +10,7 @@ class Vanagon
10
10
  :skipcheck => ['--skipcheck', 'Skip the `check` stage when building components'],
11
11
  :preserve => ['-p', '--preserve', 'Whether to tear down the VM on success or not (defaults to false)'],
12
12
  :verbose => ['-v', '--verbose', 'Verbose output (does nothing)'],
13
+ :only_build => ["--only-build COMPONENT,COMPONENT,...", Array, 'Only build this array of components']
13
14
  }.freeze
14
15
 
15
16
  def initialize(banner, options = [])
@@ -2,13 +2,16 @@ require 'vanagon/platform/dsl'
2
2
 
3
3
  class Vanagon
4
4
  class Platform
5
- attr_accessor :make, :servicedir, :defaultdir, :provisioning, :num_cores, :tar
6
- attr_accessor :build_dependencies, :name, :vmpooler_template, :cflags, :ldflags, :settings
7
- attr_accessor :servicetype, :patch, :architecture, :codename, :os_name, :os_version
8
- attr_accessor :docker_image, :ssh_port, :rpmbuild, :install, :platform_triple
9
- attr_accessor :target_user, :package_type, :find, :sort, :build_hosts, :copy, :cross_compiled
10
- attr_accessor :aws_ami, :aws_user_data, :aws_shutdown_behavior, :aws_key_name, :aws_region, :aws_key
11
- attr_accessor :aws_instance_type, :aws_vpc_id, :aws_subnet_id, :output_dir
5
+ attr_accessor :make, :servicedir, :defaultdir, :provisioning, :num_cores
6
+ attr_accessor :tar, :build_dependencies, :name, :vmpooler_template
7
+ attr_accessor :abs_resource_name, :cflags, :ldflags, :settings
8
+ attr_accessor :servicetype, :patch, :architecture, :codename, :os_name
9
+ attr_accessor :os_version, :docker_image, :ssh_port, :rpmbuild, :install
10
+ attr_accessor :platform_triple, :target_user, :package_type, :find, :sort
11
+ attr_accessor :build_hosts, :copy, :cross_compiled, :aws_ami
12
+ attr_accessor :aws_user_data, :aws_shutdown_behavior, :aws_key_name
13
+ attr_accessor :aws_region, :aws_key, :aws_instance_type, :aws_vpc_id
14
+ attr_accessor :aws_subnet_id, :output_dir
12
15
 
13
16
  # Platform names currently contain some information about the platform. Fields
14
17
  # within the name are delimited by the '-' character, and this regex can be used to