vanagon 0.4.1 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -10,10 +10,10 @@ class Vanagon
10
10
  attr_accessor :url, :sum, :file, :extension, :workdir, :cleanup
11
11
 
12
12
  # Extensions for files we intend to unpack during the build
13
- ARCHIVE_EXTENSIONS = '.tar.gz', '.tgz'
13
+ ARCHIVE_EXTENSIONS = '.tar.gz', '.tgz', '.zip'
14
14
 
15
15
  # Extensions for files we aren't going to unpack during the build
16
- NON_ARCHIVE_EXTENSIONS = '.gem', '.ru', '.txt', '.conf', '.ini', '.gpg', '.rb', '.sh', '.csh', '.xml'
16
+ NON_ARCHIVE_EXTENSIONS = '.gem', '.ru', '.txt', '.conf', '.ini', '.gpg', '.rb', '.sh', '.csh', '.xml', '.vim', '.json', '.service'
17
17
 
18
18
  # Constructor for the Http source type
19
19
  #
@@ -100,7 +100,12 @@ class Vanagon
100
100
  # @raise [RuntimeError] an exception is raised if there is no known extraction method for @extension
101
101
  def extract(tar)
102
102
  if ARCHIVE_EXTENSIONS.include?(@extension)
103
- return "gunzip -c '#{@file}' | '#{tar}' xf -"
103
+ case @extension
104
+ when ".tar.gz", ".tgz"
105
+ return "gunzip -c '#{@file}' | '#{tar}' xf -"
106
+ when ".zip"
107
+ return "unzip '#{@file}'"
108
+ end
104
109
  elsif NON_ARCHIVE_EXTENSIONS.include?(@extension)
105
110
  # Don't need to unpack gems, ru, txt, conf, ini, gpg
106
111
  return nil
@@ -12,7 +12,7 @@ class Vanagon
12
12
  include Vanagon::Utilities
13
13
  attr_accessor :platform, :project, :target, :workdir, :verbose, :preserve
14
14
 
15
- def initialize(platform, project, options = { :configdir => nil, :target => nil, :engine => nil, :components => nil })
15
+ def initialize(platform, project, options = { :configdir => nil, :target => nil, :engine => nil, :components => nil, :skipcheck => false })
16
16
  @verbose = false
17
17
  @preserve = false
18
18
 
@@ -23,16 +23,19 @@ class Vanagon
23
23
 
24
24
  @platform = Vanagon::Platform.load_platform(platform, File.join(@@configdir, "platforms"))
25
25
  @project = Vanagon::Project.load_project(project, File.join(@@configdir, "projects"), @platform, components)
26
+ @project.settings[:skipcheck] = options[:skipcheck]
27
+ @@logger = Logger.new('vanagon_hosts.log')
28
+ @@logger.progname = 'vanagon'
26
29
 
27
30
  # If a target has been given, we don't want to make any assumptions about how to tear it down.
28
31
  engine = 'base' if target
32
+ # Hardware has explicit teardown to unlock the node
33
+ engine = 'hardware' if @platform.build_hosts
29
34
  require "vanagon/engine/#{engine}"
30
35
  @engine = Object.const_get("Vanagon::Engine::#{engine.capitalize}").new(@platform, target)
31
36
 
32
- @@logger = Logger.new('vanagon_hosts.log')
33
- @@logger.progname = 'vanagon'
34
37
  rescue LoadError => e
35
- raise Vanagon::Error.wrap(e, "Could not load the desired engine '#{@engine_name}'.")
38
+ raise Vanagon::Error.wrap(e, "Could not load the desired engine '#{engine}'")
36
39
  end
37
40
 
38
41
  def cleanup_workdir
@@ -88,7 +91,11 @@ class Vanagon
88
91
  puts e
89
92
  puts e.backtrace.join("\n")
90
93
  raise e
91
- end
94
+ ensure
95
+ if @engine == "hardware"
96
+ @engine.teardown
97
+ end
98
+ end
92
99
 
93
100
  def prepare(workdir = nil)
94
101
  @workdir = workdir ? FileUtils.mkdir_p(workdir).first : Dir.mktmpdir
@@ -0,0 +1,64 @@
1
+ require 'vanagon/engine/base'
2
+ require 'json'
3
+ require 'lock_manager'
4
+
5
+ LOCK_MANAGER_HOST = ENV['LOCK_MANAGER_HOST'] || 'redis'
6
+ LOCK_MANAGER_PORT = ENV['LOCK_MANAGER_PORT'] || 6379
7
+ VANAGON_LOCK_USER = ENV['USER']
8
+
9
+
10
+ class Vanagon
11
+ class Engine
12
+ # Class to use when building on a hardware device (e.g. AIX, Switch, etc)
13
+ #
14
+ class Hardware < Base
15
+
16
+ # This method is used to obtain a vm to build upon
17
+ # For the base class we just return the target that was passed in
18
+ def select_target
19
+ @target = node_lock(@build_hosts)
20
+ end
21
+
22
+ # Poll for a lock
23
+ def polling_lock(host)
24
+ Vanagon::Driver.logger.info "Polling for a lock on #{host}."
25
+ @lockman.polling_lock(host, VANAGON_LOCK_USER, "Vanagon automated lock")
26
+ Vanagon::Driver.logger.info "Lock acquired on #{host}."
27
+ puts "Lock acquired on #{host} for #{VANAGON_LOCK_USER}."
28
+ host
29
+ end
30
+
31
+ # Iterarte over the options and find a node open to lock.
32
+ def node_lock(hosts)
33
+ hosts.each do |h|
34
+ Vanagon::Driver.logger.info "Attempting to lock #{h}."
35
+ if @lockman.lock(h, VANAGON_LOCK_USER, "Vanagon automated lock")
36
+ Vanagon::Driver.logger.info "Lock acquired on #{h}."
37
+ puts "Lock acquired on #{h} for #{VANAGON_LOCK_USER}."
38
+ return h
39
+ end
40
+ end
41
+ # If they are all locked, fall back to a polling lock on last item
42
+ polling_lock(hosts.pop)
43
+ end
44
+
45
+ # Steps needed to tear down or clean up the system after the build is
46
+ # complete. In this case, we'll attempt to unlock the hardware
47
+ def teardown
48
+ Vanagon::Driver.logger.info "Removing lock on #{@target}."
49
+ puts "Removing lock on #{@target}."
50
+ @lockman.unlock(@target, VANAGON_LOCK_USER)
51
+ end
52
+
53
+ def initialize(platform, target)
54
+ Vanagon::Driver.logger.debug "Hardware engine invoked."
55
+ @platform = platform
56
+ @build_hosts = platform.build_hosts
57
+ # Redis is the only backend supported in lock_manager currently
58
+ @lockman = LockManager.new(type: 'redis', server: LOCK_MANAGER_HOST)
59
+ super
60
+ @required_attributes << "build_hosts"
61
+ end
62
+ end
63
+ end
64
+ end
@@ -2,44 +2,27 @@ require 'optparse'
2
2
 
3
3
  class Vanagon
4
4
  class OptParse
5
+
6
+ FLAGS = {
7
+ :workdir => ['-w DIR', '--workdir DIR', "Working directory where build source should be put (defaults to a tmpdir)"],
8
+ :configdir => ['-c', '--configdir DIR', 'Configs dir (defaults to $pwd/configs)'],
9
+ :target => ['-t HOST', '--target HOST', 'Configure a target machine for build and packaging (defaults to grabbing one from the pooler)'],
10
+ :engine => ['-e ENGINE', '--engine ENGINE', "A custom engine to use (defaults to the pooler) [base, local, docker, pooler currently supported]"],
11
+ :skipcheck => ['--skipcheck', 'Ship the `check` stage when building components'],
12
+ :preserve => ['-p', '--preserve', 'Whether to tear down the VM on success or not (defaults to false)'],
13
+ :verbose => ['-v', '--verbose', 'Verbose output (does nothing)'],
14
+ }
15
+
5
16
  def initialize(banner, options = [])
6
17
  @options = Hash[options.map { |v| [v, nil] }]
7
18
  @optparse = OptionParser.new do |opts|
8
19
  opts.banner = banner
9
20
 
10
- if @options.include?(:workdir)
11
- opts.on('-w DIR', '--workdir DIR', "Working directory where build source should be put (defaults to a tmpdir)") do |dir|
12
- @options[:workdir] = dir
13
- end
14
- end
15
-
16
- if @options.include?(:configdir)
17
- opts.on('-c', '--configdir DIR', 'Configs dir (defaults to $pwd/configs)') do |dir|
18
- @options[:configdir] = dir
19
- end
20
- end
21
-
22
- if @options.include?(:target)
23
- opts.on('-t HOST', '--target HOST', 'Configure a target machine for build and packaging (defaults to grabbing one from the pooler)') do |name|
24
- @options[:target] = name
25
- end
26
- end
27
-
28
- if @options.include?(:engine)
29
- opts.on('-e ENGINE', '--engine ENGINE', "A custom engine to use (defaults to the pooler) [base, local, docker, pooler currently supported]") do |engine|
30
- @options[:engine] = engine
31
- end
32
- end
33
-
34
- if @options.include?(:preserve)
35
- opts.on('-p', '--preserve', 'Whether to tear down the VM on success or not (defaults to false)') do |flag|
36
- @options[:preserve] = flag
37
- end
38
- end
39
-
40
- if @options.include?(:verbose)
41
- opts.on('-v', '--verbose', 'Verbose output (does nothing)') do |flag|
42
- @options[:verbose] = flag
21
+ FLAGS.each_pair do |name, args|
22
+ if @options.include?(name)
23
+ opts.on(*args) do |value|
24
+ @options[name] = value
25
+ end
43
26
  end
44
27
  end
45
28
 
@@ -0,0 +1,39 @@
1
+ require 'vanagon/errors'
2
+
3
+ class Vanagon
4
+ class Patch
5
+
6
+ # @!attribute [r] path
7
+ # @return [String] The path to the patch
8
+ attr_reader :path
9
+
10
+ # @!attribute [r] strip
11
+ # @return [Integer] the number of path components to strip from the patch path
12
+ attr_reader :strip
13
+
14
+ # @!attribute [r] fuzz
15
+ # @return [Integer] The fuzz factor for applying the patch
16
+ attr_reader :fuzz
17
+
18
+ # @!attribute [r] after
19
+ # @return [String] What step should this patch be applied to, one of ["unpack", "install"]
20
+ attr_reader :after
21
+
22
+ # @!attribute [r] destination
23
+ # @return [String] The working directory where this patch will be applied. Only used for post-installation patches.
24
+ attr_reader :destination
25
+
26
+ def initialize(path, strip, fuzz, after, destination)
27
+ raise Vanagon::Error, "We can only apply patches after the source is unpacked or after installation" unless ['unpack', 'install'].include?(after)
28
+ @path = path
29
+ @strip = strip
30
+ @fuzz = fuzz
31
+ @after = after
32
+ @destination = destination
33
+ end
34
+
35
+ def cmd(platform)
36
+ "#{platform.patch} --strip=#{@strip} --fuzz=#{@fuzz} --ignore-whitespace < $(workdir)/patches/#{File.basename(@path)}"
37
+ end
38
+ end
39
+ end
@@ -6,6 +6,7 @@ class Vanagon
6
6
  attr_accessor :build_dependencies, :name, :vmpooler_template, :cflags, :ldflags, :settings
7
7
  attr_accessor :servicetype, :patch, :architecture, :codename, :os_name, :os_version
8
8
  attr_accessor :docker_image, :ssh_port, :rpmbuild, :install, :platform_triple
9
+ attr_accessor :package_type, :build_hosts
9
10
 
10
11
  # Platform names currently contain some information about the platform. Fields
11
12
  # within the name are delimited by the '-' character, and this regex can be used to
@@ -137,7 +138,7 @@ class Vanagon
137
138
  #
138
139
  # @return [true, false] true if it is a debian variety, false otherwise
139
140
  def is_deb?
140
- return !!@name.match(/^(debian|ubuntu|cumulus)-.*$/)
141
+ return !!@name.match(/^(debian|ubuntu|cumulus|huaweios)-.*$/)
141
142
  end
142
143
 
143
144
  # Utility matcher to determine is the platform is a redhat variety or
@@ -146,7 +147,7 @@ class Vanagon
146
147
  # @return [true, false] true if it is a redhat variety or uses rpm
147
148
  # under the hood, false otherwise
148
149
  def is_rpm?
149
- return !!@name.match(/^(aix|cisco-wrlinux|el|eos|fedora|huaweios|sles)-.*$/)
150
+ return !!@name.match(/^(aix|cisco-wrlinux|el|eos|fedora|sles)-.*$/)
150
151
  end
151
152
 
152
153
  # Utility matcher to determine is the platform is an enterprise linux variety
@@ -3,7 +3,7 @@ require 'vanagon/platform/rpm'
3
3
  require 'vanagon/platform/rpm/aix'
4
4
  require 'vanagon/platform/rpm/sles'
5
5
  require 'vanagon/platform/rpm/wrl'
6
- require 'vanagon/platform/swix'
6
+ require 'vanagon/platform/rpm/eos'
7
7
  require 'vanagon/platform/osx'
8
8
  require 'vanagon/platform/solaris_10'
9
9
  require 'vanagon/platform/solaris_11'
@@ -33,12 +33,10 @@ class Vanagon
33
33
  Vanagon::Platform::RPM.new(@name)
34
34
  when /^sles-/
35
35
  Vanagon::Platform::RPM::SLES.new(@name)
36
- when /^huaweios-/
37
- Vanagon::Platform::RPM::WRL.new(@name)
38
- when /^(cumulus|debian|ubuntu)-/
36
+ when /^(cumulus|debian|huaweios|ubuntu)-/
39
37
  Vanagon::Platform::DEB.new(@name)
40
38
  when /^eos-/
41
- Vanagon::Platform::RPM::Swix.new(@name)
39
+ Vanagon::Platform::RPM::EOS.new(@name)
42
40
  when /^osx-/
43
41
  Vanagon::Platform::OSX.new(@name)
44
42
  when /^solaris-10/
@@ -92,6 +90,13 @@ class Vanagon
92
90
  @platform.tar = tar_cmd
93
91
  end
94
92
 
93
+ # Set the type of package we are going to build for this platform
94
+ #
95
+ # @param pkg_type [String] The type of package we are going to build for this platform
96
+ def package_type(pkg_type)
97
+ @platform.package_type = pkg_type
98
+ end
99
+
95
100
  # Set the path to rpmbuild for the platform
96
101
  #
97
102
  # @param rpmbuild_cmd [String] Full path to rpmbuild with arguments to be used by default
@@ -156,6 +161,19 @@ class Vanagon
156
161
  @platform.servicetype = type
157
162
  end
158
163
 
164
+ # Set the list of possible host to perform a build on (when not using
165
+ # pooler or CLI flags)
166
+ #
167
+ # @param type [Array] the names of the hosts (must be resolvable)
168
+ # @rase ArgumentError if builds_hosts has no arguments
169
+ def build_hosts(*args)
170
+ raise ArgumentError, "build_hosts requires at least one host to be a build target." if args.empty?
171
+ @platform.build_hosts = Array(args).flatten
172
+ end
173
+
174
+ # Because single vs plural is annoying to remember
175
+ alias_method :build_host, :build_hosts
176
+
159
177
  # Set the name of this platform as the vm pooler expects it
160
178
  #
161
179
  # @param name [String] name of the target template to use from the vmpooler
@@ -0,0 +1,83 @@
1
+ class Vanagon
2
+ class Platform
3
+ class RPM
4
+ class EOS < Vanagon::Platform::RPM
5
+ # The specific bits used to generate an EOS package for a given project
6
+ #
7
+ # @param project [Vanagon::Project] project to build an EOS package of
8
+ # @return [Array] list of commands required to build the EOS package
9
+ # for the given project from an rpm or a swix
10
+ def generate_package(project)
11
+ # If nothing is passed in as platform type, default to building a swix package
12
+ if project.platform.package_type.nil? || project.platform.package_type.empty?
13
+ return generate_swix_package(project)
14
+ else
15
+ case project.platform.package_type
16
+ when "rpm"
17
+ return super(project)
18
+ when "swix"
19
+ return generate_swix_package(project)
20
+ else
21
+ fail "I don't know how to build package type '#{project.platform.package_type}' for EOS. Teach me?"
22
+ end
23
+ end
24
+ end
25
+
26
+ # Method to derive the package name for the project
27
+ #
28
+ # @param project [Vanagon::Project] project to name
29
+ # @return [String] name of the EOS package for this project
30
+ def package_name(project)
31
+ # If nothing is passed in as platform type, default to building a swix package
32
+ if project.platform.package_type.nil? || project.platform.package_type.empty?
33
+ return swix_package_name(project)
34
+ else
35
+ case project.platform.package_type
36
+ when "rpm"
37
+ return super(project)
38
+ when "swix"
39
+ return swix_package_name(project)
40
+ else
41
+ fail "I don't know how to name package type '#{project.platform.package_type}' for EOS. Teach me?"
42
+ end
43
+ end
44
+ end
45
+
46
+ # The specific bits used to generate an SWIX package for a given project
47
+ #
48
+ # @param project [Vanagon::Project] project to build a SWIX package of
49
+ # @return [Array] list of commands required to build the SWIX package
50
+ # for the given project from an rpm
51
+ def generate_swix_package(project)
52
+ target_dir = project.repo ? output_dir(project.repo) : output_dir
53
+ commands = ["bash -c 'mkdir -p $(tempdir)/rpmbuild/{SOURCES,SPECS,BUILD,RPMS,SRPMS}'",
54
+ "cp #{project.name}-#{project.version}.tar.gz $(tempdir)/rpmbuild/SOURCES",
55
+ "cp file-list-for-rpm $(tempdir)/rpmbuild/SOURCES",
56
+ "cp #{project.name}.spec $(tempdir)/rpmbuild/SPECS",
57
+ "PATH=/opt/freeware/bin:$$PATH #{@rpmbuild} -bb --target #{@architecture} #{rpm_defines} $(tempdir)/rpmbuild/SPECS/#{project.name}.spec",
58
+ "mkdir -p output/#{target_dir}",
59
+ "cp $(tempdir)/rpmbuild/*RPMS/**/*.rpm ./output/#{target_dir}"]
60
+
61
+
62
+ pkgname_swix = swix_package_name(project)
63
+ pkgname_rpm = pkgname_swix.sub(/swix$/, 'rpm')
64
+ commands += ["echo 'format: 1' > ./output/#{target_dir}/manifest.txt",
65
+ "echo \"primaryRpm: #{pkgname_rpm}\" >> ./output/#{target_dir}/manifest.txt",
66
+ "echo #{pkgname_rpm}-sha1: `sha1sum ./output/#{target_dir}/#{pkgname_rpm}`",
67
+ "cd ./output/#{target_dir}/ && zip #{pkgname_swix} manifest.txt #{pkgname_rpm}",
68
+ "rm ./output/#{target_dir}/manifest.txt ./output/#{target_dir}/#{pkgname_rpm}"]
69
+
70
+ commands
71
+ end
72
+
73
+ # Method to derive the package name for the project
74
+ #
75
+ # @param project [Vanagon::Project] project to name
76
+ # @return [String] name of the SWIX package for this project
77
+ def swix_package_name(project)
78
+ "#{project.name}-#{project.version}-#{project.release}.#{os_name}#{os_version}.#{project.noarch ? 'noarch' : @architecture}.swix"
79
+ end
80
+ end
81
+ end
82
+ end
83
+ end
@@ -10,6 +10,19 @@ class Vanagon
10
10
  name_and_version = "#{project.name}-#{project.version}"
11
11
  pkg_name = package_name(project)
12
12
 
13
+ # Generate list of dirs in the package and create an exlplicit search
14
+ # string for AWK to use in order to explicitly define which directories
15
+ # to put in to the prototype file. Note that the Regexp object was avoided
16
+ # because the output from Regexp would create failures from AWK as the conversion
17
+ # from a Regexp obj to a string is formatted in a sub-optimal way that would have
18
+ # required more string manipulation anyway. the string should be formatted like so:
19
+ # && ($$3 ~ /directory\/regex.*/ || $$3 ~ /another\/directory\/regex.*/)
20
+ # for as many iterations as there are directries in the package
21
+ pkgdirs = project.get_root_directories.map { |dir| dir.sub(/^\//, "").gsub(/([\/\.])+/, '\\\\\1') + '.*' }
22
+ explicit_search_string = pkgdirs.map do |dir_regex|
23
+ " $$3 ~ /" + dir_regex + "/ "
24
+ end.join("||")
25
+
13
26
  # Here we maintain backward compatibility with older vanagon versions
14
27
  # that did this by default. This shim should get removed at some point
15
28
  # in favor of just letting the makefile deliver the bill-of-materials
@@ -42,14 +55,16 @@ class Vanagon
42
55
  # - The bin directory and all bin files are owned by root:bin instead of root:sys
43
56
  # - All files under lib are owned by root:bin instead of root:sys
44
57
  # - All .so files are owned by root:bin instead of root:sys
58
+ # - Explicity only include directories in the package contents
59
+ # (this should exclude things like root/bin root/var and such)
45
60
  %((cd $(tempdir)/#{name_and_version}; pkgproto . | sort | awk ' \
46
- $$1 ~ /^d$$/ {print "d",$$2,$$3,"0755 root sys";} \
61
+ $$1 ~ /^d$$/ && (#{explicit_search_string}) {print "d",$$2,$$3,"0755 root sys";} \
47
62
  $$1 ~ /^s$$/ {print;} \
48
63
  $$1 ~ /^f$$/ {print "f",$$2,$$3,$$4,"root sys";} \
49
64
  $$1 !~ /^[dfs]$$/ {print;} ' | /opt/csw/bin/gsed \
50
- -e '/^[fd] [^ ]\\+ .*[/]s\\?bin/ {s/root sys$$/root bin/}' \
51
- -e '/^[fd] [^ ]\\+ .*[/]lib[/][^/ ]\\+ / {s/root sys$$/root bin/}' \
52
- -e '/^[fd] [^ ]\\+ .*[/][^ ]\\+[.]so / {s/root sys$$/root bin/}' >> ../packaging/proto) ),
65
+ -e '/^[fd] [^ ]\\+ .*[/]s\\?bin[^ ]\\+/ {s/root sys$$/root bin/}' \
66
+ -e '/^[fd] [^ ]\\+ .*[/]lib[/][^ ]\\+/ {s/root sys$$/root bin/}' \
67
+ -e '/^[fd] [^ ]\\+ .*[/][^ ]\\+[.]so/ {s/root sys$$/root bin/}' >> ../packaging/proto)),
53
68
  %((cd $(tempdir); #{project.get_directories.map { |dir| "/opt/csw/bin/ggrep -q 'd none #{dir.path.sub(/^\//, '')}' packaging/proto || echo 'd none #{dir.path.sub(/^\//, '')} #{dir.mode || '0755'} #{dir.owner || 'root'} #{dir.group || 'sys'}' >> packaging/proto" }.join('; ')})),
54
69
 
55
70
  # Actually build the package