vanagon 0.11.3 → 0.12.0

data/lib/vanagon/driver.rb

@@ -43,8 +43,8 @@ class Vanagon
       # flatten all the results in to one array and set project.components to that.
       @project.components = only_build.flat_map { |comp| @project.filter_component(comp) }.uniq
       if @verbose
-        puts "Only building:"
-        @project.components.each { |comp| puts comp.name }
+        $stderr.puts "Only building:"
+        @project.components.each { |comp| $stderr.puts comp.name }
       end
     end
 
@@ -117,7 +117,7 @@ class Vanagon
 
       @engine.startup(@workdir)
 
-      puts "Target is #{@engine.target}"
+      $stderr.puts "Target is #{@engine.target}"
       retry_task { install_build_dependencies }
       retry_task { @project.fetch_sources(@workdir) }
 
@@ -133,8 +133,8 @@ class Vanagon
         cleanup_workdir
       end
     rescue => e
-      puts e
-      puts e.backtrace.join("\n")
+      $stderr.puts e
+      $stderr.puts e.backtrace.join("\n")
       raise e
     ensure
       if ["hardware", "ec2"].include?(@engine.name)
@@ -148,7 +148,7 @@ class Vanagon
         raise Vanagon::Error, "Project requires a version set, all is lost."
       end
 
-      puts "rendering Makefile"
+      $stderr.puts "rendering Makefile"
       retry_task { @project.fetch_sources(@workdir) }
       @project.make_bill_of_materials(@workdir)
       @project.generate_packaging_artifacts(@workdir)
@@ -159,7 +159,7 @@ class Vanagon
       @workdir = workdir ? FileUtils.mkdir_p(workdir).first : Dir.mktmpdir
       @engine.startup(@workdir)
 
-      puts "Devkit on #{@engine.target}"
+      $stderr.puts "Devkit on #{@engine.target}"
 
       install_build_dependencies
       @project.fetch_sources(@workdir)
@@ -169,8 +169,8 @@ class Vanagon
       @engine.ship_workdir(@workdir)
       @engine.dispatch("(cd #{@engine.remote_workdir}; #{@platform.make} #{@project.name}-project)")
     rescue => e
-      puts e
-      puts e.backtrace.join("\n")
+      $stderr.puts e
+      $stderr.puts e.backtrace.join("\n")
       raise e
     end
 

data/lib/vanagon/engine/ec2.rb

@@ -57,17 +57,17 @@ class Vanagon
       end
 
       def select_target
-        puts "Instance created id: #{instance.id}"
-        puts "Created instance waiting for status ok"
+        $stderr.puts "Instance created id: #{instance.id}"
+        $stderr.puts "Created instance waiting for status ok"
         @ec2.wait_until(:instance_status_ok, instance_ids: [instance.id])
-        puts "Instance running"
+        $stderr.puts "Instance running"
         @target = instance.private_ip_address
       rescue ::Aws::Waiters::Errors::WaiterFailed => error
         fail "Failed to wait for ec2 instance to start got error #{error}"
       end
 
       def teardown
-        puts "Destroying instance on AWS id: #{instance.id}"
+        $stderr.puts "Destroying instance on AWS id: #{instance.id}"
         instances.batch_terminate!
       end
     end

data/lib/vanagon/engine/hardware.rb

@@ -23,7 +23,7 @@ class Vanagon
         Vanagon::Driver.logger.info "Polling for a lock on #{host}."
         @lockman.polling_lock(host, VANAGON_LOCK_USER, "Vanagon automated lock")
         Vanagon::Driver.logger.info "Lock acquired on #{host}."
-        puts "Lock acquired on #{host} for #{VANAGON_LOCK_USER}."
+        $stderr.puts "Lock acquired on #{host} for #{VANAGON_LOCK_USER}."
         host
       end
 
@@ -33,7 +33,7 @@ class Vanagon
           Vanagon::Driver.logger.info "Attempting  to lock #{h}."
           if @lockman.lock(h, VANAGON_LOCK_USER, "Vanagon automated lock")
             Vanagon::Driver.logger.info "Lock acquired on #{h}."
-            puts "Lock acquired on #{h} for #{VANAGON_LOCK_USER}."
+            $stderr.puts "Lock acquired on #{h} for #{VANAGON_LOCK_USER}."
             return h
           end
         end
@@ -45,7 +45,7 @@ class Vanagon
       # complete. In this case, we'll attempt to unlock the hardware
       def teardown
         Vanagon::Driver.logger.info "Removing lock on #{@target}."
-        puts "Removing lock on #{@target}."
+        $stderr.puts "Removing lock on #{@target}."
         @lockman.unlock(@target, VANAGON_LOCK_USER)
       end
 

data/lib/vanagon/engine/pooler.rb

@@ -56,7 +56,7 @@ class Vanagon
         absolute_path = File.expand_path(path)
         return nil unless File.exist?(absolute_path)
 
-        puts "Reading vmpooler token from: #{path}"
+        $stderr.puts "Reading vmpooler token from: #{path}"
         File.read(absolute_path).chomp
       end
       private :read_vanagon_token
@@ -70,7 +70,7 @@ class Vanagon
         absolute_path = File.expand_path(path)
         return nil unless File.exist?(absolute_path)
 
-        puts "Reading vmpooler token from: #{path}"
+        $stderr.puts "Reading vmpooler token from: #{path}"
         YAML.load_file(absolute_path)['token']
       end
       private :read_vmfloaty_token
@@ -119,7 +119,7 @@ class Vanagon
         )
         if response and response["ok"]
           Vanagon::Driver.logger.info "#{@target} has been destroyed"
-          puts "#{@target} has been destroyed"
+          $stderr.puts "#{@target} has been destroyed"
         else
           Vanagon::Driver.logger.info "#{@target} could not be destroyed"
           warn "#{@target} could not be destroyed"

data/lib/vanagon/optparse.rb

@@ -28,7 +28,7 @@ class Vanagon
         end
 
         opts.on('-h', '--help', 'Display help') do
-          puts opts
+          $stdout.puts opts
           exit 1
         end
       end

data/lib/vanagon/platform.rb

@@ -60,6 +60,9 @@ class Vanagon
     # Stores the local path where retrieved artifacts will be saved.
     attr_accessor :output_dir
 
+    # Stores the local path where source artifacts will be saved.
+    attr_accessor :source_output_dir
+
     # Username to use when connecting to a build target
     attr_accessor :target_user
 
@@ -126,14 +129,13 @@ class Vanagon
     # @raise if the instance_eval on Platform fails, the exception is reraised
     def self.load_platform(name, configdir)
       platfile = File.join(configdir, "#{name}.rb")
-      code = File.read(platfile)
       dsl = Vanagon::Platform::DSL.new(name)
-      dsl.instance_eval(code, __FILE__, __LINE__)
+      dsl.instance_eval(File.read(platfile), platfile, 1)
       dsl._platform
     rescue => e
-      puts "Error loading platform '#{name}' using '#{platfile}':"
-      puts e
-      puts e.backtrace.join("\n")
+      $stderr.puts "Error loading platform '#{name}' using '#{platfile}':"
+      $stderr.puts e
+      $stderr.puts e.backtrace.join("\n")
       raise e
     end
 
@@ -239,6 +241,17 @@ class Vanagon
       @output_dir ||= File.join(@os_name, @os_version, target_repo, @architecture)
     end
 
+    # Get the source dir for packages. Don't change it if it was already defined
+    # by the platform config. Defaults to output_dir unless specified otherwise
+    # (RPM specifies this)
+    #
+    # @param target_repo [String] optional repo target for built source packages
+    # defined at the project level
+    # @return [String] relative path to where source packages should be output to
+    def source_output_dir(target_repo = "")
+      @source_output_dir ||= output_dir(target_repo)
+    end
+
     # Get the value of @dist, or derive it from the value of @os_name and @os_version.
     # This is relatively RPM specific but '#codename' is defined in Platform, and that's
     # just as Deb/Ubuntu specific. All of the accessors in the top-level Platform

data/lib/vanagon/platform/deb.rb

@@ -7,6 +7,11 @@ class Vanagon
       # @return [Array] list of commands required to build a debian package for the given project from a tarball
       def generate_package(project) # rubocop:disable Metrics/AbcSize
         target_dir = project.repo ? output_dir(project.repo) : output_dir
+        if project.source_artifacts
+          copy_extensions = '*.{deb,build,tar.gz,changes,dsc}'
+        else
+          copy_extensions = '*.deb'
+        end
         pkg_arch_opt = project.noarch ? "" : "-a#{@architecture}"
         ["mkdir -p output/#{target_dir}",
         "mkdir -p $(tempdir)/#{project.name}-#{project.version}",
@@ -16,7 +21,7 @@ class Vanagon
         "gunzip -c #{project.name}-#{project.version}.tar.gz | '#{@tar}' -C '$(tempdir)/#{project.name}-#{project.version}' --strip-components 1 -xf -",
         "sed -i 's/\ /?/g' $(tempdir)/#{project.name}-#{project.version}/debian/install",
         "(cd $(tempdir)/#{project.name}-#{project.version}; debuild --no-lintian #{pkg_arch_opt} -uc -us)",
-        "cp $(tempdir)/*.deb ./output/#{target_dir}"]
+        "cp $(tempdir)/#{copy_extensions} ./output/#{target_dir}"]
       end
 
       # Method to generate the files required to build a debian package for the project

data/lib/vanagon/platform/dsl.rb

@@ -348,6 +348,10 @@ class Vanagon
         @platform.output_dir = directory
       end
 
+      def source_output_dir(directory)
+        @platform.source_output_dir = directory
+      end
+
       # Helper to setup a apt repository on a target system
       #
       # @param definition [String] the repo setup file, must be a valid uri, fetched with curl

data/lib/vanagon/platform/rpm.rb

@@ -8,15 +8,24 @@ class Vanagon
       #
       # @param project [Vanagon::Project] project to build an rpm package of
       # @return [Array] list of commands required to build an rpm package for the given project from a tarball
-      def generate_package(project)
+      def generate_package(project) # rubocop:disable Metrics/AbcSize
         target_dir = project.repo ? output_dir(project.repo) : output_dir
+        target_source_output_dir = project.repo ? source_output_dir(project.repo) : source_output_dir
+        if project.source_artifacts
+          rpmbuild = "#{@rpmbuild} -ba"
+          artifact_copy = "mkdir -p output/#{target_source_output_dir}; cp $(tempdir)/rpmbuild/RPMS/**/*.rpm ./output/#{target_dir}; cp $(tempdir)/rpmbuild/SRPMS/*.rpm ./output/#{target_source_output_dir}"
+        else
+          rpmbuild = "#{@rpmbuild} -bb"
+          artifact_copy = "cp $(tempdir)/rpmbuild/*RPMS/**/*.rpm ./output/#{target_dir}"
+        end
+
         ["bash -c 'mkdir -p $(tempdir)/rpmbuild/{SOURCES,SPECS,BUILD,RPMS,SRPMS}'",
         "cp #{project.name}-#{project.version}.tar.gz $(tempdir)/rpmbuild/SOURCES",
         "cp file-list-for-rpm $(tempdir)/rpmbuild/SOURCES",
         "cp #{project.name}.spec $(tempdir)/rpmbuild/SPECS",
-        "PATH=/opt/freeware/bin:$$PATH #{@rpmbuild} -bb --target #{@architecture} #{rpm_defines} $(tempdir)/rpmbuild/SPECS/#{project.name}.spec",
+        "PATH=/opt/freeware/bin:$$PATH #{rpmbuild} --target #{@architecture} #{rpm_defines} $(tempdir)/rpmbuild/SPECS/#{project.name}.spec",
         "mkdir -p output/#{target_dir}",
-        "cp $(tempdir)/rpmbuild/*RPMS/**/*.rpm ./output/#{target_dir}"]
+        artifact_copy]
       end
 
       # Method to generate the files required to build an rpm package for the project
@@ -40,6 +49,13 @@ class Vanagon
         super
       end
 
+      # Method to derive the directory for source artifacts
+      #
+      # @param target_repo [String] repo the source artifacts are targeting
+      def source_output_dir(target_repo = "products")
+        @source_output_dir ||= File.join(@os_name, @os_version, target_repo, 'SRPMS')
+      end
+
       def rpm_defines
         defines =  %(--define '_topdir $(tempdir)/rpmbuild' )
         # RPM doesn't allow dashes in the os_name. This was added to

data/lib/vanagon/project.rb

@@ -80,6 +80,9 @@ class Vanagon
     # them to appear in your spec/rules files!
     attr_accessor :package_overrides
 
+    # Should we include source packages?
+    attr_accessor :source_artifacts
+
     # Loads a given project from the configdir
     #
     # @param name [String] the name of the project
@@ -90,14 +93,13 @@ class Vanagon
     # @raise if the instance_eval on Project fails, the exception is reraised
     def self.load_project(name, configdir, platform, include_components = [])
       projfile = File.join(configdir, "#{name}.rb")
-      code = File.read(projfile)
       dsl = Vanagon::Project::DSL.new(name, platform, include_components)
-      dsl.instance_eval(code, __FILE__, __LINE__)
+      dsl.instance_eval(File.read(projfile), projfile, 1)
       dsl._project
     rescue => e
-      puts "Error loading project '#{name}' using '#{projfile}':"
-      puts e
-      puts e.backtrace.join("\n")
+      $stderr.puts "Error loading project '#{name}' using '#{projfile}':"
+      $stderr.puts e
+      $stderr.puts e.backtrace.join("\n")
       raise e
     end
 
@@ -122,6 +124,7 @@ class Vanagon
       @provides = []
       @conflicts = []
       @package_overrides = []
+      @source_artifacts = false
     end
 
     # Magic getter to retrieve settings in the project

data/lib/vanagon/project/dsl.rb

@@ -148,6 +148,15 @@ class Vanagon
         @project.release = rel
       end
 
+      # Generate source packages in addition to binary packages.
+      # Currently only implemented for rpm/deb packages.
+      #
+      # @param source_artifacts [Boolean] whether or not to output
+      #        source packages
+      def generate_source_artifacts(source_artifacts)
+        @project.source_artifacts = source_artifacts
+      end
+
       # Sets the version for the project based on a git describe of the
       # directory that holds the configs. Requires that a git tag be present
       # and reachable from the current commit in that repository.
@@ -228,7 +237,7 @@ class Vanagon
       #
       # @param name [String] name of component to add. must be present in configdir/components and named $name.rb currently
       def component(name)
-        puts "Loading #{name}" if @project.settings[:verbose]
+        $stderr.puts "Loading #{name}" if @project.settings[:verbose]
         if @include_components.empty? or @include_components.include?(name)
           component = Vanagon::Component.load_component(name, File.join(Vanagon::Driver.configdir, "components"), @project.settings, @project.platform)
           @project.components << component
@@ -252,7 +261,7 @@ class Vanagon
       # @param protocol [String] a supported component source type (Http, Git, ...)
       # @param rule [String, Proc] a rule used to rewrite component source urls
       def register_rewrite_rule(protocol, rule)
-        Vanagon::Component::Source.register_rewrite_rule(protocol, rule)
+        Vanagon::Component::Source::Rewrite.register_rewrite_rule(protocol, rule)
       end
 
       # Toggle to apply additional cleanup during the build for space constrained systems

data/lib/vanagon/utilities.rb

@@ -210,7 +210,7 @@ class Vanagon
     #                        output of the command if return_command_output is true
     # @raise [RuntimeError] If there is no target given or the command fails an exception is raised
     def remote_ssh_command(target, command, port = 22, return_command_output: false)
-      puts "Executing '#{command}' on '#{target}'"
+      $stderr.puts "Executing '#{command}' on '#{target}'"
       if return_command_output
         ret = %x(#{ssh_command(port)} -T #{target} '#{command.gsub("'", "'\\\\''")}').chomp
         if $CHILD_STATUS.success?
@@ -233,7 +233,7 @@ class Vanagon
     # @raise [RuntimeError] If the command fails an exception is raised
     def local_command(command, return_command_output: false)
       clean_environment do
-        puts "Executing '#{command}' locally"
+        $stderr.puts "Executing '#{command}' locally"
         if return_command_output
           ret = %x(#{command}).chomp
           if $CHILD_STATUS.success?
@@ -277,7 +277,7 @@ class Vanagon
       outfile ||= File.join(Dir.mktmpdir, File.basename(erbfile).sub(File.extname(erbfile), ""))
       output = erb_string(erbfile, opts[:binding])
       File.open(outfile, 'w') { |f| f.write output }
-      puts "Generated: #{outfile}"
+      $stderr.puts "Generated: #{outfile}"
       FileUtils.rm_rf erbfile if remove_orig
       outfile
     end

data/spec/lib/vanagon/component/source/rewrite_spec.rb

@@ -0,0 +1,55 @@
+require 'vanagon/component/source'
+
+describe "Vanagon::Component::Source::Rewrite" do
+  let(:klass) { Vanagon::Component::Source::Rewrite }
+  before(:each) { klass.rewrite_rules.clear }
+
+  describe ".parse_and_rewrite" do
+    let(:simple_rule) { Proc.new {|url| url.gsub('a', 'e') } }
+    let(:complex_rule) do
+      Proc.new do |url|
+        match = url.match(/github.com\/(.*)$/)
+        "git://github.delivery.puppetlabs.net/#{match[1].gsub('/', '-')}" if match
+      end
+    end
+
+    it 'replaces the first section of a url with a string if string is given' do
+      klass.register_rewrite_rule('http', 'http://buildsources.delivery.puppetlabs.net')
+
+      expect(klass.rewrite('http://things.and.stuff/foo.tar.gz', 'http'))
+        .to eq('http://buildsources.delivery.puppetlabs.net/foo.tar.gz')
+    end
+
+    it 'applies the rule to the url if a proc is given as the rule' do
+      klass.register_rewrite_rule('http', simple_rule)
+
+      expect(klass.rewrite('http://things.and.stuff/foo.tar.gz', 'http'))
+        .to eq('http://things.end.stuff/foo.ter.gz')
+    end
+
+    it 'applies the rule to the url if a proc is given as the rule' do
+      klass.register_rewrite_rule('git', complex_rule)
+
+      expect(klass.rewrite('git://github.com/puppetlabs/facter', 'git'))
+        .to eq('git://github.delivery.puppetlabs.net/puppetlabs-facter')
+    end
+  end
+
+  describe ".register_rewrite_rule" do
+    it 'only accepts Proc and String as rule types' do
+      expect { klass.register_rewrite_rule('http', 5) }
+        .to raise_error(Vanagon::Error)
+    end
+
+    it 'rejects invalid protocols' do
+      expect { klass.register_rewrite_rule('gopher', 'abcd') }
+        .to raise_error Vanagon::Error
+    end
+
+    before { klass.register_rewrite_rule('http', 'http://buildsources.delivery.puppetlabs.net') }
+    it 'registers the rule for the given protocol' do
+      expect(klass.rewrite_rules)
+        .to eq({'http' => 'http://buildsources.delivery.puppetlabs.net'})
+    end
+  end
+end

data/spec/lib/vanagon/component/source_spec.rb

@@ -2,7 +2,6 @@ require 'vanagon/component/source'
 
 describe "Vanagon::Component::Source" do
   let(:klass) { Vanagon::Component::Source }
-  before(:each) { klass.rewrite_rules.clear }
 
   describe ".source" do
     let(:unrecognized_uri) { "abcd://things" }
@@ -24,10 +23,8 @@ describe "Vanagon::Component::Source" do
     let(:workdir) { Dir.mktmpdir }
 
     let(:original_git_url) { "git://things.and.stuff/foo-bar.git" }
-    let(:rewritten_git_url) { "git://things.end.stuff/foo-ber.git" }
 
     let(:original_http_url) { "http://things.and.stuff/foo.tar.gz" }
-    let(:rewritten_http_url) { "http://buildsources.delivery.puppetlabs.net/foo.tar.gz" }
 
     it "fails on unrecognized URI schemes" do
       expect { klass.source(unrecognized_uri, workdir: workdir) }
@@ -69,17 +66,6 @@ describe "Vanagon::Component::Source" do
         expect(klass.source(https_git, ref: ref, workdir: workdir).class)
           .to eq Vanagon::Component::Source::Git
       end
-
-      it "rewrites git:// URLs" do
-        proc_rule = Proc.new { |url| url.gsub('a', 'e') }
-        klass.register_rewrite_rule('git', proc_rule)
-        # Vanagon::Component::Source::Git#url returns a URI object
-        # so to check its value, we cast it to a simple string. It's
-        # hacky for sure, but seems less diagreeable than mangling the
-        # return value in the class itself.
-        expect(klass.source(original_git_url, ref: ref, workdir: workdir).url.to_s)
-        .to eq rewritten_git_url
-      end
     end
 
     context "takes a HTTP/HTTPS file" do
@@ -102,15 +88,6 @@ describe "Vanagon::Component::Source" do
         expect(klass.source(https_url, sum: sum, workdir: workdir, sum_type: "md5").class)
           .to equal(Vanagon::Component::Source::Http)
       end
-
-      before do
-        klass.register_rewrite_rule 'http',
-          'http://buildsources.delivery.puppetlabs.net'
-      end
-      it "applies rewrite rules to HTTP URLs" do
-        expect(klass.source(original_http_url, sum: sum, workdir: workdir, sum_type: "md5").url)
-          .to eq(rewritten_http_url)
-      end
     end
 
     context "takes a local file" do
@@ -130,53 +107,4 @@ describe "Vanagon::Component::Source" do
       end
     end
   end
-
-  describe ".rewrite" do
-    let(:simple_rule) { Proc.new {|url| url.gsub('a', 'e') } }
-    let(:complex_rule) do
-      Proc.new do |url|
-        match = url.match(/github.com\/(.*)$/)
-        "git://github.delivery.puppetlabs.net/#{match[1].gsub('/', '-')}" if match
-      end
-    end
-
-    it 'replaces the first section of a url with a string if string is given' do
-      klass.register_rewrite_rule('http', 'http://buildsources.delivery.puppetlabs.net')
-
-      expect(klass.rewrite('http://things.and.stuff/foo.tar.gz', 'http'))
-        .to eq('http://buildsources.delivery.puppetlabs.net/foo.tar.gz')
-    end
-
-    it 'applies the rule to the url if a proc is given as the rule' do
-      klass.register_rewrite_rule('http', simple_rule)
-
-      expect(klass.rewrite('http://things.and.stuff/foo.tar.gz', 'http'))
-        .to eq('http://things.end.stuff/foo.ter.gz')
-    end
-
-    it 'applies the rule to the url if a proc is given as the rule' do
-      klass.register_rewrite_rule('git', complex_rule)
-
-      expect(klass.rewrite('git://github.com/puppetlabs/facter', 'git'))
-        .to eq('git://github.delivery.puppetlabs.net/puppetlabs-facter')
-    end
-  end
-
-  describe ".register_rewrite_rule" do
-    it 'only accepts Proc and String as rule types' do
-      expect { klass.register_rewrite_rule('http', 5) }
-        .to raise_error(Vanagon::Error)
-    end
-
-    it 'rejects invalid protocols' do
-      expect { klass.register_rewrite_rule('gopher', 'abcd') }
-        .to raise_error Vanagon::Error
-    end
-
-    before { klass.register_rewrite_rule('http', 'http://buildsources.delivery.puppetlabs.net') }
-    it 'registers the rule for the given protocol' do
-      expect(klass.rewrite_rules)
-        .to eq({'http' => 'http://buildsources.delivery.puppetlabs.net'})
-    end
-  end
 end