vanagon 0.15.35 → 0.16.1

data/lib/vanagon/platform.rb

@@ -115,6 +115,7 @@ class Vanagon
     # Docker engine specific
     attr_accessor :docker_image
     attr_accessor :docker_run_args
+    attr_accessor :use_docker_exec
 
     # AWS engine specific
     attr_accessor :aws_ami
@@ -238,6 +239,8 @@ class Vanagon
       @copy ||= "cp"
       @shasum ||= "sha1sum"
 
+      @use_docker_exec = false
+
       # Our first attempt at defining metadata about a platform
       @cross_compiled ||= false
       @valid_operators ||= ['<', '>', '<=', '>=', '=']
@@ -331,6 +334,10 @@ class Vanagon
       return !!@name.match(/^(el|redhat|redhatfips)-.*$/)
     end
 
+    def is_el8?
+      return !!@name.match(/^(el|redhat|redhatfips)-8.*$/)
+    end
+
     # Utility matcher to determine is the platform is a sles variety
     #
     # @return [true, false] true if it is a sles variety, false otherwise
@@ -482,9 +489,9 @@ class Vanagon
     def generate_compiled_archive(project)
       name_and_version = "#{project.name}-#{project.version}"
       name_and_version_and_platform = "#{name_and_version}.#{name}"
+      name_and_platform = "#{project.name}.#{name}"
       final_archive = "output/#{name_and_version_and_platform}.tar.gz"
       archive_directory = "#{project.name}-archive"
-      metadata = project.build_manifest_json(true)
 
       # previously, we weren't properly handling the case of custom BOM paths.
       # If we have a custom BOM path, during Makefile execution, the top-level
@@ -499,7 +506,6 @@ class Vanagon
         bill_of_materials_command = "mv .#{project.bill_of_materials.path}/bill-of-materials ../.."
       end
 
-      metadata.gsub!(/\n/, '\n')
       [
         "mkdir output",
         "mkdir #{archive_directory}",
@@ -507,7 +513,7 @@ class Vanagon
         "rm #{name_and_version}.tar.gz",
         "cd #{archive_directory}/#{name_and_version}; #{bill_of_materials_command}; #{tar} cf ../../#{name_and_version_and_platform}.tar *",
         "gzip -9c #{name_and_version_and_platform}.tar > #{name_and_version_and_platform}.tar.gz",
-        "echo -e \"#{metadata}\" > output/#{name_and_version_and_platform}.json",
+        "cp build_metadata.#{name_and_platform}.json output/#{name_and_version_and_platform}.json",
         "cp bill-of-materials output/#{name_and_version_and_platform}-bill-of-materials ||:",
         "cp #{name_and_version_and_platform}.tar.gz output",
         "#{shasum} #{final_archive} > #{final_archive}.sha1"

data/lib/vanagon/platform/deb.rb

@@ -140,7 +140,7 @@ class Vanagon
         @tar = "tar"
         @patch = "/usr/bin/patch"
         @num_cores = "/usr/bin/nproc"
-        @curl = "curl --silent --show-error --fail"
+        @curl = "curl --silent --show-error --fail --location"
         @valid_operators = ['<', '>', '<=', '>=', '=', '<<', '>>']
         super(name)
       end

data/lib/vanagon/platform/dsl.rb

@@ -284,6 +284,17 @@ class Vanagon
         @platform.docker_run_args = Array(args)
       end
 
+      # Specify whether to use Docker exec instead of SSH to run commands
+      #
+      # This also causes Vanagon to use `docker cp` instead of `rsync` when
+      # copying files.
+      #
+      # @param bool [Boolean] a boolean value indicating whether to use
+      #   `docker exec` and `docker cp` over `ssh` and `rsync`.
+      def use_docker_exec(bool)
+        @platform.use_docker_exec = bool
+      end
+
       # Set the ami for the platform to use
       #
       # @param ami [String] the ami id used.

data/lib/vanagon/platform/rpm.rb

@@ -103,7 +103,7 @@ class Vanagon
         @patch ||= "/usr/bin/patch"
         @num_cores ||= "/bin/grep -c 'processor' /proc/cpuinfo"
         @rpmbuild ||= "/usr/bin/rpmbuild"
-        @curl = "curl --silent --show-error --fail"
+        @curl = "curl --silent --show-error --fail --location"
         super(name)
       end
     end

data/lib/vanagon/platform/windows.rb

@@ -205,11 +205,36 @@ class Vanagon
         # "Misc Dir for versions.txt, License file and Icon file"
         misc_dir = "SourceDir/#{project.settings[:base_dir]}/#{project.settings[:company_id]}/#{project.settings[:product_id]}/misc"
         # Actual array of commands to be written to the Makefile
-        [
+        make_commands = [
           "mkdir -p output/#{target_dir}",
           "mkdir -p $(tempdir)/{SourceDir,wix/wixobj}",
           "#{@copy} -r wix/* $(tempdir)/wix/",
-          "gunzip -c #{project.name}-#{project.version}.tar.gz | '#{@tar}' -C '$(tempdir)/SourceDir' --strip-components 1 -xf -",
+          "gunzip -c #{project.name}-#{project.version}.tar.gz | '#{@tar}' -C '$(tempdir)/SourceDir' --strip-components 1 -xf -"
+        ]
+
+        unless project.extra_files_to_sign.empty?
+          begin
+            tempdir = nil
+            # Skip signing extra files if logging into the signing_host fails
+            # This enables things like CI being able to sign the additional files,
+            # but locally triggered builds by developers who don't have access to
+            # the signing host just print a message and skip the signing.
+            Vanagon::Utilities.retry_with_timeout(3, 5) do
+              tempdir = Vanagon::Utilities::remote_ssh_command("#{project.signing_username}@#{project.signing_hostname}", "#{@mktemp} 2>/dev/null", return_command_output: true)
+            end
+            project.extra_files_to_sign.each do |file|
+              file_location = File.join(tempdir, File.basename(file))
+              make_commands << [
+                "rsync -e '#{Vanagon::Utilities.ssh_command}' -rHlv --no-perms --no-owner --no-group #{File.join('$(tempdir)', 'SourceDir', file)} #{project.signing_username}@#{project.signing_hostname}:#{tempdir}",
+                "#{Vanagon::Utilities.ssh_command} #{project.signing_username}@#{project.signing_hostname} #{project.signing_command} #{file_location}",
+                "rsync -e '#{Vanagon::Utilities.ssh_command}' -rHlv -O --no-perms --no-owner --no-group #{project.signing_username}@#{project.signing_hostname}:#{file_location} #{File.join('$(tempdir)', 'SourceDir', file)}"
+              ]
+            end
+          rescue RuntimeError
+            warn "Unable to connect to #{project.signing_username}@#{project.signing_hostname}, skipping signing extra files: #{project.extra_files_to_sign.join(',')}"
+          end
+        end
+        make_commands << [
           "mkdir -p $(tempdir)/#{misc_dir}",
           # Need to use awk here to convert to DOS format so that notepad can display file correctly.
           "awk 'sub(\"$$\", \"\\r\")' $(tempdir)/SourceDir/bill-of-materials > $(tempdir)/#{misc_dir}/versions.txt",
@@ -225,6 +250,8 @@ class Vanagon
           # -loc is required for the UI localization it points to the actual localization .wxl
           "cd $(tempdir)/wix/wixobj; \"$$WIX/bin/light.exe\" #{light_flags} -b $$(cygpath -aw $(tempdir)) -loc $$(cygpath -aw $(tempdir)/wix/localization/puppet_en-us.wxl) -out $$(cygpath -aw $(workdir)/output/#{target_dir}/#{msi_package_name(project)}) *.wixobj",
         ]
+
+        make_commands.flatten
       end
 
       # Method to derive the msi (Windows Installer) package name for the project.

data/lib/vanagon/project.rb

@@ -108,6 +108,14 @@ class Vanagon
     # you just want to perform installation and pull down a file.
     attr_accessor :no_packaging
 
+    # Extra files to sign
+    # Right now just supported on windows, useful for signing powershell scripts
+    # that need to be signed between build and MSI creation
+    attr_accessor :extra_files_to_sign
+    attr_accessor :signing_hostname
+    attr_accessor :signing_username
+    attr_accessor :signing_command
+
     # Loads a given project from the configdir
     #
     # @param name [String] the name of the project
@@ -156,6 +164,10 @@ class Vanagon
       @upstream_metadata = {}
       @no_packaging = false
       @artifacts_to_fetch = []
+      @extra_files_to_sign = []
+      @signing_hostname = ''
+      @signing_username = ''
+      @signing_command = ''
     end
 
     # Magic getter to retrieve settings in the project
@@ -714,20 +726,27 @@ class Vanagon
     # Writes a json file at `ext/build_metadata.<project>.<platform>.json` containing information
     # about what went into a built artifact
     #
-    # @return [Hash] of build information
-    def save_manifest_json(platform)
+    # @param platform [String] platform we're writing metadata for
+    # @param additional_directories [String|Array[String]] additional
+    #        directories to write build_metadata.<project>.<platform>.json to
+    def save_manifest_json(platform, additional_directories = nil) # rubocop:disable Metrics/AbcSize
       manifest = build_manifest_json
       metadata = metadata_merge(manifest, @upstream_metadata)
 
       ext_directory = 'ext'
       FileUtils.mkdir_p ext_directory
 
+      directories = [ext_directory, additional_directories].compact
       metadata_file_name = "build_metadata.#{name}.#{platform.name}.json"
-      File.open(File.join(ext_directory, metadata_file_name), 'w') do |f|
-        f.write(JSON.pretty_generate(metadata))
+      directories.each do |directory|
+        File.open(File.join(directory, metadata_file_name), 'w') do |f|
+          f.write(JSON.pretty_generate(metadata))
+        end
       end
 
       ## VANAGON-132 Backwards compatibility: make a 'build_metadata.json' file
+      # No need to propagate this backwards compatibility to the new additional
+      # directories
       File.open(File.join(ext_directory, 'build_metadata.json'), 'w') do |f|
         f.write(JSON.pretty_generate(metadata))
       end

data/lib/vanagon/project/dsl.rb

@@ -363,6 +363,39 @@ class Vanagon
       def no_packaging(var)
         @project.no_packaging = var
       end
+
+      # Set to sign additional files during buildtime. Only implemented for
+      # windows. Can be specified more than once
+      #
+      # @param [String] file to sign
+      def extra_file_to_sign(file)
+        @project.extra_files_to_sign << file
+      end
+
+      # The hostname to sign additional files on. Only does anything when there
+      # are extra files to sign
+      #
+      # @param [String] hostname of the machine to run the extra file signing on
+      def signing_hostname(hostname)
+        @project.signing_hostname = hostname
+      end
+
+      # The username to log in to the signing_hostname as. Only does anything
+      # when there are extra files to sign
+      #
+      # @param [String] the username to log in to `signing_hostname` as
+      def signing_username(username)
+        @project.signing_username = username
+      end
+
+      # The command to run to sign additional files. The command should assume
+      # it will have the file path appended to the end of the command, since
+      # files end up in a temp directory.
+      #
+      # @param [String] the command to sign additional files
+      def signing_command(command)
+        @project.signing_command = command
+      end
     end
   end
 end

data/resources/rpm/project.spec.erb

@@ -33,6 +33,15 @@
 <%= var %>
 <% end -%>
 
+# This breaks on el8. This is a hack to unblock development.
+<%- if @platform.is_el8? %>
+%undefine __debug_package
+
+# Build el-8 packages without build-id files to prevent collision
+%define _build_id_links none
+<% end -%>
+
+
 # To avoid files installed but not packaged errors
 %global __os_install_post %{__os_install_post} \
     rm -rf %{buildroot}/usr/lib/debug

data/spec/lib/vanagon/cli_spec.rb

@@ -0,0 +1,80 @@
+require 'vanagon/cli'
+
+##
+## Ignore the CLI calling 'exit'
+##
+RSpec.configure do |rspec|
+  rspec.around(:example) do |ex|
+    begin
+      ex.run
+    rescue SystemExit => e
+      puts "Got SystemExit: #{e.inspect}. Ignoring"
+    end
+  end
+end
+
+describe Vanagon::CLI do
+  describe "options that don't take a value" do
+    [:skipcheck, :verbose].each do |flag|
+      it "can create an option parser that accepts the #{flag} flag" do
+        subject = described_class.new
+        expect(subject.parse(%W[build --#{flag} project platform])).to have_key(flag)
+      end
+    end
+
+    describe "short options" do
+      [["v", :verbose]].each do |short, long|
+        it "maps the short option #{short} to #{long}" do
+          subject = described_class.new
+          expect(subject.parse(%W[build -#{short} project platform])).to include(long => true)
+        end
+      end
+    end
+  end
+
+  describe "options that only allow limited values" do
+    [[:preserve, ["always", "never", "on-failure"]]].each do |option, values|
+      values.each do |value|
+        it "can create a parser that accepts \"--#{option} #{value}\"" do
+          subject = described_class.new
+          expect(subject.parse(%W[build --#{option} #{value} project platform]))
+            .to include(option => value.to_sym)
+        end
+      end
+    end
+    [[:preserve, ["bad-argument"]]].each do |option, values|
+      values.each do |value|
+        it "rejects the bad argument \"--#{option} #{value}\"" do
+          subject = described_class.new
+          expect{subject.parse(%W[build --#{option} #{value} project platform])}
+            .to raise_error(Vanagon::InvalidArgument)
+        end
+      end
+    end
+    it "preserve defaults to :on-failure" do
+      subject = described_class.new
+      expect(subject.parse([])).to include(:preserve => :'on-failure')
+    end
+  end
+
+
+  describe "options that take a value" do
+    [:workdir, :configdir, :engine].each do |option|
+      it "can create an option parser that accepts the #{option} option" do
+        subject = described_class.new
+        expect(subject.parse(%W[build --#{option} hello project platform]))
+          .to include(option => "hello")
+      end
+    end
+
+    describe "short options" do
+      [["w", :workdir], ["c", :configdir], ["e", :engine]].each do |short, long|
+        it "maps the short option #{short} to #{long}" do
+          subject = described_class.new
+          expect(subject.parse(%W[build -#{short} hello project platform]))
+            .to include(long => "hello")
+        end
+      end
+    end
+  end
+end

data/spec/lib/vanagon/engine/docker_spec.rb

@@ -1,21 +1,36 @@
 require 'vanagon/engine/docker'
 require 'vanagon/platform'
 
-describe 'Vanagon::Engine::Docker' do
-  let (:platform_with_docker_image) {
-    plat = Vanagon::Platform::DSL.new('debian-6-i386')
-    plat.instance_eval("platform 'debian-6-i386' do |plat|
-                      plat.docker_image 'debian-6-i386'
-                    end")
+describe Vanagon::Engine::Docker do
+  let (:platform_with_docker_image) do
+    plat = Vanagon::Platform::DSL.new('debian-10-amd64')
+    plat.instance_eval(<<~EOF)
+      platform 'debian-10-amd64' do |plat|
+        plat.docker_image 'debian:10-slim'
+      end
+    EOF
     plat._platform
-  }
+  end
 
-  let (:platform_without_docker_image) {
-    plat = Vanagon::Platform::DSL.new('debian-6-i386')
-    plat.instance_eval("platform 'debian-6-i386' do |plat|
-                    end")
+  let (:platform_without_docker_image) do
+    plat = Vanagon::Platform::DSL.new('debian-10-amd64')
+    plat.instance_eval(<<~EOF)
+      platform 'debian-10-amd64' do |plat|
+      end
+    EOF
+    plat._platform
+  end
+
+  let(:platform_with_docker_exec) do
+    plat = Vanagon::Platform::DSL.new('debian-10-amd64')
+    plat.instance_eval(<<~EOF)
+      platform 'debian-10-amd64' do |plat|
+        plat.docker_image 'debian:10-slim'
+        plat.use_docker_exec true
+      end
+    EOF
     plat._platform
-  }
+  end
 
   describe '#initialize' do
     it 'fails without docker installed' do
@@ -23,24 +38,67 @@ describe 'Vanagon::Engine::Docker' do
         expect(FileTest).to receive(:executable?).with(File.join(path_elem, 'docker')).and_return(false)
       end
 
-      expect { Vanagon::Engine::Docker.new(platform_with_docker_image) }.to raise_error(RuntimeError)
+      expect { described_class.new(platform_with_docker_image) }.to raise_error(RuntimeError)
     end
   end
 
   describe "#validate_platform" do
     it 'raises an error if the platform is missing a required attribute' do
       expect(Vanagon::Utilities).to receive(:find_program_on_path).with('docker').and_return('/usr/bin/docker')
-      expect { Vanagon::Engine::Docker.new(platform_without_docker_image).validate_platform }.to raise_error(Vanagon::Error)
+      expect { described_class.new(platform_without_docker_image).validate_platform }.to raise_error(Vanagon::Error)
     end
 
     it 'returns true if the platform has the required attributes' do
       expect(Vanagon::Utilities).to receive(:find_program_on_path).with('docker').and_return('/usr/bin/docker')
-      expect(Vanagon::Engine::Docker.new(platform_with_docker_image).validate_platform).to be(true)
+      expect(described_class.new(platform_with_docker_image).validate_platform).to be(true)
     end
   end
 
   it 'returns "docker" name' do
     expect(Vanagon::Utilities).to receive(:find_program_on_path).with('docker').and_return('/usr/bin/docker')
-    expect(Vanagon::Engine::Docker.new(platform_with_docker_image).name).to eq('docker')
+    expect(described_class.new(platform_with_docker_image).name).to eq('docker')
+  end
+
+  describe '#dispatch' do
+    context 'when platform has use_docker_exec set' do
+      subject { described_class.new(platform_with_docker_exec) }
+
+      it 'uses docker exec' do
+        expect(Vanagon::Utilities).to receive(:remote_ssh_command).never
+        expect(subject).to receive(:docker_exec)
+
+        subject.dispatch('true', true)
+      end
+    end
+  end
+
+  describe '#ship_workdir' do
+    context 'when platform has use_docker_exec set' do
+      subject { described_class.new(platform_with_docker_exec) }
+
+      it 'uses docker cp' do
+        expect(Vanagon::Utilities).to receive(:rsync_to).never
+        expect(subject).to receive(:docker_cp_globs_to)
+
+        subject.ship_workdir('foo/')
+      end
+    end
+  end
+
+  describe '#retrieve_built_artifact' do
+    context 'when platform has use_docker_exec set' do
+      subject { described_class.new(platform_with_docker_exec) }
+
+      before(:each) do
+        allow(FileUtils).to receive(:mkdir_p)
+      end
+
+      it 'uses docker cp' do
+        expect(Vanagon::Utilities).to receive(:rsync_from).never
+        expect(subject).to receive(:docker_cp_globs_from)
+
+        subject.retrieve_built_artifact('output/*', false)
+      end
+    end
   end
 end