packaging 0.99.0

data/spec/lib/packaging/repo_spec.rb

@@ -0,0 +1,97 @@
+# -*- ruby -*-
+require 'spec_helper'
+describe "#Pkg::Repo" do
+  let(:platform_repo_stub) do
+    [
+      {"name"=>"el-4-i386", "repo_location"=>"repos/el/4/**/i386"},
+      {"name"=>"el-5-i386", "repo_location"=>"repos/el/5/**/i386"},
+      {"name"=>"el-6-i386", "repo_location"=>"repos/el/6/**/i386"}
+    ]
+  end
+  describe "#create_signed_repo_archive" do
+    it "should change to the correct dir" do
+      allow(Pkg::Util::Tool).to receive(:check_tool).and_return("tarcommand")
+      allow(Pkg::Config).to receive(:project).and_return("project")
+      allow(Pkg::Util::Version).to receive(:dot_version).and_return("1.1.1")
+      allow(Pkg::Util::File).to receive(:empty_dir?).and_return(false)
+      allow(Pkg::Util::Execution).to receive(:capture3)
+
+      expect(Dir).to receive(:chdir).with("pkg").and_yield
+      expect(Dir).to receive(:chdir).with("project/1.1.1").and_yield
+      Pkg::Repo.create_signed_repo_archive("/path", "project-debian-6-i386", "version")
+    end
+
+    it "should use a ref if ref is specified as versioning" do
+      allow(Pkg::Util::Tool).to receive(:check_tool).and_return("tarcommand")
+      allow(Dir).to receive(:chdir).with("pkg").and_yield
+      allow(Pkg::Util::File).to receive(:empty_dir?).and_return(false)
+      allow(Pkg::Util::Execution).to receive(:capture3)
+
+      expect(Pkg::Config).to receive(:project).and_return("project")
+      expect(Pkg::Config).to receive(:ref).and_return("AAAAAAAAAAAAAAA")
+      expect(Dir).to receive(:chdir).with("project/AAAAAAAAAAAAAAA").and_yield
+      Pkg::Repo.create_signed_repo_archive("/path", "project-debian-6-i386", "ref")
+    end
+
+    it "should use dot versions if version is specified as versioning" do
+      allow(Pkg::Util::Tool).to receive(:check_tool).and_return("tarcommand")
+      allow(Dir).to receive(:chdir).with("pkg").and_yield
+      allow(Pkg::Util::File).to receive(:empty_dir?).and_return(false)
+      allow(Pkg::Util::Execution).to receive(:capture3)
+
+      expect(Pkg::Config).to receive(:project).and_return("project")
+      expect(Pkg::Util::Version).to receive(:dot_version).and_return("1.1.1")
+      expect(Dir).to receive(:chdir).with("project/1.1.1").and_yield
+      Pkg::Repo.create_signed_repo_archive("/path", "project-debian-6-i386", "version")
+    end
+
+    it "should fail if ENV['FAIL_ON_MISSING_TARGET'] is true and empty_dir? is also true" do
+      allow(Pkg::Util::Tool).to receive(:check_tool).and_return("tarcommand")
+      allow(Pkg::Config).to receive(:project).and_return("project")
+      allow(Pkg::Util::Version).to receive(:dot_version).and_return("1.1.1")
+      allow(Pkg::Util::Execution).to receive(:capture3)
+      allow(Dir).to receive(:chdir).with("pkg").and_yield
+      allow(Dir).to receive(:chdir).with("project/1.1.1").and_yield
+      allow(Pkg::Util::File).to receive(:empty_dir?).and_return(true)
+      ENV['FAIL_ON_MISSING_TARGET'] = "true"
+
+      expect{Pkg::Repo.create_signed_repo_archive("/path", "project-debian-6-i386", "version")}.to raise_error(RuntimeError, "ERROR: missing packages under /path")
+    end
+
+    it "should only warn if ENV['FAIL_ON_MISSING_TARGET'] is false and empty_dir? is true" do
+      allow(Pkg::Util::Tool).to receive(:check_tool).and_return("tarcommand")
+      allow(Pkg::Config).to receive(:project).and_return("project")
+      allow(Pkg::Util::Version).to receive(:dot_version).and_return("1.1.1")
+      allow(Pkg::Util::Execution).to receive(:capture3)
+      allow(Dir).to receive(:chdir).with("pkg").and_yield
+      allow(Dir).to receive(:chdir).with("project/1.1.1").and_yield
+      allow(Pkg::Util::File).to receive(:empty_dir?).and_return(true)
+      ENV['FAIL_ON_MISSING_TARGET'] = "false"
+
+      expect{Pkg::Repo.create_signed_repo_archive("/path", "project-debian-6-i386", "version")}.not_to raise_error
+    end
+
+    it "should invoke tar correctly" do
+      allow(Pkg::Util::Tool).to receive(:check_tool).and_return("tarcommand")
+      allow(Pkg::Config).to receive(:project).and_return("project")
+      allow(Pkg::Util::Version).to receive(:dot_version).and_return("1.1.1")
+      allow(Dir).to receive(:chdir).with("pkg").and_yield
+      allow(Dir).to receive(:chdir).with("project/1.1.1").and_yield
+      allow(Pkg::Util::File).to receive(:empty_dir?).and_return(false)
+
+      expect(Pkg::Util::Execution).to receive(:capture3).with("tarcommand --owner=0 --group=0 --create --gzip --file repos/project-debian-6-i386.tar.gz /path")
+      Pkg::Repo.create_signed_repo_archive("/path", "project-debian-6-i386", "version")
+    end
+  end
+
+  describe "#create_signed_repo_archive" do
+    it "should invoke create_signed_repo_archive correctly for multiple entries in platform_repos" do
+      allow(Pkg::Config).to receive(:platform_repos).and_return(platform_repo_stub)
+
+      expect(Pkg::Repo).to receive(:create_signed_repo_archive).with("repos/el/4/**/i386", "project-el-4-i386", "version")
+      expect(Pkg::Repo).to receive(:create_signed_repo_archive).with("repos/el/5/**/i386", "project-el-5-i386", "version")
+      expect(Pkg::Repo).to receive(:create_signed_repo_archive).with("repos/el/6/**/i386", "project-el-6-i386", "version")
+      Pkg::Repo.create_all_repo_archives("project", "version")
+    end
+  end
+end

data/spec/lib/packaging/retrieve_spec.rb

@@ -0,0 +1,61 @@
+require 'spec_helper'
+require 'packaging/retrieve'
+require 'packaging/paths'
+
+describe 'Pkg::Retrieve' do
+  local_target = 'pkg'
+  remote_target = 'repos'
+  project = 'puppet-agent'
+  ref = 'b25e64984dd505391f248fe5501ad81e2645b6d2'
+  foss_platforms = ['el-7-x86_64', 'ubuntu-16.04-amd64', 'osx-10.11-x86_64', 'windows-2012-x64']
+  platform_data = {:platform_data => {
+    'aix-6.1-power' => {:artifact => './aix/6.1/PC1/ppc/puppet-agent-5.3.2.155.gb25e649-1.aix6.1.ppc.rpm'},
+    'el-7-x86_64' => {:artifact => './el/7/PC1/x86_64/puppet-agent-5.3.2.155.gb25e649-1.el7.x86_64.rpm'},
+    'osx-10.11-x86_64' => {:artifact => './apple/10.11/PC1/x86_64/puppet-agent-5.3.2.155.gb25e649-1.osx10.11.dmg'},
+    'sles-11-s390x' => {:artifact => './sles/11/PC1/s390x/puppet-agent-5.3.2.155.gb25e649-1.sles11.s390x.rpm'},
+    'solaris-10-sparc' => {:artifact => './solaris/10/PC1/puppet-agent-5.3.2.155.gb25e649-1.sparc.pkg.gz'},
+    'ubuntu-16.04-amd64' => {:artifact => './deb/xenial/PC1/puppet-agent_5.3.2.155.gb25e649-1xenial_amd64.deb'},
+    'windows-2012-x64' => {:artifact => './windows/puppet-agent-x64.msi'},
+  }}
+  build_url = "builds.delivery.puppetlabs.net/#{project}/#{ref}"
+  build_path = "/opt/jenkins-builds/#{project}/#{ref}"
+
+  before :each do
+    allow(Pkg::Config).to receive(:project).and_return(project)
+    allow(Pkg::Config).to receive(:ref).and_return(ref)
+    allow(Pkg::Config).to receive(:foss_platforms).and_return(foss_platforms)
+    allow(File).to receive(:readable?).with("#{local_target}/#{ref}.yaml").and_return(true)
+    allow(Pkg::Util::Serialization).to receive(:load_yaml).and_return(platform_data)
+  end
+
+  describe '#foss_only_retrieve' do
+    it 'should fail without foss_platforms' do
+      allow(Pkg::Config).to receive(:foss_platforms).and_return(nil)
+      expect { Pkg::Retrieve.foss_only_retrieve(build_url, local_target) }.to raise_error(/I don't know anything about FOSS_PLATFORMS/)
+    end
+
+    it 'should fail if cannot read <ref>.yaml' do
+      allow(File).to receive(:readable?).with("#{local_target}/#{ref}.yaml").and_return(false)
+      expect { Pkg::Retrieve.foss_only_retrieve(build_url, local_target) }.to raise_error(/Couldn't read #{ref}.yaml/)
+    end
+
+    it 'should retrieve foss_only packages' do
+      expect(Pkg::Retrieve).to receive(:default_wget).exactly(1 + foss_platforms.count).times
+      Pkg::Retrieve.foss_only_retrieve(build_url, local_target)
+    end
+  end
+
+  describe '#retrieve_all' do
+    it 'should try to use wget first' do
+      expect(Pkg::Retrieve).to receive(:default_wget)
+      Pkg::Retrieve.retrieve_all(build_url, build_path, remote_target, local_target)
+    end
+
+    it 'should use rsync if wget is not found' do
+      allow(Pkg::Util::Tool).to receive(:find_tool).with('wget').and_return(nil)
+      expect(Pkg::Util::Net).to receive(:rsync_from)
+      Pkg::Retrieve.retrieve_all(build_url, build_path, remote_target, local_target)
+    end
+  end
+end
+

data/spec/lib/packaging/rpm/repo_spec.rb

@@ -0,0 +1,133 @@
+require 'spec_helper'
+
+describe "Pkg::Rpm::Repo" do
+  let(:wget)          { "/opt/tools/bin/wget" }
+  let(:builds_server) { "saturn.puppetlabs.net" }
+  let(:project)       { "rpm_repos" }
+  let(:ref)           { "1234abcd" }
+  let(:base_url)      { "http://#{builds_server}/#{project}/#{ref}" }
+  let(:mocks)         { ["el-5-i386", "el-5-x86_64", "el-5-SRPMS"] }
+  let(:wget_results)  {
+                        mocks.map do |mock|
+                          dist, version, arch = mock.split('-')
+                          "http://#{builds_server}/#{project}/#{ref}/repos/#{dist}/#{version}/products/#{arch}/repodata/"
+                        end.join("\n")
+                      }
+  let(:wget_garbage)  { "\nother things\n and an index\nhttp://somethingelse.com" }
+  let(:repo_configs)  { mocks.map { |mock| "pkg/repo_configs/rpm/pl-#{project}-#{ref}-#{mock}.repo" } }
+
+  # Setup and tear down for the tests
+  around do |example|
+    orig_server = Pkg::Config.builds_server
+    orig_host = Pkg::Config.distribution_server
+    orig_project = Pkg::Config.project
+    orig_ref = Pkg::Config.ref
+    orig_repo_path = Pkg::Config.jenkins_repo_path
+    Pkg::Config.builds_server = builds_server
+    Pkg::Config.project = project
+    Pkg::Config.ref = ref
+    example.run
+    Pkg::Config.builds_server = orig_server
+    Pkg::Config.distribution_server = orig_host
+    Pkg::Config.project = orig_project
+    Pkg::Config.ref = orig_ref
+    Pkg::Config.jenkins_repo_path = orig_repo_path
+  end
+
+  describe "#generate_repo_configs" do
+    it "fails if wget isn't available" do
+      Pkg::Util::Tool.stub(:find_tool).with("wget", {:required => true}) {false}
+      expect {Pkg::Rpm::Repo.generate_repo_configs}.to raise_error(RuntimeError)
+    end
+
+    it "warns if there are no rpm repos available for the build" do
+      Pkg::Util::Tool.should_receive(:find_tool).with("wget", {:required => true}).and_return(wget)
+      Pkg::Util::Execution.should_receive(:capture3).with("#{wget} --spider -r -l 5 --no-parent #{base_url}/repos/ 2>&1").and_return("")
+      Pkg::Rpm::Repo.should_receive(:warn).with("No rpm repos were found to generate configs from!")
+      Pkg::Rpm::Repo.generate_repo_configs
+    end
+
+    it "writes the expected repo configs to disk" do
+      Pkg::Util::Tool.should_receive(:find_tool).with("wget", {:required => true}).and_return(wget)
+      Pkg::Util::Execution.should_receive(:capture3).with("#{wget} --spider -r -l 5 --no-parent #{base_url}/repos/ 2>&1").and_return(wget_results + wget_garbage)
+      wget_results.split.each do |result|
+        cur_result = result.chomp('repodata/')
+        Pkg::Util::Execution.should_receive(:capture3).with("#{wget} --spider -r -l 1 --no-parent #{cur_result} 2>&1").and_return("#{cur_result}/thing.rpm")
+      end
+      FileUtils.should_receive(:mkdir_p).with("pkg/repo_configs/rpm")
+      config = []
+      repo_configs.each_with_index do |repo_config, i|
+        Pkg::Paths.should_receive(:tag_from_artifact_path).and_return(mocks[i])
+        Pkg::Platforms.should_receive(:parse_platform_tag).and_return(mocks[i].split('-'))
+        config[i] = double(File)
+        File.should_receive(:open).with(repo_config, 'w').and_yield(config[i])
+        config[i].should_receive(:puts)
+      end
+      Pkg::Rpm::Repo.generate_repo_configs
+    end
+  end
+
+  describe "#retrieve_repo_configs" do
+    it "fails if wget isn't available" do
+      Pkg::Util::Tool.stub(:find_tool).with("wget", {:required => true}) {false}
+      expect {Pkg::Rpm::Repo.generate_repo_configs}.to raise_error(RuntimeError)
+    end
+
+    it "fails if there are no deb repos available for the build" do
+      Pkg::Util::Tool.should_receive(:find_tool).with("wget", {:required => true}).and_return(wget)
+      FileUtils.should_receive(:mkdir_p).with("pkg/repo_configs").and_return(true)
+      Pkg::Util::Execution.should_receive(:capture3).with("#{wget} -r -np -nH --cut-dirs 3 -P pkg/repo_configs --reject 'index*' #{base_url}/repo_configs/rpm/").and_raise(RuntimeError)
+      expect {Pkg::Rpm::Repo.retrieve_repo_configs}.to raise_error(RuntimeError, /Couldn't retrieve rpm yum repo configs/)
+    end
+  end
+
+  describe "#create_local_repos" do
+    let(:command) { "/usr/bin/make some repos" }
+    let(:target_directory) { "/tmp/dir/thing" }
+
+    it "makes a repo in the target directory" do
+      Pkg::Rpm::Repo.should_receive(:repo_creation_command).with(target_directory).and_return("run this thing")
+      Pkg::Util::Execution.should_receive(:capture3).with("bash -c 'run this thing'")
+      Pkg::Rpm::Repo.create_local_repos(target_directory)
+    end
+  end
+
+  describe "#create_remote_repos" do
+    let(:command) { "/usr/bin/make some repos" }
+    let(:artifact_directory) { "/tmp/dir/thing" }
+    let(:pkg_directories) { ['el-6-i386', 'el/7/x86_64'] }
+
+    it "makes a repo in the target directory" do
+      File.stub(:join) {artifact_directory}
+      Pkg::Repo.should_receive(:directories_that_contain_packages).and_return(pkg_directories)
+      Pkg::Repo.should_receive(:populate_repo_directory)
+      Pkg::Rpm::Repo.should_receive(:repo_creation_command).and_return(command)
+      Pkg::Util::Net.should_receive(:remote_ssh_cmd).with(Pkg::Config.distribution_server, command)
+      Pkg::Rpm::Repo.should_receive(:generate_repo_configs)
+      Pkg::Rpm::Repo.should_receive(:ship_repo_configs)
+      Pkg::Util::Net.should_receive(:remote_ssh_cmd).with(Pkg::Config.distribution_server, "rm -f #{artifact_directory}/repos/.lock" )
+      Pkg::Rpm::Repo.create_remote_repos
+    end
+  end
+
+  describe "#ship_repo_configs" do
+    it "warn if there are no repo configs to ship" do
+      Pkg::Util::File.should_receive(:empty_dir?).with("pkg/repo_configs/rpm").and_return(true)
+      Pkg::Rpm::Repo.should_receive(:warn).with("No repo configs have been generated! Try pl:rpm_repo_configs.")
+      Pkg::Rpm::Repo.ship_repo_configs
+    end
+
+    it "ships repo configs to the build server" do
+      Pkg::Config.jenkins_repo_path = "/a/b/c/d"
+      Pkg::Config.project = "thing2"
+      Pkg::Config.ref = "abcd1234"
+      Pkg::Config.distribution_server = "a.host.that.wont.exist"
+      repo_dir = "#{Pkg::Config.jenkins_repo_path}/#{Pkg::Config.project}/#{Pkg::Config.ref}/repo_configs/rpm"
+      Pkg::Util::File.should_receive(:empty_dir?).with("pkg/repo_configs/rpm").and_return(false)
+      Pkg::Rpm::Repo.should_receive(:invoke_task).with("pl:fetch")
+      Pkg::Util::Net.should_receive(:remote_ssh_cmd).with(Pkg::Config.distribution_server, "mkdir -p #{repo_dir}")
+      Pkg::Util::Execution.should_receive(:retry_on_fail).with(:times => 3)
+      Pkg::Rpm::Repo.ship_repo_configs
+    end
+  end
+end

data/spec/lib/packaging/tar_spec.rb

@@ -0,0 +1,122 @@
+# -*- ruby -*-
+require 'spec_helper'
+
+describe "tar.rb" do
+  let(:project) { "packaging" }
+  let(:version) { "1.2.3" }
+  let(:files)   { [ "a", "b", "c" ] }
+  let(:templates) do
+    [
+      "ext/redhat/spec.erb",
+      { "source" => "ext/debian/control.erb", "target" => "ext/debian/not-a-control-file" },
+      "ext/debian/changelog.erb",
+      "ext/packaging/thing.erb"
+    ]
+  end
+  let(:expanded_templates) do
+    [
+      "#{PROJECT_ROOT}/ext/redhat/spec.erb",
+      { "source" => "ext/debian/control.erb", "target" => "ext/debian/not-a-control-file" },
+      "#{PROJECT_ROOT}/ext/debian/changelog.erb"
+    ]
+  end
+
+  describe "#expand_templates" do
+    it "should be invoked when Pkg::Config.templates is set" do
+      Pkg::Config.config_from_hash(
+        {
+          :templates      => templates,
+          :project        => project,
+          :version        => version,
+          :files          => files,
+          :project_root   => PROJECT_ROOT,
+          :packaging_root => "ext/packaging"
+        })
+
+      Pkg::Tar.any_instance.should_receive(:expand_templates)
+      Pkg::Tar.new
+    end
+
+    it "packaging templates should be filtered and paths should be expanded" do
+      Pkg::Config.config_from_hash(
+        {
+          :templates      => templates,
+          :project        => project,
+          :version        => version,
+          :files          => files,
+          :project_root   => PROJECT_ROOT,
+          :packaging_root => "ext/packaging"
+        })
+
+      templates.each do |temp|
+        if temp.is_a?(String)
+          Dir.stub(:glob).with(File.join(PROJECT_ROOT, temp)).and_return(File.join(PROJECT_ROOT, temp))
+        end
+      end
+
+      tar = Pkg::Tar.new
+      tar.templates = templates
+      tar.expand_templates
+      tar.templates.should eq expanded_templates
+    end
+  end
+
+  describe "#template" do
+    it "should handle hashes and strings correctly" do
+      Pkg::Config.config_from_hash(
+        {
+          :templates      => expanded_templates,
+          :project        => project,
+          :version        => version,
+          :files          => files,
+          :project_root   => PROJECT_ROOT,
+          :packaging_root => "ext/packaging"
+        })
+
+      # Set up correct stubs and expectations
+      expanded_templates.each do |temp|
+        if temp.is_a?(String)
+          full_path_temp = File.join(PROJECT_ROOT, temp)
+          target = full_path_temp.sub(File.extname(full_path_temp), "")
+        elsif temp.is_a?(Hash)
+          full_path_temp = File.join(PROJECT_ROOT, temp["source"])
+          target = File.join(PROJECT_ROOT, temp["target"])
+        end
+
+        Dir.stub(:glob).with(full_path_temp).and_return(full_path_temp)
+        File.stub(:exist?).with(full_path_temp).and_return(true)
+        Pkg::Util::File.should_receive(:erb_file).with(full_path_temp, target, true, :binding => an_instance_of(Binding))
+      end
+
+      Pkg::Tar.new.template
+    end
+
+    it "should raise an error if the template source can't be found" do
+     Pkg::Config.config_from_hash(
+        {
+          :templates      => expanded_templates,
+          :project        => project,
+          :version        => version,
+          :files          => files,
+          :project_root   => PROJECT_ROOT,
+          :packaging_root => "ext/packaging"
+        })
+
+      # Set up correct stubs and expectations
+      expanded_templates.each do |temp|
+        if temp.is_a?(String)
+          full_path_temp = File.join(PROJECT_ROOT, temp)
+          target = full_path_temp.sub(File.extname(full_path_temp), "")
+        elsif temp.is_a?(Hash)
+          full_path_temp = File.join(PROJECT_ROOT, temp["source"])
+          target = File.join(PROJECT_ROOT, temp["target"])
+        end
+
+        Dir.stub(:glob).with(full_path_temp).and_return(full_path_temp)
+        File.stub(:exist?).with(full_path_temp).and_return(false)
+      end
+
+      expect { Pkg::Tar.new.template }.to raise_error RuntimeError
+    end
+  end
+end

data/spec/lib/packaging/util/execution_spec.rb

@@ -0,0 +1,56 @@
+require 'spec_helper'
+
+describe "Pkg::Util::Execution" do
+  let(:command)    { "/usr/bin/do-something-important --arg1=thing2" }
+  let(:output)     { "the command returns some really cool stuff that may be useful later" }
+
+  describe "#success?" do
+    it "should return false on failure" do
+      %x{false}
+      Pkg::Util::Execution.success?.should be_false
+    end
+
+    it "should return true on success" do
+      %x{true}
+      Pkg::Util::Execution.success?.should be_true
+    end
+
+    it "should return false when passed an exitstatus object from a failure" do
+      %x{false}
+      Pkg::Util::Execution.success?($?).should be_false
+    end
+
+    it "should return true when passed and exitstatus object from a success" do
+      %x{true}
+      Pkg::Util::Execution.success?($?).should be_true
+    end
+  end
+
+  describe "#ex" do
+    it "should raise an error if the command fails" do
+      Pkg::Util::Execution.should_receive(:`).with(command).and_return(true)
+      Pkg::Util::Execution.should_receive(:success?).and_return(false)
+      expect{ Pkg::Util::Execution.ex(command) }.to raise_error(RuntimeError)
+    end
+
+    it "should return the output of the command for success" do
+      Pkg::Util::Execution.should_receive(:`).with(command).and_return(output)
+      Pkg::Util::Execution.should_receive(:success?).and_return(true)
+      Pkg::Util::Execution.ex(command).should == output
+    end
+  end
+
+  describe "#capture3" do
+    it "should raise an error if the command fails" do
+      Open3.should_receive(:capture3).with(command).and_return([output, '', 1])
+      Pkg::Util::Execution.should_receive(:success?).with(1).and_return(false)
+      expect{ Pkg::Util::Execution.capture3(command) }.to raise_error(RuntimeError, /#{output}/)
+    end
+
+    it "should return the output of the command for success" do
+      Open3.should_receive(:capture3).with(command).and_return([output, '', 0])
+      Pkg::Util::Execution.should_receive(:success?).with(0).and_return(true)
+      Pkg::Util::Execution.capture3(command).should == [output, '', 0]
+    end
+  end
+end