right_publish 0.2.0

data/right_publish.gemspec

@@ -0,0 +1,101 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{right_publish}
+  s.version = "0.2.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Brian Szmyd", "Tony Spataro"]
+  s.date = %q{2013-07-03}
+  s.default_executable = %q{right_publish}
+  s.description = %q{A tool for maintaining S3-based DEB, GEM and RPM packages.}
+  s.email = %q{support@rightscale.com}
+  s.executables = ["right_publish"]
+  s.extra_rdoc_files = [
+    "README.rdoc"
+  ]
+  s.files = [
+    ".rspec",
+    "Gemfile",
+    "Gemfile.lock",
+    "README.rdoc",
+    "Rakefile",
+    "VERSION",
+    "bin/right_publish",
+    "lib/right_publish.rb",
+    "lib/right_publish/profile.rb",
+    "lib/right_publish/repo.rb",
+    "lib/right_publish/repos/apt.rb",
+    "lib/right_publish/repos/gem.rb",
+    "lib/right_publish/repos/yum.rb",
+    "lib/right_publish/storage.rb",
+    "lib/right_publish/stores/local.rb",
+    "lib/right_publish/stores/s3.rb",
+    "right_publish.gemspec",
+    "spec/repo_manager_spec.rb",
+    "spec/repo_spec.rb",
+    "spec/repos/apt_spec.rb",
+    "spec/repos/gem_spec.rb",
+    "spec/repos/yum_spec.rb",
+    "spec/spec_helper.rb",
+    "spec/storage_manager_spec.rb",
+    "spec/storage_spec.rb",
+    "spec/stores/local_spec.rb",
+    "spec/stores/s3_spec.rb"
+  ]
+  s.homepage = %q{https://github.com/rightscale/right_publish}
+  s.licenses = ["Proprietary"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.7}
+  s.summary = %q{Package publishing and indexing tool}
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<builder>, [">= 0"])
+      s.add_runtime_dependency(%q<fog>, ["~> 1.9"])
+      s.add_runtime_dependency(%q<trollop>, ["~> 2.0"])
+      s.add_development_dependency(%q<rake>, ["~> 0.9"])
+      s.add_development_dependency(%q<jeweler>, ["~> 1.8.3"])
+      s.add_development_dependency(%q<right_develop>, ["~> 1.0"])
+      s.add_development_dependency(%q<rdoc>, [">= 2.4.2"])
+      s.add_development_dependency(%q<rspec>, ["~> 2.0"])
+      s.add_development_dependency(%q<flexmock>, ["~> 0.9"])
+      s.add_development_dependency(%q<simplecov>, [">= 0"])
+      s.add_development_dependency(%q<ruby-debug>, [">= 0.10"])
+      s.add_development_dependency(%q<ruby-debug19>, [">= 0.11.6"])
+    else
+      s.add_dependency(%q<builder>, [">= 0"])
+      s.add_dependency(%q<fog>, ["~> 1.9"])
+      s.add_dependency(%q<trollop>, ["~> 2.0"])
+      s.add_dependency(%q<rake>, ["~> 0.9"])
+      s.add_dependency(%q<jeweler>, ["~> 1.8.3"])
+      s.add_dependency(%q<right_develop>, ["~> 1.0"])
+      s.add_dependency(%q<rdoc>, [">= 2.4.2"])
+      s.add_dependency(%q<rspec>, ["~> 2.0"])
+      s.add_dependency(%q<flexmock>, ["~> 0.9"])
+      s.add_dependency(%q<simplecov>, [">= 0"])
+      s.add_dependency(%q<ruby-debug>, [">= 0.10"])
+      s.add_dependency(%q<ruby-debug19>, [">= 0.11.6"])
+    end
+  else
+    s.add_dependency(%q<builder>, [">= 0"])
+    s.add_dependency(%q<fog>, ["~> 1.9"])
+    s.add_dependency(%q<trollop>, ["~> 2.0"])
+    s.add_dependency(%q<rake>, ["~> 0.9"])
+    s.add_dependency(%q<jeweler>, ["~> 1.8.3"])
+    s.add_dependency(%q<right_develop>, ["~> 1.0"])
+    s.add_dependency(%q<rdoc>, [">= 2.4.2"])
+    s.add_dependency(%q<rspec>, ["~> 2.0"])
+    s.add_dependency(%q<flexmock>, ["~> 0.9"])
+    s.add_dependency(%q<simplecov>, [">= 0"])
+    s.add_dependency(%q<ruby-debug>, [">= 0.10"])
+    s.add_dependency(%q<ruby-debug19>, [">= 0.11.6"])
+  end
+end
+

data/spec/repo_manager_spec.rb

@@ -0,0 +1,65 @@
+require 'right_publish/repo'
+require 'spec_helper'
+
+describe RightPublish::RepoManager do
+
+  def is_registered?(name, key, type)
+    @repo_types = RightPublish::RepoManager.repo_types
+    @repo_types.should have_key(name)
+    @repo_types[name].should equal(key)
+    RightPublish::RepoManager.get_repository(key).should be_instance_of(type)
+  end
+
+  it "should provide a hash of known repo types" do
+    RightPublish::RepoManager.repo_types.should be_instance_of(Hash)
+  end
+
+  it "returns nil for unknown repo types" do
+    RightPublish::RepoManager.get_repository(:doesnotexist_repo).should be_equal(nil)
+  end
+
+  it "should know about the apt repository" do
+    is_registered?('apt', :apt_repo, RightPublish::AptRepo)
+  end
+
+  it "should know about the gem repository" do
+    is_registered?('gem', :gem_repo, RightPublish::GemRepo)
+  end
+
+  it "should know about the yum repository" do
+    is_registered?('yum', :yum_repo, RightPublish::YumRepo)
+  end
+
+  context ".register_repo" do
+    it "rejects repos without a REPO_KEY" do
+      class MissingKey
+      end
+      expect { RightPublish::RepoManager.register_repo(MissingKey) }.to raise_error(NameError)
+    end
+
+    it "rejects repos whose key does not end in _repo" do
+      class BadKey
+        REPO_KEY = :bad_key
+      end
+      expect { RightPublish::RepoManager.register_repo(BadKey) }.to raise_error(TypeError)
+    end
+
+    it "rejects repos who do not provide options" do
+      class NoOptions
+        REPO_KEY = :nooptions_repo
+      end
+      expect { RightPublish::RepoManager.register_repo(NoOptions) }.to raise_error(NameError)
+    end
+
+    it "accpets repos who are properly defined" do
+      class GoodRepo
+        include RightPublish::Repo
+        REPO_KEY = :good_repo
+        REPO_OPTIONS = {}
+      end
+      RightPublish::RepoManager.register_repo(GoodRepo).should be_equal(nil)
+      is_registered?('good', :good_repo, GoodRepo)
+    end
+  end
+
+end

data/spec/repo_spec.rb

@@ -0,0 +1,74 @@
+require 'right_publish/repo'
+require 'spec_helper'
+
+describe RightPublish::Repo do
+
+  class FakeRepo
+    include RightPublish::Repo
+    REPO_KEY = :fake_repo
+    REPO_OPTIONS = {}
+  end
+
+  module MockLocal
+    STORAGE_KEY = :mocklocal_storage
+    STORAGE_OPTIONS = { }
+
+    def self.get_directories()
+    end
+  end
+
+  module MockRemote
+    STORAGE_KEY = :mockremote_storage
+    STORAGE_OPTIONS = { }
+
+    def self.get_directories()
+    end
+  end
+
+  before(:all) do
+    RightPublish::StorageManager.register_storage(MockLocal)
+    RightPublish::StorageManager.register_storage(MockRemote)
+    RightPublish::RepoManager.register_repo(FakeRepo)
+  end
+
+  before(:each) do
+    flexmock(RightPublish::Profile) do |obj|
+      obj.should_receive(:log).and_return { |str,dbg|
+      }
+
+      obj.should_receive(:config).and_return {
+        { :verbose=>false,
+          :remote_storage=>{:provider=>'mockremote'},
+          :local_storage=>{:provider=>'mocklocal'},
+          :fake_repo=>{:subdir=>'fake/'}
+        }
+      }
+    end
+
+    @repo = RightPublish::RepoManager.get_repository(:fake_repo)
+  end
+
+  context "#fetch" do
+    it "fetches from remote storage to local" do
+      flexmock(RightPublish::Storage).should_receive(:sync_dirs).and_return do |src,dest,options|
+        src.should be_equal(MockRemote)
+        dest.should be_equal(MockLocal)
+        nil
+      end
+      @repo.fetch
+    end
+  end
+
+  context "#store" do
+    it "stores from local storage to remote" do
+      flexmock(RightPublish::Storage).should_receive(:sync_dirs).and_return do |src,dest,options|
+        src.should be_equal(MockLocal)
+        dest.should be_equal(MockRemote)
+        nil
+      end
+      @repo.store
+    end
+  end
+
+end
+

data/spec/repos/apt_spec.rb

@@ -0,0 +1,288 @@
+require 'right_publish/repo'
+require 'spec_helper'
+
+describe RightPublish::AptRepo do
+
+  before(:each) do
+    @cache_dir = 'somedir/'
+    @repo_subdir = 'fake/'
+
+    @repo = flexmock(RightPublish::RepoManager.get_repository(:apt_repo)) do |obj|
+      obj.should_receive(:fetch)
+      obj.should_receive(:store)
+      obj.should_receive(:do_in_subdir).and_return {|dir,block| block.call}
+      obj.should_receive(:prune_all)
+      obj.should_receive(:write_conf_file)
+    end
+
+    flexmock(RightPublish::Profile) do |obj|
+      obj.should_receive(:log).and_return { |str,dbg| }
+    end
+
+    flexmock(File) { |obj| obj.should_receive(:file?).and_return(true) }
+  end
+
+  context "#publish" do
+
+    # Test the Pool (not flat) repo profile type
+    [false, true].each do |should_sign|
+
+      context "Pool layout#{" (with signature)" if should_sign}" do
+
+        def build_expectations(pkgs, expected_dists)
+          pkgs = Array(pkgs)
+          expected_dists = Array(expected_dists)
+
+          pkgs.each do |pkg|
+            expected_dists.each do |dist|
+              @remove_expectations[dist] ||= []
+              @remove_expectations[dist] << RightPublish::AptRepo.pkg_parts(pkg)[:name]
+              
+              @install_expectations[dist] ||= []
+              @install_expectations[dist] << pkg
+            end
+          end
+
+
+
+          system_calls = (pkgs.size * expected_dists.size) * 2
+          @repo.should_receive(@system_func).times(system_calls).and_return do |cmd|
+            path, subcmd, distro, pkg = /reprepro\s+(?:--ask-passphrase )?(?:-C main )?-b\s+(.+)\s+(includedeb|includedsc|removesrc|remove)\s+([^\s]+)\s+([^\s]+)\s+2>&1 >\/dev\/null\Z/.match(cmd).captures
+            path.should be_eql(File.join(@cache_dir,@repo_subdir))
+
+            if @remove_expectations.has_key?(distro) && @remove_expectations[distro].include?(pkg)
+              subcmd.should include('remove') #can be remove or removesrc
+              @remove_expectations.should be_has_key(distro)
+              @remove_expectations[distro].delete(pkg)
+              @remove_expectations.delete(distro) if @remove_expectations[distro].empty?
+            else
+              subcmd.should be_eql((pkg.end_with?(RightPublish::AptRepo::SRC_EXTENSION) && 'includedsc') || 'includedeb')
+              @install_expectations.should be_has_key(distro)
+              @install_expectations[distro].delete(pkg)
+              @install_expectations.delete(distro) if @install_expectations[distro].empty?
+            end
+            0
+          end
+        end
+
+        before(:each) do
+          @apt_dists = ['quantal', 'precise', 'wheezy']
+          @install_expectations = {}
+          @remove_expectations = {}
+          @system_func = should_sign ? :shellout_with_password : :system
+          flexmock(RightPublish::Profile) do |obj|
+            obj.should_receive(:config).and_return {
+              { :verbose=>false,
+                :remote_storage=>{:provider=>'mockremote'},
+                :local_storage=>{:provider=>'mocklocal',:cache_dir=>@cache_dir},
+                :apt_repo=>{
+                  :auto=>true,
+                  :subdir=>@repo_subdir,
+                  :dists=>@apt_dists,
+                  :gpg_key_id=>should_sign,
+                  :gpg_password=>"fakepass"}
+              }
+            }
+          end
+        end
+
+        after(:each) do
+          @install_expectations.should be_empty
+          @remove_expectations.should be_empty
+        end
+
+        # Test for publish noarch packages (multicasting)
+        context "NoArch Packages" do
+          it "installs a single noarch deb file to all dists" do
+            deb_file = 'foo_1_all.deb'
+            build_expectations(deb_file,@apt_dists)
+            @repo.publish(deb_file, nil)
+          end
+
+          it "installs a single src dsc file to all dists" do
+            dsc_file = 'foo_1.dsc'
+            build_expectations(dsc_file,@apt_dists)
+            @repo.publish(dsc_file, nil)
+          end
+
+          it "installs a list of noarch deb files to all dists" do
+            deb_files = ['foo_1_all.deb', 'bar_1_all.deb']
+            build_expectations(deb_files,@apt_dists)
+            @repo.publish(deb_files, nil)
+          end
+
+          it "globs and installs a directory of noarch deb files to all dists" do
+            deb_files = ['foo_1_all.deb', 'bar_1_all.deb']
+            build_expectations(deb_files,@apt_dists)
+            flexmock(File) { |obj| obj.should_receive(:directory?).and_return(true) }
+            flexmock(Dir) { |obj| obj.should_receive(:glob).and_return(deb_files) }
+            @repo.publish('somedir/', nil)
+          end
+
+          it "fails with no files in a dir" do
+            flexmock(File) { |obj| obj.should_receive(:directory?).and_return(true) }
+            flexmock(Dir) { |obj| obj.should_receive(:glob).and_return([]) }
+            @repo.should_receive(@system_func).never
+            expect { @repo.publish('some_dir/', nil) }.to raise_error(RuntimeError)
+          end
+
+          it "fails with files missing a deb extenstion" do
+            deb_files = ['foo_1_all.deb', 'bar.all.rpm']
+            @repo.should_receive(@system_func).never
+            expect { @repo.publish(deb_files, nil) }.to raise_error(RuntimeError)
+          end
+
+          it "fails with a mix of noarch and binary debs" do
+            deb_files = ['foo_1_all.deb', 'bar_2_amd64.deb']
+            @repo.should_receive(@system_func).never
+            expect { @repo.publish(deb_files, nil) }.to raise_error(RuntimeError)
+          end
+        end # end NoArch context
+
+        context "Architecture Dependant Packages" do
+          before(:each) do
+            @target_distribution_str = 'quantal'
+          end
+
+          it "installs a single amd64 deb file to a dist" do
+            deb_file = 'foo_1_amd64.deb'
+            build_expectations(deb_file,@target_distribution_str)
+            @repo.publish(deb_file, @target_distribution_str)
+          end
+
+          it "installs a list of amd64 debs to a dist" do
+            deb_files = ['foo1.9_1.2~precise_amd64.deb', 'bar_1_amd64.deb']
+            build_expectations(deb_files,@target_distribution_str)
+            @repo.publish(deb_files, @target_distribution_str)
+          end
+
+          it "installs a directory mixed of amd64 and i386 debs to a dist" do
+            deb_files = ['foo_1_amd64.deb', 'bar_1_i386.deb']
+            build_expectations(deb_files,@target_distribution_str)
+            flexmock(File) { |obj| obj.should_receive(:directory?).and_return(true) }
+            flexmock(Dir) { |obj| obj.should_receive(:glob).and_return(deb_files) }
+            @repo.publish('some_dir/', @target_distribution_str)
+          end
+
+          it "fails with no files in a dir" do
+            flexmock(File) { |obj| obj.should_receive(:directory?).and_return(true) }
+            flexmock(Dir) { |obj| obj.should_receive(:glob).and_return([]) }
+            @repo.should_receive(@system_func).never
+            expect { @repo.publish('some_dir/', @target_distribution_str) }.to raise_error(RuntimeError)
+          end
+
+          it "fails with files missing a deb extension" do
+            deb_files = ['foo_1_amd64.deb', 'bar_1_i386.rpm']
+            @repo.should_receive(@system_func).never
+            expect { @repo.publish(deb_files, @target_distribution_str) }.to raise_error(RuntimeError)
+          end
+
+          it "fails with unknown architectures" do
+            deb_files = ['foo.ppc.deb', 'bar.ppc.deb']
+            @repo.should_receive(@system_func).never
+            expect { @repo.publish(deb_files, @target_distribution_str) }.to raise_error(RuntimeError)
+          end
+
+          it "fails when not provided with a target" do
+            deb_files = ['foo_1_amd64.deb', 'bar_1_i386.deb']
+            @repo.should_receive(@system_func).never
+            expect { @repo.publish(deb_files, nil) }.to raise_error(RuntimeError)
+          end
+        end # end Architecture Dependant context
+      end # end Pool context
+    end
+
+    # Test the Flat repo profile type
+    context "Flat layout" do
+      def build_expectations(pkgs)
+        pkgs = Array(pkgs)
+
+        pkgs.each do |pkg|
+          dist_path = File.join(@repo_subdir, (pkg.end_with?(RightPublish::AptRepo::BIN_EXTENSION) && 'binaries') || 'sources')
+
+          @install_expectations[dist_path] ||= []
+          @install_expectations[dist_path] << pkg
+        end
+        @repo.should_receive(:install_file).and_return do |pkg,dir|
+          @install_expectations.should be_has_key(dir)
+          @install_expectations[dir].delete(pkg)
+          @install_expectations.delete(dir) if @install_expectations[dir].empty?
+        end
+        @repo.should_receive(:system).twice.and_return(0)
+      end
+
+      before(:each) do
+        @install_expectations = {}
+        flexmock(RightPublish::Profile) do |obj|
+          obj.should_receive(:config).and_return {
+            { :verbose=>false,
+              :remote_storage=>{:provider=>'mockremote'},
+              :local_storage=>{:provider=>'mocklocal',:cache_dir=>@cache_dir},
+              :apt_repo=>{
+                :apt=>false,
+                :subdir=>@repo_subdir }
+            }
+          }
+        end
+      end
+
+      after(:each) do
+        @install_expectations.should be_empty
+      end
+
+      it "installs a single deb file to repo" do
+        deb_file = 'foo_1_all.deb'
+        build_expectations(deb_file)
+        @repo.publish(deb_file, nil)
+      end
+
+      it "installs a single src deb file to repo" do
+        dsc_file = 'foo_1.dsc'
+        src_file = 'foo_1.orig.tar.gz'
+        diff_file = 'foo_1.diff.gz'
+        all_files = [dsc_file, src_file, diff_file]
+        flexmock(Dir).should_receive(:glob).and_return([src_file])
+        build_expectations(all_files)
+        @repo.publish(dsc_file, nil)
+      end
+
+      it "installs a list of deb files to repo" do
+        deb_files = ['foo_1_amd64.deb', 'bar_1_amd64.deb']
+        build_expectations(deb_files)
+        @repo.publish(deb_files, nil)
+      end
+
+      it "globs and installs a directory of deb files to repo" do
+        deb_files = ['foo_1_all.deb', 'bar_1.dsc']
+        src_file = 'bar_1.orig.tar.gz'
+        diff_file = 'bar_1.diff.gz'
+        all_files = ([deb_files, src_file, diff_file]).flatten
+        build_expectations(all_files)
+        flexmock(File) { |obj| obj.should_receive(:directory?).and_return(true) }
+        flexmock(Dir).should_receive(:glob).and_return do |pattern|
+          (pattern.end_with?('orig.tar.gz') && [src_file]) || deb_files
+        end
+        @repo.publish('somedir/', nil)
+      end
+
+      it "succeeds with a mix of noarch and binary debs" do
+        deb_files = ['foo_1_all.deb', 'bar_1_amd64.deb']
+        build_expectations(deb_files)
+        @repo.publish(deb_files, nil)
+      end
+
+      it "fails with no files in a dir" do
+        flexmock(File) { |obj| obj.should_receive(:directory?).and_return(true) }
+        flexmock(Dir) { |obj| obj.should_receive(:glob).and_return([]) }
+        @repo.should_receive(:system).never
+        expect { @repo.publish('some_dir/', nil) }.to raise_error(RuntimeError)
+      end
+
+      it "fails with files missing a deb extenstion" do
+        deb_files = ['foo_1_i386.deb', 'bar.all.rpm']
+        @repo.should_receive(:system).never
+        expect { @repo.publish('some_dir/', nil) }.to raise_error(RuntimeError)
+      end
+    end # end Trivial context
+  end # end #publish context
+end

data/spec/repos/gem_spec.rb

@@ -0,0 +1,79 @@
+require 'right_publish/repo'
+require 'spec_helper'
+
+describe RightPublish::GemRepo do
+
+  before(:each) do
+    @install_expectations = []
+    @repo_subdir = 'fake/'
+    @repo = flexmock(RightPublish::RepoManager.get_repository(:gem_repo)) do |obj|
+      obj.should_receive(:fetch)
+      obj.should_receive(:store)
+      obj.should_receive(:do_in_subdir).and_return {|dir,block| block.call}
+    end
+
+    flexmock(File) { |obj| obj.should_receive(:file?).and_return(true) }
+
+    flexmock(RightPublish::Profile) do |obj|
+      obj.should_receive(:log).and_return { |str,dbg|
+      }
+
+      obj.should_receive(:config).and_return {
+        { :verbose=>false,
+          :remote_storage=>{:provider=>'mockremote'},
+          :local_storage=>{:provider=>'mocklocal'},
+          :gem_repo=>{:subdir=>@repo_subdir}
+        }
+      }
+    end
+  end
+
+  after(:each) do
+    @install_expectations.should be_empty
+  end
+
+  context "#publish" do
+    def build_expectations(pkgs)
+      @install_expectations = Array(pkgs)
+      @repo.should_receive(:install_file).and_return do |pkg,dir|
+        @install_expectations.delete(pkg)
+        dir.should be_eql(File.join(@repo_subdir, RightPublish::GemRepo::GEMS_SUBDIRECTORY ))
+      end
+      @repo.should_receive(:system).once.and_return(0)
+    end
+
+    it "should install a single gem file" do
+      gem_file = 'foo-1.gem'
+      build_expectations(gem_file)
+      @repo.publish(gem_file, nil)
+    end
+
+    it "should install a list of gem files" do
+      gem_files = ['foo-1.gem', 'bar-2.0.0.gem']
+      build_expectations(gem_files)
+      @repo.publish(gem_files, nil)
+    end
+
+    it "should glob a directory of gem files and install them" do
+      gem_files = ['foo-1.gem', 'bar_game-2.0.gem']
+      build_expectations(gem_files)
+      flexmock(File) { |obj| obj.should_receive(:directory?).and_return(true) }
+      flexmock(Dir) { |obj| obj.should_receive(:glob).and_return(gem_files) }
+      @repo.publish('some_dir/', nil)
+    end
+
+    it "should fail with no files in a dir" do
+      flexmock(File) { |obj| obj.should_receive(:directory?).and_return(true) }
+      flexmock(Dir) { |obj| obj.should_receive(:glob).and_return([]) }
+      @repo.should_receive(:system).never
+      expect { @repo.publish('some_dir/', nil) }.to raise_error(RuntimeError)
+    end
+
+    it "should fail with files without a gem extenstion" do
+      gem_files = ['foo-1.gem', 'bar-2.rpm']
+      @repo.should_receive(:system).never
+      expect { @repo.publish(gem_files, nil) }.to raise_error(RuntimeError)
+    end
+  end
+end
+