right_publish 0.2.0

data/lib/right_publish/repos/apt.rb

@@ -0,0 +1,178 @@
+module RightPublish
+  class AptRepo
+    include RightPublish::Repo
+
+    DEFAULT_APT_AUTO = true
+    DEFAULT_APT_DIR = 'apt/'
+    DEFAULT_DESCRIPTION = "RightScale RightLink Repository"
+    REPO_KEY = :apt_repo
+
+    REPO_OPTIONS = {
+      :dists=>:addr_optional, 
+      :description => DEFAULT_DESCRIPTION, 
+      :auto=>DEFAULT_APT_AUTO, 
+      :subdir=>DEFAULT_APT_DIR, 
+      :gpg_key_id       => :attr_optional,
+      :gpg_password     => :attr_optional }
+
+    BIN_EXTENSION = 'deb'
+    SRC_EXTENSION = 'dsc'
+    BIN_ALL_ARCH = 'all'
+
+    @@supported_archs = [ 'i386', 'amd64' ]
+    @@all_archs = [ 'i386', 'amd64', BIN_ALL_ARCH ]
+
+    def publish(file_or_dir, target)
+      repo_updates = {}
+
+      # If we specified a target, let's make sure it's in our profile
+      if target
+        fail("specified distribution is not listed in this profile!") unless repo_config[:dists].include?(target)
+      end
+
+      pkgs = []
+      get_pkg_list(file_or_dir, [BIN_EXTENSION, SRC_EXTENSION]) do |path|
+        if path.end_with? BIN_EXTENSION and repo_config[:auto]
+          arch = AptRepo.pkg_parts(path)[:arch] || fail("could not determine architecture for package: #{path}")
+          fail("you probably want to specify a distribution target for binary packages!") unless target || arch == BIN_ALL_ARCH
+        end
+        pkgs << path
+      end
+
+      # Synchronize the upstream source to our local cache
+      fetch
+
+      repo_path = File.join(Profile.config[:local_storage][:cache_dir], repo_config[:subdir])
+      write_conf_file repo_path if repo_config[:auto]
+
+      pkgs.each do |pkg|
+        Profile.log("Adding package [#{pkg}]...")
+        if repo_config[:auto]
+          # If source package, remove existing sources so we don't get a hash clash
+          auto_repo_remove(repo_path, pkg, target)
+          auto_repo_add(repo_path, pkg, target)
+        else
+          trivial_repo_add(pkg)
+        end
+      end
+      rebuild_index(repo_config[:subdir]) if not repo_config[:auto]
+
+      # Commit back to remote storage
+      store
+    end
+
+    def self.pkg_parts(pkg_name)
+      result = {:name=>nil, :version=>nil, :arch=>nil, :ext=>nil}
+      if pkg_name.end_with?(SRC_EXTENSION)
+        if /([A-Za-z0-9\.\-+]+)_([A-Za-z0-9\-\.:~]+)\.#{SRC_EXTENSION}\Z/.match(pkg_name)
+          result[:name] = $1
+          result[:version] = $2
+          result[:ext] = SRC_EXTENSION
+        end
+      else
+        if /([A-Za-z0-9\.\-+]+)_([A-Za-z0-9\-\.:~]+)_(#{@@all_archs.join('|')})\.#{BIN_EXTENSION}\Z/.match(pkg_name)
+          result[:name] = $1
+          result[:version] = $2
+          result[:arch] = $3
+          result[:ext] = BIN_EXTENSION
+        end
+      end
+      result
+    end
+
+    private
+
+    def auto_repo_add(repo_path, pkg, target)
+      targets = (target && Array(target)) || repo_config[:dists]
+      targets.each do |t|
+        sub_command = (pkg.end_with?(BIN_EXTENSION) && 'includedeb') || 'includedsc'
+        ask_passphrase = (repo_config[:gpg_key_id]) ? "--ask-passphrase " : ""
+        cmd = "reprepro #{ask_passphrase}-C main -b #{repo_path} #{sub_command} #{t} #{pkg} 2>&1 >/dev/null"
+        if repo_config[:gpg_key_id]
+          exited = shellout_with_password(cmd)
+        else
+          exited = system(cmd)
+        end
+
+        raise RuntimeError, "apt package installation failed; cannot continue publishing" unless exited
+      end
+    end
+
+    def auto_repo_remove(repo_path, pkg, target)
+      sub_command = (pkg.end_with?(BIN_EXTENSION) && 'remove') || 'removesrc'
+      pkg_name = AptRepo.pkg_parts(pkg)[:name]
+      Profile.log("Removing any existing files for #{pkg_name}")
+
+      targets = (target && Array(target)) || repo_config[:dists]
+      targets.each do |t|
+        ask_passphrase = (repo_config[:gpg_key_id]) ? "--ask-passphrase" : ""
+        cmd = "reprepro #{ask_passphrase} -b #{repo_path} #{sub_command} #{t} #{pkg_name} 2>&1 >/dev/null"
+        if repo_config[:gpg_key_id]
+          exited = shellout_with_password(cmd)
+        else
+          exited = system(cmd)
+        end
+      end
+    end
+
+    def rebuild_index(subdir)
+      Profile.log("Rebuilding repository index...")
+      do_in_subdir(subdir) do
+        indexed = system("dpkg-scanpackages binaries /dev/null 2>/dev/null | gzip -c9 > Packages.gz")
+        raise RuntimeError, "apt package installation failed; cannot continue publishing" unless indexed
+        indexed = system("dpkg-scansources sources /dev/null 2>/dev/null | gzip -c9 > Sources.gz")
+        raise RuntimeError, "apt package installation failed; cannot continue publishing" unless indexed
+      end
+    end
+
+    def trivial_repo_add(pkg)
+      pkg_info = AptRepo.pkg_parts(pkg)
+
+      if pkg_info[:ext].eql?(BIN_EXTENSION)
+        sub_dir = File.join(repo_config[:subdir], 'binaries')
+      else
+        pkg_name = pkg_info[:name]
+        src_file = Dir.glob("#{File.dirname(pkg)}/#{pkg_name}_*.orig.tar.gz")
+        raise RuntimeError, "could not find original source for #{pkg}, missing or ambiguous." unless src_file.size == 1
+
+        diff_file = pkg.sub(/#{SRC_EXTENSION}$/, 'diff.gz')
+        sub_dir = File.join(repo_config[:subdir], 'sources')
+        raise RuntimeError, "missing the debian diff file for #{pkg}." unless File.file?(diff_file)
+
+        do_in_subdir(sub_dir) { prune_all("#{pkg_info[:name]}*.orig.tar.gz") }
+        install_file(src_file[0], sub_dir)
+
+        do_in_subdir(sub_dir) { prune_all("#{pkg_info[:name]}*.diff.gz") }
+        install_file(diff_file, sub_dir)
+      end
+
+      do_in_subdir(sub_dir) { prune_all("#{pkg_info[:name]}_*#{pkg_info[:arch]}.#{pkg_info[:ext]}") }
+      install_file(pkg, sub_dir)
+    end
+
+    def write_conf_file(repo_path)
+      destination_dir = File.expand_path(File.join(repo_path, 'conf'))
+
+      FileUtils.mkdir_p destination_dir
+      config_file = open( File.join(destination_dir, 'distributions'), 'wb' )
+
+      repo_config[:dists].each { |dist| config_file << dist_conf(dist) }
+      config_file.close
+    end
+
+    def dist_conf(dist)
+      <<EOF
+Origin: RightScale
+Codename: #{dist}
+Version: 1.0
+Architectures: #{@@supported_archs.join(' ')} source
+Components: main
+Description: #{repo_config[:description]}
+#{(repo_config[:gpg_key_id] && "SignWith: #{repo_config[:gpg_key_id]}") || nil}
+
+EOF
+    end
+  end
+
+  RightPublish::RepoManager.register_repo(AptRepo)
+end

data/lib/right_publish/repos/gem.rb

@@ -0,0 +1,44 @@
+module RightPublish
+  class GemRepo
+    include RightPublish::Repo
+
+    DEFAULT_GEM_DIR = 'gems/'
+    REPO_KEY = :gem_repo
+    REPO_OPTIONS = {:subdir=>DEFAULT_GEM_DIR}
+
+    GEMS_SUBDIRECTORY = 'gems'
+    GEM_EXT = 'gem'
+
+    def publish(file_or_dir, target)
+      gems_list = get_pkg_list(file_or_dir, GEM_EXT)
+
+      # Synchronize the upstream source to our local cache
+      fetch
+
+      # Copy gem files to the repository
+      destination = File.join(repo_config[:subdir], GEMS_SUBDIRECTORY)
+      gems_list.each do |path|
+        do_in_subdir(destination) { prune_all("#{pkg_parts(path)[:name]}-*.gem") }
+        install_file(path, destination)
+      end
+
+      # Rebuild the gem index
+      Profile.log("Rebuilding Gem Index...")
+      do_in_subdir(repo_config[:subdir]) do
+        indexed = system('gem generate_index 2>&1 >/dev/null')  # requires 'builder' gem to be installed
+        raise Exception, "gem generate_index failed; cannot continue publishing" unless indexed
+      end
+
+      # Commit back to remote storage
+      store
+    end
+
+    def pkg_parts(name)
+      result = {:name=>nil, :version=>nil}
+      result[:name], result[:version] = /([A-Za-z0-9\-_]+)-([0-9\.]+)\.#{GEM_EXT}\Z/.match(name).captures
+      result
+    end
+  end
+
+  RightPublish::RepoManager.register_repo(GemRepo)
+end

data/lib/right_publish/repos/yum.rb

@@ -0,0 +1,167 @@
+
+
+module RightPublish
+
+  class YumRepo
+    include RightPublish::Repo
+
+    DEFAULT_YUM_EPEL = 1
+    DEFAULT_YUM_DIR = 'yum/'
+    DEFAULT_DESCRIPTION = "RightScale RightLink Repository"
+    REPO_KEY = :yum_repo
+    DEFAULT_EPEL_LAYOUT = "rightlink"
+    REPO_OPTIONS = {
+      :dists            => :attr_optional,
+      :epel             => DEFAULT_YUM_EPEL,
+      :epel_layout      => DEFAULT_EPEL_LAYOUT,
+      :subdir           => DEFAULT_YUM_DIR,
+      :description      => DEFAULT_DESCRIPTION,
+      :gpg_key_id       => :attr_optional,
+      :gpg_password     => :attr_optional }
+    YUM_EXT = 'rpm'
+
+    BIN_ALL_ARCH = 'noarch'
+    SRC_ALL_ARCH = 'src'
+    SRC_ALL_PATH = 'SRPMS'
+
+    ARCHITECTURES = [ 'i386', 'x86_64' ]
+    PKG_TYPES = [ ARCHITECTURES, BIN_ALL_ARCH, SRC_ALL_ARCH ]
+
+    def publish(file_or_dir, target)
+      repo_updates = {}
+
+      get_pkg_list(file_or_dir, YUM_EXT) do |path|
+        # Determine package architectures, and sort
+        appropriate_repos(path, target) do |repo_path|
+          repo_path = File.join(repo_config[:subdir], repo_path)
+          repo_updates[repo_path] ||= []
+          repo_updates[repo_path].push(path)
+        end
+      end
+      return if repo_updates.size == 0
+
+      # Synchronize the upstream source to our local cache
+      fetch
+
+      full_pkg_paths = []
+      repo_updates.each_pair do |repo_path, pkgs|
+        pkgs.each do |pkg|
+          import_pkg( pkg, repo_path )
+          full_pkg_paths << File.join(repo_path, File.basename(pkg))
+        end
+      end
+      sign_files full_pkg_paths
+
+      # Rebuild the yum index'
+      repo_updates.each_key do |repo_path|
+        regen_metadata( repo_path )
+      end
+
+      # Commit back to remote storage
+      store
+    end
+
+    def import_pkg( pkg, repo_path )
+      # Remove any instances of this package at a different versionh
+      yum_prune( pkg, repo_path )
+      install_file( pkg, repo_path )
+    end
+
+    def sign_files( pkgs )
+      if repo_config[:gpg_key_id]
+        do_in_subdir('') do
+
+          cmd = "rpm --define '%_gpg_name #{repo_config[:gpg_key_id]}' --addsign #{pkgs.join(' ')} 2>&1 >/dev/null"
+          exited = shellout_with_password(cmd)
+
+          raise Exception, "rpm signing failed; cannot continue publishing" unless exited
+        end
+      end
+    end
+
+    def pkg_parts(path)
+      @infocache ||= {}
+      return @infocache[path] if @infocache.has_key?(path)
+
+      result = {:name=>nil, :version=>nil, :arch=>nil, :release=>nil}
+
+      query = IO.popen("rpm --queryformat '%{NAME} %{VERSION} %{RELEASE} %{ARCH}' -qp #{path}")
+      result[:name], result[:version], result[:release], result[:arch] = query.readline.split(' ')
+
+      # RPM query doesn't identify package as a source package, we just
+      # have to test the filename.
+      result[:arch] = SRC_ALL_ARCH if path.end_with?('src.rpm')
+
+      @infocache[path] = result
+      result
+    end
+
+    private
+
+    def repo_dir(epel_ver, dist, dist_ver, arch)
+      if repo_config[:epel_layout] == "rightscale-software"
+        File.join(dist_ver, arch)
+      else
+        File.join(epel_ver.to_s, dist, dist_ver, arch)
+      end
+    end
+
+    def appropriate_repos(path, target)
+      unless repo_config[:epel]
+        # This is a flat repo
+        yield ''
+        return
+      end
+
+      pkg_arch = pkg_parts(path)[:arch] || fail("could not determine architecture for package: #{path}")
+
+      if target
+        #TODO Check that the specified dist is configured in the profile (sanity)
+        dist, dist_ver = target.split("/")
+        fail("dist format needs to be in form 'dist/dist_version'!") unless dist && dist_ver
+        dists = {dist => [dist_ver]}
+      else
+        unless [BIN_ALL_ARCH,SRC_ALL_ARCH].include? pkg_arch
+          fail("need to specify a distribution with binary packages!")
+        end
+        dists = repo_config[:dists]
+      end
+ 
+      dists.each_pair do |dist,dist_vers|
+        dist_vers.each do |ver|
+          # Source RPMS stored in SRPMS architecture, noarchs go to all architectures
+          case pkg_arch
+          when SRC_ALL_ARCH
+            [SRC_ALL_PATH]
+          when BIN_ALL_ARCH
+            ARCHITECTURES
+          else
+            [pkg_arch]
+          end.each { |arch| yield repo_dir(repo_config[:epel].to_s, dist.to_s, ver.to_s, arch) }
+        end
+      end
+    end
+
+    def regen_metadata(repo_path)
+      # Rebuild the yum index
+      Profile.log("Rebuilding Yum Repo [#{repo_path}]...")
+      do_in_subdir(repo_path) do
+        exit_val = system('createrepo --update -o $(pwd) $(pwd) 2>&1 >/dev/null')
+        raise Exception, "yum regen_metadata failed; cannot continue publishing" unless exit_val
+      end
+    end
+
+    def yum_prune(pkg, repo_path)
+      our_name = pkg_parts(pkg)[:name]
+      our_arch = pkg_parts(pkg)[:arch]
+
+      do_in_subdir(repo_path) do
+        Dir.glob("*.#{YUM_EXT}") do |rpm|
+          File.unlink(rpm) if pkg_parts(rpm)[:name] == our_name && pkg_parts(rpm)[:arch] == our_arch
+        end
+      end
+    end
+  end
+
+  RightPublish::RepoManager.register_repo(YumRepo)
+end

data/lib/right_publish/storage.rb

@@ -0,0 +1,90 @@
+require 'right_publish/profile'
+
+module RightPublish
+  class StorageManager
+    @@storage_type_regex = /\A(\w+)_storage/
+    @@storage_table = {}
+
+    def self.get_storage(type)
+      @@storage_table[type] if @@storage_table[type]
+    end
+
+    def self.storage_types()
+      type_hash = {}
+      @@storage_table.each_key { |k| type_hash[@@storage_type_regex.match(k.to_s)[1]] = k }
+      type_hash
+    end
+
+    def self.register_storage(module_type)
+      storage_key = module_type::STORAGE_KEY
+
+      if module_type.respond_to?(:get_directories) && @@storage_type_regex.match(storage_key.to_s)
+       @@storage_table[storage_key] = module_type
+       RightPublish::Profile.instance.register_section(@@storage_type_regex.match(storage_key.to_s)[1], module_type::STORAGE_OPTIONS)
+      else
+        raise TypeError
+      end
+      nil
+    end
+  end
+
+  module Storage
+    def self.sync_dirs(src, dest, options={})
+      src_dir = src.get_directories()
+      dest_dir = dest.get_directories()
+
+      src_map = get_etag_map(src_dir, options) {|file| file.etag}
+      dest_map = get_etag_map(dest_dir, options)
+
+      dest_map.each_pair do |hash, files|
+        files.each do |path, file|
+          unless src_map.include? hash and src_map[hash].include? path
+            Profile.log("Removing: #{path}", :debug)
+            file.destroy
+          end
+        end
+      end if options[:sweep]
+
+      src_map.each_pair do |hash, files|
+        # If we already have this data, just copy it, otherwise
+        # sync down a copy and make copies of that.
+        if dest_map.include? hash
+          local_copy = dest_map[hash].first.last
+        else
+          remote_file = files.shift
+          Profile.log("Synchronizing: #{remote_file.first}", :debug)
+          local_copy = dest_dir.files.create(:key=>remote_file.first, :body=>remote_file.last.body, :acl=>'public-read')
+        end
+
+        files.each_key do |file|
+          unless dest_map.include? hash and dest_map[hash].include? file
+            Profile.log("Duplicating: #{file}", :debug)
+            local_copy.copy(dest_dir.key, file, 'x-amz-acl'=>'public-read')
+          end
+        end
+      end
+    end
+
+    private
+
+    def self.get_etag_map(dir, options={})
+      hash_map = {}
+      dir.files.each do |file|
+        next if file.key[-1,1] == '/'
+        next if file.key[0,options[:subdir].size] != options[:subdir] if options.has_key? :subdir
+
+        md5 = dir.compute_md5(file)
+
+        if hash_map.has_key? md5
+          hash_map[md5][file.key] = file
+        else
+          hash_map[md5] = {file.key=>file}
+        end
+      end
+      hash_map
+    end
+  end
+end
+
+require 'right_publish/stores/local'
+require 'right_publish/stores/s3'

data/lib/right_publish/stores/local.rb

@@ -0,0 +1,29 @@
+require 'fog'
+require 'digest/md5'
+
+module RightPublish
+  module LocalStorage
+
+    DEFAULT_LOCAL_CACHEDIR = '~/.rp_cache'
+    STORAGE_KEY = :local_storage
+    STORAGE_OPTIONS = { :cache_dir=>DEFAULT_LOCAL_CACHEDIR}
+    def compute_md5(file)
+      Digest::MD5.file(File.join(service.path_to(self.key), file.key)).hexdigest
+    end
+
+    def self.get_directories()
+      Profile.log("Connecting to local cache.", :debug)
+      conn = Fog::Storage.new(
+        :provider=>"Local",
+        :local_root=>Profile.config[STORAGE_KEY][:cache_dir] )
+
+        Profile.log("Attaching to local cache: [#{Profile.config[STORAGE_KEY][:cache_dir]}].", :debug)
+        conn.directories.create(:key=>'.') unless File.exists? Profile.config[STORAGE_KEY][:cache_dir]
+        local_dir = conn.directories.get('.')
+        local_dir.extend(LocalStorage)
+        local_dir
+    end
+  end
+
+  RightPublish::StorageManager.register_storage(LocalStorage)
+end

data/lib/right_publish/stores/s3.rb

@@ -0,0 +1,35 @@
+require 'fog'
+
+module RightPublish
+  module S3Storage
+
+    STORAGE_KEY = :s3_storage
+    STORAGE_OPTIONS = {
+      :access_id   => :attr_optional,
+      :access_key  => :attr_optional,
+      :region      => :attr_optional,
+      :remote_path => :attr_needed,
+    }
+
+    def compute_md5(file)
+      file.etag
+    end
+
+    def self.get_directories()
+      Profile.log("Connecting to S3.", :debug)
+      conn = Fog::Storage.new(
+        :provider              => "AWS",
+        :aws_access_key_id     => Profile.config[:remote_storage][:access_id],
+        :aws_secret_access_key => Profile.config[:remote_storage][:access_key],
+        :region                => Profile.config[:remote_storage][:region]
+      )
+
+      Profile.log("Attaching to bucket: [#{Profile.config[:remote_storage][:remote_path]}].", :debug)
+      aws_bucket = conn.directories.get(Profile.config[:remote_storage][:remote_path])
+      aws_bucket.extend(S3Storage)
+      aws_bucket
+    end
+  end
+
+  RightPublish::StorageManager.register_storage(S3Storage)
+end

data/lib/right_publish.rb

@@ -0,0 +1,97 @@
+#
+# === Synopsis:
+#   RightScale RightPublish (right_publish) - (c) 2013 RightScale Inc
+#
+
+require 'trollop'
+
+require 'right_publish/profile'
+require 'right_publish/repo'
+
+module RightPublish
+  module Main
+    REPOSITORY_TYPES = RepoManager.repo_types()
+    SUB_COMMANDS = %w(publish fetch store)
+
+    def self.run()
+      options = parse_args
+
+      begin
+        profile = RightPublish::Profile.instance
+        profile.load(options[:profile])
+      rescue LoadError => e
+        puts e
+        exit -1
+      end
+
+      # These options override profile attributes.
+      profile.settings[:local_storage][:cache_dir] = options[:local_cache] if options[:local_cache]
+      profile.settings[:verbose] = true if options[:verbose]
+
+      # We already know this is a valid type, parse_args checked
+      repo = RepoManager.get_repository(REPOSITORY_TYPES[options[:repo_type]])
+      begin
+        case options[:cmd]
+        when "publish"
+          repo.publish(options[:files], options[:dist])
+        when "store"
+          repo.store
+        when "fetch"
+          repo.fetch
+        else
+        end
+      rescue RuntimeError => e
+        RightPublish::Profile.log("Fatal Error:\n\t#{e}", :error)
+        exit -1
+      end
+      RightPublish::Profile.log("Success!")
+    end
+
+    def self.parse_args()
+      options = Trollop.options do
+        version "RightPublish alpha (c) 2013 RightScale Inc"
+        banner <<-EOS
+RightPublish can manage a YUM/APT/RubyGem repository in remote storage, e.g. S3.
+
+Usage:
+       right_publish [global_options] <command> [file1, file2, ...]
+
+commands:
+  fetch
+  publish
+  store
+
+global options:
+EOS
+
+        opt :local_cache,       "Local cache location", :type => String
+        opt :profile,           "Publish profile", :type => String
+        opt :repo_type,         "Repository type: #{REPOSITORY_TYPES.keys.inspect}", :type => String
+        opt :dist,              "Target distribution. Required for binary packages. If unspecified for noarch and source packages, will copy to all distributions specified in profile.", :type => String
+        opt :verbose,           "Verbose output"
+
+        stop_on SUB_COMMANDS
+      end
+
+      options[:cmd]= ARGV.shift
+      options[:files] = expand_argv_globs
+
+      Trollop.die "argument --profile is required" unless options[:profile]
+      Trollop.die "argument --repo-type is required" unless options[:repo_type]
+      Trollop.die "profile does not exist: #{options[:profile]}" unless File.exist?(options[:profile])
+      Trollop.die "invalid repository type: #{options[:repo_type]}" unless REPOSITORY_TYPES[options[:repo_type]]
+
+      options
+    end
+
+    def self.expand_argv_globs
+      files = []
+
+      ARGV.each do |glob|
+        files += Dir.glob(glob)
+      end
+
+      files
+    end
+  end
+end