right_publish 0.2.0

data/.rspec

@@ -0,0 +1,4 @@
+--colour
+--format=nested
+--backtrace
+--debugger

data/Gemfile

@@ -0,0 +1,20 @@
+source "http://rubygems.org"
+
+gem "builder"
+gem "fog", "~> 1.9"
+gem "trollop", "~> 2.0"
+
+# Gems used to test and develop RightPublish
+group :development do
+  gem "rake", "~> 0.9"
+  gem "jeweler", "~> 1.8.3"
+  gem "right_develop", "~> 1.0",
+      :git    => "git@github.com:rightscale/right_develop.git",
+      :branch => "master"
+  gem "rdoc", ">= 2.4.2"
+  gem "rspec", "~> 2.0"
+  gem "flexmock", "~> 0.9"
+  gem "simplecov"
+  gem "ruby-debug", ">= 0.10", :platforms => :ruby_18
+  gem "ruby-debug19", ">= 0.11.6", :platforms => :ruby_19
+end

data/Gemfile.lock

@@ -0,0 +1,114 @@
+GIT
+  remote: git@github.com:rightscale/right_develop.git
+  revision: 90a8fde986145756f341088130c6e1aac4759d68
+  branch: master
+  specs:
+    right_develop (1.1.0)
+      actionpack (>= 2.3.0, < 4.0)
+      builder (~> 3.0)
+      cucumber (~> 1.0)
+      rake (>= 0.8.7, < 0.10)
+      right_support (~> 2.0)
+      rspec (>= 1.3, < 3.0)
+      trollop (>= 1.0, < 3.0)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    actionpack (2.3.18)
+      activesupport (= 2.3.18)
+      rack (~> 1.1.0)
+    activesupport (2.3.18)
+    archive-tar-minitar (0.5.2)
+    builder (3.2.0)
+    columnize (0.3.6)
+    cucumber (1.2.5)
+      builder (>= 2.1.2)
+      diff-lcs (>= 1.1.3)
+      gherkin (~> 2.11.7)
+      multi_json (~> 1.3)
+    diff-lcs (1.2.3)
+    excon (0.20.1)
+    flexmock (0.9.0)
+    fog (1.10.1)
+      builder
+      excon (~> 0.20)
+      formatador (~> 0.2.0)
+      mime-types
+      multi_json (~> 1.0)
+      net-scp (~> 1.1)
+      net-ssh (>= 2.1.3)
+      nokogiri (~> 1.5.0)
+      ruby-hmac
+    formatador (0.2.4)
+    gherkin (2.11.8)
+      multi_json (~> 1.3)
+    git (1.2.5)
+    jeweler (1.8.4)
+      bundler (~> 1.0)
+      git (>= 1.2.5)
+      rake
+      rdoc
+    json (1.7.7)
+    linecache (0.46)
+      rbx-require-relative (> 0.0.4)
+    linecache19 (0.5.12)
+      ruby_core_source (>= 0.1.4)
+    mime-types (1.22)
+    multi_json (1.7.2)
+    net-scp (1.1.0)
+      net-ssh (>= 2.6.5)
+    net-ssh (2.6.7)
+    nokogiri (1.5.9)
+    rack (1.1.6)
+    rake (0.9.6)
+    rbx-require-relative (0.0.9)
+    rdoc (4.0.1)
+      json (~> 1.4)
+    right_support (2.6.17)
+    rspec (2.13.0)
+      rspec-core (~> 2.13.0)
+      rspec-expectations (~> 2.13.0)
+      rspec-mocks (~> 2.13.0)
+    rspec-core (2.13.1)
+    rspec-expectations (2.13.0)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.13.1)
+    ruby-debug (0.10.4)
+      columnize (>= 0.1)
+      ruby-debug-base (~> 0.10.4.0)
+    ruby-debug-base (0.10.4)
+      linecache (>= 0.3)
+    ruby-debug-base19 (0.11.25)
+      columnize (>= 0.3.1)
+      linecache19 (>= 0.5.11)
+      ruby_core_source (>= 0.1.4)
+    ruby-debug19 (0.11.6)
+      columnize (>= 0.3.1)
+      linecache19 (>= 0.5.11)
+      ruby-debug-base19 (>= 0.11.19)
+    ruby-hmac (0.4.0)
+    ruby_core_source (0.1.5)
+      archive-tar-minitar (>= 0.5.2)
+    simplecov (0.7.1)
+      multi_json (~> 1.0)
+      simplecov-html (~> 0.7.1)
+    simplecov-html (0.7.1)
+    trollop (2.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  builder
+  flexmock (~> 0.9)
+  fog (~> 1.9)
+  jeweler (~> 1.8.3)
+  rake (~> 0.9)
+  rdoc (>= 2.4.2)
+  right_develop (~> 1.0)!
+  rspec (~> 2.0)
+  ruby-debug (>= 0.10)
+  ruby-debug19 (>= 0.11.6)
+  simplecov
+  trollop (~> 2.0)

data/README.rdoc

@@ -0,0 +1,113 @@
+= RightPublish
+
+= DESCRIPTION
+
+== Synopsis
+
+RightPublish facilitates the management of remote package repositories. It is in
+essense a wrapper around package repository management tools such as YUM and
+RubyGems allowing one to add/remove/replace packages stored in a repository
+residing in a remote storage location, typically Amazon's S3. The current
+incantation of RightPublish can manage RubyGem, YUM (RPM) and APT (DEB) pacakge
+repositories.
+
+== Interface
+
+RightPublish is used as a command line tool and controlled via command line options
+and a YAML configuration file known as a RightPublish <i>profile</i>.
+
+=== Profile Format
+
+  ###########################################################
+  # RightPublish Profile
+  #
+  # + Attribute is required, example shown
+  # * Attribute is optional, no default; example shown
+  # - Attribute is optional, default shown
+  #
+  # Values of the form "@@...@@" indicate an environment
+  # variable should be substituted in place of the value.
+  # These substitution sequences can appear as the entire
+  # value of an attribute, or they can appear anywhere in
+  # the attribute with static text surrounding them.
+  #
+  :verbose:                       # -: false
+  :apt_repo:
+    :dists:                       # *: [ woody, sid, etch, jaunty, lucid ]
+    :auto:                        # -: true
+    :subdir:                      # -: apt/
+    :gpg_key_id:                  # *: 9A917D05
+    :gpg_password:                # *: @@GPG_PASSWORD@@
+  :gem_repo:
+    :subdir:                      # -: gems/
+  :yum_repo:
+    :dists:                       # *: el: ['6']
+    :epel:                        # -: 1
+    :subdir:                      # -: yum/
+    :gpg_key_id:                  # *: 9A917D05
+    :gpg_password:                # *: @@GPG_PASSWORD@@
+  :local_storage:
+    :cache_dir                    # -: ~/.rp_cache
+  :remote_storage:
+    :storage_provider:            # -: AWS
+    :access_id:                   # *: @@AWS_ACCESS_KEY_ID@@
+    :access_key:                  # *: @@AWS_SECRET_ACCESS_KEY@@
+    :remote_path:                 # +: rightlink-testing
+  #
+  # End of RightPublish Profile
+  ###########################################################
+
+=== Command Line Usage
+
+  RightPublish can manage a YUM/APT/RubyGem repository in remote storage, e.g. S3.
+  If autosigning packages import the private key beforehand with gpg --import.  If 
+  no gpg_password config key is supplied but a gpg_key_id is it should  prompt you
+  for a password.
+
+  Usage:
+         right_publish [global_options] <command> [cmd_options]
+  commands:
+    fetch
+    publish
+    store
+  global options:
+      --profile, -p <s>:   Publish profile
+    --repo-type, -r <s>:   Repository type: ["apt", "gem", "yum"]
+             --dist, -d:   Target dist: ["el/6", "precise"]
+          --verbose, -v:   Verbose output
+          --version, -e:   Print version and exit
+             --help, -h:   Show this message
+
+
+== Supported Configuration
+
+RightPublish has been tested on EL6, Ubuntu 12.10 and ArchLinux (as of March `13).
+
+== Work in Progress
+
+RightPublish is work in progress, expect more documentation and examples in the near future.
+
+= LICENSE
+
+<b>RightPublish</b>
+
+Copyright:: Copyright (c) 2013 RightScale, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,48 @@
+# -*-ruby-*-
+require 'rubygems'
+require 'bundler/setup'
+
+require 'rake'
+require 'rdoc/task'
+require 'rubygems/package_task'
+
+require 'rake/clean'
+require 'rspec/core/rake_task'
+
+require 'right_develop/ci/rake_task'
+
+desc "Run unit tests"
+task :default => :spec
+
+desc "Run unit tests"
+RSpec::Core::RakeTask.new do |t|
+  t.pattern = Dir['**/*_spec.rb']
+end
+
+desc 'Generate documentation for the right_publish gem.'
+Rake::RDocTask.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'doc'
+  rdoc.title    = 'RightPublish'
+  rdoc.options << '--line-numbers' << '--inline-source'
+  rdoc.rdoc_files.include('README.rdoc')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+  rdoc.rdoc_files.exclude('features/**/*')
+  rdoc.rdoc_files.exclude('spec/**/*')
+end
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification; see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "right_publish"
+  gem.homepage = "https://github.com/rightscale/right_publish"
+  gem.license = "Proprietary"
+  gem.summary = %Q{Package publishing and indexing tool}
+  gem.description = %Q{A tool for maintaining S3-based DEB, GEM and RPM packages.}
+  gem.email = "support@rightscale.com"
+  gem.authors = ['Brian Szmyd', 'Tony Spataro']
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+CLEAN.include('pkg')
+
+RightDevelop::CI::RakeTask.new

data/VERSION

@@ -0,0 +1 @@
+0.2.0

data/bin/right_publish

@@ -0,0 +1,8 @@
+#! /usr/bin/env ruby
+#
+#
+
+$: << File.expand_path("../../lib",__FILE__)
+
+require 'right_publish'
+RightPublish::Main.run()

data/lib/right_publish/profile.rb

@@ -0,0 +1,96 @@
+require 'singleton'
+require 'yaml'
+
+module RightPublish
+
+  class Profile
+
+    # Configuration defaults
+    # =============
+    DEFAULT_VERBOSE = false
+    # =============
+
+    attr_accessor :settings
+    include Singleton
+
+    def self.config()
+      return Profile.instance.settings
+    end
+
+    def self.log(s, level=:info)
+      puts(s) if Profile.config[:verbose] || level != :debug
+    end
+
+    def load(path)
+      begin
+        if path && File.exists?(path)
+          begin
+            @settings ||= Profile.symbolize_profile(YAML.load_file(path))
+            validate_profile
+          rescue Exception => e
+            raise LoadError.new("Bad Format: #{path}\n#{e}")
+          end
+        else
+          raise LoadError.new("Missing Profile")
+        end
+      rescue LoadError => e
+        raise LoadError.new("#{e}")
+      end
+      nil
+    end
+
+    def register_section(section_key, section_options)
+      @config_map ||= {}
+      @config_map[section_key] ||= section_options
+    end
+
+    private
+
+    def self.symbolize_profile(args)
+      if args.is_a? Hash
+        Hash[ args.collect do |key, value|
+          [key.to_sym, symbolize_profile(value)]
+        end ]
+      else
+        args
+      end
+    end
+
+    SUBSTITUTION = /@@(\w+)@@/
+
+    def check_section(section, attrs)
+      provider = (attrs[:provider] && attrs[:provider]) || section
+      raise LoadError.new("Unknown provider #{attrs[:provider]} for: #{section}") unless @config_map[provider]
+
+      @settings[section] ||= {}
+      @config_map[provider].each_pair do |attr, default|
+        setting = @settings[section][attr]
+
+        if setting.nil? && (default != :attr_optional)
+          if default == :attr_needed
+            raise LoadError.new("Missing attribute #{section}:#{attr}")
+          end
+          @settings[section][attr] = default
+        end
+
+        # Perform environment-variable substitition if the setting is a leaf node
+        if setting.is_a?(String)
+          while match = SUBSTITUTION.match(setting)
+            if ENV.key?(match[1])
+              setting = setting.gsub(match[0], ENV[match[1]])
+            else
+              raise LoadError.new("Missing environment variable #{match[1]}.")
+            end
+          end
+        end
+
+        @settings[section][attr] = setting
+      end
+    end
+
+    def validate_profile()
+      @settings[:verbose] = DEFAULT_VERBOSE unless @settings[:verbose]
+      @settings.each_pair { |section, attrs| check_section(section, attrs) if attrs.is_a? Hash }
+    end
+  end
+end 

data/lib/right_publish/repo.rb

@@ -0,0 +1,159 @@
+require 'right_publish/profile'
+require 'right_publish/storage'
+require 'pty'
+require 'expect'
+
+module RightPublish
+
+  class RepoManager
+    @@repository_type_regex = /\A(\w+)_repo/
+    @@repository_table = {}
+
+    def self.get_repository(type)
+      @@repository_table[type].new(type) if @@repository_table[type]
+    end
+
+    def self.repo_types()
+      type_hash = {}
+      @@repository_table.each_key { |k| type_hash[@@repository_type_regex.match(k.to_s)[1]] = k }
+      type_hash
+    end
+
+    def self.register_repo(repo_type)
+      repo_key = repo_type::REPO_KEY
+
+      if repo_type.respond_to?(:new) && @@repository_type_regex.match(repo_key.to_s)
+        @@repository_table[repo_key] = repo_type
+        RightPublish::Profile.instance.register_section(repo_key, repo_type::REPO_OPTIONS)
+      else
+        raise TypeError
+      end
+
+      nil
+    end
+  end
+
+  module Repo
+    def initialize(option_key)
+      @repository_type ||= option_key
+    end
+
+    def fetch()
+      Profile.log("Fetching latest #{repo_human_name(@repository_type)} data...")
+      begin
+        sync_dirs(
+          get_storage(Profile.config[:remote_storage][:provider]),
+          get_storage(Profile.config[:local_storage][:provider]),
+          Profile.config[@repository_type][:subdir] )
+      rescue Exception => e
+        RightPublish::Profile.log("Could not synchronize storage:\n\t#{e}", :error)
+        raise RuntimeError, "fetch from remote failed."
+      end
+    end
+
+    def store()
+      Profile.log("Commiting local #{repo_human_name(@repository_type)} data...")
+      begin
+        sync_dirs(
+          get_storage(Profile.config[:local_storage][:provider]),
+          get_storage(Profile.config[:remote_storage][:provider]),
+          Profile.config[@repository_type][:subdir] )
+      rescue Exception => e
+        RightPublish::Profile.log("Could not sychronize storage:\n\t#{e}", :error)
+        raise RuntimeError, "store to remote failed."
+      end
+    end
+    
+    private
+
+    def build_glob(ext)
+      # Transform array into glob compliant pattern
+      '{'.concat Array(ext).join(',').concat '}'
+    end
+
+    def do_in_subdir(subdir)
+      full_path = File.expand_path(File.join(Profile.config[:local_storage][:cache_dir], subdir))
+      FileUtils.mkdir_p full_path
+
+      Dir.chdir(full_path) { yield }
+    end
+
+    def get_pkg_list(file_or_dir, ext=nil)
+      pkg_list = []
+      file_or_dir = Array(file_or_dir)
+      file_or_dir.each do |path|
+        path = path.gsub("\\", "/")
+        pkg_list << if File.directory?(path)
+          glob_filter = (ext && "*.#{build_glob(ext)}") || "*"
+          Dir.glob(File.join(path, glob_filter))
+        else
+          path.split(',')
+        end
+      end
+      pkg_list.flatten!
+
+      fail("No packages found") if pkg_list.empty?
+      pkg_list.each do |pkg_path|
+        fail("\"#{pkg_path}\" does not appear to be a valid package for the repository.") \
+          unless File.file?(pkg_path) && Array(ext).any? { |e| pkg_path.end_with?(e) }
+        yield pkg_path if block_given?
+      end if ext
+
+      pkg_list
+    end
+
+    # For automation, we want to send the password, however gpg uses getpass 
+    # c function, which interacts directly with /dev/pty instead of stdin/stdout 
+    # to hide the password as its typed in.  So, we need to allocate a pty.
+    def shellout_with_password(cmd)
+      password = repo_config[:gpg_password]
+      raise Exception, ":gpg_password must be supplied when signing packages" unless password
+
+      begin
+        PTY.spawn(cmd) do |stdin,stdout,pid|
+          while output = stdin.expect(/pass.?phrase/i, 2)
+            stdout.puts(password)
+            Profile.log(output, :debug)
+          end
+          Process.wait(pid)
+        end
+        status = $?
+      rescue PTY::ChildExited => e
+        status = e.status
+      end
+      return status.success?
+    end
+
+    def get_storage(provider)
+      type = RightPublish::StorageManager.storage_types[provider]
+      RightPublish::StorageManager.get_storage(type)
+    end
+
+    def install_file(file, dest)
+      Profile.log("#{file} => #{dest}")
+      local_dir = get_storage(Profile.config[:local_storage][:provider]).get_directories
+      File.open(file, "rb") { |chunk| local_dir.files.create(:key=>File.join(dest, File.basename(file)), :body=>chunk) }
+    end
+
+    def prune_all(glob_pattern)
+      Profile.log("Pruning: #{glob_pattern}")
+      Dir.glob(glob_pattern) { |f| File.unlink(f) }
+    end
+
+    def repo_config()
+      Profile.config[@repository_type]
+    end
+
+    def repo_human_name(type)
+      type.to_s.sub(/_/,' ')
+    end
+
+    def sync_dirs(src, dest, subdir='')
+      RightPublish::Storage.sync_dirs(src, dest, :subdir=>subdir, :sweep=>true)
+    end
+  end
+end
+
+require 'right_publish/repos/apt'
+require 'right_publish/repos/gem'
+require 'right_publish/repos/yum'