right_publish 0.3.0 → 0.4.0

data/Gemfile

@@ -2,6 +2,8 @@ source "http://rubygems.org"
 
 gem "builder"
 gem "fog", "~> 1.9"
+gem "excon", "<= 0.25.3" # We do not depend directly on excon, but fog does, and > 0.25 is incompatible with fog
+gem "mime-types", "~> 1.0"
 gem "trollop", "~> 2.0"
 
 # Gems used to test and develop RightPublish

data/Gemfile.lock

@@ -101,9 +101,11 @@ PLATFORMS
 
 DEPENDENCIES
   builder
+  excon (<= 0.25.3)
   flexmock (~> 0.9)
   fog (~> 1.9)
   jeweler (~> 1.8.3)
+  mime-types (~> 1.0)
   rake (~> 0.9)
   rdoc (>= 2.4.2)
   right_develop (~> 1.0)!

data/README.rdoc

@@ -33,28 +33,28 @@ debian distros. It expects the "expect" package to be installed on all distros.
   # value of an attribute, or they can appear anywhere in
   # the attribute with static text surrounding them.
   #
-  :verbose:                       # -: false
-  :apt_repo:
-    :dists:                       # *: [ woody, sid, etch, jaunty, lucid ]
-    :auto:                        # -: true
-    :subdir:                      # -: apt/
-    :gpg_key_id:                  # *: 9A917D05
-    :gpg_password:                # *: @@GPG_PASSWORD@@
-  :gem_repo:
-    :subdir:                      # -: gems/
-  :yum_repo:
-    :dists:                       # *: el: ['6']
-    :epel:                        # -: 1
-    :subdir:                      # -: yum/
-    :gpg_key_id:                  # *: 9A917D05
-    :gpg_password:                # *: @@GPG_PASSWORD@@
-  :local_storage:
-    :cache_dir                    # -: ~/.rp_cache
-  :remote_storage:
-    :storage_provider:            # -: AWS
-    :access_id:                   # *: @@AWS_ACCESS_KEY_ID@@
-    :access_key:                  # *: @@AWS_SECRET_ACCESS_KEY@@
-    :remote_path:                 # +: rightlink-testing
+  verbose:                       # -: false
+  apt_repo:
+    dists:                       # *: [ woody, sid, etch, jaunty, lucid ]
+    auto:                        # -: true
+    subdir:                      # -: apt/
+    gpg_key_id:                  # *: 9A917D05
+    gpg_password:                # *: @@GPG_PASSWORD@@
+  gem_repo:
+    subdir:                      # -: gems/
+  yum_repo:
+    dists:                       # *: el: ['6']
+    epel:                        # -: 1
+    subdir:                      # -: yum/
+    gpg_key_id:                  # *: 9A917D05
+    gpg_password:                # *: @@GPG_PASSWORD@@
+  local_storage:
+    cache_dir                    # -: ~/.right_publish/cache
+  remote_storage:
+    provider:                    # -: s3
+    access_id:                   # *: @@AWS_ACCESS_KEY_ID@@
+    access_key:                  # *: @@AWS_SECRET_ACCESS_KEY@@
+    remote_path:                 # +: rightlink-testing
   #
   # End of RightPublish Profile
   ###########################################################
@@ -69,12 +69,12 @@ debian distros. It expects the "expect" package to be installed on all distros.
   Usage:
          right_publish [global_options] <command> [cmd_options]
   commands:
-    fetch
+    pull
     publish
-    store
+    push
   global options:
       --profile, -p <s>:   Publish profile
-    --repo-type, -r <s>:   Repository type: ["apt", "gem", "yum"]
+    --repo-type, -r <s>:   Repository type: ["apt", "gem", "msi", "yum", "zypp"]
              --dist, -d:   Target dist: ["el/6", "precise"]
           --verbose, -v:   Verbose output
           --version, -e:   Print version and exit

data/VERSION

@@ -1 +1 @@
-0.3.0
+0.4.0

data/bin/right_publish

@@ -1,8 +1,4 @@
 #! /usr/bin/env ruby
-#
-#
-
-$: << File.expand_path("../../lib",__FILE__)
 
 require 'right_publish'
-RightPublish::Main.run()
+RightPublish::CLI.run()

data/lib/right_publish/annotation.rb

@@ -0,0 +1,101 @@
+require 'builder'
+
+RightPublish::Profile.instance.register_section(:annotation,
+                                                :title       => nil,
+                                                :scripts     => [],
+                                                :stylesheets => [])
+
+module RightPublish
+  module Annotation
+    module_function
+
+    # Generate an HTML index file for a set of packages.
+    # @param [Array] files list of relative file keys
+    # @param [Integer] strip number of path elements to strip from displayed filenames, default 0
+    # @option options [Array] :filter  a whitelist of String glob patterns to filter files, i.e. ["*.rpm", "*.deb"]
+    # @option options [String] :title Human-readable page title
+    # @option options [Array] :scripts JavaScript hrefs to embed into page as <script> tags
+    # @option options [Array] :stylesheets Stylesheet hrefs to embed into page as <link> tags
+    def generate_html(files, strip=0, options={})
+      title       = options[:title]
+      filter      = options[:filter] || []
+      scripts     = options[:scripts] || []
+      stylesheets = options[:stylesheets] || []
+
+      unless filter.nil? || filter.empty?
+        filtered  = files.select { |f| filter.any? { |ff| File.fnmatch(ff, f.key) } }
+      end
+      segmented = filtered.map { |f| f.key.split('/') }
+
+      index     = {}
+      segmented.each do |segments|
+        stripped = segments[strip..-1]
+
+        map = index
+        stripped.each_with_index do |seg, idx|
+          if idx == stripped.size - 1
+            map[seg] = "/#{segments.join('/')}"
+          else
+            map[seg] ||= {}
+            map      = map[seg]
+          end
+        end
+      end
+
+      b = Builder::XmlMarkup.new(:indent => 2)
+
+      # Declare us as an HTML5 document
+      b.declare!(:DOCTYPE, :HTML)
+
+      b.html {
+        b.head {
+          # Ensure that browsers get a valid content type and charset, even if the Web server
+          # is misconfigured. This is necessary for our doc to be considered valid HTML5.
+          b.meta(:'http-equiv'=>'Content-Type', :content=>'text/html; charset=UTF-8')
+
+          if title
+            b.title(title)
+          end
+
+          stylesheets.each do |stylesheet|
+            b.link(:rel => 'stylesheet', :type => 'text/css', :href => stylesheet)
+          end
+
+          scripts.each do |script|
+            b.script(:src => script, :type => 'text/javascript')
+          end
+        }
+        b.body {
+          if title
+            b.h1(title, :class => 'title')
+          end
+          recursive_ul(index, b)
+        }
+      }
+    end
+
+    private
+
+    def self.recursive_ul(index, b)
+      b.ul(:class => 'dirList') {
+        index.keys.sort.each do |k|
+          v = index[k]
+
+          case v
+          when Hash
+            b.li(:class => 'dir') {
+              b.text! k
+              recursive_ul(v, b)
+            }
+          else
+            base = File.basename(v)
+            ext  = File.extname(v)[1..-1]
+            b.li(:class => ext) {
+              b.a(base, :href => v)
+            }
+          end
+        end
+      }
+    end
+  end
+end

data/lib/right_publish/cli.rb

@@ -0,0 +1,91 @@
+require 'trollop'
+
+module RightPublish
+  module CLI
+    REPOSITORY_TYPES = RepoManager.repo_types()
+    SUB_COMMANDS = %w(publish pull add annotate push)
+
+    def self.run()
+      options = parse_args
+
+      begin
+        profile = RightPublish::Profile.instance
+        profile.load(options[:profile] || options[:repo_type])
+      rescue LoadError => e
+        puts e
+        exit -1
+      end
+
+      # These options override profile attributes.
+      profile.settings[:local_storage][:cache_dir] = options[:local_cache] if options[:local_cache]
+      profile.settings[:verbose] = true if options[:verbose]
+
+      # We already know this is a valid type, parse_args checked
+      repo = RepoManager.get_repository(REPOSITORY_TYPES[options[:repo_type]])
+      begin
+        case options[:cmd]
+        when 'publish'
+          repo.publish(options[:files], options[:dist])
+        when 'pull'
+          repo.pull
+        when 'add'
+          repo.add(options[:files], options[:dist])
+        when 'annotate'
+          repo.annotate
+        when 'push'
+          repo.push
+        else
+          raise ArgumentError, "Unknown command '#{options[:cmd]}'"
+        end
+      rescue RuntimeError => e
+        RightPublish::Profile.log("Fatal Error:\n\t#{e}", :error)
+        exit -1
+      end
+
+      exit 0
+    end
+
+    def self.parse_args()
+      options = Trollop.options do
+        version "RightPublish (c) 2013 RightScale Inc"
+        banner <<-EOS
+RightPublish can manage a YUM/APT/RubyGem repository in remote storage, e.g. S3.
+
+Usage:
+       right_publish [options] <command> [file1, file2, ...]
+
+basic commands:
+  publish
+
+advanced commands:
+  pull add annotate push
+
+options:
+        EOS
+
+        opt :local_cache,       "Local cache location", :type => String
+        opt :profile,           "Publish profile", :type => String, :default => nil
+        opt :repo_type,         "Repository type: #{REPOSITORY_TYPES.keys.inspect}", :type => String
+        opt :dist,              "Target distribution. Required for binary packages. If unspecified for noarch and source packages, will copy to all distributions specified in profile.", :type => String
+        opt :verbose,           "Verbose output"
+      end
+
+      options[:cmd]= ARGV.shift
+      options[:files] = expand_argv_globs
+
+      Trollop.die "invalid repository type: #{options[:repo_type]}" unless REPOSITORY_TYPES[options[:repo_type]]
+
+      options
+    end
+
+    def self.expand_argv_globs
+      files = []
+
+      ARGV.each do |glob|
+        files += Dir.glob(glob)
+      end
+
+      files
+    end
+  end
+end

data/lib/right_publish/profile.rb

@@ -2,13 +2,10 @@ require 'singleton'
 require 'yaml'
 
 module RightPublish
-
   class Profile
-
-    # Configuration defaults
-    # =============
-    DEFAULT_VERBOSE = false
-    # =============
+    # Location where per-user profiles are stored
+    USER_PROFILE_DIR = File.expand_path('~/.right_publish/profiles')
+    DEFAULT_VERBOSE  = false
 
     attr_accessor :settings
     include Singleton
@@ -21,22 +18,35 @@ module RightPublish
       puts(s) if Profile.config[:verbose] || level != :debug
     end
 
+    # Load a YML profile from disk. Profiles can be identified by their path on disk, or by the name
+    # of a YML file located in USER_PROFILE_DIR (without YML extension).
+    #
+    # @param [String] path absolute/relative path to file, or the name of per-user profile
+    # @return [true] always returns true
+    #
+    # @example per-user profile
+    #   Profile.instance.load("gem")
+    #
+    # @example profile on disk
+    #   Profile.instance.load("config/publish/nightly.yml")
+    #
+    # @see USER_PROFILE_DIR
     def load(path)
-      begin
-        if path && File.exists?(path)
-          begin
-            @settings ||= Profile.symbolize_profile(YAML.load_file(path))
-            validate_profile
-          rescue Exception => e
-            raise LoadError.new("Bad Format: #{path}\n#{e}")
-          end
-        else
-          raise LoadError.new("Missing Profile")
-        end
-      rescue LoadError => e
-        raise LoadError.new("#{e}")
+      if File.exists?(path)
+        @settings ||= Profile.symbolize_profile(YAML.load_file(path))
+        validate_profile
+      elsif (path = File.join(USER_PROFILE_DIR, "#{path}.yml")) && File.exists?(path)
+        @settings ||= Profile.symbolize_profile(YAML.load_file(path))
+        validate_profile
+      else
+        raise LoadError, "Missing profile '#{path}'"
       end
-      nil
+
+      true
+    rescue LoadError
+      raise
+    rescue Exception => e
+      raise LoadError, "Cannot load profile '#{path}' - #{e}"
     end
 
     def register_section(section_key, section_options)

data/lib/right_publish/repo.rb

@@ -5,7 +5,7 @@ module RightPublish
 
   class RepoManager
     @@repository_type_regex = /\A(\w+)_repo/
-    @@repository_table = {}
+    @@repository_table      = {}
 
     def self.get_repository(type)
       @@repository_table[type].new(type) if @@repository_table[type]
@@ -31,37 +31,85 @@ module RightPublish
     end
   end
 
-  module Repo
+  class Repo
     def initialize(option_key)
       @repository_type ||= option_key
     end
 
-    def fetch()
-      Profile.log("Fetching latest #{repo_human_name(@repository_type)} data...")
+    # Sync files from remote to local storage
+    def pull()
+      Profile.log("Fetching #{repo_human_name(@repository_type)} files from remote...")
+
       begin
         sync_dirs(
           get_storage(Profile.config[:remote_storage][:provider]),
           get_storage(Profile.config[:local_storage][:provider]),
-          Profile.config[@repository_type][:subdir] )
+          Profile.config[@repository_type][:subdir])
       rescue Exception => e
         RightPublish::Profile.log("Could not synchronize storage:\n\t#{e}", :error)
-        raise RuntimeError, "fetch from remote failed."
+        raise RuntimeError, "pull from remote failed."
       end
     end
 
-    def store()
-      Profile.log("Commiting local #{repo_human_name(@repository_type)} data...")
+    # Sync files from local to remote storage
+    def push()
+      Profile.log("Committing local #{repo_human_name(@repository_type)} files to remote...")
+
       begin
         sync_dirs(
           get_storage(Profile.config[:local_storage][:provider]),
           get_storage(Profile.config[:remote_storage][:provider]),
-          Profile.config[@repository_type][:subdir] )
+          Profile.config[@repository_type][:subdir])
       rescue Exception => e
         RightPublish::Profile.log("Could not sychronize storage:\n\t#{e}", :error)
-        raise RuntimeError, "store to remote failed."
+        raise RuntimeError, "push to remote failed."
       end
     end
-    
+
+    # Create an index.html file to enable browsing of this repo's subdir.
+    # @option options [String] :subdir a subdirectory (common prefix); only files matching this prefix will be included
+    # @option options [Array] :filter  a whitelist of String glob patterns to filter files, i.e. ["*.rpm", "*.deb"]
+    def annotate(options={})
+
+      options[:subdir] ||= repo_config[:subdir]
+      storage_type = options[:use_remote_storage] ? :remote_storage : :local_storage
+
+      Profile.log("Creating HTML directory listing from #{storage_type.to_s} storage for #{repo_human_name(@repository_type)} files...")
+
+      files = []
+      RightPublish::Storage.ls(
+        get_storage(Profile.config[storage_type][:provider]),
+        :subdir => options[:subdir]) do |file|
+        files << file
+      end
+
+      # Build merged options hash for annotation, based on profile config plus some options
+      # passed into our class.
+      html_options     = Profile.config[:annotation] ? Profile.config[:annotation].dup : {}
+      html_options[:filter] = options[:filter] if options.key?(:filter)
+
+      strip            = options[:subdir].split('/').size
+      html             = RightPublish::Annotation.generate_html(files, strip, html_options)
+
+      output    = File.join(repo_config[:subdir], 'index.html')
+      local_dir = get_storage(Profile.config[:local_storage][:provider]).get_directories
+      local_dir.files.create(:key => output, :body => html)
+    end
+
+    # Perform one-shot publish of one or more packages.
+    # "One-shot" means: pull, add, annotate and push.
+    def publish(file_or_dir, target)
+      pull
+      add(file_or_dir, target)
+      annotate
+      push
+    end
+
+    # Add a new package to local storage and reindex if necessary
+    def add(file_or_dir, targe)
+      raise NotImplementedError, "Subclass responsibility"
+    end
+
     private
 
     def build_glob(ext)
@@ -77,16 +125,16 @@ module RightPublish
     end
 
     def get_pkg_list(file_or_dir, ext=nil)
-      pkg_list = []
+      pkg_list    = []
       file_or_dir = Array(file_or_dir)
       file_or_dir.each do |path|
         path = path.gsub("\\", "/")
         pkg_list << if File.directory?(path)
-          glob_filter = (ext && "*.#{build_glob(ext)}") || "*"
-          Dir.glob(File.join(path, glob_filter))
-        else
-          path.split(',')
-        end
+                      glob_filter = (ext && "*.#{build_glob(ext)}") || "*"
+                      Dir.glob(File.join(path, glob_filter))
+                    else
+                      path.split(',')
+                    end
       end
       pkg_list.flatten!
 
@@ -100,8 +148,8 @@ module RightPublish
       pkg_list
     end
 
-    # For automation, we want to send the password, however gpg uses getpass 
-    # c function, which interacts directly with /dev/pty instead of stdin/stdout 
+    # For automation, we want to send the password, however gpg uses getpass
+    # c function, which interacts directly with /dev/pty instead of stdin/stdout
     # to hide the password as its typed in.  So, we need to allocate a pty.
     # We do this by shelling out to expect script (tcl based).  Ruby 1.8
     # implementation of "expect" is broken and has some race conditions with
@@ -109,9 +157,9 @@ module RightPublish
     def shellout_with_password(cmd)
       password = repo_config[:gpg_password]
       raise Exception, ":gpg_password must be supplied when signing packages" unless password
-      ENV['GPG_PASSWORD'] = password 
-      bin_dir = File.expand_path("../../../bin", __FILE__)
-      autosign = File.join(bin_dir, "autosign.expect")
+      ENV['GPG_PASSWORD'] = password
+      bin_dir             = File.expand_path("../../../bin", __FILE__)
+      autosign            = File.join(bin_dir, "autosign.expect")
       system("#{autosign} #{cmd}")
     end
 
@@ -123,7 +171,7 @@ module RightPublish
     def install_file(file, dest)
       Profile.log("#{file} => #{dest}")
       local_dir = get_storage(Profile.config[:local_storage][:provider]).get_directories
-      File.open(file, "rb") { |chunk| local_dir.files.create(:key=>File.join(dest, File.basename(file)), :body=>chunk) }
+      File.open(file, "rb") { |chunk| local_dir.files.create(:key => File.join(dest, File.basename(file)), :body => chunk) }
     end
 
     def prune_all(glob_pattern)
@@ -136,16 +184,17 @@ module RightPublish
     end
 
     def repo_human_name(type)
-      type.to_s.sub(/_/,' ')
+      type.to_s.sub(/_/, ' ')
     end
 
     def sync_dirs(src, dest, subdir='')
-      RightPublish::Storage.sync_dirs(src, dest, :subdir=>subdir, :sweep=>true)
+      RightPublish::Storage.sync_dirs(src, dest, :subdir => subdir, :sweep => true)
     end
   end
 end
 
 require 'right_publish/repos/apt'
 require 'right_publish/repos/gem'
+require 'right_publish/repos/msi'
 require 'right_publish/repos/yum'
 require 'right_publish/repos/zypp'