right_publish 0.3.0 → 0.4.0

data/lib/right_publish/repos/apt.rb

@@ -1,10 +1,8 @@
 module RightPublish
-  class AptRepo
-    include RightPublish::Repo
-
+  class AptRepo < Repo
     DEFAULT_APT_AUTO = true
     DEFAULT_APT_DIR = 'apt/'
-    DEFAULT_DESCRIPTION = "RightScale RightLink Repository"
+    DEFAULT_DESCRIPTION = "Apt Repository"
     REPO_KEY = :apt_repo
 
     REPO_OPTIONS = {
@@ -22,7 +20,7 @@ module RightPublish
     @@supported_archs = [ 'i386', 'amd64' ]
     @@all_archs = [ 'i386', 'amd64', BIN_ALL_ARCH ]
 
-    def publish(file_or_dir, target)
+    def add(file_or_dir, target)
       repo_updates = {}
 
       # If we specified a target, let's make sure it's in our profile
@@ -39,9 +37,6 @@ module RightPublish
         pkgs << path
       end
 
-      # Synchronize the upstream source to our local cache
-      fetch
-
       repo_path = File.join(Profile.config[:local_storage][:cache_dir], repo_config[:subdir])
       write_conf_file repo_path if repo_config[:auto]
 
@@ -56,9 +51,12 @@ module RightPublish
         end
       end
       rebuild_index(repo_config[:subdir]) if not repo_config[:auto]
+    end
 
-      # Commit back to remote storage
-      store
+    def annotate(options={})
+      options[:subdir] ||= File.join(repo_config[:subdir], 'pool', 'main')
+      options[:filter] = ['*.deb']
+      super(options)
     end
 
     def self.pkg_parts(pkg_name)

data/lib/right_publish/repos/gem.rb

@@ -1,24 +1,23 @@
 module RightPublish
-  class GemRepo
-    include RightPublish::Repo
-
+  class GemRepo < Repo
     DEFAULT_GEM_DIR = 'gems/'
     REPO_KEY = :gem_repo
-    REPO_OPTIONS = {:subdir=>DEFAULT_GEM_DIR}
+    REPO_OPTIONS = {:subdir=>DEFAULT_GEM_DIR, :prune=>false}
 
     GEMS_SUBDIRECTORY = 'gems'
     GEM_EXT = 'gem'
 
-    def publish(file_or_dir, target)
+    def add(file_or_dir, target)
       gems_list = get_pkg_list(file_or_dir, GEM_EXT)
 
-      # Synchronize the upstream source to our local cache
-      fetch
-
       # Copy gem files to the repository
       destination = File.join(repo_config[:subdir], GEMS_SUBDIRECTORY)
       gems_list.each do |path|
-        do_in_subdir(destination) { prune_all("#{pkg_parts(path)[:name]}-*.gem") }
+        # RubyGems repos do not normally prune; it must be explicitly enabled.
+        if repo_config[:prune]
+          do_in_subdir(destination) { prune_all("#{pkg_parts(path)[:name]}-*.gem") }
+        end
+
         install_file(path, destination)
       end
 
@@ -28,11 +27,16 @@ module RightPublish
         indexed = system('gem generate_index 2>&1 >/dev/null')  # requires 'builder' gem to be installed
         raise Exception, "gem generate_index failed; cannot continue publishing" unless indexed
       end
+    end
 
-      # Commit back to remote storage
-      store
+    def annotate(options={})
+      options[:subdir] ||= File.join(repo_config[:subdir], 'gems')
+      options[:filter] = ['*.gem']
+      super(options)
     end
 
+    protected
+
     def pkg_parts(name)
       result = {:name=>nil, :version=>nil}
       result[:name], result[:version] = /([A-Za-z0-9\-_]+)-([0-9\.]+)\.#{GEM_EXT}\Z/.match(name).captures

data/lib/right_publish/repos/msi.rb

@@ -0,0 +1,45 @@
+module RightPublish
+  class MsiRepo < Repo
+    DEFAULT_MSI_DIR = 'msi/'
+    REPO_KEY = :msi_repo
+    REPO_OPTIONS = {:subdir=>DEFAULT_MSI_DIR, :prune=>true}
+
+    GEMS_SUBDIRECTORY = 'gems'
+    MSI_EXT = 'msi'
+
+    def add(file_or_dir, target)
+      msi_list = get_pkg_list(file_or_dir, MSI_EXT)
+
+      # Copy MSI files to the repository
+      destination = repo_config[:subdir]
+      msi_list.each do |path|
+        # RubyGems repos do not normally prune; it must be explicitly enabled.
+        if repo_config[:prune]
+          do_in_subdir(destination) { prune_all("#{pkg_parts(path)[:name]}-*.msi") }
+        end
+
+        install_file(path, destination)
+      end
+
+      # No index generation is necessary; MSI is not a distribution mechanism. The annotated
+      # HTML will suffice to direct people to the proper download.
+    end
+
+    def annotate(options={})
+      options[:subdir] ||= repo_config[:subdir]
+      options[:filter] = ['*.msi']
+      options[:use_remote_storage] = true
+      super(options)
+    end
+
+    protected
+
+    def pkg_parts(name)
+      result = {:name=>nil, :version=>nil}
+      result[:name], result[:version] = /([A-Za-z0-9\-_]+)-([0-9\.]+)\.#{MSI_EXT}\Z/.match(name).captures
+      result
+    end
+  end
+
+  RightPublish::RepoManager.register_repo(MsiRepo)
+end

data/lib/right_publish/repos/yum.rb

@@ -1,13 +1,10 @@
 
 
 module RightPublish
-
-  class YumRepo
-    include RightPublish::Repo
-
+  class YumRepo < Repo
     DEFAULT_YUM_EPEL = 1
     DEFAULT_YUM_DIR = 'yum/'
-    DEFAULT_DESCRIPTION = "RightScale RightLink Repository"
+    DEFAULT_DESCRIPTION = "Yum Repository"
     REPO_KEY = :yum_repo
     DEFAULT_EPEL_LAYOUT = "rightlink"
     REPO_OPTIONS = {
@@ -27,7 +24,7 @@ module RightPublish
     ARCHITECTURES = [ 'i386', 'x86_64' ]
     PKG_TYPES = [ ARCHITECTURES, BIN_ALL_ARCH, SRC_ALL_ARCH ]
 
-    def publish(file_or_dir, target)
+    def add(file_or_dir, target)
       repo_updates = {}
 
       get_pkg_list(file_or_dir, YUM_EXT) do |path|
@@ -40,9 +37,6 @@ module RightPublish
       end
       return if repo_updates.size == 0
 
-      # Synchronize the upstream source to our local cache
-      fetch
-
       full_pkg_paths = []
       repo_updates.each_pair do |repo_path, pkgs|
         pkgs.each do |pkg|
@@ -57,9 +51,12 @@ module RightPublish
         regen_metadata( repo_path )
         sign_metadata( repo_path )
       end
+    end
 
-      # Commit back to remote storage
-      store
+    def annotate(options={})
+      options[:subdir] ||= File.join(repo_config[:subdir], '1')
+      options[:filter] = ['*.rpm']
+      super(options)
     end
 
     def import_pkg( pkg, repo_path )

data/lib/right_publish/storage.rb

@@ -29,11 +29,23 @@ module RightPublish
   end
 
   module Storage
+    def self.ls(bucket, options={})
+      storage = bucket.get_directories()
+      storage.files.each do |file|
+        # skip directories
+        next if file.key[-1,1] == '/'
+        # skip files not located in specified subdir
+        next if file.key[0,options[:subdir].size] != options[:subdir] if options.has_key? :subdir
+
+        yield(file)
+      end
+    end
+
     def self.sync_dirs(src, dest, options={})
       src_dir = src.get_directories()
       dest_dir = dest.get_directories()
 
-      src_map = get_etag_map(src_dir, options) {|file| file.etag}
+      src_map = get_etag_map(src_dir, options)
       dest_map = get_etag_map(dest_dir, options)
 
       dest_map.each_pair do |hash, files|
@@ -53,7 +65,9 @@ module RightPublish
         else
           remote_file = files.shift
           Profile.log("Synchronizing: #{remote_file.first}", :debug)
-          local_copy = dest_dir.files.create(:key=>remote_file.first, :body=>remote_file.last.body, :acl=>'public-read')
+          local_copy = dest_dir.files.create(:key=>remote_file.first, :body=>remote_file.last.body,
+                                             :acl=>'public-read',
+                                             :content_type => guess_mime_type(remote_file.first))
         end
 
         files.each_key do |file|
@@ -65,12 +79,37 @@ module RightPublish
       end
     end
 
-    private
+    # Given a filename, make an educated guess as to its MIME type. We rely on the mime-types
+    # gem and its built-in database of extensions and types, but many extensions are in use
+    # by multiple MIME types.
+    #
+    # If the extension maps to more than one MIME type, we will pick the first type that is
+    # a member of RightPublish::MIME_TYPES. This gives us 100% reliable behavior for MIME
+    # types that RightPublish "knows," and passable behavior for other types.
+    #
+    # @param [String] filename relative or absolute filename
+    # @return [String] the inferred MIME type for filename, based on its extension
+    def self.guess_mime_type(filename)
+      filename = File.basename(filename)
+      candidates = MIME::Types.type_for(filename)
+
+      if candidates.size == 1
+        winner = candidates.first
+      elsif preferred = candidates.detect { |c| RightPublish::MIME_TYPES.keys.include? c.to_s }
+        winner = preferred
+      else
+        winner = 'application/octet-stream'
+      end
+
+      winner.to_s
+    end
 
     def self.get_etag_map(dir, options={})
       hash_map = {}
       dir.files.each do |file|
+        # skip directories
         next if file.key[-1,1] == '/'
+        # skip files not located in specified subdir
         next if file.key[0,options[:subdir].size] != options[:subdir] if options.has_key? :subdir
 
         md5 = dir.compute_md5(file)

data/lib/right_publish/stores/local.rb

@@ -3,10 +3,10 @@ require 'digest/md5'
 
 module RightPublish
   module LocalStorage
-
-    DEFAULT_LOCAL_CACHEDIR = '~/.rp_cache'
+    DEFAULT_LOCAL_CACHEDIR = File.expand_path('~/.right_publish/cache')
     STORAGE_KEY = :local_storage
-    STORAGE_OPTIONS = { :cache_dir=>DEFAULT_LOCAL_CACHEDIR}
+    STORAGE_OPTIONS = { :cache_dir=>DEFAULT_LOCAL_CACHEDIR }
+
     def compute_md5(file)
       Digest::MD5.file(File.join(service.path_to(self.key), file.key)).hexdigest
     end

data/lib/right_publish.rb

@@ -3,95 +3,33 @@
 #   RightScale RightPublish (right_publish) - (c) 2013 RightScale Inc
 #
 
-require 'trollop'
-
-require 'right_publish/profile'
-require 'right_publish/repo'
+require 'mime/types'
 
 module RightPublish
-  module Main
-    REPOSITORY_TYPES = RepoManager.repo_types()
-    SUB_COMMANDS = %w(publish fetch store)
-
-    def self.run()
-      options = parse_args
-
-      begin
-        profile = RightPublish::Profile.instance
-        profile.load(options[:profile])
-      rescue LoadError => e
-        puts e
-        exit -1
-      end
-
-      # These options override profile attributes.
-      profile.settings[:local_storage][:cache_dir] = options[:local_cache] if options[:local_cache]
-      profile.settings[:verbose] = true if options[:verbose]
-
-      # We already know this is a valid type, parse_args checked
-      repo = RepoManager.get_repository(REPOSITORY_TYPES[options[:repo_type]])
-      begin
-        case options[:cmd]
-        when "publish"
-          repo.publish(options[:files], options[:dist])
-        when "store"
-          repo.store
-        when "fetch"
-          repo.fetch
-        else
-        end
-      rescue RuntimeError => e
-        RightPublish::Profile.log("Fatal Error:\n\t#{e}", :error)
-        exit -1
-      end
-      RightPublish::Profile.log("Success!")
-    end
-
-    def self.parse_args()
-      options = Trollop.options do
-        version "RightPublish alpha (c) 2013 RightScale Inc"
-        banner <<-EOS
-RightPublish can manage a YUM/APT/RubyGem repository in remote storage, e.g. S3.
-
-Usage:
-       right_publish [global_options] <command> [file1, file2, ...]
-
-commands:
-  fetch
-  publish
-  store
-
-global options:
-EOS
-
-        opt :local_cache,       "Local cache location", :type => String
-        opt :profile,           "Publish profile", :type => String
-        opt :repo_type,         "Repository type: #{REPOSITORY_TYPES.keys.inspect}", :type => String
-        opt :dist,              "Target distribution. Required for binary packages. If unspecified for noarch and source packages, will copy to all distributions specified in profile.", :type => String
-        opt :verbose,           "Verbose output"
-
-        stop_on SUB_COMMANDS
-      end
-
-      options[:cmd]= ARGV.shift
-      options[:files] = expand_argv_globs
-
-      Trollop.die "argument --profile is required" unless options[:profile]
-      Trollop.die "argument --repo-type is required" unless options[:repo_type]
-      Trollop.die "profile does not exist: #{options[:profile]}" unless File.exist?(options[:profile])
-      Trollop.die "invalid repository type: #{options[:repo_type]}" unless REPOSITORY_TYPES[options[:repo_type]]
-
-      options
-    end
-
-    def self.expand_argv_globs
-      files = []
-
-      ARGV.each do |glob|
-        files += Dir.glob(glob)
-      end
+  # Combined list of MIME types that should be registered with mime-types, AND preferred
+  # MIME types for this application to use, in cases where a given filename extension is
+  # used by multiple MIME types.
+  MIME_TYPES = {
+    'application/x-deb'            => 'deb',
+    'application/x-ruby-gem'       => 'gem',
+    'application/x-rpm'            => 'rpm',
+    'application/x-msi'            => 'msi',
+    'text/xml'                     => 'xml',
+  }
+end
 
-      files
+# Register some extra MIME types that the gem may not know about.
+RightPublish::MIME_TYPES.each_pair do |name, extension|
+  if MIME::Types[name].empty?
+    type = MIME::Type.new(name) do |t|
+      t.extensions = [extension]
+      t.encoding = '8bit'
     end
+    MIME::Types.add(type)
   end
 end
+
+require 'right_publish/profile'
+require 'right_publish/annotation'
+require 'right_publish/repo'
+require 'right_publish/cli'

data/right_publish.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "right_publish"
-  s.version = "0.3.0"
+  s.version = "0.4.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Brian Szmyd", "Tony Spataro"]
-  s.date = "2013-08-12"
+  s.date = "2013-11-01"
   s.description = "A tool for maintaining S3-based DEB, GEM and RPM packages."
   s.email = "support@rightscale.com"
   s.executables = ["autosign.expect", "right_publish"]
@@ -27,16 +27,20 @@ Gem::Specification.new do |s|
     "bin/autosign.expect",
     "bin/right_publish",
     "lib/right_publish.rb",
+    "lib/right_publish/annotation.rb",
+    "lib/right_publish/cli.rb",
     "lib/right_publish/profile.rb",
     "lib/right_publish/repo.rb",
     "lib/right_publish/repos/apt.rb",
     "lib/right_publish/repos/gem.rb",
+    "lib/right_publish/repos/msi.rb",
     "lib/right_publish/repos/yum.rb",
     "lib/right_publish/repos/zypp.rb",
     "lib/right_publish/storage.rb",
     "lib/right_publish/stores/local.rb",
     "lib/right_publish/stores/s3.rb",
     "right_publish.gemspec",
+    "spec/annotation_spec.rb",
     "spec/repo_manager_spec.rb",
     "spec/repo_spec.rb",
     "spec/repos/apt_spec.rb",
@@ -51,7 +55,7 @@ Gem::Specification.new do |s|
   s.homepage = "https://github.com/rightscale/right_publish"
   s.licenses = ["Proprietary"]
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.24"
+  s.rubygems_version = "1.8.23"
   s.summary = "Package publishing and indexing tool"
 
   if s.respond_to? :specification_version then
@@ -60,6 +64,8 @@ Gem::Specification.new do |s|
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<builder>, [">= 0"])
       s.add_runtime_dependency(%q<fog>, ["~> 1.9"])
+      s.add_runtime_dependency(%q<excon>, ["<= 0.25.3"])
+      s.add_runtime_dependency(%q<mime-types>, ["~> 1.0"])
       s.add_runtime_dependency(%q<trollop>, ["~> 2.0"])
       s.add_development_dependency(%q<rake>, ["~> 0.9"])
       s.add_development_dependency(%q<jeweler>, ["~> 1.8.3"])
@@ -73,6 +79,8 @@ Gem::Specification.new do |s|
     else
       s.add_dependency(%q<builder>, [">= 0"])
       s.add_dependency(%q<fog>, ["~> 1.9"])
+      s.add_dependency(%q<excon>, ["<= 0.25.3"])
+      s.add_dependency(%q<mime-types>, ["~> 1.0"])
       s.add_dependency(%q<trollop>, ["~> 2.0"])
       s.add_dependency(%q<rake>, ["~> 0.9"])
       s.add_dependency(%q<jeweler>, ["~> 1.8.3"])
@@ -87,6 +95,8 @@ Gem::Specification.new do |s|
   else
     s.add_dependency(%q<builder>, [">= 0"])
     s.add_dependency(%q<fog>, ["~> 1.9"])
+    s.add_dependency(%q<excon>, ["<= 0.25.3"])
+    s.add_dependency(%q<mime-types>, ["~> 1.0"])
     s.add_dependency(%q<trollop>, ["~> 2.0"])
     s.add_dependency(%q<rake>, ["~> 0.9"])
     s.add_dependency(%q<jeweler>, ["~> 1.8.3"])