puppet 3.5.1-x86-mingw32 → 3.6.0.rc1-x86-mingw32

data/lib/puppet/face/module/list.rb

@@ -58,85 +58,22 @@ Puppet::Face.define(:module, '1.0.0') do
     EOT
 
     when_invoked do |options|
-      environment_from_options(options).modules_by_path
-    end
+      Puppet::ModuleTool.set_option_defaults(options)
+      environment = options[:environment_instance]
 
-    when_rendering :console do |modules_by_path, options|
-      output = ''
-      environment = environment_from_options(options)
-
-      error_types = {
-        :non_semantic_version => {
-          :title => "Non semantic version dependency"
-        },
-        :missing => {
-          :title => "Missing dependency"
-        },
-        :version_mismatch => {
-          :title => "Module '%s' (v%s) fails to meet some dependencies:"
-        }
+      {
+        :environment     => environment,
+        :modules_by_path => environment.modules_by_path,
       }
+    end
 
-      @unmet_deps = {}
-      error_types.each_key do |type|
-        @unmet_deps[type] = Hash.new do |hash, key|
-          hash[key] = { :errors => [], :parent => nil }
-        end
-      end
+    when_rendering :console do |result, options|
+      environment     = result[:environment]
+      modules_by_path = result[:modules_by_path]
 
-      # Prepare the unmet dependencies for display on the console.
-      environment.modules.sort_by {|mod| mod.name}.each do |mod|
-        unmet_grouped = Hash.new { |h,k| h[k] = [] }
-        unmet_grouped = mod.unmet_dependencies.inject(unmet_grouped) do |acc, dep|
-          acc[dep[:reason]] << dep
-          acc
-        end
-        unmet_grouped.each do |type, deps|
-          unless deps.empty?
-            unmet_grouped[type].sort_by { |dep| dep[:name] }.each do |dep|
-              dep_name           = dep[:name].gsub('/', '-')
-              installed_version  = dep[:mod_details][:installed_version]
-              version_constraint = dep[:version_constraint]
-              parent_name        = dep[:parent][:name].gsub('/', '-')
-              parent_version     = dep[:parent][:version]
-
-              msg = "'#{parent_name}' (#{parent_version})"
-              msg << " requires '#{dep_name}' (#{version_constraint})"
-              @unmet_deps[type][dep[:name]][:errors] << msg
-              @unmet_deps[type][dep[:name]][:parent] = {
-                :name    => dep[:parent][:name],
-                :version => parent_version
-              }
-              @unmet_deps[type][dep[:name]][:version] = installed_version
-            end
-          end
-        end
-      end
+      output = ''
 
-      # Display unmet dependencies by category.
-      error_display_order = [:non_semantic_version, :version_mismatch, :missing]
-      error_display_order.each do |type|
-        unless @unmet_deps[type].empty?
-          @unmet_deps[type].keys.sort_by {|dep| dep }.each do |dep|
-            name    = dep.gsub('/', '-')
-            title   = error_types[type][:title]
-            errors  = @unmet_deps[type][dep][:errors]
-            version = @unmet_deps[type][dep][:version]
-
-            msg = case type
-                  when :version_mismatch
-                    title % [name, version] + "\n"
-                  when :non_semantic_version
-                    title + " '#{name}' (v#{version}):\n"
-                  else
-                    title + " '#{name}':\n"
-                  end
-
-            errors.each { |error_string| msg << "  #{error_string}\n" }
-            Puppet.warning msg.chomp
-          end
-        end
-      end
+      warn_unmet_dependencies(environment)
 
       environment.modulepath.each do |path|
         modules = modules_by_path[path]
@@ -166,14 +103,78 @@ Puppet::Face.define(:module, '1.0.0') do
     end
   end
 
-  def environment_from_options(options)
-    environments = Puppet.lookup(:environments)
-    if options[:modulepath]
-      environments.for(options[:modulepath], '')
-    elsif options[:environment]
-      environments.get(options[:environment])
-    else
-      environments.get(Puppet[:environment])
+  def warn_unmet_dependencies(environment)
+    error_types = {
+      :non_semantic_version => {
+        :title => "Non semantic version dependency"
+      },
+      :missing => {
+        :title => "Missing dependency"
+      },
+      :version_mismatch => {
+        :title => "Module '%s' (v%s) fails to meet some dependencies:"
+      }
+    }
+
+    @unmet_deps = {}
+    error_types.each_key do |type|
+      @unmet_deps[type] = Hash.new do |hash, key|
+        hash[key] = { :errors => [], :parent => nil }
+      end
+    end
+
+    # Prepare the unmet dependencies for display on the console.
+    environment.modules.sort_by {|mod| mod.name}.each do |mod|
+      unmet_grouped = Hash.new { |h,k| h[k] = [] }
+      unmet_grouped = mod.unmet_dependencies.inject(unmet_grouped) do |acc, dep|
+        acc[dep[:reason]] << dep
+        acc
+      end
+      unmet_grouped.each do |type, deps|
+        unless deps.empty?
+          unmet_grouped[type].sort_by { |dep| dep[:name] }.each do |dep|
+            dep_name           = dep[:name].gsub('/', '-')
+            installed_version  = dep[:mod_details][:installed_version]
+            version_constraint = dep[:version_constraint]
+            parent_name        = dep[:parent][:name].gsub('/', '-')
+            parent_version     = dep[:parent][:version]
+
+            msg = "'#{parent_name}' (#{parent_version})"
+            msg << " requires '#{dep_name}' (#{version_constraint})"
+            @unmet_deps[type][dep[:name]][:errors] << msg
+            @unmet_deps[type][dep[:name]][:parent] = {
+              :name    => dep[:parent][:name],
+              :version => parent_version
+            }
+            @unmet_deps[type][dep[:name]][:version] = installed_version
+          end
+        end
+      end
+    end
+
+    # Display unmet dependencies by category.
+    error_display_order = [:non_semantic_version, :version_mismatch, :missing]
+    error_display_order.each do |type|
+      unless @unmet_deps[type].empty?
+        @unmet_deps[type].keys.sort_by {|dep| dep }.each do |dep|
+          name    = dep.gsub('/', '-')
+          title   = error_types[type][:title]
+          errors  = @unmet_deps[type][dep][:errors]
+          version = @unmet_deps[type][dep][:version]
+
+          msg = case type
+                when :version_mismatch
+                  title % [name, version] + "\n"
+                when :non_semantic_version
+                  title + " '#{name}' (v#{version}):\n"
+                else
+                  title + " '#{name}':\n"
+                end
+
+          errors.each { |error_string| msg << "  #{error_string}\n" }
+          Puppet.warning msg.chomp
+        end
+      end
     end
   end
 

data/lib/puppet/face/module/search.rb

@@ -23,7 +23,7 @@ Puppet::Face.define(:module, '1.0.0') do
 
     when_invoked do |term, options|
       Puppet::ModuleTool.set_option_defaults options
-      Puppet::ModuleTool::Applications::Searcher.new(term, Puppet::Forge.new("PMT", self.version), options).run
+      Puppet::ModuleTool::Applications::Searcher.new(term, Puppet::Forge.new, options).run
     end
 
     when_rendering :console do |results, term, options|

data/lib/puppet/face/module/upgrade.rb

@@ -40,9 +40,9 @@ Puppet::Face.define(:module, '1.0.0') do
     end
 
     option "--ignore-dependencies" do
-      summary "Do not attempt to install dependencies."
+      summary "Do not attempt to install dependencies"
       description <<-EOT
-        Do not attempt to install dependencies
+        Do not attempt to install dependencies.  (Implied by --force.)
       EOT
     end
 
@@ -57,19 +57,20 @@ Puppet::Face.define(:module, '1.0.0') do
       name = name.gsub('/', '-')
       Puppet.notice "Preparing to upgrade '#{name}' ..."
       Puppet::ModuleTool.set_option_defaults options
-      Puppet::ModuleTool::Applications::Upgrader.new(name, Puppet::Forge.new("PMT", self.version), options).run
+      Puppet::ModuleTool::Applications::Upgrader.new(name, options).run
     end
 
     when_rendering :console do |return_value|
-      if return_value[:result] == :failure
+      if return_value[:result] == :noop
+        Puppet.notice return_value[:error][:multiline]
+        exit 0
+      elsif return_value[:result] == :failure
         Puppet.err(return_value[:error][:multiline])
         exit 1
-      elsif return_value[:result] == :noop
-        Puppet.err(return_value[:error][:multiline])
-        exit 0
       else
-        tree = Puppet::ModuleTool.build_tree(return_value[:affected_modules], return_value[:base_dir])
-        return_value[:base_dir] + "\n" +
+        tree = Puppet::ModuleTool.build_tree(return_value[:graph], return_value[:base_dir])
+
+        "#{return_value[:base_dir]}\n" +
         Puppet::ModuleTool.format_tree(tree)
       end
     end

data/lib/puppet/face/parser.rb

@@ -37,8 +37,9 @@ Puppet::Face.define(:parser, '0.0.1') do
           Puppet[:code] = STDIN.read
           validate_manifest
         else
-           files << Puppet[:manifest]
-           Puppet.notice "No manifest specified. Validating the default manifest #{Puppet[:manifest]}"
+          manifest = Puppet.lookup(:current_environment).manifest
+          files << manifest
+          Puppet.notice "No manifest specified. Validating the default manifest #{manifest}"
         end
       end
       missing_files = []

data/lib/puppet/face/plugin.rb

@@ -38,16 +38,20 @@ Puppet::Face.define(:plugin, '0.0.1') do
 
     when_invoked do |options|
       require 'puppet/configurer/downloader'
-      Puppet::Configurer::Downloader.new("plugin",
+      remote_environment_for_plugins = Puppet::Node::Environment.remote(Puppet[:environment])
+      result = Puppet::Configurer::Downloader.new("plugin",
                                          Puppet[:plugindest],
                                          Puppet[:pluginsource],
-                                         Puppet[:pluginsignore]).evaluate
+                                         Puppet[:pluginsignore],
+                                         remote_environment_for_plugins).evaluate
       if Puppet.features.external_facts?
-          Puppet::Configurer::Downloader.new("pluginfacts",
+          result += Puppet::Configurer::Downloader.new("pluginfacts",
                                              Puppet[:pluginfactdest],
                                              Puppet[:pluginfactsource],
-                                             Puppet[:pluginsignore]).evaluate
+                                             Puppet[:pluginsignore],
+                                             remote_environment_for_plugins).evaluate
       end
+      result
     end
 
     when_rendering :console do |value|

data/lib/puppet/file_bucket/dipper.rb

@@ -4,6 +4,7 @@ require 'puppet/file_bucket/file'
 require 'puppet/indirector/request'
 
 class Puppet::FileBucket::Dipper
+  include Puppet::Util::Checksums
   # This is a transitional implementation that uses REST
   # to access remote filebucket files.
 
@@ -23,6 +24,8 @@ class Puppet::FileBucket::Dipper
       @local_path = nil
       @rest_path = "https://#{server}:#{port}/#{environment}/file_bucket_file/"
     end
+    @checksum_type = Puppet[:digest_algorithm].to_sym
+    @digest = method(@checksum_type)
   end
 
   def local?
@@ -56,7 +59,7 @@ class Puppet::FileBucket::Dipper
 
   # Retrieve a file by sum.
   def getfile(sum)
-    source_path = "#{@rest_path}md5/#{sum}"
+    source_path = "#{@rest_path}#{@checksum_type}/#{sum}"
     file_bucket_file = Puppet::FileBucket::File.indirection.find(source_path, :bucket_path => @local_path)
 
     raise Puppet::Error, "File not found" unless file_bucket_file
@@ -68,7 +71,7 @@ class Puppet::FileBucket::Dipper
     restore = true
     file_handle = Puppet::FileSystem.pathname(file)
     if Puppet::FileSystem.exist?(file_handle)
-      cursum = Digest::MD5.hexdigest(Puppet::FileSystem.binread(file_handle))
+      cursum = @digest.call(Puppet::FileSystem.binread(file_handle))
 
       # if the checksum has changed...
       # this might be extra effort
@@ -79,7 +82,7 @@ class Puppet::FileBucket::Dipper
 
     if restore
       if newcontents = getfile(sum)
-        newsum = Digest::MD5.hexdigest(newcontents)
+        newsum = @digest.call(newcontents)
         changed = nil
         if Puppet::FileSystem.exist?(file_handle) and ! Puppet::FileSystem.writable?(file_handle)
           changed = Puppet::FileSystem.stat(file_handle).mode

data/lib/puppet/file_bucket/file.rb

@@ -32,6 +32,7 @@ class Puppet::FileBucket::File
     @contents = contents
 
     @bucket_path = options.delete(:bucket_path)
+    @checksum_type = Puppet[:digest_algorithm].to_sym
     raise ArgumentError.new("Unknown option(s): #{options.keys.join(', ')}") unless options.empty?
   end
 
@@ -46,7 +47,7 @@ class Puppet::FileBucket::File
   end
 
   def checksum_type
-    'md5'
+    @checksum_type.to_s
   end
 
   def checksum
@@ -54,7 +55,8 @@ class Puppet::FileBucket::File
   end
 
   def checksum_data
-    @checksum_data ||= Digest::MD5.hexdigest(contents)
+    algorithm = Puppet::Util::Checksums.method(@checksum_type)
+    @checksum_data ||= algorithm.call(contents)
   end
 
   def to_s

data/lib/puppet/file_serving/metadata.rb

@@ -149,7 +149,7 @@ class Puppet::FileServing::Metadata < Puppet::FileServing::Base
       @checksum_type = checksum['type']
       @checksum      = checksum['value']
     end
-    @checksum_type ||= "md5"
+    @checksum_type ||= Puppet[:digest_algorithm]
     @ftype       = data.delete('type')
     @destination = data.delete('destination')
     super(path,data)

data/lib/puppet/file_system/memory_file.rb

@@ -36,10 +36,36 @@ class Puppet::FileSystem::MemoryFile
   def executable?; @properties[:executable?]; end
 
   def each_line(&block)
-    StringIO.new(@properties[:content]).each_line(&block)
+    handle.each_line(&block)
+  end
+
+  def handle
+    raise Errno::ENOENT unless exist?
+    StringIO.new(@properties[:content] || '')
   end
 
   def duplicate_as(other_path)
     self.class.new(other_path, @properties)
   end
+
+  def absolute?
+    Pathname.new(path).absolute?
+  end
+
+  def to_path
+    path
+  end
+
+  def to_s
+    to_path
+  end
+
+  # Used by Ruby 1.8.7 file system abstractions when operating on Pathname like things.
+  def to_str
+    to_path
+  end
+
+  def inspect
+    "<Puppet::FileSystem::MemoryFile:#{to_s}>"
+  end
 end

data/lib/puppet/file_system/memory_impl.rb

@@ -28,7 +28,7 @@ class Puppet::FileSystem::MemoryImpl
   end
 
   def pathname(path)
-    find(path)
+    find(path) || Puppet::FileSystem::MemoryFile.a_missing_file(path)
   end
 
   def basename(path)
@@ -39,6 +39,20 @@ class Puppet::FileSystem::MemoryImpl
     object.path
   end
 
+  def read(path)
+    handle = assert_path(path).handle
+    handle.read
+  end
+
+  def open(path, *args, &block)
+    handle = assert_path(path).handle
+    if block_given?
+      yield handle
+    else
+      return handle
+    end
+  end
+
   def assert_path(path)
     if path.is_a?(Puppet::FileSystem::MemoryFile)
       path

data/lib/puppet/forge.rb

@@ -1,21 +1,27 @@
+require 'puppet/vendor'
+Puppet::Vendor.load_vendored
+
 require 'net/http'
-require 'open-uri'
-require 'pathname'
+require 'tempfile'
 require 'uri'
+require 'pathname'
+require 'json'
+require 'semantic'
 
-class Puppet::Forge
-  require 'puppet/forge/errors'
+class Puppet::Forge < Semantic::Dependency::Source
   require 'puppet/forge/cache'
   require 'puppet/forge/repository'
+  require 'puppet/forge/errors'
 
   include Puppet::Forge::Errors
 
-  # +consumer_name+ is a name to be used for identifying the consumer of the
-  # forge and +consumer_semver+ is a SemVer object to identify the version of
-  # the consumer
-  def initialize(consumer_name, consumer_semver)
-    @consumer_name = consumer_name
-    @consumer_semver = consumer_semver
+  USER_AGENT = "PMT/1.1.1 (v3; Net::HTTP)".freeze
+
+  attr_reader :host, :repository
+
+  def initialize(host = Puppet[:module_repository])
+    @host = host
+    @repository = Puppet::Forge::Repository.new(host, USER_AGENT)
   end
 
   # Return a list of module metadata hashes that match the search query.
@@ -46,61 +52,151 @@ class Puppet::Forge
   # @raise [Puppet::Forge::Errors::ResponseError] if the repository returns a
   #   bad HTTP response
   def search(term)
-    server = Puppet.settings[:module_repository]
-    Puppet.notice "Searching #{server} ..."
-    response = repository.make_http_request("/modules.json?q=#{URI.escape(term)}")
-
-    case response.code
-    when "200"
-      matches = PSON.parse(response.body)
-    else
-      raise ResponseError.new(:uri => uri.to_s, :input => term, :response => response)
+    matches = []
+    uri = "/v3/modules?query=#{URI.escape(term)}"
+
+    while uri
+      response = make_http_request(uri)
+
+      if response.code == '200'
+        result = JSON.parse(response.body)
+        uri = result['pagination']['next']
+        matches.concat result['results']
+      else
+        raise ResponseError.new(:uri => URI.parse(@host).merge(uri) , :input => term, :response => response)
+      end
     end
 
-    matches
+    matches.each do |mod|
+      mod['author'] = mod['owner']['username']
+      mod['tag_list'] = mod['current_release']['tags']
+      mod['full_name'] = "#{mod['author']}/#{mod['name']}"
+      mod['version'] = mod['current_release']['version']
+      mod['project_url'] = mod['homepage_url']
+      mod['desc'] = mod['current_release']['metadata']['summary'] || ''
+    end
   end
 
-  # Return a list of module metadata hashes for the module requested and all
-  # of its dependencies.
+  # Fetches {ModuleRelease} entries for each release of the named module.
   #
-  # @param author [String] module's author name
-  # @param mod_name [String] module name
-  # @param version [String] optional module version number
-  # @return [Array] module and dependency metadata
-  # @raise [Puppet::Forge::Errors::CommunicationError] if there is a network
-  #   related error
-  # @raise [Puppet::Forge::Errors::SSLVerifyError] if there is a problem
-  #   verifying the remote SSL certificate
-  # @raise [Puppet::Forge::Errors::ResponseError] if the repository returns
-  #   an error in its API response or a bad HTTP response
-  def remote_dependency_info(author, mod_name, version)
-    version_string = version ? "&version=#{version}" : ''
-    response = repository.make_http_request("/api/v1/releases.json?module=#{author}/#{mod_name}#{version_string}")
-    json = PSON.parse(response.body) rescue {}
-    case response.code
-    when "200"
-      return json
-    else
-      error = json['error']
-      if error && error =~ /^Module #{author}\/#{mod_name} has no release/
-        return []
+  # @param input [String] the module name to look up
+  # @return [Array<Semantic::Dependency::ModuleRelease>] a list of releases for
+  #         the given name
+  # @see Semantic::Dependency::Source#fetch
+  def fetch(input)
+    name = input.tr('/', '-')
+    uri = "/v3/releases?module=#{name}"
+    releases = []
+
+    while uri
+      response = make_http_request(uri)
+
+      if response.code == '200'
+        response = JSON.parse(response.body)
       else
-        raise ResponseError.new(:uri => uri.to_s, :input => "#{author}/#{mod_name}", :message => error, :response => response)
+        raise ResponseError.new(:uri => URI.parse(@host).merge(uri), :input => input, :response => response)
       end
+
+      releases.concat(process(response['results']))
+      uri = response['pagination']['next']
     end
+
+    return releases
   end
 
-  def retrieve(release)
-    repository.retrieve(release)
+  def make_http_request(*args)
+    @repository.make_http_request(*args)
   end
 
-  def uri
-    repository.uri
+  class ModuleRelease < Semantic::Dependency::ModuleRelease
+    attr_reader :install_dir, :metadata
+
+    def initialize(source, data)
+      @data = data
+      @metadata = meta = data['metadata']
+
+      name = meta['name'].tr('/', '-')
+      version = Semantic::Version.parse(meta['version'])
+
+      dependencies = (meta['dependencies'] || [])
+      dependencies.map! do |dep|
+        range = dep['version_requirement'] || dep['versionRequirement'] || '>=0'
+        [
+          dep['name'].tr('/', '-'),
+          (Semantic::VersionRange.parse(range) rescue Semantic::VersionRange::EMPTY_RANGE),
+        ]
+      end
+
+      super(source, name, version, Hash[dependencies])
+    end
+
+    def install(dir)
+      staging_dir = self.prepare
+
+      module_dir = dir + name[/-(.*)/, 1]
+      module_dir.rmtree if module_dir.exist?
+
+      # Make sure unpacked module has the same ownership as the folder we are moving it into.
+      Puppet::ModuleTool::Applications::Unpacker.harmonize_ownership(dir, staging_dir)
+
+      FileUtils.mv(staging_dir, module_dir)
+      @install_dir = dir
+
+      # Return the Pathname object representing the directory where the
+      # module release archive was unpacked the to.
+      return module_dir
+    ensure
+      staging_dir.rmtree if staging_dir.exist?
+    end
+
+    def prepare
+      return @unpacked_into if @unpacked_into
+
+      download(@data['file_uri'], tmpfile)
+      validate_checksum(tmpfile, @data['file_md5'])
+      unpack(tmpfile, tmpdir)
+
+      @unpacked_into = Pathname.new(tmpdir)
+    end
+
+    private
+
+    # Obtain a suitable temporary path for unpacking tarballs
+    #
+    # @return [Pathname] path to temporary unpacking location
+    def tmpdir
+      @dir ||= Dir.mktmpdir(name, Puppet::Forge::Cache.base_path)
+    end
+
+    def tmpfile
+      @file ||= Tempfile.new(name, Puppet::Forge::Cache.base_path).tap do |f|
+        f.binmode
+      end
+    end
+
+    def download(uri, destination)
+      @source.make_http_request(uri, destination)
+      destination.flush and destination.close
+    end
+
+    def validate_checksum(file, checksum)
+      if Digest::MD5.file(file.path).hexdigest != checksum
+        raise RuntimeError, "Downloaded release for #{name} did not match expected checksum"
+      end
+    end
+
+    def unpack(file, destination)
+      begin
+        Puppet::ModuleTool::Applications::Unpacker.unpack(file.path, destination)
+      rescue Puppet::ExecutionFailure => e
+        raise RuntimeError, "Could not extract contents of module archive: #{e.message}"
+      end
+    end
   end
 
-  def repository
-    version = "#{@consumer_name}/#{[@consumer_semver.major, @consumer_semver.minor, @consumer_semver.tiny].join('.')}#{@consumer_semver.special}"
-    @repository ||= Puppet::Forge::Repository.new(Puppet[:module_repository], version)
+  private
+
+  def process(list)
+    list.map { |release| ModuleRelease.new(self, release) }
   end
-  private :repository
 end