puppet-parse 0.0.1 → 0.0.2

data/lib/vendor/puppet/file_serving/terminus_helper.rb

@@ -0,0 +1,21 @@
+require 'puppet/file_serving'
+require 'puppet/file_serving/fileset'
+
+# Define some common methods for FileServing termini.
+module Puppet::FileServing::TerminusHelper
+  # Create model instances for all files in a fileset.
+  def path2instances(request, *paths)
+    filesets = paths.collect do |path|
+      # Filesets support indirector requests as an options collection
+      Puppet::FileServing::Fileset.new(path, request)
+    end
+
+    Puppet::FileServing::Fileset.merge(*filesets).collect do |file, base_path|
+      inst = model.new(base_path, :relative_path => file)
+      inst.checksum_type = request.options[:checksum_type] if request.options[:checksum_type]
+      inst.links = request.options[:links] if request.options[:links]
+      inst.collect
+      inst
+    end
+  end
+end

data/lib/vendor/puppet/file_serving/terminus_selector.rb

@@ -0,0 +1,28 @@
+require 'puppet/file_serving'
+
+# This module is used to pick the appropriate terminus
+# in file-serving indirections.  This is necessary because
+# the terminus varies based on the URI asked for.
+module Puppet::FileServing::TerminusSelector
+  PROTOCOL_MAP = {"puppet" => :rest, "file" => :file}
+
+  def select(request)
+    # We rely on the request's parsing of the URI.
+
+    # Short-circuit to :file if it's a fully-qualified path or specifies a 'file' protocol.
+    return PROTOCOL_MAP["file"] if Puppet::Util.absolute_path?(request.key)
+    return PROTOCOL_MAP["file"] if request.protocol == "file"
+
+    # We're heading over the wire the protocol is 'puppet' and we've got a server name or we're not named 'apply' or 'puppet'
+    if request.protocol == "puppet" and (request.server or !["puppet","apply"].include?(Puppet.settings[:name]))
+      return PROTOCOL_MAP["puppet"]
+    end
+
+    if request.protocol and PROTOCOL_MAP[request.protocol].nil?
+      raise(ArgumentError, "URI protocol '#{request.protocol}' is not currently supported for file serving")
+    end
+
+    # If we're still here, we're using the file_server or modules.
+    :file_server
+  end
+end

data/lib/vendor/puppet/file_serving.rb

@@ -0,0 +1,3 @@
+# Just a stub class.
+class Puppet::FileServing # :nodoc:
+end

data/lib/vendor/puppet/forge/cache.rb

@@ -0,0 +1,55 @@
+require 'uri'
+
+module Puppet::Forge
+  # = Cache
+  #
+  # Provides methods for reading files from local cache, filesystem or network.
+  class Cache
+
+    # Instantiate new cahe for the +repositry+ instance.
+    def initialize(repository, options = {})
+      @repository = repository
+      @options = options
+    end
+
+    # Return filename retrieved from +uri+ instance. Will download this file and
+    # cache it if needed.
+    #
+    # TODO: Add checksum support.
+    # TODO: Add error checking.
+    def retrieve(url)
+      (path + File.basename(url.to_s)).tap do |cached_file|
+        uri = url.is_a?(::URI) ? url : ::URI.parse(url)
+        unless cached_file.file?
+          if uri.scheme == 'file'
+            FileUtils.cp(URI.unescape(uri.path), cached_file)
+          else
+            # TODO: Handle HTTPS; probably should use repository.contact
+            data = read_retrieve(uri)
+            cached_file.open('wb') { |f| f.write data }
+          end
+        end
+      end
+    end
+
+    # Return contents of file at the given URI's +uri+.
+    def read_retrieve(uri)
+      return uri.read
+    end
+
+    # Return Pathname for repository's cache directory, create it if needed.
+    def path
+      (self.class.base_path + @repository.cache_key).tap{ |o| o.mkpath }
+    end
+
+    # Return the base Pathname for all the caches.
+    def self.base_path
+      Pathname(Puppet.settings[:module_working_dir]) + 'cache'
+    end
+
+    # Clean out all the caches.
+    def self.clean
+      base_path.rmtree if base_path.exist?
+    end
+  end
+end

data/lib/vendor/puppet/forge/repository.rb

@@ -0,0 +1,102 @@
+require 'net/http'
+require 'digest/sha1'
+require 'uri'
+
+module Puppet::Forge
+  # = Repository
+  #
+  # This class is a file for accessing remote repositories with modules.
+  class Repository
+
+    attr_reader :uri, :cache
+
+    # Instantiate a new repository instance rooted at the optional string
+    # +url+, else an instance of the default Puppet modules repository.
+    def initialize(url=Puppet[:module_repository])
+      @uri = url.is_a?(::URI) ? url : ::URI.parse(url.sub(/^(?!https?:\/\/)/, 'http://'))
+      @cache = Cache.new(self)
+    end
+
+    # Read HTTP proxy configurationm from Puppet's config file, or the
+    # http_proxy environment variable.
+    def http_proxy_env
+      proxy_env = ENV["http_proxy"] || ENV["HTTP_PROXY"] || nil
+      begin
+        return URI.parse(proxy_env) if proxy_env
+      rescue URI::InvalidURIError
+        return nil
+      end
+      return nil
+    end
+
+    def http_proxy_host
+      env = http_proxy_env
+
+      if env and env.host then
+        return env.host
+      end
+
+      if Puppet.settings[:http_proxy_host] == 'none'
+        return nil
+      end
+
+      return Puppet.settings[:http_proxy_host]
+    end
+
+    def http_proxy_port
+      env = http_proxy_env
+
+      if env and env.port then
+        return env.port
+      end
+
+      return Puppet.settings[:http_proxy_port]
+    end
+
+    # Return a Net::HTTPResponse read for this +request+.
+    def make_http_request(request, options = {})
+      if ! @uri.user.nil? && ! @uri.password.nil?
+        request.basic_auth(@uri.user, @uri.password)
+      end
+      return read_response(request)
+    end
+
+    # Return a Net::HTTPResponse read from this HTTPRequest +request+.
+    def read_response(request)
+      begin
+        Net::HTTP::Proxy(
+            http_proxy_host,
+            http_proxy_port
+            ).start(@uri.host, @uri.port) do |http|
+          http.request(request)
+        end
+      rescue Errno::ECONNREFUSED, SocketError
+        msg = "Error: Could not connect to #{@uri}\n"
+        msg << "  There was a network communications problem\n"
+        msg << "    Check your network connection and try again\n"
+        Puppet.err msg
+        exit(1)
+      end
+    end
+
+    # Return the local file name containing the data downloaded from the
+    # repository at +release+ (e.g. "myuser-mymodule").
+    def retrieve(release)
+      return cache.retrieve(@uri + release)
+    end
+
+    # Return the URI string for this repository.
+    def to_s
+      return @uri.to_s
+    end
+
+    # Return the cache key for this repository, this a hashed string based on
+    # the URI.
+    def cache_key
+      return @cache_key ||= [
+        @uri.to_s.gsub(/[^[:alnum:]]+/, '_').sub(/_$/, ''),
+        Digest::SHA1.hexdigest(@uri.to_s)
+      ].join('-')
+    end
+  end
+end

data/lib/vendor/puppet/forge.rb

@@ -0,0 +1,98 @@
+require 'net/http'
+require 'open-uri'
+require 'pathname'
+require 'uri'
+require 'puppet/forge/cache'
+require 'puppet/forge/repository'
+
+module Puppet::Forge
+  # Return a list of module metadata hashes that match the search query.
+  # This return value is used by the module_tool face install search,
+  # and displayed to on the console.
+  #
+  # Example return value:
+  #
+  # [
+  #   {
+  #     "author"      => "puppetlabs",
+  #     "name"        => "bacula",
+  #     "tag_list"    => ["backup", "bacula"],
+  #     "releases"    => [{"version"=>"0.0.1"}, {"version"=>"0.0.2"}],
+  #     "full_name"   => "puppetlabs/bacula",
+  #     "version"     => "0.0.2",
+  #     "project_url" => "http://github.com/puppetlabs/puppetlabs-bacula",
+  #     "desc"        => "bacula"
+  #   }
+  # ]
+  #
+  def self.search(term)
+    server = Puppet.settings[:module_repository].sub(/^(?!https?:\/\/)/, 'http://')
+    Puppet.notice "Searching #{server} ..."
+    request = Net::HTTP::Get.new("/modules.json?q=#{URI.escape(term)}")
+    response = repository.make_http_request(request)
+
+    case response.code
+    when "200"
+      matches = PSON.parse(response.body)
+    else
+      raise RuntimeError, "Could not execute search (HTTP #{response.code})"
+      matches = []
+    end
+
+    matches
+  end
+
+  def self.remote_dependency_info(author, mod_name, version)
+    version_string = version ? "&version=#{version}" : ''
+    request = Net::HTTP::Get.new("/api/v1/releases.json?module=#{author}/#{mod_name}" + version_string)
+    response = repository.make_http_request(request)
+    json = PSON.parse(response.body) rescue {}
+    case response.code
+    when "200"
+      return json
+    else
+      error = json['error'] || ''
+      if error =~ /^Module #{author}\/#{mod_name} has no release/
+        return []
+      else
+        raise RuntimeError, "Could not find release information for this module (#{author}/#{mod_name}) (HTTP #{response.code})"
+      end
+    end
+  end
+
+  def self.get_release_packages_from_repository(install_list)
+    install_list.map do |release|
+      modname, version, file = release
+      cache_path = nil
+      if file
+        begin
+          cache_path = repository.retrieve(file)
+        rescue OpenURI::HTTPError => e
+          raise RuntimeError, "Could not download module: #{e.message}"
+        end
+      else
+        raise RuntimeError, "Malformed response from module repository."
+      end
+      cache_path
+    end
+  end
+
+  # Locate a module release package on the local filesystem and move it
+  # into the `Puppet.settings[:module_working_dir]`. Do not unpack it, just
+  # return the location of the package on disk.
+  def self.get_release_package_from_filesystem(filename)
+    if File.exist?(File.expand_path(filename))
+      repository = Repository.new('file:///')
+      uri = URI.parse("file://#{URI.escape(File.expand_path(filename))}")
+      cache_path = repository.retrieve(uri)
+    else
+      raise ArgumentError, "File does not exists: #{filename}"
+    end
+
+    cache_path
+  end
+
+  def self.repository
+    @repository ||= Puppet::Forge::Repository.new
+  end
+end

data/lib/vendor/puppet/indirector/active_record.rb

@@ -0,0 +1,28 @@
+require 'puppet/indirector'
+
+class Puppet::Indirector::ActiveRecord < Puppet::Indirector::Terminus
+  class << self
+    attr_accessor :ar_model
+  end
+
+  def self.use_ar_model(klass)
+    self.ar_model = klass
+  end
+
+  def ar_model
+    self.class.ar_model
+  end
+
+  def initialize
+    Puppet::Rails.init
+  end
+
+  def find(request)
+    return nil unless instance = ar_model.find_by_name(request.key)
+    instance.to_puppet
+  end
+
+  def save(request)
+    ar_model.from_puppet(request.instance).save
+  end
+end

data/lib/vendor/puppet/indirector/catalog/active_record.rb

@@ -0,0 +1,41 @@
+require 'puppet/rails/host'
+require 'puppet/indirector/active_record'
+require 'puppet/resource/catalog'
+
+class Puppet::Resource::Catalog::ActiveRecord < Puppet::Indirector::ActiveRecord
+  use_ar_model Puppet::Rails::Host
+
+  # If we can find the host, then return a catalog with the host's resources
+  # as the vertices.
+  def find(request)
+    return nil unless request.options[:cache_integration_hack]
+    return nil unless host = ar_model.find_by_name(request.key)
+
+    catalog = Puppet::Resource::Catalog.new(host.name)
+
+    host.resources.each do |resource|
+      catalog.add_resource resource.to_transportable
+    end
+
+    catalog
+  end
+
+  # Save the values from a Facts instance as the facts on a Rails Host instance.
+  def save(request)
+    catalog = request.instance
+
+    host = ar_model.find_by_name(catalog.name) || ar_model.create(:name => catalog.name)
+
+    host.railsmark "Saved catalog to database" do
+      host.merge_resources(catalog.vertices)
+      host.last_compile = Time.now
+
+      if node = Puppet::Node.indirection.find(catalog.name)
+        host.ip = node.parameters["ipaddress"]
+        host.environment = node.environment.to_s
+      end
+
+      host.save
+    end
+  end
+end

data/lib/vendor/puppet/indirector/catalog/compiler.rb

@@ -0,0 +1,153 @@
+require 'puppet/node'
+require 'puppet/resource/catalog'
+require 'puppet/indirector/code'
+require 'yaml'
+
+class Puppet::Resource::Catalog::Compiler < Puppet::Indirector::Code
+  desc "Puppet's catalog compilation interface, and its back-end is
+    Puppet's compiler"
+
+  include Puppet::Util
+
+  attr_accessor :code
+
+  def extract_facts_from_request(request)
+    return unless text_facts = request.options[:facts]
+    raise ArgumentError, "Facts but no fact format provided for #{request.name}" unless format = request.options[:facts_format]
+
+    # If the facts were encoded as yaml, then the param reconstitution system
+    # in Network::HTTP::Handler will automagically deserialize the value.
+    if text_facts.is_a?(Puppet::Node::Facts)
+      facts = text_facts
+    else
+      facts = Puppet::Node::Facts.convert_from(format, text_facts)
+    end
+    facts.add_timestamp
+    Puppet::Node::Facts.indirection.save(facts)
+  end
+
+  # Compile a node's catalog.
+  def find(request)
+    extract_facts_from_request(request)
+
+    node = node_from_request(request)
+
+    if catalog = compile(node)
+      return catalog
+    else
+      # This shouldn't actually happen; we should either return
+      # a config or raise an exception.
+      return nil
+    end
+  end
+
+  # filter-out a catalog to remove exported resources
+  def filter(catalog)
+    return catalog.filter { |r| r.virtual? } if catalog.respond_to?(:filter)
+    catalog
+  end
+
+  def initialize
+    set_server_facts
+  end
+
+  # Is our compiler part of a network, or are we just local?
+  def networked?
+    Puppet.run_mode.master?
+  end
+
+  private
+
+  # Add any extra data necessary to the node.
+  def add_node_data(node)
+    # Merge in our server-side facts, so they can be used during compilation.
+    node.merge(@server_facts)
+  end
+
+  # Compile the actual catalog.
+  def compile(node)
+    str = "Compiled catalog for #{node.name}"
+    str += " in environment #{node.environment}" if node.environment
+    config = nil
+
+    loglevel = networked? ? :notice : :none
+
+    benchmark(loglevel, str) do
+      begin
+        config = Puppet::Parser::Compiler.compile(node)
+      rescue Puppet::Error => detail
+        Puppet.err(detail.to_s) if networked?
+        raise
+      end
+    end
+
+    config
+  end
+
+  # Turn our host name into a node object.
+  def find_node(name)
+    begin
+      return nil unless node = Puppet::Node.indirection.find(name)
+    rescue => detail
+      puts detail.backtrace if Puppet[:trace]
+      raise Puppet::Error, "Failed when searching for node #{name}: #{detail}"
+    end
+
+
+    # Add any external data to the node.
+    add_node_data(node)
+
+    node
+  end
+
+  # Extract the node from the request, or use the request
+  # to find the node.
+  def node_from_request(request)
+    if node = request.options[:use_node]
+      return node
+    end
+
+    # We rely on our authorization system to determine whether the connected
+    # node is allowed to compile the catalog's node referenced by key.
+    # By default the REST authorization system makes sure only the connected node
+    # can compile his catalog.
+    # This allows for instance monitoring systems or puppet-load to check several
+    # node's catalog with only one certificate and a modification to auth.conf 
+    # If no key is provided we can only compile the currently connected node.
+    name = request.key || request.node
+    if node = find_node(name)
+      return node
+    end
+
+    raise ArgumentError, "Could not find node '#{name}'; cannot compile"
+  end
+
+  # Initialize our server fact hash; we add these to each client, and they
+  # won't change while we're running, so it's safe to cache the values.
+  def set_server_facts
+    @server_facts = {}
+
+    # Add our server version to the fact list
+    @server_facts["serverversion"] = Puppet.version.to_s
+
+    # And then add the server name and IP
+    {"servername" => "fqdn",
+      "serverip" => "ipaddress"
+    }.each do |var, fact|
+      if value = Facter.value(fact)
+        @server_facts[var] = value
+      else
+        Puppet.warning "Could not retrieve fact #{fact}"
+      end
+    end
+
+    if @server_facts["servername"].nil?
+      host = Facter.value(:hostname)
+      if domain = Facter.value(:domain)
+        @server_facts["servername"] = [host, domain].join(".")
+      else
+        @server_facts["servername"] = host
+      end
+    end
+  end
+end

data/lib/vendor/puppet/indirector/catalog/queue.rb

@@ -0,0 +1,5 @@
+require 'puppet/resource/catalog'
+require 'puppet/indirector/queue'
+
+class Puppet::Resource::Catalog::Queue < Puppet::Indirector::Queue
+end

data/lib/vendor/puppet/indirector/catalog/rest.rb

@@ -0,0 +1,6 @@
+require 'puppet/resource/catalog'
+require 'puppet/indirector/rest'
+
+class Puppet::Resource::Catalog::Rest < Puppet::Indirector::REST
+  desc "Find resource catalogs over HTTP via REST."
+end

data/lib/vendor/puppet/indirector/catalog/static_compiler.rb

@@ -0,0 +1,136 @@
+require 'puppet/node'
+require 'puppet/resource/catalog'
+require 'puppet/indirector/code'
+
+class Puppet::Resource::Catalog::StaticCompiler < Puppet::Indirector::Code
+  def compiler
+    @compiler ||= indirection.terminus(:compiler)
+  end
+
+  def find(request)
+    return nil unless catalog = compiler.find(request)
+
+    raise "Did not get catalog back" unless catalog.is_a?(model)
+
+    catalog.resources.find_all { |res| res.type == "File" }.each do |resource|
+      next unless source = resource[:source]
+      next unless source =~ /^puppet:/
+
+      file = resource.to_ral
+      if file.recurse?
+        add_children(request.key, catalog, resource, file)
+      else
+        find_and_replace_metadata(request.key, resource, file)
+      end
+    end
+
+    catalog
+  end
+
+  def find_and_replace_metadata(host, resource, file)
+    # We remove URL info from it, so it forces a local copy
+    # rather than routing through the network.
+    # Weird, but true.
+    newsource = file[:source][0].sub("puppet:///", "")
+    file[:source][0] = newsource
+
+    raise "Could not get metadata for #{resource[:source]}" unless metadata = file.parameter(:source).metadata
+
+    replace_metadata(host, resource, metadata)
+  end
+
+  def replace_metadata(host, resource, metadata)
+    [:mode, :owner, :group].each do |param|
+      resource[param] ||= metadata.send(param)
+    end
+
+    resource[:ensure] = metadata.ftype
+    if metadata.ftype == "file"
+      unless resource[:content]
+        resource[:content] = metadata.checksum
+        resource[:checksum] = metadata.checksum_type
+      end
+    end
+
+    store_content(resource) if resource[:ensure] == "file"
+    old_source = resource.delete(:source)
+    Puppet.info "Metadata for #{resource} in catalog for '#{host}' added from '#{old_source}'"
+  end
+
+  def add_children(host, catalog, resource, file)
+    file = resource.to_ral
+
+    children = get_child_resources(host, catalog, resource, file)
+
+    remove_existing_resources(children, catalog)
+
+    children.each do |name, res|
+      catalog.add_resource res
+      catalog.add_edge(resource, res)
+    end
+  end
+
+  def get_child_resources(host, catalog, resource, file)
+    sourceselect = file[:sourceselect]
+    children = {}
+
+    source = resource[:source]
+
+    # This is largely a copy of recurse_remote in File
+    total = file[:source].collect do |source|
+      next unless result = file.perform_recursion(source)
+      return if top = result.find { |r| r.relative_path == "." } and top.ftype != "directory"
+      result.each { |data| data.source = "#{source}/#{data.relative_path}" }
+      break result if result and ! result.empty? and sourceselect == :first
+      result
+    end.flatten
+
+    # This only happens if we have sourceselect == :all
+    unless sourceselect == :first
+      found = []
+      total.reject! do |data|
+        result = found.include?(data.relative_path)
+        found << data.relative_path unless found.include?(data.relative_path)
+        result
+      end
+    end
+
+    total.each do |meta|
+      # This is the top-level parent directory
+      if meta.relative_path == "."
+        replace_metadata(host, resource, meta)
+        next
+      end
+      children[meta.relative_path] ||= Puppet::Resource.new(:file, File.join(file[:path], meta.relative_path))
+
+      # I think this is safe since it's a URL, not an actual file
+      children[meta.relative_path][:source] = source + "/" + meta.relative_path
+      replace_metadata(host, children[meta.relative_path], meta)
+    end
+
+    children
+  end
+
+  def remove_existing_resources(children, catalog)
+    existing_names = catalog.resources.collect { |r| r.to_s }
+    both = (existing_names & children.keys).inject({}) { |hash, name| hash[name] = true; hash }
+    both.each { |name| children.delete(name) }
+  end
+
+  def store_content(resource)
+    @summer ||= Object.new
+    @summer.extend(Puppet::Util::Checksums)
+
+    type = @summer.sumtype(resource[:content])
+    sum = @summer.sumdata(resource[:content])
+
+    if Puppet::FileBucket::File.indirection.find("#{type}/#{sum}")
+      Puppet.info "Content for '#{resource[:source]}' already exists"
+    else
+      Puppet.info "Storing content for source '#{resource[:source]}'"
+      content = Puppet::FileServing::Content.indirection.find(resource[:source])
+      file = Puppet::FileBucket::File.new(content.content)
+      Puppet::FileBucket::File.indirection.save(file)
+    end
+  end
+end

data/lib/vendor/puppet/indirector/catalog/store_configs.rb

@@ -0,0 +1,5 @@
+require 'puppet/indirector/store_configs'
+require 'puppet/resource/catalog'
+
+class Puppet::Resource::Catalog::StoreConfigs < Puppet::Indirector::StoreConfigs
+end