ncio 0.2.2

data/lib/ncio/http_client.rb

@@ -0,0 +1,149 @@
+require 'net/http'
+require 'socket'
+require 'openssl'
+
+module Ncio
+  ##
+  # HttpClient provides a Net::HTTP instance pre-configured to communicate with
+  # the Puppet Node Classification Service.  The client will return Ruby native
+  # objects where possible, parsing JSON responses from the service.
+  #
+  # This client implements v1 of the [Node Classification
+  # API](https://docs.puppet.com/pe/2016.1/nc_index.html).
+  class HttpClient
+    attr_reader :host
+    attr_reader :port
+    attr_reader :use_ssl
+    attr_reader :cert
+    attr_reader :key
+    attr_reader :cacert
+    attr_reader :protocol
+
+    # ApiError is raised when there are errors in the REST API repsonse.
+    class ApiError < RuntimeError; end
+
+    ssldir = '/etc/puppetlabs/puppet/ssl'
+    OPTION_DEFAULTS = {
+      host: Socket.gethostname,
+      port: 4433,
+      use_ssl: true,
+      cert: ssldir + '/certs/pe-internal-orchestrator.pem',
+      key: ssldir + '/private_keys/pe-internal-orchestrator.pem',
+      cacert: ssldir + '/certs/ca.pem'
+    }.freeze
+
+    ##
+    # initialize a new HttpClient instance
+    #
+    # @param [Hash] opts Options
+    #
+    # @option opts [String] :host The API host, e.g. `"master1.puppet.vm"`.
+    #   Defaults to the local hostname returned by `Socket.gethostname`
+    #
+    # @option opts [Fixnum] :port The API tcp port, Defaults to `4433`
+    #
+    # @option opts [String] :cert The path to the PEM encoded client
+    #   certificate.  Defaults to
+    #   `"/etc/puppetlabs/puppet/ssl/certs/pe-internal-orchestrator.pem"`
+    #
+    # @option opts [String] :key The path to the PEM encoded RSA private key
+    #   used for the SSL client connection.  Defaults to
+    #   `"/etc/puppetlabs/puppet/ssl/private_keys/pe-internal-orchestrator.pem"`
+    #
+    # @option opts [String] :cacert The path to the PEM encoded CA certificate
+    #   used to authenticate the service URL.  Defaults to
+    #   `"/etc/puppetlabs/puppet/ssl/certs/ca.pem"`
+    def initialize(opts = {})
+      opts = OPTION_DEFAULTS.merge(opts)
+      @use_ssl = opts[:use_ssl]
+      @host = opts[:host]
+      @port = opts[:port]
+      @cert = opts[:cert]
+      @key = opts[:key]
+      @cacert = opts[:cacert]
+      @protocol = use_ssl ? 'https' : 'http'
+    end
+
+    ##
+    # make a request, pass through to Net::HTTP#request
+    #
+    # @param [Net::HTTPRequest] req The HTTP request, e.g. an instance of
+    #   `Net::HTTP::Get`, `Net::HTTP::Post`, or `Net::HTTP::Head`.
+    #
+    # @param [String] body The request body, if any.
+    #
+    # @return [Net::HTTPResponse] response
+    def request(req, body = nil)
+      http.request(req, body)
+    end
+
+    ##
+    # Provide a URL to the endpoint this client connects to.  This is intended
+    # to construct URL's and add query parameters easily.
+    #
+    # @return [URI] the URI of the server this client connects to.
+    def uri
+      return @uri if @uri
+      @uri = URI("#{protocol}://#{host}:#{port}")
+    end
+
+    private
+
+    ##
+    # return a memoized HTTP object instance configured with the SSL client
+    # certificate and ready to authorize the peer service
+    #
+    # TODO: Add revocation checking.  See: [puppet/ssl/host.rb line
+    # 263](https://github.com/puppetlabs/puppet/blob/4.5.2/lib/puppet/ssl/host.rb#L263)
+    #
+    # @return [Net::HTTP]
+    def http
+      return @http if @http
+      client = Net::HTTP.new(uri.host, uri.port)
+      @http = if use_ssl
+                setup_ssl(client)
+              else
+                client
+              end
+      @http
+    end
+
+    ##
+    # Configure this client to use SSL
+    #
+    # @param [Net::HTTP] http The http instance to configure to use SSL
+    #
+    # @return [Net::HTTP] configured with SSL certificates passed to
+    # initializer.
+    def setup_ssl(http)
+      http.use_ssl = true
+      http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+      # Setup the SSL store used for this connection
+      ssl_store = ssl_store()
+      ssl_store.purpose = OpenSSL::X509::PURPOSE_ANY
+      ssl_store.add_file(cacert)
+      http.cert_store = ssl_store
+      # PEM files
+      http.cert = OpenSSL::X509::Certificate.new(read_cert)
+      http.key = OpenSSL::PKey::RSA.new(read_key)
+      http.ca_file = cacert
+      http
+    end
+
+    # helper method to stub the OpenSSL Store
+    def ssl_store
+      OpenSSL::X509::Store.new
+    end
+    ##
+
+    # helper method to stub the cert in the tests
+    def read_cert
+      File.read(cert)
+    end
+
+    # helper method to stub the cert in the tests
+    def read_key
+      File.read(key)
+    end
+  end
+end

data/lib/ncio/support.rb

@@ -0,0 +1,116 @@
+module Ncio
+  ##
+  # Support module to mix into other classes, particularly the application and
+  # API classes.  This support module provides a centralized logging
+  # configuration, and common methods to access configuration options about the
+  # behavior of the program.
+  module Support
+    attr_reader :opts
+
+    def self.reset_logging!(opts)
+      out = map_file_option(opts[:logto])
+      logger = Logger.new(out)
+      logger.level = opts[:debug] ? Logger::DEBUG : Logger::INFO
+      @log = logger
+    end
+
+    ##
+    # Logging is handled centrally, the helper methods will delegate to the
+    # centrally configured logging instance.
+    def self.log
+      @log
+    end
+
+    ##
+    # Map a file option to STDOUT, STDERR or a fully qualified file path.
+    #
+    # @param [String] filepath A relative or fully qualified file path, or the
+    #   keyword strings 'STDOUT' or 'STDERR'
+    #
+    # @return [String] file path or $stdout or $sederr
+    def self.map_file_option(filepath)
+      case filepath
+      when 'STDOUT' then $stdout
+      when 'STDERR' then $stderr
+      when 'STDIN' then $stdin
+      else File.expand_path(filepath)
+      end
+    end
+
+    def map_file_option(filepath)
+      Ncio::Support.map_file_option(filepath)
+    end
+
+    def log
+      Ncio::Support.log
+    end
+
+    ##
+    # Reset the logging system, requires command line options to have been
+    # parsed.
+    #
+    # @param [Hash<Symbol, String>] Options hash, passed to the support module
+    def reset_logging!(opts)
+      Ncio::Support.reset_logging!(opts)
+    end
+
+    ##
+    # Memoized helper method to instantiate an API instance assuming options
+    # are available.
+    def api
+      @api ||= Ncio::Api::V1.new(opts)
+    end
+
+    ##
+    # Log an info message
+    def info(msg)
+      log.info msg
+    end
+
+    ##
+    # Log a debug message
+    def debug(msg)
+      log.debug msg
+    end
+
+    def uri
+      opts[:uri]
+    end
+
+    def file
+      opts[:file]
+    end
+
+    ##
+    # Helper method to write output, used for stubbing out the tests.
+    #
+    # @param [String, IO] output the output path or a IO stream
+    def write_output(str, output)
+      if output.is_a?(IO)
+        output.puts(str)
+      else
+        File.open(output, 'w+') { |f| f.puts(str) }
+      end
+    end
+
+    ##
+    # Helper method to read from STDIN, or a file and execute an arbitrary block
+    # of code.  A block must be passed which will recieve an IO object in the
+    # event input is a readable file path.
+    def input_stream(input)
+      if input.is_a?(IO)
+        yield input
+      else
+        File.open(input, 'r') { |stream| yield stream }
+      end
+    end
+
+    ##
+    # Return the application version as a Semantic Version encoded string
+    #
+    # @return [String] the version
+    def version
+      Ncio::VERSION
+    end
+  end
+end

data/lib/ncio/support/option_parsing.rb

@@ -0,0 +1,200 @@
+require 'ncio/version'
+# rubocop:disable Metrics/ModuleLength
+module Ncio
+  module Support
+    ##
+    # Support methods intended to be mixed into the application class.  These
+    # methods are specific to command line parsing.  The model is [GLOBAL
+    # OPTIONS] SUBCOMMAND [SUBCOMMAND OPTIONS]
+    #
+    # Configuration state parsed from options is intended to be stored in a
+    # @opts hash and injected into dependencies, like API instances.
+    module OptionParsing
+      attr_reader :argv, :env, :opts
+
+      ##
+      # Reset the @opts instance variable by parsing @argv and @env.  Operates
+      # against duplicate copies of the argument vector avoid side effects.
+      #
+      # @return [Hash<Symbol, String>] Options hash
+      def reset_options!
+        @opts = parse_options(argv, env)
+      end
+
+      ##
+      # Parse options using the argument vector and the environment hash as
+      # input. Option parsing occurs in two phases, first the global options are
+      # parsed. These are the options specified before the subcommand.  The
+      # subcommand, if any, is matched, and subcommand specific options are then
+      # parsed from the remainder of the argument vector.
+      #
+      # @param [Array] argv The argument vector, passed to the option parser.
+      #
+      # @param [Hash] env The environment hash, passed to the option parser to
+      #   supply defaults not specified on the command line argument vector.
+      #
+      # @return [Hash<Symbol, String>] options hash
+      def parse_options(argv, env)
+        argv_copy = argv.dup
+        opts = parse_global_options!(argv_copy, env)
+        subcommand = parse_subcommand!(argv_copy)
+        opts[:subcommand] = subcommand
+        sub_opts = parse_subcommand_options!(subcommand, argv_copy, env)
+        opts.merge!(sub_opts)
+        opts
+      end
+
+      ##
+      # Parse out the global options, the ones specified between the main
+      # executable and the subcommand argument.
+      #
+      # Modifies argv as a side effect, shifting elements from the array until
+      # the first unknown option is found, which is assumed to be the subcommand
+      # name.
+      #
+      # @return [Hash<Symbol, String>] Global options
+      # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      def parse_global_options!(argv, env)
+        semver = Ncio::VERSION
+        host = Socket.gethostname
+        Ncio::Trollop.options(argv) do
+          stop_on_unknown
+          version "ncio #{semver} (c) 2016 Jeff McCune"
+          banner BANNER
+          uri_dfl = env['NCIO_URI'] || "https://#{host}:4433/classifier-api/v1"
+          opt :uri, 'Node Classifier service uri '\
+            '{NCIO_URI}', default: uri_dfl
+          opt :cert, CERT_MSG, default: env['NCIO_CERT'] || CERT_DEFAULT
+          opt :key, KEY_MSG, default: env['NCIO_KEY'] || KEY_DEFAULT
+          opt :cacert, CACERT_MSG, default: env['NCIO_CACERT'] || CACERT_DEFAULT
+          log_msg = 'Log file to write to or keywords '\
+            'STDOUT, STDERR {NCIO_LOGTO}'
+          opt :logto, log_msg, default: env['NCIO_LOGTO'] || 'STDERR'
+          opt :debug
+        end
+      end
+      # rubocop:enable Metrics/MethodLength, Metrics/AbcSize
+
+      ##
+      # Extract the subcommand, if any, from the arguments provided.  Modifies
+      # argv as a side effect, shifting the subcommand name if it is present.
+      #
+      # @return [String] The subcommand name, e.g. 'backup' or 'restore', or
+      #   false if no arguments remain in the argument vector.
+      def parse_subcommand!(argv)
+        argv.shift || false
+      end
+
+      ##
+      # Parse the subcommand options.  This method branches out because each
+      # subcommand can have quite different options, unlike global options which
+      # are consistent across all invocations of the application.
+      #
+      # Modifies argv as a side effect, shifting all options as things are
+      # parsed.
+      #
+      # @return [Hash<Symbol, String>] Subcommand specific options hash
+      # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      def parse_subcommand_options!(subcommand, argv, env)
+        case subcommand
+        when 'backup', 'restore'
+          Ncio::Trollop.options(argv) do
+            banner "Node Classification #{subcommand} options:"
+            groups_msg = 'Operate against NC groups.  See: https://goo.gl/QD6ZdW'
+            opt :groups, groups_msg, default: true
+            file_msg = 'File to operate against {NCIO_FILE} or STDOUT, STDERR'
+            file_default = FILE_DEFAULT_MAP[subcommand]
+            opt :file, file_msg, default: env['NCIO_FILE'] || file_default
+          end
+        when 'transform'
+          opts = Ncio::Trollop.options(argv) do
+            banner "Node Classification transformations\n"\
+              'Note: Currently only Monolithic (All-in-one) deployments are '\
+              "supported.\n\nTransformation matches against class names "\
+              'assigned to groups.  Transformation of hostnames happen '\
+              'against rules assigned to groups and class parameters for '\
+              "matching classes.\n\nOptions:"
+            hostname_msg = 'Replace the fully qualified domain name on the '\
+              'left with the right, separated with a : '\
+              'e.g --hostname master1.acme.com:master2.acme.com'
+            opt :class_matcher, 'Regexp matching classes assigned to groups. '\
+                'Passed to Regexp.new()',
+                default: '^puppet_enterprise'
+            opt :input, 'Input file path or keywords STDIN, STDOUT, STDERR',
+                default: 'STDIN'
+            opt :output, 'Output file path or keywords STDIN, STDOUT, STDERR',
+                default: 'STDOUT'
+            opt :hostname, hostname_msg, type: :strings, multi: true,
+                                         required: true
+          end
+          opts[:matcher] = Regexp.new(opts[:class_matcher])
+          if opts[:hostname_given]
+            hsh = map_hostnames(opts[:hostname].flatten)
+            opts.merge!(hostname_map: hsh)
+          end
+        else
+          Ncio::Trollop.die "Unknown subcommand: #{subcommand.inspect}"
+        end
+      end
+      # rubocop:enable Metrics/MethodLength, Metrics/AbcSize
+
+      ##
+      # Map an array of hostnames separated by a colon.  The left is the key of
+      # the map, the right is the value.  The Hash returned is "smart" in that a
+      # key that does not exist will return a value matching the key.
+      #
+      # @param [Array<String>] hostnames
+      #
+      # @return [Hash<String, String>] Hash of hostnames, left as key, right as
+      #   value.
+      def map_hostnames(hostnames)
+        smart_map = Hash.new { |_, key| key }
+        hostnames.each_with_object(smart_map) do |pair, hsh|
+          (k, v) = pair.split(':')
+          hsh[k] = v
+          hsh
+        end
+      end
+
+      BANNER = <<-'EOBANNER'.freeze
+usage: ncio [GLOBAL OPTIONS] SUBCOMMAND [ARGS]
+Sub Commands:
+
+  backup     Backup Node Classification resources
+  restore    Restore Node Classification resources
+  transform  Transform a backup, replacing hostnames
+
+Quick Start: On the host of the Node Classifier service, as root or pe-puppet
+(to read certs and keys)
+
+    /opt/puppetlabs/puppet/bin/ncio backup > groups.$(date +%s).json
+    /opt/puppetlabs/puppet/bin/ncio restore < groups.1467151827.json
+
+Transformation:
+
+    ncio --uri https://master1.puppet.vm:4433/classification-api/v1 backup \
+     | ncio transform --hostname master1.puppet.vm:master2.puppet.vm \
+     | ncio --uri https://master2.puppet.vm:4433/classification-api/v1 restore
+
+Global options: (Note, command line arguments supersede ENV vars in {}'s)
+      EOBANNER
+
+      SSLDIR = '/etc/puppetlabs/puppet/ssl'.freeze
+      CERT_MSG = 'White listed client SSL cert {NCIO_CERT} '\
+        'See: https://goo.gl/zCjncC'.freeze
+      CERT_DEFAULT = (SSLDIR + '/certs/'\
+                      'pe-internal-orchestrator.pem').freeze
+      KEY_MSG = 'Client RSA key, must match certificate '\
+        '{NCIO_KEY}'.freeze
+      KEY_DEFAULT = (SSLDIR + '/private_keys/'\
+                     'pe-internal-orchestrator.pem').freeze
+      CACERT_MSG = 'CA Cert to authenticate the service uri '\
+        '{NCIO_CACERT}'.freeze
+      CACERT_DEFAULT = (SSLDIR + '/certs/ca.pem').freeze
+
+      # Map is indexed by the subcommand
+      FILE_DEFAULT_MAP = { 'backup' => 'STDOUT', 'restore' => 'STDIN' }.freeze
+    end
+  end
+end
+# rubocop:enable Metrics/ModuleLength