ncio 0.2.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7f4228a74a79a10e47ffce674748ec755eae37ec
+  data.tar.gz: 5c80ffcca70d9ca50174b4f76cf72fed453e3794
+SHA512:
+  metadata.gz: bc29b5864c421b08faf4609baa7560e2e5e8be82c6210522e82df39338ca7c5e2290ff53f4dd6ee9b34e3e79b4fd5bb1e21a7980f4bd3f347c0723d23727884d
+  data.tar.gz: b31ba443b49e416b5980e9a78996a0c9d8f53b7836c8d7b608f5e3d65b420929999acc34de959e7b92060e0255b05a274f45d389e6faf1ed9c973930ab1b4c80

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+.DS_Store

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.rubocop.yml

@@ -0,0 +1,7 @@
+--- 
+AllCops:
+  Exclude:
+    - doc/**
+    - lib/ncio/trollop.rb
+    - ncio.gemspec
+    - Rakefile

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.2.3
+before_install: gem install bundler -v 1.12.5

data/.yardopts

@@ -0,0 +1 @@
+--markup markdown

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in ncio.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Jeff McCune
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,95 @@
+# ncio - Puppet Node Classifier backup / restore
+
+This project implements a small command line utility to backup and restore node
+classification data.  The intended purpose is to backup node classification
+groups on a Primary, monolithic PE master and restore the backup on a secondary
+monolithic PE master.  The purpose is to keep node classification groups in sync
+and ready in the event the secondary master needs to take over service from the
+primary.
+
+
+## Transformation
+
+To achieve the goal of replicating node classification groups from one PE
+monolithic master to a secondary monolithic master, certain values need to be
+transformed.  For example, consider a primary named `master1.puppet.vm` and a
+secondary named `master2.puppet.vm`  Both are monolithic masters.  When the
+backup is taken on the primary, the hostname will be embedded in the data.  This
+is problematic because it will cause mis-configuration errors when imported into
+the secondary which has a different name.
+
+To illustrate, consider the PuppetDB classification group:
+
+    {
+      "name": "PE PuppetDB",
+      "rule": [
+        "or",
+        [
+          "=",
+          "name",
+          "master1.puppet.vm"
+        ]
+      ],
+      "classes": {
+        "puppet_enterprise::profile::puppetdb": {
+        }
+      }
+    }
+
+Transformation from master1 to master2 is possible:
+
+    export PATH="/opt/pupeptlabs/puppet/bin:$PATH"
+    ncio --uri https://master1.puppet.vm:4433/classification-api/v1 backup \
+     | ncio transform --hostname master1.puppet.vm:master2.puppet.vm \
+     | ncio --uri https://master2.puppet.vm:4433/classification-api/v1 restore
+
+This method of "replicating" node classification data has some caveats.  It's
+only been tested on PE Monolithic masters.  The method assumes master1 and
+master2 share the same Certificate Authority.  By default, only the default
+`puppet_enterprise` classification groups are transformed.
+
+Additional groups and classes may be processed by chaining transfomation
+processes and getting creative with the use of the `--class-matcher` option.
+
+## Installation
+
+Install this tool on the same node running the node classification service:
+
+    $ sudo /opt/puppetlabs/puppet/bin/gem install ncio
+    Successfully installed ncio-0.1.0
+    Parsing documentation for ncio-0.1.0
+    Installing ri documentation for ncio-0.1.0
+    Done installing documentation for ncio after 0 seconds
+    1 gem installed
+
+## Usage
+
+If the file `/etc/puppetlabs/puppet/ssl/certs/pe-internal-orchestrator.pem`
+exists on the same node as the Node Classifier, then no configuration is
+necessary.  The default options will work to backup and restore node
+classification data.
+
+    sudo -H -u pe-puppet /opt/puppetlabs/puppet/bin/ncio backup > /var/tmp/backup.json
+    I, [2016-06-28T19:25:55.507684 #2992]  INFO -- : Backup completed successfully!
+
+If this file does not exist, ncio will need to use a different client
+certificate.  It is recommended to use the same certificate used by the Puppet
+Agent, which should be white-listed for node classification API access.  The
+white-list of certificates is located at
+`/etc/puppetlabs/console-services/rbac-certificate-whitelist`
+
+    sudo -H -u pe-puppet /opt/puppetlabs/puppet/bin/ncio \
+      --cert /etc/puppetlabs/puppet/ssl/certs/${HOSTNAME}.pem \
+      --key  /etc/puppetlabs/puppet/ssl/private_keys/${HOSTNAME}.pem \
+      backup > /var/tmp/backup.json
+    I, [2016-06-28T19:28:48.236257 #3148]  INFO -- : Backup completed successfully!
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at
+https://github.com/jeffmccune/ncio.
+
+## License
+
+The gem is available as open source under the terms of the [MIT
+License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,6 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'ncio'
+require 'pry'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+Pry.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/ncio

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+require 'ncio'
+require 'ncio/app'
+app = Ncio::App.new
+exit_code = app.run
+Kernel.exit(exit_code)

data/lib/ncio.rb

@@ -0,0 +1,9 @@
+require 'ncio/version'
+require 'logger'
+
+##
+# The top level module for Ncio (Node Classification Input/Output)
+# This module contains instance methods used by all of the classes in this
+# project.  Mostly for option handling, logging, I.O, etc...
+module Ncio
+end

data/lib/ncio/api.rb

@@ -0,0 +1,7 @@
+module Ncio
+  ##
+  # Placeholder for API classes.  Please see the specific class for the node
+  # classifier API version, e.g. `Ncio::Api::V1`.
+  module Api
+  end
+end

data/lib/ncio/api/v1.rb

@@ -0,0 +1,109 @@
+require 'ncio/api'
+require 'ncio/http_client'
+require 'uri'
+require 'socket'
+require 'json'
+
+module Ncio
+  module Api
+    ##
+    # Node Classifier API Version 1
+    #
+    # See [Groups
+    # endpoint](https://docs.puppet.com/pe/2016.1/nc_groups.html#get-v1groups)
+    class V1
+      attr_reader :host
+      attr_reader :port
+      attr_reader :opts
+
+      class ApiError < RuntimeError; end
+
+      DEFAULT_HEADERS = {
+        'Content-Type' => 'application/json',
+        'Transfer-Encoding' => 'chunked'
+      }.freeze
+
+      ##
+      # Initialize and return a new API instance.
+      #
+      # @param [Hash<Symbol, String>] The global options hash created by the
+      #   application instanece.
+      def initialize(opts)
+        @opts = opts
+        uri = URI(opts[:uri])
+        @host = uri.host
+        @port = uri.port
+      end
+
+      ##
+      # Return a memoized HTTP connection
+      def connection
+        return @connection if @connection
+        conn_opts = {
+          host: host,
+          port: port,
+          cert: opts[:cert],
+          key: opts[:key],
+          cacert: opts[:cacert]
+        }
+        @connection = Ncio::HttpClient.new(conn_opts)
+      end
+
+      ##
+      # Return all of the groups currently defined in the node classifier API.
+      #
+      # @param [Boolean] inherited If set to any value besides 0 or false, the
+      #   node group will include the classes, class parameters, and variables
+      #   that it inherits from its ancestors.
+      #
+      # @return [Array]
+      def groups(inherited = false)
+        uri = build_uri('groups', inherited: inherited.to_s)
+        req = Net::HTTP::Get.new(uri, DEFAULT_HEADERS)
+        resp = connection.request(req)
+        if resp.code == '200'
+          obj = JSON.parse(resp.body)
+        else
+          msg = "Expected 200 response, got #{resp.code} body: #{resp.body}"
+          raise ApiError, msg
+        end
+        obj
+      end
+
+      ##
+      # Import all of the classification groups using the POST
+      # /v1/import-hierarchy endpoint.  See: [NC Import
+      # Hierarchy](https://docs.puppet.com/pe/2016.1/nc_import-hierarchy.html)
+      #
+      # @return [Boolean] true if the upload was successful
+      def import_hierarchy(stream)
+        uri = build_uri('import-hierarchy')
+        req = Net::HTTP::Post.new(uri, DEFAULT_HEADERS)
+        req.body_stream = stream
+        resp = connection.request(req)
+        return true if resp.code == '204'
+        msg = "Expected 204 response, got #{resp.code} body: #{resp.body}"
+        raise ApiError, msg
+      end
+
+      ##
+      # Return a URI instance with the given path and parameters.  The
+      #   connection base URI is used to construct the full URI to the service.
+      #
+      # @param [String] path The path relative to the classifier base service
+      #   path and the API version, e.g. 'groups'.
+      #
+      # @param [Hash] params The query parameters to encode into the URI, e.g.
+      #   `{inherited: 'false'}`.
+      #
+      # @return [URI] The API uri with query parameters and a fully constructed
+      #   path.
+      def build_uri(path, params = {})
+        uri = connection.uri
+        uri.path = "/classifier-api/v1/#{path}"
+        uri.query = URI.encode_www_form(params)
+        uri
+      end
+    end
+  end
+end

data/lib/ncio/app.rb

@@ -0,0 +1,114 @@
+# rubocop:disable Style/IndentationWidth
+require 'ncio'
+require 'ncio/support'
+require 'ncio/support/option_parsing'
+require 'ncio/support/transform'
+require 'ncio/trollop'
+require 'ncio/version'
+require 'ncio/api/v1'
+require 'uri'
+require 'socket'
+
+module Ncio
+##
+# The main Application class.  Intended to be instantiated and called with
+# using the `run` method by the `ncio` bin script.
+class App
+  # rubocop:enable Style/IndentationWidth
+  # include support methods (option handling, logging, I/O helpers)
+  include Ncio::Support
+  include Ncio::Support::OptionParsing
+  include Ncio::Support::Transform
+
+  ##
+  # @param [Array] argv The argument vector, passed to the option parser.
+  #
+  # @param [Hash] env The environment hash, passed to the option parser to
+  #   supply defaults not specified on the command line argument vector.
+  #
+  # @return [Ncio::App] the application instance.
+  def initialize(argv = ARGV.dup, env = ENV.to_hash)
+    @argv = argv
+    @env = env
+    reset!
+  end
+
+  ##
+  # Reset all state associated with this application instance.
+  def reset!
+    reset_options!
+    reset_logging!(opts)
+    @api = nil
+  end
+
+  ##
+  # Run the application instance.  This method is responsible for the
+  # application lifecycle.  Command line arguments are parsed, unspecified
+  # options are read from the environment, and the specified subcommand is
+  # executed.
+  #
+  # @return [Fixnum] the exit code to pass to Kernel.exit in the calling
+  #   script.
+  # rubocop:disable Metrics/MethodLength
+  def run
+    case opts[:subcommand]
+    when 'backup'
+      backup_groups if opts[:groups]
+      return 0
+    when 'restore'
+      restore_groups if opts[:groups]
+      return 0
+    when 'transform'
+      transform_groups
+      return 0
+    end
+  end
+  # rubocop:enable Metrics/MethodLength
+
+  ##
+  # Backup all groups in a manner suitable for the node classification hierarchy
+  # import. See:
+  # [NC Groups](https://docs.puppet.com/pe/2016.1/nc_groups.html#get-v1groups)
+  def backup_groups
+    debug "GET #{uri}/groups"
+    str = JSON.pretty_generate(api.groups)
+    debug "Write #{str.bytesize} bytes to #{file} ..."
+    write_output(str, map_file_option(file))
+    info 'Backup completed successfully!'
+  end
+
+  ##
+  # Restore all groups in a manner suitable for the node classification
+  # hierarchy import. See: [NC Import
+  # Hierarchy](https://docs.puppet.com/pe/2016.1/nc_import-hierarchy.html)
+  def restore_groups
+    api = self.api
+    debug "Open #{file} for streaming ..."
+    input_stream(map_file_option(file)) do |stream|
+      debug "POST #{uri}/import-hierarchy"
+      api.import_hierarchy(stream)
+    end
+    info 'Successfully restored node classification groups!'
+  end
+
+  ##
+  # Transform a backup produced with backup_groups.  The transformation is
+  # intended to allow restoration of the backup on PE Infrastructure cluster
+  # different from the one the backup was produced on.
+  #
+  # Currently only one PE cluster type is supported, the Monolithic master type.
+  # rubocop:disable Metrics/AbcSize
+  def transform_groups
+    # Read input
+    groups = JSON.parse(input_stream(map_file_option(opts[:input]), &:read))
+    groups.map! do |group|
+      group_matches?(group) ? transform_group(group) : group
+    end
+    str = JSON.pretty_generate(groups)
+    debug "Write #{str.bytesize} bytes to #{opts[:output]} ..."
+    write_output(str, map_file_option(opts[:output]))
+    info 'Transformation completed successful!'
+  end
+  # rubocop:enable Metrics/AbcSize
+end
+end