ncio 1.0.1 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: aae5414153f31d081b94ba2c7db8c2861c6ef295
-  data.tar.gz: 70e4a75237aa6a37e768b6a8f67e6b1c4a245e64
+  metadata.gz: d3a023e01e8f30ad099628a652e23d8432a96e29
+  data.tar.gz: 26a22f56c26aa09821e7cc5af6b11ee92f2e9748
 SHA512:
-  metadata.gz: 6de835b4da89a650b2a39b21317ca8e662937e1df2dea90c40dc38694e7ff3c3fb32e2d58814da8f296bad71fa6c74da070439d51532d408d136361edd7e8f59
-  data.tar.gz: fba4dc609759ae67ef6c50f3f44f20c84507792482dcb3287a69e9f679e4a284af95c9669770befeff4e83a60f8e448bc649637335ea6102c8f79983cc11bf8d
+  metadata.gz: fbd161b5b74263feccfe972c9f0be4b1a12bd5e5e650d3e80a683177cd25615d9510974843c58689494442961ae712e09840f30a300336047d0017a1423d9bc5
+  data.tar.gz: 219f253cf24cbc5038160884ef0f55eecff340e3a30c92c9690b758b2e9d0885a23639108b7b76376f1f7c72d8f6d0fbf47ee0005e01b362611fada80928249c

data/CHANGELOG.md

@@ -0,0 +1,10 @@
+Version 1.1.0
+===
+
+ * Add `--retry-connections` [Issue 6](https://github.com/jeffmccune/ncio/issues/6)
+ * Add better error handling [Issue 7](https://github.com/jeffmccune/ncio/issues/7)
+
+Version 1.0.1
+===
+
+ * Initial release.  Backup / Transform / Restore

data/README.md

@@ -111,6 +111,70 @@ Log to the console using the `--no-syslog` command line option.
 The tool can only log to either syslog or the console at this time.  Multiple
 log destinations are not currently supported.
 
+## Retrying Connections
+
+It can take some time for the `pe-console-services` service to come online.  In
+an effort to make things are robust as possible consider using the
+`--retry-connections` global option.  This allows ncio to retry API connections
+while the service comes online.  This option has been added to address the
+following use case:
+
+    service start pe-console-services.service
+    ncio --retry-connections backup
+
+In this scenario ncio will retry API connections, eventually succeeding or
+timing out.  Ideally the service will come online before the timeout expires.
+
+## Replication
+
+A simple way to replicate node classification data between a primary and a
+secondary can be achieved with the following shell script.  Call this from cron
+on a periodic basis:
+
+```bash
+#! /bin/bash
+#
+# This shell script is intended to be executed from cron on a periodic basis.
+# The goal is to keep a Standby PE Monolithic master in sync with an active
+# Primary.  Pass the FQDN of the primary as ARG 1 and the FQDN of the secondary
+# as ARG 2
+#
+# In a DR situation when the secondary becomes active, block replication by
+# touching the lockfile.  This will prevent any changes made to the standby from
+# being clobbered as soon as the primary comes back online.
+#
+# Prior to re-enabling replication after a DR situation, replicate back to the
+# primary by reversing the direction of this script.
+
+set -euo pipefail
+
+PRIMARY="$1"
+STANDBY="$2"
+
+SOURCE="https://${PRIMARY}:4433/classification-api/v1"
+PATH="/opt/puppetlabs/puppet/bin:$PATH"
+lockfile='/etc/ncio_do_not_replicate'
+
+log() {
+  logger -t ncio-replicate -p daemon.warn -s "$1"
+}
+
+if [[ -e "$lockfile" ]]; then
+  log "WARN: Replication aborted, $lockfile exists!"
+  exit 1
+fi
+
+ncio --uri "$SOURCE" --retry-connections backup \
+  | ncio transform --hostname "${PRIMARY}:${STANDBY}" \
+  | ncio --retry-connections restore
+rval=$?
+
+[[ $rval -eq 0 ]] && STATUS='OK' || STATUS='ERROR'
+msg="INFO: Finished replicating puppet classification groups."
+log "$msg STATUS=${STATUS} EXITCODE=${rval} (touch $lockfile to disable)"
+exit $rval
+```
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at

data/ext/ncio-replicate

@@ -0,0 +1,41 @@
+#! /bin/bash
+#
+# This shell script is intended to be executed from cron on a periodic basis.
+# The goal is to keep a Standby PE Monolithic master in sync with an active
+# Primary.  Pass the FQDN of the primary as ARG 1 and the FQDN of the secondary
+# as ARG 2
+#
+# In a DR situation when the secondary becomes active, block replication by
+# touching the lockfile.  This will prevent any changes made to the standby from
+# being clobbered as soon as the primary comes back online.
+#
+# Prior to re-enabling replication after a DR situation, replicate back to the
+# primary by reversing the direction of this script.
+
+set -euo pipefail
+
+PRIMARY="$1"
+STANDBY="$2"
+
+SOURCE="https://${PRIMARY}:4433/classification-api/v1"
+PATH="/opt/puppetlabs/puppet/bin:$PATH"
+lockfile='/etc/ncio_do_not_replicate'
+
+log() {
+  logger -t ncio-replicate -p daemon.warn -s "$1"
+}
+
+if [[ -e "$lockfile" ]]; then
+  log "WARN: Replication aborted, $lockfile exists!"
+  exit 1
+fi
+
+ncio --uri "$SOURCE" backup \
+  | ncio transform --hostname "${PRIMARY}:${STANDBY}" \
+  | ncio restore
+rval=$?
+
+[[ $rval -eq 0 ]] && STATUS='OK' || STATUS='ERROR'
+msg="INFO: Finished replicating puppet classification groups."
+log "$msg STATUS=${STATUS} EXITCODE=${rval} (touch $lockfile to disable)"
+exit $rval

data/lib/ncio/api/v1.rb

@@ -1,5 +1,7 @@
 require 'ncio/api'
 require 'ncio/http_client'
+require 'ncio/support'
+require 'ncio/support/retry_action'
 require 'uri'
 require 'socket'
 require 'json'
@@ -17,6 +19,7 @@ module Ncio
       attr_reader :opts
 
       class ApiError < RuntimeError; end
+      class ApiAuthenticationError < RuntimeError; end
 
       DEFAULT_HEADERS = {
         'Content-Type' => 'application/json',
@@ -35,6 +38,10 @@ module Ncio
         @port = uri.port
       end
 
+      def log
+        Ncio::Support.log
+      end
+
       ##
       # Return a memoized HTTP connection
       def connection
@@ -49,6 +56,37 @@ module Ncio
         @connection = Ncio::HttpClient.new(conn_opts)
       end
 
+      ##
+      # Make a request respecting the timeout global option
+      #
+      # Assumes the timeout value is available in opts[:connect_timeout]
+      def request_with_timeout(req)
+        params = {
+          timeout: opts[:connect_timeout],
+          retry_exceptions: [Errno::ECONNREFUSED],
+          log: self.log,
+        }
+        Ncio::Support::RetryAction.retry_action(params) do
+          connection.request(req)
+        end
+      end
+
+      ##
+      # Make a request without a timeout
+      def request_without_timeout(req)
+        connection.request(req)
+      end
+
+      ##
+      # Make a request, return a response
+      def request(req)
+        if opts[:retry_connections]
+          request_with_timeout(req)
+        else
+          request_without_timeout(req)
+        end
+      end
+
       ##
       # Return all of the groups currently defined in the node classifier API.
       #
@@ -60,14 +98,27 @@ module Ncio
       def groups(inherited = false)
         uri = build_uri('groups', inherited: inherited.to_s)
         req = Net::HTTP::Get.new(uri, DEFAULT_HEADERS)
-        resp = connection.request(req)
-        if resp.code == '200'
+        resp = request(req)
+        obj = if resp.code == '200'
+                JSON.parse(resp.body)
+              else
+                raise_on_non_200(resp, 200)
+              end
+        obj
+      end
+
+      ##
+      # Handle a non 200 response.
+      def raise_on_non_200(resp, expected_code=200)
+        if resp.code == '401' && %r{rbac/user-unauthenticated}.match(resp.body)
           obj = JSON.parse(resp.body)
+          msg = obj['msg'] || '401 User Unauthenticated Error'
+          raise ApiAuthenticationError, msg
         else
-          msg = "Expected 200 response, got #{resp.code} body: #{resp.body}"
+          msg = "Expected #{expected_code} response, got #{resp.code} "\
+                "body: #{resp.body}"
           raise ApiError, msg
         end
-        obj
       end
 
       ##
@@ -80,10 +131,9 @@ module Ncio
         uri = build_uri('import-hierarchy')
         req = Net::HTTP::Post.new(uri, DEFAULT_HEADERS)
         req.body_stream = stream
-        resp = connection.request(req)
+        resp = request(req)
         return true if resp.code == '204'
-        msg = "Expected 204 response, got #{resp.code} body: #{resp.body}"
-        raise ApiError, msg
+        raise_on_non_200(resp, 204)
       end
 
       ##

data/lib/ncio/app.rb

@@ -2,6 +2,7 @@
 require 'ncio'
 require 'ncio/support'
 require 'ncio/support/option_parsing'
+require 'ncio/support/retry_action'
 require 'ncio/support/transform'
 require 'ncio/trollop'
 require 'ncio/version'
@@ -62,6 +63,11 @@ class App
       transform_groups
       return 0
     end
+  rescue Exception => e
+    msg = "ERROR: #{friendly_error(e)}"
+    fatal msg
+    $stderr.puts msg
+    return 1
   end
   # rubocop:enable Metrics/MethodLength
 
@@ -90,6 +96,7 @@ class App
   # Restore all groups in a manner suitable for the node classification
   # hierarchy import. See: [NC Import
   # Hierarchy](https://docs.puppet.com/pe/2016.1/nc_import-hierarchy.html)
+  # rubocop:disable Lint/RescueException
   def restore_groups
     warn 'Starting Node Classification Restore using '\
       "POST #{uri}/import-hierarchy"
@@ -105,6 +112,7 @@ class App
     fatal "ERROR Restoring backup: #{format_error e}"
     raise e
   end
+  # rubocop:enable Lint/RescueException
 
   ##
   # Transform a backup produced with backup_groups.  The transformation is

data/lib/ncio/support.rb

@@ -1,6 +1,7 @@
+require 'json'
 require 'logger'
+require 'stringio'
 require 'syslog/logger'
-require 'json'
 
 module Ncio
   ##
@@ -11,6 +12,10 @@ module Ncio
   module Support
     attr_reader :opts
 
+    ##
+    # Reset the global logger instance and return it as an object.
+    #
+    # @return [Logger] initialized logging instance
     def self.reset_logging!(opts)
       logger = opts[:syslog] ? syslog_logger : stream_logger(opts)
       @log = logger
@@ -38,7 +43,7 @@ module Ncio
     # Logging is handled centrally, the helper methods will delegate to the
     # centrally configured logging instance.
     def self.log
-      @log
+      @log || reset_logging!
     end
 
     ##
@@ -53,6 +58,7 @@ module Ncio
       when 'STDOUT' then $stdout
       when 'STDERR' then $stderr
       when 'STDIN' then $stdin
+      when 'STRING' then StringIO.new
       else File.expand_path(filepath)
       end
     end
@@ -155,10 +161,51 @@ module Ncio
     #
     # @param [Exception] e the exception to format
     def format_error(e)
-      data = { error: "#{e.class}", message: e.message, backtrace: e.backtrace }
+      data = { error: e.class.to_s, message: e.message, backtrace: e.backtrace }
       JSON.pretty_generate(data)
     end
 
+    ##
+    # Top level exception handler and friendly error message handler.
+    def friendly_error(e)
+      case e
+      when Ncio::Support::RetryAction::RetryException::Timeout
+        'Timeout expired connecting to the console service.  Verify it is up and running.'
+      when OpenSSL::SSL::SSLError
+        friendly_ssl_error(e)
+      when Ncio::Api::V1::ApiAuthenticationError
+        'Make sure the --cert option value is listed in the certificate whitelist, '\
+        'and you are able to run puppet agent --test on the master. '\
+        'The certificate whitelist on the master is located at '\
+        '/etc/puppetlabs/console-services/rbac-certificate-whitelist'
+      else
+        e.message
+      end
+    end
+
+    ##
+    # Handle SSL errors as a special case
+    def friendly_ssl_error(e)
+      case e.message
+      when %r{read server hello A}
+        'The socket connected, but there is no SSL service on the other side. '\
+        'This is often the case with TCP forwarding, e.g. in Vagrant '\
+        'or with SSH tunnels.'
+      when %r{state=error: certificate verify failed}
+        'The socket connected, but the certificate presented by the service could not '\
+        'be verified.  Make sure the value of the --cacert option points to an identical '\
+        'copy of the /etc/puppetlabs/puppet/ssl/certs/ca.pem file from the master.'
+      when %r{returned=5 errno=0 state=SSLv3 read finished A}
+        "The socket connected, but got back SSL error: #{e.message} "\
+        'This usually means the value of the --cert and --key options are certificates '\
+        'which are not signed by the same CA the service trusts.  This can often happen '\
+        'if the service has recently been re-installed.  Please obtain a valid cert and key '\
+        'and try again.'
+      else
+        "SSL Error: The socket is listening but something went wrong: #{e.message}"
+      end
+    end
+
     ##
     # Return the application version as a Semantic Version encoded string
     #

data/lib/ncio/support/option_parsing.rb

@@ -75,6 +75,13 @@ module Ncio
           opt :syslog, 'Log to syslog', default: true, conflicts: :logto
           opt :verbose, 'Set log level to INFO'
           opt :debug, 'Set log level to DEBUG'
+          opt :retry_connections, 'Retry API connections, '\
+              'e.g. waiting for the service to come online. '\
+              '{NCIO_RETRY_CONNECTIONS}',
+              default: (env['NCIO_RETRY_CONNECTIONS'] == 'true') || false
+          opt :connect_timeout, 'Retry <i> seconds if --retry-connections=true '\
+              '{NCIO_CONNECT_TIMEOUT}',
+              default: env['NCIO_CONNECT_TIMEOUT'] || CONNECT_TIMEOUT_DEFAULT
         end
       end
       # rubocop:enable Metrics/MethodLength, Metrics/AbcSize
@@ -198,6 +205,8 @@ Global options: (Note, command line arguments supersede ENV vars in {}'s)
 
       # Map is indexed by the subcommand
       FILE_DEFAULT_MAP = { 'backup' => 'STDOUT', 'restore' => 'STDIN' }.freeze
+
+      CONNECT_TIMEOUT_DEFAULT = 120
     end
   end
 end

data/lib/ncio/support/retry_action.rb

@@ -0,0 +1,79 @@
+require 'ncio/support'
+module Ncio
+  module Support
+    ##
+    # Provide a method to retry arbitrary code blocks with the ability to rescue
+    # certain exceptions and retry rather than failing hard on the exception.
+    #
+    # Copied from:
+    # https://github.com/puppetlabs/puppetlabs-cloud_provisioner/blob/f6cbac3/lib/puppet/cloudpack/utils.rb
+    module RetryAction
+      class RetryException < RuntimeError
+        class NoBlockGiven < RetryException; end
+        class NoTimeoutGiven < RetryException; end
+        class Timeout < RetryException; end
+      end
+
+      def self.timedout?(start, timeout)
+        return true if timeout.nil?
+        (Time.now - start) >= timeout
+      end
+
+      ##
+      # Retry an action, catching exceptions and retrying if the exception has
+      # been specified.
+      #
+      # rubocop:disable Metrics/PerceivedComplexity, Metrics/MethodLength
+      # rubocop:disable Metrics/CyclomaticComplexity, Metrics/AbcSize
+      def self.retry_action(params = { retry_exceptions: nil, timeout: nil, log: nil })
+        # Retry actions for a specified amount of time. This method will allow
+        # the final retry to complete even if that extends beyond the timeout
+        # period.
+        raise RetryException::NoBlockGiven unless block_given?
+
+        raise RetryException::NoTimeoutGiven if params[:timeout].nil?
+        params[:retry_exceptions] ||= []
+
+        # Assumes reset_logging! has been called.  This happens in the Ncio::App
+        # initialization.
+        log = params[:log] || Ncio::Support.log
+
+        start = Time.now
+        failures = 0
+
+        # rubocop:disable Lint/RescueException
+        begin
+          yield
+
+        rescue Exception => e
+          # If we were giving exceptions to catch,
+          # catch the exceptions we care about and retry.
+          # All others fail hard
+
+          raise RetryException::Timeout if timedout?(start, params[:timeout])
+
+          retry_exceptions = params[:retry_exceptions]
+
+          unless retry_exceptions.empty?
+            if retry_exceptions.include?(e.class)
+              log.warn("Retrying: #{e.class}: #{e}")
+            else
+              # If the exceptions is not in the list of retry_exceptions
+              # re-raise.
+              raise e
+            end
+          end
+          # rubocop:enable Lint/RescueException
+
+          failures += 1
+          # Increase the amount of time that we sleep after every
+          # failed retry attempt.
+          sleep(((2**failures) - 1) * 0.1)
+
+          retry
+        end
+        # rubocop:enable Metrics/PerceivedComplexity, Metrics/MethodLength
+      end
+    end
+  end
+end

data/lib/ncio/version.rb

@@ -1,3 +1,3 @@
 module Ncio
-  VERSION = '1.0.1'.freeze
+  VERSION = '1.1.0'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ncio
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.1.0
 platform: ruby
 authors:
 - Jeff McCune
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-06-29 00:00:00.000000000 Z
+date: 2016-08-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -136,6 +136,7 @@ files:
 - ".rubocop.yml"
 - ".travis.yml"
 - ".yardopts"
+- CHANGELOG.md
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -143,6 +144,7 @@ files:
 - bin/console
 - bin/setup
 - exe/ncio
+- ext/ncio-replicate
 - lib/ncio.rb
 - lib/ncio/api.rb
 - lib/ncio/api/v1.rb
@@ -150,6 +152,7 @@ files:
 - lib/ncio/http_client.rb
 - lib/ncio/support.rb
 - lib/ncio/support/option_parsing.rb
+- lib/ncio/support/retry_action.rb
 - lib/ncio/support/transform.rb
 - lib/ncio/trollop.rb
 - lib/ncio/version.rb
@@ -174,7 +177,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.2.5
 signing_key: 
 specification_version: 4
 summary: Puppet Node Classifier backup / restore / transform