octocatalog-diff 0.5.1

data/lib/octocatalog-diff/catalog-util/command.rb

@@ -0,0 +1,96 @@
+require 'fileutils'
+require 'open3'
+require 'shellwords'
+
+module OctocatalogDiff
+  module CatalogUtil
+    # Used to construct the command to run 'puppet' to construct the catalog.
+    class Command
+      # Constructor
+      def initialize(options = {}, logger = nil)
+        @options = options
+        @logger = logger
+
+        # Required parameters
+        @compilation_dir = options[:compilation_dir]
+        raise ArgumentError, 'Compile dir (:compilation_dir) must be a string' unless @compilation_dir.is_a?(String)
+        raise Errno::ENOENT, "Compile dir #{@compilation_dir} doesn't exist" unless File.exist?(@compilation_dir)
+        raise ArgumentError, "Compile dir #{@compilation_dir} not a directory" unless File.directory?(@compilation_dir)
+
+        @node = options[:node]
+        raise ArgumentError, 'Node must be specified to compile catalog' if @node.nil? || !@node.is_a?(String)
+      end
+
+      # Build up the command line to run Puppet
+      def puppet_command
+        cmdline = []
+
+        # Where is the puppet binary?
+        puppet = @options[:puppet_binary]
+        raise ArgumentError, 'Puppet binary was not supplied' if puppet.nil?
+        raise Errno::ENOENT, "Puppet binary #{puppet} doesn't exist" unless File.file?(puppet)
+        cmdline << puppet
+
+        # Node to compile
+        cmdline.concat ['master', '--compile', Shellwords.escape(@node)]
+
+        # storeconfigs?
+        if @options[:storeconfigs]
+          cmdline.concat %w(--storeconfigs --storeconfigs_backend=puppetdb)
+        else
+          cmdline << '--no-storeconfigs'
+        end
+
+        # enc?
+        if @options[:enc]
+          raise Errno::ENOENT, "Did not find ENC as expected at #{@options[:enc]}" unless File.file?(@options[:enc])
+          cmdline << "--node_terminus=exec --external_nodes=#{Shellwords.escape(@options[:enc])}"
+        end
+
+        # Future parser?
+        cmdline << '--parser=future' if @options[:parser] == :future
+
+        # Path to facts, or a specific fact file?
+        facts_terminus = @options.fetch(:facts_terminus, 'yaml')
+        if facts_terminus == 'yaml'
+          cmdline << "--factpath=#{Shellwords.escape(File.join(@compilation_dir, 'var', 'yaml', 'facts'))}"
+          if @options[:fact_file].is_a?(String) && @options[:fact_file] =~ /.*\.(\w+)$/
+            fact_file = File.join(@compilation_dir, 'var', 'yaml', 'facts', "#{@node}.#{Regexp.last_match(1)}")
+            FileUtils.cp @options[:fact_file], fact_file unless File.file?(fact_file) || @options[:fact_file] == fact_file
+          end
+          cmdline << '--facts_terminus=yaml'
+        elsif facts_terminus == 'facter'
+          cmdline << '--facts_terminus=facter'
+        else
+          raise ArgumentError, "Unrecognized facts_terminus setting: '#{facts_terminus}'"
+        end
+
+        # Some typical options for puppet
+        cmdline.concat %w(
+          --no-daemonize
+          --no-ca
+          --color=false
+          --config_version="/bin/echo catalogscript"
+          --environment=production
+        )
+
+        # For people who aren't running hiera, a hiera-config will not be generated when @options[:hiera_config]
+        # is nil. For everyone else, the hiera config was generated/copied/munged in the 'builddir' class
+        # and was installed into the compile directory and named hiera.yaml.
+        unless @options[:hiera_config].nil?
+          cmdline << "--hiera_config=#{Shellwords.escape(File.join(@compilation_dir, 'hiera.yaml'))}"
+        end
+
+        # Options with parameters
+        cmdline << "--environmentpath=#{Shellwords.escape(File.join(@compilation_dir, 'environments'))}"
+        cmdline << "--vardir=#{Shellwords.escape(File.join(@compilation_dir, 'var'))}"
+        cmdline << "--logdir=#{Shellwords.escape(File.join(@compilation_dir, 'var'))}"
+        cmdline << "--ssldir=#{Shellwords.escape(File.join(@compilation_dir, 'var', 'ssl'))}"
+        cmdline << "--confdir=#{Shellwords.escape(@compilation_dir)}"
+
+        # Return full command
+        cmdline.join(' ')
+      end
+    end
+  end
+end

data/lib/octocatalog-diff/catalog-util/enc.rb

@@ -0,0 +1,77 @@
+require_relative 'enc/noop'
+require_relative 'enc/pe'
+require_relative 'enc/script'
+
+require 'stringio'
+
+module OctocatalogDiff
+  module CatalogUtil
+    # Support a generic ENC. It must use one of the supported backends found in the
+    # 'enc' subdirectory.
+    class ENC
+      attr_reader :builder
+
+      # Constructor
+      # @param :backend [Symbol] If set, this will force a backend
+      # @param :enc [String] Path to ENC script (node_terminus = exec)
+      # @param # FIXME: Add support for PE's ENC endpoint API
+      def initialize(options = {})
+        @options = options
+
+        # Determine appropriate backend based on options supplied
+        @enc_obj = backend
+
+        # Initialize instance variables for content and error message.
+        @builder = @enc_obj.class.to_s
+
+        # Set the executed flag to false, so that it can be executed when something is retrieved.
+        @executed = false
+      end
+
+      # Retrieve content
+      # @return [String] ENC content, or nil if there was an error
+      def content
+        execute
+        @content ||= @enc_obj.content
+      end
+
+      # Retrieve error message
+      # @return [String] Error message, or nil if there was no error
+      def error_message
+        execute
+        @error_message ||= @enc_obj.error_message
+      end
+
+      private
+
+      # Backend - given options, choose an appropriate backend and construct the corresponding object.
+      # @return [?] Backend object
+      def backend
+        # Hard-coded backend
+        if @options[:backend]
+          return OctocatalogDiff::CatalogUtil::ENC::Noop.new(@options) if @options[:backend] == :noop
+          return OctocatalogDiff::CatalogUtil::ENC::PE.new(@options) if @options[:backend] == :pe
+          return OctocatalogDiff::CatalogUtil::ENC::Script.new(@options) if @options[:backend] == :script
+          raise ArgumentError, "Unknown backend :#{@options[:backend]}"
+        end
+
+        # Determine backend based on arguments
+        return OctocatalogDiff::CatalogUtil::ENC::PE.new(@options) if @options[:pe_enc_url]
+        return OctocatalogDiff::CatalogUtil::ENC::Script.new(@options) if @options[:enc]
+
+        # At this point we do not know what backend to use for the ENC
+        raise ArgumentError, 'Unable to determine ENC backend to use'
+      end
+
+      # Execute the 'execute' method of the object, but only once
+      # @param [Logger] Logger (optional) - if not supplied any logger messages will be discarded
+      def execute(logger = nil)
+        return if @executed
+        logger ||= @options[:logger]
+        logger ||= Logger.new(StringIO.new)
+        @enc_obj.execute(logger) if @enc_obj.respond_to?(:execute)
+        @executed = true
+      end
+    end
+  end
+end

data/lib/octocatalog-diff/catalog-util/enc/noop.rb

@@ -0,0 +1,22 @@
+module OctocatalogDiff
+  module CatalogUtil
+    class ENC
+      # No-op ENC.
+      class Noop
+        # Constructor
+        def initialize(_options)
+        end
+
+        # Retrieve content
+        def content
+          ''
+        end
+
+        # Error message
+        def error_message
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/octocatalog-diff/catalog-util/enc/pe.rb

@@ -0,0 +1,99 @@
+require_relative 'pe/v1'
+require_relative '../../util/httparty'
+require_relative '../facts'
+
+module OctocatalogDiff
+  module CatalogUtil
+    class ENC
+      # Support the Puppet Enterprise classification API.
+      # Documentation: https://docs.puppet.com/pe/latest/nc_index.html
+      class PE
+        # Error class that can be caught
+        class ClassificationError < RuntimeError; end
+
+        # Allow the main ENC object to retrieve these values
+        attr_reader :content, :error_message
+
+        # Constructor
+        # @param options [Hash] Options - must contain the Puppet Enterprise URL and the node
+        def initialize(options)
+          # Make sure the node is in the options
+          raise ArgumentError, 'OctocatalogDiff::CatalogUtil::ENC::PE#new requires :node' unless options.key?(:node)
+          @node = options[:node]
+
+          # Retrieve the base URL for the Puppet Enterprise ENC service
+          raise ArgumentError, 'OctocatalogDiff::CatalogUtil::ENC::PE#new requires :pe_enc_url' unless options.key?(:pe_enc_url)
+
+          # Save options
+          @options = options
+
+          # Get the object corresponding to the version of the API in use.
+          # (Right now this is hard-coded at V1 because that is the only version there is. In the future
+          # if there are different versions, this will need to be parameterized.)
+          @api = OctocatalogDiff::CatalogUtil::ENC::PE::V1.new(@options)
+
+          # Initialize the content and error message
+          @content = nil
+          @error_message = 'The execute method was never run'
+        end
+
+        # Executor
+        # @param logger [Logger] Logger object
+        def execute(logger)
+          logger.debug "Beginning OctocatalogDiff::CatalogUtil::ENC::PE#execute for #{@node}"
+
+          @options[:facts] ||= facts(logger)
+          return unless @options[:facts]
+
+          more_options = { headers: @api.headers, timeout: @options[:timeout] || 10 }
+          post_hash = @api.body
+          url = @api.url
+          response = OctocatalogDiff::Util::HTTParty.post(url, @options.merge(more_options), post_hash, 'pe_enc')
+
+          unless response[:code] == 200
+            logger.debug "PE ENC failed: #{response.inspect}"
+            logger.error "PE ENC failed: Response from #{url} was #{response[:code]}"
+            @error_message = "Response from #{url} was #{response[:code]}"
+            return
+          end
+
+          logger.debug "Response from #{url} was #{response[:code]}"
+          unless response[:parsed].is_a?(Hash)
+            logger.error "PE ENC failed: Response from #{url} was not a hash! #{response[:parsed].inspect}"
+            @error_message = "PE ENC failed: Response from #{url} was not a hash! #{response[:parsed].class}"
+            return
+          end
+
+          begin
+            @content = @api.result(response[:parsed], logger)
+            @error_message = nil
+          rescue OctocatalogDiff::CatalogUtil::ENC::PE::ClassificationError => exc
+            @error_message = exc.message
+            logger.error "PE ENC failed: #{exc.message}"
+            return
+          end
+
+          logger.debug "Completed OctocatalogDiff::CatalogUtil::ENC::PE#execute for #{@node}"
+        end
+
+        # Facts
+        # @param logger [Logger] Logger object
+        # @return [OctocatalogDiff::Facts] Facts object
+        def facts(logger)
+          facts_obj = OctocatalogDiff::CatalogUtil::Facts.new(@options, logger)
+          logger.debug "Start retrieving facts for #{@node} from #{self.class}"
+          begin
+            result = facts_obj.facts
+            logger.debug "Success retrieving facts for #{@node} from #{self.class}"
+          rescue OctocatalogDiff::Facts::FactRetrievalError, OctocatalogDiff::Facts::FactSourceError => exc
+            @content = nil
+            @error_message = "Fact retrieval failed: #{exc.class} - #{exc.message}"
+            logger.error @error_message
+            result = nil
+          end
+          result
+        end
+      end
+    end
+  end
+end

data/lib/octocatalog-diff/catalog-util/enc/pe/v1.rb

@@ -0,0 +1,61 @@
+require 'json'
+require 'uri'
+require 'yaml'
+
+module OctocatalogDiff
+  module CatalogUtil
+    class ENC
+      class PE
+        # Support the Puppet Enterprise classification API.
+        # Documentation: https://docs.puppet.com/pe/latest/nc_index.html
+        # This is version 1 of the API
+        class V1
+          # Constructor
+          # @param options [Hash] All input options
+          def initialize(options)
+            @options = options
+          end
+
+          # Return the URL to the API
+          # @return [String] API URL
+          def url
+            "#{@options[:pe_enc_url]}/v1/classified/nodes/#{@options[:node]}"
+          end
+
+          # Headers
+          # @return [Hash] Headers for request
+          def headers
+            result = {
+              'Accept' => 'application/json',
+              'Content-Type' => 'application/json'
+            }
+            result['X-Authentication'] = @options[:pe_enc_token] if @options[:pe_enc_token]
+            result
+          end
+
+          # POST body
+          # @return [String] POST body data
+          def body
+            raise ":facts required (got #{@options[:facts].class})" unless @options[:facts].is_a?(OctocatalogDiff::Facts)
+            { 'fact' => @options[:facts].facts }.to_json
+          end
+
+          # Parse response from ENC and return the final ENC data
+          # @param parsed [Parsed response] Parsed response from ENC
+          # @param logger [Logger] Logger.object
+          # @return [String] ENC data as text
+          def result(parsed, logger)
+            %w(classes parameters).each do |required_key|
+              next if parsed[required_key]
+              logger.debug parsed.keys.inspect
+              raise OctocatalogDiff::CatalogUtil::ENC::PE::ClassificationError, "Response missing: #{required_key}"
+            end
+
+            obj = { 'classes' => parsed['classes'], 'parameters' => parsed['parameters'] }
+            obj.to_yaml
+          end
+        end
+      end
+    end
+  end
+end

data/lib/octocatalog-diff/catalog-util/enc/script.rb

@@ -0,0 +1,88 @@
+require 'fileutils'
+require 'open3'
+require 'shellwords'
+require 'tempfile'
+
+module OctocatalogDiff
+  module CatalogUtil
+    class ENC
+      # Support an ENC that executes a script on this system which returns the ENC data on STDOUT.
+      class Script
+        attr_reader :content, :error_message, :script
+
+        # Constructor
+        # @param options [Hash] Options - must contain script name and node name, plus tempdir if it's a relative path
+        def initialize(options)
+          # Make sure the node is in the options
+          raise ArgumentError, 'OctocatalogDiff::CatalogUtil::ENC::Script#new requires :node' unless options.key?(:node)
+          @node = options[:node]
+
+          # Determine path to ENC and make sure it exists
+          raise ArgumentError, 'OctocatalogDiff::CatalogUtil::ENC::Script#new requires :enc' unless options.key?(:enc)
+          @script = script_path(options[:enc], options[:tempdir])
+
+          # Other options we may recognize
+          @pass_env_vars = options.fetch(:pass_env_vars, [])
+
+          # Initialize the content and error message
+          @content = nil
+          @error_message = 'The execute method was never run'
+        end
+
+        # Executor
+        # @param logger [Logger] Logger object
+        def execute(logger)
+          logger.debug "Beginning OctocatalogDiff::CatalogUtil::ENC::Script#execute for #{@node} with #{@script}"
+          logger.debug "Passing these extra environment variables: #{@pass_env_vars}" if @pass_env_vars.any?
+
+          # Copy the script and make it executable
+          # Then run the command in the restricted environment
+          raise Errno::ENOENT, "ENC #{@script} wasn't found" unless File.file?(@script)
+          file = Tempfile.open('enc.sh')
+          file.close
+          begin
+            FileUtils.cp @script, file.path
+            FileUtils.chmod 0o755, file.path
+            env = {
+              'HOME' => ENV['HOME'],
+              'PATH' => ENV['PATH'],
+              'PWD' => File.dirname(@script)
+            }
+            @pass_env_vars.each { |var| env[var] ||= ENV[var] }
+            command = [file.path, @node].map { |x| Shellwords.escape(x) }.join(' ')
+            out, err, status = Open3.capture3(env, command, unsetenv_others: true, chdir: File.dirname(@script))
+            logger.debug "ENC exited #{status.exitstatus}: #{out.length} bytes to STDOUT, #{err.length} bytes to STDERR"
+          ensure
+            file.unlink
+          end
+
+          # Analyze the output
+          if status.exitstatus.zero?
+            @content = out
+            @error_message = nil
+            logger.warn "ENC STDERR: #{err}" unless err.empty?
+          else
+            @content = nil
+            @error_message = "ENC failed with status #{status.exitstatus}: #{out} #{err}"
+            logger.error "ENC failed - Status #{status.exitstatus}"
+            logger.error "Failed ENC printed this to STDOUT: #{out}" unless out.empty?
+            logger.error "Failed ENC printed this to STDERR: #{err}" unless err.empty?
+          end
+        end
+
+        private
+
+        # Determine the script path for the incoming file -- absolute or relative
+        # @param enc [String] Path to ENC supplied by user/config
+        # @param tempdir [String]
+        # @return [String] Full path to file on system
+        def script_path(enc, tempdir)
+          return enc if enc.start_with? '/'
+          raise ArgumentError, 'OctocatalogDiff::CatalogUtil::ENC::Script#new requires :tempdir' unless tempdir.is_a?(String)
+          return File.join(tempdir, enc) if enc =~ %r{^environments/production/}
+          File.join(tempdir, 'environments', 'production', enc)
+        end
+      end
+    end
+  end
+end