abide_dev_utils 0.5.2 → 0.9.0

data/lib/abide_dev_utils/errors.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require 'abide_dev_utils/errors/base'
+require 'abide_dev_utils/errors/comply'
+require 'abide_dev_utils/errors/gcloud'
 require 'abide_dev_utils/errors/general'
 require 'abide_dev_utils/errors/jira'
 require 'abide_dev_utils/errors/xccdf'

data/lib/abide_dev_utils/gcloud.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'abide_dev_utils/errors/gcloud'
+
+module AbideDevUtils
+  module GCloud
+    include AbideDevUtils::Errors::GCloud
+
+    def self.storage_bucket(name: nil, project: nil, credentials: nil)
+      raise MissingProjectError if project.nil? && ENV['ABIDE_GCLOUD_PROJECT'].nil?
+      raise MissingCredentialsError if credentials.nil? && ENV['ABIDE_GCLOUD_CREDENTIALS'].nil?
+      raise MissingBucketNameError if name.nil? && ENV['ABIDE_GCLOUD_BUCKET'].nil?
+
+      require 'google/cloud/storage'
+      @bucket = Google::Cloud::Storage.new(
+        project_id: project || ENV['ABIDE_GCLOUD_PROJECT'],
+        credentials: credentials || JSON.parse(ENV['ABIDE_GCLOUD_CREDENTIALS'])
+      ).bucket(name || ENV['ABIDE_GCLOUD_BUCKET'])
+    end
+  end
+end

data/lib/abide_dev_utils/mixins.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module AbideDevUtils
+  module Mixins
+    # mixin methods for the Hash data type
+    module Hash
+      def deep_copy
+        Marshal.load(Marshal.dump(self))
+      end
+
+      def diff(other)
+        dup.delete_if { |k, v| other[k] == v }.merge!(other.dup.delete_if { |k, _| key?(k) })
+      end
+    end
+  end
+end

data/lib/abide_dev_utils/output.rb

@@ -22,9 +22,13 @@ module AbideDevUtils
     end
 
     def self.yaml(in_obj, console: false, file: nil)
-      AbideDevUtils::Validate.hashable(in_obj)
-      # Use object's #to_yaml method if it exists, convert to hash if not
-      yaml_out = in_obj.respond_to?(:to_yaml) ? in_obj.to_yaml : in_obj.to_h.to_yaml
+      yaml_out = if in_obj.is_a? String
+                   in_obj
+                 else
+                   AbideDevUtils::Validate.hashable(in_obj)
+                   # Use object's #to_yaml method if it exists, convert to hash if not
+                   in_obj.respond_to?(:to_yaml) ? in_obj.to_yaml : in_obj.to_h.to_yaml
+                 end
       simple(yaml_out) if console
       FWRITER.write_yaml(yaml_out, file: file) unless file.nil?
     end

data/lib/abide_dev_utils/ppt/api.rb

@@ -0,0 +1,219 @@
+# frozen_string_literal: true
+
+require 'io/console'
+require 'json'
+require 'net/http'
+require 'openssl'
+
+module AbideDevUtils
+  module Ppt
+    class ApiClient
+      attr_reader :hostname, :custom_ports
+      attr_writer :auth_token, :tls_cert_verify
+      attr_accessor :content_type
+
+      CT_JSON = 'application/json'
+      API_DEFS = {
+        codemanager: {
+          port: 8170,
+          version: 'v1',
+          base: 'code-manager',
+          paths: [
+            {
+              path: 'deploys',
+              verbs: %w[post],
+              x_auth: true
+            }
+          ]
+        },
+        classifier1: {
+          port: 4433,
+          version: 'v1',
+          base: 'classifier-api',
+          paths: [
+            {
+              path: 'groups',
+              verbs: %w[get post],
+              x_auth: true
+            }
+          ]
+        },
+        orchestrator: {
+          port: 8143,
+          version: 'v1',
+          base: 'orchestrator',
+          paths: [
+            {
+              path: 'command/deploy',
+              verbs: %w[post],
+              x_auth: true
+            },
+            {
+              path: 'command/task',
+              verbs: %w[post],
+              x_auth: true
+            },
+            {
+              path: 'jobs',
+              verbs: %w[get],
+              x_auth: true
+            }
+          ]
+        }
+      }.freeze
+
+      def initialize(hostname, auth_token: nil, content_type: CT_JSON, custom_ports: {}, verbose: false)
+        @hostname = hostname
+        @auth_token = auth_token
+        @content_type = content_type
+        @custom_ports = custom_ports
+        @verbose = verbose
+        define_api_methods
+      end
+
+      def login(username, password: nil, lifetime: '1h', label: nil)
+        label = "AbideDevUtils token for #{username} - lifetime #{lifetime}" if label.nil?
+        password = IO.console.getpass 'Password: ' if password.nil?
+        data = {
+          'login' => username,
+          'password' => password,
+          'lifetime' => lifetime,
+          'label' => label
+        }
+        uri = URI("https://#{@hostname}:4433/rbac-api/v1/auth/token")
+        result = http_request(uri, post_request(uri, x_auth: false, **data), json_out: true)
+        @auth_token = result['token']
+        log_verbose("Successfully logged in? #{auth_token?}")
+        auth_token?
+      end
+
+      def auth_token?
+        defined?(@auth_token) && !@auth_token.nil? && !@auth_token.empty?
+      end
+
+      def tls_cert_verify
+        @tls_cert_verify = defined?(@tls_cert_verify) ? @tls_cert_verify : false
+      end
+
+      def verbose?
+        @verbose
+      end
+
+      def no_verbose
+        @verbose = false
+      end
+
+      def verbose!
+        @verbose = true
+      end
+
+      private
+
+      def define_api_methods
+        api_method_data.each do |meth, data|
+          case meth
+          when /^get_.*/
+            self.class.define_method(meth) do |*args, **kwargs|
+              uri = args.empty? ? data[:uri] : URI("#{data[:uri]}/#{args.join('/')}")
+              req = get_request(uri, x_auth: data[:x_auth], **kwargs)
+              http_request(data[:uri], req, json_out: true)
+            end
+          when /^post_.*/
+            self.class.define_method(meth) do |*args, **kwargs|
+              uri = args.empty? ? data[:uri] : URI("#{data[:uri]}/#{args.join('/')}")
+              req = post_request(uri, x_auth: data[:x_auth], **kwargs)
+              http_request(data[:uri], req, json_out: true)
+            end
+          else
+            raise "Cannot define method for #{meth}"
+          end
+        end
+      end
+
+      def api_method_data
+        method_data = {}
+        API_DEFS.each do |key, val|
+          val[:paths].each do |path|
+            method_names = api_method_names(key, path)
+            method_names.each do |name|
+              method_data[name] = {
+                uri: api_method_uri(val[:port], val[:base], val[:version], path[:path]),
+                x_auth: path[:x_auth]
+              }
+            end
+          end
+        end
+        method_data
+      end
+
+      def api_method_names(api_name, path)
+        path[:verbs].each_with_object([]) do |verb, ary|
+          path_str = path[:path].split('/').join('_')
+          ary << [verb, api_name.to_s, path_str].join('_')
+        end
+      end
+
+      def api_method_uri(port, base, version, path)
+        URI("https://#{@hostname}:#{port}/#{base}/#{version}/#{path}")
+      end
+
+      def get_request(uri, x_auth: true, **qparams)
+        log_verbose('New GET request:')
+        log_verbose("request_qparams?: #{!qparams.empty?}")
+        uri.query = URI.encode_www_form(qparams) unless qparams.empty?
+        headers = init_headers(x_auth: x_auth)
+        log_verbose("request_headers: #{redact_headers(headers)}")
+        Net::HTTP::Get.new(uri, headers)
+      end
+
+      def post_request(uri, x_auth: true, **data)
+        log_verbose('New POST request:')
+        log_verbose("request_data?: #{!data.empty?}")
+        headers = init_headers(x_auth: x_auth)
+        log_verbose("request_headers: #{redact_headers(headers)}")
+        req = Net::HTTP::Post.new(uri, headers)
+        req.body = data.to_json unless data.empty?
+        req
+      end
+
+      def init_headers(x_auth: true)
+        headers = { 'Content-Type' => @content_type }
+        return headers unless x_auth
+
+        raise 'Auth token not set!' unless auth_token?
+
+        headers['X-Authentication'] = @auth_token
+        headers
+      end
+
+      def http_request(uri, req, json_out: true)
+        result = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true, verify_mode: tls_verify_mode) do |http|
+          log_verbose("use_ssl: true, verify_mode: #{tls_verify_mode}")
+          http.request(req)
+        end
+        case result.code
+        when '200', '201', '202'
+          json_out ? JSON.parse(result.body) : result
+        else
+          jbody = JSON.parse(result.body)
+          log_verbose("HTTP #{result.code} #{jbody['kind']} #{jbody['msg']} #{jbody['details']} #{uri}")
+          raise "HTTP #{result.code} #{jbody['kind']} #{jbody['msg']} #{jbody['details']} #{uri}"
+        end
+      end
+
+      def log_verbose(msg)
+        puts msg if @verbose
+      end
+
+      def redact_headers(headers)
+        r_headers = headers.dup
+        r_headers['X-Authentication'] = 'XXXXX' if r_headers.key?('X-Authentication')
+        r_headers
+      end
+
+      def tls_verify_mode
+        tls_cert_verify ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
+      end
+    end
+  end
+end

data/lib/abide_dev_utils/ppt/class_utils.rb

@@ -0,0 +1,184 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'tempfile'
+require 'abide_dev_utils/errors/ppt'
+
+module AbideDevUtils
+  module Ppt
+    module ClassUtils
+      include AbideDevUtils::Errors::Ppt
+
+      CLASS_NAME_PATTERN = /\A([a-z][a-z0-9_]*)?(::[a-z][a-z0-9_]*)*\Z/.freeze
+      CLASS_NAME_CAPTURE_PATTERN = /\A^class (?<class_name>([a-z][a-z0-9_]*)?(::[a-z][a-z0-9_]*)*).*\Z/.freeze
+
+      # Validates a Puppet class name
+      # @param name [String] Puppet class name
+      # @return [Boolean] Is the name a valid Puppet class name
+      def self.valid_class_name?(name)
+        name.match?(CLASS_NAME_PATTERN)
+      end
+
+      # Takes a full Puppet class name and returns the path
+      # of the class file. This command must be run from the
+      # root module directory if validate_path is true.
+      # @param class_name [String] full Puppet class name
+      # @return [String] path to class file
+      def self.path_from_class_name(class_name)
+        parts = class_name.split('::')
+        parts[-1] = "#{parts[-1]}.pp"
+        File.expand_path(File.join('manifests', parts[1..-1]))
+      end
+
+      # Returns the namespaced class name from a file path
+      # @param class_path [String] the path to the Puppet class
+      # @return [String] the namespaced class name
+      def self.class_name_from_path(class_path)
+        parts = class_path.split(File::SEPARATOR).map { |x| x == '' ? File::SEPARATOR : x }
+        module_root_idx = parts.find_index('manifests') - 1
+        module_root = parts[module_root_idx].split('-')[-1]
+        namespaces = parts[(module_root_idx + 2)..-2].join('::') # add 2 to module root idx to skip manifests dir
+        class_name = parts[-1].delete_suffix('.pp')
+        [module_root, namespaces, class_name].join('::')
+      end
+
+      # Takes a path to a Puppet file and extracts the class name from the class declaration in the file.
+      # This differs from class_name_from_path because we actually read the class file and search
+      # the code for a class declaration to get the class name instead of just using the path
+      # to construct a valid Puppet class name.
+      # @param path [String] the path to a Puppet file
+      # @return [String] the Puppet class name
+      # @raise [ClassDeclarationNotFoundError] if there is not class declaration in the file
+      def self.class_name_from_declaration(path)
+        File.readlines(path).each do |line|
+          next unless line.match?(/^class /)
+
+          return CLASS_NAME_CAPTURE_PATTERN.match(line)['class_name']
+        end
+        raise ClassDeclarationNotFoundError, "Path:#{path}"
+      end
+
+      # Renames a file by file move. Ensures destination path exists before moving.
+      # @param from_path [String] path of the original file
+      # @param to_path [String] path of the new file
+      # @param verbose [Boolean] Sets verbose mode on file operations
+      # @param force [Boolean] If true, file move file overwrite existing files
+      def self.rename_class_file(from_path, to_path, **kwargs)
+        verbose = kwargs.fetch(:verbose, false)
+        force = kwargs.fetch(:force, false)
+        FileUtils.mkdir_p(File.dirname(to_path), verbose: verbose)
+        FileUtils.mv(from_path, to_path, verbose: verbose, force: force)
+      end
+
+      # Renames a Puppet class in the class declaration of the given file
+      # @param from [String] the original class name
+      # @param to [String] the new class name
+      # @param file_path [String] the path to the class file
+      # @param verbose [Boolean] Sets verbose mode on file operations
+      # @param force [Boolean] If true, file move file overwrite existing files
+      # @raise [ClassDeclarationNotFoundError] if the class file does not contain the from class declaration
+      def self.rename_puppet_class_declaration(from, to, file_path, **kwargs)
+        verbose = kwargs.fetch(:verbose, false)
+        force = kwargs.fetch(:force, false)
+        temp_file = Tempfile.new
+        renamed = false
+        begin
+          File.readlines(file_path).each do |line|
+            if line.match?(/^class #{from}.*/)
+              line.gsub!(/^class #{from}/, "class #{to}")
+              renamed = true
+            end
+            temp_file.puts line
+          end
+          raise ClassDeclarationNotFoundError, "File:#{file_path},Declaration:class #{from}" unless renamed
+
+          temp_file.close
+          FileUtils.mv(temp_file.path, file_path, verbose: verbose, force: force)
+        ensure
+          temp_file.close
+          temp_file.unlink
+        end
+      end
+
+      # Determines if a Puppet class name is mismatched by constructing a class name from
+      # a path to a Puppet file and extracting the class name from the class declaration
+      # inside the file. This is useful to determine if a Puppet class file breaks the
+      # autoload path pattern.
+      # @param path [String] path to a Puppet class file
+      # @return [Boolean] if the actual class name and path-constructed class name match
+      def self.mismatched_class_declaration?(path)
+        class_name_from_path(path) != class_name_from_declaration(path)
+      end
+
+      # Finds all Puppet classes in the given directory that have class declarations
+      # that do not adhere to the autoload path pattern.
+      # @param class_dir [String] path to a directory containing Puppet class files
+      # @return [Array] paths to all Puppet class files with mismatched class names
+      def self.find_all_mismatched_class_declarations(class_dir)
+        mismatched = []
+        Dir[File.join(File.expand_path(class_dir), '*.pp')].each do |class_file|
+          mismatched << class_file if mismatched_class_declaration?(class_file)
+        end
+        mismatched.sort
+      end
+
+      # Given a directory holding Puppet manifests, returns
+      # the full namespace for all classes in that directory.
+      # @param puppet_class_dir [String] path to a dir containing Puppet manifests
+      # @return [String] The namespace for all classes in manifests in the dir
+      def self.find_class_namespace(puppet_class_dir)
+        path = Pathname.new(puppet_class_dir)
+        mod_root = nil
+        ns_parts = []
+        found_manifests = false
+        path.ascend do |p|
+          if found_manifests
+            mod_root = find_mod_root(p)
+            break
+          end
+          if File.basename(p) == 'manifests'
+            found_manifests = true
+            next
+          else
+            ns_parts << File.basename(p)
+          end
+        end
+        "#{mod_root}::#{ns_parts.reverse.join('::')}::"
+      end
+
+      # Given a Pathname object of the 'manifests' directory in a Puppet module,
+      # determines the module namespace root. Does this by consulting
+      # metadata.json, if it exists, or by using the parent directory name.
+      # @param pathname [Pathname] A Pathname object of the module's manifests dir
+      # @return [String] The module's namespace root
+      def self.find_mod_root(pathname)
+        metadata_file = nil
+        pathname.entries.each do |e|
+          metadata_file = "#{pathname}/metadata.json" if File.basename(e) == 'metadata.json'
+        end
+        if metadata_file.nil?
+          File.basename(p)
+        else
+          File.open(metadata_file) do |f|
+            file = JSON.parse(f.read)
+            File.basename(p) unless file.key?('name')
+            file['name'].split('-')[-1]
+          end
+        end
+      end
+
+      # @return [Array] An array of frozen arrays where each sub-array's
+      #   index 0 is class_name and index 1 is the full path to the file.
+      def self.find_all_classes_and_paths(puppet_class_dir)
+        all_cap = []
+        Dir.each_child(puppet_class_dir) do |c|
+          path = "#{puppet_class_dir}/#{c}"
+          next if File.directory?(path) || File.extname(path) != '.pp'
+
+          all_cap << [File.basename(path, '.pp'), path].freeze
+        end
+        all_cap
+      end
+    end
+  end
+end

data/lib/abide_dev_utils/ppt/coverage.rb

@@ -4,16 +4,15 @@ require 'json'
 require 'pathname'
 require 'yaml'
 require 'puppet_pal'
-require 'abide_dev_utils/ppt'
+require 'abide_dev_utils/ppt/class_utils'
 
 module AbideDevUtils
   module Ppt
     class CoverageReport
-      include AbideDevUtils::Ppt
       def self.generate(puppet_class_dir, hiera_path, profile = nil)
         coverage = {}
         coverage['classes'] = {}
-        all_cap = AbideDevUtils::Ppt.find_all_classes_and_paths(puppet_class_dir)
+        all_cap = ClassUtils.find_all_classes_and_paths(puppet_class_dir)
         invalid_classes = find_invalid_classes(all_cap)
         valid_classes = find_valid_classes(all_cap, invalid_classes)
         coverage['classes']['invalid'] = invalid_classes

data/lib/abide_dev_utils/ppt/score_module.rb

@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+
+require 'pathname'
+require 'metadata-json-lint'
+require 'puppet-lint'
+require 'json'
+
+module AbideDevUtils
+  module Ppt
+    class ScoreModule
+      attr_reader :module_name, :module_dir, :manifests_dir
+
+      def initialize(module_dir)
+        @module_name = module_dir.split(File::SEPARATOR)[-1]
+        @module_dir = real_module_dir(module_dir)
+        @manifests_dir = File.join(real_module_dir(module_dir), 'manifests')
+        @metadata = JSON.parse(File.join(@module_dir, 'metadata.json'))
+      end
+
+      def lint
+        linter_exit_code, linter_output = lint_manifests
+        {
+          exit_code: linter_exit_code,
+          manifests: manifest_count,
+          lines: line_count,
+          linter_version: linter_version,
+          output: linter_output
+        }.to_json
+      end
+
+      # def metadata
+
+      # end
+
+      private
+
+      def manifests
+        @manifests ||= Dir["#{manifests_dir}/**/*.pp"]
+      end
+
+      def manifest_count
+        @manifest_count ||= manifests.count
+      end
+
+      def line_count
+        @line_count ||= manifests.each_with_object([]) { |x, ary| ary << File.readlines(x).size }.sum
+      end
+
+      def lint_manifests
+        results = []
+        PuppetLint.configuration.with_filename = true
+        PuppetLint.configuration.json = true
+        PuppetLint.configuration.relative = true
+        linter_exit_code = 0
+        manifests.each do |manifest|
+          next if PuppetLint.configuration.ignore_paths.any? { |p| File.fnmatch(p, manifest) }
+
+          linter = PuppetLint.new
+          linter.file = manifest
+          linter.run
+          linter_exit_code = 1 if linter.errors? || linter.warnings?
+          results << linter.problems.reject { |x| x[:kind] == :ignored }
+        end
+        [linter_exit_code, JSON.generate(results)]
+      end
+
+      def lint_metadata
+        results = { errors: [], warnings: [] }
+        results[:errors] << metadata_schema_errors
+        dep_errors, dep_warnings = metadata_validate_deps
+        results[:errors] << dep_errors
+        results[:warnings] << dep_warnings
+        results[:errors] << metadata_deprecated_fields
+      end
+
+      def metadata_schema_errors
+        MetadataJsonLint::Schema.new.validate(@metadata).each_with_object([]) do |err, ary|
+          check = err[:field] == 'root' ? :required_fields : err[:field]
+          ary << metadata_err(check, err[:message])
+        end
+      end
+
+      def metadata_validate_deps
+        return [[], []] unless @metadata.key?('dependencies')
+
+        errors, warnings = []
+        duplicates = metadata_dep_duplicates
+        warnings << duplicates unless duplicates.empty?
+        @metadata['dependencies'].each do |dep|
+          e, w = metadata_dep_version_requirement(dep)
+          errors << e unless e.nil?
+          warnings << w unless w.nil?
+          warnings << metadata_dep_version_range(dep['name']) if dep.key?('version_range')
+        end
+        [errors.flatten, warnings.flatten]
+      end
+
+      def metadata_deprecated_fields
+        %w[types checksum].each_with_object([]) do |field, ary|
+          next unless @metadata.key?(field)
+
+          ary << metadata_err(:deprecated_fields, "Deprecated field '#{field}' found in metadata.json")
+        end
+      end
+
+      def metadata_dep_duplicates
+        results = []
+        duplicates = @metadata['dependencies'].detect { |x| @metadata['dependencies'].count(x) > 1 }
+        return results if duplicates.empty?
+
+        duplicates.each { |x| results << metadata_err(:dependencies, "Duplicate dependencies on #{x}") }
+        results
+      end
+
+      def metadata_dep_version_requirement(dependency)
+        unless dependency.key?('version_requirement')
+          return [metadata_err(:dependencies, "Invalid 'version_requirement' field in metadata.json: #{e}"), nil]
+        end
+
+        ver_req = MetadataJsonLint::VersionRequirement.new(dependency['version_requirement'])
+        return [nil, metadata_dep_open_ended(dependency['name'], dependency['version_requirement'])] if ver_req.open_ended?
+        return [nil, metadata_dep_mixed_syntax(dependency['name'], dependency['version_requirement'])] if ver_req.mixed_syntax?
+
+        [nil, nil]
+      end
+
+      def metadata_dep_open_ended(name, version_req)
+        metadata_err(:dependencies, "Dependency #{name} has an open ended dependency version requirement #{version_req}")
+      end
+
+      def metadata_dep_mixed_syntax(name, version_req)
+        msg = 'Mixing "x" or "*" version syntax with operators is not recommended in ' \
+              "metadata.json, use one style in the #{name} dependency: #{version_req}"
+        metadata_err(:dependencies, msg)
+      end
+
+      def metadata_dep_version_range(name)
+        metadata_err(:dependencies, "Dependency #{name} has a 'version_range' attribute which is no longer used by the forge.")
+      end
+
+      def metadata_err(check, msg)
+        { check: check, msg: msg }
+      end
+
+      def linter_version
+        PuppetLint::VERSION
+      end
+
+      def relative_manifests
+        Dir.glob('manifests/**/*.pp')
+      end
+
+      def real_module_dir(path)
+        return Pathname.pwd if path.nil?
+
+        return Pathname.new(path).cleanpath(consider_symlink: true) if Dir.exist?(path)
+
+        raise ArgumentError, "Path #{path} is not a directory"
+      end
+    end
+  end
+end