kafo_parsers 0.1.4 → 1.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: b0c3926a766e0ebd3f3d156451176f9d02e68a21
-  data.tar.gz: 98db6338da1c9acf47304c93dc4d56a75a07d6bb
+SHA256:
+  metadata.gz: 62b061db2fb2d7f269541ff5a7865f7482703fb4766432e3117980d51fb2cc0e
+  data.tar.gz: 2e6135c7beb9ccb754008f0ea1842c67a82b27cd8bcb96a43960f6e415f1a640
 SHA512:
-  metadata.gz: d23d9e3fc141268358563fef5ccf804d9b073a37b093cefb287652aff56a792c67c6fa222a3e42a48add29c5910fb053fd3fb7a6c66b255a755c728974ffb24b
-  data.tar.gz: 4ab94d2975ca76e8e634e655cb08d41816006539de778af31550af18f12d5973dc0dfa04d9b1c93be371beb51aa82fe60ad5c103a30257ecfe050f4a4cb1c47b
+  metadata.gz: e9ada8c7ac628dc9631a97cd9587200e2f5320449919ab7954f41c53b112c5115e436d2880fd5158bde2293538d7521c9c7b235a63fc295eeaf0155514c54c82
+  data.tar.gz: 7adf7c7d9c4851c0bc5668dfd1b34adc0e0792dccf30d4e83faf62f915cef876913cd6fe60904ed68073ee215c4a361dec99d04d034738b5346906a6e7523175

data/README.md

@@ -44,33 +44,99 @@ KafoParsers::Parsers.find_available(:logger => logger)
 
 To load a specific parser:
 ```ruby
-require 'kafo_parsers/puppet_module_parser'
-hash = KafoParsers::PuppetModuleParser.parse('/puppet/module/manifests/init.pp')
+require 'kafo_parsers/puppet_strings_module_parser'
+hash = KafoParsers::PuppetStringsModuleParser.parse('/puppet/module/manifests/init.pp')
 ```
 
-#### PuppetModuleParser
-
-The standard PuppetModuleParser loads Puppet as a regular library or gem, so it
-must be installed in the same Ruby that's running kafo_parsers.
-
-Only Puppet versions 2.6.x, 2.7.x and 3.x are supported.
-
-Add `gem 'puppet', '< 4'` to your application's Gemfile to use this.
-
 #### PuppetStringsModuleParser
 
-If you use Puppet 4 you prefer PuppetStringsModuleParser which leverages
-puppet-strings to parse puppet manifests. This requires puppet-strings 0.99 or
-higher and may be installed either as a gem in the same environment, or in a
-Puppet AIO installation.
+Leverage puppet-strings to parse puppet manifests. This requires puppet-strings
+1.2.0 or higher and may be installed either as a gem in the same environment,
+or in a Puppet AIO installation.
 
 ```ruby
 require 'kafo_parsers/puppet_strings_module_parser'
 hash = KafoParsers::PuppetStringsModuleParser.parse('/puppet/module/manifests/init.pp')
 ```
 
-Output will be similar to PuppetModuleParser, only validations are not supported,
-since they are not parsed by puppet-strings.
+## Documentation syntax
+
+### RDoc syntax
+
+Classes and defined types should be prefixed with a comment section with an RDoc
+block, containing a description, headings for different parameter groups and
+parameters laid out as shown below:
+
+```puppet
+# Example class that installs Example
+#
+# Supports version 1 to 3.
+#
+# === Parameters::
+#
+# $foo::  Sets the value of foo in the Example config
+#
+# === Advanced parameters::
+#
+# $bar::  Sets the value of bar in the advanced config
+```
+
+Parameters may have multi-line descriptions and can have extra attributes
+defined on new lines below them. Supports:
+
+```puppet
+# $foo::  Sets the value of foo in the Example config
+#         condition: $bar == 'use_foo'
+#         type: Optional[String]
+```
+
+Supports:
+
+* `condition:` an expression to determine if the parameter is used
+* `type:` the data type of the parameter
+
+Used by:
+
+* `PuppetStringsModuleParser` (but deprecated, prefer YARD)
+
+### YARD syntax
+
+Classes and defined types should be prefixed with a comment section in YARD
+following the Puppet Strings documentation standard, as shown below:
+
+```puppet
+# Example class that installs Example
+#
+# Supports version 1 to 3.
+#
+# @param foo Sets the value of foo in the Example config
+# @param bar Sets the value of bar in the advanced config
+#            group: Advanced parameters
+```
+
+Parameters may have multi-line descriptions and can have extra attributes
+defined on new lines below them. Supports:
+
+```puppet
+# @param foo Sets the value of foo in the Example config
+#            condition: $bar == 'use_foo'
+```
+
+Supports:
+
+* `condition:` an expression to determine if the parameter is used
+* `group:` comma-separated list of groups, increasing in specificity
+
+Data types are given in the parameter list of the class, or can be given inline
+for Puppet 3 compatibility:
+
+```puppet
+# @param foo [Integer] Sets the value of foo in the Example config
+```
+
+Used by:
+
+* `PuppetStringsModuleParser`
 
 # License
 

data/lib/kafo_parsers/doc_parser.rb

@@ -3,6 +3,7 @@ require 'rdoc'
 require 'rdoc/markup' # required for RDoc < 0.9.5
 require 'rdoc/markup/parser' # required for RDoc < 0.9.5
 require 'kafo_parsers/exceptions'
+require 'kafo_parsers/param_doc_parser'
 
 module KafoParsers
   class DocParser
@@ -15,7 +16,7 @@ module KafoParsers
       @docs           = {}
       @groups         = {}
       @conditions     = {}
-      @types          = Hash.new('string')
+      @types          = {}
       @rdoc           = rdoc_parser.parse(@text)
     end
 
@@ -46,34 +47,17 @@ module KafoParsers
       return if label.nil?
       key              = label.gsub(/[^A-Za-z0-9_-]/, '')
       @groups[key]     = current_groups
-      text_parts       = para.parts.first.parts.map!(&:strip)
-      attributes, docs = text_parts.partition { |line| line =~ ATTRIBUTE_LINE }
-      parse_attributes(key, attributes)
-      @docs[key] = docs
-    end
+      text_parts       = para.parts.first.parts
 
-    def parse_attributes(parameter, attributes)
-      condition = nil
-      attributes.each do |attribute|
-        data        = attribute.match(ATTRIBUTE_LINE)
-        name, value = data[1], data[2]
-
-        case name
-          when 'type'
-            @types[parameter] = value
-          when 'condition'
-            if condition.nil?
-              condition = value
-            else
-              raise KafoParsers::DocParseError, "Two or more conditions defined for #{name}"
-            end
-          else
-            raise KafoParsers::DocParseError, "Unknown attribute #{name}"
-        end
+      param_parser = ParamDocParser.new(key, text_parts)
+      @docs[key] = param_parser.doc
+      parse_attributes(key, param_parser)
+    end
 
-      end
-      condition              = [current_condition, condition].compact.join(' && ')
+    def parse_attributes(parameter, parser)
+      condition              = [current_condition, parser.condition].compact.join(' && ')
       @conditions[parameter] = condition.empty? ? nil : condition
+      @types[parameter]      = parser.type unless parser.type.nil?
     end
 
     def parse_header(heading)

data/lib/kafo_parsers/param_doc_parser.rb

@@ -0,0 +1,42 @@
+require 'kafo_parsers/exceptions'
+
+module KafoParsers
+  class ParamDocParser
+    ATTRIBUTE_LINE   = /^(condition|group|type)\s*:\s*(.*)/
+
+    def initialize(param, text)
+      @param = param
+      @metadata = {}
+      parse_paragraph([text].flatten)
+    end
+
+    attr_reader :param, :doc
+    [:condition, :group, :type].each do |attr|
+      define_method(attr) do
+        @metadata[attr]
+      end
+    end
+
+    private
+
+    def parse_paragraph(text)
+      text_parts       = text.map(&:strip)
+      attributes, docs = text_parts.partition { |line| line =~ ATTRIBUTE_LINE }
+      parse_attributes(attributes)
+      @doc = docs
+    end
+
+    def parse_attributes(attributes)
+      attributes.each do |attribute|
+        data        = attribute.match(ATTRIBUTE_LINE)
+        name, value = data[1], data[2]
+        raise KafoParsers::DocParseError, "Two or more #{name} lines defined for #{param}" if @metadata.key?(name)
+        @metadata[name.to_sym] = value
+      end
+
+      if @metadata.key?(:group)
+        @metadata[:group] = @metadata[:group].split(/\s*,\s*/)
+      end
+    end
+  end
+end

data/lib/kafo_parsers/parsers.rb

@@ -1,10 +1,9 @@
-require 'kafo_parsers/puppet_module_parser.rb'
 require 'kafo_parsers/puppet_strings_module_parser.rb'
 
 module KafoParsers
   module Parsers
     def self.all
-      [ PuppetModuleParser, PuppetStringsModuleParser ]
+      [ PuppetStringsModuleParser ]
     end
 
     def self.find_available(options = {})

data/lib/kafo_parsers/puppet_strings_module_parser.rb

@@ -1,6 +1,9 @@
 # encoding: UTF-8
 require 'json'
+require 'open3'
+
 require 'kafo_parsers/doc_parser'
+require 'kafo_parsers/param_doc_parser'
 
 module KafoParsers
   class PuppetStringsModuleParser
@@ -24,25 +27,25 @@ module KafoParsers
     end
 
     def self.available?
-      `#{puppet_bin} help strings 2>&1`
-      if $?.success?
+      _stdout, _stderr, status = run_puppet(['help', 'strings'])
+      if status.success?
         return true
       else
-        raise KafoParsers::ParserNotAvailable.new("#{puppet_bin} does not have strings module installed")
+        raise KafoParsers::ParserNotAvailable.new("#{puppet_bin} does not have strings module installed.")
       end
     end
 
     def initialize(file)
-      @file = file
-      raise KafoParsers::ModuleName, "File not found #{file}, check your answer file" unless File.exists?(file)
+      @file = file = File.expand_path(file)
+      raise KafoParsers::ModuleName, "File not found #{file}, check your answer file" unless File.exist?(file)
+
+      command = ['strings', 'generate', '--format', 'json', file]
+      @raw_json, stderr, status = self.class.run_puppet(command)
 
-      command = "#{self.class.puppet_bin} strings generate --emit-json-stdout #{file}"
-      @raw_json = `#{command}`
-      unless $?.success?
-        raise KafoParsers::ParseError, "'#{command}' returned error\n#{@raw_json}"
+      unless status.success?
+        raise KafoParsers::ParseError, "'#{command}' returned error:\n  #{@raw_json}\n  #{stderr}"
       end
 
-      Encoding.default_external = Encoding::UTF_8 if defined?(Encoding)
       begin
         @complete_hash = ::JSON.parse(@raw_json)
       rescue ::JSON::ParserError => e
@@ -53,15 +56,17 @@ module KafoParsers
       self
     end
 
-    # AIO and system default puppet bins are tested for existence, fallback to just `puppet` otherwise
     def self.puppet_bin
-      @puppet_bin ||= begin
-        found_puppet_path = (::ENV['PATH'].split(File::PATH_SEPARATOR) + ['/opt/puppetlabs/bin']).find do |path|
-          binary = File.join(path, 'puppet')
-          binary if File.executable?(binary)
-        end
-        found_puppet_path.nil? ? 'puppet' : File.join(found_puppet_path, 'puppet')
+      @puppet_bin ||= self.find_puppet_bin
+    end
+
+    # AIO and system default puppet bins are tested for existence, fallback to just `puppet` otherwise
+    def self.find_puppet_bin
+      found_puppet_path = (::ENV['PATH'].split(File::PATH_SEPARATOR) + ['/opt/puppetlabs/bin']).find do |path|
+        binary = File.join(path, 'puppet')
+        binary if File.executable?(binary) && !File.symlink?(binary)
       end
+      found_puppet_path.nil? ? 'puppet' : File.join(found_puppet_path, 'puppet')
     end
 
     def data_type
@@ -91,7 +96,7 @@ module KafoParsers
 
     # returns data in following form
     # {
-    #   :docs => { $param1 => 'documentation without types and conditions'}
+    #   :docs => { $param1 => ['documentation without types and conditions']}
     #   :types => { $param1 => 'boolean'},
     #   :groups => { $param1 => ['Parameters', 'Advanced']},
     #   :conditions => { $param1 => '$db_type == "mysql"'},
@@ -101,19 +106,52 @@ module KafoParsers
       if @parsed_hash.nil?
         raise KafoParsers::DocParseError, "no documentation found for manifest #{@file}, parsing error?"
       elsif !@parsed_hash['docstring'].nil? && !@parsed_hash['docstring']['text'].nil?
-        parser             = DocParser.new(@parsed_hash['docstring']['text']).parse
-        data[:docs]        = parser.docs
-        data[:groups]      = parser.groups
-        data[:types]       = merge_doc_types(parser.types)
-        data[:conditions]  = parser.conditions
+        # Lowest precedence: types given in the strings hash from class definition
+        tag_params.each do |param|
+          data[:types][param['name']] = param['types'][0] unless param['types'].nil?
+        end
+
+        # Next: types and other data from RDoc parser
+        rdoc_parser = DocParser.new(@parsed_hash['docstring']['text']).parse
+        data[:docs] = rdoc_parser.docs
+        data[:groups] = rdoc_parser.groups
+        data[:conditions] = rdoc_parser.conditions
+        data[:types].merge! rdoc_parser.types
+
+        # Highest precedence: data in YARD @param stored in the 'text' field
+        tag_params.each do |param|
+          param_name = param['name']
+          unless param['text'].nil? || param['text'].empty?
+            param_parser = ParamDocParser.new(param_name, param['text'].split($/))
+            data[:docs][param_name] = param_parser.doc if param_parser.doc
+            data[:groups][param_name] = param_parser.group if param_parser.group
+            data[:conditions][param_name] = param_parser.condition if param_parser.condition
+            data[:types][param_name] = param_parser.type if param_parser.type
+          end
+        end
       end
       data
     end
 
     private
 
+    def self.run_puppet(command)
+      env_vars = self.puppet_bin.start_with?('/opt/puppetlabs') ? clean_env_vars : ::ENV
+      command = command.unshift(self.puppet_bin)
+      Open3.capture3(env_vars, *command, :unsetenv_others => true)
+    end
+
+    def self.clean_env_vars
+      # Cleaning ENV vars and keeping required vars only because,
+      # When using SCL it adds GEM_HOME and GEM_PATH ENV vars.
+      whitelisted_vars = %w[HOME USER LANG]
+
+      cleaned_env = ::ENV.select { |var| whitelisted_vars.include?(var) || var.start_with?('LC_') }
+      cleaned_env['PATH'] = '/sbin:/bin:/usr/sbin:/usr/bin:/opt/puppetlabs/bin'
+      cleaned_env
+    end
+
     # default values using puppet strings includes $ symbol, e.g. "$::foreman::params::ssl"
-    # to keep the same API we strip $ if it's present
     #
     # values are reported as strings which is issue especially for :under
     # strings are double quoted
@@ -122,16 +160,11 @@ module KafoParsers
       if (value.start_with?("'") && value.end_with?("'")) || (value.start_with?('"') && value.end_with?('"'))
         value = value[1..-2]
       end
-      value = value[1..-1] if value.start_with?('$')
       value = :undef if value == 'undef'
 
       value
     end
 
-    def merge_doc_types(types)
-      Hash[tag_params.select { |param| !param['types'].nil? }.map { |param| [ param['name'], param['types'].first ] }].merge(types)
-    end
-
     def tag_params
       if @parsed_hash['docstring']['tags']
         @parsed_hash['docstring']['tags'].select { |tag| tag['tag_name'] == 'param' }

data/lib/kafo_parsers/version.rb

@@ -1,3 +1,3 @@
 module KafoParsers
-  VERSION = "0.1.4"
+  VERSION = "1.1.1"
 end

metadata

@@ -1,85 +1,85 @@
 --- !ruby/object:Gem::Specification
 name: kafo_parsers
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 1.1.1
 platform: ruby
 authors:
 - Marek Hulan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-31 00:00:00.000000000 Z
+date: 2020-08-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.5'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.5'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '11'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '11'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.9'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.9'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: ci_reporter
+  name: ci_reporter_minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.9.0
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.9.0
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
@@ -122,14 +122,13 @@ files:
 - lib/kafo_parsers.rb
 - lib/kafo_parsers/doc_parser.rb
 - lib/kafo_parsers/exceptions.rb
+- lib/kafo_parsers/param_doc_parser.rb
 - lib/kafo_parsers/parsers.rb
-- lib/kafo_parsers/puppet_module_parser.rb
 - lib/kafo_parsers/puppet_strings_module_parser.rb
-- lib/kafo_parsers/validation.rb
 - lib/kafo_parsers/version.rb
 homepage: https://github.com/theforeman/kafo_parsers
 licenses:
-- GPLv3+
+- GPL-3.0+
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -146,8 +145,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Puppet module parsers

data/lib/kafo_parsers/puppet_module_parser.rb

@@ -1,125 +0,0 @@
-# encoding: UTF-8
-require 'kafo_parsers/doc_parser'
-require 'kafo_parsers/validation'
-
-module KafoParsers
-  # Based on ideas from puppet-parse by Johan van den Dorpe
-  # we don't build any tree structure since e.g. params from doc does not
-  # have to be defined in puppet DSL and vice versa, we just gather all info
-  # we can read from the whole manifest
-  class PuppetModuleParser
-    @@puppet_initialized = false
-
-    def self.available?
-      require 'puppet'
-      [2, 3].include?(Puppet::PUPPETVERSION.to_i)
-    rescue LoadError => e
-      raise KafoParsers::ParserNotAvailable.new(e)
-    end
-
-    # You can call this method to get all supported information from a given manifest
-    #
-    # @param [ String ] manifest file path to parse
-    # @return [ Hash ] hash containing values, validations, documentation, types, groups and conditions
-    def self.parse(file)
-      content = new(file)
-      docs    = content.docs
-
-      data              = {
-          :values      => content.values,
-          :validations => content.validations
-      }
-      data[:parameters] = data[:values].keys
-      data.merge!(docs)
-      data
-    end
-
-    def initialize(file)
-      @file = file
-      raise KafoParsers::ModuleName, "File not found #{file}, check your answer file" unless File.exists?(file)
-
-      unless @@puppet_initialized
-        require 'puppet'
-        if Puppet::PUPPETVERSION.to_i >= 3
-          Puppet.initialize_settings
-        else
-          Puppet.parse_config
-        end
-        Encoding.default_external = Encoding::UTF_8 if defined?(Encoding)
-        @@puppet_initialized = true
-      end
-
-      env = Puppet::Node::Environment.new
-      parser = Puppet::Parser::Parser.new(env)
-      parser.import(@file)
-
-      # Find object corresponding to class defined in init.pp in list of hostclasses
-      ast_types = parser.environment.known_resource_types.hostclasses.map(&:last)
-      @object = ast_types.find { |ast_type| ast_type.file == file }
-
-      # Find object in list of definitions if not found among hostclasses
-      if @object.nil?
-        ast_types = parser.environment.known_resource_types.definitions.map(&:last)
-        @object = ast_types.find { |ast_type| ast_type.file == file }
-      end
-
-      parser
-    end
-
-    # TODO - store parsed object type (Puppet::Parser::AST::Variable must be dumped later)
-    def values
-      parameters = {}
-      arguments  = @object.respond_to?(:arguments) ? @object.arguments : {}
-      arguments.each { |k, v| parameters[k] = v.respond_to?(:value) ? v.value : nil }
-      parameters
-    end
-
-    def validations(param = nil)
-      return [] if @object.code.nil?
-      @object.code.select { |stmt| stmt.is_a?(Puppet::Parser::AST::Function) && stmt.name =~ /^validate_/ }.map do |v|
-        Validation.new(v.name, interpret_validation_args(v.arguments))
-      end
-    end
-
-    # returns data in following form
-    # {
-    #   :docs => { $param1 => 'documentation without types and conditions'}
-    #   :types => { $param1 => 'boolean'},
-    #   :groups => { $param1 => ['Parameters', 'Advanced']},
-    #   :conditions => { $param1 => '$db_type == "mysql"'},
-    #   :object_type => 'hostclass' # or definition
-    # }
-    def docs
-      data = { :docs => {}, :types => {}, :groups => {}, :conditions => {}, :object_type => '' }
-      if @object.nil?
-        raise KafoParsers::DocParseError, "no documentation found for manifest #{@file}, parsing error?"
-      elsif !@object.doc.nil?
-        parser             = DocParser.new(@object.doc).parse
-        data[:docs]        = parser.docs
-        data[:groups]      = parser.groups
-        data[:types]       = parser.types
-        data[:conditions]  = parser.conditions
-        data[:object_type] = @object.type.to_s
-      end
-      data
-    end
-
-    private
-
-    def interpret_validation_args(args)
-      args.map do |arg|
-        if arg.is_a?(Puppet::Parser::AST::Variable)
-          arg.to_s
-        elsif arg.is_a?(Puppet::Parser::AST::ASTArray)
-          interpret_validation_args(arg.to_a)
-        elsif arg.is_a?(Puppet::Parser::AST::Concat)
-          interpret_validation_args(arg.value).join
-        elsif arg.respond_to? :value
-          arg.value
-        else
-          arg
-        end
-      end
-    end
-  end
-end

data/lib/kafo_parsers/validation.rb

@@ -1,14 +0,0 @@
-module KafoParsers
-  class Validation
-    attr_reader :name, :arguments
-
-    def initialize(name, arguments)
-      @name = name
-      @arguments = arguments
-    end
-
-    def ==(other)
-      name == other.name && arguments == other.arguments
-    end
-  end
-end