pdk 1.16.0 → 1.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +24 -0
  3. data/lib/pdk.rb +25 -18
  4. data/lib/pdk/answer_file.rb +2 -93
  5. data/lib/pdk/cli.rb +1 -5
  6. data/lib/pdk/cli/config.rb +3 -1
  7. data/lib/pdk/cli/config/get.rb +3 -1
  8. data/lib/pdk/cli/convert.rb +1 -1
  9. data/lib/pdk/cli/exec/command.rb +13 -0
  10. data/lib/pdk/cli/exec_group.rb +78 -43
  11. data/lib/pdk/cli/get.rb +20 -0
  12. data/lib/pdk/cli/get/config.rb +24 -0
  13. data/lib/pdk/cli/util.rb +6 -3
  14. data/lib/pdk/cli/validate.rb +26 -44
  15. data/lib/pdk/config.rb +178 -4
  16. data/lib/pdk/config/ini_file.rb +183 -0
  17. data/lib/pdk/config/ini_file_setting.rb +39 -0
  18. data/lib/pdk/config/namespace.rb +25 -5
  19. data/lib/pdk/config/setting.rb +3 -2
  20. data/lib/pdk/context.rb +96 -0
  21. data/lib/pdk/context/control_repo.rb +60 -0
  22. data/lib/pdk/context/module.rb +28 -0
  23. data/lib/pdk/context/none.rb +22 -0
  24. data/lib/pdk/control_repo.rb +40 -0
  25. data/lib/pdk/generate/module.rb +8 -12
  26. data/lib/pdk/module/release.rb +2 -8
  27. data/lib/pdk/util.rb +35 -5
  28. data/lib/pdk/util/bundler.rb +1 -0
  29. data/lib/pdk/util/changelog_generator.rb +6 -1
  30. data/lib/pdk/util/template_uri.rb +4 -3
  31. data/lib/pdk/validate.rb +72 -25
  32. data/lib/pdk/validate/external_command_validator.rb +208 -0
  33. data/lib/pdk/validate/internal_ruby_validator.rb +100 -0
  34. data/lib/pdk/validate/invokable_validator.rb +216 -0
  35. data/lib/pdk/validate/metadata/metadata_json_lint_validator.rb +86 -0
  36. data/lib/pdk/validate/metadata/metadata_syntax_validator.rb +78 -0
  37. data/lib/pdk/validate/metadata/metadata_validator_group.rb +20 -0
  38. data/lib/pdk/validate/puppet/puppet_epp_validator.rb +133 -0
  39. data/lib/pdk/validate/puppet/puppet_lint_validator.rb +66 -0
  40. data/lib/pdk/validate/puppet/puppet_syntax_validator.rb +137 -0
  41. data/lib/pdk/validate/puppet/puppet_validator_group.rb +21 -0
  42. data/lib/pdk/validate/ruby/ruby_rubocop_validator.rb +80 -0
  43. data/lib/pdk/validate/ruby/ruby_validator_group.rb +19 -0
  44. data/lib/pdk/validate/tasks/tasks_metadata_lint_validator.rb +88 -0
  45. data/lib/pdk/validate/tasks/tasks_name_validator.rb +50 -0
  46. data/lib/pdk/validate/tasks/tasks_validator_group.rb +20 -0
  47. data/lib/pdk/validate/validator.rb +111 -0
  48. data/lib/pdk/validate/validator_group.rb +103 -0
  49. data/lib/pdk/validate/yaml/yaml_syntax_validator.rb +95 -0
  50. data/lib/pdk/validate/yaml/yaml_validator_group.rb +19 -0
  51. data/lib/pdk/version.rb +1 -1
  52. data/locales/pdk.pot +161 -125
  53. metadata +29 -17
  54. data/lib/pdk/validate/base_validator.rb +0 -215
  55. data/lib/pdk/validate/metadata/metadata_json_lint.rb +0 -82
  56. data/lib/pdk/validate/metadata/metadata_syntax.rb +0 -111
  57. data/lib/pdk/validate/metadata_validator.rb +0 -26
  58. data/lib/pdk/validate/puppet/puppet_epp.rb +0 -135
  59. data/lib/pdk/validate/puppet/puppet_lint.rb +0 -64
  60. data/lib/pdk/validate/puppet/puppet_syntax.rb +0 -135
  61. data/lib/pdk/validate/puppet_validator.rb +0 -26
  62. data/lib/pdk/validate/ruby/rubocop.rb +0 -72
  63. data/lib/pdk/validate/ruby_validator.rb +0 -26
  64. data/lib/pdk/validate/tasks/metadata_lint.rb +0 -130
  65. data/lib/pdk/validate/tasks/name.rb +0 -90
  66. data/lib/pdk/validate/tasks_validator.rb +0 -29
  67. data/lib/pdk/validate/yaml/syntax.rb +0 -125
  68. data/lib/pdk/validate/yaml_validator.rb +0 -28
@@ -0,0 +1,216 @@
1
+ require 'pdk'
2
+
3
+ module PDK
4
+ module Validate
5
+ # A base class for file based validators.
6
+ # This class provides base methods and helpers to help determine the file targets to validate against.
7
+ # Acutal validator implementation should inherit from other child abstract classes e.g. ExternalCommandValdiator
8
+ # @see PDK::Validate::Validator
9
+ # @abstract
10
+ class InvokableValidator < Validator
11
+ # Controls how many times the validator is invoked.
12
+ #
13
+ # :once - The validator will be invoked once and passed all the
14
+ # targets.
15
+ # :per_target - The validator will be invoked for each target
16
+ # separately.
17
+ # @abstract
18
+ def invoke_style
19
+ :once
20
+ end
21
+
22
+ # Whether this Validator can be invoked in this context. By default any Validator can work in any Context, except ::None
23
+ # @return [Boolean]
24
+ # @abstract
25
+ def valid_in_context?
26
+ !context.is_a?(PDK::Context::None)
27
+ end
28
+
29
+ # An array, or a string, of glob patterns to use to find targets
30
+ # @return [Array[String], String]
31
+ # @abstract
32
+ def pattern; end
33
+
34
+ # An array, or a string, of glob patterns to use to ignore targets
35
+ # @return [Array[String], String, Nil]
36
+ # @abstract
37
+ def pattern_ignore; end
38
+
39
+ # @see PDK::Validate::Validator.prepare_invoke!
40
+ def prepare_invoke!
41
+ return if @prepared
42
+ super
43
+
44
+ # Register the spinner
45
+ spinner
46
+ nil
47
+ end
48
+
49
+ # Parses the target strings provided from the CLI
50
+ #
51
+ # @param options [Hash] A Hash containing the input options from the CLI.
52
+ #
53
+ # @return targets [Array] An Array of Strings containing target file paths
54
+ # for the validator to validate.
55
+ # @return skipped [Array] An Array of Strings containing targets
56
+ # that are skipped due to target not containing
57
+ # any files that can be validated by the validator.
58
+ # @return invalid [Array] An Array of Strings containing targets that do
59
+ # not exist, and will not be run by validator.
60
+ def parse_targets
61
+ requested_targets = options.fetch(:targets, [])
62
+ # If no targets are specified and empty targets are allowed return with an empty list.
63
+ # It will be up to the validator (and whatever validation tool it uses) to determine the
64
+ # targets. For example, using rubocop with no targets, will allow rubocop to determine the
65
+ # target list using it's .rubocop.yml file
66
+ return [[], [], []] if requested_targets.empty? && allow_empty_targets?
67
+ # If no targets are specified, then we will run validations from the base context directory.
68
+ targets = requested_targets.empty? ? [context.root_path] : requested_targets
69
+ targets.map! { |r| r.gsub(File::ALT_SEPARATOR, File::SEPARATOR) } if File::ALT_SEPARATOR
70
+
71
+ # If this validator is not valid in this context then skip all of the targets
72
+ return [[], targets, []] unless valid_in_context?
73
+
74
+ skipped = []
75
+ invalid = []
76
+ matched = targets.map { |target|
77
+ if pattern.nil?
78
+ target
79
+ else
80
+ if PDK::Util::Filesystem.directory?(target) # rubocop:disable Style/IfInsideElse
81
+ target_root = context.root_path
82
+ pattern_glob = Array(pattern).map { |p| PDK::Util::Filesystem.glob(File.join(target_root, p), File::FNM_DOTMATCH) }
83
+ target_list = pattern_glob.flatten
84
+ .select { |glob| PDK::Util::Filesystem.fnmatch(File.join(PDK::Util::Filesystem.expand_path(PDK::Util.canonical_path(target)), '*'), glob, File::FNM_DOTMATCH) }
85
+ .map { |glob| Pathname.new(glob).relative_path_from(Pathname.new(context.root_path)).to_s }
86
+
87
+ ignore_list = ignore_pathspec
88
+ target_list = target_list.reject { |file| ignore_list.match(file) }
89
+
90
+ skipped << target if target_list.flatten.empty?
91
+ target_list
92
+ elsif PDK::Util::Filesystem.file?(target)
93
+ if Array(pattern).include? target
94
+ target
95
+ elsif Array(pattern).any? { |p| PDK::Util::Filesystem.fnmatch(PDK::Util::Filesystem.expand_path(p), PDK::Util::Filesystem.expand_path(target), File::FNM_DOTMATCH) }
96
+ target
97
+ else
98
+ skipped << target
99
+ next
100
+ end
101
+ else
102
+ invalid << target
103
+ next
104
+ end
105
+ end
106
+ }.compact.flatten.uniq
107
+ [matched, skipped, invalid]
108
+ end
109
+
110
+ # Whether the target parsing ignores "dotfiles" (e.g. .gitignore or .pdkignore) which are considered hidden files in POSIX
111
+ # @return [Boolean]
112
+ # @abstract
113
+ def ignore_dotfiles?
114
+ true
115
+ end
116
+
117
+ # @see PDK::Validate::Validator.spinner_text
118
+ # @abstract
119
+ def spinner_text
120
+ _('Running %{name} validator ...') % { name: name }
121
+ end
122
+
123
+ # @see PDK::Validate::Validator.spinner
124
+ def spinner
125
+ return nil unless spinners_enabled?
126
+ return @spinner unless @spinner.nil?
127
+ require 'pdk/cli/util/spinner'
128
+
129
+ @spinner = TTY::Spinner.new("[:spinner] #{spinner_text}", PDK::CLI::Util.spinner_opts_for_platform)
130
+ end
131
+
132
+ # Process any targets that were skipped by the validator and add the events to the validation report
133
+ # @param report [PDK::Report] The report to add the events to
134
+ # @param skipped [Array[String]] The list of skipped targets
135
+ def process_skipped(report, skipped = [])
136
+ skipped.each do |skipped_target|
137
+ PDK.logger.debug(_('%{validator}: Skipped \'%{target}\'. Target does not contain any files to validate (%{pattern}).') % { validator: name, target: skipped_target, pattern: pattern })
138
+ report.add_event(
139
+ file: skipped_target,
140
+ source: name,
141
+ message: _('Target does not contain any files to validate (%{pattern}).') % { pattern: pattern },
142
+ severity: :info,
143
+ state: :skipped,
144
+ )
145
+ end
146
+ end
147
+
148
+ # Process any targets that were invalid by the validator and add the events to the validation report
149
+ # @param report [PDK::Report] The report to add the events to
150
+ # @param invalid [Array[String]] The list of invalid targets
151
+ def process_invalid(report, invalid = [])
152
+ invalid.each do |invalid_target|
153
+ PDK.logger.debug(_('%{validator}: Skipped \'%{target}\'. Target file not found.') % { validator: name, target: invalid_target })
154
+ report.add_event(
155
+ file: invalid_target,
156
+ source: name,
157
+ message: _('File does not exist.'),
158
+ severity: :error,
159
+ state: :error,
160
+ )
161
+ end
162
+ end
163
+
164
+ # Controls how the validator behaves if not passed any targets.
165
+ #
166
+ # true - PDK will not pass the globbed targets to the validator command
167
+ # and it will instead rely on the underlying tool to find its
168
+ # own default targets.
169
+ # false - PDK will pass the globbed targets to the validator command.
170
+ # @abstract
171
+ def allow_empty_targets?
172
+ false
173
+ end
174
+
175
+ protected
176
+
177
+ # Takes the pattern used in a module context and transforms it depending on the
178
+ # context e.g. A Control Repo will use the module pattern in each module path
179
+ #
180
+ # @param [Array, String] The pattern when used in the module root. Does not start with '/'
181
+ #
182
+ # @return [Array[String]]
183
+ def contextual_pattern(module_pattern)
184
+ module_pattern = [module_pattern] unless module_pattern.is_a?(Array)
185
+ return module_pattern unless context.is_a?(PDK::Context::ControlRepo)
186
+ context.actualized_module_paths.map { |mod_path| module_pattern.map { |pat_path| mod_path + '/*/' + pat_path } }.flatten
187
+ end
188
+
189
+ private
190
+
191
+ # Helper method to collate the default ignored paths
192
+ # @return [PathSpec] Paths to ignore
193
+ def ignore_pathspec
194
+ ignore_pathspec = if context.is_a?(PDK::Context::Module)
195
+ require 'pdk/module'
196
+ PDK::Module.default_ignored_pathspec(ignore_dotfiles?)
197
+ elsif context.is_a?(PDK::Context::ControlRepo)
198
+ require 'pdk/control_repo'
199
+ PDK::ControlRepo.default_ignored_pathspec(ignore_dotfiles?)
200
+ else
201
+ PathSpec.new.tap do |ps|
202
+ ps.add('.*') if ignore_dotfiles?
203
+ end
204
+ end
205
+
206
+ unless pattern_ignore.nil?
207
+ Array(pattern_ignore).each do |pattern|
208
+ ignore_pathspec.add(pattern)
209
+ end
210
+ end
211
+
212
+ ignore_pathspec
213
+ end
214
+ end
215
+ end
216
+ end
@@ -0,0 +1,86 @@
1
+ require 'pdk'
2
+
3
+ module PDK
4
+ module Validate
5
+ module Metadata
6
+ class MetadataJSONLintValidator < ExternalCommandValidator
7
+ # Validate each metadata file separately, as metadata-json-lint does not
8
+ # support multiple targets.
9
+ def invoke_style
10
+ :per_target
11
+ end
12
+
13
+ def name
14
+ 'metadata-json-lint'
15
+ end
16
+
17
+ def cmd
18
+ 'metadata-json-lint'
19
+ end
20
+
21
+ def spinner_text_for_targets(targets)
22
+ _('Checking module metadata style (%{targets}).') % {
23
+ targets: PDK::Util.targets_relative_to_pwd(targets.flatten).join(' '),
24
+ }
25
+ end
26
+
27
+ def pattern
28
+ contextual_pattern('metadata.json')
29
+ end
30
+
31
+ def parse_options(targets)
32
+ cmd_options = ['--format', 'json']
33
+ cmd_options << '--strict-dependencies'
34
+
35
+ cmd_options.concat(targets)
36
+ end
37
+
38
+ def parse_output(report, result, targets)
39
+ raise ArgumentError, _('More than 1 target provided to PDK::Validate::MetadataJSONLintValidator.') if targets.count > 1
40
+
41
+ if result[:stdout].strip.empty?
42
+ # metadata-json-lint will print nothing if there are no problems with
43
+ # the file being linted. This should be handled separately to
44
+ # metadata-json-lint generating output that can not be parsed as JSON
45
+ # (unhandled exception in metadata-json-lint).
46
+ json_data = {}
47
+ else
48
+ begin
49
+ json_data = JSON.parse(result[:stdout])
50
+ rescue JSON::ParserError
51
+ raise PDK::Validate::ParseOutputError, result[:stdout]
52
+ end
53
+ end
54
+
55
+ if json_data.empty?
56
+ report.add_event(
57
+ file: targets.first,
58
+ source: name,
59
+ state: :passed,
60
+ severity: :ok,
61
+ )
62
+ else
63
+ json_data.delete('result')
64
+ json_data.keys.each do |type|
65
+ json_data[type].each do |offense|
66
+ # metadata-json-lint groups the offenses by type, so the type ends
67
+ # up being `warnings` or `errors`. We want to convert that to the
68
+ # singular noun for the event.
69
+ event_type = type[%r{\A(.+?)s?\Z}, 1]
70
+
71
+ report.add_event(
72
+ file: targets.first,
73
+ source: name,
74
+ message: offense['msg'],
75
+ test: offense['check'],
76
+ severity: event_type,
77
+ state: :failure,
78
+ )
79
+ end
80
+ end
81
+ end
82
+ end
83
+ end
84
+ end
85
+ end
86
+ end
@@ -0,0 +1,78 @@
1
+ require 'pdk'
2
+
3
+ module PDK
4
+ module Validate
5
+ module Metadata
6
+ class MetadataSyntaxValidator < InternalRubyValidator
7
+ def name
8
+ 'metadata-syntax'
9
+ end
10
+
11
+ def pattern
12
+ contextual_pattern(['metadata.json', 'tasks/*.json'])
13
+ end
14
+
15
+ def spinner_text
16
+ _('Checking metadata syntax (%{patterns}).') % {
17
+ patterns: pattern.join(' '),
18
+ }
19
+ end
20
+
21
+ def invoke(report)
22
+ super
23
+ ensure
24
+ JSON.parser = JSON::Ext::Parser if defined?(JSON::Ext::Parser)
25
+ end
26
+
27
+ def before_validation
28
+ # The pure ruby JSON parser gives much nicer parse error messages than
29
+ # the C extension at the cost of slightly slower parsing. We require it
30
+ # here and restore the C extension at the end of the method (if it was
31
+ # being used).
32
+ require 'json/pure'
33
+ JSON.parser = JSON::Pure::Parser
34
+ end
35
+
36
+ def validate_target(report, target)
37
+ unless PDK::Util::Filesystem.readable?(target)
38
+ report.add_event(
39
+ file: target,
40
+ source: name,
41
+ state: :failure,
42
+ severity: 'error',
43
+ message: _('Could not be read.'),
44
+ )
45
+ return 1
46
+ end
47
+
48
+ begin
49
+ JSON.parse(PDK::Util::Filesystem.read_file(target))
50
+
51
+ report.add_event(
52
+ file: target,
53
+ source: name,
54
+ state: :passed,
55
+ severity: 'ok',
56
+ )
57
+ return 0
58
+ rescue JSON::ParserError => e
59
+ # Because the message contains a raw segment of the file, we use
60
+ # String#dump here to unescape any escape characters like newlines.
61
+ # We then strip out the surrounding quotes and the exclaimation
62
+ # point that json_pure likes to put in exception messages.
63
+ sane_message = e.message.dump[%r{\A"(.+?)!?"\Z}, 1]
64
+
65
+ report.add_event(
66
+ file: target,
67
+ source: name,
68
+ state: :failure,
69
+ severity: 'error',
70
+ message: sane_message,
71
+ )
72
+ return 1
73
+ end
74
+ end
75
+ end
76
+ end
77
+ end
78
+ end
@@ -0,0 +1,20 @@
1
+ require 'pdk'
2
+
3
+ module PDK
4
+ module Validate
5
+ module Metadata
6
+ class MetadataValidatorGroup < ValidatorGroup
7
+ def name
8
+ 'metadata'
9
+ end
10
+
11
+ def validators
12
+ [
13
+ MetadataSyntaxValidator,
14
+ MetadataJSONLintValidator,
15
+ ].freeze
16
+ end
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,133 @@
1
+ require 'pdk'
2
+
3
+ module PDK
4
+ module Validate
5
+ module Puppet
6
+ class PuppetEPPValidator < ExternalCommandValidator
7
+ # In Puppet >= 5.3.4, the error context formatting was changed to facilitate localization
8
+ ERROR_CONTEXT = %r{(?:file:\s(?<file>.+?)|line:\s(?<line>.+?)|column:\s(?<column>.+?))}
9
+ # In Puppet < 5.3.3, the error context was formatted in these variations:
10
+ # - "at file_path:line_num:col_num"
11
+ # - "at file_path:line_num"
12
+ # - "at line line_num"
13
+ # - "in file_path"
14
+ ERROR_CONTEXT_LEGACY = %r{(?:at\sline\s(?<line>\d+)|at\s(?<file>.+?):(?<line>\d+):(?<column>\d+)|at\s(?<file>.+?):(?<line>\d+)|in\s(?<file>.+?))}
15
+
16
+ PUPPET_LOGGER_PREFIX = %r{^(debug|info|notice|warning|error|alert|critical):\s.+?$}i
17
+ PUPPET_SYNTAX_PATTERN = %r{^
18
+ (?<severity>.+?):\s
19
+ (?<message>.+?)
20
+ (?:
21
+ \s\(#{ERROR_CONTEXT}(,\s#{ERROR_CONTEXT})*\)| # attempt to match the new localisation friendly location
22
+ \s#{ERROR_CONTEXT_LEGACY}| # attempt to match the old " at file:line:column" location
23
+ $ # handle cases where the output has no location
24
+ )
25
+ $}x
26
+
27
+ def name
28
+ 'puppet-epp'
29
+ end
30
+
31
+ def cmd
32
+ 'puppet'
33
+ end
34
+
35
+ def pattern
36
+ contextual_pattern('**/*.epp')
37
+ end
38
+
39
+ def spinner_text_for_targets(_targets)
40
+ _('Checking Puppet EPP syntax (%{pattern}).') % { pattern: pattern.join(' ') }
41
+ end
42
+
43
+ def parse_options(targets)
44
+ # Due to PDK-1266 we need to run `puppet parser validate` with an empty
45
+ # modulepath. On *nix, Ruby treats `/dev/null` as an empty directory
46
+ # however it doesn't do so with `NUL` on Windows. The workaround for
47
+ # this to ensure consistent behaviour is to create an empty temporary
48
+ # directory and use that as the modulepath.
49
+ ['epp', 'validate', '--config', null_file, '--modulepath', validate_tmpdir].concat(targets)
50
+ end
51
+
52
+ def invoke(report)
53
+ super
54
+ ensure
55
+ remove_validate_tmpdir
56
+ end
57
+
58
+ def validate_tmpdir
59
+ require 'tmpdir'
60
+
61
+ @validate_tmpdir ||= Dir.mktmpdir('puppet-epp-validate')
62
+ end
63
+
64
+ def remove_validate_tmpdir
65
+ return unless @validate_tmpdir
66
+ return unless PDK::Util::Filesystem.directory?(@validate_tmpdir)
67
+
68
+ PDK::Util::Filesystem.remove_entry_secure(@validate_tmpdir)
69
+ @validate_tmpdir = nil
70
+ end
71
+
72
+ def null_file
73
+ Gem.win_platform? ? 'NUL' : '/dev/null'
74
+ end
75
+
76
+ def parse_output(report, result, targets)
77
+ # Due to PUP-7504, we will have to programmatically construct the json
78
+ # object from the text output for now.
79
+ output = result[:stderr].split(%r{\r?\n}).reject { |entry| entry.empty? }
80
+
81
+ results_data = []
82
+ output.each do |offense|
83
+ offense_data = parse_offense(offense)
84
+ results_data << offense_data
85
+ end
86
+
87
+ # puppet parser validate does not include files without problems in its
88
+ # output, so we need to go through the list of targets and add passing
89
+ # events to the report for any target not listed in the output.
90
+ targets.reject { |target| results_data.any? { |j| j[:file] =~ %r{#{target}} } }.each do |target|
91
+ report.add_event(
92
+ file: target,
93
+ source: name,
94
+ severity: :ok,
95
+ state: :passed,
96
+ )
97
+ end
98
+
99
+ results_data.each do |offense|
100
+ report.add_event(offense)
101
+ end
102
+ end
103
+
104
+ def parse_offense(offense)
105
+ sanitize_console_output(offense)
106
+
107
+ offense_data = {
108
+ source: name,
109
+ state: :failure,
110
+ }
111
+
112
+ if offense.match(PUPPET_LOGGER_PREFIX)
113
+ attributes = offense.match(PUPPET_SYNTAX_PATTERN)
114
+
115
+ unless attributes.nil?
116
+ attributes.names.each do |name|
117
+ offense_data[name.to_sym] = attributes[name] unless attributes[name].nil?
118
+ end
119
+ end
120
+ else
121
+ offense_data[:message] = offense
122
+ end
123
+
124
+ offense_data
125
+ end
126
+
127
+ def sanitize_console_output(line)
128
+ line.gsub!(%r{\e\[([;\d]+)?m}, '')
129
+ end
130
+ end
131
+ end
132
+ end
133
+ end