ra10ke 0.5.0 → 0.6.0

Sign up to get free protection for your applications and to get access to all the features.
data/lib/ra10ke/solve.rb CHANGED
@@ -11,218 +11,214 @@ require 'semverse/version'
11
11
  FETCH_LIMIT = 3
12
12
 
13
13
  module Ra10ke::Solve
14
- class RakeTask < ::Rake::TaskLib
15
- def initialize(*_args)
16
- namespace :r10k do
17
- desc 'Find missing or outdated module dependencies'
18
- task :solve_dependencies, [:allow_major_bump] do |_t, args|
19
- require 'r10k/puppetfile'
20
- require 'r10k/module/git'
21
- require 'r10k/module/metadata_file'
22
- require 'puppet_forge'
14
+ def define_task_solve_dependencies(*_args)
15
+ desc 'Find missing or outdated module dependencies'
16
+ task :solve_dependencies, [:allow_major_bump] do |_t, args|
17
+ require 'r10k/puppetfile'
18
+ require 'r10k/module/git'
19
+ require 'r10k/module/metadata_file'
20
+ require 'puppet_forge'
23
21
 
24
- allow_major_bump = false
25
- allow_major_bump = true if args[:allow_major_bump]
22
+ allow_major_bump = false
23
+ allow_major_bump = true if args[:allow_major_bump]
26
24
 
27
- # Same as in the dependencies task, but oh well.
28
- PuppetForge.user_agent = "ra10ke/#{Ra10ke::VERSION}"
29
- puppetfile = R10K::Puppetfile.new(Dir.pwd)
30
- puppetfile.load!
31
- PuppetForge.host = puppetfile.forge if puppetfile.forge =~ /^http/
25
+ # Same as in the dependencies task, but oh well.
26
+ PuppetForge.user_agent = "ra10ke/#{Ra10ke::VERSION}"
27
+ puppetfile = get_puppetfile
28
+ puppetfile.load!
29
+ PuppetForge.host = puppetfile.forge if puppetfile.forge =~ /^http/
32
30
 
33
- # ignore file allows for "don't tell me about this"
34
- ignore_modules = []
35
- if File.exist?('.r10kignore')
36
- ignore_modules = File.readlines('.r10kignore').each(&:chomp!)
37
- end
38
- # Actual new logic begins here:
39
- cache = (ENV['XDG_CACHE_DIR'] || File.expand_path('~/.cache'))
40
- # Metadata cache, since the Forge is slow:
41
- @metadata_cache = YAML::Store.new File.join(cache, 'ra10ke.metadata_cache')
42
- # The graph of available module versions
43
- @graph = Solve::Graph.new
44
- # Set of modules that we have already added to the graph
45
- @processed_modules = Set.new
46
- # The set of "demands" we make of the solver. Will be a list of module names
47
- # Could also demand certain version constraints to hold, but the code does not do it
48
- # Can be either "module-name" or ["module-name", "version-constraint"]
49
- @demands = Set.new
50
- # List of modules we have in the Puppetfile, as [name, version] pairs
51
- @current_modules = []
52
-
53
- puppetfile.modules.each do |puppet_module|
54
- next if ignore_modules.include? puppet_module.title
55
- if puppet_module.class == R10K::Module::Forge
56
- module_name = puppet_module.title.tr('/', '-')
57
- installed_version = puppet_module.expected_version
58
- puts "Processing Forge module #{module_name}-#{installed_version}"
59
- @current_modules << [module_name, installed_version]
60
- @graph.artifact(module_name, installed_version)
61
- constraint = '>=0.0.0'
62
- unless allow_major_bump
63
- ver = Semverse::Version.new installed_version
64
- if ver.major.zero?
65
- constraint = "~>#{installed_version}"
66
- else
67
- nver = Semverse::Version.new([ver.major + 1, 0, 0])
68
- constraint = "<#{nver}"
69
- end
70
- end
71
- puts "...Adding a demand: #{module_name} #{constraint}"
31
+ # ignore file allows for "don't tell me about this"
32
+ ignore_modules = []
33
+ if File.exist?('.r10kignore')
34
+ ignore_modules = File.readlines('.r10kignore').each(&:chomp!)
35
+ end
36
+ # Actual new logic begins here:
37
+ cache = (ENV['XDG_CACHE_DIR'] || File.expand_path('~/.cache'))
38
+ # Metadata cache, since the Forge is slow:
39
+ @metadata_cache = YAML::Store.new File.join(cache, 'ra10ke.metadata_cache')
40
+ # The graph of available module versions
41
+ @graph = Solve::Graph.new
42
+ # Set of modules that we have already added to the graph
43
+ @processed_modules = Set.new
44
+ # The set of "demands" we make of the solver. Will be a list of module names
45
+ # Could also demand certain version constraints to hold, but the code does not do it
46
+ # Can be either "module-name" or ["module-name", "version-constraint"]
47
+ @demands = Set.new
48
+ # List of modules we have in the Puppetfile, as [name, version] pairs
49
+ @current_modules = []
72
50
 
73
- @demands.add([module_name, constraint])
74
- puts '...Fetching latest release version information'
75
- forge_rel = PuppetForge::Module.find(module_name).current_release
76
- mod = @graph.artifact(module_name, forge_rel.version)
77
- puts '...Adding its requirements to the graph'
78
- meta = get_release_metadata(module_name, forge_rel)
79
- add_reqs_to_graph(mod, meta)
51
+ puppetfile.modules.each do |puppet_module|
52
+ next if ignore_modules.include? puppet_module.title
53
+ if puppet_module.class == R10K::Module::Forge
54
+ module_name = puppet_module.title.tr('/', '-')
55
+ installed_version = puppet_module.expected_version
56
+ puts "Processing Forge module #{module_name}-#{installed_version}"
57
+ @current_modules << [module_name, installed_version]
58
+ @graph.artifact(module_name, installed_version)
59
+ constraint = '>=0.0.0'
60
+ unless allow_major_bump
61
+ ver = Semverse::Version.new installed_version
62
+ if ver.major.zero?
63
+ constraint = "~>#{installed_version}"
64
+ else
65
+ nver = Semverse::Version.new([ver.major + 1, 0, 0])
66
+ constraint = "<#{nver}"
80
67
  end
81
-
82
- next unless puppet_module.class == R10K::Module::Git
83
- # This downloads the git module to modules/modulename
84
- meta = fetch_git_metadata(puppet_module)
85
- version = get_key_or_sym(meta, :version)
86
- module_name = puppet_module.title.tr('/', '-')
87
- @current_modules << [module_name, version]
88
- # We should add git modules with exact versions, or the system might recommend updating to a
89
- # Forge version.
90
- puts "Adding git module #{module_name} to the list of required modules with exact version: #{version}"
91
- @demands.add([module_name, version])
92
- mod = @graph.artifact(module_name, version)
93
- puts "...Adding requirements for git module #{module_name}-#{version}"
94
- add_reqs_to_graph(mod, meta)
95
68
  end
96
- puts
97
- puts 'Resolving dependencies...'
98
- if allow_major_bump
99
- puts 'WARNING: Potentially breaking updates are allowed for this resolution'
100
- end
101
- result = Solve.it!(@graph, @demands, sorted: true)
102
- puts
103
- print_module_diff(@current_modules, result)
69
+ puts "...Adding a demand: #{module_name} #{constraint}"
70
+
71
+ @demands.add([module_name, constraint])
72
+ puts '...Fetching latest release version information'
73
+ forge_rel = PuppetForge::Module.find(module_name).current_release
74
+ mod = @graph.artifact(module_name, forge_rel.version)
75
+ puts '...Adding its requirements to the graph'
76
+ meta = get_release_metadata(module_name, forge_rel)
77
+ add_reqs_to_graph(mod, meta)
104
78
  end
79
+
80
+ next unless puppet_module.class == R10K::Module::Git
81
+ # This downloads the git module to modules/modulename
82
+ meta = fetch_git_metadata(puppet_module)
83
+ version = get_key_or_sym(meta, :version)
84
+ module_name = puppet_module.title.tr('/', '-')
85
+ @current_modules << [module_name, version]
86
+ # We should add git modules with exact versions, or the system might recommend updating to a
87
+ # Forge version.
88
+ puts "Adding git module #{module_name} to the list of required modules with exact version: #{version}"
89
+ @demands.add([module_name, version])
90
+ mod = @graph.artifact(module_name, version)
91
+ puts "...Adding requirements for git module #{module_name}-#{version}"
92
+ add_reqs_to_graph(mod, meta)
93
+ end
94
+ puts
95
+ puts 'Resolving dependencies...'
96
+ if allow_major_bump
97
+ puts 'WARNING: Potentially breaking updates are allowed for this resolution'
105
98
  end
99
+ result = Solve.it!(@graph, @demands, sorted: true)
100
+ puts
101
+ print_module_diff(@current_modules, result)
106
102
  end
103
+ end
107
104
 
108
- def get_release_metadata(name, release)
109
- meta = nil
110
- @metadata_cache.transaction do
111
- meta = @metadata_cache["#{name}-#{release.version}"]
112
- unless meta
113
- meta = release.metadata
114
- @metadata_cache["#{name}-#{release.version}"] = meta
115
- end
105
+ private
106
+
107
+ def get_release_metadata(name, release)
108
+ meta = nil
109
+ @metadata_cache.transaction do
110
+ meta = @metadata_cache["#{name}-#{release.version}"]
111
+ unless meta
112
+ meta = release.metadata
113
+ @metadata_cache["#{name}-#{release.version}"] = meta
116
114
  end
117
- meta
118
115
  end
116
+ meta
117
+ end
119
118
 
120
- def fetch_git_metadata(puppet_module)
121
- # No caching here. I don't think it's really possible to do in a sane way.
122
- puts "Fetching git module #{puppet_module.title}, saving to modules/"
123
- puppet_module.sync
124
- metadata_path = Pathname.new(puppet_module.full_path) + 'metadata.json'
125
- unless metadata_path.exist?
126
- puts 'WARNING: metadata.json does not exist, assuming version 0.0.0 and no dependencies'
127
- return {
128
- version: '0.0.0',
129
- name: puppet_module.title,
130
- dependencies: []
131
- }
132
- end
133
- metadata = R10K::Module::MetadataFile.new(metadata_path)
134
- metadata = metadata.read
135
- {
136
- version: metadata.version,
137
- name: metadata.name,
138
- dependencies: metadata.dependencies
119
+ def fetch_git_metadata(puppet_module)
120
+ # No caching here. I don't think it's really possible to do in a sane way.
121
+ puts "Fetching git module #{puppet_module.title}, saving to modules/"
122
+ puppet_module.sync
123
+ metadata_path = Pathname.new(puppet_module.full_path) + 'metadata.json'
124
+ unless metadata_path.exist?
125
+ puts 'WARNING: metadata.json does not exist, assuming version 0.0.0 and no dependencies'
126
+ return {
127
+ version: '0.0.0',
128
+ name: puppet_module.title,
129
+ dependencies: []
139
130
  }
140
131
  end
132
+ metadata = R10K::Module::MetadataFile.new(metadata_path)
133
+ metadata = metadata.read
134
+ {
135
+ version: metadata.version,
136
+ name: metadata.name,
137
+ dependencies: metadata.dependencies
138
+ }
139
+ end
141
140
 
142
- # Is there a better way? :(
143
- def get_key_or_sym(hash, k)
144
- hash.fetch(k.to_sym, hash.fetch(k.to_s, nil))
145
- end
141
+ # Is there a better way? :(
142
+ def get_key_or_sym(hash, k)
143
+ hash.fetch(k.to_sym, hash.fetch(k.to_s, nil))
144
+ end
146
145
 
147
- # At least puppet-extlib has malformed metadata
148
- def get_version_req(dep)
149
- req = get_key_or_sym(dep, :version_requirement)
150
- req = get_key_or_sym(dep, :version_range) unless req
151
- req
152
- end
146
+ # At least puppet-extlib has malformed metadata
147
+ def get_version_req(dep)
148
+ req = get_key_or_sym(dep, :version_requirement)
149
+ req = get_key_or_sym(dep, :version_range) unless req
150
+ req
151
+ end
153
152
 
154
- def print_module_diff(current, resolution)
155
- current.sort!
156
- resolution.sort!
157
- outdated = []
158
- missing = []
159
- resolution.each do |mod|
160
- cur_mod, cur_version = current.shift
161
- mod, version = mod
162
- if (cur_mod == mod) && cur_version && (cur_version != version)
163
- outdated << [mod, cur_version, version]
164
- elsif cur_mod != mod
165
- missing << [mod, version]
166
- current.unshift [cur_mod, cur_version]
167
- end
168
- end
169
- missing.each do |m|
170
- puts format('MISSING: %-25s %s', *m)
171
- end
172
- outdated.each do |o|
173
- puts format('OUTDATED: %-25s %s -> %s', *o)
153
+ def print_module_diff(current, resolution)
154
+ current.sort!
155
+ resolution.sort!
156
+ outdated = []
157
+ missing = []
158
+ resolution.each do |mod|
159
+ cur_mod, cur_version = current.shift
160
+ mod, version = mod
161
+ if (cur_mod == mod) && cur_version && (cur_version != version)
162
+ outdated << [mod, cur_version, version]
163
+ elsif cur_mod != mod
164
+ missing << [mod, version]
165
+ current.unshift [cur_mod, cur_version]
174
166
  end
175
167
  end
168
+ missing.each do |m|
169
+ puts format('MISSING: %-25s %s', *m)
170
+ end
171
+ outdated.each do |o|
172
+ puts format('OUTDATED: %-25s %s -> %s', *o)
173
+ end
174
+ end
176
175
 
177
- def add_reqs_to_graph(artifact, metadata, no_demands = nil)
178
- deps = get_key_or_sym(metadata, :dependencies)
179
- my_name = get_key_or_sym(metadata, :name)
180
- deps.each do |dep|
181
- name = get_key_or_sym(dep, :name).tr('/', '-')
182
- # Add dependency to the global set of modules we want, so that we can
183
- # actually ask the solver for the versioned thing
184
- @demands.add(name) unless no_demands
185
- ver = get_version_req(dep)
186
- unless ver
187
- # no version specified, so anything will do
188
- ver = '>=0.0.0'
189
- end
190
- ver.split(/(?=[<])/).each do |bound|
191
- bound.strip!
192
- v = begin
193
- Semverse::Constraint.new(bound)
194
- rescue
195
- nil
196
- end
197
- if v
198
- artifact.depends(name, v.to_s)
199
- else
200
- puts "WARNING: Invalid version constraint: #{bound}"
201
- end
202
- end
203
- # Find the dependency in the forge, unless it's already been processed
204
- # and add its releases to the global graph
205
- next unless @processed_modules.add?(name)
206
- puts "Fetching module info for #{name}"
207
- mod = begin
208
- PuppetForge::Module.find(name)
209
- rescue
210
- # It's probably a git module
211
- nil
212
- end
213
- next unless mod # Git module, or non-forge dependency. Skip to next for now.
214
- # Fetching metadata for all releases takes ages (which is weird, since it's mostly static info)
215
- mod.releases.take(FETCH_LIMIT).each do |rel|
216
- meta = get_release_metadata(name, rel)
217
- rel_artifact = @graph.artifact(name, rel.version)
218
- puts "...Recursively adding requirements for dependency #{name} version #{rel.version}"
219
- # We don't want to add the requirements to the list of demands for all versions,
220
- # but we need them in the graph to be able to solve dependencies
221
- add_reqs_to_graph(rel_artifact, meta, :no_demands)
176
+ def add_reqs_to_graph(artifact, metadata, no_demands = nil)
177
+ deps = get_key_or_sym(metadata, :dependencies)
178
+ my_name = get_key_or_sym(metadata, :name)
179
+ deps.each do |dep|
180
+ name = get_key_or_sym(dep, :name).tr('/', '-')
181
+ # Add dependency to the global set of modules we want, so that we can
182
+ # actually ask the solver for the versioned thing
183
+ @demands.add(name) unless no_demands
184
+ ver = get_version_req(dep)
185
+ unless ver
186
+ # no version specified, so anything will do
187
+ ver = '>=0.0.0'
188
+ end
189
+ ver.split(/(?=[<])/).each do |bound|
190
+ bound.strip!
191
+ v = begin
192
+ Semverse::Constraint.new(bound)
193
+ rescue
194
+ nil
195
+ end
196
+ if v
197
+ artifact.depends(name, v.to_s)
198
+ else
199
+ puts "WARNING: Invalid version constraint: #{bound}"
222
200
  end
223
201
  end
202
+ # Find the dependency in the forge, unless it's already been processed
203
+ # and add its releases to the global graph
204
+ next unless @processed_modules.add?(name)
205
+ puts "Fetching module info for #{name}"
206
+ mod = begin
207
+ PuppetForge::Module.find(name)
208
+ rescue
209
+ # It's probably a git module
210
+ nil
211
+ end
212
+ next unless mod # Git module, or non-forge dependency. Skip to next for now.
213
+ # Fetching metadata for all releases takes ages (which is weird, since it's mostly static info)
214
+ mod.releases.take(FETCH_LIMIT).each do |rel|
215
+ meta = get_release_metadata(name, rel)
216
+ rel_artifact = @graph.artifact(name, rel.version)
217
+ puts "...Recursively adding requirements for dependency #{name} version #{rel.version}"
218
+ # We don't want to add the requirements to the list of demands for all versions,
219
+ # but we need them in the graph to be able to solve dependencies
220
+ add_reqs_to_graph(rel_artifact, meta, :no_demands)
221
+ end
224
222
  end
225
223
  end
226
224
  end
227
-
228
- Ra10ke::Solve::RakeTask.new
@@ -0,0 +1,18 @@
1
+ module Ra10ke::Syntax
2
+ def define_task_syntax(*_args)
3
+ desc "Syntax check Puppetfile"
4
+ task :syntax do
5
+ require 'r10k/action/puppetfile/check'
6
+
7
+ puppetfile = R10K::Action::Puppetfile::Check.new({
8
+ :root => @basedir,
9
+ :moduledir => @moduledir,
10
+ :puppetfile => @puppetfile_path
11
+ }, '')
12
+
13
+ unless puppetfile.call
14
+ abort("Puppetfile syntax check failed")
15
+ end
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,126 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'ra10ke/monkey_patches'
4
+ require 'tempfile'
5
+ require 'table_print'
6
+ require 'ra10ke/puppetfile_parser'
7
+ require 'English'
8
+
9
+ module Ra10ke
10
+ module Validate
11
+
12
+ GOOD_EMOJI = ENV['GOOD_EMOJI'] || '👍'
13
+ BAD_EMOJI = ENV['BAD_EMOJI'] || '😨'
14
+
15
+ # Validate the git urls and refs
16
+ def define_task_validate(*args)
17
+ desc 'Validate the git urls and branches, refs, or tags'
18
+ task :validate do
19
+ gitvalididation = Ra10ke::Validate::Validation.new(get_puppetfile.puppetfile_path)
20
+ exit_code = 0
21
+ if gitvalididation.bad_mods?
22
+ exit_code = 1
23
+ message = BAD_EMOJI + ' Not all modules in the Puppetfile are valid. '.red + BAD_EMOJI
24
+ else
25
+ message = GOOD_EMOJI + ' Puppetfile looks good. '.green + GOOD_EMOJI
26
+ end
27
+ tp gitvalididation.sorted_mods, :name, { url: { width: 50 } }, :ref, :status
28
+ abort(message) if exit_code.positive?
29
+ puts message
30
+ end
31
+ end
32
+
33
+ class Validation
34
+ include Ra10ke::PuppetfileParser
35
+
36
+ attr_reader :puppetfile
37
+
38
+ def initialize(file)
39
+ file ||= './Puppetfile'
40
+ @puppetfile = File.expand_path(file)
41
+ abort("Puppetfile does not exist at #{puppetfile}") unless File.readable?(puppetfile)
42
+ end
43
+
44
+ # @return [Boolean] - return true if the ref is valid
45
+ # @param url [String] - the git string either https or ssh url
46
+ # @param ref [String] - the ref object, branch name, tag name, or commit sha, defaults to HEAD
47
+ def valid_ref?(url, ref = 'HEAD')
48
+ raise ArgumentError unless ref
49
+ found = all_refs(url).find do |sha, data |
50
+ # we don't need to bother with these types
51
+ next if data[:type] == :pull || data[:type] == :merge_request
52
+ # is the name equal to the tag or branch? Is the commit sha equal?
53
+ data[:name].eql?(ref) || sha.slice(0,8).eql?(ref.slice(0,8))
54
+ end
55
+ !found.nil?
56
+ end
57
+
58
+ # @return [Hash] - a hash of all the refs associated with the remote repository
59
+ # @param url [String] - the git string either https or ssh url
60
+ # @example
61
+ # {"0ec707e431367bbe2752966be8ab915b6f0da754"=>{:ref=>"refs/heads/74110ac", :type=>:branch, :subtype=>nil, :name=>"74110ac"},
62
+ # "07bb5d2d94db222dca5860eb29c184e8970f36f4"=>{:ref=>"refs/pull/74/head", :type=>:pull, :subtype=>:head, :name=>"74"},
63
+ # "156ca9a8ea69e056e86355b27d944e59d1b3a1e1"=>{:ref=>"refs/heads/master", :type=>:branch, :subtype=>nil, :name=>"master"},
64
+ # "fcc0532bbc5a5b65f3941738339e9cc7e3d767ce"=>{:ref=>"refs/pull/249/head", :type=>:pull, :subtype=>:head, :name=>"249"},
65
+ # "8d54891fa5df75890ee15d53080c2a81b4960f92"=>{:ref=>"refs/pull/267/head", :type=>:pull, :subtype=>:head, :name=>"267"} }
66
+ def all_refs(url)
67
+ data = `git ls-remote --symref #{url}`
68
+ raise "Error downloading #{url}" unless $CHILD_STATUS.success?
69
+ data.lines.reduce({}) do |refs, line|
70
+ sha, ref = line.split("\t")
71
+ next refs if sha.eql?('ref: refs/heads/master')
72
+ _, type, name, subtype = ref.chomp.split('/')
73
+ next refs unless name
74
+ type = :tag if type.eql?('tags')
75
+ type = type.to_sym
76
+ subtype = subtype.to_sym if subtype
77
+ type = :branch if type.eql?(:heads)
78
+ refs[sha] = {ref: ref.chomp, type: type, subtype: subtype, name: name }
79
+ refs
80
+ end
81
+ end
82
+
83
+ # @return [Boolean] - return true if the commit sha is valid
84
+ # @param url [String] - the git string either https or ssh url
85
+ # @param ref [String] - the sha id
86
+ def valid_commit?(url, sha)
87
+ return false if sha.nil? || sha.empty?
88
+ return true if valid_ref?(url, sha)
89
+ Dir.mktmpdir do |dir|
90
+ `git clone --no-tags #{url} #{dir} 2>&1 > /dev/null`
91
+ Dir.chdir(dir) do
92
+ `git show #{sha} 2>&1 > /dev/null`
93
+ $CHILD_STATUS.success?
94
+ end
95
+ end
96
+ end
97
+
98
+ # @return [Array[Hash]] array of module information and git status
99
+ def all_modules
100
+ begin
101
+ git_modules(puppetfile).map do |mod|
102
+ ref = mod[:args][:ref] || mod[:args][:tag] || mod[:args][:branch]
103
+ valid_ref = valid_ref?(mod[:args][:git], ref) || valid_commit?(mod[:args][:git], mod[:args][:ref])
104
+ {
105
+ name: mod[:name],
106
+ url: mod[:args][:git],
107
+ ref: ref,
108
+ valid_ref?: valid_ref,
109
+ status: valid_ref ? Ra10ke::Validate::GOOD_EMOJI : Ra10ke::Validate::BAD_EMOJI
110
+ }
111
+ end
112
+ end
113
+ end
114
+
115
+ # @return [Boolean] - true if there are any bad mods
116
+ def bad_mods?
117
+ all_modules.find_all { |mod| !mod[:valid_ref?] }.count > 0
118
+ end
119
+
120
+ # @return [Hash] - sorts the mods based on good/bad
121
+ def sorted_mods
122
+ all_modules.sort_by { |a| a[:valid_ref?] ? 1 : 0 }
123
+ end
124
+ end
125
+ end
126
+ end
@@ -1,3 +1,3 @@
1
1
  module Ra10ke
2
- VERSION = "0.5.0"
2
+ VERSION = "0.6.0"
3
3
  end