ra10ke 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 22fd36a3a88f9a05d15fb0b4e964e3151e0eff8f
-  data.tar.gz: f9234354db77a8bf07d29e826c7570ffc6bc29d0
+  metadata.gz: 4483e49e41d8b69e98d4730662b79084a670580b
+  data.tar.gz: 0bf2f4d16f66755a60f3653852900e46983136f1
 SHA512:
-  metadata.gz: 8bba17ade9b5e494e75f604e8ec734c1c5489a127adc366554c2b35c69cade5a3e49d50b725fb036ac03e92e98db0651f18856e6eb2f4d0e1db797f1027636c4
-  data.tar.gz: 3f59f147b0d1709ae9e3c4aa5cfbd2a2e1770b2d789661b9f501a003b99e4ad7e778b667aeba4166b836ffbfe20547d98838519f2f51433426a4221d5b23c451
+  metadata.gz: de78be1d8e3146d55b207bd48f100f00cd36e310f857d3de3d2cb831dc894c63b3196fa1aa75dd8fda9964c46e1f6f0412507ebb22863a2ec8e19a37bf39fb2c
+  data.tar.gz: 36087c043bb91a4a4b034fb86fc5695c6d2ea74c4ed5ababcb3737b2477ee9714fe30afb8544741feb73a306deacc41633341a7acbf41a354203480776ca223e

data/CHANGELOG.md

@@ -1,6 +1,14 @@
 CHANGELOG
 =========
 
+0.3.0
+-----
+
+2017-10-08
+
+* [#6](https://github.com/voxpupuli/ra10ke/pull/6) Add a dependency solver based on the Ruby Solve library (thanks to [Jarkko Oranen](https://github.com/oranenj))
+* [#8](https://github.com/voxpupuli/ra10ke/pull/8) Add numeric tag convention support (thanks to [Hervé MARTIN](https://github.com/HerveMARTIN))
+
 0.2.0
 -----
 

data/README.md

@@ -40,6 +40,18 @@ Ignoring specific modules:
 Under specific conditions you may not wish to report on specific modules being out of date,
 to ignore a module create `.r10kignore` file in the same directory as your Puppetfile.
 
+### r10k:solve_dependencies
+
+Reads the Puppetfile in the current directory and uses the ruby 'solve' library to find
+missing and outdated dependencies based on their metadata.
+
+The solver does not allow major version bumps according to SemVer by default. To allow
+major upgrades, call the rake task with any parameter.
+
+The rake task will download git modules into the modules/ directory to access their metadata.json.
+It will also cache forge metadata in ̃$XDG_CACHE_DIR/ra10ke.metadata_cache in order to make subsequent
+runs faster.
+
 #### Limitations
 
   * It works only with modules from the [Forge](https://forge.puppetlabs.com), and Git.

data/lib/ra10ke/solve.rb

@@ -0,0 +1,227 @@
+require 'rake'
+require 'rake/tasklib'
+require 'ra10ke/version'
+require 'git'
+require 'set'
+require 'solve'
+require 'yaml/store'
+require 'semverse/version'
+
+# How many versions to fetch from the Forge, at most
+FETCH_LIMIT = 3
+
+module Ra10ke::Solve
+  class RakeTask < ::Rake::TaskLib
+    def initialize(*_args)
+      namespace :r10k do
+        desc 'Find missing or outdated module dependencies'
+        task :solve_dependencies, [:allow_major_bump] do |_t, args|
+          require 'r10k/puppetfile'
+          require 'r10k/module/git'
+          require 'r10k/module/metadata_file'
+          require 'puppet_forge'
+
+          allow_major_bump = false
+          allow_major_bump = true if args[:allow_major_bump]
+
+          # Same as in the dependencies task, but oh well.
+          PuppetForge.user_agent = "ra10ke/#{Ra10ke::VERSION}"
+          puppetfile = R10K::Puppetfile.new('.')
+          puppetfile.load!
+
+          # ignore file allows for "don't tell me about this"
+          ignore_modules = []
+          if File.exist?('.r10kignore')
+            ignore_modules = File.readlines('.r10kignore').each(&:chomp!)
+          end
+          # Actual new logic begins here:
+          cache = (ENV['XDG_CACHE_DIR'] || File.expand_path('~/.cache'))
+          # Metadata cache, since the Forge is slow:
+          @metadata_cache = YAML::Store.new File.join(cache, 'ra10ke.metadata_cache')
+          # The graph of available module versions
+          @graph = Solve::Graph.new
+          # Set of modules that we have already added to the graph
+          @processed_modules = Set.new
+          # The set of "demands" we make of the solver. Will be a list of module names
+          # Could also demand certain version constraints to hold, but the code does not do it
+          # Can be either "module-name" or ["module-name", "version-constraint"]
+          @demands = Set.new
+          # List of modules we have in the Puppetfile, as [name, version] pairs
+          @current_modules = []
+
+          puppetfile.modules.each do |puppet_module|
+            next if ignore_modules.include? puppet_module.title
+            if puppet_module.class == R10K::Module::Forge
+              module_name = puppet_module.title.tr('/', '-')
+              installed_version = puppet_module.expected_version
+              puts "Processing Forge module #{module_name}-#{installed_version}"
+              @current_modules << [module_name, installed_version]
+              @graph.artifact(module_name, installed_version)
+              constraint = '>=0.0.0'
+              unless allow_major_bump
+                ver = Semverse::Version.new installed_version
+                if ver.major.zero?
+                  constraint = "~>#{installed_version}"
+                else
+                  nver = Semverse::Version.new([ver.major + 1, 0, 0])
+                  constraint = "<#{nver}"
+                end
+              end
+              puts "...Adding a demand: #{module_name} #{constraint}"
+
+              @demands.add([module_name, constraint])
+              puts '...Fetching latest release version information'
+              forge_rel = PuppetForge::Module.find(module_name).current_release
+              mod = @graph.artifact(module_name, forge_rel.version)
+              puts '...Adding its requirements to the graph'
+              meta = get_release_metadata(module_name, forge_rel)
+              add_reqs_to_graph(mod, meta)
+            end
+
+            next unless puppet_module.class == R10K::Module::Git
+            # This downloads the git module to modules/modulename
+            meta = fetch_git_metadata(puppet_module)
+            version = get_key_or_sym(meta, :version)
+            module_name = puppet_module.title.tr('/', '-')
+            @current_modules << [module_name, version]
+            # We should add git modules with exact versions, or the system might recommend updating to a
+            # Forge version.
+            puts "Adding git module #{module_name} to the list of required modules with exact version: #{version}"
+            @demands.add([module_name, version])
+            mod = @graph.artifact(module_name, version)
+            puts "...Adding requirements for git module #{module_name}-#{version}"
+            add_reqs_to_graph(mod, meta)
+          end
+          puts
+          puts 'Resolving dependencies...'
+          if allow_major_bump
+            puts 'WARNING:  Potentially breaking updates are allowed for this resolution'
+          end
+          result = Solve.it!(@graph, @demands, sorted: true)
+          puts
+          print_module_diff(@current_modules, result)
+        end
+      end
+    end
+
+    def get_release_metadata(name, release)
+      meta = nil
+      @metadata_cache.transaction do
+        meta = @metadata_cache["#{name}-#{release.version}"]
+        unless meta
+          meta = release.metadata
+          @metadata_cache["#{name}-#{release.version}"] = meta
+        end
+      end
+      meta
+    end
+
+    def fetch_git_metadata(puppet_module)
+      # No caching here. I don't think it's really possible to do in a sane way.
+      puts "Fetching git module #{puppet_module.title}, saving to modules/"
+      puppet_module.sync
+      metadata_path = Pathname.new(puppet_module.full_path) + 'metadata.json'
+      unless metadata_path.exist?
+        puts 'WARNING: metadata.json does not exist, assuming version 0.0.0 and no dependencies'
+        return {
+          version: '0.0.0',
+          name: puppet_module.title,
+          dependencies: []
+        }
+      end
+      metadata = R10K::Module::MetadataFile.new(metadata_path)
+      metadata = metadata.read
+      {
+        version: metadata.version,
+        name: metadata.name,
+        dependencies: metadata.dependencies
+      }
+    end
+
+    # Is there a better way? :(
+    def get_key_or_sym(hash, k)
+      hash.fetch(k.to_sym, hash.fetch(k.to_s, nil))
+    end
+
+    # At least puppet-extlib has malformed metadata
+    def get_version_req(dep)
+      req = get_key_or_sym(dep, :version_requirement)
+      req = get_key_or_sym(dep, :version_range) unless req
+      req
+    end
+
+    def print_module_diff(current, resolution)
+      current.sort!
+      resolution.sort!
+      outdated = []
+      missing = []
+      resolution.each do |mod|
+        cur_mod, cur_version = current.shift
+        mod, version = mod
+        if (cur_mod == mod) && cur_version && (cur_version != version)
+          outdated << [mod, cur_version, version]
+        elsif cur_mod != mod
+          missing << [mod, version]
+          current.unshift [cur_mod, cur_version]
+        end
+      end
+      missing.each do |m|
+        puts format('MISSING:  %-25s %s', *m)
+      end
+      outdated.each do |o|
+        puts format('OUTDATED: %-25s %s -> %s', *o)
+      end
+    end
+
+    def add_reqs_to_graph(artifact, metadata, no_demands = nil)
+      deps = get_key_or_sym(metadata, :dependencies)
+      my_name = get_key_or_sym(metadata, :name)
+      deps.each do |dep|
+        name = get_key_or_sym(dep, :name).tr('/', '-')
+        # Add dependency to the global set of modules we want, so that we can
+        # actually ask the solver for the versioned thing
+        @demands.add(name) unless no_demands
+        ver = get_version_req(dep)
+        unless ver
+          # no version specified, so anything will do
+          ver = '>=0.0.0'
+        end
+        ver.split(/(?=[<])/).each do |bound|
+          bound.strip!
+          v = begin
+                Semverse::Constraint.new(bound)
+              rescue
+                nil
+              end
+          if v
+            artifact.depends(name, v.to_s)
+          else
+            puts "WARNING: Invalid version constraint: #{bound}"
+          end
+        end
+        # Find the dependency in the forge, unless it's already been processed
+        # and add its releases to the global graph
+        next unless @processed_modules.add?(name)
+        puts "Fetching module info for #{name}"
+        mod = begin
+                PuppetForge::Module.find(name)
+              rescue
+                # It's probably a git module
+                nil
+              end
+        next unless mod # Git module, or non-forge dependency. Skip to next for now.
+        # Fetching metadata for all releases takes ages (which is weird, since it's mostly static info)
+        mod.releases.take(FETCH_LIMIT).each do |rel|
+          meta = get_release_metadata(name, rel)
+          rel_artifact = @graph.artifact(name, rel.version)
+          puts "...Recursively adding requirements for dependency #{name} version #{rel.version}"
+          # We don't want to add the requirements to the list of demands for all versions,
+          # but we need them in the graph to be able to solve dependencies
+          add_reqs_to_graph(rel_artifact, meta, :no_demands)
+        end
+      end
+    end
+  end
+end
+
+Ra10ke::Solve::RakeTask.new

data/lib/ra10ke/version.rb

@@ -1,3 +1,3 @@
 module Ra10ke
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/lib/ra10ke.rb

@@ -1,6 +1,7 @@
 require 'rake'
 require 'rake/tasklib'
 require 'ra10ke/version'
+require 'ra10ke/solve'
 require 'git'
 
 module Ra10ke
@@ -52,12 +53,14 @@ module Ra10ke
               when 'sha'
                 latest_ref = remote_refs['head'][:sha]
               when 'tag'
-                # we have to be opinionated here, due to multiple conventions only one can reign suppreme
-                # as such tag v#.#.# is what will pick.
-                if ref.match(/^[vV]\d/)
+                # we have to be opinionated here, due to multiple conventions only the two main will be accepted
+                # v#.#.# or #.#.# is what we will pick.
+                if ref.match(/^[vV]?\d[\.\d]*/)
                   tags = remote_refs['tags']
-                  version_tags = tags.select { |f| /^[vV]\d/.match(f) }
+                  version_tags = tags.select { |f| /^[vV]?\d[\.\d]*/.match(f) }
                   latest_ref = version_tags.keys.sort.last
+                else
+                  latest_ref = "undef (tags don't match v#.#.# or #.#.#)"
                 end
               else
                 raise "Unable to determine ref_type for #{puppet_module.title}"

data/ra10ke.gemspec

@@ -19,4 +19,5 @@ Gem::Specification.new do |spec|
   spec.add_dependency "puppet_forge"
   spec.add_dependency "r10k"
   spec.add_dependency "git"
+  spec.add_dependency "solve"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ra10ke
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Theo Chatzimichos
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-01-03 00:00:00.000000000 Z
+date: 2017-10-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -66,6 +66,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: solve
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: R10K and Puppetfile rake tasks
 email:
 - tampakrap@gmail.com
@@ -79,6 +93,7 @@ files:
 - LICENSE.txt
 - README.md
 - lib/ra10ke.rb
+- lib/ra10ke/solve.rb
 - lib/ra10ke/version.rb
 - ra10ke.gemspec
 homepage: https://github.com/voxpupuli/ra10ke
@@ -101,7 +116,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: Syntax check for the Puppetfile, check for outdated installed puppet modules