generate-puppetfile 0.11.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4796755a11eed130c7ffdf5587687d38ec0d903f
-  data.tar.gz: fb597256b5e7283a42354cdf13fe9a8b572d6725
+  metadata.gz: 20ff01c403b03aacd1a849c841fef1723543d739
+  data.tar.gz: 2571ee7127adad7638a9e0cf2f1f773cc7d1bf96
 SHA512:
-  metadata.gz: 09b2664d006c404941337f80b7a35c3591804ef05b5a594dd446e8fc290319a0162c2075dd49bd6d3ad2de48ace8229c6038020185ceb73f31b64257d398e1cc
-  data.tar.gz: b8ba776dd3e63f6feb081110eb28eb3aae59433388c2bafd03d4e91fa57d53ec5ca9705ec9e487f3fed5664aeceaa3c1ad3f79197af180f68b82a377c49cba21
+  metadata.gz: a3d7878cea4c6ccb4ede987cd424cdeddae79c8f79f80ca835af479a0d28fbc944406b8e7d327bf24b42c1ad419f57f81e9eb0f3f55c7706b30cf56c6ff14e5c
+  data.tar.gz: 14eff83bc5121fefe0736e19e28fe6a5c7bd4edc8a0b85c66366ec5781e806c5ea186663267aeb2a7c914713f672aafeef537238849c9330e10a9ffc5d7366c4

data/README.md

@@ -21,7 +21,7 @@ Installing modules. This may take a few minutes.
 Your Puppetfile has been generated. Copy and paste between the markers:
 
 =======================================================================
-forge 'http://forge.puppetlabs.com'
+forge 'https://forge.puppet.com'
 
 # Modules discovered by generate-puppetfile
 mod 'echocat/nfs', '1.7.1'
@@ -48,7 +48,7 @@ Installing modules. This may take a few minutes.
 Your Puppetfile has been generated. Copy and paste between the markers:
 
 =======================================================================
-forge 'http://forge.puppetlabs.com'
+forge 'https://forge.puppet.com'
 
 # Modules discovered by generate-puppetfile
 mod 'ajjahn/dhcp', '0.2.0'
@@ -101,19 +101,10 @@ mod 'lab_config',
 =======================================================================
 ```
 
-## Changes
-Documentation of important pre-1.0.0 changes
-
-* *0.10.0*: `--fixtures` has been replaced with `--create-fixtures`. `-f` is still an alias for this argument.
-
 ## Limitations
 
-* Parsing of an existing Puppetfile is naive. Anything that doesn't look like a forge module is preserved and then added to the end of the new Puppetfile. Verify that no ordering errors are introduced before using the new file.
-
-## Thanks
-Many thanks to the following people for contributing to generate-puppetfile
-* [Tomy Lobo](https://github.com/TomyLobo)
-* [Ben Ford](https://github.com/binford2k)
+* There is no defined standard for a `Puppetfile`'s contents. `generate-puppetfile` knows about some of the most common types of module references and attempts to treat them as references instead of strings. Any unknown references will be appended as raw content following the known references.
+* Verify that no ordering errors are introduced before using the new file.
 
 ## Contributing
 

data/lib/generate_puppetfile/bin.rb

@@ -6,15 +6,21 @@ require 'tempfile'
 require 'json'
 require 'mkmf'
 require 'colorize'
+require 'uri'
 
 module GeneratePuppetfile
   # Internal: The Bin class contains the logic for calling generate_puppetfile at the command line
   class Bin
-    Module_Regex = Regexp.new("mod ['\"]([a-z0-9_]+\/[a-z0-9_]+)['\"](,\\s+['\"](\\d+\.\\d+\.\\d+)['\"])?", Regexp::IGNORECASE)
-    @options = {}         # Options hash
-    @workspace = nil      # Working directory for module download and inspection
-    @module_data = {}     # key: modulename, value: version number
-    @download_errors = '' # A list of errors encountered while downloading modules. Should remain empty.
+    Module_Regex        = %r{^\s*mod ['\"]([a-z0-9_]+\/[a-z0-9_]+)['\"](,\s+['\"](\d+\.\d+\.\d+)['\"])?\s*$}
+    Repository_Regex    = %r{^\s*mod\s+['\"](\w+)['\"]\s*,\s*$}
+    Location_Only_Regex = %r{^\s+:git\s+=>\s+['\"](\S+)['\"]\s*$}
+    Location_Plus_Regex = %r{^\s+:git\s+=>\s+['\"](\S+)['\"]\s*,\s*$}
+    Type_ID_Regex       = %r{^\s+:(\w+)\s+=>\s+['\"](\S+)['\"]\s*$}
+    Forge_Regex         = %r{^forge}
+    Blanks_Regex        = %r{^\s*$}
+    Comments_Regex      = %r{^\s*#}
+    Skipall_Regex       = %r{^forge|^\s*$|^#}
+
     Silence = ('>' + File::NULL.to_str + ' 2>&1 ').freeze
     Puppetfile_Header = '# Modules discovered by generate-puppetfile'.freeze
     Extras_Note = '# Discovered elements from existing Puppetfile'.freeze
@@ -28,6 +34,11 @@ module GeneratePuppetfile
     #   GeneratePuppetfile::Bin.new(ARGV).run
     def initialize(args)
       @args = args
+      @options = {}         # Options hash
+      @workspace = nil      # Working directory for module download and inspection
+      @module_data = {}     # key: modulename, value: version number
+      @repository_data = [] # Non-forge modules. Array of hashes containing name, location, type, and ID
+      @download_errors = '' # A list of errors encountered while downloading modules. Should remain empty.
     end
 
     # Public: Run generate-puppetfile at the command line.
@@ -60,8 +71,9 @@ module GeneratePuppetfile
       forge_module_list = []
 
       # When using --fixtures-only, simply parse the provided Puppetfile and get out
-      if @options[:fixtures_only]
+      if @options[:fixtures_only] && @options[:puppetfile]
         @module_data = generate_module_data_from_Puppetfile
+        @repository_data = generate_repository_data_from_Puppetfile
         fixtures_data = generate_fixtures_data
         write_fixtures_data(fixtures_data)
         return 0
@@ -81,6 +93,7 @@ module GeneratePuppetfile
       end
 
       puppetfile_contents = {}
+      # Currently, ALL statements not including a forge module are listed as extras. The @repository_data should be removed from the extras eventually (#54)
       extras = []
       if @options[:puppetfile]
         puts "\nProcessing the puppetfile '#{@options[:puppetfile]}'...\n\n" if @options[:debug]
@@ -110,6 +123,7 @@ module GeneratePuppetfile
         display_puppetfile(puppetfile_contents) unless @options[:silent]
 
         if @options[:create_fixtures]
+          @repository_data = generate_repository_data_from_Puppetfile if @options[:puppetfile]
           fixtures_data = generate_fixtures_data
           write_fixtures_data(fixtures_data)
         end
@@ -186,8 +200,8 @@ Your Puppetfile has been generated. Copy and paste between the markers:
           print "    #{name} looks like a forge module.\n" if @options[:debug]
           puppetfile_contents[:modules].push(name)
         else
-          next if line =~ /^forge/
-          next if line =~ /^\s+$/
+          next if line =~ Forge_Regex
+          next if line =~ Blanks_Regex
           next if line =~ /#{Puppetfile_Header}/
           next if line =~ /#{Extras_Note}/
 
@@ -212,8 +226,8 @@ Your Puppetfile has been generated. Copy and paste between the markers:
           print "    #{name} looks like a forge module.\n" if @options[:debug]
           puppetfile_contents[:modules].push([name, version])
         else
-          next if line =~ /^forge/
-          next if line =~ /^\s+$/
+          next if line =~ Forge_Regex
+          next if line =~ Blanks_Regex
           next if line =~ /#{Puppetfile_Header}/
           next if line =~ /#{Extras_Note}/
 
@@ -292,11 +306,82 @@ Your Puppetfile has been generated. Copy and paste between the markers:
       modules
     end
 
+    # Public: generate the ad hoc repository (non-forge module) data from an existing Puppetfile
+    #   Returns an array of hashes with keys name, location, type, id
+    def generate_repository_data_from_Puppetfile
+      repositories = []
+
+      # Open the Puppetfile
+      File.open(@options[:puppetfile], 'r') do |fh|
+        while (line = fh.gets) != nil
+          # Skip blank lines, comments, anything that looks like a forge module
+          next if line =~ Skipall_Regex
+          next if Module_Regex.match(line)
+          # When we see /mod 'modulename',/ it is possibly a properly formatted fixture
+          if Repository_Regex.match(line)
+            complete = false
+            name = Regexp.last_match(1)
+            while (line = fh.gets) != nil
+              next if line =~ Skipall_Regex
+              if Location_Only_Regex.match(line)
+                # The Puppetfile may specify just a location /:git => 'https://github.com/author/puppet-modulename'/
+                # We do not validate the URI protocol, just that it is a valid URI
+                location = Regexp.last_match(1)
+                puts "Found module #{name} with location #{location}" if @options[:debug]
+                unless location.match(URI.regexp)
+                  puts "#{location} is not a valid URI, skipping this repo" if @options[:debug]
+                  break
+                end
+                repositories << {name: name, location: location}
+                complete = true
+              elsif Location_Plus_Regex.match(line)
+                # Or it may provide more, with a trailing comma
+                #   :git => 'https://github.com/author/puppet-modulename',
+                #   :ref => '1.0.0'
+                location = Regexp.last_match(1)
+                while (line = fh.gets) != nil
+                  next if line =~ Skipall_Regex
+                  if Type_ID_Regex.match(line)
+                    type = Regexp.last_match(1)
+                    id   = Regexp.last_match(2)
+                    puts "Found module #{name} with location #{location}, #{type} of #{id}" if @options[:debug]
+                    unless location.match(URI.regexp)
+                      puts "#{location} is not a valid URI, skipping this repo" if @options[:debug]
+                      break
+                    end
+                    repositories << {name: name, location: location, type: type, id: id}
+                    complete = true
+                  else
+                    # If the :git line ends with a comma but no type/ID is found, ignore it, we cannot properly determine the fixture
+                    puts "Found module #{name} at location #{location}. Expected type/ID information but did not find any, skipping." if @options[:debug]
+                    complete = true
+                  end
+                  break if complete
+                end
+              else
+                # If the /mod 'modulename',/ line is not followed with a :git string, ignore it, we cannot properly determine the fixture
+                puts "Found a reference to module #{name} but no location (:git) was provided, skipping." if @options[:debug]
+                complete = true
+              end
+              break if complete
+            end
+          end
+        end
+      end
+
+      repositories
+    end
+
+
     # Public: generate the list of modules in Puppetfile format from the @workspace
     def generate_forge_module_output
+      return '' if @module_data.empty?
+
+      max_length    = @module_data.keys.max_by {|mod| mod.length}.length + 3 # room for extra chars
       module_output = ''
-      @module_data.keys.each do |modulename|
-        module_output += "mod '#{modulename}', '#{@module_data[modulename]}'\n"
+
+      @module_data.each do |modulename, version|
+        module_output += sprintf("mod %-#{max_length}s '%s'\n", "'#{modulename}',", version)
       end
       module_output
     end
@@ -307,7 +392,7 @@ Your Puppetfile has been generated. Copy and paste between the markers:
     # extras is an array of strings
     def generate_puppetfile_contents(extras)
       puppetfile_contents = <<-EOF
-forge 'http://forge.puppetlabs.com'
+forge 'https://forge.puppet.com'
 
 #{Puppetfile_Header}
       EOF
@@ -376,18 +461,42 @@ forge 'http://forge.puppetlabs.com'
         end
       end
 
-      fixtures_data += "  forge_modules:\n" if @module_data != {}
-      @module_data.keys.each do |modulename|
-        shortname = modulename.split('/')[1]
-        version = @module_data[modulename] 
-        data = <<-EOF
+      unless @repository_data.empty?
+        fixtures_data += "  repositories:\n"
+        @repository_data.each do |repodata|
+          # Each repository has two or  pieces of data
+          #   Mandatory: the module name, the URI/location
+          #   Optional: the type (ref, branch, commit, etc.) and ID (tag, branch name, commit hash, etc.)
+          name      = repodata[:name]
+          location  = repodata[:location]
+          type      = repodata[:type]
+          id        = repodata[:id]
+
+          data = <<-EOF
+    #{name}:
+      repo: "#{location}"
+          EOF
+          data += "      #{type}: \"#{id}\"\n" if (type && id)
+
+          fixtures_data += data
+        end
+      end
+
+
+      unless @module_data.empty?
+        fixtures_data += "  forge_modules:\n"
+        @module_data.keys.each do |modulename|
+          shortname = modulename.split('/')[1]
+          version = @module_data[modulename] 
+          data = <<-EOF
     #{shortname}:
       repo: "#{modulename}"
       ref: "#{version}"
-        EOF
-        data.gsub!(/^ *ref.*$\n/, '') unless version != nil
+          EOF
+          data.gsub!(/^ *ref.*$\n/, '') unless version != nil
 
-        fixtures_data += data
+          fixtures_data += data
+        end
       end
 
       fixtures_data

data/lib/generate_puppetfile/version.rb

@@ -1,3 +1,3 @@
 module GeneratePuppetfile
-  VERSION = '0.11.0'.freeze
+  VERSION = '1.0.0'.freeze
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: generate-puppetfile
 version: !ruby/object:Gem::Version
-  version: 0.11.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Rob Nelson
@@ -9,132 +9,118 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-03-22 00:00:00.000000000 Z
+date: 2018-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: colorize
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3'
 - !ruby/object:Gem::Dependency
   name: rspec-its
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1'
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1'
 - !ruby/object:Gem::Dependency
   name: puppet
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: rubocop
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 0.39.0
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 0.39.0
 - !ruby/object:Gem::Dependency
   name: github_changelog_generator
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: multi_json
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: Generate a Puppetfile for use with r10k based on an existing file or
@@ -145,8 +131,8 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- bin/generate-puppetfile
 - README.md
+- bin/generate-puppetfile
 - lib/generate_puppetfile.rb
 - lib/generate_puppetfile/bin.rb
 - lib/generate_puppetfile/optparser.rb
@@ -161,17 +147,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: 2.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14
+rubygems_version: 2.6.10
 signing_key: 
 specification_version: 4
 summary: Generate a Puppetfile