puppet-library 0.13.0 → 0.14.0

checksums.yaml

@@ -1,7 +1,7 @@
---- 
-SHA1: 
-  data.tar.gz: 52a2fcf7cb4121ce34d56a978c220b0608654351
-  metadata.gz: afdccfcce7d2c23cae0cb1c429f687c3cf6b0ba8
-SHA512: 
-  data.tar.gz: dacbade142357d35f0cd154a8a401a52c855fc0df5f2c0f5ed8d450b89e026475e3f5bfdf6e0a074e820d1c8fa702d8e586c84906b0b0a05f7096a71acdd0a34
-  metadata.gz: e268f15fe38c6c8c8d41877dd70cf3d5b38092cffd6adc0f5b2592bfcbdd82b4eb46dcc034d4c8955f273a18ca41747d73637aa36f752625247bb14f334e4a49
+---
+SHA1:
+  metadata.gz: ce0573aac08832b970f50871291d13f4487e8495
+  data.tar.gz: d5f1fd926455ca4fbb87c7570797b002b0cf4bca
+SHA512:
+  metadata.gz: 123f603ff5ec60ec88f41bb7120e338e83e349e4d566dd11a1be46be406ae5f170577203d7c21a46eb7ed426da02ef25f71304d843cfb3589d84465a6d038961
+  data.tar.gz: 948cc9e9a048709d05ec1d53a9b2c0d90225d9b0026383fa48afc5f3ee442f1ebc08bcf62deaee591e7c50ca5e6218e0295007da0e57e140cce67c377590ba48

data/.gitignore

@@ -4,7 +4,6 @@
 .bundle
 .config
 .yardoc
-Gemfile.lock
 InstalledFiles
 _yardoc
 coverage
@@ -17,3 +16,4 @@ test/tmp
 test/version_tmp
 tmp
 .rake_t_cache
+/Gemfile.lock

data/CHANGELOG.yml

@@ -91,3 +91,9 @@
 - tag: v0.13.0
   changes:
       - Fixed "Invalid gemspec" error (#5)
+- tag: v0.14.0
+  changes:
+      - Use `metadata.json` in place of `Modulefile` if it exists (and `Modulefile` doesn't exist)
+      - Ignore corrupt module files on disk
+      - Added logging for when module files can't be read from disk
+      - Recurse into module directories when serving modules from disk

data/README.md

@@ -2,12 +2,12 @@
 
 A private Puppet Forge. Compatible with [librarian-puppet](http://librarian-puppet.com).
 
-[![Build Status](https://travis-ci.org/drrb/puppet-library.png?branch=master)](https://travis-ci.org/drrb/puppet-library)
-[![Coverage Status](https://coveralls.io/repos/drrb/puppet-library/badge.png)](https://coveralls.io/r/drrb/puppet-library)
-[![Code Climate](https://codeclimate.com/github/drrb/puppet-library.png)](https://codeclimate.com/github/drrb/puppet-library)
+[![Build Status](https://api.travis-ci.org/drrb/puppet-library.svg)](https://travis-ci.org/drrb/puppet-library)
+[![Coverage Status](https://img.shields.io/coveralls/drrb/puppet-library.svg)](https://coveralls.io/r/drrb/puppet-library)
+[![Code Climate](https://img.shields.io/codeclimate/github/drrb/puppet-library.svg)](https://codeclimate.com/github/drrb/puppet-library)
 
-[![Gem Version](https://badge.fury.io/rb/puppet-library.png)](http://badge.fury.io/rb/puppet-library)
-[![Dependency Status](https://gemnasium.com/drrb/puppet-library.png)](https://gemnasium.com/drrb/puppet-library)
+[![Gem Version](https://badge.fury.io/rb/puppet-library.svg)](http://badge.fury.io/rb/puppet-library)
+[![Dependency Status](https://gemnasium.com/drrb/puppet-library.svg)](https://gemnasium.com/drrb/puppet-library)
 
 Puppet Library serves Puppet modules in the same format as [the Puppet Forge](http://forge.puppetlabs.com). This allows you to create a private Puppet Forge and manage all the modules you use completely within your infrastructure.
 
@@ -18,6 +18,10 @@ Plugins can be created to serve modules from arbitrary sources. Puppet Library c
  - serving modules from source on disk
  - serving modules from Git repositories using tags as version numbers!
 
+## Demo
+
+Puppet Library has a web UI to browse the modules. A demo is available [here](http://puppet-library.herokuapp.com).
+
 ## Installation
 
 Install the server as a Gem:
@@ -96,18 +100,30 @@ Puppet Library contains built-in support for:
 
 ## Compatibility with other tools
 
+### Dependency Resolution/Installation
+
 Puppet Library currently supports:
 - search with Puppet (`puppet module search apache`)
 - dependency resolution and installation with Puppet (`puppet module install puppetlabs/apache`)
 - dependency resolution and installation with [librarian-puppet](http://librarian-puppet.com)
 - installation with [r10k](https://github.com/adrienthebo/r10k)
 
+### Ruby
+
 Puppet Library is tested against Ruby versions:
 - 1.8.7
 - 1.9.3
 - 2.0.0
 - 2.1.0
 
+### Puppet
+
+Puppet Library implements version 1 of the Forge API, and supports reading
+module metadata from modules' Modulefiles. This means that Puppet Library
+can currently only work with Puppet < 3.6, which uses the new Forge API (v3)
+and has switched to using `metadata.json` instead of `Modulefile`. Progress
+can be tracked [here](https://github.com/drrb/puppet-library/issues?milestone=1).
+
 ## Config file (EXPERIMENTAL)
 
 Instead of specifying command-line options, you can configure Puppet Library with a configuration file.

data/TODO.yml

@@ -1,4 +1,5 @@
 features:
+  - Support Forge API v3
   - Git repo forge:
     - make it more flexible:
       - fail gracefully if we can't contact the remote repo
@@ -18,9 +19,6 @@ features:
   - Config APIs:
     - documentation
 
-bugs:
-  - Using directory forge and Puppet Forge proxy doesn't list Puppet Forge modules on the index page unless you search
-
 dubious_features:
   - Proxy modules' source in a directory (supported individually for now: should we just leave it that way?)
   - Make proxy cache TTL configurable

data/config.ru

@@ -19,10 +19,10 @@ require 'rubygems'
 require 'puppet_library'
 
 server = PuppetLibrary::Server.configure do
-    # My custom modules
-    forge :directory do
-        path "/var/puppet/modules"
-    end
+    ## My custom modules
+    #forge :directory do
+    #    path "/var/puppet/modules"
+    #end
 
     # Unreleased versions from Github
     forge :git_repository do

data/lib/puppet_library/app/views/index.haml

@@ -22,7 +22,8 @@
 
 :javascript
   $(document).ready(function() {
-    $.getJSON("modules.json?q=#{query}", function(modules) {
+    var source = "modules.json#{ query.nil? ? '' : '?q=' + query }";
+    $.getJSON(source, function(modules) {
       $.each(modules, function(index, module) {
         var item = $("<li/>");
         item.append($("<b/>").append($("<a/>").attr("href", module.full_name).text(module.full_name)));

data/lib/puppet_library/archive/archive_reader.rb

@@ -26,7 +26,7 @@ module PuppetLibrary::Archive
         def read_entry(entry_name_regex)
             tar = Gem::Package::TarReader.new(Zlib::GzipReader.open(@path))
             tar.rewind
-            entry = tar.find {|e| e.full_name =~ entry_name_regex } or raise "Couldn't find entry in archive"
+            entry = tar.find {|e| e.full_name =~ entry_name_regex } or raise "Couldn't find entry in archive matching #{entry_name_regex.inspect}"
             entry.read
         end
     end

data/lib/puppet_library/forge/directory.rb

@@ -65,16 +65,23 @@ module PuppetLibrary::Forge
             end
         end
 
+        def get_all_metadata
+            get_metadata("*", "*")
+        end
+
         def get_metadata(author, module_name)
-            Dir["#{@module_dir.path}/#{author}-#{module_name}*"].map do |module_path|
-                archive = PuppetLibrary::Archive::ArchiveReader.new(module_path)
-                metadata_file = archive.read_entry %r[[^/]+/metadata\.json$]
-                JSON.parse(metadata_file)
-            end
+            archives = Dir["#{@module_dir.path}/**/#{author}-#{module_name}-*.tar.gz"]
+            archives.map {|path| read_metadata(path) }.compact
         end
 
-        def get_all_metadata
-            get_metadata("*", "")
+        private
+        def read_metadata(archive_path)
+            archive = PuppetLibrary::Archive::ArchiveReader.new(archive_path)
+            metadata_file = archive.read_entry %r[[^/]+/metadata\.json$]
+            JSON.parse(metadata_file)
+        rescue => error
+            warn "Error reading from module archive #{archive_path}: #{error}"
+            return nil
         end
     end
 end

data/lib/puppet_library/forge/forge.rb

@@ -14,8 +14,12 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+require 'puppet_library/util/logging'
+
 module PuppetLibrary::Forge
     class Forge
+        include PuppetLibrary::Util::Logging
+
         def prime
         end
 

data/lib/puppet_library/forge/git_repository.rb

@@ -60,8 +60,8 @@ module PuppetLibrary::Forge
             super(self)
             @version_tag_regex = version_tag_regex
             @git = git
-            @modulefile_cache = PuppetLibrary::Http::Cache::InMemory.new
-            @tags_cache = PuppetLibrary::Http::Cache::InMemory.new
+            @metadata_cache = PuppetLibrary::Http::Cache::InMemory.new(60)
+            @tags_cache = PuppetLibrary::Http::Cache::InMemory.new(60)
         end
 
         def prime
@@ -75,11 +75,11 @@ module PuppetLibrary::Forge
         def get_module(author, name, version)
             return nil unless tags.include? tag_for(version)
 
-            metadata = modulefile_for(version).to_metadata
+            metadata = metadata_for(version)
             return nil unless metadata["name"] == "#{author}-#{name}"
 
-            on_tag_for(version) do
-                PuppetLibrary::Archive::Archiver.archive_dir('.', "#{metadata["name"]}-#{version}") do |archive|
+            with_tag_for(version) do |tag_path|
+                PuppetLibrary::Archive::Archiver.archive_dir(tag_path, "#{metadata["name"]}-#{version}") do |archive|
                     archive.add_file("metadata.json", 0644) do |entry|
                         entry.write metadata.to_json
                     end
@@ -89,7 +89,7 @@ module PuppetLibrary::Forge
 
         def get_all_metadata
             tags.map do |tag|
-                modulefile_for_tag(tag).to_metadata
+                metadata_for_tag(tag)
             end
         end
 
@@ -106,23 +106,33 @@ module PuppetLibrary::Forge
             @tags_cache.get do
                 tags = @git.tags
                 tags = tags.select {|tag| tag =~ @version_tag_regex }
-                tags = tags.select {|tag| @git.file_exists?("Modulefile", tag) }
+                tags = tags.select do |tag|
+                    @git.file_exists?("metadata.json", tag) || @git.file_exists?("Modulefile", tag)
+                end
             end
         end
 
-        def modulefile_for_tag(tag)
-            @modulefile_cache.get(tag) do
-                modulefile_source = @git.read_file("Modulefile", tag)
-                PuppetLibrary::PuppetModule::Modulefile.parse(modulefile_source)
+        def metadata_for_tag(tag)
+            @metadata_cache.get(tag) do
+                metadata_file_exists = @git.file_exists?("metadata.json", tag)
+                modulefile_exists = @git.file_exists?("Modulefile", tag)
+                if metadata_file_exists && !modulefile_exists
+                    metadata_file = @git.read_file("metadata.json", tag)
+                    JSON.parse(metadata_file)
+                else
+                    modulefile_source = @git.read_file("Modulefile", tag)
+                    modulefile = PuppetLibrary::PuppetModule::Modulefile.parse(modulefile_source)
+                    modulefile.to_metadata
+                end
             end
         end
 
-        def modulefile_for(version)
-            modulefile_for_tag(tag_for(version))
+        def metadata_for(version)
+            metadata_for_tag(tag_for(version))
         end
 
-        def on_tag_for(version, &block)
-            @git.on_tag(tag_for(version), &block)
+        def with_tag_for(version, &block)
+            @git.with_tag(tag_for(version), &block)
         end
 
         def tag_for(version)
@@ -131,7 +141,7 @@ module PuppetLibrary::Forge
 
         def tag_versions
             tags_to_versions = tags.map do |tag|
-                [ modulefile_for_tag(tag).get_version, tag ]
+                [ metadata_for_tag(tag)["version"], tag ]
             end
             Hash[tags_to_versions]
         end

data/lib/puppet_library/forge/source.rb

@@ -25,7 +25,8 @@ module PuppetLibrary::Forge
     # Metadata (+metadata.json+) is generated on the fly.
     #
     # <b>Note:</b>
-    # The module directory must have a +Modulefile+.
+    # The module directory must have either a +metadata.json+ or a +Modulefile+.
+    # If it contains both, +metadata.json+ will be used.
     #
     # <b>Usage:</b>
     #
@@ -52,41 +53,47 @@ module PuppetLibrary::Forge
             raise "Module directory '#{module_dir.path}' doesn't exist" unless File.directory? module_dir.path
             raise "Module directory '#{module_dir.path}' isn't readable" unless File.executable? module_dir.path
             @module_dir = module_dir
-            @cache = PuppetLibrary::Http::Cache::InMemory.new(CACHE_TTL_MILLIS)
+            @metadata_cache = PuppetLibrary::Http::Cache::InMemory.new(CACHE_TTL_MILLIS)
         end
 
         def get_module(author, name, version)
             return nil unless this_module?(author, name, version)
             PuppetLibrary::Archive::Archiver.archive_dir(@module_dir.path, "#{author}-#{name}-#{version}") do |archive|
                 archive.add_file("metadata.json", 0644) do |entry|
-                    entry.write modulefile.to_metadata.to_json
+                    entry.write metadata.to_json
                 end
             end
         end
 
         def get_metadata(author, module_name)
             return [] unless this_module?(author, module_name)
-            [ modulefile.to_metadata ]
+            [ metadata ]
         end
 
         def get_all_metadata
-            get_metadata(modulefile.get_author, modulefile.get_simple_name)
+            get_metadata(metadata["author"], metadata["name"].split("-").last)
         end
 
         private
         def this_module?(author, module_name, version = nil)
-            same_module = modulefile.get_name == "#{author}-#{module_name}"
+            same_module = metadata["name"] == "#{author}-#{module_name}"
             if version.nil?
                 return same_module
             else
-                return same_module && modulefile.get_version == version
+                return same_module && metadata["version"] == version
             end
         end
 
-        def modulefile
-            modulefile_path = File.join(@module_dir.path, "Modulefile")
-            @cache.get modulefile_path do
-                PuppetLibrary::PuppetModule::Modulefile.read(modulefile_path)
+        def metadata
+            @metadata_cache.get "metadata" do
+                metadata_file_path = File.join(@module_dir.path, "metadata.json")
+                modulefile_path = File.join(@module_dir.path, "Modulefile")
+                if File.exist?(metadata_file_path) && ! File.exist?(modulefile_path)
+                    JSON.parse(File.read(metadata_file_path))
+                else
+                    modulefile = PuppetLibrary::PuppetModule::Modulefile.read(modulefile_path)
+                    modulefile.to_metadata
+                end
             end
         end
     end

data/lib/puppet_library/util/git.rb

@@ -39,11 +39,11 @@ module PuppetLibrary::Util
             git("tag").split
         end
 
-        def on_tag(tag)
+        def with_tag(tag)
             update_cache!
             PuppetLibrary::Util::TempDir.use "git" do |path|
                 git "checkout #{tag}", path
-                yield
+                yield(path)
             end
         end
 
@@ -75,14 +75,14 @@ module PuppetLibrary::Util
         def create_cache
             @mutex.synchronize do
                 info "Cloning Git repository from #{@source} to #{@git_dir}"
-                git "clone --bare #{@source} #{@git_dir}"
+                git "clone --mirror #{@source} #{@git_dir}"
                 FileUtils.touch fetch_file
             end
         end
 
         def update_cache
             @mutex.synchronize do
-                git "fetch --tags"
+                git "fetch origin --tags --update-head-ok"
             end
         end
 

data/lib/puppet_library/util/logging.rb

@@ -18,12 +18,21 @@ require 'logger'
 
 module PuppetLibrary::Util
     module Logging
-        def log_io
-            @log_io ||= StringIO.new
+        class LogCollector < Array
+            def write(message)
+                self << message
+            end
+
+            def close
+            end
+        end
+
+        def logs
+            @logs ||= LogCollector.new
         end
 
         def logger
-            destination = ENV["TESTING"] ? log_io : STDERR
+            destination = ENV["TESTING"] ? logs : STDERR
             @logger ||= Logger.new(destination).tap do |logger|
                 logger.progname = self.class.name
                 logger.level = Logger::DEBUG

data/lib/puppet_library/util/temp_dir.rb

@@ -21,7 +21,14 @@ module PuppetLibrary::Util
     class TempDir
         attr_reader :path
 
-        def self.use(name, &block)
+        def self.use(name)
+            path = create(name)
+            yield(path)
+        ensure
+            FileUtils.rm_rf path
+        end
+
+        def self.cd(name, &block)
             path = create(name)
             Dir.chdir(path, &block)
         ensure

data/lib/puppet_library/version.rb

@@ -16,5 +16,5 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 module PuppetLibrary
-    VERSION = "0.13.0"
+    VERSION = "0.14.0"
 end

data/puppet-library.gemspec

@@ -47,11 +47,11 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "gitsu"
   spec.add_development_dependency "librarian-puppet", "0.9.10" # 0.9.12 breaks on Ruby 1.8.7
   spec.add_development_dependency "mime-types", "< 2"
-  spec.add_development_dependency "pry"
+  spec.add_development_dependency "pry", "0.9.12.6"
   spec.add_development_dependency "puppet", "~> 3.3.0"
   spec.add_development_dependency "rack-test"
   spec.add_development_dependency "rake"
-  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "rspec", "~> 3.0.0"
   spec.add_development_dependency "simplecov"
 
   # Guard has dependencies that don't work with Ruby < 1.9