puppet-library 0.10.0 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e4ad4f98c15d6ba4b89830bb6f30c72dc2ce3efd
-  data.tar.gz: 487e711b4ecdea363118803048f8320a902ed6e2
+  metadata.gz: 5501e3e79913a9181ca6b829be328e8c6b3e3c64
+  data.tar.gz: 6625c303010b85e82a6f6d121379becb92634eff
 SHA512:
-  metadata.gz: 11bb00e50d4037bb7e14c60e69fe8c5d99e5f032dd4cb76adaa670462df40c62b7fe230488b2747fa730c685b88315560bb9543dff3698ef81894e4262ace257
-  data.tar.gz: 986c55e4faa32246bc1c9387180fec9488b2c153caed62038328f10a5af4b78180880f007e548e6fe0f48bd8e597b04c89a5d48eb3753c754c08855ae04eddc4
+  metadata.gz: 2cfa2f30bc16b177d5c272fa495b3d3b5f522959507faf17612b3234c2eb13fc19835b64d8703103692102e25a186188de8b977fd7764eb3c2083ab5e4ec0908
+  data.tar.gz: 4df43a457d93c2722154f295aeec9c43e250f4dba2ef7be9e881e2d56348c7adeafe88bd8c8186ee1cf171f844cb07865be2effb4a15258b1ecd1d55dc0e01a0

data/CHANGELOG.yml

@@ -64,3 +64,14 @@
   changes:
       - Added new Git repo forge that serves modules from Git repositories, using tags for version numbers!
       - Fixed imports to enable easier embedding of Puppet Library
+- tag: v0.11.0
+  changes:
+      - Added support for nontrivial tag names as version numbers when proxying Git repositories (e.g. tags named "v1.0.1")
+      - Added REST API to clear forge caches
+      - Git repositories are cached at startup for better performance and reliability
+      - Ensured temp directories are cleaned up more reliably
+      - Improved logging
+      - Improved Ruby config API for Rack/Passenger/embedded use
+      - Fixed issue with "Too many open files" caused by Git repo forges on Ruby 1.8
+      - Fixed issue Git repo forges with tags without Modulefiles
+      - Fixed issue Git repo forges with no description in Modulefile

data/README.md

@@ -16,6 +16,7 @@ Plugins can be created to serve modules from arbitrary sources. Puppet Library c
  - proxying a remote forge (or forges)
  - caching proxied forges to disk
  - serving modules from source on disk
+ - serving modules from Git repositories using tags as version numbers!
 
 ## Installation
 
@@ -90,6 +91,7 @@ Puppet Library contains built-in support for:
  - serving packaged (`.tar.gz`) modules from a directory (or directories) of your choosing
  - proxying a remote forge (or forges)
  - serving modules from source on disk
+ - serving modules from Git repositories using tags as version numbers!
  - serving modules from a combination of the above
 
 ## Compatibility with other tools
@@ -145,16 +147,23 @@ cat > /webapps/puppet-library/config.ru <<EOF
 require "rubygems"
 require "puppet_library"
 
-# NB: this config API is not yet stable, and may change without notice
-server = PuppetLibrary::Server.configure do |library|
-    # Serve our private modules
-    library.forge PuppetLibrary::Forge::Directory do |forge|
-        forge.path = "/var/lib/modules"
+# NB: this config API is not yet stable, and may change without notice.
+# The format shown is valid from Puppet Library v0.11.0
+server = PuppetLibrary::Server.configure do
+    # Serve our private modules out of a directory on disk
+    forge :directory do
+        path = "/var/lib/modules"
+    end
+
+    # Serve the latest versions from GitHub
+    forge :git_repository do
+        source "http://github.com/example/puppetlabs-apache-fork.git"
+        include_tags /[0-9.]+/
     end
 
     # Download all other modules from the Puppet Forge
-    library.forge PuppetLibrary::Forge::Proxy do |forge|
-        forge.url = "http://forge.puppetlabs.com"
+    forge :proxy do
+        url "http://forge.puppetlabs.com"
     end
 end
 

data/Rakefile

@@ -223,32 +223,32 @@ end
 desc "Import files individually to make sure they can be imported externally"
 task "test-imports" do
     puts "Testing imports:"
-    Dir.chdir "lib"
-
-    paths = Dir["**/*.rb"].map {|f| f.sub /\.rb$/, "" }
-    paths = paths.sort_by {|p| p.count "/"}
-    errors = []
-    paths.each do |path|
-        print "importing #{path}..."
-        output = `bundle exec ruby -e '$LOAD_PATH.unshift(File.expand_path(".")); require "#{path}"' 2>&1`
-        print " ["
-        if $?.success?
-            print "OK".green
-        else
-            print "FAIL".red
-            errors << "#{path}: #{output.red}"
+    Dir.chdir "lib" do
+        paths = Dir["**/*.rb"].map {|f| f.sub /\.rb$/, "" }
+        paths = paths.sort_by {|p| p.count "/"}
+        errors = []
+        paths.each do |path|
+            print "importing #{path}..."
+            output = `bundle exec ruby -e '$LOAD_PATH.unshift(File.expand_path(".")); require "#{path}"' 2>&1`
+            print " ["
+            if $?.success?
+                print "OK".green
+            else
+                print "FAIL".red
+                errors << "#{path}: #{output.red}"
+            end
+            puts "]"
         end
-        puts "]"
-    end
 
-    unless errors.empty?
-        puts
-        puts
-        fail <<-EOFAILURE
+        unless errors.empty?
+            puts
+            puts
+            fail <<-EOFAILURE
 Failed to import some files:
 
 #{errors.join "\n\n"}
-        EOFAILURE
+            EOFAILURE
+        end
     end
 end
 

data/TODO.yml

@@ -1,10 +1,15 @@
 features:
     - Web UI:
         - asynchronous (index page loads too slowly)
-    - Proxy a git repository:
-        - make it work with remote forges (clone and fetch on query/download)
-        - make it fast (pre-clone repos on startup (maybe just warn if they can't be found). Maybe cache artifacts?)
-    - logging
+    - Git repo forge:
+        - make it more flexible:
+            - fail gracefully if we can't contact the remote repo
+    - Performance:
+        - cache stuff so it's faster
+    - logging:
+        - use Ruby logging
+        - include some verbose logging
+        - improve error messages (e.g. for when a module can't be untarred)
     - Named subforges:
         - per-subforge queries
         - upload modules (web form and API)
@@ -13,8 +18,7 @@ features:
         - delete modules from disk (web form)
     - Authentication
     - Config APIs:
-        - parameterless blocks (especially for Server. Include PuppetLibrary::Forge aliased to Forge in scope)
-        - validation of required params
+        - documentation
 
 dubious_features:
     - Proxy modules' source in a directory (supported individually for now: should we just leave it that way?)

data/bin/puppet-library

@@ -19,7 +19,6 @@ lib_path = File.join(File.dirname(__FILE__), '..', 'lib')
 $LOAD_PATH.unshift lib_path unless $LOAD_PATH.include? lib_path
 
 require "puppet_library"
-require 'optparse'
 
 module PuppetLibrary
     PuppetLibrary.new.go(ARGV)

data/config.ru

@@ -18,33 +18,31 @@
 require 'rubygems'
 require 'puppet_library'
 
-server = PuppetLibrary::Server.configure do |library|
-    Forge = PuppetLibrary::Forge
-
+server = PuppetLibrary::Server.configure do
     # My custom modules
-    library.forge Forge::Directory do |forge|
-        forge.path = "/var/lib/modules"
+    forge :directory do
+        path "/var/puppet/modules"
     end
 
     # Unreleased versions from Github
-    library.forge Forge::GitRepository do |forge|
-        forge.repository = "https://github.com/puppetlabs/puppetlabs-apache.git"
-        forge.tag_regex = /[0-9.]+/
+    forge :git_repository do
+        source "https://github.com/puppetlabs/puppetlabs-apache.git"
+        include_tags /[0-9.]+/
     end
 
-    library.forge Forge::GitRepository do |forge|
-        forge.repository = "https://github.com/puppetlabs/puppetlabs-concat.git"
-        forge.tag_regex = /[0-9.]+/
+    forge :git_repository do
+        source "https://github.com/puppetlabs/puppetlabs-concat.git"
+        include_tags /[0-9.]+/
     end
 
-    library.forge Forge::GitRepository do |forge|
-        forge.repository = "https://github.com/puppetlabs/puppetlabs-stdlib.git"
-        forge.tag_regex = /[0-9.]+/
+    forge :git_repository do
+        source "https://github.com/puppetlabs/puppetlabs-stdlib.git"
+        include_tags /[0-9.]+/
     end
 
     # Everything from The Forge
-    library.forge Forge::Proxy do |forge|
-        forge.url = "http://forge.puppetlabs.com"
+    forge :proxy do
+        url "http://forge.puppetlabs.com"
     end
 end
 

data/features/step_definitions/sinatra_steps.rb

@@ -19,32 +19,23 @@ spec_path = File.expand_path("../../spec", File.dirname(__FILE__))
 $LOAD_PATH.unshift spec_path unless $LOAD_PATH.include? spec_path
 
 require 'module_spec_helper'
+require 'spec_helper'
 require 'fileutils'
 
 include FileUtils
 
 def module_dir
-    unless @module_dir
-        file = Tempfile.new("module_dir")
-        @module_dir = file.path
-        file.delete
-    end
-    @module_dir
+    @module_dir ||= Tempdir.new("module_dir")
 end
 
 def module_writer
-    @module_writer ||= ModuleSpecHelper::ModuleWriter.new(module_dir)
+    @module_writer ||= ModuleSpecHelper::ModuleWriter.new(module_dir.path)
 end
 
 Before do
-    mkdir_p module_dir
     forge.add_forge PuppetLibrary::Forge::Directory.new(module_dir)
 end
 
-After do
-    rm_rf module_dir
-end
-
 Given /^the "(.*?)" module is available at version "(.*?)"$/ do |full_name, version|
     author, name = full_name.split "/"
     module_writer.write_module(author, name, version)

data/lib/puppet_library/forge.rb

@@ -22,6 +22,7 @@ module PuppetLibrary::Forge
     require 'puppet_library/forge/abstract'
     require 'puppet_library/forge/cache'
     require 'puppet_library/forge/directory'
+    require 'puppet_library/forge/forge'
     require 'puppet_library/forge/git_repository'
     require 'puppet_library/forge/multi'
     require 'puppet_library/forge/proxy'

data/lib/puppet_library/forge/abstract.rb

@@ -15,6 +15,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+require 'puppet_library/forge/forge'
 require 'puppet_library/forge/search_result'
 require 'puppet_library/util'
 
@@ -24,7 +25,7 @@ module PuppetLibrary::Forge
     # types.
     #
     # See PuppetLibrary::Forge::Directory for an example
-    class Abstract
+    class Abstract < Forge
         def initialize(module_repo)
             @repo = module_repo
         end

data/lib/puppet_library/forge/cache.rb

@@ -17,6 +17,8 @@
 
 require 'puppet_library/forge/proxy'
 require 'puppet_library/http/http_client'
+require 'puppet_library/util/config_api'
+
 module PuppetLibrary::Forge
 
     # A forge that proxies a remote forge. All module archives downloaded from the
@@ -32,14 +34,16 @@ module PuppetLibrary::Forge
     #        repo.path = "/var/modules/cache
     #    end
     class Cache < Proxy
-        class Config
-            attr_accessor :url, :path
-        end
 
-        def self.configure
-            config = Config.new
-            yield(config)
-            Cache.new(config.url, config.path)
+        def self.configure(&block)
+            config_api = PuppetLibrary::Util::ConfigApi.for(Cache) do
+                required :url, "URL"
+                required :path, "path" do |path|
+                    Dir.new(path)
+                end
+            end
+            config = config_api.configure(&block)
+            Cache.new(config.get_url, config.get_path)
         end
 
         # * <tt>:url</tt> - The URL of the remote forge.

data/lib/puppet_library/forge/directory.rb

@@ -17,6 +17,7 @@
 require 'json'
 require 'puppet_library/forge/abstract'
 require 'puppet_library/archive/archive_reader'
+require 'puppet_library/util/config_api'
 
 module PuppetLibrary::Forge
 
@@ -35,27 +36,28 @@ module PuppetLibrary::Forge
     #        repo.path = "/var/modules/cache
     #    end
     class Directory < PuppetLibrary::Forge::Abstract
-        class Config
-            attr_accessor :path
-        end
 
-        def self.configure
-            config = Config.new
-            yield(config)
-            Directory.new(config.path)
+        def self.configure(&block)
+            config_api = PuppetLibrary::Util::ConfigApi.for(Directory) do
+                required :path, "path to a directory to serve modules from" do |dir|
+                    Dir.new(File.expand_path(dir)).tap do |dir|
+                        raise "Module directory '#{dir}' isn't readable" unless File.executable? dir
+                    end
+                end
+            end
+            config = config_api.configure(&block)
+            Directory.new(config.get_path)
         end
 
         # * <tt>:module_dir</tt> - The directory containing the packaged modules.
         def initialize(module_dir)
             super(self)
-            raise "Module directory '#{module_dir}' doesn't exist" unless File.directory? module_dir
-            raise "Module directory '#{module_dir}' isn't readable" unless File.executable? module_dir
             @module_dir = module_dir
         end
 
         def get_module(author, name, version)
             file_name = "#{author}-#{name}-#{version}.tar.gz"
-            path = File.join(File.expand_path(@module_dir), file_name)
+            path = File.join(File.expand_path(@module_dir.path), file_name)
             if File.exist? path
                 File.open(path, 'r')
             else
@@ -63,8 +65,8 @@ module PuppetLibrary::Forge
             end
         end
 
-        def get_metadata(author = "*", module_name = "")
-            Dir["#{@module_dir}/#{author}-#{module_name}*"].map do |module_path|
+        def get_metadata(author, module_name)
+            Dir["#{@module_dir.path}/#{author}-#{module_name}*"].map do |module_path|
                 archive = PuppetLibrary::Archive::ArchiveReader.new(module_path)
                 metadata_file = archive.read_entry %r[[^/]+/metadata\.json$]
                 JSON.parse(metadata_file)
@@ -72,7 +74,7 @@ module PuppetLibrary::Forge
         end
 
         def get_all_metadata
-            get_metadata
+            get_metadata("*", "")
         end
     end
 end

data/lib/puppet_library/forge/forge.rb

@@ -0,0 +1,25 @@
+# Puppet Library
+# Copyright (C) 2014 drrb
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+module PuppetLibrary::Forge
+    class Forge
+        def prime
+        end
+
+        def clear_cache
+        end
+    end
+end

data/lib/puppet_library/forge/git_repository.rb

@@ -21,6 +21,8 @@ require 'rubygems/package'
 require 'puppet_library/forge/abstract'
 require 'puppet_library/util/git'
 require 'puppet_library/util/temp_dir'
+require 'puppet_library/util/config_api'
+require 'puppet_library/http/cache/in_memory'
 
 module PuppetLibrary::Forge
     # A forge for serving modules from a Git repository
@@ -35,27 +37,38 @@ module PuppetLibrary::Forge
     #        repo.include_tags = /[0-9.]+/
     #    end
     class GitRepository < PuppetLibrary::Forge::Abstract
-        class Config
-            attr_accessor :source
-            attr_accessor :include_tags
-        end
 
-        def self.configure
-            config = Config.new
-            yield(config)
-            GitRepository.new(config.source, config.include_tags)
+        def self.configure(&block)
+            config_api = PuppetLibrary::Util::ConfigApi.for(GitRepository) do
+                required :source, "path or URL"
+                required :include_tags, "regex" do |value|
+                    value.tap do |value|
+                        raise "not a regex" unless value.is_a? Regexp
+                    end
+                end
+            end
+
+            config = config_api.configure(&block)
+            cache_dir = PuppetLibrary::Util::TempDir.new("git-repo-cache")
+            git = PuppetLibrary::Util::Git.new(config.get_source, cache_dir)
+            GitRepository.new(git, config.get_include_tags)
         end
 
         # * <tt>:source</tt> - The URL or path of the git repository
         # * <tt>:version_tag_regex</tt> - A regex that describes which tags to serve
-        def initialize(source, version_tag_regex)
+        def initialize(git, version_tag_regex)
             super(self)
-            cache_path = PuppetLibrary::Util::TempDir.create("git-repo-cache")
             @version_tag_regex = version_tag_regex
-            @git = PuppetLibrary::Util::Git.new(source, cache_path)
+            @git = git
+            @modulefile_cache = PuppetLibrary::Http::Cache::InMemory.new
+            @tags_cache = PuppetLibrary::Http::Cache::InMemory.new
+        end
+
+        def prime
+            @git.update_cache!
         end
 
-        def destroy!
+        def clear_cache
             @git.clear_cache!
         end
 
@@ -89,14 +102,19 @@ module PuppetLibrary::Forge
         end
 
         private
-
         def tags
-            @git.tags.select {|tag| tag =~ @version_tag_regex }
+            @tags_cache.get do
+                tags = @git.tags
+                tags = tags.select {|tag| tag =~ @version_tag_regex }
+                tags = tags.select {|tag| @git.file_exists?("Modulefile", tag) }
+            end
         end
 
         def modulefile_for_tag(tag)
-            modulefile_source = @git.read_file("Modulefile", tag)
-            PuppetLibrary::PuppetModule::Modulefile.parse(modulefile_source)
+            @modulefile_cache.get(tag) do
+                modulefile_source = @git.read_file("Modulefile", tag)
+                PuppetLibrary::PuppetModule::Modulefile.parse(modulefile_source)
+            end
         end
 
         def modulefile_for(version)
@@ -108,7 +126,14 @@ module PuppetLibrary::Forge
         end
 
         def tag_for(version)
-            version
+            tag_versions[version]
+        end
+
+        def tag_versions
+            tags_to_versions = tags.map do |tag|
+                [ modulefile_for_tag(tag).get_version, tag ]
+            end
+            Hash[tags_to_versions]
         end
     end
 end