puppet-library 0.9.1 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
---- 
-SHA1: 
-  metadata.gz: 280994b916da7d8f05ba4cf98828e6459d6fdea4
-  data.tar.gz: d1654492cdb5e30855c62b5fbabf08d7c1cb38a5
-SHA512: 
-  metadata.gz: d518cbe064ed9c664fd2198d070ce7db2576c761d9526a52c3ee577d8f067c0183462027dd78420810f82c1262f8fa8d9ac5e39c64b93e6a70e19baa8aff4022
-  data.tar.gz: 28e347c2c9ed7215a6c2f2915264e463580d337e596464a6d5afddcb64d35bfec27b6a9be176712cb939f455f10b4694eea55adca0e98a90cb9d5f7337c953cc
+---
+SHA1:
+  metadata.gz: e4ad4f98c15d6ba4b89830bb6f30c72dc2ce3efd
+  data.tar.gz: 487e711b4ecdea363118803048f8320a902ed6e2
+SHA512:
+  metadata.gz: 11bb00e50d4037bb7e14c60e69fe8c5d99e5f032dd4cb76adaa670462df40c62b7fe230488b2747fa730c685b88315560bb9543dff3698ef81894e4262ace257
+  data.tar.gz: 986c55e4faa32246bc1c9387180fec9488b2c153caed62038328f10a5af4b78180880f007e548e6fe0f48bd8e597b04c89a5d48eb3753c754c08855ae04eddc4

data/CHANGELOG.yml

@@ -62,4 +62,5 @@
       - Fixed source forge bug that generated metadata incorrectly
 - tag: v0.10.0
   changes:
+      - Added new Git repo forge that serves modules from Git repositories, using tags for version numbers!
       - Fixed imports to enable easier embedding of Puppet Library

data/README.md

@@ -146,11 +146,16 @@ require "rubygems"
 require "puppet_library"
 
 # NB: this config API is not yet stable, and may change without notice
-server = PuppetLibrary::Server.set_up do |library|
+server = PuppetLibrary::Server.configure do |library|
     # Serve our private modules
-    library.forge PuppetLibrary::Forge::Directory.new("/var/lib/modules")
-    # Download all other moduls from the Puppet Forge
-    library.forge PuppetLibrary::Forge::Proxy.new("http://forge.puppetlabs.com")
+    library.forge PuppetLibrary::Forge::Directory do |forge|
+        forge.path = "/var/lib/modules"
+    end
+
+    # Download all other modules from the Puppet Forge
+    library.forge PuppetLibrary::Forge::Proxy do |forge|
+        forge.url = "http://forge.puppetlabs.com"
+    end
 end
 
 run server

data/Rakefile

@@ -17,7 +17,6 @@
 
 require 'bundler/gem_tasks'
 require 'rspec/core/rake_task'
-require 'coveralls/rake/task'
 require 'json'
 require 'yaml'
 require 'net/http'
@@ -61,7 +60,16 @@ rescue
     return true
 end
 
-Coveralls::RakeTask.new
+if RUBY_VERSION.start_with? "1.8"
+    namespace :coveralls do
+        task :push do
+            puts "Skipping Coverage pushing because this Ruby version doesn't support it"
+        end
+    end
+else
+    require 'coveralls/rake/task'
+    Coveralls::RakeTask.new
+end
 
 desc "Run the specs"
 RSpec::Core::RakeTask.new(:spec)
@@ -102,7 +110,7 @@ end
 task :default => [:test, 'coveralls:push']
 
 desc "Check it works on all local rubies"
-task :verify => 'test-imports' do
+task :verify => [ 'check-license', 'test-imports' ] do
     versions = SUPPORTED_RUBY_VERSIONS
     puts "\nRunning Specs".green
     spec_results = versions.map do |ruby_version|
@@ -122,13 +130,18 @@ task :verify => 'test-imports' do
         system "rvm #{ruby_version} do bundle exec rake features"
     end
 
+    gem_versions = versions.map do |ruby_version|
+        `rvm #{ruby_version} do gem --version`.strip
+    end
+
     puts "\nResults:\n".green
-    results = [ spec_results, integration_test_results, acceptance_test_results ].transpose
-    puts "+---------+-------+-------+-------+"
-    puts "| Version | Specs |  ITs  |  ATs  |"
-    puts "+---------+-------+-------+-------+"
-    versions.zip(results).each do |(version, (spec_result, integration_test_result, acceptance_test_result))|
+    results = [ gem_versions, spec_results, integration_test_results, acceptance_test_results ].transpose
+    puts "+---------+----------+-------+-------+-------+"
+    puts "| Version | Gem Ver. | Specs |  ITs  |  ATs  |"
+    puts "+---------+----------+-------+-------+-------+"
+    versions.zip(results).each do |(version, (gem_version, spec_result, integration_test_result, acceptance_test_result))|
         v = version.ljust(7)
+        g = gem_version.ljust(8)
         s = spec_result ? "pass".green : "fail".red
         if ruby_version_supports_integration_test? version
             i = integration_test_result ? "pass".green : "fail".red
@@ -141,35 +154,44 @@ task :verify => 'test-imports' do
         else
             a = "skip".yellow
         end
-        puts "| #{v} | #{s}  | #{i}  | #{a}  |"
+        puts "| #{v} | #{g} | #{s}  | #{i}  | #{a}  |"
     end
-    puts "+---------+-------+-------+-------+"
+    puts "+---------+----------+-------+-------+-------+"
 
-    versions.zip(results).each do |(version, (spec_result, integration_test_result))|
+    versions.zip(results).each do |(version, (gem_version, spec_result, integration_test_result, acceptance_test_result))|
         unless spec_result
             fail "Specs failed with Ruby #{version}"
         end
 
-        if ruby_version_supports_integration_test?(version)
-            unless integration_test_result
-                fail "Integration tests failed with Ruby #{version}"
-            end
+        unless integration_test_result
+            fail "Integration tests failed with Ruby #{version}"
+        end
+
+        unless acceptance_test_result
+            fail "Acceptance tests failed with Ruby #{version}"
         end
     end
 end
 
+desc "Run simple checks"
+task :check => [ "check-license", "test-imports" ]do
+end
+
 desc "Check all files for license headers"
 task "check-license" do
-    puts "Checking that all program files contain license headers"
+    print "Checking that all program files contain license headers"
 
     files = `git ls-files`.split "\n"
     ignored_files = File.read(".licenseignore").split("\n") << ".licenseignore"
     offending_files = files.reject { |file| File.read(file).include? "WITHOUT ANY WARRANTY" } - ignored_files
-    if offending_files.empty?
-        puts "Done"
+    ok = offending_files.empty?
+    result = if ok
+        "OK".green
     else
-        abort("ERROR: THE FOLLOWING FILES HAVE NO LICENSE HEADERS: \n" + offending_files.join("\n"))
+        "FAIL".red
     end
+    puts " [ #{result} ]"
+    abort("ERROR: THE FOLLOWING FILES HAVE NO LICENSE HEADERS: \n" + offending_files.join("\n")) unless ok
 end
 
 desc "Print the version number"
@@ -200,8 +222,7 @@ end
 
 desc "Import files individually to make sure they can be imported externally"
 task "test-imports" do
-    puts "Testing imports"
-    puts
+    puts "Testing imports:"
     Dir.chdir "lib"
 
     paths = Dir["**/*.rb"].map {|f| f.sub /\.rb$/, "" }
@@ -209,7 +230,7 @@ task "test-imports" do
     errors = []
     paths.each do |path|
         print "importing #{path}..."
-        output = `ruby -e '$LOAD_PATH.unshift(File.expand_path(".")); require "#{path}"' 2>&1`
+        output = `bundle exec ruby -e '$LOAD_PATH.unshift(File.expand_path(".")); require "#{path}"' 2>&1`
         print " ["
         if $?.success?
             print "OK".green
@@ -243,7 +264,7 @@ def upload_release_notes(version)
     end
 
     tag = "v#{version}"
-    changelog = YAML.load_file("CHANGELOG.yml")
+    changelog = YAML.load_file(File.expand_path("CHANGELOG.yml", File.dirname(__FILE__)))
     release_notes = changelog.find {|release| release["tag"] == tag}
     changes = release_notes ? release_notes["changes"] : []
     description = changes.map { |change| "- #{change}" }.join("\n")

data/TODO.yml

@@ -2,9 +2,6 @@ features:
     - Web UI:
         - asynchronous (index page loads too slowly)
     - Proxy a git repository:
-        - make it threadsafe (checkouts into temp dirs)
-        - make the specs fast
-        - add an integration test
         - make it work with remote forges (clone and fetch on query/download)
         - make it fast (pre-clone repos on startup (maybe just warn if they can't be found). Maybe cache artifacts?)
     - logging
@@ -15,7 +12,9 @@ features:
         - clear proxy cache (web form and API)
         - delete modules from disk (web form)
     - Authentication
-    - Make imports work properly from pry, and other projects
+    - Config APIs:
+        - parameterless blocks (especially for Server. Include PuppetLibrary::Forge aliased to Forge in scope)
+        - validation of required params
 
 dubious_features:
     - Proxy modules' source in a directory (supported individually for now: should we just leave it that way?)

data/config.ru

@@ -18,9 +18,34 @@
 require 'rubygems'
 require 'puppet_library'
 
-server = PuppetLibrary::Server.set_up do |library|
-    library.forge PuppetLibrary::Forge::Proxy.new("http://forge.puppetlabs.com")
-    library.forge PuppetLibrary::Forge::Directory.new("/var/lib/modules")
+server = PuppetLibrary::Server.configure do |library|
+    Forge = PuppetLibrary::Forge
+
+    # My custom modules
+    library.forge Forge::Directory do |forge|
+        forge.path = "/var/lib/modules"
+    end
+
+    # Unreleased versions from Github
+    library.forge Forge::GitRepository do |forge|
+        forge.repository = "https://github.com/puppetlabs/puppetlabs-apache.git"
+        forge.tag_regex = /[0-9.]+/
+    end
+
+    library.forge Forge::GitRepository do |forge|
+        forge.repository = "https://github.com/puppetlabs/puppetlabs-concat.git"
+        forge.tag_regex = /[0-9.]+/
+    end
+
+    library.forge Forge::GitRepository do |forge|
+        forge.repository = "https://github.com/puppetlabs/puppetlabs-stdlib.git"
+        forge.tag_regex = /[0-9.]+/
+    end
+
+    # Everything from The Forge
+    library.forge Forge::Proxy do |forge|
+        forge.url = "http://forge.puppetlabs.com"
+    end
 end
 
 run server

data/lib/puppet_library/forge/cache.rb

@@ -21,7 +21,26 @@ module PuppetLibrary::Forge
 
     # A forge that proxies a remote forge. All module archives downloaded from the
     # remote forged are cached to disk.
+    #
+    # <b>Usage:</b>
+    #
+    #    forge = PuppetLibrary::Forge::Cache.configure do |repo|
+    #        # The URL of the remote forge
+    #        repo.url = "http://forge.example.com
+    #
+    #        # The path to cache the files to on disk
+    #        repo.path = "/var/modules/cache
+    #    end
     class Cache < Proxy
+        class Config
+            attr_accessor :url, :path
+        end
+
+        def self.configure
+            config = Config.new
+            yield(config)
+            Cache.new(config.url, config.path)
+        end
 
         # * <tt>:url</tt> - The URL of the remote forge.
         # * <tt>:cache_dir</tt> - The directory in which to cache the downloaded artifacts.

data/lib/puppet_library/forge/directory.rb

@@ -27,7 +27,23 @@ module PuppetLibrary::Forge
     # * The modules must be named in the format <tt>author-modulename-version.tar.gz</tt>
     # * The modules must contain a +metadata.json+ file
     # That is, the format must be the same as what is produced by <tt>puppet module build</tt>
+    #
+    # <b>Usage:</b>
+    #
+    #    forge = PuppetLibrary::Forge::Directory.configure do |repo|
+    #        # The path to serve the modules from
+    #        repo.path = "/var/modules/cache
+    #    end
     class Directory < PuppetLibrary::Forge::Abstract
+        class Config
+            attr_accessor :path
+        end
+
+        def self.configure
+            config = Config.new
+            yield(config)
+            Directory.new(config.path)
+        end
 
         # * <tt>:module_dir</tt> - The directory containing the packaged modules.
         def initialize(module_dir)

data/lib/puppet_library/forge/git_repository.rb

@@ -15,92 +15,100 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 require 'json'
-require 'rubygems/package'
 require 'zlib'
+require 'open3'
+require 'rubygems/package'
 require 'puppet_library/forge/abstract'
+require 'puppet_library/util/git'
+require 'puppet_library/util/temp_dir'
 
 module PuppetLibrary::Forge
+    # A forge for serving modules from a Git repository
+    #
+    # <b>Usage:</b>
+    #
+    #    forge = PuppetLibrary::Forge::GitRepository.configure do |repo|
+    #        # The location of the git repository
+    #        repo.source = "http://example.com/mymodule.git
+    #
+    #        # A regular expression describing which tags to serve
+    #        repo.include_tags = /[0-9.]+/
+    #    end
     class GitRepository < PuppetLibrary::Forge::Abstract
-        def initialize(author, name, version_tag_regex, git_path)
+        class Config
+            attr_accessor :source
+            attr_accessor :include_tags
+        end
+
+        def self.configure
+            config = Config.new
+            yield(config)
+            GitRepository.new(config.source, config.include_tags)
+        end
+
+        # * <tt>:source</tt> - The URL or path of the git repository
+        # * <tt>:version_tag_regex</tt> - A regex that describes which tags to serve
+        def initialize(source, version_tag_regex)
             super(self)
-            @author = author
-            @name = name
-            @path = File.expand_path(git_path)
+            cache_path = PuppetLibrary::Util::TempDir.create("git-repo-cache")
             @version_tag_regex = version_tag_regex
+            @git = PuppetLibrary::Util::Git.new(source, cache_path)
+        end
+
+        def destroy!
+            @git.clear_cache!
         end
 
         def get_module(author, name, version)
-            unless author == @author && name == @name
-                return nil
-            end
-            unless tags.include? tag_name(version)
-                return nil
-            end
+            return nil unless tags.include? tag_for(version)
+
+            metadata = modulefile_for(version).to_metadata
+            return nil unless metadata["name"] == "#{author}-#{name}"
+
             on_tag_for(version) do
-                PuppetLibrary::Archive::Archiver.archive_dir(@path, "#{@author}-#{@name}-#{version}") do |archive|
+                PuppetLibrary::Archive::Archiver.archive_dir('.', "#{metadata["name"]}-#{version}") do |archive|
                     archive.add_file("metadata.json", 0644) do |entry|
-                        entry.write modulefile.to_metadata.to_json
+                        entry.write metadata.to_json
                     end
                 end
             end
         end
 
-        def get_metadata(author, module_name)
-            unless author == @author && module_name == @name
-                return []
-            end
+        def get_all_metadata
             tags.map do |tag|
                 modulefile_for_tag(tag).to_metadata
             end
         end
 
-        def get_all_metadata
-            get_metadata(@author, @name)
+        def get_metadata(author, module_name)
+            metadata = get_all_metadata
+            metadata.select do |m|
+                m["author"] == author
+                m["name"] == "#{author}-#{module_name}"
+            end
         end
 
         private
+
         def tags
-            git("tag").split.select {|tag| tag =~ @version_tag_regex }
+            @git.tags.select {|tag| tag =~ @version_tag_regex }
         end
 
         def modulefile_for_tag(tag)
-            modulefile_source = git("show refs/tags/#{tag}:Modulefile")
+            modulefile_source = @git.read_file("Modulefile", tag)
             PuppetLibrary::PuppetModule::Modulefile.parse(modulefile_source)
         end
 
-        def modulefile
-            modulefile_path = File.join(@path, "Modulefile")
-            PuppetLibrary::PuppetModule::Modulefile.read(modulefile_path)
-        end
-
-        def package_name(version)
-            "#{@author}-#{@name}-#{version}.tar.gz"
-        end
-
-        def tag_name(version)
-            version
+        def modulefile_for(version)
+            modulefile_for_tag(tag_for(version))
         end
 
         def on_tag_for(version, &block)
-            on_tag(tag_name(version), &block)
-        end
-
-        def on_tag(tag, &block)
-            git "checkout #{tag}"
-            in_dir(@path, &block)
+            @git.on_tag(tag_for(version), &block)
         end
 
-        def in_dir(dir)
-            origin = Dir.pwd
-            Dir.chdir dir
-            yield
-        ensure
-            Dir.chdir origin
-        end
-
-        def git(command)
-            #TODO: redirection will fail on windows. How do we properly discard stderr?
-            IO.popen("git --git-dir=#{@path}/.git --work-tree=#{@path} #{command} 2>/dev/null").read
+        def tag_for(version)
+            version
         end
     end
 end

data/lib/puppet_library/forge/proxy.rb

@@ -22,13 +22,27 @@ require 'puppet_library/http/cache/noop'
 module PuppetLibrary::Forge
 
     # A forge that proxies a remote forge.
+    #
+    # <b>Usage:</b>
+    #
+    #    forge = PuppetLibrary::Forge::Proxy.configure do |repo|
+    #        # The URL of the remote forge
+    #        repo.url = "http://forge.example.com
+    #    end
     class Proxy
 
+        class Config
+            attr_accessor :url
+        end
+
+        def self.configure
+            config = Config.new
+            yield(config)
+            Proxy.new(config.url)
+        end
+
         # * <tt>:url</tt> - The URL of the remote forge.
-        def initialize(url,
-                       query_cache = PuppetLibrary::Http::Cache::InMemory.new,
-                       download_cache = PuppetLibrary::Http::Cache::NoOp.new,
-                       http_client = PuppetLibrary::Http::HttpClient.new)
+        def initialize(url, query_cache = PuppetLibrary::Http::Cache::InMemory.new, download_cache = PuppetLibrary::Http::Cache::NoOp.new, http_client = PuppetLibrary::Http::HttpClient.new)
             @url = PuppetLibrary::Http::Url.normalize(url)
             @http_client = http_client
             @query_cache = query_cache

data/lib/puppet_library/forge/source.rb

@@ -27,6 +27,16 @@ module PuppetLibrary::Forge
     # <b>Note:</b>
     # The module directory must have a +Modulefile+.
     class Source < PuppetLibrary::Forge::Abstract
+        class Config
+            attr_accessor :path
+        end
+
+        def self.configure
+            config = Config.new
+            yield(config)
+            Source.new(config.path)
+        end
+
         CACHE_TTL_MILLIS = 500
 
         # * <tt>:module_dir</tt> - The directory containing the module's source.

data/lib/puppet_library/puppet_library.rb

@@ -103,7 +103,7 @@ module PuppetLibrary
             load_defaults!(options)
             process_options!(options)
 
-            Server.set_up do |server|
+            Server.configure do |server|
                 options[:forges].each do |(forge_type, config)|
                     subforge = forge_type.new(*config)
                     server.forge subforge

data/lib/puppet_library/server.rb

@@ -25,7 +25,7 @@ module PuppetLibrary
     #
     # A Rack application that can be configured as follows:
     #
-    #    server = PuppetLibrary::Server.set_up do |library|
+    #    server = PuppetLibrary::Server.configure do |library|
     #        # Look for my modules locally
     #        library.forge PuppetLibrary::Forge::Directory.new("/var/lib/modules")
     #
@@ -40,12 +40,21 @@ module PuppetLibrary
                 @forge = forge
             end
 
-            def forge(forge)
-                @forge.add_forge forge
+            def forge(forge, &block)
+                if forge.is_a? Class
+                    @forge.add_forge forge.configure(&block)
+                else
+                    @forge.add_forge forge
+                end
             end
         end
 
         def self.set_up(&config_block)
+            puts "PuppetLibrary::Server::set_up deprecated: use #configure instead"
+            self.configure(&config_block)
+        end
+
+        def self.configure
             forge = Forge::Multi.new
             yield(Config.new(forge))
             Server.new(forge)

data/lib/puppet_library/util/git.rb

@@ -0,0 +1,103 @@
+# Puppet Library
+# Copyright (C) 2014 drrb
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'fileutils'
+require 'monitor'
+require 'time'
+require 'puppet_library/util/temp_dir'
+
+module PuppetLibrary::Util
+    class Git
+        DEFAULT_CACHE_TTL_SECONDS = 60
+        def initialize(source, cache_path, cache_ttl_seconds = DEFAULT_CACHE_TTL_SECONDS)
+            @source = source
+            @cache_path = File.expand_path(cache_path)
+            @cache_ttl_seconds = cache_ttl_seconds
+            @git_dir = File.join(@cache_path, ".git")
+            @mutex = Monitor.new
+        end
+
+        def tags
+            update_cache!
+            git("tag").split
+        end
+
+        def on_tag(tag)
+            update_cache!
+            PuppetLibrary::Util::TempDir.use "git" do |path|
+                git "checkout #{tag}", path
+                yield
+            end
+        end
+
+        def read_file(path, tag)
+            update_cache!
+            git "show refs/tags/#{tag}:#{path}"
+        end
+
+        def clear_cache!
+            @mutex.synchronize do
+                FileUtils.rm_rf @cache_path
+            end
+        end
+
+        def update_cache!
+            create_cache unless cache_exists?
+            update_cache if cache_stale?
+        end
+
+        private
+        def create_cache
+            @mutex.synchronize do
+                git "clone --bare #{@source} #{@git_dir}" unless cache_exists?
+                FileUtils.touch fetch_file
+            end
+        end
+
+        def update_cache
+            @mutex.synchronize do
+                git "fetch --tags"
+            end
+        end
+
+        def cache_exists?
+            File.directory? @git_dir
+        end
+
+        def cache_stale?
+            Time.now - last_fetch > @cache_ttl_seconds
+        end
+
+        def last_fetch
+            if File.exist? fetch_file
+                File.stat(fetch_file).mtime
+            else
+                Time.at(0)
+            end
+        end
+
+        def fetch_file
+            File.join(@git_dir, "FETCH_HEAD")
+        end
+
+        def git(command, work_tree = nil)
+            work_tree = @cache_path unless work_tree
+            Open3.popen3("git --git-dir=#{@git_dir} --work-tree=#{work_tree} #{command}") do |stdin, stdout, stderr, thread|
+                stdout.read
+            end
+        end
+    end
+end