puppet-library 0.6.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,15 @@
 ---
-SHA1:
-  metadata.gz: 6f9c3cf2e116027a1009cf54a51a6ba183de0580
-  data.tar.gz: ef8af80971ad97d5aca06fd4cb011c48ed4645bf
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    ZWU3ZDcxMTYwNTRlYzU4MmI0M2UxN2Y1NWJjZDBmZTdmNjEzNTQzMg==
+  data.tar.gz: !binary |-
+    Yzc2ZGY1NDNmZGRiYzI5ZjBjYzRkZDhlNmI4YmMyNmEwYThlZWM5Zg==
 SHA512:
-  metadata.gz: a695a4a49383c10faca50142781097319f9c5f1154f02ddf32b328f1ef6a71f5a7aa203936c96e77026cc9675ab6eb79297784a303a8bd156bfd2e33f0b39533
-  data.tar.gz: 9a4c3cf05cba46d86208be6958d0c657e9858e54bf50e107fa9a7aebcac2eef7df8e0079ff71a5fe565bbd8315c50806828b7f078358b673dba02e9baeb07474
+  metadata.gz: !binary |-
+    OTBlNjVhZjI2ZGVlZDUyMDllNGNhODkxODQ4NzVlYmY2MjM0YmNmMGJlNzMy
+    MTBlMjFjOTZkYjM0MjcwY2UxNjllN2U2ZDg2MTliYjEwYzc5NWQxMzRiZjUw
+    MDg1YzMwMzEzZTJmYzY2ZTBjYjllYmM3ZTc1MzkxYzliMTFmZGI=
+  data.tar.gz: !binary |-
+    MmE5MjdiYjViMDYxYTU1ZTY5MDIwYzdhZDBkZGI5MmRlYTIyYzIwMDFmZjc4
+    OTY0ZmFkNGUyODE0ZDdjNzI0YzVjNzU4MWQ5NzE0OWE2MDEyMDY1YzA1ZDcy
+    YzlhOWY3NDhmZDIzN2Q0YjJmNTc1ZGQ2YmQyZTIyNWM0ZWY0ZTg=

data/CHANGELOG.yml

@@ -42,6 +42,11 @@
       - Added module search field to web UI
 - tag: v0.6.0
   changes:
-      - Add config API for running with Passenger
+      - Added config API for running with Passenger
       - Load configuration from a config file
       - Fixed version number bug that made the app crash with some Rubygems versions
+- tag: v0.7.0
+  changes:
+      - Added new proxy forge that caches downloaded modules on disk
+      - Added `--cache-basedir` CLI option to make proxies cache downloaded modules on disk
+      - Fixed issue with forge proxying (#1)

data/README.md

@@ -66,6 +66,10 @@ Serve modules from a remote forge as a proxy
 
     $ puppet-library --proxy http://forge.puppetlabs.com
 
+Proxy a remote forge, caching downloaded modules on disk
+
+    $ puppet-library --proxy http://forge.puppetlabs.com --cache-basedir
+
 Serve modules on a specific port
 
     $ puppet-library --port 8888

data/Rakefile

@@ -54,6 +54,13 @@ def ruby_version_supports_acceptance_tests?(version = RUBY_VERSION)
     end
 end
 
+def offline?
+    Net::HTTP.get_response(URI.parse("http://forge.puppetlabs.com"))
+    return false
+rescue
+    return true
+end
+
 Coveralls::RakeTask.new
 
 desc "Run the specs"
@@ -78,6 +85,7 @@ if ruby_version_supports_integration_test?
     desc "Run the integration tests"
     RSpec::Core::RakeTask.new(:integration_test) do |rspec|
         rspec.pattern = "test/**/*_integration_test.rb"
+        rspec.rspec_opts = "--tag ~online" if offline?
     end
 else
     task :integration_test do
@@ -185,7 +193,7 @@ task "register-release", [:version] do |task, args|
 end
 
 def upload_release_notes(version)
-    puts "Registering release notes for #{PuppetLibrary::VERSION}"
+    puts "Registering release notes for #{version}"
     github = Github.new
 
     unless version =~ /\d+\.\d+\.\d+/

data/TODO.yml

@@ -1,13 +1,10 @@
 features:
     - disk cache:
-        - Allow proxy to cache downloaded modules/API calls on disk
+        - use disk cache to respond to queries if remote API calls fail
         - |
             Accept --cache-dir option to specify a parent cache directory (people
             will need to embed the server or use a config file for more control)
-    - Validate repo dirs (make sure they're readable)
-    - Allow failover between module repositories
     - Web UI: asynchronous (index page loads too slowly)
-    - Look in /etc/puppet-library.yml for config by default
     - Named subforges:
         - per-subforge queries
         - upload modules (web form and API)
@@ -18,3 +15,5 @@ features:
 
 dubious_features:
     - Make proxy cache TTL configurable
+    - Allow failover between module repositories
+    - Look in /etc/puppet-library.yml for config by default

data/lib/puppet_library/forge.rb

@@ -20,6 +20,7 @@ module PuppetLibrary::Forge
     end
 
     require 'puppet_library/forge/abstract'
+    require 'puppet_library/forge/cache'
     require 'puppet_library/forge/directory'
     require 'puppet_library/forge/multi'
     require 'puppet_library/forge/proxy'

data/lib/puppet_library/forge/cache.rb

@@ -0,0 +1,27 @@
+# -*- encoding: utf-8 -*-
+# Puppet Library
+# Copyright (C) 2014 drrb
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'puppet_library/forge/proxy'
+require 'puppet_library/http/http_client'
+
+module PuppetLibrary::Forge
+    class Cache < Proxy
+        def initialize(url, cache_dir, http_client = PuppetLibrary::Http::HttpClient.new)
+            super(url, PuppetLibrary::Http::Cache::InMemory.new, PuppetLibrary::Http::Cache::Disk.new(cache_dir), http_client)
+        end
+    end
+end

data/lib/puppet_library/forge/directory.rb

@@ -23,6 +23,8 @@ module PuppetLibrary::Forge
     class Directory < PuppetLibrary::Forge::Abstract
         def initialize(module_dir)
             super(self)
+            raise "Module directory '#{module_dir}' doesn't exist" unless File.directory? module_dir
+            raise "Module directory '#{module_dir}' isn't readable" unless File.executable? module_dir
             @module_dir = module_dir
         end
 

data/lib/puppet_library/forge/proxy.rb

@@ -18,7 +18,6 @@
 require 'puppet_library/http/http_client'
 require 'puppet_library/http/cache/in_memory'
 require 'puppet_library/http/cache/noop'
-require 'puppet_library/http/url'
 
 module PuppetLibrary::Forge
     class Proxy
@@ -26,7 +25,7 @@ module PuppetLibrary::Forge
                        query_cache = PuppetLibrary::Http::Cache::InMemory.new,
                        download_cache = PuppetLibrary::Http::Cache::NoOp.new,
                        http_client = PuppetLibrary::Http::HttpClient.new)
-            @url = PuppetLibrary::Http::Url.normalize(url)
+            @url = url
             @http_client = http_client
             @query_cache = query_cache
             @download_cache = download_cache
@@ -42,7 +41,7 @@ module PuppetLibrary::Forge
             begin
                 version_info = get_module_version(author, name, version)
                 raise ModuleNotFound if version_info.nil?
-                download_file(version_info["file"])
+                download_module(author, name, version, version_info["file"])
             rescue OpenURI::HTTPError
                 raise ModuleNotFound
             end
@@ -59,7 +58,9 @@ module PuppetLibrary::Forge
 
         def get_module_metadata_with_dependencies(author, name, version)
             begin
-                look_up_releases(author, name, version)
+                look_up_releases(author, name, version) do |full_name, release_info|
+                    release_info["file"] = module_path_for(full_name, release_info["version"])
+                end
             rescue OpenURI::HTTPError
                 raise ModuleNotFound
             end
@@ -78,15 +79,31 @@ module PuppetLibrary::Forge
             versions["#{author}/#{name}"]
         end
 
-        def look_up_releases(author, name, version = nil)
+        def look_up_releases(author, name, version = nil, &block)
             version_query = version ? "&version=#{version}" : ""
             url = "/api/v1/releases.json?module=#{author}/#{name}#{version_query}"
-            response = get(url)
-            JSON.parse(response)
+            response_text = get(url)
+            response = JSON.parse(response_text)
+            process_releases_response(response, &block)
         end
 
-        def download_file(file)
-            @download_cache.get(file) do
+        def process_releases_response(response)
+            if block_given?
+                response.each do |full_name, release_infos|
+                    release_infos.each do |release_info|
+                        yield(full_name, release_info)
+                    end
+                end
+            end
+            return response
+        end
+
+        def module_path_for(full_name, version)
+            "/modules/#{full_name.sub("/", "-")}-#{version}.tar.gz"
+        end
+
+        def download_module(author, name, version, file)
+            @download_cache.get("#{author}-#{name}-#{version}.tar.gz") do
                 @http_client.download(url(file))
             end
         end

data/lib/puppet_library/http/cache.rb

@@ -16,6 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 module PuppetLibrary::Http::Cache
+    require 'puppet_library/http/cache/disk'
     require 'puppet_library/http/cache/in_memory'
     require 'puppet_library/http/cache/noop'
 end

data/lib/puppet_library/http/cache/disk.rb

@@ -0,0 +1,53 @@
+# -*- encoding: utf-8 -*-
+# Puppet Library
+# Copyright (C) 2014 drrb
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+module PuppetLibrary::Http::Cache
+    class Disk
+        def initialize(directory)
+            @directory = directory
+        end
+
+        def get(path)
+            unless include? path
+                buffer = yield
+                save(path, buffer)
+            end
+            retrieve(path)
+        end
+
+        private
+        def include?(path)
+            File.exist? entry_path(path)
+        end
+
+        def save(path, buffer)
+            file_path = entry_path(path)
+            FileUtils.mkdir_p File.dirname(file_path)
+            File.open(file_path, "w") do |file|
+                file.write buffer.read
+            end
+        end
+
+        def retrieve(path)
+            File.open(entry_path(path))
+        end
+
+        def entry_path(path)
+            File.join(@directory, path)
+        end
+    end
+end

data/lib/puppet_library/http/url.rb

@@ -15,10 +15,19 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+require 'uri'
+
 module PuppetLibrary::Http
     class Url
         def self.normalize(uri_string)
-            unless uri_string =~ /^https?/
+            begin
+                url = URI.parse(uri_string)
+            rescue URI::InvalidURIError => e
+                raise PuppetLibrary::ExpectedError, "Invalid URL '#{uri_string}': #{e.message}"
+            end
+            if url.scheme
+                raise PuppetLibrary::ExpectedError, "Invalid URL '#{uri_string}': unsupported protocol '#{url.scheme}'" unless url.scheme =~ /^https?$/
+            else
                 uri_string = "http://#{uri_string}"
             end
             uri_string.sub /\/$/, ""

data/lib/puppet_library/puppet_library.rb

@@ -42,6 +42,7 @@ module PuppetLibrary
             options = {}
             option_parser = OptionParser.new do |opts|
                 opts.banner = "Usage: #{File.basename $0} [options]"
+                opts.version = VERSION
 
                 opts.on("-c", "--config-file FILE", "Config file to read config values from") do |config_file|
                     options[:config_file] = config_file
@@ -73,7 +74,11 @@ module PuppetLibrary
                     options[:forges] << [Forge::Directory, module_dir]
                 end
                 opts.on("-x", "--proxy URL", "Remote forge to proxy (can be specified multiple times)") do |url|
-                    options[:forges] << [Forge::Proxy, url]
+                    options[:forges] << [Forge::Proxy, sanitize_url(url)]
+                end
+
+                opts.on("--cache-basedir DIR", "Cache all proxies' downloaded modules under this directory") do |cache_basedir|
+                    options[:cache_basedir] = cache_basedir
                 end
             end
             begin
@@ -87,14 +92,15 @@ module PuppetLibrary
 
         def build_server(options)
             if options[:config_file]
-                load_config(options)
+                load_config!(options)
             end
 
-            load_defaults(options)
+            load_defaults!(options)
+            process_options!(options)
 
             Server.set_up do |server|
                 options[:forges].each do |(forge_type, config)|
-                    subforge = forge_type.new(config)
+                    subforge = forge_type.new(*config)
                     server.forge subforge
                 end
             end
@@ -126,7 +132,7 @@ module PuppetLibrary
             )
         end
 
-        def load_config(options)
+        def load_config!(options)
             config = read_yaml_file(options[:config_file])
             options[:daemonize] = config["daemonize"]
             options[:port] = config["port"]
@@ -140,7 +146,7 @@ module PuppetLibrary
             options[:forges] = configured_forges + options[:forges]
         end
 
-        def load_defaults(options)
+        def load_defaults!(options)
             options[:daemonize] ||= false
             options[:pidfile] ||= nil
             if options[:forges].empty?
@@ -148,9 +154,34 @@ module PuppetLibrary
             end
         end
 
+        def process_options!(options)
+            options[:forges].map! do |(forge_type, config)|
+                if forge_type == Forge::Proxy
+                end
+                if forge_type == Forge::Proxy && options[:cache_basedir]
+                    cache_dir = File.join(options[:cache_basedir], url_hostname(config))
+                    [ Forge::Cache, [ config, sanitize_path(cache_dir) ] ]
+                else
+                    [ forge_type, config ]
+                end
+            end
+        end
+
         def read_yaml_file(path)
             YAML.load_file(File.expand_path(path)) || {}
         end
+
+        def sanitize_url(url)
+            Http::Url.normalize(url)
+        end
+
+        def sanitize_path(path)
+            File.expand_path path
+        end
+
+        def url_hostname(url)
+            URI.parse(Http::Url.normalize(url)).host
+        end
     end
 
     class ExpectedError < StandardError

data/lib/puppet_library/server.rb

@@ -68,12 +68,15 @@ module PuppetLibrary
             begin
                 @forge.get_module_metadata(author, module_name).to_json
             rescue Forge::ModuleNotFound
-                status 410
-                {"error" => "Could not find module \"#{module_name}\""}.to_json
+                halt 410, {"error" => "Could not find module \"#{module_name}\""}.to_json
             end
         end
 
         get "/api/v1/releases.json" do
+            unless params[:module]
+                halt 400, {"error" => "The number of version constraints in the query does not match the number of module names"}.to_json
+            end
+
             author, module_name = params[:module].split "/"
             version = params[:version]
             begin
@@ -92,11 +95,12 @@ module PuppetLibrary
             content_type "application/octet-stream"
 
             begin
-                @forge.get_module_buffer(author, name, version).tap do
+                buffer = @forge.get_module_buffer(author, name, version).tap do
                     attachment "#{author}-#{name}-#{version}.tar.gz"
                 end
+                [ 200, { "Content-Length" => buffer.size.to_s }, buffer ]
             rescue Forge::ModuleNotFound
-                status 404
+                halt 404
             end
         end
 

data/lib/puppet_library/version.rb

@@ -16,5 +16,5 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 module PuppetLibrary
-    VERSION = "0.6.0"
+    VERSION = "0.7.0"
 end

data/spec/forge/cache_spec.rb

@@ -0,0 +1,67 @@
+# -*- encoding: utf-8 -*-
+# Puppet Library
+# Copyright (C) 2014 drrb
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'spec_helper'
+
+module PuppetLibrary::Forge
+    describe Cache do
+        let(:cache_dir) { Tempdir.create("module-cache") }
+        let(:http_client) { double(PuppetLibrary::Http::HttpClient) }
+        let(:forge) { Cache.new("http://forge.example.com", cache_dir, http_client) }
+
+        it "is a proxy" do
+            expect(forge).to be_a Proxy
+        end
+
+        describe "#get_module_buffer" do
+            before do
+                allow(http_client).to receive(:get).
+                    with("http://forge.example.com/api/v1/releases.json?module=puppetlabs/apache").
+                    and_return('{"puppetlabs/apache":[{"version":"1.0.0","file":"/puppetlabs/apache.tgz","dependencies":[["puppetlabs/concat",">= 1.0.0"],["puppetlabs/stdlib","~> 2.0.0"]]},{"version":"2.0.0","dependencies":[]}]}')
+            end
+
+            context "the first time it's called" do
+                it "downloads the module to disk and serves it" do
+                    file_buffer = StringIO.new("apache module")
+                    expect(http_client).to receive(:download).
+                        with("http://forge.example.com/puppetlabs/apache.tgz").
+                        and_return(file_buffer)
+
+                    buffer = forge.get_module_buffer("puppetlabs", "apache", "1.0.0")
+
+                    downloaded_file = File.join(cache_dir, "puppetlabs-apache-1.0.0.tar.gz")
+                    expect(File.read downloaded_file).to eq "apache module"
+                    expect(buffer.read).to eq "apache module"
+                end
+            end
+
+            context "the second time it's called" do
+                it "serves the cached module from the disk" do
+                    cached_module = File.join(cache_dir, "puppetlabs-apache-1.0.0.tar.gz")
+                    File.open(cached_module, "w") do |file|
+                        file.write "apache module"
+                    end
+                    expect(http_client).not_to receive(:download)
+
+                    buffer = forge.get_module_buffer("puppetlabs", "apache", "1.0.0")
+
+                    expect(buffer.read).to eq "apache module"
+                end
+            end
+        end
+    end
+end

data/spec/forge/directory_spec.rb

@@ -19,18 +19,38 @@ require 'spec_helper'
 module PuppetLibrary::Forge
     describe Directory do
         include ModuleSpecHelper
-        include FileUtils
 
-        let(:module_dir) { Tempfile.new("module_dir").path }
+        let(:module_dir) { Tempdir.create("module_dir") }
         let(:module_repo) { Directory.new(module_dir) }
 
-        before do
+        after do
             rm_rf module_dir
-            mkdir_p module_dir
         end
 
-        after do
-            rm_rf module_dir
+        describe "#initialize" do
+            context "when the module directory doesn't exist" do
+                before do
+                    rm_rf module_dir
+                end
+
+                it "raises an error" do
+                    expect {
+                        Directory.new(module_dir)
+                    }.to raise_error /Module directory .* doesn't exist/
+                end
+            end
+
+            context "when the module directory isn't readable" do
+                before do
+                    chmod 0400, module_dir
+                end
+
+                it "raises an error" do
+                    expect {
+                        Directory.new(module_dir)
+                    }.to raise_error /Module directory .* isn't readable/
+                end
+            end
         end
 
         describe "#get_module" do

data/spec/forge/proxy_spec.rb

@@ -24,24 +24,6 @@ module PuppetLibrary::Forge
         let(:download_cache) { PuppetLibrary::Http::Cache::InMemory.new }
         let(:forge) { Proxy.new("http://puppetforge.example.com", query_cache, download_cache, http_client) }
 
-        describe "url" do
-            it "defaults to HTTP, when protocol not specified" do
-                forge = Proxy.new("forge.puppetlabs.com", query_cache, download_cache, http_client) 
-
-                expect(http_client).to receive(:get).with("http:\/\/forge.puppetlabs.com/puppetlabs/apache.json").and_return('{"puppetlabs/apache":[]}')
-
-                forge.get_module_metadata("puppetlabs", "apache")
-            end
-
-            it "copes with a trailing slash" do
-                forge = Proxy.new("forge.puppetlabs.com/", query_cache, download_cache, http_client) 
-
-                expect(http_client).to receive(:get).with("http:\/\/forge.puppetlabs.com/puppetlabs/apache.json").and_return('{"puppetlabs/apache":[]}')
-
-                forge.get_module_metadata("puppetlabs", "apache")
-            end
-        end
-
         describe "#search_modules" do
             it "forwards the request directly" do
                 search_results = '["a","b","c"]'
@@ -180,15 +162,16 @@ module PuppetLibrary::Forge
             end
 
             context "when the module is found" do
-                it "forwards the request directly" do
-                    response = '{"puppetlabs/apache":[{"version":"1.0.0","dependencies":[["puppetlabs/concat",">= 1.0.0"],["puppetlabs/stdlib","~> 2.0.0"]]},{"version":"2.0.0","dependencies":[]}]}'
+                it "forwards the request directly, but adjusts the module download locations" do
+                    original_response = '{"puppetlabs/apache":[{"version":"1.0.0","file":"/puppetlabs/apache/1.0.0.tar.gz","dependencies":[["puppetlabs/concat",">= 1.0.0"],["puppetlabs/stdlib","~> 2.0.0"]]},{"version":"2.0.0","file":"/puppetlabs/apache/2.0.0.tar.gz","dependencies":[]}]}'
+                    doctored_response = '{"puppetlabs/apache":[{"version":"1.0.0","file":"/modules/puppetlabs-apache-1.0.0.tar.gz","dependencies":[["puppetlabs/concat",">= 1.0.0"],["puppetlabs/stdlib","~> 2.0.0"]]},{"version":"2.0.0","file":"/modules/puppetlabs-apache-2.0.0.tar.gz","dependencies":[]}]}'
                     expect(http_client).to receive(:get).
                         with("http://puppetforge.example.com/api/v1/releases.json?module=puppetlabs/apache&version=1.0.0").
-                        and_return(response)
+                        and_return(original_response)
 
                     result = forge.get_module_metadata_with_dependencies("puppetlabs", "apache", "1.0.0")
 
-                    expect(result).to eq JSON.parse(response)
+                    expect(result).to eq JSON.parse(doctored_response)
                 end
 
                 it "caches the result" do

data/spec/http/cache/disk_spec.rb

@@ -0,0 +1,57 @@
+# -*- encoding: utf-8 -*-
+# Puppet Library
+# Copyright (C) 2014 drrb
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'spec_helper'
+
+module PuppetLibrary::Http::Cache
+    describe Disk do
+        let(:cache_dir) { Tempdir.new("modulecache").path }
+        let(:cache) { Disk.new(cache_dir) }
+
+        after do
+            rm_rf cache_dir
+        end
+
+        describe "#get" do
+            context "the first time it's called" do
+                it "returns the value from the block" do
+                    buffer = StringIO.new "hello"
+
+                    result = cache.get("my-file") { buffer }
+
+                    expect(result.read).to eq "hello"
+                end
+
+                it "saves the content to disk" do
+                    buffer = StringIO.new "hello"
+
+                    result = cache.get("dir/my-file") { buffer }
+
+                    path = File.join(cache_dir, "dir", "my-file")
+                    expect(File.read(path)).to eq "hello"
+                end
+            end
+
+            context "the second time it's called" do
+                it "reads the config from the disk" do
+                    cache.get("my-file") { StringIO.new "hello" }
+                    result = cache.get("my-file") { raise "shouldn't be called" }
+                end
+            end
+        end
+    end
+end

data/spec/http/cache/in_memory_spec.rb

@@ -34,7 +34,7 @@ module PuppetLibrary::Http::Cache
             context "the second time it's called" do
                 it "returns the cached value" do
                     cache.get("greeting") { "hello" }
-                    greeting = cache.get("greeting") { "hi" }
+                    greeting = cache.get("greeting") { raise "shouldn't be called" }
 
                     expect(greeting).to eq "hello"
                 end

data/spec/http/url_spec.rb

@@ -34,6 +34,22 @@ module PuppetLibrary::Http
                 result = Url.normalize "http://localhost/"
                 expect(result).to eq "http://localhost"
             end
+
+            context "when the URL has an invalid scheme" do
+                it "raises an error" do
+                    expect {
+                        Url.normalize "ftp://example.com"
+                    }.to raise_exception /Invalid URL/
+                end
+            end
+
+            context "when the URL is invalid" do
+                it "raises an error" do
+                    expect {
+                        Url.normalize "a b"
+                    }.to raise_exception /Invalid URL/
+                end
+            end
         end
     end
 end

data/spec/puppet_library_spec.rb

@@ -140,6 +140,39 @@ module PuppetLibrary
 
                     library.go(["--proxy", "http://forge1.example.com", "--proxy", "http://forge2.example.com"])
                 end
+
+                context "when no protocol is specified" do
+                    it "defaults to HTTP" do
+                        expect(Forge::Proxy).to receive(:new).with("http://forge.example.com")
+
+                        library.go(["--proxy", "forge.example.com"])
+                    end
+                end
+
+                context "when the URL contains a trailing slash" do
+                    it "removes the slash" do
+                        expect(Forge::Proxy).to receive(:new).with("http://forge.example.com")
+
+                        library.go(["--proxy", "http://forge.example.com/"])
+                    end
+                end
+            end
+
+            context "when using --cache-basedir option" do
+                it "uses the specified directory to hold cache directories for all proxies" do
+                    proxy1 = double('proxy')
+                    expect(Forge::Cache).to receive(:new).with("http://forge1.example.com", "/var/modules/forge1.example.com").and_return(proxy1)
+                    expect(forge).to receive(:add_forge).with(proxy1)
+                    expect(Rack::Server).to receive(:start).with(default_options)
+
+                    library.go(["--proxy", "http://forge1.example.com", "--cache-basedir", "/var/modules"])
+                end
+
+                it "expands the path specified" do
+                    expect(Forge::Cache).to receive(:new).with("http://forge1.example.com", "/var/modules/forge1.example.com")
+
+                    library.go(["--proxy", "http://forge1.example.com", "--cache-basedir", "/var/../var/modules"])
+                end
             end
 
             context "when using --module-dir option" do
@@ -172,8 +205,8 @@ module PuppetLibrary
                 expect(log).to receive(:puts).with(/Host: default/)
                 expect(log).to receive(:puts).with(/Server: default/)
                 expect(log).to receive(:puts).with(/Forges:/)
-                expect(log).to receive(:puts).with(/- PuppetLibrary::Forge::Directory: \.\/modules/)
-                library.go(["--module-dir", "./modules"])
+                expect(log).to receive(:puts).with(/- PuppetLibrary::Forge::Proxy: http:\/\/forge.example.com/)
+                library.go(["--proxy", "http://forge.example.com"])
             end
         end
     end

data/spec/server_spec.rb

@@ -196,6 +196,15 @@ module PuppetLibrary
         end
 
         describe "GET /api/v1/releases.json" do
+            context "when module parameter not specified" do
+                it "returns an error" do
+                    get "/api/v1/releases.json"
+
+                    expect(last_response.body).to eq('{"error":"The number of version constraints in the query does not match the number of module names"}')
+                    expect(last_response.status).to eq(400)
+                end
+            end
+
             it "gets metadata for module and dependencies" do
                 metadata = {
                     "puppetlabs/apache" => [

data/spec/spec_helper.rb

@@ -21,3 +21,21 @@ require 'module_spec_helper'
 require 'pry'
 require 'rack'
 require 'tempfile'
+require 'fileutils'
+
+include FileUtils
+
+class Tempdir
+    attr_reader :path
+
+    def self.create(name)
+        Tempdir.new(name).path
+    end
+
+    def initialize(name)
+        file = Tempfile.new(name)
+        @path = file.path
+        file.unlink
+        FileUtils.mkdir @path
+    end
+end

data/test/librarian_puppet_integration_test.rb

@@ -18,13 +18,32 @@
 require 'spec_helper'
 require 'open-uri'
 
+
+RSpec::Matchers.define :be_cached do
+    match do |mod_file|
+        ! Dir[File.join(cache_dir, mod_file)].empty?
+    end
+end
+
+RSpec::Matchers.define :be_installed do
+    match do |mod_name|
+        File.directory?(File.join(project_dir, "modules",  mod_name))
+    end
+end
+
+def write_puppetfile(content)
+    File.open("#{project_dir}/Puppetfile", "w") do |puppetfile|
+        puppetfile.puts content
+    end
+end
+
 module PuppetLibrary
-    describe "librarian-puppet integration test" do
+    describe "offline proxy" do
         include ModuleSpecHelper
-        include FileUtils
 
-        let(:module_dir) { Tempfile.new("module_dir").path }
-        let(:project_dir) { Tempfile.new("project_dir").path }
+        let(:module_dir) { Tempdir.create("module_dir") }
+        let(:project_dir) { Tempdir.create("project_dir") }
+        let(:cache_dir) { Tempdir.create("cache_dir") }
         let(:start_dir) { pwd }
         let(:disk_forge) { Forge::Directory.new(module_dir) }
         let(:disk_server) do
@@ -45,7 +64,7 @@ module PuppetLibrary
                 disk_rack_server.start
             end
         end
-        let(:proxy_forge) { Forge::Proxy.new("http://localhost:9000") }
+        let(:proxy_forge) { Forge::Cache.new("http://localhost:9000", cache_dir) }
         let(:proxy_server) do
             Server.set_up do |server|
                 server.forge proxy_forge
@@ -66,10 +85,6 @@ module PuppetLibrary
         end
 
         before do
-            rm_rf module_dir
-            rm_rf project_dir
-            mkdir_p module_dir
-            mkdir_p project_dir
             # Initialize to catch wiring errors
             disk_rack_server
             proxy_rack_server
@@ -84,15 +99,10 @@ module PuppetLibrary
         after do
             rm_rf module_dir
             rm_rf project_dir
+            rm_rf cache_dir
             cd start_dir
         end
 
-        def write_puppetfile(content)
-            File.open("#{project_dir}/Puppetfile", "w") do |puppetfile|
-                puppetfile.puts content
-            end
-        end
-
         it "queries, downloads and searches through a proxy to a directory" do
             add_module("puppetlabs", "apache", "1.0.0") do |metadata|
                 metadata["dependencies"] << { "name" => "puppetlabs/concat", "version_requirement" => ">= 2.0.0" }
@@ -108,10 +118,13 @@ module PuppetLibrary
 
             # Install modules through the proxy
             system "librarian-puppet install" or fail "call to puppet-library failed"
-            expect(File.directory? "modules").to be true
-            expect(File.directory? "modules/apache").to be true
-            expect(File.directory? "modules/concat").to be true
-            expect(File.directory? "modules/stdlib").to be true
+            expect("apache").to be_installed
+            expect("concat").to be_installed
+            expect("stdlib").to be_installed
+
+            expect("puppetlabs-apache-1.0.0.tar.gz").to be_cached
+            expect("puppetlabs-concat-2.0.0.tar.gz").to be_cached
+            expect("puppetlabs-stdlib-3.0.0.tar.gz").to be_cached
 
             # Search through the proxy
             search_results = JSON.parse(open("http://localhost:9001/modules.json").read)
@@ -123,4 +136,69 @@ module PuppetLibrary
             expect(found_modules["puppetlabs/stdlib"]).to eq "3.0.0"
         end
     end
+
+    describe "online proxy", :online => true do
+        include ModuleSpecHelper
+
+        let(:project_dir) { Tempdir.create("project_dir") }
+        let(:module_dir) { Tempdir.create("module_dir") }
+        let(:cache_dir) { Tempdir.create("cache_dir") }
+        let(:start_dir) { pwd }
+        let(:proxy_server) do
+            Server.set_up do |server|
+                server.forge Forge::Directory.new(module_dir)
+                server.forge Forge::Cache.new("http://forge.puppetlabs.com", cache_dir)
+            end
+        end
+        let(:proxy_rack_server) do
+            Rack::Server.new(
+                :app => proxy_server,
+                :Host => "localhost",
+                :Port => 9002,
+                :server => "webrick"
+            )
+        end
+        let(:proxy_server_runner) do
+            Thread.new do
+                proxy_rack_server.start
+            end
+        end
+
+        before do
+            # Initialize to catch wiring errors
+            proxy_rack_server
+
+            # Start the server
+            proxy_server_runner
+            start_dir
+            cd project_dir
+        end
+
+        after do
+            rm_rf module_dir
+            rm_rf project_dir
+            rm_rf cache_dir
+            cd start_dir
+        end
+
+        it "queries, downloads and searches through a proxy to a directory" do
+            add_module("drrb", "tomcat", "1.0.0") do |metadata|
+                metadata["dependencies"] << { "name" => "puppetlabs/apache", "version_requirement" => "0.9.0" }
+            end
+
+            write_puppetfile <<-EOF
+                forge 'http://localhost:9002'
+                mod 'drrb/tomcat'
+            EOF
+
+            # Install modules through the proxy
+            system "librarian-puppet install" or fail "call to puppet-library failed"
+            expect("tomcat").to be_installed
+            expect("apache").to be_installed
+            expect("concat").to be_installed
+            expect("stdlib").to be_installed
+
+            expect("puppetlabs-apache-*.tar.gz").to be_cached
+        end
+    end
 end

metadata

@@ -1,55 +1,55 @@
 --- !ruby/object:Gem::Specification
 name: puppet-library
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.0
 platform: ruby
 authors:
 - drrb
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-03 00:00:00.000000000 Z
+date: 2014-02-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sinatra
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: haml
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
@@ -70,42 +70,42 @@ dependencies:
   name: coveralls
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: gitsu
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: librarian-puppet
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
@@ -126,14 +126,14 @@ dependencies:
   name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
@@ -154,140 +154,140 @@ dependencies:
   name: rack-test
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: guard
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: guard-rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: terminal-notifier-guard
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: capybara
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: cucumber
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 description: A Puppet module server
@@ -323,12 +323,14 @@ files:
 - lib/puppet_library/app/views/module_not_found.haml
 - lib/puppet_library/forge.rb
 - lib/puppet_library/forge/abstract.rb
+- lib/puppet_library/forge/cache.rb
 - lib/puppet_library/forge/directory.rb
 - lib/puppet_library/forge/multi.rb
 - lib/puppet_library/forge/proxy.rb
 - lib/puppet_library/forge/search_result.rb
 - lib/puppet_library/http.rb
 - lib/puppet_library/http/cache.rb
+- lib/puppet_library/http/cache/disk.rb
 - lib/puppet_library/http/cache/in_memory.rb
 - lib/puppet_library/http/cache/noop.rb
 - lib/puppet_library/http/http_client.rb
@@ -339,9 +341,11 @@ files:
 - lib/puppet_library/version.rb
 - puppet-library.gemspec
 - spec/forge/abstract_spec.rb
+- spec/forge/cache_spec.rb
 - spec/forge/directory_spec.rb
 - spec/forge/multi_spec.rb
 - spec/forge/proxy_spec.rb
+- spec/http/cache/disk_spec.rb
 - spec/http/cache/in_memory_spec.rb
 - spec/http/cache/noop_spec.rb
 - spec/http/http_client_spec.rb
@@ -362,17 +366,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 2.2.1
 signing_key: 
 specification_version: 4
 summary: Puppet Library is a private Puppet module server that's compatible with librarian-puppet.
@@ -383,9 +387,11 @@ test_files:
 - features/step_definitions/sinatra_steps.rb
 - features/support/env.rb
 - spec/forge/abstract_spec.rb
+- spec/forge/cache_spec.rb
 - spec/forge/directory_spec.rb
 - spec/forge/multi_spec.rb
 - spec/forge/proxy_spec.rb
+- spec/http/cache/disk_spec.rb
 - spec/http/cache/in_memory_spec.rb
 - spec/http/cache/noop_spec.rb
 - spec/http/http_client_spec.rb