puppet-library 0.0.2 → 0.1.0

data/lib/puppet_library/module_repo/proxy.rb

@@ -0,0 +1,87 @@
+# -*- encoding: utf-8 -*-
+# Puppet Library
+# Copyright (C) 2014 drrb
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'puppet_library/http/http_client'
+require 'puppet_library/http/cache'
+require 'puppet_library/http/url'
+
+module PuppetLibrary::ModuleRepo
+    class Proxy
+        def initialize(url, http_client = PuppetLibrary::Http::HttpClient.new, query_cache = PuppetLibrary::Http::Cache.new)
+            @url = PuppetLibrary::Http::Url.normalize(url)
+            @http_client = http_client
+            @query_cache = query_cache
+        end
+
+        def get_module(author, name, version)
+            version_info = get_module_version(author, name, version)
+            if version_info.nil?
+                nil
+            else
+                download_file(version_info["file"])
+            end
+        end
+
+        def get_metadata(author, name)
+            module_versions = get_module_versions(author, name)
+            module_versions.map do |version_info|
+                {
+                    "name" => "#{author}-#{name}",
+                    "author" => author,
+                    "version" => version_info["version"],
+                    "dependencies" => version_info["dependencies"].map do |(dep_name, dep_spec)|
+                        { "name" => dep_name, "version_requirement" => dep_spec }
+                    end
+                }
+            end
+        rescue OpenURI::HTTPError => http_error
+            return []
+        end
+
+        private
+        def get_module_version(author, name, version)
+            module_versions = get_module_versions(author, name)
+            module_versions.find do |version_info|
+                version_info["version"] == version
+            end
+        end
+
+        def get_module_versions(author, name)
+            versions = look_up_releases(author, name)
+            versions["#{author}/#{name}"]
+        end
+
+        def look_up_releases(author, name)
+            response = get("/api/v1/releases.json?module=#{author}/#{name}")
+            JSON.parse(response)
+        end
+
+        def download_file(file)
+            @http_client.download(url(file))
+        end
+
+        def get(relative_url)
+            @query_cache.get(relative_url) do
+                @http_client.get(url(relative_url))
+            end
+        end
+
+        def url(relative_url)
+            @url + relative_url
+        end
+    end
+end

data/lib/puppet_library/puppet_library.rb

@@ -1,6 +1,6 @@
 # -*- encoding: utf-8 -*-
 # Puppet Library
-# Copyright (C) 2013 drrb
+# Copyright (C) 2014 drrb
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -16,6 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 require 'rack'
+require 'puppet_library/forge'
 require 'puppet_library/module_repo/directory'
 require 'puppet_library/module_repo/multi'
 
@@ -41,19 +42,24 @@ module PuppetLibrary
             option_parser = OptionParser.new do |opts|
                 opts.banner = "Usage: #{File.basename $0} [options]"
 
-                options[:port] = "9292"
-                opts.on("-p", "--port [PORT]", "Port to listen on (defaults to #{options[:port]})") do |port|
+                opts.on("-p", "--port [PORT]", "Port to listen on (defaults to whatever Rack wants to use)") do |port|
                     options[:port] = port
                 end
 
-                options[:hostname] = "0.0.0.0"
-                opts.on("-b", "--bind-host [HOSTNAME]", "Host name to bind to (defaults to #{options[:hostname]})") do |hostname|
+                opts.on("-s", "--server [SERVER]", "Server to use (defaults to whatever Rack wants to use)") do |server|
+                    options[:server] = server
+                end
+
+                opts.on("-b", "--bind-host [HOSTNAME]", "Host name to bind to (defaults to whatever Rack wants to use)") do |hostname|
                     options[:hostname] = hostname
                 end
 
-                options[:module_dirs] = []
+                options[:repositories] = []
                 opts.on("-m", "--module-dir [DIR]", "Directory containing the modules (can be specified multiple times. Defaults to './modules')") do |module_dir|
-                    options[:module_dirs] << module_dir
+                    options[:repositories] << [ModuleRepo::Directory, module_dir]
+                end
+                opts.on("-x", "--proxy [URL]", "Remote forge to proxy (can be specified multiple times)") do |url|
+                    options[:repositories] << [ModuleRepo::Proxy, url]
                 end
             end
             begin
@@ -62,29 +68,32 @@ module PuppetLibrary
                 raise ExpectedError, parse_error.message + "\n" + option_parser.help
             end
 
-            if options[:module_dirs].empty?
-                options[:module_dirs] << "./modules"
-            end
-
             return options
         end
 
         def build_server(options)
-            module_repo = ModuleRepo::Multi.new
-            options[:module_dirs].each do |dir|
-                subrepo = ModuleRepo::Directory.new(dir)
-                module_repo.add_repo(subrepo)
+            if options[:repositories].empty?
+                options[:repositories] << [ ModuleRepo::Proxy, "http://forge.puppetlabs.com" ]
+            end
+
+            Server.set_up do |server|
+                options[:repositories].each do |(repo_type, config)|
+                    subrepo = repo_type.new(config)
+                    server.module_repo subrepo
+                end
             end
-            Server.new(module_repo)
         end
 
         def announce_server_start(options)
+            options = options.clone
+            options.default = "default"
             @log.puts "Starting Puppet Library server:"
             @log.puts " |- Port: #{options[:port]}"
             @log.puts " |- Host: #{options[:hostname]}"
-            @log.puts " `- Module dirs:"
-            options[:module_dirs].each do |dir|
-                @log.puts "    - #{dir}"
+            @log.puts " |- Server: #{options[:server]}"
+            @log.puts " `- Repositories:"
+            options[:repositories].each do |(repo_type, config)|
+                @log.puts "    - #{repo_type}: #{config}"
             end
         end
 
@@ -92,7 +101,8 @@ module PuppetLibrary
             Rack::Server.start(
                 :app => server,
                 :Host => options[:hostname],
-                :Port => options[:port]
+                :Port => options[:port],
+                :server => options[:server]
             )
         end
     end

data/lib/puppet_library/server.rb

@@ -1,6 +1,6 @@
 # -*- encoding: utf-8 -*-
 # Puppet Library
-# Copyright (C) 2013 drrb
+# Copyright (C) 2014 drrb
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -17,35 +17,19 @@
 
 require 'sinatra/base'
 
+require 'puppet_library/forge'
 require 'puppet_library/module_metadata'
 require 'puppet_library/module_repo/multi'
 
-class Array
-    def deep_merge
-        inject({}) do |merged, map|
-            merged.deep_merge(map)
-        end
-    end
-end
-
-class Hash
-    def deep_merge(other)
-        merge(other) do |key, old_val, new_val|
-            if old_val.instance_of? Array
-                old_val + new_val
-            else
-                new_val
-            end
-        end
-    end
-end
-
 module PuppetLibrary
     class Server < Sinatra::Base
+        def self.set_up(&config_block)
+            Server.new(Forge.configure(&config_block))
+        end
 
-        def initialize(module_repo = ModuleRepo::Multi.new)
+        def initialize(forge)
             super(nil)
-            @repo = settings.respond_to?(:module_repo) ? settings.module_repo : module_repo
+            @forge = forge
         end
 
         configure do
@@ -57,7 +41,7 @@ module PuppetLibrary
             module_name = params[:module]
 
             begin
-                get_module_metadata(author, module_name).to_json
+                @forge.get_module_metadata(author, module_name).to_json
             rescue ModuleNotFound
                 status 410
                 {"error" => "Could not find module \"#{module_name}\""}.to_json
@@ -67,7 +51,7 @@ module PuppetLibrary
         get "/api/v1/releases.json" do
             author, module_name = params[:module].split "/"
             begin
-                get_module_metadata_with_dependencies(author, module_name).to_json
+                @forge.get_module_metadata_with_dependencies(author, module_name).to_json
             rescue ModuleNotFound
                 status 410
                 {"error" => "Module #{author}/#{module_name} not found"}.to_json
@@ -82,53 +66,12 @@ module PuppetLibrary
             content_type "application/octet-stream"
 
             begin
-                get_module_buffer(author, name, version).tap do
+                @forge.get_module_buffer(author, name, version).tap do
                     attachment "#{author}-#{name}-#{version}.tar.gz"
                 end
             rescue ModuleNotFound
                 status 404
             end
         end
-
-        private
-        def get_module_metadata(author, name)
-            modules = get_metadata(author, name)
-            raise ModuleNotFound if modules.empty?
-
-            module_infos = modules.map { |m| m.to_info }
-            module_infos.deep_merge
-        end
-
-        def get_module_metadata_with_dependencies(author, name)
-            full_name = "#{author}/#{name}"
-
-            module_queue = []
-            modules_versions = {}
-            module_queue << full_name
-            until module_queue.empty?
-                module_full_name = module_queue.shift
-                already_added = modules_versions.include? module_full_name
-                unless already_added
-                    author, module_name = module_full_name.split "/"
-                    module_versions = get_metadata(author, module_name)
-                    dependencies = module_versions.map {|v| v.dependency_names }.flatten
-                    module_queue += dependencies
-                    modules_versions[module_full_name] = module_versions.map { |v| v.to_version }
-                end
-            end
-            raise ModuleNotFound if modules_versions.values == [[]]
-            modules_versions
-        end
-
-        def get_module_buffer(author, name, version)
-            @repo.get_module(author, name, version) or raise ModuleNotFound
-        end
-
-        def get_metadata(author, module_name)
-            @repo.get_metadata(author, module_name).map {|metadata| ModuleMetadata.new(metadata)}
-        end
     end
 end
-
-class ModuleNotFound < Exception
-end

data/lib/puppet_library/util.rb

@@ -1,6 +1,6 @@
 # -*- encoding: utf-8 -*-
 # Puppet Library
-# Copyright (C) 2013 drrb
+# Copyright (C) 2014 drrb
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -23,7 +23,7 @@ class Array
             attribute = yield(element)
             is_duplicate = attr_to_element.include? attribute
             unless is_duplicate
-                attr_to_element[yield(element)] = element
+                attr_to_element[attribute] = element
             end
             !is_duplicate
         end

data/lib/puppet_library/version.rb

@@ -16,5 +16,5 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 module PuppetLibrary
-    VERSION = "0.0.2"
+    VERSION = "0.1.0"
 end

data/puppet-library.gemspec

@@ -1,6 +1,6 @@
 # -*- encoding: utf-8 -*-
 # Puppet Library
-# Copyright (C) 2013 drrb
+# Copyright (C) 2014 drrb
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -41,8 +41,10 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "coveralls"
+  spec.add_development_dependency "librarian-puppet"
   spec.add_development_dependency "mime-types", "< 2"
   spec.add_development_dependency "pry"
+  spec.add_development_dependency "puppet"
   spec.add_development_dependency "rack-test"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec"

data/spec/forge_spec.rb

@@ -0,0 +1,147 @@
+# -*- encoding: utf-8 -*-
+# Puppet Library
+# Copyright (C) 2014 drrb
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'spec_helper'
+
+module PuppetLibrary
+    describe Forge do
+        let(:forge) { Forge.new(module_repo) }
+        let(:module_repo) { double('module_repo') }
+
+        describe "#get_module_buffer" do
+            context "module version not found" do
+                it "raises an error" do
+                    expect(module_repo).to receive(:get_module).with("puppetlabs", "apache", "1.0.0").and_return(nil)
+
+                    expect {
+                        forge.get_module_buffer("puppetlabs", "apache", "1.0.0")
+                    }.to raise_error ModuleNotFound
+                end
+            end
+
+            context "when the module is found" do
+                it "returns a buffer containing the module archive" do
+                    file_buffer = "module content"
+                    expect(module_repo).to receive(:get_module).with("puppetlabs", "apache", "1.0.0").and_return(file_buffer)
+
+                    result = forge.get_module_buffer("puppetlabs", "apache", "1.0.0")
+                    expect(result).to eq file_buffer
+                end
+            end
+        end
+
+        describe "#get_module_metadata" do
+            context "when no modules found" do
+                it "raises an error" do
+                    expect(module_repo).to receive(:get_metadata).with("nonexistant", "nonexistant").and_return([])
+
+                    expect {
+                        forge.get_module_metadata("nonexistant", "nonexistant")
+                    }.to raise_error ModuleNotFound
+                end
+            end
+
+            context "when module versions found" do
+                it "returns metadata for all versions" do
+                    metadata = [ {
+                        "author" => "puppetlabs",
+                        "name" => "puppetlabs-apache",
+                        "description" => "Apache module",
+                        "version" => "1.0.0"
+                    }, {
+                        "author" => "puppetlabs",
+                        "name" => "puppetlabs-apache",
+                        "description" => "Apache module",
+                        "version" => "1.1.0"
+                    } ]
+                    expect(module_repo).to receive(:get_metadata).with("puppetlabs", "apache").and_return(metadata)
+
+                    metadata = forge.get_module_metadata("puppetlabs", "apache")
+
+                    expect(metadata["author"]).to eq "puppetlabs"
+                    expect(metadata["full_name"]).to eq "puppetlabs/apache"
+                    expect(metadata["name"]).to eq "apache"
+                    expect(metadata["desc"]).to eq "Apache module"
+                    expect(metadata["releases"]).to eq [
+                        {"version" => "1.0.0"},
+                        {"version" => "1.1.0"}
+                    ]
+                end
+            end
+        end
+
+        describe "#get_module_metadata_with_dependencies" do
+            context "when no module versions found" do
+                it "raises an error" do
+                    expect(module_repo).to receive(:get_metadata).with("nonexistant", "nonexistant").and_return([])
+
+                    expect {
+                        forge.get_module_metadata_with_dependencies("nonexistant", "nonexistant")
+                    }.to raise_error ModuleNotFound
+                end
+            end
+
+            context "when module versions found" do
+                it "returns metadata for module and dependencies" do
+                    apache_metadata = [ {
+                        "author" => "puppetlabs",
+                        "name" => "puppetlabs-apache",
+                        "description" => "Apache module",
+                        "version" => "1.0.0",
+                        "dependencies" => [
+                            { "name" => "puppetlabs/stdlib", "version_requirement" => ">= 2.4.0" },
+                            { "name" => "puppetlabs/concat", "version_requirement" => ">= 1.0.0" }
+                        ]
+                    }, {
+                        "author" => "puppetlabs",
+                        "name" => "puppetlabs-apache",
+                        "description" => "Apache module",
+                        "version" => "1.1.0",
+                        "dependencies" => [
+                            { "name" => "puppetlabs/stdlib", "version_requirement" => ">= 2.4.0" },
+                            { "name" => "puppetlabs/concat", "version_requirement" => ">= 1.0.0" }
+                        ]
+                    } ]
+                    stdlib_metadata = [ {
+                        "author" => "puppetlabs",
+                        "name" => "puppetlabs-stdlib",
+                        "description" => "Stdlib module",
+                        "version" => "2.0.0",
+                        "dependencies" => [ ]
+                    } ]
+                    concat_metadata = [ {
+                        "author" => "puppetlabs",
+                        "name" => "puppetlabs-concat",
+                        "description" => "Concat module",
+                        "version" => "1.0.0",
+                        "dependencies" => [ ]
+                    } ]
+                    expect(module_repo).to receive(:get_metadata).with("puppetlabs", "apache").and_return(apache_metadata)
+                    expect(module_repo).to receive(:get_metadata).with("puppetlabs", "stdlib").and_return(stdlib_metadata)
+                    expect(module_repo).to receive(:get_metadata).with("puppetlabs", "concat").and_return(concat_metadata)
+
+                    result = forge.get_module_metadata_with_dependencies("puppetlabs", "apache")
+                    expect(result.keys.sort).to eq(["puppetlabs/apache", "puppetlabs/concat", "puppetlabs/stdlib"])
+                    expect(result["puppetlabs/apache"].size).to eq(2)
+                    expect(result["puppetlabs/apache"][0]["file"]).to eq("/modules/puppetlabs-apache-1.0.0.tar.gz")
+                    expect(result["puppetlabs/apache"][0]["version"]).to eq("1.0.0")
+                    expect(result["puppetlabs/apache"][0]["version"]).to eq("1.0.0")
+                end
+            end
+        end
+    end
+end