puppet-library 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,15 @@
 ---
-SHA1:
-  metadata.gz: 6c1ea499735a82bacf4051bd1b4ea24f2003d613
-  data.tar.gz: ad4e79023760adaf7bc96becfa61d9403f2c60be
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    ZDc4NjNkZDQyZTlhNWMzZWIwZDhmMTM1MzY1ZTg4NzUyNWQyYzY2OQ==
+  data.tar.gz: !binary |-
+    MzA4N2JiOGU0MTkzMTBmZGUyYzhhOTFiNWQxZGViOGU0YTY5ODk3OA==
 SHA512:
-  metadata.gz: ebe0288bdc97d3df601dc7f1d523d49d3884662af2b756aac2a7634219175a7b7f6035b3a3ffd0bafcfd87f0c9c3c5eb847ff12409a9aca7daf4de24da4b1141
-  data.tar.gz: 81466a2f9550252a75f7f6d98d941df618eb0b611a52a2d7df0fb30e85b825dcdd0399ed3876889189417e07e14b650cb219f31d0cdb2e33b8d22cf597ce32e5
+  metadata.gz: !binary |-
+    NWUzMTQ5MDM2ZWEwOWY4MzcwYTI1NGRkNjJiNWI3ZDJiN2I0OTg5MGMwMWEz
+    MmY0ZmRhNTQ4ZWI5YWQ2ZWUxMGUxNDQ0YjhjYzMzMmJmMTBjNWRmMTlkM2M3
+    M2Y4YTU0NzQwOThjM2E0YTU2MzIzYWM2M2U4YmFmNDkyYzBkMDY=
+  data.tar.gz: !binary |-
+    MDBmNzBkZDYzZWZkZWU5YWE0ODhkZDNmMDQ5ZmRkODMxNjJjYTQ4Y2Q2M2U3
+    Njc3M2E3Zjk1NGE4MGM2ZWJmMDllMThiM2IwMDE3M2EzYjAxNDg2OTNlOGM0
+    ZGYyNGNhNjJlOWI5MTRlY2QzMjk0YzY5NmU1ZmFlNTNjNTg0ZDc=

data/.travis.yml

@@ -16,13 +16,16 @@
 
 language: ruby
 before_install:
-  - gem update --system 2.1.11
+  - gem update --system ${RUBYGEMS_VERSION}
   - gem --version
 rvm:
   - 1.8.7
   - 1.9.3
   - 2.0.0
   - 2.1.0
-matrix:
-    allow_failures:
-      - rvm: 2.1.0
+env:
+  - RUBYGEMS_VERSION=2.1.11
+  - RUBYGEMS_VERSION=1.8.24
+#matrix:
+#    allow_failures:
+#      - rvm: 2.1.0

data/CHANGELOG.yml

@@ -28,4 +28,8 @@
   changes:
       - Support more specific module version queries using 'version' parameter
       - Fixed bug on older versions of RubyGems
+- tag: v0.3.0
+  changes:
+      - Added module search
+      - Improved performance of proxy
 

data/{lib/puppet_library/module_repo/multi.rb → Guardfile}

@@ -15,29 +15,11 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-module PuppetLibrary::ModuleRepo
-    class Multi
-        def repos
-            @repos ||= []
-        end
+notification :terminal_notifier
 
-        def add_repo(repo)
-            repos << repo
-        end
-
-        def get_module(author, name, version)
-            repos.each do |repo|
-                mod = repo.get_module(author, name, version)
-                return mod unless mod.nil?
-            end
-            return nil
-        end
-
-        def get_metadata(author, name)
-            metadata_list = repos.inject([]) do |metadata_list, repo|
-                metadata_list + repo.get_metadata(author, name)
-            end
-            metadata_list.unique_by { |metadata| metadata["version"] }
-        end
-    end
+guard 'rspec', :version => 2 do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/puppet_library/(.+)\.rb$})     { |m| "spec/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
 end
+

data/README.md

@@ -74,6 +74,21 @@ See all options
 
     $ puppet-library --help
 
+## Back-Ends
+
+Puppet Library contains built-in support for:
+ - serving packaged (`.tar.gz`) modules from a directory (or directories) of your choosing
+ - proxying a remote forge (or forges)
+ - serving modules from a combination of the above
+
+## Compatibility with other tools
+
+Puppet Library currently supports:
+- search with Puppet (`puppet module search apache`)
+- dependency resolution and installation with Puppet (`puppet module install puppetlabs/apache`)
+- dependency resolution and installation with [librarian-puppet](http://librarian-puppet.com)
+- installation with [r10k](https://github.com/adrienthebo/r10k)
+
 ## Contributing
 
 1. Fork it

data/Rakefile

@@ -19,11 +19,25 @@ require 'bundler/gem_tasks'
 require 'rspec/core/rake_task'
 require 'coveralls/rake/task'
 
-if RUBY_VERSION.start_with? "1.8"
-    # The integration test doesn't work on Ruby 1.8.
-    DEFAULT_TEST_TASK = :spec
-else
+class String
+    def green
+        "\e[32m#{self}\e[0m"
+    end
+
+    def red
+        "\e[31m#{self}\e[0m"
+    end
+end
+
+def ruby_version_supports_integration_test?(version = RUBY_VERSION)
+    # The integration test doesn't work on Ruby 1.8, and Puppet doesn't work on 2.1
+    ! /^(1\.8|2\.1)/.match(version)
+end
+
+if ruby_version_supports_integration_test?
     DEFAULT_TEST_TASK = :test
+else
+    DEFAULT_TEST_TASK = :spec
 end
 
 Coveralls::RakeTask.new
@@ -31,6 +45,11 @@ Coveralls::RakeTask.new
 desc "Run the specs"
 RSpec::Core::RakeTask.new(:spec)
 
+desc "Run the integration tests"
+RSpec::Core::RakeTask.new(:integration_test) do |rspec|
+    rspec.pattern = "test/**/*_integration_test.rb"
+end
+
 desc "Run all the tests"
 RSpec::Core::RakeTask.new(:test) do |rspec|
     rspec.pattern = "{spec,test}/**/*_{spec,integration_test}.rb"
@@ -40,7 +59,43 @@ task :default => [DEFAULT_TEST_TASK, 'coveralls:push']
 
 desc "Check it works on all local rubies"
 task :verify do
-    system "rvm all do rake test"
+    versions = %w[1.8 1.9 2.0 2.1]
+    puts "\nRunning Specs".green
+    spec_results = versions.map do |ruby_version|
+        puts "\n- Ruby #{ruby_version}".green
+        system "rvm #{ruby_version} do rake spec"
+    end
+
+    puts "\nRunning Integration Tests".green
+    integration_test_results = versions.map do |ruby_version|
+        puts "\n- Ruby  #{ruby_version}".green
+        system "rvm #{ruby_version} do rake integration_test"
+    end
+
+    puts "\nResults:\n".green
+    results = spec_results.zip(integration_test_results)
+    puts "+---------+-------+-------+"
+    puts "| Version | Specs | Tests |"
+    puts "+---------+-------+-------+"
+    versions.zip(results).each do |(version, (spec_result, integration_test_result))|
+        v = version
+        s = spec_result ? "pass".green : "fail".red
+        i = integration_test_result ? "pass".green : "fail".red
+        puts "| #{v}     | #{s}  | #{i}  |"
+    end
+    puts "+---------+-------+-------+"
+
+    versions.zip(results).each do |(version, (spec_result, integration_test_result))|
+        unless spec_result
+            fail "Specs failed with Ruby #{version}"
+        end
+
+        if ruby_version_supports_integration_test?(version)
+            unless integration_test_result
+                fail "Integration tests failed with Ruby #{version}"
+            end
+        end
+    end
 end
 
 desc "Check all files for license headers"

data/config.ru

@@ -19,8 +19,8 @@ require 'rubygems'
 require 'puppet_library'
 
 server = PuppetLibrary::Server.set_up do |library|
-    library.module_repo PuppetLibrary::ModuleRepo::Proxy.new("http://forge.puppetlabs.com")
-    library.module_repo PuppetLibrary::ModuleRepo::Directory.new("/var/lib/modules")
+    library.forge PuppetLibrary::Forge::Proxy.new("http://forge.puppetlabs.com")
+    library.forge PuppetLibrary::Forge::Directory.new("/var/lib/modules")
 end
 
 run server

data/lib/puppet_library/forge/abstract.rb

@@ -0,0 +1,183 @@
+# -*- encoding: utf-8 -*-
+# Puppet Library
+# Copyright (C) 2014 drrb
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'puppet_library/forge/search_result'
+require 'puppet_library/util'
+
+module PuppetLibrary::Forge
+
+    class Abstract
+        def initialize(module_repo)
+            @repo = module_repo
+        end
+
+        def search_modules(query)
+            search = Search.new(query)
+
+            search_results = retrieve_all_metadata.select do |result|
+                search.matches? result
+            end.sort_by do |result|
+                result.version
+            end.reverse.map do |result|
+                result.to_search_result
+            end
+
+            SearchResult.merge_by_full_name(search_results)
+        end
+
+        def get_module_metadata(author, name)
+            modules = retrieve_metadata(author, name)
+
+            raise ModuleNotFound if modules.empty?
+
+            module_infos = modules.map { |m| m.to_info }
+            module_infos.deep_merge
+        end
+
+        def get_module_metadata_with_dependencies(author, name, version)
+            raise ModuleNotFound if retrieve_metadata(author, name).empty?
+
+            full_name = "#{author}/#{name}"
+            versions = collect_dependencies_versions(full_name)
+            return versions if version.nil?
+
+            versions[full_name] = versions[full_name].select do |v|
+                Gem::Dependency.new(name, version).match?(name, v["version"])
+            end
+
+            dependencies = versions[full_name].map do |v|
+                v["dependencies"].map {|(name, spec)| name}
+            end.flatten
+            versions = Hash[versions.select do |name, info|
+                name == full_name || dependencies.include?(name)
+            end]
+            return versions
+        end
+
+        def collect_dependencies_versions(module_full_name, metadata = {})
+            author, module_name = module_full_name.split "/"
+            module_versions = retrieve_metadata(author, module_name)
+            metadata[module_full_name] = module_versions.map {|v| v.to_version }
+
+            dependencies = module_versions.map {|v| v.dependency_names }.flatten
+            dependencies.each do |dependency|
+                collect_dependencies_versions(dependency, metadata) unless metadata.include? dependency
+            end
+            return metadata
+        end
+
+        def get_module_buffer(author, name, version)
+            @repo.get_module(author, name, version) or raise ModuleNotFound
+        end
+
+        def retrieve_metadata(author, module_name)
+            @repo.get_metadata(author, module_name).map {|metadata| ModuleMetadata.new(metadata)}
+        end
+
+        def retrieve_all_metadata
+            @repo.get_all_metadata.map {|metadata| ModuleMetadata.new(metadata)}
+        end
+    end
+
+    class Search
+        def initialize(query)
+            @query = query
+        end
+
+        def matches?(metadata)
+            return true if @query.nil?
+            return true if metadata.name.include? @query
+            return true if metadata.author.include? @query
+            return false
+        end
+    end
+
+    class ModuleMetadata
+        def initialize(metadata)
+            @metadata = metadata
+        end
+
+        def author
+            @metadata["name"][/^[^-]+/]
+        end
+
+        def name
+            @metadata["name"].sub(/^[^-]+-/, "")
+        end
+
+        def full_name
+            @metadata["name"].sub("-", "/")
+        end
+
+        def version
+            @metadata["version"]
+        end
+
+        def dependencies
+            @metadata["dependencies"]
+        end
+
+        def summary
+            @metadata["summary"]
+        end
+
+        def description
+            @metadata["description"]
+        end
+
+        def project_page
+            @metadata["project_page"]
+        end
+
+        def dependency_names
+            dependencies.map {|d| d["name"]}
+        end
+
+        def to_info
+            {
+                "author" => author,
+                "full_name" => full_name,
+                "name" => name,
+                "desc" => description,
+                "releases" => [ { "version" => version } ]
+            }
+        end
+
+        def to_version
+            {
+                "file" => "/modules/#{author}-#{name}-#{version}.tar.gz",
+                "version" => version,
+                "dependencies" => dependencies.map do |dependency|
+                    [ dependency["name"], dependency["version_requirement"] ]
+                end
+            }
+        end
+
+        def to_search_result
+            {
+                "author" => author,
+                "full_name" => full_name,
+                "name" => name,
+                "desc" => summary,
+                "project_url" => project_page,
+                "releases" => [{ "version" => version}],
+                "version" => version,
+                "tag_list" => [author, name]
+            }
+        end
+    end
+end

data/lib/puppet_library/{module_repo → forge}/directory.rb

@@ -17,10 +17,12 @@
 require 'json'
 require 'rubygems/package'
 require 'zlib'
+require 'puppet_library/forge/abstract'
 
-module PuppetLibrary::ModuleRepo
-    class Directory
+module PuppetLibrary::Forge
+    class Directory < PuppetLibrary::Forge::Abstract
         def initialize(module_dir)
+            super(self)
             @module_dir = module_dir
         end
 
@@ -34,7 +36,7 @@ module PuppetLibrary::ModuleRepo
             end
         end
 
-        def get_metadata(author, module_name)
+        def get_metadata(author = "*", module_name = "")
             Dir["#{@module_dir}/#{author}-#{module_name}*"].map do |module_path|
                 tar = Gem::Package::TarReader.new(Zlib::GzipReader.open(module_path))
                 tar.rewind
@@ -42,5 +44,9 @@ module PuppetLibrary::ModuleRepo
                 JSON.parse(metadata_file.read)
             end
         end
+
+        def get_all_metadata
+            get_metadata
+        end
     end
 end

data/lib/puppet_library/forge/multi.rb

@@ -0,0 +1,81 @@
+# -*- encoding: utf-8 -*-
+# Puppet Library
+# Copyright (C) 2014 drrb
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'puppet_library/forge/search_result'
+
+module PuppetLibrary::Forge
+    class Multi
+        def add_forge(forge)
+            forges << forge
+        end
+
+        def search_modules(query)
+            all_results = forges.map do |forge|
+                forge.search_modules(query)
+            end.flatten
+
+            SearchResult.merge_by_full_name(all_results)
+        end
+
+        def get_module_buffer(author, name, version)
+            forges.each do |forge|
+                begin
+                    return forge.get_module_buffer(author, name, version)
+                rescue ModuleNotFound
+                    # Try the next one
+                end
+            end
+            raise ModuleNotFound
+        end
+
+        def get_module_metadata(author, name)
+            metadata_list = forges.inject([]) do |metadata_list, forge|
+                begin
+                    metadata_list << forge.get_module_metadata(author, name)
+                rescue ModuleNotFound
+                    metadata_list
+                end
+            end
+            raise ModuleNotFound if metadata_list.empty?
+            metadata_list.deep_merge.tap do |metadata|
+                metadata["releases"] = metadata["releases"].unique_by { |release| release["version"] }
+            end
+        end
+
+        def get_module_metadata_with_dependencies(author, name, version)
+            metadata_list = []
+            forges.each do |forge|
+                begin
+                    metadata_list << forge.get_module_metadata_with_dependencies(author, name, version)
+                rescue ModuleNotFound
+                    # Try the next one
+                end
+            end
+            raise ModuleNotFound if metadata_list.empty?
+            metadata_list.deep_merge.tap do |metadata|
+                metadata.each do |module_name, releases|
+                    metadata[module_name] = releases.unique_by { |release| release["version"] }
+                end
+            end
+        end
+
+        private
+        def forges
+            @forges ||= []
+        end
+    end
+end

data/lib/puppet_library/{module_repo → forge}/proxy.rb

@@ -20,8 +20,7 @@ require 'puppet_library/http/cache/in_memory'
 require 'puppet_library/http/cache/noop'
 require 'puppet_library/http/url'
 
-module PuppetLibrary::ModuleRepo
-    #include PuppetLibrary::Http
+module PuppetLibrary::Forge
     class Proxy
         def initialize(url,
                        query_cache = PuppetLibrary::Http::Cache::InMemory.new,
@@ -33,29 +32,37 @@ module PuppetLibrary::ModuleRepo
             @download_cache = download_cache
         end
 
-        def get_module(author, name, version)
-            version_info = get_module_version(author, name, version)
-            if version_info.nil?
-                nil
-            else
+        def search_modules(query)
+            query_parameter = query.nil? ? "" : "?q=#{query}"
+            results = get("/modules.json#{query_parameter}")
+            JSON.parse results
+        end
+
+        def get_module_buffer(author, name, version)
+            begin
+                version_info = get_module_version(author, name, version)
+                raise ModuleNotFound if version_info.nil?
                 download_file(version_info["file"])
+            rescue OpenURI::HTTPError
+                raise ModuleNotFound
             end
         end
 
-        def get_metadata(author, name)
-            module_versions = get_module_versions(author, name)
-            module_versions.map do |version_info|
-                {
-                    "name" => "#{author}-#{name}",
-                    "author" => author,
-                    "version" => version_info["version"],
-                    "dependencies" => version_info["dependencies"].map do |(dep_name, dep_spec)|
-                        { "name" => dep_name, "version_requirement" => dep_spec }
-                    end
-                }
+        def get_module_metadata(author, name)
+            begin
+                response = get("/#{author}/#{name}.json")
+                JSON.parse(response)
+            rescue OpenURI::HTTPError
+                raise ModuleNotFound
+            end
+        end
+
+        def get_module_metadata_with_dependencies(author, name, version)
+            begin
+                look_up_releases(author, name, version)
+            rescue OpenURI::HTTPError
+                raise ModuleNotFound
             end
-        rescue OpenURI::HTTPError => http_error
-            return []
         end
 
         private
@@ -71,8 +78,10 @@ module PuppetLibrary::ModuleRepo
             versions["#{author}/#{name}"]
         end
 
-        def look_up_releases(author, name)
-            response = get("/api/v1/releases.json?module=#{author}/#{name}")
+        def look_up_releases(author, name, version = nil)
+            version_query = version ? "&version=#{version}" : ""
+            url = "/api/v1/releases.json?module=#{author}/#{name}#{version_query}"
+            response = get(url)
             JSON.parse(response)
         end
 
@@ -93,3 +102,4 @@ module PuppetLibrary::ModuleRepo
         end
     end
 end
+

data/lib/puppet_library/forge/search_result.rb

@@ -0,0 +1,50 @@
+# -*- encoding: utf-8 -*-
+# Puppet Library
+# Copyright (C) 2014 drrb
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'puppet_library/util'
+
+module PuppetLibrary::Forge::SearchResult
+    def self.merge_by_full_name(results)
+        results_by_module = results.group_by do |result|
+            result["full_name"]
+        end
+
+        results_by_module.values.map do |module_results|
+            combine_search_results(module_results)
+        end.flatten
+    end
+
+    def self.combine_search_results(search_results)
+        highest_version, tags, releases = search_results.inject([nil, [], []]) do |(highest_version, tags, releases), result|
+            [
+                max_version(highest_version, result["version"]),
+                tags + (result["tag_list"] || []),
+                releases + (result["releases"] || [])
+            ]
+        end
+
+        combined_result = search_results.first.tap do |result|
+            result["version"] = highest_version
+            result["tag_list"] = tags.uniq
+            result["releases"] = releases.uniq.version_sort_by {|r| r["version"]}.reverse
+        end
+    end
+
+    def self.max_version(left, right)
+        [Gem::Version.new(left), Gem::Version.new(right)].max.version
+    end
+end