puppet-library 0.0.2 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 038f863a597766d136e7db4d8f71245844334d7c
-  data.tar.gz: 0f98e71f5110f3875d4c03a401446bf400faa0bc
+  metadata.gz: dd852d5afc9f767f3adbd20e96a3d6ba37fec092
+  data.tar.gz: 9030ad628af928e81289963f28e9df751a4240b6
 SHA512:
-  metadata.gz: 210fbe7f939854063796aacf44bcdd00da819f3ce97ede62ed58d00ad8c7cd787630aba645bbd460064a5f142da86613b65c920f7995bade1ded406fa049d6b6
-  data.tar.gz: 4978484237188df4973d38aa8880dbfe3829d17753714f8f20cbb29bf85fbbb6426facb137fb71aefe4125e34dcf690142adf45747c5bffc869009abdb825458
+  metadata.gz: a168548b6050b5d34d121b1191f74dfe5a84e11024714f07d753828ef72894ef3b039ffbbd8c3c7a125179b29c4862ea8d4de17f2c45a3e0cf60527d48ae723a
+  data.tar.gz: 1c130b6732c99488fffc95442bca7b3604b0ae38213aba4415b185dd79514b705ee76cdc8710f1b53054f5428c54143b54ae80ec987c7a243888f51752602c5f

data/.travis.yml

@@ -20,6 +20,5 @@ before_install:
   - gem --version
 rvm:
   - "1.8.7"
-  - "1.9.2"
   - "1.9.3"
   - "2.0.0"

data/README.md

@@ -1,11 +1,19 @@
 # Puppet Library
 
-A server for your Puppet modules. Compatible with [librarian-puppet](http://librarian-puppet.com).
+A private server for your Puppet modules. Compatible with [librarian-puppet](http://librarian-puppet.com).
 
 [![Build Status](https://travis-ci.org/drrb/puppet-library.png?branch=master)](https://travis-ci.org/drrb/puppet-library)
 [![Coverage Status](https://coveralls.io/repos/drrb/puppet-library/badge.png)](https://coveralls.io/r/drrb/puppet-library)
+[![Code Climate](https://codeclimate.com/github/drrb/puppet-library.png)](https://codeclimate.com/github/drrb/puppet-library)
 
-Puppet library serves packaged (`tar.gz`) modules from a directory of your choosing in the same format as [the Puppet Forge](http://forge.puppetlabs.com).
+[![Gem Version](https://badge.fury.io/rb/puppet-library.png)](http://badge.fury.io/rb/puppet-library)
+[![Dependency Status](https://gemnasium.com/drrb/puppet-library.png)](https://gemnasium.com/drrb/puppet-library)
+
+Puppet Library serves Puppet modules in the same format as [the Puppet Forge](http://forge.puppetlabs.com). This allows you to create a private Puppet Forge and manage all the modules you use completely within your infrastructure.
+
+Plugins can be created to serve modules from arbitrary sources. Puppet Library contains built-in support for:
+ - serving packaged (`.tar.gz`) modules from a directory (or directories) of your choosing
+ - proxying a remote forge (or forges)
 
 ## Installation
 
@@ -13,6 +21,12 @@ Install the server as a Gem:
 
     $ gem install puppet-library
 
+Or, to get the latest, you can install from source
+
+    $ git clone https://github.com/drrb/puppet-library.git
+    $ cd puppet-library
+    $ rake install
+
 ## Getting Started
 
 Create a module directory and add the modules to it
@@ -24,7 +38,7 @@ Create a module directory and add the modules to it
 
 Start the server
 
-    $ puppet-library
+    $ puppet-library --port 9292 --module-dir ./modules
 
 Point librarian-puppet to the server
 
@@ -40,7 +54,7 @@ Resolve and download the modules
 
 ## Usage
 
-Run the server
+Run the server (proxies the Puppet Forge by default)
 
     $ puppet-library
 
@@ -48,6 +62,10 @@ Serve modules from a specific directory
 
     $ puppet-library --module-dir /var/puppet-modules
 
+Serve modules from a remote forge as a proxy
+
+    $ puppet-library --proxy http://forge.puppetlabs.com
+
 Serve modules on a specific port
 
     $ puppet-library --port 8888
@@ -73,7 +91,7 @@ This project was inspired by [dalen/simple-puppet-forge](https://github.com/dale
 ## License
 
 Puppet Library
-Copyright (C) 2013 drrb
+Copyright (C) 2014 drrb
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

data/Rakefile

@@ -1,6 +1,6 @@
 # -*- encoding: utf-8 -*-
 # Puppet Library
-# Copyright (C) 2013 drrb
+# Copyright (C) 2014 drrb
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -19,14 +19,28 @@ require 'bundler/gem_tasks'
 require 'rspec/core/rake_task'
 require 'coveralls/rake/task'
 
-RSpec::Core::RakeTask.new(:spec)
+if RUBY_VERSION.start_with? "1.8"
+    # The integration test doesn't work on Ruby 1.8.
+    DEFAULT_TEST_TASK = :spec
+else
+    DEFAULT_TEST_TASK = :test
+end
+
 Coveralls::RakeTask.new
 
-task :default => [:spec, 'coveralls:push']
+desc "Run the specs"
+RSpec::Core::RakeTask.new(:spec)
+
+desc "Run all the tests"
+RSpec::Core::RakeTask.new(:test) do |rspec|
+    rspec.pattern = "{spec,test}/**/*_{spec,integration_test}.rb"
+end
+
+task :default => [DEFAULT_TEST_TASK, 'coveralls:push']
 
 desc "Check it works on all local rubies"
 task :verify do
-    system "rvm all do rake spec check-license"
+    system "rvm all do rake test"
 end
 
 desc "Check all files for license headers"

data/TODO.yml

@@ -1,5 +1,16 @@
-Features:
-    - Configure embedded server (currently uses the default Rack server)
-    - Proxy another repository
+features:
     - Use config from a config file
+    - Allow proxy to cache downloaded modules/API calls on disk
+    - |
+        Accept --cache-dir option to specify a parent cache directory (people
+        will need to embed the server or use a config file for more control)
+    - Validate repo dirs (make sure they're readable)
+    - Allow failover between module repositories
     - Document usage with Passenger
+    - Web UI
+    - Authentication
+
+dubious_features:
+    - Make proxy cache TTL configurable
+    - REST API to clear proxy cache
+    - REST API to upload modules

data/config.ru

@@ -1,6 +1,6 @@
 # -*- encoding: utf-8 -*-
 # Puppet Library
-# Copyright (C) 2013 drrb
+# Copyright (C) 2014 drrb
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -18,5 +18,9 @@
 require 'rubygems'
 require 'puppet_library'
 
-PuppetLibrary::Server.set :module_repo, ModuleRepo::Directory.new("/var/puppet-library/modules")
-run PuppetLibrary::Server
+server = PuppetLibrary::Server.set_up do |library|
+    library.module_repo PuppetLibrary::ModuleRepo::Proxy.new("http://forge.puppetlabs.com")
+    library.module_repo PuppetLibrary::ModuleRepo::Directory.new("/var/lib/modules")
+end
+
+run server

data/lib/puppet_library.rb

@@ -1,6 +1,6 @@
 # -*- encoding: utf-8 -*-
 # Puppet Library
-# Copyright (C) 2013 drrb
+# Copyright (C) 2014 drrb
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -19,7 +19,11 @@ require 'puppet_library/version'
 
 require 'puppet_library/puppet_library'
 require 'puppet_library/server'
+require 'puppet_library/http/cache'
+require 'puppet_library/http/http_client'
+require 'puppet_library/http/url'
 require 'puppet_library/module_metadata'
 require 'puppet_library/module_repo/directory'
 require 'puppet_library/module_repo/multi'
+require 'puppet_library/module_repo/proxy'
 require 'puppet_library/util'

data/lib/puppet_library/forge.rb

@@ -0,0 +1,100 @@
+# -*- encoding: utf-8 -*-
+# Puppet Library
+# Copyright (C) 2014 drrb
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+class Array
+    def deep_merge
+        inject({}) do |merged, map|
+            merged.deep_merge(map)
+        end
+    end
+end
+
+class Hash
+    def deep_merge(other)
+        merge(other) do |key, old_val, new_val|
+            if old_val.instance_of? Array
+                old_val + new_val
+            else
+                new_val
+            end
+        end
+    end
+end
+
+module PuppetLibrary
+    class ModuleNotFound < Exception
+    end
+
+    class Forge
+        class Config
+            def initialize(module_repo)
+                @module_repo = module_repo
+            end
+
+            def module_repo(repo)
+                @module_repo.add_repo repo
+            end
+        end
+
+        def self.configure(&config_block)
+            module_repo = ModuleRepo::Multi.new
+            yield(Config.new(module_repo))
+            Forge.new(module_repo)
+        end
+
+        def initialize(module_repo)
+            @repo = module_repo
+        end
+
+        def get_module_metadata(author, name)
+            modules = get_metadata(author, name)
+
+            raise ModuleNotFound if modules.empty?
+
+            module_infos = modules.map { |m| m.to_info }
+            module_infos.deep_merge
+        end
+
+        def get_module_metadata_with_dependencies(author, name)
+            full_name = "#{author}/#{name}"
+
+            collect_dependencies_versions(full_name).tap do |versions|
+                raise ModuleNotFound if versions[full_name].empty?
+            end
+        end
+
+        def collect_dependencies_versions(module_full_name, metadata = {})
+            author, module_name = module_full_name.split "/"
+            module_versions = get_metadata(author, module_name)
+            metadata[module_full_name] = module_versions.map {|v| v.to_version }
+
+            dependencies = module_versions.map {|v| v.dependency_names }.flatten
+            dependencies.each do |dependency|
+                collect_dependencies_versions(dependency, metadata) unless metadata.include? dependency
+            end
+            return metadata
+        end
+
+        def get_module_buffer(author, name, version)
+            @repo.get_module(author, name, version) or raise ModuleNotFound
+        end
+
+        def get_metadata(author, module_name)
+            @repo.get_metadata(author, module_name).map {|metadata| ModuleMetadata.new(metadata)}
+        end
+    end
+end

data/lib/puppet_library/http/cache.rb

@@ -0,0 +1,61 @@
+# -*- encoding: utf-8 -*-
+# Puppet Library
+# Copyright (C) 2014 drrb
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+module PuppetLibrary::Http
+    class Cache
+        ARBITRARY_CACHE_TTL_MILLIS = 10 * 1000
+        def initialize(millis_to_live = ARBITRARY_CACHE_TTL_MILLIS)
+            @reaper = Reaper.new(millis_to_live)
+            @cache = {}
+        end
+
+        def get(key)
+            entry = @cache[key]
+            if entry
+                return entry.value unless @reaper.wants_to_kill? entry
+            end
+
+            value = yield
+            @cache[key] = Entry.new(value)
+            return value
+        end
+
+        class Entry
+            attr_accessor :value
+
+            def initialize(value)
+                @birth = Time.now
+                @value = value
+            end
+
+            def age_millis
+                age_seconds = Time.now - @birth
+                age_seconds * 1000
+            end
+        end
+
+        class Reaper
+            def initialize(millis_to_live)
+                @millis_to_live = millis_to_live
+            end
+
+            def wants_to_kill?(entry)
+                entry.age_millis > @millis_to_live
+            end
+        end
+    end
+end

data/lib/puppet_library/http/http_client.rb

@@ -0,0 +1,35 @@
+# -*- encoding: utf-8 -*-
+# Puppet Library
+# Copyright (C) 2014 drrb
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'open-uri'
+
+module PuppetLibrary::Http
+    class HttpClient
+        def get(url)
+            open_uri(url).read
+        end
+
+        def download(url)
+            open_uri(url)
+        end
+
+        private
+        def open_uri(url)
+            open(url)
+        end
+    end
+end

data/lib/puppet_library/http/url.rb

@@ -0,0 +1,27 @@
+# -*- encoding: utf-8 -*-
+# Puppet Library
+# Copyright (C) 2014 drrb
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+module PuppetLibrary::Http
+    class Url
+        def self.normalize(uri_string)
+            unless uri_string =~ /^https?/
+                uri_string = "http://#{uri_string}"
+            end
+            uri_string.sub /\/$/, ""
+        end
+    end
+end

data/lib/puppet_library/module_metadata.rb

@@ -1,6 +1,6 @@
 # -*- encoding: utf-8 -*-
 # Puppet Library
-# Copyright (C) 2013 drrb
+# Copyright (C) 2014 drrb
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -17,7 +17,6 @@
 
 module PuppetLibrary
     class ModuleMetadata
-
         def initialize(metadata)
             @metadata = metadata
         end
@@ -42,6 +41,10 @@ module PuppetLibrary
             @metadata["dependencies"]
         end
 
+        def description
+            @metadata["description"]
+        end
+
         def dependency_names
             dependencies.map {|d| d["name"]}
         end
@@ -51,7 +54,7 @@ module PuppetLibrary
                 "author" => author,
                 "full_name" => full_name,
                 "name" => name,
-                "desc" => @metadata["description"],
+                "desc" => description,
                 "releases" => [ { "version" => version } ]
             }
         end
@@ -60,7 +63,9 @@ module PuppetLibrary
             {
                 "file" => "/modules/#{author}-#{name}-#{version}.tar.gz",
                 "version" => version,
-                    "dependencies" => dependencies.map {|m| [ m["name"], m["version_requirement"] ]}
+                "dependencies" => dependencies.map do |dependency|
+                    [ dependency["name"], dependency["version_requirement"] ]
+                end
             }
         end
     end