puppet-library 0.8.0 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
---- 
-SHA512: 
-  metadata.gz: 2353481213ea28760a1aae078b39ba27620b720127d3be2e8a5cf340981fc348e68830f9e0cfc35f7e8d9134f3d2e9475782c500d96439ba14d3b3b4a2ebe5ea
-  data.tar.gz: 36ca9d425a611179a3f7195a27ba5741071e3b5d287c2375b48b39481804711749b8d269a2f27727117823d3b6125ef2551f796462e37baad8e17c455b2fe5b6
-SHA1: 
-  metadata.gz: a16e24fc8143344b487a3fa0dc2470585e372515
-  data.tar.gz: c8539df4b07cd7b1839f4bf8ed94aa0ba99296ed
+---
+SHA1:
+  metadata.gz: 957de9eb484e3ec881a524fe467b6ceab2575f34
+  data.tar.gz: b3a3dbc2610ec8fcdc1064f9b745ea5ba9539e3d
+SHA512:
+  metadata.gz: 9c8bf309ca1363ea52d83d144a1aea682c236c10a00ee7520a62608771f37d2cbfb7d16eb9f1267fd06ad4127aae612b2b1c0d042532a64999bfae8d27d69040
+  data.tar.gz: 965b766bcc9d58bc6939ba454a937469a5db4fc9f23b82a2dfbce089bc5b3c4817583379a2444131b94a44506b74de66e2ba9bf2a53a2627639ceb18eef1093c

data/.simplecov

@@ -1,6 +1,6 @@
 # -*- encoding: utf-8 -*-
 # Puppet Library
-# Copyright (C) 2013 drrb
+# Copyright (C) 2014 drrb
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -17,5 +17,6 @@
 
 SimpleCov.start do
     add_filter "/spec/"
+    add_filter "/test/"
     add_filter "/features/"
 end

data/CHANGELOG.yml

@@ -52,4 +52,8 @@
       - Fixed issue with forge proxying (#1)
 - tag: v0.8.0
   changes:
+      - Fixed issue with downloads (#2)
       - Fixed issue with directory-based forges on Ruby 1.8 (#3)
+- tag: v0.9.0
+  changes:
+      - Added new source forge that serves modules from directories on disk

data/Guardfile

@@ -17,7 +17,7 @@
 
 notification :terminal_notifier
 
-guard 'rspec', :version => 2 do
+guard 'rspec' do
   watch(%r{^spec/.+_spec\.rb$})
   watch(%r{^lib/puppet_library/(.+)\.rb$})     { |m| "spec/#{m[1]}_spec.rb" }
   watch('spec/spec_helper.rb')  { "spec" }

data/README.md

@@ -15,6 +15,7 @@ Plugins can be created to serve modules from arbitrary sources. Puppet Library c
  - serving packaged (`.tar.gz`) modules from a directory (or directories) of your choosing
  - proxying a remote forge (or forges)
  - caching proxied forges to disk
+ - serving modules from source on disk
 
 ## Installation
 
@@ -71,6 +72,10 @@ Proxy a remote forge, caching downloaded modules on disk
 
     $ puppet-library --proxy http://forge.puppetlabs.com --cache-basedir
 
+Serve a module from source
+
+    $ puppet-library --source-dir ~/code/puppetlabs-apache
+
 Serve modules on a specific port
 
     $ puppet-library --port 8888
@@ -84,6 +89,7 @@ See all options
 Puppet Library contains built-in support for:
  - serving packaged (`.tar.gz`) modules from a directory (or directories) of your choosing
  - proxying a remote forge (or forges)
+ - serving modules from source on disk
  - serving modules from a combination of the above
 
 ## Compatibility with other tools
@@ -118,6 +124,8 @@ daemonize: true
 pidfile: /var/run/puppetlibrary.pid
 forges:
     - Directory: /var/lib/modules
+    - Directory: /var/lib/other-modules
+    - Source: /var/code/puppetlabs-apache
     - Proxy: http://forge.puppetlabs.com
 ```
 

data/Rakefile

@@ -23,10 +23,10 @@ require 'yaml'
 require 'net/http'
 
 SUPPORTED_RUBY_VERSIONS = %w[1.8 1.9 2.0 2.1 system]
-# The integration test doesn't work on Ruby 1.8, and Puppet doesn't work on 2.1
+# Puppet doesn't work on 2.1
 INTEGRATION_TEST_INCOMPATIBLE_RUBY_VERSIONS = %w[2.1]
 # Capybara needs Nokogiri, which needs 1.9+
-ACCEPTANCE_TEST_INCOMPATIBLE_RUBY_VERSIONS = %w[1.8 system]
+ACCEPTANCE_TEST_INCOMPATIBLE_RUBY_VERSIONS = %w[1.8]
 
 class String
     def green

data/TODO.yml

@@ -9,8 +9,10 @@ features:
         - clear proxy cache (web form and API)
         - delete modules from disk (web form)
     - Authentication
+    - Make imports work properly from pry, and other projects
 
 dubious_features:
+    - Proxy modules' source in a directory (supported individually for now: should we just leave it that way?)
     - Make proxy cache TTL configurable
     - Allow failover between module repositories
     - Look in /etc/puppet-library.yml for config by default

data/lib/puppet_library/archive/archive_reader.rb

@@ -0,0 +1,33 @@
+# Puppet Library
+# Copyright (C) 2014 drrb
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'rubygems/package'
+require 'zlib'
+
+module PuppetLibrary::Archive
+    class ArchiveReader
+        def initialize(path)
+            @path = path
+        end
+
+        def read_entry(entry_name_regex)
+            tar = Gem::Package::TarReader.new(Zlib::GzipReader.open(@path))
+            tar.rewind
+            entry = tar.find {|e| e.full_name =~ entry_name_regex } or raise "Couldn't find entry in archive"
+            entry.read
+        end
+    end
+end

data/lib/puppet_library/archive/archiver.rb

@@ -0,0 +1,92 @@
+# Puppet Library
+# Copyright (C) 2014 drrb
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'rubygems/package'
+require 'zlib'
+
+module PuppetLibrary::Archive
+    # Adapted from https://gist.github.com/sinisterchipmunk/1335041
+    module Archiver
+        def self.archive_dir(dir, basedir, &block)
+            gzip(tar(dir, basedir, &block))
+        end
+
+        private
+        # gzips the underlying string in the given StringIO,
+        # returning a new StringIO representing the
+        # compressed file.
+        def self.gzip(tar_buffer)
+            zip_buffer = StringIO.new("")
+            zipper = Zlib::GzipWriter.new(zip_buffer)
+            zipper.write tar_buffer.string
+            zipper.close # this is necessary!
+            # z was closed to write the gzip footer, so
+            # now we need a new StringIO
+            StringIO.new zip_buffer.string
+        end
+
+        def self.tar(path, basedir, &block)
+            tarfile = StringIO.new("")
+            Gem::Package::TarWriter.new(tarfile) do |tar|
+                walk_directory(path) do |file|
+                    entry_name = file.sub /^#{Regexp::escape path}\/?/, "#{basedir}/"
+                    TarEntry.from(file).add_to!(tar, entry_name)
+                end
+                yield(RebasedTar.new(tar, basedir)) if block_given?
+            end
+            tarfile.rewind
+            tarfile
+        end
+
+        def self.walk_directory(basedir)
+            Dir[File.join(basedir, "**/*")].each do |file|
+                yield(file)
+            end
+        end
+
+        class RebasedTar
+            def initialize(tar, basedir)
+                @tar, @basedir = tar, basedir
+            end
+
+            def add_file(name, mode, &block)
+                entry_name = name.sub /^\/?/, "#{@basedir}/"
+                @tar.add_file(entry_name, mode, &block)
+            end
+        end
+
+        class TarEntry
+            def self.from(file)
+                TarEntry.new(file)
+            end
+
+            def initialize(file)
+                @file = file
+            end
+
+            def add_to!(tar, entry_name)
+                mode = File.stat(@file).mode
+                if File.directory?(@file)
+                    tar.mkdir(entry_name, mode)
+                else
+                    tar.add_file(entry_name, mode) do |entry|
+                        File.open(@file, "rb") { |file| entry.write file.read }
+                    end
+                end
+            end
+        end
+    end
+end

data/lib/puppet_library/archive.rb

@@ -0,0 +1,20 @@
+# Puppet Library
+# Copyright (C) 2014 drrb
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+module PuppetLibrary::Archive
+    require 'puppet_library/archive/archiver'
+    require 'puppet_library/archive/archive_reader'
+end

data/lib/puppet_library/forge/directory.rb

@@ -15,9 +15,8 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 require 'json'
-require 'rubygems/package'
-require 'zlib'
 require 'puppet_library/forge/abstract'
+require 'puppet_library/archive/archive_reader'
 
 module PuppetLibrary::Forge
     class Directory < PuppetLibrary::Forge::Abstract
@@ -40,10 +39,9 @@ module PuppetLibrary::Forge
 
         def get_metadata(author = "*", module_name = "")
             Dir["#{@module_dir}/#{author}-#{module_name}*"].map do |module_path|
-                tar = Gem::Package::TarReader.new(Zlib::GzipReader.open(module_path))
-                tar.rewind
-                metadata_file = tar.find {|e| e.full_name =~ /[^\/]+\/metadata\.json/}
-                JSON.parse(metadata_file.read)
+                archive = PuppetLibrary::Archive::ArchiveReader.new(module_path)
+                metadata_file = archive.read_entry %r[[^/]+/metadata\.json$]
+                JSON.parse(metadata_file)
             end
         end
 

data/lib/puppet_library/forge/multi.rb

@@ -19,12 +19,16 @@ require 'puppet_library/forge/search_result'
 
 module PuppetLibrary::Forge
     class Multi
+        def initialize
+            @forges = []
+        end
+
         def add_forge(forge)
-            forges << forge
+            @forges << forge
         end
 
         def search_modules(query)
-            all_results = forges.map do |forge|
+            all_results = @forges.map do |forge|
                 forge.search_modules(query)
             end.flatten
 
@@ -32,7 +36,7 @@ module PuppetLibrary::Forge
         end
 
         def get_module_buffer(author, name, version)
-            forges.each do |forge|
+            @forges.each do |forge|
                 begin
                     return forge.get_module_buffer(author, name, version)
                 rescue ModuleNotFound
@@ -43,7 +47,7 @@ module PuppetLibrary::Forge
         end
 
         def get_module_metadata(author, name)
-            metadata_list = forges.inject([]) do |metadata_list, forge|
+            metadata_list = @forges.inject([]) do |metadata_list, forge|
                 begin
                     metadata_list << forge.get_module_metadata(author, name)
                 rescue ModuleNotFound
@@ -58,7 +62,7 @@ module PuppetLibrary::Forge
 
         def get_module_metadata_with_dependencies(author, name, version)
             metadata_list = []
-            forges.each do |forge|
+            @forges.each do |forge|
                 begin
                     metadata_list << forge.get_module_metadata_with_dependencies(author, name, version)
                 rescue ModuleNotFound
@@ -72,10 +76,5 @@ module PuppetLibrary::Forge
                 end
             end
         end
-
-        private
-        def forges
-            @forges ||= []
-        end
     end
 end

data/lib/puppet_library/forge/proxy.rb

@@ -79,12 +79,12 @@ module PuppetLibrary::Forge
             versions["#{author}/#{name}"]
         end
 
-        def look_up_releases(author, name, version = nil, &block)
+        def look_up_releases(author, name, version = nil, &optional_processor)
             version_query = version ? "&version=#{version}" : ""
             url = "/api/v1/releases.json?module=#{author}/#{name}#{version_query}"
             response_text = get(url)
             response = JSON.parse(response_text)
-            process_releases_response(response, &block)
+            process_releases_response(response, &optional_processor)
         end
 
         def process_releases_response(response)

data/lib/puppet_library/forge/source.rb

@@ -0,0 +1,69 @@
+# Puppet Library
+# Copyright (C) 2014 drrb
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'puppet_library/archive/archiver'
+require 'puppet_library/puppet_module/modulefile'
+require 'json'
+
+module PuppetLibrary::Forge
+    class Source < PuppetLibrary::Forge::Abstract
+        CACHE_TTL_MILLIS = 500
+        def initialize(module_dir)
+            super(self)
+            module_dir = File.expand_path(module_dir)
+            raise "Module directory '#{module_dir}' doesn't exist" unless File.directory? module_dir
+            raise "Module directory '#{module_dir}' isn't readable" unless File.executable? module_dir
+            @module_dir = module_dir
+            @cache = PuppetLibrary::Http::Cache::InMemory.new(CACHE_TTL_MILLIS)
+        end
+
+        def get_module(author, name, version)
+            return nil unless this_module?(author, name, version)
+            PuppetLibrary::Archive::Archiver.archive_dir(@module_dir, "#{author}-#{name}-#{version}") do |archive|
+                archive.add_file("metadata.json", 0644) do |entry|
+                    entry.write modulefile.to_metadata
+                end
+            end
+        end
+
+        def get_metadata(author, module_name)
+            return [] unless this_module?(author, module_name)
+            [ modulefile.to_metadata ]
+        end
+
+        def get_all_metadata
+            get_metadata(modulefile.get_author, modulefile.get_simple_name)
+        end
+
+        private
+        def this_module?(author, module_name, version = nil)
+            same_module = modulefile.get_name == "#{author}-#{module_name}"
+            if version.nil?
+                return same_module
+            else
+                return same_module && modulefile.get_version == version
+            end
+        end
+
+        def modulefile
+            modulefile_path = File.join(@module_dir, "Modulefile")
+            @cache.get modulefile_path do
+                PuppetLibrary::PuppetModule::Modulefile.read(modulefile_path)
+            end
+        end
+    end
+end
+

data/lib/puppet_library/forge.rb

@@ -25,4 +25,5 @@ module PuppetLibrary::Forge
     require 'puppet_library/forge/multi'
     require 'puppet_library/forge/proxy'
     require 'puppet_library/forge/search_result'
+    require 'puppet_library/forge/source'
 end

data/lib/puppet_library/http/cache/noop.rb

@@ -18,11 +18,9 @@
 require 'puppet_library/http/cache/in_memory'
 
 module PuppetLibrary::Http::Cache
-    class NoOp < PuppetLibrary::Http::Cache::InMemory
-        def save(key, entry)
-        end
-
-        def retrieve(key)
+    class NoOp
+        def get(path)
+            yield
         end
     end
 end

data/lib/puppet_library/puppet_library.rb

@@ -19,6 +19,8 @@ require 'rack'
 require 'yaml'
 require 'puppet_library/forge/directory'
 require 'puppet_library/forge/multi'
+require 'puppet_library/forge/proxy'
+require 'puppet_library/forge/source'
 require 'puppet_library/version'
 
 module PuppetLibrary
@@ -70,12 +72,15 @@ module PuppetLibrary
                 end
 
                 options[:forges] = []
-                opts.on("-m", "--module-dir DIR", "Directory containing the modules (can be specified multiple times)") do |module_dir|
+                opts.on("-m", "--module-dir DIR", "Directory containing packaged modules (can be specified multiple times)") do |module_dir|
                     options[:forges] << [Forge::Directory, module_dir]
                 end
                 opts.on("-x", "--proxy URL", "Remote forge to proxy (can be specified multiple times)") do |url|
                     options[:forges] << [Forge::Proxy, sanitize_url(url)]
                 end
+                opts.on("--source-dir DIR", "Directory containing a module's source (can be specified multiple times)") do |module_dir|
+                    options[:forges] << [Forge::Source, module_dir]
+                end
 
                 opts.on("--cache-basedir DIR", "Cache all proxies' downloaded modules under this directory") do |cache_basedir|
                     options[:cache_basedir] = cache_basedir

data/lib/puppet_library/puppet_module/modulefile.rb

@@ -0,0 +1,68 @@
+# Puppet Library
+# Copyright (C) 2014 drrb
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+module PuppetLibrary::PuppetModule
+    class Modulefile
+        def self.read(modulefile_path)
+            Modulefile.new.tap do |modulefile|
+                modulefile.instance_eval(File.read(modulefile_path))
+            end
+        end
+
+        %w[name version author source summary description project_page license].each do |property|
+            class_eval <<-EOF
+                def #{property}(value)
+                    @#{property} = value
+                end
+
+                def get_#{property}
+                    @#{property}
+                end
+            EOF
+        end
+
+        def dependency(name, spec)
+            get_dependencies.push("name" => name, "version_requirement" => spec)
+        end
+
+        def get_dependencies
+            @dependencies ||= []
+        end
+
+        def get_simple_name
+            @name.split("-").last
+        end
+
+        def method_missing(name, *args, &block)
+            Modulefile.log "Unsupported config parsed from Modulefile: #{name}(#{args.join", "})"
+        end
+
+        def to_metadata
+            {
+                "name" => get_name,
+                "version" => get_version,
+                "author" => get_author,
+                "description" => get_description,
+                "dependencies" => get_dependencies
+            }
+        end
+
+        private
+        def self.log(message)
+            puts message
+        end
+    end
+end

data/lib/puppet_library/puppet_module.rb

@@ -0,0 +1,19 @@
+# Puppet Library
+# Copyright (C) 2014 drrb
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+module PuppetLibrary::PuppetModule
+    require 'puppet_library/puppet_module/modulefile'
+end

data/lib/puppet_library/server.rb

@@ -82,8 +82,7 @@ module PuppetLibrary
             begin
                 @forge.get_module_metadata_with_dependencies(author, module_name, version).to_json
             rescue Forge::ModuleNotFound
-                status 410
-                {"error" => "Module #{author}/#{module_name} not found"}.to_json
+                halt 410, {"error" => "Module #{author}/#{module_name} not found"}.to_json
             end
         end
 
@@ -98,12 +97,7 @@ module PuppetLibrary
                 buffer = @forge.get_module_buffer(author, name, version).tap do
                     attachment "#{author}-#{name}-#{version}.tar.gz"
                 end
-                if buffer.respond_to?(:size)
-                    headers = { "Content-Length" => buffer.size.to_s }
-                else
-                    headers = {}
-                end
-                [ 200, headers, buffer ]
+                download buffer
             rescue Forge::ModuleNotFound
                 halt 404
             end
@@ -117,9 +111,18 @@ module PuppetLibrary
                 metadata = @forge.get_module_metadata(author, module_name)
                 haml :module, { :locals => { "metadata" => metadata } }
             rescue Forge::ModuleNotFound
-                status 404
-                haml :module_not_found, { :locals => { "author" => author, "name" => module_name } }
+                halt 404, haml(:module_not_found, { :locals => { "author" => author, "name" => module_name } })
+            end
+        end
+
+        private
+        def download(buffer)
+            if buffer.respond_to?(:size)
+                headers = { "Content-Length" => buffer.size.to_s }
+            else
+                headers = {}
             end
+            [ 200, headers, buffer ]
         end
     end
 end

data/lib/puppet_library/version.rb

@@ -16,5 +16,5 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 module PuppetLibrary
-    VERSION = "0.8.0"
+    VERSION = "0.9.0"
 end

data/lib/puppet_library.rb

@@ -16,9 +16,11 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 module PuppetLibrary
+    require 'puppet_library/archive'
     require 'puppet_library/forge'
     require 'puppet_library/http'
     require 'puppet_library/puppet_library'
+    require 'puppet_library/puppet_module'
     require 'puppet_library/server'
     require 'puppet_library/util'
     require 'puppet_library/version'

data/puppet-library.gemspec

@@ -24,7 +24,7 @@ Gem::Specification.new do |spec|
   spec.version       = PuppetLibrary::VERSION
   spec.authors       = ["drrb"]
   spec.email         = ["drrrrrrrrrrrb@gmail.com"]
-  spec.description   = "A Puppet module server"
+  spec.description   = "A private Puppet forge"
   spec.summary       = <<-EOF
     Puppet Library is a private Puppet module server that's compatible with librarian-puppet.
   EOF
@@ -43,7 +43,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "coveralls"
   spec.add_development_dependency "gitsu"
-  spec.add_development_dependency "librarian-puppet"
+  spec.add_development_dependency "librarian-puppet", "0.9.10" # 0.9.12 breaks on Ruby 1.8.7
   spec.add_development_dependency "mime-types", "< 2"
   spec.add_development_dependency "pry"
   spec.add_development_dependency "puppet", "~> 3.3.0"