puppet-forge-mirror 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 32be687427cf9f2e8873d6b53e012c0ba4bd8c22
+  data.tar.gz: fc766fa83c54c911b7bed3bbbdea1a2ffb340ab1
+SHA512:
+  metadata.gz: 699bd9097e43fe4363b3a58016343c797d585595286bd4a62b16220a636cff94091a9a5d1679ef800a61e6bc17924ff422d760a3b53908ab0172600aee6facfa
+  data.tar.gz: ff6dfc777f3fb0c2105008df3e65ee3b8e6897d8ba439b7c7b8cdf1739e7dfb92e485697ecb72098011c7e3b7d3dbcd157e3264aba1ee56fddacca51972849de

data/.gemspec

@@ -0,0 +1,14 @@
+# -*- encoding: utf-8 -*-
+Gem::Specification.new do |s|
+  s.name        = 'puppet-forge-mirror'
+  s.version     = '0.0.1'
+  s.author      = 'Johan Haals'
+  s.email       = ['johan.haals@gmail.com']
+  s.homepage    = 'https://github.com/jhaals/puppet-forge-mirror'
+  s.summary     = "Mirror the Puppet Forge by downloading it's modules"
+  s.license     = 'Apache 2.0'
+  s.files         = `git ls-files`.split("\n")
+  s.require_paths = ['lib', 'bin']
+  s.required_ruby_version = '>= 1.9.3'
+end
+

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gem 'puppet-forge-mirror', :path => '.'

data/README.md

@@ -0,0 +1,28 @@
+# puppet-forge-mirror
+simple tool that mirrors the Puppet Forge by downloading it's modules to a local folder on disk.
+It queries the Puppet v3 API and download each module listed under `https://forgeapi.puppetlabs.com/v3/releases`.
+
+A module is only downloaded if:
+
+* less than 1MB - Some people put binary files in their modules
+* The checksum from the API != checksum of the module already stored on disk.
+
+Modules are structured `<author>/<modulename>/<author>-<modulename>-<version>.tar.gz>`
+
+### Usage
+
+    gem install puppet-forge-mirror
+
+    $ puppet-forge-mirror /path/to/modules
+    downloading https://forgeapi.puppetlabs.com/v3/files/example42-puppi-2.1.9.tar.gz
+    downloading https://forgeapi.puppetlabs.com/v3/files/puppetlabs-postgresql-3.3.3.tar.gz
+    downloading https://forgeapi.puppetlabs.com/v3/files/puppetlabs-stdlib-4.3.2.tar.gz
+    downloading https://forgeapi.puppetlabs.com/v3/files/phundisk-spacewalk-0.1.2.tar.gz
+    downloading https://forgeapi.puppetlabs.com/v3/files/puppetlabs-apache-1.0.1.tar.gz
+    downloading https://forgeapi.puppetlabs.com/v3/files/puppetlabs-inifile-1.0.3.tar.gz
+    downloading https://forgeapi.puppetlabs.com/v3/files/puppetlabs-puppetdb-3.0.1.tar.gz
+    downloading https://forgeapi.puppetlabs.com/v3/files/yguenane-repoforge-0.1.0.tar.gz
+    downloading https://forgeapi.puppetlabs.com/v3/files/puppetlabs-apt-1.2.0.tar.gz
+
+### Why would I need this?
+One example would be that you're running a huge infrastructure with a masterless Puppet setup.That could generate a several thousand requests which would make Puppetlabs servers very unhappy. Running a local cache internally would increase the download speed and reduce the upstream requests to the forge. The mirror script could for example run once a day and that would only generate ~ 300 HTTP requests.

data/bin/puppet-forge-mirror

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+require 'puppet-forge-mirror'
+
+if ARGV[0].nil?
+  puts "puppet-forge-mirror [module path]"
+  exit
+end
+
+fm = PuppetForgeMirror.new(
+  'https://forgeapi.puppetlabs.com',
+  ARGV[0])
+fm.download_modules

data/lib/puppet-forge-mirror.rb

@@ -0,0 +1,73 @@
+require 'net/http'
+require 'json'
+require 'open-uri'
+require 'digest'
+require 'fileutils'
+require 'uri'
+
+# Mirror /v3 forges
+class PuppetForgeMirror
+  def initialize(forge_url, modules_dir)
+    @modules_dir = modules_dir
+    @forge_url = forge_url
+  end
+
+  # Perform http GET request, return response.body if status code is 200
+  def http_get(url)
+    uri = URI(url)
+
+    Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https') do |http|
+      request = Net::HTTP::Get.new uri
+      response = http.request request
+
+      if response.code != '200'
+        puts "Error #{response.code}: Failed to get #{uri}"
+        return nil
+      end
+      response.body
+    end
+  end
+
+  # Download a specific module
+  def download_module(url, mod, owner, filename, checksum)
+    module_dir = File.join(@modules_dir, owner, mod)
+    filepath = File.join(module_dir, filename)
+
+    FileUtils.mkdir_p module_dir unless File.exist? module_dir
+
+    if File.exist? filepath
+      return if Digest::MD5.file(filepath).hexdigest == checksum
+    end
+
+    puts "downloading #{url}"
+    data = http_get(url)
+    return if data.nil?
+    open(filepath, 'wb') { |f| f.write(data) }
+  end
+
+  def download_modules
+    url = URI.join(@forge_url, '/v3/releases')
+    while
+      data = http_get(url)
+      break if data.nil?
+
+      r = JSON.parse(data)
+      r['results'].each do |m|
+        if m['file_size'] > 1024 * 1024 * 1
+          puts "#{m['metadata']['name']} is too big, skipping"
+          next
+        end
+        download_module(
+          URI.join(@forge_url, m['file_uri']),
+          m['module']['name'],
+          m['module']['owner']['username'],
+          m['file_uri'].split('/').last,
+          m['file_md5']
+        )
+      end
+      next_url = r['pagination']['next']
+      break if next_url.nil?
+      url = URI.join(@forge_url, next_url)
+    end
+  end
+end

metadata

@@ -0,0 +1,50 @@
+--- !ruby/object:Gem::Specification
+name: puppet-forge-mirror
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Johan Haals
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-09-03 00:00:00.000000000 Z
+dependencies: []
+description: 
+email:
+- johan.haals@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gemspec
+- Gemfile
+- README.md
+- bin/puppet-forge-mirror
+- lib/puppet-forge-mirror.rb
+homepage: https://github.com/jhaals/puppet-forge-mirror
+licenses:
+- Apache 2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+- bin
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: 1.9.3
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 4
+summary: Mirror the Puppet Forge by downloading it's modules
+test_files: []