puppet-forge-mirror 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 32be687427cf9f2e8873d6b53e012c0ba4bd8c22
4
+ data.tar.gz: fc766fa83c54c911b7bed3bbbdea1a2ffb340ab1
5
+ SHA512:
6
+ metadata.gz: 699bd9097e43fe4363b3a58016343c797d585595286bd4a62b16220a636cff94091a9a5d1679ef800a61e6bc17924ff422d760a3b53908ab0172600aee6facfa
7
+ data.tar.gz: ff6dfc777f3fb0c2105008df3e65ee3b8e6897d8ba439b7c7b8cdf1739e7dfb92e485697ecb72098011c7e3b7d3dbcd157e3264aba1ee56fddacca51972849de
@@ -0,0 +1,14 @@
1
+ # -*- encoding: utf-8 -*-
2
+ Gem::Specification.new do |s|
3
+ s.name = 'puppet-forge-mirror'
4
+ s.version = '0.0.1'
5
+ s.author = 'Johan Haals'
6
+ s.email = ['johan.haals@gmail.com']
7
+ s.homepage = 'https://github.com/jhaals/puppet-forge-mirror'
8
+ s.summary = "Mirror the Puppet Forge by downloading it's modules"
9
+ s.license = 'Apache 2.0'
10
+ s.files = `git ls-files`.split("\n")
11
+ s.require_paths = ['lib', 'bin']
12
+ s.required_ruby_version = '>= 1.9.3'
13
+ end
14
+
data/Gemfile ADDED
@@ -0,0 +1,2 @@
1
+ source 'https://rubygems.org'
2
+ gem 'puppet-forge-mirror', :path => '.'
@@ -0,0 +1,28 @@
1
+ # puppet-forge-mirror
2
+ simple tool that mirrors the Puppet Forge by downloading it's modules to a local folder on disk.
3
+ It queries the Puppet v3 API and download each module listed under `https://forgeapi.puppetlabs.com/v3/releases`.
4
+
5
+ A module is only downloaded if:
6
+
7
+ * less than 1MB - Some people put binary files in their modules
8
+ * The checksum from the API != checksum of the module already stored on disk.
9
+
10
+ Modules are structured `<author>/<modulename>/<author>-<modulename>-<version>.tar.gz>`
11
+
12
+ ### Usage
13
+
14
+ gem install puppet-forge-mirror
15
+
16
+ $ puppet-forge-mirror /path/to/modules
17
+ downloading https://forgeapi.puppetlabs.com/v3/files/example42-puppi-2.1.9.tar.gz
18
+ downloading https://forgeapi.puppetlabs.com/v3/files/puppetlabs-postgresql-3.3.3.tar.gz
19
+ downloading https://forgeapi.puppetlabs.com/v3/files/puppetlabs-stdlib-4.3.2.tar.gz
20
+ downloading https://forgeapi.puppetlabs.com/v3/files/phundisk-spacewalk-0.1.2.tar.gz
21
+ downloading https://forgeapi.puppetlabs.com/v3/files/puppetlabs-apache-1.0.1.tar.gz
22
+ downloading https://forgeapi.puppetlabs.com/v3/files/puppetlabs-inifile-1.0.3.tar.gz
23
+ downloading https://forgeapi.puppetlabs.com/v3/files/puppetlabs-puppetdb-3.0.1.tar.gz
24
+ downloading https://forgeapi.puppetlabs.com/v3/files/yguenane-repoforge-0.1.0.tar.gz
25
+ downloading https://forgeapi.puppetlabs.com/v3/files/puppetlabs-apt-1.2.0.tar.gz
26
+
27
+ ### Why would I need this?
28
+ One example would be that you're running a huge infrastructure with a masterless Puppet setup.That could generate a several thousand requests which would make Puppetlabs servers very unhappy. Running a local cache internally would increase the download speed and reduce the upstream requests to the forge. The mirror script could for example run once a day and that would only generate ~ 300 HTTP requests.
@@ -0,0 +1,12 @@
1
+ #!/usr/bin/env ruby
2
+ require 'puppet-forge-mirror'
3
+
4
+ if ARGV[0].nil?
5
+ puts "puppet-forge-mirror [module path]"
6
+ exit
7
+ end
8
+
9
+ fm = PuppetForgeMirror.new(
10
+ 'https://forgeapi.puppetlabs.com',
11
+ ARGV[0])
12
+ fm.download_modules
@@ -0,0 +1,73 @@
1
+ require 'net/http'
2
+ require 'json'
3
+ require 'open-uri'
4
+ require 'digest'
5
+ require 'fileutils'
6
+ require 'uri'
7
+
8
+ # Mirror /v3 forges
9
+ class PuppetForgeMirror
10
+ def initialize(forge_url, modules_dir)
11
+ @modules_dir = modules_dir
12
+ @forge_url = forge_url
13
+ end
14
+
15
+ # Perform http GET request, return response.body if status code is 200
16
+ def http_get(url)
17
+ uri = URI(url)
18
+
19
+ Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https') do |http|
20
+ request = Net::HTTP::Get.new uri
21
+ response = http.request request
22
+
23
+ if response.code != '200'
24
+ puts "Error #{response.code}: Failed to get #{uri}"
25
+ return nil
26
+ end
27
+ response.body
28
+ end
29
+ end
30
+
31
+ # Download a specific module
32
+ def download_module(url, mod, owner, filename, checksum)
33
+ module_dir = File.join(@modules_dir, owner, mod)
34
+ filepath = File.join(module_dir, filename)
35
+
36
+ FileUtils.mkdir_p module_dir unless File.exist? module_dir
37
+
38
+ if File.exist? filepath
39
+ return if Digest::MD5.file(filepath).hexdigest == checksum
40
+ end
41
+
42
+ puts "downloading #{url}"
43
+ data = http_get(url)
44
+ return if data.nil?
45
+ open(filepath, 'wb') { |f| f.write(data) }
46
+ end
47
+
48
+ def download_modules
49
+ url = URI.join(@forge_url, '/v3/releases')
50
+ while
51
+ data = http_get(url)
52
+ break if data.nil?
53
+
54
+ r = JSON.parse(data)
55
+ r['results'].each do |m|
56
+ if m['file_size'] > 1024 * 1024 * 1
57
+ puts "#{m['metadata']['name']} is too big, skipping"
58
+ next
59
+ end
60
+ download_module(
61
+ URI.join(@forge_url, m['file_uri']),
62
+ m['module']['name'],
63
+ m['module']['owner']['username'],
64
+ m['file_uri'].split('/').last,
65
+ m['file_md5']
66
+ )
67
+ end
68
+ next_url = r['pagination']['next']
69
+ break if next_url.nil?
70
+ url = URI.join(@forge_url, next_url)
71
+ end
72
+ end
73
+ end
metadata ADDED
@@ -0,0 +1,50 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: puppet-forge-mirror
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Johan Haals
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2014-09-03 00:00:00.000000000 Z
12
+ dependencies: []
13
+ description:
14
+ email:
15
+ - johan.haals@gmail.com
16
+ executables: []
17
+ extensions: []
18
+ extra_rdoc_files: []
19
+ files:
20
+ - .gemspec
21
+ - Gemfile
22
+ - README.md
23
+ - bin/puppet-forge-mirror
24
+ - lib/puppet-forge-mirror.rb
25
+ homepage: https://github.com/jhaals/puppet-forge-mirror
26
+ licenses:
27
+ - Apache 2.0
28
+ metadata: {}
29
+ post_install_message:
30
+ rdoc_options: []
31
+ require_paths:
32
+ - lib
33
+ - bin
34
+ required_ruby_version: !ruby/object:Gem::Requirement
35
+ requirements:
36
+ - - '>='
37
+ - !ruby/object:Gem::Version
38
+ version: 1.9.3
39
+ required_rubygems_version: !ruby/object:Gem::Requirement
40
+ requirements:
41
+ - - '>='
42
+ - !ruby/object:Gem::Version
43
+ version: '0'
44
+ requirements: []
45
+ rubyforge_project:
46
+ rubygems_version: 2.0.14
47
+ signing_key:
48
+ specification_version: 4
49
+ summary: Mirror the Puppet Forge by downloading it's modules
50
+ test_files: []