librarian-puppet-pr328 2.2.1pr328

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,172 @@
1
+ require 'uri'
2
+ require 'net/https'
3
+ require 'open-uri'
4
+ require 'json'
5
+
6
+ require 'librarian/puppet/version'
7
+ require 'librarian/puppet/source/repo'
8
+
9
+ module Librarian
10
+ module Puppet
11
+ module Source
12
+ class GitHubTarball
13
+ class Repo < Librarian::Puppet::Source::Repo
14
+ include Librarian::Puppet::Util
15
+
16
+ TOKEN_KEY = 'GITHUB_API_TOKEN'
17
+
18
+ def versions
19
+ return @versions if @versions
20
+ data = api_call("/repos/#{source.uri}/tags")
21
+ if data.nil?
22
+ raise Error, "Unable to find module '#{source.uri}' on https://github.com"
23
+ end
24
+
25
+ all_versions = data.map { |r| r['name'].gsub(/^v/, '') }.sort.reverse
26
+
27
+ all_versions.delete_if do |version|
28
+ version !~ /\A\d+\.\d+(\.\d+.*)?\z/
29
+ end
30
+
31
+ @versions = all_versions.compact
32
+ debug { " Module #{name} found versions: #{@versions.join(", ")}" }
33
+ @versions
34
+ end
35
+
36
+ def manifests
37
+ versions.map do |version|
38
+ Manifest.new(source, name, version)
39
+ end
40
+ end
41
+
42
+ def install_version!(version, install_path)
43
+ if environment.local? && !vendored?(vendored_name, version)
44
+ raise Error, "Could not find a local copy of #{source.uri} at #{version}."
45
+ end
46
+
47
+ vendor_cache(source.uri.to_s, version) unless vendored?(vendored_name, version)
48
+
49
+ cache_version_unpacked! version
50
+
51
+ if install_path.exist? && rsync? != true
52
+ install_path.rmtree
53
+ end
54
+
55
+ unpacked_path = version_unpacked_cache_path(version).children.first
56
+ cp_r(unpacked_path, install_path)
57
+ end
58
+
59
+ def cache_version_unpacked!(version)
60
+ path = version_unpacked_cache_path(version)
61
+ return if path.directory?
62
+
63
+ path.mkpath
64
+
65
+ target = vendored?(vendored_name, version) ? vendored_path(vendored_name, version) : name
66
+
67
+ Librarian::Posix.run!(%W{tar xzf #{target} -C #{path}})
68
+ end
69
+
70
+ def vendor_cache(name, version)
71
+ clean_up_old_cached_versions(vendored_name(name))
72
+
73
+ url = "https://api.github.com/repos/#{name}/tarball/#{version}"
74
+ add_api_token_to_url(url)
75
+
76
+ environment.vendor!
77
+ File.open(vendored_path(vendored_name(name), version).to_s, 'wb') do |f|
78
+ begin
79
+ debug { "Downloading <#{url}> to <#{f.path}>" }
80
+ open(url,
81
+ "User-Agent" => "librarian-puppet v#{Librarian::Puppet::VERSION}") do |res|
82
+ while buffer = res.read(8192)
83
+ f.write(buffer)
84
+ end
85
+ end
86
+ rescue OpenURI::HTTPError => e
87
+ raise e, "Error requesting <#{url}>: #{e.to_s}"
88
+ end
89
+ end
90
+ end
91
+
92
+ def clean_up_old_cached_versions(name)
93
+ Dir["#{environment.vendor_cache}/#{name}*.tar.gz"].each do |old_version|
94
+ FileUtils.rm old_version
95
+ end
96
+ end
97
+
98
+ def token_key_value
99
+ ENV[TOKEN_KEY]
100
+ end
101
+
102
+ def token_key_nil?
103
+ token_key_value.nil? || token_key_value.empty?
104
+ end
105
+
106
+ def add_api_token_to_url url
107
+ if token_key_nil?
108
+ debug { "#{TOKEN_KEY} environment value is empty or missing" }
109
+ elsif url.include? "?"
110
+ url << "&access_token=#{ENV[TOKEN_KEY]}"
111
+ else
112
+ url << "?access_token=#{ENV[TOKEN_KEY]}"
113
+ end
114
+ url
115
+ end
116
+
117
+ private
118
+
119
+ def api_call(path)
120
+ tags = []
121
+ url = "https://api.github.com#{path}?page=1&per_page=100"
122
+ while true do
123
+ debug { " Module #{name} getting tags at: #{url}" }
124
+ add_api_token_to_url(url)
125
+ response = http_get(url, :headers => {
126
+ "User-Agent" => "librarian-puppet v#{Librarian::Puppet::VERSION}"
127
+ })
128
+
129
+ code, data = response.code.to_i, response.body
130
+
131
+ if code == 200
132
+ tags.concat JSON.parse(data)
133
+ else
134
+ begin
135
+ message = JSON.parse(data)['message']
136
+ if code == 403 && message && message.include?('API rate limit exceeded')
137
+ raise Error, message + " -- increase limit by authenticating via #{TOKEN_KEY}=your-token"
138
+ elsif message
139
+ raise Error, "Error fetching #{url}: [#{code}] #{message}"
140
+ end
141
+ rescue JSON::ParserError
142
+ # response does not return json
143
+ end
144
+ raise Error, "Error fetching #{url}: [#{code}] #{response.body}"
145
+ end
146
+
147
+ # next page
148
+ break if response["link"].nil?
149
+ next_link = response["link"].split(",").select{|l| l.match /rel=.*next.*/}
150
+ break if next_link.empty?
151
+ url = next_link.first.match(/<(.*)>/)[1]
152
+ end
153
+ return tags
154
+ end
155
+
156
+ def http_get(url, options)
157
+ uri = URI.parse(url)
158
+ http = Net::HTTP.new(uri.host, uri.port)
159
+ http.use_ssl = true
160
+ request = Net::HTTP::Get.new(uri.request_uri)
161
+ options[:headers].each { |k, v| request.add_field k, v }
162
+ http.request(request)
163
+ end
164
+
165
+ def vendored_name(name = source.uri.to_s)
166
+ name.sub('/','-')
167
+ end
168
+ end
169
+ end
170
+ end
171
+ end
172
+ end
@@ -0,0 +1,190 @@
1
+ require 'librarian/puppet/util'
2
+
3
+ module Librarian
4
+ module Puppet
5
+ module Source
6
+ module Local
7
+ include Librarian::Puppet::Util
8
+
9
+ def install!(manifest)
10
+ manifest.source == self or raise ArgumentError
11
+
12
+ debug { "Installing #{manifest}" }
13
+
14
+ name, version = manifest.name, manifest.version
15
+ found_path = found_path(name)
16
+
17
+ # We only care about this if we're fetching from a Forge
18
+ if found_path || self.is_a?(Librarian::Puppet::Source::Forge)
19
+ raise Error, "Path for #{name} doesn't contain a puppet module" if found_path.nil?
20
+
21
+ unless name.include? '/' or name.include? '-'
22
+ warn { "Invalid module name '#{name}', you should qualify it with 'ORGANIZATION-#{name}' for resolution to work correctly" }
23
+ end
24
+
25
+ install_path = environment.install_path.join(module_name(name))
26
+ elsif !repository_cached
27
+ raise Error, "Could not find cached version of #{name} for installation"
28
+ else
29
+ found_path = repository_cache_path
30
+ install_path = environment.project_path + path.to_s
31
+ end
32
+
33
+ if install_path.exist? && rsync? != true
34
+ debug { "Deleting #{relative_path_to(install_path)}" }
35
+ install_path.rmtree
36
+ end
37
+
38
+ install_perform_step_copy!(found_path, install_path)
39
+ end
40
+
41
+ def fetch_version(name, extra)
42
+ cache!
43
+ found_path = found_path(name)
44
+ module_version
45
+ end
46
+
47
+ def fetch_dependencies(name, version, extra)
48
+ dependencies = Set.new
49
+
50
+ if specfile?
51
+ spec = environment.dsl(Pathname(specfile))
52
+ dependencies.merge spec.dependencies
53
+ end
54
+
55
+ parsed_metadata['dependencies'].each do |d|
56
+ if environment.use_forge
57
+ gem_requirement = Librarian::Dependency::Requirement.new(d['version_requirement']).to_gem_requirement
58
+ new_dependency = Dependency.new(d['name'], gem_requirement, forge_source)
59
+ dependencies << new_dependency
60
+ end
61
+
62
+ dependencies
63
+ end
64
+
65
+ dependencies
66
+ end
67
+
68
+ def forge_source
69
+ Forge.default
70
+ end
71
+
72
+ private
73
+
74
+ # Naming this method 'version' causes an exception to be raised.
75
+ def module_version
76
+ if parsed_metadata['version']
77
+ parsed_metadata['version']
78
+ else
79
+ warn { "Module #{to_s} does not have version, defaulting to 0.0.1" }
80
+ '0.0.1'
81
+ end
82
+ end
83
+
84
+ def require_puppet
85
+ begin
86
+ require 'puppet'
87
+ require 'puppet/module_tool'
88
+ rescue LoadError
89
+ $stderr.puts <<-EOF
90
+ Unable to load puppet, the puppet gem is required for :git and :path source.
91
+ Install it with: gem install puppet
92
+ EOF
93
+ exit 1
94
+ end
95
+ true
96
+ end
97
+
98
+ def evaluate_modulefile(modulefile)
99
+ @@require_puppet ||= require_puppet
100
+
101
+ metadata = ::Puppet::ModuleTool::Metadata.new
102
+
103
+ # Puppet 4 does not have the class
104
+ unless defined? ::Puppet::ModuleTool::ModulefileReader
105
+ warn { "Can't parse Modulefile in Puppet >= 4.0 and you are using #{Librarian::Puppet::puppet_version}. Ignoring dependencies in #{modulefile}" }
106
+ return metadata
107
+ end
108
+
109
+ begin
110
+ ::Puppet::ModuleTool::ModulefileReader.evaluate(metadata, modulefile)
111
+ raise SyntaxError, "Missing version" unless metadata.version
112
+ rescue ArgumentError, SyntaxError => error
113
+ warn { "Unable to parse #{modulefile}, ignoring: #{error}" }
114
+ if metadata.respond_to? :version=
115
+ metadata.version = '0.0.1' # puppet < 3.6
116
+ else
117
+ metadata.update({'version' => '0.0.1'}) # puppet >= 3.6
118
+ end
119
+ end
120
+ metadata
121
+ end
122
+
123
+ def parsed_metadata
124
+ if @metadata.nil?
125
+ @metadata = if metadata?
126
+ begin
127
+ JSON.parse(File.read(metadata))
128
+ rescue JSON::ParserError => e
129
+ raise Error, "Unable to parse json file #{metadata}: #{e}"
130
+ end
131
+ elsif modulefile?
132
+ # translate Modulefile to metadata.json
133
+ evaluated = evaluate_modulefile(modulefile)
134
+ {
135
+ 'version' => evaluated.version,
136
+ 'dependencies' => evaluated.dependencies.map do |dependency|
137
+ {
138
+ 'name' => dependency.instance_variable_get(:@full_module_name),
139
+ 'version_requirement' => dependency.instance_variable_get(:@version_requirement)
140
+ }
141
+ end
142
+ }
143
+ else
144
+ {}
145
+ end
146
+ @metadata['dependencies'] ||= []
147
+ end
148
+ @metadata
149
+ end
150
+
151
+ def modulefile
152
+ File.join(filesystem_path, 'Modulefile')
153
+ end
154
+
155
+ def modulefile?
156
+ File.exists?(modulefile)
157
+ end
158
+
159
+ def metadata
160
+ File.join(filesystem_path, 'metadata.json')
161
+ end
162
+
163
+ def metadata?
164
+ File.exists?(metadata)
165
+ end
166
+
167
+ def specfile
168
+ File.join(filesystem_path, environment.specfile_name)
169
+ end
170
+
171
+ def specfile?
172
+ File.exists?(specfile)
173
+ end
174
+
175
+ def install_perform_step_copy!(found_path, install_path)
176
+ debug { "Copying #{relative_path_to(found_path)} to #{relative_path_to(install_path)}" }
177
+ cp_r(found_path, install_path)
178
+ end
179
+
180
+ def manifest?(name, path)
181
+ return true if path.join('manifests').exist?
182
+ return true if path.join('lib').join('puppet').exist?
183
+ return true if path.join('lib').join('facter').exist?
184
+ debug { "Could not find manifests, lib/puppet or lib/facter under #{path}, maybe it is not a puppet module" }
185
+ true
186
+ end
187
+ end
188
+ end
189
+ end
190
+ end
@@ -0,0 +1,12 @@
1
+ require 'librarian/source/path'
2
+ require 'librarian/puppet/source/local'
3
+
4
+ module Librarian
5
+ module Puppet
6
+ module Source
7
+ class Path < Librarian::Source::Path
8
+ include Local
9
+ end
10
+ end
11
+ end
12
+ end
@@ -0,0 +1,38 @@
1
+ # parent class for githubtarball and forge source Repos
2
+ module Librarian
3
+ module Puppet
4
+ module Source
5
+ class Repo
6
+
7
+ attr_accessor :source, :name
8
+ private :source=, :name=
9
+
10
+ def initialize(source, name)
11
+ self.source = source
12
+ self.name = name
13
+ end
14
+
15
+ def environment
16
+ source.environment
17
+ end
18
+
19
+ def cache_path
20
+ @cache_path ||= source.cache_path.join(name)
21
+ end
22
+
23
+ def version_unpacked_cache_path(version)
24
+ cache_path.join(version.to_s)
25
+ end
26
+
27
+ def vendored?(name, version)
28
+ vendored_path(name, version).exist?
29
+ end
30
+
31
+ def vendored_path(name, version)
32
+ environment.vendor_cache.join("#{name}-#{version}.tar.gz")
33
+ end
34
+
35
+ end
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,25 @@
1
+ #!/usr/bin/env ruby
2
+ #^syntax detection
3
+
4
+ forge "https://forgeapi.puppetlabs.com"
5
+
6
+ # use dependencies defined in metadata.json
7
+ metadata
8
+
9
+ # use dependencies defined in Modulefile
10
+ # modulefile
11
+
12
+ # A module from the Puppet Forge
13
+ # mod 'puppetlabs-stdlib'
14
+
15
+ # A module from git
16
+ # mod 'puppetlabs-ntp',
17
+ # :git => 'git://github.com/puppetlabs/puppetlabs-ntp.git'
18
+
19
+ # A module from a git branch/tag
20
+ # mod 'puppetlabs-apt',
21
+ # :git => 'https://github.com/puppetlabs/puppetlabs-apt.git',
22
+ # :ref => '1.4.x'
23
+
24
+ # A module from Github pre-packaged tarball
25
+ # mod 'puppetlabs-apache', '0.6.0', :github_tarball => 'puppetlabs/puppetlabs-apache'
@@ -0,0 +1,64 @@
1
+ require 'rsync'
2
+
3
+ module Librarian
4
+ module Puppet
5
+
6
+ module Util
7
+
8
+ def debug(*args, &block)
9
+ environment.logger.debug(*args, &block)
10
+ end
11
+ def info(*args, &block)
12
+ environment.logger.info(*args, &block)
13
+ end
14
+ def warn(*args, &block)
15
+ environment.logger.warn(*args, &block)
16
+ end
17
+
18
+ def rsync?
19
+ environment.config_db.local['rsync'] == 'true'
20
+ end
21
+
22
+ # workaround Issue #173 FileUtils.cp_r will fail if there is a symlink that points to a missing file
23
+ # or when the symlink is copied before the target file when preserve is true
24
+ # see also https://tickets.opscode.com/browse/CHEF-833
25
+ #
26
+ # If the rsync configuration parameter is set, use rsync instead of FileUtils
27
+ def cp_r(src, dest)
28
+ if rsync?
29
+ Rsync.run(File.join(src, "/"), dest, ['-avz', '--delete'])
30
+ else
31
+ begin
32
+ FileUtils.cp_r(src, dest, :preserve => true)
33
+ rescue Errno::ENOENT, Errno::EACCES
34
+ debug { "Failed to copy from #{src} to #{dest} preserving file types, trying again without preserving them" }
35
+ FileUtils.rm_rf(dest)
36
+ FileUtils.cp_r(src, dest)
37
+ end
38
+ end
39
+ end
40
+
41
+ # Remove user and password from a URI object
42
+ def clean_uri(uri)
43
+ new_uri = uri.clone
44
+ new_uri.user = nil
45
+ new_uri.password = nil
46
+ new_uri
47
+ end
48
+
49
+ # normalize module name to use organization-module instead of organization/module
50
+ def normalize_name(name)
51
+ name.sub('/','-')
52
+ end
53
+
54
+ # get the module name from organization-module
55
+ def module_name(name)
56
+ # module name can't have dashes, so let's assume it is everything after the last dash
57
+ name.rpartition('-').last
58
+ end
59
+
60
+ # deprecated
61
+ alias :organization_name :module_name
62
+ end
63
+ end
64
+ end