librarian 0.0.25 → 0.0.26

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. data/.gitignore +4 -0
  2. data/CHANGELOG.md +21 -0
  3. data/README.md +6 -1
  4. data/lib/librarian/action/persist_resolution_mixin.rb +51 -0
  5. data/lib/librarian/action/resolve.rb +3 -38
  6. data/lib/librarian/action/update.rb +4 -38
  7. data/lib/librarian/chef/dsl.rb +1 -0
  8. data/lib/librarian/chef/source.rb +1 -0
  9. data/lib/librarian/chef/source/github.rb +27 -0
  10. data/lib/librarian/chef/source/site.rb +51 -51
  11. data/lib/librarian/cli.rb +31 -23
  12. data/lib/librarian/cli/manifest_presenter.rb +36 -22
  13. data/lib/librarian/dependency.rb +60 -0
  14. data/lib/librarian/environment.rb +13 -1
  15. data/lib/librarian/linter/source_linter.rb +55 -0
  16. data/lib/librarian/lockfile/parser.rb +39 -16
  17. data/lib/librarian/manifest.rb +8 -0
  18. data/lib/librarian/manifest_set.rb +5 -7
  19. data/lib/librarian/mock/source/mock.rb +4 -21
  20. data/lib/librarian/resolution.rb +1 -1
  21. data/lib/librarian/resolver.rb +15 -12
  22. data/lib/librarian/resolver/implementation.rb +166 -75
  23. data/lib/librarian/source/basic_api.rb +45 -0
  24. data/lib/librarian/source/git.rb +4 -22
  25. data/lib/librarian/source/git/repository.rb +1 -1
  26. data/lib/librarian/source/local.rb +0 -7
  27. data/lib/librarian/source/path.rb +4 -22
  28. data/lib/librarian/version.rb +1 -1
  29. data/librarian.gemspec +3 -3
  30. data/spec/functional/chef/source/site_spec.rb +150 -100
  31. data/spec/functional/source/git/repository_spec.rb +2 -1
  32. data/spec/{functional → integration}/chef/source/git_spec.rb +12 -3
  33. data/spec/integration/chef/source/site_spec.rb +217 -0
  34. data/spec/support/cli_macro.rb +4 -12
  35. data/spec/support/method_patch_macro.rb +30 -0
  36. data/spec/unit/config/database_spec.rb +8 -0
  37. data/spec/unit/dependency_spec.rb +176 -0
  38. data/spec/unit/environment_spec.rb +76 -7
  39. data/spec/unit/resolver_spec.rb +2 -2
  40. metadata +52 -46
data/.gitignore CHANGED
@@ -1,5 +1,9 @@
1
1
  *.gem
2
+ bin
3
+ !bin/librarian-chef
4
+ !bin/librarian-mock
2
5
  .bundle
3
6
  Gemfile.lock
4
7
  pkg/*
5
8
  tmp
9
+ vendor
@@ -1,5 +1,26 @@
1
1
  # Change Log
2
2
 
3
+ ## 0.0.26
4
+
5
+ * \#112. Prevent the git source being confused in certain cases by colorized
6
+ output from git. HT: @ericpp.
7
+
8
+ * \#115, \#116. Accommodate cookbook archives generated by `git archive`.
9
+ HT: @taqtiqa-mark.
10
+
11
+ * \#119. Support NO_PROXY, a modifier on HTTP_PROXY and friends. HT: @databus23.
12
+
13
+ * \#121. A github pseudosource. HT: @alno.
14
+
15
+ * \#122. Follow http redirect responses in the chef site source. HT: @Fleurer.
16
+
17
+ * Improve the resolver performance by detecting conflicts earlier and
18
+ backtracking more quickly. This will help avoid certain cases of long
19
+ resolutions caused by conflicts; the conflicts will still be there, but will
20
+ be detected more quickly and the resolution terminated more quickly. This
21
+ changes the resolution algorithm, and new resolutions may differ from older
22
+ resolutions.
23
+
3
24
  ## 0.0.25
4
25
 
5
26
  * \#71. Fix an error, given certain locale settings, with reading cookbook
data/README.md CHANGED
@@ -1,4 +1,4 @@
1
- Librarian [![Build Status](https://secure.travis-ci.org/applicationsonline/librarian.png)](http://travis-ci.org/applicationsonline/librarian)
1
+ Librarian [![Build Status](https://secure.travis-ci.org/applicationsonline/librarian.png)](http://travis-ci.org/applicationsonline/librarian) [![Code Climate](https://codeclimate.com/badge.png)](https://codeclimate.com/github/applicationsonline/librarian)
2
2
  =========
3
3
 
4
4
  Librarian is a framework for writing bundlers, which are tools that resolve,
@@ -136,6 +136,11 @@ we vendored it in our repository into the `cookbooks/` directory for us.
136
136
  The `:path =>` source won't be confused with the `:git =>` source's `:path =>`
137
137
  option.
138
138
 
139
+ Also, there is shortcut for cookbooks hosted on GitHub, so we may write:
140
+
141
+ cookbook "rvm",
142
+ :github => "fnichol/chef-rvm"
143
+
139
144
  ### How to Use
140
145
 
141
146
  Install Librarian-Chef:
@@ -0,0 +1,51 @@
1
+ require "librarian/error"
2
+ require "librarian/spec_change_set"
3
+
4
+ module Librarian
5
+ module Action
6
+ module PersistResolutionMixin
7
+
8
+ private
9
+
10
+ def persist_resolution(resolution)
11
+ resolution.correct? or raise Error,
12
+ "Could not resolve the dependencies."
13
+
14
+ lockfile_text = lockfile.save(resolution)
15
+ debug { "Bouncing #{lockfile_name}" }
16
+ bounced_lockfile_text = lockfile.save(lockfile.load(lockfile_text))
17
+ unless bounced_lockfile_text == lockfile_text
18
+ debug { "lockfile_text: \n#{lockfile_text}" }
19
+ debug { "bounced_lockfile_text: \n#{bounced_lockfile_text}" }
20
+ raise Error, "Cannot bounce #{lockfile_name}!"
21
+ end
22
+ lockfile_path.open('wb') { |f| f.write(lockfile_text) }
23
+ end
24
+
25
+ def specfile_name
26
+ environment.specfile_name
27
+ end
28
+
29
+ def lockfile_name
30
+ environment.lockfile_name
31
+ end
32
+
33
+ def specfile_path
34
+ environment.specfile_path
35
+ end
36
+
37
+ def lockfile_path
38
+ environment.lockfile_path
39
+ end
40
+
41
+ def specfile
42
+ environment.specfile
43
+ end
44
+
45
+ def lockfile
46
+ environment.lockfile
47
+ end
48
+
49
+ end
50
+ end
51
+ end
@@ -1,11 +1,12 @@
1
- require "librarian/error"
2
1
  require "librarian/resolver"
3
2
  require "librarian/spec_change_set"
4
3
  require "librarian/action/base"
4
+ require "librarian/action/persist_resolution_mixin"
5
5
 
6
6
  module Librarian
7
7
  module Action
8
8
  class Resolve < Base
9
+ include PersistResolutionMixin
9
10
 
10
11
  def run
11
12
  if force? || !lockfile_path.exist?
@@ -23,19 +24,7 @@ module Librarian
23
24
  end
24
25
 
25
26
  resolution = resolver.resolve(spec, manifests)
26
- unless resolution.correct?
27
- raise Error, "Could not resolve the dependencies."
28
- else
29
- lockfile_text = lockfile.save(resolution)
30
- debug { "Bouncing #{lockfile_name}" }
31
- bounced_lockfile_text = lockfile.save(lockfile.load(lockfile_text))
32
- unless bounced_lockfile_text == lockfile_text
33
- debug { "lockfile_text: \n#{lockfile_text}"}
34
- debug { "bounced_lockfile_text: \n#{bounced_lockfile_text}"}
35
- raise Error, "Cannot bounce #{lockfile_name}!"
36
- end
37
- lockfile_path.open('wb') { |f| f.write(lockfile_text) }
38
- end
27
+ persist_resolution(resolution)
39
28
  end
40
29
 
41
30
  private
@@ -44,30 +33,6 @@ module Librarian
44
33
  options[:force]
45
34
  end
46
35
 
47
- def specfile_name
48
- environment.specfile_name
49
- end
50
-
51
- def lockfile_name
52
- environment.lockfile_name
53
- end
54
-
55
- def specfile_path
56
- environment.specfile_path
57
- end
58
-
59
- def lockfile_path
60
- environment.lockfile_path
61
- end
62
-
63
- def specfile
64
- environment.specfile
65
- end
66
-
67
- def lockfile
68
- environment.lockfile
69
- end
70
-
71
36
  def resolver
72
37
  Resolver.new(environment)
73
38
  end
@@ -1,12 +1,13 @@
1
- require "librarian/error"
2
1
  require "librarian/manifest_set"
3
2
  require "librarian/resolver"
4
3
  require "librarian/spec_change_set"
5
4
  require "librarian/action/base"
5
+ require "librarian/action/persist_resolution_mixin"
6
6
 
7
7
  module Librarian
8
8
  module Action
9
9
  class Update < Base
10
+ include PersistResolutionMixin
10
11
 
11
12
  def run
12
13
  unless lockfile_path.exist?
@@ -19,20 +20,9 @@ module Librarian
19
20
  partial_manifests = ManifestSet.deep_strip(manifests, dependency_names)
20
21
  unpinnable_sources = previous_resolution.sources - partial_manifests.map(&:source)
21
22
  unpinnable_sources.each(&:unpin!)
23
+
22
24
  resolution = resolver.resolve(spec, partial_manifests)
23
- unless resolution.correct?
24
- raise Error, "Could not resolve the dependencies."
25
- else
26
- lockfile_text = lockfile.save(resolution)
27
- debug { "Bouncing #{lockfile_name}" }
28
- bounced_lockfile_text = lockfile.save(lockfile.load(lockfile_text))
29
- unless bounced_lockfile_text == lockfile_text
30
- debug { "lockfile_text: \n#{lockfile_text}"}
31
- debug { "bounced_lockfile_text: \n#{bounced_lockfile_text}"}
32
- raise Error, "Cannot bounce #{lockfile_name}!"
33
- end
34
- lockfile_path.open('wb') { |f| f.write(lockfile_text) }
35
- end
25
+ persist_resolution(resolution)
36
26
  end
37
27
 
38
28
  private
@@ -41,30 +31,6 @@ module Librarian
41
31
  options[:names]
42
32
  end
43
33
 
44
- def specfile_name
45
- environment.specfile_name
46
- end
47
-
48
- def lockfile_name
49
- environment.lockfile_name
50
- end
51
-
52
- def specfile_path
53
- environment.specfile_path
54
- end
55
-
56
- def lockfile_path
57
- environment.lockfile_path
58
- end
59
-
60
- def specfile
61
- environment.specfile
62
- end
63
-
64
- def lockfile
65
- environment.lockfile
66
- end
67
-
68
34
  def resolver
69
35
  Resolver.new(environment)
70
36
  end
@@ -9,6 +9,7 @@ module Librarian
9
9
 
10
10
  source :site => Source::Site
11
11
  source :git => Source::Git
12
+ source :github => Source::Github
12
13
  source :path => Source::Path
13
14
  end
14
15
  end
@@ -1,3 +1,4 @@
1
1
  require 'librarian/chef/source/path'
2
2
  require 'librarian/chef/source/git'
3
+ require 'librarian/chef/source/github'
3
4
  require 'librarian/chef/source/site'
@@ -0,0 +1,27 @@
1
+ require 'librarian/chef/source/git'
2
+
3
+ module Librarian
4
+ module Chef
5
+ module Source
6
+ class Github
7
+
8
+ class << self
9
+
10
+ def lock_name
11
+ Git.lock_name
12
+ end
13
+
14
+ def from_lock_options(environment, options)
15
+ Git.from_lock_options(environment, options)
16
+ end
17
+
18
+ def from_spec_args(environment, uri, options)
19
+ Git.from_spec_args(environment, "https://github.com/#{uri}", options)
20
+ end
21
+
22
+ end
23
+
24
+ end
25
+ end
26
+ end
27
+ end
@@ -8,6 +8,7 @@ require 'zlib'
8
8
  require 'securerandom'
9
9
  require 'archive/tar/minitar'
10
10
 
11
+ require 'librarian/source/basic_api'
11
12
  require 'librarian/chef/manifest_reader'
12
13
 
13
14
  module Librarian
@@ -76,8 +77,7 @@ module Librarian
76
77
  end
77
78
 
78
79
  def version_uri_metadata(version_uri)
79
- @version_uri_metadata ||= { }
80
- @version_uri_metadata[version_uri.to_s] ||= begin
80
+ memo(__method__, version_uri.to_s) do
81
81
  cache_version_uri_metadata! version_uri
82
82
  parse_local_json(version_uri_metadata_cache_path(version_uri))
83
83
  end
@@ -89,8 +89,7 @@ module Librarian
89
89
  end
90
90
 
91
91
  def version_uri_manifest(version_uri)
92
- @version_uri_manifest ||= { }
93
- @version_uri_manifest[version_uri.to_s] ||= begin
92
+ memo(__method__, version_uri.to_s) do
94
93
  cache_version_uri_unpacked! version_uri
95
94
  unpacked_path = version_uri_unpacked_cache_path(version_uri)
96
95
  manifest_path = ManifestReader.manifest_path(unpacked_path)
@@ -106,8 +105,7 @@ module Librarian
106
105
  end
107
106
 
108
107
  def to_version_uri(version)
109
- @to_version_uri ||= { }
110
- @to_version_uri[version.to_s] ||= begin
108
+ memo(__method__, version.to_s) do
111
109
  cache_version! version
112
110
  version_cache_path(version).read
113
111
  end
@@ -126,15 +124,13 @@ module Librarian
126
124
  end
127
125
 
128
126
  def version_cache_path(version)
129
- @version_cache_path ||= { }
130
- @version_cache_path[version.to_s] ||= begin
127
+ memo(__method__, version.to_s) do
131
128
  cache_path.join("version").join(version.to_s)
132
129
  end
133
130
  end
134
131
 
135
132
  def version_uri_cache_path(version_uri)
136
- @version_uri_cache_path ||= { }
137
- @version_uri_cache_path[version_uri.to_s] ||= begin
133
+ memo(__method__, version_uri.to_s) do
138
134
  cache_path.join("version-uri").join(hexdigest(version_uri))
139
135
  end
140
136
  end
@@ -145,8 +141,7 @@ module Librarian
145
141
  end
146
142
 
147
143
  def version_uri_metadata_cache_path(version_uri)
148
- @version_uri_metadata_cache_path ||= { }
149
- @version_uri_metadata_cache_path[version_uri.to_s] ||= begin
144
+ memo(__method__, version_uri.to_s) do
150
145
  version_uri_cache_path(version_uri).join("metadata.json")
151
146
  end
152
147
  end
@@ -157,8 +152,7 @@ module Librarian
157
152
  end
158
153
 
159
154
  def version_uri_package_cache_path(version_uri)
160
- @version_uri_package_cache_path ||= { }
161
- @version_uri_package_cache_path[version_uri.to_s] ||= begin
155
+ memo(__method__, version_uri.to_s) do
162
156
  version_uri_cache_path(version_uri).join("package.tar.gz")
163
157
  end
164
158
  end
@@ -169,8 +163,7 @@ module Librarian
169
163
  end
170
164
 
171
165
  def version_uri_unpacked_cache_path(version_uri)
172
- @version_uri_unpacked_cache_path ||= { }
173
- @version_uri_unpacked_cache_path[version_uri.to_s] ||= begin
166
+ memo(__method__, version_uri.to_s) do
174
167
  version_uri_cache_path(version_uri).join("package")
175
168
  end
176
169
  end
@@ -243,9 +236,6 @@ module Librarian
243
236
  debug { "Caching #{uri} to #{path}" }
244
237
 
245
238
  response = http_get(uri)
246
- unless Net::HTTPSuccess === response
247
- raise Error, "Could not get #{uri} because #{response.code} #{response.message}!"
248
- end
249
239
 
250
240
  object = response.body
251
241
  case type
@@ -273,9 +263,11 @@ module Librarian
273
263
  end
274
264
 
275
265
  # Cookbook files, as pulled from Opscode Community Site API, are
276
- # embedded in a subdirectory of the tarball.
266
+ # embedded in a subdirectory of the tarball. If created by git archive they
267
+ # can include the subfolder `pax_global_header`, which is ignored.
277
268
  subtemps = temp.children
278
269
  subtemps.empty? and raise "The package archive was empty!"
270
+ subtemps.delete_if{|pth| pth.to_s[/pax_global_header/]}
279
271
  subtemps.size > 1 and raise "The package archive has too many children!"
280
272
  subtemp = subtemps.first
281
273
  debug { "Moving #{relative_path_to(subtemp)} to #{relative_path_to(path)}" }
@@ -306,40 +298,55 @@ module Librarian
306
298
  end
307
299
 
308
300
  def http(uri)
309
- environment.net_http_class.new(uri.host, uri.port)
301
+ environment.net_http_class(uri.host).new(uri.host, uri.port)
310
302
  end
311
303
 
312
304
  def http_get(uri)
313
- http = http(uri)
314
- request = Net::HTTP::Get.new(uri.path)
315
- response = http.start{|http| http.request(request)}
316
-
317
- response
305
+ max_redirects = 10
306
+ redirects = []
307
+
308
+ loop do
309
+ debug { "Performing http-get for #{uri}" }
310
+ http = http(uri)
311
+ request = Net::HTTP::Get.new(uri.path)
312
+ response = http.start{|http| http.request(request)}
313
+
314
+ case response
315
+ when Net::HTTPSuccess
316
+ debug { "Responded with success" }
317
+ return response
318
+ when Net::HTTPRedirection
319
+ location = response["Location"]
320
+ debug { "Responded with redirect to #{uri}" }
321
+ redirects.size > max_redirects and raise Error,
322
+ "Could not get #{uri} because too many redirects!"
323
+ redirects.include?(location) and raise Error,
324
+ "Could not get #{uri} because redirect cycle!"
325
+ redirects << location
326
+ uri = URI.parse(location)
327
+ # continue the loop
328
+ else
329
+ raise Error, "Could not get #{uri} because #{response.code} #{response.message}!"
330
+ end
331
+ end
318
332
  end
319
333
 
320
- end
321
-
322
- class << self
323
-
324
- LOCK_NAME = 'SITE'
325
-
326
- def lock_name
327
- LOCK_NAME
328
- end
334
+ def memo(method, *path)
335
+ ivar = "@#{method}".to_sym
336
+ unless memo = instance_variable_get(ivar)
337
+ memo = instance_variable_set(ivar, { })
338
+ end
329
339
 
330
- def from_lock_options(environment, options)
331
- new(environment, options[:remote], options.reject{|k, v| k == :remote})
340
+ memo.key?(path) or memo[path] = yield
341
+ memo[path]
332
342
  end
333
343
 
334
- def from_spec_args(environment, uri, options)
335
- recognized_options = []
336
- unrecognized_options = options.keys - recognized_options
337
- unrecognized_options.empty? or raise Error, "unrecognized options: #{unrecognized_options.join(", ")}"
344
+ end
338
345
 
339
- new(environment, uri, options)
340
- end
346
+ include Librarian::Source::BasicApi
341
347
 
342
- end
348
+ lock_name 'SITE'
349
+ spec_options []
343
350
 
344
351
  attr_accessor :environment, :uri
345
352
  private :environment=, :uri=
@@ -395,13 +402,6 @@ module Librarian
395
402
  line(name).manifests
396
403
  end
397
404
 
398
- def manifest(name, version, dependencies)
399
- manifest = Manifest.new(self, name)
400
- manifest.version = version
401
- manifest.dependencies = dependencies
402
- manifest
403
- end
404
-
405
405
  def cache_path
406
406
  @cache_path ||= begin
407
407
  dir = Digest::MD5.hexdigest(uri)