chef-dk 0.2.1 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. checksums.yaml +4 -4
  2. data/CONTRIBUTING.md +2 -2
  3. data/README.md +25 -0
  4. data/lib/chef-dk/builtin_commands.rb +4 -0
  5. data/lib/chef-dk/cli.rb +46 -0
  6. data/lib/chef-dk/command/base.rb +4 -0
  7. data/lib/chef-dk/command/generator_commands/template.rb +2 -1
  8. data/lib/chef-dk/command/install.rb +105 -0
  9. data/lib/chef-dk/command/push.rb +123 -0
  10. data/lib/chef-dk/cookbook_profiler/identifiers.rb +5 -0
  11. data/lib/chef-dk/exceptions.rb +38 -0
  12. data/lib/chef-dk/generator.rb +16 -1
  13. data/lib/chef-dk/helpers.rb +1 -1
  14. data/lib/chef-dk/policyfile/cookbook_location_specification.rb +4 -0
  15. data/lib/chef-dk/policyfile/cookbook_locks.rb +73 -0
  16. data/lib/chef-dk/policyfile/reports/install.rb +70 -0
  17. data/lib/chef-dk/policyfile/reports/table_printer.rb +58 -0
  18. data/lib/chef-dk/policyfile/reports/upload.rb +70 -0
  19. data/lib/chef-dk/policyfile/solution_dependencies.rb +102 -8
  20. data/lib/chef-dk/policyfile/uploader.rb +37 -6
  21. data/lib/chef-dk/policyfile_compiler.rb +19 -5
  22. data/lib/chef-dk/policyfile_lock.rb +122 -9
  23. data/lib/chef-dk/policyfile_services/install.rb +131 -0
  24. data/lib/chef-dk/policyfile_services/push.rb +121 -0
  25. data/lib/chef-dk/skeletons/code_generator/recipes/cookbook.rb +6 -4
  26. data/lib/chef-dk/ui.rb +50 -0
  27. data/lib/chef-dk/version.rb +1 -1
  28. data/spec/shared/a_file_generator.rb +4 -1
  29. data/spec/test_helpers.rb +21 -0
  30. data/spec/unit/cli_spec.rb +100 -1
  31. data/spec/unit/command/base_spec.rb +23 -0
  32. data/spec/unit/command/exec_spec.rb +2 -2
  33. data/spec/unit/command/install_spec.rb +159 -0
  34. data/spec/unit/command/push_spec.rb +203 -0
  35. data/spec/unit/command/shell_init_spec.rb +1 -1
  36. data/spec/unit/policyfile/cookbook_location_specification_spec.rb +7 -0
  37. data/spec/unit/policyfile/cookbook_locks_spec.rb +13 -2
  38. data/spec/unit/policyfile/reports/install_spec.rb +115 -0
  39. data/spec/unit/policyfile/reports/upload_spec.rb +96 -0
  40. data/spec/unit/policyfile/solution_dependencies_spec.rb +1 -1
  41. data/spec/unit/policyfile/uploader_spec.rb +9 -12
  42. data/spec/unit/policyfile_lock_serialization_spec.rb +292 -0
  43. data/spec/unit/policyfile_services/install_spec.rb +170 -0
  44. data/spec/unit/policyfile_services/push_spec.rb +202 -0
  45. metadata +48 -6
@@ -18,24 +18,35 @@
18
18
  require 'chef/cookbook_uploader'
19
19
  require 'chef-dk/policyfile/read_cookbook_for_compat_mode_upload'
20
20
 
21
+ require 'chef-dk/ui'
22
+ require 'chef-dk/policyfile/reports/upload'
23
+
21
24
  module ChefDK
22
25
  module Policyfile
23
26
  class Uploader
24
27
 
28
+ LockedCookbookForUpload = Struct.new(:cookbook, :lock)
29
+
25
30
  COMPAT_MODE_DATA_BAG_NAME = "policyfiles".freeze
26
31
 
27
32
  attr_reader :policyfile_lock
28
33
  attr_reader :policy_group
29
34
  attr_reader :http_client
35
+ attr_reader :ui
30
36
 
31
- def initialize(policyfile_lock, policy_group, http_client: nil)
37
+ def initialize(policyfile_lock, policy_group, ui: nil, http_client: nil)
32
38
  @policyfile_lock = policyfile_lock
33
39
  @policy_group = policy_group
34
40
  @http_client = http_client
41
+ @ui = ui || UI.null
42
+
43
+ @cookbook_versions_for_policy = nil
35
44
  end
36
45
 
37
46
  def upload
38
- uploader.upload_cookbooks
47
+ ui.msg("WARN: Uploading policy to policy group #{policy_group} in compatibility mode")
48
+
49
+ upload_cookbooks
39
50
  data_bag_create
40
51
  data_bag_item_create
41
52
  end
@@ -63,6 +74,8 @@ module ChefDK
63
74
  }
64
75
 
65
76
  upload_lockfile_as_data_bag_item(policy_id, data_item)
77
+ ui.msg("Policy uploaded as data bag item #{COMPAT_MODE_DATA_BAG_NAME}/#{policy_id}")
78
+ true
66
79
  end
67
80
 
68
81
  def uploader
@@ -71,8 +84,10 @@ module ChefDK
71
84
  end
72
85
 
73
86
  def cookbook_versions_to_upload
74
- cookbook_versions_for_policy.reject do |cookbook|
75
- remote_already_has_cookbook?(cookbook)
87
+ cookbook_versions_for_policy.inject([]) do |versions_to_upload, cookbook_with_lock|
88
+ cb = cookbook_with_lock.cookbook
89
+ versions_to_upload << cb unless remote_already_has_cookbook?(cb)
90
+ versions_to_upload
76
91
  end
77
92
  end
78
93
 
@@ -91,14 +106,30 @@ module ChefDK
91
106
  # An Array of Chef::CookbookVersion objects representing the full set that
92
107
  # the policyfile lock requires.
93
108
  def cookbook_versions_for_policy
109
+ return @cookbook_versions_for_policy if @cookbook_versions_for_policy
94
110
  policyfile_lock.validate_cookbooks!
95
- policyfile_lock.cookbook_locks.map do |name, lock|
96
- ReadCookbookForCompatModeUpload.load(name, lock.dotted_decimal_identifier, lock.cookbook_path)
111
+ @cookbook_versions_for_policy = policyfile_lock.cookbook_locks.map do |name, lock|
112
+ cb = ReadCookbookForCompatModeUpload.load(name, lock.dotted_decimal_identifier, lock.cookbook_path)
113
+ LockedCookbookForUpload.new(cb, lock)
97
114
  end
98
115
  end
99
116
 
100
117
  private
101
118
 
119
+ def upload_cookbooks
120
+ ui.msg("WARN: Uploading cookbooks using semver compat mode")
121
+
122
+ uploader.upload_cookbooks
123
+
124
+ reused_cbs, uploaded_cbs = cookbook_versions_for_policy.partition do |cb_with_lock|
125
+ remote_already_has_cookbook?(cb_with_lock.cookbook)
126
+ end
127
+
128
+ Reports::Upload.new(reused_cbs: reused_cbs, uploaded_cbs: uploaded_cbs, ui: ui).show
129
+
130
+ true
131
+ end
132
+
102
133
  def upload_lockfile_as_data_bag_item(policy_id, data_item)
103
134
  http_client.put("data/#{COMPAT_MODE_DATA_BAG_NAME}/#{policy_id}", data_item)
104
135
  rescue Net::HTTPServerException => e
@@ -22,6 +22,8 @@ require 'chef/run_list'
22
22
 
23
23
  require 'chef-dk/policyfile/dsl'
24
24
  require 'chef-dk/policyfile_lock'
25
+ require 'chef-dk/ui'
26
+ require 'chef-dk/policyfile/reports/install'
25
27
 
26
28
  module ChefDK
27
29
 
@@ -34,8 +36,8 @@ module ChefDK
34
36
  # Cookbooks from these sources lock that cookbook to exactly one version
35
37
  SOURCE_TYPES_WITH_FIXED_VERSIONS = [:git, :path].freeze
36
38
 
37
- def self.evaluate(policyfile_string, policyfile_filename)
38
- compiler = new
39
+ def self.evaluate(policyfile_string, policyfile_filename, ui: nil)
40
+ compiler = new(ui: ui)
39
41
  compiler.evaluate_policyfile(policyfile_string, policyfile_filename)
40
42
  compiler
41
43
  end
@@ -48,11 +50,15 @@ module ChefDK
48
50
 
49
51
  attr_reader :dsl
50
52
  attr_reader :storage_config
53
+ attr_reader :install_report
51
54
 
52
- def initialize
55
+ def initialize(ui: nil)
53
56
  @storage_config = Policyfile::StorageConfig.new
54
57
  @dsl = Policyfile::DSL.new(storage_config)
55
58
  @artifact_server_cookbook_location_specs = {}
59
+
60
+ @ui = ui || UI.null
61
+ @install_report = Policyfile::Reports::Install.new(ui: @ui, policyfile_compiler: self)
56
62
  end
57
63
 
58
64
  def error!
@@ -86,6 +92,7 @@ module ChefDK
86
92
  spec = cookbook_location_spec_for(cookbook_name)
87
93
  if spec.nil? or !spec.version_fixed?
88
94
  spec = create_spec_for_cookbook(cookbook_name, version)
95
+ install_report.installing_cookbook(spec)
89
96
  spec.ensure_cached
90
97
  end
91
98
  end
@@ -213,6 +220,12 @@ module ChefDK
213
220
  self
214
221
  end
215
222
 
223
+ def fixed_version_cookbooks_specs
224
+ @fixed_version_cookbooks_specs ||= cookbook_location_specs.select do |_cookbook_name, cookbook_location_spec|
225
+ cookbook_location_spec.version_fixed?
226
+ end
227
+ end
228
+
216
229
  private
217
230
 
218
231
  def normalize_recipe(run_list_item)
@@ -228,8 +241,9 @@ module ChefDK
228
241
  def cache_fixed_version_cookbooks
229
242
  ensure_cache_dir_exists
230
243
 
231
- cookbook_location_specs.each do |_cookbook_name, cookbook_location_spec|
232
- cookbook_location_spec.ensure_cached if cookbook_location_spec.version_fixed?
244
+ fixed_version_cookbooks_specs.each do |name, cookbook_location_spec|
245
+ install_report.installing_fixed_version_cookbook(cookbook_location_spec)
246
+ cookbook_location_spec.ensure_cached
233
247
  end
234
248
  end
235
249
 
@@ -18,10 +18,51 @@
18
18
  require 'chef-dk/policyfile/storage_config'
19
19
  require 'chef-dk/policyfile/cookbook_locks'
20
20
  require 'chef-dk/policyfile/solution_dependencies'
21
+ require 'chef-dk/ui'
21
22
 
22
23
  module ChefDK
24
+
23
25
  class PolicyfileLock
24
26
 
27
+ class InstallReport
28
+
29
+ attr_reader :ui
30
+ attr_reader :policyfile_lock
31
+
32
+ def initialize(ui: ui, policyfile_lock: nil)
33
+ @ui = ui
34
+ @policyfile_lock = policyfile_lock
35
+
36
+ @cookbook_name_width = nil
37
+ @cookbook_version_width = nil
38
+ end
39
+
40
+ def installing_fixed_version_cookbook(cookbook_spec)
41
+ verb = cookbook_spec.installed? ? "Using " : "Installing"
42
+ ui.msg("#{verb} #{format_fixed_version_cookbook(cookbook_spec)}")
43
+ end
44
+
45
+ def installing_cookbook(cookbook_lock)
46
+ verb = cookbook_lock.installed? ? "Using " : "Installing"
47
+ ui.msg("#{verb} #{format_cookbook(cookbook_lock)}")
48
+ end
49
+
50
+ private
51
+
52
+ def format_cookbook(cookbook_lock)
53
+ "#{cookbook_lock.name.ljust(cookbook_name_width)} #{cookbook_lock.version.to_s.ljust(cookbook_version_width)}"
54
+ end
55
+
56
+ def cookbook_name_width
57
+ policyfile_lock.cookbook_locks.map { |name, _| name.size }.max
58
+ end
59
+
60
+ def cookbook_version_width
61
+ policyfile_lock.cookbook_locks.map { |_, lock| lock.version.size }.max
62
+ end
63
+ end
64
+
65
+ RUN_LIST_ITEM_FORMAT = /\Arecipe\[[^\s]+::[^\s]+\]\Z/.freeze
25
66
 
26
67
  def self.build(storage_config)
27
68
  lock = new(storage_config)
@@ -46,14 +87,18 @@ module ChefDK
46
87
 
47
88
  attr_reader :cookbook_locks
48
89
 
49
- def initialize(storage_config)
90
+ attr_reader :install_report
91
+
92
+ def initialize(storage_config, ui: nil)
50
93
  @name = nil
51
94
  @run_list = []
52
95
  @cookbook_locks = {}
53
96
  @relative_paths_root = Dir.pwd
54
97
  @storage_config = storage_config
98
+ @ui = ui || UI.null
55
99
 
56
100
  @solution_dependencies = Policyfile::SolutionDependencies.new
101
+ @install_report = InstallReport.new(ui: @ui, policyfile_lock: self)
57
102
  end
58
103
 
59
104
  def lock_data_for(cookbook_name)
@@ -146,14 +191,10 @@ module ChefDK
146
191
  end
147
192
 
148
193
  def build_from_lock_data(lock_data)
149
- @name = lock_data["name"]
150
- @run_list = lock_data["run_list"]
151
- lock_data["cookbook_locks"].each do |name, lock_info|
152
- build_cookbook_lock_from_lock_data(name, lock_info)
153
- end
154
-
155
- s = Policyfile::SolutionDependencies.from_lock(lock_data["solution_dependencies"])
156
- @solution_dependencies = s
194
+ set_name_from_lock_data(lock_data)
195
+ set_run_list_from_lock_data(lock_data)
196
+ set_cookbook_locks_from_lock_data(lock_data)
197
+ set_solution_dependencies_from_lock_data(lock_data)
157
198
  self
158
199
  end
159
200
 
@@ -162,6 +203,7 @@ module ChefDK
162
203
  ensure_cache_dir_exists
163
204
 
164
205
  cookbook_locks.each do |cookbook_name, cookbook_lock|
206
+ install_report.installing_cookbook(cookbook_lock)
165
207
  cookbook_lock.install_locked
166
208
  end
167
209
  end
@@ -175,7 +217,78 @@ module ChefDK
175
217
 
176
218
  private
177
219
 
220
+ def set_name_from_lock_data(lock_data)
221
+ name_attribute = lock_data["name"]
222
+
223
+ raise InvalidLockfile, "lockfile does not have a `name' attribute" if name_attribute.nil?
224
+
225
+ unless name_attribute.kind_of?(String)
226
+ raise InvalidLockfile, "lockfile's name attribute must be a String (got: #{name_attribute.inspect})"
227
+ end
228
+
229
+ if name_attribute.empty?
230
+ raise InvalidLockfile, "lockfile's name attribute cannot be an empty string"
231
+ end
232
+
233
+ @name = name_attribute
234
+
235
+ end
236
+
237
+ def set_run_list_from_lock_data(lock_data)
238
+ run_list_attribute = lock_data["run_list"]
239
+
240
+ raise InvalidLockfile, "lockfile does not have a run_list attribute" if run_list_attribute.nil?
241
+
242
+ unless run_list_attribute.kind_of?(Array)
243
+ raise InvalidLockfile, "lockfile's run_list must be an array of run list items (got: #{run_list_attribute.inspect})"
244
+ end
245
+
246
+ bad_run_list_items = run_list_attribute.select { |e| e !~ RUN_LIST_ITEM_FORMAT }
247
+
248
+ unless bad_run_list_items.empty?
249
+ msg = "lockfile's run_list items must be formatted like `recipe[$COOKBOOK_NAME::$RECIPE_NAME]'. Invalid items: `#{bad_run_list_items.join("' `")}'"
250
+ raise InvalidLockfile, msg
251
+ end
252
+
253
+ @run_list = run_list_attribute
254
+ end
255
+
256
+ def set_cookbook_locks_from_lock_data(lock_data)
257
+ cookbook_lock_data = lock_data["cookbook_locks"]
258
+
259
+ if cookbook_lock_data.nil?
260
+ raise InvalidLockfile, "lockfile does not have a cookbook_locks attribute"
261
+ end
262
+
263
+ unless cookbook_lock_data.kind_of?(Hash)
264
+ raise InvalidLockfile, "lockfile's cookbook_locks attribute must be a Hash (JSON object). (got: #{cookbook_lock_data.inspect})"
265
+ end
266
+
267
+ lock_data["cookbook_locks"].each do |name, lock_info|
268
+ build_cookbook_lock_from_lock_data(name, lock_info)
269
+ end
270
+ end
271
+
272
+ def set_solution_dependencies_from_lock_data(lock_data)
273
+ soln_deps = lock_data["solution_dependencies"]
274
+
275
+ if soln_deps.nil?
276
+ raise InvalidLockfile, "lockfile does not have a solution_dependencies attribute"
277
+ end
278
+
279
+ unless soln_deps.kind_of?(Hash)
280
+ raise InvalidLockfile, "lockfile's solution_dependencies attribute must be a Hash (JSON object). (got: #{soln_deps.inspect})"
281
+ end
282
+
283
+ s = Policyfile::SolutionDependencies.from_lock(lock_data["solution_dependencies"])
284
+ @solution_dependencies = s
285
+ end
286
+
178
287
  def build_cookbook_lock_from_lock_data(name, lock_info)
288
+ unless lock_info.kind_of?(Hash)
289
+ raise InvalidLockfile, "lockfile cookbook_locks entries must be a Hash (JSON object). (got: #{lock_info.inspect})"
290
+ end
291
+
179
292
  if lock_info["cache_key"].nil?
180
293
  local_cookbook(name).build_from_lock_data(lock_info)
181
294
  else
@@ -0,0 +1,131 @@
1
+ #
2
+ # Copyright:: Copyright (c) 2014 Chef Software Inc.
3
+ # License:: Apache License, Version 2.0
4
+ #
5
+ # Licensed under the Apache License, Version 2.0 (the "License");
6
+ # you may not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+ #
17
+
18
+ require 'chef-dk/exceptions'
19
+ require 'chef-dk/policyfile_compiler'
20
+ require 'chef-dk/policyfile/storage_config'
21
+ require 'chef-dk/policyfile_lock'
22
+
23
+ module ChefDK
24
+ module PolicyfileServices
25
+
26
+ class Install
27
+
28
+ attr_reader :root_dir
29
+ attr_reader :ui
30
+
31
+ def initialize(policyfile: nil, ui: nil, root_dir: nil)
32
+ @policyfile_relative_path = policyfile
33
+ @ui = ui
34
+ @root_dir = root_dir
35
+
36
+ @policyfile_content = nil
37
+ @policyfile_compiler = nil
38
+ end
39
+
40
+ def run
41
+ unless File.exist?(policyfile_path)
42
+ # TODO: suggest next step. Add a generator/init command? Specify path to Policyfile.rb?
43
+ # See card CC-232
44
+ raise PolicyfileNotFound, "Policyfile not found at path #{policyfile_path}"
45
+ end
46
+
47
+ if File.exist?(lockfile_path)
48
+ install_from_lock
49
+ else
50
+ generate_lock_and_install
51
+ end
52
+ end
53
+
54
+ def policyfile_relative_path
55
+ @policyfile_relative_path || "Policyfile.rb"
56
+ end
57
+
58
+ def policyfile_path
59
+ File.expand_path(policyfile_relative_path, root_dir)
60
+ end
61
+
62
+ def lockfile_relative_path
63
+ policyfile_relative_path.gsub(/\.rb\Z/, '') + ".lock.json"
64
+ end
65
+
66
+ def lockfile_path
67
+ File.expand_path(lockfile_relative_path, root_dir)
68
+ end
69
+
70
+ def policyfile_content
71
+ @policyfile_content ||= IO.read(policyfile_path)
72
+ end
73
+
74
+ def policyfile_compiler
75
+ @policyfile_compiler ||= ChefDK::PolicyfileCompiler.evaluate(policyfile_content, policyfile_path, ui: ui)
76
+ end
77
+
78
+ def expanded_run_list
79
+ policyfile_compiler.expanded_run_list.to_s
80
+ end
81
+
82
+ def policyfile_lock_content
83
+ @policyfile_lock_content ||= IO.read(lockfile_path) if File.exist?(lockfile_path)
84
+ end
85
+
86
+ def policyfile_lock
87
+ return nil if policyfile_lock_content.nil?
88
+ @policyfile_lock ||= begin
89
+ lock_data = FFI_Yajl::Parser.new.parse(policyfile_lock_content)
90
+ PolicyfileLock.new(storage_config, ui: ui).build_from_lock_data(lock_data)
91
+ end
92
+ end
93
+
94
+ def storage_config
95
+ @storage_config ||= Policyfile::StorageConfig.new(relative_paths_root: root_dir)
96
+ end
97
+
98
+ def generate_lock_and_install
99
+ policyfile_compiler.error!
100
+
101
+ ui.msg "Building policy #{policyfile_compiler.name}"
102
+ ui.msg "Expanded run list: " + expanded_run_list + "\n"
103
+
104
+ ui.msg "Caching Cookbooks..."
105
+
106
+ policyfile_compiler.install
107
+
108
+ lock_data = policyfile_compiler.lock.to_lock
109
+
110
+ File.open(lockfile_path, "w+") do |f|
111
+ f.print(FFI_Yajl::Encoder.encode(lock_data, pretty: true ))
112
+ end
113
+
114
+ ui.msg ""
115
+
116
+ ui.msg "Lockfile written to #{lockfile_path}"
117
+ rescue => error
118
+ raise PolicyfileInstallError.new("Failed to generate Policyfile.lock", error)
119
+ end
120
+
121
+ def install_from_lock
122
+ ui.msg "Installing cookbooks from lock"
123
+
124
+ policyfile_lock.install_cookbooks
125
+ rescue => error
126
+ raise PolicyfileInstallError.new("Failed to install cookbooks from lockfile", error)
127
+ end
128
+
129
+ end
130
+ end
131
+ end