chef-dk 0.2.1 → 0.3.0

data/lib/chef-dk/policyfile/uploader.rb

@@ -18,24 +18,35 @@
 require 'chef/cookbook_uploader'
 require 'chef-dk/policyfile/read_cookbook_for_compat_mode_upload'
 
+require 'chef-dk/ui'
+require 'chef-dk/policyfile/reports/upload'
+
 module ChefDK
   module Policyfile
     class Uploader
 
+      LockedCookbookForUpload = Struct.new(:cookbook, :lock)
+
       COMPAT_MODE_DATA_BAG_NAME = "policyfiles".freeze
 
       attr_reader :policyfile_lock
       attr_reader :policy_group
       attr_reader :http_client
+      attr_reader :ui
 
-      def initialize(policyfile_lock, policy_group, http_client: nil)
+      def initialize(policyfile_lock, policy_group, ui: nil, http_client: nil)
         @policyfile_lock = policyfile_lock
         @policy_group = policy_group
         @http_client = http_client
+        @ui = ui || UI.null
+
+        @cookbook_versions_for_policy = nil
       end
 
       def upload
-        uploader.upload_cookbooks
+        ui.msg("WARN: Uploading policy to policy group #{policy_group} in compatibility mode")
+
+        upload_cookbooks
         data_bag_create
         data_bag_item_create
       end
@@ -63,6 +74,8 @@ module ChefDK
         }
 
         upload_lockfile_as_data_bag_item(policy_id, data_item)
+        ui.msg("Policy uploaded as data bag item #{COMPAT_MODE_DATA_BAG_NAME}/#{policy_id}")
+        true
       end
 
       def uploader
@@ -71,8 +84,10 @@ module ChefDK
       end
 
       def cookbook_versions_to_upload
-        cookbook_versions_for_policy.reject do |cookbook|
-          remote_already_has_cookbook?(cookbook)
+        cookbook_versions_for_policy.inject([]) do |versions_to_upload, cookbook_with_lock|
+          cb = cookbook_with_lock.cookbook
+          versions_to_upload << cb unless remote_already_has_cookbook?(cb)
+          versions_to_upload
         end
       end
 
@@ -91,14 +106,30 @@ module ChefDK
       # An Array of Chef::CookbookVersion objects representing the full set that
       # the policyfile lock requires.
       def cookbook_versions_for_policy
+        return @cookbook_versions_for_policy if @cookbook_versions_for_policy
         policyfile_lock.validate_cookbooks!
-        policyfile_lock.cookbook_locks.map do |name, lock|
-          ReadCookbookForCompatModeUpload.load(name, lock.dotted_decimal_identifier, lock.cookbook_path)
+        @cookbook_versions_for_policy = policyfile_lock.cookbook_locks.map do |name, lock|
+          cb = ReadCookbookForCompatModeUpload.load(name, lock.dotted_decimal_identifier, lock.cookbook_path)
+          LockedCookbookForUpload.new(cb, lock)
         end
       end
 
       private
 
+      def upload_cookbooks
+        ui.msg("WARN: Uploading cookbooks using semver compat mode")
+
+        uploader.upload_cookbooks
+
+        reused_cbs, uploaded_cbs = cookbook_versions_for_policy.partition do |cb_with_lock|
+          remote_already_has_cookbook?(cb_with_lock.cookbook)
+        end
+
+        Reports::Upload.new(reused_cbs: reused_cbs, uploaded_cbs: uploaded_cbs, ui: ui).show
+
+        true
+      end
+
       def upload_lockfile_as_data_bag_item(policy_id, data_item)
         http_client.put("data/#{COMPAT_MODE_DATA_BAG_NAME}/#{policy_id}", data_item)
       rescue Net::HTTPServerException => e

data/lib/chef-dk/policyfile_compiler.rb

@@ -22,6 +22,8 @@ require 'chef/run_list'
 
 require 'chef-dk/policyfile/dsl'
 require 'chef-dk/policyfile_lock'
+require 'chef-dk/ui'
+require 'chef-dk/policyfile/reports/install'
 
 module ChefDK
 
@@ -34,8 +36,8 @@ module ChefDK
     # Cookbooks from these sources lock that cookbook to exactly one version
     SOURCE_TYPES_WITH_FIXED_VERSIONS = [:git, :path].freeze
 
-    def self.evaluate(policyfile_string, policyfile_filename)
-      compiler = new
+    def self.evaluate(policyfile_string, policyfile_filename, ui: nil)
+      compiler = new(ui: ui)
       compiler.evaluate_policyfile(policyfile_string, policyfile_filename)
       compiler
     end
@@ -48,11 +50,15 @@ module ChefDK
 
     attr_reader :dsl
     attr_reader :storage_config
+    attr_reader :install_report
 
-    def initialize
+    def initialize(ui: nil)
       @storage_config = Policyfile::StorageConfig.new
       @dsl = Policyfile::DSL.new(storage_config)
       @artifact_server_cookbook_location_specs = {}
+
+      @ui = ui || UI.null
+      @install_report = Policyfile::Reports::Install.new(ui: @ui, policyfile_compiler: self)
     end
 
     def error!
@@ -86,6 +92,7 @@ module ChefDK
         spec = cookbook_location_spec_for(cookbook_name)
         if spec.nil? or !spec.version_fixed?
           spec = create_spec_for_cookbook(cookbook_name, version)
+          install_report.installing_cookbook(spec)
           spec.ensure_cached
         end
       end
@@ -213,6 +220,12 @@ module ChefDK
       self
     end
 
+    def fixed_version_cookbooks_specs
+      @fixed_version_cookbooks_specs ||= cookbook_location_specs.select do |_cookbook_name, cookbook_location_spec|
+        cookbook_location_spec.version_fixed?
+      end
+    end
+
     private
 
     def normalize_recipe(run_list_item)
@@ -228,8 +241,9 @@ module ChefDK
     def cache_fixed_version_cookbooks
       ensure_cache_dir_exists
 
-      cookbook_location_specs.each do |_cookbook_name, cookbook_location_spec|
-        cookbook_location_spec.ensure_cached if cookbook_location_spec.version_fixed?
+      fixed_version_cookbooks_specs.each do |name, cookbook_location_spec|
+        install_report.installing_fixed_version_cookbook(cookbook_location_spec)
+        cookbook_location_spec.ensure_cached
       end
     end
 

data/lib/chef-dk/policyfile_lock.rb

@@ -18,10 +18,51 @@
 require 'chef-dk/policyfile/storage_config'
 require 'chef-dk/policyfile/cookbook_locks'
 require 'chef-dk/policyfile/solution_dependencies'
+require 'chef-dk/ui'
 
 module ChefDK
+
   class PolicyfileLock
 
+    class InstallReport
+
+      attr_reader :ui
+      attr_reader :policyfile_lock
+
+      def initialize(ui: ui, policyfile_lock: nil)
+        @ui = ui
+        @policyfile_lock = policyfile_lock
+
+        @cookbook_name_width = nil
+        @cookbook_version_width = nil
+      end
+
+      def installing_fixed_version_cookbook(cookbook_spec)
+        verb = cookbook_spec.installed? ? "Using     " : "Installing"
+        ui.msg("#{verb} #{format_fixed_version_cookbook(cookbook_spec)}")
+      end
+
+      def installing_cookbook(cookbook_lock)
+        verb = cookbook_lock.installed? ? "Using     " : "Installing"
+        ui.msg("#{verb} #{format_cookbook(cookbook_lock)}")
+      end
+
+      private
+
+      def format_cookbook(cookbook_lock)
+        "#{cookbook_lock.name.ljust(cookbook_name_width)} #{cookbook_lock.version.to_s.ljust(cookbook_version_width)}"
+      end
+
+      def cookbook_name_width
+        policyfile_lock.cookbook_locks.map { |name, _| name.size }.max
+      end
+
+      def cookbook_version_width
+        policyfile_lock.cookbook_locks.map { |_, lock| lock.version.size }.max
+      end
+    end
+
+    RUN_LIST_ITEM_FORMAT = /\Arecipe\[[^\s]+::[^\s]+\]\Z/.freeze
 
     def self.build(storage_config)
       lock = new(storage_config)
@@ -46,14 +87,18 @@ module ChefDK
 
     attr_reader :cookbook_locks
 
-    def initialize(storage_config)
+    attr_reader :install_report
+
+    def initialize(storage_config, ui: nil)
       @name = nil
       @run_list = []
       @cookbook_locks = {}
       @relative_paths_root = Dir.pwd
       @storage_config = storage_config
+      @ui = ui || UI.null
 
       @solution_dependencies = Policyfile::SolutionDependencies.new
+      @install_report = InstallReport.new(ui: @ui, policyfile_lock: self)
     end
 
     def lock_data_for(cookbook_name)
@@ -146,14 +191,10 @@ module ChefDK
     end
 
     def build_from_lock_data(lock_data)
-      @name = lock_data["name"]
-      @run_list = lock_data["run_list"]
-      lock_data["cookbook_locks"].each do |name, lock_info|
-        build_cookbook_lock_from_lock_data(name, lock_info)
-      end
-
-      s = Policyfile::SolutionDependencies.from_lock(lock_data["solution_dependencies"])
-      @solution_dependencies = s
+      set_name_from_lock_data(lock_data)
+      set_run_list_from_lock_data(lock_data)
+      set_cookbook_locks_from_lock_data(lock_data)
+      set_solution_dependencies_from_lock_data(lock_data)
       self
     end
 
@@ -162,6 +203,7 @@ module ChefDK
       ensure_cache_dir_exists
 
       cookbook_locks.each do |cookbook_name, cookbook_lock|
+        install_report.installing_cookbook(cookbook_lock)
         cookbook_lock.install_locked
       end
     end
@@ -175,7 +217,78 @@ module ChefDK
 
     private
 
+    def set_name_from_lock_data(lock_data)
+      name_attribute = lock_data["name"]
+
+      raise InvalidLockfile, "lockfile does not have a `name' attribute" if name_attribute.nil?
+
+      unless name_attribute.kind_of?(String)
+        raise InvalidLockfile, "lockfile's name attribute must be a String (got: #{name_attribute.inspect})"
+      end
+
+      if name_attribute.empty?
+        raise InvalidLockfile, "lockfile's name attribute cannot be an empty string"
+      end
+
+      @name = name_attribute
+
+    end
+
+    def set_run_list_from_lock_data(lock_data)
+      run_list_attribute = lock_data["run_list"]
+
+      raise InvalidLockfile, "lockfile does not have a run_list attribute" if run_list_attribute.nil?
+
+      unless run_list_attribute.kind_of?(Array)
+        raise InvalidLockfile, "lockfile's run_list must be an array of run list items (got: #{run_list_attribute.inspect})"
+      end
+
+      bad_run_list_items = run_list_attribute.select { |e| e !~ RUN_LIST_ITEM_FORMAT }
+
+      unless bad_run_list_items.empty?
+        msg = "lockfile's run_list items must be formatted like `recipe[$COOKBOOK_NAME::$RECIPE_NAME]'. Invalid items: `#{bad_run_list_items.join("' `")}'"
+        raise InvalidLockfile, msg
+      end
+
+      @run_list = run_list_attribute
+    end
+
+    def set_cookbook_locks_from_lock_data(lock_data)
+      cookbook_lock_data = lock_data["cookbook_locks"]
+
+      if cookbook_lock_data.nil?
+        raise InvalidLockfile, "lockfile does not have a cookbook_locks attribute"
+      end
+
+      unless cookbook_lock_data.kind_of?(Hash)
+        raise InvalidLockfile, "lockfile's cookbook_locks attribute must be a Hash (JSON object). (got: #{cookbook_lock_data.inspect})"
+      end
+
+      lock_data["cookbook_locks"].each do |name, lock_info|
+        build_cookbook_lock_from_lock_data(name, lock_info)
+      end
+    end
+
+    def set_solution_dependencies_from_lock_data(lock_data)
+      soln_deps = lock_data["solution_dependencies"]
+
+      if soln_deps.nil?
+        raise InvalidLockfile, "lockfile does not have a solution_dependencies attribute"
+      end
+
+      unless soln_deps.kind_of?(Hash)
+        raise InvalidLockfile, "lockfile's solution_dependencies attribute must be a Hash (JSON object). (got: #{soln_deps.inspect})"
+      end
+
+      s = Policyfile::SolutionDependencies.from_lock(lock_data["solution_dependencies"])
+      @solution_dependencies = s
+    end
+
     def build_cookbook_lock_from_lock_data(name, lock_info)
+      unless lock_info.kind_of?(Hash)
+        raise InvalidLockfile, "lockfile cookbook_locks entries must be a Hash (JSON object). (got: #{lock_info.inspect})"
+      end
+
       if lock_info["cache_key"].nil?
         local_cookbook(name).build_from_lock_data(lock_info)
       else

data/lib/chef-dk/policyfile_services/install.rb

@@ -0,0 +1,131 @@
+#
+# Copyright:: Copyright (c) 2014 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef-dk/exceptions'
+require 'chef-dk/policyfile_compiler'
+require 'chef-dk/policyfile/storage_config'
+require 'chef-dk/policyfile_lock'
+
+module ChefDK
+  module PolicyfileServices
+
+    class Install
+
+      attr_reader :root_dir
+      attr_reader :ui
+
+      def initialize(policyfile: nil, ui: nil, root_dir: nil)
+        @policyfile_relative_path = policyfile
+        @ui = ui
+        @root_dir = root_dir
+
+        @policyfile_content = nil
+        @policyfile_compiler = nil
+      end
+
+      def run
+        unless File.exist?(policyfile_path)
+          # TODO: suggest next step. Add a generator/init command? Specify path to Policyfile.rb?
+          # See card CC-232
+          raise PolicyfileNotFound, "Policyfile not found at path #{policyfile_path}"
+        end
+
+        if File.exist?(lockfile_path)
+          install_from_lock
+        else
+          generate_lock_and_install
+        end
+      end
+
+      def policyfile_relative_path
+        @policyfile_relative_path || "Policyfile.rb"
+      end
+
+      def policyfile_path
+        File.expand_path(policyfile_relative_path, root_dir)
+      end
+
+      def lockfile_relative_path
+        policyfile_relative_path.gsub(/\.rb\Z/, '') + ".lock.json"
+      end
+
+      def lockfile_path
+        File.expand_path(lockfile_relative_path, root_dir)
+      end
+
+      def policyfile_content
+        @policyfile_content ||= IO.read(policyfile_path)
+      end
+
+      def policyfile_compiler
+        @policyfile_compiler ||= ChefDK::PolicyfileCompiler.evaluate(policyfile_content, policyfile_path, ui: ui)
+      end
+
+      def expanded_run_list
+        policyfile_compiler.expanded_run_list.to_s
+      end
+
+      def policyfile_lock_content
+        @policyfile_lock_content ||= IO.read(lockfile_path) if File.exist?(lockfile_path)
+      end
+
+      def policyfile_lock
+        return nil if policyfile_lock_content.nil?
+        @policyfile_lock ||= begin
+          lock_data = FFI_Yajl::Parser.new.parse(policyfile_lock_content)
+          PolicyfileLock.new(storage_config, ui: ui).build_from_lock_data(lock_data)
+        end
+      end
+
+      def storage_config
+        @storage_config ||= Policyfile::StorageConfig.new(relative_paths_root: root_dir)
+      end
+
+      def generate_lock_and_install
+        policyfile_compiler.error!
+
+        ui.msg "Building policy #{policyfile_compiler.name}"
+        ui.msg "Expanded run list: " + expanded_run_list + "\n"
+
+        ui.msg "Caching Cookbooks..."
+
+        policyfile_compiler.install
+
+        lock_data = policyfile_compiler.lock.to_lock
+
+        File.open(lockfile_path, "w+") do |f|
+          f.print(FFI_Yajl::Encoder.encode(lock_data, pretty: true ))
+        end
+
+        ui.msg ""
+
+        ui.msg "Lockfile written to #{lockfile_path}"
+      rescue => error
+        raise PolicyfileInstallError.new("Failed to generate Policyfile.lock", error)
+      end
+
+      def install_from_lock
+        ui.msg "Installing cookbooks from lock"
+
+        policyfile_lock.install_cookbooks
+      rescue => error
+        raise PolicyfileInstallError.new("Failed to install cookbooks from lockfile", error)
+      end
+
+    end
+  end
+end