chef-apply 0.2.8 → 0.2.13

data/lib/chef_apply/action/converge_target.rb

@@ -16,9 +16,9 @@
 #
 
 require "chef_apply/action/base"
-require "chef_apply/text"
 require "pathname"
 require "tempfile"
+# FLAG: require "chef/util/path_helper"
 require "chef/util/path_helper"
 
 module ChefApply::Action
@@ -26,8 +26,8 @@ module ChefApply::Action
 
     def perform_action
       local_policy_path = config.delete :local_policy_path
-      remote_tmp = target_host.mktemp()
-      remote_dir_path = escape_windows_path(remote_tmp)
+      remote_tmp = target_host.temp_dir()
+      remote_dir_path = target_host.normalize_path(remote_tmp)
       # Ensure the directory is owned by the connecting user,
       # otherwise we won't be able to put things into it over scp as that user.
       remote_policy_path = create_remote_policy(local_policy_path, remote_dir_path)
@@ -36,11 +36,12 @@ module ChefApply::Action
       upload_trusted_certs(remote_dir_path)
 
       notify(:running_chef)
-      cmd_str = run_chef(remote_dir_path,
-                         File.basename(remote_config_path),
-                         File.basename(remote_policy_path))
+      # TODO - just teach target_host how to run_chef?
+      cmd_str = run_chef_cmd(remote_dir_path,
+                             File.basename(remote_config_path),
+                             File.basename(remote_policy_path))
       c = target_host.run_command(cmd_str)
-      target_host.run_command!("#{delete_folder} #{remote_dir_path}")
+      target_host.del_dir(remote_dir_path)
       if c.exit_status == 0
         ChefApply::Log.info(c.stdout)
         notify(:success)
@@ -73,14 +74,20 @@ module ChefApply::Action
       workstation_rb = <<~EOM
         local_mode true
         color false
-        cache_path "#{cache_path}"
-        chef_repo_path "#{cache_path}"
+        cache_path "#{target_host.ws_cache_path}"
+        chef_repo_path "#{target_host.ws_cache_path}"
         require_relative "reporter"
         reporter = ChefApply::Reporter.new
         report_handlers << reporter
         exception_handlers << reporter
       EOM
 
+      unless ChefApply::Config.chef.chef_license.nil?
+        workstation_rb << <<~EOM
+          chef_license "#{ChefApply::Config.chef.chef_license}"
+        EOM
+      end
+
       # add the target host's log level value
       # (we don't set a location because we want output to
       #   go in stdout for reporting back to chef-apply)
@@ -115,13 +122,16 @@ module ChefApply::Action
       remote_config_path
     end
 
-    def create_remote_handler(dir)
-      remote_handler_path = File.join(dir, "reporter.rb")
+    def create_remote_handler(remote_dir)
+      remote_handler_path = File.join(remote_dir, "reporter.rb")
       begin
-        handler_file = Tempfile.new
+        # TODO - why don't we upload the original remote_handler_path instead of making a temp copy?
+        handler_file = Tempfile.new()
+        # TODO - ideally this is a resource in the gem, and not placed in with source files.
         handler_file.write(File.read(File.join(__dir__, "reporter.rb")))
         handler_file.close
         target_host.upload_file(handler_file.path, remote_handler_path)
+      # TODO - should we be more specific in our error catch?
       rescue RuntimeError
         raise HandlerUploadFailed.new()
       ensure
@@ -131,41 +141,67 @@ module ChefApply::Action
     end
 
     def upload_trusted_certs(dir)
+      # TODO BOOTSTRAP - trusted certs dir and other config to be received as argument to constructor
       local_tcd = Chef::Util::PathHelper.escape_glob_dir(ChefApply::Config.chef.trusted_certs_dir)
       certs = Dir.glob(File.join(local_tcd, "*.{crt,pem}"))
       return if certs.empty?
+
       notify(:uploading_trusted_certs)
       remote_tcd = "#{dir}/trusted_certs"
-      target_host.mkdir(remote_tcd)
+      target_host.make_directory(remote_tcd)
       certs.each do |cert_file|
         target_host.upload_file(cert_file, "#{remote_tcd}/#{File.basename(cert_file)}")
       end
     end
 
+    def chef_report_path
+      @chef_report_path ||= target_host.normalize_path(File.join(target_host.ws_cache_path, "cache", "run-report.json"))
+    end
+
     def handle_ccr_error
       require "chef_apply/errors/ccr_failure_mapper"
       mapper_opts = {}
-      c = target_host.run_command(read_chef_report)
-      if c.exit_status == 0
-        report = JSON.parse(c.stdout)
+      content = target_host.fetch_file_contents(chef_report_path)
+      if content.nil?
+        report = {}
+        mapper_opts[:failed_report_path] = chef_report_path
+        ChefApply::Log.error("Could not read remote report at #{chef_report_path}")
+      else
         # We need to delete the stacktrace after copying it over. Otherwise if we get a
         # remote failure that does not write a chef stacktrace its possible to get an old
         # stale stacktrace.
-        target_host.run_command!(delete_chef_report)
+        target_host.del_file(chef_report_path)
+        report = JSON.parse(content)
         ChefApply::Log.error("Remote chef-client error follows:")
         ChefApply::Log.error(report["exception"])
-      else
-        report = {}
-        ChefApply::Log.error("Could not read remote report:")
-        ChefApply::Log.error("stdout: #{c.stdout}")
-        ChefApply::Log.error("stderr: #{c.stderr}")
-        mapper_opts[:stdout] = c.stdout
-        mapper_opts[:stderr] = c.stderr
       end
       mapper = ChefApply::Errors::CCRFailureMapper.new(report["exception"], mapper_opts)
       mapper.raise_mapped_exception!
     end
 
+    # Chef will try 'downloading' the policy from the internet unless we pass it a valid, local file
+    # in the working directory. By pointing it at a local file it will just copy it instead of trying
+    # to download it.
+    #
+    # Chef 13 on Linux requires full path specifiers for --config and --recipe-url while on Chef 13 and 14 on
+    # Windows must use relative specifiers to prevent URI from causing an error
+    # (https://github.com/chef/chef/pull/7223/files).
+    def run_chef_cmd(working_dir, config_file, policy)
+      case target_host.base_os
+      when :windows
+        "Set-Location -Path #{working_dir}; " +
+          # We must 'wait' for chef-client to finish before changing directories and Out-Null does that
+          "chef-client -z --config #{File.join(working_dir, config_file)} --recipe-url #{File.join(working_dir, policy)} | Out-Null; " +
+          # We have to leave working dir so we don't hold a lock on it, which allows us to delete this tempdir later
+          "Set-Location C:/; " +
+          "exit $LASTEXITCODE"
+      else
+        # cd is shell a builtin, so we'll invoke bash. This also means all commands are executed
+        # with sudo (as long as we are hardcoding our sudo use)
+        "bash -c 'cd #{working_dir}; chef-client -z --config #{File.join(working_dir, config_file)} --recipe-url #{File.join(working_dir, policy)}'"
+      end
+    end
+
     class ConfigUploadFailed < ChefApply::Error
       def initialize(); super("CHEFUPL003"); end
     end

data/lib/chef_apply/action/install_chef.rb

@@ -15,16 +15,105 @@
 # limitations under the License.
 #
 
-require "chef_apply/action/install_chef/base"
-require "chef_apply/action/install_chef/windows"
-require "chef_apply/action/install_chef/linux"
-
-module ChefApply::Action::InstallChef
-  def self.instance_for_target(target_host, opts = { check_only: false })
-    opts[:target_host] = target_host
-    case target_host.base_os
-    when :windows then Windows.new(opts)
-    when :linux then Linux.new(opts)
+require "chef_apply/action/base"
+require "chef_apply/minimum_chef_version"
+require "fileutils"
+
+module ChefApply
+  module Action
+    class InstallChef < ChefApply::Action::Base
+
+      def initialize(opts = { check_only: false })
+        super
+      end
+
+      def perform_action
+        if ChefApply::MinimumChefVersion.check!(target_host, config[:check_only]) == :minimum_version_met
+          notify(:already_installed)
+        else
+          perform_local_install
+        end
+      end
+
+      def upgrading?
+        @upgrading
+      end
+
+      def perform_local_install
+        package = lookup_artifact()
+        notify(:downloading)
+        local_path = download_to_workstation(package.url)
+        notify(:uploading)
+        remote_path = upload_to_target(local_path)
+        notify(:installing)
+        target_host.install_package(remote_path)
+        notify(:install_complete)
+      end
+
+      def perform_remote_install
+        # TODO BOOTSTRAP - we'll need to implement this for both platforms
+        # require "mixlib/install"
+        # installer = Mixlib::Install.new({
+        #   platform: "windows",
+        #   product_name: "chef",
+        #   channel: :stable,
+        #   shell_type: :ps1,
+        #   version: "13",
+        # })
+        target_host.run_command! installer.install_command
+        raise NotImplementedError
+      end
+
+      def lookup_artifact
+        return @artifact_info if @artifact_info
+        require "mixlib/install"
+        c = train_to_mixlib(target_host.platform)
+        Mixlib::Install.new(c).artifact_info
+      end
+
+      def version_to_install
+        lookup_artifact.version
+      end
+
+      def train_to_mixlib(platform)
+        opts = {
+          platform_version: platform.release,
+          platform: platform.name,
+          architecture: platform.arch,
+          product_name: "chef",
+          product_version: :latest,
+          channel: :stable,
+          platform_version_compatibility_mode: true,
+        }
+        case platform.name
+        when /windows/
+          opts[:platform] = "windows"
+        when "redhat", "centos"
+          opts[:platform] = "el"
+        when "suse"
+          opts[:platform] = "sles"
+        when "amazon"
+          opts[:platform] = "el"
+          if platform.release.to_i > 2010 # legacy Amazon version 1
+            opts[:platform_version] = "6"
+          else
+            opts[:platform_version] = "7"
+          end
+        end
+        opts
+      end
+
+      def download_to_workstation(url_path)
+        require "chef_apply/file_fetcher"
+        ChefApply::FileFetcher.fetch(url_path)
+      end
+
+      def upload_to_target(local_path)
+        installer_dir = target_host.temp_dir()
+        remote_path = File.join(installer_dir, File.basename(local_path))
+        target_host.upload_file(local_path, remote_path)
+        remote_path
+      end
     end
   end
 end

data/lib/chef_apply/cli.rb

@@ -36,6 +36,8 @@ require "chef_apply/telemeter"
 require "chef_apply/ui/error_printer"
 require "chef_apply/ui/terminal"
 require "chef_apply/ui/terminal/job"
+require "license_acceptance/cli_flags/mixlib_cli"
+require "license_acceptance/acceptor"
 
 module ChefApply
   class CLI
@@ -48,6 +50,7 @@ module ChefApply
     include ChefApply::CLI::Validation
     # Help and version formatting
     include ChefApply::CLI::Help
+    include LicenseAcceptance::CLIFlags::MixlibCLI
 
     RC_OK = 0
     RC_COMMAND_FAILED = 1
@@ -106,6 +109,7 @@ module ChefApply
       elsif parsed_options[:version]
         show_version
       else
+        check_license_acceptance
         validate_params(cli_arguments)
         target_hosts = resolve_targets(cli_arguments.shift, parsed_options)
         render_cookbook_setup(cli_arguments)
@@ -124,6 +128,16 @@ module ChefApply
       temp_cookbook.delete unless temp_cookbook.nil?
     end
 
+    def check_license_acceptance
+      acceptor = LicenseAcceptance::Acceptor.new(provided: ChefApply::Config.chef.chef_license)
+      begin
+        acceptor.check_and_persist("infra-client", "latest")
+      rescue LicenseAcceptance::LicenseNotAcceptedError
+        raise LicenseCheckFailed.new
+      end
+      ChefApply::Config.chef.chef_license ||= acceptor.acceptance_value
+    end
+
     def resolve_targets(host_spec, opts)
       @target_hosts = TargetResolver.new(host_spec,
                                          opts.delete(:protocol),
@@ -170,7 +184,7 @@ module ChefApply
     def install(target_host, reporter)
       context = TS.install_chef
       reporter.update(context.verifying)
-      installer = Action::InstallChef.instance_for_target(target_host, check_only: !parsed_options[:install])
+      installer = Action::InstallChef.new(target_host: target_host, check_only: !parsed_options[:install])
       installer.run do |event, data|
         case event
         when :installing
@@ -325,4 +339,8 @@ module ChefApply
     end
 
   end
+
+  class LicenseCheckFailed < ChefApply::Error
+    def initialize(); super("CHEFLIC001"); end
+  end
 end

data/lib/chef_apply/config.rb

@@ -145,6 +145,7 @@ module ChefApply
       ChefConfig::WorkstationConfigLoader.new(nil, ChefApply::Log).load
       default(:cookbook_repo_paths, [ChefConfig::Config[:cookbook_path]].flatten)
       default(:trusted_certs_dir, ChefConfig::Config[:trusted_certs_dir])
+      default(:chef_license, ChefConfig::Config[:chef_license])
       ChefConfig::Config.reset
     end
 

data/lib/chef_apply/error.rb

@@ -28,6 +28,7 @@ module ChefApply
     end
   end
 
+  # These helpers are obsolete
   class ErrorNoLogs < Error
     def initialize(id, *params)
       super

data/lib/chef_apply/errors/ccr_failure_mapper.rb

@@ -28,7 +28,7 @@ module ChefApply::Errors
 
     def raise_mapped_exception!
       if @cause_line.nil?
-        raise RemoteChefRunFailedToResolveError.new(params[:stdout], params[:stderr])
+        raise RemoteChefRunFailedToResolveError.new(params[:failed_report_path])
       else
         errid, *args = exception_args_from_cause()
         if errid.nil?
@@ -85,7 +85,7 @@ module ChefApply::Errors
     end
 
     class RemoteChefRunFailedToResolveError < ChefApply::ErrorNoStack
-      def initialize(stdout, stderr); super("CHEFCCR001", stdout, stderr); end
+      def initialize(path); super("CHEFCCR001", path); end
     end
 
   end

data/lib/chef_apply/target_host.rb

@@ -1,5 +1,5 @@
 #
-# Copyright:: Copyright (c) 2017 Chef Software Inc.
+# Copyright:: Copyright (c) 2017-2019 Chef Software Inc.
 # License:: Apache License, Version 2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,6 +18,7 @@
 require "chef_apply/log"
 require "chef_apply/error"
 require "train"
+
 module ChefApply
   class TargetHost
     attr_reader :config, :reporter, :backend, :transport_type
@@ -26,10 +27,39 @@ module ChefApply
     # See #apply_ssh_config
     SSH_CONFIG_OVERRIDE_KEYS = [:user, :port, :proxy].freeze
 
-    def self.instance_for_url(target, opts = {})
-      opts = { target: @url }
-      target_host = new(target, opts)
+    # We're borrowing a page from train here - because setting up a
+    # reliable connection for testing is a multi-step process,
+    # we'll provide this method which instantiates a TargetHost connected
+    # to a train mock backend. If the family/name provided resolves to a suported
+    # OS, this instance will mix-in the supporting methods for the given platform;
+    # otherwise those methods will raise NotImplementedError.
+    def self.mock_instance(url, family: "unknown", name: "unknown",
+                                release: "unknown", arch: "x86_64")
+      # Specifying sudo: false ensures that attempted operations
+      # don't fail because the mock platform doesn't support sudo
+      target_host = TargetHost.new(url, { sudo: false })
+
+      # Don't pull in the platform-specific mixins automatically during connect
+      # Otherwise, it will raise since it can't resolve the OS without the mock.
+      target_host.instance_variable_set(:@mocked_connection, true)
       target_host.connect!
+
+      # We need to provide this mock before invoking mix_in_target_platform,
+      # otherwise it will fail with an unknown OS (since we don't have a real connection).
+      target_host.backend.mock_os(
+        family: family,
+        name: name,
+        release: release,
+        arch: arch
+      )
+
+      # Only mix-in if we can identify the platform.  This
+      # prevents mix_in_target_platform! from raising on unknown platform during
+      # tests that validate unsupported platform behaviors.
+      if target_host.base_os != :other
+        target_host.mix_in_target_platform!
+      end
+
       target_host
     end
 
@@ -79,10 +109,19 @@ module ChefApply
       end
     end
 
+    # Establish connection to configured target.
+    #
     def connect!
+      # Keep existing connections
       return unless @backend.nil?
       @backend = train_connection.connection
       @backend.wait_until_ready
+
+      # When the testing function `mock_instance` is used, it will set
+      # this instance variable to false and handle this function call
+      # after the platform data is mocked; this will allow binding
+      # of mixin functions based on the mocked platform.
+      mix_in_target_platform! unless @mocked_connection
     rescue Train::UserError => e
       raise ConnectionFailure.new(e, config)
     rescue Train::Error => e
@@ -91,8 +130,20 @@ module ChefApply
       raise ConnectionFailure.new(e.cause || e, config)
     end
 
+    def mix_in_target_platform!
+      case base_os
+      when :linux
+        require "chef_apply/target_host/linux"
+        class << self; include ChefApply::TargetHost::Linux; end
+      when :windows
+        require "chef_apply/target_host/windows"
+        class << self; include ChefApply::TargetHost::Windows; end
+      when :other
+        raise ChefApply::TargetHost::UnsupportedTargetOS.new(platform.name)
+      end
+    end
+
     # Returns the user being used to connect. Defaults to train's default user if not specified
-    # defaulted in .ssh/config (for ssh connections), as set up in '#apply_ssh_config'.
     def user
       return config[:user] unless config[:user].nil?
       require "train/transports/ssh"
@@ -112,17 +163,16 @@ module ChefApply
     end
 
     def base_os
-      if platform.family == "windows"
+      if platform.windows?
         :windows
       elsif platform.linux?
         :linux
       else
-        # TODO - this seems like it shouldn't happen here, when
-        # all the caller is doing is asking about the OS
-        raise ChefApply::TargetHost::UnsupportedTargetOS.new(platform.name)
+        :other
       end
     end
 
+    # TODO 2019-01-29  not expose this, it's internal implemenation. Same with #backend.
     def platform
       backend.platform
     end
@@ -143,6 +193,17 @@ module ChefApply
       backend.upload(local_path, remote_path)
     end
 
+    # Retrieve the contents of a remote file. Returns nil
+    # if the file didn't exist or couldn't be read.
+    def fetch_file_contents(remote_path)
+      result = backend.file(remote_path)
+      if result.exist? && result.file?
+        result.content
+      else
+        nil
+      end
+    end
+
     # Returns the installed chef version as a Gem::Version,
     # or raised ChefNotInstalled if chef client version manifest can't
     # be found.
@@ -150,81 +211,62 @@ module ChefApply
       return @installed_chef_version if @installed_chef_version
       # Note: In the case of a very old version of chef (that has no manifest - pre 12.0?)
       #       this will report as not installed.
-      manifest = get_chef_version_manifest()
-      raise ChefNotInstalled.new if manifest == :not_found
-      # We'll split the version here because  unstable builds (where we currently
-      # install from) are in the form "Major.Minor.Build+HASH" which is not a valid
+      manifest = read_chef_version_manifest()
+
+      # We split the version here because  unstable builds install from)
+      # are in the form "Major.Minor.Build+HASH" which is not a valid
       # version string.
       @installed_chef_version = Gem::Version.new(manifest["build_version"].split("+")[0])
     end
 
-    MANIFEST_PATHS = {
-      # TODO - use a proper method to query the win installation path -
-      #        currently we're assuming the default, but this can be customized
-      #        at install time.
-      #        A working approach is below - but it runs very slowly in testing
-      #        on a virtualbox windows vm:
-      #        (over winrm) Get-WmiObject Win32_Product | Where {$_.Name -match 'Chef Client'}
-      windows: "c:\\opscode\\chef\\version-manifest.json",
-      linux: "/opt/chef/version-manifest.json",
-    }.freeze
-
-    def get_chef_version_manifest
-      path = MANIFEST_PATHS[base_os()]
-      manifest = backend.file(path)
-      return :not_found unless manifest.file?
-      JSON.parse(manifest.content)
+    def read_chef_version_manifest
+      manifest = fetch_file_contents(omnibus_manifest_path)
+      raise ChefNotInstalled.new if manifest.nil?
+      JSON.parse(manifest)
     end
 
-    # create a dir.  set owner to the connecting user if host isn't windows
-    # so that scp -- which uses the connecting user -- can upload into it.
-    def mkdir(path)
-      if base_os == :windows
-        run_command!("New-Item -ItemType Directory -Force -Path #{path}")
-      else
-        # This will also set ownership to the connecting user instead of default of
-        # root when sudo'd, so that the dir can be used to upload files using scp -
-        # which is done as the connecting user.
-        run_command!("mkdir -p #{path}")
-        chown(path, user)
-      end
-      nil
+    # Creates and caches location of temporary directory on the remote host
+    # using platform-specific implementations of make_temp_dir
+    # This will also set ownership to the connecting user instead of default of
+    # root when sudo'd, so that the dir can be used to upload files using scp
+    # as the connecting user.
+    #
+    # The base temp dir is cached and will only be created once per connection lifetime.
+    def temp_dir
+      dir = make_temp_dir()
+      chown(dir, user)
+      dir
     end
 
-    # TODO make these platform-specific classes instead of conditionals
+    # create a directory.  because we run all commands as root, this will also set group:owner
+    # to the connecting user if host isn't windows so that scp -- which uses the connecting user --
+    # will have permissions to upload into it.
+    def make_directory(path)
+      mkdir(path)
+      chown(path, user)
+      path
+    end
 
-    # Simplified chown - just sets user , defaults to connection user. Does not touch
-    # group.  Only has effect on non-windows targets
-    def chown(path, owner = nil)
-      return if base_os == :windows
-      owner ||= user
-      run_command!("chown #{owner} '#{path}'")
+    # normalizes path across OS's
+    def normalize_path(p) # NOTE BOOTSTRAP: was action::base::escape_windows_path
+      p.tr("\\", "/")
     end
 
-    MKTMP_WIN_CMD = "$parent = [System.IO.Path]::GetTempPath();" +
-      "[string] $name = [System.Guid]::NewGuid();" +
-      "$tmp = New-Item -ItemType Directory -Path " +
-      "(Join-Path $parent $name);" +
-      "$tmp.FullName"
+    # Simplified chown - just sets user, defaults to connection user. Does not touch
+    # group.  Only has effect on non-windows targets
+    def chown(path, owner); raise NotImplementedError; end
 
-    MKTMP_LINUX_CMD = "d=$(mktemp -d -p${TMPDIR:-/tmp} chef_XXXXXX); echo $d".freeze
+    # Platform-specific installation of packages
+    def install_package(target_package_path); raise NotImplementedError; end
 
-    # Create temporary dir and return the path.
-    # This will also set ownership to the connecting user instead of default of
-    # root when sudo'd, so that the dir can be used to upload files using scp -
-    # which is done as the connecting user.
-    def mktemp
-      if base_os == :windows
-        res = run_command!(MKTMP_WIN_CMD)
-        res.stdout.chomp.strip
-      else
-        # # TODO should we keep chmod 777?
-        res = run_command!("bash -c '#{MKTMP_LINUX_CMD}'")
-        path = res.stdout.chomp.strip
-        chown(path)
-        path
-      end
-    end
+    def ws_cache_path; raise NotImplementedError; end
+
+    # Recursively delete directory
+    def del_dir(path); raise NotImplementedError; end
+
+    def del_file(path); raise NotImplementedError; end
+
+    def omnibus_manifest_path(); raise NotImplementedError; end
 
     private
 
@@ -276,9 +318,7 @@ module ChefApply
         super(*(Array(init_params).flatten))
       end
     end
-
     class ChefNotInstalled < StandardError; end
-
     class UnsupportedTargetOS < ChefApply::ErrorNoLogs
       def initialize(os_name); super("CHEFTARG001", os_name); end
     end