chef-apply 0.1.2

data/lib/chef_apply/file_fetcher.rb

@@ -0,0 +1,70 @@
+#
+# Copyright:: Copyright (c) 2017 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "net/http"
+require "uri"
+require "chef_apply/config"
+require "chef_apply/log"
+
+module ChefApply
+  class FileFetcher
+    class << self
+      # Simple fetcher of an http(s) url. Returns the local path
+      # of the downloaded file.
+      def fetch(path)
+        cache_path = ChefApply::Config.cache.path
+        FileUtils.mkdir_p(cache_path)
+        url = URI.parse(path)
+        name = File.basename(url.path)
+        local_path = File.join(cache_path, name)
+
+        # TODO header check for size or checksum?
+        return local_path if File.exist?(local_path)
+
+        download_file(url, local_path)
+        local_path
+      end
+
+      def download_file(url, local_path)
+        temp_path = "#{local_path}.downloading"
+        file = open(temp_path, "wb")
+        ChefApply::Log.debug "Downloading: #{temp_path}"
+        Net::HTTP.start(url.host) do |http|
+          begin
+            http.request_get(url.path) do |resp|
+              resp.read_body do |segment|
+                file.write(segment)
+              end
+            end
+          rescue e
+            @error = true
+            raise
+          ensure
+            file.close()
+            # If any failures occurred, don't risk keeping
+            # an incomplete download that we'll see as 'cached'
+            if @error
+              FileUtils.rm_f(temp_path)
+            else
+              FileUtils.mv(temp_path, local_path)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chef_apply/log.rb

@@ -0,0 +1,42 @@
+#
+# Copyright:: Copyright (c) 2017 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "mixlib/log"
+
+module ChefApply
+  class Log
+    extend Mixlib::Log
+
+    def self.setup(location, log_level)
+      @location = location
+      if location.is_a?(String)
+        if location.casecmp("stdout") == 0
+          location = $stdout
+        else
+          location = File.open(location, "w+")
+        end
+      end
+      init(location)
+      Log.level = log_level
+    end
+
+    def self.location
+      @location
+    end
+
+  end
+end

data/lib/chef_apply/recipe_lookup.rb

@@ -0,0 +1,117 @@
+#
+# Copyright:: Copyright (c) 2017 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "chef-config/config"
+require "chef_apply/config"
+require "chef_apply/error"
+require "chef_apply/log"
+
+module ChefApply
+  # When users are trying to converge a local recipe on a remote target, there
+  # is a very specific (but expansive) set of things they can specify. This
+  # class encapsulates that logic for testing purposes. We either return
+  # a path to a recipe or we raise an error.
+  class RecipeLookup
+
+    attr_reader :cookbook_repo_paths
+    def initialize(cookbook_repo_paths)
+      @cookbook_repo_paths = cookbook_repo_paths
+    end
+
+    # The recipe specifier is provided by the customer as either a path OR
+    # a cookbook and optional recipe name.
+    def split(recipe_specifier)
+      recipe_specifier.split("::")
+    end
+
+    # Given a cookbook path or name, try to load that cookbook. Either return
+    # a cookbook object or raise an error.
+    def load_cookbook(path_or_name)
+      require "chef/exceptions"
+      if File.directory?(path_or_name)
+        cookbook_path = path_or_name
+        # First, is there a cookbook in the specified dir that matches?
+        require "chef/cookbook/cookbook_version_loader"
+        begin
+          v = Chef::Cookbook::CookbookVersionLoader.new(cookbook_path)
+          v.load!
+          cookbook = v.cookbook_version
+        rescue Chef::Exceptions::CookbookNotFoundInRepo
+          raise InvalidCookbook.new(cookbook_path)
+        end
+      else
+        cookbook_name = path_or_name
+        # Second, is there a cookbook in their local repository that matches?
+        require "chef/cookbook_loader"
+        cb_loader = Chef::CookbookLoader.new(cookbook_repo_paths)
+        cb_loader.load_cookbooks_without_shadow_warning
+
+        begin
+          cookbook = cb_loader[cookbook_name]
+        rescue Chef::Exceptions::CookbookNotFoundInRepo
+          cookbook_repo_paths.each do |repo_path|
+            cookbook_path = File.join(repo_path, cookbook_name)
+            if File.directory?(cookbook_path)
+              raise InvalidCookbook.new(cookbook_path)
+            end
+          end
+          raise CookbookNotFound.new(cookbook_name, cookbook_repo_paths)
+        end
+      end
+      cookbook
+    end
+
+    # Find the specified recipe or default recipe if none is specified.
+    # Raise an error if recipe cannot be found.
+    def find_recipe(cookbook, recipe_name = nil)
+      recipes = cookbook.recipe_filenames_by_name
+      if recipe_name.nil?
+        default_recipe = recipes["default"]
+        raise NoDefaultRecipe.new(cookbook.root_dir, cookbook.name) if default_recipe.nil?
+        default_recipe
+      else
+        recipe = recipes[recipe_name]
+        raise RecipeNotFound.new(cookbook.root_dir, recipe_name, recipes.keys, cookbook.name) if recipe.nil?
+        recipe
+      end
+    end
+
+    class InvalidCookbook < ChefApply::Error
+      def initialize(cookbook_path); super("CHEFVAL005", cookbook_path); end
+    end
+
+    class CookbookNotFound < ChefApply::Error
+      def initialize(cookbook_name, repo_paths)
+        repo_paths = repo_paths.join("\n")
+        super("CHEFVAL006", cookbook_name, repo_paths)
+      end
+    end
+
+    class NoDefaultRecipe < ChefApply::Error
+      def initialize(cookbook_path, cookbook_name); super("CHEFVAL007", cookbook_path, cookbook_name); end
+    end
+
+    class RecipeNotFound < ChefApply::Error
+      def initialize(cookbook_path, recipe_name, available_recipes, cookbook_name)
+        available_recipes.map! { |r| "'#{r}'" }
+        available_recipes = available_recipes.join(", ")
+        super("CHEFVAL008", cookbook_path, recipe_name, available_recipes, cookbook_name)
+      end
+    end
+
+  end
+end

data/lib/chef_apply/startup.rb

@@ -0,0 +1,162 @@
+#
+# Copyright:: Copyright (c) 2018 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "chef_apply/config"
+require "chef_apply/text"
+require "chef_apply/ui/terminal"
+require "chef_apply/telemeter/sender"
+module ChefApply
+  class Startup
+    attr_reader :argv
+    T = ChefApply::Text.cli
+
+    def initialize(argv)
+      @term_init = false
+      @argv = argv.clone
+      # Enable CLI output via Terminal. This comes first because other startup steps may
+      # need to output to the terminal.
+      init_terminal
+    end
+
+    def run
+      # Some tasks we do only once in an installation:
+      first_run_tasks
+
+      # Call this every time, so that if we add or change ~/.chef-workstation
+      # directory structure, we can be sure that it exists. Even with a
+      # custom configuration, the .chef-workstation directory and subdirs
+      # are required.
+      setup_workstation_user_directories
+
+      # Startup tasks that may change behavior based on configuration value
+      # must be run after load_config
+      load_config
+
+      # Init logging using log level out of config
+      setup_logging
+
+      # Begin upload of previous session telemetry. (If telemetry is not enabled,
+      # in config the uploader will clean up previous session(s) without sending)
+      start_telemeter_upload
+
+      # Launch the actual Chef Apply behavior
+      start_chef_apply
+
+    # NOTE: Because these exceptions occur outside of the
+    #       CLI handling, they won't be tracked in telemtry.
+    #       We can revisit this once the pending error handling rework
+    #       is underway.
+    rescue ConfigPathInvalid => e
+      UI::Terminal.output(T.error.bad_config_file(e.path))
+    rescue ConfigPathNotProvided
+      UI::Terminal.output(T.error.missing_config_path)
+    rescue Mixlib::Config::UnknownConfigOptionError => e
+      # Ideally we'd update the exception in mixlib to include
+      # a field with the faulty value, line number, and nested context -
+      # it's less fragile than depending on text parsing, which
+      # is what we'll do for now.
+      if e.message =~ /.*unsupported config value (.*)[.]+$/
+        # TODO - levenshteinian distance to figure out
+        # what they may have meant instead.
+        UI::Terminal.output(T.error.invalid_config_key($1, Config.location))
+      else
+        # Safety net in case the error text changes from under us.
+        UI::Terminal.output(T.error.unknown_config_error(e.message, Config.location))
+      end
+    rescue Tomlrb::ParseError => e
+      UI::Terminal.output(T.error.unknown_config_error(e.message, Config.location))
+    end
+
+    def init_terminal
+      UI::Terminal.init($stdout)
+    end
+
+    def first_run_tasks
+      return if Dir.exist?(Config::WS_BASE_PATH)
+      create_default_config
+      setup_telemetry
+    end
+
+    def create_default_config
+      UI::Terminal.output T.creating_config(Config.default_location)
+      UI::Terminal.output ""
+      FileUtils.mkdir_p(Config::WS_BASE_PATH)
+      FileUtils.touch(Config.default_location)
+    end
+
+    def setup_telemetry
+      require "securerandom"
+      installation_id = SecureRandom.uuid
+      File.write(Config.telemetry_installation_identifier_file, installation_id)
+
+      # Tell the user we're anonymously tracking, give brief opt-out
+      # and a link to detailed information.
+      UI::Terminal.output T.telemetry_enabled(Config.location)
+      UI::Terminal.output ""
+    end
+
+    def start_telemeter_upload
+      ChefApply::Telemeter::Sender.start_upload_thread()
+    end
+
+    def setup_workstation_user_directories
+      # Note that none of  these paths are customizable in config, so
+      # it's safe to do before we load config.
+      FileUtils.mkdir_p(Config::WS_BASE_PATH)
+      FileUtils.mkdir_p(Config.base_log_directory)
+      FileUtils.mkdir_p(Config.telemetry_path)
+    end
+
+    def load_config
+      path = custom_config_path
+      Config.custom_location(path) unless path.nil?
+      Config.load
+    end
+
+    # Look for a user-supplied config path by  manually parsing the option.
+    # Note that we can't use Mixlib::CLI for this.
+    # To ensure that ChefApply::CLI initializes with correct
+    # option defaults, we need to have configuraton loaded before initializing it.
+    def custom_config_path
+      argv.each_with_index do |arg, index|
+        if arg == "--config-path" || arg == "-c"
+          next_arg = argv[index + 1]
+          raise ConfigPathNotProvided.new if next_arg.nil?
+          raise ConfigPathInvalid.new(next_arg) unless File.file?(next_arg) && File.readable?(next_arg)
+          return next_arg
+        end
+      end
+      nil
+    end
+
+    def setup_logging
+      ChefApply::Log.setup(Config.log.location, Config.log.level.to_sym)
+      ChefApply::Log.info("Initialized logger")
+    end
+
+    def start_chef_apply
+      require "chef_apply/cli"
+      ChefApply::CLI.new(@argv).run
+    end
+    class ConfigPathNotProvided < StandardError; end
+    class ConfigPathInvalid < StandardError
+      attr_reader :path
+      def initialize(path)
+        @path = path
+      end
+    end
+  end
+end

data/lib/chef_apply/status_reporter.rb

@@ -0,0 +1,42 @@
+#
+# Copyright:: Copyright (c) 2017 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module ChefApply
+  class StatusReporter
+
+    def initialize(ui_element, prefix: nil, key: nil)
+      @ui_element = ui_element
+      @key = key
+      @ui_element.update(prefix: prefix)
+    end
+
+    def update(msg)
+      @ui_element.update({ @key => msg })
+    end
+
+    def success(msg)
+      update(msg)
+      @ui_element.success
+    end
+
+    def error(msg)
+      update(msg)
+      @ui_element.error
+    end
+
+  end
+end

data/lib/chef_apply/target_host.rb

@@ -0,0 +1,233 @@
+#
+# Copyright:: Copyright (c) 2017 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "chef_apply/log"
+require "chef_apply/error"
+require "train"
+module ChefApply
+  class TargetHost
+    attr_reader :config, :reporter, :backend, :transport_type
+    # These values may exist in .ssh/config but will be ignored by train
+    # in favor of its defaults unless we specify them explicitly.
+    # See #apply_ssh_config
+    SSH_CONFIG_OVERRIDE_KEYS = [:user, :port, :proxy]
+
+    def self.instance_for_url(target, opts = {})
+      opts = { target: @url }
+      target_host = new(target, opts)
+      target_host.connect!
+      target_host
+    end
+
+    def initialize(host_url, opts = {}, logger = nil)
+      @config = connection_config(host_url, opts, logger)
+      @transport_type = Train.validate_backend(@config)
+      apply_ssh_config(@config, opts) if @transport_type == "ssh"
+      @train_connection = Train.create(@transport_type, config)
+    end
+
+    def connection_config(host_url, opts_in, logger)
+      connection_opts = { target: host_url,
+                          sudo: opts_in[:sudo] === false ? false : true,
+                          www_form_encoded_password: true,
+                          key_files: opts_in[:identity_file],
+                          logger: ChefApply::Log }
+      if opts_in.has_key? :ssl
+        connection_opts[:ssl] = opts_in[:ssl]
+        connection_opts[:self_signed] = (opts_in[:ssl_verify] === false ? true : false)
+      end
+
+      [:sudo_password, :sudo, :sudo_command, :password, :user].each do |key|
+        connection_opts[key] = opts_in[key] if opts_in.has_key? key
+      end
+
+      Train.target_config(connection_opts)
+    end
+
+    def apply_ssh_config(config, opts_in)
+      # If we don't provide certain options, they will be defaulted
+      # within train - in the case of ssh, this will prevent the .ssh/config
+      # values from being picked up.
+      # Here we'll modify the returned @config to specify
+      # values that we get out of .ssh/config if present and if they haven't
+      # been explicitly given.
+      host_cfg = ssh_config_for_host(config[:host])
+      SSH_CONFIG_OVERRIDE_KEYS.each do |key|
+        if host_cfg.has_key?(key) && opts_in[key].nil?
+          config[key] = host_cfg[key]
+        end
+      end
+    end
+
+    def connect!
+      return unless @backend.nil?
+      @backend = train_connection.connection
+      @backend.wait_until_ready
+    rescue Train::UserError => e
+      raise ConnectionFailure.new(e, config)
+    rescue Train::Error => e
+      # These are typically wrapper errors for other problems,
+      # so we'll prefer to use e.cause over e if available.
+      raise ConnectionFailure.new(e.cause || e, config)
+    end
+
+    # Returns the user being used to connect. Defaults to train's default user if not specified
+    # defaulted in .ssh/config (for ssh connections), as set up in '#apply_ssh_config'.
+    def user
+      return config[:user] unless config[:user].nil?
+      require "train/transports/ssh"
+      Train::Transports::SSH.default_options[:user][:default]
+    end
+
+    def hostname
+      config[:host]
+    end
+
+    def architecture
+      platform.arch
+    end
+
+    def version
+      platform.release
+    end
+
+    def base_os
+      if platform.family == "windows"
+        :windows
+      elsif platform.linux?
+        :linux
+      else
+        # TODO - this seems like it shouldn't happen here, when
+        # all the caller is doing is asking about the OS
+        raise ChefApply::TargetHost::UnsupportedTargetOS.new(platform.name)
+      end
+    end
+
+    def platform
+      backend.platform
+    end
+
+    def run_command!(command, sudo_as_user = false)
+      result = run_command(command, sudo_as_user)
+      if result.exit_status != 0
+        raise RemoteExecutionFailed.new(@config[:host], command, result)
+      end
+      result
+    end
+
+    def run_command(command, sudo_as_user = false)
+      if config[:sudo] && sudo_as_user && base_os == :linux
+        command = "-u #{config[:user]} #{command}"
+      end
+      backend.run_command command
+    end
+
+    def upload_file(local_path, remote_path)
+      backend.upload(local_path, remote_path)
+    end
+
+    # Returns the installed chef version as a Gem::Version,
+    # or raised ChefNotInstalled if chef client version manifest can't
+    # be found.
+    def installed_chef_version
+      return @installed_chef_version if @installed_chef_version
+      # Note: In the case of a very old version of chef (that has no manifest - pre 12.0?)
+      #       this will report as not installed.
+      manifest = get_chef_version_manifest()
+      raise ChefNotInstalled.new if manifest == :not_found
+      # We'll split the version here because  unstable builds (where we currently
+      # install from) are in the form "Major.Minor.Build+HASH" which is not a valid
+      # version string.
+      @installed_chef_version = Gem::Version.new(manifest["build_version"].split("+")[0])
+    end
+
+    MANIFEST_PATHS = {
+      # TODO - use a proper method to query the win installation path -
+      #        currently we're assuming the default, but this can be customized
+      #        at install time.
+      #        A working approach is below - but it runs very slowly in testing
+      #        on a virtualbox windows vm:
+      #        (over winrm) Get-WmiObject Win32_Product | Where {$_.Name -match 'Chef Client'}
+      windows: "c:\\opscode\\chef\\version-manifest.json",
+      linux: "/opt/chef/version-manifest.json"
+    }
+
+    def get_chef_version_manifest
+      path = MANIFEST_PATHS[base_os()]
+      manifest = backend.file(path)
+      return :not_found unless manifest.file?
+      JSON.parse(manifest.content)
+    end
+
+    private
+
+    def train_connection
+      @train_connection
+    end
+
+    def ssh_config_for_host(host)
+      require "net/ssh"
+      Net::SSH::Config.for(host)
+    end
+
+    class RemoteExecutionFailed < ChefApply::ErrorNoLogs
+      attr_reader :stdout, :stderr
+      def initialize(host, command, result)
+        super("CHEFRMT001",
+              command,
+              result.exit_status,
+              host,
+              result.stderr.empty? ? result.stdout : result.stderr)
+      end
+    end
+
+    class ConnectionFailure < ChefApply::ErrorNoLogs
+      # TODO: Currently this only handles sudo-related errors;
+      # we should also look at e.cause for underlying connection errors
+      # which are presently only visible in log files.
+      def initialize(original_exception, connection_opts)
+        sudo_command = connection_opts[:sudo_command]
+        init_params =
+          #  Comments below show the original_exception.reason values to check for instead of strings,
+          #  after train 1.4.12 is consumable.
+          case original_exception.message # original_exception.reason
+          when /Sudo requires a password/ # :sudo_password_required
+            "CHEFTRN003"
+          when /Wrong sudo password/ #:bad_sudo_password
+            "CHEFTRN004"
+          when /Can't find sudo command/, /No such file/, /command not found/ # :sudo_command_not_found
+            # NOTE: In the /No such file/ case, reason will be nil - we still have
+            # to check message text. (Or PR to train to handle this case)
+            ["CHEFTRN005", sudo_command] # :sudo_command_not_found
+          when /Sudo requires a TTY.*/   # :sudo_no_tty
+            "CHEFTRN006"
+          when /has no keys added/
+            "CHEFTRN007"
+          else
+            ["CHEFTRN999", original_exception.message]
+          end
+        super(*(Array(init_params).flatten))
+      end
+    end
+
+    class ChefNotInstalled < StandardError; end
+
+    class UnsupportedTargetOS < ChefApply::ErrorNoLogs
+      def initialize(os_name); super("CHEFTARG001", os_name); end
+    end
+  end
+end