chef-apply 0.1.2

data/lib/chef_apply/target_resolver.rb

@@ -0,0 +1,202 @@
+#
+# Copyright:: Copyright (c) 2018 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "chef_apply/target_host"
+require "chef_apply/error"
+
+module ChefApply
+  class TargetResolver
+    MAX_EXPANDED_TARGETS = 24
+
+    def initialize(target, default_protocol, conn_options)
+      @default_proto = default_protocol
+      @unparsed_target = target
+      @split_targets = @unparsed_target.split(",")
+      @conn_options = conn_options.dup
+      @default_password = @conn_options.delete(:password)
+      @default_user = @conn_options.delete(:user)
+    end
+
+    # Returns the list of targets as an array of TargetHost instances,
+    # them to account for ranges embedded in the target name.
+    def targets
+      return @targets unless @targets.nil?
+      expanded_urls = []
+      @split_targets.each do |target|
+        expanded_urls = (expanded_urls | expand_targets(target))
+      end
+      @targets = expanded_urls.map do |url|
+        config = @conn_options.merge(config_for_target(url))
+        TargetHost.new(config.delete(:url), config)
+      end
+    end
+
+    def config_for_target(url)
+      prefix, target = prefix_from_target(url)
+
+      inline_password = nil
+      inline_user = nil
+      host = target
+      # Default greedy-scan of the regex means that
+      # $2 will resolve to content after the final "@"
+      # URL credentials will take precedence over the default :user
+      # in @conn_opts
+      if target =~ /(.*)@(.*)/
+        inline_credentials = $1
+        host = $2
+        # We'll use a non-greedy match to grab everthinmg up to the first ':'
+        # as username if there is no :, credentials is just the username
+        if inline_credentials =~ /(.+?):(.*)/
+          inline_user = $1
+          inline_password = $2
+        else
+          inline_user = inline_credentials
+        end
+      end
+      user, password = make_credentials(inline_user, inline_password)
+      { url: "#{prefix}#{host}",
+        user: user,
+        password: password }
+    end
+
+    # Merge the inline user/pass with the default user/pass, giving
+    # precedence to inline.
+    def make_credentials(inline_user, inline_password)
+      user = inline_user || @default_user
+      user = nil if user && user.empty?
+      password = (inline_password || @default_password)
+      password = nil if password && password.empty?
+      [user, password]
+    end
+
+    def prefix_from_target(target)
+      if target =~ /^(.+?):\/\/(.*)/
+        # We'll store the existing prefix to avoid it interfering
+        # with the check further below.
+        if ChefApply::Config::SUPPORTED_PROTOCOLS.include? $1.downcase
+          prefix = "#{$1}://"
+          target = $2
+        else
+          raise UnsupportedProtocol.new($1)
+        end
+      else
+        prefix = "#{@default_proto}://"
+      end
+      [prefix, target]
+    end
+
+    def expand_targets(target)
+      @current_target = target # Hold onto this for error reporting
+      do_parse([target.downcase])
+    end
+
+    private
+
+    # A string matching PREFIX[x:y]POSTFIX:
+    # POSTFIX can contain further ranges itself
+    # This uses a greedy match (.*) to get include every character
+    # up to the last "[" in PREFIX
+    # $1 - prefix; $2 - x, $3 - y, $4 unproccessed/remaining text
+    TARGET_WITH_RANGE = /^(.*)\[([\p{Alnum}]+):([\p{Alnum}]+)\](.*)/
+
+    def do_parse(targets, depth = 0)
+      raise TooManyRanges.new(@current_target) if depth > 2
+      new_targets = []
+      done = false
+      targets.each do |target|
+        if TARGET_WITH_RANGE =~ target
+          # $1 - prefix; $2 - x, $3 - y, $4 unprocessed/remaining text
+          expand_range(new_targets, $1, $2, $3, $4)
+        else
+          # Nothing more to expand
+          done = true
+          new_targets << target
+        end
+      end
+      if done
+        new_targets
+      else
+        do_parse(new_targets, depth + 1)
+      end
+    end
+
+    def expand_range(dest, prefix, start, stop, suffix)
+      prefix ||= ""
+      suffix ||= ""
+      start_is_int = Integer(start) >= 0 rescue false
+      stop_is_int = Integer(stop) >= 0 rescue false
+
+      if (start_is_int && !stop_is_int) || (stop_is_int && !start_is_int)
+        raise InvalidRange.new(@current_target, "[#{start}:#{stop}]")
+      end
+
+      # Ensure that a numeric range doesn't get created as a string, which
+      # would make the created Range further below fail to iterate for some values
+      # because of ASCII sorting.
+      if start_is_int
+        start = Integer(start)
+      end
+
+      if stop_is_int
+        stop = Integer(stop)
+      end
+
+      # For range to iterate correctly, the values must
+      # be low,high
+      if start > stop
+        temp = stop; stop = start; start = temp
+      end
+      Range.new(start, stop).each do |value|
+        # Ranges will resolve only numbers and letters,
+        # not other ascii characters that happen to fall between.
+        if start_is_int || /^[a-z0-9]/ =~ value
+          dest << "#{prefix}#{value}#{suffix}"
+        end
+        # Stop expanding as soon as we go over limit to prevent
+        # making the user wait for a massive accidental expansion
+        if dest.length > MAX_EXPANDED_TARGETS
+          raise TooManyTargets.new(@split_targets.length, MAX_EXPANDED_TARGETS)
+        end
+      end
+    end
+
+    class InvalidRange < ErrorNoLogs
+      def initialize(unresolved_target, given_range)
+        super("CHEFRANGE001", unresolved_target, given_range)
+      end
+    end
+
+    class TooManyRanges < ErrorNoLogs
+      def initialize(unresolved_target)
+        super("CHEFRANGE002", unresolved_target)
+      end
+    end
+
+    class TooManyTargets < ErrorNoLogs
+      def initialize(num_top_level_targets, max_targets)
+        super("CHEFRANGE003", num_top_level_targets, max_targets)
+      end
+    end
+
+    class UnsupportedProtocol < ErrorNoLogs
+      def initialize(attempted_protocol)
+        super("CHEFVAL011", attempted_protocol,
+              ChefApply::Config::SUPPORTED_PROTOCOLS.join(" "))
+      end
+    end
+  end
+end

data/lib/chef_apply/telemeter.rb

@@ -0,0 +1,162 @@
+#
+# Copyright:: Copyright (c) 2018 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "benchmark"
+require "forwardable"
+require "singleton"
+require "json"
+require "digest/sha1"
+require "securerandom"
+require "chef_apply/version"
+require "chef_apply/config"
+require "yaml"
+
+module ChefApply
+
+  # This definites the Telemeter interface. Implementation thoughts for
+  # when we unstub it:
+  # - let's track the call sequence; most of our calls will be nested inside
+  # a main 'timed_capture', and it would be good to see ordering within nested calls.
+  class Telemeter
+    include Singleton
+    DEFAULT_INSTALLATION_GUID = "00000000-0000-0000-0000-000000000000"
+
+    class << self
+      extend Forwardable
+      def_delegators :instance, :timed_capture, :capture, :commit, :timed_action_capture, :timed_run_capture
+      def_delegators :instance, :pending_event_count, :last_event, :enabled?
+      def_delegators :instance, :make_event_payload
+    end
+
+    attr_reader :events_to_send, :run_timestamp
+
+    def enabled?
+      require "telemetry/decision"
+      ChefApply::Config.telemetry.enable && !Telemetry::Decision.env_opt_out?
+    end
+
+    def initialize
+      @events_to_send = []
+      @run_timestamp =  Time.now.utc.strftime("%FT%TZ")
+    end
+
+    def timed_action_capture(action, &block)
+      # Note: we do not directly capture hostname for privacy concerns, but
+      # using a sha1 digest will allow us to anonymously see
+      # unique hosts to derive number of hosts affected by a command
+      target = action.target_host
+      target_data = { platform: {}, hostname_sha1: nil, transport_type: nil }
+      if target
+        target_data[:platform][:name] = target.base_os # :windows, :linux, eventually :macos
+        target_data[:platform][:version] = target.version
+        target_data[:platform][:architecture] = target.architecture
+        target_data[:hostname_sha1] = Digest::SHA1.hexdigest(target.hostname.downcase)
+        target_data[:transport_type] = target.transport_type
+      end
+      timed_capture(:action, { action: action.name, target: target_data }, &block)
+    end
+
+    def timed_run_capture(arguments, &block)
+      timed_capture(:run, arguments: arguments, &block)
+    end
+
+    def capture(name, data = {})
+      # Adding it to the head of the list will ensure that the
+      # sequence of events is preserved when we send the final payload
+      payload = make_event_payload(name, data)
+      @events_to_send.unshift payload
+    end
+
+    def timed_capture(name, data = {})
+      time = Benchmark.measure { yield }
+      data[:duration] = time.real
+      capture(name, data)
+    end
+
+    def commit
+      if enabled?
+        session = convert_events_to_session
+        write_session(session)
+      end
+      @events_to_send = []
+    end
+
+    def make_event_payload(name, data)
+      {
+        event: name,
+        properties: {
+          installation_id: installation_id,
+          run_timestamp: run_timestamp,
+          host_platform: host_platform,
+          event_data: data
+        }
+      }
+    end
+
+    def installation_id
+      @installation_id ||=
+        begin
+          File.read(ChefApply::Config.telemetry_installation_identifier_file).chomp
+        rescue
+          ChefApply::Log.info "could not read #{ChefApply::Config.telemetry_installation_identifier_file} - using default id"
+          DEFAULT_INSTALLATION_GUID
+        end
+    end
+
+    # For testing.
+    def pending_event_count
+      @events_to_send.length
+    end
+
+    def last_event
+      @events_to_send.last
+    end
+
+    private
+
+    def host_platform
+      @host_platform ||= case RUBY_PLATFORM
+                         when /mswin|mingw|windows/
+                           "windows"
+                         else
+                           RUBY_PLATFORM.split("-")[1]
+                         end
+    end
+
+    def convert_events_to_session
+      YAML.dump({ "version" => ChefApply::VERSION,
+                  "entries" => @events_to_send })
+    end
+
+    def write_session(session)
+      File.write(next_filename, convert_events_to_session)
+    end
+
+    def next_filename
+      id = 0
+      filename = ""
+      loop do
+        id += 1
+        filename = File.join(ChefApply::Config.telemetry_path,
+                             "telemetry-payload-#{id}.yml")
+        break unless File.exist?(filename)
+      end
+      filename
+    end
+
+  end
+end

data/lib/chef_apply/telemeter/patch.rb

@@ -0,0 +1,32 @@
+#
+# Copyright:: Copyright (c) 2018 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+class Telemetry
+  class Session
+    # The telemetry session data is normally kept in .chef, which we don't have.
+    def session_file
+      ChefApply::Config.telemetry_session_file.freeze
+    end
+  end
+
+  def deliver(data = {})
+    if ChefApply::Telemeter.instance.enabled?
+      payload = event.prepare(data)
+      client.await.fire(payload)
+    end
+  end
+end

data/lib/chef_apply/telemeter/sender.rb

@@ -0,0 +1,121 @@
+#
+# Copyright:: Copyright (c) 2018 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "telemetry"
+require "chef_apply/telemeter"
+require "chef_apply/telemeter/patch"
+require "chef_apply/log"
+require "chef_apply/version"
+
+module ChefApply
+  class Telemeter
+    class Sender
+      attr_reader :session_files
+
+      def self.start_upload_thread
+        # Find the files before we spawn the thread - otherwise
+        # we may accidentally pick up the current run's session file if it
+        # finishes before the thread scans for new files
+        session_files = Sender.find_session_files
+        sender = Sender.new(session_files)
+        Thread.new { sender.run }
+      end
+
+      def self.find_session_files
+        ChefApply::Log.info("Looking for telemetry data to submit")
+        session_search = File.join(ChefApply::Config.telemetry_path, "telemetry-payload-*.yml")
+        session_files = Dir.glob(session_search)
+        ChefApply::Log.info("Found #{session_files.length} sessions to submit")
+        session_files
+      end
+
+      def initialize(session_files)
+        @session_files = session_files
+      end
+
+      def run
+        if ChefApply::Telemeter.enabled?
+          ChefApply::Log.info("Telemetry enabled, beginning upload of previous session(s)")
+          # dev mode telemetry gets sent to a different location
+          if ChefApply::Config.telemetry.dev
+            ENV["CHEF_TELEMETRY_ENDPOINT"] ||= "https://telemetry-acceptance.chef.io"
+          end
+          session_files.each { |path| process_session(path) }
+        else
+          # If telemetry is not enabled, just clean up and return. Even though
+          # the telemetry gem will not send if disabled, log output saying that we're submitting
+          # it when it has been disabled can be alarming.
+          ChefApply::Log.info("Telemetry disabled, clearing any existing session captures without sending them.")
+          session_files.each { |path| FileUtils.rm_rf(path) }
+        end
+        FileUtils.rm_rf(ChefApply::Config.telemetry_session_file)
+        ChefApply::Log.info("Terminating, nothing more to do.")
+      rescue => e
+        ChefApply::Log.fatal "Sender thread aborted: '#{e}' failed at  #{e.backtrace[0]}"
+      end
+
+      def process_session(path)
+        ChefApply::Log.info("Processing telemetry entries from #{path}")
+        content = load_and_clear_session(path)
+        submit_session(content)
+      end
+
+      def submit_session(content)
+        # Each file contains the actions taken within a single run of the chef tool.
+        # Each run is one session, so we'll first remove remove the session file
+        # to force creating a new one.
+        FileUtils.rm_rf(ChefApply::Config.telemetry_session_file)
+        # We'll use the version captured in the sesion file
+        entries = content["entries"]
+        cli_version = content["version"]
+        total = entries.length
+        telemetry = Telemetry.new(product: "chef-workstation",
+                                  origin: "command-line",
+                                  product_version: cli_version,
+                                  install_context: "omnibus")
+        total = entries.length
+        entries.each_with_index do |entry, x|
+          submit_entry(telemetry, entry, x + 1, total)
+        end
+      end
+
+      def submit_entry(telemetry, entry, sequence, total)
+        ChefApply::Log.info("Submitting telemetry entry #{sequence}/#{total}: #{entry} ")
+        telemetry.deliver(entry)
+        ChefApply::Log.info("Entry #{sequence}/#{total} submitted.")
+      rescue => e
+        # No error handling in telemetry lib, so at least track the failrue
+        ChefApply::Log.error("Failed to send entry #{sequence}/#{total}: #{e}")
+        ChefApply::Log.error("Backtrace: #{e.backtrace} ")
+      end
+
+      private
+
+      def load_and_clear_session(path)
+        content = File.read(path)
+        # We'll remove it now instead of after we parse or submit it -
+        # if we fail to deliver, we don't want to be stuck resubmitting it if the problem
+        # was due to payload. This is a trade-off - if we get a transient error, the
+        # payload will be lost.
+        # TODO: Improve error handling so we can intelligently decide whether to
+        #       retry a failed load or failed submit.
+        FileUtils.rm_rf(path)
+        YAML.load(content)
+      end
+    end
+  end
+end