chef-core 0.0.1

data/lib/chef_core/target_resolver.rb

@@ -0,0 +1,221 @@
+#
+# Copyright:: Copyright (c) 2018 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "chef_core/target_host"
+require "chef_core/error"
+
+module ChefCore
+
+  # This class will resolve a provided target host name to
+  # one or more TargetHost instances.  The target host name passed in
+  # to the constructor can contain up to two ranges in the form [0-9] and [a-z].
+  # Multiple target host names can be passed in asa comma-separated list.
+  #
+  # Use resolver_inst.targets to get the expanded list of TargetHost instances.
+  class TargetResolver
+    MAX_EXPANDED_TARGETS = 24
+    SUPPORTED_PROTOCOLS = %w{winrm ssh}
+    def initialize(target, default_protocol, conn_options, max_expanded_targets: MAX_EXPANDED_TARGETS)
+      @max_expanded_targets = max_expanded_targets
+      @default_proto = default_protocol
+      @unparsed_target = target
+      @split_targets = @unparsed_target.split(",")
+      @conn_options = conn_options.dup
+      @default_password = @conn_options.delete(:password)
+      @default_user = @conn_options.delete(:user)
+    end
+
+    # Returns the list of targets as an array of TargetHost instances
+    def targets
+      return @targets unless @targets.nil?
+      expanded_urls = []
+      @split_targets.each do |target|
+        expand_targets(target).each { |t| expanded_urls << t }
+      end
+
+      # Apply max_expanded_targets to the total list of resolved
+      # targets, since multiple targets can be comma-separated.
+      # Limiting it only in the expression resolver could still
+      # yield more than the maximum.
+      if expanded_urls.length > @max_expanded_targets
+        raise TooManyTargets.new(expanded_urls.length, @max_expanded_targets)
+      end
+
+      @targets = expanded_urls.map do |url|
+        config = @conn_options.merge(config_for_target(url))
+        TargetHost.new(config.delete(:url), config)
+      end
+    end
+
+    def config_for_target(url)
+      prefix, target = prefix_from_target(url)
+
+      inline_password = nil
+      inline_user = nil
+      host = target
+      # Default greedy-scan of the regex means that
+      # $2 will resolve to content after the final "@"
+      # URL credentials will take precedence over the default :user
+      # in @conn_opts
+      if target =~ /(.*)@(.*)/
+        inline_credentials = $1
+        host = $2
+        # We'll use a non-greedy match to grab everthinmg up to the first ':'
+        # as username if there is no :, credentials is just the username
+        if inline_credentials =~ /(.+?):(.*)/
+          inline_user = $1
+          inline_password = $2
+        else
+          inline_user = inline_credentials
+        end
+      end
+      user, password = make_credentials(inline_user, inline_password)
+      { url: "#{prefix}#{host}",
+        user: user,
+        password: password }
+    end
+
+    # Merge the inline user/pass with the default user/pass, giving
+    # precedence to inline.
+    def make_credentials(inline_user, inline_password)
+      user = inline_user || @default_user
+      user = nil if user && user.empty?
+      password = (inline_password || @default_password)
+      password = nil if password && password.empty?
+      [user, password]
+    end
+
+    def prefix_from_target(target)
+      if target =~ /^(.+?):\/\/(.*)/
+        # We'll store the existing prefix to avoid it interfering
+        # with the check further below.
+        if SUPPORTED_PROTOCOLS.include? $1.downcase
+          prefix = "#{$1}://"
+          target = $2
+        else
+          raise UnsupportedProtocol.new($1)
+        end
+      else
+        prefix = "#{@default_proto}://"
+      end
+      [prefix, target]
+    end
+
+    def expand_targets(target)
+      @current_target = target # Hold onto this for error reporting
+      do_parse([target.downcase])
+    end
+
+    private
+
+    # A string matching PREFIX[x:y]POSTFIX:
+    # POSTFIX can contain further ranges itself
+    # This uses a greedy match (.*) to get include every character
+    # up to the last "[" in PREFIX
+    # $1 - prefix; $2 - x, $3 - y, $4 unproccessed/remaining text
+    TARGET_WITH_RANGE = /^(.*)\[([\p{Alnum}]+):([\p{Alnum}]+)\](.*)/.freeze
+
+    def do_parse(targets, depth = 0)
+      raise TooManyRanges.new(@current_target) if depth > 2
+      new_targets = []
+      done = false
+      targets.each do |target|
+        if TARGET_WITH_RANGE =~ target
+          # $1 - prefix; $2 - x, $3 - y, $4 unprocessed/remaining text
+          expand_range(new_targets, $1, $2, $3, $4)
+        else
+          # Nothing more to expand
+          done = true
+          new_targets << target
+        end
+      end
+      if done
+        new_targets
+      else
+        do_parse(new_targets, depth + 1)
+      end
+    end
+
+    def expand_range(dest, prefix, start, stop, suffix)
+      prefix ||= ""
+      suffix ||= ""
+      start_is_int = Integer(start) >= 0 rescue false
+      stop_is_int = Integer(stop) >= 0 rescue false
+
+      if (start_is_int && !stop_is_int) || (stop_is_int && !start_is_int)
+        raise InvalidRange.new(@current_target, "[#{start}:#{stop}]")
+      end
+
+      # Ensure that a numeric range doesn't get created as a string, which
+      # would make the created Range further below fail to iterate for some values
+      # because of ASCII sorting.
+      if start_is_int
+        start = Integer(start)
+      end
+
+      if stop_is_int
+        stop = Integer(stop)
+      end
+
+      # For range to iterate correctly, the values must
+      # be low,high
+      if start > stop
+        temp = stop; stop = start; start = temp
+      end
+      Range.new(start, stop).each do |value|
+        # Ranges will resolve only numbers and letters,
+        # not other ascii characters that happen to fall between.
+        if start_is_int || /^[a-z0-9]/ =~ value
+          dest << "#{prefix}#{value}#{suffix}"
+        end
+        # Stop expanding as soon as we go over limit to prevent
+        # making the user wait for a massive accidental expansion
+        if dest.length > @max_expanded_targets
+          raise TooManyTargets.new(@split_targets.length, @max_expanded_targets)
+        end
+      end
+    end
+    # Provide an error base for all targetresolver errors, to simplify
+    # handling when caller is not concerned with the specific failure.
+    class TargetResolverError  < ChefCore::Error; end
+
+    class InvalidRange < TargetResolverError
+      def initialize(unresolved_target, given_range)
+        super("CHEFRANGE001", unresolved_target, given_range)
+      end
+    end
+
+    class TooManyRanges < TargetResolverError
+      def initialize(unresolved_target)
+        super("CHEFRANGE002", unresolved_target)
+      end
+    end
+
+    class TooManyTargets < TargetResolverError
+      def initialize(num_top_level_targets, max_targets)
+        super("CHEFRANGE003", num_top_level_targets, max_targets)
+      end
+    end
+
+    class UnsupportedProtocol < TargetResolverError
+      def initialize(attempted_protocol)
+        super("CHEFVAL011", attempted_protocol,
+              ChefCore::TargetResolver::SUPPORTED_PROTOCOLS.join(" "))
+      end
+    end
+  end
+end

data/lib/chef_core/telemeter/patch.rb

@@ -0,0 +1,32 @@
+#
+# Copyright:: Copyright (c) 2018 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+class Telemetry
+  class Session
+    # The telemetry session data is normally kept in .chef, which we don't have.
+    def session_file
+      ChefCore::Config.telemetry_session_file.freeze
+    end
+  end
+
+  def deliver(data = {})
+    if ChefCore::Telemeter.instance.enabled?
+      payload = event.prepare(data)
+      client.await.fire(payload)
+    end
+  end
+end

data/lib/chef_core/telemeter/sender.rb

@@ -0,0 +1,123 @@
+#
+# Copyright:: Copyright (c) 2018 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "telemetry"
+require "chef_core/telemeter"
+require "chef_core/telemeter/patch"
+require "chef_core/log"
+require "chef_core/version"
+
+module ChefCore
+  class Telemeter
+    class Sender
+      attr_reader :session_files, :config
+
+      def self.start_upload_thread(config)
+        # Find the files before we spawn the thread - otherwise
+        # we may accidentally pick up the current run's session file if it
+        # finishes before the thread scans for new files
+        session_files = Sender.find_session_files(config)
+        sender = Sender.new(session_files, config)
+        Thread.new { sender.run }
+      end
+
+      def self.find_session_files(config)
+        ChefCore::Log.info("Looking for telemetry data to submit")
+        session_search = File.join(config[:payload_dir], "telemetry-payload-*.yml")
+        session_files = Dir.glob(session_search)
+        ChefCore::Log.info("Found #{session_files.length} sessions to submit")
+        session_files
+      end
+
+      def initialize(session_files, config)
+        @session_files = session_files
+        @config = config
+      end
+
+      def run
+        if ChefCore::Telemeter.enabled?
+          ChefCore::Log.info("Telemetry enabled, beginning upload of previous session(s)")
+          # dev mode telemetry gets sent to a different location
+
+          if config[:dev_mode]
+            ENV["CHEF_TELEMETRY_ENDPOINT"] ||= "https://telemetry-acceptance.chef.io"
+          end
+          session_files.each { |path| process_session(path) }
+        else
+          # If telemetry is not enabled, just clean up and return. Even though
+          # the telemetry gem will not send if disabled, log output saying that we're submitting
+          # it when it has been disabled can be alarming.
+          ChefCore::Log.info("Telemetry disabled, clearing any existing session captures without sending them.")
+          session_files.each { |path| FileUtils.rm_rf(path) }
+        end
+        FileUtils.rm_rf(config[:session_file])
+        ChefCore::Log.info("Terminating, nothing more to do.")
+      rescue => e
+        ChefCore::Log.fatal "Sender thread aborted: '#{e}' failed at  #{e.backtrace[0]}"
+      end
+
+      def process_session(path)
+        ChefCore::Log.info("Processing telemetry entries from #{path}")
+        content = load_and_clear_session(path)
+        submit_session(content)
+      end
+
+      def submit_session(content)
+        # Each file contains the actions taken within a single run of the chef tool.
+        # Each run is one session, so we'll first remove remove the session file
+        # to force creating a new one.
+        FileUtils.rm_rf(config[:session_file])
+        # We'll use the version captured in the sesion file
+        entries = content["entries"]
+        cli_version = content["version"]
+        total = entries.length
+        telemetry = Telemetry.new(product: "chef-workstation",
+                                  origin: "command-line",
+                                  product_version: cli_version,
+                                  install_context: "omnibus")
+        total = entries.length
+        entries.each_with_index do |entry, x|
+          submit_entry(telemetry, entry, x + 1, total)
+        end
+      end
+
+      def submit_entry(telemetry, entry, sequence, total)
+        ChefCore::Log.info("Submitting telemetry entry #{sequence}/#{total}: #{entry} ")
+        telemetry.deliver(entry)
+        ChefCore::Log.info("Entry #{sequence}/#{total} submitted.")
+      rescue => e
+        # No error handling in telemetry lib, so at least track the failrue
+        ChefCore::Log.error("Failed to send entry #{sequence}/#{total}: #{e}")
+        ChefCore::Log.error("Backtrace: #{e.backtrace} ")
+      end
+
+      private
+
+      def load_and_clear_session(path)
+        content = File.read(path)
+        # We'll remove it now instead of after we parse or submit it -
+        # if we fail to deliver, we don't want to be stuck resubmitting it if the problem
+        # was due to payload. This is a trade-off - if we get a transient error, the
+        # payload will be lost.
+        # TODO: Improve error handling so we can intelligently decide whether to
+        #       retry a failed load or failed submit.
+        FileUtils.rm_rf(path)
+        YAML.load(content)
+      end
+    end
+  end
+end

data/lib/chef_core/telemeter.rb

@@ -0,0 +1,157 @@
+#
+# Copyright:: Copyright (c) 2018 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "benchmark"
+require "forwardable"
+require "singleton"
+require "json"
+require "digest/sha1"
+require "securerandom"
+require "yaml"
+
+module ChefCore
+
+  # This definites the Telemeter interface. Implementation thoughts for
+  # when we unstub it:
+  # - let's track the call sequence; most of our calls will be nested inside
+  # a main 'timed_capture', and it would be good to see ordering within nested calls.
+  class Telemeter
+    include Singleton
+    DEFAULT_INSTALLATION_GUID = "00000000-0000-0000-0000-000000000000".freeze
+
+    class << self
+      extend Forwardable
+      def_delegators :instance, :setup, :timed_capture, :capture, :commit, :timed_run_capture
+      def_delegators :instance, :pending_event_count, :last_event, :enabled?
+      def_delegators :instance, :make_event_payload, :config
+    end
+
+    attr_reader :events_to_send, :run_timestamp, :config
+
+    def setup(config)
+      # TODO validate required & correct keys
+      # :payload_dir #required
+      # :session_file # required
+      # :installation_identifier_file # required
+      # :enabled  # false, not required
+      # :dev_mode # false, not required
+      config[:dev_mode] ||= false
+      config[:enabled] ||= false
+      require "chef_core/telemeter/sender"
+      @config = config
+      Sender.start_upload_thread(config)
+    end
+
+    def enabled?
+      require "telemetry/decision"
+      config[:enabled] && !Telemetry::Decision.env_opt_out?
+    end
+
+    def initialize
+      @events_to_send = []
+      @run_timestamp =  Time.now.utc.strftime("%FT%TZ")
+    end
+
+    def timed_run_capture(arguments, &block)
+      timed_capture(:run, arguments: arguments, &block)
+    end
+
+    def timed_capture(name, data = {})
+      time = Benchmark.measure { yield }
+      data[:duration] = time.real
+      capture(name, data)
+    end
+
+    def capture(name, data = {})
+      # Adding it to the head of the list will ensure that the
+      # sequence of events is preserved when we send the final payload
+      payload = make_event_payload(name, data)
+      @events_to_send.unshift payload
+    end
+
+    def commit
+      if enabled?
+        session = convert_events_to_session
+        write_session(session)
+      end
+      @events_to_send = []
+    end
+
+    def make_event_payload(name, data)
+      {
+        event: name,
+        properties: {
+          installation_id: installation_id,
+          run_timestamp: run_timestamp,
+          host_platform: host_platform,
+          event_data: data,
+        },
+      }
+    end
+
+    def installation_id
+      @installation_id ||=
+        begin
+          File.read(config[:installation_identifier_file]).chomp
+        rescue
+          ChefCore::Log.info "could not read #{config[:installation_identifier_file]} - using default id"
+          DEFAULT_INSTALLATION_GUID
+        end
+    end
+
+    # For testing.
+    def pending_event_count
+      @events_to_send.length
+    end
+
+    def last_event
+      @events_to_send.last
+    end
+
+    private
+
+    def host_platform
+      @host_platform ||= case RUBY_PLATFORM
+                         when /mswin|mingw|windows/
+                           "windows"
+                         else
+                           RUBY_PLATFORM.split("-")[1]
+                         end
+    end
+
+    def convert_events_to_session
+      YAML.dump({ "version" => ChefCore::VERSION,
+                  "entries" => @events_to_send })
+    end
+
+    def write_session(session)
+      File.write(next_filename, convert_events_to_session)
+    end
+
+    def next_filename
+      id = 0
+      filename = ""
+      loop do
+        id += 1
+        filename = File.join(config[:payload_dir], "telemetry-payload-#{id}.yml")
+        break unless File.exist?(filename)
+      end
+      filename
+    end
+
+  end
+end

data/lib/chef_core/text/error_translation.rb

@@ -0,0 +1,82 @@
+#
+# Copyright:: Copyright (c) 2017 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module ChefCore
+  module Text
+    # Represents an error loaded from translation, with
+    # display attributes set.
+    class ErrorTranslation
+      ATTRIBUTES = [:decorations, :header, :footer, :stack, :log].freeze
+      attr_reader :message, *ATTRIBUTES
+
+      def initialize(id, params: [])
+        error_translation = error_translation_for_id(id)
+
+        options = YAML.load(Text.errors.display_defaults, "display_defaults",
+                            symbolize_names: true)
+
+        # Display metadata is a string containing a YAML hash that is optionally under
+        # the error's 'options' attribute
+        # Note that we couldn't use :display, as that conflicts with a method on Object.
+        display_opts = if error_translation.methods.include?(:options)
+                         YAML.load(error_translation.options, @id, symbolize_names: true)
+                       else
+                         {}
+                       end
+
+        options = options.merge(display_opts) unless display_opts.nil?
+
+        @message = error_translation.text(*params)
+
+        ATTRIBUTES.each do |attribute|
+          instance_variable_set("@#{attribute}", options.delete(attribute))
+        end
+
+        if options.length > 0
+          # Anything not in ATTRIBUTES is not supported. This will also catch
+          # typos in attr names
+          raise InvalidDisplayAttributes.new(id, options)
+        end
+      end
+
+      def inspect
+        inspection = "#{self}: "
+        ATTRIBUTES.each do |attribute|
+          inspection << "#{attribute}: #{send(attribute.to_s)}; "
+        end
+        inspection << "message: #{message.gsub("\n", "\\n")}"
+        inspection
+      end
+
+      private
+
+      # This is split out for simplified unit testing of error formatting.
+      def error_translation_for_id(id)
+        Text.errors.send(id)
+      end
+
+      class InvalidDisplayAttributes < RuntimeError
+        attr_reader :invalid_attrs
+        def initialize(id, attrs)
+          @invalid_attrs = attrs
+          super("Invalid display attributes found for #{id}: #{attrs}")
+        end
+      end
+
+    end
+  end
+end

data/lib/chef_core/text/text_wrapper.rb

@@ -0,0 +1,87 @@
+#
+# Copyright:: Copyright (c) 2017 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module ChefCore
+  module Text
+    # TextWrapper is a wrapper around R18n that returns all resolved values
+    # as Strings, and raise an error when a given i18n key is not found.
+    #
+    # This simplifies behaviors when interfacing with other libraries/components
+    # that don't play nicely with TranslatedString or Untranslated out of R18n.
+    class TextWrapper
+      def initialize(translation_tree)
+        @tree = translation_tree
+        @tree.translation_keys.each do |k|
+          # Integer keys are not translatable - they're quantity indicators in the key that
+          # are instead sent as arguments. If we see one here, it means it was not correctly
+          # labeled as plural with !!pl in the parent key
+          if k.class == Integer
+            raise MissingPlural.new(@tree.instance_variable_get(:@path), k)
+          end
+          k = k.to_sym
+          define_singleton_method k do |*args|
+            subtree = @tree.send(k, *args)
+            if subtree.methods.include?(:translation_keys) && !subtree.translation_keys.empty?
+              # If there are no more possible children, just return the translated value
+              TextWrapper.new(subtree)
+            else
+              subtree.to_s
+            end
+          end
+        end
+      end
+
+      def method_missing(name, *args)
+        raise InvalidKey.new(@tree.instance_variable_get(:@path), name)
+      end
+
+      # TODO - make the checks for these conditions lint steps that run during build
+      #        instead of part of the shipped product.
+      class TextError < RuntimeError
+        attr_accessor :line
+        def set_call_context
+          # TODO - this can vary (8 isn't always right) - inspect
+          @line = caller(8, 1).first
+          if @line =~ /.*\/lib\/(.*\.rb):(\d+)/
+            @line = "File: #{$1} Line: #{$2}"
+          end
+        end
+      end
+
+      class InvalidKey < TextError
+        def initialize(path, terminus)
+          set_call_context
+          # Calling back into Text here seems icky, this is an error
+          # that only engineering should see.
+          message = "i18n key #{path}.#{terminus} does not exist.\n"
+          message << "  Referenced from #{line}"
+          super(message)
+        end
+      end
+
+      class MissingPlural < TextError
+        def initialize(path, terminus)
+          set_call_context
+          message = "i18n key #{path}.#{terminus} appears to reference a pluralization.\n"
+          message << "  Please append the plural indicator '!!pl' to the end of #{path}.\n"
+          message << "  Referenced from #{line}"
+          super(message)
+        end
+      end
+    end
+  end
+end