smart_proxy_salt 3.0.0 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 978fe04cd0b2ee65be99fc84f20599f295d51498b8aa1ba0541cc9569f56ee9a
-  data.tar.gz: 732c3b575790b37ec8570816f76d10af534b10cf3fb7ea217b7378f645c951f9
+  metadata.gz: 63743a9b37c7860b60af93db9735f210608221d87675c7d11764eb9f08348caa
+  data.tar.gz: 38fa3a0d210534f63d1f925790d0a062b20a907499dad0ba2bae1125a0bb8190
 SHA512:
-  metadata.gz: d291974d5e815bb0a8d84f46d8f4e8509d0c70dc8dc76fa9d35d0f836ff7e55f3fe7f67c3cd0d5fc1452d2747b528ccca05c0de7d881042468b652a30824dd5e
-  data.tar.gz: e41c156691bb2c1c86381658104a4fb54fb4d6a22a8e1778b7dc6ce9f39d40183d649699c49b8c835815c16b073886d68d0988bf0a019c79d8fbb001cdc7133c
+  metadata.gz: '008b29492b0f227e04ba281423df738e9334e040492f2fcf62172e866cccd931a80de5a5eac14d8e5b4c81a17dd26697e11f89aa696ab28670471f55be585e26'
+  data.tar.gz: 05a4afde3202a35328d2b2daa4cc2266808a9c7af01b53dae3876a4a8b033ef898fc07e1d86bfc9a22c05b7ab3bcdb924884c34b1d5546e1a8c93a39ec256623

data/bin/foreman-node

@@ -14,6 +14,7 @@ require 'net/http'
 require 'net/https'
 require 'etc'
 require 'timeout'
+require 'msgpack' if SETTINGS[:filecache]
 
 begin
   require 'json'
@@ -57,48 +58,37 @@ rescue Exception => e
   exit 1
 end
 
+def get_grains_from_filecache(minion)
+  # Use the grains from the salt master's filesystem based cache
+  # This requires the following settings in /etc/salt/foreman.yaml:
+  # :filecache: true
+  # :cachedir: "/path/to/master/cache" (default: "/var/cache/salt/master")
+  # Also, the msgpack rubygem needs to be present
+  cachedir = SETTINGS[:cachedir] || '/var/cache/salt/master'
+  content = File.read("#{cachedir}/minions/#{minion}/data.p")
+  data = MessagePack.unpack(content)
+  data['grains']
+end
+
+def get_grains_from_saltrun(minion)
+  result = IO.popen(['salt-run', '-l', 'quiet', '--output=json', 'cache.grains', minion], &:read)
+  data = JSON.parse(result)
+  data[minion]
+end
+
 def plain_grains(minion)
   # We have to get the grains from the cache, because the client
   # is probably running 'state.highstate' right now.
-  #
-  # salt-run doesn't support directly outputting to json,
-  # so we have resort to python to extract the grains.
-  # Based on https://github.com/saltstack/salt/issues/9444
-
-  script = <<-EOF
-#!/usr/bin/env python
-import json
-import os
-import sys
-
-import salt.config
-import salt.runner
-
-if __name__ == '__main__':
-    __opts__ = salt.config.master_config(
-            os.environ.get('SALT_MASTER_CONFIG', '/etc/salt/master'))
-    runner = salt.runner.Runner(__opts__)
-
-    stdout_bak = sys.stdout
-    with open(os.devnull, 'wb') as f:
-        sys.stdout = f
-        ret = runner.cmd('cache.grains', ['#{minion}'])
-    sys.stdout = stdout_bak
-
-    print json.dumps(ret)
-EOF
-
-  result = IO.popen('python 2>/dev/null', 'r+') do |python|
-    python.write script
-    python.close_write
-    result = python.read
-  end
 
-  grains = JSON.parse(result)
+  grains = if SETTINGS[:filecache]
+             get_grains_from_filecache(minion)
+           else
+             get_grains_from_saltrun(minion)
+           end
 
   raise 'No grains received from Salt master' unless grains
 
-  plainify(grains[minion]).flatten.inject(&:merge)
+  plainify(grains).flatten.inject(&:merge)
 end
 
 def plainify(hash, prefix = nil)

data/bin/salt_python_wrapper

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+set -u
+
+for py in 'python3' 'python'; do
+  exe=$(type -p ${py})
+  if [ -n "${exe}" ]; then
+    if ${exe} -c 'import salt.config'; then
+      ${exe} "$@"
+      exit $?
+    fi
+  fi
+done
+
+echo "No usable python version found, check if python or python3 can import salt.config!" 1>&2
+exit 1

data/lib/smart_proxy_salt/cli.rb

@@ -14,44 +14,43 @@ module Proxy
           Proxy::Salt::Plugin.settings.autosign_file
         end
 
-        def autosign_create(host)
-          FileUtils.touch(autosign_file) unless File.exist?(autosign_file)
-
-          autosign = open(autosign_file, File::RDWR)
+        def autosign_key_file
+          Proxy::Salt::Plugin.settings.autosign_key_file
+        end
 
-          found = false
-          autosign.each_line { |line| found = true if line.chomp == host }
-          autosign.puts host unless found
-          autosign.close
+        def autosign_create_hostname(hostname)
+          if append_value_to_file(autosign_file, hostname)
+            { message: 'Added hostname successfully.' }
+          else
+            { message: 'Failed to add hostname.' \
+                       ' See smart proxy error log for more information.' }
+          end
+        end
 
-          result = { :message => "Added #{host} to autosign" }
-          logger.info result[:message]
-          result
+        def autosign_remove_hostname(hostname)
+          if remove_value_from_file(autosign_file, hostname)
+            { message: 'Removed hostname successfully.' }
+          else
+            { message: 'Failed to remove hostname.' \
+                       ' See smart proxy error log for more information.' }
+          end
         end
 
-        def autosign_remove(host)
-          raise "No such file #{autosign_file}" unless File.exist?(autosign_file)
+        def autosign_create_key(key)
+          if append_value_to_file(autosign_key_file, key)
+            { message: 'Added key successfully.' }
+          else
+            { message: 'Failed to add key.' \
+                       ' See smart proxy error log for more information.' }
+          end
+        end
 
-          found = false
-          entries = open(autosign_file, File::RDONLY).readlines.collect do |l|
-            if l.chomp != host
-              l
-            else
-              found = true
-              nil
-            end
-          end.uniq.compact
-          if found
-            autosign = open(autosign_file, File::TRUNC | File::RDWR)
-            autosign.write entries.join("\n")
-            autosign.write "\n"
-            autosign.close
-            result = { :message => "Removed #{host} from autosign" }
-            logger.info result[:message]
-            result
+        def autosign_remove_key(key)
+          if remove_value_from_file(autosign_key_file, key)
+            { message: 'Removed key successfully.' }
           else
-            logger.info "Attempt to remove nonexistant client autosign for #{host}"
-            raise Proxy::Salt::NotFound.new("Attempt to remove nonexistant client autosign for #{host}")
+            { message: 'Failed to remove key.' \
+                       ' See smart proxy error log for more information.' }
           end
         end
 
@@ -62,6 +61,57 @@ module Proxy
           end.map(&:chomp)
         end
 
+        def append_value_to_file(filepath, value)
+          result = false
+          begin
+            raise "No such file: #{filepath}" unless File.exist?(filepath)
+
+            file = open(filepath, File::RDWR)
+            found = false
+            file.each_line { |line| found = true if line.chomp == value }
+            file.puts value unless found
+            file.close
+
+            logger.info "Added an entry to '#{filepath}' successfully."
+            result = true
+          rescue SystemCallError => e
+            logger.info "Attempted to add an entry to '#{filepath}', but an exception occurred: #{e}"
+          end
+          result
+        end
+
+        def remove_value_from_file(filepath, value)
+          result = false
+          begin
+            raise "No such file: #{filepath}" unless File.exist?(filepath)
+
+            found = false
+            entries = open(filepath, File::RDONLY).readlines.collect do |l|
+              entry = l.chomp
+              if entry == value
+                found = true
+                nil
+              elsif entry == ""
+                nil
+              else
+                l
+              end
+            end.uniq.compact
+            if found
+              file = open(filepath, File::TRUNC | File::RDWR)
+              file.write entries.join()
+              file.close
+              logger.info "Removed an entry from '#{filepath}' successfully."
+              result = true
+            else
+              raise Proxy::Salt::NotFound.new("Attempt to remove non-existent entry.")
+            end
+          rescue SystemCallError => e
+            logger.info "Attempted to remove an entry from '#{filepath}', but an exception occurred: #{e}"
+          end
+          result
+        end
+
         def highstate(host)
           find_salt_binaries
           cmd = [@sudo, '-u', Proxy::Salt::Plugin.settings.salt_command_user, @salt, '--async', escape_for_shell(host), 'state.highstate']

data/lib/smart_proxy_salt/salt.rb

@@ -13,11 +13,24 @@ module Proxy
       plugin 'salt', Proxy::Salt::VERSION
 
       default_settings :autosign_file      => '/etc/salt/autosign.conf',
+                       :autosign_key_file  => '/var/lib/foreman-proxy/salt/grains/autosign_key',
                        :salt_command_user  => 'root',
-                       :use_api            => false
+                       :use_api            => false,
+                       :saltfile           => '/etc/foreman-proxy/settings.d/salt.saltfile'
 
-      http_rackup_path File.expand_path('salt_http_config.ru', File.expand_path('../', __FILE__))
-      https_rackup_path File.expand_path('salt_http_config.ru', File.expand_path('../', __FILE__))
+      requires :dynflow, '>= 0.5.0'
+
+      rackup_path File.expand_path('salt_http_config.ru', __dir__)
+
+      load_classes do
+        require 'smart_proxy_dynflow'
+        require 'smart_proxy_salt/salt_runner'
+        require 'smart_proxy_salt/salt_task_launcher'
+      end
+
+      load_dependency_injection_wirings do |_container_instance, _settings|
+        Proxy::Dynflow::TaskLauncherRegistry.register('salt', SaltTaskLauncher)
+      end
     end
 
     class << self
@@ -33,6 +46,10 @@ module Proxy
           super
         end
       end
+
+      def respond_to_missing?(method, include_private = false)
+        Proxy::Salt::Rest.respond_to?(method) || Proxy::Salt::CLI.respond_to?(method) || super
+      end
     end
   end
 end

data/lib/smart_proxy_salt/salt_api.rb

@@ -9,12 +9,32 @@ module Proxy
     class Api < ::Sinatra::Base
       include ::Proxy::Log
       helpers ::Proxy::Helpers
-      authorize_with_ssl_client
+      authorize_with_trusted_hosts
+
+      post '/autosign_key/:key' do
+        content_type :json
+        begin
+          Proxy::Salt.autosign_create_key(params[:key]).to_json
+        rescue Exception => e
+          log_halt 406, "Failed to create autosign key #{params[:key]}: #{e}"
+        end
+      end
+
+      delete '/autosign_key/:key' do
+        content_type :json
+        begin
+          Proxy::Salt.autosign_remove_key(params[:key]).to_json
+        rescue Proxy::Salt::NotFound => e
+          log_halt 404, e.to_s
+        rescue Exception => e
+          log_halt 406, "Failed to remove autosign key #{params[:key]}: #{e}"
+        end
+      end
 
       post '/autosign/:host' do
         content_type :json
         begin
-          Proxy::Salt.autosign_create(params[:host]).to_json
+          Proxy::Salt.autosign_create_hostname(params[:host]).to_json
         rescue Exception => e
           log_halt 406, "Failed to create autosign for #{params[:host]}: #{e}"
         end
@@ -23,7 +43,7 @@ module Proxy
       delete '/autosign/:host' do
         content_type :json
         begin
-          Proxy::Salt.autosign_remove(params[:host]).to_json
+          Proxy::Salt.autosign_remove_hostname(params[:host]).to_json
         rescue Proxy::Salt::NotFound => e
           log_halt 404, e.to_s
         rescue Exception => e

data/lib/smart_proxy_salt/salt_runner.rb

@@ -0,0 +1,55 @@
+require 'smart_proxy_dynflow/runner/base'
+require 'smart_proxy_dynflow/runner/command_runner'
+
+module Proxy
+  module Salt
+    # Implements the SaltRunner to be used by foreman_remote_execution
+    class SaltRunner < Proxy::Dynflow::Runner::CommandRunner
+      DEFAULT_REFRESH_INTERVAL = 1
+
+      attr_reader :jid
+
+      def initialize(options, suspended_action)
+        super(options, :suspended_action => suspended_action)
+        @options = options
+      end
+
+      def start
+        command = generate_command
+        logger.debug("Running command '#{command.join(' ')}'")
+        initialize_command(*command)
+      end
+
+      def kill
+        publish_data('== TASK ABORTED BY USER ==', 'stdout')
+        publish_exit_status(1)
+        ::Process.kill('SIGTERM', @command_pid)
+      end
+
+      def publish_data(data, type)
+        if @jid.nil? && (match = data.match(/jid: ([0-9]+)/))
+          @jid = match[1]
+        end
+        super
+      end
+
+      def publish_exit_status(status)
+        # If there was no salt job associated with this run, mark the job as failed
+        status = 1 if @jid.nil?
+        super status
+      end
+
+      private
+
+      def generate_command
+        saltfile_path = ::Proxy::Salt::Plugin.settings[:saltfile]
+        command = %w[salt --show-jid]
+        command << "--saltfile=#{saltfile_path}" if File.file?(saltfile_path)
+        command << @options['name']
+        command << 'state.template_str'
+        command << @options['script']
+        command
+      end
+    end
+  end
+end

data/lib/smart_proxy_salt/salt_task_launcher.rb

@@ -0,0 +1,27 @@
+require 'smart_proxy_dynflow/task_launcher'
+
+module Proxy
+  module Salt
+    # Implements the TaskLauncher::Batch for Salt
+    class SaltTaskLauncher < ::Proxy::Dynflow::TaskLauncher::Batch
+      # Implements the Runner::Action for Salt
+      class SaltRunnerAction < ::Proxy::Dynflow::Action::Runner
+        def initiate_runner
+          additional_options = {
+            :step_id => run_step_id,
+            :uuid => execution_plan_id
+          }
+          ::Proxy::Salt::SaltRunner.new(
+            input.merge(additional_options),
+            suspended_action
+          )
+        end
+      end
+
+      def child_launcher(parent)
+        ::Proxy::Dynflow::TaskLauncher::Single.new(world, callback, :parent => parent,
+                                                                    :action_class_override => SaltRunnerAction)
+      end
+    end
+  end
+end

data/lib/smart_proxy_salt/version.rb

@@ -3,6 +3,6 @@
 module Proxy
   # Salt module
   module Salt
-    VERSION = '3.0.0'
+    VERSION = '4.0.0'.freeze
   end
 end

data/salt/minion_auth/README.md

@@ -0,0 +1,48 @@
+# Foreman Salt Minion Authentication
+
+Currently, there are two possibilites to authenticate a newly deployed minion automatically:
+1. Use the _/etc/salt/autosign.conf_ file which stores the hostnames of acceptable hosts.
+2. Use _Salt Autosign Grains_ for a more secure way which relies on a shared secret key.
+
+This README, handles the second option and how to configure it
+
+## Setup
+Add the content of 'master.snippet' to '/etc/salt/master' which configures the grains key file on the master and a reactor. The grains file holds the acceptable keys and will be written by the Smart Proxy when a new minion is deployed. The reactor initiates an interaction with Foreman Salt if a new minion was authenticated successfully.
+In case there is already a reactor configured, you need to adapt it using the options mentioned in 'master.snippet'. The directories given in 'master.snippet' are the default ones. In case you want your files in a different place, you have to change the paths accordingly.
+
+If '/srv/salt' is configured as 'file_roots' in your '/etc/salt/master' config, setup the necessary Salt runners:
+
+```
+/srv/salt/_runners/foreman_file.py
+/srv/salt/_runners/foreman_https.py
+```
+
+Check if the reactor ('foreman_minion_auth.sls') is at the appropriated location and set the correct smart proxy address in the reactor file:
+
+```
+/var/lib/foreman-proxy/salt/reactors/foreman_minion_auth.sls
+```
+
+After changing the Salt master, restart the salt-master service:
+
+```
+systemctl restart salt-master
+```
+
+After checking the reactor and runners, run the following command to make them available in the Salt environment:
+
+```
+salt-run saltutil.sync_all
+```
+
+## Procedure
+
+1. A new host, configured as Salt minion, is deployed with Foreman Salt.
+2. Foreman Salt generates a unique key for that minion and distributes it via the Provisioning Template to the host and via an API call to the Smart Proxy.
+3. The Smart Proxy makes the key available for the Salt Autosign Grains procedure by adding it to the previously defined file (by default: /var/lib/foreman-proxy/salt/grains/autosign_key).
+4. The Salt minion is started and uses the configured Salt autosign grain for authentication to the Salt master.
+5. The Salt master accepts the minion depending on the key and the corresponding auth reactor is triggered on the Salt master.
+6. The Salt master initiates an API call to Foreman Salt which marks the corresponding host status as	_authenticated_.
+7. Foreman Salt triggers an API call to Smart Proxy Salt which deletes the key from the acceptable keys list of the Salt master (since the minion was authenticated already and shall not be reused).
+
+The minion was authenticated successfully.

data/salt/minion_auth/foreman_minion_auth.sls

@@ -0,0 +1,24 @@
+{% if 'act' in data and 'id' in data %}
+{% if data['act'] == 'accept' %}
+{% if salt['saltutil.runner']('foreman_file.check_key', (data['id'], 100)) == True %}
+{%- do salt.log.info('Minion authenticated successfully, starting HTTPS request to delete autosign key.') -%}
+remove_autosign_key_custom_runner:
+  runner.foreman_https.query_cert:
+    - method: PUT
+    - host: example.proxy.com # set smart proxy address
+    - path: /salt/api/v2/salt_autosign_auth?name={{ data['id'] }}
+    - cert: /etc/pki/katello/puppet/puppet_client.crt # default cert location
+    - key: /etc/pki/katello/puppet/puppet_client.key # default key location
+    - port: 443
+# Uncomment the following lines in case you want to use username + password authentication (not recommended)
+# call_foreman_salt_custom_runner:
+#   runner.foreman_https.query_user:
+#     - method: PUT
+#     - host: example.proxy.com # set smart proxy address
+#     - path: /salt/api/v2/salt_autosign_auth?name={{ data['id']  }}
+#     - username: my_username # set username
+#     - password: my_password # set password
+#     - port: 443
+{% endif %}
+{% endif %}
+{% endif %}

data/salt/minion_auth/master.snippet

@@ -0,0 +1,5 @@
+autosign_grains_dir: /var/lib/foreman-proxy/salt/grains
+
+reactor:
+  - 'salt/auth':
+    - /var/lib/foreman-proxy/salt/reactors/foreman_minion_auth.sls

data/salt/minion_auth/srv/salt/_runners/foreman_file.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+
+import os
+import time
+
+SALT_KEY_PATH = "/etc/salt/pki/master/minions/"
+
+def time_secs(path):
+    stat = os.stat(path)
+    return stat.st_mtime
+
+
+def younger_than_secs(path, seconds):
+
+    now_time = time.time()
+    file_time = time_secs(path)
+    if now_time - file_time <= seconds:
+        return True
+    return False
+
+def check_key(hostname, seconds):
+    return younger_than_secs(SALT_KEY_PATH + hostname, seconds)

data/salt/minion_auth/srv/salt/_runners/foreman_https.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+
+from http.client import HTTPSConnection
+import ssl
+import base64
+import json
+
+
+def query_cert(host, path, port, method, cert, key,
+               payload=None, timeout=10):
+
+    headers = {"Accept": "application/json"}
+
+    if payload is not None or method.lower() in ['put', 'post']:
+        headers["Content-Type"] = "application/json"
+
+    ctx = ssl.create_default_context()
+    ctx.load_cert_chain(certfile=cert, keyfile=key)
+
+    connection = HTTPSConnection(host,
+                                 port=port,
+                                 context=ctx,
+                                 timeout=timeout)
+    if payload is None:
+        connection.request(method,
+                           path,
+                           headers=headers)
+    else:
+        payload_json = json.dumps(payload)
+        connection.request(method=method,
+                           url=path,
+                           body=payload_json,
+                           headers=headers)
+    response = connection.getresponse()
+    response_str = response.read().decode('utf-8')
+    print(response_str)
+
+
+def query_user(host, path, port, method, username, password,
+               payload=None, timeout=10):
+
+    auth = "{}:{}".format(username, password)
+    token = base64.b64encode(auth.encode('utf-8')).decode('ascii')
+    headers = {"Authorization": "Basic {}".format(token),
+               "Accept": "application/json"}
+    if payload is not None or method.lower() in ["put", "post"]:
+        headers["Content-Type"] = "application/json"
+
+    ctx = ssl._create_unverified_context()
+    connection = HTTPSConnection(host,
+                                 port=port,
+                                 context=ctx,
+                                 timeout=timeout)
+    if payload is None:
+        connection.request(method=method,
+                           url=path,
+                           headers=headers)
+    else:
+        payload_json = json.dumps(payload)
+        connection.request(method=method,
+                           url=path,
+                           body=payload_json,
+                           headers=headers)
+    response = connection.getresponse()
+    response_str = response.read().decode('utf-8')
+    print(response_str)

data/salt/report_upload/README.md

@@ -0,0 +1,31 @@
+# Foreman Salt Report Upload
+
+Currently, there are two possibilites to upload the salt report to Foreman:
+1. Use /usr/sbin/upload-salt-reports which is called by a cron job every 10 minutes by default
+2. Upload the report immediately by using a Salt Reactor.
+
+This README, handles the second option and how to configure it
+
+## Setup
+Add the content of 'master.snippet' to '/etc/salt/master' which configures a reactor. 
+In case there is already a reactor configured, you need to adapt it using the options mentioned in 'master.snippet'.
+
+In case '/srv/salt' is configured as 'file_roots' in your '/etc/salt/master' config, setup the necessary salt state file and Salt runner functions:
+
+```
+/srv/salt/foreman_report_upload.sls
+/srv/salt/_runners/foreman_report_upload.py
+```
+
+After changing the salt master run:
+
+```
+systemctl restart salt-master
+```
+
+After adding the foreman_report_upload.sls and foreman_report_upload.py, run the following:
+
+```
+salt-run saltutil.sync_all
+```
+

data/salt/report_upload/master.snippet

@@ -0,0 +1,3 @@
+reactor:
+  - 'salt/job/*/ret/*':
+    - /srv/salt/foreman_report_upload.sls

data/salt/report_upload/srv/salt/_runners/foreman_report_upload.py

@@ -0,0 +1,106 @@
+# -*- coding: utf-8 -*-
+'''
+Uploads reports from the Salt job cache to Foreman
+'''
+from __future__ import absolute_import, print_function, unicode_literals
+
+FOREMAN_CONFIG = '/etc/salt/foreman.yaml'
+
+try:
+    from http.client import HTTPConnection, HTTPSConnection
+except ImportError:
+    from httplib import HTTPSConnection, HTTPSConnection
+
+import ssl
+import json
+import yaml
+import os
+import sys
+import base64
+
+# Import python libs
+import logging
+
+log = logging.getLogger(__name__)
+
+
+def salt_config():
+    with open(FOREMAN_CONFIG, 'r') as f:
+        config = yaml.load(f.read())
+    return config
+
+
+def upload(report):
+    config = salt_config()
+    headers = {'Accept': 'application/json',
+               'Content-Type': 'application/json'}
+
+    if config[':proto'] == 'https':
+        ctx = ssl.create_default_context()
+        ctx.load_cert_chain(certfile=config[':ssl_cert'], keyfile=config[':ssl_key'])
+        if config[':ssl_ca']:
+          ctx.load_verify_locations(cafile=config[':ssl_ca'])
+        connection = HTTPSConnection(config[':host'],
+                port=config[':port'], context=ctx)
+    else:
+        connection = HTTPConnection(config[':host'],
+                port=config[':port'])
+        if ':username' in config and ':password' in config:
+            token = base64.b64encode('{}:{}'.format(config[':username'],
+                                                    config[':password']))
+            headers['Authorization'] = 'Basic {}'.format(token)
+
+    connection.request('POST', '/salt/api/v2/jobs/upload',
+            json.dumps(report), headers)
+    response = connection.getresponse()
+
+    if response.status == 200:
+        info_msg = 'Success {0}: {1}'.format(report['job']['job_id'], response.read())
+        log.info(info_msg)
+    else:
+        log.error("Unable to upload job - aborting report upload")
+        log.error(response.read())
+
+
+def create_report(json_str):
+    msg = json.loads(json_str)
+
+    if msg['fun'] == 'state.highstate':
+        return {'job':
+             {
+               'result': {
+                 msg['id']: msg['return'],
+               },
+               'function': 'state.highstate',
+               'job_id': msg['jid']
+             }
+           }
+    elif msg['fun'] == 'state.template_str':
+        for key, entry in msg['return'].items():
+            if key.startswith('module_') and entry['__id__'] == 'state.highstate':
+                return {'job':
+                     {
+                       'result': {
+                         msg['id']: entry['changes']['ret'],
+                       },
+                       'function': 'state.highstate',
+                       'job_id': msg['jid']
+                     }
+                   }
+    raise Exception('No state.highstate found')
+
+
+def now(highstate):
+    '''
+    Upload a highstate to Foreman
+    highstate :
+        dictionary containing a highstate (generated by a Salt reactor)
+    '''
+    log.debug('Upload highstate to Foreman')
+
+    try:
+        report = create_report(base64.b64decode(highstate))
+        upload(report)
+    except Exception as exc:
+        log.error('Exception encountered: %s', exc)
+

data/salt/report_upload/srv/salt/foreman_report_upload.sls

@@ -0,0 +1,10 @@
+{% if 'cmd' in data and data['cmd'] == '_return' and 'fun' in data and (
+    data['fun'] == 'state.highstate' or (data['fun'] == 'state.template_str' and
+                                         'fun_args' in data and
+                                         data['fun_args'][0].startswith('state.highstate:')
+)) %}
+ foreman_report_upload:
+  runner.foreman_report_upload.now:
+    - args:
+      - highstate: '{{ data|json|base64_encode }}'
+{% endif %}

data/sbin/upload-salt-reports

@@ -1,38 +1,55 @@
-#!/usr/bin/env python
+#!/usr/bin/salt_python_wrapper
 # Uploads reports from the Salt job cache to Foreman
 
+from __future__ import print_function
+
 LAST_UPLOADED = '/etc/salt/last_uploaded'
 FOREMAN_CONFIG = '/etc/salt/foreman.yaml'
 LOCK_FILE = '/var/lock/salt-report-upload.lock'
 
-import httplib
+try:
+    from http.client import HTTPConnection, HTTPSConnection
+except ImportError:
+    from httplib import HTTPSConnection, HTTPSConnection
 import ssl
 import json
 import yaml
+import io
 import os
 import sys
 import base64
-
 import traceback
-
 import salt.config
 import salt.runner
 
+if sys.version_info.major == 3:
+    unicode = str
+
 
 def salt_config():
-    with open(FOREMAN_CONFIG, 'r') as f:
+    with io.open(FOREMAN_CONFIG, 'r') as f:
         config = yaml.load(f.read())
     return config
 
 
 def get_job(job_id):
     result = run('jobs.lookup_jid', [job_id])
-
     # If any minion's results are strings, they're exceptions
     # and should be wrapped in a list like other errors
-    for minion, value in result.iteritems():
-        if type(value) == str:
-            result[minion] = [value]
+
+    for minion, value in result.items():
+        try:
+            if isinstance(value,str):
+                result[minion] = [value]
+            elif isinstance(value,list):
+                result[minion] = value
+            else:
+                for key, entry in value.items():
+                    if key.startswith('module_') and '__id__' in entry and entry['__id__'] == 'state.highstate':
+                        result[minion] = entry['changes']['ret']
+                        break
+        except KeyError:
+            traceback.print_exc()
 
     return {'job':
              {
@@ -47,7 +64,7 @@ def read_last_uploaded():
     if not os.path.isfile(LAST_UPLOADED):
         return 0
     else:
-        with open(LAST_UPLOADED, 'r') as f:
+        with io.open(LAST_UPLOADED, 'r') as f:
             result = f.read().strip()
         if len(result) == 20:
             try:
@@ -59,8 +76,8 @@ def read_last_uploaded():
 
 
 def write_last_uploaded(last_uploaded):
-    with open(LAST_UPLOADED, 'w+') as f:
-        f.write(last_uploaded)
+    with io.open(LAST_UPLOADED, 'w+') as f:
+        f.write(unicode(last_uploaded))
 
 
 def run(*args, **kwargs):
@@ -68,7 +85,7 @@ def run(*args, **kwargs):
             os.environ.get('SALT_MASTER_CONFIG', '/etc/salt/master'))
 
     runner = salt.runner.Runner(__opts__)
-    with open(os.devnull, 'wb') as f:
+    with io.open(os.devnull, 'w') as f:
         stdout_bak, sys.stdout = sys.stdout, f
         try:
             ret = runner.cmd(*args, **kwargs)
@@ -79,12 +96,11 @@ def run(*args, **kwargs):
 
 def jobs_to_upload():
     jobs = run('jobs.list_jobs', kwarg={
-        "search_function": "state.highstate",
+        "search_function": ["state.highstate","state.template_str"],
     })
     last_uploaded = read_last_uploaded()
 
-    job_ids = [jid for (jid, value) in jobs.iteritems()
-               if int(jid) > last_uploaded]
+    job_ids = [jid for jid in jobs.keys() if int(jid) > last_uploaded]
 
     for job_id in sorted(job_ids):
         yield job_id, get_job(job_id)
@@ -99,15 +115,17 @@ def upload(jobs):
         ctx = ssl.create_default_context()
         ctx.load_cert_chain(certfile=config[':ssl_cert'], keyfile=config[':ssl_key'])
         if config[':ssl_ca']:
-          ctx.load_verify_locations(cafile=config[':ssl_ca'])
-        connection = httplib.HTTPSConnection(config[':host'],
-                port=config[':port'], context=ctx)
+            ctx.load_verify_locations(cafile=config[':ssl_ca'])
+        connection = HTTPSConnection(config[':host'],
+                                     port=config[':port'], context=ctx)
     else:
-        connection = httplib.HTTPConnection(config[':host'],
-                port=config[':port'])
+        connection = HTTPConnection(config[':host'],
+                                    port=config[':port'])
         if ':username' in config and ':password' in config:
-            token = base64.b64encode('{}:{}'.format(config[':username'],
-                                                    config[':password']))
+            auth = '{}:{}'.format(config[':username'], config[':password'])
+            if not isinstance(auth, bytes):
+                auth = auth.encode('UTF-8')
+            token = base64.b64encode(auth)
             headers['Authorization'] = 'Basic {}'.format(token)
 
     for job_id, job in jobs:
@@ -121,23 +139,24 @@ def upload(jobs):
 
         if response.status == 200:
             write_last_uploaded(job_id)
-            print "Success %s: %s" % (job_id, response.read())
+            print("Success %s: %s" % (job_id, response.read()))
         else:
-            print "Unable to upload job - aborting report upload"
-            print response.read()
+            print("Unable to upload job - aborting report upload")
+            print(response.read())
 
 
 def get_lock():
     if os.path.isfile(LOCK_FILE):
         raise Exception("Unable to obtain lock.")
     else:
-        open(LOCK_FILE, 'w+').close()
+        io.open(LOCK_FILE, 'w+').close()
 
 
 def release_lock():
     if os.path.isfile(LOCK_FILE):
         os.remove(LOCK_FILE)
 
+
 if __name__ == '__main__':
     try:
         get_lock()

data/settings.d/salt.yml.example

@@ -1,6 +1,7 @@
 ---
 :enabled: true
 :autosign_file: /etc/salt/autosign.conf
+:autosign_key_file: /var/lib/foreman-proxy/salt/grains/autosign_key
 :salt_command_user: root
 # Some features require using the Salt API - such as listing
 # environments and retrieving state info

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: smart_proxy_salt
 version: !ruby/object:Gem::Version
-  version: 3.0.0
+  version: 4.0.0
 platform: ruby
 authors:
 - Michael Moll
@@ -9,64 +9,8 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-08-02 00:00:00.000000000 Z
+date: 2021-08-02 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: json
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rack
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '1.1'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '1.1'
-- !ruby/object:Gem::Dependency
-  name: sinatra
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: logging
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
@@ -115,28 +59,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10'
+        version: '13'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10'
+        version: '13'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.32.1
+        version: 0.50.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.32.1
+        version: 0.50.0
 - !ruby/object:Gem::Dependency
   name: rack-test
   requirement: !ruby/object:Gem::Requirement
@@ -151,10 +95,25 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: smart_proxy_dynflow
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.5.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.5.0
 description: SaltStack Plug-In for Foreman's Smart Proxy
 email: foreman-dev@googlegroups.com
 executables:
 - foreman-node
+- salt_python_wrapper
 extensions: []
 extra_rdoc_files:
 - README.md
@@ -163,6 +122,7 @@ files:
 - LICENSE
 - README.md
 - bin/foreman-node
+- bin/salt_python_wrapper
 - bundler.d/salt.rb
 - cron/smart_proxy_salt
 - etc/foreman.yaml.example
@@ -173,11 +133,18 @@ files:
 - lib/smart_proxy_salt/salt.rb
 - lib/smart_proxy_salt/salt_api.rb
 - lib/smart_proxy_salt/salt_http_config.ru
+- lib/smart_proxy_salt/salt_runner.rb
+- lib/smart_proxy_salt/salt_task_launcher.rb
 - lib/smart_proxy_salt/version.rb
-- lib/smart_proxy_salt_core.rb
-- lib/smart_proxy_salt_core/salt_runner.rb
-- lib/smart_proxy_salt_core/salt_task_launcher.rb
-- lib/smart_proxy_salt_core/version.rb
+- salt/minion_auth/README.md
+- salt/minion_auth/foreman_minion_auth.sls
+- salt/minion_auth/master.snippet
+- salt/minion_auth/srv/salt/_runners/foreman_file.py
+- salt/minion_auth/srv/salt/_runners/foreman_https.py
+- salt/report_upload/README.md
+- salt/report_upload/master.snippet
+- salt/report_upload/srv/salt/_runners/foreman_report_upload.py
+- salt/report_upload/srv/salt/foreman_report_upload.sls
 - sbin/upload-salt-reports
 - settings.d/salt.saltfile.example
 - settings.d/salt.yml.example

data/lib/smart_proxy_salt_core.rb

@@ -1,17 +0,0 @@
-require 'foreman_tasks_core'
-require 'foreman_remote_execution_core'
-
-module SmartProxySaltCore
-  extend ForemanTasksCore::SettingsLoader
-  register_settings(:salt,
-                    :saltfile => '/etc/foreman-proxy/settings.d/salt.saltfile')
-
-  if ForemanTasksCore.dynflow_present?
-    require 'smart_proxy_salt_core/salt_runner'
-    require 'smart_proxy_salt_core/salt_task_launcher'
-
-    if defined?(SmartProxyDynflowCore)
-      SmartProxyDynflowCore::TaskLauncherRegistry.register('salt', SaltTaskLauncher)
-    end
-  end
-end

data/lib/smart_proxy_salt_core/salt_runner.rb

@@ -1,51 +0,0 @@
-require 'foreman_tasks_core/runner/command_runner'
-
-module SmartProxySaltCore
-  class SaltRunner < ForemanTasksCore::Runner::CommandRunner
-    DEFAULT_REFRESH_INTERVAL = 1
-
-    attr_reader :jid
-
-    def initialize(options, suspended_action:)
-      super(options, :suspended_action => suspended_action)
-      @options = options
-    end
-
-    def start
-      command = generate_command
-      logger.debug("Running command '#{command.join(' ')}'")
-      initialize_command(*command)
-    end
-
-    def kill
-      publish_data('== TASK ABORTED BY USER ==', 'stdout')
-      publish_exit_status(1)
-      ::Process.kill('SIGTERM', @command_pid)
-    end
-
-    def publish_data(data, type)
-      if @jid.nil? && (match = data.match(/jid: ([0-9]+)/))
-        @jid = match[1]
-      end
-      super
-    end
-
-    def publish_exit_status(status)
-      # If there was no salt job associated with this run, mark the job as failed
-      status = 1 if @jid.nil?
-      super status
-    end
-
-    private
-
-    def generate_command
-      saltfile_path = SmartProxySaltCore.settings[:saltfile]
-      command = %w(salt --show-jid)
-      command << "--saltfile=#{saltfile_path}" if File.file?(saltfile_path)
-      command << @options['name']
-      command << 'state.template_str'
-      command << @options['script']
-      command
-    end
-  end
-end

data/lib/smart_proxy_salt_core/salt_task_launcher.rb

@@ -1,21 +0,0 @@
-module SmartProxySaltCore
-  class SaltTaskLauncher < ForemanTasksCore::TaskLauncher::Batch
-    class SaltRunnerAction < ForemanTasksCore::Runner::Action
-      def initiate_runner
-        additional_options = {
-          :step_id => run_step_id,
-          :uuid => execution_plan_id
-        }
-        ::SmartProxySaltCore::SaltRunner.new(
-          input.merge(additional_options),
-          :suspended_action => suspended_action
-        )
-      end
-    end
-
-    def child_launcher(parent)
-      ForemanTasksCore::TaskLauncher::Single.new(world, callback, :parent => parent,
-        :action_class_override => SaltRunnerAction)
-    end
-  end
-end

data/lib/smart_proxy_salt_core/version.rb

@@ -1,3 +0,0 @@
-module SmartProxySaltCore
-  VERSION = '0.0.1'.freeze
-end