smart_proxy_salt 3.1.0 → 5.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f3ac4ad731748d62d1ec922520aa60ce1a7db4b001b2aeaabfe9c2e0709e3f9e
-  data.tar.gz: b345c0ae059918e924a6ec771664a38e687d3b8a288559cd48274e271f69c551
+  metadata.gz: b25650758e39a88cb8683f8ebe295d343c0752d0fff229fbe21f86ee585848c6
+  data.tar.gz: 9f26eb7258a77a426c1addf5e8ba949c432332f1b4b35897e193762cf1a60d3f
 SHA512:
-  metadata.gz: 20b39c0f2ef2a6ea13cf94d996141ee1e28ff542fd8def32275560085b8f6c31e57d0d834e6179744c98a4eccfcb3efa4dc8ce74e178b0751103933253d1c465
-  data.tar.gz: d1dd34e0c168829fe049c8db0bb751782b3548a39566c530d148a00485fcf0c96c2fd52b96a7bb71b7a0ac3b9e5ea74629ab39e7b4ef166b1f0ba1163aa6fbd7
+  metadata.gz: bd79c67ac4733c9b1f19a7e5f6a62a6019e39209d37abd21d7a67f48aea4e24da875bbdaecd3c8db574e566cb451038904fd9e21d045db61896df0a96ccab51e
+  data.tar.gz: 9c8640cc30f494fe2406b0a840643aacc2211c738fc25864c8c7e1024e4d2a6b6ad72e459eac947a2af9ac2b86ee14a87ba0cc84e27018742906d8a8ad50e18b

data/bin/foreman-node

@@ -14,6 +14,7 @@ require 'net/http'
 require 'net/https'
 require 'etc'
 require 'timeout'
+require 'msgpack' if SETTINGS[:filecache]
 
 begin
   require 'json'
@@ -57,19 +58,37 @@ rescue Exception => e
   exit 1
 end
 
+def get_grains_from_filecache(minion)
+  # Use the grains from the salt master's filesystem based cache
+  # This requires the following settings in /etc/salt/foreman.yaml:
+  # :filecache: true
+  # :cachedir: "/path/to/master/cache" (default: "/var/cache/salt/master")
+  # Also, the msgpack rubygem needs to be present
+  cachedir = SETTINGS[:cachedir] || '/var/cache/salt/master'
+  content = File.read("#{cachedir}/minions/#{minion}/data.p")
+  data = MessagePack.unpack(content)
+  data['grains']
+end
+
+def get_grains_from_saltrun(minion)
+  result = IO.popen(['salt-run', '-l', 'quiet', '--output=json', 'cache.grains', minion], &:read)
+  data = JSON.parse(result)
+  data[minion]
+end
+
 def plain_grains(minion)
   # We have to get the grains from the cache, because the client
   # is probably running 'state.highstate' right now.
 
-  result = IO.popen(['salt-run', '--output=json', 'cache.grains', minion]) do |io|
-    io.read
-  end
-
-  grains = JSON.parse(result)
+  grains = if SETTINGS[:filecache]
+             get_grains_from_filecache(minion)
+           else
+             get_grains_from_saltrun(minion)
+           end
 
   raise 'No grains received from Salt master' unless grains
 
-  plainify(grains[minion]).flatten.inject(&:merge)
+  plainify(grains).flatten.inject(&:merge)
 end
 
 def plainify(hash, prefix = nil)

data/bin/salt_python_wrapper

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+set -u
+
+for py in 'python3' 'python'; do
+  exe=$(type -p ${py})
+  if [ -n "${exe}" ]; then
+    if ${exe} -c 'import salt.config'; then
+      ${exe} "$@"
+      exit $?
+    fi
+  fi
+done
+
+echo "No usable python version found, check if python or python3 can import salt.config!" 1>&2
+exit 1

data/etc/foreman.conf.example

@@ -0,0 +1,65 @@
+# /etc/salt/master.d/foreman.config Example configuration
+#
+# This file summarizes configurations for the salt-master. Modify directories and
+# parameters to fit your setup. When you're done, remove the .example from the
+# filename so the salt-master will make use of it.
+# Have a look at the [Foreman Salt Plugin Documentation](https://theforeman.org/plugins/foreman_salt/) for detailed explanations.
+#
+# After editing this file, run the following command to active the changes:
+# $ systemctl restart salt-master
+
+
+##
+# Autosign
+autosign_grains_dir: /var/lib/foreman-proxy/salt/grains
+autosign_file: /etc/salt/autosign.conf
+# Uncomment the next line to make use of the autosign host name file (not recommended)
+# permissive_pki_access: True
+
+
+##
+# Node classifier
+master_tops:
+  ext_nodes: /usr/bin/foreman-node
+
+
+##
+# Pillar data access
+ext_pillar:
+  - puppet: /usr/bin/foreman-node
+
+
+##
+# Salt API access
+external_auth:
+  pam:
+    saltuser: # Username of your salt user
+      - '@runner'
+
+rest_cherrypy:
+  port: 9191
+  ssl_key: /etc/puppet/example.key # Add the path to your Puppet ssl key here
+  ssl_crt: /etc/puppet/example.crt # Add the path to your Puppet ssl certificate here
+
+
+##
+# Remote execution provider
+publisher_acl:
+  foreman-proxy:
+    - state.template_str
+
+
+##
+# Salt environment (optional)
+file_roots:
+  base:
+   - /srv/salt
+
+ 
+##
+# Reactors
+reactor:
+  - 'salt/auth': # Autosign reactor
+    - /var/lib/foreman-proxy/salt/reactors/foreman_minion_auth.sls
+  - 'salt/job/*/ret/*': # Report reactor
+    - /var/lib/foreman-proxy/salt/reactors/foreman_report_upload.sls

data/lib/smart_proxy_salt/cli.rb

@@ -14,44 +14,43 @@ module Proxy
           Proxy::Salt::Plugin.settings.autosign_file
         end
 
-        def autosign_create(host)
-          FileUtils.touch(autosign_file) unless File.exist?(autosign_file)
-
-          autosign = open(autosign_file, File::RDWR)
+        def autosign_key_file
+          Proxy::Salt::Plugin.settings.autosign_key_file
+        end
 
-          found = false
-          autosign.each_line { |line| found = true if line.chomp == host }
-          autosign.puts host unless found
-          autosign.close
+        def autosign_create_hostname(hostname)
+          if append_value_to_file(autosign_file, hostname)
+            { message: 'Added hostname successfully.' }
+          else
+            { message: 'Failed to add hostname.' \
+                       ' See smart proxy error log for more information.' }
+          end
+        end
 
-          result = { :message => "Added #{host} to autosign" }
-          logger.info result[:message]
-          result
+        def autosign_remove_hostname(hostname)
+          if remove_value_from_file(autosign_file, hostname)
+            { message: 'Removed hostname successfully.' }
+          else
+            { message: 'Failed to remove hostname.' \
+                       ' See smart proxy error log for more information.' }
+          end
         end
 
-        def autosign_remove(host)
-          raise "No such file #{autosign_file}" unless File.exist?(autosign_file)
+        def autosign_create_key(key)
+          if append_value_to_file(autosign_key_file, key)
+            { message: 'Added key successfully.' }
+          else
+            { message: 'Failed to add key.' \
+                       ' See smart proxy error log for more information.' }
+          end
+        end
 
-          found = false
-          entries = open(autosign_file, File::RDONLY).readlines.collect do |l|
-            if l.chomp != host
-              l
-            else
-              found = true
-              nil
-            end
-          end.uniq.compact
-          if found
-            autosign = open(autosign_file, File::TRUNC | File::RDWR)
-            autosign.write entries.join("\n")
-            autosign.write "\n"
-            autosign.close
-            result = { :message => "Removed #{host} from autosign" }
-            logger.info result[:message]
-            result
+        def autosign_remove_key(key)
+          if remove_value_from_file(autosign_key_file, key)
+            { message: 'Removed key successfully.' }
           else
-            logger.info "Attempt to remove nonexistant client autosign for #{host}"
-            raise Proxy::Salt::NotFound.new("Attempt to remove nonexistant client autosign for #{host}")
+            { message: 'Failed to remove key.' \
+                       ' See smart proxy error log for more information.' }
           end
         end
 
@@ -62,6 +61,45 @@ module Proxy
           end.map(&:chomp)
         end
 
+        def append_value_to_file(filepath, value)
+          File.open(filepath, File::CREAT|File::RDWR) do |file|
+            unless file.any? { |line| line.chomp == value}
+              file.puts value
+            end
+          end
+          logger.info "Added an entry to '#{filepath}' successfully."
+          true
+        rescue IOError => e
+          logger.info "Attempted to add an entry to '#{filepath}', but an exception occurred: #{e}"
+          false
+        end
+
+        def remove_value_from_file(filepath, value)
+
+          return true unless File.exist?(filepath)
+
+          found = false
+          entries = File.readlines(filepath).collect do |line|
+            entry = line.chomp
+            if entry == value
+              found = true
+              nil
+            elsif entry == ""
+              nil
+            else
+              line
+            end
+          end.uniq.compact
+          if found
+            File.write(filepath, entries.join())
+            logger.info "Removed an entry from '#{filepath}' successfully."
+          end
+          true
+        rescue IOError => e
+          logger.info "Attempted to remove an entry from '#{filepath}', but an exception occurred: #{e}"
+          false
+        end
+
         def highstate(host)
           find_salt_binaries
           cmd = [@sudo, '-u', Proxy::Salt::Plugin.settings.salt_command_user, @salt, '--async', escape_for_shell(host), 'state.highstate']

data/lib/smart_proxy_salt/salt.rb

@@ -13,11 +13,24 @@ module Proxy
       plugin 'salt', Proxy::Salt::VERSION
 
       default_settings :autosign_file      => '/etc/salt/autosign.conf',
+                       :autosign_key_file  => '/var/lib/foreman-proxy/salt/grains/autosign_key',
                        :salt_command_user  => 'root',
-                       :use_api            => false
+                       :use_api            => false,
+                       :saltfile           => '/etc/foreman-proxy/settings.d/salt.saltfile'
 
-      http_rackup_path File.expand_path('salt_http_config.ru', File.expand_path('../', __FILE__))
-      https_rackup_path File.expand_path('salt_http_config.ru', File.expand_path('../', __FILE__))
+      requires :dynflow, '>= 0.5.0'
+
+      rackup_path File.expand_path('salt_http_config.ru', __dir__)
+
+      load_classes do
+        require 'smart_proxy_dynflow'
+        require 'smart_proxy_salt/salt_runner'
+        require 'smart_proxy_salt/salt_task_launcher'
+      end
+
+      load_dependency_injection_wirings do |_container_instance, _settings|
+        Proxy::Dynflow::TaskLauncherRegistry.register('salt', SaltTaskLauncher)
+      end
     end
 
     class << self
@@ -33,6 +46,10 @@ module Proxy
           super
         end
       end
+
+      def respond_to_missing?(method, include_private = false)
+        Proxy::Salt::Rest.respond_to?(method) || Proxy::Salt::CLI.respond_to?(method) || super
+      end
     end
   end
 end

data/lib/smart_proxy_salt/salt_api.rb

@@ -9,12 +9,32 @@ module Proxy
     class Api < ::Sinatra::Base
       include ::Proxy::Log
       helpers ::Proxy::Helpers
-      authorize_with_ssl_client
+      authorize_with_trusted_hosts
+
+      post '/autosign_key/:key' do
+        content_type :json
+        begin
+          Proxy::Salt.autosign_create_key(params[:key]).to_json
+        rescue Exception => e
+          log_halt 406, "Failed to create autosign key #{params[:key]}: #{e}"
+        end
+      end
+
+      delete '/autosign_key/:key' do
+        content_type :json
+        begin
+          Proxy::Salt.autosign_remove_key(params[:key]).to_json
+        rescue Proxy::Salt::NotFound => e
+          log_halt 404, e.to_s
+        rescue Exception => e
+          log_halt 406, "Failed to remove autosign key #{params[:key]}: #{e}"
+        end
+      end
 
       post '/autosign/:host' do
         content_type :json
         begin
-          Proxy::Salt.autosign_create(params[:host]).to_json
+          Proxy::Salt.autosign_create_hostname(params[:host]).to_json
         rescue Exception => e
           log_halt 406, "Failed to create autosign for #{params[:host]}: #{e}"
         end
@@ -23,7 +43,7 @@ module Proxy
       delete '/autosign/:host' do
         content_type :json
         begin
-          Proxy::Salt.autosign_remove(params[:host]).to_json
+          Proxy::Salt.autosign_remove_hostname(params[:host]).to_json
         rescue Proxy::Salt::NotFound => e
           log_halt 404, e.to_s
         rescue Exception => e

data/lib/smart_proxy_salt/salt_runner.rb

@@ -0,0 +1,55 @@
+require 'smart_proxy_dynflow/runner/base'
+require 'smart_proxy_dynflow/runner/command_runner'
+
+module Proxy
+  module Salt
+    # Implements the SaltRunner to be used by foreman_remote_execution
+    class SaltRunner < Proxy::Dynflow::Runner::CommandRunner
+      DEFAULT_REFRESH_INTERVAL = 1
+
+      attr_reader :jid
+
+      def initialize(options, suspended_action)
+        super(options, :suspended_action => suspended_action)
+        @options = options
+      end
+
+      def start
+        command = generate_command
+        logger.debug("Running command '#{command.join(' ')}'")
+        initialize_command(*command)
+      end
+
+      def kill
+        publish_data('== TASK ABORTED BY USER ==', 'stdout')
+        publish_exit_status(1)
+        ::Process.kill('SIGTERM', @command_pid)
+      end
+
+      def publish_data(data, type)
+        if @jid.nil? && (match = data.match(/jid: ([0-9]+)/))
+          @jid = match[1]
+        end
+        super
+      end
+
+      def publish_exit_status(status)
+        # If there was no salt job associated with this run, mark the job as failed
+        status = 1 if @jid.nil?
+        super status
+      end
+
+      private
+
+      def generate_command
+        saltfile_path = ::Proxy::Salt::Plugin.settings[:saltfile]
+        command = %w[salt --show-jid]
+        command << "--saltfile=#{saltfile_path}" if File.file?(saltfile_path)
+        command << @options['name']
+        command << 'state.template_str'
+        command << @options['script']
+        command
+      end
+    end
+  end
+end

data/lib/smart_proxy_salt/salt_task_launcher.rb

@@ -0,0 +1,27 @@
+require 'smart_proxy_dynflow/task_launcher'
+
+module Proxy
+  module Salt
+    # Implements the TaskLauncher::Batch for Salt
+    class SaltTaskLauncher < ::Proxy::Dynflow::TaskLauncher::Batch
+      # Implements the Runner::Action for Salt
+      class SaltRunnerAction < ::Proxy::Dynflow::Action::Runner
+        def initiate_runner
+          additional_options = {
+            :step_id => run_step_id,
+            :uuid => execution_plan_id
+          }
+          ::Proxy::Salt::SaltRunner.new(
+            input.merge(additional_options),
+            suspended_action
+          )
+        end
+      end
+
+      def child_launcher(parent)
+        ::Proxy::Dynflow::TaskLauncher::Single.new(world, callback, :parent => parent,
+                                                                    :action_class_override => SaltRunnerAction)
+      end
+    end
+  end
+end

data/lib/smart_proxy_salt/version.rb

@@ -3,6 +3,6 @@
 module Proxy
   # Salt module
   module Salt
-    VERSION = '3.1.0'
+    VERSION = '5.0.0'.freeze
   end
 end

data/salt/minion_auth/README.md

@@ -0,0 +1,48 @@
+# Foreman Salt Minion Authentication
+
+Currently, there are two possibilites to authenticate a newly deployed minion automatically:
+1. Use the _/etc/salt/autosign.conf_ file which stores the hostnames of acceptable hosts.
+2. Use _Salt Autosign Grains_ for a more secure way which relies on a shared secret key.
+
+This README, handles the second option and how to configure it
+
+## Setup
+Add the content of 'master.snippet' to '/etc/salt/master' which configures the grains key file on the master and a reactor. The grains file holds the acceptable keys and will be written by the Smart Proxy when a new minion is deployed. The reactor initiates an interaction with Foreman Salt if a new minion was authenticated successfully.
+In case there is already a reactor configured, you need to adapt it using the options mentioned in 'master.snippet'. The directories given in 'master.snippet' are the default ones. In case you want your files in a different place, you have to change the paths accordingly.
+
+If '/srv/salt' is configured as 'file_roots' in your '/etc/salt/master' config, setup the necessary Salt runners:
+
+```
+/srv/salt/_runners/foreman_file.py
+/srv/salt/_runners/foreman_https.py
+```
+
+Check if the reactor ('foreman_minion_auth.sls') is at the appropriated location:
+
+```
+/var/lib/foreman-proxy/salt/reactors/foreman_minion_auth.sls
+```
+
+Restart the salt-master service:
+
+```
+systemctl restart salt-master
+```
+
+After checking the reactor and runners, run the following command to make them available in the Salt environment:
+
+```
+salt-run saltutil.sync_all
+```
+
+## Procedure
+
+1. A new host, configured as Salt minion, is deployed with Foreman Salt.
+2. Foreman Salt generates a unique key for that minion and distributes it via the Provisioning Template to the host and via an API call to the Smart Proxy.
+3. The Smart Proxy makes the key available for the Salt Autosign Grains procedure by adding it to the previously defined file (by default: /var/lib/foreman-proxy/salt/grains/autosign_key).
+4. The Salt minion is started and uses the configured Salt autosign grain for authentication to the Salt master.
+5. The Salt master accepts the minion depending on the key and the corresponding auth reactor is triggered on the Salt master.
+6. The Salt master initiates an API call to Foreman Salt which marks the corresponding host status as	_authenticated_.
+7. Foreman Salt triggers an API call to Smart Proxy Salt which deletes the key from the acceptable keys list of the Salt master (since the minion was authenticated already and shall not be reused).
+
+The minion was authenticated successfully.

data/salt/minion_auth/foreman_minion_auth.sls

@@ -0,0 +1,10 @@
+{% if 'act' in data and 'id' in data %}
+{% if data['act'] == 'accept' %}
+{% if salt['saltutil.runner']('foreman_file.check_key', (data['id'], 100)) == True %}
+{%- do salt.log.info('Minion authenticated successfully, starting HTTPS request to delete autosign key.') -%}
+remove_autosign_key_custom_runner:
+  runner.foreman_https.remove_key:
+    - minion: {{ data['id'] }}
+{% endif %}
+{% endif %}
+{% endif %}

data/salt/minion_auth/master.snippet

@@ -0,0 +1,5 @@
+autosign_grains_dir: /var/lib/foreman-proxy/salt/grains
+
+reactor:
+  - 'salt/auth':
+    - /var/lib/foreman-proxy/salt/reactors/foreman_minion_auth.sls

data/salt/minion_auth/srv/salt/_runners/foreman_file.py

@@ -0,0 +1,26 @@
+"""
+Salt runner to check the age of a minion key file.
+"""
+
+import os
+import time
+
+SALT_KEY_PATH = "/etc/salt/pki/master/minions/"
+
+
+def time_secs(path):
+    stat = os.stat(path)
+    return stat.st_mtime
+
+
+def younger_than_secs(path, seconds):
+
+    now_time = time.time()
+    file_time = time_secs(path)
+    if now_time - file_time <= seconds:
+        return True
+    return False
+
+
+def check_key(hostname, seconds):
+    return younger_than_secs(SALT_KEY_PATH + hostname, seconds)

data/salt/minion_auth/srv/salt/_runners/foreman_https.py

@@ -0,0 +1,121 @@
+"""
+Salt runner to make generic https requests or perform directly
+an autosign key removal.
+"""
+
+
+from http.client import HTTPSConnection
+import ssl
+import base64
+import json
+import logging
+import yaml
+
+FOREMAN_CONFIG = '/etc/salt/foreman.yaml'
+log = logging.getLogger(__name__)
+
+
+def salt_config():
+    """
+    Read the foreman configuratoin from FOREMAN_CONFIG
+    """
+    with open(FOREMAN_CONFIG, 'r') as config_file:
+        config = yaml.load(config_file.read())
+    return config
+
+
+def remove_key(minion):
+    """
+    Perform an HTTPS request to the configured foreman host and trigger
+    the autosign key removal process.
+    """
+    config = salt_config()
+    host_name = config[':host']
+    port = config[':port']
+    timeout = config[':timeout']
+    method = 'PUT'
+    path = '/salt/api/v2/salt_autosign_auth?name=%s' % minion
+
+    # Differentiate between cert and user authentication
+    if config[':proto'] == 'https':
+        query_cert(host=host_name,
+                   path=path,
+                   port=port,
+                   method=method,
+                   cert=config[':ssl_cert'],
+                   key=config[':ssl_key'],
+                   timeout=timeout)
+    else:
+        query_user(host=host_name,
+                   path=path,
+                   port=port,
+                   method=method,
+                   username=config[':username'],
+                   password=config[':password'],
+                   timeout=timeout)
+
+
+def query_cert(host, path, port, method, cert, key,
+               payload=None, timeout=10):
+    """
+    Perform an HTTPS query with certificate credentials.
+    """
+
+    headers = {"Accept": "application/json"}
+
+    if payload is not None or method.lower() in ['put', 'post']:
+        headers["Content-Type"] = "application/json"
+
+    ctx = ssl.create_default_context()
+    ctx.load_cert_chain(certfile=cert, keyfile=key)
+
+    connection = HTTPSConnection(host,
+                                 port=port,
+                                 context=ctx,
+                                 timeout=timeout)
+    if payload is None:
+        connection.request(method,
+                           path,
+                           headers=headers)
+    else:
+        payload_json = json.dumps(payload)
+        connection.request(method=method,
+                           url=path,
+                           body=payload_json,
+                           headers=headers)
+    response = connection.getresponse()
+    response_str = response.read().decode('utf-8')
+    print(response_str)
+
+
+def query_user(host, path, port, method, username, password,
+               payload=None, timeout=10):
+    """
+    Perform an HTTPS query with user credentials.
+    """
+
+    auth = "{}:{}".format(username, password)
+    token = base64.b64encode(auth.encode('utf-8')).decode('ascii')
+    headers = {"Authorization": "Basic {}".format(token),
+               "Accept": "application/json"}
+    if payload is not None or method.lower() in ["put", "post"]:
+        headers["Content-Type"] = "application/json"
+
+    ctx = ssl._create_unverified_context()
+    connection = HTTPSConnection(host,
+                                 port=port,
+                                 context=ctx,
+                                 timeout=timeout)
+    if payload is None:
+        connection.request(method=method,
+                           url=path,
+                           headers=headers)
+    else:
+        payload_json = json.dumps(payload)
+        connection.request(method=method,
+                           url=path,
+                           body=payload_json,
+                           headers=headers)
+    response = connection.getresponse()
+    response_str = response.read().decode('utf-8')
+    print(response_str)

data/salt/report_upload/README.md

@@ -0,0 +1,36 @@
+# Foreman Salt Report Upload
+
+Currently, there are two possibilites to upload the salt report to Foreman:
+1. Use /usr/sbin/upload-salt-reports which is called by a cron job every 10 minutes by default
+2. Upload the report immediately by using a Salt Reactor.
+
+This README, handles the second option and how to configure it
+
+## Setup
+Add the content of 'master.snippet' to '/etc/salt/master' which configures a reactor. 
+In case there is already a reactor configured, you need to adapt it using the options mentioned in 'master.snippet'.
+
+Check the reactor file to be in the following folder (or a different one depending on your master configuration):
+
+```
+/var/lib/foreman-proxy/salt/reactors/foreman_report_upload.sls
+```
+
+In case '/srv/salt' is configured as 'file_roots' in your '/etc/salt/master' config, setup the necessary Salt runner:
+
+```
+/srv/salt/_runners/foreman_report_upload.py
+```
+
+After changing the salt master run:
+
+```
+systemctl restart salt-master
+```
+
+After adding the foreman_report_upload.sls and foreman_report_upload.py, run the following:
+
+```
+salt-run saltutil.sync_all
+```
+

data/salt/report_upload/foreman_report_upload.sls

@@ -0,0 +1,10 @@
+{% if 'cmd' in data and data['cmd'] == '_return' and 'fun' in data and (
+    data['fun'] == 'state.highstate' or (data['fun'] == 'state.template_str' and
+                                         'fun_args' in data and
+                                         data['fun_args'][0].startswith('state.highstate:')
+)) %}
+ foreman_report_upload:
+  runner.foreman_report_upload.now:
+    - args:
+      - highstate: '{{ data|json|base64_encode }}'
+{% endif %}

data/salt/report_upload/master.snippet

@@ -0,0 +1,3 @@
+reactor:
+  - 'salt/job/*/ret/*':
+    - /var/lib/foreman-proxy/salt/reactors/foreman_report_upload.sls

data/salt/report_upload/srv/salt/_runners/foreman_report_upload.py

@@ -0,0 +1,106 @@
+# -*- coding: utf-8 -*-
+'''
+Uploads reports from the Salt job cache to Foreman
+'''
+from __future__ import absolute_import, print_function, unicode_literals
+
+FOREMAN_CONFIG = '/etc/salt/foreman.yaml'
+
+try:
+    from http.client import HTTPConnection, HTTPSConnection
+except ImportError:
+    from httplib import HTTPSConnection, HTTPSConnection
+
+import ssl
+import json
+import yaml
+import os
+import sys
+import base64
+
+# Import python libs
+import logging
+
+log = logging.getLogger(__name__)
+
+
+def salt_config():
+    with open(FOREMAN_CONFIG, 'r') as f:
+        config = yaml.load(f.read())
+    return config
+
+
+def upload(report):
+    config = salt_config()
+    headers = {'Accept': 'application/json',
+               'Content-Type': 'application/json'}
+
+    if config[':proto'] == 'https':
+        ctx = ssl.create_default_context()
+        ctx.load_cert_chain(certfile=config[':ssl_cert'], keyfile=config[':ssl_key'])
+        if config[':ssl_ca']:
+          ctx.load_verify_locations(cafile=config[':ssl_ca'])
+        connection = HTTPSConnection(config[':host'],
+                port=config[':port'], context=ctx)
+    else:
+        connection = HTTPConnection(config[':host'],
+                port=config[':port'])
+        if ':username' in config and ':password' in config:
+            token = base64.b64encode('{}:{}'.format(config[':username'],
+                                                    config[':password']))
+            headers['Authorization'] = 'Basic {}'.format(token)
+
+    connection.request('POST', '/salt/api/v2/jobs/upload',
+            json.dumps(report), headers)
+    response = connection.getresponse()
+
+    if response.status == 200:
+        info_msg = 'Success {0}: {1}'.format(report['job']['job_id'], response.read())
+        log.info(info_msg)
+    else:
+        log.error("Unable to upload job - aborting report upload")
+        log.error(response.read())
+
+
+def create_report(json_str):
+    msg = json.loads(json_str)
+
+    if msg['fun'] == 'state.highstate':
+        return {'job':
+             {
+               'result': {
+                 msg['id']: msg['return'],
+               },
+               'function': 'state.highstate',
+               'job_id': msg['jid']
+             }
+           }
+    elif msg['fun'] == 'state.template_str':
+        for key, entry in msg['return'].items():
+            if key.startswith('module_') and entry['__id__'] == 'state.highstate':
+                return {'job':
+                     {
+                       'result': {
+                         msg['id']: entry['changes']['ret'],
+                       },
+                       'function': 'state.highstate',
+                       'job_id': msg['jid']
+                     }
+                   }
+    raise Exception('No state.highstate found')
+
+
+def now(highstate):
+    '''
+    Upload a highstate to Foreman
+    highstate :
+        dictionary containing a highstate (generated by a Salt reactor)
+    '''
+    log.debug('Upload highstate to Foreman')
+
+    try:
+        report = create_report(base64.b64decode(highstate))
+        upload(report)
+    except Exception as exc:
+        log.error('Exception encountered: %s', exc)
+

data/sbin/upload-salt-reports

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/salt_python_wrapper
 # Uploads reports from the Salt job cache to Foreman
 
 from __future__ import print_function
@@ -18,12 +18,13 @@ import io
 import os
 import sys
 import base64
-
 import traceback
-
 import salt.config
 import salt.runner
 
+if sys.version_info.major == 3:
+    unicode = str
+
 
 def salt_config():
     with io.open(FOREMAN_CONFIG, 'r') as f:
@@ -33,12 +34,22 @@ def salt_config():
 
 def get_job(job_id):
     result = run('jobs.lookup_jid', [job_id])
-
     # If any minion's results are strings, they're exceptions
     # and should be wrapped in a list like other errors
+
     for minion, value in result.items():
-        if type(value) == str:
-            result[minion] = [value]
+        try:
+            if isinstance(value,str):
+                result[minion] = [value]
+            elif isinstance(value,list):
+                result[minion] = value
+            else:
+                for key, entry in value.items():
+                    if key.startswith('module_') and '__id__' in entry and entry['__id__'] == 'state.highstate':
+                        result[minion] = entry['changes']['ret']
+                        break
+        except KeyError:
+            traceback.print_exc()
 
     return {'job':
              {
@@ -66,7 +77,7 @@ def read_last_uploaded():
 
 def write_last_uploaded(last_uploaded):
     with io.open(LAST_UPLOADED, 'w+') as f:
-        f.write(last_uploaded)
+        f.write(unicode(last_uploaded))
 
 
 def run(*args, **kwargs):
@@ -85,12 +96,11 @@ def run(*args, **kwargs):
 
 def jobs_to_upload():
     jobs = run('jobs.list_jobs', kwarg={
-        "search_function": "state.highstate",
+        "search_function": ["state.highstate","state.template_str"],
     })
     last_uploaded = read_last_uploaded()
 
-    job_ids = [jid for (jid, value) in jobs.items()
-               if int(jid) > last_uploaded]
+    job_ids = [jid for jid in jobs.keys() if int(jid) > last_uploaded]
 
     for job_id in sorted(job_ids):
         yield job_id, get_job(job_id)
@@ -112,8 +122,10 @@ def upload(jobs):
         connection = HTTPConnection(config[':host'],
                                     port=config[':port'])
         if ':username' in config and ':password' in config:
-            token = base64.b64encode('{}:{}'.format(config[':username'],
-                                                    config[':password']))
+            auth = '{}:{}'.format(config[':username'], config[':password'])
+            if not isinstance(auth, bytes):
+                auth = auth.encode('UTF-8')
+            token = base64.b64encode(auth)
             headers['Authorization'] = 'Basic {}'.format(token)
 
     for job_id, job in jobs:

data/settings.d/salt.yml.example

@@ -1,6 +1,7 @@
 ---
 :enabled: true
 :autosign_file: /etc/salt/autosign.conf
+:autosign_key_file: /var/lib/foreman-proxy/salt/grains/autosign_key
 :salt_command_user: root
 # Some features require using the Salt API - such as listing
 # environments and retrieving state info

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: smart_proxy_salt
 version: !ruby/object:Gem::Version
-  version: 3.1.0
+  version: 5.0.0
 platform: ruby
 authors:
 - Michael Moll
@@ -9,64 +9,8 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-10-31 00:00:00.000000000 Z
+date: 2022-02-14 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: json
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rack
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '1.1'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '1.1'
-- !ruby/object:Gem::Dependency
-  name: sinatra
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: logging
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
@@ -115,28 +59,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10'
+        version: '13'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10'
+        version: '13'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.32.1
+        version: 0.50.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.32.1
+        version: 0.50.0
 - !ruby/object:Gem::Dependency
   name: rack-test
   requirement: !ruby/object:Gem::Requirement
@@ -151,10 +95,25 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: smart_proxy_dynflow
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.5.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.5.0
 description: SaltStack Plug-In for Foreman's Smart Proxy
 email: foreman-dev@googlegroups.com
 executables:
 - foreman-node
+- salt_python_wrapper
 extensions: []
 extra_rdoc_files:
 - README.md
@@ -163,8 +122,10 @@ files:
 - LICENSE
 - README.md
 - bin/foreman-node
+- bin/salt_python_wrapper
 - bundler.d/salt.rb
 - cron/smart_proxy_salt
+- etc/foreman.conf.example
 - etc/foreman.yaml.example
 - lib/smart_proxy_salt.rb
 - lib/smart_proxy_salt/api_request.rb
@@ -173,11 +134,18 @@ files:
 - lib/smart_proxy_salt/salt.rb
 - lib/smart_proxy_salt/salt_api.rb
 - lib/smart_proxy_salt/salt_http_config.ru
+- lib/smart_proxy_salt/salt_runner.rb
+- lib/smart_proxy_salt/salt_task_launcher.rb
 - lib/smart_proxy_salt/version.rb
-- lib/smart_proxy_salt_core.rb
-- lib/smart_proxy_salt_core/salt_runner.rb
-- lib/smart_proxy_salt_core/salt_task_launcher.rb
-- lib/smart_proxy_salt_core/version.rb
+- salt/minion_auth/README.md
+- salt/minion_auth/foreman_minion_auth.sls
+- salt/minion_auth/master.snippet
+- salt/minion_auth/srv/salt/_runners/foreman_file.py
+- salt/minion_auth/srv/salt/_runners/foreman_https.py
+- salt/report_upload/README.md
+- salt/report_upload/foreman_report_upload.sls
+- salt/report_upload/master.snippet
+- salt/report_upload/srv/salt/_runners/foreman_report_upload.py
 - sbin/upload-salt-reports
 - settings.d/salt.saltfile.example
 - settings.d/salt.yml.example

data/lib/smart_proxy_salt_core/salt_runner.rb

@@ -1,51 +0,0 @@
-require 'foreman_tasks_core/runner/command_runner'
-
-module SmartProxySaltCore
-  class SaltRunner < ForemanTasksCore::Runner::CommandRunner
-    DEFAULT_REFRESH_INTERVAL = 1
-
-    attr_reader :jid
-
-    def initialize(options, suspended_action:)
-      super(options, :suspended_action => suspended_action)
-      @options = options
-    end
-
-    def start
-      command = generate_command
-      logger.debug("Running command '#{command.join(' ')}'")
-      initialize_command(*command)
-    end
-
-    def kill
-      publish_data('== TASK ABORTED BY USER ==', 'stdout')
-      publish_exit_status(1)
-      ::Process.kill('SIGTERM', @command_pid)
-    end
-
-    def publish_data(data, type)
-      if @jid.nil? && (match = data.match(/jid: ([0-9]+)/))
-        @jid = match[1]
-      end
-      super
-    end
-
-    def publish_exit_status(status)
-      # If there was no salt job associated with this run, mark the job as failed
-      status = 1 if @jid.nil?
-      super status
-    end
-
-    private
-
-    def generate_command
-      saltfile_path = SmartProxySaltCore.settings[:saltfile]
-      command = %w(salt --show-jid)
-      command << "--saltfile=#{saltfile_path}" if File.file?(saltfile_path)
-      command << @options['name']
-      command << 'state.template_str'
-      command << @options['script']
-      command
-    end
-  end
-end

data/lib/smart_proxy_salt_core/salt_task_launcher.rb

@@ -1,21 +0,0 @@
-module SmartProxySaltCore
-  class SaltTaskLauncher < ForemanTasksCore::TaskLauncher::Batch
-    class SaltRunnerAction < ForemanTasksCore::Runner::Action
-      def initiate_runner
-        additional_options = {
-          :step_id => run_step_id,
-          :uuid => execution_plan_id
-        }
-        ::SmartProxySaltCore::SaltRunner.new(
-          input.merge(additional_options),
-          :suspended_action => suspended_action
-        )
-      end
-    end
-
-    def child_launcher(parent)
-      ForemanTasksCore::TaskLauncher::Single.new(world, callback, :parent => parent,
-        :action_class_override => SaltRunnerAction)
-    end
-  end
-end

data/lib/smart_proxy_salt_core/version.rb

@@ -1,3 +0,0 @@
-module SmartProxySaltCore
-  VERSION = '0.0.2'.freeze
-end

data/lib/smart_proxy_salt_core.rb

@@ -1,17 +0,0 @@
-require 'foreman_tasks_core'
-require 'foreman_remote_execution_core'
-
-module SmartProxySaltCore
-  extend ForemanTasksCore::SettingsLoader
-  register_settings(:salt,
-                    :saltfile => '/etc/foreman-proxy/settings.d/salt.saltfile')
-
-  if ForemanTasksCore.dynflow_present?
-    require 'smart_proxy_salt_core/salt_runner'
-    require 'smart_proxy_salt_core/salt_task_launcher'
-
-    if defined?(SmartProxyDynflowCore)
-      SmartProxyDynflowCore::TaskLauncherRegistry.register('salt', SaltTaskLauncher)
-    end
-  end
-end