choria-mcorpc-support 2.21.1 → 2.23.1

data/lib/mcollective/util/playbook/data_stores/file_data_store.rb

@@ -0,0 +1,106 @@
+require_relative "base"
+
+require "json"
+require "yaml"
+
+module MCollective
+  module Util
+    class Playbook
+      class DataStores
+        class FileDataStore < Base
+          attr_accessor :file, :format, :create
+
+          def startup_hook
+            @file_mutex = Mutex.new
+          end
+
+          def from_hash(properties)
+            @file = properties["file"]
+            @format = properties["format"]
+            @create = !!properties["create"]
+
+            validate_configuration!
+
+            self
+          end
+
+          def validate_configuration!
+            raise("No file given to use as data source") unless @file
+            raise("No file format given") unless @format
+            raise("File format has to be one of 'json' or 'yaml'") unless ["json", "yaml"].include?(@format)
+
+            @file = File.expand_path(@file)
+
+            FileUtils.touch(@file) if @create && !File.exist?(@file)
+
+            raise("Cannot find data file %s" % @file) unless File.exist?(@file)
+            raise("Cannot read data file %s" % @file) unless File.readable?(@file)
+            raise("Cannot write data file %s" % @file) unless File.writable?(@file)
+
+            raise("The data file must contain a Hash or be empty") unless data.is_a?(Hash)
+          end
+
+          def data
+            @file_mutex.synchronize do
+              parse_data
+            end
+          end
+
+          def parse_data
+            return({}) if File.size(@file) == 0
+
+            case @format
+            when "json"
+              JSON.parse(File.read(@file))
+            when "yaml"
+              YAML.load(File.read(@file))
+            end
+          end
+
+          def save_data(raw_data)
+            File.open(@file, "w") do |f|
+              case @format
+              when "json"
+                f.print(JSON.dump(raw_data))
+              when "yaml"
+                f.print(YAML.dump(raw_data))
+              end
+            end
+          end
+
+          def read(key)
+            raise("No such key %s" % [key]) unless include?(key)
+
+            data[key]
+          end
+
+          def write(key, value)
+            @file_mutex.synchronize do
+              raw_data = parse_data
+
+              raw_data[key] = value
+
+              save_data(raw_data)
+            end
+
+            read(key)
+          end
+
+          def delete(key)
+            @file_mutex.synchronize do
+              raw_data = parse_data
+
+              raw_data.delete(key)
+
+              save_data(raw_data)
+            end
+          end
+
+          def include?(key)
+            data.include?(key)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mcollective/util/playbook/data_stores/shell_data_store.rb

@@ -0,0 +1,103 @@
+require_relative "base"
+
+module MCollective
+  module Util
+    class Playbook
+      class DataStores
+        class ShellDataStore < Base
+          attr_reader :command, :timeout, :environment, :cwd
+
+          def write(key, value)
+            run("write", key, "CHORIA_DATA_VALUE" => value)
+
+            nil
+          end
+
+          def delete(key)
+            run("delete", key)
+
+            nil
+          end
+
+          def read(key)
+            run("read", key).stdout.chomp
+          end
+
+          def run(action, key, environment={})
+            validate_key(key)
+
+            command = "%s --%s" % [@command, action]
+            options = shell_options
+
+            options["environment"].merge!(
+              environment.merge(
+                "CHORIA_DATA_KEY" => key,
+                "CHORIA_DATA_ACTION" => action
+              )
+            )
+
+            shell = run_command(command, options)
+
+            unless shell.status.exitstatus == 0
+              Log.warn("While running command %s: %s" % [command, shell.stderr])
+              raise("Could not %s key %s, got exitcode %d" % [action, key, shell.status.exitstatus])
+            end
+
+            shell
+          end
+
+          def run_command(command, options)
+            shell = Shell.new(command, options)
+            shell.runcommand
+            shell
+          end
+
+          def validate_key(key)
+            raise("Valid keys must match ^[a-zA-Z0-9_-]+$") unless key =~ /^[a-zA-Z0-9_-]+$/
+
+            true
+          end
+
+          def from_hash(properties)
+            @command = properties["command"]
+            @timeout = properties.fetch("timeout", 10)
+            @environment = properties.fetch("environment", {})
+            @cwd = properties["cwd"]
+
+            self
+          end
+
+          def validate_configuration!
+            raise("A command is required") unless @command
+            raise("Command %s is not executable" % @command) unless File.executable?(@command)
+            raise("Timeout should be an integer") unless @timeout.to_i.to_s == @timeout.to_s
+
+            if @environment
+              raise("Environment should be a hash") unless @environment.is_a?(Hash)
+
+              all_strings = @environment.map {|k, v| k.is_a?(String) && v.is_a?(String)}.all?
+              raise("All keys and values in the environment must be strings") unless all_strings
+            end
+
+            if @cwd
+              raise("cwd %s does not exist" % @cwd) unless File.exist?(@cwd)
+              raise("cwd %s is not a directory" % @cwd) unless File.directory?(@cwd)
+            end
+          end
+
+          def shell_options
+            unless @__options
+              @__options = {}
+              @__options["cwd"] = @cwd if @cwd
+              @__options["environment"] = @environment
+              @__options["timeout"] = Integer(@timeout)
+            end
+
+            # bacause environment is being edited
+            Marshal.load(Marshal.dump(@__options))
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mcollective/util/playbook/inputs.rb

@@ -0,0 +1,265 @@
+module MCollective
+  module Util
+    class Playbook
+      class Inputs
+        def initialize(playbook)
+          @playbook = playbook
+          @inputs = {}
+        end
+
+        # List of known input names
+        #
+        # @return [Array<String>]
+        def keys
+          @inputs.keys
+        end
+
+        # List of known input names that have static values
+        #
+        # @return [Array<String>]
+        def static_keys
+          keys - dynamic_keys
+        end
+
+        # List of known input names that have dynamic values
+        #
+        # @return [Array<String>]
+        def dynamic_keys
+          @inputs.select do |_, props|
+            props[:dynamic]
+          end.keys
+        end
+
+        # Creates options for each input in the application
+        #
+        # @param application [MCollective::Application]
+        # @param set_required [Boolean] when true required inputs will be required on the CLI
+        # @raise [StandardError] for invalidly defined inputs
+        def add_cli_options(application, set_required)
+          @inputs.each do |input, props|
+            i_props = props[:properties]
+
+            next if i_props["dynamic"]
+
+            type = case i_props["type"]
+                   when :string, "String"
+                     String
+                   when :fixnum, "Fixnum", "Integer"
+                     Integer
+                   when :float, "Float"
+                     Float
+                   when :numeric, "Numeric"
+                     Numeric
+                   when :array, ":array", "Array"
+                     :array
+                   when :bool, ":bool", ":boolean", "Boolean"
+                     :boolean
+                   else
+                     raise("Invalid input type %s given for input %s" % [i_props["type"], input])
+                   end
+
+            description = "%s (%s) %s" % [i_props["description"], type, i_props["default"] ? ("default: %s" % i_props["default"]) : ""]
+
+            option_params = {
+              :description => description,
+              :arguments => ["--%s %s" % [input.downcase, input.upcase]],
+              :type => type
+            }
+
+            option_params[:arguments] = ["--%s" % input.downcase] if type == :boolean
+
+            option_params[:required] = i_props["required"] if set_required && !i_props.include?("data")
+
+            application.class.option(input, option_params)
+          end
+        end
+
+        # Attempts to find values for all declared inputs
+        #
+        # During tests saving to the ds can be skipped by setting
+        # @save_during_prepare to false
+        #
+        # @param data [Hash] input data from CLI etc
+        # @return [void]
+        # @raise [StandardError] when required data could not be found
+        # @raise [StandardError] when validation fails for any data
+        def prepare(data={})
+          @inputs.each do |input, props|
+            next unless data.include?(input)
+            next if data[input].nil?
+            next if props[:properties]["dynamic"]
+
+            validate_data(input, data[input])
+
+            props[:value] = data[input]
+            props[:dynamic] = false
+          end
+
+          validate_requirements
+        end
+
+        # Checks if a input is dynamic
+        #
+        # Dynaic inputs are those where the data is sourced from
+        # a data source, an input that has a data source defined
+        # and had a specific input given will not be dynamic
+        #
+        # @return [Boolean]
+        def dynamic?(input)
+          @inputs[input][:dynamic]
+        end
+
+        def include?(input)
+          @inputs.include?(input)
+        end
+
+        # Looks up data from a datastore, returns default when not found
+        #
+        # @param input [String] input name
+        # @return [Object] value from the ds
+        # @raise [StandardError] for invalid inputs and ds errors
+        def lookup_from_datastore(input)
+          raise("Unknown input %s" % input) unless include?(input)
+
+          properties = @inputs[input][:properties]
+
+          value = @playbook.data_stores.read(properties["data"])
+          validate_data(input, value)
+
+          value
+        rescue
+          raise("Could not resolve %s for input %s: %s: %s" % [properties["data"], input, $!.class, $!.to_s]) unless properties.include?("default")
+
+          Log.warn("Could not find %s, returning default value" % properties["data"])
+
+          properties["default"]
+        end
+
+        # Saves static property values in their associated data stores for any keys with save set
+        def save_input_data
+          @inputs.each do |input, props|
+            next unless props[:properties]["save"]
+            next unless props[:properties]["data"]
+            next unless props[:value]
+            next if props[:dynamic]
+
+            save_to_datastore(input)
+          end
+        end
+
+        # Saves the value of a input to the data entry associated with it
+        #
+        # @see #save_input_data
+        # @param input [String] input name
+        # @raise [StandardError] for invalid inputs and ds errors
+        def save_to_datastore(input)
+          raise("Unknown input %s" % input) unless include?(input)
+
+          i_data = @inputs[input]
+
+          raise("Input %s has no value, cannot store it" % input) unless i_data.include?(:value)
+
+          Log.debug("Saving value for input %s to data item %s" % [input, i_data[:properties]["data"]])
+
+          @playbook.data_stores.write(i_data[:properties]["data"], i_data[:value])
+        end
+
+        # Retrieves the value for a specific input
+        #
+        # @param input [String] input name
+        # @return [Object]
+        # @raise [StandardError] for unknown inputs
+        def [](input)
+          raise("Unknown input %s" % input) unless include?(input)
+
+          props = @inputs[input][:properties]
+
+          if @inputs[input].include?(:value)
+            Log.debug("Resolving %s as static" % [input])
+            @inputs[input][:value]
+          elsif props.include?("data")
+            Log.debug("Resolving %s as dynamic" % [input])
+            lookup_from_datastore(input)
+          elsif props.include?("default")
+            Log.debug("Resolving %s as default" % [input])
+            props["default"]
+          else
+            raise("Input %s has no value, data source or default" % [input])
+          end
+        end
+
+        # Retrieves the properties for a specific input
+        #
+        # @param input [String] input name
+        # @return [Hash]
+        # @raise [StandardError] for unknown inputs
+        def input_properties(input)
+          if include?(input)
+            @inputs[input][:properties]
+          else
+            raise("Unknown input %s" % input)
+          end
+        end
+
+        # Checks all required inputs have values
+        #
+        # @raise [StandardError] when not
+        def validate_requirements
+          invalid = @inputs.map do |input, props|
+            next unless props[:properties]["required"]
+            next if props[:properties].include?("data")
+
+            unless props[:value]
+              Log.warn("Input %s requires a value but has none or nil" % input)
+              input
+            end
+          end.compact
+
+          raise("Values were required but not given for inputs: %s" % invalid.join(", ")) unless invalid.empty?
+        end
+
+        # Validates a piece of data against an input
+        #
+        # @todo this seems quite limited, we have to expand with real needs
+        # @param input [String] a valid input name
+        # @param value [Object] a value to validate
+        # @raise [StandardError] on validation failure
+        def validate_data(input, value)
+          return unless @inputs[input][:properties].include?("validation")
+
+          validator = @inputs[input][:properties]["validation"]
+
+          case validator
+          when /^:(.+)/
+            validator = $1.intern
+          when /^\/(.+)\/$/
+            validator = Regexp.new($1)
+          end
+
+          Log.debug("Validating input %s using %s validator" % [input, validator])
+
+          Validator.validate(value, validator)
+        rescue
+          Log.warn("Attempt to validate input %s with validator %s failed: %s" % [input, validator, $!.to_s]) if validator
+
+          raise("Failed to validate value for input %s: %s" % [input, $!.to_s])
+        end
+
+        def from_hash(data)
+          data.each do |input, props|
+            props["required"] = true unless props.include?("required")
+
+            Log.debug("Loading input %s" % [input])
+
+            @inputs[input] = {
+              :properties => props,
+              :dynamic => props.include?("data")
+            }
+          end
+
+          self
+        end
+      end
+    end
+  end
+end