choria-mcorpc-support 2.21.1 → 2.23.1

data/lib/mcollective/util/playbook/data_stores.rb

@@ -0,0 +1,201 @@
+require_relative "data_stores/base"
+require_relative "data_stores/consul_data_store"
+require_relative "data_stores/environment_data_store"
+require_relative "data_stores/etcd_data_store"
+require_relative "data_stores/file_data_store"
+require_relative "data_stores/shell_data_store"
+
+module MCollective
+  module Util
+    class Playbook
+      class DataStores
+        def initialize(playbook)
+          @playbook = playbook
+          @stores = {}
+        end
+
+        # Determines if a path is in the valid form and a known store
+        #
+        # @param path [String] example data_store/key_name
+        # @return [Boolean]
+        def valid_path?(path)
+          store, _ = parse_path(path)
+          include?(store)
+        rescue
+          false
+        end
+
+        # Parse a normal format data path
+        #
+        # @param path [String] example data_store/key_name
+        # @return [String, String] store name and key name
+        # @raise [StandardError] for incorrectly formatted paths
+        def parse_path(path)
+          if path =~ /^([a-zA-Z0-9\-_]+)\/(.+)$/
+            [$1, $2]
+          else
+            raise("Invalid data store path %s" % [path])
+          end
+        end
+
+        # Reads from a path
+        #
+        # @param path [String] example data_store/key_name
+        # @return [Object] data from the data store
+        # @raise [StandardError] when reading from the key fails
+        # @raise [StandardError] for unknown stores
+        def read(path)
+          store, key = parse_path(path)
+
+          Log.debug("Reading key %s from data store %s" % [key, store])
+
+          self[store].read(key)
+        rescue
+          raise("Could not read key %s: %s: %s" % [path, $!.class, $!.to_s])
+        end
+
+        # Writes to a path
+        #
+        # @param path [String] example data_store/key_name
+        # @param value [Object] data to write to the path
+        # @raise [StandardError] when reading from the key fails
+        # @raise [StandardError] for unknown stores
+        def write(path, value)
+          store, key = parse_path(path)
+
+          Log.debug("Storing data in key %s within data store %s" % [key, store])
+          self[store].write(key, value)
+        end
+
+        # Deletes a path
+        #
+        # @param path [String] example data_store/key_name
+        # @raise [StandardError] when reading from the key fails
+        # @raise [StandardError] for unknown stores
+        def delete(path)
+          store, key = parse_path(path)
+
+          Log.debug("Deleting key %s from data store %s" % [key, store])
+          self[store].delete(key)
+        end
+
+        # Members registered in a service
+        #
+        # @param path [String] example data_store/key_name
+        # @return [Array<String>]
+        # @raise [StandardError] when reading from the key fails
+        # @raise [StandardError] for unknown stores
+        def members(path)
+          store, service = parse_path(path)
+
+          Log.debug("Retrieving service members for service %s from data store %s" % [service, store])
+          self[store].members(service)
+        end
+
+        # Attempts to lock a named semaphore, waits until it succeeds
+        #
+        # @param path [String] example data_store/key_name
+        # @raise [StandardError] when obtaining a lock fails
+        # @raise [StandardError] when obtaining a lock timesout
+        # @raise [StandardError] for unknown stores
+        def lock(path)
+          store, key = parse_path(path)
+          timeout = lock_timeout(store)
+
+          Log.debug("Obtaining lock %s on data store %s with timeout %d" % [key, store, timeout])
+
+          self[store].lock(key, timeout)
+        end
+
+        # Attempts to unlock a named semaphore
+        #
+        # @param path [String] example data_store/key_name
+        # @raise [StandardError] when reading from the key fails
+        # @raise [StandardError] for unknown stores
+        def release(path)
+          store, key = parse_path(path)
+
+          Log.debug("Releasing lock %s on data store %s" % [key, store])
+
+          self[store].release(key)
+        end
+
+        # Retrieves the configured lock timeout for a store
+        #
+        # @param store [String]
+        # @return [Integer]
+        def lock_timeout(store)
+          raise("Unknown data store %s" % store) unless include?(store)
+
+          @stores[store][:lock_timeout]
+        end
+
+        # Finds a named store instance
+        #
+        # @param store [String] a store name
+        # @raise [StandardError] for unknown stores
+        def [](store)
+          raise("Unknown data store %s" % store) unless include?(store)
+
+          @stores[store][:store]
+        end
+
+        # List of known store names
+        #
+        # @return [Array<String>]
+        def keys
+          @stores.keys
+        end
+
+        # Determines if a store is known
+        #
+        # @return [Boolean]
+        def include?(store)
+          @stores.include?(store)
+        end
+
+        # Prepares all the stores
+        def prepare
+          @stores.each_value do |properties|
+            properties[:store].from_hash(properties[:properties]).prepare
+          end
+        end
+
+        # Creates a store instance for a given type
+        #
+        # @param name [String] the store instance name
+        # @param type [String] store type
+        # @return [DataStores::Base] data store instance
+        # @raise [NameError] for unknown types
+        def store_for(name, type)
+          klass_name = "%sDataStore" % type.capitalize
+
+          DataStores.const_get(klass_name).new(name, @playbook)
+        rescue NameError
+          raise("Cannot find a handler for Data Store type %s" % type)
+        end
+
+        # Initialize the data stores from playbook data
+        #
+        # @param data [Hash] playbook format data
+        # @return [DataStores]
+        def from_hash(data)
+          @stores.clear
+
+          data.each do |store, props|
+            Log.debug("Loading data store %s" % [store])
+
+            @stores[store] = {
+              :properties => props,
+              :type => props["type"],
+              :lock_timeout => Integer(props.fetch("timeout", 120)),
+              :store => store_for(store, props["type"])
+            }
+          end
+
+          self
+        end
+      end
+    end
+  end
+end

data/lib/mcollective/util/playbook/data_stores/base.rb

@@ -0,0 +1,99 @@
+module MCollective
+  module Util
+    class Playbook
+      class DataStores
+        class Base
+          def initialize(name, playbook)
+            @playbook = playbook
+            @name = name
+
+            startup_hook
+          end
+
+          # Start up processing
+          #
+          # Implementations should not create a {initialize} instead they should
+          # have this hook which will be called once initialization is done
+          def startup_hook; end
+
+          # Prepares the store for use
+          #
+          # Here you can do things like connect to data bases, set up pools etc
+          def prepare; end
+
+          # Validate store properties created using from_hash
+          #
+          # @raise [StandardError] when configuration is invalid
+          def validate_configuration!; end
+
+          # Parse the data store properties as supplied by the playbook
+          #
+          # @param properties [Hash] lock properties
+          # @return [Base]
+          def from_hash(properties)
+            @properties = properties
+
+            validate_configuration!
+
+            self
+          end
+
+          # Release a lock found in the store
+          #
+          # @note when the lock does not exist it should not raise an error
+          # @param key [String] the lock name
+          # @raise [StandardError] when releaging fails
+          def release(key)
+            raise(StandardError, "release not implemented in %s" % [self.class], caller)
+          end
+
+          # Locks a specific lock in the store
+          #
+          # @note when the lock does not exist it should be created
+          # @param key [String] the lock name
+          # @param timeout [Integer,Float] how long to attempt to get the lock for
+          # @raise [StandardError] when locking fails
+          def lock(key, timeout)
+            raise(StandardError, "lock not implemented in %s" % [self.class], caller)
+          end
+
+          # Finds the members in a service
+          #
+          # @param key [String] the service name
+          # @return [Array<String>] list of service members
+          # @raise [StandardError] when the service is unknown or general error happened
+          def members(key)
+            raise(StandardError, "members not implemented in %s" % [self.class], caller)
+          end
+
+          # Deletes a key from a data store
+          #
+          # @note deleting non existing data should not raise an error
+          # @param key [String] the key to delete
+          # @raise [StandardError] when deleting fails
+          def delete(key)
+            raise(StandardError, "delete not implemented in %s" % [self.class], caller)
+          end
+
+          # Writes a value to the key in a data store
+          #
+          # @param key [String] the key to write
+          # @param value [String] the value to write
+          # @raise [StandardError] when writing fails
+          def write(key, value)
+            raise(StandardError, "write not implemented in %s" % [self.class], caller)
+          end
+
+          # Reads a key from a data store
+          #
+          # @param key [String] the key to read
+          # @return [String] string found in the data store
+          # @raise [StandardError] when the key does not exist
+          def read(key)
+            raise(StandardError, "read not implemented in %s" % [self.class], caller)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mcollective/util/playbook/data_stores/consul_data_store.rb

@@ -0,0 +1,88 @@
+module MCollective
+  module Util
+    class Playbook
+      class DataStores
+        class ConsulDataStore < Base
+          def startup_hook
+            require "diplomat"
+
+            @session_mutex = Mutex.new
+            @session_manager = nil
+            @session_id = nil
+          rescue LoadError
+            raise("Consul Data Store is not functional. Please install the diplomat Gem.")
+          end
+
+          def lock(key, timeout)
+            Timeout.timeout(timeout) do
+              Diplomat::Lock.wait_to_acquire(key, session, nil, 2)
+            end
+          rescue Timeout::Error
+            raise("Failed to obtain lock %s after %d seconds" % [key, timeout])
+          end
+
+          def release(key)
+            Diplomat::Lock.release(key, session)
+          end
+
+          def read(key)
+            Diplomat::Kv.get(key)
+          end
+
+          def write(key, value)
+            Diplomat::Kv.put(key, value)
+          end
+
+          def delete(key)
+            Diplomat::Kv.delete(key)
+          end
+
+          def ttl
+            supplied = Integer(@properties.fetch("ttl", 10)).abs
+
+            supplied < 10 ? 10 : supplied
+          end
+
+          def renew_session
+            if @session_id
+              Log.debug("Renewing Consul session %s" % [@session_id])
+              Diplomat::Session.renew(@session_id)
+            end
+          end
+
+          def start_session_manager
+            @session_manager = Thread.new do
+              begin
+                loop do
+                  renew_session
+
+                  sleep(ttl - 5)
+                end
+              rescue
+                Log.warn("Session manager for Consul data store %s failed: %s: %s" % [@name, $!.class, $!.to_s])
+
+                sleep(1)
+
+                retry
+              end
+            end
+          end
+
+          def session
+            @session_mutex.synchronize do
+              return @session_id if @session_id
+
+              ttl_spec = "%ss" % ttl
+
+              @session_id ||= Diplomat::Session.create("TTL" => ttl_spec, "Behavior" => "delete")
+
+              start_session_manager
+
+              @session_id
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mcollective/util/playbook/data_stores/environment_data_store.rb

@@ -0,0 +1,33 @@
+require_relative "base"
+
+module MCollective
+  module Util
+    class Playbook
+      class DataStores
+        class EnvironmentDataStore < Base
+          def read(key)
+            raise("No such environment variable %s" % [key_for(key)]) unless include?(key)
+
+            ENV[key_for(key)]
+          end
+
+          def write(key, value)
+            ENV[key_for(key)] = value
+          end
+
+          def delete(key)
+            ENV.delete(key_for(key))
+          end
+
+          def key_for(key)
+            "%s%s" % [@properties["prefix"], key]
+          end
+
+          def include?(key)
+            ENV.include?(key_for(key))
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mcollective/util/playbook/data_stores/etcd_data_store.rb

@@ -0,0 +1,42 @@
+module MCollective
+  module Util
+    class Playbook
+      class DataStores
+        class EtcdDataStore < Base
+          def startup_hook
+            require "etcdv3"
+          rescue LoadError
+            raise("Etcd Data Store is not functional. Please install the etcdv3 Gem.")
+          end
+
+          def conn
+            return @_conn if @_conn
+
+            opts = {}
+            opts[:url] = @properties.fetch("url", "http://127.0.0.1:2379")
+            opts[:user] = @properties["user"] if @properties["user"]
+            opts[:password] = @properties["password"] if @properties["password"]
+
+            @_conn = Etcdv3.new(opts)
+          end
+
+          def read(key)
+            result = conn.get(key)
+
+            raise("Could not find key %s" % key) if !result || result.kvs.empty?
+
+            result.kvs[0].value
+          end
+
+          def write(key, value)
+            conn.put(key, value)
+          end
+
+          def delete(key)
+            conn.del(key)
+          end
+        end
+      end
+    end
+  end
+end